@ultimabase/realtime 0.0.0-stage → 0.1.0

package/src/reach.ts

@@ -0,0 +1,251 @@
+// Where a live read reaches (D-154): the fields its filter and order read on
+// its own rows, the relations its filter goes through, level by level, and
+// the relations its selection shows, so a write is followed only where it
+// lands.
+import type { Condition, Filter, Nested, Schema, SchemaField } from 'ultimabase'
+import type { Fields } from './changes.ts'
+import type { LivePlan } from 'ultimabase/internal'
+
+/** A relation followed from one model to the next. */
+export type Step = {
+	readonly relation: string
+	readonly many: boolean
+	readonly target: string
+	readonly inverse: string
+	/** The target stores the key: a link changes there, out of sight. */
+	readonly keyOnTarget: boolean
+}
+
+/** A level of a filter: its rows are those its steps reach from the root. */
+export type Level = {
+	readonly steps: readonly Step[]
+	readonly model: string
+	/** The fields it reads, the key back to the level above included. */
+	readonly fields: Fields
+	/** Under `NOT`, `isNot`, `every`, `none` or `is: null`: a lost link may add a row. */
+	readonly negative: boolean
+}
+
+/** A relation the selection shows, or counts. */
+export type Shown = {
+	/** The relations from the root, joined by dots. */
+	readonly path: string
+	readonly steps: readonly Step[]
+	readonly model: string
+	/** The fields of its rows that show, or decide which rows show. */
+	readonly fields: Fields
+	/** A `_count`: its rows are not in the result to track. */
+	readonly counted: boolean
+}
+
+export type Reach = {
+	/** The filter's levels, the root's first. */
+	readonly levels: readonly Level[]
+	/** The fields that place a root row: its filter's and its order's. */
+	readonly placing: Fields
+	/** The root fields that show. */
+	readonly showing: Fields
+	readonly shown: readonly Shown[]
+	/** Models a nested filter or order reaches further: any change resends all. */
+	readonly deep: ReadonlySet<string>
+}
+
+type Models = ReadonlyMap<string, ReadonlyMap<string, SchemaField>>
+
+const indexes = new WeakMap<Schema, Models>()
+
+/** `schema`'s fields by model and name. */
+export function modelsOf(schema: Schema): Models {
+	const known = indexes.get(schema)
+	if (known !== undefined) return known
+	const models = new Map(
+		schema.models.map(({ name, fields }) => [
+			name,
+			new Map(fields.map((field) => [field.name, field])),
+		]),
+	)
+	indexes.set(schema, models)
+	return models
+}
+
+/** The relation `name` of `model`, if it is one. */
+export function stepOf(
+	models: Models,
+	model: string,
+	name: string,
+): Step | undefined {
+	const field = models.get(model)?.get(name)
+	if (field === undefined || field.kind === 'scalar') return undefined
+	const back = models.get(field.target)?.get(field.inverse)
+	return {
+		relation: name,
+		many: field.kind === 'many',
+		target: field.target,
+		inverse: field.inverse,
+		keyOnTarget: back?.kind === 'one' && back.stored,
+	}
+}
+
+const relationKeys = ['is', 'isNot', 'some', 'every', 'none'] as const
+
+/** What a nested read selects. */
+type Read = Pick<Nested, 'include' | 'count' | 'fields'>
+
+const isList = (
+	value: Condition | readonly Filter[],
+): value is readonly Filter[] => Array.isArray(value)
+
+/**
+ * Adds the levels of `filter`, read on `model`'s rows, to `levels`; returns
+ * the fields it reads on them.
+ */
+export function walk(
+	models: Models,
+	model: string,
+	filter: Filter | undefined,
+	steps: readonly Step[],
+	negative: boolean,
+	levels: Level[],
+): Fields {
+	const fields = new Set<string>()
+	const back = steps.at(-1)
+	if (back !== undefined) fields.add(back.inverse)
+	let all = false
+	const visit = (part: Filter | undefined, under: boolean): void => {
+		for (const [key, value] of Object.entries(part ?? {})) {
+			if (value === undefined) continue
+			if (isList(value)) {
+				for (const inner of value) visit(inner, under || key === 'NOT')
+				continue
+			}
+			if (key === '_search') all = true
+			fields.add(key)
+			const step = stepOf(models, model, key)
+			if (step === undefined) continue
+			for (const condition of relationKeys) {
+				const inner = value[condition]
+				if (inner === undefined) continue
+				const lost =
+					condition === 'is'
+						? inner === null
+						: condition !== 'some' && !(condition === 'isNot' && inner === null)
+				const next = [...steps, step]
+				walk(models, step.target, inner ?? {}, next, under || lost, levels)
+			}
+		}
+	}
+	visit(filter, negative)
+	const read = all ? 'all' : fields
+	levels.push({ steps, model, fields: read, negative })
+	return read
+}
+
+/** The fields a nested read shows: `'all'` when it lists none. */
+function showing(nested: Read, also: readonly string[] = []): Fields {
+	if (nested.fields === undefined) return 'all'
+	return new Set([
+		...nested.fields,
+		...Object.keys(nested.include ?? {}),
+		...Object.keys(nested.count ?? {}),
+		...also,
+	])
+}
+
+const union = (a: Fields, b: Fields): Fields =>
+	a === 'all' || b === 'all' ? 'all' : new Set([...a, ...b])
+
+/** The models a nested filter or order reaches beyond `model`. */
+function further(
+	models: Models,
+	model: string,
+	where: Filter | undefined,
+	orderBy: Nested['orderBy'],
+	deep: Set<string>,
+): Fields {
+	const levels: Level[] = []
+	const fields = walk(models, model, where, [], false, levels)
+	for (const level of levels) {
+		if (level.steps.length > 0) deep.add(level.model)
+	}
+	for (const order of orderBy ?? []) {
+		let on = model
+		for (const { relation } of order.through ?? []) {
+			on = stepOf(models, on, relation)?.target ?? on
+			deep.add(on)
+		}
+		const counted = order.count && stepOf(models, on, order.field)
+		if (counted) deep.add(counted.target)
+	}
+	return fields
+}
+
+/** Adds what `nested`, read on `model`'s rows at `path`, shows. */
+function walkShown(
+	models: Models,
+	model: string,
+	nested: Read,
+	at: { readonly path: string; readonly steps: readonly Step[] },
+	shown: Shown[],
+	deep: Set<string>,
+): void {
+	const named = (name: string) => (at.path ? `${at.path}.${name}` : name)
+	for (const [name, inner] of Object.entries(nested.include ?? {})) {
+		const step = stepOf(models, model, name)
+		if (step === undefined) continue
+		const steps = [...at.steps, step]
+		const filtered = further(
+			models,
+			step.target,
+			inner.where,
+			inner.orderBy,
+			deep,
+		)
+		const fields = union(showing(inner, [step.inverse]), filtered)
+		shown.push({
+			path: named(name),
+			steps,
+			model: step.target,
+			fields,
+			counted: false,
+		})
+		walkShown(
+			models,
+			step.target,
+			inner,
+			{ path: named(name), steps },
+			shown,
+			deep,
+		)
+	}
+	for (const [name, where] of Object.entries(nested.count ?? {})) {
+		const step = stepOf(models, model, name)
+		if (step === undefined) continue
+		const filtered = further(models, step.target, where, undefined, deep)
+		shown.push({
+			path: named(`_count.${name}`),
+			steps: [...at.steps, step],
+			model: step.target,
+			fields: union(new Set([step.inverse]), filtered),
+			counted: true,
+		})
+	}
+}
+
+/** Where `plan`, a live read of `model`, reaches. */
+export function reachOf(schema: Schema, model: string, plan: LivePlan): Reach {
+	const models = modelsOf(schema)
+	const levels: Level[] = []
+	const root = walk(models, model, plan.where, [], false, levels)
+	levels.reverse()
+	const order = new Set(plan.orderBy.map(({ field }) => field))
+	const shown: Shown[] = []
+	const deep = new Set<string>()
+	walkShown(models, model, plan, { path: '', steps: [] }, shown, deep)
+	return {
+		levels,
+		placing: union(root, order),
+		showing: showing(plan),
+		shown,
+		deep,
+	}
+}

package/src/reads.ts

@@ -0,0 +1,103 @@
+// The reads a flush makes (D-154): the root rows a relation's rows reach,
+// a filter checked on a few rows, in memory when it reads the row alone,
+// and each changed row loaded once for every live read that needs it.
+import type { Filter, Nested, Row } from 'ultimabase'
+import { inMemory, matches, storedRows } from 'ultimabase/internal'
+import type { Finder } from './affected.ts'
+import type { Step } from './reach.ts'
+
+/** A filter on the root that holds when `steps` reach a row passing `inner`. */
+export const through = (steps: readonly Step[], inner: Filter): Filter =>
+	steps.reduceRight<Filter>(
+		(filter, { relation, many }) => ({
+			[relation]: many ? { some: filter } : { is: filter },
+		}),
+		inner,
+	)
+
+/** The `model` rows in `ids`, as one filter. */
+const among = (ids: readonly string[]): Filter => ({ id: { in: [...ids] } })
+
+/** Finds root rows of `model` through one indexed query per call. */
+export const finderOf =
+	(model: string): Finder =>
+	async (steps, ids, within) => {
+		const reach = through(steps, among(ids))
+		const where = within === undefined ? reach : { AND: [among(within), reach] }
+		const rows = await storedRows(model, { where, fields: ['id'] })
+		return rows.map(({ id }) => id)
+	}
+
+/**
+ * Each changed row, read once per flush whatever reads it, and each page
+ * of rows after a window's last, once for the live reads that share it.
+ */
+export class Loader {
+	readonly #rows = new Map<string, Map<string, Promise<Row | undefined>>>()
+	readonly #pages = new Map<string, Promise<readonly Row[]>>()
+
+	#known(model: string): Map<string, Promise<Row | undefined>> {
+		const known =
+			this.#rows.get(model) ?? new Map<string, Promise<Row | undefined>>()
+		this.#rows.set(model, known)
+		return known
+	}
+
+	/** The stored rows `read` finds, whose rows `load` then gives. */
+	async after(model: string, read: Nested): Promise<readonly Row[]> {
+		const key = JSON.stringify([model, read])
+		let page = this.#pages.get(key)
+		if (page === undefined) {
+			page = storedRows(model, read)
+			this.#pages.set(key, page)
+			const rows = await page
+			const known = this.#known(model)
+			for (const row of rows) {
+				if (!known.has(row.id)) known.set(row.id, Promise.resolve(row))
+			}
+		}
+		return page
+	}
+
+	/** The stored rows of `model` in `ids`; a deleted row is missing. */
+	async load(
+		model: string,
+		ids: readonly string[],
+	): Promise<ReadonlyMap<string, Row>> {
+		const known = this.#known(model)
+		const missing = ids.filter((id) => !known.has(id))
+		if (missing.length > 0) {
+			const read = storedRows(model, { where: among(missing) }).then(
+				(rows) => new Map(rows.map((row) => [row.id, row])),
+			)
+			for (const id of missing) {
+				known.set(
+					id,
+					read.then((rows) => rows.get(id)),
+				)
+			}
+		}
+		const found = new Map<string, Row>()
+		for (const id of ids) {
+			const row = await known.get(id)
+			if (row !== undefined) found.set(id, row)
+		}
+		return found
+	}
+}
+
+/** The ids of `rows` that pass `filter`: in memory when it reads the row alone. */
+export async function pass(
+	model: string,
+	rows: ReadonlyMap<string, Row>,
+	filter: Filter | undefined,
+): Promise<ReadonlySet<string>> {
+	if (filter === undefined || rows.size === 0) return new Set(rows.keys())
+	if (inMemory(model, filter)) {
+		const kept = [...rows.values()].filter((row) => matches(model, row, filter))
+		return new Set(kept.map(({ id }) => id))
+	}
+	const where = { AND: [among([...rows.keys()]), filter] }
+	const kept = await storedRows(model, { where, fields: ['id'] })
+	return new Set(kept.map(({ id }) => id))
+}

package/src/shape.ts

@@ -0,0 +1,86 @@
+// What a live read asks for (D-154): `first` and `totalCount` are the
+// engine's own; the rest is planned as a `findMany`'s, with each level's
+// `id` selected, which places and tracks the rows it reaches.
+import { PublicError } from 'ultimabase'
+
+type Args = { readonly [key: string]: unknown }
+
+const isArgs = (value: unknown): value is Args =>
+	typeof value === 'object' && value !== null && !Array.isArray(value)
+
+const invalid = (path: string, message: string): PublicError =>
+	new PublicError('VALIDATION_FAILED', message, {
+		details: { issues: [{ path: [path], message }] },
+	})
+
+/** `nested`, a relation's value in a selection, with its rows' `id`. */
+const nestedWithIds = (nested: unknown): unknown =>
+	isArgs(nested) ? withIds(nested) : nested
+
+/** `args`' selection, each level's `id` selected and never omitted. */
+function withIds(args: Args): Args {
+	const { select, include, omit, ...rest } = args
+	const out: { [key: string]: unknown } = { ...rest }
+	if (isArgs(select)) {
+		const fields = Object.entries(select).map(([field, value]) => [
+			field,
+			field === '_count' ? value : nestedWithIds(value),
+		])
+		out['select'] = { ...Object.fromEntries(fields), id: true }
+	} else if (select !== undefined) out['select'] = select
+	if (isArgs(include)) {
+		const fields = Object.entries(include).map(([field, value]) => [
+			field,
+			field === '_count' ? value : nestedWithIds(value),
+		])
+		out['include'] = Object.fromEntries(fields)
+	} else if (include !== undefined) out['include'] = include
+	if (isArgs(omit)) {
+		out['omit'] = Object.fromEntries(
+			Object.entries(omit).filter(([field]) => field !== 'id'),
+		)
+	} else if (omit !== undefined) out['omit'] = omit
+	return out
+}
+
+/** What `livePlan` reads: `args` without the engine's own keys. */
+export function liveArgs(args: unknown, row: boolean): unknown {
+	if (!isArgs(args)) return args
+	if (row) return withIds(args)
+	const { first: _first, totalCount: _totalCount, ...rest } = args
+	return withIds(rest)
+}
+
+export type Shape = {
+	readonly first: number
+	readonly where: unknown
+	readonly totalCount: boolean
+	/** The selection each window row is read with. */
+	readonly shape: Args
+}
+
+/** `args`' window and selection; a list's `first` is a positive integer. */
+export function shapeOf(args: unknown, row: boolean): Shape {
+	const read = isArgs(args) ? args : {}
+	const { where, first, totalCount } = read
+	if (
+		!row &&
+		(!Number.isInteger(first) || typeof first !== 'number' || first < 1)
+	) {
+		throw invalid('first', 'first must be a positive integer.')
+	}
+	if (!row && totalCount !== undefined && typeof totalCount !== 'boolean') {
+		throw invalid('totalCount', 'totalCount must be a boolean.')
+	}
+	const { select, include, omit } = withIds(read)
+	return {
+		first: typeof first === 'number' && !row ? first : 1,
+		where,
+		totalCount: !row && totalCount === true,
+		shape: {
+			...(select !== undefined && { select }),
+			...(include !== undefined && { include }),
+			...(omit !== undefined && { omit }),
+		},
+	}
+}

package/src/stale.ts

@@ -0,0 +1,68 @@
+// Whether a write may change what a recorded read found (D-154): a caller or
+// a live read's plan that read it is built again.
+import type { Recorded } from 'ultimabase/internal'
+import { inMemory, liveSchema, matches } from 'ultimabase/internal'
+import type { Batch, Fields } from './changes.ts'
+import { listed, touched } from './changes.ts'
+import { modelsOf, stepOf, walk } from './reach.ts'
+import type { Level } from './reach.ts'
+import type { Loader } from './reads.ts'
+
+const fieldsOf = (fields: readonly string[] | undefined): Fields =>
+	fields === undefined ? 'all' : new Set(fields)
+
+/** Whether a write in `batch` may change what `read` found. */
+async function stale(
+	read: Recorded,
+	batch: Batch,
+	loader: Loader,
+): Promise<boolean> {
+	const models = modelsOf(liveSchema())
+	if (read.relation !== undefined) {
+		const [model = '', field = ''] = read.relation.split('.')
+		const step = stepOf(models, model, field)
+		if (touched(batch.get(model), new Set([field]))) return true
+		if (step && touched(batch.get(step.target), new Set([step.inverse]))) {
+			return true
+		}
+	}
+	const { model, where, ids } = read
+	if (where !== undefined && !inMemory(model, where)) {
+		const levels: Level[] = []
+		walk(models, model, where, [], false, levels)
+		return levels.some((level) => batch.has(level.model))
+	}
+	const changes = batch.get(model)
+	if (changes === undefined) return false
+	const { inserts, deletes } = changes
+	const filters = fieldsOf(read.filters)
+	const any =
+		read.fields === undefined || filters === 'all'
+			? 'all'
+			: new Set([...read.fields, ...filters])
+	// An update that sets no field it returned or chose its rows by is none.
+	const updates = touched(changes, any) ? changes.updates : undefined
+	if (ids === undefined || inserts === 'all' || deletes === 'all') {
+		return (inserts ?? deletes ?? updates) !== undefined
+	}
+	if (updates === 'all') return true
+	const found = new Set(ids)
+	const written = [...listed(updates), ...listed(deletes)]
+	if (written.some((id) => found.has(id))) return true
+	// A row enters when inserted, or updated in a field that chose the rows.
+	const moved = touched(changes, filters) ? listed(updates) : []
+	const others = [...listed(inserts), ...moved]
+	if (others.length === 0) return false
+	if (where === undefined) return true
+	const rows = await loader.load(model, others)
+	return [...rows.values()].some((row) => matches(model, row, where))
+}
+
+export const anyStale = async (
+	reads: readonly Recorded[],
+	batch: Batch,
+	loader: Loader,
+): Promise<boolean> => {
+	for (const read of reads) if (await stale(read, batch, loader)) return true
+	return false
+}

package/src/tracking.ts

@@ -0,0 +1,61 @@
+// The related rows a live read shows (D-154): by relation path then id, the
+// window's rows that reach each one, so a write to it resends them.
+import type { Row } from 'ultimabase'
+import type { LivePlan } from 'ultimabase/internal'
+
+type Shown = { readonly [field: string]: unknown }
+
+/** The rows `include` reached from `node`, as `[path, id]`. */
+function reached(
+	node: Shown,
+	include: LivePlan['include'],
+	path: string,
+	out: (readonly [string, string])[],
+): void {
+	for (const [name, inner] of Object.entries(include ?? {})) {
+		const at = path === '' ? name : `${path}.${name}`
+		const value = node[name]
+		const rows: readonly unknown[] = Array.isArray(value) ? value : [value]
+		for (const row of rows) {
+			if (typeof row !== 'object' || row === null) continue
+			const id: unknown = Reflect.get(row, 'id')
+			if (typeof id === 'string') out.push([at, id])
+			reached(Object.fromEntries(Object.entries(row)), inner.include, at, out)
+		}
+	}
+}
+
+export class Tracking {
+	readonly tracked: Map<string, Map<string, Set<string>>> = new Map()
+	readonly #reaching = new Map<string, readonly (readonly [string, string])[]>()
+
+	clear(): void {
+		this.tracked.clear()
+		this.#reaching.clear()
+	}
+
+	/** Forgets the related rows the window's row `root` reached. */
+	untrack(root: string): void {
+		for (const [path, id] of this.#reaching.get(root) ?? []) {
+			const roots = this.tracked.get(path)?.get(id)
+			roots?.delete(root)
+			if (roots?.size === 0) this.tracked.get(path)?.delete(id)
+		}
+		this.#reaching.delete(root)
+	}
+
+	/** Keeps the related rows `node`, as read through `include`, reaches. */
+	track(node: Row, include: LivePlan['include']): void {
+		const out: (readonly [string, string])[] = []
+		reached(node, include, '', out)
+		this.untrack(node.id)
+		this.#reaching.set(node.id, out)
+		for (const [path, id] of out) {
+			const ids = this.tracked.get(path) ?? new Map<string, Set<string>>()
+			this.tracked.set(path, ids)
+			const roots = ids.get(id) ?? new Set<string>()
+			ids.set(id, roots)
+			roots.add(node.id)
+		}
+	}
+}