@bjornpagen/bumbledb 0.20.2 → 1.0.0

package/src/changes.ts

@@ -0,0 +1,361 @@
+import type { Scope } from "effect"
+import { Effect, Exit } from "effect"
+import { drainClose, releaseOwner } from "#close.ts"
+import type { SchemaId } from "#compile.ts"
+import { Schema as CoreSchema, schemaTables } from "#compile.ts"
+import type { ChangesHandle, DraftHandle } from "#db-native.ts"
+import { dbNative } from "#db-native.ts"
+import { lower } from "#lower.ts"
+import type { AnyRelation, Fact } from "#relation.ts"
+import type { CellValue } from "#rows.ts"
+import { assertHostCellFits, cellOf, recordOf } from "#rows.ts"
+import type { ExecutionPolicy } from "#runtime.ts"
+import { nativeOperationWith, policyWire, runtimeHandle } from "#runtime.ts"
+import type { CloseReport } from "#runtime-errors.ts"
+import { DbError } from "#runtime-errors.ts"
+import type { AnySchema } from "#schema.ts"
+import type { Rel } from "#shape.ts"
+
+/**
+ * `ChangeSet` — the public name of the engine's checked immutable delta
+ * (chapter 30/34): schema-fingerprint-bound canonical native bytes with
+ * one-command `(add, remove ∖ add)` normalization and exact same-fact
+ * add-wins. Immutable and reusable while open; sealing/submitting it later
+ * retains the SAME native value — no second JS row walk ever happens.
+ */
+interface ChangeSet<S extends AnySchema> {
+	readonly schemaId: SchemaId
+	/** Phantom schema brand: a ChangeSet is only ever applied to its own S. */
+	readonly schema?: S
+	close(): Effect.Effect<CloseReport>
+}
+
+/**
+ * `ChangeDraft` — the scoped, database-free construction capability.
+ * Every method constructs a LAZY effect; execution reads the then-current
+ * iterable and charges its work again on every sequential rerun (no hidden
+ * memoization, no automatic retry, no iterator replay). Input must stay
+ * stable from an ingestion effect's execution start through its Exit;
+ * after successful ingestion the accepted native bytes are independent.
+ * Failure/interruption SPENDS the draft and initiates tracked drain.
+ * Concurrent/reentrant construction refuses and spends/drains the draft.
+ * `finish` consumes the draft; later ingestion or a second finish refuses
+ * through the spent capability state.
+ */
+interface ChangeDraft<S extends AnySchema> {
+	insert<R extends Rel<S>>(relation: R, rows: Iterable<Fact<R>>): Effect.Effect<void, DbError>
+	delete<R extends Rel<S>>(relation: R, rows: Iterable<Fact<R>>): Effect.Effect<void, DbError>
+	finish(): Effect.Effect<ChangeSet<S>, DbError, Scope.Scope>
+	close(): Effect.Effect<CloseReport>
+}
+
+/**
+ * Host-copy granularity: one bounded host-to-native message per chunk, so
+ * real event-loop turns happen between chunks (each chunk completes through
+ * a native callback, not a microtask chain). These are converter
+ * granularity bounds, not database-size policy — the draft's aggregate
+ * input/working/spill budget is charged natively and is CUMULATIVE across
+ * calls and chunks (chunks never reset it).
+ */
+const CHUNK_BYTES = 65536n
+const CHUNK_ROWS = 4096
+
+interface DraftState {
+	readonly handle: DraftHandle
+	readonly policy: ExecutionPolicy
+	readonly theory: AnySchema
+	spent: boolean
+	inFlight: boolean
+}
+
+interface ChangesInternal {
+	readonly handle: ChangesHandle
+	readonly schemaId: SchemaId
+	closed: boolean
+}
+
+const changesInternals = new WeakMap<object, ChangesInternal>()
+
+/** Private cross-module accessor (db.apply retains the native ChangeSet). */
+function internalChanges(value: object): ChangesInternal | undefined {
+	return changesInternals.get(value)
+}
+
+function refusal(operation: string, reason: "SpentHandle" | "ClosedHandle" | "InvalidArgument"): DbError {
+	return new DbError({ operation, reason: { _tag: reason } })
+}
+
+interface Chunk {
+	readonly rows: bigint
+	readonly cells: readonly CellValue[]
+	readonly done: boolean
+	readonly leftover: object | undefined
+}
+
+function eventLoopTurn(): Effect.Effect<void> {
+	return Effect.callback<void>((resume) => {
+		const id = setImmediate(() => resume(Effect.void))
+		return Effect.sync(() => clearImmediate(id))
+	})
+}
+
+function hostFactCharge(relation: AnyRelation, record: Readonly<Record<string, unknown>>): bigint {
+	const data = relation.data
+	let bytes = 0n
+	for (const declared of data.fields) {
+		const value = record[declared.name]
+		if (value === undefined) {
+			throw refusal("ChangeDraft.ingest", "InvalidArgument")
+		}
+		bytes += assertHostCellFits(`relation ${data.name} field ${declared.name}`, value, CHUNK_BYTES)
+	}
+	return bytes
+}
+
+function projectFact(relation: AnyRelation, record: Readonly<Record<string, unknown>>): readonly CellValue[] {
+	const data = relation.data
+	const cells: CellValue[] = []
+	for (const declared of data.fields) {
+		const value = record[declared.name]
+		if (value === undefined) {
+			throw refusal("ChangeDraft.ingest", "InvalidArgument")
+		}
+		cells.push(cellOf(`relation ${data.name} field ${declared.name}`, declared.field, value))
+	}
+	return cells
+}
+
+/**
+ * Pulls one bounded chunk off the caller's iterator. Host length is judged
+ * before any string scan or byte copy. A leftover fact that does not fit
+ * the current chunk is returned unconverted for the next turn.
+ */
+function pullChunk(relation: AnyRelation, iterator: Iterator<object>, pending: object | undefined): Chunk {
+	const cells: CellValue[] = []
+	let rows = 0n
+	let bytes = 0n
+	let leftover: object | undefined
+	let current: object | undefined = pending
+	while (rows < BigInt(CHUNK_ROWS) && leftover === undefined) {
+		if (current === undefined) {
+			const next = iterator.next()
+			if (next.done === true) {
+				return { rows, cells, done: true, leftover: undefined }
+			}
+			current = next.value
+		}
+		const record = recordOf(current)
+		const charge = hostFactCharge(relation, record)
+		if (rows > 0n && bytes + charge > CHUNK_BYTES) {
+			leftover = current
+			break
+		}
+		cells.push(...projectFact(relation, record))
+		bytes += charge
+		rows += 1n
+		current = undefined
+	}
+	return { rows, cells, done: false, leftover }
+}
+
+function spendAndDrain(state: DraftState, operation: string): Effect.Effect<void> {
+	return Effect.suspend(() => {
+		if (state.spent) {
+			return Effect.void
+		}
+		state.spent = true
+		// Tracked drain: join the native close transition; the report is
+		// diagnostic here (the ingestion failure itself is the caller's
+		// error), but native Closing accounting is never dropped.
+		return drainClose(operation, (callback) => dbNative.runtimeDraftClose(state.handle, callback)).pipe(Effect.asVoid)
+	})
+}
+
+function ingest(
+	state: DraftState,
+	operation: "ChangeDraft.insert" | "ChangeDraft.delete",
+	relation: AnyRelation,
+	rows: Iterable<object>
+): Effect.Effect<void, DbError> {
+	const verb = operation === "ChangeDraft.insert" ? dbNative.runtimeDraftInsert : dbNative.runtimeDraftDelete
+	return Effect.gen(function* () {
+		if (state.spent) {
+			return yield* Effect.fail(refusal(operation, "SpentHandle"))
+		}
+		if (state.inFlight) {
+			// Reentrant construction refuses AND spends/drains — there is
+			// no implicit queue (chapter 35).
+			yield* spendAndDrain(state, operation)
+			return yield* Effect.fail(refusal(operation, "SpentHandle"))
+		}
+		if (state.theory.relations[relation.name] !== relation) {
+			return yield* Effect.fail(refusal(operation, "InvalidArgument"))
+		}
+		const tables = schemaTables(state.theory)
+		const relationId = tables.relationIds.get(relation.name)
+		if (relationId === undefined) {
+			return yield* Effect.fail(refusal(operation, "InvalidArgument"))
+		}
+		state.inFlight = true
+		const wire = yield* Effect.try({
+			try: () => policyWire(state.policy, operation),
+			catch: () => refusal(operation, "InvalidArgument")
+		})
+		const body = Effect.gen(function* () {
+			const iterator = rows[Symbol.iterator]()
+			let leftover: object | undefined
+			let done = false
+			while (!done) {
+				const chunk = yield* Effect.try({
+					try: () => pullChunk(relation, iterator, leftover),
+					catch: (cause) => (cause instanceof DbError ? cause : refusal(operation, "InvalidArgument"))
+				}).pipe(Effect.catch((error) => spendAndDrain(state, operation).pipe(Effect.andThen(Effect.fail(error)))))
+				leftover = chunk.leftover
+				done = chunk.done && leftover === undefined
+				if (chunk.rows === 0n) {
+					continue
+				}
+				yield* eventLoopTurn()
+				yield* nativeOperationWith(
+					operation,
+					(callback) => verb(state.handle, wire, relationId, chunk.rows, chunk.cells, callback),
+					dbNative.runtimeReportTake,
+					() => undefined
+				).pipe(
+					Effect.catch((error) =>
+						Effect.sync(() => {
+							state.spent = true
+						}).pipe(Effect.andThen(Effect.fail(error)))
+					)
+				)
+			}
+		})
+		return yield* Effect.onExit(body, (exit) => {
+			state.inFlight = false
+			if (state.spent || Exit.isSuccess(exit)) {
+				return Effect.void
+			}
+			return spendAndDrain(state, operation)
+		})
+	})
+}
+
+function makeChangeSet<S extends AnySchema>(handle: ChangesHandle, schemaId: SchemaId): ChangeSet<S> {
+	const value: ChangeSet<S> = {
+		schemaId,
+		close() {
+			return drainClose("ChangeSet.close", (callback) => dbNative.runtimeChangesClose(handle, callback))
+		}
+	}
+	Object.freeze(value)
+	changesInternals.set(value, { handle, schemaId, closed: false })
+	return value
+}
+
+function makeDraft<S extends AnySchema>(state: DraftState, schemaId: SchemaId): ChangeDraft<S> {
+	const draft: ChangeDraft<S> = {
+		insert(relation, rows) {
+			return ingest(state, "ChangeDraft.insert", relation, rows)
+		},
+		delete(relation, rows) {
+			return ingest(state, "ChangeDraft.delete", relation, rows)
+		},
+		finish() {
+			return Effect.acquireRelease(
+				Effect.gen(function* () {
+					if (state.spent) {
+						return yield* Effect.fail(refusal("ChangeDraft.finish", "SpentHandle"))
+					}
+					if (state.inFlight) {
+						yield* spendAndDrain(state, "ChangeDraft.finish")
+						return yield* Effect.fail(refusal("ChangeDraft.finish", "SpentHandle"))
+					}
+					// Finish CONSUMES the draft, success or failure.
+					state.spent = true
+					const wire = yield* nativeOperationWith(
+						"ChangeDraft.finish",
+						(callback) =>
+							dbNative.runtimeDraftFinish(state.handle, policyWire(state.policy, "ChangeDraft.finish"), callback),
+						dbNative.runtimeChangesTake,
+						(value) => value
+					)
+					return makeChangeSet<S>(wire.changes, schemaId)
+				}),
+				(changes) =>
+					Effect.suspend(() => {
+						const internal = changesInternals.get(changes)
+						if (internal === undefined || internal.closed) {
+							return Effect.void
+						}
+						internal.closed = true
+						return releaseOwner("ChangeSet.close", (callback) =>
+							dbNative.runtimeChangesClose(internal.handle, callback)
+						)
+					}),
+				{ interruptible: true }
+			)
+		},
+		close() {
+			return Effect.suspend(() => {
+				state.spent = true
+				return drainClose("ChangeDraft.close", (callback) => dbNative.runtimeDraftClose(state.handle, callback))
+			})
+		}
+	}
+	return Object.freeze(draft)
+}
+
+/**
+ * `ChangeSet.builder(schema, work)` — lazy scoped acquisition of a
+ * database-free draft (chapter 35 roster). Requires the acquired
+ * `NativeRuntime`; the draft's native resources release with its scope, and
+ * the scope finalizer surfaces incomplete/failed teardown as a
+ * `CloseFailure` defect.
+ */
+const builder = Effect.fn("ChangeSet.builder")(function* <S extends AnySchema>(schema: S, work: ExecutionPolicy) {
+	const handle = yield* runtimeHandle()
+	const compiled = yield* CoreSchema.compile(schema, work)
+	const spec = lower(schema)
+	return yield* Effect.acquireRelease(
+		Effect.gen(function* () {
+			const draftHandle = yield* nativeOperationWith(
+				"ChangeSet.builder",
+				(callback) => dbNative.runtimeDraftOpen(handle, policyWire(work, "ChangeSet.builder"), spec, callback),
+				dbNative.runtimeDraftTake,
+				(value) => value
+			)
+			const state: DraftState = {
+				handle: draftHandle,
+				policy: work,
+				theory: schema,
+				spent: false,
+				inFlight: false
+			}
+			const draft = makeDraft<S>(state, compiled.schemaId)
+			draftStates.set(draft, state)
+			return draft
+		}),
+		(draft) =>
+			Effect.suspend(() => {
+				const state = draftStates.get(draft)
+				if (state === undefined) {
+					return Effect.void
+				}
+				// Idempotent: repeated close joins the same native
+				// transition; the finalizer runs it unconditionally so an
+				// abandoned draft is always drained.
+				state.spent = true
+				return releaseOwner("ChangeDraft.close", (callback) => dbNative.runtimeDraftClose(state.handle, callback))
+			}),
+		{ interruptible: true }
+	)
+})
+
+// The draft value → state registry lets the scope finalizer reach the
+// native handle without exposing it on the public capability.
+const draftStates = new WeakMap<object, DraftState>()
+
+const ChangeSet = Object.freeze({ builder })
+
+export type { ChangeDraft }
+export { ChangeSet, internalChanges }

package/src/close.ts

@@ -0,0 +1,53 @@
+import { Effect } from "effect"
+import { finalizeClose } from "#runtime.ts"
+import type { CloseReport } from "#runtime-errors.ts"
+import { DbError } from "#runtime-errors.ts"
+/**
+ * Shared close/drain adapters for scoped core owners (drafts, changes,
+ * snapshots, sessions, results, cursors, databases). One policy, chapter
+ * 35's: early `close()` starts/joins the native close transition and
+ * returns the honest {@link CloseReport}; a scope FINALIZER runs the same
+ * close and surfaces `incomplete`/`failed` as a structured `CloseFailure`
+ * DEFECT in the finalizer Cause — never catch-and-log, never false
+ * quiescence. Teardown uses the runtime's reserved cleanup envelope
+ * natively; repeated close joins the same stored transition (idempotent).
+ */
+import type { CloseWire } from "#runtime-native.ts"
+
+function reportOf(operation: string, wire: CloseWire): CloseReport {
+	if (wire.kind === "failed") {
+		return { kind: "failed", error: new DbError({ operation, reason: { _tag: "Internal" } }) }
+	}
+	return wire
+}
+
+/**
+ * Adapts one native close verb to `Effect<CloseReport>`: registration is
+ * synchronous, completion resumes exactly once, and the whole wait is
+ * uninterruptible — a close is a bounded registration/drain handshake, not
+ * long maskable work (the native side owns the cleanup deadline).
+ */
+function drainClose(
+	operation: string,
+	start: (callback: (report: CloseWire) => void) => void
+): Effect.Effect<CloseReport> {
+	return Effect.callback<CloseReport>((resume) => {
+		try {
+			start((report) => resume(Effect.succeed(reportOf(operation, report))))
+		} catch (cause) {
+			resume(
+				Effect.succeed({
+					kind: "failed",
+					error: cause instanceof DbError ? cause : new DbError({ operation, reason: { _tag: "Internal" } })
+				})
+			)
+		}
+	}).pipe(Effect.uninterruptible)
+}
+
+/** The finalizer policy: run the close, then die on incomplete/failed (E stays never). */
+function releaseOwner(operation: string, start: (callback: (report: CloseWire) => void) => void): Effect.Effect<void> {
+	return drainClose(operation, start).pipe(Effect.flatMap((report) => finalizeClose(operation, report)))
+}
+
+export { drainClose, releaseOwner }

package/src/closed.ts

@@ -1,4 +1,4 @@
-import * as errors from "@superbuilders/errors"
+import { AuthoringError } from "#errors.ts"
 import {
 	type AnyClosedIdField,
 	type AnyField,
@@ -14,7 +14,7 @@ import type { AnyRelation, RelationField } from "#relation.ts"
 import { resolveSelection, type SelectionBinding, type SelectionInput } from "#relation.ts"
 import type { LiteralSpec } from "#spec.ts"
 
-type PayloadField = Exclude<AnyField, { readonly fresh: true }>
+type PayloadField = AnyField
 
 type PayloadColumns = Record<string, PayloadField> & { readonly id?: never }
 
@@ -178,7 +178,7 @@ function mintAxioms<Handles extends string, Cols extends Record<string, PayloadF
 	}
 	Object.freeze(out)
 	if (!axiomsMinted<Handles, Cols>(out, handles, cols)) {
-		throw errors.new(`closed relation ${name}: axiom-row minting incomplete`)
+		throw new AuthoringError({ message: `closed relation ${name}: axiom-row minting incomplete` })
 	}
 	return out
 }
@@ -202,23 +202,27 @@ function closed<Name extends string, Handles extends ClosedHandleTuple, Cols ext
 	axioms?: Axioms<Handles[number], Cols>
 ): Closed<Name, Handles, Record<never, never>> | Closed<Name, Handles, Cols> {
 	if (!Array.isArray(handles)) {
-		throw errors.new(
-			`closed relation ${name}: payload columns declared without ground axioms — the payload tier is spelled closed(name, handles, columns, axioms)`
-		)
+		throw new AuthoringError({
+			message: `closed relation ${name}: payload columns declared without ground axioms — the payload tier is spelled closed(name, handles, columns, axioms)`
+		})
 	}
 	if (!isHandleTuple(handles)) {
-		throw errors.new(`closed relation ${name}: at least one handle is required (an empty vocabulary declares nothing)`)
+		throw new AuthoringError({
+			message: `closed relation ${name}: at least one handle is required (an empty vocabulary declares nothing)`
+		})
 	}
 	if (columns === undefined) {
 		if (axioms !== undefined) {
-			throw errors.new(`closed relation ${name}: the bare tier declares no columns, so ground axioms are inadmissible`)
+			throw new AuthoringError({
+				message: `closed relation ${name}: the bare tier declares no columns, so ground axioms are inadmissible`
+			})
 		}
 		return closedBare(name, handles)
 	}
 	if (axioms === undefined) {
-		throw errors.new(
-			`closed relation ${name}: payload columns declared without ground axioms — the payload tier is spelled closed(name, handles, columns, axioms)`
-		)
+		throw new AuthoringError({
+			message: `closed relation ${name}: payload columns declared without ground axioms — the payload tier is spelled closed(name, handles, columns, axioms)`
+		})
 	}
 	return closedPayload(name, handles, columns, axioms)
 }
@@ -236,7 +240,7 @@ function closedBare<Name extends string, const Hs extends ClosedHandleTuple>(
 	}
 	Object.freeze(empty)
 	if (!axiomsMinted<Hs[number], Record<never, never>>(empty, handles, [])) {
-		throw errors.new(`closed relation ${name}: bare-tier axiom-row minting incomplete`)
+		throw new AuthoringError({ message: `closed relation ${name}: bare-tier axiom-row minting incomplete` })
 	}
 	return mintClosed(name, handles, {}, empty)
 }
@@ -256,7 +260,7 @@ function closedPayload<Name extends string, Handles extends ClosedHandleTuple, C
 		assertDeclarationOrderKey(`closed relation ${name} handle`, handle)
 	}
 	if (!handleKeysOwn(axioms, handles)) {
-		throw errors.new(`closed relation ${name}: handle enumeration incomplete`)
+		throw new AuthoringError({ message: `closed relation ${name}: handle enumeration incomplete` })
 	}
 	return mintClosed(name, handles, columns, axioms)
 }
@@ -283,7 +287,7 @@ function mintClosed<Name extends string, Handles extends ClosedHandleTuple, Cols
 	const seen = new Set<string>()
 	for (const handle of handles) {
 		if (seen.has(handle)) {
-			throw errors.new(`closed relation ${name}: duplicate handle ${handle}`)
+			throw new AuthoringError({ message: `closed relation ${name}: duplicate handle ${handle}` })
 		}
 		seen.add(handle)
 	}
@@ -293,9 +297,9 @@ function mintClosed<Name extends string, Handles extends ClosedHandleTuple, Cols
 	for (const [columnName, field] of Object.entries(columns)) {
 		assertDeclarationOrderKey(`closed relation ${name} column`, columnName)
 		if (columnName === "id") {
-			throw errors.new(
-				`closed relation ${name}: the payload column id collides with the sealed shape's synthetic id (the relation mints its own id at ordinal 0; name the column something else)`
-			)
+			throw new AuthoringError({
+				message: `closed relation ${name}: the payload column id collides with the sealed shape's synthetic id (the relation mints its own id at ordinal 0; name the column something else)`
+			})
 		}
 		cols.push(Object.freeze({ name: columnName, field }))
 	}
@@ -323,7 +327,9 @@ function mintClosed<Name extends string, Handles extends ClosedHandleTuple, Cols
 	function where(selection: ClosedSelectionInput<Cols>): SelectedClosed<Name, Handles, Cols> {
 		const owner = holder.value
 		if (owner === undefined) {
-			throw errors.new(`closed relation ${name}: self-reference read before construction completed`)
+			throw new AuthoringError({
+				message: `closed relation ${name}: self-reference read before construction completed`
+			})
 		}
 		return Object.freeze({
 			relation: owner,
@@ -334,7 +340,7 @@ function mintClosed<Name extends string, Handles extends ClosedHandleTuple, Cols
 	const value: ClosedCore<Name, Handles, Cols> =
 		cols.length > 0 ? Object.freeze({ ...core, where }) : Object.freeze(core)
 	if (!surfaceMinted(value, cols)) {
-		throw errors.new(`closed relation ${name}: ergonomic-surface minting incomplete`)
+		throw new AuthoringError({ message: `closed relation ${name}: ergonomic-surface minting incomplete` })
 	}
 	holder.value = value
 	return value