@bjornpagen/bumbledb 1.3.0 → 2.0.0-rc.1

package/COOKBOOK.md

@@ -3,7 +3,9 @@
 Worked recipes for the Effect-native surface: typed schema values,
 declared keys and laws, application-owned `Uuid` identity, one bounded
 native runtime, scoped resources, immutable final-state changes, `Option`
-reads, sealed complete results, and one-shot page streams.
+reads, sealed complete results, one-shot page streams, and durable
+`Database`s over an object store. Recipes 1 to 18 use the embedded `Db` from
+`@bjornpagen/bumbledb/engine`; recipes 19 and 20 use `Database`.
 
 Every `ts` fence below is extracted and type-checked against the real
 package surface by `test/cookbook-doc.test.ts`; the imports fence here is
@@ -11,7 +13,7 @@ prepended to every recipe. The examples are lazy Effect programs — nothing
 below runs a database at import time.
 
 ```ts
-import { Effect, Option, Result, Stream } from "effect"
+import { Effect, Layer, Option, Result, Schema as EffectSchema, Stream } from "effect"
 import {
 	alternatives,
 	bool,
@@ -21,23 +23,25 @@ import {
 	closedId,
 	Compute,
 	contained,
-	Db,
-	describeQuery,
+	Database,
+	decodeBoundaryField,
 	duration,
+	encodeBoundaryField,
 	f64,
+	fieldSchema,
+	FsStore,
 	i64,
 	uuid,
 	Uuid,
 	interval,
 	key,
 	mirrors,
-	NativeRuntime,
+	Bumble,
 	on,
 	query,
-	queryFromDescription,
 	ref,
 	relation,
-	Scalar,
+	S3Store,
 	schema,
 	select,
 	str,
@@ -46,14 +50,16 @@ import {
 	weigh,
 	within
 } from "@bjornpagen/bumbledb"
+import { Db } from "@bjornpagen/bumbledb/engine"
 import type {
-	ApplyOutcome,
 	CompleteResult,
 	Fact,
 	FloatIntervalValue,
 	IntervalValue,
-	QueryReader
+	Migrations,
+	S3Sender
 } from "@bjornpagen/bumbledb"
+import type { ApplyOutcome, QueryReader } from "@bjornpagen/bumbledb/engine"
 
 declare const localPath: string
 ```
@@ -97,7 +103,7 @@ void [Learning, scoreIsNumber, unitsAreExact]
 
 ## 2. One runtime layer; explicit create and open
 
-`NativeRuntime.layer()` acquires the single native runtime
+`Bumble.layer()` acquires the single native runtime
 with scope; reuse ONE layer value so Effect's memoization shares it.
 Optional configuration controls workers, outstanding jobs/handles, and cleanup
 reporting. Defaults are up to four workers, 128 queued jobs, 128 cleanup reports,
@@ -105,10 +111,8 @@ reporting. Defaults are up to four workers, 128 queued jobs, 128 cleanup reports
 Operations use ordinary allocation without byte/row/work quotas or execution
 deadlines. Effect interruption requests cooperative cancellation. Runtime
 inspection reports outstanding work, not memory usage.
-Database `inspect().storage` reports virtual map extent, populated file length,
-non-free LMDB pages, and allocated disk blocks (null when unavailable).
-None of these measures process heap usage or resident RAM. Log history
-inspection uses the same `StorageInspection` fields.
+Database `inspect()` reports the generation, the database's bytes on disk and
+the native operations it retains; none of these measures heap or resident RAM.
 `Db.open` never creates a missing database; `Db.create` refuses existing
 authority. Both are scoped acquisitions.
 
@@ -131,7 +135,7 @@ const createOnce = Effect.scoped(
 )
 
 // One boundary; an Effect app provides the layer in its own graph instead.
-const layer = NativeRuntime.layer()
+const layer = Bumble.layer()
 void [openExisting.pipe(Effect.provide(layer)), createOnce.pipe(Effect.provide(layer))]
 ```
 
@@ -155,15 +159,14 @@ const applyOnce = Effect.scoped(
 		const draft = yield* ChangeSet.builder(Tasks)
 		yield* draft.insert(Task, [{ id: taskId, title: "write the cookbook", done: 0n }])
 		const changes = yield* draft.finish()
-		const outcome: ApplyOutcome = yield* db.apply(changes, { expected: { kind: "any" } })
-		switch (outcome.kind) {
-			case "accepted":
-			case "no-change":
-				return outcome.witness
-			case "invariant-rejected":
+		const outcome: ApplyOutcome = yield* db.apply(changes)
+		switch (outcome._tag) {
+			case "Committed":
+				return outcome.generation
+			case "Rejected":
 				// Complete statement diagnostics, typed data — never a throw.
 				return yield* Effect.fail(outcome.violations)
-			case "moved":
+			case "Moved":
 				return outcome.current
 		}
 	})
@@ -173,9 +176,9 @@ void applyOnce
 
 For a noncommitting check, `db.judge` runs the same admission procedure and
 aborts its private candidate on the native worker. It briefly takes the
-writer; it does not hold a writer session open in JavaScript. Admitted and
-rejected judgments include the actual base and net proposed fact counts.
-Operational errors stay in the Effect error channel.
+writer; it does not hold a writer session open in JavaScript. A judgment
+carries the generation it judged against, and an admitted one also carries the
+net proposed fact counts. Operational errors stay in the Effect error channel.
 
 ```ts
 const judgeOnce = Effect.scoped(
@@ -184,15 +187,15 @@ const judgeOnce = Effect.scoped(
 		const draft = yield* ChangeSet.builder(Tasks)
 		const id = yield* Effect.sync(() => crypto.randomUUID())
 		yield* draft.insert(Task, [{ id, title: "inspect a candidate", done: 0n }])
-		return yield* db.judge(yield* draft.finish(), { expected: { kind: "any" } })
+		return yield* db.judge(yield* draft.finish())
 	})
 )
 void judgeOnce
 ```
 
 Judgment is optional: `apply` always judges for itself. To act on a judgment,
-use its base as an exact expected witness and handle `moved`; do not assume
-another writer could not change the database between these operations.
+judge and apply with the same snapshot's witness and handle `Moved`; do not
+assume another writer could not change the database between these operations.
 
 Change sets also work as inspectable values without a database. Their
 `counts` report distinct requested additions/removals, not the net change
@@ -318,39 +321,6 @@ const readTwoAuthors = (reader: QueryReader<typeof Library>) =>
 void readTwoAuthors
 ```
 
-Generated queries use the same checked rule, scalar, and execution machinery.
-`describeQuery` exposes the logical IR and its names as owned data. Its relation
-ordinals refer to the supplied schema's declaration order; variables are local
-to each rule. Intermediate names are local labels. The result-field record
-passed to `queryFromDescription` must match the actual derived head, including
-closed vocabularies, and determines the returned row type. Parameters supplied
-to a generated query are checked against their uses during execution.
-
-```ts
-const Event = relation("Event", { id: uuid, amount: u64 })
-const Events = schema("Events", { Event }, [key(Event, ["id"])])
-const countEvents = query(Events).rule((r) =>
-	r.match(Event, {}).find({ count: r.count() })
-)
-const description = describeQuery(countEvents)
-const totalEvents = queryFromDescription(
-	Events,
-	{ ...description, columns: ["total"] },
-	{ total: u64 }
-)
-const readTotal = (reader: QueryReader<typeof Events>) =>
-	Effect.scoped(Effect.gen(function* () {
-		const result = yield* reader.execute(totalEvents, {})
-		const rows: readonly { readonly total: bigint }[] = yield* result.collect()
-		return rows
-	}))
-void readTotal
-```
-
-Descriptions contain bigint and byte values, so they are not a JSON wire format.
-An accepted description is an authored query; native preparation still checks
-engine semantics. It carries no snapshot, prepared handle, or result rows.
-
 ## 6. Grouped exact aggregates
 
 Aggregates fold the group's distinct full bindings: keep the identity-bearing
@@ -458,9 +428,9 @@ void [okOrRefused, mintOnce]
 
 ## 10. Witnessed correction: exact expected state
 
-Read under a short scope, keep the copied witness, and apply with
-`expected: { kind: "exact", at: witness }`. An intervening net change moves
-the apply instead of silently overwriting.
+Read under a short scope, keep the snapshot's witness (an opaque value that
+outlives the snapshot), and pass it to `apply` as the expected state. An intervening net change moves the apply instead of
+silently overwriting.
 
 ```ts
 const Account = relation("Account", { id: uuid, balance: i64 })
@@ -485,7 +455,7 @@ const correct = (accountId: Uuid) =>
 			yield* draft.delete(Account, [observed.previous])
 			yield* draft.insert(Account, [{ ...observed.previous, balance: observed.previous.balance + 1n }])
 			const changes = yield* draft.finish()
-			return yield* db.apply(changes, { expected: { kind: "exact", at: observed.at } })
+			return yield* db.apply(changes, observed.at)
 		})
 	)
 void correct
@@ -509,44 +479,28 @@ const dense: FloatIntervalValue = { start: 0.25, end: 1.5 }
 void [Windows, discrete, dense]
 ```
 
-## 12. Scoped ownership and honest close
+## 12. Scoped ownership
 
-Every native resource is scoped; early `close()` is itself an Effect
-returning the honest `CloseReport`. A scope finalizer that cannot complete
-teardown surfaces a structured `CloseFailure` DEFECT in the Cause — never a
-silently swallowed failure, never falsely reclaimed resources.
+Every native resource belongs to the scope that acquired it: the database, snapshots, prepared
+queries, results and change sets are released when their scope closes, innermost first. There is
+no `close()`; to release something early, acquire it in a smaller scope. A release that cannot
+drain cleanly is a `CloseFailure` defect in the Cause, never a swallowed failure.
 
 ```ts
 const Item = relation("Item", { id: uuid, label: str })
 const Items = schema("Items", { Item }, [key(Item, ["id"])])
 
-const explicitClose = Effect.scoped(
+const shortLived = Effect.scoped(
 	Effect.gen(function* () {
 		const db = yield* Db.open(localPath, Items)
-		const report = yield* db.close()
-		// `closed` releases this capability's obligations; `incomplete` and
-		// `failed` retain native Closing accounting — they are never
-		// counted as reclaimed.
-		return report.kind
+		const generation = yield* Effect.scoped(Effect.map(db.snapshot(), (snapshot) => snapshot.generation))
+		return generation
 	})
 )
-void explicitClose
+void shortLived
 ```
 
-## 13. Unresolved field arithmetic is authoring metadata
-
-`Scalar.field("units")` is not a typed program. Builders accept it inside
-arithmetic. Native compilation binds it against the verified source
-schema — including empty input — before any manifest write or freeze.
-
-```ts
-const incrementUnits = Scalar.add(Scalar.field("units"), Scalar.u64(1n))
-const asFloat = Scalar.toF64(Scalar.add(Scalar.field("units"), Scalar.u64(1n)))
-void [incrementUnits, asFloat]
-```
-
-
-## 14. Derive slices, measure them, and round the total
+## 13. Derive slices, measure them, and round the total
 
 An earning range crosses two rate bands. Intersection produces a relation of
 nonempty segments, measurement produces exact integer widths, and ordinary
@@ -586,7 +540,7 @@ const calculate = Effect.scoped(Effect.gen(function* () {
 		{ id: 1n, schedule: 1n, span: { start: 0n, end: 100n }, numerator: 1n },
 		{ id: 2n, schedule: 1n, span: { start: 100n, end: 200n }, numerator: 2n }
 	])
-	yield* db.apply(yield* change.finish(), { expected: { kind: "any" } })
+	yield* db.apply(yield* change.finish())
 	const snapshot = yield* db.snapshot()
 	return yield* (yield* snapshot.execute(amounts, {})).collect()
 }))
@@ -601,7 +555,7 @@ calculation; summing separately rounded contributions can give a different resul
 Checked multiplication and aggregate result ranges still apply before the final
 `mulDiv`. The query creates no stored slice or copied-width facts.
 
-## 15. Subtract a window, coalesce coverage, and measure
+## 14. Subtract a window, coalesce coverage, and measure
 
 Binary difference returns zero, one, or two maximal nonempty pieces. A subsequent
 `pack` stage coalesces overlapping or adjacent coverage per owner; measurement
@@ -656,7 +610,7 @@ segments, not arbitrary sums of already-rounded floating lengths.
 Each difference subtracts one interval. Unioning `A minus B1` with `A minus B2`
 does not subtract their combined coverage from `A`.
 
-## 16. Exhaustive alternatives with ordinary laws
+## 15. Exhaustive alternatives with ordinary laws
 
 Declare each key once. The helper expands a closed roster into ordinary
 containment and mirrors statements in roster order. Equivalent independently
@@ -691,7 +645,7 @@ Each payload keeps its own inferred fields and ordinary relation representation.
 Evidence and ownership still need their own laws. The expansion has the same
 native descriptor, fingerprint, and admission costs as the identical manual laws.
 
-## 17. Exact integer quotients and explicit rounding
+## 16. Exact integer quotients and explicit rounding
 
 `mulDiv` requires matching integer operands and a strictly positive divisor.
 It computes the product exactly in a wider native integer and checks the public
@@ -717,3 +671,178 @@ returns `-4`. Exact quotients do not change. `u64::MAX * 2 / 2` succeeds;
 Existing checked multiply/divide retain their intermediate-overflow behavior,
 so replacing them with `mulDiv` is an explicit semantic choice. All three modes
 perform bounded work without an additional scan, index, or JavaScript evaluator.
+
+## 17. Use field codecs directly
+
+Fields carry their host-value schema and strict JSON-boundary representation.
+Use the same descriptors in relations, selected input records, and individual
+values. Integer JSON values are canonical decimal strings; float JSON values
+are canonical `$f64` bit images. Native row bytes remain a separate format.
+
+```ts
+const Method = closed("Method", ["Electronic", "Postal"])
+const input = EffectSchema.Struct({
+	amount: fieldSchema(i64),
+	method: fieldSchema(closedId(Method)),
+	window: fieldSchema(interval(i64))
+})
+const valid = EffectSchema.is(input)({
+	amount: -25n,
+	method: "Electronic",
+	window: { start: 0n, end: 60n }
+})
+const amount = Result.getOrThrow(decodeBoundaryField(i64, "-25"))
+const json = Result.getOrThrow(encodeBoundaryField(i64, amount)) // "-25"
+void [valid, json]
+```
+
+The codecs enforce integer bounds, closed handles, byte widths, valid intervals,
+and canonical floating-point values through the same interpreter used by rows.
+Application parsing of dates, currency, or nonblank text composes with the field
+schema using ordinary Effect Schema operations.
+
+## 18. Combine projections and computations in one query
+
+Every arm supplies the same ordered output columns and compatible field types.
+A stored scalar and a computed scalar can occupy the same projection column.
+
+```ts
+const Entry = relation("Entry", { id: u64, amount: i64, previous: i64, correction: bool })
+const Ledger = schema("Ledger", { Entry }, [])
+const amounts = query(Ledger)
+	.rule((r) => {
+		const { id, amount } = v(Entry)
+		return r.match(Entry, { id, amount, correction: false }).find({ id, amount })
+	})
+	.rule((r) => {
+		const { id, amount, previous } = v(Entry)
+		return r.match(Entry, { id, amount, previous, correction: true })
+			.find({ id, amount: Compute.subtract(amount, previous) })
+	})
+const totals = query(Ledger).rule((r) => {
+	const row = v(amounts)
+	return r.match(amounts, row).find({ id: row.id, total: r.sum(row.amount) })
+})
+void totals
+```
+
+Identical output facts deduplicate before the following stage aggregates them.
+An evidenced zero remains a row; an absent group has none. Interval projections
+can combine with produced interval pieces of the same element kind; a width
+refinement survives only when every arm preserves it. Aggregate operators must
+agree across arms and cannot share a column with a projection.
+
+Each expression retains its original failure boundary. Moving filters across
+an imported stage can change which arithmetic executes; a later filter cannot
+erase an upstream overflow.
+
+## 19. A durable database: idempotent submits and fresh reads
+
+A `Database` is a log in an object store read through a disposable local
+cache. Provide it once with `Database.layer` under a service key; in a Lambda,
+build the layer during init. Production opens with `onOpen: "verify"` after
+`bumbledb migrate` ran in the deploy pipeline.
+
+Mint one request id per intent and reuse it for every retry of that intent: a
+command is decided at most once. `Decided` carries the receipt (`Committed`,
+`NoChange`, `PreconditionFailed` or `InvariantRejected`). Every refusal except
+`Unknown` proves the command is not in the log; after `Unknown`, `resolve`
+answers from the log. `precondition: reader.revision` decides the command only
+if nothing committed since that read, which makes read-modify-write exact.
+
+```ts
+const Task = relation("Task", { id: uuid, title: str, done: bool })
+const Tasks = schema("Tasks", { Task }, [key(Task, ["id"])])
+const openTasks = query(Tasks).rule((r) => {
+	const { id, title } = v(Task)
+	return r.match(Task, { id, title, done: false }).find({ id, title })
+})
+
+declare const migrations: Migrations
+declare const s3: S3Sender
+
+const TasksDb = Database.tag<typeof Tasks>("app/TasksDb")
+const TasksLive = Database.layer(TasksDb, {
+	schema: Tasks,
+	migrations,
+	store: S3Store.make({
+		client: s3,
+		log: { bucket: "tasks--use1-az4--x-s3" },
+		checkpoints: { bucket: "tasks-checkpoints" },
+		prefix: "prod/"
+	}),
+	cache: { directory: "/tmp/tasks" },
+	onOpen: "verify"
+}).pipe(Layer.provide(Bumble.layer()))
+
+const completeFirst = Effect.scoped(
+	Effect.gen(function* () {
+		const db = yield* TasksDb
+		const reader = yield* db.read("latest")
+		const [first] = yield* (yield* reader.execute(openTasks, {})).collect()
+		if (first === undefined) {
+			return "nothing open"
+		}
+		const draft = yield* ChangeSet.builder(Tasks)
+		yield* draft.delete(Task, [{ ...first, done: false }])
+		yield* draft.insert(Task, [{ ...first, done: true }])
+		const requestId = Database.requestId()
+		const outcome = yield* db.submit(yield* draft.finish(), { requestId, precondition: reader.revision })
+		if (outcome._tag === "Decided") {
+			return outcome.receipt.outcome._tag
+		}
+		if (outcome.refusal._tag === "Unknown") {
+			const receipt = yield* db.resolve(requestId)
+			return Option.isSome(receipt) ? receipt.value.outcome._tag : "not decided"
+		}
+		return outcome.refusal._tag
+	})
+)
+
+void completeFirst.pipe(Effect.provide(TasksLive))
+```
+
+`read` takes `"cached"` (the cache as it is), `"latest"` (one request to find
+the log's tip) or `{ atLeast: seq }` (read-your-writes after a receipt's `seq`).
+
+## 20. One database per tenant
+
+`Database.pool` opens a database per tenant on first use and closes it after
+it has been idle for `idleTimeToLive`. Each tenant has its own log and cache;
+`FsStore` runs the same protocol on a local directory, and `MemStore` is the
+in-memory store for tests.
+
+```ts
+const Note = relation("Note", { id: uuid, text: str })
+const Notes = schema("Notes", { Note }, [key(Note, ["id"])])
+const allNotes = query(Notes).rule((r) => {
+	const { id, text } = v(Note)
+	return r.match(Note, { id, text }).find({ id, text })
+})
+
+declare const noteMigrations: Migrations
+
+const perTenant = Effect.scoped(
+	Effect.gen(function* () {
+		const pool = yield* Database.pool({
+			schema: Notes,
+			migrations: noteMigrations,
+			store: (tenant) => FsStore.make(`data/${tenant}/log`),
+			cache: (tenant) => ({ directory: `data/${tenant}/cache` }),
+			onOpen: "migrate",
+			idleTimeToLive: "10 minutes"
+		})
+		const count = (tenant: string) =>
+			Effect.scoped(
+				Effect.gen(function* () {
+					const db = yield* pool.get(tenant)
+					const reader = yield* db.read("cached")
+					return (yield* (yield* reader.execute(allNotes, {})).collect()).length
+				})
+			)
+		return yield* Effect.all([count("acme"), count("globex")])
+	})
+)
+
+void perTenant.pipe(Effect.provide(Bumble.layer()))
+```