@bjornpagen/bumbledb 1.3.1 → 2.0.0-rc.1

package/src/database/mem.ts

@@ -0,0 +1,113 @@
+import * as fs from "node:fs/promises"
+import { Effect } from "effect"
+import { DbError } from "../errors.ts"
+import type { Body, Bucket, Created, Deleted, Fetched, Listed, Millis, ObjectStore, Reply, Target } from "./io.ts"
+
+type Verb = "get" | "putIfAbsent" | "list" | "delete"
+
+/** `Fail`: the call fails before it has any effect. `Lose`: the call takes effect, then fails, like a lost response. */
+type Fault = "Fail" | "Lose"
+
+interface MemStoreOptions {
+	/** The store clock behind `Date` and `Last-Modified`. Defaults to a counter that ticks once per call. */
+	readonly clock?: () => Millis
+	/** Picks the fault, if any, for each call. */
+	readonly fault?: (call: { readonly verb: Verb; readonly bucket: Bucket; readonly key: string }) => Fault | undefined
+}
+
+interface Stored {
+	readonly bytes: Uint8Array
+	readonly lastModified: Millis
+}
+
+/** An in-memory store with the create-only semantics of S3, and deterministic fault injection for tests. */
+interface MemStore extends ObjectStore {
+	/** A copy of the objects currently in `bucket`, by key. */
+	readonly objects: (bucket: Bucket) => ReadonlyMap<string, Uint8Array>
+}
+
+function failure(operation: string): DbError {
+	return new DbError({ operation, reason: { _tag: "Io", kind: "Injected" } })
+}
+
+function make(options: MemStoreOptions = {}): MemStore {
+	let ticks = 0n
+	const clock =
+		options.clock ??
+		(() => {
+			ticks += 1n
+			return ticks
+		})
+	const buckets: Record<Bucket, Map<string, Stored>> = { Log: new Map(), Checkpoints: new Map() }
+
+	function run<R>(
+		verb: Verb,
+		bucket: Bucket,
+		key: string,
+		body: (now: Millis) => Effect.Effect<R, DbError>
+	): Effect.Effect<Reply<R>, DbError> {
+		return Effect.suspend(() => {
+			const fault = options.fault?.({ verb, bucket, key })
+			const operation = `MemStore.${verb}`
+			if (fault === "Fail") return Effect.fail(failure(operation))
+			const now = clock()
+			return Effect.flatMap(body(now), (result) =>
+				fault === "Lose" ? Effect.fail(failure(operation)) : Effect.succeed({ date: now, result })
+			)
+		})
+	}
+
+	const io = <A>(operation: string, f: () => Promise<A>) =>
+		Effect.tryPromise({
+			try: f,
+			catch: () => new DbError({ operation, reason: { _tag: "Io", kind: "File" } })
+		})
+
+	return {
+		get: (bucket: Bucket, key: string, target: Target) =>
+			run("get", bucket, key, (): Effect.Effect<Fetched, DbError> => {
+				const stored = buckets[bucket].get(key)
+				if (stored === undefined) return Effect.succeed({ _tag: "Missing" })
+				if (target._tag === "Memory") {
+					return Effect.succeed({ _tag: "Body", bytes: stored.bytes.slice(), lastModified: stored.lastModified })
+				}
+				return io("MemStore.get", () => fs.writeFile(target.path, stored.bytes)).pipe(
+					Effect.as<Fetched>({ _tag: "Saved", lastModified: stored.lastModified })
+				)
+			}),
+		putIfAbsent: (bucket: Bucket, key: string, body: Body) =>
+			run("putIfAbsent", bucket, key, (now): Effect.Effect<Created, DbError> => {
+				const bytes =
+					body._tag === "Bytes"
+						? Effect.succeed(body.bytes.slice())
+						: io("MemStore.putIfAbsent", () => fs.readFile(body.path)).pipe(
+								Effect.map((buffer) => new Uint8Array(buffer))
+							)
+				return Effect.map(bytes, (owned): Created => {
+					const objects = buckets[bucket]
+					if (objects.has(key)) return { _tag: "Occupied" }
+					objects.set(key, { bytes: owned, lastModified: now })
+					return { _tag: "Created" }
+				})
+			}),
+		list: (prefix: string, startAfter: string | null, maxKeys: number) =>
+			run("list", "Checkpoints", prefix, (): Effect.Effect<Listed> => {
+				const keys = [...buckets.Checkpoints.keys()]
+					.filter((key) => key.startsWith(prefix) && (startAfter === null || key > startAfter))
+					.sort()
+					.slice(0, maxKeys)
+				return Effect.succeed({ _tag: "Keys", keys })
+			}),
+		delete: (key: string) =>
+			run("delete", "Checkpoints", key, (): Effect.Effect<Deleted> => {
+				buckets.Checkpoints.delete(key)
+				return Effect.succeed({ _tag: "Deleted" })
+			}),
+		objects: (bucket: Bucket) => new Map([...buckets[bucket]].map(([key, stored]) => [key, stored.bytes.slice()]))
+	}
+}
+
+const MemStore = Object.freeze({ make })
+
+export type { Fault, MemStoreOptions }
+export { MemStore }

package/src/database/migration.ts

@@ -0,0 +1,107 @@
+import type { Effect, Scope } from "effect"
+import type { ChangeDraft } from "../changes.ts"
+import type { QueryReader } from "../db.ts"
+import type { DbError } from "../errors.ts"
+import { AuthoringError } from "../errors.ts"
+import type { AnySchema } from "../schema.ts"
+import { schemaDescriptor, schemasAgree } from "../schema.ts"
+
+/** What `populate` reads from (the database at the old schema) and writes into (rows at the new one). */
+interface PopulateContext<From extends AnySchema, To extends AnySchema> {
+	readonly from: QueryReader<From>
+	readonly into: ChangeDraft<To>
+}
+
+/**
+ * Computes the rows a migration adds at its new schema, reading the database at the old one.
+ * Relations that keep their name and fields are copied unchanged without code.
+ */
+type Populate<From extends AnySchema, To extends AnySchema> = (
+	context: PopulateContext<From, To>
+) => Effect.Effect<void, DbError, Scope.Scope>
+
+/**
+ * One bundled migration: its stable `id` (the directory name), the content `hash` that
+ * `bumbledb generate` wrote (64 hex digits), and the schema it migrates to. The first migration of a
+ * bundle has no `from`: it is the initial schema, and its `populate` seeds a newly created database
+ * once.
+ */
+interface Migration<From extends AnySchema | undefined = AnySchema | undefined, To extends AnySchema = AnySchema> {
+	readonly id: string
+	readonly hash: string
+	readonly from: From
+	readonly to: To
+	readonly populate?: Populate<From extends AnySchema ? From : To, To>
+}
+
+/** Any migration of a bundle; its `populate` is read back through {@link populateOf}. */
+interface AnyMigration {
+	readonly id: string
+	readonly hash: string
+	readonly from: AnySchema | undefined
+	readonly to: AnySchema
+	readonly populate?: unknown
+}
+
+/** A bundle: the initial migration first, then each migration from its predecessor's schema. */
+type Migrations = readonly [AnyMigration, ...AnyMigration[]]
+
+const HASH = /^[0-9a-f]{64}$/
+const ID = /^[0-9]{4}_[A-Za-z0-9_]+$/
+
+function make<To extends AnySchema, From extends AnySchema = never>(input: {
+	readonly id: string
+	readonly hash: string
+	readonly from?: From
+	readonly to: To
+	readonly populate?: Populate<[From] extends [never] ? To : From, To>
+}): Migration<[From] extends [never] ? undefined : From, To>
+function make(input: {
+	readonly id: string
+	readonly hash: string
+	readonly from?: AnySchema
+	readonly to: AnySchema
+	readonly populate?: Populate<AnySchema, AnySchema>
+}): AnyMigration {
+	if (!ID.test(input.id)) throw new AuthoringError({ message: `migration ${input.id}: an id is NNNN_name` })
+	if (!HASH.test(input.hash)) throw new AuthoringError({ message: `migration ${input.id}: a hash is 64 hex digits` })
+	const migration = {
+		id: input.id,
+		hash: input.hash,
+		from: input.from === undefined ? undefined : schemaDescriptor(input.from),
+		to: schemaDescriptor(input.to),
+		...(input.populate === undefined ? {} : { populate: input.populate })
+	}
+	return Object.freeze(migration)
+}
+
+/** The populate step of a migration `make` built. */
+function populateOf(migration: AnyMigration): Populate<AnySchema, AnySchema> | undefined {
+	return migration.populate as Populate<AnySchema, AnySchema> | undefined
+}
+
+/** Checks that each migration starts from its predecessor's schema and that ids are unique. */
+function verifyBundle(migrations: Migrations): void {
+	const ids = new Set<string>()
+	migrations.forEach((migration, index) => {
+		if (ids.has(migration.id)) throw new AuthoringError({ message: `migrations: ${migration.id} appears twice` })
+		ids.add(migration.id)
+		const previous = migrations[index - 1]
+		if (previous === undefined) {
+			if (migration.from !== undefined) {
+				throw new AuthoringError({ message: `migrations: the first migration ${migration.id} has no from schema` })
+			}
+			return
+		}
+		if (migration.from === undefined || !schemasAgree(migration.from, previous.to)) {
+			throw new AuthoringError({
+				message: `migrations: ${migration.id} must start from the schema ${previous.id} migrates to`
+			})
+		}
+	})
+}
+
+const Migration = Object.freeze({ make })
+
+export type { AnyMigration, Migrations, Populate, PopulateContext }
+export { Migration, populateOf, verifyBundle }

package/src/database/s3.ts

@@ -0,0 +1,231 @@
+import { createReadStream, createWriteStream } from "node:fs"
+import * as fs from "node:fs/promises"
+import type { Readable } from "node:stream"
+import { pipeline } from "node:stream/promises"
+import { Effect } from "effect"
+import { DbError } from "../errors.ts"
+import type { Body, Bucket, Created, Deleted, Fetched, Listed, Millis, ObjectStore, Reply, Target } from "./io.ts"
+
+type Sdk = typeof import("@aws-sdk/client-s3")
+
+let sdk: Promise<Sdk> | undefined
+
+/**
+ * The part of an AWS SDK v3 `S3Client` the store calls. Declared here so the package's types do not
+ * require the optional `@aws-sdk/client-s3` peer.
+ */
+interface S3Sender {
+	send(command: never, options: { readonly abortSignal: AbortSignal }): Promise<unknown>
+}
+
+interface S3StoreOptions {
+	/**
+	 * The application's own `S3Client`: it owns credentials, region, endpoint and SDK retries. Its
+	 * connections resolve DNS on libuv's thread pool, 4 threads by default, which serializes a cold
+	 * open's parallel reads where lookups are slow.
+	 */
+	readonly client: S3Sender
+	/** The commit log bucket, ideally an S3 Express directory bucket (`name--azid--x-s3`). */
+	readonly log: { readonly bucket: string }
+	/** The checkpoint bucket: an S3 Standard bucket, whose LIST is lexicographic. */
+	readonly checkpoints: { readonly bucket: string }
+	/** Prepended to every key; empty, or ending in `/`. */
+	readonly prefix: string
+}
+
+interface Answer<O> {
+	readonly output: O
+	readonly date: Millis | null
+}
+
+interface Refusal {
+	readonly status: number | undefined
+	readonly date: Millis | null
+}
+
+function dateOf(headers: unknown): Millis | null {
+	if (typeof headers !== "object" || headers === null || !("date" in headers) || typeof headers.date !== "string") {
+		return null
+	}
+	const millis = Date.parse(headers.date)
+	return Number.isNaN(millis) ? null : BigInt(millis)
+}
+
+function refusalOf(cause: unknown): Refusal {
+	if (typeof cause !== "object" || cause === null) return { status: undefined, date: null }
+	const metadata = "$metadata" in cause ? (cause.$metadata as { httpStatusCode?: number }) : undefined
+	const response = "$response" in cause ? (cause.$response as { headers?: unknown } | undefined) : undefined
+	return { status: metadata?.httpStatusCode, date: dateOf(response?.headers) }
+}
+
+/**
+ * Sends one command and reads the response's `Date` header, which the SDK does not surface on
+ * outputs. The middleware sits innermost in the deserialize step, so it sees every raw response.
+ */
+async function send<O>(
+	client: S3Sender,
+	command: { middlewareStack: { add: (...args: never[]) => void } },
+	signal: AbortSignal
+): Promise<Answer<O>> {
+	let date: Millis | null = null
+	const capture =
+		(next: (args: unknown) => Promise<{ response: unknown }>) =>
+		async (args: unknown): Promise<{ response: unknown }> => {
+			const result = await next(args)
+			const response = result.response as { headers?: unknown } | undefined
+			date = dateOf(response?.headers) ?? date
+			return result
+		}
+	const add = command.middlewareStack.add as (middleware: unknown, options: object) => void
+	add(capture, { step: "deserialize", priority: "low", name: "bumbledbResponseDate" })
+	const output = (await client.send(command as never, { abortSignal: signal })) as O
+	return { output, date }
+}
+
+/**
+ * S3 answers a create-only PUT that overlaps another conditional write to its key with 409. The
+ * SDK would retry it with backoff into a late 412. A log slot being written is taken for the log's
+ * purposes (it reads ownership back itself), so there the 409 is answered at once, as a refusal.
+ */
+const conflict = Symbol("bdb.conflict")
+
+function answerConflicts(command: { middlewareStack: { add: (...args: never[]) => void } }): void {
+	const add = command.middlewareStack.add as (middleware: unknown, options: object) => void
+	add(
+		(next: (args: unknown) => Promise<unknown>) => async (args: unknown) => {
+			try {
+				return await next(args)
+			} catch (cause) {
+				if (refusalOf(cause).status !== 409) throw cause
+				const response = "$response" in (cause as object) ? (cause as { $response?: unknown }).$response : undefined
+				return { output: { [conflict]: refusalOf(cause) }, response }
+			}
+		},
+		{ step: "deserialize", priority: "high", name: "bumbledbConflictAnswer" }
+	)
+}
+
+function millisOf(value: Date | undefined): Millis {
+	return BigInt(value?.getTime() ?? 0)
+}
+
+/** A store over the application's AWS SDK v3 client. Every key lives under `prefix`. */
+function make(options: S3StoreOptions): ObjectStore {
+	const { client, prefix } = options
+	const bucketName = (bucket: Bucket) => (bucket === "Log" ? options.log.bucket : options.checkpoints.bucket)
+	const load = () => {
+		sdk ??= import("@aws-sdk/client-s3")
+		return sdk
+	}
+
+	function attempt<R>(
+		operation: string,
+		run: (sdk: Sdk, signal: AbortSignal) => Promise<Reply<R>>,
+		refused: (refusal: Refusal) => Reply<R> | undefined
+	): Effect.Effect<Reply<R>, DbError> {
+		return Effect.tryPromise({
+			try: async (signal) => {
+				const module = await load()
+				try {
+					return await run(module, signal)
+				} catch (cause) {
+					const answer = refused(refusalOf(cause))
+					if (answer !== undefined) return answer
+					throw cause
+				}
+			},
+			catch: (cause) =>
+				new DbError({
+					operation,
+					reason: { _tag: "Io", kind: refusalOf(cause).status?.toString() ?? "Transport" }
+				})
+		})
+	}
+
+	return {
+		get: (bucket: Bucket, key: string, target: Target) =>
+			attempt<Fetched>(
+				"S3Store.get",
+				async ({ GetObjectCommand }, signal) => {
+					const answer = await send<{ Body?: unknown; LastModified?: Date }>(
+						client,
+						new GetObjectCommand({ Bucket: bucketName(bucket), Key: `${prefix}${key}` }),
+						signal
+					)
+					const body = answer.output.Body as (Readable & { transformToByteArray(): Promise<Uint8Array> }) | undefined
+					const lastModified = millisOf(answer.output.LastModified)
+					if (body === undefined) throw new Error("GetObject returned no body")
+					if (target._tag === "Memory") {
+						return {
+							date: answer.date,
+							result: { _tag: "Body", bytes: await body.transformToByteArray(), lastModified }
+						}
+					}
+					await pipeline(body, createWriteStream(target.path))
+					return { date: answer.date, result: { _tag: "Saved", lastModified } }
+				},
+				(refusal) => (refusal.status === 404 ? { date: refusal.date, result: { _tag: "Missing" } } : undefined)
+			),
+		putIfAbsent: (bucket: Bucket, key: string, body: Body) =>
+			attempt<Created>(
+				"S3Store.putIfAbsent",
+				async ({ PutObjectCommand }, signal) => {
+					const payload =
+						body._tag === "Bytes"
+							? { Body: body.bytes, ContentLength: body.bytes.byteLength }
+							: { Body: createReadStream(body.path), ContentLength: (await fs.stat(body.path)).size }
+					const command = new PutObjectCommand({
+						Bucket: bucketName(bucket),
+						Key: `${prefix}${key}`,
+						IfNoneMatch: "*",
+						...payload
+					})
+					if (bucket === "Log") answerConflicts(command)
+					const answer = await send<{ [conflict]?: Refusal }>(client, command, signal)
+					const refused = answer.output[conflict]
+					if (refused !== undefined) return { date: refused.date ?? answer.date, result: { _tag: "Occupied" } }
+					return { date: answer.date, result: { _tag: "Created" } }
+				},
+				(refusal) => (refusal.status === 412 ? { date: refusal.date, result: { _tag: "Occupied" } } : undefined)
+			),
+		list: (keyPrefix: string, startAfter: string | null, maxKeys: number) =>
+			attempt<Listed>(
+				"S3Store.list",
+				async ({ ListObjectsV2Command }, signal) => {
+					const answer = await send<{ Contents?: ReadonlyArray<{ Key?: string }> }>(
+						client,
+						new ListObjectsV2Command({
+							Bucket: options.checkpoints.bucket,
+							Prefix: `${prefix}${keyPrefix}`,
+							MaxKeys: maxKeys,
+							...(startAfter === null ? {} : { StartAfter: `${prefix}${startAfter}` })
+						}),
+						signal
+					)
+					const keys = (answer.output.Contents ?? []).flatMap((object) =>
+						object.Key?.startsWith(prefix) === true ? [object.Key.slice(prefix.length)] : []
+					)
+					return { date: answer.date, result: { _tag: "Keys", keys } }
+				},
+				() => undefined
+			),
+		delete: (key: string) =>
+			attempt<Deleted>(
+				"S3Store.delete",
+				async ({ DeleteObjectCommand }, signal) => {
+					const answer = await send(
+						client,
+						new DeleteObjectCommand({ Bucket: options.checkpoints.bucket, Key: `${prefix}${key}` }),
+						signal
+					)
+					return { date: answer.date, result: { _tag: "Deleted" } }
+				},
+				() => undefined
+			)
+	}
+}
+
+const S3Store = Object.freeze({ make })
+
+export type { S3Sender, S3StoreOptions }
+export { S3Store }