@statewalker/webrun-files-sqlite 0.10.0

package/src/blocks/stream-codec.ts

@@ -0,0 +1,132 @@
+/**
+ * A streaming compression format. Pulled, never pushed: `compress` and
+ * `decompress` read their input only as their output is consumed.
+ */
+export interface StreamCodec {
+  /** Stored in `fs_files.compression`. Implementations of one format share the name. */
+  name: string;
+  compress(input: AsyncIterable<Uint8Array>): AsyncIterable<Uint8Array>;
+  decompress(input: AsyncIterable<Uint8Array>): AsyncIterable<Uint8Array>;
+}
+
+/**
+ * zlib-wrapped Deflate over Compression Streams — browsers, Node, Deno, Bun,
+ * Workers. The source is wrapped in a pull-only ReadableStream, so the
+ * transform asks for input only when it has room, and cancelling the output
+ * closes the source.
+ */
+export function webDeflateCodec(): StreamCodec {
+  if (typeof CompressionStream === "undefined") {
+    throw new Error("webDeflateCodec: CompressionStream is not available in this runtime");
+  }
+  return {
+    name: "deflate",
+    compress: (input) =>
+      iterate(pullStream(input).pipeThrough(asPair(new CompressionStream("deflate")))),
+    decompress: (input) =>
+      iterate(pullStream(input).pipeThrough(asPair(new DecompressionStream("deflate")))),
+  };
+}
+
+/** The part of the `pako` module the streaming codec uses. */
+export interface PakoStreamModule {
+  Deflate: new (options?: { level?: number }) => PakoStream;
+  Inflate: new () => PakoStream;
+}
+
+interface PakoStream {
+  push(data: Uint8Array, flush?: boolean): boolean;
+  onData(chunk: Uint8Array): void;
+  err: number;
+  msg: string;
+  ended?: boolean;
+}
+
+/**
+ * zlib-wrapped Deflate through pako's incremental classes, for runtimes without
+ * Compression Streams. The module is injected, not imported.
+ */
+export function pakoDeflateCodec(
+  pako: PakoStreamModule,
+  opts: { level?: number } = {},
+): StreamCodec {
+  const options = opts.level === undefined ? {} : { level: opts.level };
+  return {
+    name: "deflate",
+    compress: (input) => pakoTransform(new pako.Deflate(options), input, true),
+    decompress: (input) => pakoTransform(new pako.Inflate(), input, false),
+  };
+}
+
+/**
+ * Feed one input chunk at a time and hand on whatever that produced before
+ * pulling again. pako's `push` is synchronous, so its output per chunk is
+ * bounded by the chunk.
+ */
+async function* pakoTransform(
+  stream: PakoStream,
+  input: AsyncIterable<Uint8Array>,
+  compressing: boolean,
+): AsyncGenerator<Uint8Array> {
+  let pending: Uint8Array[] = [];
+  stream.onData = (chunk) => pending.push(chunk);
+  const check = () => {
+    if (stream.err) throw new Error(`pako: ${stream.msg || `error ${stream.err}`}`);
+  };
+  for await (const chunk of input) {
+    if (chunk.length === 0) continue;
+    stream.push(chunk, false);
+    check();
+    const out = pending;
+    pending = [];
+    yield* out;
+  }
+  if (compressing) {
+    stream.push(new Uint8Array(0), true);
+    check();
+  } else if (!stream.ended) {
+    throw new Error("pako: truncated deflate stream");
+  }
+  yield* pending;
+}
+
+/** Compression streams accept any BufferSource; this pipe only ever writes Uint8Array. */
+function asPair(transform: CompressionStream | DecompressionStream) {
+  return transform as unknown as ReadableWritablePair<Uint8Array, Uint8Array>;
+}
+
+function pullStream(source: AsyncIterable<Uint8Array>): ReadableStream<Uint8Array> {
+  const iterator = source[Symbol.asyncIterator]();
+  return new ReadableStream<Uint8Array>(
+    {
+      async pull(controller) {
+        const next = await iterator.next();
+        if (next.done) controller.close();
+        else controller.enqueue(next.value);
+      },
+      async cancel(reason) {
+        await iterator.return?.(reason);
+      },
+    },
+    { highWaterMark: 0 },
+  );
+}
+
+async function* iterate(stream: ReadableStream<Uint8Array>): AsyncGenerator<Uint8Array> {
+  const reader = stream.getReader();
+  let finished = false;
+  try {
+    for (;;) {
+      const { done, value } = await reader.read();
+      if (done) {
+        finished = true;
+        return;
+      }
+      yield value;
+    }
+  } finally {
+    // An early exit cancels the pipeline, which closes the source.
+    if (!finished) await reader.cancel().catch(() => {});
+    reader.releaseLock();
+  }
+}

package/src/bytes.ts

@@ -0,0 +1,8 @@
+/** Blobs arrive as Uint8Array (node:sqlite), ArrayBuffer (Durable Objects) or number[] (D1). */
+export function toBytes(value: unknown): Uint8Array {
+  if (value instanceof Uint8Array) return value;
+  if (value instanceof ArrayBuffer) return new Uint8Array(value);
+  if (Array.isArray(value)) return Uint8Array.from(value);
+  if (typeof value === "string") return new TextEncoder().encode(value);
+  return new Uint8Array(0);
+}

package/src/codec.ts

@@ -0,0 +1,88 @@
+/**
+ * SQLAR compression.
+ *
+ * The format mandates zlib-wrapped Deflate (RFC 1950, the `78 xx` header) —
+ * what zlib's `compress()` writes. Every codec here writes exactly that, so an
+ * archive written by one is read by any other, and by other SQLAR tools.
+ */
+export interface Codec {
+  name: string;
+  /**
+   * Inputs shorter than this are stored plaintext without a deflate attempt:
+   * deflating a short input makes it longer.
+   */
+  minSize: number;
+  deflate(bytes: Uint8Array): Uint8Array | Promise<Uint8Array>;
+  inflate(data: Uint8Array): Uint8Array | Promise<Uint8Array>;
+}
+
+const MIN_SIZE = 64;
+
+/** Compression Streams — browsers, Node, Workers. No dependency. */
+export function webCodec(opts: { minSize?: number } = {}): Codec {
+  if (typeof CompressionStream === "undefined") {
+    throw new Error("webCodec: CompressionStream is not available in this runtime");
+  }
+  return {
+    name: "web",
+    minSize: opts.minSize ?? MIN_SIZE,
+    // "deflate" is the zlib-wrapped format; "deflate-raw" would not be SQLAR.
+    deflate: (bytes) => pipe(bytes, new CompressionStream("deflate")),
+    inflate: (data) => pipe(data, new DecompressionStream("deflate")),
+  };
+}
+
+/** The part of the `pako` module this codec uses. */
+export interface PakoModule {
+  deflate(data: Uint8Array, opts?: { level?: number }): Uint8Array;
+  inflate(data: Uint8Array): Uint8Array;
+}
+
+/**
+ * pako — synchronous, for runtimes without a usable CompressionStream. The
+ * module is injected so this package does not depend on it:
+ *
+ *     import * as pako from "pako";
+ *     new SqlarFilesApi(driver, { codec: pakoCodec(pako) });
+ */
+export function pakoCodec(
+  pako: PakoModule,
+  opts: { minSize?: number; level?: number } = {},
+): Codec {
+  const deflateOptions = opts.level === undefined ? {} : { level: opts.level };
+  return {
+    name: "pako",
+    minSize: opts.minSize ?? MIN_SIZE,
+    // pako.deflate is zlib-wrapped by default; deflateRaw is not.
+    deflate: (bytes) => pako.deflate(bytes, deflateOptions),
+    inflate: (data) => pako.inflate(data),
+  };
+}
+
+/**
+ * No compression — still a valid archive. Reading a compressed row through it
+ * throws rather than returning deflate bytes as file content.
+ */
+export function rawCodec(): Codec {
+  return {
+    name: "raw",
+    minSize: Number.POSITIVE_INFINITY,
+    deflate: (bytes) => bytes,
+    inflate() {
+      throw new Error("rawCodec cannot inflate: this archive holds compressed rows");
+    },
+  };
+}
+
+/** webCodec where CompressionStream exists, rawCodec elsewhere. */
+export function defaultCodec(): Codec {
+  return typeof CompressionStream === "undefined" ? rawCodec() : webCodec();
+}
+
+async function pipe(
+  bytes: Uint8Array,
+  transform: TransformStream<BufferSource, Uint8Array>,
+): Promise<Uint8Array> {
+  const stream = new Blob([bytes as BlobPart]).stream().pipeThrough(transform);
+  return new Uint8Array(await new Response(stream).arrayBuffer());
+}

package/src/drivers.ts

@@ -0,0 +1,133 @@
+import type { SqlDriver, SqlStatement } from "./sql.types.js";
+
+/** The subset of node:sqlite's `DatabaseSync` this driver uses. */
+export interface NodeSqliteDatabase {
+  exec(sql: string): void;
+  prepare(sql: string): {
+    all(...params: never[]): unknown[];
+    run(...params: never[]): { changes: number | bigint };
+  };
+}
+
+/** node:sqlite — local development, tests and any Node deployment. */
+export class NodeSqlDriver implements SqlDriver {
+  readonly #db: NodeSqliteDatabase;
+
+  constructor(db: NodeSqliteDatabase) {
+    this.#db = db;
+  }
+
+  all<T>(sql: string, ...params: unknown[]): T[] {
+    return this.#db.prepare(sql).all(...(params as never[])) as T[];
+  }
+
+  run(sql: string, ...params: unknown[]): void {
+    this.#db.prepare(sql).run(...(params as never[]));
+  }
+
+  /** Synchronous from BEGIN to COMMIT, so no other call can interleave on the connection. */
+  transaction(statements: SqlStatement[]): number[] {
+    this.#db.exec("BEGIN IMMEDIATE");
+    try {
+      const counts = statements.map(({ sql, params = [] }) =>
+        Number(this.#db.prepare(sql).run(...(params as never[])).changes),
+      );
+      this.#db.exec("COMMIT");
+      return counts;
+    } catch (error) {
+      this.#db.exec("ROLLBACK");
+      throw error;
+    }
+  }
+}
+
+/** The subset of a Durable Object's `ctx.storage` this driver uses. */
+export interface DoStorage {
+  sql: {
+    exec(query: string, ...bindings: unknown[]): { toArray(): unknown[] };
+  };
+  transactionSync<T>(fn: () => T): T;
+}
+
+/**
+ * Durable Object SQLite storage: pass `ctx.storage`. Synchronous; BLOBs come
+ * back as ArrayBuffer. Transactions use `transactionSync`, since `sql.exec`
+ * rejects `BEGIN`.
+ */
+export class DoSqlDriver implements SqlDriver {
+  readonly #storage: DoStorage;
+
+  constructor(storage: DoStorage) {
+    this.#storage = storage;
+  }
+
+  all<T>(sql: string, ...params: unknown[]): T[] {
+    return this.#storage.sql.exec(sql, ...params).toArray() as T[];
+  }
+
+  run(sql: string, ...params: unknown[]): void {
+    // The cursor is lazy; draining it makes sure the statement has run.
+    this.#storage.sql.exec(sql, ...params).toArray();
+  }
+
+  /**
+   * Counts come from SQLite's `changes()` on the same connection, not from the
+   * cursor's `rowsWritten`, which also counts every index row a write touches.
+   */
+  transaction(statements: SqlStatement[]): number[] {
+    return this.#storage.transactionSync(() =>
+      statements.map(({ sql, params = [] }) => {
+        this.#storage.sql.exec(sql, ...params).toArray();
+        const [row] = this.#storage.sql.exec("SELECT changes() AS n").toArray() as { n: number }[];
+        return row.n;
+      }),
+    );
+  }
+}
+
+/** The subset of a `D1Database` this driver uses. */
+export interface D1Database {
+  prepare(query: string): D1PreparedStatement;
+  batch(statements: D1BoundStatement[]): Promise<{ meta: { changes: number } }[]>;
+}
+
+interface D1PreparedStatement {
+  bind(...values: unknown[]): D1BoundStatement;
+}
+
+interface D1BoundStatement {
+  all(): Promise<{ results: unknown[] }>;
+  run(): Promise<unknown>;
+}
+
+/** Cloudflare D1. Asynchronous; BLOBs come back as number[]; transactions are `batch()`. */
+export class D1SqlDriver implements SqlDriver {
+  readonly #db: D1Database;
+
+  constructor(db: D1Database) {
+    this.#db = db;
+  }
+
+  async all<T>(sql: string, ...params: unknown[]): Promise<T[]> {
+    const { results } = await this.#db
+      .prepare(sql)
+      .bind(...params)
+      .all();
+    return results as T[];
+  }
+
+  async run(sql: string, ...params: unknown[]): Promise<void> {
+    await this.#db
+      .prepare(sql)
+      .bind(...params)
+      .run();
+  }
+
+  async transaction(statements: SqlStatement[]): Promise<number[]> {
+    if (statements.length === 0) return [];
+    const results = await this.#db.batch(
+      statements.map(({ sql, params = [] }) => this.#db.prepare(sql).bind(...params)),
+    );
+    return results.map((result) => result.meta.changes);
+  }
+}

package/src/index.ts

@@ -0,0 +1,25 @@
+export { SqliteFilesApi, type SqliteFilesApiOptions } from "./blocks/sqlite-files-api.js";
+export {
+  type PakoStreamModule,
+  pakoDeflateCodec,
+  type StreamCodec,
+  webDeflateCodec,
+} from "./blocks/stream-codec.js";
+export {
+  type Codec,
+  defaultCodec,
+  type PakoModule,
+  pakoCodec,
+  rawCodec,
+  webCodec,
+} from "./codec.js";
+export {
+  type D1Database,
+  D1SqlDriver,
+  DoSqlDriver,
+  type DoStorage,
+  NodeSqlDriver,
+  type NodeSqliteDatabase,
+} from "./drivers.js";
+export type { SqlDriver, SqlStatement } from "./sql.types.js";
+export { SqlarFilesApi, type SqlarFilesApiOptions } from "./sqlar-files-api.js";

package/src/sql.types.ts

@@ -0,0 +1,25 @@
+/** One statement of a transaction. */
+export interface SqlStatement {
+  sql: string;
+  params?: unknown[];
+}
+
+/**
+ * The SQL port. Each method may be sync or async, so node:sqlite, a Durable
+ * Object's storage and D1 all satisfy it through a thin driver.
+ */
+export interface SqlDriver {
+  /** Rows as objects keyed by column name. */
+  all<T = Record<string, unknown>>(sql: string, ...params: unknown[]): T[] | Promise<T[]>;
+  /** Execute one statement for its effect. */
+  run(sql: string, ...params: unknown[]): void | Promise<void>;
+  /**
+   * Run the statements in order as one transaction: all apply or none do, and
+   * a failure rethrows after rolling back. Returns, per statement, a count that
+   * is zero exactly when that statement changed nothing.
+   *
+   * No statement may depend on reading another's result — D1's `batch()` cannot
+   * pass results along — so conditions belong in SQL (`WHERE EXISTS …`).
+   */
+  transaction(statements: SqlStatement[]): number[] | Promise<number[]>;
+}