@ixo/oracle-runtime-workers 0.2.0

package/src/shell/auth.ts

@@ -0,0 +1,290 @@
+/**
+ * UCAN authentication for the Workers shell — a port of the Node runtime's
+ * `AuthHeaderMiddleware` (`modules/auth/*`), same headers, same semantics:
+ *
+ *  - Primary:  `Authorization: Bearer <invocation>` + `X-Auth-Type: ucan` — a
+ *    user-self-signed root invocation for `{ can: '*', with: 'ixo:oracle' }`
+ *    addressed to this oracle's DID. Proves WHO is calling; short TTL.
+ *  - Fallback: `x-ucan-delegation: <base64 CAR>` — accepted alone for legacy
+ *    clients and always used for downstream authorization when present. A
+ *    delegation is only trusted when its issuer equals the authenticated DID.
+ *
+ * The returned `userDid` is always the cryptographically recovered signer,
+ * never a client-claimed value. `did:ixo` keys resolve through Blocksync.
+ * Verdicts are cached per isolate keyed by a hash of the token, bounded by
+ * the token's own expiry.
+ */
+import {
+  createIxoDIDResolver,
+  createUCANValidator,
+  defineCapability,
+  InMemoryInvocationStore,
+} from '@ixo/ucan';
+
+/**
+ * One replay-protection store per isolate, WITHOUT the library's hourly
+ * `setInterval` sweep: a pending timer keeps a Durable Object resident, and
+ * this code also runs inside the user object (socket CONNECT auth), where a
+ * leaked interval per CONNECT blocked WebSocket hibernation entirely. Expired
+ * entries are dropped lazily on lookup and swept every N validations instead.
+ */
+const invocationStore = new InMemoryInvocationStore({
+  enableAutoCleanup: false,
+});
+const SWEEP_EVERY = 500;
+let validations = 0;
+
+function sweepInvocationStore(): void {
+  validations += 1;
+  if (validations % SWEEP_EVERY === 0) void invocationStore.cleanup();
+}
+
+export const DEFAULT_UCAN_AUTH_MAX_TTL_SECONDS = 900;
+const THREE_MINUTES_MS = 3 * 60 * 1000;
+const ORACLE_AUTH_RESOURCE = 'ixo:oracle';
+const OracleAuthCapability = defineCapability({ can: '*', protocol: 'ixo:' });
+
+export interface AuthConfig {
+  oracleDid: string;
+  blocksyncUri: string;
+  maxTtlSeconds?: number;
+}
+
+export interface AuthResult {
+  userDid: string;
+  /** Raw delegation CAR when the client supplied a valid one for this user. */
+  delegation?: string;
+  delegationExpiration?: number;
+  /** Which artifact authenticated the request. */
+  via: 'invocation' | 'delegation';
+}
+
+export type AuthOutcome =
+  | { ok: true; auth: AuthResult }
+  | { ok: false; status: number; error: string };
+
+interface CacheEntry<T> {
+  value: T;
+  expiresAt: number;
+}
+
+/** Tiny bounded TTL cache — per isolate, lost on eviction (that's fine). */
+class TtlCache<T> {
+  private readonly map = new Map<string, CacheEntry<T>>();
+  constructor(private readonly max = 5000) {}
+  get(key: string): T | undefined {
+    const hit = this.map.get(key);
+    if (!hit) return undefined;
+    if (hit.expiresAt <= Date.now()) {
+      this.map.delete(key);
+      return undefined;
+    }
+    return hit.value;
+  }
+  set(key: string, value: T, ttlMs: number): void {
+    if (this.map.size >= this.max) {
+      const oldest = this.map.keys().next().value;
+      if (oldest !== undefined) this.map.delete(oldest);
+    }
+    this.map.set(key, { value, expiresAt: Date.now() + ttlMs });
+  }
+}
+
+const invocationCache = new TtlCache<{ userDid: string; expiration: number }>();
+const delegationCache = new TtlCache<{
+  userDid: string;
+  expiration?: number;
+}>();
+
+async function sha256(text: string): Promise<string> {
+  const digest = await crypto.subtle.digest(
+    'SHA-256',
+    new TextEncoder().encode(text),
+  );
+  return Array.from(new Uint8Array(digest))
+    .map((b) => b.toString(16).padStart(2, '0'))
+    .join('');
+}
+
+async function validateInvocation(
+  invocation: string,
+  cfg: AuthConfig,
+): Promise<
+  | { ok: true; userDid: string; expiration: number }
+  | { ok: false; error: string }
+> {
+  const key = await sha256(invocation);
+  const cached = invocationCache.get(key);
+  if (cached) return { ok: true, ...cached };
+
+  sweepInvocationStore();
+  const validator = await createUCANValidator({
+    serverDid: cfg.oracleDid,
+    rootIssuers: ['*'],
+    didResolver: createIxoDIDResolver({ indexerUrl: cfg.blocksyncUri }),
+    invocationStore,
+  });
+  const result = await validator.validate(
+    invocation,
+    OracleAuthCapability,
+    ORACLE_AUTH_RESOURCE,
+  );
+  if (!result.ok)
+    return {
+      ok: false,
+      error: `[${result.error?.code}] ${result.error?.message}`,
+    };
+  if (!result.invoker)
+    return { ok: false, error: 'Invocation validated without an invoker DID' };
+  if (typeof result.expiration !== 'number' || !isFinite(result.expiration)) {
+    return { ok: false, error: 'Auth invocation must declare an expiration' };
+  }
+  const nowSeconds = Math.floor(Date.now() / 1000);
+  const maxTtl = cfg.maxTtlSeconds ?? DEFAULT_UCAN_AUTH_MAX_TTL_SECONDS;
+  if (result.expiration > nowSeconds + maxTtl) {
+    return {
+      ok: false,
+      error: `Auth invocation TTL exceeds maximum of ${maxTtl}s`,
+    };
+  }
+  const verdict = { userDid: result.invoker, expiration: result.expiration };
+  invocationCache.set(
+    key,
+    verdict,
+    Math.max(1000, (result.expiration - nowSeconds) * 1000),
+  );
+  return { ok: true, ...verdict };
+}
+
+async function validateDelegation(
+  header: string,
+  cfg: AuthConfig,
+): Promise<
+  | { ok: true; userDid: string; expiration?: number }
+  | { ok: false; error: string }
+> {
+  const key = await sha256(header);
+  const cached = delegationCache.get(key);
+  if (cached) return { ok: true, ...cached };
+
+  sweepInvocationStore();
+  const validator = await createUCANValidator({
+    serverDid: cfg.oracleDid,
+    rootIssuers: [],
+    didResolver: createIxoDIDResolver({ indexerUrl: cfg.blocksyncUri }),
+    invocationStore,
+  });
+  const result = await validator.validateDelegation(header);
+  if (!result.ok)
+    return {
+      ok: false,
+      error: `[${result.error?.code}] ${result.error?.message}`,
+    };
+  if (!result.invoker)
+    return { ok: false, error: 'Delegation validated without an invoker DID' };
+  const verdict = { userDid: result.invoker, expiration: result.expiration };
+  const ttlMs =
+    typeof result.expiration === 'number'
+      ? Math.max(1000, result.expiration * 1000 - Date.now())
+      : THREE_MINUTES_MS;
+  delegationCache.set(key, verdict, Math.min(ttlMs, THREE_MINUTES_MS));
+  return { ok: true, ...verdict };
+}
+
+function extractInvocation(headers: Headers): string | null {
+  const authType = headers.get('x-auth-type');
+  const authorization = headers.get('authorization');
+  if (!authorization || authType?.toLowerCase() !== 'ucan') return null;
+  const [scheme, token] = authorization.split(' ', 2);
+  if (scheme?.toLowerCase() !== 'bearer' || !token) return null;
+  return token.trim();
+}
+
+/** Authenticate one request. Never throws. */
+export async function authenticate(
+  headers: Headers,
+  cfg: AuthConfig,
+): Promise<AuthOutcome> {
+  const invocation = extractInvocation(headers);
+  const delegationHeader = headers.get('x-ucan-delegation')?.trim() || null;
+
+  if (!invocation && !delegationHeader) {
+    return {
+      ok: false,
+      status: 401,
+      error:
+        'Missing Authorization (UCAN invocation) or x-ucan-delegation header',
+    };
+  }
+
+  let userDid: string | null = null;
+  let via: AuthResult['via'] = 'delegation';
+
+  if (invocation) {
+    const inv = await validateInvocation(invocation, cfg);
+    if (!inv.ok)
+      return {
+        ok: false,
+        status: 401,
+        error: `Invalid UCAN invocation: ${inv.error}`,
+      };
+    userDid = inv.userDid;
+    via = 'invocation';
+  }
+
+  let delegation: string | undefined;
+  let delegationExpiration: number | undefined;
+  if (delegationHeader) {
+    const del = await validateDelegation(delegationHeader, cfg);
+    if (!del.ok) {
+      // With a valid invocation the delegation is only authorization material;
+      // a bad one is ignored so the caller can still chat. Alone, it's fatal.
+      if (!userDid)
+        return {
+          ok: false,
+          status: 401,
+          error: `Invalid UCAN delegation: ${del.error}`,
+        };
+    } else if (userDid && del.userDid !== userDid) {
+      // Never pair one user's invocation with another user's delegation.
+      if (!invocation) userDid = del.userDid;
+    } else {
+      userDid ??= del.userDid;
+      delegation = delegationHeader;
+      delegationExpiration = del.expiration;
+    }
+  }
+
+  if (!userDid)
+    return { ok: false, status: 401, error: 'Unable to authenticate request' };
+  return { ok: true, auth: { userDid, delegation, delegationExpiration, via } };
+}
+
+/** Normalise a route exclusion into a matcher over (method, path). */
+export interface RouteExclusion {
+  path: string;
+  method?: string;
+}
+
+export function isExcluded(
+  method: string,
+  path: string,
+  exclusions: readonly RouteExclusion[],
+): boolean {
+  const norm = (p: string) => '/' + p.replace(/^\/+/, '').replace(/\/+$/, '');
+  const target = norm(path);
+  return exclusions.some((ex) => {
+    const m = (ex.method ?? 'ALL').toUpperCase();
+    if (m !== 'ALL' && m !== method.toUpperCase()) return false;
+    const pattern = norm(ex.path);
+    if (pattern.endsWith('/*') || pattern.endsWith('/{*path}')) {
+      const base = pattern.replace(/\/(\*|\{\*path\})$/, '');
+      return target === base || target.startsWith(base + '/');
+    }
+    // `:param` segments match one path segment.
+    const pSeg = pattern.split('/');
+    const tSeg = target.split('/');
+    if (pSeg.length !== tSeg.length) return false;
+    return pSeg.every((s, i) => s.startsWith(':') || s === tSeg[i]);
+  });
+}

package/src/sqlite/blob-codec.ts

@@ -0,0 +1,100 @@
+/**
+ * Lossless per-blob gzip for the checkpointer's BLOB columns.
+ *
+ * The saver's schema already stores each message once (`messages` PK is the
+ * message id) and prunes checkpoints per thread, so the remaining size lever
+ * is the serialized JSON itself. Every blob written through this codec is
+ * gzip-compressed; reads detect the gzip magic bytes and decompress
+ * transparently, so legacy uncompressed rows (files written by the Node
+ * runtime, or by this runtime before the codec) keep loading unchanged and
+ * both kinds coexist in one file.
+ *
+ * NOT compressed: `checkpoints.metadata` — `list()` filters it SQL-side via
+ * `json(CAST(metadata AS TEXT))`, which must keep seeing plaintext JSON. It
+ * is bounded (source/step/parents) so the loss is negligible.
+ *
+ * A compressed blob is a plain gzip stream — `gunzip` on the exported file
+ * shows the original JSON. Nothing is dropped, reordered, or summarized:
+ * compress(decompress(x)) round-trips bit-exact at the value level.
+ */
+
+/** Blobs below this size gain nothing from gzip (header ≈ 20 bytes). */
+export const MIN_COMPRESS_BYTES = 256;
+
+const textEncoder = new TextEncoder();
+const textDecoder = new TextDecoder();
+
+export function isGzip(bytes: Uint8Array): boolean {
+  return bytes.length >= 2 && bytes[0] === 0x1f && bytes[1] === 0x8b;
+}
+
+async function pipeThrough(
+  bytes: Uint8Array,
+  transform: TransformStream<Uint8Array, Uint8Array>,
+): Promise<Uint8Array> {
+  const source = new ReadableStream<Uint8Array>({
+    start(controller) {
+      controller.enqueue(bytes);
+      controller.close();
+    },
+  });
+  const stream = source.pipeThrough(transform);
+  const chunks: Uint8Array[] = [];
+  let total = 0;
+  const reader = stream.getReader();
+  for (;;) {
+    const { done, value } = await reader.read();
+    if (done) break;
+    chunks.push(value);
+    total += value.length;
+  }
+  const out = new Uint8Array(total);
+  let offset = 0;
+  for (const chunk of chunks) {
+    out.set(chunk, offset);
+    offset += chunk.length;
+  }
+  return out;
+}
+
+export function gzipBytes(bytes: Uint8Array): Promise<Uint8Array> {
+  return pipeThrough(bytes, new CompressionStream('gzip'));
+}
+
+export function gunzipBytes(bytes: Uint8Array): Promise<Uint8Array> {
+  return pipeThrough(bytes, new DecompressionStream('gzip'));
+}
+
+/**
+ * Encode one value for a BLOB column: small values pass through untouched;
+ * larger ones are gzipped. An incompressible payload (already-compressed
+ * media) that would GROW is stored uncompressed.
+ */
+export async function compressForStorage(
+  data: Uint8Array | string,
+): Promise<Uint8Array> {
+  const bytes = typeof data === 'string' ? textEncoder.encode(data) : data;
+  if (bytes.length < MIN_COMPRESS_BYTES) return bytes;
+  const compressed = await gzipBytes(bytes);
+  return compressed.length < bytes.length ? compressed : bytes;
+}
+
+/**
+ * Read one BLOB/TEXT column value back to its JSON text, decompressing when
+ * the gzip magic is present. Handles every historical shape: TEXT rows,
+ * legacy plain BLOBs, and codec-written gzip BLOBs.
+ */
+export async function readBlobText(raw: Uint8Array | string): Promise<string> {
+  if (typeof raw === 'string') return raw;
+  const bytes = isGzip(raw) ? await gunzipBytes(raw) : raw;
+  return textDecoder.decode(bytes);
+}
+
+/** Decode SQLite `hex(column)` output (upper-case hex) back to bytes. */
+export function hexToBytes(hex: string): Uint8Array {
+  const out = new Uint8Array(hex.length / 2);
+  for (let i = 0; i < out.length; i++) {
+    out[i] = Number.parseInt(hex.slice(i * 2, i * 2 + 2), 16);
+  }
+  return out;
+}

package/src/sqlite/blob-compactor.ts

@@ -0,0 +1,146 @@
+/**
+ * Background re-compression of legacy (uncompressed) checkpointer blobs.
+ *
+ * A file imported from the Node runtime arrives with every blob as plain
+ * JSON. New writes are gzipped by `blob-codec.ts`, but the imported bulk —
+ * which is what makes multi-GB users multi-GB — only shrinks if rewritten.
+ * `compactStep` rewrites a bounded batch per call so the user object's alarm
+ * can chip away at arbitrarily large files without ever hogging a tick;
+ * `finishCompaction` reclaims the freed pages once nothing is left.
+ *
+ * Lossless by construction: each row's blob is read, gzipped, and written
+ * back — the read path decompresses transparently, so the decoded value is
+ * bit-identical to what the legacy row held. No row is ever deleted.
+ *
+ * Selection is by gzip magic (`substr(col, 1, 2) != X'1F8B'`) behind a
+ * per-table rowid cursor that the CALLER persists between calls. The cursor
+ * is what lets incompressible rows (embedded media that gzip would grow —
+ * left unrewritten) be passed over exactly once instead of starving the rows
+ * behind them. Rows inserted after compaction started are codec-compressed
+ * already, so a monotonic cursor cannot miss anything.
+ */
+import { gzipBytes, isGzip, MIN_COMPRESS_BYTES } from './blob-codec';
+import type { DoSqliteDatabase } from './database';
+
+interface CompactTarget {
+  table: string;
+  column: string;
+}
+
+/** `checkpoints.metadata` is deliberately absent — see blob-codec.ts. */
+const TARGETS: readonly CompactTarget[] = [
+  { table: 'messages', column: 'message' },
+  { table: 'checkpoints', column: 'checkpoint' },
+  { table: 'writes', column: 'value' },
+];
+
+const textEncoder = new TextEncoder();
+
+/** Highest rowid examined per table; persist between calls, `{}` to start. */
+export type CompactCursors = Record<string, number>;
+
+export interface CompactStepResult {
+  /** Rows rewritten this step. */
+  rewritten: number;
+  /** Bytes saved this step (pre minus post). */
+  savedBytes: number;
+  /** Rows examined (incl. incompressible ones passed over). */
+  examined: number;
+  /** True once every table's scan is exhausted. */
+  done: boolean;
+  /** Updated cursors — persist and pass into the next call. */
+  cursors: CompactCursors;
+}
+
+/**
+ * Examine up to `batch` legacy rows and rewrite the compressible ones as
+ * gzipped blobs. Call repeatedly (persisting `cursors`) until `done`.
+ */
+export async function compactStep(
+  db: DoSqliteDatabase,
+  batch = 200,
+  cursors: CompactCursors = {},
+): Promise<CompactStepResult> {
+  // One transaction per step: the batch lands atomically and cannot
+  // interleave with a turn's own checkpoint transaction.
+  return db.transaction(() => compactStepInTx(db, batch, cursors));
+}
+
+async function compactStepInTx(
+  db: DoSqliteDatabase,
+  batch: number,
+  cursors: CompactCursors,
+): Promise<CompactStepResult> {
+  let budget = batch;
+  let rewritten = 0;
+  let savedBytes = 0;
+  let examined = 0;
+  let exhausted = true;
+  const next: CompactCursors = { ...cursors };
+
+  for (const { table, column } of TARGETS) {
+    let cursor = next[table] ?? 0;
+    while (budget > 0) {
+      const rows = await db.exec<{ rowid: number; data: Uint8Array | string }>(
+        `SELECT rowid, ${column} AS data FROM ${table}
+         WHERE rowid > ?
+           AND ${column} IS NOT NULL
+           AND length(${column}) >= ?
+           AND substr(${column}, 1, 2) != X'1F8B'
+         ORDER BY rowid
+         LIMIT ?`,
+        [cursor, MIN_COMPRESS_BYTES, budget],
+      );
+      if (rows.length === 0) break;
+      for (const { rowid, data } of rows) {
+        cursor = Math.max(cursor, rowid);
+        budget -= 1;
+        examined += 1;
+        const bytes =
+          typeof data === 'string' ? textEncoder.encode(data) : data;
+        if (isGzip(bytes)) continue;
+        const compressed = await gzipBytes(bytes);
+        // Incompressible (already-compressed media): leave as-is; the cursor
+        // has moved past it, so it is never re-attempted.
+        if (compressed.length >= bytes.length) continue;
+        await db.run(`UPDATE ${table} SET ${column} = ? WHERE rowid = ?`, [
+          compressed,
+          rowid,
+        ]);
+        rewritten += 1;
+        savedBytes += bytes.length - compressed.length;
+      }
+    }
+    next[table] = cursor;
+    if (budget <= 0) {
+      // Budget ran out mid-table — cannot know whether scans are exhausted.
+      exhausted = false;
+      break;
+    }
+  }
+
+  return { rewritten, savedBytes, examined, done: exhausted, cursors: next };
+}
+
+/**
+ * Reclaim freed pages after compaction. The rebuild (`DoSqliteDatabase.compact`:
+ * `VACUUM INTO` a spill file + swap, constant memory) transiently needs
+ * about twice the current size in DO storage — refused near the cap so
+ * compaction can never be the thing that overflows it. Falls back to
+ * `incremental_vacuum` (a no-op on files whose auto_vacuum is NONE, i.e.
+ * legacy imports that have never been fully vacuumed).
+ */
+export async function finishCompaction(
+  db: DoSqliteDatabase,
+  opts: { fileSize: number; storageCapBytes?: number },
+): Promise<{ vacuumed: 'full' | 'incremental' }> {
+  const cap = opts.storageCapBytes ?? 10 * 1024 * 1024 * 1024;
+  if (opts.fileSize * 2 < cap * 0.9) {
+    await db.compact();
+    return { vacuumed: 'full' };
+  }
+  // Refuses to run inside a transaction — hold the transaction mutex (turn
+  // checkpoints queue behind) without opening one.
+  await db.withoutTransactions(() => db.run('PRAGMA incremental_vacuum'));
+  return { vacuumed: 'incremental' };
+}

package/src/sqlite/cache-config.ts

@@ -0,0 +1,45 @@
+/**
+ * `CHUNK_CACHE_BYTES`: the per-object budget of clean 64 KiB chunks the DO
+ * VFS keeps in memory (see `DoVfs`). Every user object of a script shares
+ * one isolate (128 MB), so this is the knob that trades turn latency (more
+ * cache = fewer DO storage reads) against how many active users fit.
+ */
+import { DEFAULT_CACHE_PAGES, VFS_PAGE_SIZE } from './do-vfs';
+
+export const DEFAULT_CHUNK_CACHE_BYTES = DEFAULT_CACHE_PAGES * VFS_PAGE_SIZE;
+export const MIN_CHUNK_CACHE_BYTES = 1024 * 1024;
+export const MAX_CHUNK_CACHE_BYTES = 64 * 1024 * 1024;
+
+/**
+ * Parse the env value (plain bytes, or with a `k`/`m` suffix: `4m`,
+ * `4096k`). Unset → default; malformed or out of range → default plus a
+ * warning through `warn`.
+ */
+export function parseChunkCacheBytes(
+  raw: string | undefined,
+  warn: (message: string) => void = () => undefined,
+): number {
+  if (raw === undefined || raw.trim() === '') return DEFAULT_CHUNK_CACHE_BYTES;
+  const match = /^\s*(\d+)\s*([kKmM]?)\s*$/.exec(raw);
+  if (!match) {
+    warn(
+      `CHUNK_CACHE_BYTES=${JSON.stringify(raw)} is not a byte count — using the default ${DEFAULT_CHUNK_CACHE_BYTES}`,
+    );
+    return DEFAULT_CHUNK_CACHE_BYTES;
+  }
+  const unit = match[2]?.toLowerCase();
+  const multiplier = unit === 'k' ? 1024 : unit === 'm' ? 1024 * 1024 : 1;
+  const bytes = Number(match[1]) * multiplier;
+  if (bytes < MIN_CHUNK_CACHE_BYTES || bytes > MAX_CHUNK_CACHE_BYTES) {
+    warn(
+      `CHUNK_CACHE_BYTES=${raw} is outside ${MIN_CHUNK_CACHE_BYTES}..${MAX_CHUNK_CACHE_BYTES} — using the default ${DEFAULT_CHUNK_CACHE_BYTES}`,
+    );
+    return DEFAULT_CHUNK_CACHE_BYTES;
+  }
+  return bytes;
+}
+
+/** The `cachePages` open option for a byte budget. */
+export function cachePagesForBytes(bytes: number): number {
+  return Math.max(1, Math.round(bytes / VFS_PAGE_SIZE));
+}