fieldlog 0.15.0

package/src/retain.ts

@@ -0,0 +1,267 @@
+// retain.ts — snapshot + truncate: bound the log without losing truth.
+// Only the acked prefix (the relay already holds it) is ever swept, and the
+// cutover is write-new-file + atomic rename, never in-place mutation.
+import {
+  closeSync,
+  existsSync,
+  fsyncSync,
+  openSync,
+  readFileSync,
+  renameSync,
+  unlinkSync,
+  writeSync,
+} from 'node:fs';
+import { dirname } from 'node:path';
+import { Database } from 'bun:sqlite';
+import { isMarker } from './log.js';
+import type { EventStore } from './store.js';
+
+export interface SnapshotResult {
+  snapshot: string;
+  sealedSeq: number; // acked prefix sealed into this snapshot
+  dbSeq: number; // max event seq held by the db at snapshot time
+}
+
+export interface TruncateResult {
+  removed: number;
+  kept: number;
+  sealedSeq: number;
+}
+
+export function snapshotPathFor(dbPath: string): string {
+  return dbPath.replace(/\.(db|sqlite|sqlite3)$/, '') + '.snapshot.db';
+}
+
+/** Live db paths with a snapshot currently in flight. takeSnapshot is
+ * synchronous, so a present key means re-entrant or overlapping use — fail
+ * loud instead of interleaving two VACUUM INTO + stamp sequences over the
+ * same live db (the second copy would stamp live meta out from under the
+ * first, or vice versa). */
+const snapshotsInFlight = new Set<string>();
+
+/** Cross-process single-writer guard for takeSnapshot. The in-process set
+ * above cannot see a second OS process, so a lock file next to the live db
+ * (created O_CREAT|O_EXCL) serializes writers across processes. A holder
+ * crash can leave a stale file behind; the next writer then fails loud with
+ * ERR_SNAPSHOT_IN_FLIGHT (delete the `<db>.snapshot.lock` file once no
+ * writer is running) instead of silently interleaving two VACUUM INTO +
+ * stamp sequences. Single writer only: concurrent snapshots are rejected,
+ * never queued. */
+export function snapshotLockPathFor(dbPath: string): string {
+  return `${dbPath}.snapshot.lock`;
+}
+
+function acquireSnapshotLock(dbPath: string): number | null {
+  const lockPath = snapshotLockPathFor(dbPath);
+  try {
+    return openSync(lockPath, 'wx', 0o644);
+  } catch {
+    return null;
+  }
+}
+
+function failSnapshotInFlight(dbPath: string): never {
+  throw new Error(
+    `ERR_SNAPSHOT_IN_FLIGHT: snapshot already in progress for '${dbPath}'; ` +
+      `finish it before starting another (overlapping copies would stamp live meta out of order)`,
+  );
+}
+
+ /** Online full copy (VACUUM INTO) + seal stamp in both snapshot and live meta. */
+ export function takeSnapshot(
+   store: EventStore,
+   dbPath: string,
+   sealedSeq: number,
+   dest?: string,
+ ): SnapshotResult {
+   if (snapshotsInFlight.has(dbPath)) {
+     throw new Error(
+       `ERR_SNAPSHOT_IN_FLIGHT: snapshot already in progress for '${dbPath}'; ` +
+         `finish it before starting another (overlapping copies would stamp live meta out of order)`,
+     );
+   }
+  const lockFd = acquireSnapshotLock(dbPath);
+  if (lockFd === null) failSnapshotInFlight(dbPath);
+   snapshotsInFlight.add(dbPath);
+   try {
+    const snapshot = dest ?? snapshotPathFor(dbPath);
+    try {
+      unlinkSync(snapshot); // VACUUM INTO refuses an existing target
+    } catch {
+      /* fresh path */
+    }
+    store.exec(`VACUUM INTO '${snapshot.replace(/'/g, "''")}'`);
+    // Checkpoint BEFORE stamping: capture the observed read-model tip first,
+    // so every stamp below describes the same state the copy was taken from.
+    // A write landing between the copy and this read can only push dbSeq
+    // above the copy — never below — so the seal stays conservative.
+    const rows = store.query<{ m: number | null }>(`SELECT MAX(seq) AS m FROM _events`);
+    const dbSeq: number = rows[0]?.m ?? 0;
+    const db = new Database(snapshot);
+    try {
+      db.exec('BEGIN IMMEDIATE');
+      try {
+        db.exec(
+          `INSERT INTO _meta(k,v) VALUES('snapshot.sealed_seq','${sealedSeq}') ` +
+            `ON CONFLICT(k) DO UPDATE SET v=excluded.v`,
+        );
+        db.exec(
+          `INSERT INTO _meta(k,v) VALUES('snapshot.at','${Date.now()}') ` +
+            `ON CONFLICT(k) DO UPDATE SET v=excluded.v`,
+        );
+        db.exec('COMMIT');
+      } catch (err) {
+        try {
+          db.exec('ROLLBACK');
+        } catch {
+          /* already torn down — report the original failure */
+        }
+        throw err;
+      }
+    } finally {
+      db.close();
+    }
+    // Both live stamps in one transaction: a crash must never leave
+    // snapshot.path pointing at a copy whose seal differs from
+    // snapshot.sealed_seq.
+    store.exec('BEGIN IMMEDIATE');
+    try {
+      store.setMeta('snapshot.path', snapshot);
+      store.setMeta('snapshot.sealed_seq', String(sealedSeq));
+      store.exec('COMMIT');
+    } catch (err) {
+      try {
+        store.exec('ROLLBACK');
+      } catch {
+        /* already torn down — report the original failure */
+      }
+      throw err;
+    }
+    return { snapshot, sealedSeq, dbSeq };
+  } finally {
+    snapshotsInFlight.delete(dbPath);
+    try { closeSync(lockFd); } catch { /* already closed */ }
+    try { unlinkSync(snapshotLockPathFor(dbPath)); } catch { /* released by other means */ }
+  }
+}
+
+/**
+ * Clamp a snapshot seal to what truncate may safely sweep: at most the ack
+ * cursor (never remove unacked data), and at most below the first swept seq
+ * missing from the read-model (never remove unapplied data — a stale or
+ * over-advanced ack must not turn into permanent loss). Returns 0 when
+ * nothing is safely sweepable; the caller must treat that as a no-op.
+ */
+export function clampSealToStored(
+  store: EventStore,
+  logSeqs: number[],
+  sealed: number,
+  ackSeq: number,
+): number {
+  // An empty read-model proves nothing applied: seal/ack cursors alone must
+  // never authorize a sweep, so report 0 (no-op) instead of min(sealed, ack).
+  // Same when the log holds nothing at/below the seal — there is no proven
+  // applied prefix to sweep.
+  if (logSeqs.length === 0) return 0;
+  let effective = Math.min(sealed, ackSeq);
+  if (!(effective > 0)) return 0;
+  const cands = logSeqs.filter((s) => s <= effective).sort((a, b) => a - b);
+  if (cands.length === 0) return 0;
+  const rows = store.query<{ seq: number }>(`SELECT seq FROM _events WHERE seq <= ?`, [effective]);
+  const have = new Set(rows.map((r) => r.seq));
+  for (const s of cands) {
+    if (!have.has(s)) {
+      effective = s - 1;
+      break;
+    }
+  }
+  return effective > 0 ? effective : 0;
+}
+
+/**
+ * Sweep log lines with seq <= sealedSeq. New file = marker + kept lines,
+ * fsynced, then atomically renamed over the original. The rename itself is
+ * made durable with a directory fsync (`syncDir`, injectable for tests).
+ * Only successfully parsed events at/below the seal are removed; markers
+ * supersede, and anything unparseable is preserved byte-for-byte.
+ */
+export function sweepLogFile(
+  logPath: string,
+  sealedSeq: number,
+  syncDir: (dir: string) => void = syncDirOf,
+): TruncateResult {
+  if (sealedSeq <= 0 || !existsSync(logPath)) return { removed: 0, kept: 0, sealedSeq: 0 };
+  const kept: string[] = [];
+  let removed = 0;
+  let tip: string | null = null;
+  let tipSeq = -1;
+  for (const line of readFileSync(logPath, 'utf8').split('\n')) {
+    const t = line.trim();
+    if (!t) continue;
+    let parsed: unknown;
+    try {
+      parsed = JSON.parse(t);
+    } catch {
+      kept.push(line); // corrupt bytes stay for quarantine on next open
+      continue;
+    }
+    if (isMarker(parsed)) continue; // superseded by the new marker below
+    if (
+      parsed !== null &&
+      typeof parsed === 'object' &&
+      'seq' in parsed &&
+      typeof parsed.seq === 'number' &&
+      'hash' in parsed &&
+      typeof parsed.hash === 'string' &&
+      parsed.seq <= sealedSeq
+    ) {
+      removed += 1;
+      if (parsed.seq > tipSeq) {
+        tipSeq = parsed.seq;
+        tip = parsed.hash;
+      }
+      continue;
+    }
+    kept.push(line);
+  }
+  if (removed === 0) return { removed: 0, kept: kept.length, sealedSeq };
+  const marker =
+    JSON.stringify({
+      v: 1,
+      marker: 'fieldlog-truncate',
+      truncated_before: sealedSeq + 1,
+      tip,
+      next_seq: sealedSeq + 1,
+    }) + '\n';
+  const tmp = logPath + '.tmp';
+  const fd = openSync(tmp, 'w');
+  try {
+    writeSync(fd, marker);
+    for (const l of kept) writeSync(fd, l + '\n');
+    fsyncSync(fd);
+  } finally {
+    closeSync(fd);
+  }
+  renameSync(tmp, logPath);
+  syncDir(dirname(logPath)); // make the rename itself durable before reporting success
+  return { removed, kept: kept.length, sealedSeq };
+}
+
+/**
+ * Best-effort directory fsync so a sweep rename survives a crash (the file
+ * fsync above only durables content, not the directory entry). Platforms
+ * without directory fsync fall through silently — content durability still
+ * holds via the file fsync.
+ */
+function syncDirOf(dir: string): void {
+  try {
+    const dfd = openSync(dir, 'r');
+    try {
+      fsyncSync(dfd);
+    } finally {
+      closeSync(dfd);
+    }
+  } catch {
+    /* no durable-rename primitive here */
+  }
+}

package/src/revokelog.ts

@@ -0,0 +1,291 @@
+// revokelog.ts — authenticated revoke event-log, convergent across relays.
+//
+// Closes two relay-revocation leaks: (1) revokes were unauthenticated local
+// state (relay.ts keeps a bare Set<string> plus a live 'revoked' hint any
+// relay can claim), (2) that set never converges across relays (per-relay
+// sidecar file, no merge). Here every revoke is an admin-signed event and
+// replicas sync by idempotent set-union merge, so merge is commutative and
+// snapshots converge.
+//
+// Event JSON: { v, tokenId, deviceId, epoch, admin, issuedAt, prev, hash, sig, id }
+//   tokenId  revoked capability/grant id; '*' = whole-device revoke.
+//   deviceId token owner whose capability dies.
+//   epoch    monotonic per tokenId; higher supersedes lower.
+//   admin    signer deviceId; must be in the trusted admin registry.
+//   issuedAt wall clock, display only — never authoritative.
+//   prev     hash-chain link: REVOKE_GENESIS or a known event hash (audit hint).
+//   hash     sha256 over the canonical core; id = hash (idempotency key).
+//   sig      admin ed25519 signature over hash (auth.ts signBytes).
+import { createHash } from 'node:crypto';
+import { signBytes, verifyBytes } from './auth.js';
+
+export const REVOKE_GENESIS = 'REVOKE-GENESIS';
+export const REVOKE_V = 1;
+
+export interface RevokeEvent {
+  v: 1;
+  tokenId: string;
+  deviceId: string;
+  epoch: number;
+  admin: string;
+  issuedAt: number;
+  prev: string;
+  hash: string;
+  sig: string;
+  id: string;
+}
+
+export interface RevokeInput {
+  tokenId: string;
+  deviceId: string;
+  epoch: number;
+  issuedAt?: number;
+}
+
+export type AdminRegistry = Map<string, string> | Record<string, string>;
+
+export function normalizeRegistry(reg?: AdminRegistry): Map<string, string> {
+  if (!reg) return new Map();
+  return reg instanceof Map ? new Map(reg) : new Map(Object.entries(reg));
+}
+
+type RevokeCore = Omit<RevokeEvent, 'hash' | 'sig' | 'id'>;
+
+/** Canonical bytes covered by the chain hash (hash/sig/id excluded). */
+export function canonicalRevoke(e: RevokeCore): string {
+  return JSON.stringify({
+    v: e.v,
+    tokenId: e.tokenId,
+    deviceId: e.deviceId,
+    epoch: e.epoch,
+    admin: e.admin,
+    issuedAt: e.issuedAt,
+    prev: e.prev,
+  });
+}
+
+export function hashRevoke(e: RevokeCore): string {
+  return createHash('sha256').update(canonicalRevoke(e), 'utf8').digest('hex');
+}
+
+function checkFields(e: RevokeCore): string | null {
+  if (e.v !== REVOKE_V) return `unsupported v ${e.v as number}`;
+  if (typeof e.tokenId !== 'string' || e.tokenId === '') return 'empty tokenId';
+  if (typeof e.deviceId !== 'string' || e.deviceId === '') return 'empty deviceId';
+  if (typeof e.admin !== 'string' || e.admin === '') return 'empty admin';
+  if (!Number.isInteger(e.epoch) || e.epoch < 0) return `bad epoch ${e.epoch as number}`;
+  if (typeof e.issuedAt !== 'number' || !Number.isFinite(e.issuedAt)) return 'bad issuedAt';
+  if (typeof e.prev !== 'string' || e.prev === '') return 'empty prev';
+  return null;
+}
+
+/** Build + admin-sign one revoke; prev threads the author's tip (audit hint). */
+export function createRevokeEvent(
+  adminPrivatePem: string,
+  admin: string,
+  input: RevokeInput,
+  prev: string = REVOKE_GENESIS,
+  now: number = Date.now(),
+): RevokeEvent {
+  const core: RevokeCore = {
+    v: REVOKE_V,
+    tokenId: input.tokenId,
+    deviceId: input.deviceId,
+    epoch: input.epoch,
+    admin,
+    issuedAt: input.issuedAt ?? now,
+    prev,
+  };
+  const bad = checkFields(core);
+  if (bad) throw new Error(`revoke rejected: ${bad}`);
+  const hash = hashRevoke(core);
+  return { ...core, hash, sig: signBytes(adminPrivatePem, hash), id: hash };
+}
+
+function verifyOne(admins: Map<string, string>, e: RevokeEvent): string | null {
+  const bad = checkFields(e);
+  if (bad) return bad;
+  if (e.id !== e.hash) return 'id/hash split';
+  const { hash, sig, id, ...core } = e;
+  void id;
+  if (hashRevoke(core) !== hash) return 'hash mismatch (tampered payload?)';
+  const pub = admins.get(e.admin);
+  if (!pub) return `unknown admin ${e.admin}`;
+  if (!verifyBytes(pub, hash, sig)) return 'bad admin signature';
+  return null;
+}
+
+/** Single-event auth check: content hash + known admin + signature. */
+export function verifyRevokeEvent(admins: AdminRegistry, e: RevokeEvent): boolean {
+  return verifyOne(normalizeRegistry(admins), e) === null;
+}
+
+export interface RevokeVerify {
+  ok: boolean;
+  at?: string;
+  reason?: string;
+}
+
+/**
+ * DAG replay over a batch: every event authentic and every non-genesis prev
+ * pointing at a known hash. Order-free on purpose — concurrent writers fork
+ * the prev hint, so this checks linkage, not a single linear order.
+ */
+export function verifyRevokeChain(admins: AdminRegistry, events: Iterable<RevokeEvent>): RevokeVerify {
+  const reg = normalizeRegistry(admins);
+  const list = [...events];
+  for (const e of list) {
+    const bad = verifyOne(reg, e);
+    if (bad) return { ok: false, at: e.id, reason: bad };
+  }
+  const known = new Set(list.map((e) => e.hash));
+  for (const e of list) {
+    if (e.prev !== REVOKE_GENESIS && !known.has(e.prev)) {
+      return { ok: false, at: e.id, reason: `dangling prev ${e.prev.slice(0, 12)}` };
+    }
+  }
+  return { ok: true };
+}
+
+export interface MergeResult {
+  added: number;
+  skipped: number;
+  rejected: number;
+}
+
+export type AppendOutcome = 'added' | 'duplicate';
+
+export class RevokeLog {
+  private admins: Map<string, string>;
+  private order: RevokeEvent[] = []; // first-seen order; diffSince slices this
+  private byHash = new Map<string, RevokeEvent>();
+
+  constructor(trustedAdmins?: AdminRegistry) {
+    this.admins = normalizeRegistry(trustedAdmins);
+  }
+
+  get size(): number {
+    return this.order.length;
+  }
+
+  /** Live tip hash (REVOKE_GENESIS when empty); doubles as the next prev. */
+  get tip(): string {
+    return this.order.length > 0 ? this.order[this.order.length - 1].hash : REVOKE_GENESIS;
+  }
+
+  addAdmin(deviceId: string, publicKeyPem: string): void {
+    this.admins.set(deviceId, publicKeyPem);
+  }
+
+  /** Sign + append in one step; prev threads the local tip. */
+  create(adminPrivatePem: string, admin: string, input: RevokeInput, now: number = Date.now()): RevokeEvent {
+    const e = createRevokeEvent(adminPrivatePem, admin, input, this.tip, now);
+    this.append(e);
+    return { ...e };
+  }
+
+  /** Verified single append; throws on forgery. Replay of a known hash is a no-op. */
+  append(e: RevokeEvent): AppendOutcome {
+    if (this.byHash.has(e.hash)) return 'duplicate';
+    const bad = verifyOne(this.admins, e);
+    if (bad) throw new Error(`revoke rejected: ${bad}`);
+    if (e.prev !== REVOKE_GENESIS && !this.byHash.has(e.prev)) {
+      throw new Error(`revoke rejected: dangling prev ${e.prev.slice(0, 12)}`);
+    }
+    const frozen = { ...e };
+    this.order.push(frozen);
+    this.byHash.set(frozen.hash, frozen);
+    return 'added';
+  }
+
+  /**
+   * Canonical convergent view: sorted by (epoch, hash). Byte-equal across
+   * replicas after a full bidirectional merge, regardless of arrival order.
+   */
+  snapshot(): RevokeEvent[] {
+    return [...this.order]
+      .sort((a, b) => a.epoch - b.epoch || (a.hash < b.hash ? -1 : a.hash > b.hash ? 1 : 0))
+      .map((e) => ({ ...e }));
+  }
+
+  /**
+   * First-seen suffix after cursor; cursor = order.length. Only diff your own
+   * log (it is append-only, so the cursor is stable); cross-replica sync is
+   * merge(snapshot) — set union needs no cursor.
+   */
+  diffSince(cursor: number): { events: RevokeEvent[]; cursor: number } {
+    if (!Number.isInteger(cursor) || cursor < 0 || cursor > this.order.length) {
+      throw new Error(`revoke rejected: bad cursor ${cursor as number}`);
+    }
+    return { events: this.order.slice(cursor).map((e) => ({ ...e })), cursor: this.order.length };
+  }
+
+  /**
+   * Idempotent set-union merge: commutative, so A.merge(B) then B.merge(A)
+   * converges. Unordered batches resolve to a fixpoint (prev may arrive
+   * later in the same batch); forged or dangling events count as rejected
+   * and are never stored.
+   */
+  merge(remote: Iterable<RevokeEvent>): MergeResult {
+    const res: MergeResult = { added: 0, skipped: 0, rejected: 0 };
+    const pending: RevokeEvent[] = [];
+    for (const e of remote) {
+      if (this.byHash.has(e.hash)) {
+        res.skipped += 1;
+        continue;
+      }
+      if (verifyOne(this.admins, e) !== null) {
+        res.rejected += 1;
+        continue;
+      }
+      pending.push(e);
+    }
+    let progress = true;
+    while (progress && pending.length > 0) {
+      progress = false;
+      for (let i = pending.length - 1; i >= 0; i--) {
+        const e = pending[i];
+        if (e.prev === REVOKE_GENESIS || this.byHash.has(e.prev)) {
+          pending.splice(i, 1);
+          const frozen = { ...e };
+          this.order.push(frozen);
+          this.byHash.set(frozen.hash, frozen);
+          res.added += 1;
+          progress = true;
+        }
+      }
+    }
+    res.rejected += pending.length;
+    return res;
+  }
+
+  verify(): RevokeVerify {
+    return verifyRevokeChain(this.admins, this.order);
+  }
+
+  /** Effective state: revoked iff some event for tokenId has epoch >= tokenEpoch. */
+  isRevoked(tokenId: string, tokenEpoch = 0): boolean {
+    for (const e of this.order) {
+      if (e.tokenId === tokenId && e.epoch >= tokenEpoch) return true;
+    }
+    return false;
+  }
+
+  /**
+   * One row per tokenId at its max epoch, sorted by tokenId. Epoch ties
+   * break by min event hash, mirroring snapshot()'s (epoch, hash) order,
+   * so arrival order never decides the winner across replicas.
+   */
+  revokedTokens(): Array<{ tokenId: string; deviceId: string; epoch: number }> {
+    const top = new Map<string, { tokenId: string; deviceId: string; epoch: number; hash: string }>();
+    for (const e of this.order) {
+      const cur = top.get(e.tokenId);
+      if (!cur || e.epoch > cur.epoch || (e.epoch === cur.epoch && e.hash < cur.hash)) {
+        top.set(e.tokenId, { tokenId: e.tokenId, deviceId: e.deviceId, epoch: e.epoch, hash: e.hash });
+      }
+    }
+    return [...top.values()]
+      .map(({ tokenId, deviceId, epoch }) => ({ tokenId, deviceId, epoch }))
+      .sort((a, b) => (a.tokenId < b.tokenId ? -1 : 1));
+  }
+}