cursedbelt-server 1.0.0

package/src/server/sqlite/backup.spec.ts

@@ -0,0 +1,114 @@
+import { Database } from 'bun:sqlite';
+import { afterEach, beforeEach, describe, expect, test } from 'bun:test';
+import { existsSync, mkdtempSync, readdirSync, rmSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { createSqliteBackupService, snapshotSqlite } from './backup';
+
+describe('createSqliteBackupService', () => {
+  let db: Database;
+  let dir: string;
+
+  beforeEach(() => {
+    dir = mkdtempSync(join(tmpdir(), 'cb-backup-'));
+    db = new Database(':memory:');
+    db.run('CREATE TABLE t (id INTEGER PRIMARY KEY, v TEXT)');
+    for (let i = 0; i < 50; i++) db.run('INSERT INTO t (v) VALUES (?)', [`row-${i}`]);
+  });
+
+  afterEach(() => {
+    db.close();
+    rmSync(dir, { recursive: true, force: true });
+  });
+
+  test('runNow writes a real, queryable backup and reports its size', async () => {
+    const svc = createSqliteBackupService(db, { backupDir: dir });
+    const { path, sizeMb } = await svc.runNow();
+
+    expect(existsSync(path)).toBe(true);
+    expect(sizeMb).toBeGreaterThan(0);
+
+    // The backup is a valid SQLite DB with the source data.
+    const copy = new Database(path, { readonly: true });
+    expect((copy.query('SELECT COUNT(*) AS n FROM t').get() as { n: number }).n).toBe(50);
+    copy.close();
+  });
+
+  test('listBackups reports newest first', async () => {
+    const svc = createSqliteBackupService(db, { backupDir: dir });
+    await svc.runNow();
+    await svc.runNow();
+    const backups = svc.listBackups();
+    expect(backups).toHaveLength(2);
+    expect(backups[0].createdAt.getTime()).toBeGreaterThanOrEqual(backups[1].createdAt.getTime());
+    for (const b of backups) expect(b.sizeMb).toBeGreaterThan(0);
+  });
+
+  test('prunes to maxBackups after each run', async () => {
+    const svc = createSqliteBackupService(db, { backupDir: dir, maxBackups: 2 });
+    await svc.runNow();
+    await svc.runNow();
+    await svc.runNow();
+
+    expect(svc.listBackups()).toHaveLength(2);
+    // Only the backup files remain on disk.
+    expect(readdirSync(dir).filter((f) => f.endsWith('.sqlite'))).toHaveLength(2);
+  });
+
+  test('creates the backup directory if it does not exist', async () => {
+    const nested = join(dir, 'a', 'b', 'backups');
+    const svc = createSqliteBackupService(db, { backupDir: nested });
+    const { path } = await svc.runNow();
+    expect(existsSync(path)).toBe(true);
+  });
+
+  test('listBackups returns [] for a missing directory', () => {
+    const svc = createSqliteBackupService(db, { backupDir: join(dir, 'nope') });
+    expect(svc.listBackups()).toEqual([]);
+  });
+
+  test('pruneOld is a no-op below the threshold', async () => {
+    const svc = createSqliteBackupService(db, { backupDir: dir, maxBackups: 5 });
+    await svc.runNow();
+    svc.pruneOld();
+    expect(svc.listBackups()).toHaveLength(1);
+  });
+});
+
+describe('snapshotSqlite', () => {
+  let dir: string;
+  let src: string;
+
+  beforeEach(() => {
+    dir = mkdtempSync(join(tmpdir(), 'cb-snapshot-'));
+    src = join(dir, 'source.sqlite');
+    const db = new Database(src);
+    db.run('CREATE TABLE t (id INTEGER PRIMARY KEY, v TEXT)');
+    for (let i = 0; i < 10; i++) db.run('INSERT INTO t (v) VALUES (?)', [`row-${i}`]);
+    db.close();
+  });
+
+  afterEach(() => rmSync(dir, { recursive: true, force: true }));
+
+  test('writes a consistent, queryable copy and returns its byte size', () => {
+    const dest = join(dir, 'snap.sqlite');
+    const bytes = snapshotSqlite(src, dest);
+    expect(bytes).toBeGreaterThan(0);
+
+    const copy = new Database(dest, { readonly: true });
+    expect((copy.query('SELECT COUNT(*) AS n FROM t').get() as { n: number }).n).toBe(10);
+    copy.close();
+  });
+
+  test('refuses an existing destination rather than half-overwriting it', () => {
+    const dest = join(dir, 'snap.sqlite');
+    snapshotSqlite(src, dest);
+    expect(() => snapshotSqlite(src, dest)).toThrow();
+  });
+
+  test('escapes single quotes in the destination path', () => {
+    const dest = join(dir, "o'brien.sqlite");
+    expect(snapshotSqlite(src, dest)).toBeGreaterThan(0);
+    expect(existsSync(dest)).toBe(true);
+  });
+});

package/src/server/sqlite/backup.ts

@@ -0,0 +1,154 @@
+import { Database } from 'bun:sqlite';
+import { randomBytes } from 'node:crypto';
+import { existsSync, mkdirSync, readdirSync, rmSync, statSync } from 'node:fs';
+import { join } from 'node:path';
+
+/**
+ * SQLite backup service for `bun:sqlite` — the from-scratch replacement for the apps'
+ * `backupService.ts`. It uses `VACUUM INTO`, the correct primitive for `bun:sqlite`:
+ * it writes a compact, defragmented copy with all WAL data merged in (the Node
+ * `better-sqlite3` online `.backup()` API does not exist here). Reads continue under
+ * WAL during the copy; only writes serialize behind it — negligible for normal DB
+ * sizes, so the `daily-backup` cron job (task 29) runs it off-peak on the main thread.
+ *
+ * Optional S3 mirroring uses Bun's native S3 client (credentials from the environment,
+ * never stored here). The job *definition* lives in the app; this is the reusable
+ * mechanism.
+ */
+
+const BACKUP_PREFIX = 'backup-';
+const BACKUP_EXT = '.sqlite';
+const BYTES_PER_MB = 1_048_576;
+
+export interface SqliteBackupS3Opts {
+  bucket: string;
+  /** Key prefix within the bucket, e.g. `'ca/backups'`. */
+  prefix: string;
+  /** AWS region; falls back to Bun's env resolution when omitted. */
+  region?: string;
+  /** Custom endpoint (e.g. an S3-compatible store like R2/MinIO). */
+  endpoint?: string;
+}
+
+export interface SqliteBackupOpts {
+  /** Directory backups are written to (created if missing). */
+  backupDir: string;
+  /** How many local backups to retain after each run. Default: 7. */
+  maxBackups?: number;
+  /** Mirror each backup to S3 after writing it locally. */
+  s3?: SqliteBackupS3Opts;
+}
+
+export interface BackupInfo {
+  path: string;
+  createdAt: Date;
+  sizeMb: number;
+}
+
+export interface SqliteBackupService {
+  /** Write a fresh backup now, mirror to S3 (if configured), prune, and describe it. */
+  runNow(): Promise<{ path: string; sizeMb: number }>;
+  /** Local backups, newest first. */
+  listBackups(): BackupInfo[];
+  /** Delete the oldest backups beyond `maxBackups`. */
+  pruneOld(): void;
+}
+
+/**
+ * `VACUUM INTO` a source db path to a fresh destination file — the path-level
+ * snapshot primitive behind `createSqliteBackupService` and the satellites' own
+ * backup scripts. It is transactionally consistent even while the db is being
+ * written under WAL (unlike `cp`), and it REFUSES an existing target, so a
+ * same-instant re-run errors rather than half-overwriting a previous snapshot.
+ * Opens the source read-only, closes it, and returns the snapshot's byte size.
+ *
+ * Use this when the caller chooses the exact destination path (a dated dir, a
+ * `latest.sqlite`, a remote-pull staging file); use {@link createSqliteBackupService}
+ * when you want a managed backup DIRECTORY with timestamped names + retention + S3.
+ */
+export function snapshotSqlite(sourcePath: string, destPath: string): number {
+  const source = new Database(sourcePath, { readonly: true });
+  try {
+    source.run(`VACUUM INTO '${destPath.replaceAll("'", "''")}'`);
+  } finally {
+    source.close();
+  }
+  return statSync(destPath).size;
+}
+
+const hasBunS3 = (): boolean => typeof (Bun as { S3Client?: unknown }).S3Client === 'function';
+
+async function uploadToS3(localPath: string, s3: SqliteBackupS3Opts): Promise<void> {
+  if (!hasBunS3()) {
+    console.warn(
+      '[sqlite-backup] S3 is configured but Bun.S3Client is unavailable; skipping upload. ' +
+        'Upgrade Bun (≥1.1) or remove the s3 option.',
+    );
+    return;
+  }
+  const client = new Bun.S3Client({
+    bucket: s3.bucket,
+    region: s3.region,
+    endpoint: s3.endpoint,
+  });
+  const key = `${s3.prefix.replace(/\/+$/, '')}/${localPath.split('/').pop()}`;
+  await client.file(key).write(Bun.file(localPath));
+}
+
+export function createSqliteBackupService(
+  db: Database,
+  opts: SqliteBackupOpts,
+): SqliteBackupService {
+  const { backupDir, maxBackups = 7, s3 } = opts;
+
+  if (s3 && !hasBunS3()) {
+    console.warn(
+      '[sqlite-backup] S3 mirroring requested but Bun.S3Client is unavailable in this runtime; ' +
+        'backups will be written locally only.',
+    );
+  }
+
+  const service: SqliteBackupService = {
+    async runNow() {
+      mkdirSync(backupDir, { recursive: true });
+      // ISO timestamp (filesystem-safe) + a short random suffix so rapid successive
+      // runs in the same millisecond never collide onto one path.
+      const stamp = new Date().toISOString().replace(/[:.]/g, '-');
+      const outPath = join(
+        backupDir,
+        `${BACKUP_PREFIX}${stamp}-${randomBytes(3).toString('hex')}${BACKUP_EXT}`,
+      );
+
+      // VACUUM INTO checkpoints the WAL and writes a clean, compact copy.
+      db.run('VACUUM INTO ?', [outPath]);
+      const sizeMb = statSync(outPath).size / BYTES_PER_MB;
+
+      if (s3) await uploadToS3(outPath, s3);
+
+      service.pruneOld();
+      return { path: outPath, sizeMb };
+    },
+
+    listBackups() {
+      if (!existsSync(backupDir)) return [];
+      return readdirSync(backupDir)
+        .filter((f) => f.startsWith(BACKUP_PREFIX) && f.endsWith(BACKUP_EXT))
+        .map((f) => {
+          const path = join(backupDir, f);
+          const stat = statSync(path);
+          return { path, createdAt: stat.mtime, sizeMb: stat.size / BYTES_PER_MB };
+        })
+        .sort(
+          (a, b) => b.createdAt.getTime() - a.createdAt.getTime() || b.path.localeCompare(a.path),
+        );
+    },
+
+    pruneOld() {
+      for (const old of service.listBackups().slice(maxBackups)) {
+        rmSync(old.path, { force: true });
+      }
+    },
+  };
+
+  return service;
+}

package/src/server/sqlite/browse.spec.ts

@@ -0,0 +1,70 @@
+import { Database } from 'bun:sqlite';
+import { describe, expect, test } from 'bun:test';
+import {
+  isKnownTable,
+  listSqliteTables,
+  NotReadOnlyError,
+  quoteIdent,
+  readSqliteTable,
+  runReadonlySqlite,
+  UnknownTableError,
+} from './browse';
+
+function seed(): Database {
+  const db = new Database(':memory:');
+  db.run('CREATE TABLE notes (id INTEGER PRIMARY KEY, body TEXT)');
+  db.run('CREATE TABLE contacts (id INTEGER PRIMARY KEY, name TEXT)');
+  for (let i = 1; i <= 5; i++)
+    db.run('INSERT INTO notes (id, body) VALUES (?, ?)', [i, `note ${i}`]);
+  return db;
+}
+
+describe('listSqliteTables', () => {
+  test('returns user tables with row counts, excluding sqlite_*', () => {
+    const db = seed();
+    db.run('CREATE TABLE x (id INTEGER PRIMARY KEY AUTOINCREMENT)'); // creates sqlite_sequence
+    const items = listSqliteTables(db);
+    expect(items.map((t) => t.name).sort()).toEqual(['contacts', 'notes', 'x']);
+    expect(items.find((t) => t.name === 'notes')?.count).toBe(5);
+    expect(items.some((t) => t.name.startsWith('sqlite_'))).toBe(false);
+  });
+});
+
+describe('readSqliteTable', () => {
+  test('clamps and paginates', () => {
+    const page = readSqliteTable(seed(), 'notes', { limit: 2, offset: 1 });
+    expect(page.items).toHaveLength(2);
+    expect(page).toMatchObject({ total: 5, limit: 2, offset: 1 });
+  });
+
+  test('clamps an over-large limit to 1000 and negatives to defaults', () => {
+    expect(readSqliteTable(seed(), 'notes', { limit: 99999 }).limit).toBe(1000);
+    expect(readSqliteTable(seed(), 'notes', { offset: -4 }).offset).toBe(0);
+  });
+
+  test('throws UnknownTableError for a non-table (injection-safe)', () => {
+    const db = seed();
+    expect(() => readSqliteTable(db, 'notes"; DROP TABLE notes;--')).toThrow(UnknownTableError);
+    expect(isKnownTable(db, 'notes')).toBe(true);
+  });
+});
+
+describe('runReadonlySqlite', () => {
+  test('runs a bound SELECT', () => {
+    expect(runReadonlySqlite(seed(), 'SELECT body FROM notes WHERE id = ?', [3])).toEqual([
+      { body: 'note 3' },
+    ]);
+  });
+
+  test('rejects non-SELECT with NotReadOnlyError (nothing runs)', () => {
+    const db = seed();
+    expect(() => runReadonlySqlite(db, 'DELETE FROM notes')).toThrow(NotReadOnlyError);
+    expect(db.query('SELECT COUNT(*) AS n FROM notes').get()).toEqual({ n: 5 });
+  });
+});
+
+describe('quoteIdent', () => {
+  test('double-quotes and escapes embedded quotes', () => {
+    expect(quoteIdent('a"b')).toBe('"a""b"');
+  });
+});

package/src/server/sqlite/browse.ts

@@ -0,0 +1,120 @@
+import type { Database } from 'bun:sqlite';
+import { clampPageLimit } from 'cwip/math';
+import { isReadOnlySql } from './readonlySql';
+
+/**
+ * Hono-free read-only SQLite browse primitives — the shared core beneath
+ * `createSqliteViewerMiddleware` (viewer.ts) AND any consumer that can't mount a
+ * Hono sub-app (e.g. a cursedshell plugin's `defineService`, whose routes are pure
+ * data, or an app with a bespoke transport). One primitive per concern: the
+ * middleware is now a thin HTTP wrapper over these functions.
+ *
+ * Every path is read-only and injection-safe: table names are validated against
+ * `sqlite_master` before interpolation, all values are bound, and ad-hoc SQL is
+ * gated by {@link isReadOnlySql} (SELECT/WITH only). Authn/authz is the caller's job.
+ */
+
+/** A sane admin page-size window. */
+const MAX_PAGE = 1000;
+
+/** Clamp a requested page size into `[1, MAX_PAGE]`, falling back when unparseable. */
+export function clampLimit(raw: number | string | undefined, fallback: number): number {
+  return clampPageLimit(raw, { max: MAX_PAGE, fallback });
+}
+
+/** Clamp an offset to a non-negative integer (0 when unparseable). */
+export function clampOffset(raw: number | string | undefined): number {
+  return clampPageLimit(raw, { min: 0, max: Number.POSITIVE_INFINITY, fallback: 0 });
+}
+
+/** Is `name` an actual table in this DB? (The injection guard for `SELECT * FROM`.) */
+export function isKnownTable(db: Database, name: string): boolean {
+  const row = db
+    .query<{ n: number }, [string]>(
+      "SELECT COUNT(*) AS n FROM sqlite_master WHERE type='table' AND name = ?",
+    )
+    .get(name);
+  return (row?.n ?? 0) > 0;
+}
+
+/** Double-quote a name already proven to exist in `sqlite_master`. */
+export function quoteIdent(name: string): string {
+  return `"${name.replace(/"/g, '""')}"`;
+}
+
+export interface SqliteTableInfo {
+  name: string;
+  count: number;
+}
+
+/** User tables (FTS shadow + `sqlite_*` internals excluded) with row counts. */
+export function listSqliteTables(db: Database): SqliteTableInfo[] {
+  const names = db
+    .query<{ name: string }, []>(
+      "SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%' ORDER BY name",
+    )
+    .all()
+    .map((r) => r.name);
+  return names.map((name) => ({
+    name,
+    count:
+      db.query<{ n: number }, []>(`SELECT COUNT(*) AS n FROM ${quoteIdent(name)}`).get()?.n ?? 0,
+  }));
+}
+
+export interface SqliteTablePage {
+  items: Record<string, unknown>[];
+  total: number;
+  limit: number;
+  offset: number;
+}
+
+export class UnknownTableError extends Error {
+  constructor(public readonly table: string) {
+    super(`Unknown table: ${table}`);
+    this.name = 'UnknownTableError';
+  }
+}
+
+/**
+ * A clamped, paginated window over one table. Throws {@link UnknownTableError} for a
+ * table that isn't in `sqlite_master` (callers map that to a 404).
+ */
+export function readSqliteTable(
+  db: Database,
+  table: string,
+  opts: { limit?: number | string; offset?: number | string } = {},
+): SqliteTablePage {
+  if (!isKnownTable(db, table)) throw new UnknownTableError(table);
+  const limit = clampLimit(opts.limit, 50);
+  const offset = clampOffset(opts.offset);
+  const items = db
+    .query<Record<string, unknown>, [number, number]>(
+      `SELECT * FROM ${quoteIdent(table)} LIMIT ? OFFSET ?`,
+    )
+    .all(limit, offset);
+  const total =
+    db.query<{ n: number }, []>(`SELECT COUNT(*) AS n FROM ${quoteIdent(table)}`).get()?.n ?? 0;
+  return { items, total, limit, offset };
+}
+
+export class NotReadOnlyError extends Error {
+  constructor() {
+    super('Only read-only SELECT/WITH queries are allowed');
+    this.name = 'NotReadOnlyError';
+  }
+}
+
+/**
+ * Run an ad-hoc read-only query with bound params. Throws {@link NotReadOnlyError}
+ * for anything the {@link isReadOnlySql} guard rejects (non-SELECT, stacked
+ * statements, PRAGMA, …); SQLite errors (bad column/table) propagate as `Error`.
+ */
+export function runReadonlySqlite(
+  db: Database,
+  sql: string,
+  params: unknown[] = [],
+): Record<string, unknown>[] {
+  if (typeof sql !== 'string' || !isReadOnlySql(sql)) throw new NotReadOnlyError();
+  return db.query(sql).all(...(params as never[])) as Record<string, unknown>[];
+}

package/src/server/sqlite/export.spec.ts

@@ -0,0 +1,43 @@
+import { Database } from 'bun:sqlite';
+import { describe, expect, test } from 'bun:test';
+import { UnknownTableError } from './browse';
+import { exportSqliteSnapshot, exportSqliteTable } from './export';
+
+function seed(): Database {
+  const db = new Database(':memory:');
+  db.run('CREATE TABLE notes (id INTEGER PRIMARY KEY, body TEXT)');
+  db.run("INSERT INTO notes (id, body) VALUES (1, 'a'), (2, 'b, with comma')");
+  return db;
+}
+
+describe('exportSqliteTable', () => {
+  test('csv includes a header and RFC-4180 quoting', () => {
+    const out = exportSqliteTable(seed(), 'notes', 'csv');
+    expect(out).toMatchObject({ filename: 'notes.csv', mime: 'text/csv' });
+    expect(out.text.startsWith('id,body')).toBe(true);
+    expect(out.text).toContain('"b, with comma"');
+  });
+
+  test('json round-trips the rows', () => {
+    const out = exportSqliteTable(seed(), 'notes', 'json');
+    expect(out).toMatchObject({ filename: 'notes.json', mime: 'application/json' });
+    expect(JSON.parse(out.text)).toEqual([
+      { id: 1, body: 'a' },
+      { id: 2, body: 'b, with comma' },
+    ]);
+  });
+
+  test('throws UnknownTableError for a bad table', () => {
+    expect(() => exportSqliteTable(seed(), 'nope', 'csv')).toThrow(UnknownTableError);
+  });
+});
+
+describe('exportSqliteSnapshot', () => {
+  test('produces re-openable .sqlite bytes carrying the data', () => {
+    const snap = exportSqliteSnapshot(seed());
+    expect(snap.filename.endsWith('.sqlite')).toBe(true);
+    expect(snap.bytes.byteLength).toBeGreaterThan(0);
+    // SQLite files start with the "SQLite format 3\0" magic.
+    expect(new TextDecoder().decode(snap.bytes.slice(0, 15))).toBe('SQLite format 3');
+  });
+});

package/src/server/sqlite/export.ts

@@ -0,0 +1,74 @@
+import type { Database } from 'bun:sqlite';
+import { readFileSync, rmSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { toCsv } from 'cwip/csv';
+import { isKnownTable, quoteIdent, UnknownTableError } from './browse';
+
+/**
+ * Export primitives for `bun:sqlite` — the missing "get the data out" half of the
+ * SQLite toolkit. Table export composes the ONE canonical serializers (`toCsv` from
+ * `cwip/csv`, `JSON.stringify` for JSON — no third CSV/JSON path); snapshot export
+ * reuses the `VACUUM INTO` primitive (see `backup.ts`) to produce a portable `.sqlite`.
+ *
+ * These return in-memory payloads (text / bytes) rather than streaming to a Response,
+ * so they're transport-agnostic: a cursedshell plugin returns them over its JSON RPC
+ * (base64 for bytes), a Hono app streams them as a `Content-Disposition: attachment`
+ * download. Admin-scale — a whole-DB snapshot is base64'd through JSON, fine for
+ * typical DB sizes; very large DBs should stream a file route instead.
+ */
+
+export type TableExportFormat = 'csv' | 'json';
+
+export interface TableExport {
+  filename: string;
+  mime: string;
+  text: string;
+}
+
+/** All rows of one table serialized to CSV or JSON. Throws {@link UnknownTableError}. */
+export function exportSqliteTable(
+  db: Database,
+  table: string,
+  format: TableExportFormat,
+): TableExport {
+  if (!isKnownTable(db, table)) throw new UnknownTableError(table);
+  const rows = db.query<Record<string, unknown>, []>(`SELECT * FROM ${quoteIdent(table)}`).all();
+  if (format === 'json') {
+    return {
+      filename: `${table}.json`,
+      mime: 'application/json',
+      text: JSON.stringify(rows, null, 2),
+    };
+  }
+  return {
+    filename: `${table}.csv`,
+    mime: 'text/csv',
+    // toCsv needs at least the columns; an empty table yields an empty string, which
+    // is a valid (header-less) CSV for a zero-row export.
+    text: toCsv(rows),
+  };
+}
+
+export interface SnapshotExport {
+  filename: string;
+  mime: string;
+  bytes: Uint8Array;
+}
+
+/**
+ * A compact, WAL-merged copy of the whole database as `.sqlite` bytes, via
+ * `VACUUM INTO` a throwaway temp file (the same primitive `createSqliteBackupService`
+ * uses). The temp file is always removed.
+ */
+export function exportSqliteSnapshot(db: Database): SnapshotExport {
+  const stamp = new Date().toISOString().replace(/[:.]/g, '-');
+  const tmp = join(tmpdir(), `cb-export-${stamp}-${Math.random().toString(36).slice(2)}.sqlite`);
+  try {
+    db.run('VACUUM INTO ?', [tmp]);
+    const bytes = new Uint8Array(readFileSync(tmp));
+    return { filename: `snapshot-${stamp}.sqlite`, mime: 'application/x-sqlite3', bytes };
+  } finally {
+    rmSync(tmp, { force: true });
+  }
+}