@bluefields/cli 0.2.0

package/dist/store.js

@@ -0,0 +1,233 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+/**
+ * The local metadata store — an embedded Postgres (PGlite, WASM, in-process)
+ * that satisfies the `DbClient` the engine injects. NO hosted Postgres, no
+ * network, no KMS. The full Drizzle schema is applied to a file-backed PGlite
+ * data directory under `<store>/db`, so snapshot history survives across CLI
+ * invocations and `log`/`diff`/`watch`/`export` can query it with real SQL.
+ *
+ * This reuses the QA real-DB harness pattern (statement-by-statement, tolerant
+ * apply): a handful of migrations use Neon/pg_cron/pg_partman features PGlite
+ * lacks; those statements skip harmlessly while the engine tables are created
+ * from plain DDL. Two extra local adaptations then run:
+ *
+ *   1. DEFAULT partitions — `snapshots` / `cost_records` / `events` /
+ *      `audit_log` are declared `PARTITION BY RANGE`, and without pg_partman
+ *      they have no child partitions, so INSERTs would fail. We attach a
+ *      `<table>_default DEFAULT` partition to each so rows land locally.
+ *   2. A single seed `customers` row — `cost_records.customer_id` is a NOT-NULL
+ *      FK to `customers`, and the engine writes a cost row per fetch. One fixed
+ *      local customer keeps the engine's billing plumbing satisfied with zero
+ *      behavioural change.
+ */
+import { mkdirSync, readFileSync, readdirSync } from 'node:fs';
+import { dirname, resolve } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { customers } from '@bluefields/db';
+import { PGlite } from '@electric-sql/pglite';
+import { drizzle } from 'drizzle-orm/pglite';
+/**
+ * Fixed local identity for the single-user store. `cost_records` and
+ * `snapshots` are attributed to this synthetic customer; it never leaves the
+ * machine and carries no billing meaning.
+ */
+export const LOCAL_CUSTOMER_ID = '00000000-0000-4000-8000-000000000001';
+export const LOCAL_CUSTOMER_EMAIL = 'local@bluefield.invalid';
+export const LOCAL_SUBSCRIPTION_ID = null;
+/** Partitioned parents that need a DEFAULT partition to accept rows locally. */
+const PARTITIONED_TABLES = ['snapshots', 'cost_records', 'events', 'audit_log'];
+/** Resolve the engine's migrations directory (overridable for tests/packaging). */
+function migrationsDir() {
+    const override = process.env.BF_MIGRATIONS_DIR;
+    if (override?.trim())
+        return resolve(override.trim());
+    // Works from both `src/` (tsx) and `dist/` — both are direct children of
+    // `packages/cli`, so `../../db/src/migrations` lands in `@bluefields/db`.
+    const here = dirname(fileURLToPath(import.meta.url));
+    return resolve(here, '../../db/src/migrations');
+}
+/**
+ * Open (or create) the file-backed local store at `dbDir` and return an
+ * engine-compatible `DbClient`. Idempotent: re-opening an existing store
+ * re-applies DDL tolerantly (`IF NOT EXISTS` / already-exists skips) and
+ * re-seeds via `ON CONFLICT DO NOTHING`.
+ */
+export async function openStore(dbDir) {
+    // PGlite's node fs backend creates only the leaf data dir, not its parents.
+    mkdirSync(dbDir, { recursive: true });
+    const client = new PGlite(dbDir);
+    await client.waitReady;
+    let skipped = 0;
+    const alreadyInitialized = await tableExists(client, 'snapshots');
+    if (!alreadyInitialized) {
+        skipped = await applyMigrations(client);
+    }
+    await ensureDefaultPartitions(client);
+    const db = drizzle(client);
+    await seedLocalCustomer(db);
+    return {
+        db,
+        client,
+        skipped,
+        async close() {
+            await client.close();
+        },
+    };
+}
+/** True if a public-schema table exists. */
+async function tableExists(client, table) {
+    const r = await client.query(`SELECT to_regclass('public.${table}') AS reg;`);
+    return Boolean(r.rows[0]?.reg);
+}
+/** Apply every migration statement-by-statement with per-statement tolerance. */
+async function applyMigrations(client) {
+    const dir = migrationsDir();
+    const files = readdirSync(dir)
+        .filter((f) => f.endsWith('.sql'))
+        .sort();
+    let skipped = 0;
+    for (const f of files) {
+        const text = readFileSync(resolve(dir, f), 'utf8');
+        for (const stmt of splitSql(text)) {
+            const bare = stmt.replace(/--[^\n]*/g, ' ').trim();
+            if (!bare)
+                continue;
+            // Drop transaction control so one incompatible DDL can't abort a whole
+            // file: each statement applies in its own autocommit.
+            if (/^(begin|commit|rollback|start\s+transaction)$/i.test(bare))
+                continue;
+            try {
+                await client.exec(stmt);
+            }
+            catch {
+                skipped += 1; // Neon/pg_cron/partman-only — irrelevant to the engine tables.
+                try {
+                    await client.exec('ROLLBACK');
+                }
+                catch {
+                    /* no transaction open */
+                }
+            }
+        }
+    }
+    return skipped;
+}
+/** Attach a DEFAULT partition to each partitioned parent (idempotent). */
+async function ensureDefaultPartitions(client) {
+    for (const table of PARTITIONED_TABLES) {
+        if (!(await tableExists(client, table)))
+            continue;
+        try {
+            await client.exec(`CREATE TABLE IF NOT EXISTS ${table}_default PARTITION OF ${table} DEFAULT;`);
+        }
+        catch {
+            // Already partitioned by partman, or not actually partitioned in this
+            // build — either way rows can land. Clear any aborted tx state.
+            try {
+                await client.exec('ROLLBACK');
+            }
+            catch {
+                /* no transaction open */
+            }
+        }
+    }
+}
+/** Insert the fixed local customer if absent. */
+async function seedLocalCustomer(db) {
+    await db
+        .insert(customers)
+        .values({ id: LOCAL_CUSTOMER_ID, email: LOCAL_CUSTOMER_EMAIL, plan: 'free' })
+        .onConflictDoNothing();
+}
+/**
+ * Split a migration file into individual statements, respecting `$tag$…$tag$`
+ * dollar-quoted bodies (functions / DO blocks) so we don't split inside them.
+ * Ported verbatim from the QA real-DB harness.
+ */
+function splitSql(sqlText) {
+    const out = [];
+    let cur = '';
+    let dollarTag = null;
+    let inLine = false; // -- line comment
+    let inBlock = false; // /* block comment */
+    let inStr = false; // '...' string literal
+    for (let i = 0; i < sqlText.length; i++) {
+        const ch = sqlText[i];
+        const next = sqlText[i + 1];
+        if (inLine) {
+            cur += ch;
+            if (ch === '\n')
+                inLine = false;
+            continue;
+        }
+        if (inBlock) {
+            cur += ch;
+            if (ch === '*' && next === '/') {
+                cur += next;
+                i += 1;
+                inBlock = false;
+            }
+            continue;
+        }
+        if (inStr) {
+            cur += ch;
+            if (ch === "'") {
+                if (next === "'") {
+                    cur += next;
+                    i += 1;
+                }
+                else
+                    inStr = false;
+            }
+            continue;
+        }
+        if (dollarTag) {
+            if (ch === '$') {
+                const m = sqlText.slice(i).match(/^\$[A-Za-z0-9_]*\$/);
+                if (m && m[0] === dollarTag) {
+                    cur += m[0];
+                    i += m[0].length - 1;
+                    dollarTag = null;
+                    continue;
+                }
+            }
+            cur += ch;
+            continue;
+        }
+        // Not inside any comment/string/dollar-quote:
+        if (ch === '-' && next === '-') {
+            inLine = true;
+            cur += ch;
+            continue;
+        }
+        if (ch === '/' && next === '*') {
+            inBlock = true;
+            cur += ch;
+            continue;
+        }
+        if (ch === "'") {
+            inStr = true;
+            cur += ch;
+            continue;
+        }
+        if (ch === '$') {
+            const m = sqlText.slice(i).match(/^\$[A-Za-z0-9_]*\$/);
+            if (m) {
+                dollarTag = m[0];
+                cur += m[0];
+                i += m[0].length - 1;
+                continue;
+            }
+        }
+        if (ch === ';') {
+            out.push(cur);
+            cur = '';
+            continue;
+        }
+        cur += ch;
+    }
+    if (cur.trim())
+        out.push(cur);
+    return out;
+}
+//# sourceMappingURL=store.js.map

package/dist/store.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"store.js","sourceRoot":"","sources":["../src/store.ts"],"names":[],"mappings":"AAAA,6CAA6C;AAC7C;;;;;;;;;;;;;;;;;;;;GAoBG;AAEH,OAAO,EAAE,SAAS,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,SAAS,CAAC;AAC/D,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAC7C,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AAEzC,OAAO,EAAiB,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAC1D,OAAO,EAAE,MAAM,EAAE,MAAM,sBAAsB,CAAC;AAC9C,OAAO,EAAE,OAAO,EAAE,MAAM,oBAAoB,CAAC;AAE7C;;;;GAIG;AACH,MAAM,CAAC,MAAM,iBAAiB,GAAG,sCAA+C,CAAC;AACjF,MAAM,CAAC,MAAM,oBAAoB,GAAG,yBAAkC,CAAC;AACvE,MAAM,CAAC,MAAM,qBAAqB,GAAG,IAAI,CAAC;AAE1C,gFAAgF;AAChF,MAAM,kBAAkB,GAAG,CAAC,WAAW,EAAE,cAAc,EAAE,QAAQ,EAAE,WAAW,CAAU,CAAC;AAEzF,mFAAmF;AACnF,SAAS,aAAa;IACpB,MAAM,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,iBAAiB,CAAC;IAC/C,IAAI,QAAQ,EAAE,IAAI,EAAE;QAAE,OAAO,OAAO,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC;IACtD,yEAAyE;IACzE,0EAA0E;IAC1E,MAAM,IAAI,GAAG,OAAO,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;IACrD,OAAO,OAAO,CAAC,IAAI,EAAE,yBAAyB,CAAC,CAAC;AAClD,CAAC;AAWD;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,KAAa;IAC3C,4EAA4E;IAC5E,SAAS,CAAC,KAAK,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACtC,MAAM,MAAM,GAAG,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC;IACjC,MAAM,MAAM,CAAC,SAAS,CAAC;IAEvB,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,MAAM,kBAAkB,GAAG,MAAM,WAAW,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC;IAClE,IAAI,CAAC,kBAAkB,EAAE,CAAC;QACxB,OAAO,GAAG,MAAM,eAAe,CAAC,MAAM,CAAC,CAAC;IAC1C,CAAC;IAED,MAAM,uBAAuB,CAAC,MAAM,CAAC,CAAC;IAEtC,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,CAAwB,CAAC;IAClD,MAAM,iBAAiB,CAAC,EAAE,CAAC,CAAC;IAE5B,OAAO;QACL,EAAE;QACF,MAAM;QACN,OAAO;QACP,KAAK,CAAC,KAAK;YACT,MAAM,MAAM,CAAC,KAAK,EAAE,CAAC;QACvB,CAAC;KACF,CAAC;AACJ,CAAC;AAED,4CAA4C;AAC5C,KAAK,UAAU,WAAW,CAAC,MAAc,EAAE,KAAa;IACtD,MAAM,CAAC,GAAG,MAAM,MAAM,CAAC,KAAK,CAC1B,8BAA8B,KAAK,YAAY,CAChD,CAAC;IACF,OAAO,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;AACjC,CAAC;AAED,iFAAiF;AACjF,KAAK,UAAU,eAAe,CAAC,MAAc;IAC3C,MAAM,GAAG,GAAG,aAAa,EAAE,CAAC;IAC5B,MAAM,KAAK,GAAG,WAAW,CAAC,GAAG,CAAC;SAC3B,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;SACjC,IAAI,EAAE,CAAC;IACV,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC;QACtB,MAAM,IAAI,GAAG,YAAY,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC;QACnD,KAAK,MAAM,IAAI,IAAI,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;YAClC,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,WAAW,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;YACnD,IAAI,CAAC,IAAI;gBAAE,SAAS;YACpB,uEAAuE;YACvE,sDAAsD;YACtD,IAAI,gDAAgD,CAAC,IAAI,CAAC,IAAI,CAAC;gBAAE,SAAS;YAC1E,IAAI,CAAC;gBACH,MAAM,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC1B,CAAC;YAAC,MAAM,CAAC;gBACP,OAAO,IAAI,CAAC,CAAC,CAAC,+DAA+D;gBAC7E,IAAI,CAAC;oBACH,MAAM,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;gBAChC,CAAC;gBAAC,MAAM,CAAC;oBACP,yBAAyB;gBAC3B,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,0EAA0E;AAC1E,KAAK,UAAU,uBAAuB,CAAC,MAAc;IACnD,KAAK,MAAM,KAAK,IAAI,kBAAkB,EAAE,CAAC;QACvC,IAAI,CAAC,CAAC,MAAM,WAAW,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;YAAE,SAAS;QAClD,IAAI,CAAC;YACH,MAAM,MAAM,CAAC,IAAI,CACf,8BAA8B,KAAK,yBAAyB,KAAK,WAAW,CAC7E,CAAC;QACJ,CAAC;QAAC,MAAM,CAAC;YACP,sEAAsE;YACtE,gEAAgE;YAChE,IAAI,CAAC;gBACH,MAAM,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;YAChC,CAAC;YAAC,MAAM,CAAC;gBACP,yBAAyB;YAC3B,CAAC;QACH,CAAC;IACH,CAAC;AACH,CAAC;AAED,iDAAiD;AACjD,KAAK,UAAU,iBAAiB,CAAC,EAAY;IAC3C,MAAM,EAAE;SACL,MAAM,CAAC,SAAS,CAAC;SACjB,MAAM,CAAC,EAAE,EAAE,EAAE,iBAAiB,EAAE,KAAK,EAAE,oBAAoB,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC;SAC5E,mBAAmB,EAAE,CAAC;AAC3B,CAAC;AAED;;;;GAIG;AACH,SAAS,QAAQ,CAAC,OAAe;IAC/B,MAAM,GAAG,GAAa,EAAE,CAAC;IACzB,IAAI,GAAG,GAAG,EAAE,CAAC;IACb,IAAI,SAAS,GAAkB,IAAI,CAAC;IACpC,IAAI,MAAM,GAAG,KAAK,CAAC,CAAC,kBAAkB;IACtC,IAAI,OAAO,GAAG,KAAK,CAAC,CAAC,sBAAsB;IAC3C,IAAI,KAAK,GAAG,KAAK,CAAC,CAAC,uBAAuB;IAC1C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACxC,MAAM,EAAE,GAAG,OAAO,CAAC,CAAC,CAAW,CAAC;QAChC,MAAM,IAAI,GAAG,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QAC5B,IAAI,MAAM,EAAE,CAAC;YACX,GAAG,IAAI,EAAE,CAAC;YACV,IAAI,EAAE,KAAK,IAAI;gBAAE,MAAM,GAAG,KAAK,CAAC;YAChC,SAAS;QACX,CAAC;QACD,IAAI,OAAO,EAAE,CAAC;YACZ,GAAG,IAAI,EAAE,CAAC;YACV,IAAI,EAAE,KAAK,GAAG,IAAI,IAAI,KAAK,GAAG,EAAE,CAAC;gBAC/B,GAAG,IAAI,IAAI,CAAC;gBACZ,CAAC,IAAI,CAAC,CAAC;gBACP,OAAO,GAAG,KAAK,CAAC;YAClB,CAAC;YACD,SAAS;QACX,CAAC;QACD,IAAI,KAAK,EAAE,CAAC;YACV,GAAG,IAAI,EAAE,CAAC;YACV,IAAI,EAAE,KAAK,GAAG,EAAE,CAAC;gBACf,IAAI,IAAI,KAAK,GAAG,EAAE,CAAC;oBACjB,GAAG,IAAI,IAAI,CAAC;oBACZ,CAAC,IAAI,CAAC,CAAC;gBACT,CAAC;;oBAAM,KAAK,GAAG,KAAK,CAAC;YACvB,CAAC;YACD,SAAS;QACX,CAAC;QACD,IAAI,SAAS,EAAE,CAAC;YACd,IAAI,EAAE,KAAK,GAAG,EAAE,CAAC;gBACf,MAAM,CAAC,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,oBAAoB,CAAC,CAAC;gBACvD,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,KAAK,SAAS,EAAE,CAAC;oBAC5B,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;oBACZ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;oBACrB,SAAS,GAAG,IAAI,CAAC;oBACjB,SAAS;gBACX,CAAC;YACH,CAAC;YACD,GAAG,IAAI,EAAE,CAAC;YACV,SAAS;QACX,CAAC;QACD,8CAA8C;QAC9C,IAAI,EAAE,KAAK,GAAG,IAAI,IAAI,KAAK,GAAG,EAAE,CAAC;YAC/B,MAAM,GAAG,IAAI,CAAC;YACd,GAAG,IAAI,EAAE,CAAC;YACV,SAAS;QACX,CAAC;QACD,IAAI,EAAE,KAAK,GAAG,IAAI,IAAI,KAAK,GAAG,EAAE,CAAC;YAC/B,OAAO,GAAG,IAAI,CAAC;YACf,GAAG,IAAI,EAAE,CAAC;YACV,SAAS;QACX,CAAC;QACD,IAAI,EAAE,KAAK,GAAG,EAAE,CAAC;YACf,KAAK,GAAG,IAAI,CAAC;YACb,GAAG,IAAI,EAAE,CAAC;YACV,SAAS;QACX,CAAC;QACD,IAAI,EAAE,KAAK,GAAG,EAAE,CAAC;YACf,MAAM,CAAC,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,oBAAoB,CAAC,CAAC;YACvD,IAAI,CAAC,EAAE,CAAC;gBACN,SAAS,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;gBACjB,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;gBACZ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;gBACrB,SAAS;YACX,CAAC;QACH,CAAC;QACD,IAAI,EAAE,KAAK,GAAG,EAAE,CAAC;YACf,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACd,GAAG,GAAG,EAAE,CAAC;YACT,SAAS;QACX,CAAC;QACD,GAAG,IAAI,EAAE,CAAC;IACZ,CAAC;IACD,IAAI,GAAG,CAAC,IAAI,EAAE;QAAE,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAC9B,OAAO,GAAG,CAAC;AACb,CAAC"}

package/package.json

@@ -0,0 +1,63 @@
+{
+  "name": "@bluefields/cli",
+  "version": "0.2.0",
+  "description": "Bluefield — git for web data. A local-first CLI + library that snapshots web pages into a content-addressed store on your own disk and signs each capture with an offline-verifiable Ed25519 provenance manifest. No server, no account, no telemetry.",
+  "license": "AGPL-3.0-or-later",
+  "type": "module",
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "bin": {
+    "bf": "dist/cli.js",
+    "bluefields": "dist/cli.js"
+  },
+  "files": [
+    "dist",
+    "README.md",
+    "LICENSE"
+  ],
+  "engines": {
+    "node": ">=22"
+  },
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/danielvhofmann/bluefield.git",
+    "directory": "packages/cli"
+  },
+  "keywords": [
+    "bluefield",
+    "web-scraping",
+    "provenance",
+    "attestation",
+    "content-addressed",
+    "snapshot",
+    "watch",
+    "diff",
+    "local-first",
+    "offline",
+    "ed25519",
+    "git-for-web-data",
+    "mcp"
+  ],
+  "publishConfig": {
+    "access": "public"
+  },
+  "dependencies": {
+    "@electric-sql/pglite": "^0.5.3",
+    "drizzle-orm": "0.36.4",
+    "@bluefields/attest": "0.2.0",
+    "@bluefields/db": "0.2.0",
+    "@bluefields/extractor": "0.2.0",
+    "@bluefields/fetcher": "0.2.0",
+    "@bluefields/differ": "0.2.0",
+    "@bluefields/primitives": "0.2.0"
+  },
+  "devDependencies": {
+    "@types/node": "22.9.0"
+  },
+  "scripts": {
+    "build": "tsc -p .",
+    "start": "tsx src/cli.ts",
+    "typecheck": "tsc --noEmit -p .",
+    "test": "vitest run"
+  }
+}