@sdods/db 0.2.0

package/dist/switch/import.js

@@ -0,0 +1,106 @@
+import { createReadStream, existsSync, readdirSync } from 'node:fs';
+import { join } from 'node:path';
+import { createInterface } from 'node:readline';
+import { sql } from 'kysely';
+import { TABLES_IN_FK_ORDER } from '../schema.js';
+import { isTdTable, listTdColumns } from '../test-data.js';
+const BOOL_COLUMNS = new Set([
+    'archived',
+    'is_default',
+    'flaky',
+    'will_be_retried',
+    'succeeded',
+    'accepted',
+    'quarantined',
+    'enabled',
+    'active',
+    'catch_up',
+]);
+/**
+ * Load `<dir>/<table>.jsonl` files in FK order inside one transaction, `ON CONFLICT DO NOTHING`.
+ * td_* tables are created on the fly (text columns) when missing.
+ */
+export async function importAll(adb, dir, opts = {}) {
+    const batch = opts.batch ?? 500;
+    const files = readdirSync(dir).filter((f) => f.endsWith('.jsonl'));
+    const known = new Set(TABLES_IN_FK_ORDER);
+    const ordered = [
+        ...TABLES_IN_FK_ORDER.filter((t) => files.includes(`${t}.jsonl`)),
+        ...files
+            .map((f) => f.replace(/\.jsonl$/, ''))
+            .filter((t) => !known.has(t) && isTdTable(t))
+            .sort(),
+    ];
+    const result = { tables: [] };
+    await adb.db.transaction().execute(async (trx) => {
+        for (const table of ordered) {
+            const file = join(dir, `${table}.jsonl`);
+            if (!existsSync(file))
+                continue;
+            const rows = [];
+            let inserted = 0;
+            let skipped = 0;
+            const flush = async () => {
+                if (rows.length === 0)
+                    return;
+                const values = rows.map((r) => denormalizeRow(adb, r));
+                if (isTdTable(table))
+                    await ensureTdTable(trx, adb, table, Object.keys(values[0]));
+                const q = trx.insertInto(table).values(values);
+                const res = isTdTable(table)
+                    ? await q.execute()
+                    : await q.onConflict((oc) => oc.doNothing()).execute();
+                const n = Number(res?.[0]?.numInsertedOrUpdatedRows ?? rows.length);
+                inserted += n;
+                skipped += rows.length - n;
+                rows.length = 0;
+            };
+            const rl = createInterface({ input: createReadStream(file), crlfDelay: Infinity });
+            for await (const line of rl) {
+                const t = line.trim();
+                if (!t)
+                    continue;
+                rows.push(JSON.parse(t));
+                if (rows.length >= batch)
+                    await flush();
+            }
+            await flush();
+            result.tables.push({ table, rows: inserted, skipped });
+        }
+    });
+    return result;
+}
+function denormalizeRow(adb, row) {
+    const out = {};
+    for (const [k, v] of Object.entries(row)) {
+        if (v && typeof v === 'object' && '$base64' in v)
+            out[k] = Buffer.from(v.$base64, 'base64');
+        else if (k.endsWith('_json'))
+            out[k] = v == null ? null : JSON.stringify(v);
+        else if (BOOL_COLUMNS.has(k) && v != null)
+            out[k] = adb.driver === 'postgres' ? Boolean(v) : v ? 1 : 0;
+        else
+            out[k] = v;
+    }
+    return out;
+}
+async function ensureTdTable(trx, adb, table, columns) {
+    const existing = await listTdColumns(trx, adb.driver, table);
+    if (existing.length === 0) {
+        let b = trx.schema
+            .createTable(table)
+            .addColumn('row_index', 'integer', (cb) => cb.notNull())
+            .addColumn('env', 'text', (cb) => cb.notNull().defaultTo('*'));
+        for (const c of columns)
+            if (c !== 'row_index' && c !== 'env')
+                b = b.addColumn(c, 'text');
+        await b.execute();
+    }
+    else {
+        for (const c of columns)
+            if (!existing.includes(c))
+                await trx.schema.alterTable(table).addColumn(c, 'text').execute();
+    }
+    await sql `select 1`.execute(trx);
+}
+//# sourceMappingURL=import.js.map

package/dist/switch/index.d.ts

@@ -0,0 +1,5 @@
+export * from './export.js';
+export * from './import.js';
+export * from './switch.js';
+export * from './prune.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/switch/index.js

@@ -0,0 +1,5 @@
+export * from './export.js';
+export * from './import.js';
+export * from './switch.js';
+export * from './prune.js';
+//# sourceMappingURL=index.js.map

package/dist/switch/prune.d.ts

@@ -0,0 +1,19 @@
+import type { SdodsDb } from '../create-db.js';
+export interface PruneOptions {
+    keepRuns?: number;
+    keepDays?: number;
+    projectSlug?: string;
+    artifactsRoot?: string;
+    dryRun?: boolean;
+}
+export interface PruneResult {
+    deleted: Array<{
+        id: string;
+        projectSlug: string;
+        createdAt: string;
+    }>;
+    kept: number;
+}
+/** Delete old runs (and their artifact directories) beyond `keepRuns` per project or older than `keepDays`. */
+export declare function prune(adb: SdodsDb, opts: PruneOptions): Promise<PruneResult>;
+//# sourceMappingURL=prune.d.ts.map

package/dist/switch/prune.js

@@ -0,0 +1,50 @@
+import { existsSync, rmSync } from 'node:fs';
+import { join } from 'node:path';
+import { sql } from 'kysely';
+import { deleteRun } from '../repos/runs.js';
+/** Delete old runs (and their artifact directories) beyond `keepRuns` per project or older than `keepDays`. */
+export async function prune(adb, opts) {
+    const { db } = adb;
+    let q = db
+        .selectFrom('runs')
+        .innerJoin('projects', 'projects.id', 'runs.project_id')
+        .select([
+        'runs.id as id',
+        'projects.slug as slug',
+        'runs.created_at as created_at',
+        'runs.started_at as started_at',
+        'runs.artifacts_dir as artifacts_dir',
+    ])
+        .orderBy(sql `coalesce(runs.started_at, runs.created_at)`, 'desc');
+    if (opts.projectSlug)
+        q = q.where('projects.slug', '=', opts.projectSlug);
+    const rows = await q.execute();
+    const cutoff = opts.keepDays ? Date.now() - opts.keepDays * 86_400_000 : null;
+    const perProject = new Map();
+    const victims = [];
+    for (const r of rows) {
+        const n = (perProject.get(r.slug) ?? 0) + 1;
+        perProject.set(r.slug, n);
+        const tooMany = opts.keepRuns !== undefined && n > opts.keepRuns;
+        const tooOld = cutoff !== null && new Date(String(r.created_at)).getTime() < cutoff;
+        if (tooMany || tooOld)
+            victims.push(r);
+    }
+    if (!opts.dryRun) {
+        for (const v of victims) {
+            await deleteRun(db, v.id);
+            const dir = v.artifacts_dir ?? (opts.artifactsRoot ? join(opts.artifactsRoot, v.id) : null);
+            if (dir && existsSync(dir))
+                rmSync(dir, { recursive: true, force: true });
+        }
+    }
+    return {
+        deleted: victims.map((v) => ({
+            id: v.id,
+            projectSlug: v.slug,
+            createdAt: String(v.created_at),
+        })),
+        kept: rows.length - victims.length,
+    };
+}
+//# sourceMappingURL=prune.js.map

package/dist/switch/switch.d.ts

@@ -0,0 +1,34 @@
+import { type SdodsDb } from '../create-db.js';
+import { type Driver, type DriverConfig } from '../driver.js';
+export interface SwitchOptions {
+    source: SdodsDb;
+    target: Driver;
+    targetUrl?: string;
+    targetPath?: string;
+    dryRun?: boolean;
+    /** clear the target's platform tables before importing (required when the target has data) */
+    truncate?: boolean;
+    envFile?: string;
+    keepExport?: boolean;
+}
+export interface SwitchResult {
+    target: DriverConfig;
+    dryRun: boolean;
+    tables: Array<{
+        table: string;
+        source: number;
+        target: number;
+        ok: boolean;
+    }>;
+    ok: boolean;
+    envFile?: string;
+    envChanges?: string[];
+}
+/**
+ * Copy the platform database to the other driver: migrate target → export → import → verify counts →
+ * (unless dry-run) rewrite DB_DRIVER/DATABASE_URL/SQLITE_PATH in `.env`.
+ */
+export declare function switchDriver(opts: SwitchOptions): Promise<SwitchResult>;
+/** Rewrite DB_DRIVER and the matching URL/path in a dotenv file, preserving other lines. */
+export declare function rewriteEnv(envFile: string, cfg: DriverConfig): string[];
+//# sourceMappingURL=switch.d.ts.map

package/dist/switch/switch.js

@@ -0,0 +1,133 @@
+import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join, resolve } from 'node:path';
+import { createDb } from '../create-db.js';
+import { DEFAULT_SQLITE_PATH } from '../driver.js';
+import { migrateToLatest } from '../migrate.js';
+import { TABLES_IN_FK_ORDER } from '../schema.js';
+import { listTdTables } from '../test-data.js';
+import { countRows, exportAll } from './export.js';
+import { importAll } from './import.js';
+/**
+ * Copy the platform database to the other driver: migrate target → export → import → verify counts →
+ * (unless dry-run) rewrite DB_DRIVER/DATABASE_URL/SQLITE_PATH in `.env`.
+ */
+export async function switchDriver(opts) {
+    const targetCfg = opts.target === 'postgres'
+        ? { driver: 'postgres', databaseUrl: opts.targetUrl ?? process.env.DATABASE_URL }
+        : {
+            driver: 'sqlite',
+            sqlitePath: opts.targetPath ?? process.env.SQLITE_PATH ?? DEFAULT_SQLITE_PATH,
+        };
+    if (targetCfg.driver === 'postgres' && !targetCfg.databaseUrl)
+        throw new Error('switch to postgres needs --target-url or DATABASE_URL');
+    if (opts.source.driver === targetCfg.driver && sameLocation(opts.source.config, targetCfg)) {
+        throw new Error(`Source and target are the same ${targetCfg.driver} database.`);
+    }
+    const target = createDb(targetCfg);
+    const dir = mkdtempSync(join(tmpdir(), 'sdods-switch-'));
+    try {
+        await migrateToLatest(target);
+        // A target that already holds platform rows cannot be merged safely: slug conflicts would keep
+        // the target's ids while child rows reference the source's ids (foreign key failures).
+        const occupied = [];
+        for (const table of TABLES_IN_FK_ORDER) {
+            const n = await countRows(target, table).catch(() => 0);
+            if (n > 0)
+                occupied.push(`${table} (${n})`);
+        }
+        if (occupied.length && !opts.truncate) {
+            throw new Error(`Target ${targetCfg.driver} database already has data in ${occupied.join(', ')}. ` +
+                'Point at an empty database, or pass --truncate to clear its platform tables first.');
+        }
+        if (occupied.length && opts.truncate && !opts.dryRun) {
+            for (const table of [...TABLES_IN_FK_ORDER].reverse()) {
+                await target.db.deleteFrom(table).execute();
+            }
+        }
+        const tables = [
+            ...TABLES_IN_FK_ORDER,
+            ...(await listTdTables(opts.source.db, opts.source.driver)),
+        ];
+        if (opts.dryRun) {
+            // Verify without writing: report what would be copied and confirm the target is (or will be) empty.
+            const rows = [];
+            for (const table of tables) {
+                const s = await countRows(opts.source, table);
+                const t = await countRows(target, table).catch(() => 0);
+                rows.push({
+                    table,
+                    source: s,
+                    target: opts.truncate ? 0 : t,
+                    ok: opts.truncate || t === 0,
+                });
+            }
+            return { target: targetCfg, dryRun: true, tables: rows, ok: rows.every((r) => r.ok) };
+        }
+        await exportAll(opts.source, dir);
+        await importAll(target, dir);
+        const rows = [];
+        for (const table of tables) {
+            const s = await countRows(opts.source, table);
+            const t = await countRows(target, table).catch(() => -1);
+            rows.push({ table, source: s, target: t, ok: t >= s });
+        }
+        const ok = rows.every((r) => r.ok);
+        const result = {
+            target: targetCfg,
+            dryRun: false,
+            tables: rows,
+            ok,
+        };
+        if (!ok)
+            return result;
+        const envFile = resolve(opts.envFile ?? '.env');
+        result.envFile = envFile;
+        result.envChanges = rewriteEnv(envFile, targetCfg);
+        return result;
+    }
+    finally {
+        await target.close();
+        if (!opts.keepExport)
+            rmSync(dir, { recursive: true, force: true });
+    }
+}
+function sameLocation(a, b) {
+    if (a.driver !== b.driver)
+        return false;
+    if (a.driver === 'postgres')
+        return a.databaseUrl === b.databaseUrl;
+    return resolve(a.sqlitePath ?? '') === resolve(b.sqlitePath ?? '');
+}
+/** Rewrite DB_DRIVER and the matching URL/path in a dotenv file, preserving other lines. */
+export function rewriteEnv(envFile, cfg) {
+    const lines = existsSync(envFile)
+        ? readFileSync(envFile, 'utf8')
+            .split(/\r?\n/)
+            .filter((l, i, arr) => !(i === arr.length - 1 && l === ''))
+        : [];
+    const set = (key, value) => {
+        const idx = lines.findIndex((l) => new RegExp(`^\\s*#?\\s*${key}=`).test(l));
+        if (value === undefined) {
+            if (idx >= 0 && !lines[idx].trim().startsWith('#'))
+                lines[idx] = `# ${lines[idx]}`;
+            return `${key} commented out`;
+        }
+        const line = `${key}=${value}`;
+        if (idx >= 0)
+            lines[idx] = line;
+        else
+            lines.push(line);
+        return line;
+    };
+    const changes = [set('DB_DRIVER', cfg.driver)];
+    if (cfg.driver === 'postgres') {
+        changes.push(set('DATABASE_URL', cfg.databaseUrl));
+    }
+    else {
+        changes.push(set('SQLITE_PATH', cfg.sqlitePath));
+    }
+    writeFileSync(envFile, lines.join('\n').replace(/\n*$/, '\n'));
+    return changes;
+}
+//# sourceMappingURL=switch.js.map

package/dist/test-data.d.ts

@@ -0,0 +1,22 @@
+import { type Kysely } from 'kysely';
+import type { Driver } from './driver.js';
+export type Row = Record<string, unknown>;
+/** `td_<slug>_<name>` — user-defined test-data tables share the platform DB on both dialects. */
+export declare function tdTableName(projectSlug: string, dataset: string): string;
+export declare function isTdTable(name: string): boolean;
+/** Create (if needed) and fill a test-data table. Columns are text; `env` and `row_index` are always present. */
+export declare function importRowsToTable(db: Kysely<any>, driver: Driver, input: {
+    projectSlug: string;
+    dataset: string;
+    env?: string;
+    rows: Row[];
+    truncate?: boolean;
+}): Promise<{
+    table: string;
+    inserted: number;
+    columns: string[];
+}>;
+export declare function readTableRows(db: Kysely<any>, table: string, env?: string): Promise<Row[]>;
+export declare function listTdTables(db: Kysely<any>, driver: Driver): Promise<string[]>;
+export declare function listTdColumns(db: Kysely<any>, driver: Driver, table: string): Promise<string[]>;
+//# sourceMappingURL=test-data.d.ts.map

package/dist/test-data.js

@@ -0,0 +1,79 @@
+import { sql } from 'kysely';
+/** `td_<slug>_<name>` — user-defined test-data tables share the platform DB on both dialects. */
+export function tdTableName(projectSlug, dataset) {
+    const clean = (s) => s
+        .toLowerCase()
+        .replace(/[^a-z0-9]+/g, '_')
+        .replace(/^_+|_+$/g, '');
+    return `td_${clean(projectSlug)}_${clean(dataset)}`;
+}
+export function isTdTable(name) {
+    return /^td_[a-z0-9_]+$/.test(name);
+}
+/** Create (if needed) and fill a test-data table. Columns are text; `env` and `row_index` are always present. */
+export async function importRowsToTable(db, driver, input) {
+    const table = tdTableName(input.projectSlug, input.dataset);
+    const env = input.env ?? '*';
+    const columns = [...new Set(input.rows.flatMap((r) => Object.keys(r)))].filter((c) => c !== 'env' && c !== 'row_index');
+    const existing = await listTdColumns(db, driver, table);
+    if (existing.length === 0) {
+        let b = db.schema
+            .createTable(table)
+            .addColumn('row_index', 'integer', (cb) => cb.notNull())
+            .addColumn('env', 'text', (cb) => cb.notNull().defaultTo('*'));
+        for (const c of columns)
+            b = b.addColumn(c, 'text');
+        await b.execute();
+        await db.schema
+            .createIndex(`idx_${table}_env`)
+            .on(table)
+            .columns(['env', 'row_index'])
+            .execute();
+    }
+    else {
+        for (const c of columns)
+            if (!existing.includes(c))
+                await db.schema.alterTable(table).addColumn(c, 'text').execute();
+    }
+    if (input.truncate)
+        await db.deleteFrom(table).where('env', '=', env).execute();
+    const values = input.rows.map((r, i) => {
+        const out = { row_index: i, env };
+        for (const c of columns)
+            out[c] = r[c] == null ? null : typeof r[c] === 'object' ? JSON.stringify(r[c]) : String(r[c]);
+        return out;
+    });
+    for (let i = 0; i < values.length; i += 500) {
+        await db
+            .insertInto(table)
+            .values(values.slice(i, i + 500))
+            .execute();
+    }
+    return { table, inserted: values.length, columns };
+}
+export async function readTableRows(db, table, env) {
+    let q = db.selectFrom(table).selectAll().orderBy('row_index');
+    if (env)
+        q = q.where('env', 'in', [env, '*']);
+    const rows = (await q.execute());
+    return rows.map(({ row_index: _i, env: _e, ...rest }) => rest);
+}
+export async function listTdTables(db, driver) {
+    if (driver === 'postgres') {
+        const res = await sql `select table_name from information_schema.tables where table_schema = current_schema() and table_name like 'td\_%'`.execute(db);
+        return res.rows.map((r) => r.table_name).filter(isTdTable);
+    }
+    const res = await sql `select name from sqlite_master where type = 'table' and name like 'td\\_%' escape '\\'`.execute(db);
+    return res.rows.map((r) => r.name).filter(isTdTable);
+}
+export async function listTdColumns(db, driver, table) {
+    if (!isTdTable(table))
+        return [];
+    if (driver === 'postgres') {
+        const res = await sql `select column_name from information_schema.columns where table_schema = current_schema() and table_name = ${table} order by ordinal_position`.execute(db);
+        return res.rows.map((r) => r.column_name);
+    }
+    const res = await sql `select name from pragma_table_info(${table})`.execute(db);
+    return res.rows.map((r) => r.name);
+}
+//# sourceMappingURL=test-data.js.map

package/package.json

@@ -0,0 +1,50 @@
+{
+  "name": "@sdods/db",
+  "version": "0.2.0",
+  "description": "SDODS platform database: one schema on SQLite or Postgres, migrations, result ingest and insights.",
+  "license": "Apache-2.0",
+  "author": "SDODS <admin@sdods.com>",
+  "type": "module",
+  "exports": {
+    ".": "./dist/index.js",
+    "./schema": "./dist/schema.js",
+    "./ingest": "./dist/ingest/index.js",
+    "./switch": "./dist/switch/index.js",
+    "./insights": "./dist/insights.js",
+    "./test-data": "./dist/test-data.js"
+  },
+  "publishConfig": {
+    "access": "public"
+  },
+  "files": [
+    "dist"
+  ],
+  "engines": {
+    "node": ">=22"
+  },
+  "dependencies": {
+    "@sdods/contracts": "0.2.0",
+    "better-sqlite3": "^13.0.3",
+    "kysely": "^0.29.5",
+    "pg": "^8.23.0",
+    "zod": "^4.5.4"
+  },
+  "homepage": "https://sdods.com",
+  "bugs": {
+    "url": "https://github.com/siri1410/SDODS/issues"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/siri1410/SDODS.git",
+    "directory": "packages/db"
+  },
+  "keywords": [
+    "sdods",
+    "automation",
+    "orchestration",
+    "testing",
+    "bdd",
+    "ai-agents",
+    "mcp"
+  ]
+}