@chronicle.app/etl-sqlite 0.4.0 → 0.5.0

package/dist/SqliteExtractor.d.ts

@@ -1,6 +1,13 @@
 import { DatabaseSync } from 'node:sqlite';
 import { z } from 'zod';
-import { type Delivery, Extractor } from '@chronicle.app/etl';
+import { type DatabaseColumns, type DatabaseSchema, type Delivery, Extractor } from '@chronicle.app/etl';
+/**
+ * Refuse an open database unless it has every column one of `databases`
+ * needs, naming `path` as what was given. Its columns, when it has them. For
+ * a reader that opens its database itself rather than through a
+ * {@link SqliteExtractor}.
+ */
+export declare function assertDatabaseSchema(db: DatabaseSync, databases: DatabaseSchema[], path: string): DatabaseColumns;
 /** Read an existing source database without changing its journal mode. */
 export declare abstract class SqliteExtractor<SelfClass extends typeof SqliteExtractor = typeof SqliteExtractor> extends Extractor<SelfClass> {
     static delivery: Delivery;
@@ -21,8 +28,27 @@ export declare abstract class SqliteExtractor<SelfClass extends typeof SqliteExt
         until?: Date | undefined;
         limit?: number | undefined;
     }>;
+    /**
+     * The tables a reader reads and the columns its queries use, which its
+     * manifest's `databases` repeats for `scan`. A reader that declares them
+     * reads a snapshot of its input, and only an input with every column a
+     * schema needs.
+     */
+    static databases?: DatabaseSchema[];
     protected db: DatabaseSync | null;
+    /** The checked database's columns, for {@link has}. */
+    private columns;
+    private snapshotDir;
     setup(): Promise<void>;
+    /**
+     * Refuse the open database unless it has every column one of `databases`
+     * needs, naming `path` as what was given. `setup()` checks a declared
+     * schema; a reader that declares none, as one whose input is a copy
+     * already, can check one after it.
+     */
+    protected assertSchema(databases: DatabaseSchema[], path: string): void;
+    /** Whether the database has a column a schema marks as optional, as `table.column`. */
+    protected has(column: string): boolean;
     /**
      * Open a database read-only without changing its journal mode. `setup()`
      * opens `input` with this; subclasses can call it to read another file, such

package/dist/SqliteExtractor.js

@@ -1,19 +1,80 @@
+import { rmSync } from 'node:fs';
+import { mkdtemp } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
 import { DatabaseSync } from 'node:sqlite';
 import { z } from 'zod';
-import { Extractor, assertReadable } from '@chronicle.app/etl';
+import { Extractor, InputNotFound, assertReadable, columnsOf, matchSchema, } from '@chronicle.app/etl';
+import { snapshot } from './snapshot.js';
+/**
+ * Refuse an open database unless it has every column one of `databases`
+ * needs, naming `path` as what was given. Its columns, when it has them. For
+ * a reader that opens its database itself rather than through a
+ * {@link SqliteExtractor}.
+ */
+export function assertDatabaseSchema(db, databases, path) {
+    const columns = columnsOf(db);
+    const match = matchSchema(columns, databases);
+    if (!match || match.missing.length > 0 || match.schema.supported === false) {
+        throw new InputNotFound(match ? 'That’s a version of the database this can’t read' : 'That isn’t the database', {
+            path,
+            ...(match && { fields: { missing: match.missing } }),
+            hint: 'Run `chronicle scan <path>` to see what it is.',
+        });
+    }
+    return columns;
+}
 /** Read an existing source database without changing its journal mode. */
 export class SqliteExtractor extends Extractor {
     static delivery = 'local';
     static schema = Extractor.schema.extend({
         input: z.string().min(1).describe('Path to an existing SQLite database'),
     });
+    /**
+     * The tables a reader reads and the columns its queries use, which its
+     * manifest's `databases` repeats for `scan`. A reader that declares them
+     * reads a snapshot of its input, and only an input with every column a
+     * schema needs.
+     */
+    static databases;
     db = null;
+    /** The checked database's columns, for {@link has}. */
+    columns = null;
+    snapshotDir = null;
     async setup() {
         if (this.db)
             throw new Error('Database already initialized');
         await super.setup();
         const { input } = this.config;
-        this.db = this.openDatabase(input);
+        const { databases } = this.constructor;
+        if (!databases) {
+            this.db = this.openDatabase(input);
+            return;
+        }
+        assertReadable(input, 'database');
+        this.snapshotDir = await mkdtemp(join(tmpdir(), 'chronicle-sqlite-'));
+        try {
+            this.db = this.openDatabase(await snapshot(input, this.snapshotDir));
+            this.assertSchema(databases, input);
+        }
+        catch (error) {
+            await this.teardown();
+            throw error;
+        }
+    }
+    /**
+     * Refuse the open database unless it has every column one of `databases`
+     * needs, naming `path` as what was given. `setup()` checks a declared
+     * schema; a reader that declares none, as one whose input is a copy
+     * already, can check one after it.
+     */
+    assertSchema(databases, path) {
+        this.columns = assertDatabaseSchema(this.db, databases, path);
+    }
+    /** Whether the database has a column a schema marks as optional, as `table.column`. */
+    has(column) {
+        const [table, name] = column.toLowerCase().split('.');
+        return this.columns?.get(table)?.has(name) ?? false;
     }
     /**
      * Open a database read-only without changing its journal mode. `setup()`
@@ -31,6 +92,11 @@ export class SqliteExtractor extends Extractor {
             this.db.close();
             this.db = null;
         }
+        if (this.snapshotDir) {
+            rmSync(this.snapshotDir, { recursive: true, force: true });
+            this.snapshotDir = null;
+        }
+        this.columns = null;
         await super.teardown();
     }
 }

package/dist/index.d.ts

@@ -1,6 +1,8 @@
 export * from './SqliteExtractor.js';
+export * from './snapshot.js';
 export * from './statements.js';
 export * from './timeRange.js';
 export * from './appleEpoch.js';
 export * from './chromeEpoch.js';
+export * from './months.js';
 //# sourceMappingURL=index.d.ts.map

package/dist/index.js

@@ -1,5 +1,7 @@
 export * from './SqliteExtractor.js';
+export * from './snapshot.js';
 export * from './statements.js';
 export * from './timeRange.js';
 export * from './appleEpoch.js';
 export * from './chromeEpoch.js';
+export * from './months.js';

package/dist/months.d.ts

@@ -0,0 +1,14 @@
+/**
+ * A month bucket in SQL, `YYYY-MM` in local time, as a survey's `byMonth`
+ * keys it, from an expression in Unix seconds.
+ * @param unixSeconds A trusted, developer-supplied SQL expression
+ */
+export declare function monthSql(unixSeconds: string): string;
+/** Rows of a `GROUP BY` month, as a survey's `byMonth`: a row without a month is left out. */
+export declare function countsByMonth(rows: Array<{
+    month: string | null;
+    n: number;
+}>): {
+    [month: string]: number;
+};
+//# sourceMappingURL=months.d.ts.map

package/dist/months.js

@@ -0,0 +1,16 @@
+/**
+ * A month bucket in SQL, `YYYY-MM` in local time, as a survey's `byMonth`
+ * keys it, from an expression in Unix seconds.
+ * @param unixSeconds A trusted, developer-supplied SQL expression
+ */
+export function monthSql(unixSeconds) {
+    return `strftime('%Y-%m', ${unixSeconds}, 'unixepoch', 'localtime')`;
+}
+/** Rows of a `GROUP BY` month, as a survey's `byMonth`: a row without a month is left out. */
+export function countsByMonth(rows) {
+    const byMonth = {};
+    for (const { month, n } of rows)
+        if (month)
+            byMonth[month] = n;
+    return byMonth;
+}

package/dist/snapshot.d.ts

@@ -0,0 +1,21 @@
+import { type DatabaseSchema } from '@chronicle.app/etl';
+/**
+ * Copy a database and its `-wal` into `dir`, to read it without writing
+ * beside it, waiting on an app's lock, or missing what's only in the
+ * `-wal` so far. The copy is a clone where the filesystem can make one, as
+ * APFS can, which is instant, and a full copy elsewhere, made off the event
+ * loop. A copy that caught a save is taken again.
+ */
+export declare function snapshot(database: string, dir: string, { attempts, retryMs }?: {
+    attempts?: number;
+    retryMs?: number;
+}): Promise<string>;
+/**
+ * Write an empty database with only a schema's tables and columns, for a
+ * test that a reader's queries need nothing it doesn't declare. Without
+ * `optional`, the columns it does without are left out too.
+ */
+export declare function writeSchemaDatabase(path: string, schema: DatabaseSchema, { optional }?: {
+    optional?: boolean;
+}): void;
+//# sourceMappingURL=snapshot.d.ts.map

package/dist/snapshot.js

@@ -0,0 +1,69 @@
+import { constants as fsConstants, rmSync, statSync } from 'node:fs';
+import { copyFile } from 'node:fs/promises';
+import { basename, join } from 'node:path';
+import { DatabaseSync } from 'node:sqlite';
+import { EXIT_CODES, ExtractorError, delay } from '@chronicle.app/etl';
+/** The files a database is saved in: itself, and its write-ahead log. */
+const SAVED = ['', '-wal'];
+/**
+ * Copy a database and its `-wal` into `dir`, to read it without writing
+ * beside it, waiting on an app's lock, or missing what's only in the
+ * `-wal` so far. The copy is a clone where the filesystem can make one, as
+ * APFS can, which is instant, and a full copy elsewhere, made off the event
+ * loop. A copy that caught a save is taken again.
+ */
+export async function snapshot(database, dir, { attempts = 3, retryMs = 200 } = {}) {
+    const copy = join(dir, basename(database));
+    for (let attempt = 1;; attempt++) {
+        const before = savedState(database);
+        for (const suffix of SAVED) {
+            rmSync(copy + suffix, { force: true });
+            try {
+                await copyFile(database + suffix, copy + suffix, fsConstants.COPYFILE_FICLONE);
+            }
+            catch (error) {
+                // A database can have no -wal, and one gone by the time its copy
+                // starts is a save, which the state after catches.
+                if (!suffix || error.code !== 'ENOENT')
+                    throw error;
+            }
+        }
+        if (before === savedState(database))
+            return copy;
+        if (attempt >= attempts) {
+            throw new ExtractorError('The database kept changing while it was copied', {
+                code: 'input-busy',
+                exitCode: EXIT_CODES.transient,
+                hint: 'Run it again in a moment.',
+                fields: { path: database },
+            });
+        }
+        await delay(retryMs);
+    }
+}
+/** The size and time each of a database's files was saved, to tell whether one changed. */
+function savedState(database) {
+    return SAVED.map(suffix => {
+        const file = statSync(database + suffix, { throwIfNoEntry: false });
+        return file ? `${file.size}:${file.mtimeMs}` : '-';
+    }).join(' ');
+}
+/**
+ * Write an empty database with only a schema's tables and columns, for a
+ * test that a reader's queries need nothing it doesn't declare. Without
+ * `optional`, the columns it does without are left out too.
+ */
+export function writeSchemaDatabase(path, schema, { optional = true } = {}) {
+    const db = new DatabaseSync(path);
+    try {
+        for (const [table, columns] of Object.entries(schema.tables)) {
+            const kept = columns
+                .filter(column => optional || !column.endsWith('?'))
+                .map(column => `"${column.replace(/\?$/, '')}"`);
+            db.exec(`CREATE TABLE "${table}" (${kept.join(', ')})`);
+        }
+    }
+    finally {
+        db.close();
+    }
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@chronicle.app/etl-sqlite",
-  "version": "0.4.0",
+  "version": "0.5.0",
   "main": "./dist/index.js",
   "types": "./dist/index.d.ts",
   "files": [
@@ -19,8 +19,8 @@
   "type": "module",
   "description": "Read-only SQLite extraction using built-in node:sqlite",
   "devDependencies": {
-    "@chronicle.app/etl": "0.4.0",
-    "@chronicle.app/tsconfig": "0.4.0",
+    "@chronicle.app/etl": "0.5.0",
+    "@chronicle.app/tsconfig": "0.5.0",
     "@types/node": "^22.10.1",
     "typescript": "^5.7.2"
   },