@chronicle.app/etl-sqlite 0.3.0 → 0.5.0

package/README.md

@@ -32,6 +32,8 @@ The connection is read-only: missing databases are not created, and journal mode
 
 Apple epoch helpers retain the existing API: iOS inputs/outputs use nanoseconds since 2001, Safari inputs/outputs use seconds since 2001. `iosToUnixTimestamp` returns Unix milliseconds; other Unix arguments/results use seconds.
 
+Chrome epoch helpers cover Chromium's microseconds since 1601. `chromeToUnixMsSql(column)` returns a SQL expression in Unix milliseconds, since current Chrome times are too large to read as numbers. `unixMsToChromeTimestamp(ms)` converts a bound back, as a `timeRangeConditions` `convert`.
+
 Node SQLite returns large INTEGER values only when `statement.setReadBigInts(true)` is enabled. Use that for source nanosecond timestamps and convert deliberately at the application boundary.
 
 ## Porting from better-sqlite3

package/dist/SqliteExtractor.d.ts

@@ -1,6 +1,13 @@
 import { DatabaseSync } from 'node:sqlite';
 import { z } from 'zod';
-import { type Delivery, Extractor } from '@chronicle.app/etl';
+import { type DatabaseColumns, type DatabaseSchema, type Delivery, Extractor } from '@chronicle.app/etl';
+/**
+ * Refuse an open database unless it has every column one of `databases`
+ * needs, naming `path` as what was given. Its columns, when it has them. For
+ * a reader that opens its database itself rather than through a
+ * {@link SqliteExtractor}.
+ */
+export declare function assertDatabaseSchema(db: DatabaseSync, databases: DatabaseSchema[], path: string): DatabaseColumns;
 /** Read an existing source database without changing its journal mode. */
 export declare abstract class SqliteExtractor<SelfClass extends typeof SqliteExtractor = typeof SqliteExtractor> extends Extractor<SelfClass> {
     static delivery: Delivery;
@@ -21,8 +28,27 @@ export declare abstract class SqliteExtractor<SelfClass extends typeof SqliteExt
         until?: Date | undefined;
         limit?: number | undefined;
     }>;
+    /**
+     * The tables a reader reads and the columns its queries use, which its
+     * manifest's `databases` repeats for `scan`. A reader that declares them
+     * reads a snapshot of its input, and only an input with every column a
+     * schema needs.
+     */
+    static databases?: DatabaseSchema[];
     protected db: DatabaseSync | null;
+    /** The checked database's columns, for {@link has}. */
+    private columns;
+    private snapshotDir;
     setup(): Promise<void>;
+    /**
+     * Refuse the open database unless it has every column one of `databases`
+     * needs, naming `path` as what was given. `setup()` checks a declared
+     * schema; a reader that declares none, as one whose input is a copy
+     * already, can check one after it.
+     */
+    protected assertSchema(databases: DatabaseSchema[], path: string): void;
+    /** Whether the database has a column a schema marks as optional, as `table.column`. */
+    protected has(column: string): boolean;
     /**
      * Open a database read-only without changing its journal mode. `setup()`
      * opens `input` with this; subclasses can call it to read another file, such

package/dist/SqliteExtractor.js

@@ -1,19 +1,80 @@
+import { rmSync } from 'node:fs';
+import { mkdtemp } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
 import { DatabaseSync } from 'node:sqlite';
 import { z } from 'zod';
-import { Extractor } from '@chronicle.app/etl';
+import { Extractor, InputNotFound, assertReadable, columnsOf, matchSchema, } from '@chronicle.app/etl';
+import { snapshot } from './snapshot.js';
+/**
+ * Refuse an open database unless it has every column one of `databases`
+ * needs, naming `path` as what was given. Its columns, when it has them. For
+ * a reader that opens its database itself rather than through a
+ * {@link SqliteExtractor}.
+ */
+export function assertDatabaseSchema(db, databases, path) {
+    const columns = columnsOf(db);
+    const match = matchSchema(columns, databases);
+    if (!match || match.missing.length > 0 || match.schema.supported === false) {
+        throw new InputNotFound(match ? 'That’s a version of the database this can’t read' : 'That isn’t the database', {
+            path,
+            ...(match && { fields: { missing: match.missing } }),
+            hint: 'Run `chronicle scan <path>` to see what it is.',
+        });
+    }
+    return columns;
+}
 /** Read an existing source database without changing its journal mode. */
 export class SqliteExtractor extends Extractor {
     static delivery = 'local';
     static schema = Extractor.schema.extend({
         input: z.string().min(1).describe('Path to an existing SQLite database'),
     });
+    /**
+     * The tables a reader reads and the columns its queries use, which its
+     * manifest's `databases` repeats for `scan`. A reader that declares them
+     * reads a snapshot of its input, and only an input with every column a
+     * schema needs.
+     */
+    static databases;
     db = null;
+    /** The checked database's columns, for {@link has}. */
+    columns = null;
+    snapshotDir = null;
     async setup() {
         if (this.db)
             throw new Error('Database already initialized');
         await super.setup();
         const { input } = this.config;
-        this.db = this.openDatabase(input);
+        const { databases } = this.constructor;
+        if (!databases) {
+            this.db = this.openDatabase(input);
+            return;
+        }
+        assertReadable(input, 'database');
+        this.snapshotDir = await mkdtemp(join(tmpdir(), 'chronicle-sqlite-'));
+        try {
+            this.db = this.openDatabase(await snapshot(input, this.snapshotDir));
+            this.assertSchema(databases, input);
+        }
+        catch (error) {
+            await this.teardown();
+            throw error;
+        }
+    }
+    /**
+     * Refuse the open database unless it has every column one of `databases`
+     * needs, naming `path` as what was given. `setup()` checks a declared
+     * schema; a reader that declares none, as one whose input is a copy
+     * already, can check one after it.
+     */
+    assertSchema(databases, path) {
+        this.columns = assertDatabaseSchema(this.db, databases, path);
+    }
+    /** Whether the database has a column a schema marks as optional, as `table.column`. */
+    has(column) {
+        const [table, name] = column.toLowerCase().split('.');
+        return this.columns?.get(table)?.has(name) ?? false;
     }
     /**
      * Open a database read-only without changing its journal mode. `setup()`
@@ -21,6 +82,9 @@ export class SqliteExtractor extends Extractor {
      * as a copy of a locked database, after closing and clearing `this.db`.
      */
     openDatabase(path) {
+        // SQLite says only "unable to open database file"; say whether the file
+        // is missing or macOS privacy controls refused it.
+        assertReadable(path, 'database');
         return new DatabaseSync(path, { readOnly: true });
     }
     async teardown() {
@@ -28,6 +92,11 @@ export class SqliteExtractor extends Extractor {
             this.db.close();
             this.db = null;
         }
+        if (this.snapshotDir) {
+            rmSync(this.snapshotDir, { recursive: true, force: true });
+            this.snapshotDir = null;
+        }
+        this.columns = null;
         await super.teardown();
     }
 }

package/dist/chromeEpoch.d.ts

@@ -0,0 +1,28 @@
+/**
+ * Chrome epoch timestamp utilities.
+ *
+ * Chromium stores times (History `visits.visit_time`, `urls.last_visit_time`)
+ * as microseconds since January 1, 1601 00:00:00 UTC, the Windows FILETIME
+ * epoch. Unix timestamps count from January 1, 1970 00:00:00 UTC. The
+ * difference is 11,644,473,600 seconds.
+ *
+ * Current Chrome times exceed Number.MAX_SAFE_INTEGER, so `node:sqlite` throws
+ * ERR_OUT_OF_RANGE when reading them as numbers. Convert in SQL with
+ * `chromeToUnixMsSql(column)`, or read with `statement.setReadBigInts(true)`.
+ */
+export declare const CHROME_EPOCH_OFFSET_MS = 11644473600000;
+/**
+ * A SQL expression converting a Chrome time column to Unix milliseconds
+ * (integer division, so sub-millisecond precision is dropped).
+ * @param column A trusted, developer-supplied column name
+ */
+export declare function chromeToUnixMsSql(column: string): string;
+/**
+ * Convert Unix milliseconds to a Chrome timestamp, for binding against a Chrome
+ * time column. The result is exact until the year 3800 although it exceeds
+ * Number.MAX_SAFE_INTEGER: it is a multiple of 1000, and so of 8, below 2^56.
+ * @param unixMs Unix timestamp in milliseconds
+ * @returns Chrome timestamp in microseconds since Jan 1, 1601
+ */
+export declare function unixMsToChromeTimestamp(unixMs: number): number;
+//# sourceMappingURL=chromeEpoch.d.ts.map

package/dist/chromeEpoch.js

@@ -0,0 +1,31 @@
+/**
+ * Chrome epoch timestamp utilities.
+ *
+ * Chromium stores times (History `visits.visit_time`, `urls.last_visit_time`)
+ * as microseconds since January 1, 1601 00:00:00 UTC, the Windows FILETIME
+ * epoch. Unix timestamps count from January 1, 1970 00:00:00 UTC. The
+ * difference is 11,644,473,600 seconds.
+ *
+ * Current Chrome times exceed Number.MAX_SAFE_INTEGER, so `node:sqlite` throws
+ * ERR_OUT_OF_RANGE when reading them as numbers. Convert in SQL with
+ * `chromeToUnixMsSql(column)`, or read with `statement.setReadBigInts(true)`.
+ */
+export const CHROME_EPOCH_OFFSET_MS = 11_644_473_600_000;
+/**
+ * A SQL expression converting a Chrome time column to Unix milliseconds
+ * (integer division, so sub-millisecond precision is dropped).
+ * @param column A trusted, developer-supplied column name
+ */
+export function chromeToUnixMsSql(column) {
+    return `(${column} / 1000 - ${CHROME_EPOCH_OFFSET_MS})`;
+}
+/**
+ * Convert Unix milliseconds to a Chrome timestamp, for binding against a Chrome
+ * time column. The result is exact until the year 3800 although it exceeds
+ * Number.MAX_SAFE_INTEGER: it is a multiple of 1000, and so of 8, below 2^56.
+ * @param unixMs Unix timestamp in milliseconds
+ * @returns Chrome timestamp in microseconds since Jan 1, 1601
+ */
+export function unixMsToChromeTimestamp(unixMs) {
+    return (Math.floor(unixMs) + CHROME_EPOCH_OFFSET_MS) * 1000;
+}

package/dist/index.d.ts

@@ -1,5 +1,8 @@
 export * from './SqliteExtractor.js';
+export * from './snapshot.js';
 export * from './statements.js';
 export * from './timeRange.js';
 export * from './appleEpoch.js';
+export * from './chromeEpoch.js';
+export * from './months.js';
 //# sourceMappingURL=index.d.ts.map

package/dist/index.js

@@ -1,4 +1,7 @@
 export * from './SqliteExtractor.js';
+export * from './snapshot.js';
 export * from './statements.js';
 export * from './timeRange.js';
 export * from './appleEpoch.js';
+export * from './chromeEpoch.js';
+export * from './months.js';

package/dist/months.d.ts

@@ -0,0 +1,14 @@
+/**
+ * A month bucket in SQL, `YYYY-MM` in local time, as a survey's `byMonth`
+ * keys it, from an expression in Unix seconds.
+ * @param unixSeconds A trusted, developer-supplied SQL expression
+ */
+export declare function monthSql(unixSeconds: string): string;
+/** Rows of a `GROUP BY` month, as a survey's `byMonth`: a row without a month is left out. */
+export declare function countsByMonth(rows: Array<{
+    month: string | null;
+    n: number;
+}>): {
+    [month: string]: number;
+};
+//# sourceMappingURL=months.d.ts.map

package/dist/months.js

@@ -0,0 +1,16 @@
+/**
+ * A month bucket in SQL, `YYYY-MM` in local time, as a survey's `byMonth`
+ * keys it, from an expression in Unix seconds.
+ * @param unixSeconds A trusted, developer-supplied SQL expression
+ */
+export function monthSql(unixSeconds) {
+    return `strftime('%Y-%m', ${unixSeconds}, 'unixepoch', 'localtime')`;
+}
+/** Rows of a `GROUP BY` month, as a survey's `byMonth`: a row without a month is left out. */
+export function countsByMonth(rows) {
+    const byMonth = {};
+    for (const { month, n } of rows)
+        if (month)
+            byMonth[month] = n;
+    return byMonth;
+}

package/dist/snapshot.d.ts

@@ -0,0 +1,21 @@
+import { type DatabaseSchema } from '@chronicle.app/etl';
+/**
+ * Copy a database and its `-wal` into `dir`, to read it without writing
+ * beside it, waiting on an app's lock, or missing what's only in the
+ * `-wal` so far. The copy is a clone where the filesystem can make one, as
+ * APFS can, which is instant, and a full copy elsewhere, made off the event
+ * loop. A copy that caught a save is taken again.
+ */
+export declare function snapshot(database: string, dir: string, { attempts, retryMs }?: {
+    attempts?: number;
+    retryMs?: number;
+}): Promise<string>;
+/**
+ * Write an empty database with only a schema's tables and columns, for a
+ * test that a reader's queries need nothing it doesn't declare. Without
+ * `optional`, the columns it does without are left out too.
+ */
+export declare function writeSchemaDatabase(path: string, schema: DatabaseSchema, { optional }?: {
+    optional?: boolean;
+}): void;
+//# sourceMappingURL=snapshot.d.ts.map

package/dist/snapshot.js

@@ -0,0 +1,69 @@
+import { constants as fsConstants, rmSync, statSync } from 'node:fs';
+import { copyFile } from 'node:fs/promises';
+import { basename, join } from 'node:path';
+import { DatabaseSync } from 'node:sqlite';
+import { EXIT_CODES, ExtractorError, delay } from '@chronicle.app/etl';
+/** The files a database is saved in: itself, and its write-ahead log. */
+const SAVED = ['', '-wal'];
+/**
+ * Copy a database and its `-wal` into `dir`, to read it without writing
+ * beside it, waiting on an app's lock, or missing what's only in the
+ * `-wal` so far. The copy is a clone where the filesystem can make one, as
+ * APFS can, which is instant, and a full copy elsewhere, made off the event
+ * loop. A copy that caught a save is taken again.
+ */
+export async function snapshot(database, dir, { attempts = 3, retryMs = 200 } = {}) {
+    const copy = join(dir, basename(database));
+    for (let attempt = 1;; attempt++) {
+        const before = savedState(database);
+        for (const suffix of SAVED) {
+            rmSync(copy + suffix, { force: true });
+            try {
+                await copyFile(database + suffix, copy + suffix, fsConstants.COPYFILE_FICLONE);
+            }
+            catch (error) {
+                // A database can have no -wal, and one gone by the time its copy
+                // starts is a save, which the state after catches.
+                if (!suffix || error.code !== 'ENOENT')
+                    throw error;
+            }
+        }
+        if (before === savedState(database))
+            return copy;
+        if (attempt >= attempts) {
+            throw new ExtractorError('The database kept changing while it was copied', {
+                code: 'input-busy',
+                exitCode: EXIT_CODES.transient,
+                hint: 'Run it again in a moment.',
+                fields: { path: database },
+            });
+        }
+        await delay(retryMs);
+    }
+}
+/** The size and time each of a database's files was saved, to tell whether one changed. */
+function savedState(database) {
+    return SAVED.map(suffix => {
+        const file = statSync(database + suffix, { throwIfNoEntry: false });
+        return file ? `${file.size}:${file.mtimeMs}` : '-';
+    }).join(' ');
+}
+/**
+ * Write an empty database with only a schema's tables and columns, for a
+ * test that a reader's queries need nothing it doesn't declare. Without
+ * `optional`, the columns it does without are left out too.
+ */
+export function writeSchemaDatabase(path, schema, { optional = true } = {}) {
+    const db = new DatabaseSync(path);
+    try {
+        for (const [table, columns] of Object.entries(schema.tables)) {
+            const kept = columns
+                .filter(column => optional || !column.endsWith('?'))
+                .map(column => `"${column.replace(/\?$/, '')}"`);
+            db.exec(`CREATE TABLE "${table}" (${kept.join(', ')})`);
+        }
+    }
+    finally {
+        db.close();
+    }
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@chronicle.app/etl-sqlite",
-  "version": "0.3.0",
+  "version": "0.5.0",
   "main": "./dist/index.js",
   "types": "./dist/index.d.ts",
   "files": [
@@ -19,8 +19,8 @@
   "type": "module",
   "description": "Read-only SQLite extraction using built-in node:sqlite",
   "devDependencies": {
-    "@chronicle.app/etl": "0.3.0",
-    "@chronicle.app/tsconfig": "0.3.0",
+    "@chronicle.app/etl": "0.5.0",
+    "@chronicle.app/tsconfig": "0.5.0",
     "@types/node": "^22.10.1",
     "typescript": "^5.7.2"
   },