@chronicle.app/etl-sqlite 0.2.0 → 0.4.0

package/README.md

@@ -4,10 +4,46 @@ SQLite extraction with built-in `node:sqlite` on Node >=22.13.0.
 
 Extend `SqliteExtractor`, declare `source` and `strategy`, and implement `extract()` using the protected `db` connection. Pass an existing database path as `input`. Always call `teardown()` in a `finally` block, including after setup or extraction fails.
 
+`setup()` opens `input` with the protected `openDatabase(path)`. Call it from a subclass to read a different file, such as a temporary copy of a database the source app keeps exclusively locked, or a database extracted from a backup. Close and clear `this.db` before reopening; `teardown()` closes whichever connection is open.
+
+```ts
+override async setup(): Promise<void> {
+  await super.setup();
+  try {
+    this.db!.prepare('SELECT 1 FROM sqlite_schema').get();
+  } catch (err) {
+    if (!isSqliteBusy(err)) throw err;
+    this.db!.close();
+    this.db = null;
+    const copy = join(await mkdtemp(join(tmpdir(), 'source-')), 'copy.sqlite');
+    await copyFile(this.config.input, copy);
+    this.db = this.openDatabase(copy);
+  }
+}
+```
+
+`isSqliteBusy(err)` reports whether an error is SQLITE_BUSY (including extended busy codes). Opening a locked database succeeds; the first read fails.
+
+`allRows<T>(stmt, ...params)`, `getRow<T>(stmt, ...params)` and `iterateRows<T>(stmt, ...params)` wrap `all()`, `get()` and `iterate()` with a row type, so call sites need no `as unknown as T[]`. The type is not checked at runtime. `iterateRows` also keeps its statement referenced until iteration ends, so iterating `db.prepare(sql)` inline is safe. On Node 22.13, `iterate()` alone does not, and garbage collection mid-iteration throws "statement has been finalized".
+
 The connection is read-only: missing databases are not created, and journal mode is not changed. Live WAL databases require access to the database and its sidecar files; a copied database must include uncheckpointed WAL contents or be a consistent SQLite backup.
 
 `timeRangeConditions(column, range, options)` returns parameterized SQL conditions and values. The column is a trusted developer-supplied SQL identifier, not user input. Bounds are exclusive by default; conversion functions and inclusive operators are supported. Numeric dates are Unix milliseconds; the default SQL values are Unix seconds.
 
 Apple epoch helpers retain the existing API: iOS inputs/outputs use nanoseconds since 2001, Safari inputs/outputs use seconds since 2001. `iosToUnixTimestamp` returns Unix milliseconds; other Unix arguments/results use seconds.
 
+Chrome epoch helpers cover Chromium's microseconds since 1601. `chromeToUnixMsSql(column)` returns a SQL expression in Unix milliseconds, since current Chrome times are too large to read as numbers. `unixMsToChromeTimestamp(ms)` converts a bound back, as a `timeRangeConditions` `convert`.
+
 Node SQLite returns large INTEGER values only when `statement.setReadBigInts(true)` is enabled. Use that for source nanosecond timestamps and convert deliberately at the application boundary.
+
+## Porting from better-sqlite3
+
+`node:sqlite` differs from better-sqlite3 in ways that fail at runtime rather than at compile time. All of these hold on Node 22.13.0:
+
+- Parameters are spread, not passed as an array: `stmt.all(...values)`. `stmt.all(values)` throws `Unknown named parameter '0'`.
+- Booleans can't be bound (`ERR_INVALID_ARG_TYPE`). Bind `1` or `0`.
+- Reading an integer beyond `Number.MAX_SAFE_INTEGER` throws `ERR_OUT_OF_RANGE`. Call `stmt.setReadBigInts(true)`, or select the column with `CAST(column AS TEXT)`.
+- BLOBs come back as `Uint8Array`, not `Buffer`; `Buffer.isBuffer()` is false. Wrap with `Buffer.from(value.buffer, value.byteOffset, value.byteLength)` where a `Buffer` is needed.
+- Databases open read-only and their journal mode is never changed. Don't set `journal_mode` or other writing pragmas on source databases.
+
+There is no query builder. Write SQL with `?` placeholders, and build `IN (...)` lists and optional `WHERE` clauses from arrays of conditions and values, as `timeRangeConditions` does.

package/dist/SqliteExtractor.d.ts

@@ -23,6 +23,12 @@ export declare abstract class SqliteExtractor<SelfClass extends typeof SqliteExt
     }>;
     protected db: DatabaseSync | null;
     setup(): Promise<void>;
+    /**
+     * Open a database read-only without changing its journal mode. `setup()`
+     * opens `input` with this; subclasses can call it to read another file, such
+     * as a copy of a locked database, after closing and clearing `this.db`.
+     */
+    protected openDatabase(path: string): DatabaseSync;
     teardown(): Promise<void>;
 }
 //# sourceMappingURL=SqliteExtractor.d.ts.map

package/dist/SqliteExtractor.js

@@ -1,6 +1,6 @@
 import { DatabaseSync } from 'node:sqlite';
 import { z } from 'zod';
-import { Extractor } from '@chronicle.app/etl';
+import { Extractor, assertReadable } from '@chronicle.app/etl';
 /** Read an existing source database without changing its journal mode. */
 export class SqliteExtractor extends Extractor {
     static delivery = 'local';
@@ -13,7 +13,18 @@ export class SqliteExtractor extends Extractor {
             throw new Error('Database already initialized');
         await super.setup();
         const { input } = this.config;
-        this.db = new DatabaseSync(input, { readOnly: true });
+        this.db = this.openDatabase(input);
+    }
+    /**
+     * Open a database read-only without changing its journal mode. `setup()`
+     * opens `input` with this; subclasses can call it to read another file, such
+     * as a copy of a locked database, after closing and clearing `this.db`.
+     */
+    openDatabase(path) {
+        // SQLite says only "unable to open database file"; say whether the file
+        // is missing or macOS privacy controls refused it.
+        assertReadable(path, 'database');
+        return new DatabaseSync(path, { readOnly: true });
     }
     async teardown() {
         if (this.db) {

package/dist/chromeEpoch.d.ts

@@ -0,0 +1,28 @@
+/**
+ * Chrome epoch timestamp utilities.
+ *
+ * Chromium stores times (History `visits.visit_time`, `urls.last_visit_time`)
+ * as microseconds since January 1, 1601 00:00:00 UTC, the Windows FILETIME
+ * epoch. Unix timestamps count from January 1, 1970 00:00:00 UTC. The
+ * difference is 11,644,473,600 seconds.
+ *
+ * Current Chrome times exceed Number.MAX_SAFE_INTEGER, so `node:sqlite` throws
+ * ERR_OUT_OF_RANGE when reading them as numbers. Convert in SQL with
+ * `chromeToUnixMsSql(column)`, or read with `statement.setReadBigInts(true)`.
+ */
+export declare const CHROME_EPOCH_OFFSET_MS = 11644473600000;
+/**
+ * A SQL expression converting a Chrome time column to Unix milliseconds
+ * (integer division, so sub-millisecond precision is dropped).
+ * @param column A trusted, developer-supplied column name
+ */
+export declare function chromeToUnixMsSql(column: string): string;
+/**
+ * Convert Unix milliseconds to a Chrome timestamp, for binding against a Chrome
+ * time column. The result is exact until the year 3800 although it exceeds
+ * Number.MAX_SAFE_INTEGER: it is a multiple of 1000, and so of 8, below 2^56.
+ * @param unixMs Unix timestamp in milliseconds
+ * @returns Chrome timestamp in microseconds since Jan 1, 1601
+ */
+export declare function unixMsToChromeTimestamp(unixMs: number): number;
+//# sourceMappingURL=chromeEpoch.d.ts.map

package/dist/chromeEpoch.js

@@ -0,0 +1,31 @@
+/**
+ * Chrome epoch timestamp utilities.
+ *
+ * Chromium stores times (History `visits.visit_time`, `urls.last_visit_time`)
+ * as microseconds since January 1, 1601 00:00:00 UTC, the Windows FILETIME
+ * epoch. Unix timestamps count from January 1, 1970 00:00:00 UTC. The
+ * difference is 11,644,473,600 seconds.
+ *
+ * Current Chrome times exceed Number.MAX_SAFE_INTEGER, so `node:sqlite` throws
+ * ERR_OUT_OF_RANGE when reading them as numbers. Convert in SQL with
+ * `chromeToUnixMsSql(column)`, or read with `statement.setReadBigInts(true)`.
+ */
+export const CHROME_EPOCH_OFFSET_MS = 11_644_473_600_000;
+/**
+ * A SQL expression converting a Chrome time column to Unix milliseconds
+ * (integer division, so sub-millisecond precision is dropped).
+ * @param column A trusted, developer-supplied column name
+ */
+export function chromeToUnixMsSql(column) {
+    return `(${column} / 1000 - ${CHROME_EPOCH_OFFSET_MS})`;
+}
+/**
+ * Convert Unix milliseconds to a Chrome timestamp, for binding against a Chrome
+ * time column. The result is exact until the year 3800 although it exceeds
+ * Number.MAX_SAFE_INTEGER: it is a multiple of 1000, and so of 8, below 2^56.
+ * @param unixMs Unix timestamp in milliseconds
+ * @returns Chrome timestamp in microseconds since Jan 1, 1601
+ */
+export function unixMsToChromeTimestamp(unixMs) {
+    return (Math.floor(unixMs) + CHROME_EPOCH_OFFSET_MS) * 1000;
+}

package/dist/index.d.ts

@@ -1,4 +1,6 @@
 export * from './SqliteExtractor.js';
+export * from './statements.js';
 export * from './timeRange.js';
 export * from './appleEpoch.js';
+export * from './chromeEpoch.js';
 //# sourceMappingURL=index.d.ts.map

package/dist/index.js

@@ -1,3 +1,5 @@
 export * from './SqliteExtractor.js';
+export * from './statements.js';
 export * from './timeRange.js';
 export * from './appleEpoch.js';
+export * from './chromeEpoch.js';

package/dist/statements.d.ts

@@ -0,0 +1,29 @@
+import type { SQLInputValue, StatementSync } from 'node:sqlite';
+type Parameters = SQLInputValue[] | [namedParameters: Record<string, SQLInputValue>, ...anonymousParameters: SQLInputValue[]];
+/**
+ * Run a statement and type its rows. `node:sqlite` returns
+ * `Record<string, SQLOutputValue>`; `T` is the caller's description of the
+ * selected columns and is not checked at runtime.
+ */
+export declare function allRows<T>(statement: StatementSync, ...params: Parameters): T[];
+/** Run a statement and type its first row, or `undefined` when there is none. */
+export declare function getRow<T>(statement: StatementSync, ...params: Parameters): T | undefined;
+/**
+ * Run a statement and iterate its rows one at a time, typed as `T`.
+ *
+ * The generator keeps `statement` referenced until iteration ends. On Node
+ * 22.13, the iterator from `iterate()` does not, so a statement prepared inline
+ * (`iterateRows(db.prepare(sql))`) can be garbage collected mid-iteration, and
+ * the next row throws "statement has been finalized".
+ */
+export declare function iterateRows<T>(statement: StatementSync, ...params: Parameters): Generator<T>;
+/**
+ * Whether an error is SQLITE_BUSY, which `node:sqlite` reports as
+ * `{ code: 'ERR_SQLITE_ERROR', errcode: 5 }` when another connection holds a
+ * lock (for example an app that keeps its database exclusively locked).
+ * Extended result codes (primary code in the low byte), such as
+ * SQLITE_BUSY_SNAPSHOT, also match.
+ */
+export declare function isSqliteBusy(err: unknown): boolean;
+export {};
+//# sourceMappingURL=statements.d.ts.map

package/dist/statements.js

@@ -0,0 +1,37 @@
+/**
+ * Run a statement and type its rows. `node:sqlite` returns
+ * `Record<string, SQLOutputValue>`; `T` is the caller's description of the
+ * selected columns and is not checked at runtime.
+ */
+export function allRows(statement, ...params) {
+    return statement.all(...params);
+}
+/** Run a statement and type its first row, or `undefined` when there is none. */
+export function getRow(statement, ...params) {
+    return statement.get(...params);
+}
+/**
+ * Run a statement and iterate its rows one at a time, typed as `T`.
+ *
+ * The generator keeps `statement` referenced until iteration ends. On Node
+ * 22.13, the iterator from `iterate()` does not, so a statement prepared inline
+ * (`iterateRows(db.prepare(sql))`) can be garbage collected mid-iteration, and
+ * the next row throws "statement has been finalized".
+ */
+export function* iterateRows(statement, ...params) {
+    yield* statement.iterate(...params);
+}
+const SQLITE_BUSY = 5;
+/**
+ * Whether an error is SQLITE_BUSY, which `node:sqlite` reports as
+ * `{ code: 'ERR_SQLITE_ERROR', errcode: 5 }` when another connection holds a
+ * lock (for example an app that keeps its database exclusively locked).
+ * Extended result codes (primary code in the low byte), such as
+ * SQLITE_BUSY_SNAPSHOT, also match.
+ */
+export function isSqliteBusy(err) {
+    if (typeof err !== 'object' || err === null)
+        return false;
+    const { code, errcode } = err;
+    return (code === 'ERR_SQLITE_ERROR' && typeof errcode === 'number' && errcode % 256 === SQLITE_BUSY);
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@chronicle.app/etl-sqlite",
-  "version": "0.2.0",
+  "version": "0.4.0",
   "main": "./dist/index.js",
   "types": "./dist/index.d.ts",
   "files": [
@@ -19,8 +19,8 @@
   "type": "module",
   "description": "Read-only SQLite extraction using built-in node:sqlite",
   "devDependencies": {
-    "@chronicle.app/etl": "0.2.0",
-    "@chronicle.app/tsconfig": "0.2.0",
+    "@chronicle.app/etl": "0.4.0",
+    "@chronicle.app/tsconfig": "0.4.0",
     "@types/node": "^22.10.1",
     "typescript": "^5.7.2"
   },