@innovatorstack/harness 0.0.0-stage → 0.1.0

package/dist/storage/sqlite/storage.js

@@ -0,0 +1,644 @@
+import { Database } from "bun:sqlite";
+import { renameSync, rmSync, writeFileSync } from "node:fs";
+import { resolve } from "node:path";
+import { assertDataTree, getOwn } from "../../core/data.js";
+import { setupBytesMax } from "../../core/limits.js";
+import { selectQuery } from "../../queries/types.js";
+import { selectDocument } from "../documents.js";
+import { changeMetadata, metadataEpoch, trackMetadata } from "../metadata-epoch.js";
+import { packageTransaction } from "../package-transaction.js";
+import { CatalogEntries } from "./catalog.js";
+import { takeConnection, retainConnection } from "./connections.js";
+import { StorageError } from "./errors.js";
+import { decodeDefinition, decodePresentProperties } from "./metadata.js";
+import { SqliteQueries } from "./queries.js";
+import { assertCompatible, columnsFor, columnFor, normalizeValue, quoteIdentifier, tableName, validateDefinition, } from "./schema.js";
+import { SqliteStatements } from "./statements.js";
+import { SqliteTransactions, setCheckpoint, restoreCheckpoint, retainCheckpoint, } from "./transactions.js";
+const schemaVersion = 1;
+const applicationId = 0x49_48_4e_53;
+function openDatabase(filename) {
+    const database = new Database(filename, { create: true, strict: true });
+    try {
+        database.run("PRAGMA foreign_keys = ON");
+        database.run("PRAGMA journal_mode = DELETE");
+    }
+    catch (error) {
+        database.close();
+        throw error;
+    }
+    return database;
+}
+export class SqliteStorage {
+    catalog = new CatalogEntries();
+    boundary = new SqliteTransactions(() => this.database);
+    statements = new SqliteStatements(() => this.database);
+    selections = new SqliteQueries(this, () => this.database, (definition, row) => this.decodeItem(definition, row));
+    [selectQuery](type, predicate, spend) {
+        return this.selections.select(type, predicate, spend);
+    }
+    transactions = Object.freeze({
+        begin: () => {
+            this.boundary.begin();
+            this.transactionMetadata = metadataEpoch(this);
+            if (this.transactionMetadata === undefined) {
+                this.definitions.clear();
+            }
+        },
+        commit: () => {
+            this.boundary.commit();
+        },
+        rollback: () => {
+            this.boundary.rollback();
+            if (this.transactionMetadata === undefined ||
+                this.transactionMetadata !== metadataEpoch(this)) {
+                this.clearStatements();
+            }
+            this.definitions.clear();
+            changeMetadata(this);
+        },
+    });
+    connection;
+    closed = false;
+    packageWriteActive = false;
+    transactionMetadata;
+    filename;
+    reusableImage;
+    definitionList;
+    definitions = new Map();
+    constructor(filename = ":memory:") {
+        this.filename = filename === ":memory:" ? filename : resolve(filename);
+        this.connection = openDatabase(this.filename);
+        try {
+            this.initialize();
+            if (this.filename === ":memory:") {
+                trackMetadata(this);
+            }
+        }
+        catch (error) {
+            this.connection.close();
+            throw error;
+        }
+    }
+    defineItemType(definition) {
+        assertStorageData(definition, "INVALID_SCHEMA");
+        validateDefinition(definition);
+        const database = this.database;
+        if (!database.inTransaction) {
+            this.definitions.clear();
+        }
+        this.writeAtomically(() => {
+            const previous = this.getItemType(definition.name);
+            if (previous &&
+                this.definitions.get(definition.name.toLowerCase())?.encoded ===
+                    JSON.stringify(definition)) {
+                return;
+            }
+            if (previous) {
+                assertCompatible(previous, definition);
+            }
+            this.assertUniqueTypeId(definition);
+            this.defineTable(definition, previous);
+            database
+                .query("INSERT INTO _harness_item_types (id, name, definition) VALUES (?, ?, ?) ON CONFLICT(name) DO UPDATE SET id = excluded.id, definition = excluded.definition")
+                .run(definition.id, definition.name, JSON.stringify(definition));
+            this.definitions.delete(definition.name.toLowerCase());
+            changeMetadata(this);
+        });
+    }
+    getItemType(name) {
+        // An active transaction has one metadata view; schema writes invalidate their entry.
+        // Outside it, read SQLite so other connections and direct storage callers remain visible.
+        const cached = this.definitions.get(name.toLowerCase());
+        if ((this.database.inTransaction || metadataEpoch(this) !== undefined) && cached) {
+            return cached.value;
+        }
+        const row = this.database
+            .query("SELECT name, definition FROM _harness_item_types WHERE name = ? COLLATE NOCASE")
+            .get(name);
+        if (row) {
+            return this.readDefinition(row);
+        }
+        return undefined;
+    }
+    listItemTypes() {
+        const database = this.database;
+        const epoch = metadataEpoch(this);
+        if (epoch !== undefined && this.definitionList?.epoch === epoch) {
+            return this.definitionList.values;
+        }
+        const values = Object.freeze(database
+            .query("SELECT name, definition FROM _harness_item_types ORDER BY name COLLATE BINARY")
+            .all()
+            .map((row) => this.readDefinition(row)));
+        if (epoch !== undefined) {
+            this.definitionList = { epoch, values };
+        }
+        return values;
+    }
+    insertItem(item) {
+        const definition = this.requireItemType(item.type);
+        this.validateId(item.id);
+        const properties = this.normalizeProperties(definition, item.properties);
+        this.writeAtomically(() => {
+            const names = ["id", ...Object.keys(properties)];
+            const values = [item.id, ...Object.values(properties)];
+            const result = this.statements
+                .query(`INSERT INTO ${quoteIdentifier(tableName(definition))} ` +
+                `(${names.map(quoteIdentifier).join(", ")}) ` +
+                `VALUES (${names.map(() => "?").join(", ")}) ` +
+                "ON CONFLICT(id) DO NOTHING")
+                .run(...values);
+            if (result.changes === 0) {
+                throw new StorageError("ITEM_EXISTS", `${definition.name} item ${item.id} already exists.`);
+            }
+            this.writePresence(definition.name, item.id, Object.keys(properties));
+        });
+    }
+    updateItem(type, id, values) {
+        const definition = this.requireItemType(type);
+        const properties = this.normalizeProperties(definition, values);
+        this.writeAtomically(() => {
+            const existing = this.getItem(definition.name, id);
+            if (!existing) {
+                throw new StorageError("ITEM_NOT_FOUND", `${definition.name} item ${id} does not exist.`);
+            }
+            const entries = Object.entries(properties);
+            if (!entries.length) {
+                return;
+            }
+            this.statements
+                .query(`UPDATE ${quoteIdentifier(tableName(definition))} SET ${entries
+                .map(([name]) => `${quoteIdentifier(name)} = ?`)
+                .join(", ")} WHERE id = ?`)
+                .run(...entries.map(([, value]) => value), id);
+            this.writePresence(definition.name, existing.id, [
+                ...new Set([...Object.keys(existing.properties), ...Object.keys(properties)]),
+            ]);
+        });
+    }
+    getItem(type, id) {
+        const definition = this.requireItemType(type);
+        const row = this.statements
+            .query(`SELECT r.*, p.properties AS _harness_presence FROM ${quoteIdentifier(tableName(definition))} AS r ` +
+            "LEFT JOIN _harness_presence AS p ON p.type = ? COLLATE NOCASE AND p.id = r.id COLLATE NOCASE WHERE r.id = ?")
+            .get(definition.name, id);
+        if (row) {
+            return this.decodeItem(definition, row);
+        }
+        return undefined;
+    }
+    listItems(type) {
+        const definition = this.requireItemType(type);
+        return this.statements
+            .query(`SELECT r.*, p.properties AS _harness_presence FROM ${quoteIdentifier(tableName(definition))} AS r ` +
+            "LEFT JOIN _harness_presence AS p ON p.type = ? COLLATE NOCASE AND p.id = r.id COLLATE NOCASE ORDER BY r.id COLLATE BINARY")
+            .all(definition.name)
+            .map((row) => this.decodeItem(definition, row));
+    }
+    deleteItem(type, id) {
+        const definition = this.requireItemType(type);
+        const database = this.database;
+        this.writeAtomically(() => {
+            if (!this.getItem(type, id)) {
+                throw new StorageError("ITEM_NOT_FOUND", `${definition.name} item ${id} does not exist.`);
+            }
+            this.statements
+                .query(`DELETE FROM ${quoteIdentifier(tableName(definition))} WHERE id = ?`)
+                .run(id);
+            database
+                .query("DELETE FROM _harness_presence WHERE type = ? COLLATE NOCASE AND id = ? COLLATE NOCASE")
+                .run(definition.name, id);
+        });
+    }
+    putCatalogEntry(entry) {
+        changeMetadata(this);
+        assertStorageData(entry, "INVALID_ITEM");
+        this.validateId(entry.id);
+        this.database
+            .query("INSERT INTO _harness_catalog (type, id, properties, attributes, source) VALUES (?, ?, ?, ?, ?) ON CONFLICT(type, id) DO UPDATE SET properties = excluded.properties, attributes = excluded.attributes, source = excluded.source")
+            .run(entry.type, entry.id, JSON.stringify(entry.properties), JSON.stringify(entry.attributes), JSON.stringify(entry.source));
+    }
+    getCatalogEntry(type, id) {
+        const row = this.database
+            .query("SELECT * FROM _harness_catalog WHERE type = ? COLLATE NOCASE AND id = ? COLLATE NOCASE")
+            .get(type, id);
+        if (row) {
+            return this.catalog.read(row, metadataEpoch(this));
+        }
+        return undefined;
+    }
+    listCatalogEntries(type) {
+        const database = this.database;
+        return this.catalog.select(type, metadataEpoch(this), () => type === undefined
+            ? database
+                .query("SELECT * FROM _harness_catalog ORDER BY type COLLATE BINARY, id COLLATE BINARY")
+                .all()
+            : database
+                .query("SELECT * FROM _harness_catalog WHERE type = ? COLLATE NOCASE ORDER BY id COLLATE BINARY")
+                .all(type));
+    }
+    deleteCatalogEntry(type, id) {
+        changeMetadata(this);
+        this.database
+            .query("DELETE FROM _harness_catalog WHERE type = ? COLLATE NOCASE AND id = ? COLLATE NOCASE")
+            .run(type, id);
+    }
+    putDocument(document) {
+        changeMetadata(this);
+        this.database
+            .query("INSERT INTO _harness_documents (path, sha256, xml) VALUES (?, ?, ?) ON CONFLICT(path) DO UPDATE SET sha256 = excluded.sha256, xml = excluded.xml")
+            .run(document.path, document.sha256, document.xml);
+    }
+    listDocuments() {
+        return this.database
+            .query("SELECT path, sha256, xml FROM _harness_documents ORDER BY path COLLATE BINARY")
+            .all();
+    }
+    [selectDocument](path) {
+        return (this.database
+            .query("SELECT path, sha256, xml FROM _harness_documents WHERE path = ?")
+            .get(path) ?? undefined);
+    }
+    snapshot() {
+        this.requireNoTransaction("snapshot");
+        this.boundary.flush();
+        const pages = this.database
+            .query("PRAGMA page_count")
+            .get()?.page_count;
+        const pageSize = this.database
+            .query("PRAGMA page_size")
+            .get()?.page_size;
+        if (pages === undefined || pageSize === undefined || pages * pageSize > setupBytesMax) {
+            throw new StorageError("INVALID_SNAPSHOT", "The database exceeds its snapshot byte limit.");
+        }
+        return new Uint8Array(this.database.serialize());
+    }
+    [setCheckpoint](image) {
+        if (this.filename === ":memory:") {
+            this.boundary.mark(image);
+        }
+    }
+    [restoreCheckpoint](image) {
+        if (this.filename !== ":memory:") {
+            return false;
+        }
+        this.statements.clear();
+        if (!this.boundary.restore(image)) {
+            return false;
+        }
+        this.definitions.clear();
+        this.definitionList = undefined;
+        this.catalog.clear();
+        changeMetadata(this);
+        return true;
+    }
+    [retainCheckpoint](image) {
+        if (this.filename === ":memory:" && this.boundary.owns(image)) {
+            this.reusableImage = image;
+        }
+    }
+    restore(snapshot) {
+        this.requireNoTransaction("restore");
+        changeMetadata(this);
+        if (snapshot.byteLength > setupBytesMax) {
+            throw new StorageError("INVALID_SNAPSHOT", "The snapshot exceeds its byte limit.");
+        }
+        this.definitions.clear();
+        const leased = this.filename === ":memory:" ? takeConnection(snapshot) : undefined;
+        let restored;
+        try {
+            restored =
+                leased ??
+                    Database.deserialize(new Uint8Array(snapshot), {
+                        strict: true,
+                    });
+            if (!leased) {
+                this.validateSnapshot(restored);
+            }
+        }
+        catch (error) {
+            restored?.close();
+            throw new StorageError("INVALID_SNAPSHOT", "Cannot restore an invalid InnovatorHarness database snapshot.", { cause: error });
+        }
+        if (this.filename === ":memory:") {
+            this.statements.clear();
+            this.connection.close();
+            this.connection = restored;
+            this.boundary.forget();
+            if (leased) {
+                this.boundary.adopt(snapshot);
+            }
+            this.connection.run("PRAGMA foreign_keys = ON");
+            return;
+        }
+        const temporaryPath = `${this.filename}.${crypto.randomUUID()}.restore`;
+        try {
+            writeFileSync(temporaryPath, restored.serialize(), { flag: "wx" });
+            this.statements.clear();
+            this.connection.close();
+            try {
+                renameSync(temporaryPath, this.filename);
+            }
+            finally {
+                this.connection = openDatabase(this.filename);
+            }
+        }
+        finally {
+            restored.close();
+            rmSync(temporaryPath, { force: true });
+        }
+    }
+    close() {
+        if (this.closed) {
+            return;
+        }
+        const failures = [];
+        let retained = false;
+        try {
+            retained = this.retainCleanConnection();
+        }
+        catch (error) {
+            failures.push(error);
+        }
+        this.closed = true;
+        this.definitions.clear();
+        this.catalog.clear();
+        this.definitionList = undefined;
+        if (retained) {
+            return;
+        }
+        try {
+            if (this.connection.inTransaction) {
+                this.connection.run("ROLLBACK");
+            }
+        }
+        catch (error) {
+            failures.push(error);
+        }
+        try {
+            this.statements.clear();
+            this.connection.close();
+        }
+        catch (error) {
+            failures.push(error);
+        }
+        if (failures.length > 0) {
+            throw new AggregateError(failures, "Storage cleanup failed.");
+        }
+    }
+    [packageTransaction](write) {
+        if (this.packageWriteActive) {
+            throw new StorageError("TRANSACTION_STATE", "Package write scopes cannot overlap.");
+        }
+        try {
+            return this.database.transaction(() => {
+                this.packageWriteActive = true;
+                try {
+                    return write();
+                }
+                finally {
+                    this.packageWriteActive = false;
+                }
+            })();
+        }
+        catch (error) {
+            // A rolled-back ALTER can leave cached result columns describing the temporary schema.
+            this.clearStatements();
+            this.definitions.clear();
+            this.definitionList = undefined;
+            this.catalog.clear();
+            changeMetadata(this);
+            throw error;
+        }
+    }
+    writeAtomically(write) {
+        if (this.packageWriteActive) {
+            return write();
+        }
+        return this.database.transaction(write)();
+    }
+    clearStatements() {
+        this.statements.clear();
+        this.connection.clearQueryCache();
+    }
+    retainCleanConnection() {
+        const image = this.reusableImage;
+        this.reusableImage = undefined;
+        if (!image || !this.boundary.owns(image)) {
+            return false;
+        }
+        this.statements.clear();
+        this.boundary.restore(image);
+        if (!retainConnection(image, this.connection)) {
+            return false;
+        }
+        this.boundary.forget();
+        return true;
+    }
+    defineTable(definition, previous) {
+        const database = this.database;
+        const physicalName = tableName(definition);
+        const columns = columnsFor(definition);
+        if (previous) {
+            const previousNames = new Set(columnsFor(previous).map((column) => column.name.toLowerCase()));
+            for (const column of columns) {
+                if (!previousNames.has(column.name.toLowerCase())) {
+                    database.run(`ALTER TABLE ${quoteIdentifier(physicalName)} ` +
+                        `ADD COLUMN ${quoteIdentifier(column.name)} ${column.declaration}`);
+                }
+            }
+        }
+        else {
+            const existing = database
+                .query("SELECT name FROM sqlite_schema WHERE name = ? COLLATE NOCASE")
+                .get(physicalName);
+            if (existing) {
+                throw new StorageError("INVALID_SCHEMA", `Table name ${physicalName} is already in use.`);
+            }
+            const declarations = columns.map((column) => {
+                const primaryKey = column.name === "id" ? " NOT NULL PRIMARY KEY COLLATE NOCASE" : "";
+                return `${quoteIdentifier(column.name)} ${column.declaration}${primaryKey}`;
+            });
+            database.run(`CREATE TABLE ${quoteIdentifier(physicalName)} (${declarations.join(", ")}) STRICT`);
+        }
+    }
+    assertUniqueTypeId(definition) {
+        const database = this.database;
+        const duplicate = database
+            .query("SELECT name FROM _harness_item_types WHERE id = ? COLLATE NOCASE")
+            .get(definition.id);
+        if (duplicate && duplicate.name !== definition.name) {
+            throw new StorageError("INVALID_SCHEMA", `ItemType id ${definition.id} already belongs to ${duplicate.name}.`);
+        }
+    }
+    get database() {
+        if (this.closed) {
+            throw new StorageError("STORAGE_CLOSED", "SQLite storage is closed.");
+        }
+        return this.connection;
+    }
+    initialize() {
+        const version = this.database
+            .query("PRAGMA user_version")
+            .get()?.user_version;
+        const identity = this.database
+            .query("PRAGMA application_id")
+            .get()?.application_id;
+        const hasTables = this.database
+            .query("SELECT name FROM sqlite_schema WHERE type = 'table' LIMIT 1")
+            .get() !== null;
+        if ((identity !== 0 && identity !== applicationId) ||
+            (identity !== applicationId && hasTables)) {
+            throw new StorageError("INCOMPATIBLE_SCHEMA", "Database is not an InnovatorHarness storage database.");
+        }
+        if (version !== 0 && version !== schemaVersion) {
+            throw new StorageError("INCOMPATIBLE_SCHEMA", `Unsupported database schema version ${version}; expected ${schemaVersion}.`);
+        }
+        this.database.transaction(() => {
+            this.database.run(`
+        CREATE TABLE IF NOT EXISTS _harness_item_types (
+          id TEXT NOT NULL PRIMARY KEY COLLATE NOCASE,
+          name TEXT NOT NULL UNIQUE COLLATE NOCASE,
+          definition TEXT NOT NULL
+        ) STRICT;
+        CREATE TABLE IF NOT EXISTS _harness_catalog (
+          type TEXT NOT NULL COLLATE NOCASE,
+          id TEXT NOT NULL COLLATE NOCASE,
+          properties TEXT NOT NULL,
+          attributes TEXT NOT NULL,
+          source TEXT NOT NULL,
+          PRIMARY KEY (type, id)
+        ) STRICT;
+        CREATE TABLE IF NOT EXISTS _harness_documents (
+          path TEXT NOT NULL PRIMARY KEY,
+          sha256 TEXT NOT NULL,
+          xml TEXT NOT NULL
+        ) STRICT;
+        CREATE TABLE IF NOT EXISTS _harness_presence (
+          type TEXT NOT NULL COLLATE NOCASE,
+          id TEXT NOT NULL COLLATE NOCASE,
+          properties TEXT NOT NULL,
+          PRIMARY KEY (type, id)
+        ) STRICT;
+        PRAGMA user_version = ${schemaVersion};
+        PRAGMA application_id = ${applicationId};
+      `);
+        })();
+    }
+    validateSnapshot(database) {
+        const version = database
+            .query("PRAGMA user_version")
+            .get()?.user_version;
+        const identity = database
+            .query("PRAGMA application_id")
+            .get()?.application_id;
+        const check = database
+            .query("PRAGMA quick_check")
+            .get()?.quick_check;
+        if (version !== schemaVersion || identity !== applicationId || check !== "ok") {
+            throw new StorageError("INVALID_SNAPSHOT", "Snapshot has an incompatible schema or failed SQLite integrity checking.");
+        }
+        database.query("SELECT id, name, definition FROM _harness_item_types LIMIT 0").all();
+        database
+            .query("SELECT type, id, properties, attributes, source FROM _harness_catalog LIMIT 0")
+            .all();
+        database.query("SELECT path, sha256, xml FROM _harness_documents LIMIT 0").all();
+        database.query("SELECT type, id, properties FROM _harness_presence LIMIT 0").all();
+    }
+    requireNoTransaction(operation) {
+        this.boundary.assertIdle(operation);
+    }
+    requireItemType(name) {
+        const definition = this.getItemType(name);
+        if (!definition) {
+            throw new StorageError("UNKNOWN_ITEM_TYPE", `ItemType ${name} is not defined. Load its package before accessing its items.`);
+        }
+        return definition;
+    }
+    readDefinition(row) {
+        // Lists and standalone lookups compare the stored representation before reuse.
+        const key = row.name.toLowerCase();
+        const cached = this.definitions.get(key);
+        if (cached?.encoded === row.definition) {
+            return cached.value;
+        }
+        const definition = decodeDefinition(row.definition);
+        for (const property of definition.properties) {
+            if (property.source) {
+                Object.freeze(property.source);
+            }
+            Object.freeze(property);
+        }
+        if (definition.source) {
+            Object.freeze(definition.source);
+        }
+        Object.freeze(definition.properties);
+        Object.freeze(definition);
+        this.definitions.set(key, { encoded: row.definition, value: definition });
+        return definition;
+    }
+    validateId(id) {
+        if (typeof id !== "string" || !id.trim()) {
+            throw new StorageError("INVALID_ITEM", "Item id must be a non-empty string.");
+        }
+    }
+    normalizeProperties(definition, properties) {
+        assertStorageData(properties, "INVALID_ITEM");
+        const result = {};
+        for (const [name, value] of Object.entries(properties)) {
+            const column = columnFor(definition, name);
+            if (!column || name === "id") {
+                throw new StorageError("INVALID_ITEM", `Unknown or immutable property ${definition.name}.${name}.`);
+            }
+            Object.defineProperty(result, name, {
+                value: normalizeValue(column, value),
+                enumerable: true,
+                configurable: true,
+                writable: true,
+            });
+        }
+        return result;
+    }
+    writePresence(type, id, properties) {
+        this.database
+            .query("INSERT INTO _harness_presence (type, id, properties) VALUES (?, ?, ?) ON CONFLICT(type, id) DO UPDATE SET properties = excluded.properties")
+            .run(type, id, JSON.stringify(properties));
+    }
+    decodeItem(definition, row) {
+        const id = row.id;
+        if (typeof id !== "string") {
+            throw new StorageError("CORRUPT_STORAGE", `Invalid id stored in table ${tableName(definition)}.`);
+        }
+        const presence = getOwn(row, "_harness_presence");
+        if (typeof presence !== "string") {
+            throw new StorageError("CORRUPT_STORAGE", `Missing property presence metadata for ${definition.name} ${id}.`);
+        }
+        const properties = {};
+        for (const name of decodePresentProperties(presence)) {
+            const value = getOwn(row, name);
+            if (value === undefined || name === "_harness_presence") {
+                throw new StorageError("CORRUPT_STORAGE", `Missing column ${definition.name}.${name}.`);
+            }
+            Object.defineProperty(properties, name, {
+                value,
+                enumerable: true,
+                configurable: true,
+                writable: true,
+            });
+        }
+        return { type: definition.name, id, properties };
+    }
+}
+function assertStorageData(value, code) {
+    try {
+        assertDataTree(value);
+    }
+    catch (error) {
+        if (!(error instanceof TypeError)) {
+            throw error;
+        }
+        throw new StorageError(code, `Invalid storage data: ${error.message}`, { cause: error });
+    }
+}
+//# sourceMappingURL=storage.js.map