@innovatorstack/harness 0.0.0-stage → 0.1.1

package/src/storage/sqlite/predicate.ts

@@ -0,0 +1,301 @@
+import { validatePredicate } from "../../queries/predicates.js";
+import { Comparison } from "../../queries/types.js";
+import type { Predicate, QueryField } from "../../queries/types.js";
+import { valueKey } from "../../queries/values.js";
+import type { HarnessStorage, ItemTypeDefinition, StorageValue } from "../types.js";
+import { quoteIdentifier, tableName } from "./schema.js";
+
+export interface CompiledPredicate {
+    readonly sql: string;
+    readonly parameters: readonly StorageValue[];
+}
+
+interface Bounds {
+    readonly yes: string;
+    readonly no: string;
+}
+
+const uncertain: Bounds = { yes: "1", no: "1" };
+
+/** Conservative pushdown: SQL may retain extra rows but must never exclude a true predicate. */
+export function compilePredicate(
+    storage: HarnessStorage,
+    definition: ItemTypeDefinition,
+    root: Predicate,
+): CompiledPredicate {
+    validatePredicate(root);
+
+    return new PredicateCompiler(storage, definition).compile(root);
+}
+
+class PredicateCompiler {
+    private readonly parameters: StorageValue[] = [];
+    private readonly joins: string[] = [];
+    private readonly paths = new Map<string, string>();
+
+    constructor(
+        private readonly storage: HarnessStorage,
+        private readonly definition: ItemTypeDefinition,
+    ) {}
+
+    compile(root: Predicate): CompiledPredicate {
+        const fragments = new Map<Predicate, Bounds>();
+        const pending = [{ node: root, expanded: false }];
+
+        while (pending.length > 0) {
+            const frame = pending.pop();
+
+            if (!frame) {
+                throw new Error("A SQL predicate frame must exist while compilation is pending.");
+            }
+
+            const node = frame.node;
+
+            if (
+                !frame.expanded &&
+                (node.kind === "and" || node.kind === "or" || node.kind === "not")
+            ) {
+                pending.push({ node, expanded: true });
+                const children = node.kind === "not" ? [node.term] : node.terms;
+
+                for (const child of children) {
+                    pending.push({ node: child, expanded: false });
+                }
+            } else {
+                fragments.set(node, this.expression(node, fragments));
+            }
+        }
+
+        const where = fragment(fragments, root).yes;
+        const table = quoteIdentifier(tableName(this.definition));
+        this.parameters.push(this.definition.name);
+        const presence = `LEFT JOIN _harness_presence AS p ON p.type = ?${this.parameters.length} COLLATE NOCASE AND p.id = r.id COLLATE NOCASE`;
+
+        return {
+            sql: `SELECT r.*, p.properties AS _harness_presence FROM ${table} AS r ${this.joins.join(" ")} ${presence} WHERE ${where} ORDER BY r.id COLLATE BINARY`,
+            parameters: this.parameters,
+        };
+    }
+
+    private expression(node: Predicate, fragments: ReadonlyMap<Predicate, Bounds>): Bounds {
+        if (node.kind === "constant") {
+            return { yes: node.value ? "1" : "0", no: node.value ? "0" : "1" };
+        }
+
+        if (node.kind === "and" || node.kind === "or") {
+            const values = node.terms.map((term) => fragment(fragments, term));
+
+            return {
+                yes: joined(
+                    values.map((value) => value.yes),
+                    node.kind,
+                ),
+                no: joined(
+                    values.map((value) => value.no),
+                    node.kind === "and" ? "or" : "and",
+                ),
+            };
+        }
+
+        if (node.kind === "not") {
+            const child = fragment(fragments, node.term);
+
+            return { yes: child.no, no: child.yes };
+        }
+
+        const column = this.column(node.field);
+
+        if (node.kind === "null") {
+            return exact(`${column} IS ${node.negate ? "NOT " : ""}NULL`);
+        }
+
+        if (node.field.kind === "date") {
+            return uncertain;
+        }
+
+        if (node.kind === "like") {
+            if (node.pattern.length > 50_000 || node.pattern.includes("\0")) {
+                return uncertain;
+            }
+
+            const expression = `${column} LIKE ${this.parameter(node.pattern)}`;
+            const bounds = { yes: exact(expression).yes, no: "1" };
+
+            if (node.field.kind === "identifier") {
+                return uncertain;
+            }
+
+            return node.negate ? { yes: bounds.no, no: bounds.yes } : bounds;
+        }
+
+        if (node.kind === "set") {
+            return this.set(node, column);
+        }
+
+        return this.comparison(node, column);
+    }
+
+    private comparison(node: Extract<Predicate, { kind: "compare" }>, column: string): Bounds {
+        if (node.field.kind === "decimal") {
+            const value = this.parameter(node.value);
+            const equal = `CAST(${column} AS REAL) = CAST(${value} AS REAL)`;
+
+            if (node.operator === Comparison.Equal) {
+                return { yes: exact(equal).yes, no: "1" };
+            }
+
+            if (node.operator === Comparison.NotEqual) {
+                return { yes: "1", no: exact(equal).yes };
+            }
+
+            const greater = [Comparison.Greater, Comparison.GreaterOrEqual].includes(node.operator);
+
+            return {
+                yes: exact(
+                    `CAST(${column} AS REAL) ${greater ? ">=" : "<="} CAST(${value} AS REAL)`,
+                ).yes,
+                no: exact(`CAST(${column} AS REAL) ${greater ? "<=" : ">="} CAST(${value} AS REAL)`)
+                    .yes,
+            };
+        }
+
+        const value = this.parameter(valueKey(node.field.kind, node.value));
+        const collation = node.field.kind === "identifier" ? "NOCASE" : "BINARY";
+        const bounds = exact(
+            `${column} COLLATE ${collation} ${sqlOperator(node.operator)} ${value}`,
+        );
+
+        return node.field.kind === "identifier" ? asciiBounds(column, bounds) : bounds;
+    }
+
+    private set(node: Extract<Predicate, { kind: "set" }>, column: string): Bounds {
+        if (node.values.length === 0) {
+            return node.negate ? { yes: "1", no: "0" } : { yes: "0", no: "1" };
+        }
+
+        const decimal = node.field.kind === "decimal";
+        const values = node.values.map((value) => this.parameter(valueKey(node.field.kind, value)));
+        const left = decimal ? `CAST(${column} AS REAL)` : column;
+        const right = decimal ? values.map((value) => `CAST(${value} AS REAL)`) : values;
+        const collation = node.field.kind === "identifier" ? "NOCASE" : "BINARY";
+        let bounds = exact(`${left} COLLATE ${collation} IN (${right.join(", ")})`);
+
+        if (decimal) {
+            bounds = { yes: bounds.yes, no: "1" };
+        } else if (node.field.kind === "identifier") {
+            bounds = asciiBounds(column, bounds);
+        }
+
+        return node.negate ? { yes: bounds.no, no: bounds.yes } : bounds;
+    }
+
+    private column(field: QueryField): string {
+        let parent = "r";
+        let definition = this.definition;
+        const path: string[] = [];
+
+        for (const reference of field.path) {
+            const target = this.storage.getItemType(reference.target);
+
+            if (
+                !target ||
+                !definition.properties.some((property) => property.name === reference.property)
+            ) {
+                throw new TypeError("A foreign query path must refer to imported metadata.");
+            }
+
+            path.push(reference.property, target.name);
+            const key = JSON.stringify(path);
+            let alias = this.paths.get(key);
+
+            if (!alias) {
+                if (this.paths.size >= 48) {
+                    throw new RangeError("Query exceeds the SQLite join budget.");
+                }
+
+                alias = `q${this.paths.size + 1}`;
+                this.paths.set(key, alias);
+                const table = quoteIdentifier(tableName(target));
+                const source = `${parent}.${quoteIdentifier(reference.property)}`;
+                this.joins.push(
+                    `LEFT JOIN ${table} AS ${alias} ON ${alias}.id = ${source} COLLATE NOCASE`,
+                );
+            }
+
+            parent = alias;
+            definition = target;
+        }
+
+        if (
+            field.column !== "id" &&
+            !definition.properties.some((property) => property.name === field.column)
+        ) {
+            throw new TypeError("A query column must refer to imported metadata.");
+        }
+
+        return `${parent}.${quoteIdentifier(field.column)}`;
+    }
+
+    private parameter(value: StorageValue): string {
+        this.parameters.push(value);
+
+        return `?${this.parameters.length}`;
+    }
+}
+
+function fragment(fragments: ReadonlyMap<Predicate, Bounds>, node: Predicate): Bounds {
+    const result = fragments.get(node);
+
+    if (!result) {
+        throw new Error("SQL operands must be compiled before their parent.");
+    }
+
+    return result;
+}
+
+function joined(terms: readonly string[], kind: "and" | "or"): string {
+    if (terms.length === 0) {
+        return kind === "and" ? "1" : "0";
+    }
+
+    return `(${terms.join(kind === "and" ? " AND " : " OR ")})`;
+}
+
+function exact(expression: string): Bounds {
+    return { yes: `((${expression}) IS TRUE)`, no: `((${expression}) IS FALSE)` };
+}
+
+function asciiBounds(column: string, bounds: Bounds): Bounds {
+    return {
+        yes: `(${column} GLOB '*[^ -~]*' OR ${bounds.yes})`,
+        no: `(${column} GLOB '*[^ -~]*' OR ${bounds.no})`,
+    };
+}
+
+function sqlOperator(operator: Comparison): string {
+    switch (operator) {
+        case Comparison.Equal: {
+            return "=";
+        }
+        case Comparison.NotEqual: {
+            return "<>";
+        }
+        case Comparison.Greater: {
+            return ">";
+        }
+        case Comparison.GreaterOrEqual: {
+            return ">=";
+        }
+        case Comparison.Less: {
+            return "<";
+        }
+        case Comparison.LessOrEqual: {
+            return "<=";
+        }
+        default: {
+            const unexpected: never = operator;
+
+            throw new TypeError(`Unknown SQL comparison '${String(unexpected)}'.`);
+        }
+    }
+}

package/src/storage/sqlite/queries.ts

@@ -0,0 +1,48 @@
+import type { Database } from "bun:sqlite";
+
+import type { Predicate } from "../../queries/types.js";
+import type { HarnessStorage, ItemTypeDefinition, StorageValue, StoredItem } from "../types.js";
+import { compilePredicate } from "./predicate.js";
+
+const candidatesMax = 100_000;
+
+/** SQLite narrows candidates; the shared evaluator owns the final, precision-preserving result. */
+export class SqliteQueries {
+    constructor(
+        private readonly storage: HarnessStorage,
+        private readonly connection: () => Database,
+        private readonly decode: (
+            definition: ItemTypeDefinition,
+            row: Record<string, StorageValue>,
+        ) => StoredItem,
+    ) {}
+
+    select(
+        type: string,
+        predicate: Predicate,
+        spend: (units: number) => void,
+    ): readonly StoredItem[] {
+        const definition = this.storage.getItemType(type);
+
+        if (!definition) {
+            throw new TypeError(`Query ItemType '${type}' must be imported.`);
+        }
+
+        const compiled = compilePredicate(this.storage, definition, predicate);
+        const statement = this.connection().query<Record<string, StorageValue>, StorageValue[]>(
+            `${compiled.sql} LIMIT ${candidatesMax + 1}`,
+        );
+        // Conservative bounds can discard a branch with the last allocated placeholders.
+        // SQLite's count includes numbered gaps and excludes unused trailing parameters.
+        const rows = statement.all(...compiled.parameters.slice(0, statement.paramsCount));
+        spend(rows.length);
+
+        if (rows.length > candidatesMax) {
+            throw new RangeError(
+                "Query candidate limit exceeded. Narrow the selector before paging.",
+            );
+        }
+
+        return rows.map((row) => this.decode(definition, row));
+    }
+}

package/src/storage/sqlite/schema.ts

@@ -0,0 +1,288 @@
+import { isBaselineSource } from "../../baseline/source.js";
+import { isFrozenData } from "../../core/data.js";
+import type { ItemTypeDefinition, PropertyDefinition, StorageValue } from "../types.js";
+import { StorageError } from "./errors.js";
+
+type ColumnKind = "text" | "integer" | "boolean" | "float" | "decimal";
+
+export interface StorageColumn {
+    readonly name: string;
+    readonly kind: ColumnKind;
+    readonly declaration: "TEXT" | "INTEGER" | "REAL";
+}
+
+const textDataTypes = new Set([
+    "string",
+    "text",
+    "formatted text",
+    "ml_string",
+    "item",
+    "date",
+    "image",
+    "color",
+    "list",
+    "filter list",
+    "sequence",
+    "mv_list",
+    "md5",
+]);
+
+interface ColumnLayout {
+    readonly columns: readonly StorageColumn[];
+    readonly byName: ReadonlyMap<string, StorageColumn>;
+}
+
+const columnLayouts = new WeakMap<ItemTypeDefinition, ColumnLayout>();
+
+export function quoteIdentifier(name: string): string {
+    // oxlint-disable-next-line no-control-regex -- SQL identifiers must reject control characters.
+    if (!name || /[\u0000-\u001F\u007F]/u.test(name)) {
+        throw new StorageError("INVALID_SCHEMA", `Invalid SQL identifier ${JSON.stringify(name)}.`);
+    }
+
+    return `"${name.replaceAll('"', '""')}"`;
+}
+
+export function tableName(definition: ItemTypeDefinition): string {
+    return definition.tableName ?? definition.name;
+}
+
+function propertyColumn(property: PropertyDefinition): StorageColumn {
+    const dataType = property.dataType.toLowerCase();
+
+    if (textDataTypes.has(dataType)) {
+        return { name: property.name, kind: "text", declaration: "TEXT" };
+    }
+
+    switch (dataType) {
+        case "integer": {
+            return { name: property.name, kind: "integer", declaration: "INTEGER" };
+        }
+        case "boolean": {
+            return { name: property.name, kind: "boolean", declaration: "INTEGER" };
+        }
+        case "float": {
+            return { name: property.name, kind: "float", declaration: "REAL" };
+        }
+        case "decimal": {
+            return { name: property.name, kind: "decimal", declaration: "TEXT" };
+        }
+        default: {
+            throw new StorageError(
+                "INVALID_SCHEMA",
+                `Property ${property.name} uses unsupported storage data type ${JSON.stringify(property.dataType)}.`,
+            );
+        }
+    }
+}
+
+export function columnsFor(definition: ItemTypeDefinition): readonly StorageColumn[] {
+    return columnLayout(definition).columns;
+}
+
+export function columnFor(definition: ItemTypeDefinition, name: string): StorageColumn | undefined {
+    return columnLayout(definition).byName.get(name);
+}
+
+function columnLayout(definition: ItemTypeDefinition): ColumnLayout {
+    const immutable = isFrozenData(definition);
+    const cached = immutable ? columnLayouts.get(definition) : undefined;
+
+    if (cached) {
+        return cached;
+    }
+
+    const columns = compileColumns(definition);
+
+    if (immutable) {
+        for (const column of columns) {
+            Object.freeze(column);
+        }
+
+        Object.freeze(columns);
+    }
+
+    const layout = { columns, byName: new Map(columns.map((column) => [column.name, column])) };
+
+    if (immutable) {
+        columnLayouts.set(definition, layout);
+    }
+
+    return layout;
+}
+
+function compileColumns(definition: ItemTypeDefinition): StorageColumn[] {
+    const columns: StorageColumn[] = [{ name: "id", kind: "text", declaration: "TEXT" }];
+
+    if (definition.isRelationship) {
+        columns.push(
+            { name: "source_id", kind: "text", declaration: "TEXT" },
+            { name: "related_id", kind: "text", declaration: "TEXT" },
+        );
+    }
+
+    const declaredNames = new Set<string>();
+
+    for (const property of definition.properties) {
+        quoteIdentifier(property.name);
+        const key = property.name.toLowerCase();
+
+        if (declaredNames.has(key)) {
+            throw new StorageError(
+                "INVALID_SCHEMA",
+                `Duplicate property ${property.name} on ${definition.name}.`,
+            );
+        }
+
+        declaredNames.add(key);
+
+        if (property.dataType.toLowerCase() === "foreign" || property.isFederated) {
+            continue;
+        }
+
+        if (key.startsWith("_harness_") || ["rowid", "_rowid_", "oid"].includes(key)) {
+            throw new StorageError("INVALID_SCHEMA", `Property name ${property.name} is reserved.`);
+        }
+
+        const column = propertyColumn(property);
+        const systemColumn = columns.find((existing) => existing.name.toLowerCase() === key);
+
+        if (systemColumn) {
+            if (column.kind !== "text") {
+                throw new StorageError(
+                    "INVALID_SCHEMA",
+                    `System property ${property.name} must use a text storage type.`,
+                );
+            }
+        } else {
+            columns.push(column);
+        }
+    }
+
+    return columns;
+}
+
+export function validateDefinition(definition: ItemTypeDefinition): void {
+    if (!definition.id || !definition.name) {
+        throw new StorageError("INVALID_SCHEMA", "ItemType id and name must be non-empty strings.");
+    }
+
+    const name = tableName(definition);
+    quoteIdentifier(name);
+
+    if (/^(?:_harness_|sqlite_)/iu.test(name)) {
+        throw new StorageError("INVALID_SCHEMA", `Table name ${name} is reserved.`);
+    }
+
+    columnsFor(definition);
+}
+
+export function assertCompatible(previous: ItemTypeDefinition, next: ItemTypeDefinition): void {
+    const replacingBaseline = isBaselineSource(previous.source) && !isBaselineSource(next.source);
+
+    if (
+        (previous.id !== next.id && !replacingBaseline) ||
+        previous.name !== next.name ||
+        tableName(previous).toLowerCase() !== tableName(next).toLowerCase() ||
+        (!replacingBaseline && Boolean(previous.isRelationship) !== Boolean(next.isRelationship))
+    ) {
+        throw new StorageError(
+            "INCOMPATIBLE_SCHEMA",
+            `Cannot change the identity, table, or relationship kind of ${previous.name}.`,
+        );
+    }
+
+    if (replacingBaseline) {
+        const previousColumns = columnsFor(previous);
+        const nextColumns = columnsFor(next);
+
+        for (const column of previousColumns) {
+            const replacement = nextColumns.find((candidate) => candidate.name === column.name);
+
+            if (!replacement || replacement.declaration !== column.declaration) {
+                throw new StorageError(
+                    "INCOMPATIBLE_SCHEMA",
+                    `Native metadata cannot change the storage column ${previous.name}.${column.name}.`,
+                );
+            }
+        }
+
+        return;
+    }
+
+    for (const property of previous.properties) {
+        const replacement = next.properties.find((candidate) => candidate.name === property.name);
+
+        if (
+            !replacement ||
+            property.dataType.toLowerCase() !== replacement.dataType.toLowerCase() ||
+            (property.dataSource !== replacement.dataSource &&
+                !isBaselineSource(property.source)) ||
+            (property.foreignProperty !== replacement.foreignProperty &&
+                !isBaselineSource(property.source))
+        ) {
+            throw new StorageError(
+                "INCOMPATIBLE_SCHEMA",
+                `Cannot remove or change property ${previous.name}.${property.name}.`,
+            );
+        }
+    }
+}
+
+export function normalizeValue(column: StorageColumn, value: StorageValue): StorageValue {
+    if (value === null) {
+        return null;
+    }
+
+    switch (column.kind) {
+        case "text": {
+            if (typeof value === "string") {
+                return value;
+            }
+
+            break;
+        }
+        case "integer": {
+            if (typeof value === "number" && Number.isSafeInteger(value)) {
+                return value;
+            }
+
+            break;
+        }
+        case "boolean": {
+            if (value === 0 || value === 1) {
+                return value;
+            }
+
+            break;
+        }
+        case "float": {
+            if (typeof value === "number" && Number.isFinite(value)) {
+                return value;
+            }
+
+            break;
+        }
+        case "decimal": {
+            if (typeof value === "string" && /^[+-]?\d+(?:\.\d+)?$/u.test(value)) {
+                return value;
+            }
+
+            if (typeof value === "number" && Number.isFinite(value)) {
+                return String(value);
+            }
+
+            break;
+        }
+        default: {
+            const unexpected: never = column.kind;
+
+            throw new Error(`Unknown storage column kind: ${String(unexpected)}`);
+        }
+    }
+
+    throw new StorageError(
+        "INVALID_ITEM",
+        `Property ${column.name} requires a ${column.kind} value; received ${JSON.stringify(value)}.`,
+    );
+}

package/src/storage/sqlite/statements.ts

@@ -0,0 +1,51 @@
+import type { Database, Statement } from "bun:sqlite";
+
+import type { StorageValue } from "../types.js";
+
+type ItemStatement = Statement<Record<string, StorageValue>, StorageValue[]>;
+
+const statementsMax = 128;
+
+/** Bounded, connection-owned SQL plans. These retain no item data or database copies. */
+export class SqliteStatements {
+    private readonly statements = new Map<string, ItemStatement>();
+
+    constructor(private readonly connection: () => Database) {}
+
+    query(sql: string): ItemStatement {
+        const existing = this.statements.get(sql);
+
+        if (existing) {
+            this.statements.delete(sql);
+            this.statements.set(sql, existing);
+
+            return existing;
+        }
+
+        if (this.statements.size === statementsMax) {
+            const oldest = this.statements.entries().next().value;
+
+            if (!oldest) {
+                throw new Error("A full statement cache must contain an eviction candidate.");
+            }
+
+            oldest[1].finalize();
+            this.statements.delete(oldest[0]);
+        }
+
+        const statement = this.connection().prepare<Record<string, StorageValue>, StorageValue[]>(
+            sql,
+        );
+        this.statements.set(sql, statement);
+
+        return statement;
+    }
+
+    clear(): void {
+        for (const statement of this.statements.values()) {
+            statement.finalize();
+        }
+
+        this.statements.clear();
+    }
+}