dsh-plugin-dbgraph-store 0.0.0-stage → 0.1.0

package/lib/terms.js

@@ -0,0 +1,146 @@
+/**
+ * The semantic layer of the index: `terms` rows and the FTS `search` documents.
+ *
+ * Two lookups answer the same question from opposite directions:
+ *
+ * - `terms` is the resolver's table — one row per (word, object[, column]) with a weight that says how
+ *   strong the evidence is: a column's name token beats its comment, an alias expansion beats neither,
+ *   a Chinese bigram beats nothing at all, it is partial evidence by construction.
+ * - `search` is the FTS5 document — the tokenizer's output for an object's name and comment (one row)
+ *   and for each column's name and comment (one row per column), used for free-text listing and as a
+ *   recall net behind the resolver.
+ *
+ * Weights live in one exported table so the resolver's expectations and the writer's output cannot
+ * drift apart silently.
+ * @module dsh-plugin-dbgraph-store/terms
+ */
+import { searchDocument, tokenizeForSearch } from 'dsh-plugin-dbgraph-service';
+import { aliasGroupTokens, aliasGroupsIn } from "./aliases.js";
+import { isAsciiToken, normalizeComment } from "./normalize.js";
+/**
+ * Evidence weight per token source. A column-level row is finally weighed again by the resolver
+ * (a column hit is weaker evidence about the *table* than the table's own name), so these numbers are
+ * "how much does this word describe the name it was found in", not final confidence.
+ */
+export const TERM_WEIGHTS = {
+    /** A token of the object's own name. */
+    objectNameAscii: 1,
+    /** A CJK bigram of the object's name — partial by nature. */
+    objectNameCjk: 0.75,
+    /** An ASCII token of the object's comment. */
+    objectCommentAscii: 0.75,
+    /** A CJK bigram of the object's comment. */
+    objectCommentCjk: 0.5,
+    /** An alias-group expansion on an object's name or comment. */
+    objectAlias: 0.6,
+    /** An ASCII token of the column's own name. */
+    columnNameAscii: 0.9,
+    /** A CJK bigram of the column's name. */
+    columnNameCjk: 0.75,
+    /** An ASCII token of the column's comment. */
+    columnCommentAscii: 0.7,
+    /** A CJK bigram of the column's comment. */
+    columnCommentCjk: 0.5,
+    /** An alias-group expansion on a column's name or comment. */
+    columnAlias: 0.65,
+};
+/**
+ * Assign the ids the writer will use, so terms and rows cannot disagree about identity.
+ * @param catalog - the snapshot being written.
+ * @returns object ids by position and column ids by object/column position.
+ */
+export function assignCatalogIds(catalog) {
+    const objects = [];
+    const columns = [];
+    let nextColumnId = 1;
+    for (let objectIndex = 0; objectIndex < catalog.objects.length; objectIndex += 1) {
+        const object = catalog.objects[objectIndex];
+        objects.push(objectIndex + 1);
+        const ids = [];
+        for (let columnIndex = 0; columnIndex < (object?.columns.length ?? 0); columnIndex += 1) {
+            ids.push(nextColumnId);
+            nextColumnId += 1;
+        }
+        columns.push(ids);
+    }
+    return { objects, columns };
+}
+/**
+ * Pick the weight for one token given whether it is an ASCII identifier part or a CJK bigram.
+ * @param token - the token.
+ * @param ascii - weight for an ASCII token.
+ * @param cjk - weight for a CJK token.
+ * @returns the chosen weight.
+ */
+function weightFor(token, ascii, cjk) {
+    return isAsciiToken(token) ? ascii : cjk;
+}
+/**
+ * Build every `terms` row and both `search` documents for one catalog.
+ * @param catalog - the snapshot being written.
+ * @param ids - ids from {@link assignCatalogIds}.
+ * @param aliases - the alias table handed in by the caller.
+ * @returns deduplicated term rows and search documents.
+ */
+export function buildTerms(catalog, ids, aliases) {
+    const terms = new Map();
+    const search = [];
+    const add = (term, weight, objectId, columnId) => {
+        if (term === '')
+            return;
+        const key = `${term}\u0000${objectId}\u0000${columnId ?? ''}`;
+        const existing = terms.get(key);
+        if (existing !== undefined && existing.weight >= weight)
+            return;
+        terms.set(key, { term, weight, objectId, columnId });
+    };
+    const addTokens = (tokens, asciiWeight, cjkWeight, objectId, columnId) => {
+        for (const token of tokens)
+            add(token, weightFor(token, asciiWeight, cjkWeight), objectId, columnId);
+    };
+    for (let objectIndex = 0; objectIndex < catalog.objects.length; objectIndex += 1) {
+        const object = catalog.objects[objectIndex];
+        const objectId = ids.objects[objectIndex];
+        if (object === undefined || objectId === undefined)
+            continue;
+        const comment = normalizeComment(object.comment);
+        addTokens(tokenizeForSearch(object.name), TERM_WEIGHTS.objectNameAscii, TERM_WEIGHTS.objectNameCjk, objectId, null);
+        if (comment !== undefined) {
+            addTokens(tokenizeForSearch(comment), TERM_WEIGHTS.objectCommentAscii, TERM_WEIGHTS.objectCommentCjk, objectId, null);
+        }
+        for (const group of aliasGroupsIn([object.name, comment], aliases)) {
+            for (const token of aliasGroupTokens(group)) {
+                add(token, TERM_WEIGHTS.objectAlias, objectId, null);
+            }
+        }
+        search.push({
+            text: searchDocument([object.name, comment]),
+            objectId,
+            columnId: null,
+        });
+        const columnIds = ids.columns[objectIndex] ?? [];
+        for (let columnIndex = 0; columnIndex < object.columns.length; columnIndex += 1) {
+            const column = object.columns[columnIndex];
+            const columnId = columnIds[columnIndex];
+            if (column === undefined || columnId === undefined)
+                continue;
+            const columnComment = normalizeComment(column.comment);
+            addTokens(tokenizeForSearch(column.name), TERM_WEIGHTS.columnNameAscii, TERM_WEIGHTS.columnNameCjk, objectId, columnId);
+            if (columnComment !== undefined) {
+                addTokens(tokenizeForSearch(columnComment), TERM_WEIGHTS.columnCommentAscii, TERM_WEIGHTS.columnCommentCjk, objectId, columnId);
+            }
+            for (const group of aliasGroupsIn([column.name, columnComment], aliases)) {
+                for (const token of aliasGroupTokens(group)) {
+                    add(token, TERM_WEIGHTS.columnAlias, objectId, columnId);
+                }
+            }
+            search.push({
+                text: searchDocument([column.name, columnComment]),
+                objectId: null,
+                columnId,
+            });
+        }
+    }
+    return { terms: [...terms.values()], search };
+}
+//# sourceMappingURL=terms.js.map

package/lib/writer-js.d.ts

@@ -0,0 +1,42 @@
+/**
+ * The JavaScript write path: take one normalized catalog snapshot and produce a complete `index.db`.
+ *
+ * This path is two things at once, and both constrain how it may be written:
+ *
+ * - the **parity reference** for the native writer — fixture expectations are generated by running
+ *   this code, so it must be deterministic, and it must execute {@link INDEX_SCHEMA_DDL} verbatim
+ *   rather than carrying any schema shape of its own;
+ * - the **degradation path** — a host without the native addon still gets a full index, at the cost of
+ *   blocking the event loop for the duration (`node:sqlite` is synchronous). `IndexReport.elapsedMs`
+ *   is measured here precisely so that cost can be compared against the native kernel instead of
+ *   argued about.
+ *
+ * The file is written to `<index>.<pid>-<nonce>.tmp`, closed, and only then renamed over `index.db`.
+ * The nonce keeps concurrent runs from sharing one staging file; a run that dies leaves the previous
+ * index untouched, and an abandoned staging file is swept by a later run once it is an hour old.
+ * @module dsh-plugin-dbgraph-store/writer-js
+ */
+import type { AliasGroup, ConnectionIdType, IndexReport, RawCatalog } from 'dsh-plugin-dbgraph-service';
+/** What one JavaScript write needs. */
+export interface JsWriteInput {
+    /** Connection identity; also the directory name under the root. */
+    readonly connection: ConnectionIdType;
+    /** The snapshot to write. */
+    readonly catalog: RawCatalog;
+    /** Alias table expanded into `terms`. */
+    readonly aliases: readonly AliasGroup[];
+    /** Index root (`~/.dsh/dbgraph`, or `DSH_DBGRAPH_ROOT`). */
+    readonly root: string;
+    /** Epoch milliseconds recorded as the write time; defaults to now. */
+    readonly now?: number;
+}
+/**
+ * Write a complete index for one connection.
+ *
+ * The returned counts are what was actually inserted, not what the catalog claimed, so a dropped edge
+ * or a duplicate term is visible rather than assumed away.
+ * @param input - the snapshot, identity, aliases and root.
+ * @returns the run's report; `elapsedMs` covers schema creation, every insert, and the rename.
+ */
+export declare function writeIndexJs(input: JsWriteInput): IndexReport;
+//# sourceMappingURL=writer-js.d.ts.map

package/lib/writer-js.js

@@ -0,0 +1,269 @@
+/**
+ * The JavaScript write path: take one normalized catalog snapshot and produce a complete `index.db`.
+ *
+ * This path is two things at once, and both constrain how it may be written:
+ *
+ * - the **parity reference** for the native writer — fixture expectations are generated by running
+ *   this code, so it must be deterministic, and it must execute {@link INDEX_SCHEMA_DDL} verbatim
+ *   rather than carrying any schema shape of its own;
+ * - the **degradation path** — a host without the native addon still gets a full index, at the cost of
+ *   blocking the event loop for the duration (`node:sqlite` is synchronous). `IndexReport.elapsedMs`
+ *   is measured here precisely so that cost can be compared against the native kernel instead of
+ *   argued about.
+ *
+ * The file is written to `<index>.<pid>-<nonce>.tmp`, closed, and only then renamed over `index.db`.
+ * The nonce keeps concurrent runs from sharing one staging file; a run that dies leaves the previous
+ * index untouched, and an abandoned staging file is swept by a later run once it is an hour old.
+ * @module dsh-plugin-dbgraph-store/writer-js
+ */
+import { mkdirSync, renameSync, rmSync } from 'node:fs';
+import { dirname } from 'node:path';
+import { performance } from 'node:perf_hooks';
+import { DatabaseSync } from 'node:sqlite';
+import { DbgraphError, INDEX_META_KEYS, INDEX_SCHEMA_DDL, INDEX_SCHEMA_VERSION, isValidConnectionId, } from 'dsh-plugin-dbgraph-service';
+import { normalizeComment, typeClassOf } from "./normalize.js";
+import { indexPath, stagingIndexPath, sweepStaleStaging } from "./paths.js";
+import { assignCatalogIds, buildTerms } from "./terms.js";
+/**
+ * Execute the contract DDL and translate any failure into a structured error.
+ * @param db - the staging database.
+ */
+function applySchema(db) {
+    for (const statement of INDEX_SCHEMA_DDL) {
+        try {
+            db.exec(statement);
+        }
+        catch (error) {
+            throw new DbgraphError(`creating the index schema failed: ${messageOf(error)}`, 'DBGRAPH_WRITE_FAILED', { cause: error });
+        }
+    }
+}
+/**
+ * One thrown value's message.
+ * @param error - the thrown value.
+ * @returns a non-empty message.
+ */
+function messageOf(error) {
+    if (error instanceof Error)
+        return error.message;
+    if (typeof error === 'string')
+        return error;
+    return JSON.stringify(error) ?? 'unknown error';
+}
+/**
+ * Index the objects being written by every lookup the edge resolver needs.
+ * @param objects - the catalog objects, in write order.
+ * @param objectIds - their ids.
+ * @returns display-qualified and bare-name lookups.
+ */
+function buildObjectSlots(objects, objectIds, columnIds) {
+    const byDisplay = new Map();
+    const byName = new Map();
+    for (let index = 0; index < objects.length; index += 1) {
+        const object = objects[index];
+        const id = objectIds[index];
+        if (object === undefined || id === undefined)
+            continue;
+        const schema = object.schema ?? '';
+        const exact = new Map();
+        const lower = new Map();
+        for (let columnIndex = 0; columnIndex < object.columns.length; columnIndex += 1) {
+            const column = object.columns[columnIndex];
+            const columnId = (columnIds[index] ?? [])[columnIndex];
+            if (column === undefined || columnId === undefined)
+                continue;
+            exact.set(column.name, columnId);
+            lower.set(column.name.toLowerCase(), columnId);
+        }
+        const slot = { id, schema, name: object.name, columns: exact, columnsLower: lower };
+        byDisplay.set(schema === '' ? object.name : `${schema}.${object.name}`, slot);
+        const bucket = byName.get(object.name);
+        if (bucket === undefined)
+            byName.set(object.name, [slot]);
+        else
+            bucket.push(slot);
+    }
+    return { byDisplay, byName };
+}
+/**
+ * Resolve a referenced object name against the objects in this snapshot.
+ * @param reference - the name the catalog reported, possibly schema-qualified.
+ * @param lookups - the lookup tables from {@link buildObjectSlots}.
+ * @returns the slot, or undefined when the target was not indexed.
+ */
+function findReferencedObject(reference, lookups) {
+    const direct = lookups.byDisplay.get(reference);
+    if (direct !== undefined)
+        return direct;
+    const bare = lookups.byName.get(reference);
+    if (bare !== undefined && bare.length === 1)
+        return bare[0];
+    const lower = reference.toLowerCase();
+    for (const [display, slot] of lookups.byDisplay) {
+        if (display.toLowerCase() === lower)
+            return slot;
+    }
+    return undefined;
+}
+/**
+ * Resolve a column name inside one object.
+ * @param slot - the object.
+ * @param name - the declared column name.
+ * @returns the column id, or undefined.
+ */
+function findColumnId(slot, name) {
+    return slot.columns.get(name) ?? slot.columnsLower.get(name.toLowerCase());
+}
+/**
+ * Write a complete index for one connection.
+ *
+ * The returned counts are what was actually inserted, not what the catalog claimed, so a dropped edge
+ * or a duplicate term is visible rather than assumed away.
+ * @param input - the snapshot, identity, aliases and root.
+ * @returns the run's report; `elapsedMs` covers schema creation, every insert, and the rename.
+ */
+export function writeIndexJs(input) {
+    const startedAt = performance.now();
+    const connection = input.connection;
+    if (!isValidConnectionId(connection)) {
+        throw new DbgraphError(`"${connection}" is not a usable connection id`, 'DBGRAPH_INVALID_PROVIDER');
+    }
+    const now = input.now ?? Date.now();
+    const finalPath = indexPath(connection, input.root);
+    const stagingPath = stagingIndexPath(connection, input.root);
+    mkdirSync(dirname(finalPath), { recursive: true });
+    // A killed run leaves its per-run staging file behind; nothing would ever reuse that name.
+    sweepStaleStaging(finalPath);
+    rmSync(stagingPath, { force: true });
+    const catalog = input.catalog;
+    const ids = assignCatalogIds(catalog);
+    const built = buildTerms(catalog, ids, input.aliases);
+    const warnings = [...(catalog.warnings ?? [])];
+    const lookups = buildObjectSlots(catalog.objects, ids.objects, ids.columns);
+    let database;
+    let objectCount = 0;
+    let columnCount = 0;
+    let edgeCount = 0;
+    let skippedEdges = 0;
+    try {
+        database = new DatabaseSync(stagingPath);
+        // The staging file is disposable until the rename: there is no reader to protect and no
+        // transaction to replay. The old index stays untouched, so durability buys nothing here.
+        database.exec('PRAGMA journal_mode = OFF');
+        database.exec('PRAGMA synchronous = OFF');
+        applySchema(database);
+        database.exec('BEGIN');
+        const setMeta = database.prepare('INSERT OR REPLACE INTO meta (key, value) VALUES (?, ?)');
+        setMeta.run(INDEX_META_KEYS.schemaVersion, String(INDEX_SCHEMA_VERSION));
+        setMeta.run(INDEX_META_KEYS.connection, connection);
+        setMeta.run(INDEX_META_KEYS.engine, catalog.engine);
+        setMeta.run(INDEX_META_KEYS.database, catalog.database);
+        if (catalog.serverVersion !== undefined) {
+            setMeta.run(INDEX_META_KEYS.serverVersion, catalog.serverVersion);
+        }
+        setMeta.run(INDEX_META_KEYS.level, catalog.level);
+        setMeta.run(INDEX_META_KEYS.fetchedAt, String(catalog.fetchedAt));
+        setMeta.run(INDEX_META_KEYS.l1IndexedAt, String(catalog.fetchedAt));
+        if (catalog.level === 'l2')
+            setMeta.run(INDEX_META_KEYS.l2IndexedAt, String(catalog.fetchedAt));
+        setMeta.run(INDEX_META_KEYS.writer, 'js');
+        setMeta.run(INDEX_META_KEYS.writtenAt, String(now));
+        const insertObject = database.prepare(`INSERT INTO objects (id, kind, schema_name, name, comment, row_estimate, updated_at)
+       VALUES (?, ?, ?, ?, ?, ?, ?)`);
+        const insertColumn = database.prepare(`INSERT INTO columns (id, object_id, ordinal, name, type, type_class, nullable, default_value,
+         comment, is_primary_key, is_unique, enum_values_json, updated_at)
+       VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`);
+        for (let index = 0; index < catalog.objects.length; index += 1) {
+            const object = catalog.objects[index];
+            const objectId = ids.objects[index];
+            if (object === undefined || objectId === undefined)
+                continue;
+            insertObject.run(objectId, object.kind, object.schema ?? '', object.name, normalizeComment(object.comment) ?? null, object.rowEstimate === undefined || !Number.isFinite(object.rowEstimate)
+                ? null
+                : Math.trunc(object.rowEstimate), now);
+            objectCount += 1;
+            const columnIds = ids.columns[index] ?? [];
+            for (let columnIndex = 0; columnIndex < object.columns.length; columnIndex += 1) {
+                const column = object.columns[columnIndex];
+                const columnId = columnIds[columnIndex];
+                if (column === undefined || columnId === undefined)
+                    continue;
+                const enumValues = column.enumValues !== undefined && column.enumValues.length > 0
+                    ? JSON.stringify(column.enumValues)
+                    : null;
+                insertColumn.run(columnId, objectId, column.ordinal, column.name, column.type, typeClassOf(column.type), column.isPrimaryKey === true ? 0 : column.nullable === false ? 0 : 1, column.defaultValue ?? null, normalizeComment(column.comment) ?? null, column.isPrimaryKey === true ? 1 : 0, column.isUnique === true ? 1 : 0, enumValues, now);
+                columnCount += 1;
+            }
+        }
+        const insertEdge = database.prepare(`INSERT INTO edges (src_object, src_column_id, kind, dst_object, dst_column_id, name)
+       VALUES (?, ?, ?, ?, ?, ?)`);
+        for (let index = 0; index < catalog.objects.length; index += 1) {
+            const object = catalog.objects[index];
+            const srcId = ids.objects[index];
+            if (object === undefined || srcId === undefined)
+                continue;
+            const srcSlot = lookups.byDisplay.get((object.schema ?? '') === '' ? object.name : `${object.schema}.${object.name}`);
+            for (const edge of object.edges ?? []) {
+                const destination = edge.refObject === undefined ? undefined : findReferencedObject(edge.refObject, lookups);
+                if (edge.refObject !== undefined && destination === undefined) {
+                    skippedEdges += 1;
+                    continue;
+                }
+                const rowCount = Math.max(1, edge.columns.length);
+                for (let columnIndex = 0; columnIndex < rowCount; columnIndex += 1) {
+                    const sourceColumn = edge.columns[columnIndex];
+                    const srcColumnId = srcSlot === undefined || sourceColumn === undefined
+                        ? undefined
+                        : findColumnId(srcSlot, sourceColumn);
+                    if (sourceColumn !== undefined && srcColumnId === undefined) {
+                        skippedEdges += 1;
+                        continue;
+                    }
+                    const referencedColumn = edge.refColumns?.[columnIndex];
+                    const dstColumnId = destination === undefined || referencedColumn === undefined
+                        ? undefined
+                        : findColumnId(destination, referencedColumn);
+                    insertEdge.run(srcId, srcColumnId ?? null, edge.kind, destination?.id ?? null, dstColumnId ?? null, edge.name ?? null);
+                    edgeCount += 1;
+                }
+            }
+        }
+        const insertTerm = database.prepare('INSERT INTO terms (term, weight, object_id, column_id) VALUES (?, ?, ?, ?)');
+        for (const row of built.terms)
+            insertTerm.run(row.term, row.weight, row.objectId, row.columnId);
+        const insertSearch = database.prepare('INSERT INTO search (text, object_id, column_id) VALUES (?, ?, ?)');
+        for (const row of built.search)
+            insertSearch.run(row.text, row.objectId, row.columnId);
+        database.exec('COMMIT');
+        database.close();
+        database = undefined;
+        renameSync(stagingPath, finalPath);
+    }
+    catch (error) {
+        try {
+            database?.close();
+        }
+        catch {
+            // The staging handle is already unusable; the remaining work is cleanup, not diagnosis.
+        }
+        rmSync(stagingPath, { force: true });
+        if (error instanceof DbgraphError)
+            throw error;
+        throw new DbgraphError(`writing the index for "${connection}" failed: ${messageOf(error)}`, 'DBGRAPH_WRITE_FAILED', { cause: error });
+    }
+    if (skippedEdges > 0) {
+        warnings.push(`${skippedEdges} edge row(s) skipped: the referenced object or column is not in this snapshot`);
+    }
+    return {
+        connection,
+        level: catalog.level,
+        objects: objectCount,
+        columns: columnCount,
+        edges: edgeCount,
+        terms: built.terms.length,
+        writer: 'js',
+        elapsedMs: Math.round(performance.now() - startedAt),
+        warnings,
+    };
+}
+//# sourceMappingURL=writer-js.js.map

package/package.json

@@ -1,6 +1,67 @@
 {
   "name": "dsh-plugin-dbgraph-store",
-  "version": "0.0.0-stage",
-  "stub": true,
-  "description": "Temporary package placeholder for staged publishing"
+  "version": "0.1.0",
+  "description": "On-disk index for dsh-plugin-dbgraph: schema, the JavaScript write path kept as the parity reference and fallback, and the read path over the global ~/.dsh/dbgraph index",
+  "keywords": [
+    "dsh",
+    "dsh-plugin",
+    "deepseek-harness",
+    "cordis",
+    "cordis-plugin",
+    "database",
+    "sql",
+    "sqlite",
+    "mysql",
+    "postgresql",
+    "mongodb",
+    "redis",
+    "schema",
+    "ai-agent",
+    "llm-tools"
+  ],
+  "license": "MIT",
+  "author": "CC ZHAO",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/CC19990113/dsh-plugin-dbgraph.git",
+    "directory": "packages/store"
+  },
+  "homepage": "https://github.com/CC19990113/dsh-plugin-dbgraph#readme",
+  "bugs": "https://github.com/CC19990113/dsh-plugin-dbgraph/issues",
+  "type": "module",
+  "main": "lib/index.js",
+  "types": "lib/index.d.ts",
+  "exports": {
+    ".": {
+      "types": "./lib/index.d.ts",
+      "default": "./lib/index.js"
+    },
+    "./invariant": {
+      "types": "./lib/invariant.d.ts",
+      "default": "./lib/invariant.js"
+    },
+    "./package.json": "./package.json"
+  },
+  "files": [
+    "lib/**/*.js",
+    "lib/**/*.d.ts"
+  ],
+  "publishConfig": {
+    "access": "public",
+    "provenance": true
+  },
+  "peerDependencies": {
+    "@deepseek-ai/cordis": ">=4.0.1",
+    "@deepseek-ai/dsh-invariants": ">=0.1.0-rc.6",
+    "@deepseek-ai/dsh-llm": ">=0.1.0-rc.6"
+  },
+  "devDependencies": {
+    "@deepseek-ai/cordis": "^4.0.1",
+    "@deepseek-ai/dsh-invariants": "0.1.0-rc.6",
+    "@deepseek-ai/dsh-llm": "0.1.0-rc.6",
+    "dsh-plugin-dbgraph-service": "^0.1.0"
+  },
+  "dependencies": {
+    "dsh-plugin-dbgraph-service": "^0.1.0"
+  }
 }

package/README.md

@@ -1,3 +0,0 @@
-# Temporary Holding Version
-
-This version is a temporary placeholder for this package. An operational version to replace this has been submitted for review and is awaiting a staged release.