homegraph 1.2.0 → 1.4.1

package/dist/db/queries.js

@@ -46,6 +46,7 @@ const query_utils_1 = require("../search/query-utils");
 const query_parser_1 = require("../search/query-parser");
 const generated_detection_1 = require("../extraction/generated-detection");
 const arkts_1 = require("../extraction/languages/arkts");
+const identifier_segments_1 = require("../search/identifier-segments");
 /**
  * Path-only heuristic for files that should not be candidates for
  * "dominant file" detection: test/spec files and tool-generated files.
@@ -69,6 +70,14 @@ function isLowValueFile(filePath) {
         (0, generated_detection_1.isGeneratedFile)(filePath));
 }
 const SQLITE_PARAM_CHUNK_SIZE = 500;
+/**
+ * Last segment of a (possibly dotted/qualified) reference name — the part a
+ * new symbol's plain node name could match: 'util.greet' → 'greet'.
+ */
+function referenceNameTail(referenceName) {
+    const idx = Math.max(referenceName.lastIndexOf('.'), referenceName.lastIndexOf(':'));
+    return idx >= 0 ? referenceName.slice(idx + 1) : referenceName;
+}
 /**
  * Convert database row to Node object
  */
@@ -136,6 +145,8 @@ class QueryBuilder {
     // whole project, not a symbol, so it carries no discriminative signal (#720).
     // Set once by the HomeGraph instance; empty by default (no down-weighting).
     projectNameTokens = new Set();
+    segmentedNames = new Set();
+    static MAX_SEGMENTED_NAMES = 65536;
     // Node cache for frequently accessed nodes (LRU-style, max 1000 entries)
     nodeCache = new Map();
     maxCacheSize = 1000;
@@ -254,6 +265,10 @@ class QueryBuilder {
             returnType: node.returnType ?? null,
             updatedAt: node.updatedAt ?? Date.now(),
         });
+        // Segment vocabulary rides the same write path so it never drifts ahead of
+        // the nodes it describes. File/import nodes are excluded (#1144).
+        if (this.isSegmentableKind(node.kind))
+            this.insertNameSegments(node.name);
     }
     /**
      * Insert multiple nodes in a transaction
@@ -325,6 +340,9 @@ class QueryBuilder {
             returnType: node.returnType ?? null,
             updatedAt: node.updatedAt ?? Date.now(),
         });
+        // Framework post-extract renames must reach the segment vocabulary (#1141).
+        if (this.isSegmentableKind(node.kind))
+            this.insertNameSegments(node.name);
     }
     /**
      * Delete a node by ID
@@ -352,6 +370,104 @@ class QueryBuilder {
         }
         this.stmts.deleteNodesByFile.run(filePath);
     }
+    /** Which node kinds contribute their name to the segment vocabulary. */
+    isSegmentableKind(kind) {
+        return kind !== 'file' && kind !== 'import';
+    }
+    /** Write `name`'s segments into name_segment_vocab (idempotent). */
+    insertNameSegments(name) {
+        if (this.segmentedNames.has(name))
+            return;
+        if (this.segmentedNames.size >= QueryBuilder.MAX_SEGMENTED_NAMES)
+            this.segmentedNames.clear();
+        this.segmentedNames.add(name);
+        if (!this.stmts.insertNameSegment) {
+            this.stmts.insertNameSegment = this.db.prepare('INSERT OR IGNORE INTO name_segment_vocab (segment, name) VALUES (?, ?)');
+        }
+        for (const segment of (0, identifier_segments_1.splitIdentifierSegments)(name)) {
+            this.stmts.insertNameSegment.run(segment, name);
+        }
+    }
+    // ===========================================================================
+    // Name-segment vocabulary (prompt-hook graph-derived gate)
+    // ===========================================================================
+    /** Wipe the segment vocabulary. A full index calls this at its start; the
+     *  node write path repopulates it as files (re-)index, so the end state is
+     *  exactly the current names with no orphan rows. */
+    clearNameSegmentVocab() {
+        this.db.exec('DELETE FROM name_segment_vocab');
+        this.segmentedNames.clear();
+    }
+    /** True when the vocab has no rows — an index built before the table existed.
+     *  `sync` uses this to heal such databases (see rebuildNameSegmentVocabFrom). */
+    isNameSegmentVocabEmpty() {
+        const row = this.db.prepare('SELECT 1 FROM name_segment_vocab LIMIT 1').get();
+        return row === undefined;
+    }
+    /** One page of distinct segmentable node names, for batched vocab rebuilds
+     *  (file basenames and import specifiers are excluded from the vocab — see
+     *  insertNode). */
+    getDistinctNodeNames(limit, offset) {
+        const rows = this.db
+            .prepare("SELECT DISTINCT name FROM nodes WHERE kind NOT IN ('file', 'import') ORDER BY name LIMIT ? OFFSET ?")
+            .all(limit, offset);
+        return rows.map((r) => r.name);
+    }
+    /** Insert segments for a batch of names in one transaction (vocab heal path). */
+    insertNameSegmentsBatch(names) {
+        this.db.transaction(() => {
+            for (const name of names)
+                this.insertNameSegments(name);
+        })();
+    }
+    /**
+     * Names whose segments cover at least `minWords` distinct PROMPT WORDS —
+     * the co-occurrence probe behind the prompt hook's medium tier: the words
+     * "state" and "machine" both being segments of `OrderStateMachine` is strong
+     * evidence the prompt names that symbol in prose. Ordered by coverage.
+     *
+     * Takes (segment variant → original word) pairs and folds variants back to
+     * their word INSIDE the SQL: a name matching both `service` and `services`
+     * counts ONE word, not two. Counting raw variants let plural-variant pairs
+     * of a single word tie with genuine two-word matches and — because ORDER
+     * BY/LIMIT run here, before any JS-side re-check — crowd a real match past
+     * the LIMIT on vocab-heavy repos (#1146).
+     */
+    getSegmentCoOccurrence(variants, minWords, limit) {
+        if (variants.length === 0)
+            return [];
+        const placeholders = variants.map(() => '?').join(', ');
+        const whens = variants.map(() => 'WHEN ? THEN ?').join(' ');
+        const rows = this.db
+            .prepare(`SELECT name, COUNT(DISTINCT CASE segment ${whens} END) AS matches
+         FROM name_segment_vocab
+         WHERE segment IN (${placeholders})
+         GROUP BY name
+         HAVING matches >= ?
+         ORDER BY matches DESC, length(name) ASC
+         LIMIT ?`)
+            .all(...variants.flatMap((v) => [v.segment, v.word]), ...variants.map((v) => v.segment), minWords, limit);
+        return rows;
+    }
+    /** How many distinct names each segment appears in — the rarity signal that
+     *  separates a discriminative word ("checkout") from a ubiquitous one ("state"). */
+    getSegmentNameCounts(segments) {
+        if (segments.length === 0)
+            return new Map();
+        const placeholders = segments.map(() => '?').join(', ');
+        const rows = this.db
+            .prepare(`SELECT segment, COUNT(*) AS n FROM name_segment_vocab
+         WHERE segment IN (${placeholders}) GROUP BY segment`)
+            .all(...segments);
+        return new Map(rows.map((r) => [r.segment, r.n]));
+    }
+    /** Names containing the given segment (rare-single-word tier). */
+    getNamesForSegment(segment, limit) {
+        const rows = this.db
+            .prepare('SELECT name FROM name_segment_vocab WHERE segment = ? ORDER BY length(name) ASC LIMIT ?')
+            .all(segment, limit);
+        return rows.map((r) => r.name);
+    }
     /**
      * Get a node by ID
      */
@@ -645,6 +761,33 @@ class QueryBuilder {
             yield rowToNode(row);
         }
     }
+    /**
+     * Stream nodes of a language whose `decorators` JSON array contains a
+     * quoted decorator name. LIKE is a substring filter (a decorator name can
+     * appear as a substring of another), so callers must still exact-check
+     * `node.decorators.includes(decorator)`. Exists so the kotlin expect/actual
+     * synthesizer never materializes the whole node table the way
+     * `getAllNodes().filter(...)` did — that array alone OOM'd Node's default
+     * heap on a 2M-node graph (#1212).
+     */
+    *iterateNodesByLanguageWithDecorator(language, decorator) {
+        // Fresh statement per call — an iterator holds an open cursor (see
+        // iterateNodesByKind).
+        const stmt = this.db.prepare("SELECT * FROM nodes WHERE language = ? AND decorators LIKE '%' || ? || '%'");
+        for (const row of stmt.iterate(language, `"${decorator}"`)) {
+            yield rowToNode(row);
+        }
+    }
+    /**
+     * Distinct languages present in the files table. One indexed aggregate —
+     * lets the dynamic-edge synthesizers skip passes for languages the project
+     * doesn't contain at all (a Kotlin pass has no work on a pure-C repo), so
+     * their cost is zero rather than a full-graph scan that finds nothing (#1212).
+     */
+    getDistinctFileLanguages() {
+        const rows = this.db.prepare('SELECT DISTINCT language FROM files').all();
+        return new Set(rows.map((r) => r.language));
+    }
     /**
      * Get all nodes in the database
      */
@@ -1259,12 +1402,23 @@ class QueryBuilder {
     /**
      * Get incoming edges to a node
      */
-    getIncomingEdges(targetId, kinds) {
+    getIncomingEdges(targetId, kinds, limit) {
+        const cap = typeof limit === 'number' && Number.isFinite(limit)
+            ? Math.max(1, Math.min(Math.floor(limit), 2000))
+            : undefined;
         if (kinds && kinds.length > 0) {
-            const sql = `SELECT * FROM edges WHERE target = ? AND kind IN (${kinds.map(() => '?').join(',')})`;
+            let sql = `SELECT * FROM edges WHERE target = ? AND kind IN (${kinds.map(() => '?').join(',')})`;
+            if (cap !== undefined)
+                sql += ` LIMIT ${cap}`;
             const rows = this.db.prepare(sql).all(targetId, ...kinds);
             return rows.map(rowToEdge);
         }
+        if (cap !== undefined) {
+            const rows = this.db
+                .prepare('SELECT * FROM edges WHERE target = ? LIMIT ?')
+                .all(targetId, cap);
+            return rows.map(rowToEdge);
+        }
         if (!this.stmts.getEdgesByTarget) {
             this.stmts.getEdgesByTarget = this.db.prepare('SELECT * FROM edges WHERE target = ?');
         }
@@ -1343,7 +1497,8 @@ class QueryBuilder {
      * {@link getDependentFilePaths}: all kinds except `contains`.
      */
     getCrossFileIncomingEdgesWithTarget(filePath) {
-        const sql = `SELECT e.*, tgt.name AS target_name, tgt.kind AS target_kind
+        const sql = `SELECT e.*, tgt.name AS target_name, tgt.kind AS target_kind,
+        src.file_path AS source_file_path, src.language AS source_language
       FROM edges e
       JOIN nodes tgt ON tgt.id = e.target
       JOIN nodes src ON src.id = e.source
@@ -1355,6 +1510,8 @@ class QueryBuilder {
             ...rowToEdge(row),
             targetName: row.target_name,
             targetKind: row.target_kind,
+            sourceFilePath: row.source_file_path,
+            sourceLanguage: row.source_language,
         }));
     }
     /**
@@ -1541,7 +1698,7 @@ class QueryBuilder {
      */
     getUnresolvedReferencesCount() {
         if (!this.stmts.getUnresolvedCount) {
-            this.stmts.getUnresolvedCount = this.db.prepare('SELECT COUNT(*) as count FROM unresolved_refs');
+            this.stmts.getUnresolvedCount = this.db.prepare("SELECT COUNT(*) as count FROM unresolved_refs WHERE status = 'pending'");
         }
         const row = this.stmts.getUnresolvedCount.get();
         return row.count;
@@ -1552,7 +1709,7 @@ class QueryBuilder {
      */
     getUnresolvedReferencesBatch(offset, limit) {
         if (!this.stmts.getUnresolvedBatch) {
-            this.stmts.getUnresolvedBatch = this.db.prepare('SELECT * FROM unresolved_refs LIMIT ? OFFSET ?');
+            this.stmts.getUnresolvedBatch = this.db.prepare("SELECT * FROM unresolved_refs WHERE status = 'pending' LIMIT ? OFFSET ?");
         }
         const rows = this.stmts.getUnresolvedBatch.all(limit, offset);
         return rows.map((row) => ({
@@ -1586,6 +1743,135 @@ class QueryBuilder {
         const rows = this.stmts.getAllNodeNames.all();
         return rows.map((r) => r.name);
     }
+    /**
+     * Stream distinct node names one row at a time — for resolver cache warm-up
+     * that needs to yield to the event loop mid-scan.
+     */
+    *iterateNodeNames() {
+        const stmt = this.db.prepare('SELECT DISTINCT name FROM nodes');
+        for (const row of stmt.iterate()) {
+            yield row.name;
+        }
+    }
+    /**
+     * Park refs as status='failed' so drain loops skip them while sync can retry
+     * when a later file change introduces a matching symbol.
+     */
+    markReferencesFailed(refs) {
+        if (refs.length === 0)
+            return;
+        const stmt = this.db.prepare("UPDATE unresolved_refs SET status = 'failed', name_tail = ? WHERE from_node_id = ? AND reference_name = ? AND reference_kind = ?");
+        const markMany = this.db.transaction((items) => {
+            for (const ref of items) {
+                stmt.run(referenceNameTail(ref.referenceName), ref.fromNodeId, ref.referenceName, ref.referenceKind);
+            }
+        });
+        markMany(refs);
+    }
+    /**
+     * Delete unresolved-ref rows by row id — the precise cleanup for refs a
+     * resolution pass actually processed. The key-tuple variant above also
+     * deletes SIBLING rows (same caller calling the same callee at other lines)
+     * that a later batch hasn't attempted yet, so when a batch boundary split a
+     * caller's same-named call sites, the later sites' edges were silently never
+     * created (#1269).
+     */
+    deleteReferencesByRowIds(rowIds) {
+        if (rowIds.length === 0)
+            return;
+        for (let i = 0; i < rowIds.length; i += SQLITE_PARAM_CHUNK_SIZE) {
+            const chunk = rowIds.slice(i, i + SQLITE_PARAM_CHUNK_SIZE);
+            const placeholders = chunk.map(() => '?').join(',');
+            this.db.prepare(`DELETE FROM unresolved_refs WHERE id IN (${placeholders})`).run(...chunk);
+        }
+    }
+    /**
+     * Park refs as status='failed' by row id — the precise counterpart of
+     * markReferencesFailed, for the same reason as deleteReferencesByRowIds:
+     * the key-tuple variant also flips same-key sibling rows in later batches
+     * to 'failed' before they were ever attempted (#1269). Resolution outcome
+     * can differ per call site (receiver-type inference reads the ref's line),
+     * so a sibling must not inherit this row's failure.
+     */
+    markReferencesFailedByRowIds(refs) {
+        if (refs.length === 0)
+            return;
+        const stmt = this.db.prepare("UPDATE unresolved_refs SET status = 'failed', name_tail = ? WHERE id = ?");
+        const markMany = this.db.transaction((items) => {
+            for (const ref of items) {
+                stmt.run(referenceNameTail(ref.referenceName), ref.rowId);
+            }
+        });
+        markMany(refs);
+    }
+    /**
+     * Failed refs whose name tail matches one of the given symbol names — the
+     * candidates a sync should retry after files carrying those names changed
+     * (#1240). Names matching more than `perNameCeiling` failed refs are
+     * skipped entirely: at that population a name is external/builtin noise
+     * (`get`, `map`, …) that one new definition won't resolve — the same
+     * rationale as resolution's AMBIGUOUS_NAME_CEILING (#999) — and retrying an
+     * arbitrary subset would be both wasted work and incoherent coverage.
+     */
+    getRetryableFailedReferences(names, perNameCeiling = 500) {
+        if (names.length === 0)
+            return [];
+        // Pass 1: per-tail counts, chunked under the SQLite parameter limit.
+        const retryNames = [];
+        for (let i = 0; i < names.length; i += SQLITE_PARAM_CHUNK_SIZE) {
+            const chunk = names.slice(i, i + SQLITE_PARAM_CHUNK_SIZE);
+            const placeholders = chunk.map(() => '?').join(',');
+            const counts = this.db
+                .prepare(`SELECT name_tail, COUNT(*) as count FROM unresolved_refs WHERE status = 'failed' AND name_tail IN (${placeholders}) GROUP BY name_tail`)
+                .all(...chunk);
+            for (const row of counts) {
+                if (row.count <= perNameCeiling)
+                    retryNames.push(row.name_tail);
+            }
+        }
+        if (retryNames.length === 0)
+            return [];
+        // Pass 2: load the surviving rows.
+        const rows = [];
+        for (let i = 0; i < retryNames.length; i += SQLITE_PARAM_CHUNK_SIZE) {
+            const chunk = retryNames.slice(i, i + SQLITE_PARAM_CHUNK_SIZE);
+            const placeholders = chunk.map(() => '?').join(',');
+            const chunkRows = this.db
+                .prepare(`SELECT * FROM unresolved_refs WHERE status = 'failed' AND name_tail IN (${placeholders})`)
+                .all(...chunk);
+            rows.push(...chunkRows);
+        }
+        return rows.map((row) => ({
+            fromNodeId: row.from_node_id,
+            referenceName: row.reference_name,
+            referenceKind: row.reference_kind,
+            line: row.line,
+            column: row.col,
+            candidates: row.candidates ? (0, utils_1.safeJsonParse)(row.candidates, undefined) : undefined,
+            filePath: row.file_path,
+            language: row.language,
+            rowId: row.id,
+        }));
+    }
+    /**
+     * Distinct node names present in the given files — the symbol names a sync
+     * pass uses to look up retryable failed refs after those files changed.
+     */
+    getNodeNamesByFiles(filePaths) {
+        if (filePaths.length === 0)
+            return [];
+        const names = new Set();
+        for (let i = 0; i < filePaths.length; i += SQLITE_PARAM_CHUNK_SIZE) {
+            const chunk = filePaths.slice(i, i + SQLITE_PARAM_CHUNK_SIZE);
+            const placeholders = chunk.map(() => '?').join(',');
+            const rows = this.db
+                .prepare(`SELECT DISTINCT name FROM nodes WHERE file_path IN (${placeholders})`)
+                .all(...chunk);
+            for (const row of rows)
+                names.add(row.name);
+        }
+        return [...names];
+    }
     /**
      * Get unresolved references scoped to specific file paths.
      * Uses the idx_unresolved_file_path index for efficient lookup.
@@ -1602,7 +1888,7 @@ class QueryBuilder {
             const chunk = filePaths.slice(i, i + SQLITE_PARAM_CHUNK_SIZE);
             const placeholders = chunk.map(() => '?').join(',');
             const chunkRows = this.db
-                .prepare(`SELECT * FROM unresolved_refs WHERE file_path IN (${placeholders})`)
+                .prepare(`SELECT * FROM unresolved_refs WHERE status = 'pending' AND file_path IN (${placeholders})`)
                 .all(...chunk);
             rows.push(...chunkRows);
         }
@@ -1752,6 +2038,13 @@ class QueryBuilder {
             .get(cacheKey);
         return row ?? null;
     }
+    /** All cache keys — used to warm the in-memory key index after daemon restart. */
+    listMcpQueryCacheKeys() {
+        const rows = this.db
+            .prepare('SELECT cache_key FROM mcp_query_cache')
+            .all();
+        return rows.map((r) => r.cache_key);
+    }
     setMcpQueryCache(cacheKey, tool, response, createdAt) {
         this.db
             .prepare('INSERT INTO mcp_query_cache (cache_key, tool, response, created_at) VALUES (?, ?, ?, ?) ' +