@davesheffer/hunch 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +241 -0
- package/dist/cli/index.js +587 -0
- package/dist/cli/invocation.js +23 -0
- package/dist/core/format.js +46 -0
- package/dist/core/glob.js +62 -0
- package/dist/core/ids.js +39 -0
- package/dist/core/io.js +44 -0
- package/dist/core/migrate.js +59 -0
- package/dist/core/paths.js +41 -0
- package/dist/core/types.js +142 -0
- package/dist/extractors/diff.js +198 -0
- package/dist/extractors/git.js +136 -0
- package/dist/extractors/indexer.js +271 -0
- package/dist/extractors/parse.js +176 -0
- package/dist/integrations/claudemd.js +77 -0
- package/dist/integrations/hooks.js +80 -0
- package/dist/integrations/mergeDriver.js +41 -0
- package/dist/integrations/scaffold.js +74 -0
- package/dist/mcp/server.js +233 -0
- package/dist/store/compact.js +100 -0
- package/dist/store/db.js +19 -0
- package/dist/store/embedder.js +133 -0
- package/dist/store/hunchStore.js +469 -0
- package/dist/store/jsonStore.js +268 -0
- package/dist/store/merge.js +179 -0
- package/dist/store/schema.js +100 -0
- package/dist/synthesis/provider.js +488 -0
- package/dist/synthesis/synthesize.js +312 -0
- package/package.json +68 -0
|
@@ -0,0 +1,133 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Pluggable embedder for the semantic-search READ path (DESIGN.md §6 — the
|
|
3
|
+
* "add embeddings once keyword search proves insufficient" upgrade).
|
|
4
|
+
*
|
|
5
|
+
* Embeddings are LOCAL and FREE. Anthropic has no embeddings endpoint and the
|
|
6
|
+
* project is subscription-only (see synthesis/provider.ts), so we run a small
|
|
7
|
+
* sentence-transformer locally via transformers.js. That library is an OPTIONAL
|
|
8
|
+
* dependency, dynamically imported — if it isn't installed, `selectEmbedder()`
|
|
9
|
+
* returns null and the whole feature degrades to pure FTS (the lean-install
|
|
10
|
+
* default). This mirrors the synthesis provider: an interface so callers never
|
|
11
|
+
* know which implementation ran, plus a deterministic stub for tests.
|
|
12
|
+
*
|
|
13
|
+
* Vectors are L2-NORMALIZED, so cosine similarity == dot product downstream.
|
|
14
|
+
*/
|
|
15
|
+
import { createRequire } from "node:module";
|
|
16
|
+
// The transformers.js package was renamed @xenova → @huggingface at v3. Probe both.
|
|
17
|
+
const PACKAGES = ["@huggingface/transformers", "@xenova/transformers"];
|
|
18
|
+
const HF_MODEL = "Xenova/all-MiniLM-L6-v2"; // 384-dim MiniLM; quantized ~23MB
|
|
19
|
+
/** Is one of the transformers packages resolvable WITHOUT executing it? Cheap
|
|
20
|
+
* availability probe (no onnxruntime init) so `selectEmbedder()` can return null
|
|
21
|
+
* fast on the lean install. */
|
|
22
|
+
function installedPackage() {
|
|
23
|
+
const req = createRequire(import.meta.url);
|
|
24
|
+
for (const p of PACKAGES) {
|
|
25
|
+
try {
|
|
26
|
+
req.resolve(p);
|
|
27
|
+
return p;
|
|
28
|
+
}
|
|
29
|
+
catch {
|
|
30
|
+
/* not installed — try next */
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
return null;
|
|
34
|
+
}
|
|
35
|
+
/** Run `fn` with stray stdout writes rerouted to stderr. transformers.js / its
|
|
36
|
+
* backends may log during model load; on the MCP stdio channel a single stray
|
|
37
|
+
* byte on stdout corrupts JSON-RPC. The guard is active only for the duration of
|
|
38
|
+
* `fn` (model load), which is the only noisy phase. */
|
|
39
|
+
async function withStdoutGuarded(fn) {
|
|
40
|
+
const orig = process.stdout.write.bind(process.stdout);
|
|
41
|
+
const toErr = process.stderr.write.bind(process.stderr);
|
|
42
|
+
process.stdout.write = toErr;
|
|
43
|
+
try {
|
|
44
|
+
return await fn();
|
|
45
|
+
}
|
|
46
|
+
finally {
|
|
47
|
+
process.stdout.write = orig;
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
export class TransformersEmbedder {
|
|
51
|
+
dim = 384;
|
|
52
|
+
id = "all-MiniLM-L6-v2";
|
|
53
|
+
extractor = null;
|
|
54
|
+
load() {
|
|
55
|
+
if (!this.extractor) {
|
|
56
|
+
this.extractor = withStdoutGuarded(async () => {
|
|
57
|
+
const pkg = installedPackage();
|
|
58
|
+
if (!pkg)
|
|
59
|
+
throw new Error("transformers.js not installed");
|
|
60
|
+
// String var (not a literal) so tsc doesn't require the optional dep to be
|
|
61
|
+
// present to typecheck/build, and the import resolves at runtime when it is.
|
|
62
|
+
const mod = (await import(pkg));
|
|
63
|
+
return (await mod.pipeline("feature-extraction", HF_MODEL));
|
|
64
|
+
});
|
|
65
|
+
}
|
|
66
|
+
return this.extractor;
|
|
67
|
+
}
|
|
68
|
+
async embed(texts) {
|
|
69
|
+
if (!texts.length)
|
|
70
|
+
return [];
|
|
71
|
+
const extractor = await this.load();
|
|
72
|
+
// pooling:mean + normalize:true → a [n, dim] tensor; .data is one flat
|
|
73
|
+
// Float32Array of length n*dim. Slice per row to a TIGHT copy (the tensor
|
|
74
|
+
// backing buffer is shared — a subarray view would alias it).
|
|
75
|
+
const res = await extractor(texts, { pooling: "mean", normalize: true });
|
|
76
|
+
const out = [];
|
|
77
|
+
for (let i = 0; i < texts.length; i++) {
|
|
78
|
+
out.push(res.data.slice(i * this.dim, (i + 1) * this.dim));
|
|
79
|
+
}
|
|
80
|
+
return out;
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
/** Deterministic, dependency-free embedder for tests (no model download).
|
|
84
|
+
* Bag-of-token-hashes → L2-normalized vector: shared tokens ⇒ similar vectors,
|
|
85
|
+
* enough to exercise hybrid ranking without semantics. Intentionally returns
|
|
86
|
+
* SUBARRAY VIEWS into one backing buffer, so any caller that fails to copy
|
|
87
|
+
* tightly before persisting will corrupt data — that keeps the BLOB round-trip
|
|
88
|
+
* honest. */
|
|
89
|
+
export class StubEmbedder {
|
|
90
|
+
dim;
|
|
91
|
+
id = "stub-v1";
|
|
92
|
+
constructor(dim = 32) {
|
|
93
|
+
this.dim = dim;
|
|
94
|
+
}
|
|
95
|
+
async embed(texts) {
|
|
96
|
+
const backing = new Float32Array(texts.length * this.dim);
|
|
97
|
+
const out = [];
|
|
98
|
+
texts.forEach((t, row) => {
|
|
99
|
+
const off = row * this.dim;
|
|
100
|
+
for (const tok of t.toLowerCase().match(/[a-z0-9]+/g) ?? []) {
|
|
101
|
+
let h = 2166136261;
|
|
102
|
+
for (let i = 0; i < tok.length; i++) {
|
|
103
|
+
h ^= tok.charCodeAt(i);
|
|
104
|
+
h = Math.imul(h, 16777619);
|
|
105
|
+
}
|
|
106
|
+
const idx = off + ((h >>> 0) % this.dim);
|
|
107
|
+
backing[idx] = (backing[idx] ?? 0) + 1;
|
|
108
|
+
}
|
|
109
|
+
let norm = 0;
|
|
110
|
+
for (let i = 0; i < this.dim; i++)
|
|
111
|
+
norm += (backing[off + i] ?? 0) ** 2;
|
|
112
|
+
norm = Math.sqrt(norm) || 1;
|
|
113
|
+
for (let i = 0; i < this.dim; i++)
|
|
114
|
+
backing[off + i] = (backing[off + i] ?? 0) / norm;
|
|
115
|
+
out.push(backing.subarray(off, off + this.dim)); // view, not copy (on purpose)
|
|
116
|
+
});
|
|
117
|
+
return out;
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
/** Choose an embedder, or null to signal "no semantic search → use pure FTS".
|
|
121
|
+
* Never throws. `HUNCH_EMBEDDER=stub` forces the test stub; `=none` forces the
|
|
122
|
+
* null (FTS-only) path; otherwise use the local model iff its optional dep is
|
|
123
|
+
* installed. */
|
|
124
|
+
export async function selectEmbedder() {
|
|
125
|
+
switch (process.env.HUNCH_EMBEDDER) {
|
|
126
|
+
case "stub":
|
|
127
|
+
return new StubEmbedder();
|
|
128
|
+
case "none":
|
|
129
|
+
return null;
|
|
130
|
+
}
|
|
131
|
+
return installedPackage() ? new TransformersEmbedder() : null;
|
|
132
|
+
}
|
|
133
|
+
//# sourceMappingURL=embedder.js.map
|
|
@@ -0,0 +1,469 @@
|
|
|
1
|
+
import { ENTITY_KINDS } from "../core/types.js";
|
|
2
|
+
import { openDb } from "./db.js";
|
|
3
|
+
import { RESET_SQL, embedHash } from "./schema.js";
|
|
4
|
+
import { selectEmbedder } from "./embedder.js";
|
|
5
|
+
import { JsonStore } from "./jsonStore.js";
|
|
6
|
+
import { pathMatchesGlob } from "../core/glob.js";
|
|
7
|
+
export class HunchStore {
|
|
8
|
+
paths;
|
|
9
|
+
json;
|
|
10
|
+
_db = null;
|
|
11
|
+
constructor(paths) {
|
|
12
|
+
this.paths = paths;
|
|
13
|
+
this.json = new JsonStore(paths);
|
|
14
|
+
}
|
|
15
|
+
get db() {
|
|
16
|
+
if (!this._db)
|
|
17
|
+
this._db = openDb(this.paths.sqlite);
|
|
18
|
+
return this._db;
|
|
19
|
+
}
|
|
20
|
+
close() {
|
|
21
|
+
this._db?.close();
|
|
22
|
+
this._db = null;
|
|
23
|
+
}
|
|
24
|
+
// ---- write path ---------------------------------------------------------
|
|
25
|
+
/** Rebuild the entire SQLite index + FTS from the JSON source of truth. */
|
|
26
|
+
reindex() {
|
|
27
|
+
const db = this.db;
|
|
28
|
+
const counts = {};
|
|
29
|
+
const tx = db.transaction(() => {
|
|
30
|
+
db.exec(RESET_SQL);
|
|
31
|
+
const j = (s) => s; // readability marker for JSON-encoded columns
|
|
32
|
+
const comps = this.json.loadAll("components");
|
|
33
|
+
const insComp = db.prepare(`INSERT INTO components VALUES (@id,@kind,@name,@responsibility,@paths,@status,@owners,@fragility,@ps,@pc,@pe,@created_at,@updated_at)`);
|
|
34
|
+
for (const c of comps) {
|
|
35
|
+
insComp.run({
|
|
36
|
+
id: c.id, kind: c.kind, name: c.name, responsibility: c.responsibility,
|
|
37
|
+
paths: JSON.stringify(c.paths), status: c.status, owners: JSON.stringify(c.owners),
|
|
38
|
+
fragility: c.fragility, ps: c.provenance.source, pc: c.provenance.confidence,
|
|
39
|
+
pe: JSON.stringify(c.provenance.evidence), created_at: c.created_at, updated_at: c.updated_at,
|
|
40
|
+
});
|
|
41
|
+
this.fts(db, c.id, "components", c.name, `${c.responsibility} ${c.paths.join(" ")}`);
|
|
42
|
+
}
|
|
43
|
+
counts.components = comps.length;
|
|
44
|
+
const edges = this.json.loadAll("edges");
|
|
45
|
+
const insEdge = db.prepare(`INSERT INTO edges VALUES (@id,@from,@to,@type,@reason,@strength,@ps,@pc,@pe)`);
|
|
46
|
+
for (const e of edges) {
|
|
47
|
+
insEdge.run({ id: e.id, from: e.from, to: e.to, type: e.type, reason: e.reason, strength: e.strength,
|
|
48
|
+
ps: e.provenance.source, pc: e.provenance.confidence, pe: JSON.stringify(e.provenance.evidence) });
|
|
49
|
+
}
|
|
50
|
+
counts.edges = edges.length;
|
|
51
|
+
const syms = this.json.loadAll("symbols");
|
|
52
|
+
const insSym = db.prepare(`INSERT INTO symbols VALUES (@id,@file,@name,@kind,@sh,@calls,@called_by,@loc,@churn,@bug,@fanin,@fanout,@last)`);
|
|
53
|
+
for (const s of syms) {
|
|
54
|
+
insSym.run({ id: s.id, file: s.file, name: s.name, kind: s.kind, sh: s.signature_hash,
|
|
55
|
+
calls: JSON.stringify(s.calls), called_by: JSON.stringify(s.called_by),
|
|
56
|
+
loc: s.metrics.loc, churn: s.metrics.churn_90d, bug: s.metrics.bug_count,
|
|
57
|
+
fanin: s.metrics.fan_in, fanout: s.metrics.fan_out, last: s.last_changed });
|
|
58
|
+
this.fts(db, s.id, "symbols", `${s.name} (${s.kind})`, s.file);
|
|
59
|
+
}
|
|
60
|
+
counts.symbols = syms.length;
|
|
61
|
+
const decs = this.json.loadAll("decisions");
|
|
62
|
+
const insDec = db.prepare(`INSERT INTO decisions VALUES (@id,@title,@status,@context,@decision,@cons,@alts,@rc,@rf,@sup,@cbb,@commit,@ps,@pc,@pe,@date)`);
|
|
63
|
+
for (const d of decs) {
|
|
64
|
+
insDec.run({ id: d.id, title: d.title, status: d.status, context: d.context, decision: d.decision,
|
|
65
|
+
cons: JSON.stringify(d.consequences), alts: JSON.stringify(d.alternatives_rejected),
|
|
66
|
+
rc: JSON.stringify(d.related_components), rf: JSON.stringify(d.related_files),
|
|
67
|
+
sup: d.supersedes, cbb: d.caused_by_bug, commit: d.commit,
|
|
68
|
+
ps: d.provenance.source, pc: d.provenance.confidence, pe: JSON.stringify(d.provenance.evidence), date: d.date });
|
|
69
|
+
this.fts(db, d.id, "decisions", d.title, `${d.context} ${d.decision} ${d.consequences.join(" ")}`);
|
|
70
|
+
}
|
|
71
|
+
counts.decisions = decs.length;
|
|
72
|
+
const bugs = this.json.loadAll("bugs");
|
|
73
|
+
const insBug = db.prepare(`INSERT INTO bugs VALUES (@id,@title,@symptom,@rc,@sev,@status,@af,@as,@lin,@ps,@pc,@pe)`);
|
|
74
|
+
for (const b of bugs) {
|
|
75
|
+
insBug.run({ id: b.id, title: b.title, symptom: b.symptom, rc: b.root_cause, sev: b.severity, status: b.status,
|
|
76
|
+
af: JSON.stringify(b.affected_files), as: JSON.stringify(b.affected_symbols), lin: JSON.stringify(b.lineage),
|
|
77
|
+
ps: b.provenance.source, pc: b.provenance.confidence, pe: JSON.stringify(b.provenance.evidence) });
|
|
78
|
+
this.fts(db, b.id, "bugs", b.title, `${b.symptom} ${b.root_cause}`);
|
|
79
|
+
}
|
|
80
|
+
counts.bugs = bugs.length;
|
|
81
|
+
const cons = this.json.loadAll("constraints");
|
|
82
|
+
const insCon = db.prepare(`INSERT INTO constraints VALUES (@id,@type,@statement,@scope,@sev,@enf,@rat,@sd,@viol,@ps,@pc,@pe)`);
|
|
83
|
+
for (const c of cons) {
|
|
84
|
+
insCon.run({ id: c.id, type: c.type, statement: c.statement, scope: JSON.stringify(c.scope),
|
|
85
|
+
sev: c.severity, enf: c.enforcement, rat: c.rationale, sd: c.source_decision, viol: JSON.stringify(c.violations),
|
|
86
|
+
ps: c.provenance.source, pc: c.provenance.confidence, pe: JSON.stringify(c.provenance.evidence) });
|
|
87
|
+
this.fts(db, c.id, "constraints", c.statement, `${c.rationale} ${c.scope.join(" ")}`);
|
|
88
|
+
}
|
|
89
|
+
counts.constraints = cons.length;
|
|
90
|
+
void j;
|
|
91
|
+
});
|
|
92
|
+
tx();
|
|
93
|
+
// Reconcile embeddings AFTER the FTS rebuild (model-free): drop vectors whose
|
|
94
|
+
// source doc vanished or whose text changed. Embeddings are NOT in RESET_SQL,
|
|
95
|
+
// so this is what keeps them coherent across the many reindex() call sites.
|
|
96
|
+
this.pruneStaleEmbeddings();
|
|
97
|
+
return { counts };
|
|
98
|
+
}
|
|
99
|
+
fts(db, ref, kind, title, body) {
|
|
100
|
+
db.prepare(`INSERT INTO search (ref, kind, title, body) VALUES (?,?,?,?)`).run(ref, kind, title, body ?? "");
|
|
101
|
+
}
|
|
102
|
+
// ---- read path ----------------------------------------------------------
|
|
103
|
+
/** FTS5 ranked search (hunch_query). Falls back to LIKE if the query has no
|
|
104
|
+
* FTS-tokenizable terms. */
|
|
105
|
+
search(query, limit = 12) {
|
|
106
|
+
const match = toFtsQuery(query);
|
|
107
|
+
// No FTS-tokenizable terms (e.g. a CJK-only query) — degrade to LIKE rather
|
|
108
|
+
// than silently returning nothing (the documented fallback).
|
|
109
|
+
if (!match)
|
|
110
|
+
return this.likeSearch(query, limit);
|
|
111
|
+
try {
|
|
112
|
+
const rows = this.db.prepare(`SELECT ref, kind, title, snippet(search, 3, '[', ']', '…', 12) AS snip, bm25(search) AS score
|
|
113
|
+
FROM search WHERE search MATCH ? ORDER BY score LIMIT ?`).all(match, limit);
|
|
114
|
+
return rows.map((r) => ({ ref: r.ref, kind: r.kind, title: r.title, snippet: r.snip, score: r.score }));
|
|
115
|
+
}
|
|
116
|
+
catch {
|
|
117
|
+
// Malformed FTS expression — degrade to a LIKE scan over titles/bodies.
|
|
118
|
+
return this.likeSearch(query, limit);
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
/** Substring fallback over titles/bodies (handles non-ASCII / malformed FTS). */
|
|
122
|
+
likeSearch(query, limit) {
|
|
123
|
+
const like = `%${query.replace(/[%_]/g, "")}%`;
|
|
124
|
+
const rows = this.db.prepare(`SELECT ref, kind, title, substr(body,1,120) AS snip FROM search
|
|
125
|
+
WHERE title LIKE ? OR body LIKE ? LIMIT ?`).all(like, like, limit);
|
|
126
|
+
return rows.map((r) => ({ ref: r.ref, kind: r.kind, title: r.title, snippet: r.snip, score: 0 }));
|
|
127
|
+
}
|
|
128
|
+
// ---- semantic search (opt-in embeddings) --------------------------------
|
|
129
|
+
/** The exact (ref, kind, title, body) docs that feed FTS — and thus embeddings.
|
|
130
|
+
* A single source so FTS, the doc_hash, and the stored vectors never disagree. */
|
|
131
|
+
searchDocs() {
|
|
132
|
+
return this.db.prepare(`SELECT ref, kind, title, body FROM search`).all();
|
|
133
|
+
}
|
|
134
|
+
/** Delete embedding rows whose source doc was removed or whose text changed
|
|
135
|
+
* (doc_hash mismatch). Model-free and cheap; run at the end of every reindex()
|
|
136
|
+
* so vectors track the JSON truth without ever being reset. Returns the count. */
|
|
137
|
+
pruneStaleEmbeddings() {
|
|
138
|
+
const live = new Map(); // ref -> current doc_hash
|
|
139
|
+
for (const d of this.searchDocs())
|
|
140
|
+
live.set(d.ref, embedHash(d.title, d.body));
|
|
141
|
+
const rows = this.db.prepare(`SELECT ref, doc_hash FROM embeddings`).all();
|
|
142
|
+
const del = this.db.prepare(`DELETE FROM embeddings WHERE ref = ?`);
|
|
143
|
+
let pruned = 0;
|
|
144
|
+
const tx = this.db.transaction(() => {
|
|
145
|
+
for (const r of rows)
|
|
146
|
+
if (live.get(r.ref) !== r.doc_hash) {
|
|
147
|
+
del.run(r.ref);
|
|
148
|
+
pruned++;
|
|
149
|
+
}
|
|
150
|
+
});
|
|
151
|
+
tx();
|
|
152
|
+
return pruned;
|
|
153
|
+
}
|
|
154
|
+
/** Embedding coverage for a model: up-to-date vectors vs total docs (doctor). */
|
|
155
|
+
embeddingStats(model) {
|
|
156
|
+
const total = this.db.prepare(`SELECT count(*) c FROM search`).get().c;
|
|
157
|
+
const embedded = this.db.prepare(`SELECT count(*) c FROM embeddings WHERE model = ?`).get(model).c;
|
|
158
|
+
return { embedded, total };
|
|
159
|
+
}
|
|
160
|
+
/** Generate/refresh embeddings for every doc missing an up-to-date vector for
|
|
161
|
+
* this embedder's model. Batched + flushed per batch so a Ctrl-C leaves a
|
|
162
|
+
* coherent partial index that a re-run resumes. Assumes reindex() ran first. */
|
|
163
|
+
async embedAll(embedder, opts = {}) {
|
|
164
|
+
const model = embedder.id;
|
|
165
|
+
const current = new Map(); // ref -> stored doc_hash for this model
|
|
166
|
+
for (const r of this.db.prepare(`SELECT ref, doc_hash FROM embeddings WHERE model = ?`).all(model)) {
|
|
167
|
+
current.set(r.ref, r.doc_hash);
|
|
168
|
+
}
|
|
169
|
+
const docs = this.searchDocs().map((d) => ({ ...d, hash: embedHash(d.title, d.body) }));
|
|
170
|
+
const todo = docs.filter((d) => current.get(d.ref) !== d.hash);
|
|
171
|
+
const ins = this.db.prepare(`INSERT OR REPLACE INTO embeddings (ref, kind, model, dim, doc_hash, vec) VALUES (?,?,?,?,?,?)`);
|
|
172
|
+
const batchSize = opts.batch ?? 32;
|
|
173
|
+
let done = 0;
|
|
174
|
+
for (let i = 0; i < todo.length; i += batchSize) {
|
|
175
|
+
const slice = todo.slice(i, i + batchSize);
|
|
176
|
+
const vecs = await embedder.embed(slice.map((d) => `${d.title}\n${d.body}`));
|
|
177
|
+
const tx = this.db.transaction(() => {
|
|
178
|
+
slice.forEach((d, j) => {
|
|
179
|
+
const v = vecs[j];
|
|
180
|
+
if (v)
|
|
181
|
+
ins.run(d.ref, d.kind, model, embedder.dim, d.hash, vecToBlob(v));
|
|
182
|
+
});
|
|
183
|
+
});
|
|
184
|
+
tx();
|
|
185
|
+
done += slice.length;
|
|
186
|
+
opts.onProgress?.(done, todo.length);
|
|
187
|
+
}
|
|
188
|
+
return { embedded: todo.length, skipped: docs.length - todo.length, total: docs.length };
|
|
189
|
+
}
|
|
190
|
+
/** Hybrid search (hunch_query / `hunch query --semantic`): FTS bm25 fused with
|
|
191
|
+
* cosine over stored embeddings via Reciprocal Rank Fusion. Degrades to pure
|
|
192
|
+
* sync FTS (zero added latency) when there's no embedder or no vectors yet, so
|
|
193
|
+
* the lean install and fallback regressions are unaffected. Pass
|
|
194
|
+
* `embedder: null` to FORCE FTS-only without auto-selecting. */
|
|
195
|
+
async hybridSearch(query, limit = 12, opts = {}) {
|
|
196
|
+
const embedder = opts.embedder !== undefined ? opts.embedder : await selectEmbedder();
|
|
197
|
+
if (!embedder)
|
|
198
|
+
return this.search(query, limit);
|
|
199
|
+
const count = this.db.prepare(`SELECT count(*) c FROM embeddings WHERE model = ?`).get(embedder.id).c;
|
|
200
|
+
if (count === 0)
|
|
201
|
+
return this.search(query, limit);
|
|
202
|
+
const fts = this.search(query, Math.max(limit, 50));
|
|
203
|
+
let qvec;
|
|
204
|
+
try {
|
|
205
|
+
[qvec] = await embedder.embed([query]);
|
|
206
|
+
}
|
|
207
|
+
catch {
|
|
208
|
+
return fts.slice(0, limit); // embedding failed at query time → lexical only
|
|
209
|
+
}
|
|
210
|
+
if (!qvec)
|
|
211
|
+
return fts.slice(0, limit);
|
|
212
|
+
const sem = this.cosineRank(qvec, embedder.id, 50);
|
|
213
|
+
return this.rrfFuse(fts, sem, limit);
|
|
214
|
+
}
|
|
215
|
+
/** Brute-force exact cosine top-n over stored vectors for one model. Vectors are
|
|
216
|
+
* pre-normalized, so cosine == dot product. */
|
|
217
|
+
cosineRank(qvec, model, n) {
|
|
218
|
+
const rows = this.db.prepare(`SELECT ref, kind, vec FROM embeddings WHERE model = ?`).all(model);
|
|
219
|
+
const dim = qvec.length;
|
|
220
|
+
const scored = rows.map((r) => {
|
|
221
|
+
const v = blobToVec(r.vec, dim);
|
|
222
|
+
let dot = 0;
|
|
223
|
+
for (let i = 0; i < dim; i++)
|
|
224
|
+
dot += qvec[i] * v[i];
|
|
225
|
+
return { ref: r.ref, kind: r.kind, score: dot };
|
|
226
|
+
});
|
|
227
|
+
scored.sort((a, b) => b.score - a.score);
|
|
228
|
+
return scored.slice(0, n).map((s) => {
|
|
229
|
+
const row = this.db.prepare(`SELECT title, body FROM search WHERE ref = ?`).get(s.ref);
|
|
230
|
+
return { ref: s.ref, kind: s.kind, title: row?.title ?? s.ref, snippet: (row?.body ?? "").slice(0, 120), score: s.score };
|
|
231
|
+
});
|
|
232
|
+
}
|
|
233
|
+
/** Rank-based Reciprocal Rank Fusion of the FTS and semantic lists. Ranks (not
|
|
234
|
+
* raw scores) erase the bm25-vs-cosine scale mismatch; a small lexical weight
|
|
235
|
+
* keeps exact symbol/path matches from being displaced by paraphrase hits. */
|
|
236
|
+
rrfFuse(fts, sem, limit) {
|
|
237
|
+
const acc = new Map();
|
|
238
|
+
const add = (list, weight) => list.forEach((hit, i) => {
|
|
239
|
+
const e = acc.get(hit.ref) ?? { hit, score: 0 };
|
|
240
|
+
e.score += weight / (RRF_K + i + 1);
|
|
241
|
+
acc.set(hit.ref, e);
|
|
242
|
+
});
|
|
243
|
+
add(fts, RRF_W_FTS);
|
|
244
|
+
add(sem, RRF_W_SEM);
|
|
245
|
+
return [...acc.values()].sort((a, b) => b.score - a.score).slice(0, limit).map((e) => ({ ...e.hit, score: e.score }));
|
|
246
|
+
}
|
|
247
|
+
/** All decisions/bugs/constraints/symbols/components touching a file path or
|
|
248
|
+
* symbol name (hunch_why). */
|
|
249
|
+
why(target) {
|
|
250
|
+
const decisions = this.json.loadAll("decisions");
|
|
251
|
+
const bugs = this.json.loadAll("bugs");
|
|
252
|
+
const constraints = this.json.loadAll("constraints");
|
|
253
|
+
const symbols = this.json.loadAll("symbols");
|
|
254
|
+
const components = this.json.loadAll("components");
|
|
255
|
+
const matchedSymbols = symbols.filter((s) => s.file === target || s.name === target || s.id === target || s.file.endsWith(target));
|
|
256
|
+
const symIds = new Set(matchedSymbols.map((s) => s.id));
|
|
257
|
+
const fileSet = new Set(matchedSymbols.map((s) => s.file));
|
|
258
|
+
const isPath = target.includes("/") || target.includes(".");
|
|
259
|
+
const fileMatch = (files) => files.some((f) => f === target || (isPath && (f.endsWith(target) || target.endsWith(f))) || fileSet.has(f));
|
|
260
|
+
return {
|
|
261
|
+
target,
|
|
262
|
+
decisions: decisions.filter((d) => fileMatch(d.related_files) || d.related_components.some((c) => components.find((x) => x.id === c && fileMatch(x.paths)))),
|
|
263
|
+
bugs: bugs.filter((b) => fileMatch(b.affected_files) || b.affected_symbols.some((s) => symIds.has(s))),
|
|
264
|
+
constraints: constraints.filter((c) => c.scope.some((g) => pathMatchesGlob(target, g) || [...fileSet].some((f) => pathMatchesGlob(f, g)))),
|
|
265
|
+
symbols: matchedSymbols,
|
|
266
|
+
components: components.filter((c) => c.paths.some((g) => pathMatchesGlob(target, g) || [...fileSet].some((f) => pathMatchesGlob(f, g)))),
|
|
267
|
+
};
|
|
268
|
+
}
|
|
269
|
+
/** Transitive blast radius: every symbol/component that (in)directly depends on
|
|
270
|
+
* `id`, via a recursive CTE over the edges graph (hunch_get_dependents). We
|
|
271
|
+
* walk edges BACKWARD (edges.to = current) following call/dep/import/contains. */
|
|
272
|
+
getDependents(id, maxDepth = 6) {
|
|
273
|
+
const rows = this.db.prepare(
|
|
274
|
+
/* sql */ `
|
|
275
|
+
WITH RECURSIVE up(node, depth) AS (
|
|
276
|
+
SELECT ?, 0
|
|
277
|
+
UNION
|
|
278
|
+
SELECT e."from", up.depth + 1
|
|
279
|
+
FROM edges e JOIN up ON e."to" = up.node
|
|
280
|
+
WHERE e.type IN ('calls','depends_on','imports','contains') AND up.depth < ?
|
|
281
|
+
)
|
|
282
|
+
SELECT DISTINCT up.node AS id, MIN(up.depth) AS depth FROM up
|
|
283
|
+
WHERE up.node <> ? GROUP BY up.node ORDER BY depth, id`).all(id, maxDepth, id);
|
|
284
|
+
return rows.map((r) => ({ id: r.id, depth: r.depth, via: this.labelFor(r.id) }));
|
|
285
|
+
}
|
|
286
|
+
/** Symbols/components this id depends ON (forward walk) — used for refactor blast radius. */
|
|
287
|
+
getDependencies(id, maxDepth = 6) {
|
|
288
|
+
const rows = this.db.prepare(
|
|
289
|
+
/* sql */ `
|
|
290
|
+
WITH RECURSIVE down(node, depth) AS (
|
|
291
|
+
SELECT ?, 0
|
|
292
|
+
UNION
|
|
293
|
+
SELECT e."to", down.depth + 1
|
|
294
|
+
FROM edges e JOIN down ON e."from" = down.node
|
|
295
|
+
WHERE e.type IN ('calls','depends_on','imports','contains') AND down.depth < ?
|
|
296
|
+
)
|
|
297
|
+
SELECT DISTINCT down.node AS id, MIN(down.depth) AS depth FROM down
|
|
298
|
+
WHERE down.node <> ? GROUP BY down.node ORDER BY depth, id`).all(id, maxDepth, id);
|
|
299
|
+
return rows.map((r) => ({ id: r.id, depth: r.depth, via: this.labelFor(r.id) }));
|
|
300
|
+
}
|
|
301
|
+
labelFor(id) {
|
|
302
|
+
if (id.startsWith("sym_")) {
|
|
303
|
+
const r = this.db.prepare(`SELECT name, file FROM symbols WHERE id=?`).get(id);
|
|
304
|
+
return r ? `${r.name} @ ${r.file}` : id;
|
|
305
|
+
}
|
|
306
|
+
if (id.startsWith("cmp_")) {
|
|
307
|
+
const r = this.db.prepare(`SELECT name FROM components WHERE id=?`).get(id);
|
|
308
|
+
return r ? r.name : id;
|
|
309
|
+
}
|
|
310
|
+
return id;
|
|
311
|
+
}
|
|
312
|
+
/** Constraints whose scope glob matches a path/glob (hunch_check_constraints). */
|
|
313
|
+
checkConstraints(scope) {
|
|
314
|
+
const all = this.json.loadAll("constraints");
|
|
315
|
+
return all
|
|
316
|
+
.filter((c) => c.scope.some((g) => pathMatchesGlob(scope, g) || pathMatchesGlob(g, scope) || g === scope))
|
|
317
|
+
.sort((a, b) => sev(b.severity) - sev(a.severity));
|
|
318
|
+
}
|
|
319
|
+
/** Bugs matching a symptom (FTS over bugs) or a symbol, with lineage (hunch_bug_lineage). */
|
|
320
|
+
bugLineage(symptomOrSymbol) {
|
|
321
|
+
const bugs = this.json.loadAll("bugs");
|
|
322
|
+
const direct = bugs.filter((b) => b.affected_symbols.includes(symptomOrSymbol) || b.affected_files.includes(symptomOrSymbol));
|
|
323
|
+
if (direct.length)
|
|
324
|
+
return direct;
|
|
325
|
+
// fall back to fts over bug titles/symptoms
|
|
326
|
+
const hits = this.search(symptomOrSymbol).filter((h) => h.kind === "bugs").map((h) => h.ref);
|
|
327
|
+
const byHit = bugs.filter((b) => hits.includes(b.id));
|
|
328
|
+
if (byHit.length)
|
|
329
|
+
return byHit;
|
|
330
|
+
// last resort: naive substring over symptom/root_cause
|
|
331
|
+
const q = symptomOrSymbol.toLowerCase();
|
|
332
|
+
return bugs.filter((b) => `${b.title} ${b.symptom} ${b.root_cause}`.toLowerCase().includes(q));
|
|
333
|
+
}
|
|
334
|
+
/** Ranked fragility report (hunch fragile). fragility = weighted churn + bugs + fan-in. */
|
|
335
|
+
fragility(limit = 15) {
|
|
336
|
+
const syms = this.json.loadAll("symbols");
|
|
337
|
+
const bugs = this.json.loadAll("bugs");
|
|
338
|
+
// bug counts per symbol from actual bug records (authoritative over stale metric)
|
|
339
|
+
const bugBySym = new Map();
|
|
340
|
+
for (const b of bugs)
|
|
341
|
+
for (const s of b.affected_symbols)
|
|
342
|
+
bugBySym.set(s, (bugBySym.get(s) ?? 0) + 1);
|
|
343
|
+
const maxChurn = Math.max(1, ...syms.map((s) => s.metrics.churn_90d));
|
|
344
|
+
const maxFanIn = Math.max(1, ...syms.map((s) => s.metrics.fan_in));
|
|
345
|
+
const scored = syms.map((s) => {
|
|
346
|
+
const bugCount = Math.max(s.metrics.bug_count, bugBySym.get(s.id) ?? 0);
|
|
347
|
+
const churnN = s.metrics.churn_90d / maxChurn;
|
|
348
|
+
const fanInN = s.metrics.fan_in / maxFanIn;
|
|
349
|
+
// weighted: bugs dominate, then churn, then centrality
|
|
350
|
+
const score = 0.5 * Math.min(1, bugCount / 3) + 0.3 * churnN + 0.2 * fanInN;
|
|
351
|
+
const evidence = [];
|
|
352
|
+
if (bugCount)
|
|
353
|
+
evidence.push(`${bugCount} bug(s)`);
|
|
354
|
+
if (s.metrics.churn_90d)
|
|
355
|
+
evidence.push(`churn ${s.metrics.churn_90d}/90d`);
|
|
356
|
+
if (s.metrics.fan_in)
|
|
357
|
+
evidence.push(`fan-in ${s.metrics.fan_in}`);
|
|
358
|
+
return { id: s.id, file: s.file, name: s.name, score: round(score), churn_90d: s.metrics.churn_90d,
|
|
359
|
+
bug_count: bugCount, fan_in: s.metrics.fan_in, evidence };
|
|
360
|
+
});
|
|
361
|
+
return scored.filter((s) => s.score > 0).sort((a, b) => b.score - a.score).slice(0, limit);
|
|
362
|
+
}
|
|
363
|
+
/** Convenience: load a single entity from JSON by id (any kind). */
|
|
364
|
+
resolve(id) {
|
|
365
|
+
for (const kind of ENTITY_KINDS) {
|
|
366
|
+
const rec = this.json.get(kind, id);
|
|
367
|
+
if (rec)
|
|
368
|
+
return { kind, record: rec };
|
|
369
|
+
}
|
|
370
|
+
return undefined;
|
|
371
|
+
}
|
|
372
|
+
/** All edges (for graph export). */
|
|
373
|
+
allEdges() {
|
|
374
|
+
return this.json.loadAll("edges");
|
|
375
|
+
}
|
|
376
|
+
/** Drift detection (DESIGN §9 "staleness kills trust"): a decision/constraint
|
|
377
|
+
* is STALE when a file in its scope changed AFTER it was last verified. The
|
|
378
|
+
* caller supplies `lastChange(file) -> ISO date | ""` (git-backed). */
|
|
379
|
+
staleness(lastChange) {
|
|
380
|
+
const out = [];
|
|
381
|
+
const check = (kind, id, files, verified) => {
|
|
382
|
+
if (!verified)
|
|
383
|
+
return; // never verified → not flagged as drift (it's just new)
|
|
384
|
+
const vt = Date.parse(verified);
|
|
385
|
+
if (Number.isNaN(vt))
|
|
386
|
+
return;
|
|
387
|
+
let newest = "";
|
|
388
|
+
for (const f of files) {
|
|
389
|
+
const d = lastChange(f);
|
|
390
|
+
if (d && Date.parse(d) > vt && d > newest)
|
|
391
|
+
newest = d;
|
|
392
|
+
}
|
|
393
|
+
if (newest)
|
|
394
|
+
out.push({ kind, id, last_verified: verified, changed_at: newest, files: files.slice(0, 8) });
|
|
395
|
+
};
|
|
396
|
+
for (const d of this.json.loadAll("decisions"))
|
|
397
|
+
check("decision", d.id, d.related_files, d.provenance.last_verified);
|
|
398
|
+
for (const c of this.json.loadAll("constraints"))
|
|
399
|
+
check("constraint", c.id, c.scope, c.provenance.last_verified);
|
|
400
|
+
return out.sort((a, b) => b.changed_at.localeCompare(a.changed_at));
|
|
401
|
+
}
|
|
402
|
+
/** The Context Assembler (DESIGN §2.1/§6): the MINIMAL relevant Hunch slice for
|
|
403
|
+
* a task on `target`, ordered by what matters most — invariants first, then the
|
|
404
|
+
* why, then blast radius and bug history — trimmed to a rough token budget. */
|
|
405
|
+
assembleContext(target, budget = 1500) {
|
|
406
|
+
const w = this.why(target);
|
|
407
|
+
const symIds = w.symbols.map((s) => s.id);
|
|
408
|
+
const blast = new Map();
|
|
409
|
+
for (const id of symIds) {
|
|
410
|
+
for (const d of this.getDependents(id)) {
|
|
411
|
+
const prev = blast.get(d.id);
|
|
412
|
+
if (!prev || d.depth < prev.depth)
|
|
413
|
+
blast.set(d.id, d); // keep the MIN depth across start symbols
|
|
414
|
+
}
|
|
415
|
+
}
|
|
416
|
+
const bugs = w.bugs.length ? w.bugs : this.bugLineage(target);
|
|
417
|
+
const ctx = {
|
|
418
|
+
target,
|
|
419
|
+
constraints: w.constraints.sort((a, b) => sev(b.severity) - sev(a.severity)),
|
|
420
|
+
decisions: w.decisions.sort((a, b) => (b.provenance.confidence ?? 0) - (a.provenance.confidence ?? 0)),
|
|
421
|
+
bugs,
|
|
422
|
+
blast_radius: [...blast.values()].sort((a, b) => a.depth - b.depth).slice(0, 12),
|
|
423
|
+
components: w.components,
|
|
424
|
+
budget_tokens: budget,
|
|
425
|
+
};
|
|
426
|
+
return ctx;
|
|
427
|
+
}
|
|
428
|
+
}
|
|
429
|
+
function sev(s) {
|
|
430
|
+
return { blocking: 3, warning: 2, advisory: 1 }[s] ?? 0;
|
|
431
|
+
}
|
|
432
|
+
function round(n) {
|
|
433
|
+
return Math.round(n * 100) / 100;
|
|
434
|
+
}
|
|
435
|
+
// --- semantic-search helpers ---------------------------------------------
|
|
436
|
+
/** RRF tuning (env-overridable). Lexical weight ≥ semantic so exact matches win
|
|
437
|
+
* ties while semantic adds paraphrase recall. */
|
|
438
|
+
const RRF_K = numEnv("HUNCH_RRF_K", 60);
|
|
439
|
+
const RRF_W_FTS = numEnv("HUNCH_RRF_W_FTS", 1);
|
|
440
|
+
const RRF_W_SEM = numEnv("HUNCH_RRF_W_SEM", 0.7);
|
|
441
|
+
function numEnv(name, dflt) {
|
|
442
|
+
const v = Number(process.env[name]);
|
|
443
|
+
return Number.isFinite(v) && v > 0 ? v : dflt;
|
|
444
|
+
}
|
|
445
|
+
/** Pack a vector's exact bytes for SQLite. Explicit offset+length so a SUBARRAY
|
|
446
|
+
* view (byteOffset != 0) writes only its slice, not the whole backing buffer.
|
|
447
|
+
* better-sqlite3 copies on bind, so the returned view never aliases the row. */
|
|
448
|
+
function vecToBlob(v) {
|
|
449
|
+
return Buffer.from(v.buffer, v.byteOffset, v.byteLength);
|
|
450
|
+
}
|
|
451
|
+
/** Decode a stored BLOB into an ALIGNED Float32Array — copy bytes into a fresh
|
|
452
|
+
* ArrayBuffer rather than viewing the (possibly mis-aligned, pooled) Buffer,
|
|
453
|
+
* which would throw RangeError on a non-4-multiple byteOffset. */
|
|
454
|
+
function blobToVec(buf, dim) {
|
|
455
|
+
const ab = new ArrayBuffer(dim * 4);
|
|
456
|
+
new Uint8Array(ab).set(new Uint8Array(buf.buffer, buf.byteOffset, dim * 4));
|
|
457
|
+
return new Float32Array(ab);
|
|
458
|
+
}
|
|
459
|
+
/** Turn a free-text question into a tolerant FTS5 MATCH expression: split on
|
|
460
|
+
* non-word chars, OR the terms with prefix matching. Returns null if empty. */
|
|
461
|
+
function toFtsQuery(q) {
|
|
462
|
+
// Unicode word chars so accented/non-Latin terms still tokenize for FTS.
|
|
463
|
+
const terms = q.toLowerCase().match(/[\p{L}\p{N}_]+/gu);
|
|
464
|
+
if (!terms || terms.length === 0)
|
|
465
|
+
return null;
|
|
466
|
+
// quote each term and add prefix '*' for partial matches; OR them so recall is high
|
|
467
|
+
return terms.map((t) => `"${t}"*`).join(" OR ");
|
|
468
|
+
}
|
|
469
|
+
//# sourceMappingURL=hunchStore.js.map
|