@djolex999/vir-cli 0.14.1 → 0.16.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +281 -264
- package/dist/cli/embedSetup.js +62 -0
- package/dist/cli/embedSetup.js.map +1 -0
- package/dist/cli/queries.js +44 -0
- package/dist/cli/queries.js.map +1 -0
- package/dist/cli.js +143 -20
- package/dist/cli.js.map +1 -1
- package/dist/config.js +10 -0
- package/dist/config.js.map +1 -1
- package/dist/diagnostics/doctor.js +79 -0
- package/dist/diagnostics/doctor.js.map +1 -1
- package/dist/mcp/server.js +19 -3
- package/dist/mcp/server.js.map +1 -1
- package/dist/pipeline/embeddingSweep.js +17 -11
- package/dist/pipeline/embeddingSweep.js.map +1 -1
- package/dist/pipeline/run.js +19 -7
- package/dist/pipeline/run.js.map +1 -1
- package/dist/pipeline/writer.js +43 -23
- package/dist/pipeline/writer.js.map +1 -1
- package/dist/search/embedder.js +75 -15
- package/dist/search/embedder.js.map +1 -1
- package/dist/search/localProvider.js +99 -0
- package/dist/search/localProvider.js.map +1 -0
- package/dist/search/provider.js +72 -0
- package/dist/search/provider.js.map +1 -0
- package/dist/search/queryLog.js +123 -0
- package/dist/search/queryLog.js.map +1 -0
- package/dist/search/retriever.js +61 -17
- package/dist/search/retriever.js.map +1 -1
- package/dist/search/thresholds.js +18 -0
- package/dist/search/thresholds.js.map +1 -0
- package/dist/state/db.js +65 -17
- package/dist/state/db.js.map +1 -1
- package/package.json +1 -1
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
import { spawnSync } from "node:child_process";
|
|
2
|
+
import { existsSync, mkdirSync, writeFileSync } from "node:fs";
|
|
3
|
+
import { join } from "node:path";
|
|
4
|
+
import { stdin, stdout } from "node:process";
|
|
5
|
+
import { createInterface } from "node:readline/promises";
|
|
6
|
+
import { LOCAL_PROVIDER_DIR, LocalProvider, buildSetupPlan, isLocalProviderInstalled, } from "../search/localProvider.js";
|
|
7
|
+
import * as ui from "../ui/display.js";
|
|
8
|
+
// `vir embed --setup`: the one-command activation path for semantic search
|
|
9
|
+
// without Ollama. States the disk cost and waits for consent BEFORE touching
|
|
10
|
+
// disk or network; --yes skips the prompt, never the cost statement.
|
|
11
|
+
export async function runEmbedSetup(yes) {
|
|
12
|
+
ui.header("embed --setup");
|
|
13
|
+
ui.blank();
|
|
14
|
+
const plan = buildSetupPlan(isLocalProviderInstalled(), false);
|
|
15
|
+
for (const l of plan.lines)
|
|
16
|
+
ui.line(ui.text(l));
|
|
17
|
+
ui.blank();
|
|
18
|
+
if (!yes) {
|
|
19
|
+
const rl = createInterface({ input: stdin, output: stdout });
|
|
20
|
+
const answer = (await rl.question("Proceed? [y/N] ")).trim().toLowerCase();
|
|
21
|
+
rl.close();
|
|
22
|
+
if (answer !== "y" && answer !== "yes") {
|
|
23
|
+
ui.line(ui.dim("aborted — nothing was installed"));
|
|
24
|
+
return;
|
|
25
|
+
}
|
|
26
|
+
}
|
|
27
|
+
if (plan.needsInstall) {
|
|
28
|
+
mkdirSync(LOCAL_PROVIDER_DIR, { recursive: true });
|
|
29
|
+
const pkgPath = join(LOCAL_PROVIDER_DIR, "package.json");
|
|
30
|
+
if (!existsSync(pkgPath)) {
|
|
31
|
+
writeFileSync(pkgPath, `${JSON.stringify({ name: "vir-embedder", private: true }, null, 2)}\n`);
|
|
32
|
+
}
|
|
33
|
+
ui.line(ui.dim("installing fastembed (npm output follows)…"));
|
|
34
|
+
const res = spawnSync("npm", [
|
|
35
|
+
"install",
|
|
36
|
+
"fastembed@^2",
|
|
37
|
+
"--prefix",
|
|
38
|
+
LOCAL_PROVIDER_DIR,
|
|
39
|
+
"--no-fund",
|
|
40
|
+
"--no-audit",
|
|
41
|
+
], { stdio: "inherit" });
|
|
42
|
+
if (res.error && res.error.code === "ENOENT") {
|
|
43
|
+
ui.row(ui.errorColor(ui.CROSS), ui.text("npm not found on PATH"));
|
|
44
|
+
ui.line(ui.dim("install Node.js/npm, then re-run vir embed --setup"));
|
|
45
|
+
process.exitCode = 1;
|
|
46
|
+
return;
|
|
47
|
+
}
|
|
48
|
+
if (res.status !== 0) {
|
|
49
|
+
ui.row(ui.errorColor(ui.CROSS), ui.text("npm install failed — see output above"));
|
|
50
|
+
process.exitCode = 1;
|
|
51
|
+
return;
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
ui.line(ui.dim("downloading the model and warming up (first embed)…"));
|
|
55
|
+
const t0 = Date.now();
|
|
56
|
+
const provider = new LocalProvider();
|
|
57
|
+
await provider.embedDoc("vir local embedding setup probe");
|
|
58
|
+
const secs = Math.round((Date.now() - t0) / 1000);
|
|
59
|
+
ui.row(ui.success(ui.CHECK), ui.text(`local provider ready: ${provider.modelName}, ${provider.dimensions}d (${secs}s)`));
|
|
60
|
+
ui.line(ui.dim("vir uses it automatically when Ollama is not running; `vir embed` backfills notes."));
|
|
61
|
+
}
|
|
62
|
+
//# sourceMappingURL=embedSetup.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"embedSetup.js","sourceRoot":"","sources":["../../src/cli/embedSetup.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,oBAAoB,CAAC;AAC/C,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AAC/D,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,cAAc,CAAC;AAC7C,OAAO,EAAE,eAAe,EAAE,MAAM,wBAAwB,CAAC;AACzD,OAAO,EACL,kBAAkB,EAClB,aAAa,EACb,cAAc,EACd,wBAAwB,GACzB,MAAM,4BAA4B,CAAC;AACpC,OAAO,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAEvC,2EAA2E;AAC3E,6EAA6E;AAC7E,qEAAqE;AACrE,MAAM,CAAC,KAAK,UAAU,aAAa,CAAC,GAAY;IAC9C,EAAE,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC;IAC3B,EAAE,CAAC,KAAK,EAAE,CAAC;IACX,MAAM,IAAI,GAAG,cAAc,CAAC,wBAAwB,EAAE,EAAE,KAAK,CAAC,CAAC;IAC/D,KAAK,MAAM,CAAC,IAAI,IAAI,CAAC,KAAK;QAAE,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;IAChD,EAAE,CAAC,KAAK,EAAE,CAAC;IAEX,IAAI,CAAC,GAAG,EAAE,CAAC;QACT,MAAM,EAAE,GAAG,eAAe,CAAC,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC;QAC7D,MAAM,MAAM,GAAG,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,iBAAiB,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;QAC3E,EAAE,CAAC,KAAK,EAAE,CAAC;QACX,IAAI,MAAM,KAAK,GAAG,IAAI,MAAM,KAAK,KAAK,EAAE,CAAC;YACvC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,GAAG,CAAC,iCAAiC,CAAC,CAAC,CAAC;YACnD,OAAO;QACT,CAAC;IACH,CAAC;IAED,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;QACtB,SAAS,CAAC,kBAAkB,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QACnD,MAAM,OAAO,GAAG,IAAI,CAAC,kBAAkB,EAAE,cAAc,CAAC,CAAC;QACzD,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,EAAE,CAAC;YACzB,aAAa,CACX,OAAO,EACP,GAAG,IAAI,CAAC,SAAS,CAAC,EAAE,IAAI,EAAE,cAAc,EAAE,OAAO,EAAE,IAAI,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,CACxE,CAAC;QACJ,CAAC;QACD,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,GAAG,CAAC,4CAA4C,CAAC,CAAC,CAAC;QAC9D,MAAM,GAAG,GAAG,SAAS,CACnB,KAAK,EACL;YACE,SAAS;YACT,cAAc;YACd,UAAU;YACV,kBAAkB;YAClB,WAAW;YACX,YAAY;SACb,EACD,EAAE,KAAK,EAAE,SAAS,EAAE,CACrB,CAAC;QACF,IAAI,GAAG,CAAC,KAAK,IAAK,GAAG,CAAC,KAA+B,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;YACxE,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,UAAU,CAAC,EAAE,CAAC,KAAK,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,uBAAuB,CAAC,CAAC,CAAC;YAClE,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,GAAG,CAAC,oDAAoD,CAAC,CAAC,CAAC;YACtE,OAAO,CAAC,QAAQ,GAAG,CAAC,CAAC;YACrB,OAAO;QACT,CAAC;QACD,IAAI,GAAG,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACrB,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,UAAU,CAAC,EAAE,CAAC,KAAK,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,uCAAuC,CAAC,CAAC,CAAC;YAClF,OAAO,CAAC,QAAQ,GAAG,CAAC,CAAC;YACrB,OAAO;QACT,CAAC;IACH,CAAC;IAED,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,GAAG,CAAC,qDAAqD,CAAC,CAAC,CAAC;IACvE,MAAM,EAAE,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACtB,MAAM,QAAQ,GAAG,IAAI,aAAa,EAAE,CAAC;IACrC,MAAM,QAAQ,CAAC,QAAQ,CAAC,iCAAiC,CAAC,CAAC;IAC3D,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,EAAE,CAAC,GAAG,IAAI,CAAC,CAAC;IAClD,EAAE,CAAC,GAAG,CACJ,EAAE,CAAC,OAAO,CAAC,EAAE,CAAC,KAAK,CAAC,EACpB,EAAE,CAAC,IAAI,CAAC,yBAAyB,QAAQ,CAAC,SAAS,KAAK,QAAQ,CAAC,UAAU,MAAM,IAAI,IAAI,CAAC,CAC3F,CAAC;IACF,EAAE,CAAC,IAAI,CACL,EAAE,CAAC,GAAG,CAAC,oFAAoF,CAAC,CAC7F,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
// Below this many logged queries, "never surfaced" is dominated by "never
|
|
2
|
+
// asked about" — a handful of queries can only touch a handful of notes, so
|
|
3
|
+
// presenting the rest as dead weight is noise dressed as signal. 20 queries
|
|
4
|
+
// at topK 5–8 give every genuinely useful note ~100+ chances to surface.
|
|
5
|
+
export const MIN_DEAD_WEIGHT_SAMPLE = 20;
|
|
6
|
+
export function buildQueriesReport(records, allSlugs, topN = 10) {
|
|
7
|
+
const surfaced = new Map();
|
|
8
|
+
let embedding = 0;
|
|
9
|
+
let tfidf = 0;
|
|
10
|
+
let degraded = 0;
|
|
11
|
+
for (const r of records) {
|
|
12
|
+
if (r.method === "embedding")
|
|
13
|
+
embedding += 1;
|
|
14
|
+
else
|
|
15
|
+
tfidf += 1;
|
|
16
|
+
if (r.degraded)
|
|
17
|
+
degraded += 1;
|
|
18
|
+
for (const h of r.hits) {
|
|
19
|
+
const s = surfaced.get(h.slug) ?? { count: 0, rankSum: 0 };
|
|
20
|
+
s.count += 1;
|
|
21
|
+
s.rankSum += h.rank;
|
|
22
|
+
surfaced.set(h.slug, s);
|
|
23
|
+
}
|
|
24
|
+
}
|
|
25
|
+
const topNotes = [...surfaced.entries()]
|
|
26
|
+
.map(([slug, s]) => ({
|
|
27
|
+
slug,
|
|
28
|
+
count: s.count,
|
|
29
|
+
avgRank: Math.round((s.rankSum / s.count) * 100) / 100,
|
|
30
|
+
}))
|
|
31
|
+
.sort((a, b) => b.count - a.count || a.avgRank - b.avgRank)
|
|
32
|
+
.slice(0, topN);
|
|
33
|
+
return {
|
|
34
|
+
total: records.length,
|
|
35
|
+
byMethod: { embedding, tfidf },
|
|
36
|
+
degraded,
|
|
37
|
+
degradedRate: records.length === 0 ? 0 : degraded / records.length,
|
|
38
|
+
topNotes,
|
|
39
|
+
deadWeight: records.length < MIN_DEAD_WEIGHT_SAMPLE
|
|
40
|
+
? null
|
|
41
|
+
: allSlugs.filter((s) => !surfaced.has(s)),
|
|
42
|
+
};
|
|
43
|
+
}
|
|
44
|
+
//# sourceMappingURL=queries.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"queries.js","sourceRoot":"","sources":["../../src/cli/queries.ts"],"names":[],"mappings":"AAEA,0EAA0E;AAC1E,4EAA4E;AAC5E,4EAA4E;AAC5E,yEAAyE;AACzE,MAAM,CAAC,MAAM,sBAAsB,GAAG,EAAE,CAAC;AAczC,MAAM,UAAU,kBAAkB,CAChC,OAAyB,EACzB,QAAkB,EAClB,IAAI,GAAG,EAAE;IAET,MAAM,QAAQ,GAAG,IAAI,GAAG,EAA8C,CAAC;IACvE,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,IAAI,QAAQ,GAAG,CAAC,CAAC;IACjB,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;QACxB,IAAI,CAAC,CAAC,MAAM,KAAK,WAAW;YAAE,SAAS,IAAI,CAAC,CAAC;;YACxC,KAAK,IAAI,CAAC,CAAC;QAChB,IAAI,CAAC,CAAC,QAAQ;YAAE,QAAQ,IAAI,CAAC,CAAC;QAC9B,KAAK,MAAM,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC;YACvB,MAAM,CAAC,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC;YAC3D,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC;YACb,CAAC,CAAC,OAAO,IAAI,CAAC,CAAC,IAAI,CAAC;YACpB,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;QAC1B,CAAC;IACH,CAAC;IAED,MAAM,QAAQ,GAAG,CAAC,GAAG,QAAQ,CAAC,OAAO,EAAE,CAAC;SACrC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QACnB,IAAI;QACJ,KAAK,EAAE,CAAC,CAAC,KAAK;QACd,OAAO,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,OAAO,GAAG,CAAC,CAAC,KAAK,CAAC,GAAG,GAAG,CAAC,GAAG,GAAG;KACvD,CAAC,CAAC;SACF,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,OAAO,GAAG,CAAC,CAAC,OAAO,CAAC;SAC1D,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;IAElB,OAAO;QACL,KAAK,EAAE,OAAO,CAAC,MAAM;QACrB,QAAQ,EAAE,EAAE,SAAS,EAAE,KAAK,EAAE;QAC9B,QAAQ;QACR,YAAY,EAAE,OAAO,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,GAAG,OAAO,CAAC,MAAM;QAClE,QAAQ;QACR,UAAU,EACR,OAAO,CAAC,MAAM,GAAG,sBAAsB;YACrC,CAAC,CAAC,IAAI;YACN,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;KAC/C,CAAC;AACJ,CAAC"}
|
package/dist/cli.js
CHANGED
|
@@ -29,9 +29,11 @@ import { composeFromSources, estimateComposeCostTokens, gatherSources, } from ".
|
|
|
29
29
|
import { computeCost } from "./cost/pricing.js";
|
|
30
30
|
import { countByCategory, summarizeAll, summarizeProject, } from "./pipeline/summarizer.js";
|
|
31
31
|
import { buildPeriodPrompt, estimatePeriodCostTokens, periodLabel, periodRange, selectNotesInPeriod, summarizePeriod, } from "./pipeline/periodSummary.js";
|
|
32
|
-
import {
|
|
32
|
+
import { embedNoteWithProvider, resolveEmbeddingProvider, } from "./search/provider.js";
|
|
33
33
|
import { acquireLock, LockHeldError, releaseLock } from "./pipeline/lock.js";
|
|
34
|
-
import {
|
|
34
|
+
import { loadIndex, searchWithOutcome, vaultRoot } from "./search/retriever.js";
|
|
35
|
+
import { QUERY_LOG_PATH, readQueryLog, recordQueryEvent, } from "./search/queryLog.js";
|
|
36
|
+
import { MIN_DEAD_WEIGHT_SAMPLE, buildQueriesReport } from "./cli/queries.js";
|
|
35
37
|
import { buildQueryResults, errorPayload } from "./output/json.js";
|
|
36
38
|
import { synthesize } from "./search/synthesizer.js";
|
|
37
39
|
import { runMcpServer } from "./mcp/server.js";
|
|
@@ -39,6 +41,7 @@ import { runReview } from "./cli/review.js";
|
|
|
39
41
|
import { runAction } from "./cli/runAction.js";
|
|
40
42
|
import { buildInitConfig } from "./cli/initConfig.js";
|
|
41
43
|
import { runReconcile } from "./cli/reconcile.js";
|
|
44
|
+
import { runEmbedSetup } from "./cli/embedSetup.js";
|
|
42
45
|
import { installToClaudeCode, isClaudeAvailable, isInstalled, uninstallFromClaudeCode, } from "./mcp/install.js";
|
|
43
46
|
import { install as installDaemon, status as daemonStatus, uninstall as uninstallDaemon, } from "./daemon/index.js";
|
|
44
47
|
import { StateDb } from "./state/db.js";
|
|
@@ -216,6 +219,54 @@ program
|
|
|
216
219
|
ui.line(` ${ui.dim(ui.BULLET)} ${ui.text(label.padEnd(42))} ${ui.dim(`${s.calls}×`)} ${ui.warn(ui.formatUsd(s.cost))}`);
|
|
217
220
|
}
|
|
218
221
|
}));
|
|
222
|
+
program
|
|
223
|
+
.command("queries")
|
|
224
|
+
.description("Report on logged retrievals from ~/.vir/queries.jsonl")
|
|
225
|
+
.option("--json", "Emit the full report as JSON for scripting")
|
|
226
|
+
.option("--top <n>", "How many top-surfaced notes to show (default 10)", "10")
|
|
227
|
+
.action(runAction(async (opts) => {
|
|
228
|
+
const cfg = loadConfig();
|
|
229
|
+
const topN = Math.max(1, Number(opts.top ?? "10") || 10);
|
|
230
|
+
const records = readQueryLog();
|
|
231
|
+
// loadIndex walks exactly the retrievable notes (skips summaries/,
|
|
232
|
+
// .rejected/, archived/, index/log) — the honest denominator for
|
|
233
|
+
// dead weight.
|
|
234
|
+
const allSlugs = loadIndex(cfg).map((d) => d.relPath.replace(/\.md$/, ""));
|
|
235
|
+
const report = buildQueriesReport(records, allSlugs, topN);
|
|
236
|
+
if (opts.json) {
|
|
237
|
+
// NOT a plugin contract — scripting output like `vir projects --json`.
|
|
238
|
+
process.stdout.write(JSON.stringify(report) + "\n");
|
|
239
|
+
return;
|
|
240
|
+
}
|
|
241
|
+
ui.header("queries");
|
|
242
|
+
ui.blank();
|
|
243
|
+
if (report.total === 0) {
|
|
244
|
+
ui.row(ui.warn(ui.WARN_GLYPH), ui.text("no logged queries yet"));
|
|
245
|
+
ui.line(ui.dim(` ${QUERY_LOG_PATH} fills as you run \`vir query\` or use the MCP tools`));
|
|
246
|
+
return;
|
|
247
|
+
}
|
|
248
|
+
ui.stat("queries", report.total);
|
|
249
|
+
ui.stat("method", `${report.byMethod.embedding} embedding · ${report.byMethod.tfidf} tfidf`);
|
|
250
|
+
ui.stat("degraded", `${report.degraded} (${Math.round(report.degradedRate * 100)}%)`, report.degraded > 0 ? ui.warn : undefined);
|
|
251
|
+
ui.blank();
|
|
252
|
+
ui.line(ui.dim(` top ${Math.min(topN, report.topNotes.length)} surfaced notes`));
|
|
253
|
+
for (const n of report.topNotes) {
|
|
254
|
+
ui.line(` ${ui.dim(ui.BULLET)} ${ui.text(n.slug.padEnd(52))} ${ui.dim(`${n.count}× · avg rank ${n.avgRank}`)}`);
|
|
255
|
+
}
|
|
256
|
+
ui.blank();
|
|
257
|
+
if (report.deadWeight === null) {
|
|
258
|
+
ui.line(ui.dim(` dead weight: suppressed — ${report.total} quer${report.total === 1 ? "y" : "ies"} logged, ${MIN_DEAD_WEIGHT_SAMPLE} needed before "never surfaced" means unused rather than unasked`));
|
|
259
|
+
}
|
|
260
|
+
else {
|
|
261
|
+
ui.line(ui.dim(` dead weight: ${report.deadWeight.length} of ${allSlugs.length} notes never surfaced`));
|
|
262
|
+
for (const slug of report.deadWeight.slice(0, topN)) {
|
|
263
|
+
ui.line(` ${ui.dim(ui.BULLET)} ${ui.muted(slug)}`);
|
|
264
|
+
}
|
|
265
|
+
if (report.deadWeight.length > topN) {
|
|
266
|
+
ui.line(ui.dim(` … ${report.deadWeight.length - topN} more (\`vir queries --json\` for all)`));
|
|
267
|
+
}
|
|
268
|
+
}
|
|
269
|
+
}));
|
|
219
270
|
program
|
|
220
271
|
.command("calibrate <sessionId>")
|
|
221
272
|
.description("Distill ONE session to stdout for A/B model comparison — never writes vault or DB")
|
|
@@ -643,25 +694,64 @@ program
|
|
|
643
694
|
}));
|
|
644
695
|
program
|
|
645
696
|
.command("embed")
|
|
646
|
-
.description("Generate
|
|
697
|
+
.description("Generate embeddings for distilled notes")
|
|
647
698
|
.option("--force", "Regenerate even if embedding already exists")
|
|
699
|
+
.option("--setup", "Install the local embedding provider (fastembed + bge-small) on demand")
|
|
700
|
+
.option("--yes", "Skip the --setup disk-cost confirmation")
|
|
648
701
|
.action(runAction(async (opts) => {
|
|
702
|
+
if (opts.setup) {
|
|
703
|
+
await runEmbedSetup(opts.yes === true);
|
|
704
|
+
return;
|
|
705
|
+
}
|
|
649
706
|
const cfg = loadConfig();
|
|
650
707
|
ui.header("embed");
|
|
651
708
|
ui.blank();
|
|
652
|
-
|
|
653
|
-
|
|
654
|
-
ui.
|
|
655
|
-
ui.line(ui.dim("
|
|
656
|
-
ui.line(ui.dim(" ollama serve"));
|
|
709
|
+
const provider = await resolveEmbeddingProvider(cfg.embeddingProvider);
|
|
710
|
+
if (!provider) {
|
|
711
|
+
ui.row(ui.errorColor(ui.CROSS), ui.text("no embedding provider available"));
|
|
712
|
+
ui.line(ui.dim(" vir embed --setup # local provider, no Ollama needed (~233 MB)"));
|
|
713
|
+
ui.line(ui.dim(" or: brew install ollama && ollama pull nomic-embed-text && ollama serve"));
|
|
657
714
|
process.exitCode = 1;
|
|
658
715
|
return;
|
|
659
716
|
}
|
|
717
|
+
const provenance = provider.provenance();
|
|
660
718
|
const db = new StateDb();
|
|
661
719
|
try {
|
|
662
720
|
const rows = db.listDistilled();
|
|
663
721
|
const root = join(cfg.vaultPath, cfg.outputDir);
|
|
664
|
-
const
|
|
722
|
+
const embeddedRows = [
|
|
723
|
+
...db.getEmbeddings(root),
|
|
724
|
+
...db.getArticleEmbeddings(),
|
|
725
|
+
...db.getTopicEmbeddings(root, cfg.topicsDir),
|
|
726
|
+
...db.getPdfEmbeddings(),
|
|
727
|
+
];
|
|
728
|
+
// Crossing a model boundary is an index invalidation, not a gap fill —
|
|
729
|
+
// it needs --force AND explicit consent, with the cost stated first.
|
|
730
|
+
// Without --force, mismatched rows stay excluded from vector search
|
|
731
|
+
// (counted, reported) and only NULL-embedding gaps are filled.
|
|
732
|
+
const mismatched = embeddedRows.filter((r) => r.embeddingModel !== provider.modelName).length;
|
|
733
|
+
if (mismatched > 0 && !opts.force) {
|
|
734
|
+
ui.row(ui.warn(ui.WARN_GLYPH), ui.text(`${mismatched} note(s) are embedded under a different model than the active one (${provider.modelName})`));
|
|
735
|
+
ui.line(ui.dim("they are excluded from vector search until re-embedded"));
|
|
736
|
+
ui.line(ui.dim("run `vir embed --force` to re-embed the whole index"));
|
|
737
|
+
}
|
|
738
|
+
if (mismatched > 0 && opts.force) {
|
|
739
|
+
const estSecs = Math.max(1, Math.round(rows.length * 0.15));
|
|
740
|
+
ui.row(ui.warn(ui.WARN_GLYPH), ui.text(`model boundary: re-embedding ${rows.length} note(s) under ${provider.modelName} (~${estSecs}s), replacing vectors from other models`));
|
|
741
|
+
if (!opts.yes) {
|
|
742
|
+
const rl = createInterface({ input: stdin, output: stdout });
|
|
743
|
+
const answer = (await rl.question("Proceed? [y/N] ")).trim().toLowerCase();
|
|
744
|
+
rl.close();
|
|
745
|
+
if (answer !== "y" && answer !== "yes") {
|
|
746
|
+
ui.line(ui.dim("aborted — index unchanged"));
|
|
747
|
+
return;
|
|
748
|
+
}
|
|
749
|
+
}
|
|
750
|
+
}
|
|
751
|
+
const existing = new Set(db
|
|
752
|
+
.getEmbeddings(root)
|
|
753
|
+
.filter((r) => r.embeddingModel === provider.modelName)
|
|
754
|
+
.map((r) => r.sessionId));
|
|
665
755
|
const target = opts.force
|
|
666
756
|
? rows
|
|
667
757
|
: rows.filter((r) => !existing.has(r.sessionId));
|
|
@@ -702,12 +792,12 @@ program
|
|
|
702
792
|
skipped += 1;
|
|
703
793
|
continue;
|
|
704
794
|
}
|
|
705
|
-
const vec = await
|
|
795
|
+
const vec = await embedNoteWithProvider(provider, r.content);
|
|
706
796
|
if (!vec) {
|
|
707
797
|
errors += 1;
|
|
708
798
|
continue;
|
|
709
799
|
}
|
|
710
|
-
db.storeEmbedding(r.sessionId, vec);
|
|
800
|
+
db.storeEmbedding(r.sessionId, vec, provenance);
|
|
711
801
|
embedded += 1;
|
|
712
802
|
sp.text = ui.dim(`embedding notes (${embedded}/${total})`);
|
|
713
803
|
}
|
|
@@ -716,12 +806,12 @@ program
|
|
|
716
806
|
skipped += 1;
|
|
717
807
|
continue;
|
|
718
808
|
}
|
|
719
|
-
const vec = await
|
|
809
|
+
const vec = await embedNoteWithProvider(provider, t.content);
|
|
720
810
|
if (!vec) {
|
|
721
811
|
errors += 1;
|
|
722
812
|
continue;
|
|
723
813
|
}
|
|
724
|
-
db.storeTopicEmbedding(t.id, vec);
|
|
814
|
+
db.storeTopicEmbedding(t.id, vec, provenance);
|
|
725
815
|
embedded += 1;
|
|
726
816
|
sp.text = ui.dim(`embedding notes (${embedded}/${total})`);
|
|
727
817
|
}
|
|
@@ -730,12 +820,12 @@ program
|
|
|
730
820
|
skipped += 1;
|
|
731
821
|
continue;
|
|
732
822
|
}
|
|
733
|
-
const vec = await
|
|
823
|
+
const vec = await embedNoteWithProvider(provider, a.content);
|
|
734
824
|
if (!vec) {
|
|
735
825
|
errors += 1;
|
|
736
826
|
continue;
|
|
737
827
|
}
|
|
738
|
-
db.storeArticleEmbedding(a.path, vec);
|
|
828
|
+
db.storeArticleEmbedding(a.path, vec, provenance);
|
|
739
829
|
embedded += 1;
|
|
740
830
|
sp.text = ui.dim(`embedding notes (${embedded}/${total})`);
|
|
741
831
|
}
|
|
@@ -744,12 +834,12 @@ program
|
|
|
744
834
|
skipped += 1;
|
|
745
835
|
continue;
|
|
746
836
|
}
|
|
747
|
-
const vec = await
|
|
837
|
+
const vec = await embedNoteWithProvider(provider, p.content);
|
|
748
838
|
if (!vec) {
|
|
749
839
|
errors += 1;
|
|
750
840
|
continue;
|
|
751
841
|
}
|
|
752
|
-
db.storePdfEmbedding(p.path, vec);
|
|
842
|
+
db.storePdfEmbedding(p.path, vec, provenance);
|
|
753
843
|
embedded += 1;
|
|
754
844
|
sp.text = ui.dim(`embedding notes (${embedded}/${total})`);
|
|
755
845
|
}
|
|
@@ -792,8 +882,18 @@ async function runQueryJson(question, limit) {
|
|
|
792
882
|
}
|
|
793
883
|
const db = new StateDb();
|
|
794
884
|
try {
|
|
795
|
-
const
|
|
796
|
-
const
|
|
885
|
+
const t0 = Date.now();
|
|
886
|
+
const outcome = await searchWithOutcome(cfg, db, question, limit);
|
|
887
|
+
recordQueryEvent({
|
|
888
|
+
logQueries: cfg.logQueries,
|
|
889
|
+
source: "cli",
|
|
890
|
+
query: question,
|
|
891
|
+
type: "all",
|
|
892
|
+
hits: outcome.hits,
|
|
893
|
+
search: outcome,
|
|
894
|
+
latencyMs: Date.now() - t0,
|
|
895
|
+
});
|
|
896
|
+
const results = buildQueryResults(outcome.hits, vaultRoot(cfg), cfg.topicsDir);
|
|
797
897
|
process.stdout.write(JSON.stringify(results) + "\n");
|
|
798
898
|
}
|
|
799
899
|
catch (err) {
|
|
@@ -829,6 +929,7 @@ program
|
|
|
829
929
|
// served the hits.
|
|
830
930
|
const sp = ui.spinner("searching vault").start();
|
|
831
931
|
let outcome;
|
|
932
|
+
const t0 = Date.now();
|
|
832
933
|
try {
|
|
833
934
|
outcome = await searchWithOutcome(cfg, db, question, 8);
|
|
834
935
|
sp.stop();
|
|
@@ -837,10 +938,28 @@ program
|
|
|
837
938
|
sp.fail(ui.errorColor(err.message));
|
|
838
939
|
return;
|
|
839
940
|
}
|
|
941
|
+
recordQueryEvent({
|
|
942
|
+
logQueries: cfg.logQueries,
|
|
943
|
+
source: "cli",
|
|
944
|
+
query: question,
|
|
945
|
+
type: "all",
|
|
946
|
+
hits: outcome.hits,
|
|
947
|
+
search: outcome,
|
|
948
|
+
latencyMs: Date.now() - t0,
|
|
949
|
+
});
|
|
840
950
|
const hits = outcome.hits;
|
|
841
951
|
if (outcome.degraded) {
|
|
842
952
|
ui.row(ui.warn(ui.WARN_GLYPH), ui.text(`embedding search failed — results are TF-IDF fallback (${outcome.embedError})`));
|
|
843
953
|
}
|
|
954
|
+
else if (outcome.noProvider) {
|
|
955
|
+
// A supported mode with a one-line pointer, not a warning — different
|
|
956
|
+
// state than a failed provider, different fix.
|
|
957
|
+
ui.line(ui.dim("no embedding provider — using keyword search"));
|
|
958
|
+
ui.line(ui.dim(" semantic search: `vir embed --setup` (one command, ~1 min, 233 MB), or install Ollama"));
|
|
959
|
+
}
|
|
960
|
+
if (outcome.excludedMismatched > 0) {
|
|
961
|
+
ui.row(ui.warn(ui.WARN_GLYPH), ui.text(`${outcome.excludedMismatched} note(s) embedded under a different model were excluded — run \`vir embed\` to re-embed them`));
|
|
962
|
+
}
|
|
844
963
|
if (hits.length === 0) {
|
|
845
964
|
ui.row(ui.warn(ui.WARN_GLYPH), ui.text("no documents matched"));
|
|
846
965
|
return;
|
|
@@ -861,7 +980,11 @@ program
|
|
|
861
980
|
ui.summary({
|
|
862
981
|
sources: { value: relevant.length, color: ui.info },
|
|
863
982
|
via: {
|
|
864
|
-
value: outcome.degraded
|
|
983
|
+
value: outcome.degraded
|
|
984
|
+
? "tfidf (embeddings failed)"
|
|
985
|
+
: outcome.noProvider
|
|
986
|
+
? "tfidf (no provider)"
|
|
987
|
+
: method,
|
|
865
988
|
color: outcome.degraded ? ui.warn : ui.accent,
|
|
866
989
|
},
|
|
867
990
|
searched: { value: totalNotes, color: ui.muted },
|