@wei840222/qmd 2026.8.23
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +1373 -0
- package/LICENSE +45 -0
- package/README.md +1439 -0
- package/THIRD_PARTY_NOTICES.md +31 -0
- package/bin/qmd +192 -0
- package/dist/ast.d.ts +65 -0
- package/dist/ast.js +334 -0
- package/dist/bench/bench.d.ts +35 -0
- package/dist/bench/bench.js +338 -0
- package/dist/bench/cjk-baseline.d.ts +36 -0
- package/dist/bench/cjk-baseline.js +111 -0
- package/dist/bench/fixture.d.ts +2 -0
- package/dist/bench/fixture.js +84 -0
- package/dist/bench/score.d.ts +38 -0
- package/dist/bench/score.js +107 -0
- package/dist/bench/types.d.ts +110 -0
- package/dist/bench/types.js +8 -0
- package/dist/cli/build-info.json +4 -0
- package/dist/cli/embed-lock.d.ts +24 -0
- package/dist/cli/embed-lock.js +94 -0
- package/dist/cli/embedding-owner.d.ts +10 -0
- package/dist/cli/embedding-owner.js +20 -0
- package/dist/cli/formatter.d.ts +120 -0
- package/dist/cli/formatter.js +355 -0
- package/dist/cli/mcp-pid.d.ts +25 -0
- package/dist/cli/mcp-pid.js +86 -0
- package/dist/cli/qmd.d.ts +72 -0
- package/dist/cli/qmd.js +4806 -0
- package/dist/cli/version.d.ts +42 -0
- package/dist/cli/version.js +80 -0
- package/dist/collections.d.ts +200 -0
- package/dist/collections.js +433 -0
- package/dist/db.d.ts +65 -0
- package/dist/db.js +143 -0
- package/dist/diagnostics.d.ts +62 -0
- package/dist/diagnostics.js +260 -0
- package/dist/embedding/config.d.ts +52 -0
- package/dist/embedding/config.js +229 -0
- package/dist/embedding/identity.d.ts +58 -0
- package/dist/embedding/identity.js +321 -0
- package/dist/embedding/local-identity.d.ts +1 -0
- package/dist/embedding/local-identity.js +15 -0
- package/dist/embedding/local.d.ts +34 -0
- package/dist/embedding/local.js +290 -0
- package/dist/embedding/openai.d.ts +79 -0
- package/dist/embedding/openai.js +477 -0
- package/dist/embedding/owner.d.ts +13 -0
- package/dist/embedding/owner.js +36 -0
- package/dist/embedding/provider.d.ts +68 -0
- package/dist/embedding/provider.js +16 -0
- package/dist/embedding/remote-chunking.d.ts +22 -0
- package/dist/embedding/remote-chunking.js +83 -0
- package/dist/embedding/remote-embedding.d.ts +15 -0
- package/dist/embedding/remote-embedding.js +77 -0
- package/dist/hybrid-llm.d.ts +18 -0
- package/dist/hybrid-llm.js +53 -0
- package/dist/index.d.ts +244 -0
- package/dist/index.js +418 -0
- package/dist/llm.d.ts +566 -0
- package/dist/llm.js +1847 -0
- package/dist/maintenance.d.ts +33 -0
- package/dist/maintenance.js +52 -0
- package/dist/mcp/origin-guard.d.ts +67 -0
- package/dist/mcp/origin-guard.js +137 -0
- package/dist/mcp/server.d.ts +116 -0
- package/dist/mcp/server.js +919 -0
- package/dist/paths.d.ts +1 -0
- package/dist/paths.js +4 -0
- package/dist/remote-llm.d.ts +52 -0
- package/dist/remote-llm.js +464 -0
- package/dist/search/cjk-analyzer.d.ts +33 -0
- package/dist/search/cjk-analyzer.js +158 -0
- package/dist/search/cjk-index.d.ts +104 -0
- package/dist/search/cjk-index.js +1031 -0
- package/dist/search/jieba-loader.d.ts +23 -0
- package/dist/search/jieba-loader.js +79 -0
- package/dist/search/query-expansion.d.ts +23 -0
- package/dist/search/query-expansion.js +43 -0
- package/dist/search/zh-dict.txt +624013 -0
- package/dist/store.d.ts +1218 -0
- package/dist/store.js +6076 -0
- package/dist/trust.d.ts +152 -0
- package/dist/trust.js +249 -0
- package/package.json +139 -0
- package/scripts/build.mjs +83 -0
- package/scripts/check-package-grammars.mjs +29 -0
- package/scripts/package-smoke.mjs +205 -0
- package/scripts/sync-zh-dict.mjs +187 -0
- package/scripts/test-all.mjs +45 -0
- package/skills/qmd/SKILL.md +324 -0
- package/skills/qmd/references/mcp-setup.md +119 -0
- package/skills/release/SKILL.md +141 -0
- package/skills/release/scripts/install-hooks.sh +38 -0
- package/skills/release/scripts/release-context.sh +129 -0
package/dist/cli/qmd.js
ADDED
|
@@ -0,0 +1,4806 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
import { isBun, openDatabase, loadSqliteVec } from "../db.js";
|
|
3
|
+
import fastGlob from "fast-glob";
|
|
4
|
+
import { spawn as nodeSpawn } from "child_process";
|
|
5
|
+
import { isQmdMcpPid, mcpDaemonStateFiles } from "./mcp-pid.js";
|
|
6
|
+
import { embedLockPathForDb, tryAcquireEmbedLock, EMBED_LOCK_BUSY_MESSAGE } from "./embed-lock.js";
|
|
7
|
+
import { fileURLToPath } from "url";
|
|
8
|
+
import { basename, dirname, join as pathJoin, relative as relativePath, resolve as pathResolve } from "path";
|
|
9
|
+
import { parseArgs } from "util";
|
|
10
|
+
import { readFileSync, readdirSync, realpathSync, statSync, existsSync, unlinkSync, writeFileSync, openSync, closeSync, mkdirSync, lstatSync, rmSync, symlinkSync, readlinkSync, copyFileSync } from "fs";
|
|
11
|
+
import { createInterface } from "readline/promises";
|
|
12
|
+
import { getPwd, getRealPath, isPathInsideDir, homedir, resolve, enableProductionMode, searchFTS, extractSnippet, getContextForFile, getContextForPath, listCollections, findSimilarFiles, findDocument, resolveCommaListName, matchFilesByGlob, getHashesNeedingEmbedding, clearAllEmbeddings, insertEmbedding, getStatus, hashContent, extractTitle, formatDocForEmbedding, getEmbeddingFingerprint, chunkDocumentByTokens, clearCache, getCacheKey, getCachedResult, setCachedResult, getIndexHealth, parseVirtualPath, buildVirtualPath, isVirtualPath, isDocid, resolveVirtualPath, toVirtualPath, insertContent, insertDocument, insertDocumentWithContent, findActiveDocument, findOrMigrateLegacyDocument, updateDocumentTitle, updateDocument, updateDocumentWithContent, deactivateDocument, getActiveDocumentPaths, cleanupOrphanedContent, countOrphanedVectors, previewCleanup, runCleanup, getCollectionsWithoutContext, getTopLevelPathsWithoutContext, handelize, escapeLikePattern, hybridQuery, vectorSearchQuery, structuredSearch, addLineNumbers, DEFAULT_EMBED_MODEL, DEFAULT_EMBED_MAX_BATCH_BYTES, DEFAULT_EMBED_MAX_DOCS_PER_BATCH, DEFAULT_RERANK_MODEL, DEFAULT_QUERY_MODEL, DEFAULT_GLOB, splitGlobMask, DEFAULT_MULTI_GET_MAX_BYTES, createStore, getDefaultDbPath, reindexCollection, generateEmbeddings, getPendingEmbeddingDocsReadOnly, syncConfigToDb, } from "../store.js";
|
|
13
|
+
import { disposeDefaultLlamaCpp, getDefaultLlamaCpp, setDefaultLlamaCpp, LlamaCpp, withLLMSession, pullModels, DEFAULT_MODEL_CACHE_DIR, resolveEmbedModel, resolveGenerateModel, resolveRerankModel, resolveModels, inspectGgufFile, isDarwinMetalMitigationActive } from "../llm.js";
|
|
14
|
+
import { rebuildCjkLexicalIndex } from "../search/cjk-index.js";
|
|
15
|
+
import { RemoteLLM } from "../remote-llm.js";
|
|
16
|
+
import { HybridLLM } from "../hybrid-llm.js";
|
|
17
|
+
import { EmbeddingConfigError, OPENAI_EMBEDDING_MODEL, readCanonicalEmbeddingConfig, resolveEmbeddingConfig, writeCanonicalEmbeddingConfig, } from "../embedding/config.js";
|
|
18
|
+
import { OpenAIEmbeddingProvider, UnavailableOpenAIEmbeddingProvider, } from "../embedding/openai.js";
|
|
19
|
+
import { createCliEmbeddingProviderOwner } from "./embedding-owner.js";
|
|
20
|
+
import { readStoredEmbeddingIdentity } from "../embedding/identity.js";
|
|
21
|
+
import { authorizeRemoteEmbeddingRequest, remoteEmbeddingIdentity, } from "../embedding/remote-embedding.js";
|
|
22
|
+
import { inspectIndexDiagnostics } from "../diagnostics.js";
|
|
23
|
+
import { formatSearchResults, formatDocuments, escapeXml, escapeCSV, } from "./formatter.js";
|
|
24
|
+
import { resolveCommit } from "./version.js";
|
|
25
|
+
import { getCollection as getCollectionFromYaml, listCollections as yamlListCollections, getDefaultCollectionNames, addContext as yamlAddContext, removeContext as yamlRemoveContext, removeCollection as yamlRemoveCollectionFn, renameCollection as yamlRenameCollectionFn, setGlobalContext, listAllContexts, setConfigIndexName, loadConfig, saveConfig, setConfigSource, findLocalConfigPath, getLocalDbPath, getConfigPath, configExists, } from "../collections.js";
|
|
26
|
+
import { decideLocalConfigGate, gatedItems, hasGatedItems, isCollectionPathInsideProject, isLocalConfigPath, isLocalConfigTrustOptedIn, isTrusted, listTrusted, recordTrust, revokeTrust, sensitiveDigest, } from "../trust.js";
|
|
27
|
+
// NOTE: enableProductionMode() is intentionally NOT called at module scope here.
|
|
28
|
+
// Importing this module for its exports (e.g. buildEditorUri, termLink from
|
|
29
|
+
// test/cli.test.ts) must not flip the global production flag, as that leaks
|
|
30
|
+
// into unrelated tests that rely on the default (development) database path
|
|
31
|
+
// resolution. The flag is flipped inside the CLI's main-module guard below so
|
|
32
|
+
// it only fires when qmd is actually invoked as a script.
|
|
33
|
+
// =============================================================================
|
|
34
|
+
// Store/DB lifecycle (no legacy singletons in store.ts)
|
|
35
|
+
// =============================================================================
|
|
36
|
+
let store = null;
|
|
37
|
+
let cliEmbeddingOwner = null;
|
|
38
|
+
let storeDbPathOverride;
|
|
39
|
+
let currentIndexName = "index";
|
|
40
|
+
function getStore() {
|
|
41
|
+
if (!store) {
|
|
42
|
+
store = createStore(storeDbPathOverride);
|
|
43
|
+
let cliLlama;
|
|
44
|
+
let config;
|
|
45
|
+
try {
|
|
46
|
+
config = loadConfig();
|
|
47
|
+
syncConfigToDb(store.db, config);
|
|
48
|
+
const dbEmbeddingConfig = readCanonicalEmbeddingConfig(store.db);
|
|
49
|
+
const activeModels = ensureModelsConfiguredForCli();
|
|
50
|
+
const modelsForLlm = localConfigIsFullyTrusted() ? activeModels : resolveModels();
|
|
51
|
+
const embedding = resolveEmbeddingConfig({
|
|
52
|
+
config,
|
|
53
|
+
dbConfig: dbEmbeddingConfig,
|
|
54
|
+
env: process.env,
|
|
55
|
+
defaultLocalModel: modelsForLlm.embed,
|
|
56
|
+
});
|
|
57
|
+
writeCanonicalEmbeddingConfig(store.db, embedding.canonical);
|
|
58
|
+
cliLlama = new LlamaCpp({
|
|
59
|
+
embedModel: embedding.canonical.provider === "local"
|
|
60
|
+
? embedding.canonical.model
|
|
61
|
+
: modelsForLlm.embed,
|
|
62
|
+
generateModel: modelsForLlm.generate,
|
|
63
|
+
rerankModel: modelsForLlm.rerank,
|
|
64
|
+
});
|
|
65
|
+
setDefaultLlamaCpp(cliLlama);
|
|
66
|
+
if (embedding.canonical.provider === "openai") {
|
|
67
|
+
const apiKey = config?.models?.embed_api_key?.trim() ||
|
|
68
|
+
process.env.OPENAI_API_KEY?.trim();
|
|
69
|
+
const configuredModel = embedding.canonical.model;
|
|
70
|
+
const configuredDimension = embedding.canonical.dimension;
|
|
71
|
+
const configuredBaseUrl = embedding.canonical.baseUrl;
|
|
72
|
+
const canConstruct = (apiKey != null && apiKey !== "") || embedding.credentialAvailable;
|
|
73
|
+
const provider = canConstruct
|
|
74
|
+
? new OpenAIEmbeddingProvider({
|
|
75
|
+
apiKey: apiKey || undefined,
|
|
76
|
+
model: configuredModel,
|
|
77
|
+
dimension: configuredDimension,
|
|
78
|
+
baseUrl: configuredBaseUrl,
|
|
79
|
+
authorizeRequest: request => {
|
|
80
|
+
const activeProvider = store?.embeddingProvider;
|
|
81
|
+
if (!activeProvider?.remote) {
|
|
82
|
+
throw new EmbeddingConfigError("Remote embedding provider is not active.");
|
|
83
|
+
}
|
|
84
|
+
const storedIdentity = readStoredEmbeddingIdentity(store.db);
|
|
85
|
+
const requestIdentity = [
|
|
86
|
+
storedIdentity,
|
|
87
|
+
remoteEmbeddingIdentity(activeProvider, "regex"),
|
|
88
|
+
remoteEmbeddingIdentity(activeProvider, "auto"),
|
|
89
|
+
].find(identity => identity?.fingerprint === request.fingerprint);
|
|
90
|
+
if (!requestIdentity) {
|
|
91
|
+
throw new EmbeddingConfigError("Remote request fingerprint does not match an active embedding identity.");
|
|
92
|
+
}
|
|
93
|
+
authorizeRemoteEmbeddingRequest(store.db, requestIdentity, request.purpose, {
|
|
94
|
+
lease: request.buildLease,
|
|
95
|
+
requestFingerprint: request.fingerprint,
|
|
96
|
+
});
|
|
97
|
+
},
|
|
98
|
+
})
|
|
99
|
+
: new UnavailableOpenAIEmbeddingProvider({ model: configuredModel, dimension: configuredDimension, baseUrl: configuredBaseUrl });
|
|
100
|
+
cliEmbeddingOwner = createCliEmbeddingProviderOwner(embedding.canonical, cliLlama, provider);
|
|
101
|
+
store.embeddingProvider = provider;
|
|
102
|
+
store.authorizeRemoteRequest = (purpose, context) => {
|
|
103
|
+
authorizeRemoteEmbeddingRequest(store.db, context.identity ?? remoteEmbeddingIdentity(provider), purpose, { lease: context.lease });
|
|
104
|
+
};
|
|
105
|
+
store.authorizeRemoteBuildStart = identity => {
|
|
106
|
+
authorizeRemoteEmbeddingRequest(store.db, identity, "capability-probe");
|
|
107
|
+
};
|
|
108
|
+
}
|
|
109
|
+
else {
|
|
110
|
+
cliEmbeddingOwner = createCliEmbeddingProviderOwner(embedding.canonical, cliLlama);
|
|
111
|
+
store.embeddingProvider = cliEmbeddingOwner.provider;
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
catch {
|
|
115
|
+
// Config may not exist yet — that's fine, DB works without it
|
|
116
|
+
}
|
|
117
|
+
const rawGenerateUrl = config?.models?.generate_url ?? config?.models?.generate_base_url ?? config?.models?.generate_api_url;
|
|
118
|
+
const rawRerankUrl = config?.models?.rerank_url ?? config?.models?.rerank_base_url ?? config?.models?.rerank_api_url;
|
|
119
|
+
const hasRemoteLLM = Boolean(rawGenerateUrl || rawRerankUrl);
|
|
120
|
+
const remoteLlm = hasRemoteLLM
|
|
121
|
+
? new RemoteLLM({
|
|
122
|
+
generateUrl: config?.models?.generate_url,
|
|
123
|
+
generateBaseUrl: config?.models?.generate_base_url,
|
|
124
|
+
generateApiUrl: config?.models?.generate_api_url,
|
|
125
|
+
generateApiModel: config?.models?.generate_api_model,
|
|
126
|
+
generateApiKey: config?.models?.generate_api_key,
|
|
127
|
+
rerankUrl: config?.models?.rerank_url,
|
|
128
|
+
rerankBaseUrl: config?.models?.rerank_base_url,
|
|
129
|
+
rerankApiUrl: config?.models?.rerank_api_url,
|
|
130
|
+
rerankApiModel: config?.models?.rerank_api_model,
|
|
131
|
+
rerankApiKey: config?.models?.rerank_api_key,
|
|
132
|
+
})
|
|
133
|
+
: undefined;
|
|
134
|
+
if (cliLlama) {
|
|
135
|
+
store.llm = remoteLlm ? new HybridLLM(cliLlama, remoteLlm) : cliLlama;
|
|
136
|
+
}
|
|
137
|
+
}
|
|
138
|
+
return store;
|
|
139
|
+
}
|
|
140
|
+
function getDoctorStore() {
|
|
141
|
+
if (!store) {
|
|
142
|
+
const dbPath = getDbPath();
|
|
143
|
+
store = existsSync(dbPath)
|
|
144
|
+
? createStore(dbPath, { readOnly: true })
|
|
145
|
+
: createStore(":memory:");
|
|
146
|
+
}
|
|
147
|
+
return store;
|
|
148
|
+
}
|
|
149
|
+
function getDb() {
|
|
150
|
+
return getStore().db;
|
|
151
|
+
}
|
|
152
|
+
/** Re-sync YAML config into SQLite after CLI mutations (add/remove/rename collection, context changes) */
|
|
153
|
+
function resyncConfig() {
|
|
154
|
+
const s = getStore();
|
|
155
|
+
syncConfigToDb(s.db, loadConfig());
|
|
156
|
+
}
|
|
157
|
+
function closeDb() {
|
|
158
|
+
// A provider owner may still have queued work that depends on this store.
|
|
159
|
+
// The common CLI cleanup closes provider/runtime before closing the database.
|
|
160
|
+
if (cliEmbeddingOwner)
|
|
161
|
+
return;
|
|
162
|
+
if (store) {
|
|
163
|
+
store.close();
|
|
164
|
+
store = null;
|
|
165
|
+
}
|
|
166
|
+
}
|
|
167
|
+
export function createCliResourceCloser(options) {
|
|
168
|
+
let closePromise = null;
|
|
169
|
+
return () => {
|
|
170
|
+
if (!closePromise) {
|
|
171
|
+
closePromise = (async () => {
|
|
172
|
+
const owner = options.getOwner();
|
|
173
|
+
try {
|
|
174
|
+
if (owner)
|
|
175
|
+
await owner.close();
|
|
176
|
+
else
|
|
177
|
+
await options.disposeFallback();
|
|
178
|
+
}
|
|
179
|
+
finally {
|
|
180
|
+
options.clearOwner();
|
|
181
|
+
options.closeStore();
|
|
182
|
+
}
|
|
183
|
+
})();
|
|
184
|
+
}
|
|
185
|
+
return closePromise;
|
|
186
|
+
};
|
|
187
|
+
}
|
|
188
|
+
const closeCliResources = createCliResourceCloser({
|
|
189
|
+
getOwner: () => cliEmbeddingOwner,
|
|
190
|
+
clearOwner: () => { cliEmbeddingOwner = null; },
|
|
191
|
+
disposeFallback: disposeDefaultLlamaCpp,
|
|
192
|
+
closeStore: () => {
|
|
193
|
+
if (store) {
|
|
194
|
+
store.close();
|
|
195
|
+
store = null;
|
|
196
|
+
}
|
|
197
|
+
},
|
|
198
|
+
});
|
|
199
|
+
function getDbPath() {
|
|
200
|
+
return store?.dbPath ?? storeDbPathOverride ?? getDefaultDbPath();
|
|
201
|
+
}
|
|
202
|
+
function getActiveIndexName() {
|
|
203
|
+
return currentIndexName;
|
|
204
|
+
}
|
|
205
|
+
function mcpDaemonPaths() {
|
|
206
|
+
const cacheDir = process.env.XDG_CACHE_HOME
|
|
207
|
+
? resolve(process.env.XDG_CACHE_HOME, "qmd")
|
|
208
|
+
: resolve(homedir(), ".cache", "qmd");
|
|
209
|
+
const { pidFile, logFile } = mcpDaemonStateFiles(getActiveIndexName());
|
|
210
|
+
return {
|
|
211
|
+
cacheDir,
|
|
212
|
+
pidPath: resolve(cacheDir, pidFile),
|
|
213
|
+
logPath: resolve(cacheDir, logFile),
|
|
214
|
+
};
|
|
215
|
+
}
|
|
216
|
+
function setIndexName(name) {
|
|
217
|
+
let normalizedName = name;
|
|
218
|
+
// Normalize relative paths to prevent malformed database paths
|
|
219
|
+
if (name && name.includes('/')) {
|
|
220
|
+
const absolutePath = pathResolve(process.cwd(), name);
|
|
221
|
+
// Replace path separators with underscores to create a valid filename
|
|
222
|
+
normalizedName = absolutePath.replace(/\//g, '_').replace(/^_/, '');
|
|
223
|
+
}
|
|
224
|
+
currentIndexName = normalizedName || "index";
|
|
225
|
+
storeDbPathOverride = normalizedName ? getDefaultDbPath(normalizedName) : undefined;
|
|
226
|
+
// Reset open handle so next use opens the new index
|
|
227
|
+
closeDb();
|
|
228
|
+
}
|
|
229
|
+
function ensureVecTable(_db, dimensions) {
|
|
230
|
+
// Store owns the DB; ignore `_db` and ensure vec table on the active store
|
|
231
|
+
getStore().ensureVecTable(dimensions);
|
|
232
|
+
}
|
|
233
|
+
// Terminal colors (respects NO_COLOR env)
|
|
234
|
+
const useColor = !process.env.NO_COLOR && process.stdout.isTTY;
|
|
235
|
+
const c = {
|
|
236
|
+
reset: useColor ? "\x1b[0m" : "",
|
|
237
|
+
dim: useColor ? "\x1b[2m" : "",
|
|
238
|
+
bold: useColor ? "\x1b[1m" : "",
|
|
239
|
+
cyan: useColor ? "\x1b[36m" : "",
|
|
240
|
+
yellow: useColor ? "\x1b[33m" : "",
|
|
241
|
+
green: useColor ? "\x1b[32m" : "",
|
|
242
|
+
magenta: useColor ? "\x1b[35m" : "",
|
|
243
|
+
blue: useColor ? "\x1b[34m" : "",
|
|
244
|
+
};
|
|
245
|
+
// Terminal cursor control
|
|
246
|
+
const cursor = {
|
|
247
|
+
hide() { process.stderr.write('\x1b[?25l'); },
|
|
248
|
+
show() { process.stderr.write('\x1b[?25h'); },
|
|
249
|
+
};
|
|
250
|
+
async function flushWritable(stream) {
|
|
251
|
+
await new Promise((resolve) => {
|
|
252
|
+
stream.write("", () => resolve());
|
|
253
|
+
});
|
|
254
|
+
}
|
|
255
|
+
/**
|
|
256
|
+
* Finish a successful CLI command after output has been flushed.
|
|
257
|
+
*
|
|
258
|
+
* We deliberately do NOT call `process.exit(0)`. `process.exit()` skips
|
|
259
|
+
* Node's `beforeExit` event, and node-llama-cpp registers a `beforeExit` hook
|
|
260
|
+
* that auto-disposes its native handles. On darwin, without that hook firing,
|
|
261
|
+
* libggml-metal's static `ggml_metal_device` destructor asserts on a
|
|
262
|
+
* non-empty residency-set collection during `__cxa_finalize_ranges` and
|
|
263
|
+
* dumps a multi-kB backtrace (upstream ggml-org/llama.cpp#22593, fix open as
|
|
264
|
+
* PR #22595). Empirically, even with explicit `disposeDefaultLlamaCpp()` the
|
|
265
|
+
* direct `process.exit(0)` path still trips the assertion — letting the
|
|
266
|
+
* event loop drain naturally is what actually clears the rsets.
|
|
267
|
+
*
|
|
268
|
+
* So: set `process.exitCode = 0` and return. The main module finishes, the
|
|
269
|
+
* event loop drains, `beforeExit` fires, native resources tear down in
|
|
270
|
+
* order, and the process exits cleanly. The `GGML_METAL_NO_RESIDENCY=1` env
|
|
271
|
+
* var that `bin/qmd` exports is a defense-in-depth safety net for paths
|
|
272
|
+
* that still call `process.exit()` after loading the native binding
|
|
273
|
+
* (signal handlers, error paths, `bun test`).
|
|
274
|
+
*
|
|
275
|
+
* If the caller passes an explicit `exit` for testability, we honor it —
|
|
276
|
+
* the lifecycle tests verify the legacy flush → cleanup → exit ordering.
|
|
277
|
+
* Production callers must not pass `exit`.
|
|
278
|
+
*/
|
|
279
|
+
export async function finishSuccessfulCliCommand(options) {
|
|
280
|
+
const stderr = options.stderr ?? process.stderr;
|
|
281
|
+
await flushWritable(options.stdout ?? process.stdout);
|
|
282
|
+
try {
|
|
283
|
+
await (options.cleanup ?? disposeDefaultLlamaCpp)();
|
|
284
|
+
}
|
|
285
|
+
catch (error) {
|
|
286
|
+
stderr.write(`QMD Warning: cleanup after successful output failed (${error instanceof Error ? error.message : String(error)}); exiting 0 because command output completed.\n`);
|
|
287
|
+
}
|
|
288
|
+
await flushWritable(stderr);
|
|
289
|
+
if (options.exit) {
|
|
290
|
+
options.exit(0);
|
|
291
|
+
return;
|
|
292
|
+
}
|
|
293
|
+
process.exitCode = 0;
|
|
294
|
+
}
|
|
295
|
+
// Ensure cursor is restored on exit
|
|
296
|
+
process.on('SIGINT', () => { cursor.show(); process.exit(130); });
|
|
297
|
+
process.on('SIGTERM', () => { cursor.show(); process.exit(143); });
|
|
298
|
+
// Terminal progress bar using OSC 9;4 escape sequence (TTY only)
|
|
299
|
+
const isTTY = process.stderr.isTTY;
|
|
300
|
+
const progress = {
|
|
301
|
+
set(percent) {
|
|
302
|
+
if (isTTY)
|
|
303
|
+
process.stderr.write(`\x1b]9;4;1;${Math.round(percent)}\x07`);
|
|
304
|
+
},
|
|
305
|
+
clear() {
|
|
306
|
+
if (isTTY)
|
|
307
|
+
process.stderr.write(`\x1b]9;4;0\x07`);
|
|
308
|
+
},
|
|
309
|
+
indeterminate() {
|
|
310
|
+
if (isTTY)
|
|
311
|
+
process.stderr.write(`\x1b]9;4;3\x07`);
|
|
312
|
+
},
|
|
313
|
+
error() {
|
|
314
|
+
if (isTTY)
|
|
315
|
+
process.stderr.write(`\x1b]9;4;2\x07`);
|
|
316
|
+
},
|
|
317
|
+
};
|
|
318
|
+
// Format seconds into human-readable ETA
|
|
319
|
+
function formatETA(seconds) {
|
|
320
|
+
if (seconds < 60)
|
|
321
|
+
return `${Math.round(seconds)}s`;
|
|
322
|
+
if (seconds < 3600)
|
|
323
|
+
return `${Math.floor(seconds / 60)}m ${Math.round(seconds % 60)}s`;
|
|
324
|
+
return `${Math.floor(seconds / 3600)}h ${Math.floor((seconds % 3600) / 60)}m`;
|
|
325
|
+
}
|
|
326
|
+
// Check index health and print warnings/tips
|
|
327
|
+
function checkIndexHealth(db, model = resolveEmbedModelForCli()) {
|
|
328
|
+
const { needsEmbedding, totalDocs, daysStale } = getIndexHealth(db, model);
|
|
329
|
+
// Warn if many docs need embedding
|
|
330
|
+
if (needsEmbedding > 0) {
|
|
331
|
+
const pct = Math.round((needsEmbedding / totalDocs) * 100);
|
|
332
|
+
if (pct >= 10) {
|
|
333
|
+
process.stderr.write(`${c.yellow}Warning: ${needsEmbedding} documents (${pct}%) need embeddings. Run 'qmd embed' for better results.${c.reset}\n`);
|
|
334
|
+
}
|
|
335
|
+
else {
|
|
336
|
+
process.stderr.write(`${c.dim}Tip: ${needsEmbedding} documents need embeddings. Run 'qmd embed' to index them.${c.reset}\n`);
|
|
337
|
+
}
|
|
338
|
+
}
|
|
339
|
+
// Check if most recent document update is older than 2 weeks
|
|
340
|
+
if (daysStale !== null && daysStale >= 14) {
|
|
341
|
+
process.stderr.write(`${c.dim}Tip: Index last updated ${daysStale} days ago. Run 'qmd update' to refresh.${c.reset}\n`);
|
|
342
|
+
}
|
|
343
|
+
}
|
|
344
|
+
// Compute unique display path for a document
|
|
345
|
+
// Always include at least parent folder + filename, add more parent dirs until unique
|
|
346
|
+
function computeDisplayPath(filepath, collectionPath, existingPaths) {
|
|
347
|
+
// Get path relative to collection (include collection dir name)
|
|
348
|
+
const collectionDir = collectionPath.replace(/\/$/, '');
|
|
349
|
+
const collectionName = collectionDir.split('/').pop() || '';
|
|
350
|
+
let relativePath;
|
|
351
|
+
if (filepath.startsWith(collectionDir + '/')) {
|
|
352
|
+
// filepath is under collection: use collection name + relative path
|
|
353
|
+
relativePath = collectionName + filepath.slice(collectionDir.length);
|
|
354
|
+
}
|
|
355
|
+
else {
|
|
356
|
+
// Fallback: just use the filepath
|
|
357
|
+
relativePath = filepath;
|
|
358
|
+
}
|
|
359
|
+
const parts = relativePath.split('/').filter(p => p.length > 0);
|
|
360
|
+
// Always include at least parent folder + filename (minimum 2 parts if available)
|
|
361
|
+
// Then add more parent dirs until unique
|
|
362
|
+
const minParts = Math.min(2, parts.length);
|
|
363
|
+
for (let i = parts.length - minParts; i >= 0; i--) {
|
|
364
|
+
const candidate = parts.slice(i).join('/');
|
|
365
|
+
if (!existingPaths.has(candidate)) {
|
|
366
|
+
return candidate;
|
|
367
|
+
}
|
|
368
|
+
}
|
|
369
|
+
// Absolute fallback: use full path (should be unique)
|
|
370
|
+
return filepath;
|
|
371
|
+
}
|
|
372
|
+
function formatTimeAgo(date) {
|
|
373
|
+
const seconds = Math.floor((Date.now() - date.getTime()) / 1000);
|
|
374
|
+
if (seconds < 60)
|
|
375
|
+
return `${seconds}s ago`;
|
|
376
|
+
const minutes = Math.floor(seconds / 60);
|
|
377
|
+
if (minutes < 60)
|
|
378
|
+
return `${minutes}m ago`;
|
|
379
|
+
const hours = Math.floor(minutes / 60);
|
|
380
|
+
if (hours < 24)
|
|
381
|
+
return `${hours}h ago`;
|
|
382
|
+
const days = Math.floor(hours / 24);
|
|
383
|
+
return `${days}d ago`;
|
|
384
|
+
}
|
|
385
|
+
function formatMs(ms) {
|
|
386
|
+
if (ms < 1000)
|
|
387
|
+
return `${ms}ms`;
|
|
388
|
+
return `${(ms / 1000).toFixed(1)}s`;
|
|
389
|
+
}
|
|
390
|
+
function formatBytes(bytes) {
|
|
391
|
+
if (bytes < 1024)
|
|
392
|
+
return `${bytes} B`;
|
|
393
|
+
if (bytes < 1024 * 1024)
|
|
394
|
+
return `${(bytes / 1024).toFixed(1)} KB`;
|
|
395
|
+
if (bytes < 1024 * 1024 * 1024)
|
|
396
|
+
return `${(bytes / (1024 * 1024)).toFixed(1)} MB`;
|
|
397
|
+
return `${(bytes / (1024 * 1024 * 1024)).toFixed(1)} GB`;
|
|
398
|
+
}
|
|
399
|
+
function sameDirectory(a, b) {
|
|
400
|
+
try {
|
|
401
|
+
return realpathSync(a) === realpathSync(b);
|
|
402
|
+
}
|
|
403
|
+
catch {
|
|
404
|
+
return pathResolve(a) === pathResolve(b);
|
|
405
|
+
}
|
|
406
|
+
}
|
|
407
|
+
function initLocalIndex() {
|
|
408
|
+
const cwd = getPwd();
|
|
409
|
+
if (sameDirectory(cwd, homedir())) {
|
|
410
|
+
throw new Error("Refusing to initialize a local index in $HOME. The global index is automatically created; run `qmd collection add <path>` for the global index, or run `qmd init` inside a project folder.");
|
|
411
|
+
}
|
|
412
|
+
const qmdDir = pathJoin(cwd, ".qmd");
|
|
413
|
+
const ymlPath = pathJoin(qmdDir, "index.yml");
|
|
414
|
+
const yamlPath = pathJoin(qmdDir, "index.yaml");
|
|
415
|
+
const configPath = existsSync(yamlPath) ? yamlPath : ymlPath;
|
|
416
|
+
const dbPath = pathJoin(qmdDir, "index.sqlite");
|
|
417
|
+
mkdirSync(qmdDir, { recursive: true });
|
|
418
|
+
setConfigSource({ configPath });
|
|
419
|
+
storeDbPathOverride = dbPath;
|
|
420
|
+
closeDb();
|
|
421
|
+
if (!existsSync(configPath)) {
|
|
422
|
+
saveConfig({
|
|
423
|
+
collections: {},
|
|
424
|
+
models: resolveModels(),
|
|
425
|
+
});
|
|
426
|
+
}
|
|
427
|
+
else {
|
|
428
|
+
ensureModelsConfiguredForCli();
|
|
429
|
+
}
|
|
430
|
+
const localStore = createStore(dbPath);
|
|
431
|
+
syncConfigToDb(localStore.db, loadConfig());
|
|
432
|
+
localStore.close();
|
|
433
|
+
console.log("ready to go with new local index");
|
|
434
|
+
}
|
|
435
|
+
function isForceCpuEnabled() {
|
|
436
|
+
const value = process.env.QMD_FORCE_CPU;
|
|
437
|
+
return !!value && !["false", "off", "none", "disable", "disabled", "0"].includes(value.trim().toLowerCase());
|
|
438
|
+
}
|
|
439
|
+
function configuredGpuModeLabel() {
|
|
440
|
+
return isForceCpuEnabled()
|
|
441
|
+
? "CPU forced (QMD_FORCE_CPU)"
|
|
442
|
+
: (process.env.QMD_LLAMA_GPU?.trim() || "auto");
|
|
443
|
+
}
|
|
444
|
+
function summarizeDeviceNames(names) {
|
|
445
|
+
const counts = new Map();
|
|
446
|
+
for (const name of names) {
|
|
447
|
+
counts.set(name, (counts.get(name) || 0) + 1);
|
|
448
|
+
}
|
|
449
|
+
return Array.from(counts.entries())
|
|
450
|
+
.map(([name, count]) => count > 1 ? `${count}× ${name}` : name)
|
|
451
|
+
.join(", ");
|
|
452
|
+
}
|
|
453
|
+
function sanitizeDiagnosticMessage(message) {
|
|
454
|
+
const home = homedir();
|
|
455
|
+
return message
|
|
456
|
+
.replaceAll(home, "~")
|
|
457
|
+
.replaceAll(process.cwd(), ".")
|
|
458
|
+
.split("\n")
|
|
459
|
+
.map(line => line.trim())
|
|
460
|
+
.filter(Boolean)
|
|
461
|
+
.slice(0, 3)
|
|
462
|
+
.join("; ");
|
|
463
|
+
}
|
|
464
|
+
/** Hint after `qmd update` when orphaned embedding chunks exceed this share of vectors (#768). */
|
|
465
|
+
const ORPHAN_VECTOR_HINT_RATIO = 0.1;
|
|
466
|
+
function formatOrphanedVectorHint(orphaned, total) {
|
|
467
|
+
const pct = total > 0 ? Math.round((orphaned / total) * 100) : 0;
|
|
468
|
+
return `${orphaned} orphaned embedding chunks (${pct}% of vectors) — run 'qmd cleanup' to reclaim space`;
|
|
469
|
+
}
|
|
470
|
+
async function showStatus() {
|
|
471
|
+
const dbPath = getDbPath();
|
|
472
|
+
const db = getDoctorStore().db;
|
|
473
|
+
// Collections are defined in YAML; no duplicate cleanup needed.
|
|
474
|
+
// Collections are defined in YAML; no duplicate cleanup needed.
|
|
475
|
+
// Index size
|
|
476
|
+
let indexSize = 0;
|
|
477
|
+
try {
|
|
478
|
+
const stat = statSync(dbPath).size;
|
|
479
|
+
indexSize = stat;
|
|
480
|
+
}
|
|
481
|
+
catch { }
|
|
482
|
+
// Collections info (from YAML + database stats)
|
|
483
|
+
const collections = listCollections(db);
|
|
484
|
+
// Overall stats
|
|
485
|
+
const totalDocs = db.prepare(`SELECT COUNT(*) as count FROM documents WHERE active = 1`).get();
|
|
486
|
+
const vectorCount = db.prepare(`SELECT COUNT(*) as count FROM content_vectors`).get();
|
|
487
|
+
const statusConfig = loadConfig();
|
|
488
|
+
const statusEmbedModel = statusConfig.models?.embed ?? resolveModels().embed;
|
|
489
|
+
const statusEmbedding = resolveEmbeddingConfig({
|
|
490
|
+
config: statusConfig,
|
|
491
|
+
dbConfig: readCanonicalEmbeddingConfig(db),
|
|
492
|
+
env: process.env,
|
|
493
|
+
defaultLocalModel: statusEmbedModel,
|
|
494
|
+
}).canonical;
|
|
495
|
+
const diagnostics = inspectIndexDiagnostics(db, {
|
|
496
|
+
fallbackModel: statusEmbedding.model,
|
|
497
|
+
provider: statusEmbedding.provider === "openai"
|
|
498
|
+
? new UnavailableOpenAIEmbeddingProvider({
|
|
499
|
+
model: statusEmbedding.model,
|
|
500
|
+
dimension: statusEmbedding.dimension ?? undefined,
|
|
501
|
+
baseUrl: statusEmbedding.baseUrl,
|
|
502
|
+
})
|
|
503
|
+
: undefined,
|
|
504
|
+
keyConfigured: Boolean(process.env.OPENAI_API_KEY?.trim()),
|
|
505
|
+
configuredProvider: {
|
|
506
|
+
id: statusEmbedding.provider === "openai" ? "openai" : "local-llama-cpp",
|
|
507
|
+
remote: statusEmbedding.provider === "openai",
|
|
508
|
+
model: statusEmbedding.model,
|
|
509
|
+
dimension: statusEmbedding.dimension,
|
|
510
|
+
},
|
|
511
|
+
});
|
|
512
|
+
const needsEmbedding = diagnostics.embedding.chunks.pendingDocuments;
|
|
513
|
+
// Most recent update across all collections
|
|
514
|
+
const mostRecent = db.prepare(`SELECT MAX(modified_at) as latest FROM documents WHERE active = 1`).get();
|
|
515
|
+
console.log(`${c.bold}QMD Status${c.reset}\n`);
|
|
516
|
+
console.log(`Index: ${dbPath}`);
|
|
517
|
+
console.log(`Size: ${formatBytes(indexSize)}`);
|
|
518
|
+
// MCP daemon status (check PID file liveness; scoped per --index)
|
|
519
|
+
const { pidPath: mcpPidPath } = mcpDaemonPaths();
|
|
520
|
+
if (existsSync(mcpPidPath)) {
|
|
521
|
+
const mcpPid = parseInt(readFileSync(mcpPidPath, "utf-8").trim());
|
|
522
|
+
if (isQmdMcpPid(mcpPid)) {
|
|
523
|
+
console.log(`MCP: ${c.green}running${c.reset} (PID ${mcpPid})`);
|
|
524
|
+
}
|
|
525
|
+
else {
|
|
526
|
+
try {
|
|
527
|
+
unlinkSync(mcpPidPath);
|
|
528
|
+
}
|
|
529
|
+
catch { /* ignore */ }
|
|
530
|
+
// Stale / recycled PID file cleaned up silently
|
|
531
|
+
}
|
|
532
|
+
}
|
|
533
|
+
console.log("");
|
|
534
|
+
const orphanedVectors = countOrphanedVectors(db);
|
|
535
|
+
console.log(`${c.bold}Documents${c.reset}`);
|
|
536
|
+
console.log(` Total: ${totalDocs.count} files indexed`);
|
|
537
|
+
console.log(` Vectors: ${vectorCount.count} embedded`);
|
|
538
|
+
if (orphanedVectors > 0) {
|
|
539
|
+
const pct = vectorCount.count > 0 ? Math.round((orphanedVectors / vectorCount.count) * 100) : 0;
|
|
540
|
+
console.log(` ${c.yellow}Orphaned: ${orphanedVectors} embedding chunks (${pct}%)${c.reset} — run 'qmd cleanup'`);
|
|
541
|
+
}
|
|
542
|
+
if (needsEmbedding > 0) {
|
|
543
|
+
console.log(` ${c.yellow}Pending: ${needsEmbedding} need embedding${c.reset} (run 'qmd embed')`);
|
|
544
|
+
}
|
|
545
|
+
if (mostRecent.latest) {
|
|
546
|
+
const lastUpdate = new Date(mostRecent.latest);
|
|
547
|
+
console.log(` Updated: ${formatTimeAgo(lastUpdate)}`);
|
|
548
|
+
}
|
|
549
|
+
console.log(`\n${c.bold}Embedding${c.reset}`);
|
|
550
|
+
console.log(` Provider: ${diagnostics.embedding.provider.id ?? "unknown"}`);
|
|
551
|
+
console.log(` Model: ${diagnostics.embedding.provider.model}`);
|
|
552
|
+
console.log(` Dimension: ${diagnostics.embedding.provider.dimension ?? "unknown"}`);
|
|
553
|
+
console.log(` Identity: ${diagnostics.embedding.identity.shortFingerprint ?? "unknown"} (${diagnostics.embedding.build.state})`);
|
|
554
|
+
console.log(` Pending: ${diagnostics.embedding.chunks.pendingDocuments}; metadata-only=${diagnostics.embedding.chunks.metadataOnly}, vector-only=${diagnostics.embedding.chunks.vectorOnly}`);
|
|
555
|
+
if (diagnostics.embedding.repairCommand)
|
|
556
|
+
console.log(` Repair: ${diagnostics.embedding.repairCommand}`);
|
|
557
|
+
console.log(`\n${c.bold}CJK Lexical${c.reset}`);
|
|
558
|
+
const jiebaDisplay = diagnostics.lexical.jiebaCapability === "available"
|
|
559
|
+
? `${c.green}active${c.reset}`
|
|
560
|
+
: diagnostics.lexical.jiebaCapability === "unavailable"
|
|
561
|
+
? `${c.yellow}unavailable${c.reset}`
|
|
562
|
+
: `${c.dim}${diagnostics.lexical.jiebaCapability}${c.reset}`;
|
|
563
|
+
console.log(` Jieba: ${jiebaDisplay}`);
|
|
564
|
+
console.log(` Analyzer: ${diagnostics.lexical.analyzerFingerprint.slice(0, 12)} (${diagnostics.lexical.state})`);
|
|
565
|
+
console.log(` Channels: char=${diagnostics.lexical.channels.char}, word=${diagnostics.lexical.channels.word}, bigram=${diagnostics.lexical.channels.bigram}`);
|
|
566
|
+
if (diagnostics.lexical.rebuildReason)
|
|
567
|
+
console.log(` Reason: ${diagnostics.lexical.rebuildReason}`);
|
|
568
|
+
if (diagnostics.lexical.repairCommand)
|
|
569
|
+
console.log(` Repair: ${diagnostics.lexical.repairCommand}`);
|
|
570
|
+
// Get all contexts grouped by collection (from YAML)
|
|
571
|
+
const allContexts = listAllContexts();
|
|
572
|
+
const contextsByCollection = new Map();
|
|
573
|
+
for (const ctx of allContexts) {
|
|
574
|
+
// Group contexts by collection name
|
|
575
|
+
if (!contextsByCollection.has(ctx.collection)) {
|
|
576
|
+
contextsByCollection.set(ctx.collection, []);
|
|
577
|
+
}
|
|
578
|
+
contextsByCollection.get(ctx.collection).push({
|
|
579
|
+
path_prefix: ctx.path,
|
|
580
|
+
context: ctx.context
|
|
581
|
+
});
|
|
582
|
+
}
|
|
583
|
+
// AST chunking status
|
|
584
|
+
try {
|
|
585
|
+
const { getASTStatus } = await import("../ast.js");
|
|
586
|
+
const ast = await getASTStatus();
|
|
587
|
+
console.log(`\n${c.bold}AST Chunking${c.reset}`);
|
|
588
|
+
if (ast.available) {
|
|
589
|
+
const ok = ast.languages.filter(l => l.available).map(l => l.language);
|
|
590
|
+
const fail = ast.languages.filter(l => !l.available);
|
|
591
|
+
console.log(` Status: ${c.green}active${c.reset}`);
|
|
592
|
+
console.log(` Languages: ${ok.join(", ")}`);
|
|
593
|
+
if (fail.length > 0) {
|
|
594
|
+
for (const f of fail) {
|
|
595
|
+
console.log(` ${c.yellow}Unavailable: ${f.language} (${f.error})${c.reset}`);
|
|
596
|
+
}
|
|
597
|
+
}
|
|
598
|
+
}
|
|
599
|
+
else {
|
|
600
|
+
console.log(` Status: ${c.yellow}unavailable${c.reset} (falling back to regex chunking)`);
|
|
601
|
+
for (const l of ast.languages) {
|
|
602
|
+
if (l.error)
|
|
603
|
+
console.log(` ${c.dim}${l.language}: ${l.error}${c.reset}`);
|
|
604
|
+
}
|
|
605
|
+
}
|
|
606
|
+
}
|
|
607
|
+
catch {
|
|
608
|
+
console.log(`\n${c.bold}AST Chunking${c.reset}`);
|
|
609
|
+
console.log(` Status: ${c.dim}not available${c.reset}`);
|
|
610
|
+
}
|
|
611
|
+
if (collections.length > 0) {
|
|
612
|
+
console.log(`\n${c.bold}Collections${c.reset}`);
|
|
613
|
+
for (const col of collections) {
|
|
614
|
+
const lastMod = col.last_modified ? formatTimeAgo(new Date(col.last_modified)) : "never";
|
|
615
|
+
const contexts = contextsByCollection.get(col.name) || [];
|
|
616
|
+
console.log(` ${c.cyan}${col.name}${c.reset} ${c.dim}(qmd://${col.name}/)${c.reset}`);
|
|
617
|
+
console.log(` ${c.dim}Pattern:${c.reset} ${col.glob_pattern}`);
|
|
618
|
+
console.log(` ${c.dim}Files:${c.reset} ${col.active_count} (updated ${lastMod})`);
|
|
619
|
+
if (contexts.length > 0) {
|
|
620
|
+
console.log(` ${c.dim}Contexts:${c.reset} ${contexts.length}`);
|
|
621
|
+
for (const ctx of contexts) {
|
|
622
|
+
// Handle both empty string and '/' as root context
|
|
623
|
+
const pathDisplay = (ctx.path_prefix === '' || ctx.path_prefix === '/') ? '/' : `/${ctx.path_prefix}`;
|
|
624
|
+
const contextPreview = ctx.context.length > 60
|
|
625
|
+
? ctx.context.substring(0, 57) + '...'
|
|
626
|
+
: ctx.context;
|
|
627
|
+
console.log(` ${c.dim}${pathDisplay}:${c.reset} ${contextPreview}`);
|
|
628
|
+
}
|
|
629
|
+
}
|
|
630
|
+
}
|
|
631
|
+
// Show examples of virtual paths
|
|
632
|
+
console.log(`\n${c.bold}Examples${c.reset}`);
|
|
633
|
+
console.log(` ${c.dim}# List files in a collection${c.reset}`);
|
|
634
|
+
if (collections.length > 0 && collections[0]) {
|
|
635
|
+
console.log(` qmd ls ${collections[0].name}`);
|
|
636
|
+
}
|
|
637
|
+
console.log(` ${c.dim}# Get a document${c.reset}`);
|
|
638
|
+
if (collections.length > 0 && collections[0]) {
|
|
639
|
+
console.log(` qmd get qmd://${collections[0].name}/path/to/file.md`);
|
|
640
|
+
}
|
|
641
|
+
console.log(` ${c.dim}# Search within a collection${c.reset}`);
|
|
642
|
+
if (collections.length > 0 && collections[0]) {
|
|
643
|
+
console.log(` qmd search "query" -c ${collections[0].name}`);
|
|
644
|
+
}
|
|
645
|
+
}
|
|
646
|
+
else {
|
|
647
|
+
console.log(`\n${c.dim}No collections. Run 'qmd collection add .' to index markdown files.${c.reset}`);
|
|
648
|
+
}
|
|
649
|
+
// Models
|
|
650
|
+
{
|
|
651
|
+
// hf:org/repo/file.gguf → https://huggingface.co/org/repo
|
|
652
|
+
const hfLink = (uri) => {
|
|
653
|
+
const match = uri.match(/^hf:([^/]+\/[^/]+)\//);
|
|
654
|
+
return match ? `https://huggingface.co/${match[1]}` : uri;
|
|
655
|
+
};
|
|
656
|
+
const activeModels = {
|
|
657
|
+
...resolveModels(statusConfig.models),
|
|
658
|
+
embed: statusEmbedding.model,
|
|
659
|
+
};
|
|
660
|
+
console.log(`\n${c.bold}Models${c.reset}`);
|
|
661
|
+
console.log(` Embedding: ${hfLink(activeModels.embed)}`);
|
|
662
|
+
console.log(` Reranking: ${hfLink(activeModels.rerank)}`);
|
|
663
|
+
console.log(` Generation: ${hfLink(activeModels.generate)}`);
|
|
664
|
+
}
|
|
665
|
+
// Tips section
|
|
666
|
+
const tips = [];
|
|
667
|
+
// Check for collections without context
|
|
668
|
+
const collectionsWithoutContext = collections.filter(col => {
|
|
669
|
+
const contexts = contextsByCollection.get(col.name) || [];
|
|
670
|
+
return contexts.length === 0;
|
|
671
|
+
});
|
|
672
|
+
if (collectionsWithoutContext.length > 0) {
|
|
673
|
+
const names = collectionsWithoutContext.map(c => c.name).slice(0, 3).join(', ');
|
|
674
|
+
const more = collectionsWithoutContext.length > 3 ? ` +${collectionsWithoutContext.length - 3} more` : '';
|
|
675
|
+
tips.push(`Add context to collections for better search results: ${names}${more}`);
|
|
676
|
+
tips.push(` ${c.dim}qmd context add qmd://<name>/ "What this collection contains"${c.reset}`);
|
|
677
|
+
tips.push(` ${c.dim}qmd context add qmd://<name>/meeting-notes "Weekly team meeting notes"${c.reset}`);
|
|
678
|
+
}
|
|
679
|
+
// Check for collections without update commands
|
|
680
|
+
const collectionsWithoutUpdate = collections.filter(col => {
|
|
681
|
+
const yamlCol = getCollectionFromYaml(col.name);
|
|
682
|
+
return !yamlCol?.update;
|
|
683
|
+
});
|
|
684
|
+
if (collectionsWithoutUpdate.length > 0 && collections.length > 1) {
|
|
685
|
+
const names = collectionsWithoutUpdate.map(c => c.name).slice(0, 3).join(', ');
|
|
686
|
+
const more = collectionsWithoutUpdate.length > 3 ? ` +${collectionsWithoutUpdate.length - 3} more` : '';
|
|
687
|
+
tips.push(`Add update commands to keep collections fresh: ${names}${more}`);
|
|
688
|
+
tips.push(` ${c.dim}qmd collection update-cmd <name> 'git stash && git pull --rebase --ff-only && git stash pop'${c.reset}`);
|
|
689
|
+
}
|
|
690
|
+
if (tips.length > 0) {
|
|
691
|
+
console.log(`\n${c.bold}Tips${c.reset}`);
|
|
692
|
+
for (const tip of tips) {
|
|
693
|
+
console.log(` ${tip}`);
|
|
694
|
+
}
|
|
695
|
+
}
|
|
696
|
+
closeDb();
|
|
697
|
+
}
|
|
698
|
+
function builtinModels() {
|
|
699
|
+
return {
|
|
700
|
+
embed: DEFAULT_EMBED_MODEL,
|
|
701
|
+
rerank: DEFAULT_RERANK_MODEL,
|
|
702
|
+
generate: DEFAULT_QUERY_MODEL,
|
|
703
|
+
};
|
|
704
|
+
}
|
|
705
|
+
/**
|
|
706
|
+
* Gated surface of the active YAML: hooks, collection paths, models.
|
|
707
|
+
* Read from the YAML rather than the synced SQLite copy so `qmd trust`
|
|
708
|
+
* and `qmd update` always digest the same set.
|
|
709
|
+
*/
|
|
710
|
+
function collectSensitiveSnapshot() {
|
|
711
|
+
const config = loadConfig();
|
|
712
|
+
return {
|
|
713
|
+
hooks: Object.entries(config.collections ?? {})
|
|
714
|
+
.filter(([, col]) => !!col.update)
|
|
715
|
+
.map(([name, col]) => ({ collection: name, command: col.update })),
|
|
716
|
+
paths: Object.entries(config.collections ?? {}).map(([name, col]) => ({
|
|
717
|
+
collection: name,
|
|
718
|
+
path: col.path,
|
|
719
|
+
})),
|
|
720
|
+
models: {
|
|
721
|
+
embed: config.models?.embed,
|
|
722
|
+
rerank: config.models?.rerank,
|
|
723
|
+
generate: config.models?.generate,
|
|
724
|
+
},
|
|
725
|
+
};
|
|
726
|
+
}
|
|
727
|
+
function localConfigDigest(configPath, snapshot) {
|
|
728
|
+
return sensitiveDigest(snapshot, configPath, builtinModels());
|
|
729
|
+
}
|
|
730
|
+
function localConfigGated(configPath, snapshot) {
|
|
731
|
+
return gatedItems(configPath, snapshot, builtinModels());
|
|
732
|
+
}
|
|
733
|
+
/** True when this process may use the local config's gated fields. */
|
|
734
|
+
function localConfigIsFullyTrusted() {
|
|
735
|
+
const configPath = getConfigPath();
|
|
736
|
+
if (!isLocalConfigPath(configPath))
|
|
737
|
+
return true;
|
|
738
|
+
if (isLocalConfigTrustOptedIn())
|
|
739
|
+
return true;
|
|
740
|
+
const snapshot = collectSensitiveSnapshot();
|
|
741
|
+
const gated = localConfigGated(configPath, snapshot);
|
|
742
|
+
if (!hasGatedItems(gated))
|
|
743
|
+
return true;
|
|
744
|
+
return isTrusted(configPath, localConfigDigest(configPath, snapshot));
|
|
745
|
+
}
|
|
746
|
+
/** Record the active config's current gated set as approved. */
|
|
747
|
+
function trustCurrentConfig() {
|
|
748
|
+
const configPath = getConfigPath();
|
|
749
|
+
if (!isLocalConfigPath(configPath))
|
|
750
|
+
return;
|
|
751
|
+
recordTrust(configPath, localConfigDigest(configPath, collectSensitiveSnapshot()));
|
|
752
|
+
}
|
|
753
|
+
/** Ask the user a yes/no question. Only called when stdin and stdout are TTYs. */
|
|
754
|
+
async function confirmOnTty(question) {
|
|
755
|
+
const { createInterface } = await import("node:readline/promises");
|
|
756
|
+
const rl = createInterface({ input: process.stdin, output: process.stdout });
|
|
757
|
+
try {
|
|
758
|
+
const answer = (await rl.question(question)).trim().toLowerCase();
|
|
759
|
+
return answer === "y" || answer === "yes";
|
|
760
|
+
}
|
|
761
|
+
finally {
|
|
762
|
+
rl.close();
|
|
763
|
+
}
|
|
764
|
+
}
|
|
765
|
+
function printGatedItems(gated) {
|
|
766
|
+
if (gated.hooks.length > 0) {
|
|
767
|
+
console.log(`${c.yellow}This project's config defines update commands:${c.reset}`);
|
|
768
|
+
for (const hook of gated.hooks) {
|
|
769
|
+
console.log(` ${c.bold}${hook.collection}${c.reset}: ${hook.command}`);
|
|
770
|
+
}
|
|
771
|
+
}
|
|
772
|
+
if (gated.paths.length > 0) {
|
|
773
|
+
console.log(`${c.yellow}Collection paths outside this project:${c.reset}`);
|
|
774
|
+
for (const item of gated.paths) {
|
|
775
|
+
console.log(` ${c.bold}${item.collection}${c.reset}: ${item.path}`);
|
|
776
|
+
}
|
|
777
|
+
}
|
|
778
|
+
if (gated.models.length > 0) {
|
|
779
|
+
console.log(`${c.yellow}Custom models:${c.reset}`);
|
|
780
|
+
for (const item of gated.models) {
|
|
781
|
+
console.log(` ${c.bold}${item.slot}${c.reset}: ${item.uri}`);
|
|
782
|
+
}
|
|
783
|
+
}
|
|
784
|
+
}
|
|
785
|
+
/**
|
|
786
|
+
* Decide whether this run may use gated fields from a project-local config
|
|
787
|
+
* (update hooks, out-of-project collection paths, custom model URIs).
|
|
788
|
+
* Returns false when those must be skipped — in-project indexing still
|
|
789
|
+
* proceeds, since that is what the caller asked for.
|
|
790
|
+
*/
|
|
791
|
+
async function resolveLocalConfigTrust() {
|
|
792
|
+
const configPath = getConfigPath();
|
|
793
|
+
const snapshot = collectSensitiveSnapshot();
|
|
794
|
+
const gated = localConfigGated(configPath, snapshot);
|
|
795
|
+
if (!hasGatedItems(gated))
|
|
796
|
+
return true;
|
|
797
|
+
const decision = decideLocalConfigGate({
|
|
798
|
+
configPath,
|
|
799
|
+
snapshot,
|
|
800
|
+
builtins: builtinModels(),
|
|
801
|
+
isInteractive: !!process.stdin.isTTY && !!process.stdout.isTTY,
|
|
802
|
+
});
|
|
803
|
+
if (decision.action === "run")
|
|
804
|
+
return true;
|
|
805
|
+
printGatedItems(gated);
|
|
806
|
+
console.log(`${c.dim}Config that came with a checkout is not trusted by default.${c.reset}`);
|
|
807
|
+
if (decision.action === "skip") {
|
|
808
|
+
console.log(`${c.yellow}Skipping them — no terminal to confirm on. Indexing of this project continues.${c.reset}`);
|
|
809
|
+
console.log(`${c.dim}Approve with 'qmd trust', or set QMD_TRUST_LOCAL_CONFIG=1 for unattended runs.${c.reset}\n`);
|
|
810
|
+
return false;
|
|
811
|
+
}
|
|
812
|
+
const approved = await confirmOnTty("Trust this project's .qmd config? [y/N] ");
|
|
813
|
+
if (!approved) {
|
|
814
|
+
console.log(`${c.yellow}Skipped. Indexing of this project continues.${c.reset}\n`);
|
|
815
|
+
return false;
|
|
816
|
+
}
|
|
817
|
+
recordTrust(configPath, decision.digest);
|
|
818
|
+
console.log(`${c.green}Trusted ${configPath}.${c.reset} ${c.dim}Editing a hook, path, or model will ask again.${c.reset}\n`);
|
|
819
|
+
return true;
|
|
820
|
+
}
|
|
821
|
+
/**
|
|
822
|
+
* `qmd trust [list|revoke]` — approve, inspect or drop the approval for
|
|
823
|
+
* the project-local config in scope (hooks, out-of-project paths, custom models).
|
|
824
|
+
*/
|
|
825
|
+
function manageTrust(subcommand) {
|
|
826
|
+
const configPath = getConfigPath();
|
|
827
|
+
if (subcommand === "list") {
|
|
828
|
+
const records = listTrusted();
|
|
829
|
+
if (records.length === 0) {
|
|
830
|
+
console.log(`${c.dim}No trusted project configs.${c.reset}`);
|
|
831
|
+
return;
|
|
832
|
+
}
|
|
833
|
+
for (const record of records) {
|
|
834
|
+
console.log(`${record.path} ${c.dim}(trusted ${record.trustedAt})${c.reset}`);
|
|
835
|
+
}
|
|
836
|
+
return;
|
|
837
|
+
}
|
|
838
|
+
if (subcommand === "revoke") {
|
|
839
|
+
if (!isLocalConfigPath(configPath)) {
|
|
840
|
+
console.log(`${c.dim}No project-local .qmd config in scope — ${configPath} is your own config and is never gated.${c.reset}`);
|
|
841
|
+
console.log(`${c.dim}Run this from inside the project, or see 'qmd trust list'.${c.reset}`);
|
|
842
|
+
return;
|
|
843
|
+
}
|
|
844
|
+
if (revokeTrust(configPath)) {
|
|
845
|
+
console.log(`${c.green}✓ Revoked trust for ${configPath}${c.reset}`);
|
|
846
|
+
}
|
|
847
|
+
else {
|
|
848
|
+
console.log(`${c.dim}${configPath} was not trusted.${c.reset}`);
|
|
849
|
+
}
|
|
850
|
+
return;
|
|
851
|
+
}
|
|
852
|
+
if (subcommand) {
|
|
853
|
+
console.error(`Usage: qmd trust [list|revoke]`);
|
|
854
|
+
process.exit(1);
|
|
855
|
+
}
|
|
856
|
+
if (!isLocalConfigPath(configPath)) {
|
|
857
|
+
console.log(`${c.dim}${configPath} is your own config — it is never gated. Nothing to trust.${c.reset}`);
|
|
858
|
+
return;
|
|
859
|
+
}
|
|
860
|
+
const snapshot = collectSensitiveSnapshot();
|
|
861
|
+
const gated = localConfigGated(configPath, snapshot);
|
|
862
|
+
if (!hasGatedItems(gated)) {
|
|
863
|
+
console.log(`${c.dim}${configPath} defines no update commands, out-of-project collection paths, or custom models. Nothing to trust.${c.reset}`);
|
|
864
|
+
return;
|
|
865
|
+
}
|
|
866
|
+
printGatedItems(gated);
|
|
867
|
+
recordTrust(configPath, localConfigDigest(configPath, snapshot));
|
|
868
|
+
console.log(`${c.green}✓ Trusted ${configPath}${c.reset}`);
|
|
869
|
+
console.log(`${c.dim}Editing a hook, out-of-project path, or custom model will ask again. Revoke with 'qmd trust revoke'.${c.reset}`);
|
|
870
|
+
}
|
|
871
|
+
async function updateCollections() {
|
|
872
|
+
// Prompt before opening the store so an approval is visible to getStore (#889).
|
|
873
|
+
const allowed = await resolveLocalConfigTrust();
|
|
874
|
+
const db = getDb();
|
|
875
|
+
const storeInstance = getStore();
|
|
876
|
+
// Collections are defined in YAML; no duplicate cleanup needed.
|
|
877
|
+
// Clear Ollama cache on update
|
|
878
|
+
clearCache(db);
|
|
879
|
+
const collections = listCollections(db);
|
|
880
|
+
if (collections.length === 0) {
|
|
881
|
+
console.log(`${c.dim}No collections found. Run 'qmd collection add .' to index markdown files.${c.reset}`);
|
|
882
|
+
closeDb();
|
|
883
|
+
return;
|
|
884
|
+
}
|
|
885
|
+
// A project-local .qmd/index.yml travels with a `git clone`, so its `update:`
|
|
886
|
+
// hooks, out-of-project collection paths, and custom model URIs are somebody
|
|
887
|
+
// else's choices until the user says otherwise (#886, #889).
|
|
888
|
+
const hooksAllowed = allowed;
|
|
889
|
+
const configPath = getConfigPath();
|
|
890
|
+
console.log(`${c.bold}Updating ${collections.length} collection(s)...${c.reset}\n`);
|
|
891
|
+
for (let i = 0; i < collections.length; i++) {
|
|
892
|
+
const col = collections[i];
|
|
893
|
+
if (!col)
|
|
894
|
+
continue;
|
|
895
|
+
console.log(`${c.cyan}[${i + 1}/${collections.length}]${c.reset} ${c.bold}${col.name}${c.reset} ${c.dim}(${col.glob_pattern})${c.reset}`);
|
|
896
|
+
// Execute custom update command if specified in YAML
|
|
897
|
+
const yamlCol = getCollectionFromYaml(col.name);
|
|
898
|
+
const rawPath = yamlCol?.path ?? col.pwd;
|
|
899
|
+
if (!hooksAllowed && isLocalConfigPath(configPath) && !isCollectionPathInsideProject(configPath, rawPath)) {
|
|
900
|
+
console.log(`${c.yellow}Skipping collection '${col.name}' — path ${rawPath} is outside this project and this .qmd config is not trusted.${c.reset}`);
|
|
901
|
+
console.log(`${c.dim}Approve with 'qmd trust'.${c.reset}\n`);
|
|
902
|
+
continue;
|
|
903
|
+
}
|
|
904
|
+
if (yamlCol?.update && hooksAllowed) {
|
|
905
|
+
console.log(`${c.dim} Running update command: ${yamlCol.update}${c.reset}`);
|
|
906
|
+
try {
|
|
907
|
+
const proc = nodeSpawn("bash", ["-c", yamlCol.update], {
|
|
908
|
+
cwd: col.pwd,
|
|
909
|
+
stdio: ["ignore", "pipe", "pipe"],
|
|
910
|
+
});
|
|
911
|
+
const [output, errorOutput, exitCode] = await new Promise((resolve, reject) => {
|
|
912
|
+
let out = "";
|
|
913
|
+
let err = "";
|
|
914
|
+
proc.stdout?.on("data", (d) => { out += d.toString(); });
|
|
915
|
+
proc.stderr?.on("data", (d) => { err += d.toString(); });
|
|
916
|
+
proc.on("error", reject);
|
|
917
|
+
proc.on("close", (code) => resolve([out, err, code ?? 1]));
|
|
918
|
+
});
|
|
919
|
+
if (output.trim()) {
|
|
920
|
+
console.log(output.trim().split('\n').map(l => ` ${l}`).join('\n'));
|
|
921
|
+
}
|
|
922
|
+
if (errorOutput.trim()) {
|
|
923
|
+
console.log(errorOutput.trim().split('\n').map(l => ` ${l}`).join('\n'));
|
|
924
|
+
}
|
|
925
|
+
if (exitCode !== 0) {
|
|
926
|
+
console.log(`${c.yellow}✗ Update command failed with exit code ${exitCode}${c.reset}`);
|
|
927
|
+
process.exit(exitCode);
|
|
928
|
+
}
|
|
929
|
+
}
|
|
930
|
+
catch (err) {
|
|
931
|
+
console.log(`${c.yellow}✗ Update command failed: ${err}${c.reset}`);
|
|
932
|
+
process.exit(1);
|
|
933
|
+
}
|
|
934
|
+
}
|
|
935
|
+
const startTime = Date.now();
|
|
936
|
+
console.log(`Collection: ${col.pwd} (${col.glob_pattern})`);
|
|
937
|
+
progress.indeterminate();
|
|
938
|
+
const result = await reindexCollection(storeInstance, col.pwd, col.glob_pattern, col.name, {
|
|
939
|
+
ignorePatterns: yamlCol?.ignore,
|
|
940
|
+
onProgress: (info) => {
|
|
941
|
+
progress.set((info.current / info.total) * 100);
|
|
942
|
+
const elapsed = (Date.now() - startTime) / 1000;
|
|
943
|
+
const rate = info.current / elapsed;
|
|
944
|
+
const remaining = (info.total - info.current) / rate;
|
|
945
|
+
const eta = info.current > 2 ? ` ETA: ${formatETA(remaining)}` : "";
|
|
946
|
+
if (isTTY)
|
|
947
|
+
process.stderr.write(`\rIndexing: ${info.current}/${info.total}${eta} `);
|
|
948
|
+
},
|
|
949
|
+
});
|
|
950
|
+
progress.clear();
|
|
951
|
+
console.log(`\nIndexed: ${result.indexed} new, ${result.updated} updated, ${result.unchanged} unchanged, ${result.removed} removed`);
|
|
952
|
+
reportSkippedReads(result.skippedFiles);
|
|
953
|
+
if (result.orphanedCleaned > 0) {
|
|
954
|
+
console.log(`Cleaned up ${result.orphanedCleaned} orphaned content hash(es)`);
|
|
955
|
+
}
|
|
956
|
+
console.log("");
|
|
957
|
+
}
|
|
958
|
+
// Check if any documents need embedding (show once at end)
|
|
959
|
+
const needsEmbedding = getHashesNeedingEmbedding(db);
|
|
960
|
+
await rebuildCjkLexicalIndex(getDbPath());
|
|
961
|
+
const vectorTotal = db.prepare(`SELECT COUNT(*) as count FROM content_vectors`).get().count;
|
|
962
|
+
const orphanedVectors = countOrphanedVectors(db);
|
|
963
|
+
closeDb();
|
|
964
|
+
console.log(`${c.green}✓ All collections updated.${c.reset}`);
|
|
965
|
+
if (needsEmbedding > 0) {
|
|
966
|
+
console.log(`\nRun 'qmd embed' to update embeddings (${needsEmbedding} unique hashes need vectors)`);
|
|
967
|
+
}
|
|
968
|
+
if (vectorTotal > 0 && orphanedVectors / vectorTotal >= ORPHAN_VECTOR_HINT_RATIO) {
|
|
969
|
+
console.log(`\n${formatOrphanedVectorHint(orphanedVectors, vectorTotal)}`);
|
|
970
|
+
}
|
|
971
|
+
}
|
|
972
|
+
/**
|
|
973
|
+
* Detect which collection (if any) contains the given filesystem path.
|
|
974
|
+
* Returns { collectionId, collectionName, relativePath } or null if not in any collection.
|
|
975
|
+
*/
|
|
976
|
+
function detectCollectionFromPath(db, fsPath) {
|
|
977
|
+
const realPath = getRealPath(fsPath);
|
|
978
|
+
// Find collections that this path is under from YAML
|
|
979
|
+
const allCollections = yamlListCollections();
|
|
980
|
+
// Find longest matching path
|
|
981
|
+
let bestMatch = null;
|
|
982
|
+
for (const coll of allCollections) {
|
|
983
|
+
if (realPath.startsWith(coll.path + '/') || realPath === coll.path) {
|
|
984
|
+
if (!bestMatch || coll.path.length > bestMatch.path.length) {
|
|
985
|
+
bestMatch = { name: coll.name, path: coll.path };
|
|
986
|
+
}
|
|
987
|
+
}
|
|
988
|
+
}
|
|
989
|
+
if (!bestMatch)
|
|
990
|
+
return null;
|
|
991
|
+
// Calculate relative path
|
|
992
|
+
let relativePath = realPath;
|
|
993
|
+
if (relativePath.startsWith(bestMatch.path + '/')) {
|
|
994
|
+
relativePath = relativePath.slice(bestMatch.path.length + 1);
|
|
995
|
+
}
|
|
996
|
+
else if (relativePath === bestMatch.path) {
|
|
997
|
+
relativePath = '';
|
|
998
|
+
}
|
|
999
|
+
return {
|
|
1000
|
+
collectionName: bestMatch.name,
|
|
1001
|
+
relativePath
|
|
1002
|
+
};
|
|
1003
|
+
}
|
|
1004
|
+
async function contextAdd(pathArg, contextText) {
|
|
1005
|
+
const db = getDb();
|
|
1006
|
+
// Handle "/" as global context (applies to all collections)
|
|
1007
|
+
if (pathArg === '/') {
|
|
1008
|
+
setGlobalContext(contextText);
|
|
1009
|
+
resyncConfig();
|
|
1010
|
+
console.log(`${c.green}✓${c.reset} Set global context`);
|
|
1011
|
+
console.log(`${c.dim}Context: ${contextText}${c.reset}`);
|
|
1012
|
+
closeDb();
|
|
1013
|
+
return;
|
|
1014
|
+
}
|
|
1015
|
+
// Resolve path - defaults to current directory if not provided
|
|
1016
|
+
let fsPath = pathArg || '.';
|
|
1017
|
+
if (fsPath === '.' || fsPath === './') {
|
|
1018
|
+
fsPath = getPwd();
|
|
1019
|
+
}
|
|
1020
|
+
else if (fsPath.startsWith('~/')) {
|
|
1021
|
+
fsPath = homedir() + fsPath.slice(1);
|
|
1022
|
+
}
|
|
1023
|
+
else if (!fsPath.startsWith('/') && !fsPath.startsWith('qmd://')) {
|
|
1024
|
+
fsPath = resolve(getPwd(), fsPath);
|
|
1025
|
+
}
|
|
1026
|
+
// Handle virtual paths (qmd://collection/path)
|
|
1027
|
+
if (isVirtualPath(fsPath)) {
|
|
1028
|
+
const parsed = parseVirtualPath(fsPath);
|
|
1029
|
+
if (!parsed) {
|
|
1030
|
+
console.error(`${c.yellow}Invalid virtual path: ${fsPath}${c.reset}`);
|
|
1031
|
+
process.exit(1);
|
|
1032
|
+
}
|
|
1033
|
+
const coll = getCollectionFromYaml(parsed.collectionName);
|
|
1034
|
+
if (!coll) {
|
|
1035
|
+
console.error(`${c.yellow}Collection not found: ${parsed.collectionName}${c.reset}`);
|
|
1036
|
+
process.exit(1);
|
|
1037
|
+
}
|
|
1038
|
+
yamlAddContext(parsed.collectionName, parsed.path, contextText);
|
|
1039
|
+
resyncConfig();
|
|
1040
|
+
const displayPath = parsed.path
|
|
1041
|
+
? `qmd://${parsed.collectionName}/${parsed.path}`
|
|
1042
|
+
: `qmd://${parsed.collectionName}/ (collection root)`;
|
|
1043
|
+
console.log(`${c.green}✓${c.reset} Added context for: ${displayPath}`);
|
|
1044
|
+
console.log(`${c.dim}Context: ${contextText}${c.reset}`);
|
|
1045
|
+
closeDb();
|
|
1046
|
+
return;
|
|
1047
|
+
}
|
|
1048
|
+
// Detect collection from filesystem path
|
|
1049
|
+
const detected = detectCollectionFromPath(db, fsPath);
|
|
1050
|
+
if (!detected) {
|
|
1051
|
+
console.error(`${c.yellow}Path is not in any indexed collection: ${fsPath}${c.reset}`);
|
|
1052
|
+
console.error(`${c.dim}Run 'qmd status' to see indexed collections${c.reset}`);
|
|
1053
|
+
process.exit(1);
|
|
1054
|
+
}
|
|
1055
|
+
yamlAddContext(detected.collectionName, detected.relativePath, contextText);
|
|
1056
|
+
resyncConfig();
|
|
1057
|
+
const displayPath = detected.relativePath ? `qmd://${detected.collectionName}/${detected.relativePath}` : `qmd://${detected.collectionName}/`;
|
|
1058
|
+
console.log(`${c.green}✓${c.reset} Added context for: ${displayPath}`);
|
|
1059
|
+
console.log(`${c.dim}Context: ${contextText}${c.reset}`);
|
|
1060
|
+
closeDb();
|
|
1061
|
+
}
|
|
1062
|
+
function contextList() {
|
|
1063
|
+
const db = getDb();
|
|
1064
|
+
const allContexts = listAllContexts();
|
|
1065
|
+
if (allContexts.length === 0) {
|
|
1066
|
+
console.log(`${c.dim}No contexts configured. Use 'qmd context add' to add one.${c.reset}`);
|
|
1067
|
+
closeDb();
|
|
1068
|
+
return;
|
|
1069
|
+
}
|
|
1070
|
+
console.log(`\n${c.bold}Configured Contexts${c.reset}\n`);
|
|
1071
|
+
let lastCollection = '';
|
|
1072
|
+
for (const ctx of allContexts) {
|
|
1073
|
+
if (ctx.collection !== lastCollection) {
|
|
1074
|
+
console.log(`${c.cyan}${ctx.collection}${c.reset}`);
|
|
1075
|
+
lastCollection = ctx.collection;
|
|
1076
|
+
}
|
|
1077
|
+
const displayPath = ctx.path ? ` ${ctx.path}` : ' / (root)';
|
|
1078
|
+
console.log(`${displayPath}`);
|
|
1079
|
+
console.log(` ${c.dim}${ctx.context}${c.reset}`);
|
|
1080
|
+
}
|
|
1081
|
+
closeDb();
|
|
1082
|
+
}
|
|
1083
|
+
function contextRemove(pathArg) {
|
|
1084
|
+
if (pathArg === '/') {
|
|
1085
|
+
// Remove global context
|
|
1086
|
+
setGlobalContext(undefined);
|
|
1087
|
+
// Resync so SQLite store_config is updated
|
|
1088
|
+
resyncConfig();
|
|
1089
|
+
closeDb();
|
|
1090
|
+
console.log(`${c.green}✓${c.reset} Removed global context`);
|
|
1091
|
+
return;
|
|
1092
|
+
}
|
|
1093
|
+
// Handle virtual paths
|
|
1094
|
+
if (isVirtualPath(pathArg)) {
|
|
1095
|
+
const parsed = parseVirtualPath(pathArg);
|
|
1096
|
+
if (!parsed) {
|
|
1097
|
+
console.error(`${c.yellow}Invalid virtual path: ${pathArg}${c.reset}`);
|
|
1098
|
+
process.exit(1);
|
|
1099
|
+
}
|
|
1100
|
+
const coll = getCollectionFromYaml(parsed.collectionName);
|
|
1101
|
+
if (!coll) {
|
|
1102
|
+
console.error(`${c.yellow}Collection not found: ${parsed.collectionName}${c.reset}`);
|
|
1103
|
+
process.exit(1);
|
|
1104
|
+
}
|
|
1105
|
+
const success = yamlRemoveContext(coll.name, parsed.path);
|
|
1106
|
+
if (!success) {
|
|
1107
|
+
console.error(`${c.yellow}No context found for: ${pathArg}${c.reset}`);
|
|
1108
|
+
process.exit(1);
|
|
1109
|
+
}
|
|
1110
|
+
resyncConfig();
|
|
1111
|
+
closeDb();
|
|
1112
|
+
console.log(`${c.green}✓${c.reset} Removed context for: ${pathArg}`);
|
|
1113
|
+
return;
|
|
1114
|
+
}
|
|
1115
|
+
// Handle filesystem paths
|
|
1116
|
+
let fsPath = pathArg;
|
|
1117
|
+
if (fsPath === '.' || fsPath === './') {
|
|
1118
|
+
fsPath = getPwd();
|
|
1119
|
+
}
|
|
1120
|
+
else if (fsPath.startsWith('~/')) {
|
|
1121
|
+
fsPath = homedir() + fsPath.slice(1);
|
|
1122
|
+
}
|
|
1123
|
+
else if (!fsPath.startsWith('/')) {
|
|
1124
|
+
fsPath = resolve(getPwd(), fsPath);
|
|
1125
|
+
}
|
|
1126
|
+
const db = getDb();
|
|
1127
|
+
const detected = detectCollectionFromPath(db, fsPath);
|
|
1128
|
+
closeDb();
|
|
1129
|
+
if (!detected) {
|
|
1130
|
+
console.error(`${c.yellow}Path is not in any indexed collection: ${fsPath}${c.reset}`);
|
|
1131
|
+
process.exit(1);
|
|
1132
|
+
}
|
|
1133
|
+
const success = yamlRemoveContext(detected.collectionName, detected.relativePath);
|
|
1134
|
+
if (!success) {
|
|
1135
|
+
console.error(`${c.yellow}No context found for: qmd://${detected.collectionName}/${detected.relativePath}${c.reset}`);
|
|
1136
|
+
process.exit(1);
|
|
1137
|
+
}
|
|
1138
|
+
resyncConfig();
|
|
1139
|
+
closeDb();
|
|
1140
|
+
console.log(`${c.green}✓${c.reset} Removed context for: qmd://${detected.collectionName}/${detected.relativePath}`);
|
|
1141
|
+
}
|
|
1142
|
+
/**
|
|
1143
|
+
* Render an absolute filesystem path for human display under --full-path.
|
|
1144
|
+
*
|
|
1145
|
+
* If the path is the current working directory or a subpath of it, return a
|
|
1146
|
+
* "./"-prefixed relative path so it is unambiguously a filesystem path (not a
|
|
1147
|
+
* bare collection-relative string that could be confused for a `qmd://`
|
|
1148
|
+
* fragment). Otherwise return the absolute realpath so symlinks resolve
|
|
1149
|
+
* consistently. Returns `null` if the path could not be normalized — callers
|
|
1150
|
+
* fall back to whatever they had before.
|
|
1151
|
+
*/
|
|
1152
|
+
function renderFullPath(absolutePath, cwd = process.cwd()) {
|
|
1153
|
+
let real;
|
|
1154
|
+
try {
|
|
1155
|
+
real = realpathSync(absolutePath);
|
|
1156
|
+
}
|
|
1157
|
+
catch {
|
|
1158
|
+
real = absolutePath;
|
|
1159
|
+
}
|
|
1160
|
+
const cwdReal = (() => { try {
|
|
1161
|
+
return realpathSync(cwd);
|
|
1162
|
+
}
|
|
1163
|
+
catch {
|
|
1164
|
+
return cwd;
|
|
1165
|
+
} })();
|
|
1166
|
+
if (real === cwdReal)
|
|
1167
|
+
return "./";
|
|
1168
|
+
if (real.startsWith(cwdReal + "/")) {
|
|
1169
|
+
const rel = relativePath(cwdReal, real);
|
|
1170
|
+
if (rel && !rel.startsWith(".."))
|
|
1171
|
+
return `./${rel}`;
|
|
1172
|
+
}
|
|
1173
|
+
return real;
|
|
1174
|
+
}
|
|
1175
|
+
/**
|
|
1176
|
+
* Report rows that `--full-path` could not turn into an on-disk path.
|
|
1177
|
+
*
|
|
1178
|
+
* The flag's whole job is producing openable paths, so falling back to a
|
|
1179
|
+
* `qmd://` URI is worth saying out loud: it means the file moved or was
|
|
1180
|
+
* deleted since the last index, not that the path was normalized away. The
|
|
1181
|
+
* notice goes to stderr so stdout stays machine-readable.
|
|
1182
|
+
*/
|
|
1183
|
+
function warnUnresolvedFullPaths(unresolved, total) {
|
|
1184
|
+
if (unresolved <= 0)
|
|
1185
|
+
return;
|
|
1186
|
+
const subject = total === 1
|
|
1187
|
+
? "the file"
|
|
1188
|
+
: `${unresolved} of ${total} results`;
|
|
1189
|
+
console.error(`${c.yellow}warning:${c.reset} --full-path could not resolve ${subject} on disk ` +
|
|
1190
|
+
`(moved or deleted since indexing); showing qmd:// + docid instead. ` +
|
|
1191
|
+
`Run 'qmd update' to refresh the index.`);
|
|
1192
|
+
}
|
|
1193
|
+
function getDocument(filename, fromLine, maxLines, lineNumbers, fullPath = false) {
|
|
1194
|
+
// Parse :line suffix from filename. Two forms:
|
|
1195
|
+
// "file.md:100" -> start at line 100
|
|
1196
|
+
// "file.md:100:40" -> start at line 100, read 40 lines
|
|
1197
|
+
// The :// in virtual paths is never matched because we anchor digits to $.
|
|
1198
|
+
// Explicit --from/-l flags always win over values parsed from the path.
|
|
1199
|
+
let inputPath = filename;
|
|
1200
|
+
const rangeMatch = inputPath.match(/:(\d+):(\d+)$/);
|
|
1201
|
+
if (rangeMatch) {
|
|
1202
|
+
if (fromLine === undefined)
|
|
1203
|
+
fromLine = parseInt(rangeMatch[1], 10);
|
|
1204
|
+
if (maxLines === undefined)
|
|
1205
|
+
maxLines = parseInt(rangeMatch[2], 10);
|
|
1206
|
+
inputPath = inputPath.slice(0, -rangeMatch[0].length);
|
|
1207
|
+
}
|
|
1208
|
+
else {
|
|
1209
|
+
const colonMatch = inputPath.match(/:(\d+)$/);
|
|
1210
|
+
if (colonMatch) {
|
|
1211
|
+
const matched = colonMatch[1];
|
|
1212
|
+
if (matched) {
|
|
1213
|
+
if (fromLine === undefined)
|
|
1214
|
+
fromLine = parseInt(matched, 10);
|
|
1215
|
+
inputPath = inputPath.slice(0, -colonMatch[0].length);
|
|
1216
|
+
}
|
|
1217
|
+
}
|
|
1218
|
+
}
|
|
1219
|
+
if (fromLine !== undefined)
|
|
1220
|
+
fromLine = Math.max(1, fromLine);
|
|
1221
|
+
const parsedIndexPath = isVirtualPath(inputPath) ? parseVirtualPath(inputPath) : null;
|
|
1222
|
+
if (parsedIndexPath) {
|
|
1223
|
+
if (parsedIndexPath.indexName) {
|
|
1224
|
+
setIndexName(parsedIndexPath.indexName);
|
|
1225
|
+
setConfigIndexName(parsedIndexPath.indexName);
|
|
1226
|
+
}
|
|
1227
|
+
inputPath = buildVirtualPath(parsedIndexPath.collectionName, parsedIndexPath.path);
|
|
1228
|
+
}
|
|
1229
|
+
const db = getDb();
|
|
1230
|
+
const doc = findDocument(db, inputPath, { includeBody: true });
|
|
1231
|
+
if ("error" in doc) {
|
|
1232
|
+
if (doc.error === "excluded_by_ignore") {
|
|
1233
|
+
console.error(`Document is excluded by ignore rule: ${filename}`);
|
|
1234
|
+
console.error(`Collection: ${doc.collection}`);
|
|
1235
|
+
console.error(`Matched path: ${doc.path}`);
|
|
1236
|
+
console.error(`Ignore rule: ${doc.rule}`);
|
|
1237
|
+
}
|
|
1238
|
+
else {
|
|
1239
|
+
console.error(`Document not found: ${filename}`);
|
|
1240
|
+
if (doc.similarFiles.length > 0) {
|
|
1241
|
+
console.error("Similar files:");
|
|
1242
|
+
for (const file of doc.similarFiles)
|
|
1243
|
+
console.error(` ${file}`);
|
|
1244
|
+
}
|
|
1245
|
+
}
|
|
1246
|
+
closeDb();
|
|
1247
|
+
process.exit(1);
|
|
1248
|
+
}
|
|
1249
|
+
// `findDocument` already computes the docid (first 6 hash chars) and the
|
|
1250
|
+
// canonical display path, so we reuse them here instead of a second lookup.
|
|
1251
|
+
const docid = doc.docid;
|
|
1252
|
+
const canonicalPath = `qmd://${doc.displayPath}`;
|
|
1253
|
+
// --full-path: show the on-disk path instead of the qmd:// URL + docid, when
|
|
1254
|
+
// the file actually exists. Fall back to the canonical header otherwise, and
|
|
1255
|
+
// say so on stderr — a fallback here means the index is stale.
|
|
1256
|
+
let header;
|
|
1257
|
+
if (fullPath) {
|
|
1258
|
+
const fsPath = resolveVirtualPath(db, canonicalPath);
|
|
1259
|
+
if (fsPath && existsSync(fsPath)) {
|
|
1260
|
+
header = renderFullPath(fsPath);
|
|
1261
|
+
}
|
|
1262
|
+
else {
|
|
1263
|
+
header = docid ? `${canonicalPath} #${docid}` : canonicalPath;
|
|
1264
|
+
warnUnresolvedFullPaths(1, 1);
|
|
1265
|
+
}
|
|
1266
|
+
}
|
|
1267
|
+
else {
|
|
1268
|
+
header = docid ? `${canonicalPath} #${docid}` : canonicalPath;
|
|
1269
|
+
}
|
|
1270
|
+
let output = doc.body || "";
|
|
1271
|
+
const startLine = fromLine || 1;
|
|
1272
|
+
// Apply line filtering if specified
|
|
1273
|
+
if (fromLine !== undefined || maxLines !== undefined) {
|
|
1274
|
+
const lines = output.split('\n');
|
|
1275
|
+
const start = startLine - 1; // Convert to 0-indexed
|
|
1276
|
+
const end = maxLines !== undefined ? start + maxLines : lines.length;
|
|
1277
|
+
output = lines.slice(start, end).join('\n');
|
|
1278
|
+
}
|
|
1279
|
+
// Line numbers are on by default (disable with --no-line-numbers) so the
|
|
1280
|
+
// model can cite exact lines and request follow-up ranges via path:from:count.
|
|
1281
|
+
if (lineNumbers) {
|
|
1282
|
+
output = addLineNumbers(output, startLine);
|
|
1283
|
+
}
|
|
1284
|
+
// Header: identify the document (path + docid, or the on-disk path with
|
|
1285
|
+
// --full-path), then optional context.
|
|
1286
|
+
console.log(header);
|
|
1287
|
+
if (doc.context) {
|
|
1288
|
+
console.log(`Folder Context: ${doc.context}`);
|
|
1289
|
+
}
|
|
1290
|
+
console.log("---\n");
|
|
1291
|
+
console.log(output);
|
|
1292
|
+
closeDb();
|
|
1293
|
+
}
|
|
1294
|
+
// Multi-get: fetch multiple documents by glob pattern or comma-separated list
|
|
1295
|
+
function multiGet(pattern, maxLines, maxBytes = DEFAULT_MULTI_GET_MAX_BYTES, format = "cli", lineNumbers = true, fullPath = false) {
|
|
1296
|
+
const db = getDb();
|
|
1297
|
+
// Check if it's a comma-separated list or a glob pattern
|
|
1298
|
+
const isCommaSeparated = pattern.includes(',') && !pattern.includes('*') && !pattern.includes('?') && !pattern.includes('{');
|
|
1299
|
+
const isSingleDocid = isDocid(pattern);
|
|
1300
|
+
let files;
|
|
1301
|
+
if (isCommaSeparated || isSingleDocid) {
|
|
1302
|
+
// Comma-separated list of files (can be virtual paths or relative paths)
|
|
1303
|
+
const names = isCommaSeparated
|
|
1304
|
+
? pattern.split(',').map(s => s.trim()).filter(Boolean)
|
|
1305
|
+
: [pattern.trim()].filter(Boolean);
|
|
1306
|
+
files = [];
|
|
1307
|
+
for (const name of names) {
|
|
1308
|
+
const resolved = resolveCommaListName(db, name);
|
|
1309
|
+
if (resolved.ok) {
|
|
1310
|
+
files.push({
|
|
1311
|
+
filepath: resolved.match.virtualPath,
|
|
1312
|
+
displayPath: resolved.match.virtualPath,
|
|
1313
|
+
bodyLength: resolved.match.bodyLength,
|
|
1314
|
+
collection: resolved.match.collection,
|
|
1315
|
+
path: resolved.match.path
|
|
1316
|
+
});
|
|
1317
|
+
}
|
|
1318
|
+
else {
|
|
1319
|
+
console.error(resolved.error);
|
|
1320
|
+
}
|
|
1321
|
+
}
|
|
1322
|
+
if (isSingleDocid && files.length === 0) {
|
|
1323
|
+
closeDb();
|
|
1324
|
+
process.exit(1);
|
|
1325
|
+
}
|
|
1326
|
+
}
|
|
1327
|
+
else {
|
|
1328
|
+
// Glob pattern - matchFilesByGlob now returns virtual paths
|
|
1329
|
+
files = matchFilesByGlob(db, pattern).map(f => ({
|
|
1330
|
+
...f,
|
|
1331
|
+
collection: undefined, // Will be fetched later if needed
|
|
1332
|
+
path: undefined
|
|
1333
|
+
}));
|
|
1334
|
+
if (files.length === 0) {
|
|
1335
|
+
console.error(`No files matched pattern: ${pattern}`);
|
|
1336
|
+
closeDb();
|
|
1337
|
+
process.exit(1);
|
|
1338
|
+
}
|
|
1339
|
+
}
|
|
1340
|
+
// Collect results for structured output
|
|
1341
|
+
const results = [];
|
|
1342
|
+
for (const file of files) {
|
|
1343
|
+
// Parse virtual path to get collection info if not already available
|
|
1344
|
+
let collection = file.collection;
|
|
1345
|
+
let path = file.path;
|
|
1346
|
+
if (!collection || !path) {
|
|
1347
|
+
const parsed = parseVirtualPath(file.filepath);
|
|
1348
|
+
if (parsed) {
|
|
1349
|
+
collection = parsed.collectionName;
|
|
1350
|
+
path = parsed.path;
|
|
1351
|
+
}
|
|
1352
|
+
}
|
|
1353
|
+
// Get context using collection-scoped function
|
|
1354
|
+
const context = collection && path ? getContextForPath(db, collection, path) : null;
|
|
1355
|
+
// Resolve docid (first 6 chars of content hash) so every entry can be cited.
|
|
1356
|
+
const docidRow = collection && path ? db.prepare(`
|
|
1357
|
+
SELECT d.hash as hash
|
|
1358
|
+
FROM documents d
|
|
1359
|
+
WHERE d.collection = ? AND d.path = ? AND d.active = 1
|
|
1360
|
+
`).get(collection, path) : null;
|
|
1361
|
+
const docid = docidRow?.hash ? docidRow.hash.slice(0, 6) : undefined;
|
|
1362
|
+
// --full-path: resolve the on-disk path when it exists (else fall back).
|
|
1363
|
+
// Display as ./-prefixed relative path when under $PWD; absolute realpath
|
|
1364
|
+
// otherwise. See renderFullPath() for the policy.
|
|
1365
|
+
let fsPath;
|
|
1366
|
+
if (fullPath) {
|
|
1367
|
+
const resolved = resolveVirtualPath(db, file.filepath);
|
|
1368
|
+
if (resolved && existsSync(resolved))
|
|
1369
|
+
fsPath = renderFullPath(resolved);
|
|
1370
|
+
}
|
|
1371
|
+
// Check size limit
|
|
1372
|
+
if (file.bodyLength > maxBytes) {
|
|
1373
|
+
results.push({
|
|
1374
|
+
file: file.filepath,
|
|
1375
|
+
displayPath: file.displayPath,
|
|
1376
|
+
fsPath,
|
|
1377
|
+
docid,
|
|
1378
|
+
title: file.displayPath.split('/').pop() || file.displayPath,
|
|
1379
|
+
body: "",
|
|
1380
|
+
context,
|
|
1381
|
+
skipped: true,
|
|
1382
|
+
skipReason: `File too large (${Math.round(file.bodyLength / 1024)}KB > ${Math.round(maxBytes / 1024)}KB). Use 'qmd get ${file.displayPath}' to retrieve.`,
|
|
1383
|
+
});
|
|
1384
|
+
continue;
|
|
1385
|
+
}
|
|
1386
|
+
// Fetch document content using collection and path
|
|
1387
|
+
if (!collection || !path)
|
|
1388
|
+
continue;
|
|
1389
|
+
const doc = db.prepare(`
|
|
1390
|
+
SELECT content.doc as body, d.title
|
|
1391
|
+
FROM documents d
|
|
1392
|
+
JOIN content ON content.hash = d.hash
|
|
1393
|
+
WHERE d.collection = ? AND d.path = ? AND d.active = 1
|
|
1394
|
+
`).get(collection, path);
|
|
1395
|
+
if (!doc)
|
|
1396
|
+
continue;
|
|
1397
|
+
let body = doc.body;
|
|
1398
|
+
// Apply line limit if specified
|
|
1399
|
+
if (maxLines !== undefined) {
|
|
1400
|
+
const lines = body.split('\n');
|
|
1401
|
+
body = lines.slice(0, maxLines).join('\n');
|
|
1402
|
+
if (lines.length > maxLines) {
|
|
1403
|
+
body += `\n\n[... truncated ${lines.length - maxLines} more lines]`;
|
|
1404
|
+
}
|
|
1405
|
+
}
|
|
1406
|
+
// Line numbers on by default (disable with --no-line-numbers).
|
|
1407
|
+
if (lineNumbers) {
|
|
1408
|
+
body = addLineNumbers(body);
|
|
1409
|
+
}
|
|
1410
|
+
results.push({
|
|
1411
|
+
file: file.filepath,
|
|
1412
|
+
displayPath: file.displayPath,
|
|
1413
|
+
fsPath,
|
|
1414
|
+
docid,
|
|
1415
|
+
title: doc.title || file.displayPath.split('/').pop() || file.displayPath,
|
|
1416
|
+
body,
|
|
1417
|
+
context,
|
|
1418
|
+
skipped: false,
|
|
1419
|
+
});
|
|
1420
|
+
}
|
|
1421
|
+
closeDb();
|
|
1422
|
+
// --full-path replaces the qmd:// path + docid with the on-disk path (when it
|
|
1423
|
+
// resolved). Per result: pick the identifier and whether to show the docid.
|
|
1424
|
+
const identOf = (r) => (fullPath && r.fsPath) ? r.fsPath : r.displayPath;
|
|
1425
|
+
const docidOf = (r) => (fullPath && r.fsPath) ? undefined : r.docid;
|
|
1426
|
+
const unresolvedCount = fullPath ? results.filter(r => !r.fsPath).length : 0;
|
|
1427
|
+
// Output based on format
|
|
1428
|
+
if (format === "json") {
|
|
1429
|
+
const output = results.map(r => {
|
|
1430
|
+
const docidVal = docidOf(r);
|
|
1431
|
+
return {
|
|
1432
|
+
file: identOf(r),
|
|
1433
|
+
...(docidVal && { docid: `#${docidVal}` }),
|
|
1434
|
+
title: r.title,
|
|
1435
|
+
...(r.context && { context: r.context }),
|
|
1436
|
+
...(r.skipped ? { skipped: true, reason: r.skipReason } : { body: r.body }),
|
|
1437
|
+
};
|
|
1438
|
+
});
|
|
1439
|
+
console.log(JSON.stringify(output, null, 2));
|
|
1440
|
+
}
|
|
1441
|
+
else if (format === "csv") {
|
|
1442
|
+
const escapeField = (val) => {
|
|
1443
|
+
if (val === null || val === undefined)
|
|
1444
|
+
return "";
|
|
1445
|
+
const str = String(val);
|
|
1446
|
+
if (str.includes(",") || str.includes('"') || str.includes("\n")) {
|
|
1447
|
+
return `"${str.replace(/"/g, '""')}"`;
|
|
1448
|
+
}
|
|
1449
|
+
return str;
|
|
1450
|
+
};
|
|
1451
|
+
console.log("docid,file,title,context,skipped,body");
|
|
1452
|
+
for (const r of results) {
|
|
1453
|
+
const docidVal = docidOf(r);
|
|
1454
|
+
console.log([docidVal ? `#${docidVal}` : "", identOf(r), r.title, r.context, r.skipped ? "true" : "false", r.skipped ? r.skipReason : r.body].map(escapeField).join(","));
|
|
1455
|
+
}
|
|
1456
|
+
}
|
|
1457
|
+
else if (format === "files") {
|
|
1458
|
+
// Headerless CSV: docid,filepath[,context][,status] — docid is its own
|
|
1459
|
+
// field (comma-separated), matching search --format files shape so naive
|
|
1460
|
+
// comma-splitting stays usable (#760).
|
|
1461
|
+
for (const r of results) {
|
|
1462
|
+
const docidVal = docidOf(r);
|
|
1463
|
+
const id = docidVal ? `#${docidVal},` : "";
|
|
1464
|
+
const ctx = r.context ? `,"${r.context.replace(/"/g, '""')}"` : "";
|
|
1465
|
+
const status = r.skipped ? "[SKIPPED]" : "";
|
|
1466
|
+
console.log(`${id}${identOf(r)}${ctx}${status ? `,${status}` : ""}`);
|
|
1467
|
+
}
|
|
1468
|
+
}
|
|
1469
|
+
else if (format === "md") {
|
|
1470
|
+
for (const r of results) {
|
|
1471
|
+
const docidVal = docidOf(r);
|
|
1472
|
+
console.log(`## ${identOf(r)}\n`);
|
|
1473
|
+
if (docidVal)
|
|
1474
|
+
console.log(`**docid:** \`#${docidVal}\`\n`);
|
|
1475
|
+
if (r.title && r.title !== r.displayPath)
|
|
1476
|
+
console.log(`**Title:** ${r.title}\n`);
|
|
1477
|
+
if (r.context)
|
|
1478
|
+
console.log(`**Context:** ${r.context}\n`);
|
|
1479
|
+
if (r.skipped) {
|
|
1480
|
+
console.log(`> ${r.skipReason}\n`);
|
|
1481
|
+
}
|
|
1482
|
+
else {
|
|
1483
|
+
console.log("```");
|
|
1484
|
+
console.log(r.body);
|
|
1485
|
+
console.log("```\n");
|
|
1486
|
+
}
|
|
1487
|
+
}
|
|
1488
|
+
}
|
|
1489
|
+
else if (format === "xml") {
|
|
1490
|
+
console.log('<?xml version="1.0" encoding="UTF-8"?>');
|
|
1491
|
+
console.log("<documents>");
|
|
1492
|
+
for (const r of results) {
|
|
1493
|
+
const docidVal = docidOf(r);
|
|
1494
|
+
const docidAttr = docidVal ? ` docid="#${docidVal}"` : "";
|
|
1495
|
+
console.log(` <document${docidAttr}>`);
|
|
1496
|
+
console.log(` <file>${escapeXml(identOf(r))}</file>`);
|
|
1497
|
+
console.log(` <title>${escapeXml(r.title)}</title>`);
|
|
1498
|
+
if (r.context)
|
|
1499
|
+
console.log(` <context>${escapeXml(r.context)}</context>`);
|
|
1500
|
+
if (r.skipped) {
|
|
1501
|
+
console.log(` <skipped>true</skipped>`);
|
|
1502
|
+
console.log(` <reason>${escapeXml(r.skipReason || "")}</reason>`);
|
|
1503
|
+
}
|
|
1504
|
+
else {
|
|
1505
|
+
console.log(` <body>${escapeXml(r.body)}</body>`);
|
|
1506
|
+
}
|
|
1507
|
+
console.log(" </document>");
|
|
1508
|
+
}
|
|
1509
|
+
console.log("</documents>");
|
|
1510
|
+
}
|
|
1511
|
+
else {
|
|
1512
|
+
// CLI format (default)
|
|
1513
|
+
for (const r of results) {
|
|
1514
|
+
const docidVal = docidOf(r);
|
|
1515
|
+
const id = docidVal ? ` #${docidVal}` : "";
|
|
1516
|
+
console.log(`\n${'='.repeat(60)}`);
|
|
1517
|
+
console.log(`File: ${identOf(r)}${id}`);
|
|
1518
|
+
console.log(`${'='.repeat(60)}\n`);
|
|
1519
|
+
if (r.skipped) {
|
|
1520
|
+
console.log(`[SKIPPED: ${r.skipReason}]`);
|
|
1521
|
+
continue;
|
|
1522
|
+
}
|
|
1523
|
+
if (r.context) {
|
|
1524
|
+
console.log(`Folder Context: ${r.context}\n---\n`);
|
|
1525
|
+
}
|
|
1526
|
+
console.log(r.body);
|
|
1527
|
+
}
|
|
1528
|
+
}
|
|
1529
|
+
warnUnresolvedFullPaths(unresolvedCount, results.length);
|
|
1530
|
+
}
|
|
1531
|
+
// List files in virtual file tree
|
|
1532
|
+
function listFiles(pathArg) {
|
|
1533
|
+
const db = getDb();
|
|
1534
|
+
if (!pathArg) {
|
|
1535
|
+
// No argument - list all collections
|
|
1536
|
+
const yamlCollections = yamlListCollections();
|
|
1537
|
+
if (yamlCollections.length === 0) {
|
|
1538
|
+
console.log("No collections found. Run 'qmd collection add .' to index files.");
|
|
1539
|
+
closeDb();
|
|
1540
|
+
return;
|
|
1541
|
+
}
|
|
1542
|
+
// Get file counts from database for each collection
|
|
1543
|
+
const collections = yamlCollections.map(coll => {
|
|
1544
|
+
const stats = db.prepare(`
|
|
1545
|
+
SELECT COUNT(*) as file_count
|
|
1546
|
+
FROM documents d
|
|
1547
|
+
WHERE d.collection = ? AND d.active = 1
|
|
1548
|
+
`).get(coll.name);
|
|
1549
|
+
return {
|
|
1550
|
+
name: coll.name,
|
|
1551
|
+
file_count: stats?.file_count || 0
|
|
1552
|
+
};
|
|
1553
|
+
});
|
|
1554
|
+
console.log(`${c.bold}Collections:${c.reset}\n`);
|
|
1555
|
+
for (const coll of collections) {
|
|
1556
|
+
console.log(` ${c.dim}qmd://${c.reset}${c.cyan}${coll.name}/${c.reset} ${c.dim}(${coll.file_count} files)${c.reset}`);
|
|
1557
|
+
}
|
|
1558
|
+
closeDb();
|
|
1559
|
+
return;
|
|
1560
|
+
}
|
|
1561
|
+
// Parse the path argument
|
|
1562
|
+
let collectionName;
|
|
1563
|
+
let pathPrefix = null;
|
|
1564
|
+
const afterScheme = pathArg.startsWith('qmd://') ? pathArg.slice('qmd://'.length) : null;
|
|
1565
|
+
if (afterScheme !== null && afterScheme.startsWith('/')) {
|
|
1566
|
+
// Absolute-path collection: qmd:///Users/foo/bar — normalizeVirtualPath would corrupt
|
|
1567
|
+
// this by stripping all leading slashes, so bypass parseVirtualPath entirely.
|
|
1568
|
+
const normalized = afterScheme.replace(/\/$/, '');
|
|
1569
|
+
const allColls = yamlListCollections();
|
|
1570
|
+
const match = allColls
|
|
1571
|
+
.filter(c => normalized === c.name || normalized.startsWith(c.name + '/'))
|
|
1572
|
+
.sort((a, b) => b.name.length - a.name.length)[0];
|
|
1573
|
+
if (match) {
|
|
1574
|
+
collectionName = match.name;
|
|
1575
|
+
const rest = normalized.slice(match.name.length).replace(/^\//, '');
|
|
1576
|
+
pathPrefix = rest || null;
|
|
1577
|
+
}
|
|
1578
|
+
else {
|
|
1579
|
+
// Preserve the historical qmd:////collection/path alias behavior for normal
|
|
1580
|
+
// collections when no absolute-path collection matches.
|
|
1581
|
+
const parsed = parseVirtualPath(pathArg);
|
|
1582
|
+
if (!parsed) {
|
|
1583
|
+
console.error(`Invalid virtual path: ${pathArg}`);
|
|
1584
|
+
closeDb();
|
|
1585
|
+
process.exit(1);
|
|
1586
|
+
}
|
|
1587
|
+
collectionName = parsed.collectionName;
|
|
1588
|
+
pathPrefix = parsed.path;
|
|
1589
|
+
}
|
|
1590
|
+
}
|
|
1591
|
+
else if (afterScheme !== null) {
|
|
1592
|
+
// Normal virtual path: qmd://collection-name/path
|
|
1593
|
+
const parsed = parseVirtualPath(pathArg);
|
|
1594
|
+
if (!parsed) {
|
|
1595
|
+
console.error(`Invalid virtual path: ${pathArg}`);
|
|
1596
|
+
closeDb();
|
|
1597
|
+
process.exit(1);
|
|
1598
|
+
}
|
|
1599
|
+
collectionName = parsed.collectionName;
|
|
1600
|
+
pathPrefix = parsed.path;
|
|
1601
|
+
}
|
|
1602
|
+
else if (pathArg.startsWith('/')) {
|
|
1603
|
+
// Raw absolute filesystem path — longest-prefix match against collection names
|
|
1604
|
+
const normalized = pathArg.replace(/\/$/, '');
|
|
1605
|
+
const allColls = yamlListCollections();
|
|
1606
|
+
const match = allColls
|
|
1607
|
+
.filter(c => normalized === c.name || normalized.startsWith(c.name + '/'))
|
|
1608
|
+
.sort((a, b) => b.name.length - a.name.length)[0];
|
|
1609
|
+
if (match) {
|
|
1610
|
+
collectionName = match.name;
|
|
1611
|
+
const rest = normalized.slice(match.name.length).replace(/^\//, '');
|
|
1612
|
+
pathPrefix = rest || null;
|
|
1613
|
+
}
|
|
1614
|
+
else {
|
|
1615
|
+
collectionName = normalized;
|
|
1616
|
+
}
|
|
1617
|
+
}
|
|
1618
|
+
else {
|
|
1619
|
+
// Short collection name or name/path
|
|
1620
|
+
const parts = pathArg.split('/');
|
|
1621
|
+
collectionName = parts[0] || '';
|
|
1622
|
+
if (parts.length > 1) {
|
|
1623
|
+
pathPrefix = parts.slice(1).join('/');
|
|
1624
|
+
}
|
|
1625
|
+
}
|
|
1626
|
+
// Get the collection
|
|
1627
|
+
const coll = getCollectionFromYaml(collectionName);
|
|
1628
|
+
if (!coll) {
|
|
1629
|
+
console.error(`Collection not found: ${collectionName}`);
|
|
1630
|
+
console.error(`Run 'qmd ls' to see available collections.`);
|
|
1631
|
+
closeDb();
|
|
1632
|
+
process.exit(1);
|
|
1633
|
+
}
|
|
1634
|
+
// List files in the collection with size and modification time
|
|
1635
|
+
let query;
|
|
1636
|
+
let params;
|
|
1637
|
+
if (pathPrefix) {
|
|
1638
|
+
// List files under a specific path
|
|
1639
|
+
query = `
|
|
1640
|
+
SELECT d.path, d.title, d.modified_at, LENGTH(ct.doc) as size
|
|
1641
|
+
FROM documents d
|
|
1642
|
+
JOIN content ct ON d.hash = ct.hash
|
|
1643
|
+
WHERE d.collection = ? AND d.path LIKE ? ESCAPE '#' AND d.active = 1
|
|
1644
|
+
ORDER BY d.path
|
|
1645
|
+
`;
|
|
1646
|
+
params = [coll.name, `${escapeLikePattern(pathPrefix)}%`];
|
|
1647
|
+
}
|
|
1648
|
+
else {
|
|
1649
|
+
// List all files in the collection
|
|
1650
|
+
query = `
|
|
1651
|
+
SELECT d.path, d.title, d.modified_at, LENGTH(ct.doc) as size
|
|
1652
|
+
FROM documents d
|
|
1653
|
+
JOIN content ct ON d.hash = ct.hash
|
|
1654
|
+
WHERE d.collection = ? AND d.active = 1
|
|
1655
|
+
ORDER BY d.path
|
|
1656
|
+
`;
|
|
1657
|
+
params = [coll.name];
|
|
1658
|
+
}
|
|
1659
|
+
const files = db.prepare(query).all(...params);
|
|
1660
|
+
if (files.length === 0) {
|
|
1661
|
+
if (pathPrefix) {
|
|
1662
|
+
console.log(`No files found under qmd://${collectionName}/${pathPrefix}`);
|
|
1663
|
+
}
|
|
1664
|
+
else {
|
|
1665
|
+
console.log(`No files found in collection: ${collectionName}`);
|
|
1666
|
+
}
|
|
1667
|
+
closeDb();
|
|
1668
|
+
return;
|
|
1669
|
+
}
|
|
1670
|
+
// Calculate max widths for alignment
|
|
1671
|
+
const maxSize = Math.max(...files.map(f => formatBytes(f.size).length));
|
|
1672
|
+
// Output in ls -l style
|
|
1673
|
+
for (const file of files) {
|
|
1674
|
+
const sizeStr = formatBytes(file.size).padStart(maxSize);
|
|
1675
|
+
const date = new Date(file.modified_at);
|
|
1676
|
+
const timeStr = formatLsTime(date);
|
|
1677
|
+
// Dim the qmd:// prefix, highlight the filename
|
|
1678
|
+
console.log(`${sizeStr} ${timeStr} ${c.dim}qmd://${collectionName}/${c.reset}${c.cyan}${file.path}${c.reset}`);
|
|
1679
|
+
}
|
|
1680
|
+
closeDb();
|
|
1681
|
+
}
|
|
1682
|
+
// Format date/time like ls -l
|
|
1683
|
+
function formatLsTime(date) {
|
|
1684
|
+
const now = new Date();
|
|
1685
|
+
const sixMonthsAgo = new Date(now.getTime() - 6 * 30 * 24 * 60 * 60 * 1000);
|
|
1686
|
+
const months = ['Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec'];
|
|
1687
|
+
const month = months[date.getMonth()];
|
|
1688
|
+
const day = date.getDate().toString().padStart(2, ' ');
|
|
1689
|
+
// If file is older than 6 months, show year instead of time
|
|
1690
|
+
if (date < sixMonthsAgo) {
|
|
1691
|
+
const year = date.getFullYear();
|
|
1692
|
+
return `${month} ${day} ${year}`;
|
|
1693
|
+
}
|
|
1694
|
+
else {
|
|
1695
|
+
const hours = date.getHours().toString().padStart(2, '0');
|
|
1696
|
+
const minutes = date.getMinutes().toString().padStart(2, '0');
|
|
1697
|
+
return `${month} ${day} ${hours}:${minutes}`;
|
|
1698
|
+
}
|
|
1699
|
+
}
|
|
1700
|
+
// Collection management commands
|
|
1701
|
+
function collectionList() {
|
|
1702
|
+
const db = getDb();
|
|
1703
|
+
const collections = listCollections(db);
|
|
1704
|
+
if (collections.length === 0) {
|
|
1705
|
+
console.log("No collections found. Run 'qmd collection add .' to create one.");
|
|
1706
|
+
closeDb();
|
|
1707
|
+
return;
|
|
1708
|
+
}
|
|
1709
|
+
console.log(`${c.bold}Collections (${collections.length}):${c.reset}\n`);
|
|
1710
|
+
for (const coll of collections) {
|
|
1711
|
+
const updatedAt = coll.last_modified ? new Date(coll.last_modified) : new Date();
|
|
1712
|
+
const timeAgo = formatTimeAgo(updatedAt);
|
|
1713
|
+
// Get YAML config to check includeByDefault
|
|
1714
|
+
const yamlColl = getCollectionFromYaml(coll.name);
|
|
1715
|
+
const excluded = yamlColl?.includeByDefault === false;
|
|
1716
|
+
const excludeTag = excluded ? ` ${c.yellow}[excluded]${c.reset}` : '';
|
|
1717
|
+
console.log(`${c.cyan}${coll.name}${c.reset} ${c.dim}(qmd://${coll.name}/)${c.reset}${excludeTag}`);
|
|
1718
|
+
console.log(` ${c.dim}Pattern:${c.reset} ${coll.glob_pattern}`);
|
|
1719
|
+
if (yamlColl?.ignore?.length) {
|
|
1720
|
+
console.log(` ${c.dim}Ignore:${c.reset} ${yamlColl.ignore.join(', ')}`);
|
|
1721
|
+
}
|
|
1722
|
+
console.log(` ${c.dim}Files:${c.reset} ${coll.active_count}`);
|
|
1723
|
+
console.log(` ${c.dim}Updated:${c.reset} ${timeAgo}`);
|
|
1724
|
+
console.log();
|
|
1725
|
+
}
|
|
1726
|
+
closeDb();
|
|
1727
|
+
}
|
|
1728
|
+
/** Canonical --mask, with --glob as the alias OpenClaw and others already pass (#536). */
|
|
1729
|
+
function collectionGlobFromCli(values) {
|
|
1730
|
+
const mask = typeof values.mask === "string" && values.mask.length > 0 ? values.mask : undefined;
|
|
1731
|
+
const glob = typeof values.glob === "string" && values.glob.length > 0 ? values.glob : undefined;
|
|
1732
|
+
return mask ?? glob ?? DEFAULT_GLOB;
|
|
1733
|
+
}
|
|
1734
|
+
async function collectionAdd(pwd, globPattern, name) {
|
|
1735
|
+
// If name not provided, generate from pwd basename
|
|
1736
|
+
let collName = name;
|
|
1737
|
+
if (!collName) {
|
|
1738
|
+
const parts = pwd.split('/').filter(Boolean);
|
|
1739
|
+
collName = parts[parts.length - 1] || 'root';
|
|
1740
|
+
}
|
|
1741
|
+
// Check if collection with this name already exists in YAML
|
|
1742
|
+
const existing = getCollectionFromYaml(collName);
|
|
1743
|
+
if (existing) {
|
|
1744
|
+
console.error(`${c.yellow}Collection '${collName}' already exists.${c.reset}`);
|
|
1745
|
+
console.error(`Use a different name with --name <name>`);
|
|
1746
|
+
process.exit(1);
|
|
1747
|
+
}
|
|
1748
|
+
// Check if a collection with this pwd+glob already exists in YAML
|
|
1749
|
+
const allCollections = yamlListCollections();
|
|
1750
|
+
const existingPwdGlob = allCollections.find(c => c.path === pwd && c.pattern === globPattern);
|
|
1751
|
+
if (existingPwdGlob) {
|
|
1752
|
+
console.error(`${c.yellow}A collection already exists for this path and pattern:${c.reset}`);
|
|
1753
|
+
console.error(` Name: ${existingPwdGlob.name} (qmd://${existingPwdGlob.name}/)`);
|
|
1754
|
+
console.error(` Pattern: ${globPattern}`);
|
|
1755
|
+
console.error(`\nUse 'qmd update' to re-index it, or remove it first with 'qmd collection remove ${existingPwdGlob.name}'`);
|
|
1756
|
+
process.exit(1);
|
|
1757
|
+
}
|
|
1758
|
+
// Add to YAML config + sync to SQLite
|
|
1759
|
+
const { addCollection } = await import("../collections.js");
|
|
1760
|
+
addCollection(collName, pwd, globPattern);
|
|
1761
|
+
// The user just typed this path, so it needs no separate approval (#889).
|
|
1762
|
+
trustCurrentConfig();
|
|
1763
|
+
resyncConfig();
|
|
1764
|
+
// Create the collection and index files
|
|
1765
|
+
console.log(`Creating collection '${collName}'...`);
|
|
1766
|
+
const newColl = getCollectionFromYaml(collName);
|
|
1767
|
+
await indexFiles(pwd, globPattern, collName, false, newColl?.ignore);
|
|
1768
|
+
console.log(`${c.green}✓${c.reset} Collection '${collName}' created successfully`);
|
|
1769
|
+
}
|
|
1770
|
+
function collectionRemove(name) {
|
|
1771
|
+
// Check if collection exists in YAML
|
|
1772
|
+
const coll = getCollectionFromYaml(name);
|
|
1773
|
+
if (!coll) {
|
|
1774
|
+
console.error(`${c.yellow}Collection not found: ${name}${c.reset}`);
|
|
1775
|
+
console.error(`Run 'qmd collection list' to see available collections.`);
|
|
1776
|
+
process.exit(1);
|
|
1777
|
+
}
|
|
1778
|
+
const db = getDb();
|
|
1779
|
+
const deletedDocs = Number(db.prepare(`SELECT COUNT(*) AS count FROM documents WHERE collection = ?`).get(name).count);
|
|
1780
|
+
const cleanedHashes = Number(db.prepare(`
|
|
1781
|
+
SELECT COUNT(*) AS count
|
|
1782
|
+
FROM content c
|
|
1783
|
+
WHERE NOT EXISTS (
|
|
1784
|
+
SELECT 1 FROM documents d
|
|
1785
|
+
WHERE d.hash = c.hash AND d.collection <> ?
|
|
1786
|
+
)
|
|
1787
|
+
`).get(name).count);
|
|
1788
|
+
// External config is authoritative; reconciliation performs the SQLite mutation.
|
|
1789
|
+
yamlRemoveCollectionFn(name);
|
|
1790
|
+
resyncConfig();
|
|
1791
|
+
closeDb();
|
|
1792
|
+
console.log(`${c.green}✓${c.reset} Removed collection '${name}'`);
|
|
1793
|
+
console.log(` Deleted ${deletedDocs} documents`);
|
|
1794
|
+
if (cleanedHashes > 0) {
|
|
1795
|
+
console.log(` Cleaned up ${cleanedHashes} orphaned content hashes`);
|
|
1796
|
+
}
|
|
1797
|
+
}
|
|
1798
|
+
function collectionRename(oldName, newName) {
|
|
1799
|
+
// Check if old collection exists in YAML
|
|
1800
|
+
const coll = getCollectionFromYaml(oldName);
|
|
1801
|
+
if (!coll) {
|
|
1802
|
+
console.error(`${c.yellow}Collection not found: ${oldName}${c.reset}`);
|
|
1803
|
+
console.error(`Run 'qmd collection list' to see available collections.`);
|
|
1804
|
+
process.exit(1);
|
|
1805
|
+
}
|
|
1806
|
+
// Check if new name already exists in YAML
|
|
1807
|
+
const existing = getCollectionFromYaml(newName);
|
|
1808
|
+
if (existing) {
|
|
1809
|
+
console.error(`${c.yellow}Collection name already exists: ${newName}${c.reset}`);
|
|
1810
|
+
console.error(`Choose a different name or remove the existing collection first.`);
|
|
1811
|
+
process.exit(1);
|
|
1812
|
+
}
|
|
1813
|
+
getDb();
|
|
1814
|
+
// External config is authoritative; reconciliation performs the SQLite mutation.
|
|
1815
|
+
yamlRenameCollectionFn(oldName, newName);
|
|
1816
|
+
resyncConfig();
|
|
1817
|
+
closeDb();
|
|
1818
|
+
console.log(`${c.green}✓${c.reset} Renamed collection '${oldName}' to '${newName}'`);
|
|
1819
|
+
console.log(` Virtual paths updated: ${c.cyan}qmd://${oldName}/${c.reset} → ${c.cyan}qmd://${newName}/${c.reset}`);
|
|
1820
|
+
}
|
|
1821
|
+
async function indexFiles(pwd, globPattern = DEFAULT_GLOB, collectionName, suppressEmbedNotice = false, ignorePatterns) {
|
|
1822
|
+
const db = getDb();
|
|
1823
|
+
const resolvedPwd = pwd || getPwd();
|
|
1824
|
+
const now = new Date().toISOString();
|
|
1825
|
+
const excludeDirs = ["node_modules", ".git", ".cache", "vendor", "dist", "build"];
|
|
1826
|
+
// Clear Ollama cache on index
|
|
1827
|
+
clearCache(db);
|
|
1828
|
+
// Collection name must be provided (from YAML)
|
|
1829
|
+
if (!collectionName) {
|
|
1830
|
+
throw new Error("Collection name is required. Collections must be defined in ~/.config/qmd/index.yml");
|
|
1831
|
+
}
|
|
1832
|
+
console.log(`Collection: ${resolvedPwd} (${globPattern})`);
|
|
1833
|
+
progress.indeterminate();
|
|
1834
|
+
const allIgnore = [
|
|
1835
|
+
...excludeDirs.map(d => `**/${d}/**`),
|
|
1836
|
+
...(ignorePatterns || []),
|
|
1837
|
+
];
|
|
1838
|
+
const allFiles = await fastGlob(splitGlobMask(globPattern), {
|
|
1839
|
+
cwd: resolvedPwd,
|
|
1840
|
+
onlyFiles: true,
|
|
1841
|
+
followSymbolicLinks: false,
|
|
1842
|
+
dot: false,
|
|
1843
|
+
ignore: allIgnore,
|
|
1844
|
+
});
|
|
1845
|
+
// Filter hidden files/folders (dot: false handles top-level but not nested)
|
|
1846
|
+
const files = allFiles.filter(file => {
|
|
1847
|
+
const parts = file.split("/");
|
|
1848
|
+
return !parts.some(part => part.startsWith("."));
|
|
1849
|
+
});
|
|
1850
|
+
const total = files.length;
|
|
1851
|
+
const hasNoFiles = total === 0;
|
|
1852
|
+
if (hasNoFiles) {
|
|
1853
|
+
progress.clear();
|
|
1854
|
+
console.log("No files found matching pattern.");
|
|
1855
|
+
// Continue so the deactivation pass can mark previously indexed docs as inactive.
|
|
1856
|
+
}
|
|
1857
|
+
let indexed = 0, updated = 0, unchanged = 0, processed = 0;
|
|
1858
|
+
const skippedFiles = [];
|
|
1859
|
+
const seenPaths = new Set();
|
|
1860
|
+
// Literal paths of every file in this scan. Passed to the legacy-path
|
|
1861
|
+
// migration so it never adopts a row that still belongs to a live file.
|
|
1862
|
+
const livePaths = new Set(files.map(f => f.replace(/\\/g, '/')));
|
|
1863
|
+
const startTime = Date.now();
|
|
1864
|
+
for (const relativeFile of files) {
|
|
1865
|
+
const filepath = getRealPath(resolve(resolvedPwd, relativeFile));
|
|
1866
|
+
// Store the literal relative path — handelize() is NOT applied at index time.
|
|
1867
|
+
const path = relativeFile.replace(/\\/g, '/');
|
|
1868
|
+
if (!isPathInsideDir(resolvedPwd, filepath)) {
|
|
1869
|
+
processed++;
|
|
1870
|
+
skippedFiles.push({ file: relativeFile, code: "OUTSIDE_COLLECTION" });
|
|
1871
|
+
progress.set((processed / total) * 100);
|
|
1872
|
+
continue;
|
|
1873
|
+
}
|
|
1874
|
+
seenPaths.add(path);
|
|
1875
|
+
let content;
|
|
1876
|
+
try {
|
|
1877
|
+
content = readFileSync(filepath, "utf-8");
|
|
1878
|
+
}
|
|
1879
|
+
catch (err) {
|
|
1880
|
+
// Skip files that can't be read (ETIMEDOUT, EAGAIN, EACCES, …) (#460)
|
|
1881
|
+
processed++;
|
|
1882
|
+
skippedFiles.push({ file: relativeFile, code: fsErrorCode(err) });
|
|
1883
|
+
progress.set((processed / total) * 100);
|
|
1884
|
+
continue;
|
|
1885
|
+
}
|
|
1886
|
+
// Skip empty files - nothing useful to index
|
|
1887
|
+
if (!content.trim()) {
|
|
1888
|
+
processed++;
|
|
1889
|
+
continue;
|
|
1890
|
+
}
|
|
1891
|
+
const hash = await hashContent(content);
|
|
1892
|
+
const title = extractTitle(content, relativeFile);
|
|
1893
|
+
// Check if document exists (also migrates legacy lowercase paths)
|
|
1894
|
+
const existing = findOrMigrateLegacyDocument(db, collectionName, path, livePaths);
|
|
1895
|
+
if (existing) {
|
|
1896
|
+
if (existing.hash === hash) {
|
|
1897
|
+
// Hash unchanged, but check if title needs updating
|
|
1898
|
+
if (existing.title !== title) {
|
|
1899
|
+
updateDocumentTitle(db, existing.id, title, now);
|
|
1900
|
+
updated++;
|
|
1901
|
+
}
|
|
1902
|
+
else {
|
|
1903
|
+
unchanged++;
|
|
1904
|
+
}
|
|
1905
|
+
}
|
|
1906
|
+
else {
|
|
1907
|
+
// Content changed - insert new content hash and update document
|
|
1908
|
+
const stat = statSync(filepath);
|
|
1909
|
+
updateDocumentWithContent(db, hash, content, now, existing.id, title, stat ? new Date(stat.mtime).toISOString() : now);
|
|
1910
|
+
updated++;
|
|
1911
|
+
}
|
|
1912
|
+
}
|
|
1913
|
+
else {
|
|
1914
|
+
// New document - insert content and document
|
|
1915
|
+
indexed++;
|
|
1916
|
+
const stat = statSync(filepath);
|
|
1917
|
+
insertDocumentWithContent(db, hash, content, now, collectionName, path, title, stat ? new Date(stat.birthtime).toISOString() : now, stat ? new Date(stat.mtime).toISOString() : now);
|
|
1918
|
+
}
|
|
1919
|
+
processed++;
|
|
1920
|
+
progress.set((processed / total) * 100);
|
|
1921
|
+
const elapsed = (Date.now() - startTime) / 1000;
|
|
1922
|
+
const rate = processed / elapsed;
|
|
1923
|
+
const remaining = (total - processed) / rate;
|
|
1924
|
+
const eta = processed > 2 ? ` ETA: ${formatETA(remaining)}` : "";
|
|
1925
|
+
if (isTTY)
|
|
1926
|
+
process.stderr.write(`\rIndexing: ${processed}/${total}${eta} `);
|
|
1927
|
+
}
|
|
1928
|
+
// Deactivate documents in this collection that no longer exist
|
|
1929
|
+
const allActive = getActiveDocumentPaths(db, collectionName);
|
|
1930
|
+
let removed = 0;
|
|
1931
|
+
for (const path of allActive) {
|
|
1932
|
+
if (!seenPaths.has(path)) {
|
|
1933
|
+
deactivateDocument(db, collectionName, path);
|
|
1934
|
+
removed++;
|
|
1935
|
+
}
|
|
1936
|
+
}
|
|
1937
|
+
// Clean up orphaned content hashes (content not referenced by any document)
|
|
1938
|
+
const orphanedContent = cleanupOrphanedContent(db);
|
|
1939
|
+
// Check if vector index needs updating
|
|
1940
|
+
const needsEmbedding = getHashesNeedingEmbedding(db);
|
|
1941
|
+
progress.clear();
|
|
1942
|
+
console.log(`\nIndexed: ${indexed} new, ${updated} updated, ${unchanged} unchanged, ${removed} removed`);
|
|
1943
|
+
reportSkippedReads(skippedFiles);
|
|
1944
|
+
if (orphanedContent > 0) {
|
|
1945
|
+
console.log(`Cleaned up ${orphanedContent} orphaned content hash(es)`);
|
|
1946
|
+
}
|
|
1947
|
+
if (needsEmbedding > 0 && !suppressEmbedNotice) {
|
|
1948
|
+
console.log(`\nRun 'qmd embed' to update embeddings (${needsEmbedding} unique hashes need vectors)`);
|
|
1949
|
+
}
|
|
1950
|
+
await rebuildCjkLexicalIndex(getDbPath());
|
|
1951
|
+
closeDb();
|
|
1952
|
+
}
|
|
1953
|
+
function fsErrorCode(err) {
|
|
1954
|
+
if (err && typeof err === "object" && "code" in err) {
|
|
1955
|
+
const code = err.code;
|
|
1956
|
+
if (typeof code === "string" && code.length > 0)
|
|
1957
|
+
return code;
|
|
1958
|
+
}
|
|
1959
|
+
return "ERROR";
|
|
1960
|
+
}
|
|
1961
|
+
function reportSkippedReads(skippedFiles) {
|
|
1962
|
+
if (skippedFiles.length === 0)
|
|
1963
|
+
return;
|
|
1964
|
+
for (const skipped of skippedFiles) {
|
|
1965
|
+
if (skipped.code === "OUTSIDE_COLLECTION") {
|
|
1966
|
+
console.warn(`⚠ Skipped file outside collection: ${skipped.file}`);
|
|
1967
|
+
}
|
|
1968
|
+
else {
|
|
1969
|
+
console.warn(`⚠ Skipped unreadable file: ${skipped.file} (${skipped.code})`);
|
|
1970
|
+
}
|
|
1971
|
+
}
|
|
1972
|
+
const escaped = skippedFiles.filter(f => f.code === "OUTSIDE_COLLECTION").length;
|
|
1973
|
+
const unreadable = skippedFiles.length - escaped;
|
|
1974
|
+
if (escaped)
|
|
1975
|
+
console.warn(`Skipped ${escaped} file(s) outside the collection root`);
|
|
1976
|
+
if (unreadable)
|
|
1977
|
+
console.warn(`Skipped ${unreadable} unreadable file(s)`);
|
|
1978
|
+
}
|
|
1979
|
+
function renderProgressBar(percent, width = 30) {
|
|
1980
|
+
const filled = Math.round((percent / 100) * width);
|
|
1981
|
+
const empty = width - filled;
|
|
1982
|
+
const bar = "█".repeat(filled) + "░".repeat(empty);
|
|
1983
|
+
return bar;
|
|
1984
|
+
}
|
|
1985
|
+
function parseEmbedBatchOption(name, value) {
|
|
1986
|
+
if (value === undefined)
|
|
1987
|
+
return undefined;
|
|
1988
|
+
const parsed = Number(value);
|
|
1989
|
+
if (!Number.isInteger(parsed) || parsed < 1) {
|
|
1990
|
+
throw new Error(`${name} must be a positive integer`);
|
|
1991
|
+
}
|
|
1992
|
+
return parsed;
|
|
1993
|
+
}
|
|
1994
|
+
function parseChunkStrategy(value) {
|
|
1995
|
+
if (value === undefined)
|
|
1996
|
+
return undefined;
|
|
1997
|
+
const s = String(value);
|
|
1998
|
+
if (s === "auto" || s === "regex")
|
|
1999
|
+
return s;
|
|
2000
|
+
throw new Error(`--chunk-strategy must be "auto" or "regex" (got "${s}")`);
|
|
2001
|
+
}
|
|
2002
|
+
// --timeout for `qmd embed`: a cap on the whole embed session, in minutes. Returns
|
|
2003
|
+
// the value in milliseconds, or undefined to use the default. 0 disables the cap.
|
|
2004
|
+
function parseEmbedTimeoutOption(value) {
|
|
2005
|
+
if (value === undefined)
|
|
2006
|
+
return undefined;
|
|
2007
|
+
const minutes = Number(value);
|
|
2008
|
+
if (!Number.isFinite(minutes) || minutes < 0) {
|
|
2009
|
+
throw new Error(`--timeout must be a non-negative number of minutes (0 = no limit)`);
|
|
2010
|
+
}
|
|
2011
|
+
return minutes * 60 * 1000;
|
|
2012
|
+
}
|
|
2013
|
+
function ensureModelsConfiguredForCli() {
|
|
2014
|
+
try {
|
|
2015
|
+
const config = loadConfig();
|
|
2016
|
+
const current = config.models ?? {};
|
|
2017
|
+
const defaultLocalModels = {
|
|
2018
|
+
embed: current.embed || process.env.QMD_EMBED_MODEL || DEFAULT_EMBED_MODEL,
|
|
2019
|
+
generate: current.generate || process.env.QMD_GENERATE_MODEL || DEFAULT_QUERY_MODEL,
|
|
2020
|
+
rerank: current.rerank || process.env.QMD_RERANK_MODEL || DEFAULT_RERANK_MODEL,
|
|
2021
|
+
};
|
|
2022
|
+
if (current.embed !== defaultLocalModels.embed || current.generate !== defaultLocalModels.generate || current.rerank !== defaultLocalModels.rerank) {
|
|
2023
|
+
saveConfig({
|
|
2024
|
+
...config,
|
|
2025
|
+
models: {
|
|
2026
|
+
...current,
|
|
2027
|
+
embed: defaultLocalModels.embed,
|
|
2028
|
+
generate: defaultLocalModels.generate,
|
|
2029
|
+
rerank: defaultLocalModels.rerank,
|
|
2030
|
+
},
|
|
2031
|
+
});
|
|
2032
|
+
}
|
|
2033
|
+
return resolveModels(config.models);
|
|
2034
|
+
}
|
|
2035
|
+
catch {
|
|
2036
|
+
return resolveModels();
|
|
2037
|
+
}
|
|
2038
|
+
}
|
|
2039
|
+
export function resolveEmbedModelForCli() {
|
|
2040
|
+
try {
|
|
2041
|
+
const config = loadConfig();
|
|
2042
|
+
const resolved = resolveEmbeddingConfig({
|
|
2043
|
+
config,
|
|
2044
|
+
defaultLocalModel: process.env.QMD_EMBED_MODEL || DEFAULT_EMBED_MODEL,
|
|
2045
|
+
});
|
|
2046
|
+
return resolved.canonical.model;
|
|
2047
|
+
}
|
|
2048
|
+
catch {
|
|
2049
|
+
return ensureModelsConfiguredForCli().embed;
|
|
2050
|
+
}
|
|
2051
|
+
}
|
|
2052
|
+
export function resolveGenerateModelForCli() {
|
|
2053
|
+
return ensureModelsConfiguredForCli().generate;
|
|
2054
|
+
}
|
|
2055
|
+
export function resolveRerankModelForCli() {
|
|
2056
|
+
return ensureModelsConfiguredForCli().rerank;
|
|
2057
|
+
}
|
|
2058
|
+
function resolveModelsForCli() {
|
|
2059
|
+
return ensureModelsConfiguredForCli();
|
|
2060
|
+
}
|
|
2061
|
+
/** Models that may actually be loaded. Falls back to defaults/env when a
|
|
2062
|
+
* project-local config's custom URIs are not trusted (#889). */
|
|
2063
|
+
function resolveModelsForRuntime() {
|
|
2064
|
+
const configured = ensureModelsConfiguredForCli();
|
|
2065
|
+
if (localConfigIsFullyTrusted())
|
|
2066
|
+
return configured;
|
|
2067
|
+
return resolveModels();
|
|
2068
|
+
}
|
|
2069
|
+
async function vectorIndex(model = resolveEmbedModelForCli(), force = false, batchOptions) {
|
|
2070
|
+
const storeInstance = getStore();
|
|
2071
|
+
const db = storeInstance.db;
|
|
2072
|
+
const provider = storeInstance.embeddingProvider;
|
|
2073
|
+
if (provider?.remote) {
|
|
2074
|
+
if (force && batchOptions?.collection) {
|
|
2075
|
+
throw new EmbeddingConfigError("Remote destructive embedding rebuilds cannot be collection-scoped.");
|
|
2076
|
+
}
|
|
2077
|
+
const identity = remoteEmbeddingIdentity(provider, batchOptions?.chunkStrategy);
|
|
2078
|
+
const pending = getPendingEmbeddingDocsReadOnly(db, batchOptions?.collection, identity.model, identity.fingerprint);
|
|
2079
|
+
if (pending.length === 0 && !force) {
|
|
2080
|
+
console.log(`${c.green}✓ All content hashes already have embeddings.${c.reset}`);
|
|
2081
|
+
closeDb();
|
|
2082
|
+
return;
|
|
2083
|
+
}
|
|
2084
|
+
if (provider instanceof UnavailableOpenAIEmbeddingProvider) {
|
|
2085
|
+
throw new EmbeddingConfigError("OpenAI document embedding is authorized, but OPENAI_API_KEY is not configured.");
|
|
2086
|
+
}
|
|
2087
|
+
model = provider.model;
|
|
2088
|
+
}
|
|
2089
|
+
// Exclusive process lock — concurrent embeds race on vectors_vec (#825)
|
|
2090
|
+
const embedLock = tryAcquireEmbedLock(embedLockPathForDb(getDbPath()));
|
|
2091
|
+
if (!embedLock) {
|
|
2092
|
+
console.log(EMBED_LOCK_BUSY_MESSAGE);
|
|
2093
|
+
closeDb();
|
|
2094
|
+
return;
|
|
2095
|
+
}
|
|
2096
|
+
try {
|
|
2097
|
+
if (force) {
|
|
2098
|
+
console.log(`${c.yellow}Force re-indexing: clearing all vectors...${c.reset}`);
|
|
2099
|
+
}
|
|
2100
|
+
// Check if there's work to do before starting
|
|
2101
|
+
const hashesToEmbed = provider?.remote
|
|
2102
|
+
? getPendingEmbeddingDocsReadOnly(db, batchOptions?.collection, provider.model, remoteEmbeddingIdentity(provider, batchOptions?.chunkStrategy).fingerprint).length
|
|
2103
|
+
: getHashesNeedingEmbedding(db, batchOptions?.collection, model);
|
|
2104
|
+
if (hashesToEmbed === 0 && !force) {
|
|
2105
|
+
console.log(`${c.green}✓ All content hashes already have embeddings.${c.reset}`);
|
|
2106
|
+
closeDb();
|
|
2107
|
+
return;
|
|
2108
|
+
}
|
|
2109
|
+
console.log(`${c.dim}Model: ${shortModelName(model)}${c.reset}\n`);
|
|
2110
|
+
if (batchOptions?.maxDocsPerBatch !== undefined || batchOptions?.maxBatchBytes !== undefined) {
|
|
2111
|
+
const maxDocsPerBatch = batchOptions.maxDocsPerBatch ?? DEFAULT_EMBED_MAX_DOCS_PER_BATCH;
|
|
2112
|
+
const maxBatchBytes = batchOptions.maxBatchBytes ?? DEFAULT_EMBED_MAX_BATCH_BYTES;
|
|
2113
|
+
console.log(`${c.dim}Batch: ${maxDocsPerBatch} docs / ${formatBytes(maxBatchBytes)}${c.reset}\n`);
|
|
2114
|
+
}
|
|
2115
|
+
cursor.hide();
|
|
2116
|
+
progress.indeterminate();
|
|
2117
|
+
const startTime = Date.now();
|
|
2118
|
+
const result = await generateEmbeddings(storeInstance, {
|
|
2119
|
+
force,
|
|
2120
|
+
model,
|
|
2121
|
+
collection: batchOptions?.collection,
|
|
2122
|
+
maxDocsPerBatch: batchOptions?.maxDocsPerBatch,
|
|
2123
|
+
maxBatchBytes: batchOptions?.maxBatchBytes,
|
|
2124
|
+
chunkStrategy: batchOptions?.chunkStrategy,
|
|
2125
|
+
maxDurationMs: batchOptions?.maxDurationMs,
|
|
2126
|
+
onProgress: (info) => {
|
|
2127
|
+
if (info.totalBytes === 0)
|
|
2128
|
+
return;
|
|
2129
|
+
// Progress is measured by input bytes, not by chunks. The final chunk
|
|
2130
|
+
// count is discovered lazily batch-by-batch, so displaying
|
|
2131
|
+
// chunksEmbedded/totalChunks makes the percent look wrong when a few
|
|
2132
|
+
// large documents remain. Show chunks as a count and label the byte
|
|
2133
|
+
// percentage explicitly as input progress.
|
|
2134
|
+
const percent = Math.min(100, (info.bytesProcessed / info.totalBytes) * 100);
|
|
2135
|
+
progress.set(percent);
|
|
2136
|
+
const elapsed = (Date.now() - startTime) / 1000;
|
|
2137
|
+
const bytesPerSec = elapsed > 0 ? info.bytesProcessed / elapsed : 0;
|
|
2138
|
+
const remainingBytes = Math.max(0, info.totalBytes - info.bytesProcessed);
|
|
2139
|
+
const etaSec = bytesPerSec > 0 ? remainingBytes / bytesPerSec : Number.POSITIVE_INFINITY;
|
|
2140
|
+
const bar = renderProgressBar(percent);
|
|
2141
|
+
const percentStr = percent.toFixed(0).padStart(3);
|
|
2142
|
+
const throughput = bytesPerSec > 0 ? `${formatBytes(bytesPerSec)}/s` : ".../s";
|
|
2143
|
+
const eta = elapsed > 2 && Number.isFinite(etaSec) ? formatETA(etaSec) : "...";
|
|
2144
|
+
const inputStr = `${formatBytes(info.bytesProcessed)}/${formatBytes(info.totalBytes)} input`;
|
|
2145
|
+
const chunkStr = `${formatCount(info.chunksEmbedded)} chunks`;
|
|
2146
|
+
const errStr = info.errors > 0 ? ` ${c.yellow}${formatCount(info.errors)} err${c.reset}` : "";
|
|
2147
|
+
if (isTTY)
|
|
2148
|
+
process.stderr.write(`\r${c.cyan}${bar}${c.reset} ${c.bold}${percentStr}% input${c.reset} ${c.dim}${chunkStr}${errStr} · ${inputStr} · ${throughput} · ETA ${eta}${c.reset} `);
|
|
2149
|
+
},
|
|
2150
|
+
});
|
|
2151
|
+
progress.clear();
|
|
2152
|
+
cursor.show();
|
|
2153
|
+
const totalTimeSec = result.durationMs / 1000;
|
|
2154
|
+
if (result.chunksEmbedded === 0 && result.docsProcessed === 0) {
|
|
2155
|
+
console.log(`${c.green}✓ No non-empty documents to embed.${c.reset}`);
|
|
2156
|
+
}
|
|
2157
|
+
else {
|
|
2158
|
+
console.log(`\r${c.green}${renderProgressBar(100)}${c.reset} ${c.bold}100%${c.reset} `);
|
|
2159
|
+
console.log(`\n${c.green}✓ Done!${c.reset} Embedded ${c.bold}${result.chunksEmbedded}${c.reset} chunks from ${c.bold}${result.docsProcessed}${c.reset} documents in ${c.bold}${formatETA(totalTimeSec)}${c.reset}`);
|
|
2160
|
+
if (result.errors > 0) {
|
|
2161
|
+
console.log(`${c.yellow}⚠ ${formatCount(result.errors)} chunks still failed after retries${c.reset}`);
|
|
2162
|
+
for (const failure of (result.failures ?? []).slice(0, 8)) {
|
|
2163
|
+
console.log(` ${c.dim}${failure.path}#${failure.seq} (${failure.attempts} attempts): ${failure.reason}${c.reset}`);
|
|
2164
|
+
}
|
|
2165
|
+
if ((result.failures?.length ?? 0) > 8) {
|
|
2166
|
+
console.log(` ${c.dim}...and ${formatCount((result.failures?.length ?? 0) - 8)} more${c.reset}`);
|
|
2167
|
+
}
|
|
2168
|
+
}
|
|
2169
|
+
}
|
|
2170
|
+
closeDb();
|
|
2171
|
+
}
|
|
2172
|
+
finally {
|
|
2173
|
+
embedLock.release();
|
|
2174
|
+
}
|
|
2175
|
+
}
|
|
2176
|
+
// Sanitize a term for FTS5: remove punctuation except apostrophes
|
|
2177
|
+
function sanitizeFTS5Term(term) {
|
|
2178
|
+
// Remove all non-alphanumeric except apostrophes (for contractions like "don't")
|
|
2179
|
+
return term.replace(/[^\w']/g, '').trim();
|
|
2180
|
+
}
|
|
2181
|
+
// Build FTS5 query: phrase-aware with fallback to individual terms
|
|
2182
|
+
function buildFTS5Query(query) {
|
|
2183
|
+
// Sanitize the full query for phrase matching
|
|
2184
|
+
const sanitizedQuery = query.replace(/[^\w\s']/g, '').trim();
|
|
2185
|
+
const terms = query
|
|
2186
|
+
.split(/\s+/)
|
|
2187
|
+
.map(sanitizeFTS5Term)
|
|
2188
|
+
.filter(term => term.length >= 2); // Skip single chars and empty
|
|
2189
|
+
if (terms.length === 0)
|
|
2190
|
+
return "";
|
|
2191
|
+
if (terms.length === 1)
|
|
2192
|
+
return `"${terms[0].replace(/"/g, '""')}"`;
|
|
2193
|
+
// Strategy: exact phrase OR proximity match OR individual terms
|
|
2194
|
+
// Exact phrase matches rank highest, then close proximity, then any term
|
|
2195
|
+
const phrase = `"${sanitizedQuery.replace(/"/g, '""')}"`;
|
|
2196
|
+
const quotedTerms = terms.map(t => `"${t.replace(/"/g, '""')}"`);
|
|
2197
|
+
// FTS5 NEAR syntax: NEAR(term1 term2, distance)
|
|
2198
|
+
const nearPhrase = `NEAR(${quotedTerms.join(' ')}, 10)`;
|
|
2199
|
+
const orTerms = quotedTerms.join(' OR ');
|
|
2200
|
+
// Exact phrase > proximity > any term
|
|
2201
|
+
return `(${phrase}) OR (${nearPhrase}) OR (${orTerms})`;
|
|
2202
|
+
}
|
|
2203
|
+
// Normalize BM25 score to 0-1 range using sigmoid
|
|
2204
|
+
function normalizeBM25(score) {
|
|
2205
|
+
// BM25 scores are negative in SQLite (lower = better)
|
|
2206
|
+
// Typical range: -15 (excellent) to -2 (weak match)
|
|
2207
|
+
// Map to 0-1 where higher is better
|
|
2208
|
+
const absScore = Math.abs(score);
|
|
2209
|
+
// Sigmoid-ish normalization: maps ~2-15 range to ~0.1-0.95
|
|
2210
|
+
return 1 / (1 + Math.exp(-(absScore - 5) / 3));
|
|
2211
|
+
}
|
|
2212
|
+
// Highlight query terms in text (skip short words < 3 chars)
|
|
2213
|
+
function highlightTerms(text, query) {
|
|
2214
|
+
if (!useColor)
|
|
2215
|
+
return text;
|
|
2216
|
+
const terms = query.toLowerCase().split(/\s+/).filter(t => t.length >= 3);
|
|
2217
|
+
let result = text;
|
|
2218
|
+
for (const term of terms) {
|
|
2219
|
+
const regex = new RegExp(`(${term.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')})`, 'gi');
|
|
2220
|
+
result = result.replace(regex, `${c.yellow}${c.bold}$1${c.reset}`);
|
|
2221
|
+
}
|
|
2222
|
+
return result;
|
|
2223
|
+
}
|
|
2224
|
+
// Format score with color based on value
|
|
2225
|
+
function formatScore(score) {
|
|
2226
|
+
const pct = (score * 100).toFixed(0).padStart(3);
|
|
2227
|
+
if (!useColor)
|
|
2228
|
+
return `${pct}%`;
|
|
2229
|
+
if (score >= 0.7)
|
|
2230
|
+
return `${c.green}${pct}%${c.reset}`;
|
|
2231
|
+
if (score >= 0.4)
|
|
2232
|
+
return `${c.yellow}${pct}%${c.reset}`;
|
|
2233
|
+
return `${c.dim}${pct}%${c.reset}`;
|
|
2234
|
+
}
|
|
2235
|
+
function formatExplainNumber(value) {
|
|
2236
|
+
return value.toFixed(4);
|
|
2237
|
+
}
|
|
2238
|
+
// Shorten directory path for display - relative to $HOME (used for context paths, not documents)
|
|
2239
|
+
function shortPath(dirpath) {
|
|
2240
|
+
const home = homedir();
|
|
2241
|
+
if (dirpath.startsWith(home)) {
|
|
2242
|
+
return '~' + dirpath.slice(home.length);
|
|
2243
|
+
}
|
|
2244
|
+
return dirpath;
|
|
2245
|
+
}
|
|
2246
|
+
// Emit format-safe empty output for search commands.
|
|
2247
|
+
function printEmptySearchResults(format, reason = "no_results") {
|
|
2248
|
+
if (format === "json") {
|
|
2249
|
+
console.log("[]");
|
|
2250
|
+
return;
|
|
2251
|
+
}
|
|
2252
|
+
if (format === "csv") {
|
|
2253
|
+
console.log("docid,score,file,title,context,line,snippet");
|
|
2254
|
+
return;
|
|
2255
|
+
}
|
|
2256
|
+
if (format === "xml") {
|
|
2257
|
+
console.log("<results></results>");
|
|
2258
|
+
return;
|
|
2259
|
+
}
|
|
2260
|
+
if (format === "md" || format === "files") {
|
|
2261
|
+
return;
|
|
2262
|
+
}
|
|
2263
|
+
if (reason === "min_score") {
|
|
2264
|
+
console.log("No results found above minimum score threshold.");
|
|
2265
|
+
return;
|
|
2266
|
+
}
|
|
2267
|
+
console.log("No results found.");
|
|
2268
|
+
}
|
|
2269
|
+
const DEFAULT_EDITOR_URI_TEMPLATE = "vscode://file/{path}:{line}:{col}";
|
|
2270
|
+
function encodePathForEditorUri(absolutePath) {
|
|
2271
|
+
return encodeURI(absolutePath)
|
|
2272
|
+
.replace(/\?/g, "%3F")
|
|
2273
|
+
.replace(/#/g, "%23");
|
|
2274
|
+
}
|
|
2275
|
+
function getEditorUriTemplate() {
|
|
2276
|
+
const envTemplate = process.env.QMD_EDITOR_URI?.trim();
|
|
2277
|
+
if (envTemplate)
|
|
2278
|
+
return envTemplate;
|
|
2279
|
+
try {
|
|
2280
|
+
const config = loadConfig();
|
|
2281
|
+
const configTemplate = (config.editor_uri
|
|
2282
|
+
|| config.editor_uri_template
|
|
2283
|
+
|| config.editorUri
|
|
2284
|
+
|| config["editor-uri"])?.trim();
|
|
2285
|
+
if (configTemplate)
|
|
2286
|
+
return configTemplate;
|
|
2287
|
+
}
|
|
2288
|
+
catch {
|
|
2289
|
+
// Ignore config parsing issues and use default template.
|
|
2290
|
+
}
|
|
2291
|
+
return DEFAULT_EDITOR_URI_TEMPLATE;
|
|
2292
|
+
}
|
|
2293
|
+
export function buildEditorUri(template, absolutePath, line, col) {
|
|
2294
|
+
const safeLine = Number.isFinite(line) && line > 0 ? Math.floor(line) : 1;
|
|
2295
|
+
const safeCol = Number.isFinite(col) && col > 0 ? Math.floor(col) : 1;
|
|
2296
|
+
const encodedPath = encodePathForEditorUri(absolutePath);
|
|
2297
|
+
return template
|
|
2298
|
+
.replace(/\{path\}/g, encodedPath)
|
|
2299
|
+
.replace(/\{line\}/g, String(safeLine))
|
|
2300
|
+
.replace(/\{col\}/g, String(safeCol))
|
|
2301
|
+
.replace(/\{column\}/g, String(safeCol));
|
|
2302
|
+
}
|
|
2303
|
+
export function termLink(text, url, isTTY = !!process.stdout.isTTY) {
|
|
2304
|
+
if (!isTTY)
|
|
2305
|
+
return text;
|
|
2306
|
+
return `\x1b]8;;${url}\x07${text}\x1b]8;;\x07`;
|
|
2307
|
+
}
|
|
2308
|
+
function outputResults(results, query, opts) {
|
|
2309
|
+
const filtered = results.filter(r => r.score >= opts.minScore).slice(0, opts.limit);
|
|
2310
|
+
if (filtered.length === 0) {
|
|
2311
|
+
printEmptySearchResults(opts.format, "min_score");
|
|
2312
|
+
return;
|
|
2313
|
+
}
|
|
2314
|
+
// Helper to create qmd:// URI from displayPath
|
|
2315
|
+
const toQmdPath = (displayPath) => {
|
|
2316
|
+
const [collectionName, ...segments] = displayPath.split("/");
|
|
2317
|
+
if (!collectionName || segments.length === 0) {
|
|
2318
|
+
return `qmd://${displayPath}`;
|
|
2319
|
+
}
|
|
2320
|
+
const indexName = getActiveIndexName();
|
|
2321
|
+
return buildVirtualPath(collectionName, segments.join("/"), indexName === "index" ? undefined : indexName);
|
|
2322
|
+
};
|
|
2323
|
+
// Resolve every row's visible identifier up front. With --full-path we swap
|
|
2324
|
+
// the qmd:// URI for the file's on-disk path via renderFullPath() (./-
|
|
2325
|
+
// prefixed relative when under $PWD, absolute realpath otherwise). A row
|
|
2326
|
+
// whose file is gone from disk falls back to qmd:// and *keeps its docid*,
|
|
2327
|
+
// so it stays addressable — the same per-row rule multiGet() uses. Resolving
|
|
2328
|
+
// eagerly also means unresolved rows are counted before anything is printed.
|
|
2329
|
+
const linkDbForPaths = opts.fullPath ? getDb() : null;
|
|
2330
|
+
const resolutions = new Map();
|
|
2331
|
+
for (const row of filtered) {
|
|
2332
|
+
// Always rebuild from displayPath so the active index name is included
|
|
2333
|
+
// as ?index=… for non-default indexes. row.file may not carry it.
|
|
2334
|
+
const qmdUri = toQmdPath(row.displayPath);
|
|
2335
|
+
let resolution = { ident: qmdUri, resolved: false };
|
|
2336
|
+
if (opts.fullPath && linkDbForPaths) {
|
|
2337
|
+
const absolute = resolveVirtualPath(linkDbForPaths, qmdUri);
|
|
2338
|
+
if (absolute && existsSync(absolute)) {
|
|
2339
|
+
resolution = { ident: renderFullPath(absolute), resolved: true };
|
|
2340
|
+
}
|
|
2341
|
+
}
|
|
2342
|
+
resolutions.set(row, resolution);
|
|
2343
|
+
}
|
|
2344
|
+
const unresolvedCount = opts.fullPath
|
|
2345
|
+
? filtered.filter(row => !resolutions.get(row)?.resolved).length
|
|
2346
|
+
: 0;
|
|
2347
|
+
const displayPathFor = (row) => resolutions.get(row)?.ident ?? toQmdPath(row.displayPath);
|
|
2348
|
+
// Show the docid whenever it is still the row's identifier: always without
|
|
2349
|
+
// --full-path, and with it only for rows that have no on-disk path to show.
|
|
2350
|
+
const showDocid = (row) => !opts.fullPath || !resolutions.get(row)?.resolved;
|
|
2351
|
+
if (opts.format === "json") {
|
|
2352
|
+
// JSON output for LLM consumption
|
|
2353
|
+
const output = filtered.map(row => {
|
|
2354
|
+
const docid = row.docid || (row.hash ? row.hash.slice(0, 6) : undefined);
|
|
2355
|
+
const snippetInfo = extractSnippet(row.body, query, 300, row.chunkPos, row.chunkLen, opts.intent);
|
|
2356
|
+
let body = opts.full ? row.body : undefined;
|
|
2357
|
+
let snippet = !opts.full ? snippetInfo.snippet : undefined;
|
|
2358
|
+
if (opts.lineNumbers) {
|
|
2359
|
+
if (body)
|
|
2360
|
+
body = addLineNumbers(body);
|
|
2361
|
+
if (snippet)
|
|
2362
|
+
snippet = addLineNumbers(snippet);
|
|
2363
|
+
}
|
|
2364
|
+
// With --full-path, omit docid (the on-disk path is the identifier) —
|
|
2365
|
+
// unless the path could not be resolved, in which case it is all the
|
|
2366
|
+
// caller has.
|
|
2367
|
+
return {
|
|
2368
|
+
...(docid && showDocid(row) && { docid: `#${docid}` }),
|
|
2369
|
+
score: Math.round(row.score * 100) / 100,
|
|
2370
|
+
file: displayPathFor(row),
|
|
2371
|
+
line: snippetInfo.line,
|
|
2372
|
+
title: row.title,
|
|
2373
|
+
...(row.context && { context: row.context }),
|
|
2374
|
+
...(body && { body }),
|
|
2375
|
+
...(snippet && { snippet }),
|
|
2376
|
+
...(opts.explain && row.explain && { explain: row.explain }),
|
|
2377
|
+
};
|
|
2378
|
+
});
|
|
2379
|
+
console.log(JSON.stringify(output, null, 2));
|
|
2380
|
+
}
|
|
2381
|
+
else if (opts.format === "files") {
|
|
2382
|
+
// Simple docid,score,filepath,context output
|
|
2383
|
+
for (const row of filtered) {
|
|
2384
|
+
const docid = row.docid || (row.hash ? row.hash.slice(0, 6) : "");
|
|
2385
|
+
const ctx = row.context ? `,"${row.context.replace(/"/g, '""')}"` : "";
|
|
2386
|
+
if (opts.fullPath) {
|
|
2387
|
+
// --full-path: drop the docid, the on-disk path is the identifier.
|
|
2388
|
+
console.log(`${row.score.toFixed(2)},${displayPathFor(row)}${ctx}`);
|
|
2389
|
+
}
|
|
2390
|
+
else {
|
|
2391
|
+
console.log(`#${docid},${row.score.toFixed(2)},${displayPathFor(row)}${ctx}`);
|
|
2392
|
+
}
|
|
2393
|
+
}
|
|
2394
|
+
}
|
|
2395
|
+
else if (opts.format === "cli") {
|
|
2396
|
+
const editorUriTemplate = getEditorUriTemplate();
|
|
2397
|
+
const linkDb = getDb();
|
|
2398
|
+
for (let i = 0; i < filtered.length; i++) {
|
|
2399
|
+
const row = filtered[i];
|
|
2400
|
+
if (!row)
|
|
2401
|
+
continue;
|
|
2402
|
+
const { line, snippet } = extractSnippet(row.body, query, 500, row.chunkPos, row.chunkLen, opts.intent);
|
|
2403
|
+
const docid = row.docid || (row.hash ? row.hash.slice(0, 6) : undefined);
|
|
2404
|
+
// Line 1: filepath with docid
|
|
2405
|
+
// Default: show the full qmd:// URI so the user can see which collection
|
|
2406
|
+
// a hit lives in and can pipe the same string straight back into
|
|
2407
|
+
// `qmd get`. A bare collection-relative path like `sources/foo.md` is
|
|
2408
|
+
// ambiguous: it's not a real filesystem path, not a URI, and not a
|
|
2409
|
+
// shell-friendly identifier on its own.
|
|
2410
|
+
// With --full-path the visible label is the file's on-disk path
|
|
2411
|
+
// ($PWD-relative when in a subfolder; absolute realpath otherwise),
|
|
2412
|
+
// and the docid is omitted because the path is the identifier.
|
|
2413
|
+
const virtualPath = toQmdPath(row.displayPath);
|
|
2414
|
+
const parsed = parseVirtualPath(virtualPath);
|
|
2415
|
+
const absolutePath = resolveVirtualPath(linkDb, virtualPath);
|
|
2416
|
+
const visiblePath = displayPathFor(row);
|
|
2417
|
+
// Only show :line if we actually found a term match in the snippet body (exclude header line).
|
|
2418
|
+
const snippetBody = snippet.split("\n").slice(1).join("\n").toLowerCase();
|
|
2419
|
+
const hasMatch = query.toLowerCase().split(/\s+/).some(t => t.length > 0 && snippetBody.includes(t));
|
|
2420
|
+
const lineInfo = hasMatch ? `:${line}` : "";
|
|
2421
|
+
const docidStr = (docid && showDocid(row)) ? ` ${c.dim}#${docid}${c.reset}` : "";
|
|
2422
|
+
if (process.stdout.isTTY && absolutePath && parsed?.path) {
|
|
2423
|
+
const linkLine = hasMatch ? line : 1;
|
|
2424
|
+
const linkTarget = buildEditorUri(editorUriTemplate, absolutePath, linkLine, 1);
|
|
2425
|
+
const clickable = termLink(`${visiblePath}${lineInfo}`, linkTarget);
|
|
2426
|
+
console.log(`${c.cyan}${clickable}${c.reset}${docidStr}`);
|
|
2427
|
+
}
|
|
2428
|
+
else {
|
|
2429
|
+
console.log(`${c.cyan}${visiblePath}${c.dim}${lineInfo}${c.reset}${docidStr}`);
|
|
2430
|
+
}
|
|
2431
|
+
// Line 2: Title (if available)
|
|
2432
|
+
if (row.title) {
|
|
2433
|
+
console.log(`${c.bold}Title: ${row.title}${c.reset}`);
|
|
2434
|
+
}
|
|
2435
|
+
// Line 3: Context (if available)
|
|
2436
|
+
if (row.context) {
|
|
2437
|
+
console.log(`${c.dim}Context: ${row.context}${c.reset}`);
|
|
2438
|
+
}
|
|
2439
|
+
// Line 4: Score
|
|
2440
|
+
const score = formatScore(row.score);
|
|
2441
|
+
console.log(`Score: ${c.bold}${score}${c.reset}`);
|
|
2442
|
+
if (opts.explain && row.explain) {
|
|
2443
|
+
const explain = row.explain;
|
|
2444
|
+
const ftsScores = explain.ftsScores.length > 0
|
|
2445
|
+
? explain.ftsScores.map(formatExplainNumber).join(", ")
|
|
2446
|
+
: "none";
|
|
2447
|
+
const vecScores = explain.vectorScores.length > 0
|
|
2448
|
+
? explain.vectorScores.map(formatExplainNumber).join(", ")
|
|
2449
|
+
: "none";
|
|
2450
|
+
const contribSummary = explain.rrf.contributions
|
|
2451
|
+
.slice()
|
|
2452
|
+
.sort((a, b) => b.rrfContribution - a.rrfContribution)
|
|
2453
|
+
.slice(0, 3)
|
|
2454
|
+
.map(c => `${c.source}/${c.queryType}#${c.rank}:${formatExplainNumber(c.rrfContribution)}`)
|
|
2455
|
+
.join(" | ");
|
|
2456
|
+
console.log(`${c.dim}Explain: fts=[${ftsScores}] vec=[${vecScores}]${c.reset}`);
|
|
2457
|
+
console.log(`${c.dim} RRF: total=${formatExplainNumber(explain.rrf.totalScore)} base=${formatExplainNumber(explain.rrf.baseScore)} bonus=${formatExplainNumber(explain.rrf.topRankBonus)} rank=${explain.rrf.rank}${c.reset}`);
|
|
2458
|
+
console.log(`${c.dim} Blend: ${Math.round(explain.rrf.weight * 100)}%*${formatExplainNumber(explain.rrf.positionScore)} + ${Math.round((1 - explain.rrf.weight) * 100)}%*${formatExplainNumber(explain.rerankScore)} = ${formatExplainNumber(explain.blendedScore)}${c.reset}`);
|
|
2459
|
+
if (contribSummary.length > 0) {
|
|
2460
|
+
console.log(`${c.dim} Top RRF contributions: ${contribSummary}${c.reset}`);
|
|
2461
|
+
}
|
|
2462
|
+
}
|
|
2463
|
+
console.log();
|
|
2464
|
+
// Snippet with highlighting (diff-style header included)
|
|
2465
|
+
const content = opts.full ? row.body : snippet;
|
|
2466
|
+
const displayContent = opts.lineNumbers ? addLineNumbers(content, opts.full ? 1 : line) : content;
|
|
2467
|
+
const highlighted = highlightTerms(displayContent, query);
|
|
2468
|
+
console.log(highlighted);
|
|
2469
|
+
// Double empty line between results
|
|
2470
|
+
if (i < filtered.length - 1)
|
|
2471
|
+
console.log('\n');
|
|
2472
|
+
}
|
|
2473
|
+
}
|
|
2474
|
+
else if (opts.format === "md") {
|
|
2475
|
+
for (let i = 0; i < filtered.length; i++) {
|
|
2476
|
+
const row = filtered[i];
|
|
2477
|
+
if (!row)
|
|
2478
|
+
continue;
|
|
2479
|
+
const visiblePath = displayPathFor(row);
|
|
2480
|
+
const heading = row.title || visiblePath;
|
|
2481
|
+
const docid = row.docid || (row.hash ? row.hash.slice(0, 6) : undefined);
|
|
2482
|
+
let content = opts.full ? row.body : extractSnippet(row.body, query, 500, row.chunkPos, row.chunkLen, opts.intent).snippet;
|
|
2483
|
+
if (opts.lineNumbers) {
|
|
2484
|
+
content = addLineNumbers(content);
|
|
2485
|
+
}
|
|
2486
|
+
const fileLine = `**file:** \`${visiblePath}\`\n`;
|
|
2487
|
+
// With --full-path the on-disk path is the identifier; drop the docid
|
|
2488
|
+
// line unless this row had no path to show.
|
|
2489
|
+
const docidLine = (docid && showDocid(row)) ? `**docid:** \`#${docid}\`\n` : "";
|
|
2490
|
+
const contextLine = row.context ? `**context:** ${row.context}\n` : "";
|
|
2491
|
+
console.log(`---\n# ${heading}\n${fileLine}${docidLine}${contextLine}\n${content}\n`);
|
|
2492
|
+
}
|
|
2493
|
+
}
|
|
2494
|
+
else if (opts.format === "xml") {
|
|
2495
|
+
for (const row of filtered) {
|
|
2496
|
+
const titleAttr = row.title ? ` title="${row.title.replace(/"/g, '"')}"` : "";
|
|
2497
|
+
const contextAttr = row.context ? ` context="${row.context.replace(/"/g, '"')}"` : "";
|
|
2498
|
+
const docid = row.docid || (row.hash ? row.hash.slice(0, 6) : "");
|
|
2499
|
+
let content = opts.full ? row.body : extractSnippet(row.body, query, 500, row.chunkPos, row.chunkLen, opts.intent).snippet;
|
|
2500
|
+
if (opts.lineNumbers) {
|
|
2501
|
+
content = addLineNumbers(content);
|
|
2502
|
+
}
|
|
2503
|
+
const docidAttr = showDocid(row) ? ` docid="#${docid}"` : "";
|
|
2504
|
+
console.log(`<file${docidAttr} name="${displayPathFor(row)}"${titleAttr}${contextAttr}>\n${content}\n</file>\n`);
|
|
2505
|
+
}
|
|
2506
|
+
}
|
|
2507
|
+
else {
|
|
2508
|
+
// CSV format. The docid column is always present — under --full-path it is
|
|
2509
|
+
// empty for rows whose on-disk path was found and carries the docid for
|
|
2510
|
+
// rows that fell back to a qmd:// URI, so the columns stay positional.
|
|
2511
|
+
// (multi-get's CSV already emits the docid column unconditionally.)
|
|
2512
|
+
console.log("docid,score,file,title,context,line,snippet");
|
|
2513
|
+
for (const row of filtered) {
|
|
2514
|
+
const { line, snippet } = extractSnippet(row.body, query, 500, row.chunkPos, row.chunkLen, opts.intent);
|
|
2515
|
+
let content = opts.full ? row.body : snippet;
|
|
2516
|
+
if (opts.lineNumbers) {
|
|
2517
|
+
content = addLineNumbers(content, opts.full ? 1 : line);
|
|
2518
|
+
}
|
|
2519
|
+
const docid = row.docid || (row.hash ? row.hash.slice(0, 6) : "");
|
|
2520
|
+
const snippetText = content || "";
|
|
2521
|
+
const path = escapeCSV(displayPathFor(row));
|
|
2522
|
+
const tail = `${path},${escapeCSV(row.title || "")},${escapeCSV(row.context || "")},${line},${escapeCSV(snippetText)}`;
|
|
2523
|
+
const docidField = (docid && showDocid(row)) ? `#${docid}` : "";
|
|
2524
|
+
console.log(`${docidField},${row.score.toFixed(4)},${tail}`);
|
|
2525
|
+
}
|
|
2526
|
+
}
|
|
2527
|
+
warnUnresolvedFullPaths(unresolvedCount, filtered.length);
|
|
2528
|
+
}
|
|
2529
|
+
// Resolve -c collection filter: supports single string, array, or undefined.
|
|
2530
|
+
// Returns validated collection names (exits on unknown collection).
|
|
2531
|
+
function resolveCollectionFilter(raw, useDefaults = false) {
|
|
2532
|
+
// If no filter specified and useDefaults is true, use default collections
|
|
2533
|
+
if (!raw && useDefaults) {
|
|
2534
|
+
return getDefaultCollectionNames();
|
|
2535
|
+
}
|
|
2536
|
+
if (!raw)
|
|
2537
|
+
return [];
|
|
2538
|
+
const names = Array.isArray(raw) ? raw : [raw];
|
|
2539
|
+
const validated = [];
|
|
2540
|
+
for (const name of names) {
|
|
2541
|
+
const coll = getCollectionFromYaml(name);
|
|
2542
|
+
if (!coll) {
|
|
2543
|
+
console.error(`Collection not found: ${name}`);
|
|
2544
|
+
closeDb();
|
|
2545
|
+
process.exit(1);
|
|
2546
|
+
}
|
|
2547
|
+
validated.push(name);
|
|
2548
|
+
}
|
|
2549
|
+
return validated;
|
|
2550
|
+
}
|
|
2551
|
+
// Pass 0/1/N collection names through to store search (search each, then merge).
|
|
2552
|
+
function collectionSearchFilter(names) {
|
|
2553
|
+
if (names.length === 0)
|
|
2554
|
+
return undefined;
|
|
2555
|
+
if (names.length === 1)
|
|
2556
|
+
return names[0];
|
|
2557
|
+
return names;
|
|
2558
|
+
}
|
|
2559
|
+
export function parseStructuredQuery(query) {
|
|
2560
|
+
const rawLines = query.split('\n').map((line, idx) => ({
|
|
2561
|
+
raw: line,
|
|
2562
|
+
trimmed: line.trim(),
|
|
2563
|
+
number: idx + 1,
|
|
2564
|
+
})).filter(line => line.trimmed.length > 0);
|
|
2565
|
+
if (rawLines.length === 0)
|
|
2566
|
+
return null;
|
|
2567
|
+
const prefixRe = /^(lex|vec|hyde):\s*/i;
|
|
2568
|
+
const expandRe = /^expand:\s*/i;
|
|
2569
|
+
const intentRe = /^intent:\s*/i;
|
|
2570
|
+
const typed = [];
|
|
2571
|
+
let intent;
|
|
2572
|
+
for (const line of rawLines) {
|
|
2573
|
+
if (expandRe.test(line.trimmed)) {
|
|
2574
|
+
if (rawLines.length > 1) {
|
|
2575
|
+
throw new Error(`Line ${line.number} starts with expand:, but query documents cannot mix expand with typed lines. Submit a single expand query instead.`);
|
|
2576
|
+
}
|
|
2577
|
+
const text = line.trimmed.replace(expandRe, '').trim();
|
|
2578
|
+
if (!text) {
|
|
2579
|
+
throw new Error('expand: query must include text.');
|
|
2580
|
+
}
|
|
2581
|
+
return null; // treat as standalone expand query
|
|
2582
|
+
}
|
|
2583
|
+
// Parse intent: lines
|
|
2584
|
+
if (intentRe.test(line.trimmed)) {
|
|
2585
|
+
if (intent !== undefined) {
|
|
2586
|
+
throw new Error(`Line ${line.number}: only one intent: line is allowed per query document.`);
|
|
2587
|
+
}
|
|
2588
|
+
const text = line.trimmed.replace(intentRe, '').trim();
|
|
2589
|
+
if (!text) {
|
|
2590
|
+
throw new Error(`Line ${line.number}: intent: must include text.`);
|
|
2591
|
+
}
|
|
2592
|
+
intent = text;
|
|
2593
|
+
continue;
|
|
2594
|
+
}
|
|
2595
|
+
const match = line.trimmed.match(prefixRe);
|
|
2596
|
+
if (match) {
|
|
2597
|
+
const type = match[1].toLowerCase();
|
|
2598
|
+
const text = line.trimmed.slice(match[0].length).trim();
|
|
2599
|
+
if (!text) {
|
|
2600
|
+
throw new Error(`Line ${line.number} (${type}:) must include text.`);
|
|
2601
|
+
}
|
|
2602
|
+
if (/\r|\n/.test(text)) {
|
|
2603
|
+
throw new Error(`Line ${line.number} (${type}:) contains a newline. Keep each query on a single line.`);
|
|
2604
|
+
}
|
|
2605
|
+
if (type === 'lex' && rawLines.length === 1) {
|
|
2606
|
+
return null;
|
|
2607
|
+
}
|
|
2608
|
+
typed.push({ type, query: text, line: line.number });
|
|
2609
|
+
continue;
|
|
2610
|
+
}
|
|
2611
|
+
if (rawLines.length === 1) {
|
|
2612
|
+
// Single plain line -> implicit expand
|
|
2613
|
+
return null;
|
|
2614
|
+
}
|
|
2615
|
+
throw new Error(`Line ${line.number} is missing a lex:/vec:/hyde:/intent: prefix. Each line in a query document must start with one.`);
|
|
2616
|
+
}
|
|
2617
|
+
// intent: alone is not a valid query — must have at least one search
|
|
2618
|
+
if (intent && typed.length === 0) {
|
|
2619
|
+
throw new Error('intent: cannot appear alone. Add at least one lex:, vec:, or hyde: line.');
|
|
2620
|
+
}
|
|
2621
|
+
return typed.length > 0 ? { searches: typed, intent } : null;
|
|
2622
|
+
}
|
|
2623
|
+
function search(query, opts) {
|
|
2624
|
+
const db = getDb();
|
|
2625
|
+
// Validate collection filter (supports multiple -c flags)
|
|
2626
|
+
// Use default collections if none specified
|
|
2627
|
+
const collectionNames = resolveCollectionFilter(opts.collection, true);
|
|
2628
|
+
// Use large limit for --all, otherwise fetch more than needed and let outputResults filter
|
|
2629
|
+
const fetchLimit = opts.all ? 100000 : Math.max(50, opts.limit * 2);
|
|
2630
|
+
const results = searchFTS(db, query, fetchLimit, collectionNames);
|
|
2631
|
+
// Add context to results
|
|
2632
|
+
const resultsWithContext = results.map(r => ({
|
|
2633
|
+
file: r.filepath,
|
|
2634
|
+
displayPath: r.displayPath,
|
|
2635
|
+
title: r.title,
|
|
2636
|
+
body: r.body || "",
|
|
2637
|
+
score: r.score,
|
|
2638
|
+
context: getContextForFile(db, r.filepath),
|
|
2639
|
+
hash: r.hash,
|
|
2640
|
+
docid: r.docid,
|
|
2641
|
+
}));
|
|
2642
|
+
closeDb();
|
|
2643
|
+
if (resultsWithContext.length === 0) {
|
|
2644
|
+
printEmptySearchResults(opts.format);
|
|
2645
|
+
return;
|
|
2646
|
+
}
|
|
2647
|
+
outputResults(resultsWithContext, query, opts);
|
|
2648
|
+
}
|
|
2649
|
+
// Log query expansion as a tree to stderr (CLI progress feedback)
|
|
2650
|
+
function logExpansionTree(originalQuery, expanded) {
|
|
2651
|
+
const lines = [];
|
|
2652
|
+
lines.push(`${c.dim}├─ ${originalQuery}${c.reset}`);
|
|
2653
|
+
for (const q of expanded) {
|
|
2654
|
+
let preview = q.query.replace(/\n/g, ' ');
|
|
2655
|
+
if (preview.length > 72)
|
|
2656
|
+
preview = preview.substring(0, 69) + '...';
|
|
2657
|
+
lines.push(`${c.dim}├─ ${q.type}: ${preview}${c.reset}`);
|
|
2658
|
+
}
|
|
2659
|
+
if (lines.length > 0) {
|
|
2660
|
+
lines[lines.length - 1] = lines[lines.length - 1].replace('├─', '└─');
|
|
2661
|
+
}
|
|
2662
|
+
for (const line of lines)
|
|
2663
|
+
process.stderr.write(line + '\n');
|
|
2664
|
+
}
|
|
2665
|
+
async function vectorSearch(query, opts, _model = DEFAULT_EMBED_MODEL) {
|
|
2666
|
+
const store = getStore();
|
|
2667
|
+
// Validate collection filter (supports multiple -c flags)
|
|
2668
|
+
// Use default collections if none specified
|
|
2669
|
+
const collectionNames = resolveCollectionFilter(opts.collection, true);
|
|
2670
|
+
checkIndexHealth(store.db);
|
|
2671
|
+
await withLLMSession(async () => {
|
|
2672
|
+
const results = await vectorSearchQuery(store, query, {
|
|
2673
|
+
collection: collectionNames,
|
|
2674
|
+
limit: opts.all ? 500 : (opts.limit || 10),
|
|
2675
|
+
minScore: opts.minScore || 0.3,
|
|
2676
|
+
expansionContext: opts.intent,
|
|
2677
|
+
hooks: {
|
|
2678
|
+
onExpand: (original, expanded) => {
|
|
2679
|
+
logExpansionTree(original, expanded);
|
|
2680
|
+
process.stderr.write(`${c.dim}Searching ${expanded.length + 1} vector queries...${c.reset}\n`);
|
|
2681
|
+
},
|
|
2682
|
+
},
|
|
2683
|
+
});
|
|
2684
|
+
closeDb();
|
|
2685
|
+
if (results.length === 0) {
|
|
2686
|
+
printEmptySearchResults(opts.format);
|
|
2687
|
+
return;
|
|
2688
|
+
}
|
|
2689
|
+
outputResults(results.map(r => ({
|
|
2690
|
+
file: r.file,
|
|
2691
|
+
displayPath: r.displayPath,
|
|
2692
|
+
title: r.title,
|
|
2693
|
+
body: r.body,
|
|
2694
|
+
score: r.score,
|
|
2695
|
+
context: r.context,
|
|
2696
|
+
docid: r.docid,
|
|
2697
|
+
})), query, { ...opts, limit: results.length });
|
|
2698
|
+
}, { maxDuration: 10 * 60 * 1000, name: 'vectorSearch' });
|
|
2699
|
+
}
|
|
2700
|
+
async function querySearch(query, opts, _embedModel = DEFAULT_EMBED_MODEL, _rerankModel = DEFAULT_RERANK_MODEL) {
|
|
2701
|
+
const store = getStore();
|
|
2702
|
+
// Validate collection filter (supports multiple -c flags)
|
|
2703
|
+
// Use default collections if none specified
|
|
2704
|
+
const collectionNames = resolveCollectionFilter(opts.collection, true);
|
|
2705
|
+
checkIndexHealth(store.db);
|
|
2706
|
+
// Check for structured query syntax (lex:/vec:/hyde:/intent: prefixes)
|
|
2707
|
+
const parsed = parseStructuredQuery(query);
|
|
2708
|
+
// Intent can come from --intent flag or from intent: line in query document
|
|
2709
|
+
const intent = opts.intent || parsed?.intent;
|
|
2710
|
+
if (opts.expansion === "force" && parsed?.searches.some(search => search.type === "lex")) {
|
|
2711
|
+
throw new Error("conflicting expansion directives: --expand cannot be combined with lex:");
|
|
2712
|
+
}
|
|
2713
|
+
await withLLMSession(async () => {
|
|
2714
|
+
let results;
|
|
2715
|
+
if (parsed) {
|
|
2716
|
+
const structuredQueries = parsed.searches;
|
|
2717
|
+
// Structured search — user provided their own query expansions
|
|
2718
|
+
const typeLabels = structuredQueries.map(s => s.type).join('+');
|
|
2719
|
+
process.stderr.write(`${c.dim}Structured search: ${structuredQueries.length} queries (${typeLabels})${c.reset}\n`);
|
|
2720
|
+
if (intent) {
|
|
2721
|
+
process.stderr.write(`${c.dim}├─ intent: ${intent}${c.reset}\n`);
|
|
2722
|
+
}
|
|
2723
|
+
// Log each sub-query
|
|
2724
|
+
for (const s of structuredQueries) {
|
|
2725
|
+
let preview = s.query.replace(/\n/g, ' ');
|
|
2726
|
+
if (preview.length > 72)
|
|
2727
|
+
preview = preview.substring(0, 69) + '...';
|
|
2728
|
+
process.stderr.write(`${c.dim}├─ ${s.type}: ${preview}${c.reset}\n`);
|
|
2729
|
+
}
|
|
2730
|
+
process.stderr.write(`${c.dim}└─ Searching...${c.reset}\n`);
|
|
2731
|
+
results = await structuredSearch(store, structuredQueries, {
|
|
2732
|
+
collections: collectionNames.length > 0 ? collectionNames : undefined,
|
|
2733
|
+
limit: opts.all ? 500 : (opts.limit || 10),
|
|
2734
|
+
minScore: opts.minScore || 0,
|
|
2735
|
+
candidateLimit: opts.candidateLimit,
|
|
2736
|
+
skipRerank: opts.skipRerank,
|
|
2737
|
+
explain: !!opts.explain,
|
|
2738
|
+
rerankContext: intent,
|
|
2739
|
+
chunkStrategy: opts.chunkStrategy,
|
|
2740
|
+
hooks: {
|
|
2741
|
+
onEmbedStart: (count) => {
|
|
2742
|
+
process.stderr.write(`${c.dim}Embedding ${count} ${count === 1 ? 'query' : 'queries'}...${c.reset}`);
|
|
2743
|
+
},
|
|
2744
|
+
onEmbedDone: (ms) => {
|
|
2745
|
+
process.stderr.write(`${c.dim} (${formatMs(ms)})${c.reset}\n`);
|
|
2746
|
+
},
|
|
2747
|
+
onRerankStart: (chunkCount) => {
|
|
2748
|
+
process.stderr.write(`${c.dim}Reranking ${chunkCount} chunks...${c.reset}`);
|
|
2749
|
+
progress.indeterminate();
|
|
2750
|
+
},
|
|
2751
|
+
onRerankDone: (ms) => {
|
|
2752
|
+
progress.clear();
|
|
2753
|
+
process.stderr.write(`${c.dim} (${formatMs(ms)})${c.reset}\n`);
|
|
2754
|
+
},
|
|
2755
|
+
},
|
|
2756
|
+
});
|
|
2757
|
+
}
|
|
2758
|
+
else {
|
|
2759
|
+
// Standard hybrid query with automatic expansion
|
|
2760
|
+
results = await hybridQuery(store, query, {
|
|
2761
|
+
collections: collectionNames.length > 0 ? collectionNames : undefined,
|
|
2762
|
+
limit: opts.all ? 500 : (opts.limit || 10),
|
|
2763
|
+
minScore: opts.minScore || 0,
|
|
2764
|
+
candidateLimit: opts.candidateLimit,
|
|
2765
|
+
skipRerank: opts.skipRerank,
|
|
2766
|
+
explain: !!opts.explain,
|
|
2767
|
+
rerankContext: intent,
|
|
2768
|
+
expansion: opts.expansion,
|
|
2769
|
+
chunkStrategy: opts.chunkStrategy,
|
|
2770
|
+
hooks: {
|
|
2771
|
+
onExpansionDecision: (decision) => {
|
|
2772
|
+
if (decision.reason !== "auto-expand" && decision.reason !== "strong-signal") {
|
|
2773
|
+
process.stderr.write(`${c.dim}Expansion policy: ${decision.reason}${c.reset}\n`);
|
|
2774
|
+
}
|
|
2775
|
+
},
|
|
2776
|
+
onExpansionError: (event) => {
|
|
2777
|
+
process.stderr.write(`${c.dim}Expansion error: ${event.reason}${c.reset}\n`);
|
|
2778
|
+
},
|
|
2779
|
+
onStrongSignal: (score) => {
|
|
2780
|
+
process.stderr.write(`${c.dim}Strong BM25 signal (${score.toFixed(2)}) — skipping expansion${c.reset}\n`);
|
|
2781
|
+
},
|
|
2782
|
+
onExpandStart: () => {
|
|
2783
|
+
process.stderr.write(`${c.dim}Expanding query...${c.reset}`);
|
|
2784
|
+
},
|
|
2785
|
+
onExpand: (original, expanded, ms) => {
|
|
2786
|
+
process.stderr.write(`${c.dim} (${formatMs(ms)})${c.reset}\n`);
|
|
2787
|
+
logExpansionTree(original, expanded);
|
|
2788
|
+
process.stderr.write(`${c.dim}Searching ${expanded.length + 1} queries...${c.reset}\n`);
|
|
2789
|
+
},
|
|
2790
|
+
onEmbedStart: (count) => {
|
|
2791
|
+
process.stderr.write(`${c.dim}Embedding ${count} ${count === 1 ? 'query' : 'queries'}...${c.reset}`);
|
|
2792
|
+
},
|
|
2793
|
+
onEmbedDone: (ms) => {
|
|
2794
|
+
process.stderr.write(`${c.dim} (${formatMs(ms)})${c.reset}\n`);
|
|
2795
|
+
},
|
|
2796
|
+
onRerankStart: (chunkCount) => {
|
|
2797
|
+
process.stderr.write(`${c.dim}Reranking ${chunkCount} chunks...${c.reset}`);
|
|
2798
|
+
progress.indeterminate();
|
|
2799
|
+
},
|
|
2800
|
+
onRerankDone: (ms) => {
|
|
2801
|
+
progress.clear();
|
|
2802
|
+
process.stderr.write(`${c.dim} (${formatMs(ms)})${c.reset}\n`);
|
|
2803
|
+
},
|
|
2804
|
+
},
|
|
2805
|
+
});
|
|
2806
|
+
}
|
|
2807
|
+
closeDb();
|
|
2808
|
+
if (results.length === 0) {
|
|
2809
|
+
printEmptySearchResults(opts.format);
|
|
2810
|
+
return;
|
|
2811
|
+
}
|
|
2812
|
+
// Use first lex/vec query for output context, or original query
|
|
2813
|
+
const structuredQueries = parsed?.searches;
|
|
2814
|
+
const displayQuery = structuredQueries
|
|
2815
|
+
? (structuredQueries.find(s => s.type === 'lex')?.query || structuredQueries.find(s => s.type === 'vec')?.query || query)
|
|
2816
|
+
: query;
|
|
2817
|
+
outputResults(results.map(r => ({
|
|
2818
|
+
file: r.file,
|
|
2819
|
+
displayPath: r.displayPath,
|
|
2820
|
+
title: r.title,
|
|
2821
|
+
body: r.body,
|
|
2822
|
+
chunkPos: r.bestChunkPos,
|
|
2823
|
+
chunkLen: r.bestChunk.length,
|
|
2824
|
+
score: r.score,
|
|
2825
|
+
context: r.context,
|
|
2826
|
+
docid: r.docid,
|
|
2827
|
+
explain: r.explain,
|
|
2828
|
+
})), displayQuery, { ...opts, limit: results.length });
|
|
2829
|
+
}, { maxDuration: 10 * 60 * 1000, name: 'querySearch' });
|
|
2830
|
+
}
|
|
2831
|
+
// Parse CLI arguments using util.parseArgs
|
|
2832
|
+
function parseCLI() {
|
|
2833
|
+
const { values, positionals } = parseArgs({
|
|
2834
|
+
args: process.argv.slice(2), // Skip node and script path
|
|
2835
|
+
options: {
|
|
2836
|
+
// Global options
|
|
2837
|
+
index: {
|
|
2838
|
+
type: "string",
|
|
2839
|
+
},
|
|
2840
|
+
context: {
|
|
2841
|
+
type: "string",
|
|
2842
|
+
},
|
|
2843
|
+
help: { type: "boolean", short: "h" },
|
|
2844
|
+
version: { type: "boolean", short: "v" },
|
|
2845
|
+
skill: { type: "boolean" },
|
|
2846
|
+
global: { type: "boolean" },
|
|
2847
|
+
yes: { type: "boolean" },
|
|
2848
|
+
// Search options
|
|
2849
|
+
n: { type: "string" },
|
|
2850
|
+
"min-score": { type: "string" },
|
|
2851
|
+
all: { type: "boolean" },
|
|
2852
|
+
full: { type: "boolean" },
|
|
2853
|
+
format: { type: "string" }, // preferred: --format cli|json|csv|md|xml|files
|
|
2854
|
+
// Legacy boolean format aliases. Kept working for back-compat but
|
|
2855
|
+
// omitted from the documented help; prefer `--format <kind>`.
|
|
2856
|
+
csv: { type: "boolean" },
|
|
2857
|
+
md: { type: "boolean" },
|
|
2858
|
+
xml: { type: "boolean" },
|
|
2859
|
+
files: { type: "boolean" },
|
|
2860
|
+
json: { type: "boolean" },
|
|
2861
|
+
explain: { type: "boolean" },
|
|
2862
|
+
collection: { type: "string", short: "c", multiple: true }, // Filter by collection(s)
|
|
2863
|
+
// Collection options
|
|
2864
|
+
name: { type: "string" }, // collection name
|
|
2865
|
+
mask: { type: "string" }, // glob pattern
|
|
2866
|
+
glob: { type: "string" }, // alias for --mask (OpenClaw / #536)
|
|
2867
|
+
// Embed options
|
|
2868
|
+
force: { type: "boolean", short: "f" },
|
|
2869
|
+
"max-docs-per-batch": { type: "string" },
|
|
2870
|
+
"max-batch-mb": { type: "string" },
|
|
2871
|
+
timeout: { type: "string" }, // embed session cap in minutes (0 = no limit; default 30)
|
|
2872
|
+
// Update options
|
|
2873
|
+
pull: { type: "boolean" }, // git pull before update
|
|
2874
|
+
refresh: { type: "boolean" },
|
|
2875
|
+
progress: { type: "boolean" }, // qmd pull: show node-llama-cpp download progress bar
|
|
2876
|
+
"dry-run": { type: "boolean" }, // cleanup: report what would be removed
|
|
2877
|
+
// Get options
|
|
2878
|
+
l: { type: "string" }, // max lines
|
|
2879
|
+
from: { type: "string" }, // start line
|
|
2880
|
+
"max-bytes": { type: "string" }, // max bytes for multi-get
|
|
2881
|
+
"line-numbers": { type: "boolean" }, // add line numbers to output (search; default on for get/multi-get)
|
|
2882
|
+
"no-line-numbers": { type: "boolean" }, // disable line numbers for get/multi-get
|
|
2883
|
+
"full-path": { type: "boolean" }, // show on-disk paths instead of qmd:// (get/multi-get/search/query)
|
|
2884
|
+
// Query options
|
|
2885
|
+
"candidate-limit": { type: "string", short: "C" },
|
|
2886
|
+
"no-rerank": { type: "boolean", default: false },
|
|
2887
|
+
expand: { type: "boolean", default: false },
|
|
2888
|
+
"no-gpu": { type: "boolean", default: false },
|
|
2889
|
+
intent: { type: "string" },
|
|
2890
|
+
// Chunking options
|
|
2891
|
+
"chunk-strategy": { type: "string" }, // "regex" (default) or "auto" (AST for code files)
|
|
2892
|
+
// MCP HTTP transport options
|
|
2893
|
+
http: { type: "boolean" },
|
|
2894
|
+
daemon: { type: "boolean" },
|
|
2895
|
+
port: { type: "string" },
|
|
2896
|
+
host: { type: "string" },
|
|
2897
|
+
},
|
|
2898
|
+
allowPositionals: true,
|
|
2899
|
+
strict: false, // Allow unknown options to pass through
|
|
2900
|
+
});
|
|
2901
|
+
if (values["no-gpu"]) {
|
|
2902
|
+
process.env.QMD_FORCE_CPU = "1";
|
|
2903
|
+
}
|
|
2904
|
+
// Select index name (default: "index"). If no explicit --index is supplied,
|
|
2905
|
+
// a project-local .qmd/index.yaml overrides the global config/cache paths.
|
|
2906
|
+
const indexName = values.index;
|
|
2907
|
+
if (indexName) {
|
|
2908
|
+
setIndexName(indexName);
|
|
2909
|
+
setConfigIndexName(indexName);
|
|
2910
|
+
setConfigSource();
|
|
2911
|
+
}
|
|
2912
|
+
else {
|
|
2913
|
+
const localConfigPath = findLocalConfigPath();
|
|
2914
|
+
if (localConfigPath) {
|
|
2915
|
+
setConfigSource({ configPath: localConfigPath });
|
|
2916
|
+
storeDbPathOverride = getLocalDbPath(localConfigPath);
|
|
2917
|
+
closeDb();
|
|
2918
|
+
}
|
|
2919
|
+
else {
|
|
2920
|
+
setConfigSource();
|
|
2921
|
+
}
|
|
2922
|
+
}
|
|
2923
|
+
// Determine output format. Prefer --format <kind>; fall back to the
|
|
2924
|
+
// legacy boolean aliases (--csv/--md/--xml/--files/--json) which remain
|
|
2925
|
+
// wired up for back-compat but are no longer documented.
|
|
2926
|
+
let format = "cli";
|
|
2927
|
+
const rawFormat = typeof values.format === "string" ? values.format.toLowerCase().trim() : "";
|
|
2928
|
+
const VALID_FORMATS = ["cli", "json", "csv", "md", "xml", "files"];
|
|
2929
|
+
if (rawFormat) {
|
|
2930
|
+
if (VALID_FORMATS.includes(rawFormat)) {
|
|
2931
|
+
format = rawFormat;
|
|
2932
|
+
}
|
|
2933
|
+
else {
|
|
2934
|
+
console.error(`Unknown --format value: ${values.format}`);
|
|
2935
|
+
console.error(`Valid: ${VALID_FORMATS.join(", ")}`);
|
|
2936
|
+
process.exit(1);
|
|
2937
|
+
}
|
|
2938
|
+
}
|
|
2939
|
+
else if (values.csv)
|
|
2940
|
+
format = "csv";
|
|
2941
|
+
else if (values.md)
|
|
2942
|
+
format = "md";
|
|
2943
|
+
else if (values.xml)
|
|
2944
|
+
format = "xml";
|
|
2945
|
+
else if (values.files)
|
|
2946
|
+
format = "files";
|
|
2947
|
+
else if (values.json)
|
|
2948
|
+
format = "json";
|
|
2949
|
+
// Default limit: 20 for --files/--json, 5 otherwise
|
|
2950
|
+
// --all means return all results (use very large limit)
|
|
2951
|
+
const defaultLimit = (format === "files" || format === "json") ? 20 : 5;
|
|
2952
|
+
const isAll = !!values.all;
|
|
2953
|
+
const opts = {
|
|
2954
|
+
format,
|
|
2955
|
+
full: !!values.full,
|
|
2956
|
+
limit: isAll ? 100000 : (values.n ? parseInt(String(values.n), 10) || defaultLimit : defaultLimit),
|
|
2957
|
+
minScore: values["min-score"] ? parseFloat(String(values["min-score"])) || 0 : 0,
|
|
2958
|
+
all: isAll,
|
|
2959
|
+
collection: values.collection,
|
|
2960
|
+
lineNumbers: !!values["line-numbers"],
|
|
2961
|
+
candidateLimit: values["candidate-limit"] ? parseInt(String(values["candidate-limit"]), 10) : undefined,
|
|
2962
|
+
skipRerank: !!values["no-rerank"],
|
|
2963
|
+
explain: !!values.explain,
|
|
2964
|
+
intent: values.intent,
|
|
2965
|
+
expansion: values.expand ? "force" : "auto",
|
|
2966
|
+
chunkStrategy: parseChunkStrategy(values["chunk-strategy"]),
|
|
2967
|
+
fullPath: !!values["full-path"],
|
|
2968
|
+
};
|
|
2969
|
+
return {
|
|
2970
|
+
command: positionals[0] || "",
|
|
2971
|
+
args: positionals.slice(1),
|
|
2972
|
+
query: positionals.slice(1).join(" "),
|
|
2973
|
+
opts,
|
|
2974
|
+
values,
|
|
2975
|
+
};
|
|
2976
|
+
}
|
|
2977
|
+
function getSkillInstallDir(globalInstall) {
|
|
2978
|
+
return globalInstall
|
|
2979
|
+
? resolve(homedir(), ".agents", "skills", "qmd")
|
|
2980
|
+
: resolve(getPwd(), ".agents", "skills", "qmd");
|
|
2981
|
+
}
|
|
2982
|
+
function getClaudeSkillLinkPath(globalInstall) {
|
|
2983
|
+
return globalInstall
|
|
2984
|
+
? resolve(homedir(), ".claude", "skills", "qmd")
|
|
2985
|
+
: resolve(getPwd(), ".claude", "skills", "qmd");
|
|
2986
|
+
}
|
|
2987
|
+
function pathExists(path) {
|
|
2988
|
+
try {
|
|
2989
|
+
lstatSync(path);
|
|
2990
|
+
return true;
|
|
2991
|
+
}
|
|
2992
|
+
catch {
|
|
2993
|
+
return false;
|
|
2994
|
+
}
|
|
2995
|
+
}
|
|
2996
|
+
function removePath(path) {
|
|
2997
|
+
const stat = lstatSync(path);
|
|
2998
|
+
if (stat.isDirectory() && !stat.isSymbolicLink()) {
|
|
2999
|
+
rmSync(path, { recursive: true, force: true });
|
|
3000
|
+
}
|
|
3001
|
+
else {
|
|
3002
|
+
unlinkSync(path);
|
|
3003
|
+
}
|
|
3004
|
+
}
|
|
3005
|
+
const SKILL_DIR = "skills";
|
|
3006
|
+
function findPackageRoot() {
|
|
3007
|
+
if (process.env.QMD_SKILLS_DIR) {
|
|
3008
|
+
return null;
|
|
3009
|
+
}
|
|
3010
|
+
const start = dirname(fileURLToPath(import.meta.url));
|
|
3011
|
+
let current = start;
|
|
3012
|
+
while (true) {
|
|
3013
|
+
if (existsSync(resolve(current, SKILL_DIR))) {
|
|
3014
|
+
return current;
|
|
3015
|
+
}
|
|
3016
|
+
const parent = dirname(current);
|
|
3017
|
+
if (parent === current)
|
|
3018
|
+
break;
|
|
3019
|
+
current = parent;
|
|
3020
|
+
}
|
|
3021
|
+
return null;
|
|
3022
|
+
}
|
|
3023
|
+
function getSkillSearchDirs(_runtimeOnly = false) {
|
|
3024
|
+
if (process.env.QMD_SKILLS_DIR) {
|
|
3025
|
+
return [process.env.QMD_SKILLS_DIR];
|
|
3026
|
+
}
|
|
3027
|
+
const root = findPackageRoot();
|
|
3028
|
+
if (!root)
|
|
3029
|
+
return [];
|
|
3030
|
+
const dir = resolve(root, SKILL_DIR);
|
|
3031
|
+
return existsSync(dir) ? [dir] : [];
|
|
3032
|
+
}
|
|
3033
|
+
function parseSkillFrontmatter(content) {
|
|
3034
|
+
const trimmed = content.trimStart();
|
|
3035
|
+
if (!trimmed.startsWith("---"))
|
|
3036
|
+
return null;
|
|
3037
|
+
const end = trimmed.slice(3).indexOf("\n---");
|
|
3038
|
+
if (end < 0)
|
|
3039
|
+
return null;
|
|
3040
|
+
const frontmatter = trimmed.slice(3, 3 + end);
|
|
3041
|
+
let name = "";
|
|
3042
|
+
let description = "";
|
|
3043
|
+
let hidden = false;
|
|
3044
|
+
const lines = frontmatter.split(/\r?\n/);
|
|
3045
|
+
for (let i = 0; i < lines.length; i++) {
|
|
3046
|
+
const line = lines[i];
|
|
3047
|
+
if (line.startsWith("name:")) {
|
|
3048
|
+
name = line.slice("name:".length).trim();
|
|
3049
|
+
}
|
|
3050
|
+
else if (line.startsWith("description:")) {
|
|
3051
|
+
const parts = [line.slice("description:".length).trim()];
|
|
3052
|
+
while (i + 1 < lines.length && /^\s+\S/.test(lines[i + 1])) {
|
|
3053
|
+
i++;
|
|
3054
|
+
parts.push(lines[i].trim());
|
|
3055
|
+
}
|
|
3056
|
+
description = parts.join(" ");
|
|
3057
|
+
}
|
|
3058
|
+
else if (line.startsWith("hidden:")) {
|
|
3059
|
+
const value = line.slice("hidden:".length).trim().toLowerCase();
|
|
3060
|
+
hidden = value === "true" || value === "yes";
|
|
3061
|
+
}
|
|
3062
|
+
}
|
|
3063
|
+
if (!name)
|
|
3064
|
+
return null;
|
|
3065
|
+
return { name, description, hidden };
|
|
3066
|
+
}
|
|
3067
|
+
function discoverSkills(runtimeOnly = false) {
|
|
3068
|
+
const skills = [];
|
|
3069
|
+
for (const dir of getSkillSearchDirs(runtimeOnly)) {
|
|
3070
|
+
let entries = [];
|
|
3071
|
+
try {
|
|
3072
|
+
entries = readdirSync(dir);
|
|
3073
|
+
}
|
|
3074
|
+
catch {
|
|
3075
|
+
continue;
|
|
3076
|
+
}
|
|
3077
|
+
for (const entry of entries) {
|
|
3078
|
+
const skillDir = resolve(dir, entry);
|
|
3079
|
+
const skillPath = resolve(skillDir, "SKILL.md");
|
|
3080
|
+
if (!existsSync(skillPath))
|
|
3081
|
+
continue;
|
|
3082
|
+
let content = "";
|
|
3083
|
+
try {
|
|
3084
|
+
content = readFileSync(skillPath, "utf-8");
|
|
3085
|
+
}
|
|
3086
|
+
catch {
|
|
3087
|
+
continue;
|
|
3088
|
+
}
|
|
3089
|
+
const parsed = parseSkillFrontmatter(content);
|
|
3090
|
+
if (!parsed)
|
|
3091
|
+
continue;
|
|
3092
|
+
skills.push({ ...parsed, dir: skillDir });
|
|
3093
|
+
}
|
|
3094
|
+
}
|
|
3095
|
+
return skills.sort((a, b) => a.name.localeCompare(b.name));
|
|
3096
|
+
}
|
|
3097
|
+
function findSkill(name, runtimeOnly = false) {
|
|
3098
|
+
return discoverSkills(runtimeOnly).find((skill) => skill.name === name) ?? null;
|
|
3099
|
+
}
|
|
3100
|
+
function readSkillContent(skill) {
|
|
3101
|
+
return readFileSync(resolve(skill.dir, "SKILL.md"), "utf-8");
|
|
3102
|
+
}
|
|
3103
|
+
function collectSkillFiles(skill) {
|
|
3104
|
+
const files = [];
|
|
3105
|
+
for (const subdirName of ["references", "templates", "scripts"]) {
|
|
3106
|
+
const subdir = resolve(skill.dir, subdirName);
|
|
3107
|
+
if (!existsSync(subdir))
|
|
3108
|
+
continue;
|
|
3109
|
+
for (const entry of readdirSync(subdir).sort()) {
|
|
3110
|
+
const filePath = resolve(subdir, entry);
|
|
3111
|
+
try {
|
|
3112
|
+
if (!statSync(filePath).isFile())
|
|
3113
|
+
continue;
|
|
3114
|
+
files.push({ relativePath: `${subdirName}/${basename(filePath)}`, content: readFileSync(filePath, "utf-8") });
|
|
3115
|
+
}
|
|
3116
|
+
catch {
|
|
3117
|
+
// Ignore unreadable supplementary files.
|
|
3118
|
+
}
|
|
3119
|
+
}
|
|
3120
|
+
}
|
|
3121
|
+
return files;
|
|
3122
|
+
}
|
|
3123
|
+
function showSkill() {
|
|
3124
|
+
const skill = findSkill("qmd");
|
|
3125
|
+
if (!skill) {
|
|
3126
|
+
throw new Error("QMD skill not found. Reinstall qmd or set QMD_SKILLS_DIR.");
|
|
3127
|
+
}
|
|
3128
|
+
console.log("QMD Skill");
|
|
3129
|
+
console.log("");
|
|
3130
|
+
const content = readSkillContent(skill);
|
|
3131
|
+
process.stdout.write(content.endsWith("\n") ? content : content + "\n");
|
|
3132
|
+
}
|
|
3133
|
+
function copyDirectoryContents(sourceDir, targetDir) {
|
|
3134
|
+
mkdirSync(targetDir, { recursive: true });
|
|
3135
|
+
for (const entry of readdirSync(sourceDir)) {
|
|
3136
|
+
const sourcePath = resolve(sourceDir, entry);
|
|
3137
|
+
const targetPath = resolve(targetDir, entry);
|
|
3138
|
+
const stat = statSync(sourcePath);
|
|
3139
|
+
if (stat.isDirectory()) {
|
|
3140
|
+
copyDirectoryContents(sourcePath, targetPath);
|
|
3141
|
+
}
|
|
3142
|
+
else if (stat.isFile()) {
|
|
3143
|
+
copyFileSync(sourcePath, targetPath);
|
|
3144
|
+
}
|
|
3145
|
+
}
|
|
3146
|
+
}
|
|
3147
|
+
function installedSkillStubContent() {
|
|
3148
|
+
return `---
|
|
3149
|
+
name: qmd
|
|
3150
|
+
description: Bootstrap QMD search instructions from the installed qmd CLI. Use when users ask to find notes, retrieve documents, inspect a wiki, or answer from indexed local markdown.
|
|
3151
|
+
license: MIT
|
|
3152
|
+
compatibility: Requires qmd CLI. Run \`qmd skill show\` for version-matched instructions.
|
|
3153
|
+
allowed-tools: Bash(qmd:*), mcp__qmd__*
|
|
3154
|
+
---
|
|
3155
|
+
|
|
3156
|
+
# QMD - Query Markdown Documents
|
|
3157
|
+
|
|
3158
|
+
This installed skill is intentionally a small bootstrap so it does not go stale
|
|
3159
|
+
when the qmd package updates.
|
|
3160
|
+
|
|
3161
|
+
Load the full, version-matched QMD instructions from the CLI:
|
|
3162
|
+
|
|
3163
|
+
!\`qmd skill show\`
|
|
3164
|
+
|
|
3165
|
+
If your agent does not support bang-command expansion, run:
|
|
3166
|
+
|
|
3167
|
+
\`\`\`bash
|
|
3168
|
+
qmd skill show
|
|
3169
|
+
\`\`\`
|
|
3170
|
+
|
|
3171
|
+
Then follow those instructions. In short: search first, fetch full sources with
|
|
3172
|
+
\`qmd get\` or \`qmd multi-get\`, and answer from retrieved text rather than snippets.
|
|
3173
|
+
`;
|
|
3174
|
+
}
|
|
3175
|
+
function writeSkillInstall(targetDir, force) {
|
|
3176
|
+
if (pathExists(targetDir)) {
|
|
3177
|
+
if (!force) {
|
|
3178
|
+
throw new Error(`Skill already exists: ${targetDir} (use --force to replace it)`);
|
|
3179
|
+
}
|
|
3180
|
+
removePath(targetDir);
|
|
3181
|
+
}
|
|
3182
|
+
const skill = findSkill("qmd");
|
|
3183
|
+
if (!skill) {
|
|
3184
|
+
throw new Error("QMD skill not found. Reinstall qmd or set QMD_SKILLS_DIR.");
|
|
3185
|
+
}
|
|
3186
|
+
copyDirectoryContents(skill.dir, targetDir);
|
|
3187
|
+
writeFileSync(resolve(targetDir, "SKILL.md"), installedSkillStubContent(), "utf-8");
|
|
3188
|
+
}
|
|
3189
|
+
function outputSkillsJson(payload) {
|
|
3190
|
+
console.log(JSON.stringify(payload));
|
|
3191
|
+
}
|
|
3192
|
+
function runSkillsCommand(args, jsonMode, fullOption = false, allOption = false) {
|
|
3193
|
+
const subcommand = args[0] ?? "list";
|
|
3194
|
+
const runtimeSkills = () => discoverSkills(true).filter((skill) => !skill.hidden);
|
|
3195
|
+
switch (subcommand) {
|
|
3196
|
+
case "list": {
|
|
3197
|
+
const skills = runtimeSkills();
|
|
3198
|
+
if (jsonMode) {
|
|
3199
|
+
outputSkillsJson({ success: true, data: skills.map(({ name, description }) => ({ name, description })) });
|
|
3200
|
+
return;
|
|
3201
|
+
}
|
|
3202
|
+
if (skills.length === 0) {
|
|
3203
|
+
console.log("No skills found");
|
|
3204
|
+
return;
|
|
3205
|
+
}
|
|
3206
|
+
const maxName = Math.max(...skills.map((skill) => skill.name.length));
|
|
3207
|
+
for (const skill of skills) {
|
|
3208
|
+
console.log(` ${skill.name.padEnd(maxName)} ${skill.description}`);
|
|
3209
|
+
}
|
|
3210
|
+
return;
|
|
3211
|
+
}
|
|
3212
|
+
case "get": {
|
|
3213
|
+
const full = fullOption || args.includes("--full");
|
|
3214
|
+
const getAll = allOption || args.includes("--all");
|
|
3215
|
+
const names = args.slice(1).filter((arg) => arg !== "--full" && arg !== "--all");
|
|
3216
|
+
const targets = getAll ? runtimeSkills() : names.map((name) => {
|
|
3217
|
+
const skill = findSkill(name, true);
|
|
3218
|
+
if (!skill) {
|
|
3219
|
+
throw new Error(`Skill not found: ${name}`);
|
|
3220
|
+
}
|
|
3221
|
+
return skill;
|
|
3222
|
+
});
|
|
3223
|
+
if (targets.length === 0) {
|
|
3224
|
+
throw new Error("No skill name provided. Usage: qmd skills get <name>");
|
|
3225
|
+
}
|
|
3226
|
+
if (jsonMode) {
|
|
3227
|
+
outputSkillsJson({
|
|
3228
|
+
success: true,
|
|
3229
|
+
data: targets.map((skill) => ({
|
|
3230
|
+
name: skill.name,
|
|
3231
|
+
content: readSkillContent(skill),
|
|
3232
|
+
...(full ? { files: collectSkillFiles(skill).map((file) => ({ path: file.relativePath, content: file.content })) } : {}),
|
|
3233
|
+
})),
|
|
3234
|
+
});
|
|
3235
|
+
return;
|
|
3236
|
+
}
|
|
3237
|
+
targets.forEach((skill, index) => {
|
|
3238
|
+
if (index > 0)
|
|
3239
|
+
console.log("\n---\n");
|
|
3240
|
+
const content = readSkillContent(skill);
|
|
3241
|
+
process.stdout.write(content.endsWith("\n") ? content : content + "\n");
|
|
3242
|
+
if (full) {
|
|
3243
|
+
for (const file of collectSkillFiles(skill)) {
|
|
3244
|
+
console.log(`\n--- ${file.relativePath} ---\n`);
|
|
3245
|
+
process.stdout.write(file.content.endsWith("\n") ? file.content : file.content + "\n");
|
|
3246
|
+
}
|
|
3247
|
+
}
|
|
3248
|
+
});
|
|
3249
|
+
return;
|
|
3250
|
+
}
|
|
3251
|
+
case "path": {
|
|
3252
|
+
const name = args[1];
|
|
3253
|
+
if (!name) {
|
|
3254
|
+
const paths = getSkillSearchDirs(true);
|
|
3255
|
+
if (jsonMode)
|
|
3256
|
+
outputSkillsJson({ success: true, data: { paths } });
|
|
3257
|
+
else
|
|
3258
|
+
paths.forEach((path) => console.log(path));
|
|
3259
|
+
return;
|
|
3260
|
+
}
|
|
3261
|
+
const skill = findSkill(name, true);
|
|
3262
|
+
if (!skill) {
|
|
3263
|
+
throw new Error(`Skill not found: ${name}`);
|
|
3264
|
+
}
|
|
3265
|
+
if (jsonMode)
|
|
3266
|
+
outputSkillsJson({ success: true, data: { name: skill.name, path: skill.dir } });
|
|
3267
|
+
else
|
|
3268
|
+
console.log(skill.dir);
|
|
3269
|
+
return;
|
|
3270
|
+
}
|
|
3271
|
+
case "help": {
|
|
3272
|
+
showSkillsHelp();
|
|
3273
|
+
return;
|
|
3274
|
+
}
|
|
3275
|
+
default:
|
|
3276
|
+
throw new Error(`Unknown skills subcommand: ${subcommand}`);
|
|
3277
|
+
}
|
|
3278
|
+
}
|
|
3279
|
+
function showSkillsHelp() {
|
|
3280
|
+
console.log("Usage: qmd skills <list|get|path> [options]");
|
|
3281
|
+
console.log("");
|
|
3282
|
+
console.log("Commands:");
|
|
3283
|
+
console.log(" list List bundled runtime skills");
|
|
3284
|
+
console.log(" get <name> Print a bundled runtime skill");
|
|
3285
|
+
console.log(" get <name> --full Include references/templates/scripts");
|
|
3286
|
+
console.log(" get --all Print all bundled runtime skills");
|
|
3287
|
+
console.log(" path [name] Print runtime skill directory path(s)");
|
|
3288
|
+
console.log("");
|
|
3289
|
+
console.log("Options:");
|
|
3290
|
+
console.log(" --json Print structured JSON");
|
|
3291
|
+
}
|
|
3292
|
+
function ensureClaudeSymlink(linkPath, targetDir, force) {
|
|
3293
|
+
const parentDir = dirname(linkPath);
|
|
3294
|
+
if (pathExists(parentDir)) {
|
|
3295
|
+
const resolvedTargetDir = realpathSync(dirname(targetDir));
|
|
3296
|
+
const resolvedLinkParent = realpathSync(parentDir);
|
|
3297
|
+
// If .claude/skills already resolves to the same directory as .agents/skills,
|
|
3298
|
+
// the skill is already visible to Claude and creating qmd -> qmd would loop.
|
|
3299
|
+
if (resolvedTargetDir === resolvedLinkParent) {
|
|
3300
|
+
return false;
|
|
3301
|
+
}
|
|
3302
|
+
}
|
|
3303
|
+
const linkTarget = relativePath(parentDir, targetDir) || ".";
|
|
3304
|
+
mkdirSync(parentDir, { recursive: true });
|
|
3305
|
+
if (pathExists(linkPath)) {
|
|
3306
|
+
const stat = lstatSync(linkPath);
|
|
3307
|
+
if (stat.isSymbolicLink() && readlinkSync(linkPath) === linkTarget) {
|
|
3308
|
+
return true;
|
|
3309
|
+
}
|
|
3310
|
+
if (!force) {
|
|
3311
|
+
throw new Error(`Claude skill path already exists: ${linkPath} (use --force to replace it)`);
|
|
3312
|
+
}
|
|
3313
|
+
removePath(linkPath);
|
|
3314
|
+
}
|
|
3315
|
+
symlinkSync(linkTarget, linkPath, "dir");
|
|
3316
|
+
return true;
|
|
3317
|
+
}
|
|
3318
|
+
async function shouldCreateClaudeSymlink(linkPath, autoYes) {
|
|
3319
|
+
if (autoYes) {
|
|
3320
|
+
return true;
|
|
3321
|
+
}
|
|
3322
|
+
if (!process.stdin.isTTY || !process.stdout.isTTY) {
|
|
3323
|
+
console.log(`Tip: create a Claude symlink manually at ${linkPath}`);
|
|
3324
|
+
return false;
|
|
3325
|
+
}
|
|
3326
|
+
const rl = createInterface({
|
|
3327
|
+
input: process.stdin,
|
|
3328
|
+
output: process.stdout,
|
|
3329
|
+
});
|
|
3330
|
+
try {
|
|
3331
|
+
const answer = await rl.question(`Create a symlink in ${linkPath}? [y/N] `);
|
|
3332
|
+
const normalized = answer.trim().toLowerCase();
|
|
3333
|
+
return normalized === "y" || normalized === "yes";
|
|
3334
|
+
}
|
|
3335
|
+
finally {
|
|
3336
|
+
rl.close();
|
|
3337
|
+
}
|
|
3338
|
+
}
|
|
3339
|
+
async function installSkill(globalInstall, force, autoYes) {
|
|
3340
|
+
const installDir = getSkillInstallDir(globalInstall);
|
|
3341
|
+
writeSkillInstall(installDir, force);
|
|
3342
|
+
console.log(`✓ Installed QMD skill to ${installDir}`);
|
|
3343
|
+
const claudeLinkPath = getClaudeSkillLinkPath(globalInstall);
|
|
3344
|
+
if (!(await shouldCreateClaudeSymlink(claudeLinkPath, autoYes))) {
|
|
3345
|
+
return;
|
|
3346
|
+
}
|
|
3347
|
+
const linked = ensureClaudeSymlink(claudeLinkPath, installDir, force);
|
|
3348
|
+
if (linked) {
|
|
3349
|
+
console.log(`✓ Linked Claude skill at ${claudeLinkPath}`);
|
|
3350
|
+
}
|
|
3351
|
+
else {
|
|
3352
|
+
console.log(`✓ Claude already sees the skill via ${dirname(claudeLinkPath)}`);
|
|
3353
|
+
}
|
|
3354
|
+
}
|
|
3355
|
+
function showHelp() {
|
|
3356
|
+
console.log("qmd — Quick Markdown Search");
|
|
3357
|
+
console.log("");
|
|
3358
|
+
console.log("Usage:");
|
|
3359
|
+
console.log(" qmd <command> [options]");
|
|
3360
|
+
console.log("");
|
|
3361
|
+
console.log("Primary commands:");
|
|
3362
|
+
console.log(" qmd query <query> - Hybrid search with auto expansion + reranking (recommended)");
|
|
3363
|
+
console.log(" qmd query 'lex:..\\nvec:...' - Structured query document (you provide lex/vec/hyde lines)");
|
|
3364
|
+
console.log(" qmd search <query> - Full-text BM25 keywords (no LLM)");
|
|
3365
|
+
console.log(" qmd vsearch <query> - Vector similarity only");
|
|
3366
|
+
console.log(" qmd get <file>[:from[:count]] - Show a document (line-numbered; #docid in header)");
|
|
3367
|
+
console.log(" qmd multi-get <pattern> - Batch fetch via glob or comma-separated list");
|
|
3368
|
+
console.log(" qmd skills list/get/path - List and retrieve bundled runtime skills");
|
|
3369
|
+
console.log(" qmd skill show/install - Show or install the QMD skill");
|
|
3370
|
+
console.log(" qmd mcp - Start the MCP server (stdio transport for AI agents)");
|
|
3371
|
+
console.log(" qmd bench <fixture.json> - Run search quality benchmarks against a fixture file");
|
|
3372
|
+
console.log("");
|
|
3373
|
+
console.log("Collections & context:");
|
|
3374
|
+
console.log(" qmd collection add/list/remove/rename/show - Manage indexed folders");
|
|
3375
|
+
console.log(" qmd context add/list/rm - Attach human-written summaries");
|
|
3376
|
+
console.log(" qmd ls [collection[/path]] - Inspect indexed files");
|
|
3377
|
+
console.log("");
|
|
3378
|
+
console.log("Maintenance:");
|
|
3379
|
+
console.log(" qmd init - Create a project-local .qmd index");
|
|
3380
|
+
console.log(" qmd status - View index + collection health");
|
|
3381
|
+
console.log(" qmd update [--pull] - Re-index collections (optionally git pull first)");
|
|
3382
|
+
console.log(" qmd trust [list|revoke] - Approve a checked-in .qmd config's hooks/paths/models");
|
|
3383
|
+
console.log(" qmd embed [-f] [-c <name>] - Generate/refresh vector embeddings");
|
|
3384
|
+
console.log(" --max-docs-per-batch <n> - Cap docs loaded into memory per embedding batch");
|
|
3385
|
+
console.log(" --max-batch-mb <n> - Cap UTF-8 MB loaded into memory per embedding batch");
|
|
3386
|
+
console.log(" --timeout <minutes> - Embed session cap in minutes (0 = no limit; default 30)");
|
|
3387
|
+
console.log(" qmd pull [--refresh] [--progress] - Download embedding/generation/rerank models");
|
|
3388
|
+
console.log(" qmd cleanup [--dry-run] - Drop inactive docs/orphans, compact FTS, vacuum");
|
|
3389
|
+
console.log("");
|
|
3390
|
+
console.log("Query syntax (qmd query):");
|
|
3391
|
+
console.log(" QMD queries are either a single policy query or a multi-line");
|
|
3392
|
+
console.log(" document where every line is typed with lex:, vec:, or hyde:. This grammar");
|
|
3393
|
+
console.log(" matches the docs in docs/SYNTAX.md and is enforced in the CLI.");
|
|
3394
|
+
console.log("");
|
|
3395
|
+
const grammar = [
|
|
3396
|
+
`query = policy_query | query_document ;`,
|
|
3397
|
+
`policy_query = [ "lex:" ] text | explicit_expand ;`,
|
|
3398
|
+
`explicit_expand= "expand:" text ;`,
|
|
3399
|
+
`query_document = [ intent_line ] { typed_line } ;`,
|
|
3400
|
+
`intent_line = "intent:" text newline ;`,
|
|
3401
|
+
`typed_line = type ":" text newline ;`,
|
|
3402
|
+
`type = "lex" | "vec" | "hyde" ;`,
|
|
3403
|
+
`text = quoted_phrase | plain_text ;`,
|
|
3404
|
+
`quoted_phrase = '"' { character } '"' ;`,
|
|
3405
|
+
`plain_text = { character } ;`,
|
|
3406
|
+
`newline = "\\n" ;`,
|
|
3407
|
+
];
|
|
3408
|
+
console.log(" Grammar:");
|
|
3409
|
+
for (const line of grammar) {
|
|
3410
|
+
console.log(` ${line}`);
|
|
3411
|
+
}
|
|
3412
|
+
console.log("");
|
|
3413
|
+
console.log(" Examples:");
|
|
3414
|
+
console.log(" qmd query \"how does auth work\" # shared auto policy");
|
|
3415
|
+
console.log(" qmd query --expand \"how does auth work\" # force expansion");
|
|
3416
|
+
console.log(" qmd query \"lex: authentication\" # skip expansion");
|
|
3417
|
+
console.log(" qmd query $'lex: CAP theorem\\nvec: consistency' # typed query document");
|
|
3418
|
+
console.log(" qmd query $'lex: \"exact matches\" sports -baseball' # phrase + negation lex search");
|
|
3419
|
+
console.log(" qmd query $'hyde: Hypothetical answer text' # hyde-only document");
|
|
3420
|
+
console.log("");
|
|
3421
|
+
console.log(" Constraints:");
|
|
3422
|
+
console.log(" - auto skips expansion for CJK or a strong lexical signal; otherwise it expands.");
|
|
3423
|
+
console.log(" - --expand or expand: selects force; standalone lex: selects skip.");
|
|
3424
|
+
console.log(" - --expand combined with lex: is an error; explicit directives beat auto.");
|
|
3425
|
+
console.log(" - Standalone policy queries cannot mix with typed lines.");
|
|
3426
|
+
console.log(" - Query documents allow only lex:, vec:, or hyde: prefixes.");
|
|
3427
|
+
console.log(" - Each typed line must be single-line text with balanced quotes.");
|
|
3428
|
+
console.log("");
|
|
3429
|
+
console.log("AI agents & integrations:");
|
|
3430
|
+
console.log(" - Run `qmd mcp` to expose the MCP server (stdio) to agents/IDEs.");
|
|
3431
|
+
console.log(" - Run `qmd skills get qmd --full` for version-matched agent instructions.");
|
|
3432
|
+
console.log(" - `qmd skill install` installs the QMD skill into ./.agents/skills/qmd.");
|
|
3433
|
+
console.log(" - Use `qmd skill install --global` for ~/.agents/skills/qmd.");
|
|
3434
|
+
console.log(" - `qmd --skill` is kept as an alias for `qmd skill show`.");
|
|
3435
|
+
console.log(" - Advanced: `qmd mcp --http ...` and `qmd mcp --http --daemon` are optional for custom transports.");
|
|
3436
|
+
console.log("");
|
|
3437
|
+
console.log("Global options:");
|
|
3438
|
+
console.log(" --index <name> - Use a named index (default: index)");
|
|
3439
|
+
console.log(" QMD_EDITOR_URI - Editor link template for clickable TTY search output");
|
|
3440
|
+
console.log("");
|
|
3441
|
+
console.log("Search options:");
|
|
3442
|
+
console.log(" -n <num> - Max results (default 5, or 20 for --format files|json)");
|
|
3443
|
+
console.log(" --all - Return all matches (pair with --min-score)");
|
|
3444
|
+
console.log(" --min-score <num> - Minimum similarity score");
|
|
3445
|
+
console.log(" --full - Output full document instead of snippet");
|
|
3446
|
+
console.log(" -C, --candidate-limit <n> - Max candidates to rerank (default 40, lower = faster)");
|
|
3447
|
+
console.log(" --no-rerank - Skip LLM reranking (use RRF scores only, much faster on CPU)");
|
|
3448
|
+
console.log(" --no-gpu - Force CPU mode for llama.cpp operations (same as QMD_FORCE_CPU=1)");
|
|
3449
|
+
console.log(" --line-numbers - Include line numbers (search; get/multi-get are on by default)");
|
|
3450
|
+
console.log(" --no-line-numbers - Disable line numbers for get/multi-get");
|
|
3451
|
+
console.log(" --full-path - Show on-disk paths instead of qmd:// + docid (get/multi-get/search/query)");
|
|
3452
|
+
console.log(" Paths are ./-prefixed when under $PWD, absolute otherwise");
|
|
3453
|
+
console.log(" Results whose file is gone keep qmd:// + docid and warn on stderr");
|
|
3454
|
+
console.log(" --explain - Include retrieval score traces (query, CLI/--format json)");
|
|
3455
|
+
console.log(" --format <kind> - Output format: cli (default) | json | csv | md | xml | files");
|
|
3456
|
+
console.log(" -c, --collection <name> - Filter by one or more collections");
|
|
3457
|
+
console.log("");
|
|
3458
|
+
console.log("Embed/query options:");
|
|
3459
|
+
console.log(" --chunk-strategy <auto|regex> - Chunking mode (default: regex; auto uses AST for code files)");
|
|
3460
|
+
console.log(" --timeout <minutes> - Embed session cap in minutes (0 = no limit; default 30)");
|
|
3461
|
+
console.log(" --expand - Force query expansion (auto is the default; lex: skips)");
|
|
3462
|
+
console.log("");
|
|
3463
|
+
console.log("Embedding providers & disclosure:");
|
|
3464
|
+
console.log(" - Local embedding is the default. OpenAI requires explicit provider configuration and OPENAI_API_KEY.");
|
|
3465
|
+
console.log(" - Remote document builds send titles and deterministic UTF-8 document chunks; vector/hybrid queries send formatted query text.");
|
|
3466
|
+
console.log(" - Deleting old vectors has no vector rollback; lexical search remains available while rebuilding.");
|
|
3467
|
+
console.log(" - If Jieba or the analyzed index is unavailable, CJK search omits word and bigram channels and uses character-only fallback.");
|
|
3468
|
+
console.log("");
|
|
3469
|
+
console.log("Multi-get options:");
|
|
3470
|
+
console.log(" -l <num> - Maximum lines per file");
|
|
3471
|
+
console.log(" --max-bytes <num> - Skip files larger than N bytes (default 65536)");
|
|
3472
|
+
console.log(" --format <kind> - Same formats as search");
|
|
3473
|
+
console.log("");
|
|
3474
|
+
console.log(`Index: ${getDbPath()}`);
|
|
3475
|
+
}
|
|
3476
|
+
function doctorCheck(label, ok, details) {
|
|
3477
|
+
const mark = ok ? `${c.green}✓${c.reset}` : `${c.yellow}⚠${c.reset}`;
|
|
3478
|
+
console.log(`${mark} ${label}: ${details}`);
|
|
3479
|
+
}
|
|
3480
|
+
function formatCount(n) {
|
|
3481
|
+
return n.toLocaleString("en-US");
|
|
3482
|
+
}
|
|
3483
|
+
function shortModelName(model) {
|
|
3484
|
+
if (model.startsWith("hf:")) {
|
|
3485
|
+
return model.split("/").pop() || model;
|
|
3486
|
+
}
|
|
3487
|
+
return model.length > 56 ? `${model.slice(0, 53)}...` : model;
|
|
3488
|
+
}
|
|
3489
|
+
function normalizedDoctorNextSteps(steps) {
|
|
3490
|
+
const unique = Array.from(new Set(steps));
|
|
3491
|
+
const hasForceEmbed = unique.some(step => step.includes("qmd embed --force"));
|
|
3492
|
+
if (!hasForceEmbed)
|
|
3493
|
+
return unique;
|
|
3494
|
+
return unique.filter(step => !step.includes("qmd embed") || step.startsWith("Run `qmd embed --force`"));
|
|
3495
|
+
}
|
|
3496
|
+
function shortHashSeq(hashSeq) {
|
|
3497
|
+
const idx = hashSeq.lastIndexOf("_");
|
|
3498
|
+
if (idx < 0)
|
|
3499
|
+
return hashSeq.length > 18 ? `${hashSeq.slice(0, 18)}...` : hashSeq;
|
|
3500
|
+
return `${hashSeq.slice(0, 12)}_${hashSeq.slice(idx + 1)}`;
|
|
3501
|
+
}
|
|
3502
|
+
function decodeStoredEmbedding(bytes) {
|
|
3503
|
+
return new Float32Array(bytes.buffer.slice(bytes.byteOffset, bytes.byteOffset + bytes.byteLength));
|
|
3504
|
+
}
|
|
3505
|
+
function cosineDistance(a, b) {
|
|
3506
|
+
if (a.length !== b.length || a.length === 0)
|
|
3507
|
+
return Number.POSITIVE_INFINITY;
|
|
3508
|
+
let dot = 0;
|
|
3509
|
+
let normA = 0;
|
|
3510
|
+
let normB = 0;
|
|
3511
|
+
for (let i = 0; i < a.length; i++) {
|
|
3512
|
+
const av = a[i] ?? 0;
|
|
3513
|
+
const bv = b[i] ?? 0;
|
|
3514
|
+
dot += av * bv;
|
|
3515
|
+
normA += av * av;
|
|
3516
|
+
normB += bv * bv;
|
|
3517
|
+
}
|
|
3518
|
+
if (normA === 0 || normB === 0)
|
|
3519
|
+
return Number.POSITIVE_INFINITY;
|
|
3520
|
+
return 1 - (dot / (Math.sqrt(normA) * Math.sqrt(normB)));
|
|
3521
|
+
}
|
|
3522
|
+
function formatModelDiagnosticPath(path) {
|
|
3523
|
+
return sanitizeDiagnosticMessage(path);
|
|
3524
|
+
}
|
|
3525
|
+
function findCachedModelInspection(model) {
|
|
3526
|
+
const invalid = [];
|
|
3527
|
+
if (model.startsWith("hf:")) {
|
|
3528
|
+
const filename = model.split("/").pop();
|
|
3529
|
+
if (!filename || !existsSync(DEFAULT_MODEL_CACHE_DIR))
|
|
3530
|
+
return { path: null, invalid };
|
|
3531
|
+
const entries = readdirSync(DEFAULT_MODEL_CACHE_DIR, { withFileTypes: true });
|
|
3532
|
+
for (const entry of entries) {
|
|
3533
|
+
// Only consider real `.gguf` blobs. `qmd pull` writes a `<filename>.etag`
|
|
3534
|
+
// HTTP sidecar next to each download; that name also satisfies
|
|
3535
|
+
// `includes(filename)`, so inspecting it as GGUF false-positives
|
|
3536
|
+
// "invalid model" in `qmd doctor` whenever readdir yields the sidecar
|
|
3537
|
+
// before the blob (#812).
|
|
3538
|
+
if (!entry.isFile() || !entry.name.endsWith(".gguf") || !entry.name.includes(filename))
|
|
3539
|
+
continue;
|
|
3540
|
+
const candidate = pathJoin(DEFAULT_MODEL_CACHE_DIR, entry.name);
|
|
3541
|
+
const inspection = inspectGgufFile(candidate);
|
|
3542
|
+
if (inspection.valid)
|
|
3543
|
+
return { path: candidate, invalid };
|
|
3544
|
+
invalid.push(`${formatModelDiagnosticPath(candidate)}: ${inspection.details}`);
|
|
3545
|
+
}
|
|
3546
|
+
return { path: null, invalid };
|
|
3547
|
+
}
|
|
3548
|
+
const inspection = inspectGgufFile(model);
|
|
3549
|
+
if (inspection.valid)
|
|
3550
|
+
return { path: model, invalid };
|
|
3551
|
+
if (inspection.exists)
|
|
3552
|
+
invalid.push(`${formatModelDiagnosticPath(model)}: ${inspection.details}`);
|
|
3553
|
+
return { path: null, invalid };
|
|
3554
|
+
}
|
|
3555
|
+
function envValueForDisplay(value) {
|
|
3556
|
+
const sanitized = sanitizeDiagnosticMessage(value);
|
|
3557
|
+
return sanitized.length > 96 ? `${sanitized.slice(0, 93)}...` : sanitized;
|
|
3558
|
+
}
|
|
3559
|
+
function collectEnvironmentOverrides(activeModels, configModels = {}) {
|
|
3560
|
+
const overrides = [];
|
|
3561
|
+
const add = (name, consequence) => {
|
|
3562
|
+
const raw = process.env[name]?.trim();
|
|
3563
|
+
if (!raw)
|
|
3564
|
+
return;
|
|
3565
|
+
overrides.push({ name, value: envValueForDisplay(raw), consequence });
|
|
3566
|
+
};
|
|
3567
|
+
const addModel = (name, key, active) => {
|
|
3568
|
+
const raw = process.env[name]?.trim();
|
|
3569
|
+
if (!raw)
|
|
3570
|
+
return;
|
|
3571
|
+
const configured = configModels[key];
|
|
3572
|
+
const consequence = configured && configured !== raw
|
|
3573
|
+
? `set but ignored because index models.${key} is configured as ${configured}`
|
|
3574
|
+
: `sets the active ${key} model to ${active}; changes embedding/search semantics and may require \`qmd pull\` plus \`qmd embed\``;
|
|
3575
|
+
overrides.push({ name, value: envValueForDisplay(raw), consequence });
|
|
3576
|
+
};
|
|
3577
|
+
add("INDEX_PATH", "overrides the SQLite index path; QMD reads/writes a different database");
|
|
3578
|
+
add("QMD_CONFIG_DIR", "overrides the QMD config directory and takes precedence over XDG_CONFIG_HOME");
|
|
3579
|
+
add("XDG_CONFIG_HOME", "moves QMD config to $XDG_CONFIG_HOME/qmd when QMD_CONFIG_DIR is not set");
|
|
3580
|
+
add("XDG_CACHE_HOME", "moves the default index cache, model cache, and MCP daemon PID files");
|
|
3581
|
+
addModel("QMD_EMBED_MODEL", "embed", activeModels.embed);
|
|
3582
|
+
addModel("QMD_GENERATE_MODEL", "generate", activeModels.generate);
|
|
3583
|
+
addModel("QMD_RERANK_MODEL", "rerank", activeModels.rerank);
|
|
3584
|
+
add("QMD_FORCE_CPU", "forces llama.cpp to bypass GPU backends; embeddings/query will be slower but GPU crashes are avoided");
|
|
3585
|
+
add("QMD_LLAMA_GPU", "selects llama.cpp GPU backend (metal/cuda/vulkan) or disables GPU when set to false/off/0");
|
|
3586
|
+
add("QMD_DOCTOR_DEVICE_PROBE", "controls qmd doctor native device probing; 0/off skips GPU probing");
|
|
3587
|
+
add("QMD_EMBED_PARALLELISM", "overrides embedding parallel context count; too high can exhaust RAM/VRAM");
|
|
3588
|
+
add("QMD_EXPAND_CONTEXT_SIZE", "overrides query expansion context size; larger values use more memory");
|
|
3589
|
+
add("QMD_RERANK_CONTEXT_SIZE", "overrides reranker context size; larger values use more memory");
|
|
3590
|
+
add("QMD_EMBED_CONTEXT_SIZE", "overrides embed context size; larger values use more memory");
|
|
3591
|
+
add("QMD_EDITOR_URI", "overrides clickable editor link template in terminal output");
|
|
3592
|
+
add("QMD_SKILLS_DIR", "overrides where qmd skills are discovered from");
|
|
3593
|
+
add("QMD_METAL_KEEP_RESIDENCY", "opts back into libggml-metal residency sets on darwin; restores ~0ms perf wins for long-lived processes but re-exposes the static-destructor backtrace dump at process exit (ggml-org/llama.cpp#22593)");
|
|
3594
|
+
add("GGML_METAL_NO_RESIDENCY", "set automatically by the launcher on darwin to disable Metal residency sets (avoids ggml-org/llama.cpp#22593); override via QMD_METAL_KEEP_RESIDENCY=1");
|
|
3595
|
+
add("NO_COLOR", "disables colored terminal output");
|
|
3596
|
+
add("CI", "disables real LLM operations inside QMD's LlamaCpp wrapper");
|
|
3597
|
+
add("HF_ENDPOINT", "changes Hugging Face download endpoint used when pulling models");
|
|
3598
|
+
add("QMD_WRAPPER_CAPTURE", "test/debug hook for the qmd shell wrapper; should not be set in normal use");
|
|
3599
|
+
add("WSL_DISTRO_NAME", "enables WSL path handling heuristics");
|
|
3600
|
+
add("WSL_INTEROP", "enables WSL path handling heuristics");
|
|
3601
|
+
return overrides;
|
|
3602
|
+
}
|
|
3603
|
+
function checkDoctorIndexConfig(nextSteps) {
|
|
3604
|
+
try {
|
|
3605
|
+
const config = loadConfig();
|
|
3606
|
+
const collectionCount = Object.keys(config.collections ?? {}).length;
|
|
3607
|
+
if (collectionCount === 0) {
|
|
3608
|
+
doctorCheck("index config", false, "no collections configured. Next: `qmd collection add .`");
|
|
3609
|
+
nextSteps.push("Run `qmd collection add . --name <name>` from the folder you want to index, or edit .qmd/index.yml manually.");
|
|
3610
|
+
}
|
|
3611
|
+
else {
|
|
3612
|
+
doctorCheck("index config", true, `${formatCount(collectionCount)} ${collectionCount === 1 ? "collection" : "collections"} configured`);
|
|
3613
|
+
}
|
|
3614
|
+
return { config, valid: true };
|
|
3615
|
+
}
|
|
3616
|
+
catch (error) {
|
|
3617
|
+
const message = error instanceof Error ? sanitizeDiagnosticMessage(error.message) : sanitizeDiagnosticMessage(String(error));
|
|
3618
|
+
const configPath = getConfigPath();
|
|
3619
|
+
doctorCheck("index config", false, `invalid index.yml at ${configPath}: ${message}. Next: fix the YAML and rerun \`qmd doctor\``);
|
|
3620
|
+
nextSteps.push(`Fix invalid YAML in ${configPath}, then rerun \`qmd doctor\`.`);
|
|
3621
|
+
return { config: null, valid: false };
|
|
3622
|
+
}
|
|
3623
|
+
}
|
|
3624
|
+
function checkEnvironmentOverrides(activeModels, configModels = {}) {
|
|
3625
|
+
const overrides = collectEnvironmentOverrides(activeModels, configModels);
|
|
3626
|
+
if (overrides.length === 0) {
|
|
3627
|
+
doctorCheck("environment overrides", true, "none");
|
|
3628
|
+
return;
|
|
3629
|
+
}
|
|
3630
|
+
doctorCheck("environment overrides", false, `${overrides.length} set`);
|
|
3631
|
+
for (const override of overrides) {
|
|
3632
|
+
console.log(` - ${override.name}=${override.value}: ${override.consequence}`);
|
|
3633
|
+
}
|
|
3634
|
+
}
|
|
3635
|
+
function checkModelDefaults(activeModels, configModels = {}, doctorEmbedding) {
|
|
3636
|
+
const isRemoteEmbed = Boolean(doctorEmbedding?.provider === "openai"
|
|
3637
|
+
|| configModels.embed_api_url
|
|
3638
|
+
|| configModels.embed_base_url
|
|
3639
|
+
|| configModels.embed_url
|
|
3640
|
+
|| configModels.embed_api_model);
|
|
3641
|
+
const isRemoteGen = Boolean(configModels.generate_api_url
|
|
3642
|
+
|| configModels.generate_base_url
|
|
3643
|
+
|| configModels.generate_url
|
|
3644
|
+
|| configModels.generate_api_model);
|
|
3645
|
+
const isRemoteRerank = Boolean(configModels.rerank_api_url
|
|
3646
|
+
|| configModels.rerank_base_url
|
|
3647
|
+
|| configModels.rerank_url
|
|
3648
|
+
|| configModels.rerank_api_model);
|
|
3649
|
+
const checks = [
|
|
3650
|
+
{ role: "embedding", key: "embed", isRemote: isRemoteEmbed, active: activeModels.embed, configured: configModels.embed, defaultModel: DEFAULT_EMBED_MODEL, envName: "QMD_EMBED_MODEL", envValue: process.env.QMD_EMBED_MODEL },
|
|
3651
|
+
{ role: "generation", key: "generate", isRemote: isRemoteGen, active: activeModels.generate, configured: configModels.generate, defaultModel: DEFAULT_QUERY_MODEL, envName: "QMD_GENERATE_MODEL", envValue: process.env.QMD_GENERATE_MODEL },
|
|
3652
|
+
{ role: "reranking", key: "rerank", isRemote: isRemoteRerank, active: activeModels.rerank, configured: configModels.rerank, defaultModel: DEFAULT_RERANK_MODEL, envName: "QMD_RERANK_MODEL", envValue: process.env.QMD_RERANK_MODEL },
|
|
3653
|
+
];
|
|
3654
|
+
const notes = [];
|
|
3655
|
+
for (const check of checks) {
|
|
3656
|
+
if (check.isRemote)
|
|
3657
|
+
continue;
|
|
3658
|
+
const envValue = check.envValue?.trim();
|
|
3659
|
+
if (envValue && check.active === envValue) {
|
|
3660
|
+
notes.push(`${check.role}: env ${check.envName}=${check.active} (default ${check.defaultModel}; might be ok)`);
|
|
3661
|
+
}
|
|
3662
|
+
else if (check.configured && check.configured !== check.defaultModel) {
|
|
3663
|
+
notes.push(`${check.role}: index ${check.configured} (default ${check.defaultModel}; might be ok)`);
|
|
3664
|
+
}
|
|
3665
|
+
else if (envValue && check.active !== envValue) {
|
|
3666
|
+
notes.push(`${check.role}: ${check.envName} is set to ${envValue} but index config uses ${check.active}`);
|
|
3667
|
+
}
|
|
3668
|
+
}
|
|
3669
|
+
if (notes.length === 0) {
|
|
3670
|
+
doctorCheck("model defaults", true, "using QMD codebase defaults");
|
|
3671
|
+
return;
|
|
3672
|
+
}
|
|
3673
|
+
doctorCheck("model defaults", false, `non-default model configuration: ${notes.join("; ")}`);
|
|
3674
|
+
}
|
|
3675
|
+
function checkModelCache(activeModels, nextSteps) {
|
|
3676
|
+
const models = [
|
|
3677
|
+
["embedding", activeModels.embed],
|
|
3678
|
+
["generation", activeModels.generate],
|
|
3679
|
+
["reranking", activeModels.rerank],
|
|
3680
|
+
];
|
|
3681
|
+
const unique = new Map();
|
|
3682
|
+
for (const [role, model] of models) {
|
|
3683
|
+
unique.set(model, [...(unique.get(model) ?? []), role]);
|
|
3684
|
+
}
|
|
3685
|
+
const missing = [];
|
|
3686
|
+
const cached = [];
|
|
3687
|
+
const invalid = [];
|
|
3688
|
+
for (const [model, roles] of unique) {
|
|
3689
|
+
const label = `${roles.join("+")}: ${model}`;
|
|
3690
|
+
const isRemoteApi = !model.startsWith("hf:") && !existsSync(model) && !model.endsWith(".gguf");
|
|
3691
|
+
if (isRemoteApi) {
|
|
3692
|
+
cached.push(`${label} (remote API)`);
|
|
3693
|
+
continue;
|
|
3694
|
+
}
|
|
3695
|
+
const inspection = findCachedModelInspection(model);
|
|
3696
|
+
invalid.push(...inspection.invalid.map(detail => `${label} (${detail})`));
|
|
3697
|
+
if (inspection.path) {
|
|
3698
|
+
cached.push(label);
|
|
3699
|
+
}
|
|
3700
|
+
else {
|
|
3701
|
+
missing.push(label);
|
|
3702
|
+
}
|
|
3703
|
+
}
|
|
3704
|
+
if (missing.length === 0 && invalid.length === 0) {
|
|
3705
|
+
doctorCheck("model cache", true, `${cached.length} active ${cached.length === 1 ? "model is" : "models are"} downloaded and valid GGUF`);
|
|
3706
|
+
return;
|
|
3707
|
+
}
|
|
3708
|
+
const parts = [];
|
|
3709
|
+
if (invalid.length > 0)
|
|
3710
|
+
parts.push(`invalid ${invalid.length}: ${invalid.join("; ")}`);
|
|
3711
|
+
if (missing.length > 0)
|
|
3712
|
+
parts.push(`missing ${missing.length}/${unique.size}: ${missing.join("; ")}`);
|
|
3713
|
+
const next = invalid.length > 0
|
|
3714
|
+
? "Next: run `qmd pull --refresh` (or remove the bad cached file)"
|
|
3715
|
+
: "Next: run `qmd pull`";
|
|
3716
|
+
doctorCheck("model cache", false, `${parts.join("; ")}. ${next}`);
|
|
3717
|
+
if (invalid.length > 0) {
|
|
3718
|
+
nextSteps.push("Run `qmd pull --refresh` to replace invalid cached model files, or delete the listed file and rerun `qmd pull`.");
|
|
3719
|
+
}
|
|
3720
|
+
else {
|
|
3721
|
+
nextSteps.push("Run `qmd pull` to download missing embedding/generation/reranking models before `qmd embed` or `qmd query`.");
|
|
3722
|
+
}
|
|
3723
|
+
}
|
|
3724
|
+
async function checkEmbeddingVectorSamples(db, model, fingerprint, sampleSize = 3) {
|
|
3725
|
+
const activeDocs = db.prepare(`SELECT COUNT(*) AS count FROM documents WHERE active = 1`).get().count;
|
|
3726
|
+
if (activeDocs === 0) {
|
|
3727
|
+
return { ok: true, details: "no active documents indexed" };
|
|
3728
|
+
}
|
|
3729
|
+
const vecTableExists = db.prepare(`SELECT 1 FROM sqlite_master WHERE type='table' AND name='vectors_vec'`).get();
|
|
3730
|
+
if (!vecTableExists) {
|
|
3731
|
+
return { ok: false, details: "no vector table to test; please run qmd embed again" };
|
|
3732
|
+
}
|
|
3733
|
+
const samples = db.prepare(`
|
|
3734
|
+
SELECT cv.hash, cv.seq, c.doc AS body, MIN(d.path) AS path
|
|
3735
|
+
FROM content_vectors cv
|
|
3736
|
+
JOIN documents d ON d.hash = cv.hash AND d.active = 1
|
|
3737
|
+
JOIN content c ON c.hash = cv.hash
|
|
3738
|
+
WHERE cv.model = ? AND cv.embed_fingerprint = ?
|
|
3739
|
+
GROUP BY cv.hash, cv.seq, c.doc
|
|
3740
|
+
ORDER BY random()
|
|
3741
|
+
LIMIT ?
|
|
3742
|
+
`).all(model, fingerprint, sampleSize);
|
|
3743
|
+
if (samples.length === 0) {
|
|
3744
|
+
return { ok: false, details: "no current embedded chunks to test; please run qmd embed again" };
|
|
3745
|
+
}
|
|
3746
|
+
const threshold = 0.0001;
|
|
3747
|
+
const mismatches = [];
|
|
3748
|
+
await withLLMSession(async (session) => {
|
|
3749
|
+
for (const sample of samples) {
|
|
3750
|
+
const hashSeq = `${sample.hash}_${sample.seq}`;
|
|
3751
|
+
const chunks = await chunkDocumentByTokens(sample.body, undefined, undefined, undefined, sample.path, undefined, session.signal);
|
|
3752
|
+
const chunk = chunks[sample.seq];
|
|
3753
|
+
if (!chunk) {
|
|
3754
|
+
mismatches.push(`${shortHashSeq(hashSeq)}: chunk no longer exists`);
|
|
3755
|
+
continue;
|
|
3756
|
+
}
|
|
3757
|
+
const title = extractTitle(sample.body, sample.path);
|
|
3758
|
+
const result = await session.embed(formatDocForEmbedding(chunk.text, title, model), { model });
|
|
3759
|
+
if (!result) {
|
|
3760
|
+
mismatches.push(`${shortHashSeq(hashSeq)}: embedding failed`);
|
|
3761
|
+
continue;
|
|
3762
|
+
}
|
|
3763
|
+
const stored = db.prepare(`SELECT embedding FROM vectors_vec WHERE hash_seq = ?`).get(hashSeq);
|
|
3764
|
+
if (!stored) {
|
|
3765
|
+
mismatches.push(`${shortHashSeq(hashSeq)}: stored vector missing`);
|
|
3766
|
+
continue;
|
|
3767
|
+
}
|
|
3768
|
+
const distance = cosineDistance(result.embedding, decodeStoredEmbedding(stored.embedding));
|
|
3769
|
+
if (distance > threshold) {
|
|
3770
|
+
mismatches.push(`${shortHashSeq(hashSeq)}: stored vector distance ${distance.toFixed(6)}`);
|
|
3771
|
+
}
|
|
3772
|
+
}
|
|
3773
|
+
}, { maxDuration: 10 * 60 * 1000, name: "doctorEmbeddingVectorSample" });
|
|
3774
|
+
if (mismatches.length > 0) {
|
|
3775
|
+
return {
|
|
3776
|
+
ok: false,
|
|
3777
|
+
details: `${mismatches.length}/${samples.length} sampled chunks differ from stored vectors (${mismatches[0]}). Rebuild with \`qmd embed --force\``,
|
|
3778
|
+
};
|
|
3779
|
+
}
|
|
3780
|
+
return {
|
|
3781
|
+
ok: true,
|
|
3782
|
+
details: `${samples.length} sampled ${samples.length === 1 ? "chunk" : "chunks"} reproduce stored vectors`,
|
|
3783
|
+
};
|
|
3784
|
+
}
|
|
3785
|
+
function hasLibraryInDirs(libraryBaseName, dirs) {
|
|
3786
|
+
for (const dir of dirs) {
|
|
3787
|
+
if (!dir || !existsSync(dir))
|
|
3788
|
+
continue;
|
|
3789
|
+
try {
|
|
3790
|
+
for (const entry of readdirSync(dir)) {
|
|
3791
|
+
if (entry === libraryBaseName || entry.startsWith(`${libraryBaseName}.`))
|
|
3792
|
+
return true;
|
|
3793
|
+
}
|
|
3794
|
+
}
|
|
3795
|
+
catch { /* ignore unreadable system library dirs */ }
|
|
3796
|
+
}
|
|
3797
|
+
return false;
|
|
3798
|
+
}
|
|
3799
|
+
function linuxCudaRuntimeDiagnostic() {
|
|
3800
|
+
if (process.platform !== "linux")
|
|
3801
|
+
return null;
|
|
3802
|
+
const dirs = new Set();
|
|
3803
|
+
for (const value of [process.env.LD_LIBRARY_PATH, process.env.CUDA_PATH]) {
|
|
3804
|
+
for (const part of (value ?? "").split(":")) {
|
|
3805
|
+
if (part)
|
|
3806
|
+
dirs.add(part);
|
|
3807
|
+
}
|
|
3808
|
+
}
|
|
3809
|
+
if (process.env.CUDA_PATH) {
|
|
3810
|
+
dirs.add(pathJoin(process.env.CUDA_PATH, "lib64"));
|
|
3811
|
+
dirs.add(pathJoin(process.env.CUDA_PATH, "targets", "x86_64-linux", "lib"));
|
|
3812
|
+
}
|
|
3813
|
+
for (const dir of ["/usr/lib", "/usr/lib64", "/usr/lib/x86_64-linux-gnu", "/usr/local/cuda/lib64", "/usr/local/cuda/targets/x86_64-linux/lib"]) {
|
|
3814
|
+
dirs.add(dir);
|
|
3815
|
+
}
|
|
3816
|
+
try {
|
|
3817
|
+
for (const entry of readdirSync("/usr/local")) {
|
|
3818
|
+
if (!entry.toLowerCase().startsWith("cuda-"))
|
|
3819
|
+
continue;
|
|
3820
|
+
const cudaRoot = pathJoin("/usr/local", entry);
|
|
3821
|
+
dirs.add(pathJoin(cudaRoot, "lib64"));
|
|
3822
|
+
dirs.add(pathJoin(cudaRoot, "targets", "x86_64-linux", "lib"));
|
|
3823
|
+
}
|
|
3824
|
+
}
|
|
3825
|
+
catch { /* /usr/local may not be readable in restricted environments */ }
|
|
3826
|
+
const searchDirs = [...dirs];
|
|
3827
|
+
const hasDriver = hasLibraryInDirs("libcuda.so", searchDirs) || hasLibraryInDirs("libnvidia-ml.so", searchDirs);
|
|
3828
|
+
if (!hasDriver)
|
|
3829
|
+
return null;
|
|
3830
|
+
const cudaLibraries = [
|
|
3831
|
+
["libcudart.so", "CUDA runtime"],
|
|
3832
|
+
["libcublas.so", "cuBLAS"],
|
|
3833
|
+
["libcublasLt.so", "cuBLASLt"],
|
|
3834
|
+
];
|
|
3835
|
+
const missing = cudaLibraries
|
|
3836
|
+
.filter(([library]) => !hasLibraryInDirs(library, searchDirs))
|
|
3837
|
+
.map(([, label]) => label);
|
|
3838
|
+
if (missing.length === 0)
|
|
3839
|
+
return null;
|
|
3840
|
+
return `NVIDIA driver libraries are visible, but CUDA user-space libraries are missing from loader paths (${missing.join(", ")})`;
|
|
3841
|
+
}
|
|
3842
|
+
async function runDoctorDeviceChecks(nextSteps) {
|
|
3843
|
+
const mode = configuredGpuModeLabel();
|
|
3844
|
+
doctorCheck("device mode", true, mode);
|
|
3845
|
+
const skipProbe = ["0", "false", "off", "no", "skip"].includes((process.env.QMD_DOCTOR_DEVICE_PROBE ?? "").trim().toLowerCase());
|
|
3846
|
+
if (skipProbe) {
|
|
3847
|
+
doctorCheck("device probe", false, "skipped by QMD_DOCTOR_DEVICE_PROBE=0. Next: unset it and rerun `qmd doctor` to verify GPU/CPU acceleration");
|
|
3848
|
+
nextSteps.push("Unset `QMD_DOCTOR_DEVICE_PROBE` and rerun `qmd doctor` when you want to verify llama.cpp device acceleration.");
|
|
3849
|
+
return;
|
|
3850
|
+
}
|
|
3851
|
+
const crashHint = "Probing native llama backend now. If qmd crashes here, rerun with `QMD_FORCE_CPU=1 qmd doctor` (or `QMD_DOCTOR_DEVICE_PROBE=0 qmd doctor` to skip this probe).";
|
|
3852
|
+
if (process.stdout.isTTY) {
|
|
3853
|
+
process.stdout.write(`${c.dim}${crashHint}${c.reset}`);
|
|
3854
|
+
}
|
|
3855
|
+
try {
|
|
3856
|
+
const device = await getDefaultLlamaCpp().getDeviceInfo({ allowBuild: false });
|
|
3857
|
+
if (process.stdout.isTTY) {
|
|
3858
|
+
process.stdout.write(`\r${" ".repeat(crashHint.length)}\r`);
|
|
3859
|
+
}
|
|
3860
|
+
if (device.gpu) {
|
|
3861
|
+
const gpuLabel = device.gpu === "metal" && process.platform === "darwin"
|
|
3862
|
+
? "metal (macOS Metal backend)"
|
|
3863
|
+
: String(device.gpu);
|
|
3864
|
+
const parts = [`GPU ${gpuLabel}`, `offloading ${device.gpuOffloading ? "enabled" : "disabled"}`];
|
|
3865
|
+
if (device.gpuDevices.length > 0)
|
|
3866
|
+
parts.push(`devices: ${summarizeDeviceNames(device.gpuDevices)}`);
|
|
3867
|
+
if (device.vram)
|
|
3868
|
+
parts.push(`VRAM ${formatBytes(device.vram.free)} free / ${formatBytes(device.vram.total)} total`);
|
|
3869
|
+
parts.push(`${device.cpuCores} CPU math cores`);
|
|
3870
|
+
doctorCheck("device probe", device.gpuOffloading, device.gpuOffloading
|
|
3871
|
+
? parts.join("; ")
|
|
3872
|
+
: `${parts.join("; ")}. Next: check QMD_LLAMA_GPU and llama.cpp backend support`);
|
|
3873
|
+
if (!device.gpuOffloading) {
|
|
3874
|
+
nextSteps.push("GPU was detected but offloading is disabled; check `QMD_LLAMA_GPU=metal|cuda|vulkan` and rerun `qmd doctor`.");
|
|
3875
|
+
}
|
|
3876
|
+
// Surface the darwin residency-set mitigation. libggml-metal's
|
|
3877
|
+
// process-static device dtor asserts on un-expired residency sets
|
|
3878
|
+
// during libc exit() (ggml-org/llama.cpp#22593), producing a giant
|
|
3879
|
+
// stderr backtrace after correct output. The bin/qmd launcher exports
|
|
3880
|
+
// GGML_METAL_NO_RESIDENCY=1 on darwin to skip the assertion entirely.
|
|
3881
|
+
// No measurable perf cost on short-lived CLI calls.
|
|
3882
|
+
if (device.gpu === "metal" && process.platform === "darwin") {
|
|
3883
|
+
if (isDarwinMetalMitigationActive()) {
|
|
3884
|
+
doctorCheck("darwin metal residency", true, "GGML_METAL_NO_RESIDENCY=1 set by launcher; clean process exit (avoids ggml-org/llama.cpp#22593). Opt back in with QMD_METAL_KEEP_RESIDENCY=1 if you run long-lived qmd processes.");
|
|
3885
|
+
}
|
|
3886
|
+
else {
|
|
3887
|
+
doctorCheck("darwin metal residency", false, "residency sets active (QMD_METAL_KEEP_RESIDENCY=1 or launcher bypassed); llama-using commands may dump a libggml-metal backtrace at exit (ggml-org/llama.cpp#22593) even when output succeeded.");
|
|
3888
|
+
nextSteps.push("Unset `QMD_METAL_KEEP_RESIDENCY` so the launcher can disable Metal residency sets; without this, query/vsearch/embed dump a stack trace at exit even on success.");
|
|
3889
|
+
}
|
|
3890
|
+
}
|
|
3891
|
+
}
|
|
3892
|
+
else {
|
|
3893
|
+
const cudaDiagnostic = linuxCudaRuntimeDiagnostic();
|
|
3894
|
+
const diagnosticSuffix = cudaDiagnostic ? ` ${cudaDiagnostic}.` : "";
|
|
3895
|
+
doctorCheck("device probe", false, `running on CPU (${device.cpuCores} math cores).${diagnosticSuffix} Next: install/configure Metal, CUDA, or Vulkan for faster embeddings, or set QMD_FORCE_CPU=1 to make CPU mode explicit`);
|
|
3896
|
+
if (cudaDiagnostic) {
|
|
3897
|
+
nextSteps.push(`${cudaDiagnostic}; install CUDA runtime/cuBLAS libraries or add their directory to LD_LIBRARY_PATH, then rerun \`qmd doctor\`.`);
|
|
3898
|
+
}
|
|
3899
|
+
else {
|
|
3900
|
+
nextSteps.push("Vector operations are running on CPU; install/configure Metal, CUDA, or Vulkan if embedding/query performance is too slow.");
|
|
3901
|
+
}
|
|
3902
|
+
}
|
|
3903
|
+
}
|
|
3904
|
+
catch (error) {
|
|
3905
|
+
if (process.stdout.isTTY) {
|
|
3906
|
+
process.stdout.write(`\r${" ".repeat(crashHint.length)}\r`);
|
|
3907
|
+
}
|
|
3908
|
+
const message = error instanceof Error ? sanitizeDiagnosticMessage(error.message) : sanitizeDiagnosticMessage(String(error));
|
|
3909
|
+
doctorCheck("device probe", false, `probe failed: ${message}. Next: run with QMD_FORCE_CPU=1 to bypass GPU probing, or set QMD_LLAMA_GPU=metal|cuda|vulkan and retry`);
|
|
3910
|
+
nextSteps.push("GPU probe failed; try `QMD_FORCE_CPU=1 qmd doctor` to confirm CPU fallback, then fix GPU drivers/backend if acceleration is expected.");
|
|
3911
|
+
}
|
|
3912
|
+
}
|
|
3913
|
+
async function showDoctor() {
|
|
3914
|
+
const storeInstance = getDoctorStore();
|
|
3915
|
+
const db = storeInstance.db;
|
|
3916
|
+
const pkg = readPackageJson();
|
|
3917
|
+
const activeModels = resolveModelsForCli();
|
|
3918
|
+
let doctorConfig;
|
|
3919
|
+
try {
|
|
3920
|
+
doctorConfig = loadConfig();
|
|
3921
|
+
}
|
|
3922
|
+
catch {
|
|
3923
|
+
// The dedicated index-config check below reports parse errors. Keep the
|
|
3924
|
+
// remaining diagnostics available by falling back to DB/default config.
|
|
3925
|
+
}
|
|
3926
|
+
const doctorEmbedding = resolveEmbeddingConfig({
|
|
3927
|
+
config: doctorConfig,
|
|
3928
|
+
dbConfig: readCanonicalEmbeddingConfig(db),
|
|
3929
|
+
env: process.env,
|
|
3930
|
+
defaultLocalModel: activeModels.embed,
|
|
3931
|
+
}).canonical;
|
|
3932
|
+
const embedModel = doctorEmbedding.model;
|
|
3933
|
+
const nextSteps = [];
|
|
3934
|
+
console.log(`${c.bold}QMD Doctor${c.reset}\n`);
|
|
3935
|
+
console.log(`Index: ${getDbPath()}`);
|
|
3936
|
+
console.log(`Runtime: ${isBun ? "bun:sqlite" : "better-sqlite3"}`);
|
|
3937
|
+
try {
|
|
3938
|
+
const row = db.prepare(`SELECT sqlite_version() AS version`).get();
|
|
3939
|
+
doctorCheck("SQLite runtime", true, row.version);
|
|
3940
|
+
}
|
|
3941
|
+
catch (error) {
|
|
3942
|
+
doctorCheck("SQLite runtime", false, error instanceof Error ? error.message : String(error));
|
|
3943
|
+
}
|
|
3944
|
+
const betterSqliteVersion = pkg.dependencies?.["better-sqlite3"] ?? pkg.devDependencies?.["better-sqlite3"] ?? "not declared";
|
|
3945
|
+
doctorCheck("better-sqlite3 package", true, String(betterSqliteVersion));
|
|
3946
|
+
try {
|
|
3947
|
+
loadSqliteVec(db);
|
|
3948
|
+
const row = db.prepare(`SELECT vec_version() AS version`).get();
|
|
3949
|
+
doctorCheck("sqlite-vec", true, row.version);
|
|
3950
|
+
}
|
|
3951
|
+
catch (error) {
|
|
3952
|
+
doctorCheck("sqlite-vec", false, error instanceof Error ? error.message : String(error));
|
|
3953
|
+
}
|
|
3954
|
+
const configCheck = checkDoctorIndexConfig(nextSteps);
|
|
3955
|
+
if (configCheck.valid) {
|
|
3956
|
+
try {
|
|
3957
|
+
const row = db.prepare(`
|
|
3958
|
+
SELECT value FROM store_config WHERE key = 'config_sync_diagnostic'
|
|
3959
|
+
`).get();
|
|
3960
|
+
const diagnostic = row ? JSON.parse(row.value) : null;
|
|
3961
|
+
if (diagnostic?.reconciled) {
|
|
3962
|
+
const changes = [
|
|
3963
|
+
diagnostic.collections.added.length > 0
|
|
3964
|
+
? `added ${diagnostic.collections.added.join(", ")}`
|
|
3965
|
+
: null,
|
|
3966
|
+
diagnostic.collections.updated.length > 0
|
|
3967
|
+
? `updated ${diagnostic.collections.updated.join(", ")}`
|
|
3968
|
+
: null,
|
|
3969
|
+
diagnostic.collections.removed.length > 0
|
|
3970
|
+
? `removed ${diagnostic.collections.removed.join(", ")}`
|
|
3971
|
+
: null,
|
|
3972
|
+
diagnostic.globalContextUpdated ? "updated global context" : null,
|
|
3973
|
+
].filter((part) => part !== null);
|
|
3974
|
+
doctorCheck("config reconciliation", false, `startup repaired store_collections drift: ${sanitizeDiagnosticMessage(changes.join("; "))}`);
|
|
3975
|
+
}
|
|
3976
|
+
else {
|
|
3977
|
+
doctorCheck("config reconciliation", true, "external config and SQLite metadata are in sync");
|
|
3978
|
+
}
|
|
3979
|
+
}
|
|
3980
|
+
catch (error) {
|
|
3981
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
3982
|
+
doctorCheck("config reconciliation", false, `diagnostic is unreadable: ${sanitizeDiagnosticMessage(message)}`);
|
|
3983
|
+
}
|
|
3984
|
+
}
|
|
3985
|
+
const configModels = configCheck.config?.models ?? {};
|
|
3986
|
+
checkEnvironmentOverrides(activeModels, configModels);
|
|
3987
|
+
checkModelDefaults(activeModels, configModels, doctorEmbedding);
|
|
3988
|
+
checkModelCache(activeModels, nextSteps);
|
|
3989
|
+
const isRemoteEmbed = Boolean(doctorEmbedding.provider === "openai"
|
|
3990
|
+
|| configModels.embed_api_url
|
|
3991
|
+
|| configModels.embed_base_url
|
|
3992
|
+
|| configModels.embed_url
|
|
3993
|
+
|| configModels.embed_api_model);
|
|
3994
|
+
if (isRemoteEmbed) {
|
|
3995
|
+
const embedEndpoint = (doctorEmbedding.provider === "openai" ? doctorEmbedding.baseUrl : undefined)
|
|
3996
|
+
?? configModels.embed_api_url
|
|
3997
|
+
?? configModels.embed_base_url
|
|
3998
|
+
?? configModels.embed_url
|
|
3999
|
+
?? (process.env.OPENAI_BASE_URL || "https://api.openai.com/v1");
|
|
4000
|
+
doctorCheck("openai embedding", true, `${doctorEmbedding.model} (endpoint: ${embedEndpoint})`);
|
|
4001
|
+
}
|
|
4002
|
+
const isRemoteGen = Boolean(configModels.generate_api_url
|
|
4003
|
+
|| configModels.generate_base_url
|
|
4004
|
+
|| configModels.generate_url
|
|
4005
|
+
|| configModels.generate_api_model);
|
|
4006
|
+
if (isRemoteGen) {
|
|
4007
|
+
const genEndpoint = configModels.generate_api_url
|
|
4008
|
+
?? configModels.generate_base_url
|
|
4009
|
+
?? configModels.generate_url
|
|
4010
|
+
?? (process.env.OPENAI_BASE_URL || "https://api.openai.com/v1");
|
|
4011
|
+
const genModel = configModels.generate_api_model ?? activeModels.generate;
|
|
4012
|
+
doctorCheck("query expansion", true, `${genModel} (endpoint: ${genEndpoint})`);
|
|
4013
|
+
}
|
|
4014
|
+
const isRemoteRerank = Boolean(configModels.rerank_api_url
|
|
4015
|
+
|| configModels.rerank_base_url
|
|
4016
|
+
|| configModels.rerank_url
|
|
4017
|
+
|| configModels.rerank_api_model);
|
|
4018
|
+
if (isRemoteRerank) {
|
|
4019
|
+
const rerankEndpoint = configModels.rerank_api_url
|
|
4020
|
+
?? configModels.rerank_base_url
|
|
4021
|
+
?? configModels.rerank_url
|
|
4022
|
+
?? (process.env.OPENAI_BASE_URL || "https://api.openai.com/v1");
|
|
4023
|
+
const rerankModel = configModels.rerank_api_model ?? activeModels.rerank;
|
|
4024
|
+
doctorCheck("reranking model", true, `${rerankModel} (endpoint: ${rerankEndpoint})`);
|
|
4025
|
+
}
|
|
4026
|
+
await runDoctorDeviceChecks(nextSteps);
|
|
4027
|
+
const diagnostics = inspectIndexDiagnostics(db, {
|
|
4028
|
+
fallbackModel: embedModel,
|
|
4029
|
+
provider: doctorEmbedding.provider === "openai"
|
|
4030
|
+
? new UnavailableOpenAIEmbeddingProvider({
|
|
4031
|
+
model: doctorEmbedding.model,
|
|
4032
|
+
dimension: doctorEmbedding.dimension ?? undefined,
|
|
4033
|
+
baseUrl: doctorEmbedding.baseUrl,
|
|
4034
|
+
})
|
|
4035
|
+
: undefined,
|
|
4036
|
+
keyConfigured: Boolean(process.env.OPENAI_API_KEY?.trim()),
|
|
4037
|
+
configuredProvider: {
|
|
4038
|
+
id: doctorEmbedding.provider === "openai" ? "openai" : "local-llama-cpp",
|
|
4039
|
+
remote: doctorEmbedding.provider === "openai",
|
|
4040
|
+
model: doctorEmbedding.model,
|
|
4041
|
+
dimension: doctorEmbedding.dimension,
|
|
4042
|
+
},
|
|
4043
|
+
});
|
|
4044
|
+
const fingerprint = diagnostics.embedding.identity.fullFingerprint;
|
|
4045
|
+
const embeddingHealthy = diagnostics.embedding.build.state === "ready"
|
|
4046
|
+
|| diagnostics.embedding.build.state === "empty";
|
|
4047
|
+
doctorCheck("embedding identity", embeddingHealthy, `${diagnostics.embedding.provider.id ?? "unknown"}/${diagnostics.embedding.provider.model}; state=${diagnostics.embedding.build.state}; fingerprint=${diagnostics.embedding.identity.shortFingerprint ?? "none"}`);
|
|
4048
|
+
const lexicalHealthy = diagnostics.lexical.rebuildReason == null;
|
|
4049
|
+
doctorCheck("CJK lexical channels", lexicalHealthy, `char=${diagnostics.lexical.channels.char}, word=${diagnostics.lexical.channels.word}, bigram=${diagnostics.lexical.channels.bigram}${diagnostics.lexical.rebuildReason ? `; reason=${diagnostics.lexical.rebuildReason}` : ""}`);
|
|
4050
|
+
if (diagnostics.embedding.repairCommand && !embeddingHealthy) {
|
|
4051
|
+
nextSteps.push(`Repair embeddings with \`${diagnostics.embedding.repairCommand}\`.`);
|
|
4052
|
+
}
|
|
4053
|
+
if (diagnostics.lexical.repairCommand && !lexicalHealthy) {
|
|
4054
|
+
nextSteps.push(`Repair CJK lexical indexes with \`${diagnostics.lexical.repairCommand}\`.`);
|
|
4055
|
+
}
|
|
4056
|
+
try {
|
|
4057
|
+
const pending = diagnostics.embedding.chunks.pendingDocuments;
|
|
4058
|
+
doctorCheck("embedding freshness", pending === 0, pending === 0 ? "all active documents match current fingerprint" : `${formatCount(pending)} active documents need embeddings. Next: \`qmd embed\``);
|
|
4059
|
+
if (pending > 0) {
|
|
4060
|
+
nextSteps.push(`Run \`qmd embed\` to generate ${formatCount(pending)} missing/stale document embeddings.`);
|
|
4061
|
+
}
|
|
4062
|
+
}
|
|
4063
|
+
catch (error) {
|
|
4064
|
+
doctorCheck("embedding freshness", false, error instanceof Error ? error.message : String(error));
|
|
4065
|
+
}
|
|
4066
|
+
try {
|
|
4067
|
+
const rows = db.prepare(`
|
|
4068
|
+
SELECT model, embed_fingerprint AS fingerprint, COUNT(DISTINCT hash) AS docs, COUNT(*) AS chunks
|
|
4069
|
+
FROM content_vectors
|
|
4070
|
+
GROUP BY model, embed_fingerprint
|
|
4071
|
+
ORDER BY chunks DESC, model, embed_fingerprint
|
|
4072
|
+
`).all();
|
|
4073
|
+
const uniqueFingerprints = new Set(rows.map(row => row.fingerprint));
|
|
4074
|
+
const offCurrent = rows.filter(row => row.model === embedModel && row.fingerprint !== fingerprint);
|
|
4075
|
+
const ok = rows.length === 0 || (fingerprint != null
|
|
4076
|
+
&& uniqueFingerprints.size === 1
|
|
4077
|
+
&& rows[0]?.fingerprint === fingerprint
|
|
4078
|
+
&& offCurrent.length === 0);
|
|
4079
|
+
const currentDocs = rows
|
|
4080
|
+
.filter(row => row.model === embedModel && row.fingerprint === fingerprint)
|
|
4081
|
+
.reduce((sum, row) => sum + row.docs, 0);
|
|
4082
|
+
const otherDocs = rows.reduce((sum, row) => sum + row.docs, 0) - currentDocs;
|
|
4083
|
+
const groups = rows.map(row => {
|
|
4084
|
+
const label = row.fingerprint === fingerprint ? "current" : (row.fingerprint || "legacy");
|
|
4085
|
+
return `${shortModelName(row.model)}:${label} ${formatCount(row.docs)} docs/${formatCount(row.chunks)} chunks`;
|
|
4086
|
+
}).join("; ");
|
|
4087
|
+
const namedFingerprintRows = rows.filter(row => row.fingerprint);
|
|
4088
|
+
const namedFingerprints = [...new Set(namedFingerprintRows.map(row => row.fingerprint))];
|
|
4089
|
+
if (namedFingerprints.length > 1) {
|
|
4090
|
+
const namedGroups = namedFingerprintRows
|
|
4091
|
+
.map(row => `${row.fingerprint}${row.fingerprint === fingerprint ? " (current)" : ""}: ${shortModelName(row.model)} ${formatCount(row.docs)} docs/${formatCount(row.chunks)} chunks`)
|
|
4092
|
+
.join("; ");
|
|
4093
|
+
doctorCheck("mixed named embedding fingerprints", false, `content_vectors contains ${namedFingerprints.length} named fingerprints: ${namedGroups}. Next: \`qmd embed\` or \`qmd embed --force\``);
|
|
4094
|
+
nextSteps.push("Run `qmd embed` to converge mixed named embedding fingerprints; use `qmd embed --force` if old named fingerprints or vector sample mismatches remain.");
|
|
4095
|
+
}
|
|
4096
|
+
const details = rows.length === 0
|
|
4097
|
+
? `no vectors yet; current fingerprint ${fingerprint ?? "unknown until dimension is known"}`
|
|
4098
|
+
: ok
|
|
4099
|
+
? `${formatCount(currentDocs)} docs on current fingerprint (${fingerprint})`
|
|
4100
|
+
: `${formatCount(currentDocs)} docs current, ${formatCount(otherDocs)} docs legacy/stale. ${groups}. Next: \`qmd embed\``;
|
|
4101
|
+
doctorCheck("embedding fingerprints", ok, details);
|
|
4102
|
+
if (!ok) {
|
|
4103
|
+
nextSteps.push("Run `qmd embed` to migrate active documents to the current embedding fingerprint; use `qmd embed --force` if vector samples still fail afterward.");
|
|
4104
|
+
}
|
|
4105
|
+
}
|
|
4106
|
+
catch (error) {
|
|
4107
|
+
doctorCheck("embedding fingerprints", false, error instanceof Error ? error.message : String(error));
|
|
4108
|
+
}
|
|
4109
|
+
if (diagnostics.embedding.provider.remote) {
|
|
4110
|
+
doctorCheck("embedding vector sample", true, "skipped for remote provider; doctor sends no remote embedding request");
|
|
4111
|
+
}
|
|
4112
|
+
else if (fingerprint != null) {
|
|
4113
|
+
try {
|
|
4114
|
+
const vectorSample = await checkEmbeddingVectorSamples(db, embedModel, fingerprint);
|
|
4115
|
+
doctorCheck("embedding vector sample", vectorSample.ok, vectorSample.details);
|
|
4116
|
+
if (!vectorSample.ok) {
|
|
4117
|
+
nextSteps.push("Run `qmd embed --force` to rebuild existing vectors that no longer reproduce under the current embedding pipeline.");
|
|
4118
|
+
}
|
|
4119
|
+
}
|
|
4120
|
+
catch (error) {
|
|
4121
|
+
const message = error instanceof Error ? sanitizeDiagnosticMessage(error.message) : sanitizeDiagnosticMessage(String(error));
|
|
4122
|
+
doctorCheck("embedding vector sample", false, `${message}; rebuild with \`qmd embed --force\``);
|
|
4123
|
+
nextSteps.push("Run `qmd embed --force` to rebuild existing vectors, then rerun `qmd doctor`.");
|
|
4124
|
+
}
|
|
4125
|
+
}
|
|
4126
|
+
else {
|
|
4127
|
+
doctorCheck("embedding vector sample", true, "skipped until the local embedding dimension is known");
|
|
4128
|
+
}
|
|
4129
|
+
const steps = normalizedDoctorNextSteps(nextSteps);
|
|
4130
|
+
if (steps.length > 0) {
|
|
4131
|
+
console.log(`\n${c.bold}Recommended next step${steps.length === 1 ? "" : "s"}${c.reset}`);
|
|
4132
|
+
for (const step of steps) {
|
|
4133
|
+
console.log(` - ${step}`);
|
|
4134
|
+
}
|
|
4135
|
+
}
|
|
4136
|
+
closeDb();
|
|
4137
|
+
}
|
|
4138
|
+
function printDoctorHint() {
|
|
4139
|
+
console.error("If qmd still behaves unexpectedly, run 'qmd doctor' for diagnostics.");
|
|
4140
|
+
}
|
|
4141
|
+
function exitWithError(error, code = 1) {
|
|
4142
|
+
console.error(error instanceof Error ? error.message : String(error));
|
|
4143
|
+
printDoctorHint();
|
|
4144
|
+
process.exit(code);
|
|
4145
|
+
}
|
|
4146
|
+
function readPackageJson() {
|
|
4147
|
+
const scriptDir = dirname(fileURLToPath(import.meta.url));
|
|
4148
|
+
const pkgPath = resolve(scriptDir, "..", "..", "package.json");
|
|
4149
|
+
return JSON.parse(readFileSync(pkgPath, "utf-8"));
|
|
4150
|
+
}
|
|
4151
|
+
function showVersion() {
|
|
4152
|
+
const scriptDir = dirname(fileURLToPath(import.meta.url));
|
|
4153
|
+
const pkg = readPackageJson();
|
|
4154
|
+
// Prefer the commit stamped in at build time; fall back to the checkout's
|
|
4155
|
+
// HEAD only when this really is qmd's own checkout. See src/cli/version.ts.
|
|
4156
|
+
const commit = resolveCommit(scriptDir, pathResolve(scriptDir, "..", ".."));
|
|
4157
|
+
const versionStr = commit ? `${pkg.version} (${commit})` : pkg.version;
|
|
4158
|
+
console.log(`qmd ${versionStr}`);
|
|
4159
|
+
}
|
|
4160
|
+
// Main CLI - only run if this is the main module
|
|
4161
|
+
const __filename = fileURLToPath(import.meta.url);
|
|
4162
|
+
const argv1 = process.argv[1];
|
|
4163
|
+
const isMain = argv1 === __filename
|
|
4164
|
+
|| argv1?.endsWith("/qmd.ts")
|
|
4165
|
+
|| argv1?.endsWith("/qmd.js")
|
|
4166
|
+
|| (argv1 != null && realpathSync(argv1) === __filename);
|
|
4167
|
+
if (isMain) {
|
|
4168
|
+
// Flip to production mode only when this module is executed as the CLI
|
|
4169
|
+
// entrypoint, not when imported for its exports. Tests must set INDEX_PATH
|
|
4170
|
+
// or use createStore() with an explicit path.
|
|
4171
|
+
enableProductionMode();
|
|
4172
|
+
const cli = parseCLI();
|
|
4173
|
+
if (cli.values.version) {
|
|
4174
|
+
showVersion();
|
|
4175
|
+
process.exit(0);
|
|
4176
|
+
}
|
|
4177
|
+
if (cli.values.skill) {
|
|
4178
|
+
showSkill();
|
|
4179
|
+
process.exit(0);
|
|
4180
|
+
}
|
|
4181
|
+
if (cli.values.help && cli.command === "skill") {
|
|
4182
|
+
console.log("Usage: qmd skill <show|install> [options]");
|
|
4183
|
+
console.log("");
|
|
4184
|
+
console.log("Commands:");
|
|
4185
|
+
console.log(" show Print the QMD skill");
|
|
4186
|
+
console.log(" install Install QMD skill into ./.agents/skills/qmd");
|
|
4187
|
+
console.log("");
|
|
4188
|
+
console.log("Options:");
|
|
4189
|
+
console.log(" --global Install into ~/.agents/skills/qmd");
|
|
4190
|
+
console.log(" --yes Also create the .claude/skills/qmd symlink");
|
|
4191
|
+
console.log(" -f, --force Replace existing install or symlink");
|
|
4192
|
+
process.exit(0);
|
|
4193
|
+
}
|
|
4194
|
+
if (!cli.command || cli.values.help) {
|
|
4195
|
+
showHelp();
|
|
4196
|
+
process.exit(cli.values.help ? 0 : 1);
|
|
4197
|
+
}
|
|
4198
|
+
switch (cli.command) {
|
|
4199
|
+
case "context": {
|
|
4200
|
+
const subcommand = cli.args[0];
|
|
4201
|
+
if (!subcommand) {
|
|
4202
|
+
console.error("Usage: qmd context <add|list|rm>");
|
|
4203
|
+
console.error("");
|
|
4204
|
+
console.error("Commands:");
|
|
4205
|
+
console.error(" qmd context add [path] \"text\" - Add context (defaults to current dir)");
|
|
4206
|
+
console.error(" qmd context add / \"text\" - Add global context to all collections");
|
|
4207
|
+
console.error(" qmd context list - List all contexts");
|
|
4208
|
+
console.error(" qmd context rm <path> - Remove context");
|
|
4209
|
+
process.exit(1);
|
|
4210
|
+
}
|
|
4211
|
+
switch (subcommand) {
|
|
4212
|
+
case "add": {
|
|
4213
|
+
if (cli.args.length < 2) {
|
|
4214
|
+
console.error("Usage: qmd context add [path] \"text\"");
|
|
4215
|
+
console.error("");
|
|
4216
|
+
console.error("Examples:");
|
|
4217
|
+
console.error(" qmd context add \"Context for current directory\"");
|
|
4218
|
+
console.error(" qmd context add . \"Context for current directory\"");
|
|
4219
|
+
console.error(" qmd context add /subfolder \"Context for subfolder\"");
|
|
4220
|
+
console.error(" qmd context add / \"Global context for all collections\"");
|
|
4221
|
+
console.error("");
|
|
4222
|
+
console.error(" Using virtual paths:");
|
|
4223
|
+
console.error(" qmd context add qmd://journals/ \"Context for entire journals collection\"");
|
|
4224
|
+
console.error(" qmd context add qmd://journals/2024 \"Context for 2024 journals\"");
|
|
4225
|
+
process.exit(1);
|
|
4226
|
+
}
|
|
4227
|
+
let pathArg;
|
|
4228
|
+
let contextText;
|
|
4229
|
+
// Check if first arg looks like a path or if it's the context text
|
|
4230
|
+
const firstArg = cli.args[1] || '';
|
|
4231
|
+
const secondArg = cli.args[2];
|
|
4232
|
+
if (secondArg) {
|
|
4233
|
+
// Two args: path + context
|
|
4234
|
+
pathArg = firstArg;
|
|
4235
|
+
contextText = cli.args.slice(2).join(" ");
|
|
4236
|
+
}
|
|
4237
|
+
else {
|
|
4238
|
+
// One arg: context only (use current directory)
|
|
4239
|
+
pathArg = undefined;
|
|
4240
|
+
contextText = firstArg;
|
|
4241
|
+
}
|
|
4242
|
+
await contextAdd(pathArg, contextText);
|
|
4243
|
+
break;
|
|
4244
|
+
}
|
|
4245
|
+
case "list": {
|
|
4246
|
+
contextList();
|
|
4247
|
+
break;
|
|
4248
|
+
}
|
|
4249
|
+
case "rm":
|
|
4250
|
+
case "remove": {
|
|
4251
|
+
if (cli.args.length < 2 || !cli.args[1]) {
|
|
4252
|
+
console.error("Usage: qmd context rm <path>");
|
|
4253
|
+
console.error("Examples:");
|
|
4254
|
+
console.error(" qmd context rm /");
|
|
4255
|
+
console.error(" qmd context rm qmd://journals/2024");
|
|
4256
|
+
process.exit(1);
|
|
4257
|
+
}
|
|
4258
|
+
contextRemove(cli.args[1]);
|
|
4259
|
+
break;
|
|
4260
|
+
}
|
|
4261
|
+
default:
|
|
4262
|
+
console.error(`Unknown subcommand: ${subcommand}`);
|
|
4263
|
+
console.error("Available: add, list, rm");
|
|
4264
|
+
process.exit(1);
|
|
4265
|
+
}
|
|
4266
|
+
break;
|
|
4267
|
+
}
|
|
4268
|
+
case "get": {
|
|
4269
|
+
if (!cli.args[0]) {
|
|
4270
|
+
console.error("Usage: qmd get <filepath>[:from[:count]] [--from <line>] [-l <lines>] [--no-line-numbers] [--full-path]");
|
|
4271
|
+
process.exit(1);
|
|
4272
|
+
}
|
|
4273
|
+
const fromLine = cli.values.from ? parseInt(cli.values.from, 10) : undefined;
|
|
4274
|
+
const maxLines = cli.values.l ? parseInt(cli.values.l, 10) : undefined;
|
|
4275
|
+
// Line numbers default ON for get; opt out with --no-line-numbers.
|
|
4276
|
+
const getLineNumbers = !cli.values["no-line-numbers"];
|
|
4277
|
+
getDocument(cli.args[0], fromLine, maxLines, getLineNumbers, !!cli.values["full-path"]);
|
|
4278
|
+
break;
|
|
4279
|
+
}
|
|
4280
|
+
case "multi-get": {
|
|
4281
|
+
if (!cli.args[0]) {
|
|
4282
|
+
console.error("Usage: qmd multi-get <pattern> [-l <lines>] [--max-bytes <bytes>] [--no-line-numbers] [--full-path] [--format json|csv|md|xml|files]");
|
|
4283
|
+
console.error(" pattern: glob (e.g., 'journals/2025-05*.md') or comma-separated list");
|
|
4284
|
+
process.exit(1);
|
|
4285
|
+
}
|
|
4286
|
+
const maxLinesMulti = cli.values.l ? parseInt(cli.values.l, 10) : undefined;
|
|
4287
|
+
const maxBytes = cli.values["max-bytes"] ? parseInt(cli.values["max-bytes"], 10) : DEFAULT_MULTI_GET_MAX_BYTES;
|
|
4288
|
+
// Line numbers default ON for multi-get; opt out with --no-line-numbers.
|
|
4289
|
+
const mgLineNumbers = !cli.values["no-line-numbers"];
|
|
4290
|
+
multiGet(cli.args[0], maxLinesMulti, maxBytes, cli.opts.format, mgLineNumbers, !!cli.values["full-path"]);
|
|
4291
|
+
break;
|
|
4292
|
+
}
|
|
4293
|
+
case "ls": {
|
|
4294
|
+
listFiles(cli.args[0]);
|
|
4295
|
+
break;
|
|
4296
|
+
}
|
|
4297
|
+
case "collection": {
|
|
4298
|
+
const subcommand = cli.args[0];
|
|
4299
|
+
switch (subcommand) {
|
|
4300
|
+
case "list": {
|
|
4301
|
+
collectionList();
|
|
4302
|
+
break;
|
|
4303
|
+
}
|
|
4304
|
+
case "add": {
|
|
4305
|
+
const pwd = cli.args[1];
|
|
4306
|
+
if (!pwd) {
|
|
4307
|
+
console.error("Usage: qmd collection add <path> [--name NAME] [--mask GLOB]");
|
|
4308
|
+
console.error(" Pass '.' to index the current directory.");
|
|
4309
|
+
console.error(" --mask / --glob: glob (default **/*.md), brace group, or comma-separated list");
|
|
4310
|
+
process.exit(1);
|
|
4311
|
+
}
|
|
4312
|
+
const resolvedPwd = pwd === '.' ? getPwd() : getRealPath(resolve(pwd));
|
|
4313
|
+
const globPattern = collectionGlobFromCli(cli.values);
|
|
4314
|
+
const name = cli.values.name;
|
|
4315
|
+
if (!existsSync(resolvedPwd)) {
|
|
4316
|
+
console.error(`${c.yellow}Collection path does not exist.${c.reset}`);
|
|
4317
|
+
console.error(` Received: ${pwd}`);
|
|
4318
|
+
console.error(` Resolved: ${resolvedPwd}`);
|
|
4319
|
+
console.error("Provide an existing directory and run 'qmd collection add <path>' again.");
|
|
4320
|
+
process.exit(1);
|
|
4321
|
+
}
|
|
4322
|
+
if (!statSync(resolvedPwd).isDirectory()) {
|
|
4323
|
+
console.error(`${c.yellow}Collection path is not a directory.${c.reset}`);
|
|
4324
|
+
console.error(` Received: ${pwd}`);
|
|
4325
|
+
console.error(` Resolved: ${resolvedPwd}`);
|
|
4326
|
+
console.error("Choose a directory and run 'qmd collection add <path>' again.");
|
|
4327
|
+
process.exit(1);
|
|
4328
|
+
}
|
|
4329
|
+
await collectionAdd(resolvedPwd, globPattern, name);
|
|
4330
|
+
break;
|
|
4331
|
+
}
|
|
4332
|
+
case "remove":
|
|
4333
|
+
case "rm": {
|
|
4334
|
+
if (!cli.args[1]) {
|
|
4335
|
+
console.error("Usage: qmd collection remove <name>");
|
|
4336
|
+
console.error(" Use 'qmd collection list' to see available collections");
|
|
4337
|
+
process.exit(1);
|
|
4338
|
+
}
|
|
4339
|
+
collectionRemove(cli.args[1]);
|
|
4340
|
+
break;
|
|
4341
|
+
}
|
|
4342
|
+
case "rename":
|
|
4343
|
+
case "mv": {
|
|
4344
|
+
if (!cli.args[1] || !cli.args[2]) {
|
|
4345
|
+
console.error("Usage: qmd collection rename <old-name> <new-name>");
|
|
4346
|
+
console.error(" Use 'qmd collection list' to see available collections");
|
|
4347
|
+
process.exit(1);
|
|
4348
|
+
}
|
|
4349
|
+
collectionRename(cli.args[1], cli.args[2]);
|
|
4350
|
+
break;
|
|
4351
|
+
}
|
|
4352
|
+
case "set-update":
|
|
4353
|
+
case "update-cmd": {
|
|
4354
|
+
const name = cli.args[1];
|
|
4355
|
+
const cmd = cli.args.slice(2).join(' ') || null;
|
|
4356
|
+
if (!name) {
|
|
4357
|
+
console.error("Usage: qmd collection update-cmd <name> [command]");
|
|
4358
|
+
console.error(" Set the command to run before indexing (e.g., 'git pull')");
|
|
4359
|
+
console.error(" Omit command to clear it");
|
|
4360
|
+
process.exit(1);
|
|
4361
|
+
}
|
|
4362
|
+
const { updateCollectionSettings, getCollection } = await import("../collections.js");
|
|
4363
|
+
const col = getCollection(name);
|
|
4364
|
+
if (!col) {
|
|
4365
|
+
console.error(`Collection not found: ${name}`);
|
|
4366
|
+
process.exit(1);
|
|
4367
|
+
}
|
|
4368
|
+
updateCollectionSettings(name, { update: cmd });
|
|
4369
|
+
resyncConfig();
|
|
4370
|
+
// The user just typed this command, so it needs no separate approval;
|
|
4371
|
+
// re-record so the digest covers the new hook set (#886).
|
|
4372
|
+
trustCurrentConfig();
|
|
4373
|
+
if (cmd) {
|
|
4374
|
+
console.log(`✓ Set update command for '${name}': ${cmd}`);
|
|
4375
|
+
}
|
|
4376
|
+
else {
|
|
4377
|
+
console.log(`✓ Cleared update command for '${name}'`);
|
|
4378
|
+
}
|
|
4379
|
+
break;
|
|
4380
|
+
}
|
|
4381
|
+
case "include":
|
|
4382
|
+
case "exclude": {
|
|
4383
|
+
const name = cli.args[1];
|
|
4384
|
+
if (!name) {
|
|
4385
|
+
console.error(`Usage: qmd collection ${subcommand} <name>`);
|
|
4386
|
+
console.error(` ${subcommand === 'include' ? 'Include' : 'Exclude'} collection in default queries`);
|
|
4387
|
+
process.exit(1);
|
|
4388
|
+
}
|
|
4389
|
+
const { updateCollectionSettings, getCollection } = await import("../collections.js");
|
|
4390
|
+
const col = getCollection(name);
|
|
4391
|
+
if (!col) {
|
|
4392
|
+
console.error(`Collection not found: ${name}`);
|
|
4393
|
+
process.exit(1);
|
|
4394
|
+
}
|
|
4395
|
+
const include = subcommand === 'include';
|
|
4396
|
+
updateCollectionSettings(name, { includeByDefault: include });
|
|
4397
|
+
resyncConfig();
|
|
4398
|
+
console.log(`✓ Collection '${name}' ${include ? 'included in' : 'excluded from'} default queries`);
|
|
4399
|
+
break;
|
|
4400
|
+
}
|
|
4401
|
+
case "show":
|
|
4402
|
+
case "info": {
|
|
4403
|
+
const name = cli.args[1];
|
|
4404
|
+
if (!name) {
|
|
4405
|
+
console.error("Usage: qmd collection show <name>");
|
|
4406
|
+
process.exit(1);
|
|
4407
|
+
}
|
|
4408
|
+
const { getCollection } = await import("../collections.js");
|
|
4409
|
+
const col = getCollection(name);
|
|
4410
|
+
if (!col) {
|
|
4411
|
+
console.error(`Collection not found: ${name}`);
|
|
4412
|
+
process.exit(1);
|
|
4413
|
+
}
|
|
4414
|
+
console.log(`Collection: ${name}`);
|
|
4415
|
+
console.log(` Path: ${col.path}`);
|
|
4416
|
+
console.log(` Pattern: ${col.pattern}`);
|
|
4417
|
+
console.log(` Include: ${col.includeByDefault !== false ? 'yes (default)' : 'no'}`);
|
|
4418
|
+
if (col.update) {
|
|
4419
|
+
console.log(` Update: ${col.update}`);
|
|
4420
|
+
}
|
|
4421
|
+
if (col.context) {
|
|
4422
|
+
const ctxCount = Object.keys(col.context).length;
|
|
4423
|
+
console.log(` Contexts: ${ctxCount}`);
|
|
4424
|
+
}
|
|
4425
|
+
break;
|
|
4426
|
+
}
|
|
4427
|
+
case "help":
|
|
4428
|
+
case undefined: {
|
|
4429
|
+
console.log("Usage: qmd collection <command> [options]");
|
|
4430
|
+
console.log("");
|
|
4431
|
+
console.log("Commands:");
|
|
4432
|
+
console.log(" list List all collections");
|
|
4433
|
+
console.log(" add <path> [--name NAME] [--mask|--glob GLOB] Add a collection");
|
|
4434
|
+
console.log(" remove <name> Remove a collection");
|
|
4435
|
+
console.log(" rename <old> <new> Rename a collection");
|
|
4436
|
+
console.log(" show <name> Show collection details");
|
|
4437
|
+
console.log(" update-cmd <name> [cmd] Set pre-update command (e.g., 'git pull')");
|
|
4438
|
+
console.log(" include <name> Include in default queries");
|
|
4439
|
+
console.log(" exclude <name> Exclude from default queries");
|
|
4440
|
+
console.log("");
|
|
4441
|
+
console.log("Examples:");
|
|
4442
|
+
console.log(" qmd collection add ~/notes --name notes");
|
|
4443
|
+
console.log(" qmd collection add ~/notes --name notes --mask 'a.md,journals/*.md'");
|
|
4444
|
+
console.log(" qmd collection update-cmd brain 'git pull'");
|
|
4445
|
+
console.log(" qmd collection exclude archive");
|
|
4446
|
+
process.exit(0);
|
|
4447
|
+
}
|
|
4448
|
+
default:
|
|
4449
|
+
console.error(`Unknown subcommand: ${subcommand}`);
|
|
4450
|
+
console.error("Run 'qmd collection help' for usage");
|
|
4451
|
+
printDoctorHint();
|
|
4452
|
+
process.exit(1);
|
|
4453
|
+
}
|
|
4454
|
+
break;
|
|
4455
|
+
}
|
|
4456
|
+
case "init":
|
|
4457
|
+
try {
|
|
4458
|
+
initLocalIndex();
|
|
4459
|
+
}
|
|
4460
|
+
catch (error) {
|
|
4461
|
+
exitWithError(error);
|
|
4462
|
+
}
|
|
4463
|
+
break;
|
|
4464
|
+
case "status":
|
|
4465
|
+
await showStatus();
|
|
4466
|
+
break;
|
|
4467
|
+
case "doctor":
|
|
4468
|
+
await showDoctor();
|
|
4469
|
+
break;
|
|
4470
|
+
case "update":
|
|
4471
|
+
await updateCollections();
|
|
4472
|
+
break;
|
|
4473
|
+
case "trust":
|
|
4474
|
+
manageTrust(cli.args[0]);
|
|
4475
|
+
break;
|
|
4476
|
+
case "embed":
|
|
4477
|
+
try {
|
|
4478
|
+
await resolveLocalConfigTrust();
|
|
4479
|
+
const maxDocsPerBatch = parseEmbedBatchOption("maxDocsPerBatch", cli.values["max-docs-per-batch"]);
|
|
4480
|
+
const maxBatchMb = parseEmbedBatchOption("maxBatchBytes", cli.values["max-batch-mb"]);
|
|
4481
|
+
const embedChunkStrategy = parseChunkStrategy(cli.values["chunk-strategy"]);
|
|
4482
|
+
const embedMaxDurationMs = parseEmbedTimeoutOption(cli.values["timeout"]);
|
|
4483
|
+
// Validate -c against configured collections before dispatching, so a
|
|
4484
|
+
// typo errors with "Collection not found: X" instead of silently
|
|
4485
|
+
// reporting success because no pending docs match a nonexistent name.
|
|
4486
|
+
// embed operates on a single collection; only the first value is used.
|
|
4487
|
+
const embedValidatedCollections = resolveCollectionFilter(cli.opts.collection, false);
|
|
4488
|
+
const embedCollection = embedValidatedCollections[0];
|
|
4489
|
+
await vectorIndex(resolveEmbedModelForCli(), !!cli.values.force, {
|
|
4490
|
+
maxDocsPerBatch,
|
|
4491
|
+
maxBatchBytes: maxBatchMb === undefined ? undefined : maxBatchMb * 1024 * 1024,
|
|
4492
|
+
chunkStrategy: embedChunkStrategy,
|
|
4493
|
+
collection: embedCollection,
|
|
4494
|
+
maxDurationMs: embedMaxDurationMs,
|
|
4495
|
+
});
|
|
4496
|
+
}
|
|
4497
|
+
catch (error) {
|
|
4498
|
+
exitWithError(error);
|
|
4499
|
+
}
|
|
4500
|
+
break;
|
|
4501
|
+
case "pull": {
|
|
4502
|
+
await resolveLocalConfigTrust();
|
|
4503
|
+
const refresh = cli.values.refresh === undefined ? false : Boolean(cli.values.refresh);
|
|
4504
|
+
const activeModels = resolveModelsForRuntime();
|
|
4505
|
+
const models = [
|
|
4506
|
+
activeModels.embed,
|
|
4507
|
+
activeModels.generate,
|
|
4508
|
+
activeModels.rerank,
|
|
4509
|
+
];
|
|
4510
|
+
console.log(`${c.bold}Pulling models${c.reset}`);
|
|
4511
|
+
const results = await pullModels(models, {
|
|
4512
|
+
refresh,
|
|
4513
|
+
cacheDir: DEFAULT_MODEL_CACHE_DIR,
|
|
4514
|
+
cli: Boolean(cli.values.progress),
|
|
4515
|
+
});
|
|
4516
|
+
for (const result of results) {
|
|
4517
|
+
const size = formatBytes(result.sizeBytes);
|
|
4518
|
+
const note = result.refreshed ? "refreshed" : "cached/checked";
|
|
4519
|
+
console.log(`- ${result.model} -> ${result.path} (${size}, ${note})`);
|
|
4520
|
+
}
|
|
4521
|
+
break;
|
|
4522
|
+
}
|
|
4523
|
+
case "search":
|
|
4524
|
+
if (!cli.query) {
|
|
4525
|
+
console.error("Usage: qmd search [options] <query>");
|
|
4526
|
+
process.exit(1);
|
|
4527
|
+
}
|
|
4528
|
+
search(cli.query, cli.opts);
|
|
4529
|
+
break;
|
|
4530
|
+
case "vsearch":
|
|
4531
|
+
case "vector-search": // undocumented alias
|
|
4532
|
+
if (!cli.query) {
|
|
4533
|
+
console.error("Usage: qmd vsearch [options] <query>");
|
|
4534
|
+
process.exit(1);
|
|
4535
|
+
}
|
|
4536
|
+
// Default min-score for vector search is 0.3
|
|
4537
|
+
if (!cli.values["min-score"]) {
|
|
4538
|
+
cli.opts.minScore = 0.3;
|
|
4539
|
+
}
|
|
4540
|
+
await resolveLocalConfigTrust();
|
|
4541
|
+
await vectorSearch(cli.query, cli.opts);
|
|
4542
|
+
break;
|
|
4543
|
+
case "query":
|
|
4544
|
+
case "deep-search": // undocumented alias
|
|
4545
|
+
if (!cli.query) {
|
|
4546
|
+
console.error("Usage: qmd query [options] <query>");
|
|
4547
|
+
process.exit(1);
|
|
4548
|
+
}
|
|
4549
|
+
await resolveLocalConfigTrust();
|
|
4550
|
+
await querySearch(cli.query, cli.opts);
|
|
4551
|
+
break;
|
|
4552
|
+
case "bench": {
|
|
4553
|
+
const fixturePath = cli.args[0];
|
|
4554
|
+
if (!fixturePath) {
|
|
4555
|
+
console.error("Usage: qmd bench <fixture.json> [--json] [-c collection]");
|
|
4556
|
+
console.error("");
|
|
4557
|
+
console.error("Run search quality benchmarks against a fixture file.");
|
|
4558
|
+
console.error("See src/bench/fixtures/example.json for the fixture format.");
|
|
4559
|
+
process.exit(1);
|
|
4560
|
+
}
|
|
4561
|
+
const { runBenchmark } = await import("../bench/bench.js");
|
|
4562
|
+
const benchCollection = cli.opts.collection;
|
|
4563
|
+
try {
|
|
4564
|
+
await runBenchmark(fixturePath, {
|
|
4565
|
+
json: !!cli.values.json,
|
|
4566
|
+
collection: Array.isArray(benchCollection) ? benchCollection[0] : benchCollection,
|
|
4567
|
+
dbPath: getDbPath(),
|
|
4568
|
+
configPath: configExists() ? getConfigPath() : undefined,
|
|
4569
|
+
});
|
|
4570
|
+
}
|
|
4571
|
+
catch (error) {
|
|
4572
|
+
exitWithError(error);
|
|
4573
|
+
}
|
|
4574
|
+
break;
|
|
4575
|
+
}
|
|
4576
|
+
case "mcp": {
|
|
4577
|
+
const sub = cli.args[0]; // stop | status | undefined
|
|
4578
|
+
// Cache dir for PID/log files — scoped per --index so named daemons
|
|
4579
|
+
// do not collide with the default index (#772).
|
|
4580
|
+
const { cacheDir, pidPath, logPath } = mcpDaemonPaths();
|
|
4581
|
+
// Subcommands take priority over flags
|
|
4582
|
+
if (sub === "stop") {
|
|
4583
|
+
if (!existsSync(pidPath)) {
|
|
4584
|
+
console.log("Not running (no PID file).");
|
|
4585
|
+
process.exit(0);
|
|
4586
|
+
}
|
|
4587
|
+
const pid = parseInt(readFileSync(pidPath, "utf-8").trim());
|
|
4588
|
+
if (!isQmdMcpPid(pid)) {
|
|
4589
|
+
try {
|
|
4590
|
+
unlinkSync(pidPath);
|
|
4591
|
+
}
|
|
4592
|
+
catch { /* ignore */ }
|
|
4593
|
+
console.log("Cleaned up stale PID file (server was not running).");
|
|
4594
|
+
process.exit(0);
|
|
4595
|
+
}
|
|
4596
|
+
try {
|
|
4597
|
+
process.kill(pid, "SIGTERM");
|
|
4598
|
+
unlinkSync(pidPath);
|
|
4599
|
+
console.log(`Stopped QMD MCP server (PID ${pid}).`);
|
|
4600
|
+
}
|
|
4601
|
+
catch {
|
|
4602
|
+
try {
|
|
4603
|
+
unlinkSync(pidPath);
|
|
4604
|
+
}
|
|
4605
|
+
catch { /* ignore */ }
|
|
4606
|
+
console.log("Cleaned up stale PID file (server was not running).");
|
|
4607
|
+
}
|
|
4608
|
+
process.exit(0);
|
|
4609
|
+
}
|
|
4610
|
+
if (cli.values.http) {
|
|
4611
|
+
const port = Number(cli.values.port) || 8181;
|
|
4612
|
+
// --host overrides the default localhost bind; QMD_HOST env is the
|
|
4613
|
+
// fallback (resolved in startMcpHttpServer). Use "0.0.0.0" to accept
|
|
4614
|
+
// off-host connections, e.g. a container liveness probe.
|
|
4615
|
+
const host = cli.values.host ? String(cli.values.host) : undefined;
|
|
4616
|
+
if (cli.values.daemon) {
|
|
4617
|
+
// Guard: check if already running (identity-checked — recycled PIDs are stale)
|
|
4618
|
+
if (existsSync(pidPath)) {
|
|
4619
|
+
const existingPid = parseInt(readFileSync(pidPath, "utf-8").trim());
|
|
4620
|
+
if (isQmdMcpPid(existingPid)) {
|
|
4621
|
+
console.error(`Already running (PID ${existingPid}). Run 'qmd mcp stop' first.`);
|
|
4622
|
+
process.exit(1);
|
|
4623
|
+
}
|
|
4624
|
+
// Stale or recycled PID file — remove and continue
|
|
4625
|
+
try {
|
|
4626
|
+
unlinkSync(pidPath);
|
|
4627
|
+
}
|
|
4628
|
+
catch { /* ignore */ }
|
|
4629
|
+
}
|
|
4630
|
+
mkdirSync(cacheDir, { recursive: true });
|
|
4631
|
+
const logFd = openSync(logPath, "w"); // truncate — fresh log per daemon run
|
|
4632
|
+
const selfPath = fileURLToPath(import.meta.url);
|
|
4633
|
+
const indexArgs = cli.values.index ? ["--index", String(cli.values.index)] : [];
|
|
4634
|
+
const hostArgs = host ? ["--host", host] : [];
|
|
4635
|
+
const isBunRuntime = typeof process.versions.bun === "string";
|
|
4636
|
+
const spawnArgs = selfPath.endsWith(".ts")
|
|
4637
|
+
? isBunRuntime
|
|
4638
|
+
? [selfPath, ...indexArgs, "mcp", "--http", "--port", String(port), ...hostArgs]
|
|
4639
|
+
: ["--import", pathJoin(dirname(selfPath), "..", "..", "node_modules", "tsx", "dist", "esm", "index.mjs"), selfPath, ...indexArgs, "mcp", "--http", "--port", String(port), ...hostArgs]
|
|
4640
|
+
: [selfPath, ...indexArgs, "mcp", "--http", "--port", String(port), ...hostArgs];
|
|
4641
|
+
const child = nodeSpawn(process.execPath, spawnArgs, {
|
|
4642
|
+
stdio: ["ignore", logFd, logFd],
|
|
4643
|
+
detached: true,
|
|
4644
|
+
env: {
|
|
4645
|
+
...process.env,
|
|
4646
|
+
// Explicit resolved DB path so the child does not depend on
|
|
4647
|
+
// re-parsing --index (and cannot inherit a stale INDEX_PATH).
|
|
4648
|
+
INDEX_PATH: getDbPath(),
|
|
4649
|
+
},
|
|
4650
|
+
});
|
|
4651
|
+
child.unref();
|
|
4652
|
+
closeSync(logFd); // parent's copy; child inherited the fd
|
|
4653
|
+
writeFileSync(pidPath, String(child.pid));
|
|
4654
|
+
console.log(`Started on http://${host ?? "localhost"}:${port}/mcp (PID ${child.pid})`);
|
|
4655
|
+
console.log(`Logs: ${logPath}`);
|
|
4656
|
+
process.exit(0);
|
|
4657
|
+
}
|
|
4658
|
+
// Foreground HTTP mode — remove top-level cursor handlers so the
|
|
4659
|
+
// async cleanup handlers in startMcpHttpServer actually run.
|
|
4660
|
+
process.removeAllListeners("SIGTERM");
|
|
4661
|
+
process.removeAllListeners("SIGINT");
|
|
4662
|
+
// Best-effort: if this process owns the daemon pidfile, unlink on exit
|
|
4663
|
+
// (covers SIGTERM/SIGINT via startMcpHttpServer's process.exit).
|
|
4664
|
+
const unlinkOwnPidfile = () => {
|
|
4665
|
+
try {
|
|
4666
|
+
if (!existsSync(pidPath))
|
|
4667
|
+
return;
|
|
4668
|
+
const written = parseInt(readFileSync(pidPath, "utf-8").trim());
|
|
4669
|
+
if (written === process.pid)
|
|
4670
|
+
unlinkSync(pidPath);
|
|
4671
|
+
}
|
|
4672
|
+
catch { /* ignore */ }
|
|
4673
|
+
};
|
|
4674
|
+
process.on("exit", unlinkOwnPidfile);
|
|
4675
|
+
const { startMcpHttpServer } = await import("../mcp/server.js");
|
|
4676
|
+
try {
|
|
4677
|
+
await startMcpHttpServer(port, { dbPath: getDbPath(), host });
|
|
4678
|
+
}
|
|
4679
|
+
catch (e) {
|
|
4680
|
+
if (typeof e === "object" && e !== null && "code" in e && e.code === "EADDRINUSE") {
|
|
4681
|
+
console.error(`Port ${port} already in use. Try a different port with --port.`);
|
|
4682
|
+
process.exit(1);
|
|
4683
|
+
}
|
|
4684
|
+
throw e;
|
|
4685
|
+
}
|
|
4686
|
+
}
|
|
4687
|
+
else {
|
|
4688
|
+
// Default: stdio transport
|
|
4689
|
+
const { startMcpServer } = await import("../mcp/server.js");
|
|
4690
|
+
await startMcpServer({ dbPath: getDbPath() });
|
|
4691
|
+
}
|
|
4692
|
+
break;
|
|
4693
|
+
}
|
|
4694
|
+
case "skills": {
|
|
4695
|
+
try {
|
|
4696
|
+
if (cli.values.help || cli.args[0] === "help") {
|
|
4697
|
+
showSkillsHelp();
|
|
4698
|
+
}
|
|
4699
|
+
else {
|
|
4700
|
+
runSkillsCommand(cli.args, Boolean(cli.values.json), Boolean(cli.values.full), Boolean(cli.values.all));
|
|
4701
|
+
}
|
|
4702
|
+
}
|
|
4703
|
+
catch (error) {
|
|
4704
|
+
if (cli.values.json) {
|
|
4705
|
+
outputSkillsJson({ success: false, error: error instanceof Error ? error.message : String(error) });
|
|
4706
|
+
}
|
|
4707
|
+
else {
|
|
4708
|
+
console.error(error instanceof Error ? error.message : String(error));
|
|
4709
|
+
}
|
|
4710
|
+
process.exit(1);
|
|
4711
|
+
}
|
|
4712
|
+
break;
|
|
4713
|
+
}
|
|
4714
|
+
case "skill": {
|
|
4715
|
+
const subcommand = cli.args[0];
|
|
4716
|
+
switch (subcommand) {
|
|
4717
|
+
case "show": {
|
|
4718
|
+
showSkill();
|
|
4719
|
+
break;
|
|
4720
|
+
}
|
|
4721
|
+
case "install": {
|
|
4722
|
+
try {
|
|
4723
|
+
await installSkill(Boolean(cli.values.global), Boolean(cli.values.force), Boolean(cli.values.yes));
|
|
4724
|
+
}
|
|
4725
|
+
catch (error) {
|
|
4726
|
+
exitWithError(error);
|
|
4727
|
+
}
|
|
4728
|
+
break;
|
|
4729
|
+
}
|
|
4730
|
+
case "help":
|
|
4731
|
+
case undefined: {
|
|
4732
|
+
console.log("Usage: qmd skill <show|install> [options]");
|
|
4733
|
+
console.log("");
|
|
4734
|
+
console.log("Commands:");
|
|
4735
|
+
console.log(" show Print the QMD skill");
|
|
4736
|
+
console.log(" install Install QMD skill into ./.agents/skills/qmd");
|
|
4737
|
+
console.log("");
|
|
4738
|
+
console.log("Options:");
|
|
4739
|
+
console.log(" --global Install into ~/.agents/skills/qmd");
|
|
4740
|
+
console.log(" --yes Also create the .claude/skills/qmd symlink");
|
|
4741
|
+
console.log(" -f, --force Replace existing install or symlink");
|
|
4742
|
+
process.exit(0);
|
|
4743
|
+
}
|
|
4744
|
+
default:
|
|
4745
|
+
console.error(`Unknown subcommand: ${subcommand}`);
|
|
4746
|
+
console.error("Run 'qmd skill help' for usage");
|
|
4747
|
+
printDoctorHint();
|
|
4748
|
+
process.exit(1);
|
|
4749
|
+
}
|
|
4750
|
+
break;
|
|
4751
|
+
}
|
|
4752
|
+
case "cleanup": {
|
|
4753
|
+
const db = getDb();
|
|
4754
|
+
const dryRun = Boolean(cli.values["dry-run"]);
|
|
4755
|
+
if (dryRun) {
|
|
4756
|
+
const stats = previewCleanup(db);
|
|
4757
|
+
console.log("Dry run — no changes made.\n");
|
|
4758
|
+
console.log(`Would clear ${stats.cacheCount} cached API responses`);
|
|
4759
|
+
if (stats.orphanedVectors > 0) {
|
|
4760
|
+
console.log(`Would remove ${stats.orphanedVectors} orphaned embedding chunks`);
|
|
4761
|
+
}
|
|
4762
|
+
else {
|
|
4763
|
+
console.log(`${c.dim}No orphaned embeddings to remove${c.reset}`);
|
|
4764
|
+
}
|
|
4765
|
+
if (stats.inactiveDocs > 0) {
|
|
4766
|
+
console.log(`Would remove ${stats.inactiveDocs} inactive document records`);
|
|
4767
|
+
}
|
|
4768
|
+
if (stats.orphanedContent > 0) {
|
|
4769
|
+
console.log(`Would remove ${stats.orphanedContent} orphaned content hashes`);
|
|
4770
|
+
}
|
|
4771
|
+
console.log("Would compact FTS and vacuum the database");
|
|
4772
|
+
closeDb();
|
|
4773
|
+
break;
|
|
4774
|
+
}
|
|
4775
|
+
const stats = runCleanup(db);
|
|
4776
|
+
console.log(`${c.green}✓${c.reset} Cleared ${stats.cacheCount} cached API responses`);
|
|
4777
|
+
if (stats.orphanedVectors > 0) {
|
|
4778
|
+
console.log(`${c.green}✓${c.reset} Removed ${stats.orphanedVectors} orphaned embedding chunks`);
|
|
4779
|
+
}
|
|
4780
|
+
else {
|
|
4781
|
+
console.log(`${c.dim}No orphaned embeddings to remove${c.reset}`);
|
|
4782
|
+
}
|
|
4783
|
+
if (stats.inactiveDocs > 0) {
|
|
4784
|
+
console.log(`${c.green}✓${c.reset} Removed ${stats.inactiveDocs} inactive document records`);
|
|
4785
|
+
}
|
|
4786
|
+
if (stats.orphanedContent > 0) {
|
|
4787
|
+
console.log(`${c.green}✓${c.reset} Removed ${stats.orphanedContent} orphaned content hashes`);
|
|
4788
|
+
}
|
|
4789
|
+
console.log(`${c.green}✓${c.reset} FTS compacted, database vacuumed`);
|
|
4790
|
+
closeDb();
|
|
4791
|
+
break;
|
|
4792
|
+
}
|
|
4793
|
+
default:
|
|
4794
|
+
console.error(`Unknown command: ${cli.command}`);
|
|
4795
|
+
console.error("Run 'qmd --help' for usage.");
|
|
4796
|
+
printDoctorHint();
|
|
4797
|
+
process.exit(1);
|
|
4798
|
+
}
|
|
4799
|
+
if (cli.command !== "mcp") {
|
|
4800
|
+
await finishSuccessfulCliCommand({
|
|
4801
|
+
command: cli.command,
|
|
4802
|
+
format: cli.opts.format,
|
|
4803
|
+
cleanup: closeCliResources,
|
|
4804
|
+
});
|
|
4805
|
+
}
|
|
4806
|
+
} // end if (main module)
|