@poolzin/pool-bot 2026.3.4 → 2026.3.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +10 -0
- package/assets/pool-bot-icon-dark.png +0 -0
- package/assets/pool-bot-logo-1.png +0 -0
- package/assets/pool-bot-mascot.png +0 -0
- package/dist/agents/pi-embedded-runner/tool-result-truncation.js +62 -7
- package/dist/agents/pi-tools.js +32 -2
- package/dist/agents/poolbot-tools.js +12 -0
- package/dist/agents/session-write-lock.js +93 -8
- package/dist/agents/tools/pdf-native-providers.js +102 -0
- package/dist/agents/tools/pdf-tool.helpers.js +86 -0
- package/dist/agents/tools/pdf-tool.js +508 -0
- package/dist/auto-reply/reply/get-reply.js +6 -0
- package/dist/auto-reply/reply/message-preprocess-hooks.js +17 -0
- package/dist/build-info.json +3 -3
- package/dist/cli/banner.js +20 -1
- package/dist/cli/security-cli.js +211 -2
- package/dist/cli/tagline.js +7 -0
- package/dist/config/types.cli.js +1 -0
- package/dist/config/types.security.js +33 -0
- package/dist/config/zod-schema.js +15 -0
- package/dist/config/zod-schema.providers-core.js +1 -0
- package/dist/config/zod-schema.security.js +113 -0
- package/dist/cron/normalize.js +3 -0
- package/dist/cron/service/jobs.js +48 -0
- package/dist/discord/monitor/message-handler.preflight.js +11 -2
- package/dist/gateway/http-common.js +6 -1
- package/dist/gateway/protocol/schema/cron.js +3 -0
- package/dist/gateway/server-channels.js +99 -14
- package/dist/gateway/server-cron.js +89 -0
- package/dist/gateway/server-health-probes.js +55 -0
- package/dist/gateway/server-http.js +5 -0
- package/dist/hooks/bundled/session-memory/handler.js +8 -2
- package/dist/hooks/fire-and-forget.js +6 -0
- package/dist/hooks/internal-hooks.js +64 -19
- package/dist/hooks/message-hook-mappers.js +179 -0
- package/dist/infra/abort-signal.js +12 -0
- package/dist/infra/boundary-file-read.js +118 -0
- package/dist/infra/boundary-path.js +594 -0
- package/dist/infra/file-identity.js +12 -0
- package/dist/infra/fs-safe.js +377 -12
- package/dist/infra/hardlink-guards.js +30 -0
- package/dist/infra/json-utf8-bytes.js +8 -0
- package/dist/infra/net/fetch-guard.js +63 -13
- package/dist/infra/net/proxy-env.js +17 -0
- package/dist/infra/net/ssrf.js +74 -272
- package/dist/infra/path-alias-guards.js +21 -0
- package/dist/infra/path-guards.js +13 -1
- package/dist/infra/ports-probe.js +19 -0
- package/dist/infra/prototype-keys.js +4 -0
- package/dist/infra/restart-stale-pids.js +254 -0
- package/dist/infra/safe-open-sync.js +71 -0
- package/dist/infra/secure-random.js +7 -0
- package/dist/media/ffmpeg-limits.js +4 -0
- package/dist/media/input-files.js +6 -2
- package/dist/media/temp-files.js +12 -0
- package/dist/memory/embedding-chunk-limits.js +5 -2
- package/dist/memory/embeddings-ollama.js +91 -138
- package/dist/memory/embeddings-remote-fetch.js +11 -10
- package/dist/memory/embeddings.js +25 -9
- package/dist/memory/manager-embedding-ops.js +1 -1
- package/dist/memory/post-json.js +23 -0
- package/dist/memory/qmd-manager.js +272 -77
- package/dist/memory/remote-http.js +33 -0
- package/dist/plugin-sdk/windows-spawn.js +214 -0
- package/dist/security/capability-guards.js +89 -0
- package/dist/security/capability-manager.js +76 -0
- package/dist/security/capability.js +147 -0
- package/dist/security/index.js +7 -0
- package/dist/security/middleware.js +105 -0
- package/dist/shared/net/ip-test-fixtures.js +1 -0
- package/dist/shared/net/ip.js +303 -0
- package/dist/shared/net/ipv4.js +8 -11
- package/dist/shared/pid-alive.js +59 -2
- package/dist/slack/monitor/context.js +1 -0
- package/dist/slack/monitor/message-handler/dispatch.js +14 -1
- package/dist/slack/monitor/provider.js +2 -0
- package/dist/test-helpers/ssrf.js +13 -0
- package/dist/tui/tui.js +9 -4
- package/dist/utils/fetch-timeout.js +12 -1
- package/docs/adr/003-feature-gap-analysis.md +112 -0
- package/package.json +10 -4
|
@@ -44,18 +44,34 @@ async function createLocalEmbeddingProvider(options) {
|
|
|
44
44
|
let llama = null;
|
|
45
45
|
let embeddingModel = null;
|
|
46
46
|
let embeddingContext = null;
|
|
47
|
+
let initPromise = null;
|
|
47
48
|
const ensureContext = async () => {
|
|
48
|
-
if (
|
|
49
|
-
|
|
49
|
+
if (embeddingContext) {
|
|
50
|
+
return embeddingContext;
|
|
50
51
|
}
|
|
51
|
-
if (
|
|
52
|
-
|
|
53
|
-
embeddingModel = await llama.loadModel({ modelPath: resolved });
|
|
52
|
+
if (initPromise) {
|
|
53
|
+
return initPromise;
|
|
54
54
|
}
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
55
|
+
initPromise = (async () => {
|
|
56
|
+
try {
|
|
57
|
+
if (!llama) {
|
|
58
|
+
llama = await getLlama({ logLevel: LlamaLogLevel.error });
|
|
59
|
+
}
|
|
60
|
+
if (!embeddingModel) {
|
|
61
|
+
const resolved = await resolveModelFile(modelPath, modelCacheDir || undefined);
|
|
62
|
+
embeddingModel = await llama.loadModel({ modelPath: resolved });
|
|
63
|
+
}
|
|
64
|
+
if (!embeddingContext) {
|
|
65
|
+
embeddingContext = await embeddingModel.createEmbeddingContext();
|
|
66
|
+
}
|
|
67
|
+
return embeddingContext;
|
|
68
|
+
}
|
|
69
|
+
catch (err) {
|
|
70
|
+
initPromise = null;
|
|
71
|
+
throw err;
|
|
72
|
+
}
|
|
73
|
+
})();
|
|
74
|
+
return initPromise;
|
|
59
75
|
};
|
|
60
76
|
return {
|
|
61
77
|
id: "local",
|
|
@@ -524,7 +524,7 @@ export class MemoryManagerEmbeddingOps extends MemoryManagerSyncOps {
|
|
|
524
524
|
return;
|
|
525
525
|
}
|
|
526
526
|
const content = options.content ?? (await fs.readFile(entry.absPath, "utf-8"));
|
|
527
|
-
const chunks = enforceEmbeddingMaxInputTokens(this.provider, chunkMarkdown(content, this.settings.chunking).filter((chunk) => chunk.text.trim().length > 0));
|
|
527
|
+
const chunks = enforceEmbeddingMaxInputTokens(this.provider, chunkMarkdown(content, this.settings.chunking).filter((chunk) => chunk.text.trim().length > 0), EMBEDDING_BATCH_MAX_TOKENS);
|
|
528
528
|
if (options.source === "sessions" && "lineMap" in entry) {
|
|
529
529
|
remapChunkLines(chunks, entry.lineMap);
|
|
530
530
|
}
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
import { withRemoteHttpResponse } from "./remote-http.js";
|
|
2
|
+
export async function postJson(params) {
|
|
3
|
+
return await withRemoteHttpResponse({
|
|
4
|
+
url: params.url,
|
|
5
|
+
ssrfPolicy: params.ssrfPolicy,
|
|
6
|
+
init: {
|
|
7
|
+
method: "POST",
|
|
8
|
+
headers: params.headers,
|
|
9
|
+
body: JSON.stringify(params.body),
|
|
10
|
+
},
|
|
11
|
+
onResponse: async (res) => {
|
|
12
|
+
if (!res.ok) {
|
|
13
|
+
const text = await res.text();
|
|
14
|
+
const err = new Error(`${params.errorPrefix}: ${res.status} ${text}`);
|
|
15
|
+
if (params.attachStatus) {
|
|
16
|
+
err.status = res.status;
|
|
17
|
+
}
|
|
18
|
+
throw err;
|
|
19
|
+
}
|
|
20
|
+
return await params.parse(await res.json());
|
|
21
|
+
},
|
|
22
|
+
});
|
|
23
|
+
}
|
|
@@ -5,12 +5,15 @@ import path from "node:path";
|
|
|
5
5
|
import readline from "node:readline";
|
|
6
6
|
import { resolveAgentWorkspaceDir } from "../agents/agent-scope.js";
|
|
7
7
|
import { resolveStateDir } from "../config/paths.js";
|
|
8
|
+
import { writeFileWithinRoot } from "../infra/fs-safe.js";
|
|
8
9
|
import { createSubsystemLogger } from "../logging/subsystem.js";
|
|
10
|
+
import { materializeWindowsSpawnProgram, resolveWindowsSpawnProgram, } from "../plugin-sdk/windows-spawn.js";
|
|
9
11
|
import { isFileMissingError, statRegularFile } from "./fs-utils.js";
|
|
10
12
|
import { deriveQmdScopeChannel, deriveQmdScopeChatType, isQmdScopeAllowed } from "./qmd-scope.js";
|
|
11
13
|
import { listSessionFilesForAgent, buildSessionEntry, } from "./session-files.js";
|
|
12
14
|
import { requireNodeSqlite } from "./sqlite.js";
|
|
13
15
|
import { parseQmdQueryJson } from "./qmd-query-parser.js";
|
|
16
|
+
import { extractKeywords } from "./query-expansion.js";
|
|
14
17
|
const log = createSubsystemLogger("memory");
|
|
15
18
|
const SNIPPET_HEADER_RE = /@@\s*-([0-9]+),([0-9]+)/;
|
|
16
19
|
const SEARCH_PENDING_UPDATE_WAIT_MS = 500;
|
|
@@ -18,7 +21,70 @@ const MAX_QMD_OUTPUT_CHARS = 200_000;
|
|
|
18
21
|
const NUL_MARKER_RE = /(?:\^@|\\0|\\x00|\\u0000|null\s*byte|nul\s*byte)/i;
|
|
19
22
|
const QMD_EMBED_BACKOFF_BASE_MS = 60_000;
|
|
20
23
|
const QMD_EMBED_BACKOFF_MAX_MS = 60 * 60 * 1000;
|
|
24
|
+
const HAN_SCRIPT_RE = /[\u3400-\u9fff]/u;
|
|
25
|
+
const QMD_BM25_HAN_KEYWORD_LIMIT = 12;
|
|
21
26
|
let qmdEmbedQueueTail = Promise.resolve();
|
|
27
|
+
function resolveWindowsCommandShim(command) {
|
|
28
|
+
if (process.platform !== "win32") {
|
|
29
|
+
return command;
|
|
30
|
+
}
|
|
31
|
+
const trimmed = command.trim();
|
|
32
|
+
if (!trimmed) {
|
|
33
|
+
return command;
|
|
34
|
+
}
|
|
35
|
+
const ext = path.extname(trimmed).toLowerCase();
|
|
36
|
+
if (ext === ".cmd" || ext === ".exe" || ext === ".bat") {
|
|
37
|
+
return command;
|
|
38
|
+
}
|
|
39
|
+
const base = path.basename(trimmed).toLowerCase();
|
|
40
|
+
if (base === "qmd" || base === "mcporter") {
|
|
41
|
+
return `${trimmed}.cmd`;
|
|
42
|
+
}
|
|
43
|
+
return command;
|
|
44
|
+
}
|
|
45
|
+
function resolveSpawnInvocation(params) {
|
|
46
|
+
const program = resolveWindowsSpawnProgram({
|
|
47
|
+
command: resolveWindowsCommandShim(params.command),
|
|
48
|
+
platform: process.platform,
|
|
49
|
+
env: params.env,
|
|
50
|
+
execPath: process.execPath,
|
|
51
|
+
packageName: params.packageName,
|
|
52
|
+
allowShellFallback: true,
|
|
53
|
+
});
|
|
54
|
+
return materializeWindowsSpawnProgram(program, params.args);
|
|
55
|
+
}
|
|
56
|
+
function hasHanScript(value) {
|
|
57
|
+
return HAN_SCRIPT_RE.test(value);
|
|
58
|
+
}
|
|
59
|
+
function normalizeHanBm25Query(query) {
|
|
60
|
+
const trimmed = query.trim();
|
|
61
|
+
if (!trimmed || !hasHanScript(trimmed)) {
|
|
62
|
+
return trimmed;
|
|
63
|
+
}
|
|
64
|
+
const keywords = extractKeywords(trimmed);
|
|
65
|
+
const normalizedKeywords = [];
|
|
66
|
+
const seen = new Set();
|
|
67
|
+
for (const keyword of keywords) {
|
|
68
|
+
const token = keyword.trim();
|
|
69
|
+
if (!token || seen.has(token)) {
|
|
70
|
+
continue;
|
|
71
|
+
}
|
|
72
|
+
const includesHan = hasHanScript(token);
|
|
73
|
+
// Han unigrams are usually too broad for BM25 and can drown signal.
|
|
74
|
+
if (includesHan && Array.from(token).length < 2) {
|
|
75
|
+
continue;
|
|
76
|
+
}
|
|
77
|
+
if (!includesHan && token.length < 2) {
|
|
78
|
+
continue;
|
|
79
|
+
}
|
|
80
|
+
seen.add(token);
|
|
81
|
+
normalizedKeywords.push(token);
|
|
82
|
+
if (normalizedKeywords.length >= QMD_BM25_HAN_KEYWORD_LIMIT) {
|
|
83
|
+
break;
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
return normalizedKeywords.length > 0 ? normalizedKeywords.join(" ") : trimmed;
|
|
87
|
+
}
|
|
22
88
|
async function runWithQmdEmbedLock(task) {
|
|
23
89
|
const previous = qmdEmbedQueueTail;
|
|
24
90
|
let release;
|
|
@@ -54,6 +120,7 @@ export class QmdMemoryManager {
|
|
|
54
120
|
xdgCacheHome;
|
|
55
121
|
indexPath;
|
|
56
122
|
env;
|
|
123
|
+
managedCollectionNames;
|
|
57
124
|
collectionRoots = new Map();
|
|
58
125
|
sources = new Set();
|
|
59
126
|
docPathCache = new Map();
|
|
@@ -89,6 +156,9 @@ export class QmdMemoryManager {
|
|
|
89
156
|
this.env = {
|
|
90
157
|
...process.env,
|
|
91
158
|
XDG_CONFIG_HOME: this.xdgConfigHome,
|
|
159
|
+
// workaround for upstream bug https://github.com/tobi/qmd/issues/132
|
|
160
|
+
// QMD doesn't respect XDG_CONFIG_HOME:
|
|
161
|
+
QMD_CONFIG_DIR: this.xdgConfigHome,
|
|
92
162
|
XDG_CACHE_HOME: this.xdgCacheHome,
|
|
93
163
|
NO_COLOR: "1",
|
|
94
164
|
};
|
|
@@ -112,6 +182,7 @@ export class QmdMemoryManager {
|
|
|
112
182
|
},
|
|
113
183
|
];
|
|
114
184
|
}
|
|
185
|
+
this.managedCollectionNames = this.computeManagedCollectionNames();
|
|
115
186
|
}
|
|
116
187
|
async initialize(mode) {
|
|
117
188
|
this.bootstrapCollections();
|
|
@@ -171,29 +242,9 @@ export class QmdMemoryManager {
|
|
|
171
242
|
const result = await this.runQmd(["collection", "list", "--json"], {
|
|
172
243
|
timeoutMs: this.qmd.update.commandTimeoutMs,
|
|
173
244
|
});
|
|
174
|
-
const parsed =
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
if (typeof entry === "string") {
|
|
178
|
-
existing.set(entry, {});
|
|
179
|
-
}
|
|
180
|
-
else if (entry && typeof entry === "object") {
|
|
181
|
-
const name = entry.name;
|
|
182
|
-
if (typeof name === "string") {
|
|
183
|
-
const listedPath = entry.path;
|
|
184
|
-
const listedPattern = entry.pattern;
|
|
185
|
-
const listedMask = entry.mask;
|
|
186
|
-
existing.set(name, {
|
|
187
|
-
path: typeof listedPath === "string" ? listedPath : undefined,
|
|
188
|
-
pattern: typeof listedPattern === "string"
|
|
189
|
-
? listedPattern
|
|
190
|
-
: typeof listedMask === "string"
|
|
191
|
-
? listedMask
|
|
192
|
-
: undefined,
|
|
193
|
-
});
|
|
194
|
-
}
|
|
195
|
-
}
|
|
196
|
-
}
|
|
245
|
+
const parsed = this.parseListedCollections(result.stdout);
|
|
246
|
+
for (const [name, details] of parsed) {
|
|
247
|
+
existing.set(name, details);
|
|
197
248
|
}
|
|
198
249
|
}
|
|
199
250
|
catch {
|
|
@@ -292,6 +343,18 @@ export class QmdMemoryManager {
|
|
|
292
343
|
const lower = message.toLowerCase();
|
|
293
344
|
return (lower.includes("not found") || lower.includes("does not exist") || lower.includes("missing"));
|
|
294
345
|
}
|
|
346
|
+
isMissingCollectionSearchError(err) {
|
|
347
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
348
|
+
return this.isCollectionMissingError(message) && message.toLowerCase().includes("collection");
|
|
349
|
+
}
|
|
350
|
+
async tryRepairMissingCollectionSearch(err) {
|
|
351
|
+
if (!this.isMissingCollectionSearchError(err)) {
|
|
352
|
+
return false;
|
|
353
|
+
}
|
|
354
|
+
log.warn("qmd search failed because a managed collection is missing; repairing collections and retrying once");
|
|
355
|
+
await this.ensureCollections();
|
|
356
|
+
return true;
|
|
357
|
+
}
|
|
295
358
|
async addCollection(pathArg, name, pattern) {
|
|
296
359
|
await this.runQmd(["collection", "add", pathArg, "--name", name, "--mask", pattern], {
|
|
297
360
|
timeoutMs: this.qmd.update.commandTimeoutMs,
|
|
@@ -302,6 +365,90 @@ export class QmdMemoryManager {
|
|
|
302
365
|
timeoutMs: this.qmd.update.commandTimeoutMs,
|
|
303
366
|
});
|
|
304
367
|
}
|
|
368
|
+
parseListedCollections(output) {
|
|
369
|
+
const listed = new Map();
|
|
370
|
+
const trimmed = output.trim();
|
|
371
|
+
if (!trimmed) {
|
|
372
|
+
return listed;
|
|
373
|
+
}
|
|
374
|
+
try {
|
|
375
|
+
const parsed = JSON.parse(trimmed);
|
|
376
|
+
if (Array.isArray(parsed)) {
|
|
377
|
+
for (const entry of parsed) {
|
|
378
|
+
if (typeof entry === "string") {
|
|
379
|
+
listed.set(entry, {});
|
|
380
|
+
continue;
|
|
381
|
+
}
|
|
382
|
+
if (!entry || typeof entry !== "object") {
|
|
383
|
+
continue;
|
|
384
|
+
}
|
|
385
|
+
const name = entry.name;
|
|
386
|
+
if (typeof name !== "string") {
|
|
387
|
+
continue;
|
|
388
|
+
}
|
|
389
|
+
const listedPath = entry.path;
|
|
390
|
+
const listedPattern = entry.pattern;
|
|
391
|
+
const listedMask = entry.mask;
|
|
392
|
+
listed.set(name, {
|
|
393
|
+
path: typeof listedPath === "string" ? listedPath : undefined,
|
|
394
|
+
pattern: typeof listedPattern === "string"
|
|
395
|
+
? listedPattern
|
|
396
|
+
: typeof listedMask === "string"
|
|
397
|
+
? listedMask
|
|
398
|
+
: undefined,
|
|
399
|
+
});
|
|
400
|
+
}
|
|
401
|
+
return listed;
|
|
402
|
+
}
|
|
403
|
+
}
|
|
404
|
+
catch {
|
|
405
|
+
// Some qmd builds ignore `--json` and still print table output.
|
|
406
|
+
}
|
|
407
|
+
let currentName = null;
|
|
408
|
+
for (const rawLine of output.split(/\r?\n/)) {
|
|
409
|
+
const line = rawLine.trimEnd();
|
|
410
|
+
if (!line.trim()) {
|
|
411
|
+
currentName = null;
|
|
412
|
+
continue;
|
|
413
|
+
}
|
|
414
|
+
const collectionLine = /^\s*([a-z0-9._-]+)\s+\(qmd:\/\/[^)]+\)\s*$/i.exec(line);
|
|
415
|
+
if (collectionLine) {
|
|
416
|
+
currentName = collectionLine[1];
|
|
417
|
+
if (!listed.has(currentName)) {
|
|
418
|
+
listed.set(currentName, {});
|
|
419
|
+
}
|
|
420
|
+
continue;
|
|
421
|
+
}
|
|
422
|
+
if (/^\s*collections\b/i.test(line)) {
|
|
423
|
+
continue;
|
|
424
|
+
}
|
|
425
|
+
const bareNameLine = /^\s*([a-z0-9._-]+)\s*$/i.exec(line);
|
|
426
|
+
if (bareNameLine && !line.includes(":")) {
|
|
427
|
+
currentName = bareNameLine[1];
|
|
428
|
+
if (!listed.has(currentName)) {
|
|
429
|
+
listed.set(currentName, {});
|
|
430
|
+
}
|
|
431
|
+
continue;
|
|
432
|
+
}
|
|
433
|
+
if (!currentName) {
|
|
434
|
+
continue;
|
|
435
|
+
}
|
|
436
|
+
const patternLine = /^\s*(?:pattern|mask)\s*:\s*(.+?)\s*$/i.exec(line);
|
|
437
|
+
if (patternLine) {
|
|
438
|
+
const existing = listed.get(currentName) ?? {};
|
|
439
|
+
existing.pattern = patternLine[1].trim();
|
|
440
|
+
listed.set(currentName, existing);
|
|
441
|
+
continue;
|
|
442
|
+
}
|
|
443
|
+
const pathLine = /^\s*path\s*:\s*(.+?)\s*$/i.exec(line);
|
|
444
|
+
if (pathLine) {
|
|
445
|
+
const existing = listed.get(currentName) ?? {};
|
|
446
|
+
existing.path = pathLine[1].trim();
|
|
447
|
+
listed.set(currentName, existing);
|
|
448
|
+
}
|
|
449
|
+
}
|
|
450
|
+
return listed;
|
|
451
|
+
}
|
|
305
452
|
shouldRebindCollection(collection, listed) {
|
|
306
453
|
if (!listed.path) {
|
|
307
454
|
// Older qmd versions may only return names from `collection list --json`.
|
|
@@ -381,26 +528,25 @@ export class QmdMemoryManager {
|
|
|
381
528
|
}
|
|
382
529
|
const qmdSearchCommand = this.qmd.searchMode;
|
|
383
530
|
const mcporterEnabled = this.qmd.mcporter.enabled;
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
parsed = await this.runQmdSearchViaMcporter({
|
|
531
|
+
const runSearchAttempt = async (allowMissingCollectionRepair) => {
|
|
532
|
+
try {
|
|
533
|
+
if (mcporterEnabled) {
|
|
534
|
+
const tool = qmdSearchCommand === "search"
|
|
535
|
+
? "search"
|
|
536
|
+
: qmdSearchCommand === "vsearch"
|
|
537
|
+
? "vector_search"
|
|
538
|
+
: "deep_search";
|
|
539
|
+
const minScore = opts?.minScore ?? 0;
|
|
540
|
+
if (collectionNames.length > 1) {
|
|
541
|
+
return await this.runMcporterAcrossCollections({
|
|
542
|
+
tool,
|
|
543
|
+
query: trimmed,
|
|
544
|
+
limit,
|
|
545
|
+
minScore,
|
|
546
|
+
collectionNames,
|
|
547
|
+
});
|
|
548
|
+
}
|
|
549
|
+
return await this.runQmdSearchViaMcporter({
|
|
404
550
|
mcporter: this.qmd.mcporter,
|
|
405
551
|
tool,
|
|
406
552
|
query: trimmed,
|
|
@@ -410,47 +556,54 @@ export class QmdMemoryManager {
|
|
|
410
556
|
timeoutMs: this.qmd.limits.timeoutMs,
|
|
411
557
|
});
|
|
412
558
|
}
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
}
|
|
417
|
-
else {
|
|
559
|
+
if (collectionNames.length > 1) {
|
|
560
|
+
return await this.runQueryAcrossCollections(trimmed, limit, collectionNames, qmdSearchCommand);
|
|
561
|
+
}
|
|
418
562
|
const args = this.buildSearchArgs(qmdSearchCommand, trimmed, limit);
|
|
419
563
|
args.push(...this.buildCollectionFilterArgs(collectionNames));
|
|
420
564
|
// Always scope to managed collections (default + custom). Even for `search`/`vsearch`,
|
|
421
565
|
// pass collection filters; if a given QMD build rejects these flags, we fall back to `query`.
|
|
422
566
|
const result = await this.runQmd(args, { timeoutMs: this.qmd.limits.timeoutMs });
|
|
423
|
-
|
|
567
|
+
return parseQmdQueryJson(result.stdout, result.stderr);
|
|
424
568
|
}
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
569
|
+
catch (err) {
|
|
570
|
+
if (allowMissingCollectionRepair && this.isMissingCollectionSearchError(err)) {
|
|
571
|
+
throw err;
|
|
572
|
+
}
|
|
573
|
+
if (!mcporterEnabled &&
|
|
574
|
+
qmdSearchCommand !== "query" &&
|
|
575
|
+
this.isUnsupportedQmdOptionError(err)) {
|
|
576
|
+
log.warn(`qmd ${qmdSearchCommand} does not support configured flags; retrying search with qmd query`);
|
|
577
|
+
try {
|
|
578
|
+
if (collectionNames.length > 1) {
|
|
579
|
+
return await this.runQueryAcrossCollections(trimmed, limit, collectionNames, "query");
|
|
580
|
+
}
|
|
436
581
|
const fallbackArgs = this.buildSearchArgs("query", trimmed, limit);
|
|
437
582
|
fallbackArgs.push(...this.buildCollectionFilterArgs(collectionNames));
|
|
438
583
|
const fallback = await this.runQmd(fallbackArgs, {
|
|
439
584
|
timeoutMs: this.qmd.limits.timeoutMs,
|
|
440
585
|
});
|
|
441
|
-
|
|
586
|
+
return parseQmdQueryJson(fallback.stdout, fallback.stderr);
|
|
587
|
+
}
|
|
588
|
+
catch (fallbackErr) {
|
|
589
|
+
log.warn(`qmd query fallback failed: ${String(fallbackErr)}`);
|
|
590
|
+
throw fallbackErr instanceof Error ? fallbackErr : new Error(String(fallbackErr));
|
|
442
591
|
}
|
|
443
592
|
}
|
|
444
|
-
catch (fallbackErr) {
|
|
445
|
-
log.warn(`qmd query fallback failed: ${String(fallbackErr)}`);
|
|
446
|
-
throw fallbackErr instanceof Error ? fallbackErr : new Error(String(fallbackErr));
|
|
447
|
-
}
|
|
448
|
-
}
|
|
449
|
-
else {
|
|
450
593
|
const label = mcporterEnabled ? "mcporter/qmd" : `qmd ${qmdSearchCommand}`;
|
|
451
594
|
log.warn(`${label} failed: ${String(err)}`);
|
|
452
595
|
throw err instanceof Error ? err : new Error(String(err));
|
|
453
596
|
}
|
|
597
|
+
};
|
|
598
|
+
let parsed;
|
|
599
|
+
try {
|
|
600
|
+
parsed = await runSearchAttempt(true);
|
|
601
|
+
}
|
|
602
|
+
catch (err) {
|
|
603
|
+
if (!(await this.tryRepairMissingCollectionSearch(err))) {
|
|
604
|
+
throw err instanceof Error ? err : new Error(String(err));
|
|
605
|
+
}
|
|
606
|
+
parsed = await runSearchAttempt(false);
|
|
454
607
|
}
|
|
455
608
|
const results = [];
|
|
456
609
|
for (const entry of parsed) {
|
|
@@ -603,7 +756,10 @@ export class QmdMemoryManager {
|
|
|
603
756
|
if (this.shouldRunEmbed(force)) {
|
|
604
757
|
try {
|
|
605
758
|
await runWithQmdEmbedLock(async () => {
|
|
606
|
-
await this.runQmd(["embed"], {
|
|
759
|
+
await this.runQmd(["embed"], {
|
|
760
|
+
timeoutMs: this.qmd.update.embedTimeoutMs,
|
|
761
|
+
discardOutput: true,
|
|
762
|
+
});
|
|
607
763
|
});
|
|
608
764
|
this.lastEmbedAt = Date.now();
|
|
609
765
|
this.embedBackoffUntil = null;
|
|
@@ -641,13 +797,19 @@ export class QmdMemoryManager {
|
|
|
641
797
|
}
|
|
642
798
|
async runQmdUpdateOnce(reason) {
|
|
643
799
|
try {
|
|
644
|
-
await this.runQmd(["update"], {
|
|
800
|
+
await this.runQmd(["update"], {
|
|
801
|
+
timeoutMs: this.qmd.update.updateTimeoutMs,
|
|
802
|
+
discardOutput: true,
|
|
803
|
+
});
|
|
645
804
|
}
|
|
646
805
|
catch (err) {
|
|
647
806
|
if (!(await this.tryRepairNullByteCollections(err, reason))) {
|
|
648
807
|
throw err;
|
|
649
808
|
}
|
|
650
|
-
await this.runQmd(["update"], {
|
|
809
|
+
await this.runQmd(["update"], {
|
|
810
|
+
timeoutMs: this.qmd.update.updateTimeoutMs,
|
|
811
|
+
discardOutput: true,
|
|
812
|
+
});
|
|
651
813
|
}
|
|
652
814
|
}
|
|
653
815
|
isRetryableUpdateError(err) {
|
|
@@ -759,14 +921,26 @@ export class QmdMemoryManager {
|
|
|
759
921
|
}
|
|
760
922
|
async runQmd(args, opts) {
|
|
761
923
|
return await new Promise((resolve, reject) => {
|
|
762
|
-
const
|
|
924
|
+
const spawnInvocation = resolveSpawnInvocation({
|
|
925
|
+
command: this.qmd.command,
|
|
926
|
+
args,
|
|
927
|
+
env: this.env,
|
|
928
|
+
packageName: "qmd",
|
|
929
|
+
});
|
|
930
|
+
const child = spawn(spawnInvocation.command, spawnInvocation.argv, {
|
|
763
931
|
env: this.env,
|
|
764
932
|
cwd: this.workspaceDir,
|
|
933
|
+
shell: spawnInvocation.shell,
|
|
934
|
+
windowsHide: spawnInvocation.windowsHide,
|
|
765
935
|
});
|
|
766
936
|
let stdout = "";
|
|
767
937
|
let stderr = "";
|
|
768
938
|
let stdoutTruncated = false;
|
|
769
939
|
let stderrTruncated = false;
|
|
940
|
+
// When discardOutput is set, skip stdout accumulation entirely and keep
|
|
941
|
+
// only a small stderr tail for diagnostics -- never fail on truncation.
|
|
942
|
+
// This prevents large `qmd update` runs from hitting the output cap.
|
|
943
|
+
const discard = opts?.discardOutput === true;
|
|
770
944
|
const timer = opts?.timeoutMs
|
|
771
945
|
? setTimeout(() => {
|
|
772
946
|
child.kill("SIGKILL");
|
|
@@ -774,6 +948,9 @@ export class QmdMemoryManager {
|
|
|
774
948
|
}, opts.timeoutMs)
|
|
775
949
|
: null;
|
|
776
950
|
child.stdout.on("data", (data) => {
|
|
951
|
+
if (discard) {
|
|
952
|
+
return; // drain without accumulating
|
|
953
|
+
}
|
|
777
954
|
const next = appendOutputWithCap(stdout, data.toString("utf8"), this.maxQmdOutputChars);
|
|
778
955
|
stdout = next.text;
|
|
779
956
|
stdoutTruncated = stdoutTruncated || next.truncated;
|
|
@@ -793,7 +970,7 @@ export class QmdMemoryManager {
|
|
|
793
970
|
if (timer) {
|
|
794
971
|
clearTimeout(timer);
|
|
795
972
|
}
|
|
796
|
-
if (stdoutTruncated || stderrTruncated) {
|
|
973
|
+
if (!discard && (stdoutTruncated || stderrTruncated)) {
|
|
797
974
|
reject(new Error(`qmd ${args.join(" ")} produced too much output (limit ${this.maxQmdOutputChars} chars)`));
|
|
798
975
|
return;
|
|
799
976
|
}
|
|
@@ -835,10 +1012,18 @@ export class QmdMemoryManager {
|
|
|
835
1012
|
}
|
|
836
1013
|
async runMcporter(args, opts) {
|
|
837
1014
|
return await new Promise((resolve, reject) => {
|
|
838
|
-
const
|
|
1015
|
+
const spawnInvocation = resolveSpawnInvocation({
|
|
1016
|
+
command: "mcporter",
|
|
1017
|
+
args,
|
|
1018
|
+
env: this.env,
|
|
1019
|
+
packageName: "mcporter",
|
|
1020
|
+
});
|
|
1021
|
+
const child = spawn(spawnInvocation.command, spawnInvocation.argv, {
|
|
839
1022
|
// Keep mcporter and direct qmd commands on the same agent-scoped XDG state.
|
|
840
1023
|
env: this.env,
|
|
841
1024
|
cwd: this.workspaceDir,
|
|
1025
|
+
shell: spawnInvocation.shell,
|
|
1026
|
+
windowsHide: spawnInvocation.windowsHide,
|
|
842
1027
|
});
|
|
843
1028
|
let stdout = "";
|
|
844
1029
|
let stderr = "";
|
|
@@ -1011,11 +1196,17 @@ export class QmdMemoryManager {
|
|
|
1011
1196
|
if (cutoff && entry.mtimeMs < cutoff) {
|
|
1012
1197
|
continue;
|
|
1013
1198
|
}
|
|
1014
|
-
const
|
|
1199
|
+
const targetName = `${path.basename(sessionFile, ".jsonl")}.md`;
|
|
1200
|
+
const target = path.join(exportDir, targetName);
|
|
1015
1201
|
tracked.add(sessionFile);
|
|
1016
1202
|
const state = this.exportedSessionState.get(sessionFile);
|
|
1017
1203
|
if (!state || state.hash !== entry.hash || state.mtimeMs !== entry.mtimeMs) {
|
|
1018
|
-
await
|
|
1204
|
+
await writeFileWithinRoot({
|
|
1205
|
+
rootDir: exportDir,
|
|
1206
|
+
relativePath: targetName,
|
|
1207
|
+
data: this.renderSessionMarkdown(entry),
|
|
1208
|
+
encoding: "utf-8",
|
|
1209
|
+
});
|
|
1019
1210
|
}
|
|
1020
1211
|
this.exportedSessionState.set(sessionFile, {
|
|
1021
1212
|
hash: entry.hash,
|
|
@@ -1439,6 +1630,9 @@ export class QmdMemoryManager {
|
|
|
1439
1630
|
return [...bestByDocId.values()].toSorted((a, b) => (b.score ?? 0) - (a.score ?? 0));
|
|
1440
1631
|
}
|
|
1441
1632
|
listManagedCollectionNames() {
|
|
1633
|
+
return this.managedCollectionNames;
|
|
1634
|
+
}
|
|
1635
|
+
computeManagedCollectionNames() {
|
|
1442
1636
|
const seen = new Set();
|
|
1443
1637
|
const names = [];
|
|
1444
1638
|
for (const collection of this.qmd.collections) {
|
|
@@ -1459,10 +1653,11 @@ export class QmdMemoryManager {
|
|
|
1459
1653
|
return names.flatMap((name) => ["-c", name]);
|
|
1460
1654
|
}
|
|
1461
1655
|
buildSearchArgs(command, query, limit) {
|
|
1656
|
+
const normalizedQuery = command === "search" ? normalizeHanBm25Query(query) : query;
|
|
1462
1657
|
if (command === "query") {
|
|
1463
|
-
return ["query",
|
|
1658
|
+
return ["query", normalizedQuery, "--json", "-n", String(limit)];
|
|
1464
1659
|
}
|
|
1465
|
-
return [command,
|
|
1660
|
+
return [command, normalizedQuery, "--json", "-n", String(limit)];
|
|
1466
1661
|
}
|
|
1467
1662
|
}
|
|
1468
1663
|
function appendOutputWithCap(current, chunk, maxChars) {
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
import { fetchWithSsrFGuard } from "../infra/net/fetch-guard.js";
|
|
2
|
+
export function buildRemoteBaseUrlPolicy(baseUrl) {
|
|
3
|
+
const trimmed = baseUrl.trim();
|
|
4
|
+
if (!trimmed) {
|
|
5
|
+
return undefined;
|
|
6
|
+
}
|
|
7
|
+
try {
|
|
8
|
+
const parsed = new URL(trimmed);
|
|
9
|
+
if (parsed.protocol !== "http:" && parsed.protocol !== "https:") {
|
|
10
|
+
return undefined;
|
|
11
|
+
}
|
|
12
|
+
// Keep policy tied to the configured host so private operator endpoints
|
|
13
|
+
// continue to work, while cross-host redirects stay blocked.
|
|
14
|
+
return { allowedHostnames: [parsed.hostname] };
|
|
15
|
+
}
|
|
16
|
+
catch {
|
|
17
|
+
return undefined;
|
|
18
|
+
}
|
|
19
|
+
}
|
|
20
|
+
export async function withRemoteHttpResponse(params) {
|
|
21
|
+
const { response, release } = await fetchWithSsrFGuard({
|
|
22
|
+
url: params.url,
|
|
23
|
+
init: params.init,
|
|
24
|
+
policy: params.ssrfPolicy,
|
|
25
|
+
auditContext: params.auditContext ?? "memory-remote",
|
|
26
|
+
});
|
|
27
|
+
try {
|
|
28
|
+
return await params.onResponse(response);
|
|
29
|
+
}
|
|
30
|
+
finally {
|
|
31
|
+
await release();
|
|
32
|
+
}
|
|
33
|
+
}
|