akm-cli 0.9.0-beta.45 → 0.9.0-beta.46
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli/shared.js +28 -0
- package/dist/cli.js +1 -2
- package/dist/commands/env/env-cli.js +16 -24
- package/dist/commands/env/secret-cli.js +12 -20
- package/dist/commands/graph/graph-cli.js +5 -13
- package/dist/commands/graph/graph.js +3 -3
- package/dist/commands/improve/consolidate/chunking.js +141 -0
- package/dist/commands/improve/consolidate/eligibility.js +64 -0
- package/dist/commands/improve/consolidate/merge.js +145 -0
- package/dist/commands/improve/consolidate/sanitize.js +231 -0
- package/dist/commands/improve/consolidate/types.js +4 -0
- package/dist/commands/improve/consolidate.js +20 -571
- package/dist/commands/improve/distill.js +5 -9
- package/dist/commands/improve/eligibility.js +434 -0
- package/dist/commands/improve/extract-cli.js +9 -1
- package/dist/commands/improve/extract.js +5 -19
- package/dist/commands/improve/improve-auto-accept.js +4 -8
- package/dist/commands/improve/improve-cli.js +35 -60
- package/dist/commands/improve/improve-result-file.js +5 -23
- package/dist/commands/improve/improve-session.js +58 -0
- package/dist/commands/improve/improve.js +107 -3606
- package/dist/commands/improve/locks.js +154 -0
- package/dist/commands/improve/loop-stages.js +1079 -0
- package/dist/commands/improve/preparation.js +1963 -0
- package/dist/commands/improve/recombine.js +6 -12
- package/dist/commands/improve/reflect.js +29 -34
- package/dist/commands/proposal/drain.js +25 -48
- package/dist/commands/proposal/proposal-cli.js +21 -31
- package/dist/commands/proposal/validators/proposals.js +3 -7
- package/dist/commands/read/curate.js +70 -14
- package/dist/commands/read/knowledge.js +2 -2
- package/dist/commands/sources/self-update.js +2 -2
- package/dist/commands/sources/stash-cli.js +9 -37
- package/dist/commands/tasks/tasks-cli.js +19 -27
- package/dist/commands/wiki-cli.js +21 -35
- package/dist/core/config/config.js +18 -2
- package/dist/core/events.js +3 -7
- package/dist/core/logs-db.js +6 -63
- package/dist/core/state/migrations.js +714 -0
- package/dist/core/state-db.js +28 -779
- package/dist/indexer/db/db.js +82 -216
- package/dist/indexer/indexer.js +11 -112
- package/dist/indexer/passes/dir-staleness.js +114 -0
- package/dist/indexer/search/search-source.js +10 -24
- package/dist/integrations/agent/runner-dispatch.js +59 -0
- package/dist/llm/client.js +22 -11
- package/dist/llm/graph-extract.js +28 -39
- package/dist/llm/memory-infer.js +34 -22
- package/dist/llm/metadata-enhance.js +35 -30
- package/dist/llm/structured-call.js +49 -0
- package/dist/output/shapes/passthrough.js +0 -1
- package/dist/registry/providers/skills-sh.js +21 -147
- package/dist/registry/providers/static-index.js +15 -157
- package/dist/registry/resolve.js +22 -9
- package/dist/scripts/migrate-storage.js +892 -1186
- package/dist/scripts/migrations/import-fs-improve-runs-to-db.js +214 -179
- package/dist/setup/setup.js +26 -5
- package/dist/sources/providers/filesystem.js +0 -1
- package/dist/sources/providers/git-install.js +206 -0
- package/dist/sources/providers/git-provider.js +234 -0
- package/dist/sources/providers/git-stash.js +248 -0
- package/dist/sources/providers/git.js +10 -671
- package/dist/sources/providers/npm.js +2 -6
- package/dist/sources/providers/sync-from-ref.js +9 -1
- package/dist/sources/providers/website.js +2 -3
- package/dist/sources/website-ingest.js +51 -9
- package/dist/sources/wiki-fetchers/registry.js +53 -0
- package/dist/sources/wiki-fetchers/youtube.js +181 -0
- package/dist/storage/database.js +45 -10
- package/dist/storage/managed-db.js +82 -0
- package/dist/storage/repositories/registry-cache.js +92 -0
- package/dist/tasks/runner.js +5 -13
- package/dist/workflows/runtime/runs.js +1 -117
- package/dist/workflows/runtime/workflow-asset-loader.js +125 -0
- package/package.json +5 -5
- package/dist/commands/db-cli.js +0 -23
- package/dist/indexer/db/db-backup.js +0 -376
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
// This Source Code Form is subject to the terms of the Mozilla Public
|
|
2
|
+
// License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
3
|
+
// file, You can obtain one at https://mozilla.org/MPL/2.0/.
|
|
4
|
+
/**
|
|
5
|
+
* Incremental dir-staleness engine.
|
|
6
|
+
*
|
|
7
|
+
* Decides, per stash directory, whether the directory's indexed rows are still
|
|
8
|
+
* fresh relative to what is on disk — so an incremental `akm index` run can
|
|
9
|
+
* skip unchanged directories instead of regenerating their metadata.
|
|
10
|
+
*
|
|
11
|
+
* Two persisted signals back the decision:
|
|
12
|
+
* 1. The `entries` rows already indexed for the directory (`getEntriesByDir`).
|
|
13
|
+
* 2. The `index_dir_state` fingerprint row (`getIndexDirState`), which caches
|
|
14
|
+
* the file-set hash + max mtime for directories that legitimately produced
|
|
15
|
+
* zero rows, so they are not rescanned every run.
|
|
16
|
+
*
|
|
17
|
+
* `computeDirFingerprint` derives the fingerprint (basename set + max mtime)
|
|
18
|
+
* that both the freshness check and the persisted `index_dir_state` row use.
|
|
19
|
+
*/
|
|
20
|
+
import fs from "node:fs";
|
|
21
|
+
import path from "node:path";
|
|
22
|
+
import { getEntriesByDir, getIndexDirState } from "../db/db.js";
|
|
23
|
+
export function getDirIndexState(db, dirPath, files, builtAtMs) {
|
|
24
|
+
const prevEntries = getEntriesByDir(db, dirPath);
|
|
25
|
+
const fingerprint = computeDirFingerprint(dirPath, files);
|
|
26
|
+
if (prevEntries.length > 0) {
|
|
27
|
+
const staleReason = getDirStaleReason(dirPath, files, prevEntries, builtAtMs);
|
|
28
|
+
if (!staleReason) {
|
|
29
|
+
return { stale: false, reason: { kind: "unchanged" }, persistedRowCount: prevEntries.length };
|
|
30
|
+
}
|
|
31
|
+
return { stale: true, reason: staleReason, persistedRowCount: prevEntries.length };
|
|
32
|
+
}
|
|
33
|
+
const cachedState = getIndexDirState(db, dirPath);
|
|
34
|
+
if (cachedState &&
|
|
35
|
+
cachedState.fileSetHash === fingerprint.fileSetHash &&
|
|
36
|
+
cachedState.fileMtimeMaxMs === fingerprint.fileMtimeMaxMs) {
|
|
37
|
+
return {
|
|
38
|
+
stale: false,
|
|
39
|
+
reason: { kind: "cached-zero-row-state", detail: cachedState.reason },
|
|
40
|
+
persistedRowCount: 0,
|
|
41
|
+
};
|
|
42
|
+
}
|
|
43
|
+
return {
|
|
44
|
+
stale: true,
|
|
45
|
+
reason: { kind: "no-previous-rows", detail: cachedState ? `cached=${cachedState.reason}` : undefined },
|
|
46
|
+
persistedRowCount: 0,
|
|
47
|
+
};
|
|
48
|
+
}
|
|
49
|
+
export function getCachedZeroRowDirState(db, dirPath, files, builtAtMs, priorDirsChanged) {
|
|
50
|
+
const state = getDirIndexState(db, dirPath, files, builtAtMs);
|
|
51
|
+
if (state.stale || state.reason.kind !== "cached-zero-row-state")
|
|
52
|
+
return undefined;
|
|
53
|
+
if (!canUseIncrementalSkip(state, priorDirsChanged))
|
|
54
|
+
return undefined;
|
|
55
|
+
return state;
|
|
56
|
+
}
|
|
57
|
+
export function canUseIncrementalSkip(state, priorDirsChanged) {
|
|
58
|
+
return !(priorDirsChanged &&
|
|
59
|
+
state.reason.kind === "cached-zero-row-state" &&
|
|
60
|
+
state.reason.detail === "deduped-zero-row");
|
|
61
|
+
}
|
|
62
|
+
export function computeDirFingerprint(_dirPath, files) {
|
|
63
|
+
const normalizedFiles = [...new Set(files.map((file) => path.basename(file)))].sort();
|
|
64
|
+
let fileMtimeMaxMs = 0;
|
|
65
|
+
for (const file of files) {
|
|
66
|
+
try {
|
|
67
|
+
fileMtimeMaxMs = Math.max(fileMtimeMaxMs, fs.statSync(file).mtimeMs);
|
|
68
|
+
}
|
|
69
|
+
catch {
|
|
70
|
+
fileMtimeMaxMs = Number.POSITIVE_INFINITY;
|
|
71
|
+
break;
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
return {
|
|
75
|
+
fileSetHash: normalizedFiles.join("\0"),
|
|
76
|
+
fileMtimeMaxMs,
|
|
77
|
+
};
|
|
78
|
+
}
|
|
79
|
+
function getDirStaleReason(_dirPath, currentFiles, previousEntries, builtAtMs) {
|
|
80
|
+
const prevFileNames = new Set(previousEntries
|
|
81
|
+
.map((ie) => {
|
|
82
|
+
const fromPath = path.basename(ie.filePath);
|
|
83
|
+
return fromPath || ie.entry.filename;
|
|
84
|
+
})
|
|
85
|
+
.filter((e) => !!e));
|
|
86
|
+
const currFileNames = new Set(currentFiles.map((f) => path.basename(f)));
|
|
87
|
+
if (prevFileNames.size !== currFileNames.size) {
|
|
88
|
+
return { kind: "file-set-changed", detail: `${prevFileNames.size} -> ${currFileNames.size} files` };
|
|
89
|
+
}
|
|
90
|
+
for (const name of currFileNames) {
|
|
91
|
+
if (!prevFileNames.has(name))
|
|
92
|
+
return { kind: "file-set-changed", detail: name };
|
|
93
|
+
}
|
|
94
|
+
for (const file of currentFiles) {
|
|
95
|
+
try {
|
|
96
|
+
if (fs.statSync(file).mtimeMs > builtAtMs)
|
|
97
|
+
return { kind: "mtime-changed", detail: path.basename(file) };
|
|
98
|
+
}
|
|
99
|
+
catch {
|
|
100
|
+
return { kind: "missing-file", detail: path.basename(file) };
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
return undefined;
|
|
104
|
+
}
|
|
105
|
+
export function inferZeroRowReason(stash, priorReason, warnings, dirPath, dedupedRows) {
|
|
106
|
+
if (dedupedRows > 0)
|
|
107
|
+
return "deduped-zero-row";
|
|
108
|
+
const workflowNoise = warnings.some((warning) => warning.startsWith("Skipped workflow ") && warning.includes(dirPath));
|
|
109
|
+
if (workflowNoise)
|
|
110
|
+
return "workflow-noise";
|
|
111
|
+
if (!stash || stash.entries.length === 0)
|
|
112
|
+
return "empty-generated-set";
|
|
113
|
+
return `zero-row:${priorReason?.kind ?? "unknown"}`;
|
|
114
|
+
}
|
|
@@ -5,13 +5,11 @@ import fs from "node:fs";
|
|
|
5
5
|
import path from "node:path";
|
|
6
6
|
import { resolveStashDir } from "../../core/common.js";
|
|
7
7
|
import { getSources, loadConfig } from "../../core/config/config.js";
|
|
8
|
-
import { resolveSourceProviderFactory } from "../../sources/provider-factory.js";
|
|
8
|
+
import { resolveSourceProviderFactory, resolveSourceProviders } from "../../sources/provider-factory.js";
|
|
9
9
|
// Eager side-effect imports so all built-in source providers self-register
|
|
10
10
|
// before resolveEntryContentDir() runs.
|
|
11
11
|
import "../../sources/providers/index.js";
|
|
12
12
|
import { warn } from "../../core/warn.js";
|
|
13
|
-
import { ensureGitMirror, getCachePaths, parseGitRepoUrl } from "../../sources/providers/git.js";
|
|
14
|
-
import { ensureWebsiteMirror } from "../../sources/website-ingest.js";
|
|
15
13
|
// Legacy "context-hub" / "github" type aliases are normalized to "git" at
|
|
16
14
|
// config-load time (see src/config.ts), so this set only contains the canonical
|
|
17
15
|
// type.
|
|
@@ -261,31 +259,19 @@ function isValidDirectory(dir) {
|
|
|
261
259
|
export async function ensureSourceCaches(config, options) {
|
|
262
260
|
const cfg = config ?? loadConfig();
|
|
263
261
|
const force = options?.force === true;
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
262
|
+
// Polymorphic refresh: walk every enabled source through its registered
|
|
263
|
+
// provider and call `sync()`. Every cache-backed kind (git, website, npm)
|
|
264
|
+
// refreshes the same way — a bad source warns and is skipped without
|
|
265
|
+
// aborting the others. The git content/-subdir layout convention stays in
|
|
266
|
+
// resolveEntryContentDir.
|
|
267
|
+
for (const provider of resolveSourceProviders(cfg)) {
|
|
268
|
+
if (!provider.sync)
|
|
267
269
|
continue;
|
|
268
270
|
try {
|
|
269
|
-
|
|
270
|
-
const cachePaths = getCachePaths(repo.canonicalUrl);
|
|
271
|
-
await ensureGitMirror(repo, cachePaths, {
|
|
272
|
-
requireRepoDir: true,
|
|
273
|
-
writable: entry.writable === true,
|
|
274
|
-
force,
|
|
275
|
-
});
|
|
276
|
-
}
|
|
277
|
-
catch (err) {
|
|
278
|
-
warn(`Warning: failed to refresh git mirror for "${entry.url}": ${err instanceof Error ? err.message : String(err)}`);
|
|
279
|
-
}
|
|
280
|
-
}
|
|
281
|
-
for (const entry of entries) {
|
|
282
|
-
if (entry.type !== "website" || !entry.url || entry.enabled === false)
|
|
283
|
-
continue;
|
|
284
|
-
try {
|
|
285
|
-
await ensureWebsiteMirror(entry, { requireStashDir: true, force });
|
|
271
|
+
await provider.sync({ force });
|
|
286
272
|
}
|
|
287
273
|
catch (err) {
|
|
288
|
-
warn(`Warning: failed to refresh
|
|
274
|
+
warn(`Warning: failed to refresh ${provider.kind} source "${provider.name}": ${err instanceof Error ? err.message : String(err)}`);
|
|
289
275
|
}
|
|
290
276
|
}
|
|
291
277
|
}
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
// This Source Code Form is subject to the terms of the Mozilla Public
|
|
2
|
+
// License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
3
|
+
// file, You can obtain one at https://mozilla.org/MPL/2.0/.
|
|
4
|
+
/**
|
|
5
|
+
* X3 — the ONE dispatch seam for the {@link RunnerSpec} tagged union.
|
|
6
|
+
*
|
|
7
|
+
* The improve slice dispatches a `RunnerSpec` (`llm | agent | sdk`) in several
|
|
8
|
+
* places (`reflect.ts`, `proposal/drain.ts`, …). Before this module each site
|
|
9
|
+
* re-rolled the identical 3-arm switch and re-declared its own per-kind test
|
|
10
|
+
* seams (`chat`, `runAgentFn`, `runSdkFn`). `executeRunner` collapses that into
|
|
11
|
+
* one switch + one {@link RunnerSeams} object.
|
|
12
|
+
*
|
|
13
|
+
* Scoping (behavior-preserving):
|
|
14
|
+
* - The `agent` and `sdk` arms are byte-identical across call sites: invoke
|
|
15
|
+
* the profile runner (`runAgent` / `runOpencodeSdk`) with the per-call
|
|
16
|
+
* `RunAgentOptions` the caller passes. Those default runners live here so
|
|
17
|
+
* callers stop importing `runAgent` / `runOpencodeSdk` for dispatch. The
|
|
18
|
+
* `opts` (incl. any `timeoutMs`) is constructed by the caller and passed
|
|
19
|
+
* through unchanged, so each site keeps its exact option set.
|
|
20
|
+
* - The `llm` arm is irreducibly caller-specific (reflect wraps
|
|
21
|
+
* `runReflectViaLlm`, which returns reflect's iteration shape; drain wraps a
|
|
22
|
+
* plain `chatCompletion`). It is therefore a REQUIRED seam — there is no
|
|
23
|
+
* default `llm` handler — so neither caller's bespoke behavior is changed.
|
|
24
|
+
* - The `assertNever` exhaustiveness arm is kept so a 4th `RunnerSpec` kind is
|
|
25
|
+
* a compile error here instead of a silent runtime fall-through.
|
|
26
|
+
*
|
|
27
|
+
* The return type is {@link AgentRunResult} so a later `callStructured` layer
|
|
28
|
+
* (X2) can wrap `executeRunner` without changing this contract.
|
|
29
|
+
*/
|
|
30
|
+
import { assertNever } from "../../core/assert.js";
|
|
31
|
+
import { runOpencodeSdk } from "../harnesses/opencode-sdk/index.js";
|
|
32
|
+
import { runAgent } from "./spawn.js";
|
|
33
|
+
/**
|
|
34
|
+
* Dispatch a {@link RunnerSpec} to its runner and return the raw
|
|
35
|
+
* {@link AgentRunResult}. `opts` is the {@link RunAgentOptions} for the profile
|
|
36
|
+
* (`agent` / `sdk`) arms; it is passed through unchanged so each caller keeps
|
|
37
|
+
* its exact option set (incl. any `timeoutMs` the caller chose to apply).
|
|
38
|
+
*/
|
|
39
|
+
export async function executeRunner(spec, prompt, opts, seams = {}) {
|
|
40
|
+
switch (spec.kind) {
|
|
41
|
+
case "llm": {
|
|
42
|
+
if (!seams.llm) {
|
|
43
|
+
throw new Error("executeRunner: an `llm` runner requires a `seams.llm` handler (no default LLM dispatch).");
|
|
44
|
+
}
|
|
45
|
+
return seams.llm(spec, prompt);
|
|
46
|
+
}
|
|
47
|
+
case "agent": {
|
|
48
|
+
const run = seams.runAgent ?? runAgent;
|
|
49
|
+
return run(spec.profile, prompt, opts);
|
|
50
|
+
}
|
|
51
|
+
case "sdk": {
|
|
52
|
+
const run = seams.runSdk ?? runOpencodeSdk;
|
|
53
|
+
return run(spec.profile, prompt, opts);
|
|
54
|
+
}
|
|
55
|
+
default:
|
|
56
|
+
// Exhaustiveness arm: a 4th RunnerSpec kind becomes a compile error here.
|
|
57
|
+
return assertNever(spec);
|
|
58
|
+
}
|
|
59
|
+
}
|
package/dist/llm/client.js
CHANGED
|
@@ -103,18 +103,29 @@ function retryBackoffMs() {
|
|
|
103
103
|
return RETRY_BACKOFF_MIN_MS + Math.random() * (RETRY_BACKOFF_MAX_MS - RETRY_BACKOFF_MIN_MS);
|
|
104
104
|
}
|
|
105
105
|
/**
|
|
106
|
-
* Detect whether an error message indicates a context
|
|
107
|
-
*
|
|
108
|
-
*
|
|
106
|
+
* Detect whether an error message indicates a context size exceeded condition.
|
|
107
|
+
* Covers common patterns from OpenAI-compatible APIs (LM Studio, Ollama, etc).
|
|
108
|
+
*
|
|
109
|
+
* Requires BOTH a context keyword AND token-count/overflow evidence so that
|
|
110
|
+
* model prose merely mentioning "context size" / "context length" (e.g. gemma
|
|
111
|
+
* narrating about a document) does not get misclassified as a provider
|
|
112
|
+
* context-limit error (#496).
|
|
113
|
+
*
|
|
114
|
+
* Canonical home: `graph-extract.ts` re-exports this so the index-pass
|
|
115
|
+
* graph extractor and the retry classifier (`isRetryable`) share one
|
|
116
|
+
* definition — retrying a context overflow cannot shrink the input, so it
|
|
117
|
+
* must never be retried.
|
|
109
118
|
*/
|
|
110
|
-
function
|
|
119
|
+
export function isContextSizeError(message) {
|
|
111
120
|
const lower = message.toLowerCase();
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
121
|
+
const contextKw = /context (size|length|window)|prompt too long|exceeds.*context/.test(lower);
|
|
122
|
+
if (!contextKw) {
|
|
123
|
+
return false;
|
|
124
|
+
}
|
|
125
|
+
const evidence = /\b\d+\s*(token|tokens|tk)\b/.test(lower) ||
|
|
126
|
+
/max(imum)?\s+(context|token|input)/.test(lower) ||
|
|
127
|
+
/exceeded|over.*limit|too.*long/.test(lower);
|
|
128
|
+
return evidence;
|
|
118
129
|
}
|
|
119
130
|
/**
|
|
120
131
|
* Decide whether a first-attempt {@link LlmCallError} is eligible for a single
|
|
@@ -138,7 +149,7 @@ function looksLikeContextOverflow(message) {
|
|
|
138
149
|
* failure in the improve/reflect and capability-probe flows.
|
|
139
150
|
*/
|
|
140
151
|
function isRetryable(err) {
|
|
141
|
-
if (
|
|
152
|
+
if (isContextSizeError(err.message))
|
|
142
153
|
return false;
|
|
143
154
|
if (err.code === "provider_error") {
|
|
144
155
|
return typeof err.statusCode === "number" && err.statusCode >= 500;
|
|
@@ -24,8 +24,9 @@ import systemPromptTemplate from "../assets/prompts/graph-extract-system.md" wit
|
|
|
24
24
|
import userPromptTemplate from "../assets/prompts/graph-extract-user-prompt.md" with { type: "text" };
|
|
25
25
|
import { toErrorMessage } from "../core/common.js";
|
|
26
26
|
import { warn, warnVerbose } from "../core/warn.js";
|
|
27
|
-
import { chatCompletion,
|
|
27
|
+
import { chatCompletion, isContextSizeError, parseEmbeddedJsonResponse } from "./client.js";
|
|
28
28
|
import { tryLlmFeature } from "./feature-gate.js";
|
|
29
|
+
import { callStructured } from "./structured-call.js";
|
|
29
30
|
/**
|
|
30
31
|
* Separator token used between assets in a batch prompt.
|
|
31
32
|
* Chosen to be visually clear and unlikely to appear verbatim in asset bodies.
|
|
@@ -46,26 +47,11 @@ const SYSTEM_PROMPT = systemPromptTemplate;
|
|
|
46
47
|
const USER_PROMPT_PREFIX = userPromptTemplate
|
|
47
48
|
.replace("{{MAX_ENTITIES}}", String(MAX_ENTITIES_PER_ASSET))
|
|
48
49
|
.replace("{{MAX_RELATIONS}}", String(MAX_RELATIONS_PER_ASSET));
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
* model prose merely mentioning "context size" / "context length" (e.g. gemma
|
|
55
|
-
* narrating about a document) does not get misclassified as a provider
|
|
56
|
-
* context-limit error (#496).
|
|
57
|
-
*/
|
|
58
|
-
export function isContextSizeError(message) {
|
|
59
|
-
const lower = message.toLowerCase();
|
|
60
|
-
const contextKw = /context (size|length|window)|prompt too long|exceeds.*context/.test(lower);
|
|
61
|
-
if (!contextKw) {
|
|
62
|
-
return false;
|
|
63
|
-
}
|
|
64
|
-
const evidence = /\b\d+\s*(token|tokens|tk)\b/.test(lower) ||
|
|
65
|
-
/max(imum)?\s+(context|token|input)/.test(lower) ||
|
|
66
|
-
/exceeded|over.*limit|too.*long/.test(lower);
|
|
67
|
-
return evidence;
|
|
68
|
-
}
|
|
50
|
+
// `isContextSizeError` is defined in `./client` and re-exported here so the
|
|
51
|
+
// graph extractor and the retry classifier (`isRetryable`) share one
|
|
52
|
+
// definition (#496). Re-exported (not just imported) to preserve existing
|
|
53
|
+
// importers of this module — including its unit test.
|
|
54
|
+
export { isContextSizeError } from "./client.js";
|
|
69
55
|
const GENERIC_ENTITIES = new Set([
|
|
70
56
|
"agent",
|
|
71
57
|
"application",
|
|
@@ -703,17 +689,21 @@ export async function extractGraphFromBody(llmConfig, body, signal, akmConfig, o
|
|
|
703
689
|
return merged;
|
|
704
690
|
}
|
|
705
691
|
const userPrompt = `${USER_PROMPT_PREFIX}${trimmedBody}`;
|
|
706
|
-
return
|
|
707
|
-
|
|
708
|
-
|
|
709
|
-
|
|
710
|
-
|
|
711
|
-
|
|
712
|
-
|
|
713
|
-
|
|
714
|
-
|
|
715
|
-
|
|
716
|
-
|
|
692
|
+
return callStructured({
|
|
693
|
+
feature: "graph_extraction",
|
|
694
|
+
akmConfig,
|
|
695
|
+
config: llmConfig,
|
|
696
|
+
messages: [
|
|
697
|
+
{ role: "system", content: SYSTEM_PROMPT },
|
|
698
|
+
{ role: "user", content: userPrompt },
|
|
699
|
+
],
|
|
700
|
+
request: {
|
|
701
|
+
temperature: 0.1,
|
|
702
|
+
timeoutMs: llmConfig.timeoutMs,
|
|
703
|
+
signal,
|
|
704
|
+
onRetryAttempt: () => bumpTelemetry(options.telemetry, "retryAttempts"),
|
|
705
|
+
},
|
|
706
|
+
parse: (raw) => {
|
|
717
707
|
if (!raw)
|
|
718
708
|
return empty();
|
|
719
709
|
const parsed = parseEmbeddedJsonResponse(raw);
|
|
@@ -729,16 +719,16 @@ export async function extractGraphFromBody(llmConfig, body, signal, akmConfig, o
|
|
|
729
719
|
if (extraction.status === "failed")
|
|
730
720
|
bumpTelemetry(options.telemetry, "failureCount");
|
|
731
721
|
return extraction;
|
|
732
|
-
}
|
|
733
|
-
|
|
722
|
+
},
|
|
723
|
+
onError: (cls, err) => {
|
|
734
724
|
const errMsg = toErrorMessage(err);
|
|
735
|
-
if (
|
|
725
|
+
if (cls === "context_limit") {
|
|
736
726
|
bumpTelemetry(options.telemetry, "failureCount");
|
|
737
727
|
warn(`graph extraction: context size exceeded for asset; promptChars=${userPrompt.length}${formatContextHint(llmConfig)}. ` +
|
|
738
728
|
`Consider increasing llm.contextLength in config.json.`);
|
|
739
729
|
return empty("context_limit", "failed");
|
|
740
730
|
}
|
|
741
|
-
else if (
|
|
731
|
+
else if (cls === "html") {
|
|
742
732
|
bumpTelemetry(options.telemetry, "htmlErrorCount");
|
|
743
733
|
warn(`graph extraction: provider returned HTML instead of JSON for asset; promptChars=${userPrompt.length}${formatContextHint(llmConfig)}: ${errMsg}`);
|
|
744
734
|
return empty("llm_error", "failed");
|
|
@@ -748,9 +738,8 @@ export async function extractGraphFromBody(llmConfig, body, signal, akmConfig, o
|
|
|
748
738
|
warn(`graph extraction failed for asset; promptChars=${userPrompt.length}${formatContextHint(llmConfig)}: ${errMsg}`);
|
|
749
739
|
return empty("llm_error", "failed");
|
|
750
740
|
}
|
|
751
|
-
}
|
|
752
|
-
|
|
753
|
-
timeoutMs: llmConfig.timeoutMs,
|
|
741
|
+
},
|
|
742
|
+
fallback: empty(),
|
|
754
743
|
onFallback,
|
|
755
744
|
});
|
|
756
745
|
}
|
package/dist/llm/memory-infer.js
CHANGED
|
@@ -22,8 +22,8 @@ import memoryInferSystemPrompt from "../assets/prompts/memory-infer-system.md" w
|
|
|
22
22
|
import memoryInferUserPrompt from "../assets/prompts/memory-infer-user.md" with { type: "text" };
|
|
23
23
|
import { toErrorMessage } from "../core/common.js";
|
|
24
24
|
import { warn } from "../core/warn.js";
|
|
25
|
-
import {
|
|
26
|
-
import {
|
|
25
|
+
import { parseEmbeddedJsonResponse } from "./client.js";
|
|
26
|
+
import { callStructured } from "./structured-call.js";
|
|
27
27
|
/** Hard cap on body chars sent to the model — pragmatic and matches `runLlmEnrich`. */
|
|
28
28
|
const MAX_BODY_CHARS = 4000;
|
|
29
29
|
const SYSTEM_PROMPT = memoryInferSystemPrompt;
|
|
@@ -59,26 +59,39 @@ const DERIVED_MEMORY_JSON_SCHEMA = {
|
|
|
59
59
|
* Errors are logged via `warn()` but never thrown — a failed split for one memory
|
|
60
60
|
* must not abort the rest of the index pass.
|
|
61
61
|
*
|
|
62
|
-
* Routes through `
|
|
63
|
-
* and onFallback hook are honoured uniformly
|
|
62
|
+
* Routes through `callStructured({ feature: "memory_inference", ... })` so the
|
|
63
|
+
* feature gate, error classification, and onFallback hook are honoured uniformly
|
|
64
|
+
* (Fix C5).
|
|
64
65
|
*/
|
|
65
66
|
export async function compressMemoryToDerivedMemory(llmConfig, body, signal, akmConfig, onFallback, telemetry, onRetryAttempt) {
|
|
66
67
|
const trimmedBody = body.trim();
|
|
67
68
|
if (!trimmedBody)
|
|
68
69
|
return undefined;
|
|
69
70
|
const userPrompt = `${USER_PROMPT_PREFIX}${trimmedBody.slice(0, MAX_BODY_CHARS)}`;
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
71
|
+
// Memory-inference is ALWAYS gated: no `akmConfig` ⇒ gate closed (no chat,
|
|
72
|
+
// `disabled` fallback), never the seam's ungated/propagate path (which is for
|
|
73
|
+
// direct callers like `enhanceMetadata`). This is the gate-closed branch
|
|
74
|
+
// `tryLlmFeature(_, undefined, _)` took before the migration.
|
|
75
|
+
if (!akmConfig) {
|
|
76
|
+
onFallback?.({ feature: "memory_inference", reason: "disabled" });
|
|
77
|
+
return undefined;
|
|
78
|
+
}
|
|
79
|
+
return callStructured({
|
|
80
|
+
feature: "memory_inference",
|
|
81
|
+
akmConfig,
|
|
82
|
+
config: llmConfig,
|
|
83
|
+
messages: [
|
|
84
|
+
{ role: "system", content: SYSTEM_PROMPT },
|
|
85
|
+
{ role: "user", content: userPrompt },
|
|
86
|
+
],
|
|
87
|
+
request: {
|
|
88
|
+
temperature: 0.1,
|
|
89
|
+
timeoutMs: llmConfig.timeoutMs,
|
|
90
|
+
signal,
|
|
91
|
+
responseSchema: DERIVED_MEMORY_JSON_SCHEMA,
|
|
92
|
+
onRetryAttempt,
|
|
93
|
+
},
|
|
94
|
+
parse: (raw) => {
|
|
82
95
|
if (!raw)
|
|
83
96
|
return undefined;
|
|
84
97
|
const parsed = parseEmbeddedJsonResponse(raw);
|
|
@@ -108,9 +121,9 @@ export async function compressMemoryToDerivedMemory(llmConfig, body, signal, akm
|
|
|
108
121
|
return undefined;
|
|
109
122
|
}
|
|
110
123
|
return { title, description, tags, searchHints, content };
|
|
111
|
-
}
|
|
112
|
-
|
|
113
|
-
if (
|
|
124
|
+
},
|
|
125
|
+
onError: (cls, err) => {
|
|
126
|
+
if (cls === "html") {
|
|
114
127
|
if (telemetry)
|
|
115
128
|
telemetry.htmlErrorCount = (telemetry.htmlErrorCount ?? 0) + 1;
|
|
116
129
|
warn(`memory inference: provider returned HTML instead of JSON; skipping memory: ${toErrorMessage(err)}`);
|
|
@@ -118,9 +131,8 @@ export async function compressMemoryToDerivedMemory(llmConfig, body, signal, akm
|
|
|
118
131
|
}
|
|
119
132
|
warn(`memory inference failed: ${toErrorMessage(err)}`);
|
|
120
133
|
return undefined;
|
|
121
|
-
}
|
|
122
|
-
|
|
123
|
-
timeoutMs: llmConfig.timeoutMs,
|
|
134
|
+
},
|
|
135
|
+
fallback: undefined,
|
|
124
136
|
onFallback,
|
|
125
137
|
});
|
|
126
138
|
}
|
|
@@ -9,8 +9,8 @@
|
|
|
9
9
|
* transport client in `client.ts`.
|
|
10
10
|
*/
|
|
11
11
|
import metadataEnhanceSystemPrompt from "../assets/prompts/metadata-enhance-system.md" with { type: "text" };
|
|
12
|
-
import {
|
|
13
|
-
import {
|
|
12
|
+
import { parseJsonResponse } from "./client.js";
|
|
13
|
+
import { callStructured } from "./structured-call.js";
|
|
14
14
|
const SYSTEM_PROMPT = metadataEnhanceSystemPrompt;
|
|
15
15
|
/**
|
|
16
16
|
* Use an LLM to enhance a stash entry's metadata: improve description,
|
|
@@ -41,34 +41,39 @@ Generate improved metadata for this ${entry.type}. Return JSON with these fields
|
|
|
41
41
|
- "tags": an array of 3-8 relevant keyword tags
|
|
42
42
|
|
|
43
43
|
Return ONLY the JSON object, no explanation.`;
|
|
44
|
-
|
|
45
|
-
|
|
44
|
+
// `parse` owns the raw response: the `!raw`/unparseable case ⇒ `{}`, plus the
|
|
45
|
+
// description/searchHints/tags shaping. `enhanceMetadata` never warns and
|
|
46
|
+
// never bumps telemetry, so `onError` (gated path only) just swallows to `{}`
|
|
47
|
+
// — identical to the surrounding control flow's pre-migration behaviour. The
|
|
48
|
+
// ungated path (akmConfig === undefined) propagates errors via callStructured.
|
|
49
|
+
return callStructured({
|
|
50
|
+
feature: "metadata_enhance",
|
|
51
|
+
akmConfig,
|
|
52
|
+
config,
|
|
53
|
+
messages: [
|
|
46
54
|
{ role: "system", content: SYSTEM_PROMPT },
|
|
47
55
|
{ role: "user", content: userPrompt },
|
|
48
|
-
],
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
result
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
.
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
return runLlm();
|
|
72
|
-
}
|
|
73
|
-
return tryLlmFeature("metadata_enhance", akmConfig, runLlm, {}, { timeoutMs: config.timeoutMs });
|
|
56
|
+
],
|
|
57
|
+
request: { signal, timeoutMs: config.timeoutMs },
|
|
58
|
+
parse: (raw) => {
|
|
59
|
+
const parsed = raw ? parseJsonResponse(raw) : undefined;
|
|
60
|
+
if (!parsed)
|
|
61
|
+
return {};
|
|
62
|
+
const result = {};
|
|
63
|
+
if (typeof parsed.description === "string" && parsed.description) {
|
|
64
|
+
result.description = parsed.description;
|
|
65
|
+
}
|
|
66
|
+
if (Array.isArray(parsed.searchHints)) {
|
|
67
|
+
result.searchHints = parsed.searchHints
|
|
68
|
+
.filter((s) => typeof s === "string" && s.trim().length > 0)
|
|
69
|
+
.slice(0, 8);
|
|
70
|
+
}
|
|
71
|
+
if (Array.isArray(parsed.tags)) {
|
|
72
|
+
result.tags = parsed.tags.filter((s) => typeof s === "string" && s.trim().length > 0).slice(0, 10);
|
|
73
|
+
}
|
|
74
|
+
return result;
|
|
75
|
+
},
|
|
76
|
+
onError: () => ({}),
|
|
77
|
+
fallback: {},
|
|
78
|
+
});
|
|
74
79
|
}
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
// This Source Code Form is subject to the terms of the Mozilla Public
|
|
2
|
+
// License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
3
|
+
// file, You can obtain one at https://mozilla.org/MPL/2.0/.
|
|
4
|
+
import { chatCompletion, isContextSizeError, LlmCallError } from "./client.js";
|
|
5
|
+
import { tryLlmFeature } from "./feature-gate.js";
|
|
6
|
+
/**
|
|
7
|
+
* Classify a thrown LLM error into one of the three buckets. This is the single
|
|
8
|
+
* home for the `isContextSizeError -> html -> other` ladder that was previously
|
|
9
|
+
* inlined at every call site.
|
|
10
|
+
*/
|
|
11
|
+
export function classifyLlmError(err) {
|
|
12
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
13
|
+
if (isContextSizeError(message))
|
|
14
|
+
return "context_limit";
|
|
15
|
+
if (err instanceof LlmCallError && err.code === "provider_html_error")
|
|
16
|
+
return "html";
|
|
17
|
+
return "other";
|
|
18
|
+
}
|
|
19
|
+
export async function callStructured(opts) {
|
|
20
|
+
const { feature, akmConfig, config, messages, request, parse, onError, fallback, onFallback } = opts;
|
|
21
|
+
const chat = request?.chat ?? chatCompletion;
|
|
22
|
+
const chatOptions = {
|
|
23
|
+
temperature: request?.temperature,
|
|
24
|
+
timeoutMs: request?.timeoutMs,
|
|
25
|
+
signal: request?.signal,
|
|
26
|
+
responseSchema: request?.responseSchema,
|
|
27
|
+
onRetryAttempt: request?.onRetryAttempt,
|
|
28
|
+
};
|
|
29
|
+
// UNGATED: run the chat+parse directly. Errors propagate — no `onError`
|
|
30
|
+
// funnel — matching the pre-gate behaviour of direct callers.
|
|
31
|
+
if (akmConfig === undefined) {
|
|
32
|
+
const raw = await chat(config, messages, chatOptions);
|
|
33
|
+
return parse(raw);
|
|
34
|
+
}
|
|
35
|
+
// GATED: run through `tryLlmFeature`. A throw inside is classified ONCE and
|
|
36
|
+
// routed to `onError`; `tryLlmFeature` returns `fallback` on disablement/timeout.
|
|
37
|
+
return tryLlmFeature(feature, akmConfig, async () => {
|
|
38
|
+
try {
|
|
39
|
+
const raw = await chat(config, messages, chatOptions);
|
|
40
|
+
return parse(raw);
|
|
41
|
+
}
|
|
42
|
+
catch (err) {
|
|
43
|
+
return onError(classifyLlmError(err), err);
|
|
44
|
+
}
|
|
45
|
+
}, fallback, {
|
|
46
|
+
timeoutMs: request?.timeoutMs,
|
|
47
|
+
onFallback,
|
|
48
|
+
});
|
|
49
|
+
}
|