akm-cli 0.9.0-beta.45 → 0.9.0-beta.47

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (81) hide show
  1. package/dist/assets/wiki/ingest-workflow-template.md +17 -10
  2. package/dist/cli/shared.js +28 -0
  3. package/dist/cli.js +1 -2
  4. package/dist/commands/env/env-cli.js +16 -24
  5. package/dist/commands/env/secret-cli.js +12 -20
  6. package/dist/commands/graph/graph-cli.js +5 -13
  7. package/dist/commands/graph/graph.js +3 -3
  8. package/dist/commands/improve/consolidate/chunking.js +141 -0
  9. package/dist/commands/improve/consolidate/eligibility.js +64 -0
  10. package/dist/commands/improve/consolidate/merge.js +145 -0
  11. package/dist/commands/improve/consolidate/sanitize.js +231 -0
  12. package/dist/commands/improve/consolidate/types.js +4 -0
  13. package/dist/commands/improve/consolidate.js +20 -571
  14. package/dist/commands/improve/distill.js +5 -9
  15. package/dist/commands/improve/eligibility.js +434 -0
  16. package/dist/commands/improve/extract-cli.js +9 -1
  17. package/dist/commands/improve/extract.js +5 -19
  18. package/dist/commands/improve/improve-auto-accept.js +4 -8
  19. package/dist/commands/improve/improve-cli.js +35 -60
  20. package/dist/commands/improve/improve-result-file.js +5 -23
  21. package/dist/commands/improve/improve-session.js +58 -0
  22. package/dist/commands/improve/improve.js +107 -3606
  23. package/dist/commands/improve/locks.js +154 -0
  24. package/dist/commands/improve/loop-stages.js +1079 -0
  25. package/dist/commands/improve/preparation.js +1963 -0
  26. package/dist/commands/improve/recombine.js +6 -12
  27. package/dist/commands/improve/reflect.js +29 -34
  28. package/dist/commands/proposal/drain.js +25 -48
  29. package/dist/commands/proposal/proposal-cli.js +21 -31
  30. package/dist/commands/proposal/validators/proposals.js +3 -7
  31. package/dist/commands/read/curate.js +70 -14
  32. package/dist/commands/read/knowledge.js +2 -2
  33. package/dist/commands/sources/self-update.js +2 -2
  34. package/dist/commands/sources/stash-cli.js +9 -37
  35. package/dist/commands/tasks/tasks-cli.js +19 -27
  36. package/dist/commands/wiki-cli.js +21 -35
  37. package/dist/core/config/config.js +18 -2
  38. package/dist/core/events.js +3 -7
  39. package/dist/core/logs-db.js +6 -63
  40. package/dist/core/state/migrations.js +714 -0
  41. package/dist/core/state-db.js +28 -779
  42. package/dist/indexer/db/db.js +82 -216
  43. package/dist/indexer/indexer.js +11 -112
  44. package/dist/indexer/passes/dir-staleness.js +114 -0
  45. package/dist/indexer/search/search-source.js +10 -24
  46. package/dist/indexer/search/semantic-status.js +4 -0
  47. package/dist/integrations/agent/runner-dispatch.js +59 -0
  48. package/dist/llm/client.js +22 -11
  49. package/dist/llm/embedder.js +15 -0
  50. package/dist/llm/embedders/deterministic.js +66 -0
  51. package/dist/llm/graph-extract.js +28 -39
  52. package/dist/llm/memory-infer.js +34 -22
  53. package/dist/llm/metadata-enhance.js +35 -30
  54. package/dist/llm/structured-call.js +49 -0
  55. package/dist/output/shapes/passthrough.js +0 -1
  56. package/dist/registry/providers/skills-sh.js +21 -147
  57. package/dist/registry/providers/static-index.js +15 -157
  58. package/dist/registry/resolve.js +22 -9
  59. package/dist/scripts/migrate-storage.js +892 -1186
  60. package/dist/scripts/migrations/import-fs-improve-runs-to-db.js +214 -179
  61. package/dist/setup/setup.js +26 -5
  62. package/dist/sources/providers/filesystem.js +0 -1
  63. package/dist/sources/providers/git-install.js +206 -0
  64. package/dist/sources/providers/git-provider.js +234 -0
  65. package/dist/sources/providers/git-stash.js +248 -0
  66. package/dist/sources/providers/git.js +10 -671
  67. package/dist/sources/providers/npm.js +2 -6
  68. package/dist/sources/providers/sync-from-ref.js +9 -1
  69. package/dist/sources/providers/website.js +2 -3
  70. package/dist/sources/website-ingest.js +51 -9
  71. package/dist/sources/wiki-fetchers/registry.js +53 -0
  72. package/dist/sources/wiki-fetchers/youtube.js +185 -0
  73. package/dist/storage/database.js +45 -10
  74. package/dist/storage/managed-db.js +82 -0
  75. package/dist/storage/repositories/registry-cache.js +92 -0
  76. package/dist/tasks/runner.js +5 -13
  77. package/dist/workflows/runtime/runs.js +1 -117
  78. package/dist/workflows/runtime/workflow-asset-loader.js +125 -0
  79. package/package.json +5 -5
  80. package/dist/commands/db-cli.js +0 -23
  81. package/dist/indexer/db/db-backup.js +0 -376
@@ -0,0 +1,114 @@
1
+ // This Source Code Form is subject to the terms of the Mozilla Public
2
+ // License, v. 2.0. If a copy of the MPL was not distributed with this
3
+ // file, You can obtain one at https://mozilla.org/MPL/2.0/.
4
+ /**
5
+ * Incremental dir-staleness engine.
6
+ *
7
+ * Decides, per stash directory, whether the directory's indexed rows are still
8
+ * fresh relative to what is on disk — so an incremental `akm index` run can
9
+ * skip unchanged directories instead of regenerating their metadata.
10
+ *
11
+ * Two persisted signals back the decision:
12
+ * 1. The `entries` rows already indexed for the directory (`getEntriesByDir`).
13
+ * 2. The `index_dir_state` fingerprint row (`getIndexDirState`), which caches
14
+ * the file-set hash + max mtime for directories that legitimately produced
15
+ * zero rows, so they are not rescanned every run.
16
+ *
17
+ * `computeDirFingerprint` derives the fingerprint (basename set + max mtime)
18
+ * that both the freshness check and the persisted `index_dir_state` row use.
19
+ */
20
+ import fs from "node:fs";
21
+ import path from "node:path";
22
+ import { getEntriesByDir, getIndexDirState } from "../db/db.js";
23
+ export function getDirIndexState(db, dirPath, files, builtAtMs) {
24
+ const prevEntries = getEntriesByDir(db, dirPath);
25
+ const fingerprint = computeDirFingerprint(dirPath, files);
26
+ if (prevEntries.length > 0) {
27
+ const staleReason = getDirStaleReason(dirPath, files, prevEntries, builtAtMs);
28
+ if (!staleReason) {
29
+ return { stale: false, reason: { kind: "unchanged" }, persistedRowCount: prevEntries.length };
30
+ }
31
+ return { stale: true, reason: staleReason, persistedRowCount: prevEntries.length };
32
+ }
33
+ const cachedState = getIndexDirState(db, dirPath);
34
+ if (cachedState &&
35
+ cachedState.fileSetHash === fingerprint.fileSetHash &&
36
+ cachedState.fileMtimeMaxMs === fingerprint.fileMtimeMaxMs) {
37
+ return {
38
+ stale: false,
39
+ reason: { kind: "cached-zero-row-state", detail: cachedState.reason },
40
+ persistedRowCount: 0,
41
+ };
42
+ }
43
+ return {
44
+ stale: true,
45
+ reason: { kind: "no-previous-rows", detail: cachedState ? `cached=${cachedState.reason}` : undefined },
46
+ persistedRowCount: 0,
47
+ };
48
+ }
49
+ export function getCachedZeroRowDirState(db, dirPath, files, builtAtMs, priorDirsChanged) {
50
+ const state = getDirIndexState(db, dirPath, files, builtAtMs);
51
+ if (state.stale || state.reason.kind !== "cached-zero-row-state")
52
+ return undefined;
53
+ if (!canUseIncrementalSkip(state, priorDirsChanged))
54
+ return undefined;
55
+ return state;
56
+ }
57
+ export function canUseIncrementalSkip(state, priorDirsChanged) {
58
+ return !(priorDirsChanged &&
59
+ state.reason.kind === "cached-zero-row-state" &&
60
+ state.reason.detail === "deduped-zero-row");
61
+ }
62
+ export function computeDirFingerprint(_dirPath, files) {
63
+ const normalizedFiles = [...new Set(files.map((file) => path.basename(file)))].sort();
64
+ let fileMtimeMaxMs = 0;
65
+ for (const file of files) {
66
+ try {
67
+ fileMtimeMaxMs = Math.max(fileMtimeMaxMs, fs.statSync(file).mtimeMs);
68
+ }
69
+ catch {
70
+ fileMtimeMaxMs = Number.POSITIVE_INFINITY;
71
+ break;
72
+ }
73
+ }
74
+ return {
75
+ fileSetHash: normalizedFiles.join("\0"),
76
+ fileMtimeMaxMs,
77
+ };
78
+ }
79
+ function getDirStaleReason(_dirPath, currentFiles, previousEntries, builtAtMs) {
80
+ const prevFileNames = new Set(previousEntries
81
+ .map((ie) => {
82
+ const fromPath = path.basename(ie.filePath);
83
+ return fromPath || ie.entry.filename;
84
+ })
85
+ .filter((e) => !!e));
86
+ const currFileNames = new Set(currentFiles.map((f) => path.basename(f)));
87
+ if (prevFileNames.size !== currFileNames.size) {
88
+ return { kind: "file-set-changed", detail: `${prevFileNames.size} -> ${currFileNames.size} files` };
89
+ }
90
+ for (const name of currFileNames) {
91
+ if (!prevFileNames.has(name))
92
+ return { kind: "file-set-changed", detail: name };
93
+ }
94
+ for (const file of currentFiles) {
95
+ try {
96
+ if (fs.statSync(file).mtimeMs > builtAtMs)
97
+ return { kind: "mtime-changed", detail: path.basename(file) };
98
+ }
99
+ catch {
100
+ return { kind: "missing-file", detail: path.basename(file) };
101
+ }
102
+ }
103
+ return undefined;
104
+ }
105
+ export function inferZeroRowReason(stash, priorReason, warnings, dirPath, dedupedRows) {
106
+ if (dedupedRows > 0)
107
+ return "deduped-zero-row";
108
+ const workflowNoise = warnings.some((warning) => warning.startsWith("Skipped workflow ") && warning.includes(dirPath));
109
+ if (workflowNoise)
110
+ return "workflow-noise";
111
+ if (!stash || stash.entries.length === 0)
112
+ return "empty-generated-set";
113
+ return `zero-row:${priorReason?.kind ?? "unknown"}`;
114
+ }
@@ -5,13 +5,11 @@ import fs from "node:fs";
5
5
  import path from "node:path";
6
6
  import { resolveStashDir } from "../../core/common.js";
7
7
  import { getSources, loadConfig } from "../../core/config/config.js";
8
- import { resolveSourceProviderFactory } from "../../sources/provider-factory.js";
8
+ import { resolveSourceProviderFactory, resolveSourceProviders } from "../../sources/provider-factory.js";
9
9
  // Eager side-effect imports so all built-in source providers self-register
10
10
  // before resolveEntryContentDir() runs.
11
11
  import "../../sources/providers/index.js";
12
12
  import { warn } from "../../core/warn.js";
13
- import { ensureGitMirror, getCachePaths, parseGitRepoUrl } from "../../sources/providers/git.js";
14
- import { ensureWebsiteMirror } from "../../sources/website-ingest.js";
15
13
  // Legacy "context-hub" / "github" type aliases are normalized to "git" at
16
14
  // config-load time (see src/config.ts), so this set only contains the canonical
17
15
  // type.
@@ -261,31 +259,19 @@ function isValidDirectory(dir) {
261
259
  export async function ensureSourceCaches(config, options) {
262
260
  const cfg = config ?? loadConfig();
263
261
  const force = options?.force === true;
264
- const entries = getSources(cfg);
265
- for (const entry of entries) {
266
- if (!GIT_STASH_TYPES.has(entry.type) || !entry.url || entry.enabled === false)
262
+ // Polymorphic refresh: walk every enabled source through its registered
263
+ // provider and call `sync()`. Every cache-backed kind (git, website, npm)
264
+ // refreshes the same way a bad source warns and is skipped without
265
+ // aborting the others. The git content/-subdir layout convention stays in
266
+ // resolveEntryContentDir.
267
+ for (const provider of resolveSourceProviders(cfg)) {
268
+ if (!provider.sync)
267
269
  continue;
268
270
  try {
269
- const repo = parseGitRepoUrl(entry.url);
270
- const cachePaths = getCachePaths(repo.canonicalUrl);
271
- await ensureGitMirror(repo, cachePaths, {
272
- requireRepoDir: true,
273
- writable: entry.writable === true,
274
- force,
275
- });
276
- }
277
- catch (err) {
278
- warn(`Warning: failed to refresh git mirror for "${entry.url}": ${err instanceof Error ? err.message : String(err)}`);
279
- }
280
- }
281
- for (const entry of entries) {
282
- if (entry.type !== "website" || !entry.url || entry.enabled === false)
283
- continue;
284
- try {
285
- await ensureWebsiteMirror(entry, { requireStashDir: true, force });
271
+ await provider.sync({ force });
286
272
  }
287
273
  catch (err) {
288
- warn(`Warning: failed to refresh website stash for "${entry.url}": ${err instanceof Error ? err.message : String(err)}`);
274
+ warn(`Warning: failed to refresh ${provider.kind} source "${provider.name}": ${err instanceof Error ? err.message : String(err)}`);
289
275
  }
290
276
  }
291
277
  }
@@ -4,8 +4,12 @@
4
4
  import fs from "node:fs";
5
5
  import { writeFileAtomic } from "../../core/common.js";
6
6
  import { getCacheDir, getSemanticStatusPath } from "../../core/paths.js";
7
+ import { DETERMINISTIC_EMBED_MODEL_ID, isDeterministicEmbedEnabled } from "../../llm/embedders/deterministic.js";
7
8
  import { DEFAULT_LOCAL_MODEL } from "../../llm/embedders/local.js";
8
9
  export function deriveSemanticProviderFingerprint(embedding) {
10
+ if (isDeterministicEmbedEnabled()) {
11
+ return `deterministic:${DETERMINISTIC_EMBED_MODEL_ID}`;
12
+ }
9
13
  if (embedding?.endpoint) {
10
14
  return `remote:${embedding.endpoint}|${embedding.model}|${embedding.dimension ?? "default"}`;
11
15
  }
@@ -0,0 +1,59 @@
1
+ // This Source Code Form is subject to the terms of the Mozilla Public
2
+ // License, v. 2.0. If a copy of the MPL was not distributed with this
3
+ // file, You can obtain one at https://mozilla.org/MPL/2.0/.
4
+ /**
5
+ * X3 — the ONE dispatch seam for the {@link RunnerSpec} tagged union.
6
+ *
7
+ * The improve slice dispatches a `RunnerSpec` (`llm | agent | sdk`) in several
8
+ * places (`reflect.ts`, `proposal/drain.ts`, …). Before this module each site
9
+ * re-rolled the identical 3-arm switch and re-declared its own per-kind test
10
+ * seams (`chat`, `runAgentFn`, `runSdkFn`). `executeRunner` collapses that into
11
+ * one switch + one {@link RunnerSeams} object.
12
+ *
13
+ * Scoping (behavior-preserving):
14
+ * - The `agent` and `sdk` arms are byte-identical across call sites: invoke
15
+ * the profile runner (`runAgent` / `runOpencodeSdk`) with the per-call
16
+ * `RunAgentOptions` the caller passes. Those default runners live here so
17
+ * callers stop importing `runAgent` / `runOpencodeSdk` for dispatch. The
18
+ * `opts` (incl. any `timeoutMs`) is constructed by the caller and passed
19
+ * through unchanged, so each site keeps its exact option set.
20
+ * - The `llm` arm is irreducibly caller-specific (reflect wraps
21
+ * `runReflectViaLlm`, which returns reflect's iteration shape; drain wraps a
22
+ * plain `chatCompletion`). It is therefore a REQUIRED seam — there is no
23
+ * default `llm` handler — so neither caller's bespoke behavior is changed.
24
+ * - The `assertNever` exhaustiveness arm is kept so a 4th `RunnerSpec` kind is
25
+ * a compile error here instead of a silent runtime fall-through.
26
+ *
27
+ * The return type is {@link AgentRunResult} so a later `callStructured` layer
28
+ * (X2) can wrap `executeRunner` without changing this contract.
29
+ */
30
+ import { assertNever } from "../../core/assert.js";
31
+ import { runOpencodeSdk } from "../harnesses/opencode-sdk/index.js";
32
+ import { runAgent } from "./spawn.js";
33
+ /**
34
+ * Dispatch a {@link RunnerSpec} to its runner and return the raw
35
+ * {@link AgentRunResult}. `opts` is the {@link RunAgentOptions} for the profile
36
+ * (`agent` / `sdk`) arms; it is passed through unchanged so each caller keeps
37
+ * its exact option set (incl. any `timeoutMs` the caller chose to apply).
38
+ */
39
+ export async function executeRunner(spec, prompt, opts, seams = {}) {
40
+ switch (spec.kind) {
41
+ case "llm": {
42
+ if (!seams.llm) {
43
+ throw new Error("executeRunner: an `llm` runner requires a `seams.llm` handler (no default LLM dispatch).");
44
+ }
45
+ return seams.llm(spec, prompt);
46
+ }
47
+ case "agent": {
48
+ const run = seams.runAgent ?? runAgent;
49
+ return run(spec.profile, prompt, opts);
50
+ }
51
+ case "sdk": {
52
+ const run = seams.runSdk ?? runOpencodeSdk;
53
+ return run(spec.profile, prompt, opts);
54
+ }
55
+ default:
56
+ // Exhaustiveness arm: a 4th RunnerSpec kind becomes a compile error here.
57
+ return assertNever(spec);
58
+ }
59
+ }
@@ -103,18 +103,29 @@ function retryBackoffMs() {
103
103
  return RETRY_BACKOFF_MIN_MS + Math.random() * (RETRY_BACKOFF_MAX_MS - RETRY_BACKOFF_MIN_MS);
104
104
  }
105
105
  /**
106
- * Detect whether an error message indicates a context-size-exceeded condition.
107
- * Mirrors the heuristic in `graph-extract.ts` retrying a context overflow
108
- * cannot shrink the input, so it must not be retried.
106
+ * Detect whether an error message indicates a context size exceeded condition.
107
+ * Covers common patterns from OpenAI-compatible APIs (LM Studio, Ollama, etc).
108
+ *
109
+ * Requires BOTH a context keyword AND token-count/overflow evidence so that
110
+ * model prose merely mentioning "context size" / "context length" (e.g. gemma
111
+ * narrating about a document) does not get misclassified as a provider
112
+ * context-limit error (#496).
113
+ *
114
+ * Canonical home: `graph-extract.ts` re-exports this so the index-pass
115
+ * graph extractor and the retry classifier (`isRetryable`) share one
116
+ * definition — retrying a context overflow cannot shrink the input, so it
117
+ * must never be retried.
109
118
  */
110
- function looksLikeContextOverflow(message) {
119
+ export function isContextSizeError(message) {
111
120
  const lower = message.toLowerCase();
112
- return (lower.includes("context") &&
113
- (lower.includes("context size") ||
114
- lower.includes("context length") ||
115
- lower.includes("context_window") ||
116
- lower.includes("prompt too long") ||
117
- lower.includes("exceeds")));
121
+ const contextKw = /context (size|length|window)|prompt too long|exceeds.*context/.test(lower);
122
+ if (!contextKw) {
123
+ return false;
124
+ }
125
+ const evidence = /\b\d+\s*(token|tokens|tk)\b/.test(lower) ||
126
+ /max(imum)?\s+(context|token|input)/.test(lower) ||
127
+ /exceeded|over.*limit|too.*long/.test(lower);
128
+ return evidence;
118
129
  }
119
130
  /**
120
131
  * Decide whether a first-attempt {@link LlmCallError} is eligible for a single
@@ -138,7 +149,7 @@ function looksLikeContextOverflow(message) {
138
149
  * failure in the improve/reflect and capability-probe flows.
139
150
  */
140
151
  function isRetryable(err) {
141
- if (looksLikeContextOverflow(err.message))
152
+ if (isContextSizeError(err.message))
142
153
  return false;
143
154
  if (err.code === "provider_error") {
144
155
  return typeof err.statusCode === "number" && err.statusCode >= 500;
@@ -2,6 +2,7 @@
2
2
  // License, v. 2.0. If a copy of the MPL was not distributed with this
3
3
  // file, You can obtain one at https://mozilla.org/MPL/2.0/.
4
4
  import { embedCacheKey, getCachedEmbedding, setCachedEmbedding } from "./embedders/cache.js";
5
+ import { DETERMINISTIC_EMBED_MODEL_ID, deterministicEmbed, isDeterministicEmbedEnabled, } from "./embedders/deterministic.js";
5
6
  import { DEFAULT_LOCAL_MODEL, isTransformersAvailable, LocalEmbedder } from "./embedders/local.js";
6
7
  import { hasRemoteEndpoint, RemoteEmbedder } from "./embedders/remote.js";
7
8
  // ── Re-exports (public API) ─────────────────────────────────────────────────
@@ -39,6 +40,10 @@ export function resetLocalEmbedder() {
39
40
  * and embedding config. Repeated identical queries return the cached vector.
40
41
  */
41
42
  export async function embed(text, embeddingConfig, signal) {
43
+ // Deterministic mode (env-gated, test/bench only): model-free, stable.
44
+ if (isDeterministicEmbedEnabled()) {
45
+ return deterministicEmbed(text);
46
+ }
42
47
  const key = embedCacheKey(text, embeddingConfig);
43
48
  const cached = getCachedEmbedding(key);
44
49
  if (cached)
@@ -58,6 +63,10 @@ export async function embed(text, embeddingConfig, signal) {
58
63
  export async function embedBatch(texts, embeddingConfig, signal) {
59
64
  if (texts.length === 0)
60
65
  return [];
66
+ // Deterministic mode (env-gated, test/bench only): model-free, stable.
67
+ if (isDeterministicEmbedEnabled()) {
68
+ return texts.map((t) => deterministicEmbed(t));
69
+ }
61
70
  if (embeddingConfig && hasRemoteEndpoint(embeddingConfig)) {
62
71
  return new RemoteEmbedder(embeddingConfig).embedBatch(texts, signal);
63
72
  }
@@ -95,6 +104,8 @@ export { cosineSimilarity } from "./embedders/types.js";
95
104
  * - No config: use `DEFAULT_LOCAL_MODEL` (the shared singleton model).
96
105
  */
97
106
  export function resolveEmbeddingModelId(embeddingConfig) {
107
+ if (isDeterministicEmbedEnabled())
108
+ return DETERMINISTIC_EMBED_MODEL_ID;
98
109
  if (!embeddingConfig)
99
110
  return DEFAULT_LOCAL_MODEL;
100
111
  if (hasRemoteEndpoint(embeddingConfig))
@@ -106,6 +117,10 @@ export function resolveEmbeddingModelId(embeddingConfig) {
106
117
  * Check whether embedding is available with a detailed reason on failure.
107
118
  */
108
119
  export async function checkEmbeddingAvailability(embeddingConfig) {
120
+ // Deterministic mode (env-gated): always available — no model, no network.
121
+ if (isDeterministicEmbedEnabled()) {
122
+ return { available: true };
123
+ }
109
124
  if (embeddingConfig && hasRemoteEndpoint(embeddingConfig)) {
110
125
  try {
111
126
  await new RemoteEmbedder(embeddingConfig).embed("test");
@@ -0,0 +1,66 @@
1
+ // This Source Code Form is subject to the terms of the Mozilla Public
2
+ // License, v. 2.0. If a copy of the MPL was not distributed with this
3
+ // file, You can obtain one at https://mozilla.org/MPL/2.0/.
4
+ /** Env var that switches the whole embedding facade into deterministic mode. */
5
+ export const DETERMINISTIC_EMBED_ENV = "AKM_EMBED_DETERMINISTIC";
6
+ /**
7
+ * Vector width. Matches the default local model (`bge-small`, 384 dims) so the
8
+ * index DB's embedding column and sqlite-vec table dimensions line up without
9
+ * any extra config.
10
+ */
11
+ export const DETERMINISTIC_EMBED_DIM = 384;
12
+ /**
13
+ * Stable model id reported for deterministic mode. Used as the embedding
14
+ * `model_id` and folded into the provider fingerprint so a deterministic index
15
+ * is never confused with a real-model index (and vice versa).
16
+ */
17
+ export const DETERMINISTIC_EMBED_MODEL_ID = "akm-deterministic-hash-v1";
18
+ /** True when deterministic embedding is enabled via env. */
19
+ export function isDeterministicEmbedEnabled() {
20
+ return process.env[DETERMINISTIC_EMBED_ENV] === "1";
21
+ }
22
+ /** FNV-1a 32-bit hash. Platform- and version-stable. */
23
+ function fnv1a(str) {
24
+ let h = 0x811c9dc5;
25
+ for (let i = 0; i < str.length; i++) {
26
+ h ^= str.charCodeAt(i);
27
+ // 32-bit FNV prime multiply via shifts to stay in uint32.
28
+ h = (h + ((h << 1) + (h << 4) + (h << 7) + (h << 8) + (h << 24))) >>> 0;
29
+ }
30
+ return h >>> 0;
31
+ }
32
+ /** Lowercase, split on non-alphanumeric, drop empties. */
33
+ function tokenize(text) {
34
+ return text
35
+ .toLowerCase()
36
+ .split(/[^a-z0-9]+/)
37
+ .filter((t) => t.length > 0);
38
+ }
39
+ /**
40
+ * Deterministically embed `text` into a unit-length vector of width `dim`
41
+ * using feature hashing. Empty / token-less input returns a fixed unit
42
+ * vector so cosine similarity never sees a zero vector (NaN guard).
43
+ */
44
+ export function deterministicEmbed(text, dim = DETERMINISTIC_EMBED_DIM) {
45
+ const vec = new Array(dim).fill(0);
46
+ const tokens = tokenize(text);
47
+ for (const tok of tokens) {
48
+ const h = fnv1a(tok);
49
+ const idx = h % dim;
50
+ // Use a higher bit for the sign so it is independent of the bucket index.
51
+ const sign = (h >>> 16) & 1 ? 1 : -1;
52
+ vec[idx] += sign;
53
+ }
54
+ let norm = 0;
55
+ for (const v of vec)
56
+ norm += v * v;
57
+ norm = Math.sqrt(norm);
58
+ if (norm === 0) {
59
+ // No usable tokens — return a fixed, stable unit vector.
60
+ vec[0] = 1;
61
+ return vec;
62
+ }
63
+ for (let i = 0; i < dim; i++)
64
+ vec[i] /= norm;
65
+ return vec;
66
+ }
@@ -24,8 +24,9 @@ import systemPromptTemplate from "../assets/prompts/graph-extract-system.md" wit
24
24
  import userPromptTemplate from "../assets/prompts/graph-extract-user-prompt.md" with { type: "text" };
25
25
  import { toErrorMessage } from "../core/common.js";
26
26
  import { warn, warnVerbose } from "../core/warn.js";
27
- import { chatCompletion, LlmCallError, parseEmbeddedJsonResponse } from "./client.js";
27
+ import { chatCompletion, isContextSizeError, parseEmbeddedJsonResponse } from "./client.js";
28
28
  import { tryLlmFeature } from "./feature-gate.js";
29
+ import { callStructured } from "./structured-call.js";
29
30
  /**
30
31
  * Separator token used between assets in a batch prompt.
31
32
  * Chosen to be visually clear and unlikely to appear verbatim in asset bodies.
@@ -46,26 +47,11 @@ const SYSTEM_PROMPT = systemPromptTemplate;
46
47
  const USER_PROMPT_PREFIX = userPromptTemplate
47
48
  .replace("{{MAX_ENTITIES}}", String(MAX_ENTITIES_PER_ASSET))
48
49
  .replace("{{MAX_RELATIONS}}", String(MAX_RELATIONS_PER_ASSET));
49
- /**
50
- * Detect whether an error message indicates a context size exceeded condition.
51
- * Covers common patterns from OpenAI-compatible APIs (LM Studio, Ollama, etc).
52
- *
53
- * Requires BOTH a context keyword AND token-count/overflow evidence so that
54
- * model prose merely mentioning "context size" / "context length" (e.g. gemma
55
- * narrating about a document) does not get misclassified as a provider
56
- * context-limit error (#496).
57
- */
58
- export function isContextSizeError(message) {
59
- const lower = message.toLowerCase();
60
- const contextKw = /context (size|length|window)|prompt too long|exceeds.*context/.test(lower);
61
- if (!contextKw) {
62
- return false;
63
- }
64
- const evidence = /\b\d+\s*(token|tokens|tk)\b/.test(lower) ||
65
- /max(imum)?\s+(context|token|input)/.test(lower) ||
66
- /exceeded|over.*limit|too.*long/.test(lower);
67
- return evidence;
68
- }
50
+ // `isContextSizeError` is defined in `./client` and re-exported here so the
51
+ // graph extractor and the retry classifier (`isRetryable`) share one
52
+ // definition (#496). Re-exported (not just imported) to preserve existing
53
+ // importers of this module — including its unit test.
54
+ export { isContextSizeError } from "./client.js";
69
55
  const GENERIC_ENTITIES = new Set([
70
56
  "agent",
71
57
  "application",
@@ -703,17 +689,21 @@ export async function extractGraphFromBody(llmConfig, body, signal, akmConfig, o
703
689
  return merged;
704
690
  }
705
691
  const userPrompt = `${USER_PROMPT_PREFIX}${trimmedBody}`;
706
- return tryLlmFeature("graph_extraction", akmConfig, async () => {
707
- try {
708
- const raw = await chatCompletion(llmConfig, [
709
- { role: "system", content: SYSTEM_PROMPT },
710
- { role: "user", content: userPrompt },
711
- ], {
712
- temperature: 0.1,
713
- timeoutMs: llmConfig.timeoutMs,
714
- signal,
715
- onRetryAttempt: () => bumpTelemetry(options.telemetry, "retryAttempts"),
716
- });
692
+ return callStructured({
693
+ feature: "graph_extraction",
694
+ akmConfig,
695
+ config: llmConfig,
696
+ messages: [
697
+ { role: "system", content: SYSTEM_PROMPT },
698
+ { role: "user", content: userPrompt },
699
+ ],
700
+ request: {
701
+ temperature: 0.1,
702
+ timeoutMs: llmConfig.timeoutMs,
703
+ signal,
704
+ onRetryAttempt: () => bumpTelemetry(options.telemetry, "retryAttempts"),
705
+ },
706
+ parse: (raw) => {
717
707
  if (!raw)
718
708
  return empty();
719
709
  const parsed = parseEmbeddedJsonResponse(raw);
@@ -729,16 +719,16 @@ export async function extractGraphFromBody(llmConfig, body, signal, akmConfig, o
729
719
  if (extraction.status === "failed")
730
720
  bumpTelemetry(options.telemetry, "failureCount");
731
721
  return extraction;
732
- }
733
- catch (err) {
722
+ },
723
+ onError: (cls, err) => {
734
724
  const errMsg = toErrorMessage(err);
735
- if (isContextSizeError(errMsg)) {
725
+ if (cls === "context_limit") {
736
726
  bumpTelemetry(options.telemetry, "failureCount");
737
727
  warn(`graph extraction: context size exceeded for asset; promptChars=${userPrompt.length}${formatContextHint(llmConfig)}. ` +
738
728
  `Consider increasing llm.contextLength in config.json.`);
739
729
  return empty("context_limit", "failed");
740
730
  }
741
- else if (err instanceof LlmCallError && err.code === "provider_html_error") {
731
+ else if (cls === "html") {
742
732
  bumpTelemetry(options.telemetry, "htmlErrorCount");
743
733
  warn(`graph extraction: provider returned HTML instead of JSON for asset; promptChars=${userPrompt.length}${formatContextHint(llmConfig)}: ${errMsg}`);
744
734
  return empty("llm_error", "failed");
@@ -748,9 +738,8 @@ export async function extractGraphFromBody(llmConfig, body, signal, akmConfig, o
748
738
  warn(`graph extraction failed for asset; promptChars=${userPrompt.length}${formatContextHint(llmConfig)}: ${errMsg}`);
749
739
  return empty("llm_error", "failed");
750
740
  }
751
- }
752
- }, empty(), {
753
- timeoutMs: llmConfig.timeoutMs,
741
+ },
742
+ fallback: empty(),
754
743
  onFallback,
755
744
  });
756
745
  }
@@ -22,8 +22,8 @@ import memoryInferSystemPrompt from "../assets/prompts/memory-infer-system.md" w
22
22
  import memoryInferUserPrompt from "../assets/prompts/memory-infer-user.md" with { type: "text" };
23
23
  import { toErrorMessage } from "../core/common.js";
24
24
  import { warn } from "../core/warn.js";
25
- import { chatCompletion, LlmCallError, parseEmbeddedJsonResponse } from "./client.js";
26
- import { tryLlmFeature } from "./feature-gate.js";
25
+ import { parseEmbeddedJsonResponse } from "./client.js";
26
+ import { callStructured } from "./structured-call.js";
27
27
  /** Hard cap on body chars sent to the model — pragmatic and matches `runLlmEnrich`. */
28
28
  const MAX_BODY_CHARS = 4000;
29
29
  const SYSTEM_PROMPT = memoryInferSystemPrompt;
@@ -59,26 +59,39 @@ const DERIVED_MEMORY_JSON_SCHEMA = {
59
59
  * Errors are logged via `warn()` but never thrown — a failed split for one memory
60
60
  * must not abort the rest of the index pass.
61
61
  *
62
- * Routes through `tryLlmFeature("memory_inference", ...)` so the feature gate
63
- * and onFallback hook are honoured uniformly (Fix C5).
62
+ * Routes through `callStructured({ feature: "memory_inference", ... })` so the
63
+ * feature gate, error classification, and onFallback hook are honoured uniformly
64
+ * (Fix C5).
64
65
  */
65
66
  export async function compressMemoryToDerivedMemory(llmConfig, body, signal, akmConfig, onFallback, telemetry, onRetryAttempt) {
66
67
  const trimmedBody = body.trim();
67
68
  if (!trimmedBody)
68
69
  return undefined;
69
70
  const userPrompt = `${USER_PROMPT_PREFIX}${trimmedBody.slice(0, MAX_BODY_CHARS)}`;
70
- return tryLlmFeature("memory_inference", akmConfig, async () => {
71
- try {
72
- const raw = await chatCompletion(llmConfig, [
73
- { role: "system", content: SYSTEM_PROMPT },
74
- { role: "user", content: userPrompt },
75
- ], {
76
- temperature: 0.1,
77
- timeoutMs: llmConfig.timeoutMs,
78
- signal,
79
- responseSchema: DERIVED_MEMORY_JSON_SCHEMA,
80
- onRetryAttempt,
81
- });
71
+ // Memory-inference is ALWAYS gated: no `akmConfig` gate closed (no chat,
72
+ // `disabled` fallback), never the seam's ungated/propagate path (which is for
73
+ // direct callers like `enhanceMetadata`). This is the gate-closed branch
74
+ // `tryLlmFeature(_, undefined, _)` took before the migration.
75
+ if (!akmConfig) {
76
+ onFallback?.({ feature: "memory_inference", reason: "disabled" });
77
+ return undefined;
78
+ }
79
+ return callStructured({
80
+ feature: "memory_inference",
81
+ akmConfig,
82
+ config: llmConfig,
83
+ messages: [
84
+ { role: "system", content: SYSTEM_PROMPT },
85
+ { role: "user", content: userPrompt },
86
+ ],
87
+ request: {
88
+ temperature: 0.1,
89
+ timeoutMs: llmConfig.timeoutMs,
90
+ signal,
91
+ responseSchema: DERIVED_MEMORY_JSON_SCHEMA,
92
+ onRetryAttempt,
93
+ },
94
+ parse: (raw) => {
82
95
  if (!raw)
83
96
  return undefined;
84
97
  const parsed = parseEmbeddedJsonResponse(raw);
@@ -108,9 +121,9 @@ export async function compressMemoryToDerivedMemory(llmConfig, body, signal, akm
108
121
  return undefined;
109
122
  }
110
123
  return { title, description, tags, searchHints, content };
111
- }
112
- catch (err) {
113
- if (err instanceof LlmCallError && err.code === "provider_html_error") {
124
+ },
125
+ onError: (cls, err) => {
126
+ if (cls === "html") {
114
127
  if (telemetry)
115
128
  telemetry.htmlErrorCount = (telemetry.htmlErrorCount ?? 0) + 1;
116
129
  warn(`memory inference: provider returned HTML instead of JSON; skipping memory: ${toErrorMessage(err)}`);
@@ -118,9 +131,8 @@ export async function compressMemoryToDerivedMemory(llmConfig, body, signal, akm
118
131
  }
119
132
  warn(`memory inference failed: ${toErrorMessage(err)}`);
120
133
  return undefined;
121
- }
122
- }, undefined, {
123
- timeoutMs: llmConfig.timeoutMs,
134
+ },
135
+ fallback: undefined,
124
136
  onFallback,
125
137
  });
126
138
  }