akm-cli 0.9.0-beta.2 → 0.9.0-beta.26

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (111) hide show
  1. package/CHANGELOG.md +614 -0
  2. package/dist/assets/prompts/consolidate-system.md +23 -0
  3. package/dist/assets/prompts/contradiction-judge.md +33 -0
  4. package/dist/assets/prompts/distill-knowledge-system.md +22 -0
  5. package/dist/assets/prompts/distill-lesson-system.md +36 -0
  6. package/dist/assets/prompts/extract-session.md +5 -1
  7. package/dist/assets/prompts/graph-extract-system.md +1 -0
  8. package/dist/assets/prompts/memory-infer-system.md +1 -0
  9. package/dist/assets/prompts/memory-infer-user.md +5 -0
  10. package/dist/assets/prompts/metadata-enhance-system.md +1 -0
  11. package/dist/assets/prompts/procedural-system.md +44 -0
  12. package/dist/assets/prompts/recombine-system.md +40 -0
  13. package/dist/assets/prompts/staleness-detect-system.md +6 -0
  14. package/dist/assets/prompts/validate-summary-judge.md +1 -0
  15. package/dist/assets/templates/html/default.html +78 -0
  16. package/dist/assets/templates/html/health.html +730 -0
  17. package/dist/assets/templates/html/vendor/echarts.min.js +45 -0
  18. package/dist/cli/shared.js +21 -5
  19. package/dist/cli.js +47 -5
  20. package/dist/commands/agent/contribute-cli.js +16 -3
  21. package/dist/commands/feedback-cli.js +15 -6
  22. package/dist/commands/graph/graph.js +75 -71
  23. package/dist/commands/health/checks.js +48 -0
  24. package/dist/commands/health/html-report.js +790 -0
  25. package/dist/commands/health.js +478 -15
  26. package/dist/commands/improve/calibration.js +161 -0
  27. package/dist/commands/improve/consolidate.js +634 -111
  28. package/dist/commands/improve/dedup.js +482 -0
  29. package/dist/commands/improve/distill.js +145 -69
  30. package/dist/commands/improve/encoding-salience.js +205 -0
  31. package/dist/commands/improve/extract-cli.js +115 -1
  32. package/dist/commands/improve/extract-prompt.js +33 -2
  33. package/dist/commands/improve/extract-watch.js +140 -0
  34. package/dist/commands/improve/extract.js +280 -35
  35. package/dist/commands/improve/feedback-valence.js +54 -0
  36. package/dist/commands/improve/homeostatic.js +467 -0
  37. package/dist/commands/improve/improve-auto-accept.js +139 -6
  38. package/dist/commands/improve/improve-profiles.js +12 -0
  39. package/dist/commands/improve/improve.js +1851 -515
  40. package/dist/commands/improve/memory/memory-contradiction-detect.js +23 -28
  41. package/dist/commands/improve/outcome-loop.js +256 -0
  42. package/dist/commands/improve/proactive-maintenance.js +87 -0
  43. package/dist/commands/improve/procedural.js +409 -0
  44. package/dist/commands/improve/recombine.js +488 -0
  45. package/dist/commands/improve/reflect-noise.js +0 -0
  46. package/dist/commands/improve/reflect.js +51 -1
  47. package/dist/commands/improve/related-sessions.js +120 -0
  48. package/dist/commands/improve/salience.js +386 -0
  49. package/dist/commands/improve/triage.js +95 -0
  50. package/dist/commands/lint/agent-linter.js +19 -24
  51. package/dist/commands/lint/base-linter.js +173 -60
  52. package/dist/commands/lint/command-linter.js +19 -24
  53. package/dist/commands/lint/env-key-rules.js +34 -1
  54. package/dist/commands/lint/index.js +30 -13
  55. package/dist/commands/lint/memory-linter.js +1 -1
  56. package/dist/commands/lint/registry.js +5 -2
  57. package/dist/commands/lint/task-linter.js +3 -3
  58. package/dist/commands/lint/workflow-linter.js +26 -1
  59. package/dist/commands/proposal/drain.js +73 -6
  60. package/dist/commands/proposal/proposal-cli.js +22 -10
  61. package/dist/commands/proposal/proposal.js +17 -1
  62. package/dist/commands/proposal/validators/proposals.js +369 -329
  63. package/dist/commands/read/curate.js +294 -79
  64. package/dist/commands/read/search-cli.js +7 -0
  65. package/dist/commands/read/search.js +1 -0
  66. package/dist/commands/remember.js +6 -2
  67. package/dist/commands/sources/installed-stashes.js +5 -1
  68. package/dist/commands/sources/stash-cli.js +10 -2
  69. package/dist/core/asset/frontmatter.js +166 -167
  70. package/dist/core/asset/markdown.js +8 -0
  71. package/dist/core/config/config-schema.js +241 -0
  72. package/dist/core/config/config.js +2 -2
  73. package/dist/core/logs-db.js +305 -0
  74. package/dist/core/paths.js +3 -0
  75. package/dist/core/state-db.js +706 -42
  76. package/dist/indexer/db/db.js +347 -38
  77. package/dist/indexer/db/graph-db.js +81 -86
  78. package/dist/indexer/ensure-index.js +152 -17
  79. package/dist/indexer/graph/graph-boost.js +51 -41
  80. package/dist/indexer/index-writer-lock.js +99 -0
  81. package/dist/indexer/indexer.js +114 -111
  82. package/dist/indexer/passes/memory-inference.js +71 -25
  83. package/dist/indexer/passes/staleness-detect.js +2 -5
  84. package/dist/indexer/search/db-search.js +15 -4
  85. package/dist/indexer/search/ranking.js +4 -0
  86. package/dist/integrations/harnesses/claude/session-log.js +27 -5
  87. package/dist/integrations/harnesses/opencode/session-log.js +9 -0
  88. package/dist/integrations/session-logs/index.js +16 -0
  89. package/dist/llm/client.js +38 -4
  90. package/dist/llm/embedder.js +27 -3
  91. package/dist/llm/embedders/local.js +66 -2
  92. package/dist/llm/graph-extract.js +2 -1
  93. package/dist/llm/memory-infer.js +4 -8
  94. package/dist/llm/metadata-enhance.js +9 -1
  95. package/dist/llm/usage-persist.js +77 -0
  96. package/dist/llm/usage-telemetry.js +103 -0
  97. package/dist/output/context.js +3 -2
  98. package/dist/output/html-render.js +73 -0
  99. package/dist/output/shapes/curate.js +14 -2
  100. package/dist/output/shapes/helpers.js +17 -1
  101. package/dist/output/text/helpers.js +78 -1
  102. package/dist/runtime.js +25 -1
  103. package/dist/scripts/migrate-storage.js +1194 -607
  104. package/dist/scripts/migrations/import-fs-improve-runs-to-db.js +455 -270
  105. package/dist/sources/providers/tar-utils.js +16 -8
  106. package/dist/storage/sqlite-pragmas.js +146 -0
  107. package/dist/tasks/runner.js +99 -16
  108. package/dist/workflows/db.js +5 -2
  109. package/dist/workflows/validate-summary.js +2 -7
  110. package/docs/data-and-telemetry.md +1 -0
  111. package/package.json +7 -5
@@ -119,6 +119,26 @@ export async function runMemoryInferencePass(ctx) {
119
119
  // 2026-05-26).
120
120
  if (signal?.aborted)
121
121
  return { aborted: true };
122
+ // Pre-check (#588): when `<parent>.derived.md` is already on disk the
123
+ // inference is by definition complete — the parent only looks pending
124
+ // because `markParentProcessed` never ran (process killed between the
125
+ // child write and the mark) or the child was created externally (e.g.
126
+ // consolidation). Skip the LLM/cache call entirely and mark the parent
127
+ // so it never re-pends. Before this check, production measurements
128
+ // showed ~55% of the pass's LLM budget re-deriving such parents only to
129
+ // discover the existing child after the fact.
130
+ if (fs.existsSync(derivedChildPath(record))) {
131
+ markParentProcessed(record);
132
+ return {
133
+ skipped: false,
134
+ splitParent: false,
135
+ written: 0,
136
+ fromCache: false,
137
+ retryAttempts: 0,
138
+ childExists: true,
139
+ precheck: true,
140
+ };
141
+ }
122
142
  // Incremental cache: skip LLM call when body hash is unchanged and
123
143
  // --re-enrich was not requested. The cache ref is the absolute file path.
124
144
  const validate = (raw) => {
@@ -171,23 +191,30 @@ export async function runMemoryInferencePass(ctx) {
171
191
  return { skipped: false, splitParent: true, written: writeOutcome.written, fromCache, retryAttempts };
172
192
  }
173
193
  // LLM produced a valid derived draft but no file was written — either
174
- // because `<parent>.derived.md` already exists on disk or
175
- // `writeAssetToSource` threw. Categorise as `childExists` so the
176
- // attempt is accounted for in health metrics rather than vanishing
177
- // into the freshAttempts denominator.
194
+ // because `<parent>.derived.md` appeared on disk after the pre-check
195
+ // above (a rare mid-flight race) or `writeAssetToSource` threw.
196
+ // Categorise as `childExists` so the consumed attempt is accounted for
197
+ // in health metrics rather than vanishing into the freshAttempts
198
+ // denominator.
178
199
  //
179
- // When the child already exists on disk the inference is, by definition,
180
- // already complete — so mark the parent processed here too (#550).
181
- // Without this, `isPendingMemory()` re-queues the same parent every run
182
- // (the `written > 0` path was previously the only site that marks it),
183
- // causing permanent re-queueing and wasted LLM calls. A genuine write
184
- // *failure* (`writeAssetToSource` threw) must NOT mark the parent it
185
- // should be retried next run — so we key off the explicit `childExists`
186
- // outcome rather than the conflated `written === 0`.
200
+ // When the child exists the inference is, by definition, complete — so
201
+ // mark the parent processed here too (#550), otherwise
202
+ // `isPendingMemory()` re-queues the same parent every run. A genuine
203
+ // write *failure* (`writeAssetToSource` threw) must NOT mark the parent
204
+ // it should be retried next run so we key off the explicit
205
+ // `childExists` outcome rather than the conflated `written === 0`.
187
206
  if (writeOutcome.childExists) {
188
207
  markParentProcessed(record);
189
208
  }
190
- return { skipped: false, splitParent: false, written: 0, fromCache, retryAttempts, childExists: true };
209
+ return {
210
+ skipped: false,
211
+ splitParent: false,
212
+ written: 0,
213
+ fromCache,
214
+ retryAttempts,
215
+ childExists: true,
216
+ precheck: false,
217
+ };
191
218
  },
192
219
  // Default concurrency of 4 for cloud APIs. Set `llm.concurrency: 1`
193
220
  // in config.json for local model servers (LM Studio, Ollama).
@@ -224,11 +251,16 @@ export async function runMemoryInferencePass(ctx) {
224
251
  result.writtenFacts += res.written;
225
252
  }
226
253
  else if ("childExists" in res && res.childExists) {
227
- // LLM call was consumed but the derived file already existed (or the
228
- // write threw). Track separately so this category is observable in
229
- // health output and stops bleeding into the freshAttempts denominator.
254
+ // Derived child already on disk. Track separately so this category is
255
+ // observable in health output and stops bleeding into the
256
+ // freshAttempts denominator. Pre-check skips (#588) are the routine
257
+ // self-healing path — no LLM attempt was consumed and the parent has
258
+ // been marked processed — so only the rare post-LLM case (mid-flight
259
+ // race or write failure) warrants a per-ref warning.
230
260
  result.skippedChildExists += 1;
231
- warn(`memory inference: derived child for ${pending[i]?.ref ?? "<unknown>"} already existed or write failed; counted as skippedChildExists`);
261
+ if (!res.precheck) {
262
+ warn(`memory inference: derived child for ${pending[i]?.ref ?? "<unknown>"} already existed or write failed; counted as skippedChildExists`);
263
+ }
232
264
  }
233
265
  else {
234
266
  // The per-record state machine should cover every outcome. A hit here
@@ -324,6 +356,14 @@ function toMemoryName(memoriesDir, filePath) {
324
356
  // user has organised under memories/.
325
357
  return rel.replace(/\\/g, "/").replace(/\.md$/i, "");
326
358
  }
359
+ /**
360
+ * Absolute path of the derived child for a parent memory. Single source of
361
+ * truth for the `<parent>.derived.md` naming convention — used both by the
362
+ * pre-LLM existence check (#588) and the write path.
363
+ */
364
+ function derivedChildPath(parent) {
365
+ return path.join(parent.stashRoot, "memories", `${parent.name}.derived.md`);
366
+ }
327
367
  async function writeDerivedMemory(parent, derived) {
328
368
  const writeTarget = {
329
369
  kind: "filesystem",
@@ -338,11 +378,10 @@ async function writeDerivedMemory(parent, derived) {
338
378
  };
339
379
  const childName = `${parent.name}.derived`;
340
380
  const childRefStr = `memory:${childName}`;
341
- const childPath = path.join(parent.stashRoot, "memories", `${childName}.md`);
342
- if (fs.existsSync(childPath)) {
343
- // The derived child is already on disk inference for this parent is
344
- // complete. Report `childExists` so the caller marks the parent processed
345
- // (#550) instead of re-queueing it forever.
381
+ if (fs.existsSync(derivedChildPath(parent))) {
382
+ // The derived child appeared on disk after the caller's pre-check (#588)
383
+ // a rare mid-flight race. Report `childExists` so the caller marks the
384
+ // parent processed (#550) instead of re-queueing it forever.
346
385
  return { written: 0, childExists: true };
347
386
  }
348
387
  try {
@@ -386,10 +425,17 @@ function markParentProcessed(parent) {
386
425
  warn(`memory inference: failed to re-read parent ${parent.filePath}: ${err instanceof Error ? err.message : String(err)}`);
387
426
  return;
388
427
  }
389
- const updatedFm = { ...parent.data, [FM_INFERENCE_PROCESSED]: true };
390
428
  const block = parseFrontmatterBlock(raw);
391
- const body = block?.content ?? raw;
392
- const next = assembleAsset(updatedFm, body);
429
+ if (!block) {
430
+ // Cannot safely rewrite malformed frontmatter — skip marking so the memory
431
+ // is retried on the next run once the frontmatter is repaired. Writing with
432
+ // `body = raw` would wrap the entire file (including the bad frontmatter)
433
+ // in a new block, producing a duplicate-frontmatter corruption.
434
+ warn(`memory inference: skipping markParentProcessed for ${parent.filePath} — could not parse frontmatter block`);
435
+ return;
436
+ }
437
+ const updatedFm = { ...parent.data, [FM_INFERENCE_PROCESSED]: true };
438
+ const next = assembleAsset(updatedFm, block.content);
393
439
  try {
394
440
  fs.writeFileSync(parent.filePath, next, "utf8");
395
441
  }
@@ -44,6 +44,7 @@
44
44
  import { createHash } from "node:crypto";
45
45
  import fs from "node:fs";
46
46
  import path from "node:path";
47
+ import stalenessDetectSystemPrompt from "../../assets/prompts/staleness-detect-system.md" with { type: "text" };
47
48
  import { assembleAsset } from "../../core/asset/asset-serialize.js";
48
49
  import { parseFrontmatter, parseFrontmatterBlock } from "../../core/asset/frontmatter.js";
49
50
  import { concurrentMap } from "../../core/concurrent.js";
@@ -319,11 +320,7 @@ function pickSimilar(candidate, all) {
319
320
  return scored.slice(0, TOP_K_SIMILAR).map((s) => s.snap);
320
321
  }
321
322
  // ── LLM dispatch ────────────────────────────────────────────────────────────
322
- const SYSTEM_PROMPT = "You are a belief-state classifier for a memory store. Given a candidate memory and a list of more-recent similar memories from the same store, decide whether the candidate is still current or has been superseded.\n\n" +
323
- "Respond on the first line with exactly YES or NO.\n" +
324
- "If YES, the second line MUST be of the form `SUPERSEDED_BY: <ref>` where <ref> is the exact ref of the superseding memory from the list provided. Do NOT invent refs.\n" +
325
- "If NO, do not include any additional lines.\n" +
326
- "No prose, no preamble, no markdown.";
323
+ const SYSTEM_PROMPT = stalenessDetectSystemPrompt;
327
324
  async function askValidator(connection, candidate, allMemories, signal, timeoutMs) {
328
325
  const similar = pickSimilar(candidate, allMemories);
329
326
  if (similar.length === 0) {
@@ -65,6 +65,7 @@ export async function searchLocal(input) {
65
65
  const includeProposed = input.includeProposed === true;
66
66
  const beliefFilter = input.beliefFilter ?? "all";
67
67
  const restrictToSources = input.restrictToSources === true;
68
+ const includeExcludedTypes = input.includeExcludedTypes === true;
68
69
  const rendererRegistry = input.rendererRegistry ?? defaultRendererRegistry;
69
70
  const allSourceDirs = sources.map((s) => s.path);
70
71
  const rawStatus = readSemanticStatus();
@@ -114,7 +115,7 @@ export async function searchLocal(input) {
114
115
  mode: "keyword",
115
116
  };
116
117
  }
117
- const { hits, embedMs, rankMs } = await searchDatabase(db, query, searchType, limit, stashDir, allSourceDirs, config, sources, rendererRegistry, filters, includeProposed, beliefFilter, restrictToSources);
118
+ const { hits, embedMs, rankMs } = await searchDatabase(db, query, searchType, limit, stashDir, allSourceDirs, config, sources, rendererRegistry, filters, includeProposed, beliefFilter, restrictToSources, includeExcludedTypes);
118
119
  return {
119
120
  hits,
120
121
  tip: hits.length === 0
@@ -131,14 +132,19 @@ export async function searchLocal(input) {
131
132
  }
132
133
  }
133
134
  // ── Database search ─────────────────────────────────────────────────────────
134
- async function searchDatabase(db, query, searchType, limit, stashDir, allSourceDirs, config, sources, rendererRegistry = defaultRendererRegistry, filters, includeProposed = false, beliefFilter = "all", restrictToSources = false) {
135
+ async function searchDatabase(db, query, searchType, limit, stashDir, allSourceDirs, config, sources, rendererRegistry = defaultRendererRegistry, filters, includeProposed = false, beliefFilter = "all", restrictToSources = false, includeExcludedTypes = false) {
135
136
  const hasSearchableTokens = query.length > 0 && sanitizeFtsQuery(query).length > 0;
137
+ // #627 — resolve the default type-exclusion policy. It applies ONLY on the
138
+ // untyped ('any') path and only when the caller did not opt back in via
139
+ // `includeExcludedTypes`. When the config key is ABSENT a built-in default of
140
+ // ['session'] is applied; an explicit empty list disables exclusion.
141
+ const defaultExcludes = searchType === "any" && !includeExcludedTypes ? (config.search?.defaultExcludeTypes ?? ["session"]) : [];
136
142
  // Empty queries — including ones that sanitize down to no searchable FTS
137
143
  // tokens such as "." — should enumerate matching entries instead of
138
144
  // returning an empty result set from FTS.
139
145
  if (!hasSearchableTokens) {
140
146
  const typeFilter = searchType === "any" ? undefined : searchType;
141
- const allEntries = getAllEntries(db, typeFilter);
147
+ const allEntries = getAllEntries(db, typeFilter, defaultExcludes);
142
148
  // Deduplicate by file path — multiple entries can share the same file
143
149
  const seenFilePaths = new Set();
144
150
  const uniqueEntries = allEntries.filter((ie) => {
@@ -187,7 +193,7 @@ async function searchDatabase(db, query, searchType, limit, stashDir, allSourceD
187
193
  const typeFilter = searchType === "any" ? undefined : searchType;
188
194
  const tEmbed0 = Date.now();
189
195
  const embeddingPromise = tryVecScores(db, query, limit * 3, config);
190
- const ftsResults = searchFts(db, query, limit * 3, typeFilter);
196
+ const ftsResults = searchFts(db, query, limit * 3, typeFilter, defaultExcludes);
191
197
  const embeddingScores = await embeddingPromise;
192
198
  const embedMs = Date.now() - tEmbed0;
193
199
  const tRank0 = Date.now();
@@ -208,6 +214,11 @@ async function searchDatabase(db, query, searchType, limit, stashDir, allSourceD
208
214
  embedScoreMap,
209
215
  getEntryById: (id) => getEntryById(db, id) ?? undefined,
210
216
  typeFilter,
217
+ // #627 — also exclude default-hidden types from the vector-only branch so a
218
+ // session asset that is a top-k vector neighbor (but not an FTS match) does
219
+ // not leak into default ('any') results. defaultExcludes is already []
220
+ // unless this is the untyped path without includeExcludedTypes.
221
+ excludeTypes: defaultExcludes,
211
222
  });
212
223
  // ── Scoring Phase ──────────────────────────────────────────────────────
213
224
  // Apply boosts as multiplicative factors (all boosts in a single phase
@@ -20,6 +20,7 @@ export function normalizeFtsScores(results) {
20
20
  export function combineSearchScores(options) {
21
21
  const FTS_WEIGHT = 0.7;
22
22
  const VEC_WEIGHT = 0.3;
23
+ const excludeTypeSet = options.excludeTypes && options.excludeTypes.length > 0 ? new Set(options.excludeTypes) : null;
23
24
  const scored = [];
24
25
  const seenIds = new Set();
25
26
  for (const [id, { score: ftsScore, result }] of options.ftsScoreMap) {
@@ -42,6 +43,9 @@ export function combineSearchScores(options) {
42
43
  continue;
43
44
  if (options.typeFilter && found.entry.type !== options.typeFilter)
44
45
  continue;
46
+ // #627 — drop vector-only neighbors whose type is excluded on the default path.
47
+ if (excludeTypeSet?.has(found.entry.type))
48
+ continue;
45
49
  scored.push({
46
50
  id,
47
51
  entry: found.entry,
@@ -5,7 +5,19 @@ import fs from "node:fs";
5
5
  import os from "node:os";
6
6
  import path from "node:path";
7
7
  import { extractInlineRefMentions } from "../../session-logs/inline-refs.js";
8
- const CLAUDE_PROJECTS_DIR = path.join(os.homedir(), ".claude", "projects");
8
+ /**
9
+ * Root directory holding Claude Code's per-project JSONL session logs.
10
+ *
11
+ * Resolved per call (not memoized at module load) so the `AKM_CLAUDE_PROJECTS_DIR`
12
+ * override can be set after import. The override exists so tests — and the
13
+ * isolated-storage sandbox — can point the scan at an empty fixture directory
14
+ * instead of the real `~/.claude/projects`, which on an actively-used machine
15
+ * holds many large session files and would make `akm health` (which scans it
16
+ * synchronously) slow and non-hermetic.
17
+ */
18
+ function claudeProjectsDir() {
19
+ return process.env.AKM_CLAUDE_PROJECTS_DIR ?? path.join(os.homedir(), ".claude", "projects");
20
+ }
9
21
  /**
10
22
  * Parse a single Claude Code JSONL event into a normalized {@link SessionEvent}.
11
23
  * Returns `undefined` for events that don't carry textual content (file
@@ -93,11 +105,21 @@ export class ClaudeCodeProvider {
93
105
  // HARNESS_BY_ID.get("claude").runtimeId.
94
106
  name = "claude-code";
95
107
  isAvailable() {
96
- return fs.existsSync(CLAUDE_PROJECTS_DIR);
108
+ return fs.existsSync(claudeProjectsDir());
109
+ }
110
+ /**
111
+ * Directory holding Claude Code's per-project session JSONL files
112
+ * (`~/.claude/projects`, honoring `AKM_CLAUDE_PROJECTS_DIR`). Returns `[]`
113
+ * when the directory does not exist on this machine. See {@link
114
+ * SessionLogHarness.watchRoots}.
115
+ */
116
+ watchRoots() {
117
+ const dir = claudeProjectsDir();
118
+ return fs.existsSync(dir) ? [dir] : [];
97
119
  }
98
120
  *readEvents(input) {
99
121
  try {
100
- for (const jsonlPath of this.#walkJsonl(CLAUDE_PROJECTS_DIR)) {
122
+ for (const jsonlPath of this.#walkJsonl(claudeProjectsDir())) {
101
123
  const stat = fs.statSync(jsonlPath);
102
124
  if (stat.mtimeMs < input.sinceMs)
103
125
  continue;
@@ -128,7 +150,7 @@ export class ClaudeCodeProvider {
128
150
  }
129
151
  }
130
152
  listSessions(input = {}) {
131
- const root = input.location ?? CLAUDE_PROJECTS_DIR;
153
+ const root = input.location ?? claudeProjectsDir();
132
154
  const sinceMs = input.sinceMs ?? 0;
133
155
  const summaries = [];
134
156
  try {
@@ -286,7 +308,7 @@ export class ClaudeCodeProvider {
286
308
  const full = path.join(dir, entry.name);
287
309
  if (entry.isDirectory())
288
310
  yield* this.#walkJsonl(full);
289
- else if (entry.name.endsWith(".jsonl"))
311
+ else if (entry.name.endsWith(".jsonl") && entry.name !== "journal.jsonl")
290
312
  yield full;
291
313
  }
292
314
  }
@@ -26,6 +26,15 @@ export class OpenCodeProvider {
26
26
  isAvailable() {
27
27
  return fs.existsSync(this.#baseDir);
28
28
  }
29
+ /**
30
+ * Directory holding opencode's per-project session metadata files
31
+ * (`<base>/storage/session`). Returns `[]` when it does not exist on this
32
+ * machine. See {@link SessionLogHarness.watchRoots}.
33
+ */
34
+ watchRoots() {
35
+ const sessionRoot = path.join(this.#baseDir, "storage", "session");
36
+ return fs.existsSync(sessionRoot) ? [sessionRoot] : [];
37
+ }
29
38
  *readEvents(input) {
30
39
  // Legacy behavior: stream raw log lines from the top-level dir and `log/`
31
40
  // subdirectory. Kept to keep `getExecutionLogCandidates` working without
@@ -28,6 +28,22 @@ const ERROR_PATTERNS = /error|failed|exception|cannot|undefined|null pointer|ENO
28
28
  export function getAvailableHarnesses() {
29
29
  return HARNESSES.filter((harness) => harness.isAvailable());
30
30
  }
31
+ /**
32
+ * Map each available harness to its `{ harnessName, roots }` watch target,
33
+ * skipping harnesses that expose no roots (absent `watchRoots()` or an empty
34
+ * result). This is the one stable entry point the watcher uses so it never
35
+ * reaches into providers directly.
36
+ */
37
+ export function getWatchTargets() {
38
+ const targets = [];
39
+ for (const harness of getAvailableHarnesses()) {
40
+ const roots = harness.watchRoots?.() ?? [];
41
+ if (roots.length === 0)
42
+ continue;
43
+ targets.push({ harnessName: harness.name, roots });
44
+ }
45
+ return targets;
46
+ }
31
47
  export function normalizeSessionTopic(text) {
32
48
  const normalized = text.replace(/\s+/g, " ").trim().toLowerCase();
33
49
  if (normalized.length < 10)
@@ -14,6 +14,7 @@ import { fetchWithTimeout } from "../core/common.js";
14
14
  import { resolveSecret } from "../core/config/config.js";
15
15
  import { escapeJsonStringControls, parseJsonResponse, stripCodeFences, stripThinkBlocks } from "../core/parse.js";
16
16
  import { warnVerbose } from "../core/warn.js";
17
+ import { emitLlmUsage, extractUsageTokens } from "./usage-telemetry.js";
17
18
  // Re-export shared parse utilities so existing importers of `client.ts` continue
18
19
  // to resolve `parseJsonResponse` and `parseEmbeddedJsonResponse` from this module.
19
20
  export { escapeJsonStringControls, parseEmbeddedJsonResponse, parseJsonResponse, stripCodeFences, stripThinkBlocks, } from "../core/parse.js";
@@ -118,9 +119,23 @@ function looksLikeContextOverflow(message) {
118
119
  /**
119
120
  * Decide whether a first-attempt {@link LlmCallError} is eligible for a single
120
121
  * retry. Retryable: HTTP 5xx (`provider_error` with statusCode >= 500) and
121
- * `network_error` whose message looks like a transient connection reset
122
- * (ECONNRESET / EPIPE / "fetch failed"). NOT retryable: 4xx, `rate_limited`
123
- * (429), `timeout`, `parse_error`, and context-overflow-classified errors.
122
+ * `network_error` whose message looks like a transient connection drop.
123
+ * NOT retryable: 4xx, `rate_limited` (429), `timeout`, `parse_error`, and
124
+ * context-overflow-classified errors.
125
+ *
126
+ * The connection-drop heuristic covers the substrings emitted across runtimes
127
+ * for a mid-flight socket close:
128
+ * - `ECONNRESET` / `EPIPE` — Node/libuv socket reset codes
129
+ * - `fetch failed` — undici's generic wrapper message
130
+ * - `socket connection was closed` — Bun's message for a dropped connection
131
+ * (e.g. "The socket connection was closed unexpectedly.")
132
+ * - `terminated` / `other side closed` — undici's phrasings for the same
133
+ *
134
+ * These all describe a transient transport failure where a second attempt can
135
+ * legitimately succeed, which is exactly the case a single bounded retry is
136
+ * meant to absorb. Before this list was widened, Bun's "socket connection was
137
+ * closed unexpectedly" fell through unretried and surfaced as a recurring
138
+ * failure in the improve/reflect and capability-probe flows.
124
139
  */
125
140
  function isRetryable(err) {
126
141
  if (looksLikeContextOverflow(err.message))
@@ -130,7 +145,12 @@ function isRetryable(err) {
130
145
  }
131
146
  if (err.code === "network_error") {
132
147
  const lower = err.message.toLowerCase();
133
- return lower.includes("econnreset") || lower.includes("epipe") || lower.includes("fetch failed");
148
+ return (lower.includes("econnreset") ||
149
+ lower.includes("epipe") ||
150
+ lower.includes("fetch failed") ||
151
+ lower.includes("socket connection was closed") ||
152
+ lower.includes("terminated") ||
153
+ lower.includes("other side closed"));
134
154
  }
135
155
  return false;
136
156
  }
@@ -179,6 +199,10 @@ async function chatCompletionAttempt(config, messages, options, timeoutMs) {
179
199
  const responseFormat = options?.responseSchema && config.supportsJsonSchema
180
200
  ? { response_format: { type: "json_schema", json_schema: { schema: options.responseSchema, strict: true } } }
181
201
  : {};
202
+ // Wall-clock start for per-call usage telemetry (#576). Captured here so the
203
+ // emitted duration covers the full request/response/parse cycle of a single
204
+ // attempt, not the retry-wrapping `chatCompletion`.
205
+ const requestStartedAt = Date.now();
182
206
  let response;
183
207
  try {
184
208
  response = await fetchWithTimeout(config.endpoint, {
@@ -241,6 +265,16 @@ async function chatCompletionAttempt(config, messages, options, timeoutMs) {
241
265
  catch {
242
266
  throw new LlmCallError(`LLM response was not valid JSON ${config.endpoint}: ${redactErrorBody(rawOkBody)}`, "parse_error", response.status);
243
267
  }
268
+ // Per-call usage telemetry (#576). Best-effort and fully isolated: a missing
269
+ // or garbled usage block still records duration + model, and a throwing sink
270
+ // can never fail the call (emitLlmUsage swallows its own errors). The stage
271
+ // is supplied ambiently by emitLlmUsage; no `stage` param is threaded here.
272
+ emitLlmUsage({
273
+ model: typeof json.model === "string" && json.model ? json.model : config.model,
274
+ durationMs: Date.now() - requestStartedAt,
275
+ finishReason: typeof json.choices?.[0]?.finish_reason === "string" ? json.choices[0].finish_reason : undefined,
276
+ ...extractUsageTokens(json.usage),
277
+ });
244
278
  const content = (json.choices?.[0]?.message?.content ?? "").trim();
245
279
  const reasoning = (json.choices?.[0]?.message?.reasoning_content ?? "").trim();
246
280
  return content || reasoning;
@@ -2,7 +2,7 @@
2
2
  // License, v. 2.0. If a copy of the MPL was not distributed with this
3
3
  // file, You can obtain one at https://mozilla.org/MPL/2.0/.
4
4
  import { embedCacheKey, getCachedEmbedding, setCachedEmbedding } from "./embedders/cache.js";
5
- import { isTransformersAvailable, LocalEmbedder } from "./embedders/local.js";
5
+ import { DEFAULT_LOCAL_MODEL, isTransformersAvailable, LocalEmbedder } from "./embedders/local.js";
6
6
  import { hasRemoteEndpoint, RemoteEmbedder } from "./embedders/remote.js";
7
7
  // ── Re-exports (public API) ─────────────────────────────────────────────────
8
8
  export { clearEmbeddingCache } from "./embedders/cache.js";
@@ -52,7 +52,8 @@ export async function embed(text, embeddingConfig, signal) {
52
52
  /**
53
53
  * Generate embeddings for multiple texts in batch.
54
54
  * Uses the OpenAI-compatible batch API for remote endpoints (batches of 100).
55
- * Falls back to sequential embedding for the local transformer pipeline.
55
+ * Uses the LocalEmbedder.embedBatch path for the local transformer pipeline,
56
+ * which processes texts in chunks of 32 for genuine batched inference.
56
57
  */
57
58
  export async function embedBatch(texts, embeddingConfig, signal) {
58
59
  if (texts.length === 0)
@@ -60,8 +61,13 @@ export async function embedBatch(texts, embeddingConfig, signal) {
60
61
  if (embeddingConfig && hasRemoteEndpoint(embeddingConfig)) {
61
62
  return new RemoteEmbedder(embeddingConfig).embedBatch(texts, signal);
62
63
  }
63
- // Local transformer: process sequentially (pipeline handles one at a time)
64
+ // Local transformer: use the batched path (chunks of 32 via LocalEmbedder).
65
+ // When a localModel override is set we cannot share the singleton (which uses
66
+ // the default model), so fall back to per-text embedWithModel in that case.
64
67
  const localModel = embeddingConfig?.localModel;
68
+ if (!localModel) {
69
+ return getLocalEmbedder().embedBatch(texts, signal);
70
+ }
65
71
  const results = [];
66
72
  for (const text of texts) {
67
73
  if (signal?.aborted) {
@@ -77,6 +83,24 @@ export async function embedBatch(texts, embeddingConfig, signal) {
77
83
  // facade and its `@huggingface/transformers` import chain. Re-export
78
84
  // preserves the existing public API.
79
85
  export { cosineSimilarity } from "./embedders/types.js";
86
+ // ── Model ID resolution ─────────────────────────────────────────────────────
87
+ /**
88
+ * Derive a stable string identifier for the embedding model in use.
89
+ * This is the `model_id` stored in `body_embeddings` (and used for the
90
+ * drop-all-on-mismatch purge when the model changes).
91
+ *
92
+ * Rules:
93
+ * - Remote endpoint: use `config.model` (the API-level model name).
94
+ * - Local transformers: use `config.localModel ?? DEFAULT_LOCAL_MODEL`.
95
+ * - No config: use `DEFAULT_LOCAL_MODEL` (the shared singleton model).
96
+ */
97
+ export function resolveEmbeddingModelId(embeddingConfig) {
98
+ if (!embeddingConfig)
99
+ return DEFAULT_LOCAL_MODEL;
100
+ if (hasRemoteEndpoint(embeddingConfig))
101
+ return embeddingConfig.model ?? "remote";
102
+ return embeddingConfig.localModel ?? DEFAULT_LOCAL_MODEL;
103
+ }
80
104
  // ── Availability check ──────────────────────────────────────────────────────
81
105
  /**
82
106
  * Check whether embedding is available with a detailed reason on failure.
@@ -19,8 +19,24 @@ import { getDirname, resolveModule } from "../../runtime.js";
19
19
  * `all-MiniLM-L6-v2` at the same 384-dimension footprint.
20
20
  */
21
21
  export const DEFAULT_LOCAL_MODEL = "Xenova/bge-small-en-v1.5";
22
+ /** Type-guard: true when the value looks like a batch Tensor (has .dims). */
23
+ function isBatchTensor(v) {
24
+ return (v !== null &&
25
+ typeof v === "object" &&
26
+ "data" in v &&
27
+ "dims" in v &&
28
+ Array.isArray(v.dims) &&
29
+ v.dims.length >= 2);
30
+ }
22
31
  const LOCAL_EMBEDDER_DTYPE = "fp32";
23
32
  const LOCAL_EMBEDDER_FALLBACK_DTYPE = "auto";
33
+ /**
34
+ * Maximum texts per batch for the local transformers pipeline. The pipeline
35
+ * can run genuine batched inference over a string array; 32 is a safe default
36
+ * that fits well inside most model context budgets while providing 10–50×
37
+ * throughput improvement over one-at-a-time calls on the cold minority.
38
+ */
39
+ const LOCAL_BATCH_SIZE = 32;
24
40
  /**
25
41
  * Return the local model name that will be used for embedding.
26
42
  * When `overrideModel` is provided it takes precedence; otherwise
@@ -77,15 +93,63 @@ export class LocalEmbedder {
77
93
  }
78
94
  return this.embedWithModel(text, this.defaultModel);
79
95
  }
96
+ /**
97
+ * Embed a batch of texts. Processes in chunks of `LOCAL_BATCH_SIZE` (32) so
98
+ * the transformers pipeline can run genuine batched inference rather than one
99
+ * call per text. Falls back to one-at-a-time if the pipeline does not support
100
+ * array input (older versions of @huggingface/transformers). Each chunk is
101
+ * checked against the AbortSignal between calls.
102
+ */
80
103
  async embedBatch(texts, signal) {
81
104
  if (texts.length === 0)
82
105
  return [];
106
+ if (signal?.aborted) {
107
+ throw signal.reason instanceof Error ? signal.reason : new Error("embedding interrupted");
108
+ }
109
+ const pipeline = await this.getPipeline(this.defaultModel);
83
110
  const results = [];
84
- for (const text of texts) {
111
+ for (let i = 0; i < texts.length; i += LOCAL_BATCH_SIZE) {
85
112
  if (signal?.aborted) {
86
113
  throw signal.reason instanceof Error ? signal.reason : new Error("embedding interrupted");
87
114
  }
88
- results.push(await this.embedWithModel(text, this.defaultModel));
115
+ const chunk = texts.slice(i, i + LOCAL_BATCH_SIZE);
116
+ try {
117
+ // @huggingface/transformers feature-extraction pipeline accepts a
118
+ // string[] and returns a batch Tensor (NOT an Array<{data}>).
119
+ // The Tensor has .data (flat Float32Array, length = batch * dim) and
120
+ // .dims = [batch, dim]. Slice .data into per-row vectors using .dims.
121
+ const batchResult = await pipeline(chunk, {
122
+ pooling: "mean",
123
+ normalize: true,
124
+ });
125
+ if (isBatchTensor(batchResult)) {
126
+ const dim = batchResult.dims[1];
127
+ for (let row = 0; row < chunk.length; row++) {
128
+ results.push(Array.from(batchResult.data.subarray(row * dim, (row + 1) * dim)));
129
+ }
130
+ }
131
+ else if (Array.isArray(batchResult)) {
132
+ // Older versions of @huggingface/transformers returned Array<{data}>.
133
+ for (const r of batchResult) {
134
+ results.push(Array.from(r.data));
135
+ }
136
+ }
137
+ else {
138
+ // Single-text result returned for a chunk — should not happen for
139
+ // string[] input, but handle defensively.
140
+ throw new Error("unexpected pipeline return shape for batch input");
141
+ }
142
+ }
143
+ catch {
144
+ // Fallback: process one-at-a-time (older pipeline versions or mismatched
145
+ // return type). Fail-open per text: a single failure aborts the chunk.
146
+ for (const text of chunk) {
147
+ if (signal?.aborted) {
148
+ throw signal.reason instanceof Error ? signal.reason : new Error("embedding interrupted");
149
+ }
150
+ results.push(await this.embedWithModel(text, this.defaultModel));
151
+ }
152
+ }
89
153
  }
90
154
  return results;
91
155
  }
@@ -20,6 +20,7 @@
20
20
  * the connection via `resolveIndexPassLLM("graph", config)` and pass it
21
21
  * straight through.
22
22
  */
23
+ import systemPromptTemplate from "../assets/prompts/graph-extract-system.md" with { type: "text" };
23
24
  import userPromptTemplate from "../assets/prompts/graph-extract-user-prompt.md" with { type: "text" };
24
25
  import { toErrorMessage } from "../core/common.js";
25
26
  import { warn, warnVerbose } from "../core/warn.js";
@@ -41,7 +42,7 @@ const NON_ARRAY_BATCH_DISABLE_THRESHOLD = 2;
41
42
  const MAX_ENTITIES_PER_ASSET = 32;
42
43
  /** Hard cap on relations returned per asset. */
43
44
  const MAX_RELATIONS_PER_ASSET = 32;
44
- const SYSTEM_PROMPT = "You extract a knowledge graph from developer notes. Return ONLY valid JSON — no prose, no markdown fences, no preamble.";
45
+ const SYSTEM_PROMPT = systemPromptTemplate;
45
46
  const USER_PROMPT_PREFIX = userPromptTemplate
46
47
  .replace("{{MAX_ENTITIES}}", String(MAX_ENTITIES_PER_ASSET))
47
48
  .replace("{{MAX_RELATIONS}}", String(MAX_RELATIONS_PER_ASSET));
@@ -18,20 +18,16 @@
18
18
  * the connection via `resolveIndexPassLLM("memory", config)` and pass it
19
19
  * straight through.
20
20
  */
21
+ import memoryInferSystemPrompt from "../assets/prompts/memory-infer-system.md" with { type: "text" };
22
+ import memoryInferUserPrompt from "../assets/prompts/memory-infer-user.md" with { type: "text" };
21
23
  import { toErrorMessage } from "../core/common.js";
22
24
  import { warn } from "../core/warn.js";
23
25
  import { chatCompletion, LlmCallError, parseEmbeddedJsonResponse } from "./client.js";
24
26
  import { tryLlmFeature } from "./feature-gate.js";
25
27
  /** Hard cap on body chars sent to the model — pragmatic and matches `runLlmEnrich`. */
26
28
  const MAX_BODY_CHARS = 4000;
27
- const SYSTEM_PROMPT = "You compress a developer memory into one high-signal derived memory for later retrieval. " +
28
- "Return only valid JSON. No prose outside the JSON object. No markdown fences.";
29
- const USER_PROMPT_PREFIX = `Compress the memory below into one derived memory. Output ONLY JSON:
30
- {"title":"short title string","description":"one sentence summary string","tags":["tag1","tag2"],"searchHints":["search phrase 1","search phrase 2"],"content":"2-3 sentence compressed body preserving key facts verbatim"}
31
- Rules: be specific, no vague generalizations, preserve key facts (names/versions/paths/config keys verbatim), merge related points, 3-8 tags, 3-6 searchHints. The content field must be a plain string with 2-3 sentences.
32
-
33
- Memory:
34
- `;
29
+ const SYSTEM_PROMPT = memoryInferSystemPrompt;
30
+ const USER_PROMPT_PREFIX = memoryInferUserPrompt;
35
31
  /**
36
32
  * Strict JSON Schema for the derived-memory payload. Sent to providers that
37
33
  * opt in via `LlmConnectionConfig.supportsJsonSchema = true`; the client
@@ -1,9 +1,17 @@
1
1
  // This Source Code Form is subject to the terms of the Mozilla Public
2
2
  // License, v. 2.0. If a copy of the MPL was not distributed with this
3
3
  // file, You can obtain one at https://mozilla.org/MPL/2.0/.
4
+ /**
5
+ * LLM-driven metadata enhancement for stash entries.
6
+ *
7
+ * Split out of `llm.ts` so the higher-level workflow (prompting the LLM to
8
+ * improve descriptions/tags/searchHints) lives separately from the low-level
9
+ * transport client in `client.ts`.
10
+ */
11
+ import metadataEnhanceSystemPrompt from "../assets/prompts/metadata-enhance-system.md" with { type: "text" };
4
12
  import { chatCompletion, parseJsonResponse } from "./client.js";
5
13
  import { tryLlmFeature } from "./feature-gate.js";
6
- const SYSTEM_PROMPT = `You are a metadata generator for a developer asset registry. Given a script/skill/command/agent entry, generate improved metadata. Respond with ONLY valid JSON, no markdown fencing.`;
14
+ const SYSTEM_PROMPT = metadataEnhanceSystemPrompt;
7
15
  /**
8
16
  * Use an LLM to enhance a stash entry's metadata: improve description,
9
17
  * generate searchHints, and suggest tags.