akm-cli 0.9.0-beta.4 → 0.9.0-beta.40

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (131) hide show
  1. package/CHANGELOG.md +626 -0
  2. package/dist/assets/prompts/consolidate-system.md +23 -0
  3. package/dist/assets/prompts/contradiction-judge.md +33 -0
  4. package/dist/assets/prompts/distill-knowledge-system.md +22 -0
  5. package/dist/assets/prompts/distill-lesson-system.md +36 -0
  6. package/dist/assets/prompts/extract-session.md +6 -2
  7. package/dist/assets/prompts/graph-extract-system.md +1 -0
  8. package/dist/assets/prompts/memory-infer-system.md +1 -0
  9. package/dist/assets/prompts/memory-infer-user.md +5 -0
  10. package/dist/assets/prompts/metadata-enhance-system.md +1 -0
  11. package/dist/assets/prompts/procedural-system.md +44 -0
  12. package/dist/assets/prompts/recombine-system.md +40 -0
  13. package/dist/assets/prompts/staleness-detect-system.md +6 -0
  14. package/dist/assets/prompts/validate-summary-judge.md +1 -0
  15. package/dist/assets/stash-skeleton/facts/conventions/assets/agent.md +22 -0
  16. package/dist/assets/stash-skeleton/facts/conventions/assets/command.md +22 -0
  17. package/dist/assets/stash-skeleton/facts/conventions/assets/fact.md +24 -0
  18. package/dist/assets/stash-skeleton/facts/conventions/assets/knowledge.md +22 -0
  19. package/dist/assets/stash-skeleton/facts/conventions/assets/lesson.md +25 -0
  20. package/dist/assets/stash-skeleton/facts/conventions/assets/memory.md +21 -0
  21. package/dist/assets/stash-skeleton/facts/conventions/assets/script.md +21 -0
  22. package/dist/assets/stash-skeleton/facts/conventions/assets/skill.md +23 -0
  23. package/dist/assets/stash-skeleton/facts/conventions/assets/workflow.md +22 -0
  24. package/dist/assets/templates/html/health.html +281 -111
  25. package/dist/cli.js +14 -3
  26. package/dist/commands/agent/contribute-cli.js +16 -3
  27. package/dist/commands/feedback-cli.js +15 -6
  28. package/dist/commands/graph/graph.js +75 -71
  29. package/dist/commands/health/checks.js +48 -0
  30. package/dist/commands/health/html-report.js +422 -80
  31. package/dist/commands/health.js +381 -9
  32. package/dist/commands/improve/calibration.js +161 -0
  33. package/dist/commands/improve/consolidate.js +631 -111
  34. package/dist/commands/improve/dedup.js +482 -0
  35. package/dist/commands/improve/distill.js +163 -69
  36. package/dist/commands/improve/encoding-salience.js +205 -0
  37. package/dist/commands/improve/extract-cli.js +115 -1
  38. package/dist/commands/improve/extract-prompt.js +39 -2
  39. package/dist/commands/improve/extract-watch.js +140 -0
  40. package/dist/commands/improve/extract.js +403 -40
  41. package/dist/commands/improve/feedback-valence.js +54 -0
  42. package/dist/commands/improve/homeostatic.js +467 -0
  43. package/dist/commands/improve/improve-auto-accept.js +113 -6
  44. package/dist/commands/improve/improve-profiles.js +12 -0
  45. package/dist/commands/improve/improve.js +2042 -612
  46. package/dist/commands/improve/memory/memory-contradiction-detect.js +23 -28
  47. package/dist/commands/improve/outcome-loop.js +256 -0
  48. package/dist/commands/improve/proactive-maintenance.js +115 -0
  49. package/dist/commands/improve/procedural.js +418 -0
  50. package/dist/commands/improve/recombine.js +602 -0
  51. package/dist/commands/improve/reflect-noise.js +0 -0
  52. package/dist/commands/improve/reflect.js +46 -4
  53. package/dist/commands/improve/related-sessions.js +120 -0
  54. package/dist/commands/improve/salience.js +438 -0
  55. package/dist/commands/improve/triage.js +93 -0
  56. package/dist/commands/lint/agent-linter.js +19 -24
  57. package/dist/commands/lint/base-linter.js +173 -60
  58. package/dist/commands/lint/command-linter.js +19 -24
  59. package/dist/commands/lint/env-key-rules.js +34 -1
  60. package/dist/commands/lint/fact-linter.js +39 -0
  61. package/dist/commands/lint/index.js +31 -13
  62. package/dist/commands/lint/memory-linter.js +1 -1
  63. package/dist/commands/lint/registry.js +7 -2
  64. package/dist/commands/lint/task-linter.js +3 -3
  65. package/dist/commands/lint/workflow-linter.js +26 -1
  66. package/dist/commands/proposal/drain-policies.js +5 -0
  67. package/dist/commands/proposal/drain.js +17 -1
  68. package/dist/commands/proposal/proposal.js +5 -0
  69. package/dist/commands/proposal/propose.js +5 -0
  70. package/dist/commands/proposal/validators/proposal-quality-validators.js +9 -8
  71. package/dist/commands/proposal/validators/proposals.js +187 -57
  72. package/dist/commands/read/curate.js +344 -80
  73. package/dist/commands/read/search-cli.js +7 -0
  74. package/dist/commands/read/search.js +1 -0
  75. package/dist/commands/read/show.js +67 -2
  76. package/dist/commands/sources/init.js +36 -9
  77. package/dist/commands/sources/installed-stashes.js +5 -1
  78. package/dist/commands/sources/schema-repair.js +13 -1
  79. package/dist/commands/sources/stash-cli.js +19 -3
  80. package/dist/commands/sources/stash-skeleton.js +23 -8
  81. package/dist/core/asset/asset-registry.js +2 -0
  82. package/dist/core/asset/asset-spec.js +14 -0
  83. package/dist/core/asset/frontmatter.js +166 -167
  84. package/dist/core/asset/markdown.js +8 -0
  85. package/dist/core/authoring-rules.js +83 -0
  86. package/dist/core/config/config-schema.js +274 -2
  87. package/dist/core/config/config.js +2 -2
  88. package/dist/core/logs-db.js +4 -3
  89. package/dist/core/paths.js +3 -0
  90. package/dist/core/standards/resolve-standards-context.js +87 -0
  91. package/dist/core/standards/resolve-stash-standards.js +99 -0
  92. package/dist/core/standards/resolve-type-conventions.js +66 -0
  93. package/dist/core/state-db.js +691 -30
  94. package/dist/indexer/db/db.js +364 -38
  95. package/dist/indexer/db/graph-db.js +129 -86
  96. package/dist/indexer/ensure-index.js +152 -17
  97. package/dist/indexer/graph/graph-boost.js +51 -41
  98. package/dist/indexer/graph/graph-extraction.js +203 -3
  99. package/dist/indexer/index-writer-lock.js +99 -0
  100. package/dist/indexer/indexer.js +114 -111
  101. package/dist/indexer/passes/memory-inference.js +10 -3
  102. package/dist/indexer/passes/staleness-detect.js +2 -5
  103. package/dist/indexer/search/db-search.js +15 -4
  104. package/dist/indexer/search/ranking-contributors.js +22 -0
  105. package/dist/indexer/search/ranking.js +4 -0
  106. package/dist/indexer/walk/matchers.js +9 -0
  107. package/dist/integrations/agent/prompts.js +33 -0
  108. package/dist/integrations/harnesses/claude/session-log.js +11 -1
  109. package/dist/integrations/harnesses/opencode/session-log.js +173 -3
  110. package/dist/integrations/session-logs/index.js +16 -0
  111. package/dist/llm/client.js +23 -4
  112. package/dist/llm/embedder.js +27 -3
  113. package/dist/llm/embedders/local.js +66 -2
  114. package/dist/llm/feature-gate.js +8 -4
  115. package/dist/llm/graph-extract.js +2 -1
  116. package/dist/llm/memory-infer.js +4 -8
  117. package/dist/llm/metadata-enhance.js +9 -1
  118. package/dist/output/renderers.js +73 -1
  119. package/dist/output/shapes/curate.js +14 -2
  120. package/dist/output/text/helpers.js +16 -1
  121. package/dist/runtime.js +25 -1
  122. package/dist/scripts/migrate-storage.js +1378 -599
  123. package/dist/scripts/migrations/import-fs-improve-runs-to-db.js +479 -270
  124. package/dist/setup/setup.js +3 -3
  125. package/dist/sources/providers/tar-utils.js +16 -8
  126. package/dist/storage/sqlite-pragmas.js +146 -0
  127. package/dist/wiki/wiki.js +37 -0
  128. package/dist/workflows/db.js +3 -4
  129. package/dist/workflows/validate-summary.js +2 -7
  130. package/docs/data-and-telemetry.md +1 -0
  131. package/package.json +8 -6
@@ -4,6 +4,7 @@
4
4
  import fs from "node:fs";
5
5
  import os from "node:os";
6
6
  import path from "node:path";
7
+ import { openDatabase } from "../../../storage/database.js";
7
8
  import { extractInlineRefMentions } from "../../session-logs/inline-refs.js";
8
9
  function getOpenCodeBaseDir() {
9
10
  if (process.platform === "darwin") {
@@ -12,20 +13,49 @@ function getOpenCodeBaseDir() {
12
13
  return path.join(os.homedir(), ".local", "share", "opencode");
13
14
  }
14
15
  /**
15
- * Opencode storage layout (observed 2026-05):
16
- * <base>/storage/session/<projectId>/<sessionId>.json — metadata
17
- * <base>/storage/message/<sessionId>/<messageId>.json one per message
16
+ * Opencode storage layouts:
17
+ *
18
+ * SQLite (current, observed 2026-06): `<base>/opencode.db` a Drizzle-managed
19
+ * database with `session` / `message` / `part` tables. Message text lives in
20
+ * `part` rows (`data` JSON, `type: "text"`); `message.data` holds role/timing.
21
+ * This is the layout current opencode builds write; it is preferred whenever
22
+ * `opencode.db` exists.
23
+ *
24
+ * JSON files (legacy, observed 2026-05): `<base>/storage/session/<projectId>/
25
+ * <sessionId>.json` (metadata) + `<base>/storage/message/<sessionId>/
26
+ * <messageId>.json` (one per message). Read only when `opencode.db` is absent.
18
27
  *
19
28
  * Older builds wrote logs directly into `<base>/log/` and `<base>/*.log`;
20
29
  * those are still scanned by {@link OpenCodeProvider.readEvents} for
21
30
  * backward compatibility with the existing failure-pattern aggregator.
22
31
  */
32
+ /** Filename of opencode's SQLite session store, relative to its base dir. */
33
+ const OPENCODE_DB_FILENAME = "opencode.db";
23
34
  export class OpenCodeProvider {
24
35
  name = "opencode";
25
36
  #baseDir = getOpenCodeBaseDir();
26
37
  isAvailable() {
27
38
  return fs.existsSync(this.#baseDir);
28
39
  }
40
+ /** Absolute path to opencode's SQLite store under `base`. */
41
+ #dbPath(base) {
42
+ return path.join(base, OPENCODE_DB_FILENAME);
43
+ }
44
+ /**
45
+ * Directories/files opencode writes session data under. Returns the base dir
46
+ * when the SQLite store (`opencode.db`) exists, the legacy JSON session root
47
+ * (`<base>/storage/session`) when present, or both during a migration overlap.
48
+ * Empty when neither exists. See {@link SessionLogHarness.watchRoots}.
49
+ */
50
+ watchRoots() {
51
+ const roots = [];
52
+ if (fs.existsSync(this.#dbPath(this.#baseDir)))
53
+ roots.push(this.#baseDir);
54
+ const sessionRoot = path.join(this.#baseDir, "storage", "session");
55
+ if (fs.existsSync(sessionRoot))
56
+ roots.push(sessionRoot);
57
+ return roots;
58
+ }
29
59
  *readEvents(input) {
30
60
  // Legacy behavior: stream raw log lines from the top-level dir and `log/`
31
61
  // subdirectory. Kept to keep `getExecutionLogCandidates` working without
@@ -82,6 +112,9 @@ export class OpenCodeProvider {
82
112
  listSessions(input = {}) {
83
113
  const base = input.location ?? this.#baseDir;
84
114
  const sinceMs = input.sinceMs ?? 0;
115
+ const dbPath = this.#dbPath(base);
116
+ if (fs.existsSync(dbPath))
117
+ return this.#listSessionsFromDb(dbPath, sinceMs);
85
118
  const sessionRoot = path.join(base, "storage", "session");
86
119
  if (!fs.existsSync(sessionRoot))
87
120
  return [];
@@ -142,6 +175,8 @@ export class OpenCodeProvider {
142
175
  return summaries.sort((a, b) => (b.endedAt ?? 0) - (a.endedAt ?? 0));
143
176
  }
144
177
  readSession(ref) {
178
+ if (path.basename(ref.filePath) === OPENCODE_DB_FILENAME)
179
+ return this.#readSessionFromDb(ref);
145
180
  let meta = {};
146
181
  try {
147
182
  meta = JSON.parse(fs.readFileSync(ref.filePath, "utf8"));
@@ -199,6 +234,141 @@ export class OpenCodeProvider {
199
234
  inlineRefs,
200
235
  };
201
236
  }
237
+ /**
238
+ * List sessions from the SQLite store. `filePath` on each summary is the
239
+ * `opencode.db` path so {@link readSession} can route back to the DB reader.
240
+ * Returns `[]` (never throws) when the DB is unreadable or lacks the expected
241
+ * schema — callers treat a missing harness as "no sessions".
242
+ */
243
+ #listSessionsFromDb(dbPath, sinceMs) {
244
+ let db;
245
+ try {
246
+ db = openDatabase(dbPath, { readonly: true, create: false });
247
+ }
248
+ catch {
249
+ return [];
250
+ }
251
+ try {
252
+ const rows = db
253
+ .prepare("SELECT id, title, directory, time_created, time_updated FROM session WHERE time_updated >= ? ORDER BY time_updated DESC")
254
+ .all(sinceMs);
255
+ return rows.map((r) => {
256
+ const startedAt = typeof r.time_created === "number" ? r.time_created : undefined;
257
+ const endedAt = typeof r.time_updated === "number" ? r.time_updated : undefined;
258
+ const title = typeof r.title === "string" && r.title.length > 0 ? r.title : undefined;
259
+ const projectHint = typeof r.directory === "string" && r.directory.length > 0 ? r.directory : undefined;
260
+ return {
261
+ harness: this.name,
262
+ sessionId: r.id,
263
+ filePath: dbPath,
264
+ ...(startedAt !== undefined ? { startedAt } : {}),
265
+ ...(endedAt !== undefined ? { endedAt } : {}),
266
+ ...(projectHint ? { projectHint } : {}),
267
+ ...(title ? { title } : {}),
268
+ };
269
+ });
270
+ }
271
+ catch {
272
+ // Missing `session` table / unexpected schema — treat as no sessions.
273
+ return [];
274
+ }
275
+ finally {
276
+ db.close();
277
+ }
278
+ }
279
+ /**
280
+ * Read one session from the SQLite store. Message text lives in `part` rows
281
+ * (`type: "text"`); `message.data` carries role + timing. One event per
282
+ * message, text-parts concatenated in time order. Returns empty events
283
+ * (never throws) when the DB is unreadable.
284
+ */
285
+ #readSessionFromDb(ref) {
286
+ const emptyRef = { harness: this.name, sessionId: ref.sessionId, filePath: ref.filePath };
287
+ let db;
288
+ try {
289
+ db = openDatabase(ref.filePath, { readonly: true, create: false });
290
+ }
291
+ catch {
292
+ return { ref: emptyRef, events: [], inlineRefs: [] };
293
+ }
294
+ try {
295
+ const meta = db
296
+ .prepare("SELECT title, directory, time_created, time_updated FROM session WHERE id = ?")
297
+ .get(ref.sessionId);
298
+ const startedAt = typeof meta?.time_created === "number" ? meta.time_created : undefined;
299
+ const endedAt = typeof meta?.time_updated === "number" ? meta.time_updated : undefined;
300
+ const title = typeof meta?.title === "string" && meta.title.length > 0 ? meta.title : undefined;
301
+ const projectHint = typeof meta?.directory === "string" && meta.directory.length > 0 ? meta.directory : undefined;
302
+ const messages = db
303
+ .prepare("SELECT id, data, time_created FROM message WHERE session_id = ? ORDER BY time_created ASC, id ASC")
304
+ .all(ref.sessionId);
305
+ const parts = db
306
+ .prepare("SELECT message_id, data FROM part WHERE session_id = ? ORDER BY time_created ASC, id ASC")
307
+ .all(ref.sessionId);
308
+ // Group text-part bodies by their parent message.
309
+ const textByMessage = new Map();
310
+ for (const part of parts) {
311
+ let parsed;
312
+ try {
313
+ parsed = JSON.parse(part.data);
314
+ }
315
+ catch {
316
+ continue;
317
+ }
318
+ if (parsed?.type !== "text")
319
+ continue;
320
+ const text = parsed.text;
321
+ if (typeof text !== "string" || text.length < 1)
322
+ continue;
323
+ const bucket = textByMessage.get(part.message_id) ?? [];
324
+ bucket.push(text);
325
+ textByMessage.set(part.message_id, bucket);
326
+ }
327
+ const events = [];
328
+ const inlineRefs = [];
329
+ for (const message of messages) {
330
+ let mdata = {};
331
+ try {
332
+ mdata = JSON.parse(message.data);
333
+ }
334
+ catch {
335
+ // role/timing unavailable — fall through with defaults
336
+ }
337
+ const role = typeof mdata.role === "string" ? mdata.role : "unknown";
338
+ const mtime = mdata.time?.created;
339
+ const ts = typeof mtime === "number"
340
+ ? mtime
341
+ : typeof message.time_created === "number"
342
+ ? message.time_created
343
+ : undefined;
344
+ const text = (textByMessage.get(message.id) ?? []).join("\n").trim();
345
+ if (text.length < 1)
346
+ continue;
347
+ events.push({ harness: this.name, text, ts, sessionId: ref.sessionId, role, filePath: ref.filePath });
348
+ inlineRefs.push(...extractInlineRefMentions(text, ts));
349
+ }
350
+ events.sort((a, b) => (a.ts ?? 0) - (b.ts ?? 0));
351
+ return {
352
+ ref: {
353
+ harness: this.name,
354
+ sessionId: ref.sessionId,
355
+ filePath: ref.filePath,
356
+ ...(startedAt !== undefined ? { startedAt } : {}),
357
+ ...(endedAt !== undefined ? { endedAt } : {}),
358
+ ...(projectHint ? { projectHint } : {}),
359
+ ...(title ? { title } : {}),
360
+ },
361
+ events,
362
+ inlineRefs,
363
+ };
364
+ }
365
+ catch {
366
+ return { ref: emptyRef, events: [], inlineRefs: [] };
367
+ }
368
+ finally {
369
+ db.close();
370
+ }
371
+ }
202
372
  /**
203
373
  * Derive opencode base dir from a session metadata file path so a caller
204
374
  * passing a custom `--location` can still find the message dir.
@@ -28,6 +28,22 @@ const ERROR_PATTERNS = /error|failed|exception|cannot|undefined|null pointer|ENO
28
28
  export function getAvailableHarnesses() {
29
29
  return HARNESSES.filter((harness) => harness.isAvailable());
30
30
  }
31
+ /**
32
+ * Map each available harness to its `{ harnessName, roots }` watch target,
33
+ * skipping harnesses that expose no roots (absent `watchRoots()` or an empty
34
+ * result). This is the one stable entry point the watcher uses so it never
35
+ * reaches into providers directly.
36
+ */
37
+ export function getWatchTargets() {
38
+ const targets = [];
39
+ for (const harness of getAvailableHarnesses()) {
40
+ const roots = harness.watchRoots?.() ?? [];
41
+ if (roots.length === 0)
42
+ continue;
43
+ targets.push({ harnessName: harness.name, roots });
44
+ }
45
+ return targets;
46
+ }
31
47
  export function normalizeSessionTopic(text) {
32
48
  const normalized = text.replace(/\s+/g, " ").trim().toLowerCase();
33
49
  if (normalized.length < 10)
@@ -119,9 +119,23 @@ function looksLikeContextOverflow(message) {
119
119
  /**
120
120
  * Decide whether a first-attempt {@link LlmCallError} is eligible for a single
121
121
  * retry. Retryable: HTTP 5xx (`provider_error` with statusCode >= 500) and
122
- * `network_error` whose message looks like a transient connection reset
123
- * (ECONNRESET / EPIPE / "fetch failed"). NOT retryable: 4xx, `rate_limited`
124
- * (429), `timeout`, `parse_error`, and context-overflow-classified errors.
122
+ * `network_error` whose message looks like a transient connection drop.
123
+ * NOT retryable: 4xx, `rate_limited` (429), `timeout`, `parse_error`, and
124
+ * context-overflow-classified errors.
125
+ *
126
+ * The connection-drop heuristic covers the substrings emitted across runtimes
127
+ * for a mid-flight socket close:
128
+ * - `ECONNRESET` / `EPIPE` — Node/libuv socket reset codes
129
+ * - `fetch failed` — undici's generic wrapper message
130
+ * - `socket connection was closed` — Bun's message for a dropped connection
131
+ * (e.g. "The socket connection was closed unexpectedly.")
132
+ * - `terminated` / `other side closed` — undici's phrasings for the same
133
+ *
134
+ * These all describe a transient transport failure where a second attempt can
135
+ * legitimately succeed, which is exactly the case a single bounded retry is
136
+ * meant to absorb. Before this list was widened, Bun's "socket connection was
137
+ * closed unexpectedly" fell through unretried and surfaced as a recurring
138
+ * failure in the improve/reflect and capability-probe flows.
125
139
  */
126
140
  function isRetryable(err) {
127
141
  if (looksLikeContextOverflow(err.message))
@@ -131,7 +145,12 @@ function isRetryable(err) {
131
145
  }
132
146
  if (err.code === "network_error") {
133
147
  const lower = err.message.toLowerCase();
134
- return lower.includes("econnreset") || lower.includes("epipe") || lower.includes("fetch failed");
148
+ return (lower.includes("econnreset") ||
149
+ lower.includes("epipe") ||
150
+ lower.includes("fetch failed") ||
151
+ lower.includes("socket connection was closed") ||
152
+ lower.includes("terminated") ||
153
+ lower.includes("other side closed"));
135
154
  }
136
155
  return false;
137
156
  }
@@ -2,7 +2,7 @@
2
2
  // License, v. 2.0. If a copy of the MPL was not distributed with this
3
3
  // file, You can obtain one at https://mozilla.org/MPL/2.0/.
4
4
  import { embedCacheKey, getCachedEmbedding, setCachedEmbedding } from "./embedders/cache.js";
5
- import { isTransformersAvailable, LocalEmbedder } from "./embedders/local.js";
5
+ import { DEFAULT_LOCAL_MODEL, isTransformersAvailable, LocalEmbedder } from "./embedders/local.js";
6
6
  import { hasRemoteEndpoint, RemoteEmbedder } from "./embedders/remote.js";
7
7
  // ── Re-exports (public API) ─────────────────────────────────────────────────
8
8
  export { clearEmbeddingCache } from "./embedders/cache.js";
@@ -52,7 +52,8 @@ export async function embed(text, embeddingConfig, signal) {
52
52
  /**
53
53
  * Generate embeddings for multiple texts in batch.
54
54
  * Uses the OpenAI-compatible batch API for remote endpoints (batches of 100).
55
- * Falls back to sequential embedding for the local transformer pipeline.
55
+ * Uses the LocalEmbedder.embedBatch path for the local transformer pipeline,
56
+ * which processes texts in chunks of 32 for genuine batched inference.
56
57
  */
57
58
  export async function embedBatch(texts, embeddingConfig, signal) {
58
59
  if (texts.length === 0)
@@ -60,8 +61,13 @@ export async function embedBatch(texts, embeddingConfig, signal) {
60
61
  if (embeddingConfig && hasRemoteEndpoint(embeddingConfig)) {
61
62
  return new RemoteEmbedder(embeddingConfig).embedBatch(texts, signal);
62
63
  }
63
- // Local transformer: process sequentially (pipeline handles one at a time)
64
+ // Local transformer: use the batched path (chunks of 32 via LocalEmbedder).
65
+ // When a localModel override is set we cannot share the singleton (which uses
66
+ // the default model), so fall back to per-text embedWithModel in that case.
64
67
  const localModel = embeddingConfig?.localModel;
68
+ if (!localModel) {
69
+ return getLocalEmbedder().embedBatch(texts, signal);
70
+ }
65
71
  const results = [];
66
72
  for (const text of texts) {
67
73
  if (signal?.aborted) {
@@ -77,6 +83,24 @@ export async function embedBatch(texts, embeddingConfig, signal) {
77
83
  // facade and its `@huggingface/transformers` import chain. Re-export
78
84
  // preserves the existing public API.
79
85
  export { cosineSimilarity } from "./embedders/types.js";
86
+ // ── Model ID resolution ─────────────────────────────────────────────────────
87
+ /**
88
+ * Derive a stable string identifier for the embedding model in use.
89
+ * This is the `model_id` stored in `body_embeddings` (and used for the
90
+ * drop-all-on-mismatch purge when the model changes).
91
+ *
92
+ * Rules:
93
+ * - Remote endpoint: use `config.model` (the API-level model name).
94
+ * - Local transformers: use `config.localModel ?? DEFAULT_LOCAL_MODEL`.
95
+ * - No config: use `DEFAULT_LOCAL_MODEL` (the shared singleton model).
96
+ */
97
+ export function resolveEmbeddingModelId(embeddingConfig) {
98
+ if (!embeddingConfig)
99
+ return DEFAULT_LOCAL_MODEL;
100
+ if (hasRemoteEndpoint(embeddingConfig))
101
+ return embeddingConfig.model ?? "remote";
102
+ return embeddingConfig.localModel ?? DEFAULT_LOCAL_MODEL;
103
+ }
80
104
  // ── Availability check ──────────────────────────────────────────────────────
81
105
  /**
82
106
  * Check whether embedding is available with a detailed reason on failure.
@@ -19,8 +19,24 @@ import { getDirname, resolveModule } from "../../runtime.js";
19
19
  * `all-MiniLM-L6-v2` at the same 384-dimension footprint.
20
20
  */
21
21
  export const DEFAULT_LOCAL_MODEL = "Xenova/bge-small-en-v1.5";
22
+ /** Type-guard: true when the value looks like a batch Tensor (has .dims). */
23
+ function isBatchTensor(v) {
24
+ return (v !== null &&
25
+ typeof v === "object" &&
26
+ "data" in v &&
27
+ "dims" in v &&
28
+ Array.isArray(v.dims) &&
29
+ v.dims.length >= 2);
30
+ }
22
31
  const LOCAL_EMBEDDER_DTYPE = "fp32";
23
32
  const LOCAL_EMBEDDER_FALLBACK_DTYPE = "auto";
33
+ /**
34
+ * Maximum texts per batch for the local transformers pipeline. The pipeline
35
+ * can run genuine batched inference over a string array; 32 is a safe default
36
+ * that fits well inside most model context budgets while providing 10–50×
37
+ * throughput improvement over one-at-a-time calls on the cold minority.
38
+ */
39
+ const LOCAL_BATCH_SIZE = 32;
24
40
  /**
25
41
  * Return the local model name that will be used for embedding.
26
42
  * When `overrideModel` is provided it takes precedence; otherwise
@@ -77,15 +93,63 @@ export class LocalEmbedder {
77
93
  }
78
94
  return this.embedWithModel(text, this.defaultModel);
79
95
  }
96
+ /**
97
+ * Embed a batch of texts. Processes in chunks of `LOCAL_BATCH_SIZE` (32) so
98
+ * the transformers pipeline can run genuine batched inference rather than one
99
+ * call per text. Falls back to one-at-a-time if the pipeline does not support
100
+ * array input (older versions of @huggingface/transformers). Each chunk is
101
+ * checked against the AbortSignal between calls.
102
+ */
80
103
  async embedBatch(texts, signal) {
81
104
  if (texts.length === 0)
82
105
  return [];
106
+ if (signal?.aborted) {
107
+ throw signal.reason instanceof Error ? signal.reason : new Error("embedding interrupted");
108
+ }
109
+ const pipeline = await this.getPipeline(this.defaultModel);
83
110
  const results = [];
84
- for (const text of texts) {
111
+ for (let i = 0; i < texts.length; i += LOCAL_BATCH_SIZE) {
85
112
  if (signal?.aborted) {
86
113
  throw signal.reason instanceof Error ? signal.reason : new Error("embedding interrupted");
87
114
  }
88
- results.push(await this.embedWithModel(text, this.defaultModel));
115
+ const chunk = texts.slice(i, i + LOCAL_BATCH_SIZE);
116
+ try {
117
+ // @huggingface/transformers feature-extraction pipeline accepts a
118
+ // string[] and returns a batch Tensor (NOT an Array<{data}>).
119
+ // The Tensor has .data (flat Float32Array, length = batch * dim) and
120
+ // .dims = [batch, dim]. Slice .data into per-row vectors using .dims.
121
+ const batchResult = await pipeline(chunk, {
122
+ pooling: "mean",
123
+ normalize: true,
124
+ });
125
+ if (isBatchTensor(batchResult)) {
126
+ const dim = batchResult.dims[1];
127
+ for (let row = 0; row < chunk.length; row++) {
128
+ results.push(Array.from(batchResult.data.subarray(row * dim, (row + 1) * dim)));
129
+ }
130
+ }
131
+ else if (Array.isArray(batchResult)) {
132
+ // Older versions of @huggingface/transformers returned Array<{data}>.
133
+ for (const r of batchResult) {
134
+ results.push(Array.from(r.data));
135
+ }
136
+ }
137
+ else {
138
+ // Single-text result returned for a chunk — should not happen for
139
+ // string[] input, but handle defensively.
140
+ throw new Error("unexpected pipeline return shape for batch input");
141
+ }
142
+ }
143
+ catch {
144
+ // Fallback: process one-at-a-time (older pipeline versions or mismatched
145
+ // return type). Fail-open per text: a single failure aborts the chunk.
146
+ for (const text of chunk) {
147
+ if (signal?.aborted) {
148
+ throw signal.reason instanceof Error ? signal.reason : new Error("embedding interrupted");
149
+ }
150
+ results.push(await this.embedWithModel(text, this.defaultModel));
151
+ }
152
+ }
89
153
  }
90
154
  return results;
91
155
  }
@@ -30,10 +30,14 @@ const FEATURE_LOCATION = {
30
30
  proposal_quality_gate: (cfg) => cfg.profiles?.improve?.default?.processes?.reflect?.qualityGate?.enabled ?? false,
31
31
  // Legacy default: false
32
32
  memory_contradiction_detection: (cfg) => cfg.profiles?.improve?.default?.processes?.consolidate?.contradictionDetection?.enabled ?? false,
33
- // Default: true. Session extraction replaces the akm-plugin checkpoint hook
34
- // and is the primary path for capturing durable signal from real sessions.
35
- // Opt out via `profiles.improve.default.processes.extract.enabled: false`.
36
- session_extraction: (cfg) => cfg.profiles?.improve?.default?.processes?.extract?.enabled ?? true,
33
+ // Always on at the LLM-wrapper level. Enablement is decided ONCE at the
34
+ // extract entry point (`akmExtract`): the `extract.enabled` process toggle
35
+ // gates extract as a STAGE of `akm improve` (the active improve profile, per
36
+ // #593/#594), while an explicit `akm extract` command always runs. Gating the
37
+ // inner LLM calls on `default.processes.extract.enabled` here was a footgun —
38
+ // dropping extract from the daily improve profile silently disabled the
39
+ // standalone `akm extract` command. (cfg unused — kept for resolver signature.)
40
+ session_extraction: (_cfg) => true,
37
41
  };
38
42
  /**
39
43
  * Pure predicate: is the named feature gate enabled in `config`?
@@ -20,6 +20,7 @@
20
20
  * the connection via `resolveIndexPassLLM("graph", config)` and pass it
21
21
  * straight through.
22
22
  */
23
+ import systemPromptTemplate from "../assets/prompts/graph-extract-system.md" with { type: "text" };
23
24
  import userPromptTemplate from "../assets/prompts/graph-extract-user-prompt.md" with { type: "text" };
24
25
  import { toErrorMessage } from "../core/common.js";
25
26
  import { warn, warnVerbose } from "../core/warn.js";
@@ -41,7 +42,7 @@ const NON_ARRAY_BATCH_DISABLE_THRESHOLD = 2;
41
42
  const MAX_ENTITIES_PER_ASSET = 32;
42
43
  /** Hard cap on relations returned per asset. */
43
44
  const MAX_RELATIONS_PER_ASSET = 32;
44
- const SYSTEM_PROMPT = "You extract a knowledge graph from developer notes. Return ONLY valid JSON — no prose, no markdown fences, no preamble.";
45
+ const SYSTEM_PROMPT = systemPromptTemplate;
45
46
  const USER_PROMPT_PREFIX = userPromptTemplate
46
47
  .replace("{{MAX_ENTITIES}}", String(MAX_ENTITIES_PER_ASSET))
47
48
  .replace("{{MAX_RELATIONS}}", String(MAX_RELATIONS_PER_ASSET));
@@ -18,20 +18,16 @@
18
18
  * the connection via `resolveIndexPassLLM("memory", config)` and pass it
19
19
  * straight through.
20
20
  */
21
+ import memoryInferSystemPrompt from "../assets/prompts/memory-infer-system.md" with { type: "text" };
22
+ import memoryInferUserPrompt from "../assets/prompts/memory-infer-user.md" with { type: "text" };
21
23
  import { toErrorMessage } from "../core/common.js";
22
24
  import { warn } from "../core/warn.js";
23
25
  import { chatCompletion, LlmCallError, parseEmbeddedJsonResponse } from "./client.js";
24
26
  import { tryLlmFeature } from "./feature-gate.js";
25
27
  /** Hard cap on body chars sent to the model — pragmatic and matches `runLlmEnrich`. */
26
28
  const MAX_BODY_CHARS = 4000;
27
- const SYSTEM_PROMPT = "You compress a developer memory into one high-signal derived memory for later retrieval. " +
28
- "Return only valid JSON. No prose outside the JSON object. No markdown fences.";
29
- const USER_PROMPT_PREFIX = `Compress the memory below into one derived memory. Output ONLY JSON:
30
- {"title":"short title string","description":"one sentence summary string","tags":["tag1","tag2"],"searchHints":["search phrase 1","search phrase 2"],"content":"2-3 sentence compressed body preserving key facts verbatim"}
31
- Rules: be specific, no vague generalizations, preserve key facts (names/versions/paths/config keys verbatim), merge related points, 3-8 tags, 3-6 searchHints. The content field must be a plain string with 2-3 sentences.
32
-
33
- Memory:
34
- `;
29
+ const SYSTEM_PROMPT = memoryInferSystemPrompt;
30
+ const USER_PROMPT_PREFIX = memoryInferUserPrompt;
35
31
  /**
36
32
  * Strict JSON Schema for the derived-memory payload. Sent to providers that
37
33
  * opt in via `LlmConnectionConfig.supportsJsonSchema = true`; the client
@@ -1,9 +1,17 @@
1
1
  // This Source Code Form is subject to the terms of the Mozilla Public
2
2
  // License, v. 2.0. If a copy of the MPL was not distributed with this
3
3
  // file, You can obtain one at https://mozilla.org/MPL/2.0/.
4
+ /**
5
+ * LLM-driven metadata enhancement for stash entries.
6
+ *
7
+ * Split out of `llm.ts` so the higher-level workflow (prompting the LLM to
8
+ * improve descriptions/tags/searchHints) lives separately from the low-level
9
+ * transport client in `client.ts`.
10
+ */
11
+ import metadataEnhanceSystemPrompt from "../assets/prompts/metadata-enhance-system.md" with { type: "text" };
4
12
  import { chatCompletion, parseJsonResponse } from "./client.js";
5
13
  import { tryLlmFeature } from "./feature-gate.js";
6
- const SYSTEM_PROMPT = `You are a metadata generator for a developer asset registry. Given a script/skill/command/agent entry, generate improved metadata. Respond with ONLY valid JSON, no markdown fencing.`;
14
+ const SYSTEM_PROMPT = metadataEnhanceSystemPrompt;
7
15
  /**
8
16
  * Use an LLM to enhance a stash entry's metadata: improve description,
9
17
  * generate searchHints, and suggest tags.
@@ -468,6 +468,44 @@ const sessionMdRenderer = {
468
468
  };
469
469
  },
470
470
  };
471
+ // ── 9. fact-md ───────────────────────────────────────────────────────────────
472
+ /**
473
+ * Renderer for the `fact` asset type. A fact is durable stash-level semantic
474
+ * knowledge (personal/team/project details, coding conventions, stash-meta).
475
+ * It carries `category` (personal|team|project|convention|meta) and an
476
+ * optional `pinned` flag marking it as part of the always-injected core. The
477
+ * renderer surfaces a one-liner (category + pinned marker) so an agent can tell
478
+ * at a glance what kind of fact it is and whether it is core context.
479
+ */
480
+ const factMdRenderer = {
481
+ name: "fact-md",
482
+ buildShowResponse(ctx) {
483
+ const name = deriveName(ctx);
484
+ const parsed = parseFrontmatter(ctx.content());
485
+ const fm = parsed.data;
486
+ const category = asNonEmptyString(fm.category);
487
+ const description = asNonEmptyString(fm.description);
488
+ const pinned = fm.pinned === true;
489
+ const headerParts = [
490
+ category ? `category: ${category}` : undefined,
491
+ pinned ? "pinned (core context)" : undefined,
492
+ ].filter((p) => !!p);
493
+ const action = [
494
+ "Durable stash fact — apply it as background context.",
495
+ headerParts.length > 0 ? headerParts.join(" ") : undefined,
496
+ ]
497
+ .filter((p) => !!p)
498
+ .join("\n");
499
+ return {
500
+ type: "fact",
501
+ name,
502
+ path: ctx.absPath,
503
+ action,
504
+ description,
505
+ content: parsed.content,
506
+ };
507
+ },
508
+ };
471
509
  function applySessionMetadata(entry, ctx) {
472
510
  try {
473
511
  const fm = applyFrontmatterDescriptionAndTags(entry, ctx);
@@ -501,6 +539,34 @@ function applyTocMetadata(entry, ctx) {
501
539
  // Non-fatal: skip TOC if file can't be read
502
540
  }
503
541
  }
542
+ /**
543
+ * Fact metadata: surface `category` and the `pinned` core marker as tags +
544
+ * search hints (no dedicated DB columns — same encoding pattern as session /
545
+ * task). `pinned` is mirrored to both a `pinned` tag and a `pinned` search
546
+ * hint so the ranking contributor can detect it and queries can target it.
547
+ */
548
+ function applyFactMetadata(entry, ctx) {
549
+ try {
550
+ const fm = applyFrontmatterDescriptionAndTags(entry, ctx);
551
+ const tags = new Set([...(entry.tags ?? []), "fact"]);
552
+ const hints = new Set(entry.searchHints ?? []);
553
+ const category = asNonEmptyString(fm.category);
554
+ if (category) {
555
+ tags.add(category);
556
+ hints.add(`category:${category}`);
557
+ }
558
+ if (fm.pinned === true) {
559
+ tags.add("pinned");
560
+ hints.add("pinned");
561
+ }
562
+ entry.tags = Array.from(tags).filter(Boolean);
563
+ if (hints.size > 0)
564
+ entry.searchHints = Array.from(hints).filter(Boolean);
565
+ }
566
+ catch {
567
+ // Non-fatal: skip metadata extraction on parse error
568
+ }
569
+ }
504
570
  /**
505
571
  * Parse frontmatter, apply description (if not already set) and merge tags
506
572
  * into `entry`. Returns the raw frontmatter data object so callers can access
@@ -660,6 +726,11 @@ registerMetadataContributor({
660
726
  appliesTo: ({ rendererName }) => rendererName === "session-md",
661
727
  contribute: (entry, ctx) => applySessionMetadata(entry, ctx.renderContext),
662
728
  });
729
+ registerMetadataContributor({
730
+ name: "fact-md-metadata",
731
+ appliesTo: ({ rendererName }) => rendererName === "fact-md",
732
+ contribute: (entry, ctx) => applyFactMetadata(entry, ctx.renderContext),
733
+ });
663
734
  // ── Registration ─────────────────────────────────────────────────────────────
664
735
  /** All built-in renderers. */
665
736
  const builtinRenderers = [
@@ -676,6 +747,7 @@ const builtinRenderers = [
676
747
  secretFileRenderer,
677
748
  taskMdRenderer,
678
749
  sessionMdRenderer,
750
+ factMdRenderer,
679
751
  ];
680
752
  /**
681
753
  * Register all built-in renderers with the file-context registry.
@@ -687,4 +759,4 @@ export function registerBuiltinRenderers() {
687
759
  }
688
760
  }
689
761
  // ── Named exports for testing ────────────────────────────────────────────────
690
- export { agentMdRenderer, commandMdRenderer, envFileRenderer, INTERPRETER_MAP, knowledgeMdRenderer, lessonMdRenderer, memoryMdRenderer, SETUP_SIGNALS, scriptSourceRenderer, secretFileRenderer, skillMdRenderer, wikiMdRenderer, workflowMdRenderer, };
762
+ export { agentMdRenderer, commandMdRenderer, envFileRenderer, factMdRenderer, INTERPRETER_MAP, knowledgeMdRenderer, lessonMdRenderer, memoryMdRenderer, SETUP_SIGNALS, scriptSourceRenderer, secretFileRenderer, skillMdRenderer, wikiMdRenderer, workflowMdRenderer, };
@@ -5,7 +5,7 @@ import { capDescription, NORMAL_DESCRIPTION_LIMIT, pickFields } from "./helpers.
5
5
  // Curation is a small, high-signal top-N. Even at `brief` we keep `followUp`
6
6
  // (the actionable `akm show <ref>` command) and `reason` (why this asset was
7
7
  // selected) — these are the point of curate, unlike a bulk search listing.
8
- const BRIEF_FIELDS = ["source", "type", "name", "ref", "id", "followUp", "reason"];
8
+ const BRIEF_FIELDS = ["source", "type", "name", "ref", "id", "supportRefs", "followUp", "reason"];
9
9
  const NORMAL_FIELDS = [
10
10
  "source",
11
11
  "type",
@@ -17,12 +17,24 @@ const NORMAL_FIELDS = [
17
17
  "keys",
18
18
  "parameters",
19
19
  "run",
20
+ "supportRefs",
20
21
  "followUp",
21
22
  "reason",
22
23
  "score",
23
24
  ];
24
25
  // Agent shape: the minimal field set an LLM needs to decide and act.
25
- const AGENT_FIELDS = ["source", "type", "name", "ref", "id", "description", "followUp", "reason", "score"];
26
+ const AGENT_FIELDS = [
27
+ "source",
28
+ "type",
29
+ "name",
30
+ "ref",
31
+ "id",
32
+ "description",
33
+ "supportRefs",
34
+ "followUp",
35
+ "reason",
36
+ "score",
37
+ ];
26
38
  function shapeCurateItem(item, detail, shape) {
27
39
  if (shape === "agent") {
28
40
  return capDescription(pickFields(item, AGENT_FIELDS), NORMAL_DESCRIPTION_LIMIT);