akm-cli 0.9.12 → 0.9.14-beta.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (48) hide show
  1. package/CHANGELOG.md +100 -0
  2. package/dist/assets/workflows/workflow-template.md +4 -0
  3. package/dist/commands/improve/eligibility.js +27 -15
  4. package/dist/commands/improve/improve.js +1 -0
  5. package/dist/commands/lint/base-linter.js +10 -0
  6. package/dist/commands/proposal/drain.js +48 -6
  7. package/dist/commands/proposal/proposal-cli.js +1 -0
  8. package/dist/commands/read/curate.js +3 -2
  9. package/dist/commands/read/show.js +26 -9
  10. package/dist/core/adapter/adapters/akm-adapter.js +5 -1
  11. package/dist/core/asset/markdown-fragments.js +146 -0
  12. package/dist/core/config/config-walker.js +7 -3
  13. package/dist/core/config/config.js +21 -12
  14. package/dist/core/config/schema/primitives.js +8 -2
  15. package/dist/core/errors.js +2 -0
  16. package/dist/core/lexical-score.js +25 -0
  17. package/dist/core/type-presentation.js +36 -4
  18. package/dist/indexer/index-written-assets.js +4 -0
  19. package/dist/indexer/indexer.js +5 -2
  20. package/dist/indexer/passes/metadata.js +64 -1
  21. package/dist/indexer/scan/doc-to-entry.js +3 -0
  22. package/dist/indexer/scan/drain-dir.js +33 -22
  23. package/dist/indexer/search/db-search.js +72 -14
  24. package/dist/indexer/search/name-match.js +35 -0
  25. package/dist/indexer/search/ranking-contributors.js +15 -12
  26. package/dist/indexer/search/ranking.js +42 -18
  27. package/dist/indexer/usage/show-usage.js +14 -2
  28. package/dist/llm/client.js +12 -8
  29. package/dist/llm/embedders/remote.js +3 -2
  30. package/dist/llm/graph-extract.js +18 -67
  31. package/dist/output/shapes.js +46 -1
  32. package/dist/output/text/proposal-format.js +5 -0
  33. package/dist/scripts/akm-migrate-node.js +648 -253
  34. package/dist/scripts/akm-migrate.js +648 -253
  35. package/dist/storage/repositories/index-connection.js +23 -8
  36. package/dist/storage/repositories/index-entries-repository.js +3 -2
  37. package/dist/storage/repositories/index-entry-schema.js +43 -3
  38. package/dist/storage/repositories/index-fts-repository.js +160 -14
  39. package/dist/storage/repositories/index-schema.js +8 -18
  40. package/dist/storage/repositories/workflow-runs-repository.js +118 -10
  41. package/dist/workflows/exec/run-workflow.js +1 -1
  42. package/dist/workflows/exec/step-work.js +41 -0
  43. package/dist/workflows/parser.js +1 -1
  44. package/dist/workflows/runtime/runs.js +29 -5
  45. package/docs/migration/release-notes/0.9.14.md +26 -0
  46. package/docs/migration/release-notes/README.md +2 -0
  47. package/docs/reference/cli.md +18 -0
  48. package/package.json +1 -1
@@ -27,6 +27,7 @@ import { hasRegistryUrlCredentials, REGISTRY_CREDENTIALS_UNSUPPORTED } from "../
27
27
  import { warnOnce } from "../warn.js";
28
28
  import { AkmConfigBaseSchema, EngineConfigSchema, listTopLevelConfigKeys } from "./config-schema.js";
29
29
  import { deepMergeConfig } from "./deep-merge.js";
30
+ import { isApiKeyReference } from "./schema/primitives.js";
30
31
  /**
31
32
  * Parse a dotted path into segments. Empty segments are rejected. Bracket
32
33
  * notation (e.g. `sources[0]`) is NOT supported — arrays are set as JSON.
@@ -214,9 +215,12 @@ export function configSet(config, dotted, raw) {
214
215
  const parsed = path[0] === "engines" && path.length === 2
215
216
  ? EngineConfigSchema.safeParse(value)
216
217
  : symbolicApiKey
217
- ? /^\$[A-Za-z_][A-Za-z0-9_]*$|^\$\{[A-Za-z_][A-Za-z0-9_]*\}$/.test(raw)
218
+ ? isApiKeyReference(raw)
218
219
  ? { success: true, data: value }
219
- : { success: false, error: { issues: [{ path: [], message: `apiKey must be $VAR or \${VAR}` }] } }
220
+ : {
221
+ success: false,
222
+ error: { issues: [{ path: [], message: "apiKey must be $VAR, ${VAR}, or secret://<name>" }] },
223
+ }
220
224
  : isUnknownKey
221
225
  ? { success: true, data: value }
222
226
  : schema.safeParse(candidate);
@@ -297,7 +301,7 @@ function rejectLiteralApiKeyInWholeObjectSet(path, raw, dotted) {
297
301
  }
298
302
  if (!isRecord(parsed) || typeof parsed.apiKey !== "string")
299
303
  return;
300
- if (/^\$[A-Za-z_][A-Za-z0-9_]*$|^\$\{[A-Za-z_][A-Za-z0-9_]*\}$/.test(parsed.apiKey))
304
+ if (isApiKeyReference(parsed.apiKey))
301
305
  return;
302
306
  throw new UsageError(`apiKey cannot be persisted in config; export ${recipeForApiKey([...path, "apiKey"], `${dotted}.apiKey`)} instead. (key: ${dotted}.apiKey)`, "INVALID_FLAG_VALUE", "Storing API keys in config.json leaks them through backups, logs, and version control. " +
303
307
  "Use the corresponding environment variable. AKM reads it at request time.");
@@ -12,6 +12,7 @@ import { bundlesToSourceEntries } from "./config-sources.js";
12
12
  import { upgradeConfigVersion } from "./config-version-shim.js";
13
13
  import { deepMergeConfig } from "./deep-merge.js";
14
14
  import { migrateLegacySourceShape } from "./legacy-source-shape-shim.js";
15
+ import { isApiKeyReference, SECRET_STORE_REFERENCE_PATTERN } from "./schema/primitives.js";
15
16
  export { stripJsonComments } from "./config-io.js";
16
17
  import { getConfigPath } from "../paths.js";
17
18
  import { warn, warnOnce } from "../warn.js";
@@ -299,7 +300,7 @@ export function sanitizeConfigForWrite(config) {
299
300
  const stripped = [];
300
301
  if (config.embedding?.apiKey !== undefined) {
301
302
  const apiKey = config.embedding.apiKey;
302
- if (isEnvReference(apiKey)) {
303
+ if (isApiKeyReference(apiKey)) {
303
304
  // Preserve reference verbatim — not a secret.
304
305
  sanitized.embedding = { ...config.embedding };
305
306
  }
@@ -316,7 +317,7 @@ export function sanitizeConfigForWrite(config) {
316
317
  if (config.engines) {
317
318
  const engines = {};
318
319
  for (const [name, engine] of Object.entries(config.engines)) {
319
- if (engine.kind !== "llm" || engine.apiKey === undefined || isEnvReference(engine.apiKey)) {
320
+ if (engine.kind !== "llm" || engine.apiKey === undefined || isApiKeyReference(engine.apiKey)) {
320
321
  engines[name] = { ...engine };
321
322
  continue;
322
323
  }
@@ -345,19 +346,15 @@ export function sanitizeConfigForWrite(config) {
345
346
  }
346
347
  return sanitized;
347
348
  }
348
- /** Matches the only 0.9 symbolic secret forms: `${VAR}` or `$VAR`. */
349
- function isEnvReference(value) {
350
- return /^\$\{[A-Za-z_][A-Za-z0-9_]*\}$|^\$[A-Za-z_][A-Za-z0-9_]*$/.test(value);
351
- }
352
349
  export function updateConfig(partial) {
353
350
  return mutateConfig((current) => deepMergeConfig(current, partial)).config;
354
351
  }
355
- // ── Helpers ─────────────────────────────────────────────────────────────────
356
352
  /**
357
- * Resolve a single secret value by expanding `${VAR}` / `$VAR` references
358
- * against `process.env`. Use this at apiKey /
359
- * authorization-header consumption sites (LLM client, embedder, agent SDK
360
- * runner) — NOT on the load path. Non-string inputs pass through unchanged.
353
+ * Resolve a single secret value: expand `${VAR}` / `$VAR` against
354
+ * `process.env`, or look up `secret://<name>` via `resolveFromStore`. Use this
355
+ * at apiKey / authorization-header consumption sites (LLM client, embedder,
356
+ * agent SDK runner) — NOT on the load path. Non-string inputs pass through
357
+ * unchanged.
361
358
  *
362
359
  * Returns the input unchanged when no substitution markers are present, so
363
360
  * literal API key strings (already-resolved secrets) are zero-cost.
@@ -365,12 +362,24 @@ export function updateConfig(partial) {
365
362
  * Other config string values (URLs, endpoints, model names, prompts) are
366
363
  * preserved verbatim on read — only fields explicitly routed through this
367
364
  * helper are expanded.
365
+ *
366
+ * A `secret://<name>` value that fails to resolve throws `ConfigError`
367
+ * (naming the ref, never the secret) rather than silently sending an unusable
368
+ * credential.
368
369
  */
369
- export function resolveSecret(value) {
370
+ export function resolveSecret(value, resolveFromStore) {
370
371
  if (value === undefined)
371
372
  return undefined;
372
373
  if (typeof value !== "string")
373
374
  return value;
375
+ const storeRef = SECRET_STORE_REFERENCE_PATTERN.exec(value)?.[1];
376
+ if (storeRef !== undefined) {
377
+ const resolved = resolveFromStore?.(storeRef) ?? null;
378
+ if (resolved === null) {
379
+ throw new ConfigError(`Secret store reference "${value}" did not resolve to a stored value.`, "SECRET_REFERENCE_UNRESOLVED");
380
+ }
381
+ return resolved;
382
+ }
374
383
  if (!value.includes("$"))
375
384
  return value;
376
385
  return value.replace(/\$\{([A-Za-z_][A-Za-z0-9_]*)\}|\$([A-Za-z_][A-Za-z0-9_]*)/g, (_match, braced, bare) => {
@@ -28,15 +28,21 @@ export const httpUrl = z.string().refine((v) => v.startsWith("http://") || v.sta
28
28
  });
29
29
  const ENGINE_NAME_PATTERN = new RegExp(ENGINE_NAME_PATTERN_SOURCE);
30
30
  export const ENV_REFERENCE_PATTERN = /^\$[A-Za-z_][A-Za-z0-9_]*$|^\$\{[A-Za-z_][A-Za-z0-9_]*\}$/;
31
+ /** `secret://<name>` — an apiKey reference into the akm secret store, resolved via `resolveSecretFromStore`. */
32
+ export const SECRET_STORE_REFERENCE_PATTERN = /^secret:\/\/(.+)$/;
31
33
  export const engineName = z
32
34
  .string()
33
35
  .max(63)
34
36
  .regex(ENGINE_NAME_PATTERN, "names must be lowercase kebab-case and must not begin with reserved akm-");
37
+ /** The two symbolic apiKey forms akm accepts: an env-var reference or a secret-store reference. Never matches a literal key. */
38
+ export function isApiKeyReference(value) {
39
+ return ENV_REFERENCE_PATTERN.test(value) || SECRET_STORE_REFERENCE_PATTERN.test(value);
40
+ }
35
41
  export function symbolicOrWarnApiKey(label) {
36
42
  return z.string().superRefine((value) => {
37
- if (ENV_REFERENCE_PATTERN.test(value))
43
+ if (isApiKeyReference(value))
38
44
  return;
39
- warnOnce(`config:literal-api-key:${label}`, `A ${label} in config.json is a literal API key, not a $VAR/\${VAR} reference; using it as configured. Prefer \`akm config set ...apiKey '$VAR'\` (with the corresponding env var set) — see docs/reference/data-and-telemetry.md.`);
45
+ warnOnce(`config:literal-api-key:${label}`, `A ${label} in config.json is a literal API key, not a $VAR/\${VAR}/secret:// reference; using it as configured. Prefer \`akm config set ...apiKey '$VAR'\` (with the corresponding env var set) or \`secret://<name>\` (with \`akm secret set <name> ...\`) — see docs/reference/data-and-telemetry.md.`);
40
46
  });
41
47
  }
42
48
  export const chatCompletionsEndpoint = z.string().superRefine((value, ctx) => {
@@ -17,6 +17,7 @@ const CONFIG_HINTS = {
17
17
  UNSAFE_STASH_DIR: "Choose a path inside your home directory (e.g. ~/akm) or another empty workspace. The bundle directory cannot be the filesystem root, your home directory itself, or a sensitive system path like /etc, /var, ~/.config, or ~/.ssh.",
18
18
  UNKNOWN_IMPROVE_STRATEGY: "Pass one of the listed strategy names to `--strategy`, or define it under `improve.strategies`. Names are case-sensitive.",
19
19
  EXECUTION_NOT_AUTHORIZED: "Change the selected tools or update the machine/user execution policy, then retry.",
20
+ SECRET_REFERENCE_UNRESOLVED: "Check the secret exists (`akm secret list`) and the name after `secret://` matches, or run `akm secret set <name> <value>` to store it.",
20
21
  };
21
22
  // Code-review finding: COMPOSITION_INVALID covers several unrelated causes
22
23
  // (a rejected with:, a multi-job source, a composition cycle/depth/size
@@ -76,6 +77,7 @@ const USAGE_HINTS = {
76
77
  WORKFLOW_IR_VERSION_UNSUPPORTED: "Abandon the run with `akm workflow abandon <id>`, then start it again from the workflow source — a frozen plan this akm cannot execute is not re-executable in place.",
77
78
  // P3b (docs/plans/specs/p3b-child-executor.md §4.3).
78
79
  WORKFLOW_OUTPUT_INVALID: "Check each `outputs:` entry's `from:` against the step artifact it names, and its `schema:` against the value that step actually promotes.",
80
+ RUN_LEASE_HELD: "Wait for the named engine invocation to finish or for the lease to expire, then retry. `akm workflow status <id>` shows the current lease.",
79
81
  };
80
82
  /** Default hint for each NotFoundError code. */
81
83
  const NOT_FOUND_HINTS = {
@@ -0,0 +1,25 @@
1
+ // This Source Code Form is subject to the terms of the Mozilla Public
2
+ // License, v. 2.0. If a copy of the MPL was not distributed with this
3
+ // file, You can obtain one at https://mozilla.org/MPL/2.0/.
4
+ /** Dependency-safe fixed FTS5 calibration shared by storage and ranking. */
5
+ const SCORE_FLOOR = 0.3;
6
+ const PARENT_CEILING = 0.8;
7
+ // Fragment BM25 is from a distinct FTS population. Its separately calibrated
8
+ // ceiling is an explicit evidence policy, not a cross-table comparability claim:
9
+ // when a body fragment independently proves the query, retain that actionable
10
+ // selector beside the same parent's length-penalized whole-body row. Metadata
11
+ // and cross-fragment conjunctions cannot enter this population.
12
+ const FRAGMENT_CEILING = 0.8;
13
+ const BM25_REFERENCE = 0.000001;
14
+ const LOG_SHAPE = 3;
15
+ export function stableFtsScore(bm25Score, population = "parent") {
16
+ const ceiling = population === "fragment" ? FRAGMENT_CEILING : PARENT_CEILING;
17
+ if (bm25Score === Number.NEGATIVE_INFINITY)
18
+ return ceiling;
19
+ if (!Number.isFinite(bm25Score) || bm25Score >= 0)
20
+ return SCORE_FLOOR;
21
+ const scaled = Math.log1p(-bm25Score / BM25_REFERENCE);
22
+ if (!Number.isFinite(scaled))
23
+ return ceiling;
24
+ return SCORE_FLOOR + (ceiling - SCORE_FLOOR) * (scaled / (scaled + LOG_SHAPE));
25
+ }
@@ -51,20 +51,41 @@ function buildWorkflowAction(ref) {
51
51
  * see the file header.
52
52
  */
53
53
  export const TYPE_PRESENTATION = {
54
- skill: { label: "Skill", renderer: "skill-md", action: (ref) => `akm show ${ref} -> follow the instructions` },
54
+ skill: {
55
+ label: "Skill",
56
+ renderer: "skill-md",
57
+ action: (ref) => `akm show ${ref} -> follow the instructions`,
58
+ fragmentRef: false,
59
+ },
55
60
  command: {
56
61
  label: "Command",
57
62
  renderer: "command-md",
58
63
  action: (ref) => `akm show ${ref} -> fill placeholders and dispatch`,
64
+ fragmentRef: false,
65
+ },
66
+ agent: {
67
+ label: "Agent",
68
+ renderer: "agent-md",
69
+ action: (ref) => `akm show ${ref} -> dispatch with full prompt`,
70
+ fragmentRef: false,
59
71
  },
60
- agent: { label: "Agent", renderer: "agent-md", action: (ref) => `akm show ${ref} -> dispatch with full prompt` },
61
72
  knowledge: {
62
73
  label: "Knowledge",
63
74
  renderer: "knowledge-md",
64
75
  action: (ref) => `akm show ${ref} -> read reference material`,
65
76
  },
66
- workflow: { label: "Workflow", renderer: "workflow-md", action: (ref) => buildWorkflowAction(ref) },
67
- script: { label: "Script", renderer: "script-source", action: (ref) => `akm show ${ref} -> execute the run command` },
77
+ workflow: {
78
+ label: "Workflow",
79
+ renderer: "workflow-md",
80
+ action: (ref) => buildWorkflowAction(ref),
81
+ fragmentRef: false,
82
+ },
83
+ script: {
84
+ label: "Script",
85
+ renderer: "script-source",
86
+ action: (ref) => `akm show ${ref} -> execute the run command`,
87
+ fragmentRef: false,
88
+ },
68
89
  memory: { label: "Memory", renderer: "memory-md", action: (ref) => `akm show ${ref} -> recall context` },
69
90
  env: {
70
91
  label: "Env",
@@ -85,6 +106,7 @@ export const TYPE_PRESENTATION = {
85
106
  label: "Task",
86
107
  renderer: "task-yaml",
87
108
  action: (ref) => `akm show ${ref} -> inspect; akm task run <id> -> run now; edit the file + akm task sync -> unschedule`,
109
+ fragmentRef: false,
88
110
  },
89
111
  session: {
90
112
  label: "Session",
@@ -105,6 +127,7 @@ export const TYPE_PRESENTATION = {
105
127
  label: "Instruction",
106
128
  renderer: "knowledge-md",
107
129
  action: (ref) => `akm show ${ref} -> read the project instructions`,
130
+ fragmentRef: false,
108
131
  },
109
132
  };
110
133
  /** Generic fallback for a type outside {@link KNOWN_TYPES} — never `undefined`, never a throw. */
@@ -120,6 +143,15 @@ export function presentationFor(type) {
120
143
  return TYPE_PRESENTATION[type];
121
144
  return DEFAULT_PRESENTATION;
122
145
  }
146
+ /**
147
+ * A missing declaration means a read-like reference or foreign type can expose
148
+ * a safe selector. Built-in executable and instruction-bearing types opt out
149
+ * explicitly above: their search match can be a fragment, but the advertised
150
+ * action needs the parent canonical ref and its complete context.
151
+ */
152
+ export function allowsFragmentRef(type) {
153
+ return presentationFor(type).fragmentRef !== false;
154
+ }
123
155
  export const defaultRendererRegistry = {
124
156
  rendererNameFor(type) {
125
157
  return presentationFor(type).renderer;
@@ -31,6 +31,7 @@ import { closeDatabase, openExistingDatabase } from "../storage/repositories/ind
31
31
  import { deleteEntriesByIds, getEntryCount, upsertEntry } from "../storage/repositories/index-entries-repository.js";
32
32
  import { deriveEntryProvenance, deriveInstallations } from "./installations.js";
33
33
  import { generateEmbeddingsForDb, publishTargetedEmbeddingMeta } from "./materialize-embeddings.js";
34
+ import { getMarkdownFragmentContent, hasMarkdownFragmentContent, setMarkdownFragmentContent, } from "./passes/metadata.js";
34
35
  import { drainDirDocuments } from "./scan/drain-dir.js";
35
36
  import { buildSearchText } from "./search/search-fields.js";
36
37
  import { buildFileContext } from "./walk/file-context.js";
@@ -158,6 +159,9 @@ export async function indexWrittenAssets(stashDir, filePaths, options = {}) {
158
159
  let entryWithSize = entry;
159
160
  try {
160
161
  entryWithSize = { ...entry, fileSize: fs.statSync(file).size };
162
+ if (hasMarkdownFragmentContent(entry)) {
163
+ setMarkdownFragmentContent(entryWithSize, getMarkdownFragmentContent(entry));
164
+ }
161
165
  }
162
166
  catch {
163
167
  // stat raced a delete — index without the size, like the full walk does.
@@ -31,7 +31,7 @@ import { deriveEntryProvenance, deriveInstallations } from "./installations.js";
31
31
  import { indexedPathMatchesOwner, resolveAdapterConceptOwner, } from "./lookup/adapter-concept-owner.js";
32
32
  import { generateEmbeddingsForDb } from "./materialize-embeddings.js";
33
33
  import { canUseIncrementalSkip, computeDirFingerprint, getCachedDirState, getDirIndexState, inferZeroRowReason, } from "./passes/dir-staleness.js";
34
- import { isEnrichmentComplete, isWorkflowSkipWarning } from "./passes/metadata.js";
34
+ import { getMarkdownFragmentContent, hasMarkdownFragmentContent, isEnrichmentComplete, isWorkflowSkipWarning, setMarkdownFragmentContent, } from "./passes/metadata.js";
35
35
  import { drainDirDocuments } from "./scan/drain-dir.js";
36
36
  import { buildSearchText } from "./search/search-fields.js";
37
37
  import { purgeOldUsageEvents } from "./usage/usage-events.js";
@@ -1504,7 +1504,10 @@ export function createEnrichmentDeadline(timeoutMs, totalEntries) {
1504
1504
  // ── Helpers ─────────────────────────────────────────────────────────────────
1505
1505
  function attachFileSize(entry, entryPath) {
1506
1506
  try {
1507
- return { ...entry, fileSize: fs.statSync(entryPath).size };
1507
+ const sized = { ...entry, fileSize: fs.statSync(entryPath).size };
1508
+ if (hasMarkdownFragmentContent(entry))
1509
+ setMarkdownFragmentContent(sized, getMarkdownFragmentContent(entry));
1510
+ return sized;
1508
1511
  }
1509
1512
  catch {
1510
1513
  return entry;
@@ -1179,6 +1179,67 @@ export function projectMarkdownContent(body, truncationInfo) {
1179
1179
  truncationInfo.truncated = text.length > MARKDOWN_CONTENT_MAX_CHARS;
1180
1180
  return truncateUnicodeSafe(text, MARKDOWN_CONTENT_MAX_CHARS);
1181
1181
  }
1182
+ // Fragment text is intentionally not an IndexDocument field. IndexDocument is
1183
+ // an adapter/search payload boundary; this is an internal, derived index input
1184
+ // that is persisted separately by the entries repository for deterministic FTS
1185
+ // rebuilds. The WeakMap follows the already-read document through recognition
1186
+ // without making safe body bytes observable through public payloads.
1187
+ const markdownFragmentContentByEntry = new WeakMap();
1188
+ const markdownFragmentProjectionEntries = new WeakSet();
1189
+ export function setMarkdownFragmentContent(entry, content) {
1190
+ markdownFragmentProjectionEntries.add(entry);
1191
+ if (content)
1192
+ markdownFragmentContentByEntry.set(entry, content);
1193
+ }
1194
+ export function getMarkdownFragmentContent(entry) {
1195
+ return markdownFragmentContentByEntry.get(entry);
1196
+ }
1197
+ export function hasMarkdownFragmentContent(entry) {
1198
+ return markdownFragmentProjectionEntries.has(entry);
1199
+ }
1200
+ /**
1201
+ * Produce a safe, structure-preserving projection for fragment indexing.
1202
+ * Excluded source lines are retained as blank lines so fragment locations map
1203
+ * exactly to authored line numbers. This must be called from both indexing and
1204
+ * `show`; it deliberately never performs a storage-layer file reread.
1205
+ */
1206
+ export function projectMarkdownFragmentContent(raw) {
1207
+ const lines = raw.split(/\r?\n/);
1208
+ const parsed = parseFrontmatter(raw);
1209
+ const start = parsed.frontmatter ? parsed.bodyStartLine - 1 : 0;
1210
+ const projected = lines.map(() => "");
1211
+ let fence;
1212
+ const htmlComment = { inComment: false };
1213
+ for (let index = start; index < lines.length; index++) {
1214
+ const rawLine = lines[index];
1215
+ if (fence) {
1216
+ if (isMarkdownFenceClosing(rawLine, fence))
1217
+ fence = undefined;
1218
+ continue;
1219
+ }
1220
+ if (!htmlComment.inComment) {
1221
+ const opening = parseMarkdownFenceOpening(rawLine);
1222
+ if (opening) {
1223
+ fence = opening;
1224
+ continue;
1225
+ }
1226
+ }
1227
+ let safe = stripMarkdownHtmlComments(rawLine, htmlComment);
1228
+ const opening = parseMarkdownFenceOpening(safe.trim());
1229
+ if (opening) {
1230
+ fence = opening;
1231
+ continue;
1232
+ }
1233
+ // Reference link destinations and standalone HTML are not retrieval
1234
+ // evidence and can contain credential-bearing URLs.
1235
+ if (/^\s*\[[^\]]+\]:\s*\S+/.test(safe) || /^\s*<[^>]+>\s*$/.test(safe))
1236
+ continue;
1237
+ safe = stripMarkdownLinkDestinations(safe).replace(/<[^>]+>/g, " ");
1238
+ projected[index] = safe.replace(/[ \t]+$/g, "");
1239
+ }
1240
+ const text = projected.join("\n");
1241
+ return text.trim() ? text : undefined;
1242
+ }
1182
1243
  // ── Metadata Generation ─────────────────────────────────────────────────────
1183
1244
  /**
1184
1245
  * Priorities 1-2 of the metadata pipeline — package.json (P1), `.md`
@@ -1220,7 +1281,9 @@ export function applyPreContributorFields(entry, file, ctx, pkgMeta) {
1220
1281
  applyProvenanceFrontmatter(entry, parsed.data);
1221
1282
  // Native Markdown has one bounded low-weight body projection. Sensitive
1222
1283
  // types and raw session/checkpoint material never cross this boundary.
1223
- if (entry.type !== "env" && entry.type !== "session" && !hasSessionMemoryMarker(parsed.data, parsed.content)) {
1284
+ const safeForFragments = entry.type !== "env" && entry.type !== "session" && !hasSessionMemoryMarker(parsed.data, parsed.content);
1285
+ setMarkdownFragmentContent(entry, safeForFragments ? projectMarkdownFragmentContent(content) : undefined);
1286
+ if (safeForFragments) {
1224
1287
  const truncationInfo = { truncated: false };
1225
1288
  const contentProjection = projectMarkdownContent(parsed.content, truncationInfo);
1226
1289
  if (contentProjection) {
@@ -34,6 +34,7 @@
34
34
  * Pure, type-only imports (no cycle participation).
35
35
  */
36
36
  import path from "node:path";
37
+ import { getMarkdownFragmentContent, hasMarkdownFragmentContent, setMarkdownFragmentContent, } from "../passes/metadata.js";
37
38
  /**
38
39
  * Reconstruct the `IndexDocument` an `IndexDocument` was mapped from. First-class
39
40
  * IndexDocument members and the `documentJson`-carried extras are both restored;
@@ -60,6 +61,8 @@ export function indexDocumentToStashEntry(doc) {
60
61
  entry.content = doc.content;
61
62
  if (doc.contentTruncated !== undefined)
62
63
  entry.contentTruncated = doc.contentTruncated;
64
+ if (hasMarkdownFragmentContent(doc))
65
+ setMarkdownFragmentContent(entry, getMarkdownFragmentContent(doc));
63
66
  if (doc.ownsPresentation !== undefined)
64
67
  entry.ownsPresentation = doc.ownsPresentation;
65
68
  if (doc.updated !== undefined)
@@ -30,7 +30,7 @@ import path from "node:path";
30
30
  import { akmAdapter } from "../../core/adapter/adapters/akm-adapter.js";
31
31
  import { compareCodePoints } from "../../core/common.js";
32
32
  import { canonicalizeWorkflowName } from "../../core/recognition-util.js";
33
- import { resolveUniqueWorkflowSource, WorkflowSourceRejectionError, workflowNameForSourcePath, } from "../../workflows/source-files.js";
33
+ import { resolveWorkflowSourceDomains, workflowNameForSourcePath } from "../../workflows/source-files.js";
34
34
  import { compileWorkflowSource } from "../../workflows/source-ir/compile.js";
35
35
  import { buildMetadataSkipWarning } from "../passes/metadata.js";
36
36
  import { buildFileContext } from "../walk/file-context.js";
@@ -53,32 +53,41 @@ export function drainDirDocuments(adapter, component, fileContexts) {
53
53
  const conceptIdByFile = new Map();
54
54
  const rejectedPaths = new Set();
55
55
  const rejectedConceptIds = new Set();
56
- const workflowLookups = new Map();
57
- for (const file of fileContexts) {
56
+ // A full directory drain may contain both peer workflow formats for one
57
+ // canonical ref. Ownership arbitration must happen *before* recognition:
58
+ // otherwise both documents reach the persistence fold and SQLite's final
59
+ // row is determined by the walk/readdir order. Resolve exactly the paths
60
+ // this drain owns, so a full scan retains only the deterministic `.md`
61
+ // winner while a targeted one-file reindex deliberately keeps its written
62
+ // source and therefore marks an existing peer row stale for read fallback.
63
+ const workflowOwnerPathByCanonicalName = new Map(resolveWorkflowSourceDomains(component.root, adapter.id, fileContexts.map((file) => file.absPath))
64
+ .filter((resolution) => resolution.source !== undefined)
65
+ .map((resolution) => [resolution.canonicalName, path.resolve(resolution.source.path)]));
66
+ const invalidWorkflowOwnerNames = new Set();
67
+ const orderedFileContexts = [...fileContexts].sort((left, right) => {
68
+ const leftName = workflowNameForSourcePath(component.root, adapter.id, left.absPath);
69
+ const rightName = workflowNameForSourcePath(component.root, adapter.id, right.absPath);
70
+ const leftOwner = leftName !== undefined &&
71
+ workflowOwnerPathByCanonicalName.get(canonicalizeWorkflowName(leftName)) === path.resolve(left.absPath);
72
+ const rightOwner = rightName !== undefined &&
73
+ workflowOwnerPathByCanonicalName.get(canonicalizeWorkflowName(rightName)) === path.resolve(right.absPath);
74
+ if (leftOwner !== rightOwner)
75
+ return leftOwner ? -1 : 1;
76
+ return compareCodePoints(left.absPath, right.absPath);
77
+ });
78
+ for (const file of orderedFileContexts) {
79
+ if (rejectedPaths.has(file.absPath))
80
+ continue;
58
81
  const workflowName = workflowNameForSourcePath(component.root, adapter.id, file.absPath);
59
82
  if (workflowName !== undefined) {
60
83
  const canonicalName = canonicalizeWorkflowName(workflowName);
61
- if (!workflowLookups.has(canonicalName))
62
- workflowLookups.set(canonicalName, workflowName);
63
- }
64
- }
65
- for (const [canonicalName, workflowName] of [...workflowLookups].sort(([left], [right]) => compareCodePoints(left, right))) {
66
- try {
67
- resolveUniqueWorkflowSource(component.root, adapter.id, workflowName);
68
- }
69
- catch (error) {
70
- if (!(error instanceof WorkflowSourceRejectionError))
71
- throw error;
72
- rejectedConceptIds.add(adapter.id === "akm" ? `workflows/${canonicalName}` : canonicalName);
73
- for (const relativePath of error.sourcePaths) {
74
- rejectedPaths.add(path.join(component.root, relativePath));
84
+ const ownerPath = workflowOwnerPathByCanonicalName.get(canonicalName);
85
+ if (ownerPath !== undefined &&
86
+ ownerPath !== path.resolve(file.absPath) &&
87
+ !invalidWorkflowOwnerNames.has(canonicalName)) {
88
+ continue;
75
89
  }
76
- warnings.push(error.message);
77
90
  }
78
- }
79
- for (const file of fileContexts) {
80
- if (rejectedPaths.has(file.absPath))
81
- continue;
82
91
  const doc = adapter.recognize(component, file);
83
92
  if (doc === null)
84
93
  continue;
@@ -92,6 +101,8 @@ export function drainDirDocuments(adapter, component, fileContexts) {
92
101
  const dropWarning = handleWorkflowDoc(doc, file, component.root);
93
102
  if (dropWarning !== null) {
94
103
  warnings.push(dropWarning);
104
+ if (workflowName !== undefined)
105
+ invalidWorkflowOwnerNames.add(canonicalizeWorkflowName(workflowName));
95
106
  continue;
96
107
  }
97
108
  if (doc.hash !== undefined)
@@ -17,10 +17,11 @@ import path from "node:path";
17
17
  import { buildActionFromContributors, defaultActionContributors } from "../../core/action-contributors.js";
18
18
  import { stashDirFor } from "../../core/asset/asset-placement.js";
19
19
  import { displayRef } from "../../core/asset/resolve-ref.js";
20
+ import { compareCodePoints } from "../../core/common.js";
20
21
  import { classifyPathAccess } from "../../core/path-access.js";
21
22
  import { getDbPath } from "../../core/paths.js";
22
23
  import { systemErrorCode } from "../../core/system-error.js";
23
- import { defaultRendererRegistry } from "../../core/type-presentation.js";
24
+ import { allowsFragmentRef, defaultRendererRegistry } from "../../core/type-presentation.js";
24
25
  import { normalizeEmbeddingEndpoint } from "../../llm/embedders/remote.js";
25
26
  import { assertIndexPathReadable, closeDatabase, openExistingDatabase, } from "../../storage/repositories/index-connection.js";
26
27
  import { getAllEntries, getBaseBeliefStatesForDerivedTwins, getEntryById, getEntryCount, getPositiveFeedbackCountsByIds, } from "../../storage/repositories/index-entries-repository.js";
@@ -186,13 +187,47 @@ export async function searchLocal(input) {
186
187
  }
187
188
  // ── Database search ─────────────────────────────────────────────────────────
188
189
  /**
189
- * Keep one deterministic ranking order before stable path deduplication. Exact
190
- * names survive the public score ceiling, while raw contributor differences
191
- * are quantized so utility-recency epsilon cannot reorder visible ties.
190
+ * Keep public scores in [0, 1] without flattening every boosted result to the
191
+ * same hard-clamped value. The ranking pipeline deliberately keeps its raw
192
+ * score for deterministic ordering before stable path deduplication; this
193
+ * monotone display projection preserves that order and leaves visible
194
+ * separation for graph, type, and project-context signals.
192
195
  */
196
+ function displaySearchScore(score) {
197
+ return 1 - Math.exp(-Math.max(0, score));
198
+ }
199
+ /**
200
+ * A final deterministic key for genuinely tied candidates. It deliberately
201
+ * excludes the asset name, filename, path, durable ref, and SQLite id: callers
202
+ * such as the memory-pack adapter generate each of those from an opaque source
203
+ * id, so using one here makes an otherwise equal search depend on that id.
204
+ *
205
+ * The normal AKM Markdown adapter keeps an H1 title in `content`; strip that
206
+ * one synthetic title too, because the adapter may derive it from the opaque
207
+ * filename. Identical remaining bodies are semantically indistinguishable at
208
+ * this ranking stage and intentionally continue to the existing name/path
209
+ * fallback for repeatable local presentation.
210
+ */
211
+ function asciiCaseFold(value) {
212
+ // SQLite's built-in lower() folds ASCII only unless a build opts into ICU.
213
+ // Keep this key deliberately in that portable shared subset instead of
214
+ // introducing locale-dependent JavaScript ordering for non-ASCII content.
215
+ return value.replace(/[A-Z]/g, (letter) => String.fromCharCode(letter.charCodeAt(0) + 32));
216
+ }
217
+ /** The portable byte-level title/body rule mirrored in index-fts-repository. */
218
+ export function canonicalContentTieKey(entry) {
219
+ const content = entry.content ?? "";
220
+ const newline = content.startsWith("# ") ? content.indexOf("\n") : -1;
221
+ // SQLite uses ltrim(value, char(13) || char(10) || ' ') after an exact '# '
222
+ // title and trim(value, ' ') otherwise. Keep exactly that deliberately
223
+ // narrow byte contract; do not use locale or Unicode-whitespace helpers.
224
+ const body = newline >= 0 ? content.slice(newline + 1).replace(/^[\r\n ]+/, "") : content;
225
+ const source = (body || entry.description || "").replace(/^ +| +$/g, "");
226
+ return Buffer.from(asciiCaseFold(source), "utf8").toString("hex");
227
+ }
193
228
  function buildSearchResultComparator(query) {
194
229
  const queryTokens = buildLexicalQueryPlan(query).tokens.map((token) => token.toLowerCase());
195
- const displayScore = (score) => Math.round(Math.min(1, Math.max(0, score)) * 10000) / 10000;
230
+ const displayScore = (score) => Math.round(displaySearchScore(score) * 10000) / 10000;
196
231
  const stableRankScore = (score) => Math.round(score * 10000) / 10000;
197
232
  return (a, b) => {
198
233
  const aNameTier = lexicalNameMatchTier(a.entry, queryTokens);
@@ -208,11 +243,28 @@ function buildSearchResultComparator(query) {
208
243
  const rawScoreDiff = stableRankScore(b.score) - stableRankScore(a.score);
209
244
  if (rawScoreDiff !== 0)
210
245
  return rawScoreDiff;
246
+ // Ceiling values are intentionally allowed to demote visibility, but not
247
+ // to erase relevance. Prefer the score before a relaxed body-only ceiling;
248
+ // a later belief-state ceiling has its own minScore handoff and must not
249
+ // overwrite this ordering evidence. Belief-only ceilings fall back to
250
+ // their `preCeilingScore`.
251
+ const preCeilingRelevance = (item) => item.preRelaxedCeilingScore ?? item.preCeilingScore ?? item.score;
252
+ const ceilingDiff = stableRankScore(preCeilingRelevance(b)) - stableRankScore(preCeilingRelevance(a));
253
+ if (ceilingDiff !== 0)
254
+ return ceilingDiff;
211
255
  const nameDiff = bNameTier - aNameTier;
212
256
  if (nameDiff !== 0)
213
257
  return nameDiff;
214
258
  const typeDiff = typeBoostFor(b.entry.type) - typeBoostFor(a.entry.type);
215
- return typeDiff || a.filePath.localeCompare(b.filePath);
259
+ if (typeDiff !== 0)
260
+ return typeDiff;
261
+ // Keep opaque generated IDs out of the final relevance tie-break. This
262
+ // runs only after every ranking contributor (including the #940 preserved
263
+ // pre-ceiling evidence), exact-name, and type comparison has tied.
264
+ const contentDiff = compareCodePoints(canonicalContentTieKey(a.entry), canonicalContentTieKey(b.entry));
265
+ if (contentDiff !== 0)
266
+ return contentDiff;
267
+ return a.filePath.localeCompare(b.filePath);
216
268
  };
217
269
  }
218
270
  async function searchDatabase(db, query, searchType, limit, stashDir, allSourceDirs, config, sources, rendererRegistry = defaultRendererRegistry, filters, includeProposed = false, beliefFilter = "all", restrictToSources = false, includeExcludedTypes = false, disableProjectContext = false, disableScopedUtility = false) {
@@ -279,8 +331,9 @@ async function searchDatabase(db, query, searchType, limit, stashDir, allSourceD
279
331
  const { ftsResults, embeddingScores, embedMs, mode, semanticWarning } = await collectSearchSignals(db, query, limit * 3, typeFilter, defaultExcludes, config);
280
332
  const tRank0 = Date.now();
281
333
  // ── Score normalization ──────────────────────────────────────────────
282
- // Normalized BM25 + cosine similarity with weighted addition
283
- // (FTS 0.7, vector 0.3) for well-differentiated combined scores.
334
+ // Stable bounded BM25 transform + cosine similarity with weighted addition
335
+ // (FTS 0.7, vector 0.3). The lexical transform is per-row, so widening the
336
+ // candidate set cannot alter a pre-existing row's base score.
284
337
  const ftsScoreMap = normalizeFtsScores(ftsResults);
285
338
  // Build embedding score map (cosine similarities already 0-1)
286
339
  const embedScoreMap = new Map();
@@ -400,11 +453,11 @@ async function searchDatabase(db, query, searchType, limit, stashDir, allSourceD
400
453
  const selected = beliefFiltered.slice(0, limit);
401
454
  const hits = await Promise.all(selected.map((ranked) => {
402
455
  const { entry, filePath, score, rankingMode, utilityBoosted } = ranked;
403
- // CLAUDE.md locks SearchHit.score in [0,1]. The boost loop above can
404
- // exceed 1.0 (this was a pre-existing breach that #207's graph boost
405
- // up to ~1.05 additive contribution made detectable); clamp here
406
- // so the score handed to buildDbHit always satisfies the spec.
407
- const finalScore = Math.min(1, Math.max(0, score));
456
+ // CLAUDE.md locks SearchHit.score in [0,1]. The boost loop deliberately
457
+ // remains raw for ranking, then takes a monotone bounded projection at
458
+ // the public boundary so contributors do not collapse into hard-clamped
459
+ // ties.
460
+ const finalScore = displaySearchScore(score);
408
461
  return buildDbHit({
409
462
  entry,
410
463
  path: filePath,
@@ -413,6 +466,7 @@ async function searchDatabase(db, query, searchType, limit, stashDir, allSourceD
413
466
  query,
414
467
  rankingMode,
415
468
  lexicalMatch: ranked.lexicalMatch,
469
+ fragmentId: ranked.fragmentId,
416
470
  defaultStashDir: stashDir,
417
471
  allSourceDirs,
418
472
  sources,
@@ -728,7 +782,11 @@ export async function buildDbHit(input) {
728
782
  (source && path.resolve(source.path) === path.resolve(input.defaultStashDir)
729
783
  ? (input.bundleId ?? undefined)
730
784
  : undefined);
731
- const ref = resolveSearchHitRef(input.entry, input, defaultBundleId);
785
+ const parentRef = resolveSearchHitRef(input.entry, input, defaultBundleId);
786
+ // Fragments prove lexical relevance, but executable assets must retain the
787
+ // parent ref consumed by their advertised action (for example workflow run).
788
+ // The central type-presentation contract opts those types out explicitly.
789
+ const ref = input.fragmentId && allowsFragmentRef(input.entry.type) ? `${parentRef}#${input.fragmentId}` : parentRef;
732
790
  const editable = isEditable(absolutePath, input.config, input.sources);
733
791
  const estimatedTokens = typeof input.entry.fileSize === "number" ? Math.round(input.entry.fileSize / 4) : undefined;
734
792
  const hit = {