akm-cli 0.9.12 → 0.9.14-beta.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (48) hide show
  1. package/CHANGELOG.md +100 -0
  2. package/dist/assets/workflows/workflow-template.md +4 -0
  3. package/dist/commands/improve/eligibility.js +27 -15
  4. package/dist/commands/improve/improve.js +1 -0
  5. package/dist/commands/lint/base-linter.js +10 -0
  6. package/dist/commands/proposal/drain.js +48 -6
  7. package/dist/commands/proposal/proposal-cli.js +1 -0
  8. package/dist/commands/read/curate.js +3 -2
  9. package/dist/commands/read/show.js +26 -9
  10. package/dist/core/adapter/adapters/akm-adapter.js +5 -1
  11. package/dist/core/asset/markdown-fragments.js +146 -0
  12. package/dist/core/config/config-walker.js +7 -3
  13. package/dist/core/config/config.js +21 -12
  14. package/dist/core/config/schema/primitives.js +8 -2
  15. package/dist/core/errors.js +2 -0
  16. package/dist/core/lexical-score.js +25 -0
  17. package/dist/core/type-presentation.js +36 -4
  18. package/dist/indexer/index-written-assets.js +4 -0
  19. package/dist/indexer/indexer.js +5 -2
  20. package/dist/indexer/passes/metadata.js +64 -1
  21. package/dist/indexer/scan/doc-to-entry.js +3 -0
  22. package/dist/indexer/scan/drain-dir.js +33 -22
  23. package/dist/indexer/search/db-search.js +72 -14
  24. package/dist/indexer/search/name-match.js +35 -0
  25. package/dist/indexer/search/ranking-contributors.js +15 -12
  26. package/dist/indexer/search/ranking.js +42 -18
  27. package/dist/indexer/usage/show-usage.js +14 -2
  28. package/dist/llm/client.js +12 -8
  29. package/dist/llm/embedders/remote.js +3 -2
  30. package/dist/llm/graph-extract.js +18 -67
  31. package/dist/output/shapes.js +46 -1
  32. package/dist/output/text/proposal-format.js +5 -0
  33. package/dist/scripts/akm-migrate-node.js +648 -253
  34. package/dist/scripts/akm-migrate.js +648 -253
  35. package/dist/storage/repositories/index-connection.js +23 -8
  36. package/dist/storage/repositories/index-entries-repository.js +3 -2
  37. package/dist/storage/repositories/index-entry-schema.js +43 -3
  38. package/dist/storage/repositories/index-fts-repository.js +160 -14
  39. package/dist/storage/repositories/index-schema.js +8 -18
  40. package/dist/storage/repositories/workflow-runs-repository.js +118 -10
  41. package/dist/workflows/exec/run-workflow.js +1 -1
  42. package/dist/workflows/exec/step-work.js +41 -0
  43. package/dist/workflows/parser.js +1 -1
  44. package/dist/workflows/runtime/runs.js +29 -5
  45. package/docs/migration/release-notes/0.9.14.md +26 -0
  46. package/docs/migration/release-notes/README.md +2 -0
  47. package/docs/reference/cli.md +18 -0
  48. package/package.json +1 -1
package/CHANGELOG.md CHANGED
@@ -4,6 +4,106 @@ All notable changes to this project will be documented in this file.
4
4
 
5
5
  The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
6
6
 
7
+ ## [0.9.14-beta.1] - 2026-09-04
8
+
9
+ ### Added
10
+
11
+ - **Long Markdown bodies can return the matching lexical fragment (#937).**
12
+ The derived index now keeps a separate, safe fragment population for lexical
13
+ retrieval. Search can return an addressable `#akm-fragment-…` ref, and `akm
14
+ show` resolves that ref to the exact indexed projection. Headingless and
15
+ oversized documents split at paragraph or word boundaries, so a fact in the
16
+ middle of a long body is no longer represented only by its parent document.
17
+
18
+ ### Changed
19
+
20
+ - **Index generations are checked before use (#934).** A newer index is never
21
+ queried or rebuilt by an older binary; it reports that akm must be upgraded.
22
+ An older derived index is rebuilt from the materialized sources by the current
23
+ binary. A current-generation stamp is accepted only when the canonical entry,
24
+ parent FTS, and fragment surfaces all match, and the stamp is written only
25
+ after schema creation succeeds. This release advances the derived index from
26
+ v22 to v23 for fragment retrieval.
27
+ - **Lexical relevance remains stable through scoring and relaxed-query ties
28
+ (#933, #940).** Lexical scores use a fixed monotone calibration rather than a
29
+ result-set-relative scale, preserving score headroom. Relaxed matches retain
30
+ body relevance as tie evidence, including when a belief-state ceiling also
31
+ applies. This compound-safe implementation supersedes PR #941.
32
+ - **The frozen W0 lexical weight matrix remains the shipped policy (#930).**
33
+ The W1 and W2 alternatives were measured and rejected; no unvalidated weight
34
+ change is included in this release.
35
+
36
+ ### Fixed
37
+
38
+ - **Fuzzy name matches require structural identity.** Short or opaque name
39
+ fragments no longer create a false identity match merely because their text
40
+ overlaps a stored name.
41
+ - **Full indexing honors canonical workflow-source ownership.** When peer `.md`
42
+ and `.yml` sources map to one workflow ref, the index now persists the same
43
+ deterministic `.md` winner used by lookup and execution instead of allowing
44
+ filesystem enumeration order to select the stored row.
45
+
46
+ ## [0.9.13] - 2026-09-04
47
+
48
+ ### Added
49
+
50
+ - **A stable `results` alias on every list-returning command (#922).** `search`,
51
+ `curate`, `proposal list`, `bundle list`, `env list`, `secret list`,
52
+ `registry search`, `registry list`, `workflow list`, `task history` and
53
+ `log list` each keep their existing semantic key (`hits`, `items`,
54
+ `proposals`, …) and now also expose the same array as `results`, in every
55
+ shape including `--shape agent`. It is the same array, not a copy. A caller
56
+ reading `hits` from a `curate` response previously got nothing back and could
57
+ reasonably read that as "no results" — there was no error and the `summary`
58
+ alongside it still reported that results were selected. Commands carrying
59
+ several heterogeneous collections (`health`, `task doctor`) are deliberately
60
+ excluded.
61
+ - **Engine and embedding credentials can resolve from the secret store
62
+ (#917).** `apiKey` accepts `secret://<name>` alongside `$VAR` / `${VAR}`,
63
+ resolved through the existing store-backed resolver. A detached SessionEnd
64
+ hook or a cron job — the two contexts least able to supply an environment
65
+ variable and most likely to need extraction — no longer requires editing the
66
+ login environment for a value akm already stores. Literal keys in
67
+ `config.json` are still refused, and an unresolvable reference fails loudly,
68
+ naming the reference and never the secret.
69
+ - **`akm proposal drain` reports what failed (#921).** The envelope carries a
70
+ `failed[]` naming each proposal and why it was refused (stale target,
71
+ validation), instead of reporting `failed: 0` while the same run printed five
72
+ failures to stderr. `--dry-run` now applies the same stale-target check the
73
+ real run does, so its prediction stops disagreeing with the outcome. The
74
+ refusals themselves are correct and unchanged: declining to overwrite a target
75
+ modified since the proposal was created is the desired behaviour. `akm
76
+ improve`'s triage pre-pass reports the same count.
77
+ - **Workflow step output is checked against its declared schema (#923).** A step
78
+ whose returned object does not match its `outputSchema` now says so, naming
79
+ the step and the specific problem (a missing required field, say). This is a
80
+ warning: the run continues and its status is unchanged, because a workflow is
81
+ a guide rather than a contract. Previously an author got neither enforcement
82
+ nor feedback — the only signal was a notice saying akm was not checking, which
83
+ is a different fact from whether the output matched.
84
+
85
+ ### Changed
86
+
87
+ - **A held run lease no longer reports as database corruption (#924).** `akm
88
+ workflow run` surfaced raw SQLite text — `database is locked`, `database disk
89
+ image is malformed`, `disk I/O error` — for what was simply another
90
+ invocation holding the lease. `disk image is malformed` in particular reads as
91
+ data loss and sent one reporter through integrity checks and a WAL review
92
+ before finding the real cause. The lease message now appears at default
93
+ verbosity and carries a dedicated `RUN_LEASE_HELD` code, so a wrapper can tell
94
+ "retry shortly" from "you passed bad input". Genuine SQLite errors still
95
+ report as themselves.
96
+ - **`akm lint` stops flagging templated output paths (#927).** A documented
97
+ run-time filename such as `reports/review-<timestamp>.md` is no longer
98
+ reported as a `stale-path` broken reference. Angle-bracket and brace
99
+ placeholders, `${VAR}`, date-format runs like `YYYYMMDD`, and glob characters
100
+ are all recognised as parameterised rather than missing. Genuinely broken
101
+ literal paths are still reported.
102
+ - **The workflow level-2 heading rule is discoverable before you trip it
103
+ (#926).** The `workflow create` template states that `##` headings are step
104
+ ids and points at `###` for cross-cutting notes, and the compiler's rejection
105
+ now carries the remedy rather than only the diagnosis.
106
+
7
107
  ## [0.9.12] - 2026-09-03
8
108
 
9
109
  ### Added
@@ -16,6 +16,10 @@ steps:
16
16
  Free preamble prose describing what this workflow does. It is indexed for
17
17
  search and shown in `akm show`, but it is never dispatched to a step.
18
18
 
19
+ Level-2 (`##`) headings below are step ids and must exactly match one
20
+ declared in `steps:` above — for cross-cutting notes that aren't a step
21
+ (shared context, prerequisites), use a level-3 (`###`) heading instead.
22
+
19
23
  ## first-step
20
24
 
21
25
  Describe what to do in this step. Refer to run parameters in plain
@@ -6,7 +6,7 @@ import path from "node:path";
6
6
  import { parseFrontmatter } from "../../core/asset/frontmatter.js";
7
7
  import { conceptIdFromTypeName, parseRefInput, resolveRef } from "../../core/asset/resolve-ref.js";
8
8
  import { loadConfig } from "../../core/config/config.js";
9
- import { NotFoundError, rethrowIfTestIsolationError, UsageError } from "../../core/errors.js";
9
+ import { ConfigError, NotFoundError, rethrowIfTestIsolationError, UsageError } from "../../core/errors.js";
10
10
  import { readEvents } from "../../core/events.js";
11
11
  import { isPathAbsent } from "../../core/path-access.js";
12
12
  import { getDbPath } from "../../core/paths.js";
@@ -47,19 +47,11 @@ function describeIndexSnapshot(readOnly, status) {
47
47
  reason: readOnly ? "loaded a non-mutating point-in-time copy of the existing index" : "loaded the prepared index",
48
48
  };
49
49
  }
50
- if (status === "missing") {
51
- return {
52
- status,
53
- reason: readOnly
54
- ? "index.db is missing; dry-run uses an empty snapshot and does not create it"
55
- : "index.db is missing after index preparation; the selector uses an empty snapshot",
56
- };
57
- }
58
50
  return {
59
51
  status,
60
52
  reason: readOnly
61
- ? "index.db has no entries table; dry-run uses an empty snapshot and does not migrate it"
62
- : "index.db has no entries table; the selector uses an empty snapshot",
53
+ ? "index.db is missing; dry-run uses an empty snapshot and does not create it"
54
+ : "index.db is missing after index preparation; the selector uses an empty snapshot",
63
55
  };
64
56
  }
65
57
  function describeUnavailableSnapshot(error) {
@@ -68,6 +60,26 @@ function describeUnavailableSnapshot(error) {
68
60
  reason: `index.db cannot provide a stable non-mutating snapshot (${error.message}); dry-run uses an empty snapshot`,
69
61
  };
70
62
  }
63
+ /**
64
+ * A dry-run is explicitly a non-mutating planning operation, so it may report
65
+ * an unusable derived index as an empty snapshot. This is deliberately a
66
+ * typed boundary mapping: readers otherwise receive no incompatible handle,
67
+ * and we must not turn an arbitrary SQLite error into a successful plan by
68
+ * matching its text.
69
+ */
70
+ function isIncompatibleIndexError(error) {
71
+ return error instanceof ConfigError && error.code === "INDEX_SCHEMA_INCOMPATIBLE";
72
+ }
73
+ function describeIncompatibleIndexSnapshot(error) {
74
+ // `INDEX_SCHEMA_INCOMPATIBLE` establishes that this is the derived-index
75
+ // boundary, and the error's hint preserves whether this binary should
76
+ // rebuild an older/unknown generation or upgrade for a newer one.
77
+ const action = error.hint() ?? error.message;
78
+ return {
79
+ status: "incompatible",
80
+ reason: `index.db is incompatible; ${action} Dry-run uses an empty snapshot and does not migrate it.`,
81
+ };
82
+ }
71
83
  export function resolveImproveScope(scope) {
72
84
  const trimmed = scope?.trim();
73
85
  if (!trimmed)
@@ -194,12 +206,12 @@ async function collectEligibleRefsFromIndex(scope, stashDir, improveProfile, rea
194
206
  indexSnapshot: describeUnavailableSnapshot(error),
195
207
  };
196
208
  }
197
- if (error instanceof Error && /no such table:\s*entries/i.test(error.message)) {
209
+ if (readOnly && isIncompatibleIndexError(error)) {
198
210
  return {
199
211
  plannedRefs: [],
200
212
  memorySummary: { eligible: 0, derived: 0 },
201
213
  strategyFilteredRefs: [],
202
- indexSnapshot: describeIndexSnapshot(readOnly, "incompatible"),
214
+ indexSnapshot: describeIncompatibleIndexSnapshot(error),
203
215
  };
204
216
  }
205
217
  throw error;
@@ -300,12 +312,12 @@ async function collectEligibleRefsFromIndex(scope, stashDir, improveProfile, rea
300
312
  indexSnapshot: describeUnavailableSnapshot(error),
301
313
  };
302
314
  }
303
- if (error instanceof Error && /no such table:\s*entries/i.test(error.message)) {
315
+ if (readOnly && isIncompatibleIndexError(error)) {
304
316
  return {
305
317
  plannedRefs: [],
306
318
  memorySummary: { eligible: 0, derived: 0 },
307
319
  strategyFilteredRefs: [],
308
- indexSnapshot: describeIndexSnapshot(readOnly, "incompatible"),
320
+ indexSnapshot: describeIncompatibleIndexSnapshot(error),
309
321
  };
310
322
  }
311
323
  throw error;
@@ -1380,6 +1380,7 @@ function finalizeImproveResult(args) {
1380
1380
  promoted: triageDrain.promoted.length,
1381
1381
  rejected: triageDrain.rejected.length,
1382
1382
  deferred: triageDrain.deferred.length,
1383
+ failed: triageDrain.failed.length,
1383
1384
  skippedByCap: triageDrain.skippedByCap.length,
1384
1385
  },
1385
1386
  }
@@ -132,6 +132,14 @@ function fixMissingUpdated(raw, mtime) {
132
132
  return spliceFrontmatterLine(raw, `updated: ${localDateStamp(mtime)}`) ?? raw;
133
133
  }
134
134
  // ── stale-path helpers ────────────────────────────────────────────────────────
135
+ /**
136
+ * A path segment shaped like a run-time filename template rather than a
137
+ * literal reference: `<timestamp>`-style angle brackets, `{stamp}`/`${VAR}`
138
+ * braces, a `YYYYMMDD`/`HHMMSS`-style run of date-format letters, or a glob
139
+ * character. Such a path never exists under its literal spelling, so
140
+ * `stale-path` skips it instead of flagging it.
141
+ */
142
+ const PATH_PLACEHOLDER_PATTERN = /[<{]|[*?]|[YMDHS]{4,}/;
135
143
  function checkStalePath(body) {
136
144
  const pathRe = /(?:\/home\/|\/tmp\/|\/var\/|\/root\/|\/opt\/)[^\s"'`)\]>,\n]+/g;
137
145
  let match;
@@ -139,6 +147,8 @@ function checkStalePath(body) {
139
147
  // biome-ignore lint/suspicious/noAssignInExpressions: idiomatic regex loop
140
148
  while ((match = pathRe.exec(body)) !== null) {
141
149
  const candidate = match[0];
150
+ if (PATH_PLACEHOLDER_PATTERN.test(candidate))
151
+ continue;
142
152
  if (!fs.existsSync(candidate)) {
143
153
  stale.push(candidate);
144
154
  }
@@ -123,6 +123,42 @@ export function classifyProposal(proposal, policy, maxDiffLines) {
123
123
  function deferReasonForSource(source) {
124
124
  return source === "distill" ? "possible-dup" : "mid-band";
125
125
  }
126
+ /**
127
+ * Map a thrown error's message to one of `DrainResult.failed`'s stable reason
128
+ * codes, falling back to `fallback` for anything not specifically recognized.
129
+ * Recognizes the write-time guards a proposal can trip during promotion
130
+ * (see repository.ts's `promoteProposalWithLease` / `preflightProposalPromotion`).
131
+ */
132
+ function categorizeDrainFailure(message, fallback) {
133
+ if (/target (?:changed after|was created after) proposal/.test(message))
134
+ return "stale-target";
135
+ if (/failed validation:/.test(message))
136
+ return "validation";
137
+ return fallback;
138
+ }
139
+ function pushDrainFailure(result, id, err, fallbackReason) {
140
+ const message = err instanceof Error ? err.message : String(err);
141
+ result.failed.push({ id, reason: categorizeDrainFailure(message, fallbackReason), detail: message });
142
+ return message;
143
+ }
144
+ /**
145
+ * Mirror repository.ts's `promoteProposalWithLease` stale-target guard so a
146
+ * dry-run preflight predicts the same refusal a real promote would hit,
147
+ * without writing anything. `assetPath` is the path `preflightProposalPromotion`
148
+ * already resolved for this proposal.
149
+ */
150
+ function assertProposalTargetFresh(proposal, assetPath) {
151
+ const backup = fs.existsSync(assetPath) ? fs.readFileSync(assetPath) : undefined;
152
+ const currentHash = backup ? createHash("sha256").update(backup).digest("hex") : undefined;
153
+ if (proposal.beforeHash !== undefined && (!backup || currentHash !== proposal.beforeHash)) {
154
+ throw new Error(`Proposal target changed after proposal ${proposal.id} was created; refusing to overwrite newer content.`);
155
+ }
156
+ if (proposal.beforeHash === undefined &&
157
+ backup !== undefined &&
158
+ proposal.changes.some((change) => change.op === "create")) {
159
+ throw new Error(`Proposal target was created after proposal ${proposal.id} was created; refusing to overwrite newer content.`);
160
+ }
161
+ }
126
162
  // ---------------------------------------------------------------------------
127
163
  // Judgment tier (Phase 3)
128
164
  // ---------------------------------------------------------------------------
@@ -493,6 +529,7 @@ export async function drainProposals(opts, promoteFn = akmProposalAccept, reject
493
529
  deferred: classification.deferred,
494
530
  skippedByCap: [],
495
531
  staged: [],
532
+ failed: [],
496
533
  };
497
534
  // A configured judgment runner makes every deferred item dispatch-eligible.
498
535
  // Validate its symbolic credentials before applying any deterministic gate,
@@ -518,7 +555,8 @@ export async function drainProposals(opts, promoteFn = akmProposalAccept, reject
518
555
  result.rejected.push(target.id);
519
556
  }
520
557
  catch (err) {
521
- warn(`[triage] reject failed for ${target.id}: ${err instanceof Error ? err.message : String(err)}`);
558
+ const message = pushDrainFailure(result, target.id, err, "reject-error");
559
+ warn(`[triage] reject failed for ${target.id}: ${message}`);
522
560
  }
523
561
  }
524
562
  // --- Accept ceiling: enforced BEFORE the promote loop ---
@@ -550,13 +588,15 @@ export async function drainProposals(opts, promoteFn = akmProposalAccept, reject
550
588
  deterministicPromoted += 1;
551
589
  }
552
590
  catch (err) {
553
- warn(`[triage] promote failed for ${id}: ${err instanceof Error ? err.message : String(err)}`);
591
+ const message = pushDrainFailure(result, id, err, "promote-error");
592
+ warn(`[triage] promote failed for ${id}: ${message}`);
554
593
  }
555
594
  }
556
595
  }
557
596
  else if (opts.applyMode === "promote" && opts.dryRun) {
558
- // Exercise the same stamped candidate and lint boundary as real promotion.
559
- // Tests that omit config retain the classification-only seam.
597
+ // Exercise the same stamped candidate, lint, and stale-target boundary as
598
+ // real promotion so a dry-run's predicted promotions match what a real
599
+ // run would do. Tests that omit config retain the classification-only seam.
560
600
  const byId = new Map(pending.map((proposal) => [proposal.id, proposal]));
561
601
  for (const id of withinCap) {
562
602
  try {
@@ -564,7 +604,7 @@ export async function drainProposals(opts, promoteFn = akmProposalAccept, reject
564
604
  const proposal = byId.get(id);
565
605
  if (!proposal)
566
606
  throw new Error(`Proposal ${id} disappeared during drain preflight.`);
567
- preflightProposalPromotion(opts.config, proposal, {
607
+ const preflight = preflightProposalPromotion(opts.config, proposal, {
568
608
  ...(opts.target ? { target: opts.target } : {}),
569
609
  gateDecision: {
570
610
  outcome: "auto-accepted",
@@ -572,12 +612,14 @@ export async function drainProposals(opts, promoteFn = akmProposalAccept, reject
572
612
  gate: gateLabel,
573
613
  },
574
614
  });
615
+ assertProposalTargetFresh(proposal, preflight.assetPath);
575
616
  }
576
617
  result.promoted.push(id);
577
618
  deterministicPromoted += 1;
578
619
  }
579
620
  catch (err) {
580
- warn(`[triage] preflight failed for ${id}: ${err instanceof Error ? err.message : String(err)}`);
621
+ const message = pushDrainFailure(result, id, err, "promote-error");
622
+ warn(`[triage] preflight failed for ${id}: ${message}`);
581
623
  }
582
624
  }
583
625
  }
@@ -473,6 +473,7 @@ const proposalDrainCommand = defineJsonCommand({
473
473
  deferred: result.deferred,
474
474
  skippedByCap: result.skippedByCap,
475
475
  staged: result.staged,
476
+ failed: result.failed,
476
477
  });
477
478
  },
478
479
  });
@@ -529,8 +529,9 @@ function getCurateFamily(ref) {
529
529
  try {
530
530
  // F4b: `ref` is a search-hit ref in the 0.9.0 conceptId grammar — parse via
531
531
  // the new-grammar `parseRefInput` so skill/reference family grouping still
532
- // recognizes it.
533
- const parsed = parseRefInput(ref);
532
+ // recognizes it. Search may add an opaque Markdown selector; identity and
533
+ // family ownership are on the parent asset, not that selector.
534
+ const parsed = parseRefInput(ref.split("#", 1)[0]);
534
535
  if (parsed.type === "skill") {
535
536
  return { key: parsed.name, role: "root" };
536
537
  }
@@ -23,6 +23,7 @@ import { assetPathForName, stashDirFor } from "../../core/asset/asset-placement.
23
23
  import { makeBundleRef, parseBundleRef } from "../../core/asset/asset-ref.js";
24
24
  import { parseFrontmatter } from "../../core/asset/frontmatter.js";
25
25
  import { extractSection, markdownFragmentSlugs } from "../../core/asset/markdown.js";
26
+ import { fragmentForSelector } from "../../core/asset/markdown-fragments.js";
26
27
  import { displayRef, typeNameFromConceptId } from "../../core/asset/resolve-ref.js";
27
28
  import { META_DIR, parseMetaRef, readMetaFile } from "../../core/asset/stash-meta.js";
28
29
  import { asNonEmptyString, isWithin } from "../../core/common.js";
@@ -36,6 +37,7 @@ import { hasGraphData } from "../../indexer/db/graph-db.js";
36
37
  import { listRelatedPathsForFile } from "../../indexer/graph/graph-boost.js";
37
38
  import { extractGraphForSingleFile } from "../../indexer/graph/graph-extraction.js";
38
39
  import { lookupBundleRef, lookupBundleRefWithResolution } from "../../indexer/indexer.js";
40
+ import { projectMarkdownFragmentContent } from "../../indexer/passes/metadata.js";
39
41
  import { ensurePrimaryIndexForRead, resolveReadSources } from "../../indexer/read-preflight.js";
40
42
  import { buildEditHint, findSourceForPath, isEditable, resolveSourceEntries, } from "../../indexer/search/search-source.js";
41
43
  import { recentShowCount, recordShowUsage } from "../../indexer/usage/show-usage.js";
@@ -45,6 +47,7 @@ import { resolveSourcesForOrigin } from "../../registry/origin-resolve.js";
45
47
  import { resolveStorageLocations } from "../../storage/locations.js";
46
48
  import { closeDatabase, openExistingDatabase } from "../../storage/repositories/index-connection.js";
47
49
  import { TELEMETRY_BUSY_TIMEOUT_MS, withIndexDb } from "../../storage/repositories/index-db.js";
50
+ import { getIndexedMarkdownFragment } from "../../storage/repositories/index-fts-repository.js";
48
51
  import { computeBodyHash } from "../../storage/repositories/index-llm-cache-repository.js";
49
52
  // Eagerly import source providers to trigger self-registration.
50
53
  import "../../sources/providers/index.js";
@@ -76,9 +79,15 @@ export async function akmShowUnified(input) {
76
79
  // (env — key names only; secret — never rendered), fragment or not. Warn
77
80
  // and ignore the fragment rather than refusing the whole show.
78
81
  warnSensitiveFragmentUnsupported(parseBundleRef(ref));
79
- // Auto-index when stale so the index is current before lookup.
80
- const { primarySource } = resolveReadSources();
81
- await ensurePrimaryIndexForRead(primarySource);
82
+ // An opaque fragment selector is an indexed revision handle. Do not refresh
83
+ // it away between search and show if disk changed concurrently; the stored
84
+ // safe substrate below is its source of truth. Friendly heading selectors
85
+ // intentionally retain the normal source-live read behavior.
86
+ const parsedRef = parseBundleRef(ref);
87
+ if (!parsedRef.fragment?.startsWith("akm-fragment-")) {
88
+ const { primarySource } = resolveReadSources();
89
+ await ensurePrimaryIndexForRead(primarySource);
90
+ }
82
91
  // Try local filesystem (FTS5 index lookup)
83
92
  const result = await showLocal(input);
84
93
  // Scope filter narrows resolution: if a scope filter was supplied, the
@@ -226,11 +235,14 @@ export async function showLocal(input) {
226
235
  }
227
236
  const fileCtx = buildFileContext(sourceStashDir, assetPath);
228
237
  const presentedName = indexedEntry.name;
238
+ const indexedFragment = parsed.fragment?.startsWith("akm-fragment-")
239
+ ? withIndexDb((db) => getIndexedMarkdownFragment(db, indexedEntry.itemRef, parsed.fragment))
240
+ : undefined;
229
241
  const indexedRenderer = rendererForIndexedEntry(indexedEntry, fileCtx);
230
242
  let response;
231
243
  try {
232
244
  if (indexedRenderer === null) {
233
- response = buildIndexedProjectionResponse(indexedEntry, assetPath, parsed.fragment);
245
+ response = buildIndexedProjectionResponse(indexedEntry, assetPath, parsed.fragment, indexedFragment?.content);
234
246
  }
235
247
  else {
236
248
  const match = typeof indexedRenderer === "string" ? indexedMatch(indexedEntry, indexedRenderer) : recognizeMatch(fileCtx);
@@ -251,7 +263,7 @@ export async function showLocal(input) {
251
263
  warn(`Fragment "#${parsed.fragment}" was ignored: ${makeBundleRef(parsed.bundle, parsed.conceptId)} is not a Markdown document, so heading fragments do not apply. Showing the whole asset.`);
252
264
  }
253
265
  else {
254
- applyMarkdownFragment(response, fileCtx.content(), parsed.fragment, presentedName);
266
+ applyMarkdownFragment(response, fileCtx.content(), parsed.fragment, presentedName, indexedFragment?.content);
255
267
  }
256
268
  }
257
269
  }
@@ -495,7 +507,7 @@ function rendererForIndexedEntry(entry, _file) {
495
507
  function indexedMatch(entry, renderer) {
496
508
  return { type: entry.type, specificity: Number.MAX_SAFE_INTEGER, renderer, meta: { name: entry.name } };
497
509
  }
498
- function buildIndexedProjectionResponse(entry, assetPath, fragment) {
510
+ function buildIndexedProjectionResponse(entry, assetPath, fragment, indexedFragmentContent) {
499
511
  const isMarkdown = path.extname(assetPath).toLowerCase() === ".md";
500
512
  if (fragment !== undefined && !isMarkdown) {
501
513
  warn(`Fragment "#${fragment}" was ignored: ${entry.conceptId} is not a Markdown document, so heading fragments do not apply. Showing the whole asset.`);
@@ -503,7 +515,7 @@ function buildIndexedProjectionResponse(entry, assetPath, fragment) {
503
515
  const raw = fs.readFileSync(assetPath, "utf8");
504
516
  const parsed = parseFrontmatter(raw);
505
517
  const content = fragment !== undefined && isMarkdown
506
- ? requireMarkdownSection(parsed.content, fragment, entry.name).content
518
+ ? (indexedFragmentContent ?? requireMarkdownSection(raw, fragment, entry.name).content)
507
519
  : parsed.content;
508
520
  const description = entry.document?.description ?? asNonEmptyString(parsed.data.description);
509
521
  const tags = entry.document?.tags ??
@@ -520,8 +532,8 @@ function buildIndexedProjectionResponse(entry, assetPath, fragment) {
520
532
  ...(tags && tags.length > 0 ? { tags } : {}),
521
533
  };
522
534
  }
523
- function applyMarkdownFragment(response, raw, fragment, name) {
524
- const section = requireMarkdownSection(parseFrontmatter(raw).content, fragment, name).content;
535
+ function applyMarkdownFragment(response, raw, fragment, name, indexedFragmentContent) {
536
+ const section = indexedFragmentContent ?? requireMarkdownSection(raw, fragment, name).content;
525
537
  if (response.template !== undefined)
526
538
  response.template = section;
527
539
  else if (response.prompt !== undefined)
@@ -533,6 +545,11 @@ function requireMarkdownSection(content, fragment, name) {
533
545
  const section = extractSection(content, fragment);
534
546
  if (section)
535
547
  return section;
548
+ const indexed = projectMarkdownFragmentContent(content);
549
+ const safeFragment = indexed ? fragmentForSelector(indexed, fragment) : undefined;
550
+ if (safeFragment) {
551
+ return { content: safeFragment.text, startLine: safeFragment.startLine, endLine: safeFragment.endLine };
552
+ }
536
553
  const available = markdownFragmentSlugs(content);
537
554
  throw new NotFoundError(`Fragment "#${fragment}" not found in ${name}.` +
538
555
  (available.length > 0 ? ` Available fragments: ${available.map((slug) => `#${slug}`).join(", ")}.` : ""));
@@ -81,7 +81,7 @@
81
81
  */
82
82
  import fs from "node:fs";
83
83
  import path from "node:path";
84
- import { applyPostContributorFields, applyPreContributorFields, extractPackageMetadata, } from "../../../indexer/passes/metadata.js";
84
+ import { applyPostContributorFields, applyPreContributorFields, extractPackageMetadata, getMarkdownFragmentContent, hasMarkdownFragmentContent, setMarkdownFragmentContent, } from "../../../indexer/passes/metadata.js";
85
85
  import { assetPathCandidatesForName, assetPathForName, deriveCanonicalAssetNameFromStashRoot, placementTypes, stashDirFor, stashDirNames, } from "../../asset/asset-placement.js";
86
86
  import { parseFrontmatter } from "../../asset/frontmatter.js";
87
87
  import { executionDefaultsFromFrontmatter, renderMarkdownExecutionSource } from "../execution-source.js";
@@ -244,6 +244,10 @@ function indexDocumentFromEntry(entry, base, rendererName) {
244
244
  doc.lessonStrength = entry.lessonStrength;
245
245
  if (entry.derivedFrom !== undefined)
246
246
  doc.derivedFrom = entry.derivedFrom;
247
+ // Internal fragment substrate follows the recognition projection without
248
+ // becoming an IndexDocument field or serialized search payload.
249
+ if (hasMarkdownFragmentContent(entry))
250
+ setMarkdownFragmentContent(doc, getMarkdownFragmentContent(entry));
247
251
  return doc;
248
252
  }
249
253
  function conceptIdForRecognizedType(root, filePath, type) {
@@ -0,0 +1,146 @@
1
+ // This Source Code Form is subject to the terms of the Mozilla Public
2
+ // License, v. 2.0. If a copy of the MPL was not distributed with this
3
+ // file, You can obtain one at https://mozilla.org/MPL/2.0/.
4
+ /**
5
+ * Deterministic, addressable Markdown fragments.
6
+ *
7
+ * Input is the safe, line-preserving Markdown projection, never the raw file.
8
+ * Keeping that distinction here means search can only emit selectors that
9
+ * `show` can reproduce without disclosing fenced/commented/link-target bytes.
10
+ */
11
+ import { createHash } from "node:crypto";
12
+ import { markdownHeadingSlug, parseMarkdownToc } from "./markdown.js";
13
+ export const MARKDOWN_FRAGMENT_MAX_CHARS = 1600;
14
+ export const MARKDOWN_FRAGMENT_PREFIX = "akm-fragment-";
15
+ function hash(text) {
16
+ return createHash("sha256").update(text).digest("hex");
17
+ }
18
+ function uniqueSlugs(body) {
19
+ const out = new Map();
20
+ const seen = new Set();
21
+ for (const heading of parseMarkdownToc(body).headings) {
22
+ const base = markdownHeadingSlug(heading.text);
23
+ if (!base)
24
+ continue;
25
+ let slug = base;
26
+ for (let suffix = 1; seen.has(slug); suffix++)
27
+ slug = `${base}-${suffix}`;
28
+ seen.add(slug);
29
+ out.set(heading.line, slug);
30
+ }
31
+ return out;
32
+ }
33
+ function textOf(lines) {
34
+ return lines.join("\n").trim();
35
+ }
36
+ /** Split a large sequence only at paragraph, then word, boundaries. */
37
+ function splitPiece(piece, maxChars) {
38
+ if (textOf(piece.lines).length <= maxChars)
39
+ return [piece];
40
+ const pieces = [];
41
+ let start = 0;
42
+ while (start < piece.lines.length) {
43
+ let end = start;
44
+ let chars = 0;
45
+ while (end < piece.lines.length) {
46
+ const next = piece.lines[end];
47
+ // Let the single-line word-window path below own an oversized first
48
+ // line. Without this guard it is consumed whole before that path can
49
+ // run, silently defeating the fragment bound for transcripts/logs.
50
+ if (end === start && next.length > maxChars)
51
+ break;
52
+ if (end > start && chars + next.length + 1 > maxChars)
53
+ break;
54
+ chars += next.length + (end > start ? 1 : 0);
55
+ end++;
56
+ }
57
+ // A single long authored line needs word windows, but its source range is
58
+ // intentionally the same line for every window.
59
+ if (end === start) {
60
+ const line = piece.lines[start];
61
+ let offset = 0;
62
+ while (offset < line.length) {
63
+ let cut = Math.min(offset + maxChars, line.length);
64
+ if (cut < line.length) {
65
+ const space = line.lastIndexOf(" ", cut);
66
+ if (space > offset + Math.floor(maxChars * 0.55))
67
+ cut = space;
68
+ }
69
+ pieces.push({ lines: [line.slice(offset, cut).trim()], startLine: piece.startLine + start });
70
+ offset = cut;
71
+ while (line[offset] === " ")
72
+ offset++;
73
+ }
74
+ start++;
75
+ continue;
76
+ }
77
+ // Prefer not to split a paragraph if a blank boundary fits before `end`.
78
+ let preferred = -1;
79
+ for (let i = start + 1; i < end; i++)
80
+ if (!piece.lines[i].trim())
81
+ preferred = i;
82
+ if (preferred > start)
83
+ end = preferred;
84
+ pieces.push({ lines: piece.lines.slice(start, end), startLine: piece.startLine + start });
85
+ start = end;
86
+ while (start < piece.lines.length && !piece.lines[start].trim())
87
+ start++;
88
+ }
89
+ return pieces.filter((candidate) => textOf(candidate.lines));
90
+ }
91
+ /**
92
+ * Heading sections first, then paragraph/word windows. `startLine`/`endLine`
93
+ * always refer to the authored file's line numbers because the projection
94
+ * preserves one line per source line (with excluded bytes blanked out).
95
+ */
96
+ export function splitMarkdownFragmentStats(body, maxChars = MARKDOWN_FRAGMENT_MAX_CHARS) {
97
+ const lines = body.split(/\r?\n/);
98
+ const headings = parseMarkdownToc(body).headings;
99
+ const boundaries = [1, ...headings.map((heading) => heading.line), lines.length + 1]
100
+ .filter((line, index, all) => index === 0 || line !== all[index - 1])
101
+ .sort((left, right) => left - right);
102
+ const slugs = uniqueSlugs(body);
103
+ const pieces = [];
104
+ let sectionCount = 0;
105
+ for (let i = 0; i < boundaries.length - 1; i++) {
106
+ const startLine = boundaries[i];
107
+ const end = boundaries[i + 1] - 1;
108
+ const section = { lines: lines.slice(startLine - 1, end), startLine, headingSlug: slugs.get(startLine) };
109
+ if (textOf(section.lines)) {
110
+ sectionCount++;
111
+ pieces.push(...splitPiece(section, maxChars));
112
+ }
113
+ }
114
+ // Whether a heading section survived as one fragment is a property of the
115
+ // complete piece set. Count once before materialization instead of scanning
116
+ // every piece for every fragment (which made many headed documents O(N²)).
117
+ const piecesPerHeading = new Map();
118
+ for (const piece of pieces) {
119
+ if (piece.headingSlug)
120
+ piecesPerHeading.set(piece.headingSlug, (piecesPerHeading.get(piece.headingSlug) ?? 0) + 1);
121
+ }
122
+ const fragments = pieces.map((piece, ordinal) => {
123
+ const text = textOf(piece.lines);
124
+ const contentLines = piece.lines.map((line, index) => ({ line, index })).filter(({ line }) => line.trim());
125
+ const first = contentLines[0]?.index ?? 0;
126
+ const last = contentLines.at(-1)?.index ?? 0;
127
+ const digest = hash(text);
128
+ const unsplitHeading = piece.headingSlug && piecesPerHeading.get(piece.headingSlug) === 1;
129
+ return {
130
+ fragmentId: `${MARKDOWN_FRAGMENT_PREFIX}${ordinal + 1}-${digest.slice(0, 12)}`,
131
+ ordinal,
132
+ startLine: piece.startLine + first,
133
+ endLine: piece.startLine + last,
134
+ ...(unsplitHeading ? { headingSlug: piece.headingSlug } : {}),
135
+ text,
136
+ hash: digest,
137
+ };
138
+ });
139
+ return { fragments, hardSplitCount: Math.max(0, pieces.length - sectionCount) };
140
+ }
141
+ export function splitMarkdownFragments(body, maxChars = MARKDOWN_FRAGMENT_MAX_CHARS) {
142
+ return splitMarkdownFragmentStats(body, maxChars).fragments;
143
+ }
144
+ export function fragmentForSelector(body, selector) {
145
+ return splitMarkdownFragments(body).find((fragment) => fragment.fragmentId === selector || fragment.headingSlug === selector);
146
+ }