akm-cli 0.9.13 → 0.9.14-beta.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/CHANGELOG.md CHANGED
@@ -4,6 +4,45 @@ All notable changes to this project will be documented in this file.
4
4
 
5
5
  The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
6
6
 
7
+ ## [0.9.14-beta.1] - 2026-09-04
8
+
9
+ ### Added
10
+
11
+ - **Long Markdown bodies can return the matching lexical fragment (#937).**
12
+ The derived index now keeps a separate, safe fragment population for lexical
13
+ retrieval. Search can return an addressable `#akm-fragment-…` ref, and `akm
14
+ show` resolves that ref to the exact indexed projection. Headingless and
15
+ oversized documents split at paragraph or word boundaries, so a fact in the
16
+ middle of a long body is no longer represented only by its parent document.
17
+
18
+ ### Changed
19
+
20
+ - **Index generations are checked before use (#934).** A newer index is never
21
+ queried or rebuilt by an older binary; it reports that akm must be upgraded.
22
+ An older derived index is rebuilt from the materialized sources by the current
23
+ binary. A current-generation stamp is accepted only when the canonical entry,
24
+ parent FTS, and fragment surfaces all match, and the stamp is written only
25
+ after schema creation succeeds. This release advances the derived index from
26
+ v22 to v23 for fragment retrieval.
27
+ - **Lexical relevance remains stable through scoring and relaxed-query ties
28
+ (#933, #940).** Lexical scores use a fixed monotone calibration rather than a
29
+ result-set-relative scale, preserving score headroom. Relaxed matches retain
30
+ body relevance as tie evidence, including when a belief-state ceiling also
31
+ applies. This compound-safe implementation supersedes PR #941.
32
+ - **The frozen W0 lexical weight matrix remains the shipped policy (#930).**
33
+ The W1 and W2 alternatives were measured and rejected; no unvalidated weight
34
+ change is included in this release.
35
+
36
+ ### Fixed
37
+
38
+ - **Fuzzy name matches require structural identity.** Short or opaque name
39
+ fragments no longer create a false identity match merely because their text
40
+ overlaps a stored name.
41
+ - **Full indexing honors canonical workflow-source ownership.** When peer `.md`
42
+ and `.yml` sources map to one workflow ref, the index now persists the same
43
+ deterministic `.md` winner used by lookup and execution instead of allowing
44
+ filesystem enumeration order to select the stored row.
45
+
7
46
  ## [0.9.13] - 2026-09-04
8
47
 
9
48
  ### Added
@@ -6,7 +6,7 @@ import path from "node:path";
6
6
  import { parseFrontmatter } from "../../core/asset/frontmatter.js";
7
7
  import { conceptIdFromTypeName, parseRefInput, resolveRef } from "../../core/asset/resolve-ref.js";
8
8
  import { loadConfig } from "../../core/config/config.js";
9
- import { NotFoundError, rethrowIfTestIsolationError, UsageError } from "../../core/errors.js";
9
+ import { ConfigError, NotFoundError, rethrowIfTestIsolationError, UsageError } from "../../core/errors.js";
10
10
  import { readEvents } from "../../core/events.js";
11
11
  import { isPathAbsent } from "../../core/path-access.js";
12
12
  import { getDbPath } from "../../core/paths.js";
@@ -47,19 +47,11 @@ function describeIndexSnapshot(readOnly, status) {
47
47
  reason: readOnly ? "loaded a non-mutating point-in-time copy of the existing index" : "loaded the prepared index",
48
48
  };
49
49
  }
50
- if (status === "missing") {
51
- return {
52
- status,
53
- reason: readOnly
54
- ? "index.db is missing; dry-run uses an empty snapshot and does not create it"
55
- : "index.db is missing after index preparation; the selector uses an empty snapshot",
56
- };
57
- }
58
50
  return {
59
51
  status,
60
52
  reason: readOnly
61
- ? "index.db has no entries table; dry-run uses an empty snapshot and does not migrate it"
62
- : "index.db has no entries table; the selector uses an empty snapshot",
53
+ ? "index.db is missing; dry-run uses an empty snapshot and does not create it"
54
+ : "index.db is missing after index preparation; the selector uses an empty snapshot",
63
55
  };
64
56
  }
65
57
  function describeUnavailableSnapshot(error) {
@@ -68,6 +60,26 @@ function describeUnavailableSnapshot(error) {
68
60
  reason: `index.db cannot provide a stable non-mutating snapshot (${error.message}); dry-run uses an empty snapshot`,
69
61
  };
70
62
  }
63
+ /**
64
+ * A dry-run is explicitly a non-mutating planning operation, so it may report
65
+ * an unusable derived index as an empty snapshot. This is deliberately a
66
+ * typed boundary mapping: readers otherwise receive no incompatible handle,
67
+ * and we must not turn an arbitrary SQLite error into a successful plan by
68
+ * matching its text.
69
+ */
70
+ function isIncompatibleIndexError(error) {
71
+ return error instanceof ConfigError && error.code === "INDEX_SCHEMA_INCOMPATIBLE";
72
+ }
73
+ function describeIncompatibleIndexSnapshot(error) {
74
+ // `INDEX_SCHEMA_INCOMPATIBLE` establishes that this is the derived-index
75
+ // boundary, and the error's hint preserves whether this binary should
76
+ // rebuild an older/unknown generation or upgrade for a newer one.
77
+ const action = error.hint() ?? error.message;
78
+ return {
79
+ status: "incompatible",
80
+ reason: `index.db is incompatible; ${action} Dry-run uses an empty snapshot and does not migrate it.`,
81
+ };
82
+ }
71
83
  export function resolveImproveScope(scope) {
72
84
  const trimmed = scope?.trim();
73
85
  if (!trimmed)
@@ -194,12 +206,12 @@ async function collectEligibleRefsFromIndex(scope, stashDir, improveProfile, rea
194
206
  indexSnapshot: describeUnavailableSnapshot(error),
195
207
  };
196
208
  }
197
- if (error instanceof Error && /no such table:\s*entries/i.test(error.message)) {
209
+ if (readOnly && isIncompatibleIndexError(error)) {
198
210
  return {
199
211
  plannedRefs: [],
200
212
  memorySummary: { eligible: 0, derived: 0 },
201
213
  strategyFilteredRefs: [],
202
- indexSnapshot: describeIndexSnapshot(readOnly, "incompatible"),
214
+ indexSnapshot: describeIncompatibleIndexSnapshot(error),
203
215
  };
204
216
  }
205
217
  throw error;
@@ -300,12 +312,12 @@ async function collectEligibleRefsFromIndex(scope, stashDir, improveProfile, rea
300
312
  indexSnapshot: describeUnavailableSnapshot(error),
301
313
  };
302
314
  }
303
- if (error instanceof Error && /no such table:\s*entries/i.test(error.message)) {
315
+ if (readOnly && isIncompatibleIndexError(error)) {
304
316
  return {
305
317
  plannedRefs: [],
306
318
  memorySummary: { eligible: 0, derived: 0 },
307
319
  strategyFilteredRefs: [],
308
- indexSnapshot: describeIndexSnapshot(readOnly, "incompatible"),
320
+ indexSnapshot: describeIncompatibleIndexSnapshot(error),
309
321
  };
310
322
  }
311
323
  throw error;
@@ -529,8 +529,9 @@ function getCurateFamily(ref) {
529
529
  try {
530
530
  // F4b: `ref` is a search-hit ref in the 0.9.0 conceptId grammar — parse via
531
531
  // the new-grammar `parseRefInput` so skill/reference family grouping still
532
- // recognizes it.
533
- const parsed = parseRefInput(ref);
532
+ // recognizes it. Search may add an opaque Markdown selector; identity and
533
+ // family ownership are on the parent asset, not that selector.
534
+ const parsed = parseRefInput(ref.split("#", 1)[0]);
534
535
  if (parsed.type === "skill") {
535
536
  return { key: parsed.name, role: "root" };
536
537
  }
@@ -23,6 +23,7 @@ import { assetPathForName, stashDirFor } from "../../core/asset/asset-placement.
23
23
  import { makeBundleRef, parseBundleRef } from "../../core/asset/asset-ref.js";
24
24
  import { parseFrontmatter } from "../../core/asset/frontmatter.js";
25
25
  import { extractSection, markdownFragmentSlugs } from "../../core/asset/markdown.js";
26
+ import { fragmentForSelector } from "../../core/asset/markdown-fragments.js";
26
27
  import { displayRef, typeNameFromConceptId } from "../../core/asset/resolve-ref.js";
27
28
  import { META_DIR, parseMetaRef, readMetaFile } from "../../core/asset/stash-meta.js";
28
29
  import { asNonEmptyString, isWithin } from "../../core/common.js";
@@ -36,6 +37,7 @@ import { hasGraphData } from "../../indexer/db/graph-db.js";
36
37
  import { listRelatedPathsForFile } from "../../indexer/graph/graph-boost.js";
37
38
  import { extractGraphForSingleFile } from "../../indexer/graph/graph-extraction.js";
38
39
  import { lookupBundleRef, lookupBundleRefWithResolution } from "../../indexer/indexer.js";
40
+ import { projectMarkdownFragmentContent } from "../../indexer/passes/metadata.js";
39
41
  import { ensurePrimaryIndexForRead, resolveReadSources } from "../../indexer/read-preflight.js";
40
42
  import { buildEditHint, findSourceForPath, isEditable, resolveSourceEntries, } from "../../indexer/search/search-source.js";
41
43
  import { recentShowCount, recordShowUsage } from "../../indexer/usage/show-usage.js";
@@ -45,6 +47,7 @@ import { resolveSourcesForOrigin } from "../../registry/origin-resolve.js";
45
47
  import { resolveStorageLocations } from "../../storage/locations.js";
46
48
  import { closeDatabase, openExistingDatabase } from "../../storage/repositories/index-connection.js";
47
49
  import { TELEMETRY_BUSY_TIMEOUT_MS, withIndexDb } from "../../storage/repositories/index-db.js";
50
+ import { getIndexedMarkdownFragment } from "../../storage/repositories/index-fts-repository.js";
48
51
  import { computeBodyHash } from "../../storage/repositories/index-llm-cache-repository.js";
49
52
  // Eagerly import source providers to trigger self-registration.
50
53
  import "../../sources/providers/index.js";
@@ -76,9 +79,15 @@ export async function akmShowUnified(input) {
76
79
  // (env — key names only; secret — never rendered), fragment or not. Warn
77
80
  // and ignore the fragment rather than refusing the whole show.
78
81
  warnSensitiveFragmentUnsupported(parseBundleRef(ref));
79
- // Auto-index when stale so the index is current before lookup.
80
- const { primarySource } = resolveReadSources();
81
- await ensurePrimaryIndexForRead(primarySource);
82
+ // An opaque fragment selector is an indexed revision handle. Do not refresh
83
+ // it away between search and show if disk changed concurrently; the stored
84
+ // safe substrate below is its source of truth. Friendly heading selectors
85
+ // intentionally retain the normal source-live read behavior.
86
+ const parsedRef = parseBundleRef(ref);
87
+ if (!parsedRef.fragment?.startsWith("akm-fragment-")) {
88
+ const { primarySource } = resolveReadSources();
89
+ await ensurePrimaryIndexForRead(primarySource);
90
+ }
82
91
  // Try local filesystem (FTS5 index lookup)
83
92
  const result = await showLocal(input);
84
93
  // Scope filter narrows resolution: if a scope filter was supplied, the
@@ -226,11 +235,14 @@ export async function showLocal(input) {
226
235
  }
227
236
  const fileCtx = buildFileContext(sourceStashDir, assetPath);
228
237
  const presentedName = indexedEntry.name;
238
+ const indexedFragment = parsed.fragment?.startsWith("akm-fragment-")
239
+ ? withIndexDb((db) => getIndexedMarkdownFragment(db, indexedEntry.itemRef, parsed.fragment))
240
+ : undefined;
229
241
  const indexedRenderer = rendererForIndexedEntry(indexedEntry, fileCtx);
230
242
  let response;
231
243
  try {
232
244
  if (indexedRenderer === null) {
233
- response = buildIndexedProjectionResponse(indexedEntry, assetPath, parsed.fragment);
245
+ response = buildIndexedProjectionResponse(indexedEntry, assetPath, parsed.fragment, indexedFragment?.content);
234
246
  }
235
247
  else {
236
248
  const match = typeof indexedRenderer === "string" ? indexedMatch(indexedEntry, indexedRenderer) : recognizeMatch(fileCtx);
@@ -251,7 +263,7 @@ export async function showLocal(input) {
251
263
  warn(`Fragment "#${parsed.fragment}" was ignored: ${makeBundleRef(parsed.bundle, parsed.conceptId)} is not a Markdown document, so heading fragments do not apply. Showing the whole asset.`);
252
264
  }
253
265
  else {
254
- applyMarkdownFragment(response, fileCtx.content(), parsed.fragment, presentedName);
266
+ applyMarkdownFragment(response, fileCtx.content(), parsed.fragment, presentedName, indexedFragment?.content);
255
267
  }
256
268
  }
257
269
  }
@@ -495,7 +507,7 @@ function rendererForIndexedEntry(entry, _file) {
495
507
  function indexedMatch(entry, renderer) {
496
508
  return { type: entry.type, specificity: Number.MAX_SAFE_INTEGER, renderer, meta: { name: entry.name } };
497
509
  }
498
- function buildIndexedProjectionResponse(entry, assetPath, fragment) {
510
+ function buildIndexedProjectionResponse(entry, assetPath, fragment, indexedFragmentContent) {
499
511
  const isMarkdown = path.extname(assetPath).toLowerCase() === ".md";
500
512
  if (fragment !== undefined && !isMarkdown) {
501
513
  warn(`Fragment "#${fragment}" was ignored: ${entry.conceptId} is not a Markdown document, so heading fragments do not apply. Showing the whole asset.`);
@@ -503,7 +515,7 @@ function buildIndexedProjectionResponse(entry, assetPath, fragment) {
503
515
  const raw = fs.readFileSync(assetPath, "utf8");
504
516
  const parsed = parseFrontmatter(raw);
505
517
  const content = fragment !== undefined && isMarkdown
506
- ? requireMarkdownSection(parsed.content, fragment, entry.name).content
518
+ ? (indexedFragmentContent ?? requireMarkdownSection(raw, fragment, entry.name).content)
507
519
  : parsed.content;
508
520
  const description = entry.document?.description ?? asNonEmptyString(parsed.data.description);
509
521
  const tags = entry.document?.tags ??
@@ -520,8 +532,8 @@ function buildIndexedProjectionResponse(entry, assetPath, fragment) {
520
532
  ...(tags && tags.length > 0 ? { tags } : {}),
521
533
  };
522
534
  }
523
- function applyMarkdownFragment(response, raw, fragment, name) {
524
- const section = requireMarkdownSection(parseFrontmatter(raw).content, fragment, name).content;
535
+ function applyMarkdownFragment(response, raw, fragment, name, indexedFragmentContent) {
536
+ const section = indexedFragmentContent ?? requireMarkdownSection(raw, fragment, name).content;
525
537
  if (response.template !== undefined)
526
538
  response.template = section;
527
539
  else if (response.prompt !== undefined)
@@ -533,6 +545,11 @@ function requireMarkdownSection(content, fragment, name) {
533
545
  const section = extractSection(content, fragment);
534
546
  if (section)
535
547
  return section;
548
+ const indexed = projectMarkdownFragmentContent(content);
549
+ const safeFragment = indexed ? fragmentForSelector(indexed, fragment) : undefined;
550
+ if (safeFragment) {
551
+ return { content: safeFragment.text, startLine: safeFragment.startLine, endLine: safeFragment.endLine };
552
+ }
536
553
  const available = markdownFragmentSlugs(content);
537
554
  throw new NotFoundError(`Fragment "#${fragment}" not found in ${name}.` +
538
555
  (available.length > 0 ? ` Available fragments: ${available.map((slug) => `#${slug}`).join(", ")}.` : ""));
@@ -81,7 +81,7 @@
81
81
  */
82
82
  import fs from "node:fs";
83
83
  import path from "node:path";
84
- import { applyPostContributorFields, applyPreContributorFields, extractPackageMetadata, } from "../../../indexer/passes/metadata.js";
84
+ import { applyPostContributorFields, applyPreContributorFields, extractPackageMetadata, getMarkdownFragmentContent, hasMarkdownFragmentContent, setMarkdownFragmentContent, } from "../../../indexer/passes/metadata.js";
85
85
  import { assetPathCandidatesForName, assetPathForName, deriveCanonicalAssetNameFromStashRoot, placementTypes, stashDirFor, stashDirNames, } from "../../asset/asset-placement.js";
86
86
  import { parseFrontmatter } from "../../asset/frontmatter.js";
87
87
  import { executionDefaultsFromFrontmatter, renderMarkdownExecutionSource } from "../execution-source.js";
@@ -244,6 +244,10 @@ function indexDocumentFromEntry(entry, base, rendererName) {
244
244
  doc.lessonStrength = entry.lessonStrength;
245
245
  if (entry.derivedFrom !== undefined)
246
246
  doc.derivedFrom = entry.derivedFrom;
247
+ // Internal fragment substrate follows the recognition projection without
248
+ // becoming an IndexDocument field or serialized search payload.
249
+ if (hasMarkdownFragmentContent(entry))
250
+ setMarkdownFragmentContent(doc, getMarkdownFragmentContent(entry));
247
251
  return doc;
248
252
  }
249
253
  function conceptIdForRecognizedType(root, filePath, type) {
@@ -0,0 +1,146 @@
1
+ // This Source Code Form is subject to the terms of the Mozilla Public
2
+ // License, v. 2.0. If a copy of the MPL was not distributed with this
3
+ // file, You can obtain one at https://mozilla.org/MPL/2.0/.
4
+ /**
5
+ * Deterministic, addressable Markdown fragments.
6
+ *
7
+ * Input is the safe, line-preserving Markdown projection, never the raw file.
8
+ * Keeping that distinction here means search can only emit selectors that
9
+ * `show` can reproduce without disclosing fenced/commented/link-target bytes.
10
+ */
11
+ import { createHash } from "node:crypto";
12
+ import { markdownHeadingSlug, parseMarkdownToc } from "./markdown.js";
13
+ export const MARKDOWN_FRAGMENT_MAX_CHARS = 1600;
14
+ export const MARKDOWN_FRAGMENT_PREFIX = "akm-fragment-";
15
+ function hash(text) {
16
+ return createHash("sha256").update(text).digest("hex");
17
+ }
18
+ function uniqueSlugs(body) {
19
+ const out = new Map();
20
+ const seen = new Set();
21
+ for (const heading of parseMarkdownToc(body).headings) {
22
+ const base = markdownHeadingSlug(heading.text);
23
+ if (!base)
24
+ continue;
25
+ let slug = base;
26
+ for (let suffix = 1; seen.has(slug); suffix++)
27
+ slug = `${base}-${suffix}`;
28
+ seen.add(slug);
29
+ out.set(heading.line, slug);
30
+ }
31
+ return out;
32
+ }
33
+ function textOf(lines) {
34
+ return lines.join("\n").trim();
35
+ }
36
+ /** Split a large sequence only at paragraph, then word, boundaries. */
37
+ function splitPiece(piece, maxChars) {
38
+ if (textOf(piece.lines).length <= maxChars)
39
+ return [piece];
40
+ const pieces = [];
41
+ let start = 0;
42
+ while (start < piece.lines.length) {
43
+ let end = start;
44
+ let chars = 0;
45
+ while (end < piece.lines.length) {
46
+ const next = piece.lines[end];
47
+ // Let the single-line word-window path below own an oversized first
48
+ // line. Without this guard it is consumed whole before that path can
49
+ // run, silently defeating the fragment bound for transcripts/logs.
50
+ if (end === start && next.length > maxChars)
51
+ break;
52
+ if (end > start && chars + next.length + 1 > maxChars)
53
+ break;
54
+ chars += next.length + (end > start ? 1 : 0);
55
+ end++;
56
+ }
57
+ // A single long authored line needs word windows, but its source range is
58
+ // intentionally the same line for every window.
59
+ if (end === start) {
60
+ const line = piece.lines[start];
61
+ let offset = 0;
62
+ while (offset < line.length) {
63
+ let cut = Math.min(offset + maxChars, line.length);
64
+ if (cut < line.length) {
65
+ const space = line.lastIndexOf(" ", cut);
66
+ if (space > offset + Math.floor(maxChars * 0.55))
67
+ cut = space;
68
+ }
69
+ pieces.push({ lines: [line.slice(offset, cut).trim()], startLine: piece.startLine + start });
70
+ offset = cut;
71
+ while (line[offset] === " ")
72
+ offset++;
73
+ }
74
+ start++;
75
+ continue;
76
+ }
77
+ // Prefer not to split a paragraph if a blank boundary fits before `end`.
78
+ let preferred = -1;
79
+ for (let i = start + 1; i < end; i++)
80
+ if (!piece.lines[i].trim())
81
+ preferred = i;
82
+ if (preferred > start)
83
+ end = preferred;
84
+ pieces.push({ lines: piece.lines.slice(start, end), startLine: piece.startLine + start });
85
+ start = end;
86
+ while (start < piece.lines.length && !piece.lines[start].trim())
87
+ start++;
88
+ }
89
+ return pieces.filter((candidate) => textOf(candidate.lines));
90
+ }
91
+ /**
92
+ * Heading sections first, then paragraph/word windows. `startLine`/`endLine`
93
+ * always refer to the authored file's line numbers because the projection
94
+ * preserves one line per source line (with excluded bytes blanked out).
95
+ */
96
+ export function splitMarkdownFragmentStats(body, maxChars = MARKDOWN_FRAGMENT_MAX_CHARS) {
97
+ const lines = body.split(/\r?\n/);
98
+ const headings = parseMarkdownToc(body).headings;
99
+ const boundaries = [1, ...headings.map((heading) => heading.line), lines.length + 1]
100
+ .filter((line, index, all) => index === 0 || line !== all[index - 1])
101
+ .sort((left, right) => left - right);
102
+ const slugs = uniqueSlugs(body);
103
+ const pieces = [];
104
+ let sectionCount = 0;
105
+ for (let i = 0; i < boundaries.length - 1; i++) {
106
+ const startLine = boundaries[i];
107
+ const end = boundaries[i + 1] - 1;
108
+ const section = { lines: lines.slice(startLine - 1, end), startLine, headingSlug: slugs.get(startLine) };
109
+ if (textOf(section.lines)) {
110
+ sectionCount++;
111
+ pieces.push(...splitPiece(section, maxChars));
112
+ }
113
+ }
114
+ // Whether a heading section survived as one fragment is a property of the
115
+ // complete piece set. Count once before materialization instead of scanning
116
+ // every piece for every fragment (which made many headed documents O(N²)).
117
+ const piecesPerHeading = new Map();
118
+ for (const piece of pieces) {
119
+ if (piece.headingSlug)
120
+ piecesPerHeading.set(piece.headingSlug, (piecesPerHeading.get(piece.headingSlug) ?? 0) + 1);
121
+ }
122
+ const fragments = pieces.map((piece, ordinal) => {
123
+ const text = textOf(piece.lines);
124
+ const contentLines = piece.lines.map((line, index) => ({ line, index })).filter(({ line }) => line.trim());
125
+ const first = contentLines[0]?.index ?? 0;
126
+ const last = contentLines.at(-1)?.index ?? 0;
127
+ const digest = hash(text);
128
+ const unsplitHeading = piece.headingSlug && piecesPerHeading.get(piece.headingSlug) === 1;
129
+ return {
130
+ fragmentId: `${MARKDOWN_FRAGMENT_PREFIX}${ordinal + 1}-${digest.slice(0, 12)}`,
131
+ ordinal,
132
+ startLine: piece.startLine + first,
133
+ endLine: piece.startLine + last,
134
+ ...(unsplitHeading ? { headingSlug: piece.headingSlug } : {}),
135
+ text,
136
+ hash: digest,
137
+ };
138
+ });
139
+ return { fragments, hardSplitCount: Math.max(0, pieces.length - sectionCount) };
140
+ }
141
+ export function splitMarkdownFragments(body, maxChars = MARKDOWN_FRAGMENT_MAX_CHARS) {
142
+ return splitMarkdownFragmentStats(body, maxChars).fragments;
143
+ }
144
+ export function fragmentForSelector(body, selector) {
145
+ return splitMarkdownFragments(body).find((fragment) => fragment.fragmentId === selector || fragment.headingSlug === selector);
146
+ }
@@ -0,0 +1,25 @@
1
+ // This Source Code Form is subject to the terms of the Mozilla Public
2
+ // License, v. 2.0. If a copy of the MPL was not distributed with this
3
+ // file, You can obtain one at https://mozilla.org/MPL/2.0/.
4
+ /** Dependency-safe fixed FTS5 calibration shared by storage and ranking. */
5
+ const SCORE_FLOOR = 0.3;
6
+ const PARENT_CEILING = 0.8;
7
+ // Fragment BM25 is from a distinct FTS population. Its separately calibrated
8
+ // ceiling is an explicit evidence policy, not a cross-table comparability claim:
9
+ // when a body fragment independently proves the query, retain that actionable
10
+ // selector beside the same parent's length-penalized whole-body row. Metadata
11
+ // and cross-fragment conjunctions cannot enter this population.
12
+ const FRAGMENT_CEILING = 0.8;
13
+ const BM25_REFERENCE = 0.000001;
14
+ const LOG_SHAPE = 3;
15
+ export function stableFtsScore(bm25Score, population = "parent") {
16
+ const ceiling = population === "fragment" ? FRAGMENT_CEILING : PARENT_CEILING;
17
+ if (bm25Score === Number.NEGATIVE_INFINITY)
18
+ return ceiling;
19
+ if (!Number.isFinite(bm25Score) || bm25Score >= 0)
20
+ return SCORE_FLOOR;
21
+ const scaled = Math.log1p(-bm25Score / BM25_REFERENCE);
22
+ if (!Number.isFinite(scaled))
23
+ return ceiling;
24
+ return SCORE_FLOOR + (ceiling - SCORE_FLOOR) * (scaled / (scaled + LOG_SHAPE));
25
+ }
@@ -51,20 +51,41 @@ function buildWorkflowAction(ref) {
51
51
  * see the file header.
52
52
  */
53
53
  export const TYPE_PRESENTATION = {
54
- skill: { label: "Skill", renderer: "skill-md", action: (ref) => `akm show ${ref} -> follow the instructions` },
54
+ skill: {
55
+ label: "Skill",
56
+ renderer: "skill-md",
57
+ action: (ref) => `akm show ${ref} -> follow the instructions`,
58
+ fragmentRef: false,
59
+ },
55
60
  command: {
56
61
  label: "Command",
57
62
  renderer: "command-md",
58
63
  action: (ref) => `akm show ${ref} -> fill placeholders and dispatch`,
64
+ fragmentRef: false,
65
+ },
66
+ agent: {
67
+ label: "Agent",
68
+ renderer: "agent-md",
69
+ action: (ref) => `akm show ${ref} -> dispatch with full prompt`,
70
+ fragmentRef: false,
59
71
  },
60
- agent: { label: "Agent", renderer: "agent-md", action: (ref) => `akm show ${ref} -> dispatch with full prompt` },
61
72
  knowledge: {
62
73
  label: "Knowledge",
63
74
  renderer: "knowledge-md",
64
75
  action: (ref) => `akm show ${ref} -> read reference material`,
65
76
  },
66
- workflow: { label: "Workflow", renderer: "workflow-md", action: (ref) => buildWorkflowAction(ref) },
67
- script: { label: "Script", renderer: "script-source", action: (ref) => `akm show ${ref} -> execute the run command` },
77
+ workflow: {
78
+ label: "Workflow",
79
+ renderer: "workflow-md",
80
+ action: (ref) => buildWorkflowAction(ref),
81
+ fragmentRef: false,
82
+ },
83
+ script: {
84
+ label: "Script",
85
+ renderer: "script-source",
86
+ action: (ref) => `akm show ${ref} -> execute the run command`,
87
+ fragmentRef: false,
88
+ },
68
89
  memory: { label: "Memory", renderer: "memory-md", action: (ref) => `akm show ${ref} -> recall context` },
69
90
  env: {
70
91
  label: "Env",
@@ -85,6 +106,7 @@ export const TYPE_PRESENTATION = {
85
106
  label: "Task",
86
107
  renderer: "task-yaml",
87
108
  action: (ref) => `akm show ${ref} -> inspect; akm task run <id> -> run now; edit the file + akm task sync -> unschedule`,
109
+ fragmentRef: false,
88
110
  },
89
111
  session: {
90
112
  label: "Session",
@@ -105,6 +127,7 @@ export const TYPE_PRESENTATION = {
105
127
  label: "Instruction",
106
128
  renderer: "knowledge-md",
107
129
  action: (ref) => `akm show ${ref} -> read the project instructions`,
130
+ fragmentRef: false,
108
131
  },
109
132
  };
110
133
  /** Generic fallback for a type outside {@link KNOWN_TYPES} — never `undefined`, never a throw. */
@@ -120,6 +143,15 @@ export function presentationFor(type) {
120
143
  return TYPE_PRESENTATION[type];
121
144
  return DEFAULT_PRESENTATION;
122
145
  }
146
+ /**
147
+ * A missing declaration means a read-like reference or foreign type can expose
148
+ * a safe selector. Built-in executable and instruction-bearing types opt out
149
+ * explicitly above: their search match can be a fragment, but the advertised
150
+ * action needs the parent canonical ref and its complete context.
151
+ */
152
+ export function allowsFragmentRef(type) {
153
+ return presentationFor(type).fragmentRef !== false;
154
+ }
123
155
  export const defaultRendererRegistry = {
124
156
  rendererNameFor(type) {
125
157
  return presentationFor(type).renderer;
@@ -31,6 +31,7 @@ import { closeDatabase, openExistingDatabase } from "../storage/repositories/ind
31
31
  import { deleteEntriesByIds, getEntryCount, upsertEntry } from "../storage/repositories/index-entries-repository.js";
32
32
  import { deriveEntryProvenance, deriveInstallations } from "./installations.js";
33
33
  import { generateEmbeddingsForDb, publishTargetedEmbeddingMeta } from "./materialize-embeddings.js";
34
+ import { getMarkdownFragmentContent, hasMarkdownFragmentContent, setMarkdownFragmentContent, } from "./passes/metadata.js";
34
35
  import { drainDirDocuments } from "./scan/drain-dir.js";
35
36
  import { buildSearchText } from "./search/search-fields.js";
36
37
  import { buildFileContext } from "./walk/file-context.js";
@@ -158,6 +159,9 @@ export async function indexWrittenAssets(stashDir, filePaths, options = {}) {
158
159
  let entryWithSize = entry;
159
160
  try {
160
161
  entryWithSize = { ...entry, fileSize: fs.statSync(file).size };
162
+ if (hasMarkdownFragmentContent(entry)) {
163
+ setMarkdownFragmentContent(entryWithSize, getMarkdownFragmentContent(entry));
164
+ }
161
165
  }
162
166
  catch {
163
167
  // stat raced a delete — index without the size, like the full walk does.
@@ -31,7 +31,7 @@ import { deriveEntryProvenance, deriveInstallations } from "./installations.js";
31
31
  import { indexedPathMatchesOwner, resolveAdapterConceptOwner, } from "./lookup/adapter-concept-owner.js";
32
32
  import { generateEmbeddingsForDb } from "./materialize-embeddings.js";
33
33
  import { canUseIncrementalSkip, computeDirFingerprint, getCachedDirState, getDirIndexState, inferZeroRowReason, } from "./passes/dir-staleness.js";
34
- import { isEnrichmentComplete, isWorkflowSkipWarning } from "./passes/metadata.js";
34
+ import { getMarkdownFragmentContent, hasMarkdownFragmentContent, isEnrichmentComplete, isWorkflowSkipWarning, setMarkdownFragmentContent, } from "./passes/metadata.js";
35
35
  import { drainDirDocuments } from "./scan/drain-dir.js";
36
36
  import { buildSearchText } from "./search/search-fields.js";
37
37
  import { purgeOldUsageEvents } from "./usage/usage-events.js";
@@ -1504,7 +1504,10 @@ export function createEnrichmentDeadline(timeoutMs, totalEntries) {
1504
1504
  // ── Helpers ─────────────────────────────────────────────────────────────────
1505
1505
  function attachFileSize(entry, entryPath) {
1506
1506
  try {
1507
- return { ...entry, fileSize: fs.statSync(entryPath).size };
1507
+ const sized = { ...entry, fileSize: fs.statSync(entryPath).size };
1508
+ if (hasMarkdownFragmentContent(entry))
1509
+ setMarkdownFragmentContent(sized, getMarkdownFragmentContent(entry));
1510
+ return sized;
1508
1511
  }
1509
1512
  catch {
1510
1513
  return entry;
@@ -1179,6 +1179,67 @@ export function projectMarkdownContent(body, truncationInfo) {
1179
1179
  truncationInfo.truncated = text.length > MARKDOWN_CONTENT_MAX_CHARS;
1180
1180
  return truncateUnicodeSafe(text, MARKDOWN_CONTENT_MAX_CHARS);
1181
1181
  }
1182
+ // Fragment text is intentionally not an IndexDocument field. IndexDocument is
1183
+ // an adapter/search payload boundary; this is an internal, derived index input
1184
+ // that is persisted separately by the entries repository for deterministic FTS
1185
+ // rebuilds. The WeakMap follows the already-read document through recognition
1186
+ // without making safe body bytes observable through public payloads.
1187
+ const markdownFragmentContentByEntry = new WeakMap();
1188
+ const markdownFragmentProjectionEntries = new WeakSet();
1189
+ export function setMarkdownFragmentContent(entry, content) {
1190
+ markdownFragmentProjectionEntries.add(entry);
1191
+ if (content)
1192
+ markdownFragmentContentByEntry.set(entry, content);
1193
+ }
1194
+ export function getMarkdownFragmentContent(entry) {
1195
+ return markdownFragmentContentByEntry.get(entry);
1196
+ }
1197
+ export function hasMarkdownFragmentContent(entry) {
1198
+ return markdownFragmentProjectionEntries.has(entry);
1199
+ }
1200
+ /**
1201
+ * Produce a safe, structure-preserving projection for fragment indexing.
1202
+ * Excluded source lines are retained as blank lines so fragment locations map
1203
+ * exactly to authored line numbers. This must be called from both indexing and
1204
+ * `show`; it deliberately never performs a storage-layer file reread.
1205
+ */
1206
+ export function projectMarkdownFragmentContent(raw) {
1207
+ const lines = raw.split(/\r?\n/);
1208
+ const parsed = parseFrontmatter(raw);
1209
+ const start = parsed.frontmatter ? parsed.bodyStartLine - 1 : 0;
1210
+ const projected = lines.map(() => "");
1211
+ let fence;
1212
+ const htmlComment = { inComment: false };
1213
+ for (let index = start; index < lines.length; index++) {
1214
+ const rawLine = lines[index];
1215
+ if (fence) {
1216
+ if (isMarkdownFenceClosing(rawLine, fence))
1217
+ fence = undefined;
1218
+ continue;
1219
+ }
1220
+ if (!htmlComment.inComment) {
1221
+ const opening = parseMarkdownFenceOpening(rawLine);
1222
+ if (opening) {
1223
+ fence = opening;
1224
+ continue;
1225
+ }
1226
+ }
1227
+ let safe = stripMarkdownHtmlComments(rawLine, htmlComment);
1228
+ const opening = parseMarkdownFenceOpening(safe.trim());
1229
+ if (opening) {
1230
+ fence = opening;
1231
+ continue;
1232
+ }
1233
+ // Reference link destinations and standalone HTML are not retrieval
1234
+ // evidence and can contain credential-bearing URLs.
1235
+ if (/^\s*\[[^\]]+\]:\s*\S+/.test(safe) || /^\s*<[^>]+>\s*$/.test(safe))
1236
+ continue;
1237
+ safe = stripMarkdownLinkDestinations(safe).replace(/<[^>]+>/g, " ");
1238
+ projected[index] = safe.replace(/[ \t]+$/g, "");
1239
+ }
1240
+ const text = projected.join("\n");
1241
+ return text.trim() ? text : undefined;
1242
+ }
1182
1243
  // ── Metadata Generation ─────────────────────────────────────────────────────
1183
1244
  /**
1184
1245
  * Priorities 1-2 of the metadata pipeline — package.json (P1), `.md`
@@ -1220,7 +1281,9 @@ export function applyPreContributorFields(entry, file, ctx, pkgMeta) {
1220
1281
  applyProvenanceFrontmatter(entry, parsed.data);
1221
1282
  // Native Markdown has one bounded low-weight body projection. Sensitive
1222
1283
  // types and raw session/checkpoint material never cross this boundary.
1223
- if (entry.type !== "env" && entry.type !== "session" && !hasSessionMemoryMarker(parsed.data, parsed.content)) {
1284
+ const safeForFragments = entry.type !== "env" && entry.type !== "session" && !hasSessionMemoryMarker(parsed.data, parsed.content);
1285
+ setMarkdownFragmentContent(entry, safeForFragments ? projectMarkdownFragmentContent(content) : undefined);
1286
+ if (safeForFragments) {
1224
1287
  const truncationInfo = { truncated: false };
1225
1288
  const contentProjection = projectMarkdownContent(parsed.content, truncationInfo);
1226
1289
  if (contentProjection) {