akm-cli 0.9.13 → 0.9.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +39 -0
- package/dist/commands/improve/eligibility.js +27 -15
- package/dist/commands/read/curate.js +3 -2
- package/dist/commands/read/show.js +26 -9
- package/dist/core/adapter/adapters/akm-adapter.js +5 -1
- package/dist/core/asset/markdown-fragments.js +146 -0
- package/dist/core/lexical-score.js +25 -0
- package/dist/core/type-presentation.js +36 -4
- package/dist/indexer/index-written-assets.js +4 -0
- package/dist/indexer/indexer.js +5 -2
- package/dist/indexer/passes/metadata.js +64 -1
- package/dist/indexer/scan/doc-to-entry.js +3 -0
- package/dist/indexer/scan/drain-dir.js +33 -22
- package/dist/indexer/search/db-search.js +72 -14
- package/dist/indexer/search/name-match.js +35 -0
- package/dist/indexer/search/ranking-contributors.js +15 -12
- package/dist/indexer/search/ranking.js +42 -18
- package/dist/indexer/usage/show-usage.js +14 -2
- package/dist/llm/graph-extract.js +18 -67
- package/dist/scripts/akm-migrate-node.js +334 -90
- package/dist/scripts/akm-migrate.js +334 -90
- package/dist/storage/repositories/index-connection.js +23 -8
- package/dist/storage/repositories/index-entries-repository.js +3 -2
- package/dist/storage/repositories/index-entry-schema.js +43 -3
- package/dist/storage/repositories/index-fts-repository.js +160 -14
- package/dist/storage/repositories/index-schema.js +8 -18
- package/docs/migration/release-notes/0.9.14.md +26 -0
- package/docs/migration/release-notes/README.md +2 -0
- package/package.json +1 -1
package/CHANGELOG.md
CHANGED
|
@@ -4,6 +4,45 @@ All notable changes to this project will be documented in this file.
|
|
|
4
4
|
|
|
5
5
|
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
|
|
6
6
|
|
|
7
|
+
## [0.9.14] - 2026-09-04
|
|
8
|
+
|
|
9
|
+
### Added
|
|
10
|
+
|
|
11
|
+
- **Long Markdown bodies can return the matching lexical fragment (#937).**
|
|
12
|
+
The derived index now keeps a separate, safe fragment population for lexical
|
|
13
|
+
retrieval. Search can return an addressable `#akm-fragment-…` ref, and `akm
|
|
14
|
+
show` resolves that ref to the exact indexed projection. Headingless and
|
|
15
|
+
oversized documents split at paragraph or word boundaries, so a fact in the
|
|
16
|
+
middle of a long body is no longer represented only by its parent document.
|
|
17
|
+
|
|
18
|
+
### Changed
|
|
19
|
+
|
|
20
|
+
- **Index generations are checked before use (#934).** A newer index is never
|
|
21
|
+
queried or rebuilt by an older binary; it reports that akm must be upgraded.
|
|
22
|
+
An older derived index is rebuilt from the materialized sources by the current
|
|
23
|
+
binary. A current-generation stamp is accepted only when the canonical entry,
|
|
24
|
+
parent FTS, and fragment surfaces all match, and the stamp is written only
|
|
25
|
+
after schema creation succeeds. This release advances the derived index from
|
|
26
|
+
v22 to v23 for fragment retrieval.
|
|
27
|
+
- **Lexical relevance remains stable through scoring and relaxed-query ties
|
|
28
|
+
(#933, #940).** Lexical scores use a fixed monotone calibration rather than a
|
|
29
|
+
result-set-relative scale, preserving score headroom. Relaxed matches retain
|
|
30
|
+
body relevance as tie evidence, including when a belief-state ceiling also
|
|
31
|
+
applies. This compound-safe implementation supersedes PR #941.
|
|
32
|
+
- **The frozen W0 lexical weight matrix remains the shipped policy (#930).**
|
|
33
|
+
The W1 and W2 alternatives were measured and rejected; no unvalidated weight
|
|
34
|
+
change is included in this release.
|
|
35
|
+
|
|
36
|
+
### Fixed
|
|
37
|
+
|
|
38
|
+
- **Fuzzy name matches require structural identity.** Short or opaque name
|
|
39
|
+
fragments no longer create a false identity match merely because their text
|
|
40
|
+
overlaps a stored name.
|
|
41
|
+
- **Full indexing honors canonical workflow-source ownership.** When peer `.md`
|
|
42
|
+
and `.yml` sources map to one workflow ref, the index now persists the same
|
|
43
|
+
deterministic `.md` winner used by lookup and execution instead of allowing
|
|
44
|
+
filesystem enumeration order to select the stored row.
|
|
45
|
+
|
|
7
46
|
## [0.9.13] - 2026-09-04
|
|
8
47
|
|
|
9
48
|
### Added
|
|
@@ -6,7 +6,7 @@ import path from "node:path";
|
|
|
6
6
|
import { parseFrontmatter } from "../../core/asset/frontmatter.js";
|
|
7
7
|
import { conceptIdFromTypeName, parseRefInput, resolveRef } from "../../core/asset/resolve-ref.js";
|
|
8
8
|
import { loadConfig } from "../../core/config/config.js";
|
|
9
|
-
import { NotFoundError, rethrowIfTestIsolationError, UsageError } from "../../core/errors.js";
|
|
9
|
+
import { ConfigError, NotFoundError, rethrowIfTestIsolationError, UsageError } from "../../core/errors.js";
|
|
10
10
|
import { readEvents } from "../../core/events.js";
|
|
11
11
|
import { isPathAbsent } from "../../core/path-access.js";
|
|
12
12
|
import { getDbPath } from "../../core/paths.js";
|
|
@@ -47,19 +47,11 @@ function describeIndexSnapshot(readOnly, status) {
|
|
|
47
47
|
reason: readOnly ? "loaded a non-mutating point-in-time copy of the existing index" : "loaded the prepared index",
|
|
48
48
|
};
|
|
49
49
|
}
|
|
50
|
-
if (status === "missing") {
|
|
51
|
-
return {
|
|
52
|
-
status,
|
|
53
|
-
reason: readOnly
|
|
54
|
-
? "index.db is missing; dry-run uses an empty snapshot and does not create it"
|
|
55
|
-
: "index.db is missing after index preparation; the selector uses an empty snapshot",
|
|
56
|
-
};
|
|
57
|
-
}
|
|
58
50
|
return {
|
|
59
51
|
status,
|
|
60
52
|
reason: readOnly
|
|
61
|
-
? "index.db
|
|
62
|
-
: "index.db
|
|
53
|
+
? "index.db is missing; dry-run uses an empty snapshot and does not create it"
|
|
54
|
+
: "index.db is missing after index preparation; the selector uses an empty snapshot",
|
|
63
55
|
};
|
|
64
56
|
}
|
|
65
57
|
function describeUnavailableSnapshot(error) {
|
|
@@ -68,6 +60,26 @@ function describeUnavailableSnapshot(error) {
|
|
|
68
60
|
reason: `index.db cannot provide a stable non-mutating snapshot (${error.message}); dry-run uses an empty snapshot`,
|
|
69
61
|
};
|
|
70
62
|
}
|
|
63
|
+
/**
|
|
64
|
+
* A dry-run is explicitly a non-mutating planning operation, so it may report
|
|
65
|
+
* an unusable derived index as an empty snapshot. This is deliberately a
|
|
66
|
+
* typed boundary mapping: readers otherwise receive no incompatible handle,
|
|
67
|
+
* and we must not turn an arbitrary SQLite error into a successful plan by
|
|
68
|
+
* matching its text.
|
|
69
|
+
*/
|
|
70
|
+
function isIncompatibleIndexError(error) {
|
|
71
|
+
return error instanceof ConfigError && error.code === "INDEX_SCHEMA_INCOMPATIBLE";
|
|
72
|
+
}
|
|
73
|
+
function describeIncompatibleIndexSnapshot(error) {
|
|
74
|
+
// `INDEX_SCHEMA_INCOMPATIBLE` establishes that this is the derived-index
|
|
75
|
+
// boundary, and the error's hint preserves whether this binary should
|
|
76
|
+
// rebuild an older/unknown generation or upgrade for a newer one.
|
|
77
|
+
const action = error.hint() ?? error.message;
|
|
78
|
+
return {
|
|
79
|
+
status: "incompatible",
|
|
80
|
+
reason: `index.db is incompatible; ${action} Dry-run uses an empty snapshot and does not migrate it.`,
|
|
81
|
+
};
|
|
82
|
+
}
|
|
71
83
|
export function resolveImproveScope(scope) {
|
|
72
84
|
const trimmed = scope?.trim();
|
|
73
85
|
if (!trimmed)
|
|
@@ -194,12 +206,12 @@ async function collectEligibleRefsFromIndex(scope, stashDir, improveProfile, rea
|
|
|
194
206
|
indexSnapshot: describeUnavailableSnapshot(error),
|
|
195
207
|
};
|
|
196
208
|
}
|
|
197
|
-
if (
|
|
209
|
+
if (readOnly && isIncompatibleIndexError(error)) {
|
|
198
210
|
return {
|
|
199
211
|
plannedRefs: [],
|
|
200
212
|
memorySummary: { eligible: 0, derived: 0 },
|
|
201
213
|
strategyFilteredRefs: [],
|
|
202
|
-
indexSnapshot:
|
|
214
|
+
indexSnapshot: describeIncompatibleIndexSnapshot(error),
|
|
203
215
|
};
|
|
204
216
|
}
|
|
205
217
|
throw error;
|
|
@@ -300,12 +312,12 @@ async function collectEligibleRefsFromIndex(scope, stashDir, improveProfile, rea
|
|
|
300
312
|
indexSnapshot: describeUnavailableSnapshot(error),
|
|
301
313
|
};
|
|
302
314
|
}
|
|
303
|
-
if (
|
|
315
|
+
if (readOnly && isIncompatibleIndexError(error)) {
|
|
304
316
|
return {
|
|
305
317
|
plannedRefs: [],
|
|
306
318
|
memorySummary: { eligible: 0, derived: 0 },
|
|
307
319
|
strategyFilteredRefs: [],
|
|
308
|
-
indexSnapshot:
|
|
320
|
+
indexSnapshot: describeIncompatibleIndexSnapshot(error),
|
|
309
321
|
};
|
|
310
322
|
}
|
|
311
323
|
throw error;
|
|
@@ -529,8 +529,9 @@ function getCurateFamily(ref) {
|
|
|
529
529
|
try {
|
|
530
530
|
// F4b: `ref` is a search-hit ref in the 0.9.0 conceptId grammar — parse via
|
|
531
531
|
// the new-grammar `parseRefInput` so skill/reference family grouping still
|
|
532
|
-
// recognizes it.
|
|
533
|
-
|
|
532
|
+
// recognizes it. Search may add an opaque Markdown selector; identity and
|
|
533
|
+
// family ownership are on the parent asset, not that selector.
|
|
534
|
+
const parsed = parseRefInput(ref.split("#", 1)[0]);
|
|
534
535
|
if (parsed.type === "skill") {
|
|
535
536
|
return { key: parsed.name, role: "root" };
|
|
536
537
|
}
|
|
@@ -23,6 +23,7 @@ import { assetPathForName, stashDirFor } from "../../core/asset/asset-placement.
|
|
|
23
23
|
import { makeBundleRef, parseBundleRef } from "../../core/asset/asset-ref.js";
|
|
24
24
|
import { parseFrontmatter } from "../../core/asset/frontmatter.js";
|
|
25
25
|
import { extractSection, markdownFragmentSlugs } from "../../core/asset/markdown.js";
|
|
26
|
+
import { fragmentForSelector } from "../../core/asset/markdown-fragments.js";
|
|
26
27
|
import { displayRef, typeNameFromConceptId } from "../../core/asset/resolve-ref.js";
|
|
27
28
|
import { META_DIR, parseMetaRef, readMetaFile } from "../../core/asset/stash-meta.js";
|
|
28
29
|
import { asNonEmptyString, isWithin } from "../../core/common.js";
|
|
@@ -36,6 +37,7 @@ import { hasGraphData } from "../../indexer/db/graph-db.js";
|
|
|
36
37
|
import { listRelatedPathsForFile } from "../../indexer/graph/graph-boost.js";
|
|
37
38
|
import { extractGraphForSingleFile } from "../../indexer/graph/graph-extraction.js";
|
|
38
39
|
import { lookupBundleRef, lookupBundleRefWithResolution } from "../../indexer/indexer.js";
|
|
40
|
+
import { projectMarkdownFragmentContent } from "../../indexer/passes/metadata.js";
|
|
39
41
|
import { ensurePrimaryIndexForRead, resolveReadSources } from "../../indexer/read-preflight.js";
|
|
40
42
|
import { buildEditHint, findSourceForPath, isEditable, resolveSourceEntries, } from "../../indexer/search/search-source.js";
|
|
41
43
|
import { recentShowCount, recordShowUsage } from "../../indexer/usage/show-usage.js";
|
|
@@ -45,6 +47,7 @@ import { resolveSourcesForOrigin } from "../../registry/origin-resolve.js";
|
|
|
45
47
|
import { resolveStorageLocations } from "../../storage/locations.js";
|
|
46
48
|
import { closeDatabase, openExistingDatabase } from "../../storage/repositories/index-connection.js";
|
|
47
49
|
import { TELEMETRY_BUSY_TIMEOUT_MS, withIndexDb } from "../../storage/repositories/index-db.js";
|
|
50
|
+
import { getIndexedMarkdownFragment } from "../../storage/repositories/index-fts-repository.js";
|
|
48
51
|
import { computeBodyHash } from "../../storage/repositories/index-llm-cache-repository.js";
|
|
49
52
|
// Eagerly import source providers to trigger self-registration.
|
|
50
53
|
import "../../sources/providers/index.js";
|
|
@@ -76,9 +79,15 @@ export async function akmShowUnified(input) {
|
|
|
76
79
|
// (env — key names only; secret — never rendered), fragment or not. Warn
|
|
77
80
|
// and ignore the fragment rather than refusing the whole show.
|
|
78
81
|
warnSensitiveFragmentUnsupported(parseBundleRef(ref));
|
|
79
|
-
//
|
|
80
|
-
|
|
81
|
-
|
|
82
|
+
// An opaque fragment selector is an indexed revision handle. Do not refresh
|
|
83
|
+
// it away between search and show if disk changed concurrently; the stored
|
|
84
|
+
// safe substrate below is its source of truth. Friendly heading selectors
|
|
85
|
+
// intentionally retain the normal source-live read behavior.
|
|
86
|
+
const parsedRef = parseBundleRef(ref);
|
|
87
|
+
if (!parsedRef.fragment?.startsWith("akm-fragment-")) {
|
|
88
|
+
const { primarySource } = resolveReadSources();
|
|
89
|
+
await ensurePrimaryIndexForRead(primarySource);
|
|
90
|
+
}
|
|
82
91
|
// Try local filesystem (FTS5 index lookup)
|
|
83
92
|
const result = await showLocal(input);
|
|
84
93
|
// Scope filter narrows resolution: if a scope filter was supplied, the
|
|
@@ -226,11 +235,14 @@ export async function showLocal(input) {
|
|
|
226
235
|
}
|
|
227
236
|
const fileCtx = buildFileContext(sourceStashDir, assetPath);
|
|
228
237
|
const presentedName = indexedEntry.name;
|
|
238
|
+
const indexedFragment = parsed.fragment?.startsWith("akm-fragment-")
|
|
239
|
+
? withIndexDb((db) => getIndexedMarkdownFragment(db, indexedEntry.itemRef, parsed.fragment))
|
|
240
|
+
: undefined;
|
|
229
241
|
const indexedRenderer = rendererForIndexedEntry(indexedEntry, fileCtx);
|
|
230
242
|
let response;
|
|
231
243
|
try {
|
|
232
244
|
if (indexedRenderer === null) {
|
|
233
|
-
response = buildIndexedProjectionResponse(indexedEntry, assetPath, parsed.fragment);
|
|
245
|
+
response = buildIndexedProjectionResponse(indexedEntry, assetPath, parsed.fragment, indexedFragment?.content);
|
|
234
246
|
}
|
|
235
247
|
else {
|
|
236
248
|
const match = typeof indexedRenderer === "string" ? indexedMatch(indexedEntry, indexedRenderer) : recognizeMatch(fileCtx);
|
|
@@ -251,7 +263,7 @@ export async function showLocal(input) {
|
|
|
251
263
|
warn(`Fragment "#${parsed.fragment}" was ignored: ${makeBundleRef(parsed.bundle, parsed.conceptId)} is not a Markdown document, so heading fragments do not apply. Showing the whole asset.`);
|
|
252
264
|
}
|
|
253
265
|
else {
|
|
254
|
-
applyMarkdownFragment(response, fileCtx.content(), parsed.fragment, presentedName);
|
|
266
|
+
applyMarkdownFragment(response, fileCtx.content(), parsed.fragment, presentedName, indexedFragment?.content);
|
|
255
267
|
}
|
|
256
268
|
}
|
|
257
269
|
}
|
|
@@ -495,7 +507,7 @@ function rendererForIndexedEntry(entry, _file) {
|
|
|
495
507
|
function indexedMatch(entry, renderer) {
|
|
496
508
|
return { type: entry.type, specificity: Number.MAX_SAFE_INTEGER, renderer, meta: { name: entry.name } };
|
|
497
509
|
}
|
|
498
|
-
function buildIndexedProjectionResponse(entry, assetPath, fragment) {
|
|
510
|
+
function buildIndexedProjectionResponse(entry, assetPath, fragment, indexedFragmentContent) {
|
|
499
511
|
const isMarkdown = path.extname(assetPath).toLowerCase() === ".md";
|
|
500
512
|
if (fragment !== undefined && !isMarkdown) {
|
|
501
513
|
warn(`Fragment "#${fragment}" was ignored: ${entry.conceptId} is not a Markdown document, so heading fragments do not apply. Showing the whole asset.`);
|
|
@@ -503,7 +515,7 @@ function buildIndexedProjectionResponse(entry, assetPath, fragment) {
|
|
|
503
515
|
const raw = fs.readFileSync(assetPath, "utf8");
|
|
504
516
|
const parsed = parseFrontmatter(raw);
|
|
505
517
|
const content = fragment !== undefined && isMarkdown
|
|
506
|
-
? requireMarkdownSection(
|
|
518
|
+
? (indexedFragmentContent ?? requireMarkdownSection(raw, fragment, entry.name).content)
|
|
507
519
|
: parsed.content;
|
|
508
520
|
const description = entry.document?.description ?? asNonEmptyString(parsed.data.description);
|
|
509
521
|
const tags = entry.document?.tags ??
|
|
@@ -520,8 +532,8 @@ function buildIndexedProjectionResponse(entry, assetPath, fragment) {
|
|
|
520
532
|
...(tags && tags.length > 0 ? { tags } : {}),
|
|
521
533
|
};
|
|
522
534
|
}
|
|
523
|
-
function applyMarkdownFragment(response, raw, fragment, name) {
|
|
524
|
-
const section = requireMarkdownSection(
|
|
535
|
+
function applyMarkdownFragment(response, raw, fragment, name, indexedFragmentContent) {
|
|
536
|
+
const section = indexedFragmentContent ?? requireMarkdownSection(raw, fragment, name).content;
|
|
525
537
|
if (response.template !== undefined)
|
|
526
538
|
response.template = section;
|
|
527
539
|
else if (response.prompt !== undefined)
|
|
@@ -533,6 +545,11 @@ function requireMarkdownSection(content, fragment, name) {
|
|
|
533
545
|
const section = extractSection(content, fragment);
|
|
534
546
|
if (section)
|
|
535
547
|
return section;
|
|
548
|
+
const indexed = projectMarkdownFragmentContent(content);
|
|
549
|
+
const safeFragment = indexed ? fragmentForSelector(indexed, fragment) : undefined;
|
|
550
|
+
if (safeFragment) {
|
|
551
|
+
return { content: safeFragment.text, startLine: safeFragment.startLine, endLine: safeFragment.endLine };
|
|
552
|
+
}
|
|
536
553
|
const available = markdownFragmentSlugs(content);
|
|
537
554
|
throw new NotFoundError(`Fragment "#${fragment}" not found in ${name}.` +
|
|
538
555
|
(available.length > 0 ? ` Available fragments: ${available.map((slug) => `#${slug}`).join(", ")}.` : ""));
|
|
@@ -81,7 +81,7 @@
|
|
|
81
81
|
*/
|
|
82
82
|
import fs from "node:fs";
|
|
83
83
|
import path from "node:path";
|
|
84
|
-
import { applyPostContributorFields, applyPreContributorFields, extractPackageMetadata, } from "../../../indexer/passes/metadata.js";
|
|
84
|
+
import { applyPostContributorFields, applyPreContributorFields, extractPackageMetadata, getMarkdownFragmentContent, hasMarkdownFragmentContent, setMarkdownFragmentContent, } from "../../../indexer/passes/metadata.js";
|
|
85
85
|
import { assetPathCandidatesForName, assetPathForName, deriveCanonicalAssetNameFromStashRoot, placementTypes, stashDirFor, stashDirNames, } from "../../asset/asset-placement.js";
|
|
86
86
|
import { parseFrontmatter } from "../../asset/frontmatter.js";
|
|
87
87
|
import { executionDefaultsFromFrontmatter, renderMarkdownExecutionSource } from "../execution-source.js";
|
|
@@ -244,6 +244,10 @@ function indexDocumentFromEntry(entry, base, rendererName) {
|
|
|
244
244
|
doc.lessonStrength = entry.lessonStrength;
|
|
245
245
|
if (entry.derivedFrom !== undefined)
|
|
246
246
|
doc.derivedFrom = entry.derivedFrom;
|
|
247
|
+
// Internal fragment substrate follows the recognition projection without
|
|
248
|
+
// becoming an IndexDocument field or serialized search payload.
|
|
249
|
+
if (hasMarkdownFragmentContent(entry))
|
|
250
|
+
setMarkdownFragmentContent(doc, getMarkdownFragmentContent(entry));
|
|
247
251
|
return doc;
|
|
248
252
|
}
|
|
249
253
|
function conceptIdForRecognizedType(root, filePath, type) {
|
|
@@ -0,0 +1,146 @@
|
|
|
1
|
+
// This Source Code Form is subject to the terms of the Mozilla Public
|
|
2
|
+
// License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
3
|
+
// file, You can obtain one at https://mozilla.org/MPL/2.0/.
|
|
4
|
+
/**
|
|
5
|
+
* Deterministic, addressable Markdown fragments.
|
|
6
|
+
*
|
|
7
|
+
* Input is the safe, line-preserving Markdown projection, never the raw file.
|
|
8
|
+
* Keeping that distinction here means search can only emit selectors that
|
|
9
|
+
* `show` can reproduce without disclosing fenced/commented/link-target bytes.
|
|
10
|
+
*/
|
|
11
|
+
import { createHash } from "node:crypto";
|
|
12
|
+
import { markdownHeadingSlug, parseMarkdownToc } from "./markdown.js";
|
|
13
|
+
export const MARKDOWN_FRAGMENT_MAX_CHARS = 1600;
|
|
14
|
+
export const MARKDOWN_FRAGMENT_PREFIX = "akm-fragment-";
|
|
15
|
+
function hash(text) {
|
|
16
|
+
return createHash("sha256").update(text).digest("hex");
|
|
17
|
+
}
|
|
18
|
+
function uniqueSlugs(body) {
|
|
19
|
+
const out = new Map();
|
|
20
|
+
const seen = new Set();
|
|
21
|
+
for (const heading of parseMarkdownToc(body).headings) {
|
|
22
|
+
const base = markdownHeadingSlug(heading.text);
|
|
23
|
+
if (!base)
|
|
24
|
+
continue;
|
|
25
|
+
let slug = base;
|
|
26
|
+
for (let suffix = 1; seen.has(slug); suffix++)
|
|
27
|
+
slug = `${base}-${suffix}`;
|
|
28
|
+
seen.add(slug);
|
|
29
|
+
out.set(heading.line, slug);
|
|
30
|
+
}
|
|
31
|
+
return out;
|
|
32
|
+
}
|
|
33
|
+
function textOf(lines) {
|
|
34
|
+
return lines.join("\n").trim();
|
|
35
|
+
}
|
|
36
|
+
/** Split a large sequence only at paragraph, then word, boundaries. */
|
|
37
|
+
function splitPiece(piece, maxChars) {
|
|
38
|
+
if (textOf(piece.lines).length <= maxChars)
|
|
39
|
+
return [piece];
|
|
40
|
+
const pieces = [];
|
|
41
|
+
let start = 0;
|
|
42
|
+
while (start < piece.lines.length) {
|
|
43
|
+
let end = start;
|
|
44
|
+
let chars = 0;
|
|
45
|
+
while (end < piece.lines.length) {
|
|
46
|
+
const next = piece.lines[end];
|
|
47
|
+
// Let the single-line word-window path below own an oversized first
|
|
48
|
+
// line. Without this guard it is consumed whole before that path can
|
|
49
|
+
// run, silently defeating the fragment bound for transcripts/logs.
|
|
50
|
+
if (end === start && next.length > maxChars)
|
|
51
|
+
break;
|
|
52
|
+
if (end > start && chars + next.length + 1 > maxChars)
|
|
53
|
+
break;
|
|
54
|
+
chars += next.length + (end > start ? 1 : 0);
|
|
55
|
+
end++;
|
|
56
|
+
}
|
|
57
|
+
// A single long authored line needs word windows, but its source range is
|
|
58
|
+
// intentionally the same line for every window.
|
|
59
|
+
if (end === start) {
|
|
60
|
+
const line = piece.lines[start];
|
|
61
|
+
let offset = 0;
|
|
62
|
+
while (offset < line.length) {
|
|
63
|
+
let cut = Math.min(offset + maxChars, line.length);
|
|
64
|
+
if (cut < line.length) {
|
|
65
|
+
const space = line.lastIndexOf(" ", cut);
|
|
66
|
+
if (space > offset + Math.floor(maxChars * 0.55))
|
|
67
|
+
cut = space;
|
|
68
|
+
}
|
|
69
|
+
pieces.push({ lines: [line.slice(offset, cut).trim()], startLine: piece.startLine + start });
|
|
70
|
+
offset = cut;
|
|
71
|
+
while (line[offset] === " ")
|
|
72
|
+
offset++;
|
|
73
|
+
}
|
|
74
|
+
start++;
|
|
75
|
+
continue;
|
|
76
|
+
}
|
|
77
|
+
// Prefer not to split a paragraph if a blank boundary fits before `end`.
|
|
78
|
+
let preferred = -1;
|
|
79
|
+
for (let i = start + 1; i < end; i++)
|
|
80
|
+
if (!piece.lines[i].trim())
|
|
81
|
+
preferred = i;
|
|
82
|
+
if (preferred > start)
|
|
83
|
+
end = preferred;
|
|
84
|
+
pieces.push({ lines: piece.lines.slice(start, end), startLine: piece.startLine + start });
|
|
85
|
+
start = end;
|
|
86
|
+
while (start < piece.lines.length && !piece.lines[start].trim())
|
|
87
|
+
start++;
|
|
88
|
+
}
|
|
89
|
+
return pieces.filter((candidate) => textOf(candidate.lines));
|
|
90
|
+
}
|
|
91
|
+
/**
|
|
92
|
+
* Heading sections first, then paragraph/word windows. `startLine`/`endLine`
|
|
93
|
+
* always refer to the authored file's line numbers because the projection
|
|
94
|
+
* preserves one line per source line (with excluded bytes blanked out).
|
|
95
|
+
*/
|
|
96
|
+
export function splitMarkdownFragmentStats(body, maxChars = MARKDOWN_FRAGMENT_MAX_CHARS) {
|
|
97
|
+
const lines = body.split(/\r?\n/);
|
|
98
|
+
const headings = parseMarkdownToc(body).headings;
|
|
99
|
+
const boundaries = [1, ...headings.map((heading) => heading.line), lines.length + 1]
|
|
100
|
+
.filter((line, index, all) => index === 0 || line !== all[index - 1])
|
|
101
|
+
.sort((left, right) => left - right);
|
|
102
|
+
const slugs = uniqueSlugs(body);
|
|
103
|
+
const pieces = [];
|
|
104
|
+
let sectionCount = 0;
|
|
105
|
+
for (let i = 0; i < boundaries.length - 1; i++) {
|
|
106
|
+
const startLine = boundaries[i];
|
|
107
|
+
const end = boundaries[i + 1] - 1;
|
|
108
|
+
const section = { lines: lines.slice(startLine - 1, end), startLine, headingSlug: slugs.get(startLine) };
|
|
109
|
+
if (textOf(section.lines)) {
|
|
110
|
+
sectionCount++;
|
|
111
|
+
pieces.push(...splitPiece(section, maxChars));
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
// Whether a heading section survived as one fragment is a property of the
|
|
115
|
+
// complete piece set. Count once before materialization instead of scanning
|
|
116
|
+
// every piece for every fragment (which made many headed documents O(N²)).
|
|
117
|
+
const piecesPerHeading = new Map();
|
|
118
|
+
for (const piece of pieces) {
|
|
119
|
+
if (piece.headingSlug)
|
|
120
|
+
piecesPerHeading.set(piece.headingSlug, (piecesPerHeading.get(piece.headingSlug) ?? 0) + 1);
|
|
121
|
+
}
|
|
122
|
+
const fragments = pieces.map((piece, ordinal) => {
|
|
123
|
+
const text = textOf(piece.lines);
|
|
124
|
+
const contentLines = piece.lines.map((line, index) => ({ line, index })).filter(({ line }) => line.trim());
|
|
125
|
+
const first = contentLines[0]?.index ?? 0;
|
|
126
|
+
const last = contentLines.at(-1)?.index ?? 0;
|
|
127
|
+
const digest = hash(text);
|
|
128
|
+
const unsplitHeading = piece.headingSlug && piecesPerHeading.get(piece.headingSlug) === 1;
|
|
129
|
+
return {
|
|
130
|
+
fragmentId: `${MARKDOWN_FRAGMENT_PREFIX}${ordinal + 1}-${digest.slice(0, 12)}`,
|
|
131
|
+
ordinal,
|
|
132
|
+
startLine: piece.startLine + first,
|
|
133
|
+
endLine: piece.startLine + last,
|
|
134
|
+
...(unsplitHeading ? { headingSlug: piece.headingSlug } : {}),
|
|
135
|
+
text,
|
|
136
|
+
hash: digest,
|
|
137
|
+
};
|
|
138
|
+
});
|
|
139
|
+
return { fragments, hardSplitCount: Math.max(0, pieces.length - sectionCount) };
|
|
140
|
+
}
|
|
141
|
+
export function splitMarkdownFragments(body, maxChars = MARKDOWN_FRAGMENT_MAX_CHARS) {
|
|
142
|
+
return splitMarkdownFragmentStats(body, maxChars).fragments;
|
|
143
|
+
}
|
|
144
|
+
export function fragmentForSelector(body, selector) {
|
|
145
|
+
return splitMarkdownFragments(body).find((fragment) => fragment.fragmentId === selector || fragment.headingSlug === selector);
|
|
146
|
+
}
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
// This Source Code Form is subject to the terms of the Mozilla Public
|
|
2
|
+
// License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
3
|
+
// file, You can obtain one at https://mozilla.org/MPL/2.0/.
|
|
4
|
+
/** Dependency-safe fixed FTS5 calibration shared by storage and ranking. */
|
|
5
|
+
const SCORE_FLOOR = 0.3;
|
|
6
|
+
const PARENT_CEILING = 0.8;
|
|
7
|
+
// Fragment BM25 is from a distinct FTS population. Its separately calibrated
|
|
8
|
+
// ceiling is an explicit evidence policy, not a cross-table comparability claim:
|
|
9
|
+
// when a body fragment independently proves the query, retain that actionable
|
|
10
|
+
// selector beside the same parent's length-penalized whole-body row. Metadata
|
|
11
|
+
// and cross-fragment conjunctions cannot enter this population.
|
|
12
|
+
const FRAGMENT_CEILING = 0.8;
|
|
13
|
+
const BM25_REFERENCE = 0.000001;
|
|
14
|
+
const LOG_SHAPE = 3;
|
|
15
|
+
export function stableFtsScore(bm25Score, population = "parent") {
|
|
16
|
+
const ceiling = population === "fragment" ? FRAGMENT_CEILING : PARENT_CEILING;
|
|
17
|
+
if (bm25Score === Number.NEGATIVE_INFINITY)
|
|
18
|
+
return ceiling;
|
|
19
|
+
if (!Number.isFinite(bm25Score) || bm25Score >= 0)
|
|
20
|
+
return SCORE_FLOOR;
|
|
21
|
+
const scaled = Math.log1p(-bm25Score / BM25_REFERENCE);
|
|
22
|
+
if (!Number.isFinite(scaled))
|
|
23
|
+
return ceiling;
|
|
24
|
+
return SCORE_FLOOR + (ceiling - SCORE_FLOOR) * (scaled / (scaled + LOG_SHAPE));
|
|
25
|
+
}
|
|
@@ -51,20 +51,41 @@ function buildWorkflowAction(ref) {
|
|
|
51
51
|
* see the file header.
|
|
52
52
|
*/
|
|
53
53
|
export const TYPE_PRESENTATION = {
|
|
54
|
-
skill: {
|
|
54
|
+
skill: {
|
|
55
|
+
label: "Skill",
|
|
56
|
+
renderer: "skill-md",
|
|
57
|
+
action: (ref) => `akm show ${ref} -> follow the instructions`,
|
|
58
|
+
fragmentRef: false,
|
|
59
|
+
},
|
|
55
60
|
command: {
|
|
56
61
|
label: "Command",
|
|
57
62
|
renderer: "command-md",
|
|
58
63
|
action: (ref) => `akm show ${ref} -> fill placeholders and dispatch`,
|
|
64
|
+
fragmentRef: false,
|
|
65
|
+
},
|
|
66
|
+
agent: {
|
|
67
|
+
label: "Agent",
|
|
68
|
+
renderer: "agent-md",
|
|
69
|
+
action: (ref) => `akm show ${ref} -> dispatch with full prompt`,
|
|
70
|
+
fragmentRef: false,
|
|
59
71
|
},
|
|
60
|
-
agent: { label: "Agent", renderer: "agent-md", action: (ref) => `akm show ${ref} -> dispatch with full prompt` },
|
|
61
72
|
knowledge: {
|
|
62
73
|
label: "Knowledge",
|
|
63
74
|
renderer: "knowledge-md",
|
|
64
75
|
action: (ref) => `akm show ${ref} -> read reference material`,
|
|
65
76
|
},
|
|
66
|
-
workflow: {
|
|
67
|
-
|
|
77
|
+
workflow: {
|
|
78
|
+
label: "Workflow",
|
|
79
|
+
renderer: "workflow-md",
|
|
80
|
+
action: (ref) => buildWorkflowAction(ref),
|
|
81
|
+
fragmentRef: false,
|
|
82
|
+
},
|
|
83
|
+
script: {
|
|
84
|
+
label: "Script",
|
|
85
|
+
renderer: "script-source",
|
|
86
|
+
action: (ref) => `akm show ${ref} -> execute the run command`,
|
|
87
|
+
fragmentRef: false,
|
|
88
|
+
},
|
|
68
89
|
memory: { label: "Memory", renderer: "memory-md", action: (ref) => `akm show ${ref} -> recall context` },
|
|
69
90
|
env: {
|
|
70
91
|
label: "Env",
|
|
@@ -85,6 +106,7 @@ export const TYPE_PRESENTATION = {
|
|
|
85
106
|
label: "Task",
|
|
86
107
|
renderer: "task-yaml",
|
|
87
108
|
action: (ref) => `akm show ${ref} -> inspect; akm task run <id> -> run now; edit the file + akm task sync -> unschedule`,
|
|
109
|
+
fragmentRef: false,
|
|
88
110
|
},
|
|
89
111
|
session: {
|
|
90
112
|
label: "Session",
|
|
@@ -105,6 +127,7 @@ export const TYPE_PRESENTATION = {
|
|
|
105
127
|
label: "Instruction",
|
|
106
128
|
renderer: "knowledge-md",
|
|
107
129
|
action: (ref) => `akm show ${ref} -> read the project instructions`,
|
|
130
|
+
fragmentRef: false,
|
|
108
131
|
},
|
|
109
132
|
};
|
|
110
133
|
/** Generic fallback for a type outside {@link KNOWN_TYPES} — never `undefined`, never a throw. */
|
|
@@ -120,6 +143,15 @@ export function presentationFor(type) {
|
|
|
120
143
|
return TYPE_PRESENTATION[type];
|
|
121
144
|
return DEFAULT_PRESENTATION;
|
|
122
145
|
}
|
|
146
|
+
/**
|
|
147
|
+
* A missing declaration means a read-like reference or foreign type can expose
|
|
148
|
+
* a safe selector. Built-in executable and instruction-bearing types opt out
|
|
149
|
+
* explicitly above: their search match can be a fragment, but the advertised
|
|
150
|
+
* action needs the parent canonical ref and its complete context.
|
|
151
|
+
*/
|
|
152
|
+
export function allowsFragmentRef(type) {
|
|
153
|
+
return presentationFor(type).fragmentRef !== false;
|
|
154
|
+
}
|
|
123
155
|
export const defaultRendererRegistry = {
|
|
124
156
|
rendererNameFor(type) {
|
|
125
157
|
return presentationFor(type).renderer;
|
|
@@ -31,6 +31,7 @@ import { closeDatabase, openExistingDatabase } from "../storage/repositories/ind
|
|
|
31
31
|
import { deleteEntriesByIds, getEntryCount, upsertEntry } from "../storage/repositories/index-entries-repository.js";
|
|
32
32
|
import { deriveEntryProvenance, deriveInstallations } from "./installations.js";
|
|
33
33
|
import { generateEmbeddingsForDb, publishTargetedEmbeddingMeta } from "./materialize-embeddings.js";
|
|
34
|
+
import { getMarkdownFragmentContent, hasMarkdownFragmentContent, setMarkdownFragmentContent, } from "./passes/metadata.js";
|
|
34
35
|
import { drainDirDocuments } from "./scan/drain-dir.js";
|
|
35
36
|
import { buildSearchText } from "./search/search-fields.js";
|
|
36
37
|
import { buildFileContext } from "./walk/file-context.js";
|
|
@@ -158,6 +159,9 @@ export async function indexWrittenAssets(stashDir, filePaths, options = {}) {
|
|
|
158
159
|
let entryWithSize = entry;
|
|
159
160
|
try {
|
|
160
161
|
entryWithSize = { ...entry, fileSize: fs.statSync(file).size };
|
|
162
|
+
if (hasMarkdownFragmentContent(entry)) {
|
|
163
|
+
setMarkdownFragmentContent(entryWithSize, getMarkdownFragmentContent(entry));
|
|
164
|
+
}
|
|
161
165
|
}
|
|
162
166
|
catch {
|
|
163
167
|
// stat raced a delete — index without the size, like the full walk does.
|
package/dist/indexer/indexer.js
CHANGED
|
@@ -31,7 +31,7 @@ import { deriveEntryProvenance, deriveInstallations } from "./installations.js";
|
|
|
31
31
|
import { indexedPathMatchesOwner, resolveAdapterConceptOwner, } from "./lookup/adapter-concept-owner.js";
|
|
32
32
|
import { generateEmbeddingsForDb } from "./materialize-embeddings.js";
|
|
33
33
|
import { canUseIncrementalSkip, computeDirFingerprint, getCachedDirState, getDirIndexState, inferZeroRowReason, } from "./passes/dir-staleness.js";
|
|
34
|
-
import { isEnrichmentComplete, isWorkflowSkipWarning } from "./passes/metadata.js";
|
|
34
|
+
import { getMarkdownFragmentContent, hasMarkdownFragmentContent, isEnrichmentComplete, isWorkflowSkipWarning, setMarkdownFragmentContent, } from "./passes/metadata.js";
|
|
35
35
|
import { drainDirDocuments } from "./scan/drain-dir.js";
|
|
36
36
|
import { buildSearchText } from "./search/search-fields.js";
|
|
37
37
|
import { purgeOldUsageEvents } from "./usage/usage-events.js";
|
|
@@ -1504,7 +1504,10 @@ export function createEnrichmentDeadline(timeoutMs, totalEntries) {
|
|
|
1504
1504
|
// ── Helpers ─────────────────────────────────────────────────────────────────
|
|
1505
1505
|
function attachFileSize(entry, entryPath) {
|
|
1506
1506
|
try {
|
|
1507
|
-
|
|
1507
|
+
const sized = { ...entry, fileSize: fs.statSync(entryPath).size };
|
|
1508
|
+
if (hasMarkdownFragmentContent(entry))
|
|
1509
|
+
setMarkdownFragmentContent(sized, getMarkdownFragmentContent(entry));
|
|
1510
|
+
return sized;
|
|
1508
1511
|
}
|
|
1509
1512
|
catch {
|
|
1510
1513
|
return entry;
|
|
@@ -1179,6 +1179,67 @@ export function projectMarkdownContent(body, truncationInfo) {
|
|
|
1179
1179
|
truncationInfo.truncated = text.length > MARKDOWN_CONTENT_MAX_CHARS;
|
|
1180
1180
|
return truncateUnicodeSafe(text, MARKDOWN_CONTENT_MAX_CHARS);
|
|
1181
1181
|
}
|
|
1182
|
+
// Fragment text is intentionally not an IndexDocument field. IndexDocument is
|
|
1183
|
+
// an adapter/search payload boundary; this is an internal, derived index input
|
|
1184
|
+
// that is persisted separately by the entries repository for deterministic FTS
|
|
1185
|
+
// rebuilds. The WeakMap follows the already-read document through recognition
|
|
1186
|
+
// without making safe body bytes observable through public payloads.
|
|
1187
|
+
const markdownFragmentContentByEntry = new WeakMap();
|
|
1188
|
+
const markdownFragmentProjectionEntries = new WeakSet();
|
|
1189
|
+
export function setMarkdownFragmentContent(entry, content) {
|
|
1190
|
+
markdownFragmentProjectionEntries.add(entry);
|
|
1191
|
+
if (content)
|
|
1192
|
+
markdownFragmentContentByEntry.set(entry, content);
|
|
1193
|
+
}
|
|
1194
|
+
export function getMarkdownFragmentContent(entry) {
|
|
1195
|
+
return markdownFragmentContentByEntry.get(entry);
|
|
1196
|
+
}
|
|
1197
|
+
export function hasMarkdownFragmentContent(entry) {
|
|
1198
|
+
return markdownFragmentProjectionEntries.has(entry);
|
|
1199
|
+
}
|
|
1200
|
+
/**
|
|
1201
|
+
* Produce a safe, structure-preserving projection for fragment indexing.
|
|
1202
|
+
* Excluded source lines are retained as blank lines so fragment locations map
|
|
1203
|
+
* exactly to authored line numbers. This must be called from both indexing and
|
|
1204
|
+
* `show`; it deliberately never performs a storage-layer file reread.
|
|
1205
|
+
*/
|
|
1206
|
+
export function projectMarkdownFragmentContent(raw) {
|
|
1207
|
+
const lines = raw.split(/\r?\n/);
|
|
1208
|
+
const parsed = parseFrontmatter(raw);
|
|
1209
|
+
const start = parsed.frontmatter ? parsed.bodyStartLine - 1 : 0;
|
|
1210
|
+
const projected = lines.map(() => "");
|
|
1211
|
+
let fence;
|
|
1212
|
+
const htmlComment = { inComment: false };
|
|
1213
|
+
for (let index = start; index < lines.length; index++) {
|
|
1214
|
+
const rawLine = lines[index];
|
|
1215
|
+
if (fence) {
|
|
1216
|
+
if (isMarkdownFenceClosing(rawLine, fence))
|
|
1217
|
+
fence = undefined;
|
|
1218
|
+
continue;
|
|
1219
|
+
}
|
|
1220
|
+
if (!htmlComment.inComment) {
|
|
1221
|
+
const opening = parseMarkdownFenceOpening(rawLine);
|
|
1222
|
+
if (opening) {
|
|
1223
|
+
fence = opening;
|
|
1224
|
+
continue;
|
|
1225
|
+
}
|
|
1226
|
+
}
|
|
1227
|
+
let safe = stripMarkdownHtmlComments(rawLine, htmlComment);
|
|
1228
|
+
const opening = parseMarkdownFenceOpening(safe.trim());
|
|
1229
|
+
if (opening) {
|
|
1230
|
+
fence = opening;
|
|
1231
|
+
continue;
|
|
1232
|
+
}
|
|
1233
|
+
// Reference link destinations and standalone HTML are not retrieval
|
|
1234
|
+
// evidence and can contain credential-bearing URLs.
|
|
1235
|
+
if (/^\s*\[[^\]]+\]:\s*\S+/.test(safe) || /^\s*<[^>]+>\s*$/.test(safe))
|
|
1236
|
+
continue;
|
|
1237
|
+
safe = stripMarkdownLinkDestinations(safe).replace(/<[^>]+>/g, " ");
|
|
1238
|
+
projected[index] = safe.replace(/[ \t]+$/g, "");
|
|
1239
|
+
}
|
|
1240
|
+
const text = projected.join("\n");
|
|
1241
|
+
return text.trim() ? text : undefined;
|
|
1242
|
+
}
|
|
1182
1243
|
// ── Metadata Generation ─────────────────────────────────────────────────────
|
|
1183
1244
|
/**
|
|
1184
1245
|
* Priorities 1-2 of the metadata pipeline — package.json (P1), `.md`
|
|
@@ -1220,7 +1281,9 @@ export function applyPreContributorFields(entry, file, ctx, pkgMeta) {
|
|
|
1220
1281
|
applyProvenanceFrontmatter(entry, parsed.data);
|
|
1221
1282
|
// Native Markdown has one bounded low-weight body projection. Sensitive
|
|
1222
1283
|
// types and raw session/checkpoint material never cross this boundary.
|
|
1223
|
-
|
|
1284
|
+
const safeForFragments = entry.type !== "env" && entry.type !== "session" && !hasSessionMemoryMarker(parsed.data, parsed.content);
|
|
1285
|
+
setMarkdownFragmentContent(entry, safeForFragments ? projectMarkdownFragmentContent(content) : undefined);
|
|
1286
|
+
if (safeForFragments) {
|
|
1224
1287
|
const truncationInfo = { truncated: false };
|
|
1225
1288
|
const contentProjection = projectMarkdownContent(parsed.content, truncationInfo);
|
|
1226
1289
|
if (contentProjection) {
|