akm-cli 0.9.12 → 0.9.14-beta.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +100 -0
- package/dist/assets/workflows/workflow-template.md +4 -0
- package/dist/commands/improve/eligibility.js +27 -15
- package/dist/commands/improve/improve.js +1 -0
- package/dist/commands/lint/base-linter.js +10 -0
- package/dist/commands/proposal/drain.js +48 -6
- package/dist/commands/proposal/proposal-cli.js +1 -0
- package/dist/commands/read/curate.js +3 -2
- package/dist/commands/read/show.js +26 -9
- package/dist/core/adapter/adapters/akm-adapter.js +5 -1
- package/dist/core/asset/markdown-fragments.js +146 -0
- package/dist/core/config/config-walker.js +7 -3
- package/dist/core/config/config.js +21 -12
- package/dist/core/config/schema/primitives.js +8 -2
- package/dist/core/errors.js +2 -0
- package/dist/core/lexical-score.js +25 -0
- package/dist/core/type-presentation.js +36 -4
- package/dist/indexer/index-written-assets.js +4 -0
- package/dist/indexer/indexer.js +5 -2
- package/dist/indexer/passes/metadata.js +64 -1
- package/dist/indexer/scan/doc-to-entry.js +3 -0
- package/dist/indexer/scan/drain-dir.js +33 -22
- package/dist/indexer/search/db-search.js +72 -14
- package/dist/indexer/search/name-match.js +35 -0
- package/dist/indexer/search/ranking-contributors.js +15 -12
- package/dist/indexer/search/ranking.js +42 -18
- package/dist/indexer/usage/show-usage.js +14 -2
- package/dist/llm/client.js +12 -8
- package/dist/llm/embedders/remote.js +3 -2
- package/dist/llm/graph-extract.js +18 -67
- package/dist/output/shapes.js +46 -1
- package/dist/output/text/proposal-format.js +5 -0
- package/dist/scripts/akm-migrate-node.js +648 -253
- package/dist/scripts/akm-migrate.js +648 -253
- package/dist/storage/repositories/index-connection.js +23 -8
- package/dist/storage/repositories/index-entries-repository.js +3 -2
- package/dist/storage/repositories/index-entry-schema.js +43 -3
- package/dist/storage/repositories/index-fts-repository.js +160 -14
- package/dist/storage/repositories/index-schema.js +8 -18
- package/dist/storage/repositories/workflow-runs-repository.js +118 -10
- package/dist/workflows/exec/run-workflow.js +1 -1
- package/dist/workflows/exec/step-work.js +41 -0
- package/dist/workflows/parser.js +1 -1
- package/dist/workflows/runtime/runs.js +29 -5
- package/docs/migration/release-notes/0.9.14.md +26 -0
- package/docs/migration/release-notes/README.md +2 -0
- package/docs/reference/cli.md +18 -0
- package/package.json +1 -1
|
@@ -27,6 +27,7 @@ import { hasRegistryUrlCredentials, REGISTRY_CREDENTIALS_UNSUPPORTED } from "../
|
|
|
27
27
|
import { warnOnce } from "../warn.js";
|
|
28
28
|
import { AkmConfigBaseSchema, EngineConfigSchema, listTopLevelConfigKeys } from "./config-schema.js";
|
|
29
29
|
import { deepMergeConfig } from "./deep-merge.js";
|
|
30
|
+
import { isApiKeyReference } from "./schema/primitives.js";
|
|
30
31
|
/**
|
|
31
32
|
* Parse a dotted path into segments. Empty segments are rejected. Bracket
|
|
32
33
|
* notation (e.g. `sources[0]`) is NOT supported — arrays are set as JSON.
|
|
@@ -214,9 +215,12 @@ export function configSet(config, dotted, raw) {
|
|
|
214
215
|
const parsed = path[0] === "engines" && path.length === 2
|
|
215
216
|
? EngineConfigSchema.safeParse(value)
|
|
216
217
|
: symbolicApiKey
|
|
217
|
-
?
|
|
218
|
+
? isApiKeyReference(raw)
|
|
218
219
|
? { success: true, data: value }
|
|
219
|
-
: {
|
|
220
|
+
: {
|
|
221
|
+
success: false,
|
|
222
|
+
error: { issues: [{ path: [], message: "apiKey must be $VAR, ${VAR}, or secret://<name>" }] },
|
|
223
|
+
}
|
|
220
224
|
: isUnknownKey
|
|
221
225
|
? { success: true, data: value }
|
|
222
226
|
: schema.safeParse(candidate);
|
|
@@ -297,7 +301,7 @@ function rejectLiteralApiKeyInWholeObjectSet(path, raw, dotted) {
|
|
|
297
301
|
}
|
|
298
302
|
if (!isRecord(parsed) || typeof parsed.apiKey !== "string")
|
|
299
303
|
return;
|
|
300
|
-
if (
|
|
304
|
+
if (isApiKeyReference(parsed.apiKey))
|
|
301
305
|
return;
|
|
302
306
|
throw new UsageError(`apiKey cannot be persisted in config; export ${recipeForApiKey([...path, "apiKey"], `${dotted}.apiKey`)} instead. (key: ${dotted}.apiKey)`, "INVALID_FLAG_VALUE", "Storing API keys in config.json leaks them through backups, logs, and version control. " +
|
|
303
307
|
"Use the corresponding environment variable. AKM reads it at request time.");
|
|
@@ -12,6 +12,7 @@ import { bundlesToSourceEntries } from "./config-sources.js";
|
|
|
12
12
|
import { upgradeConfigVersion } from "./config-version-shim.js";
|
|
13
13
|
import { deepMergeConfig } from "./deep-merge.js";
|
|
14
14
|
import { migrateLegacySourceShape } from "./legacy-source-shape-shim.js";
|
|
15
|
+
import { isApiKeyReference, SECRET_STORE_REFERENCE_PATTERN } from "./schema/primitives.js";
|
|
15
16
|
export { stripJsonComments } from "./config-io.js";
|
|
16
17
|
import { getConfigPath } from "../paths.js";
|
|
17
18
|
import { warn, warnOnce } from "../warn.js";
|
|
@@ -299,7 +300,7 @@ export function sanitizeConfigForWrite(config) {
|
|
|
299
300
|
const stripped = [];
|
|
300
301
|
if (config.embedding?.apiKey !== undefined) {
|
|
301
302
|
const apiKey = config.embedding.apiKey;
|
|
302
|
-
if (
|
|
303
|
+
if (isApiKeyReference(apiKey)) {
|
|
303
304
|
// Preserve reference verbatim — not a secret.
|
|
304
305
|
sanitized.embedding = { ...config.embedding };
|
|
305
306
|
}
|
|
@@ -316,7 +317,7 @@ export function sanitizeConfigForWrite(config) {
|
|
|
316
317
|
if (config.engines) {
|
|
317
318
|
const engines = {};
|
|
318
319
|
for (const [name, engine] of Object.entries(config.engines)) {
|
|
319
|
-
if (engine.kind !== "llm" || engine.apiKey === undefined ||
|
|
320
|
+
if (engine.kind !== "llm" || engine.apiKey === undefined || isApiKeyReference(engine.apiKey)) {
|
|
320
321
|
engines[name] = { ...engine };
|
|
321
322
|
continue;
|
|
322
323
|
}
|
|
@@ -345,19 +346,15 @@ export function sanitizeConfigForWrite(config) {
|
|
|
345
346
|
}
|
|
346
347
|
return sanitized;
|
|
347
348
|
}
|
|
348
|
-
/** Matches the only 0.9 symbolic secret forms: `${VAR}` or `$VAR`. */
|
|
349
|
-
function isEnvReference(value) {
|
|
350
|
-
return /^\$\{[A-Za-z_][A-Za-z0-9_]*\}$|^\$[A-Za-z_][A-Za-z0-9_]*$/.test(value);
|
|
351
|
-
}
|
|
352
349
|
export function updateConfig(partial) {
|
|
353
350
|
return mutateConfig((current) => deepMergeConfig(current, partial)).config;
|
|
354
351
|
}
|
|
355
|
-
// ── Helpers ─────────────────────────────────────────────────────────────────
|
|
356
352
|
/**
|
|
357
|
-
* Resolve a single secret value
|
|
358
|
-
*
|
|
359
|
-
* authorization-header consumption sites (LLM client, embedder,
|
|
360
|
-
* runner) — NOT on the load path. Non-string inputs pass through
|
|
353
|
+
* Resolve a single secret value: expand `${VAR}` / `$VAR` against
|
|
354
|
+
* `process.env`, or look up `secret://<name>` via `resolveFromStore`. Use this
|
|
355
|
+
* at apiKey / authorization-header consumption sites (LLM client, embedder,
|
|
356
|
+
* agent SDK runner) — NOT on the load path. Non-string inputs pass through
|
|
357
|
+
* unchanged.
|
|
361
358
|
*
|
|
362
359
|
* Returns the input unchanged when no substitution markers are present, so
|
|
363
360
|
* literal API key strings (already-resolved secrets) are zero-cost.
|
|
@@ -365,12 +362,24 @@ export function updateConfig(partial) {
|
|
|
365
362
|
* Other config string values (URLs, endpoints, model names, prompts) are
|
|
366
363
|
* preserved verbatim on read — only fields explicitly routed through this
|
|
367
364
|
* helper are expanded.
|
|
365
|
+
*
|
|
366
|
+
* A `secret://<name>` value that fails to resolve throws `ConfigError`
|
|
367
|
+
* (naming the ref, never the secret) rather than silently sending an unusable
|
|
368
|
+
* credential.
|
|
368
369
|
*/
|
|
369
|
-
export function resolveSecret(value) {
|
|
370
|
+
export function resolveSecret(value, resolveFromStore) {
|
|
370
371
|
if (value === undefined)
|
|
371
372
|
return undefined;
|
|
372
373
|
if (typeof value !== "string")
|
|
373
374
|
return value;
|
|
375
|
+
const storeRef = SECRET_STORE_REFERENCE_PATTERN.exec(value)?.[1];
|
|
376
|
+
if (storeRef !== undefined) {
|
|
377
|
+
const resolved = resolveFromStore?.(storeRef) ?? null;
|
|
378
|
+
if (resolved === null) {
|
|
379
|
+
throw new ConfigError(`Secret store reference "${value}" did not resolve to a stored value.`, "SECRET_REFERENCE_UNRESOLVED");
|
|
380
|
+
}
|
|
381
|
+
return resolved;
|
|
382
|
+
}
|
|
374
383
|
if (!value.includes("$"))
|
|
375
384
|
return value;
|
|
376
385
|
return value.replace(/\$\{([A-Za-z_][A-Za-z0-9_]*)\}|\$([A-Za-z_][A-Za-z0-9_]*)/g, (_match, braced, bare) => {
|
|
@@ -28,15 +28,21 @@ export const httpUrl = z.string().refine((v) => v.startsWith("http://") || v.sta
|
|
|
28
28
|
});
|
|
29
29
|
const ENGINE_NAME_PATTERN = new RegExp(ENGINE_NAME_PATTERN_SOURCE);
|
|
30
30
|
export const ENV_REFERENCE_PATTERN = /^\$[A-Za-z_][A-Za-z0-9_]*$|^\$\{[A-Za-z_][A-Za-z0-9_]*\}$/;
|
|
31
|
+
/** `secret://<name>` — an apiKey reference into the akm secret store, resolved via `resolveSecretFromStore`. */
|
|
32
|
+
export const SECRET_STORE_REFERENCE_PATTERN = /^secret:\/\/(.+)$/;
|
|
31
33
|
export const engineName = z
|
|
32
34
|
.string()
|
|
33
35
|
.max(63)
|
|
34
36
|
.regex(ENGINE_NAME_PATTERN, "names must be lowercase kebab-case and must not begin with reserved akm-");
|
|
37
|
+
/** The two symbolic apiKey forms akm accepts: an env-var reference or a secret-store reference. Never matches a literal key. */
|
|
38
|
+
export function isApiKeyReference(value) {
|
|
39
|
+
return ENV_REFERENCE_PATTERN.test(value) || SECRET_STORE_REFERENCE_PATTERN.test(value);
|
|
40
|
+
}
|
|
35
41
|
export function symbolicOrWarnApiKey(label) {
|
|
36
42
|
return z.string().superRefine((value) => {
|
|
37
|
-
if (
|
|
43
|
+
if (isApiKeyReference(value))
|
|
38
44
|
return;
|
|
39
|
-
warnOnce(`config:literal-api-key:${label}`, `A ${label} in config.json is a literal API key, not a $VAR/\${VAR} reference; using it as configured. Prefer \`akm config set ...apiKey '$VAR'\` (with the corresponding env var set) — see docs/reference/data-and-telemetry.md.`);
|
|
45
|
+
warnOnce(`config:literal-api-key:${label}`, `A ${label} in config.json is a literal API key, not a $VAR/\${VAR}/secret:// reference; using it as configured. Prefer \`akm config set ...apiKey '$VAR'\` (with the corresponding env var set) or \`secret://<name>\` (with \`akm secret set <name> ...\`) — see docs/reference/data-and-telemetry.md.`);
|
|
40
46
|
});
|
|
41
47
|
}
|
|
42
48
|
export const chatCompletionsEndpoint = z.string().superRefine((value, ctx) => {
|
package/dist/core/errors.js
CHANGED
|
@@ -17,6 +17,7 @@ const CONFIG_HINTS = {
|
|
|
17
17
|
UNSAFE_STASH_DIR: "Choose a path inside your home directory (e.g. ~/akm) or another empty workspace. The bundle directory cannot be the filesystem root, your home directory itself, or a sensitive system path like /etc, /var, ~/.config, or ~/.ssh.",
|
|
18
18
|
UNKNOWN_IMPROVE_STRATEGY: "Pass one of the listed strategy names to `--strategy`, or define it under `improve.strategies`. Names are case-sensitive.",
|
|
19
19
|
EXECUTION_NOT_AUTHORIZED: "Change the selected tools or update the machine/user execution policy, then retry.",
|
|
20
|
+
SECRET_REFERENCE_UNRESOLVED: "Check the secret exists (`akm secret list`) and the name after `secret://` matches, or run `akm secret set <name> <value>` to store it.",
|
|
20
21
|
};
|
|
21
22
|
// Code-review finding: COMPOSITION_INVALID covers several unrelated causes
|
|
22
23
|
// (a rejected with:, a multi-job source, a composition cycle/depth/size
|
|
@@ -76,6 +77,7 @@ const USAGE_HINTS = {
|
|
|
76
77
|
WORKFLOW_IR_VERSION_UNSUPPORTED: "Abandon the run with `akm workflow abandon <id>`, then start it again from the workflow source — a frozen plan this akm cannot execute is not re-executable in place.",
|
|
77
78
|
// P3b (docs/plans/specs/p3b-child-executor.md §4.3).
|
|
78
79
|
WORKFLOW_OUTPUT_INVALID: "Check each `outputs:` entry's `from:` against the step artifact it names, and its `schema:` against the value that step actually promotes.",
|
|
80
|
+
RUN_LEASE_HELD: "Wait for the named engine invocation to finish or for the lease to expire, then retry. `akm workflow status <id>` shows the current lease.",
|
|
79
81
|
};
|
|
80
82
|
/** Default hint for each NotFoundError code. */
|
|
81
83
|
const NOT_FOUND_HINTS = {
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
// This Source Code Form is subject to the terms of the Mozilla Public
|
|
2
|
+
// License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
3
|
+
// file, You can obtain one at https://mozilla.org/MPL/2.0/.
|
|
4
|
+
/** Dependency-safe fixed FTS5 calibration shared by storage and ranking. */
|
|
5
|
+
const SCORE_FLOOR = 0.3;
|
|
6
|
+
const PARENT_CEILING = 0.8;
|
|
7
|
+
// Fragment BM25 is from a distinct FTS population. Its separately calibrated
|
|
8
|
+
// ceiling is an explicit evidence policy, not a cross-table comparability claim:
|
|
9
|
+
// when a body fragment independently proves the query, retain that actionable
|
|
10
|
+
// selector beside the same parent's length-penalized whole-body row. Metadata
|
|
11
|
+
// and cross-fragment conjunctions cannot enter this population.
|
|
12
|
+
const FRAGMENT_CEILING = 0.8;
|
|
13
|
+
const BM25_REFERENCE = 0.000001;
|
|
14
|
+
const LOG_SHAPE = 3;
|
|
15
|
+
export function stableFtsScore(bm25Score, population = "parent") {
|
|
16
|
+
const ceiling = population === "fragment" ? FRAGMENT_CEILING : PARENT_CEILING;
|
|
17
|
+
if (bm25Score === Number.NEGATIVE_INFINITY)
|
|
18
|
+
return ceiling;
|
|
19
|
+
if (!Number.isFinite(bm25Score) || bm25Score >= 0)
|
|
20
|
+
return SCORE_FLOOR;
|
|
21
|
+
const scaled = Math.log1p(-bm25Score / BM25_REFERENCE);
|
|
22
|
+
if (!Number.isFinite(scaled))
|
|
23
|
+
return ceiling;
|
|
24
|
+
return SCORE_FLOOR + (ceiling - SCORE_FLOOR) * (scaled / (scaled + LOG_SHAPE));
|
|
25
|
+
}
|
|
@@ -51,20 +51,41 @@ function buildWorkflowAction(ref) {
|
|
|
51
51
|
* see the file header.
|
|
52
52
|
*/
|
|
53
53
|
export const TYPE_PRESENTATION = {
|
|
54
|
-
skill: {
|
|
54
|
+
skill: {
|
|
55
|
+
label: "Skill",
|
|
56
|
+
renderer: "skill-md",
|
|
57
|
+
action: (ref) => `akm show ${ref} -> follow the instructions`,
|
|
58
|
+
fragmentRef: false,
|
|
59
|
+
},
|
|
55
60
|
command: {
|
|
56
61
|
label: "Command",
|
|
57
62
|
renderer: "command-md",
|
|
58
63
|
action: (ref) => `akm show ${ref} -> fill placeholders and dispatch`,
|
|
64
|
+
fragmentRef: false,
|
|
65
|
+
},
|
|
66
|
+
agent: {
|
|
67
|
+
label: "Agent",
|
|
68
|
+
renderer: "agent-md",
|
|
69
|
+
action: (ref) => `akm show ${ref} -> dispatch with full prompt`,
|
|
70
|
+
fragmentRef: false,
|
|
59
71
|
},
|
|
60
|
-
agent: { label: "Agent", renderer: "agent-md", action: (ref) => `akm show ${ref} -> dispatch with full prompt` },
|
|
61
72
|
knowledge: {
|
|
62
73
|
label: "Knowledge",
|
|
63
74
|
renderer: "knowledge-md",
|
|
64
75
|
action: (ref) => `akm show ${ref} -> read reference material`,
|
|
65
76
|
},
|
|
66
|
-
workflow: {
|
|
67
|
-
|
|
77
|
+
workflow: {
|
|
78
|
+
label: "Workflow",
|
|
79
|
+
renderer: "workflow-md",
|
|
80
|
+
action: (ref) => buildWorkflowAction(ref),
|
|
81
|
+
fragmentRef: false,
|
|
82
|
+
},
|
|
83
|
+
script: {
|
|
84
|
+
label: "Script",
|
|
85
|
+
renderer: "script-source",
|
|
86
|
+
action: (ref) => `akm show ${ref} -> execute the run command`,
|
|
87
|
+
fragmentRef: false,
|
|
88
|
+
},
|
|
68
89
|
memory: { label: "Memory", renderer: "memory-md", action: (ref) => `akm show ${ref} -> recall context` },
|
|
69
90
|
env: {
|
|
70
91
|
label: "Env",
|
|
@@ -85,6 +106,7 @@ export const TYPE_PRESENTATION = {
|
|
|
85
106
|
label: "Task",
|
|
86
107
|
renderer: "task-yaml",
|
|
87
108
|
action: (ref) => `akm show ${ref} -> inspect; akm task run <id> -> run now; edit the file + akm task sync -> unschedule`,
|
|
109
|
+
fragmentRef: false,
|
|
88
110
|
},
|
|
89
111
|
session: {
|
|
90
112
|
label: "Session",
|
|
@@ -105,6 +127,7 @@ export const TYPE_PRESENTATION = {
|
|
|
105
127
|
label: "Instruction",
|
|
106
128
|
renderer: "knowledge-md",
|
|
107
129
|
action: (ref) => `akm show ${ref} -> read the project instructions`,
|
|
130
|
+
fragmentRef: false,
|
|
108
131
|
},
|
|
109
132
|
};
|
|
110
133
|
/** Generic fallback for a type outside {@link KNOWN_TYPES} — never `undefined`, never a throw. */
|
|
@@ -120,6 +143,15 @@ export function presentationFor(type) {
|
|
|
120
143
|
return TYPE_PRESENTATION[type];
|
|
121
144
|
return DEFAULT_PRESENTATION;
|
|
122
145
|
}
|
|
146
|
+
/**
|
|
147
|
+
* A missing declaration means a read-like reference or foreign type can expose
|
|
148
|
+
* a safe selector. Built-in executable and instruction-bearing types opt out
|
|
149
|
+
* explicitly above: their search match can be a fragment, but the advertised
|
|
150
|
+
* action needs the parent canonical ref and its complete context.
|
|
151
|
+
*/
|
|
152
|
+
export function allowsFragmentRef(type) {
|
|
153
|
+
return presentationFor(type).fragmentRef !== false;
|
|
154
|
+
}
|
|
123
155
|
export const defaultRendererRegistry = {
|
|
124
156
|
rendererNameFor(type) {
|
|
125
157
|
return presentationFor(type).renderer;
|
|
@@ -31,6 +31,7 @@ import { closeDatabase, openExistingDatabase } from "../storage/repositories/ind
|
|
|
31
31
|
import { deleteEntriesByIds, getEntryCount, upsertEntry } from "../storage/repositories/index-entries-repository.js";
|
|
32
32
|
import { deriveEntryProvenance, deriveInstallations } from "./installations.js";
|
|
33
33
|
import { generateEmbeddingsForDb, publishTargetedEmbeddingMeta } from "./materialize-embeddings.js";
|
|
34
|
+
import { getMarkdownFragmentContent, hasMarkdownFragmentContent, setMarkdownFragmentContent, } from "./passes/metadata.js";
|
|
34
35
|
import { drainDirDocuments } from "./scan/drain-dir.js";
|
|
35
36
|
import { buildSearchText } from "./search/search-fields.js";
|
|
36
37
|
import { buildFileContext } from "./walk/file-context.js";
|
|
@@ -158,6 +159,9 @@ export async function indexWrittenAssets(stashDir, filePaths, options = {}) {
|
|
|
158
159
|
let entryWithSize = entry;
|
|
159
160
|
try {
|
|
160
161
|
entryWithSize = { ...entry, fileSize: fs.statSync(file).size };
|
|
162
|
+
if (hasMarkdownFragmentContent(entry)) {
|
|
163
|
+
setMarkdownFragmentContent(entryWithSize, getMarkdownFragmentContent(entry));
|
|
164
|
+
}
|
|
161
165
|
}
|
|
162
166
|
catch {
|
|
163
167
|
// stat raced a delete — index without the size, like the full walk does.
|
package/dist/indexer/indexer.js
CHANGED
|
@@ -31,7 +31,7 @@ import { deriveEntryProvenance, deriveInstallations } from "./installations.js";
|
|
|
31
31
|
import { indexedPathMatchesOwner, resolveAdapterConceptOwner, } from "./lookup/adapter-concept-owner.js";
|
|
32
32
|
import { generateEmbeddingsForDb } from "./materialize-embeddings.js";
|
|
33
33
|
import { canUseIncrementalSkip, computeDirFingerprint, getCachedDirState, getDirIndexState, inferZeroRowReason, } from "./passes/dir-staleness.js";
|
|
34
|
-
import { isEnrichmentComplete, isWorkflowSkipWarning } from "./passes/metadata.js";
|
|
34
|
+
import { getMarkdownFragmentContent, hasMarkdownFragmentContent, isEnrichmentComplete, isWorkflowSkipWarning, setMarkdownFragmentContent, } from "./passes/metadata.js";
|
|
35
35
|
import { drainDirDocuments } from "./scan/drain-dir.js";
|
|
36
36
|
import { buildSearchText } from "./search/search-fields.js";
|
|
37
37
|
import { purgeOldUsageEvents } from "./usage/usage-events.js";
|
|
@@ -1504,7 +1504,10 @@ export function createEnrichmentDeadline(timeoutMs, totalEntries) {
|
|
|
1504
1504
|
// ── Helpers ─────────────────────────────────────────────────────────────────
|
|
1505
1505
|
function attachFileSize(entry, entryPath) {
|
|
1506
1506
|
try {
|
|
1507
|
-
|
|
1507
|
+
const sized = { ...entry, fileSize: fs.statSync(entryPath).size };
|
|
1508
|
+
if (hasMarkdownFragmentContent(entry))
|
|
1509
|
+
setMarkdownFragmentContent(sized, getMarkdownFragmentContent(entry));
|
|
1510
|
+
return sized;
|
|
1508
1511
|
}
|
|
1509
1512
|
catch {
|
|
1510
1513
|
return entry;
|
|
@@ -1179,6 +1179,67 @@ export function projectMarkdownContent(body, truncationInfo) {
|
|
|
1179
1179
|
truncationInfo.truncated = text.length > MARKDOWN_CONTENT_MAX_CHARS;
|
|
1180
1180
|
return truncateUnicodeSafe(text, MARKDOWN_CONTENT_MAX_CHARS);
|
|
1181
1181
|
}
|
|
1182
|
+
// Fragment text is intentionally not an IndexDocument field. IndexDocument is
|
|
1183
|
+
// an adapter/search payload boundary; this is an internal, derived index input
|
|
1184
|
+
// that is persisted separately by the entries repository for deterministic FTS
|
|
1185
|
+
// rebuilds. The WeakMap follows the already-read document through recognition
|
|
1186
|
+
// without making safe body bytes observable through public payloads.
|
|
1187
|
+
const markdownFragmentContentByEntry = new WeakMap();
|
|
1188
|
+
const markdownFragmentProjectionEntries = new WeakSet();
|
|
1189
|
+
export function setMarkdownFragmentContent(entry, content) {
|
|
1190
|
+
markdownFragmentProjectionEntries.add(entry);
|
|
1191
|
+
if (content)
|
|
1192
|
+
markdownFragmentContentByEntry.set(entry, content);
|
|
1193
|
+
}
|
|
1194
|
+
export function getMarkdownFragmentContent(entry) {
|
|
1195
|
+
return markdownFragmentContentByEntry.get(entry);
|
|
1196
|
+
}
|
|
1197
|
+
export function hasMarkdownFragmentContent(entry) {
|
|
1198
|
+
return markdownFragmentProjectionEntries.has(entry);
|
|
1199
|
+
}
|
|
1200
|
+
/**
|
|
1201
|
+
* Produce a safe, structure-preserving projection for fragment indexing.
|
|
1202
|
+
* Excluded source lines are retained as blank lines so fragment locations map
|
|
1203
|
+
* exactly to authored line numbers. This must be called from both indexing and
|
|
1204
|
+
* `show`; it deliberately never performs a storage-layer file reread.
|
|
1205
|
+
*/
|
|
1206
|
+
export function projectMarkdownFragmentContent(raw) {
|
|
1207
|
+
const lines = raw.split(/\r?\n/);
|
|
1208
|
+
const parsed = parseFrontmatter(raw);
|
|
1209
|
+
const start = parsed.frontmatter ? parsed.bodyStartLine - 1 : 0;
|
|
1210
|
+
const projected = lines.map(() => "");
|
|
1211
|
+
let fence;
|
|
1212
|
+
const htmlComment = { inComment: false };
|
|
1213
|
+
for (let index = start; index < lines.length; index++) {
|
|
1214
|
+
const rawLine = lines[index];
|
|
1215
|
+
if (fence) {
|
|
1216
|
+
if (isMarkdownFenceClosing(rawLine, fence))
|
|
1217
|
+
fence = undefined;
|
|
1218
|
+
continue;
|
|
1219
|
+
}
|
|
1220
|
+
if (!htmlComment.inComment) {
|
|
1221
|
+
const opening = parseMarkdownFenceOpening(rawLine);
|
|
1222
|
+
if (opening) {
|
|
1223
|
+
fence = opening;
|
|
1224
|
+
continue;
|
|
1225
|
+
}
|
|
1226
|
+
}
|
|
1227
|
+
let safe = stripMarkdownHtmlComments(rawLine, htmlComment);
|
|
1228
|
+
const opening = parseMarkdownFenceOpening(safe.trim());
|
|
1229
|
+
if (opening) {
|
|
1230
|
+
fence = opening;
|
|
1231
|
+
continue;
|
|
1232
|
+
}
|
|
1233
|
+
// Reference link destinations and standalone HTML are not retrieval
|
|
1234
|
+
// evidence and can contain credential-bearing URLs.
|
|
1235
|
+
if (/^\s*\[[^\]]+\]:\s*\S+/.test(safe) || /^\s*<[^>]+>\s*$/.test(safe))
|
|
1236
|
+
continue;
|
|
1237
|
+
safe = stripMarkdownLinkDestinations(safe).replace(/<[^>]+>/g, " ");
|
|
1238
|
+
projected[index] = safe.replace(/[ \t]+$/g, "");
|
|
1239
|
+
}
|
|
1240
|
+
const text = projected.join("\n");
|
|
1241
|
+
return text.trim() ? text : undefined;
|
|
1242
|
+
}
|
|
1182
1243
|
// ── Metadata Generation ─────────────────────────────────────────────────────
|
|
1183
1244
|
/**
|
|
1184
1245
|
* Priorities 1-2 of the metadata pipeline — package.json (P1), `.md`
|
|
@@ -1220,7 +1281,9 @@ export function applyPreContributorFields(entry, file, ctx, pkgMeta) {
|
|
|
1220
1281
|
applyProvenanceFrontmatter(entry, parsed.data);
|
|
1221
1282
|
// Native Markdown has one bounded low-weight body projection. Sensitive
|
|
1222
1283
|
// types and raw session/checkpoint material never cross this boundary.
|
|
1223
|
-
|
|
1284
|
+
const safeForFragments = entry.type !== "env" && entry.type !== "session" && !hasSessionMemoryMarker(parsed.data, parsed.content);
|
|
1285
|
+
setMarkdownFragmentContent(entry, safeForFragments ? projectMarkdownFragmentContent(content) : undefined);
|
|
1286
|
+
if (safeForFragments) {
|
|
1224
1287
|
const truncationInfo = { truncated: false };
|
|
1225
1288
|
const contentProjection = projectMarkdownContent(parsed.content, truncationInfo);
|
|
1226
1289
|
if (contentProjection) {
|
|
@@ -34,6 +34,7 @@
|
|
|
34
34
|
* Pure, type-only imports (no cycle participation).
|
|
35
35
|
*/
|
|
36
36
|
import path from "node:path";
|
|
37
|
+
import { getMarkdownFragmentContent, hasMarkdownFragmentContent, setMarkdownFragmentContent, } from "../passes/metadata.js";
|
|
37
38
|
/**
|
|
38
39
|
* Reconstruct the `IndexDocument` an `IndexDocument` was mapped from. First-class
|
|
39
40
|
* IndexDocument members and the `documentJson`-carried extras are both restored;
|
|
@@ -60,6 +61,8 @@ export function indexDocumentToStashEntry(doc) {
|
|
|
60
61
|
entry.content = doc.content;
|
|
61
62
|
if (doc.contentTruncated !== undefined)
|
|
62
63
|
entry.contentTruncated = doc.contentTruncated;
|
|
64
|
+
if (hasMarkdownFragmentContent(doc))
|
|
65
|
+
setMarkdownFragmentContent(entry, getMarkdownFragmentContent(doc));
|
|
63
66
|
if (doc.ownsPresentation !== undefined)
|
|
64
67
|
entry.ownsPresentation = doc.ownsPresentation;
|
|
65
68
|
if (doc.updated !== undefined)
|
|
@@ -30,7 +30,7 @@ import path from "node:path";
|
|
|
30
30
|
import { akmAdapter } from "../../core/adapter/adapters/akm-adapter.js";
|
|
31
31
|
import { compareCodePoints } from "../../core/common.js";
|
|
32
32
|
import { canonicalizeWorkflowName } from "../../core/recognition-util.js";
|
|
33
|
-
import {
|
|
33
|
+
import { resolveWorkflowSourceDomains, workflowNameForSourcePath } from "../../workflows/source-files.js";
|
|
34
34
|
import { compileWorkflowSource } from "../../workflows/source-ir/compile.js";
|
|
35
35
|
import { buildMetadataSkipWarning } from "../passes/metadata.js";
|
|
36
36
|
import { buildFileContext } from "../walk/file-context.js";
|
|
@@ -53,32 +53,41 @@ export function drainDirDocuments(adapter, component, fileContexts) {
|
|
|
53
53
|
const conceptIdByFile = new Map();
|
|
54
54
|
const rejectedPaths = new Set();
|
|
55
55
|
const rejectedConceptIds = new Set();
|
|
56
|
-
|
|
57
|
-
|
|
56
|
+
// A full directory drain may contain both peer workflow formats for one
|
|
57
|
+
// canonical ref. Ownership arbitration must happen *before* recognition:
|
|
58
|
+
// otherwise both documents reach the persistence fold and SQLite's final
|
|
59
|
+
// row is determined by the walk/readdir order. Resolve exactly the paths
|
|
60
|
+
// this drain owns, so a full scan retains only the deterministic `.md`
|
|
61
|
+
// winner while a targeted one-file reindex deliberately keeps its written
|
|
62
|
+
// source and therefore marks an existing peer row stale for read fallback.
|
|
63
|
+
const workflowOwnerPathByCanonicalName = new Map(resolveWorkflowSourceDomains(component.root, adapter.id, fileContexts.map((file) => file.absPath))
|
|
64
|
+
.filter((resolution) => resolution.source !== undefined)
|
|
65
|
+
.map((resolution) => [resolution.canonicalName, path.resolve(resolution.source.path)]));
|
|
66
|
+
const invalidWorkflowOwnerNames = new Set();
|
|
67
|
+
const orderedFileContexts = [...fileContexts].sort((left, right) => {
|
|
68
|
+
const leftName = workflowNameForSourcePath(component.root, adapter.id, left.absPath);
|
|
69
|
+
const rightName = workflowNameForSourcePath(component.root, adapter.id, right.absPath);
|
|
70
|
+
const leftOwner = leftName !== undefined &&
|
|
71
|
+
workflowOwnerPathByCanonicalName.get(canonicalizeWorkflowName(leftName)) === path.resolve(left.absPath);
|
|
72
|
+
const rightOwner = rightName !== undefined &&
|
|
73
|
+
workflowOwnerPathByCanonicalName.get(canonicalizeWorkflowName(rightName)) === path.resolve(right.absPath);
|
|
74
|
+
if (leftOwner !== rightOwner)
|
|
75
|
+
return leftOwner ? -1 : 1;
|
|
76
|
+
return compareCodePoints(left.absPath, right.absPath);
|
|
77
|
+
});
|
|
78
|
+
for (const file of orderedFileContexts) {
|
|
79
|
+
if (rejectedPaths.has(file.absPath))
|
|
80
|
+
continue;
|
|
58
81
|
const workflowName = workflowNameForSourcePath(component.root, adapter.id, file.absPath);
|
|
59
82
|
if (workflowName !== undefined) {
|
|
60
83
|
const canonicalName = canonicalizeWorkflowName(workflowName);
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
try {
|
|
67
|
-
resolveUniqueWorkflowSource(component.root, adapter.id, workflowName);
|
|
68
|
-
}
|
|
69
|
-
catch (error) {
|
|
70
|
-
if (!(error instanceof WorkflowSourceRejectionError))
|
|
71
|
-
throw error;
|
|
72
|
-
rejectedConceptIds.add(adapter.id === "akm" ? `workflows/${canonicalName}` : canonicalName);
|
|
73
|
-
for (const relativePath of error.sourcePaths) {
|
|
74
|
-
rejectedPaths.add(path.join(component.root, relativePath));
|
|
84
|
+
const ownerPath = workflowOwnerPathByCanonicalName.get(canonicalName);
|
|
85
|
+
if (ownerPath !== undefined &&
|
|
86
|
+
ownerPath !== path.resolve(file.absPath) &&
|
|
87
|
+
!invalidWorkflowOwnerNames.has(canonicalName)) {
|
|
88
|
+
continue;
|
|
75
89
|
}
|
|
76
|
-
warnings.push(error.message);
|
|
77
90
|
}
|
|
78
|
-
}
|
|
79
|
-
for (const file of fileContexts) {
|
|
80
|
-
if (rejectedPaths.has(file.absPath))
|
|
81
|
-
continue;
|
|
82
91
|
const doc = adapter.recognize(component, file);
|
|
83
92
|
if (doc === null)
|
|
84
93
|
continue;
|
|
@@ -92,6 +101,8 @@ export function drainDirDocuments(adapter, component, fileContexts) {
|
|
|
92
101
|
const dropWarning = handleWorkflowDoc(doc, file, component.root);
|
|
93
102
|
if (dropWarning !== null) {
|
|
94
103
|
warnings.push(dropWarning);
|
|
104
|
+
if (workflowName !== undefined)
|
|
105
|
+
invalidWorkflowOwnerNames.add(canonicalizeWorkflowName(workflowName));
|
|
95
106
|
continue;
|
|
96
107
|
}
|
|
97
108
|
if (doc.hash !== undefined)
|
|
@@ -17,10 +17,11 @@ import path from "node:path";
|
|
|
17
17
|
import { buildActionFromContributors, defaultActionContributors } from "../../core/action-contributors.js";
|
|
18
18
|
import { stashDirFor } from "../../core/asset/asset-placement.js";
|
|
19
19
|
import { displayRef } from "../../core/asset/resolve-ref.js";
|
|
20
|
+
import { compareCodePoints } from "../../core/common.js";
|
|
20
21
|
import { classifyPathAccess } from "../../core/path-access.js";
|
|
21
22
|
import { getDbPath } from "../../core/paths.js";
|
|
22
23
|
import { systemErrorCode } from "../../core/system-error.js";
|
|
23
|
-
import { defaultRendererRegistry } from "../../core/type-presentation.js";
|
|
24
|
+
import { allowsFragmentRef, defaultRendererRegistry } from "../../core/type-presentation.js";
|
|
24
25
|
import { normalizeEmbeddingEndpoint } from "../../llm/embedders/remote.js";
|
|
25
26
|
import { assertIndexPathReadable, closeDatabase, openExistingDatabase, } from "../../storage/repositories/index-connection.js";
|
|
26
27
|
import { getAllEntries, getBaseBeliefStatesForDerivedTwins, getEntryById, getEntryCount, getPositiveFeedbackCountsByIds, } from "../../storage/repositories/index-entries-repository.js";
|
|
@@ -186,13 +187,47 @@ export async function searchLocal(input) {
|
|
|
186
187
|
}
|
|
187
188
|
// ── Database search ─────────────────────────────────────────────────────────
|
|
188
189
|
/**
|
|
189
|
-
* Keep
|
|
190
|
-
*
|
|
191
|
-
*
|
|
190
|
+
* Keep public scores in [0, 1] without flattening every boosted result to the
|
|
191
|
+
* same hard-clamped value. The ranking pipeline deliberately keeps its raw
|
|
192
|
+
* score for deterministic ordering before stable path deduplication; this
|
|
193
|
+
* monotone display projection preserves that order and leaves visible
|
|
194
|
+
* separation for graph, type, and project-context signals.
|
|
192
195
|
*/
|
|
196
|
+
function displaySearchScore(score) {
|
|
197
|
+
return 1 - Math.exp(-Math.max(0, score));
|
|
198
|
+
}
|
|
199
|
+
/**
|
|
200
|
+
* A final deterministic key for genuinely tied candidates. It deliberately
|
|
201
|
+
* excludes the asset name, filename, path, durable ref, and SQLite id: callers
|
|
202
|
+
* such as the memory-pack adapter generate each of those from an opaque source
|
|
203
|
+
* id, so using one here makes an otherwise equal search depend on that id.
|
|
204
|
+
*
|
|
205
|
+
* The normal AKM Markdown adapter keeps an H1 title in `content`; strip that
|
|
206
|
+
* one synthetic title too, because the adapter may derive it from the opaque
|
|
207
|
+
* filename. Identical remaining bodies are semantically indistinguishable at
|
|
208
|
+
* this ranking stage and intentionally continue to the existing name/path
|
|
209
|
+
* fallback for repeatable local presentation.
|
|
210
|
+
*/
|
|
211
|
+
function asciiCaseFold(value) {
|
|
212
|
+
// SQLite's built-in lower() folds ASCII only unless a build opts into ICU.
|
|
213
|
+
// Keep this key deliberately in that portable shared subset instead of
|
|
214
|
+
// introducing locale-dependent JavaScript ordering for non-ASCII content.
|
|
215
|
+
return value.replace(/[A-Z]/g, (letter) => String.fromCharCode(letter.charCodeAt(0) + 32));
|
|
216
|
+
}
|
|
217
|
+
/** The portable byte-level title/body rule mirrored in index-fts-repository. */
|
|
218
|
+
export function canonicalContentTieKey(entry) {
|
|
219
|
+
const content = entry.content ?? "";
|
|
220
|
+
const newline = content.startsWith("# ") ? content.indexOf("\n") : -1;
|
|
221
|
+
// SQLite uses ltrim(value, char(13) || char(10) || ' ') after an exact '# '
|
|
222
|
+
// title and trim(value, ' ') otherwise. Keep exactly that deliberately
|
|
223
|
+
// narrow byte contract; do not use locale or Unicode-whitespace helpers.
|
|
224
|
+
const body = newline >= 0 ? content.slice(newline + 1).replace(/^[\r\n ]+/, "") : content;
|
|
225
|
+
const source = (body || entry.description || "").replace(/^ +| +$/g, "");
|
|
226
|
+
return Buffer.from(asciiCaseFold(source), "utf8").toString("hex");
|
|
227
|
+
}
|
|
193
228
|
function buildSearchResultComparator(query) {
|
|
194
229
|
const queryTokens = buildLexicalQueryPlan(query).tokens.map((token) => token.toLowerCase());
|
|
195
|
-
const displayScore = (score) => Math.round(
|
|
230
|
+
const displayScore = (score) => Math.round(displaySearchScore(score) * 10000) / 10000;
|
|
196
231
|
const stableRankScore = (score) => Math.round(score * 10000) / 10000;
|
|
197
232
|
return (a, b) => {
|
|
198
233
|
const aNameTier = lexicalNameMatchTier(a.entry, queryTokens);
|
|
@@ -208,11 +243,28 @@ function buildSearchResultComparator(query) {
|
|
|
208
243
|
const rawScoreDiff = stableRankScore(b.score) - stableRankScore(a.score);
|
|
209
244
|
if (rawScoreDiff !== 0)
|
|
210
245
|
return rawScoreDiff;
|
|
246
|
+
// Ceiling values are intentionally allowed to demote visibility, but not
|
|
247
|
+
// to erase relevance. Prefer the score before a relaxed body-only ceiling;
|
|
248
|
+
// a later belief-state ceiling has its own minScore handoff and must not
|
|
249
|
+
// overwrite this ordering evidence. Belief-only ceilings fall back to
|
|
250
|
+
// their `preCeilingScore`.
|
|
251
|
+
const preCeilingRelevance = (item) => item.preRelaxedCeilingScore ?? item.preCeilingScore ?? item.score;
|
|
252
|
+
const ceilingDiff = stableRankScore(preCeilingRelevance(b)) - stableRankScore(preCeilingRelevance(a));
|
|
253
|
+
if (ceilingDiff !== 0)
|
|
254
|
+
return ceilingDiff;
|
|
211
255
|
const nameDiff = bNameTier - aNameTier;
|
|
212
256
|
if (nameDiff !== 0)
|
|
213
257
|
return nameDiff;
|
|
214
258
|
const typeDiff = typeBoostFor(b.entry.type) - typeBoostFor(a.entry.type);
|
|
215
|
-
|
|
259
|
+
if (typeDiff !== 0)
|
|
260
|
+
return typeDiff;
|
|
261
|
+
// Keep opaque generated IDs out of the final relevance tie-break. This
|
|
262
|
+
// runs only after every ranking contributor (including the #940 preserved
|
|
263
|
+
// pre-ceiling evidence), exact-name, and type comparison has tied.
|
|
264
|
+
const contentDiff = compareCodePoints(canonicalContentTieKey(a.entry), canonicalContentTieKey(b.entry));
|
|
265
|
+
if (contentDiff !== 0)
|
|
266
|
+
return contentDiff;
|
|
267
|
+
return a.filePath.localeCompare(b.filePath);
|
|
216
268
|
};
|
|
217
269
|
}
|
|
218
270
|
async function searchDatabase(db, query, searchType, limit, stashDir, allSourceDirs, config, sources, rendererRegistry = defaultRendererRegistry, filters, includeProposed = false, beliefFilter = "all", restrictToSources = false, includeExcludedTypes = false, disableProjectContext = false, disableScopedUtility = false) {
|
|
@@ -279,8 +331,9 @@ async function searchDatabase(db, query, searchType, limit, stashDir, allSourceD
|
|
|
279
331
|
const { ftsResults, embeddingScores, embedMs, mode, semanticWarning } = await collectSearchSignals(db, query, limit * 3, typeFilter, defaultExcludes, config);
|
|
280
332
|
const tRank0 = Date.now();
|
|
281
333
|
// ── Score normalization ──────────────────────────────────────────────
|
|
282
|
-
//
|
|
283
|
-
// (FTS 0.7, vector 0.3)
|
|
334
|
+
// Stable bounded BM25 transform + cosine similarity with weighted addition
|
|
335
|
+
// (FTS 0.7, vector 0.3). The lexical transform is per-row, so widening the
|
|
336
|
+
// candidate set cannot alter a pre-existing row's base score.
|
|
284
337
|
const ftsScoreMap = normalizeFtsScores(ftsResults);
|
|
285
338
|
// Build embedding score map (cosine similarities already 0-1)
|
|
286
339
|
const embedScoreMap = new Map();
|
|
@@ -400,11 +453,11 @@ async function searchDatabase(db, query, searchType, limit, stashDir, allSourceD
|
|
|
400
453
|
const selected = beliefFiltered.slice(0, limit);
|
|
401
454
|
const hits = await Promise.all(selected.map((ranked) => {
|
|
402
455
|
const { entry, filePath, score, rankingMode, utilityBoosted } = ranked;
|
|
403
|
-
// CLAUDE.md locks SearchHit.score in [0,1]. The boost loop
|
|
404
|
-
//
|
|
405
|
-
//
|
|
406
|
-
//
|
|
407
|
-
const finalScore =
|
|
456
|
+
// CLAUDE.md locks SearchHit.score in [0,1]. The boost loop deliberately
|
|
457
|
+
// remains raw for ranking, then takes a monotone bounded projection at
|
|
458
|
+
// the public boundary so contributors do not collapse into hard-clamped
|
|
459
|
+
// ties.
|
|
460
|
+
const finalScore = displaySearchScore(score);
|
|
408
461
|
return buildDbHit({
|
|
409
462
|
entry,
|
|
410
463
|
path: filePath,
|
|
@@ -413,6 +466,7 @@ async function searchDatabase(db, query, searchType, limit, stashDir, allSourceD
|
|
|
413
466
|
query,
|
|
414
467
|
rankingMode,
|
|
415
468
|
lexicalMatch: ranked.lexicalMatch,
|
|
469
|
+
fragmentId: ranked.fragmentId,
|
|
416
470
|
defaultStashDir: stashDir,
|
|
417
471
|
allSourceDirs,
|
|
418
472
|
sources,
|
|
@@ -728,7 +782,11 @@ export async function buildDbHit(input) {
|
|
|
728
782
|
(source && path.resolve(source.path) === path.resolve(input.defaultStashDir)
|
|
729
783
|
? (input.bundleId ?? undefined)
|
|
730
784
|
: undefined);
|
|
731
|
-
const
|
|
785
|
+
const parentRef = resolveSearchHitRef(input.entry, input, defaultBundleId);
|
|
786
|
+
// Fragments prove lexical relevance, but executable assets must retain the
|
|
787
|
+
// parent ref consumed by their advertised action (for example workflow run).
|
|
788
|
+
// The central type-presentation contract opts those types out explicitly.
|
|
789
|
+
const ref = input.fragmentId && allowsFragmentRef(input.entry.type) ? `${parentRef}#${input.fragmentId}` : parentRef;
|
|
732
790
|
const editable = isEditable(absolutePath, input.config, input.sources);
|
|
733
791
|
const estimatedTokens = typeof input.entry.fileSize === "number" ? Math.round(input.entry.fileSize / 4) : undefined;
|
|
734
792
|
const hit = {
|