akm-cli 0.9.16-alpha.1 → 0.9.16
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +56 -132
- package/dist/assets/hints/cli-hints-full.md +13 -6
- package/dist/assets/tasks/core/index-refresh.yml +1 -1
- package/dist/assets/tasks/improve/akm-improve-catchup.yml +3 -6
- package/dist/cli/retired-commands.js +0 -4
- package/dist/cli/unknown-flags.js +3 -36
- package/dist/commands/env/env-binding.js +4 -4
- package/dist/commands/env/env-cli.js +3 -3
- package/dist/commands/improve/collapse-detector.js +2 -2
- package/dist/commands/improve/consolidate.js +4 -6
- package/dist/commands/improve/improve-cli.js +20 -15
- package/dist/commands/improve/reflect.js +23 -2
- package/dist/commands/lint/base-linter.js +9 -0
- package/dist/commands/lint/env-key-rules.js +2 -2
- package/dist/commands/proposal/propose.js +15 -1
- package/dist/commands/proposal/repository.js +3 -12
- package/dist/commands/proposal/validators/proposal-quality-validators.js +40 -3
- package/dist/commands/proposal/validators/proposal-validators.js +5 -4
- package/dist/commands/read/curate.js +44 -34
- package/dist/commands/read/search.js +35 -54
- package/dist/commands/read/show.js +21 -2
- package/dist/commands/registry-cli.js +5 -5
- package/dist/commands/sources/add-cli.js +59 -16
- package/dist/commands/sources/bundle-cli.js +35 -11
- package/dist/commands/sources/bundle-config-ops.js +30 -0
- package/dist/commands/sources/dangerous-env-audit.js +4 -4
- package/dist/commands/sources/info.js +8 -8
- package/dist/commands/sources/installed-stashes.js +55 -61
- package/dist/commands/sources/source-add.js +39 -38
- package/dist/commands/sources/source-manage.js +34 -12
- package/dist/commands/sources/stash-cli.js +111 -119
- package/dist/commands/sources/stash-skeleton.js +6 -3
- package/dist/commands/tasks/explain.js +4 -1
- package/dist/commands/tasks/tasks-cli.js +31 -9
- package/dist/commands/tasks/tasks.js +239 -194
- package/dist/commands/tasks/validate.js +20 -32
- package/dist/core/activation-policy.js +4 -4
- package/dist/core/adapter/adapters/akm-adapter.js +8 -35
- package/dist/core/adapter/adapters/akm-metadata.js +1 -11
- package/dist/core/adapter/execution-source.js +10 -29
- package/dist/core/asset/asset-placement.js +0 -35
- package/dist/core/config/config-schema.js +64 -8
- package/dist/core/config/config-sources.js +96 -2
- package/dist/core/config/config.js +190 -24
- package/dist/core/config/legacy-source-shape-shim.js +9 -0
- package/dist/core/config/schema/embedding.js +30 -7
- package/dist/core/config/schema/execution.js +23 -0
- package/dist/core/config/schema/experimental.js +1 -1
- package/dist/core/config/schema/scheduler.js +20 -0
- package/dist/core/config/schema/search.js +10 -12
- package/dist/core/config/schema/sources-bundles.js +32 -1
- package/dist/core/content-safety.js +52 -0
- package/dist/core/errors.js +2 -5
- package/dist/core/maintenance-barrier.js +11 -13
- package/dist/core/paths.js +11 -0
- package/dist/core/run-lock.js +2 -5
- package/dist/core/state/migrations.js +1 -26
- package/dist/core/state-db.js +27 -63
- package/dist/core/type-presentation.js +1 -1
- package/dist/core/write-source.js +13 -8
- package/dist/indexer/bundle-identity-guard.js +45 -8
- package/dist/indexer/ensure-index.js +0 -5
- package/dist/indexer/index-db-contention.js +56 -0
- package/dist/indexer/index-rebuild-lock.js +73 -0
- package/dist/indexer/index-written-assets.js +171 -133
- package/dist/indexer/indexer.js +1621 -458
- package/dist/indexer/lookup/adapter-concept-owner.js +5 -19
- package/dist/indexer/materialize-embeddings.js +785 -0
- package/dist/indexer/passes/dir-staleness.js +161 -0
- package/dist/indexer/passes/metadata.js +1 -18
- package/dist/indexer/scan/drain-dir.js +70 -27
- package/dist/indexer/search/db-search.js +89 -373
- package/dist/indexer/search/ranking-contributors.js +16 -21
- package/dist/indexer/search/ranking.js +57 -135
- package/dist/indexer/search/search-source.js +29 -11
- package/dist/integrations/agent/execution-lowering.js +3 -2
- package/dist/integrations/agent/execution-preparation.js +32 -1
- package/dist/integrations/agent/prompts.js +1 -1
- package/dist/integrations/agent/request-lowering.js +3 -2
- package/dist/llm/client.js +3 -11
- package/dist/llm/embedder.js +3 -10
- package/dist/llm/embedders/remote.js +104 -133
- package/dist/llm/feature-gate.js +2 -4
- package/dist/llm/rerank-client.js +3 -3
- package/dist/output/html-render.js +2 -1
- package/dist/output/shapes/passthrough.js +2 -1
- package/dist/output/stdout.js +24 -0
- package/dist/output/text/command-format.js +13 -19
- package/dist/output/text/helpers.js +1 -1
- package/dist/output/text/index.js +2 -5
- package/dist/output/text.js +4 -3
- package/dist/registry/resolve.js +37 -10
- package/dist/scripts/akm-migrate-node.js +15197 -11351
- package/dist/scripts/akm-migrate.js +15514 -11668
- package/dist/setup/semantic-assets.js +2 -2
- package/dist/setup/setup.js +3 -3
- package/dist/setup/steps/connection.js +2 -3
- package/dist/setup/steps/tasks.js +29 -36
- package/dist/sources/providers/git-install.js +17 -11
- package/dist/sources/providers/git-provider.js +12 -5
- package/dist/sources/providers/git-stash.js +38 -16
- package/dist/sources/snapshot-fetchers/website-ingest.js +3 -3
- package/dist/storage/repositories/embedding-salvage-repository.js +184 -0
- package/dist/storage/repositories/index-connection.js +3 -1
- package/dist/storage/repositories/index-entries-repository.js +68 -77
- package/dist/storage/repositories/index-entry-schema.js +25 -16
- package/dist/storage/repositories/index-fts-repository.js +263 -29
- package/dist/storage/repositories/index-meta-repository.js +29 -0
- package/dist/storage/repositories/index-schema.js +122 -115
- package/dist/storage/repositories/index-utility-repository.js +1 -1
- package/dist/storage/repositories/index-vec-repository.js +435 -22
- package/dist/tasks/activation-config.js +90 -0
- package/dist/tasks/backends/cron.js +9 -0
- package/dist/tasks/backends/launchd.js +1 -0
- package/dist/tasks/backends/schtasks.js +2 -0
- package/dist/tasks/embedded.js +4 -5
- package/dist/tasks/scheduler-binding.js +2 -2
- package/dist/tasks/scheduler-sync-preview.js +8 -1
- package/dist/tasks/scheduler-sync.js +19 -10
- package/dist/tasks/source/parse-task-source.js +10 -113
- package/dist/tasks/source/project-v4.js +2 -2
- package/dist/tasks/source/task-source-v4.js +4 -12
- package/dist/tasks/source/task-to-v3.js +4 -12
- package/dist/tasks/source/task-to-v4.js +40 -7
- package/docs/migration/README.md +1 -0
- package/docs/migration/release-notes/0.9.15.md +36 -34
- package/docs/migration/release-notes/0.9.16.md +60 -98
- package/docs/migration/release-notes/README.md +0 -5
- package/docs/migration/v0.9.1-to-v0.9.2.md +6 -9
- package/docs/reference/cli.md +124 -122
- package/docs/reference/configuration.md +137 -133
- package/docs/reference/data-and-telemetry.md +1 -2
- package/docs/reference/tasks.md +34 -29
- package/package.json +1 -1
- package/schemas/akm-config.json +170 -6
- package/schemas/akm-task.json +1 -2
- package/dist/commands/sources/index-status.js +0 -99
- package/dist/core/hash.js +0 -18
- package/dist/indexer/drain.js +0 -306
- package/dist/indexer/embedding-identity.js +0 -20
- package/dist/indexer/enrich.js +0 -260
- package/dist/indexer/reconcile.js +0 -890
- package/dist/indexer/scan/parse-file.js +0 -66
- package/dist/indexer/units/unit.js +0 -159
- package/dist/llm/embedders/provider-limits.js +0 -288
- package/dist/storage/repositories/files-repository.js +0 -181
- package/dist/storage/repositories/units-repository.js +0 -510
|
@@ -16,6 +16,7 @@ import fs from "node:fs";
|
|
|
16
16
|
import { placementTypes, stashDirFor } from "../../core/asset/asset-placement.js";
|
|
17
17
|
import { parseRefInput } from "../../core/asset/resolve-ref.js";
|
|
18
18
|
import { resolveStashDir } from "../../core/common.js";
|
|
19
|
+
import { generatedContentRejection } from "../../core/content-safety.js";
|
|
19
20
|
import { UsageError } from "../../core/errors.js";
|
|
20
21
|
import { appendEvent } from "../../core/events.js";
|
|
21
22
|
import { resolveStandardsContext } from "../../core/standards/resolve-standards-context.js";
|
|
@@ -217,7 +218,20 @@ export async function akmPropose(options) {
|
|
|
217
218
|
};
|
|
218
219
|
}
|
|
219
220
|
}
|
|
220
|
-
|
|
221
|
+
const unsafeContent = generatedContentRejection(payload.content, redactWithLoweredExecutionDispatchLease(lease, payload.content));
|
|
222
|
+
if (unsafeContent) {
|
|
223
|
+
return {
|
|
224
|
+
schemaVersion: 2,
|
|
225
|
+
ok: false,
|
|
226
|
+
reason: "parse_error",
|
|
227
|
+
error: unsafeContent,
|
|
228
|
+
type: options.type,
|
|
229
|
+
name: options.name,
|
|
230
|
+
engine: engineName,
|
|
231
|
+
exitCode: result.exitCode,
|
|
232
|
+
...noticeFields(notices),
|
|
233
|
+
};
|
|
234
|
+
}
|
|
221
235
|
// 6. Insert the proposal. Note: we allow the agent's `ref` to normalise the
|
|
222
236
|
// asset name (e.g. path-cleanup), but only after validating that the ref is
|
|
223
237
|
// well-formed and the type still matches the requested type.
|
|
@@ -1027,19 +1027,10 @@ async function finalizeProposalTransaction(txn, target, proposal, ctx) {
|
|
|
1027
1027
|
}
|
|
1028
1028
|
let accepted = getProposal(p.stashDir, p.proposalId, ctx);
|
|
1029
1029
|
if (txn.journal.phase === "proposal-persisted") {
|
|
1030
|
-
if (await indexWrittenAssets(txn.journal.root, [p.assetPath], { bundleId: target.source.name })) {
|
|
1031
|
-
|
|
1032
|
-
}
|
|
1033
|
-
else {
|
|
1034
|
-
// `indexWrittenAssets`'s contract (index-written-assets.ts): `false`
|
|
1035
|
-
// means the asset write already stands, but the index itself needs a
|
|
1036
|
-
// manual `akm index` — warn and continue, the same as `source clone`
|
|
1037
|
-
// treats the same return, instead of failing the whole accept/revert.
|
|
1038
|
-
// Leave the phase at "proposal-persisted" (do not advance) so a later
|
|
1039
|
-
// recovery of this journal retries the index update rather than
|
|
1040
|
-
// skipping it as already done.
|
|
1041
|
-
warn(`${p.operation === "accept" ? "Accept" : "Revert"} of ${p.ref} succeeded, but its index update failed; run \`akm index\` to refresh it.`);
|
|
1030
|
+
if (!(await indexWrittenAssets(txn.journal.root, [p.assetPath], { bundleId: target.source.name }))) {
|
|
1031
|
+
throw new Error(`Proposal ${p.proposalId} index finalization failed.`);
|
|
1042
1032
|
}
|
|
1033
|
+
advanceTxn(txn, "index-finalized");
|
|
1043
1034
|
}
|
|
1044
1035
|
if (txn.journal.phase === "index-finalized") {
|
|
1045
1036
|
accepted = getProposal(p.stashDir, p.proposalId, ctx);
|
|
@@ -61,6 +61,7 @@
|
|
|
61
61
|
import { parseFrontmatter } from "../../../core/asset/frontmatter.js";
|
|
62
62
|
import { parseRefInput } from "../../../core/asset/resolve-ref.js";
|
|
63
63
|
import { DESCRIPTION_MAX_CHARS, DESCRIPTION_MIN_CHARS, WHEN_TO_USE_MAX_CHARS, WHEN_TO_USE_MIN_CHARS, } from "../../../core/authoring-rules.js";
|
|
64
|
+
import { containsRedactedContent, containsReflectPromptScaffolding, REDACTED_CONTENT_MARKER, REFLECT_AVOID_PATTERNS_HEADING, } from "../../../core/content-safety.js";
|
|
64
65
|
import { proposalContent } from "../../../core/file-change.js";
|
|
65
66
|
/**
|
|
66
67
|
* The canonical asset NAME an inputRef names, lower-cased — the tail the
|
|
@@ -399,6 +400,40 @@ const reflectTruncationMarkerValidator = {
|
|
|
399
400
|
];
|
|
400
401
|
},
|
|
401
402
|
};
|
|
403
|
+
/** Never promote proposal text that already contains an output-redaction marker (#962). */
|
|
404
|
+
const redactedContentValidator = {
|
|
405
|
+
name: "redacted-content",
|
|
406
|
+
appliesTo(proposal) {
|
|
407
|
+
return typeof proposal.payload?.content === "string";
|
|
408
|
+
},
|
|
409
|
+
validate(proposal) {
|
|
410
|
+
if (!containsRedactedContent(proposalContent(proposal)))
|
|
411
|
+
return [];
|
|
412
|
+
return [
|
|
413
|
+
{
|
|
414
|
+
kind: "redacted-content",
|
|
415
|
+
message: `Proposal ${proposal.id} (${proposal.ref}) contains ${REDACTED_CONTENT_MARKER}. Restore the original non-secret prose and create a clean proposal; redacted output cannot be promoted.`,
|
|
416
|
+
},
|
|
417
|
+
];
|
|
418
|
+
},
|
|
419
|
+
};
|
|
420
|
+
/** Defense in depth when a reflect proposal bypasses creation-time sanitization (#963). */
|
|
421
|
+
const reflectPromptScaffoldingValidator = {
|
|
422
|
+
name: "reflect-prompt-scaffolding",
|
|
423
|
+
appliesTo(proposal) {
|
|
424
|
+
return proposal.source === "reflect" && typeof proposal.payload?.content === "string";
|
|
425
|
+
},
|
|
426
|
+
validate(proposal) {
|
|
427
|
+
if (!containsReflectPromptScaffolding(proposalContent(proposal)))
|
|
428
|
+
return [];
|
|
429
|
+
return [
|
|
430
|
+
{
|
|
431
|
+
kind: "reflect-prompt-scaffolding",
|
|
432
|
+
message: `Proposal ${proposal.id} (${proposal.ref}) still contains the run-only "${REFLECT_AVOID_PATTERNS_HEADING}" prompt section. Reflect the asset again before promotion.`,
|
|
433
|
+
},
|
|
434
|
+
];
|
|
435
|
+
},
|
|
436
|
+
};
|
|
402
437
|
/**
|
|
403
438
|
* Report a validator's findings as advisory.
|
|
404
439
|
*
|
|
@@ -424,9 +459,9 @@ function advisory(validator) {
|
|
|
424
459
|
* Full set of quality validators in registration order. Appended onto
|
|
425
460
|
* {@link defaultProposalValidators} so they run inside `validateProposal` on
|
|
426
461
|
* `proposal accept` automatically. All prose-quality checks report without
|
|
427
|
-
* blocking (see {@link advisory})
|
|
428
|
-
*
|
|
429
|
-
* prose quality.
|
|
462
|
+
* blocking (see {@link advisory}). The truncation-marker, redacted-content,
|
|
463
|
+
* and reflected-prompt-scaffolding validators block because they protect
|
|
464
|
+
* durable content rather than judging prose quality.
|
|
430
465
|
*/
|
|
431
466
|
export const defaultProposalQualityValidators = [
|
|
432
467
|
...[
|
|
@@ -436,4 +471,6 @@ export const defaultProposalQualityValidators = [
|
|
|
436
471
|
reflectSizeGuardValidator,
|
|
437
472
|
].map(advisory),
|
|
438
473
|
reflectTruncationMarkerValidator,
|
|
474
|
+
redactedContentValidator,
|
|
475
|
+
reflectPromptScaffoldingValidator,
|
|
439
476
|
];
|
|
@@ -131,14 +131,15 @@ export const defaultProposalValidators = [
|
|
|
131
131
|
* and was previously safe to run in full there because every quality
|
|
132
132
|
* validator was advisory (`advisory()` downgrades findings to `severity:
|
|
133
133
|
* "warn"`, which {@link runProposalValidators}'s `ok` never treats as
|
|
134
|
-
* failing). #952
|
|
135
|
-
*
|
|
136
|
-
*
|
|
134
|
+
* failing). Blocking durable-content validators (the #952 truncation marker,
|
|
135
|
+
* #962 redaction marker, and #963 reflected prompt scaffolding) deliberately
|
|
136
|
+
* remain outside this subset, so running the full
|
|
137
|
+
* {@link defaultProposalValidators} list at mint time could throw
|
|
137
138
|
* `invalid_canonical_structure` for any lesson/task/workflow reflect
|
|
138
139
|
* proposal whose body leaks the truncation marker — instead of letting
|
|
139
140
|
* `sanitizeReflectPayload` mint the proposal and defer it with
|
|
140
141
|
* `reflect-truncation-leak`, per the #952 design. Quality validators (prose
|
|
141
|
-
* shape, reflect size ratio,
|
|
142
|
+
* shape, reflect size ratio, and durable-content guards) belong at
|
|
142
143
|
* `proposal accept` / drain-promotion time, which already calls
|
|
143
144
|
* {@link validateProposal} (the full list) via `preflightProposalPromotion`
|
|
144
145
|
* / `promoteProposalWithLease`.
|
|
@@ -25,6 +25,8 @@ import { copySearchHitAttribution, getSearchHitAttribution, usageEventAttributio
|
|
|
25
25
|
import { findSourceForPath, resolveSourceEntries } from "../../indexer/search/search-source.js";
|
|
26
26
|
import { insertUsageEvent } from "../../indexer/usage/usage-events.js";
|
|
27
27
|
import { estimateTokenCount } from "../../llm/embedders/remote.js";
|
|
28
|
+
import { tryLlmFeature } from "../../llm/feature-gate.js";
|
|
29
|
+
import { rerankDocuments } from "../../llm/rerank-client.js";
|
|
28
30
|
import { truncateDescription } from "../../output/shapes/helpers.js";
|
|
29
31
|
import { TELEMETRY_BUSY_TIMEOUT_MS, withIndexDb } from "../../storage/repositories/index-db.js";
|
|
30
32
|
import { findEntryIdByRef, getItemRefById } from "../../storage/repositories/index-entries-repository.js";
|
|
@@ -145,7 +147,7 @@ export async function curateSearchResults(query, result, limit, selectedType, ev
|
|
|
145
147
|
// fixtures) with a `SearchResponse` that was never type-filtered.
|
|
146
148
|
const stashHits = selectedType && selectedType !== "any" ? allStashHits.filter((hit) => hit.type === selectedType) : allStashHits;
|
|
147
149
|
const selected = selectCuratedStashHits(query, stashHits, limit);
|
|
148
|
-
const selectedStashHits = selected.selected;
|
|
150
|
+
const selectedStashHits = await maybeRerankCuratedStashHits(query, selected.selected);
|
|
149
151
|
const supportRefsByRef = selected.supportRefsByRef;
|
|
150
152
|
// F4/R-019: respect `--limit` for registry fill instead of hard-capping it
|
|
151
153
|
// at a bare literal 2 — the remaining slots after stash hits ARE the cap.
|
|
@@ -169,16 +171,10 @@ export async function curateSearchResults(query, result, limit, selectedType, ev
|
|
|
169
171
|
/**
|
|
170
172
|
* Pack a curate result's stash hits into a single token-budgeted blob:
|
|
171
173
|
* resolve each hit's content via the SAME path `akm show` uses
|
|
172
|
-
* (`akmShowUnified
|
|
173
|
-
*
|
|
174
|
-
*
|
|
175
|
-
*
|
|
176
|
-
* the ranking order `curateSearchResults` already produced, until the next
|
|
177
|
-
* hit would exceed `budgetTokens`. Each packed item's `ref` is that SAME
|
|
178
|
-
* fetched ref (`item.selectedRef ?? item.ref`), mirroring
|
|
179
|
-
* `enrichCuratedStashHit`'s own `contentRef` convention — labelling a packed
|
|
180
|
-
* fragment with the bare entry ref would let a later `akm show <ref>` return
|
|
181
|
-
* a different, larger document than what was actually packed and budgeted.
|
|
174
|
+
* (`akmShowUnified` — this also means a `ref#fragment` hit packs just the
|
|
175
|
+
* matched section), then greedily accumulate hits, in the ranking order
|
|
176
|
+
* `curateSearchResults` already produced, until the next hit would exceed
|
|
177
|
+
* `budgetTokens`.
|
|
182
178
|
*
|
|
183
179
|
* Registry hits are never packed — only `CuratedStashItem`s (locked
|
|
184
180
|
* contract, AGENTS.md: registry results stay separate/opt-in).
|
|
@@ -192,17 +188,9 @@ export async function packCuratedHits(result, budgetTokens) {
|
|
|
192
188
|
const packed = [];
|
|
193
189
|
let used = 0;
|
|
194
190
|
for (const item of stashItems) {
|
|
195
|
-
// item 4 — record the ref actually FETCHED (`contentRef`), mirroring
|
|
196
|
-
// `enrichCuratedStashHit`'s own convention: `item.ref` is now always the
|
|
197
|
-
// bare entry ref, but a body-only match's content came from the
|
|
198
|
-
// fragment-qualified `selectedRef`. Recording `item.ref` here labelled a
|
|
199
|
-
// packed fragment with the whole entry's ref, so a consumer that later
|
|
200
|
-
// ran `akm show <that ref>` got a different, larger document than what
|
|
201
|
-
// was actually packed and budgeted.
|
|
202
|
-
const contentRef = item.selectedRef ?? item.ref;
|
|
203
191
|
let shown;
|
|
204
192
|
try {
|
|
205
|
-
shown = await akmShowUnified({ ref:
|
|
193
|
+
shown = await akmShowUnified({ ref: item.ref, skipLogging: true });
|
|
206
194
|
}
|
|
207
195
|
catch {
|
|
208
196
|
continue;
|
|
@@ -210,7 +198,7 @@ export async function packCuratedHits(result, budgetTokens) {
|
|
|
210
198
|
const content = shown.content ?? shown.template ?? shown.prompt ?? "";
|
|
211
199
|
const tokens = estimateTokenCount(content);
|
|
212
200
|
if (used + tokens <= budgetTokens) {
|
|
213
|
-
packed.push({ ref:
|
|
201
|
+
packed.push({ ref: item.ref, tokens, content });
|
|
214
202
|
used += tokens;
|
|
215
203
|
continue;
|
|
216
204
|
}
|
|
@@ -218,7 +206,7 @@ export async function packCuratedHits(result, budgetTokens) {
|
|
|
218
206
|
const remaining = budgetTokens - used;
|
|
219
207
|
if (remaining > 0) {
|
|
220
208
|
const truncated = content.slice(0, remaining * 4);
|
|
221
|
-
packed.push({ ref:
|
|
209
|
+
packed.push({ ref: item.ref, tokens: estimateTokenCount(truncated), content: truncated });
|
|
222
210
|
used += estimateTokenCount(truncated);
|
|
223
211
|
}
|
|
224
212
|
}
|
|
@@ -227,15 +215,9 @@ export async function packCuratedHits(result, budgetTokens) {
|
|
|
227
215
|
return { query: result.query, budget: budgetTokens, tokens: used, items: packed };
|
|
228
216
|
}
|
|
229
217
|
async function enrichCuratedStashHit(query, hit, supportRefs, selectedRefs, eventSource) {
|
|
230
|
-
// index-redesign-contract.md B5f item 1 — `hit.ref` is always the bare entry
|
|
231
|
-
// ref now; `contentRef` is the fragment-qualified ref when the search match
|
|
232
|
-
// was a Markdown fragment (`hit.selectedRef`), so the preview/description
|
|
233
|
-
// resolved below and the curated item's own `followUp` still land on the
|
|
234
|
-
// matched section instead of regressing to the whole entry.
|
|
235
|
-
const contentRef = hit.selectedRef ?? hit.ref;
|
|
236
218
|
let shown;
|
|
237
219
|
try {
|
|
238
|
-
shown = await akmShowUnified({ ref:
|
|
220
|
+
shown = await akmShowUnified({ ref: hit.ref, eventSource, skipLogging: true });
|
|
239
221
|
}
|
|
240
222
|
catch {
|
|
241
223
|
shown = undefined;
|
|
@@ -253,13 +235,10 @@ async function enrichCuratedStashHit(query, hit, supportRefs, selectedRefs, even
|
|
|
253
235
|
type: shown?.type ?? hit.type,
|
|
254
236
|
name: shown?.name ?? hit.name,
|
|
255
237
|
ref: hit.ref,
|
|
256
|
-
...(hit.selectedRef ? { selectedRef: hit.selectedRef } : {}),
|
|
257
238
|
path: shown?.path ?? hit.path,
|
|
258
239
|
editable: shown?.editable ?? hit.editable ?? false,
|
|
259
240
|
...((shown?.editable ?? hit.editable ?? false) === false
|
|
260
|
-
? {
|
|
261
|
-
editHint: shown?.editHint ?? hit.editHint ?? `This asset is read-only. Inspect it with: akm show ${hit.ref}`,
|
|
262
|
-
}
|
|
241
|
+
? { editHint: shown?.editHint ?? hit.editHint ?? `This asset is read-only. Inspect it with: akm show ${hit.ref}` }
|
|
263
242
|
: {}),
|
|
264
243
|
...(description ? { description } : {}),
|
|
265
244
|
...(preview ? { preview } : {}),
|
|
@@ -267,7 +246,7 @@ async function enrichCuratedStashHit(query, hit, supportRefs, selectedRefs, even
|
|
|
267
246
|
...(shown?.parameters?.length ? { parameters: shown.parameters } : {}),
|
|
268
247
|
...(shown?.run ? { run: shown.run } : {}),
|
|
269
248
|
...(mergedSupportRefs.length > 0 ? { supportRefs: mergedSupportRefs } : {}),
|
|
270
|
-
followUp: `akm show ${
|
|
249
|
+
followUp: `akm show ${hit.ref}`,
|
|
271
250
|
reason: buildCuratedReason(query, shown?.type ?? hit.type),
|
|
272
251
|
...(hit.score !== undefined ? { score: hit.score } : {}),
|
|
273
252
|
};
|
|
@@ -617,6 +596,37 @@ function appendCurateSupportRef(supportRefsByRef, ownerRef, supportRef) {
|
|
|
617
596
|
return;
|
|
618
597
|
supportRefsByRef.set(ownerRef, [...existing, supportRef]);
|
|
619
598
|
}
|
|
599
|
+
/** Default number of `selectCuratedStashHits` candidates sent to the reranker when `search.curateRerank.topN` isn't set. */
|
|
600
|
+
const DEFAULT_CURATE_RERANK_TOP_N = 8;
|
|
601
|
+
/**
|
|
602
|
+
* Optional cross-encoder rerank pass over curate's already-selected, already-
|
|
603
|
+
* ranked candidates (#951). Disabled by default (`search.curateRerank.enabled`
|
|
604
|
+
* is falsy) and, when enabled, best-effort: any failure (misconfigured
|
|
605
|
+
* endpoint, network error, timeout, malformed response) falls back to
|
|
606
|
+
* `selectCuratedStashHits`'s own ranking unchanged — a reranker outage must
|
|
607
|
+
* never turn into a curate failure.
|
|
608
|
+
*
|
|
609
|
+
* Only the top `topN` (default {@link DEFAULT_CURATE_RERANK_TOP_N}) already-
|
|
610
|
+
* selected hits are sent (bounded request size); anything past that keeps its
|
|
611
|
+
* original position appended after the reranked prefix.
|
|
612
|
+
*/
|
|
613
|
+
async function maybeRerankCuratedStashHits(query, hits) {
|
|
614
|
+
if (hits.length <= 1)
|
|
615
|
+
return hits;
|
|
616
|
+
const config = loadConfig();
|
|
617
|
+
const rerankConfig = config.search?.curateRerank;
|
|
618
|
+
return tryLlmFeature("curate_rerank", config, async () => {
|
|
619
|
+
const topN = rerankConfig?.topN ?? DEFAULT_CURATE_RERANK_TOP_N;
|
|
620
|
+
const head = hits.slice(0, topN);
|
|
621
|
+
const tail = hits.slice(topN);
|
|
622
|
+
const documents = head.map((hit) => [hit.name, hit.description].filter(Boolean).join(" — "));
|
|
623
|
+
const ranked = await rerankDocuments(rerankConfig ?? {}, query, documents);
|
|
624
|
+
const rerankedHead = ranked
|
|
625
|
+
.map(({ index }) => head[index])
|
|
626
|
+
.filter((hit) => hit !== undefined);
|
|
627
|
+
return [...rerankedHead, ...tail];
|
|
628
|
+
}, hits, { timeoutMs: rerankConfig?.timeoutMs ?? null });
|
|
629
|
+
}
|
|
620
630
|
function selectCuratedStashHits(query, hits, limit) {
|
|
621
631
|
const intent = parseCurateIntent(query);
|
|
622
632
|
const collapsed = collapseCurateFamilies(query, hits);
|
|
@@ -17,8 +17,6 @@ import { appendEvent } from "../../core/events.js";
|
|
|
17
17
|
import { resolveReadSources } from "../../indexer/read-preflight.js";
|
|
18
18
|
import { searchLocal } from "../../indexer/search/db-search.js";
|
|
19
19
|
import { getSearchHitAttribution, usageEventAttributionMetadata, } from "../../indexer/search/search-attribution.js";
|
|
20
|
-
import { tryLlmFeature } from "../../llm/feature-gate.js";
|
|
21
|
-
import { rerankDocuments } from "../../llm/rerank-client.js";
|
|
22
20
|
import { getEntryIdByFilePath, getItemRefById } from "../../storage/repositories/index-entries-repository.js";
|
|
23
21
|
// Eagerly import source providers to trigger self-registration before the
|
|
24
22
|
// indexer or path-resolution code runs.
|
|
@@ -28,6 +26,24 @@ import { insertUsageEvent } from "../../indexer/usage/usage-events.js";
|
|
|
28
26
|
import { TELEMETRY_BUSY_TIMEOUT_MS, withIndexDb } from "../../storage/repositories/index-db.js";
|
|
29
27
|
import { searchRegistry } from "./registry-search.js";
|
|
30
28
|
const DEFAULT_LIMIT = 20;
|
|
29
|
+
function duplicateConceptWarnings(hits, defaultBundle) {
|
|
30
|
+
const ownersByConcept = new Map();
|
|
31
|
+
for (const hit of hits) {
|
|
32
|
+
const displayRef = (hit.parentRef ?? hit.ref).split("#", 1)[0] ?? hit.ref;
|
|
33
|
+
const boundary = displayRef.indexOf("//");
|
|
34
|
+
const conceptId = boundary >= 0 ? displayRef.slice(boundary + 2) : displayRef;
|
|
35
|
+
const owner = hit.origin ?? defaultBundle ?? "working-bundle";
|
|
36
|
+
const owners = ownersByConcept.get(conceptId) ?? [];
|
|
37
|
+
if (!owners.includes(owner))
|
|
38
|
+
owners.push(owner);
|
|
39
|
+
ownersByConcept.set(conceptId, owners);
|
|
40
|
+
}
|
|
41
|
+
return [...ownersByConcept]
|
|
42
|
+
.filter(([, owners]) => owners.length > 1)
|
|
43
|
+
.sort(([left], [right]) => left.localeCompare(right))
|
|
44
|
+
.map(([conceptId, owners]) => `Multiple bundles provide "${conceptId}": ${owners.join(", ")}. ` +
|
|
45
|
+
"Unqualified refs resolve by configured bundle priority; use a bundle-qualified ref to select explicitly.");
|
|
46
|
+
}
|
|
31
47
|
export async function akmSearch(input) {
|
|
32
48
|
const t0 = Date.now();
|
|
33
49
|
const query = input.query.trim();
|
|
@@ -113,21 +129,12 @@ export async function akmSearch(input) {
|
|
|
113
129
|
disableProjectContext: input.disableProjectContext === true,
|
|
114
130
|
disableScopedUtility: input.disableScopedUtility === true,
|
|
115
131
|
});
|
|
116
|
-
// #951 (moved from curate in 0.9.16 — the pass was always meant for
|
|
117
|
-
// search). Applied ONCE here, to LOCAL hits only, before the source
|
|
118
|
-
// branches below divide the same `localResult.hits` between the "local"
|
|
119
|
-
// and "all" responses — so both get the rerank and neither double-applies
|
|
120
|
-
// it. `localResult` is `undefined` for `source === "registry"`, so a
|
|
121
|
-
// registry-only search never reaches this call and never pays for the
|
|
122
|
-
// reranker's HTTP request; registry hits are never reranked (registry
|
|
123
|
-
// results staying separate from stash hits is a locked contract,
|
|
124
|
-
// AGENTS.md).
|
|
125
|
-
const rerankedLocalHits = localResult ? await maybeRerankSearchHits(query, localResult.hits, config) : undefined;
|
|
126
132
|
const registryResult = source === "local"
|
|
127
133
|
? undefined
|
|
128
134
|
: await searchRegistry(query, { limit, includeAssets: input.assets === true, registries: config.registries });
|
|
129
135
|
if (source === "local") {
|
|
130
|
-
const localHits =
|
|
136
|
+
const localHits = localResult?.hits ?? [];
|
|
137
|
+
const warnings = [...(localResult?.warnings ?? []), ...duplicateConceptWarnings(localHits, config.defaultBundle)];
|
|
131
138
|
const hasResults = localHits.length > 0;
|
|
132
139
|
const response = {
|
|
133
140
|
schemaVersion: 1,
|
|
@@ -135,7 +142,7 @@ export async function akmSearch(input) {
|
|
|
135
142
|
source,
|
|
136
143
|
hits: localHits,
|
|
137
144
|
tip: hasResults ? undefined : localResult?.tip,
|
|
138
|
-
warnings:
|
|
145
|
+
warnings: warnings.length ? warnings : undefined,
|
|
139
146
|
searchMode: localResult?.mode ?? "keyword",
|
|
140
147
|
timing: { totalMs: Date.now() - t0, rankMs: localResult?.rankMs, embedMs: localResult?.embedMs },
|
|
141
148
|
};
|
|
@@ -173,8 +180,12 @@ export async function akmSearch(input) {
|
|
|
173
180
|
return response;
|
|
174
181
|
}
|
|
175
182
|
// source === "all"
|
|
176
|
-
const allStashHits = (
|
|
177
|
-
const warnings = [
|
|
183
|
+
const allStashHits = (localResult?.hits ?? []).slice(0, limit);
|
|
184
|
+
const warnings = [
|
|
185
|
+
...(localResult?.warnings ?? []),
|
|
186
|
+
...duplicateConceptWarnings(allStashHits, config.defaultBundle),
|
|
187
|
+
...(registryResult?.warnings ?? []),
|
|
188
|
+
];
|
|
178
189
|
const hasResults = allStashHits.length > 0 || registryHits.length > 0;
|
|
179
190
|
const response = {
|
|
180
191
|
schemaVersion: 1,
|
|
@@ -194,42 +205,6 @@ export async function akmSearch(input) {
|
|
|
194
205
|
function usageSearchMode(mode) {
|
|
195
206
|
return mode === "semantic" ? "semantic" : "keyword";
|
|
196
207
|
}
|
|
197
|
-
/** Default number of `searchLocal`'s already-ranked LOCAL hits sent to the reranker when `search.rerank.topN` isn't set. */
|
|
198
|
-
const DEFAULT_SEARCH_RERANK_TOP_N = 8;
|
|
199
|
-
/**
|
|
200
|
-
* Optional cross-encoder rerank pass over `akm search`'s already-ranked LOCAL
|
|
201
|
-
* hits (#951, moved from `akm curate` in 0.9.16 — the pass was always meant
|
|
202
|
-
* for search). Disabled by default (`search.rerank.enabled` is falsy) and,
|
|
203
|
-
* when enabled, best-effort: any failure (misconfigured endpoint, network
|
|
204
|
-
* error, timeout, malformed response, an out-of-range or duplicate index in
|
|
205
|
-
* the response) falls back to `searchLocal`'s own ranking unchanged — a
|
|
206
|
-
* reranker outage must never turn into a search failure.
|
|
207
|
-
*
|
|
208
|
-
* Only the top `topN` (default {@link DEFAULT_SEARCH_RERANK_TOP_N}) hits are
|
|
209
|
-
* sent (bounded request size); anything past that keeps its original
|
|
210
|
-
* position appended after the reranked prefix. The reranker changes ARRAY
|
|
211
|
-
* ORDER only — each hit's own `score` is left untouched as the retrieval
|
|
212
|
-
* score (see docs/reference/cli.md and docs/reference/configuration.md for
|
|
213
|
-
* why: `SearchHit.score` is a locked `[0,1]` contract downstream consumers
|
|
214
|
-
* compare and threshold on, while ordering is the field a rerank-aware
|
|
215
|
-
* consumer reads).
|
|
216
|
-
*/
|
|
217
|
-
async function maybeRerankSearchHits(query, hits, config) {
|
|
218
|
-
if (hits.length <= 1)
|
|
219
|
-
return hits;
|
|
220
|
-
const rerankConfig = config.search?.rerank;
|
|
221
|
-
return tryLlmFeature("search_rerank", config, async () => {
|
|
222
|
-
const topN = rerankConfig?.topN ?? DEFAULT_SEARCH_RERANK_TOP_N;
|
|
223
|
-
const head = hits.slice(0, topN);
|
|
224
|
-
const tail = hits.slice(topN);
|
|
225
|
-
const documents = head.map((hit) => [hit.name, hit.description].filter(Boolean).join(" — "));
|
|
226
|
-
const ranked = await rerankDocuments(rerankConfig ?? {}, query, documents);
|
|
227
|
-
const rerankedHead = ranked
|
|
228
|
-
.map(({ index }) => head[index])
|
|
229
|
-
.filter((hit) => hit !== undefined);
|
|
230
|
-
return [...rerankedHead, ...tail];
|
|
231
|
-
}, hits, { timeoutMs: rerankConfig?.timeoutMs ?? null });
|
|
232
|
-
}
|
|
233
208
|
function maybeLogSearchEvent(input, query, response, mode) {
|
|
234
209
|
if (input.skipLogging)
|
|
235
210
|
return;
|
|
@@ -349,9 +324,15 @@ function logSearchEvent(query, response, mode = "keyword", eventSource = "user",
|
|
|
349
324
|
*/
|
|
350
325
|
function assertNamedSourceExists(config, namedSourceName) {
|
|
351
326
|
const configSources = getSources(config);
|
|
352
|
-
const foundInConfig = configSources.
|
|
327
|
+
const foundInConfig = configSources.find((source) => source.name === namedSourceName || source.path === namedSourceName);
|
|
328
|
+
if (foundInConfig?.enabled === false) {
|
|
329
|
+
throw new UsageError(`Source "${namedSourceName}" is disabled.`, "INVALID_SOURCE_VALUE");
|
|
330
|
+
}
|
|
353
331
|
if (!foundInConfig) {
|
|
354
|
-
const validNames = configSources
|
|
332
|
+
const validNames = configSources
|
|
333
|
+
.filter((source) => source.enabled !== false)
|
|
334
|
+
.map((source) => source.name)
|
|
335
|
+
.filter((name) => Boolean(name));
|
|
355
336
|
const hint = validNames.length > 0
|
|
356
337
|
? `Known source names: ${validNames.join(", ")}`
|
|
357
338
|
: "No named sources are configured. Run `akm bundle list` to see installed bundles.";
|
|
@@ -75,6 +75,11 @@ export async function akmShowUnified(input) {
|
|
|
75
75
|
if (metaRef)
|
|
76
76
|
return showStashMeta(metaRef);
|
|
77
77
|
}
|
|
78
|
+
const legacyReplacement = legacyColonRefReplacement(ref);
|
|
79
|
+
if (legacyReplacement) {
|
|
80
|
+
throw new NotFoundError(`The legacy colon ref "${ref}" was removed in 0.9.0. Use the slash form instead: ` +
|
|
81
|
+
`akm show ${legacyReplacement}`);
|
|
82
|
+
}
|
|
78
83
|
// Env/secret bodies have no safe fragment surface, and a fragment cannot
|
|
79
84
|
// widen what the env/secret renderers expose: both always omit the body
|
|
80
85
|
// (env — key names only; secret — never rendered), fragment or not. Warn
|
|
@@ -110,6 +115,20 @@ export async function akmShowUnified(input) {
|
|
|
110
115
|
}
|
|
111
116
|
return result;
|
|
112
117
|
}
|
|
118
|
+
/** Actionable migration guidance for the retired `[bundle//]type:name` spelling. */
|
|
119
|
+
function legacyColonRefReplacement(ref) {
|
|
120
|
+
const match = /^(?:(?<bundle>[^/#]+)\/\/)?(?<type>[a-z][a-z0-9-]*):(?<name>[^#]+)(?<fragment>#.*)?$/i.exec(ref);
|
|
121
|
+
const type = match?.groups?.type?.toLowerCase();
|
|
122
|
+
const name = match?.groups?.name;
|
|
123
|
+
if (!type || !name)
|
|
124
|
+
return undefined;
|
|
125
|
+
const stashDir = stashDirFor(type);
|
|
126
|
+
if (!stashDir)
|
|
127
|
+
return undefined;
|
|
128
|
+
const bundle = match?.groups?.bundle;
|
|
129
|
+
const fragment = match?.groups?.fragment ?? "";
|
|
130
|
+
return `${bundle ? `${bundle}//` : ""}${stashDir}/${name}${fragment}`;
|
|
131
|
+
}
|
|
113
132
|
/**
|
|
114
133
|
* Resolve a stash `.meta/` doc and return it as a lightweight ShowResponse.
|
|
115
134
|
*
|
|
@@ -288,7 +307,7 @@ export async function showLocal(input) {
|
|
|
288
307
|
throw new UsageError(`Renderer "${match.renderer}" not found for asset: ${makeBundleRef(parsed.bundle, parsed.conceptId)}`);
|
|
289
308
|
}
|
|
290
309
|
const renderBundle = indexedEntry.bundleId;
|
|
291
|
-
const renderDefaultBundle = config.defaultBundle ?? (source?.
|
|
310
|
+
const renderDefaultBundle = config.defaultBundle ?? (source?.isDefault === true ? renderBundle : undefined);
|
|
292
311
|
const renderCtx = buildRenderContext(fileCtx, match, allSourceDirs, renderBundle, renderDefaultBundle);
|
|
293
312
|
response = renderer.buildShowResponse(renderCtx);
|
|
294
313
|
if (parsed.fragment !== undefined) {
|
|
@@ -306,7 +325,7 @@ export async function showLocal(input) {
|
|
|
306
325
|
}
|
|
307
326
|
response.type = indexedEntry.type;
|
|
308
327
|
response.name = indexedEntry.name;
|
|
309
|
-
const isPrimaryStash = source
|
|
328
|
+
const isPrimaryStash = source?.isDefault === true;
|
|
310
329
|
const canonicalRef = displayRef({
|
|
311
330
|
type: indexedEntry.type,
|
|
312
331
|
name: presentedName,
|
|
@@ -25,7 +25,7 @@ export const registryCommand = defineGroupCommand({
|
|
|
25
25
|
name: { type: "string", description: "Human-friendly name for the registry" },
|
|
26
26
|
provider: { type: "string", description: "Provider type (e.g. static-index, skills-sh)" },
|
|
27
27
|
options: { type: "string", description: "Provider-specific options as JSON." },
|
|
28
|
-
"allow-insecure": {
|
|
28
|
+
"allow-insecure-transport": {
|
|
29
29
|
type: "boolean",
|
|
30
30
|
description: "Allow a plain HTTP registry URL (otherwise rejected)",
|
|
31
31
|
default: false,
|
|
@@ -41,12 +41,12 @@ export const registryCommand = defineGroupCommand({
|
|
|
41
41
|
throw new UsageError("Registry URL must start with http:// or https://");
|
|
42
42
|
}
|
|
43
43
|
if (args.url.startsWith("http://")) {
|
|
44
|
-
const
|
|
45
|
-
if (!
|
|
44
|
+
const allowInsecureTransport = args["allow-insecure-transport"];
|
|
45
|
+
if (!allowInsecureTransport) {
|
|
46
46
|
throw new UsageError("Registry URL uses plain HTTP (not HTTPS). An on-path attacker could substitute a malicious index. " +
|
|
47
|
-
"Use https:// or pass --allow-insecure if you have explicitly accepted the risk.");
|
|
47
|
+
"Use https:// or pass --allow-insecure-transport if you have explicitly accepted the risk.");
|
|
48
48
|
}
|
|
49
|
-
warn("Warning: registry URL uses plain HTTP (not HTTPS). --allow-insecure was set; an on-path attacker could substitute a malicious index.");
|
|
49
|
+
warn("Warning: registry URL uses plain HTTP (not HTTPS). --allow-insecure-transport was set; an on-path attacker could substitute a malicious index.");
|
|
50
50
|
}
|
|
51
51
|
const entry = { url: args.url };
|
|
52
52
|
if (args.name)
|