akm-cli 0.9.17-alpha.2 → 0.9.17-alpha.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +756 -0
- package/dist/akm +94 -196
- package/dist/cli/shared.js +6 -2
- package/dist/cli.js +22 -9
- package/dist/commands/agent/agent-dispatch.js +1 -1
- package/dist/commands/command/command-execution.js +24 -62
- package/dist/commands/feedback-cli.js +0 -1
- package/dist/commands/health/accept-rate.js +2 -2
- package/dist/commands/health/checks.js +30 -75
- package/dist/commands/health/config-skew.js +38 -0
- package/dist/commands/health/egress.js +54 -0
- package/dist/commands/health/html-report.js +0 -38
- package/dist/commands/health/improve-metrics.js +123 -562
- package/dist/commands/health/plugin-staleness.js +53 -3
- package/dist/commands/health/renderers.js +12 -4
- package/dist/commands/health/report-view-model.js +11 -106
- package/dist/commands/health/types-improve.js +4 -19
- package/dist/commands/health/windows.js +64 -73
- package/dist/commands/health.js +122 -143
- package/dist/commands/improve/consolidate/chunking.js +25 -100
- package/dist/commands/improve/consolidate/sanitize.js +54 -149
- package/dist/commands/improve/consolidate.js +538 -1075
- package/dist/commands/improve/content-hash.js +16 -24
- package/dist/commands/improve/distill/content-repair.js +18 -100
- package/dist/commands/improve/distill-guards.js +20 -81
- package/dist/commands/improve/distill-promotion-policy.js +23 -243
- package/dist/commands/improve/distill.js +608 -1075
- package/dist/commands/improve/eligibility.js +126 -400
- package/dist/commands/improve/execution.js +3 -5
- package/dist/commands/improve/extract.js +487 -1046
- package/dist/commands/improve/feedback-valence.js +0 -25
- package/dist/commands/improve/improve-cli.js +29 -166
- package/dist/commands/improve/improve-result-file.js +10 -66
- package/dist/commands/improve/improve-strategies.js +12 -7
- package/dist/commands/improve/improve-usage-report.js +18 -64
- package/dist/commands/improve/improve.js +443 -1063
- package/dist/commands/improve/ledger.js +114 -0
- package/dist/commands/improve/locks.js +2 -8
- package/dist/commands/improve/loop-stages.js +459 -1172
- package/dist/commands/improve/memory/derived-ref.js +12 -77
- package/dist/commands/improve/memory/memory-belief.js +14 -118
- package/dist/commands/improve/memory/memory-improve.js +4 -3
- package/dist/commands/improve/outcome-loop.js +28 -156
- package/dist/commands/improve/planner.js +5 -10
- package/dist/commands/improve/preparation.js +851 -2339
- package/dist/commands/improve/proactive-maintenance.js +34 -101
- package/dist/commands/improve/reflect-noise.js +104 -280
- package/dist/commands/improve/reflect.js +621 -1367
- package/dist/commands/improve/salience.js +46 -232
- package/dist/commands/improve/session-asset.js +19 -100
- package/dist/commands/improve/stage.js +323 -0
- package/dist/commands/proposal/drain.js +251 -644
- package/dist/commands/proposal/proposal-cli.js +3 -18
- package/dist/commands/proposal/proposal-types.js +20 -41
- package/dist/commands/proposal/proposal.js +1 -2
- package/dist/commands/proposal/propose.js +134 -160
- package/dist/commands/proposal/repository.js +502 -1487
- package/dist/commands/proposal/validators/proposal-quality-validators.js +71 -174
- package/dist/commands/proposal/validators/proposal-validators.js +1 -1
- package/dist/commands/proposal/validators/proposals.js +13 -89
- package/dist/commands/read/curate.js +63 -413
- package/dist/commands/read/search-cli.js +16 -33
- package/dist/commands/read/search.js +17 -23
- package/dist/commands/read/show.js +2 -13
- package/dist/commands/sources/bundle-cli.js +25 -2
- package/dist/commands/sources/bundle-config-ops.js +7 -0
- package/dist/commands/sources/dangerous-env-audit.js +1 -2
- package/dist/commands/sources/info.js +2 -11
- package/dist/commands/sources/installed-stashes.js +197 -746
- package/dist/commands/sources/schema-repair.js +98 -129
- package/dist/commands/sources/source-add.js +62 -12
- package/dist/commands/sources/stash-cli.js +1 -1
- package/dist/commands/tasks/explain.js +10 -13
- package/dist/commands/tasks/tasks-cli.js +9 -8
- package/dist/commands/tasks/tasks.js +326 -930
- package/dist/commands/tasks/validate.js +42 -21
- package/dist/commands/workflow/plan.js +22 -29
- package/dist/commands/workflow-cli.js +4 -4
- package/dist/core/adapter/adapters/akm-adapter.js +0 -1
- package/dist/core/adapter/adapters/akm-lint.js +2 -3
- package/dist/core/adapter/adapters/akm-metadata.js +11 -12
- package/dist/core/adapter/adapters/akm-workflow-adapter.js +1 -1
- package/dist/core/adapter/execution-source.js +17 -29
- package/dist/core/asset/resolve-ref.js +1 -1
- package/dist/core/bundle-id.js +42 -5
- package/dist/core/bundle-rename.js +291 -0
- package/dist/core/config/config-io.js +1 -2
- package/dist/core/config/config-schema.js +1 -33
- package/dist/core/config/config-walker.js +1 -1
- package/dist/core/config/config.js +163 -68
- package/dist/core/config/legacy-source-shape-shim.js +38 -9
- package/dist/core/config/schema/embedding.js +20 -5
- package/dist/core/config/schema/engines.js +5 -0
- package/dist/core/config/schema/execution.js +1 -1
- package/dist/core/config/schema/experimental.js +1 -1
- package/dist/core/config/schema/improve-processes.js +21 -95
- package/dist/core/config/schema/improve.js +4 -42
- package/dist/core/config/schema/scheduler.js +12 -12
- package/dist/core/config/schema/search.js +6 -22
- package/dist/core/env-secret-ref.js +0 -1
- package/dist/core/errors.js +8 -9
- package/dist/core/file-lock.js +76 -173
- package/dist/core/logs-db.js +2 -2
- package/dist/core/paths.js +0 -27
- package/dist/core/redaction.js +109 -2
- package/dist/core/run-lock.js +2 -5
- package/dist/core/spawn-env.js +1 -1
- package/dist/core/state/migrations.js +108 -61
- package/dist/core/state-db-scope.js +2 -4
- package/dist/core/state-db.js +126 -692
- package/dist/core/type-presentation.js +1 -9
- package/dist/core/write-source.js +293 -1012
- package/dist/execution/input-contract.js +1 -1
- package/dist/execution/resolved-request.js +135 -689
- package/dist/execution/source.js +63 -257
- package/dist/execution/target-ref.js +1 -1
- package/dist/indexer/bundle-identity-guard.js +2 -2
- package/dist/indexer/db/graph-db.js +106 -46
- package/dist/indexer/ensure-index.js +44 -85
- package/dist/indexer/graph/graph-extraction.js +340 -562
- package/dist/indexer/graph/graph-related.js +130 -0
- package/dist/indexer/index-rebuild-lock.js +3 -11
- package/dist/indexer/index-writer-lock.js +8 -17
- package/dist/indexer/index-written-assets.js +139 -151
- package/dist/indexer/indexer.js +524 -846
- package/dist/indexer/materialize-embeddings.js +60 -397
- package/dist/indexer/passes/memory-inference.js +81 -90
- package/dist/indexer/passes/metadata.js +132 -200
- package/dist/indexer/read-preflight.js +0 -7
- package/dist/indexer/scan/doc-to-entry.js +1 -3
- package/dist/indexer/scan/drain-dir.js +1 -1
- package/dist/indexer/search/db-search.js +181 -590
- package/dist/indexer/search/fts-query.js +30 -41
- package/dist/indexer/search/ranking.js +28 -154
- package/dist/indexer/search/search-attribution.js +12 -32
- package/dist/indexer/search/search-fields.js +11 -15
- package/dist/indexer/search/search-hit-enrichers.js +54 -85
- package/dist/indexer/search/search-source.js +1 -4
- package/dist/indexer/usage/usage-events.js +2 -7
- package/dist/integrations/agent/engine-fallback.js +23 -40
- package/dist/integrations/agent/engine-resolution.js +93 -183
- package/dist/integrations/agent/execution.js +507 -0
- package/dist/integrations/agent/model-map.js +28 -156
- package/dist/integrations/agent/request-lowering.js +66 -141
- package/dist/integrations/agent/runner-dispatch.js +143 -321
- package/dist/integrations/agent/runner.js +54 -14
- package/dist/integrations/lockfile.js +53 -101
- package/dist/llm/embedders/deterministic.js +2 -3
- package/dist/llm/embedders/profile.js +71 -0
- package/dist/llm/embedders/remote.js +10 -15
- package/dist/llm/graph-extract.js +3 -12
- package/dist/llm/index-passes.js +3 -5
- package/dist/llm/memory-infer.js +1 -2
- package/dist/llm/metadata-enhance.js +1 -2
- package/dist/llm/structured-call.js +5 -24
- package/dist/output/generic-render.js +23 -11
- package/dist/output/html-render.js +13 -10
- package/dist/output/render-registry.js +3 -32
- package/dist/output/shapes/helpers.js +2 -34
- package/dist/output/shapes/passthrough.js +1 -9
- package/dist/{indexer/search/ranking-types.js → output/text/bundle-rename.js} +4 -1
- package/dist/output/text/command-format.js +60 -23
- package/dist/output/text/helpers.js +1 -1
- package/dist/output/text/migrate.js +5 -14
- package/dist/output/text/proposal-format.js +1 -2
- package/dist/output/text/workflow-format.js +0 -32
- package/dist/output/text.js +2 -0
- package/dist/registry/factory.js +4 -19
- package/dist/registry/network.js +66 -220
- package/dist/registry/providers/index.js +0 -2
- package/dist/registry/providers/skills-sh.js +3 -14
- package/dist/registry/providers/static-index.js +24 -26
- package/dist/registry/resolve.js +55 -131
- package/dist/scripts/akm-migrate-node.js +43937 -93313
- package/dist/scripts/akm-migrate.js +43697 -93071
- package/dist/setup/registry-stash-loader.js +4 -13
- package/dist/setup/semantic-assets.js +3 -44
- package/dist/setup/setup.js +1 -1
- package/dist/setup/steps/tasks.js +25 -15
- package/dist/sources/provider-factory.js +17 -18
- package/dist/sources/providers/filesystem.js +2 -3
- package/dist/sources/providers/git-install.js +7 -1
- package/dist/sources/providers/git-provider.js +0 -3
- package/dist/sources/providers/git-stash.js +0 -17
- package/dist/sources/providers/npm.js +2 -4
- package/dist/sources/providers/provider-utils.js +5 -10
- package/dist/sources/providers/website.js +0 -2
- package/dist/sources/snapshot-fetchers/website-ingest.js +1 -1
- package/dist/sources/website-url.js +2 -2
- package/dist/storage/database.js +9 -35
- package/dist/storage/repositories/improve-ledger-repository.js +168 -0
- package/dist/storage/repositories/index-connection.js +34 -70
- package/dist/storage/repositories/index-entries-repository.js +69 -111
- package/dist/storage/repositories/index-entry-mapper.js +1 -2
- package/dist/storage/repositories/index-entry-schema.js +83 -269
- package/dist/storage/repositories/index-fts-repository.js +86 -256
- package/dist/storage/repositories/index-llm-cache-repository.js +17 -0
- package/dist/storage/repositories/index-meta-repository.js +6 -4
- package/dist/storage/repositories/index-schema.js +192 -220
- package/dist/storage/repositories/index-utility-repository.js +8 -29
- package/dist/storage/repositories/index-vec-repository.js +133 -414
- package/dist/storage/repositories/outcome-repository.js +2 -1
- package/dist/storage/repositories/proposals-repository.js +35 -0
- package/dist/storage/repositories/registry-index-cache-repository.js +100 -0
- package/dist/storage/repositories/task-history-repository.js +26 -4
- package/dist/storage/repositories/workflow-runs-repository.js +53 -244
- package/dist/storage/sqlite-migrations.js +136 -0
- package/dist/storage/sqlite-pragmas.js +11 -9
- package/dist/storage/sqlite-transaction.js +170 -0
- package/dist/storage/state-db-integrity.js +34 -27
- package/dist/tasks/activation-config.js +134 -62
- package/dist/tasks/backends/cron.js +129 -277
- package/dist/tasks/backends/exec-utils.js +2 -5
- package/dist/tasks/backends/launchd.js +125 -745
- package/dist/tasks/backends/schtasks.js +101 -620
- package/dist/tasks/prepare/prepare-support.js +5 -15
- package/dist/tasks/prepare/prepare.js +0 -2
- package/dist/tasks/resolve-akm-bin.js +20 -79
- package/dist/tasks/run/attempt-lifecycle.js +0 -1
- package/dist/tasks/scheduler-binding.js +18 -238
- package/dist/tasks/scheduler-invocation.js +52 -52
- package/dist/tasks/scheduler-lock.js +53 -0
- package/dist/tasks/scheduler-sync.js +363 -679
- package/dist/tasks/source/parse-task-source.js +160 -10
- package/dist/tasks/source/task-source-v3-frozen.js +3 -4
- package/dist/tasks/source/task-to-v4.js +2 -2
- package/dist/workflows/authoring/authoring.js +3 -12
- package/dist/workflows/compile.js +211 -0
- package/dist/workflows/concurrency-policy.js +13 -74
- package/dist/workflows/exec/child-invocation.js +3 -17
- package/dist/workflows/exec/child-workflow.js +32 -141
- package/dist/workflows/exec/dispatch-redaction.js +13 -53
- package/dist/workflows/exec/environment.js +98 -0
- package/dist/workflows/exec/exec-unit.js +33 -140
- package/dist/workflows/exec/frozen-judge.js +7 -59
- package/dist/workflows/exec/native-executor.js +82 -341
- package/dist/workflows/exec/param-secrets.js +29 -47
- package/dist/workflows/exec/run-workflow.js +154 -387
- package/dist/workflows/exec/scheduler.js +9 -36
- package/dist/workflows/exec/step-work.js +127 -430
- package/dist/workflows/exec/unit-dispatch.js +11 -63
- package/dist/workflows/exec/unit-writer.js +8 -52
- package/dist/workflows/exec/worktree.js +39 -273
- package/dist/workflows/freeze/child-output-references.js +4 -15
- package/dist/workflows/freeze/environment.js +99 -92
- package/dist/workflows/freeze/freeze.js +172 -0
- package/dist/workflows/freeze/step-values.js +19 -21
- package/dist/workflows/freeze/targets/child-workflow.js +23 -92
- package/dist/workflows/freeze/targets/command.js +10 -33
- package/dist/workflows/freeze/targets/script.js +5 -12
- package/dist/workflows/freeze/targets/shell.js +3 -6
- package/dist/workflows/freeze/targets/task.js +25 -80
- package/dist/workflows/freeze/task-bindings.js +20 -67
- package/dist/workflows/{source-ir/github-yaml.js → github-yaml.js} +88 -206
- package/dist/workflows/ir/params.js +6 -51
- package/dist/workflows/ir/plan-hash.js +2 -34
- package/dist/workflows/parser.js +140 -43
- package/dist/{commands/improve/consolidate/types.js → workflows/plan.js} +2 -1
- package/dist/workflows/renderer.js +36 -69
- package/dist/workflows/resource-limits.js +12 -120
- package/dist/workflows/runtime/agent-identity.js +8 -40
- package/dist/workflows/runtime/run-outputs.js +3 -6
- package/dist/workflows/runtime/run-plan.js +316 -0
- package/dist/workflows/runtime/runs.js +48 -200
- package/dist/workflows/runtime/workflow-asset-loader.js +24 -57
- package/dist/workflows/{source-ir/semantics.js → source-semantics.js} +16 -20
- package/dist/workflows/validate-summary.js +2 -7
- package/docs/integration/bundling-akm.md +49 -42
- package/docs/migration/README.md +1 -0
- package/docs/migration/release-notes/0.9.17.md +41 -0
- package/docs/migration/v0.9.1-to-v0.9.2.md +19 -7
- package/docs/reference/cli.md +182 -125
- package/docs/reference/configuration.md +49 -56
- package/docs/reference/data-and-telemetry.md +19 -20
- package/docs/reference/tasks.md +86 -38
- package/docs/reference/workflow-schema.md +14 -18
- package/docs/reference/workflows.md +6 -9
- package/package.json +1 -1
- package/schemas/akm-config.json +87 -406
- package/dist/commands/health/advisories.js +0 -150
- package/dist/commands/health/metrics.js +0 -329
- package/dist/commands/health/surfaces.js +0 -102
- package/dist/commands/improve/anti-collapse.js +0 -83
- package/dist/commands/improve/collapse-detector.js +0 -432
- package/dist/commands/improve/consolidate/eligibility.js +0 -48
- package/dist/commands/improve/consolidate/merge.js +0 -146
- package/dist/commands/improve/distill/promote-memory.js +0 -329
- package/dist/commands/improve/distill/quality-gate.js +0 -500
- package/dist/commands/improve/memory/memory-contradiction-detect.js +0 -291
- package/dist/commands/improve/proposal-envelope.js +0 -31
- package/dist/commands/improve/run-context.js +0 -123
- package/dist/commands/improve/shared.js +0 -21
- package/dist/commands/improve/source-identity.js +0 -28
- package/dist/commands/improve/triage.js +0 -96
- package/dist/commands/proposal/drain-policies.js +0 -151
- package/dist/commands/sources/update-transaction.js +0 -220
- package/dist/core/action-contributors.js +0 -28
- package/dist/core/config/config-version-shim.js +0 -101
- package/dist/core/config/retired-experimental-keys-shim.js +0 -62
- package/dist/core/fs-txn.js +0 -405
- package/dist/core/lexical-score.js +0 -25
- package/dist/core/maintenance-barrier.js +0 -167
- package/dist/execution/executable-identity.js +0 -105
- package/dist/execution/guarded-source.js +0 -427
- package/dist/indexer/graph/graph-boost.js +0 -427
- package/dist/indexer/graph/graph-dedup.js +0 -95
- package/dist/indexer/search/name-match.js +0 -35
- package/dist/indexer/search/ranking-contributors.js +0 -515
- package/dist/indexer/walk/project-context.js +0 -192
- package/dist/integrations/agent/execution-cascade.js +0 -566
- package/dist/integrations/agent/execution-definitions.js +0 -202
- package/dist/integrations/agent/execution-lowering.js +0 -841
- package/dist/integrations/agent/execution-preparation.js +0 -98
- package/dist/integrations/agent/inline-execution.js +0 -74
- package/dist/registry/create-provider-registry.js +0 -29
- package/dist/registry/pinned-request-helper.js +0 -247
- package/dist/registry/pinned-transport.js +0 -717
- package/dist/sources/providers/index.js +0 -14
- package/dist/storage/engines/sqlite-migrations.js +0 -271
- package/dist/storage/repositories/canaries-repository.js +0 -107
- package/dist/storage/repositories/embedding-salvage-repository.js +0 -184
- package/dist/storage/repositories/registry-cache.js +0 -113
- package/dist/tasks/scheduler-sync-preview.js +0 -52
- package/dist/workflows/freeze/resolve-steps.js +0 -86
- package/dist/workflows/freeze/source-freeze.js +0 -64
- package/dist/workflows/ir/compile.js +0 -321
- package/dist/workflows/ir/environment-v4.js +0 -330
- package/dist/workflows/ir/freeze-v4.js +0 -153
- package/dist/workflows/ir/schema-v4.js +0 -745
- package/dist/workflows/ir/schema.js +0 -354
- package/dist/workflows/program/schema.js +0 -78
- package/dist/workflows/runtime/checkin.js +0 -57
- package/dist/workflows/runtime/plan-classifier.js +0 -196
- package/dist/workflows/runtime/unit-checkin.js +0 -45
- package/dist/workflows/runtime/unit-phases.js +0 -20
- package/dist/workflows/schema.js +0 -4
- package/dist/workflows/source-ir/compile.js +0 -200
- package/dist/workflows/source-ir/program.js +0 -50
- package/dist/workflows/source-ir/result.js +0 -26
- package/dist/workflows/source-ir/schema.js +0 -786
- package/dist/workflows/source-ir/triggers.js +0 -79
- package/dist/workflows/source-ir/uses.js +0 -40
- package/dist/workflows/validator.js +0 -60
|
@@ -8,24 +8,27 @@
|
|
|
8
8
|
* high-signal set of stash + registry hits and enrich each with the data
|
|
9
9
|
* needed to act (ref, run, parameters, follow-up command).
|
|
10
10
|
*
|
|
11
|
-
*
|
|
12
|
-
*
|
|
13
|
-
*
|
|
14
|
-
*
|
|
11
|
+
* Curation is one search with the fused ranking, the top `limit` hits, and
|
|
12
|
+
* per-hit enrichment (preview, run and parameters, graph support refs). An
|
|
13
|
+
* optional reranker reorders the top fused candidates first.
|
|
14
|
+
*
|
|
15
|
+
* The exported `akmCurate()` API is the single entry point; tests can also
|
|
16
|
+
* drive `curateSearchResults` with a fixture search response.
|
|
15
17
|
*/
|
|
16
18
|
import fs from "node:fs";
|
|
17
19
|
import { parseFrontmatter } from "../../core/asset/frontmatter.js";
|
|
18
|
-
import { parseRefInput } from "../../core/asset/resolve-ref.js";
|
|
19
20
|
import { getIndexPassConfig, loadConfig } from "../../core/config/config.js";
|
|
20
21
|
import { rethrowIfTestIsolationError, UsageError } from "../../core/errors.js";
|
|
21
22
|
import { appendEvent } from "../../core/events.js";
|
|
23
|
+
import { redactCredentialPatterns } from "../../core/redaction.js";
|
|
22
24
|
import { withStateDbTelemetry } from "../../core/state-db.js";
|
|
23
25
|
import { enqueueGraphExtraction, hasGraphData } from "../../indexer/db/graph-db.js";
|
|
26
|
+
import { searchHitContent } from "../../indexer/search/db-search.js";
|
|
24
27
|
import { copySearchHitAttribution, getSearchHitAttribution, usageEventAttributionMetadata, } from "../../indexer/search/search-attribution.js";
|
|
25
28
|
import { findSourceForPath, resolveSourceEntries } from "../../indexer/search/search-source.js";
|
|
26
29
|
import { insertUsageEvent } from "../../indexer/usage/usage-events.js";
|
|
27
30
|
import { estimateTokenCount } from "../../llm/embedders/remote.js";
|
|
28
|
-
import { tryLlmFeature } from "../../llm/feature-gate.js";
|
|
31
|
+
import { isLlmFeatureEnabled, tryLlmFeature } from "../../llm/feature-gate.js";
|
|
29
32
|
import { rerankDocuments } from "../../llm/rerank-client.js";
|
|
30
33
|
import { truncateDescription } from "../../output/shapes/helpers.js";
|
|
31
34
|
import { TELEMETRY_BUSY_TIMEOUT_MS, withIndexDb } from "../../storage/repositories/index-db.js";
|
|
@@ -33,40 +36,23 @@ import { findEntryIdByRef, getItemRefById } from "../../storage/repositories/ind
|
|
|
33
36
|
import { computeBodyHash } from "../../storage/repositories/index-llm-cache-repository.js";
|
|
34
37
|
import { akmSearch, parseSearchSource } from "./search.js";
|
|
35
38
|
import { akmShowUnified } from "./show.js";
|
|
36
|
-
const CURATE_FALLBACK_FILTER_WORDS = new Set([
|
|
37
|
-
"a",
|
|
38
|
-
"an",
|
|
39
|
-
"and",
|
|
40
|
-
"for",
|
|
41
|
-
"how",
|
|
42
|
-
"i",
|
|
43
|
-
"in",
|
|
44
|
-
"of",
|
|
45
|
-
"or",
|
|
46
|
-
"the",
|
|
47
|
-
"to",
|
|
48
|
-
"with",
|
|
49
|
-
]);
|
|
50
|
-
const CURATE_SHORT_FALLBACK_TOKENS = new Set(["ai", "ci", "cd", "go", "js", "ts"]);
|
|
51
|
-
const MIN_CURATE_FALLBACK_TOKEN_LENGTH = 3;
|
|
52
|
-
const MAX_CURATE_FALLBACK_KEYWORDS = 6;
|
|
53
|
-
export const CURATE_SEARCH_LIMIT_MULTIPLIER = 4;
|
|
54
|
-
export const MIN_CURATE_SEARCH_LIMIT = 12;
|
|
55
39
|
const DEFAULT_CURATE_LIMIT = 4;
|
|
56
|
-
const CURATE_CLOSE_SCORE_BAND = 0.12;
|
|
57
|
-
// Used by `shouldRunCurateFallback` to judge whether the initial search
|
|
58
|
-
// already returned strong results (a different purpose from ranking/keeping
|
|
59
|
-
// hits — see that function for its own rationale).
|
|
60
|
-
const CURATE_RELATIVE_SCORE_FLOOR = 0.7;
|
|
61
|
-
const CURATE_FALLBACK_TOP_SCORE_THRESHOLD = 0.8;
|
|
62
|
-
const CURATE_FALLBACK_STRONG_SCORE_FLOOR = 0.35;
|
|
63
40
|
const MAX_CURATE_SUPPORT_REFS = 2;
|
|
64
|
-
|
|
41
|
+
/** Fused candidates the reranker reorders when `search.curateRerank.topN` is unset. */
|
|
42
|
+
const DEFAULT_CURATE_RERANK_TOP_N = 30;
|
|
43
|
+
/** Characters of name, description and content sent to the reranker per candidate. */
|
|
44
|
+
const RERANK_DOCUMENT_CHARS = 2000;
|
|
65
45
|
/**
|
|
66
46
|
* Fire-and-forget: log a curate event to the usage_events table and events.jsonl.
|
|
67
47
|
* Never blocks the caller; errors are silently ignored.
|
|
68
48
|
*/
|
|
69
|
-
function logCurateEvent(
|
|
49
|
+
function logCurateEvent(rawQuery, result, eventSource = "user", attributionProjection = "full") {
|
|
50
|
+
// Credentials pasted into a query (e.g. by the Claude Code hook that
|
|
51
|
+
// curates every user prompt) must never reach state.db verbatim — see
|
|
52
|
+
// `redactCredentialPatterns`. Redacted once here so every persistence call
|
|
53
|
+
// below (events.metadata_json via appendEvent, usage_events.query via
|
|
54
|
+
// insertUsageEvent) gets the same scrubbed text.
|
|
55
|
+
const query = redactCredentialPatterns(rawQuery);
|
|
70
56
|
const itemRefs = result.items.map((item) => ("ref" in item ? item.ref : `registry:${item.id}`));
|
|
71
57
|
appendEvent({
|
|
72
58
|
eventType: "curate",
|
|
@@ -121,11 +107,13 @@ export async function akmCurate(options) {
|
|
|
121
107
|
const limit = options.limit && options.limit > 0 ? options.limit : DEFAULT_CURATE_LIMIT;
|
|
122
108
|
const source = options.source ?? parseSearchSource("local");
|
|
123
109
|
const searchResponse = options.searchResponse ??
|
|
124
|
-
(await
|
|
110
|
+
(await akmSearch({
|
|
125
111
|
query: options.query,
|
|
126
112
|
type: options.type,
|
|
127
|
-
|
|
113
|
+
// An enabled reranker reorders the top fused candidates, not just the final `limit`.
|
|
114
|
+
limit: Math.max(limit, rerankTopN(loadConfig())),
|
|
128
115
|
source,
|
|
116
|
+
skipLogging: true,
|
|
129
117
|
}));
|
|
130
118
|
const result = await curateSearchResults(options.query, searchResponse, limit, options.type, options.eventSource);
|
|
131
119
|
if (!options.skipLogging) {
|
|
@@ -136,27 +124,18 @@ export async function akmCurate(options) {
|
|
|
136
124
|
export async function curateSearchResults(query, result, limit, selectedType, eventSource) {
|
|
137
125
|
const allStashHits = result.hits.filter((hit) => hit.type !== "registry");
|
|
138
126
|
const registryHits = result.registryHits ?? [];
|
|
139
|
-
// F3/R-018: `--type`
|
|
140
|
-
//
|
|
141
|
-
//
|
|
142
|
-
//
|
|
143
|
-
// then running the SAME pipeline over it gives `--type` its intended
|
|
144
|
-
// "narrow, don't disable" semantics — and matters even when the caller's
|
|
145
|
-
// search already applied a type filter at the DB layer, because
|
|
146
|
-
// `curateSearchResults` is also driven directly (tests, `searchResponse`
|
|
147
|
-
// fixtures) with a `SearchResponse` that was never type-filtered.
|
|
127
|
+
// F3/R-018: `--type` NARROWS the candidate pool. The caller's search
|
|
128
|
+
// usually applied the filter already, but `curateSearchResults` is also
|
|
129
|
+
// driven directly (tests, `searchResponse` fixtures) with a
|
|
130
|
+
// `SearchResponse` that was never type-filtered.
|
|
148
131
|
const stashHits = selectedType && selectedType !== "any" ? allStashHits.filter((hit) => hit.type === selectedType) : allStashHits;
|
|
149
|
-
const
|
|
150
|
-
const selectedStashHits = await maybeRerankCuratedStashHits(query, selected.selected);
|
|
151
|
-
const supportRefsByRef = selected.supportRefsByRef;
|
|
132
|
+
const selectedStashHits = (await maybeRerankCuratedStashHits(query, stashHits)).slice(0, limit);
|
|
152
133
|
// F4/R-019: respect `--limit` for registry fill instead of hard-capping it
|
|
153
134
|
// at a bare literal 2 — the remaining slots after stash hits ARE the cap.
|
|
154
135
|
const selectedRegistryHits = selectedStashHits.length >= limit ? [] : registryHits.slice(0, limit - selectedStashHits.length);
|
|
155
136
|
const selectedRefs = new Set(selectedStashHits.map((hit) => hit.ref));
|
|
156
137
|
const items = [
|
|
157
|
-
...(await Promise.all(selectedStashHits
|
|
158
|
-
.slice(0, limit)
|
|
159
|
-
.map((hit) => enrichCuratedStashHit(query, hit, supportRefsByRef.get(hit.ref) ?? [], selectedRefs, eventSource)))),
|
|
138
|
+
...(await Promise.all(selectedStashHits.map((hit) => enrichCuratedStashHit(query, hit, selectedRefs, eventSource)))),
|
|
160
139
|
...selectedRegistryHits.map((hit) => buildCuratedRegistryItem(query, hit)),
|
|
161
140
|
].slice(0, limit);
|
|
162
141
|
return {
|
|
@@ -214,7 +193,7 @@ export async function packCuratedHits(result, budgetTokens) {
|
|
|
214
193
|
}
|
|
215
194
|
return { query: result.query, budget: budgetTokens, tokens: used, items: packed };
|
|
216
195
|
}
|
|
217
|
-
async function enrichCuratedStashHit(query, hit,
|
|
196
|
+
async function enrichCuratedStashHit(query, hit, selectedRefs, eventSource) {
|
|
218
197
|
let shown;
|
|
219
198
|
try {
|
|
220
199
|
shown = await akmShowUnified({ ref: hit.ref, eventSource, skipLogging: true });
|
|
@@ -229,7 +208,7 @@ async function enrichCuratedStashHit(query, hit, supportRefs, selectedRefs, even
|
|
|
229
208
|
maybeEnqueueLazyGraph(shown.path);
|
|
230
209
|
const description = shown?.description ?? hit.description;
|
|
231
210
|
const preview = buildCuratedPreview(shown, hit);
|
|
232
|
-
const
|
|
211
|
+
const supportRefs = buildCurateSupportRefs(shown?.related?.hits, selectedRefs, hit.ref);
|
|
233
212
|
const item = {
|
|
234
213
|
source: "local",
|
|
235
214
|
type: shown?.type ?? hit.type,
|
|
@@ -245,7 +224,7 @@ async function enrichCuratedStashHit(query, hit, supportRefs, selectedRefs, even
|
|
|
245
224
|
...(shown?.keys?.length ? { keys: shown.keys } : {}),
|
|
246
225
|
...(shown?.parameters?.length ? { parameters: shown.parameters } : {}),
|
|
247
226
|
...(shown?.run ? { run: shown.run } : {}),
|
|
248
|
-
...(
|
|
227
|
+
...(supportRefs.length > 0 ? { supportRefs } : {}),
|
|
249
228
|
followUp: `akm show ${hit.ref}`,
|
|
250
229
|
reason: buildCuratedReason(query, shown?.type ?? hit.type),
|
|
251
230
|
...(hit.score !== undefined ? { score: hit.score } : {}),
|
|
@@ -341,274 +320,20 @@ function buildCurateSummary(query, items) {
|
|
|
341
320
|
const labels = items.map((item) => ("ref" in item ? item.ref : `${item.type}:${item.name}`));
|
|
342
321
|
return `Selected ${items.length} curated result${items.length === 1 ? "" : "s"}: ${labels.join(", ")}.`;
|
|
343
322
|
}
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
const tokens = Array.from(new Set(query
|
|
350
|
-
.toLowerCase()
|
|
351
|
-
.split(/[^a-z0-9]+/)
|
|
352
|
-
.map((token) => token.trim())
|
|
353
|
-
.filter((token) => token.length > 0 &&
|
|
354
|
-
!CURATE_FALLBACK_FILTER_WORDS.has(token) &&
|
|
355
|
-
(token.length >= MIN_CURATE_FALLBACK_TOKEN_LENGTH || CURATE_SHORT_FALLBACK_TOKENS.has(token))))).slice(0, MAX_CURATE_FALLBACK_KEYWORDS);
|
|
356
|
-
if (tokens.length === 1 && tokens[0] === normalizedWhole)
|
|
357
|
-
return [];
|
|
358
|
-
return tokens;
|
|
359
|
-
}
|
|
360
|
-
export function mergeCurateSearchResponses(base, extras) {
|
|
361
|
-
// The base (full-query) ranking is authoritative by default; keyword
|
|
362
|
-
// fallbacks add recall without comparing their raw scores to the full-query
|
|
363
|
-
// score scale. Base order is preserved and fallback-only hits append below,
|
|
364
|
-
// except for the narrow weak-provenance promotion documented below. Duplicate
|
|
365
|
-
// refs keep their base position but take the maximum score for the downstream
|
|
366
|
-
// relevance floor.
|
|
367
|
-
const bestExtraStashScore = new Map();
|
|
368
|
-
for (const result of extras) {
|
|
369
|
-
for (const hit of result.hits.filter((entry) => entry.type !== "registry")) {
|
|
370
|
-
const prev = bestExtraStashScore.get(hit.ref);
|
|
371
|
-
if (prev === undefined || (hit.score ?? 0) > prev)
|
|
372
|
-
bestExtraStashScore.set(hit.ref, hit.score ?? 0);
|
|
373
|
-
}
|
|
374
|
-
}
|
|
375
|
-
const baseRefs = new Set();
|
|
376
|
-
const baseStash = [];
|
|
377
|
-
for (const hit of base.hits.filter((entry) => entry.type !== "registry")) {
|
|
378
|
-
baseRefs.add(hit.ref);
|
|
379
|
-
const extraScore = bestExtraStashScore.get(hit.ref);
|
|
380
|
-
baseStash.push(extraScore !== undefined && extraScore > (hit.score ?? 0) ? { ...hit, score: extraScore } : hit);
|
|
381
|
-
}
|
|
382
|
-
const extraOnly = new Map();
|
|
383
|
-
for (const result of extras) {
|
|
384
|
-
for (const hit of result.hits.filter((entry) => entry.type !== "registry")) {
|
|
385
|
-
if (baseRefs.has(hit.ref))
|
|
386
|
-
continue;
|
|
387
|
-
const existing = extraOnly.get(hit.ref);
|
|
388
|
-
if (!existing || (hit.score ?? 0) > (existing.score ?? 0))
|
|
389
|
-
extraOnly.set(hit.ref, hit);
|
|
390
|
-
}
|
|
391
|
-
}
|
|
392
|
-
// Fallback-only hits ordinarily stay below every full-query hit through the
|
|
393
|
-
// downstream selector, which re-sorts by score and derives its relevance
|
|
394
|
-
// floor from the leader. Restamping them below the minimum base score keeps a
|
|
395
|
-
// single-token score from leapfrogging contextual results. The one exception
|
|
396
|
-
// is a strong name match repairing a lexically weak base, selected below.
|
|
397
|
-
const sortedExtra = [...extraOnly.values()].sort((a, b) => (b.score ?? 0) - (a.score ?? 0));
|
|
398
|
-
const lexicalAttribution = (hit) => getSearchHitAttribution(hit)?.lexical;
|
|
399
|
-
// The full-query base remains authoritative unless its lexical provenance is
|
|
400
|
-
// itself weak: relaxed recovery, or no hit whose name covers the query. In
|
|
401
|
-
// that case one fallback discovery with strong name coverage may lead. This
|
|
402
|
-
// preserves contextual base ranking while allowing measured recall to repair
|
|
403
|
-
// a thin match without comparing unrelated raw FTS score scales.
|
|
404
|
-
const weakLexicalBase = baseStash.length > 0 &&
|
|
405
|
-
baseStash.every((hit) => {
|
|
406
|
-
const lexical = lexicalAttribution(hit);
|
|
407
|
-
return lexical !== undefined && (lexical.execution === "relaxed" || lexical.nameMatchTier < 2);
|
|
408
|
-
});
|
|
409
|
-
const promotedExtraIndex = weakLexicalBase
|
|
410
|
-
? sortedExtra.findIndex((hit) => (lexicalAttribution(hit)?.nameMatchTier ?? 0) >= 2)
|
|
411
|
-
: -1;
|
|
412
|
-
const promotedExtra = promotedExtraIndex >= 0 ? sortedExtra[promotedExtraIndex] : undefined;
|
|
413
|
-
const remainingExtra = promotedExtraIndex >= 0 ? sortedExtra.filter((_, index) => index !== promotedExtraIndex) : sortedExtra;
|
|
414
|
-
const minBaseScore = baseStash.length
|
|
415
|
-
? Math.min(...baseStash.map((hit) => hit.score ?? 0))
|
|
416
|
-
: Number.POSITIVE_INFINITY;
|
|
417
|
-
const cappedExtra = baseStash.length
|
|
418
|
-
? remainingExtra.map((hit, i) => ({ ...hit, score: minBaseScore - 1e-6 * (i + 1) }))
|
|
419
|
-
: remainingExtra;
|
|
420
|
-
const mergedHits = [
|
|
421
|
-
...(promotedExtra ? [{ ...promotedExtra, score: minBaseScore }] : []),
|
|
422
|
-
...baseStash,
|
|
423
|
-
...cappedExtra,
|
|
424
|
-
];
|
|
425
|
-
// Registry hits are supplemental fill — same rule: base first (max score on
|
|
426
|
-
// dups), then fallback-only registry hits appended by score.
|
|
427
|
-
const bestExtraRegScore = new Map();
|
|
428
|
-
for (const result of extras) {
|
|
429
|
-
for (const hit of result.registryHits ?? []) {
|
|
430
|
-
const prev = bestExtraRegScore.get(hit.id);
|
|
431
|
-
if (prev === undefined || (hit.score ?? 0) > prev)
|
|
432
|
-
bestExtraRegScore.set(hit.id, hit.score ?? 0);
|
|
433
|
-
}
|
|
434
|
-
}
|
|
435
|
-
const baseRegIds = new Set();
|
|
436
|
-
const baseReg = [];
|
|
437
|
-
for (const hit of base.registryHits ?? []) {
|
|
438
|
-
baseRegIds.add(hit.id);
|
|
439
|
-
const extraScore = bestExtraRegScore.get(hit.id);
|
|
440
|
-
baseReg.push(extraScore !== undefined && extraScore > (hit.score ?? 0) ? { ...hit, score: extraScore } : hit);
|
|
441
|
-
}
|
|
442
|
-
const extraRegOnly = new Map();
|
|
443
|
-
for (const result of extras) {
|
|
444
|
-
for (const hit of result.registryHits ?? []) {
|
|
445
|
-
if (baseRegIds.has(hit.id))
|
|
446
|
-
continue;
|
|
447
|
-
const existing = extraRegOnly.get(hit.id);
|
|
448
|
-
if (!existing || (hit.score ?? 0) > (existing.score ?? 0))
|
|
449
|
-
extraRegOnly.set(hit.id, hit);
|
|
450
|
-
}
|
|
451
|
-
}
|
|
452
|
-
const warnings = Array.from(new Set([...(base.warnings ?? []), ...extras.flatMap((result) => result.warnings ?? [])]));
|
|
453
|
-
const mergedRegistryHits = [
|
|
454
|
-
...baseReg,
|
|
455
|
-
...[...extraRegOnly.values()].sort((a, b) => (b.score ?? 0) - (a.score ?? 0)),
|
|
456
|
-
];
|
|
457
|
-
const modes = [base.searchMode, ...extras.map((result) => result.searchMode)].filter((mode) => mode !== undefined);
|
|
458
|
-
const searchMode = modes.includes("fts-fallback")
|
|
459
|
-
? "fts-fallback"
|
|
460
|
-
: modes.includes("semantic")
|
|
461
|
-
? "semantic"
|
|
462
|
-
: modes.includes("keyword")
|
|
463
|
-
? "keyword"
|
|
464
|
-
: undefined;
|
|
465
|
-
return {
|
|
466
|
-
...base,
|
|
467
|
-
hits: mergedHits,
|
|
468
|
-
...(mergedRegistryHits.length > 0 ? { registryHits: mergedRegistryHits } : {}),
|
|
469
|
-
...(warnings.length > 0 ? { warnings } : {}),
|
|
470
|
-
...(searchMode ? { searchMode } : {}),
|
|
471
|
-
...(mergedHits.length > 0 || mergedRegistryHits.length > 0 ? { tip: undefined } : {}),
|
|
472
|
-
};
|
|
473
|
-
}
|
|
474
|
-
export async function searchForCuration(input) {
|
|
475
|
-
const initial = await akmSearch({ ...input, skipLogging: true });
|
|
476
|
-
if (!shouldRunCurateFallback(initial, input.limit))
|
|
477
|
-
return initial;
|
|
478
|
-
const fallbackQueries = deriveCurateFallbackQueries(input.query);
|
|
479
|
-
if (fallbackQueries.length === 0)
|
|
480
|
-
return initial;
|
|
481
|
-
const fallbackResults = await Promise.all(fallbackQueries.map((token) => akmSearch({
|
|
482
|
-
query: token,
|
|
483
|
-
type: input.type,
|
|
484
|
-
limit: input.limit,
|
|
485
|
-
source: input.source,
|
|
486
|
-
skipLogging: true,
|
|
487
|
-
})));
|
|
488
|
-
return mergeCurateSearchResponses(initial, fallbackResults);
|
|
489
|
-
}
|
|
490
|
-
function parseCurateIntent(query) {
|
|
491
|
-
const lower = query.toLowerCase();
|
|
492
|
-
return {
|
|
493
|
-
executionHeavy: /(run|script|bash|shell|cli|execute|automation|deploy|build|test|lint)/.test(lower),
|
|
494
|
-
multiStep: /(plan|workflow|steps?|procedure|rollout|review|migration|release|checklist)/.test(lower),
|
|
495
|
-
delegation: /(agent|assistant|planner|reviewer|architect|prompt)/.test(lower),
|
|
496
|
-
recall: /(memory|context|recall|remember)/.test(lower),
|
|
497
|
-
reference: CURATE_REFERENCE_QUERY_RE.test(lower),
|
|
498
|
-
};
|
|
499
|
-
}
|
|
500
|
-
function computeCurateTypeNudge(type, intent) {
|
|
501
|
-
let nudge = 0;
|
|
502
|
-
if (intent.executionHeavy) {
|
|
503
|
-
if (type === "script")
|
|
504
|
-
nudge += 0.06;
|
|
505
|
-
else if (type === "command")
|
|
506
|
-
nudge += 0.04;
|
|
507
|
-
else if (type === "memory")
|
|
508
|
-
nudge -= 0.04;
|
|
509
|
-
}
|
|
510
|
-
if (intent.multiStep) {
|
|
511
|
-
if (type === "workflow")
|
|
512
|
-
nudge += 0.06;
|
|
513
|
-
else if (type === "skill")
|
|
514
|
-
nudge += 0.04;
|
|
515
|
-
else if (type === "knowledge")
|
|
516
|
-
nudge -= 0.02;
|
|
517
|
-
}
|
|
518
|
-
if (intent.delegation && type === "agent")
|
|
519
|
-
nudge += 0.06;
|
|
520
|
-
if (intent.recall && type === "memory")
|
|
521
|
-
nudge += 0.08;
|
|
522
|
-
if (intent.reference) {
|
|
523
|
-
if (type === "knowledge")
|
|
524
|
-
nudge += 0.05;
|
|
525
|
-
else if (type === "skill")
|
|
526
|
-
nudge += 0.02;
|
|
527
|
-
}
|
|
528
|
-
return nudge;
|
|
529
|
-
}
|
|
530
|
-
function getCurateFamily(ref) {
|
|
531
|
-
try {
|
|
532
|
-
// F4b: `ref` is a search-hit ref in the 0.9.0 conceptId grammar — parse via
|
|
533
|
-
// the new-grammar `parseRefInput` so skill/reference family grouping still
|
|
534
|
-
// recognizes it. Search may add an opaque Markdown selector; identity and
|
|
535
|
-
// family ownership are on the parent asset, not that selector.
|
|
536
|
-
const parsed = parseRefInput(ref.split("#", 1)[0]);
|
|
537
|
-
if (parsed.type === "skill") {
|
|
538
|
-
return { key: parsed.name, role: "root" };
|
|
539
|
-
}
|
|
540
|
-
if (parsed.type !== "knowledge")
|
|
541
|
-
return undefined;
|
|
542
|
-
const match = /^skills\/(.+?)\/references\/(.+)$/.exec(parsed.name);
|
|
543
|
-
if (!match)
|
|
544
|
-
return undefined;
|
|
545
|
-
return {
|
|
546
|
-
key: match[1],
|
|
547
|
-
role: "reference",
|
|
548
|
-
topicTokens: match[2]
|
|
549
|
-
.split(/[^a-z0-9]+/i)
|
|
550
|
-
.map((token) => token.trim().toLowerCase())
|
|
551
|
-
.filter(Boolean),
|
|
552
|
-
};
|
|
553
|
-
}
|
|
554
|
-
catch {
|
|
555
|
-
return undefined;
|
|
556
|
-
}
|
|
557
|
-
}
|
|
558
|
-
function annotateCurateHit(query, hit, index, intent) {
|
|
559
|
-
const rawScore = hit.score ?? 0;
|
|
560
|
-
const family = getCurateFamily(hit.ref);
|
|
561
|
-
let adjustedScore = rawScore + computeCurateTypeNudge(hit.type, intent);
|
|
562
|
-
if (family?.role === "root" && !isNarrowReferenceFamilyQuery(query, family))
|
|
563
|
-
adjustedScore += 0.07;
|
|
564
|
-
if (family?.role === "reference" && isNarrowReferenceFamilyQuery(query, family))
|
|
565
|
-
adjustedScore += 0.07;
|
|
566
|
-
return {
|
|
567
|
-
hit,
|
|
568
|
-
rawScore,
|
|
569
|
-
adjustedScore,
|
|
570
|
-
originalIndex: index,
|
|
571
|
-
family,
|
|
572
|
-
};
|
|
573
|
-
}
|
|
574
|
-
function compareCurateHits(a, b) {
|
|
575
|
-
const rawDiff = b.rawScore - a.rawScore;
|
|
576
|
-
if (Math.abs(rawDiff) > CURATE_CLOSE_SCORE_BAND)
|
|
577
|
-
return rawDiff;
|
|
578
|
-
const adjustedDiff = b.adjustedScore - a.adjustedScore;
|
|
579
|
-
if (adjustedDiff !== 0)
|
|
580
|
-
return adjustedDiff;
|
|
581
|
-
if (rawDiff !== 0)
|
|
582
|
-
return rawDiff;
|
|
583
|
-
return a.originalIndex - b.originalIndex;
|
|
584
|
-
}
|
|
585
|
-
function isNarrowReferenceFamilyQuery(query, family) {
|
|
586
|
-
if (!family || family.role !== "reference")
|
|
587
|
-
return false;
|
|
588
|
-
const lower = query.toLowerCase();
|
|
589
|
-
if (CURATE_REFERENCE_QUERY_RE.test(lower))
|
|
590
|
-
return true;
|
|
591
|
-
return family.topicTokens.some((token) => token.length >= 3 && lower.includes(token));
|
|
323
|
+
/** How many fused candidates curate fetches for the reranker; 0 when reranking is off. */
|
|
324
|
+
function rerankTopN(config) {
|
|
325
|
+
if (!isLlmFeatureEnabled(config, "curate_rerank"))
|
|
326
|
+
return 0;
|
|
327
|
+
return config.search?.curateRerank?.topN ?? DEFAULT_CURATE_RERANK_TOP_N;
|
|
592
328
|
}
|
|
593
|
-
function appendCurateSupportRef(supportRefsByRef, ownerRef, supportRef) {
|
|
594
|
-
const existing = supportRefsByRef.get(ownerRef) ?? [];
|
|
595
|
-
if (existing.some((entry) => entry.ref === supportRef.ref))
|
|
596
|
-
return;
|
|
597
|
-
supportRefsByRef.set(ownerRef, [...existing, supportRef]);
|
|
598
|
-
}
|
|
599
|
-
/** Default number of `selectCuratedStashHits` candidates sent to the reranker when `search.curateRerank.topN` isn't set. */
|
|
600
|
-
const DEFAULT_CURATE_RERANK_TOP_N = 8;
|
|
601
329
|
/**
|
|
602
|
-
* Optional cross-encoder rerank
|
|
603
|
-
*
|
|
604
|
-
*
|
|
605
|
-
*
|
|
606
|
-
*
|
|
607
|
-
*
|
|
608
|
-
*
|
|
609
|
-
* Only the top `topN` (default {@link DEFAULT_CURATE_RERANK_TOP_N}) already-
|
|
610
|
-
* selected hits are sent (bounded request size); anything past that keeps its
|
|
611
|
-
* original position appended after the reranked prefix.
|
|
330
|
+
* Optional cross-encoder rerank of the top fused candidates (#951). Disabled
|
|
331
|
+
* by default (`search.curateRerank.enabled`) and, when enabled, best-effort:
|
|
332
|
+
* any failure (misconfigured endpoint, network error, timeout, malformed
|
|
333
|
+
* response) keeps the fused order — a reranker outage must never turn into a
|
|
334
|
+
* curate failure. The top `topN` candidates (default
|
|
335
|
+
* {@link DEFAULT_CURATE_RERANK_TOP_N}) are sent as name, description and the
|
|
336
|
+
* start of the indexed content; the rest keep their fused order after them.
|
|
612
337
|
*/
|
|
613
338
|
async function maybeRerankCuratedStashHits(query, hits) {
|
|
614
339
|
if (hits.length <= 1)
|
|
@@ -616,109 +341,34 @@ async function maybeRerankCuratedStashHits(query, hits) {
|
|
|
616
341
|
const config = loadConfig();
|
|
617
342
|
const rerankConfig = config.search?.curateRerank;
|
|
618
343
|
return tryLlmFeature("curate_rerank", config, async () => {
|
|
619
|
-
const
|
|
620
|
-
const
|
|
621
|
-
const
|
|
622
|
-
const documents = head.map((hit) => [hit.name, hit.description].filter(Boolean).join(" — "));
|
|
623
|
-
const ranked = await rerankDocuments(rerankConfig ?? {}, query, documents);
|
|
344
|
+
const head = hits.slice(0, rerankTopN(config));
|
|
345
|
+
const tail = hits.slice(head.length);
|
|
346
|
+
const ranked = await rerankDocuments(rerankConfig ?? {}, query, rerankDocumentTexts(head));
|
|
624
347
|
const rerankedHead = ranked
|
|
625
348
|
.map(({ index }) => head[index])
|
|
626
349
|
.filter((hit) => hit !== undefined);
|
|
627
350
|
return [...rerankedHead, ...tail];
|
|
628
351
|
}, hits, { timeoutMs: rerankConfig?.timeoutMs ?? null });
|
|
629
352
|
}
|
|
630
|
-
|
|
631
|
-
|
|
632
|
-
|
|
633
|
-
|
|
634
|
-
|
|
635
|
-
|
|
636
|
-
|
|
637
|
-
return { selected: ranked.slice(0, limit).map((entry) => entry.hit), supportRefsByRef };
|
|
638
|
-
}
|
|
639
|
-
function collapseCurateFamilies(query, hits) {
|
|
640
|
-
const passthrough = [];
|
|
641
|
-
const supportRefsByRef = new Map();
|
|
642
|
-
const groups = new Map();
|
|
643
|
-
for (const [index, hit] of hits.entries()) {
|
|
644
|
-
const family = getCurateFamily(hit.ref);
|
|
645
|
-
if (!family) {
|
|
646
|
-
passthrough.push({ hit, originalIndex: index });
|
|
647
|
-
continue;
|
|
648
|
-
}
|
|
649
|
-
const group = groups.get(family.key) ?? { references: [] };
|
|
650
|
-
if (family.role === "root") {
|
|
651
|
-
if (!group.root)
|
|
652
|
-
group.root = { hit, originalIndex: index };
|
|
653
|
-
}
|
|
654
|
-
else {
|
|
655
|
-
group.references.push({ hit, originalIndex: index });
|
|
656
|
-
}
|
|
657
|
-
groups.set(family.key, group);
|
|
658
|
-
}
|
|
659
|
-
const collapsedFamilies = [];
|
|
660
|
-
for (const group of groups.values()) {
|
|
661
|
-
const bestReference = group.references[0];
|
|
662
|
-
const representative = group.root && !isNarrowReferenceFamilyQuery(query, getCurateFamily(bestReference?.hit.ref ?? group.root.hit.ref))
|
|
663
|
-
? group.root
|
|
664
|
-
: (bestReference ?? group.root);
|
|
665
|
-
if (!representative)
|
|
666
|
-
continue;
|
|
667
|
-
collapsedFamilies.push(representative);
|
|
668
|
-
const supportCandidates = [group.root, ...group.references]
|
|
669
|
-
.filter((entry) => {
|
|
670
|
-
return entry !== undefined && entry.hit.ref !== representative.hit.ref;
|
|
671
|
-
})
|
|
672
|
-
.sort((a, b) => {
|
|
673
|
-
if (a === group.root)
|
|
674
|
-
return -1;
|
|
675
|
-
if (b === group.root)
|
|
676
|
-
return 1;
|
|
677
|
-
return a.hit.ref.localeCompare(b.hit.ref);
|
|
678
|
-
});
|
|
679
|
-
for (const support of supportCandidates) {
|
|
680
|
-
appendCurateSupportRef(supportRefsByRef, representative.hit.ref, {
|
|
681
|
-
ref: support.hit.ref,
|
|
682
|
-
type: support.hit.type,
|
|
683
|
-
reason: "Related family asset to inspect next.",
|
|
684
|
-
});
|
|
685
|
-
}
|
|
686
|
-
}
|
|
687
|
-
return {
|
|
688
|
-
hits: [...passthrough, ...collapsedFamilies].sort((a, b) => a.originalIndex - b.originalIndex),
|
|
689
|
-
supportRefsByRef,
|
|
690
|
-
};
|
|
353
|
+
/**
|
|
354
|
+
* Name, description and the start of each hit's indexed content, capped for
|
|
355
|
+
* the reranker. The content is the index's safe projection (env and secret
|
|
356
|
+
* values never reach it), never the raw file.
|
|
357
|
+
*/
|
|
358
|
+
function rerankDocumentTexts(hits) {
|
|
359
|
+
return hits.map((hit) => [hit.name, hit.description, searchHitContent(hit)].filter(Boolean).join("\n").slice(0, RERANK_DOCUMENT_CHARS));
|
|
691
360
|
}
|
|
692
|
-
|
|
693
|
-
|
|
694
|
-
|
|
695
|
-
|
|
696
|
-
continue;
|
|
697
|
-
if (merged.some((existing) => existing.ref === entry.ref))
|
|
698
|
-
continue;
|
|
699
|
-
merged.push(entry);
|
|
700
|
-
if (merged.length >= MAX_CURATE_SUPPORT_REFS)
|
|
701
|
-
return merged;
|
|
702
|
-
}
|
|
703
|
-
if (!Array.isArray(relatedHits))
|
|
704
|
-
return merged;
|
|
705
|
-
for (const hit of relatedHits) {
|
|
361
|
+
/** Up to {@link MAX_CURATE_SUPPORT_REFS} graph-related assets not already selected. */
|
|
362
|
+
function buildCurateSupportRefs(relatedHits, selectedRefs, ownerRef) {
|
|
363
|
+
const supportRefs = [];
|
|
364
|
+
for (const hit of relatedHits ?? []) {
|
|
706
365
|
if (!hit.ref || hit.ref === ownerRef || selectedRefs.has(hit.ref))
|
|
707
366
|
continue;
|
|
708
|
-
if (
|
|
367
|
+
if (supportRefs.some((existing) => existing.ref === hit.ref))
|
|
709
368
|
continue;
|
|
710
|
-
|
|
711
|
-
if (
|
|
369
|
+
supportRefs.push({ ref: hit.ref, type: hit.type, reason: "Related asset via shared entities." });
|
|
370
|
+
if (supportRefs.length >= MAX_CURATE_SUPPORT_REFS)
|
|
712
371
|
break;
|
|
713
372
|
}
|
|
714
|
-
return
|
|
715
|
-
}
|
|
716
|
-
function shouldRunCurateFallback(initial, desiredCount) {
|
|
717
|
-
const stashHits = initial.hits.filter((hit) => hit.type !== "registry");
|
|
718
|
-
if (stashHits.length === 0)
|
|
719
|
-
return true;
|
|
720
|
-
const topScore = stashHits[0]?.score ?? 0;
|
|
721
|
-
const strongFloor = Math.max(CURATE_FALLBACK_STRONG_SCORE_FLOOR, topScore * CURATE_RELATIVE_SCORE_FLOOR);
|
|
722
|
-
const strongCount = stashHits.filter((hit) => (hit.score ?? 0) >= strongFloor).length;
|
|
723
|
-
return !(topScore >= CURATE_FALLBACK_TOP_SCORE_THRESHOLD && strongCount >= Math.min(2, desiredCount));
|
|
373
|
+
return supportRefs;
|
|
724
374
|
}
|