akm-cli 0.9.16 → 0.9.17-alpha.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +2101 -0
- package/STABILITY.md +11 -10
- package/dist/akm +124 -193
- package/dist/akm-migrate +38 -19
- package/dist/assets/hints/cli-hints-full.md +6 -7
- package/dist/assets/improve-strategies/catchup.json +0 -3
- package/dist/assets/improve-strategies/consolidate.json +0 -1
- package/dist/assets/improve-strategies/default.json +1 -2
- package/dist/assets/improve-strategies/proactive-maintenance.json +1 -2
- package/dist/assets/improve-strategies/quick.json +1 -2
- package/dist/assets/improve-strategies/reflect-distill.json +1 -2
- package/dist/assets/improve-strategies/thorough.json +0 -3
- package/dist/assets/prompts/consolidate-pair.md +20 -0
- package/dist/assets/prompts/consolidate-system.md +4 -11
- package/dist/assets/prompts/retrieval-relevance-judge.md +6 -0
- package/dist/assets/stash-skeleton/facts/conventions/backlinks.md +20 -20
- package/dist/assets/stash-skeleton/facts/conventions/domains.md +2 -2
- package/dist/assets/templates/html/health.html +3 -5
- package/dist/cli/retired-commands.js +1 -1
- package/dist/cli/shared.js +6 -2
- package/dist/cli/unknown-flags.js +24 -1
- package/dist/cli.js +68 -10
- package/dist/commands/agent/agent-dispatch.js +1 -1
- package/dist/commands/command/command-execution.js +24 -62
- package/dist/commands/feedback-cli.js +0 -1
- package/dist/commands/health/accept-rate.js +6 -0
- package/dist/commands/health/archive-usage.js +92 -0
- package/dist/commands/health/checks.js +83 -74
- package/dist/commands/health/config-skew.js +38 -0
- package/dist/commands/health/data-dir-usage.js +25 -13
- package/dist/commands/health/egress.js +54 -0
- package/dist/commands/health/html-report.js +1 -42
- package/dist/commands/health/improve-metrics.js +136 -591
- package/dist/commands/health/md-report.js +1 -6
- package/dist/commands/health/plugin-staleness.js +53 -3
- package/dist/commands/health/renderers.js +12 -4
- package/dist/commands/health/report-view-model.js +14 -120
- package/dist/commands/health/types-improve.js +4 -19
- package/dist/commands/health/windows.js +64 -74
- package/dist/commands/health.js +145 -143
- package/dist/commands/improve/consolidate/chunking.js +26 -117
- package/dist/commands/improve/consolidate/continuity-check.js +137 -0
- package/dist/commands/improve/consolidate/pair-pass.js +791 -0
- package/dist/commands/improve/consolidate/sanitize.js +54 -149
- package/dist/commands/improve/consolidate.js +589 -1127
- package/dist/commands/improve/content-hash.js +16 -24
- package/dist/commands/improve/distill/content-repair.js +18 -100
- package/dist/commands/improve/distill-guards.js +20 -81
- package/dist/commands/improve/distill-promotion-policy.js +23 -243
- package/dist/commands/improve/distill.js +608 -1041
- package/dist/commands/improve/eligibility.js +126 -390
- package/dist/commands/improve/execution.js +8 -10
- package/dist/commands/improve/extract-prompt.js +1 -2
- package/dist/commands/improve/extract.js +487 -1046
- package/dist/commands/improve/feedback-valence.js +0 -25
- package/dist/commands/improve/improve-cli.js +75 -169
- package/dist/commands/improve/improve-result-file.js +10 -66
- package/dist/commands/improve/improve-strategies.js +52 -4
- package/dist/commands/improve/improve-usage-report.js +18 -64
- package/dist/commands/improve/improve.js +480 -1074
- package/dist/commands/improve/ledger.js +119 -0
- package/dist/commands/improve/locks.js +2 -8
- package/dist/commands/improve/loop-stages.js +415 -1073
- package/dist/commands/improve/memory/derived-ref.js +12 -77
- package/dist/commands/improve/memory/memory-belief.js +16 -118
- package/dist/commands/improve/memory/memory-improve.js +266 -14
- package/dist/commands/improve/outcome-loop.js +28 -156
- package/dist/commands/improve/planner.js +5 -15
- package/dist/commands/improve/preparation.js +779 -2319
- package/dist/commands/improve/proactive-maintenance.js +34 -101
- package/dist/commands/improve/reflect-noise.js +104 -280
- package/dist/commands/improve/reflect.js +642 -1353
- package/dist/commands/improve/retrieval-gate.js +127 -0
- package/dist/commands/improve/retrieval-scope.js +92 -0
- package/dist/commands/improve/salience.js +41 -240
- package/dist/commands/improve/session-asset.js +19 -100
- package/dist/commands/improve/stage.js +322 -0
- package/dist/commands/lint/base-linter.js +37 -15
- package/dist/commands/proposal/drain.js +261 -578
- package/dist/commands/proposal/proposal-cli.js +19 -20
- package/dist/commands/proposal/proposal-types.js +31 -24
- package/dist/commands/proposal/proposal.js +38 -8
- package/dist/commands/proposal/propose.js +134 -160
- package/dist/commands/proposal/repository.js +1097 -1394
- package/dist/commands/proposal/validators/proposal-quality-validators.js +71 -174
- package/dist/commands/proposal/validators/proposal-validators.js +1 -1
- package/dist/commands/proposal/validators/proposals.js +22 -89
- package/dist/commands/read/curate.js +105 -462
- package/dist/commands/read/knowledge.js +3 -2
- package/dist/commands/read/search-cli.js +16 -33
- package/dist/commands/read/search.js +17 -23
- package/dist/commands/read/show.js +57 -108
- package/dist/commands/sources/bundle-cli.js +25 -2
- package/dist/commands/sources/bundle-config-ops.js +4 -0
- package/dist/commands/sources/dangerous-env-audit.js +1 -2
- package/dist/commands/sources/info.js +127 -29
- package/dist/commands/sources/installed-stashes.js +197 -746
- package/dist/commands/sources/schema-repair.js +98 -129
- package/dist/commands/sources/source-add.js +62 -12
- package/dist/commands/sources/source-manage.js +9 -2
- package/dist/commands/sources/stash-cli.js +24 -4
- package/dist/commands/tasks/explain.js +10 -13
- package/dist/commands/tasks/tasks-cli.js +12 -13
- package/dist/commands/tasks/tasks.js +350 -936
- package/dist/commands/tasks/validate.js +26 -24
- package/dist/commands/workflow/plan.js +22 -29
- package/dist/commands/workflow-cli.js +4 -4
- package/dist/core/adapter/adapters/akm-adapter.js +2 -1
- package/dist/core/adapter/adapters/akm-lint.js +2 -3
- package/dist/core/adapter/adapters/akm-metadata.js +42 -12
- package/dist/core/adapter/adapters/akm-task-adapter.js +29 -8
- package/dist/core/adapter/adapters/akm-workflow-adapter.js +1 -1
- package/dist/core/adapter/execution-source.js +17 -29
- package/dist/core/asset/asset-placement.js +4 -13
- package/dist/core/asset/frontmatter.js +106 -1
- package/dist/core/asset/resolve-ref.js +1 -1
- package/dist/core/bundle-id.js +42 -5
- package/dist/core/bundle-rename.js +285 -0
- package/dist/core/config/config-io.js +1 -2
- package/dist/core/config/config-schema.js +9 -34
- package/dist/core/config/config-walker.js +1 -1
- package/dist/core/config/config.js +184 -111
- package/dist/core/config/engine-semantics.js +0 -2
- package/dist/core/config/legacy-source-shape-shim.js +38 -9
- package/dist/core/config/schema/embedding.js +20 -5
- package/dist/core/config/schema/engines.js +5 -0
- package/dist/core/config/schema/execution.js +1 -1
- package/dist/core/config/schema/experimental.js +1 -1
- package/dist/core/config/schema/improve-processes.js +54 -125
- package/dist/core/config/schema/improve.js +4 -42
- package/dist/core/config/schema/index-config.js +9 -48
- package/dist/core/config/schema/scheduler.js +12 -12
- package/dist/core/config/schema/search.js +6 -22
- package/dist/core/env-secret-ref.js +0 -1
- package/dist/core/errors.js +8 -9
- package/dist/core/file-change.js +13 -5
- package/dist/core/file-lock.js +76 -173
- package/dist/core/improve-result.js +35 -7
- package/dist/core/improve-types.js +0 -1
- package/dist/core/logs-db.js +2 -2
- package/dist/core/loopback.js +7 -12
- package/dist/core/non-task-input.js +20 -0
- package/dist/core/parse.js +13 -16
- package/dist/core/paths.js +0 -24
- package/dist/core/redaction.js +109 -2
- package/dist/core/run-lock.js +2 -5
- package/dist/core/spawn-env.js +1 -1
- package/dist/core/state/migrations.js +123 -61
- package/dist/core/state-db-scope.js +2 -4
- package/dist/core/state-db.js +126 -692
- package/dist/core/time.js +0 -20
- package/dist/core/type-presentation.js +1 -9
- package/dist/core/write-source.js +294 -1005
- package/dist/execution/input-contract.js +1 -1
- package/dist/execution/resolved-request.js +135 -689
- package/dist/execution/source.js +63 -257
- package/dist/execution/target-ref.js +1 -1
- package/dist/indexer/bundle-identity-guard.js +2 -2
- package/dist/indexer/db/llm-cache.js +2 -2
- package/dist/indexer/ensure-index.js +77 -73
- package/dist/indexer/index-rebuild-lock.js +3 -11
- package/dist/indexer/index-writer-lock.js +8 -17
- package/dist/indexer/index-written-assets.js +141 -154
- package/dist/indexer/indexer.js +400 -1124
- package/dist/indexer/links/declared-links.js +90 -0
- package/dist/indexer/materialize-embeddings.js +60 -397
- package/dist/indexer/passes/memory-inference.js +96 -90
- package/dist/indexer/passes/metadata.js +132 -219
- package/dist/indexer/read-preflight.js +0 -7
- package/dist/indexer/scan/doc-to-entry.js +2 -3
- package/dist/indexer/scan/drain-dir.js +1 -1
- package/dist/indexer/search/db-search.js +190 -590
- package/dist/indexer/search/fts-query.js +30 -41
- package/dist/indexer/search/ranking.js +28 -154
- package/dist/indexer/search/search-attribution.js +12 -32
- package/dist/indexer/search/search-fields.js +11 -15
- package/dist/indexer/search/search-hit-enrichers.js +54 -85
- package/dist/indexer/search/search-source.js +1 -4
- package/dist/indexer/usage/usage-events.js +36 -7
- package/dist/indexer/walk/walker.js +3 -4
- package/dist/integrations/agent/engine-fallback.js +23 -40
- package/dist/integrations/agent/engine-resolution.js +93 -183
- package/dist/integrations/agent/execution.js +507 -0
- package/dist/integrations/agent/model-map.js +28 -156
- package/dist/integrations/agent/request-lowering.js +66 -141
- package/dist/integrations/agent/runner-dispatch.js +143 -321
- package/dist/integrations/agent/runner.js +54 -14
- package/dist/integrations/lockfile.js +53 -101
- package/dist/llm/client.js +18 -6
- package/dist/llm/embedders/deterministic.js +2 -3
- package/dist/llm/embedders/profile.js +71 -0
- package/dist/llm/embedders/remote.js +11 -17
- package/dist/llm/feature-gate.js +0 -8
- package/dist/llm/index-passes.js +3 -5
- package/dist/llm/memory-infer.js +1 -2
- package/dist/llm/structured-call.js +5 -24
- package/dist/output/generic-render.js +23 -11
- package/dist/output/html-render.js +13 -10
- package/dist/output/render-registry.js +3 -32
- package/dist/output/shapes/helpers.js +25 -38
- package/dist/output/shapes/passthrough.js +1 -9
- package/dist/{indexer/graph/graph-types.js → output/text/bundle-rename.js} +4 -1
- package/dist/output/text/command-format.js +69 -31
- package/dist/output/text/helpers.js +1 -1
- package/dist/output/text/migrate.js +5 -14
- package/dist/output/text/proposal-format.js +48 -3
- package/dist/output/text/show-format.js +13 -17
- package/dist/output/text/workflow-format.js +0 -32
- package/dist/output/text.js +2 -0
- package/dist/registry/factory.js +4 -19
- package/dist/registry/network.js +66 -220
- package/dist/registry/providers/index.js +0 -2
- package/dist/registry/providers/skills-sh.js +3 -14
- package/dist/registry/providers/static-index.js +24 -26
- package/dist/registry/resolve.js +55 -131
- package/dist/scripts/akm-migrate-node.js +42948 -92369
- package/dist/scripts/akm-migrate.js +42935 -92354
- package/dist/setup/registry-stash-loader.js +4 -13
- package/dist/setup/semantic-assets.js +3 -44
- package/dist/setup/setup.js +1 -1
- package/dist/setup/steps/connection.js +5 -6
- package/dist/setup/steps/platforms.js +2 -2
- package/dist/setup/steps/tasks.js +25 -15
- package/dist/sources/provider-factory.js +17 -18
- package/dist/sources/providers/filesystem.js +2 -3
- package/dist/sources/providers/git-install.js +7 -1
- package/dist/sources/providers/git-provider.js +0 -3
- package/dist/sources/providers/git-stash.js +83 -21
- package/dist/sources/providers/npm.js +2 -4
- package/dist/sources/providers/provider-utils.js +5 -10
- package/dist/sources/providers/website.js +0 -2
- package/dist/sources/snapshot-fetchers/website-ingest.js +1 -1
- package/dist/sources/website-url.js +2 -2
- package/dist/storage/database.js +9 -35
- package/dist/storage/repositories/improve-ledger-repository.js +209 -0
- package/dist/storage/repositories/index-connection.js +39 -72
- package/dist/storage/repositories/index-entries-repository.js +131 -129
- package/dist/storage/repositories/index-entry-mapper.js +1 -2
- package/dist/storage/repositories/index-entry-schema.js +101 -268
- package/dist/storage/repositories/index-fts-repository.js +86 -256
- package/dist/storage/repositories/index-links-repository.js +143 -0
- package/dist/storage/repositories/index-llm-cache-repository.js +7 -9
- package/dist/storage/repositories/index-meta-repository.js +6 -4
- package/dist/storage/repositories/index-schema.js +257 -325
- package/dist/storage/repositories/index-utility-repository.js +8 -29
- package/dist/storage/repositories/index-vec-repository.js +133 -414
- package/dist/storage/repositories/outcome-repository.js +2 -1
- package/dist/storage/repositories/proposals-repository.js +104 -1
- package/dist/storage/repositories/registry-index-cache-repository.js +100 -0
- package/dist/storage/repositories/salience-repository.js +1 -19
- package/dist/storage/repositories/task-history-repository.js +26 -4
- package/dist/storage/repositories/workflow-runs-repository.js +53 -244
- package/dist/storage/sqlite-migrations.js +136 -0
- package/dist/storage/sqlite-pragmas.js +11 -9
- package/dist/storage/sqlite-transaction.js +170 -0
- package/dist/storage/state-db-integrity.js +130 -0
- package/dist/tasks/activation-config.js +134 -62
- package/dist/tasks/backends/cron.js +191 -302
- package/dist/tasks/backends/exec-utils.js +2 -5
- package/dist/tasks/backends/launchd.js +141 -748
- package/dist/tasks/backends/schtasks.js +119 -623
- package/dist/tasks/prepare/prepare-support.js +5 -15
- package/dist/tasks/prepare/prepare.js +0 -2
- package/dist/tasks/resolve-akm-bin.js +20 -79
- package/dist/tasks/run/attempt-lifecycle.js +0 -1
- package/dist/tasks/run/load-task.js +1 -1
- package/dist/tasks/scheduler-binding.js +20 -238
- package/dist/tasks/scheduler-invocation.js +136 -244
- package/dist/tasks/scheduler-lock.js +53 -0
- package/dist/tasks/scheduler-sync.js +368 -679
- package/dist/tasks/source/parse-task-source.js +55 -9
- package/dist/tasks/source/task-source-v3-frozen.js +3 -4
- package/dist/tasks/source/task-to-v4.js +464 -88
- package/dist/workflows/authoring/authoring.js +3 -12
- package/dist/workflows/compile.js +211 -0
- package/dist/workflows/concurrency-policy.js +13 -74
- package/dist/workflows/exec/child-invocation.js +3 -17
- package/dist/workflows/exec/child-workflow.js +32 -141
- package/dist/workflows/exec/dispatch-redaction.js +13 -53
- package/dist/workflows/exec/environment.js +98 -0
- package/dist/workflows/exec/exec-unit.js +33 -140
- package/dist/workflows/exec/frozen-judge.js +7 -59
- package/dist/workflows/exec/native-executor.js +82 -341
- package/dist/workflows/exec/param-secrets.js +29 -47
- package/dist/workflows/exec/run-workflow.js +154 -387
- package/dist/workflows/exec/scheduler.js +9 -36
- package/dist/workflows/exec/step-work.js +127 -430
- package/dist/workflows/exec/unit-dispatch.js +11 -63
- package/dist/workflows/exec/unit-writer.js +8 -52
- package/dist/workflows/exec/worktree.js +39 -273
- package/dist/workflows/freeze/child-output-references.js +4 -15
- package/dist/workflows/freeze/environment.js +99 -92
- package/dist/workflows/freeze/freeze.js +172 -0
- package/dist/workflows/freeze/step-values.js +19 -21
- package/dist/workflows/freeze/targets/child-workflow.js +23 -92
- package/dist/workflows/freeze/targets/command.js +10 -33
- package/dist/workflows/freeze/targets/script.js +5 -12
- package/dist/workflows/freeze/targets/shell.js +3 -6
- package/dist/workflows/freeze/targets/task.js +25 -80
- package/dist/workflows/freeze/task-bindings.js +20 -67
- package/dist/workflows/{source-ir/github-yaml.js → github-yaml.js} +88 -206
- package/dist/workflows/ir/params.js +6 -51
- package/dist/workflows/ir/plan-hash.js +2 -34
- package/dist/workflows/parser.js +140 -43
- package/dist/{commands/improve/consolidate/types.js → workflows/plan.js} +2 -1
- package/dist/workflows/renderer.js +36 -69
- package/dist/workflows/resource-limits.js +12 -120
- package/dist/workflows/runtime/agent-identity.js +8 -40
- package/dist/workflows/runtime/run-outputs.js +3 -6
- package/dist/workflows/runtime/run-plan.js +316 -0
- package/dist/workflows/runtime/runs.js +48 -200
- package/dist/workflows/runtime/workflow-asset-loader.js +24 -57
- package/dist/workflows/{source-ir/semantics.js → source-semantics.js} +16 -20
- package/dist/workflows/validate-summary.js +2 -7
- package/docs/integration/bundling-akm.md +49 -42
- package/docs/migration/README.md +1 -0
- package/docs/migration/release-notes/0.9.17.md +43 -0
- package/docs/migration/v0.9.1-to-v0.9.2.md +23 -7
- package/docs/reference/cli.md +232 -135
- package/docs/reference/configuration.md +71 -57
- package/docs/reference/data-and-telemetry.md +20 -21
- package/docs/reference/tasks.md +105 -39
- package/docs/reference/workflow-schema.md +14 -18
- package/docs/reference/workflows.md +6 -9
- package/package.json +1 -1
- package/schemas/akm-config.json +115 -738
- package/schemas/akm-workflow.json +1 -0
- package/dist/assets/improve-strategies/graph-refresh.json +0 -15
- package/dist/assets/prompts/contradiction-judge.md +0 -33
- package/dist/assets/prompts/graph-extract-system.md +0 -1
- package/dist/assets/prompts/graph-extract-user-prompt.md +0 -35
- package/dist/assets/prompts/metadata-enhance-system.md +0 -1
- package/dist/assets/tasks/improve/akm-graph-refresh-weekly.yml +0 -4
- package/dist/commands/health/advisories.js +0 -150
- package/dist/commands/health/metrics.js +0 -329
- package/dist/commands/health/surfaces.js +0 -102
- package/dist/commands/improve/anti-collapse.js +0 -83
- package/dist/commands/improve/collapse-detector.js +0 -432
- package/dist/commands/improve/consolidate/eligibility.js +0 -48
- package/dist/commands/improve/consolidate/merge.js +0 -149
- package/dist/commands/improve/distill/promote-memory.js +0 -291
- package/dist/commands/improve/distill/quality-gate.js +0 -337
- package/dist/commands/improve/eval-cases.js +0 -52
- package/dist/commands/improve/memory/memory-contradiction-detect.js +0 -291
- package/dist/commands/improve/proposal-envelope.js +0 -31
- package/dist/commands/improve/run-context.js +0 -123
- package/dist/commands/improve/shared.js +0 -31
- package/dist/commands/improve/source-identity.js +0 -28
- package/dist/commands/improve/triage.js +0 -96
- package/dist/commands/proposal/drain-policies.js +0 -151
- package/dist/commands/sources/update-transaction.js +0 -220
- package/dist/core/action-contributors.js +0 -28
- package/dist/core/config/config-version-shim.js +0 -101
- package/dist/core/fs-txn.js +0 -405
- package/dist/core/lexical-score.js +0 -25
- package/dist/core/maintenance-barrier.js +0 -167
- package/dist/execution/executable-identity.js +0 -105
- package/dist/execution/guarded-source.js +0 -427
- package/dist/indexer/db/graph-db.js +0 -444
- package/dist/indexer/graph/graph-boost.js +0 -427
- package/dist/indexer/graph/graph-dedup.js +0 -95
- package/dist/indexer/graph/graph-extraction.js +0 -1108
- package/dist/indexer/search/name-match.js +0 -35
- package/dist/indexer/search/ranking-contributors.js +0 -515
- package/dist/indexer/search/ranking-types.js +0 -4
- package/dist/indexer/walk/project-context.js +0 -192
- package/dist/integrations/agent/execution-cascade.js +0 -566
- package/dist/integrations/agent/execution-definitions.js +0 -202
- package/dist/integrations/agent/execution-lowering.js +0 -841
- package/dist/integrations/agent/execution-preparation.js +0 -98
- package/dist/integrations/agent/inline-execution.js +0 -74
- package/dist/llm/graph-extract.js +0 -728
- package/dist/llm/metadata-enhance.js +0 -96
- package/dist/registry/create-provider-registry.js +0 -29
- package/dist/registry/pinned-request-helper.js +0 -247
- package/dist/registry/pinned-transport.js +0 -717
- package/dist/sources/providers/index.js +0 -14
- package/dist/storage/engines/sqlite-migrations.js +0 -271
- package/dist/storage/repositories/canaries-repository.js +0 -107
- package/dist/storage/repositories/embedding-salvage-repository.js +0 -184
- package/dist/storage/repositories/registry-cache.js +0 -113
- package/dist/tasks/scheduler-sync-preview.js +0 -52
- package/dist/tasks/source/task-to-v3.js +0 -507
- package/dist/workflows/freeze/resolve-steps.js +0 -86
- package/dist/workflows/freeze/source-freeze.js +0 -64
- package/dist/workflows/ir/compile.js +0 -321
- package/dist/workflows/ir/environment-v4.js +0 -330
- package/dist/workflows/ir/freeze-v4.js +0 -153
- package/dist/workflows/ir/schema-v4.js +0 -745
- package/dist/workflows/ir/schema.js +0 -354
- package/dist/workflows/program/schema.js +0 -77
- package/dist/workflows/runtime/checkin.js +0 -57
- package/dist/workflows/runtime/plan-classifier.js +0 -196
- package/dist/workflows/runtime/unit-checkin.js +0 -45
- package/dist/workflows/runtime/unit-phases.js +0 -20
- package/dist/workflows/schema.js +0 -4
- package/dist/workflows/source-ir/compile.js +0 -200
- package/dist/workflows/source-ir/program.js +0 -50
- package/dist/workflows/source-ir/result.js +0 -26
- package/dist/workflows/source-ir/schema.js +0 -786
- package/dist/workflows/source-ir/triggers.js +0 -79
- package/dist/workflows/source-ir/uses.js +0 -40
- package/dist/workflows/validator.js +0 -60
|
@@ -2,41 +2,34 @@
|
|
|
2
2
|
// License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
3
3
|
// file, You can obtain one at https://mozilla.org/MPL/2.0/.
|
|
4
4
|
/**
|
|
5
|
-
* Database-backed (SQLite +
|
|
5
|
+
* Database-backed (SQLite FTS5 + vector) search.
|
|
6
6
|
*
|
|
7
|
-
*
|
|
8
|
-
*
|
|
9
|
-
*
|
|
10
|
-
*
|
|
11
|
-
* sources/providers/filesystem.ts also imports `searchLocal` from here.
|
|
12
|
-
*
|
|
13
|
-
* Renamed from `local-search.ts` to signal that this is the DB-layer search
|
|
14
|
-
* implementation, not a "local vs. remote" distinction.
|
|
7
|
+
* Ranking is two candidate channels fused by reciprocal rank (`ranking.ts`):
|
|
8
|
+
* BM25 over whole documents matching any query word, and the nearest document
|
|
9
|
+
* vectors to the query embedding. Filters narrow the fused list; nothing else
|
|
10
|
+
* reorders it.
|
|
15
11
|
*/
|
|
16
12
|
import path from "node:path";
|
|
17
|
-
import { buildActionFromContributors, defaultActionContributors } from "../../core/action-contributors.js";
|
|
18
13
|
import { stashDirFor } from "../../core/asset/asset-placement.js";
|
|
19
14
|
import { displayRef } from "../../core/asset/resolve-ref.js";
|
|
20
15
|
import { compareCodePoints } from "../../core/common.js";
|
|
21
16
|
import { classifyPathAccess } from "../../core/path-access.js";
|
|
22
17
|
import { getDbPath } from "../../core/paths.js";
|
|
23
18
|
import { systemErrorCode } from "../../core/system-error.js";
|
|
24
|
-
import {
|
|
19
|
+
import { presentationFor } from "../../core/type-presentation.js";
|
|
20
|
+
import { embed } from "../../llm/embedder.js";
|
|
21
|
+
import { applyEmbeddingTemplate, resolveEmbeddingProfile } from "../../llm/embedders/profile.js";
|
|
25
22
|
import { normalizeEmbeddingEndpoint } from "../../llm/embedders/remote.js";
|
|
26
23
|
import { assertIndexPathReadable, closeDatabase, openExistingDatabase, } from "../../storage/repositories/index-connection.js";
|
|
27
|
-
import { getAllEntries, getBaseBeliefStatesForDerivedTwins, getEntryById, getEntryCount,
|
|
28
|
-
import {
|
|
24
|
+
import { getAllEntries, getBaseBeliefStatesForDerivedTwins, getEntryById, getEntryCount, getEntryRefsAndTypes, } from "../../storage/repositories/index-entries-repository.js";
|
|
25
|
+
import { searchFts } from "../../storage/repositories/index-fts-repository.js";
|
|
29
26
|
import { getMeta } from "../../storage/repositories/index-meta-repository.js";
|
|
30
27
|
import { getEmbeddingCount, searchVec } from "../../storage/repositories/index-vec-repository.js";
|
|
31
|
-
import { getCurrentWorkflowScopeKey } from "../../workflows/authoring/scope-key.js";
|
|
32
28
|
import { ensureIndex } from "../ensure-index.js";
|
|
33
|
-
import { collectGraphRelatedHit, loadGraphBoostContext } from "../graph/graph-boost.js";
|
|
34
29
|
import { isProposedQuality } from "../passes/metadata.js";
|
|
35
|
-
import {
|
|
36
|
-
import {
|
|
37
|
-
import {
|
|
38
|
-
import { typeBoostFor } from "./ranking-contributors.js";
|
|
39
|
-
import { attachSearchHitAttribution, copySearchHitAttribution, getSearchHitAttribution } from "./search-attribution.js";
|
|
30
|
+
import { ftsQueryTokens, parseRefPrefixQuery, parseRetiredTypePrefixQuery } from "./fts-query.js";
|
|
31
|
+
import { reciprocalRankFusion } from "./ranking.js";
|
|
32
|
+
import { attachSearchHitAttribution } from "./search-attribution.js";
|
|
40
33
|
import { enrichSearchHit } from "./search-hit-enrichers.js";
|
|
41
34
|
import { buildEditHint, findSourceForPath, isEditable } from "./search-source.js";
|
|
42
35
|
/**
|
|
@@ -47,6 +40,16 @@ import { buildEditHint, findSourceForPath, isEditable } from "./search-source.js
|
|
|
47
40
|
* actionable without re-introducing read-triggered reindexing.
|
|
48
41
|
*/
|
|
49
42
|
const STALE_INDEX_HINT_MS = 7 * 24 * 60 * 60 * 1000;
|
|
43
|
+
/** Candidates each channel contributes to fusion. */
|
|
44
|
+
const CHANNEL_DEPTH = 100;
|
|
45
|
+
/** How long search waits for the query embedding unless `embedding.queryTimeoutMs` says otherwise. */
|
|
46
|
+
export const DEFAULT_QUERY_EMBED_TIMEOUT_MS = 3000;
|
|
47
|
+
/** Each search hit's indexed content, kept off the output object for curate's reranker. */
|
|
48
|
+
const hitContent = new WeakMap();
|
|
49
|
+
/** The indexed (safe-projected) content of a hit this process's search returned. */
|
|
50
|
+
export function searchHitContent(hit) {
|
|
51
|
+
return hitContent.get(hit);
|
|
52
|
+
}
|
|
50
53
|
function hasIndexedProvenance(entry) {
|
|
51
54
|
return Boolean(entry.itemRef && entry.bundleId && entry.conceptId);
|
|
52
55
|
}
|
|
@@ -65,11 +68,8 @@ function buildStaleIndexHint(db) {
|
|
|
65
68
|
return undefined;
|
|
66
69
|
}
|
|
67
70
|
}
|
|
68
|
-
function
|
|
69
|
-
return
|
|
70
|
-
}
|
|
71
|
-
export function buildLocalAction(type, ref, registry = defaultRendererRegistry) {
|
|
72
|
-
return buildActionFromContributors({ type, ref }, defaultActionContributors(registry)) ?? `akm show ${ref}`;
|
|
71
|
+
export function buildLocalAction(type, ref) {
|
|
72
|
+
return presentationFor(type).action?.(ref) ?? `akm show ${ref}`;
|
|
73
73
|
}
|
|
74
74
|
function resolveSearchHitRef(entry, provenance, defaultBundleId) {
|
|
75
75
|
return displayRef({
|
|
@@ -79,26 +79,6 @@ function resolveSearchHitRef(entry, provenance, defaultBundleId) {
|
|
|
79
79
|
bundleId: provenance.bundleId,
|
|
80
80
|
}, defaultBundleId);
|
|
81
81
|
}
|
|
82
|
-
function resolveSearchHitOrigin(source) {
|
|
83
|
-
return source?.registryId ?? null;
|
|
84
|
-
}
|
|
85
|
-
/**
|
|
86
|
-
* Phase 2A / Rec 5: gate for the per-search `getPositiveFeedbackCountsByIds`
|
|
87
|
-
* lookup. Returns `true` only when the user has explicitly opted into
|
|
88
|
-
* `improve.utilityDecay` AND configured a `feedbackStabilityBoost > 1.0`.
|
|
89
|
-
* Either condition being false makes the DB query pure overhead (the ranking
|
|
90
|
-
* contributor ignores `positiveFeedbackCounts` when `utilityDecayConfig` is
|
|
91
|
-
* absent, and `1.0^count == 1` collapses the boost into a no-op).
|
|
92
|
-
*
|
|
93
|
-
* Exported for unit testing — keeps the gate decision pinned so a future edit
|
|
94
|
-
* can't quietly broaden the hot path.
|
|
95
|
-
*/
|
|
96
|
-
export function shouldQueryPositiveFeedbackCounts(utilityDecayRaw) {
|
|
97
|
-
if (utilityDecayRaw === undefined)
|
|
98
|
-
return false;
|
|
99
|
-
const boost = utilityDecayRaw.feedbackStabilityBoost ?? 1.5;
|
|
100
|
-
return boost > 1.0;
|
|
101
|
-
}
|
|
102
82
|
// ── Main search entrypoint ───────────────────────────────────────────────────
|
|
103
83
|
/**
|
|
104
84
|
* Whether an embedding provider is actually configured.
|
|
@@ -116,18 +96,9 @@ function hasConfiguredEmbeddingProvider(config) {
|
|
|
116
96
|
return Boolean(config.embedding?.endpoint && config.embedding?.model);
|
|
117
97
|
}
|
|
118
98
|
export async function searchLocal(input) {
|
|
119
|
-
const { query,
|
|
120
|
-
const filters = input.filters;
|
|
121
|
-
const includeProposed = input.includeProposed === true;
|
|
122
|
-
const beliefFilter = input.beliefFilter ?? "all";
|
|
123
|
-
const restrictToSources = input.restrictToSources === true;
|
|
124
|
-
const includeExcludedTypes = input.includeExcludedTypes === true;
|
|
125
|
-
const disableProjectContext = input.disableProjectContext === true;
|
|
126
|
-
const disableScopedUtility = input.disableScopedUtility === true;
|
|
127
|
-
const rendererRegistry = input.rendererRegistry ?? defaultRendererRegistry;
|
|
128
|
-
const allSourceDirs = sources.map((s) => s.path);
|
|
99
|
+
const { query, stashDir, config } = input;
|
|
129
100
|
const warnings = [];
|
|
130
|
-
// Semantic search is attempted fresh on every query (see `
|
|
101
|
+
// Semantic search is attempted fresh on every query (see `startVectorChannel`);
|
|
131
102
|
// there is no cached readiness verdict to consult here. The only thing
|
|
132
103
|
// worth flagging ahead of the attempt is a config that can never succeed.
|
|
133
104
|
if (config.semanticSearchMode === "auto" && !hasConfiguredEmbeddingProvider(config)) {
|
|
@@ -167,7 +138,7 @@ export async function searchLocal(input) {
|
|
|
167
138
|
const staleHint = buildStaleIndexHint(db);
|
|
168
139
|
if (staleHint)
|
|
169
140
|
warnings.push(staleHint);
|
|
170
|
-
const { hits, embedMs, rankMs, mode, semanticWarning } = await searchDatabase(db,
|
|
141
|
+
const { hits, embedMs, rankMs, mode, semanticWarning } = await searchDatabase(db, input);
|
|
171
142
|
if (semanticWarning)
|
|
172
143
|
warnings.push(semanticWarning);
|
|
173
144
|
return {
|
|
@@ -177,7 +148,7 @@ export async function searchLocal(input) {
|
|
|
177
148
|
embedMs,
|
|
178
149
|
rankMs,
|
|
179
150
|
// Report the mode the search ACTUALLY used, carried explicitly from the
|
|
180
|
-
// vector
|
|
151
|
+
// vector channel — not inferred from elapsed embedding milliseconds.
|
|
181
152
|
mode,
|
|
182
153
|
};
|
|
183
154
|
}
|
|
@@ -186,326 +157,142 @@ export async function searchLocal(input) {
|
|
|
186
157
|
}
|
|
187
158
|
}
|
|
188
159
|
// ── Database search ─────────────────────────────────────────────────────────
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
}
|
|
199
|
-
/**
|
|
200
|
-
* A final deterministic key for genuinely tied candidates. It deliberately
|
|
201
|
-
* excludes the asset name, filename, path, durable ref, and SQLite id: callers
|
|
202
|
-
* such as the memory-pack adapter generate each of those from an opaque source
|
|
203
|
-
* id, so using one here makes an otherwise equal search depend on that id.
|
|
204
|
-
*
|
|
205
|
-
* The normal AKM Markdown adapter keeps an H1 title in `content`; strip that
|
|
206
|
-
* one synthetic title too, because the adapter may derive it from the opaque
|
|
207
|
-
* filename. Identical remaining bodies are semantically indistinguishable at
|
|
208
|
-
* this ranking stage and intentionally continue to the existing name/path
|
|
209
|
-
* fallback for repeatable local presentation.
|
|
210
|
-
*/
|
|
211
|
-
function asciiCaseFold(value) {
|
|
212
|
-
// SQLite's built-in lower() folds ASCII only unless a build opts into ICU.
|
|
213
|
-
// Keep this key deliberately in that portable shared subset instead of
|
|
214
|
-
// introducing locale-dependent JavaScript ordering for non-ASCII content.
|
|
215
|
-
return value.replace(/[A-Z]/g, (letter) => String.fromCharCode(letter.charCodeAt(0) + 32));
|
|
216
|
-
}
|
|
217
|
-
/** The portable byte-level title/body rule mirrored in index-fts-repository. */
|
|
218
|
-
export function canonicalContentTieKey(entry) {
|
|
219
|
-
const content = entry.content ?? "";
|
|
220
|
-
const newline = content.startsWith("# ") ? content.indexOf("\n") : -1;
|
|
221
|
-
// SQLite uses ltrim(value, char(13) || char(10) || ' ') after an exact '# '
|
|
222
|
-
// title and trim(value, ' ') otherwise. Keep exactly that deliberately
|
|
223
|
-
// narrow byte contract; do not use locale or Unicode-whitespace helpers.
|
|
224
|
-
const body = newline >= 0 ? content.slice(newline + 1).replace(/^[\r\n ]+/, "") : content;
|
|
225
|
-
const source = (body || entry.description || "").replace(/^ +| +$/g, "");
|
|
226
|
-
return Buffer.from(asciiCaseFold(source), "utf8").toString("hex");
|
|
227
|
-
}
|
|
228
|
-
function buildSearchResultComparator(query) {
|
|
229
|
-
const queryTokens = buildLexicalQueryPlan(query).tokens.map((token) => token.toLowerCase());
|
|
230
|
-
const displayScore = (score) => Math.round(displaySearchScore(score) * 10000) / 10000;
|
|
231
|
-
const stableRankScore = (score) => Math.round(score * 10000) / 10000;
|
|
232
|
-
return (a, b) => {
|
|
233
|
-
const aNameTier = lexicalNameMatchTier(a.entry, queryTokens);
|
|
234
|
-
const bNameTier = lexicalNameMatchTier(b.entry, queryTokens);
|
|
235
|
-
if (aNameTier === 3 || bNameTier === 3) {
|
|
236
|
-
const nameDiff = bNameTier - aNameTier;
|
|
237
|
-
if (nameDiff !== 0)
|
|
238
|
-
return nameDiff;
|
|
239
|
-
}
|
|
240
|
-
const scoreDiff = displayScore(b.score) - displayScore(a.score);
|
|
241
|
-
if (scoreDiff !== 0)
|
|
242
|
-
return scoreDiff;
|
|
243
|
-
const rawScoreDiff = stableRankScore(b.score) - stableRankScore(a.score);
|
|
244
|
-
if (rawScoreDiff !== 0)
|
|
245
|
-
return rawScoreDiff;
|
|
246
|
-
// Ceiling values are intentionally allowed to demote visibility, but not
|
|
247
|
-
// to erase relevance. Prefer the score before a relaxed body-only ceiling;
|
|
248
|
-
// a later belief-state ceiling has its own minScore handoff and must not
|
|
249
|
-
// overwrite this ordering evidence. Belief-only ceilings fall back to
|
|
250
|
-
// their `preCeilingScore`.
|
|
251
|
-
const preCeilingRelevance = (item) => item.preRelaxedCeilingScore ?? item.preCeilingScore ?? item.score;
|
|
252
|
-
const ceilingDiff = stableRankScore(preCeilingRelevance(b)) - stableRankScore(preCeilingRelevance(a));
|
|
253
|
-
if (ceilingDiff !== 0)
|
|
254
|
-
return ceilingDiff;
|
|
255
|
-
const nameDiff = bNameTier - aNameTier;
|
|
256
|
-
if (nameDiff !== 0)
|
|
257
|
-
return nameDiff;
|
|
258
|
-
const typeDiff = typeBoostFor(b.entry.type) - typeBoostFor(a.entry.type);
|
|
259
|
-
if (typeDiff !== 0)
|
|
260
|
-
return typeDiff;
|
|
261
|
-
// Keep opaque generated IDs out of the final relevance tie-break. This
|
|
262
|
-
// runs only after every ranking contributor (including the #940 preserved
|
|
263
|
-
// pre-ceiling evidence), exact-name, and type comparison has tied.
|
|
264
|
-
const contentDiff = compareCodePoints(canonicalContentTieKey(a.entry), canonicalContentTieKey(b.entry));
|
|
265
|
-
if (contentDiff !== 0)
|
|
266
|
-
return contentDiff;
|
|
267
|
-
return a.filePath.localeCompare(b.filePath);
|
|
160
|
+
async function searchDatabase(db, input) {
|
|
161
|
+
const { query, searchType, limit, stashDir, sources, config, filters } = input;
|
|
162
|
+
const filterOptions = {
|
|
163
|
+
db,
|
|
164
|
+
sources,
|
|
165
|
+
restrictToSources: input.restrictToSources === true,
|
|
166
|
+
filters,
|
|
167
|
+
includeProposed: input.includeProposed === true,
|
|
168
|
+
beliefFilter: input.beliefFilter ?? "all",
|
|
268
169
|
};
|
|
269
|
-
}
|
|
270
|
-
async function searchDatabase(db, query, searchType, limit, stashDir, allSourceDirs, config, sources, rendererRegistry = defaultRendererRegistry, filters, includeProposed = false, beliefFilter = "all", restrictToSources = false, includeExcludedTypes = false, disableProjectContext = false, disableScopedUtility = false) {
|
|
271
|
-
const hasSearchableTokens = query.length > 0 && buildLexicalQueryPlan(query).tokens.length > 0;
|
|
272
170
|
// #627 — resolve the default type-exclusion policy. It applies ONLY on the
|
|
273
171
|
// untyped ('any') path and only when the caller did not opt back in via
|
|
274
172
|
// `includeExcludedTypes`. When the config key is ABSENT a built-in default of
|
|
275
173
|
// ['session'] is applied; an explicit empty list disables exclusion.
|
|
276
|
-
const defaultExcludes = searchType === "any" && !includeExcludedTypes ? (config.search?.defaultExcludeTypes ?? ["session"]) : [];
|
|
174
|
+
const defaultExcludes = searchType === "any" && !input.includeExcludedTypes ? (config.search?.defaultExcludeTypes ?? ["session"]) : [];
|
|
277
175
|
// D4 — conceptId-prefix queries (`memories/projecta/`, `bundle//`,
|
|
278
176
|
// `bundle//skills/`) translate to a deterministic enumeration narrowed by
|
|
279
|
-
// conceptId
|
|
280
|
-
// sanitized form would produce ("memories projecta" — noise). The branch
|
|
177
|
+
// conceptId instead of a keyword search over their path words. The branch
|
|
281
178
|
// fires only on the untyped path: an explicit `--type` flag expresses
|
|
282
179
|
// stronger intent and wins. The PREFIX is itself explicit intent, so
|
|
283
180
|
// `defaultExcludeTypes` does not apply — `sessions/` enumerates sessions
|
|
284
181
|
// exactly like `--type session` does, and `bundle//` means the whole bundle.
|
|
285
182
|
const refPrefix = searchType === "any" ? parseRefPrefixQuery(query) : null;
|
|
286
|
-
// Shared args for the two browse paths below; browse never runs semantic
|
|
287
|
-
// ranking, so both return usedSemantic: false.
|
|
288
|
-
const browseArgs = {
|
|
289
|
-
db,
|
|
290
|
-
query,
|
|
291
|
-
limit,
|
|
292
|
-
stashDir,
|
|
293
|
-
allSourceDirs,
|
|
294
|
-
sources,
|
|
295
|
-
config,
|
|
296
|
-
rendererRegistry,
|
|
297
|
-
filters,
|
|
298
|
-
includeProposed,
|
|
299
|
-
beliefFilter,
|
|
300
|
-
restrictToSources,
|
|
301
|
-
};
|
|
302
183
|
if (refPrefix) {
|
|
303
|
-
// Browse path (conceptId-prefix enumeration).
|
|
304
184
|
return {
|
|
305
|
-
|
|
306
|
-
...
|
|
185
|
+
hits: await enumerateEntries({
|
|
186
|
+
...filterOptions,
|
|
187
|
+
limit,
|
|
188
|
+
stashDir,
|
|
189
|
+
config,
|
|
307
190
|
excludeTypes: [],
|
|
308
191
|
conceptIdPrefix: refPrefix.conceptIdPrefix,
|
|
309
192
|
...(refPrefix.bundle !== undefined ? { bundle: refPrefix.bundle } : {}),
|
|
310
|
-
})
|
|
193
|
+
}),
|
|
311
194
|
mode: "keyword",
|
|
312
195
|
};
|
|
313
196
|
}
|
|
314
|
-
// Empty queries — including ones
|
|
315
|
-
//
|
|
316
|
-
|
|
317
|
-
if (!hasSearchableTokens) {
|
|
318
|
-
// Browse path (empty/unsearchable query).
|
|
197
|
+
// Empty queries — including ones with no searchable token such as "." —
|
|
198
|
+
// enumerate matching entries instead of returning nothing.
|
|
199
|
+
if (ftsQueryTokens(query).length === 0) {
|
|
319
200
|
return {
|
|
320
|
-
|
|
321
|
-
...
|
|
201
|
+
hits: await enumerateEntries({
|
|
202
|
+
...filterOptions,
|
|
203
|
+
limit,
|
|
204
|
+
stashDir,
|
|
205
|
+
config,
|
|
322
206
|
typeFilter: searchType === "any" ? undefined : searchType,
|
|
323
207
|
excludeTypes: defaultExcludes,
|
|
324
|
-
})
|
|
208
|
+
}),
|
|
325
209
|
mode: "keyword",
|
|
326
210
|
};
|
|
327
211
|
}
|
|
328
|
-
// Start the async embedding request without awaiting, then run FTS
|
|
329
|
-
// synchronously while the HTTP/local embedding request is in-flight.
|
|
330
212
|
const typeFilter = searchType === "any" ? undefined : searchType;
|
|
331
|
-
const
|
|
213
|
+
const startedAt = Date.now();
|
|
214
|
+
// The query embedding request goes out first, so FTS runs while it is in flight.
|
|
215
|
+
const vectorChannel = startVectorChannel(db, query, config);
|
|
216
|
+
const lexical = searchFts(db, query, CHANNEL_DEPTH, typeFilter, defaultExcludes);
|
|
217
|
+
const vector = await vectorChannel;
|
|
218
|
+
const embedMs = Date.now() - startedAt;
|
|
332
219
|
const tRank0 = Date.now();
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
const
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
220
|
+
const vectorCandidates = vector.neighbors ? keepAllowedTypes(db, vector.neighbors, typeFilter, defaultExcludes) : [];
|
|
221
|
+
const fused = reciprocalRankFusion([lexical, vectorCandidates]);
|
|
222
|
+
const selected = selectFusedEntries(db, fused, limit, filterOptions);
|
|
223
|
+
const rankMs = Date.now() - tRank0;
|
|
224
|
+
const hits = await Promise.all(selected.map(({ candidate, row }) => buildDbHit({
|
|
225
|
+
entry: row.entry,
|
|
226
|
+
path: row.filePath,
|
|
227
|
+
itemRef: row.itemRef,
|
|
228
|
+
bundleId: row.bundleId,
|
|
229
|
+
conceptId: row.conceptId,
|
|
230
|
+
score: Math.round(candidate.score * 1e6) / 1e6,
|
|
231
|
+
whyMatched: describeRanks(candidate),
|
|
232
|
+
defaultStashDir: stashDir,
|
|
233
|
+
sources,
|
|
234
|
+
config,
|
|
235
|
+
db,
|
|
236
|
+
})));
|
|
237
|
+
const mode = vector.warning ? "fts-fallback" : vector.neighbors ? "semantic" : "keyword";
|
|
238
|
+
return { embedMs, rankMs, hits, mode, semanticWarning: vector.warning };
|
|
239
|
+
}
|
|
240
|
+
/**
|
|
241
|
+
* Walk the fused list in order, loading entries a batch at a time, and keep
|
|
242
|
+
* the first `limit` that survive path deduplication and the filters. An entry
|
|
243
|
+
* whose indexed content is identical to a kept one's (the same body saved
|
|
244
|
+
* under another name or in another bundle) is a duplicate and is skipped.
|
|
245
|
+
*/
|
|
246
|
+
function selectFusedEntries(db, fused, limit, filterOptions) {
|
|
247
|
+
const selected = [];
|
|
248
|
+
const seenPaths = new Set();
|
|
249
|
+
const seenContent = new Set();
|
|
250
|
+
const batchSize = Math.max(limit * 2, 20);
|
|
251
|
+
for (let offset = 0; offset < fused.length && selected.length < limit; offset += batchSize) {
|
|
252
|
+
const batch = [];
|
|
253
|
+
for (const candidate of fused.slice(offset, offset + batchSize)) {
|
|
254
|
+
const row = getEntryById(db, candidate.id);
|
|
255
|
+
if (!row || !hasIndexedProvenance(row) || seenPaths.has(row.filePath))
|
|
256
|
+
continue;
|
|
257
|
+
seenPaths.add(row.filePath);
|
|
258
|
+
batch.push({ ...row, id: candidate.id, candidate });
|
|
259
|
+
}
|
|
260
|
+
for (const kept of applyEntryFilters(batch, filterOptions)) {
|
|
261
|
+
const { candidate, ...row } = kept;
|
|
262
|
+
const content = row.entry.content?.replace(/\s+/g, " ").trim();
|
|
263
|
+
if (content) {
|
|
264
|
+
if (seenContent.has(content))
|
|
265
|
+
continue;
|
|
266
|
+
seenContent.add(content);
|
|
267
|
+
}
|
|
268
|
+
selected.push({ candidate, row });
|
|
343
269
|
}
|
|
344
270
|
}
|
|
345
|
-
|
|
346
|
-
const scored = combineSearchScores({
|
|
347
|
-
ftsScoreMap,
|
|
348
|
-
embedScoreMap,
|
|
349
|
-
getEntryById: (id) => getEntryById(db, id) ?? undefined,
|
|
350
|
-
typeFilter,
|
|
351
|
-
// #627 — also exclude default-hidden types from the vector-only branch so a
|
|
352
|
-
// session asset that is a top-k vector neighbor (but not an FTS match) does
|
|
353
|
-
// not leak into default ('any') results. defaultExcludes is already []
|
|
354
|
-
// unless this is the untyped path without includeExcludedTypes.
|
|
355
|
-
excludeTypes: defaultExcludes,
|
|
356
|
-
}).filter(hasIndexedProvenance);
|
|
357
|
-
// ── Scoring Phase ──────────────────────────────────────────────────────
|
|
358
|
-
// Apply boosts as multiplicative factors (all boosts in a single phase
|
|
359
|
-
// so that sort order and displayed scores are always consistent).
|
|
360
|
-
// Ranking philosophy: the goal is to surface the MOST USEFUL result for the
|
|
361
|
-
// user's intent. An exact name match is the strongest signal. Actionable
|
|
362
|
-
// asset types (skills, commands, agents) are more useful than passive
|
|
363
|
-
// reference docs. Curated metadata is more reliable than auto-generated.
|
|
364
|
-
// Graph boost context (#207). Built once per query and reused across
|
|
365
|
-
// every scored entry so the disk read + JSON parse only happens once
|
|
366
|
-
// per search invocation. `null` when no graph file is present, when
|
|
367
|
-
// the schema doesn't match, or when no query token matches a graph
|
|
368
|
-
// entity — in all of those cases the per-entry call is skipped and
|
|
369
|
-
// graph contributes nothing. The graph signal feeds this single
|
|
370
|
-
// FTS5+boosts loop as ONE additive component (CLAUDE.md / spec §6:
|
|
371
|
-
// one scoring pipeline, no parallel SearchHit scorer).
|
|
372
|
-
const graphContext = (() => {
|
|
373
|
-
// Search across all source dirs; the graph file lives next to the
|
|
374
|
-
// primary source root. Cache misses are silent — the helper handles
|
|
375
|
-
// missing files internally and returns `null` instead of throwing.
|
|
376
|
-
if (allSourceDirs.length === 0)
|
|
377
|
-
return null;
|
|
378
|
-
return loadGraphBoostContext(allSourceDirs, query, config, db);
|
|
379
|
-
})();
|
|
380
|
-
// Resolve project-context tokens from the current working directory once
|
|
381
|
-
// per search invocation. Returns null when running from home dir / /tmp,
|
|
382
|
-
// or when the caller passed `--no-project-context` (disableProjectContext).
|
|
383
|
-
const projectContext = disableProjectContext ? null : resolveProjectContext(process.cwd());
|
|
384
|
-
// Phase 2A / Rec 5: resolve forgetting-curve config and skip the feedback
|
|
385
|
-
// count query when the boost cannot make a difference (default ≤ 1.0 means
|
|
386
|
-
// boost^count == 1 — zero overhead for the common case).
|
|
387
|
-
const utilityDecayRaw = config.improve?.utilityDecay;
|
|
388
|
-
const halfLifeDays = utilityDecayRaw?.halfLifeDays ?? 30;
|
|
389
|
-
const feedbackStabilityBoost = utilityDecayRaw?.feedbackStabilityBoost ?? 1.5;
|
|
390
|
-
const utilityDecayConfig = utilityDecayRaw !== undefined ? { halfLifeDays, feedbackStabilityBoost } : undefined;
|
|
391
|
-
// Gate the feedback-count query on the user having explicitly opted into
|
|
392
|
-
// utilityDecay. Without an opt-in, `utilityDecayConfig` is undefined and the
|
|
393
|
-
// ranking contributor ignores `positiveFeedbackCounts` — so running the DB
|
|
394
|
-
// query here would be pure overhead. The boost > 1.0 sub-gate then skips the
|
|
395
|
-
// query when the configured boost is a no-op (1.5^count when boost==1 is 1).
|
|
396
|
-
const positiveFeedbackCounts = shouldQueryPositiveFeedbackCounts(utilityDecayRaw)
|
|
397
|
-
? getPositiveFeedbackCountsByIds(scored.map((item) => item.id))
|
|
398
|
-
: undefined;
|
|
399
|
-
// Resolve per-project scope key for scoped utility scoring.
|
|
400
|
-
// `disableScopedUtility` (wired from `akm search --no-project-context`)
|
|
401
|
-
// opts out (e.g. for registry searches or tests).
|
|
402
|
-
let scopeKey;
|
|
403
|
-
try {
|
|
404
|
-
scopeKey = disableScopedUtility ? undefined : getCurrentWorkflowScopeKey();
|
|
405
|
-
}
|
|
406
|
-
catch {
|
|
407
|
-
// Non-fatal — ranking proceeds without scoped utility on any error.
|
|
408
|
-
}
|
|
409
|
-
// 03-R3: derived twins inherit their base's demoting belief state before
|
|
410
|
-
// ranking, so the (03) belief-state ranker demotes a stale flag-free twin.
|
|
411
|
-
inheritDerivedTwinBeliefStates(db, scored);
|
|
412
|
-
applyRankingRules({
|
|
413
|
-
db,
|
|
414
|
-
query,
|
|
415
|
-
items: scored,
|
|
416
|
-
graphContext,
|
|
417
|
-
projectContext,
|
|
418
|
-
utilityDecayConfig,
|
|
419
|
-
positiveFeedbackCounts,
|
|
420
|
-
scopeKey,
|
|
421
|
-
});
|
|
422
|
-
// ── minScore floor ──────────────────────────────────────────────────────
|
|
423
|
-
// Drop semantic-only hits (cosine-only, no FTS match) whose score falls
|
|
424
|
-
// below the configured floor. FTS hits and hybrid hits are always kept.
|
|
425
|
-
// Default floor: 0.2. Set search.minScore = 0 in config to disable.
|
|
426
|
-
// Judged on the PRE-ceiling score when a demoting belief state clamped the
|
|
427
|
-
// item (`preCeilingScore`): the belief ceilings can sit below this floor
|
|
428
|
-
// (archived 0.15 < 0.2), and a demotion must rank the hit last, not
|
|
429
|
-
// silently remove a result that would otherwise have listed.
|
|
430
|
-
const minScore = config.search?.minScore ?? 0.2;
|
|
431
|
-
const preFilter = minScore > 0
|
|
432
|
-
? scored.filter((item) => item.rankingMode !== "semantic" || (item.preCeilingScore ?? item.score) >= minScore)
|
|
433
|
-
: scored;
|
|
434
|
-
preFilter.sort(buildSearchResultComparator(query));
|
|
435
|
-
// Deduplicate by file path — keep only the highest-scored entry per file.
|
|
436
|
-
const deduped = deduplicateByPath(preFilter);
|
|
437
|
-
// Source → scope → proposed-quality → derived-twin belief inheritance →
|
|
438
|
-
// belief: the post-candidate filter chain shared with enumerateEntries (see
|
|
439
|
-
// applyEntryFilters). Applied AFTER ranking so filtering narrows the result
|
|
440
|
-
// set without touching the single FTS5+boosts scoring pipeline. The twin
|
|
441
|
-
// inheritance inside the chain re-runs here as an idempotent no-op — it
|
|
442
|
-
// already ran on the full candidate pool before ranking (:460) to feed the
|
|
443
|
-
// belief-state ranker.
|
|
444
|
-
const beliefFiltered = applyEntryFilters(deduped, {
|
|
445
|
-
db,
|
|
446
|
-
sources,
|
|
447
|
-
restrictToSources,
|
|
448
|
-
filters,
|
|
449
|
-
includeProposed,
|
|
450
|
-
beliefFilter,
|
|
451
|
-
});
|
|
452
|
-
const rankMs = Date.now() - tRank0;
|
|
453
|
-
const selected = beliefFiltered.slice(0, limit);
|
|
454
|
-
const fragmentSelections = selected.flatMap((ranked) => ranked.fragmentId && allowsFragmentRef(ranked.entry.type) && hasIndexedProvenance(ranked)
|
|
455
|
-
? [{ entryId: ranked.id, itemRef: ranked.itemRef, fragmentId: ranked.fragmentId }]
|
|
456
|
-
: []);
|
|
457
|
-
const selectedFragments = getIndexedMarkdownFragments(db, fragmentSelections);
|
|
458
|
-
const selectedFragmentByEntryId = new Map();
|
|
459
|
-
fragmentSelections.forEach((selection, index) => {
|
|
460
|
-
selectedFragmentByEntryId.set(selection.entryId, selectedFragments[index]);
|
|
461
|
-
});
|
|
462
|
-
const hits = await Promise.all(selected.map((ranked) => {
|
|
463
|
-
const { entry, filePath, score, rankingMode, utilityBoosted } = ranked;
|
|
464
|
-
// CLAUDE.md locks SearchHit.score in [0,1]. The boost loop deliberately
|
|
465
|
-
// remains raw for ranking, then takes a monotone bounded projection at
|
|
466
|
-
// the public boundary so contributors do not collapse into hard-clamped
|
|
467
|
-
// ties.
|
|
468
|
-
const finalScore = displaySearchScore(score);
|
|
469
|
-
return buildDbHit({
|
|
470
|
-
entry,
|
|
471
|
-
path: filePath,
|
|
472
|
-
...indexedProvenance(ranked),
|
|
473
|
-
score: Math.round(finalScore * 10000) / 10000,
|
|
474
|
-
query,
|
|
475
|
-
rankingMode,
|
|
476
|
-
lexicalMatch: ranked.lexicalMatch,
|
|
477
|
-
fragmentId: ranked.fragmentId,
|
|
478
|
-
indexedFragment: ranked.fragmentId ? (selectedFragmentByEntryId.get(ranked.id) ?? null) : undefined,
|
|
479
|
-
defaultStashDir: stashDir,
|
|
480
|
-
allSourceDirs,
|
|
481
|
-
sources,
|
|
482
|
-
config,
|
|
483
|
-
utilityBoosted,
|
|
484
|
-
graphContext,
|
|
485
|
-
attributionSource: ranked,
|
|
486
|
-
rendererRegistry,
|
|
487
|
-
db,
|
|
488
|
-
});
|
|
489
|
-
}));
|
|
490
|
-
return { embedMs, rankMs, hits, mode, semanticWarning };
|
|
271
|
+
return selected.slice(0, limit);
|
|
491
272
|
}
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
const
|
|
495
|
-
const
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
273
|
+
/** Vector candidates of the requested type, nearest first; equal distances are ordered by ref. */
|
|
274
|
+
function keepAllowedTypes(db, neighbors, typeFilter, excludeTypes) {
|
|
275
|
+
const rows = getEntryRefsAndTypes(db, neighbors.map((neighbor) => neighbor.id));
|
|
276
|
+
const excluded = new Set(excludeTypes);
|
|
277
|
+
return neighbors
|
|
278
|
+
.flatMap(({ id, distance }) => {
|
|
279
|
+
const row = rows.get(id);
|
|
280
|
+
if (!row)
|
|
281
|
+
return [];
|
|
282
|
+
if (typeFilter ? row.type !== typeFilter : excluded.has(row.type))
|
|
283
|
+
return [];
|
|
284
|
+
return [{ id, itemRef: row.itemRef, distance }];
|
|
285
|
+
})
|
|
286
|
+
.sort((a, b) => a.distance - b.distance || compareCodePoints(a.itemRef, b.itemRef))
|
|
287
|
+
.map(({ id, itemRef }) => ({ id, itemRef }));
|
|
288
|
+
}
|
|
289
|
+
/** `whyMatched` for a fused hit: its rank in each channel that returned it. */
|
|
290
|
+
function describeRanks(candidate) {
|
|
291
|
+
const [lexicalRank, vectorRank] = candidate.ranks;
|
|
292
|
+
return [
|
|
293
|
+
...(lexicalRank !== undefined ? [`lexical rank ${lexicalRank}`] : []),
|
|
294
|
+
...(vectorRank !== undefined ? [`vector rank ${vectorRank}`] : []),
|
|
295
|
+
];
|
|
509
296
|
}
|
|
510
297
|
/**
|
|
511
298
|
* The no-hits tip. A query in the retired `<type>:` / `<type>:<prefix>/` browse
|
|
@@ -526,13 +313,13 @@ function emptyResultTip(query) {
|
|
|
526
313
|
/**
|
|
527
314
|
* Enumerate index entries without FTS scoring — the browse path shared by
|
|
528
315
|
* empty/unsearchable queries and D4 conceptId-prefix queries (`memories/`,
|
|
529
|
-
* `bundle//`, `bundle//skills/`). Applies the same
|
|
316
|
+
* `bundle//`, `bundle//skills/`). Applies the same filters as the scored
|
|
530
317
|
* path (source narrowing, scope, proposed-quality, belief) before the limit
|
|
531
318
|
* slice. Hits carry the fixed browse score 1 in type-then-name order — this is
|
|
532
319
|
* a deterministic listing, not a relevance ranking.
|
|
533
320
|
*/
|
|
534
321
|
async function enumerateEntries(opts) {
|
|
535
|
-
const { db,
|
|
322
|
+
const { db, sources, config } = opts;
|
|
536
323
|
const allEntries = getAllEntries(db, opts.typeFilter, opts.excludeTypes).filter(hasIndexedProvenance);
|
|
537
324
|
// Explicit listing order: type, then name, then filePath. The underlying
|
|
538
325
|
// SELECT carries no ORDER BY, so its row order tracks the query plan and the
|
|
@@ -552,78 +339,34 @@ async function enumerateEntries(opts) {
|
|
|
552
339
|
const prefixFiltered = conceptIdPrefix.length > 0
|
|
553
340
|
? bundleFiltered.filter((ie) => ie.conceptId.toLowerCase().startsWith(conceptIdPrefix))
|
|
554
341
|
: bundleFiltered;
|
|
555
|
-
// Deduplicate by file path — multiple entries can share the same file
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
seenFilePaths.add(ie.filePath);
|
|
561
|
-
return true;
|
|
562
|
-
});
|
|
563
|
-
// Source → scope → proposed-quality → derived-twin belief inheritance →
|
|
564
|
-
// belief: the post-candidate filter chain shared with searchDatabase's
|
|
565
|
-
// scored path (see applyEntryFilters). Filtering happens BEFORE the limit
|
|
566
|
-
// slice so a restrictive filter still returns up to `limit` results. On this
|
|
567
|
-
// path the twin inheritance is the ONLY place it runs (there is no ranking
|
|
568
|
-
// pass), keeping the belief filter and reported hit state consistent with
|
|
569
|
-
// the scored path.
|
|
570
|
-
const beliefFiltered = applyEntryFilters(uniqueEntries, {
|
|
571
|
-
db,
|
|
572
|
-
sources,
|
|
573
|
-
restrictToSources: opts.restrictToSources,
|
|
574
|
-
filters,
|
|
575
|
-
includeProposed: opts.includeProposed,
|
|
576
|
-
beliefFilter,
|
|
577
|
-
});
|
|
578
|
-
const selected = beliefFiltered.slice(0, opts.limit);
|
|
579
|
-
const hits = await Promise.all(selected.map((ie) => buildDbHit({
|
|
342
|
+
// Deduplicate by file path — multiple entries can share the same file.
|
|
343
|
+
// Filtering happens BEFORE the limit slice so a restrictive filter still
|
|
344
|
+
// returns up to `limit` results.
|
|
345
|
+
const selected = applyEntryFilters(deduplicateByPath(prefixFiltered), opts).slice(0, opts.limit);
|
|
346
|
+
return Promise.all(selected.map((ie) => buildDbHit({
|
|
580
347
|
entry: ie.entry,
|
|
581
348
|
path: ie.filePath,
|
|
582
349
|
itemRef: ie.itemRef,
|
|
583
350
|
bundleId: ie.bundleId,
|
|
584
351
|
conceptId: ie.conceptId,
|
|
585
352
|
score: 1,
|
|
586
|
-
query,
|
|
587
|
-
rankingMode: "fts",
|
|
588
353
|
defaultStashDir: opts.stashDir,
|
|
589
|
-
allSourceDirs: opts.allSourceDirs,
|
|
590
354
|
sources,
|
|
591
355
|
config,
|
|
592
|
-
rendererRegistry,
|
|
593
356
|
db,
|
|
594
357
|
})));
|
|
595
|
-
return { hits };
|
|
596
358
|
}
|
|
597
359
|
/**
|
|
598
|
-
*
|
|
599
|
-
*
|
|
600
|
-
*
|
|
601
|
-
* belief inheritance → belief filter. Extracting the chain removes the two
|
|
602
|
-
* paths' formerly-duplicated filter sequences so the predicates, their order,
|
|
603
|
-
* and the twin-inheritance placement can never drift apart (plan §4.3).
|
|
604
|
-
*
|
|
605
|
-
* What this does NOT unify — and deliberately leaves divergent — is CANDIDATE-
|
|
606
|
-
* POOL construction, which is inherent search-vs-browse semantics: the scored
|
|
607
|
-
* path's pool is `searchFts`/vector matches for the query's own tokens (FTS
|
|
608
|
-
* includes structured fields and bounded adapter content), while the
|
|
609
|
-
* enumerate path's pool is `getAllEntries` for the type, independent of query
|
|
610
|
-
* text. A derived twin sharing no indexed token with the query is therefore an
|
|
611
|
-
* enumerate-path candidate but never a scored-path candidate. (A golden
|
|
612
|
-
* fixture used to pin that divergence; the golden suites were deleted in
|
|
613
|
-
* 0.9.8, so this comment is now the record of it.)
|
|
614
|
-
*
|
|
615
|
-
* `inheritDerivedTwinBeliefStates` is idempotent, so running it here is safe on
|
|
616
|
-
* the scored path, which must ALSO call it before ranking (the belief-state
|
|
617
|
-
* ranker demotes inherited states): by the time this chain runs, those twins
|
|
618
|
-
* already carry a state and the call here is a no-op for them. The enumerate
|
|
619
|
-
* path never ranks, so this is the only place it inherits.
|
|
360
|
+
* Filter chain shared by the scored and browse paths, in this order:
|
|
361
|
+
* source-narrowing → scope → proposed-quality → derived-twin belief
|
|
362
|
+
* inheritance → belief filter.
|
|
620
363
|
*/
|
|
621
364
|
function applyEntryFilters(items, opts) {
|
|
622
365
|
const { filters } = opts;
|
|
623
366
|
// Source filter: when the caller narrowed `sources` via `--from <name>`,
|
|
624
367
|
// drop entries whose filePath does not live under any requested source. The
|
|
625
|
-
//
|
|
626
|
-
//
|
|
368
|
+
// index spans every configured source, so without this filter a narrowed
|
|
369
|
+
// --from request would still leak results from other sources.
|
|
627
370
|
const sourceFiltered = opts.restrictToSources
|
|
628
371
|
? items.filter((item) => findSourceForPath(item.filePath, opts.sources) !== undefined)
|
|
629
372
|
: items;
|
|
@@ -645,14 +388,12 @@ function applyEntryFilters(items, opts) {
|
|
|
645
388
|
}
|
|
646
389
|
/**
|
|
647
390
|
* 03-R3: let each `.derived` twin inherit its base memory's demoting belief
|
|
648
|
-
* state
|
|
649
|
-
*
|
|
650
|
-
*
|
|
651
|
-
*
|
|
652
|
-
*
|
|
653
|
-
*
|
|
654
|
-
* improve run, erasing it. Only twins with no state of their own inherit; an
|
|
655
|
-
* explicit twin state always wins. Reuses the (03) belief-state ranker + filter.
|
|
391
|
+
* state, so `--belief` treats a stale flag-free twin like its corrected base.
|
|
392
|
+
* The base carries the flag; its near-duplicate `.derived` twin carries none.
|
|
393
|
+
* Done in-memory at search time — NOT by writing the twin's frontmatter —
|
|
394
|
+
* because the SCC belief resolver refreshes any non-frozen state written to a
|
|
395
|
+
* derived memory back to `active` on the next improve run, erasing it. Only
|
|
396
|
+
* twins with no state of their own inherit; an explicit twin state always wins.
|
|
656
397
|
*/
|
|
657
398
|
function inheritDerivedTwinBeliefStates(db, items) {
|
|
658
399
|
const DEMOTING = new Set(["contradicted", "superseded", "deprecated", "archived"]);
|
|
@@ -687,33 +428,36 @@ function matchBeliefFilter(beliefState, filter) {
|
|
|
687
428
|
beliefState === "deprecated" ||
|
|
688
429
|
beliefState === "archived");
|
|
689
430
|
}
|
|
690
|
-
// ── Vector
|
|
691
|
-
|
|
431
|
+
// ── Vector channel ──────────────────────────────────────────────────────────
|
|
432
|
+
/**
|
|
433
|
+
* Embed the query and return its nearest document ids, best first. The
|
|
434
|
+
* embedding request is dispatched before this returns, so the caller's FTS
|
|
435
|
+
* query overlaps it. A slow or failing embedder degrades the search to keyword
|
|
436
|
+
* ranking with a warning after `embedding.queryTimeoutMs`.
|
|
437
|
+
*/
|
|
438
|
+
function startVectorChannel(db, query, config) {
|
|
692
439
|
if (config.semanticSearchMode === "off")
|
|
693
|
-
return {
|
|
694
|
-
// A real-time completeness fact, not a cached verdict: skip the
|
|
695
|
-
//
|
|
696
|
-
//
|
|
697
|
-
//
|
|
698
|
-
// `semanticWarning` below instead of silently skipping with no signal.
|
|
440
|
+
return Promise.resolve({ neighbors: null });
|
|
441
|
+
// A real-time completeness fact, not a cached verdict: skip the round trip
|
|
442
|
+
// only when the index has never embedded anything. A PARTIAL failure still
|
|
443
|
+
// attempts — and if the endpoint is genuinely down, the failure surfaces as
|
|
444
|
+
// a live warning instead of silently skipping with no signal.
|
|
699
445
|
if (getEmbeddingCount(db) === 0)
|
|
700
|
-
return {
|
|
701
|
-
|
|
702
|
-
|
|
703
|
-
|
|
704
|
-
|
|
705
|
-
|
|
706
|
-
|
|
707
|
-
|
|
708
|
-
|
|
709
|
-
|
|
710
|
-
|
|
711
|
-
|
|
712
|
-
|
|
713
|
-
|
|
714
|
-
|
|
715
|
-
return { scores: null, warning: buildVectorFallbackWarning(config, error) };
|
|
716
|
-
}
|
|
446
|
+
return Promise.resolve({ neighbors: null });
|
|
447
|
+
const timeoutMs = config.embedding?.queryTimeoutMs ?? DEFAULT_QUERY_EMBED_TIMEOUT_MS;
|
|
448
|
+
const controller = new AbortController();
|
|
449
|
+
let timer;
|
|
450
|
+
const timeout = new Promise((_, reject) => {
|
|
451
|
+
timer = setTimeout(() => {
|
|
452
|
+
controller.abort();
|
|
453
|
+
reject(new Error(`Query embedding timed out after ${timeoutMs}ms`));
|
|
454
|
+
}, timeoutMs);
|
|
455
|
+
});
|
|
456
|
+
const queryText = applyEmbeddingTemplate(resolveEmbeddingProfile(config.embedding).queryTemplate, query);
|
|
457
|
+
return Promise.race([embed(queryText, config.embedding, controller.signal), timeout])
|
|
458
|
+
.then((vector) => ({ neighbors: searchVec(db, vector, CHANNEL_DEPTH) }))
|
|
459
|
+
.catch((error) => ({ neighbors: null, warning: buildVectorFallbackWarning(config, error) }))
|
|
460
|
+
.finally(() => clearTimeout(timer));
|
|
717
461
|
}
|
|
718
462
|
function buildVectorFallbackWarning(config, error) {
|
|
719
463
|
const endpoint = safeEmbeddingEndpoint(config);
|
|
@@ -770,99 +514,39 @@ function classifyVectorFailure(error) {
|
|
|
770
514
|
}
|
|
771
515
|
// ── Hit building ────────────────────────────────────────────────────────────
|
|
772
516
|
export async function buildDbHit(input) {
|
|
773
|
-
const rendererRegistry = input.rendererRegistry ?? defaultRendererRegistry;
|
|
774
517
|
const absolutePath = path.resolve(input.path);
|
|
775
|
-
const entryStashDir = findSourceForPath(absolutePath, input.sources)?.path ?? input.defaultStashDir;
|
|
776
|
-
// Quality and confidence boosts are now applied in the main scoring
|
|
777
|
-
// phase (searchDatabase). buildDbHit receives the already-final score and
|
|
778
|
-
// passes it through without further multiplication. We still compute the
|
|
779
|
-
// boost values here for buildWhyMatched reporting.
|
|
780
|
-
// Mirrors the boost computation in `searchDatabase`; only `curated`
|
|
781
|
-
// contributes a positive boost. Used for `whyMatched` reporting only.
|
|
782
|
-
const qualityBoost = input.entry.quality === "curated" ? 0.05 : 0;
|
|
783
|
-
const confidenceBoost = typeof input.entry.confidence === "number" ? Math.min(0.05, Math.max(0, input.entry.confidence) * 0.05) : 0;
|
|
784
|
-
// Round to 4 decimal places, no boost multiplication
|
|
785
|
-
const score = Math.round(input.score * 10000) / 10000;
|
|
786
|
-
const graphBoost = getSearchHitAttribution(input.attributionSource ?? {})?.graphExtraction?.boost ?? 0;
|
|
787
|
-
const whyMatched = buildWhyMatched(input.entry, input.query, input.rankingMode, qualityBoost, confidenceBoost, input.utilityBoosted, graphBoost, input.lexicalMatch);
|
|
788
|
-
const graphHit = input.graphContext ? collectGraphRelatedHit(input.graphContext, absolutePath) : null;
|
|
789
518
|
const source = findSourceForPath(absolutePath, input.sources);
|
|
519
|
+
const entryStashDir = source?.path ?? input.defaultStashDir;
|
|
790
520
|
const defaultBundleId = input.config?.defaultBundle ??
|
|
791
521
|
(source && path.resolve(source.path) === path.resolve(input.defaultStashDir)
|
|
792
522
|
? (input.bundleId ?? undefined)
|
|
793
523
|
: undefined);
|
|
794
|
-
const
|
|
795
|
-
// Fragments prove lexical relevance, but executable assets must retain the
|
|
796
|
-
// parent ref consumed by their advertised action (for example workflow run).
|
|
797
|
-
// The central type-presentation contract opts those types out explicitly.
|
|
798
|
-
const ref = input.fragmentId && allowsFragmentRef(input.entry.type) ? `${parentRef}#${input.fragmentId}` : parentRef;
|
|
524
|
+
const ref = resolveSearchHitRef(input.entry, input, defaultBundleId);
|
|
799
525
|
const editable = isEditable(absolutePath, input.config, input.sources);
|
|
800
|
-
const
|
|
801
|
-
? input.fragmentId && input.db
|
|
802
|
-
? getIndexedMarkdownFragment(input.db, input.itemRef, input.fragmentId)
|
|
803
|
-
: undefined
|
|
804
|
-
: (input.indexedFragment ?? undefined);
|
|
805
|
-
const selectedRef = input.fragmentId && ref !== parentRef ? `${parentRef}#${input.fragmentId}` : undefined;
|
|
806
|
-
const parentEstimatedTokens = typeof input.entry.fileSize === "number"
|
|
807
|
-
? Math.round(input.entry.fileSize / 4)
|
|
808
|
-
: indexedFragment
|
|
809
|
-
? Math.round(indexedFragment.parentChars / 4)
|
|
810
|
-
: undefined;
|
|
811
|
-
const fragmentEstimatedTokens = indexedFragment ? Math.round(indexedFragment.fragmentChars / 4) : undefined;
|
|
812
|
-
const estimatedTokens = selectedRef === ref && fragmentEstimatedTokens !== undefined ? fragmentEstimatedTokens : parentEstimatedTokens;
|
|
526
|
+
const estimatedTokens = typeof input.entry.fileSize === "number" ? Math.round(input.entry.fileSize / 4) : undefined;
|
|
813
527
|
const hit = {
|
|
814
528
|
type: input.entry.type,
|
|
815
529
|
name: input.entry.name,
|
|
816
530
|
path: absolutePath,
|
|
817
531
|
ref,
|
|
818
|
-
origin:
|
|
532
|
+
origin: source?.registryId ?? null,
|
|
819
533
|
editable,
|
|
820
534
|
...(!editable ? { editHint: buildEditHint(ref) } : {}),
|
|
821
535
|
description: input.entry.description,
|
|
822
536
|
tags: input.entry.tags,
|
|
823
537
|
size: deriveSize(input.entry.fileSize),
|
|
824
|
-
action: buildLocalAction(input.entry.type, ref
|
|
825
|
-
score,
|
|
826
|
-
whyMatched,
|
|
538
|
+
action: buildLocalAction(input.entry.type, ref),
|
|
539
|
+
score: input.score,
|
|
540
|
+
...(input.whyMatched ? { whyMatched: input.whyMatched } : {}),
|
|
827
541
|
...(estimatedTokens !== undefined ? { estimatedTokens } : {}),
|
|
828
|
-
...(selectedRef
|
|
829
|
-
? {
|
|
830
|
-
selectedRef,
|
|
831
|
-
parentRef,
|
|
832
|
-
...(indexedFragment
|
|
833
|
-
? {
|
|
834
|
-
fragmentOrdinal: indexedFragment.ordinal + 1,
|
|
835
|
-
fragmentCount: indexedFragment.count,
|
|
836
|
-
startLine: indexedFragment.startLine,
|
|
837
|
-
endLine: indexedFragment.endLine,
|
|
838
|
-
...(indexedFragment.previousFragmentId
|
|
839
|
-
? { previousRef: `${parentRef}#${indexedFragment.previousFragmentId}` }
|
|
840
|
-
: {}),
|
|
841
|
-
...(indexedFragment.nextFragmentId
|
|
842
|
-
? { nextRef: `${parentRef}#${indexedFragment.nextFragmentId}` }
|
|
843
|
-
: {}),
|
|
844
|
-
fragmentChars: indexedFragment.fragmentChars,
|
|
845
|
-
fragmentEstimatedTokens,
|
|
846
|
-
parentChars: indexedFragment.parentChars,
|
|
847
|
-
...(parentEstimatedTokens !== undefined ? { parentEstimatedTokens } : {}),
|
|
848
|
-
}
|
|
849
|
-
: parentEstimatedTokens !== undefined
|
|
850
|
-
? { parentEstimatedTokens }
|
|
851
|
-
: {}),
|
|
852
|
-
}
|
|
853
|
-
: {}),
|
|
854
542
|
// Surface optional quality (v1 spec §4.2). Omitted when entry has
|
|
855
543
|
// no `quality` field so payloads stay compact for the common case.
|
|
856
544
|
...(input.entry.quality ? { quality: input.entry.quality } : {}),
|
|
857
545
|
...(input.entry.beliefState ? { beliefState: input.entry.beliefState } : {}),
|
|
858
546
|
...(input.entry.currentBeliefRefs ? { currentBeliefRefs: input.entry.currentBeliefRefs } : {}),
|
|
859
|
-
...(graphHit ? { graph: { entities: graphHit.entities, relations: graphHit.relations } } : {}),
|
|
860
|
-
// Which stage of the progressive AND->OR lexical ladder produced this
|
|
861
|
-
// hit. Omitted when the hit has no FTS component (pure-semantic hybrid
|
|
862
|
-
// contribution).
|
|
863
|
-
...(input.lexicalMatch ? { matchStage: input.lexicalMatch } : {}),
|
|
864
547
|
};
|
|
865
|
-
|
|
548
|
+
if (input.entry.content)
|
|
549
|
+
hitContent.set(hit, input.entry.content);
|
|
866
550
|
if (input.entry.derivedFrom) {
|
|
867
551
|
attachSearchHitAttribution(hit, {
|
|
868
552
|
memoryInference: { exposure: "direct" },
|
|
@@ -872,90 +556,10 @@ export async function buildDbHit(input) {
|
|
|
872
556
|
type: input.entry.type,
|
|
873
557
|
stashDir: entryStashDir,
|
|
874
558
|
bundleId: input.bundleId,
|
|
875
|
-
rendererRegistry,
|
|
876
559
|
db: input.db,
|
|
877
560
|
});
|
|
878
561
|
return hit;
|
|
879
562
|
}
|
|
880
|
-
function attachDbHitAttribution(hit, input) {
|
|
881
|
-
if (input.lexicalMatch) {
|
|
882
|
-
attachSearchHitAttribution(hit, {
|
|
883
|
-
lexical: {
|
|
884
|
-
execution: input.lexicalMatch,
|
|
885
|
-
nameMatchTier: lexicalNameMatchTier(input.entry, buildLexicalQueryPlan(input.query).tokens),
|
|
886
|
-
},
|
|
887
|
-
});
|
|
888
|
-
}
|
|
889
|
-
if (input.attributionSource)
|
|
890
|
-
copySearchHitAttribution(input.attributionSource, hit);
|
|
891
|
-
}
|
|
892
|
-
export function buildWhyMatched(entry, query,
|
|
893
|
-
// "hybrid" ranking mode
|
|
894
|
-
rankingMode, qualityBoost, confidenceBoost, utilityBoosted, graphBoost, lexicalMatch) {
|
|
895
|
-
const reasons = [
|
|
896
|
-
rankingMode === "hybrid"
|
|
897
|
-
? "hybrid (fts + semantic)"
|
|
898
|
-
: rankingMode === "semantic"
|
|
899
|
-
? "semantic similarity"
|
|
900
|
-
: "fts bm25 relevance",
|
|
901
|
-
];
|
|
902
|
-
if (lexicalMatch === "relaxed")
|
|
903
|
-
reasons.push("lexical recovery after strict query returned no hits");
|
|
904
|
-
if (lexicalMatch === "prefix")
|
|
905
|
-
reasons.push("prefix match after strict query returned no hits");
|
|
906
|
-
const tokens = query.toLowerCase().split(/\s+/).filter(Boolean);
|
|
907
|
-
const queryLower = query.toLowerCase().trim();
|
|
908
|
-
const name = entry.name.toLowerCase();
|
|
909
|
-
const nameBase = name.split("/").pop() ?? name;
|
|
910
|
-
const tags = entry.tags?.join(" ").toLowerCase() ?? "";
|
|
911
|
-
const searchHints = entry.searchHints?.join(" ").toLowerCase() ?? "";
|
|
912
|
-
const aliases = entry.aliases?.join(" ").toLowerCase() ?? "";
|
|
913
|
-
const desc = entry.description?.toLowerCase() ?? "";
|
|
914
|
-
// Name match quality
|
|
915
|
-
if (nameBase === queryLower || name === queryLower) {
|
|
916
|
-
reasons.push("exact name match");
|
|
917
|
-
}
|
|
918
|
-
else if (nameBase.includes(queryLower) || queryLower.includes(nameBase)) {
|
|
919
|
-
reasons.push("near-exact name match");
|
|
920
|
-
}
|
|
921
|
-
else if (tokens.some((t) => nameBase.includes(t))) {
|
|
922
|
-
reasons.push("matched name tokens");
|
|
923
|
-
}
|
|
924
|
-
// Type relevance
|
|
925
|
-
if (entry.type === "skill" || entry.type === "command" || entry.type === "agent") {
|
|
926
|
-
reasons.push(`${entry.type} type boost`);
|
|
927
|
-
}
|
|
928
|
-
if (tokens.some((t) => tags.includes(t)))
|
|
929
|
-
reasons.push("matched tags");
|
|
930
|
-
if (tokens.some((t) => searchHints.includes(t)))
|
|
931
|
-
reasons.push("matched searchHints");
|
|
932
|
-
if (tokens.some((t) => aliases.includes(t)))
|
|
933
|
-
reasons.push("matched aliases");
|
|
934
|
-
if (tokens.some((t) => desc.includes(t)))
|
|
935
|
-
reasons.push("matched description");
|
|
936
|
-
if (qualityBoost > 0)
|
|
937
|
-
reasons.push("curated metadata boost");
|
|
938
|
-
if (confidenceBoost > 0)
|
|
939
|
-
reasons.push("metadata confidence boost");
|
|
940
|
-
if (entry.beliefState === "active")
|
|
941
|
-
reasons.push("active belief state");
|
|
942
|
-
if (entry.beliefState === "asserted")
|
|
943
|
-
reasons.push("asserted belief state");
|
|
944
|
-
if (entry.beliefState === "contradicted")
|
|
945
|
-
reasons.push("contradicted belief state");
|
|
946
|
-
if (entry.beliefState === "superseded")
|
|
947
|
-
reasons.push("superseded belief state");
|
|
948
|
-
if (entry.beliefState === "deprecated")
|
|
949
|
-
reasons.push("deprecated belief state");
|
|
950
|
-
if (entry.beliefState === "archived")
|
|
951
|
-
reasons.push("archived belief state");
|
|
952
|
-
if (utilityBoosted)
|
|
953
|
-
reasons.push("usage history boost");
|
|
954
|
-
if (typeof graphBoost === "number" && graphBoost > 0) {
|
|
955
|
-
reasons.push(`graph boost +${graphBoost.toFixed(2)}`);
|
|
956
|
-
}
|
|
957
|
-
return reasons;
|
|
958
|
-
}
|
|
959
563
|
// ── Utilities ────────────────────────────────────────────────────────────────
|
|
960
564
|
export function deriveSize(bytes) {
|
|
961
565
|
if (bytes === undefined)
|
|
@@ -966,11 +570,7 @@ export function deriveSize(bytes) {
|
|
|
966
570
|
return "medium";
|
|
967
571
|
return "large";
|
|
968
572
|
}
|
|
969
|
-
/**
|
|
970
|
-
* Deduplicate the already-ranked result stream by file path. The caller owns
|
|
971
|
-
* the one ranking order; re-sorting here would silently discard exact-name and
|
|
972
|
-
* relaxed-recovery ordering in favor of an internal pre-clamp score.
|
|
973
|
-
*/
|
|
573
|
+
/** Keep the first entry per file path; the caller owns the order. */
|
|
974
574
|
function deduplicateByPath(items) {
|
|
975
575
|
const seen = new Set();
|
|
976
576
|
return items.filter((item) => {
|