akm-cli 0.9.1 → 0.9.2-alpha.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +88 -0
- package/README.md +3 -1
- package/SECURITY.md +1 -1
- package/STABILITY.md +1 -1
- package/dist/akm +2 -2
- package/dist/akm-migrate +2 -2
- package/dist/assets/hints/cli-hints-full.md +14 -9
- package/dist/assets/improve-strategies/proactive-maintenance.json +1 -1
- package/dist/assets/improve-strategies/reflect-distill.json +1 -1
- package/dist/assets/models.json +35 -0
- package/dist/assets/tasks/core/extract.yml +6 -5
- package/dist/assets/tasks/core/improve.yml +6 -5
- package/dist/assets/tasks/core/index-refresh.yml +6 -5
- package/dist/assets/tasks/core/sync.yml +6 -5
- package/dist/assets/tasks/core/version-check.yml +6 -5
- package/dist/assets/tasks/improve/akm-graph-refresh-weekly.yml +6 -5
- package/dist/assets/tasks/improve/akm-improve-catchup.yml +6 -5
- package/dist/assets/tasks/improve/akm-improve-consolidate.yml +6 -5
- package/dist/assets/tasks/improve/akm-improve-frequent.yml +6 -5
- package/dist/assets/tasks/improve/akm-improve-nightly.yml +6 -5
- package/dist/cli/confirm.js +2 -2
- package/dist/cli/parse-args.js +3 -24
- package/dist/cli/retired-commands.js +1 -1
- package/dist/cli/shared.js +2 -2
- package/dist/cli.js +11 -9
- package/dist/commands/agent/agent-dispatch.js +55 -89
- package/dist/commands/agent/contribute-cli.js +12 -45
- package/dist/commands/command/builtin-action.js +32 -0
- package/dist/commands/command/command-cli.js +99 -0
- package/dist/commands/command/command-execution.js +308 -0
- package/dist/commands/command/execution-source-loader.js +176 -0
- package/dist/commands/command/portable-template.js +60 -0
- package/dist/commands/config-cli.js +10 -4
- package/dist/commands/env/env.js +4 -2
- package/dist/commands/feedback-cli.js +1 -1
- package/dist/commands/health/checks.js +241 -29
- package/dist/commands/health/html-report.js +0 -14
- package/dist/commands/health/report-view-model.js +0 -1
- package/dist/commands/health/surfaces.js +6 -7
- package/dist/commands/health/types.js +0 -2
- package/dist/commands/health.js +63 -18
- package/dist/commands/improve/collapse-detector.js +2 -2
- package/dist/commands/improve/consolidate.js +251 -214
- package/dist/commands/improve/distill/promote-memory.js +71 -34
- package/dist/commands/improve/distill/quality-gate.js +17 -5
- package/dist/commands/improve/distill.js +232 -155
- package/dist/commands/improve/eligibility.js +112 -79
- package/dist/commands/improve/execution.js +57 -0
- package/dist/commands/improve/extract-cli.js +5 -5
- package/dist/commands/improve/extract.js +521 -342
- package/dist/commands/improve/improve-strategies.js +43 -14
- package/dist/commands/improve/improve.js +249 -29
- package/dist/commands/improve/loop-stages.js +11 -17
- package/dist/commands/improve/memory/memory-contradiction-detect.js +90 -66
- package/dist/commands/improve/outcome-loop.js +22 -38
- package/dist/commands/improve/planner.js +134 -0
- package/dist/commands/improve/preparation.js +730 -409
- package/dist/commands/improve/reflect.js +386 -223
- package/dist/commands/improve/run-context.js +3 -4
- package/dist/commands/improve/salience.js +6 -58
- package/dist/commands/improve/session-asset.js +12 -12
- package/dist/commands/lint/index.js +101 -29
- package/dist/commands/migrate-cli.js +11 -69
- package/dist/commands/migration-tool.js +6 -9
- package/dist/commands/models-cli.js +27 -0
- package/dist/commands/proposal/drain.js +258 -186
- package/dist/commands/proposal/proposal-cli.js +32 -10
- package/dist/commands/proposal/proposal.js +2 -5
- package/dist/commands/proposal/propose.js +192 -172
- package/dist/commands/proposal/repository.js +54 -91
- package/dist/commands/proposal/validators/proposal-validators.js +9 -7
- package/dist/commands/read/curate.js +10 -0
- package/dist/commands/read/registry-search.js +25 -9
- package/dist/commands/read/remember-cli.js +14 -2
- package/dist/commands/read/search.js +10 -4
- package/dist/commands/read/show.js +139 -153
- package/dist/commands/registry-cli.js +16 -7
- package/dist/commands/remember.js +33 -18
- package/dist/commands/sources/add-cli.js +19 -178
- package/dist/commands/sources/bundle-cli.js +15 -3
- package/dist/commands/sources/dangerous-env-audit.js +135 -0
- package/dist/commands/sources/info.js +2 -1
- package/dist/commands/sources/installed-stashes.js +835 -152
- package/dist/commands/sources/schema-repair.js +174 -95
- package/dist/commands/sources/self-update.js +30 -74
- package/dist/commands/sources/source-add.js +3 -5
- package/dist/commands/sources/sources-cli.js +2 -15
- package/dist/commands/sources/update-transaction.js +220 -0
- package/dist/commands/tasks/tasks-cli.js +3 -3
- package/dist/commands/tasks/tasks.js +736 -317
- package/dist/commands/workflow-cli.js +2 -2
- package/dist/core/adapter/adapters/agent-skills-adapter.js +3 -0
- package/dist/core/adapter/adapters/akm-adapter.js +83 -34
- package/dist/core/adapter/adapters/akm-lint.js +54 -39
- package/dist/core/adapter/adapters/akm-metadata.js +45 -45
- package/dist/core/adapter/adapters/akm-task-adapter.js +32 -49
- package/dist/core/adapter/adapters/akm-workflow-adapter.js +38 -23
- package/dist/core/adapter/adapters/dotenv-adapter.js +30 -1
- package/dist/core/adapter/adapters/generic-files-adapter.js +11 -0
- package/dist/core/adapter/adapters/index.js +0 -9
- package/dist/core/adapter/adapters/llm-wiki-adapter.js +4 -0
- package/dist/core/adapter/adapters/okf-adapter.js +4 -0
- package/dist/core/adapter/adapters/opencode-adapter.js +5 -8
- package/dist/core/adapter/adapters/tool-dir-shared.js +63 -6
- package/dist/core/adapter/adapters/website-snapshot-adapter.js +4 -0
- package/dist/core/adapter/execution-source.js +308 -0
- package/dist/core/adapter/recognize-match.js +36 -13
- package/dist/core/adapter/registry.js +0 -9
- package/dist/core/asset/stash-meta.js +94 -4
- package/dist/core/common.js +6 -11
- package/dist/core/config/config-io.js +3 -3
- package/dist/core/config/config-schema.js +18 -40
- package/dist/core/config/config-sources.js +11 -21
- package/dist/core/config/config-walker.js +31 -13
- package/dist/core/config/config.js +21 -20
- package/dist/core/config/schema/engines.js +8 -7
- package/dist/core/config/schema/improve-processes.js +29 -5
- package/dist/core/config/schema/primitives.js +1 -23
- package/dist/core/config/schema/sources-bundles.js +13 -16
- package/dist/core/errors.js +2 -0
- package/dist/core/events.js +68 -32
- package/dist/core/extra-params.js +1 -0
- package/dist/core/improve-result.js +315 -0
- package/dist/core/lesson-lint.js +0 -6
- package/dist/core/maintenance-barrier.js +4 -4
- package/dist/core/network-policy.js +152 -0
- package/dist/core/paths.js +1 -1
- package/dist/core/recognition-util.js +4 -4
- package/dist/core/registry-url.js +456 -0
- package/dist/core/state/migrations.js +161 -47
- package/dist/core/state-db.js +453 -80
- package/dist/core/system-error.js +32 -0
- package/dist/core/time.js +2 -12
- package/dist/core/write-source.js +0 -18
- package/dist/execution/directory-identity.js +52 -0
- package/dist/execution/executable-identity.js +107 -0
- package/dist/execution/guarded-source.js +398 -0
- package/dist/execution/json.js +95 -0
- package/dist/{commands/health/types-session-log.js → execution/limits.js} +2 -1
- package/dist/execution/record.js +55 -0
- package/dist/execution/resolved-request.js +730 -0
- package/dist/execution/source.js +320 -0
- package/dist/indexer/bundle-identity-guard.js +5 -4
- package/dist/indexer/db/graph-db.js +33 -0
- package/dist/indexer/graph/graph-boost.js +3 -4
- package/dist/indexer/graph/graph-extraction.js +562 -373
- package/dist/indexer/index-written-assets.js +65 -34
- package/dist/indexer/indexer.js +443 -222
- package/dist/indexer/installations.js +6 -0
- package/dist/indexer/lookup/adapter-concept-owner.js +283 -0
- package/dist/indexer/passes/memory-inference.js +227 -174
- package/dist/indexer/scan/doc-to-entry.js +7 -9
- package/dist/indexer/scan/drain-dir.js +51 -23
- package/dist/indexer/search/db-search.js +98 -22
- package/dist/indexer/search/search-fields.js +1 -1
- package/dist/indexer/search/search-hit-enrichers.js +1 -1
- package/dist/indexer/search/search-source.js +7 -16
- package/dist/indexer/search/semantic-status.js +10 -1
- package/dist/indexer/usage/show-usage.js +105 -0
- package/dist/indexer/usage/usage-events.js +7 -2
- package/dist/indexer/walk/matchers.js +40 -10
- package/dist/indexer/walk/path-resolver.js +5 -2
- package/dist/indexer/walk/walker.js +20 -2
- package/dist/integrations/agent/builder-shared.js +3 -6
- package/dist/integrations/agent/conversation-fallback.js +16 -0
- package/dist/integrations/agent/engine-resolution.js +87 -87
- package/dist/integrations/agent/execution-cascade.js +566 -0
- package/dist/integrations/agent/execution-definitions.js +211 -0
- package/dist/integrations/agent/execution-lowering.js +811 -0
- package/dist/integrations/agent/execution-preparation.js +67 -0
- package/dist/integrations/agent/index.js +0 -2
- package/dist/integrations/agent/inline-execution.js +74 -0
- package/dist/integrations/agent/model-map.js +515 -0
- package/dist/integrations/agent/persona-fallback.js +30 -0
- package/dist/integrations/agent/request-lowering.js +186 -0
- package/dist/integrations/agent/runner-dispatch.js +230 -37
- package/dist/integrations/agent/runner.js +12 -83
- package/dist/integrations/harnesses/aider/agent-builder.js +8 -0
- package/dist/integrations/harnesses/aider/index.js +0 -1
- package/dist/integrations/harnesses/amazonq/agent-builder.js +8 -0
- package/dist/integrations/harnesses/amazonq/index.js +0 -1
- package/dist/integrations/harnesses/claude/agent-builder.js +14 -1
- package/dist/integrations/harnesses/claude/index.js +1 -5
- package/dist/integrations/harnesses/claude/session-log.js +3 -33
- package/dist/integrations/harnesses/codex/agent-builder.js +8 -0
- package/dist/integrations/harnesses/codex/index.js +0 -1
- package/dist/integrations/harnesses/copilot/agent-builder.js +8 -0
- package/dist/integrations/harnesses/copilot/index.js +0 -1
- package/dist/integrations/harnesses/gemini/agent-builder.js +8 -0
- package/dist/integrations/harnesses/gemini/index.js +0 -1
- package/dist/integrations/harnesses/index.js +4 -44
- package/dist/integrations/harnesses/opencode/agent-builder.js +16 -9
- package/dist/integrations/harnesses/opencode/index.js +0 -2
- package/dist/integrations/harnesses/opencode/session-log.js +14 -204
- package/dist/integrations/harnesses/opencode-sdk/harness.js +12 -1
- package/dist/integrations/harnesses/opencode-sdk/sdk-runner.js +40 -42
- package/dist/integrations/harnesses/openhands/agent-builder.js +8 -0
- package/dist/integrations/harnesses/openhands/index.js +0 -1
- package/dist/integrations/harnesses/pi/agent-builder.js +8 -0
- package/dist/integrations/harnesses/pi/index.js +0 -1
- package/dist/integrations/harnesses/shared.js +0 -1
- package/dist/integrations/harnesses/types.js +1 -3
- package/dist/integrations/lockfile.js +82 -79
- package/dist/integrations/session-logs/index.js +6 -17
- package/dist/integrations/session-logs/provider-base.js +1 -29
- package/dist/llm/client.js +10 -5
- package/dist/llm/embedder.js +6 -7
- package/dist/llm/embedders/local.js +37 -88
- package/dist/llm/embedders/types.js +1 -1
- package/dist/llm/graph-extract.js +75 -50
- package/dist/llm/index-passes.js +43 -5
- package/dist/llm/memory-infer.js +8 -6
- package/dist/llm/metadata-enhance.js +5 -3
- package/dist/llm/structured-call.js +122 -25
- package/dist/output/format-exempt.js +1 -1
- package/dist/output/render-registry.js +0 -16
- package/dist/output/renderers.js +12 -7
- package/dist/output/shapes/curate.js +1 -0
- package/dist/output/shapes/helpers.js +10 -2
- package/dist/output/shapes/passthrough.js +2 -0
- package/dist/output/text/command-format.js +28 -32
- package/dist/output/text/health-format.js +1 -29
- package/dist/output/text/migrate.js +6 -56
- package/dist/output/text/proposal-format.js +16 -1
- package/dist/output/text/workflow-format.js +16 -0
- package/dist/registry/network.js +279 -0
- package/dist/registry/pinned-request-helper.js +247 -0
- package/dist/registry/pinned-transport.js +717 -0
- package/dist/registry/providers/skills-sh.js +18 -6
- package/dist/registry/providers/static-index.js +20 -7
- package/dist/registry/resolve.js +53 -28
- package/dist/scripts/akm-migrate-node.js +19122 -52218
- package/dist/scripts/akm-migrate.js +19064 -51567
- package/dist/setup/registry-stash-loader.js +64 -20
- package/dist/setup/semantic-assets.js +9 -34
- package/dist/setup/setup.js +12 -30
- package/dist/setup/source-identity.js +17 -0
- package/dist/setup/steps/sources.js +36 -15
- package/dist/setup/steps/tasks.js +39 -11
- package/dist/sources/providers/git-provider.js +3 -3
- package/dist/sources/providers/npm.js +2 -2
- package/dist/sources/providers/provider-utils.js +4 -3
- package/dist/sources/providers/website.js +11 -7
- package/dist/sources/snapshot-fetchers/host-guard.js +9 -136
- package/dist/sources/snapshot-fetchers/website-ingest.js +25 -109
- package/dist/sources/website-url.js +73 -0
- package/dist/storage/engines/sqlite-migrations.js +81 -26
- package/dist/storage/managed-db.js +27 -24
- package/dist/storage/repositories/events-repository.js +3 -0
- package/dist/storage/repositories/index-connection.js +42 -10
- package/dist/storage/repositories/index-entries-repository.js +167 -207
- package/dist/storage/repositories/index-entry-mapper.js +8 -12
- package/dist/storage/repositories/index-entry-schema.js +255 -0
- package/dist/storage/repositories/index-fts-repository.js +10 -10
- package/dist/storage/repositories/index-llm-cache-repository.js +8 -13
- package/dist/storage/repositories/index-meta-repository.js +0 -11
- package/dist/storage/repositories/index-schema.js +71 -342
- package/dist/storage/repositories/index-utility-repository.js +12 -17
- package/dist/storage/repositories/index-vec-repository.js +1 -1
- package/dist/storage/repositories/proposals-repository.js +4 -127
- package/dist/storage/repositories/registry-cache.js +2 -1
- package/dist/storage/repositories/task-history-repository.js +20 -40
- package/dist/storage/repositories/workflow-runs-repository.js +228 -129
- package/dist/storage/sqlite-read-snapshot.js +148 -0
- package/dist/tasks/backends/cron.js +170 -42
- package/dist/tasks/backends/index.js +1 -1
- package/dist/tasks/backends/launchd.js +787 -202
- package/dist/tasks/backends/schtasks.js +282 -83
- package/dist/tasks/embedded.js +7 -7
- package/dist/tasks/frozen-script.js +50 -0
- package/dist/tasks/resolve-akm-bin.js +5 -1
- package/dist/tasks/runner.js +239 -251
- package/dist/tasks/runtime-v3.js +281 -0
- package/dist/tasks/scheduler-binding.js +272 -0
- package/dist/tasks/scheduler-invocation.js +57 -43
- package/dist/tasks/scheduler-sync.js +654 -0
- package/dist/tasks/source-v3.js +752 -0
- package/dist/tasks/standalone-script-entry.js +5 -0
- package/dist/tasks/task-id.js +29 -0
- package/dist/workflows/authoring/authoring.js +15 -32
- package/dist/workflows/exec/dispatch-redaction.js +14 -8
- package/dist/workflows/exec/exec-unit.js +7 -28
- package/dist/workflows/exec/frozen-judge.js +57 -89
- package/dist/workflows/exec/lowering-notices.js +23 -0
- package/dist/workflows/exec/native-executor.js +301 -458
- package/dist/workflows/exec/param-secrets.js +4 -3
- package/dist/workflows/exec/run-workflow.js +26 -32
- package/dist/workflows/exec/step-work.js +105 -109
- package/dist/workflows/exec/unit-dispatch.js +103 -27
- package/dist/workflows/exec/unit-writer.js +3 -3
- package/dist/workflows/exec/worktree.js +2 -2
- package/dist/workflows/ir/compile.js +86 -72
- package/dist/workflows/ir/environment-v4.js +328 -0
- package/dist/workflows/ir/freeze-v4.js +122 -0
- package/dist/workflows/ir/plan-hash.js +13 -7
- package/dist/workflows/ir/schema-v4.js +525 -0
- package/dist/workflows/ir/schema.js +25 -284
- package/dist/workflows/ir/source-freeze-v4.js +506 -0
- package/dist/workflows/parser.js +27 -24
- package/dist/workflows/program/schema.js +1 -2
- package/dist/workflows/renderer.js +42 -29
- package/dist/workflows/resource-limits.js +4 -5
- package/dist/workflows/runtime/agent-identity.js +11 -13
- package/dist/workflows/runtime/plan-classifier.js +8 -8
- package/dist/workflows/runtime/runs.js +27 -43
- package/dist/workflows/runtime/workflow-asset-loader.js +45 -205
- package/dist/workflows/source-files.js +373 -0
- package/dist/workflows/source-ir/compile.js +196 -0
- package/dist/workflows/source-ir/github-yaml.js +577 -0
- package/dist/workflows/source-ir/ordering.js +38 -0
- package/dist/workflows/source-ir/program.js +50 -0
- package/dist/workflows/source-ir/result.js +26 -0
- package/dist/workflows/source-ir/schema.js +772 -0
- package/dist/workflows/source-ir/semantics.js +242 -0
- package/dist/workflows/source-ir/uses.js +14 -0
- package/docs/README.md +2 -0
- package/docs/migration/README.md +3 -1
- package/docs/migration/release-notes/0.9.2.md +55 -0
- package/docs/migration/release-notes/README.md +5 -0
- package/docs/migration/v0.8-to-v0.9.md +76 -1077
- package/docs/migration/v0.9.0-troubleshooting.md +104 -516
- package/docs/migration/v0.9.1-to-v0.9.2.md +150 -0
- package/docs/reference/README.md +1 -0
- package/docs/reference/cli.md +230 -98
- package/docs/reference/configuration.md +153 -27
- package/docs/reference/data-and-telemetry.md +19 -1
- package/docs/reference/supported-formats.md +23 -3
- package/docs/reference/tasks.md +182 -0
- package/docs/reference/workflow-schema.md +91 -40
- package/docs/reference/workflows.md +33 -6
- package/package.json +10 -6
- package/schemas/akm-config.json +372 -216
- package/schemas/akm-task.json +324 -80
- package/schemas/akm-workflow.json +6 -9
- package/dist/core/migration-operation.js +0 -75
- package/dist/integrations/agent/model-aliases.js +0 -74
- package/dist/tasks/parser.js +0 -380
- package/dist/tasks/schema.js +0 -123
- package/dist/tasks/validator.js +0 -80
- package/dist/workflows/ir/freeze.js +0 -320
- package/dist/workflows/runtime/document-cache.js +0 -13
package/dist/indexer/indexer.js
CHANGED
|
@@ -7,25 +7,30 @@ import { detectAdapterId } from "../core/adapter/detect-adapter.js";
|
|
|
7
7
|
import { adapterForId } from "../core/adapter/registry.js";
|
|
8
8
|
import { isHttpUrl, toErrorMessage } from "../core/common.js";
|
|
9
9
|
import { concurrentMap } from "../core/concurrent.js";
|
|
10
|
+
import { ConfigError } from "../core/errors.js";
|
|
10
11
|
import { isLoopbackEndpoint } from "../core/loopback.js";
|
|
11
12
|
import { classifyPathAccess, describeInaccessiblePath } from "../core/path-access.js";
|
|
12
13
|
import { getDbPath } from "../core/paths.js";
|
|
13
14
|
import { SCRIPT_EXTENSIONS } from "../core/recognition-util.js";
|
|
14
15
|
import { withStateDb } from "../core/state-db.js";
|
|
15
16
|
import { isVerbose, warn, warnVerbose } from "../core/warn.js";
|
|
16
|
-
import {
|
|
17
|
+
import { disposeLoweredExecutionDispatchLease, } from "../integrations/agent/execution-lowering.js";
|
|
18
|
+
import { isLlmFeatureEnabled } from "../llm/feature-gate.js";
|
|
19
|
+
import { resolveIndexPassExecution } from "../llm/index-passes.js";
|
|
20
|
+
import { preflightStructuredLlmRunner } from "../llm/structured-call.js";
|
|
17
21
|
import { resolveSourcesForOrigin } from "../registry/origin-resolve.js";
|
|
18
|
-
import { closeDatabase, openExistingDatabase, openIndexDatabase } from "../storage/repositories/index-connection.js";
|
|
19
|
-
import {
|
|
22
|
+
import { closeDatabase, openExistingDatabase, openIndexDatabase, openReadonlyExistingDatabase, } from "../storage/repositories/index-connection.js";
|
|
23
|
+
import { deleteEntriesByBundle, deleteEntriesByDirAndBundle, deleteEntriesByDirExceptRefs, deleteEntriesByIds, deleteUsageEventsByEntryIds, findEntryIdByRef, getAllEntries, getEmbeddableEntryCount, getEntryCount, getIndexedBundleIdsByDir, getIndexedDirPathsByBundleId, relinkUsageEvents, upsertEntry, } from "../storage/repositories/index-entries-repository.js";
|
|
20
24
|
import { rebuildFts } from "../storage/repositories/index-fts-repository.js";
|
|
21
|
-
import { clearStaleCacheEntries } from "../storage/repositories/index-llm-cache-repository.js";
|
|
22
|
-
import { deleteIndexDirState,
|
|
25
|
+
import { clearStaleCacheEntries, computeBodyHash, getLlmCacheEntry, } from "../storage/repositories/index-llm-cache-repository.js";
|
|
26
|
+
import { deleteIndexDirState, deleteMeta, getMeta, setMeta, upsertIndexDirState, } from "../storage/repositories/index-meta-repository.js";
|
|
23
27
|
import { upsertUtilityScore } from "../storage/repositories/index-utility-repository.js";
|
|
24
28
|
import { getAllEntriesForEmbedding, getEmbeddingCount, isVecAvailable, isVecFastPathReady, purgeEmbeddings, setVecFastPathReady, upsertEmbedding, warnIfVecMissing, } from "../storage/repositories/index-vec-repository.js";
|
|
25
|
-
import {
|
|
29
|
+
import { assertIndexedWorkflowSourceIdentity, WorkflowSourceIdentityError } from "../workflows/source-files.js";
|
|
26
30
|
import { deleteStoredGraph } from "./db/graph-db.js";
|
|
27
31
|
import { withIndexWriterLease } from "./index-writer-lock.js";
|
|
28
32
|
import { deriveEntryProvenance, deriveInstallations } from "./installations.js";
|
|
33
|
+
import { indexedPathMatchesOwner, resolveAdapterConceptOwner, } from "./lookup/adapter-concept-owner.js";
|
|
29
34
|
import { canUseIncrementalSkip, computeDirFingerprint, getCachedZeroRowDirState, getDirIndexState, inferZeroRowReason, } from "./passes/dir-staleness.js";
|
|
30
35
|
import { isEnrichmentComplete, isWorkflowSkipWarning } from "./passes/metadata.js";
|
|
31
36
|
import { drainDirDocuments } from "./scan/drain-dir.js";
|
|
@@ -33,6 +38,16 @@ import { buildSearchText } from "./search/search-fields.js";
|
|
|
33
38
|
import { classifySemanticFailure, clearSemanticStatus, deriveSemanticProviderFingerprint, writeSemanticStatus, } from "./search/semantic-status.js";
|
|
34
39
|
import { purgeOldUsageEvents } from "./usage/usage-events.js";
|
|
35
40
|
import { walkStashFlatWithStatus } from "./walk/walker.js";
|
|
41
|
+
function collectLoweringNotices(target, notices) {
|
|
42
|
+
const keys = new Set(target.map((notice) => JSON.stringify(notice)));
|
|
43
|
+
for (const notice of notices) {
|
|
44
|
+
const key = JSON.stringify(notice);
|
|
45
|
+
if (keys.has(key))
|
|
46
|
+
continue;
|
|
47
|
+
keys.add(key);
|
|
48
|
+
target.push(notice);
|
|
49
|
+
}
|
|
50
|
+
}
|
|
36
51
|
function throwIfAborted(signal) {
|
|
37
52
|
if (signal?.aborted) {
|
|
38
53
|
throw signal.reason instanceof Error ? signal.reason : new Error("index interrupted");
|
|
@@ -56,51 +71,67 @@ export function getDefaultLlmConcurrency(llmConfig) {
|
|
|
56
71
|
// auto-derived 1/2 is what runs (see docs/architecture/internals/indexing.md).
|
|
57
72
|
return 2;
|
|
58
73
|
}
|
|
59
|
-
|
|
74
|
+
function sourceOwners(sources) {
|
|
75
|
+
const installations = deriveInstallations([...sources]);
|
|
76
|
+
return sources.flatMap((source, index) => {
|
|
77
|
+
const installation = installations[index];
|
|
78
|
+
return installation ? [{ bundleId: installation.id, sourceRoot: path.resolve(source.path) }] : [];
|
|
79
|
+
});
|
|
80
|
+
}
|
|
81
|
+
function parseStoredSourceOwners(raw) {
|
|
82
|
+
if (!raw)
|
|
83
|
+
return [];
|
|
84
|
+
try {
|
|
85
|
+
const parsed = JSON.parse(raw);
|
|
86
|
+
if (!Array.isArray(parsed) ||
|
|
87
|
+
parsed.some((owner) => typeof owner !== "object" ||
|
|
88
|
+
owner === null ||
|
|
89
|
+
typeof owner.bundleId !== "string" ||
|
|
90
|
+
typeof owner.sourceRoot !== "string")) {
|
|
91
|
+
warn("index_meta sourceOwners value is invalid — treating as empty");
|
|
92
|
+
return [];
|
|
93
|
+
}
|
|
94
|
+
return parsed.map((owner) => {
|
|
95
|
+
const stored = owner;
|
|
96
|
+
return { bundleId: stored.bundleId, sourceRoot: path.resolve(stored.sourceRoot) };
|
|
97
|
+
});
|
|
98
|
+
}
|
|
99
|
+
catch {
|
|
100
|
+
warn("index_meta sourceOwners value is corrupt JSON — treating as empty");
|
|
101
|
+
return [];
|
|
102
|
+
}
|
|
103
|
+
}
|
|
60
104
|
/**
|
|
61
105
|
* Source cache phase: ensure git stash caches are up to date and purge orphaned
|
|
62
106
|
* entries from removed sources (incremental only).
|
|
63
107
|
*/
|
|
64
108
|
async function runSourceCachePhase(ctx) {
|
|
65
|
-
const { db,
|
|
109
|
+
const { db, isIncremental, full, sources } = ctx;
|
|
66
110
|
if (isIncremental && !full) {
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
}
|
|
77
|
-
else {
|
|
78
|
-
warn("index_meta stashDirs value is not an array — treating as empty");
|
|
79
|
-
}
|
|
80
|
-
}
|
|
81
|
-
catch {
|
|
82
|
-
warn("index_meta stashDirs value is corrupt JSON — treating as empty");
|
|
83
|
-
}
|
|
84
|
-
const currentSet = new Set(sourceDirs);
|
|
85
|
-
for (const dir of prevStashDirs) {
|
|
86
|
-
if (!currentSet.has(dir)) {
|
|
87
|
-
ctx.hadRemovedSources = true;
|
|
88
|
-
ctx.removedSourceDirs.push(dir);
|
|
89
|
-
}
|
|
111
|
+
const currentByBundle = new Map(sourceOwners(sources).map((owner) => [owner.bundleId, owner]));
|
|
112
|
+
for (const previous of parseStoredSourceOwners(getMeta(db, "sourceOwners"))) {
|
|
113
|
+
const current = currentByBundle.get(previous.bundleId);
|
|
114
|
+
if (!current || current.sourceRoot !== previous.sourceRoot) {
|
|
115
|
+
ctx.hadRemovedSources = true;
|
|
116
|
+
ctx.removedSources.push({
|
|
117
|
+
...previous,
|
|
118
|
+
removeBundleEntries: current === undefined,
|
|
119
|
+
});
|
|
90
120
|
}
|
|
91
121
|
}
|
|
92
122
|
}
|
|
93
123
|
// Source caches are hydrated before akmIndex() calls this phase; nothing
|
|
94
124
|
// further to do here. The flag is exposed on ctx for runWalkPhase().
|
|
95
|
-
void config;
|
|
96
125
|
}
|
|
97
126
|
function applyRemovedSources(ctx) {
|
|
98
127
|
if (!ctx.scanComplete)
|
|
99
128
|
return;
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
129
|
+
const currentRoots = new Set(sourceOwners(ctx.sources).map((owner) => owner.sourceRoot));
|
|
130
|
+
for (const removed of ctx.removedSources) {
|
|
131
|
+
if (removed.removeBundleEntries)
|
|
132
|
+
deleteEntriesByBundle(ctx.db, removed.bundleId);
|
|
133
|
+
if (!currentRoots.has(removed.sourceRoot))
|
|
134
|
+
deleteStoredGraph(ctx.db, removed.sourceRoot);
|
|
104
135
|
}
|
|
105
136
|
}
|
|
106
137
|
/**
|
|
@@ -115,7 +146,14 @@ async function runWalkPhase(ctx) {
|
|
|
115
146
|
throwIfAborted(signal);
|
|
116
147
|
ctx.timing.tWalkStart = Date.now();
|
|
117
148
|
const doFullDelete = full || !isIncremental;
|
|
118
|
-
const { scannedDirs, skippedDirs, generatedCount, dirsNeedingLlm, warnings, complete } = await indexEntries(db, sources, isIncremental, builtAtMs, hadRemovedSources, doFullDelete, onProgress, !clean)
|
|
149
|
+
const { scannedDirs, skippedDirs, generatedCount, dirsNeedingLlm, warnings, complete } = await indexEntries(db, sources, isIncremental, builtAtMs, hadRemovedSources, doFullDelete, onProgress, !clean, async (dirRecords, ownersByRoot) => {
|
|
150
|
+
const runner = ctx.enrichmentExecution.runner;
|
|
151
|
+
if (runner &&
|
|
152
|
+
isLlmFeatureEnabled(config, "metadata_enhance") &&
|
|
153
|
+
dirRecordsNeedMetadataDispatch(db, dirRecords, ownersByRoot)) {
|
|
154
|
+
ctx.enrichmentLease = await preflightStructuredLlmRunner(runner);
|
|
155
|
+
}
|
|
156
|
+
});
|
|
119
157
|
ctx.scannedDirs = scannedDirs;
|
|
120
158
|
ctx.skippedDirs = skippedDirs;
|
|
121
159
|
ctx.generatedCount = generatedCount;
|
|
@@ -142,10 +180,10 @@ async function runWalkPhase(ctx) {
|
|
|
142
180
|
ctx.timing.tWalkEnd = Date.now();
|
|
143
181
|
throwIfAborted(signal);
|
|
144
182
|
// LLM enrichment for directories that need it
|
|
145
|
-
await enhanceDirsWithLlm(db, config, dirsNeedingLlm, onProgress, signal);
|
|
183
|
+
await enhanceDirsWithLlm(db, config, ctx.enrichmentExecution, dirsNeedingLlm, onProgress, signal, (notices) => collectLoweringNotices(ctx.loweringNotices, notices), ctx.enrichmentLease);
|
|
146
184
|
onProgress({
|
|
147
185
|
phase: "llm",
|
|
148
|
-
message:
|
|
186
|
+
message: ctx.enrichmentExecution.runner
|
|
149
187
|
? `LLM enhancement reviewed ${dirsNeedingLlm.length} ${dirsNeedingLlm.length === 1 ? "directory" : "directories"}.`
|
|
150
188
|
: "LLM enhancement disabled.",
|
|
151
189
|
});
|
|
@@ -170,7 +208,7 @@ async function runEmbeddingPhase(ctx) {
|
|
|
170
208
|
* Finalize phase: rebuild FTS, re-link usage events, recompute utility scores,
|
|
171
209
|
* regenerate wiki indexes, update index metadata, and emit the verify event.
|
|
172
210
|
*/
|
|
173
|
-
async function runFinalizePhase(ctx) {
|
|
211
|
+
async function runFinalizePhase(ctx, deferredUpdateTransaction) {
|
|
174
212
|
const { db, config, sources, sourceDirs, isIncremental, stashDir, signal, onProgress } = ctx;
|
|
175
213
|
ctx.timing.tFinalizeStart = Date.now();
|
|
176
214
|
// Rebuild FTS after all inserts. Use incremental mode when this whole
|
|
@@ -182,20 +220,27 @@ async function runFinalizePhase(ctx) {
|
|
|
182
220
|
message: isIncremental ? "Rebuilt full-text search index (dirty rows only)." : "Rebuilt full-text search index.",
|
|
183
221
|
});
|
|
184
222
|
ctx.timing.tFtsEnd = Date.now();
|
|
185
|
-
// Re-link
|
|
186
|
-
//
|
|
187
|
-
//
|
|
188
|
-
|
|
189
|
-
//
|
|
190
|
-
// Chunk-8 WI-8.3: usage_events lives in state.db now (index.db no longer holds
|
|
191
|
-
// it), so these cross-DB passes take both handles — entries in `db` (index.db),
|
|
192
|
-
// usage_events in the loaned state.db.
|
|
193
|
-
withStateDb((stateDb) => {
|
|
223
|
+
// Re-link state.db usage events to the regenerated index and recompute the
|
|
224
|
+
// derived utility cache. Stored refs already use the current item-ref grammar,
|
|
225
|
+
// so this idempotent pass only restores derived entry ids.
|
|
226
|
+
const mutateState = (stateDb, stateSchema) => {
|
|
194
227
|
onProgress({ phase: "finalize", message: "Relinking usage events." });
|
|
195
|
-
relinkUsageEvents(db, stateDb, { sources, defaultStashDir: stashDir });
|
|
228
|
+
relinkUsageEvents(db, stateDb, { sources, defaultStashDir: stashDir, stateSchema });
|
|
196
229
|
onProgress({ phase: "finalize", message: "Recomputing utility scores." });
|
|
197
|
-
recomputeUtilityScores(db, stateDb);
|
|
198
|
-
}
|
|
230
|
+
recomputeUtilityScores(db, stateDb, { stateSchema });
|
|
231
|
+
};
|
|
232
|
+
if (deferredUpdateTransaction) {
|
|
233
|
+
if (deferredUpdateTransaction.db !== db || !db.inTransaction) {
|
|
234
|
+
throw new Error("Source update index finalization requires its borrowed unified transaction.");
|
|
235
|
+
}
|
|
236
|
+
// state.db is ATTACHed to this same index connection before the outer
|
|
237
|
+
// BEGIN IMMEDIATE. Index and state mutations therefore share one SQLite
|
|
238
|
+
// commit/rollback decision rather than an unsafe two-connection ordering.
|
|
239
|
+
mutateState(db, deferredUpdateTransaction.stateSchema);
|
|
240
|
+
}
|
|
241
|
+
else {
|
|
242
|
+
withStateDb(mutateState);
|
|
243
|
+
}
|
|
199
244
|
// Purge LLM cache entries for assets that no longer exist in the index.
|
|
200
245
|
try {
|
|
201
246
|
onProgress({ phase: "finalize", message: "Clearing stale LLM cache entries." });
|
|
@@ -213,6 +258,7 @@ async function runFinalizePhase(ctx) {
|
|
|
213
258
|
setMeta(db, "builtAt", new Date().toISOString());
|
|
214
259
|
setMeta(db, "stashDir", stashDir);
|
|
215
260
|
setMeta(db, "stashDirs", JSON.stringify(sourceDirs));
|
|
261
|
+
setMeta(db, "sourceOwners", JSON.stringify(sourceOwners(sources)));
|
|
216
262
|
}
|
|
217
263
|
setMeta(db, "hasEmbeddings", embeddingResult.success ? "1" : "0");
|
|
218
264
|
// Stash-organization conventions (SPEC-8): track which `index.indexBodyOpening`
|
|
@@ -225,10 +271,11 @@ async function runFinalizePhase(ctx) {
|
|
|
225
271
|
const semanticEntryCount = getEmbeddableEntryCount(db);
|
|
226
272
|
onProgress({ phase: "finalize", message: "Verifying semantic search state." });
|
|
227
273
|
const verification = verifyIndexState(db, config, semanticEntryCount, embeddingResult);
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
274
|
+
const persistSemanticStatus = () => {
|
|
275
|
+
if (config.semanticSearchMode === "off") {
|
|
276
|
+
clearSemanticStatus();
|
|
277
|
+
return;
|
|
278
|
+
}
|
|
232
279
|
writeSemanticStatus({
|
|
233
280
|
status: verification.semanticStatus === "disabled" ? "pending" : verification.semanticStatus,
|
|
234
281
|
...(embeddingResult.reason ? { reason: embeddingResult.reason } : {}),
|
|
@@ -238,6 +285,12 @@ async function runFinalizePhase(ctx) {
|
|
|
238
285
|
entryCount: verification.entryCount,
|
|
239
286
|
embeddingCount: verification.embeddingCount,
|
|
240
287
|
});
|
|
288
|
+
};
|
|
289
|
+
if (deferredUpdateTransaction) {
|
|
290
|
+
deferredUpdateTransaction.afterCommit = persistSemanticStatus;
|
|
291
|
+
}
|
|
292
|
+
else {
|
|
293
|
+
persistSemanticStatus();
|
|
241
294
|
}
|
|
242
295
|
onProgress({ phase: "verify", message: verification.message });
|
|
243
296
|
// Store verification result and totalEntries on ctx for the caller to use
|
|
@@ -299,7 +352,7 @@ export function reconcileBodyOpeningIndexState(db, flagEnabled, isFullWalk) {
|
|
|
299
352
|
* reported instead.
|
|
300
353
|
*/
|
|
301
354
|
function runCleanPass(db, dryRun) {
|
|
302
|
-
const allEntries = db.prepare("SELECT id,
|
|
355
|
+
const allEntries = db.prepare("SELECT id, item_ref AS ref, file_path AS path FROM entries").all();
|
|
303
356
|
// Only check entries that have a non-empty local path (skip remote/virtual).
|
|
304
357
|
const localEntries = allEntries.filter((e) => typeof e.path === "string" && e.path.trim() !== "");
|
|
305
358
|
const missing = [];
|
|
@@ -335,9 +388,23 @@ export function _setAkmIndexForTests(fake) {
|
|
|
335
388
|
akmIndexOverride = fake;
|
|
336
389
|
}
|
|
337
390
|
export async function akmIndex(options) {
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
391
|
+
try {
|
|
392
|
+
const override = akmIndexOverride;
|
|
393
|
+
return override ? await override(options) : await akmIndexReal(options);
|
|
394
|
+
}
|
|
395
|
+
catch (error) {
|
|
396
|
+
const updateDb = options.deferredUpdateTransaction?.db;
|
|
397
|
+
if (updateDb?.inTransaction) {
|
|
398
|
+
try {
|
|
399
|
+
updateDb.exec("ROLLBACK");
|
|
400
|
+
}
|
|
401
|
+
catch {
|
|
402
|
+
// Preserve the indexing error. The update coordinator will retry
|
|
403
|
+
// rollback before closing its borrowed unified handle.
|
|
404
|
+
}
|
|
405
|
+
}
|
|
406
|
+
throw error;
|
|
407
|
+
}
|
|
341
408
|
}
|
|
342
409
|
let indexTransactionHookForTests;
|
|
343
410
|
/**
|
|
@@ -385,7 +452,7 @@ function detectAndPersistBundleAdapters(allSourceEntries, config, mutateConfig,
|
|
|
385
452
|
}
|
|
386
453
|
}
|
|
387
454
|
const persistedAdapters = {};
|
|
388
|
-
if (detectedByBundle.size === 0)
|
|
455
|
+
if (detectedByBundle.size === 0 || !opts.persist)
|
|
389
456
|
return { config, persistedAdapters };
|
|
390
457
|
const nextConfig = mutateConfig((current) => {
|
|
391
458
|
if (!current.bundles)
|
|
@@ -418,6 +485,38 @@ function detectAndPersistBundleAdapters(allSourceEntries, config, mutateConfig,
|
|
|
418
485
|
}
|
|
419
486
|
return { config: nextConfig, persistedAdapters };
|
|
420
487
|
}
|
|
488
|
+
function createIndexRunContext(options) {
|
|
489
|
+
const prevStashDir = getMeta(options.db, "stashDir");
|
|
490
|
+
const prevBuiltAt = getMeta(options.db, "builtAt");
|
|
491
|
+
const isIncremental = !options.full && prevStashDir === options.stashDir && !!prevBuiltAt;
|
|
492
|
+
const builtAtMs = isIncremental && prevBuiltAt ? new Date(prevBuiltAt).getTime() : 0;
|
|
493
|
+
const { t0, ...context } = options;
|
|
494
|
+
return {
|
|
495
|
+
...context,
|
|
496
|
+
loweringNotices: [...options.enrichmentExecution.notices],
|
|
497
|
+
timing: {
|
|
498
|
+
t0,
|
|
499
|
+
tWalkStart: t0,
|
|
500
|
+
tWalkEnd: t0,
|
|
501
|
+
tLlmEnd: t0,
|
|
502
|
+
tFtsEnd: t0,
|
|
503
|
+
tEmbedEnd: t0,
|
|
504
|
+
tFinalizeStart: t0,
|
|
505
|
+
tFinalizeEnd: t0,
|
|
506
|
+
},
|
|
507
|
+
isIncremental,
|
|
508
|
+
builtAtMs,
|
|
509
|
+
hadRemovedSources: false,
|
|
510
|
+
removedSources: [],
|
|
511
|
+
scanComplete: true,
|
|
512
|
+
scannedDirs: 0,
|
|
513
|
+
skippedDirs: 0,
|
|
514
|
+
generatedCount: 0,
|
|
515
|
+
walkWarnings: [],
|
|
516
|
+
dirsNeedingLlm: [],
|
|
517
|
+
embeddingResult: null,
|
|
518
|
+
};
|
|
519
|
+
}
|
|
421
520
|
async function akmIndexReal(options) {
|
|
422
521
|
// R-022: `dryRun` only ever gated the `--clean` stale-entry removal pass
|
|
423
522
|
// (see `runCleanPass` below) — every other phase (walk, LLM enrichment,
|
|
@@ -460,7 +559,8 @@ async function akmIndexReal(options) {
|
|
|
460
559
|
// Durable state must be runtime-compatible before source hydration,
|
|
461
560
|
// adapter persistence, or index.db creation can mutate the installation.
|
|
462
561
|
onProgress({ phase: "preflight", message: "Validating durable state." });
|
|
463
|
-
|
|
562
|
+
if (!options.deferredUpdateTransaction)
|
|
563
|
+
withStateDb(() => undefined);
|
|
464
564
|
// Ensure git stash caches are extracted before resolving stash dirs,
|
|
465
565
|
// so their content directories exist on disk for the walker to discover.
|
|
466
566
|
const sourceCacheStart = Date.now();
|
|
@@ -482,6 +582,7 @@ async function akmIndexReal(options) {
|
|
|
482
582
|
const allSourceEntries = resolveSourceEntries(stashDir, config);
|
|
483
583
|
const detected = detectAndPersistBundleAdapters(allSourceEntries, config, mutateConfig, {
|
|
484
584
|
announce: options.implicit !== true,
|
|
585
|
+
persist: options.persistDetectedAdapters !== false,
|
|
485
586
|
});
|
|
486
587
|
config = detected.config;
|
|
487
588
|
const persistedAdapters = detected.persistedAdapters;
|
|
@@ -491,20 +592,22 @@ async function akmIndexReal(options) {
|
|
|
491
592
|
message: `Resolved ${allSourceDirs.length} stash source${allSourceDirs.length === 1 ? "" : "s"}.`,
|
|
492
593
|
});
|
|
493
594
|
const t0 = Date.now();
|
|
595
|
+
const enrichmentExecution = resolveIndexPassExecution("enrichment", config);
|
|
494
596
|
// Open database — pass embedding dimension from config if available
|
|
495
597
|
const dbPath = getDbPath();
|
|
496
598
|
const embeddingDim = config.embedding?.dimension;
|
|
497
|
-
const
|
|
599
|
+
const borrowedUpdateDb = options.deferredUpdateTransaction?.db;
|
|
600
|
+
const db = borrowedUpdateDb ?? openIndexDatabase(dbPath, embeddingDim ? { embeddingDim } : undefined);
|
|
601
|
+
if (borrowedUpdateDb && !borrowedUpdateDb.inTransaction) {
|
|
602
|
+
throw new Error("Source update index requires an active borrowed index transaction.");
|
|
603
|
+
}
|
|
604
|
+
let indexRunContext;
|
|
498
605
|
try {
|
|
499
|
-
// Determine incremental vs full mode
|
|
500
|
-
const prevStashDir = getMeta(db, "stashDir");
|
|
501
|
-
const prevBuiltAt = getMeta(db, "builtAt");
|
|
502
|
-
const isIncremental = !full && prevStashDir === stashDir && !!prevBuiltAt;
|
|
503
|
-
const builtAtMs = isIncremental && prevBuiltAt ? new Date(prevBuiltAt).getTime() : 0;
|
|
504
606
|
// Assemble the run context
|
|
505
|
-
const ctx = {
|
|
607
|
+
const ctx = createIndexRunContext({
|
|
506
608
|
db,
|
|
507
609
|
config,
|
|
610
|
+
enrichmentExecution,
|
|
508
611
|
sources: allSourceEntries,
|
|
509
612
|
sourceDirs: allSourceDirs,
|
|
510
613
|
full,
|
|
@@ -512,36 +615,17 @@ async function akmIndexReal(options) {
|
|
|
512
615
|
stashDir,
|
|
513
616
|
onProgress,
|
|
514
617
|
signal,
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
tWalkEnd: t0,
|
|
519
|
-
tLlmEnd: t0,
|
|
520
|
-
tFtsEnd: t0,
|
|
521
|
-
tEmbedEnd: t0,
|
|
522
|
-
tFinalizeStart: t0,
|
|
523
|
-
tFinalizeEnd: t0,
|
|
524
|
-
},
|
|
525
|
-
isIncremental,
|
|
526
|
-
builtAtMs,
|
|
527
|
-
hadRemovedSources: false,
|
|
528
|
-
removedSourceDirs: [],
|
|
529
|
-
scanComplete: true,
|
|
530
|
-
scannedDirs: 0,
|
|
531
|
-
skippedDirs: 0,
|
|
532
|
-
generatedCount: 0,
|
|
533
|
-
walkWarnings: [],
|
|
534
|
-
dirsNeedingLlm: [],
|
|
535
|
-
embeddingResult: null,
|
|
536
|
-
};
|
|
618
|
+
t0,
|
|
619
|
+
});
|
|
620
|
+
indexRunContext = ctx;
|
|
537
621
|
onProgress({
|
|
538
622
|
phase: "summary",
|
|
539
623
|
message: buildIndexSummaryMessage({
|
|
540
|
-
mode: isIncremental ? "incremental" : "full",
|
|
624
|
+
mode: ctx.isIncremental ? "incremental" : "full",
|
|
541
625
|
sourcesCount: allSourceDirs.length,
|
|
542
626
|
semanticSearchMode: config.semanticSearchMode,
|
|
543
627
|
embeddingProvider: getEmbeddingProvider(config.embedding),
|
|
544
|
-
llmEnabled: !!
|
|
628
|
+
llmEnabled: !!enrichmentExecution.runner,
|
|
545
629
|
vecAvailable: isVecAvailable(db),
|
|
546
630
|
}),
|
|
547
631
|
});
|
|
@@ -550,7 +634,7 @@ async function akmIndexReal(options) {
|
|
|
550
634
|
await runWalkPhase(ctx);
|
|
551
635
|
applyRemovedSources(ctx);
|
|
552
636
|
await runEmbeddingPhase(ctx);
|
|
553
|
-
await runFinalizePhase(ctx);
|
|
637
|
+
await runFinalizePhase(ctx, options.deferredUpdateTransaction);
|
|
554
638
|
// ────────────────────────────────────────────────────────────────────────
|
|
555
639
|
// runFinalizePhase always populates these before returning.
|
|
556
640
|
const verification = ctx.verification;
|
|
@@ -581,10 +665,11 @@ async function akmIndexReal(options) {
|
|
|
581
665
|
totalEntries,
|
|
582
666
|
generatedMetadata: ctx.generatedCount,
|
|
583
667
|
indexPath: dbPath,
|
|
584
|
-
mode: isIncremental ? "incremental" : "full",
|
|
668
|
+
mode: ctx.isIncremental ? "incremental" : "full",
|
|
585
669
|
directoriesScanned: ctx.scannedDirs,
|
|
586
670
|
directoriesSkipped: ctx.skippedDirs,
|
|
587
671
|
...(ctx.walkWarnings.length > 0 ? { warnings: ctx.walkWarnings } : {}),
|
|
672
|
+
...(ctx.loweringNotices.length > 0 ? { notices: Object.freeze([...ctx.loweringNotices]) } : {}),
|
|
588
673
|
...(Object.keys(persistedAdapters).length > 0
|
|
589
674
|
? { configUpdated: { detectedAdapters: persistedAdapters } }
|
|
590
675
|
: {}),
|
|
@@ -606,10 +691,69 @@ async function akmIndexReal(options) {
|
|
|
606
691
|
};
|
|
607
692
|
}
|
|
608
693
|
finally {
|
|
609
|
-
|
|
694
|
+
if (indexRunContext?.enrichmentLease) {
|
|
695
|
+
disposeLoweredExecutionDispatchLease(indexRunContext.enrichmentLease);
|
|
696
|
+
}
|
|
697
|
+
if (!borrowedUpdateDb)
|
|
698
|
+
closeDatabase(db);
|
|
610
699
|
}
|
|
611
700
|
});
|
|
612
701
|
}
|
|
702
|
+
function buildIndexedSourceOwners(sources) {
|
|
703
|
+
const installations = deriveInstallations([...sources]);
|
|
704
|
+
const owners = new Map();
|
|
705
|
+
sources.forEach((source, index) => {
|
|
706
|
+
const installation = installations[index];
|
|
707
|
+
if (!installation)
|
|
708
|
+
return;
|
|
709
|
+
const component = installation.components[0];
|
|
710
|
+
owners.set(path.resolve(source.path), {
|
|
711
|
+
bundleId: installation.id,
|
|
712
|
+
componentId: component?.id ?? installation.id,
|
|
713
|
+
adapterId: component?.adapter ?? "akm",
|
|
714
|
+
});
|
|
715
|
+
});
|
|
716
|
+
return owners;
|
|
717
|
+
}
|
|
718
|
+
/** Read-only mirror of the enrichment cache gate used before entry persistence. */
|
|
719
|
+
function dirRecordsNeedMetadataDispatch(db, records, ownersByRoot) {
|
|
720
|
+
for (const record of records) {
|
|
721
|
+
if (record.skip || record.remove || !record.stash)
|
|
722
|
+
continue;
|
|
723
|
+
const owner = ownersByRoot.get(path.resolve(record.currentStashDir));
|
|
724
|
+
if (!owner)
|
|
725
|
+
throw new Error(`Missing bundle provenance for indexed source ${record.currentStashDir}`);
|
|
726
|
+
for (const entry of record.stash.entries) {
|
|
727
|
+
if (entry.quality !== "generated" || isEnrichmentComplete(entry))
|
|
728
|
+
continue;
|
|
729
|
+
const entryFile = entry.filename ? path.join(record.dirPath, entry.filename) : undefined;
|
|
730
|
+
if (!entryFile)
|
|
731
|
+
continue;
|
|
732
|
+
const adapterConceptId = record.conceptIdByFile?.get(entryFile);
|
|
733
|
+
if (!adapterConceptId)
|
|
734
|
+
continue;
|
|
735
|
+
let fileContent;
|
|
736
|
+
try {
|
|
737
|
+
fileContent = fs.readFileSync(entryFile, "utf8");
|
|
738
|
+
}
|
|
739
|
+
catch {
|
|
740
|
+
// The dispatch path uses the same deterministic metadata fallback.
|
|
741
|
+
}
|
|
742
|
+
const bodyHash = computeBodyHash(fileContent ?? `${entry.name}\n${entry.description ?? ""}`);
|
|
743
|
+
const cacheKey = deriveEntryProvenance(owner, entry.type, entry.name, adapterConceptId).itemRef;
|
|
744
|
+
const cached = getLlmCacheEntry(db, cacheKey, bodyHash);
|
|
745
|
+
if (!cached)
|
|
746
|
+
return true;
|
|
747
|
+
try {
|
|
748
|
+
JSON.parse(cached.resultJson);
|
|
749
|
+
}
|
|
750
|
+
catch {
|
|
751
|
+
return true;
|
|
752
|
+
}
|
|
753
|
+
}
|
|
754
|
+
}
|
|
755
|
+
return false;
|
|
756
|
+
}
|
|
613
757
|
function removalsFirst(records) {
|
|
614
758
|
return [...records.filter((record) => record.remove), ...records.filter((record) => !record.remove)];
|
|
615
759
|
}
|
|
@@ -654,8 +798,8 @@ function groupFileContextsByDir(fileContexts) {
|
|
|
654
798
|
}
|
|
655
799
|
return groups;
|
|
656
800
|
}
|
|
657
|
-
function sourceSnapshotRemovals(db, currentStashDir, currentDirs,
|
|
658
|
-
const indexedDirs =
|
|
801
|
+
function sourceSnapshotRemovals(db, currentStashDir, bundleId, currentDirs, allIndexedDirsByBundle) {
|
|
802
|
+
const indexedDirs = allIndexedDirsByBundle?.get(bundleId) ?? getIndexedDirPathsByBundleId(db, bundleId);
|
|
659
803
|
return [...indexedDirs]
|
|
660
804
|
.map((dirPath) => path.resolve(dirPath))
|
|
661
805
|
.filter((dirPath) => !currentDirs.has(dirPath))
|
|
@@ -688,6 +832,9 @@ function buildSourceScanPlans(db, allSourceEntries, isIncremental, reconcileMiss
|
|
|
688
832
|
}
|
|
689
833
|
const walked = walkStashFlatWithStatus(currentStashDir, {
|
|
690
834
|
includeAllDirectories: component.adapter === "okf",
|
|
835
|
+
...(component.adapter === "akm" || component.adapter === "akm-workflow"
|
|
836
|
+
? { workflowSymlinkAdapter: component.adapter }
|
|
837
|
+
: {}),
|
|
691
838
|
});
|
|
692
839
|
const dirGroups = groupFileContextsByDir(walked.files);
|
|
693
840
|
const adapter = adapterForId(component.adapter);
|
|
@@ -723,20 +870,19 @@ function buildSourceScanPlans(db, allSourceEntries, isIncremental, reconcileMiss
|
|
|
723
870
|
// A full, globally-complete run uses the atomic table wipe below. Every
|
|
724
871
|
// other run reconciles only sources that produced trustworthy snapshots.
|
|
725
872
|
if (reconcileMissingDirs && (isIncremental || !allComplete)) {
|
|
726
|
-
const
|
|
727
|
-
if (
|
|
873
|
+
const allIndexedDirsByBundle = !isIncremental ? new Map() : undefined;
|
|
874
|
+
if (allIndexedDirsByBundle) {
|
|
728
875
|
for (const entry of getAllEntries(db)) {
|
|
729
|
-
const
|
|
730
|
-
|
|
731
|
-
|
|
732
|
-
allIndexedDirsBySource.set(sourceRoot, dirs);
|
|
876
|
+
const dirs = allIndexedDirsByBundle.get(entry.bundleId) ?? new Set();
|
|
877
|
+
dirs.add(path.dirname(path.resolve(entry.filePath)));
|
|
878
|
+
allIndexedDirsByBundle.set(entry.bundleId, dirs);
|
|
733
879
|
}
|
|
734
880
|
}
|
|
735
881
|
for (const plan of plans) {
|
|
736
882
|
if (!plan.walkComplete || !plan.adapter)
|
|
737
883
|
continue;
|
|
738
884
|
const currentDirs = new Set([...plan.dirGroups.keys()].map((dirPath) => path.resolve(dirPath)));
|
|
739
|
-
for (const removal of sourceSnapshotRemovals(db, plan.currentStashDir, currentDirs,
|
|
885
|
+
for (const removal of sourceSnapshotRemovals(db, plan.currentStashDir, plan.component.id, currentDirs, allIndexedDirsByBundle)) {
|
|
740
886
|
addRemoval(plan, removal.dirPath, removal.currentStashDir);
|
|
741
887
|
}
|
|
742
888
|
}
|
|
@@ -748,15 +894,18 @@ function buildSourceScanPlans(db, allSourceEntries, isIncremental, reconcileMiss
|
|
|
748
894
|
// The first configured source that exposes a physical directory owns it.
|
|
749
895
|
// Remove rows left by a prior owner even when both adapters are identical.
|
|
750
896
|
const claimedDirs = new Set();
|
|
897
|
+
const sourcePathByBundle = new Map(plans.map((plan) => [plan.component.id, plan.currentStashDir]));
|
|
751
898
|
for (const plan of plans) {
|
|
752
899
|
for (const dirPath of plan.dirGroups.keys()) {
|
|
753
900
|
const resolvedDir = path.resolve(dirPath);
|
|
754
901
|
if (claimedDirs.has(resolvedDir))
|
|
755
902
|
continue;
|
|
756
903
|
claimedDirs.add(resolvedDir);
|
|
757
|
-
for (const
|
|
758
|
-
if (
|
|
759
|
-
|
|
904
|
+
for (const priorOwnerBundle of getIndexedBundleIdsByDir(db, dirPath)) {
|
|
905
|
+
if (priorOwnerBundle !== plan.component.id) {
|
|
906
|
+
const priorOwnerPath = sourcePathByBundle.get(priorOwnerBundle);
|
|
907
|
+
if (priorOwnerPath)
|
|
908
|
+
addRemoval(plan, dirPath, priorOwnerPath);
|
|
760
909
|
}
|
|
761
910
|
}
|
|
762
911
|
}
|
|
@@ -876,7 +1025,7 @@ async function scanSourceDirs(db, allSourceEntries, isIncremental, builtAtMs, ha
|
|
|
876
1025
|
// abstains on its own bundle's walked files. The core walk keeps only the
|
|
877
1026
|
// universal hygiene `walkStashFlat` already applies (.git/dot-dirs/etc.).
|
|
878
1027
|
const indexableFiles = ctxs.map((ctx) => ctx.absPath);
|
|
879
|
-
const forceScan = handoffDirs.has(path.resolve(dirPath));
|
|
1028
|
+
const forceScan = handoffDirs.has(path.resolve(dirPath)) || requiresWorkflowSourcePreflight(ctxs);
|
|
880
1029
|
if (markSeenOrSkipDuplicate(dirPath, currentStashDir, indexableFiles))
|
|
881
1030
|
continue;
|
|
882
1031
|
if (indexableFiles.length === 0) {
|
|
@@ -931,6 +1080,16 @@ async function scanSourceDirs(db, allSourceEntries, isIncremental, builtAtMs, ha
|
|
|
931
1080
|
complete: plans.every((plan) => plan.walkComplete && plan.adapter !== undefined),
|
|
932
1081
|
};
|
|
933
1082
|
}
|
|
1083
|
+
function requiresWorkflowSourcePreflight(ctxs) {
|
|
1084
|
+
return ctxs.some((ctx) => {
|
|
1085
|
+
try {
|
|
1086
|
+
return fs.lstatSync(ctx.absPath).isSymbolicLink();
|
|
1087
|
+
}
|
|
1088
|
+
catch {
|
|
1089
|
+
return true;
|
|
1090
|
+
}
|
|
1091
|
+
});
|
|
1092
|
+
}
|
|
934
1093
|
function preserveExistingIndex(doFullDelete, dirRecords, sourceRoots) {
|
|
935
1094
|
if (!doFullDelete)
|
|
936
1095
|
return false;
|
|
@@ -1020,8 +1179,13 @@ function persistDirRecords(db, dirRecords, doFullDelete, warnings, sourceRoots,
|
|
|
1020
1179
|
indexTransactionHook("full-delete-applied");
|
|
1021
1180
|
}
|
|
1022
1181
|
for (const { dirPath, currentStashDir, files, stash, skip, reason, hashByFile, conceptIdByFile, indexVariant, remove, pruneMissing, } of dirRecords) {
|
|
1182
|
+
const bundle = bundleByRoot.get(path.resolve(currentStashDir));
|
|
1183
|
+
if (!bundle)
|
|
1184
|
+
throw new Error(`Missing bundle provenance for indexed source ${currentStashDir}`);
|
|
1023
1185
|
if (remove) {
|
|
1024
|
-
const removedIds =
|
|
1186
|
+
const removedIds = deleteEntriesByDirAndBundle(db, dirPath, bundle.bundleId, {
|
|
1187
|
+
cleanupUsageEvents: false,
|
|
1188
|
+
});
|
|
1025
1189
|
addEntryIds(deletedUsageEntryIds, removedIds);
|
|
1026
1190
|
deleteIndexDirState(db, dirPath);
|
|
1027
1191
|
continue;
|
|
@@ -1041,15 +1205,12 @@ function persistDirRecords(db, dirRecords, doFullDelete, warnings, sourceRoots,
|
|
|
1041
1205
|
// Diff-persist (F4a M-core-2, spec §14.2): upsert the current file set
|
|
1042
1206
|
// FIRST (ON CONFLICT preserving `entries.id` so embeddings / utility /
|
|
1043
1207
|
// usage stay attached to unchanged rows), tracking every upserted
|
|
1044
|
-
// `
|
|
1208
|
+
// durable `item_ref`, then prune only the departed rows below. Replaces the old
|
|
1045
1209
|
// `deleteEntriesByDir` truncate-and-reinsert (which discarded ids).
|
|
1046
|
-
const
|
|
1210
|
+
const keptItemRefs = new Set();
|
|
1047
1211
|
let persistedRows = 0;
|
|
1048
1212
|
let dedupedRows = 0;
|
|
1049
1213
|
if (stash) {
|
|
1050
|
-
const bundle = bundleByRoot.get(path.resolve(currentStashDir));
|
|
1051
|
-
if (!bundle)
|
|
1052
|
-
throw new Error(`Missing bundle provenance for indexed source ${currentStashDir}`);
|
|
1053
1214
|
const ownerIdentity = bundle.bundleId;
|
|
1054
1215
|
for (const entry of stash.entries) {
|
|
1055
1216
|
const entryPath = entry.filename ? path.join(dirPath, entry.filename) : null;
|
|
@@ -1070,24 +1231,15 @@ function persistDirRecords(db, dirRecords, doFullDelete, warnings, sourceRoots,
|
|
|
1070
1231
|
continue;
|
|
1071
1232
|
}
|
|
1072
1233
|
indexedAssetIdentities.add(identityKey);
|
|
1073
|
-
const entryKey = bundle?.adapterId !== "akm" && adapterConceptId
|
|
1074
|
-
? `${currentStashDir}:concept:${adapterConceptId}`
|
|
1075
|
-
: `${currentStashDir}:${entry.type}:${entry.name}`;
|
|
1076
|
-
keptEntryKeys.add(entryKey);
|
|
1077
1234
|
const searchText = buildSearchText(entry);
|
|
1078
1235
|
const entryWithSize = attachFileSize(entry, entryPath);
|
|
1079
1236
|
// content_hash = doc.hash from the drain, keyed by the recognized
|
|
1080
1237
|
// file's path. A missing hash preserves the existing value on upsert.
|
|
1081
1238
|
const contentHash = hashByFile?.get(entryPath);
|
|
1082
1239
|
const provenance = deriveEntryProvenance(bundle, entry.type, entry.name, adapterConceptId);
|
|
1083
|
-
|
|
1240
|
+
keptItemRefs.add(provenance.itemRef);
|
|
1241
|
+
upsertEntry(db, entryPath, entryWithSize, searchText, provenance, contentHash);
|
|
1084
1242
|
persistedRows++;
|
|
1085
|
-
if (entry.type === "workflow") {
|
|
1086
|
-
const doc = takeWorkflowDocument(entry);
|
|
1087
|
-
if (doc) {
|
|
1088
|
-
upsertWorkflowDocument(db, entryId, doc, fs.readFileSync(entryPath));
|
|
1089
|
-
}
|
|
1090
|
-
}
|
|
1091
1243
|
}
|
|
1092
1244
|
// Collect dirs needing LLM enhancement during the first walk.
|
|
1093
1245
|
// Only dirs with "generated" entries need enrichment.
|
|
@@ -1100,7 +1252,7 @@ function persistDirRecords(db, dirRecords, doFullDelete, warnings, sourceRoots,
|
|
|
1100
1252
|
// an empty kept-set this deletes every row for the dir — the exact net
|
|
1101
1253
|
// effect of the old unconditional `deleteEntriesByDir`, minus the id churn.
|
|
1102
1254
|
if (pruneMissing !== false) {
|
|
1103
|
-
addEntryIds(deletedUsageEntryIds,
|
|
1255
|
+
addEntryIds(deletedUsageEntryIds, deleteEntriesByDirExceptRefs(db, dirPath, bundle.bundleId, keptItemRefs, { cleanupUsageEvents: false }));
|
|
1104
1256
|
}
|
|
1105
1257
|
const fingerprint = computeDirFingerprint(dirPath, files, indexVariant);
|
|
1106
1258
|
const persistedReason = persistedRows === 0
|
|
@@ -1139,32 +1291,21 @@ function persistDirRecords(db, dirRecords, doFullDelete, warnings, sourceRoots,
|
|
|
1139
1291
|
deleteUsageEventsByEntryIds([...deletedUsageEntryIds]);
|
|
1140
1292
|
return { dirsNeedingLlm };
|
|
1141
1293
|
}
|
|
1142
|
-
async function indexEntries(db, allSourceEntries, isIncremental, builtAtMs, hadRemovedSources, doFullDelete = false, onProgress, reconcileMissingDirs = true) {
|
|
1294
|
+
async function indexEntries(db, allSourceEntries, isIncremental, builtAtMs, hadRemovedSources, doFullDelete = false, onProgress, reconcileMissingDirs = true, beforePersist) {
|
|
1143
1295
|
// Phase 1 (async): walk directories and pre-generate all metadata outside the
|
|
1144
1296
|
// transaction.
|
|
1145
1297
|
const { dirRecords, scannedDirs, skippedDirs, generatedCount, warnings, complete } = await scanSourceDirs(db, allSourceEntries, isIncremental, builtAtMs, hadRemovedSources, onProgress, reconcileMissingDirs);
|
|
1298
|
+
const bundleByRoot = buildIndexedSourceOwners(allSourceEntries);
|
|
1299
|
+
await beforePersist?.(dirRecords, bundleByRoot);
|
|
1146
1300
|
// Phase 2 (sync): write all pre-generated metadata inside a single transaction.
|
|
1147
1301
|
// Source roots feed the #624-P1 zero-document preflight (a full-rebuild wipe
|
|
1148
1302
|
// is suppressed when the scan is empty because roots are unreadable).
|
|
1149
1303
|
const sourceRoots = allSourceEntries.map((s) => s.path);
|
|
1150
|
-
//
|
|
1151
|
-
//
|
|
1152
|
-
//
|
|
1304
|
+
// Map each source root → its durable bundle id so the writer can persist
|
|
1305
|
+
// `item_ref = <bundle>//<conceptId>` and canonical component/adapter
|
|
1306
|
+
// provenance. `deriveInstallations`
|
|
1153
1307
|
// preserves source order, so a positional zip yields the SAME bundle id the
|
|
1154
1308
|
// dispatched `adapter.recognize` emits as `IndexDocument.ref` for that root.
|
|
1155
|
-
const installations = deriveInstallations(allSourceEntries);
|
|
1156
|
-
const bundleByRoot = new Map();
|
|
1157
|
-
allSourceEntries.forEach((source, i) => {
|
|
1158
|
-
const inst = installations[i];
|
|
1159
|
-
if (!inst)
|
|
1160
|
-
return;
|
|
1161
|
-
const component = inst.components[0];
|
|
1162
|
-
bundleByRoot.set(path.resolve(source.path), {
|
|
1163
|
-
bundleId: inst.id,
|
|
1164
|
-
componentId: component?.id ?? inst.id,
|
|
1165
|
-
adapterId: component?.adapter ?? "akm",
|
|
1166
|
-
});
|
|
1167
|
-
});
|
|
1168
1309
|
const { dirsNeedingLlm } = persistDirRecords(db, dirRecords, doFullDelete, warnings, sourceRoots, complete, bundleByRoot);
|
|
1169
1310
|
return { scannedDirs, skippedDirs, generatedCount, warnings, dirsNeedingLlm, complete };
|
|
1170
1311
|
}
|
|
@@ -1184,12 +1325,11 @@ function indexedProvenanceForFile(db, filePath) {
|
|
|
1184
1325
|
adapterId: row.adapterId,
|
|
1185
1326
|
};
|
|
1186
1327
|
}
|
|
1187
|
-
async function enhanceDirsWithLlm(db, config, dirsNeedingLlm, onProgress, signal) {
|
|
1188
|
-
//
|
|
1189
|
-
//
|
|
1190
|
-
|
|
1191
|
-
|
|
1192
|
-
if (!llmConfig || dirsNeedingLlm.length === 0)
|
|
1328
|
+
async function enhanceDirsWithLlm(db, config, execution, dirsNeedingLlm, onProgress, signal, onNotices, lease) {
|
|
1329
|
+
// The invocation owns one frozen symbolic selection. Summary reporting and
|
|
1330
|
+
// every enrichment dispatch consume this same snapshot.
|
|
1331
|
+
const llmRunner = execution.runner;
|
|
1332
|
+
if (!llmRunner || dirsNeedingLlm.length === 0)
|
|
1193
1333
|
return;
|
|
1194
1334
|
// Aggregate per-entry failures so a misconfigured LLM endpoint surfaces
|
|
1195
1335
|
// as a single visible warning instead of silently degrading every entry
|
|
@@ -1212,7 +1352,7 @@ async function enhanceDirsWithLlm(db, config, dirsNeedingLlm, onProgress, signal
|
|
|
1212
1352
|
// engine's timeoutMs (or 10 minutes if not set). Users can extend it via
|
|
1213
1353
|
// `index.enrichment.timeoutMs` (or `index.defaults.timeoutMs`, or the
|
|
1214
1354
|
// engine's own `engines.<name>.timeoutMs`) — no separate knob needed.
|
|
1215
|
-
const enrichDeadline = createEnrichmentDeadline(
|
|
1355
|
+
const enrichDeadline = createEnrichmentDeadline(llmRunner.timeoutMs, totalEntries);
|
|
1216
1356
|
let deadlineHit = false;
|
|
1217
1357
|
const enrichSignal = (() => {
|
|
1218
1358
|
if (!enrichDeadline)
|
|
@@ -1233,12 +1373,13 @@ async function enhanceDirsWithLlm(db, config, dirsNeedingLlm, onProgress, signal
|
|
|
1233
1373
|
onProgress?.({
|
|
1234
1374
|
phase: "llm",
|
|
1235
1375
|
message: `LLM enhancement starting for ${totalEntries} entr${totalEntries === 1 ? "y" : "ies"} ` +
|
|
1236
|
-
`across ${totalDirs} director${totalDirs === 1 ? "y" : "ies"} (concurrency ${getDefaultLlmConcurrency(
|
|
1376
|
+
`across ${totalDirs} director${totalDirs === 1 ? "y" : "ies"} (concurrency ${getDefaultLlmConcurrency(llmRunner.connection)}).`,
|
|
1237
1377
|
processed: 0,
|
|
1238
1378
|
total: totalEntries,
|
|
1239
1379
|
});
|
|
1240
1380
|
}
|
|
1241
1381
|
let currentDirLabel;
|
|
1382
|
+
let configFailure;
|
|
1242
1383
|
let lastProgressAt = Date.now();
|
|
1243
1384
|
let heartbeatTimer;
|
|
1244
1385
|
if (totalEntries > 0 && onProgress) {
|
|
@@ -1284,30 +1425,42 @@ async function enhanceDirsWithLlm(db, config, dirsNeedingLlm, onProgress, signal
|
|
|
1284
1425
|
});
|
|
1285
1426
|
lastProgressAt = Date.now();
|
|
1286
1427
|
const targetStash = { entries: entriesToEnhance };
|
|
1287
|
-
const
|
|
1288
|
-
|
|
1289
|
-
|
|
1290
|
-
lastProgressAt = Date.now();
|
|
1291
|
-
onProgress?.({
|
|
1292
|
-
phase: "llm",
|
|
1293
|
-
message: `Enhanced ${completedEntries}/${totalEntries} entr${totalEntries === 1 ? "y" : "ies"}; ` +
|
|
1294
|
-
`${completedDirs}/${totalDirs} director${totalDirs === 1 ? "y" : "ies"} complete` +
|
|
1295
|
-
(event.entryName ? `; current ${event.entryName}` : "") +
|
|
1296
|
-
(currentDirLabel ? ` in ${currentDirLabel}` : "") +
|
|
1297
|
-
(event.outcome === "cache-hit" ? " (cache hit)" : ""),
|
|
1298
|
-
processed: completedEntries,
|
|
1299
|
-
total: totalEntries,
|
|
1300
|
-
});
|
|
1428
|
+
const itemRefs = entriesToEnhance.map((entry) => {
|
|
1429
|
+
const entryPath = entry.filename ? path.join(dirPath, entry.filename) : files[0] || dirPath;
|
|
1430
|
+
return indexedProvenanceForFile(db, entryPath).itemRef;
|
|
1301
1431
|
});
|
|
1432
|
+
let enhanced;
|
|
1433
|
+
try {
|
|
1434
|
+
enhanced = await enhanceStashWithLlm(llmRunner, targetStash, files, summary, enrichSignal, db, itemRefs, config, (event) => {
|
|
1435
|
+
completedEntries++;
|
|
1436
|
+
lastProgressAt = Date.now();
|
|
1437
|
+
onProgress?.({
|
|
1438
|
+
phase: "llm",
|
|
1439
|
+
message: `Enhanced ${completedEntries}/${totalEntries} entr${totalEntries === 1 ? "y" : "ies"}; ` +
|
|
1440
|
+
`${completedDirs}/${totalDirs} director${totalDirs === 1 ? "y" : "ies"} complete` +
|
|
1441
|
+
(event.entryName ? `; current ${event.entryName}` : "") +
|
|
1442
|
+
(currentDirLabel ? ` in ${currentDirLabel}` : "") +
|
|
1443
|
+
(event.outcome === "cache-hit" ? " (cache hit)" : ""),
|
|
1444
|
+
processed: completedEntries,
|
|
1445
|
+
total: totalEntries,
|
|
1446
|
+
});
|
|
1447
|
+
}, onNotices, lease);
|
|
1448
|
+
}
|
|
1449
|
+
catch (err) {
|
|
1450
|
+
if (err instanceof ConfigError) {
|
|
1451
|
+
configFailure ??= err;
|
|
1452
|
+
return undefined;
|
|
1453
|
+
}
|
|
1454
|
+
throw err;
|
|
1455
|
+
}
|
|
1302
1456
|
// Re-upsert the enhanced entries in a single transaction so a crash
|
|
1303
1457
|
// cannot leave half the entries updated and the rest stale.
|
|
1304
1458
|
db.transaction(() => {
|
|
1305
1459
|
for (const entry of enhanced.entries) {
|
|
1306
1460
|
const entryPath = entry.filename ? path.join(dirPath, entry.filename) : files[0] || dirPath;
|
|
1307
|
-
const entryKey = `${currentStashDir}:${entry.type}:${entry.name}`;
|
|
1308
1461
|
const searchText = buildSearchText(entry);
|
|
1309
1462
|
const provenance = indexedProvenanceForFile(db, entryPath);
|
|
1310
|
-
upsertEntry(db,
|
|
1463
|
+
upsertEntry(db, entryPath, attachFileSize(entry, entryPath), searchText, provenance);
|
|
1311
1464
|
}
|
|
1312
1465
|
})();
|
|
1313
1466
|
completedDirs++;
|
|
@@ -1325,7 +1478,9 @@ async function enhanceDirsWithLlm(db, config, dirsNeedingLlm, onProgress, signal
|
|
|
1325
1478
|
// Studio, Ollama run one inference at a time — parallel requests cause
|
|
1326
1479
|
// "Model reloaded" / 500 errors). No config override reaches this path:
|
|
1327
1480
|
// `resolveLlmEngineUse` does not forward `engines.<name>.concurrency`.
|
|
1328
|
-
getDefaultLlmConcurrency(
|
|
1481
|
+
getDefaultLlmConcurrency(llmRunner.connection));
|
|
1482
|
+
if (configFailure)
|
|
1483
|
+
throw configFailure;
|
|
1329
1484
|
}
|
|
1330
1485
|
finally {
|
|
1331
1486
|
if (heartbeatTimer)
|
|
@@ -1401,7 +1556,7 @@ async function generateEmbeddingsForDb(db, config, onProgress, signal) {
|
|
|
1401
1556
|
const batchNum = Math.floor(i / EMBED_BATCH_SIZE) + 1;
|
|
1402
1557
|
const chars = texts[i].length;
|
|
1403
1558
|
const tokens = estimateTokenCount(texts[i]);
|
|
1404
|
-
const ref = allEntries[i].
|
|
1559
|
+
const ref = allEntries[i].itemRef;
|
|
1405
1560
|
warnVerbose(`[embed] ${ref} (${chars} chars, est. ${tokens} tokens) → batch ${batchNum}/${totalBatches}`);
|
|
1406
1561
|
}
|
|
1407
1562
|
}
|
|
@@ -1586,9 +1741,10 @@ function resolveIndexedFiles(dirPath, files, stash) {
|
|
|
1586
1741
|
}
|
|
1587
1742
|
return resolved.size > 0 ? [...resolved] : files;
|
|
1588
1743
|
}
|
|
1589
|
-
async function enhanceStashWithLlm(
|
|
1744
|
+
async function enhanceStashWithLlm(llmRunner, stash, files, summary, signal, db, itemRefs, akmConfig, onEntryDone, onNotices, lease) {
|
|
1590
1745
|
const { enhanceMetadata } = await import("../llm/metadata-enhance.js");
|
|
1591
1746
|
const { computeBodyHash, getLlmCacheEntry, upsertLlmCacheEntry } = await import("../storage/repositories/index-llm-cache-repository.js");
|
|
1747
|
+
let configFailure;
|
|
1592
1748
|
const results = await concurrentMap(stash.entries, async (entry, idx) => {
|
|
1593
1749
|
if (signal?.aborted)
|
|
1594
1750
|
return entry;
|
|
@@ -1607,11 +1763,12 @@ async function enhanceStashWithLlm(llmConfig, stash, files, summary, signal, db,
|
|
|
1607
1763
|
}
|
|
1608
1764
|
}
|
|
1609
1765
|
// Incremental cache: skip LLM call when file body is unchanged. The
|
|
1610
|
-
//
|
|
1611
|
-
// across index runs.
|
|
1766
|
+
// Cache metadata enrichment by the canonical durable item ref.
|
|
1612
1767
|
const cacheBody = fileContent ?? `${entry.name}\n${entry.description ?? ""}`;
|
|
1613
1768
|
const bodyHash = computeBodyHash(cacheBody);
|
|
1614
|
-
const cacheKey =
|
|
1769
|
+
const cacheKey = itemRefs?.[idx];
|
|
1770
|
+
if (!cacheKey)
|
|
1771
|
+
throw new Error(`Missing canonical item ref for enrichment entry ${entry.name}.`);
|
|
1615
1772
|
if (db) {
|
|
1616
1773
|
const cached = getLlmCacheEntry(db, cacheKey, bodyHash);
|
|
1617
1774
|
if (cached) {
|
|
@@ -1634,7 +1791,7 @@ async function enhanceStashWithLlm(llmConfig, stash, files, summary, signal, db,
|
|
|
1634
1791
|
}
|
|
1635
1792
|
}
|
|
1636
1793
|
}
|
|
1637
|
-
const outcome = await enhanceMetadata(
|
|
1794
|
+
const outcome = await enhanceMetadata(llmRunner, entry, fileContent, signal, akmConfig, onNotices, lease);
|
|
1638
1795
|
if (outcome.status !== "enriched") {
|
|
1639
1796
|
// Not a genuine LLM success: the gate was closed (`skipped`) or the
|
|
1640
1797
|
// call errored/timed out (`failed`). Do NOT mark the entry enriched
|
|
@@ -1682,6 +1839,10 @@ async function enhanceStashWithLlm(llmConfig, stash, files, summary, signal, db,
|
|
|
1682
1839
|
return updated;
|
|
1683
1840
|
}
|
|
1684
1841
|
catch (err) {
|
|
1842
|
+
if (err instanceof ConfigError) {
|
|
1843
|
+
configFailure ??= err;
|
|
1844
|
+
return entry;
|
|
1845
|
+
}
|
|
1685
1846
|
const msg = toErrorMessage(err);
|
|
1686
1847
|
// failureSamples is bounded to 3 items, so a linear scan is cheaper
|
|
1687
1848
|
// than maintaining a parallel Set for membership checks (#177 review).
|
|
@@ -1694,7 +1855,9 @@ async function enhanceStashWithLlm(llmConfig, stash, files, summary, signal, db,
|
|
|
1694
1855
|
},
|
|
1695
1856
|
// Defaults: 2 for remote LLM APIs, 1 for local model servers. No config
|
|
1696
1857
|
// override reaches this path (see getDefaultLlmConcurrency).
|
|
1697
|
-
getDefaultLlmConcurrency(
|
|
1858
|
+
getDefaultLlmConcurrency(llmRunner.connection));
|
|
1859
|
+
if (configFailure)
|
|
1860
|
+
throw configFailure;
|
|
1698
1861
|
// concurrentMap returns Array<T | undefined>; filter out undefined slots
|
|
1699
1862
|
// (which can only occur if the callback itself returned undefined, which
|
|
1700
1863
|
// it never does above — but TypeScript needs the filter for type safety).
|
|
@@ -1711,58 +1874,111 @@ async function resolveLookupSources() {
|
|
|
1711
1874
|
}
|
|
1712
1875
|
function resolveLookupScope(bundle, sources) {
|
|
1713
1876
|
if (!bundle)
|
|
1714
|
-
return {
|
|
1715
|
-
return {
|
|
1877
|
+
return { candidateSources: sources, qualified: false };
|
|
1878
|
+
return { candidateSources: resolveSourcesForOrigin(bundle, sources), qualified: true };
|
|
1716
1879
|
}
|
|
1717
|
-
|
|
1718
|
-
export async function lookupBundleRef(ref) {
|
|
1880
|
+
async function lookupBundleRefWithResolutionUsing(ref, openLookupDatabase) {
|
|
1719
1881
|
const sources = await resolveLookupSources();
|
|
1720
1882
|
if (sources.length === 0)
|
|
1721
|
-
return null;
|
|
1722
|
-
const
|
|
1723
|
-
|
|
1724
|
-
|
|
1725
|
-
|
|
1883
|
+
return { entry: null };
|
|
1884
|
+
const bundleBySourcePath = new Map(deriveInstallations(sources).map((installation, index) => [path.resolve(sources[index].path), installation.id]));
|
|
1885
|
+
const { candidateSources, qualified } = resolveLookupScope(ref.bundle, sources);
|
|
1886
|
+
if (candidateSources.length === 0)
|
|
1887
|
+
return { entry: null };
|
|
1888
|
+
let db;
|
|
1889
|
+
let indexError;
|
|
1726
1890
|
try {
|
|
1727
|
-
|
|
1728
|
-
|
|
1729
|
-
|
|
1730
|
-
|
|
1731
|
-
|
|
1732
|
-
|
|
1733
|
-
|
|
1734
|
-
|
|
1735
|
-
|
|
1736
|
-
|
|
1737
|
-
|
|
1738
|
-
|
|
1739
|
-
|
|
1740
|
-
|
|
1741
|
-
|
|
1742
|
-
|
|
1743
|
-
|
|
1744
|
-
|
|
1891
|
+
db = openLookupDatabase(getDbPath());
|
|
1892
|
+
}
|
|
1893
|
+
catch (error) {
|
|
1894
|
+
indexError = error;
|
|
1895
|
+
}
|
|
1896
|
+
try {
|
|
1897
|
+
for (const source of candidateSources) {
|
|
1898
|
+
const adapterId = source.adapterId ?? detectAdapterId(source.path);
|
|
1899
|
+
const owner = resolveAdapterConceptOwner(source.path, adapterId, ref.conceptId);
|
|
1900
|
+
const lookupConceptId = owner?.conceptId ?? ref.conceptId;
|
|
1901
|
+
const inputRef = makeBundleRef(qualified ? ref.bundle : undefined, lookupConceptId);
|
|
1902
|
+
const sourceBundleId = bundleBySourcePath.get(path.resolve(source.path));
|
|
1903
|
+
const id = db && sourceBundleId ? findEntryIdByRef(db, inputRef, sourceBundleId) : undefined;
|
|
1904
|
+
if (id !== undefined && owner && db) {
|
|
1905
|
+
const entry = readLookupEntry(db, id, ref.conceptId, source.path);
|
|
1906
|
+
if (entry) {
|
|
1907
|
+
if (owner.workflowSource) {
|
|
1908
|
+
assertIndexedWorkflowSourceIdentity(inputRef, entry.filePath, owner.workflowSource);
|
|
1909
|
+
if (entry.adapterId !== adapterId) {
|
|
1910
|
+
throw new WorkflowSourceIdentityError(inputRef, entry.filePath, owner.path);
|
|
1911
|
+
}
|
|
1912
|
+
}
|
|
1913
|
+
else if (entry.adapterId !== adapterId || !indexedPathMatchesOwner(entry.filePath, owner)) {
|
|
1914
|
+
return { entry: null, owner, ...(indexError === undefined ? {} : { indexError }) };
|
|
1915
|
+
}
|
|
1916
|
+
return { entry, owner, ...(indexError === undefined ? {} : { indexError }) };
|
|
1917
|
+
}
|
|
1745
1918
|
}
|
|
1746
|
-
|
|
1747
|
-
|
|
1748
|
-
|
|
1749
|
-
|
|
1750
|
-
filePath: row.filePath,
|
|
1751
|
-
stashDir: row.stashDir,
|
|
1752
|
-
type: row.type,
|
|
1753
|
-
name: document?.name ?? ref.conceptId.split("/").pop() ?? ref.conceptId,
|
|
1754
|
-
adapterId: row.adapterId,
|
|
1755
|
-
document,
|
|
1756
|
-
itemRef: row.itemRef,
|
|
1757
|
-
bundleId: row.bundleId,
|
|
1758
|
-
conceptId: row.conceptId,
|
|
1759
|
-
};
|
|
1919
|
+
// A physical owner with a missing/incomplete index row still owns this
|
|
1920
|
+
// unqualified concept. Stop here so a later source cannot retarget it.
|
|
1921
|
+
if (owner)
|
|
1922
|
+
return { entry: null, owner, ...(indexError === undefined ? {} : { indexError }) };
|
|
1760
1923
|
}
|
|
1761
|
-
return null;
|
|
1924
|
+
return { entry: null, ...(indexError === undefined ? {} : { indexError }) };
|
|
1762
1925
|
}
|
|
1763
1926
|
finally {
|
|
1764
|
-
|
|
1927
|
+
if (db)
|
|
1928
|
+
closeDatabase(db);
|
|
1929
|
+
}
|
|
1930
|
+
}
|
|
1931
|
+
export async function lookupBundleRefWithResolution(ref) {
|
|
1932
|
+
return lookupBundleRefWithResolutionUsing(ref, openExistingDatabase);
|
|
1933
|
+
}
|
|
1934
|
+
/** Resolve an adapter-owned `[bundle//]conceptId` without interpreting its path as an AKM type. */
|
|
1935
|
+
export async function lookupBundleRef(ref) {
|
|
1936
|
+
const resolution = await lookupBundleRefWithResolution(ref);
|
|
1937
|
+
if (resolution.indexError !== undefined)
|
|
1938
|
+
throw resolution.indexError;
|
|
1939
|
+
return resolution.entry;
|
|
1940
|
+
}
|
|
1941
|
+
/**
|
|
1942
|
+
* Resolve one execution source without opening the live index database for
|
|
1943
|
+
* write or allowing SQLite read-lock bookkeeping to touch its SHM file.
|
|
1944
|
+
*/
|
|
1945
|
+
export async function lookupBundleRefReadonly(ref) {
|
|
1946
|
+
const resolution = await lookupBundleRefWithResolutionUsing(ref, (dbPath) => {
|
|
1947
|
+
const db = openReadonlyExistingDatabase(dbPath, { isolatedSnapshot: true });
|
|
1948
|
+
if (!db)
|
|
1949
|
+
throw new Error(`Index database not found at ${dbPath}. Run 'akm index' to build it.`);
|
|
1950
|
+
return db;
|
|
1951
|
+
});
|
|
1952
|
+
if (resolution.indexError !== undefined)
|
|
1953
|
+
throw resolution.indexError;
|
|
1954
|
+
return resolution.entry;
|
|
1955
|
+
}
|
|
1956
|
+
function readLookupEntry(db, id, fallbackConceptId, sourceRoot) {
|
|
1957
|
+
const row = db
|
|
1958
|
+
.prepare("SELECT file_path AS filePath, type, document_json AS documentJson, " +
|
|
1959
|
+
"item_ref AS itemRef, bundle_id AS bundleId, concept_id AS conceptId, " +
|
|
1960
|
+
"adapter_id AS adapterId FROM entries WHERE id = ?")
|
|
1961
|
+
.get(id);
|
|
1962
|
+
if (!row)
|
|
1963
|
+
return null;
|
|
1964
|
+
let document;
|
|
1965
|
+
try {
|
|
1966
|
+
document = JSON.parse(row.documentJson);
|
|
1765
1967
|
}
|
|
1968
|
+
catch {
|
|
1969
|
+
// Corrupt optional projection does not erase the durable path identity.
|
|
1970
|
+
}
|
|
1971
|
+
return {
|
|
1972
|
+
filePath: row.filePath,
|
|
1973
|
+
stashDir: sourceRoot,
|
|
1974
|
+
type: row.type,
|
|
1975
|
+
name: document?.name ?? fallbackConceptId.split("/").pop() ?? fallbackConceptId,
|
|
1976
|
+
adapterId: row.adapterId,
|
|
1977
|
+
document,
|
|
1978
|
+
itemRef: row.itemRef,
|
|
1979
|
+
bundleId: row.bundleId,
|
|
1980
|
+
conceptId: row.conceptId,
|
|
1981
|
+
};
|
|
1766
1982
|
}
|
|
1767
1983
|
/**
|
|
1768
1984
|
* Look up a single asset by ref. Spec §6.2 — `akm show` queries this and
|
|
@@ -1795,11 +2011,16 @@ const USAGE_EVENT_RETENTION_DAYS = 90;
|
|
|
1795
2011
|
*
|
|
1796
2012
|
* Called during `akm index` after FTS rebuild.
|
|
1797
2013
|
*/
|
|
1798
|
-
export function recomputeUtilityScores(db, stateDb) {
|
|
2014
|
+
export function recomputeUtilityScores(db, stateDb, options) {
|
|
1799
2015
|
const EMA_DECAY = 0.7;
|
|
2016
|
+
const stateSchema = options?.stateSchema;
|
|
2017
|
+
if (stateSchema !== undefined && !/^[A-Za-z_][A-Za-z0-9_]*$/.test(stateSchema)) {
|
|
2018
|
+
throw new Error("Invalid attached state schema name.");
|
|
2019
|
+
}
|
|
2020
|
+
const usageEvents = stateSchema === undefined ? "usage_events" : `"${stateSchema}".usage_events`;
|
|
1800
2021
|
// Purge stale usage events (90-day retention). usage_events lives in state.db
|
|
1801
2022
|
// (Chunk-8 WI-8.3); its table is created by state migration 020.
|
|
1802
|
-
purgeOldUsageEvents(stateDb, USAGE_EVENT_RETENTION_DAYS);
|
|
2023
|
+
purgeOldUsageEvents(stateDb, USAGE_EVENT_RETENTION_DAYS, { stateSchema });
|
|
1803
2024
|
// Time-proportional decay: apply one round of EMA per elapsed day so
|
|
1804
2025
|
// indexing frequency doesn't affect how fast scores decay.
|
|
1805
2026
|
const lastComputedAt = getMeta(db, "last_utility_computed_at");
|
|
@@ -1830,7 +2051,7 @@ export function recomputeUtilityScores(db, stateDb) {
|
|
|
1830
2051
|
ELSE NULL
|
|
1831
2052
|
END
|
|
1832
2053
|
) AS last_used_at
|
|
1833
|
-
FROM
|
|
2054
|
+
FROM ${usageEvents} u
|
|
1834
2055
|
WHERE u.entry_id IS NOT NULL
|
|
1835
2056
|
AND u.source = 'user'
|
|
1836
2057
|
GROUP BY u.entry_id
|