akm-cli 0.9.17-alpha.3 → 0.9.17-alpha.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +731 -0
- package/dist/akm +94 -196
- package/dist/cli/shared.js +6 -2
- package/dist/cli.js +22 -9
- package/dist/commands/agent/agent-dispatch.js +1 -1
- package/dist/commands/command/command-execution.js +24 -62
- package/dist/commands/feedback-cli.js +0 -1
- package/dist/commands/health/accept-rate.js +2 -2
- package/dist/commands/health/checks.js +30 -75
- package/dist/commands/health/config-skew.js +38 -0
- package/dist/commands/health/egress.js +54 -0
- package/dist/commands/health/html-report.js +0 -38
- package/dist/commands/health/improve-metrics.js +123 -562
- package/dist/commands/health/plugin-staleness.js +53 -3
- package/dist/commands/health/renderers.js +12 -4
- package/dist/commands/health/report-view-model.js +11 -106
- package/dist/commands/health/types-improve.js +4 -19
- package/dist/commands/health/windows.js +64 -73
- package/dist/commands/health.js +122 -143
- package/dist/commands/improve/consolidate/chunking.js +25 -100
- package/dist/commands/improve/consolidate/sanitize.js +54 -149
- package/dist/commands/improve/consolidate.js +538 -1075
- package/dist/commands/improve/content-hash.js +16 -24
- package/dist/commands/improve/distill/content-repair.js +18 -100
- package/dist/commands/improve/distill-guards.js +20 -81
- package/dist/commands/improve/distill-promotion-policy.js +23 -243
- package/dist/commands/improve/distill.js +608 -1075
- package/dist/commands/improve/eligibility.js +126 -400
- package/dist/commands/improve/execution.js +3 -5
- package/dist/commands/improve/extract.js +487 -1046
- package/dist/commands/improve/feedback-valence.js +0 -25
- package/dist/commands/improve/improve-cli.js +29 -166
- package/dist/commands/improve/improve-result-file.js +10 -66
- package/dist/commands/improve/improve-strategies.js +12 -7
- package/dist/commands/improve/improve-usage-report.js +18 -64
- package/dist/commands/improve/improve.js +443 -1063
- package/dist/commands/improve/ledger.js +114 -0
- package/dist/commands/improve/locks.js +2 -8
- package/dist/commands/improve/loop-stages.js +459 -1172
- package/dist/commands/improve/memory/derived-ref.js +12 -77
- package/dist/commands/improve/memory/memory-belief.js +14 -118
- package/dist/commands/improve/memory/memory-improve.js +4 -3
- package/dist/commands/improve/outcome-loop.js +28 -156
- package/dist/commands/improve/planner.js +5 -10
- package/dist/commands/improve/preparation.js +851 -2339
- package/dist/commands/improve/proactive-maintenance.js +34 -101
- package/dist/commands/improve/reflect-noise.js +104 -280
- package/dist/commands/improve/reflect.js +621 -1367
- package/dist/commands/improve/salience.js +46 -232
- package/dist/commands/improve/session-asset.js +19 -100
- package/dist/commands/improve/stage.js +323 -0
- package/dist/commands/proposal/drain.js +251 -644
- package/dist/commands/proposal/proposal-cli.js +3 -18
- package/dist/commands/proposal/proposal-types.js +20 -41
- package/dist/commands/proposal/proposal.js +1 -2
- package/dist/commands/proposal/propose.js +134 -160
- package/dist/commands/proposal/repository.js +502 -1487
- package/dist/commands/proposal/validators/proposal-quality-validators.js +71 -174
- package/dist/commands/proposal/validators/proposal-validators.js +1 -1
- package/dist/commands/proposal/validators/proposals.js +13 -89
- package/dist/commands/read/curate.js +63 -413
- package/dist/commands/read/search-cli.js +16 -33
- package/dist/commands/read/search.js +17 -23
- package/dist/commands/read/show.js +2 -13
- package/dist/commands/sources/bundle-cli.js +25 -2
- package/dist/commands/sources/bundle-config-ops.js +7 -0
- package/dist/commands/sources/dangerous-env-audit.js +1 -2
- package/dist/commands/sources/info.js +2 -11
- package/dist/commands/sources/installed-stashes.js +197 -746
- package/dist/commands/sources/schema-repair.js +98 -129
- package/dist/commands/sources/source-add.js +62 -12
- package/dist/commands/sources/stash-cli.js +1 -1
- package/dist/commands/tasks/explain.js +10 -13
- package/dist/commands/tasks/tasks-cli.js +9 -8
- package/dist/commands/tasks/tasks.js +326 -930
- package/dist/commands/tasks/validate.js +42 -21
- package/dist/commands/workflow/plan.js +22 -29
- package/dist/commands/workflow-cli.js +4 -4
- package/dist/core/adapter/adapters/akm-adapter.js +0 -1
- package/dist/core/adapter/adapters/akm-lint.js +2 -3
- package/dist/core/adapter/adapters/akm-metadata.js +11 -12
- package/dist/core/adapter/adapters/akm-workflow-adapter.js +1 -1
- package/dist/core/adapter/execution-source.js +17 -29
- package/dist/core/asset/resolve-ref.js +1 -1
- package/dist/core/bundle-id.js +42 -5
- package/dist/core/bundle-rename.js +291 -0
- package/dist/core/config/config-io.js +1 -2
- package/dist/core/config/config-schema.js +1 -33
- package/dist/core/config/config-walker.js +1 -1
- package/dist/core/config/config.js +163 -68
- package/dist/core/config/legacy-source-shape-shim.js +38 -9
- package/dist/core/config/schema/embedding.js +20 -5
- package/dist/core/config/schema/engines.js +5 -0
- package/dist/core/config/schema/execution.js +1 -1
- package/dist/core/config/schema/experimental.js +1 -1
- package/dist/core/config/schema/improve-processes.js +21 -95
- package/dist/core/config/schema/improve.js +4 -42
- package/dist/core/config/schema/scheduler.js +12 -12
- package/dist/core/config/schema/search.js +6 -22
- package/dist/core/env-secret-ref.js +0 -1
- package/dist/core/errors.js +8 -9
- package/dist/core/file-lock.js +76 -173
- package/dist/core/logs-db.js +2 -2
- package/dist/core/paths.js +0 -27
- package/dist/core/redaction.js +109 -2
- package/dist/core/run-lock.js +2 -5
- package/dist/core/spawn-env.js +1 -1
- package/dist/core/state/migrations.js +108 -61
- package/dist/core/state-db-scope.js +2 -4
- package/dist/core/state-db.js +126 -692
- package/dist/core/type-presentation.js +1 -9
- package/dist/core/write-source.js +293 -1012
- package/dist/execution/input-contract.js +1 -1
- package/dist/execution/resolved-request.js +135 -689
- package/dist/execution/source.js +63 -257
- package/dist/execution/target-ref.js +1 -1
- package/dist/indexer/bundle-identity-guard.js +2 -2
- package/dist/indexer/db/graph-db.js +106 -46
- package/dist/indexer/ensure-index.js +44 -85
- package/dist/indexer/graph/graph-extraction.js +340 -562
- package/dist/indexer/graph/graph-related.js +130 -0
- package/dist/indexer/index-rebuild-lock.js +3 -11
- package/dist/indexer/index-writer-lock.js +8 -17
- package/dist/indexer/index-written-assets.js +139 -151
- package/dist/indexer/indexer.js +524 -846
- package/dist/indexer/materialize-embeddings.js +60 -397
- package/dist/indexer/passes/memory-inference.js +81 -90
- package/dist/indexer/passes/metadata.js +132 -200
- package/dist/indexer/read-preflight.js +0 -7
- package/dist/indexer/scan/doc-to-entry.js +1 -3
- package/dist/indexer/scan/drain-dir.js +1 -1
- package/dist/indexer/search/db-search.js +181 -590
- package/dist/indexer/search/fts-query.js +30 -41
- package/dist/indexer/search/ranking.js +28 -154
- package/dist/indexer/search/search-attribution.js +12 -32
- package/dist/indexer/search/search-fields.js +11 -15
- package/dist/indexer/search/search-hit-enrichers.js +54 -85
- package/dist/indexer/search/search-source.js +1 -4
- package/dist/indexer/usage/usage-events.js +2 -7
- package/dist/integrations/agent/engine-fallback.js +23 -40
- package/dist/integrations/agent/engine-resolution.js +93 -183
- package/dist/integrations/agent/execution.js +507 -0
- package/dist/integrations/agent/model-map.js +28 -156
- package/dist/integrations/agent/request-lowering.js +66 -141
- package/dist/integrations/agent/runner-dispatch.js +143 -321
- package/dist/integrations/agent/runner.js +54 -14
- package/dist/integrations/lockfile.js +53 -101
- package/dist/llm/embedders/deterministic.js +2 -3
- package/dist/llm/embedders/profile.js +71 -0
- package/dist/llm/embedders/remote.js +10 -15
- package/dist/llm/graph-extract.js +3 -12
- package/dist/llm/index-passes.js +3 -5
- package/dist/llm/memory-infer.js +1 -2
- package/dist/llm/metadata-enhance.js +1 -2
- package/dist/llm/structured-call.js +5 -24
- package/dist/output/generic-render.js +23 -11
- package/dist/output/html-render.js +13 -10
- package/dist/output/render-registry.js +3 -32
- package/dist/output/shapes/helpers.js +2 -34
- package/dist/output/shapes/passthrough.js +1 -9
- package/dist/{indexer/search/ranking-types.js → output/text/bundle-rename.js} +4 -1
- package/dist/output/text/command-format.js +60 -23
- package/dist/output/text/helpers.js +1 -1
- package/dist/output/text/migrate.js +5 -14
- package/dist/output/text/proposal-format.js +1 -2
- package/dist/output/text/workflow-format.js +0 -32
- package/dist/output/text.js +2 -0
- package/dist/registry/factory.js +4 -19
- package/dist/registry/network.js +66 -220
- package/dist/registry/providers/index.js +0 -2
- package/dist/registry/providers/skills-sh.js +3 -14
- package/dist/registry/providers/static-index.js +24 -26
- package/dist/registry/resolve.js +55 -131
- package/dist/scripts/akm-migrate-node.js +43937 -93313
- package/dist/scripts/akm-migrate.js +43697 -93071
- package/dist/setup/registry-stash-loader.js +4 -13
- package/dist/setup/semantic-assets.js +3 -44
- package/dist/setup/setup.js +1 -1
- package/dist/setup/steps/tasks.js +25 -15
- package/dist/sources/provider-factory.js +17 -18
- package/dist/sources/providers/filesystem.js +2 -3
- package/dist/sources/providers/git-install.js +7 -1
- package/dist/sources/providers/git-provider.js +0 -3
- package/dist/sources/providers/git-stash.js +0 -17
- package/dist/sources/providers/npm.js +2 -4
- package/dist/sources/providers/provider-utils.js +5 -10
- package/dist/sources/providers/website.js +0 -2
- package/dist/sources/snapshot-fetchers/website-ingest.js +1 -1
- package/dist/sources/website-url.js +2 -2
- package/dist/storage/database.js +9 -35
- package/dist/storage/repositories/improve-ledger-repository.js +168 -0
- package/dist/storage/repositories/index-connection.js +34 -70
- package/dist/storage/repositories/index-entries-repository.js +69 -111
- package/dist/storage/repositories/index-entry-mapper.js +1 -2
- package/dist/storage/repositories/index-entry-schema.js +83 -269
- package/dist/storage/repositories/index-fts-repository.js +86 -256
- package/dist/storage/repositories/index-llm-cache-repository.js +17 -0
- package/dist/storage/repositories/index-meta-repository.js +6 -4
- package/dist/storage/repositories/index-schema.js +192 -220
- package/dist/storage/repositories/index-utility-repository.js +8 -29
- package/dist/storage/repositories/index-vec-repository.js +133 -414
- package/dist/storage/repositories/outcome-repository.js +2 -1
- package/dist/storage/repositories/proposals-repository.js +35 -0
- package/dist/storage/repositories/registry-index-cache-repository.js +100 -0
- package/dist/storage/repositories/task-history-repository.js +26 -4
- package/dist/storage/repositories/workflow-runs-repository.js +53 -244
- package/dist/storage/sqlite-migrations.js +136 -0
- package/dist/storage/sqlite-pragmas.js +11 -9
- package/dist/storage/sqlite-transaction.js +170 -0
- package/dist/storage/state-db-integrity.js +34 -27
- package/dist/tasks/activation-config.js +134 -62
- package/dist/tasks/backends/cron.js +129 -277
- package/dist/tasks/backends/exec-utils.js +2 -5
- package/dist/tasks/backends/launchd.js +125 -745
- package/dist/tasks/backends/schtasks.js +101 -620
- package/dist/tasks/prepare/prepare-support.js +5 -15
- package/dist/tasks/prepare/prepare.js +0 -2
- package/dist/tasks/resolve-akm-bin.js +20 -79
- package/dist/tasks/run/attempt-lifecycle.js +0 -1
- package/dist/tasks/scheduler-binding.js +18 -238
- package/dist/tasks/scheduler-invocation.js +52 -52
- package/dist/tasks/scheduler-lock.js +53 -0
- package/dist/tasks/scheduler-sync.js +361 -751
- package/dist/tasks/source/parse-task-source.js +160 -10
- package/dist/tasks/source/task-source-v3-frozen.js +3 -4
- package/dist/tasks/source/task-to-v4.js +2 -2
- package/dist/workflows/authoring/authoring.js +3 -12
- package/dist/workflows/compile.js +211 -0
- package/dist/workflows/concurrency-policy.js +13 -74
- package/dist/workflows/exec/child-invocation.js +3 -17
- package/dist/workflows/exec/child-workflow.js +32 -141
- package/dist/workflows/exec/dispatch-redaction.js +13 -53
- package/dist/workflows/exec/environment.js +98 -0
- package/dist/workflows/exec/exec-unit.js +33 -140
- package/dist/workflows/exec/frozen-judge.js +7 -59
- package/dist/workflows/exec/native-executor.js +82 -341
- package/dist/workflows/exec/param-secrets.js +29 -47
- package/dist/workflows/exec/run-workflow.js +154 -387
- package/dist/workflows/exec/scheduler.js +9 -36
- package/dist/workflows/exec/step-work.js +127 -430
- package/dist/workflows/exec/unit-dispatch.js +11 -63
- package/dist/workflows/exec/unit-writer.js +8 -52
- package/dist/workflows/exec/worktree.js +39 -273
- package/dist/workflows/freeze/child-output-references.js +4 -15
- package/dist/workflows/freeze/environment.js +99 -92
- package/dist/workflows/freeze/freeze.js +172 -0
- package/dist/workflows/freeze/step-values.js +19 -21
- package/dist/workflows/freeze/targets/child-workflow.js +23 -92
- package/dist/workflows/freeze/targets/command.js +10 -33
- package/dist/workflows/freeze/targets/script.js +5 -12
- package/dist/workflows/freeze/targets/shell.js +3 -6
- package/dist/workflows/freeze/targets/task.js +25 -80
- package/dist/workflows/freeze/task-bindings.js +20 -67
- package/dist/workflows/{source-ir/github-yaml.js → github-yaml.js} +88 -206
- package/dist/workflows/ir/params.js +6 -51
- package/dist/workflows/ir/plan-hash.js +2 -34
- package/dist/workflows/parser.js +140 -43
- package/dist/{commands/improve/consolidate/types.js → workflows/plan.js} +2 -1
- package/dist/workflows/renderer.js +36 -69
- package/dist/workflows/resource-limits.js +12 -120
- package/dist/workflows/runtime/agent-identity.js +8 -40
- package/dist/workflows/runtime/run-outputs.js +3 -6
- package/dist/workflows/runtime/run-plan.js +316 -0
- package/dist/workflows/runtime/runs.js +48 -200
- package/dist/workflows/runtime/workflow-asset-loader.js +24 -57
- package/dist/workflows/{source-ir/semantics.js → source-semantics.js} +16 -20
- package/dist/workflows/validate-summary.js +2 -7
- package/docs/integration/bundling-akm.md +49 -42
- package/docs/migration/README.md +1 -0
- package/docs/migration/release-notes/0.9.17.md +41 -0
- package/docs/migration/v0.9.1-to-v0.9.2.md +19 -7
- package/docs/reference/cli.md +182 -125
- package/docs/reference/configuration.md +49 -56
- package/docs/reference/data-and-telemetry.md +19 -20
- package/docs/reference/tasks.md +86 -38
- package/docs/reference/workflow-schema.md +14 -18
- package/docs/reference/workflows.md +6 -9
- package/package.json +1 -1
- package/schemas/akm-config.json +87 -406
- package/dist/commands/health/advisories.js +0 -150
- package/dist/commands/health/metrics.js +0 -329
- package/dist/commands/health/surfaces.js +0 -102
- package/dist/commands/improve/anti-collapse.js +0 -83
- package/dist/commands/improve/collapse-detector.js +0 -432
- package/dist/commands/improve/consolidate/eligibility.js +0 -48
- package/dist/commands/improve/consolidate/merge.js +0 -146
- package/dist/commands/improve/distill/promote-memory.js +0 -329
- package/dist/commands/improve/distill/quality-gate.js +0 -500
- package/dist/commands/improve/memory/memory-contradiction-detect.js +0 -291
- package/dist/commands/improve/proposal-envelope.js +0 -31
- package/dist/commands/improve/run-context.js +0 -123
- package/dist/commands/improve/shared.js +0 -21
- package/dist/commands/improve/source-identity.js +0 -28
- package/dist/commands/improve/triage.js +0 -96
- package/dist/commands/proposal/drain-policies.js +0 -151
- package/dist/commands/sources/update-transaction.js +0 -220
- package/dist/core/action-contributors.js +0 -28
- package/dist/core/config/config-version-shim.js +0 -101
- package/dist/core/config/retired-experimental-keys-shim.js +0 -62
- package/dist/core/fs-txn.js +0 -405
- package/dist/core/lexical-score.js +0 -25
- package/dist/core/maintenance-barrier.js +0 -167
- package/dist/execution/executable-identity.js +0 -105
- package/dist/execution/guarded-source.js +0 -441
- package/dist/indexer/graph/graph-boost.js +0 -427
- package/dist/indexer/graph/graph-dedup.js +0 -95
- package/dist/indexer/search/name-match.js +0 -35
- package/dist/indexer/search/ranking-contributors.js +0 -515
- package/dist/indexer/walk/project-context.js +0 -192
- package/dist/integrations/agent/execution-cascade.js +0 -566
- package/dist/integrations/agent/execution-definitions.js +0 -202
- package/dist/integrations/agent/execution-lowering.js +0 -841
- package/dist/integrations/agent/execution-preparation.js +0 -98
- package/dist/integrations/agent/inline-execution.js +0 -74
- package/dist/registry/create-provider-registry.js +0 -29
- package/dist/registry/pinned-request-helper.js +0 -247
- package/dist/registry/pinned-transport.js +0 -717
- package/dist/sources/providers/index.js +0 -14
- package/dist/storage/engines/sqlite-migrations.js +0 -271
- package/dist/storage/repositories/canaries-repository.js +0 -107
- package/dist/storage/repositories/embedding-salvage-repository.js +0 -184
- package/dist/storage/repositories/registry-cache.js +0 -113
- package/dist/tasks/scheduler-sync-preview.js +0 -52
- package/dist/workflows/freeze/resolve-steps.js +0 -86
- package/dist/workflows/freeze/source-freeze.js +0 -64
- package/dist/workflows/ir/compile.js +0 -321
- package/dist/workflows/ir/environment-v4.js +0 -330
- package/dist/workflows/ir/freeze-v4.js +0 -153
- package/dist/workflows/ir/schema-v4.js +0 -745
- package/dist/workflows/ir/schema.js +0 -354
- package/dist/workflows/program/schema.js +0 -78
- package/dist/workflows/runtime/checkin.js +0 -57
- package/dist/workflows/runtime/plan-classifier.js +0 -196
- package/dist/workflows/runtime/unit-checkin.js +0 -45
- package/dist/workflows/runtime/unit-phases.js +0 -20
- package/dist/workflows/schema.js +0 -4
- package/dist/workflows/source-ir/compile.js +0 -200
- package/dist/workflows/source-ir/program.js +0 -50
- package/dist/workflows/source-ir/result.js +0 -26
- package/dist/workflows/source-ir/schema.js +0 -786
- package/dist/workflows/source-ir/triggers.js +0 -79
- package/dist/workflows/source-ir/uses.js +0 -40
- package/dist/workflows/validator.js +0 -60
|
@@ -1,14 +0,0 @@
|
|
|
1
|
-
// This Source Code Form is subject to the terms of the Mozilla Public
|
|
2
|
-
// License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
3
|
-
// file, You can obtain one at https://mozilla.org/MPL/2.0/.
|
|
4
|
-
/**
|
|
5
|
-
* Centralized source provider registration.
|
|
6
|
-
*
|
|
7
|
-
* Import this module (side-effect import) to register all built-in source
|
|
8
|
-
* providers with the provider registry. This replaces the individual
|
|
9
|
-
* side-effect imports that were duplicated in source-search.ts and source-show.ts.
|
|
10
|
-
*/
|
|
11
|
-
import "./filesystem.js";
|
|
12
|
-
import "./git.js";
|
|
13
|
-
import "./npm.js";
|
|
14
|
-
import "./website.js";
|
|
@@ -1,271 +0,0 @@
|
|
|
1
|
-
// This Source Code Form is subject to the terms of the Mozilla Public
|
|
2
|
-
// License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
3
|
-
// file, You can obtain one at https://mozilla.org/MPL/2.0/.
|
|
4
|
-
/**
|
|
5
|
-
* Shared SQLite migration engine.
|
|
6
|
-
*
|
|
7
|
-
* SQLite schemas evolve through this transaction-per-migration runner backed
|
|
8
|
-
* by a `schema_migrations` ledger.
|
|
9
|
-
*
|
|
10
|
-
* This module factors that runner out once. Each caller supplies only its own
|
|
11
|
-
* `MIGRATIONS` array.
|
|
12
|
-
*
|
|
13
|
-
* Ledger/transaction contract:
|
|
14
|
-
* - `id` is permanent and must never be reused.
|
|
15
|
-
* - Applied IDs must be an exact ordered prefix of the registry.
|
|
16
|
-
* - Each `up` body and its ledger insert commit in the same transaction.
|
|
17
|
-
* - Migration bodies run only after `BEGIN IMMEDIATE` is observably active,
|
|
18
|
-
* and the transaction must remain active through the body's completion.
|
|
19
|
-
* - The caller owns semantic safety classification and any policy gate;
|
|
20
|
-
* this generic engine intentionally does not infer risk from SQL text.
|
|
21
|
-
*/
|
|
22
|
-
import { sleepSync } from "../../runtime.js";
|
|
23
|
-
export function assertMigrationRegistry(migrations) {
|
|
24
|
-
const seen = new Set();
|
|
25
|
-
for (const migration of migrations) {
|
|
26
|
-
if (seen.has(migration.id))
|
|
27
|
-
throw new Error(`Migration registry contains duplicate ID ${migration.id}.`);
|
|
28
|
-
seen.add(migration.id);
|
|
29
|
-
}
|
|
30
|
-
}
|
|
31
|
-
export function migrationLedgerExists(db) {
|
|
32
|
-
return !!db.prepare("SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'schema_migrations'").get();
|
|
33
|
-
}
|
|
34
|
-
/** Inspect the database's applied IDs against the exact ordered registry prefix. */
|
|
35
|
-
function inspectLedgerAgainst(db, registryIds) {
|
|
36
|
-
if (!migrationLedgerExists(db))
|
|
37
|
-
return { status: registryIds.length === 0 ? "current" : "old", migrationIds: [] };
|
|
38
|
-
const rows = db.prepare("SELECT id FROM schema_migrations ORDER BY rowid").all();
|
|
39
|
-
const migrationIds = rows.map((row) => row.id);
|
|
40
|
-
for (const [index, row] of rows.entries()) {
|
|
41
|
-
const expectedId = registryIds[index];
|
|
42
|
-
// Every id this binary knows matched in order and the ledger carries more:
|
|
43
|
-
// the database was migrated by a newer akm. Nothing here is applicable —
|
|
44
|
-
// this binary's whole registry is already applied — so this is version
|
|
45
|
-
// skew, not divergence.
|
|
46
|
-
if (!expectedId) {
|
|
47
|
-
return {
|
|
48
|
-
status: "newer",
|
|
49
|
-
migrationIds,
|
|
50
|
-
detail: `applied migration ID${rows.length - registryIds.length === 1 ? "" : "s"} ${migrationIds
|
|
51
|
-
.slice(registryIds.length)
|
|
52
|
-
.join(", ")} unknown to this akm`,
|
|
53
|
-
};
|
|
54
|
-
}
|
|
55
|
-
// A mismatch at a position this binary has a migration for is divergence,
|
|
56
|
-
// whether or not the id is one this binary knows later: this binary's
|
|
57
|
-
// migration at `index` was never applied, and something else was.
|
|
58
|
-
if (row.id !== expectedId) {
|
|
59
|
-
return {
|
|
60
|
-
status: "inconsistent",
|
|
61
|
-
migrationIds,
|
|
62
|
-
detail: `migration ledger is not an exact ordered prefix at position ${index + 1} (found '${row.id}', expected '${expectedId}'). ` +
|
|
63
|
-
`Applied, in order: [${migrationIds.join(", ")}]. This akm's expected order: [${registryIds.join(", ")}].`,
|
|
64
|
-
};
|
|
65
|
-
}
|
|
66
|
-
}
|
|
67
|
-
return {
|
|
68
|
-
status: rows.length === registryIds.length ? "current" : "old",
|
|
69
|
-
migrationIds,
|
|
70
|
-
};
|
|
71
|
-
}
|
|
72
|
-
export function inspectMigrationLedger(db, migrations) {
|
|
73
|
-
assertMigrationRegistry(migrations);
|
|
74
|
-
return inspectLedgerAgainst(db, migrations.map((migration) => migration.id));
|
|
75
|
-
}
|
|
76
|
-
/**
|
|
77
|
-
* Reject only a ledger this binary cannot reason about at all.
|
|
78
|
-
*
|
|
79
|
-
* A `newer` ledger — an exact ordered prefix of this binary's registry plus
|
|
80
|
-
* migrations a later akm added — is NOT rejected. Two akm versions sharing one
|
|
81
|
-
* data directory is a supported deployment (a bundled CLI alongside a newer
|
|
82
|
-
* global install), and refusing the open bricked the older one for every
|
|
83
|
-
* command while protecting nothing: its entire registry is already applied, so
|
|
84
|
-
* it has no pending migration to run. Callers that want to tell an operator
|
|
85
|
-
* about the skew read {@link MigrationLedgerState.status}.
|
|
86
|
-
*
|
|
87
|
-
* A `inconsistent` ledger is different: this binary has a migration that was
|
|
88
|
-
* never applied and something else was applied in its place, so running the
|
|
89
|
-
* pending set could conflict with schema it cannot see. That still refuses.
|
|
90
|
-
*/
|
|
91
|
-
export function assertMigrationLedger(db, migrations) {
|
|
92
|
-
const state = inspectMigrationLedger(db, migrations);
|
|
93
|
-
if (state.status === "inconsistent") {
|
|
94
|
-
throw new Error(`Refusing a database whose migrations are not an exact ordered prefix: ${state.detail} ` +
|
|
95
|
-
"Applying this binary's missing migration now could run it against a schema a later migration already " +
|
|
96
|
-
"changed underneath it, which is a real risk of producing a wrong schema — not something akm can guess " +
|
|
97
|
-
"its way out of safely. This usually means the database was migrated by an incompatible akm build or " +
|
|
98
|
-
"fork, or schema_migrations was edited by hand. Restore this file from a backup taken before the " +
|
|
99
|
-
"divergence, or — if there is no backup and the data is not needed — delete it and let akm rebuild it " +
|
|
100
|
-
"from scratch (a derived index.db regenerates from your sources on the next 'akm index'; state.db loses " +
|
|
101
|
-
"durable history such as improve/proposal state and must be treated as a last resort).");
|
|
102
|
-
}
|
|
103
|
-
return state;
|
|
104
|
-
}
|
|
105
|
-
/**
|
|
106
|
-
* Create the migrations ledger table if it does not exist. Must be called
|
|
107
|
-
* unconditionally on every open so a fresh database bootstraps correctly.
|
|
108
|
-
*/
|
|
109
|
-
export function ensureMigrationsTable(db) {
|
|
110
|
-
db.exec(`
|
|
111
|
-
CREATE TABLE IF NOT EXISTS schema_migrations (
|
|
112
|
-
id TEXT PRIMARY KEY,
|
|
113
|
-
applied_at TEXT NOT NULL DEFAULT (datetime('now'))
|
|
114
|
-
);
|
|
115
|
-
`);
|
|
116
|
-
}
|
|
117
|
-
/**
|
|
118
|
-
* Apply every pending migration, normally one transaction per migration.
|
|
119
|
-
*
|
|
120
|
-
* Each migration is applied in its own transaction so a failure in migration N
|
|
121
|
-
* does not roll back already-applied migrations 1..N-1. The migration row is
|
|
122
|
-
* inserted after the DDL succeeds in the same transaction, so a crash rolls
|
|
123
|
-
* back both that migration's SQL and its ledger row. A caller may explicitly
|
|
124
|
-
* group the initial prefix when ledger initialization and dependent early
|
|
125
|
-
* migrations form one safety boundary.
|
|
126
|
-
*
|
|
127
|
-
* @param db The open SQLite database.
|
|
128
|
-
* @param migrations The module's ordered, append-only migration list.
|
|
129
|
-
* @param opts Migration execution options.
|
|
130
|
-
*/
|
|
131
|
-
export function runMigrations(db, migrations, opts) {
|
|
132
|
-
assertMigrationRegistry(migrations);
|
|
133
|
-
const lockedPrefixThrough = opts?.lockInitialMigrationPrefixThrough;
|
|
134
|
-
const lockedPrefixEnd = lockedPrefixThrough
|
|
135
|
-
? migrations.findIndex((migration) => migration.id === lockedPrefixThrough)
|
|
136
|
-
: -1;
|
|
137
|
-
if (lockedPrefixThrough && lockedPrefixEnd < 0) {
|
|
138
|
-
throw new Error(`Initial locked migration prefix ends at unknown migration ID ${lockedPrefixThrough}.`);
|
|
139
|
-
}
|
|
140
|
-
if (lockedPrefixEnd >= 0) {
|
|
141
|
-
withImmediateWriteLock(db, () => {
|
|
142
|
-
if (!migrationLedgerExists(db)) {
|
|
143
|
-
opts?.beforeLedgerInitializationLocked?.(db);
|
|
144
|
-
ensureMigrationsTable(db);
|
|
145
|
-
}
|
|
146
|
-
assertMigrationLedger(db, migrations);
|
|
147
|
-
for (const migration of migrations.slice(0, lockedPrefixEnd + 1)) {
|
|
148
|
-
const already = db.prepare("SELECT 1 FROM schema_migrations WHERE id = ?").get(migration.id);
|
|
149
|
-
if (already)
|
|
150
|
-
continue;
|
|
151
|
-
opts?.beforeMigration?.(migration);
|
|
152
|
-
opts?.beforeMigrationLocked?.(migration, db);
|
|
153
|
-
db.exec(migration.up);
|
|
154
|
-
db.prepare("INSERT INTO schema_migrations (id) VALUES (?)").run(migration.id);
|
|
155
|
-
}
|
|
156
|
-
});
|
|
157
|
-
assertMigrationLedger(db, migrations);
|
|
158
|
-
}
|
|
159
|
-
else if (migrationLedgerExists(db)) {
|
|
160
|
-
assertMigrationLedger(db, migrations);
|
|
161
|
-
}
|
|
162
|
-
else {
|
|
163
|
-
withImmediateWriteLock(db, () => {
|
|
164
|
-
if (migrationLedgerExists(db))
|
|
165
|
-
return;
|
|
166
|
-
opts?.beforeLedgerInitializationLocked?.(db);
|
|
167
|
-
ensureMigrationsTable(db);
|
|
168
|
-
});
|
|
169
|
-
assertMigrationLedger(db, migrations);
|
|
170
|
-
}
|
|
171
|
-
const appliedRows = db.prepare("SELECT id FROM schema_migrations ORDER BY rowid").all();
|
|
172
|
-
const applied = new Set(appliedRows.map((r) => r.id));
|
|
173
|
-
for (const migration of migrations) {
|
|
174
|
-
if (applied.has(migration.id))
|
|
175
|
-
continue;
|
|
176
|
-
opts?.beforeMigration?.(migration);
|
|
177
|
-
withImmediateWriteLock(db, () => {
|
|
178
|
-
// Re-check under the write lock. `applied` is a snapshot taken before the
|
|
179
|
-
// loop, so two processes bootstrapping the same fresh DB concurrently
|
|
180
|
-
// (both see existed=false) could each decide
|
|
181
|
-
// to apply migration N. The first commits; the second must not re-run the
|
|
182
|
-
// DDL and must not hit a UNIQUE violation on the ledger insert.
|
|
183
|
-
const already = db.prepare("SELECT 1 FROM schema_migrations WHERE id = ?").get(migration.id);
|
|
184
|
-
if (already)
|
|
185
|
-
return;
|
|
186
|
-
opts?.beforeMigrationLocked?.(migration, db);
|
|
187
|
-
db.exec(migration.up);
|
|
188
|
-
db.prepare("INSERT INTO schema_migrations (id) VALUES (?)").run(migration.id);
|
|
189
|
-
});
|
|
190
|
-
applied.add(migration.id);
|
|
191
|
-
}
|
|
192
|
-
}
|
|
193
|
-
/** Attempts to acquire the write lock before giving up to the caller. */
|
|
194
|
-
const IMMEDIATE_LOCK_MAX_ATTEMPTS = 5;
|
|
195
|
-
function isRetryableImmediateBeginError(error) {
|
|
196
|
-
const message = (error instanceof Error ? error.message : String(error)).toLowerCase();
|
|
197
|
-
return (message.includes("database is locked") ||
|
|
198
|
-
message.includes("database table is locked") ||
|
|
199
|
-
message.includes("did not open a transaction"));
|
|
200
|
-
}
|
|
201
|
-
function sleepImmediateRetry(ms) {
|
|
202
|
-
if (ms <= 0)
|
|
203
|
-
return;
|
|
204
|
-
sleepSync(ms);
|
|
205
|
-
}
|
|
206
|
-
/**
|
|
207
|
-
* Run `fn` inside a `BEGIN IMMEDIATE` transaction.
|
|
208
|
-
*
|
|
209
|
-
* The write lock is taken up front rather than upgraded from a read lock, so a
|
|
210
|
-
* second process bootstrapping the same database WAITS for the first to commit
|
|
211
|
-
* instead of racing it. `db.transaction()` opens a DEFERRED transaction, which
|
|
212
|
-
* only takes the write lock on first write — leaving the read-then-write gap
|
|
213
|
-
* this guards.
|
|
214
|
-
*
|
|
215
|
-
* Deliberately local rather than reusing `withImmediateTransaction` from
|
|
216
|
-
* core/state-db: that module imports this one, so the dependency cannot be
|
|
217
|
-
* pointed the other way.
|
|
218
|
-
*/
|
|
219
|
-
function withImmediateWriteLock(db, fn) {
|
|
220
|
-
if (db.inTransaction) {
|
|
221
|
-
fn();
|
|
222
|
-
return;
|
|
223
|
-
}
|
|
224
|
-
let lastBeginErr;
|
|
225
|
-
for (let attempt = 1; attempt <= IMMEDIATE_LOCK_MAX_ATTEMPTS; attempt++) {
|
|
226
|
-
try {
|
|
227
|
-
db.exec("BEGIN IMMEDIATE");
|
|
228
|
-
if (!db.inTransaction) {
|
|
229
|
-
throw new Error("BEGIN IMMEDIATE did not open a transaction (phantom contention state)");
|
|
230
|
-
}
|
|
231
|
-
}
|
|
232
|
-
catch (err) {
|
|
233
|
-
lastBeginErr = err;
|
|
234
|
-
if (isRetryableImmediateBeginError(err) && attempt < IMMEDIATE_LOCK_MAX_ATTEMPTS) {
|
|
235
|
-
if (db.inTransaction) {
|
|
236
|
-
try {
|
|
237
|
-
db.exec("ROLLBACK");
|
|
238
|
-
}
|
|
239
|
-
catch {
|
|
240
|
-
// Transaction already gone; retrying is safe because fn has not run.
|
|
241
|
-
}
|
|
242
|
-
}
|
|
243
|
-
sleepImmediateRetry(2 ** (attempt - 1));
|
|
244
|
-
continue;
|
|
245
|
-
}
|
|
246
|
-
throw err;
|
|
247
|
-
}
|
|
248
|
-
try {
|
|
249
|
-
fn();
|
|
250
|
-
if (!db.inTransaction) {
|
|
251
|
-
throw new Error("Migration write lock invariant violated: transaction opened by BEGIN IMMEDIATE was no longer active after the migration body ran; refusing to COMMIT (writes may have escaped serialization)");
|
|
252
|
-
}
|
|
253
|
-
db.exec("COMMIT");
|
|
254
|
-
return;
|
|
255
|
-
}
|
|
256
|
-
catch (err) {
|
|
257
|
-
if (db.inTransaction) {
|
|
258
|
-
try {
|
|
259
|
-
db.exec("ROLLBACK");
|
|
260
|
-
}
|
|
261
|
-
catch {
|
|
262
|
-
// Already rolled back by SQLite (e.g. the statement aborted the txn).
|
|
263
|
-
}
|
|
264
|
-
}
|
|
265
|
-
throw err;
|
|
266
|
-
}
|
|
267
|
-
}
|
|
268
|
-
throw lastBeginErr instanceof Error
|
|
269
|
-
? lastBeginErr
|
|
270
|
-
: new Error(`could not acquire the migration write lock after ${IMMEDIATE_LOCK_MAX_ATTEMPTS} attempts`);
|
|
271
|
-
}
|
|
@@ -1,107 +0,0 @@
|
|
|
1
|
-
// This Source Code Form is subject to the terms of the Mozilla Public
|
|
2
|
-
// License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
3
|
-
// file, You can obtain one at https://mozilla.org/MPL/2.0/.
|
|
4
|
-
/** Insert a freshly minted canary set (all rows active, one shared set id). */
|
|
5
|
-
export function insertCanaries(db, canarySetId, canaries, now) {
|
|
6
|
-
if (canaries.length === 0)
|
|
7
|
-
return;
|
|
8
|
-
const ts = now ?? new Date().toISOString();
|
|
9
|
-
const stmt = db.prepare(`
|
|
10
|
-
INSERT INTO canary_queries (canary_set_id, anchor_ref, query, source, active, created_at)
|
|
11
|
-
VALUES (?, ?, ?, ?, 1, ?)
|
|
12
|
-
`);
|
|
13
|
-
db.transaction(() => {
|
|
14
|
-
for (const c of canaries) {
|
|
15
|
-
stmt.run(canarySetId, c.anchorRef, c.query, c.source ?? "auto", ts);
|
|
16
|
-
}
|
|
17
|
-
})();
|
|
18
|
-
}
|
|
19
|
-
/** Load the active canary set (empty array = never minted). */
|
|
20
|
-
export function getActiveCanaries(db) {
|
|
21
|
-
// Scope to the NEWEST active set: if an interrupted refresh (or a bug) ever
|
|
22
|
-
// leaves two sets active, mixing their rows would silently corrupt the
|
|
23
|
-
// recall/entropy trend baselines. The newest set wins; stale-active rows are
|
|
24
|
-
// simply never returned.
|
|
25
|
-
return db
|
|
26
|
-
.prepare(`SELECT * FROM canary_queries
|
|
27
|
-
WHERE active = 1 AND canary_set_id = (
|
|
28
|
-
SELECT canary_set_id FROM canary_queries WHERE active = 1
|
|
29
|
-
ORDER BY created_at DESC, id DESC LIMIT 1
|
|
30
|
-
)
|
|
31
|
-
ORDER BY id`)
|
|
32
|
-
.all();
|
|
33
|
-
}
|
|
34
|
-
/** Load one canary set's rows by its exact set id (any active state), insertion order. */
|
|
35
|
-
export function getCanariesBySetId(db, canarySetId) {
|
|
36
|
-
return db
|
|
37
|
-
.prepare(`SELECT * FROM canary_queries WHERE canary_set_id = ? ORDER BY id`)
|
|
38
|
-
.all(canarySetId);
|
|
39
|
-
}
|
|
40
|
-
/** List every distinct canary_set_id that still has active rows. */
|
|
41
|
-
export function listActiveCanarySetIds(db) {
|
|
42
|
-
const rows = db.prepare(`SELECT DISTINCT canary_set_id FROM canary_queries WHERE active = 1`).all();
|
|
43
|
-
return rows.map((r) => r.canary_set_id);
|
|
44
|
-
}
|
|
45
|
-
/**
|
|
46
|
-
* Deactivate every canary row in a set. Rows are RETAINED (active = 0) so
|
|
47
|
-
* historical improve_cycle_metrics rows keyed on the old canary_set_id stay
|
|
48
|
-
* interpretable; only `bun scripts/refresh-canary-set.ts --refresh` calls this.
|
|
49
|
-
*/
|
|
50
|
-
export function deactivateCanarySet(db, canarySetId) {
|
|
51
|
-
const result = db
|
|
52
|
-
.prepare(`UPDATE canary_queries SET active = 0 WHERE canary_set_id = ? AND active = 1`)
|
|
53
|
-
.run(canarySetId);
|
|
54
|
-
const changes = result.changes ?? 0;
|
|
55
|
-
return typeof changes === "bigint" ? Number(changes) : changes;
|
|
56
|
-
}
|
|
57
|
-
/** Persist one qualifying cycle's store-health snapshot. */
|
|
58
|
-
export function insertCycleMetrics(db, row) {
|
|
59
|
-
db.prepare(`
|
|
60
|
-
INSERT INTO improve_cycle_metrics
|
|
61
|
-
(run_id, ts, pass, canary_set_id, mean_recall, mean_ndcg, mean_mrr,
|
|
62
|
-
canary_ranks_json, store_total, store_by_type_json, distinct_content_ratio,
|
|
63
|
-
mean_bigram_diversity, over_generation_count, accepted_actions,
|
|
64
|
-
merge_floor_violations, alerts_json)
|
|
65
|
-
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
66
|
-
`).run(row.run_id, row.ts, row.pass, row.canary_set_id, row.mean_recall, row.mean_ndcg, row.mean_mrr, row.canary_ranks_json, row.store_total, row.store_by_type_json, row.distinct_content_ratio, row.mean_bigram_diversity, row.over_generation_count, row.accepted_actions, row.merge_floor_violations, row.alerts_json);
|
|
67
|
-
}
|
|
68
|
-
/**
|
|
69
|
-
* Load the most recent cycle rows for one canary set, OLDEST-first (the alert
|
|
70
|
-
* evaluator's window order). Scoped by canary_set_id so trends never compare
|
|
71
|
-
* across canary re-mints.
|
|
72
|
-
*/
|
|
73
|
-
export function queryRecentCycleMetrics(db, canarySetId, limit) {
|
|
74
|
-
const rows = db
|
|
75
|
-
.prepare(`SELECT run_id, ts, pass, canary_set_id, mean_recall, mean_ndcg, mean_mrr,
|
|
76
|
-
canary_ranks_json, store_total, store_by_type_json, distinct_content_ratio,
|
|
77
|
-
mean_bigram_diversity, over_generation_count, accepted_actions,
|
|
78
|
-
merge_floor_violations, alerts_json
|
|
79
|
-
FROM improve_cycle_metrics WHERE canary_set_id = ?
|
|
80
|
-
ORDER BY ts DESC, id DESC LIMIT ?`)
|
|
81
|
-
.all(canarySetId, Math.max(0, limit));
|
|
82
|
-
return rows.reverse();
|
|
83
|
-
}
|
|
84
|
-
/** Load the single most recent cycle row across all canary sets (health surface). */
|
|
85
|
-
export function getLatestCycleMetrics(db) {
|
|
86
|
-
const row = db
|
|
87
|
-
.prepare(`SELECT run_id, ts, pass, canary_set_id, mean_recall, mean_ndcg, mean_mrr,
|
|
88
|
-
canary_ranks_json, store_total, store_by_type_json, distinct_content_ratio,
|
|
89
|
-
mean_bigram_diversity, over_generation_count, accepted_actions,
|
|
90
|
-
merge_floor_violations, alerts_json
|
|
91
|
-
FROM improve_cycle_metrics ORDER BY ts DESC, id DESC LIMIT 1`)
|
|
92
|
-
.get();
|
|
93
|
-
return row == null ? undefined : row;
|
|
94
|
-
}
|
|
95
|
-
/**
|
|
96
|
-
* Delete cycle rows older than `retentionDays` (default 365 — owner-approved;
|
|
97
|
-
* a slow collapse needs a longer trend window than the 90-day events log).
|
|
98
|
-
* Returns the purged row count. canary_queries rows are never purged.
|
|
99
|
-
*/
|
|
100
|
-
export function purgeOldCycleMetrics(db, retentionDays = 365) {
|
|
101
|
-
if (!Number.isFinite(retentionDays) || retentionDays <= 0)
|
|
102
|
-
return 0;
|
|
103
|
-
const cutoff = new Date(Date.now() - retentionDays * 86_400_000).toISOString();
|
|
104
|
-
const result = db.prepare("DELETE FROM improve_cycle_metrics WHERE ts < ?").run(cutoff);
|
|
105
|
-
const changes = result.changes ?? 0;
|
|
106
|
-
return typeof changes === "bigint" ? Number(changes) : changes;
|
|
107
|
-
}
|
|
@@ -1,184 +0,0 @@
|
|
|
1
|
-
// This Source Code Form is subject to the terms of the Mozilla Public
|
|
2
|
-
// License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
3
|
-
// file, You can obtain one at https://mozilla.org/MPL/2.0/.
|
|
4
|
-
/**
|
|
5
|
-
* `index.db` embedding salvage (#955) — a transient, self-emptying table
|
|
6
|
-
* that lets a full rebuild or an index-generation bump reuse vectors instead
|
|
7
|
-
* of re-embedding a corpus whose content did not change.
|
|
8
|
-
*
|
|
9
|
-
* Zero steady-state cost by design: this is NOT a second embedding cache.
|
|
10
|
-
* Rows are copied aside only at the moment they would otherwise be discarded
|
|
11
|
-
* wholesale — a full-index wipe (`persistDirRecords`) or a generation bump
|
|
12
|
-
* (`rebuildIncompatibleIndexGeneration`) — and are consumed by the very next
|
|
13
|
-
* embedding pass (`generateEmbeddingsForDb`). A pass that completes without
|
|
14
|
-
* abort or circuit-break purges whatever is left; an interrupted pass leaves
|
|
15
|
-
* the table for the next attempt to pick up.
|
|
16
|
-
*
|
|
17
|
-
* Reuse is keyed on `sha256(search_text)` plus the fingerprint the vector was
|
|
18
|
-
* generated under — a fingerprint mismatch or a single-byte content change
|
|
19
|
-
* both correctly fall through to a real provider call. `content_hash` is the
|
|
20
|
-
* PRIMARY KEY (not `(content_hash, fingerprint)`) so relabeling a whole
|
|
21
|
-
* generation's fingerprint after a canary "keep" verdict is one UPDATE, and a
|
|
22
|
-
* hash colliding across two discards simply keeps the most recent copy —
|
|
23
|
-
* salvage is a best-effort optimization, not a durable multi-generation
|
|
24
|
-
* archive.
|
|
25
|
-
*/
|
|
26
|
-
import { createHash } from "node:crypto";
|
|
27
|
-
import { blobToEmbedding } from "./embeddings-repository.js";
|
|
28
|
-
import { getMeta } from "./index-meta-repository.js";
|
|
29
|
-
import { SQLITE_CHUNK_SIZE } from "./index-sql.js";
|
|
30
|
-
/**
|
|
31
|
-
* Create the salvage table. Additive-only DDL: it carries no bearing on the
|
|
32
|
-
* `entries` generation fingerprint (`hasCanonicalEntrySchema`), so adding it
|
|
33
|
-
* does not require an index-generation bump.
|
|
34
|
-
*/
|
|
35
|
-
export function ensureEmbeddingSalvageTable(db) {
|
|
36
|
-
db.exec(`
|
|
37
|
-
CREATE TABLE IF NOT EXISTS embedding_salvage (
|
|
38
|
-
content_hash TEXT PRIMARY KEY,
|
|
39
|
-
fingerprint TEXT NOT NULL,
|
|
40
|
-
embedding BLOB NOT NULL,
|
|
41
|
-
salvaged_at TEXT NOT NULL
|
|
42
|
-
);
|
|
43
|
-
`);
|
|
44
|
-
}
|
|
45
|
-
/** The one hash function salvage writes and reuse lookups must agree on. */
|
|
46
|
-
export function hashEmbeddableText(searchText) {
|
|
47
|
-
return createHash("sha256").update(searchText, "utf8").digest("hex");
|
|
48
|
-
}
|
|
49
|
-
function tableExists(db, name) {
|
|
50
|
-
return db.prepare("SELECT 1 FROM sqlite_master WHERE type='table' AND name=?").get(name) != null;
|
|
51
|
-
}
|
|
52
|
-
function tableHasColumn(db, table, column) {
|
|
53
|
-
const columns = db.prepare(`PRAGMA table_info(${table})`).all();
|
|
54
|
-
return columns.some((c) => c.name === column);
|
|
55
|
-
}
|
|
56
|
-
/**
|
|
57
|
-
* Copy every (hash of search_text, embedding) pair about to be discarded
|
|
58
|
-
* wholesale into `embedding_salvage`, tagged with the `embeddingFingerprint`
|
|
59
|
-
* the discarded vectors were generated under. The caller MUST run this
|
|
60
|
-
* inside the same transaction as the discard that follows it, so the copy
|
|
61
|
-
* and the delete commit or roll back together.
|
|
62
|
-
*
|
|
63
|
-
* Streams `entries JOIN embeddings` in id-ordered pages of
|
|
64
|
-
* {@link SQLITE_CHUNK_SIZE} instead of loading every row into memory before
|
|
65
|
-
* hashing anything — a full rebuild of a large stash otherwise held the
|
|
66
|
-
* entire corpus's search text and vectors in memory at once just to copy
|
|
67
|
-
* them aside (#955, field-report follow-up).
|
|
68
|
-
*
|
|
69
|
-
* A no-op (returns 0) when there is no stored `embeddingFingerprint` to tag
|
|
70
|
-
* rows with (nothing was ever verified against a provider, so there is
|
|
71
|
-
* nothing worth reusing later) or the generation being discarded predates
|
|
72
|
-
* the `entries.search_text` column or has no `embeddings` table at all — an
|
|
73
|
-
* older generation than that has nothing this can safely read.
|
|
74
|
-
*/
|
|
75
|
-
export function salvageEmbeddingsBeforeDiscard(db) {
|
|
76
|
-
const fingerprint = getMeta(db, "embeddingFingerprint");
|
|
77
|
-
if (!fingerprint)
|
|
78
|
-
return 0;
|
|
79
|
-
if (!tableExists(db, "entries") || !tableExists(db, "embeddings"))
|
|
80
|
-
return 0;
|
|
81
|
-
if (!tableHasColumn(db, "entries", "search_text"))
|
|
82
|
-
return 0;
|
|
83
|
-
const page = db.prepare("SELECT e.id AS id, e.search_text AS searchText, em.embedding AS embedding " +
|
|
84
|
-
"FROM entries e JOIN embeddings em ON em.id = e.id WHERE e.id > ? ORDER BY e.id LIMIT ?");
|
|
85
|
-
const insert = db.prepare("INSERT OR REPLACE INTO embedding_salvage (content_hash, fingerprint, embedding, salvaged_at) VALUES (?, ?, ?, ?)");
|
|
86
|
-
const salvagedAt = new Date().toISOString();
|
|
87
|
-
let lastId = 0;
|
|
88
|
-
let total = 0;
|
|
89
|
-
for (;;) {
|
|
90
|
-
const rows = page.all(lastId, SQLITE_CHUNK_SIZE);
|
|
91
|
-
if (rows.length === 0)
|
|
92
|
-
break;
|
|
93
|
-
for (const row of rows) {
|
|
94
|
-
insert.run(hashEmbeddableText(row.searchText), fingerprint, row.embedding, salvagedAt);
|
|
95
|
-
}
|
|
96
|
-
total += rows.length;
|
|
97
|
-
lastId = rows[rows.length - 1]?.id ?? lastId;
|
|
98
|
-
if (rows.length < SQLITE_CHUNK_SIZE)
|
|
99
|
-
break;
|
|
100
|
-
}
|
|
101
|
-
return total;
|
|
102
|
-
}
|
|
103
|
-
/**
|
|
104
|
-
* Remove every salvage row. Called after an embedding pass completes without
|
|
105
|
-
* abort or circuit-break (the salvaged generation has now either been reused
|
|
106
|
-
* or superseded), and by `--reembed` / a canary "rebuild" verdict (the
|
|
107
|
-
* salvaged vectors belong to a different model and are never reusable).
|
|
108
|
-
*/
|
|
109
|
-
export function purgeEmbeddingSalvage(db) {
|
|
110
|
-
db.exec("DELETE FROM embedding_salvage");
|
|
111
|
-
}
|
|
112
|
-
/**
|
|
113
|
-
* A canary "keep" verdict means the model did not actually change — only its
|
|
114
|
-
* fingerprint STRING did (e.g. a gateway rename). Salvage rows tagged with
|
|
115
|
-
* the old string are still valid vectors; rewrite them to the new string so
|
|
116
|
-
* they remain reusable instead of silently going stale.
|
|
117
|
-
*/
|
|
118
|
-
export function relabelEmbeddingSalvageFingerprint(db, fromFingerprint, toFingerprint) {
|
|
119
|
-
db.prepare("UPDATE embedding_salvage SET fingerprint = ? WHERE fingerprint = ?").run(toFingerprint, fromFingerprint);
|
|
120
|
-
}
|
|
121
|
-
/**
|
|
122
|
-
* Reuse salvaged vectors for `entries` whose `searchText` hash matches a
|
|
123
|
-
* salvage row tagged with the CURRENT `fingerprint` — never across
|
|
124
|
-
* fingerprints, and never when `search_text` differs by even one byte (the
|
|
125
|
-
* hash is exact-match only, by design). Matches are written via
|
|
126
|
-
* `writeReused` in chunks of {@link SQLITE_CHUNK_SIZE}, each its own
|
|
127
|
-
* transaction, mirroring the main pass's per-batch commit (#955) so an
|
|
128
|
-
* interruption partway through the reuse step keeps whatever already wrote.
|
|
129
|
-
*
|
|
130
|
-
* The steady state of every ordinary run is an EMPTY salvage table (nothing
|
|
131
|
-
* was just discarded), so this checks that first with one indexed lookup —
|
|
132
|
-
* `SELECT 1 ... LIMIT 1` — before hashing a single pending entry. Hashing
|
|
133
|
-
* every entry up front to look up a table that is empty 100% of the time
|
|
134
|
-
* outside a rebuild was pure wasted work on the common path (#955,
|
|
135
|
-
* field-report follow-up).
|
|
136
|
-
*/
|
|
137
|
-
export function reuseSalvagedEmbeddings(db, entries, fingerprint, writeReused) {
|
|
138
|
-
if (entries.length === 0)
|
|
139
|
-
return { reusedCount: 0, remaining: [] };
|
|
140
|
-
const anySalvageForFingerprint = db
|
|
141
|
-
.prepare("SELECT 1 FROM embedding_salvage WHERE fingerprint = ? LIMIT 1")
|
|
142
|
-
.get(fingerprint);
|
|
143
|
-
if (!anySalvageForFingerprint)
|
|
144
|
-
return { reusedCount: 0, remaining: [...entries] };
|
|
145
|
-
const hashes = entries.map((entry) => hashEmbeddableText(entry.searchText));
|
|
146
|
-
const salvageByHash = new Map();
|
|
147
|
-
const uniqueHashes = [...new Set(hashes)];
|
|
148
|
-
for (let offset = 0; offset < uniqueHashes.length; offset += SQLITE_CHUNK_SIZE) {
|
|
149
|
-
const chunk = uniqueHashes.slice(offset, offset + SQLITE_CHUNK_SIZE);
|
|
150
|
-
const placeholders = chunk.map(() => "?").join(",");
|
|
151
|
-
const rows = db
|
|
152
|
-
.prepare(`SELECT content_hash AS contentHash, embedding FROM embedding_salvage WHERE fingerprint = ? AND content_hash IN (${placeholders})`)
|
|
153
|
-
.all(fingerprint, ...chunk);
|
|
154
|
-
for (const row of rows)
|
|
155
|
-
salvageByHash.set(row.contentHash, row.embedding);
|
|
156
|
-
}
|
|
157
|
-
if (salvageByHash.size === 0)
|
|
158
|
-
return { reusedCount: 0, remaining: [...entries] };
|
|
159
|
-
let reusedCount = 0;
|
|
160
|
-
const remaining = [];
|
|
161
|
-
for (let offset = 0; offset < entries.length; offset += SQLITE_CHUNK_SIZE) {
|
|
162
|
-
const end = Math.min(offset + SQLITE_CHUNK_SIZE, entries.length);
|
|
163
|
-
const chunkMatches = [];
|
|
164
|
-
for (let i = offset; i < end; i++) {
|
|
165
|
-
const entry = entries[i];
|
|
166
|
-
const blob = salvageByHash.get(hashes[i]);
|
|
167
|
-
if (blob)
|
|
168
|
-
chunkMatches.push({ entry, blob });
|
|
169
|
-
else
|
|
170
|
-
remaining.push(entry);
|
|
171
|
-
}
|
|
172
|
-
if (chunkMatches.length === 0)
|
|
173
|
-
continue;
|
|
174
|
-
db.transaction(() => {
|
|
175
|
-
for (const { entry, blob } of chunkMatches) {
|
|
176
|
-
if (writeReused(entry, blobToEmbedding(blob)))
|
|
177
|
-
reusedCount++;
|
|
178
|
-
else
|
|
179
|
-
remaining.push(entry);
|
|
180
|
-
}
|
|
181
|
-
})();
|
|
182
|
-
}
|
|
183
|
-
return { reusedCount, remaining };
|
|
184
|
-
}
|