akm-cli 0.9.0-beta.6 → 0.9.0-rc.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +663 -0
- package/README.md +12 -4
- package/dist/akm +38 -0
- package/dist/akm-migrate-storage +38 -0
- package/dist/assets/help/help-improve.md +9 -6
- package/dist/assets/hints/cli-hints-full.md +6 -5
- package/dist/assets/profiles/default.json +9 -4
- package/dist/assets/profiles/frequent.json +1 -1
- package/dist/assets/profiles/memory-focus.json +1 -1
- package/dist/assets/profiles/proactive-maintenance.json +25 -0
- package/dist/assets/profiles/quick.json +1 -1
- package/dist/assets/profiles/recombine-only.json +21 -0
- package/dist/assets/profiles/reflect-distill.json +30 -0
- package/dist/assets/profiles/synthesize.json +15 -0
- package/dist/assets/profiles/thorough.json +1 -1
- package/dist/assets/prompts/consolidate-system.md +23 -0
- package/dist/assets/prompts/contradiction-judge.md +33 -0
- package/dist/assets/prompts/distill-knowledge-system.md +22 -0
- package/dist/assets/prompts/distill-lesson-system.md +36 -0
- package/dist/assets/prompts/extract-session.md +11 -3
- package/dist/assets/prompts/graph-extract-system.md +1 -0
- package/dist/assets/prompts/graph-extract-user-prompt.md +1 -1
- package/dist/assets/prompts/memory-infer-system.md +1 -0
- package/dist/assets/prompts/memory-infer-user.md +5 -0
- package/dist/assets/prompts/metadata-enhance-system.md +1 -0
- package/dist/assets/prompts/procedural-system.md +44 -0
- package/dist/assets/prompts/recombine-system.md +40 -0
- package/dist/assets/prompts/staleness-detect-system.md +6 -0
- package/dist/assets/prompts/validate-summary-judge.md +1 -0
- package/dist/assets/stash-skeleton/facts/conventions/assets/agent.md +38 -0
- package/dist/assets/stash-skeleton/facts/conventions/assets/command.md +38 -0
- package/dist/assets/stash-skeleton/facts/conventions/assets/fact.md +39 -0
- package/dist/assets/stash-skeleton/facts/conventions/assets/knowledge.md +40 -0
- package/dist/assets/stash-skeleton/facts/conventions/assets/lesson.md +43 -0
- package/dist/assets/stash-skeleton/facts/conventions/assets/memory.md +38 -0
- package/dist/assets/stash-skeleton/facts/conventions/assets/script.md +43 -0
- package/dist/assets/stash-skeleton/facts/conventions/assets/skill.md +40 -0
- package/dist/assets/stash-skeleton/facts/conventions/assets/workflow.md +43 -0
- package/dist/assets/templates/html/health.html +281 -111
- package/dist/assets/wiki/ingest-workflow-template.md +45 -16
- package/dist/assets/wiki/schema-template.md +4 -4
- package/dist/cli/clack.js +56 -0
- package/dist/cli/config-migrate.js +7 -1
- package/dist/cli/confirm.js +1 -1
- package/dist/cli/parse-args.js +46 -1
- package/dist/cli/shared.js +28 -0
- package/dist/cli.js +25 -14
- package/dist/commands/agent/agent-dispatch.js +3 -2
- package/dist/commands/agent/agent-support.js +0 -7
- package/dist/commands/agent/contribute-cli.js +26 -7
- package/dist/commands/config-cli.js +26 -13
- package/dist/commands/env/child-env.js +47 -0
- package/dist/commands/env/env-cli.js +220 -227
- package/dist/commands/env/env.js +14 -67
- package/dist/commands/env/secret-cli.js +140 -138
- package/dist/commands/feedback-cli.js +153 -147
- package/dist/commands/graph/graph-cli.js +5 -13
- package/dist/commands/graph/graph.js +76 -72
- package/dist/commands/health/advisories.js +151 -0
- package/dist/commands/health/checks.js +103 -16
- package/dist/commands/health/html-report.js +447 -81
- package/dist/commands/health/improve-metrics.js +771 -0
- package/dist/commands/health/llm-usage.js +65 -0
- package/dist/commands/health/md-report.js +103 -0
- package/dist/commands/health/metrics.js +278 -0
- package/dist/commands/health/stash-exposure.js +46 -0
- package/dist/commands/health/surfaces.js +216 -0
- package/dist/commands/health/task-runs.js +135 -0
- package/dist/commands/health/types.js +26 -0
- package/dist/commands/health/windows.js +195 -0
- package/dist/commands/health.js +91 -1083
- package/dist/commands/improve/anti-collapse.js +170 -0
- package/dist/commands/improve/calibration.js +161 -0
- package/dist/commands/improve/collapse-detector.js +421 -0
- package/dist/commands/improve/consolidate/chunking.js +141 -0
- package/dist/commands/improve/consolidate/eligibility.js +64 -0
- package/dist/commands/improve/consolidate/merge.js +145 -0
- package/dist/commands/improve/consolidate/sanitize.js +231 -0
- package/dist/commands/{lint.js → improve/consolidate/types.js} +1 -1
- package/dist/commands/improve/consolidate.js +1313 -1278
- package/dist/commands/improve/dedup.js +482 -0
- package/dist/commands/improve/distill/content-repair.js +202 -0
- package/dist/commands/improve/distill/promote-memory.js +229 -0
- package/dist/commands/improve/distill/quality-gate.js +236 -0
- package/dist/commands/improve/distill-guards.js +127 -0
- package/dist/commands/improve/distill-promotion-policy.js +826 -167
- package/dist/commands/improve/distill.js +243 -599
- package/dist/commands/improve/eligibility.js +434 -0
- package/dist/commands/improve/encoding-salience.js +205 -0
- package/dist/commands/improve/extract-cli.js +179 -59
- package/dist/commands/improve/extract-prompt.js +55 -4
- package/dist/commands/improve/extract-watch.js +140 -0
- package/dist/commands/improve/extract.js +409 -43
- package/dist/commands/improve/feedback-valence.js +54 -0
- package/dist/commands/improve/hot-probation.js +45 -0
- package/dist/commands/improve/improve-auto-accept.js +160 -7
- package/dist/commands/improve/improve-cli.js +115 -73
- package/dist/commands/improve/improve-profiles.js +32 -8
- package/dist/commands/improve/improve-result-file.js +15 -25
- package/dist/commands/improve/improve-session.js +58 -0
- package/dist/commands/improve/improve.js +510 -2537
- package/dist/commands/improve/locks.js +154 -0
- package/dist/commands/improve/loop-stages.js +1100 -0
- package/dist/commands/improve/memory/memory-belief.js +14 -15
- package/dist/commands/improve/memory/memory-contradiction-detect.js +83 -60
- package/dist/commands/improve/memory/memory-improve.js +27 -27
- package/dist/commands/improve/outcome-loop.js +270 -0
- package/dist/commands/improve/preparation.js +2002 -0
- package/dist/commands/improve/proactive-maintenance.js +115 -0
- package/dist/commands/improve/procedural.js +398 -0
- package/dist/commands/improve/recombine.js +818 -0
- package/dist/commands/improve/reflect-noise.js +0 -0
- package/dist/commands/improve/reflect.js +212 -45
- package/dist/commands/improve/salience.js +455 -0
- package/dist/commands/improve/schema-similarity-gate.js +168 -0
- package/dist/commands/improve/shared.js +51 -0
- package/dist/commands/improve/triage.js +93 -0
- package/dist/commands/lint/agent-linter.js +19 -24
- package/dist/commands/lint/base-linter.js +173 -60
- package/dist/commands/lint/command-linter.js +19 -24
- package/dist/commands/lint/env-key-rules.js +38 -1
- package/dist/commands/lint/fact-linter.js +39 -0
- package/dist/commands/lint/index.js +31 -13
- package/dist/commands/lint/memory-linter.js +1 -1
- package/dist/commands/lint/registry.js +7 -2
- package/dist/commands/lint/task-linter.js +3 -3
- package/dist/commands/lint/workflow-linter.js +26 -1
- package/dist/commands/observability-cli.js +4 -4
- package/dist/commands/proposal/drain-policies.js +13 -4
- package/dist/commands/proposal/drain.js +45 -51
- package/dist/commands/proposal/legacy-import.js +115 -0
- package/dist/commands/proposal/proposal-cli.js +24 -34
- package/dist/commands/proposal/proposal.js +7 -1
- package/dist/commands/proposal/propose.js +8 -3
- package/dist/commands/proposal/repository.js +829 -0
- package/dist/commands/proposal/validators/proposal-quality-validators.js +9 -8
- package/dist/commands/proposal/validators/proposals.js +93 -882
- package/dist/commands/read/curate.js +419 -103
- package/dist/commands/read/knowledge.js +10 -3
- package/dist/commands/read/remember-cli.js +133 -138
- package/dist/commands/read/search-cli.js +15 -8
- package/dist/commands/read/search.js +22 -11
- package/dist/commands/read/show.js +106 -14
- package/dist/commands/registry-cli.js +76 -87
- package/dist/commands/remember.js +11 -12
- package/dist/commands/sources/add-cli.js +91 -95
- package/dist/commands/sources/history.js +1 -1
- package/dist/commands/sources/init.js +66 -18
- package/dist/commands/sources/installed-stashes.js +11 -3
- package/dist/commands/sources/schema-repair.js +44 -46
- package/dist/commands/sources/self-update.js +2 -2
- package/dist/commands/sources/source-add.js +7 -3
- package/dist/commands/sources/sources-cli.js +3 -3
- package/dist/commands/sources/stash-cli.js +29 -41
- package/dist/commands/sources/stash-skeleton.js +57 -8
- package/dist/commands/tasks/default-tasks.js +15 -2
- package/dist/commands/tasks/tasks-cli.js +20 -29
- package/dist/commands/tasks/tasks.js +39 -11
- package/dist/commands/wiki-cli.js +23 -38
- package/dist/commands/workflow-cli.js +15 -1
- package/dist/core/asset/asset-registry.js +3 -1
- package/dist/core/asset/asset-spec.js +21 -4
- package/dist/core/asset/frontmatter.js +188 -167
- package/dist/core/asset/markdown.js +8 -0
- package/dist/core/authoring-rules.js +92 -0
- package/dist/core/common.js +4 -23
- package/dist/core/concurrent.js +10 -1
- package/dist/core/config/config-io.js +10 -1
- package/dist/core/config/config-migration.js +18 -40
- package/dist/core/config/config-schema.js +389 -58
- package/dist/core/config/config-types.js +3 -3
- package/dist/core/config/config.js +67 -22
- package/dist/core/deep-merge.js +38 -0
- package/dist/core/errors.js +1 -0
- package/dist/core/eval/rank-metrics.js +113 -0
- package/dist/core/events.js +4 -7
- package/dist/core/improve-types.js +47 -8
- package/dist/core/logs-db.js +14 -75
- package/dist/core/parse.js +36 -16
- package/dist/core/paths.js +21 -18
- package/dist/core/standards/resolve-standards-context.js +87 -0
- package/dist/core/standards/resolve-stash-standards.js +99 -0
- package/dist/core/standards/resolve-type-conventions.js +66 -0
- package/dist/core/state/migrations.js +770 -0
- package/dist/core/state-db.js +142 -1091
- package/dist/core/structured.js +69 -0
- package/dist/core/time.js +53 -0
- package/dist/core/warn.js +21 -0
- package/dist/core/write-source.js +37 -0
- package/dist/indexer/db/db.js +356 -780
- package/dist/indexer/db/entry-mapper.js +41 -0
- package/dist/indexer/db/graph-db.js +129 -86
- package/dist/indexer/db/llm-cache.js +2 -2
- package/dist/indexer/db/schema.js +516 -0
- package/dist/indexer/ensure-index.js +103 -24
- package/dist/indexer/feedback/utility-policy.js +75 -0
- package/dist/indexer/graph/graph-boost.js +51 -41
- package/dist/indexer/graph/graph-extraction.js +207 -4
- package/dist/indexer/index-writer-lock.js +106 -0
- package/dist/indexer/index-written-assets.js +105 -0
- package/dist/indexer/indexer.js +291 -310
- package/dist/indexer/passes/dir-staleness.js +114 -0
- package/dist/indexer/passes/memory-inference.js +13 -5
- package/dist/indexer/passes/metadata.js +20 -0
- package/dist/indexer/read-preflight.js +23 -0
- package/dist/indexer/search/db-search.js +89 -13
- package/dist/indexer/search/fts-query.js +51 -0
- package/dist/indexer/search/ranking-contributors.js +95 -9
- package/dist/indexer/search/ranking.js +79 -3
- package/dist/indexer/search/search-fields.js +6 -0
- package/dist/indexer/search/search-source.js +32 -21
- package/dist/indexer/search/semantic-status.js +4 -0
- package/dist/indexer/walk/matchers.js +9 -0
- package/dist/indexer/walk/walker.js +21 -13
- package/dist/integrations/agent/builders.js +39 -13
- package/dist/integrations/agent/config.js +20 -59
- package/dist/integrations/agent/detect.js +9 -0
- package/dist/integrations/agent/index.js +3 -19
- package/dist/integrations/agent/model-aliases.js +7 -2
- package/dist/integrations/agent/profiles.js +7 -1
- package/dist/integrations/agent/prompts.js +75 -9
- package/dist/integrations/agent/runner-dispatch.js +59 -0
- package/dist/integrations/agent/runner.js +13 -9
- package/dist/integrations/agent/spawn.js +69 -67
- package/dist/integrations/harnesses/claude/agent-builder.js +1 -1
- package/dist/integrations/harnesses/claude/index.js +2 -0
- package/dist/integrations/harnesses/claude/session-log.js +11 -1
- package/dist/integrations/harnesses/index.js +2 -3
- package/dist/integrations/harnesses/opencode/agent-builder.js +1 -1
- package/dist/integrations/harnesses/opencode/index.js +2 -0
- package/dist/integrations/harnesses/opencode/session-log.js +173 -3
- package/dist/integrations/harnesses/opencode-sdk/index.js +2 -2
- package/dist/integrations/harnesses/opencode-sdk/sdk-runner.js +98 -17
- package/dist/integrations/harnesses/types.js +1 -0
- package/dist/integrations/session-logs/index.js +16 -0
- package/dist/llm/call-ai.js +2 -2
- package/dist/llm/client.js +57 -15
- package/dist/llm/embedder.js +67 -4
- package/dist/llm/embedders/cache.js +3 -1
- package/dist/llm/embedders/deterministic.js +66 -0
- package/dist/llm/embedders/local.js +73 -3
- package/dist/llm/feature-gate.js +16 -15
- package/dist/llm/graph-extract.js +67 -44
- package/dist/llm/memory-infer-impl.js +138 -0
- package/dist/llm/memory-infer.js +1 -127
- package/dist/llm/metadata-enhance.js +44 -31
- package/dist/llm/structured-call.js +49 -0
- package/dist/migrate-storage-node.mjs +8 -0
- package/dist/output/context.js +5 -5
- package/dist/output/renderers.js +85 -14
- package/dist/output/shapes/curate.js +14 -2
- package/dist/output/shapes/helpers.js +0 -3
- package/dist/output/shapes/passthrough.js +2 -1
- package/dist/output/text/helpers.js +29 -1
- package/dist/output/text/workflow.js +1 -0
- package/dist/registry/providers/skills-sh.js +21 -147
- package/dist/registry/providers/static-index.js +15 -157
- package/dist/registry/resolve.js +27 -9
- package/dist/runtime.js +25 -1
- package/dist/scripts/migrate-storage.js +2718 -2354
- package/dist/scripts/migrations/import-fs-improve-runs-to-db.js +891 -597
- package/dist/setup/detect.js +9 -0
- package/dist/setup/legacy-config.js +106 -0
- package/dist/setup/prompt.js +57 -0
- package/dist/setup/providers.js +14 -0
- package/dist/setup/registry-stash-loader.js +12 -0
- package/dist/setup/semantic-assets.js +124 -0
- package/dist/setup/setup.js +52 -1614
- package/dist/setup/steps/connection.js +734 -0
- package/dist/setup/steps/output.js +31 -0
- package/dist/setup/steps/platforms.js +124 -0
- package/dist/setup/steps/semantic.js +27 -0
- package/dist/setup/steps/sources.js +222 -0
- package/dist/setup/steps/stashdir.js +42 -0
- package/dist/setup/steps/tasks.js +152 -0
- package/dist/sources/include.js +6 -2
- package/dist/sources/providers/filesystem.js +0 -1
- package/dist/sources/providers/git-install.js +210 -0
- package/dist/sources/providers/git-provider.js +234 -0
- package/dist/sources/providers/git-stash.js +248 -0
- package/dist/sources/providers/git.js +10 -661
- package/dist/sources/providers/npm.js +2 -6
- package/dist/sources/providers/provider-utils.js +13 -7
- package/dist/sources/providers/sync-from-ref.js +9 -1
- package/dist/sources/providers/tar-utils.js +16 -8
- package/dist/sources/providers/website.js +9 -5
- package/dist/sources/website-ingest.js +187 -29
- package/dist/sources/wiki-fetchers/registry.js +53 -0
- package/dist/sources/wiki-fetchers/youtube.js +239 -0
- package/dist/storage/database.js +45 -10
- package/dist/storage/managed-db.js +82 -0
- package/dist/storage/repositories/canaries-repository.js +107 -0
- package/dist/storage/repositories/consolidation-repository.js +38 -0
- package/dist/storage/repositories/embeddings-repository.js +72 -0
- package/dist/storage/repositories/events-repository.js +187 -0
- package/dist/storage/repositories/extract-sessions-repository.js +96 -0
- package/dist/storage/repositories/improve-runs-repository.js +146 -0
- package/dist/storage/repositories/index-db.js +14 -8
- package/dist/storage/repositories/proposals-repository.js +220 -0
- package/dist/storage/repositories/recombine-repository.js +213 -0
- package/dist/storage/repositories/registry-cache.js +93 -0
- package/dist/storage/repositories/registry-index-cache-repository.js +46 -0
- package/dist/storage/repositories/task-history-repository.js +93 -0
- package/dist/storage/sqlite-pragmas.js +146 -0
- package/dist/tasks/backends/cron.js +1 -1
- package/dist/tasks/backends/index.js +9 -0
- package/dist/tasks/backends/launchd.js +1 -1
- package/dist/tasks/backends/schtasks.js +1 -1
- package/dist/tasks/{resolveAkmBin.js → resolve-akm-bin.js} +2 -2
- package/dist/tasks/runner.js +15 -13
- package/dist/text-import-hook.mjs +0 -0
- package/dist/wiki/wiki.js +52 -11
- package/dist/workflows/cli.js +1 -0
- package/dist/workflows/db.js +3 -4
- package/dist/workflows/runtime/runs.js +43 -118
- package/dist/workflows/runtime/workflow-asset-loader.js +125 -0
- package/dist/workflows/validate-summary.js +2 -7
- package/docs/README.md +69 -18
- package/docs/data-and-telemetry.md +5 -4
- package/docs/migration/release-notes/0.7.0.md +1 -1
- package/docs/migration/release-notes/0.9.0.md +39 -0
- package/package.json +10 -10
- package/dist/assets/tasks/core/update-stashes.yml +0 -4
- package/dist/commands/db-cli.js +0 -23
- package/dist/indexer/db/db-backup.js +0 -376
- package/dist/indexer/passes/staleness-detect.js +0 -488
|
@@ -0,0 +1,213 @@
|
|
|
1
|
+
// This Source Code Form is subject to the terms of the Mozilla Public
|
|
2
|
+
// License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
3
|
+
// file, You can obtain one at https://mozilla.org/MPL/2.0/.
|
|
4
|
+
/**
|
|
5
|
+
* Record an induction of a recombine hypothesis and return the new consecutive
|
|
6
|
+
* count. INSERT … ON CONFLICT increments the streak, but the `last_run` guard
|
|
7
|
+
* makes a repeated call within the SAME run idempotent (no double-increment if
|
|
8
|
+
* the same ref appears twice in one run). On insert the streak starts at 1.
|
|
9
|
+
*/
|
|
10
|
+
export function recordRecombineInduction(db, input) {
|
|
11
|
+
const row = db
|
|
12
|
+
.prepare(`
|
|
13
|
+
INSERT INTO recombine_hypotheses
|
|
14
|
+
(hypothesis_ref, signature, member_key, consecutive_count, first_seen_at, last_seen_at, last_run)
|
|
15
|
+
VALUES (?, ?, ?, 1, ?, ?, ?)
|
|
16
|
+
ON CONFLICT(hypothesis_ref) DO UPDATE SET
|
|
17
|
+
consecutive_count = consecutive_count + (CASE WHEN last_run IS excluded.last_run THEN 0 ELSE 1 END),
|
|
18
|
+
last_seen_at = excluded.last_seen_at,
|
|
19
|
+
last_run = excluded.last_run,
|
|
20
|
+
signature = excluded.signature,
|
|
21
|
+
member_key = excluded.member_key
|
|
22
|
+
RETURNING consecutive_count
|
|
23
|
+
`)
|
|
24
|
+
.get(input.hypothesisRef, input.signature, input.memberKey, input.seenAt, input.seenAt, input.run);
|
|
25
|
+
return row?.consecutive_count ?? 0;
|
|
26
|
+
}
|
|
27
|
+
/**
|
|
28
|
+
* #633 — find an existing pending (non-promoted) hypothesis row whose cluster
|
|
29
|
+
* is the SAME generalization as a newly-induced one, matched by SIGNATURE plus
|
|
30
|
+
* a Jaccard membership-overlap test, rather than an exact member-set hash.
|
|
31
|
+
*
|
|
32
|
+
* In a growing stash any added/removed memory changes the exact member set, so
|
|
33
|
+
* the ref hash (and member_key) shift every run → a fresh row at count=1 → the
|
|
34
|
+
* streak never reaches `confirmThreshold` and nothing ever promotes. Matching
|
|
35
|
+
* on overlap lets a drifting-but-stable cluster keep accumulating under one row.
|
|
36
|
+
*
|
|
37
|
+
* Returns the matched row with the HIGHEST overlap (ties broken by most-recent
|
|
38
|
+
* `last_seen_at`), or `undefined` when none clears `minOverlap`. Already-promoted
|
|
39
|
+
* rows are ignored so a confirmed lesson is not reopened by a later induction.
|
|
40
|
+
*
|
|
41
|
+
* @param memberKey the candidate cluster's membership fingerprint
|
|
42
|
+
* (sorted member entryKeys joined by `|`).
|
|
43
|
+
* @param minOverlap Jaccard threshold in [0,1]; a candidate matches when
|
|
44
|
+
* |A∩B| / |A∪B| >= minOverlap.
|
|
45
|
+
*/
|
|
46
|
+
export function findMatchingRecombineHypothesis(db, input) {
|
|
47
|
+
const candidateMembers = new Set(input.memberKey.split("|").filter((m) => m.length > 0));
|
|
48
|
+
if (candidateMembers.size === 0)
|
|
49
|
+
return undefined;
|
|
50
|
+
const rows = db
|
|
51
|
+
.prepare("SELECT * FROM recombine_hypotheses WHERE signature = ? AND promoted_at IS NULL ORDER BY last_seen_at DESC")
|
|
52
|
+
.all(input.signature);
|
|
53
|
+
let best;
|
|
54
|
+
let bestOverlap = -1;
|
|
55
|
+
for (const row of rows) {
|
|
56
|
+
const rowMembers = row.member_key.split("|").filter((m) => m.length > 0);
|
|
57
|
+
if (rowMembers.length === 0)
|
|
58
|
+
continue;
|
|
59
|
+
let intersection = 0;
|
|
60
|
+
for (const m of rowMembers) {
|
|
61
|
+
if (candidateMembers.has(m))
|
|
62
|
+
intersection += 1;
|
|
63
|
+
}
|
|
64
|
+
const union = candidateMembers.size + rowMembers.length - intersection;
|
|
65
|
+
const overlap = union === 0 ? 0 : intersection / union;
|
|
66
|
+
// rows are ordered last_seen_at DESC, so a strict `>` keeps the most-recent
|
|
67
|
+
// row on ties.
|
|
68
|
+
if (overlap >= input.minOverlap && overlap > bestOverlap) {
|
|
69
|
+
best = row;
|
|
70
|
+
bestOverlap = overlap;
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
return best;
|
|
74
|
+
}
|
|
75
|
+
/**
|
|
76
|
+
* Fetch a single recombine hypothesis row, or `undefined` when the ref has
|
|
77
|
+
* never been induced. Normalizes bun:sqlite null → undefined like
|
|
78
|
+
* {@link getExtractedSession}.
|
|
79
|
+
*/
|
|
80
|
+
export function getRecombineHypothesis(db, hypothesisRef) {
|
|
81
|
+
const row = db
|
|
82
|
+
.prepare("SELECT * FROM recombine_hypotheses WHERE hypothesis_ref = ?")
|
|
83
|
+
.get(hypothesisRef);
|
|
84
|
+
return row ?? undefined;
|
|
85
|
+
}
|
|
86
|
+
/**
|
|
87
|
+
* Mark a hypothesis promoted: stamp `promoted_at` and reset the consecutive
|
|
88
|
+
* count to 0, so it must re-accumulate a full confirmation streak before it can
|
|
89
|
+
* promote again. The `promoted_at` non-null state is the double-promotion guard.
|
|
90
|
+
*/
|
|
91
|
+
export function markRecombineHypothesisPromoted(db, hypothesisRef, promotedAt) {
|
|
92
|
+
db.prepare("UPDATE recombine_hypotheses SET promoted_at = ?, consecutive_count = 0 WHERE hypothesis_ref = ?").run(promotedAt, hypothesisRef);
|
|
93
|
+
}
|
|
94
|
+
/**
|
|
95
|
+
* #658 — does any current-run cluster match this hypothesis row under the SAME
|
|
96
|
+
* signature + Jaccard-overlap rule used for re-induction? A match means the
|
|
97
|
+
* cluster genuinely re-formed this run (it was merely cap-displaced out of the
|
|
98
|
+
* processed top-`maxClustersPerRun` slice), so its streak must NOT be reset.
|
|
99
|
+
*/
|
|
100
|
+
function hypothesisMatchesAnyPresentCluster(row, presentClusters, minOverlap) {
|
|
101
|
+
const rowMembers = row.member_key.split("|").filter((m) => m.length > 0);
|
|
102
|
+
if (rowMembers.length === 0)
|
|
103
|
+
return false;
|
|
104
|
+
const rowSet = new Set(rowMembers);
|
|
105
|
+
for (const cluster of presentClusters) {
|
|
106
|
+
if (cluster.signature !== row.signature)
|
|
107
|
+
continue;
|
|
108
|
+
const clusterMembers = cluster.memberKey.split("|").filter((m) => m.length > 0);
|
|
109
|
+
if (clusterMembers.length === 0)
|
|
110
|
+
continue;
|
|
111
|
+
let intersection = 0;
|
|
112
|
+
for (const m of clusterMembers) {
|
|
113
|
+
if (rowSet.has(m))
|
|
114
|
+
intersection += 1;
|
|
115
|
+
}
|
|
116
|
+
const union = rowSet.size + clusterMembers.length - intersection;
|
|
117
|
+
const overlap = union === 0 ? 0 : intersection / union;
|
|
118
|
+
if (overlap >= minOverlap)
|
|
119
|
+
return true;
|
|
120
|
+
}
|
|
121
|
+
return false;
|
|
122
|
+
}
|
|
123
|
+
/**
|
|
124
|
+
* Decay-to-zero every NON-promoted hypothesis NOT re-induced in the current run.
|
|
125
|
+
*
|
|
126
|
+
* A generalization that stops being supported by the corpus has lost its
|
|
127
|
+
* confirmation streak, so we hard-reset `consecutive_count` to 0 (the
|
|
128
|
+
* alternative — `count - 1` floored at 0 — tolerates a single noisy run but
|
|
129
|
+
* blurs the "consecutive" semantics; hard-reset is the conservative choice).
|
|
130
|
+
*
|
|
131
|
+
* Only rows whose `hypothesis_ref` is NOT in `seenRefs` AND whose `last_run` is
|
|
132
|
+
* NOT the current run are decayed. Already-promoted rows are left alone.
|
|
133
|
+
*
|
|
134
|
+
* #658 — CAP-AWARE decay. The recombine pass only re-inducts (and thus marks
|
|
135
|
+
* `seen`) the top-`maxClustersPerRun` clusters, but a cluster genuinely
|
|
136
|
+
* re-forms every run even when it is displaced below that cap. Resetting such a
|
|
137
|
+
* row treats a SCHEDULING miss as a SUBSTANCE miss and traps the hypothesis
|
|
138
|
+
* below `confirmThreshold` forever. When `opts.presentClusters` is supplied, a
|
|
139
|
+
* row is SPARED from decay if it Jaccard-matches any present cluster (same
|
|
140
|
+
* signature, overlap >= `opts.minOverlap`) — i.e. its cluster re-formed this run
|
|
141
|
+
* but was cap-displaced. This does NOT advance the streak (only re-induction in
|
|
142
|
+
* the processed slice does that, via {@link recordRecombineInduction}), so the
|
|
143
|
+
* recurrence bar for promotion is unchanged; it only stops the cap from
|
|
144
|
+
* manufacturing artificial misses. Omitting `presentClusters` preserves the
|
|
145
|
+
* pre-#658 hard-reset-after-one-miss behaviour exactly.
|
|
146
|
+
*
|
|
147
|
+
* Returns the number of rows reset.
|
|
148
|
+
*/
|
|
149
|
+
export function decayUnseenRecombineHypotheses(db, currentRun, seenRefs, opts) {
|
|
150
|
+
// #658 — when cap-aware sparing is requested, fold the cap-displaced rows into
|
|
151
|
+
// the "seen" exclusion set: the underlying reset SQL already protects every
|
|
152
|
+
// ref it is given, so sparing == treating a spared row exactly like a seen
|
|
153
|
+
// row for this sweep (its count is left untouched, never advanced).
|
|
154
|
+
let effectiveSeen = seenRefs;
|
|
155
|
+
if (opts && opts.presentClusters.length > 0) {
|
|
156
|
+
const candidates = db
|
|
157
|
+
.prepare("SELECT hypothesis_ref, signature, member_key FROM recombine_hypotheses WHERE promoted_at IS NULL AND (last_run IS NULL OR last_run != ?) AND consecutive_count != 0")
|
|
158
|
+
.all(currentRun);
|
|
159
|
+
const seenSet = new Set(seenRefs);
|
|
160
|
+
for (const row of candidates) {
|
|
161
|
+
if (seenSet.has(row.hypothesis_ref))
|
|
162
|
+
continue;
|
|
163
|
+
if (hypothesisMatchesAnyPresentCluster(row, opts.presentClusters, opts.minOverlap)) {
|
|
164
|
+
seenSet.add(row.hypothesis_ref);
|
|
165
|
+
}
|
|
166
|
+
}
|
|
167
|
+
effectiveSeen = [...seenSet];
|
|
168
|
+
}
|
|
169
|
+
return decayUnseenRecombineHypothesesInner(db, currentRun, effectiveSeen);
|
|
170
|
+
}
|
|
171
|
+
/**
|
|
172
|
+
* The raw reset sweep shared by the cap-aware wrapper above. Resets every
|
|
173
|
+
* non-promoted row from a prior run whose ref is NOT in `seenRefs`. Kept private
|
|
174
|
+
* so the param-ceiling chunking logic lives in one place.
|
|
175
|
+
*/
|
|
176
|
+
function decayUnseenRecombineHypothesesInner(db, currentRun, seenRefs) {
|
|
177
|
+
// Reset every eligible row, then exclude the seen refs in chunks to respect
|
|
178
|
+
// the ~999 SQLite param ceiling. With no seen refs we reset all non-promoted
|
|
179
|
+
// rows from prior runs in a single statement.
|
|
180
|
+
if (seenRefs.length === 0) {
|
|
181
|
+
const res = db
|
|
182
|
+
.prepare("UPDATE recombine_hypotheses SET consecutive_count = 0 WHERE promoted_at IS NULL AND (last_run IS NULL OR last_run != ?) AND consecutive_count != 0")
|
|
183
|
+
.run(currentRun);
|
|
184
|
+
return Number(res.changes);
|
|
185
|
+
}
|
|
186
|
+
// A single NOT IN keeps the exclusion atomic (a chunked NOT IN would let a ref
|
|
187
|
+
// excluded by one chunk still be reset by another chunk's statement). The
|
|
188
|
+
// recombine pass caps RE-INDUCED clusters at `maxClustersPerRun` (a handful) —
|
|
189
|
+
// but with #658 cap-aware sparing the caller folds every cap-displaced
|
|
190
|
+
// (present-but-unprocessed) hypothesis into `effectiveSeen` too, so on a large
|
|
191
|
+
// stash `seenRefs` here can carry MANY spared refs, not just the handful that
|
|
192
|
+
// were processed. We cap defensively at ~900 (under SQLite's ~999 param
|
|
193
|
+
// ceiling): if `effectiveSeen` somehow exceeds it we fall back to resetting all
|
|
194
|
+
// eligible rows — which re-introduces the cap-displacement trap for THAT run
|
|
195
|
+
// (spared rows get decayed because the NOT IN protection is dropped). That is a
|
|
196
|
+
// rare, bounded degradation; a stash with >900 simultaneously-spared
|
|
197
|
+
// hypotheses is far beyond current scale.
|
|
198
|
+
if (seenRefs.length > 900) {
|
|
199
|
+
const res = db
|
|
200
|
+
.prepare("UPDATE recombine_hypotheses SET consecutive_count = 0 WHERE promoted_at IS NULL AND (last_run IS NULL OR last_run != ?) AND consecutive_count != 0")
|
|
201
|
+
.run(currentRun);
|
|
202
|
+
return Number(res.changes);
|
|
203
|
+
}
|
|
204
|
+
const placeholders = seenRefs.map(() => "?").join(",");
|
|
205
|
+
const res = db
|
|
206
|
+
.prepare(`UPDATE recombine_hypotheses SET consecutive_count = 0
|
|
207
|
+
WHERE promoted_at IS NULL
|
|
208
|
+
AND (last_run IS NULL OR last_run != ?)
|
|
209
|
+
AND consecutive_count != 0
|
|
210
|
+
AND hypothesis_ref NOT IN (${placeholders})`)
|
|
211
|
+
.run(currentRun, ...seenRefs);
|
|
212
|
+
return Number(res.changes);
|
|
213
|
+
}
|
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
// This Source Code Form is subject to the terms of the Mozilla Public
|
|
2
|
+
// License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
3
|
+
// file, You can obtain one at https://mozilla.org/MPL/2.0/.
|
|
4
|
+
import { rethrowIfTestIsolationError } from "../../core/errors.js";
|
|
5
|
+
import { closeDatabase, openIndexDatabase } from "../../indexer/db/db.js";
|
|
6
|
+
import { getRegistryIndexCache, upsertRegistryIndexCache } from "./registry-index-cache-repository.js";
|
|
7
|
+
/**
|
|
8
|
+
* RAII-style lifecycle helper for the registry cache DB. Opens the DB (treating
|
|
9
|
+
* a failed open exactly like the legacy fall-through: the bun-test isolation
|
|
10
|
+
* guard is re-thrown, any other failure yields `db = undefined`), runs `fn`,
|
|
11
|
+
* and guarantees the DB is closed in a `finally` after `fn` has fully settled
|
|
12
|
+
* (the await is required: the callbacks are async, and closing before they
|
|
13
|
+
* settle would tear the DB down mid-write).
|
|
14
|
+
*/
|
|
15
|
+
export async function withRegistryCacheDb(fn) {
|
|
16
|
+
let db;
|
|
17
|
+
try {
|
|
18
|
+
db = openIndexDatabase();
|
|
19
|
+
}
|
|
20
|
+
catch (err) {
|
|
21
|
+
// Never mask the bun-test isolation guard as "DB unavailable".
|
|
22
|
+
rethrowIfTestIsolationError(err);
|
|
23
|
+
db = undefined;
|
|
24
|
+
}
|
|
25
|
+
try {
|
|
26
|
+
return await fn(db);
|
|
27
|
+
}
|
|
28
|
+
finally {
|
|
29
|
+
if (db) {
|
|
30
|
+
try {
|
|
31
|
+
closeDatabase(db);
|
|
32
|
+
}
|
|
33
|
+
catch {
|
|
34
|
+
/* ignore */
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
/**
|
|
40
|
+
* Shared registry index-cache fetch template. Opens the cache DB, returns a
|
|
41
|
+
* fresh cache hit when present, otherwise fetches live (writing the result back
|
|
42
|
+
* to the cache best-effort), and falls back to a stale cache row when the fetch
|
|
43
|
+
* fails. Behaviour-preserving extraction of the logic previously duplicated in
|
|
44
|
+
* `skills-sh.ts` (`fetchSkills`) and `static-index.ts` (`loadIndex`).
|
|
45
|
+
*/
|
|
46
|
+
export async function fetchCachedJson(opts) {
|
|
47
|
+
const { cacheKey, ttlMs, parseCache, fetchFresh } = opts;
|
|
48
|
+
return withRegistryCacheDb(async (db) => {
|
|
49
|
+
// ── Step 1: Try DB cache (index.db) ─────────────────────────────────────
|
|
50
|
+
let dbCacheResult;
|
|
51
|
+
try {
|
|
52
|
+
if (db) {
|
|
53
|
+
dbCacheResult = getRegistryIndexCache(db, cacheKey, ttlMs);
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
catch (err) {
|
|
57
|
+
// Never mask the bun-test isolation guard as "DB unavailable" — see
|
|
58
|
+
// rethrowIfTestIsolationError in src/core/errors.ts. Without this, a
|
|
59
|
+
// leaky test silently gets a cold cache instead of the loud
|
|
60
|
+
// TEST_ISOLATION_MISSING failure the guard intends.
|
|
61
|
+
rethrowIfTestIsolationError(err);
|
|
62
|
+
// index.db read failed (pre-migration install or test env) — fall through
|
|
63
|
+
}
|
|
64
|
+
if (dbCacheResult) {
|
|
65
|
+
const cached = parseCache(dbCacheResult.indexJson, { stale: false });
|
|
66
|
+
if (cached !== undefined) {
|
|
67
|
+
return cached;
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
// ── Step 2: Fetch fresh ──────────────────────────────────────────────────
|
|
71
|
+
try {
|
|
72
|
+
const { value, cacheJson, cacheOpts } = await fetchFresh(db);
|
|
73
|
+
if (db) {
|
|
74
|
+
try {
|
|
75
|
+
upsertRegistryIndexCache(db, cacheKey, cacheJson, cacheOpts);
|
|
76
|
+
}
|
|
77
|
+
catch {
|
|
78
|
+
/* best-effort */
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
return value;
|
|
82
|
+
}
|
|
83
|
+
catch (err) {
|
|
84
|
+
// Fetch failed — use stale DB cache if available.
|
|
85
|
+
if (dbCacheResult) {
|
|
86
|
+
const stale = parseCache(dbCacheResult.indexJson, { stale: true });
|
|
87
|
+
if (stale !== undefined)
|
|
88
|
+
return stale;
|
|
89
|
+
}
|
|
90
|
+
throw err;
|
|
91
|
+
}
|
|
92
|
+
});
|
|
93
|
+
}
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
// This Source Code Form is subject to the terms of the Mozilla Public
|
|
2
|
+
// License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
3
|
+
// file, You can obtain one at https://mozilla.org/MPL/2.0/.
|
|
4
|
+
/**
|
|
5
|
+
* Upsert a registry index cache entry in index.db.
|
|
6
|
+
*
|
|
7
|
+
* @param db - Open index.db connection (from openDatabase / openExistingDatabase).
|
|
8
|
+
* @param registryUrl - Canonical URL of the registry (used as primary key).
|
|
9
|
+
* @param indexJson - Serialised registry index document (JSON string).
|
|
10
|
+
* @param opts.etag - HTTP ETag from the response (optional).
|
|
11
|
+
* @param opts.lastModified - HTTP Last-Modified from the response (optional).
|
|
12
|
+
*/
|
|
13
|
+
export function upsertRegistryIndexCache(db, registryUrl, indexJson, opts) {
|
|
14
|
+
db.prepare(`
|
|
15
|
+
INSERT INTO registry_index_cache (registry_url, fetched_at, etag, last_modified, index_json)
|
|
16
|
+
VALUES (?, ?, ?, ?, ?)
|
|
17
|
+
ON CONFLICT(registry_url) DO UPDATE SET
|
|
18
|
+
fetched_at = excluded.fetched_at,
|
|
19
|
+
etag = excluded.etag,
|
|
20
|
+
last_modified = excluded.last_modified,
|
|
21
|
+
index_json = excluded.index_json
|
|
22
|
+
`).run(registryUrl, new Date().toISOString(), opts?.etag ?? null, opts?.lastModified ?? null, indexJson);
|
|
23
|
+
}
|
|
24
|
+
/**
|
|
25
|
+
* Look up a cached registry index entry from index.db.
|
|
26
|
+
* Returns undefined when not found or when the entry is older than `maxAgeMs`.
|
|
27
|
+
*
|
|
28
|
+
* TTL check: if `Date.now() - new Date(fetched_at).getTime() > maxAgeMs` the
|
|
29
|
+
* entry is considered a cache miss and undefined is returned.
|
|
30
|
+
*
|
|
31
|
+
* @param db - Open index.db connection.
|
|
32
|
+
* @param registryUrl - Canonical URL of the registry (primary key).
|
|
33
|
+
* @param maxAgeMs - Maximum age in milliseconds before the entry is stale (default: 1 hour).
|
|
34
|
+
*/
|
|
35
|
+
export function getRegistryIndexCache(db, registryUrl, maxAgeMs = 3_600_000 /* 1 hour */) {
|
|
36
|
+
const row = db
|
|
37
|
+
.prepare(`SELECT fetched_at, etag, last_modified, index_json
|
|
38
|
+
FROM registry_index_cache WHERE registry_url = ?`)
|
|
39
|
+
.get(registryUrl);
|
|
40
|
+
if (!row)
|
|
41
|
+
return undefined;
|
|
42
|
+
const fetchedAt = Date.parse(row.fetched_at);
|
|
43
|
+
if (Number.isNaN(fetchedAt) || Date.now() - fetchedAt > maxAgeMs)
|
|
44
|
+
return undefined;
|
|
45
|
+
return { indexJson: row.index_json, etag: row.etag, lastModified: row.last_modified };
|
|
46
|
+
}
|
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
// This Source Code Form is subject to the terms of the Mozilla Public
|
|
2
|
+
// License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
3
|
+
// file, You can obtain one at https://mozilla.org/MPL/2.0/.
|
|
4
|
+
/**
|
|
5
|
+
* Upsert a task history row.
|
|
6
|
+
*/
|
|
7
|
+
export function upsertTaskHistory(db, row) {
|
|
8
|
+
// INSERT OR IGNORE: if a run with the same (task_id, started_at) was already
|
|
9
|
+
// imported (e.g. by the migration script), skip it silently.
|
|
10
|
+
db.prepare(`
|
|
11
|
+
INSERT OR IGNORE INTO task_history
|
|
12
|
+
(task_id, status, started_at, completed_at, failed_at, log_path,
|
|
13
|
+
target_kind, target_ref, metadata_json)
|
|
14
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
15
|
+
`).run(row.task_id, row.status, row.started_at, row.completed_at ?? null, row.failed_at ?? null, row.log_path ?? null, row.target_kind ?? null, row.target_ref ?? null, row.metadata_json);
|
|
16
|
+
}
|
|
17
|
+
/**
|
|
18
|
+
* Look up a task history row by task_id. Returns undefined when not found.
|
|
19
|
+
*/
|
|
20
|
+
/**
|
|
21
|
+
* Return the most recent run for a given task_id, or undefined if no runs exist.
|
|
22
|
+
*/
|
|
23
|
+
export function getTaskHistory(db, taskId) {
|
|
24
|
+
return db
|
|
25
|
+
.prepare(`SELECT id, task_id, status, started_at, completed_at, failed_at, log_path,
|
|
26
|
+
target_kind, target_ref, metadata_json
|
|
27
|
+
FROM task_history WHERE task_id = ? ORDER BY started_at DESC LIMIT 1`)
|
|
28
|
+
.get(taskId);
|
|
29
|
+
}
|
|
30
|
+
/**
|
|
31
|
+
* Return all runs for a given task_id, newest first.
|
|
32
|
+
*/
|
|
33
|
+
export function getTaskHistoryRuns(db, taskId, limit = 50) {
|
|
34
|
+
return db
|
|
35
|
+
.prepare(`SELECT id, task_id, status, started_at, completed_at, failed_at, log_path,
|
|
36
|
+
target_kind, target_ref, metadata_json
|
|
37
|
+
FROM task_history WHERE task_id = ? ORDER BY started_at DESC LIMIT ?`)
|
|
38
|
+
.all(taskId, limit);
|
|
39
|
+
}
|
|
40
|
+
/**
|
|
41
|
+
* Query task history rows by started_at range and/or status.
|
|
42
|
+
*/
|
|
43
|
+
export function queryTaskHistory(db, options = {}) {
|
|
44
|
+
const conditions = [];
|
|
45
|
+
const params = [];
|
|
46
|
+
if (options.since) {
|
|
47
|
+
conditions.push("started_at >= ?");
|
|
48
|
+
params.push(options.since);
|
|
49
|
+
}
|
|
50
|
+
if (options.until) {
|
|
51
|
+
conditions.push("started_at <= ?");
|
|
52
|
+
params.push(options.until);
|
|
53
|
+
}
|
|
54
|
+
if (options.status) {
|
|
55
|
+
conditions.push("status = ?");
|
|
56
|
+
params.push(options.status);
|
|
57
|
+
}
|
|
58
|
+
if (options.targetKind) {
|
|
59
|
+
conditions.push("target_kind = ?");
|
|
60
|
+
params.push(options.targetKind);
|
|
61
|
+
}
|
|
62
|
+
if (options.targetRef) {
|
|
63
|
+
conditions.push("target_ref = ?");
|
|
64
|
+
params.push(options.targetRef);
|
|
65
|
+
}
|
|
66
|
+
const where = conditions.length > 0 ? `WHERE ${conditions.join(" AND ")}` : "";
|
|
67
|
+
return db
|
|
68
|
+
.prepare(`SELECT task_id, status, started_at, completed_at, failed_at, log_path,
|
|
69
|
+
target_kind, target_ref, metadata_json
|
|
70
|
+
FROM task_history ${where} ORDER BY started_at DESC`)
|
|
71
|
+
.all(...params);
|
|
72
|
+
}
|
|
73
|
+
/**
|
|
74
|
+
* Read COMPLETED `akm-improve` task_history runs whose `started_at` falls in
|
|
75
|
+
* `[since, until)` (or `started_at >= since` when `until` is omitted), ordered
|
|
76
|
+
* oldest-first by `started_at`. Only rows with a non-null `completed_at` are
|
|
77
|
+
* returned (in-flight runs are excluded). The `task_id = 'akm-improve'`
|
|
78
|
+
* predicate is fixed because the only caller (commands/health.ts
|
|
79
|
+
* `loadTaskIntervals`) builds wall-time intervals for the improve cron task.
|
|
80
|
+
*
|
|
81
|
+
* Owns the SQL formerly inlined in commands/health.ts. Note the bound is
|
|
82
|
+
* EXCLUSIVE on the upper end (`started_at < ?`) — callers pass an already
|
|
83
|
+
* widened window; this helper does not widen.
|
|
84
|
+
*
|
|
85
|
+
* Connection-lifetime rule (WS5): `.all()` materializes a plain array before
|
|
86
|
+
* returning.
|
|
87
|
+
*/
|
|
88
|
+
export function queryCompletedTaskIntervals(db, since, until) {
|
|
89
|
+
const sql = until
|
|
90
|
+
? "SELECT started_at, completed_at FROM task_history WHERE task_id = 'akm-improve' AND started_at >= ? AND started_at < ? AND completed_at IS NOT NULL ORDER BY started_at"
|
|
91
|
+
: "SELECT started_at, completed_at FROM task_history WHERE task_id = 'akm-improve' AND started_at >= ? AND completed_at IS NOT NULL ORDER BY started_at";
|
|
92
|
+
return (until ? db.prepare(sql).all(since, until) : db.prepare(sql).all(since));
|
|
93
|
+
}
|
|
@@ -0,0 +1,146 @@
|
|
|
1
|
+
// This Source Code Form is subject to the terms of the Mozilla Public
|
|
2
|
+
// License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
3
|
+
// file, You can obtain one at https://mozilla.org/MPL/2.0/.
|
|
4
|
+
/**
|
|
5
|
+
* Shared SQLite PRAGMA application + journal-mode resolution (#628).
|
|
6
|
+
*
|
|
7
|
+
* Every AKM SQLite opener applies the same opening PRAGMAs: a journal mode, a
|
|
8
|
+
* 30 s busy_timeout, and (for most) foreign_keys = ON. Historically each opener
|
|
9
|
+
* hard-coded `PRAGMA journal_mode = WAL`. WAL is impossible on network
|
|
10
|
+
* filesystems (NFS/SMB) because its `-shm` shared-memory wal-index cannot be
|
|
11
|
+
* mmap'd over a network mount, so AKM could not run with its data dir on a
|
|
12
|
+
* network share.
|
|
13
|
+
*
|
|
14
|
+
* This module centralises that PRAGMA block behind {@link applyStandardPragmas}
|
|
15
|
+
* and makes the journal mode configurable via the `AKM_SQLITE_JOURNAL_MODE`
|
|
16
|
+
* env var (WAL | DELETE | TRUNCATE; default WAL = unchanged behaviour). When the
|
|
17
|
+
* mode is the WAL default and the data directory is detected to live on a
|
|
18
|
+
* network filesystem, it auto-falls-back to DELETE with a one-line warning.
|
|
19
|
+
*
|
|
20
|
+
* Boundary note: this is a PLAIN module, not a runtime-boundary file. It only
|
|
21
|
+
* does pure string work plus `db.exec()` (allowed everywhere). The single
|
|
22
|
+
* runtime primitive it needs — a filesystem-type probe (statfs) — lives in
|
|
23
|
+
* src/runtime.ts and is injected here, keeping the network-FS classifier a
|
|
24
|
+
* pure, unit-testable function.
|
|
25
|
+
*
|
|
26
|
+
* @module storage/sqlite-pragmas
|
|
27
|
+
*/
|
|
28
|
+
import { warn } from "../core/warn.js";
|
|
29
|
+
import { statfsType } from "../runtime.js";
|
|
30
|
+
const VALID_MODES = new Set(["WAL", "DELETE", "TRUNCATE"]);
|
|
31
|
+
// One-shot warning guards so a misconfigured env var or a network-FS fallback
|
|
32
|
+
// each emit AT MOST ONCE per process rather than on every db open.
|
|
33
|
+
let warnedInvalid = false;
|
|
34
|
+
let warnedNetworkFallback = false;
|
|
35
|
+
/**
|
|
36
|
+
* Resolve a raw `AKM_SQLITE_JOURNAL_MODE` value to a canonical {@link JournalMode}.
|
|
37
|
+
*
|
|
38
|
+
* PURE and unit-testable: the raw string is passed in (not read from
|
|
39
|
+
* `process.env` here). Trims + uppercases; an empty/undefined value yields the
|
|
40
|
+
* WAL default; a recognised value yields its canonical form; any other
|
|
41
|
+
* non-empty value warns once and falls back to WAL. Never throws.
|
|
42
|
+
*/
|
|
43
|
+
export function resolveJournalMode(raw) {
|
|
44
|
+
if (raw === undefined)
|
|
45
|
+
return "WAL";
|
|
46
|
+
const normalized = raw.trim().toUpperCase();
|
|
47
|
+
if (normalized === "")
|
|
48
|
+
return "WAL";
|
|
49
|
+
if (VALID_MODES.has(normalized)) {
|
|
50
|
+
return normalized;
|
|
51
|
+
}
|
|
52
|
+
warnInvalidJournalModeOnce(raw);
|
|
53
|
+
return "WAL";
|
|
54
|
+
}
|
|
55
|
+
/**
|
|
56
|
+
* The single env-reading seam: resolve the configured journal mode from
|
|
57
|
+
* `process.env.AKM_SQLITE_JOURNAL_MODE`. Read at call time (per open) so tests
|
|
58
|
+
* that set the env per-case see the right value and we avoid stale-env flakes.
|
|
59
|
+
*/
|
|
60
|
+
export function resolveConfiguredJournalMode(env = process.env) {
|
|
61
|
+
return resolveJournalMode(env.AKM_SQLITE_JOURNAL_MODE);
|
|
62
|
+
}
|
|
63
|
+
function warnInvalidJournalModeOnce(raw) {
|
|
64
|
+
if (warnedInvalid)
|
|
65
|
+
return;
|
|
66
|
+
warnedInvalid = true;
|
|
67
|
+
warn(`[akm] invalid AKM_SQLITE_JOURNAL_MODE=${JSON.stringify(raw)} — using WAL (valid: WAL, DELETE, TRUNCATE)`);
|
|
68
|
+
}
|
|
69
|
+
// Known Linux f_type magic numbers for network filesystems. node's statfs
|
|
70
|
+
// normalises `type` to this numeric f_type magic on all platforms.
|
|
71
|
+
const FS_MAGIC_NFS = 0x6969; // 26985 — NFS
|
|
72
|
+
const FS_MAGIC_SMB = 0x517b; // 20859 — older SMB_SUPER_MAGIC
|
|
73
|
+
const FS_MAGIC_CIFS = 0xff534d42; // 4283649346 — SMB/CIFS
|
|
74
|
+
const FS_MAGIC_SMB2 = 0xfe534d42; // 4267272514 — SMB2
|
|
75
|
+
const FS_MAGIC_FUSE = 0x65735546; // 1702057286 — FUSE (sshfs + many network FUSE mounts)
|
|
76
|
+
const NETWORK_FS_MAGICS = new Set([
|
|
77
|
+
FS_MAGIC_NFS,
|
|
78
|
+
FS_MAGIC_SMB,
|
|
79
|
+
FS_MAGIC_CIFS,
|
|
80
|
+
FS_MAGIC_SMB2,
|
|
81
|
+
// FUSE is a judgment call: it backs BOTH network mounts (sshfs) and local
|
|
82
|
+
// mounts (some encrypted/overlay FS). Treating it as network falls back to
|
|
83
|
+
// DELETE — conservative-but-safe (DELETE works everywhere; the only cost is
|
|
84
|
+
// losing WAL concurrency). An operator can always force WAL via the env var.
|
|
85
|
+
FS_MAGIC_FUSE,
|
|
86
|
+
]);
|
|
87
|
+
/**
|
|
88
|
+
* PURE classifier: is `fsType` a known network-filesystem magic number?
|
|
89
|
+
* Returns false for `undefined` (probe failed/unsupported) and for local
|
|
90
|
+
* magics (ext4 0xEF53, btrfs, xfs, tmpfs, apfs, …). Unit-testable with
|
|
91
|
+
* injected magic numbers — no real mount required.
|
|
92
|
+
*/
|
|
93
|
+
export function isNetworkFilesystem(fsType) {
|
|
94
|
+
if (fsType === undefined)
|
|
95
|
+
return false;
|
|
96
|
+
return NETWORK_FS_MAGICS.has(fsType);
|
|
97
|
+
}
|
|
98
|
+
/**
|
|
99
|
+
* Apply AKM's standard opening PRAGMAs to `db`, in order:
|
|
100
|
+
* 1. `journal_mode` = the configured mode (with WAL→DELETE network-FS fallback)
|
|
101
|
+
* 2. `busy_timeout = 30000`
|
|
102
|
+
* 3. `foreign_keys = ON` (unless `opts.foreignKeys === false`)
|
|
103
|
+
* 4. `synchronous = FULL` (only in a rollback-journal mode — DELETE/TRUNCATE)
|
|
104
|
+
*
|
|
105
|
+
* Returns the effective {@link JournalMode} so callers/tests can assert it.
|
|
106
|
+
*
|
|
107
|
+
* `synchronous = FULL` is set explicitly only in DELETE/TRUNCATE so durability
|
|
108
|
+
* intent is clear: rollback journals need FULL for crash-durability across
|
|
109
|
+
* power loss, whereas WAL is durable at NORMAL. SQLite's default synchronous is
|
|
110
|
+
* already FULL when unset, so this never changes WAL-default behaviour — the
|
|
111
|
+
* WAL path emits no `synchronous` pragma, exactly as before.
|
|
112
|
+
*/
|
|
113
|
+
export function applyStandardPragmas(db, opts = {}) {
|
|
114
|
+
let mode = resolveConfiguredJournalMode(opts.env);
|
|
115
|
+
// Network-FS fallback only fires for the WAL default and only when we have a
|
|
116
|
+
// directory to probe. An explicitly-requested DELETE/TRUNCATE is never
|
|
117
|
+
// overridden, and a failed/unsupported probe (undefined) keeps WAL.
|
|
118
|
+
if (mode === "WAL" && opts.dataDir) {
|
|
119
|
+
const probe = opts.fsTypeProbe ?? statfsType;
|
|
120
|
+
if (isNetworkFilesystem(probe(opts.dataDir))) {
|
|
121
|
+
mode = "DELETE";
|
|
122
|
+
warnNetworkFallbackOnce(opts.dataDir);
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
// PRAGMAs must run before any DDL or DML. busy_timeout is applied FIRST so a
|
|
126
|
+
// journal-mode change that must reclaim a leftover `-wal` file (WAL→DELETE on
|
|
127
|
+
// reopen of an unclean WAL db, e.g. after a crash) can wait out a transient
|
|
128
|
+
// lock instead of failing immediately with SQLITE_BUSY. For the WAL default
|
|
129
|
+
// this is a no-op (WAL→WAL changes nothing), so byte-identical behaviour is
|
|
130
|
+
// preserved.
|
|
131
|
+
db.exec("PRAGMA busy_timeout = 30000");
|
|
132
|
+
db.exec(`PRAGMA journal_mode = ${mode}`);
|
|
133
|
+
if (opts.foreignKeys !== false) {
|
|
134
|
+
db.exec("PRAGMA foreign_keys = ON");
|
|
135
|
+
}
|
|
136
|
+
if (mode !== "WAL") {
|
|
137
|
+
db.exec("PRAGMA synchronous = FULL");
|
|
138
|
+
}
|
|
139
|
+
return mode;
|
|
140
|
+
}
|
|
141
|
+
function warnNetworkFallbackOnce(dataDir) {
|
|
142
|
+
if (warnedNetworkFallback)
|
|
143
|
+
return;
|
|
144
|
+
warnedNetworkFallback = true;
|
|
145
|
+
warn(`[akm] network filesystem detected at ${dataDir} — WAL unsupported, using DELETE journal mode`);
|
|
146
|
+
}
|
|
@@ -29,7 +29,7 @@ import fs from "node:fs";
|
|
|
29
29
|
import path from "node:path";
|
|
30
30
|
import { ConfigError } from "../../core/errors.js";
|
|
31
31
|
import { getTaskLogDir } from "../../core/paths.js";
|
|
32
|
-
import { resolveAkmInvocation } from "../
|
|
32
|
+
import { resolveAkmInvocation } from "../resolve-akm-bin.js";
|
|
33
33
|
import { parseSchedule, translateToCron } from "../schedule.js";
|
|
34
34
|
const BEGIN = (id) => `# akm:task ${id} BEGIN`;
|
|
35
35
|
const END = (id) => `# akm:task ${id} END`;
|
|
@@ -4,7 +4,14 @@
|
|
|
4
4
|
import { CRON_BACKEND } from "./cron.js";
|
|
5
5
|
import { LAUNCHD_BACKEND } from "./launchd.js";
|
|
6
6
|
import { SCHTASKS_BACKEND } from "./schtasks.js";
|
|
7
|
+
let backendsOverrides;
|
|
8
|
+
/** TEST-ONLY. Swap backend selection; pass undefined to restore the real implementations. */
|
|
9
|
+
export function _setBackendsForTests(fakes) {
|
|
10
|
+
backendsOverrides = fakes;
|
|
11
|
+
}
|
|
7
12
|
export function selectBackend(options = {}) {
|
|
13
|
+
if (backendsOverrides?.selectBackend)
|
|
14
|
+
return backendsOverrides.selectBackend(options);
|
|
8
15
|
const platform = options.platform ?? process.platform;
|
|
9
16
|
switch (platform) {
|
|
10
17
|
case "win32":
|
|
@@ -16,6 +23,8 @@ export function selectBackend(options = {}) {
|
|
|
16
23
|
}
|
|
17
24
|
}
|
|
18
25
|
export function backendNameForPlatform(platform = process.platform) {
|
|
26
|
+
if (backendsOverrides?.backendNameForPlatform)
|
|
27
|
+
return backendsOverrides.backendNameForPlatform(platform);
|
|
19
28
|
if (platform === "win32")
|
|
20
29
|
return "schtasks";
|
|
21
30
|
if (platform === "darwin")
|
|
@@ -27,7 +27,7 @@ import path from "node:path";
|
|
|
27
27
|
import launchdTemplate from "../../assets/backends/launchd-template.xml" with { type: "text" };
|
|
28
28
|
import { ConfigError } from "../../core/errors.js";
|
|
29
29
|
import { getTaskLogDir } from "../../core/paths.js";
|
|
30
|
-
import { resolveAkmInvocation } from "../
|
|
30
|
+
import { resolveAkmInvocation } from "../resolve-akm-bin.js";
|
|
31
31
|
import { parseSchedule, translateToLaunchd } from "../schedule.js";
|
|
32
32
|
import { escapeXml, nodeExec, nodeFs } from "./exec-utils.js";
|
|
33
33
|
export const LAUNCHD_LABEL_PREFIX = "com.akm.task.";
|
|
@@ -35,7 +35,7 @@ import path from "node:path";
|
|
|
35
35
|
import schtasksTemplate from "../../assets/backends/schtasks-template.xml" with { type: "text" };
|
|
36
36
|
import { ConfigError } from "../../core/errors.js";
|
|
37
37
|
import { getTaskLogDir } from "../../core/paths.js";
|
|
38
|
-
import { resolveAkmInvocation } from "../
|
|
38
|
+
import { resolveAkmInvocation } from "../resolve-akm-bin.js";
|
|
39
39
|
import { parseSchedule, translateToSchtasks } from "../schedule.js";
|
|
40
40
|
import { escapeXml, nodeExec, nodeFs } from "./exec-utils.js";
|
|
41
41
|
export const DEFAULT_FOLDER_PREFIX = "\\akm\\";
|
|
@@ -44,8 +44,8 @@ export function resolveAkmInvocation(options = {}) {
|
|
|
44
44
|
/**
|
|
45
45
|
* From the URL of a module inside `src/tasks/` figure out the CLI entry.
|
|
46
46
|
*
|
|
47
|
-
* • dev `…/src/tasks/
|
|
48
|
-
* • build `…/dist/tasks/
|
|
47
|
+
* • dev `…/src/tasks/resolve-akm-bin.ts` → `…/src/cli.ts`
|
|
48
|
+
* • build `…/dist/tasks/resolve-akm-bin.js` → `…/dist/cli.js`
|
|
49
49
|
*/
|
|
50
50
|
function resolveCliEntry(moduleUrl) {
|
|
51
51
|
let modulePath;
|