akm-cli 0.9.0-beta.9 → 0.9.0-rc.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +592 -0
- package/README.md +12 -4
- package/dist/akm +38 -0
- package/dist/akm-migrate-storage +38 -0
- package/dist/assets/help/help-improve.md +9 -6
- package/dist/assets/hints/cli-hints-full.md +6 -5
- package/dist/assets/profiles/default.json +9 -4
- package/dist/assets/profiles/frequent.json +1 -1
- package/dist/assets/profiles/memory-focus.json +1 -1
- package/dist/assets/profiles/proactive-maintenance.json +25 -0
- package/dist/assets/profiles/quick.json +1 -1
- package/dist/assets/profiles/recombine-only.json +21 -0
- package/dist/assets/profiles/reflect-distill.json +30 -0
- package/dist/assets/profiles/synthesize.json +15 -0
- package/dist/assets/profiles/thorough.json +1 -1
- package/dist/assets/prompts/consolidate-system.md +23 -0
- package/dist/assets/prompts/contradiction-judge.md +33 -0
- package/dist/assets/prompts/distill-knowledge-system.md +22 -0
- package/dist/assets/prompts/distill-lesson-system.md +36 -0
- package/dist/assets/prompts/extract-session.md +11 -3
- package/dist/assets/prompts/graph-extract-system.md +1 -0
- package/dist/assets/prompts/graph-extract-user-prompt.md +1 -1
- package/dist/assets/prompts/memory-infer-system.md +1 -0
- package/dist/assets/prompts/memory-infer-user.md +5 -0
- package/dist/assets/prompts/metadata-enhance-system.md +1 -0
- package/dist/assets/prompts/procedural-system.md +44 -0
- package/dist/assets/prompts/recombine-system.md +40 -0
- package/dist/assets/prompts/staleness-detect-system.md +6 -0
- package/dist/assets/prompts/validate-summary-judge.md +1 -0
- package/dist/assets/stash-skeleton/facts/conventions/assets/agent.md +38 -0
- package/dist/assets/stash-skeleton/facts/conventions/assets/command.md +38 -0
- package/dist/assets/stash-skeleton/facts/conventions/assets/fact.md +39 -0
- package/dist/assets/stash-skeleton/facts/conventions/assets/knowledge.md +40 -0
- package/dist/assets/stash-skeleton/facts/conventions/assets/lesson.md +43 -0
- package/dist/assets/stash-skeleton/facts/conventions/assets/memory.md +38 -0
- package/dist/assets/stash-skeleton/facts/conventions/assets/script.md +43 -0
- package/dist/assets/stash-skeleton/facts/conventions/assets/skill.md +40 -0
- package/dist/assets/stash-skeleton/facts/conventions/assets/workflow.md +43 -0
- package/dist/assets/templates/html/health.html +281 -111
- package/dist/assets/wiki/ingest-workflow-template.md +45 -16
- package/dist/assets/wiki/schema-template.md +4 -4
- package/dist/cli/clack.js +56 -0
- package/dist/cli/config-migrate.js +7 -1
- package/dist/cli/confirm.js +1 -1
- package/dist/cli/parse-args.js +46 -1
- package/dist/cli/shared.js +28 -0
- package/dist/cli.js +25 -21
- package/dist/commands/agent/agent-dispatch.js +3 -2
- package/dist/commands/agent/agent-support.js +0 -7
- package/dist/commands/agent/contribute-cli.js +26 -7
- package/dist/commands/config-cli.js +26 -13
- package/dist/commands/env/child-env.js +47 -0
- package/dist/commands/env/env-cli.js +220 -227
- package/dist/commands/env/env.js +14 -67
- package/dist/commands/env/secret-cli.js +140 -138
- package/dist/commands/feedback-cli.js +156 -155
- package/dist/commands/graph/graph-cli.js +5 -13
- package/dist/commands/graph/graph.js +3 -3
- package/dist/commands/health/advisories.js +151 -0
- package/dist/commands/health/checks.js +103 -16
- package/dist/commands/health/html-report.js +447 -81
- package/dist/commands/health/improve-metrics.js +771 -0
- package/dist/commands/health/llm-usage.js +65 -0
- package/dist/commands/health/md-report.js +103 -0
- package/dist/commands/health/metrics.js +278 -0
- package/dist/commands/health/stash-exposure.js +46 -0
- package/dist/commands/health/surfaces.js +216 -0
- package/dist/commands/health/task-runs.js +135 -0
- package/dist/commands/health/types.js +26 -0
- package/dist/commands/health/windows.js +195 -0
- package/dist/commands/health.js +91 -1091
- package/dist/commands/improve/anti-collapse.js +170 -0
- package/dist/commands/improve/calibration.js +161 -0
- package/dist/commands/improve/collapse-detector.js +421 -0
- package/dist/commands/improve/consolidate/chunking.js +141 -0
- package/dist/commands/improve/consolidate/eligibility.js +64 -0
- package/dist/commands/improve/consolidate/merge.js +145 -0
- package/dist/commands/improve/consolidate/sanitize.js +231 -0
- package/dist/commands/{lint.js → improve/consolidate/types.js} +1 -1
- package/dist/commands/improve/consolidate.js +1295 -1277
- package/dist/commands/improve/dedup.js +482 -0
- package/dist/commands/improve/distill/content-repair.js +202 -0
- package/dist/commands/improve/distill/promote-memory.js +229 -0
- package/dist/commands/improve/distill/quality-gate.js +236 -0
- package/dist/commands/improve/distill-guards.js +127 -0
- package/dist/commands/improve/distill-promotion-policy.js +826 -167
- package/dist/commands/improve/distill.js +228 -605
- package/dist/commands/improve/eligibility.js +434 -0
- package/dist/commands/improve/encoding-salience.js +205 -0
- package/dist/commands/improve/extract-cli.js +179 -59
- package/dist/commands/improve/extract-prompt.js +54 -3
- package/dist/commands/improve/extract-watch.js +140 -0
- package/dist/commands/improve/extract.js +409 -43
- package/dist/commands/improve/feedback-valence.js +54 -0
- package/dist/commands/improve/hot-probation.js +45 -0
- package/dist/commands/improve/improve-auto-accept.js +157 -10
- package/dist/commands/improve/improve-cli.js +115 -73
- package/dist/commands/improve/improve-profiles.js +28 -8
- package/dist/commands/improve/improve-result-file.js +15 -25
- package/dist/commands/improve/improve-session.js +58 -0
- package/dist/commands/improve/improve.js +485 -2764
- package/dist/commands/improve/locks.js +154 -0
- package/dist/commands/improve/loop-stages.js +1100 -0
- package/dist/commands/improve/memory/memory-belief.js +14 -15
- package/dist/commands/improve/memory/memory-contradiction-detect.js +83 -60
- package/dist/commands/improve/memory/memory-improve.js +27 -27
- package/dist/commands/improve/outcome-loop.js +270 -0
- package/dist/commands/improve/preparation.js +2002 -0
- package/dist/commands/improve/proactive-maintenance.js +37 -35
- package/dist/commands/improve/procedural.js +398 -0
- package/dist/commands/improve/recombine.js +818 -0
- package/dist/commands/improve/reflect-noise.js +0 -0
- package/dist/commands/improve/reflect.js +206 -45
- package/dist/commands/improve/salience.js +455 -0
- package/dist/commands/improve/schema-similarity-gate.js +168 -0
- package/dist/commands/improve/shared.js +51 -0
- package/dist/commands/improve/triage.js +93 -0
- package/dist/commands/lint/agent-linter.js +19 -24
- package/dist/commands/lint/base-linter.js +173 -60
- package/dist/commands/lint/command-linter.js +19 -24
- package/dist/commands/lint/env-key-rules.js +38 -1
- package/dist/commands/lint/fact-linter.js +39 -0
- package/dist/commands/lint/index.js +31 -13
- package/dist/commands/lint/memory-linter.js +1 -1
- package/dist/commands/lint/registry.js +7 -2
- package/dist/commands/lint/task-linter.js +3 -3
- package/dist/commands/lint/workflow-linter.js +26 -1
- package/dist/commands/observability-cli.js +4 -4
- package/dist/commands/proposal/drain-policies.js +13 -4
- package/dist/commands/proposal/drain.js +45 -51
- package/dist/commands/proposal/legacy-import.js +115 -0
- package/dist/commands/proposal/proposal-cli.js +24 -34
- package/dist/commands/proposal/proposal.js +2 -1
- package/dist/commands/proposal/propose.js +8 -3
- package/dist/commands/proposal/repository.js +829 -0
- package/dist/commands/proposal/validators/proposal-quality-validators.js +9 -8
- package/dist/commands/proposal/validators/proposals.js +93 -895
- package/dist/commands/read/curate.js +410 -111
- package/dist/commands/read/knowledge.js +10 -3
- package/dist/commands/read/remember-cli.js +133 -138
- package/dist/commands/read/search-cli.js +15 -8
- package/dist/commands/read/search.js +22 -11
- package/dist/commands/read/show.js +106 -14
- package/dist/commands/registry-cli.js +76 -87
- package/dist/commands/remember.js +11 -12
- package/dist/commands/sources/add-cli.js +91 -95
- package/dist/commands/sources/history.js +1 -1
- package/dist/commands/sources/init.js +66 -18
- package/dist/commands/sources/installed-stashes.js +11 -3
- package/dist/commands/sources/schema-repair.js +44 -46
- package/dist/commands/sources/self-update.js +2 -2
- package/dist/commands/sources/source-add.js +7 -3
- package/dist/commands/sources/sources-cli.js +3 -3
- package/dist/commands/sources/stash-cli.js +19 -39
- package/dist/commands/sources/stash-skeleton.js +57 -8
- package/dist/commands/tasks/default-tasks.js +15 -2
- package/dist/commands/tasks/tasks-cli.js +20 -29
- package/dist/commands/tasks/tasks.js +39 -11
- package/dist/commands/wiki-cli.js +23 -38
- package/dist/commands/workflow-cli.js +15 -1
- package/dist/core/asset/asset-registry.js +3 -1
- package/dist/core/asset/asset-spec.js +21 -4
- package/dist/core/asset/frontmatter.js +188 -167
- package/dist/core/asset/markdown.js +8 -0
- package/dist/core/authoring-rules.js +92 -0
- package/dist/core/common.js +4 -23
- package/dist/core/concurrent.js +10 -1
- package/dist/core/config/config-io.js +10 -1
- package/dist/core/config/config-migration.js +18 -40
- package/dist/core/config/config-schema.js +382 -62
- package/dist/core/config/config-types.js +3 -3
- package/dist/core/config/config.js +67 -22
- package/dist/core/deep-merge.js +38 -0
- package/dist/core/errors.js +1 -0
- package/dist/core/eval/rank-metrics.js +113 -0
- package/dist/core/events.js +4 -7
- package/dist/core/improve-types.js +47 -8
- package/dist/core/logs-db.js +14 -75
- package/dist/core/parse.js +36 -16
- package/dist/core/paths.js +18 -18
- package/dist/core/standards/resolve-standards-context.js +87 -0
- package/dist/core/standards/resolve-stash-standards.js +99 -0
- package/dist/core/standards/resolve-type-conventions.js +66 -0
- package/dist/core/state/migrations.js +770 -0
- package/dist/core/state-db.js +132 -1126
- package/dist/core/structured.js +69 -0
- package/dist/core/time.js +53 -0
- package/dist/core/warn.js +21 -0
- package/dist/core/write-source.js +37 -0
- package/dist/indexer/db/db.js +259 -769
- package/dist/indexer/db/entry-mapper.js +41 -0
- package/dist/indexer/db/graph-db.js +129 -86
- package/dist/indexer/db/llm-cache.js +2 -2
- package/dist/indexer/db/schema.js +516 -0
- package/dist/indexer/ensure-index.js +36 -92
- package/dist/indexer/feedback/utility-policy.js +75 -0
- package/dist/indexer/graph/graph-boost.js +51 -41
- package/dist/indexer/graph/graph-extraction.js +207 -4
- package/dist/indexer/index-writer-lock.js +18 -11
- package/dist/indexer/index-written-assets.js +105 -0
- package/dist/indexer/indexer.js +182 -204
- package/dist/indexer/passes/dir-staleness.js +114 -0
- package/dist/indexer/passes/memory-inference.js +13 -5
- package/dist/indexer/passes/metadata.js +20 -0
- package/dist/indexer/read-preflight.js +23 -0
- package/dist/indexer/search/db-search.js +89 -13
- package/dist/indexer/search/fts-query.js +51 -0
- package/dist/indexer/search/ranking-contributors.js +95 -9
- package/dist/indexer/search/ranking.js +79 -3
- package/dist/indexer/search/search-fields.js +6 -0
- package/dist/indexer/search/search-source.js +32 -21
- package/dist/indexer/search/semantic-status.js +4 -0
- package/dist/indexer/walk/matchers.js +9 -0
- package/dist/indexer/walk/walker.js +21 -13
- package/dist/integrations/agent/builders.js +39 -13
- package/dist/integrations/agent/config.js +20 -59
- package/dist/integrations/agent/detect.js +9 -0
- package/dist/integrations/agent/index.js +3 -19
- package/dist/integrations/agent/model-aliases.js +7 -2
- package/dist/integrations/agent/profiles.js +7 -1
- package/dist/integrations/agent/prompts.js +75 -9
- package/dist/integrations/agent/runner-dispatch.js +59 -0
- package/dist/integrations/agent/runner.js +13 -9
- package/dist/integrations/agent/spawn.js +69 -67
- package/dist/integrations/harnesses/claude/agent-builder.js +1 -1
- package/dist/integrations/harnesses/claude/index.js +2 -0
- package/dist/integrations/harnesses/claude/session-log.js +10 -0
- package/dist/integrations/harnesses/index.js +2 -3
- package/dist/integrations/harnesses/opencode/agent-builder.js +1 -1
- package/dist/integrations/harnesses/opencode/index.js +2 -0
- package/dist/integrations/harnesses/opencode/session-log.js +173 -3
- package/dist/integrations/harnesses/opencode-sdk/index.js +2 -2
- package/dist/integrations/harnesses/opencode-sdk/sdk-runner.js +98 -17
- package/dist/integrations/harnesses/types.js +1 -0
- package/dist/integrations/session-logs/index.js +16 -0
- package/dist/llm/call-ai.js +2 -2
- package/dist/llm/client.js +34 -11
- package/dist/llm/embedder.js +67 -4
- package/dist/llm/embedders/cache.js +3 -1
- package/dist/llm/embedders/deterministic.js +66 -0
- package/dist/llm/embedders/local.js +73 -3
- package/dist/llm/feature-gate.js +16 -15
- package/dist/llm/graph-extract.js +67 -44
- package/dist/llm/memory-infer-impl.js +138 -0
- package/dist/llm/memory-infer.js +1 -127
- package/dist/llm/metadata-enhance.js +44 -31
- package/dist/llm/structured-call.js +49 -0
- package/dist/migrate-storage-node.mjs +8 -0
- package/dist/output/context.js +5 -5
- package/dist/output/renderers.js +85 -14
- package/dist/output/shapes/curate.js +14 -2
- package/dist/output/shapes/helpers.js +0 -3
- package/dist/output/shapes/passthrough.js +2 -1
- package/dist/output/text/helpers.js +29 -1
- package/dist/output/text/workflow.js +1 -0
- package/dist/registry/providers/skills-sh.js +21 -147
- package/dist/registry/providers/static-index.js +15 -157
- package/dist/registry/resolve.js +27 -9
- package/dist/runtime.js +25 -1
- package/dist/scripts/migrate-storage.js +2661 -2369
- package/dist/scripts/migrations/import-fs-improve-runs-to-db.js +883 -596
- package/dist/setup/detect.js +9 -0
- package/dist/setup/legacy-config.js +106 -0
- package/dist/setup/prompt.js +57 -0
- package/dist/setup/providers.js +14 -0
- package/dist/setup/registry-stash-loader.js +12 -0
- package/dist/setup/semantic-assets.js +124 -0
- package/dist/setup/setup.js +52 -1614
- package/dist/setup/steps/connection.js +734 -0
- package/dist/setup/steps/output.js +31 -0
- package/dist/setup/steps/platforms.js +124 -0
- package/dist/setup/steps/semantic.js +27 -0
- package/dist/setup/steps/sources.js +222 -0
- package/dist/setup/steps/stashdir.js +42 -0
- package/dist/setup/steps/tasks.js +152 -0
- package/dist/sources/include.js +6 -2
- package/dist/sources/providers/filesystem.js +0 -1
- package/dist/sources/providers/git-install.js +210 -0
- package/dist/sources/providers/git-provider.js +234 -0
- package/dist/sources/providers/git-stash.js +248 -0
- package/dist/sources/providers/git.js +10 -661
- package/dist/sources/providers/npm.js +2 -6
- package/dist/sources/providers/provider-utils.js +13 -7
- package/dist/sources/providers/sync-from-ref.js +9 -1
- package/dist/sources/providers/website.js +9 -5
- package/dist/sources/website-ingest.js +187 -29
- package/dist/sources/wiki-fetchers/registry.js +53 -0
- package/dist/sources/wiki-fetchers/youtube.js +239 -0
- package/dist/storage/database.js +45 -10
- package/dist/storage/managed-db.js +82 -0
- package/dist/storage/repositories/canaries-repository.js +107 -0
- package/dist/storage/repositories/consolidation-repository.js +38 -0
- package/dist/storage/repositories/embeddings-repository.js +72 -0
- package/dist/storage/repositories/events-repository.js +187 -0
- package/dist/storage/repositories/extract-sessions-repository.js +96 -0
- package/dist/storage/repositories/improve-runs-repository.js +146 -0
- package/dist/storage/repositories/index-db.js +14 -8
- package/dist/storage/repositories/proposals-repository.js +220 -0
- package/dist/storage/repositories/recombine-repository.js +213 -0
- package/dist/storage/repositories/registry-cache.js +93 -0
- package/dist/storage/repositories/registry-index-cache-repository.js +46 -0
- package/dist/storage/repositories/task-history-repository.js +93 -0
- package/dist/storage/sqlite-pragmas.js +146 -0
- package/dist/tasks/backends/cron.js +1 -1
- package/dist/tasks/backends/index.js +9 -0
- package/dist/tasks/backends/launchd.js +1 -1
- package/dist/tasks/backends/schtasks.js +1 -1
- package/dist/tasks/{resolveAkmBin.js → resolve-akm-bin.js} +2 -2
- package/dist/tasks/runner.js +15 -13
- package/dist/text-import-hook.mjs +0 -0
- package/dist/wiki/wiki.js +52 -11
- package/dist/workflows/cli.js +1 -0
- package/dist/workflows/db.js +3 -4
- package/dist/workflows/runtime/runs.js +43 -118
- package/dist/workflows/runtime/workflow-asset-loader.js +125 -0
- package/dist/workflows/validate-summary.js +2 -7
- package/docs/README.md +69 -18
- package/docs/data-and-telemetry.md +5 -4
- package/docs/migration/release-notes/0.7.0.md +1 -1
- package/docs/migration/release-notes/0.9.0.md +39 -0
- package/package.json +10 -10
- package/dist/assets/tasks/core/update-stashes.yml +0 -4
- package/dist/commands/db-cli.js +0 -23
- package/dist/indexer/db/db-backup.js +0 -376
- package/dist/indexer/passes/staleness-detect.js +0 -488
|
@@ -0,0 +1,434 @@
|
|
|
1
|
+
// This Source Code Form is subject to the terms of the Mozilla Public
|
|
2
|
+
// License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
3
|
+
// file, You can obtain one at https://mozilla.org/MPL/2.0/.
|
|
4
|
+
import fs from "node:fs";
|
|
5
|
+
import path from "node:path";
|
|
6
|
+
import { makeAssetRef, parseAssetRef } from "../../core/asset/asset-ref.js";
|
|
7
|
+
import { parseFrontmatter } from "../../core/asset/frontmatter.js";
|
|
8
|
+
import { isAssetType } from "../../core/common.js";
|
|
9
|
+
import { NotFoundError, rethrowIfTestIsolationError, UsageError } from "../../core/errors.js";
|
|
10
|
+
import { readEvents } from "../../core/events.js";
|
|
11
|
+
import { closeDatabase, getAllEntries, getUtilityScoresByIds, openExistingDatabase } from "../../indexer/db/db.js";
|
|
12
|
+
import { getWritableStashDirs, resolveSourceEntries } from "../../indexer/search/search-source.js";
|
|
13
|
+
import { resolveAssetPath } from "../../indexer/walk/path-resolver.js";
|
|
14
|
+
import { isDistillRefusedInputType } from "./distill.js";
|
|
15
|
+
import { isProfileFilteredForAllPasses } from "./improve-profiles.js";
|
|
16
|
+
// Eligibility / candidate-selection predicates for improve. Free functions
|
|
17
|
+
// (no akmImprove closure state) extracted from improve.ts to shrink the
|
|
18
|
+
// orchestrator and make candidate selection independently testable.
|
|
19
|
+
export function resolveImproveScope(scope) {
|
|
20
|
+
const trimmed = scope?.trim();
|
|
21
|
+
if (!trimmed)
|
|
22
|
+
return { mode: "all" };
|
|
23
|
+
try {
|
|
24
|
+
parseAssetRef(trimmed);
|
|
25
|
+
return { mode: "ref", value: trimmed };
|
|
26
|
+
}
|
|
27
|
+
catch {
|
|
28
|
+
if (!isAssetType(trimmed)) {
|
|
29
|
+
throw new UsageError(`Unknown asset type: "${trimmed}". Valid types: memory, knowledge, skill, lesson, workflow, agent, command, script, wiki, env, secret, task.\n` +
|
|
30
|
+
`If you passed --format to akm improve, that flag is not supported — use it with akm search or akm show instead.`, "INVALID_FLAG_VALUE");
|
|
31
|
+
}
|
|
32
|
+
return { mode: "type", value: trimmed };
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
/**
|
|
36
|
+
* Dedupe a list of eligible refs by `ref`, preserving first-seen order. Used to
|
|
37
|
+
* merge the three eligibility sources (feedback-signal, P0-A high-retrieval,
|
|
38
|
+
* Layer-2 proactive-maintenance) without admitting a ref into the loop twice.
|
|
39
|
+
*/
|
|
40
|
+
export function dedupeRefs(refs) {
|
|
41
|
+
const seen = new Set();
|
|
42
|
+
const out = [];
|
|
43
|
+
for (const r of refs) {
|
|
44
|
+
if (seen.has(r.ref))
|
|
45
|
+
continue;
|
|
46
|
+
seen.add(r.ref);
|
|
47
|
+
out.push(r);
|
|
48
|
+
}
|
|
49
|
+
return out;
|
|
50
|
+
}
|
|
51
|
+
export async function collectEligibleRefs(scope, stashDir, improveProfile) {
|
|
52
|
+
if (scope.mode === "ref" && scope.value) {
|
|
53
|
+
const parsed = parseAssetRef(scope.value);
|
|
54
|
+
const writableDirs = new Set(getWritableStashDirs(stashDir).map((dir) => path.resolve(dir)));
|
|
55
|
+
const filePath = await findAssetFilePath(scope.value, stashDir, writableDirs);
|
|
56
|
+
if (!filePath) {
|
|
57
|
+
return {
|
|
58
|
+
plannedRefs: [],
|
|
59
|
+
memorySummary: { eligible: 0, derived: 0 },
|
|
60
|
+
profileFilteredRefs: [],
|
|
61
|
+
};
|
|
62
|
+
}
|
|
63
|
+
return {
|
|
64
|
+
plannedRefs: [{ ref: scope.value, reason: "scope-ref", filePath }],
|
|
65
|
+
memorySummary: {
|
|
66
|
+
eligible: parsed.type === "memory" ? 1 : 0,
|
|
67
|
+
derived: parsed.type === "memory" && parsed.name.endsWith(".derived") ? 1 : 0,
|
|
68
|
+
},
|
|
69
|
+
profileFilteredRefs: [],
|
|
70
|
+
};
|
|
71
|
+
}
|
|
72
|
+
let sources;
|
|
73
|
+
try {
|
|
74
|
+
sources = resolveSourceEntries(stashDir);
|
|
75
|
+
}
|
|
76
|
+
catch {
|
|
77
|
+
return { plannedRefs: [], memorySummary: { eligible: 0, derived: 0 }, profileFilteredRefs: [] };
|
|
78
|
+
}
|
|
79
|
+
if (sources.length === 0) {
|
|
80
|
+
return { plannedRefs: [], memorySummary: { eligible: 0, derived: 0 }, profileFilteredRefs: [] };
|
|
81
|
+
}
|
|
82
|
+
// Only operate on writable sources — never mutate read-only registry caches
|
|
83
|
+
// or remote stashes that the user did not mark writable.
|
|
84
|
+
let writableDirs;
|
|
85
|
+
try {
|
|
86
|
+
writableDirs = getWritableStashDirs(stashDir);
|
|
87
|
+
}
|
|
88
|
+
catch {
|
|
89
|
+
writableDirs = sources.slice(0, 1).map((s) => s.path); // fallback: primary only
|
|
90
|
+
}
|
|
91
|
+
const writableDirSet = new Set(writableDirs.map((d) => path.resolve(d)));
|
|
92
|
+
let db;
|
|
93
|
+
try {
|
|
94
|
+
db = openExistingDatabase();
|
|
95
|
+
const entries = getAllEntries(db, scope.mode === "type" ? scope.value : undefined).filter((indexed) => {
|
|
96
|
+
// First apply the existing stashDir-scope filter (no-op when stashDir is unset).
|
|
97
|
+
if (!isEntryInScope(indexed.stashDir, indexed.filePath, stashDir))
|
|
98
|
+
return false;
|
|
99
|
+
// Then restrict to writable sources only.
|
|
100
|
+
return isEntryInWritableSource(indexed.stashDir, indexed.filePath, writableDirSet);
|
|
101
|
+
});
|
|
102
|
+
const planned = new Map();
|
|
103
|
+
const profileFiltered = new Map();
|
|
104
|
+
let memoryEligible = 0;
|
|
105
|
+
let memoryDerived = 0;
|
|
106
|
+
for (const indexed of entries) {
|
|
107
|
+
const ref = makeAssetRef(indexed.entry.type, indexed.entry.name);
|
|
108
|
+
const isDerived = indexed.entry.name.endsWith(".derived");
|
|
109
|
+
// `.derived` memories are LLM-inferred and intentionally skip reflect
|
|
110
|
+
// (see the synthetic `derived-memory-reflect-skipped` branch in the
|
|
111
|
+
// improve loop). Enqueueing them here just produced one synthetic skip
|
|
112
|
+
// per derived memory per hour with no real work — pure churn observed
|
|
113
|
+
// 2026-05-21: 11 derived refs re-planned every hour during idle periods.
|
|
114
|
+
// The cleanup phase (analyzeMemoryCleanup) inspects derived memories
|
|
115
|
+
// independently of `plannedRefs`, so dropping them here loses nothing.
|
|
116
|
+
if (!isDerived && !planned.has(ref) && !profileFiltered.has(ref)) {
|
|
117
|
+
// 2026-05-27: extend the .derived precedent to profile-incompatible
|
|
118
|
+
// refs. If every per-ref pass (reflect + distill) on the active
|
|
119
|
+
// profile would refuse this ref, drop it from `plannedRefs`. The
|
|
120
|
+
// caller emits `improve_skipped { reason: profile_filtered_all_passes }`
|
|
121
|
+
// once `eventsCtx` is available so the audit trail is preserved in a
|
|
122
|
+
// single event per ref instead of 2× synthetic actions per run.
|
|
123
|
+
// Background: see /tmp/akm-health-investigations/planner-profile-metrics-deep-analysis.md
|
|
124
|
+
if (improveProfile && isProfileFilteredForAllPasses(ref, improveProfile)) {
|
|
125
|
+
profileFiltered.set(ref, {
|
|
126
|
+
ref,
|
|
127
|
+
reason: "profile_filtered_all_passes",
|
|
128
|
+
filePath: indexed.filePath,
|
|
129
|
+
});
|
|
130
|
+
}
|
|
131
|
+
else {
|
|
132
|
+
planned.set(ref, {
|
|
133
|
+
ref,
|
|
134
|
+
reason: scope.mode === "type" ? "scope-type" : indexed.entry.type === "memory" ? "memory-cleanup" : "scope-type",
|
|
135
|
+
filePath: indexed.filePath,
|
|
136
|
+
});
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
if (indexed.entry.type === "memory") {
|
|
140
|
+
memoryEligible += 1;
|
|
141
|
+
if (isDerived)
|
|
142
|
+
memoryDerived += 1;
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
return {
|
|
146
|
+
plannedRefs: [...planned.values()],
|
|
147
|
+
memorySummary: { eligible: memoryEligible, derived: memoryDerived },
|
|
148
|
+
profileFilteredRefs: [...profileFiltered.values()],
|
|
149
|
+
};
|
|
150
|
+
}
|
|
151
|
+
catch (error) {
|
|
152
|
+
// The bun-test isolation guard must never be downgraded to "empty plan".
|
|
153
|
+
rethrowIfTestIsolationError(error);
|
|
154
|
+
if (error instanceof NotFoundError || error instanceof Error) {
|
|
155
|
+
return { plannedRefs: [], memorySummary: { eligible: 0, derived: 0 }, profileFilteredRefs: [] };
|
|
156
|
+
}
|
|
157
|
+
throw error;
|
|
158
|
+
}
|
|
159
|
+
finally {
|
|
160
|
+
if (db)
|
|
161
|
+
closeDatabase(db);
|
|
162
|
+
}
|
|
163
|
+
}
|
|
164
|
+
export function isEntryInScope(entryStashDir, filePath, stashDir) {
|
|
165
|
+
if (!stashDir)
|
|
166
|
+
return true;
|
|
167
|
+
const resolvedEntryStashDir = path.resolve(entryStashDir);
|
|
168
|
+
const resolvedFilePath = path.resolve(filePath);
|
|
169
|
+
const resolvedScopeStashDir = path.resolve(stashDir);
|
|
170
|
+
return (resolvedEntryStashDir === resolvedScopeStashDir ||
|
|
171
|
+
resolvedEntryStashDir.startsWith(`${resolvedScopeStashDir}${path.sep}`) ||
|
|
172
|
+
resolvedFilePath.startsWith(`${resolvedScopeStashDir}${path.sep}`));
|
|
173
|
+
}
|
|
174
|
+
/**
|
|
175
|
+
* Return true when the indexed entry belongs to one of the writable source
|
|
176
|
+
* directories. Entries from read-only registry caches or remote stashes that
|
|
177
|
+
* the user has not marked writable must never enter the improve/distill loop.
|
|
178
|
+
*/
|
|
179
|
+
export function isEntryInWritableSource(entryStashDir, filePath, writableDirSet) {
|
|
180
|
+
const resolvedEntryStashDir = path.resolve(entryStashDir);
|
|
181
|
+
const resolvedFilePath = path.resolve(filePath);
|
|
182
|
+
for (const writableDir of writableDirSet) {
|
|
183
|
+
if (resolvedEntryStashDir === writableDir ||
|
|
184
|
+
resolvedEntryStashDir.startsWith(`${writableDir}${path.sep}`) ||
|
|
185
|
+
resolvedFilePath.startsWith(`${writableDir}${path.sep}`)) {
|
|
186
|
+
return true;
|
|
187
|
+
}
|
|
188
|
+
}
|
|
189
|
+
return false;
|
|
190
|
+
}
|
|
191
|
+
export function memoryCleanupParentRef(scope, stashDir) {
|
|
192
|
+
if (scope.mode !== "ref" || !scope.value)
|
|
193
|
+
return undefined;
|
|
194
|
+
const parsed = parseAssetRef(scope.value);
|
|
195
|
+
if (parsed.type !== "memory")
|
|
196
|
+
return undefined;
|
|
197
|
+
if (!parsed.name.endsWith(".derived"))
|
|
198
|
+
return scope.value;
|
|
199
|
+
const sources = resolveSourceEntries(stashDir);
|
|
200
|
+
for (const source of sources) {
|
|
201
|
+
const candidate = path.join(source.path, "memories", `${parsed.name}.md`);
|
|
202
|
+
if (!fs.existsSync(candidate))
|
|
203
|
+
continue;
|
|
204
|
+
const raw = fs.readFileSync(candidate, "utf8");
|
|
205
|
+
const fm = parseFrontmatter(raw).data;
|
|
206
|
+
const sourceRef = typeof fm.source === "string" ? fm.source : undefined;
|
|
207
|
+
if (sourceRef) {
|
|
208
|
+
try {
|
|
209
|
+
const parent = parseAssetRef(sourceRef.trim());
|
|
210
|
+
if (parent.type === "memory")
|
|
211
|
+
return makeAssetRef(parent.type, parent.name);
|
|
212
|
+
}
|
|
213
|
+
catch { }
|
|
214
|
+
}
|
|
215
|
+
}
|
|
216
|
+
return makeAssetRef("memory", parsed.name.slice(0, -".derived".length));
|
|
217
|
+
}
|
|
218
|
+
export function isLessonCandidate(ref) {
|
|
219
|
+
// Only lesson assets need lesson-schema validation (description + when_to_use).
|
|
220
|
+
// Memories have their own distill path via shouldDistillMemoryRef.
|
|
221
|
+
// All other types go through reflect, not distill.
|
|
222
|
+
return parseAssetRef(ref).type === "lesson";
|
|
223
|
+
}
|
|
224
|
+
/**
|
|
225
|
+
* Planner-side check: should this ref enter the distill queue?
|
|
226
|
+
*
|
|
227
|
+
* Distill produces lessons from non-lesson sources. Two cases are eligible:
|
|
228
|
+
*
|
|
229
|
+
* 1. Memory refs that pass {@link shouldDistillMemoryRef} (the existing
|
|
230
|
+
* memory→lesson/knowledge promotion path).
|
|
231
|
+
*
|
|
232
|
+
* Refs whose `type` is in {@link DISTILL_REFUSED_INPUT_TYPES} (currently
|
|
233
|
+
* `lesson:*`) are explicitly excluded — distill refuses them at runtime and
|
|
234
|
+
* queuing them just produces a no-op `skipped` outcome per ref per hour. That
|
|
235
|
+
* planner waste was the bug fixed in commit
|
|
236
|
+
* fix(improve): drop distill-refused types from planner.
|
|
237
|
+
*
|
|
238
|
+
* Note: prior to this fix the gate used `isLessonCandidate(ref)` directly,
|
|
239
|
+
* which was true *only* for `lesson:*` refs — exactly the set distill refuses.
|
|
240
|
+
* The result: every hourly run re-queued the same lesson refs, the same skip
|
|
241
|
+
* message returned, and no work was ever done. See
|
|
242
|
+
* `tests/commands/improve-distill-planner-skip-lessons.test.ts`.
|
|
243
|
+
*/
|
|
244
|
+
export function isDistillCandidateRef(ref, stashDir) {
|
|
245
|
+
const parsed = parseAssetRef(ref);
|
|
246
|
+
if (isDistillRefusedInputType(parsed.type))
|
|
247
|
+
return false;
|
|
248
|
+
return shouldDistillMemoryRef(ref, stashDir);
|
|
249
|
+
}
|
|
250
|
+
export function shouldDistillMemoryRef(ref, stashDir) {
|
|
251
|
+
const parsed = parseAssetRef(ref);
|
|
252
|
+
if (parsed.type !== "memory")
|
|
253
|
+
return false;
|
|
254
|
+
const sources = resolveSourceEntries(stashDir);
|
|
255
|
+
for (const source of sources) {
|
|
256
|
+
const candidate = `${source.path}/memories/${parsed.name}.md`;
|
|
257
|
+
if (!fs.existsSync(candidate))
|
|
258
|
+
continue;
|
|
259
|
+
const raw = fs.readFileSync(candidate, "utf8");
|
|
260
|
+
const fm = parseFrontmatter(raw).data;
|
|
261
|
+
const quality = typeof fm.quality === "string" ? fm.quality : undefined;
|
|
262
|
+
if (quality === "proposed")
|
|
263
|
+
return false;
|
|
264
|
+
return !parsed.name.endsWith(".derived");
|
|
265
|
+
}
|
|
266
|
+
return !parsed.name.endsWith(".derived");
|
|
267
|
+
}
|
|
268
|
+
// ── Signal-delta eligibility helpers (0.8.0) ────────────────────────────────
|
|
269
|
+
//
|
|
270
|
+
// The 0.8.0 redesign replaced flat time-based cooldowns for reflect/distill
|
|
271
|
+
// with a *signal-delta* gate: a ref is re-eligible iff new feedback has
|
|
272
|
+
// landed since the last proposal was generated for it. These helpers build
|
|
273
|
+
// the two timestamp maps the gate needs in bulk, so the planner avoids
|
|
274
|
+
// N+1 queries across the full postCleanupRefs set.
|
|
275
|
+
/**
|
|
276
|
+
* Latest feedback event timestamp per ref in the active window. Reads all
|
|
277
|
+
* `feedback` events newer than `sinceIso` in one query and indexes by ref,
|
|
278
|
+
* keeping the maximum `ts` per ref.
|
|
279
|
+
*
|
|
280
|
+
* Only events with a meaningful payload count as "signal" — `metadata.signal`
|
|
281
|
+
* (positive/negative) OR `metadata.note` (a free-form annotation). Empty
|
|
282
|
+
* metadata events are ignored so a stray `akm feedback <ref>` invocation
|
|
283
|
+
* without a flag doesn't trigger downstream re-processing.
|
|
284
|
+
*/
|
|
285
|
+
export function buildLatestFeedbackTsMap(refs, sinceIso) {
|
|
286
|
+
const out = new Map();
|
|
287
|
+
if (refs.length === 0)
|
|
288
|
+
return out;
|
|
289
|
+
const refSet = new Set(refs);
|
|
290
|
+
const { events } = readEvents({ type: "feedback", since: sinceIso });
|
|
291
|
+
for (const e of events) {
|
|
292
|
+
const ref = e.ref;
|
|
293
|
+
if (!ref || !refSet.has(ref))
|
|
294
|
+
continue;
|
|
295
|
+
const meta = e.metadata;
|
|
296
|
+
const hasSignal = meta !== undefined && (typeof meta.signal === "string" || typeof meta.note === "string");
|
|
297
|
+
if (!hasSignal)
|
|
298
|
+
continue;
|
|
299
|
+
const ts = e.ts ?? "";
|
|
300
|
+
if (ts > (out.get(ref) ?? ""))
|
|
301
|
+
out.set(ref, ts);
|
|
302
|
+
}
|
|
303
|
+
return out;
|
|
304
|
+
}
|
|
305
|
+
/**
|
|
306
|
+
* Latest proposal timestamp per input-ref, filtered by source ('reflect' or
|
|
307
|
+
* 'distill'). Reads the corresponding `*_invoked` events from state.db —
|
|
308
|
+
* these events are emitted at proposal creation time and carry the *input*
|
|
309
|
+
* asset ref (memory:foo, skill:bar, etc.) directly. We use them rather than
|
|
310
|
+
* `listProposals` because distill proposals are keyed by the derived
|
|
311
|
+
* lesson/knowledge ref, not the source memory — joining back through the
|
|
312
|
+
* payload would be fragile.
|
|
313
|
+
*/
|
|
314
|
+
export function buildLatestProposalTsMap(refs, source) {
|
|
315
|
+
const out = new Map();
|
|
316
|
+
if (refs.length === 0)
|
|
317
|
+
return out;
|
|
318
|
+
const refSet = new Set(refs);
|
|
319
|
+
const eventType = source === "reflect" ? "reflect_invoked" : "distill_invoked";
|
|
320
|
+
const { events } = readEvents({ type: eventType });
|
|
321
|
+
for (const e of events) {
|
|
322
|
+
const ref = e.ref;
|
|
323
|
+
if (!ref || !refSet.has(ref))
|
|
324
|
+
continue;
|
|
325
|
+
// For distill_invoked we only count attempts that produced (or attempted
|
|
326
|
+
// to produce) a real proposal — config_disabled / parse-error outcomes
|
|
327
|
+
// should not move the signal-delta cursor forward.
|
|
328
|
+
if (eventType === "distill_invoked") {
|
|
329
|
+
const outcome = e.metadata?.outcome;
|
|
330
|
+
if (outcome !== "queued" && outcome !== "skipped" && outcome !== "validation_failed")
|
|
331
|
+
continue;
|
|
332
|
+
}
|
|
333
|
+
const ts = e.ts ?? "";
|
|
334
|
+
if (ts > (out.get(ref) ?? ""))
|
|
335
|
+
out.set(ref, ts);
|
|
336
|
+
}
|
|
337
|
+
return out;
|
|
338
|
+
}
|
|
339
|
+
/**
|
|
340
|
+
* Signal-delta eligibility predicate.
|
|
341
|
+
*
|
|
342
|
+
* True iff `latestFeedback[ref]` is defined AND either no prior proposal
|
|
343
|
+
* exists for this (ref, source) OR `latestFeedback[ref] > lastProposal[ref]`.
|
|
344
|
+
*
|
|
345
|
+
* Refs with no feedback signal at all are ineligible by definition — the
|
|
346
|
+
* high-retrieval fallback path (see `noFeedbackCandidates` later in the
|
|
347
|
+
* planner) handles never-touched-but-frequently-read assets separately.
|
|
348
|
+
*/
|
|
349
|
+
export function isSignalDeltaEligible(ref, latestFeedback, lastProposal) {
|
|
350
|
+
const fb = latestFeedback.get(ref);
|
|
351
|
+
if (!fb)
|
|
352
|
+
return false;
|
|
353
|
+
const lp = lastProposal.get(ref);
|
|
354
|
+
if (!lp)
|
|
355
|
+
return true;
|
|
356
|
+
return fb > lp;
|
|
357
|
+
}
|
|
358
|
+
/**
|
|
359
|
+
* H7 (#566): cooperative budget watchdog with a captured, RAII-cleared hard-kill.
|
|
360
|
+
*
|
|
361
|
+
* When the wall-clock budget expires, `onExhausted` (normally an
|
|
362
|
+
* `AbortController.abort`) signals cooperative cancellation so the run can drain
|
|
363
|
+
* its in-flight log/`state.db` flush and unwind naturally. A second hard-kill
|
|
364
|
+
* timer is then armed as a watchdog: it only `exit(0)`s if the drain itself
|
|
365
|
+
* overruns `hardKillGraceMs`, preventing the process from outliving the task
|
|
366
|
+
* timeout window (lock-cascade fix).
|
|
367
|
+
*
|
|
368
|
+
* Both timers are captured; the returned dispose() clears whichever is still
|
|
369
|
+
* pending. Callers invoke it from a `finally`, so a *clean* drain reaches the
|
|
370
|
+
* `finally` and cancels the pending hard-kill before it can fire — the previous
|
|
371
|
+
* detached `setTimeout(() => process.exit(0), 5000)` always fired, truncating a
|
|
372
|
+
* clean flush. The hard-kill timer is `unref()`-ed so it never keeps the event
|
|
373
|
+
* loop alive on its own: once the run drains it exits with its own code, not the
|
|
374
|
+
* forced 0.
|
|
375
|
+
*
|
|
376
|
+
* Dependencies are injectable purely so the concurrency-sensitive timing
|
|
377
|
+
* contract can be exercised deterministically in unit tests.
|
|
378
|
+
*/
|
|
379
|
+
export function shouldAnalyzeMemoryCleanup(scope, eligibleMemories, primaryStashDir) {
|
|
380
|
+
if (!primaryStashDir || eligibleMemories === 0)
|
|
381
|
+
return false;
|
|
382
|
+
if (scope.mode === "all")
|
|
383
|
+
return true;
|
|
384
|
+
if (scope.mode === "type")
|
|
385
|
+
return scope.value === "memory";
|
|
386
|
+
if (!scope.value)
|
|
387
|
+
return false;
|
|
388
|
+
return parseAssetRef(scope.value).type === "memory";
|
|
389
|
+
}
|
|
390
|
+
export function buildUtilityMap(refs) {
|
|
391
|
+
const map = new Map();
|
|
392
|
+
if (refs.length === 0)
|
|
393
|
+
return map;
|
|
394
|
+
const refSet = new Set(refs.map((r) => r.ref));
|
|
395
|
+
let db;
|
|
396
|
+
try {
|
|
397
|
+
db = openExistingDatabase();
|
|
398
|
+
const allDbEntries = getAllEntries(db);
|
|
399
|
+
const idToRef = new Map();
|
|
400
|
+
for (const indexed of allDbEntries) {
|
|
401
|
+
const ref = makeAssetRef(indexed.entry.type, indexed.entry.name);
|
|
402
|
+
if (refSet.has(ref))
|
|
403
|
+
idToRef.set(indexed.id, ref);
|
|
404
|
+
}
|
|
405
|
+
const ids = [...idToRef.keys()];
|
|
406
|
+
if (ids.length > 0) {
|
|
407
|
+
const { global: scores } = getUtilityScoresByIds(db, ids);
|
|
408
|
+
for (const [id, score] of scores) {
|
|
409
|
+
const ref = idToRef.get(id);
|
|
410
|
+
if (ref)
|
|
411
|
+
map.set(ref, score.utility);
|
|
412
|
+
}
|
|
413
|
+
}
|
|
414
|
+
}
|
|
415
|
+
catch (err) {
|
|
416
|
+
rethrowIfTestIsolationError(err);
|
|
417
|
+
// best-effort: if DB unavailable, all utilities default to 0
|
|
418
|
+
}
|
|
419
|
+
finally {
|
|
420
|
+
if (db)
|
|
421
|
+
closeDatabase(db);
|
|
422
|
+
}
|
|
423
|
+
return map;
|
|
424
|
+
}
|
|
425
|
+
export async function findAssetFilePath(ref, stashDir, writableDirSet) {
|
|
426
|
+
return resolveAssetPath(ref, {
|
|
427
|
+
stashDir,
|
|
428
|
+
mode: "disk-only",
|
|
429
|
+
writableDirSet,
|
|
430
|
+
directoryIndexNames: ["SKILL.md"],
|
|
431
|
+
preserveDirectNameFallback: true,
|
|
432
|
+
honorOrigin: false,
|
|
433
|
+
});
|
|
434
|
+
}
|
|
@@ -0,0 +1,205 @@
|
|
|
1
|
+
// This Source Code Form is subject to the terms of the Mozilla Public
|
|
2
|
+
// License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
3
|
+
// file, You can obtain one at https://mozilla.org/MPL/2.0/.
|
|
4
|
+
/**
|
|
5
|
+
* Encoding-time salience scoring for issue #608.
|
|
6
|
+
*
|
|
7
|
+
* Pure module — no I/O. All inputs are pre-fetched by the caller.
|
|
8
|
+
* Implements the three-signal model: novelty × 0.40 + magnitude × 0.35 +
|
|
9
|
+
* predictionError × 0.25, clamped to [0, 1].
|
|
10
|
+
*/
|
|
11
|
+
// ── Weights ────────────────────────────────────────────────────────────────────
|
|
12
|
+
const W_NOVELTY = 0.4;
|
|
13
|
+
const W_MAGNITUDE = 0.35;
|
|
14
|
+
const W_PREDICTION_ERROR = 0.25;
|
|
15
|
+
// Guard: weights must sum to 1.0 (consistent with the pattern in salience.ts).
|
|
16
|
+
// This check fires at module load and catches any future re-tuning mistakes.
|
|
17
|
+
if (Math.abs(W_NOVELTY + W_MAGNITUDE + W_PREDICTION_ERROR - 1.0) > 1e-9) {
|
|
18
|
+
throw new Error(`encoding-salience.ts: sub-signal weights must sum to 1.0 (got ${W_NOVELTY + W_MAGNITUDE + W_PREDICTION_ERROR})`);
|
|
19
|
+
}
|
|
20
|
+
// ── Novelty type floors ────────────────────────────────────────────────────────
|
|
21
|
+
const TYPE_NOVELTY_FLOOR = Object.freeze({
|
|
22
|
+
skill: 0.8,
|
|
23
|
+
agent: 0.8,
|
|
24
|
+
memory: 0.4,
|
|
25
|
+
});
|
|
26
|
+
const DEFAULT_NOVELTY_FLOOR = 0.5;
|
|
27
|
+
// ── Magnitude keyword sets ──────────────────────────────────────────────────────
|
|
28
|
+
const SEVERITY_BUCKET = new Set(["error", "critical", "warning", "incident", "regression"]);
|
|
29
|
+
// Note: "fails" and "urgent" are from the spec's initial keyword list and are included
|
|
30
|
+
// here as strong constraint signals, even though the spec's final bucket table omits them.
|
|
31
|
+
const CONSTRAINT_BUCKET = new Set(["must", "never", "always", "blocked", "breaking", "deprecated", "fails", "urgent"]);
|
|
32
|
+
const ALL_MAGNITUDE_KEYWORDS = new Set([...SEVERITY_BUCKET, ...CONSTRAINT_BUCKET]);
|
|
33
|
+
// Number of distinct keyword matches to reach full magnitude score.
|
|
34
|
+
const MAGNITUDE_FULL_SCORE_THRESHOLD = 4;
|
|
35
|
+
// Single-bucket magnitude cap: when only one semantic bucket matches.
|
|
36
|
+
const MAGNITUDE_SINGLE_BUCKET_CAP = 0.5;
|
|
37
|
+
// ── Stop-words (excluded from bigram tokenization) ────────────────────────────
|
|
38
|
+
const STOP_WORDS = new Set([
|
|
39
|
+
"a",
|
|
40
|
+
"an",
|
|
41
|
+
"the",
|
|
42
|
+
"and",
|
|
43
|
+
"or",
|
|
44
|
+
"but",
|
|
45
|
+
"in",
|
|
46
|
+
"on",
|
|
47
|
+
"at",
|
|
48
|
+
"to",
|
|
49
|
+
"for",
|
|
50
|
+
"of",
|
|
51
|
+
"with",
|
|
52
|
+
"by",
|
|
53
|
+
"from",
|
|
54
|
+
"as",
|
|
55
|
+
"is",
|
|
56
|
+
"it",
|
|
57
|
+
"its",
|
|
58
|
+
"be",
|
|
59
|
+
"are",
|
|
60
|
+
"was",
|
|
61
|
+
"were",
|
|
62
|
+
"has",
|
|
63
|
+
"have",
|
|
64
|
+
"had",
|
|
65
|
+
"do",
|
|
66
|
+
"does",
|
|
67
|
+
"did",
|
|
68
|
+
"not",
|
|
69
|
+
"this",
|
|
70
|
+
"that",
|
|
71
|
+
"these",
|
|
72
|
+
"those",
|
|
73
|
+
"i",
|
|
74
|
+
"we",
|
|
75
|
+
"you",
|
|
76
|
+
"he",
|
|
77
|
+
"she",
|
|
78
|
+
"they",
|
|
79
|
+
"my",
|
|
80
|
+
"our",
|
|
81
|
+
"your",
|
|
82
|
+
"his",
|
|
83
|
+
"her",
|
|
84
|
+
"their",
|
|
85
|
+
"can",
|
|
86
|
+
"will",
|
|
87
|
+
"would",
|
|
88
|
+
"should",
|
|
89
|
+
"could",
|
|
90
|
+
"may",
|
|
91
|
+
"might",
|
|
92
|
+
"shall",
|
|
93
|
+
"so",
|
|
94
|
+
"if",
|
|
95
|
+
"then",
|
|
96
|
+
"than",
|
|
97
|
+
"when",
|
|
98
|
+
"while",
|
|
99
|
+
"what",
|
|
100
|
+
"which",
|
|
101
|
+
"who",
|
|
102
|
+
"how",
|
|
103
|
+
"no",
|
|
104
|
+
"any",
|
|
105
|
+
"all",
|
|
106
|
+
"more",
|
|
107
|
+
"also",
|
|
108
|
+
]);
|
|
109
|
+
// ── Tokenization helpers ──────────────────────────────────────────────────────
|
|
110
|
+
/**
|
|
111
|
+
* Tokenize text into lowercase words, splitting on whitespace and punctuation,
|
|
112
|
+
* dropping stop-words. Used for body bigram extraction (novelty signal).
|
|
113
|
+
*/
|
|
114
|
+
function tokenizeBody(text) {
|
|
115
|
+
return text
|
|
116
|
+
.toLowerCase()
|
|
117
|
+
.split(/[\s\p{P}\p{S}\-_/]+/u)
|
|
118
|
+
.filter((t) => t.length > 1 && !STOP_WORDS.has(t));
|
|
119
|
+
}
|
|
120
|
+
/**
|
|
121
|
+
* Tokenize a ref name or tag into lowercase words without stop-word filtering.
|
|
122
|
+
* Ref names are identifiers, not prose — all tokens are significant.
|
|
123
|
+
*/
|
|
124
|
+
function tokenizeRef(ref) {
|
|
125
|
+
return ref
|
|
126
|
+
.toLowerCase()
|
|
127
|
+
.split(/[\s\p{P}\p{S}\-_/]+/u)
|
|
128
|
+
.filter((t) => t.length > 0);
|
|
129
|
+
}
|
|
130
|
+
/**
|
|
131
|
+
* Generate all consecutive bigrams from an array of tokens.
|
|
132
|
+
* Returns them as "token1 token2" strings.
|
|
133
|
+
*/
|
|
134
|
+
function bigrams(tokens) {
|
|
135
|
+
const result = [];
|
|
136
|
+
for (let i = 0; i < tokens.length - 1; i++) {
|
|
137
|
+
result.push(`${tokens[i]} ${tokens[i + 1]}`);
|
|
138
|
+
}
|
|
139
|
+
return result;
|
|
140
|
+
}
|
|
141
|
+
/**
|
|
142
|
+
* Build the existing-ref vocabulary: a Set of bigrams derived from tokenizing
|
|
143
|
+
* the provided ref names (and tags). Pass this to `scoreEncodingSalience` as
|
|
144
|
+
* `existingRefVocabulary` so the novelty signal can measure how much of the
|
|
145
|
+
* asset body is already represented in the stash vocabulary.
|
|
146
|
+
*/
|
|
147
|
+
export function buildRefVocabulary(refs) {
|
|
148
|
+
const vocab = new Set();
|
|
149
|
+
for (const ref of refs) {
|
|
150
|
+
const tokens = tokenizeRef(ref);
|
|
151
|
+
for (const bg of bigrams(tokens)) {
|
|
152
|
+
vocab.add(bg);
|
|
153
|
+
}
|
|
154
|
+
}
|
|
155
|
+
return vocab;
|
|
156
|
+
}
|
|
157
|
+
// ── Sub-signal computations ────────────────────────────────────────────────────
|
|
158
|
+
function computeNovelty(body, type, vocab) {
|
|
159
|
+
const tokens = tokenizeBody(body);
|
|
160
|
+
const bgs = bigrams(tokens);
|
|
161
|
+
const floor = TYPE_NOVELTY_FLOOR[type] ?? DEFAULT_NOVELTY_FLOOR;
|
|
162
|
+
if (bgs.length === 0)
|
|
163
|
+
return floor;
|
|
164
|
+
const novelCount = bgs.filter((bg) => !vocab.has(bg)).length;
|
|
165
|
+
const bigramNoveltyFraction = novelCount / bgs.length;
|
|
166
|
+
return Math.max(floor, bigramNoveltyFraction);
|
|
167
|
+
}
|
|
168
|
+
function computeMagnitude(body) {
|
|
169
|
+
const lowerBody = body.toLowerCase();
|
|
170
|
+
// Split on word boundaries to avoid partial matches (e.g. "errors" matching "error").
|
|
171
|
+
const words = new Set(lowerBody.split(/\W+/).filter((w) => w.length > 0));
|
|
172
|
+
const matched = [...ALL_MAGNITUDE_KEYWORDS].filter((kw) => words.has(kw));
|
|
173
|
+
if (matched.length === 0)
|
|
174
|
+
return 0;
|
|
175
|
+
const distinctCount = matched.length;
|
|
176
|
+
const rawMagnitude = Math.min(1.0, distinctCount / MAGNITUDE_FULL_SCORE_THRESHOLD);
|
|
177
|
+
// Require at least one match from each bucket to lift the single-bucket cap.
|
|
178
|
+
const hasSeverity = matched.some((kw) => SEVERITY_BUCKET.has(kw));
|
|
179
|
+
const hasConstraint = matched.some((kw) => CONSTRAINT_BUCKET.has(kw));
|
|
180
|
+
if (hasSeverity && hasConstraint)
|
|
181
|
+
return rawMagnitude;
|
|
182
|
+
return Math.min(rawMagnitude, MAGNITUDE_SINGLE_BUCKET_CAP);
|
|
183
|
+
}
|
|
184
|
+
function computePredictionError(revisionCount) {
|
|
185
|
+
if (revisionCount === 0)
|
|
186
|
+
return 1.0;
|
|
187
|
+
return 1 / (1 + Math.log(1 + revisionCount));
|
|
188
|
+
}
|
|
189
|
+
// ── Main export ────────────────────────────────────────────────────────────────
|
|
190
|
+
/**
|
|
191
|
+
* Compute the encoding-time salience score for an asset.
|
|
192
|
+
*
|
|
193
|
+
* Three sub-signals:
|
|
194
|
+
* novelty (0.40) — how much of the body is absent from the stash vocabulary.
|
|
195
|
+
* magnitude (0.35) — presence of severity/constraint keywords.
|
|
196
|
+
* predictionError (0.25) — surprise: high for new assets, decays with revisions.
|
|
197
|
+
*/
|
|
198
|
+
export function scoreEncodingSalience(inputs) {
|
|
199
|
+
const novelty = computeNovelty(inputs.body, inputs.type, inputs.existingRefVocabulary);
|
|
200
|
+
const magnitude = computeMagnitude(inputs.body);
|
|
201
|
+
const predictionError = computePredictionError(inputs.revisionCount);
|
|
202
|
+
const raw = W_NOVELTY * novelty + W_MAGNITUDE * magnitude + W_PREDICTION_ERROR * predictionError;
|
|
203
|
+
const score = Math.min(1, Math.max(0, raw));
|
|
204
|
+
return { score, novelty, magnitude, predictionError };
|
|
205
|
+
}
|