akm-cli 0.9.0-beta.45 → 0.9.0-beta.47
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/assets/wiki/ingest-workflow-template.md +17 -10
- package/dist/cli/shared.js +28 -0
- package/dist/cli.js +1 -2
- package/dist/commands/env/env-cli.js +16 -24
- package/dist/commands/env/secret-cli.js +12 -20
- package/dist/commands/graph/graph-cli.js +5 -13
- package/dist/commands/graph/graph.js +3 -3
- package/dist/commands/improve/consolidate/chunking.js +141 -0
- package/dist/commands/improve/consolidate/eligibility.js +64 -0
- package/dist/commands/improve/consolidate/merge.js +145 -0
- package/dist/commands/improve/consolidate/sanitize.js +231 -0
- package/dist/commands/improve/consolidate/types.js +4 -0
- package/dist/commands/improve/consolidate.js +20 -571
- package/dist/commands/improve/distill.js +5 -9
- package/dist/commands/improve/eligibility.js +434 -0
- package/dist/commands/improve/extract-cli.js +9 -1
- package/dist/commands/improve/extract.js +5 -19
- package/dist/commands/improve/improve-auto-accept.js +4 -8
- package/dist/commands/improve/improve-cli.js +35 -60
- package/dist/commands/improve/improve-result-file.js +5 -23
- package/dist/commands/improve/improve-session.js +58 -0
- package/dist/commands/improve/improve.js +107 -3606
- package/dist/commands/improve/locks.js +154 -0
- package/dist/commands/improve/loop-stages.js +1079 -0
- package/dist/commands/improve/preparation.js +1963 -0
- package/dist/commands/improve/recombine.js +6 -12
- package/dist/commands/improve/reflect.js +29 -34
- package/dist/commands/proposal/drain.js +25 -48
- package/dist/commands/proposal/proposal-cli.js +21 -31
- package/dist/commands/proposal/validators/proposals.js +3 -7
- package/dist/commands/read/curate.js +70 -14
- package/dist/commands/read/knowledge.js +2 -2
- package/dist/commands/sources/self-update.js +2 -2
- package/dist/commands/sources/stash-cli.js +9 -37
- package/dist/commands/tasks/tasks-cli.js +19 -27
- package/dist/commands/wiki-cli.js +21 -35
- package/dist/core/config/config.js +18 -2
- package/dist/core/events.js +3 -7
- package/dist/core/logs-db.js +6 -63
- package/dist/core/state/migrations.js +714 -0
- package/dist/core/state-db.js +28 -779
- package/dist/indexer/db/db.js +82 -216
- package/dist/indexer/indexer.js +11 -112
- package/dist/indexer/passes/dir-staleness.js +114 -0
- package/dist/indexer/search/search-source.js +10 -24
- package/dist/indexer/search/semantic-status.js +4 -0
- package/dist/integrations/agent/runner-dispatch.js +59 -0
- package/dist/llm/client.js +22 -11
- package/dist/llm/embedder.js +15 -0
- package/dist/llm/embedders/deterministic.js +66 -0
- package/dist/llm/graph-extract.js +28 -39
- package/dist/llm/memory-infer.js +34 -22
- package/dist/llm/metadata-enhance.js +35 -30
- package/dist/llm/structured-call.js +49 -0
- package/dist/output/shapes/passthrough.js +0 -1
- package/dist/registry/providers/skills-sh.js +21 -147
- package/dist/registry/providers/static-index.js +15 -157
- package/dist/registry/resolve.js +22 -9
- package/dist/scripts/migrate-storage.js +892 -1186
- package/dist/scripts/migrations/import-fs-improve-runs-to-db.js +214 -179
- package/dist/setup/setup.js +26 -5
- package/dist/sources/providers/filesystem.js +0 -1
- package/dist/sources/providers/git-install.js +206 -0
- package/dist/sources/providers/git-provider.js +234 -0
- package/dist/sources/providers/git-stash.js +248 -0
- package/dist/sources/providers/git.js +10 -671
- package/dist/sources/providers/npm.js +2 -6
- package/dist/sources/providers/sync-from-ref.js +9 -1
- package/dist/sources/providers/website.js +2 -3
- package/dist/sources/website-ingest.js +51 -9
- package/dist/sources/wiki-fetchers/registry.js +53 -0
- package/dist/sources/wiki-fetchers/youtube.js +185 -0
- package/dist/storage/database.js +45 -10
- package/dist/storage/managed-db.js +82 -0
- package/dist/storage/repositories/registry-cache.js +92 -0
- package/dist/tasks/runner.js +5 -13
- package/dist/workflows/runtime/runs.js +1 -117
- package/dist/workflows/runtime/workflow-asset-loader.js +125 -0
- package/package.json +5 -5
- package/dist/commands/db-cli.js +0 -23
- package/dist/indexer/db/db-backup.js +0 -376
|
@@ -66,9 +66,9 @@ import { appendEvent, readEvents } from "../../core/events.js";
|
|
|
66
66
|
import { lintLessonContent } from "../../core/lesson-lint.js";
|
|
67
67
|
import { getDbPath } from "../../core/paths.js";
|
|
68
68
|
import { resolveStashStandards } from "../../core/standards/resolve-stash-standards.js";
|
|
69
|
-
import {
|
|
69
|
+
import { withStateDb } from "../../core/state-db.js";
|
|
70
70
|
import { warnVerbose } from "../../core/warn.js";
|
|
71
|
-
import { closeDatabase, getAllEntries,
|
|
71
|
+
import { closeDatabase, getAllEntries, openIndexDatabase } from "../../indexer/db/db.js";
|
|
72
72
|
import { resolveAssetPath } from "../../indexer/walk/path-resolver.js";
|
|
73
73
|
import { chatCompletion, parseEmbeddedJsonResponse } from "../../llm/client.js";
|
|
74
74
|
import { isLlmFeatureEnabled, tryLlmFeature } from "../../llm/feature-gate.js";
|
|
@@ -668,7 +668,7 @@ export async function akmDistill(options) {
|
|
|
668
668
|
let existingRefVocabulary = new Set();
|
|
669
669
|
try {
|
|
670
670
|
const embCfg = config?.embedding;
|
|
671
|
-
const indexDb =
|
|
671
|
+
const indexDb = openIndexDatabase(getDbPath(), embCfg?.dimension ? { embeddingDim: embCfg.dimension } : undefined);
|
|
672
672
|
try {
|
|
673
673
|
const allRefs = getAllEntries(indexDb).map((e) => e.entryKey);
|
|
674
674
|
existingRefVocabulary = buildRefVocabulary(allRefs);
|
|
@@ -694,8 +694,7 @@ export async function akmDistill(options) {
|
|
|
694
694
|
}
|
|
695
695
|
// 2. Persist encoding_salience to state.db.
|
|
696
696
|
try {
|
|
697
|
-
|
|
698
|
-
try {
|
|
697
|
+
withStateDb((stateDb) => {
|
|
699
698
|
const vector = computeSalience({
|
|
700
699
|
ref: inputRef,
|
|
701
700
|
type: parsedRef.type,
|
|
@@ -703,10 +702,7 @@ export async function akmDistill(options) {
|
|
|
703
702
|
encodingSalience: salienceResult.score,
|
|
704
703
|
});
|
|
705
704
|
upsertAssetSalience(stateDb, inputRef, vector);
|
|
706
|
-
}
|
|
707
|
-
finally {
|
|
708
|
-
stateDb.close();
|
|
709
|
-
}
|
|
705
|
+
});
|
|
710
706
|
}
|
|
711
707
|
catch {
|
|
712
708
|
// State DB unavailable — frontmatter mirror is the only persistence.
|
|
@@ -0,0 +1,434 @@
|
|
|
1
|
+
// This Source Code Form is subject to the terms of the Mozilla Public
|
|
2
|
+
// License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
3
|
+
// file, You can obtain one at https://mozilla.org/MPL/2.0/.
|
|
4
|
+
import fs from "node:fs";
|
|
5
|
+
import path from "node:path";
|
|
6
|
+
import { makeAssetRef, parseAssetRef } from "../../core/asset/asset-ref.js";
|
|
7
|
+
import { parseFrontmatter } from "../../core/asset/frontmatter.js";
|
|
8
|
+
import { isAssetType } from "../../core/common.js";
|
|
9
|
+
import { NotFoundError, rethrowIfTestIsolationError, UsageError } from "../../core/errors.js";
|
|
10
|
+
import { readEvents } from "../../core/events.js";
|
|
11
|
+
import { closeDatabase, getAllEntries, getUtilityScoresByIds, openExistingDatabase } from "../../indexer/db/db.js";
|
|
12
|
+
import { getWritableStashDirs, resolveSourceEntries } from "../../indexer/search/search-source.js";
|
|
13
|
+
import { resolveAssetPath } from "../../indexer/walk/path-resolver.js";
|
|
14
|
+
import { isDistillRefusedInputType } from "./distill.js";
|
|
15
|
+
import { isProfileFilteredForAllPasses } from "./improve-profiles.js";
|
|
16
|
+
// Eligibility / candidate-selection predicates for improve. Free functions
|
|
17
|
+
// (no akmImprove closure state) extracted from improve.ts to shrink the
|
|
18
|
+
// orchestrator and make candidate selection independently testable.
|
|
19
|
+
export function resolveImproveScope(scope) {
|
|
20
|
+
const trimmed = scope?.trim();
|
|
21
|
+
if (!trimmed)
|
|
22
|
+
return { mode: "all" };
|
|
23
|
+
try {
|
|
24
|
+
parseAssetRef(trimmed);
|
|
25
|
+
return { mode: "ref", value: trimmed };
|
|
26
|
+
}
|
|
27
|
+
catch {
|
|
28
|
+
if (!isAssetType(trimmed)) {
|
|
29
|
+
throw new UsageError(`Unknown asset type: "${trimmed}". Valid types: memory, knowledge, skill, lesson, workflow, agent, command, script, wiki, env, secret, task.\n` +
|
|
30
|
+
`If you passed --format to akm improve, that flag is not supported — use it with akm search or akm show instead.`, "INVALID_FLAG_VALUE");
|
|
31
|
+
}
|
|
32
|
+
return { mode: "type", value: trimmed };
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
/**
|
|
36
|
+
* Dedupe a list of eligible refs by `ref`, preserving first-seen order. Used to
|
|
37
|
+
* merge the three eligibility sources (feedback-signal, P0-A high-retrieval,
|
|
38
|
+
* Layer-2 proactive-maintenance) without admitting a ref into the loop twice.
|
|
39
|
+
*/
|
|
40
|
+
export function dedupeRefs(refs) {
|
|
41
|
+
const seen = new Set();
|
|
42
|
+
const out = [];
|
|
43
|
+
for (const r of refs) {
|
|
44
|
+
if (seen.has(r.ref))
|
|
45
|
+
continue;
|
|
46
|
+
seen.add(r.ref);
|
|
47
|
+
out.push(r);
|
|
48
|
+
}
|
|
49
|
+
return out;
|
|
50
|
+
}
|
|
51
|
+
export async function collectEligibleRefs(scope, stashDir, improveProfile) {
|
|
52
|
+
if (scope.mode === "ref" && scope.value) {
|
|
53
|
+
const parsed = parseAssetRef(scope.value);
|
|
54
|
+
const writableDirs = new Set(getWritableStashDirs(stashDir).map((dir) => path.resolve(dir)));
|
|
55
|
+
const filePath = await findAssetFilePath(scope.value, stashDir, writableDirs);
|
|
56
|
+
if (!filePath) {
|
|
57
|
+
return {
|
|
58
|
+
plannedRefs: [],
|
|
59
|
+
memorySummary: { eligible: 0, derived: 0 },
|
|
60
|
+
profileFilteredRefs: [],
|
|
61
|
+
};
|
|
62
|
+
}
|
|
63
|
+
return {
|
|
64
|
+
plannedRefs: [{ ref: scope.value, reason: "scope-ref", filePath }],
|
|
65
|
+
memorySummary: {
|
|
66
|
+
eligible: parsed.type === "memory" ? 1 : 0,
|
|
67
|
+
derived: parsed.type === "memory" && parsed.name.endsWith(".derived") ? 1 : 0,
|
|
68
|
+
},
|
|
69
|
+
profileFilteredRefs: [],
|
|
70
|
+
};
|
|
71
|
+
}
|
|
72
|
+
let sources;
|
|
73
|
+
try {
|
|
74
|
+
sources = resolveSourceEntries(stashDir);
|
|
75
|
+
}
|
|
76
|
+
catch {
|
|
77
|
+
return { plannedRefs: [], memorySummary: { eligible: 0, derived: 0 }, profileFilteredRefs: [] };
|
|
78
|
+
}
|
|
79
|
+
if (sources.length === 0) {
|
|
80
|
+
return { plannedRefs: [], memorySummary: { eligible: 0, derived: 0 }, profileFilteredRefs: [] };
|
|
81
|
+
}
|
|
82
|
+
// Only operate on writable sources — never mutate read-only registry caches
|
|
83
|
+
// or remote stashes that the user did not mark writable.
|
|
84
|
+
let writableDirs;
|
|
85
|
+
try {
|
|
86
|
+
writableDirs = getWritableStashDirs(stashDir);
|
|
87
|
+
}
|
|
88
|
+
catch {
|
|
89
|
+
writableDirs = sources.slice(0, 1).map((s) => s.path); // fallback: primary only
|
|
90
|
+
}
|
|
91
|
+
const writableDirSet = new Set(writableDirs.map((d) => path.resolve(d)));
|
|
92
|
+
let db;
|
|
93
|
+
try {
|
|
94
|
+
db = openExistingDatabase();
|
|
95
|
+
const entries = getAllEntries(db, scope.mode === "type" ? scope.value : undefined).filter((indexed) => {
|
|
96
|
+
// First apply the existing stashDir-scope filter (no-op when stashDir is unset).
|
|
97
|
+
if (!isEntryInScope(indexed.stashDir, indexed.filePath, stashDir))
|
|
98
|
+
return false;
|
|
99
|
+
// Then restrict to writable sources only.
|
|
100
|
+
return isEntryInWritableSource(indexed.stashDir, indexed.filePath, writableDirSet);
|
|
101
|
+
});
|
|
102
|
+
const planned = new Map();
|
|
103
|
+
const profileFiltered = new Map();
|
|
104
|
+
let memoryEligible = 0;
|
|
105
|
+
let memoryDerived = 0;
|
|
106
|
+
for (const indexed of entries) {
|
|
107
|
+
const ref = makeAssetRef(indexed.entry.type, indexed.entry.name);
|
|
108
|
+
const isDerived = indexed.entry.name.endsWith(".derived");
|
|
109
|
+
// `.derived` memories are LLM-inferred and intentionally skip reflect
|
|
110
|
+
// (see the synthetic `derived-memory-reflect-skipped` branch in the
|
|
111
|
+
// improve loop). Enqueueing them here just produced one synthetic skip
|
|
112
|
+
// per derived memory per hour with no real work — pure churn observed
|
|
113
|
+
// 2026-05-21: 11 derived refs re-planned every hour during idle periods.
|
|
114
|
+
// The cleanup phase (analyzeMemoryCleanup) inspects derived memories
|
|
115
|
+
// independently of `plannedRefs`, so dropping them here loses nothing.
|
|
116
|
+
if (!isDerived && !planned.has(ref) && !profileFiltered.has(ref)) {
|
|
117
|
+
// 2026-05-27: extend the .derived precedent to profile-incompatible
|
|
118
|
+
// refs. If every per-ref pass (reflect + distill) on the active
|
|
119
|
+
// profile would refuse this ref, drop it from `plannedRefs`. The
|
|
120
|
+
// caller emits `improve_skipped { reason: profile_filtered_all_passes }`
|
|
121
|
+
// once `eventsCtx` is available so the audit trail is preserved in a
|
|
122
|
+
// single event per ref instead of 2× synthetic actions per run.
|
|
123
|
+
// Background: see /tmp/akm-health-investigations/planner-profile-metrics-deep-analysis.md
|
|
124
|
+
if (improveProfile && isProfileFilteredForAllPasses(ref, improveProfile)) {
|
|
125
|
+
profileFiltered.set(ref, {
|
|
126
|
+
ref,
|
|
127
|
+
reason: "profile_filtered_all_passes",
|
|
128
|
+
filePath: indexed.filePath,
|
|
129
|
+
});
|
|
130
|
+
}
|
|
131
|
+
else {
|
|
132
|
+
planned.set(ref, {
|
|
133
|
+
ref,
|
|
134
|
+
reason: scope.mode === "type" ? "scope-type" : indexed.entry.type === "memory" ? "memory-cleanup" : "scope-type",
|
|
135
|
+
filePath: indexed.filePath,
|
|
136
|
+
});
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
if (indexed.entry.type === "memory") {
|
|
140
|
+
memoryEligible += 1;
|
|
141
|
+
if (isDerived)
|
|
142
|
+
memoryDerived += 1;
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
return {
|
|
146
|
+
plannedRefs: [...planned.values()],
|
|
147
|
+
memorySummary: { eligible: memoryEligible, derived: memoryDerived },
|
|
148
|
+
profileFilteredRefs: [...profileFiltered.values()],
|
|
149
|
+
};
|
|
150
|
+
}
|
|
151
|
+
catch (error) {
|
|
152
|
+
// The bun-test isolation guard must never be downgraded to "empty plan".
|
|
153
|
+
rethrowIfTestIsolationError(error);
|
|
154
|
+
if (error instanceof NotFoundError || error instanceof Error) {
|
|
155
|
+
return { plannedRefs: [], memorySummary: { eligible: 0, derived: 0 }, profileFilteredRefs: [] };
|
|
156
|
+
}
|
|
157
|
+
throw error;
|
|
158
|
+
}
|
|
159
|
+
finally {
|
|
160
|
+
if (db)
|
|
161
|
+
closeDatabase(db);
|
|
162
|
+
}
|
|
163
|
+
}
|
|
164
|
+
export function isEntryInScope(entryStashDir, filePath, stashDir) {
|
|
165
|
+
if (!stashDir)
|
|
166
|
+
return true;
|
|
167
|
+
const resolvedEntryStashDir = path.resolve(entryStashDir);
|
|
168
|
+
const resolvedFilePath = path.resolve(filePath);
|
|
169
|
+
const resolvedScopeStashDir = path.resolve(stashDir);
|
|
170
|
+
return (resolvedEntryStashDir === resolvedScopeStashDir ||
|
|
171
|
+
resolvedEntryStashDir.startsWith(`${resolvedScopeStashDir}${path.sep}`) ||
|
|
172
|
+
resolvedFilePath.startsWith(`${resolvedScopeStashDir}${path.sep}`));
|
|
173
|
+
}
|
|
174
|
+
/**
|
|
175
|
+
* Return true when the indexed entry belongs to one of the writable source
|
|
176
|
+
* directories. Entries from read-only registry caches or remote stashes that
|
|
177
|
+
* the user has not marked writable must never enter the improve/distill loop.
|
|
178
|
+
*/
|
|
179
|
+
export function isEntryInWritableSource(entryStashDir, filePath, writableDirSet) {
|
|
180
|
+
const resolvedEntryStashDir = path.resolve(entryStashDir);
|
|
181
|
+
const resolvedFilePath = path.resolve(filePath);
|
|
182
|
+
for (const writableDir of writableDirSet) {
|
|
183
|
+
if (resolvedEntryStashDir === writableDir ||
|
|
184
|
+
resolvedEntryStashDir.startsWith(`${writableDir}${path.sep}`) ||
|
|
185
|
+
resolvedFilePath.startsWith(`${writableDir}${path.sep}`)) {
|
|
186
|
+
return true;
|
|
187
|
+
}
|
|
188
|
+
}
|
|
189
|
+
return false;
|
|
190
|
+
}
|
|
191
|
+
export function memoryCleanupParentRef(scope, stashDir) {
|
|
192
|
+
if (scope.mode !== "ref" || !scope.value)
|
|
193
|
+
return undefined;
|
|
194
|
+
const parsed = parseAssetRef(scope.value);
|
|
195
|
+
if (parsed.type !== "memory")
|
|
196
|
+
return undefined;
|
|
197
|
+
if (!parsed.name.endsWith(".derived"))
|
|
198
|
+
return scope.value;
|
|
199
|
+
const sources = resolveSourceEntries(stashDir);
|
|
200
|
+
for (const source of sources) {
|
|
201
|
+
const candidate = path.join(source.path, "memories", `${parsed.name}.md`);
|
|
202
|
+
if (!fs.existsSync(candidate))
|
|
203
|
+
continue;
|
|
204
|
+
const raw = fs.readFileSync(candidate, "utf8");
|
|
205
|
+
const fm = parseFrontmatter(raw).data;
|
|
206
|
+
const sourceRef = typeof fm.source === "string" ? fm.source : undefined;
|
|
207
|
+
if (sourceRef) {
|
|
208
|
+
try {
|
|
209
|
+
const parent = parseAssetRef(sourceRef.trim());
|
|
210
|
+
if (parent.type === "memory")
|
|
211
|
+
return makeAssetRef(parent.type, parent.name);
|
|
212
|
+
}
|
|
213
|
+
catch { }
|
|
214
|
+
}
|
|
215
|
+
}
|
|
216
|
+
return makeAssetRef("memory", parsed.name.slice(0, -".derived".length));
|
|
217
|
+
}
|
|
218
|
+
export function isLessonCandidate(ref) {
|
|
219
|
+
// Only lesson assets need lesson-schema validation (description + when_to_use).
|
|
220
|
+
// Memories have their own distill path via shouldDistillMemoryRef.
|
|
221
|
+
// All other types go through reflect, not distill.
|
|
222
|
+
return parseAssetRef(ref).type === "lesson";
|
|
223
|
+
}
|
|
224
|
+
/**
|
|
225
|
+
* Planner-side check: should this ref enter the distill queue?
|
|
226
|
+
*
|
|
227
|
+
* Distill produces lessons from non-lesson sources. Two cases are eligible:
|
|
228
|
+
*
|
|
229
|
+
* 1. Memory refs that pass {@link shouldDistillMemoryRef} (the existing
|
|
230
|
+
* memory→lesson/knowledge promotion path).
|
|
231
|
+
*
|
|
232
|
+
* Refs whose `type` is in {@link DISTILL_REFUSED_INPUT_TYPES} (currently
|
|
233
|
+
* `lesson:*`) are explicitly excluded — distill refuses them at runtime and
|
|
234
|
+
* queuing them just produces a no-op `skipped` outcome per ref per hour. That
|
|
235
|
+
* planner waste was the bug fixed in commit
|
|
236
|
+
* fix(improve): drop distill-refused types from planner.
|
|
237
|
+
*
|
|
238
|
+
* Note: prior to this fix the gate used `isLessonCandidate(ref)` directly,
|
|
239
|
+
* which was true *only* for `lesson:*` refs — exactly the set distill refuses.
|
|
240
|
+
* The result: every hourly run re-queued the same lesson refs, the same skip
|
|
241
|
+
* message returned, and no work was ever done. See
|
|
242
|
+
* `tests/commands/improve-distill-planner-skip-lessons.test.ts`.
|
|
243
|
+
*/
|
|
244
|
+
export function isDistillCandidateRef(ref, stashDir) {
|
|
245
|
+
const parsed = parseAssetRef(ref);
|
|
246
|
+
if (isDistillRefusedInputType(parsed.type))
|
|
247
|
+
return false;
|
|
248
|
+
return shouldDistillMemoryRef(ref, stashDir);
|
|
249
|
+
}
|
|
250
|
+
export function shouldDistillMemoryRef(ref, stashDir) {
|
|
251
|
+
const parsed = parseAssetRef(ref);
|
|
252
|
+
if (parsed.type !== "memory")
|
|
253
|
+
return false;
|
|
254
|
+
const sources = resolveSourceEntries(stashDir);
|
|
255
|
+
for (const source of sources) {
|
|
256
|
+
const candidate = `${source.path}/memories/${parsed.name}.md`;
|
|
257
|
+
if (!fs.existsSync(candidate))
|
|
258
|
+
continue;
|
|
259
|
+
const raw = fs.readFileSync(candidate, "utf8");
|
|
260
|
+
const fm = parseFrontmatter(raw).data;
|
|
261
|
+
const quality = typeof fm.quality === "string" ? fm.quality : undefined;
|
|
262
|
+
if (quality === "proposed")
|
|
263
|
+
return false;
|
|
264
|
+
return !parsed.name.endsWith(".derived");
|
|
265
|
+
}
|
|
266
|
+
return !parsed.name.endsWith(".derived");
|
|
267
|
+
}
|
|
268
|
+
// ── Signal-delta eligibility helpers (0.8.0) ────────────────────────────────
|
|
269
|
+
//
|
|
270
|
+
// The 0.8.0 redesign replaced flat time-based cooldowns for reflect/distill
|
|
271
|
+
// with a *signal-delta* gate: a ref is re-eligible iff new feedback has
|
|
272
|
+
// landed since the last proposal was generated for it. These helpers build
|
|
273
|
+
// the two timestamp maps the gate needs in bulk, so the planner avoids
|
|
274
|
+
// N+1 queries across the full postCleanupRefs set.
|
|
275
|
+
/**
|
|
276
|
+
* Latest feedback event timestamp per ref in the active window. Reads all
|
|
277
|
+
* `feedback` events newer than `sinceIso` in one query and indexes by ref,
|
|
278
|
+
* keeping the maximum `ts` per ref.
|
|
279
|
+
*
|
|
280
|
+
* Only events with a meaningful payload count as "signal" — `metadata.signal`
|
|
281
|
+
* (positive/negative) OR `metadata.note` (a free-form annotation). Empty
|
|
282
|
+
* metadata events are ignored so a stray `akm feedback <ref>` invocation
|
|
283
|
+
* without a flag doesn't trigger downstream re-processing.
|
|
284
|
+
*/
|
|
285
|
+
export function buildLatestFeedbackTsMap(refs, sinceIso) {
|
|
286
|
+
const out = new Map();
|
|
287
|
+
if (refs.length === 0)
|
|
288
|
+
return out;
|
|
289
|
+
const refSet = new Set(refs);
|
|
290
|
+
const { events } = readEvents({ type: "feedback", since: sinceIso });
|
|
291
|
+
for (const e of events) {
|
|
292
|
+
const ref = e.ref;
|
|
293
|
+
if (!ref || !refSet.has(ref))
|
|
294
|
+
continue;
|
|
295
|
+
const meta = e.metadata;
|
|
296
|
+
const hasSignal = meta !== undefined && (typeof meta.signal === "string" || typeof meta.note === "string");
|
|
297
|
+
if (!hasSignal)
|
|
298
|
+
continue;
|
|
299
|
+
const ts = e.ts ?? "";
|
|
300
|
+
if (ts > (out.get(ref) ?? ""))
|
|
301
|
+
out.set(ref, ts);
|
|
302
|
+
}
|
|
303
|
+
return out;
|
|
304
|
+
}
|
|
305
|
+
/**
|
|
306
|
+
* Latest proposal timestamp per input-ref, filtered by source ('reflect' or
|
|
307
|
+
* 'distill'). Reads the corresponding `*_invoked` events from state.db —
|
|
308
|
+
* these events are emitted at proposal creation time and carry the *input*
|
|
309
|
+
* asset ref (memory:foo, skill:bar, etc.) directly. We use them rather than
|
|
310
|
+
* `listProposals` because distill proposals are keyed by the derived
|
|
311
|
+
* lesson/knowledge ref, not the source memory — joining back through the
|
|
312
|
+
* payload would be fragile.
|
|
313
|
+
*/
|
|
314
|
+
export function buildLatestProposalTsMap(refs, source) {
|
|
315
|
+
const out = new Map();
|
|
316
|
+
if (refs.length === 0)
|
|
317
|
+
return out;
|
|
318
|
+
const refSet = new Set(refs);
|
|
319
|
+
const eventType = source === "reflect" ? "reflect_invoked" : "distill_invoked";
|
|
320
|
+
const { events } = readEvents({ type: eventType });
|
|
321
|
+
for (const e of events) {
|
|
322
|
+
const ref = e.ref;
|
|
323
|
+
if (!ref || !refSet.has(ref))
|
|
324
|
+
continue;
|
|
325
|
+
// For distill_invoked we only count attempts that produced (or attempted
|
|
326
|
+
// to produce) a real proposal — config_disabled / parse-error outcomes
|
|
327
|
+
// should not move the signal-delta cursor forward.
|
|
328
|
+
if (eventType === "distill_invoked") {
|
|
329
|
+
const outcome = e.metadata?.outcome;
|
|
330
|
+
if (outcome !== "queued" && outcome !== "skipped" && outcome !== "validation_failed")
|
|
331
|
+
continue;
|
|
332
|
+
}
|
|
333
|
+
const ts = e.ts ?? "";
|
|
334
|
+
if (ts > (out.get(ref) ?? ""))
|
|
335
|
+
out.set(ref, ts);
|
|
336
|
+
}
|
|
337
|
+
return out;
|
|
338
|
+
}
|
|
339
|
+
/**
|
|
340
|
+
* Signal-delta eligibility predicate.
|
|
341
|
+
*
|
|
342
|
+
* True iff `latestFeedback[ref]` is defined AND either no prior proposal
|
|
343
|
+
* exists for this (ref, source) OR `latestFeedback[ref] > lastProposal[ref]`.
|
|
344
|
+
*
|
|
345
|
+
* Refs with no feedback signal at all are ineligible by definition — the
|
|
346
|
+
* high-retrieval fallback path (see `noFeedbackCandidates` later in the
|
|
347
|
+
* planner) handles never-touched-but-frequently-read assets separately.
|
|
348
|
+
*/
|
|
349
|
+
export function isSignalDeltaEligible(ref, latestFeedback, lastProposal) {
|
|
350
|
+
const fb = latestFeedback.get(ref);
|
|
351
|
+
if (!fb)
|
|
352
|
+
return false;
|
|
353
|
+
const lp = lastProposal.get(ref);
|
|
354
|
+
if (!lp)
|
|
355
|
+
return true;
|
|
356
|
+
return fb > lp;
|
|
357
|
+
}
|
|
358
|
+
/**
|
|
359
|
+
* H7 (#566): cooperative budget watchdog with a captured, RAII-cleared hard-kill.
|
|
360
|
+
*
|
|
361
|
+
* When the wall-clock budget expires, `onExhausted` (normally an
|
|
362
|
+
* `AbortController.abort`) signals cooperative cancellation so the run can drain
|
|
363
|
+
* its in-flight log/`state.db` flush and unwind naturally. A second hard-kill
|
|
364
|
+
* timer is then armed as a watchdog: it only `exit(0)`s if the drain itself
|
|
365
|
+
* overruns `hardKillGraceMs`, preventing the process from outliving the task
|
|
366
|
+
* timeout window (lock-cascade fix).
|
|
367
|
+
*
|
|
368
|
+
* Both timers are captured; the returned dispose() clears whichever is still
|
|
369
|
+
* pending. Callers invoke it from a `finally`, so a *clean* drain reaches the
|
|
370
|
+
* `finally` and cancels the pending hard-kill before it can fire — the previous
|
|
371
|
+
* detached `setTimeout(() => process.exit(0), 5000)` always fired, truncating a
|
|
372
|
+
* clean flush. The hard-kill timer is `unref()`-ed so it never keeps the event
|
|
373
|
+
* loop alive on its own: once the run drains it exits with its own code, not the
|
|
374
|
+
* forced 0.
|
|
375
|
+
*
|
|
376
|
+
* Dependencies are injectable purely so the concurrency-sensitive timing
|
|
377
|
+
* contract can be exercised deterministically in unit tests.
|
|
378
|
+
*/
|
|
379
|
+
export function shouldAnalyzeMemoryCleanup(scope, eligibleMemories, primaryStashDir) {
|
|
380
|
+
if (!primaryStashDir || eligibleMemories === 0)
|
|
381
|
+
return false;
|
|
382
|
+
if (scope.mode === "all")
|
|
383
|
+
return true;
|
|
384
|
+
if (scope.mode === "type")
|
|
385
|
+
return scope.value === "memory";
|
|
386
|
+
if (!scope.value)
|
|
387
|
+
return false;
|
|
388
|
+
return parseAssetRef(scope.value).type === "memory";
|
|
389
|
+
}
|
|
390
|
+
export function buildUtilityMap(refs) {
|
|
391
|
+
const map = new Map();
|
|
392
|
+
if (refs.length === 0)
|
|
393
|
+
return map;
|
|
394
|
+
const refSet = new Set(refs.map((r) => r.ref));
|
|
395
|
+
let db;
|
|
396
|
+
try {
|
|
397
|
+
db = openExistingDatabase();
|
|
398
|
+
const allDbEntries = getAllEntries(db);
|
|
399
|
+
const idToRef = new Map();
|
|
400
|
+
for (const indexed of allDbEntries) {
|
|
401
|
+
const ref = makeAssetRef(indexed.entry.type, indexed.entry.name);
|
|
402
|
+
if (refSet.has(ref))
|
|
403
|
+
idToRef.set(indexed.id, ref);
|
|
404
|
+
}
|
|
405
|
+
const ids = [...idToRef.keys()];
|
|
406
|
+
if (ids.length > 0) {
|
|
407
|
+
const { global: scores } = getUtilityScoresByIds(db, ids);
|
|
408
|
+
for (const [id, score] of scores) {
|
|
409
|
+
const ref = idToRef.get(id);
|
|
410
|
+
if (ref)
|
|
411
|
+
map.set(ref, score.utility);
|
|
412
|
+
}
|
|
413
|
+
}
|
|
414
|
+
}
|
|
415
|
+
catch (err) {
|
|
416
|
+
rethrowIfTestIsolationError(err);
|
|
417
|
+
// best-effort: if DB unavailable, all utilities default to 0
|
|
418
|
+
}
|
|
419
|
+
finally {
|
|
420
|
+
if (db)
|
|
421
|
+
closeDatabase(db);
|
|
422
|
+
}
|
|
423
|
+
return map;
|
|
424
|
+
}
|
|
425
|
+
export async function findAssetFilePath(ref, stashDir, writableDirSet) {
|
|
426
|
+
return resolveAssetPath(ref, {
|
|
427
|
+
stashDir,
|
|
428
|
+
mode: "disk-only",
|
|
429
|
+
writableDirSet,
|
|
430
|
+
directoryIndexNames: ["SKILL.md"],
|
|
431
|
+
preserveDirectNameFallback: true,
|
|
432
|
+
honorOrigin: false,
|
|
433
|
+
});
|
|
434
|
+
}
|
|
@@ -17,7 +17,7 @@
|
|
|
17
17
|
import fs from "node:fs";
|
|
18
18
|
import path from "node:path";
|
|
19
19
|
import { defineCommand } from "citty";
|
|
20
|
-
import { output, runWithJsonErrors } from "../../cli/shared.js";
|
|
20
|
+
import { EXIT_CODES, output, runWithJsonErrors } from "../../cli/shared.js";
|
|
21
21
|
import { UsageError } from "../../core/errors.js";
|
|
22
22
|
import { getAvailableHarnesses, getWatchTargets } from "../../integrations/session-logs/index.js";
|
|
23
23
|
import { akmExtract } from "./extract.js";
|
|
@@ -141,10 +141,18 @@ export const extractCommand = defineCommand({
|
|
|
141
141
|
totalProposals,
|
|
142
142
|
results,
|
|
143
143
|
});
|
|
144
|
+
// Signal failure to callers/cron when every harness failed. output()
|
|
145
|
+
// only renders; without this the */30 cron exits 0 on a total failure
|
|
146
|
+
// and the breakage is invisible to exit-code monitoring. process.exitCode
|
|
147
|
+
// (not process.exit) lets stdout flush and the watcher/timers settle.
|
|
148
|
+
if (!ok)
|
|
149
|
+
process.exitCode = EXIT_CODES.GENERAL;
|
|
144
150
|
return;
|
|
145
151
|
}
|
|
146
152
|
const result = await akmExtract({ type, ...commonOptions });
|
|
147
153
|
output("extract", result);
|
|
154
|
+
if (!result.ok)
|
|
155
|
+
process.exitCode = EXIT_CODES.GENERAL;
|
|
148
156
|
});
|
|
149
157
|
},
|
|
150
158
|
});
|
|
@@ -32,7 +32,7 @@ import { ConfigError, UsageError } from "../../core/errors.js";
|
|
|
32
32
|
import { appendEvent } from "../../core/events.js";
|
|
33
33
|
import { probeLock, releaseLock, tryAcquireLockSync } from "../../core/file-lock.js";
|
|
34
34
|
import { resolveStashStandards } from "../../core/standards/resolve-stash-standards.js";
|
|
35
|
-
import { getExtractedSessionsMap, getLastExtractRunAt, getStateDbPath, openStateDatabase, shouldSkipAlreadyExtractedSession, upsertExtractedSession, } from "../../core/state-db.js";
|
|
35
|
+
import { getExtractedSessionsMap, getLastExtractRunAt, getStateDbPath, openStateDatabase, shouldSkipAlreadyExtractedSession, upsertExtractedSession, withStateDb, } from "../../core/state-db.js";
|
|
36
36
|
import { repairTruncatedDescription } from "../../core/text-truncation.js";
|
|
37
37
|
import { warn } from "../../core/warn.js";
|
|
38
38
|
import { resolveImproveProcessRunnerFromProfile, runnerIsLlm } from "../../integrations/agent/runner.js";
|
|
@@ -94,29 +94,15 @@ function resolveDefaultSinceMs(harnessName, now, opts) {
|
|
|
94
94
|
const floor = now - DEFAULT_SINCE_FLOOR_MS;
|
|
95
95
|
if (opts.skipTracking)
|
|
96
96
|
return floor;
|
|
97
|
-
let db = opts.stateDb;
|
|
98
|
-
let opened = false;
|
|
99
97
|
try {
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
}
|
|
104
|
-
const lastRun = getLastExtractRunAt(db, harnessName);
|
|
105
|
-
return lastRun != null ? Math.min(lastRun, floor) : floor;
|
|
98
|
+
return withStateDb((db) => {
|
|
99
|
+
const lastRun = getLastExtractRunAt(db, harnessName);
|
|
100
|
+
return lastRun != null ? Math.min(lastRun, floor) : floor;
|
|
101
|
+
}, { path: opts.stateDbPath, borrowed: opts.stateDb });
|
|
106
102
|
}
|
|
107
103
|
catch {
|
|
108
104
|
return floor;
|
|
109
105
|
}
|
|
110
|
-
finally {
|
|
111
|
-
if (opened && db) {
|
|
112
|
-
try {
|
|
113
|
-
db.close();
|
|
114
|
-
}
|
|
115
|
-
catch {
|
|
116
|
-
// best-effort close
|
|
117
|
-
}
|
|
118
|
-
}
|
|
119
|
-
}
|
|
120
106
|
}
|
|
121
107
|
/** Filesystem-safe per-session lock path, co-located with the state.db. */
|
|
122
108
|
function getExtractSessionLockPath(harness, sessionId, stateDbPath) {
|
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
// file, You can obtain one at https://mozilla.org/MPL/2.0/.
|
|
4
4
|
import { loadConfig } from "../../core/config/config.js";
|
|
5
5
|
import { appendEvent } from "../../core/events.js";
|
|
6
|
-
import { getPhaseThreshold,
|
|
6
|
+
import { getPhaseThreshold, withStateDb } from "../../core/state-db.js";
|
|
7
7
|
import { info, warn } from "../../core/warn.js";
|
|
8
8
|
import { promoteProposal, recordGateDecision } from "../proposal/validators/proposals.js";
|
|
9
9
|
/**
|
|
@@ -195,13 +195,9 @@ export function makeGateConfig(phase, shared, overrides = {}) {
|
|
|
195
195
|
let phaseThreshold;
|
|
196
196
|
if (shared.stateDbPath && shared.globalThreshold !== undefined) {
|
|
197
197
|
try {
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
}
|
|
202
|
-
finally {
|
|
203
|
-
db.close();
|
|
204
|
-
}
|
|
198
|
+
phaseThreshold = withStateDb((db) => getPhaseThreshold(db, phase) ?? undefined, {
|
|
199
|
+
path: shared.stateDbPath,
|
|
200
|
+
});
|
|
205
201
|
}
|
|
206
202
|
catch {
|
|
207
203
|
// DB unavailable — fall back to globalThreshold silently.
|