akm-cli 0.7.5 → 0.8.0-rc2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.github/CHANGELOG.md +1 -1
- package/dist/cli/parse-args.js +43 -0
- package/dist/cli.js +853 -479
- package/dist/commands/agent-dispatch.js +102 -0
- package/dist/commands/agent-support.js +62 -0
- package/dist/commands/config-cli.js +68 -84
- package/dist/commands/consolidate.js +823 -0
- package/dist/commands/distill-promotion-policy.js +658 -0
- package/dist/commands/distill.js +244 -52
- package/dist/commands/eval-cases.js +40 -0
- package/dist/commands/events.js +2 -23
- package/dist/commands/graph.js +222 -0
- package/dist/commands/health.js +376 -0
- package/dist/commands/help/help-accept.md +9 -0
- package/dist/commands/help/help-improve.md +53 -0
- package/dist/commands/help/help-proposals.md +15 -0
- package/dist/commands/help/help-propose.md +17 -0
- package/dist/commands/help/help-reject.md +8 -0
- package/dist/commands/history.js +3 -30
- package/dist/commands/improve.js +1170 -0
- package/dist/commands/info.js +2 -2
- package/dist/commands/init.js +2 -2
- package/dist/commands/install-audit.js +5 -1
- package/dist/commands/installed-stashes.js +118 -138
- package/dist/commands/knowledge.js +133 -0
- package/dist/commands/lint/agent-linter.js +46 -0
- package/dist/commands/lint/base-linter.js +285 -0
- package/dist/commands/lint/command-linter.js +46 -0
- package/dist/commands/lint/default-linter.js +13 -0
- package/dist/commands/lint/index.js +107 -0
- package/dist/commands/lint/knowledge-linter.js +13 -0
- package/dist/commands/lint/memory-linter.js +58 -0
- package/dist/commands/lint/registry.js +33 -0
- package/dist/commands/lint/skill-linter.js +42 -0
- package/dist/commands/lint/task-linter.js +47 -0
- package/dist/commands/lint/types.js +1 -0
- package/dist/commands/lint/workflow-linter.js +53 -0
- package/dist/commands/lint.js +1 -0
- package/dist/commands/proposal.js +8 -7
- package/dist/commands/propose.js +78 -28
- package/dist/commands/reflect.js +143 -35
- package/dist/commands/registry-search.js +2 -2
- package/dist/commands/remember.js +54 -0
- package/dist/commands/schema-repair.js +130 -0
- package/dist/commands/search.js +21 -5
- package/dist/commands/show.js +121 -17
- package/dist/commands/source-add.js +10 -10
- package/dist/commands/source-manage.js +11 -19
- package/dist/commands/tasks.js +385 -0
- package/dist/commands/url-checker.js +39 -0
- package/dist/commands/vault.js +8 -26
- package/dist/core/action-contributors.js +25 -0
- package/dist/core/asset-ref.js +4 -0
- package/dist/core/asset-registry.js +4 -16
- package/dist/core/asset-spec.js +10 -0
- package/dist/core/common.js +94 -0
- package/dist/core/concurrent.js +22 -0
- package/dist/core/config.js +222 -128
- package/dist/core/events.js +73 -126
- package/dist/core/frontmatter.js +3 -1
- package/dist/core/markdown.js +17 -0
- package/dist/core/memory-improve.js +678 -0
- package/dist/core/parse.js +155 -0
- package/dist/core/paths.js +101 -3
- package/dist/core/proposal-validators.js +61 -0
- package/dist/core/proposals.js +49 -38
- package/dist/core/state-db.js +775 -0
- package/dist/core/time.js +51 -0
- package/dist/core/warn.js +59 -1
- package/dist/indexer/db-search.js +52 -238
- package/dist/indexer/db.js +378 -1
- package/dist/indexer/ensure-index.js +61 -0
- package/dist/indexer/graph-boost.js +247 -94
- package/dist/indexer/graph-db.js +201 -0
- package/dist/indexer/graph-dedup.js +99 -0
- package/dist/indexer/graph-extraction.js +409 -76
- package/dist/indexer/index-context.js +10 -0
- package/dist/indexer/indexer.js +442 -290
- package/dist/indexer/llm-cache.js +47 -0
- package/dist/indexer/match-contributors.js +141 -0
- package/dist/indexer/matchers.js +24 -190
- package/dist/indexer/memory-inference.js +63 -29
- package/dist/indexer/metadata-contributors.js +26 -0
- package/dist/indexer/metadata.js +194 -175
- package/dist/indexer/path-resolver.js +89 -0
- package/dist/indexer/ranking-contributors.js +204 -0
- package/dist/indexer/ranking.js +74 -0
- package/dist/indexer/search-hit-enrichers.js +22 -0
- package/dist/indexer/search-source.js +24 -9
- package/dist/indexer/semantic-status.js +2 -16
- package/dist/indexer/walker.js +25 -0
- package/dist/integrations/agent/config.js +175 -3
- package/dist/integrations/agent/index.js +3 -1
- package/dist/integrations/agent/pipeline.js +39 -0
- package/dist/integrations/agent/profiles.js +67 -5
- package/dist/integrations/agent/prompts.js +77 -72
- package/dist/integrations/agent/runners.js +31 -0
- package/dist/integrations/agent/sdk-runner.js +120 -0
- package/dist/integrations/agent/spawn.js +71 -16
- package/dist/integrations/lockfile.js +10 -18
- package/dist/integrations/session-logs/index.js +65 -0
- package/dist/integrations/session-logs/providers/claude-code.js +56 -0
- package/dist/integrations/session-logs/providers/opencode.js +52 -0
- package/dist/integrations/session-logs/types.js +1 -0
- package/dist/llm/call-ai.js +74 -0
- package/dist/llm/client.js +61 -122
- package/dist/llm/feature-gate.js +27 -16
- package/dist/llm/graph-extract.js +297 -62
- package/dist/llm/memory-infer.js +49 -71
- package/dist/llm/metadata-enhance.js +39 -22
- package/dist/llm/prompts/graph-extract-user-prompt.md +12 -0
- package/dist/output/cli-hints-full.md +277 -0
- package/dist/output/cli-hints-short.md +65 -0
- package/dist/output/cli-hints.js +2 -318
- package/dist/output/renderers.js +190 -123
- package/dist/output/shapes.js +33 -0
- package/dist/output/text.js +239 -2
- package/dist/registry/providers/skills-sh.js +61 -49
- package/dist/registry/providers/static-index.js +44 -48
- package/dist/setup/setup.js +510 -11
- package/dist/sources/provider-factory.js +2 -1
- package/dist/sources/providers/git.js +2 -2
- package/dist/sources/website-ingest.js +4 -0
- package/dist/tasks/backends/cron.js +200 -0
- package/dist/tasks/backends/exec-utils.js +25 -0
- package/dist/tasks/backends/index.js +32 -0
- package/dist/tasks/backends/launchd-template.xml +19 -0
- package/dist/tasks/backends/launchd.js +184 -0
- package/dist/tasks/backends/schtasks-template.xml +29 -0
- package/dist/tasks/backends/schtasks.js +212 -0
- package/dist/tasks/parser.js +198 -0
- package/dist/tasks/resolveAkmBin.js +84 -0
- package/dist/tasks/runner.js +432 -0
- package/dist/tasks/schedule.js +208 -0
- package/dist/tasks/schema.js +13 -0
- package/dist/tasks/validator.js +59 -0
- package/dist/wiki/index-template.md +12 -0
- package/dist/wiki/ingest-workflow-template.md +54 -0
- package/dist/wiki/log-template.md +8 -0
- package/dist/wiki/schema-template.md +61 -0
- package/dist/wiki/wiki-templates.js +12 -0
- package/dist/wiki/wiki.js +10 -61
- package/dist/workflows/authoring.js +5 -25
- package/dist/workflows/renderer.js +8 -3
- package/dist/workflows/runs.js +59 -91
- package/dist/workflows/validator.js +1 -1
- package/dist/workflows/workflow-template.md +24 -0
- package/docs/README.md +3 -0
- package/docs/migration/release-notes/0.7.0.md +1 -1
- package/docs/migration/release-notes/0.8.0.md +43 -0
- package/package.json +3 -2
- package/dist/templates/wiki-templates.js +0 -100
|
@@ -0,0 +1,1170 @@
|
|
|
1
|
+
import fs from "node:fs";
|
|
2
|
+
import path from "node:path";
|
|
3
|
+
import { makeAssetRef, parseAssetRef } from "../core/asset-ref";
|
|
4
|
+
import { loadConfig } from "../core/config";
|
|
5
|
+
import { ConfigError, NotFoundError } from "../core/errors";
|
|
6
|
+
import { appendEvent, readEvents } from "../core/events";
|
|
7
|
+
import { parseFrontmatter } from "../core/frontmatter";
|
|
8
|
+
import { analyzeMemoryCleanup, applyMemoryCleanup, } from "../core/memory-improve";
|
|
9
|
+
import { getDbPath } from "../core/paths";
|
|
10
|
+
import { listProposals } from "../core/proposals";
|
|
11
|
+
import { info, warn } from "../core/warn";
|
|
12
|
+
import { closeDatabase, getAllEntries, getRetrievalCounts, getUtilityScoresByIds, getZeroResultSearches, openDatabase, openExistingDatabase, } from "../indexer/db";
|
|
13
|
+
import { ensureIndex } from "../indexer/ensure-index";
|
|
14
|
+
import { runGraphExtractionPass } from "../indexer/graph-extraction";
|
|
15
|
+
import { akmIndex } from "../indexer/indexer";
|
|
16
|
+
import { runMemoryInferencePass, } from "../indexer/memory-inference";
|
|
17
|
+
import { resolveAssetPath } from "../indexer/path-resolver";
|
|
18
|
+
import { getWritableStashDirs, resolveSourceEntries } from "../indexer/search-source";
|
|
19
|
+
import { getExecutionLogCandidates } from "../integrations/session-logs";
|
|
20
|
+
import { akmConsolidate } from "./consolidate";
|
|
21
|
+
import { akmDistill, deriveLessonRef } from "./distill";
|
|
22
|
+
import { countEvalCases, writeEvalCase } from "./eval-cases";
|
|
23
|
+
import { akmLint } from "./lint/index";
|
|
24
|
+
import { akmReflect } from "./reflect";
|
|
25
|
+
import { runSchemaRepairPass } from "./schema-repair";
|
|
26
|
+
import { checkDeadUrls } from "./url-checker";
|
|
27
|
+
function resolveImproveScope(scope) {
|
|
28
|
+
const trimmed = scope?.trim();
|
|
29
|
+
if (!trimmed)
|
|
30
|
+
return { mode: "all" };
|
|
31
|
+
try {
|
|
32
|
+
parseAssetRef(trimmed);
|
|
33
|
+
return { mode: "ref", value: trimmed };
|
|
34
|
+
}
|
|
35
|
+
catch {
|
|
36
|
+
return { mode: "type", value: trimmed };
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
async function collectEligibleRefs(scope, stashDir) {
|
|
40
|
+
if (scope.mode === "ref" && scope.value) {
|
|
41
|
+
const parsed = parseAssetRef(scope.value);
|
|
42
|
+
const writableDirs = new Set(getWritableStashDirs(stashDir).map((dir) => path.resolve(dir)));
|
|
43
|
+
const filePath = await findAssetFilePath(scope.value, stashDir, writableDirs);
|
|
44
|
+
if (!filePath) {
|
|
45
|
+
return {
|
|
46
|
+
plannedRefs: [],
|
|
47
|
+
memorySummary: { eligible: 0, derived: 0 },
|
|
48
|
+
};
|
|
49
|
+
}
|
|
50
|
+
return {
|
|
51
|
+
plannedRefs: [{ ref: scope.value, reason: "scope-ref" }],
|
|
52
|
+
memorySummary: {
|
|
53
|
+
eligible: parsed.type === "memory" ? 1 : 0,
|
|
54
|
+
derived: parsed.type === "memory" && parsed.name.endsWith(".derived") ? 1 : 0,
|
|
55
|
+
},
|
|
56
|
+
};
|
|
57
|
+
}
|
|
58
|
+
let sources;
|
|
59
|
+
try {
|
|
60
|
+
sources = resolveSourceEntries(stashDir);
|
|
61
|
+
}
|
|
62
|
+
catch {
|
|
63
|
+
return { plannedRefs: [], memorySummary: { eligible: 0, derived: 0 } };
|
|
64
|
+
}
|
|
65
|
+
if (sources.length === 0) {
|
|
66
|
+
return { plannedRefs: [], memorySummary: { eligible: 0, derived: 0 } };
|
|
67
|
+
}
|
|
68
|
+
// Only operate on writable sources — never mutate read-only registry caches
|
|
69
|
+
// or remote stashes that the user did not mark writable.
|
|
70
|
+
let writableDirs;
|
|
71
|
+
try {
|
|
72
|
+
writableDirs = getWritableStashDirs(stashDir);
|
|
73
|
+
}
|
|
74
|
+
catch {
|
|
75
|
+
writableDirs = sources.slice(0, 1).map((s) => s.path); // fallback: primary only
|
|
76
|
+
}
|
|
77
|
+
const writableDirSet = new Set(writableDirs.map((d) => path.resolve(d)));
|
|
78
|
+
let db;
|
|
79
|
+
try {
|
|
80
|
+
db = openExistingDatabase();
|
|
81
|
+
const entries = getAllEntries(db, scope.mode === "type" ? scope.value : undefined).filter((indexed) => {
|
|
82
|
+
// First apply the existing stashDir-scope filter (no-op when stashDir is unset).
|
|
83
|
+
if (!isEntryInScope(indexed.stashDir, indexed.filePath, stashDir))
|
|
84
|
+
return false;
|
|
85
|
+
// Then restrict to writable sources only.
|
|
86
|
+
return isEntryInWritableSource(indexed.stashDir, indexed.filePath, writableDirSet);
|
|
87
|
+
});
|
|
88
|
+
const planned = new Map();
|
|
89
|
+
let memoryEligible = 0;
|
|
90
|
+
let memoryDerived = 0;
|
|
91
|
+
for (const indexed of entries) {
|
|
92
|
+
const ref = makeAssetRef(indexed.entry.type, indexed.entry.name);
|
|
93
|
+
if (!planned.has(ref)) {
|
|
94
|
+
planned.set(ref, {
|
|
95
|
+
ref,
|
|
96
|
+
reason: scope.mode === "type" ? "scope-type" : indexed.entry.type === "memory" ? "memory-cleanup" : "scope-type",
|
|
97
|
+
});
|
|
98
|
+
}
|
|
99
|
+
if (indexed.entry.type === "memory") {
|
|
100
|
+
memoryEligible += 1;
|
|
101
|
+
if (indexed.entry.name.endsWith(".derived"))
|
|
102
|
+
memoryDerived += 1;
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
return {
|
|
106
|
+
plannedRefs: [...planned.values()],
|
|
107
|
+
memorySummary: { eligible: memoryEligible, derived: memoryDerived },
|
|
108
|
+
};
|
|
109
|
+
}
|
|
110
|
+
catch (error) {
|
|
111
|
+
if (error instanceof NotFoundError || error instanceof Error) {
|
|
112
|
+
return { plannedRefs: [], memorySummary: { eligible: 0, derived: 0 } };
|
|
113
|
+
}
|
|
114
|
+
throw error;
|
|
115
|
+
}
|
|
116
|
+
finally {
|
|
117
|
+
if (db)
|
|
118
|
+
closeDatabase(db);
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
function isEntryInScope(entryStashDir, filePath, stashDir) {
|
|
122
|
+
if (!stashDir)
|
|
123
|
+
return true;
|
|
124
|
+
const resolvedEntryStashDir = path.resolve(entryStashDir);
|
|
125
|
+
const resolvedFilePath = path.resolve(filePath);
|
|
126
|
+
const resolvedScopeStashDir = path.resolve(stashDir);
|
|
127
|
+
return (resolvedEntryStashDir === resolvedScopeStashDir ||
|
|
128
|
+
resolvedEntryStashDir.startsWith(`${resolvedScopeStashDir}${path.sep}`) ||
|
|
129
|
+
resolvedFilePath.startsWith(`${resolvedScopeStashDir}${path.sep}`));
|
|
130
|
+
}
|
|
131
|
+
/**
|
|
132
|
+
* Return true when the indexed entry belongs to one of the writable source
|
|
133
|
+
* directories. Entries from read-only registry caches or remote stashes that
|
|
134
|
+
* the user has not marked writable must never enter the improve/distill loop.
|
|
135
|
+
*/
|
|
136
|
+
function isEntryInWritableSource(entryStashDir, filePath, writableDirSet) {
|
|
137
|
+
const resolvedEntryStashDir = path.resolve(entryStashDir);
|
|
138
|
+
const resolvedFilePath = path.resolve(filePath);
|
|
139
|
+
for (const writableDir of writableDirSet) {
|
|
140
|
+
if (resolvedEntryStashDir === writableDir ||
|
|
141
|
+
resolvedEntryStashDir.startsWith(`${writableDir}${path.sep}`) ||
|
|
142
|
+
resolvedFilePath.startsWith(`${writableDir}${path.sep}`)) {
|
|
143
|
+
return true;
|
|
144
|
+
}
|
|
145
|
+
}
|
|
146
|
+
return false;
|
|
147
|
+
}
|
|
148
|
+
function memoryCleanupParentRef(scope, stashDir) {
|
|
149
|
+
if (scope.mode !== "ref" || !scope.value)
|
|
150
|
+
return undefined;
|
|
151
|
+
const parsed = parseAssetRef(scope.value);
|
|
152
|
+
if (parsed.type !== "memory")
|
|
153
|
+
return undefined;
|
|
154
|
+
if (!parsed.name.endsWith(".derived"))
|
|
155
|
+
return scope.value;
|
|
156
|
+
const sources = resolveSourceEntries(stashDir);
|
|
157
|
+
for (const source of sources) {
|
|
158
|
+
const candidate = path.join(source.path, "memories", `${parsed.name}.md`);
|
|
159
|
+
if (!fs.existsSync(candidate))
|
|
160
|
+
continue;
|
|
161
|
+
const raw = fs.readFileSync(candidate, "utf8");
|
|
162
|
+
const fm = parseFrontmatter(raw).data;
|
|
163
|
+
const sourceRef = typeof fm.source === "string" ? fm.source : undefined;
|
|
164
|
+
if (sourceRef) {
|
|
165
|
+
try {
|
|
166
|
+
const parent = parseAssetRef(sourceRef.trim());
|
|
167
|
+
if (parent.type === "memory")
|
|
168
|
+
return makeAssetRef(parent.type, parent.name);
|
|
169
|
+
}
|
|
170
|
+
catch { }
|
|
171
|
+
}
|
|
172
|
+
}
|
|
173
|
+
return makeAssetRef("memory", parsed.name.slice(0, -".derived".length));
|
|
174
|
+
}
|
|
175
|
+
function filterRemovedPlannedRefs(plannedRefs, archivedRefs) {
|
|
176
|
+
if (archivedRefs.length === 0)
|
|
177
|
+
return plannedRefs;
|
|
178
|
+
const removed = new Set(archivedRefs);
|
|
179
|
+
return plannedRefs.filter((planned) => !removed.has(planned.ref));
|
|
180
|
+
}
|
|
181
|
+
function isLessonCandidate(ref) {
|
|
182
|
+
const parsed = parseAssetRef(ref);
|
|
183
|
+
return parsed.type !== "lesson" && parsed.type !== "memory";
|
|
184
|
+
}
|
|
185
|
+
function shouldDistillMemoryRef(ref, stashDir) {
|
|
186
|
+
const parsed = parseAssetRef(ref);
|
|
187
|
+
if (parsed.type !== "memory")
|
|
188
|
+
return false;
|
|
189
|
+
const sources = resolveSourceEntries(stashDir);
|
|
190
|
+
for (const source of sources) {
|
|
191
|
+
const candidate = `${source.path}/memories/${parsed.name}.md`;
|
|
192
|
+
if (!fs.existsSync(candidate))
|
|
193
|
+
continue;
|
|
194
|
+
const raw = fs.readFileSync(candidate, "utf8");
|
|
195
|
+
const fm = parseFrontmatter(raw).data;
|
|
196
|
+
const quality = typeof fm.quality === "string" ? fm.quality : undefined;
|
|
197
|
+
if (quality === "proposed")
|
|
198
|
+
return false;
|
|
199
|
+
return !parsed.name.endsWith(".derived");
|
|
200
|
+
}
|
|
201
|
+
return !parsed.name.endsWith(".derived");
|
|
202
|
+
}
|
|
203
|
+
export async function akmImprove(options = {}) {
|
|
204
|
+
const scope = resolveImproveScope(options.scope);
|
|
205
|
+
const { plannedRefs, memorySummary } = await collectEligibleRefs(scope, options.stashDir);
|
|
206
|
+
const reflectFn = options.reflectFn ?? akmReflect;
|
|
207
|
+
const distillFn = options.distillFn ?? akmDistill;
|
|
208
|
+
const ensureIndexFn = options.ensureIndexFn ?? ensureIndex;
|
|
209
|
+
const reindexFn = options.reindexFn ?? akmIndex;
|
|
210
|
+
let primaryStashDir;
|
|
211
|
+
try {
|
|
212
|
+
primaryStashDir = resolveSourceEntries(options.stashDir)[0]?.path;
|
|
213
|
+
}
|
|
214
|
+
catch {
|
|
215
|
+
primaryStashDir = undefined;
|
|
216
|
+
}
|
|
217
|
+
const cleanupParentRef = memoryCleanupParentRef(scope, options.stashDir);
|
|
218
|
+
const memoryCleanupPlan = shouldAnalyzeMemoryCleanup(scope, memorySummary.eligible, primaryStashDir)
|
|
219
|
+
? analyzeMemoryCleanup(primaryStashDir, cleanupParentRef ? { parentRef: cleanupParentRef } : undefined)
|
|
220
|
+
: undefined;
|
|
221
|
+
const guidance = memorySummary.eligible > 0
|
|
222
|
+
? "Improve folds memory cleanup into the same proposal queue: speculative promotions still go through reflect/distill proposals, while high-confidence redundant derived memories are moved into a recoverable cleanup archive instead of being left active in the stash."
|
|
223
|
+
: undefined;
|
|
224
|
+
if (options.dryRun) {
|
|
225
|
+
const result = {
|
|
226
|
+
schemaVersion: 1,
|
|
227
|
+
ok: true,
|
|
228
|
+
scope,
|
|
229
|
+
dryRun: true,
|
|
230
|
+
...(guidance ? { guidance } : {}),
|
|
231
|
+
memorySummary,
|
|
232
|
+
...(memoryCleanupPlan ? { memoryCleanup: shapeMemoryCleanup(memoryCleanupPlan) } : {}),
|
|
233
|
+
plannedRefs,
|
|
234
|
+
};
|
|
235
|
+
return result;
|
|
236
|
+
}
|
|
237
|
+
const resolvedLockPath = primaryStashDir
|
|
238
|
+
? path.join(primaryStashDir, ".akm", "improve.lock")
|
|
239
|
+
: path.join(options.stashDir ?? ".", ".akm", "improve.lock");
|
|
240
|
+
let staleLock = false;
|
|
241
|
+
if (fs.existsSync(resolvedLockPath)) {
|
|
242
|
+
let lock = null;
|
|
243
|
+
try {
|
|
244
|
+
lock = JSON.parse(fs.readFileSync(resolvedLockPath, "utf8"));
|
|
245
|
+
}
|
|
246
|
+
catch {
|
|
247
|
+
staleLock = true;
|
|
248
|
+
}
|
|
249
|
+
if (lock !== null) {
|
|
250
|
+
try {
|
|
251
|
+
process.kill(lock.pid, 0);
|
|
252
|
+
throw new ConfigError(`akm improve is already running (pid ${lock.pid}, started ${lock.startedAt}). Use SIGTERM to stop it.`, "INVALID_CONFIG_FILE");
|
|
253
|
+
}
|
|
254
|
+
catch (err) {
|
|
255
|
+
if (err instanceof ConfigError)
|
|
256
|
+
throw err;
|
|
257
|
+
staleLock = true;
|
|
258
|
+
}
|
|
259
|
+
}
|
|
260
|
+
if (staleLock) {
|
|
261
|
+
try {
|
|
262
|
+
fs.unlinkSync(resolvedLockPath);
|
|
263
|
+
}
|
|
264
|
+
catch {
|
|
265
|
+
// ignore
|
|
266
|
+
}
|
|
267
|
+
}
|
|
268
|
+
}
|
|
269
|
+
fs.mkdirSync(path.dirname(resolvedLockPath), { recursive: true });
|
|
270
|
+
fs.writeFileSync(resolvedLockPath, JSON.stringify({ pid: process.pid, startedAt: new Date().toISOString() }));
|
|
271
|
+
const budgetMs = options.timeoutMs ?? 2 * 60 * 60 * 1000; // default 2 hours
|
|
272
|
+
const startMs = Date.now();
|
|
273
|
+
try {
|
|
274
|
+
const preparation = await runImprovePreparationStage({
|
|
275
|
+
scope,
|
|
276
|
+
options,
|
|
277
|
+
plannedRefs,
|
|
278
|
+
memoryCleanupPlan,
|
|
279
|
+
primaryStashDir,
|
|
280
|
+
memorySummary,
|
|
281
|
+
ensureIndexFn,
|
|
282
|
+
reindexFn,
|
|
283
|
+
startMs,
|
|
284
|
+
budgetMs,
|
|
285
|
+
});
|
|
286
|
+
const { crossStepErrorsInjected, memoryRefsForInference } = await runImproveLoopStage({
|
|
287
|
+
scope,
|
|
288
|
+
options,
|
|
289
|
+
primaryStashDir,
|
|
290
|
+
reflectFn,
|
|
291
|
+
distillFn,
|
|
292
|
+
loopRefs: preparation.loopRefs,
|
|
293
|
+
actions: preparation.actions,
|
|
294
|
+
signalBearingSet: preparation.signalBearingSet,
|
|
295
|
+
distillCooledRefs: preparation.distillCooledRefs,
|
|
296
|
+
recentErrors: preparation.recentErrors,
|
|
297
|
+
startMs,
|
|
298
|
+
budgetMs,
|
|
299
|
+
});
|
|
300
|
+
const { allWarnings, consolidation, deadUrls, memoryInference, graphExtraction, maintenanceActions } = await runImprovePostLoopStage({
|
|
301
|
+
scope,
|
|
302
|
+
options,
|
|
303
|
+
primaryStashDir,
|
|
304
|
+
actionableRefs: preparation.actionableRefs,
|
|
305
|
+
appliedCleanup: preparation.appliedCleanup,
|
|
306
|
+
cleanupWarnings: preparation.cleanupWarnings,
|
|
307
|
+
memorySummary,
|
|
308
|
+
memoryRefsForInference,
|
|
309
|
+
reindexFn,
|
|
310
|
+
});
|
|
311
|
+
const finalActions = maintenanceActions && maintenanceActions.length > 0
|
|
312
|
+
? [...preparation.actions, ...maintenanceActions]
|
|
313
|
+
: preparation.actions;
|
|
314
|
+
const result = {
|
|
315
|
+
schemaVersion: 1,
|
|
316
|
+
ok: true,
|
|
317
|
+
scope,
|
|
318
|
+
dryRun: false,
|
|
319
|
+
...(guidance ? { guidance } : {}),
|
|
320
|
+
memorySummary,
|
|
321
|
+
...(memoryCleanupPlan
|
|
322
|
+
? {
|
|
323
|
+
memoryCleanup: {
|
|
324
|
+
...shapeMemoryCleanup(memoryCleanupPlan),
|
|
325
|
+
...(preparation.appliedCleanup
|
|
326
|
+
? {
|
|
327
|
+
archived: preparation.appliedCleanup.archived,
|
|
328
|
+
...(preparation.appliedCleanup.transitionLogPath
|
|
329
|
+
? { transitionLogPath: preparation.appliedCleanup.transitionLogPath }
|
|
330
|
+
: {}),
|
|
331
|
+
...(preparation.appliedCleanup.transitionLogEntries !== undefined
|
|
332
|
+
? { transitionLogEntries: preparation.appliedCleanup.transitionLogEntries }
|
|
333
|
+
: {}),
|
|
334
|
+
...(allWarnings.length > 0 ? { warnings: allWarnings } : {}),
|
|
335
|
+
}
|
|
336
|
+
: preparation.cleanupWarnings.length > 0
|
|
337
|
+
? { warnings: preparation.cleanupWarnings }
|
|
338
|
+
: {}),
|
|
339
|
+
},
|
|
340
|
+
}
|
|
341
|
+
: {}),
|
|
342
|
+
plannedRefs: preparation.actionableRefs,
|
|
343
|
+
actions: finalActions,
|
|
344
|
+
...(preparation.validationFailures.length > 0 ? { validationFailures: preparation.validationFailures } : {}),
|
|
345
|
+
...(preparation.schemaRepairs.length > 0 ? { schemaRepairs: preparation.schemaRepairs } : {}),
|
|
346
|
+
...(consolidation.processed > 0 || consolidation.warnings.length > 0 ? { consolidation } : {}),
|
|
347
|
+
...(preparation.lintSummary !== undefined ? { lintSummary: preparation.lintSummary } : {}),
|
|
348
|
+
...(preparation.memoryIndexHealth !== undefined ? { memoryIndexHealth: preparation.memoryIndexHealth } : {}),
|
|
349
|
+
feedbackRatioUsed: preparation.feedbackRatioUsed,
|
|
350
|
+
...(preparation.coverageGaps.length > 0 ? { coverageGaps: preparation.coverageGaps } : {}),
|
|
351
|
+
...(preparation.executionLogCandidates.length > 0
|
|
352
|
+
? { executionLogCandidates: preparation.executionLogCandidates }
|
|
353
|
+
: {}),
|
|
354
|
+
...(primaryStashDir !== undefined ? { evalCasesWritten: countEvalCases(primaryStashDir) } : {}),
|
|
355
|
+
...(deadUrls !== undefined && deadUrls.length > 0 ? { deadUrls } : {}),
|
|
356
|
+
...(crossStepErrorsInjected > 0 ? { crossStepErrorsInjected } : {}),
|
|
357
|
+
...(memoryInference ? { memoryInference } : {}),
|
|
358
|
+
...(graphExtraction ? { graphExtraction } : {}),
|
|
359
|
+
};
|
|
360
|
+
if (!result.dryRun)
|
|
361
|
+
emitImproveCompletedEvent(result);
|
|
362
|
+
return result;
|
|
363
|
+
}
|
|
364
|
+
finally {
|
|
365
|
+
try {
|
|
366
|
+
fs.unlinkSync(resolvedLockPath);
|
|
367
|
+
}
|
|
368
|
+
catch {
|
|
369
|
+
// ignore
|
|
370
|
+
}
|
|
371
|
+
}
|
|
372
|
+
}
|
|
373
|
+
function emitImproveCompletedEvent(result) {
|
|
374
|
+
const actionCounts = {
|
|
375
|
+
reflect: 0,
|
|
376
|
+
distill: 0,
|
|
377
|
+
distillSkipped: 0,
|
|
378
|
+
memoryPrune: 0,
|
|
379
|
+
memoryInference: 0,
|
|
380
|
+
graphExtraction: 0,
|
|
381
|
+
error: 0,
|
|
382
|
+
};
|
|
383
|
+
for (const action of result.actions ?? []) {
|
|
384
|
+
switch (action.mode) {
|
|
385
|
+
case "reflect":
|
|
386
|
+
actionCounts.reflect += 1;
|
|
387
|
+
break;
|
|
388
|
+
case "distill":
|
|
389
|
+
actionCounts.distill += 1;
|
|
390
|
+
break;
|
|
391
|
+
case "distill-skipped":
|
|
392
|
+
actionCounts.distillSkipped += 1;
|
|
393
|
+
break;
|
|
394
|
+
case "memory-prune":
|
|
395
|
+
actionCounts.memoryPrune += 1;
|
|
396
|
+
break;
|
|
397
|
+
case "memory-inference":
|
|
398
|
+
actionCounts.memoryInference += 1;
|
|
399
|
+
break;
|
|
400
|
+
case "graph-extraction":
|
|
401
|
+
actionCounts.graphExtraction += 1;
|
|
402
|
+
break;
|
|
403
|
+
case "error":
|
|
404
|
+
actionCounts.error += 1;
|
|
405
|
+
break;
|
|
406
|
+
}
|
|
407
|
+
}
|
|
408
|
+
appendEvent({
|
|
409
|
+
eventType: "improve_completed",
|
|
410
|
+
ref: result.scope.mode === "ref" ? result.scope.value : `improve:${result.scope.mode}:${result.scope.value ?? "all"}`,
|
|
411
|
+
metadata: {
|
|
412
|
+
plannedRefs: result.plannedRefs.length,
|
|
413
|
+
reflectActions: actionCounts.reflect,
|
|
414
|
+
distillActions: actionCounts.distill,
|
|
415
|
+
distillSkippedActions: actionCounts.distillSkipped,
|
|
416
|
+
memoryPruneActions: actionCounts.memoryPrune,
|
|
417
|
+
memoryInferenceActions: actionCounts.memoryInference,
|
|
418
|
+
graphExtractionActions: actionCounts.graphExtraction,
|
|
419
|
+
errorActions: actionCounts.error,
|
|
420
|
+
crossStepErrorsInjected: result.crossStepErrorsInjected ?? 0,
|
|
421
|
+
feedbackRatioUsed: result.feedbackRatioUsed,
|
|
422
|
+
coverageGapCount: result.coverageGaps?.length ?? 0,
|
|
423
|
+
executionLogCandidateCount: result.executionLogCandidates?.length ?? 0,
|
|
424
|
+
evalCasesWritten: result.evalCasesWritten ?? 0,
|
|
425
|
+
deadUrlCount: result.deadUrls?.length ?? 0,
|
|
426
|
+
memoryEligible: result.memorySummary.eligible,
|
|
427
|
+
memoryDerived: result.memorySummary.derived,
|
|
428
|
+
memoryCleanupPruneCandidates: result.memoryCleanup?.pruneCandidates.length ?? 0,
|
|
429
|
+
memoryCleanupContradictionCandidates: result.memoryCleanup?.contradictionCandidates.length ?? 0,
|
|
430
|
+
memoryCleanupBeliefStateTransitions: result.memoryCleanup?.beliefStateTransitions.length ?? 0,
|
|
431
|
+
memoryCleanupConsolidationCandidates: result.memoryCleanup?.consolidationCandidates.length ?? 0,
|
|
432
|
+
memoryCleanupArchived: result.memoryCleanup?.archived?.length ?? 0,
|
|
433
|
+
memoryCleanupWarnings: result.memoryCleanup?.warnings?.length ?? 0,
|
|
434
|
+
consolidationProcessed: result.consolidation?.processed ?? 0,
|
|
435
|
+
consolidationDurationMs: result.consolidation?.durationMs ?? 0,
|
|
436
|
+
memoryInferenceWrites: result.memoryInference?.writtenFacts ?? 0,
|
|
437
|
+
memoryInferenceDurationMs: 0,
|
|
438
|
+
graphExtractionExtractedFiles: result.graphExtraction?.quality.extractedFiles ?? 0,
|
|
439
|
+
graphExtractionDurationMs: 0,
|
|
440
|
+
},
|
|
441
|
+
});
|
|
442
|
+
}
|
|
443
|
+
async function runImprovePreparationStage(args) {
|
|
444
|
+
const { scope, options, plannedRefs, memoryCleanupPlan, primaryStashDir, ensureIndexFn, reindexFn, startMs, budgetMs, } = args;
|
|
445
|
+
const actions = [];
|
|
446
|
+
const cleanupWarnings = [];
|
|
447
|
+
// Phase 0 — MEMORY.md budget check (200-line cap; warn at 180)
|
|
448
|
+
let memoryIndexHealth;
|
|
449
|
+
if (primaryStashDir) {
|
|
450
|
+
const memoryMdPath = path.join(primaryStashDir, "memories", "MEMORY.md");
|
|
451
|
+
if (fs.existsSync(memoryMdPath)) {
|
|
452
|
+
try {
|
|
453
|
+
const lines = fs.readFileSync(memoryMdPath, "utf8").split("\n").length;
|
|
454
|
+
const overBudget = lines >= 180;
|
|
455
|
+
memoryIndexHealth = { lineCount: lines, overBudget };
|
|
456
|
+
if (overBudget) {
|
|
457
|
+
cleanupWarnings.push(`MEMORY.md has ${lines} lines (budget: 200). Consolidation strongly recommended.`);
|
|
458
|
+
}
|
|
459
|
+
}
|
|
460
|
+
catch {
|
|
461
|
+
// best-effort
|
|
462
|
+
}
|
|
463
|
+
}
|
|
464
|
+
}
|
|
465
|
+
// Phase 0 — execution log synthesis
|
|
466
|
+
let executionLogCandidates = [];
|
|
467
|
+
try {
|
|
468
|
+
const logEntries = getExecutionLogCandidates(7);
|
|
469
|
+
executionLogCandidates = logEntries.filter((e) => e.isFailurePattern).map((e) => e.topic);
|
|
470
|
+
}
|
|
471
|
+
catch {
|
|
472
|
+
// best-effort
|
|
473
|
+
}
|
|
474
|
+
appendEvent({
|
|
475
|
+
eventType: "improve_invoked",
|
|
476
|
+
ref: scope.mode === "ref" ? scope.value : `improve:${scope.mode}:${scope.value ?? "all"}`,
|
|
477
|
+
metadata: { scope, dryRun: options.dryRun ?? false, assetCount: plannedRefs.length },
|
|
478
|
+
});
|
|
479
|
+
if (primaryStashDir) {
|
|
480
|
+
try {
|
|
481
|
+
await ensureIndexFn(primaryStashDir);
|
|
482
|
+
}
|
|
483
|
+
catch (err) {
|
|
484
|
+
cleanupWarnings.push(`ensureIndex failed: ${err instanceof Error ? err.message : String(err)}`);
|
|
485
|
+
}
|
|
486
|
+
}
|
|
487
|
+
let appliedCleanup;
|
|
488
|
+
try {
|
|
489
|
+
appliedCleanup =
|
|
490
|
+
primaryStashDir && memoryCleanupPlan ? applyMemoryCleanup(primaryStashDir, memoryCleanupPlan) : undefined;
|
|
491
|
+
}
|
|
492
|
+
catch (err) {
|
|
493
|
+
cleanupWarnings.push(`applyMemoryCleanup failed: ${err instanceof Error ? err.message : String(err)}`);
|
|
494
|
+
}
|
|
495
|
+
const archivedRefs = appliedCleanup?.archived.map((record) => record.ref) ?? [];
|
|
496
|
+
const postCleanupRefs = filterRemovedPlannedRefs(plannedRefs, archivedRefs);
|
|
497
|
+
// Gap 6: only surface feedback signals from the last 30 days so that
|
|
498
|
+
// ancient one-off feedback events don't permanently lock an asset into
|
|
499
|
+
// every improve run. Assets with only stale signals fall through to the
|
|
500
|
+
// high-retrieval path (P0-A) or are skipped until new signals arrive.
|
|
501
|
+
const FEEDBACK_SIGNAL_WINDOW_DAYS = 30;
|
|
502
|
+
const feedbackSinceCutoff = new Date(Date.now() - FEEDBACK_SIGNAL_WINDOW_DAYS * 24 * 60 * 60 * 1000).toISOString();
|
|
503
|
+
const signalFiltered = postCleanupRefs.filter((candidate) => {
|
|
504
|
+
const { events } = readEvents({ type: "feedback", ref: candidate.ref });
|
|
505
|
+
return events.some((e) => (e.ts ?? "") >= feedbackSinceCutoff &&
|
|
506
|
+
((e.metadata !== undefined && typeof e.metadata.signal === "string") ||
|
|
507
|
+
(e.metadata !== undefined && typeof e.metadata.note === "string")));
|
|
508
|
+
});
|
|
509
|
+
// P0-A: also surface zero-feedback assets that have been retrieved many times.
|
|
510
|
+
const RETRIEVAL_COUNT_THRESHOLD = options.minRetrievalCount ?? 5;
|
|
511
|
+
const signalBearingSet = new Set(signalFiltered.map((r) => r.ref));
|
|
512
|
+
const noFeedbackCandidates = postCleanupRefs.filter((r) => !signalBearingSet.has(r.ref));
|
|
513
|
+
let highRetrievalRefs = [];
|
|
514
|
+
let dbForRetrieval;
|
|
515
|
+
try {
|
|
516
|
+
dbForRetrieval = openExistingDatabase();
|
|
517
|
+
const showEventCount = dbForRetrieval.prepare("SELECT COUNT(*) AS cnt FROM usage_events WHERE event_type = 'show'").get().cnt;
|
|
518
|
+
if (showEventCount === 0) {
|
|
519
|
+
warn("Warning: show events not yet in usage_events — zero-feedback fallback will match only search-retrieved assets.");
|
|
520
|
+
}
|
|
521
|
+
const retrievalCounts = getRetrievalCounts(dbForRetrieval, noFeedbackCandidates.map((r) => r.ref));
|
|
522
|
+
highRetrievalRefs = noFeedbackCandidates.filter((r) => (retrievalCounts.get(r.ref) ?? 0) >= RETRIEVAL_COUNT_THRESHOLD);
|
|
523
|
+
}
|
|
524
|
+
catch {
|
|
525
|
+
// best-effort: if DB unavailable, highRetrievalRefs stays empty
|
|
526
|
+
}
|
|
527
|
+
finally {
|
|
528
|
+
if (dbForRetrieval)
|
|
529
|
+
closeDatabase(dbForRetrieval);
|
|
530
|
+
}
|
|
531
|
+
// If the user explicitly scoped to a single ref, always act on it —
|
|
532
|
+
// skip the signal/retrieval filter entirely. The filter exists to avoid
|
|
533
|
+
// noisy "improve everything" runs; it should not gate an intentional
|
|
534
|
+
// per-ref invocation where the user's explicit choice is the signal.
|
|
535
|
+
//
|
|
536
|
+
// For type/all scope with no signals yet (fresh environment), fall back
|
|
537
|
+
// to all postCleanupRefs so that the first improve run is not a no-op.
|
|
538
|
+
const signalAndRetrievalRefs = [...signalFiltered, ...highRetrievalRefs];
|
|
539
|
+
const mergedRefs = scope.mode === "ref"
|
|
540
|
+
? postCleanupRefs
|
|
541
|
+
: options.requireFeedbackSignal
|
|
542
|
+
? signalFiltered
|
|
543
|
+
: signalAndRetrievalRefs.length === 0
|
|
544
|
+
? postCleanupRefs
|
|
545
|
+
: signalAndRetrievalRefs;
|
|
546
|
+
const utilityMap = buildUtilityMap(mergedRefs);
|
|
547
|
+
// Load feedback ratio per ref and blend into sort key
|
|
548
|
+
const feedbackRatios = new Map();
|
|
549
|
+
for (const ref of mergedRefs) {
|
|
550
|
+
const { events } = readEvents({ type: "feedback", ref: ref.ref });
|
|
551
|
+
const positive = events.filter((e) => e.metadata?.signal === "positive").length;
|
|
552
|
+
const negative = events.filter((e) => e.metadata?.signal === "negative").length;
|
|
553
|
+
const total = positive + negative;
|
|
554
|
+
// ratio = negative proportion (high = needs more improvement)
|
|
555
|
+
feedbackRatios.set(ref.ref, total > 0 ? negative / total : 0);
|
|
556
|
+
}
|
|
557
|
+
// Sort: combine utility (desc) with feedback negativity (desc) — high-negative assets rank higher
|
|
558
|
+
const sorted = [...mergedRefs].sort((a, b) => {
|
|
559
|
+
const utilA = utilityMap.get(a.ref) ?? 0;
|
|
560
|
+
const utilB = utilityMap.get(b.ref) ?? 0;
|
|
561
|
+
const ratioA = feedbackRatios.get(a.ref) ?? 0;
|
|
562
|
+
const ratioB = feedbackRatios.get(b.ref) ?? 0;
|
|
563
|
+
// Combined score: 70% utility, 30% negative ratio
|
|
564
|
+
const scoreA = utilA * 0.7 + ratioA * 0.3;
|
|
565
|
+
const scoreB = utilB * 0.7 + ratioB * 0.3;
|
|
566
|
+
return scoreB - scoreA;
|
|
567
|
+
});
|
|
568
|
+
const feedbackRatioUsed = true;
|
|
569
|
+
// Phase 0: surface coverage gaps from zero-result search queries
|
|
570
|
+
let coverageGaps = [];
|
|
571
|
+
try {
|
|
572
|
+
const dbForGaps = openExistingDatabase();
|
|
573
|
+
try {
|
|
574
|
+
coverageGaps = getZeroResultSearches(dbForGaps);
|
|
575
|
+
}
|
|
576
|
+
finally {
|
|
577
|
+
closeDatabase(dbForGaps);
|
|
578
|
+
}
|
|
579
|
+
}
|
|
580
|
+
catch {
|
|
581
|
+
// best-effort
|
|
582
|
+
}
|
|
583
|
+
const actionableRefs = options.limit ? sorted.slice(0, options.limit) : sorted;
|
|
584
|
+
if (appliedCleanup) {
|
|
585
|
+
for (const candidate of memoryCleanupPlan?.pruneCandidates ?? []) {
|
|
586
|
+
const archived = appliedCleanup.archived.find((record) => record.ref === candidate.ref);
|
|
587
|
+
if (!archived)
|
|
588
|
+
continue;
|
|
589
|
+
actions.push({
|
|
590
|
+
ref: candidate.ref,
|
|
591
|
+
mode: "memory-prune",
|
|
592
|
+
result: { ok: true, pruned: true, reason: candidate.reason },
|
|
593
|
+
});
|
|
594
|
+
}
|
|
595
|
+
if ((appliedCleanup.archived.length > 0 || appliedCleanup.beliefStateTransitions.length > 0) && primaryStashDir) {
|
|
596
|
+
try {
|
|
597
|
+
await reindexFn({ stashDir: primaryStashDir });
|
|
598
|
+
}
|
|
599
|
+
catch (err) {
|
|
600
|
+
cleanupWarnings.push(`reindex after cleanup failed: ${err instanceof Error ? err.message : String(err)}`);
|
|
601
|
+
}
|
|
602
|
+
}
|
|
603
|
+
}
|
|
604
|
+
const validationFailures = [];
|
|
605
|
+
for (const candidate of actionableRefs) {
|
|
606
|
+
try {
|
|
607
|
+
const filePath = await findAssetFilePath(candidate.ref, options.stashDir);
|
|
608
|
+
if (!filePath) {
|
|
609
|
+
validationFailures.push({ ref: candidate.ref, reason: "file not found on disk" });
|
|
610
|
+
continue;
|
|
611
|
+
}
|
|
612
|
+
if (isLessonCandidate(candidate.ref)) {
|
|
613
|
+
const raw = fs.readFileSync(filePath, "utf8");
|
|
614
|
+
const fm = parseFrontmatter(raw).data;
|
|
615
|
+
if (!fm.description)
|
|
616
|
+
validationFailures.push({ ref: candidate.ref, reason: "missing description" });
|
|
617
|
+
}
|
|
618
|
+
}
|
|
619
|
+
catch (e) {
|
|
620
|
+
validationFailures.push({ ref: candidate.ref, reason: String(e) });
|
|
621
|
+
}
|
|
622
|
+
}
|
|
623
|
+
if (validationFailures.length > 0) {
|
|
624
|
+
info(`[improve] ${validationFailures.length} assets have validation issues (will be skipped):`);
|
|
625
|
+
for (const f of validationFailures)
|
|
626
|
+
info(` ${f.ref}: ${f.reason}`);
|
|
627
|
+
}
|
|
628
|
+
let schemaRepairs = [];
|
|
629
|
+
let repairedRefs = new Set();
|
|
630
|
+
// Schema repair pass: attempt to fix validation failures via LLM before skipping.
|
|
631
|
+
if (validationFailures.length > 0 && options.repairValidationFailures !== false) {
|
|
632
|
+
const baseConfigForRepair = options.config ?? loadConfig();
|
|
633
|
+
const llmCfg = baseConfigForRepair.llm;
|
|
634
|
+
if (llmCfg) {
|
|
635
|
+
const result = await runSchemaRepairPass(validationFailures, {
|
|
636
|
+
startMs,
|
|
637
|
+
budgetMs,
|
|
638
|
+
llmConfig: llmCfg,
|
|
639
|
+
stashDir: options.stashDir,
|
|
640
|
+
findFilePath: findAssetFilePath,
|
|
641
|
+
isLessonCandidateFn: isLessonCandidate,
|
|
642
|
+
});
|
|
643
|
+
schemaRepairs = result.repairs;
|
|
644
|
+
repairedRefs = result.repairedRefs;
|
|
645
|
+
}
|
|
646
|
+
}
|
|
647
|
+
const validationFailureRefs = new Set(validationFailures.filter((f) => !repairedRefs.has(f.ref)).map((f) => f.ref));
|
|
648
|
+
// Phase 0.5 — structural hygiene pass
|
|
649
|
+
let lintSummary;
|
|
650
|
+
if (primaryStashDir) {
|
|
651
|
+
try {
|
|
652
|
+
const lintResult = akmLint({ fix: true, dir: primaryStashDir });
|
|
653
|
+
lintSummary = { fixed: lintResult.summary.fixed, flagged: lintResult.summary.flagged };
|
|
654
|
+
}
|
|
655
|
+
catch {
|
|
656
|
+
// lint is best-effort; never block improve
|
|
657
|
+
}
|
|
658
|
+
}
|
|
659
|
+
const recentErrors = []; // rolling window, last 3 failures
|
|
660
|
+
const RECENT_ERRORS_CAP = 3;
|
|
661
|
+
// Seed the rolling window from any schema repair errors that occurred before the main loop.
|
|
662
|
+
for (const repair of schemaRepairs) {
|
|
663
|
+
if (repair.outcome === "error") {
|
|
664
|
+
const errMsg = repair.error ?? `schema repair error: ${repair.reason}`;
|
|
665
|
+
recentErrors.push(errMsg);
|
|
666
|
+
if (recentErrors.length > RECENT_ERRORS_CAP)
|
|
667
|
+
recentErrors.shift();
|
|
668
|
+
}
|
|
669
|
+
}
|
|
670
|
+
// ── Cooldown pre-filter ───────────────────────────────────────────────────
|
|
671
|
+
// Read all cooldown-relevant events in 4 bulk queries and materialise two
|
|
672
|
+
// Sets that the loop checks with O(1) Set.has() instead of N per-ref
|
|
673
|
+
// readEvents() + listProposals() calls. This eliminates the N×3 DB/FS
|
|
674
|
+
// round trips that caused per-asset "reflect cooldown" noise for every
|
|
675
|
+
// asset in the stash.
|
|
676
|
+
//
|
|
677
|
+
// SM-2 tier for reflect uses promoted/rejected events (recorded by
|
|
678
|
+
// `akm proposal accept/reject`) rather than the per-ref listProposals()
|
|
679
|
+
// filesystem scan, giving identical tier logic without touching the disk.
|
|
680
|
+
const REFLECT_COOLDOWN_DAYS = options.reflectCooldownDays ?? 7;
|
|
681
|
+
const DISTILL_COOLDOWN_DAYS = options.distillCooldownDays ?? 30;
|
|
682
|
+
const reflectCooledRefs = new Set();
|
|
683
|
+
const distillCooledRefs = new Set();
|
|
684
|
+
if (REFLECT_COOLDOWN_DAYS > 0 || DISTILL_COOLDOWN_DAYS > 0) {
|
|
685
|
+
const bulkWindowMs = Math.max(REFLECT_COOLDOWN_DAYS, DISTILL_COOLDOWN_DAYS, 14) * 24 * 60 * 60 * 1000;
|
|
686
|
+
const bulkSince = new Date(Date.now() - bulkWindowMs).toISOString();
|
|
687
|
+
const bulkReflects = readEvents({ type: "reflect_invoked", since: bulkSince }).events;
|
|
688
|
+
const bulkDistills = readEvents({ type: "distill_invoked", since: bulkSince }).events;
|
|
689
|
+
const bulkPromoted = readEvents({ type: "promoted", since: bulkSince }).events;
|
|
690
|
+
const bulkRejected = readEvents({ type: "rejected", since: bulkSince }).events;
|
|
691
|
+
const promotedTs = new Map();
|
|
692
|
+
for (const e of bulkPromoted) {
|
|
693
|
+
if (e.ref && (e.ts ?? "") > (promotedTs.get(e.ref) ?? ""))
|
|
694
|
+
promotedTs.set(e.ref, e.ts ?? "");
|
|
695
|
+
}
|
|
696
|
+
const rejectedTs = new Map();
|
|
697
|
+
for (const e of bulkRejected) {
|
|
698
|
+
if (e.ref && (e.ts ?? "") > (rejectedTs.get(e.ref) ?? ""))
|
|
699
|
+
rejectedTs.set(e.ref, e.ts ?? "");
|
|
700
|
+
}
|
|
701
|
+
if (REFLECT_COOLDOWN_DAYS > 0) {
|
|
702
|
+
const latestReflect = new Map();
|
|
703
|
+
for (const e of bulkReflects) {
|
|
704
|
+
if (e.ref && (e.ts ?? "") > (latestReflect.get(e.ref) ?? ""))
|
|
705
|
+
latestReflect.set(e.ref, e.ts ?? "");
|
|
706
|
+
}
|
|
707
|
+
for (const [ref, lastTs] of latestReflect) {
|
|
708
|
+
if (!lastTs)
|
|
709
|
+
continue;
|
|
710
|
+
const hasAccepted = (promotedTs.get(ref) ?? "") > lastTs;
|
|
711
|
+
const hasRejected = (rejectedTs.get(ref) ?? "") > lastTs;
|
|
712
|
+
let effectiveCooldownDays = REFLECT_COOLDOWN_DAYS;
|
|
713
|
+
if (hasAccepted)
|
|
714
|
+
continue;
|
|
715
|
+
else if (hasRejected)
|
|
716
|
+
effectiveCooldownDays = Math.min(REFLECT_COOLDOWN_DAYS, 3);
|
|
717
|
+
if (Date.now() - new Date(lastTs).getTime() < effectiveCooldownDays * 24 * 60 * 60 * 1000) {
|
|
718
|
+
reflectCooledRefs.add(ref);
|
|
719
|
+
}
|
|
720
|
+
}
|
|
721
|
+
}
|
|
722
|
+
if (DISTILL_COOLDOWN_DAYS > 0) {
|
|
723
|
+
const distillCooldownMs = DISTILL_COOLDOWN_DAYS * 24 * 60 * 60 * 1000;
|
|
724
|
+
const latestQueuedDistill = new Map();
|
|
725
|
+
for (const e of bulkDistills) {
|
|
726
|
+
if (e.ref && e.metadata?.outcome === "queued" && (e.ts ?? "") > (latestQueuedDistill.get(e.ref) ?? "")) {
|
|
727
|
+
latestQueuedDistill.set(e.ref, e.ts ?? "");
|
|
728
|
+
}
|
|
729
|
+
}
|
|
730
|
+
for (const [ref, lastTs] of latestQueuedDistill) {
|
|
731
|
+
if (lastTs && Date.now() - new Date(lastTs).getTime() < distillCooldownMs) {
|
|
732
|
+
distillCooledRefs.add(ref);
|
|
733
|
+
}
|
|
734
|
+
}
|
|
735
|
+
}
|
|
736
|
+
}
|
|
737
|
+
const loopRefs = actionableRefs.filter((r) => !reflectCooledRefs.has(r.ref) && !validationFailureRefs.has(r.ref));
|
|
738
|
+
const reflectCooledLoop = actionableRefs.filter((r) => reflectCooledRefs.has(r.ref));
|
|
739
|
+
if (reflectCooledLoop.length > 0) {
|
|
740
|
+
info(`[improve] ${reflectCooledLoop.length}/${actionableRefs.length} assets on reflect cooldown — skipping`);
|
|
741
|
+
for (const r of reflectCooledLoop) {
|
|
742
|
+
actions.push({
|
|
743
|
+
ref: r.ref,
|
|
744
|
+
mode: "distill-skipped",
|
|
745
|
+
result: { ok: true, reason: "reflect cooldown (pre-filtered)" },
|
|
746
|
+
});
|
|
747
|
+
appendEvent({ eventType: "improve_skipped", ref: r.ref, metadata: { reason: "reflect_cooldown" } });
|
|
748
|
+
}
|
|
749
|
+
}
|
|
750
|
+
if (validationFailureRefs.size > 0) {
|
|
751
|
+
info(`[improve] ${validationFailureRefs.size} assets with validation failures excluded from loop`);
|
|
752
|
+
}
|
|
753
|
+
return {
|
|
754
|
+
actions,
|
|
755
|
+
cleanupWarnings,
|
|
756
|
+
appliedCleanup,
|
|
757
|
+
memoryIndexHealth,
|
|
758
|
+
executionLogCandidates,
|
|
759
|
+
actionableRefs,
|
|
760
|
+
signalBearingSet,
|
|
761
|
+
validationFailures,
|
|
762
|
+
schemaRepairs,
|
|
763
|
+
lintSummary,
|
|
764
|
+
loopRefs,
|
|
765
|
+
distillCooledRefs,
|
|
766
|
+
feedbackRatioUsed,
|
|
767
|
+
coverageGaps,
|
|
768
|
+
recentErrors,
|
|
769
|
+
};
|
|
770
|
+
}
|
|
771
|
+
async function runImproveLoopStage(args) {
|
|
772
|
+
const { scope, options, primaryStashDir, reflectFn, distillFn, loopRefs, actions, signalBearingSet, distillCooledRefs, recentErrors, startMs, budgetMs, } = args;
|
|
773
|
+
const RECENT_ERRORS_CAP = 3;
|
|
774
|
+
const DISTILL_COOLDOWN_DAYS = options.distillCooldownDays ?? 30;
|
|
775
|
+
let completedCount = 0;
|
|
776
|
+
let crossStepErrorsInjected = 0;
|
|
777
|
+
const memoryRefsForInference = new Set();
|
|
778
|
+
for (const planned of loopRefs) {
|
|
779
|
+
if (Date.now() - startMs >= budgetMs) {
|
|
780
|
+
const remaining = loopRefs.length - completedCount;
|
|
781
|
+
info(`[improve] budget exhausted after ${Math.round((Date.now() - startMs) / 60000)}min — ${remaining} assets skipped`);
|
|
782
|
+
appendEvent({
|
|
783
|
+
eventType: "improve_skipped",
|
|
784
|
+
ref: planned.ref,
|
|
785
|
+
metadata: {
|
|
786
|
+
reason: "budget_exhausted",
|
|
787
|
+
remaining,
|
|
788
|
+
},
|
|
789
|
+
});
|
|
790
|
+
actions.push({
|
|
791
|
+
ref: planned.ref,
|
|
792
|
+
mode: "error",
|
|
793
|
+
result: { ok: false, error: "timeout: improve wall-clock budget exhausted" },
|
|
794
|
+
});
|
|
795
|
+
break;
|
|
796
|
+
}
|
|
797
|
+
try {
|
|
798
|
+
if (DISTILL_COOLDOWN_DAYS > 0 &&
|
|
799
|
+
distillCooledRefs.has(planned.ref) &&
|
|
800
|
+
(isLessonCandidate(planned.ref) || shouldDistillMemoryRef(planned.ref, options.stashDir))) {
|
|
801
|
+
actions.push({
|
|
802
|
+
ref: planned.ref,
|
|
803
|
+
mode: "distill-skipped",
|
|
804
|
+
result: { ok: true, reason: "distill cooldown" },
|
|
805
|
+
});
|
|
806
|
+
completedCount++;
|
|
807
|
+
appendEvent({
|
|
808
|
+
eventType: "improve_skipped",
|
|
809
|
+
ref: planned.ref,
|
|
810
|
+
metadata: { reason: "distill_cooldown", cooldownDays: DISTILL_COOLDOWN_DAYS },
|
|
811
|
+
});
|
|
812
|
+
info(`[improve] ${completedCount}/${loopRefs.length} ${planned.ref} (distill cooldown)`);
|
|
813
|
+
continue;
|
|
814
|
+
}
|
|
815
|
+
if (recentErrors.length > 0)
|
|
816
|
+
crossStepErrorsInjected++;
|
|
817
|
+
const reflectResult = await reflectFn({
|
|
818
|
+
ref: planned.ref,
|
|
819
|
+
task: options.task,
|
|
820
|
+
...(options.stashDir ? { stashDir: options.stashDir } : {}),
|
|
821
|
+
...(recentErrors.length > 0 ? { avoidPatterns: [...recentErrors] } : {}),
|
|
822
|
+
agentProcess: options.agentProcess ?? "reflect",
|
|
823
|
+
});
|
|
824
|
+
actions.push({ ref: planned.ref, mode: "reflect", result: reflectResult });
|
|
825
|
+
if (!reflectResult.ok) {
|
|
826
|
+
const errMsg = reflectResult.error ?? reflectResult.reason ?? "unknown reflect error";
|
|
827
|
+
recentErrors.push(errMsg);
|
|
828
|
+
if (recentErrors.length > RECENT_ERRORS_CAP)
|
|
829
|
+
recentErrors.shift();
|
|
830
|
+
}
|
|
831
|
+
const parsedPlannedRef = parseAssetRef(planned.ref);
|
|
832
|
+
const hasRecentFeedbackSignal = signalBearingSet.has(planned.ref);
|
|
833
|
+
const explicitRefScope = scope.mode === "ref";
|
|
834
|
+
const shouldAttemptDistill = isLessonCandidate(planned.ref) || shouldDistillMemoryRef(planned.ref, options.stashDir);
|
|
835
|
+
const skipMemoryDistillForWeakSignal = parsedPlannedRef.type === "memory" && !hasRecentFeedbackSignal && !explicitRefScope;
|
|
836
|
+
if (shouldAttemptDistill && !skipMemoryDistillForWeakSignal) {
|
|
837
|
+
const lessonRef = deriveLessonRef(planned.ref);
|
|
838
|
+
const dedupeStashDir = primaryStashDir ?? options.stashDir;
|
|
839
|
+
if (dedupeStashDir) {
|
|
840
|
+
const existingProposals = listProposals(dedupeStashDir, { ref: lessonRef });
|
|
841
|
+
if (existingProposals.some((p) => p.status === "pending")) {
|
|
842
|
+
actions.push({
|
|
843
|
+
ref: planned.ref,
|
|
844
|
+
mode: "distill-skipped",
|
|
845
|
+
result: { ok: true, reason: "pending proposal exists" },
|
|
846
|
+
});
|
|
847
|
+
completedCount++;
|
|
848
|
+
info(`[improve] ${completedCount}/${loopRefs.length} ${planned.ref}`);
|
|
849
|
+
continue;
|
|
850
|
+
}
|
|
851
|
+
}
|
|
852
|
+
const distillResult = await distillFn({
|
|
853
|
+
ref: planned.ref,
|
|
854
|
+
...(parsedPlannedRef.type === "memory" ? { proposalKind: "auto" } : {}),
|
|
855
|
+
...(options.stashDir ? { stashDir: options.stashDir } : {}),
|
|
856
|
+
});
|
|
857
|
+
actions.push({ ref: planned.ref, mode: "distill", result: distillResult });
|
|
858
|
+
if (parsedPlannedRef.type === "memory") {
|
|
859
|
+
const promotedToKnowledge = distillResult.outcome === "queued" && distillResult.proposalKind === "knowledge";
|
|
860
|
+
if (!promotedToKnowledge)
|
|
861
|
+
memoryRefsForInference.add(planned.ref);
|
|
862
|
+
}
|
|
863
|
+
if (distillResult.outcome === "quality_rejected" && primaryStashDir) {
|
|
864
|
+
const slug = planned.ref
|
|
865
|
+
.replace(/[^a-z0-9]/gi, "-")
|
|
866
|
+
.toLowerCase()
|
|
867
|
+
.slice(0, 60);
|
|
868
|
+
writeEvalCase(primaryStashDir, {
|
|
869
|
+
ref: planned.ref,
|
|
870
|
+
failureReason: distillResult.reason ?? "quality gate rejected",
|
|
871
|
+
assetType: parseAssetRef(planned.ref).type ?? "unknown",
|
|
872
|
+
rejectedAt: Date.now(),
|
|
873
|
+
source: "distill_quality_rejected",
|
|
874
|
+
slug: `${slug}-${Date.now()}`,
|
|
875
|
+
});
|
|
876
|
+
}
|
|
877
|
+
const rejectedProposals = readEvents({ type: "proposal_rejected", ref: planned.ref }).events.filter((e) => new Date(e.ts).getTime() >= Date.now() - 30 * 24 * 60 * 60 * 1000);
|
|
878
|
+
if (rejectedProposals.length > 0 && primaryStashDir) {
|
|
879
|
+
const slug = planned.ref
|
|
880
|
+
.replace(/[^a-z0-9]/gi, "-")
|
|
881
|
+
.toLowerCase()
|
|
882
|
+
.slice(0, 60);
|
|
883
|
+
writeEvalCase(primaryStashDir, {
|
|
884
|
+
ref: planned.ref,
|
|
885
|
+
failureReason: rejectedProposals[0].metadata?.reason ?? "proposal rejected",
|
|
886
|
+
assetType: parseAssetRef(planned.ref).type ?? "unknown",
|
|
887
|
+
rejectedAt: new Date(rejectedProposals[0].ts).getTime(),
|
|
888
|
+
source: "proposal_rejected",
|
|
889
|
+
slug: `${slug}-rejected`,
|
|
890
|
+
});
|
|
891
|
+
}
|
|
892
|
+
}
|
|
893
|
+
else if (skipMemoryDistillForWeakSignal) {
|
|
894
|
+
actions.push({
|
|
895
|
+
ref: planned.ref,
|
|
896
|
+
mode: "distill-skipped",
|
|
897
|
+
result: { ok: true, reason: "memory requires recent feedback signal" },
|
|
898
|
+
});
|
|
899
|
+
appendEvent({
|
|
900
|
+
eventType: "improve_skipped",
|
|
901
|
+
ref: planned.ref,
|
|
902
|
+
metadata: { reason: "memory_distill_requires_feedback" },
|
|
903
|
+
});
|
|
904
|
+
}
|
|
905
|
+
}
|
|
906
|
+
catch (err) {
|
|
907
|
+
actions.push({
|
|
908
|
+
ref: planned.ref,
|
|
909
|
+
mode: "error",
|
|
910
|
+
result: { ok: false, error: err instanceof Error ? err.message : String(err) },
|
|
911
|
+
});
|
|
912
|
+
}
|
|
913
|
+
completedCount++;
|
|
914
|
+
info(`[improve] ${completedCount}/${loopRefs.length} ${planned.ref}`);
|
|
915
|
+
}
|
|
916
|
+
return { crossStepErrorsInjected, memoryRefsForInference };
|
|
917
|
+
}
|
|
918
|
+
async function runImprovePostLoopStage(args) {
|
|
919
|
+
const { scope, options, primaryStashDir, actionableRefs, appliedCleanup, cleanupWarnings, memorySummary, memoryRefsForInference, reindexFn, } = args;
|
|
920
|
+
const allWarnings = [...cleanupWarnings, ...(appliedCleanup?.warnings ?? [])];
|
|
921
|
+
const baseConfig = options.config ?? loadConfig();
|
|
922
|
+
const MEMORY_VOLUME_THRESHOLD = options.memoryVolumeConsolidationThreshold ?? 100;
|
|
923
|
+
const hasLlm = !!(baseConfig.llm || baseConfig.agent);
|
|
924
|
+
const volumeTriggered = typeof memorySummary.eligible === "number" && memorySummary.eligible > MEMORY_VOLUME_THRESHOLD && hasLlm;
|
|
925
|
+
const consolidationConfig = volumeTriggered
|
|
926
|
+
? {
|
|
927
|
+
...baseConfig,
|
|
928
|
+
...(baseConfig.llm
|
|
929
|
+
? {
|
|
930
|
+
llm: {
|
|
931
|
+
...baseConfig.llm,
|
|
932
|
+
features: { ...baseConfig.llm.features, memory_consolidation: true },
|
|
933
|
+
},
|
|
934
|
+
}
|
|
935
|
+
: {}),
|
|
936
|
+
}
|
|
937
|
+
: baseConfig;
|
|
938
|
+
const consolidateCooldownDays = options.consolidateCooldownDays ?? 14;
|
|
939
|
+
const CONSOLIDATE_COOLDOWN_MS = consolidateCooldownDays * 24 * 60 * 60 * 1000;
|
|
940
|
+
const recentConsolidations = readEvents({ type: "consolidate_completed" });
|
|
941
|
+
const lastConsolidation = recentConsolidations.events
|
|
942
|
+
.filter((e) => e.metadata?.processed && Number(e.metadata.processed) > 0)
|
|
943
|
+
.sort((a, b) => new Date(b.ts ?? 0).getTime() - new Date(a.ts ?? 0).getTime())[0];
|
|
944
|
+
const consolidationOnCooldown = !volumeTriggered &&
|
|
945
|
+
consolidateCooldownDays > 0 &&
|
|
946
|
+
lastConsolidation?.ts &&
|
|
947
|
+
Date.now() - new Date(lastConsolidation.ts).getTime() < CONSOLIDATE_COOLDOWN_MS;
|
|
948
|
+
let consolidation = {
|
|
949
|
+
schemaVersion: 1,
|
|
950
|
+
ok: true,
|
|
951
|
+
shape: "consolidate-result",
|
|
952
|
+
dryRun: false,
|
|
953
|
+
previewOnly: false,
|
|
954
|
+
target: "",
|
|
955
|
+
processed: 0,
|
|
956
|
+
merged: 0,
|
|
957
|
+
deleted: 0,
|
|
958
|
+
promoted: [],
|
|
959
|
+
warnings: [],
|
|
960
|
+
durationMs: 0,
|
|
961
|
+
};
|
|
962
|
+
if (!consolidationOnCooldown) {
|
|
963
|
+
consolidation = await akmConsolidate({
|
|
964
|
+
...options.consolidateOptions,
|
|
965
|
+
config: consolidationConfig,
|
|
966
|
+
stashDir: options.stashDir,
|
|
967
|
+
autoTriggered: volumeTriggered,
|
|
968
|
+
autoAccept: "safe",
|
|
969
|
+
});
|
|
970
|
+
if (consolidation.processed > 0) {
|
|
971
|
+
appendEvent({
|
|
972
|
+
eventType: "consolidate_completed",
|
|
973
|
+
ref: "memory:_consolidation",
|
|
974
|
+
metadata: { processed: consolidation.processed, merged: consolidation.merged },
|
|
975
|
+
});
|
|
976
|
+
}
|
|
977
|
+
}
|
|
978
|
+
else {
|
|
979
|
+
const daysAgo = Math.round((Date.now() - new Date(lastConsolidation?.ts ?? 0).getTime()) / 86400000);
|
|
980
|
+
appendEvent({
|
|
981
|
+
eventType: "improve_skipped",
|
|
982
|
+
ref: "memory:_consolidation",
|
|
983
|
+
metadata: {
|
|
984
|
+
reason: "consolidation_cooldown",
|
|
985
|
+
cooldownDays: 14,
|
|
986
|
+
lastEventTs: lastConsolidation?.ts ?? null,
|
|
987
|
+
},
|
|
988
|
+
});
|
|
989
|
+
info(`[improve] consolidation skipped (last ran ${daysAgo}d ago, cooldown 14d)`);
|
|
990
|
+
}
|
|
991
|
+
info("[improve] post-loop maintenance starting");
|
|
992
|
+
const maintenanceResult = await runImproveMaintenancePasses({
|
|
993
|
+
options,
|
|
994
|
+
primaryStashDir,
|
|
995
|
+
actionableRefs,
|
|
996
|
+
memoryRefsForInference,
|
|
997
|
+
allWarnings,
|
|
998
|
+
reindexFn,
|
|
999
|
+
});
|
|
1000
|
+
let deadUrls;
|
|
1001
|
+
if (scope.mode === "all" && primaryStashDir && actionableRefs.length > 0) {
|
|
1002
|
+
try {
|
|
1003
|
+
const knowledgeEntries = actionableRefs
|
|
1004
|
+
.filter((r) => {
|
|
1005
|
+
try {
|
|
1006
|
+
return parseAssetRef(r.ref).type === "knowledge";
|
|
1007
|
+
}
|
|
1008
|
+
catch {
|
|
1009
|
+
return false;
|
|
1010
|
+
}
|
|
1011
|
+
})
|
|
1012
|
+
.slice(0, 10)
|
|
1013
|
+
.map((r) => ({ ref: r.ref, body: "" }));
|
|
1014
|
+
if (knowledgeEntries.length > 0) {
|
|
1015
|
+
info(`[improve] checking URLs in ${knowledgeEntries.length} knowledge refs`);
|
|
1016
|
+
deadUrls = await checkDeadUrls(primaryStashDir, knowledgeEntries);
|
|
1017
|
+
info(`[improve] URL check complete (${deadUrls.length} dead/timeout URLs)`);
|
|
1018
|
+
}
|
|
1019
|
+
}
|
|
1020
|
+
catch {
|
|
1021
|
+
// best-effort
|
|
1022
|
+
}
|
|
1023
|
+
}
|
|
1024
|
+
return {
|
|
1025
|
+
allWarnings,
|
|
1026
|
+
consolidation,
|
|
1027
|
+
deadUrls,
|
|
1028
|
+
...(maintenanceResult.memoryInference ? { memoryInference: maintenanceResult.memoryInference } : {}),
|
|
1029
|
+
...(maintenanceResult.graphExtraction ? { graphExtraction: maintenanceResult.graphExtraction } : {}),
|
|
1030
|
+
...(maintenanceResult.actions && maintenanceResult.actions.length > 0
|
|
1031
|
+
? { maintenanceActions: maintenanceResult.actions }
|
|
1032
|
+
: {}),
|
|
1033
|
+
};
|
|
1034
|
+
}
|
|
1035
|
+
async function runImproveMaintenancePasses(args) {
|
|
1036
|
+
const { options, primaryStashDir, memoryRefsForInference, allWarnings, reindexFn } = args;
|
|
1037
|
+
if (!primaryStashDir)
|
|
1038
|
+
return {};
|
|
1039
|
+
const config = options.config ?? loadConfig();
|
|
1040
|
+
const sources = resolveSourceEntries(options.stashDir, config);
|
|
1041
|
+
const memoryInferenceFn = options.memoryInferenceFn ?? runMemoryInferencePass;
|
|
1042
|
+
const graphExtractionFn = options.graphExtractionFn ?? runGraphExtractionPass;
|
|
1043
|
+
let db;
|
|
1044
|
+
let memoryInference;
|
|
1045
|
+
let graphExtraction;
|
|
1046
|
+
let reindexedAfterInference = false;
|
|
1047
|
+
const actions = [];
|
|
1048
|
+
try {
|
|
1049
|
+
db = openDatabase(getDbPath(), config.embedding?.dimension ? { embeddingDim: config.embedding.dimension } : undefined);
|
|
1050
|
+
if (memoryRefsForInference.size > 0) {
|
|
1051
|
+
info(`[improve] memory inference starting (${memoryRefsForInference.size} candidate refs)`);
|
|
1052
|
+
try {
|
|
1053
|
+
memoryInference = await memoryInferenceFn(config, sources, undefined, db, false, (event) => {
|
|
1054
|
+
const current = event.currentRef ? ` ${event.currentRef}` : "";
|
|
1055
|
+
info(`[improve] memory inference ${event.processed}/${event.total}${current} (written ${event.writtenFacts}, skipped ${event.skippedNoFacts})`);
|
|
1056
|
+
}, {
|
|
1057
|
+
candidateRefs: memoryRefsForInference,
|
|
1058
|
+
});
|
|
1059
|
+
actions.push({ ref: "memory:_inference", mode: "memory-inference", result: memoryInference });
|
|
1060
|
+
info(`[improve] memory inference complete (${memoryInference.writtenFacts} facts written from ${memoryInference.splitParents} parents)`);
|
|
1061
|
+
}
|
|
1062
|
+
catch (err) {
|
|
1063
|
+
allWarnings.push(`memory inference failed: ${err instanceof Error ? err.message : String(err)}`);
|
|
1064
|
+
}
|
|
1065
|
+
}
|
|
1066
|
+
if (memoryInference && (memoryInference.splitParents > 0 || memoryInference.writtenFacts > 0)) {
|
|
1067
|
+
info("[improve] reindexing after memory inference writes");
|
|
1068
|
+
try {
|
|
1069
|
+
await reindexFn({ stashDir: primaryStashDir });
|
|
1070
|
+
reindexedAfterInference = true;
|
|
1071
|
+
info("[improve] reindex after memory inference complete");
|
|
1072
|
+
}
|
|
1073
|
+
catch (err) {
|
|
1074
|
+
allWarnings.push(`reindex after memory inference failed: ${err instanceof Error ? err.message : String(err)}`);
|
|
1075
|
+
}
|
|
1076
|
+
}
|
|
1077
|
+
if (sources.length > 0) {
|
|
1078
|
+
info("[improve] graph extraction starting");
|
|
1079
|
+
try {
|
|
1080
|
+
if (db && reindexedAfterInference) {
|
|
1081
|
+
closeDatabase(db);
|
|
1082
|
+
db = openDatabase(getDbPath(), config.embedding?.dimension ? { embeddingDim: config.embedding.dimension } : undefined);
|
|
1083
|
+
}
|
|
1084
|
+
graphExtraction = await graphExtractionFn(config, sources, undefined, db, false, (event) => {
|
|
1085
|
+
const current = event.currentPath ? ` ${path.basename(event.currentPath)}` : "";
|
|
1086
|
+
info(`[improve] graph extraction ${event.processed}/${event.total}${current} (extracted ${event.extracted}, entities ${event.totalEntities}, relations ${event.totalRelations})`);
|
|
1087
|
+
});
|
|
1088
|
+
actions.push({ ref: "graph:_artifact", mode: "graph-extraction", result: graphExtraction });
|
|
1089
|
+
info(`[improve] graph extraction complete (${graphExtraction.quality.extractedFiles} files, ${graphExtraction.quality.entityCount} entities, ${graphExtraction.quality.relationCount} relations)`);
|
|
1090
|
+
}
|
|
1091
|
+
catch (err) {
|
|
1092
|
+
allWarnings.push(`graph extraction failed: ${err instanceof Error ? err.message : String(err)}`);
|
|
1093
|
+
}
|
|
1094
|
+
}
|
|
1095
|
+
}
|
|
1096
|
+
finally {
|
|
1097
|
+
if (db)
|
|
1098
|
+
closeDatabase(db);
|
|
1099
|
+
}
|
|
1100
|
+
return {
|
|
1101
|
+
...(memoryInference ? { memoryInference } : {}),
|
|
1102
|
+
...(graphExtraction ? { graphExtraction } : {}),
|
|
1103
|
+
...(actions.length > 0 ? { actions } : {}),
|
|
1104
|
+
};
|
|
1105
|
+
}
|
|
1106
|
+
function shouldAnalyzeMemoryCleanup(scope, eligibleMemories, primaryStashDir) {
|
|
1107
|
+
if (!primaryStashDir || eligibleMemories === 0)
|
|
1108
|
+
return false;
|
|
1109
|
+
if (scope.mode === "all")
|
|
1110
|
+
return true;
|
|
1111
|
+
if (scope.mode === "type")
|
|
1112
|
+
return scope.value === "memory";
|
|
1113
|
+
if (!scope.value)
|
|
1114
|
+
return false;
|
|
1115
|
+
return parseAssetRef(scope.value).type === "memory";
|
|
1116
|
+
}
|
|
1117
|
+
function shapeMemoryCleanup(plan) {
|
|
1118
|
+
return {
|
|
1119
|
+
analyzedDerived: plan.analyzedDerived,
|
|
1120
|
+
pruneCandidates: plan.pruneCandidates,
|
|
1121
|
+
contradictionCandidates: plan.contradictionCandidates,
|
|
1122
|
+
beliefStateTransitions: plan.beliefStateTransitions,
|
|
1123
|
+
consolidationCandidates: plan.consolidationCandidates,
|
|
1124
|
+
...(plan.relativeDateCandidates.length > 0 ? { relativeDateCandidates: plan.relativeDateCandidates } : {}),
|
|
1125
|
+
};
|
|
1126
|
+
}
|
|
1127
|
+
function buildUtilityMap(refs) {
|
|
1128
|
+
const map = new Map();
|
|
1129
|
+
if (refs.length === 0)
|
|
1130
|
+
return map;
|
|
1131
|
+
const refSet = new Set(refs.map((r) => r.ref));
|
|
1132
|
+
let db;
|
|
1133
|
+
try {
|
|
1134
|
+
db = openExistingDatabase();
|
|
1135
|
+
const allDbEntries = getAllEntries(db);
|
|
1136
|
+
const idToRef = new Map();
|
|
1137
|
+
for (const indexed of allDbEntries) {
|
|
1138
|
+
const ref = makeAssetRef(indexed.entry.type, indexed.entry.name);
|
|
1139
|
+
if (refSet.has(ref))
|
|
1140
|
+
idToRef.set(indexed.id, ref);
|
|
1141
|
+
}
|
|
1142
|
+
const ids = [...idToRef.keys()];
|
|
1143
|
+
if (ids.length > 0) {
|
|
1144
|
+
const scores = getUtilityScoresByIds(db, ids);
|
|
1145
|
+
for (const [id, score] of scores) {
|
|
1146
|
+
const ref = idToRef.get(id);
|
|
1147
|
+
if (ref)
|
|
1148
|
+
map.set(ref, score.utility);
|
|
1149
|
+
}
|
|
1150
|
+
}
|
|
1151
|
+
}
|
|
1152
|
+
catch {
|
|
1153
|
+
// best-effort: if DB unavailable, all utilities default to 0
|
|
1154
|
+
}
|
|
1155
|
+
finally {
|
|
1156
|
+
if (db)
|
|
1157
|
+
closeDatabase(db);
|
|
1158
|
+
}
|
|
1159
|
+
return map;
|
|
1160
|
+
}
|
|
1161
|
+
async function findAssetFilePath(ref, stashDir, writableDirSet) {
|
|
1162
|
+
return resolveAssetPath(ref, {
|
|
1163
|
+
stashDir,
|
|
1164
|
+
mode: "disk-only",
|
|
1165
|
+
writableDirSet,
|
|
1166
|
+
directoryIndexNames: ["SKILL.md"],
|
|
1167
|
+
preserveDirectNameFallback: true,
|
|
1168
|
+
honorOrigin: false,
|
|
1169
|
+
});
|
|
1170
|
+
}
|