@nowcrew/daemon 0.6.101-beta.3 → 0.6.101-beta.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cloud-report-upload.js +1 -10
- package/dist/execution-journal-lock.js +51 -2
- package/dist/execution-journal.js +3 -1
- package/dist/execution-protocol.js +1 -1
- package/dist/execution-runner.js +4 -1
- package/dist/host-execution-coordinator.js +10 -0
- package/dist/local-execution-prompt.js +8 -3
- package/dist/local-executor.js +78 -55
- package/dist/local-memory/commit-telemetry.js +30 -0
- package/dist/local-memory/execution-workspace.js +117 -0
- package/dist/local-memory/full-index-search.js +47 -0
- package/dist/local-memory/index-budget.js +9 -3
- package/dist/local-memory/initialization-identity.js +36 -0
- package/dist/local-memory/inventory-cli.js +52 -8
- package/dist/local-memory/inventory-source.js +15 -8
- package/dist/local-memory/inventory-v2.js +362 -0
- package/dist/local-memory/legacy-adapter.js +233 -0
- package/dist/local-memory/migration-access.js +242 -0
- package/dist/local-memory/migration-cli.js +54 -0
- package/dist/local-memory/migration-io.js +199 -0
- package/dist/local-memory/migration-pending.js +9 -0
- package/dist/local-memory/migration-plan.js +432 -0
- package/dist/local-memory/migration-schema.js +58 -0
- package/dist/local-memory/migration-state-invariants.js +65 -0
- package/dist/local-memory/migration-store.js +414 -0
- package/dist/local-memory/migration-tool-entry.js +21 -0
- package/dist/local-memory/pending-query.js +5 -2
- package/dist/local-memory/pending-schema.js +1 -1
- package/dist/local-memory/pending.js +16 -2
- package/dist/local-memory/prompt.js +21 -5
- package/dist/local-memory/read-cli.js +98 -76
- package/dist/local-memory/release-artifact-guard.js +55 -0
- package/dist/local-memory/rollout.js +79 -11
- package/dist/local-memory/runtime.js +191 -17
- package/dist/local-memory/source-path.js +13 -0
- package/dist/local-memory/storage-format.js +249 -0
- package/dist/local-memory/write.js +190 -21
- package/dist/local-memory/writer-mode.js +13 -0
- package/dist/local-memory-diagnostics.js +96 -15
- package/dist/local-memory-telemetry.js +31 -3
- package/dist/machine-info.js +6 -1
- package/dist/machine-runtime-environment.js +1 -0
- package/dist/marketplace-protocol.js +1 -1
- package/dist/marketplace-remote.js +1 -1
- package/dist/marketplace-runner.js +1 -1
- package/dist/memory-migration-controller.js +89 -0
- package/dist/memory-migration-inspect.js +116 -0
- package/dist/memory-migration-journal.js +14 -0
- package/dist/memory-migration-protocol.js +14 -0
- package/dist/memory-migration-runtime.js +367 -0
- package/dist/memory-prune-diagnostics.js +74 -4
- package/dist/message-file-preview.js +11 -1
- package/dist/prompt.js +9 -17
- package/dist/runner.js +1 -1
- package/dist/runtime-tool-update-bridge.js +126 -0
- package/dist/runtime-tool-update-controller.js +151 -0
- package/dist/runtime-tool-update-protocol.js +24 -0
- package/dist/runtime-tool-update-serve.js +54 -0
- package/dist/runtimes/claude-thread-runner.js +25 -4
- package/dist/serve.js +47 -3
- package/dist/token.js +1 -0
- package/dist/workspace-control-handler.js +16 -3
- package/dist/workspace.js +23 -4
- package/package.json +1 -1
|
@@ -0,0 +1,233 @@
|
|
|
1
|
+
import { Buffer } from "node:buffer";
|
|
2
|
+
import { realpath } from "node:fs/promises";
|
|
3
|
+
import { inventoryLimits, readMemoryInventorySource, scanMemorySources } from "./inventory-source.js";
|
|
4
|
+
import { detectLocalMemoryStorageVersion, LOCAL_MEMORY_GENERATED_INDEX_PATTERN } from "./storage-format.js";
|
|
5
|
+
const CURSOR_VERSION = 1;
|
|
6
|
+
const MAX_METADATA_PATH_BYTES = 4 * 1024;
|
|
7
|
+
const MAX_METADATA_WARNING_BYTES = 2 * 1024;
|
|
8
|
+
const excerpt = (text, size) => {
|
|
9
|
+
const characters = Array.from(text);
|
|
10
|
+
return characters.length > size ? `${characters.slice(0, size).join("")}...` : text;
|
|
11
|
+
};
|
|
12
|
+
function originalLowercaseIndex(text, index) {
|
|
13
|
+
let originalIndex = 0;
|
|
14
|
+
let lowerIndex = 0;
|
|
15
|
+
for (const character of text) {
|
|
16
|
+
lowerIndex += character.toLowerCase().length;
|
|
17
|
+
if (index < lowerIndex)
|
|
18
|
+
return originalIndex;
|
|
19
|
+
originalIndex += character.length;
|
|
20
|
+
}
|
|
21
|
+
return text.length;
|
|
22
|
+
}
|
|
23
|
+
function encodeCursor(cursor) {
|
|
24
|
+
return Buffer.from(JSON.stringify(cursor), "utf8").toString("base64url");
|
|
25
|
+
}
|
|
26
|
+
function decodeCursor(value) {
|
|
27
|
+
let parsed;
|
|
28
|
+
try {
|
|
29
|
+
parsed = JSON.parse(Buffer.from(value, "base64url").toString("utf8"));
|
|
30
|
+
}
|
|
31
|
+
catch {
|
|
32
|
+
throw new Error("memory_cursor_invalid");
|
|
33
|
+
}
|
|
34
|
+
if (!parsed || typeof parsed !== "object")
|
|
35
|
+
throw new Error("memory_cursor_invalid");
|
|
36
|
+
const record = parsed;
|
|
37
|
+
if (record.version !== CURSOR_VERSION || typeof record.file !== "string"
|
|
38
|
+
|| !/^[a-f0-9]{64}$/u.test(String(record.sha256))
|
|
39
|
+
|| typeof record.afterLine !== "number" || !Number.isSafeInteger(record.afterLine) || record.afterLine < 0) {
|
|
40
|
+
throw new Error("memory_cursor_invalid");
|
|
41
|
+
}
|
|
42
|
+
return { version: 1, file: record.file, sha256: String(record.sha256), afterLine: record.afterLine };
|
|
43
|
+
}
|
|
44
|
+
function boundStringList(values, maxBytes) {
|
|
45
|
+
const bounded = [];
|
|
46
|
+
for (const value of values) {
|
|
47
|
+
const candidate = [...bounded, value];
|
|
48
|
+
if (Buffer.byteLength(JSON.stringify(candidate), "utf8") > maxBytes)
|
|
49
|
+
break;
|
|
50
|
+
bounded.push(value);
|
|
51
|
+
}
|
|
52
|
+
return { values: bounded, truncated: bounded.length < values.length };
|
|
53
|
+
}
|
|
54
|
+
function sourceMatches(source, query, afterLine, limit, outputBytes) {
|
|
55
|
+
const terms = query.normalize("NFKC").toLowerCase().split(/\s+/u).filter(Boolean);
|
|
56
|
+
const matches = [];
|
|
57
|
+
let line = 0;
|
|
58
|
+
let offset = 0;
|
|
59
|
+
let heading = "";
|
|
60
|
+
let matchCount = 0;
|
|
61
|
+
let hasMore = false;
|
|
62
|
+
let outputTooLarge = false;
|
|
63
|
+
let firstMatchLine = null;
|
|
64
|
+
let remaining = outputBytes;
|
|
65
|
+
const generatedIndex = source.path === "MEMORY.md"
|
|
66
|
+
? LOCAL_MEMORY_GENERATED_INDEX_PATTERN.exec(source.content)
|
|
67
|
+
: null;
|
|
68
|
+
const generatedStart = generatedIndex === null ? -1 : Array.from(source.content.slice(0, generatedIndex.index)).length;
|
|
69
|
+
const generatedEnd = generatedIndex === null ? -1 : generatedStart + Array.from(generatedIndex[0]).length;
|
|
70
|
+
for (const match of source.content.matchAll(/[^\n]*(?:\n|$)/gu)) {
|
|
71
|
+
if (match[0] === "")
|
|
72
|
+
break;
|
|
73
|
+
line += 1;
|
|
74
|
+
const raw = match[0];
|
|
75
|
+
const lineStart = offset;
|
|
76
|
+
const lineEnd = offset + Array.from(raw).length;
|
|
77
|
+
const content = raw.replace(/\r?\n$/u, "");
|
|
78
|
+
const title = /^#{1,6}\s+(.+?)\s*$/u.exec(content)?.[1];
|
|
79
|
+
if (title !== undefined)
|
|
80
|
+
heading = excerpt(title, 96);
|
|
81
|
+
const lower = content.normalize("NFKC").toLowerCase();
|
|
82
|
+
const insideGeneratedIndex = generatedStart >= 0 && lineStart < generatedEnd && lineEnd > generatedStart;
|
|
83
|
+
if (!insideGeneratedIndex && terms.every(term => lower.includes(term))) {
|
|
84
|
+
matchCount += 1;
|
|
85
|
+
if (line > afterLine) {
|
|
86
|
+
firstMatchLine ??= line;
|
|
87
|
+
const matchIndex = lower.indexOf(terms[0] ?? "");
|
|
88
|
+
let start = Math.max(0, originalLowercaseIndex(content, matchIndex) - 60);
|
|
89
|
+
if (start > 0 && /[\uDC00-\uDFFF]/u.test(content[start]))
|
|
90
|
+
start -= 1;
|
|
91
|
+
const item = { file: source.path, sourceSha256: source.sha256, legacy: true, opaque: true,
|
|
92
|
+
line, offset, heading, text: `${start > 0 ? "..." : ""}${excerpt(content.slice(start), 240)}` };
|
|
93
|
+
const bytes = Buffer.byteLength(JSON.stringify(item), "utf8") + 1;
|
|
94
|
+
if (matches.length >= limit)
|
|
95
|
+
hasMore = true;
|
|
96
|
+
else if (bytes > remaining) {
|
|
97
|
+
hasMore = true;
|
|
98
|
+
outputTooLarge = matches.length === 0;
|
|
99
|
+
}
|
|
100
|
+
else {
|
|
101
|
+
matches.push(item);
|
|
102
|
+
remaining -= bytes;
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
offset += Array.from(raw).length;
|
|
107
|
+
}
|
|
108
|
+
return { matches, matchCount, hasMore, outputTooLarge, firstMatchLine,
|
|
109
|
+
bytesUsed: outputBytes - remaining,
|
|
110
|
+
nextAfterLine: hasMore ? matches.at(-1)?.line ?? null : null };
|
|
111
|
+
}
|
|
112
|
+
function issueCode(error) {
|
|
113
|
+
if (error instanceof TypeError)
|
|
114
|
+
return "invalid_utf8";
|
|
115
|
+
if (error instanceof Error)
|
|
116
|
+
return error.code ?? error.message;
|
|
117
|
+
return "read_failed";
|
|
118
|
+
}
|
|
119
|
+
async function readRequestedSources(agentDir, files, excludeFiles) {
|
|
120
|
+
const root = await realpath(agentDir);
|
|
121
|
+
const sources = [];
|
|
122
|
+
const issues = [];
|
|
123
|
+
let remaining = inventoryLimits.totalBytes;
|
|
124
|
+
for (const file of [...new Set(files)].sort((left, right) => left.localeCompare(right))) {
|
|
125
|
+
if (excludeFiles?.has(file))
|
|
126
|
+
continue;
|
|
127
|
+
try {
|
|
128
|
+
const source = await readMemoryInventorySource(root, file, remaining);
|
|
129
|
+
remaining -= source.bytes;
|
|
130
|
+
sources.push(source);
|
|
131
|
+
}
|
|
132
|
+
catch (error) {
|
|
133
|
+
issues.push({ code: issueCode(error), path: file });
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
return { sources, issues };
|
|
137
|
+
}
|
|
138
|
+
/**
|
|
139
|
+
* Read-only adapter for old MEMORY.md/notes content. It is only called from
|
|
140
|
+
* explicit search commands; prompt preparation never invokes it.
|
|
141
|
+
*/
|
|
142
|
+
export async function searchLegacyMemorySources(input) {
|
|
143
|
+
const scan = input.files === undefined
|
|
144
|
+
? await scanMemorySources(input.agentDir, { includeOperationalFiles: false })
|
|
145
|
+
: null;
|
|
146
|
+
const requestedResult = input.files === undefined
|
|
147
|
+
? null
|
|
148
|
+
: await readRequestedSources(input.agentDir, input.files, input.excludeFiles);
|
|
149
|
+
const requested = input.files === undefined
|
|
150
|
+
? scan.sources.filter(source => source.path === "MEMORY.md" || source.path.startsWith("notes/"))
|
|
151
|
+
: requestedResult.sources;
|
|
152
|
+
const sources = input.files === undefined
|
|
153
|
+
? requested.filter(source => !(input.excludeFiles?.has(source.path) ?? false))
|
|
154
|
+
: requested;
|
|
155
|
+
const detectedIndexVersion = await (async () => {
|
|
156
|
+
try {
|
|
157
|
+
const memory = sources.find(source => source.path === "MEMORY.md")
|
|
158
|
+
?? await readMemoryInventorySource(await realpath(input.agentDir), "MEMORY.md", inventoryLimits.fileBytes);
|
|
159
|
+
return detectLocalMemoryStorageVersion(memory.content);
|
|
160
|
+
}
|
|
161
|
+
catch {
|
|
162
|
+
return null;
|
|
163
|
+
}
|
|
164
|
+
})();
|
|
165
|
+
const relevantIssues = input.files === undefined
|
|
166
|
+
? scan.issues.filter(issue => issue.path === "MEMORY.md" || issue.path.startsWith("notes/"))
|
|
167
|
+
: requestedResult.issues;
|
|
168
|
+
const cursor = input.cursor === undefined ? null : decodeCursor(input.cursor);
|
|
169
|
+
if (cursor !== null && !sources.some(source => source.path === cursor.file))
|
|
170
|
+
throw new Error("memory_cursor_invalid");
|
|
171
|
+
const ordered = [...sources].sort((left, right) => left.path.localeCompare(right.path));
|
|
172
|
+
const matches = [];
|
|
173
|
+
let matchCount = 0;
|
|
174
|
+
let hasMore = false;
|
|
175
|
+
let nextCursor = null;
|
|
176
|
+
let started = cursor === null;
|
|
177
|
+
let pageFull = false;
|
|
178
|
+
const allUnindexedFiles = ordered.map(source => source.path);
|
|
179
|
+
const boundedFiles = boundStringList(allUnindexedFiles, MAX_METADATA_PATH_BYTES);
|
|
180
|
+
const allWarnings = relevantIssues.map(issue => ({ code: issue.code, path: issue.path }));
|
|
181
|
+
const boundedWarnings = boundStringList(allWarnings.map(warning => `${warning.code}\u0000${warning.path}`), MAX_METADATA_WARNING_BYTES);
|
|
182
|
+
const warnings = boundedWarnings.values.map(value => {
|
|
183
|
+
const separator = value.indexOf("\u0000");
|
|
184
|
+
return { code: separator < 0 ? value : value.slice(0, separator), path: separator < 0 ? "" : value.slice(separator + 1) };
|
|
185
|
+
});
|
|
186
|
+
const outputReservation = Buffer.byteLength(JSON.stringify({
|
|
187
|
+
source: "legacy", legacy: true, opaque: true, readerMode: "legacy", storageVersion: detectedIndexVersion,
|
|
188
|
+
detectedIndexVersion,
|
|
189
|
+
...(ordered.length === 1 ? { file: ordered[0].path, sha256: ordered[0].sha256 } : {}),
|
|
190
|
+
matches: [], matchCount: 0, hasMore: false, complete: false, nextAfterLine: null,
|
|
191
|
+
unindexedFiles: boundedFiles.values, unindexedFileCount: allUnindexedFiles.length,
|
|
192
|
+
unindexedFilesTruncated: boundedFiles.truncated, nextCursor: null,
|
|
193
|
+
warnings, warningCount: allWarnings.length, warningsTruncated: boundedWarnings.truncated,
|
|
194
|
+
}), "utf8") + 256;
|
|
195
|
+
let remainingOutputBytes = Math.max(0, input.outputBytes - outputReservation);
|
|
196
|
+
for (const source of ordered) {
|
|
197
|
+
if (!started) {
|
|
198
|
+
if (source.path !== cursor.file)
|
|
199
|
+
continue;
|
|
200
|
+
if (source.sha256 !== cursor.sha256)
|
|
201
|
+
throw new Error("memory_file_changed");
|
|
202
|
+
started = true;
|
|
203
|
+
}
|
|
204
|
+
const afterLine = source.path === cursor?.file ? cursor.afterLine
|
|
205
|
+
: cursor === null && source.path === ordered[0]?.path ? input.afterLine ?? 0 : 0;
|
|
206
|
+
const result = sourceMatches(source, input.query, afterLine, pageFull ? 0 : Math.max(0, input.limit - matches.length), remainingOutputBytes);
|
|
207
|
+
matches.push(...result.matches);
|
|
208
|
+
matchCount += result.matchCount;
|
|
209
|
+
remainingOutputBytes -= result.bytesUsed;
|
|
210
|
+
if (result.hasMore) {
|
|
211
|
+
hasMore = true;
|
|
212
|
+
pageFull = true;
|
|
213
|
+
if (result.outputTooLarge && matches.length === 0)
|
|
214
|
+
throw new Error("memory_output_too_large");
|
|
215
|
+
if (nextCursor === null) {
|
|
216
|
+
const last = result.matches.at(-1);
|
|
217
|
+
nextCursor = encodeCursor({ version: 1, file: source.path, sha256: source.sha256,
|
|
218
|
+
afterLine: Number(last?.line ?? Math.max(0, result.firstMatchLine ?? 1) - 1) });
|
|
219
|
+
}
|
|
220
|
+
}
|
|
221
|
+
}
|
|
222
|
+
const complete = !hasMore && relevantIssues.length === 0;
|
|
223
|
+
return {
|
|
224
|
+
source: "legacy", legacy: true, opaque: true, readerMode: "legacy",
|
|
225
|
+
storageVersion: detectedIndexVersion, detectedIndexVersion,
|
|
226
|
+
...(ordered.length === 1 ? { file: ordered[0].path, sha256: ordered[0].sha256 } : {}),
|
|
227
|
+
matches, matchCount, hasMore, complete, nextAfterLine: hasMore && ordered.length === 1
|
|
228
|
+
? Number(matches.at(-1)?.line ?? 0) : null,
|
|
229
|
+
unindexedFiles: boundedFiles.values, unindexedFileCount: allUnindexedFiles.length,
|
|
230
|
+
unindexedFilesTruncated: boundedFiles.truncated, nextCursor, warnings,
|
|
231
|
+
warningCount: allWarnings.length, warningsTruncated: boundedWarnings.truncated,
|
|
232
|
+
};
|
|
233
|
+
}
|
|
@@ -0,0 +1,242 @@
|
|
|
1
|
+
import { lstat, mkdir, open, readFile, realpath } from "node:fs/promises";
|
|
2
|
+
import { join } from "node:path";
|
|
3
|
+
import { z } from "zod";
|
|
4
|
+
import { createJournalLease, createJournalLeaseRegistry, JournalLockedError } from "../execution-journal-lock.js";
|
|
5
|
+
import { defaultProcessController } from "../execution-journal.js";
|
|
6
|
+
import { createHash } from "node:crypto";
|
|
7
|
+
import { MigrationStateSchema } from "./migration-schema.js";
|
|
8
|
+
import { detectLocalMemoryStorageVersion, parseFullMemoryIndex } from "./storage-format.js";
|
|
9
|
+
import { inventoryLimits, readMemoryInventorySource, scanMemorySources } from "./inventory-source.js";
|
|
10
|
+
import { inspectV2MemoryInventory } from "./inventory-v2.js";
|
|
11
|
+
import { MEMORY_STATE_GUARD_CAPABILITY, MEMORY_STORAGE_V2_CAPABILITY } from "../memory-migration-protocol.js";
|
|
12
|
+
import { isLocalMemoryInitializationIdentity, localMemoryInitializationIdentity } from "./initialization-identity.js";
|
|
13
|
+
import { assertMigrationStateIntegrity } from "./migration-state-invariants.js";
|
|
14
|
+
export const LOCAL_MEMORY_V2_WRITER_CAPABILITIES = [MEMORY_STORAGE_V2_CAPABILITY, MEMORY_STATE_GUARD_CAPABILITY];
|
|
15
|
+
/**
|
|
16
|
+
* Keep an execution that started before migration from writing a committed V2
|
|
17
|
+
* catalog with the legacy result writer. Server-side daemon capability checks
|
|
18
|
+
* stop new work, but an already-running old process can still reach its local
|
|
19
|
+
* result commit after the migration has finished.
|
|
20
|
+
*/
|
|
21
|
+
export async function assertMemoryWriterAllowed(agentDir, capabilities = []) {
|
|
22
|
+
const root = await realpath(agentDir);
|
|
23
|
+
let state;
|
|
24
|
+
try {
|
|
25
|
+
const source = await readMemoryInventorySource(root, ".crew/memory/migration.v1.json", inventoryLimits.fileBytes);
|
|
26
|
+
state = StateGate.parse(JSON.parse(source.content));
|
|
27
|
+
}
|
|
28
|
+
catch (error) {
|
|
29
|
+
if (error.code === "ENOENT") {
|
|
30
|
+
// A missing state file is only compatible with an old, unmarked store.
|
|
31
|
+
// Once MEMORY.md carries a schema2 index, absence of the state must fail closed.
|
|
32
|
+
try {
|
|
33
|
+
const memory = await readMemoryInventorySource(root, "MEMORY.md", inventoryLimits.fileBytes);
|
|
34
|
+
if (detectLocalMemoryStorageVersion(memory.content) === 2) {
|
|
35
|
+
throw new Error("memory_migration_state_missing");
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
catch (memoryError) {
|
|
39
|
+
if (memoryError.code === "ENOENT")
|
|
40
|
+
return;
|
|
41
|
+
throw memoryError;
|
|
42
|
+
}
|
|
43
|
+
return;
|
|
44
|
+
}
|
|
45
|
+
if (error instanceof z.ZodError || error instanceof SyntaxError)
|
|
46
|
+
throw new Error("memory_migration_state_invalid");
|
|
47
|
+
throw error;
|
|
48
|
+
}
|
|
49
|
+
if (state.phase === "applying" || state.phase === "repair_required" || state.phase === "prepared") {
|
|
50
|
+
throw new Error("memory_migration_in_progress");
|
|
51
|
+
}
|
|
52
|
+
if (state.phase !== "committed")
|
|
53
|
+
return;
|
|
54
|
+
// Reuse the committed-state integrity checks before admitting any writer;
|
|
55
|
+
// a truncated committed state must fail closed instead of looking legacy.
|
|
56
|
+
await assertMemoryAccessAllowed(root);
|
|
57
|
+
if (state.storageVersion !== 2)
|
|
58
|
+
return;
|
|
59
|
+
if (LOCAL_MEMORY_V2_WRITER_CAPABILITIES.some((capability) => !capabilities.includes(capability))) {
|
|
60
|
+
throw new Error("memory_v2_writer_capability_missing");
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
/**
|
|
64
|
+
* Check the committed V2 catalog immediately before a write is persisted.
|
|
65
|
+
* Keep this separate from the boot/state gate so legacy writer-specific
|
|
66
|
+
* validation (size limits, entry limits, note limits) retains its established
|
|
67
|
+
* error precedence.
|
|
68
|
+
*/
|
|
69
|
+
export async function assertMemoryInventoryHealth(agentDir) {
|
|
70
|
+
const root = await realpath(agentDir);
|
|
71
|
+
let state;
|
|
72
|
+
try {
|
|
73
|
+
const source = await readMemoryInventorySource(root, ".crew/memory/migration.v1.json", inventoryLimits.fileBytes);
|
|
74
|
+
state = StateGate.parse(JSON.parse(source.content));
|
|
75
|
+
}
|
|
76
|
+
catch (error) {
|
|
77
|
+
if (error.code === "ENOENT")
|
|
78
|
+
return;
|
|
79
|
+
if (error instanceof z.ZodError || error instanceof SyntaxError)
|
|
80
|
+
throw new Error("memory_migration_state_invalid");
|
|
81
|
+
throw error;
|
|
82
|
+
}
|
|
83
|
+
// Legacy/schema1 writers do not need a V2 inventory scan. In particular,
|
|
84
|
+
// a large pending directory must keep the old writer's normal latency and
|
|
85
|
+
// budget behavior.
|
|
86
|
+
if (state.phase !== "committed" || state.storageVersion !== 2)
|
|
87
|
+
return;
|
|
88
|
+
const scan = await scanMemorySources(root, { includeOperationalFiles: false });
|
|
89
|
+
const health = await inspectV2MemoryInventory(root, scan.sources, scan.issues.length === 0);
|
|
90
|
+
const blockingNoteIssues = health.analysis.issues.filter(issue => issue.path.startsWith("notes/"));
|
|
91
|
+
if (scan.issues.length === 0
|
|
92
|
+
&& !health.analysis.issues.some(issue => issue.path === "MEMORY.md")
|
|
93
|
+
&& blockingNoteIssues.length > 0) {
|
|
94
|
+
throw new Error("memory_migration_inventory_invalid");
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
const StateGate = z.object({
|
|
98
|
+
schemaVersion: z.literal(1), migrationId: z.string().min(1),
|
|
99
|
+
phase: z.enum(["prepared", "applying", "committed", "rolled_back", "repair_required"]),
|
|
100
|
+
storageVersion: z.union([z.literal(0), z.literal(1), z.literal(2)]).optional(),
|
|
101
|
+
}).passthrough();
|
|
102
|
+
async function checkedDirectory(parent, name) {
|
|
103
|
+
const directory = join(parent, name);
|
|
104
|
+
await mkdir(directory, { recursive: true, mode: 0o700 });
|
|
105
|
+
const stat = await lstat(directory);
|
|
106
|
+
if (!stat.isDirectory() || stat.isSymbolicLink() || await realpath(directory) !== directory) {
|
|
107
|
+
throw new Error("memory_migration_path_invalid");
|
|
108
|
+
}
|
|
109
|
+
return directory;
|
|
110
|
+
}
|
|
111
|
+
export async function assertMemoryAccessAllowed(agentDir, options = {}) {
|
|
112
|
+
let state = null;
|
|
113
|
+
try {
|
|
114
|
+
const source = await readMemoryInventorySource(agentDir, ".crew/memory/migration.v1.json", inventoryLimits.fileBytes);
|
|
115
|
+
state = StateGate.parse(JSON.parse(source.content));
|
|
116
|
+
if (state.phase === "prepared" || state.phase === "applying" || state.phase === "repair_required") {
|
|
117
|
+
if (options.pendingOnly && (state.phase === "prepared" || state.phase === "applying")) {
|
|
118
|
+
const fullState = MigrationStateSchema.parse(state);
|
|
119
|
+
assertMigrationStateIntegrity(fullState, fullState.identity);
|
|
120
|
+
// The migration inventory owns the pending directory while its journal
|
|
121
|
+
// is active. Allowing an execution to append a candidate here would
|
|
122
|
+
// make the next classify() see an unplanned file and force
|
|
123
|
+
// repair_required. Keep the journal fail-closed and let pending-only
|
|
124
|
+
// apply only to the marker-only legacy state (no active journal).
|
|
125
|
+
throw new Error("memory_migration_in_progress");
|
|
126
|
+
}
|
|
127
|
+
const recovery = options.initializedRecoveryIdentity;
|
|
128
|
+
if (recovery === undefined)
|
|
129
|
+
throw new Error("memory_migration_in_progress");
|
|
130
|
+
const fullState = MigrationStateSchema.parse(state);
|
|
131
|
+
assertMigrationStateIntegrity(fullState, fullState.identity);
|
|
132
|
+
const binding = recovery.workspaceId !== undefined && recovery.machineId !== undefined
|
|
133
|
+
? { workspaceId: recovery.workspaceId, machineId: recovery.machineId }
|
|
134
|
+
: undefined;
|
|
135
|
+
const identity = await localMemoryInitializationIdentity(agentDir, recovery.agentId, binding);
|
|
136
|
+
const legacyIdentity = binding === undefined
|
|
137
|
+
? identity
|
|
138
|
+
: await localMemoryInitializationIdentity(agentDir, recovery.agentId);
|
|
139
|
+
if (fullState.origin !== "initialized"
|
|
140
|
+
|| (fullState.phase === "prepared" && fullState.intentReceipt !== undefined)
|
|
141
|
+
|| (!isLocalMemoryInitializationIdentity(fullState.identity, identity)
|
|
142
|
+
&& !isLocalMemoryInitializationIdentity(fullState.identity, legacyIdentity))) {
|
|
143
|
+
throw new Error("memory_migration_in_progress");
|
|
144
|
+
}
|
|
145
|
+
return;
|
|
146
|
+
}
|
|
147
|
+
}
|
|
148
|
+
catch (error) {
|
|
149
|
+
if (error instanceof Error && error.message === "memory_migration_in_progress")
|
|
150
|
+
throw error;
|
|
151
|
+
if (error.code !== "ENOENT")
|
|
152
|
+
throw new Error("memory_migration_state_invalid");
|
|
153
|
+
}
|
|
154
|
+
let source;
|
|
155
|
+
try {
|
|
156
|
+
source = await readMemoryInventorySource(agentDir, "MEMORY.md", inventoryLimits.fileBytes);
|
|
157
|
+
}
|
|
158
|
+
catch (error) {
|
|
159
|
+
if (error.code === "ENOENT" && state === null)
|
|
160
|
+
return;
|
|
161
|
+
if (error instanceof Error && error.message === "read_limit_exceeded")
|
|
162
|
+
throw new Error("memory_file_too_large");
|
|
163
|
+
throw error;
|
|
164
|
+
}
|
|
165
|
+
const version = detectLocalMemoryStorageVersion(source.content);
|
|
166
|
+
if (version === 2 || state?.phase === "committed") {
|
|
167
|
+
if (!state)
|
|
168
|
+
throw new Error("memory_migration_state_missing");
|
|
169
|
+
const verified = MigrationStateSchema.parse(state);
|
|
170
|
+
assertMigrationStateIntegrity(verified, verified.identity);
|
|
171
|
+
if (verified.phase !== "committed" || verified.storageVersion !== 2 || version !== 2
|
|
172
|
+
|| verified.identity.rootFingerprint !== createHash("sha256").update(await realpath(agentDir)).digest("hex")
|
|
173
|
+
|| verified.identity.ownerIdentity !== String((await lstat(agentDir)).uid)) {
|
|
174
|
+
throw new Error("memory_migration_state_invalid");
|
|
175
|
+
}
|
|
176
|
+
parseFullMemoryIndex(source.content);
|
|
177
|
+
}
|
|
178
|
+
}
|
|
179
|
+
/** 复用现有进程身份锁;仅保护一次本地IO,不持锁等待模型,避免子CLI自锁。 */
|
|
180
|
+
export async function withMemoryAccess(agentDir, operation, options = {}) {
|
|
181
|
+
let created = false;
|
|
182
|
+
if (options.create) {
|
|
183
|
+
created = await lstat(agentDir).then(() => false, (e) => { if (e.code === "ENOENT")
|
|
184
|
+
return true; throw e; });
|
|
185
|
+
await mkdir(agentDir, { recursive: true, mode: 0o700 });
|
|
186
|
+
}
|
|
187
|
+
const root = await realpath(agentDir);
|
|
188
|
+
if ((await lstat(agentDir)).isSymbolicLink())
|
|
189
|
+
throw new Error("memory_migration_path_invalid");
|
|
190
|
+
const crew = await checkedDirectory(root, ".crew");
|
|
191
|
+
const memory = await checkedDirectory(crew, "memory");
|
|
192
|
+
const directory = await checkedDirectory(memory, "access.lock");
|
|
193
|
+
const lease = createJournalLease({ directory, currentPid: process.pid,
|
|
194
|
+
captureCurrentIdentity: async () => {
|
|
195
|
+
const identity = await defaultProcessController.inspectIdentity(process.pid);
|
|
196
|
+
if (!identity)
|
|
197
|
+
throw new Error("memory_migration_process_unknown");
|
|
198
|
+
return identity;
|
|
199
|
+
},
|
|
200
|
+
inspectIdentity: pid => defaultProcessController.inspectIdentity(pid),
|
|
201
|
+
registry: createJournalLeaseRegistry(),
|
|
202
|
+
syncDirectory: async (path) => {
|
|
203
|
+
if (process.platform === "win32")
|
|
204
|
+
return;
|
|
205
|
+
const handle = await open(path, "r");
|
|
206
|
+
try {
|
|
207
|
+
await handle.sync();
|
|
208
|
+
}
|
|
209
|
+
finally {
|
|
210
|
+
await handle.close();
|
|
211
|
+
}
|
|
212
|
+
},
|
|
213
|
+
});
|
|
214
|
+
const deadline = Date.now() + (options.waitMs ?? 2000);
|
|
215
|
+
for (;;) {
|
|
216
|
+
try {
|
|
217
|
+
await lease.acquire();
|
|
218
|
+
break;
|
|
219
|
+
}
|
|
220
|
+
catch (error) {
|
|
221
|
+
if (!(error instanceof JournalLockedError))
|
|
222
|
+
throw error;
|
|
223
|
+
if (Date.now() >= deadline)
|
|
224
|
+
throw new Error("memory_migration_access_busy");
|
|
225
|
+
await new Promise(resolve => setTimeout(resolve, 25));
|
|
226
|
+
}
|
|
227
|
+
}
|
|
228
|
+
try {
|
|
229
|
+
if (!options.maintenance)
|
|
230
|
+
await assertMemoryAccessAllowed(root, {
|
|
231
|
+
...(options.initializedRecoveryIdentity === undefined ? {} : {
|
|
232
|
+
initializedRecoveryIdentity: options.initializedRecoveryIdentity,
|
|
233
|
+
}),
|
|
234
|
+
...(options.pendingOnly ? { pendingOnly: true } : {}),
|
|
235
|
+
});
|
|
236
|
+
lease.assertUsable();
|
|
237
|
+
return await operation({ created });
|
|
238
|
+
}
|
|
239
|
+
finally {
|
|
240
|
+
await lease.close();
|
|
241
|
+
}
|
|
242
|
+
}
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
import { isAbsolute, resolve } from 'node:path';
|
|
2
|
+
import { fileURLToPath } from 'node:url';
|
|
3
|
+
import { z } from 'zod';
|
|
4
|
+
import { applyMemoryMigration, prepareMemoryMigration, recoverMemoryMigration, statusMemoryMigration } from './migration-store.js';
|
|
5
|
+
import { MigrationError, MigrationHistorySchema, MigrationIdentitySchema, MigrationIntentReceiptSchema, MigrationProcessStoppedReceiptSchema, MigrationRecordMappingSchema } from './migration-schema.js';
|
|
6
|
+
const base = z.object({ agentDir: z.string().refine(isAbsolute), identity: MigrationIdentitySchema });
|
|
7
|
+
export const MigrationCommandSchema = z.discriminatedUnion('command', [
|
|
8
|
+
base.extend({ command: z.literal('prepare'), serverHistory: MigrationHistorySchema, recordMappings: z.array(MigrationRecordMappingSchema).max(10000).optional() }).strict(),
|
|
9
|
+
base.extend({ command: z.literal('status') }).strict(),
|
|
10
|
+
base.extend({ command: z.literal('apply'), intentReceipt: MigrationIntentReceiptSchema }).strict(),
|
|
11
|
+
base.extend({ command: z.literal('recover'), serverHistory: MigrationHistorySchema.optional(), intentReceipt: MigrationIntentReceiptSchema.optional(), stoppedReceipt: MigrationProcessStoppedReceiptSchema.optional(), cancelled: z.boolean() }).strict(),
|
|
12
|
+
]);
|
|
13
|
+
export function summarizeMemoryMigration(state) {
|
|
14
|
+
return { migrationId: state?.migrationId ?? null, phase: state?.phase ?? null, storageVersion: state?.storageVersion ?? null, sourceDigest: state?.sourceDigest ?? null, planDigest: state?.planDigest ?? null,
|
|
15
|
+
issues: state?.issues ?? [], result: state === null ? 'missing' : state.phase === 'repair_required' ? 'repair_required' : 'success' };
|
|
16
|
+
}
|
|
17
|
+
/** The caller owns the execution/maintenance lease. This boundary never accepts or reads credentials. */
|
|
18
|
+
export async function runMemoryMigrationCommand(raw) {
|
|
19
|
+
let migrationId = null;
|
|
20
|
+
try {
|
|
21
|
+
const input = MigrationCommandSchema.parse(raw);
|
|
22
|
+
migrationId = input.identity.migrationId;
|
|
23
|
+
const state = input.command === 'prepare' ? await prepareMemoryMigration(input) : input.command === 'apply' ? await applyMemoryMigration(input)
|
|
24
|
+
: input.command === 'recover' ? await recoverMemoryMigration(input) : await statusMemoryMigration(input);
|
|
25
|
+
return summarizeMemoryMigration(state);
|
|
26
|
+
}
|
|
27
|
+
catch (error) {
|
|
28
|
+
const reason = error instanceof MigrationError ? error.code : error instanceof z.ZodError ? 'migration_input_invalid' : 'migration_operation_failed';
|
|
29
|
+
return { ...summarizeMemoryMigration(null), migrationId, result: reason === 'migration_needs_review' ? 'needs_review' : reason.includes('repair_required') ? 'repair_required' : 'failed',
|
|
30
|
+
issues: error instanceof MigrationError && error.issues.length ? error.issues : [{ path: '', reason }] };
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
async function main() {
|
|
34
|
+
try {
|
|
35
|
+
const chunks = [];
|
|
36
|
+
let bytes = 0;
|
|
37
|
+
for await (const chunk of process.stdin) {
|
|
38
|
+
const buffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk);
|
|
39
|
+
bytes += buffer.length;
|
|
40
|
+
if (bytes > 8 * 1024 * 1024)
|
|
41
|
+
throw new Error('input_limit');
|
|
42
|
+
chunks.push(buffer);
|
|
43
|
+
}
|
|
44
|
+
const result = await runMemoryMigrationCommand(JSON.parse(Buffer.concat(chunks).toString('utf8')));
|
|
45
|
+
process.stdout.write(`${JSON.stringify(result)}\n`);
|
|
46
|
+
process.exitCode = result.result === 'success' || result.result === 'missing' ? 0 : 1;
|
|
47
|
+
}
|
|
48
|
+
catch {
|
|
49
|
+
process.stdout.write(`${JSON.stringify({ ...summarizeMemoryMigration(null), result: 'failed', issues: [{ path: '', reason: 'migration_input_invalid' }] })}\n`);
|
|
50
|
+
process.exitCode = 1;
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url))
|
|
54
|
+
void main();
|