@nowcrew/daemon 0.6.97-beta.1 → 0.6.97-beta.2

package/dist/local-memory/migration-access.js

@@ -0,0 +1,242 @@
+import { lstat, mkdir, open, readFile, realpath } from "node:fs/promises";
+import { join } from "node:path";
+import { z } from "zod";
+import { createJournalLease, createJournalLeaseRegistry, JournalLockedError } from "../execution-journal-lock.js";
+import { defaultProcessController } from "../execution-journal.js";
+import { createHash } from "node:crypto";
+import { MigrationStateSchema } from "./migration-schema.js";
+import { detectLocalMemoryStorageVersion, parseFullMemoryIndex } from "./storage-format.js";
+import { inventoryLimits, readMemoryInventorySource, scanMemorySources } from "./inventory-source.js";
+import { inspectV2MemoryInventory } from "./inventory-v2.js";
+import { MEMORY_STATE_GUARD_CAPABILITY, MEMORY_STORAGE_V2_CAPABILITY } from "../memory-migration-protocol.js";
+import { isLocalMemoryInitializationIdentity, localMemoryInitializationIdentity } from "./initialization-identity.js";
+import { assertMigrationStateIntegrity } from "./migration-state-invariants.js";
+export const LOCAL_MEMORY_V2_WRITER_CAPABILITIES = [MEMORY_STORAGE_V2_CAPABILITY, MEMORY_STATE_GUARD_CAPABILITY];
+/**
+ * Keep an execution that started before migration from writing a committed V2
+ * catalog with the legacy result writer.  Server-side daemon capability checks
+ * stop new work, but an already-running old process can still reach its local
+ * result commit after the migration has finished.
+ */
+export async function assertMemoryWriterAllowed(agentDir, capabilities = []) {
+    const root = await realpath(agentDir);
+    let state;
+    try {
+        const source = await readMemoryInventorySource(root, ".crew/memory/migration.v1.json", inventoryLimits.fileBytes);
+        state = StateGate.parse(JSON.parse(source.content));
+    }
+    catch (error) {
+        if (error.code === "ENOENT") {
+            // A missing state file is only compatible with an old, unmarked store.
+            // Once MEMORY.md carries a schema2 index, absence of the state must fail closed.
+            try {
+                const memory = await readMemoryInventorySource(root, "MEMORY.md", inventoryLimits.fileBytes);
+                if (detectLocalMemoryStorageVersion(memory.content) === 2) {
+                    throw new Error("memory_migration_state_missing");
+                }
+            }
+            catch (memoryError) {
+                if (memoryError.code === "ENOENT")
+                    return;
+                throw memoryError;
+            }
+            return;
+        }
+        if (error instanceof z.ZodError || error instanceof SyntaxError)
+            throw new Error("memory_migration_state_invalid");
+        throw error;
+    }
+    if (state.phase === "applying" || state.phase === "repair_required" || state.phase === "prepared") {
+        throw new Error("memory_migration_in_progress");
+    }
+    if (state.phase !== "committed")
+        return;
+    // Reuse the committed-state integrity checks before admitting any writer;
+    // a truncated committed state must fail closed instead of looking legacy.
+    await assertMemoryAccessAllowed(root);
+    if (state.storageVersion !== 2)
+        return;
+    if (LOCAL_MEMORY_V2_WRITER_CAPABILITIES.some((capability) => !capabilities.includes(capability))) {
+        throw new Error("memory_v2_writer_capability_missing");
+    }
+}
+/**
+ * Check the committed V2 catalog immediately before a write is persisted.
+ * Keep this separate from the boot/state gate so legacy writer-specific
+ * validation (size limits, entry limits, note limits) retains its established
+ * error precedence.
+ */
+export async function assertMemoryInventoryHealth(agentDir) {
+    const root = await realpath(agentDir);
+    let state;
+    try {
+        const source = await readMemoryInventorySource(root, ".crew/memory/migration.v1.json", inventoryLimits.fileBytes);
+        state = StateGate.parse(JSON.parse(source.content));
+    }
+    catch (error) {
+        if (error.code === "ENOENT")
+            return;
+        if (error instanceof z.ZodError || error instanceof SyntaxError)
+            throw new Error("memory_migration_state_invalid");
+        throw error;
+    }
+    // Legacy/schema1 writers do not need a V2 inventory scan.  In particular,
+    // a large pending directory must keep the old writer's normal latency and
+    // budget behavior.
+    if (state.phase !== "committed" || state.storageVersion !== 2)
+        return;
+    const scan = await scanMemorySources(root, { includeOperationalFiles: false });
+    const health = await inspectV2MemoryInventory(root, scan.sources, scan.issues.length === 0);
+    const blockingNoteIssues = health.analysis.issues.filter(issue => issue.path.startsWith("notes/"));
+    if (scan.issues.length === 0
+        && !health.analysis.issues.some(issue => issue.path === "MEMORY.md")
+        && blockingNoteIssues.length > 0) {
+        throw new Error("memory_migration_inventory_invalid");
+    }
+}
+const StateGate = z.object({
+    schemaVersion: z.literal(1), migrationId: z.string().min(1),
+    phase: z.enum(["prepared", "applying", "committed", "rolled_back", "repair_required"]),
+    storageVersion: z.union([z.literal(0), z.literal(1), z.literal(2)]).optional(),
+}).passthrough();
+async function checkedDirectory(parent, name) {
+    const directory = join(parent, name);
+    await mkdir(directory, { recursive: true, mode: 0o700 });
+    const stat = await lstat(directory);
+    if (!stat.isDirectory() || stat.isSymbolicLink() || await realpath(directory) !== directory) {
+        throw new Error("memory_migration_path_invalid");
+    }
+    return directory;
+}
+export async function assertMemoryAccessAllowed(agentDir, options = {}) {
+    let state = null;
+    try {
+        const source = await readMemoryInventorySource(agentDir, ".crew/memory/migration.v1.json", inventoryLimits.fileBytes);
+        state = StateGate.parse(JSON.parse(source.content));
+        if (state.phase === "prepared" || state.phase === "applying" || state.phase === "repair_required") {
+            if (options.pendingOnly && (state.phase === "prepared" || state.phase === "applying")) {
+                const fullState = MigrationStateSchema.parse(state);
+                assertMigrationStateIntegrity(fullState, fullState.identity);
+                // The migration inventory owns the pending directory while its journal
+                // is active.  Allowing an execution to append a candidate here would
+                // make the next classify() see an unplanned file and force
+                // repair_required.  Keep the journal fail-closed and let pending-only
+                // apply only to the marker-only legacy state (no active journal).
+                throw new Error("memory_migration_in_progress");
+            }
+            const recovery = options.initializedRecoveryIdentity;
+            if (recovery === undefined)
+                throw new Error("memory_migration_in_progress");
+            const fullState = MigrationStateSchema.parse(state);
+            assertMigrationStateIntegrity(fullState, fullState.identity);
+            const binding = recovery.workspaceId !== undefined && recovery.machineId !== undefined
+                ? { workspaceId: recovery.workspaceId, machineId: recovery.machineId }
+                : undefined;
+            const identity = await localMemoryInitializationIdentity(agentDir, recovery.agentId, binding);
+            const legacyIdentity = binding === undefined
+                ? identity
+                : await localMemoryInitializationIdentity(agentDir, recovery.agentId);
+            if (fullState.origin !== "initialized"
+                || (fullState.phase === "prepared" && fullState.intentReceipt !== undefined)
+                || (!isLocalMemoryInitializationIdentity(fullState.identity, identity)
+                    && !isLocalMemoryInitializationIdentity(fullState.identity, legacyIdentity))) {
+                throw new Error("memory_migration_in_progress");
+            }
+            return;
+        }
+    }
+    catch (error) {
+        if (error instanceof Error && error.message === "memory_migration_in_progress")
+            throw error;
+        if (error.code !== "ENOENT")
+            throw new Error("memory_migration_state_invalid");
+    }
+    let source;
+    try {
+        source = await readMemoryInventorySource(agentDir, "MEMORY.md", inventoryLimits.fileBytes);
+    }
+    catch (error) {
+        if (error.code === "ENOENT" && state === null)
+            return;
+        if (error instanceof Error && error.message === "read_limit_exceeded")
+            throw new Error("memory_file_too_large");
+        throw error;
+    }
+    const version = detectLocalMemoryStorageVersion(source.content);
+    if (version === 2 || state?.phase === "committed") {
+        if (!state)
+            throw new Error("memory_migration_state_missing");
+        const verified = MigrationStateSchema.parse(state);
+        assertMigrationStateIntegrity(verified, verified.identity);
+        if (verified.phase !== "committed" || verified.storageVersion !== 2 || version !== 2
+            || verified.identity.rootFingerprint !== createHash("sha256").update(await realpath(agentDir)).digest("hex")
+            || verified.identity.ownerIdentity !== String((await lstat(agentDir)).uid)) {
+            throw new Error("memory_migration_state_invalid");
+        }
+        parseFullMemoryIndex(source.content);
+    }
+}
+/** 复用现有进程身份锁；仅保护一次本地IO，不持锁等待模型，避免子CLI自锁。 */
+export async function withMemoryAccess(agentDir, operation, options = {}) {
+    let created = false;
+    if (options.create) {
+        created = await lstat(agentDir).then(() => false, (e) => { if (e.code === "ENOENT")
+            return true; throw e; });
+        await mkdir(agentDir, { recursive: true, mode: 0o700 });
+    }
+    const root = await realpath(agentDir);
+    if ((await lstat(agentDir)).isSymbolicLink())
+        throw new Error("memory_migration_path_invalid");
+    const crew = await checkedDirectory(root, ".crew");
+    const memory = await checkedDirectory(crew, "memory");
+    const directory = await checkedDirectory(memory, "access.lock");
+    const lease = createJournalLease({ directory, currentPid: process.pid,
+        captureCurrentIdentity: async () => {
+            const identity = await defaultProcessController.inspectIdentity(process.pid);
+            if (!identity)
+                throw new Error("memory_migration_process_unknown");
+            return identity;
+        },
+        inspectIdentity: pid => defaultProcessController.inspectIdentity(pid),
+        registry: createJournalLeaseRegistry(),
+        syncDirectory: async (path) => {
+            if (process.platform === "win32")
+                return;
+            const handle = await open(path, "r");
+            try {
+                await handle.sync();
+            }
+            finally {
+                await handle.close();
+            }
+        },
+    });
+    const deadline = Date.now() + (options.waitMs ?? 2000);
+    for (;;) {
+        try {
+            await lease.acquire();
+            break;
+        }
+        catch (error) {
+            if (!(error instanceof JournalLockedError))
+                throw error;
+            if (Date.now() >= deadline)
+                throw new Error("memory_migration_access_busy");
+            await new Promise(resolve => setTimeout(resolve, 25));
+        }
+    }
+    try {
+        if (!options.maintenance)
+            await assertMemoryAccessAllowed(root, {
+                ...(options.initializedRecoveryIdentity === undefined ? {} : {
+                    initializedRecoveryIdentity: options.initializedRecoveryIdentity,
+                }),
+                ...(options.pendingOnly ? { pendingOnly: true } : {}),
+            });
+        lease.assertUsable();
+        return await operation({ created });
+    }
+    finally {
+        await lease.close();
+    }
+}

package/dist/local-memory/migration-cli.js

@@ -0,0 +1,54 @@
+import { isAbsolute, resolve } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { z } from 'zod';
+import { applyMemoryMigration, prepareMemoryMigration, recoverMemoryMigration, statusMemoryMigration } from './migration-store.js';
+import { MigrationError, MigrationHistorySchema, MigrationIdentitySchema, MigrationIntentReceiptSchema, MigrationProcessStoppedReceiptSchema, MigrationRecordMappingSchema } from './migration-schema.js';
+const base = z.object({ agentDir: z.string().refine(isAbsolute), identity: MigrationIdentitySchema });
+export const MigrationCommandSchema = z.discriminatedUnion('command', [
+    base.extend({ command: z.literal('prepare'), serverHistory: MigrationHistorySchema, recordMappings: z.array(MigrationRecordMappingSchema).max(10000).optional() }).strict(),
+    base.extend({ command: z.literal('status') }).strict(),
+    base.extend({ command: z.literal('apply'), intentReceipt: MigrationIntentReceiptSchema }).strict(),
+    base.extend({ command: z.literal('recover'), serverHistory: MigrationHistorySchema.optional(), intentReceipt: MigrationIntentReceiptSchema.optional(), stoppedReceipt: MigrationProcessStoppedReceiptSchema.optional(), cancelled: z.boolean() }).strict(),
+]);
+export function summarizeMemoryMigration(state) {
+    return { migrationId: state?.migrationId ?? null, phase: state?.phase ?? null, storageVersion: state?.storageVersion ?? null, sourceDigest: state?.sourceDigest ?? null, planDigest: state?.planDigest ?? null,
+        issues: state?.issues ?? [], result: state === null ? 'missing' : state.phase === 'repair_required' ? 'repair_required' : 'success' };
+}
+/** The caller owns the execution/maintenance lease. This boundary never accepts or reads credentials. */
+export async function runMemoryMigrationCommand(raw) {
+    let migrationId = null;
+    try {
+        const input = MigrationCommandSchema.parse(raw);
+        migrationId = input.identity.migrationId;
+        const state = input.command === 'prepare' ? await prepareMemoryMigration(input) : input.command === 'apply' ? await applyMemoryMigration(input)
+            : input.command === 'recover' ? await recoverMemoryMigration(input) : await statusMemoryMigration(input);
+        return summarizeMemoryMigration(state);
+    }
+    catch (error) {
+        const reason = error instanceof MigrationError ? error.code : error instanceof z.ZodError ? 'migration_input_invalid' : 'migration_operation_failed';
+        return { ...summarizeMemoryMigration(null), migrationId, result: reason === 'migration_needs_review' ? 'needs_review' : reason.includes('repair_required') ? 'repair_required' : 'failed',
+            issues: error instanceof MigrationError && error.issues.length ? error.issues : [{ path: '', reason }] };
+    }
+}
+async function main() {
+    try {
+        const chunks = [];
+        let bytes = 0;
+        for await (const chunk of process.stdin) {
+            const buffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk);
+            bytes += buffer.length;
+            if (bytes > 8 * 1024 * 1024)
+                throw new Error('input_limit');
+            chunks.push(buffer);
+        }
+        const result = await runMemoryMigrationCommand(JSON.parse(Buffer.concat(chunks).toString('utf8')));
+        process.stdout.write(`${JSON.stringify(result)}\n`);
+        process.exitCode = result.result === 'success' || result.result === 'missing' ? 0 : 1;
+    }
+    catch {
+        process.stdout.write(`${JSON.stringify({ ...summarizeMemoryMigration(null), result: 'failed', issues: [{ path: '', reason: 'migration_input_invalid' }] })}\n`);
+        process.exitCode = 1;
+    }
+}
+if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url))
+    void main();

package/dist/local-memory/migration-io.js

@@ -0,0 +1,199 @@
+import { createHash, randomUUID } from 'node:crypto';
+import { constants } from 'node:fs';
+import { lstat, mkdir, open, opendir, realpath, rename, rm } from 'node:fs/promises';
+import { dirname, isAbsolute, join, relative, resolve, sep } from 'node:path';
+import { MigrationError, MigrationIdentitySchema } from './migration-schema.js';
+import { LOCAL_MEMORY_NOTE_BYTE_CAP, LOCAL_MEMORY_READ_BYTE_CAP, LOCAL_MEMORY_STORAGE_BYTE_CAP } from './storage-format.js';
+export const digest = (data) => createHash('sha256').update(data).digest('hex');
+export const STATE_PATH = '.crew/memory/migration.v1.json';
+export async function validateMigrationRoot(agentDir, raw) {
+    const identity = MigrationIdentitySchema.parse(raw);
+    if (!isAbsolute(agentDir) || resolve(agentDir) !== agentDir)
+        throw new MigrationError('migration_root_invalid');
+    // macOS /var -> /private/var is a trusted system alias; the agent root itself may not be a symlink.
+    const stat = await lstat(agentDir);
+    if (!stat.isDirectory() || stat.isSymbolicLink() || String(stat.uid) !== identity.ownerIdentity)
+        throw new MigrationError('migration_owner_invalid');
+    if (digest(await realpath(agentDir)) !== identity.rootFingerprint)
+        throw new MigrationError('migration_root_fingerprint_invalid');
+    return identity;
+}
+export async function safePath(root, path, createParents = false) {
+    const target = resolve(root, path);
+    const rel = relative(root, target);
+    if (!rel || rel.startsWith(`..${sep}`) || rel === '..' || isAbsolute(rel))
+        throw new MigrationError('migration_path_invalid', [{ path, reason: 'outside_root' }]);
+    const parts = rel.split(sep);
+    let current = root;
+    for (let i = 0; i < parts.length; i++) {
+        current = join(current, parts[i]);
+        let stat = await lstat(current).catch((error) => {
+            if (error.code === 'ENOENT')
+                return null;
+            throw error;
+        });
+        if (!stat && createParents && i < parts.length - 1) {
+            await mkdir(current, { mode: 0o700 });
+            stat = await lstat(current);
+        }
+        if (stat?.isSymbolicLink() || (stat && i < parts.length - 1 && !stat.isDirectory()))
+            throw new MigrationError('migration_path_unsafe', [{ path, reason: 'symlink_or_non_directory' }]);
+    }
+    return target;
+}
+export async function readBytes(root, path, cap = LOCAL_MEMORY_NOTE_BYTE_CAP) {
+    const target = await safePath(root, path);
+    const file = await open(target, constants.O_RDONLY | constants.O_NOFOLLOW).catch((error) => {
+        if (error.code === 'ENOENT')
+            return null;
+        throw error;
+    });
+    if (!file)
+        return null;
+    try {
+        const stat = await file.stat();
+        if (!stat.isFile() || stat.size > cap)
+            throw new MigrationError('migration_read_limit', [{ path, reason: 'invalid_file_or_size' }]);
+        const data = Buffer.alloc(stat.size + 1);
+        const { bytesRead } = await file.read(data, 0, data.length, 0);
+        if (bytesRead !== stat.size || (await file.stat()).size !== stat.size)
+            throw new MigrationError('migration_source_changed', [{ path, reason: 'changed_during_read' }]);
+        return data.subarray(0, bytesRead);
+    }
+    finally {
+        await file.close();
+    }
+}
+export async function atomicBytes(root, path, data, options = {}) {
+    const target = await safePath(root, path, true);
+    const temporaryPath = options.temporaryPath ?? `${path}.tmp-${randomUUID()}`;
+    const temp = await safePath(root, temporaryPath, true);
+    if ((await lstat(dirname(target))).dev !== (await lstat(dirname(temp))).dev) {
+        throw new MigrationError('migration_publication_filesystem_mismatch');
+    }
+    const existing = options.temporaryPath ? await readBytes(root, temporaryPath) : null;
+    if (existing && digest(existing) !== digest(data)) {
+        throw new MigrationError('migration_publication_copy_unknown', [{ path: temporaryPath, reason: 'publication_hash_mismatch' }]);
+    }
+    if (!existing) {
+        // The plan-recognized name is only published after every byte is complete and synced.
+        // An interrupted unpublished file is never a recovery source and is preserved as evidence.
+        const writePath = options.temporaryPath
+            ? `${dirname(dirname(temporaryPath))}/unpublished/${randomUUID()}.tmp`
+            : temporaryPath;
+        const unpublished = await safePath(root, writePath, true);
+        if ((await lstat(dirname(unpublished))).dev !== (await lstat(dirname(temp))).dev) {
+            throw new MigrationError('migration_publication_filesystem_mismatch');
+        }
+        const file = await open(unpublished, 'wx', 0o600);
+        try {
+            if (options.temporaryPath) {
+                for (let offset = 0; offset < data.length; offset += 64 * 1024) {
+                    const end = Math.min(offset + 64 * 1024, data.length);
+                    await file.writeFile(data.subarray(offset, end));
+                    await options.afterWriteChunk?.(writePath, end, data.length);
+                }
+            }
+            else
+                await file.writeFile(data);
+            await file.sync();
+        }
+        finally {
+            await file.close();
+        }
+        if (options.temporaryPath) {
+            const complete = await readBytes(root, writePath);
+            if (!complete || digest(complete) !== digest(data))
+                throw new MigrationError('migration_unpublished_copy_invalid');
+            const appeared = await readBytes(root, temporaryPath);
+            if (appeared && digest(appeared) !== digest(data))
+                throw new MigrationError('migration_publication_copy_unknown');
+            if (!appeared) {
+                await safePath(root, temporaryPath);
+                await rename(unpublished, temp);
+                await syncDirectory(dirname(temp));
+                await syncDirectory(dirname(unpublished));
+            }
+        }
+    }
+    try {
+        await options.beforeRename?.();
+        await safePath(root, path);
+        if (options.temporaryPath) {
+            const verified = await readBytes(root, temporaryPath);
+            if (!verified || digest(verified) !== digest(data))
+                throw new MigrationError('migration_publication_copy_unknown');
+        }
+        await rename(temp, target);
+        await syncDirectory(dirname(target));
+    }
+    finally {
+        // Controlled copies survive interruption; recovery verifies their plan-bound name and content before reuse.
+        if (!options.temporaryPath)
+            await rm(temp, { force: true });
+    }
+}
+export async function syncDirectory(path) {
+    const fd = await open(path, 'r');
+    try {
+        await fd.sync();
+    }
+    finally {
+        await fd.close();
+    }
+}
+export async function inventory(root) {
+    const files = new Map();
+    const directories = [];
+    let bytes = 0;
+    let fileCount = 0;
+    let directoryCount = 0;
+    function reserveFile() {
+        if (++fileCount > 20000)
+            throw new MigrationError('migration_file_limit');
+    }
+    function reserveDirectory() {
+        if (++directoryCount > 10000)
+            throw new MigrationError('migration_directory_limit');
+    }
+    async function scan(path, reserved = false) {
+        const target = await safePath(root, path);
+        const stat = await lstat(target).catch((error) => {
+            if (error.code === 'ENOENT')
+                return null;
+            throw error;
+        });
+        if (!stat)
+            return;
+        if (stat.isDirectory()) {
+            if (!reserved)
+                reserveDirectory();
+            directories.push(path);
+            const children = [];
+            for await (const entry of await opendir(target)) {
+                if (entry.isDirectory())
+                    reserveDirectory();
+                else
+                    reserveFile();
+                children.push(entry.name);
+            }
+            for (const name of children.sort())
+                await scan(`${path}/${name}`, true);
+        }
+        else {
+            if (!reserved)
+                reserveFile();
+            const content = await readBytes(root, path, path === 'MEMORY.md' ? LOCAL_MEMORY_STORAGE_BYTE_CAP : LOCAL_MEMORY_NOTE_BYTE_CAP);
+            if (content) {
+                bytes += content.length;
+                if (bytes > LOCAL_MEMORY_READ_BYTE_CAP)
+                    throw new MigrationError('migration_read_limit');
+                files.set(path, content);
+            }
+        }
+    }
+    await scan('MEMORY.md');
+    await scan('notes');
+    await scan('.crew/memory/pending');
+    return { files, directories: directories.sort() };
+}

package/dist/local-memory/migration-pending.js

@@ -0,0 +1,9 @@
+import { detectLocalMemoryStorageVersion } from "./storage-format.js";
+export function shouldUseMigrationDeferredPending(input) {
+    return input.pruneTraceId !== null
+        && input.writerMode === "blocked"
+        && input.rolloutEnabled
+        && input.legacyFallback !== true
+        && input.failure !== "inspection_failed"
+        && detectLocalMemoryStorageVersion(input.memory) === 1;
+}