@nowcrew/daemon 0.6.100 → 0.6.101-beta.2

package/dist/local-memory/inventory-cli.js

@@ -2,9 +2,9 @@ const CommandSchema = z.object({
     "agent-dir": z.string().min(1).refine(isAbsolute),
     "dry-run": z.literal(true),
 }).strict();
-const USAGE = "只读记忆盘点与转换预演，不写文件，不支持 --apply。\n"
+const USAGE = "只读记忆盘点、schema0/1 转换预演与 schema2 完整性校验，不写文件，不支持 --apply。\n"
     + "pnpm --filter @nowcrew/daemon exec tsx src/local-memory/inventory-cli.ts --agent-dir <单个Agent绝对路径> --dry-run\n"
-    + "报告包含原始记忆，需作为敏感本地资料保存；退出码0仅代表盘点及格式预演无阻塞，不代表批准迁移。\n";
+    + "报告包含原始记忆，需作为敏感本地资料保存；退出码0仅代表盘点、格式预演或完整性校验无阻塞，不代表批准迁移。\n";
 export async function runMemoryInventoryCommand(args, io) {
     if (args.length === 1 && args[0] === "--help") {
         io.out(USAGE);
@@ -24,19 +24,60 @@ export async function runMemoryInventoryCommand(args, io) {
     try {
         const scan = await scanMemorySources(command["agent-dir"]);
         const scanComplete = scan.issues.length === 0;
-        const analysis = previewMemoryInventory(scan.sources, scanComplete);
+        const memory = scan.sources.find((source) => source.path === "MEMORY.md")?.content ?? "";
+        let storageVersion = null;
+        let detectionIssue = null;
+        try {
+            storageVersion = detectLocalMemoryStorageVersion(memory);
+        }
+        catch (error) {
+            detectionIssue = {
+                code: error instanceof Error && /^[a-z0-9_]+$/u.test(error.message)
+                    ? error.message
+                    : "local_memory_storage_version_invalid",
+                path: "MEMORY.md",
+            };
+        }
+        const v2 = storageVersion === 2
+            ? await inspectV2MemoryInventory(scan.root, scan.sources, scanComplete)
+            : null;
+        const analysis = detectionIssue
+            ? { entries: [], issues: [detectionIssue], preview: null, rawMemory: memory }
+            : v2?.analysis ?? previewMemoryInventory(scan.sources, scanComplete);
         const complete = scanComplete && !analysis.issues.some((issue) => issue.code === "missing_detail");
         const issues = [...scan.issues, ...analysis.issues];
-        const referenced = new Set(analysis.entries.map((entry) => entry.sourceFields.file));
-        const files = scan.sources.map(({ path, bytes, sha256 }) => ({ path, bytes, sha256 }));
+        const referenced = new Set([
+            ...analysis.entries.map((entry) => entry.sourceFields.file),
+            ...(v2?.preservedSourcePaths ?? []),
+        ]);
+        const reportSources = v2?.stateSource ? [...scan.sources, v2.stateSource] : scan.sources;
+        const files = reportSources.map(({ path, bytes, sha256 }) => ({ path, bytes, sha256 }));
+        const deferredFiles = files.filter((file) => isDeferredMemoryArtifactPath(file.path)).map((file) => file.path);
+        const orphanNotes = files.filter((file) => file.path.startsWith("notes/") && !isDeferredMemoryArtifactPath(file.path) && !referenced.has(file.path)).map((file) => file.path);
+        const indexedEntries = analysis.entries.filter((entry) => entry.format === "managed").length;
+        const legacyEntries = analysis.entries.length - indexedEntries;
         const report = {
             schemaVersion: 1, mode: "dry-run", agentDir: scan.root, applySupported: false,
             complete, scanComplete, requiresHumanReview: true,
-            coverage: ["MEMORY.md", "notes/**", ".crew/memory/pending/**", "根目录的 .memory-result.v1-*.json"],
+            // 只读派生观测，不改变资格判断，也不新增持久化状态。
+            // A dry-run does not create a shadow index or server state. Existing
+            // schema1 cards therefore remain legacy; indexedEntries is only coverage
+            // information, not proof that the Agent entered shadow mode.
+            storageMode: storageVersion === 2 ? (v2?.writeReady ? "v2_write" : "repair_required")
+                : detectionIssue ? "repair_required" : "legacy",
+            contextMode: storageVersion === 2 ? (v2?.writeReady ? "bounded_preview" : "blocked")
+                : detectionIssue ? "blocked" : "legacy_context",
+            readerMode: storageVersion === 2 ? "v2" : detectionIssue ? "blocked" : "legacy",
+            detectedIndexVersion: storageVersion,
+            migrationCoverage: { detectedIndexVersion: storageVersion, storageVersion, indexedEntries, legacyEntries, unindexedFiles: orphanNotes.length,
+                totalEntries: analysis.entries.length, complete },
+            coverage: ["MEMORY.md", "notes/**", ...(storageVersion === 2 ? [".crew/memory/migration.v1.json"] : []),
+                ".crew/memory/pending/**", "根目录的 .memory-result.v1-*.json"],
             limitations: ["不扫描历史执行目录；result清单仅覆盖Agent根目录。", "两遍摘要比对不等同跨文件原子快照，正式迁移仍需停止写入并重新核对。"],
-            limits: inventoryLimits, totalBytes: scan.totalBytes, files, ...analysis, issues,
+            limits: inventoryLimits, totalBytes: reportSources.reduce((total, source) => total + source.bytes, 0), files, ...analysis, issues,
             pending: files.filter((file) => file.path.startsWith(".crew/memory/pending/")).map((file) => file.path),
-            orphanNotes: files.filter((file) => file.path.startsWith("notes/") && !referenced.has(file.path)).map((file) => file.path),
+            deferredFiles,
+            orphanNotes,
             resultFiles: files.filter((file) => file.path.startsWith(".memory-result.v1-")).map((file) => file.path),
         };
         io.out(JSON.stringify(report, null, 2) + "\n");
@@ -59,3 +100,6 @@ import { parseArgs } from "node:util";
 import { z } from "zod";
 import { inventoryLimits, scanMemorySources } from "./inventory-source.js";
 import { previewMemoryInventory } from "./inventory-preview.js";
+import { inspectV2MemoryInventory } from "./inventory-v2.js";
+import { detectLocalMemoryStorageVersion } from "./storage-format.js";
+import { isDeferredMemoryArtifactPath } from "./source-path.js";

package/dist/local-memory/inventory-source.js

@@ -73,7 +73,8 @@ export async function readMemoryInventorySource(root, file, budget) {
         await handle.close();
     }
 }
-export async function scanMemorySources(agentDir) {
+export async function scanMemorySources(agentDir, options = {}) {
+    const includeOperationalFiles = options.includeOperationalFiles ?? true;
     const root = await realpath(agentDir);
     if ((await lstat(agentDir)).isSymbolicLink())
         throw new Error("agent_root_symlink");
@@ -124,10 +125,14 @@ export async function scanMemorySources(agentDir) {
     }
     await visit("MEMORY.md");
     await visit("notes", true);
-    await visit(".crew/memory/pending", true);
-    const resultNames = (await readdir(root)).filter((name) => /^\.memory-result\.v1-.*\.json$/u.test(name)).sort();
-    for (const name of resultNames)
-        await visit(name);
+    const resultNames = includeOperationalFiles
+        ? (await readdir(root)).filter((name) => /^\.memory-result\.v1-.*\.json$/u.test(name)).sort()
+        : [];
+    if (includeOperationalFiles) {
+        await visit(".crew/memory/pending", true);
+        for (const name of resultNames)
+            await visit(name);
+    }
     // 第二遍比对摘要和目录清单，发现扫描期间的变化就拒绝完整性结论；这不是跨文件事务锁。
     for (const source of sources) {
         try {
@@ -150,9 +155,11 @@ export async function scanMemorySources(agentDir) {
                 addIssue("source_changed", file);
         }
     }
-    const currentResults = (await readdir(root)).filter((name) => /^\.memory-result\.v1-.*\.json$/u.test(name)).sort();
-    if (JSON.stringify(currentResults) !== JSON.stringify(resultNames))
-        addIssue("source_changed", ".memory-result.v1-*.json");
+    if (includeOperationalFiles) {
+        const currentResults = (await readdir(root)).filter((name) => /^\.memory-result\.v1-.*\.json$/u.test(name)).sort();
+        if (JSON.stringify(currentResults) !== JSON.stringify(resultNames))
+            addIssue("source_changed", ".memory-result.v1-*.json");
+    }
     if (await realpath(agentDir) !== root)
         addIssue("source_changed", "agent_root");
     return { root, sources: sources.sort((a, b) => a.path.localeCompare(b.path)), issues, totalBytes };

package/dist/local-memory/inventory-v2.js

@@ -0,0 +1,362 @@
+import { createHash } from "node:crypto";
+import { lstat, realpath } from "node:fs/promises";
+import { renderLocalMemoryIndexCard } from "./index-budget.js";
+import { inventoryLimits, readMemoryInventorySource } from "./inventory-source.js";
+import { MigrationStateSchema } from "./migration-schema.js";
+import { assertMigrationStateIntegrity } from "./migration-state-invariants.js";
+import { isHarmlessMigrationRemainder } from "./migration-plan.js";
+import { parseFullMemoryIndex, selectLocalMemoryPreview } from "./storage-format.js";
+import { isDeferredMemoryArtifactPath } from "./source-path.js";
+const STATE_PATH = ".crew/memory/migration.v1.json";
+const DETAIL_MARKER_PATTERN = /<!--\s*NOWWORK:MEMORY_ENTRY:(?:BEGIN\s+id=([A-Za-z0-9_-]+)\s+revision=([1-9][0-9]*)|END\s+id=([A-Za-z0-9_-]+))\s*-->/gu;
+const CONTROL_MARKER_PATTERN = /NOWWORK:MEMORY_(?:ENTRY|INDEX):/u;
+function errorCode(error, fallback) {
+    if (error instanceof Error && /^[a-z0-9_]+$/u.test(error.message))
+        return error.message;
+    return fallback;
+}
+function sourceFields(entry) {
+    return {
+        id: entry.id,
+        file: entry.file,
+        type: entry.type,
+        status: entry.status,
+        summary: entry.summary,
+        triggers: entry.triggers.join(", "),
+        priority: entry.priority,
+        revision: String(entry.revision),
+        anchor: entry.anchor ?? "",
+        updated: entry.updated,
+        ...(entry.reviewAfter === null ? {} : { review_after: entry.reviewAfter }),
+    };
+}
+function exactManagedDetailBodies(detail, targetId) {
+    const escapedId = targetId.replace(/[.*+?^${}()|[\]\\]/gu, "\\$&");
+    const pattern = new RegExp(`<!--\\s*NOWWORK:MEMORY_ENTRY:BEGIN\\s+id=${escapedId}\\s+revision=[1-9][0-9]*\\s*-->[\\s\\S]*?<!--\\s*NOWWORK:MEMORY_ENTRY:END\\s+id=${escapedId}\\s*-->`, "gu");
+    return [...detail.matchAll(pattern)].map(match => {
+        const beginEnd = match[0].indexOf("-->") + 3;
+        const endStart = match[0].lastIndexOf("<!--");
+        return match[0].slice(beginEnd, endStart);
+    });
+}
+function harmlessDetailRemainder(text) {
+    let cursor = 0;
+    let lineStart = true;
+    const whitespace = /\s+/uy;
+    while (cursor < text.length) {
+        whitespace.lastIndex = cursor;
+        const spacing = whitespace.exec(text);
+        if (spacing !== null) {
+            const lastNewline = spacing[0].lastIndexOf("\n");
+            lineStart = lastNewline === spacing[0].length - 1;
+            cursor = whitespace.lastIndex;
+            continue;
+        }
+        if (text.startsWith("<!--", cursor)) {
+            const end = text.indexOf("-->", cursor + 4);
+            if (end === -1)
+                return false;
+            cursor = end + 3;
+            continue;
+        }
+        if (lineStart && text[cursor] === "#") {
+            const end = text.indexOf("\n", cursor + 1);
+            if (end === -1)
+                return true;
+            cursor = end + 1;
+            lineStart = true;
+            continue;
+        }
+        return false;
+    }
+    return true;
+}
+/**
+ * A migrated legacy note can remain on disk for external references.  It is
+ * safe to exclude it from the V2 orphan scan only when the immutable
+ * migration mapping proves that the complete, unchanged source was copied to
+ * the indexed target entry.
+ */
+export function findControlledPreservedSourcePaths(entries, sources, mappings, sourceHashes = new Map()) {
+    const entriesById = new Map(entries.map(entry => [entry.id, entry]));
+    const sourcesByPath = new Map(sources.map(source => [source.path, source]));
+    const grouped = new Map();
+    for (const mapping of mappings) {
+        if (!mapping.sourceFile.startsWith("notes/"))
+            continue;
+        const current = grouped.get(mapping.sourceFile) ?? [];
+        grouped.set(mapping.sourceFile, [...current, mapping]);
+    }
+    const preserved = [];
+    for (const [path, values] of grouped) {
+        const source = sourcesByPath.get(path);
+        if (source === undefined || values.length === 0)
+            continue;
+        const ranges = [...values].sort((left, right) => left.start - right.start);
+        let cursor = 0;
+        let valid = true;
+        for (const mapping of ranges) {
+            const targetFile = `notes/${mapping.category}.md`;
+            const target = mapping.targetId === undefined ? undefined : entriesById.get(mapping.targetId);
+            const targetSource = target === undefined ? undefined : sourcesByPath.get(target.file);
+            const expectedSourceSha256 = mapping.sourceSha256 ?? sourceHashes.get(path);
+            const sourceBody = source.content.slice(mapping.start, mapping.end);
+            const renderedBody = mapping.sourceFile === targetFile
+                ? sourceBody
+                : `\n## ${mapping.title}\n\nScope: ${mapping.scope}\n\n${sourceBody}\n`;
+            const sourceBlockBodies = mapping.targetId === undefined
+                ? []
+                : exactManagedDetailBodies(sourceBody, mapping.targetId);
+            const expectedBody = mapping.sourceFile === targetFile && sourceBlockBodies.length === 1
+                ? sourceBlockBodies[0]
+                : renderedBody;
+            const targetBodies = mapping.targetId === undefined || targetSource === undefined
+                ? []
+                : exactManagedDetailBodies(targetSource.content, mapping.targetId);
+            if (expectedSourceSha256 === undefined || expectedSourceSha256 !== source.sha256
+                || mapping.start < cursor || !isHarmlessMigrationRemainder(source.content.slice(cursor, mapping.start))
+                || mapping.end > source.content.length
+                || target === undefined || target.file !== targetFile || targetSource === undefined
+                || targetBodies.length !== 1 || targetBodies[0] !== expectedBody) {
+                valid = false;
+                break;
+            }
+            cursor = mapping.end;
+        }
+        if (valid && isHarmlessMigrationRemainder(source.content.slice(cursor)))
+            preserved.push(path);
+    }
+    return preserved.sort((left, right) => left.localeCompare(right));
+}
+export function* scanManagedDetailMarkers(detail) {
+    for (const marker of detail.matchAll(DETAIL_MARKER_PATTERN)) {
+        const beginId = marker[1];
+        yield {
+            start: marker.index,
+            end: marker.index + marker[0].length,
+            kind: beginId === undefined ? "end" : "begin",
+            id: beginId ?? marker[3],
+            revision: beginId === undefined ? null : Number(marker[2]),
+        };
+    }
+}
+/** Parse one note once, then let every index entry use constant-time lookups. */
+export function analyzeManagedDetailFile(detail, expectedIds, scan = scanManagedDetailMarkers) {
+    const revisionsById = new Map();
+    const remainder = [];
+    let cursor = 0;
+    let markerInvalid = false;
+    let unindexed = false;
+    let open = null;
+    for (const marker of scan(detail)) {
+        const between = detail.slice(cursor, marker.start);
+        if (CONTROL_MARKER_PATTERN.test(between))
+            markerInvalid = true;
+        if (open === null)
+            remainder.push(between);
+        cursor = marker.end;
+        if (marker.kind === "begin") {
+            if (open !== null || marker.revision === null)
+                markerInvalid = true;
+            else
+                open = { id: marker.id, revision: marker.revision };
+            continue;
+        }
+        if (open === null || marker.id !== open.id) {
+            markerInvalid = true;
+            continue;
+        }
+        const revisions = revisionsById.get(open.id) ?? [];
+        revisions.push(open.revision);
+        revisionsById.set(open.id, revisions);
+        if (!expectedIds.has(open.id))
+            unindexed = true;
+        open = null;
+    }
+    const tail = detail.slice(cursor);
+    if (CONTROL_MARKER_PATTERN.test(tail) || open !== null)
+        markerInvalid = true;
+    if (open === null)
+        remainder.push(tail);
+    const uncovered = remainder.join("");
+    const coverageIssue = markerInvalid
+        ? "managed_anchor_invalid"
+        : unindexed
+            ? "unindexed_detail"
+            : harmlessDetailRemainder(uncovered)
+                ? null
+                : "unindexed_text_requires_mapping";
+    return { revisionsById, coverageIssue };
+}
+function inspectDetailEntry(entry, analysis) {
+    if (analysis === undefined)
+        return "missing_detail";
+    if (analysis.coverageIssue === "managed_anchor_invalid")
+        return "managed_anchor_invalid";
+    const revisions = analysis.revisionsById.get(entry.id) ?? [];
+    if (revisions.length !== 1)
+        return "managed_anchor_invalid";
+    return revisions[0] > entry.revision ? "managed_revision_invalid" : null;
+}
+export function inspectManagedDetailFiles(entries, sources, analyze = analyzeManagedDetailFile) {
+    return inspectManagedDetailFilesWithMappings(entries, sources, [], new Map(), analyze);
+}
+export function inspectManagedDetailFilesWithMappings(entries, sources, mappings, sourceHashes = new Map(), analyze = analyzeManagedDetailFile) {
+    const sourceMap = new Map(sources.map(source => [source.path, source]));
+    const preservedSourcePaths = findControlledPreservedSourcePaths(entries, sources, mappings, sourceHashes);
+    const preserved = new Set(preservedSourcePaths);
+    const expectedByFile = new Map();
+    for (const entry of entries) {
+        const expected = expectedByFile.get(entry.file) ?? new Set();
+        expected.add(entry.id);
+        expectedByFile.set(entry.file, expected);
+    }
+    const paths = new Set([
+        ...expectedByFile.keys(),
+        ...sources.filter(source => source.path.startsWith("notes/") && !isDeferredMemoryArtifactPath(source.path)).map(source => source.path),
+    ]);
+    const analyses = new Map();
+    const fileIssues = [];
+    for (const path of paths) {
+        const source = sourceMap.get(path);
+        if (source === undefined)
+            continue;
+        const analysis = analyze(source.content, expectedByFile.get(path) ?? new Set());
+        analyses.set(path, analysis);
+        if (analysis.coverageIssue !== null
+            && !(preserved.has(path) && analysis.coverageIssue === "unindexed_text_requires_mapping")) {
+            fileIssues.push({ code: analysis.coverageIssue, path });
+        }
+    }
+    const entryIssues = new Map();
+    for (const entry of entries) {
+        const issue = inspectDetailEntry(entry, analyses.get(entry.file));
+        if (issue !== null)
+            entryIssues.set(entry.id, issue);
+    }
+    return { entryIssues, fileIssues, preservedSourcePaths };
+}
+async function inspectCommittedState(root, budget) {
+    let source;
+    try {
+        source = await readMemoryInventorySource(root, STATE_PATH, Math.min(inventoryLimits.fileBytes, budget));
+    }
+    catch (error) {
+        return {
+            valid: false,
+            source: null,
+            state: null,
+            issue: {
+                code: error.code === "ENOENT"
+                    ? "memory_migration_state_missing"
+                    : errorCode(error, "memory_migration_state_invalid"),
+                path: STATE_PATH,
+            },
+        };
+    }
+    try {
+        const state = MigrationStateSchema.parse(JSON.parse(source.content));
+        const fingerprint = createHash("sha256").update(await realpath(root)).digest("hex");
+        const ownerIdentity = String((await lstat(root)).uid);
+        if (state.phase !== "committed" || state.storageVersion !== 2
+            || state.identity.rootFingerprint !== fingerprint
+            || state.identity.ownerIdentity !== ownerIdentity) {
+            throw new Error("memory_migration_state_invalid");
+        }
+        assertMigrationStateIntegrity(state, state.identity);
+        const confirmed = await readMemoryInventorySource(root, STATE_PATH, Math.min(inventoryLimits.fileBytes, budget));
+        if (confirmed.sha256 !== source.sha256) {
+            return { valid: false, source, state: null, issue: { code: "source_changed", path: STATE_PATH } };
+        }
+        return { valid: true, source, state, issue: null };
+    }
+    catch (error) {
+        return {
+            valid: false,
+            source,
+            state: null,
+            issue: {
+                code: "memory_migration_state_invalid",
+                path: STATE_PATH,
+            },
+        };
+    }
+}
+/** Validate an already-committed schema2 store without sending it through the schema1 migration preview. */
+export async function inspectV2MemoryInventory(root, sources, readComplete) {
+    const sourceMap = new Map(sources.map(source => [source.path, source]));
+    const memory = sourceMap.get("MEMORY.md")?.content ?? "";
+    const issues = [];
+    const addIssue = (code, path) => {
+        if (!issues.some(issue => issue.code === code && issue.path === path))
+            issues.push({ code, path });
+    };
+    let entries = [];
+    try {
+        entries = parseFullMemoryIndex(memory);
+    }
+    catch (error) {
+        addIssue(errorCode(error, "local_memory_storage_entry_invalid"), "MEMORY.md");
+    }
+    const scannedBytes = sources.reduce((total, source) => total + source.bytes, 0);
+    const state = await inspectCommittedState(root, Math.max(0, inventoryLimits.totalBytes - scannedBytes));
+    if (state.issue)
+        addIssue(state.issue.code, state.issue.path);
+    const sourceHashes = new Map((state.state?.manifest.files ?? [])
+        .flatMap(file => file.before.exists ? [[file.path, file.before.sha256]] : []));
+    const detailInspection = inspectManagedDetailFilesWithMappings(entries, sources, state.state?.manifest.recordMappings ?? [], sourceHashes);
+    for (const issue of detailInspection.fileIssues)
+        addIssue(issue.code, issue.path);
+    for (const entry of entries) {
+        const issue = detailInspection.entryIssues.get(entry.id);
+        if (issue !== undefined)
+            addIssue(issue, entry.file);
+    }
+    let preview = null;
+    if (readComplete && !issues.some(issue => issue.path === "MEMORY.md")) {
+        try {
+            const selected = selectLocalMemoryPreview(memory);
+            preview = {
+                memory: selected.memory,
+                budget: {
+                    entries: selected.entries.length,
+                    omittedEntries: selected.omittedIds.length,
+                    // The bounded projection itself fits; reason records normal long-tail omission.
+                    fits: true,
+                    reason: selected.reason,
+                },
+                writerCompatible: state.valid && issues.length === 0,
+                notes: [],
+            };
+        }
+        catch (error) {
+            addIssue(errorCode(error, "local_memory_preview_invalid"), "MEMORY.md");
+        }
+    }
+    const shownDetails = new Set();
+    const writeReady = readComplete && state.valid && issues.length === 0;
+    return {
+        stateValid: state.valid,
+        writeReady,
+        stateSource: state.source,
+        preservedSourcePaths: detailInspection.preservedSourcePaths,
+        analysis: {
+            entries: entries.map(entry => {
+                const detail = shownDetails.has(entry.file) ? null : sourceMap.get(entry.file)?.content ?? null;
+                shownDetails.add(entry.file);
+                return {
+                    title: entry.title,
+                    format: "managed",
+                    disposition: detailInspection.entryIssues.has(entry.id) || !writeReady ? "blocked" : "review_required",
+                    sourceFields: sourceFields(entry),
+                    rawCard: renderLocalMemoryIndexCard(entry),
+                    detail,
+                    proposed: entry,
+                };
+            }),
+            issues,
+            preview,
+            rawMemory: memory,
+        },
+    };
+}