@nowcrew/daemon 0.6.51 → 0.6.52

package/dist/storage-manifest.js

@@ -0,0 +1,440 @@
+import { createHash } from "node:crypto";
+import { createReadStream } from "node:fs";
+import { lstat, readdir, readlink, realpath } from "node:fs/promises";
+import { dirname, isAbsolute, posix, relative, resolve, win32 } from "node:path";
+import { z } from "zod";
+import { validateStorageEnvironment } from "./storage-layout.js";
+export const STORAGE_MANIFEST_SCHEMA = "nowwork.storage.migration-manifest";
+export const STORAGE_MANIFEST_VERSION = 1;
+export const MAX_STORAGE_MANIFEST_ENTRIES = 100_000;
+const SourceKindSchema = z.enum(["daemon-home", "agents-root", "legacy-agents-root"]);
+const HashSchema = z.string().regex(/^[0-9a-f]{64}$/u);
+const ManifestEntrySchema = z.object({
+    sourceKind: SourceKindSchema,
+    sourcePath: z.string().min(1),
+    logicalDestination: z.string().min(1).nullable(),
+    type: z.enum(["file", "directory", "symlink", "unsupported"]),
+    disposition: z.enum(["copy", "skip", "transform"]),
+    reason: z.string().min(1),
+    mode: z.string().regex(/^[0-7]{4}$/u),
+    sizeBytes: z.number().int().nonnegative(),
+    sha256: HashSchema.optional(),
+    transform: z.literal("redact_live_like_tokens_v1").optional(),
+    targetSha256: HashSchema.optional(),
+    targetMode: z.string().regex(/^[0-7]{4}$/u).optional(),
+    link: z.object({
+        classification: z.enum(["internal", "external", "broken"]),
+        portable: z.boolean(),
+        targetFingerprint: z.string().regex(/^sha256:[0-9a-f]{64}$/u),
+        relativeTarget: z.string().min(1).optional(),
+    }).strict().optional(),
+}).strict();
+const StorageManifestSchema = z.object({
+    schema: z.literal(STORAGE_MANIFEST_SCHEMA),
+    formatVersion: z.literal(STORAGE_MANIFEST_VERSION),
+    migrationId: z.string().uuid(),
+    generatedAt: z.string().datetime({ offset: true }),
+    environment: z.string().min(1),
+    workspaceId: z.string().uuid(),
+    targetWorkspaceRoot: z.string().min(1),
+    sourceKinds: z.array(SourceKindSchema),
+    entries: z.array(ManifestEntrySchema).max(MAX_STORAGE_MANIFEST_ENTRIES),
+    summary: z.object({
+        records: z.number().int().nonnegative(),
+        files: z.number().int().nonnegative(),
+        directories: z.number().int().nonnegative(),
+        symlinks: z.number().int().nonnegative(),
+        copied: z.number().int().nonnegative(),
+        skipped: z.number().int().nonnegative(),
+        transformations: z.number().int().nonnegative(),
+        externalSymlinks: z.number().int().nonnegative(),
+        bytesToCopy: z.number().int().nonnegative(),
+        errors: z.number().int().nonnegative(),
+    }).strict(),
+}).strict();
+const CONTROL = /[\0\r\n]/u;
+function normalizedRelativePath(value) {
+    const normalized = value.replaceAll("\\", "/");
+    if (normalized.length === 0 || CONTROL.test(normalized) || posix.isAbsolute(normalized) || win32.isAbsolute(normalized)) {
+        throw new Error(`migration path must be normalized relative path: ${value}`);
+    }
+    const segments = normalized.split("/");
+    if (segments.some((segment) => segment.length === 0 || segment === "." || segment === "..")) {
+        throw new Error(`migration path must be normalized relative path: ${value}`);
+    }
+    if (posix.normalize(normalized) !== normalized)
+        throw new Error(`migration path must be normalized relative path: ${value}`);
+    return normalized;
+}
+function modeString(mode) {
+    return (mode & 0o7777).toString(8).padStart(4, "0");
+}
+function pathWithin(root, candidate) {
+    const child = relative(root, candidate);
+    return child === "" || (!child.startsWith("..") && !isAbsolute(child));
+}
+function pathReason(sourceKind, sourcePath) {
+    const wrapped = `/${sourcePath}/`;
+    if (sourceKind === "daemon-home" && wrapped.startsWith("/profiles/"))
+        return "credential_material";
+    if (wrapped.includes("/.home/.codex/auth.json/")
+        || wrapped.includes("/.home/auth.json/")
+        || wrapped.includes("/.home/.codex/sessions/")
+        || wrapped.includes("/.home/.config/")
+        || wrapped.includes("/.home/.local/share/"))
+        return "credential_material";
+    if (wrapped.includes("/.home/.cache/")
+        || wrapped.includes("/.home/.codex/tmp/")
+        || wrapped.includes("/.crew/ability-cache/")
+        || wrapped.includes("/.crew/executions/")
+        || wrapped.includes("/.execution-telemetry/")
+        || wrapped.includes("/tmp/"))
+        return "local_runtime";
+    return null;
+}
+async function fileDigest(path) {
+    const before = await lstat(path);
+    const digest = createHash("sha256");
+    await new Promise((resolvePromise, reject) => {
+        const stream = createReadStream(path);
+        stream.on("data", (chunk) => digest.update(chunk));
+        stream.once("end", resolvePromise);
+        stream.once("error", reject);
+    });
+    const after = await lstat(path);
+    if (before.ino !== after.ino || before.size !== after.size || before.mtimeMs !== after.mtimeMs) {
+        throw new Error(`source changed while hashing: ${path}`);
+    }
+    return digest.digest("hex");
+}
+export function assertStorageManifestEntryLimit(count) {
+    if (!Number.isSafeInteger(count) || count < 0 || count > MAX_STORAGE_MANIFEST_ENTRIES) {
+        throw new Error(`storage manifest entry limit exceeded: ${count}`);
+    }
+}
+function fingerprint(value) {
+    return `sha256:${createHash("sha256").update(value).digest("hex")}`;
+}
+class CanonicalHashWriter {
+    hash = createHash("sha256");
+    chunks = [];
+    bufferedBytes = 0;
+    append(value) {
+        this.chunks.push(value);
+        this.bufferedBytes += Buffer.byteLength(value);
+        if (this.bufferedBytes >= 64 * 1024)
+            this.flush();
+    }
+    flush() {
+        if (this.chunks.length === 0)
+            return;
+        this.hash.update(this.chunks.join(""));
+        this.chunks = [];
+        this.bufferedBytes = 0;
+    }
+    write(value) {
+        if (value === null) {
+            this.append("null");
+            return;
+        }
+        if (typeof value === "string" || typeof value === "boolean") {
+            this.append(JSON.stringify(value));
+            return;
+        }
+        if (typeof value === "number") {
+            if (!Number.isFinite(value))
+                throw new Error("storage manifest contains a non-finite number");
+            this.append(JSON.stringify(value));
+            return;
+        }
+        if (Array.isArray(value)) {
+            this.append("[");
+            value.forEach((child, index) => {
+                if (index > 0)
+                    this.append(",");
+                this.write(child);
+            });
+            this.append("]");
+            return;
+        }
+        if (typeof value === "object") {
+            const record = value;
+            const keys = Object.keys(record)
+                .filter((key) => record[key] !== undefined)
+                .sort((left, right) => left.localeCompare(right));
+            this.append("{");
+            keys.forEach((key, index) => {
+                if (index > 0)
+                    this.append(",");
+                this.append(JSON.stringify(key));
+                this.append(":");
+                this.write(record[key]);
+            });
+            this.append("}");
+            return;
+        }
+        throw new Error("storage manifest contains a non-JSON value");
+    }
+    digest() {
+        this.flush();
+        return this.hash.digest("hex");
+    }
+}
+function validatePolicy(policy) {
+    const exclusions = policy.exclusions.map((entry) => ({
+        sourceKind: SourceKindSchema.parse(entry.sourceKind),
+        sourcePath: normalizedRelativePath(entry.sourcePath),
+        reason: z.string().min(1).max(256).parse(entry.reason),
+    }));
+    const transformations = policy.transformations.map((entry) => ({
+        sourceKind: SourceKindSchema.parse(entry.sourceKind),
+        sourcePath: normalizedRelativePath(entry.sourcePath),
+        logicalDestination: normalizedRelativePath(entry.logicalDestination),
+        sourceSha256: HashSchema.parse(entry.sourceSha256),
+        targetSha256: HashSchema.parse(entry.targetSha256),
+        targetMode: z.string().regex(/^[0-7]{4}$/u).parse(entry.targetMode),
+        transform: z.literal("redact_live_like_tokens_v1").parse(entry.transform),
+    }));
+    const identities = new Set();
+    for (const entry of [...exclusions, ...transformations]) {
+        const identity = `${entry.sourceKind}:${entry.sourcePath}`;
+        if (identities.has(identity))
+            throw new Error(`duplicate migration policy path: ${identity}`);
+        identities.add(identity);
+    }
+    return Object.freeze({ exclusions: Object.freeze(exclusions), transformations: Object.freeze(transformations) });
+}
+const policyKey = (sourceKind, sourcePath) => `${sourceKind}\0${sourcePath}`;
+function indexPolicy(policy) {
+    return Object.freeze({
+        exclusions: new Map(policy.exclusions.map((entry) => [policyKey(entry.sourceKind, entry.sourcePath), entry])),
+        transformations: new Map(policy.transformations.map((entry) => [policyKey(entry.sourceKind, entry.sourcePath), entry])),
+    });
+}
+function excludedBy(index, sourceKind, sourcePath) {
+    let candidate = sourcePath;
+    for (;;) {
+        const exclusion = index.exclusions.get(policyKey(sourceKind, candidate));
+        if (exclusion !== undefined)
+            return exclusion;
+        const separator = candidate.lastIndexOf("/");
+        if (separator < 0)
+            return undefined;
+        candidate = candidate.slice(0, separator);
+    }
+}
+function transformedBy(index, sourceKind, sourcePath) {
+    return index.transformations.get(policyKey(sourceKind, sourcePath));
+}
+async function linkEvidence(root, path) {
+    const rawTarget = await readlink(path);
+    const resolvedTarget = resolve(dirname(path), rawTarget);
+    const canonicalRoot = await realpath(root);
+    let canonicalTarget;
+    try {
+        canonicalTarget = await realpath(resolvedTarget);
+    }
+    catch (error) {
+        if (error.code !== "ENOENT")
+            throw error;
+        return Object.freeze({ classification: "broken", portable: false, targetFingerprint: fingerprint(rawTarget) });
+    }
+    if (!pathWithin(canonicalRoot, canonicalTarget)) {
+        return Object.freeze({ classification: "external", portable: false, targetFingerprint: fingerprint(canonicalTarget) });
+    }
+    const portable = !isAbsolute(rawTarget);
+    return Object.freeze({
+        classification: "internal",
+        portable,
+        targetFingerprint: fingerprint(canonicalTarget),
+        ...(portable ? { relativeTarget: rawTarget.replaceAll("\\", "/") } : {}),
+    });
+}
+async function scanSource(input) {
+    const entries = [];
+    const add = (entry) => {
+        if (entries.length >= input.remainingEntries) {
+            throw new Error(`storage manifest entry limit exceeded: ${entries.length + 1}`);
+        }
+        entries.push(entry);
+    };
+    const pending = [input.root];
+    while (pending.length > 0) {
+        const directory = pending.pop();
+        const children = await readdir(directory, { withFileTypes: true });
+        children.sort((left, right) => left.name.localeCompare(right.name));
+        for (const child of children) {
+            const path = resolve(directory, child.name);
+            const sourcePath = normalizedRelativePath(relative(input.root, path));
+            const stat = await lstat(path);
+            const exclusion = excludedBy(input.policy, input.sourceKind, sourcePath);
+            const transformation = transformedBy(input.policy, input.sourceKind, sourcePath);
+            const classifiedReason = pathReason(input.sourceKind, sourcePath);
+            const defaultDestination = normalizedRelativePath(`${input.logicalPrefix}/${sourcePath}`);
+            let type = "unsupported";
+            if (stat.isDirectory())
+                type = "directory";
+            else if (stat.isFile())
+                type = "file";
+            else if (stat.isSymbolicLink())
+                type = "symlink";
+            if (stat.isDirectory())
+                pending.push(path);
+            if (exclusion !== undefined) {
+                add(ManifestEntrySchema.parse({
+                    sourceKind: input.sourceKind,
+                    sourcePath,
+                    logicalDestination: null,
+                    type,
+                    disposition: "skip",
+                    reason: exclusion.reason,
+                    mode: modeString(stat.mode),
+                    sizeBytes: stat.size,
+                }));
+                continue;
+            }
+            if (transformation !== undefined) {
+                if (type !== "file")
+                    throw new Error(`transformation source is not a file: ${sourcePath}`);
+                const digest = await fileDigest(path);
+                if (digest !== transformation.sourceSha256)
+                    throw new Error(`transformation source drift: ${sourcePath}`);
+                add(ManifestEntrySchema.parse({
+                    sourceKind: input.sourceKind,
+                    sourcePath,
+                    logicalDestination: transformation.logicalDestination,
+                    type,
+                    disposition: "transform",
+                    reason: "approved_target_only_transformation",
+                    mode: modeString(stat.mode),
+                    sizeBytes: stat.size,
+                    sha256: digest,
+                    transform: transformation.transform,
+                    targetSha256: transformation.targetSha256,
+                    targetMode: transformation.targetMode,
+                }));
+                continue;
+            }
+            if (type === "symlink") {
+                const link = await linkEvidence(input.root, path);
+                const reason = classifiedReason
+                    ?? (link.classification === "internal" && link.portable ? "portable_internal_symlink" : "unsupported_symlink");
+                const copy = reason === "portable_internal_symlink";
+                add(ManifestEntrySchema.parse({
+                    sourceKind: input.sourceKind,
+                    sourcePath,
+                    logicalDestination: copy ? defaultDestination : null,
+                    type,
+                    disposition: copy ? "copy" : "skip",
+                    reason,
+                    mode: modeString(stat.mode),
+                    sizeBytes: stat.size,
+                    link,
+                }));
+                continue;
+            }
+            if (classifiedReason !== null || type === "unsupported") {
+                add(ManifestEntrySchema.parse({
+                    sourceKind: input.sourceKind,
+                    sourcePath,
+                    logicalDestination: null,
+                    type,
+                    disposition: "skip",
+                    reason: classifiedReason ?? "unsupported_file_type",
+                    mode: modeString(stat.mode),
+                    sizeBytes: stat.size,
+                }));
+                continue;
+            }
+            add(ManifestEntrySchema.parse({
+                sourceKind: input.sourceKind,
+                sourcePath,
+                logicalDestination: defaultDestination,
+                type,
+                disposition: "copy",
+                reason: "durable_workspace_data",
+                mode: modeString(stat.mode),
+                sizeBytes: stat.size,
+                ...(type === "file" ? { sha256: await fileDigest(path) } : {}),
+            }));
+        }
+    }
+    return entries;
+}
+export async function generateStorageManifest(input) {
+    z.string().uuid().parse(input.migrationId);
+    z.string().datetime({ offset: true }).parse(input.generatedAt);
+    z.string().uuid().parse(input.workspaceId);
+    if (!isAbsolute(input.targetWorkspaceRoot) || CONTROL.test(input.targetWorkspaceRoot)) {
+        throw new Error("target workspace root must be an absolute single-line path");
+    }
+    const environment = validateStorageEnvironment(input.environment);
+    const policy = validatePolicy(input.policy);
+    const policyIndex = indexPolicy(policy);
+    const entryLimit = z.number().int().positive().max(MAX_STORAGE_MANIFEST_ENTRIES)
+        .parse(input.entryLimit ?? MAX_STORAGE_MANIFEST_ENTRIES);
+    const sourceKinds = input.sources.map((source) => SourceKindSchema.parse(source.kind));
+    if (new Set(sourceKinds).size !== sourceKinds.length)
+        throw new Error("duplicate migration source kind");
+    const entries = [];
+    for (const source of input.sources) {
+        if (!isAbsolute(source.root))
+            throw new Error(`migration source root must be absolute: ${source.kind}`);
+        entries.push(...await scanSource({
+            sourceKind: source.kind,
+            root: source.root,
+            logicalPrefix: normalizedRelativePath(source.logicalPrefix),
+            policy: policyIndex,
+            remainingEntries: entryLimit - entries.length,
+        }));
+        if (entries.length > entryLimit)
+            throw new Error(`storage manifest entry limit exceeded: ${entries.length}`);
+    }
+    entries.sort((left, right) => (left.sourceKind.localeCompare(right.sourceKind) || left.sourcePath.localeCompare(right.sourcePath)));
+    const destinations = new Set();
+    for (const entry of entries) {
+        if (entry.logicalDestination === null)
+            continue;
+        if (destinations.has(entry.logicalDestination))
+            throw new Error(`duplicate migration destination: ${entry.logicalDestination}`);
+        destinations.add(entry.logicalDestination);
+    }
+    return StorageManifestSchema.parse({
+        schema: STORAGE_MANIFEST_SCHEMA,
+        formatVersion: STORAGE_MANIFEST_VERSION,
+        migrationId: input.migrationId,
+        generatedAt: input.generatedAt,
+        environment,
+        workspaceId: input.workspaceId.toLowerCase(),
+        targetWorkspaceRoot: input.targetWorkspaceRoot,
+        sourceKinds,
+        entries,
+        summary: {
+            records: entries.length,
+            files: entries.filter((entry) => entry.type === "file").length,
+            directories: entries.filter((entry) => entry.type === "directory").length,
+            symlinks: entries.filter((entry) => entry.type === "symlink").length,
+            copied: entries.filter((entry) => entry.disposition === "copy").length,
+            skipped: entries.filter((entry) => entry.disposition === "skip").length,
+            transformations: entries.filter((entry) => entry.disposition === "transform").length,
+            externalSymlinks: entries.filter((entry) => entry.link?.classification === "external").length,
+            bytesToCopy: entries.filter((entry) => entry.type === "file" && entry.disposition !== "skip")
+                .reduce((total, entry) => total + entry.sizeBytes, 0),
+            errors: 0,
+        },
+    });
+}
+export function parseStorageManifest(raw) {
+    try {
+        return StorageManifestSchema.parse(JSON.parse(raw));
+    }
+    catch (error) {
+        throw new Error("Invalid storage migration manifest", { cause: error });
+    }
+}
+export function storageManifestDigest(manifest) {
+    const { generatedAt: _generatedAt, ...stable } = manifest;
+    const writer = new CanonicalHashWriter();
+    writer.write(stable);
+    return writer.digest();
+}