@vgai/editor 0.4.0 → 0.4.1

package/server/project-output-writer.ts

@@ -1,5 +1,5 @@
 /**
- * Atomic persistence boundary for files produced by project operations.
+ * Atomic persistence boundary for files produced by project tools.
  *
  * Generators remain ordinary project/package code. They hand their completed
  * bytes to this writer only when they are ready to commit; the editor host
@@ -8,20 +8,58 @@
  * visible to the asset watcher.
  */
 
-import { randomUUID } from 'node:crypto';
-import { copyFile, mkdir, rename, rm, stat, writeFile } from 'node:fs/promises';
-import { dirname, posix, resolve, sep } from 'node:path';
+import { createHash, randomUUID } from 'node:crypto';
+import { createReadStream } from 'node:fs';
+import { mkdir, readdir, readFile, rename, rm, stat, writeFile } from 'node:fs/promises';
+import { dirname, extname, posix, relative, resolve, sep } from 'node:path';
+import {
+  emptyProjectProvenanceDocument,
+  PROJECT_PROVENANCE_PATH,
+  ProjectProvenanceDocumentSchema,
+  ProjectProvenanceOperationSchema,
+} from '../../vgai-sdk/src/project/provenance';
 import type {
   ProjectGeneratedOutput,
   ProjectGeneratedOutputFile,
   ProjectOutputFile,
+  ProjectOutputProvenanceContext,
   ProjectOutputWriter,
 } from '../../vgai-sdk/src/types';
-
-const MAX_OUTPUT_BYTES = 64 * 1024 * 1024;
+import { consumeGenerativeExecutions } from './generative-execution-context';
 
 type NormalizedOutputFile = ProjectOutputFile & { path: string };
 
+interface HashedProjectFile extends ProjectGeneratedOutputFile {
+  sha256: string;
+}
+
+const projectCommitQueues = new Map<string, Promise<unknown>>();
+
+function withCapturedExecutions(
+  provenanceContext: ProjectOutputProvenanceContext,
+): ProjectOutputProvenanceContext {
+  const captured = consumeGenerativeExecutions();
+  const explicit = provenanceContext.executions
+    ? [...provenanceContext.executions]
+    : provenanceContext.execution
+      ? [provenanceContext.execution]
+      : [];
+  if (
+    captured.length > 0 &&
+    explicit.length > 0 &&
+    JSON.stringify(captured) !== JSON.stringify(explicit)
+  ) {
+    throw new Error('Explicit and automatically captured provider execution facts disagree.');
+  }
+  const executions = captured.length > 0 ? captured : explicit;
+  const effective = { ...provenanceContext };
+  delete effective.execution;
+  delete effective.executions;
+  if (executions.length === 1) effective.execution = executions[0]!;
+  if (executions.length > 1) effective.executions = executions;
+  return effective;
+}
+
 function normalizeOutputPath(path: string): string {
   const slashPath = path.split(sep).join('/');
   const normalized = posix.normalize(slashPath).replace(/^\.\//, '');
@@ -44,12 +82,24 @@ function byteLength(content: string | Uint8Array): number {
   return typeof content === 'string' ? Buffer.byteLength(content) : content.byteLength;
 }
 
-async function exists(path: string): Promise<boolean> {
+function sha256(content: string | Uint8Array): string {
+  return createHash('sha256').update(content).digest('hex');
+}
+
+async function sha256File(path: string): Promise<string> {
+  const hash = createHash('sha256');
+  for await (const chunk of createReadStream(path)) hash.update(chunk as Buffer);
+  return hash.digest('hex');
+}
+
+async function existingFile(path: string): Promise<boolean> {
   try {
-    await stat(path);
+    const info = await stat(path);
+    if (!info.isFile()) throw new Error(`Generated output target is not a file: ${path}`);
     return true;
-  } catch {
-    return false;
+  } catch (error) {
+    if (error instanceof Error && 'code' in error && error.code === 'ENOENT') return false;
+    throw error;
   }
 }
 
@@ -71,11 +121,6 @@ function validateBatch(files: readonly ProjectOutputFile[]): {
     paths.add(file.path);
     totalBytes += byteLength(file.content);
   }
-  if (totalBytes > MAX_OUTPUT_BYTES) {
-    throw new Error(
-      `Generated output batch is ${totalBytes} bytes; the atomic project-output limit is ${MAX_OUTPUT_BYTES} bytes.`,
-    );
-  }
   return {
     normalized,
     totalBytes,
@@ -110,24 +155,152 @@ async function commitBatch(
       const backup = resolve(backupRoot, file.path);
       await mkdir(dirname(target), { recursive: true });
       let prior: string | null = null;
-      if (await exists(target)) {
+      if (await existingFile(target)) {
         await mkdir(dirname(backup), { recursive: true });
-        await copyFile(target, backup);
+        await rename(target, backup);
         prior = backup;
       }
-      await rename(staged, target);
+      try {
+        await rename(staged, target);
+      } catch (error) {
+        if (prior) await rename(prior, target);
+        throw error;
+      }
       committed.push({ target, backup: prior });
     }
   } catch (error) {
     for (const entry of committed.reverse()) {
-      if (entry.backup) await copyFile(entry.backup, entry.target);
-      else await rm(entry.target, { force: true });
+      await rm(entry.target, { force: true });
+      if (entry.backup) await rename(entry.backup, entry.target);
     }
     throw error;
   }
 }
 
-export function createProjectOutputWriter(projectRoot: string): ProjectOutputWriter {
+async function readProvenance(projectRoot: string) {
+  try {
+    const parsed: unknown = JSON.parse(
+      await readFile(resolve(projectRoot, PROJECT_PROVENANCE_PATH), 'utf8'),
+    );
+    return ProjectProvenanceDocumentSchema.parse(parsed);
+  } catch (error) {
+    if (!(error instanceof Error) || !('code' in error) || error.code !== 'ENOENT') throw error;
+    return emptyProjectProvenanceDocument();
+  }
+}
+
+const SECRET_FIELD = /(authorization|credential|password|secret|token|api.?key|fal.?key)/i;
+const SIGNED_URL_PARAMETER = /(signature|credential|token|expires|x-amz-)/i;
+
+function sanitizeString(value: string): string {
+  try {
+    const url = new URL(value);
+    if (
+      (url.protocol === 'http:' || url.protocol === 'https:') &&
+      [...url.searchParams.keys()].some((key) => SIGNED_URL_PARAMETER.test(key))
+    ) {
+      return `${url.origin}${url.pathname}?[redacted signed query]`;
+    }
+  } catch {
+    // Most provenance strings are not URLs.
+  }
+  return value;
+}
+
+function isBinaryValue(value: object): boolean {
+  return (
+    value instanceof ArrayBuffer ||
+    ArrayBuffer.isView(value) ||
+    (typeof Blob !== 'undefined' && value instanceof Blob)
+  );
+}
+
+function sanitizeProvenanceValue(value: unknown, seen: WeakSet<object>): unknown {
+  if (typeof value === 'string') return sanitizeString(value);
+  if (value === null || typeof value === 'boolean' || typeof value === 'number') return value;
+  if (value === undefined) return undefined;
+  if (typeof value !== 'object') return `[omitted ${typeof value}]`;
+  if (isBinaryValue(value)) return '[binary omitted]';
+  if (seen.has(value)) return '[circular omitted]';
+  seen.add(value);
+  if (Array.isArray(value)) {
+    return value.map((entry) => sanitizeProvenanceValue(entry, seen) ?? null);
+  }
+  const sanitized: Record<string, unknown> = {};
+  for (const [key, entry] of Object.entries(value)) {
+    if (SECRET_FIELD.test(key)) {
+      sanitized[key] = '[redacted]';
+      continue;
+    }
+    const next = sanitizeProvenanceValue(entry, seen);
+    if (next !== undefined) sanitized[key] = next;
+  }
+  return sanitized;
+}
+
+function jsonSafeInput(input: unknown): unknown {
+  if (input === undefined) return undefined;
+  return sanitizeProvenanceValue(input, new WeakSet());
+}
+
+function provenanceOperation(
+  provenanceContext: ProjectOutputProvenanceContext,
+  outputs: readonly HashedProjectFile[],
+) {
+  return ProjectProvenanceOperationSchema.parse({
+    createdAt: new Date().toISOString(),
+    operation: {
+      name: provenanceContext.operationName,
+      ...(provenanceContext.operationSource ? { source: provenanceContext.operationSource } : {}),
+    },
+    ...(provenanceContext.execution ? { execution: provenanceContext.execution } : {}),
+    ...(provenanceContext.executions ? { executions: provenanceContext.executions } : {}),
+    ...(provenanceContext.input === undefined
+      ? {}
+      : { input: jsonSafeInput(provenanceContext.input) }),
+    outputs,
+  });
+}
+
+async function nextProvenanceFile(
+  projectRoot: string,
+  provenanceOperationId: string,
+  provenanceContext: ProjectOutputProvenanceContext,
+  outputs: readonly HashedProjectFile[],
+): Promise<NormalizedOutputFile> {
+  const provenance = await readProvenance(projectRoot);
+  const nextProvenance = ProjectProvenanceDocumentSchema.parse({
+    ...provenance,
+    operations: {
+      ...provenance.operations,
+      [provenanceOperationId]: provenanceOperation(provenanceContext, outputs),
+    },
+  });
+  return {
+    path: PROJECT_PROVENANCE_PATH,
+    content: `${JSON.stringify(nextProvenance, null, 2)}\n`,
+    mediaType: 'application/json',
+    role: 'provenance',
+  };
+}
+
+async function serializeProjectCommit<T>(projectRoot: string, work: () => Promise<T>): Promise<T> {
+  const previous = projectCommitQueues.get(projectRoot) ?? Promise.resolve();
+  const current = previous.catch(() => undefined).then(work);
+  projectCommitQueues.set(projectRoot, current);
+  try {
+    return await current;
+  } finally {
+    if (projectCommitQueues.get(projectRoot) === current) projectCommitQueues.delete(projectRoot);
+  }
+}
+
+export function createProjectOutputWriter(
+  projectRoot: string,
+  provenanceContext: ProjectOutputProvenanceContext = {
+    operationName: 'project.output.write',
+  },
+): ProjectOutputWriter {
   return {
     async write(files, options = {}): Promise<ProjectGeneratedOutput> {
       const { normalized, summaries, totalBytes } = validateBatch(files);
@@ -135,22 +308,253 @@ export function createProjectOutputWriter(projectRoot: string): ProjectOutputWri
         return { files: summaries, totalBytes, dryRun: true };
       }
 
-      const transactionRoot = resolve(
-        projectRoot,
-        '.vgai',
-        'tmp',
-        `project-output-${randomUUID()}`,
-      );
-      const stagedRoot = resolve(transactionRoot, 'staged');
-      const backupRoot = resolve(transactionRoot, 'backup');
-      try {
-        await stageBatch(stagedRoot, normalized);
-        await commitBatch(projectRoot, stagedRoot, backupRoot, normalized);
-      } finally {
-        await rm(transactionRoot, { recursive: true, force: true });
-      }
+      const effectiveProvenanceContext = withCapturedExecutions(provenanceContext);
+
+      return serializeProjectCommit(projectRoot, async () => {
+        const provenanceOperationId = randomUUID();
+        const hashedFiles: HashedProjectFile[] = normalized.map((file) => ({
+          path: file.path,
+          bytes: byteLength(file.content),
+          sha256: sha256(file.content),
+          ...(file.mediaType ? { mediaType: file.mediaType } : {}),
+          ...(file.role ? { role: file.role } : {}),
+        }));
+        const provenanceFile = await nextProvenanceFile(
+          projectRoot,
+          provenanceOperationId,
+          effectiveProvenanceContext,
+          hashedFiles,
+        );
+        const committedFiles = [...normalized, provenanceFile];
+        const transactionRoot = resolve(
+          projectRoot,
+          '.vgai',
+          'tmp',
+          `project-output-${provenanceOperationId}`,
+        );
+        const stagedRoot = resolve(transactionRoot, 'staged');
+        const backupRoot = resolve(transactionRoot, 'backup');
+        try {
+          await stageBatch(stagedRoot, committedFiles);
+          await commitBatch(projectRoot, stagedRoot, backupRoot, committedFiles);
+        } finally {
+          await rm(transactionRoot, { recursive: true, force: true });
+        }
 
-      return { files: summaries, totalBytes, dryRun: false };
+        return {
+          files: summaries,
+          totalBytes,
+          dryRun: false,
+          provenanceOperationId,
+        };
+      });
     },
   };
 }
+
+function stagedMediaType(path: string): string | undefined {
+  switch (extname(path).toLowerCase()) {
+    case '.glb':
+      return 'model/gltf-binary';
+    case '.gltf':
+      return 'model/gltf+json';
+    case '.png':
+      return 'image/png';
+    case '.webp':
+      return 'image/webp';
+    case '.jpg':
+    case '.jpeg':
+      return 'image/jpeg';
+    case '.json':
+      return 'application/json';
+    case '.hdr':
+      return 'image/vnd.radiance';
+    default:
+      return undefined;
+  }
+}
+
+function stagedRole(path: string): ProjectOutputFile['role'] {
+  if (path.endsWith('.prefab.json')) return 'prefab';
+  if (path.endsWith('/.vgai-thumbnail.webp')) return 'other';
+  return 'asset';
+}
+
+async function describeStagedDirectory(
+  stagedDirectory: string,
+  destinationProjectPath: string,
+): Promise<HashedProjectFile[]> {
+  const outputs: HashedProjectFile[] = [];
+  async function walk(directory: string): Promise<void> {
+    const entries = await readdir(directory, { withFileTypes: true });
+    for (const entry of entries) {
+      const absolute = resolve(directory, entry.name);
+      if (entry.isDirectory()) {
+        await walk(absolute);
+        continue;
+      }
+      if (!entry.isFile()) {
+        throw new Error(
+          `Staged project output contains an unsupported filesystem entry: ${absolute}`,
+        );
+      }
+      const relativeFile = relative(stagedDirectory, absolute).split(sep).join('/');
+      const path = posix.join(destinationProjectPath, relativeFile);
+      const info = await stat(absolute);
+      const mediaType = stagedMediaType(path);
+      outputs.push({
+        path,
+        bytes: info.size,
+        sha256: await sha256File(absolute),
+        ...(mediaType ? { mediaType } : {}),
+        role: stagedRole(path),
+      });
+    }
+  }
+  await walk(stagedDirectory);
+  outputs.sort((left, right) => left.path.localeCompare(right.path));
+  if (outputs.length === 0) throw new Error('Staged project output directory is empty.');
+  return outputs;
+}
+
+async function pathExists(path: string): Promise<boolean> {
+  try {
+    await stat(path);
+    return true;
+  } catch (error) {
+    if (error instanceof Error && 'code' in error && error.code === 'ENOENT') return false;
+    throw error;
+  }
+}
+
+/**
+ * Commits an already-streamed directory and the central provenance rewrite as
+ * one rollback-safe project transaction. Catalog acquisition uses this path so
+ * large assets never need to be buffered into ProjectOutputFile.content.
+ */
+export async function commitStagedProjectDirectory(options: {
+  projectRoot: string;
+  stagedDirectory: string;
+  destination: string;
+  provenance: ProjectOutputProvenanceContext;
+}): Promise<ProjectGeneratedOutput> {
+  const destinationProjectPath = normalizeOutputPath(
+    relative(options.projectRoot, options.destination).split(sep).join('/'),
+  );
+  const outputs = await describeStagedDirectory(options.stagedDirectory, destinationProjectPath);
+  const summaries: ProjectGeneratedOutputFile[] = outputs.map(
+    ({ sha256: _sha256, ...file }) => file,
+  );
+  const totalBytes = summaries.reduce((total, file) => total + file.bytes, 0);
+
+  return serializeProjectCommit(options.projectRoot, async () => {
+    if (await pathExists(options.destination)) {
+      throw new Error(`Destination already exists and was not overwritten: ${options.destination}`);
+    }
+    const provenanceOperationId = randomUUID();
+    const provenanceFile = await nextProvenanceFile(
+      options.projectRoot,
+      provenanceOperationId,
+      options.provenance,
+      outputs,
+    );
+    const transactionRoot = resolve(
+      options.projectRoot,
+      '.vgai',
+      'tmp',
+      `project-directory-${provenanceOperationId}`,
+    );
+    const stagedRoot = resolve(transactionRoot, 'staged');
+    const backupRoot = resolve(transactionRoot, 'backup');
+    let directoryCommitted = false;
+    try {
+      await stageBatch(stagedRoot, [provenanceFile]);
+      await mkdir(dirname(options.destination), { recursive: true });
+      await rename(options.stagedDirectory, options.destination);
+      directoryCommitted = true;
+      await commitBatch(options.projectRoot, stagedRoot, backupRoot, [provenanceFile]);
+    } catch (error) {
+      if (directoryCommitted) {
+        try {
+          await rename(options.destination, options.stagedDirectory);
+        } catch (rollbackError) {
+          throw new AggregateError(
+            [error, rollbackError],
+            'Project directory transaction and its rollback both failed.',
+          );
+        }
+      }
+      throw error;
+    } finally {
+      await rm(transactionRoot, { recursive: true, force: true });
+    }
+    return {
+      files: summaries,
+      totalBytes,
+      dryRun: false,
+      provenanceOperationId,
+    };
+  });
+}
+
+/** Atomically replaces an imported asset directory and appends its reimport provenance. */
+export async function replaceStagedProjectDirectory(options: {
+  projectRoot: string;
+  stagedDirectory: string;
+  destination: string;
+  provenance: ProjectOutputProvenanceContext;
+}): Promise<ProjectGeneratedOutput> {
+  const destinationProjectPath = normalizeOutputPath(
+    relative(options.projectRoot, options.destination).split(sep).join('/'),
+  );
+  const outputs = await describeStagedDirectory(options.stagedDirectory, destinationProjectPath);
+  const summaries: ProjectGeneratedOutputFile[] = outputs.map(
+    ({ sha256: _sha256, ...file }) => file,
+  );
+  const totalBytes = summaries.reduce((total, file) => total + file.bytes, 0);
+
+  return serializeProjectCommit(options.projectRoot, async () => {
+    if (!(await pathExists(options.destination))) {
+      throw new Error(`Reimport target does not exist: ${options.destination}`);
+    }
+    const provenanceOperationId = randomUUID();
+    const provenanceFile = await nextProvenanceFile(
+      options.projectRoot,
+      provenanceOperationId,
+      options.provenance,
+      outputs,
+    );
+    const transactionRoot = resolve(
+      options.projectRoot,
+      '.vgai',
+      'tmp',
+      `project-reimport-${provenanceOperationId}`,
+    );
+    const stagedRoot = resolve(transactionRoot, 'staged');
+    const backupRoot = resolve(transactionRoot, 'backup');
+    const directoryBackup = resolve(transactionRoot, 'previous-asset');
+    let previousMoved = false;
+    let replacementCommitted = false;
+    try {
+      await stageBatch(stagedRoot, [provenanceFile]);
+      await mkdir(dirname(directoryBackup), { recursive: true });
+      await rename(options.destination, directoryBackup);
+      previousMoved = true;
+      await rename(options.stagedDirectory, options.destination);
+      replacementCommitted = true;
+      await commitBatch(options.projectRoot, stagedRoot, backupRoot, [provenanceFile]);
+    } catch (error) {
+      if (replacementCommitted) await rm(options.destination, { recursive: true, force: true });
+      if (previousMoved) await rename(directoryBackup, options.destination);
+      throw error;
+    } finally {
+      await rm(transactionRoot, { recursive: true, force: true });
+    }
+    return {
+      files: summaries,
+      totalBytes,
+      dryRun: false,
+      provenanceOperationId,
+    };
+  });
+}