@xenonbyte/da-vinci-workflow 0.1.24 → 0.1.26

package/lib/pen-persistence.js

@@ -3,13 +3,13 @@ const os = require("os");
 const path = require("path");
 const crypto = require("crypto");
 const { spawnSync } = require("child_process");
+const { isPlainObject, readJsonFile, writeFileAtomic } = require("./utils");
+const { runModuleExportInWorker } = require("./async-offload");
 
 const DEFAULT_PEN_VERSION = "2.9";
 const DEFAULT_READ_DEPTH = 50;
-
-function isPlainObject(value) {
-  return Boolean(value) && typeof value === "object" && !Array.isArray(value);
-}
+const DEFAULT_PENCIL_INTERACTIVE_MAX_BUFFER = 50 * 1024 * 1024;
+const DEFAULT_PENCIL_INTERACTIVE_TIMEOUT_MS = 2 * 60 * 1000;
 
 function extractFirstJson(text) {
   const source = String(text || "");
@@ -18,10 +18,6 @@ function extractFirstJson(text) {
 }
 
 class JSONDecoder {
-  constructor() {
-    this.decoder = JSON;
-  }
-
   decodeFirst(text) {
     for (let index = 0; index < text.length; index += 1) {
       const ch = text[index];
@@ -44,6 +40,7 @@ class JSONDecoder {
     let end = 0;
     let inString = false;
     let escape = false;
+    let unicodeDigitsRemaining = 0;
     let depth = 0;
     const opening = text[0];
     const closing = opening === "{" ? "}" : "]";
@@ -52,7 +49,15 @@ class JSONDecoder {
       const ch = text[index];
 
       if (inString) {
+        if (unicodeDigitsRemaining > 0) {
+          unicodeDigitsRemaining -= 1;
+          continue;
+        }
+
         if (escape) {
+          if (ch === "u") {
+            unicodeDigitsRemaining = 4;
+          }
           escape = false;
         } else if (ch === "\\") {
           escape = true;
@@ -110,7 +115,52 @@ function hasTruncatedChildren(value) {
   return Object.values(value).some((item) => hasTruncatedChildren(item));
 }
 
+function looksLikePenStatePayload(payload) {
+  if (!isPlainObject(payload)) {
+    return false;
+  }
+
+  const hasLiveNodes = Array.isArray(payload.nodes);
+  if (hasLiveNodes) {
+    return false;
+  }
+
+  const hasSnapshotHash = typeof payload.snapshotHash === "string" && payload.snapshotHash.length > 0;
+  const hasPenPath = typeof payload.penPath === "string" && payload.penPath.length > 0;
+  const hasStateCounters =
+    Object.prototype.hasOwnProperty.call(payload, "topLevelCount") ||
+    Object.prototype.hasOwnProperty.call(payload, "topLevelIds");
+  return hasSnapshotHash && hasPenPath && hasStateCounters;
+}
+
+function assertNodesPayloadIsLiveSnapshot(payload, sourcePath) {
+  if (!looksLikePenStatePayload(payload)) {
+    return;
+  }
+
+  const location = sourcePath ? ` (${formatPathForError(sourcePath)})` : "";
+  throw new Error(
+    `\`--nodes-file\` payload${location} appears to be pen state metadata, not a live MCP nodes snapshot. ` +
+      "Use JSON from `batch_get` (an array or an object with `nodes`)."
+  );
+}
+
+function formatPathForError(sourcePath) {
+  const resolved = path.resolve(sourcePath);
+  const relative = path.relative(process.cwd(), resolved);
+  if (!relative || relative === "") {
+    return ".";
+  }
+
+  if (!relative.startsWith("..") && !path.isAbsolute(relative)) {
+    return relative;
+  }
+
+  return resolved;
+}
+
 function normalizeNodesPayload(payload) {
+  assertNodesPayloadIsLiveSnapshot(payload);
   const nodes = getNodesArray(payload);
 
   if (!nodes) {
@@ -207,29 +257,13 @@ function getStandardPenStatePath(outputPath) {
 
 function writePenDocumentAtomic(outputPath, document) {
   const targetPath = path.resolve(outputPath);
-  const targetDir = path.dirname(targetPath);
-  const tempPath = path.join(
-    targetDir,
-    `.${path.basename(targetPath)}.tmp-${process.pid}-${Date.now()}`
-  );
-
-  fs.mkdirSync(targetDir, { recursive: true });
-  fs.writeFileSync(tempPath, JSON.stringify(document, null, 2) + "\n", "utf8");
-  fs.renameSync(tempPath, targetPath);
+  writeFileAtomic(targetPath, JSON.stringify(document, null, 2) + "\n");
   return targetPath;
 }
 
 function writeJsonFileAtomic(outputPath, payload) {
   const targetPath = path.resolve(outputPath);
-  const targetDir = path.dirname(targetPath);
-  const tempPath = path.join(
-    targetDir,
-    `.${path.basename(targetPath)}.tmp-${process.pid}-${Date.now()}`
-  );
-
-  fs.mkdirSync(targetDir, { recursive: true });
-  fs.writeFileSync(tempPath, JSON.stringify(payload, null, 2) + "\n", "utf8");
-  fs.renameSync(tempPath, targetPath);
+  writeFileAtomic(targetPath, JSON.stringify(payload, null, 2) + "\n");
   return targetPath;
 }
 
@@ -262,42 +296,88 @@ function readPenState(outputPath) {
 
 function runPencilInteractive(inputPath, commands, options = {}) {
   const pencilBin = options.pencilBin || "pencil";
+  const configuredMaxBuffer = Number(options.maxBuffer);
+  const configuredTimeoutMs = Number(options.pencilTimeoutMs || options.timeoutMs);
+  const maxBuffer =
+    Number.isFinite(configuredMaxBuffer) && configuredMaxBuffer > 0
+      ? configuredMaxBuffer
+      : DEFAULT_PENCIL_INTERACTIVE_MAX_BUFFER;
+  const timeoutMs =
+    Number.isFinite(configuredTimeoutMs) && configuredTimeoutMs > 0
+      ? configuredTimeoutMs
+      : DEFAULT_PENCIL_INTERACTIVE_TIMEOUT_MS;
   const unusedOutput = path.join(
     os.tmpdir(),
     `da-vinci-verify-${process.pid}-${Date.now()}-${Math.random().toString(36).slice(2)}.pen`
   );
   const payload = Array.isArray(commands) ? commands.join("\n") : String(commands || "");
-  const result = spawnSync(
-    pencilBin,
-    ["interactive", "-i", path.resolve(inputPath), "-o", unusedOutput],
-    {
+  let result;
+  try {
+    result = spawnSync(pencilBin, ["interactive", "-i", path.resolve(inputPath), "-o", unusedOutput], {
       input: `${payload}\nexit()\n`,
       encoding: "utf8",
-      maxBuffer: 8 * 1024 * 1024
-    }
-  );
-
-  fs.rmSync(unusedOutput, { force: true });
+      maxBuffer,
+      timeout: timeoutMs
+    });
+  } finally {
+    fs.rmSync(unusedOutput, { force: true });
+  }
 
   const stdout = result.stdout || "";
   const stderr = result.stderr || "";
   if (result.status !== 0) {
+    const errorMessage =
+      result.error && result.error.message
+        ? `\n${result.error.message}`
+        : "";
     throw new Error(
-      `Pencil interactive failed for ${inputPath}.\n${stdout}${stderr}`.trim()
+      `Pencil interactive failed for ${inputPath}.${errorMessage}\n${stdout}${stderr}`.trim()
     );
   }
 
   return `${stdout}${stderr}`;
 }
 
+function formatPencilOutputPreview(output, maxChars = 2400) {
+  const text = String(output || "").trim();
+  if (!text) {
+    return "(no stdout/stderr output)";
+  }
+
+  if (text.length <= maxChars) {
+    return text;
+  }
+
+  const headLength = Math.floor(maxChars / 2);
+  const tailLength = maxChars - headLength;
+  return `${text.slice(0, headLength)}\n...\n${text.slice(text.length - tailLength)}`;
+}
+
 function verifyPenFileWithPencil(filePath, options = {}) {
   const output = runPencilInteractive(filePath, ['batch_get({ readDepth: 1 })'], options);
-  const payload = extractFirstJson(output);
+  let payload;
+  try {
+    payload = extractFirstJson(output);
+  } catch (error) {
+    throw new Error(
+      [
+        `Pencil reopen verification could not parse JSON for ${filePath}: ${error.message || String(error)}`,
+        "Pencil output preview:",
+        formatPencilOutputPreview(output)
+      ].join("\n")
+    );
+  }
   const nodes = getNodesArray(payload);
   const expectedTopLevelIds = options.expectedTopLevelIds || [];
 
   if (!nodes) {
-    throw new Error("Pencil reopen verification did not return a `nodes` array.");
+    throw new Error(
+      [
+        `Pencil reopen verification did not return a \`nodes\` array for ${filePath}.`,
+        "Pencil output preview:",
+        formatPencilOutputPreview(output)
+      ].join("\n")
+    );
   }
 
   if (expectedTopLevelIds.length > 0) {
@@ -318,6 +398,7 @@ function verifyPenFileWithPencil(filePath, options = {}) {
 function writePenFromPayloadFiles(options) {
   const outputPath = path.resolve(options.outputPath);
   const nodes = readJsonPayload(options.nodesFile);
+  assertNodesPayloadIsLiveSnapshot(nodes, options.nodesFile);
   const variables = options.variablesFile ? readJsonPayload(options.variablesFile) : undefined;
   const document = buildPenDocument({
     version: options.version || DEFAULT_PEN_VERSION,
@@ -346,7 +427,7 @@ function writePenFromPayloadFiles(options) {
 }
 
 function readPenVersion(inputPath) {
-  const payload = JSON.parse(fs.readFileSync(inputPath, "utf8"));
+  const payload = readJsonFile(inputPath, `.pen document JSON at ${path.resolve(inputPath)}`);
   return payload.version || DEFAULT_PEN_VERSION;
 }
 
@@ -402,6 +483,201 @@ function readPenDocument(inputPath) {
   return readJsonPayload(inputPath);
 }
 
+function assertPenDocumentShape(document, inputPath) {
+  if (!isPlainObject(document) || !Array.isArray(document.children)) {
+    const location = inputPath ? `: ${path.resolve(inputPath)}` : "";
+    throw new Error(`Invalid .pen document (missing \`children\` array)${location}`);
+  }
+  return document;
+}
+
+function assertPenPathReadable(penPath) {
+  const resolvedPath = path.resolve(penPath);
+
+  if (!fs.existsSync(resolvedPath)) {
+    throw new Error(`.pen file does not exist: ${resolvedPath}`);
+  }
+
+  const stat = fs.statSync(resolvedPath);
+  if (!stat.isFile()) {
+    throw new Error(`.pen path is not a file: ${resolvedPath}`);
+  }
+
+  return {
+    resolvedPath,
+    sizeBytes: stat.size
+  };
+}
+
+function hashFileSha256(filePath) {
+  return crypto.createHash("sha256").update(fs.readFileSync(filePath)).digest("hex");
+}
+
+function inspectPenSource(penPath) {
+  const { resolvedPath, sizeBytes } = assertPenPathReadable(penPath);
+  const document = assertPenDocumentShape(readPenDocument(resolvedPath), resolvedPath);
+  const snapshotHash = hashPenDocument(document);
+  const state = readPenState(resolvedPath);
+  const stateHash =
+    state && typeof state.snapshotHash === "string" && state.snapshotHash.trim()
+      ? String(state.snapshotHash).trim()
+      : "";
+
+  return {
+    path: resolvedPath,
+    sizeBytes,
+    fileSha256: hashFileSha256(resolvedPath),
+    snapshotHash,
+    statePath: getStandardPenStatePath(resolvedPath),
+    stateHash: stateHash || null,
+    stateMatchesSnapshot: stateHash ? stateHash === snapshotHash : null
+  };
+}
+
+function normalizePenPathList(values) {
+  const list = Array.isArray(values) ? values : [];
+  const normalized = [];
+  const seen = new Set();
+
+  for (const value of list) {
+    if (!value) {
+      continue;
+    }
+    const resolvedPath = path.resolve(String(value));
+    if (seen.has(resolvedPath)) {
+      continue;
+    }
+    seen.add(resolvedPath);
+    normalized.push(resolvedPath);
+  }
+
+  return normalized;
+}
+
+function comparePenBaselineAlignment(options) {
+  if (!options.penPath) {
+    throw new Error("A primary `.pen` path is required for baseline alignment checks.");
+  }
+  const penPath = path.resolve(options.penPath);
+
+  const baselinePaths = normalizePenPathList(options.baselinePaths).filter(
+    (candidatePath) => candidatePath !== penPath
+  );
+  if (baselinePaths.length === 0) {
+    throw new Error("Baseline alignment requires at least one comparison `.pen` path.");
+  }
+
+  const preferredSourcePath = options.preferredSource
+    ? path.resolve(options.preferredSource)
+    : null;
+  const knownPaths = new Set([penPath, ...baselinePaths]);
+  if (preferredSourcePath && !knownPaths.has(preferredSourcePath)) {
+    throw new Error(
+      `Preferred source is not part of the compared paths. Add it as \`--pen\` or \`--baseline\`: ${preferredSourcePath}`
+    );
+  }
+
+  const primary = inspectPenSource(penPath);
+  const baselines = baselinePaths.map((candidatePath) => {
+    const source = inspectPenSource(candidatePath);
+    return {
+      ...source,
+      matchesPrimary: source.snapshotHash === primary.snapshotHash
+    };
+  });
+  const inSync = baselines.every((source) => source.matchesPrimary);
+
+  let status = "PASS";
+  let decision = "aligned";
+  let message = "All compared `.pen` sources match the project-local baseline hash.";
+
+  if (!inSync) {
+    if (!preferredSourcePath) {
+      status = "BLOCK";
+      decision = "diverged";
+      message =
+        "Compared `.pen` sources diverged. Confirm source priority before continuing (`--prefer-source`).";
+    } else if (preferredSourcePath === penPath) {
+      status = "WARN";
+      decision = "diverged_prefer_pen";
+      message =
+        "Compared `.pen` sources diverged, but source priority explicitly keeps the project-local `.pen` as baseline.";
+    } else {
+      status = "BLOCK";
+      decision = "diverged_prefer_external";
+      message =
+        "Compared `.pen` sources diverged and preferred source points to a non-project file. Sync preferred source into the project path before continuing.";
+    }
+  }
+
+  return {
+    penPath,
+    baselinePaths,
+    preferredSourcePath,
+    primary,
+    baselines,
+    inSync,
+    status,
+    decision,
+    message
+  };
+}
+
+function formatPenBaselineAlignmentReport(result) {
+  const lines = [
+    `Primary .pen: ${result.primary.path}`,
+    `Primary snapshot hash: ${result.primary.snapshotHash}`,
+    `Primary file sha256: ${result.primary.fileSha256}`,
+    `Primary size: ${result.primary.sizeBytes} bytes`
+  ];
+
+  for (const source of result.baselines) {
+    lines.push("");
+    lines.push(`Baseline: ${source.path}`);
+    lines.push(`- Snapshot hash: ${source.snapshotHash}`);
+    lines.push(`- File sha256: ${source.fileSha256}`);
+    lines.push(`- Size: ${source.sizeBytes} bytes`);
+    lines.push(`- Matches primary: ${source.matchesPrimary ? "yes" : "no"}`);
+  }
+
+  lines.push("");
+  lines.push(`Status: ${result.status}`);
+  lines.push(`Decision: ${result.decision}`);
+  lines.push(`Summary: ${result.message}`);
+  if (result.preferredSourcePath) {
+    lines.push(`Preferred source: ${result.preferredSourcePath}`);
+  }
+
+  return lines.join("\n");
+}
+
+function syncPenSource(options) {
+  if (!options.sourcePath || !options.targetPath) {
+    throw new Error("Both source and target `.pen` paths are required for source sync.");
+  }
+  const sourcePath = path.resolve(options.sourcePath);
+  const targetPath = path.resolve(options.targetPath);
+
+  const sourceInfo = inspectPenSource(sourcePath);
+  const sourceDocument = assertPenDocumentShape(readPenDocument(sourcePath), sourcePath);
+  writePenDocumentAtomic(targetPath, sourceDocument);
+  const state = buildPenState(targetPath, sourceDocument, {
+    source: options.stateSource || "sync-pen-source"
+  });
+  const statePath = writePenStateAtomic(targetPath, state);
+
+  const targetInfo = inspectPenSource(targetPath);
+
+  return {
+    sourcePath,
+    targetPath,
+    sourceInfo,
+    targetInfo,
+    state,
+    statePath
+  };
+}
+
 function ensurePenFile(options) {
   const outputPath = path.resolve(options.outputPath);
   const exists = fs.existsSync(outputPath);
@@ -416,7 +692,7 @@ function ensurePenFile(options) {
   } else {
     document = buildPenDocument({
       version: options.version || DEFAULT_PEN_VERSION,
-      nodes: { nodes: [] },
+      nodes: [],
       variables: {}
     });
     writePenDocumentAtomic(outputPath, document);
@@ -447,6 +723,7 @@ function ensurePenFile(options) {
 
 function hashPayloadFiles(options) {
   const nodes = readJsonPayload(options.nodesFile);
+  assertNodesPayloadIsLiveSnapshot(nodes, options.nodesFile);
   const variables = options.variablesFile ? readJsonPayload(options.variablesFile) : undefined;
   const document = buildPenDocument({
     version: options.version || DEFAULT_PEN_VERSION,
@@ -466,30 +743,61 @@ function comparePenSync(options) {
     throw new Error(`Registered .pen file does not exist: ${penPath}`);
   }
 
-  const persistedDocument = readPenDocument(penPath);
+  const persistedDocument = assertPenDocumentShape(readPenDocument(penPath), penPath);
+  const diskHash = hashPenDocument(persistedDocument);
   const live = hashPayloadFiles({
     ...options,
     version: options.version || persistedDocument.version || DEFAULT_PEN_VERSION
   });
   const persistedState = readPenState(penPath);
-  const persistedHash = persistedState && persistedState.snapshotHash
-    ? String(persistedState.snapshotHash)
-    : hashPenDocument(persistedDocument);
+  const stateHash =
+    persistedState && persistedState.snapshotHash ? String(persistedState.snapshotHash).trim() : "";
+  const stateMatchesDisk = stateHash ? stateHash === diskHash : null;
+  const persistedHash = stateHash && stateMatchesDisk ? stateHash : diskHash;
 
   return {
     penPath,
     statePath: getStandardPenStatePath(penPath),
+    diskHash,
+    stateHash: stateHash || null,
+    stateMatchesDisk,
     persistedHash,
     liveHash: live.snapshotHash,
     inSync: persistedHash === live.snapshotHash,
-    usedStateFile: Boolean(persistedState && persistedState.snapshotHash),
+    usedStateFile: Boolean(stateHash && stateMatchesDisk),
     state: persistedState
   };
 }
 
+function writePenFromPayloadFilesAsync(options) {
+  return runModuleExportInWorker(__filename, "writePenFromPayloadFiles", [options]);
+}
+
+function snapshotPenFileAsync(options) {
+  return runModuleExportInWorker(__filename, "snapshotPenFile", [options]);
+}
+
+function ensurePenFileAsync(options) {
+  return runModuleExportInWorker(__filename, "ensurePenFile", [options]);
+}
+
+function comparePenSyncAsync(options) {
+  return runModuleExportInWorker(__filename, "comparePenSync", [options]);
+}
+
+function comparePenBaselineAlignmentAsync(options) {
+  return runModuleExportInWorker(__filename, "comparePenBaselineAlignment", [options]);
+}
+
+function syncPenSourceAsync(options) {
+  return runModuleExportInWorker(__filename, "syncPenSource", [options]);
+}
+
 module.exports = {
   DEFAULT_PEN_VERSION,
   DEFAULT_READ_DEPTH,
+  DEFAULT_PENCIL_INTERACTIVE_MAX_BUFFER,
+  DEFAULT_PENCIL_INTERACTIVE_TIMEOUT_MS,
   extractFirstJson,
   readJsonPayload,
   normalizeNodesPayload,
@@ -506,10 +814,21 @@ module.exports = {
   runPencilInteractive,
   verifyPenFileWithPencil,
   writePenFromPayloadFiles,
+  writePenFromPayloadFilesAsync,
   capturePenSnapshot,
   snapshotPenFile,
+  snapshotPenFileAsync,
   readPenDocument,
+  assertPenDocumentShape,
   ensurePenFile,
+  ensurePenFileAsync,
   hashPayloadFiles,
-  comparePenSync
+  comparePenSync,
+  comparePenSyncAsync,
+  inspectPenSource,
+  comparePenBaselineAlignment,
+  comparePenBaselineAlignmentAsync,
+  formatPenBaselineAlignmentReport,
+  syncPenSource,
+  syncPenSourceAsync
 };