@hassangameryt/freeflow 0.3.0 → 0.5.0

package/router/dist/processing/engine.js

@@ -1,435 +1,533 @@
 import { createHash } from "node:crypto";
 import { readFile, stat } from "node:fs/promises";
 import { relative } from "node:path";
+import { resolveLocalPath } from "../local/local-traversal.js";
 import { resolveRepoPath } from "../repo/repo-traversal.js";
 import { renderProcessingResult, classifyProcessingRecovery } from "./renderers.js";
 import { selectProcessingReducer } from "./reducers.js";
-import { processingScriptNotConfigured, processingScriptUnavailableForUnloadedSource, runProcessingScript, } from "./scripts.js";
-import { createVault, readOutputText, readVaultRecord, storeRepoFileReference, storeTextOutput } from "../vault/vault.js";
+import {
+  processingScriptNotConfigured,
+  processingScriptUnavailableForUnloadedSource,
+  runProcessingScript,
+} from "./scripts.js";
+import {
+  createVault,
+  readOutputText,
+  readVaultRecord,
+  storeRepoFileReference,
+  storeTextOutput,
+} from "../vault/vault.js";
 export const PROCESSING_ENGINE_IMPLEMENTATION = "processing-engine-skeleton-v1";
 const DEFAULT_PROCESSING_LIMITS = {
-    maxSourceBytes: 2 * 1024 * 1024,
-    maxVisibleBytes: 4_096,
+  maxSourceBytes: 2 * 1024 * 1024,
+  maxVisibleBytes: 4_096,
 };
 export async function loadProcessingSource(source, options = {}) {
-    const limits = normalizeLimits(options.limits);
-    switch (source.kind) {
-        case "repo-file":
-            return loadRepoFileSource(source, options, limits);
-        case "vault-output":
-            return loadVaultOutputSource(source, options, limits);
-        case "command-output":
-            return loadCapturedCommandOutputSource(source, limits);
-    }
+  const limits = normalizeLimits(options.limits);
+  switch (source.kind) {
+    case "repo-file":
+      return loadRepoFileSource(source, options, limits);
+    case "vault-output":
+      return loadVaultOutputSource(source, options, limits);
+    case "local-file":
+      return loadLocalFileSource(source, limits);
+    case "command-output":
+      return loadCapturedCommandOutputSource(source, limits);
+  }
 }
 export async function processSource(source, options = {}) {
-    const loaded = await loadProcessingSource(source, options);
-    const limits = normalizeLimits(options.limits);
-    const reducer = notSelectedReducer("Source was not loaded; reducer selection was skipped.");
-    const script = options.script
-        ? processingScriptUnavailableForUnloadedSource(options.script, "Source was not loaded; script processing was skipped.")
-        : processingScriptNotConfigured();
-    if (loaded.status === "blocked") {
-        return {
-            implementation: PROCESSING_ENGINE_IMPLEMENTATION,
-            status: "blocked",
-            source: loaded.source,
-            reason: loaded.reason,
-            policy: loaded.policy,
-            visibleText: renderProcessingResult({
-                status: "blocked",
-                source: loaded.source,
-                facts: [],
-                maxVisibleBytes: limits.maxVisibleBytes,
-                failure: { policy: loaded.policy, reason: loaded.reason },
-            }),
-            facts: [],
-            reducer,
-            script,
-        };
-    }
-    if (loaded.status === "unavailable") {
-        return {
-            implementation: PROCESSING_ENGINE_IMPLEMENTATION,
-            status: "unavailable",
-            source: loaded.source,
-            reason: loaded.reason,
-            visibleText: renderProcessingResult({
-                status: "unavailable",
-                source: loaded.source,
-                facts: [],
-                maxVisibleBytes: limits.maxVisibleBytes,
-                failure: { reason: loaded.reason },
-            }),
-            facts: [],
-            reducer,
-            script,
-        };
-    }
-    const selectedScript = options.script
-        ? await runProcessingScript({
-            loaded,
-            script: options.script,
-            ...(options.scriptTransform !== undefined ? { scriptTransform: options.scriptTransform } : {}),
-            ...(options.localConfig !== undefined ? { localConfig: options.localConfig } : {}),
-            ...(options.scriptSandboxAdapters !== undefined ? { adapters: options.scriptSandboxAdapters } : {}),
-        })
-        : processingScriptNotConfigured();
-    const selectedReducer = scriptSkipsReducer(selectedScript)
-        ? notSelectedReducer(scriptReducerSkipReason(selectedScript))
-        : selectProcessingReducer({ text: loaded.text, ...(options.goal !== undefined ? { goal: options.goal } : {}) });
-    const facts = selectedReducer.status === "selected" ? [...selectedReducer.result.facts, ...sourceFacts(loaded)] : sourceFacts(loaded);
-    const visibleText = renderProcessingResult({
-        status: "ok",
+  const loaded = await loadProcessingSource(source, options);
+  const limits = normalizeLimits(options.limits);
+  const reducer = notSelectedReducer("Source was not loaded; reducer selection was skipped.");
+  const script = options.script
+    ? processingScriptUnavailableForUnloadedSource(
+        options.script,
+        "Source was not loaded; script processing was skipped.",
+      )
+    : processingScriptNotConfigured();
+  if (loaded.status === "blocked") {
+    return {
+      implementation: PROCESSING_ENGINE_IMPLEMENTATION,
+      status: "blocked",
+      source: loaded.source,
+      reason: loaded.reason,
+      policy: loaded.policy,
+      visibleText: renderProcessingResult({
+        status: "blocked",
         source: loaded.source,
-        stats: loaded.stats,
-        facts,
-        reducer: selectedReducer,
-        script: selectedScript,
-        ...(loaded.recovery !== undefined ? { recovery: loaded.recovery } : {}),
-        ...(loaded.persistence !== undefined ? { persistence: loaded.persistence } : {}),
-        recoveryClass: classifyProcessingRecovery({
-            ...(loaded.recovery !== undefined ? { recovery: loaded.recovery } : {}),
-            ...(loaded.persistence !== undefined ? { persistence: loaded.persistence } : {}),
-            resultWillBePersisted: options.sessionId !== undefined,
-        }),
+        facts: [],
         maxVisibleBytes: limits.maxVisibleBytes,
-    });
-    const persisted = await persistProcessingResultText({
-        resultText: selectedScript.status === "executed" ? selectedScript.outputText : visibleText,
-        loaded,
-        options,
-        operation: selectedScript.status === "executed" ? `processing-script:${selectedScript.language}` : "processing-fact-summary",
-        producerName: selectedScript.status === "executed" ? `processing-script:${selectedScript.language}` : "processing-engine",
-        operationHashSeed: selectedScript.status === "executed"
-            ? { source: loaded.source, stats: loaded.stats, script: { language: selectedScript.language, codeHashSha256: selectedScript.codeHashSha256 } }
-            : { source: loaded.source, stats: loaded.stats },
-    });
-    const result = {
-        implementation: PROCESSING_ENGINE_IMPLEMENTATION,
-        status: "ok",
+        failure: { policy: loaded.policy, reason: loaded.reason },
+      }),
+      facts: [],
+      reducer,
+      script,
+    };
+  }
+  if (loaded.status === "unavailable") {
+    return {
+      implementation: PROCESSING_ENGINE_IMPLEMENTATION,
+      status: "unavailable",
+      source: loaded.source,
+      reason: loaded.reason,
+      visibleText: renderProcessingResult({
+        status: "unavailable",
         source: loaded.source,
-        stats: loaded.stats,
-        visibleText,
-        facts,
-        reducer: selectedReducer,
-        script: selectedScript,
+        facts: [],
+        maxVisibleBytes: limits.maxVisibleBytes,
+        failure: { reason: loaded.reason },
+      }),
+      facts: [],
+      reducer,
+      script,
     };
-    const lineage = persisted.lineage ?? loaded.lineage;
-    const persistence = persisted.persistence ?? loaded.persistence;
-    const recovery = persisted.recovery ?? loaded.recovery;
-    if (lineage !== undefined) {
-        result.lineage = lineage;
-    }
-    if (persistence !== undefined) {
-        result.persistence = persistence;
-    }
-    if (recovery !== undefined) {
-        result.recovery = recovery;
-    }
-    return result;
+  }
+  const selectedScript = options.script
+    ? await runProcessingScript({
+        loaded,
+        script: options.script,
+        ...(options.scriptTransform !== undefined ? { scriptTransform: options.scriptTransform } : {}),
+        ...(options.localConfig !== undefined ? { localConfig: options.localConfig } : {}),
+        ...(options.scriptSandboxAdapters !== undefined ? { adapters: options.scriptSandboxAdapters } : {}),
+      })
+    : processingScriptNotConfigured();
+  const selectedReducer = scriptSkipsReducer(selectedScript)
+    ? notSelectedReducer(scriptReducerSkipReason(selectedScript))
+    : selectProcessingReducer({ text: loaded.text, ...(options.goal !== undefined ? { goal: options.goal } : {}) });
+  const facts =
+    selectedReducer.status === "selected"
+      ? [...selectedReducer.result.facts, ...sourceFacts(loaded)]
+      : sourceFacts(loaded);
+  const visibleText = renderProcessingResult({
+    status: "ok",
+    source: loaded.source,
+    stats: loaded.stats,
+    facts,
+    reducer: selectedReducer,
+    script: selectedScript,
+    ...(loaded.recovery !== undefined ? { recovery: loaded.recovery } : {}),
+    ...(loaded.persistence !== undefined ? { persistence: loaded.persistence } : {}),
+    recoveryClass: classifyProcessingRecovery({
+      ...(loaded.recovery !== undefined ? { recovery: loaded.recovery } : {}),
+      ...(loaded.persistence !== undefined ? { persistence: loaded.persistence } : {}),
+      resultWillBePersisted: options.sessionId !== undefined,
+    }),
+    maxVisibleBytes: limits.maxVisibleBytes,
+  });
+  const persisted = await persistProcessingResultText({
+    resultText: selectedScript.status === "executed" ? selectedScript.outputText : visibleText,
+    loaded,
+    options,
+    operation:
+      selectedScript.status === "executed" ? `processing-script:${selectedScript.language}` : "processing-fact-summary",
+    producerName:
+      selectedScript.status === "executed" ? `processing-script:${selectedScript.language}` : "processing-engine",
+    operationHashSeed:
+      selectedScript.status === "executed"
+        ? {
+            source: loaded.source,
+            stats: loaded.stats,
+            script: { language: selectedScript.language, codeHashSha256: selectedScript.codeHashSha256 },
+          }
+        : { source: loaded.source, stats: loaded.stats },
+  });
+  const result = {
+    implementation: PROCESSING_ENGINE_IMPLEMENTATION,
+    status: "ok",
+    source: loaded.source,
+    stats: loaded.stats,
+    visibleText,
+    facts,
+    reducer: selectedReducer,
+    script: selectedScript,
+  };
+  const lineage = persisted.lineage ?? loaded.lineage;
+  const persistence = persisted.persistence ?? loaded.persistence;
+  const recovery = persisted.recovery ?? loaded.recovery;
+  if (lineage !== undefined) {
+    result.lineage = lineage;
+  }
+  if (persistence !== undefined) {
+    result.persistence = persistence;
+  }
+  if (recovery !== undefined) {
+    result.recovery = recovery;
+  }
+  return result;
 }
 async function loadRepoFileSource(source, options, limits) {
-    const descriptor = repoSourceDescriptor(source.path);
-    let resolved;
-    try {
-        resolved = await resolveRepoPath(source.root, source.path);
-    }
-    catch (error) {
-        const message = errorMessage(error);
-        const blocked = message.includes("escapes root");
-        return blocked
-            ? { status: "blocked", source: descriptor, policy: "repo_containment", reason: message }
-            : { status: "unavailable", source: descriptor, reason: message };
-    }
-    const resolvedDescriptor = repoSourceDescriptor(resolved.relativePath || ".");
-    const fileStat = await stat(resolved.absolutePath);
-    if (!fileStat.isFile()) {
-        return { status: "unavailable", source: resolvedDescriptor, reason: `Repo source is not a file: ${resolved.relativePath || "."}` };
-    }
-    if (fileStat.size > limits.maxSourceBytes) {
-        return {
-            status: "blocked",
-            source: resolvedDescriptor,
-            policy: "source_limit",
-            reason: `Repo source ${resolved.relativePath} is ${fileStat.size} bytes, above maxSourceBytes=${limits.maxSourceBytes}.`,
-            stats: { bytes: fileStat.size },
-        };
-    }
-    const text = await readFile(resolved.absolutePath, "utf8");
-    const stats = textStats(text);
-    const lineage = await persistRepoFileReferenceIfRequested({ sourcePath: resolved.relativePath, text, options });
+  const descriptor = repoSourceDescriptor(source.path);
+  let resolved;
+  try {
+    resolved = await resolveRepoPath(source.root, source.path);
+  } catch (error) {
+    const message = errorMessage(error);
+    const blocked = message.includes("escapes root");
+    return blocked
+      ? { status: "blocked", source: descriptor, policy: "repo_containment", reason: message }
+      : { status: "unavailable", source: descriptor, reason: message };
+  }
+  const resolvedDescriptor = repoSourceDescriptor(resolved.relativePath || ".");
+  const fileStat = await stat(resolved.absolutePath);
+  if (!fileStat.isFile()) {
     return {
-        status: "ok",
-        source: resolvedDescriptor,
-        text,
-        stats,
-        ...(lineage !== undefined ? { lineage } : {}),
+      status: "unavailable",
+      source: resolvedDescriptor,
+      reason: `Repo source is not a file: ${resolved.relativePath || "."}`,
     };
-}
-async function loadVaultOutputSource(source, options, limits) {
-    const requestedStream = source.stream;
-    const unavailableDescriptor = vaultSourceDescriptor(source.outputId, requestedStream ?? "combined");
-    const vaultOptions = {};
-    const vaultRoot = source.vaultRoot ?? options.vaultRoot;
-    if (vaultRoot !== undefined) {
-        vaultOptions.root = vaultRoot;
-    }
-    if (options.vaultRetention !== undefined) {
-        vaultOptions.retention = options.vaultRetention;
-    }
-    const vault = createVault(vaultOptions);
-    let record;
-    try {
-        record = await readVaultRecord(vault, source.sessionId, source.outputId);
-    }
-    catch (error) {
-        return { status: "unavailable", source: unavailableDescriptor, reason: errorMessage(error) };
-    }
-    const stream = resolveProcessingVaultStream(record, requestedStream);
-    const descriptor = vaultSourceDescriptor(source.outputId, stream);
-    const sourceBytes = byteCountForVaultStream(record, stream);
-    if (sourceBytes !== undefined && sourceBytes > limits.maxSourceBytes) {
-        return {
-            status: "blocked",
-            source: descriptor,
-            policy: "source_limit",
-            reason: `Vault source ${source.outputId}:${stream} is ${sourceBytes} bytes, above maxSourceBytes=${limits.maxSourceBytes}.`,
-            stats: { bytes: sourceBytes },
-        };
-    }
-    try {
-        const text = await readOutputText(vault, source.sessionId, source.outputId, stream);
-        const stats = textStats(text);
-        if (stats.bytes > limits.maxSourceBytes) {
-            return {
-                status: "blocked",
-                source: descriptor,
-                policy: "source_limit",
-                reason: `Vault source ${source.outputId}:${stream} is ${stats.bytes} bytes, above maxSourceBytes=${limits.maxSourceBytes}.`,
-                stats: { bytes: stats.bytes },
-            };
-        }
-        const lineage = {
-            sourceRecordIds: record.recordId ? [record.recordId] : [],
-            sourceOutputIds: [source.outputId],
-            operation: "processing-source-load",
-        };
-        return {
-            status: "ok",
-            source: descriptor,
-            text,
-            stats,
-            lineage,
-            recovery: { how: `Recover source with freeflow_search action=retrieve outputId=${source.outputId} stream=${stream}.`, outputId: source.outputId },
-        };
-    }
-    catch (error) {
-        return { status: "unavailable", source: descriptor, reason: errorMessage(error) };
-    }
-}
-function resolveProcessingVaultStream(record, requested) {
-    if (requested !== undefined) {
-        return requested;
-    }
-    return record.kind === "text" ? "raw" : "combined";
-}
-function vaultSourceDescriptor(outputId, stream) {
+  }
+  if (fileStat.size > limits.maxSourceBytes) {
     return {
-        kind: "vault-output",
-        ref: { kind: "vault", outputId, stream },
-        displayPath: `${outputId}:${stream}`,
-        stream,
+      status: "blocked",
+      source: resolvedDescriptor,
+      policy: "source_limit",
+      reason: `Repo source ${resolved.relativePath} is ${fileStat.size} bytes, above maxSourceBytes=${limits.maxSourceBytes}.`,
+      stats: { bytes: fileStat.size },
     };
+  }
+  const text = await readFile(resolved.absolutePath, "utf8");
+  const stats = textStats(text);
+  const lineage = await persistRepoFileReferenceIfRequested({ sourcePath: resolved.relativePath, text, options });
+  return {
+    status: "ok",
+    source: resolvedDescriptor,
+    text,
+    stats,
+    ...(lineage !== undefined ? { lineage } : {}),
+  };
 }
-function loadCapturedCommandOutputSource(source, limits) {
-    const stream = source.stream ?? "combined";
-    const descriptor = {
-        kind: "command-output",
-        ref: { kind: "native", tool: "command", outputId: source.outputId ?? "inline-command-output" },
-        displayPath: source.outputId ? `${source.outputId}:${stream}` : `inline-command-output:${stream}`,
-        stream,
+async function loadLocalFileSource(source, limits) {
+  const descriptor = localSourceDescriptor(source.root, source.path);
+  let resolved;
+  try {
+    resolved = await resolveLocalPath(source.root, source.path);
+  } catch (error) {
+    const message = errorMessage(error);
+    const blocked =
+      message.includes("escapes root") ||
+      message.includes("absolute path") ||
+      message.includes("too broad") ||
+      message.includes("source.root");
+    return blocked
+      ? { status: "blocked", source: descriptor, policy: "repo_containment", reason: message }
+      : { status: "unavailable", source: descriptor, reason: message };
+  }
+  const resolvedDescriptor = localSourceDescriptor(resolved.root, resolved.relativePath || ".");
+  const fileStat = await stat(resolved.absolutePath);
+  if (!fileStat.isFile()) {
+    return {
+      status: "unavailable",
+      source: resolvedDescriptor,
+      reason: `Local source is not a file: ${resolved.relativePath || "."}`,
     };
-    const text = commandOutputStreamText(source, stream);
-    const stats = textStats(text);
-    if (stats.bytes > limits.maxSourceBytes) {
-        return {
-            status: "blocked",
-            source: descriptor,
-            policy: "source_limit",
-            reason: `Captured command output ${descriptor.displayPath} is ${stats.bytes} bytes, above maxSourceBytes=${limits.maxSourceBytes}.`,
-            stats: { bytes: stats.bytes },
-        };
-    }
-    const lineage = source.outputId
-        ? { sourceOutputIds: [source.outputId], operation: "processing-source-load" }
-        : undefined;
+  }
+  if (fileStat.size > limits.maxSourceBytes) {
     return {
-        status: "ok",
-        source: descriptor,
-        text,
-        stats,
-        ...(lineage !== undefined ? { lineage } : {}),
-        ...(source.outputId !== undefined
-            ? { recovery: { how: `Recover source from captured command outputId=${source.outputId} stream=${stream}.`, outputId: source.outputId } }
-            : {}),
+      status: "blocked",
+      source: resolvedDescriptor,
+      policy: "source_limit",
+      reason: `Local source ${resolved.relativePath} is ${fileStat.size} bytes, above maxSourceBytes=${limits.maxSourceBytes}.`,
+      stats: { bytes: fileStat.size },
     };
+  }
+  const text = await readFile(resolved.absolutePath, "utf8");
+  const stats = textStats(text);
+  return {
+    status: "ok",
+    source: resolvedDescriptor,
+    text,
+    stats,
+    recovery: {
+      how: `Recover source with freeflow_search action=retrieve source.kind=local root=${resolved.root} path=${resolved.relativePath}.`,
+    },
+  };
 }
-async function persistRepoFileReferenceIfRequested(input) {
-    if (input.options.sessionId === undefined) {
-        return undefined;
-    }
-    const vaultOptions = {};
-    if (input.options.vaultRoot !== undefined) {
-        vaultOptions.root = input.options.vaultRoot;
-    }
-    if (input.options.vaultRetention !== undefined) {
-        vaultOptions.retention = input.options.vaultRetention;
-    }
-    const record = await storeRepoFileReference(createVault(vaultOptions), {
-        sessionId: input.options.sessionId,
-        path: input.sourcePath,
-        hashSha256: sha256(input.text),
-        producer: { kind: "repo", name: "processing-source-loader" },
-        persistence: { status: "metadata_only", recoverability: "metadata_only" },
-    });
+async function loadVaultOutputSource(source, options, limits) {
+  const requestedStream = source.stream;
+  const unavailableDescriptor = vaultSourceDescriptor(source.outputId, requestedStream ?? "combined");
+  const vaultOptions = {};
+  const vaultRoot = source.vaultRoot ?? options.vaultRoot;
+  if (vaultRoot !== undefined) {
+    vaultOptions.root = vaultRoot;
+  }
+  if (options.vaultRetention !== undefined) {
+    vaultOptions.retention = options.vaultRetention;
+  }
+  const vault = createVault(vaultOptions);
+  let record;
+  try {
+    record = await readVaultRecord(vault, source.sessionId, source.outputId);
+  } catch (error) {
+    return { status: "unavailable", source: unavailableDescriptor, reason: errorMessage(error) };
+  }
+  const stream = resolveProcessingVaultStream(record, requestedStream);
+  const descriptor = vaultSourceDescriptor(source.outputId, stream);
+  const sourceBytes = byteCountForVaultStream(record, stream);
+  if (sourceBytes !== undefined && sourceBytes > limits.maxSourceBytes) {
     return {
-        sourceRecordIds: [record.recordId],
-        sourceOutputIds: [record.outputId],
-        operation: "processing-source-load",
+      status: "blocked",
+      source: descriptor,
+      policy: "source_limit",
+      reason: `Vault source ${source.outputId}:${stream} is ${sourceBytes} bytes, above maxSourceBytes=${limits.maxSourceBytes}.`,
+      stats: { bytes: sourceBytes },
     };
-}
-async function persistProcessingResultText(input) {
-    if (input.options.sessionId === undefined) {
-        return {};
-    }
-    const vaultOptions = {};
-    if (input.options.vaultRoot !== undefined) {
-        vaultOptions.root = input.options.vaultRoot;
-    }
-    if (input.options.vaultRetention !== undefined) {
-        vaultOptions.retention = input.options.vaultRetention;
+  }
+  try {
+    const text = await readOutputText(vault, source.sessionId, source.outputId, stream);
+    const stats = textStats(text);
+    if (stats.bytes > limits.maxSourceBytes) {
+      return {
+        status: "blocked",
+        source: descriptor,
+        policy: "source_limit",
+        reason: `Vault source ${source.outputId}:${stream} is ${stats.bytes} bytes, above maxSourceBytes=${limits.maxSourceBytes}.`,
+        stats: { bytes: stats.bytes },
+      };
     }
     const lineage = {
-        ...(input.loaded.lineage?.sourceRecordIds !== undefined ? { sourceRecordIds: input.loaded.lineage.sourceRecordIds } : {}),
-        ...(input.loaded.lineage?.sourceOutputIds !== undefined ? { sourceOutputIds: input.loaded.lineage.sourceOutputIds } : {}),
-        operation: input.operation,
-        operationHash: sha256(JSON.stringify(input.operationHashSeed)),
+      sourceRecordIds: record.recordId ? [record.recordId] : [],
+      sourceOutputIds: [source.outputId],
+      operation: "processing-source-load",
     };
-    const record = await storeTextOutput(createVault(vaultOptions), {
-        sessionId: input.options.sessionId,
-        raw: input.resultText,
-        sourceKind: "transform",
-        producer: { kind: "transform", name: input.producerName },
-        persistence: { status: "vaulted", recoverability: "exact" },
-        lineage,
-    });
-    const persistence = { status: "vaulted", recoverability: "exact", outputId: record.outputId };
     return {
-        lineage,
-        persistence,
-        recovery: { how: `Recover processing result with freeflow_search action=retrieve outputId=${record.outputId} stream=raw.`, outputId: record.outputId },
+      status: "ok",
+      source: descriptor,
+      text,
+      stats,
+      lineage,
+      recovery: {
+        how: `Recover source with freeflow_search action=retrieve outputId=${source.outputId} stream=${stream}.`,
+        outputId: source.outputId,
+      },
     };
+  } catch (error) {
+    return { status: "unavailable", source: descriptor, reason: errorMessage(error) };
+  }
 }
-function notSelectedReducer(reason) {
+function resolveProcessingVaultStream(record, requested) {
+  if (requested !== undefined) {
+    return requested;
+  }
+  return record.kind === "text" ? "raw" : "combined";
+}
+function vaultSourceDescriptor(outputId, stream) {
+  return {
+    kind: "vault-output",
+    ref: { kind: "vault", outputId, stream },
+    displayPath: `${outputId}:${stream}`,
+    stream,
+  };
+}
+function loadCapturedCommandOutputSource(source, limits) {
+  const stream = source.stream ?? "combined";
+  const descriptor = {
+    kind: "command-output",
+    ref: { kind: "native", tool: "command", outputId: source.outputId ?? "inline-command-output" },
+    displayPath: source.outputId ? `${source.outputId}:${stream}` : `inline-command-output:${stream}`,
+    stream,
+  };
+  const text = commandOutputStreamText(source, stream);
+  const stats = textStats(text);
+  if (stats.bytes > limits.maxSourceBytes) {
     return {
-        status: "not_selected",
-        candidates: [],
-        reason,
+      status: "blocked",
+      source: descriptor,
+      policy: "source_limit",
+      reason: `Captured command output ${descriptor.displayPath} is ${stats.bytes} bytes, above maxSourceBytes=${limits.maxSourceBytes}.`,
+      stats: { bytes: stats.bytes },
     };
+  }
+  const lineage = source.outputId
+    ? { sourceOutputIds: [source.outputId], operation: "processing-source-load" }
+    : undefined;
+  return {
+    status: "ok",
+    source: descriptor,
+    text,
+    stats,
+    ...(lineage !== undefined ? { lineage } : {}),
+    ...(source.outputId !== undefined
+      ? {
+          recovery: {
+            how: `Recover source from captured command outputId=${source.outputId} stream=${stream}.`,
+            outputId: source.outputId,
+          },
+        }
+      : {}),
+  };
+}
+async function persistRepoFileReferenceIfRequested(input) {
+  if (input.options.sessionId === undefined) {
+    return undefined;
+  }
+  const vaultOptions = {};
+  if (input.options.vaultRoot !== undefined) {
+    vaultOptions.root = input.options.vaultRoot;
+  }
+  if (input.options.vaultRetention !== undefined) {
+    vaultOptions.retention = input.options.vaultRetention;
+  }
+  const record = await storeRepoFileReference(createVault(vaultOptions), {
+    sessionId: input.options.sessionId,
+    path: input.sourcePath,
+    hashSha256: sha256(input.text),
+    producer: { kind: "repo", name: "processing-source-loader" },
+    persistence: { status: "metadata_only", recoverability: "metadata_only" },
+  });
+  return {
+    sourceRecordIds: [record.recordId],
+    sourceOutputIds: [record.outputId],
+    operation: "processing-source-load",
+  };
+}
+async function persistProcessingResultText(input) {
+  if (input.options.sessionId === undefined) {
+    return {};
+  }
+  const vaultOptions = {};
+  if (input.options.vaultRoot !== undefined) {
+    vaultOptions.root = input.options.vaultRoot;
+  }
+  if (input.options.vaultRetention !== undefined) {
+    vaultOptions.retention = input.options.vaultRetention;
+  }
+  const lineage = {
+    ...(input.loaded.lineage?.sourceRecordIds !== undefined
+      ? { sourceRecordIds: input.loaded.lineage.sourceRecordIds }
+      : {}),
+    ...(input.loaded.lineage?.sourceOutputIds !== undefined
+      ? { sourceOutputIds: input.loaded.lineage.sourceOutputIds }
+      : {}),
+    operation: input.operation,
+    operationHash: sha256(JSON.stringify(input.operationHashSeed)),
+  };
+  const record = await storeTextOutput(createVault(vaultOptions), {
+    sessionId: input.options.sessionId,
+    raw: input.resultText,
+    sourceKind: "transform",
+    producer: { kind: "transform", name: input.producerName },
+    persistence: { status: "vaulted", recoverability: "exact" },
+    lineage,
+  });
+  const persistence = { status: "vaulted", recoverability: "exact", outputId: record.outputId };
+  return {
+    lineage,
+    persistence,
+    recovery: {
+      how: `Recover processing result with freeflow_search action=retrieve outputId=${record.outputId} stream=raw.`,
+      outputId: record.outputId,
+    },
+  };
+}
+function notSelectedReducer(reason) {
+  return {
+    status: "not_selected",
+    candidates: [],
+    reason,
+  };
 }
 function scriptSkipsReducer(script) {
-    return script.status === "executed" || ("policy" in script && script.policy === "unsafe-unsandboxed");
+  return script.status === "executed" || ("policy" in script && script.policy === "unsafe-unsandboxed");
 }
 function scriptReducerSkipReason(script) {
-    if (script.status === "executed" && script.policy === "unsafe-unsandboxed") {
-        return "Unsafe unsandboxed script processing produced output; reducer selection was skipped.";
-    }
-    if (script.status === "executed") {
-        return "Sandboxed script processing produced output; reducer selection was skipped.";
-    }
-    return "Unsafe unsandboxed script processing was requested; reducer fallback was skipped to avoid silent fallback.";
+  if (script.status === "executed" && script.policy === "unsafe-unsandboxed") {
+    return "Unsafe unsandboxed script processing produced output; reducer selection was skipped.";
+  }
+  if (script.status === "executed") {
+    return "Sandboxed script processing produced output; reducer selection was skipped.";
+  }
+  return "Unsafe unsandboxed script processing was requested; reducer fallback was skipped to avoid silent fallback.";
 }
 function sourceFacts(loaded) {
-    return [
-        { name: "source.kind", value: loaded.source.kind },
-        { name: "source.path", value: loaded.source.displayPath },
-        { name: "source.bytes", value: loaded.stats.bytes },
-        { name: "source.lines", value: loaded.stats.lines },
-        { name: "source.sha256", value: loaded.stats.sha256 },
-    ];
+  return [
+    { name: "source.kind", value: loaded.source.kind },
+    { name: "source.path", value: loaded.source.displayPath },
+    { name: "source.bytes", value: loaded.stats.bytes },
+    { name: "source.lines", value: loaded.stats.lines },
+    { name: "source.sha256", value: loaded.stats.sha256 },
+  ];
 }
 function commandOutputStreamText(source, stream) {
+  if (stream === "stdout") {
+    return source.stdout ?? "";
+  }
+  if (stream === "stderr") {
+    return source.stderr ?? "";
+  }
+  return source.combined ?? combineOutput(source.stdout ?? "", source.stderr ?? "");
+}
+function byteCountForVaultStream(record, stream) {
+  if (record.kind === "command") {
     if (stream === "stdout") {
-        return source.stdout ?? "";
+      return record.byteCounts.stdout;
     }
     if (stream === "stderr") {
-        return source.stderr ?? "";
-    }
-    return source.combined ?? combineOutput(source.stdout ?? "", source.stderr ?? "");
-}
-function byteCountForVaultStream(record, stream) {
-    if (record.kind === "command") {
-        if (stream === "stdout") {
-            return record.byteCounts.stdout;
-        }
-        if (stream === "stderr") {
-            return record.byteCounts.stderr;
-        }
-        if (stream === "combined" || stream === "raw") {
-            return record.byteCounts.combined;
-        }
+      return record.byteCounts.stderr;
     }
-    if (record.kind === "text" && stream === "raw") {
-        return record.byteCounts.raw;
+    if (stream === "combined" || stream === "raw") {
+      return record.byteCounts.combined;
     }
-    return undefined;
+  }
+  if (record.kind === "text" && stream === "raw") {
+    return record.byteCounts.raw;
+  }
+  return undefined;
 }
 function normalizeLimits(limits) {
-    return {
-        maxSourceBytes: positiveInteger(limits?.maxSourceBytes) ?? DEFAULT_PROCESSING_LIMITS.maxSourceBytes,
-        maxVisibleBytes: positiveInteger(limits?.maxVisibleBytes) ?? DEFAULT_PROCESSING_LIMITS.maxVisibleBytes,
-    };
+  return {
+    maxSourceBytes: positiveInteger(limits?.maxSourceBytes) ?? DEFAULT_PROCESSING_LIMITS.maxSourceBytes,
+    maxVisibleBytes: positiveInteger(limits?.maxVisibleBytes) ?? DEFAULT_PROCESSING_LIMITS.maxVisibleBytes,
+  };
 }
 function positiveInteger(value) {
-    return typeof value === "number" && Number.isFinite(value) && value > 0 ? Math.floor(value) : undefined;
+  return typeof value === "number" && Number.isFinite(value) && value > 0 ? Math.floor(value) : undefined;
 }
 function repoSourceDescriptor(displayPath) {
-    return {
-        kind: "repo-file",
-        ref: { kind: "repo", path: displayPath },
-        displayPath,
-    };
+  return {
+    kind: "repo-file",
+    ref: { kind: "repo", path: displayPath },
+    displayPath,
+  };
+}
+function localSourceDescriptor(root, displayPath) {
+  return {
+    kind: "local-file",
+    ref: { kind: "local", root, path: displayPath },
+    displayPath,
+  };
 }
 function textStats(text) {
-    return {
-        bytes: byteLength(text),
-        lines: countLines(text),
-        sha256: sha256(text),
-    };
+  return {
+    bytes: byteLength(text),
+    lines: countLines(text),
+    sha256: sha256(text),
+  };
 }
 function byteLength(text) {
-    return Buffer.byteLength(text, "utf8");
+  return Buffer.byteLength(text, "utf8");
 }
 function countLines(text) {
-    if (text.length === 0) {
-        return 0;
-    }
-    return text.split(/\r?\n/).length;
+  if (text.length === 0) {
+    return 0;
+  }
+  return text.split(/\r?\n/).length;
 }
 function sha256(text) {
-    return createHash("sha256").update(text).digest("hex");
+  return createHash("sha256").update(text).digest("hex");
 }
 function combineOutput(stdout, stderr) {
-    if (stdout && stderr) {
-        return `[stdout]\n${stdout}\n[stderr]\n${stderr}`;
-    }
-    return stdout || stderr;
+  if (stdout && stderr) {
+    return `[stdout]\n${stdout}\n[stderr]\n${stderr}`;
+  }
+  return stdout || stderr;
 }
 function errorMessage(error) {
-    return error instanceof Error ? error.message : String(error);
+  return error instanceof Error ? error.message : String(error);
 }
 export function isProcessingPathInsideRoot(root, absolutePath) {
-    const relativePath = relative(root, absolutePath);
-    return relativePath === "" || (!relativePath.startsWith("..") && !relativePath.startsWith("/") && !/^[A-Za-z]:/.test(relativePath));
+  const relativePath = relative(root, absolutePath);
+  return (
+    relativePath === "" ||
+    (!relativePath.startsWith("..") && !relativePath.startsWith("/") && !/^[A-Za-z]:/.test(relativePath))
+  );
 }