@hassangameryt/freeflow 0.4.0 → 0.6.0

package/router/dist/processing/engine.js

@@ -1,533 +0,0 @@
-import { createHash } from "node:crypto";
-import { readFile, stat } from "node:fs/promises";
-import { relative } from "node:path";
-import { resolveLocalPath } from "../local/local-traversal.js";
-import { resolveRepoPath } from "../repo/repo-traversal.js";
-import { renderProcessingResult, classifyProcessingRecovery } from "./renderers.js";
-import { selectProcessingReducer } from "./reducers.js";
-import {
-  processingScriptNotConfigured,
-  processingScriptUnavailableForUnloadedSource,
-  runProcessingScript,
-} from "./scripts.js";
-import {
-  createVault,
-  readOutputText,
-  readVaultRecord,
-  storeRepoFileReference,
-  storeTextOutput,
-} from "../vault/vault.js";
-export const PROCESSING_ENGINE_IMPLEMENTATION = "processing-engine-skeleton-v1";
-const DEFAULT_PROCESSING_LIMITS = {
-  maxSourceBytes: 2 * 1024 * 1024,
-  maxVisibleBytes: 4_096,
-};
-export async function loadProcessingSource(source, options = {}) {
-  const limits = normalizeLimits(options.limits);
-  switch (source.kind) {
-    case "repo-file":
-      return loadRepoFileSource(source, options, limits);
-    case "vault-output":
-      return loadVaultOutputSource(source, options, limits);
-    case "local-file":
-      return loadLocalFileSource(source, limits);
-    case "command-output":
-      return loadCapturedCommandOutputSource(source, limits);
-  }
-}
-export async function processSource(source, options = {}) {
-  const loaded = await loadProcessingSource(source, options);
-  const limits = normalizeLimits(options.limits);
-  const reducer = notSelectedReducer("Source was not loaded; reducer selection was skipped.");
-  const script = options.script
-    ? processingScriptUnavailableForUnloadedSource(
-        options.script,
-        "Source was not loaded; script processing was skipped.",
-      )
-    : processingScriptNotConfigured();
-  if (loaded.status === "blocked") {
-    return {
-      implementation: PROCESSING_ENGINE_IMPLEMENTATION,
-      status: "blocked",
-      source: loaded.source,
-      reason: loaded.reason,
-      policy: loaded.policy,
-      visibleText: renderProcessingResult({
-        status: "blocked",
-        source: loaded.source,
-        facts: [],
-        maxVisibleBytes: limits.maxVisibleBytes,
-        failure: { policy: loaded.policy, reason: loaded.reason },
-      }),
-      facts: [],
-      reducer,
-      script,
-    };
-  }
-  if (loaded.status === "unavailable") {
-    return {
-      implementation: PROCESSING_ENGINE_IMPLEMENTATION,
-      status: "unavailable",
-      source: loaded.source,
-      reason: loaded.reason,
-      visibleText: renderProcessingResult({
-        status: "unavailable",
-        source: loaded.source,
-        facts: [],
-        maxVisibleBytes: limits.maxVisibleBytes,
-        failure: { reason: loaded.reason },
-      }),
-      facts: [],
-      reducer,
-      script,
-    };
-  }
-  const selectedScript = options.script
-    ? await runProcessingScript({
-        loaded,
-        script: options.script,
-        ...(options.scriptTransform !== undefined ? { scriptTransform: options.scriptTransform } : {}),
-        ...(options.localConfig !== undefined ? { localConfig: options.localConfig } : {}),
-        ...(options.scriptSandboxAdapters !== undefined ? { adapters: options.scriptSandboxAdapters } : {}),
-      })
-    : processingScriptNotConfigured();
-  const selectedReducer = scriptSkipsReducer(selectedScript)
-    ? notSelectedReducer(scriptReducerSkipReason(selectedScript))
-    : selectProcessingReducer({ text: loaded.text, ...(options.goal !== undefined ? { goal: options.goal } : {}) });
-  const facts =
-    selectedReducer.status === "selected"
-      ? [...selectedReducer.result.facts, ...sourceFacts(loaded)]
-      : sourceFacts(loaded);
-  const visibleText = renderProcessingResult({
-    status: "ok",
-    source: loaded.source,
-    stats: loaded.stats,
-    facts,
-    reducer: selectedReducer,
-    script: selectedScript,
-    ...(loaded.recovery !== undefined ? { recovery: loaded.recovery } : {}),
-    ...(loaded.persistence !== undefined ? { persistence: loaded.persistence } : {}),
-    recoveryClass: classifyProcessingRecovery({
-      ...(loaded.recovery !== undefined ? { recovery: loaded.recovery } : {}),
-      ...(loaded.persistence !== undefined ? { persistence: loaded.persistence } : {}),
-      resultWillBePersisted: options.sessionId !== undefined,
-    }),
-    maxVisibleBytes: limits.maxVisibleBytes,
-  });
-  const persisted = await persistProcessingResultText({
-    resultText: selectedScript.status === "executed" ? selectedScript.outputText : visibleText,
-    loaded,
-    options,
-    operation:
-      selectedScript.status === "executed" ? `processing-script:${selectedScript.language}` : "processing-fact-summary",
-    producerName:
-      selectedScript.status === "executed" ? `processing-script:${selectedScript.language}` : "processing-engine",
-    operationHashSeed:
-      selectedScript.status === "executed"
-        ? {
-            source: loaded.source,
-            stats: loaded.stats,
-            script: { language: selectedScript.language, codeHashSha256: selectedScript.codeHashSha256 },
-          }
-        : { source: loaded.source, stats: loaded.stats },
-  });
-  const result = {
-    implementation: PROCESSING_ENGINE_IMPLEMENTATION,
-    status: "ok",
-    source: loaded.source,
-    stats: loaded.stats,
-    visibleText,
-    facts,
-    reducer: selectedReducer,
-    script: selectedScript,
-  };
-  const lineage = persisted.lineage ?? loaded.lineage;
-  const persistence = persisted.persistence ?? loaded.persistence;
-  const recovery = persisted.recovery ?? loaded.recovery;
-  if (lineage !== undefined) {
-    result.lineage = lineage;
-  }
-  if (persistence !== undefined) {
-    result.persistence = persistence;
-  }
-  if (recovery !== undefined) {
-    result.recovery = recovery;
-  }
-  return result;
-}
-async function loadRepoFileSource(source, options, limits) {
-  const descriptor = repoSourceDescriptor(source.path);
-  let resolved;
-  try {
-    resolved = await resolveRepoPath(source.root, source.path);
-  } catch (error) {
-    const message = errorMessage(error);
-    const blocked = message.includes("escapes root");
-    return blocked
-      ? { status: "blocked", source: descriptor, policy: "repo_containment", reason: message }
-      : { status: "unavailable", source: descriptor, reason: message };
-  }
-  const resolvedDescriptor = repoSourceDescriptor(resolved.relativePath || ".");
-  const fileStat = await stat(resolved.absolutePath);
-  if (!fileStat.isFile()) {
-    return {
-      status: "unavailable",
-      source: resolvedDescriptor,
-      reason: `Repo source is not a file: ${resolved.relativePath || "."}`,
-    };
-  }
-  if (fileStat.size > limits.maxSourceBytes) {
-    return {
-      status: "blocked",
-      source: resolvedDescriptor,
-      policy: "source_limit",
-      reason: `Repo source ${resolved.relativePath} is ${fileStat.size} bytes, above maxSourceBytes=${limits.maxSourceBytes}.`,
-      stats: { bytes: fileStat.size },
-    };
-  }
-  const text = await readFile(resolved.absolutePath, "utf8");
-  const stats = textStats(text);
-  const lineage = await persistRepoFileReferenceIfRequested({ sourcePath: resolved.relativePath, text, options });
-  return {
-    status: "ok",
-    source: resolvedDescriptor,
-    text,
-    stats,
-    ...(lineage !== undefined ? { lineage } : {}),
-  };
-}
-async function loadLocalFileSource(source, limits) {
-  const descriptor = localSourceDescriptor(source.root, source.path);
-  let resolved;
-  try {
-    resolved = await resolveLocalPath(source.root, source.path);
-  } catch (error) {
-    const message = errorMessage(error);
-    const blocked =
-      message.includes("escapes root") ||
-      message.includes("absolute path") ||
-      message.includes("too broad") ||
-      message.includes("source.root");
-    return blocked
-      ? { status: "blocked", source: descriptor, policy: "repo_containment", reason: message }
-      : { status: "unavailable", source: descriptor, reason: message };
-  }
-  const resolvedDescriptor = localSourceDescriptor(resolved.root, resolved.relativePath || ".");
-  const fileStat = await stat(resolved.absolutePath);
-  if (!fileStat.isFile()) {
-    return {
-      status: "unavailable",
-      source: resolvedDescriptor,
-      reason: `Local source is not a file: ${resolved.relativePath || "."}`,
-    };
-  }
-  if (fileStat.size > limits.maxSourceBytes) {
-    return {
-      status: "blocked",
-      source: resolvedDescriptor,
-      policy: "source_limit",
-      reason: `Local source ${resolved.relativePath} is ${fileStat.size} bytes, above maxSourceBytes=${limits.maxSourceBytes}.`,
-      stats: { bytes: fileStat.size },
-    };
-  }
-  const text = await readFile(resolved.absolutePath, "utf8");
-  const stats = textStats(text);
-  return {
-    status: "ok",
-    source: resolvedDescriptor,
-    text,
-    stats,
-    recovery: {
-      how: `Recover source with freeflow_search action=retrieve source.kind=local root=${resolved.root} path=${resolved.relativePath}.`,
-    },
-  };
-}
-async function loadVaultOutputSource(source, options, limits) {
-  const requestedStream = source.stream;
-  const unavailableDescriptor = vaultSourceDescriptor(source.outputId, requestedStream ?? "combined");
-  const vaultOptions = {};
-  const vaultRoot = source.vaultRoot ?? options.vaultRoot;
-  if (vaultRoot !== undefined) {
-    vaultOptions.root = vaultRoot;
-  }
-  if (options.vaultRetention !== undefined) {
-    vaultOptions.retention = options.vaultRetention;
-  }
-  const vault = createVault(vaultOptions);
-  let record;
-  try {
-    record = await readVaultRecord(vault, source.sessionId, source.outputId);
-  } catch (error) {
-    return { status: "unavailable", source: unavailableDescriptor, reason: errorMessage(error) };
-  }
-  const stream = resolveProcessingVaultStream(record, requestedStream);
-  const descriptor = vaultSourceDescriptor(source.outputId, stream);
-  const sourceBytes = byteCountForVaultStream(record, stream);
-  if (sourceBytes !== undefined && sourceBytes > limits.maxSourceBytes) {
-    return {
-      status: "blocked",
-      source: descriptor,
-      policy: "source_limit",
-      reason: `Vault source ${source.outputId}:${stream} is ${sourceBytes} bytes, above maxSourceBytes=${limits.maxSourceBytes}.`,
-      stats: { bytes: sourceBytes },
-    };
-  }
-  try {
-    const text = await readOutputText(vault, source.sessionId, source.outputId, stream);
-    const stats = textStats(text);
-    if (stats.bytes > limits.maxSourceBytes) {
-      return {
-        status: "blocked",
-        source: descriptor,
-        policy: "source_limit",
-        reason: `Vault source ${source.outputId}:${stream} is ${stats.bytes} bytes, above maxSourceBytes=${limits.maxSourceBytes}.`,
-        stats: { bytes: stats.bytes },
-      };
-    }
-    const lineage = {
-      sourceRecordIds: record.recordId ? [record.recordId] : [],
-      sourceOutputIds: [source.outputId],
-      operation: "processing-source-load",
-    };
-    return {
-      status: "ok",
-      source: descriptor,
-      text,
-      stats,
-      lineage,
-      recovery: {
-        how: `Recover source with freeflow_search action=retrieve outputId=${source.outputId} stream=${stream}.`,
-        outputId: source.outputId,
-      },
-    };
-  } catch (error) {
-    return { status: "unavailable", source: descriptor, reason: errorMessage(error) };
-  }
-}
-function resolveProcessingVaultStream(record, requested) {
-  if (requested !== undefined) {
-    return requested;
-  }
-  return record.kind === "text" ? "raw" : "combined";
-}
-function vaultSourceDescriptor(outputId, stream) {
-  return {
-    kind: "vault-output",
-    ref: { kind: "vault", outputId, stream },
-    displayPath: `${outputId}:${stream}`,
-    stream,
-  };
-}
-function loadCapturedCommandOutputSource(source, limits) {
-  const stream = source.stream ?? "combined";
-  const descriptor = {
-    kind: "command-output",
-    ref: { kind: "native", tool: "command", outputId: source.outputId ?? "inline-command-output" },
-    displayPath: source.outputId ? `${source.outputId}:${stream}` : `inline-command-output:${stream}`,
-    stream,
-  };
-  const text = commandOutputStreamText(source, stream);
-  const stats = textStats(text);
-  if (stats.bytes > limits.maxSourceBytes) {
-    return {
-      status: "blocked",
-      source: descriptor,
-      policy: "source_limit",
-      reason: `Captured command output ${descriptor.displayPath} is ${stats.bytes} bytes, above maxSourceBytes=${limits.maxSourceBytes}.`,
-      stats: { bytes: stats.bytes },
-    };
-  }
-  const lineage = source.outputId
-    ? { sourceOutputIds: [source.outputId], operation: "processing-source-load" }
-    : undefined;
-  return {
-    status: "ok",
-    source: descriptor,
-    text,
-    stats,
-    ...(lineage !== undefined ? { lineage } : {}),
-    ...(source.outputId !== undefined
-      ? {
-          recovery: {
-            how: `Recover source from captured command outputId=${source.outputId} stream=${stream}.`,
-            outputId: source.outputId,
-          },
-        }
-      : {}),
-  };
-}
-async function persistRepoFileReferenceIfRequested(input) {
-  if (input.options.sessionId === undefined) {
-    return undefined;
-  }
-  const vaultOptions = {};
-  if (input.options.vaultRoot !== undefined) {
-    vaultOptions.root = input.options.vaultRoot;
-  }
-  if (input.options.vaultRetention !== undefined) {
-    vaultOptions.retention = input.options.vaultRetention;
-  }
-  const record = await storeRepoFileReference(createVault(vaultOptions), {
-    sessionId: input.options.sessionId,
-    path: input.sourcePath,
-    hashSha256: sha256(input.text),
-    producer: { kind: "repo", name: "processing-source-loader" },
-    persistence: { status: "metadata_only", recoverability: "metadata_only" },
-  });
-  return {
-    sourceRecordIds: [record.recordId],
-    sourceOutputIds: [record.outputId],
-    operation: "processing-source-load",
-  };
-}
-async function persistProcessingResultText(input) {
-  if (input.options.sessionId === undefined) {
-    return {};
-  }
-  const vaultOptions = {};
-  if (input.options.vaultRoot !== undefined) {
-    vaultOptions.root = input.options.vaultRoot;
-  }
-  if (input.options.vaultRetention !== undefined) {
-    vaultOptions.retention = input.options.vaultRetention;
-  }
-  const lineage = {
-    ...(input.loaded.lineage?.sourceRecordIds !== undefined
-      ? { sourceRecordIds: input.loaded.lineage.sourceRecordIds }
-      : {}),
-    ...(input.loaded.lineage?.sourceOutputIds !== undefined
-      ? { sourceOutputIds: input.loaded.lineage.sourceOutputIds }
-      : {}),
-    operation: input.operation,
-    operationHash: sha256(JSON.stringify(input.operationHashSeed)),
-  };
-  const record = await storeTextOutput(createVault(vaultOptions), {
-    sessionId: input.options.sessionId,
-    raw: input.resultText,
-    sourceKind: "transform",
-    producer: { kind: "transform", name: input.producerName },
-    persistence: { status: "vaulted", recoverability: "exact" },
-    lineage,
-  });
-  const persistence = { status: "vaulted", recoverability: "exact", outputId: record.outputId };
-  return {
-    lineage,
-    persistence,
-    recovery: {
-      how: `Recover processing result with freeflow_search action=retrieve outputId=${record.outputId} stream=raw.`,
-      outputId: record.outputId,
-    },
-  };
-}
-function notSelectedReducer(reason) {
-  return {
-    status: "not_selected",
-    candidates: [],
-    reason,
-  };
-}
-function scriptSkipsReducer(script) {
-  return script.status === "executed" || ("policy" in script && script.policy === "unsafe-unsandboxed");
-}
-function scriptReducerSkipReason(script) {
-  if (script.status === "executed" && script.policy === "unsafe-unsandboxed") {
-    return "Unsafe unsandboxed script processing produced output; reducer selection was skipped.";
-  }
-  if (script.status === "executed") {
-    return "Sandboxed script processing produced output; reducer selection was skipped.";
-  }
-  return "Unsafe unsandboxed script processing was requested; reducer fallback was skipped to avoid silent fallback.";
-}
-function sourceFacts(loaded) {
-  return [
-    { name: "source.kind", value: loaded.source.kind },
-    { name: "source.path", value: loaded.source.displayPath },
-    { name: "source.bytes", value: loaded.stats.bytes },
-    { name: "source.lines", value: loaded.stats.lines },
-    { name: "source.sha256", value: loaded.stats.sha256 },
-  ];
-}
-function commandOutputStreamText(source, stream) {
-  if (stream === "stdout") {
-    return source.stdout ?? "";
-  }
-  if (stream === "stderr") {
-    return source.stderr ?? "";
-  }
-  return source.combined ?? combineOutput(source.stdout ?? "", source.stderr ?? "");
-}
-function byteCountForVaultStream(record, stream) {
-  if (record.kind === "command") {
-    if (stream === "stdout") {
-      return record.byteCounts.stdout;
-    }
-    if (stream === "stderr") {
-      return record.byteCounts.stderr;
-    }
-    if (stream === "combined" || stream === "raw") {
-      return record.byteCounts.combined;
-    }
-  }
-  if (record.kind === "text" && stream === "raw") {
-    return record.byteCounts.raw;
-  }
-  return undefined;
-}
-function normalizeLimits(limits) {
-  return {
-    maxSourceBytes: positiveInteger(limits?.maxSourceBytes) ?? DEFAULT_PROCESSING_LIMITS.maxSourceBytes,
-    maxVisibleBytes: positiveInteger(limits?.maxVisibleBytes) ?? DEFAULT_PROCESSING_LIMITS.maxVisibleBytes,
-  };
-}
-function positiveInteger(value) {
-  return typeof value === "number" && Number.isFinite(value) && value > 0 ? Math.floor(value) : undefined;
-}
-function repoSourceDescriptor(displayPath) {
-  return {
-    kind: "repo-file",
-    ref: { kind: "repo", path: displayPath },
-    displayPath,
-  };
-}
-function localSourceDescriptor(root, displayPath) {
-  return {
-    kind: "local-file",
-    ref: { kind: "local", root, path: displayPath },
-    displayPath,
-  };
-}
-function textStats(text) {
-  return {
-    bytes: byteLength(text),
-    lines: countLines(text),
-    sha256: sha256(text),
-  };
-}
-function byteLength(text) {
-  return Buffer.byteLength(text, "utf8");
-}
-function countLines(text) {
-  if (text.length === 0) {
-    return 0;
-  }
-  return text.split(/\r?\n/).length;
-}
-function sha256(text) {
-  return createHash("sha256").update(text).digest("hex");
-}
-function combineOutput(stdout, stderr) {
-  if (stdout && stderr) {
-    return `[stdout]\n${stdout}\n[stderr]\n${stderr}`;
-  }
-  return stdout || stderr;
-}
-function errorMessage(error) {
-  return error instanceof Error ? error.message : String(error);
-}
-export function isProcessingPathInsideRoot(root, absolutePath) {
-  const relativePath = relative(root, absolutePath);
-  return (
-    relativePath === "" ||
-    (!relativePath.startsWith("..") && !relativePath.startsWith("/") && !/^[A-Za-z]:/.test(relativePath))
-  );
-}