@hassangameryt/freeflow 0.3.0 → 0.5.0

package/evals/scripts/run-jq-wasm-proof-spike.js

@@ -1,299 +0,0 @@
-#!/usr/bin/env node
-import { readFile, mkdir, writeFile } from "node:fs/promises";
-import { dirname, join, resolve } from "node:path";
-import { Worker } from "node:worker_threads";
-
-import { SCRIPT_SANDBOX_REQUIRED_PROOFS, scriptSandboxProofFixturesForLanguage } from "../../router/dist/index.js";
-
-const DEFAULT_REPORT_PATH = "evals/reports/runtime/jq-wasm-proof-spike-1-report.md";
-const DEFAULT_TIMEOUT_MS = 500;
-const DEFAULT_OUTPUT_BYTES = 4096;
-const SECRET_SENTINEL = "FREEFLOW_SANDBOX_SECRET_SENTINEL_VALUE";
-const HOST_HOME = process.env.HOME ?? "__no_home__";
-
-async function main() {
-  process.env.FREEFLOW_SANDBOX_SECRET_SENTINEL = SECRET_SENTINEL;
-  const args = parseArgs(process.argv.slice(2));
-  const root = args.jqWasmRoot ?? process.env.JQ_WASM_ROOT;
-  if (!root) {
-    throw new Error("Missing --jq-wasm-root or JQ_WASM_ROOT. Install jq-wasm in a temp directory and pass its package root.");
-  }
-
-  const reportPath = args.report ?? DEFAULT_REPORT_PATH;
-  const timeoutMs = args.timeoutMs ?? DEFAULT_TIMEOUT_MS;
-  const outputBytes = args.outputBytes ?? DEFAULT_OUTPUT_BYTES;
-  const packageRoot = resolve(root);
-  const packageJson = JSON.parse(await readFile(join(packageRoot, "package.json"), "utf8"));
-  const jqPath = join(packageRoot, packageJson.main ?? "dist/index.js");
-  const packageFiles = await collectPackageFiles(packageRoot);
-
-  const version = await runJqWorker({ jqPath, query: "__version__", timeoutMs, outputBytes });
-  const positive = await runJqWorker({ jqPath, query: ".test_log", json: { test_log: "INFO setup\nERROR target\n" }, timeoutMs, outputBytes });
-
-  const fixtures = scriptSandboxProofFixturesForLanguage("jq");
-  const results = [];
-  for (const fixture of fixtures) {
-    const run = await runJqWorker({ jqPath, query: fixture.program, json: { test_log: "INFO setup\nERROR target\n" }, timeoutMs, outputBytes });
-    results.push({ fixture, run, pass: assessProof(fixture.proof, run, { timeoutMs, outputBytes }) });
-  }
-
-  const failed = results.filter((result) => !result.pass.ok);
-  const report = renderReport({
-    packageJson,
-    packageRoot,
-    jqPath,
-    packageFiles,
-    timeoutMs,
-    outputBytes,
-    version,
-    positive,
-    results,
-    failed,
-  });
-  await mkdir(dirname(reportPath), { recursive: true });
-  await writeFile(reportPath, report, "utf8");
-  console.log(`Wrote ${reportPath}`);
-  console.log(`jq-wasm proof fixtures: ${results.length - failed.length}/${results.length} passed`);
-  if (failed.length > 0) {
-    for (const result of failed) {
-      console.log(`FAIL ${result.fixture.proof}: ${result.pass.reason}`);
-    }
-    process.exitCode = 1;
-  }
-}
-
-async function collectPackageFiles(packageRoot) {
-  const { readdir, stat } = await import("node:fs/promises");
-  const files = [];
-  async function walk(dir) {
-    for (const entry of await readdir(dir, { withFileTypes: true })) {
-      const full = join(dir, entry.name);
-      if (entry.isDirectory()) {
-        await walk(full);
-      } else if (entry.isFile()) {
-        const info = await stat(full);
-        files.push({ path: full.slice(packageRoot.length + 1), bytes: info.size });
-      }
-    }
-  }
-  await walk(packageRoot);
-  return files.sort((a, b) => a.path.localeCompare(b.path));
-}
-
-async function runJqWorker({ jqPath, query, json = {}, timeoutMs, outputBytes }) {
-  const start = Date.now();
-  const workerSource = `
-    const { parentPort, workerData } = require('node:worker_threads');
-    const jq = require(workerData.jqPath);
-    const capText = (value, cap) => {
-      const text = String(value ?? '');
-      const buffer = Buffer.from(text, 'utf8');
-      return {
-        text: buffer.byteLength > cap ? buffer.subarray(0, cap).toString('utf8') : text,
-        bytes: buffer.byteLength,
-        truncated: buffer.byteLength > cap,
-      };
-    };
-    (async () => {
-      try {
-        if (workerData.query === '__version__') {
-          const version = await jq.version();
-          parentPort.postMessage({ status: 'success', version, stdout: '', stderr: '', exitCode: 0, truncated: false, rawStdoutBytes: 0, rawStderrBytes: 0 });
-          return;
-        }
-        const result = await jq.raw(workerData.json, workerData.query, ['-c']);
-        const stdoutCap = Math.max(1, Math.floor(workerData.outputBytes / 2));
-        const stderrCap = Math.max(1, workerData.outputBytes - stdoutCap);
-        const stdout = capText(result.stdout, stdoutCap);
-        const stderr = capText(result.stderr, stderrCap);
-        parentPort.postMessage({
-          status: 'success',
-          stdout: stdout.text,
-          stderr: stderr.text,
-          exitCode: result.exitCode,
-          truncated: stdout.truncated || stderr.truncated,
-          rawStdoutBytes: stdout.bytes,
-          rawStderrBytes: stderr.bytes,
-        });
-      } catch (error) {
-        const message = String(error && error.stack || error);
-        const stderr = capText(message, workerData.outputBytes);
-        parentPort.postMessage({ status: 'error', stdout: '', stderr: stderr.text, exitCode: null, truncated: stderr.truncated, rawStdoutBytes: 0, rawStderrBytes: stderr.bytes });
-      }
-    })();
-  `;
-  const worker = new Worker(workerSource, {
-    eval: true,
-    workerData: { jqPath, query, json, outputBytes },
-    resourceLimits: {
-      maxOldGenerationSizeMb: 64,
-      maxYoungGenerationSizeMb: 16,
-    },
-  });
-
-  return new Promise((resolve) => {
-    let settled = false;
-    const finish = (result) => {
-      if (settled) return;
-      settled = true;
-      clearTimeout(timer);
-      resolve({
-        ...result,
-        durationMs: Date.now() - start,
-        outputBytes: Buffer.byteLength(`${result.stdout ?? ""}${result.stderr ?? ""}`, "utf8"),
-        stdoutBytes: Buffer.byteLength(result.stdout ?? "", "utf8"),
-        stderrBytes: Buffer.byteLength(result.stderr ?? "", "utf8"),
-      });
-    };
-    const timer = setTimeout(() => {
-      if (settled) return;
-      finish({ status: "timed_out", stdout: "", stderr: "", exitCode: null, truncated: false, rawStdoutBytes: 0, rawStderrBytes: 0 });
-      void worker.terminate();
-    }, timeoutMs);
-    worker.on("message", (message) => finish(message));
-    worker.on("error", (error) => finish({ status: "error", stdout: "", stderr: String(error?.stack ?? error), exitCode: null, truncated: false, rawStdoutBytes: 0, rawStderrBytes: Buffer.byteLength(String(error?.stack ?? error), "utf8") }));
-    worker.on("exit", (code) => {
-      if (!settled && code !== 0) {
-        finish({ status: "error", stdout: "", stderr: `worker exited with code ${code}`, exitCode: code, truncated: false, rawStdoutBytes: 0, rawStderrBytes: 0 });
-      }
-    });
-  });
-}
-
-function assessProof(proof, run, limits) {
-  const combined = `${run.stdout ?? ""}\n${run.stderr ?? ""}`;
-  switch (proof) {
-    case "env_access_denied":
-      return passIf(!combined.includes(SECRET_SENTINEL) && !combined.includes("process.env"), "jq exposed only fixed runtime env, not ambient host env or secrets");
-    case "home_access_denied":
-      return passIf(!combined.includes(HOST_HOME) && !combined.match(/PRIVATE KEY|\.ssh/), "home/secret paths were not exposed");
-    case "repo_access_denied":
-      return passIf(!combined.includes("@hassangameryt/freeflow") && !combined.includes("package.json"), "repo files were not exposed");
-    case "vault_access_denied":
-      return passIf(!combined.includes("ffout_") && !combined.includes("ffrec_"), "vault records were not exposed");
-    case "network_access_denied":
-      return passIf(run.exitCode !== 0 && combined.includes("network unavailable"), "jq has no ambient network primitive; synthetic fetch errored boundedly");
-    case "input_read_only":
-      return passIf(run.exitCode !== 0 && combined.includes("module not found"), "input mutation include was unavailable");
-    case "output_escape_denied":
-      return passIf(run.exitCode !== 0 && combined.includes("module not found"), "output escape include was unavailable");
-    case "stdout_stderr_bounded":
-      return passIf(
-        run.status !== "error" && run.truncated && run.outputBytes <= limits.outputBytes && run.stdoutBytes > 0 && run.stderrBytes > 0 && run.rawStdoutBytes > limits.outputBytes && run.rawStderrBytes > limits.outputBytes,
-        "stdout and stderr flood output was truncated before crossing the worker boundary",
-      );
-    case "timeout_enforced":
-      return passIf(run.status === "timed_out" && run.durationMs < limits.timeoutMs * 5, "recursive jq loop was interrupted by worker termination");
-    default:
-      return { ok: false, reason: `No assessment rule for proof ${proof}` };
-  }
-}
-
-function passIf(condition, reason) {
-  return condition ? { ok: true, reason } : { ok: false, reason: `Failed: ${reason}` };
-}
-
-function renderReport(data) {
-  const lines = [];
-  lines.push("# jq-wasm Sandbox Proof Spike Report");
-  lines.push("");
-  lines.push(`> **Date:** ${new Date().toISOString().slice(0, 10)}`);
-  lines.push("> **Status:** " + (data.failed.length === 0 ? "Passed proof spike for jq candidate" : "Failed proof spike for jq candidate"));
-  lines.push("> **Scope:** Proof-only evaluation. No Freeflow script execution path is enabled.");
-  lines.push("");
-  lines.push("## Candidate");
-  lines.push("");
-  lines.push(`- Package: \`${data.packageJson.name}@${data.packageJson.version}\``);
-  lines.push(`- License: ${data.packageJson.license ?? "unknown"}`);
-  lines.push(`- Package root: \`${data.packageRoot}\``);
-  lines.push(`- Entry point: \`${data.jqPath}\``);
-  lines.push(`- Runtime version: ${data.version.version ? `\`${data.version.version}\`` : "unavailable"}`);
-  lines.push(`- Timeout: ${data.timeoutMs}ms via Worker termination`);
-  lines.push(`- Output cap: ${data.outputBytes} bytes across stdout + stderr before Worker result crosses to host`);
-  lines.push("- Worker resource limits: `maxOldGenerationSizeMb=64`, `maxYoungGenerationSizeMb=16`");
-  lines.push("");
-  lines.push("## Package Files");
-  lines.push("");
-  for (const file of data.packageFiles) {
-    lines.push(`- \`${file.path}\` — ${file.bytes} bytes`);
-  }
-  lines.push("");
-  lines.push("## Positive API Probe");
-  lines.push("");
-  lines.push(`- Status: ${data.positive.status}`);
-  lines.push(`- Output: \`${escapeInline(data.positive.stdout)}\``);
-  lines.push(`- Error: ${data.positive.stderr ? `\`${escapeInline(data.positive.stderr)}\`` : "none"}`);
-  lines.push("");
-  lines.push("## Required Proof Results");
-  lines.push("");
-  lines.push("| Proof | Result | Evidence |");
-  lines.push("| --- | --- | --- |");
-  for (const result of data.results) {
-    const evidence = [
-      result.pass.reason,
-      `status=${result.run.status}`,
-      `exitCode=${result.run.exitCode}`,
-      `durationMs=${result.run.durationMs}`,
-      `outputBytes=${result.run.outputBytes}`,
-      `stdoutBytes=${result.run.stdoutBytes}`,
-      `stderrBytes=${result.run.stderrBytes}`,
-      `rawStdoutBytes=${result.run.rawStdoutBytes}`,
-      `rawStderrBytes=${result.run.rawStderrBytes}`,
-      `truncated=${result.run.truncated}`,
-    ].join("; ");
-    lines.push(`| ${result.fixture.proof} | ${result.pass.ok ? "pass" : "fail"} | ${escapeTable(evidence)} |`);
-  }
-  lines.push("");
-  lines.push("## Notes");
-  lines.push("");
-  lines.push("- This proof runner uses a temporary installed `jq-wasm` package root passed explicitly by the caller.");
-  lines.push("- It does not add repo dependencies and does not wire the adapter into `freeflow_search action=transform` execution.");
-  lines.push("- Timeout proof uses Node Worker termination because in-thread recursive jq blocks the event loop.");
-  lines.push("- Output proof caps what crosses the Worker boundary; `jq-wasm` itself can still generate large in-Worker strings before the wrapper truncates them.");
-  lines.push("- Passing this spike only supports jq adapter feasibility; Python remains unavailable until a Python candidate passes proofs.");
-  lines.push("- Before product execution, this must still go through implementation/security review and source-plan update.");
-  lines.push("");
-  lines.push("## Required Proof Set");
-  lines.push("");
-  for (const proof of SCRIPT_SANDBOX_REQUIRED_PROOFS) {
-    lines.push(`- ${proof}`);
-  }
-  return `${lines.join("\n")}\n`;
-}
-
-function escapeInline(value) {
-  return String(value ?? "").replaceAll("`", "\\`").replaceAll("\n", "\\n");
-}
-
-function escapeTable(value) {
-  return escapeInline(value).replaceAll("|", "\\|");
-}
-
-function parseArgs(argv) {
-  const args = {};
-  for (let index = 0; index < argv.length; index += 1) {
-    const arg = argv[index];
-    const next = argv[index + 1];
-    if (arg === "--jq-wasm-root") {
-      args.jqWasmRoot = next;
-      index += 1;
-    } else if (arg === "--report") {
-      args.report = next;
-      index += 1;
-    } else if (arg === "--timeout-ms") {
-      args.timeoutMs = Number(next);
-      index += 1;
-    } else if (arg === "--output-bytes") {
-      args.outputBytes = Number(next);
-      index += 1;
-    } else {
-      throw new Error(`Unknown argument ${arg}`);
-    }
-  }
-  return args;
-}
-
-main().catch((error) => {
-  console.error(error instanceof Error ? error.stack : error);
-  process.exit(1);
-});

package/evals/scripts/run-output-router-transform-eval.js

@@ -1,115 +0,0 @@
-#!/usr/bin/env node
-import assert from "node:assert/strict";
-import { mkdtemp, rm, writeFile } from "node:fs/promises";
-import { join } from "node:path";
-import { tmpdir } from "node:os";
-
-import {
-  createVault,
-  freeflowTransform,
-  readOutputText,
-  storeCommandOutput,
-} from "../../router/dist/index.js";
-
-const REPORT_PATH = "evals/reports/runtime/output-router-transform-eval-1-report.md";
-const DATE = "2026-06-28";
-
-function bytes(value) {
-  return Buffer.byteLength(String(value), "utf8");
-}
-
-function evidenceBytes(result) {
-  return (result.evidence ?? []).reduce((total, packet) => total + bytes(packet.excerpt ?? ""), 0);
-}
-
-function passMark(value) {
-  return value ? "pass" : "fail";
-}
-
-async function runTransformComparison(vaultRoot) {
-  const vault = createVault({ root: vaultRoot });
-  const log = Array.from({ length: 160 }, (_, index) => {
-    if (index === 37) return "ERROR target alpha failed in payment retry path";
-    if (index === 119) return "ERROR target beta failed in invoice retry path";
-    return `INFO noisy line ${index + 1}: background worker heartbeat`;
-  }).join("\n");
-
-  const source = await storeCommandOutput(vault, {
-    sessionId: "transform-long-log-eval",
-    command: "node noisy-log-fixture.js",
-    stdout: log,
-    stderr: "",
-    combined: log,
-    executionStatus: "success",
-    exitCode: 0,
-    createdAt: "2026-06-23T00:00:00.000Z",
-  });
-
-  const filtered = await freeflowTransform({
-    sessionId: "transform-long-log-eval",
-    vaultRoot,
-    source: { kind: "vault", outputId: source.outputId, stream: "combined" },
-    operation: { kind: "regexFilter", pattern: "ERROR target", contextLines: 1, maxMatches: 10 },
-    thresholds: { largeOutputLines: 8, largeOutputBytes: 420 },
-    preserve: "important",
-  });
-  const counted = await freeflowTransform({
-    sessionId: "transform-long-log-eval",
-    vaultRoot,
-    source: { kind: "vault", outputId: source.outputId, stream: "combined" },
-    operation: { kind: "countMatches", pattern: "ERROR target" },
-    preserve: "important",
-  });
-
-  const recoveredFiltered = await readOutputText(vault, "transform-long-log-eval", filtered.outputId, "raw");
-  const gates = {
-    manualBaselineWouldReadWholeLog: bytes(log) > evidenceBytes(filtered),
-    filteredTargetFacts: recoveredFiltered.includes("ERROR target alpha") && recoveredFiltered.includes("ERROR target beta"),
-    countMatches: /matches: 2\b/.test(await readOutputText(vault, "transform-long-log-eval", counted.outputId, "raw")),
-    lineage: filtered.lineage.sourceOutputIds.includes(source.outputId),
-  };
-
-  return {
-    id: "long-log-manual-inspection-vs-freeflow-transform",
-    baseline: `${bytes(log)} raw log bytes for manual inspection`,
-    freeflow: `${evidenceBytes(filtered)} filtered evidence bytes; countMatches=2; exact transformed recovery`,
-    gates,
-  };
-}
-
-function assertAllPassed(results) {
-  for (const result of results) {
-    for (const [name, passed] of Object.entries(result.gates)) {
-      assert.equal(passed, true, `${result.id} gate failed: ${name}`);
-    }
-  }
-}
-
-function renderReport(results) {
-  const totalGates = results.reduce((count, result) => count + Object.keys(result.gates).length, 0);
-  const passedGates = results.reduce((count, result) => count + Object.values(result.gates).filter(Boolean).length, 0);
-  const rows = results.map((result) => {
-    const gates = Object.entries(result.gates).map(([name, passed]) => `${name} ${passed ? "✓" : "✗"}`).join("; ");
-    return `| ${result.id} | ${result.baseline} | ${result.freeflow} | ${passMark(Object.values(result.gates).every(Boolean))} | ${gates} |`;
-  }).join("\n");
-
-  return `# Output Router Transform Eval - Iteration 1\n\nDate: ${DATE}\n\n## Scope\n\nTargeted deterministic eval for transformed-output routing. It compares direct/manual long-log inspection against explicit Freeflow transform operations.\n\n## Command\n\n\`\`\`sh\nnpm run build && node evals/scripts/run-output-router-transform-eval.js\n\`\`\`\n\n## Summary\n\n- Fixtures: ${results.length}\n- Objective gates passed: ${passedGates}/${totalGates}\n\n## Results\n\n| fixture | direct/raw baseline | Freeflow routed behavior | status | gates |\n| --- | --- | --- | --- | --- |\n${rows}\n\n## Result\n\nAll targeted transform gates passed for these deterministic fixtures.\n`;
-}
-
-async function main() {
-  const tmpRoot = await mkdtemp(join(tmpdir(), "freeflow-transform-eval-"));
-  try {
-    const vaultRoot = join(tmpRoot, "vault");
-    const results = [await runTransformComparison(vaultRoot)];
-    assertAllPassed(results);
-    await writeFile(REPORT_PATH, renderReport(results), "utf8");
-    console.log(`Wrote ${REPORT_PATH}`);
-  } finally {
-    await rm(tmpRoot, { recursive: true, force: true });
-  }
-}
-
-main().catch((error) => {
-  console.error(error);
-  process.exitCode = 1;
-});