@midhunxavier/code-flow 0.0.0-stage → 0.1.0-preview.3

package/src/ssh-projects/team-status.mjs

@@ -0,0 +1,39 @@
+// Read-only probe. Never installs, starts or reconnects a team server, and never
+// emits its owner token or the secret path in a device-access link.
+import { readFile } from "node:fs/promises";
+import { homedir } from "node:os";
+import { join } from "node:path";
+
+async function status() {
+  let runtime;
+  try { runtime = JSON.parse(await readFile(join(homedir(), ".codeflow/team-server/runtime.json"), "utf8")); }
+  catch (error) { return { state: error.code === "ENOENT" ? "not-installed" : "unavailable" }; }
+  if (runtime?.protocol !== 1 || !Number.isInteger(runtime.port) || runtime.port < 1 || runtime.port > 65535
+    || !/^[a-f0-9]{64}$/.test(runtime.token ?? "") || !/^[a-f0-9]{32}$/.test(runtime.instance ?? "")) return { state: "unavailable" };
+  const get = async path => {
+    const response = await fetch(`http://127.0.0.1:${runtime.port}${path}`, {
+      headers: { authorization: `Bearer ${runtime.token}` }, signal: AbortSignal.timeout(4000), redirect: "error",
+    });
+    if (!response.ok) throw new Error("Status unavailable");
+    return response.json();
+  };
+  try {
+    const identity = await get("/api/team-host-runtime");
+    if (identity.instance !== runtime.instance) return { state: "unavailable" };
+    const remote = await get("/api/remote");
+    let address;
+    if (remote.enabled && remote.lan && remote.secure && !remote.lanError) {
+      try {
+        const url = new URL(remote.tailscaleUrl ?? remote.lanUrls?.[0]);
+        if (url.protocol === "https:" && !url.username && !url.password) address = url.origin;
+      } catch { /* An absent or invalid address cannot be shared. */ }
+    }
+    return {
+      state: "running",
+      access: !remote.enabled || !remote.lan ? "disabled" : remote.lanError ? "error" : address ? "ready" : "incomplete",
+      ...(address ? { address } : {}),
+      network: remote.transport === "tailscale" ? "Tailscale" : "Private network",
+    };
+  } catch { return { state: "unavailable" }; }
+}
+console.log("CODEFLOW_TEAM_STATUS " + JSON.stringify(await status()));

package/src/ssh-projects/transport.js

@@ -0,0 +1,133 @@
+import { spawn } from "node:child_process";
+import { mkdtemp, rm } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { isAbsolute, join } from "node:path";
+import { Duplex } from "node:stream";
+const validSshHost = (host) => typeof host === "string" && /^[A-Za-z0-9][A-Za-z0-9._@-]{0,199}$/.test(host);
+function validSshOptions(options) {
+  return (options.port === void 0 || Number.isInteger(options.port) && Number(options.port) >= 1 && Number(options.port) <= 65535) && (options.identityFile === void 0 || typeof options.identityFile === "string" && options.identityFile.length <= 4096 && (isAbsolute(options.identityFile) || options.identityFile.startsWith("~/")) && !/[\x00-\x1f\x7f%]/.test(options.identityFile));
+}
+const quote = (value) => `'${value.replaceAll("'", "'\\''")}'`;
+const OPTIONS = ["-T", "-a", "-k", "-o", "GSSAPIDelegateCredentials=no", "-o", "SendEnv=-*", "-o", "BatchMode=yes", "-o", "StrictHostKeyChecking=yes", "-o", "ForwardAgent=no", "-o", "ForwardX11=no", "-o", "PermitLocalCommand=no", "-o", "RemoteCommand=none", "-o", "ClearAllForwardings=yes", "-o", "ConnectTimeout=15", "-o", "ServerAliveInterval=15", "-o", "ServerAliveCountMax=3"];
+class SshTransport {
+  constructor(host, binary = "ssh", options = {}) {
+    this.host = host;
+    this.binary = binary;
+    this.options = options;
+    if (!validSshHost(host)) throw new Error("Enter an SSH host alias or user@hostname");
+    if (!validSshOptions(options)) throw new Error("Use a port from 1 to 65535 and an absolute or ~/ SSH key path without control characters or % tokens");
+  }
+  directory = "";
+  children = /* @__PURE__ */ new Set();
+  closed = false;
+  async init() {
+    this.directory = await mkdtemp(join(tmpdir(), "cf-ssh-"));
+  }
+  start(args, command) {
+    if (this.closed) throw new Error("SSH connection is closed");
+    const env = {};
+    for (const name of ["PATH", "HOME", "USER", "LOGNAME", "SSH_AUTH_SOCK", "SYSTEMROOT", "SystemRoot", "TMPDIR"]) if (process.env[name]) env[name] = process.env[name];
+    const connection = [...this.options.port ? ["-p", String(this.options.port)] : [], ...this.options.identityFile ? ["-i", this.options.identityFile] : []];
+    const child = spawn(this.binary, [...OPTIONS, ...connection, "-o", "ControlMaster=auto", "-o", "ControlPersist=60", "-o", `ControlPath=${join(this.directory, "socket")}`, ...args, this.host, ...command ? [command] : []], { env, stdio: ["pipe", "pipe", "pipe"] });
+    this.children.add(child);
+    child.once("close", () => this.children.delete(child));
+    return child;
+  }
+  async bootstrap(code, payload, teamServer = false) {
+    const command = `exec sh -c ${quote(`exec "\${SHELL:-/bin/sh}" -lc ${quote(`exec node --input-type=module -e ${quote(code)}${teamServer ? " -- --team-server" : ""}`)}`)}`;
+    const child = this.start([], command);
+    return new Promise((resolve, reject) => {
+      let stdout = "", stderr = "", settled = false;
+      const finish = (error) => {
+        if (settled) return;
+        settled = true;
+        clearTimeout(timer);
+        error ? reject(error) : resolve(stdout);
+      };
+      const timer = setTimeout(() => {
+        child.kill();
+        finish(new Error("SSH setup timed out. Check the server's SSH login, Node.js and ~/.codeflow/ssh-host/host.log."));
+      }, 12e4);
+      child.stdout.on("data", (data) => {
+        stdout += data;
+        if (stdout.length > 64 * 1024) {
+          child.kill();
+          finish(new Error("SSH setup produced too much output"));
+        }
+      });
+      child.stderr.on("data", (data) => {
+        stderr = (stderr + data).slice(-8192);
+      });
+      child.once("error", finish);
+      child.stdin.on("error", () => void 0);
+      child.once("close", (code2) => finish(code2 === 0 ? void 0 : new Error(`SSH setup failed: ${stderr.trim() || `exit ${code2}`}. Verify ${this.host} with the same port and key in an SSH terminal first.`)));
+      child.stdin.end(payload);
+    });
+  }
+  open(port, hostname = "127.0.0.1") {
+    if (!Number.isInteger(port) || port < 1 || port > 65535) throw new Error("Invalid remote port");
+    if (!["localhost", "127.0.0.1", "::1", "[::1]"].includes(hostname)) throw new Error("SSH forwards only remote loopback addresses");
+    const host = hostname.includes(":") ? "[::1]" : hostname;
+    const child = this.start(["-W", `${host}:${port}`]);
+    const stream = new Duplex({
+      read() {
+        child.stdout.resume();
+      },
+      write(chunk, encoding, callback) {
+        child.stdin.write(chunk, encoding, callback);
+      },
+      final(callback) {
+        child.stdin.end(callback);
+      },
+      destroy(error, callback) {
+        child.stdin.destroy();
+        child.stdout.destroy();
+        child.kill();
+        callback(error);
+      }
+    });
+    child.stdout.on("data", (chunk) => {
+      if (!stream.push(chunk)) child.stdout.pause();
+    });
+    child.stdout.on("end", () => stream.push(null));
+    child.stdout.on("error", (error) => stream.destroy(error));
+    stream.on("error", () => void 0);
+    child.stdin.on("error", () => void 0);
+    let stderr = "";
+    child.stderr.on("data", (data) => {
+      stderr = (stderr + data).slice(-2048);
+    });
+    child.once("error", (error) => stream.destroy(error));
+    child.once("close", (code) => {
+      if (code) stream.destroy(new Error(`SSH connection failed: ${stderr.trim() || `exit ${code}`}`));
+    });
+    return stream;
+  }
+  async close() {
+    if (this.closed) return;
+    const child = this.start(["-O", "exit"]);
+    child.on("error", () => void 0);
+    child.stdout.resume();
+    child.stderr.resume();
+    child.stdin.end();
+    await new Promise((resolve) => {
+      const timer = setTimeout(() => {
+        child.kill();
+        resolve();
+      }, 2e3);
+      child.once("close", () => {
+        clearTimeout(timer);
+        resolve();
+      });
+    });
+    this.closed = true;
+    for (const process2 of this.children) process2.kill();
+    await rm(this.directory, { recursive: true, force: true });
+  }
+}
+export {
+  SshTransport,
+  quote,
+  validSshHost,
+  validSshOptions
+};

package/src/stats.js

@@ -0,0 +1,139 @@
+import { runPassed, checksLabel } from "./run-status.js";
+const tokensOf = (r) => (r.usage.inputTokens ?? 0) + (r.usage.outputTokens ?? 0);
+const lastLine = (stat) => stat?.trim().split("\n").pop()?.trim();
+function computeStats(all, filter = {}) {
+  const runs = all.filter((r) => (!filter.repo || r.repo === filter.repo) && (!filter.since || r.createdAt >= filter.since));
+  const agents = /* @__PURE__ */ new Map();
+  const patterns = /* @__PURE__ */ new Map();
+  const cohorts = /* @__PURE__ */ new Map();
+  const agent = (id) => {
+    let a = agents.get(id);
+    if (!a) {
+      a = { agent: id, runs: 0, steps: 0, writeSteps: 0, failures: 0, inputTokens: 0, outputTokens: 0, cacheReadTokens: 0, costUsd: 0, ms: 0, judged: 0, passed: 0, attributedJudged: 0, attributedPassed: 0, sharedJudged: 0, confidence: { low: 0, high: 1, samples: 0 }, raceEntries: 0, raceWins: 0, loopSolves: 0, findings: 0, agreedFindings: 0 };
+      agents.set(id, a);
+    }
+    return a;
+  };
+  for (const r of runs) {
+    const sourceKind = r.source?.kind ?? "task";
+    const key = JSON.stringify([r.repo, r.pattern, sourceKind]);
+    const cohort = cohorts.get(key) ?? { key, repo: r.repo, pattern: r.pattern, sourceKind, runs: 0, judged: 0, passed: 0, tokens: 0, costUsd: 0, ms: 0 };
+    cohorts.set(key, cohort);
+    cohort.runs++;
+    if (r.status === "done" && ["Passed", "Failed"].includes(checksLabel(r))) {
+      cohort.judged++;
+      if (runPassed(r)) cohort.passed++;
+    }
+    cohort.tokens += tokensOf(r);
+    cohort.costUsd += r.usage.costUsd ?? 0;
+    cohort.ms += r.endedAt ? r.endedAt - r.createdAt : 0;
+    const p = patterns.get(r.pattern) ?? { pattern: r.pattern, runs: 0, done: 0, passed: 0, failing: 0, failed: 0, stopped: 0, flagged: 0, proven: 0, weak: 0, unproven: 0, ended: 0, ms: 0, steps: 0, tokens: 0, costUsd: 0 };
+    patterns.set(r.pattern, p);
+    p.runs++;
+    if (r.status === "done") p.done++;
+    if (r.status === "failed") p.failed++;
+    if (r.status === "stopped") p.stopped++;
+    if (runPassed(r)) p.passed++;
+    if (checksLabel(r) === "Failed") p.failing++;
+    if (r.integrity && Object.keys(r.integrity).length) p.flagged++;
+    if (r.facts?.proof === "proven") p.proven++;
+    if (r.facts?.proof === "weak") p.weak++;
+    if (r.facts?.proof === "unproven") p.unproven++;
+    if (r.endedAt) {
+      p.ended++;
+      p.ms += r.endedAt - r.createdAt;
+    }
+    p.steps += r.usage.agentSteps;
+    p.tokens += tokensOf(r);
+    p.costUsd += r.usage.costUsd ?? 0;
+    const writers = Object.values(r.usage.byAgent ?? {}).filter((u) => u && u.writeSteps > 0).length;
+    for (const [id, u] of Object.entries(r.usage.byAgent ?? {})) {
+      if (!u) continue;
+      const a = agent(id);
+      a.runs++;
+      a.steps += u.steps;
+      a.writeSteps += u.writeSteps;
+      a.failures += u.failures;
+      a.inputTokens += u.inputTokens;
+      a.outputTokens += u.outputTokens;
+      a.cacheReadTokens += u.cacheReadTokens;
+      a.costUsd += u.costUsd;
+      a.ms += u.ms;
+      if (u.writeSteps && r.status === "done" && ["Passed", "Failed"].includes(checksLabel(r))) {
+        a.judged++;
+        if (runPassed(r)) a.passed++;
+        if (writers === 1) {
+          a.attributedJudged++;
+          if (runPassed(r)) a.attributedPassed++;
+        } else a.sharedJudged++;
+      }
+    }
+    for (const id of r.facts?.entrants ?? []) agent(id).raceEntries++;
+    if (r.facts?.winner) agent(r.facts.winner).raceWins++;
+    if (r.facts?.solvedBy) agent(r.facts.solvedBy).loopSolves++;
+    for (const f of r.review?.findings ?? []) {
+      for (const id of f.raisedBy) {
+        agent(id).findings++;
+        if (f.raisedBy.length > 1) agent(id).agreedFindings++;
+      }
+    }
+  }
+  for (const a of agents.values()) a.confidence = { ...confidenceInterval(a.attributedPassed, a.attributedJudged), samples: a.attributedJudged };
+  const groups = /* @__PURE__ */ new Map();
+  for (const r of runs) {
+    if (!r.group) continue;
+    const g = groups.get(r.group.id) ?? { id: r.group.id, task: r.task, createdAt: r.createdAt, runs: [] };
+    groups.set(r.group.id, g);
+    g.createdAt = Math.min(g.createdAt, r.createdAt);
+    g.runs.push({
+      id: r.id,
+      label: r.group.label,
+      pattern: r.pattern,
+      roles: r.roles,
+      status: r.status,
+      verdict: runPassed(r) ? "passed" : checksLabel(r) === "Failed" ? "failing" : void 0,
+      ms: r.endedAt ? r.endedAt - r.createdAt : void 0,
+      steps: r.usage.agentSteps,
+      tokens: tokensOf(r),
+      costUsd: r.usage.costUsd ?? 0,
+      flagged: Object.values(r.integrity ?? {}).flat().length,
+      proof: r.facts?.proof,
+      stat: lastLine(r.artifacts.find((a) => a.stat?.trim())?.stat)
+    });
+  }
+  return {
+    runs: runs.length,
+    tokens: runs.reduce((n, r) => n + tokensOf(r), 0),
+    costUsd: runs.reduce((n, r) => n + (r.usage.costUsd ?? 0), 0),
+    ms: runs.reduce((n, r) => n + (r.endedAt ? r.endedAt - r.createdAt : 0), 0),
+    agents: [...agents.values()].sort((a, b) => b.runs - a.runs),
+    patterns: [...patterns.values()].sort((a, b) => b.runs - a.runs),
+    groups: [...groups.values()].sort((a, b) => b.createdAt - a.createdAt),
+    cohorts: [...cohorts.values()].sort((a, b) => b.runs - a.runs)
+  };
+}
+const MIN_SAMPLES = 3;
+function rankAgents(runs, repo, cohort = {}) {
+  const comparable = runs.filter((r) => (!cohort.pattern || r.pattern === cohort.pattern) && (!cohort.sourceKind || (r.source?.kind ?? "task") === cohort.sourceKind));
+  const stats = computeStats(comparable, { repo }).agents.filter((a) => a.attributedJudged >= MIN_SAMPLES);
+  if (stats.length < 2) return { order: [], why: {} };
+  const ranked = stats.sort((a, b) => b.confidence.low - a.confidence.low || b.attributedJudged - a.attributedJudged || a.agent.localeCompare(b.agent));
+  return {
+    order: ranked.map((a) => a.agent),
+    why: Object.fromEntries(ranked.map((a) => [a.agent, `${a.attributedPassed} of ${a.attributedJudged} runs passed in this repo with this provider as the sole writer; 95% interval ${Math.round(100 * a.confidence.low)}\u2013${Math.round(100 * a.confidence.high)}%${a.sharedJudged ? `; ${a.sharedJudged} shared outcomes excluded` : ""}${cohort.pattern ? `; ${cohort.pattern} tasks` : "; task difficulty varies"}`]))
+  };
+}
+function confidenceInterval(passed, samples) {
+  if (!samples) return { low: 0, high: 1 };
+  const z = 1.959963984540054;
+  const p = Math.max(0, Math.min(samples, passed)) / samples;
+  const denominator = 1 + z * z / samples;
+  const center = (p + z * z / (2 * samples)) / denominator;
+  const margin = z * Math.sqrt(p * (1 - p) / samples + z * z / (4 * samples * samples)) / denominator;
+  return { low: Math.max(0, center - margin), high: Math.min(1, center + margin) };
+}
+export {
+  computeStats,
+  confidenceInterval,
+  rankAgents
+};