agent-hitch 0.1.1 → 0.2.1

package/src/fs.js

@@ -1,43 +0,0 @@
-import { mkdir, open, readFile, rename, rm, writeFile } from "node:fs/promises";
-import { randomBytes } from "node:crypto";
-import path from "node:path";
-
-export async function ensureDir(directory) {
-  await mkdir(directory, { recursive: true });
-  return directory;
-}
-
-export async function readJSON(file, fallback = undefined) {
-  try {
-    return JSON.parse(await readFile(file, "utf8"));
-  } catch (error) {
-    if (error?.code === "ENOENT" && fallback !== undefined) return fallback;
-    throw error;
-  }
-}
-
-export async function atomicWriteJSON(file, value, mode = 0o600) {
-  await ensureDir(path.dirname(file));
-  const temporary = `${file}.${process.pid}.${Date.now()}.${randomBytes(6).toString("hex")}.tmp`;
-  await writeFile(temporary, `${JSON.stringify(value, null, 2)}\n`, { mode });
-  await rename(temporary, file);
-}
-
-export async function appendLine(file, value) {
-  await ensureDir(path.dirname(file));
-  const handle = await open(file, "a", 0o600);
-  try {
-    await handle.write(`${value}\n`);
-  } finally {
-    await handle.close();
-  }
-}
-
-export async function removeIfExists(file) {
-  await rm(file, { force: true });
-}
-
-export async function writePrivateFile(file, value) {
-  await ensureDir(path.dirname(file));
-  await writeFile(file, value, { mode: 0o600 });
-}

package/src/harbor-backend.js

@@ -1,285 +0,0 @@
-import { spawn } from "node:child_process";
-import { createWriteStream } from "node:fs";
-import { readdir, stat } from "node:fs/promises";
-import path from "node:path";
-import { fileURLToPath } from "node:url";
-import { HitchError, invalidInput } from "./errors.js";
-import { atomicWriteJSON, ensureDir, readJSON } from "./fs.js";
-import { consumeLines } from "./line-stream.js";
-import { terminateProcess } from "./process.js";
-import { detectVersion, fingerprintExecutable } from "./registry.js";
-import { HARBOR_CREDENTIAL_ENV, locateHarbor } from "./eval-tools.js";
-
-const BRIDGE_DIRECTORY = fileURLToPath(new URL("../integrations/harbor", import.meta.url));
-export async function runHarborBackend({
-  evalDirectory,
-  request,
-  root,
-  resolvedRevision,
-  runtimeDirectory,
-  env = process.env,
-  harborExecutable,
-  signal,
-  emit = () => {},
-}) {
-  const backendDirectory = await ensureDir(path.join(evalDirectory, "harbor"));
-  const executable = await discoverHarbor(harborExecutable, root, env);
-  const version = await detectVersion(executable, ["--version"]);
-  const identity = await fingerprintExecutable(executable);
-  const jobName = "job";
-  const configPath = path.join(backendDirectory, "job.json");
-  const jobDirectory = path.join(backendDirectory, jobName);
-  const resultPath = path.join(jobDirectory, "result.json");
-  const config = await buildHarborJobConfig({
-    request,
-    resolvedRevision,
-    runtimeDirectory,
-    backendDirectory,
-    jobName,
-    env,
-  });
-  await atomicWriteJSON(configPath, config);
-  emit({ type: "eval.backend.started", backend: "harbor", executable, version: version || null });
-
-  const invocation = await invokeHarbor(executable, ["run", "--config", configPath, "--yes"], {
-    cwd: evalDirectory,
-    env: withBridgePythonPath(env),
-    stdoutPath: path.join(backendDirectory, "stdout.log"),
-    stderrPath: path.join(backendDirectory, "stderr.log"),
-    signal,
-    emit,
-  });
-  const jobResult = await readJSON(resultPath, null);
-  const rawResult = jobResult ? await attachTrialResults(jobDirectory, jobResult) : null;
-  emit({
-    type: "eval.backend.completed",
-    backend: "harbor",
-    process_exit_code: invocation.code,
-    signal: invocation.signal,
-    result_available: rawResult !== null,
-  });
-
-  return {
-    backend: {
-      name: "harbor",
-      executable,
-      version: version || null,
-      identity,
-      config_path: configPath,
-      result_path: rawResult ? resultPath : null,
-      stdout_path: path.join(backendDirectory, "stdout.log"),
-      stderr_path: path.join(backendDirectory, "stderr.log"),
-      process_exit_code: invocation.code,
-      signal: invocation.signal,
-    },
-    rawResult,
-    summary: rawResult ? normalizeHarborResult(rawResult) : null,
-  };
-}
-
-async function attachTrialResults(jobDirectory, jobResult) {
-  if (Array.isArray(jobResult.trial_results)) return jobResult;
-  let entries;
-  try {
-    entries = await readdir(jobDirectory, { withFileTypes: true });
-  } catch (error) {
-    if (error?.code === "ENOENT") return jobResult;
-    throw error;
-  }
-  const trialResults = [];
-  for (const entry of entries.sort((left, right) => left.name.localeCompare(right.name))) {
-    if (!entry.isDirectory()) continue;
-    const trial = await readJSON(path.join(jobDirectory, entry.name, "result.json"), null);
-    if (trial?.trial_name && trial?.task_name) trialResults.push(trial);
-  }
-  return { ...jobResult, trial_results: trialResults };
-}
-
-export async function buildHarborJobConfig({
-  request,
-  resolvedRevision,
-  runtimeDirectory,
-  backendDirectory,
-  jobName = "job",
-  env = process.env,
-}) {
-  const timeoutSeconds = request.timeout_ms > 0 ? Math.ceil(request.timeout_ms / 1_000) : null;
-  const setupTimeoutSeconds = request.setup_timeout_ms > 0 ? Math.ceil(request.setup_timeout_ms / 1_000) : null;
-  const agent = {
-    import_path: "hitch_harbor_agent:HitchHarborAgent",
-    model_name: request.model || null,
-    kwargs: {
-      candidate_id: "candidate-1",
-      harness_ref: lockedHarnessRef(resolvedRevision),
-      revision_identity: resolvedRevision.identity,
-      hitch_runtime_dir: runtimeDirectory,
-      hitch_timeout_ms: request.timeout_ms,
-      agent_args: request.agent_args,
-      workdir: "/app",
-    },
-    env: credentialEnvironment(request.pass_env, env),
-    include_logs: ["hitch-*"],
-  };
-  if (timeoutSeconds !== null) agent.override_timeout_sec = timeoutSeconds + 30;
-  if (setupTimeoutSeconds !== null) agent.override_setup_timeout_sec = setupTimeoutSeconds;
-
-  return compact({
-    job_name: jobName,
-    jobs_dir: backendDirectory,
-    n_attempts: request.attempts,
-    n_concurrent_trials: request.max_concurrent,
-    environment: { type: "docker", delete: true },
-    agents: [agent],
-    datasets: [await datasetConfig(request.dataset)],
-    tasks: [],
-  });
-}
-
-export function lockedHarnessRef(resolvedRevision) {
-  if (resolvedRevision?.revision?.type === "version" && resolvedRevision.revision.version) {
-    return `${resolvedRevision.harness_id}@version:${resolvedRevision.revision.version}`;
-  }
-  if (resolvedRevision?.revision?.type === "commit" && resolvedRevision.revision.commit) {
-    return `${resolvedRevision.harness_id}@commit:${resolvedRevision.revision.commit}`;
-  }
-  throw invalidInput("Harbor eval requires a resolved exact version or commit");
-}
-
-export function normalizeHarborResult(raw) {
-  const trials = Array.isArray(raw?.trial_results) ? raw.trial_results : [];
-  const rewardValues = new Map();
-  const normalizedTrials = trials.map((trial) => {
-    const rewards = trial?.verifier_result?.rewards || {};
-    for (const [name, value] of Object.entries(rewards)) {
-      if (typeof value !== "number" || !Number.isFinite(value)) continue;
-      const values = rewardValues.get(name) || [];
-      values.push(value);
-      rewardValues.set(name, values);
-    }
-    return {
-      task_name: trial?.task_name || null,
-      trial_name: trial?.trial_name || null,
-      status: trial?.exception_info ? "errored" : "completed",
-      rewards,
-      exception: trial?.exception_info || null,
-    };
-  });
-  const rewards = Object.fromEntries([...rewardValues.entries()].sort(([left], [right]) => left.localeCompare(right)).map(([name, values]) => [name, {
-    count: values.length,
-    mean: values.reduce((sum, value) => sum + value, 0) / values.length,
-    min: Math.min(...values),
-    max: Math.max(...values),
-  }]));
-  const primary = rewards.reward || Object.values(rewards)[0] || null;
-  const stats = raw?.stats || {};
-  return {
-    n_trials: raw?.n_total_trials ?? trials.length,
-    n_completed: stats.n_completed_trials ?? normalizedTrials.filter((trial) => trial.status === "completed").length,
-    n_errored: stats.n_errored_trials ?? normalizedTrials.filter((trial) => trial.status === "errored").length,
-    n_cancelled: stats.n_cancelled_trials ?? 0,
-    primary_reward: primary?.mean ?? null,
-    rewards,
-    trials: normalizedTrials,
-  };
-}
-
-async function discoverHarbor(explicit, root, env) {
-  const located = await locateHarbor({ root, explicit, env });
-  if (!located.executable) {
-    throw new HitchError(`Harbor executable not found: ${located.requested}`, {
-      code: "harbor_unavailable",
-      exitCode: 3,
-    });
-  }
-  return located.executable;
-}
-
-async function datasetConfig(value) {
-  if (typeof value !== "string" || !value.trim()) throw invalidInput("dataset must be a non-empty string");
-  const raw = value.trim();
-  const localPath = path.resolve(raw);
-  try {
-    if ((await stat(localPath)).isDirectory()) return { path: localPath };
-  } catch (error) {
-    if (error?.code !== "ENOENT") throw error;
-  }
-  const separator = raw.lastIndexOf("@");
-  const name = separator > 0 ? raw.slice(0, separator) : raw;
-  const version = separator > 0 ? raw.slice(separator + 1) : "";
-  if (!name) throw invalidInput(`invalid Harbor dataset: ${value}`);
-  return name.includes("/")
-    ? { name, ref: version || "latest" }
-    : compact({ name, version: version || undefined });
-}
-
-function credentialEnvironment(explicitNames, env) {
-  const names = new Set(HARBOR_CREDENTIAL_ENV.filter((name) => env[name] !== undefined));
-  for (const name of explicitNames || []) {
-    if (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(name)) throw invalidInput(`invalid environment variable name: ${name}`);
-    if (env[name] === undefined) throw invalidInput(`environment variable is not set: ${name}`);
-    names.add(name);
-  }
-  return Object.fromEntries([...names].sort().map((name) => [name, `\${${name}}`]));
-}
-
-function withBridgePythonPath(env) {
-  return {
-    ...env,
-    PYTHONPATH: [BRIDGE_DIRECTORY, env.PYTHONPATH].filter(Boolean).join(path.delimiter),
-  };
-}
-
-function invokeHarbor(executable, args, { cwd, env, stdoutPath, stderrPath, signal, emit }) {
-  return new Promise((resolve, reject) => {
-    const stdout = createWriteStream(stdoutPath, { flags: "w", mode: 0o600 });
-    const stderr = createWriteStream(stderrPath, { flags: "w", mode: 0o600 });
-    const child = spawn(executable, args, {
-      cwd,
-      env,
-      stdio: ["ignore", "pipe", "pipe"],
-      detached: process.platform !== "win32",
-      windowsHide: true,
-    });
-    child.stdout.pipe(stdout);
-    child.stderr.pipe(stderr);
-    consumeLines(child.stdout, (line) => emit({ type: "eval.backend.output", stream: "stdout", text: line }));
-    consumeLines(child.stderr, (line) => emit({ type: "eval.backend.output", stream: "stderr", text: line }));
-    let settled = false;
-    const abort = () => terminateProcess(child).catch(() => {});
-    signal?.addEventListener("abort", abort, { once: true });
-    child.once("error", (error) => {
-      if (settled) return;
-      settled = true;
-      signal?.removeEventListener("abort", abort);
-      stdout.destroy();
-      stderr.destroy();
-      reject(new HitchError(`failed to launch Harbor: ${error.message}`, {
-        code: "harbor_launch_failed",
-        exitCode: 6,
-        cause: error,
-      }));
-    });
-    child.once("close", (code, processSignal) => {
-      if (settled) return;
-      settled = true;
-      signal?.removeEventListener("abort", abort);
-      Promise.all([closeWriteStream(stdout), closeWriteStream(stderr)])
-        .then(() => resolve({ code, signal: processSignal }))
-        .catch(reject);
-    });
-    if (signal?.aborted) abort();
-  });
-}
-
-function closeWriteStream(stream) {
-  return new Promise((resolve, reject) => {
-    if (stream.closed) return resolve();
-    stream.once("error", reject);
-    stream.once("close", resolve);
-    stream.end();
-  });
-}
-
-function compact(value) {
-  return Object.fromEntries(Object.entries(value).filter(([, entry]) => entry !== undefined && entry !== null));
-}

package/src/harness-reference.js

@@ -1,78 +0,0 @@
-import { fileURLToPath, pathToFileURL } from "node:url";
-import path from "node:path";
-import { invalidInput } from "./errors.js";
-
-const VERSION_PATTERN = /^\d+\.\d+\.\d+(?:-[0-9A-Za-z.-]+)?(?:\+[0-9A-Za-z.-]+)?$/;
-const COMMIT_PATTERN = /^[0-9a-fA-F]{7,64}$/;
-const HARNESS_ID_PATTERN = /^[a-z][a-z0-9-]*$/;
-
-export function parseHarnessReference(value) {
-  if (typeof value !== "string" || !value.trim()) throw invalidInput("harness reference must be a non-empty string");
-  const raw = value.trim();
-  const separator = raw.indexOf("@");
-  const harnessId = separator < 0 ? raw : raw.slice(0, separator);
-  const revision = separator < 0 ? "installed" : raw.slice(separator + 1);
-  if (!HARNESS_ID_PATTERN.test(harnessId)) throw invalidInput(`invalid harness name: ${harnessId || raw}`);
-  if (!revision) throw invalidInput(`harness reference is missing a selector: ${raw}`);
-
-  if (revision === "installed") {
-    return canonicalReference(raw, harnessId, { type: "installed" });
-  }
-  if (revision.startsWith("version:")) {
-    const version = revision.slice("version:".length);
-    if (!VERSION_PATTERN.test(version)) {
-      throw invalidInput(`version selector must be an exact semantic version: ${version || "(empty)"}`);
-    }
-    return canonicalReference(raw, harnessId, { type: "version", value: version });
-  }
-  if (revision.startsWith("commit:")) {
-    const commit = revision.slice("commit:".length);
-    assertCommit(commit);
-    return canonicalReference(raw, harnessId, { type: "commit", value: commit.toLowerCase() });
-  }
-  if (revision.startsWith("git+")) {
-    const hash = revision.lastIndexOf("#");
-    if (hash < 0) throw invalidInput("explicit Git references require #<commit>");
-    const sourceValue = revision.slice("git+".length, hash);
-    const commit = revision.slice(hash + 1);
-    assertCommit(commit);
-    let source;
-    try {
-      source = new URL(sourceValue);
-    } catch (error) {
-      throw invalidInput(`invalid Git source URL: ${sourceValue}`, { cause: error });
-    }
-    if (source.protocol !== "file:") {
-      throw invalidInput("explicit Git sources are limited to file:// URLs; use the registered source for remote commits");
-    }
-    const localPath = path.resolve(fileURLToPath(source));
-    const sourceUrl = pathToFileURL(localPath).href;
-    return canonicalReference(raw, harnessId, {
-      type: "commit",
-      value: commit.toLowerCase(),
-      source: { type: "git", url: sourceUrl, local_path: localPath, explicit: true },
-    });
-  }
-
-  throw invalidInput(`unsupported harness selector in ${raw}; use installed, version:<exact>, or commit:<sha>`);
-}
-
-function canonicalReference(raw, harnessId, selector) {
-  const suffix = selector.type === "installed"
-    ? "installed"
-    : selector.source
-      ? `git+${selector.source.url}#${selector.value}`
-      : `${selector.type}:${selector.value}`;
-  return {
-    raw,
-    canonical: `${harnessId}@${suffix}`,
-    harness_id: harnessId,
-    selector,
-  };
-}
-
-function assertCommit(value) {
-  if (!COMMIT_PATTERN.test(value || "")) {
-    throw invalidInput(`commit selector must be a 7-64 character hexadecimal ID: ${value || "(empty)"}`);
-  }
-}

package/src/line-stream.js

@@ -1,17 +0,0 @@
-export function consumeLines(stream, onLine) {
-  let buffer = "";
-  stream.setEncoding("utf8");
-  stream.on("data", (chunk) => {
-    buffer += chunk;
-    for (;;) {
-      const newline = buffer.indexOf("\n");
-      if (newline < 0) break;
-      const line = buffer.slice(0, newline).replace(/\r$/, "");
-      buffer = buffer.slice(newline + 1);
-      onLine(line);
-    }
-  });
-  stream.on("end", () => {
-    if (buffer.length > 0) onLine(buffer.replace(/\r$/, ""));
-  });
-}

package/src/locks.js

@@ -1,33 +0,0 @@
-import { randomBytes } from "node:crypto";
-import { open, readFile, rm } from "node:fs/promises";
-
-// Serializes stale-lock reclamation so an observation about an old owner can
-// never be used to remove a lock that a new owner has just acquired.
-export async function reclaimStaleLock(file, isStale) {
-  const guard = `${file}.reclaim`;
-  const owner = randomBytes(12).toString("hex");
-  let handle;
-  try {
-    handle = await open(guard, "wx", 0o600);
-  } catch (error) {
-    if (error?.code === "EEXIST") return false;
-    throw error;
-  }
-
-  try {
-    await handle.writeFile(`${JSON.stringify({ owner, pid: process.pid, created_at: new Date().toISOString() })}\n`);
-    await handle.sync();
-    if (!await isStale(file)) return false;
-    await rm(file, { force: true });
-    return true;
-  } finally {
-    await handle.close().catch(() => {});
-    let current;
-    try {
-      current = JSON.parse(await readFile(guard, "utf8"));
-    } catch {
-      current = null;
-    }
-    if (current?.owner === owner) await rm(guard, { force: true });
-  }
-}

package/src/process.js

@@ -1,49 +0,0 @@
-import { spawn } from "node:child_process";
-
-export async function terminateProcess(child, graceMs = 3_000) {
-  if (!child || child.exitCode !== null || child.signalCode !== null) return;
-
-  if (process.platform === "win32") {
-    const killer = spawn("taskkill", ["/pid", String(child.pid), "/t", "/f"], {
-      stdio: "ignore",
-      windowsHide: true,
-    });
-    await waitForExitOrTimeout(killer, graceMs);
-    return;
-  }
-
-  try {
-    process.kill(-child.pid, "SIGTERM");
-  } catch (error) {
-    if (error?.code !== "ESRCH") throw error;
-    return;
-  }
-
-  await waitForExitOrTimeout(child, graceMs);
-  if (child.exitCode === null && child.signalCode === null) {
-    try {
-      process.kill(-child.pid, "SIGKILL");
-    } catch (error) {
-      if (error?.code !== "ESRCH") throw error;
-    }
-  }
-}
-
-export function delay(milliseconds) {
-  return new Promise((resolve) => setTimeout(resolve, milliseconds));
-}
-
-function waitForExitOrTimeout(child, milliseconds) {
-  return new Promise((resolve) => {
-    let settled = false;
-    const finish = () => {
-      if (settled) return;
-      settled = true;
-      clearTimeout(timer);
-      child.removeListener("exit", finish);
-      resolve();
-    };
-    const timer = setTimeout(finish, milliseconds);
-    child.once("exit", finish);
-  });
-}

package/src/registry.js

@@ -1,84 +0,0 @@
-import { constants, createReadStream } from "node:fs";
-import { access, lstat, realpath } from "node:fs/promises";
-import { delimiter, isAbsolute, join, resolve } from "node:path";
-import { spawn } from "node:child_process";
-import { createHash } from "node:crypto";
-import { getAdapter, listDefinitions, publicDefinition } from "./adapters.js";
-
-export async function discoverAgents({ env = process.env, probeVersions = true } = {}) {
-  return Promise.all(listDefinitions().map((definition) => inspectDefinition(definition, { env, probeVersions })));
-}
-
-export async function inspectAgent(id, { env = process.env, probeVersions = true } = {}) {
-  const definition = publicDefinition(getAdapter(id));
-  return inspectDefinition(definition, { env, probeVersions });
-}
-
-async function inspectDefinition(definition, { env, probeVersions }) {
-  const configured = env[definition.path_env]?.trim() || definition.command;
-  const executable = await resolveExecutable(configured, env.PATH || "");
-  if (!executable) return { ...definition, status: "unavailable" };
-
-  const version = probeVersions
-    ? await detectVersion(executable, getAdapter(definition.id).version_args)
-    : "";
-  const identity = await fingerprintExecutable(executable);
-  return { ...definition, status: "available", executable, version, identity };
-}
-
-export async function resolveExecutable(command, searchPath) {
-  const candidates = isAbsolute(command) || command.includes("/") || command.includes("\\")
-    ? [resolve(command)]
-    : searchPath.split(delimiter).filter(Boolean).map((directory) => join(directory, command));
-
-  for (const candidate of candidates) {
-    try {
-      await access(candidate, constants.X_OK);
-      const info = await lstat(candidate);
-      if (info.isDirectory()) continue;
-      return await realpath(candidate);
-    } catch {
-      // Keep probing PATH entries.
-    }
-  }
-  return null;
-}
-
-export async function detectVersion(executable, args, timeoutMs = 5_000) {
-  return new Promise((resolveVersion) => {
-    const child = spawn(executable, args, { stdio: ["ignore", "pipe", "pipe"], windowsHide: true });
-    let stdout = "";
-    let stderr = "";
-    const timer = setTimeout(() => child.kill("SIGKILL"), timeoutMs);
-    child.stdout.on("data", (chunk) => { stdout += chunk; });
-    child.stderr.on("data", (chunk) => { stderr += chunk; });
-    child.once("error", () => {
-      clearTimeout(timer);
-      resolveVersion("");
-    });
-    child.once("close", () => {
-      clearTimeout(timer);
-      resolveVersion(selectVersionLine(stdout, stderr));
-    });
-  });
-}
-
-export async function fingerprintExecutable(executable) {
-  const hash = createHash("sha256");
-  for await (const chunk of createReadStream(executable)) hash.update(chunk);
-  const digest = hash.digest("hex");
-  return `sha256:${digest}`;
-}
-
-export function selectVersionLine(stdout, stderr) {
-  const streams = [stdout, stderr];
-  const versionPattern = /(?:^|\s)v?\d+\.\d+\.\d+(?:[-+][0-9A-Za-z.-]+)?(?:\s|$)/;
-  for (const stream of streams) {
-    const match = String(stream || "")
-      .split(/\r?\n/)
-      .map((line) => line.trim())
-      .find((line) => versionPattern.test(line));
-    if (match) return match;
-  }
-  return "";
-}