lastlight-evals 0.2.1 → 0.3.0

package/dist/seed.js

@@ -2,16 +2,25 @@
  * Deterministic workspace seeding for the code-fix tier.
  *
  * Pre-populates the run's sandbox workspace (`<stateDir>/sandboxes/<taskId>`,
- * the exact dir `setupTaskWorktree` would create) with a fixture repo checked
- * out at a fixed base commit, and points its `origin` at a LOCAL bare repo —
- * so the real workflow's `git push origin HEAD` succeeds fully offline with NO
- * GitHub clone. Because the eval calls `runWorkflow` with no
- * `ctx.prePopulateBranch`, the runner never triggers its own GitHub clone and
- * the agent works directly in this seeded dir.
+ * the exact dir `setupTaskWorktree` would create) with a repo checked out at a
+ * base commit, and points its `origin` at a LOCAL bare repo — so the real
+ * workflow's `git push origin HEAD` succeeds fully offline with NO GitHub clone.
+ * Because the eval calls `runWorkflow` with no `ctx.prePopulateBranch`, the
+ * runner never triggers its own GitHub clone and the agent works directly in
+ * this seeded dir.
+ *
+ * Two provenances, same end state:
+ *   - {@link seedWorkspace}        — a vendored fixture dir (`repos/<id>/`).
+ *   - {@link seedWorkspaceFromGit} — a real repo cloned into a repo-local cache
+ *                                    and checked out at `base_commit`.
+ *
+ * The git-source clone is a HARNESS SETUP action, not the workflow cloning
+ * GitHub — it touches the network only on a cache miss; the workflow itself
+ * still operates on a pre-seeded dir with an offline `file://` origin.
  */
 import { execFileSync } from "node:child_process";
-import { mkdirSync, cpSync } from "node:fs";
-import { resolve } from "node:path";
+import { mkdirSync, cpSync, existsSync } from "node:fs";
+import { resolve, dirname } from "node:path";
 const FIXED = "2026-01-01T00:00:00 +0000";
 const GIT_ENV = {
     ...process.env,
@@ -25,6 +34,30 @@ const GIT_ENV = {
 function git(cwd, args) {
     return execFileSync("git", args, { cwd, env: GIT_ENV, stdio: ["ignore", "pipe", "pipe"] }).toString();
 }
+/** Where git-source repos are mirrored. Repo-local (NOT `~`), gitignored —
+ * overridable with `LASTLIGHT_EVALS_CACHE`. */
+export function resolveCacheDir(override) {
+    const root = override ?? process.env.LASTLIGHT_EVALS_CACHE ?? resolve(process.cwd(), ".eval-cache");
+    return resolve(root, "repos");
+}
+/** Point `workDir`'s `origin` at a fresh LOCAL bare repo and push the current
+ * HEAD as the default branch, so the workflow can `git push` fully offline. */
+function setupOfflineOrigin(workDir, stateDir, taskId, def) {
+    const originsDir = resolve(stateDir, "origins");
+    mkdirSync(originsDir, { recursive: true });
+    const originDir = resolve(originsDir, `${taskId}.git`);
+    git(workDir, ["init", "--bare", "-q", originDir]);
+    // A git-source clone already has an `origin` (the cache) — replace it.
+    try {
+        git(workDir, ["remote", "remove", "origin"]);
+    }
+    catch {
+        /* no existing origin (fixture path) — fine */
+    }
+    git(workDir, ["remote", "add", "origin", `file://${originDir}`]);
+    git(workDir, ["push", "-q", "origin", `HEAD:refs/heads/${def}`]);
+    return originDir;
+}
 export function seedWorkspace(opts) {
     const def = opts.defaultBranch ?? "main";
     const sandboxBase = resolve(opts.stateDir, "sandboxes");
@@ -35,15 +68,66 @@ export function seedWorkspace(opts) {
     git(workDir, ["add", "-A"]);
     git(workDir, ["commit", "-q", "-m", "base"]);
     const baseCommit = git(workDir, ["rev-parse", "HEAD"]).trim();
-    const originsDir = resolve(opts.stateDir, "origins");
-    mkdirSync(originsDir, { recursive: true });
-    const originDir = resolve(originsDir, `${opts.taskId}.git`);
-    git(workDir, ["init", "--bare", "-q", originDir]);
-    git(workDir, ["remote", "add", "origin", `file://${originDir}`]);
-    git(workDir, ["push", "-q", "origin", `HEAD:refs/heads/${def}`]);
+    const originDir = setupOfflineOrigin(workDir, opts.stateDir, opts.taskId, def);
     const branch = opts.branch ?? def;
     if (branch !== def)
         git(workDir, ["checkout", "-q", "-b", branch]);
     return { workDir, originDir, baseCommit, branch };
 }
+/** True if `sha` is a 40-hex non-zero commit id (a real git-source base). */
+export function isRealSha(sha) {
+    return !!sha && /^[0-9a-f]{40}$/i.test(sha) && !/^0+$/.test(sha);
+}
+/** Ensure a repo-local bare mirror of `repo` exists and contains `baseCommit`
+ * (clone on miss, fetch if the commit is absent). Returns the cache dir. Run
+ * this SERIALLY per repo before a parallel batch — concurrent clones of the same
+ * repo race. Network is touched only here, only on a miss. */
+export function ensureRepoCache(opts) {
+    const [owner, name] = opts.repo.split("/");
+    if (!owner || !name)
+        throw new Error(`seedWorkspaceFromGit: repo must be "owner/name", got "${opts.repo}"`);
+    const cacheDir = resolveCacheDir(opts.cacheDir);
+    const mirror = resolve(cacheDir, `${owner}__${name}.git`);
+    if (!existsSync(mirror)) {
+        mkdirSync(dirname(mirror), { recursive: true });
+        git(dirname(mirror), ["clone", "--bare", "--quiet", `https://github.com/${owner}/${name}.git`, mirror]);
+    }
+    // Fetch only if the wanted commit isn't already in the mirror.
+    if (opts.baseCommit) {
+        const present = (() => {
+            try {
+                git(mirror, ["cat-file", "-e", `${opts.baseCommit}^{commit}`]);
+                return true;
+            }
+            catch {
+                return false;
+            }
+        })();
+        if (!present)
+            git(mirror, ["fetch", "--quiet", "origin", "+refs/heads/*:refs/heads/*"]);
+    }
+    return mirror;
+}
+/**
+ * Seed the sandbox from a real GitHub repo at `baseCommit`, with the same offline
+ * end state as {@link seedWorkspace}: a checked-out base, a feature branch, and a
+ * local bare `origin` to push to. Uses the repo-local mirror from
+ * {@link ensureRepoCache} so per-run checkout is offline and parallel-safe.
+ */
+export function seedWorkspaceFromGit(opts) {
+    const def = opts.defaultBranch ?? "main";
+    const mirror = ensureRepoCache({ repo: opts.repo, baseCommit: opts.baseCommit, cacheDir: opts.cacheDir });
+    const sandboxBase = resolve(opts.stateDir, "sandboxes");
+    const workDir = resolve(sandboxBase, opts.taskId);
+    mkdirSync(sandboxBase, { recursive: true });
+    // Plain local clone (not --shared) so the sandbox owns its objects/refs and
+    // parallel runs never touch the cache's object store.
+    git(sandboxBase, ["clone", "--quiet", `file://${mirror}`, workDir]);
+    git(workDir, ["checkout", "--quiet", "--detach", opts.baseCommit]);
+    const originDir = setupOfflineOrigin(workDir, opts.stateDir, opts.taskId, def);
+    const branch = opts.branch ?? def;
+    if (branch !== def)
+        git(workDir, ["checkout", "-q", "-b", branch]);
+    return { workDir, originDir, baseCommit: opts.baseCommit, branch };
+}
 //# sourceMappingURL=seed.js.map

package/dist/seed.js.map

@@ -1 +1 @@
-{"version":3,"file":"seed.js","sourceRoot":"","sources":["../src/seed.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,SAAS,CAAC;AAC5C,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAEpC,MAAM,KAAK,GAAG,2BAA2B,CAAC;AAC1C,MAAM,OAAO,GAAsB;IACjC,GAAG,OAAO,CAAC,GAAG;IACd,eAAe,EAAE,MAAM;IACvB,gBAAgB,EAAE,kBAAkB;IACpC,kBAAkB,EAAE,MAAM;IAC1B,mBAAmB,EAAE,kBAAkB;IACvC,eAAe,EAAE,KAAK;IACtB,kBAAkB,EAAE,KAAK;CAC1B,CAAC;AAEF,SAAS,GAAG,CAAC,GAAW,EAAE,IAAc;IACtC,OAAO,YAAY,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE,GAAG,EAAE,GAAG,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC;AACxG,CAAC;AASD,MAAM,UAAU,aAAa,CAAC,IAQ7B;IACC,MAAM,GAAG,GAAG,IAAI,CAAC,aAAa,IAAI,MAAM,CAAC;IACzC,MAAM,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC,QAAQ,EAAE,WAAW,CAAC,CAAC;IACxD,MAAM,OAAO,GAAG,OAAO,CAAC,WAAW,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;IAClD,SAAS,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACxC,MAAM,CAAC,IAAI,CAAC,UAAU,EAAE,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAEtD,GAAG,CAAC,OAAO,EAAE,CAAC,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,GAAG,CAAC,CAAC,CAAC;IACxC,GAAG,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC,CAAC;IAC5B,GAAG,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC;IAC7C,MAAM,UAAU,GAAG,GAAG,CAAC,OAAO,EAAE,CAAC,WAAW,EAAE,MAAM,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IAE9D,MAAM,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC;IACrD,SAAS,CAAC,UAAU,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAC3C,MAAM,SAAS,GAAG,OAAO,CAAC,UAAU,EAAE,GAAG,IAAI,CAAC,MAAM,MAAM,CAAC,CAAC;IAC5D,GAAG,CAAC,OAAO,EAAE,CAAC,MAAM,EAAE,QAAQ,EAAE,IAAI,EAAE,SAAS,CAAC,CAAC,CAAC;IAClD,GAAG,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE,KAAK,EAAE,QAAQ,EAAE,UAAU,SAAS,EAAE,CAAC,CAAC,CAAC;IACjE,GAAG,CAAC,OAAO,EAAE,CAAC,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,mBAAmB,GAAG,EAAE,CAAC,CAAC,CAAC;IAEjE,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,IAAI,GAAG,CAAC;IAClC,IAAI,MAAM,KAAK,GAAG;QAAE,GAAG,CAAC,OAAO,EAAE,CAAC,UAAU,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC;IAEnE,OAAO,EAAE,OAAO,EAAE,SAAS,EAAE,UAAU,EAAE,MAAM,EAAE,CAAC;AACpD,CAAC"}
+{"version":3,"file":"seed.js","sourceRoot":"","sources":["../src/seed.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;GAmBG;AAEH,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AACxD,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAE7C,MAAM,KAAK,GAAG,2BAA2B,CAAC;AAC1C,MAAM,OAAO,GAAsB;IACjC,GAAG,OAAO,CAAC,GAAG;IACd,eAAe,EAAE,MAAM;IACvB,gBAAgB,EAAE,kBAAkB;IACpC,kBAAkB,EAAE,MAAM;IAC1B,mBAAmB,EAAE,kBAAkB;IACvC,eAAe,EAAE,KAAK;IACtB,kBAAkB,EAAE,KAAK;CAC1B,CAAC;AAEF,SAAS,GAAG,CAAC,GAAW,EAAE,IAAc;IACtC,OAAO,YAAY,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE,GAAG,EAAE,GAAG,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC;AACxG,CAAC;AASD;+CAC+C;AAC/C,MAAM,UAAU,eAAe,CAAC,QAAiB;IAC/C,MAAM,IAAI,GAAG,QAAQ,IAAI,OAAO,CAAC,GAAG,CAAC,qBAAqB,IAAI,OAAO,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,aAAa,CAAC,CAAC;IACpG,OAAO,OAAO,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;AAChC,CAAC;AAED;+EAC+E;AAC/E,SAAS,kBAAkB,CAAC,OAAe,EAAE,QAAgB,EAAE,MAAc,EAAE,GAAW;IACxF,MAAM,UAAU,GAAG,OAAO,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC;IAChD,SAAS,CAAC,UAAU,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAC3C,MAAM,SAAS,GAAG,OAAO,CAAC,UAAU,EAAE,GAAG,MAAM,MAAM,CAAC,CAAC;IACvD,GAAG,CAAC,OAAO,EAAE,CAAC,MAAM,EAAE,QAAQ,EAAE,IAAI,EAAE,SAAS,CAAC,CAAC,CAAC;IAClD,uEAAuE;IACvE,IAAI,CAAC;QACH,GAAG,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAC;IAC/C,CAAC;IAAC,MAAM,CAAC;QACP,8CAA8C;IAChD,CAAC;IACD,GAAG,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE,KAAK,EAAE,QAAQ,EAAE,UAAU,SAAS,EAAE,CAAC,CAAC,CAAC;IACjE,GAAG,CAAC,OAAO,EAAE,CAAC,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,mBAAmB,GAAG,EAAE,CAAC,CAAC,CAAC;IACjE,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,IAQ7B;IACC,MAAM,GAAG,GAAG,IAAI,CAAC,aAAa,IAAI,MAAM,CAAC;IACzC,MAAM,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC,QAAQ,EAAE,WAAW,CAAC,CAAC;IACxD,MAAM,OAAO,GAAG,OAAO,CAAC,WAAW,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;IAClD,SAAS,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACxC,MAAM,CAAC,IAAI,CAAC,UAAU,EAAE,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAEtD,GAAG,CAAC,OAAO,EAAE,CAAC,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,GAAG,CAAC,CAAC,CAAC;IACxC,GAAG,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC,CAAC;IAC5B,GAAG,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC;IAC7C,MAAM,UAAU,GAAG,GAAG,CAAC,OAAO,EAAE,CAAC,WAAW,EAAE,MAAM,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IAE9D,MAAM,SAAS,GAAG,kBAAkB,CAAC,OAAO,EAAE,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;IAE/E,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,IAAI,GAAG,CAAC;IAClC,IAAI,MAAM,KAAK,GAAG;QAAE,GAAG,CAAC,OAAO,EAAE,CAAC,UAAU,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC;IAEnE,OAAO,EAAE,OAAO,EAAE,SAAS,EAAE,UAAU,EAAE,MAAM,EAAE,CAAC;AACpD,CAAC;AAED,6EAA6E;AAC7E,MAAM,UAAU,SAAS,CAAC,GAAuB;IAC/C,OAAO,CAAC,CAAC,GAAG,IAAI,iBAAiB,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AACnE,CAAC;AAED;;;8DAG8D;AAC9D,MAAM,UAAU,eAAe,CAAC,IAA8D;IAC5F,MAAM,CAAC,KAAK,EAAE,IAAI,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IAC3C,IAAI,CAAC,KAAK,IAAI,CAAC,IAAI;QAAE,MAAM,IAAI,KAAK,CAAC,yDAAyD,IAAI,CAAC,IAAI,GAAG,CAAC,CAAC;IAC5G,MAAM,QAAQ,GAAG,eAAe,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IAChD,MAAM,MAAM,GAAG,OAAO,CAAC,QAAQ,EAAE,GAAG,KAAK,KAAK,IAAI,MAAM,CAAC,CAAC;IAE1D,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC;QACxB,SAAS,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAChD,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,sBAAsB,KAAK,IAAI,IAAI,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;IAC1G,CAAC;IACD,+DAA+D;IAC/D,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;QACpB,MAAM,OAAO,GAAG,CAAC,GAAG,EAAE;YACpB,IAAI,CAAC;gBACH,GAAG,CAAC,MAAM,EAAE,CAAC,UAAU,EAAE,IAAI,EAAE,GAAG,IAAI,CAAC,UAAU,WAAW,CAAC,CAAC,CAAC;gBAC/D,OAAO,IAAI,CAAC;YACd,CAAC;YAAC,MAAM,CAAC;gBACP,OAAO,KAAK,CAAC;YACf,CAAC;QACH,CAAC,CAAC,EAAE,CAAC;QACL,IAAI,CAAC,OAAO;YAAE,GAAG,CAAC,MAAM,EAAE,CAAC,OAAO,EAAE,SAAS,EAAE,QAAQ,EAAE,4BAA4B,CAAC,CAAC,CAAC;IAC1F,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,oBAAoB,CAAC,IAQpC;IACC,MAAM,GAAG,GAAG,IAAI,CAAC,aAAa,IAAI,MAAM,CAAC;IACzC,MAAM,MAAM,GAAG,eAAe,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,UAAU,EAAE,IAAI,CAAC,UAAU,EAAE,QAAQ,EAAE,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC;IAE1G,MAAM,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC,QAAQ,EAAE,WAAW,CAAC,CAAC;IACxD,MAAM,OAAO,GAAG,OAAO,CAAC,WAAW,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;IAClD,SAAS,CAAC,WAAW,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAE5C,4EAA4E;IAC5E,sDAAsD;IACtD,GAAG,CAAC,WAAW,EAAE,CAAC,OAAO,EAAE,SAAS,EAAE,UAAU,MAAM,EAAE,EAAE,OAAO,CAAC,CAAC,CAAC;IACpE,GAAG,CAAC,OAAO,EAAE,CAAC,UAAU,EAAE,SAAS,EAAE,UAAU,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC;IAEnE,MAAM,SAAS,GAAG,kBAAkB,CAAC,OAAO,EAAE,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;IAE/E,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,IAAI,GAAG,CAAC;IAClC,IAAI,MAAM,KAAK,GAAG;QAAE,GAAG,CAAC,OAAO,EAAE,CAAC,UAAU,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC;IAEnE,OAAO,EAAE,OAAO,EAAE,SAAS,EAAE,UAAU,EAAE,IAAI,CAAC,UAAU,EAAE,MAAM,EAAE,CAAC;AACrE,CAAC"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "lastlight-evals",
-  "version": "0.2.1",
+  "version": "0.3.0",
   "description": "Eval harness for Last Light workflows — drives the real production workflows against a mocked GitHub and grades deterministically (SWE-bench compatible).",
   "type": "module",
   "bin": {
@@ -43,7 +43,7 @@
   "dependencies": {
     "@clack/prompts": "^1.2.0",
     "chalk": "^5.6.2",
-    "lastlight": "^0.5.0",
+    "lastlight": "^0.7.2",
     "yaml": "^2.5.0"
   },
   "peerDependencies": {