dtu-github-actions 0.7.0 → 0.8.0

package/dist/server/routes/actions/action-tarball.test.js

@@ -188,6 +188,29 @@ describe("Action Tarball Cache", () => {
         const data = await res.json();
         expect(data.actions).toEqual({});
     });
+    it("should skip local actions without crashing", async () => {
+        const baseUrl = `http://localhost:${PORT}`;
+        const res = await fetch(`${baseUrl}/_apis/distributedtask/hubs/Hub/plans/Plan/actiondownloadinfo`, {
+            method: "POST",
+            headers: { "Content-Type": "application/json" },
+            body: JSON.stringify({
+                actions: [
+                    { nameWithOwner: "./.github/actions/shared-node-cache", ref: "" },
+                    { nameWithOwner: "", ref: "" },
+                    { nameWithOwner: "actions/checkout", ref: "v4" },
+                ],
+            }),
+        });
+        expect(res.status).toBe(200);
+        const data = await res.json();
+        // Local and empty actions should be skipped
+        expect(data.actions["./.github/actions/shared-node-cache@"]).toBeUndefined();
+        expect(data.actions["@"]).toBeUndefined();
+        // Remote action should still be present
+        const checkoutInfo = data.actions["actions/checkout@v4"];
+        expect(checkoutInfo).toBeDefined();
+        expect(checkoutInfo.tarballUrl).toBe(`${baseUrl}/_dtu/action-tarball/actions/checkout/v4`);
+    });
 });
 // ── writeStepOutputLines group filtering ─────────────────────────────────────
 // The writeStepOutputLines function is internal to registerActionRoutes, so we

package/dist/server/routes/actions/generators.js

@@ -38,6 +38,19 @@ export function toContextData(obj) {
 // TemplateTokenJsonConverter uses "type" key (integer) NOT the contextData "t" key.
 // TokenType.Mapping = 2. Items are serialized as {Key: scalarToken, Value: templateToken}.
 // Strings without file/line/col are serialized as bare string values.
+/**
+ * Convert a string value to the appropriate TemplateToken.
+ * If the value is a pure `${{ expr }}` expression, encode it as a
+ * BasicExpressionToken (type 6) so the runner evaluates it at execution time.
+ * Otherwise, return a bare string (StringToken).
+ */
+function toTemplateTokenValue(v) {
+    const exprMatch = v.match(/^\$\{\{\s*([\s\S]+?)\s*\}\}$/);
+    if (exprMatch) {
+        return { type: 3, expr: exprMatch[1] };
+    }
+    return v;
+}
 export function toTemplateTokenMapping(obj) {
     const entries = Object.entries(obj);
     if (entries.length === 0) {
@@ -45,7 +58,7 @@ export function toTemplateTokenMapping(obj) {
     }
     return {
         type: 2,
-        map: entries.map(([k, v]) => ({ Key: k, Value: v })),
+        map: entries.map(([k, v]) => ({ Key: k, Value: toTemplateTokenValue(v) })),
     };
 }
 /**
@@ -96,6 +109,7 @@ export function createJobResponse(jobId, payload, baseUrl, planId) {
             id: step.Id || step.id || crypto.randomUUID(),
             name: step.Name || step.name || `step-${index}`,
             displayName: step.DisplayName || step.Name || step.name || `step-${index}`,
+            contextName: step.ContextName || step.contextName || undefined,
             type: (step.Type || "Action").toLowerCase(),
             reference: (() => {
                 const refTypeSource = step.Reference?.Type || "Script";
@@ -128,6 +142,12 @@ export function createJobResponse(jobId, payload, baseUrl, planId) {
     const ownerName = payload.repository?.owner?.login || "redwoodjs";
     const repoName = payload.repository?.name || repoFullName.split("/")[1] || "";
     const workspacePath = `/home/runner/_work/${repoName}/${repoName}`;
+    const headSha = payload.headSha && payload.headSha !== "HEAD"
+        ? payload.headSha
+        : "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa";
+    // realHeadSha is the actual HEAD commit SHA, even when headSha is unset
+    // (dirty workspace mode). Used for push event context (before/after).
+    const realHeadSha = payload.realHeadSha || headSha;
     const Variables = {
         // Standard GitHub Actions environment variables — always set by real runners.
         // CI=true is required by many scripts that branch on CI vs local (e.g. default DB_HOST).
@@ -145,6 +165,8 @@ export function createJobResponse(jobId, payload, baseUrl, planId) {
         GITHUB_RUN_NUMBER: { Value: "1", IsSecret: false },
         GITHUB_JOB: { Value: payload.name || "local-job", IsSecret: false },
         GITHUB_EVENT_NAME: { Value: "push", IsSecret: false },
+        GITHUB_API_URL: { Value: baseUrl, IsSecret: false },
+        GITHUB_SERVER_URL: { Value: "https://github.com", IsSecret: false },
         GITHUB_REF_NAME: { Value: "main", IsSecret: false },
         GITHUB_WORKFLOW: { Value: payload.workflowName || "local-workflow", IsSecret: false },
         GITHUB_WORKSPACE: { Value: workspacePath, IsSecret: false },
@@ -154,30 +176,24 @@ export function createJobResponse(jobId, payload, baseUrl, planId) {
         "system.github.repository": { Value: repoFullName, IsSecret: false },
         "github.repository": { Value: repoFullName, IsSecret: false },
         "github.actor": { Value: ownerName, IsSecret: false },
-        "github.sha": {
-            Value: payload.headSha && payload.headSha !== "HEAD"
-                ? payload.headSha
-                : "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
-            IsSecret: false,
-        },
+        "github.sha": { Value: realHeadSha, IsSecret: false },
         "github.ref": { Value: "refs/heads/main", IsSecret: false },
         repository: { Value: repoFullName, IsSecret: false },
         GITHUB_REPOSITORY: { Value: repoFullName, IsSecret: false },
         GITHUB_ACTOR: { Value: ownerName, IsSecret: false },
-        GITHUB_SHA: {
-            Value: payload.headSha && payload.headSha !== "HEAD"
-                ? payload.headSha
-                : "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
-            IsSecret: false,
-        },
+        GITHUB_SHA: { Value: realHeadSha, IsSecret: false },
         "build.repository.name": { Value: repoFullName, IsSecret: false },
         "build.repository.uri": { Value: `https://github.com/${repoFullName}`, IsSecret: false },
     };
-    // Merge all step-level env: vars into the job Variables.
+    // Merge job-level env: into Variables first, then step-level env: (step wins on conflict).
     // The runner exports every Variable as a process env var for all steps, so this is the
-    // reliable mechanism to get DB_HOST=mysql, DB_PORT=3306 etc. into the step subprocess.
-    // (Step-scoped env would require full template compilation; job-level Variables are sufficient
-    //  for DB credentials / CI flags that are consistent across steps.)
+    // reliable mechanism to get AGENT_CI_LOCAL, DB_HOST, DB_PORT etc. into the step subprocess
+    // and into the runner's expression engine (${{ env.AGENT_CI_LOCAL }}).
+    if (payload.env && typeof payload.env === "object") {
+        for (const [key, val] of Object.entries(payload.env)) {
+            Variables[key] = { Value: String(val), IsSecret: false };
+        }
+    }
     for (const step of payload.steps || []) {
         if (step.Env && typeof step.Env === "object") {
             for (const [key, val] of Object.entries(step.Env)) {
@@ -188,12 +204,11 @@ export function createJobResponse(jobId, payload, baseUrl, planId) {
     const githubContext = {
         repository: repoFullName,
         actor: ownerName,
-        sha: payload.headSha && payload.headSha !== "HEAD"
-            ? payload.headSha
-            : "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
+        sha: realHeadSha,
         ref: "refs/heads/main",
+        event_name: "push",
         server_url: "https://github.com",
-        api_url: `${baseUrl}/_apis`,
+        api_url: `${baseUrl}`,
         graphql_url: `${baseUrl}/_graphql`,
         workspace: workspacePath,
         action: "__run",
@@ -212,16 +227,19 @@ export function createJobResponse(jobId, payload, baseUrl, planId) {
                 name: repoName,
                 owner: { login: ownerName },
             },
+            before: payload.baseSha || "0000000000000000000000000000000000000000",
+            after: realHeadSha,
         };
     }
-    // Collect env vars from all steps (job-level env context seen by the runner's expression engine).
-    // Step-level `env:` blocks in the workflow YAML need to be exposed via ContextData.env so the
-    // runner can evaluate them. We merge all step envs — slightly broader than per-step scoping but
-    // correct for typical use (DB_HOST, DB_PORT, CI flags etc.).
-    const mergedStepEnv = {};
+    // Collect env vars from job-level and all steps (seen by the runner's expression engine).
+    // Job-level env is applied first, then step-level env wins on conflict.
+    const mergedEnv = {};
+    if (payload.env && typeof payload.env === "object") {
+        Object.assign(mergedEnv, payload.env);
+    }
     for (const step of payload.steps || []) {
         if (step.Env) {
-            Object.assign(mergedStepEnv, step.Env);
+            Object.assign(mergedEnv, step.Env);
         }
     }
     const ContextData = {
@@ -230,9 +248,9 @@ export function createJobResponse(jobId, payload, baseUrl, planId) {
         needs: { t: 2, d: [] }, // Empty needs context
         strategy: { t: 2, d: [] }, // Empty strategy context
         matrix: { t: 2, d: [] }, // Empty matrix context
-        // env context: merged from all step-level env: blocks so the runner's expression engine
-        // can substitute ${{ env.DB_HOST }} etc. during step execution.
-        ...(Object.keys(mergedStepEnv).length > 0 ? { env: toContextData(mergedStepEnv) } : {}),
+        // env context: merged from job-level + step-level env: blocks so the runner's expression
+        // engine can substitute ${{ env.AGENT_CI_LOCAL }}, ${{ env.DB_HOST }} etc.
+        ...(Object.keys(mergedEnv).length > 0 ? { env: toContextData(mergedEnv) } : {}),
     };
     const generatedJobId = crypto.randomUUID();
     const mockToken = createMockJwt(planId, generatedJobId);
@@ -303,7 +321,7 @@ export function createJobResponse(jobId, payload, baseUrl, planId) {
         // EnvironmentVariables is IList<TemplateToken> in the runner — each element is a MappingToken.
         // The runner evaluates each MappingToken and merges into Global.EnvironmentVariables (last wins),
         // which then populates ExpressionValues["env"] → subprocess env vars.
-        EnvironmentVariables: Object.keys(mergedStepEnv).length > 0 ? [toTemplateTokenMapping(mergedStepEnv)] : [],
+        EnvironmentVariables: Object.keys(mergedEnv).length > 0 ? [toTemplateTokenMapping(mergedEnv)] : [],
     };
     return {
         MessageId: 1,

package/dist/server/routes/actions/generators.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/server/routes/actions/generators.test.js

@@ -0,0 +1,71 @@
+import { describe, it, expect } from "vitest";
+import { createJobResponse } from "./generators.js";
+describe("createJobResponse", () => {
+    const basePayload = {
+        id: "1",
+        name: "test-job",
+        githubRepo: "owner/repo",
+        steps: [],
+    };
+    it("propagates job-level env into Variables", () => {
+        const payload = {
+            ...basePayload,
+            env: { AGENT_CI_LOCAL: "true", MY_VAR: "hello" },
+        };
+        const response = createJobResponse("1", payload, "http://localhost:3000", "plan-1");
+        const body = JSON.parse(response.Body);
+        const vars = body.Variables;
+        expect(vars.AGENT_CI_LOCAL).toEqual({ Value: "true", IsSecret: false });
+        expect(vars.MY_VAR).toEqual({ Value: "hello", IsSecret: false });
+    });
+    it("propagates job-level env into ContextData.env", () => {
+        const payload = {
+            ...basePayload,
+            env: { AGENT_CI_LOCAL: "true" },
+        };
+        const response = createJobResponse("1", payload, "http://localhost:3000", "plan-1");
+        const body = JSON.parse(response.Body);
+        // ContextData.env should be a ContextData object with type 2 (mapping)
+        expect(body.ContextData.env).toBeDefined();
+        expect(body.ContextData.env.t).toBe(2);
+        const entries = body.ContextData.env.d;
+        const localEntry = entries.find((e) => e.k === "AGENT_CI_LOCAL");
+        expect(localEntry).toBeDefined();
+        expect(localEntry.v).toEqual({ t: 0, s: "true" });
+    });
+    it("propagates job-level env into EnvironmentVariables", () => {
+        const payload = {
+            ...basePayload,
+            env: { AGENT_CI_LOCAL: "true" },
+        };
+        const response = createJobResponse("1", payload, "http://localhost:3000", "plan-1");
+        const body = JSON.parse(response.Body);
+        expect(body.EnvironmentVariables).toHaveLength(1);
+        const mapping = body.EnvironmentVariables[0];
+        expect(mapping.type).toBe(2);
+        const entry = mapping.map.find((e) => e.Key === "AGENT_CI_LOCAL");
+        expect(entry).toBeDefined();
+        expect(entry.Value).toBe("true");
+    });
+    it("step-level env overrides job-level env on conflict", () => {
+        const payload = {
+            ...basePayload,
+            env: { SHARED: "from-job" },
+            steps: [{ name: "step1", run: "echo hi", Env: { SHARED: "from-step" } }],
+        };
+        const response = createJobResponse("1", payload, "http://localhost:3000", "plan-1");
+        const body = JSON.parse(response.Body);
+        // Variables should have the step-level value (last-write wins)
+        expect(body.Variables.SHARED).toEqual({ Value: "from-step", IsSecret: false });
+        // ContextData.env should also have the step-level value
+        const entries = body.ContextData.env.d;
+        const entry = entries.find((e) => e.k === "SHARED");
+        expect(entry.v).toEqual({ t: 0, s: "from-step" });
+    });
+    it("omits env from ContextData when no env is provided", () => {
+        const response = createJobResponse("1", basePayload, "http://localhost:3000", "plan-1");
+        const body = JSON.parse(response.Body);
+        expect(body.ContextData.env).toBeUndefined();
+        expect(body.EnvironmentVariables).toEqual([]);
+    });
+});

package/dist/server/routes/actions/index.js

@@ -573,6 +573,11 @@ export function registerActionRoutes(app) {
         const result = { actions: {} };
         const baseUrl = getBaseUrl(req);
         for (const action of actions) {
+            // Local actions (RepositoryType: "self") are resolved from the workspace by the
+            // runner — they never need a tarball download. Skip them to avoid parsing errors.
+            if (!action.nameWithOwner || action.nameWithOwner.startsWith("./")) {
+                continue;
+            }
             const key = `${action.nameWithOwner}@${action.ref}`;
             // Strip sub-path from nameWithOwner (e.g. "actions/cache/save" → "actions/cache")
             // Sub-path actions share the same repo tarball as the parent action.

package/dist/server/routes/dtu.js

@@ -23,6 +23,10 @@ export function registerDtuRoutes(app) {
                     Id: crypto.randomUUID(),
                 }));
                 const jobPayload = { ...payload, steps: mappedSteps };
+                // Track the original repo root for git operations (e.g. compare commits)
+                if (payload.repoRoot) {
+                    state.repoRoot = payload.repoRoot;
+                }
                 // Store the job for dispatch. Runner-targeted jobs go ONLY into runnerJobs
                 // to prevent other runners from stealing them via the generic pool fallback.
                 // Jobs without a runnerName go into the generic pool for any runner to pick up.

package/dist/server/routes/github.js

@@ -94,7 +94,116 @@ export function registerGithubRoutes(app) {
     };
     app.post("/actions/runner-registration", globalRunnerRegistrationHandler);
     app.post("/api/v3/actions/runner-registration", globalRunnerRegistrationHandler);
-    // 7. Tarball route — actions/checkout downloads repos via this endpoint.
+    // 7. Compare commits — used by actions that detect changed files (e.g. Khan/actions@get-changed-files).
+    // Runs `git diff` on the original repo root and returns a GitHub-compatible response.
+    const compareHandler = (req, res) => {
+        const basehead = req.params.basehead;
+        // GitHub format: "base...head" (three dots) or "base..head" (two dots)
+        const parts = basehead.split(/\.{2,3}/);
+        const [base, head] = parts;
+        if (!base || !head) {
+            res.writeHead(422, { "Content-Type": "application/json" });
+            res.end(JSON.stringify({ message: "Invalid basehead format" }));
+            return;
+        }
+        const repoRoot = state.repoRoot;
+        if (!repoRoot) {
+            // No repo root available — return empty comparison
+            console.warn("[DTU] Compare: no repoRoot available, returning empty file list");
+            res.writeHead(200, { "Content-Type": "application/json" });
+            res.end(JSON.stringify({ status: "identical", files: [], total_commits: 0, commits: [] }));
+            return;
+        }
+        try {
+            const output = execSync(`git diff --name-status ${base} ${head}`, {
+                cwd: repoRoot,
+                stdio: "pipe",
+                timeout: 10000,
+            }).toString();
+            const statusMap = {
+                A: "added",
+                M: "modified",
+                D: "removed",
+                R: "renamed",
+                C: "copied",
+                T: "changed",
+            };
+            const files = output
+                .trim()
+                .split("\n")
+                .filter((line) => line.length > 0)
+                .map((line) => {
+                // Format: "M\tfilename" or "R100\told\tnew"
+                const parts = line.split("\t");
+                const rawStatus = parts[0];
+                const statusChar = rawStatus[0];
+                const filename = rawStatus.startsWith("R") ? parts[2] : parts[1];
+                const previousFilename = rawStatus.startsWith("R") ? parts[1] : undefined;
+                return {
+                    sha: "0000000000000000000000000000000000000000",
+                    filename,
+                    status: statusMap[statusChar] || "modified",
+                    ...(previousFilename ? { previous_filename: previousFilename } : {}),
+                    additions: 0,
+                    deletions: 0,
+                    changes: 0,
+                };
+            });
+            res.writeHead(200, { "Content-Type": "application/json" });
+            res.end(JSON.stringify({
+                status: files.length > 0 ? "ahead" : "identical",
+                total_commits: 1,
+                commits: [],
+                files,
+            }));
+        }
+        catch (err) {
+            console.warn(`[DTU] Compare failed (${base}...${head}):`, err.message);
+            // Fall back to listing all tracked files as "added"
+            try {
+                const allFiles = execSync("git ls-files", {
+                    cwd: repoRoot,
+                    stdio: "pipe",
+                    timeout: 10000,
+                }).toString();
+                const files = allFiles
+                    .trim()
+                    .split("\n")
+                    .filter((f) => f.length > 0)
+                    .map((filename) => ({
+                    sha: "0000000000000000000000000000000000000000",
+                    filename,
+                    status: "added",
+                    additions: 0,
+                    deletions: 0,
+                    changes: 0,
+                }));
+                res.writeHead(200, { "Content-Type": "application/json" });
+                res.end(JSON.stringify({
+                    status: "ahead",
+                    total_commits: 1,
+                    commits: [],
+                    files,
+                }));
+            }
+            catch {
+                res.writeHead(500, { "Content-Type": "application/json" });
+                res.end(JSON.stringify({ message: "Failed to compute diff" }));
+            }
+        }
+    };
+    app.get("/repos/:owner/:repo/compare/:basehead", compareHandler);
+    app.get("/_apis/repos/:owner/:repo/compare/:basehead", compareHandler);
+    // 8. List pull requests associated with a commit — used by some changed-files actions
+    // when the push event has an all-zeros `before` (new branch push).
+    const listPrsForCommitHandler = (req, res) => {
+        console.log(`[DTU] List PRs for commit ${req.params.sha} (mock: returning empty)`);
+        res.writeHead(200, { "Content-Type": "application/json" });
+        res.end(JSON.stringify([]));
+    };
+    app.get("/repos/:owner/:repo/commits/:sha/pulls", listPrsForCommitHandler);
+    app.get("/_apis/repos/:owner/:repo/commits/:sha/pulls", listPrsForCommitHandler);
+    // 9. Tarball route — actions/checkout downloads repos via this endpoint.
     // Return an empty tar.gz since the workspace is already bind-mounted.
     const tarballHandler = (req, res) => {
         console.log(`[DTU] Serving empty tarball for ${req.url}`);

package/dist/server/store.d.ts

@@ -19,6 +19,7 @@ export declare const state: {
     planToLogDir: Map<string, string>;
     timelineToLogDir: Map<string, string>;
     currentInProgressStep: Map<string, string>;
+    repoRoot: string | undefined;
     virtualCachePatterns: Set<string>;
     caches: Map<string, {
         version: string;

package/dist/server/store.js

@@ -37,6 +37,8 @@ export const state = {
     // timelineId → sanitized name of the currently in-progress step
     // (used as fallback when the feed recordId is a Job-level ID)
     currentInProgressStep: new Map(),
+    // Original repo root on the host (for git operations like compare)
+    repoRoot: undefined,
     // Substring patterns for cache keys that should always return a synthetic hit
     // with an empty archive (e.g. "pnpm" for bind-mounted pnpm stores).
     virtualCachePatterns: new Set(),
@@ -94,6 +96,7 @@ export const state = {
         this.planToLogDir.clear();
         this.timelineToLogDir.clear();
         this.currentInProgressStep.clear();
+        this.repoRoot = undefined;
         this.virtualCachePatterns.clear();
         this.caches.clear();
         this.pendingCaches.clear();

package/dist/server.test.js

@@ -439,6 +439,51 @@ describe("Artifact v4 upload/download", () => {
         expect(res.body.lockedUntil).toBeDefined();
         expect(new Date(res.body.lockedUntil).getTime()).toBeGreaterThan(Date.now());
     });
+    it("should let runner B steal runner A's job when seeded WITHOUT runnerName (generic pool bug)", async () => {
+        // REPRODUCTION for issue #103:
+        // When local-job.ts seeds a job without setting runnerName, the job lands
+        // in the generic state.jobs pool. If another runner (from a different
+        // concurrent workflow) polls before runner A, it steals the job — causing
+        // runner A to hang forever waiting for a job that will never arrive.
+        const runnerA = "agent-ci-repro-A";
+        const runnerB = "agent-ci-repro-B";
+        // Register both runners
+        await request("POST", "/_dtu/start-runner", {
+            runnerName: runnerA,
+            logDir: "/tmp/agent-ci-repro-A-logs",
+            timelineDir: "/tmp/agent-ci-repro-A-logs",
+        });
+        await request("POST", "/_dtu/start-runner", {
+            runnerName: runnerB,
+            logDir: "/tmp/agent-ci-repro-B-logs",
+            timelineDir: "/tmp/agent-ci-repro-B-logs",
+        });
+        // Seed a job intended for runner A, but WITHOUT runnerName (the bug).
+        // This goes into the generic state.jobs pool.
+        await request("POST", "/_dtu/seed", {
+            id: 4001,
+            name: "job-intended-for-A",
+            // BUG: no runnerName — job lands in generic pool
+        });
+        // Runner B creates a session and polls — it shouldn't get A's job,
+        // but because the job is in the generic pool, B steals it.
+        const sessionB = await request("POST", "/_apis/distributedtask/pools/1/sessions", {
+            agent: { name: runnerB },
+        });
+        const pollB = await request("GET", `/_apis/distributedtask/pools/1/messages?sessionId=${sessionB.body.sessionId}`);
+        // BUG CONFIRMED: runner B stole the job from the generic pool
+        expect(pollB.status).toBe(200);
+        const bodyB = JSON.parse(pollB.body.Body);
+        expect(bodyB.JobDisplayName).toBe("job-intended-for-A");
+        // Now runner A creates a session and polls — the job is gone
+        await request("POST", "/_apis/distributedtask/pools/1/sessions", {
+            agent: { name: runnerA },
+        });
+        // Runner A's poll will hang (long-poll timeout) because its job was stolen.
+        // We verify by checking state: no jobs remain for A.
+        const aHasJob = state.runnerJobs.has(runnerA) || state.jobs.size > 0;
+        expect(aHasJob).toBe(false); // A's job is gone — it will hang forever
+    }, 10_000);
     it("should handle job request finish (PATCH with result + finishTime)", async () => {
         const finishTime = new Date().toISOString();
         const res = await request("PATCH", "/_apis/distributedtask/jobrequests", {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "dtu-github-actions",
-  "version": "0.7.0",
+  "version": "0.8.0",
   "description": "Digital Twin Universe - GitHub Actions Mock and Simulation",
   "keywords": [
     "ci",