@rtrentjones/greenlight 0.2.10 → 0.2.12

package/dist/bin.js

@@ -414,7 +414,7 @@ function tokensForTool(tool) {
 }
 
 // src/version.ts
-var MODULE_REF = "v0.2.10";
+var MODULE_REF = "v0.2.12";
 var MODULE_SOURCE_BASE = "git::https://github.com/RTrentJones/greenlight.git//infra/modules";
 function moduleSource(module, ref = MODULE_REF) {
   return `${MODULE_SOURCE_BASE}/${module}?ref=${ref}`;
@@ -1099,10 +1099,17 @@ function vendorDeps(vendorDir) {
   }
   return out;
 }
-function starterVerifyConfig(lane) {
-  const spec = lane === "mcp" ? "{ mode: 'mcp', expectTools: [] }" : "{ mode: 'api', checks: [{ path: '/', status: 200 }] }";
+function starterVerifyConfig(lane, target) {
+  const spec = lane === "mcp" ? "mode: 'mcp', expectTools: []" : "mode: 'api', checks: [{ path: '/', status: 200 }]";
+  const logHint = target === "vercel" ? 'vercel logs "$GREENLIGHT_VERIFY_URL" --token "$VERCEL_API_TOKEN" 2>&1 | head -40 || true' : 'curl -sS -i "$GREENLIGHT_VERIFY_URL" 2>&1 | head -30 || true';
   return `// Greenlight verify spec \u2014 edit to assert this tool's real contract.
-export default ${spec};
+export default {
+  ${spec},
+  // Telemetry-into-verify: a shell command run ONLY when this report FAILS; its last ~50 lines
+  // attach to the report so the agent/CI sees the "why" in-loop. $GREENLIGHT_VERIFY_URL is the
+  // failing URL (no hard-coding). Best-effort (never fails the gate). Uncomment + adjust:
+  // logsOnFailure: '${logHint}',
+};
 `;
 }
 function infraTf(name, domain, lane, target, data, envs, slug) {
@@ -1294,26 +1301,8 @@ jobs:
             -F client_payload[sha]=\${{ github.sha }}
 `;
 }
-function deployListenerYml(name, toolRepo) {
-  return `name: greenlight-deploy-${name}
-
-# Option B: ${toolRepo} fires repository_dispatch(deploy-${name}) after pushing a new image.
-on:
-  repository_dispatch:
-    types: [deploy-${name}]
-  workflow_dispatch:
-
-permissions:
-  contents: read
-
-jobs:
-  deploy:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-      - uses: jdx/mise-action@v2
-      - run: pnpm install --frozen-lockfile
-      - run: pip install --quiet oci-cli
+function ociDeployAndVerifySteps(name) {
+  return `      - run: pip install --quiet oci-cli
       - name: Deploy (resolve instance OCID by name -> restart -> re-pull GHCR image)
         env:
           # The OCI CLI reuses the SAME TF_VAR_OCI_* secrets the apply uses \u2014 one secret set.
@@ -1346,7 +1335,33 @@ jobs:
         # The deploy "succeeds" only if the NEW image is actually serving. verify has a built-in
         # readiness wait (re-pull + container start). A failure here fails the job \u2192 the status
         # posted back is red. oci is verify-gated direct-to-prod (no cheap standing beta on free A1).
-        run: pnpm exec greenlight verify ${name} --env prod
+        run: pnpm exec greenlight verify ${name} --env prod`;
+}
+function deployListenerYml(name, toolRepo) {
+  return `name: greenlight-deploy-${name}
+
+# Option B: ${toolRepo} fires repository_dispatch(deploy-${name}) after pushing a new image.
+on:
+  repository_dispatch:
+    types: [deploy-${name}]
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+# Share the self-heal workflow's group so a deploy and a remediation never run at the same time.
+concurrency:
+  group: deploy-${name}
+  cancel-in-progress: false
+
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: jdx/mise-action@v2
+      - run: pnpm install --frozen-lockfile
+${ociDeployAndVerifySteps(name)}
       - name: Report status back to ${toolRepo}
         if: \${{ always() && github.event.client_payload.sha != '' }}
         env:
@@ -1360,6 +1375,65 @@ jobs:
             -f description="\${{ job.status }}"
 `;
 }
+function remediateYml(name) {
+  return `name: greenlight-remediate-${name}
+
+# Auto-heal: the keepalive Worker dispatches remediate-${name} when ${name} (oci) is unreachable.
+on:
+  repository_dispatch:
+    types: [remediate-${name}]
+  workflow_dispatch:
+
+permissions:
+  contents: read
+  issues: write
+
+# Same group as greenlight-deploy-${name}: a self-heal never overlaps a deploy or another heal, and
+# re-applying an already-healthy instance is idempotent (no diff) \u2014 anti-flap with no extra state.
+concurrency:
+  group: deploy-${name}
+  cancel-in-progress: false
+
+jobs:
+  remediate:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: jdx/mise-action@v2
+      - run: pnpm install --frozen-lockfile
+      - uses: hashicorp/setup-terraform@v3
+        with:
+          terraform_version: '~1.10'
+          terraform_wrapper: false
+      - name: Re-apply the instance (recreate it if OCI idle-reclaimed the Always-Free box)
+        env:
+          TF_TOKEN_app_terraform_io: \${{ secrets.TF_API_TOKEN }} # HCP state backend auth
+          TF_VAR_oci_tenancy_ocid: \${{ secrets.TF_VAR_OCI_TENANCY_OCID }}
+          TF_VAR_oci_user_ocid: \${{ secrets.TF_VAR_OCI_USER_OCID }}
+          TF_VAR_oci_fingerprint: \${{ secrets.TF_VAR_OCI_FINGERPRINT }}
+          TF_VAR_oci_private_key: \${{ secrets.TF_VAR_OCI_PRIVATE_KEY }}
+          TF_VAR_oci_region: \${{ secrets.TF_VAR_OCI_REGION }}
+          TF_VAR_oci_compartment_id: \${{ secrets.TF_VAR_OCI_COMPARTMENT_ID }}
+        run: |
+          if [ -z "$TF_TOKEN_app_terraform_io" ]; then
+            echo "::warning::no TF_API_TOKEN \u2014 skipping re-apply; will still attempt a restart below"
+            exit 0
+          fi
+          terraform -chdir=infra init -input=false
+          # -target pulls in the instance's deps (the ${name}_network module) automatically.
+          terraform -chdir=infra apply -input=false -auto-approve -target=module.${name}_instance
+${ociDeployAndVerifySteps(name)}
+      - name: Escalate if the self-heal failed
+        if: \${{ failure() }}
+        env:
+          GH_TOKEN: \${{ secrets.GITHUB_TOKEN }}
+        run: |
+          gh issue create --repo \${{ github.repository }} \\
+            --title "remediate-${name}: self-heal FAILED" \\
+            --body "Automatic remediation for ${name} (reason: \${{ github.event.client_payload.reason }}) did not bring prod back. Manual attention needed." \\
+            --label keepalive || true
+`;
+}
 function verifyWorkflowYml(name) {
   return `name: greenlight-verify
 
@@ -1405,12 +1479,18 @@ function nextVerifyConfig(name) {
 // Unit tests belong in this repo's PR CI; to also gate the deploy on them, add
 // { mode: 'test', command: 'pnpm test' } + a tolerant deps-install step in greenlight-verify.yml.
 const bypass = process.env.VERCEL_AUTOMATION_BYPASS_SECRET;
+// Telemetry-into-verify: on a FAILED report, fetch the Vercel deployment's runtime logs and attach
+// the tail to the report (best-effort, never fails the gate). $GREENLIGHT_VERIFY_URL is the exact
+// failing deployment URL (injected \u2014 no hard-coding); needs VERCEL_API_TOKEN in CI.
+const logsOnFailure =
+  'vercel logs "$GREENLIGHT_VERIFY_URL" --token "$VERCEL_API_TOKEN" 2>&1 | head -40 || true';
 const api = bypass
   ? {
       mode: 'api',
       checks: [{ path: '/', status: 200, requestHeaders: { 'x-vercel-protection-bypass': bypass } }],
+      logsOnFailure,
     }
-  : { mode: 'api', checks: [{ path: '/', status: 401 }] };
+  : { mode: 'api', checks: [{ path: '/', status: 401 }], logsOnFailure };
 const agentWeb = process.env.ANTHROPIC_API_KEY
   ? [
       {
@@ -1512,7 +1592,7 @@ async function adoptWrapper(ctx) {
   if (target !== "vercel") {
     writeIfAbsent(
       join2(cwd, `verify/${name}.config.ts`),
-      starterVerifyConfig(lane),
+      starterVerifyConfig(lane, target),
       `verify/${name}.config.ts`
     );
   }
@@ -1529,6 +1609,11 @@ async function adoptWrapper(ctx) {
       deployListenerYml(name, slug),
       `.github/workflows/greenlight-deploy-${name}.yml (wrapper deploy listener)`
     );
+    writeIfAbsent(
+      join2(cwd, `.github/workflows/greenlight-remediate-${name}.yml`),
+      remediateYml(name),
+      `.github/workflows/greenlight-remediate-${name}.yml (wrapper self-heal listener)`
+    );
     writeIfAbsent(
       join2(dest, ".github/workflows/greenlight-build.yml"),
       containerBuildYml(name, wrapperSlug),
@@ -2120,6 +2205,7 @@ import { resolve as resolve10 } from "path";
 import { setTimeout as sleep } from "timers/promises";
 
 // src/commands/verify.ts
+import { spawnSync } from "child_process";
 import { resolve as resolve9 } from "path";
 function defaultSpec(lane) {
   switch (lane) {
@@ -2139,6 +2225,39 @@ function printReport(report) {
   }
   console.log(`
 ${report.pass ? "\u2714 PASS" : "\u2718 FAIL"}`);
+  if (!report.pass && report.logs) {
+    console.log(`
+--- recent logs (${report.mode}) ---
+${report.logs}
+--- end logs ---`);
+  }
+}
+var LOG_TAIL_LINES = 50;
+function attachFailureLogs(reports, specs, toolDir) {
+  reports.forEach((report, i) => {
+    if (report.pass) return;
+    const cmd = specs[i]?.logsOnFailure;
+    if (!cmd) {
+      report.logs = "(no logsOnFailure configured for this spec)";
+      return;
+    }
+    try {
+      const res = spawnSync(cmd, {
+        shell: true,
+        cwd: toolDir,
+        timeout: 3e4,
+        encoding: "utf8",
+        maxBuffer: 10 * 1024 * 1024,
+        // Let the command target the exact failing URL without hard-coding it.
+        env: { ...process.env, GREENLIGHT_VERIFY_URL: report.url }
+      });
+      const out = `${res.stdout ?? ""}${res.stderr ?? ""}`.trimEnd();
+      const tail = out.split("\n").slice(-LOG_TAIL_LINES).join("\n");
+      report.logs = tail || `(logsOnFailure produced no output${res.error ? `: ${res.error.message}` : ""})`;
+    } catch (e) {
+      report.logs = `(log fetch failed: ${e instanceof Error ? e.message : String(e)})`;
+    }
+  });
 }
 function flag6(args, name) {
   const i = args.indexOf(name);
@@ -2157,6 +2276,7 @@ async function verifyCommand(args) {
       reachableTimeoutMs: waitMs,
       toolDir: process.cwd()
     });
+    attachFailureLogs(reports2, specs2, process.cwd());
     for (const report of reports2) printReport(report);
     const pass2 = allPass(reports2);
     if (reports2.length > 1)
@@ -2194,6 +2314,7 @@ ${pass2 ? "\u2714 ALL PASS" : "\u2718 FAIL"} (${reports2.length} specs)`);
   }
   const toolDir = resolve9(process.cwd(), entry.dir ?? ".");
   const reports = await verifyAll(url, specs, { reachableTimeoutMs, toolDir });
+  attachFailureLogs(reports, specs, toolDir);
   for (const report of reports) printReport(report);
   const pass = allPass(reports);
   if (reports.length > 1)

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@rtrentjones/greenlight",
-  "version": "0.2.10",
+  "version": "0.2.12",
   "description": "Greenlight CLI — setup and lifecycle for the harness.",
   "license": "MIT",
   "repository": {
@@ -31,10 +31,10 @@
     "@anthropic-ai/sdk": "^0.69.0"
   },
   "devDependencies": {
-    "@rtrentjones/greenlight-loop": "0.2.4",
+    "@rtrentjones/greenlight-adapters": "0.2.4",
     "@rtrentjones/greenlight-shared": "0.2.4",
-    "@rtrentjones/greenlight-verify": "0.2.4",
-    "@rtrentjones/greenlight-adapters": "0.2.4"
+    "@rtrentjones/greenlight-loop": "0.2.4",
+    "@rtrentjones/greenlight-verify": "0.2.4"
   },
   "scripts": {
     "build": "node scripts/copy-assets.mjs && tsup",