@guilz-dev/sdlc-gh 0.1.0

package/scripts/emit-telemetry-artifact.mjs

@@ -0,0 +1,99 @@
+#!/usr/bin/env node
+/**
+ * Emit a machine-readable telemetry artifact for inner-loop workflows.
+ * Workflows set TELEMETRY_SOURCE and context env vars, then upload telemetry-artifacts/.
+ */
+import {
+  buildTelemetryArtifact,
+  diffStatsFromGit,
+  mapNameToWallFailureType,
+  missingRequiredFields,
+  wallFailureTypeFromJobResults,
+  writeTelemetryArtifact,
+} from "./lib/telemetry-artifact.mjs";
+
+function parseJobResults(raw) {
+  if (!raw) return {};
+  try {
+    return JSON.parse(raw);
+  } catch {
+    return {};
+  }
+}
+
+function main() {
+  const source = process.env.TELEMETRY_SOURCE;
+  if (!source) {
+    console.error("::error::TELEMETRY_SOURCE is required");
+    process.exit(1);
+  }
+
+  const prNumber = Number(process.env.PR_NUMBER || 0) || 0;
+  const skipWithoutPr = process.env.TELEMETRY_SKIP_WITHOUT_PR === "1";
+  if (skipWithoutPr && !prNumber) {
+    console.log("No PR context; skipping telemetry artifact");
+    return;
+  }
+
+  const overrides = {
+    source,
+    TELEMETRY_SOURCE: source,
+    GITHUB_REPOSITORY: process.env.GITHUB_REPOSITORY,
+    GITHUB_RUN_ID: process.env.GITHUB_RUN_ID,
+    GITHUB_RUN_ATTEMPT: process.env.GITHUB_RUN_ATTEMPT,
+    GITHUB_WORKFLOW: process.env.GITHUB_WORKFLOW,
+    GITHUB_EVENT_NAME: process.env.GITHUB_EVENT_NAME,
+    PR_NUMBER: prNumber,
+    PR_BODY: process.env.PR_BODY,
+    PR_LABELS: process.env.PR_LABELS,
+    agent_type: process.env.AGENT_TYPE,
+    execution_mode: process.env.EXECUTION_MODE,
+    model: process.env.MODEL,
+    task_class: process.env.TASK_CLASS,
+    autonomy_level: process.env.AUTONOMY_LEVEL,
+    tool_calls: process.env.TOOL_CALLS,
+    retry_count: process.env.RETRY_COUNT,
+    wall_failure_type: process.env.WALL_FAILURE_TYPE,
+    cost: process.env.COST,
+    elapsed_time: process.env.ELAPSED_TIME,
+    changed_files: process.env.CHANGED_FILES,
+    diff_loc: process.env.DIFF_LOC,
+    final_outcome: process.env.FINAL_OUTCOME,
+    review_outcome: process.env.REVIEW_OUTCOME,
+  };
+
+  if (!process.env.CHANGED_FILES && !process.env.DIFF_LOC && process.env.BASE_SHA) {
+    const stats = diffStatsFromGit(process.env.BASE_SHA);
+    overrides.changed_files = stats.changed_files;
+    overrides.diff_loc = stats.diff_loc;
+  }
+
+  if (!process.env.WALL_FAILURE_TYPE && process.env.JOB_RESULTS) {
+    const wallType = wallFailureTypeFromJobResults(parseJobResults(process.env.JOB_RESULTS));
+    if (wallType) overrides.wall_failure_type = wallType;
+  } else if (process.env.WALL_FAILURE_TYPE) {
+    const mapped = mapNameToWallFailureType(process.env.WALL_FAILURE_TYPE);
+    overrides.wall_failure_type = mapped || process.env.WALL_FAILURE_TYPE;
+  }
+
+  const artifact = buildTelemetryArtifact(overrides);
+  const missing = missingRequiredFields(artifact.payload);
+  if (missing.length) {
+    console.error(`::error::Telemetry payload missing fields: ${missing.join(", ")}`);
+    process.exit(1);
+  }
+
+  const strict = process.env.HARNESS_STRICT_TELEMETRY === "1";
+  if (strict && artifact.placeholders.length) {
+    console.error(
+      `::error::Strict telemetry: unresolved placeholders: ${artifact.placeholders.join(", ")}`,
+    );
+    process.exit(1);
+  }
+
+  const path = writeTelemetryArtifact(artifact);
+  console.log(`Wrote telemetry artifact: ${path}`);
+  console.log(`::notice::telemetry_placeholders=${artifact.placeholders.join(",") || "none"}`);
+}
+
+main();

package/scripts/fetch-telemetry-artifacts.mjs

@@ -0,0 +1,176 @@
+#!/usr/bin/env node
+/**
+ * Fetch inner-loop telemetry JSON artifacts from recent workflow runs.
+ * Output directory: telemetry-collected/ (default)
+ */
+import { execSync } from "node:child_process";
+import { mkdirSync, readdirSync, readFileSync, statSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
+import { pathToFileURL } from "node:url";
+
+export const DEFAULT_COLLECT_DIR = "telemetry-collected";
+
+export const TELEMETRY_WORKFLOWS = [
+  { workflow: "harness-ci.yml", artifactPrefix: "harness-telemetry-" },
+  { workflow: "eval-ci.yml", artifactPrefix: "eval-telemetry-" },
+  { workflow: "agent-retry-orchestrator.yml", artifactPrefix: "retry-telemetry-" },
+  { workflow: "pr-context-comment.yml", artifactPrefix: "pr-context-telemetry-" },
+];
+
+function ghJson(cmd) {
+  const out = execSync(cmd, {
+    encoding: "utf8",
+    env: process.env,
+    stdio: ["pipe", "pipe", "pipe"],
+  });
+  return JSON.parse(out);
+}
+
+function runQuiet(cmd) {
+  try {
+    execSync(cmd, { encoding: "utf8", env: process.env, stdio: ["pipe", "pipe", "pipe"] });
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+/**
+ * @param {string | Date} createdAt
+ * @param {number} windowHours
+ * @returns {boolean}
+ */
+export function isWithinWindow(createdAt, windowHours) {
+  const created = new Date(createdAt).getTime();
+  const cutoff = Date.now() - windowHours * 60 * 60 * 1000;
+  return created >= cutoff;
+}
+
+/**
+ * @param {string} dir
+ * @returns {Record<string, unknown>[]}
+ */
+export function loadTelemetryJsonFiles(dir) {
+  const records = [];
+  let rootStat;
+  try {
+    rootStat = statSync(dir);
+  } catch {
+    return records;
+  }
+  if (!rootStat.isDirectory()) return records;
+
+  const walk = (current) => {
+    for (const entry of readdirSync(current)) {
+      const path = join(current, entry);
+      const info = statSync(path);
+      if (info.isDirectory()) {
+        walk(path);
+        continue;
+      }
+      if (!entry.endsWith(".json")) continue;
+      try {
+        const parsed = JSON.parse(readFileSync(path, "utf8"));
+        if (parsed?.payload) records.push(parsed);
+      } catch {
+        // skip invalid files
+      }
+    }
+  };
+  walk(dir);
+  return records;
+}
+
+/**
+ * @param {{ repo: string, outDir: string, windowHours: number }} options
+ * @returns {{ downloadedRuns: number, records: Record<string, unknown>[], manifestPath: string }}
+ */
+export function fetchTelemetryArtifacts(options) {
+  const repo = options.repo;
+  const outDir = options.outDir ?? DEFAULT_COLLECT_DIR;
+  const windowHours = Number(options.windowHours ?? 24);
+  mkdirSync(outDir, { recursive: true });
+
+  const manifest = {
+    fetched_at: new Date().toISOString(),
+    repo,
+    window_hours: windowHours,
+    runs: [],
+    errors: [],
+  };
+
+  let downloadedRuns = 0;
+
+  for (const { workflow, artifactPrefix } of TELEMETRY_WORKFLOWS) {
+    let runs = [];
+    try {
+      runs = ghJson(
+        `gh run list --repo ${repo} --workflow ${workflow} --limit 100 --json databaseId,createdAt,conclusion`,
+      );
+    } catch (error) {
+      manifest.errors.push({ workflow, message: String(error.message || error) });
+      continue;
+    }
+
+    for (const run of runs) {
+      if (!isWithinWindow(run.createdAt, windowHours)) continue;
+      const runId = run.databaseId;
+      const artifactName = `${artifactPrefix}${runId}`;
+      const targetDir = join(outDir, `${workflow.replace(/\.yml$/, "")}-${runId}`);
+      mkdirSync(targetDir, { recursive: true });
+
+      const ok = runQuiet(
+        `gh run download ${runId} --repo ${repo} -n ${artifactName} -D "${targetDir}"`,
+      );
+      if (!ok) continue;
+
+      downloadedRuns += 1;
+      manifest.runs.push({
+        workflow,
+        run_id: runId,
+        artifact: artifactName,
+        conclusion: run.conclusion,
+        path: targetDir,
+      });
+    }
+  }
+
+  const manifestPath = join(outDir, "fetch-manifest.json");
+  writeFileSync(manifestPath, `${JSON.stringify(manifest, null, 2)}\n`, "utf8");
+  const records = loadTelemetryJsonFiles(outDir);
+  return { downloadedRuns, records, manifestPath };
+}
+
+function main() {
+  const repo = process.env.GITHUB_REPOSITORY;
+  if (!repo) {
+    console.error("::error::GITHUB_REPOSITORY is required");
+    process.exit(1);
+  }
+  if (!process.env.GH_TOKEN && !process.env.GITHUB_TOKEN) {
+    console.error("::error::GH_TOKEN or GITHUB_TOKEN is required");
+    process.exit(1);
+  }
+
+  const outDir = process.env.TELEMETRY_COLLECT_DIR || DEFAULT_COLLECT_DIR;
+  const windowHours = Number(process.env.WINDOW_HOURS || 24);
+  const { downloadedRuns, records, manifestPath } = fetchTelemetryArtifacts({
+    repo,
+    outDir,
+    windowHours,
+  });
+
+  console.log(`Downloaded telemetry from ${downloadedRuns} workflow run(s)`);
+  console.log(`Loaded ${records.length} telemetry record(s) from ${outDir}`);
+  console.log(`Manifest: ${manifestPath}`);
+  if (records.length === 0) {
+    console.warn(
+      "::warning::No telemetry artifacts found in window — inner-loop emitters may not have run yet",
+    );
+  }
+  console.log(`::notice::telemetry_records=${records.length}`);
+}
+
+const isMain =
+  process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href;
+if (isMain) main();

package/scripts/harness-drift-report.mjs

@@ -0,0 +1,99 @@
+#!/usr/bin/env node
+/**
+ * Manifest of harness paths for sync; optional --against <repo-path> for drift diff.
+ */
+import { createHash } from "node:crypto";
+import { appendFileSync, readFileSync, existsSync, readdirSync, statSync } from "node:fs";
+import { join, relative, resolve } from "node:path";
+
+const ROOT = process.cwd();
+const againstArg = process.argv.indexOf("--against");
+const AGAINST = againstArg >= 0 ? resolve(process.argv[againstArg + 1] || "") : null;
+
+const PATHS = [
+  ".github/agents",
+  ".github/hooks",
+  ".github/instructions",
+  ".github/skills",
+  ".github/copilot-instructions.md",
+  ".github/labels.yml",
+  "scripts/validate-harness.mjs",
+  "scripts/check-diff-size.mjs",
+  "scripts/check-issue-spec.mjs",
+  "scripts/lib/ccsd-contract.mjs",
+  "docs/operations.md",
+];
+
+function hashFile(p) {
+  return createHash("sha256").update(readFileSync(p)).digest("hex").slice(0, 12);
+}
+
+function walk(dir, files = []) {
+  if (!existsSync(dir)) return files;
+  const st = statSync(dir);
+  if (st.isFile()) {
+    files.push(dir);
+    return files;
+  }
+  for (const e of readdirSync(dir)) walk(join(dir, e), files);
+  return files;
+}
+
+function buildReport(root) {
+  const report = new Map();
+  for (const p of PATHS) {
+    const full = join(root, p);
+    if (!existsSync(full)) {
+      report.set(p, { status: "missing" });
+      continue;
+    }
+    for (const f of walk(full)) {
+      const rel = relative(root, f);
+      report.set(rel, { sha256_12: hashFile(f) });
+    }
+  }
+  return report;
+}
+
+const report = buildReport(ROOT);
+const lines = ["## Harness drift manifest", "", "| Path | SHA (12) | Drift |", "|------|----------|-------|"];
+
+let againstReport;
+if (AGAINST) {
+  if (!existsSync(AGAINST)) {
+    console.error(`--against path not found: ${AGAINST}`);
+    process.exit(1);
+  }
+  againstReport = buildReport(AGAINST);
+  lines[0] = `## Harness drift vs \`${AGAINST}\``;
+}
+
+for (const [path, r] of [...report.entries()].sort((a, b) => a[0].localeCompare(b[0]))) {
+  let drift = "";
+  if (againstReport) {
+    const other = againstReport.get(path);
+    if (!other || other.status === "missing") drift = "⚠️ missing downstream";
+    else if (other.sha256_12 !== r.sha256_12) drift = "❌ differs";
+    else drift = "✅";
+  }
+  if (r.status) lines.push(`| ${path} | _${r.status}_ | ${drift} |`);
+  else lines.push(`| ${path} | \`${r.sha256_12}\` | ${drift} |`);
+}
+
+const text = lines.join("\n");
+const out = process.env.GITHUB_STEP_SUMMARY;
+if (out) {
+  appendFileSync(out, text + "\n");
+} else {
+  console.log(text);
+}
+
+if (againstReport) {
+  const drifts = [...report.entries()].filter(([path, r]) => {
+    const other = againstReport.get(path);
+    return r.sha256_12 && (!other || other.sha256_12 !== r.sha256_12);
+  });
+  if (drifts.length > 0) {
+    console.warn(`::warning::${drifts.length} path(s) drift from --against repo`);
+  }
+}

package/scripts/lib/bootstrap-copy.mjs

@@ -0,0 +1,123 @@
+/**
+ * Canonical lists for bootstrap copy and validate-harness consistency checks.
+ * Keep in sync with scripts/bootstrap-harness.sh copy loops.
+ */
+
+export const BOOTSTRAP_LIB_FILES = [
+  "bootstrap-copy.mjs",
+  "merge-harness-package.mjs",
+  "stacks.mjs",
+  "harness-ci-fragments.mjs",
+  "ccsd-contract.mjs",
+  "github-config.mjs",
+  "diff-size.mjs",
+  "e2e-manifest.mjs",
+  "doctor-local.mjs",
+  "l1-readiness.mjs",
+  "telemetry-artifact.mjs",
+  "harness-review.mjs",
+  "harness-review-routing.mjs",
+  "gh-aw-dogfood.mjs",
+  "setup-wizard.mjs",
+  "template-root.mjs",
+  "npm-package.mjs",
+];
+
+export const BOOTSTRAP_SCRIPT_MJS = [
+  "validate-harness.mjs",
+  "check-diff-size.mjs",
+  "check-issue-spec.mjs",
+  "select-eval-jobs.mjs",
+  "check-e2e-manifest.mjs",
+  "validate-telemetry.mjs",
+  "emit-telemetry-artifact.mjs",
+  "fetch-telemetry-artifacts.mjs",
+  "aggregate-harness-review.mjs",
+  "route-harness-review.mjs",
+  "check-l1-readiness.mjs",
+  "check-gh-aw-dogfood-scope.mjs",
+  "validate-gh-aw-compile.mjs",
+  "emit-gh-aw-dogfood-report.mjs",
+  "check-open-pr-limit.mjs",
+  "merge-harness-package.mjs",
+  "test-hooks-scenarios.mjs",
+  "test-issue-spec-scenarios.mjs",
+  "test-diff-size-scenarios.mjs",
+  "test-e2e-manifest-scenarios.mjs",
+  "test-setup-github-scenarios.mjs",
+  "test-doctor-scenarios.mjs",
+  "test-telemetry-artifact-scenarios.mjs",
+  "test-harness-review-scenarios.mjs",
+  "test-harness-review-routing-scenarios.mjs",
+  "test-gh-aw-dogfood-scenarios.mjs",
+  "test-bootstrap-guidance-scenarios.mjs",
+  "test-merge-harness-package-scenarios.mjs",
+  "test-l1-readiness-scenarios.mjs",
+  "test-setup-wizard-scenarios.mjs",
+  "harness-drift-report.mjs",
+  "check-eval-score-drift.mjs",
+  "run-e2e-bench.mjs",
+  "doctor.mjs",
+  "setup-github.mjs",
+  "setup-wizard.mjs",
+  "sdlc-gh-cli.mjs",
+  "test-sdlc-gh-cli-scenarios.mjs",
+  "test-npm-package-scenarios.mjs",
+  "trim-harness-ci.mjs",
+];
+
+export const BOOTSTRAP_SCRIPT_SH = [
+  "bootstrap-harness.sh",
+  "setup-github.sh",
+  "verify-bootstrap-stacks.sh",
+];
+
+export const BOOTSTRAP_DOCS = [
+  "operations.md",
+  "adoption.md",
+  "auth-boundaries.md",
+  "failure-taxonomy.md",
+  "telemetry-schema.md",
+  "telemetry-artifacts.md",
+  "nightly-harness-review.md",
+  "gh-aw-dogfood.md",
+  "shared-config.md",
+  "coding-agent-l1.md",
+  "kpi-baseline.md",
+  "revert-playbook.md",
+  "arch.md",
+];
+
+/** script entrypoint -> required lib/*.mjs basenames */
+export const SCRIPT_LIB_IMPORTS = {
+  "check-diff-size.mjs": ["diff-size.mjs"],
+  "check-e2e-manifest.mjs": ["e2e-manifest.mjs"],
+  "doctor.mjs": ["doctor-local.mjs", "github-config.mjs"],
+  "setup-github.mjs": ["doctor-local.mjs", "github-config.mjs"],
+  "test-diff-size-scenarios.mjs": ["diff-size.mjs"],
+  "test-e2e-manifest-scenarios.mjs": ["e2e-manifest.mjs"],
+  "test-setup-github-scenarios.mjs": ["doctor-local.mjs", "github-config.mjs"],
+  "test-doctor-scenarios.mjs": ["doctor-local.mjs"],
+  "setup-wizard.mjs": ["setup-wizard.mjs", "stacks.mjs", "npm-package.mjs"],
+  "test-setup-wizard-scenarios.mjs": ["setup-wizard.mjs", "doctor-local.mjs"],
+  "sdlc-gh-cli.mjs": ["template-root.mjs"],
+  "test-sdlc-gh-cli-scenarios.mjs": ["template-root.mjs"],
+  "test-npm-package-scenarios.mjs": ["npm-package.mjs", "stacks.mjs"],
+  "validate-harness.mjs": ["npm-package.mjs", "stacks.mjs"],
+  "emit-telemetry-artifact.mjs": ["telemetry-artifact.mjs", "ccsd-contract.mjs", "diff-size.mjs"],
+  "validate-telemetry.mjs": ["telemetry-artifact.mjs"],
+  "test-telemetry-artifact-scenarios.mjs": ["telemetry-artifact.mjs", "harness-ci-fragments.mjs"],
+  "fetch-telemetry-artifacts.mjs": [],
+  "aggregate-harness-review.mjs": ["harness-review.mjs"],
+  "test-harness-review-scenarios.mjs": ["harness-review.mjs"],
+  "route-harness-review.mjs": ["harness-review-routing.mjs"],
+  "check-l1-readiness.mjs": ["l1-readiness.mjs", "doctor-local.mjs", "setup-wizard.mjs"],
+  "test-l1-readiness-scenarios.mjs": ["l1-readiness.mjs"],
+  "merge-harness-package.mjs": ["merge-harness-package.mjs"],
+  "test-merge-harness-package-scenarios.mjs": ["merge-harness-package.mjs"],
+  "test-harness-review-routing-scenarios.mjs": ["harness-review-routing.mjs"],
+  "check-gh-aw-dogfood-scope.mjs": ["gh-aw-dogfood.mjs"],
+  "validate-gh-aw-compile.mjs": ["gh-aw-dogfood.mjs"],
+  "emit-gh-aw-dogfood-report.mjs": ["gh-aw-dogfood.mjs"],
+  "test-gh-aw-dogfood-scenarios.mjs": ["gh-aw-dogfood.mjs"],
+};