@guilz-dev/sdlc-gh 0.1.0

package/.github/workflows/agent-retry-orchestrator.yml

@@ -0,0 +1,161 @@
+name: Agent retry orchestrator
+
+on:
+  check_suite:
+    types: [completed]
+
+permissions:
+  issues: write
+  pull-requests: write
+  checks: read
+
+jobs:
+  retry:
+    if: github.event.check_suite.conclusion == 'failure'
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Evaluate retry policy
+        id: retry
+        uses: actions/github-script@v7
+        with:
+          script: |
+            const MAX_RETRIES = 3;
+            const suite = context.payload.check_suite;
+            const linked = suite.pull_requests || [];
+            let pr = linked[0];
+            if (!pr) {
+              core.info('No PR associated with failed check suite');
+              core.setOutput('skip_telemetry', '1');
+              return;
+            }
+            const prNumber = pr.number;
+            if (!prNumber) {
+              core.info('Could not resolve PR number');
+              core.setOutput('skip_telemetry', '1');
+              return;
+            }
+            core.setOutput('pr_number', String(prNumber));
+            const { data: fullPr } = await github.rest.pulls.get({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              pull_number: prNumber,
+            });
+            const labels = fullPr.labels.map((l) => l.name);
+            core.setOutput('pr_labels', labels.join(','));
+            core.setOutput('pr_body', fullPr.body || '');
+            const retryLabel = labels.find((l) => l.startsWith('retry:'));
+            const count = retryLabel ? parseInt(retryLabel.split(':')[1], 10) || 0 : 0;
+
+            const { data: checkRuns } = await github.rest.checks.listForRef({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              ref: suite.head_sha,
+            });
+            const failed = checkRuns.check_runs.filter((c) => c.conclusion === 'failure');
+            const failureType = failed[0]?.name || suite.app?.slug || 'ci';
+            const sig = failed.map((c) => `${c.name}:${c.conclusion}`).sort().join('|') || failureType;
+            core.setOutput('wall_failure_type', failureType);
+            core.setOutput('retry_count', String(count));
+
+            const noRetry = /security|zizmor|codeql/i.test(failureType);
+            if (noRetry) {
+              core.setOutput('final_outcome', 'escalated');
+              await github.rest.issues.createComment({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                issue_number: prNumber,
+                body: `⛔ \`${failureType}\` failures are not auto-retried. Human escalation required.\n\nfailure_sig: \`${sig}\``,
+              });
+              return;
+            }
+
+            const { data: comments } = await github.rest.issues.listComments({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              issue_number: prNumber,
+            });
+            const sigMatches = comments.filter((c) => c.body?.includes(`failure_sig: \`${sig}\``));
+            if (sigMatches.length >= 1) {
+              core.setOutput('final_outcome', 'escalated');
+              await github.rest.issues.createComment({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                issue_number: prNumber,
+                body: `⛔ Same failure signature detected twice (\`${sig}\`). Stopping auto-retry per docs/operations.md.`,
+              });
+              return;
+            }
+
+            if (count >= MAX_RETRIES) {
+              core.setOutput('final_outcome', 'escalated');
+              await github.rest.issues.createComment({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                issue_number: prNumber,
+                body: `⛔ Max retries (${MAX_RETRIES}) reached for \`${failureType}\`. Human escalation required.\n\nfailure_sig: \`${sig}\``,
+              });
+              return;
+            }
+            const newCount = count + 1;
+            core.setOutput('retry_count', String(newCount));
+            core.setOutput('final_outcome', 'in_progress');
+            if (retryLabel) {
+              await github.rest.issues.removeLabel({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                issue_number: prNumber,
+                name: retryLabel,
+              });
+            }
+            await github.rest.issues.addLabels({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              issue_number: prNumber,
+              labels: [`retry:${newCount}`],
+            });
+            await github.rest.issues.createComment({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              issue_number: prNumber,
+              body: [
+                `🔄 CI failed (\`${failureType}\`). Retry **${newCount}/${MAX_RETRIES}**.`,
+                '',
+                'Re-trigger coding agent or push a fix to the same PR.',
+                `wall_failure_type: ${failureType}`,
+                `failure_sig: \`${sig}\``,
+              ].join('\n'),
+            });
+
+      - uses: actions/setup-node@v4
+        if: steps.retry.outputs.skip_telemetry != '1'
+        with:
+          node-version: "22"
+
+      - name: Emit retry telemetry artifact
+        if: steps.retry.outputs.skip_telemetry != '1'
+        env:
+          TELEMETRY_SOURCE: agent-retry-orchestrator
+          GITHUB_REPOSITORY: ${{ github.repository }}
+          GITHUB_RUN_ID: ${{ github.run_id }}
+          GITHUB_RUN_ATTEMPT: ${{ github.run_attempt }}
+          GITHUB_WORKFLOW: ${{ github.workflow }}
+          GITHUB_EVENT_NAME: ${{ github.event_name }}
+          PR_NUMBER: ${{ steps.retry.outputs.pr_number }}
+          PR_BODY: ${{ steps.retry.outputs.pr_body }}
+          PR_LABELS: ${{ steps.retry.outputs.pr_labels }}
+          WALL_FAILURE_TYPE: ${{ steps.retry.outputs.wall_failure_type }}
+          RETRY_COUNT: ${{ steps.retry.outputs.retry_count }}
+          FINAL_OUTCOME: ${{ steps.retry.outputs.final_outcome }}
+          AGENT_TYPE: orchestrator
+          EXECUTION_MODE: gh_aw
+        run: node scripts/emit-telemetry-artifact.mjs
+
+      - name: Upload retry telemetry artifact
+        if: steps.retry.outputs.skip_telemetry != '1'
+        uses: actions/upload-artifact@v4
+        with:
+          name: retry-telemetry-${{ github.run_id }}
+          path: telemetry-artifacts/
+          if-no-files-found: error

package/.github/workflows/copilot-setup-steps.yml

@@ -0,0 +1,64 @@
+name: Copilot setup
+
+on:
+  workflow_dispatch:
+
+jobs:
+  copilot-setup-steps:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-node@v4
+        if: hashFiles('sample/ts/package-lock.json') != ''
+        with:
+          node-version: "22"
+
+      - uses: actions/setup-python@v5
+        if: hashFiles('sample/python/requirements-dev.txt') != ''
+        with:
+          python-version: "3.12"
+
+      - uses: actions/setup-go@v5
+        if: hashFiles('sample/go/go.mod') != ''
+        with:
+          go-version: "1.22"
+
+      - uses: ruby/setup-ruby@v1
+        if: hashFiles('sample/ruby/Gemfile.lock') != ''
+        with:
+          ruby-version: "3.3"
+          bundler-cache: true
+          working-directory: sample/ruby
+
+      - uses: shivammathur/setup-php@v2
+        if: hashFiles('sample/php/composer.lock') != ''
+        with:
+          php-version: "8.2"
+          coverage: none
+          tools: composer
+
+      - name: Install TS sample deps
+        if: hashFiles('sample/ts/package-lock.json') != ''
+        working-directory: sample/ts
+        run: npm ci
+
+      - name: Install Python sample deps
+        if: hashFiles('sample/python/requirements-dev.txt') != ''
+        working-directory: sample/python
+        run: pip install -r requirements-dev.txt
+
+      - name: Install Go sample deps
+        if: hashFiles('sample/go/go.mod') != ''
+        working-directory: sample/go
+        run: go mod download
+
+      - name: Install Ruby sample deps
+        if: hashFiles('sample/ruby/Gemfile.lock') != ''
+        working-directory: sample/ruby
+        run: bundle install
+
+      - name: Install PHP sample deps
+        if: hashFiles('sample/php/composer.lock') != ''
+        working-directory: sample/php
+        run: composer install --no-interaction --prefer-dist

package/.github/workflows/eval-ci.yml

@@ -0,0 +1,169 @@
+name: Eval CI
+
+on:
+  pull_request:
+    paths:
+      - ".github/**"
+      - "AGENTS.md"
+      - "prompts/**"
+      - "evals/**"
+  schedule:
+    - cron: "0 6 * * 1"
+
+jobs:
+  select:
+    runs-on: ubuntu-latest
+    outputs:
+      jobs: ${{ steps.sel.outputs.jobs }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - uses: actions/setup-node@v4
+        with:
+          node-version: "22"
+      - id: sel
+        env:
+          BASE_SHA: origin/${{ github.base_ref }}
+        run: node scripts/select-eval-jobs.mjs
+
+  prompt-eval:
+    needs: select
+    if: contains(needs.select.outputs.jobs, 'prompt-eval')
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Prompt eval (gh models)
+        run: |
+          shopt -s nullglob
+          files=(prompts/*.prompt.yml)
+          if [ ${#files[@]} -eq 0 ]; then
+            echo "No .prompt.yml files; skipping"
+            exit 0
+          fi
+          if ! command -v gh >/dev/null; then
+            echo "::error::gh CLI required when prompt files exist"
+            exit 1
+          fi
+          for f in "${files[@]}"; do
+            echo "Evaluating $f"
+            gh models eval "$f"
+          done
+
+  agent-policy:
+    needs: select
+    if: contains(needs.select.outputs.jobs, 'agent-policy')
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
+        with:
+          node-version: "22"
+      - name: Validate agent definitions
+        run: node scripts/validate-harness.mjs
+
+  trajectory-conventions:
+    needs: select
+    if: contains(needs.select.outputs.jobs, 'trajectory-conventions')
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+      - name: Harness convention tests
+        run: |
+          pip install pytest
+          pytest evals/trajectories -q
+
+  trajectory-task:
+    needs: select
+    if: contains(needs.select.outputs.jobs, 'trajectory-task')
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+      - name: Skill / task rubric tests
+        run: |
+          pip install pytest
+          pytest evals/trajectories -q
+
+  meta-eval:
+    needs: select
+    if: contains(needs.select.outputs.jobs, 'meta-eval')
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
+        with:
+          node-version: "22"
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+      - name: E2E manifest and task definitions
+        run: |
+          node scripts/check-e2e-manifest.mjs
+          node scripts/run-e2e-bench.mjs
+          pip install pytest
+          pytest evals/trajectories -q
+
+  e2e-bench:
+    if: github.event_name == 'schedule'
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
+        with:
+          node-version: "22"
+      - name: E2E bench manifest and tasks
+        run: |
+          node scripts/check-e2e-manifest.mjs
+          node scripts/run-e2e-bench.mjs
+
+  telemetry:
+    name: telemetry-artifact
+    runs-on: ubuntu-latest
+    if: always() && github.event_name == 'pull_request'
+    needs:
+      - select
+      - prompt-eval
+      - agent-policy
+      - trajectory-conventions
+      - trajectory-task
+      - meta-eval
+      - e2e-bench
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - uses: actions/setup-node@v4
+        with:
+          node-version: "22"
+
+      - name: Emit eval CI telemetry artifact
+        env:
+          TELEMETRY_SOURCE: eval-ci
+          GITHUB_REPOSITORY: ${{ github.repository }}
+          GITHUB_RUN_ID: ${{ github.run_id }}
+          GITHUB_RUN_ATTEMPT: ${{ github.run_attempt }}
+          GITHUB_WORKFLOW: ${{ github.workflow }}
+          GITHUB_EVENT_NAME: ${{ github.event_name }}
+          PR_NUMBER: ${{ github.event.pull_request.number || 0 }}
+          PR_BODY: ${{ github.event.pull_request.body }}
+          PR_LABELS: ${{ join(github.event.pull_request.labels.*.name, ',') }}
+          BASE_SHA: origin/${{ github.base_ref }}
+          JOB_RESULTS: ${{ toJSON(needs) }}
+          AGENT_TYPE: eval
+          EXECUTION_MODE: ci
+          FINAL_OUTCOME: in_progress
+        run: node scripts/emit-telemetry-artifact.mjs
+
+      - name: Upload eval telemetry artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: eval-telemetry-${{ github.run_id }}
+          path: telemetry-artifacts/
+          if-no-files-found: error

package/.github/workflows/eval-drift.yml

@@ -0,0 +1,75 @@
+name: Eval drift check
+
+on:
+  schedule:
+    - cron: "0 7 * * 1"
+  workflow_dispatch:
+
+permissions:
+  issues: write
+  contents: read
+
+jobs:
+  e2e-staleness:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
+        with:
+          node-version: "22"
+      - name: Check E2E manifest freshness
+        id: e2e
+        run: |
+          node scripts/check-e2e-manifest.mjs 2>&1 | tee /tmp/e2e.out || true
+          if grep -q warning /tmp/e2e.out; then echo "stale=true" >> "$GITHUB_OUTPUT"; fi
+
+      - name: Check eval score drift (15pt threshold)
+        id: score
+        run: |
+          set +e
+          set -o pipefail
+          node scripts/check-eval-score-drift.mjs 2>&1 | tee /tmp/score.out
+          SC=$?
+          if [ "$SC" = "2" ]; then echo "drift=true" >> "$GITHUB_OUTPUT"; fi
+
+      - name: Open bench review issue if stale
+        if: steps.e2e.outputs.stale == 'true'
+        uses: actions/github-script@v7
+        with:
+          script: |
+            const title = 'E2E bench rotation due (quarterly 20%)';
+            const { data: issues } = await github.rest.issues.listForRepo({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              state: 'open',
+              labels: 'harness:eval-drift',
+            });
+            if (issues.find((i) => i.title === title)) return;
+            await github.rest.issues.create({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              title,
+              labels: ['harness:eval-drift', 'task:infra'],
+              body: 'E2E manifest not rotated in 90 days. Review evals/e2e-bench/manifest.json per docs/operations.md.',
+            });
+
+      - name: Open eval score drift issue
+        if: steps.score.outputs.drift == 'true'
+        uses: actions/github-script@v7
+        with:
+          script: |
+            const title = 'Eval/production acceptance gap exceeds 15pt';
+            const { data: issues } = await github.rest.issues.listForRepo({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              state: 'open',
+              labels: 'harness:eval-drift',
+            });
+            if (issues.find((i) => i.title === title)) return;
+            await github.rest.issues.create({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              title,
+              labels: ['harness:eval-drift', 'task:infra'],
+              body: 'Eval pass rate exceeds production acceptance by >15pt. Review evals/.score-baseline.json and e2e bench per docs/operations.md.',
+            });

package/.github/workflows/gh-aw-dogfood-ci.yml

@@ -0,0 +1,73 @@
+name: gh-aw dogfood CI
+
+on:
+  pull_request:
+    paths:
+      - ".github/workflows/nightly-harness-review.md"
+      - ".github/workflows/nightly-harness-review.lock.yml"
+      - ".github/workflows/weekly-redteam.md"
+      - ".github/workflows/weekly-redteam.lock.yml"
+      - ".github/workflows/gh-aw-dogfood-ci.yml"
+      - ".github/labels.yml"
+      - ".github/aw/**"
+      - "scripts/**gh-aw**"
+      - "scripts/lib/gh-aw-dogfood.mjs"
+      - "docs/gh-aw-dogfood.md"
+      - "docs/nightly-harness-review.md"
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  dogfood:
+    name: validate-gh-aw-track
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - uses: actions/setup-node@v4
+        with:
+          node-version: "22"
+
+      - name: Install gh aw extension
+        env:
+          GH_AW_VERSION: v0.81.6
+        run: |
+          if gh extension list 2>/dev/null | grep -q 'github/gh-aw'; then
+            gh extension upgrade github/gh-aw --pin "$GH_AW_VERSION"
+          else
+            gh extension install github/gh-aw --pin "$GH_AW_VERSION"
+          fi
+
+      - name: Dogfood scenario tests
+        run: node scripts/test-gh-aw-dogfood-scenarios.mjs
+
+      - name: Enforce dogfood path scope
+        if: github.event_name == 'pull_request'
+        env:
+          PR_LABELS: ${{ join(github.event.pull_request.labels.*.name, ',') }}
+          BASE_SHA: origin/${{ github.base_ref }}
+        run: node scripts/check-gh-aw-dogfood-scope.mjs
+
+      - name: Validate gh-aw compile (when CLI available)
+        env:
+          GH_AW_COMPILE_REQUIRED: "0"
+        run: node scripts/validate-gh-aw-compile.mjs
+
+      - name: Emit dogfood evaluation report
+        env:
+          GITHUB_REPOSITORY: ${{ github.repository }}
+          PR_LABELS: ${{ join(github.event.pull_request.labels.*.name, ',') }}
+          BASE_SHA: origin/${{ github.base_ref }}
+          DOGFOOD_REPORT_DIR: dogfood-report
+        run: node scripts/emit-gh-aw-dogfood-report.mjs
+
+      - name: Upload dogfood report
+        uses: actions/upload-artifact@v4
+        with:
+          name: gh-aw-dogfood-${{ github.run_id }}
+          path: dogfood-report/
+          if-no-files-found: error