forgedock 1.16.2 → 1.18.0

package/commands/test-gate.md

@@ -439,23 +439,38 @@ BASELINE_RESULTS=""
 
 # Run the same tests against the base branch state to establish baseline
 # If base-branch tests also fail, those are pre-existing failures — not batch-introduced
-git stash 2>/dev/null || true
-git checkout "origin/${BASE_BRANCH}" -- . 2>/dev/null || true
+# Use a detached worktree at the base branch — never stash/checkout in the main checkout
+BASE_WT=$(mktemp -d)
+git -C "$REPO_PATH" worktree add --detach "$BASE_WT" "origin/${BASE_BRANCH}" 2>/dev/null \
+  || { echo "WARNING: could not create baseline worktree — baseline unavailable (failures will count as batch-introduced)"; rmdir "$BASE_WT" 2>/dev/null; BASE_WT=""; }
+# A fresh worktree has no installed deps or local env files: link the main checkout's (read-only use), at the repo
+# root and in every cluster working_dir, so a baseline cluster fails only for code reasons, never for a missing install.
+if [ -n "$BASE_WT" ]; then
+  { echo .; yq '.verification.integration_tests[].working_dir // "."' "$CONFIG_FILE" 2>/dev/null; } | sort -u | while IFS= read -r _wd; do
+    for _dep in node_modules .venv venv .env .env.local .env.test; do
+      [ -e "${REPO_PATH}/${_wd}/${_dep}" ] && [ ! -e "${BASE_WT}/${_wd}/${_dep}" ] && [ -d "${BASE_WT}/${_wd}" ] \
+        && ln -s "${REPO_PATH}/${_wd}/${_dep}" "${BASE_WT}/${_wd}/${_dep}"
+    done
+  done   # one working_dir per line: a path with spaces is never word-split
+fi
 
 for i in $(seq 0 $((CLUSTER_COUNT - 1))); do
   CLUSTER=$(yq ".verification.integration_tests[${i}].cluster" "$CONFIG_FILE")
   CMD=$(yq ".verification.integration_tests[${i}].command" "$CONFIG_FILE")
   WORKING_DIR=$(yq ".verification.integration_tests[${i}].working_dir // \".\"" "$CONFIG_FILE")
 
-  if (cd "${REPO_PATH}/${WORKING_DIR}" && eval "$CMD") 2>&1; then
+  if [ -z "$BASE_WT" ]; then
+    # No baseline: never claim pre-existing. A bundle failure in this cluster counts as batch-introduced (fail closed).
+    BASELINE_RESULTS="${BASELINE_RESULTS}\n- ${CLUSTER}: UNAVAILABLE"
+  elif (cd "${BASE_WT}/${WORKING_DIR}" && eval "$CMD") 2>&1; then
     BASELINE_RESULTS="${BASELINE_RESULTS}\n- ${CLUSTER}: PASS"
   else
     BASELINE_RESULTS="${BASELINE_RESULTS}\n- ${CLUSTER}: FAIL (pre-existing)"
   fi
 done
 
-# Restore working state
-git checkout HEAD -- . 2>/dev/null || true
+# Remove the baseline worktree (the main checkout was never touched)
+[ -n "$BASE_WT" ] && git -C "$REPO_PATH" worktree remove --force "$BASE_WT" 2>/dev/null || true
 
 echo "Baseline results:"
 echo -e "$BASELINE_RESULTS"
@@ -475,10 +490,10 @@ while IFS= read -r result_line; do
   if echo "$STATUS" | grep -q "^FAIL"; then
     # Check if this cluster also failed in baseline
     BASELINE_STATUS=$(echo -e "$BASELINE_RESULTS" | grep "^- ${CLUSTER}:" | grep -oP ':\s+\K.+' || echo "PASS")
-    if echo "$BASELINE_STATUS" | grep -q "^PASS"; then
-      # This cluster passed in baseline but fails in the bundle — batch-introduced failure
+    if ! echo "$BASELINE_STATUS" | grep -q "^FAIL"; then
+      # Baseline PASS (or UNAVAILABLE: no proof it was pre-existing) but fails in the bundle — batch-introduced failure
       BATCH_FAILURES="${BATCH_FAILURES}\n${CLUSTER}"
-      echo "BATCH-INTRODUCED FAILURE: cluster '${CLUSTER}' (baseline: PASS → bundle: FAIL)"
+      echo "BATCH-INTRODUCED FAILURE: cluster '${CLUSTER}' (baseline: ${BASELINE_STATUS} → bundle: FAIL)"
     else
       echo "PRE-EXISTING FAILURE: cluster '${CLUSTER}' (baseline: FAIL → bundle: FAIL — not this batch)"
     fi
@@ -534,7 +549,7 @@ else
     cat <<ISSUE_EOF > "$TESTGATE_FAIL_BODY_FILE"
 ## Problem
 
-The \`/test-gate\` command detected a runtime failure in cluster \`${cluster}\` that was introduced by the staging→${DEFAULT_BRANCH} bundle. The failure did not exist on \`${BASE_BRANCH}\` (baseline PASS), confirming it is batch-introduced, not pre-existing.
+The \`/test-gate\` command detected a runtime failure in cluster \`${cluster}\` that was introduced by the staging→${DEFAULT_BRANCH} bundle. It was not observed failing on \`${BASE_BRANCH}\` (baseline: PASS, or UNAVAILABLE when the baseline could not run — then treated as batch-introduced, fail closed).
 
 ## Root Cause (if known)
 
@@ -574,7 +589,7 @@ $(echo -e "$SOURCE_ISSUES_LIST")
    \`\`\`
 3. Observe failure. Compare against \`${BASE_BRANCH}\` baseline:
    \`\`\`bash
-   git checkout origin/${BASE_BRANCH} -- .
+   git worktree add --detach "$BASE_WT" origin/${BASE_BRANCH}   # run the command inside $BASE_WT
    $(yq ".verification.integration_tests[] | select(.cluster == \"${cluster}\") | .command" "$CONFIG_FILE")
    \`\`\`
 ISSUE_EOF

package/commands/work-on/build/architect.md

@@ -719,6 +719,15 @@ gh issue comment {NUMBER} {GH_FLAG} --body "<!-- FORGE:ARCHITECT -->
 |---|------|----------------|-----------------|-----|
 | 1 | {FILE} | {FUNCTION} | {CHANGE} | {REASON} |
 
+### Scope Baseline
+<!-- Machine-readable scope baseline read by build B5.5's post-build scope check (forge#3862). Every
+     path in ### Affected Paths (including tests, docs and config the plan changes), one per line as
+     - path (repo-relative; a trailing slash means every file under that directory). The keys sit alone
+     on their lines. ESTIMATED_DIFF_LINES is the plan's own estimate (code plus tests, a whole number). -->
+PLANNED_FILES:
+- {FILE}
+ESTIMATED_DIFF_LINES: {N}
+
 ### Pattern Sweep
 <!-- Pass-through from the FORGE:INVESTIGATOR '### Pattern Sweep' table (Phase A2.7).
      Omit this section when the investigation report had none. -->
@@ -799,13 +808,16 @@ SKIP_BODY=$(cat <<SKIP_EOF
 
 **Skipped**: {SKIP_REASON}. No cross-path consistency risk — the builder proceeds with the investigation report + context briefing.
 
+### Scope Baseline
+PLANNED_FILES:
+
 <!-- FORGE:ARCHITECT:COMPLETE -->
 SKIP_EOF
 )
 gh issue comment {NUMBER} {GH_FLAG} --body "$SKIP_BODY" # <!-- allowlist:check-command-side-effects -->
 ```
 
-The empty-plan-with-marker changes no downstream behavior — the builder already falls back to the investigation report + contract when the plan is empty (see Integration Point below). The marker only makes the legitimate skip **observable** to the marker-only engine gate. This intentionally does **not** weaken crash detection: a genuinely interrupted architect run still posts no marker at all, so `detectOutcome` still fails-and-escalates it correctly.
+The empty `PLANNED_FILES:` list means "no plan baseline": build B5.5 then skips the out-of-plan check (proceed, never block) and falls back to the investigation's `ESTIMATED_DIFF_LINES`. The empty-plan-with-marker changes no downstream behavior — the builder already falls back to the investigation report + contract when the plan is empty (see Integration Point below). The marker only makes the legitimate skip **observable** to the marker-only engine gate. This intentionally does **not** weaken crash detection: a genuinely interrupted architect run still posts no marker at all, so `detectOutcome` still fails-and-escalates it correctly.
 
 ---
 

package/commands/work-on/build/implement.md

@@ -69,15 +69,36 @@ gh api repos/{GH_REPO}/issues/{NUMBER}/comments \
   --jq '.[] | select(.body | contains("FORGE:CONTEXT")) | .body'
 ```
 
-**Resume check**:
-- If `<!-- FORGE:BUILDER:COMPLETE -->` is present in a BUILDER comment → implementation is done — EXIT and print `IMPLEMENT_RESULT: status: ALREADY_DONE` as the final reply.
-- If `<!-- FORGE:BUILDER -->` exists BUT `<!-- FORGE:BUILDER:COMPLETE -->` is ABSENT → implementation was interrupted after the comment was posted but before the commit (validate.md V5). Delete the partial comment and restart from Phase I2:
+**Resume check** (current-cycle slice only, same as build.md B0 <!-- Added: forge#3666 -->): every read below runs on the comments AFTER the newest trusted `<!-- FORGE:CYCLE: n -->` marker, so a cycle-1 `BUILDER:COMPLETE` never yields `ALREADY_DONE` in cycle 2 (the build would then proceed on an empty branch). `cycle_slice` is declared verbatim from build.md B0:
   ```bash
-  PARTIAL_ID=$(gh api repos/{GH_REPO}/issues/{NUMBER}/comments \
-    --jq '[.[] | select(.body | contains("FORGE:BUILDER") and (contains("FORGE:BUILDER:COMPLETE") | not))] | last | .id // ""')
+  cycle_slice() {
+    jq -s --arg assoc "${FORGE_TRAIL_TRUSTED_ASSOCIATIONS-OWNER,MEMBER,COLLABORATOR}" --arg logins "${FORGE_TRAIL_TRUSTED_LOGINS-}" '[.[][]] as $c
+      | ($assoc | split(",") | map(gsub("^\\s+|\\s+$"; "")) | map(select(length > 0))) as $A
+      | ($logins | split(",") | map(gsub("^\\s+|\\s+$"; "")) | map(select(length > 0))) as $L
+      | ([$c | to_entries[]
+          | select(((.value.author_association // "") as $x | $A | index($x) != null) or ((.value.user.type // "") == "Bot") or ((.value.user.login // "") as $x | $L | index($x) != null))
+          | select((.value.body // "") | test("\\A\\s*<!--\\s*FORGE:CYCLE:") and test("(?m)^[ \\t]*<!--[ \\t]*FORGE:CYCLE:[ \\t]*[0-9]+[ \\t]*-->[ \\t]*$"))
+          | .key] | last) as $i
+      | if $i == null then $c else $c[($i + 1):] end'
+  }
+  # Capture the read first (as build.md B0): piped straight into jq -s, a failed gh call still yields `[]` and exit 0.
+  COMMENTS_JSON=$(gh api --paginate repos/{GH_REPO}/issues/{NUMBER}/comments) \
+    && CYCLE_SLICE=$(printf '%s' "$COMMENTS_JSON" | cycle_slice) && [ -n "$CYCLE_SLICE" ] \
+    || { echo "IMPLEMENT_RESULT: status: BLOCKED (cycle-lookup-unavailable: could not read FORGE:CYCLE)"; exit 1; }  # a failed read is NEVER "no marker"
+  BUILDER_DONE=$(printf '%s' "$CYCLE_SLICE" | jq '[.[] | select((.body // "") | startswith("<!-- FORGE:BUILDER")) | select((.user.type // "") == "Bot" or ((.author_association // "") | IN("OWNER","MEMBER","COLLABORATOR"))) | select(.body | contains("<!-- FORGE:BUILDER:COMPLETE -->"))] | length')
+  ```
+- If `BUILDER_DONE` > 0 (a `<!-- FORGE:BUILDER:COMPLETE -->` BUILDER comment in the current-cycle slice) → implementation is done — EXIT and print `IMPLEMENT_RESULT: status: ALREADY_DONE` as the final reply.
+- If `<!-- FORGE:BUILDER -->` exists in the current-cycle slice BUT `<!-- FORGE:BUILDER:COMPLETE -->` is ABSENT → implementation was interrupted after the comment was posted but before the commit (validate.md V5). Delete the partial comment and restart from Phase I2:
+  Select with ONE anchored, trusted selector (body must start with the marker; author must be a Bot or OWNER/MEMBER/COLLABORATOR; mirrors `scripts/trusted-comments.sh`). A comment that merely mentions the marker (INVESTIGATOR, CONTEXT, ARCHITECT) never matches. Delete ONLY when exactly one non-complete candidate exists and it carries no other FORGE phase marker; unparsable, zero, or multiple matches delete nothing (fail closed). <!-- Added: forge#3668 -->
+  ```bash
+  # only a current-cycle partial is ever deleted; prior-cycle comments are never touched
+  BLD=$(printf '%s' "$CYCLE_SLICE" | jq -s '[.[][] | select((.body // "") | startswith("<!-- FORGE:BUILDER -->")) | select((.user.type // "") == "Bot" or ((.author_association // "") | IN("OWNER","MEMBER","COLLABORATOR"))) | {id: .id, complete: (.body | contains("<!-- FORGE:BUILDER:COMPLETE -->")), other: (.body | test("<!-- FORGE:(INVESTIGATOR|CONTEXT|CONTRACT|ARCHITECT)"))}]') || BLD=""
+  PARTIAL_ID=$(printf '%s' "$BLD" | jq -r '[.[] | select(.complete | not)] | if length == 1 and (.[0].other | not) then .[0].id else empty end' 2>/dev/null)
   if [ -n "$PARTIAL_ID" ]; then
     gh api repos/{GH_REPO}/issues/comments/$PARTIAL_ID -X DELETE
     echo "Deleted partial FORGE:BUILDER comment (no FORGE:BUILDER:COMPLETE) — restarting implementation"
+  else
+    echo "builder resume: no single safe partial BUILDER comment — deleting nothing"
   fi
   ```
 
@@ -144,6 +165,7 @@ Work in `{WORKTREE_PATH}`. Follow the contract deliverables table exactly — im
 - For each acceptance criterion in the contract: implement it, then mentally verify it's met
 - Do NOT add unrequested scope — contract out-of-scope items stay out of scope
 - **Library callback verification**: When writing a lambda or callable that will be passed to a library/framework parameter (e.g., `prepared_statement_name_func=lambda: ""`, `key=lambda x: ...`), you MUST verify the expected calling convention BEFORE writing it. Check the library's default value for that parameter, its documentation, or its source code. A lambda with wrong arity causes `TypeError` at runtime — this is invisible to static analysis and linting. The P0 incident from PR #14391 was caused by `lambda _: ""` (1 arg) passed where SQLAlchemy expects 0 args.
+- **Scratch placement (venvs, caches)**: create virtualenvs, package caches and other large scratch data under `"$TMPDIR/forge-venv-{NUMBER}"` (the engine exports a per-run, disk-backed `TMPDIR`) or inside the worktree. Never use a literal `/tmp/<name>` path: a size-limited `/tmp` fails concurrent lanes with EDQUOT/ENOSPC. Remove what you created when the run finishes. <!-- Added: forge#3572 -->
 - **Worktree-aware path derivation**: When writing shell code that derives repository paths, ALWAYS use `git rev-parse --show-toplevel` for the repo root in regular checkouts, or `git rev-parse --git-common-dir` (then `dirname`) to get the shared `.git` directory when the context may be a linked worktree. For worktree cleanup code specifically, ALWAYS use `--git-common-dir` — `--show-toplevel` returns the worktree path itself, NOT the main repo root. NEVER use `pwd`, relative paths, or `dirname` chains on `--show-toplevel` output for repo root derivation. Test path logic for both regular checkouts and linked worktrees. (Ref: review-findings #104, #105 — 4 PATH_DERIVATION defects, 15% of all review findings)
 - **State machine completeness verification**: When implementing routing logic, state transition tables, or phase dispatch code (e.g., adding a `Skill()` call in a routing loop, adding a new phase to a state machine, creating a new subcommand file), you MUST run three checks BEFORE staging: (1) **Routing target existence** — for each `Skill("subcommand", ...)` call or file reference in routing logic, verify the target file exists at `commands/{subcommand-path}.md`; (2) **State reachability** — for each declared state or phase, verify at least one prior state or entry condition in the router routes to it; (3) **Subcommand invocation wiring** — for each `commands/work-on/*.md` file that declares its invocation condition (e.g., `Invoked by work-on.md routing loop, when X`), verify that condition is actually handled in the router. A missing target file, an unreachable state, or a declared-but-unwired subcommand will not surface until review. (Ref: review-findings #85, #116, #137 — 4 ROUTING defects, 15% of all review findings)
 - **Migration safety checklist** *(trigger: diff includes `*.sql` files or files under a `migrations/` path)*: When any migration file is in the diff, verify ALL of the following BEFORE staging. Fix each violation inline — do not defer to review or the quality gate: (a) **Rollback file**: a corresponding down/rollback migration file exists (e.g. `0042_down_*.sql`, `rollback_*.sql`) or the migration is explicitly self-reversing (DROP of a previously-added column); (b) **NOT NULL safety**: any `ADD COLUMN ... NOT NULL` either includes a `DEFAULT` clause or is preceded by a backfill step — a NOT NULL column without DEFAULT locks the table and fails on existing rows; (c) **Constraint name consistency**: constraint names in the migration match ORM model declarations — a mismatch causes `alembic stamp` and FK introspection to silently diverge; (d) **CREATE TRIGGER idempotency**: every `CREATE TRIGGER` uses `CREATE OR REPLACE TRIGGER` or is preceded by `DROP TRIGGER IF EXISTS` — a bare `CREATE TRIGGER` fails on re-run in test and CI environments; (e) **Migration prefix uniqueness**: confirm the new file's numeric prefix does not already exist in the `infra/migrations/` tree (`ls infra/migrations/*.sql | xargs -n1 basename | grep -oE '^[0-9]+' | sort | uniq -d` prints duplicates) — a duplicate prefix hard-fails the deploy gate regardless of file content. <!-- Added: forge#373 -->
@@ -284,7 +306,7 @@ git add {CHANGED_FILES}
 
 Do NOT run `git commit` here. The commit (with conventional commit message and issue reference) is made by validate.md Phase V5 after `GATE_PASSED=true`.
 
-**Informational diff size** <!-- Added: forge#3450 -->: after staging, read the changed-line count so the builder comment and result carry it. This is a hint only. Implement never gates or decides: the diff-size gate lives in build B5.5, which re-measures authoritatively. Leave `DIFF_LINES` empty when the script is unavailable or fails.
+**Informational diff size** <!-- Added: forge#3450 -->: after staging, read the changed-line count so the builder comment and result carry it. This is a hint only. Implement never gates or decides: the scope check lives in build B5.5, which re-measures authoritatively against the plan (it needs only the `diff_lines=` key here). Leave `DIFF_LINES` empty when the script is unavailable or fails.
 
 ```bash
 # <Script resolution block from work-on/build.md, verbatim>

package/commands/work-on/build/validate.md

@@ -153,6 +153,8 @@ gh issue edit {NUMBER} {GH_FLAG} --add-label "needs-human" # allowlist:check-com
 
 Run after quality gate passes. All tool commands are read from `forge.yaml → verification.commands`. `forge.yaml` is usually gitignored, so it is absent from the worktree: every block below resolves `FORGE_CFG` to the worktree copy if present, else the main checkout's `forge.yaml` (via `git rev-parse --git-common-dir`). Before this, every verification command, learned test command and the SOPS chain check were silently skipped in worktree builds; each step logs `SKIPPED — not configured` when the corresponding key is absent rather than silently passing.
 
+**Scratch placement (venvs, caches)** <!-- Added: forge#3572 -->: any virtualenv, package cache or other large scratch data a verification command needs goes under `"$TMPDIR/forge-venv-{NUMBER}"` (the engine exports a per-run, disk-backed `TMPDIR`) or inside the worktree, never a literal `/tmp/<name>` path. A size-limited `/tmp` fails concurrent lanes with EDQUOT/ENOSPC. Remove what you created when validation finishes.
+
 **Track skipped checks** — initialize before any check runs:
 ```bash
 SKIPPED_CHECKS=""
@@ -710,8 +712,9 @@ else
 fi
 
 # Find the FORGE:BUILDER comment posted by implement.md Phase I6
-BUILDER_COMMENT_ID=$(gh api repos/{GH_REPO}/issues/{NUMBER}/comments \
-  --jq '[.[] | select(.body | contains("FORGE:BUILDER") and (contains("FORGE:BUILDER:COMPLETE") | not))] | last | .id // ""')
+# Anchored, trusted, single-match selector (forge#3668): zero/many/unparsable → empty → append skipped.
+BLD=$(gh api --paginate repos/{GH_REPO}/issues/{NUMBER}/comments | jq -s '[.[][] | select((.body // "") | startswith("<!-- FORGE:BUILDER -->")) | select((.user.type // "") == "Bot" or ((.author_association // "") | IN("OWNER","MEMBER","COLLABORATOR"))) | {id: .id, complete: (.body | contains("<!-- FORGE:BUILDER:COMPLETE -->")), other: (.body | test("<!-- FORGE:(INVESTIGATOR|CONTEXT|CONTRACT|ARCHITECT)"))}]') || BLD=""
+BUILDER_COMMENT_ID=$(printf '%s' "$BLD" | jq -r '[.[] | select(.complete | not)] | if length == 1 and (.[0].other | not) then .[0].id else empty end' 2>/dev/null)
 
 if [ -n "$BUILDER_COMMENT_ID" ]; then
   CURRENT_BODY=$(gh api repos/{GH_REPO}/issues/comments/$BUILDER_COMMENT_ID --jq '.body')