amsd-pipeline-shared 0.5.0 → 0.5.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (50) hide show
  1. package/docs/gateway-baseline-2.0.80.md5 +47 -0
  2. package/docs/gateway-files.txt +47 -0
  3. package/orchestrations/config/repo-artifacts.json +23 -0
  4. package/orchestrations/config/resume-preserves.json +25 -0
  5. package/orchestrations/config/run-modes.json +47 -0
  6. package/orchestrations/scripts/lib/checkpoints.sh +175 -0
  7. package/orchestrations/scripts/lib/codeline-lanes.sh +1664 -0
  8. package/orchestrations/scripts/lib/codeline-packages.js +67 -0
  9. package/orchestrations/scripts/lib/codeline-write-perimeter.sh +318 -0
  10. package/orchestrations/scripts/lib/common.sh +66 -0
  11. package/orchestrations/scripts/lib/control-plane.sh +506 -0
  12. package/orchestrations/scripts/lib/ecosystem-registry.js +148 -0
  13. package/orchestrations/scripts/lib/engine-paths.sh +52 -0
  14. package/orchestrations/scripts/lib/flags.sh +38 -0
  15. package/orchestrations/scripts/lib/git-ops.sh +975 -0
  16. package/orchestrations/scripts/lib/halt-recovery.sh +56 -0
  17. package/orchestrations/scripts/lib/handlers/_read-input.js +88 -0
  18. package/orchestrations/scripts/lib/handlers/carry-plan.js +25 -0
  19. package/orchestrations/scripts/lib/handlers/cl-entries.js +10 -0
  20. package/orchestrations/scripts/lib/handlers/codeline-ecosystem.js +164 -0
  21. package/orchestrations/scripts/lib/handlers/codeline-manifests.js +148 -0
  22. package/orchestrations/scripts/lib/handlers/filtered-prd.js +85 -0
  23. package/orchestrations/scripts/lib/handlers/merge-lane-into-canonical.js +39 -0
  24. package/orchestrations/scripts/lib/handlers/package-engines-node.js +25 -0
  25. package/orchestrations/scripts/lib/handlers/prd-phases.js +46 -0
  26. package/orchestrations/scripts/lib/handlers/rebuild-cross-contract.js +7 -0
  27. package/orchestrations/scripts/lib/handlers/record-declared-pass.js +23 -0
  28. package/orchestrations/scripts/lib/handlers/release-line-prd.js +31 -0
  29. package/orchestrations/scripts/lib/handlers/release-lines.js +31 -0
  30. package/orchestrations/scripts/lib/handlers/repo-exclude-patterns.js +108 -0
  31. package/orchestrations/scripts/lib/handlers/resume-spec-output-present.js +16 -0
  32. package/orchestrations/scripts/lib/handlers/sibling-providers.js +43 -0
  33. package/orchestrations/scripts/lib/handlers/spanning-stories-incomplete.js +50 -0
  34. package/orchestrations/scripts/lib/handlers/spec-inputs-digest.js +48 -0
  35. package/orchestrations/scripts/lib/handlers/spec-predates-plan.js +40 -0
  36. package/orchestrations/scripts/lib/handlers/stories-lacking-spec.js +64 -0
  37. package/orchestrations/scripts/lib/handlers/write-missing-plans.js +92 -0
  38. package/orchestrations/scripts/lib/orchestration-resume.sh +74 -0
  39. package/orchestrations/scripts/lib/phase-exit.sh +90 -0
  40. package/orchestrations/scripts/lib/pipeline-commit.sh +19 -0
  41. package/orchestrations/scripts/lib/plan-hash.js +19 -0
  42. package/orchestrations/scripts/lib/plan-scope.js +73 -0
  43. package/orchestrations/scripts/lib/qa-gate-evidence.sh +235 -0
  44. package/orchestrations/scripts/lib/release-lines.sh +166 -0
  45. package/orchestrations/scripts/lib/resume-semantics.sh +31 -0
  46. package/orchestrations/scripts/lib/run-checkpoint.sh +842 -0
  47. package/orchestrations/scripts/lib/run-modes.sh +93 -0
  48. package/orchestrations/scripts/lib/story-outputs.sh +322 -0
  49. package/orchestrations/scripts/lib/story-retry-state.sh +355 -0
  50. package/package.json +1 -1
@@ -0,0 +1,1664 @@
1
+ #!/usr/bin/env bash
2
+ # One more pass per release line a shared codeline's story plan names (_run_release_lines, called by the lane loops below).
3
+ . "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/release-lines.sh"
4
+ . "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/story-outputs.sh"
5
+ # lane_declared_passed <codeline> — the operator declared this lane already passed in this run
6
+ # (EPAM_PASSED_LANES, space/comma separated). AMSD-2974, 2026-10-04: cx-shared reached GO before
7
+ # lanes recorded it; a resume re-ran it (--reset) and was OOM-killed. Operator-chosen.
8
+ lane_declared_passed() {
9
+ local _l
10
+ for _l in ${EPAM_PASSED_LANES//,/ }; do [ "$_l" = "${1:-}" ] && return 0; done
11
+ return 1
12
+ }
13
+
14
+ # codeline-lanes.sh — moved verbatim out of run-agent-orchestration.sh by tools/split-main-into-modules.py
15
+ # (12 functions). Sourced by run-agent-orchestration.sh; SCRIPT_DIR and the globals it sets
16
+ # are in scope exactly as they were. A move, not an edit: every body is byte-identical to
17
+ # the golden recorded at the move (see the identity test).
18
+
19
+ # ── WHICH ROLE IS THIS PROCESS? ──────────────────────────────────────────────
20
+ #
21
+ # This one script is BOTH the parent orchestrator and, re-invoked once per codeline with
22
+ # JIRA_CODELINE_RUN=1, each lane. That is a deliberate design — a lane gets the identical
23
+ # pipeline — but it means every per-run resource is allocated twice, and every parent-only
24
+ # step needs a guard that somebody has to REMEMBER to write. Two defects came from exactly
25
+ # that omission:
26
+ #
27
+ # - the resume block sat below the dispatch and ran in neither role correctly, so every
28
+ # "resume" silently started a fresh run;
29
+ # - the control-plane port derived identically in both roles, so the first lane killed the
30
+ # parent's control plane and took its port.
31
+ #
32
+ # The role is named here, once, so a step can DECLARE which role it belongs to instead of
33
+ # re-deriving it from a raw environment variable at each site. Defined at the very top
34
+ # because entry guards and port resolution both run before anything else.
35
+ #
36
+ # A lane is a lane by virtue of JIRA_CODELINE_RUN — that variable is the contract between the
37
+ # parent's re-invocation and this block, and is read NOWHERE ELSE. See the guard test in
38
+ # test/unit/orchestration/orchestrator-role-is-explicit.test.ts.
39
+ orch_role() {
40
+ if [ -n "${JIRA_CODELINE_RUN:-}" ]; then printf 'lane'; else printf 'parent'; fi
41
+ }
42
+
43
+ # is_lane — true in a per-codeline re-invocation. Guard work that is scoped to one repository.
44
+ is_lane() { [ "$(orch_role)" = 'lane' ]; }
45
+
46
+ # Detect the Node.js binary — checks fnm, nvm, and PATH in order.
47
+ # ── Which Node the CODELINE requires, and which one is installed ─────────────
48
+ #
49
+ # detect_node() listed four literal paths (v20.20.2 / v20.20.0 under fnm and nvm) and checked them
50
+ # BEFORE whatever `node` is on PATH. next.metrolinx.com declares Node 22 in three places — .nvmrc,
51
+ # package.json engines, and the .epam/codeline-facts.json this pipeline generated itself, which
52
+ # says "Node 22.x is required; other versions may not work" — and the engine ran its 245-file jest
53
+ # suite under 20 anyway. Any failure then reads as a defect in the story's code, and no env var
54
+ # existed to correct it.
55
+ #
56
+ # The requirement is a fact of the CODELINE. The runtime is DISCOVERED from what is installed.
57
+ # Neither is a literal here: replacing the v20 list with a v22 list would just move the defect.
58
+ _codeline_node_requirement() {
59
+ local _root="${1:-${PROJECT_ROOT:-}}"
60
+ [ -n "$_root" ] || return 0
61
+ # .nvmrc exists for exactly this purpose, so it is asked first.
62
+ if [ -f "$_root/.nvmrc" ]; then
63
+ head -1 "$_root/.nvmrc" | sed -E 's/^[[:space:]]*v?//; s/[^0-9].*$//' | tr -d '[:space:]'
64
+ return 0
65
+ fi
66
+ # Otherwise the manifest's own engines field. Major only: a codeline pinning a patch is
67
+ # pinning its own business, and the engine matches what it can actually satisfy.
68
+ if [ -f "$_root/package.json" ] && command -v jq >/dev/null 2>&1; then
69
+ jq -r '.engines.node // ""' "$_root/package.json" 2>/dev/null \
70
+ | sed -E 's/[^0-9]*([0-9]+).*/\1/' | tr -d '[:space:]'
71
+ return 0
72
+ fi
73
+ printf ''
74
+ }
75
+
76
+ # detect_node [codeline_root]
77
+ #
78
+ # Install ROOTS are locations of standard version managers, overridable with
79
+ # EPAM_NODE_INSTALL_ROOTS (colon-separated). Paths, not versions — adding a Node release requires
80
+ # no edit here.
81
+ detect_node() {
82
+ local _root="${1:-${PROJECT_ROOT:-}}"
83
+ local _want; _want="$(_codeline_node_requirement "$_root" 2>/dev/null)"
84
+ # A codeline that declares nothing inherits the ENGINE's own declaration — this repo states it
85
+ # in .nvmrc and package.json engines, so it is read the same way rather than written here.
86
+ # Without this, "declares nothing" would mean "highest installed", silently moving a green
87
+ # codeline (mock3 declares neither) onto a major nobody chose.
88
+ if [ -z "$_want" ]; then
89
+ _want="$(_codeline_node_requirement "${EPAM_ENGINE_ROOT:-$SCRIPT_DIR/../..}" 2>/dev/null)"
90
+ fi
91
+ local _roots="${EPAM_NODE_INSTALL_ROOTS:-$HOME/.local/share/fnm/node-versions:$HOME/.nvm/versions/node}"
92
+
93
+ local _cands="" _d _vd _ver _bin _major
94
+ local _oifs="$IFS"; IFS=':'
95
+ for _d in $_roots; do
96
+ IFS="$_oifs"
97
+ [ -d "$_d" ] || continue
98
+ for _vd in "$_d"/v*; do
99
+ [ -d "$_vd" ] || continue
100
+ _ver="$(basename "$_vd")"
101
+ _bin=""
102
+ [ -x "$_vd/bin/node" ] && _bin="$_vd/bin/node"
103
+ [ -z "$_bin" ] && [ -x "$_vd/installation/bin/node" ] && _bin="$_vd/installation/bin/node"
104
+ [ -n "$_bin" ] || continue
105
+ _major="${_ver#v}"; _major="${_major%%.*}"
106
+ if [ -n "$_want" ] && [ "$_major" != "$_want" ]; then continue; fi
107
+ _cands="${_cands}${_ver}\t${_bin}\n"
108
+ done
109
+ IFS=':'
110
+ done
111
+ IFS="$_oifs"
112
+
113
+ # Highest version wins within whatever survived the requirement filter.
114
+ if [ -n "$_cands" ]; then
115
+ printf '%b' "$_cands" | sort -V | tail -1 | cut -f2
116
+ return 0
117
+ fi
118
+
119
+ # PATH node only when it SATISFIES the requirement. Handing back a runtime the codeline said
120
+ # will not work is how a green suite stops meaning anything.
121
+ local _p; _p="$(command -v node 2>/dev/null || true)"
122
+ if [ -n "$_p" ] && [ -x "$_p" ]; then
123
+ if [ -z "$_want" ]; then echo "$_p"; return 0; fi
124
+ local _pv; _pv="$("$_p" --version 2>/dev/null)"; _pv="${_pv#v}"; _pv="${_pv%%.*}"
125
+ [ "$_pv" = "$_want" ] && { echo "$_p"; return 0; }
126
+ fi
127
+ echo ""
128
+ return 1
129
+ }
130
+
131
+ # resolve_codeline_node <codeline_root>
132
+ # Resolves a node binary satisfying <codeline_root>/package.json's OWN
133
+ # "engines.node" declaration, installing it on demand via fnm if not already
134
+ # present. Fully data-driven — the required version comes entirely from the
135
+ # codeline's own manifest; no version number is ever hardcoded here, so this
136
+ # works for any Node version any codeline happens to declare.
137
+ #
138
+ # Live bug this closes (2026-07-22): the regression guard ran a codeline's
139
+ # vitest using detect_node()'s orchestrator-side Node (whatever fnm/nvm
140
+ # version happens to be active for THIS shell — v24.14.1 at the time), not
141
+ # the Node version the codeline itself was built/tested against
142
+ # ("engines": {"node": "^22"}). Running vitest under a mismatched major Node
143
+ # version crashed outright (SIGBUS/segfault from a native module ABI break),
144
+ # which the regression guard then reported as "tests broken" even though the
145
+ # codeline's own tests were never actually exercised.
146
+ #
147
+ # Falls back to detect_node()'s existing generic candidate search if the
148
+ # codeline declares no engines.node, fnm is unavailable, or the declared
149
+ # range can't be resolved/installed for any reason — this must never be the
150
+ # thing that blocks a run outright.
151
+ # provision_env_local_from_sample <codeline_root> <dest_.env.local_path>
152
+ #
153
+ # Some client apps throw at config-load time (e.g. a CMS SDK guard) when an
154
+ # expected env var is merely ABSENT, well before any network call is made —
155
+ # blocking local tooling (type-check/lint via a pre-commit hook) even though
156
+ # nothing here ever talks to the real service. Values only need to be
157
+ # PRESENT, never real credentials.
158
+ #
159
+ # This used to be a per-codeline block hand-written into an engine-side
160
+ # env-vars.json (added 2026-08-02, commit 359f7fa) — keys picked by trial and
161
+ # error against whatever `tsc`/lint failure was visible at the time. It
162
+ # missed MANAGEMENT_TOKEN entirely (AMSD-2041, 2026-08-05) because nothing
163
+ # ever went back to keep that hand-picked list in sync with what the
164
+ # codeline's OWN config actually reads. A codeline that already declares its
165
+ # full set of expected vars in its own `.env.local.sample` makes any
166
+ # engine-side copy of that list redundant and guaranteed to drift.
167
+ #
168
+ # So: read the codeline's own sample file and derive placeholders from ITS
169
+ # keys, not a list a human maintains here. A new var the client adds is
170
+ # picked up on the next run with zero edits to this repo. A codeline with no
171
+ # sample file gets nothing — inventing keys nobody declared would be the same
172
+ # mistake in the other direction.
173
+ provision_env_local_from_sample() {
174
+ local codeline_root="$1" dest="$2"
175
+ local sample="$codeline_root/.env.local.sample"
176
+ [ -f "$sample" ] || return 0
177
+
178
+ # Every `KEY=` line (KEY is the only part that means anything — the
179
+ # sample's own values are just whatever placeholder or blank the client
180
+ # left there, not credentials to reuse). One deterministic placeholder
181
+ # per key, not a fixed table, so this needs no maintenance as the
182
+ # codeline's own required-var set changes.
183
+ grep -E '^[A-Za-z_][A-Za-z0-9_]*=' "$sample" | while IFS='=' read -r key _; do
184
+ printf '%s=sandbox-placeholder-%s\n' "$key" "$(printf '%s' "$key" | tr '[:upper:]_' '[:lower:]-')"
185
+ done > "$dest"
186
+ }
187
+
188
+ # provision_lane_worktree <codeline_root> <worktree_path>
189
+ #
190
+ # A brownfield lane's worktree gets what `git worktree add` cannot give it — the untracked state the
191
+ # pipeline provisioned into the codeline: the .epam/ declaration (copied), the env file (provisioned
192
+ # from the codeline's own sample, as the lane does), and the vendor directory provisioning populates
193
+ # (the declaration's FIRST vendorDir, linked). Nothing is installed: an install would reach the
194
+ # codeline's package registry, a client system (AMSD-2974, 2026-10-02). Same pattern as the baseline
195
+ # gates' worktrees (lib/tsc-baseline-gate.sh). Build output is not linked — it stays in the worktree.
196
+ provision_lane_worktree() {
197
+ local _root="${1:-}" _wt="${2:-}"
198
+ [ -n "$_root" ] && [ -d "$_root" ] && [ -n "$_wt" ] && [ -d "$_wt" ] || return 0
199
+ if [ -d "$_root/.epam" ] && [ ! -e "$_wt/.epam" ]; then
200
+ cp -r "$_root/.epam" "$_wt/.epam" 2>/dev/null || warning " [worktree] could not copy .epam/ into $_wt"
201
+ fi
202
+ if [ ! -e "$_wt/.env.local" ]; then
203
+ provision_env_local_from_sample "$_wt" "$_wt/.env.local"
204
+ fi
205
+ # The declaration is read where the baseline gate reads it (_bg_vendor_dirs): the project's own
206
+ # first, then the codeline's — ensure_story_branch cleans the checkout's untracked .epam/.
207
+ local _cfg="${EPAM_PROJECT_CONFIG_DIR:+$EPAM_PROJECT_CONFIG_DIR/dependency-check.json}" _vd=""
208
+ [ -n "$_cfg" ] && [ -f "$_cfg" ] || _cfg="$_root/.epam/dependency-check.json"
209
+ [ -f "$_cfg" ] && _vd=$(jq -r '.vendorDirs[0]? // empty' "$_cfg" 2>/dev/null || true)
210
+ if [ -n "$_vd" ] && [ -d "$_root/$_vd" ] && [ ! -e "$_wt/$_vd" ]; then
211
+ ln -s "$_root/$_vd" "$_wt/$_vd" || warning " [worktree] could not link $_vd into $_wt"
212
+ fi
213
+ [ -n "$_vd" ] && link_sibling_builds "$_root" "$_wt" "$_vd"
214
+ return 0
215
+ }
216
+
217
+ # link_sibling_builds <root> <worktree> <vendor-dir> — A SITE BUILDS AGAINST ITS SIBLING'S CHANGE.
218
+ #
219
+ # Operator, 2026-10-03: nothing installed, packed, published or fetched — "shared gets updated then
220
+ # dependent codelines get the branch". A codeline of a spanning story that declares a package another
221
+ # codeline of the SAME story provides (lib/handlers/sibling-providers.js) tests against that sibling's
222
+ # checkout — its story branch, its lane already merged — after the sibling's OWN declared build ran
223
+ # there (its ecosystem's buildCommand). The worktree gets its own vendor directory: a link to every
224
+ # entry of the codeline's, except the sibling's package, linked to the sibling's checkout. The
225
+ # codeline's own vendor directory is never touched, so its baseline keeps the installed release and
226
+ # a reproducing test still fails there.
227
+ # _link_entry <entry> <target> — point a vendor entry the worktree's own directory holds at <target>.
228
+ # The entry is only ever a link this file made (to the codeline's entry), so only a link is removed —
229
+ # never a directory, never anything whose path did not resolve.
230
+ _link_entry() {
231
+ local _at="$1" _to="$2"
232
+ [ -n "$_at" ] && [ -n "$_to" ] || return 1
233
+ if [ -L "$_at" ]; then rm -f -- "$_at"; elif [ -e "$_at" ]; then warning " [sibling] $_at is not a link — left as it is"; return 1; fi
234
+ ln -s "$_to" "$_at"
235
+ }
236
+
237
+ # _place_sibling_package <at> <provider_checkout> <vendor_dir> — THE SIBLING'S PACKAGE, AS AN INSTALL HAS IT.
238
+ # A hard-linked copy of every entry of the provider's checkout except its own dependency directory and
239
+ # .git: real files inside the lane, so Node and Jest resolve the package's dependencies from the SITE.
240
+ # A link to the checkout loaded cx-shared's own React beside gotransit's (AMSD-2974, 2026-10-04): two
241
+ # React instances, hooks broken, the site's tests failing whatever the writer did. Same disk, no copy
242
+ # cost, nothing fetched. Only a copy this function made (its marker) is ever replaced.
243
+ _place_sibling_package() {
244
+ local _at="$1" _from="$2" _vd="$3" _release="${4:-}" _e _n
245
+ [ -n "$_at" ] && [ -n "$_vd" ] && [ -d "$_from" ] || return 1
246
+ if [ -L "$_at" ]; then
247
+ rm -f -- "$_at"
248
+ elif [ -e "$_at" ]; then
249
+ if [ -f "$_at/.epam-sibling-copy" ]; then
250
+ rm -rf -- "${_at:?}"
251
+ else
252
+ warning " [sibling] $_at is not a copy this pipeline made — left as it is"
253
+ return 1
254
+ fi
255
+ fi
256
+ mkdir -p "$_at" || return 1
257
+ # SEEDED FROM THE RELEASE THE SITE HAS INSTALLED: what a release ships and the sibling's own build
258
+ # does not regenerate (cx-shared's migration-tool-build, built by its release process) stays.
259
+ if [ -n "$_release" ] && [ -d "$_release" ]; then
260
+ for _e in "$_release"/* "$_release"/.[!.]*; do
261
+ [ -e "$_e" ] || continue
262
+ cp -al "$_e" "$_at/$(basename "$_e")" 2>/dev/null || cp -a "$_e" "$_at/$(basename "$_e")"
263
+ done
264
+ fi
265
+ # The sibling's change on top: each of its entries replaces the release's.
266
+ for _e in "$_from"/* "$_from"/.[!.]*; do
267
+ [ -e "$_e" ] || continue
268
+ _n=$(basename "$_e")
269
+ case "$_n" in "$_vd"|.git) continue ;; esac
270
+ [ -e "$_at/$_n" ] || [ -L "$_at/$_n" ] && rm -rf -- "${_at:?}/${_n:?}"
271
+ cp -al "$_e" "$_at/$_n" 2>/dev/null || cp -a "$_e" "$_at/$_n"
272
+ done
273
+ : > "$_at/.epam-sibling-copy"
274
+ }
275
+
276
+ link_sibling_builds() {
277
+ local _root="$1" _wt="$2" _vd="$3" _pairs _name _prov _bc _scope _base _e _stamp _suffix _story _lb _ld
278
+ [ -n "${PRD_FILE:-}" ] && [ -f "$PRD_FILE" ] && [ -d "$_root/$_vd" ] || return 0
279
+ _pairs=$("${NODE_BIN:-node}" "$SCRIPT_DIR/lib/handlers/sibling-providers.js" "$PRD_FILE" "$_root" 2>/dev/null) || return 0
280
+ [ -n "$_pairs" ] || return 0
281
+ # The worktree's own vendor directory: every entry of the codeline's, linked.
282
+ if [ -L "$_wt/$_vd" ]; then rm -f "$_wt/$_vd"; fi
283
+ mkdir -p "$_wt/$_vd"
284
+ for _e in "$_root/$_vd"/* "$_root/$_vd"/.[!.]*; do
285
+ [ -e "$_e" ] || [ -L "$_e" ] || continue
286
+ [ -e "$_wt/$_vd/$(basename "$_e")" ] || ln -s "$_e" "$_wt/$_vd/$(basename "$_e")"
287
+ done
288
+ local _fresh_vd
289
+ while IFS=$'\t' read -r _name _prov _suffix _story; do
290
+ [ -n "$_name" ] && [ -d "$_prov" ] || continue
291
+ _fresh_vd=0
292
+ # THE PROVIDER'S RELEASE LINE THIS CONSUMER RECEIVES (the story's plan): built from that line's story branch, in a
293
+ # worktree of its own, so the consumer is proved against the back-port it will actually get.
294
+ if [ -n "${_suffix:-}" ] && [ -n "${_story:-}" ]; then
295
+ _lb="${EPAM_BRANCH_PREFIX:-}AI-${_story}${_suffix}"
296
+ _ld="${_prov}-line${_suffix}"
297
+ if git -C "$_prov" rev-parse -q --verify "refs/heads/${_lb}" >/dev/null 2>&1; then
298
+ if [ -d "$_ld" ]; then git -C "$_ld" checkout -q --detach "$_lb" 2>/dev/null || true
299
+ else git -C "$_prov" worktree add -q --detach "$_ld" "$_lb" 2>/dev/null || true; fi
300
+ # ITS DEPENDENCIES INSIDE IT, as the provider's checkout has them: a hard-linked copy, never a link —
301
+ # a bundler resolves real paths, and through a link it laid the line's build out from the parent
302
+ # folder (build/cx-shared-line-7.x/src/...), failing every consumer import (AMSD-2974 run 3).
303
+ if [ -L "$_ld/$_vd" ]; then rm -f -- "$_ld/$_vd"; fi
304
+ if [ ! -e "$_ld/$_vd" ] && [ -d "$_prov/$_vd" ] && [ -d "$_ld" ]; then
305
+ cp -al "$_prov/$_vd" "$_ld/$_vd" 2>/dev/null || cp -a "$_prov/$_vd" "$_ld/$_vd"
306
+ _fresh_vd=1
307
+ fi
308
+ [ -d "$_ld" ] && _prov="$_ld"
309
+ else
310
+ warning " [sibling] ${_name}: release line branch ${_lb} does not exist yet — testing against ${_prov} as it stands"
311
+ fi
312
+ fi
313
+ # The sibling's own build, once per checkout state: the only command it runs is the one the
314
+ # sibling declares; a failure is reported and the lane tests against what it has.
315
+ _bc=$("${NODE_BIN:-node}" "$SCRIPT_DIR/lib/handlers/codeline-ecosystem.js" "$_prov" 2>/dev/null | jq -r '.buildCommand // empty' 2>/dev/null)
316
+ _stamp="${LOG_DIR:-/tmp}/sibling-build-$(printf '%s' "$_prov" | md5sum | cut -c1-12)-$(git -C "$_prov" rev-parse --short HEAD 2>/dev/null || echo none)"
317
+ # A build made against another dependency directory is not this checkout's build.
318
+ [ "${_fresh_vd:-0}" = 1 ] && rm -f -- "$_stamp"
319
+ if [ -n "$_bc" ] && [ ! -f "$_stamp" ]; then
320
+ log " [sibling] building ${_name} from ${_prov} ($(git -C "$_prov" branch --show-current 2>/dev/null)): $_bc"
321
+ if (cd "$_prov" && bash -c "$_bc") > "${_stamp}.log" 2>&1; then
322
+ : > "$_stamp"
323
+ else
324
+ warning " [sibling] the build of ${_name} failed — see ${_stamp}.log; the lane tests against what ${_prov} holds"
325
+ fi
326
+ fi
327
+ # Link the sibling's package into the worktree's vendor directory, scope by scope.
328
+ case "$_name" in
329
+ */*) _scope="${_name%%/*}"; _base="${_name#*/}"
330
+ if [ -L "$_wt/$_vd/$_scope" ]; then
331
+ rm -f "$_wt/$_vd/$_scope"; mkdir -p "$_wt/$_vd/$_scope"
332
+ for _e in "$_root/$_vd/$_scope"/*; do [ -e "$_e" ] && ln -s "$_e" "$_wt/$_vd/$_scope/$(basename "$_e")"; done
333
+ fi
334
+ mkdir -p "$_wt/$_vd/$_scope"
335
+ _place_sibling_package "$_wt/$_vd/$_scope/$_base" "$_prov" "$_vd" "$_root/$_vd/$_scope/$_base" ;;
336
+ *) _place_sibling_package "$_wt/$_vd/$_name" "$_prov" "$_vd" "$_root/$_vd/$_name" ;;
337
+ esac
338
+ log " [sibling] ${_name} → ${_prov} in $(basename "$_wt") (the codeline keeps its installed release)"
339
+ done <<< "$_pairs"
340
+ return 0
341
+ }
342
+
343
+ resolve_codeline_node() {
344
+ local codeline_root="$1"
345
+ local pkg="$codeline_root/package.json"
346
+ local required=""
347
+
348
+ if [ -f "$pkg" ]; then
349
+ required=$(node "$SCRIPT_DIR/lib/handlers/package-engines-node.js" "$pkg" 2>/dev/null || true)
350
+ fi
351
+
352
+ if [ -z "$required" ] || ! command -v fnm &>/dev/null; then
353
+ detect_node
354
+ return
355
+ fi
356
+
357
+ # fnm install/exec want a partial semver (e.g. "22", "20.10"), not a full
358
+ # range operator like "^22" or ">=22 <23" — extract the first version-like
359
+ # token from whatever the codeline declares. This is a heuristic, not a
360
+ # full semver-range resolver, but covers the common declaration shapes
361
+ # (^N, ~N, >=N, N.x, exact N.N.N) without hardcoding any specific version.
362
+ local fnm_version
363
+ fnm_version=$(echo "$required" | grep -oE '[0-9]+(\.[0-9]+){0,2}' | head -1)
364
+
365
+ if [ -z "$fnm_version" ]; then
366
+ detect_node
367
+ return
368
+ fi
369
+
370
+ fnm install "$fnm_version" >/dev/null 2>&1 || true
371
+
372
+ local resolved_bin
373
+ resolved_bin=$(fnm exec --using="$fnm_version" -- node -e 'process.stdout.write(process.execPath)' 2>/dev/null || true)
374
+
375
+ if [ -n "$resolved_bin" ] && [ -x "$resolved_bin" ]; then
376
+ echo "$resolved_bin"
377
+ return 0
378
+ fi
379
+
380
+ detect_node
381
+ }
382
+
383
+ # ensure_node_modules_healthy <codeline_root> <node_bin> <test_bin>
384
+ # Detects a missing or CORRUPTED dependency install (not just "missing") and
385
+ # repairs it via detect_and_install_dependencies() — never by editing any
386
+ # manifest file, which stays purely a manual, case-by-case decision (see the
387
+ # azure.commerce.cdts cx-shared incident, 2026-07-22 — that repair required
388
+ # temporarily stripping a private-registry dependency and was done as an
389
+ # explicit, user-approved one-off, not baked into automated tooling).
390
+ #
391
+ # Live bug this closes: a prior interrupted/killed install left node_modules
392
+ # with truncated native binaries (esbuild, rollup) — present on disk, correct
393
+ # file names, but silently corrupted. A plain "does node_modules exist" check
394
+ # would have missed this entirely; the failure only surfaced when vitest
395
+ # actually tried to load them and crashed outright (SIGBUS/segfault) instead
396
+ # of a normal test failure. This function directly smoke-tests the SAME
397
+ # binary the regression guard is about to invoke ("<test_bin> --version"),
398
+ # which is the cheapest, most direct way to know whether it will actually
399
+ # work — no need to guess at which native files might be corrupted, or fetch
400
+ # reference file sizes from a registry.
401
+ #
402
+ # Returns 0 (healthy or successfully repaired) or 1 (still broken after a
403
+ # genuine repair attempt — e.g. a private-registry auth wall with no
404
+ # credentials available). Never silent: logs what it found and did either way.
405
+ ensure_node_modules_healthy() {
406
+ local codeline_root="$1"
407
+ local node_bin="$2"
408
+ # Set for the child process invoked below, or read by a script that sources this file.
409
+ # ShellCheck cannot see the consumer, so it reports these unused; removing them takes the value away.
410
+ # shellcheck disable=SC2034
411
+ local test_bin="$3" # legacy: an arbitrary .bin entry, no longer trusted
412
+
413
+ # Probe the runner the PROJECT DECLARES, not whatever sorts first in
414
+ # node_modules/.bin. Live metrolinx 2026-07-29: the old probe picked
415
+ # `escodegen` (alphabetically first), ran `node escodegen --version`, got
416
+ # "Invalid option '--version'" — and condemned three codelines whose trees
417
+ # were fine (`jest --version` -> 29.5.0 on the same tree). It had always
418
+ # behaved this way; the caller's `|| true` hid it until that mask came off
419
+ # and every lane stopped.
420
+ local _declared _runner
421
+ _declared="$(jq -r '.scripts.test // ""' "$codeline_root/package.json" 2>/dev/null || echo "")"
422
+ # First word of the declared command is the runner: "jest --ci" -> jest.
423
+ # Derived from what the project says, so an unknown stack needs no changes.
424
+ _runner="${_declared%% *}"
425
+
426
+ if [ -z "$_runner" ]; then
427
+ # INCONCLUSIVE, not broken. We cannot identify a runner, so we cannot
428
+ # claim the tree is unusable — and halting on "cannot tell" is the
429
+ # escodegen bug with a different trigger. Step 5 runs the project's real
430
+ # test command next, which is the actual question anyway.
431
+ warning " [node-modules-health] could not determine health: $codeline_root declares no test script — deferring to the real test command"
432
+ return 0
433
+ fi
434
+
435
+ local _runner_bin="$codeline_root/node_modules/.bin/$_runner"
436
+ if [ -x "$_runner_bin" ]; then
437
+ if "$node_bin" "$_runner_bin" --version >/dev/null 2>&1; then
438
+ return 0
439
+ fi
440
+ warning " [node-modules-health] declared runner '$_runner' exists but failed --version — dependencies present but corrupted, attempting repair..."
441
+ else
442
+ warning " [node-modules-health] declared runner '$_runner' not found in $codeline_root/node_modules/.bin — attempting install..."
443
+ fi
444
+
445
+ detect_and_install_dependencies "$codeline_root" "$node_bin"
446
+ }
447
+
448
+ # ── Codeline bridge agent ─────────────────────────────────────────────────────
449
+ # Runs after a codeline completes. Extracts its exported API surface and writes
450
+ # a cross-codeline contract to logs/cross-codeline-<cl>.md.
451
+ # Exports CROSS_CODELINE_CONTRACT_<CL_UPPER> so the next codeline's re-exec
452
+ # can inject the contract into its story agents' context.
453
+ #
454
+ # Retry policy: up to BRIDGE_MAX_RETRIES (default 2) re-attempts on failure or
455
+ # missing/empty output, with a corrective re-prompt on each retry explaining
456
+ # exactly what went wrong. After all retries:
457
+ # CODELINE_BRIDGE_BLOCK_ON_FAILURE=true → exit 1 (hard abort)
458
+ # CODELINE_BRIDGE_BLOCK_ON_FAILURE=false → warn and continue (fe proceeds
459
+ # without the contract — degraded but not pipeline-corrupting)
460
+ #
461
+ # $1 — completed codeline name (e.g. 'be')
462
+ # $2 — completed codeline worktree path
463
+ # $3 — filtered PRD path for the completed codeline
464
+ _run_codeline_bridge() {
465
+ local _bcl="$1" _bwt="$2" _bprd="$3"
466
+ local _bridge_out="${LOG_DIR}/cross-codeline-${_bcl}.md"
467
+ local _profiles_file="${EPAM_AGENTS_DIR:-$AUTOMATION_DIR/agents}/profiles.json"
468
+ local _bridge_max_retries="${BRIDGE_MAX_RETRIES:-2}"
469
+
470
+ log "[bridge] Extracting cross-codeline contract for '${_bcl}' → ${_bridge_out}"
471
+
472
+ # The bridge agent's identity is its SI (seam codeline-bridge-agent), resolved by the hub — never a
473
+ # persona pasted into the prompt (operator, 2026-09-30: one path).
474
+
475
+ _cp_vals=$(mktemp "${TMPDIR:-/tmp}/codeline-bridge-vals-XXXXXX.json")
476
+ jq_vals \
477
+ --arg bridge_out "${_bridge_out}" \
478
+ --arg bprd "${_bprd}" \
479
+ --arg bcl "${_bcl}" \
480
+ --arg bwt "${_bwt}" \
481
+ '{"__BRIDGE_OUT__":$bridge_out,"__BPRD__":$bprd,"__BCL__":$bcl,"__BWT__":$bwt}' > "$_cp_vals"
482
+ local _base_prompt
483
+ _base_prompt="$(render_engine_prompt codeline-bridge "$_cp_vals")"
484
+ rm -f "$_cp_vals"
485
+
486
+ local _bridge_attempt=0 _bridge_ok=0 _corrective_note=""
487
+ while [ "$_bridge_attempt" -le "$_bridge_max_retries" ]; do
488
+ # Prepend corrective note on retries so the agent knows exactly what failed
489
+ local _bridge_prompt="${_corrective_note:+CORRECTION REQUIRED — YOUR PREVIOUS ATTEMPT FAILED: ${_corrective_note}
490
+
491
+ }${_base_prompt}"
492
+
493
+ local _bridge_rc=0
494
+ rm -f "$_bridge_out"
495
+ # Tools granted below with no restricted allowlist (unlike the read-only
496
+ # QA gates' ORCH_GATE_ALLOWED_TOOLS): this agent's whole job is
497
+ # to read real source files in BRIDGE_SRC_DIR and WriteFile the extracted
498
+ # contract to BRIDGE_OUT_FILE — the read-only allowlist would let it read
499
+ # but never persist its own output. Found live (2026-07-31 agent audit):
500
+ # this call went through plain run_orch_prompt with no tool grant at all,
501
+ # --no-tools — the same class of gap already fixed once for
502
+ # code-graph-detective/failure-analyst. Budgeted like every other
503
+ # tool-bearing gate call in this file.
504
+ local _bridge_reply; _bridge_reply="$(mktemp "${TMPDIR:-/tmp}/codeline-bridge-reply-XXXXXX")"
505
+ AI_GATE_ALLOW_TOOLS=1 EPAM_MAX_TOOL_CALLS="${CODELINE_BRIDGE_MAX_TOOL_CALLS:-10}" \
506
+ run_orch_prompt "$_bridge_prompt" "codeline-bridge-agent" "$_bcl" > "$_bridge_reply" || _bridge_rc=$?
507
+ # THE ENGINE PERFORMS THE WRITE. The contract is the agent's ANSWER; the file lies outside the
508
+ # folder the agent may write in, so on copilot every lane's three attempts returned the full
509
+ # contract as text and "failed" (AMSD-2974, 2026-10-01). A file the agent wrote itself still wins.
510
+ if [ "$_bridge_rc" = "0" ] && [ ! -s "$_bridge_out" ] && [ -s "$_bridge_reply" ]; then
511
+ cp "$_bridge_reply" "$_bridge_out"
512
+ fi
513
+ rm -f "$_bridge_reply"
514
+
515
+ # Determine success: exit 0 AND file exists AND file is non-empty
516
+ if [ "$_bridge_rc" = "0" ] && [ -f "$_bridge_out" ] && [ -s "$_bridge_out" ]; then
517
+ _bridge_ok=1
518
+ break
519
+ fi
520
+
521
+ # Diagnose the specific failure for the corrective re-prompt
522
+ if [ "$_bridge_rc" != "0" ]; then
523
+ _corrective_note="run_orch_prompt exited with code ${_bridge_rc}. You must call WriteFile to write the contract to ${_bridge_out} before finishing."
524
+ elif [ ! -f "$_bridge_out" ]; then
525
+ _corrective_note="The file ${_bridge_out} was NOT written. You MUST call WriteFile with path=${_bridge_out} to produce the contract — do not finish without writing it."
526
+ else
527
+ _corrective_note="The file ${_bridge_out} was written but is EMPTY. The source directory ${_bwt} contains TypeScript files — read them and extract their exports before writing the contract."
528
+ fi
529
+
530
+ _bridge_attempt=$(( _bridge_attempt + 1 ))
531
+ if [ "$_bridge_attempt" -le "$_bridge_max_retries" ]; then
532
+ warning "[bridge] Attempt ${_bridge_attempt}/${_bridge_max_retries} failed for '${_bcl}' — retrying with corrective prompt"
533
+ fi
534
+ done
535
+
536
+ if [ "$_bridge_ok" = "1" ]; then
537
+ local _cl_upper="${_bcl^^}"
538
+ export "CROSS_CODELINE_CONTRACT_${_cl_upper}=${_bridge_out}"
539
+ export CROSS_CODELINE_CONTRACT="$_bridge_out"
540
+ log "[bridge] ✓ Contract written: ${_bridge_out} (exported CROSS_CODELINE_CONTRACT_${_cl_upper})"
541
+ else
542
+ local _total_attempts=$(( _bridge_max_retries + 1 ))
543
+ if [ "${CODELINE_BRIDGE_BLOCK_ON_FAILURE:-false}" = "true" ]; then
544
+ error "[bridge] FATAL — contract extraction for '${_bcl}' failed after ${_total_attempts} attempt(s) and CODELINE_BRIDGE_BLOCK_ON_FAILURE=true. Aborting pipeline."
545
+ return 1
546
+ else
547
+ warning "[bridge] Contract extraction for '${_bcl}' failed after ${_total_attempts} attempt(s) — downstream codelines proceed without cross-codeline contract (set CODELINE_BRIDGE_BLOCK_ON_FAILURE=true to abort instead)"
548
+ fi
549
+ fi
550
+ }
551
+
552
+ # Stop a lane and everything it spawned.
553
+ #
554
+ # A lane subshell is not the thing doing the work: it backgrounds `bash "$0"`,
555
+ # which runs the node LLM calls. `kill <lane pid>` orphans those rather than
556
+ # stopping them, so the halt reported an abort while the lane kept running and
557
+ # kept billing for ten more minutes (live AMSD-2041 2026-07-30). It only died
558
+ # by tripping over a PRD the parent's cleanup had already removed.
559
+ #
560
+ # Walks children before parents so nothing is reparented to init and left
561
+ # running. Deliberately NOT a process-group kill: lanes share this script's
562
+ # group, so `kill -- -$pgid` would take the orchestrator down with them.
563
+ _kill_lane_tree() {
564
+ local _target="$1" _child
565
+ [ -z "$_target" ] && return 0
566
+ # Never accept our own PID or our parent's — that is self-termination
567
+ # wearing a lane's clothes.
568
+ if [ "$_target" = "$$" ] || [ "$_target" = "${BASHPID:-}" ] || [ "$_target" = "$PPID" ]; then
569
+ return 0
570
+ fi
571
+ for _child in $(pgrep -P "$_target" 2>/dev/null); do
572
+ _kill_lane_tree "$_child"
573
+ done
574
+ kill -TERM "$_target" 2>/dev/null || true
575
+ return 0
576
+ }
577
+
578
+ # ── Run-scoped working directory ────────────────────────────────────────────
579
+ # Lane working files (per-codeline PRDs, cross-lane state) used to be written to a FLAT
580
+ # machine-global namespace: /tmp/orch-<codeline>-prd-<pid>.json. Every project and every
581
+ # concurrent run shared it, and archive-run-artifacts.sh then picked "the newest matching
582
+ # file" — so a clean mock1 run archived metrolinx's PRD, describing the wrong project, the
583
+ # wrong story and the wrong day (live 2026-08-05).
584
+ #
585
+ # Scoped to THIS run instead. Falls back to a private mktemp -d rather than the shared
586
+ # namespace when no run directory can be derived: a temp dir nobody else can glob is still
587
+ # isolated, which the old path never was.
588
+ _run_work_dir() {
589
+ local _base="${EPAM_PROJECT_CONFIG_DIR:-}"
590
+ if [ -n "$_base" ] && [ -n "${ORCH_RUN_ID:-}" ]; then
591
+ local _d="$_base/runs/$ORCH_RUN_ID/work"
592
+ mkdir -p "$_d" 2>/dev/null && { printf '%s' "$_d"; return 0; }
593
+ fi
594
+ mktemp -d "${TMPDIR:-/tmp}/orch-run-XXXXXX"
595
+ }
596
+
597
+ # archive_prior_lane_state — ONCE PER RUN, before any lane works: every file in the lane log folders (logs/lanes/<lane>,
598
+ # lines/<line>) older than this run's own start moves to logs/archive/lanes-before-<run>/ — never deleted. The folders
599
+ # survive every launch (the fresh-launch reset clears only logs/), so a run inherited another run's retry counters,
600
+ # ladder positions, rejection records, self-heal ledger and review/gate history (AMSD-2974 run 2 at pause 2,
601
+ # 2026-10-09: upexpress at attempt 10 of 12, 17 old self-heal events). What this run wrote is newer and stays.
602
+ archive_prior_lane_state() {
603
+ local _lanes="${LOG_DIR:-}/lanes"
604
+ [ -d "$_lanes" ] || return 0
605
+ local _run="${EPAM_RESUME_RUN:-${ORCH_RUN_ID:-}}"
606
+ [ -n "$_run" ] || return 0
607
+ local _marker=""
608
+ if [ -n "${EPAM_PROJECT_CONFIG_DIR:-}" ]; then
609
+ _marker="${EPAM_PROJECT_CONFIG_DIR}/runs/${_run}/lane-state-archived"
610
+ [ -f "$_marker" ] && return 0
611
+ fi
612
+ # The run's start is in its id (YYYYMMDDTHHMMSSZ, UTC).
613
+ local _start
614
+ _start=$(printf '%s' "$_run" | sed -nE 's/^([0-9]{4})([0-9]{2})([0-9]{2})T([0-9]{2})([0-9]{2})([0-9]{2})Z?$/\1-\2-\3 \4:\5:\6 UTC/p')
615
+ [ -n "$_start" ] || return 0
616
+ local _ref; _ref=$(mktemp "${TMPDIR:-/tmp}/run-start-XXXXXX") || return 0
617
+ touch -d "$_start" "$_ref" 2>/dev/null || { rm -f "$_ref"; return 0; }
618
+ local _dest="${LOG_DIR}/archive/lanes-before-${_run}" _n=0 _f _rel
619
+ while IFS= read -r -d '' _f; do
620
+ _rel="${_f#"$_lanes"/}"
621
+ mkdir -p "$_dest/$(dirname "$_rel")" 2>/dev/null && mv "$_f" "$_dest/$_rel" 2>/dev/null && _n=$((_n + 1))
622
+ done < <(find "$_lanes" -type f ! -newer "$_ref" -print0 2>/dev/null)
623
+ rm -f "$_ref"
624
+ [ "$_n" -gt 0 ] && log "[orch] archived ${_n} file(s) of earlier runs' lane state → ${_dest} (this run starts with none of it)"
625
+ [ -n "$_marker" ] && { mkdir -p "${_marker%/*}" 2>/dev/null; date -u +%Y-%m-%dT%H:%M:%SZ > "$_marker"; }
626
+ return 0
627
+ }
628
+
629
+ # base_fresh_story_branches <prd> — ONCE PER RUN, before anything reads them, every codeline a brownfield story may be
630
+ # worked in (its own and every one discovery grounded, lib/plan-scope.js) is put on the story's branch, freshly based on
631
+ # the codeline's baseline. ensure_story_branch pins any commit the reset would orphan under epam-rescue/*. Once per RUN,
632
+ # not per launch: with the operator's pauses the lanes are first reached by a resumed leg, so "not a resume" never held;
633
+ # the run's own marker (runs/<id>/story-branches-based) keeps every later leg — a resume after the writer included —
634
+ # off the branches, where this run's own work is.
635
+ base_fresh_story_branches() {
636
+ local _prd="${1:-}"
637
+ [ "${EPAM_BROWNFIELD:-0}" = "1" ] && [ -f "$_prd" ] || return 0
638
+ command -v ensure_story_branch >/dev/null 2>&1 || return 0
639
+ local _run="${EPAM_RESUME_RUN:-${ORCH_RUN_ID:-}}"
640
+ local _marker=""
641
+ if [ -n "${EPAM_PROJECT_CONFIG_DIR:-}" ] && [ -n "$_run" ]; then
642
+ _marker="${EPAM_PROJECT_CONFIG_DIR}/runs/${_run}/story-branches-based"
643
+ [ -f "$_marker" ] && return 0
644
+ fi
645
+ local _line _root _story
646
+ while IFS=$'\t' read -r _root _story; do
647
+ [ -n "$_root" ] && [ -n "$_story" ] || continue
648
+ log "[orch] fresh run: basing '${_root##*/}' on its story branch for ${_story} before anything reads it"
649
+ ensure_story_branch "$_root" "$_story" "${JIRA_BASELINE_BRANCH:-}" || true
650
+ done < <("$NODE_BIN" -e '
651
+ const fs=require("fs"),path=require("path");const [prdF,cfg,lib]=process.argv.slice(1);
652
+ const prd=JSON.parse(fs.readFileSync(prdF,"utf8"));let facts=null;
653
+ try{facts=cfg?JSON.parse(fs.readFileSync(path.join(cfg,"codeline-facts.json"),"utf8")):null}catch{}
654
+ const {planScope}=require(lib);const seen=new Set();
655
+ for(const s of prd.stories||[]){if(!s||!s.id)continue;for(const d of planScope(s,prd,facts).outputDirs){const k=d.path+"\t"+s.id;if(!seen.has(k)){seen.add(k);console.log(k)}}}
656
+ ' "$_prd" "${EPAM_PROJECT_CONFIG_DIR:-}" "$SCRIPT_DIR/lib/plan-scope.js" 2>/dev/null)
657
+ [ -n "$_marker" ] && { mkdir -p "${_marker%/*}" 2>/dev/null; date -u +%Y-%m-%dT%H:%M:%SZ > "$_marker"; }
658
+ return 0
659
+ }
660
+
661
+ # lane_paused_since <lane_log_dir> <marker> — did this lane (or a release line of it) record a pause after <marker>
662
+ # (touched when this run's lanes started)? A lane paused before its writer is not a lane that finished: it recorded
663
+ # paused-run.json in its OWN log dir, which no launcher reads. A record older than the marker is an earlier launch's.
664
+ lane_paused_since() {
665
+ local _dir="${1:-}" _marker="${2:-}"
666
+ [ -d "$_dir" ] && [ -f "$_marker" ] || return 1
667
+ [ -n "$(find "$_dir" -name paused-run.json -newer "$_marker" 2>/dev/null | head -1)" ]
668
+ }
669
+
670
+ # record_lanes_pause <marker> — when any lane paused during this run, the RUN is paused: recorded where every launcher
671
+ # reads it (lib/checkpoints.sh record_run_pause, in LOG_DIR), with how to resume. Returns 1 when no lane paused.
672
+ # Found rehearsing AMSD-2974, 2026-10-09: every lane paused before its writer and the run reported "Pipeline complete".
673
+ record_lanes_pause() {
674
+ local _marker="${1:-}"
675
+ [ -f "$_marker" ] && [ -d "${LOG_DIR:-}/lanes" ] || return 1
676
+ local _recs; _recs=$(find "${LOG_DIR}/lanes" -name paused-run.json -newer "$_marker" 2>/dev/null)
677
+ [ -n "$_recs" ] || return 1
678
+ local _n _stage
679
+ _n=$(printf '%s\n' "$_recs" | awk -F/lanes/ '{split($2,a,"/"); print a[1]}' | sort -u | grep -c .)
680
+ _stage=$(jq -r '.stage // ""' "$(printf '%s\n' "$_recs" | head -1)" 2>/dev/null)
681
+ record_run_pause "${_stage:-pre-writer}"
682
+ log "[orch] ⏸ Run paused — ${_n} lane(s) paused at ${_stage:-a pause}. Resume with: EPAM_RESUME_RUN=${ORCH_RUN_ID:-} <your launcher>"
683
+ return 0
684
+ }
685
+
686
+ # _lane_phase_args — the argument a lane's FIRST phase call passes run-agent-orchestration.sh.
687
+ # A RESUMED LANE DOES NOT RESET ITS STORIES. --reset sets every completed story of the phase back to pending
688
+ # (run-agent-orchestration.sh:964), and the orchestrator's own rule is that "a resume runs without --reset"
689
+ # (:1026). The lane passed --reset unconditionally, so the story story_resume_completed_by_commit had just
690
+ # marked completed was un-marked 33 s later and a paid writer re-wrote approved work on every resume
691
+ # (AMSD-2974 run 9, resumes 9-12). A lane resumed from its checkpoint (_lane_spec_mode=0 under EPAM_RESUME_RUN)
692
+ # passes nothing; every other lane, and every retry after gate remediation, still resets.
693
+ _lane_phase_args() {
694
+ if [ -n "${EPAM_RESUME_RUN:-}" ] && [ "${_lane_spec_mode:-}" = "0" ]; then
695
+ printf ''
696
+ else
697
+ printf '%s' '--reset'
698
+ fi
699
+ }
700
+
701
+ _run_codeline_loop() {
702
+ local _prd_path="$1"
703
+ local _log_file="${2:-/tmp/orch-$(date +%Y%m%dT%H%M%S).log}"
704
+ # When the tier3 launcher (or any external phase-managing caller) drives the
705
+ # phase loop itself (calling us once per phase via --phase X), set this to
706
+ # run only that phase across all codelines. Empty = run every PRD phase.
707
+ local _phase_filter="${3:-}"
708
+ # When this run's lanes started: a lane pause recorded after it is this run's (lane_paused_since, record_lanes_pause).
709
+ local _lanes_started="${LOG_DIR:-/tmp}/.lanes-started"
710
+ mkdir -p "${LOG_DIR:-/tmp}" 2>/dev/null; : > "$_lanes_started"
711
+
712
+ # ── THE PLAN COMES FIRST, AND IT DECIDES THE LANES ─────────────────────────
713
+ # A fresh run's codelines are put on their story branches, freshly based, BEFORE anything reads them: the detective
714
+ # and the spec pass read the checkout, and a fresh AMSD-2974 run read earlier runs' commits as existing code
715
+ # (2026-10-09). A codeline a killed line pass left on the line's branch is put back. Then the story without a plan
716
+ # gets one (lib/handlers/write-missing-plans.js), shown every codeline discovery grounded; the codelines the plan
717
+ # names join the story and the run's outputDirs (lib/plan-scope.js) — so the lane list below is the plan's.
718
+ archive_prior_lane_state
719
+ base_fresh_story_branches "$_prd_path"
720
+ _return_codelines_from_release_lines "$_prd_path"
721
+ "$NODE_BIN" "$SCRIPT_DIR/lib/handlers/write-missing-plans.js" "$_prd_path" || true
722
+
723
+ # Extract codeline:path entries from project.outputDirs.
724
+ # Falls back to project.outputDir + JIRA_DEFAULT_CODELINE for single-codeline PRDs.
725
+ local _cl_entries=()
726
+ mapfile -t _cl_entries < <("$NODE_BIN" "$SCRIPT_DIR/lib/handlers/cl-entries.js" "${_prd_path}" 2>/dev/null)
727
+
728
+ # CODELINE SELECTION — run a subset of the story's codelines, one launch at a time.
729
+ #
730
+ # A story spanning several repositories otherwise runs all of them in one launch. Naming a
731
+ # subset makes each launch a natural pause: a failure blasts one lane instead of every lane,
732
+ # each pause is an inspection point, and spend is bounded per launch instead of per story.
733
+ #
734
+ # Filtered HERE because this array is the single source every downstream path reads — the
735
+ # parallel fan-out, the sequential loop, and the post-run merges. Filtering at any one of those
736
+ # would leave the others running codelines nobody asked for.
737
+ #
738
+ # UNSET OR EMPTY MEANS ALL, so a run that does not use this is byte-for-byte today's run.
739
+ # An unmatched selection yields NOTHING rather than everything: failing open here would run
740
+ # every codeline when the operator asked for one, which is the expensive direction of a typo,
741
+ # and the caller below already treats an empty list as a hard error rather than a silent pass.
742
+ #
743
+ # Matching is on the codeline NAME (the part before the colon), never the path, so a selection
744
+ # cannot accidentally match a directory that merely contains the same text.
745
+ # NO OPERATOR FILTER HERE. The lane list already IS the run's scope: it is built from the
746
+ # PRD's project.outputDirs, which resolve-codeline-scope.sh writes for every project however
747
+ # its PRD arrived. Filtering it again with a hand-typed EPAM_ONLY_CODELINES asked a human to
748
+ # restate a fact the PRD already carried, and could only ever disagree with it.
749
+
750
+ if [ ${#_cl_entries[@]} -eq 0 ]; then
751
+ error "[orch] No codeline/worktree entries found in PRD: ${_prd_path}"
752
+ error "[orch] Add project.outputDirs to the PRD or set JIRA_DEFAULT_CODELINE + project.outputDir"
753
+ return 1
754
+ fi
755
+
756
+ # ── Codeline health: assess every lane BEFORE spending anything ────────────
757
+ # Live AMSD-2041, 2026-07-28: all three discovered codelines declared a test
758
+ # script and a runner, and none could resolve one — two had no node_modules at
759
+ # all. Until that morning Step 5 skipped silently on exactly this, so an
760
+ # unverified baseline was accepted once per lane. Making it fail was right, but
761
+ # it fails INSIDE the phase, after the spec pass is already paid for: the next
762
+ # launch would have cost a full spec pass per lane to discover a dependency
763
+ # problem visible in seconds.
764
+ #
765
+ # Runs on whatever DISCOVERY returned. The codelines are resolved per ticket at
766
+ # runtime, so preparing a fixed list would hardcode discovery's output.
767
+ # lib/codeline-health.sh knows no package manager, runner or language — it
768
+ # reads what each codeline declares and prepares it accordingly.
769
+ if [ -f "$SCRIPT_DIR/lib/codeline-health.sh" ]; then
770
+ local _ch_paths=()
771
+ for _entry in "${_cl_entries[@]}"; do _ch_paths+=("${_entry#*:}"); done
772
+ log "[orch] Assessing health of ${#_ch_paths[@]} codeline(s) before starting work..."
773
+ if ! NODE_BIN="$NODE_BIN" bash "$SCRIPT_DIR/lib/codeline-health.sh" "${_ch_paths[@]}"; then
774
+ error "[orch] One or more codelines are UNHEALTHY — aborting before any spend."
775
+ error "[orch] A codeline that cannot resolve its own declared tooling cannot run its gates,"
776
+ error "[orch] so the run would accept an unverified baseline for that lane."
777
+ error "[orch] Set SKIP_CODELINE_HEALTH=1 to proceed knowing the gates cannot run."
778
+ return 1
779
+ fi
780
+ fi
781
+
782
+ # Tear down and re-scaffold every codeline worktree so each run starts clean.
783
+ # Pre-existing worktrees from prior runs poison the next run (stale package.json,
784
+ # accumulated artifacts, mid-run file mutations). Full deletion is the only safe state.
785
+ for _entry in "${_cl_entries[@]}"; do
786
+ local _cl="${_entry%%:*}" _wt="${_entry#*:}"
787
+ log "[orch] Codeline '${_cl}' → ${_wt}"
788
+
789
+ if [ "${EPAM_BROWNFIELD:-0}" != "1" ]; then
790
+ # ── Greenfield: full teardown + git init so every run starts clean ──
791
+ # Stale package.json, accumulated artifacts, and mid-run file mutations from
792
+ # prior runs all poison the next run. Full deletion is the only safe state.
793
+ # Some node_modules subdirs end up 0444/0555 after npm install — force-write
794
+ # access first so rm -rf is never blocked by restrictive permissions.
795
+ if [ -d "$_wt" ]; then
796
+ log "[orch] Tearing down '${_cl}' worktree at ${_wt}..."
797
+ chmod -R u+w "$_wt" 2>/dev/null || true
798
+ rm -rf "$_wt"
799
+ fi
800
+
801
+ local _scaffold="$SCRIPT_DIR/scaffold-${_cl}-repo.sh"
802
+ if [ -f "$_scaffold" ]; then
803
+ log "[orch] Scaffolding '${_cl}' repo at ${_wt}..."
804
+ bash "$_scaffold" "$_wt" 2>&1 | tee -a "$_log_file"
805
+ else
806
+ # No scaffold script — create a bare git repo; agents fill in the stack.
807
+ log "[orch] No scaffold-${_cl}-repo.sh found — creating bare git repo at ${_wt}..."
808
+ mkdir -p "$_wt"
809
+ git -C "$_wt" init --quiet
810
+ git -C "$_wt" config user.email "epam-cli@local"
811
+ git -C "$_wt" config user.name "epam-cli"
812
+ git -C "$_wt" commit --allow-empty -m "init: ${_cl} codeline worktree" --quiet
813
+ fi
814
+ else
815
+ # ── Brownfield: verify the existing repo; no teardown ──────────────────
816
+ # The worktree must already exist as a git repository — agents modify
817
+ # existing files in-place rather than building from scratch.
818
+ if [ ! -d "$_wt" ]; then
819
+ error "[orch] Brownfield worktree does not exist: ${_wt}"
820
+ error "[orch] Codeline '${_cl}' must point to an existing local repository."
821
+ return 1
822
+ fi
823
+ if [ ! -e "$_wt/.git" ]; then
824
+ error "[orch] Brownfield worktree is not a git repository: ${_wt}"
825
+ error "[orch] Ensure the path contains a .git directory."
826
+ return 1
827
+ fi
828
+ log "[orch] Brownfield: using existing worktree '${_cl}' at ${_wt}"
829
+
830
+ # Capture baseline SHA so gate diff oracles (review-ranger, mutant-hunter,
831
+ # fuzz-weaver, sast-sentinel) diff only the story's changes against the
832
+ # project's main branch — not the full commit history.
833
+ local _baseline_branch="${JIRA_BASELINE_BRANCH:-main}"
834
+ local _baseline_sha=""
835
+ # --verify --quiet: fail cleanly for a ref that does not exist. Without it,
836
+ # `git rev-parse origin/develop` ECHOES the literal "origin/develop" to stdout and exits
837
+ # 128, so the || chain runs the next command and the substitution captures BOTH — a
838
+ # two-line value whose first line is not a SHA. Every gate diff oracle reads this file
839
+ # (review-ranger, mutant-hunter, fuzz-weaver, sast-sentinel), which is the known
840
+ # "reviewers saw 0 files" failure. brownfield-preflight-reset.sh already guards this
841
+ # exact shape; this call site did not. A repo with no remote hits it on every lane.
842
+ _baseline_sha=$(git -C "$_wt" rev-parse --verify --quiet "origin/${_baseline_branch}" 2>/dev/null || \
843
+ git -C "$_wt" rev-parse --verify --quiet "${_baseline_branch}" 2>/dev/null || \
844
+ git -C "$_wt" rev-parse --verify --quiet HEAD 2>/dev/null || echo "")
845
+ if [ -n "$_baseline_sha" ]; then
846
+ # NOT WRITTEN HERE. See above: nothing reads this path, and the value means something
847
+ # different from the one Step 8 writes into the lane's own directory.
848
+ log "[orch] Brownfield baseline branch: ${_baseline_branch} @ ${_baseline_sha:0:8}"
849
+ else
850
+ warning "[orch] Could not resolve baseline branch '${_baseline_branch}' — gate diffs will use HEAD"
851
+ fi
852
+ fi
853
+ # THE CODELINE'S DECLARATION IS DETECTED, NEVER AUTHORED HERE.
854
+ #
855
+ # This wrote three .epam/ manifests from heredocs -- 80 lines asserting, of a repository the
856
+ # engine had never inspected, that its manifest is package.json, its sources are .ts/.tsx/.js,
857
+ # its vendor directory is node_modules, and that it must carry typescript, @types/node, vitest
858
+ # and tsx. Seventeen scripts read dependency-check.json as the codeline's OWN declaration, so
859
+ # fabricating it defeated every generic component downstream at the source: a non-Node codeline
860
+ # was handed a document saying it is TypeScript, in client-repo space, with facts nobody
861
+ # detected.
862
+ #
863
+ # lib/handlers/codeline-manifests.js assembles them from the provider whose manifest the
864
+ # repository actually carries. A codeline no provider recognises gets NOTHING and is reported
865
+ # -- an undeclared codeline is a state to surface, never one to invent an answer for.
866
+ #
867
+ # A DECLARATION THE CODELINE CONTRADICTS IS RE-DERIVED. A greenfield launch seeds .epam/ from
868
+ # the project directory before the codeline exists, so those files are the project author's
869
+ # word about a stack not yet built. When the codeline then carries a manifest a provider
870
+ # recognises and the seeded declaration names a manifest file that is not there, the codeline
871
+ # is the truth and the seed was a guess: live 2026-09-12, a Python greenfield project carried
872
+ # package.json / npm / vitest manifests copied from another project, and every generic gate
873
+ # would have judged a pytest repository as TypeScript. Derived, and the seeded siblings the
874
+ # provider does not declare are removed with it — they described the same wrong stack.
875
+ local _clm_contradicted=0
876
+ if [ -f "$_wt/.epam/dependency-check.json" ]; then
877
+ local _clm_declared_manifest
878
+ _clm_declared_manifest=$(jq -r '.manifestFile // ""' "$_wt/.epam/dependency-check.json" 2>/dev/null || echo "")
879
+ if [ -n "$_clm_declared_manifest" ] && [ ! -f "$_wt/$_clm_declared_manifest" ] \
880
+ && "$NODE_BIN" -e 'process.exit(require(process.argv[1]).resolveEcosystem(process.argv[2]) ? 0 : 1)' \
881
+ "$SCRIPT_DIR/lib/handlers/codeline-manifests.js" "$_wt" 2>/dev/null; then
882
+ _clm_contradicted=1
883
+ warning "[orch] ${_wt}: .epam/dependency-check.json declares manifest '${_clm_declared_manifest}', which this codeline does not carry — the declaration is re-derived from the ecosystem the codeline actually has"
884
+ fi
885
+ fi
886
+ if [ ! -f "$_wt/.epam/dependency-check.json" ] || [ "$_clm_contradicted" = "1" ]; then
887
+ local _clm _clm_rc=0
888
+ _clm=$("$NODE_BIN" "$SCRIPT_DIR/lib/handlers/codeline-manifests.js" "$_wt" 2>&1) || _clm_rc=$?
889
+ if [ "$_clm_rc" -ne 0 ]; then
890
+ warning "[orch] ${_wt}: no provider declares how this codeline is checked — .epam/ manifests NOT written"
891
+ printf '%s\n' "$_clm" | sed 's/^/ /' >&2
892
+ else
893
+ mkdir -p "$_wt/.epam"
894
+ if [ "$_clm_contradicted" = "1" ]; then
895
+ local _clm_stale
896
+ for _clm_stale in dependency-check.json contract-generation.json known-fixes.json; do
897
+ if [ -f "$_wt/.epam/$_clm_stale" ] && ! printf '%s' "$_clm" | jq -e --arg k "$_clm_stale" 'has($k)' >/dev/null 2>&1; then
898
+ rm -f "$_wt/.epam/$_clm_stale"
899
+ log "[orch] Removed seeded .epam/${_clm_stale}: it described a stack this codeline is not, and its ecosystem declares no replacement"
900
+ fi
901
+ done
902
+ fi
903
+ printf '%s' "$_clm" | jq -r 'keys[]' | while IFS= read -r _mf; do
904
+ [ -z "$_mf" ] && continue
905
+ printf '%s' "$_clm" | jq --arg k "$_mf" '.[$k]' > "$_wt/.epam/$_mf"
906
+ done
907
+ log "[orch] Wrote .epam/ manifests to ${_wt} from the resolved ecosystem provider"
908
+ fi
909
+ fi
910
+ # A DECLARATION MISSING A FACT ITS ECOSYSTEM DECLARES IS COMPLETED. The plug-in gained
911
+ # provisionCommand and runEnvironment on 2026-09-16; a manifest written before that carries
912
+ # neither, and a worktree provisioned from it has no environment. Only ABSENT keys are added
913
+ # — every key the manifest already holds is the project's word and stays as it is.
914
+ if [ -f "$_wt/.epam/dependency-check.json" ]; then
915
+ local _clm_more _clm_more_rc=0
916
+ _clm_more=$("$NODE_BIN" "$SCRIPT_DIR/lib/handlers/codeline-manifests.js" "$_wt" 2>/dev/null) || _clm_more_rc=$?
917
+ if [ "$_clm_more_rc" -eq 0 ]; then
918
+ local _clm_added
919
+ _clm_added=$(jq -n --argjson have "$(cat "$_wt/.epam/dependency-check.json")" --argjson derive "$(printf '%s' "$_clm_more" | jq '."dependency-check.json" // {}')" \
920
+ '[($derive | keys[]) as $k | select(($have | has($k)) | not) | $k]' 2>/dev/null || echo '[]')
921
+ if [ "$(printf '%s' "$_clm_added" | jq 'length')" -gt 0 ]; then
922
+ jq -s '.[1] * .[0]' "$_wt/.epam/dependency-check.json" <(printf '%s' "$_clm_more" | jq '."dependency-check.json"') > "$_wt/.epam/dependency-check.json.tmp" \
923
+ && mv "$_wt/.epam/dependency-check.json.tmp" "$_wt/.epam/dependency-check.json"
924
+ log "[orch] Completed .epam/dependency-check.json in ${_wt} with $(printf '%s' "$_clm_added" | jq -r 'join(", ")') from its ecosystem provider"
925
+ fi
926
+ fi
927
+ fi
928
+
929
+ # ── Plugin provisioning: config-driven, zero project-specific hardcoding ──
930
+ # This script has no idea what a "plugin" IS or does, with ONE exception:
931
+ # CodeGraph's query tool (orchestrations/plugins/codegraph-plugin.js) ships
932
+ # with epam-cli itself — the same way ReadFile/Bash are always available
933
+ # regardless of project — so it's provisioned for EVERY codeline
934
+ # unconditionally, merged with whatever the project's own plugins.json
935
+ # adds on top (never overwritten, never required to list it manually).
936
+ # Everything else remains purely config-driven: EPAM_PROJECT_CONFIG_DIR/
937
+ # codeline-facts.json (keyed by codeline name) has its entry for THIS
938
+ # codeline extracted into .epam/codeline-facts.json. Adding, removing, or
939
+ # repointing a PROJECT-specific plugin is purely a config edit in the
940
+ # project's own directory — never a change to this script.
941
+ local _project_tools_json="[]"
942
+ if [ -n "${EPAM_PROJECT_CONFIG_DIR:-}" ] && [ -f "${EPAM_PROJECT_CONFIG_DIR}/plugins.json" ]; then
943
+ _project_tools_json=$(jq -c '.tools // []' "${EPAM_PROJECT_CONFIG_DIR}/plugins.json" 2>/dev/null || echo "[]")
944
+ fi
945
+ local _codegraph_plugin_abs=""
946
+ local _codegraph_plugin_src="${SCRIPT_DIR}/../plugins/codegraph-plugin.js"
947
+ if [ -f "$_codegraph_plugin_src" ]; then
948
+ _codegraph_plugin_abs="$(cd "$(dirname "$_codegraph_plugin_src")" 2>/dev/null && pwd)/$(basename "$_codegraph_plugin_src")"
949
+ fi
950
+ if [ -n "$_codegraph_plugin_abs" ] || [ "$_project_tools_json" != "[]" ]; then
951
+ mkdir -p "$_wt/.epam"
952
+ jq -n --argjson project "$_project_tools_json" --arg cg "$_codegraph_plugin_abs" \
953
+ '{tools: (((if $cg != "" then [$cg] else [] end) + $project) | unique)}' \
954
+ > "$_wt/.epam/settings.json"
955
+ log "[orch] Provisioned .epam/settings.json (plugins, incl. built-in CodeGraph tool) for '${_cl}'"
956
+ fi
957
+
958
+ if [ -n "${EPAM_PROJECT_CONFIG_DIR:-}" ]; then
959
+ local _facts_cfg="${EPAM_PROJECT_CONFIG_DIR}/codeline-facts.json"
960
+ if [ -f "$_facts_cfg" ]; then
961
+ local _cl_facts
962
+ _cl_facts=$(jq -c --arg cl "$_cl" '.[$cl] // empty' "$_facts_cfg" 2>/dev/null)
963
+ if [ -n "$_cl_facts" ]; then
964
+ mkdir -p "$_wt/.epam"
965
+ echo "$_cl_facts" > "$_wt/.epam/codeline-facts.json"
966
+ log "[orch] Provisioned .epam/codeline-facts.json for '${_cl}' from ${_facts_cfg}"
967
+ else
968
+ # SAY SO. This skipped in silence, so a file in the wrong SHAPE was indistinguishable
969
+ # from no file and from an agent that observed nothing — three different problems with
970
+ # one symptom, which is none. The file is keyed by codeline name at the top level;
971
+ # a hand-written one nesting them under another key parses fine and yields nothing for
972
+ # every codeline, which is exactly what happened to mock3's before discovery produced it.
973
+ warning "[orch] '${_facts_cfg}' exists but has no entry for codeline '${_cl}' — agents there will work from the source alone. Codeline names must be TOP-LEVEL keys; the engine reads one with jq '.[\$cl]'."
974
+ fi
975
+ fi
976
+
977
+ # .env.local: derived from the codeline's OWN .env.local.sample, not an
978
+ # engine-side list. See provision_env_local_from_sample().
979
+ provision_env_local_from_sample "$_wt" "$_wt/.env.local"
980
+ if [ -s "$_wt/.env.local" ]; then
981
+ log "[orch] Provisioned .env.local for '${_cl}' from ${_wt}/.env.local.sample"
982
+ fi
983
+
984
+ # anti-patterns.json (optional, per-project): not keyed by codeline —
985
+ # copied whole into .epam/ so the check_anti_patterns plugin tool (and
986
+ # run_anti_pattern_check's deterministic gate) both read the same,
987
+ # already-provisioned file instead of reaching across worktree
988
+ # boundaries into EPAM_PROJECT_CONFIG_DIR directly.
989
+ local _antipatterns_cfg="${EPAM_PROJECT_CONFIG_DIR}/anti-patterns.json"
990
+ if [ -f "$_antipatterns_cfg" ]; then
991
+ mkdir -p "$_wt/.epam"
992
+ cp "$_antipatterns_cfg" "$_wt/.epam/anti-patterns.json"
993
+ log "[orch] Provisioned .epam/anti-patterns.json for '${_cl}' from ${_antipatterns_cfg}"
994
+ fi
995
+ fi
996
+ done
997
+
998
+
999
+
1000
+ # Per-codeline execution loop
1001
+ local _work_dir; _work_dir="$(_run_work_dir)"
1002
+ local _overall=0 _completed_list="" _cross_prd="$_work_dir/cross.json"
1003
+ local _cl_prds=()
1004
+
1005
+
1006
+ # Rebuild cumulative cross-codeline PRD so later codelines can check
1007
+ # completed stories from earlier codelines (is_story_completed fallback).
1008
+ _rebuild_cross() {
1009
+ [ -z "$_completed_list" ] && { unset CROSS_CODELINE_PRD; return; }
1010
+ COMPLETED_LIST="$_completed_list" \
1011
+ "$NODE_BIN" "$SCRIPT_DIR/lib/handlers/rebuild-cross-contract.js" "${_cross_prd}" 2>/dev/null
1012
+ export CROSS_CODELINE_PRD="$_cross_prd"
1013
+ }
1014
+
1015
+ # Build filtered PRD containing only stories for codeline $1, written to $2.
1016
+ # Returns story count on stdout.
1017
+ _filtered_prd() {
1018
+ local _fcl="$1" _fout="$2" _fsrc="$3"
1019
+ JIRA_DEFAULT_CODELINE="${JIRA_DEFAULT_CODELINE:-}" \
1020
+ "$NODE_BIN" "$SCRIPT_DIR/lib/handlers/filtered-prd.js" "${_fsrc}" "${_fout}" "${_fcl}" 2>/dev/null
1021
+ }
1022
+
1023
+ # A LANE RESUMES FROM ITS OWN CHECKPOINT. On a resume, a lane whose own checkpoint was taken
1024
+ # pre-writer has finished its spec pass, and that checkpoint's PRD is the spec the operator
1025
+ # approved at pause 2. Rebuilding the lane's PRD from the canonical one instead — which a resume
1026
+ # restores from the run's post-roster checkpoint, taken before any spec — handed every lane a
1027
+ # PRD with no spec, so every lane ran its spec pass again, paid, and overwrote what had been
1028
+ # approved (AMSD-2974 writer relaunch, 2026-10-02). An edit the operator made at the pause to the
1029
+ # lane's working PRD is kept. A resume sent back to an earlier stage (EPAM_RESUME_FROM_STAGE)
1030
+ # redoes the work after it, so it never takes this path.
1031
+ # Sets _lane_spec_mode=0 and returns 0 when the lane resumes from its checkpoint.
1032
+ _lane_resume_prd() {
1033
+ local _rcl="$1" _rout="$2"
1034
+ _lane_spec_mode=""
1035
+ [ -n "${EPAM_RESUME_RUN:-}" ] || return 1
1036
+ local _rck="${EPAM_PROJECT_CONFIG_DIR:-${PROJECT_CONFIG_DIR:-}}/runs/${EPAM_RESUME_RUN}/lanes/${_rcl}/checkpoint"
1037
+ [ -f "$_rck/prd.json" ] && [ -f "$_rck/checkpoint.json" ] || return 1
1038
+ # POST-SPEC IS ENOUGH: a lane checkpointed after its spec pass holds its spec. Requiring pre-writer took a specified
1039
+ # lane for one with no checkpoint, and it specified again — gotransit and metrolinx, $22.33, AMSD-2974 run 9
1040
+ # (2026-10-14; operator: "not permitted"). What follows the spec pass still runs, and the pauses still decide.
1041
+ local _want; _want=$(_stage_rank post-spec)
1042
+ [ "$(_stage_rank "$(jq -r '.stage // empty' "$_rck/checkpoint.json" 2>/dev/null)")" -ge "$_want" ] || return 1
1043
+ if [ -n "${EPAM_RESUME_FROM_STAGE:-}" ] && [ "$(_stage_rank "$EPAM_RESUME_FROM_STAGE")" -lt "$_want" ]; then
1044
+ return 1
1045
+ fi
1046
+ # The approved PRD first, when the working copy lost its spec (lib/run-checkpoint.sh lane_restore_approved_prd).
1047
+ lane_restore_approved_prd "$_rck" "$_rout"
1048
+ # A SPEC BUILT FROM INPUTS THAT NO LONGER HOLD IS SPECIFIED AGAIN (lib/run-checkpoint.sh).
1049
+ if ! lane_spec_is_current "$_rck" "$_rout"; then
1050
+ log "[orch] Codeline '${_rcl}': not resuming its pre-writer checkpoint — the lane is specified again"
1051
+ return 1
1052
+ fi
1053
+ if _operator_edited "$_rout" "$_rck/reviewed/$(basename "$_rout")"; then
1054
+ log "[orch] Codeline '${_rcl}': resuming from its pre-writer checkpoint — KEEPING its PRD, edited at the pause"
1055
+ else
1056
+ cp "$_rck/prd.json" "$_rout" || return 1
1057
+ log "[orch] Codeline '${_rcl}': resuming from its pre-writer checkpoint — its spec pass is done; the PRD approved at the pause is the lane's PRD"
1058
+ fi
1059
+ _lane_spec_mode=0
1060
+ return 0
1061
+ }
1062
+
1063
+ # THE PATH THE CALLER PASSED, ACTUALLY FORWARDED.
1064
+ #
1065
+ # This took "$1" and never used it, so the handler read no PRD; and `2>/dev/null` hid the error
1066
+ # it raised about that. The phase list came back EMPTY, every lane's phase loop ran zero times,
1067
+ # and each lane reported "✓ completed" in five seconds having invoked nothing. Live 2026-08-17,
1068
+ # mock3: two lanes, two pending stories, no writer, no commit, "Pipeline complete".
1069
+ #
1070
+ # An empty phase list is a real answer ("this PRD declares no phases") and indistinguishable
1071
+ # from a failure to read one — so the failure must be loud, and is.
1072
+ _prd_phases() {
1073
+ local _pp_prd="${1:-${PRD_FILE:-}}"
1074
+ local _pp_err _pp_out _pp_rc=0
1075
+ _pp_err=$(mktemp "${TMPDIR:-/tmp}/prd-phases-err-XXXXXX")
1076
+ _pp_out=$("$NODE_BIN" "$SCRIPT_DIR/lib/handlers/prd-phases.js" "$_pp_prd" 2>"$_pp_err") || _pp_rc=$?
1077
+ if [ "$_pp_rc" -ne 0 ]; then
1078
+ error "[orch] could not read the phases of ${_pp_prd}: $(cat "$_pp_err" 2>/dev/null)"
1079
+ error "[orch] Refusing to treat that as 'no phases' — a lane with no phases does nothing and reports success."
1080
+ rm -f "$_pp_err"
1081
+ return 1
1082
+ fi
1083
+ rm -f "$_pp_err"
1084
+ printf '%s' "$_pp_out"
1085
+ }
1086
+
1087
+ # ── Lane execution: parallel by default ────────────────────────────────────
1088
+ # Lanes of a spanning story are independent units of work; running them in
1089
+ # sequence costs (N-1) x lane-duration in wall clock. On AMSD-2041 three
1090
+ # ~20-minute lanes take an hour to do 20 minutes of work.
1091
+ #
1092
+ # THREE THINGS SEQUENCING GAVE FOR FREE, and how each is kept:
1093
+ #
1094
+ # 1. The canonical PRD merge. Every lane writes the same file, so concurrent
1095
+ # merges would clobber each other. Workers therefore never merge: each
1096
+ # writes only its own filtered PRD, and the merges run here, after the
1097
+ # wait, one at a time in declared order.
1098
+ # 2. The halt rule. Sequencing prevented spend outright. In parallel the
1099
+ # lanes are already running, so the equivalent is to abort the survivors
1100
+ # the moment one fails. Money already spent cannot be recovered; money not
1101
+ # yet spent still can.
1102
+ # 3. The cross-codeline contract. _run_codeline_bridge feeds one lane's
1103
+ # exported API into the NEXT lane's prompt, which presupposes an upstream
1104
+ # that has already finished. In parallel there is no upstream, so it is
1105
+ # skipped with a warning rather than silently writing a contract nobody
1106
+ # consumed. Lanes that genuinely integrate must set
1107
+ # EPAM_PARALLEL_CODELINES=0.
1108
+ _EPAM_PARALLEL_LANES="${EPAM_PARALLEL_CODELINES:-1}"
1109
+ if [ "$_EPAM_PARALLEL_LANES" = "1" ] && [ "${#_cl_entries[@]}" -gt 1 ]; then
1110
+ local _p_cls=() _p_wts=() _p_prds=() _p_pids=() _p_spec=() _lane_spec_mode=""
1111
+ local _p_statusdir; _p_statusdir="$(mktemp -d /tmp/orch-lanes-XXXXXX)"
1112
+
1113
+ for _entry in "${_cl_entries[@]}"; do
1114
+ local _cl="${_entry%%:*}" _wt="${_entry#*:}"
1115
+ local _cl_prd="$_work_dir/${_cl}-prd.json"
1116
+ _cl_prds+=("$_cl_prd")
1117
+ local _n_stories
1118
+ if _lane_resume_prd "$_cl" "$_cl_prd"; then
1119
+ story_resume_completed_by_commit "$_wt" "${LOG_DIR:-/tmp}/lanes/${_cl}" "$_cl_prd"
1120
+ _n_stories=$(jq '(.stories // []) | length' "$_cl_prd" 2>/dev/null || echo 0)
1121
+ else
1122
+ _n_stories=$(_filtered_prd "$_cl" "$_cl_prd" "$_prd_path")
1123
+ fi
1124
+ # The lane's work copy carries the story's plan (a checkpoint may predate it; the checkpoint is not edited).
1125
+ "$NODE_BIN" "$SCRIPT_DIR/lib/handlers/carry-plan.js" "$_cl_prd" "$_prd_path" || true
1126
+ if [ "${_n_stories:-0}" -eq 0 ]; then
1127
+ log "[orch] Codeline '${_cl}': no stories — skipping"
1128
+ continue
1129
+ fi
1130
+ if lane_declared_passed "$_cl"; then
1131
+ log "[orch] Codeline '${_cl}': declared passed in this run (EPAM_PASSED_LANES) — its own line is not run again"
1132
+ # THE DECLARATION IS THE CODELINE'S RESULT (lib/handlers/record-declared-pass.js).
1133
+ "$NODE_BIN" "$SCRIPT_DIR/lib/handlers/record-declared-pass.js" "$_prd_path" "$_cl" || true
1134
+ # ITS OTHER RELEASE LINES STILL SHIP (lib/release-lines.sh): passed is the operator's word for the codeline's own line.
1135
+ _run_release_lines "$_cl" "$_cl_prd" "$_wt" "${LOG_DIR:-/tmp}/lanes/${_cl}" "$_log_file" "$_phase_filter" "$0" "$_prd_path" || _overall=1
1136
+ continue
1137
+ fi
1138
+ _p_cls+=("$_cl"); _p_wts+=("$_wt"); _p_prds+=("$_cl_prd"); _p_spec+=("${_lane_spec_mode}")
1139
+ log "[orch] Codeline '${_cl}' — ${_n_stories} stories → ${_wt}"
1140
+ done
1141
+
1142
+ if [ "${#_p_cls[@]}" -gt 1 ]; then
1143
+ warning "[orch] Parallel lanes: cross-codeline contract bridge SKIPPED — no lane is upstream of another."
1144
+ warning "[orch] Set EPAM_PARALLEL_CODELINES=0 if these lanes integrate with each other."
1145
+ fi
1146
+
1147
+ log "[orch] Launching ${#_p_cls[@]} codeline(s) in PARALLEL..."
1148
+ _lane_idx=0
1149
+ while [ "$_lane_idx" -lt "${#_p_cls[@]}" ]; do
1150
+ _cl="${_p_cls[$_lane_idx]}"
1151
+ _wt="${_p_wts[$_lane_idx]}"
1152
+ _cl_prd="${_p_prds[$_lane_idx]}"
1153
+ _lane_spec_mode="${_p_spec[$_lane_idx]:-}"
1154
+ _lane_status="${_p_statusdir}/${_cl}.status"
1155
+ (
1156
+ _log_file="${LOG_DIR:-/tmp}/lane-${_cl}.log"
1157
+ # ── Per-lane LOG_DIR ──────────────────────────────────────────────
1158
+ # Lanes used to inherit ONE LOG_DIR, and several files in it are read
1159
+ # back as STATE rather than merely written as logs. The proven case is
1160
+ # phase-baseline-sha.txt, the git SHA every diff-based gate uses to
1161
+ # decide what a story changed. With three lanes on three different
1162
+ # repositories the last writer won, so two lanes diffed against a commit
1163
+ # that does not exist in them — an empty diff, and review-ranger /
1164
+ # mutant-hunter / team-lead-review passing on ZERO files. A false pass on
1165
+ # unreviewed code, which is worse than a crash because it looks like
1166
+ # success. Live metrolinx 2026-07-29, killed on discovery of this.
1167
+ #
1168
+ # Scoping LOG_DIR fixes every such file at once — the baseline SHA, the
1169
+ # review-incomplete-<phase> flag (PHASE is identical across lanes), the
1170
+ # story-outputs manifest and the worktree logs — instead of patching each
1171
+ # call site in seven scripts and missing the eighth.
1172
+ _lane_log_dir="${LOG_DIR:-/tmp}/lanes/${_cl}"
1173
+ mkdir -p "$_lane_log_dir" 2>/dev/null || true
1174
+ _cl_failed=0
1175
+ local _phases=()
1176
+ mapfile -t _phases < <(_prd_phases "$_cl_prd")
1177
+ local _cl_failed=0
1178
+
1179
+ for _phase in "${_phases[@]}"; do
1180
+ [ -z "$_phase" ] && continue
1181
+ # When the tier3 launcher (or any external caller) manages phases externally
1182
+ # by calling run-agent-orchestration.sh once per phase, honour its filter so
1183
+ # only the requested phase runs. Without this, a --phase scaffold call would
1184
+ # still execute every phase across every codeline (double-execution bug).
1185
+ if [ -n "$_phase_filter" ] && [ "$_phase" != "$_phase_filter" ]; then
1186
+ log "[orch] Phase '${_phase}' — skipping (caller phase filter: '${_phase_filter}')"
1187
+ continue
1188
+ fi
1189
+ log "[orch] Phase '${_phase}' — codeline '${_cl}'..."
1190
+ local _lane_reset; _lane_reset="$(_lane_phase_args)"
1191
+ local _pex=0
1192
+ JIRA_CODELINE_RUN=1 \
1193
+ EPAM_CODELINE="$_cl" \
1194
+ LOG_DIR="$_lane_log_dir" \
1195
+ AGENT_IO_DIR="$_lane_log_dir/agent-io" \
1196
+ PRD_FILE="$_cl_prd" \
1197
+ EPAM_SPEC_MODE="${_lane_spec_mode:-${EPAM_SPEC_MODE:-}}" \
1198
+ PROJECT_ROOT="$_wt" \
1199
+ OUTPUT_DIR="$_wt" \
1200
+ PHASE="$_phase" \
1201
+ CROSS_CODELINE_PRD="${CROSS_CODELINE_PRD:-}" \
1202
+ bash "$0" ${_lane_reset:+"$_lane_reset"} 2>&1 | tee -a "$_log_file"
1203
+ # No pipefail in this script, so `| tee || _pex=` tests tee's exit (0) and never
1204
+ # fires — a phase exit-2 (gate block) was masked to _pex=0 → "done" → PASSED
1205
+ # (live AMSD-1820 run #3). Capture the inner orch's real exit; tee exits 0 so set -e is fine.
1206
+ _pex=${PIPESTATUS[0]}
1207
+
1208
+ # exit 2 = gate remediation applied — reset stories and retry once (mirrors tier3 launcher)
1209
+ # WHILE THE GATES REMEDIATE, THE PHASE IS RETRIED — remediation stays on (AMSD-2974 run 9, 2026-10-14: the one retry
1210
+ # ran with remediation off, so a second, different finding — mutant-hunter after spec-validator — could not heal and
1211
+ # ended the line). Exit 2 means a gate added something new to the story; the remediator adds nothing already
1212
+ # covered, so the loop ends when there is nothing new to fix (or the story's own budget does).
1213
+ while phase_exit_is_retryable "$_pex"; do
1214
+ log "[orch] '${_phase}' ('${_cl}'): $(phase_exit_describe "$_pex") — retrying the phase"
1215
+ # A test-only remediation keeps the stories' work (lib/phase-exit.sh phase_exit_retry_args).
1216
+ _rargs=(); mapfile -t _rargs < <(phase_exit_retry_args "$_pex")
1217
+ _pex=0; _rmark=$(phase_retry_mark "$_wt")
1218
+ JIRA_CODELINE_RUN=1 \
1219
+ EPAM_CODELINE="$_cl" \
1220
+ LOG_DIR="$_lane_log_dir" \
1221
+ AGENT_IO_DIR="$_lane_log_dir/agent-io" \
1222
+ PRD_FILE="$_cl_prd" \
1223
+ EPAM_SPEC_MODE="${_lane_spec_mode:-${EPAM_SPEC_MODE:-}}" \
1224
+ PROJECT_ROOT="$_wt" \
1225
+ OUTPUT_DIR="$_wt" \
1226
+ PHASE="$_phase" \
1227
+ CROSS_CODELINE_PRD="${CROSS_CODELINE_PRD:-}" \
1228
+ bash "$0" ${_rargs[@]+"${_rargs[@]}"} 2>&1 | tee -a "$_log_file"
1229
+ _pex=${PIPESTATUS[0]}
1230
+ if [ "$_pex" -eq 0 ]; then
1231
+ log "[orch] Self-healing retry succeeded for '${_phase}' ('${_cl}')"
1232
+ elif ! phase_exit_is_retryable "$_pex"; then
1233
+ error "[orch] Phase '${_phase}' for '${_cl}' failed after self-healing retry (exit $_pex)"
1234
+ elif ! phase_retry_progressed "$_wt" "$_rmark"; then
1235
+ # THE PIPELINE NEVER LOOPS (lib/phase-exit.sh): the retry changed nothing in the codeline, so another is no different.
1236
+ error "[orch] Phase '${_phase}' for '${_cl}': the self-healing retry changed nothing in the codeline (still ${_rmark:0:9}) — not retrying again"
1237
+ _pex=1
1238
+ fi
1239
+ done
1240
+
1241
+ if [ "$_pex" -ne 0 ]; then
1242
+ error "[orch] Phase '${_phase}' for '${_cl}' failed (exit $_pex)"
1243
+ _cl_failed=1; _overall=1; break
1244
+ fi
1245
+ if lane_paused_since "$_lane_log_dir" "$_lanes_started"; then
1246
+ log "[orch] Phase '${_phase}' — '${_cl}' PAUSED (inputs ready, writer not started)."
1247
+ else
1248
+ log "[orch] Phase '${_phase}' — '${_cl}' done."
1249
+ fi
1250
+ done
1251
+
1252
+ # A SHARED CODELINE SHIPS ON EVERY RELEASE LINE ITS STORY'S PLAN NAMES (lib/release-lines.sh): its stories,
1253
+ # done on its own base, are worked once more per line off it — 0, 1 or N, as the plan says.
1254
+ if [ "$_cl_failed" = "0" ]; then
1255
+ _run_release_lines "$_cl" "$_cl_prd" "$_wt" "$_lane_log_dir" "$_log_file" "$_phase_filter" "$0" "$_prd_path" || { _cl_failed=1; _overall=1; }
1256
+ fi
1257
+
1258
+ [ "$_cl_failed" = "0" ] && \
1259
+ _completed_list="${_completed_list:+${_completed_list}:}${_cl_prd}"
1260
+ echo "$_cl_failed" > "$_lane_status"
1261
+ ) &
1262
+ _p_pids+=("$!")
1263
+ _lane_idx=$(( _lane_idx + 1 ))
1264
+ done
1265
+
1266
+ _p_any_failed=0
1267
+ _p_running=1
1268
+ while [ "$_p_running" = "1" ]; do
1269
+ _p_running=0
1270
+ for _pid in "${_p_pids[@]}"; do
1271
+ kill -0 "$_pid" 2>/dev/null && _p_running=1
1272
+ done
1273
+ for _cl in "${_p_cls[@]}"; do
1274
+ if [ -f "${_p_statusdir}/${_cl}.status" ] && [ "$(cat "${_p_statusdir}/${_cl}.status" 2>/dev/null)" != "0" ]; then
1275
+ _p_any_failed=1
1276
+ fi
1277
+ done
1278
+ # Default: let sibling lanes run to natural completion even after one
1279
+ # lane has failed. Found live 2026-08-02 (Writer Retest, AMSD-2041):
1280
+ # gotransit failed on a deterministic gate blocking on a pre-existing,
1281
+ # unrelated broken import (see run_relative_import_check's scope fix,
1282
+ # same incident) — a FALSE failure — which killed upexpress and
1283
+ # metrolinx mid-attempt via SIGTERM, discarding real, valid,
1284
+ # independent work those lanes were producing. The per-lane fold-back
1285
+ # logic below already records each lane's own outcome independently
1286
+ # (a spanning story is complete only when NO lane is outstanding) —
1287
+ # nothing downstream needed the early kill. Set
1288
+ # EPAM_CASCADE_ABORT_ON_LANE_FAILURE=1 to restore the old behavior
1289
+ # (stop spending immediately once any lane's outcome is known to have
1290
+ # failed) for cost-conscious runs where sibling lanes' work has no
1291
+ # independent value once one lane is lost.
1292
+ if [ "${EPAM_CASCADE_ABORT_ON_LANE_FAILURE:-0}" = "1" ] && [ "$_p_any_failed" = "1" ] && [ "$_p_running" = "1" ]; then
1293
+ error "[orch] HALT: a codeline failed after its retries and self-heal completed."
1294
+ error "[orch] Aborting the codeline(s) still running (EPAM_CASCADE_ABORT_ON_LANE_FAILURE=1) —"
1295
+ error "[orch] recovery is exhausted, so letting them finish would spend on a run already decided."
1296
+ for _pid in "${_p_pids[@]}"; do _kill_lane_tree "$_pid"; done
1297
+ break
1298
+ fi
1299
+ [ "$_p_running" = "1" ] && sleep 5
1300
+ done
1301
+ wait 2>/dev/null || true
1302
+
1303
+ _lane_idx=0
1304
+ while [ "$_lane_idx" -lt "${#_p_cls[@]}" ]; do
1305
+ _cl="${_p_cls[$_lane_idx]}"
1306
+ _cl_prd="${_p_prds[$_lane_idx]}"
1307
+ _cl_failed=1
1308
+ [ -f "${_p_statusdir}/${_cl}.status" ] && _cl_failed="$(cat "${_p_statusdir}/${_cl}.status" 2>/dev/null || echo 1)"
1309
+ # ── Fold this lane's ledgers back into the parent ────────────────────
1310
+ # Per-lane LOG_DIR fixed cross-lane state corruption, and fragmented the
1311
+ # cost ledger as a side effect: every reader of the canonical path — the
1312
+ # dashboard, validate-dashboards.sh, the run report — saw an EMPTY
1313
+ # phase-cost.jsonl for a parallel run while the real records sat in
1314
+ # lanes/<codeline>/. Cost tracking that silently reports zero is worse than
1315
+ # none, so the append-only ledgers are concatenated back, in declared lane
1316
+ # order, once the lane has finished writing them.
1317
+ for _ledger in phase-cost.jsonl agent-activity.jsonl healing-events.jsonl phase-gates.jsonl; do
1318
+ if [ -n "${LOG_DIR:-}" ] && [ -s "${LOG_DIR}/lanes/${_cl}/${_ledger}" ]; then
1319
+ cat "${LOG_DIR}/lanes/${_cl}/${_ledger}" >> "${LOG_DIR}/${_ledger}" 2>/dev/null || true
1320
+ fi
1321
+ done
1322
+
1323
+ # PER-LANE OUTCOME, RECORDED. Lanes have their own log dir, PRD, worktree, investigator
1324
+ # and writer, run in parallel, and this script states that no lane is upstream of
1325
+ # another — then the result collapsed into one exit code, so one lane's gate decision
1326
+ # failed a run in which the others had cleared. Live 2026-08-07: two lanes reached the
1327
+ # writer pause cleanly and the run reported failure because a third was blocked by the
1328
+ # spec review gate. Nothing said which, or that two-thirds of the work was fine.
1329
+ _LANE_OUTCOMES="${_LANE_OUTCOMES:+${_LANE_OUTCOMES}
1330
+ }${_cl} $([ "$_cl_failed" = "0" ] && echo ok || echo blocked)"
1331
+ if [ "$_cl_failed" = "0" ]; then
1332
+ _completed_list="${_completed_list:+${_completed_list}:}${_cl_prd}"
1333
+ else
1334
+ # Always explain a failed lane, even when every lane had already
1335
+ # finished by the time the abort poll noticed. Without this the run ends
1336
+ # non-zero with no statement of which lane died or why the others were
1337
+ # stopped — the operator is left diffing timestamps.
1338
+ error "[orch] codeline '${_cl}' did not complete — its retries and self-heal are exhausted."
1339
+ # LANE INDEPENDENCE IS ABOUT WHAT KEEPS RUNNING — NOT ABOUT THE EXIT CODE.
1340
+ #
1341
+ # A lane that does not complete does not invalidate the ones that did, and it must not
1342
+ # kill a sibling still doing real work: that independence is real, and
1343
+ # EPAM_CASCADE_ABORT_ON_LANE_FAILURE=1 is the knob that restores the old cascade.
1344
+ #
1345
+ # But independence stops at the exit status. This briefly returned 0 when a lane had
1346
+ # failed, which is the silent-failure class this pipeline exists to eliminate: every
1347
+ # automated caller, wrapper and CI reader takes exit 0 as "the work is done", and the
1348
+ # per-lane summary printed below is prose no caller parses. A run in which any lane
1349
+ # exhausted its retries, its ladder and its self-heal is a failed run — it is reported
1350
+ # as one, and the summary says which lanes did complete so nothing is thrown away.
1351
+ error "[orch] HALT: codeline '${_cl}' failed — the run is reported as failed."
1352
+ error "[orch] Lanes that completed keep their work on their own per-story branch;"
1353
+ error "[orch] the lane summary below states exactly which."
1354
+ _overall=1
1355
+ fi
1356
+ # Merge this codeline's final story state (status/completed/completedAt/
1357
+ # testCriteria/etc — whatever claude.sh/TC-writer wrote into the filtered
1358
+ # temp copy during real execution) back into the canonical PRD. Without
1359
+ # this, _cl_prd is the only file that ever held the real completion
1360
+ # result, and it was being deleted at the end of the loop — so the
1361
+ # canonical PRD (the file every downstream consumer, dashboard, and test
1362
+ # actually reads) stayed "pending" forever even after a real, successful
1363
+ # run. Found live 2026-07-23 via mock1.
1364
+ # The merge itself is lib/story-merge.js: it is the same operation on both the
1365
+ # sequential and the parallel path, it decides what survives a multi-lane run,
1366
+ # and inline in a heredoc it could not be tested. See that module for why a
1367
+ # spanning story cannot be merged wholesale.
1368
+ if "$NODE_BIN" "$SCRIPT_DIR/lib/handlers/merge-lane-into-canonical.js" \
1369
+ "$SCRIPT_DIR" "${_prd_path}" "${_cl_prd}" "${_cl}"; then
1370
+ log "[orch] Merged codeline '${_cl}' story state back into canonical PRD"
1371
+ else
1372
+ # Previously 2>/dev/null: a failed merge was indistinguishable from a
1373
+ # successful one that logged nothing, and it silently discards this lane's
1374
+ # entire outcome — status, criteria and all.
1375
+ error "[orch] FAILED to merge codeline '${_cl}' back into the canonical PRD;"
1376
+ error "[orch] that lane's status and verification criteria are NOT recorded."
1377
+ _overall=1
1378
+ fi
1379
+ _lane_idx=$(( _lane_idx + 1 ))
1380
+ done
1381
+ rm -rf "$_p_statusdir" 2>/dev/null || true
1382
+ else
1383
+ for _entry in "${_cl_entries[@]}"; do
1384
+ local _cl="${_entry%%:*}" _wt="${_entry#*:}"
1385
+ local _cl_prd="$_work_dir/${_cl}-prd.json"
1386
+ _cl_prds+=("$_cl_prd")
1387
+
1388
+ local _n_stories _lane_spec_mode=""
1389
+ if _lane_resume_prd "$_cl" "$_cl_prd"; then
1390
+ story_resume_completed_by_commit "$_wt" "${LOG_DIR:-/tmp}/lanes/${_cl}" "$_cl_prd"
1391
+ _n_stories=$(jq '(.stories // []) | length' "$_cl_prd" 2>/dev/null || echo 0)
1392
+ else
1393
+ _n_stories=$(_filtered_prd "$_cl" "$_cl_prd" "$_prd_path")
1394
+ fi
1395
+ # The lane's work copy carries the story's plan (a checkpoint may predate it; the checkpoint is not edited).
1396
+ "$NODE_BIN" "$SCRIPT_DIR/lib/handlers/carry-plan.js" "$_cl_prd" "$_prd_path" || true
1397
+
1398
+ if [ "${_n_stories:-0}" -eq 0 ]; then
1399
+ log "[orch] Codeline '${_cl}': no stories — skipping"
1400
+ continue
1401
+ fi
1402
+ if lane_declared_passed "$_cl"; then
1403
+ log "[orch] Codeline '${_cl}': declared passed in this run (EPAM_PASSED_LANES) — its own line is not run again"
1404
+ # THE DECLARATION IS THE CODELINE'S RESULT (lib/handlers/record-declared-pass.js).
1405
+ "$NODE_BIN" "$SCRIPT_DIR/lib/handlers/record-declared-pass.js" "$_prd_path" "$_cl" || true
1406
+ # ITS OTHER RELEASE LINES STILL SHIP (lib/release-lines.sh): passed is the operator's word for the codeline's own line.
1407
+ _run_release_lines "$_cl" "$_cl_prd" "$_wt" "${LOG_DIR:-/tmp}/lanes/${_cl}" "$_log_file" "$_phase_filter" "$0" "$_prd_path" || _overall=1
1408
+ continue
1409
+ fi
1410
+
1411
+ _rebuild_cross
1412
+ log "[orch] Codeline '${_cl}' — ${_n_stories} stories → ${_wt}"
1413
+
1414
+ # A SEQUENTIAL LANE GETS ITS OWN LOG_DIR TOO. The parallel loop scoped it (2026-07-29: state
1415
+ # files read back across lanes); this loop did not, so every lane shared phase-baseline-sha.txt
1416
+ # and story-retry-state/. Live AMSD-2974 2026-10-05: cx-shared's baseline commit was recorded in
1417
+ # the upexpress and metrolinx checkpoints (a commit their repos do not have), and the gotransit
1418
+ # writer resumed at attempt 10/12 on a counter the lanes had shared.
1419
+ local _lane_log_dir="${LOG_DIR:-/tmp}/lanes/${_cl}"
1420
+ mkdir -p "$_lane_log_dir" 2>/dev/null || true
1421
+
1422
+ local _phases=()
1423
+ mapfile -t _phases < <(_prd_phases "$_cl_prd")
1424
+ local _cl_failed=0
1425
+
1426
+ for _phase in "${_phases[@]}"; do
1427
+ [ -z "$_phase" ] && continue
1428
+ # When the tier3 launcher (or any external caller) manages phases externally
1429
+ # by calling run-agent-orchestration.sh once per phase, honour its filter so
1430
+ # only the requested phase runs. Without this, a --phase scaffold call would
1431
+ # still execute every phase across every codeline (double-execution bug).
1432
+ if [ -n "$_phase_filter" ] && [ "$_phase" != "$_phase_filter" ]; then
1433
+ log "[orch] Phase '${_phase}' — skipping (caller phase filter: '${_phase_filter}')"
1434
+ continue
1435
+ fi
1436
+ log "[orch] Phase '${_phase}' — codeline '${_cl}'..."
1437
+ local _lane_reset; _lane_reset="$(_lane_phase_args)"
1438
+ local _pex=0
1439
+ JIRA_CODELINE_RUN=1 \
1440
+ EPAM_CODELINE="$_cl" \
1441
+ LOG_DIR="$_lane_log_dir" \
1442
+ AGENT_IO_DIR="$_lane_log_dir/agent-io" \
1443
+ PRD_FILE="$_cl_prd" \
1444
+ EPAM_SPEC_MODE="${_lane_spec_mode:-${EPAM_SPEC_MODE:-}}" \
1445
+ PROJECT_ROOT="$_wt" \
1446
+ OUTPUT_DIR="$_wt" \
1447
+ PHASE="$_phase" \
1448
+ CROSS_CODELINE_PRD="${CROSS_CODELINE_PRD:-}" \
1449
+ bash "$0" ${_lane_reset:+"$_lane_reset"} 2>&1 | tee -a "$_log_file"
1450
+ # No pipefail in this script, so `| tee || _pex=` tests tee's exit (0) and never
1451
+ # fires — a phase exit-2 (gate block) was masked to _pex=0 → "done" → PASSED
1452
+ # (live AMSD-1820 run #3). Capture the inner orch's real exit; tee exits 0 so set -e is fine.
1453
+ _pex=${PIPESTATUS[0]}
1454
+
1455
+ # exit 2 = gate remediation applied — reset stories and retry once (mirrors tier3 launcher)
1456
+ # WHILE THE GATES REMEDIATE, THE PHASE IS RETRIED — remediation stays on (AMSD-2974 run 9, 2026-10-14: the one retry
1457
+ # ran with remediation off, so a second, different finding — mutant-hunter after spec-validator — could not heal and
1458
+ # ended the line). Exit 2 means a gate added something new to the story; the remediator adds nothing already
1459
+ # covered, so the loop ends when there is nothing new to fix (or the story's own budget does).
1460
+ while phase_exit_is_retryable "$_pex"; do
1461
+ log "[orch] '${_phase}' ('${_cl}'): $(phase_exit_describe "$_pex") — retrying the phase"
1462
+ # A test-only remediation keeps the stories' work (lib/phase-exit.sh phase_exit_retry_args).
1463
+ _rargs=(); mapfile -t _rargs < <(phase_exit_retry_args "$_pex")
1464
+ _pex=0; _rmark=$(phase_retry_mark "$_wt")
1465
+ JIRA_CODELINE_RUN=1 \
1466
+ EPAM_CODELINE="$_cl" \
1467
+ LOG_DIR="$_lane_log_dir" \
1468
+ AGENT_IO_DIR="$_lane_log_dir/agent-io" \
1469
+ PRD_FILE="$_cl_prd" \
1470
+ EPAM_SPEC_MODE="${_lane_spec_mode:-${EPAM_SPEC_MODE:-}}" \
1471
+ PROJECT_ROOT="$_wt" \
1472
+ OUTPUT_DIR="$_wt" \
1473
+ PHASE="$_phase" \
1474
+ CROSS_CODELINE_PRD="${CROSS_CODELINE_PRD:-}" \
1475
+ bash "$0" ${_rargs[@]+"${_rargs[@]}"} 2>&1 | tee -a "$_log_file"
1476
+ _pex=${PIPESTATUS[0]}
1477
+ if [ "$_pex" -eq 0 ]; then
1478
+ log "[orch] Self-healing retry succeeded for '${_phase}' ('${_cl}')"
1479
+ elif ! phase_exit_is_retryable "$_pex"; then
1480
+ error "[orch] Phase '${_phase}' for '${_cl}' failed after self-healing retry (exit $_pex)"
1481
+ elif ! phase_retry_progressed "$_wt" "$_rmark"; then
1482
+ # THE PIPELINE NEVER LOOPS (lib/phase-exit.sh): the retry changed nothing in the codeline, so another is no different.
1483
+ error "[orch] Phase '${_phase}' for '${_cl}': the self-healing retry changed nothing in the codeline (still ${_rmark:0:9}) — not retrying again"
1484
+ _pex=1
1485
+ fi
1486
+ done
1487
+
1488
+ if [ "$_pex" -ne 0 ]; then
1489
+ error "[orch] Phase '${_phase}' for '${_cl}' failed (exit $_pex)"
1490
+ _cl_failed=1; _overall=1; break
1491
+ fi
1492
+ if lane_paused_since "$_lane_log_dir" "$_lanes_started"; then
1493
+ log "[orch] Phase '${_phase}' — '${_cl}' PAUSED (inputs ready, writer not started)."
1494
+ else
1495
+ log "[orch] Phase '${_phase}' — '${_cl}' done."
1496
+ fi
1497
+ done
1498
+
1499
+ # A SHARED CODELINE SHIPS ON EVERY RELEASE LINE ITS STORY'S PLAN NAMES (lib/release-lines.sh): its stories,
1500
+ # done on its own base, are worked once more per line off it — 0, 1 or N, as the plan says.
1501
+ if [ "$_cl_failed" = "0" ]; then
1502
+ _run_release_lines "$_cl" "$_cl_prd" "$_wt" "$_lane_log_dir" "$_log_file" "$_phase_filter" "$0" "$_prd_path" || { _cl_failed=1; _overall=1; }
1503
+ fi
1504
+
1505
+ # The append-only ledgers back into the parent, as the parallel loop folds them.
1506
+ for _ledger in phase-cost.jsonl agent-activity.jsonl healing-events.jsonl phase-gates.jsonl; do
1507
+ if [ -n "${LOG_DIR:-}" ] && [ -s "${_lane_log_dir}/${_ledger}" ]; then
1508
+ cat "${_lane_log_dir}/${_ledger}" >> "${LOG_DIR}/${_ledger}" 2>/dev/null || true
1509
+ fi
1510
+ done
1511
+
1512
+ [ "$_cl_failed" = "0" ] && \
1513
+ _completed_list="${_completed_list:+${_completed_list}:}${_cl_prd}"
1514
+
1515
+ # Run codeline-bridge agent after each successful codeline (multi-codeline PRDs only).
1516
+ # Extracts exported types/functions/endpoints and writes a cross-codeline contract
1517
+ # consumed by downstream codeline agents via CROSS_CODELINE_CONTRACT_<CL_UPPER>.
1518
+ if [ "$_cl_failed" = "0" ] && [ "${#_cl_entries[@]}" -gt 1 ]; then
1519
+ _run_codeline_bridge "$_cl" "$_wt" "$_cl_prd"
1520
+ fi
1521
+
1522
+ # Merge this codeline's final story state (status/completed/completedAt/
1523
+ # testCriteria/etc — whatever claude.sh/TC-writer wrote into the filtered
1524
+ # temp copy during real execution) back into the canonical PRD. Without
1525
+ # this, _cl_prd is the only file that ever held the real completion
1526
+ # result, and it was being deleted at the end of the loop — so the
1527
+ # canonical PRD (the file every downstream consumer, dashboard, and test
1528
+ # actually reads) stayed "pending" forever even after a real, successful
1529
+ # run. Found live 2026-07-23 via mock1.
1530
+ # The merge itself is lib/story-merge.js: it is the same operation on both the
1531
+ # sequential and the parallel path, it decides what survives a multi-lane run,
1532
+ # and inline in a heredoc it could not be tested. See that module for why a
1533
+ # spanning story cannot be merged wholesale.
1534
+ if "$NODE_BIN" "$SCRIPT_DIR/lib/handlers/merge-lane-into-canonical.js" \
1535
+ "$SCRIPT_DIR" "${_prd_path}" "${_cl_prd}" "${_cl}"; then
1536
+ log "[orch] Merged codeline '${_cl}' story state back into canonical PRD"
1537
+ else
1538
+ # Previously 2>/dev/null: a failed merge was indistinguishable from a
1539
+ # successful one that logged nothing, and it silently discards this lane's
1540
+ # entire outcome — status, criteria and all.
1541
+ error "[orch] FAILED to merge codeline '${_cl}' back into the canonical PRD;"
1542
+ error "[orch] that lane's status and verification criteria are NOT recorded."
1543
+ _overall=1
1544
+ fi
1545
+
1546
+ # ── Halt once a lane has finally failed ───────────────────────────────────
1547
+ # The failure path inside the PHASE loop above ends in a bare `break`, which
1548
+ # leaves that inner loop only — the CODELINE loop carried on to the next
1549
+ # lane. Live AMSD-2041 (2026-07-28): 'gotransit' hit a FATAL after 3/3 spec
1550
+ # attempts and the self-heal retry, and five seconds later the run started
1551
+ # 'upexpress'. A comment below this loop asserted "Lane failures already stop
1552
+ # the loop"; it was never true, and being written down is what kept anyone
1553
+ # from checking.
1554
+ #
1555
+ # By this point retries, the ladder and self-heal have ALL completed and the
1556
+ # step is still failed — the standing mandate is to stop there. The merge
1557
+ # above runs first deliberately, so the canonical PRD still records where the
1558
+ # run died. Nothing is guessed about WHY it failed: a lane that reached here
1559
+ # exhausted every recovery the pipeline has.
1560
+ if [ "$_cl_failed" = "1" ]; then
1561
+ error "[orch] HALT: codeline '${_cl}' failed."
1562
+ _halt_recovery_state "${_cl_failed_story:-}"
1563
+ error "[orch] Not starting the remaining codeline(s)."
1564
+ break
1565
+ fi
1566
+ done
1567
+ fi
1568
+
1569
+ # ── Partial coverage is a failure, not a pass ──────────────────────────────
1570
+ # A spanning story names the codelines it must be delivered in. Lane failures
1571
+ # already stop the loop; this catches the quieter case — a lane that never ran,
1572
+ # or ran and produced no result for this story — where every lane "succeeded",
1573
+ # the pipeline reports complete, and part of the work simply never happened.
1574
+ # Checked against what the story DECLARED, not against how many lanes we
1575
+ # happened to execute.
1576
+ if [ "$_overall" = "0" ] && [ -f "$_prd_path" ]; then
1577
+ _mc_incomplete=$("$NODE_BIN" "$SCRIPT_DIR/lib/handlers/spanning-stories-incomplete.js" "$_prd_path" 2>/dev/null || true)
1578
+ if [ -n "$_mc_incomplete" ]; then
1579
+ error "[orch] Spanning story INCOMPLETE — a declared codeline never ran: ${_mc_incomplete}"
1580
+ error "[orch] The run touched fewer codelines than the story requires; this is not a success."
1581
+ _overall=1
1582
+ fi
1583
+ fi
1584
+
1585
+ # NOT deleted. This `rm -f` was /tmp hygiene: the lane PRDs used to live in a shared,
1586
+ # machine-global namespace and had to be swept. They now live in this run's OWN directory
1587
+ # (<project-config>/runs/<run-id>/work/), which makes them run EVIDENCE — the archiver
1588
+ # reads the working PRD from there, and deleting it first is why working-prd.json came
1589
+ # back "missing" on run 20260805T182214Z after the isolation fix.
1590
+ #
1591
+ # The cross-lane scratch file has no evidentiary value and is still removed.
1592
+ rm -f "$_cross_prd" 2>/dev/null || true
1593
+ unset CROSS_CODELINE_PRD
1594
+
1595
+ # THE SUMMARY IS THE POINT. A story spanning codelines can now finish with some lanes
1596
+ # complete and some not, and the one thing that must never happen is someone merging two of
1597
+ # three without knowing the third is missing.
1598
+ if [ -n "${_LANE_OUTCOMES:-}" ]; then
1599
+ local _ok_n _blocked_n
1600
+ _ok_n=$(printf '%s\n' "$_LANE_OUTCOMES" | grep -c 'ok$' || true)
1601
+ _blocked_n=$(printf '%s\n' "$_LANE_OUTCOMES" | grep -c 'blocked$' || true)
1602
+ log "[orch] Lane outcomes — ${_ok_n} completed, ${_blocked_n} did not:"
1603
+ printf '%s\n' "$_LANE_OUTCOMES" | while IFS=$'\t' read -r _lc _ls; do
1604
+ [ -n "$_lc" ] || continue
1605
+ if [ "$_ls" = "ok" ]; then log "[orch] ✓ ${_lc}"; else error "[orch] ✗ ${_lc} — did not complete"; fi
1606
+ done
1607
+ if [ "${_blocked_n:-0}" != "0" ] && [ "${_ok_n:-0}" != "0" ]; then
1608
+ warning "[orch] This story spans codelines and did NOT complete on all of them."
1609
+ warning "[orch] Work sits on a per-story branch in each completed codeline; merging is yours to decide."
1610
+ warning "[orch] The run is reported as FAILED because a lane did not complete; the"
1611
+ warning "[orch] completed lanes' work is intact on their branches and is yours to merge."
1612
+ fi
1613
+ fi
1614
+
1615
+ [ "$_overall" = "0" ] \
1616
+ && # A RUN THAT SPENT MONEY MUST NOT REPORT NOTHING. Checked here, at the end, where the run knows
1617
+ # both what it did and what it recorded — see lib/cost-ledger.sh for the run this exists for.
1618
+ assert_cost_ledger_not_silently_empty || true
1619
+ # THE RECORDING IS NOT THE ARTEFACT. Langfuse holds the traces; a cassette is a directory on
1620
+ # disk, and it exists only if something writes it. Nothing did — the exporter had no caller —
1621
+ # so four runs on 2026-09-09 recorded 278 traces and left nothing replayable once the machine
1622
+ # restarted and Langfuse went down with it.
1623
+ #
1624
+ # Here, at completion, because that is when the run knows its own id and the recording is
1625
+ # whole. It cannot fail the run: the work is finished by this point, and a run that succeeded
1626
+ # must not be reported failed because a recording could not be fetched.
1627
+ # A RUN WHOSE LANES PAUSED IS PAUSED, NOT COMPLETE; and `log ... || error ...` always took the first branch (log returns
1628
+ # 0), so a failed run printed "Pipeline complete" too (live AMSD-2974 run 9: "complete", then "aborting").
1629
+ if [ "$_overall" = "0" ] && record_lanes_pause "$_lanes_started"; then
1630
+ :
1631
+ elif [ "$_overall" = "0" ]; then
1632
+ log "[orch] ✅ Pipeline complete."
1633
+ else
1634
+ error "[orch] ⚠️ Pipeline completed with errors."
1635
+ fi
1636
+
1637
+ return $_overall
1638
+ }
1639
+
1640
+ # derive_sandbox_base_image <prd_file>
1641
+ # Maps a project's OWN project.stack.language/.runtime (already written by
1642
+ # the LLM-based `epam new generate` PRD pipeline for every scaffolded
1643
+ # project — see src/scaffold/ManifestAnalyzer.ts's generatePrd(), currently
1644
+ # unread by anything downstream) to a Docker base image for the sandbox.
1645
+ # Generic pattern match, same shape as resolve_model_provider()'s
1646
+ # EPAM_MODEL_PROVIDER_MAP glob convention elsewhere in this pipeline — no
1647
+ # specific PROJECT is ever named here, only a small number of well-known
1648
+ # language keywords. Falls back to node:20-slim (also correct for THIS
1649
+ # project) when project.stack is missing/unrecognized, so an older PRD
1650
+ # without stack data still gets a sane default rather than a build failure.
1651
+ derive_sandbox_base_image() {
1652
+ local prd_file="$1"
1653
+ local stack_text=""
1654
+ if [ -f "$prd_file" ]; then
1655
+ stack_text=$(jq -r '[.project.stack.language, .project.stack.runtime] | map(select(. != null)) | join(" ")' "$prd_file" 2>/dev/null | tr '[:upper:]' '[:lower:]')
1656
+ fi
1657
+ case "$stack_text" in
1658
+ *python*) echo "python:3.11-slim" ;;
1659
+ *golang*|*"go "*|*go) echo "golang:1.22-bookworm" ;;
1660
+ *rust*) echo "rust:1.75-slim" ;;
1661
+ *node*|*typescript*|*javascript*) echo "node:20-slim" ;;
1662
+ *) echo "node:20-slim" ;;
1663
+ esac
1664
+ }