amsd-pipeline-shared 0.5.0 → 0.5.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/docs/gateway-baseline-2.0.80.md5 +47 -0
- package/docs/gateway-files.txt +47 -0
- package/orchestrations/config/repo-artifacts.json +23 -0
- package/orchestrations/config/resume-preserves.json +25 -0
- package/orchestrations/config/run-modes.json +47 -0
- package/orchestrations/scripts/lib/checkpoints.sh +175 -0
- package/orchestrations/scripts/lib/codeline-lanes.sh +1664 -0
- package/orchestrations/scripts/lib/codeline-packages.js +67 -0
- package/orchestrations/scripts/lib/codeline-write-perimeter.sh +318 -0
- package/orchestrations/scripts/lib/common.sh +66 -0
- package/orchestrations/scripts/lib/control-plane.sh +506 -0
- package/orchestrations/scripts/lib/ecosystem-registry.js +148 -0
- package/orchestrations/scripts/lib/engine-paths.sh +52 -0
- package/orchestrations/scripts/lib/flags.sh +38 -0
- package/orchestrations/scripts/lib/git-ops.sh +975 -0
- package/orchestrations/scripts/lib/halt-recovery.sh +56 -0
- package/orchestrations/scripts/lib/handlers/_read-input.js +88 -0
- package/orchestrations/scripts/lib/handlers/carry-plan.js +25 -0
- package/orchestrations/scripts/lib/handlers/cl-entries.js +10 -0
- package/orchestrations/scripts/lib/handlers/codeline-ecosystem.js +164 -0
- package/orchestrations/scripts/lib/handlers/codeline-manifests.js +148 -0
- package/orchestrations/scripts/lib/handlers/filtered-prd.js +85 -0
- package/orchestrations/scripts/lib/handlers/merge-lane-into-canonical.js +39 -0
- package/orchestrations/scripts/lib/handlers/package-engines-node.js +25 -0
- package/orchestrations/scripts/lib/handlers/prd-phases.js +46 -0
- package/orchestrations/scripts/lib/handlers/rebuild-cross-contract.js +7 -0
- package/orchestrations/scripts/lib/handlers/record-declared-pass.js +23 -0
- package/orchestrations/scripts/lib/handlers/release-line-prd.js +31 -0
- package/orchestrations/scripts/lib/handlers/release-lines.js +31 -0
- package/orchestrations/scripts/lib/handlers/repo-exclude-patterns.js +108 -0
- package/orchestrations/scripts/lib/handlers/resume-spec-output-present.js +16 -0
- package/orchestrations/scripts/lib/handlers/sibling-providers.js +43 -0
- package/orchestrations/scripts/lib/handlers/spanning-stories-incomplete.js +50 -0
- package/orchestrations/scripts/lib/handlers/spec-inputs-digest.js +48 -0
- package/orchestrations/scripts/lib/handlers/spec-predates-plan.js +40 -0
- package/orchestrations/scripts/lib/handlers/stories-lacking-spec.js +64 -0
- package/orchestrations/scripts/lib/handlers/write-missing-plans.js +92 -0
- package/orchestrations/scripts/lib/orchestration-resume.sh +74 -0
- package/orchestrations/scripts/lib/phase-exit.sh +90 -0
- package/orchestrations/scripts/lib/pipeline-commit.sh +19 -0
- package/orchestrations/scripts/lib/plan-hash.js +19 -0
- package/orchestrations/scripts/lib/plan-scope.js +73 -0
- package/orchestrations/scripts/lib/qa-gate-evidence.sh +235 -0
- package/orchestrations/scripts/lib/release-lines.sh +166 -0
- package/orchestrations/scripts/lib/resume-semantics.sh +31 -0
- package/orchestrations/scripts/lib/run-checkpoint.sh +842 -0
- package/orchestrations/scripts/lib/run-modes.sh +93 -0
- package/orchestrations/scripts/lib/story-outputs.sh +322 -0
- package/orchestrations/scripts/lib/story-retry-state.sh +355 -0
- package/package.json +1 -1
|
@@ -0,0 +1,1664 @@
|
|
|
1
|
+
#!/usr/bin/env bash
|
|
2
|
+
# One more pass per release line a shared codeline's story plan names (_run_release_lines, called by the lane loops below).
|
|
3
|
+
. "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/release-lines.sh"
|
|
4
|
+
. "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/story-outputs.sh"
|
|
5
|
+
# lane_declared_passed <codeline> — the operator declared this lane already passed in this run
|
|
6
|
+
# (EPAM_PASSED_LANES, space/comma separated). AMSD-2974, 2026-10-04: cx-shared reached GO before
|
|
7
|
+
# lanes recorded it; a resume re-ran it (--reset) and was OOM-killed. Operator-chosen.
|
|
8
|
+
lane_declared_passed() {
|
|
9
|
+
local _l
|
|
10
|
+
for _l in ${EPAM_PASSED_LANES//,/ }; do [ "$_l" = "${1:-}" ] && return 0; done
|
|
11
|
+
return 1
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
# codeline-lanes.sh — moved verbatim out of run-agent-orchestration.sh by tools/split-main-into-modules.py
|
|
15
|
+
# (12 functions). Sourced by run-agent-orchestration.sh; SCRIPT_DIR and the globals it sets
|
|
16
|
+
# are in scope exactly as they were. A move, not an edit: every body is byte-identical to
|
|
17
|
+
# the golden recorded at the move (see the identity test).
|
|
18
|
+
|
|
19
|
+
# ── WHICH ROLE IS THIS PROCESS? ──────────────────────────────────────────────
|
|
20
|
+
#
|
|
21
|
+
# This one script is BOTH the parent orchestrator and, re-invoked once per codeline with
|
|
22
|
+
# JIRA_CODELINE_RUN=1, each lane. That is a deliberate design — a lane gets the identical
|
|
23
|
+
# pipeline — but it means every per-run resource is allocated twice, and every parent-only
|
|
24
|
+
# step needs a guard that somebody has to REMEMBER to write. Two defects came from exactly
|
|
25
|
+
# that omission:
|
|
26
|
+
#
|
|
27
|
+
# - the resume block sat below the dispatch and ran in neither role correctly, so every
|
|
28
|
+
# "resume" silently started a fresh run;
|
|
29
|
+
# - the control-plane port derived identically in both roles, so the first lane killed the
|
|
30
|
+
# parent's control plane and took its port.
|
|
31
|
+
#
|
|
32
|
+
# The role is named here, once, so a step can DECLARE which role it belongs to instead of
|
|
33
|
+
# re-deriving it from a raw environment variable at each site. Defined at the very top
|
|
34
|
+
# because entry guards and port resolution both run before anything else.
|
|
35
|
+
#
|
|
36
|
+
# A lane is a lane by virtue of JIRA_CODELINE_RUN — that variable is the contract between the
|
|
37
|
+
# parent's re-invocation and this block, and is read NOWHERE ELSE. See the guard test in
|
|
38
|
+
# test/unit/orchestration/orchestrator-role-is-explicit.test.ts.
|
|
39
|
+
orch_role() {
|
|
40
|
+
if [ -n "${JIRA_CODELINE_RUN:-}" ]; then printf 'lane'; else printf 'parent'; fi
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
# is_lane — true in a per-codeline re-invocation. Guard work that is scoped to one repository.
|
|
44
|
+
is_lane() { [ "$(orch_role)" = 'lane' ]; }
|
|
45
|
+
|
|
46
|
+
# Detect the Node.js binary — checks fnm, nvm, and PATH in order.
|
|
47
|
+
# ── Which Node the CODELINE requires, and which one is installed ─────────────
|
|
48
|
+
#
|
|
49
|
+
# detect_node() listed four literal paths (v20.20.2 / v20.20.0 under fnm and nvm) and checked them
|
|
50
|
+
# BEFORE whatever `node` is on PATH. next.metrolinx.com declares Node 22 in three places — .nvmrc,
|
|
51
|
+
# package.json engines, and the .epam/codeline-facts.json this pipeline generated itself, which
|
|
52
|
+
# says "Node 22.x is required; other versions may not work" — and the engine ran its 245-file jest
|
|
53
|
+
# suite under 20 anyway. Any failure then reads as a defect in the story's code, and no env var
|
|
54
|
+
# existed to correct it.
|
|
55
|
+
#
|
|
56
|
+
# The requirement is a fact of the CODELINE. The runtime is DISCOVERED from what is installed.
|
|
57
|
+
# Neither is a literal here: replacing the v20 list with a v22 list would just move the defect.
|
|
58
|
+
_codeline_node_requirement() {
|
|
59
|
+
local _root="${1:-${PROJECT_ROOT:-}}"
|
|
60
|
+
[ -n "$_root" ] || return 0
|
|
61
|
+
# .nvmrc exists for exactly this purpose, so it is asked first.
|
|
62
|
+
if [ -f "$_root/.nvmrc" ]; then
|
|
63
|
+
head -1 "$_root/.nvmrc" | sed -E 's/^[[:space:]]*v?//; s/[^0-9].*$//' | tr -d '[:space:]'
|
|
64
|
+
return 0
|
|
65
|
+
fi
|
|
66
|
+
# Otherwise the manifest's own engines field. Major only: a codeline pinning a patch is
|
|
67
|
+
# pinning its own business, and the engine matches what it can actually satisfy.
|
|
68
|
+
if [ -f "$_root/package.json" ] && command -v jq >/dev/null 2>&1; then
|
|
69
|
+
jq -r '.engines.node // ""' "$_root/package.json" 2>/dev/null \
|
|
70
|
+
| sed -E 's/[^0-9]*([0-9]+).*/\1/' | tr -d '[:space:]'
|
|
71
|
+
return 0
|
|
72
|
+
fi
|
|
73
|
+
printf ''
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
# detect_node [codeline_root]
|
|
77
|
+
#
|
|
78
|
+
# Install ROOTS are locations of standard version managers, overridable with
|
|
79
|
+
# EPAM_NODE_INSTALL_ROOTS (colon-separated). Paths, not versions — adding a Node release requires
|
|
80
|
+
# no edit here.
|
|
81
|
+
detect_node() {
|
|
82
|
+
local _root="${1:-${PROJECT_ROOT:-}}"
|
|
83
|
+
local _want; _want="$(_codeline_node_requirement "$_root" 2>/dev/null)"
|
|
84
|
+
# A codeline that declares nothing inherits the ENGINE's own declaration — this repo states it
|
|
85
|
+
# in .nvmrc and package.json engines, so it is read the same way rather than written here.
|
|
86
|
+
# Without this, "declares nothing" would mean "highest installed", silently moving a green
|
|
87
|
+
# codeline (mock3 declares neither) onto a major nobody chose.
|
|
88
|
+
if [ -z "$_want" ]; then
|
|
89
|
+
_want="$(_codeline_node_requirement "${EPAM_ENGINE_ROOT:-$SCRIPT_DIR/../..}" 2>/dev/null)"
|
|
90
|
+
fi
|
|
91
|
+
local _roots="${EPAM_NODE_INSTALL_ROOTS:-$HOME/.local/share/fnm/node-versions:$HOME/.nvm/versions/node}"
|
|
92
|
+
|
|
93
|
+
local _cands="" _d _vd _ver _bin _major
|
|
94
|
+
local _oifs="$IFS"; IFS=':'
|
|
95
|
+
for _d in $_roots; do
|
|
96
|
+
IFS="$_oifs"
|
|
97
|
+
[ -d "$_d" ] || continue
|
|
98
|
+
for _vd in "$_d"/v*; do
|
|
99
|
+
[ -d "$_vd" ] || continue
|
|
100
|
+
_ver="$(basename "$_vd")"
|
|
101
|
+
_bin=""
|
|
102
|
+
[ -x "$_vd/bin/node" ] && _bin="$_vd/bin/node"
|
|
103
|
+
[ -z "$_bin" ] && [ -x "$_vd/installation/bin/node" ] && _bin="$_vd/installation/bin/node"
|
|
104
|
+
[ -n "$_bin" ] || continue
|
|
105
|
+
_major="${_ver#v}"; _major="${_major%%.*}"
|
|
106
|
+
if [ -n "$_want" ] && [ "$_major" != "$_want" ]; then continue; fi
|
|
107
|
+
_cands="${_cands}${_ver}\t${_bin}\n"
|
|
108
|
+
done
|
|
109
|
+
IFS=':'
|
|
110
|
+
done
|
|
111
|
+
IFS="$_oifs"
|
|
112
|
+
|
|
113
|
+
# Highest version wins within whatever survived the requirement filter.
|
|
114
|
+
if [ -n "$_cands" ]; then
|
|
115
|
+
printf '%b' "$_cands" | sort -V | tail -1 | cut -f2
|
|
116
|
+
return 0
|
|
117
|
+
fi
|
|
118
|
+
|
|
119
|
+
# PATH node only when it SATISFIES the requirement. Handing back a runtime the codeline said
|
|
120
|
+
# will not work is how a green suite stops meaning anything.
|
|
121
|
+
local _p; _p="$(command -v node 2>/dev/null || true)"
|
|
122
|
+
if [ -n "$_p" ] && [ -x "$_p" ]; then
|
|
123
|
+
if [ -z "$_want" ]; then echo "$_p"; return 0; fi
|
|
124
|
+
local _pv; _pv="$("$_p" --version 2>/dev/null)"; _pv="${_pv#v}"; _pv="${_pv%%.*}"
|
|
125
|
+
[ "$_pv" = "$_want" ] && { echo "$_p"; return 0; }
|
|
126
|
+
fi
|
|
127
|
+
echo ""
|
|
128
|
+
return 1
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
# resolve_codeline_node <codeline_root>
|
|
132
|
+
# Resolves a node binary satisfying <codeline_root>/package.json's OWN
|
|
133
|
+
# "engines.node" declaration, installing it on demand via fnm if not already
|
|
134
|
+
# present. Fully data-driven — the required version comes entirely from the
|
|
135
|
+
# codeline's own manifest; no version number is ever hardcoded here, so this
|
|
136
|
+
# works for any Node version any codeline happens to declare.
|
|
137
|
+
#
|
|
138
|
+
# Live bug this closes (2026-07-22): the regression guard ran a codeline's
|
|
139
|
+
# vitest using detect_node()'s orchestrator-side Node (whatever fnm/nvm
|
|
140
|
+
# version happens to be active for THIS shell — v24.14.1 at the time), not
|
|
141
|
+
# the Node version the codeline itself was built/tested against
|
|
142
|
+
# ("engines": {"node": "^22"}). Running vitest under a mismatched major Node
|
|
143
|
+
# version crashed outright (SIGBUS/segfault from a native module ABI break),
|
|
144
|
+
# which the regression guard then reported as "tests broken" even though the
|
|
145
|
+
# codeline's own tests were never actually exercised.
|
|
146
|
+
#
|
|
147
|
+
# Falls back to detect_node()'s existing generic candidate search if the
|
|
148
|
+
# codeline declares no engines.node, fnm is unavailable, or the declared
|
|
149
|
+
# range can't be resolved/installed for any reason — this must never be the
|
|
150
|
+
# thing that blocks a run outright.
|
|
151
|
+
# provision_env_local_from_sample <codeline_root> <dest_.env.local_path>
|
|
152
|
+
#
|
|
153
|
+
# Some client apps throw at config-load time (e.g. a CMS SDK guard) when an
|
|
154
|
+
# expected env var is merely ABSENT, well before any network call is made —
|
|
155
|
+
# blocking local tooling (type-check/lint via a pre-commit hook) even though
|
|
156
|
+
# nothing here ever talks to the real service. Values only need to be
|
|
157
|
+
# PRESENT, never real credentials.
|
|
158
|
+
#
|
|
159
|
+
# This used to be a per-codeline block hand-written into an engine-side
|
|
160
|
+
# env-vars.json (added 2026-08-02, commit 359f7fa) — keys picked by trial and
|
|
161
|
+
# error against whatever `tsc`/lint failure was visible at the time. It
|
|
162
|
+
# missed MANAGEMENT_TOKEN entirely (AMSD-2041, 2026-08-05) because nothing
|
|
163
|
+
# ever went back to keep that hand-picked list in sync with what the
|
|
164
|
+
# codeline's OWN config actually reads. A codeline that already declares its
|
|
165
|
+
# full set of expected vars in its own `.env.local.sample` makes any
|
|
166
|
+
# engine-side copy of that list redundant and guaranteed to drift.
|
|
167
|
+
#
|
|
168
|
+
# So: read the codeline's own sample file and derive placeholders from ITS
|
|
169
|
+
# keys, not a list a human maintains here. A new var the client adds is
|
|
170
|
+
# picked up on the next run with zero edits to this repo. A codeline with no
|
|
171
|
+
# sample file gets nothing — inventing keys nobody declared would be the same
|
|
172
|
+
# mistake in the other direction.
|
|
173
|
+
provision_env_local_from_sample() {
|
|
174
|
+
local codeline_root="$1" dest="$2"
|
|
175
|
+
local sample="$codeline_root/.env.local.sample"
|
|
176
|
+
[ -f "$sample" ] || return 0
|
|
177
|
+
|
|
178
|
+
# Every `KEY=` line (KEY is the only part that means anything — the
|
|
179
|
+
# sample's own values are just whatever placeholder or blank the client
|
|
180
|
+
# left there, not credentials to reuse). One deterministic placeholder
|
|
181
|
+
# per key, not a fixed table, so this needs no maintenance as the
|
|
182
|
+
# codeline's own required-var set changes.
|
|
183
|
+
grep -E '^[A-Za-z_][A-Za-z0-9_]*=' "$sample" | while IFS='=' read -r key _; do
|
|
184
|
+
printf '%s=sandbox-placeholder-%s\n' "$key" "$(printf '%s' "$key" | tr '[:upper:]_' '[:lower:]-')"
|
|
185
|
+
done > "$dest"
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
# provision_lane_worktree <codeline_root> <worktree_path>
|
|
189
|
+
#
|
|
190
|
+
# A brownfield lane's worktree gets what `git worktree add` cannot give it — the untracked state the
|
|
191
|
+
# pipeline provisioned into the codeline: the .epam/ declaration (copied), the env file (provisioned
|
|
192
|
+
# from the codeline's own sample, as the lane does), and the vendor directory provisioning populates
|
|
193
|
+
# (the declaration's FIRST vendorDir, linked). Nothing is installed: an install would reach the
|
|
194
|
+
# codeline's package registry, a client system (AMSD-2974, 2026-10-02). Same pattern as the baseline
|
|
195
|
+
# gates' worktrees (lib/tsc-baseline-gate.sh). Build output is not linked — it stays in the worktree.
|
|
196
|
+
provision_lane_worktree() {
|
|
197
|
+
local _root="${1:-}" _wt="${2:-}"
|
|
198
|
+
[ -n "$_root" ] && [ -d "$_root" ] && [ -n "$_wt" ] && [ -d "$_wt" ] || return 0
|
|
199
|
+
if [ -d "$_root/.epam" ] && [ ! -e "$_wt/.epam" ]; then
|
|
200
|
+
cp -r "$_root/.epam" "$_wt/.epam" 2>/dev/null || warning " [worktree] could not copy .epam/ into $_wt"
|
|
201
|
+
fi
|
|
202
|
+
if [ ! -e "$_wt/.env.local" ]; then
|
|
203
|
+
provision_env_local_from_sample "$_wt" "$_wt/.env.local"
|
|
204
|
+
fi
|
|
205
|
+
# The declaration is read where the baseline gate reads it (_bg_vendor_dirs): the project's own
|
|
206
|
+
# first, then the codeline's — ensure_story_branch cleans the checkout's untracked .epam/.
|
|
207
|
+
local _cfg="${EPAM_PROJECT_CONFIG_DIR:+$EPAM_PROJECT_CONFIG_DIR/dependency-check.json}" _vd=""
|
|
208
|
+
[ -n "$_cfg" ] && [ -f "$_cfg" ] || _cfg="$_root/.epam/dependency-check.json"
|
|
209
|
+
[ -f "$_cfg" ] && _vd=$(jq -r '.vendorDirs[0]? // empty' "$_cfg" 2>/dev/null || true)
|
|
210
|
+
if [ -n "$_vd" ] && [ -d "$_root/$_vd" ] && [ ! -e "$_wt/$_vd" ]; then
|
|
211
|
+
ln -s "$_root/$_vd" "$_wt/$_vd" || warning " [worktree] could not link $_vd into $_wt"
|
|
212
|
+
fi
|
|
213
|
+
[ -n "$_vd" ] && link_sibling_builds "$_root" "$_wt" "$_vd"
|
|
214
|
+
return 0
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
# link_sibling_builds <root> <worktree> <vendor-dir> — A SITE BUILDS AGAINST ITS SIBLING'S CHANGE.
|
|
218
|
+
#
|
|
219
|
+
# Operator, 2026-10-03: nothing installed, packed, published or fetched — "shared gets updated then
|
|
220
|
+
# dependent codelines get the branch". A codeline of a spanning story that declares a package another
|
|
221
|
+
# codeline of the SAME story provides (lib/handlers/sibling-providers.js) tests against that sibling's
|
|
222
|
+
# checkout — its story branch, its lane already merged — after the sibling's OWN declared build ran
|
|
223
|
+
# there (its ecosystem's buildCommand). The worktree gets its own vendor directory: a link to every
|
|
224
|
+
# entry of the codeline's, except the sibling's package, linked to the sibling's checkout. The
|
|
225
|
+
# codeline's own vendor directory is never touched, so its baseline keeps the installed release and
|
|
226
|
+
# a reproducing test still fails there.
|
|
227
|
+
# _link_entry <entry> <target> — point a vendor entry the worktree's own directory holds at <target>.
|
|
228
|
+
# The entry is only ever a link this file made (to the codeline's entry), so only a link is removed —
|
|
229
|
+
# never a directory, never anything whose path did not resolve.
|
|
230
|
+
_link_entry() {
|
|
231
|
+
local _at="$1" _to="$2"
|
|
232
|
+
[ -n "$_at" ] && [ -n "$_to" ] || return 1
|
|
233
|
+
if [ -L "$_at" ]; then rm -f -- "$_at"; elif [ -e "$_at" ]; then warning " [sibling] $_at is not a link — left as it is"; return 1; fi
|
|
234
|
+
ln -s "$_to" "$_at"
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
# _place_sibling_package <at> <provider_checkout> <vendor_dir> — THE SIBLING'S PACKAGE, AS AN INSTALL HAS IT.
|
|
238
|
+
# A hard-linked copy of every entry of the provider's checkout except its own dependency directory and
|
|
239
|
+
# .git: real files inside the lane, so Node and Jest resolve the package's dependencies from the SITE.
|
|
240
|
+
# A link to the checkout loaded cx-shared's own React beside gotransit's (AMSD-2974, 2026-10-04): two
|
|
241
|
+
# React instances, hooks broken, the site's tests failing whatever the writer did. Same disk, no copy
|
|
242
|
+
# cost, nothing fetched. Only a copy this function made (its marker) is ever replaced.
|
|
243
|
+
_place_sibling_package() {
|
|
244
|
+
local _at="$1" _from="$2" _vd="$3" _release="${4:-}" _e _n
|
|
245
|
+
[ -n "$_at" ] && [ -n "$_vd" ] && [ -d "$_from" ] || return 1
|
|
246
|
+
if [ -L "$_at" ]; then
|
|
247
|
+
rm -f -- "$_at"
|
|
248
|
+
elif [ -e "$_at" ]; then
|
|
249
|
+
if [ -f "$_at/.epam-sibling-copy" ]; then
|
|
250
|
+
rm -rf -- "${_at:?}"
|
|
251
|
+
else
|
|
252
|
+
warning " [sibling] $_at is not a copy this pipeline made — left as it is"
|
|
253
|
+
return 1
|
|
254
|
+
fi
|
|
255
|
+
fi
|
|
256
|
+
mkdir -p "$_at" || return 1
|
|
257
|
+
# SEEDED FROM THE RELEASE THE SITE HAS INSTALLED: what a release ships and the sibling's own build
|
|
258
|
+
# does not regenerate (cx-shared's migration-tool-build, built by its release process) stays.
|
|
259
|
+
if [ -n "$_release" ] && [ -d "$_release" ]; then
|
|
260
|
+
for _e in "$_release"/* "$_release"/.[!.]*; do
|
|
261
|
+
[ -e "$_e" ] || continue
|
|
262
|
+
cp -al "$_e" "$_at/$(basename "$_e")" 2>/dev/null || cp -a "$_e" "$_at/$(basename "$_e")"
|
|
263
|
+
done
|
|
264
|
+
fi
|
|
265
|
+
# The sibling's change on top: each of its entries replaces the release's.
|
|
266
|
+
for _e in "$_from"/* "$_from"/.[!.]*; do
|
|
267
|
+
[ -e "$_e" ] || continue
|
|
268
|
+
_n=$(basename "$_e")
|
|
269
|
+
case "$_n" in "$_vd"|.git) continue ;; esac
|
|
270
|
+
[ -e "$_at/$_n" ] || [ -L "$_at/$_n" ] && rm -rf -- "${_at:?}/${_n:?}"
|
|
271
|
+
cp -al "$_e" "$_at/$_n" 2>/dev/null || cp -a "$_e" "$_at/$_n"
|
|
272
|
+
done
|
|
273
|
+
: > "$_at/.epam-sibling-copy"
|
|
274
|
+
}
|
|
275
|
+
|
|
276
|
+
link_sibling_builds() {
|
|
277
|
+
local _root="$1" _wt="$2" _vd="$3" _pairs _name _prov _bc _scope _base _e _stamp _suffix _story _lb _ld
|
|
278
|
+
[ -n "${PRD_FILE:-}" ] && [ -f "$PRD_FILE" ] && [ -d "$_root/$_vd" ] || return 0
|
|
279
|
+
_pairs=$("${NODE_BIN:-node}" "$SCRIPT_DIR/lib/handlers/sibling-providers.js" "$PRD_FILE" "$_root" 2>/dev/null) || return 0
|
|
280
|
+
[ -n "$_pairs" ] || return 0
|
|
281
|
+
# The worktree's own vendor directory: every entry of the codeline's, linked.
|
|
282
|
+
if [ -L "$_wt/$_vd" ]; then rm -f "$_wt/$_vd"; fi
|
|
283
|
+
mkdir -p "$_wt/$_vd"
|
|
284
|
+
for _e in "$_root/$_vd"/* "$_root/$_vd"/.[!.]*; do
|
|
285
|
+
[ -e "$_e" ] || [ -L "$_e" ] || continue
|
|
286
|
+
[ -e "$_wt/$_vd/$(basename "$_e")" ] || ln -s "$_e" "$_wt/$_vd/$(basename "$_e")"
|
|
287
|
+
done
|
|
288
|
+
local _fresh_vd
|
|
289
|
+
while IFS=$'\t' read -r _name _prov _suffix _story; do
|
|
290
|
+
[ -n "$_name" ] && [ -d "$_prov" ] || continue
|
|
291
|
+
_fresh_vd=0
|
|
292
|
+
# THE PROVIDER'S RELEASE LINE THIS CONSUMER RECEIVES (the story's plan): built from that line's story branch, in a
|
|
293
|
+
# worktree of its own, so the consumer is proved against the back-port it will actually get.
|
|
294
|
+
if [ -n "${_suffix:-}" ] && [ -n "${_story:-}" ]; then
|
|
295
|
+
_lb="${EPAM_BRANCH_PREFIX:-}AI-${_story}${_suffix}"
|
|
296
|
+
_ld="${_prov}-line${_suffix}"
|
|
297
|
+
if git -C "$_prov" rev-parse -q --verify "refs/heads/${_lb}" >/dev/null 2>&1; then
|
|
298
|
+
if [ -d "$_ld" ]; then git -C "$_ld" checkout -q --detach "$_lb" 2>/dev/null || true
|
|
299
|
+
else git -C "$_prov" worktree add -q --detach "$_ld" "$_lb" 2>/dev/null || true; fi
|
|
300
|
+
# ITS DEPENDENCIES INSIDE IT, as the provider's checkout has them: a hard-linked copy, never a link —
|
|
301
|
+
# a bundler resolves real paths, and through a link it laid the line's build out from the parent
|
|
302
|
+
# folder (build/cx-shared-line-7.x/src/...), failing every consumer import (AMSD-2974 run 3).
|
|
303
|
+
if [ -L "$_ld/$_vd" ]; then rm -f -- "$_ld/$_vd"; fi
|
|
304
|
+
if [ ! -e "$_ld/$_vd" ] && [ -d "$_prov/$_vd" ] && [ -d "$_ld" ]; then
|
|
305
|
+
cp -al "$_prov/$_vd" "$_ld/$_vd" 2>/dev/null || cp -a "$_prov/$_vd" "$_ld/$_vd"
|
|
306
|
+
_fresh_vd=1
|
|
307
|
+
fi
|
|
308
|
+
[ -d "$_ld" ] && _prov="$_ld"
|
|
309
|
+
else
|
|
310
|
+
warning " [sibling] ${_name}: release line branch ${_lb} does not exist yet — testing against ${_prov} as it stands"
|
|
311
|
+
fi
|
|
312
|
+
fi
|
|
313
|
+
# The sibling's own build, once per checkout state: the only command it runs is the one the
|
|
314
|
+
# sibling declares; a failure is reported and the lane tests against what it has.
|
|
315
|
+
_bc=$("${NODE_BIN:-node}" "$SCRIPT_DIR/lib/handlers/codeline-ecosystem.js" "$_prov" 2>/dev/null | jq -r '.buildCommand // empty' 2>/dev/null)
|
|
316
|
+
_stamp="${LOG_DIR:-/tmp}/sibling-build-$(printf '%s' "$_prov" | md5sum | cut -c1-12)-$(git -C "$_prov" rev-parse --short HEAD 2>/dev/null || echo none)"
|
|
317
|
+
# A build made against another dependency directory is not this checkout's build.
|
|
318
|
+
[ "${_fresh_vd:-0}" = 1 ] && rm -f -- "$_stamp"
|
|
319
|
+
if [ -n "$_bc" ] && [ ! -f "$_stamp" ]; then
|
|
320
|
+
log " [sibling] building ${_name} from ${_prov} ($(git -C "$_prov" branch --show-current 2>/dev/null)): $_bc"
|
|
321
|
+
if (cd "$_prov" && bash -c "$_bc") > "${_stamp}.log" 2>&1; then
|
|
322
|
+
: > "$_stamp"
|
|
323
|
+
else
|
|
324
|
+
warning " [sibling] the build of ${_name} failed — see ${_stamp}.log; the lane tests against what ${_prov} holds"
|
|
325
|
+
fi
|
|
326
|
+
fi
|
|
327
|
+
# Link the sibling's package into the worktree's vendor directory, scope by scope.
|
|
328
|
+
case "$_name" in
|
|
329
|
+
*/*) _scope="${_name%%/*}"; _base="${_name#*/}"
|
|
330
|
+
if [ -L "$_wt/$_vd/$_scope" ]; then
|
|
331
|
+
rm -f "$_wt/$_vd/$_scope"; mkdir -p "$_wt/$_vd/$_scope"
|
|
332
|
+
for _e in "$_root/$_vd/$_scope"/*; do [ -e "$_e" ] && ln -s "$_e" "$_wt/$_vd/$_scope/$(basename "$_e")"; done
|
|
333
|
+
fi
|
|
334
|
+
mkdir -p "$_wt/$_vd/$_scope"
|
|
335
|
+
_place_sibling_package "$_wt/$_vd/$_scope/$_base" "$_prov" "$_vd" "$_root/$_vd/$_scope/$_base" ;;
|
|
336
|
+
*) _place_sibling_package "$_wt/$_vd/$_name" "$_prov" "$_vd" "$_root/$_vd/$_name" ;;
|
|
337
|
+
esac
|
|
338
|
+
log " [sibling] ${_name} → ${_prov} in $(basename "$_wt") (the codeline keeps its installed release)"
|
|
339
|
+
done <<< "$_pairs"
|
|
340
|
+
return 0
|
|
341
|
+
}
|
|
342
|
+
|
|
343
|
+
resolve_codeline_node() {
|
|
344
|
+
local codeline_root="$1"
|
|
345
|
+
local pkg="$codeline_root/package.json"
|
|
346
|
+
local required=""
|
|
347
|
+
|
|
348
|
+
if [ -f "$pkg" ]; then
|
|
349
|
+
required=$(node "$SCRIPT_DIR/lib/handlers/package-engines-node.js" "$pkg" 2>/dev/null || true)
|
|
350
|
+
fi
|
|
351
|
+
|
|
352
|
+
if [ -z "$required" ] || ! command -v fnm &>/dev/null; then
|
|
353
|
+
detect_node
|
|
354
|
+
return
|
|
355
|
+
fi
|
|
356
|
+
|
|
357
|
+
# fnm install/exec want a partial semver (e.g. "22", "20.10"), not a full
|
|
358
|
+
# range operator like "^22" or ">=22 <23" — extract the first version-like
|
|
359
|
+
# token from whatever the codeline declares. This is a heuristic, not a
|
|
360
|
+
# full semver-range resolver, but covers the common declaration shapes
|
|
361
|
+
# (^N, ~N, >=N, N.x, exact N.N.N) without hardcoding any specific version.
|
|
362
|
+
local fnm_version
|
|
363
|
+
fnm_version=$(echo "$required" | grep -oE '[0-9]+(\.[0-9]+){0,2}' | head -1)
|
|
364
|
+
|
|
365
|
+
if [ -z "$fnm_version" ]; then
|
|
366
|
+
detect_node
|
|
367
|
+
return
|
|
368
|
+
fi
|
|
369
|
+
|
|
370
|
+
fnm install "$fnm_version" >/dev/null 2>&1 || true
|
|
371
|
+
|
|
372
|
+
local resolved_bin
|
|
373
|
+
resolved_bin=$(fnm exec --using="$fnm_version" -- node -e 'process.stdout.write(process.execPath)' 2>/dev/null || true)
|
|
374
|
+
|
|
375
|
+
if [ -n "$resolved_bin" ] && [ -x "$resolved_bin" ]; then
|
|
376
|
+
echo "$resolved_bin"
|
|
377
|
+
return 0
|
|
378
|
+
fi
|
|
379
|
+
|
|
380
|
+
detect_node
|
|
381
|
+
}
|
|
382
|
+
|
|
383
|
+
# ensure_node_modules_healthy <codeline_root> <node_bin> <test_bin>
|
|
384
|
+
# Detects a missing or CORRUPTED dependency install (not just "missing") and
|
|
385
|
+
# repairs it via detect_and_install_dependencies() — never by editing any
|
|
386
|
+
# manifest file, which stays purely a manual, case-by-case decision (see the
|
|
387
|
+
# azure.commerce.cdts cx-shared incident, 2026-07-22 — that repair required
|
|
388
|
+
# temporarily stripping a private-registry dependency and was done as an
|
|
389
|
+
# explicit, user-approved one-off, not baked into automated tooling).
|
|
390
|
+
#
|
|
391
|
+
# Live bug this closes: a prior interrupted/killed install left node_modules
|
|
392
|
+
# with truncated native binaries (esbuild, rollup) — present on disk, correct
|
|
393
|
+
# file names, but silently corrupted. A plain "does node_modules exist" check
|
|
394
|
+
# would have missed this entirely; the failure only surfaced when vitest
|
|
395
|
+
# actually tried to load them and crashed outright (SIGBUS/segfault) instead
|
|
396
|
+
# of a normal test failure. This function directly smoke-tests the SAME
|
|
397
|
+
# binary the regression guard is about to invoke ("<test_bin> --version"),
|
|
398
|
+
# which is the cheapest, most direct way to know whether it will actually
|
|
399
|
+
# work — no need to guess at which native files might be corrupted, or fetch
|
|
400
|
+
# reference file sizes from a registry.
|
|
401
|
+
#
|
|
402
|
+
# Returns 0 (healthy or successfully repaired) or 1 (still broken after a
|
|
403
|
+
# genuine repair attempt — e.g. a private-registry auth wall with no
|
|
404
|
+
# credentials available). Never silent: logs what it found and did either way.
|
|
405
|
+
ensure_node_modules_healthy() {
|
|
406
|
+
local codeline_root="$1"
|
|
407
|
+
local node_bin="$2"
|
|
408
|
+
# Set for the child process invoked below, or read by a script that sources this file.
|
|
409
|
+
# ShellCheck cannot see the consumer, so it reports these unused; removing them takes the value away.
|
|
410
|
+
# shellcheck disable=SC2034
|
|
411
|
+
local test_bin="$3" # legacy: an arbitrary .bin entry, no longer trusted
|
|
412
|
+
|
|
413
|
+
# Probe the runner the PROJECT DECLARES, not whatever sorts first in
|
|
414
|
+
# node_modules/.bin. Live metrolinx 2026-07-29: the old probe picked
|
|
415
|
+
# `escodegen` (alphabetically first), ran `node escodegen --version`, got
|
|
416
|
+
# "Invalid option '--version'" — and condemned three codelines whose trees
|
|
417
|
+
# were fine (`jest --version` -> 29.5.0 on the same tree). It had always
|
|
418
|
+
# behaved this way; the caller's `|| true` hid it until that mask came off
|
|
419
|
+
# and every lane stopped.
|
|
420
|
+
local _declared _runner
|
|
421
|
+
_declared="$(jq -r '.scripts.test // ""' "$codeline_root/package.json" 2>/dev/null || echo "")"
|
|
422
|
+
# First word of the declared command is the runner: "jest --ci" -> jest.
|
|
423
|
+
# Derived from what the project says, so an unknown stack needs no changes.
|
|
424
|
+
_runner="${_declared%% *}"
|
|
425
|
+
|
|
426
|
+
if [ -z "$_runner" ]; then
|
|
427
|
+
# INCONCLUSIVE, not broken. We cannot identify a runner, so we cannot
|
|
428
|
+
# claim the tree is unusable — and halting on "cannot tell" is the
|
|
429
|
+
# escodegen bug with a different trigger. Step 5 runs the project's real
|
|
430
|
+
# test command next, which is the actual question anyway.
|
|
431
|
+
warning " [node-modules-health] could not determine health: $codeline_root declares no test script — deferring to the real test command"
|
|
432
|
+
return 0
|
|
433
|
+
fi
|
|
434
|
+
|
|
435
|
+
local _runner_bin="$codeline_root/node_modules/.bin/$_runner"
|
|
436
|
+
if [ -x "$_runner_bin" ]; then
|
|
437
|
+
if "$node_bin" "$_runner_bin" --version >/dev/null 2>&1; then
|
|
438
|
+
return 0
|
|
439
|
+
fi
|
|
440
|
+
warning " [node-modules-health] declared runner '$_runner' exists but failed --version — dependencies present but corrupted, attempting repair..."
|
|
441
|
+
else
|
|
442
|
+
warning " [node-modules-health] declared runner '$_runner' not found in $codeline_root/node_modules/.bin — attempting install..."
|
|
443
|
+
fi
|
|
444
|
+
|
|
445
|
+
detect_and_install_dependencies "$codeline_root" "$node_bin"
|
|
446
|
+
}
|
|
447
|
+
|
|
448
|
+
# ── Codeline bridge agent ─────────────────────────────────────────────────────
|
|
449
|
+
# Runs after a codeline completes. Extracts its exported API surface and writes
|
|
450
|
+
# a cross-codeline contract to logs/cross-codeline-<cl>.md.
|
|
451
|
+
# Exports CROSS_CODELINE_CONTRACT_<CL_UPPER> so the next codeline's re-exec
|
|
452
|
+
# can inject the contract into its story agents' context.
|
|
453
|
+
#
|
|
454
|
+
# Retry policy: up to BRIDGE_MAX_RETRIES (default 2) re-attempts on failure or
|
|
455
|
+
# missing/empty output, with a corrective re-prompt on each retry explaining
|
|
456
|
+
# exactly what went wrong. After all retries:
|
|
457
|
+
# CODELINE_BRIDGE_BLOCK_ON_FAILURE=true → exit 1 (hard abort)
|
|
458
|
+
# CODELINE_BRIDGE_BLOCK_ON_FAILURE=false → warn and continue (fe proceeds
|
|
459
|
+
# without the contract — degraded but not pipeline-corrupting)
|
|
460
|
+
#
|
|
461
|
+
# $1 — completed codeline name (e.g. 'be')
|
|
462
|
+
# $2 — completed codeline worktree path
|
|
463
|
+
# $3 — filtered PRD path for the completed codeline
|
|
464
|
+
_run_codeline_bridge() {
|
|
465
|
+
local _bcl="$1" _bwt="$2" _bprd="$3"
|
|
466
|
+
local _bridge_out="${LOG_DIR}/cross-codeline-${_bcl}.md"
|
|
467
|
+
local _profiles_file="${EPAM_AGENTS_DIR:-$AUTOMATION_DIR/agents}/profiles.json"
|
|
468
|
+
local _bridge_max_retries="${BRIDGE_MAX_RETRIES:-2}"
|
|
469
|
+
|
|
470
|
+
log "[bridge] Extracting cross-codeline contract for '${_bcl}' → ${_bridge_out}"
|
|
471
|
+
|
|
472
|
+
# The bridge agent's identity is its SI (seam codeline-bridge-agent), resolved by the hub — never a
|
|
473
|
+
# persona pasted into the prompt (operator, 2026-09-30: one path).
|
|
474
|
+
|
|
475
|
+
_cp_vals=$(mktemp "${TMPDIR:-/tmp}/codeline-bridge-vals-XXXXXX.json")
|
|
476
|
+
jq_vals \
|
|
477
|
+
--arg bridge_out "${_bridge_out}" \
|
|
478
|
+
--arg bprd "${_bprd}" \
|
|
479
|
+
--arg bcl "${_bcl}" \
|
|
480
|
+
--arg bwt "${_bwt}" \
|
|
481
|
+
'{"__BRIDGE_OUT__":$bridge_out,"__BPRD__":$bprd,"__BCL__":$bcl,"__BWT__":$bwt}' > "$_cp_vals"
|
|
482
|
+
local _base_prompt
|
|
483
|
+
_base_prompt="$(render_engine_prompt codeline-bridge "$_cp_vals")"
|
|
484
|
+
rm -f "$_cp_vals"
|
|
485
|
+
|
|
486
|
+
local _bridge_attempt=0 _bridge_ok=0 _corrective_note=""
|
|
487
|
+
while [ "$_bridge_attempt" -le "$_bridge_max_retries" ]; do
|
|
488
|
+
# Prepend corrective note on retries so the agent knows exactly what failed
|
|
489
|
+
local _bridge_prompt="${_corrective_note:+CORRECTION REQUIRED — YOUR PREVIOUS ATTEMPT FAILED: ${_corrective_note}
|
|
490
|
+
|
|
491
|
+
}${_base_prompt}"
|
|
492
|
+
|
|
493
|
+
local _bridge_rc=0
|
|
494
|
+
rm -f "$_bridge_out"
|
|
495
|
+
# Tools granted below with no restricted allowlist (unlike the read-only
|
|
496
|
+
# QA gates' ORCH_GATE_ALLOWED_TOOLS): this agent's whole job is
|
|
497
|
+
# to read real source files in BRIDGE_SRC_DIR and WriteFile the extracted
|
|
498
|
+
# contract to BRIDGE_OUT_FILE — the read-only allowlist would let it read
|
|
499
|
+
# but never persist its own output. Found live (2026-07-31 agent audit):
|
|
500
|
+
# this call went through plain run_orch_prompt with no tool grant at all,
|
|
501
|
+
# --no-tools — the same class of gap already fixed once for
|
|
502
|
+
# code-graph-detective/failure-analyst. Budgeted like every other
|
|
503
|
+
# tool-bearing gate call in this file.
|
|
504
|
+
local _bridge_reply; _bridge_reply="$(mktemp "${TMPDIR:-/tmp}/codeline-bridge-reply-XXXXXX")"
|
|
505
|
+
AI_GATE_ALLOW_TOOLS=1 EPAM_MAX_TOOL_CALLS="${CODELINE_BRIDGE_MAX_TOOL_CALLS:-10}" \
|
|
506
|
+
run_orch_prompt "$_bridge_prompt" "codeline-bridge-agent" "$_bcl" > "$_bridge_reply" || _bridge_rc=$?
|
|
507
|
+
# THE ENGINE PERFORMS THE WRITE. The contract is the agent's ANSWER; the file lies outside the
|
|
508
|
+
# folder the agent may write in, so on copilot every lane's three attempts returned the full
|
|
509
|
+
# contract as text and "failed" (AMSD-2974, 2026-10-01). A file the agent wrote itself still wins.
|
|
510
|
+
if [ "$_bridge_rc" = "0" ] && [ ! -s "$_bridge_out" ] && [ -s "$_bridge_reply" ]; then
|
|
511
|
+
cp "$_bridge_reply" "$_bridge_out"
|
|
512
|
+
fi
|
|
513
|
+
rm -f "$_bridge_reply"
|
|
514
|
+
|
|
515
|
+
# Determine success: exit 0 AND file exists AND file is non-empty
|
|
516
|
+
if [ "$_bridge_rc" = "0" ] && [ -f "$_bridge_out" ] && [ -s "$_bridge_out" ]; then
|
|
517
|
+
_bridge_ok=1
|
|
518
|
+
break
|
|
519
|
+
fi
|
|
520
|
+
|
|
521
|
+
# Diagnose the specific failure for the corrective re-prompt
|
|
522
|
+
if [ "$_bridge_rc" != "0" ]; then
|
|
523
|
+
_corrective_note="run_orch_prompt exited with code ${_bridge_rc}. You must call WriteFile to write the contract to ${_bridge_out} before finishing."
|
|
524
|
+
elif [ ! -f "$_bridge_out" ]; then
|
|
525
|
+
_corrective_note="The file ${_bridge_out} was NOT written. You MUST call WriteFile with path=${_bridge_out} to produce the contract — do not finish without writing it."
|
|
526
|
+
else
|
|
527
|
+
_corrective_note="The file ${_bridge_out} was written but is EMPTY. The source directory ${_bwt} contains TypeScript files — read them and extract their exports before writing the contract."
|
|
528
|
+
fi
|
|
529
|
+
|
|
530
|
+
_bridge_attempt=$(( _bridge_attempt + 1 ))
|
|
531
|
+
if [ "$_bridge_attempt" -le "$_bridge_max_retries" ]; then
|
|
532
|
+
warning "[bridge] Attempt ${_bridge_attempt}/${_bridge_max_retries} failed for '${_bcl}' — retrying with corrective prompt"
|
|
533
|
+
fi
|
|
534
|
+
done
|
|
535
|
+
|
|
536
|
+
if [ "$_bridge_ok" = "1" ]; then
|
|
537
|
+
local _cl_upper="${_bcl^^}"
|
|
538
|
+
export "CROSS_CODELINE_CONTRACT_${_cl_upper}=${_bridge_out}"
|
|
539
|
+
export CROSS_CODELINE_CONTRACT="$_bridge_out"
|
|
540
|
+
log "[bridge] ✓ Contract written: ${_bridge_out} (exported CROSS_CODELINE_CONTRACT_${_cl_upper})"
|
|
541
|
+
else
|
|
542
|
+
local _total_attempts=$(( _bridge_max_retries + 1 ))
|
|
543
|
+
if [ "${CODELINE_BRIDGE_BLOCK_ON_FAILURE:-false}" = "true" ]; then
|
|
544
|
+
error "[bridge] FATAL — contract extraction for '${_bcl}' failed after ${_total_attempts} attempt(s) and CODELINE_BRIDGE_BLOCK_ON_FAILURE=true. Aborting pipeline."
|
|
545
|
+
return 1
|
|
546
|
+
else
|
|
547
|
+
warning "[bridge] Contract extraction for '${_bcl}' failed after ${_total_attempts} attempt(s) — downstream codelines proceed without cross-codeline contract (set CODELINE_BRIDGE_BLOCK_ON_FAILURE=true to abort instead)"
|
|
548
|
+
fi
|
|
549
|
+
fi
|
|
550
|
+
}
|
|
551
|
+
|
|
552
|
+
# Stop a lane and everything it spawned.
|
|
553
|
+
#
|
|
554
|
+
# A lane subshell is not the thing doing the work: it backgrounds `bash "$0"`,
|
|
555
|
+
# which runs the node LLM calls. `kill <lane pid>` orphans those rather than
|
|
556
|
+
# stopping them, so the halt reported an abort while the lane kept running and
|
|
557
|
+
# kept billing for ten more minutes (live AMSD-2041 2026-07-30). It only died
|
|
558
|
+
# by tripping over a PRD the parent's cleanup had already removed.
|
|
559
|
+
#
|
|
560
|
+
# Walks children before parents so nothing is reparented to init and left
|
|
561
|
+
# running. Deliberately NOT a process-group kill: lanes share this script's
|
|
562
|
+
# group, so `kill -- -$pgid` would take the orchestrator down with them.
|
|
563
|
+
_kill_lane_tree() {
|
|
564
|
+
local _target="$1" _child
|
|
565
|
+
[ -z "$_target" ] && return 0
|
|
566
|
+
# Never accept our own PID or our parent's — that is self-termination
|
|
567
|
+
# wearing a lane's clothes.
|
|
568
|
+
if [ "$_target" = "$$" ] || [ "$_target" = "${BASHPID:-}" ] || [ "$_target" = "$PPID" ]; then
|
|
569
|
+
return 0
|
|
570
|
+
fi
|
|
571
|
+
for _child in $(pgrep -P "$_target" 2>/dev/null); do
|
|
572
|
+
_kill_lane_tree "$_child"
|
|
573
|
+
done
|
|
574
|
+
kill -TERM "$_target" 2>/dev/null || true
|
|
575
|
+
return 0
|
|
576
|
+
}
|
|
577
|
+
|
|
578
|
+
# ── Run-scoped working directory ────────────────────────────────────────────
|
|
579
|
+
# Lane working files (per-codeline PRDs, cross-lane state) used to be written to a FLAT
|
|
580
|
+
# machine-global namespace: /tmp/orch-<codeline>-prd-<pid>.json. Every project and every
|
|
581
|
+
# concurrent run shared it, and archive-run-artifacts.sh then picked "the newest matching
|
|
582
|
+
# file" — so a clean mock1 run archived metrolinx's PRD, describing the wrong project, the
|
|
583
|
+
# wrong story and the wrong day (live 2026-08-05).
|
|
584
|
+
#
|
|
585
|
+
# Scoped to THIS run instead. Falls back to a private mktemp -d rather than the shared
|
|
586
|
+
# namespace when no run directory can be derived: a temp dir nobody else can glob is still
|
|
587
|
+
# isolated, which the old path never was.
|
|
588
|
+
_run_work_dir() {
|
|
589
|
+
local _base="${EPAM_PROJECT_CONFIG_DIR:-}"
|
|
590
|
+
if [ -n "$_base" ] && [ -n "${ORCH_RUN_ID:-}" ]; then
|
|
591
|
+
local _d="$_base/runs/$ORCH_RUN_ID/work"
|
|
592
|
+
mkdir -p "$_d" 2>/dev/null && { printf '%s' "$_d"; return 0; }
|
|
593
|
+
fi
|
|
594
|
+
mktemp -d "${TMPDIR:-/tmp}/orch-run-XXXXXX"
|
|
595
|
+
}
|
|
596
|
+
|
|
597
|
+
# archive_prior_lane_state — ONCE PER RUN, before any lane works: every file in the lane log folders (logs/lanes/<lane>,
|
|
598
|
+
# lines/<line>) older than this run's own start moves to logs/archive/lanes-before-<run>/ — never deleted. The folders
|
|
599
|
+
# survive every launch (the fresh-launch reset clears only logs/), so a run inherited another run's retry counters,
|
|
600
|
+
# ladder positions, rejection records, self-heal ledger and review/gate history (AMSD-2974 run 2 at pause 2,
|
|
601
|
+
# 2026-10-09: upexpress at attempt 10 of 12, 17 old self-heal events). What this run wrote is newer and stays.
|
|
602
|
+
archive_prior_lane_state() {
|
|
603
|
+
local _lanes="${LOG_DIR:-}/lanes"
|
|
604
|
+
[ -d "$_lanes" ] || return 0
|
|
605
|
+
local _run="${EPAM_RESUME_RUN:-${ORCH_RUN_ID:-}}"
|
|
606
|
+
[ -n "$_run" ] || return 0
|
|
607
|
+
local _marker=""
|
|
608
|
+
if [ -n "${EPAM_PROJECT_CONFIG_DIR:-}" ]; then
|
|
609
|
+
_marker="${EPAM_PROJECT_CONFIG_DIR}/runs/${_run}/lane-state-archived"
|
|
610
|
+
[ -f "$_marker" ] && return 0
|
|
611
|
+
fi
|
|
612
|
+
# The run's start is in its id (YYYYMMDDTHHMMSSZ, UTC).
|
|
613
|
+
local _start
|
|
614
|
+
_start=$(printf '%s' "$_run" | sed -nE 's/^([0-9]{4})([0-9]{2})([0-9]{2})T([0-9]{2})([0-9]{2})([0-9]{2})Z?$/\1-\2-\3 \4:\5:\6 UTC/p')
|
|
615
|
+
[ -n "$_start" ] || return 0
|
|
616
|
+
local _ref; _ref=$(mktemp "${TMPDIR:-/tmp}/run-start-XXXXXX") || return 0
|
|
617
|
+
touch -d "$_start" "$_ref" 2>/dev/null || { rm -f "$_ref"; return 0; }
|
|
618
|
+
local _dest="${LOG_DIR}/archive/lanes-before-${_run}" _n=0 _f _rel
|
|
619
|
+
while IFS= read -r -d '' _f; do
|
|
620
|
+
_rel="${_f#"$_lanes"/}"
|
|
621
|
+
mkdir -p "$_dest/$(dirname "$_rel")" 2>/dev/null && mv "$_f" "$_dest/$_rel" 2>/dev/null && _n=$((_n + 1))
|
|
622
|
+
done < <(find "$_lanes" -type f ! -newer "$_ref" -print0 2>/dev/null)
|
|
623
|
+
rm -f "$_ref"
|
|
624
|
+
[ "$_n" -gt 0 ] && log "[orch] archived ${_n} file(s) of earlier runs' lane state → ${_dest} (this run starts with none of it)"
|
|
625
|
+
[ -n "$_marker" ] && { mkdir -p "${_marker%/*}" 2>/dev/null; date -u +%Y-%m-%dT%H:%M:%SZ > "$_marker"; }
|
|
626
|
+
return 0
|
|
627
|
+
}
|
|
628
|
+
|
|
629
|
+
# base_fresh_story_branches <prd> — ONCE PER RUN, before anything reads them, every codeline a brownfield story may be
|
|
630
|
+
# worked in (its own and every one discovery grounded, lib/plan-scope.js) is put on the story's branch, freshly based on
|
|
631
|
+
# the codeline's baseline. ensure_story_branch pins any commit the reset would orphan under epam-rescue/*. Once per RUN,
|
|
632
|
+
# not per launch: with the operator's pauses the lanes are first reached by a resumed leg, so "not a resume" never held;
|
|
633
|
+
# the run's own marker (runs/<id>/story-branches-based) keeps every later leg — a resume after the writer included —
|
|
634
|
+
# off the branches, where this run's own work is.
|
|
635
|
+
base_fresh_story_branches() {
|
|
636
|
+
local _prd="${1:-}"
|
|
637
|
+
[ "${EPAM_BROWNFIELD:-0}" = "1" ] && [ -f "$_prd" ] || return 0
|
|
638
|
+
command -v ensure_story_branch >/dev/null 2>&1 || return 0
|
|
639
|
+
local _run="${EPAM_RESUME_RUN:-${ORCH_RUN_ID:-}}"
|
|
640
|
+
local _marker=""
|
|
641
|
+
if [ -n "${EPAM_PROJECT_CONFIG_DIR:-}" ] && [ -n "$_run" ]; then
|
|
642
|
+
_marker="${EPAM_PROJECT_CONFIG_DIR}/runs/${_run}/story-branches-based"
|
|
643
|
+
[ -f "$_marker" ] && return 0
|
|
644
|
+
fi
|
|
645
|
+
local _line _root _story
|
|
646
|
+
while IFS=$'\t' read -r _root _story; do
|
|
647
|
+
[ -n "$_root" ] && [ -n "$_story" ] || continue
|
|
648
|
+
log "[orch] fresh run: basing '${_root##*/}' on its story branch for ${_story} before anything reads it"
|
|
649
|
+
ensure_story_branch "$_root" "$_story" "${JIRA_BASELINE_BRANCH:-}" || true
|
|
650
|
+
done < <("$NODE_BIN" -e '
|
|
651
|
+
const fs=require("fs"),path=require("path");const [prdF,cfg,lib]=process.argv.slice(1);
|
|
652
|
+
const prd=JSON.parse(fs.readFileSync(prdF,"utf8"));let facts=null;
|
|
653
|
+
try{facts=cfg?JSON.parse(fs.readFileSync(path.join(cfg,"codeline-facts.json"),"utf8")):null}catch{}
|
|
654
|
+
const {planScope}=require(lib);const seen=new Set();
|
|
655
|
+
for(const s of prd.stories||[]){if(!s||!s.id)continue;for(const d of planScope(s,prd,facts).outputDirs){const k=d.path+"\t"+s.id;if(!seen.has(k)){seen.add(k);console.log(k)}}}
|
|
656
|
+
' "$_prd" "${EPAM_PROJECT_CONFIG_DIR:-}" "$SCRIPT_DIR/lib/plan-scope.js" 2>/dev/null)
|
|
657
|
+
[ -n "$_marker" ] && { mkdir -p "${_marker%/*}" 2>/dev/null; date -u +%Y-%m-%dT%H:%M:%SZ > "$_marker"; }
|
|
658
|
+
return 0
|
|
659
|
+
}
|
|
660
|
+
|
|
661
|
+
# lane_paused_since <lane_log_dir> <marker> — did this lane (or a release line of it) record a pause after <marker>
|
|
662
|
+
# (touched when this run's lanes started)? A lane paused before its writer is not a lane that finished: it recorded
|
|
663
|
+
# paused-run.json in its OWN log dir, which no launcher reads. A record older than the marker is an earlier launch's.
|
|
664
|
+
lane_paused_since() {
|
|
665
|
+
local _dir="${1:-}" _marker="${2:-}"
|
|
666
|
+
[ -d "$_dir" ] && [ -f "$_marker" ] || return 1
|
|
667
|
+
[ -n "$(find "$_dir" -name paused-run.json -newer "$_marker" 2>/dev/null | head -1)" ]
|
|
668
|
+
}
|
|
669
|
+
|
|
670
|
+
# record_lanes_pause <marker> — when any lane paused during this run, the RUN is paused: recorded where every launcher
|
|
671
|
+
# reads it (lib/checkpoints.sh record_run_pause, in LOG_DIR), with how to resume. Returns 1 when no lane paused.
|
|
672
|
+
# Found rehearsing AMSD-2974, 2026-10-09: every lane paused before its writer and the run reported "Pipeline complete".
|
|
673
|
+
record_lanes_pause() {
|
|
674
|
+
local _marker="${1:-}"
|
|
675
|
+
[ -f "$_marker" ] && [ -d "${LOG_DIR:-}/lanes" ] || return 1
|
|
676
|
+
local _recs; _recs=$(find "${LOG_DIR}/lanes" -name paused-run.json -newer "$_marker" 2>/dev/null)
|
|
677
|
+
[ -n "$_recs" ] || return 1
|
|
678
|
+
local _n _stage
|
|
679
|
+
_n=$(printf '%s\n' "$_recs" | awk -F/lanes/ '{split($2,a,"/"); print a[1]}' | sort -u | grep -c .)
|
|
680
|
+
_stage=$(jq -r '.stage // ""' "$(printf '%s\n' "$_recs" | head -1)" 2>/dev/null)
|
|
681
|
+
record_run_pause "${_stage:-pre-writer}"
|
|
682
|
+
log "[orch] ⏸ Run paused — ${_n} lane(s) paused at ${_stage:-a pause}. Resume with: EPAM_RESUME_RUN=${ORCH_RUN_ID:-} <your launcher>"
|
|
683
|
+
return 0
|
|
684
|
+
}
|
|
685
|
+
|
|
686
|
+
# _lane_phase_args — the argument a lane's FIRST phase call passes run-agent-orchestration.sh.
|
|
687
|
+
# A RESUMED LANE DOES NOT RESET ITS STORIES. --reset sets every completed story of the phase back to pending
|
|
688
|
+
# (run-agent-orchestration.sh:964), and the orchestrator's own rule is that "a resume runs without --reset"
|
|
689
|
+
# (:1026). The lane passed --reset unconditionally, so the story story_resume_completed_by_commit had just
|
|
690
|
+
# marked completed was un-marked 33 s later and a paid writer re-wrote approved work on every resume
|
|
691
|
+
# (AMSD-2974 run 9, resumes 9-12). A lane resumed from its checkpoint (_lane_spec_mode=0 under EPAM_RESUME_RUN)
|
|
692
|
+
# passes nothing; every other lane, and every retry after gate remediation, still resets.
|
|
693
|
+
_lane_phase_args() {
|
|
694
|
+
if [ -n "${EPAM_RESUME_RUN:-}" ] && [ "${_lane_spec_mode:-}" = "0" ]; then
|
|
695
|
+
printf ''
|
|
696
|
+
else
|
|
697
|
+
printf '%s' '--reset'
|
|
698
|
+
fi
|
|
699
|
+
}
|
|
700
|
+
|
|
701
|
+
_run_codeline_loop() {
|
|
702
|
+
local _prd_path="$1"
|
|
703
|
+
local _log_file="${2:-/tmp/orch-$(date +%Y%m%dT%H%M%S).log}"
|
|
704
|
+
# When the tier3 launcher (or any external phase-managing caller) drives the
|
|
705
|
+
# phase loop itself (calling us once per phase via --phase X), set this to
|
|
706
|
+
# run only that phase across all codelines. Empty = run every PRD phase.
|
|
707
|
+
local _phase_filter="${3:-}"
|
|
708
|
+
# When this run's lanes started: a lane pause recorded after it is this run's (lane_paused_since, record_lanes_pause).
|
|
709
|
+
local _lanes_started="${LOG_DIR:-/tmp}/.lanes-started"
|
|
710
|
+
mkdir -p "${LOG_DIR:-/tmp}" 2>/dev/null; : > "$_lanes_started"
|
|
711
|
+
|
|
712
|
+
# ── THE PLAN COMES FIRST, AND IT DECIDES THE LANES ─────────────────────────
|
|
713
|
+
# A fresh run's codelines are put on their story branches, freshly based, BEFORE anything reads them: the detective
|
|
714
|
+
# and the spec pass read the checkout, and a fresh AMSD-2974 run read earlier runs' commits as existing code
|
|
715
|
+
# (2026-10-09). A codeline a killed line pass left on the line's branch is put back. Then the story without a plan
|
|
716
|
+
# gets one (lib/handlers/write-missing-plans.js), shown every codeline discovery grounded; the codelines the plan
|
|
717
|
+
# names join the story and the run's outputDirs (lib/plan-scope.js) — so the lane list below is the plan's.
|
|
718
|
+
archive_prior_lane_state
|
|
719
|
+
base_fresh_story_branches "$_prd_path"
|
|
720
|
+
_return_codelines_from_release_lines "$_prd_path"
|
|
721
|
+
"$NODE_BIN" "$SCRIPT_DIR/lib/handlers/write-missing-plans.js" "$_prd_path" || true
|
|
722
|
+
|
|
723
|
+
# Extract codeline:path entries from project.outputDirs.
|
|
724
|
+
# Falls back to project.outputDir + JIRA_DEFAULT_CODELINE for single-codeline PRDs.
|
|
725
|
+
local _cl_entries=()
|
|
726
|
+
mapfile -t _cl_entries < <("$NODE_BIN" "$SCRIPT_DIR/lib/handlers/cl-entries.js" "${_prd_path}" 2>/dev/null)
|
|
727
|
+
|
|
728
|
+
# CODELINE SELECTION — run a subset of the story's codelines, one launch at a time.
|
|
729
|
+
#
|
|
730
|
+
# A story spanning several repositories otherwise runs all of them in one launch. Naming a
|
|
731
|
+
# subset makes each launch a natural pause: a failure blasts one lane instead of every lane,
|
|
732
|
+
# each pause is an inspection point, and spend is bounded per launch instead of per story.
|
|
733
|
+
#
|
|
734
|
+
# Filtered HERE because this array is the single source every downstream path reads — the
|
|
735
|
+
# parallel fan-out, the sequential loop, and the post-run merges. Filtering at any one of those
|
|
736
|
+
# would leave the others running codelines nobody asked for.
|
|
737
|
+
#
|
|
738
|
+
# UNSET OR EMPTY MEANS ALL, so a run that does not use this is byte-for-byte today's run.
|
|
739
|
+
# An unmatched selection yields NOTHING rather than everything: failing open here would run
|
|
740
|
+
# every codeline when the operator asked for one, which is the expensive direction of a typo,
|
|
741
|
+
# and the caller below already treats an empty list as a hard error rather than a silent pass.
|
|
742
|
+
#
|
|
743
|
+
# Matching is on the codeline NAME (the part before the colon), never the path, so a selection
|
|
744
|
+
# cannot accidentally match a directory that merely contains the same text.
|
|
745
|
+
# NO OPERATOR FILTER HERE. The lane list already IS the run's scope: it is built from the
|
|
746
|
+
# PRD's project.outputDirs, which resolve-codeline-scope.sh writes for every project however
|
|
747
|
+
# its PRD arrived. Filtering it again with a hand-typed EPAM_ONLY_CODELINES asked a human to
|
|
748
|
+
# restate a fact the PRD already carried, and could only ever disagree with it.
|
|
749
|
+
|
|
750
|
+
if [ ${#_cl_entries[@]} -eq 0 ]; then
|
|
751
|
+
error "[orch] No codeline/worktree entries found in PRD: ${_prd_path}"
|
|
752
|
+
error "[orch] Add project.outputDirs to the PRD or set JIRA_DEFAULT_CODELINE + project.outputDir"
|
|
753
|
+
return 1
|
|
754
|
+
fi
|
|
755
|
+
|
|
756
|
+
# ── Codeline health: assess every lane BEFORE spending anything ────────────
|
|
757
|
+
# Live AMSD-2041, 2026-07-28: all three discovered codelines declared a test
|
|
758
|
+
# script and a runner, and none could resolve one — two had no node_modules at
|
|
759
|
+
# all. Until that morning Step 5 skipped silently on exactly this, so an
|
|
760
|
+
# unverified baseline was accepted once per lane. Making it fail was right, but
|
|
761
|
+
# it fails INSIDE the phase, after the spec pass is already paid for: the next
|
|
762
|
+
# launch would have cost a full spec pass per lane to discover a dependency
|
|
763
|
+
# problem visible in seconds.
|
|
764
|
+
#
|
|
765
|
+
# Runs on whatever DISCOVERY returned. The codelines are resolved per ticket at
|
|
766
|
+
# runtime, so preparing a fixed list would hardcode discovery's output.
|
|
767
|
+
# lib/codeline-health.sh knows no package manager, runner or language — it
|
|
768
|
+
# reads what each codeline declares and prepares it accordingly.
|
|
769
|
+
if [ -f "$SCRIPT_DIR/lib/codeline-health.sh" ]; then
|
|
770
|
+
local _ch_paths=()
|
|
771
|
+
for _entry in "${_cl_entries[@]}"; do _ch_paths+=("${_entry#*:}"); done
|
|
772
|
+
log "[orch] Assessing health of ${#_ch_paths[@]} codeline(s) before starting work..."
|
|
773
|
+
if ! NODE_BIN="$NODE_BIN" bash "$SCRIPT_DIR/lib/codeline-health.sh" "${_ch_paths[@]}"; then
|
|
774
|
+
error "[orch] One or more codelines are UNHEALTHY — aborting before any spend."
|
|
775
|
+
error "[orch] A codeline that cannot resolve its own declared tooling cannot run its gates,"
|
|
776
|
+
error "[orch] so the run would accept an unverified baseline for that lane."
|
|
777
|
+
error "[orch] Set SKIP_CODELINE_HEALTH=1 to proceed knowing the gates cannot run."
|
|
778
|
+
return 1
|
|
779
|
+
fi
|
|
780
|
+
fi
|
|
781
|
+
|
|
782
|
+
# Tear down and re-scaffold every codeline worktree so each run starts clean.
|
|
783
|
+
# Pre-existing worktrees from prior runs poison the next run (stale package.json,
|
|
784
|
+
# accumulated artifacts, mid-run file mutations). Full deletion is the only safe state.
|
|
785
|
+
for _entry in "${_cl_entries[@]}"; do
|
|
786
|
+
local _cl="${_entry%%:*}" _wt="${_entry#*:}"
|
|
787
|
+
log "[orch] Codeline '${_cl}' → ${_wt}"
|
|
788
|
+
|
|
789
|
+
if [ "${EPAM_BROWNFIELD:-0}" != "1" ]; then
|
|
790
|
+
# ── Greenfield: full teardown + git init so every run starts clean ──
|
|
791
|
+
# Stale package.json, accumulated artifacts, and mid-run file mutations from
|
|
792
|
+
# prior runs all poison the next run. Full deletion is the only safe state.
|
|
793
|
+
# Some node_modules subdirs end up 0444/0555 after npm install — force-write
|
|
794
|
+
# access first so rm -rf is never blocked by restrictive permissions.
|
|
795
|
+
if [ -d "$_wt" ]; then
|
|
796
|
+
log "[orch] Tearing down '${_cl}' worktree at ${_wt}..."
|
|
797
|
+
chmod -R u+w "$_wt" 2>/dev/null || true
|
|
798
|
+
rm -rf "$_wt"
|
|
799
|
+
fi
|
|
800
|
+
|
|
801
|
+
local _scaffold="$SCRIPT_DIR/scaffold-${_cl}-repo.sh"
|
|
802
|
+
if [ -f "$_scaffold" ]; then
|
|
803
|
+
log "[orch] Scaffolding '${_cl}' repo at ${_wt}..."
|
|
804
|
+
bash "$_scaffold" "$_wt" 2>&1 | tee -a "$_log_file"
|
|
805
|
+
else
|
|
806
|
+
# No scaffold script — create a bare git repo; agents fill in the stack.
|
|
807
|
+
log "[orch] No scaffold-${_cl}-repo.sh found — creating bare git repo at ${_wt}..."
|
|
808
|
+
mkdir -p "$_wt"
|
|
809
|
+
git -C "$_wt" init --quiet
|
|
810
|
+
git -C "$_wt" config user.email "epam-cli@local"
|
|
811
|
+
git -C "$_wt" config user.name "epam-cli"
|
|
812
|
+
git -C "$_wt" commit --allow-empty -m "init: ${_cl} codeline worktree" --quiet
|
|
813
|
+
fi
|
|
814
|
+
else
|
|
815
|
+
# ── Brownfield: verify the existing repo; no teardown ──────────────────
|
|
816
|
+
# The worktree must already exist as a git repository — agents modify
|
|
817
|
+
# existing files in-place rather than building from scratch.
|
|
818
|
+
if [ ! -d "$_wt" ]; then
|
|
819
|
+
error "[orch] Brownfield worktree does not exist: ${_wt}"
|
|
820
|
+
error "[orch] Codeline '${_cl}' must point to an existing local repository."
|
|
821
|
+
return 1
|
|
822
|
+
fi
|
|
823
|
+
if [ ! -e "$_wt/.git" ]; then
|
|
824
|
+
error "[orch] Brownfield worktree is not a git repository: ${_wt}"
|
|
825
|
+
error "[orch] Ensure the path contains a .git directory."
|
|
826
|
+
return 1
|
|
827
|
+
fi
|
|
828
|
+
log "[orch] Brownfield: using existing worktree '${_cl}' at ${_wt}"
|
|
829
|
+
|
|
830
|
+
# Capture baseline SHA so gate diff oracles (review-ranger, mutant-hunter,
|
|
831
|
+
# fuzz-weaver, sast-sentinel) diff only the story's changes against the
|
|
832
|
+
# project's main branch — not the full commit history.
|
|
833
|
+
local _baseline_branch="${JIRA_BASELINE_BRANCH:-main}"
|
|
834
|
+
local _baseline_sha=""
|
|
835
|
+
# --verify --quiet: fail cleanly for a ref that does not exist. Without it,
|
|
836
|
+
# `git rev-parse origin/develop` ECHOES the literal "origin/develop" to stdout and exits
|
|
837
|
+
# 128, so the || chain runs the next command and the substitution captures BOTH — a
|
|
838
|
+
# two-line value whose first line is not a SHA. Every gate diff oracle reads this file
|
|
839
|
+
# (review-ranger, mutant-hunter, fuzz-weaver, sast-sentinel), which is the known
|
|
840
|
+
# "reviewers saw 0 files" failure. brownfield-preflight-reset.sh already guards this
|
|
841
|
+
# exact shape; this call site did not. A repo with no remote hits it on every lane.
|
|
842
|
+
_baseline_sha=$(git -C "$_wt" rev-parse --verify --quiet "origin/${_baseline_branch}" 2>/dev/null || \
|
|
843
|
+
git -C "$_wt" rev-parse --verify --quiet "${_baseline_branch}" 2>/dev/null || \
|
|
844
|
+
git -C "$_wt" rev-parse --verify --quiet HEAD 2>/dev/null || echo "")
|
|
845
|
+
if [ -n "$_baseline_sha" ]; then
|
|
846
|
+
# NOT WRITTEN HERE. See above: nothing reads this path, and the value means something
|
|
847
|
+
# different from the one Step 8 writes into the lane's own directory.
|
|
848
|
+
log "[orch] Brownfield baseline branch: ${_baseline_branch} @ ${_baseline_sha:0:8}"
|
|
849
|
+
else
|
|
850
|
+
warning "[orch] Could not resolve baseline branch '${_baseline_branch}' — gate diffs will use HEAD"
|
|
851
|
+
fi
|
|
852
|
+
fi
|
|
853
|
+
# THE CODELINE'S DECLARATION IS DETECTED, NEVER AUTHORED HERE.
|
|
854
|
+
#
|
|
855
|
+
# This wrote three .epam/ manifests from heredocs -- 80 lines asserting, of a repository the
|
|
856
|
+
# engine had never inspected, that its manifest is package.json, its sources are .ts/.tsx/.js,
|
|
857
|
+
# its vendor directory is node_modules, and that it must carry typescript, @types/node, vitest
|
|
858
|
+
# and tsx. Seventeen scripts read dependency-check.json as the codeline's OWN declaration, so
|
|
859
|
+
# fabricating it defeated every generic component downstream at the source: a non-Node codeline
|
|
860
|
+
# was handed a document saying it is TypeScript, in client-repo space, with facts nobody
|
|
861
|
+
# detected.
|
|
862
|
+
#
|
|
863
|
+
# lib/handlers/codeline-manifests.js assembles them from the provider whose manifest the
|
|
864
|
+
# repository actually carries. A codeline no provider recognises gets NOTHING and is reported
|
|
865
|
+
# -- an undeclared codeline is a state to surface, never one to invent an answer for.
|
|
866
|
+
#
|
|
867
|
+
# A DECLARATION THE CODELINE CONTRADICTS IS RE-DERIVED. A greenfield launch seeds .epam/ from
|
|
868
|
+
# the project directory before the codeline exists, so those files are the project author's
|
|
869
|
+
# word about a stack not yet built. When the codeline then carries a manifest a provider
|
|
870
|
+
# recognises and the seeded declaration names a manifest file that is not there, the codeline
|
|
871
|
+
# is the truth and the seed was a guess: live 2026-09-12, a Python greenfield project carried
|
|
872
|
+
# package.json / npm / vitest manifests copied from another project, and every generic gate
|
|
873
|
+
# would have judged a pytest repository as TypeScript. Derived, and the seeded siblings the
|
|
874
|
+
# provider does not declare are removed with it — they described the same wrong stack.
|
|
875
|
+
local _clm_contradicted=0
|
|
876
|
+
if [ -f "$_wt/.epam/dependency-check.json" ]; then
|
|
877
|
+
local _clm_declared_manifest
|
|
878
|
+
_clm_declared_manifest=$(jq -r '.manifestFile // ""' "$_wt/.epam/dependency-check.json" 2>/dev/null || echo "")
|
|
879
|
+
if [ -n "$_clm_declared_manifest" ] && [ ! -f "$_wt/$_clm_declared_manifest" ] \
|
|
880
|
+
&& "$NODE_BIN" -e 'process.exit(require(process.argv[1]).resolveEcosystem(process.argv[2]) ? 0 : 1)' \
|
|
881
|
+
"$SCRIPT_DIR/lib/handlers/codeline-manifests.js" "$_wt" 2>/dev/null; then
|
|
882
|
+
_clm_contradicted=1
|
|
883
|
+
warning "[orch] ${_wt}: .epam/dependency-check.json declares manifest '${_clm_declared_manifest}', which this codeline does not carry — the declaration is re-derived from the ecosystem the codeline actually has"
|
|
884
|
+
fi
|
|
885
|
+
fi
|
|
886
|
+
if [ ! -f "$_wt/.epam/dependency-check.json" ] || [ "$_clm_contradicted" = "1" ]; then
|
|
887
|
+
local _clm _clm_rc=0
|
|
888
|
+
_clm=$("$NODE_BIN" "$SCRIPT_DIR/lib/handlers/codeline-manifests.js" "$_wt" 2>&1) || _clm_rc=$?
|
|
889
|
+
if [ "$_clm_rc" -ne 0 ]; then
|
|
890
|
+
warning "[orch] ${_wt}: no provider declares how this codeline is checked — .epam/ manifests NOT written"
|
|
891
|
+
printf '%s\n' "$_clm" | sed 's/^/ /' >&2
|
|
892
|
+
else
|
|
893
|
+
mkdir -p "$_wt/.epam"
|
|
894
|
+
if [ "$_clm_contradicted" = "1" ]; then
|
|
895
|
+
local _clm_stale
|
|
896
|
+
for _clm_stale in dependency-check.json contract-generation.json known-fixes.json; do
|
|
897
|
+
if [ -f "$_wt/.epam/$_clm_stale" ] && ! printf '%s' "$_clm" | jq -e --arg k "$_clm_stale" 'has($k)' >/dev/null 2>&1; then
|
|
898
|
+
rm -f "$_wt/.epam/$_clm_stale"
|
|
899
|
+
log "[orch] Removed seeded .epam/${_clm_stale}: it described a stack this codeline is not, and its ecosystem declares no replacement"
|
|
900
|
+
fi
|
|
901
|
+
done
|
|
902
|
+
fi
|
|
903
|
+
printf '%s' "$_clm" | jq -r 'keys[]' | while IFS= read -r _mf; do
|
|
904
|
+
[ -z "$_mf" ] && continue
|
|
905
|
+
printf '%s' "$_clm" | jq --arg k "$_mf" '.[$k]' > "$_wt/.epam/$_mf"
|
|
906
|
+
done
|
|
907
|
+
log "[orch] Wrote .epam/ manifests to ${_wt} from the resolved ecosystem provider"
|
|
908
|
+
fi
|
|
909
|
+
fi
|
|
910
|
+
# A DECLARATION MISSING A FACT ITS ECOSYSTEM DECLARES IS COMPLETED. The plug-in gained
|
|
911
|
+
# provisionCommand and runEnvironment on 2026-09-16; a manifest written before that carries
|
|
912
|
+
# neither, and a worktree provisioned from it has no environment. Only ABSENT keys are added
|
|
913
|
+
# — every key the manifest already holds is the project's word and stays as it is.
|
|
914
|
+
if [ -f "$_wt/.epam/dependency-check.json" ]; then
|
|
915
|
+
local _clm_more _clm_more_rc=0
|
|
916
|
+
_clm_more=$("$NODE_BIN" "$SCRIPT_DIR/lib/handlers/codeline-manifests.js" "$_wt" 2>/dev/null) || _clm_more_rc=$?
|
|
917
|
+
if [ "$_clm_more_rc" -eq 0 ]; then
|
|
918
|
+
local _clm_added
|
|
919
|
+
_clm_added=$(jq -n --argjson have "$(cat "$_wt/.epam/dependency-check.json")" --argjson derive "$(printf '%s' "$_clm_more" | jq '."dependency-check.json" // {}')" \
|
|
920
|
+
'[($derive | keys[]) as $k | select(($have | has($k)) | not) | $k]' 2>/dev/null || echo '[]')
|
|
921
|
+
if [ "$(printf '%s' "$_clm_added" | jq 'length')" -gt 0 ]; then
|
|
922
|
+
jq -s '.[1] * .[0]' "$_wt/.epam/dependency-check.json" <(printf '%s' "$_clm_more" | jq '."dependency-check.json"') > "$_wt/.epam/dependency-check.json.tmp" \
|
|
923
|
+
&& mv "$_wt/.epam/dependency-check.json.tmp" "$_wt/.epam/dependency-check.json"
|
|
924
|
+
log "[orch] Completed .epam/dependency-check.json in ${_wt} with $(printf '%s' "$_clm_added" | jq -r 'join(", ")') from its ecosystem provider"
|
|
925
|
+
fi
|
|
926
|
+
fi
|
|
927
|
+
fi
|
|
928
|
+
|
|
929
|
+
# ── Plugin provisioning: config-driven, zero project-specific hardcoding ──
|
|
930
|
+
# This script has no idea what a "plugin" IS or does, with ONE exception:
|
|
931
|
+
# CodeGraph's query tool (orchestrations/plugins/codegraph-plugin.js) ships
|
|
932
|
+
# with epam-cli itself — the same way ReadFile/Bash are always available
|
|
933
|
+
# regardless of project — so it's provisioned for EVERY codeline
|
|
934
|
+
# unconditionally, merged with whatever the project's own plugins.json
|
|
935
|
+
# adds on top (never overwritten, never required to list it manually).
|
|
936
|
+
# Everything else remains purely config-driven: EPAM_PROJECT_CONFIG_DIR/
|
|
937
|
+
# codeline-facts.json (keyed by codeline name) has its entry for THIS
|
|
938
|
+
# codeline extracted into .epam/codeline-facts.json. Adding, removing, or
|
|
939
|
+
# repointing a PROJECT-specific plugin is purely a config edit in the
|
|
940
|
+
# project's own directory — never a change to this script.
|
|
941
|
+
local _project_tools_json="[]"
|
|
942
|
+
if [ -n "${EPAM_PROJECT_CONFIG_DIR:-}" ] && [ -f "${EPAM_PROJECT_CONFIG_DIR}/plugins.json" ]; then
|
|
943
|
+
_project_tools_json=$(jq -c '.tools // []' "${EPAM_PROJECT_CONFIG_DIR}/plugins.json" 2>/dev/null || echo "[]")
|
|
944
|
+
fi
|
|
945
|
+
local _codegraph_plugin_abs=""
|
|
946
|
+
local _codegraph_plugin_src="${SCRIPT_DIR}/../plugins/codegraph-plugin.js"
|
|
947
|
+
if [ -f "$_codegraph_plugin_src" ]; then
|
|
948
|
+
_codegraph_plugin_abs="$(cd "$(dirname "$_codegraph_plugin_src")" 2>/dev/null && pwd)/$(basename "$_codegraph_plugin_src")"
|
|
949
|
+
fi
|
|
950
|
+
if [ -n "$_codegraph_plugin_abs" ] || [ "$_project_tools_json" != "[]" ]; then
|
|
951
|
+
mkdir -p "$_wt/.epam"
|
|
952
|
+
jq -n --argjson project "$_project_tools_json" --arg cg "$_codegraph_plugin_abs" \
|
|
953
|
+
'{tools: (((if $cg != "" then [$cg] else [] end) + $project) | unique)}' \
|
|
954
|
+
> "$_wt/.epam/settings.json"
|
|
955
|
+
log "[orch] Provisioned .epam/settings.json (plugins, incl. built-in CodeGraph tool) for '${_cl}'"
|
|
956
|
+
fi
|
|
957
|
+
|
|
958
|
+
if [ -n "${EPAM_PROJECT_CONFIG_DIR:-}" ]; then
|
|
959
|
+
local _facts_cfg="${EPAM_PROJECT_CONFIG_DIR}/codeline-facts.json"
|
|
960
|
+
if [ -f "$_facts_cfg" ]; then
|
|
961
|
+
local _cl_facts
|
|
962
|
+
_cl_facts=$(jq -c --arg cl "$_cl" '.[$cl] // empty' "$_facts_cfg" 2>/dev/null)
|
|
963
|
+
if [ -n "$_cl_facts" ]; then
|
|
964
|
+
mkdir -p "$_wt/.epam"
|
|
965
|
+
echo "$_cl_facts" > "$_wt/.epam/codeline-facts.json"
|
|
966
|
+
log "[orch] Provisioned .epam/codeline-facts.json for '${_cl}' from ${_facts_cfg}"
|
|
967
|
+
else
|
|
968
|
+
# SAY SO. This skipped in silence, so a file in the wrong SHAPE was indistinguishable
|
|
969
|
+
# from no file and from an agent that observed nothing — three different problems with
|
|
970
|
+
# one symptom, which is none. The file is keyed by codeline name at the top level;
|
|
971
|
+
# a hand-written one nesting them under another key parses fine and yields nothing for
|
|
972
|
+
# every codeline, which is exactly what happened to mock3's before discovery produced it.
|
|
973
|
+
warning "[orch] '${_facts_cfg}' exists but has no entry for codeline '${_cl}' — agents there will work from the source alone. Codeline names must be TOP-LEVEL keys; the engine reads one with jq '.[\$cl]'."
|
|
974
|
+
fi
|
|
975
|
+
fi
|
|
976
|
+
|
|
977
|
+
# .env.local: derived from the codeline's OWN .env.local.sample, not an
|
|
978
|
+
# engine-side list. See provision_env_local_from_sample().
|
|
979
|
+
provision_env_local_from_sample "$_wt" "$_wt/.env.local"
|
|
980
|
+
if [ -s "$_wt/.env.local" ]; then
|
|
981
|
+
log "[orch] Provisioned .env.local for '${_cl}' from ${_wt}/.env.local.sample"
|
|
982
|
+
fi
|
|
983
|
+
|
|
984
|
+
# anti-patterns.json (optional, per-project): not keyed by codeline —
|
|
985
|
+
# copied whole into .epam/ so the check_anti_patterns plugin tool (and
|
|
986
|
+
# run_anti_pattern_check's deterministic gate) both read the same,
|
|
987
|
+
# already-provisioned file instead of reaching across worktree
|
|
988
|
+
# boundaries into EPAM_PROJECT_CONFIG_DIR directly.
|
|
989
|
+
local _antipatterns_cfg="${EPAM_PROJECT_CONFIG_DIR}/anti-patterns.json"
|
|
990
|
+
if [ -f "$_antipatterns_cfg" ]; then
|
|
991
|
+
mkdir -p "$_wt/.epam"
|
|
992
|
+
cp "$_antipatterns_cfg" "$_wt/.epam/anti-patterns.json"
|
|
993
|
+
log "[orch] Provisioned .epam/anti-patterns.json for '${_cl}' from ${_antipatterns_cfg}"
|
|
994
|
+
fi
|
|
995
|
+
fi
|
|
996
|
+
done
|
|
997
|
+
|
|
998
|
+
|
|
999
|
+
|
|
1000
|
+
# Per-codeline execution loop
|
|
1001
|
+
local _work_dir; _work_dir="$(_run_work_dir)"
|
|
1002
|
+
local _overall=0 _completed_list="" _cross_prd="$_work_dir/cross.json"
|
|
1003
|
+
local _cl_prds=()
|
|
1004
|
+
|
|
1005
|
+
|
|
1006
|
+
# Rebuild cumulative cross-codeline PRD so later codelines can check
|
|
1007
|
+
# completed stories from earlier codelines (is_story_completed fallback).
|
|
1008
|
+
_rebuild_cross() {
|
|
1009
|
+
[ -z "$_completed_list" ] && { unset CROSS_CODELINE_PRD; return; }
|
|
1010
|
+
COMPLETED_LIST="$_completed_list" \
|
|
1011
|
+
"$NODE_BIN" "$SCRIPT_DIR/lib/handlers/rebuild-cross-contract.js" "${_cross_prd}" 2>/dev/null
|
|
1012
|
+
export CROSS_CODELINE_PRD="$_cross_prd"
|
|
1013
|
+
}
|
|
1014
|
+
|
|
1015
|
+
# Build filtered PRD containing only stories for codeline $1, written to $2.
|
|
1016
|
+
# Returns story count on stdout.
|
|
1017
|
+
_filtered_prd() {
|
|
1018
|
+
local _fcl="$1" _fout="$2" _fsrc="$3"
|
|
1019
|
+
JIRA_DEFAULT_CODELINE="${JIRA_DEFAULT_CODELINE:-}" \
|
|
1020
|
+
"$NODE_BIN" "$SCRIPT_DIR/lib/handlers/filtered-prd.js" "${_fsrc}" "${_fout}" "${_fcl}" 2>/dev/null
|
|
1021
|
+
}
|
|
1022
|
+
|
|
1023
|
+
# A LANE RESUMES FROM ITS OWN CHECKPOINT. On a resume, a lane whose own checkpoint was taken
|
|
1024
|
+
# pre-writer has finished its spec pass, and that checkpoint's PRD is the spec the operator
|
|
1025
|
+
# approved at pause 2. Rebuilding the lane's PRD from the canonical one instead — which a resume
|
|
1026
|
+
# restores from the run's post-roster checkpoint, taken before any spec — handed every lane a
|
|
1027
|
+
# PRD with no spec, so every lane ran its spec pass again, paid, and overwrote what had been
|
|
1028
|
+
# approved (AMSD-2974 writer relaunch, 2026-10-02). An edit the operator made at the pause to the
|
|
1029
|
+
# lane's working PRD is kept. A resume sent back to an earlier stage (EPAM_RESUME_FROM_STAGE)
|
|
1030
|
+
# redoes the work after it, so it never takes this path.
|
|
1031
|
+
# Sets _lane_spec_mode=0 and returns 0 when the lane resumes from its checkpoint.
|
|
1032
|
+
_lane_resume_prd() {
|
|
1033
|
+
local _rcl="$1" _rout="$2"
|
|
1034
|
+
_lane_spec_mode=""
|
|
1035
|
+
[ -n "${EPAM_RESUME_RUN:-}" ] || return 1
|
|
1036
|
+
local _rck="${EPAM_PROJECT_CONFIG_DIR:-${PROJECT_CONFIG_DIR:-}}/runs/${EPAM_RESUME_RUN}/lanes/${_rcl}/checkpoint"
|
|
1037
|
+
[ -f "$_rck/prd.json" ] && [ -f "$_rck/checkpoint.json" ] || return 1
|
|
1038
|
+
# POST-SPEC IS ENOUGH: a lane checkpointed after its spec pass holds its spec. Requiring pre-writer took a specified
|
|
1039
|
+
# lane for one with no checkpoint, and it specified again — gotransit and metrolinx, $22.33, AMSD-2974 run 9
|
|
1040
|
+
# (2026-10-14; operator: "not permitted"). What follows the spec pass still runs, and the pauses still decide.
|
|
1041
|
+
local _want; _want=$(_stage_rank post-spec)
|
|
1042
|
+
[ "$(_stage_rank "$(jq -r '.stage // empty' "$_rck/checkpoint.json" 2>/dev/null)")" -ge "$_want" ] || return 1
|
|
1043
|
+
if [ -n "${EPAM_RESUME_FROM_STAGE:-}" ] && [ "$(_stage_rank "$EPAM_RESUME_FROM_STAGE")" -lt "$_want" ]; then
|
|
1044
|
+
return 1
|
|
1045
|
+
fi
|
|
1046
|
+
# The approved PRD first, when the working copy lost its spec (lib/run-checkpoint.sh lane_restore_approved_prd).
|
|
1047
|
+
lane_restore_approved_prd "$_rck" "$_rout"
|
|
1048
|
+
# A SPEC BUILT FROM INPUTS THAT NO LONGER HOLD IS SPECIFIED AGAIN (lib/run-checkpoint.sh).
|
|
1049
|
+
if ! lane_spec_is_current "$_rck" "$_rout"; then
|
|
1050
|
+
log "[orch] Codeline '${_rcl}': not resuming its pre-writer checkpoint — the lane is specified again"
|
|
1051
|
+
return 1
|
|
1052
|
+
fi
|
|
1053
|
+
if _operator_edited "$_rout" "$_rck/reviewed/$(basename "$_rout")"; then
|
|
1054
|
+
log "[orch] Codeline '${_rcl}': resuming from its pre-writer checkpoint — KEEPING its PRD, edited at the pause"
|
|
1055
|
+
else
|
|
1056
|
+
cp "$_rck/prd.json" "$_rout" || return 1
|
|
1057
|
+
log "[orch] Codeline '${_rcl}': resuming from its pre-writer checkpoint — its spec pass is done; the PRD approved at the pause is the lane's PRD"
|
|
1058
|
+
fi
|
|
1059
|
+
_lane_spec_mode=0
|
|
1060
|
+
return 0
|
|
1061
|
+
}
|
|
1062
|
+
|
|
1063
|
+
# THE PATH THE CALLER PASSED, ACTUALLY FORWARDED.
|
|
1064
|
+
#
|
|
1065
|
+
# This took "$1" and never used it, so the handler read no PRD; and `2>/dev/null` hid the error
|
|
1066
|
+
# it raised about that. The phase list came back EMPTY, every lane's phase loop ran zero times,
|
|
1067
|
+
# and each lane reported "✓ completed" in five seconds having invoked nothing. Live 2026-08-17,
|
|
1068
|
+
# mock3: two lanes, two pending stories, no writer, no commit, "Pipeline complete".
|
|
1069
|
+
#
|
|
1070
|
+
# An empty phase list is a real answer ("this PRD declares no phases") and indistinguishable
|
|
1071
|
+
# from a failure to read one — so the failure must be loud, and is.
|
|
1072
|
+
_prd_phases() {
|
|
1073
|
+
local _pp_prd="${1:-${PRD_FILE:-}}"
|
|
1074
|
+
local _pp_err _pp_out _pp_rc=0
|
|
1075
|
+
_pp_err=$(mktemp "${TMPDIR:-/tmp}/prd-phases-err-XXXXXX")
|
|
1076
|
+
_pp_out=$("$NODE_BIN" "$SCRIPT_DIR/lib/handlers/prd-phases.js" "$_pp_prd" 2>"$_pp_err") || _pp_rc=$?
|
|
1077
|
+
if [ "$_pp_rc" -ne 0 ]; then
|
|
1078
|
+
error "[orch] could not read the phases of ${_pp_prd}: $(cat "$_pp_err" 2>/dev/null)"
|
|
1079
|
+
error "[orch] Refusing to treat that as 'no phases' — a lane with no phases does nothing and reports success."
|
|
1080
|
+
rm -f "$_pp_err"
|
|
1081
|
+
return 1
|
|
1082
|
+
fi
|
|
1083
|
+
rm -f "$_pp_err"
|
|
1084
|
+
printf '%s' "$_pp_out"
|
|
1085
|
+
}
|
|
1086
|
+
|
|
1087
|
+
# ── Lane execution: parallel by default ────────────────────────────────────
|
|
1088
|
+
# Lanes of a spanning story are independent units of work; running them in
|
|
1089
|
+
# sequence costs (N-1) x lane-duration in wall clock. On AMSD-2041 three
|
|
1090
|
+
# ~20-minute lanes take an hour to do 20 minutes of work.
|
|
1091
|
+
#
|
|
1092
|
+
# THREE THINGS SEQUENCING GAVE FOR FREE, and how each is kept:
|
|
1093
|
+
#
|
|
1094
|
+
# 1. The canonical PRD merge. Every lane writes the same file, so concurrent
|
|
1095
|
+
# merges would clobber each other. Workers therefore never merge: each
|
|
1096
|
+
# writes only its own filtered PRD, and the merges run here, after the
|
|
1097
|
+
# wait, one at a time in declared order.
|
|
1098
|
+
# 2. The halt rule. Sequencing prevented spend outright. In parallel the
|
|
1099
|
+
# lanes are already running, so the equivalent is to abort the survivors
|
|
1100
|
+
# the moment one fails. Money already spent cannot be recovered; money not
|
|
1101
|
+
# yet spent still can.
|
|
1102
|
+
# 3. The cross-codeline contract. _run_codeline_bridge feeds one lane's
|
|
1103
|
+
# exported API into the NEXT lane's prompt, which presupposes an upstream
|
|
1104
|
+
# that has already finished. In parallel there is no upstream, so it is
|
|
1105
|
+
# skipped with a warning rather than silently writing a contract nobody
|
|
1106
|
+
# consumed. Lanes that genuinely integrate must set
|
|
1107
|
+
# EPAM_PARALLEL_CODELINES=0.
|
|
1108
|
+
_EPAM_PARALLEL_LANES="${EPAM_PARALLEL_CODELINES:-1}"
|
|
1109
|
+
if [ "$_EPAM_PARALLEL_LANES" = "1" ] && [ "${#_cl_entries[@]}" -gt 1 ]; then
|
|
1110
|
+
local _p_cls=() _p_wts=() _p_prds=() _p_pids=() _p_spec=() _lane_spec_mode=""
|
|
1111
|
+
local _p_statusdir; _p_statusdir="$(mktemp -d /tmp/orch-lanes-XXXXXX)"
|
|
1112
|
+
|
|
1113
|
+
for _entry in "${_cl_entries[@]}"; do
|
|
1114
|
+
local _cl="${_entry%%:*}" _wt="${_entry#*:}"
|
|
1115
|
+
local _cl_prd="$_work_dir/${_cl}-prd.json"
|
|
1116
|
+
_cl_prds+=("$_cl_prd")
|
|
1117
|
+
local _n_stories
|
|
1118
|
+
if _lane_resume_prd "$_cl" "$_cl_prd"; then
|
|
1119
|
+
story_resume_completed_by_commit "$_wt" "${LOG_DIR:-/tmp}/lanes/${_cl}" "$_cl_prd"
|
|
1120
|
+
_n_stories=$(jq '(.stories // []) | length' "$_cl_prd" 2>/dev/null || echo 0)
|
|
1121
|
+
else
|
|
1122
|
+
_n_stories=$(_filtered_prd "$_cl" "$_cl_prd" "$_prd_path")
|
|
1123
|
+
fi
|
|
1124
|
+
# The lane's work copy carries the story's plan (a checkpoint may predate it; the checkpoint is not edited).
|
|
1125
|
+
"$NODE_BIN" "$SCRIPT_DIR/lib/handlers/carry-plan.js" "$_cl_prd" "$_prd_path" || true
|
|
1126
|
+
if [ "${_n_stories:-0}" -eq 0 ]; then
|
|
1127
|
+
log "[orch] Codeline '${_cl}': no stories — skipping"
|
|
1128
|
+
continue
|
|
1129
|
+
fi
|
|
1130
|
+
if lane_declared_passed "$_cl"; then
|
|
1131
|
+
log "[orch] Codeline '${_cl}': declared passed in this run (EPAM_PASSED_LANES) — its own line is not run again"
|
|
1132
|
+
# THE DECLARATION IS THE CODELINE'S RESULT (lib/handlers/record-declared-pass.js).
|
|
1133
|
+
"$NODE_BIN" "$SCRIPT_DIR/lib/handlers/record-declared-pass.js" "$_prd_path" "$_cl" || true
|
|
1134
|
+
# ITS OTHER RELEASE LINES STILL SHIP (lib/release-lines.sh): passed is the operator's word for the codeline's own line.
|
|
1135
|
+
_run_release_lines "$_cl" "$_cl_prd" "$_wt" "${LOG_DIR:-/tmp}/lanes/${_cl}" "$_log_file" "$_phase_filter" "$0" "$_prd_path" || _overall=1
|
|
1136
|
+
continue
|
|
1137
|
+
fi
|
|
1138
|
+
_p_cls+=("$_cl"); _p_wts+=("$_wt"); _p_prds+=("$_cl_prd"); _p_spec+=("${_lane_spec_mode}")
|
|
1139
|
+
log "[orch] Codeline '${_cl}' — ${_n_stories} stories → ${_wt}"
|
|
1140
|
+
done
|
|
1141
|
+
|
|
1142
|
+
if [ "${#_p_cls[@]}" -gt 1 ]; then
|
|
1143
|
+
warning "[orch] Parallel lanes: cross-codeline contract bridge SKIPPED — no lane is upstream of another."
|
|
1144
|
+
warning "[orch] Set EPAM_PARALLEL_CODELINES=0 if these lanes integrate with each other."
|
|
1145
|
+
fi
|
|
1146
|
+
|
|
1147
|
+
log "[orch] Launching ${#_p_cls[@]} codeline(s) in PARALLEL..."
|
|
1148
|
+
_lane_idx=0
|
|
1149
|
+
while [ "$_lane_idx" -lt "${#_p_cls[@]}" ]; do
|
|
1150
|
+
_cl="${_p_cls[$_lane_idx]}"
|
|
1151
|
+
_wt="${_p_wts[$_lane_idx]}"
|
|
1152
|
+
_cl_prd="${_p_prds[$_lane_idx]}"
|
|
1153
|
+
_lane_spec_mode="${_p_spec[$_lane_idx]:-}"
|
|
1154
|
+
_lane_status="${_p_statusdir}/${_cl}.status"
|
|
1155
|
+
(
|
|
1156
|
+
_log_file="${LOG_DIR:-/tmp}/lane-${_cl}.log"
|
|
1157
|
+
# ── Per-lane LOG_DIR ──────────────────────────────────────────────
|
|
1158
|
+
# Lanes used to inherit ONE LOG_DIR, and several files in it are read
|
|
1159
|
+
# back as STATE rather than merely written as logs. The proven case is
|
|
1160
|
+
# phase-baseline-sha.txt, the git SHA every diff-based gate uses to
|
|
1161
|
+
# decide what a story changed. With three lanes on three different
|
|
1162
|
+
# repositories the last writer won, so two lanes diffed against a commit
|
|
1163
|
+
# that does not exist in them — an empty diff, and review-ranger /
|
|
1164
|
+
# mutant-hunter / team-lead-review passing on ZERO files. A false pass on
|
|
1165
|
+
# unreviewed code, which is worse than a crash because it looks like
|
|
1166
|
+
# success. Live metrolinx 2026-07-29, killed on discovery of this.
|
|
1167
|
+
#
|
|
1168
|
+
# Scoping LOG_DIR fixes every such file at once — the baseline SHA, the
|
|
1169
|
+
# review-incomplete-<phase> flag (PHASE is identical across lanes), the
|
|
1170
|
+
# story-outputs manifest and the worktree logs — instead of patching each
|
|
1171
|
+
# call site in seven scripts and missing the eighth.
|
|
1172
|
+
_lane_log_dir="${LOG_DIR:-/tmp}/lanes/${_cl}"
|
|
1173
|
+
mkdir -p "$_lane_log_dir" 2>/dev/null || true
|
|
1174
|
+
_cl_failed=0
|
|
1175
|
+
local _phases=()
|
|
1176
|
+
mapfile -t _phases < <(_prd_phases "$_cl_prd")
|
|
1177
|
+
local _cl_failed=0
|
|
1178
|
+
|
|
1179
|
+
for _phase in "${_phases[@]}"; do
|
|
1180
|
+
[ -z "$_phase" ] && continue
|
|
1181
|
+
# When the tier3 launcher (or any external caller) manages phases externally
|
|
1182
|
+
# by calling run-agent-orchestration.sh once per phase, honour its filter so
|
|
1183
|
+
# only the requested phase runs. Without this, a --phase scaffold call would
|
|
1184
|
+
# still execute every phase across every codeline (double-execution bug).
|
|
1185
|
+
if [ -n "$_phase_filter" ] && [ "$_phase" != "$_phase_filter" ]; then
|
|
1186
|
+
log "[orch] Phase '${_phase}' — skipping (caller phase filter: '${_phase_filter}')"
|
|
1187
|
+
continue
|
|
1188
|
+
fi
|
|
1189
|
+
log "[orch] Phase '${_phase}' — codeline '${_cl}'..."
|
|
1190
|
+
local _lane_reset; _lane_reset="$(_lane_phase_args)"
|
|
1191
|
+
local _pex=0
|
|
1192
|
+
JIRA_CODELINE_RUN=1 \
|
|
1193
|
+
EPAM_CODELINE="$_cl" \
|
|
1194
|
+
LOG_DIR="$_lane_log_dir" \
|
|
1195
|
+
AGENT_IO_DIR="$_lane_log_dir/agent-io" \
|
|
1196
|
+
PRD_FILE="$_cl_prd" \
|
|
1197
|
+
EPAM_SPEC_MODE="${_lane_spec_mode:-${EPAM_SPEC_MODE:-}}" \
|
|
1198
|
+
PROJECT_ROOT="$_wt" \
|
|
1199
|
+
OUTPUT_DIR="$_wt" \
|
|
1200
|
+
PHASE="$_phase" \
|
|
1201
|
+
CROSS_CODELINE_PRD="${CROSS_CODELINE_PRD:-}" \
|
|
1202
|
+
bash "$0" ${_lane_reset:+"$_lane_reset"} 2>&1 | tee -a "$_log_file"
|
|
1203
|
+
# No pipefail in this script, so `| tee || _pex=` tests tee's exit (0) and never
|
|
1204
|
+
# fires — a phase exit-2 (gate block) was masked to _pex=0 → "done" → PASSED
|
|
1205
|
+
# (live AMSD-1820 run #3). Capture the inner orch's real exit; tee exits 0 so set -e is fine.
|
|
1206
|
+
_pex=${PIPESTATUS[0]}
|
|
1207
|
+
|
|
1208
|
+
# exit 2 = gate remediation applied — reset stories and retry once (mirrors tier3 launcher)
|
|
1209
|
+
# WHILE THE GATES REMEDIATE, THE PHASE IS RETRIED — remediation stays on (AMSD-2974 run 9, 2026-10-14: the one retry
|
|
1210
|
+
# ran with remediation off, so a second, different finding — mutant-hunter after spec-validator — could not heal and
|
|
1211
|
+
# ended the line). Exit 2 means a gate added something new to the story; the remediator adds nothing already
|
|
1212
|
+
# covered, so the loop ends when there is nothing new to fix (or the story's own budget does).
|
|
1213
|
+
while phase_exit_is_retryable "$_pex"; do
|
|
1214
|
+
log "[orch] '${_phase}' ('${_cl}'): $(phase_exit_describe "$_pex") — retrying the phase"
|
|
1215
|
+
# A test-only remediation keeps the stories' work (lib/phase-exit.sh phase_exit_retry_args).
|
|
1216
|
+
_rargs=(); mapfile -t _rargs < <(phase_exit_retry_args "$_pex")
|
|
1217
|
+
_pex=0; _rmark=$(phase_retry_mark "$_wt")
|
|
1218
|
+
JIRA_CODELINE_RUN=1 \
|
|
1219
|
+
EPAM_CODELINE="$_cl" \
|
|
1220
|
+
LOG_DIR="$_lane_log_dir" \
|
|
1221
|
+
AGENT_IO_DIR="$_lane_log_dir/agent-io" \
|
|
1222
|
+
PRD_FILE="$_cl_prd" \
|
|
1223
|
+
EPAM_SPEC_MODE="${_lane_spec_mode:-${EPAM_SPEC_MODE:-}}" \
|
|
1224
|
+
PROJECT_ROOT="$_wt" \
|
|
1225
|
+
OUTPUT_DIR="$_wt" \
|
|
1226
|
+
PHASE="$_phase" \
|
|
1227
|
+
CROSS_CODELINE_PRD="${CROSS_CODELINE_PRD:-}" \
|
|
1228
|
+
bash "$0" ${_rargs[@]+"${_rargs[@]}"} 2>&1 | tee -a "$_log_file"
|
|
1229
|
+
_pex=${PIPESTATUS[0]}
|
|
1230
|
+
if [ "$_pex" -eq 0 ]; then
|
|
1231
|
+
log "[orch] Self-healing retry succeeded for '${_phase}' ('${_cl}')"
|
|
1232
|
+
elif ! phase_exit_is_retryable "$_pex"; then
|
|
1233
|
+
error "[orch] Phase '${_phase}' for '${_cl}' failed after self-healing retry (exit $_pex)"
|
|
1234
|
+
elif ! phase_retry_progressed "$_wt" "$_rmark"; then
|
|
1235
|
+
# THE PIPELINE NEVER LOOPS (lib/phase-exit.sh): the retry changed nothing in the codeline, so another is no different.
|
|
1236
|
+
error "[orch] Phase '${_phase}' for '${_cl}': the self-healing retry changed nothing in the codeline (still ${_rmark:0:9}) — not retrying again"
|
|
1237
|
+
_pex=1
|
|
1238
|
+
fi
|
|
1239
|
+
done
|
|
1240
|
+
|
|
1241
|
+
if [ "$_pex" -ne 0 ]; then
|
|
1242
|
+
error "[orch] Phase '${_phase}' for '${_cl}' failed (exit $_pex)"
|
|
1243
|
+
_cl_failed=1; _overall=1; break
|
|
1244
|
+
fi
|
|
1245
|
+
if lane_paused_since "$_lane_log_dir" "$_lanes_started"; then
|
|
1246
|
+
log "[orch] Phase '${_phase}' — '${_cl}' PAUSED (inputs ready, writer not started)."
|
|
1247
|
+
else
|
|
1248
|
+
log "[orch] Phase '${_phase}' — '${_cl}' done."
|
|
1249
|
+
fi
|
|
1250
|
+
done
|
|
1251
|
+
|
|
1252
|
+
# A SHARED CODELINE SHIPS ON EVERY RELEASE LINE ITS STORY'S PLAN NAMES (lib/release-lines.sh): its stories,
|
|
1253
|
+
# done on its own base, are worked once more per line off it — 0, 1 or N, as the plan says.
|
|
1254
|
+
if [ "$_cl_failed" = "0" ]; then
|
|
1255
|
+
_run_release_lines "$_cl" "$_cl_prd" "$_wt" "$_lane_log_dir" "$_log_file" "$_phase_filter" "$0" "$_prd_path" || { _cl_failed=1; _overall=1; }
|
|
1256
|
+
fi
|
|
1257
|
+
|
|
1258
|
+
[ "$_cl_failed" = "0" ] && \
|
|
1259
|
+
_completed_list="${_completed_list:+${_completed_list}:}${_cl_prd}"
|
|
1260
|
+
echo "$_cl_failed" > "$_lane_status"
|
|
1261
|
+
) &
|
|
1262
|
+
_p_pids+=("$!")
|
|
1263
|
+
_lane_idx=$(( _lane_idx + 1 ))
|
|
1264
|
+
done
|
|
1265
|
+
|
|
1266
|
+
_p_any_failed=0
|
|
1267
|
+
_p_running=1
|
|
1268
|
+
while [ "$_p_running" = "1" ]; do
|
|
1269
|
+
_p_running=0
|
|
1270
|
+
for _pid in "${_p_pids[@]}"; do
|
|
1271
|
+
kill -0 "$_pid" 2>/dev/null && _p_running=1
|
|
1272
|
+
done
|
|
1273
|
+
for _cl in "${_p_cls[@]}"; do
|
|
1274
|
+
if [ -f "${_p_statusdir}/${_cl}.status" ] && [ "$(cat "${_p_statusdir}/${_cl}.status" 2>/dev/null)" != "0" ]; then
|
|
1275
|
+
_p_any_failed=1
|
|
1276
|
+
fi
|
|
1277
|
+
done
|
|
1278
|
+
# Default: let sibling lanes run to natural completion even after one
|
|
1279
|
+
# lane has failed. Found live 2026-08-02 (Writer Retest, AMSD-2041):
|
|
1280
|
+
# gotransit failed on a deterministic gate blocking on a pre-existing,
|
|
1281
|
+
# unrelated broken import (see run_relative_import_check's scope fix,
|
|
1282
|
+
# same incident) — a FALSE failure — which killed upexpress and
|
|
1283
|
+
# metrolinx mid-attempt via SIGTERM, discarding real, valid,
|
|
1284
|
+
# independent work those lanes were producing. The per-lane fold-back
|
|
1285
|
+
# logic below already records each lane's own outcome independently
|
|
1286
|
+
# (a spanning story is complete only when NO lane is outstanding) —
|
|
1287
|
+
# nothing downstream needed the early kill. Set
|
|
1288
|
+
# EPAM_CASCADE_ABORT_ON_LANE_FAILURE=1 to restore the old behavior
|
|
1289
|
+
# (stop spending immediately once any lane's outcome is known to have
|
|
1290
|
+
# failed) for cost-conscious runs where sibling lanes' work has no
|
|
1291
|
+
# independent value once one lane is lost.
|
|
1292
|
+
if [ "${EPAM_CASCADE_ABORT_ON_LANE_FAILURE:-0}" = "1" ] && [ "$_p_any_failed" = "1" ] && [ "$_p_running" = "1" ]; then
|
|
1293
|
+
error "[orch] HALT: a codeline failed after its retries and self-heal completed."
|
|
1294
|
+
error "[orch] Aborting the codeline(s) still running (EPAM_CASCADE_ABORT_ON_LANE_FAILURE=1) —"
|
|
1295
|
+
error "[orch] recovery is exhausted, so letting them finish would spend on a run already decided."
|
|
1296
|
+
for _pid in "${_p_pids[@]}"; do _kill_lane_tree "$_pid"; done
|
|
1297
|
+
break
|
|
1298
|
+
fi
|
|
1299
|
+
[ "$_p_running" = "1" ] && sleep 5
|
|
1300
|
+
done
|
|
1301
|
+
wait 2>/dev/null || true
|
|
1302
|
+
|
|
1303
|
+
_lane_idx=0
|
|
1304
|
+
while [ "$_lane_idx" -lt "${#_p_cls[@]}" ]; do
|
|
1305
|
+
_cl="${_p_cls[$_lane_idx]}"
|
|
1306
|
+
_cl_prd="${_p_prds[$_lane_idx]}"
|
|
1307
|
+
_cl_failed=1
|
|
1308
|
+
[ -f "${_p_statusdir}/${_cl}.status" ] && _cl_failed="$(cat "${_p_statusdir}/${_cl}.status" 2>/dev/null || echo 1)"
|
|
1309
|
+
# ── Fold this lane's ledgers back into the parent ────────────────────
|
|
1310
|
+
# Per-lane LOG_DIR fixed cross-lane state corruption, and fragmented the
|
|
1311
|
+
# cost ledger as a side effect: every reader of the canonical path — the
|
|
1312
|
+
# dashboard, validate-dashboards.sh, the run report — saw an EMPTY
|
|
1313
|
+
# phase-cost.jsonl for a parallel run while the real records sat in
|
|
1314
|
+
# lanes/<codeline>/. Cost tracking that silently reports zero is worse than
|
|
1315
|
+
# none, so the append-only ledgers are concatenated back, in declared lane
|
|
1316
|
+
# order, once the lane has finished writing them.
|
|
1317
|
+
for _ledger in phase-cost.jsonl agent-activity.jsonl healing-events.jsonl phase-gates.jsonl; do
|
|
1318
|
+
if [ -n "${LOG_DIR:-}" ] && [ -s "${LOG_DIR}/lanes/${_cl}/${_ledger}" ]; then
|
|
1319
|
+
cat "${LOG_DIR}/lanes/${_cl}/${_ledger}" >> "${LOG_DIR}/${_ledger}" 2>/dev/null || true
|
|
1320
|
+
fi
|
|
1321
|
+
done
|
|
1322
|
+
|
|
1323
|
+
# PER-LANE OUTCOME, RECORDED. Lanes have their own log dir, PRD, worktree, investigator
|
|
1324
|
+
# and writer, run in parallel, and this script states that no lane is upstream of
|
|
1325
|
+
# another — then the result collapsed into one exit code, so one lane's gate decision
|
|
1326
|
+
# failed a run in which the others had cleared. Live 2026-08-07: two lanes reached the
|
|
1327
|
+
# writer pause cleanly and the run reported failure because a third was blocked by the
|
|
1328
|
+
# spec review gate. Nothing said which, or that two-thirds of the work was fine.
|
|
1329
|
+
_LANE_OUTCOMES="${_LANE_OUTCOMES:+${_LANE_OUTCOMES}
|
|
1330
|
+
}${_cl} $([ "$_cl_failed" = "0" ] && echo ok || echo blocked)"
|
|
1331
|
+
if [ "$_cl_failed" = "0" ]; then
|
|
1332
|
+
_completed_list="${_completed_list:+${_completed_list}:}${_cl_prd}"
|
|
1333
|
+
else
|
|
1334
|
+
# Always explain a failed lane, even when every lane had already
|
|
1335
|
+
# finished by the time the abort poll noticed. Without this the run ends
|
|
1336
|
+
# non-zero with no statement of which lane died or why the others were
|
|
1337
|
+
# stopped — the operator is left diffing timestamps.
|
|
1338
|
+
error "[orch] codeline '${_cl}' did not complete — its retries and self-heal are exhausted."
|
|
1339
|
+
# LANE INDEPENDENCE IS ABOUT WHAT KEEPS RUNNING — NOT ABOUT THE EXIT CODE.
|
|
1340
|
+
#
|
|
1341
|
+
# A lane that does not complete does not invalidate the ones that did, and it must not
|
|
1342
|
+
# kill a sibling still doing real work: that independence is real, and
|
|
1343
|
+
# EPAM_CASCADE_ABORT_ON_LANE_FAILURE=1 is the knob that restores the old cascade.
|
|
1344
|
+
#
|
|
1345
|
+
# But independence stops at the exit status. This briefly returned 0 when a lane had
|
|
1346
|
+
# failed, which is the silent-failure class this pipeline exists to eliminate: every
|
|
1347
|
+
# automated caller, wrapper and CI reader takes exit 0 as "the work is done", and the
|
|
1348
|
+
# per-lane summary printed below is prose no caller parses. A run in which any lane
|
|
1349
|
+
# exhausted its retries, its ladder and its self-heal is a failed run — it is reported
|
|
1350
|
+
# as one, and the summary says which lanes did complete so nothing is thrown away.
|
|
1351
|
+
error "[orch] HALT: codeline '${_cl}' failed — the run is reported as failed."
|
|
1352
|
+
error "[orch] Lanes that completed keep their work on their own per-story branch;"
|
|
1353
|
+
error "[orch] the lane summary below states exactly which."
|
|
1354
|
+
_overall=1
|
|
1355
|
+
fi
|
|
1356
|
+
# Merge this codeline's final story state (status/completed/completedAt/
|
|
1357
|
+
# testCriteria/etc — whatever claude.sh/TC-writer wrote into the filtered
|
|
1358
|
+
# temp copy during real execution) back into the canonical PRD. Without
|
|
1359
|
+
# this, _cl_prd is the only file that ever held the real completion
|
|
1360
|
+
# result, and it was being deleted at the end of the loop — so the
|
|
1361
|
+
# canonical PRD (the file every downstream consumer, dashboard, and test
|
|
1362
|
+
# actually reads) stayed "pending" forever even after a real, successful
|
|
1363
|
+
# run. Found live 2026-07-23 via mock1.
|
|
1364
|
+
# The merge itself is lib/story-merge.js: it is the same operation on both the
|
|
1365
|
+
# sequential and the parallel path, it decides what survives a multi-lane run,
|
|
1366
|
+
# and inline in a heredoc it could not be tested. See that module for why a
|
|
1367
|
+
# spanning story cannot be merged wholesale.
|
|
1368
|
+
if "$NODE_BIN" "$SCRIPT_DIR/lib/handlers/merge-lane-into-canonical.js" \
|
|
1369
|
+
"$SCRIPT_DIR" "${_prd_path}" "${_cl_prd}" "${_cl}"; then
|
|
1370
|
+
log "[orch] Merged codeline '${_cl}' story state back into canonical PRD"
|
|
1371
|
+
else
|
|
1372
|
+
# Previously 2>/dev/null: a failed merge was indistinguishable from a
|
|
1373
|
+
# successful one that logged nothing, and it silently discards this lane's
|
|
1374
|
+
# entire outcome — status, criteria and all.
|
|
1375
|
+
error "[orch] FAILED to merge codeline '${_cl}' back into the canonical PRD;"
|
|
1376
|
+
error "[orch] that lane's status and verification criteria are NOT recorded."
|
|
1377
|
+
_overall=1
|
|
1378
|
+
fi
|
|
1379
|
+
_lane_idx=$(( _lane_idx + 1 ))
|
|
1380
|
+
done
|
|
1381
|
+
rm -rf "$_p_statusdir" 2>/dev/null || true
|
|
1382
|
+
else
|
|
1383
|
+
for _entry in "${_cl_entries[@]}"; do
|
|
1384
|
+
local _cl="${_entry%%:*}" _wt="${_entry#*:}"
|
|
1385
|
+
local _cl_prd="$_work_dir/${_cl}-prd.json"
|
|
1386
|
+
_cl_prds+=("$_cl_prd")
|
|
1387
|
+
|
|
1388
|
+
local _n_stories _lane_spec_mode=""
|
|
1389
|
+
if _lane_resume_prd "$_cl" "$_cl_prd"; then
|
|
1390
|
+
story_resume_completed_by_commit "$_wt" "${LOG_DIR:-/tmp}/lanes/${_cl}" "$_cl_prd"
|
|
1391
|
+
_n_stories=$(jq '(.stories // []) | length' "$_cl_prd" 2>/dev/null || echo 0)
|
|
1392
|
+
else
|
|
1393
|
+
_n_stories=$(_filtered_prd "$_cl" "$_cl_prd" "$_prd_path")
|
|
1394
|
+
fi
|
|
1395
|
+
# The lane's work copy carries the story's plan (a checkpoint may predate it; the checkpoint is not edited).
|
|
1396
|
+
"$NODE_BIN" "$SCRIPT_DIR/lib/handlers/carry-plan.js" "$_cl_prd" "$_prd_path" || true
|
|
1397
|
+
|
|
1398
|
+
if [ "${_n_stories:-0}" -eq 0 ]; then
|
|
1399
|
+
log "[orch] Codeline '${_cl}': no stories — skipping"
|
|
1400
|
+
continue
|
|
1401
|
+
fi
|
|
1402
|
+
if lane_declared_passed "$_cl"; then
|
|
1403
|
+
log "[orch] Codeline '${_cl}': declared passed in this run (EPAM_PASSED_LANES) — its own line is not run again"
|
|
1404
|
+
# THE DECLARATION IS THE CODELINE'S RESULT (lib/handlers/record-declared-pass.js).
|
|
1405
|
+
"$NODE_BIN" "$SCRIPT_DIR/lib/handlers/record-declared-pass.js" "$_prd_path" "$_cl" || true
|
|
1406
|
+
# ITS OTHER RELEASE LINES STILL SHIP (lib/release-lines.sh): passed is the operator's word for the codeline's own line.
|
|
1407
|
+
_run_release_lines "$_cl" "$_cl_prd" "$_wt" "${LOG_DIR:-/tmp}/lanes/${_cl}" "$_log_file" "$_phase_filter" "$0" "$_prd_path" || _overall=1
|
|
1408
|
+
continue
|
|
1409
|
+
fi
|
|
1410
|
+
|
|
1411
|
+
_rebuild_cross
|
|
1412
|
+
log "[orch] Codeline '${_cl}' — ${_n_stories} stories → ${_wt}"
|
|
1413
|
+
|
|
1414
|
+
# A SEQUENTIAL LANE GETS ITS OWN LOG_DIR TOO. The parallel loop scoped it (2026-07-29: state
|
|
1415
|
+
# files read back across lanes); this loop did not, so every lane shared phase-baseline-sha.txt
|
|
1416
|
+
# and story-retry-state/. Live AMSD-2974 2026-10-05: cx-shared's baseline commit was recorded in
|
|
1417
|
+
# the upexpress and metrolinx checkpoints (a commit their repos do not have), and the gotransit
|
|
1418
|
+
# writer resumed at attempt 10/12 on a counter the lanes had shared.
|
|
1419
|
+
local _lane_log_dir="${LOG_DIR:-/tmp}/lanes/${_cl}"
|
|
1420
|
+
mkdir -p "$_lane_log_dir" 2>/dev/null || true
|
|
1421
|
+
|
|
1422
|
+
local _phases=()
|
|
1423
|
+
mapfile -t _phases < <(_prd_phases "$_cl_prd")
|
|
1424
|
+
local _cl_failed=0
|
|
1425
|
+
|
|
1426
|
+
for _phase in "${_phases[@]}"; do
|
|
1427
|
+
[ -z "$_phase" ] && continue
|
|
1428
|
+
# When the tier3 launcher (or any external caller) manages phases externally
|
|
1429
|
+
# by calling run-agent-orchestration.sh once per phase, honour its filter so
|
|
1430
|
+
# only the requested phase runs. Without this, a --phase scaffold call would
|
|
1431
|
+
# still execute every phase across every codeline (double-execution bug).
|
|
1432
|
+
if [ -n "$_phase_filter" ] && [ "$_phase" != "$_phase_filter" ]; then
|
|
1433
|
+
log "[orch] Phase '${_phase}' — skipping (caller phase filter: '${_phase_filter}')"
|
|
1434
|
+
continue
|
|
1435
|
+
fi
|
|
1436
|
+
log "[orch] Phase '${_phase}' — codeline '${_cl}'..."
|
|
1437
|
+
local _lane_reset; _lane_reset="$(_lane_phase_args)"
|
|
1438
|
+
local _pex=0
|
|
1439
|
+
JIRA_CODELINE_RUN=1 \
|
|
1440
|
+
EPAM_CODELINE="$_cl" \
|
|
1441
|
+
LOG_DIR="$_lane_log_dir" \
|
|
1442
|
+
AGENT_IO_DIR="$_lane_log_dir/agent-io" \
|
|
1443
|
+
PRD_FILE="$_cl_prd" \
|
|
1444
|
+
EPAM_SPEC_MODE="${_lane_spec_mode:-${EPAM_SPEC_MODE:-}}" \
|
|
1445
|
+
PROJECT_ROOT="$_wt" \
|
|
1446
|
+
OUTPUT_DIR="$_wt" \
|
|
1447
|
+
PHASE="$_phase" \
|
|
1448
|
+
CROSS_CODELINE_PRD="${CROSS_CODELINE_PRD:-}" \
|
|
1449
|
+
bash "$0" ${_lane_reset:+"$_lane_reset"} 2>&1 | tee -a "$_log_file"
|
|
1450
|
+
# No pipefail in this script, so `| tee || _pex=` tests tee's exit (0) and never
|
|
1451
|
+
# fires — a phase exit-2 (gate block) was masked to _pex=0 → "done" → PASSED
|
|
1452
|
+
# (live AMSD-1820 run #3). Capture the inner orch's real exit; tee exits 0 so set -e is fine.
|
|
1453
|
+
_pex=${PIPESTATUS[0]}
|
|
1454
|
+
|
|
1455
|
+
# exit 2 = gate remediation applied — reset stories and retry once (mirrors tier3 launcher)
|
|
1456
|
+
# WHILE THE GATES REMEDIATE, THE PHASE IS RETRIED — remediation stays on (AMSD-2974 run 9, 2026-10-14: the one retry
|
|
1457
|
+
# ran with remediation off, so a second, different finding — mutant-hunter after spec-validator — could not heal and
|
|
1458
|
+
# ended the line). Exit 2 means a gate added something new to the story; the remediator adds nothing already
|
|
1459
|
+
# covered, so the loop ends when there is nothing new to fix (or the story's own budget does).
|
|
1460
|
+
while phase_exit_is_retryable "$_pex"; do
|
|
1461
|
+
log "[orch] '${_phase}' ('${_cl}'): $(phase_exit_describe "$_pex") — retrying the phase"
|
|
1462
|
+
# A test-only remediation keeps the stories' work (lib/phase-exit.sh phase_exit_retry_args).
|
|
1463
|
+
_rargs=(); mapfile -t _rargs < <(phase_exit_retry_args "$_pex")
|
|
1464
|
+
_pex=0; _rmark=$(phase_retry_mark "$_wt")
|
|
1465
|
+
JIRA_CODELINE_RUN=1 \
|
|
1466
|
+
EPAM_CODELINE="$_cl" \
|
|
1467
|
+
LOG_DIR="$_lane_log_dir" \
|
|
1468
|
+
AGENT_IO_DIR="$_lane_log_dir/agent-io" \
|
|
1469
|
+
PRD_FILE="$_cl_prd" \
|
|
1470
|
+
EPAM_SPEC_MODE="${_lane_spec_mode:-${EPAM_SPEC_MODE:-}}" \
|
|
1471
|
+
PROJECT_ROOT="$_wt" \
|
|
1472
|
+
OUTPUT_DIR="$_wt" \
|
|
1473
|
+
PHASE="$_phase" \
|
|
1474
|
+
CROSS_CODELINE_PRD="${CROSS_CODELINE_PRD:-}" \
|
|
1475
|
+
bash "$0" ${_rargs[@]+"${_rargs[@]}"} 2>&1 | tee -a "$_log_file"
|
|
1476
|
+
_pex=${PIPESTATUS[0]}
|
|
1477
|
+
if [ "$_pex" -eq 0 ]; then
|
|
1478
|
+
log "[orch] Self-healing retry succeeded for '${_phase}' ('${_cl}')"
|
|
1479
|
+
elif ! phase_exit_is_retryable "$_pex"; then
|
|
1480
|
+
error "[orch] Phase '${_phase}' for '${_cl}' failed after self-healing retry (exit $_pex)"
|
|
1481
|
+
elif ! phase_retry_progressed "$_wt" "$_rmark"; then
|
|
1482
|
+
# THE PIPELINE NEVER LOOPS (lib/phase-exit.sh): the retry changed nothing in the codeline, so another is no different.
|
|
1483
|
+
error "[orch] Phase '${_phase}' for '${_cl}': the self-healing retry changed nothing in the codeline (still ${_rmark:0:9}) — not retrying again"
|
|
1484
|
+
_pex=1
|
|
1485
|
+
fi
|
|
1486
|
+
done
|
|
1487
|
+
|
|
1488
|
+
if [ "$_pex" -ne 0 ]; then
|
|
1489
|
+
error "[orch] Phase '${_phase}' for '${_cl}' failed (exit $_pex)"
|
|
1490
|
+
_cl_failed=1; _overall=1; break
|
|
1491
|
+
fi
|
|
1492
|
+
if lane_paused_since "$_lane_log_dir" "$_lanes_started"; then
|
|
1493
|
+
log "[orch] Phase '${_phase}' — '${_cl}' PAUSED (inputs ready, writer not started)."
|
|
1494
|
+
else
|
|
1495
|
+
log "[orch] Phase '${_phase}' — '${_cl}' done."
|
|
1496
|
+
fi
|
|
1497
|
+
done
|
|
1498
|
+
|
|
1499
|
+
# A SHARED CODELINE SHIPS ON EVERY RELEASE LINE ITS STORY'S PLAN NAMES (lib/release-lines.sh): its stories,
|
|
1500
|
+
# done on its own base, are worked once more per line off it — 0, 1 or N, as the plan says.
|
|
1501
|
+
if [ "$_cl_failed" = "0" ]; then
|
|
1502
|
+
_run_release_lines "$_cl" "$_cl_prd" "$_wt" "$_lane_log_dir" "$_log_file" "$_phase_filter" "$0" "$_prd_path" || { _cl_failed=1; _overall=1; }
|
|
1503
|
+
fi
|
|
1504
|
+
|
|
1505
|
+
# The append-only ledgers back into the parent, as the parallel loop folds them.
|
|
1506
|
+
for _ledger in phase-cost.jsonl agent-activity.jsonl healing-events.jsonl phase-gates.jsonl; do
|
|
1507
|
+
if [ -n "${LOG_DIR:-}" ] && [ -s "${_lane_log_dir}/${_ledger}" ]; then
|
|
1508
|
+
cat "${_lane_log_dir}/${_ledger}" >> "${LOG_DIR}/${_ledger}" 2>/dev/null || true
|
|
1509
|
+
fi
|
|
1510
|
+
done
|
|
1511
|
+
|
|
1512
|
+
[ "$_cl_failed" = "0" ] && \
|
|
1513
|
+
_completed_list="${_completed_list:+${_completed_list}:}${_cl_prd}"
|
|
1514
|
+
|
|
1515
|
+
# Run codeline-bridge agent after each successful codeline (multi-codeline PRDs only).
|
|
1516
|
+
# Extracts exported types/functions/endpoints and writes a cross-codeline contract
|
|
1517
|
+
# consumed by downstream codeline agents via CROSS_CODELINE_CONTRACT_<CL_UPPER>.
|
|
1518
|
+
if [ "$_cl_failed" = "0" ] && [ "${#_cl_entries[@]}" -gt 1 ]; then
|
|
1519
|
+
_run_codeline_bridge "$_cl" "$_wt" "$_cl_prd"
|
|
1520
|
+
fi
|
|
1521
|
+
|
|
1522
|
+
# Merge this codeline's final story state (status/completed/completedAt/
|
|
1523
|
+
# testCriteria/etc — whatever claude.sh/TC-writer wrote into the filtered
|
|
1524
|
+
# temp copy during real execution) back into the canonical PRD. Without
|
|
1525
|
+
# this, _cl_prd is the only file that ever held the real completion
|
|
1526
|
+
# result, and it was being deleted at the end of the loop — so the
|
|
1527
|
+
# canonical PRD (the file every downstream consumer, dashboard, and test
|
|
1528
|
+
# actually reads) stayed "pending" forever even after a real, successful
|
|
1529
|
+
# run. Found live 2026-07-23 via mock1.
|
|
1530
|
+
# The merge itself is lib/story-merge.js: it is the same operation on both the
|
|
1531
|
+
# sequential and the parallel path, it decides what survives a multi-lane run,
|
|
1532
|
+
# and inline in a heredoc it could not be tested. See that module for why a
|
|
1533
|
+
# spanning story cannot be merged wholesale.
|
|
1534
|
+
if "$NODE_BIN" "$SCRIPT_DIR/lib/handlers/merge-lane-into-canonical.js" \
|
|
1535
|
+
"$SCRIPT_DIR" "${_prd_path}" "${_cl_prd}" "${_cl}"; then
|
|
1536
|
+
log "[orch] Merged codeline '${_cl}' story state back into canonical PRD"
|
|
1537
|
+
else
|
|
1538
|
+
# Previously 2>/dev/null: a failed merge was indistinguishable from a
|
|
1539
|
+
# successful one that logged nothing, and it silently discards this lane's
|
|
1540
|
+
# entire outcome — status, criteria and all.
|
|
1541
|
+
error "[orch] FAILED to merge codeline '${_cl}' back into the canonical PRD;"
|
|
1542
|
+
error "[orch] that lane's status and verification criteria are NOT recorded."
|
|
1543
|
+
_overall=1
|
|
1544
|
+
fi
|
|
1545
|
+
|
|
1546
|
+
# ── Halt once a lane has finally failed ───────────────────────────────────
|
|
1547
|
+
# The failure path inside the PHASE loop above ends in a bare `break`, which
|
|
1548
|
+
# leaves that inner loop only — the CODELINE loop carried on to the next
|
|
1549
|
+
# lane. Live AMSD-2041 (2026-07-28): 'gotransit' hit a FATAL after 3/3 spec
|
|
1550
|
+
# attempts and the self-heal retry, and five seconds later the run started
|
|
1551
|
+
# 'upexpress'. A comment below this loop asserted "Lane failures already stop
|
|
1552
|
+
# the loop"; it was never true, and being written down is what kept anyone
|
|
1553
|
+
# from checking.
|
|
1554
|
+
#
|
|
1555
|
+
# By this point retries, the ladder and self-heal have ALL completed and the
|
|
1556
|
+
# step is still failed — the standing mandate is to stop there. The merge
|
|
1557
|
+
# above runs first deliberately, so the canonical PRD still records where the
|
|
1558
|
+
# run died. Nothing is guessed about WHY it failed: a lane that reached here
|
|
1559
|
+
# exhausted every recovery the pipeline has.
|
|
1560
|
+
if [ "$_cl_failed" = "1" ]; then
|
|
1561
|
+
error "[orch] HALT: codeline '${_cl}' failed."
|
|
1562
|
+
_halt_recovery_state "${_cl_failed_story:-}"
|
|
1563
|
+
error "[orch] Not starting the remaining codeline(s)."
|
|
1564
|
+
break
|
|
1565
|
+
fi
|
|
1566
|
+
done
|
|
1567
|
+
fi
|
|
1568
|
+
|
|
1569
|
+
# ── Partial coverage is a failure, not a pass ──────────────────────────────
|
|
1570
|
+
# A spanning story names the codelines it must be delivered in. Lane failures
|
|
1571
|
+
# already stop the loop; this catches the quieter case — a lane that never ran,
|
|
1572
|
+
# or ran and produced no result for this story — where every lane "succeeded",
|
|
1573
|
+
# the pipeline reports complete, and part of the work simply never happened.
|
|
1574
|
+
# Checked against what the story DECLARED, not against how many lanes we
|
|
1575
|
+
# happened to execute.
|
|
1576
|
+
if [ "$_overall" = "0" ] && [ -f "$_prd_path" ]; then
|
|
1577
|
+
_mc_incomplete=$("$NODE_BIN" "$SCRIPT_DIR/lib/handlers/spanning-stories-incomplete.js" "$_prd_path" 2>/dev/null || true)
|
|
1578
|
+
if [ -n "$_mc_incomplete" ]; then
|
|
1579
|
+
error "[orch] Spanning story INCOMPLETE — a declared codeline never ran: ${_mc_incomplete}"
|
|
1580
|
+
error "[orch] The run touched fewer codelines than the story requires; this is not a success."
|
|
1581
|
+
_overall=1
|
|
1582
|
+
fi
|
|
1583
|
+
fi
|
|
1584
|
+
|
|
1585
|
+
# NOT deleted. This `rm -f` was /tmp hygiene: the lane PRDs used to live in a shared,
|
|
1586
|
+
# machine-global namespace and had to be swept. They now live in this run's OWN directory
|
|
1587
|
+
# (<project-config>/runs/<run-id>/work/), which makes them run EVIDENCE — the archiver
|
|
1588
|
+
# reads the working PRD from there, and deleting it first is why working-prd.json came
|
|
1589
|
+
# back "missing" on run 20260805T182214Z after the isolation fix.
|
|
1590
|
+
#
|
|
1591
|
+
# The cross-lane scratch file has no evidentiary value and is still removed.
|
|
1592
|
+
rm -f "$_cross_prd" 2>/dev/null || true
|
|
1593
|
+
unset CROSS_CODELINE_PRD
|
|
1594
|
+
|
|
1595
|
+
# THE SUMMARY IS THE POINT. A story spanning codelines can now finish with some lanes
|
|
1596
|
+
# complete and some not, and the one thing that must never happen is someone merging two of
|
|
1597
|
+
# three without knowing the third is missing.
|
|
1598
|
+
if [ -n "${_LANE_OUTCOMES:-}" ]; then
|
|
1599
|
+
local _ok_n _blocked_n
|
|
1600
|
+
_ok_n=$(printf '%s\n' "$_LANE_OUTCOMES" | grep -c 'ok$' || true)
|
|
1601
|
+
_blocked_n=$(printf '%s\n' "$_LANE_OUTCOMES" | grep -c 'blocked$' || true)
|
|
1602
|
+
log "[orch] Lane outcomes — ${_ok_n} completed, ${_blocked_n} did not:"
|
|
1603
|
+
printf '%s\n' "$_LANE_OUTCOMES" | while IFS=$'\t' read -r _lc _ls; do
|
|
1604
|
+
[ -n "$_lc" ] || continue
|
|
1605
|
+
if [ "$_ls" = "ok" ]; then log "[orch] ✓ ${_lc}"; else error "[orch] ✗ ${_lc} — did not complete"; fi
|
|
1606
|
+
done
|
|
1607
|
+
if [ "${_blocked_n:-0}" != "0" ] && [ "${_ok_n:-0}" != "0" ]; then
|
|
1608
|
+
warning "[orch] This story spans codelines and did NOT complete on all of them."
|
|
1609
|
+
warning "[orch] Work sits on a per-story branch in each completed codeline; merging is yours to decide."
|
|
1610
|
+
warning "[orch] The run is reported as FAILED because a lane did not complete; the"
|
|
1611
|
+
warning "[orch] completed lanes' work is intact on their branches and is yours to merge."
|
|
1612
|
+
fi
|
|
1613
|
+
fi
|
|
1614
|
+
|
|
1615
|
+
[ "$_overall" = "0" ] \
|
|
1616
|
+
&& # A RUN THAT SPENT MONEY MUST NOT REPORT NOTHING. Checked here, at the end, where the run knows
|
|
1617
|
+
# both what it did and what it recorded — see lib/cost-ledger.sh for the run this exists for.
|
|
1618
|
+
assert_cost_ledger_not_silently_empty || true
|
|
1619
|
+
# THE RECORDING IS NOT THE ARTEFACT. Langfuse holds the traces; a cassette is a directory on
|
|
1620
|
+
# disk, and it exists only if something writes it. Nothing did — the exporter had no caller —
|
|
1621
|
+
# so four runs on 2026-09-09 recorded 278 traces and left nothing replayable once the machine
|
|
1622
|
+
# restarted and Langfuse went down with it.
|
|
1623
|
+
#
|
|
1624
|
+
# Here, at completion, because that is when the run knows its own id and the recording is
|
|
1625
|
+
# whole. It cannot fail the run: the work is finished by this point, and a run that succeeded
|
|
1626
|
+
# must not be reported failed because a recording could not be fetched.
|
|
1627
|
+
# A RUN WHOSE LANES PAUSED IS PAUSED, NOT COMPLETE; and `log ... || error ...` always took the first branch (log returns
|
|
1628
|
+
# 0), so a failed run printed "Pipeline complete" too (live AMSD-2974 run 9: "complete", then "aborting").
|
|
1629
|
+
if [ "$_overall" = "0" ] && record_lanes_pause "$_lanes_started"; then
|
|
1630
|
+
:
|
|
1631
|
+
elif [ "$_overall" = "0" ]; then
|
|
1632
|
+
log "[orch] ✅ Pipeline complete."
|
|
1633
|
+
else
|
|
1634
|
+
error "[orch] ⚠️ Pipeline completed with errors."
|
|
1635
|
+
fi
|
|
1636
|
+
|
|
1637
|
+
return $_overall
|
|
1638
|
+
}
|
|
1639
|
+
|
|
1640
|
+
# derive_sandbox_base_image <prd_file>
|
|
1641
|
+
# Maps a project's OWN project.stack.language/.runtime (already written by
|
|
1642
|
+
# the LLM-based `epam new generate` PRD pipeline for every scaffolded
|
|
1643
|
+
# project — see src/scaffold/ManifestAnalyzer.ts's generatePrd(), currently
|
|
1644
|
+
# unread by anything downstream) to a Docker base image for the sandbox.
|
|
1645
|
+
# Generic pattern match, same shape as resolve_model_provider()'s
|
|
1646
|
+
# EPAM_MODEL_PROVIDER_MAP glob convention elsewhere in this pipeline — no
|
|
1647
|
+
# specific PROJECT is ever named here, only a small number of well-known
|
|
1648
|
+
# language keywords. Falls back to node:20-slim (also correct for THIS
|
|
1649
|
+
# project) when project.stack is missing/unrecognized, so an older PRD
|
|
1650
|
+
# without stack data still gets a sane default rather than a build failure.
|
|
1651
|
+
derive_sandbox_base_image() {
|
|
1652
|
+
local prd_file="$1"
|
|
1653
|
+
local stack_text=""
|
|
1654
|
+
if [ -f "$prd_file" ]; then
|
|
1655
|
+
stack_text=$(jq -r '[.project.stack.language, .project.stack.runtime] | map(select(. != null)) | join(" ")' "$prd_file" 2>/dev/null | tr '[:upper:]' '[:lower:]')
|
|
1656
|
+
fi
|
|
1657
|
+
case "$stack_text" in
|
|
1658
|
+
*python*) echo "python:3.11-slim" ;;
|
|
1659
|
+
*golang*|*"go "*|*go) echo "golang:1.22-bookworm" ;;
|
|
1660
|
+
*rust*) echo "rust:1.75-slim" ;;
|
|
1661
|
+
*node*|*typescript*|*javascript*) echo "node:20-slim" ;;
|
|
1662
|
+
*) echo "node:20-slim" ;;
|
|
1663
|
+
esac
|
|
1664
|
+
}
|