@ai-dossier/sched 0.3.0 → 0.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +97 -3
- package/dist/attribution.d.ts +132 -0
- package/dist/attribution.d.ts.map +1 -0
- package/dist/attribution.js +268 -0
- package/dist/attribution.js.map +1 -0
- package/dist/bisect.d.ts +67 -0
- package/dist/bisect.d.ts.map +1 -0
- package/dist/bisect.js +122 -0
- package/dist/bisect.js.map +1 -0
- package/dist/dispatch.d.ts +22 -0
- package/dist/dispatch.d.ts.map +1 -1
- package/dist/dispatch.js +50 -1
- package/dist/dispatch.js.map +1 -1
- package/dist/enqueue.d.ts +9 -0
- package/dist/enqueue.d.ts.map +1 -1
- package/dist/enqueue.js +96 -17
- package/dist/enqueue.js.map +1 -1
- package/dist/groundtruth.d.ts +6 -0
- package/dist/groundtruth.d.ts.map +1 -1
- package/dist/groundtruth.js +11 -3
- package/dist/groundtruth.js.map +1 -1
- package/dist/index.d.ts +6 -3
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +36 -2
- package/dist/index.js.map +1 -1
- package/dist/json.d.ts +8 -0
- package/dist/json.d.ts.map +1 -0
- package/dist/json.js +18 -0
- package/dist/json.js.map +1 -0
- package/dist/recovery.d.ts +263 -0
- package/dist/recovery.d.ts.map +1 -0
- package/dist/recovery.js +730 -0
- package/dist/recovery.js.map +1 -0
- package/dist/scheduler.d.ts.map +1 -1
- package/dist/scheduler.js +6 -28
- package/dist/scheduler.js.map +1 -1
- package/dist/state.d.ts +55 -3
- package/dist/state.d.ts.map +1 -1
- package/dist/state.js +205 -6
- package/dist/state.js.map +1 -1
- package/dist/types.d.ts +114 -2
- package/dist/types.d.ts.map +1 -1
- package/dist/types.js +33 -3
- package/dist/types.js.map +1 -1
- package/package.json +2 -2
package/dist/recovery.js
ADDED
|
@@ -0,0 +1,730 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* Batch failure recovery (#472, RFC-0001 §F.2/F.8/F.9): what happens when the
|
|
4
|
+
* aggregate suite goes red or the batch PR will not merge.
|
|
5
|
+
*
|
|
6
|
+
* ```
|
|
7
|
+
* validating → attributing → fixing (ONE bounded attempt) → validating
|
|
8
|
+
* → evicting (revert the member's commits) → validating
|
|
9
|
+
* > ⅓ evicted, or a revert conflict → dissolving → members requeued
|
|
10
|
+
* awaiting-merge (CONFLICTING | auto-merge-blocked)
|
|
11
|
+
* → rebasing → re-validating → shipping
|
|
12
|
+
* → (2nd occurrence) dissolving into two half-batches
|
|
13
|
+
* ```
|
|
14
|
+
*
|
|
15
|
+
* The rules this module exists to enforce:
|
|
16
|
+
*
|
|
17
|
+
* - **Nothing green is discarded.** Shipped and terminal members keep their
|
|
18
|
+
* outcome through every eviction and dissolve; only active work requeues.
|
|
19
|
+
* - **One fix attempt per member.** A second red suite evicts — it never
|
|
20
|
+
* re-dispatches, so a batch cannot burn its budget on one broken member.
|
|
21
|
+
* - **A revert is never left mid-conflict.** The conflicting revert is aborted
|
|
22
|
+
* so the worktree is clean, but the reverts that already landed stay applied
|
|
23
|
+
* — which is exactly why the batch dissolves and the branch is abandoned
|
|
24
|
+
* rather than reused.
|
|
25
|
+
* - **The scheduler never calls an LLM.** `beginFixAttempt` returns the command
|
|
26
|
+
* and prompt for the caller to spawn (dispatch.ts), exactly like #464.
|
|
27
|
+
*
|
|
28
|
+
* State transitions are the typed rails in state.ts; every effect (git, the
|
|
29
|
+
* suite runner, milestone posting, the journal) is injected, so the whole
|
|
30
|
+
* module is testable against scratch repos and fakes.
|
|
31
|
+
*/
|
|
32
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
33
|
+
exports.createExecMilestonePoster = createExecMilestonePoster;
|
|
34
|
+
exports.beginAttribution = beginAttribution;
|
|
35
|
+
exports.beginFixAttempt = beginFixAttempt;
|
|
36
|
+
exports.resolveFixAttempt = resolveFixAttempt;
|
|
37
|
+
exports.expandEvictionGroups = expandEvictionGroups;
|
|
38
|
+
exports.evictMembers = evictMembers;
|
|
39
|
+
exports.checkDissolveTrigger = checkDissolveTrigger;
|
|
40
|
+
exports.dissolveBatch = dissolveBatch;
|
|
41
|
+
exports.handlePrConflict = handlePrConflict;
|
|
42
|
+
const attribution_1 = require("./attribution");
|
|
43
|
+
const bisect_1 = require("./bisect");
|
|
44
|
+
const dispatch_1 = require("./dispatch");
|
|
45
|
+
const journal_1 = require("./journal");
|
|
46
|
+
const state_1 = require("./state");
|
|
47
|
+
const types_1 = require("./types");
|
|
48
|
+
/** Milestone values carry no spaces (the dossier rule) — collapse them to `-`. */
|
|
49
|
+
function milestoneValue(value) {
|
|
50
|
+
return value.trim().replace(/\s+/g, '-');
|
|
51
|
+
}
|
|
52
|
+
/**
|
|
53
|
+
* The default poster: shells `ai-dossier runstate post`, which validates the
|
|
54
|
+
* phase/status/key contract and refuses a malformed milestone. Never throws —
|
|
55
|
+
* a failed post is journaled by the caller and degrades the audit trail, it
|
|
56
|
+
* does not fail the recovery.
|
|
57
|
+
*/
|
|
58
|
+
function createExecMilestonePoster(exec, opts = {}) {
|
|
59
|
+
const bin = opts.bin ?? 'ai-dossier';
|
|
60
|
+
return (anchor, run, milestone) => {
|
|
61
|
+
const args = [
|
|
62
|
+
'runstate',
|
|
63
|
+
'post',
|
|
64
|
+
'--issue',
|
|
65
|
+
String(anchor),
|
|
66
|
+
'--phase',
|
|
67
|
+
milestone.phase,
|
|
68
|
+
'--status',
|
|
69
|
+
milestone.status,
|
|
70
|
+
'--run',
|
|
71
|
+
run,
|
|
72
|
+
];
|
|
73
|
+
for (const [key, value] of Object.entries(milestone.kv)) {
|
|
74
|
+
args.push('--kv', `${key}=${milestoneValue(value)}`);
|
|
75
|
+
}
|
|
76
|
+
return exec(bin, args, opts.repoDir) !== null;
|
|
77
|
+
};
|
|
78
|
+
}
|
|
79
|
+
function clock(deps) {
|
|
80
|
+
return deps.now ? deps.now() : new Date();
|
|
81
|
+
}
|
|
82
|
+
/**
|
|
83
|
+
* Re-run the aggregate suite, treating a runner that THROWS as a red suite.
|
|
84
|
+
*
|
|
85
|
+
* The re-run happens after `git revert` has already rewritten the branch, and
|
|
86
|
+
* an exception escaping here would discard the whole eviction — reverted
|
|
87
|
+
* commits on the branch, but a state file still calling the members healthy.
|
|
88
|
+
*/
|
|
89
|
+
function runSuite(deps, batchId, now) {
|
|
90
|
+
if (!deps.runSuite)
|
|
91
|
+
return null;
|
|
92
|
+
try {
|
|
93
|
+
return deps.runSuite();
|
|
94
|
+
}
|
|
95
|
+
catch (err) {
|
|
96
|
+
const detail = `suite runner threw: ${err.message}`;
|
|
97
|
+
journal(deps, (0, journal_1.unitEvent)('suite-failed', `batch:${batchId}`, { detail }), now);
|
|
98
|
+
return { ok: false, failing: [], detail };
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
function journal(deps, event, now) {
|
|
102
|
+
deps.journal?.append(event, now);
|
|
103
|
+
}
|
|
104
|
+
function batchOrThrow(state, batchId) {
|
|
105
|
+
const batch = (0, state_1.findBatch)(state, batchId);
|
|
106
|
+
if (!batch)
|
|
107
|
+
throw new types_1.SchedNotFoundError(`Batch not found: ${batchId}`);
|
|
108
|
+
return batch;
|
|
109
|
+
}
|
|
110
|
+
/**
|
|
111
|
+
* Run one git command, journaling it when it fails.
|
|
112
|
+
*
|
|
113
|
+
* `ExecFn` collapses every failure into `null` — a genuine merge conflict, a
|
|
114
|
+
* bad object, a missing binary and an expired lock all look identical. The
|
|
115
|
+
* decisions here (evict? dissolve?) hang off those nulls, so the command that
|
|
116
|
+
* produced one is always recorded; without it an operator sees a dissolve with
|
|
117
|
+
* no way to tell a conflict from a broken environment.
|
|
118
|
+
*/
|
|
119
|
+
function git(deps, args, batchId, now) {
|
|
120
|
+
const out = deps.exec('git', args, deps.repoDir);
|
|
121
|
+
if (out === null) {
|
|
122
|
+
journal(deps, (0, journal_1.unitEvent)('git-failed', `batch:${batchId}`, { detail: `git ${args.join(' ')}` }), now);
|
|
123
|
+
}
|
|
124
|
+
return out;
|
|
125
|
+
}
|
|
126
|
+
/**
|
|
127
|
+
* Post a milestone (AC5), or journal the one that could not be posted.
|
|
128
|
+
*
|
|
129
|
+
* A milestone is the operator's only record of an eviction or dissolve, so
|
|
130
|
+
* every path that fails to post one says so in the journal — including the
|
|
131
|
+
* "no anchor / no run id / no poster" skips, which are silent by construction
|
|
132
|
+
* otherwise. The journalled detail carries the full key set so the post can be
|
|
133
|
+
* reconstructed by hand.
|
|
134
|
+
*/
|
|
135
|
+
function post(deps, batch, milestone, now) {
|
|
136
|
+
const rendered = Object.entries(milestone.kv)
|
|
137
|
+
.map(([key, value]) => `${key}=${milestoneValue(value)}`)
|
|
138
|
+
.join(' ');
|
|
139
|
+
const describe = (why) => {
|
|
140
|
+
journal(deps, (0, journal_1.unitEvent)('milestone-post-failed', `batch:${batch.id}`, {
|
|
141
|
+
detail: `${milestone.phase} ${milestone.status} ${rendered} (${why})`,
|
|
142
|
+
}), now);
|
|
143
|
+
};
|
|
144
|
+
if (batch.anchor === null) {
|
|
145
|
+
describe('batch has no anchor issue to post to');
|
|
146
|
+
return false;
|
|
147
|
+
}
|
|
148
|
+
if (batch.run_id === null) {
|
|
149
|
+
describe('batch has no run id — runstate post requires one');
|
|
150
|
+
return false;
|
|
151
|
+
}
|
|
152
|
+
if (!deps.postMilestone) {
|
|
153
|
+
describe('no milestone poster configured');
|
|
154
|
+
return false;
|
|
155
|
+
}
|
|
156
|
+
const ok = deps.postMilestone(batch.anchor, batch.run_id, milestone);
|
|
157
|
+
if (!ok)
|
|
158
|
+
describe('runstate post failed');
|
|
159
|
+
return ok;
|
|
160
|
+
}
|
|
161
|
+
/**
|
|
162
|
+
* `validating → attributing`: map the red suite's failures onto members (AC1).
|
|
163
|
+
* Overlap first; `git bisect` only when overlap left something ambiguous or
|
|
164
|
+
* unattributed, and only ever to ADD an offender — a bisect that lands on a
|
|
165
|
+
* non-member commit reports `unattributable` and adds nobody.
|
|
166
|
+
*/
|
|
167
|
+
function beginAttribution(state, batchId, input, deps) {
|
|
168
|
+
const now = clock(deps);
|
|
169
|
+
const batch = batchOrThrow(state, batchId);
|
|
170
|
+
let next = state;
|
|
171
|
+
if (batch.status !== 'attributing') {
|
|
172
|
+
next = (0, state_1.transitionBatch)(next, batchId, 'attributing', {}, now);
|
|
173
|
+
}
|
|
174
|
+
journal(deps, (0, journal_1.unitEvent)('suite-failed', `batch:${batchId}`, {
|
|
175
|
+
// Zero failing tests reaching attribution is not "the suite passed" — it
|
|
176
|
+
// is a report nothing could be read out of, which looks identical to
|
|
177
|
+
// green unless the line says so.
|
|
178
|
+
detail: input.failing.length === 0
|
|
179
|
+
? '0 failing — nothing to attribute (the suite report may be empty or unparseable)'
|
|
180
|
+
: `${input.failing.length} failing`,
|
|
181
|
+
}), now);
|
|
182
|
+
const overlap = (0, attribution_1.attributeByOverlap)(input.failing, input.footprints);
|
|
183
|
+
const attributed = new Map(overlap.attributed);
|
|
184
|
+
let bisect = null;
|
|
185
|
+
let method = attributed.size > 0 ? 'overlap' : 'none';
|
|
186
|
+
const needsBisect = overlap.ambiguous.length > 0 || overlap.unattributed.length > 0;
|
|
187
|
+
if (needsBisect && input.bisect) {
|
|
188
|
+
bisect = (0, bisect_1.runAttributionBisect)(deps.exec, {
|
|
189
|
+
repoDir: deps.repoDir,
|
|
190
|
+
good: input.bisect.good,
|
|
191
|
+
bad: input.bisect.bad,
|
|
192
|
+
testCommand: input.bisect.testCommand,
|
|
193
|
+
boundary: input.bisect.boundary,
|
|
194
|
+
onWarn: (detail) => journal(deps, (0, journal_1.unitEvent)('git-failed', `batch:${batchId}`, { detail }), now),
|
|
195
|
+
});
|
|
196
|
+
if (bisect.kind === 'first-bad') {
|
|
197
|
+
const unresolved = [...overlap.ambiguous.map((a) => a.test), ...overlap.unattributed];
|
|
198
|
+
attributed.set(bisect.issue, [...(attributed.get(bisect.issue) ?? []), ...unresolved]);
|
|
199
|
+
method = 'bisect';
|
|
200
|
+
}
|
|
201
|
+
}
|
|
202
|
+
const outcome = {
|
|
203
|
+
method,
|
|
204
|
+
offenders: (0, attribution_1.offendersOf)(attributed),
|
|
205
|
+
attributed,
|
|
206
|
+
ambiguous: overlap.ambiguous,
|
|
207
|
+
unattributed: overlap.unattributed,
|
|
208
|
+
bisect,
|
|
209
|
+
};
|
|
210
|
+
// `method=none offenders=none` is the state that precedes a blanket dissolve,
|
|
211
|
+
// so the line has to say WHY nothing was attributed — whether bisect was
|
|
212
|
+
// never asked for, errored, or landed on a non-member commit.
|
|
213
|
+
const bisectNote = bisect === null
|
|
214
|
+
? needsBisect
|
|
215
|
+
? 'bisect=not-requested'
|
|
216
|
+
: 'bisect=not-needed'
|
|
217
|
+
: `bisect=${bisect.kind}${'sha' in bisect ? `@${bisect.sha}` : ''}${'detail' in bisect ? ` (${bisect.detail})` : ''}`;
|
|
218
|
+
journal(deps, (0, journal_1.unitEvent)('attributed', `batch:${batchId}`, {
|
|
219
|
+
detail: `method=${outcome.method} offenders=${outcome.offenders.join(',') || 'none'} ` +
|
|
220
|
+
`ambiguous=${outcome.ambiguous.length} unattributed=${outcome.unattributed.length} ${bisectNote}`,
|
|
221
|
+
}), now);
|
|
222
|
+
return { state: next, outcome };
|
|
223
|
+
}
|
|
224
|
+
/**
|
|
225
|
+
* `attributing → fixing`: hand back the ONE bounded fix dispatch for a member
|
|
226
|
+
* (AC2). Returns `dispatch: null` when the member already had its attempt —
|
|
227
|
+
* the caller's next step is then `evictMembers`, never a second dispatch.
|
|
228
|
+
*/
|
|
229
|
+
function beginFixAttempt(state, batchId, issue, deps, opts = {}) {
|
|
230
|
+
const now = clock(deps);
|
|
231
|
+
const batch = batchOrThrow(state, batchId);
|
|
232
|
+
const attempts = batch.fix_attempts.filter((a) => a.issue === issue).length;
|
|
233
|
+
if (attempts >= types_1.MAX_FIX_ATTEMPTS_PER_MEMBER) {
|
|
234
|
+
// Journaled, not silent: otherwise the trail shows one `fix-dispatched`
|
|
235
|
+
// and then a `member-evicted` with nothing explaining why no second fix
|
|
236
|
+
// was tried.
|
|
237
|
+
journal(deps, (0, journal_1.unitEvent)('fix-resolved', `batch:${batchId}`, {
|
|
238
|
+
issue,
|
|
239
|
+
detail: `refused: ${attempts}/${types_1.MAX_FIX_ATTEMPTS_PER_MEMBER} attempts already used — next step is eviction`,
|
|
240
|
+
}), now);
|
|
241
|
+
return { state, dispatch: null };
|
|
242
|
+
}
|
|
243
|
+
const tier = types_1.FIX_ATTEMPT_TIER;
|
|
244
|
+
// `max_slots` is irrelevant here — only the command/prompt/tier-model parts
|
|
245
|
+
// of the resolved dispatch are used — but SchedConfig requires it.
|
|
246
|
+
const resolved = (0, dispatch_1.resolveDispatch)(opts.config ?? { max_slots: types_1.DEFAULT_MAX_SLOTS });
|
|
247
|
+
const dispatch = {
|
|
248
|
+
issue,
|
|
249
|
+
tier,
|
|
250
|
+
command: (0, dispatch_1.buildAgentCommand)(resolved.command, tier, issue, {
|
|
251
|
+
...dispatch_1.DEFAULT_TIER_MODELS,
|
|
252
|
+
...resolved.tierModels,
|
|
253
|
+
}),
|
|
254
|
+
prompt: (0, dispatch_1.buildFixPrompt)(resolved.fixPrompt, issue, batchId, (opts.tests ?? []).map((t) => t.id)),
|
|
255
|
+
};
|
|
256
|
+
const record = { issue, tier, outcome: 'dispatched', at: now.toISOString() };
|
|
257
|
+
let next = state;
|
|
258
|
+
if (batch.status !== 'fixing') {
|
|
259
|
+
next = (0, state_1.transitionBatch)(next, batchId, 'fixing', {}, now);
|
|
260
|
+
}
|
|
261
|
+
next = (0, state_1.patchBatch)(next, batchId, { fix_attempts: [...batch.fix_attempts, record] }, now);
|
|
262
|
+
journal(deps, (0, journal_1.unitEvent)('fix-dispatched', `batch:${batchId}`, { issue, tier }), now);
|
|
263
|
+
return { state: next, dispatch };
|
|
264
|
+
}
|
|
265
|
+
/**
|
|
266
|
+
* `fixing → validating`: record how the fix attempt ended and return the batch
|
|
267
|
+
* to validation, where the suite re-run decides what happens next (AC2). A
|
|
268
|
+
* `red` outcome does NOT evict here — the caller re-runs the suite and calls
|
|
269
|
+
* `evictMembers` if it is still failing, so eviction is always driven by
|
|
270
|
+
* evidence rather than by the fix agent's own claim.
|
|
271
|
+
*/
|
|
272
|
+
function resolveFixAttempt(state, batchId, issue, outcome, deps) {
|
|
273
|
+
const now = clock(deps);
|
|
274
|
+
const batch = batchOrThrow(state, batchId);
|
|
275
|
+
// The most recent still-open attempt for this member.
|
|
276
|
+
const idx = batch.fix_attempts.findLastIndex((a) => a.issue === issue && a.outcome === 'dispatched');
|
|
277
|
+
const fixAttempts = idx === -1
|
|
278
|
+
? batch.fix_attempts
|
|
279
|
+
: batch.fix_attempts.map((a, i) => (i === idx ? { ...a, outcome } : a));
|
|
280
|
+
let next = (0, state_1.patchBatch)(state, batchId, { fix_attempts: fixAttempts }, now);
|
|
281
|
+
if (batch.status === 'fixing') {
|
|
282
|
+
next = (0, state_1.transitionBatch)(next, batchId, 'validating', {}, now);
|
|
283
|
+
}
|
|
284
|
+
journal(deps, (0, journal_1.unitEvent)('fix-resolved', `batch:${batchId}`, {
|
|
285
|
+
issue,
|
|
286
|
+
detail: idx === -1 ? `${outcome} (no matching dispatched attempt on record)` : outcome,
|
|
287
|
+
}), now);
|
|
288
|
+
return { state: next };
|
|
289
|
+
}
|
|
290
|
+
/**
|
|
291
|
+
* Every member that must leave with `issues`: an eviction group reverts
|
|
292
|
+
* together (RFC-0001 §E.4), because a member built on another's API cannot
|
|
293
|
+
* survive that API's revert.
|
|
294
|
+
*/
|
|
295
|
+
function expandEvictionGroups(batch, issues) {
|
|
296
|
+
const out = new Set(issues);
|
|
297
|
+
let grew = true;
|
|
298
|
+
while (grew) {
|
|
299
|
+
grew = false;
|
|
300
|
+
for (const group of batch.eviction_groups) {
|
|
301
|
+
if (group.some((m) => out.has(m))) {
|
|
302
|
+
for (const member of group) {
|
|
303
|
+
if (!out.has(member)) {
|
|
304
|
+
out.add(member);
|
|
305
|
+
grew = true;
|
|
306
|
+
}
|
|
307
|
+
}
|
|
308
|
+
}
|
|
309
|
+
}
|
|
310
|
+
}
|
|
311
|
+
// Restricted to actual members: a group naming a stray issue would otherwise
|
|
312
|
+
// produce an eviction record for a non-member, which then counts against
|
|
313
|
+
// `members.length` in the dissolve trigger and dissolves the batch early.
|
|
314
|
+
return [...out].filter((issue) => batch.members.includes(issue)).sort((a, b) => a - b);
|
|
315
|
+
}
|
|
316
|
+
/**
|
|
317
|
+
* The targets' commits in the order a revert must walk them: newest first
|
|
318
|
+
* ACROSS members, not member by member.
|
|
319
|
+
*
|
|
320
|
+
* Grouping by member loses branch order, and reverting an older commit while a
|
|
321
|
+
* newer one still sits on top of it is precisely what makes `git revert`
|
|
322
|
+
* conflict — which dissolves the whole batch. For a branch `A1 B1 A2 B2`,
|
|
323
|
+
* evicting both members reverts `B2 A2 B1 A1`.
|
|
324
|
+
*/
|
|
325
|
+
function orderRevertCommits(ranges, targets) {
|
|
326
|
+
return ranges
|
|
327
|
+
.filter((range) => targets.includes(range.issue))
|
|
328
|
+
.flatMap((range) => range.commits.map((sha, i) => ({ sha, position: range.positions[i] ?? i })))
|
|
329
|
+
.sort((a, b) => b.position - a.position)
|
|
330
|
+
.map((commit) => commit.sha);
|
|
331
|
+
}
|
|
332
|
+
/**
|
|
333
|
+
* `attributing → evicting`: revert the offending members' commits, requeue them
|
|
334
|
+
* as full-cycle with their failure evidence, re-run the suite, and dissolve if
|
|
335
|
+
* the batch has now lost more than a third of its members (AC2/AC3/AC5).
|
|
336
|
+
*
|
|
337
|
+
* A conflicting revert aborts and dissolves immediately — a half-reverted
|
|
338
|
+
* worktree is not a state any later step can reason about.
|
|
339
|
+
*/
|
|
340
|
+
function evictMembers(state, batchId, input, deps) {
|
|
341
|
+
const now = clock(deps);
|
|
342
|
+
const batch = batchOrThrow(state, batchId);
|
|
343
|
+
const targets = expandEvictionGroups(batch, input.issues);
|
|
344
|
+
// An eviction group can pull in a member that already shipped. Reverting its
|
|
345
|
+
// commits would destroy merged work while `requeueMember` (rightly) refuses
|
|
346
|
+
// to requeue it — the batch would then report a member as shipped with its
|
|
347
|
+
// code gone. Dissolving instead keeps the invariant: nothing green is
|
|
348
|
+
// discarded, and the unshipped members are requeued intact.
|
|
349
|
+
const shipped = targets.filter((issue) => {
|
|
350
|
+
const entry = state.entries.find((e) => e.issue === issue);
|
|
351
|
+
return entry !== undefined && (0, state_1.isPreservedMember)(entry);
|
|
352
|
+
});
|
|
353
|
+
if (shipped.length > 0) {
|
|
354
|
+
journal(deps, (0, journal_1.unitEvent)('revert-conflict', `batch:${batchId}`, {
|
|
355
|
+
detail: `eviction group pulls in already-shipped member(s) ${shipped.join(',')} — dissolving instead of reverting merged work`,
|
|
356
|
+
}), now);
|
|
357
|
+
const dissolve = dissolveBatch(state, batchId, { strategy: 'full', reason: 'evicts-shipped-member' }, deps);
|
|
358
|
+
return {
|
|
359
|
+
state: dissolve.state,
|
|
360
|
+
evicted: [],
|
|
361
|
+
requeued: dissolve.requeued,
|
|
362
|
+
reverted: [],
|
|
363
|
+
conflict: true,
|
|
364
|
+
dissolved: true,
|
|
365
|
+
dissolve,
|
|
366
|
+
suite: null,
|
|
367
|
+
};
|
|
368
|
+
}
|
|
369
|
+
let next = state;
|
|
370
|
+
if (batch.status !== 'evicting') {
|
|
371
|
+
next = (0, state_1.transitionBatch)(next, batchId, 'evicting', {}, now);
|
|
372
|
+
}
|
|
373
|
+
const ordered = orderRevertCommits(input.ranges, targets);
|
|
374
|
+
// Validate the WHOLE plan before touching the repo: rejecting a malformed
|
|
375
|
+
// sha mid-loop would leave the earlier reverts committed.
|
|
376
|
+
const malformed = ordered.find((sha) => !attribution_1.SHA_RE.test(sha));
|
|
377
|
+
if (malformed !== undefined) {
|
|
378
|
+
return revertConflict(next, batchId, targets, deps, now, [], `invalid commit sha ${malformed}`);
|
|
379
|
+
}
|
|
380
|
+
const revertedByMember = new Map();
|
|
381
|
+
for (const range of input.ranges) {
|
|
382
|
+
if (targets.includes(range.issue))
|
|
383
|
+
revertedByMember.set(range.issue, []);
|
|
384
|
+
}
|
|
385
|
+
const reverted = [];
|
|
386
|
+
// Reverts land as commits before any state is persisted, so a crash between
|
|
387
|
+
// the two would re-revert on the next run — and a double revert re-applies
|
|
388
|
+
// the broken change. Skip what the branch already carries.
|
|
389
|
+
const history = deps.exec('git', ['log', '--format=%B', '-n', '500'], deps.repoDir) ?? '';
|
|
390
|
+
for (const sha of ordered) {
|
|
391
|
+
const owner = input.ranges.find((r) => r.commits.includes(sha));
|
|
392
|
+
const record = () => {
|
|
393
|
+
reverted.push(sha);
|
|
394
|
+
if (owner) {
|
|
395
|
+
revertedByMember.set(owner.issue, [...(revertedByMember.get(owner.issue) ?? []), sha]);
|
|
396
|
+
}
|
|
397
|
+
};
|
|
398
|
+
if (history.includes(`This reverts commit ${sha}`)) {
|
|
399
|
+
record();
|
|
400
|
+
continue;
|
|
401
|
+
}
|
|
402
|
+
if (git(deps, ['revert', '--no-edit', sha], batchId, now) === null) {
|
|
403
|
+
// Leave no half-applied revert behind: abort, then `--quit` plus a hard
|
|
404
|
+
// reset as the fallback for git versions that refuse the abort outside a
|
|
405
|
+
// sequence (`--quit` alone KEEPS the conflicted index).
|
|
406
|
+
let clean = git(deps, ['revert', '--abort'], batchId, now) !== null;
|
|
407
|
+
if (!clean) {
|
|
408
|
+
git(deps, ['revert', '--quit'], batchId, now);
|
|
409
|
+
clean = git(deps, ['reset', '--hard', 'HEAD'], batchId, now) !== null;
|
|
410
|
+
}
|
|
411
|
+
return revertConflict(next, batchId, targets, deps, now, reverted, `git revert ${sha} conflicted${clean ? '' : ' — and the cleanup failed; this checkout is left mid-revert and must not be reused'}`);
|
|
412
|
+
}
|
|
413
|
+
record();
|
|
414
|
+
}
|
|
415
|
+
// Requeue every reverted member with its evidence attached (AC2).
|
|
416
|
+
const requeued = [];
|
|
417
|
+
const records = [];
|
|
418
|
+
for (const issue of targets) {
|
|
419
|
+
const tests = input.failingByMember?.get(issue) ?? [];
|
|
420
|
+
const memberCommits = revertedByMember.get(issue) ?? [];
|
|
421
|
+
const evidence = {
|
|
422
|
+
batch: batchId,
|
|
423
|
+
reason: input.reason,
|
|
424
|
+
failing_tests: tests.map((t) => t.id),
|
|
425
|
+
attribution: input.attribution,
|
|
426
|
+
reverted_commits: memberCommits,
|
|
427
|
+
at: now.toISOString(),
|
|
428
|
+
};
|
|
429
|
+
const result = (0, state_1.requeueMember)(next, issue, { mode: 'full', batch: null }, input.reason, now, {
|
|
430
|
+
failure_evidence: evidence,
|
|
431
|
+
});
|
|
432
|
+
next = result.state;
|
|
433
|
+
if (result.requeued)
|
|
434
|
+
requeued.push(issue);
|
|
435
|
+
records.push({
|
|
436
|
+
issue,
|
|
437
|
+
reason: input.reason,
|
|
438
|
+
attribution: input.attribution,
|
|
439
|
+
reverted_commits: memberCommits,
|
|
440
|
+
group: targets.filter((t) => t !== issue),
|
|
441
|
+
at: now.toISOString(),
|
|
442
|
+
});
|
|
443
|
+
journal(deps, (0, journal_1.unitEvent)('member-evicted', `batch:${batchId}`, {
|
|
444
|
+
issue,
|
|
445
|
+
// A member with no commits is NOT a clean eviction: its work is still
|
|
446
|
+
// on the branch while the queue says it was evicted, so the batch would
|
|
447
|
+
// ship code it believes it removed. `reverted=0` alone reads as "an
|
|
448
|
+
// empty range", which is why this says it in words.
|
|
449
|
+
detail: memberCommits.length === 0
|
|
450
|
+
? `${input.reason} reverted=0 — NO commits found for this member on the batch branch; its work was NOT reverted`
|
|
451
|
+
: `${input.reason} reverted=${memberCommits.length}`,
|
|
452
|
+
}), now);
|
|
453
|
+
}
|
|
454
|
+
next = (0, state_1.patchBatch)(next, batchId, { evictions: [...batch.evictions, ...records] }, now);
|
|
455
|
+
const withoutCommits = targets.filter((t) => (revertedByMember.get(t) ?? []).length === 0);
|
|
456
|
+
post(deps, batchOrThrow(next, batchId), {
|
|
457
|
+
phase: 'batch-validate',
|
|
458
|
+
status: 'blocked',
|
|
459
|
+
kv: {
|
|
460
|
+
reason: input.reason,
|
|
461
|
+
evicted: targets.join(',') || 'none',
|
|
462
|
+
requeued: requeued.join(',') || 'none',
|
|
463
|
+
reverted: String(reverted.length),
|
|
464
|
+
attribution: input.attribution,
|
|
465
|
+
...(withoutCommits.length > 0 ? { no_commits: withoutCommits.join(',') } : {}),
|
|
466
|
+
},
|
|
467
|
+
}, now);
|
|
468
|
+
// Back to validation with the suite re-run (AC2: "re-run the suite").
|
|
469
|
+
const suite = runSuite(deps, batchId, now);
|
|
470
|
+
next = (0, state_1.transitionBatch)(next, batchId, 'validating', {}, now);
|
|
471
|
+
if (checkDissolveTrigger(batchOrThrow(next, batchId))) {
|
|
472
|
+
const dissolve = dissolveBatch(next, batchId, { strategy: 'full', reason: 'eviction-threshold' }, deps);
|
|
473
|
+
return {
|
|
474
|
+
state: dissolve.state,
|
|
475
|
+
evicted: targets,
|
|
476
|
+
// The surviving members the dissolve requeued belong here too — a caller
|
|
477
|
+
// reading `requeued` must see everything that went back on the queue.
|
|
478
|
+
requeued: [...new Set([...requeued, ...dissolve.requeued])].sort((a, b) => a - b),
|
|
479
|
+
reverted,
|
|
480
|
+
conflict: false,
|
|
481
|
+
dissolved: true,
|
|
482
|
+
dissolve,
|
|
483
|
+
suite,
|
|
484
|
+
};
|
|
485
|
+
}
|
|
486
|
+
return {
|
|
487
|
+
state: next,
|
|
488
|
+
evicted: targets,
|
|
489
|
+
requeued,
|
|
490
|
+
reverted,
|
|
491
|
+
conflict: false,
|
|
492
|
+
dissolved: false,
|
|
493
|
+
suite,
|
|
494
|
+
};
|
|
495
|
+
}
|
|
496
|
+
/**
|
|
497
|
+
* A conflicting (or unusable) revert: abandon the batch rather than continue on
|
|
498
|
+
* a partly-reverted branch (AC3).
|
|
499
|
+
*
|
|
500
|
+
* The reverts that already succeeded stay committed — they are named in the
|
|
501
|
+
* journal line because they ride along on the abandoned branch, which is
|
|
502
|
+
* exactly why the branch is abandoned rather than reused.
|
|
503
|
+
*/
|
|
504
|
+
function revertConflict(state, batchId, targets, deps, now, reverted, detail) {
|
|
505
|
+
journal(deps, (0, journal_1.unitEvent)('revert-conflict', `batch:${batchId}`, {
|
|
506
|
+
detail: `${detail}; already applied and left on the abandoned branch: ${reverted.join(',') || 'none'}`,
|
|
507
|
+
}), now);
|
|
508
|
+
const dissolve = dissolveBatch(state, batchId, { strategy: 'full', reason: 'revert-conflict' }, deps);
|
|
509
|
+
return {
|
|
510
|
+
state: dissolve.state,
|
|
511
|
+
evicted: [...targets],
|
|
512
|
+
requeued: dissolve.requeued,
|
|
513
|
+
reverted,
|
|
514
|
+
conflict: true,
|
|
515
|
+
dissolved: true,
|
|
516
|
+
dissolve,
|
|
517
|
+
suite: null,
|
|
518
|
+
};
|
|
519
|
+
}
|
|
520
|
+
/**
|
|
521
|
+
* Whether the batch has lost STRICTLY more than a third of its members
|
|
522
|
+
* (RFC-0001 §F.8). Counted over distinct evicted members, so a member evicted
|
|
523
|
+
* once and recorded twice never inflates the trigger.
|
|
524
|
+
*/
|
|
525
|
+
function checkDissolveTrigger(batch) {
|
|
526
|
+
if (batch.members.length === 0)
|
|
527
|
+
return false;
|
|
528
|
+
const evicted = new Set(batch.evictions.map((e) => e.issue)).size;
|
|
529
|
+
return evicted > batch.members.length * types_1.DISSOLVE_EVICTION_FRACTION;
|
|
530
|
+
}
|
|
531
|
+
/**
|
|
532
|
+
* Dissolve a batch (AC3): mark it `dissolved`, requeue every unshipped member,
|
|
533
|
+
* and report what was preserved. Shipped/terminal members are never touched —
|
|
534
|
+
* "nothing green is discarded" is the whole point of dissolving rather than
|
|
535
|
+
* failing the batch.
|
|
536
|
+
*
|
|
537
|
+
* No git runs here: the batch branch is simply left behind unmerged. Sched
|
|
538
|
+
* deletes nothing, so an operator can still inspect what the batch built.
|
|
539
|
+
*/
|
|
540
|
+
function dissolveBatch(state, batchId, opts, deps) {
|
|
541
|
+
const now = clock(deps);
|
|
542
|
+
const batch = batchOrThrow(state, batchId);
|
|
543
|
+
if (types_1.TERMINAL_BATCH_STATUSES.has(batch.status)) {
|
|
544
|
+
// The batch WAS found — this is an illegal edge, not a missing id, and a
|
|
545
|
+
// caller catching SchedNotFoundError to mean "unknown batch" would misroute it.
|
|
546
|
+
throw new types_1.IllegalTransitionError('batch', batch.status, 'dissolving');
|
|
547
|
+
}
|
|
548
|
+
const unshipped = [];
|
|
549
|
+
const preserved = [];
|
|
550
|
+
for (const issue of batch.members) {
|
|
551
|
+
const entry = state.entries.find((e) => e.issue === issue);
|
|
552
|
+
if (!entry)
|
|
553
|
+
continue;
|
|
554
|
+
if ((0, state_1.isPreservedMember)(entry)) {
|
|
555
|
+
preserved.push(issue);
|
|
556
|
+
}
|
|
557
|
+
else {
|
|
558
|
+
unshipped.push(issue);
|
|
559
|
+
}
|
|
560
|
+
}
|
|
561
|
+
// Why the member is back on the queue, carried on the entry itself — a
|
|
562
|
+
// dissolve requeue is otherwise indistinguishable from a fresh enqueue.
|
|
563
|
+
const evidence = {
|
|
564
|
+
batch: batchId,
|
|
565
|
+
reason: opts.reason,
|
|
566
|
+
failing_tests: [],
|
|
567
|
+
attribution: 'none',
|
|
568
|
+
reverted_commits: [],
|
|
569
|
+
at: now.toISOString(),
|
|
570
|
+
};
|
|
571
|
+
let next = (0, state_1.transitionBatch)(state, batchId, 'dissolving', {}, now);
|
|
572
|
+
const newBatches = [];
|
|
573
|
+
const requeued = [];
|
|
574
|
+
if (opts.strategy === 'halved' && unshipped.length > 0) {
|
|
575
|
+
// Split by POSITION, not by coupling: the halves are the first and second
|
|
576
|
+
// half of `unshipped` in member order. A coupled eviction group straddling
|
|
577
|
+
// the pivot is therefore broken up — accepted, because both halves re-run
|
|
578
|
+
// through the same validate/evict rails, where a member that cannot stand
|
|
579
|
+
// alone is evicted rather than silently shipped.
|
|
580
|
+
const pivot = Math.ceil(unshipped.length / 2);
|
|
581
|
+
const taken = new Set(next.batches.map((b) => b.id));
|
|
582
|
+
// A colliding id would produce two batches answering to one name:
|
|
583
|
+
// `findBatch` would return the first and `validateState` would refuse to
|
|
584
|
+
// load the file at all on the next start.
|
|
585
|
+
const freeId = (base) => {
|
|
586
|
+
let id = base;
|
|
587
|
+
let n = 2;
|
|
588
|
+
while (taken.has(id))
|
|
589
|
+
id = `${base}${n++}`;
|
|
590
|
+
taken.add(id);
|
|
591
|
+
return id;
|
|
592
|
+
};
|
|
593
|
+
const halves = [
|
|
594
|
+
{ id: freeId(`${batchId}-a`), members: unshipped.slice(0, pivot) },
|
|
595
|
+
{ id: freeId(`${batchId}-b`), members: unshipped.slice(pivot) },
|
|
596
|
+
].filter((h) => h.members.length > 0);
|
|
597
|
+
for (const half of halves) {
|
|
598
|
+
next = {
|
|
599
|
+
...next,
|
|
600
|
+
batches: [
|
|
601
|
+
...next.batches,
|
|
602
|
+
(0, state_1.createBatch)(half.id, half.members, now, {
|
|
603
|
+
base_branch: batch.base_branch,
|
|
604
|
+
anchor: batch.anchor ?? undefined,
|
|
605
|
+
run_id: batch.run_id ?? undefined,
|
|
606
|
+
// Groups survive the split, restricted to the members that landed
|
|
607
|
+
// in this half — a group spanning both halves is no longer a group.
|
|
608
|
+
eviction_groups: batch.eviction_groups
|
|
609
|
+
.map((group) => group.filter((m) => half.members.includes(m)))
|
|
610
|
+
.filter((group) => group.length > 1),
|
|
611
|
+
}),
|
|
612
|
+
],
|
|
613
|
+
};
|
|
614
|
+
newBatches.push(half.id);
|
|
615
|
+
for (const issue of half.members) {
|
|
616
|
+
const result = (0, state_1.requeueMember)(next, issue, { mode: 'slot', batch: half.id }, opts.reason, now, { failure_evidence: { ...evidence, batch: half.id } });
|
|
617
|
+
next = result.state;
|
|
618
|
+
if (result.requeued)
|
|
619
|
+
requeued.push(issue);
|
|
620
|
+
}
|
|
621
|
+
}
|
|
622
|
+
journal(deps, (0, journal_1.unitEvent)('batch-split', `batch:${batchId}`, { detail: newBatches.join(',') }), now);
|
|
623
|
+
}
|
|
624
|
+
else {
|
|
625
|
+
for (const issue of unshipped) {
|
|
626
|
+
const result = (0, state_1.requeueMember)(next, issue, { mode: 'full', batch: null }, opts.reason, now, {
|
|
627
|
+
failure_evidence: evidence,
|
|
628
|
+
});
|
|
629
|
+
next = result.state;
|
|
630
|
+
if (result.requeued)
|
|
631
|
+
requeued.push(issue);
|
|
632
|
+
}
|
|
633
|
+
}
|
|
634
|
+
next = (0, state_1.transitionBatch)(next, batchId, 'dissolved', {}, now);
|
|
635
|
+
journal(deps, (0, journal_1.unitEvent)('batch-dissolved', `batch:${batchId}`, {
|
|
636
|
+
// Ids, not counts: "which members went back on the queue, and which kept
|
|
637
|
+
// their result?" is the question a dissolve has to answer afterwards.
|
|
638
|
+
detail: `${opts.reason} strategy=${opts.strategy} requeued=${requeued.join(',') || 'none'} ` +
|
|
639
|
+
`preserved=${preserved.join(',') || 'none'}` +
|
|
640
|
+
(newBatches.length > 0 ? ` split_into=${newBatches.join(',')}` : ''),
|
|
641
|
+
}), now);
|
|
642
|
+
post(deps, batchOrThrow(next, batchId), {
|
|
643
|
+
phase: opts.milestonePhase ?? 'batch-validate',
|
|
644
|
+
status: 'blocked',
|
|
645
|
+
kv: {
|
|
646
|
+
reason: opts.reason,
|
|
647
|
+
dissolved: 'true',
|
|
648
|
+
strategy: opts.strategy,
|
|
649
|
+
requeued: requeued.join(',') || 'none',
|
|
650
|
+
preserved: preserved.join(',') || 'none',
|
|
651
|
+
...(newBatches.length > 0 ? { split_into: newBatches.join(',') } : {}),
|
|
652
|
+
},
|
|
653
|
+
}, now);
|
|
654
|
+
return { state: next, requeued, preserved, newBatches };
|
|
655
|
+
}
|
|
656
|
+
/**
|
|
657
|
+
* The batch PR came back CONFLICTING or `auto-merge-blocked` (AC4).
|
|
658
|
+
*
|
|
659
|
+
* First occurrence: rebase the batch branch onto the base, re-run the suite,
|
|
660
|
+
* and re-ship once. Second occurrence — or a rebase that conflicts, or a suite
|
|
661
|
+
* that is red after a clean rebase — dissolves the batch into two half-batches:
|
|
662
|
+
* the work is kept, the batch that could not land is not retried a third time.
|
|
663
|
+
*/
|
|
664
|
+
function handlePrConflict(state, batchId, deps, opts = {}) {
|
|
665
|
+
const now = clock(deps);
|
|
666
|
+
const batch = batchOrThrow(state, batchId);
|
|
667
|
+
const reason = opts.reason ?? 'pr-conflict';
|
|
668
|
+
// Both paths enter `rebasing` — it is the honest state for "the merge came
|
|
669
|
+
// back blocked" — and only the attempt count decides whether a rebase runs.
|
|
670
|
+
// Guarded, so a retry against a batch already in `rebasing` is idempotent
|
|
671
|
+
// rather than an IllegalTransitionError out of a recovery call.
|
|
672
|
+
let next = state;
|
|
673
|
+
if (batch.status !== 'rebasing') {
|
|
674
|
+
next = (0, state_1.transitionBatch)(next, batchId, 'rebasing', {}, now);
|
|
675
|
+
}
|
|
676
|
+
/** Every give-up path here ends the same way: halve the batch, keep the work. */
|
|
677
|
+
const bailToHalves = (from, bailReason, rebased = false, suite = null) => {
|
|
678
|
+
const dissolve = dissolveBatch(from, batchId, { strategy: 'halved', reason: bailReason, milestonePhase: 'batch-ship' }, deps);
|
|
679
|
+
return { state: dissolve.state, action: 'dissolved', rebased, suite, dissolve };
|
|
680
|
+
};
|
|
681
|
+
if (batch.rebase_attempts >= types_1.MAX_REBASE_ATTEMPTS) {
|
|
682
|
+
return bailToHalves(next, `${reason}-recurred`);
|
|
683
|
+
}
|
|
684
|
+
next = (0, state_1.patchBatch)(next, batchId, { rebase_attempts: batch.rebase_attempts + 1 }, now);
|
|
685
|
+
const base = batch.base_branch;
|
|
686
|
+
if (!attribution_1.SAFE_REF_RE.test(base)) {
|
|
687
|
+
return bailToHalves(next, 'invalid-base-branch');
|
|
688
|
+
}
|
|
689
|
+
// Rebase what the batch actually owns. `git rebase` acts on whatever HEAD
|
|
690
|
+
// happens to be, so a checkout left on another branch (or detached by an
|
|
691
|
+
// earlier step) would otherwise rewrite the wrong ref and "re-ship" something
|
|
692
|
+
// that is not this batch.
|
|
693
|
+
if (batch.branch !== null) {
|
|
694
|
+
const head = git(deps, ['symbolic-ref', '--quiet', '--short', 'HEAD'], batchId, now);
|
|
695
|
+
if (head !== batch.branch) {
|
|
696
|
+
journal(deps, (0, journal_1.unitEvent)('git-failed', `batch:${batchId}`, {
|
|
697
|
+
detail: `checkout is on '${head ?? 'detached HEAD'}', expected the batch branch '${batch.branch}'`,
|
|
698
|
+
}), now);
|
|
699
|
+
return bailToHalves(next, 'wrong-branch-checked-out');
|
|
700
|
+
}
|
|
701
|
+
}
|
|
702
|
+
// A failed fetch is not a conflict: rebasing onto a stale `origin/<base>`
|
|
703
|
+
// silently produces a batch that will conflict again at merge time, and
|
|
704
|
+
// reporting it as `rebase-conflict` sends the operator after the wrong cause.
|
|
705
|
+
if (git(deps, ['fetch', 'origin', '--', base], batchId, now) === null) {
|
|
706
|
+
return bailToHalves(next, 'fetch-failed');
|
|
707
|
+
}
|
|
708
|
+
if (git(deps, ['rebase', `origin/${base}`], batchId, now) === null) {
|
|
709
|
+
// Never leave the worktree mid-rebase.
|
|
710
|
+
git(deps, ['rebase', '--abort'], batchId, now);
|
|
711
|
+
return bailToHalves(next, 'rebase-conflict');
|
|
712
|
+
}
|
|
713
|
+
journal(deps, (0, journal_1.unitEvent)('batch-rebased', `batch:${batchId}`, { detail: base }), now);
|
|
714
|
+
next = (0, state_1.transitionBatch)(next, batchId, 're-validating', {}, now);
|
|
715
|
+
const suite = runSuite(deps, batchId, now);
|
|
716
|
+
if (suite !== null && !suite.ok) {
|
|
717
|
+
return bailToHalves(next, 'rebase-suite-red', true, suite);
|
|
718
|
+
}
|
|
719
|
+
next = (0, state_1.transitionBatch)(next, batchId, 'shipping', {}, now);
|
|
720
|
+
post(deps, batchOrThrow(next, batchId), {
|
|
721
|
+
phase: 'batch-ship',
|
|
722
|
+
// NOT `blocked`: the rebase worked and the batch is re-shipping. `blocked`
|
|
723
|
+
// stamps `next=done` on a run that is still going, and counts toward the
|
|
724
|
+
// runstate resume-loop cap.
|
|
725
|
+
status: 'awaiting-merge',
|
|
726
|
+
kv: { reason, rebased: base, rebase_attempts: String(batch.rebase_attempts + 1) },
|
|
727
|
+
}, now);
|
|
728
|
+
return { state: next, action: 'reship', rebased: true, suite };
|
|
729
|
+
}
|
|
730
|
+
//# sourceMappingURL=recovery.js.map
|