@indigoai-us/hq-cli 5.149.1 → 5.151.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +10 -0
- package/dist/command-catalog.generated.d.ts +19 -0
- package/dist/command-catalog.generated.js +25 -0
- package/dist/commands/lanes.js +495 -246
- package/dist/commands/report.d.ts +15 -1
- package/dist/commands/report.js +41 -6
- package/dist/lib/lanes/admission.d.ts +24 -0
- package/dist/lib/lanes/admission.js +193 -0
- package/dist/lib/lanes/delivery.js +5 -3
- package/dist/lib/lanes/index.d.ts +2 -0
- package/dist/lib/lanes/index.js +1 -0
- package/dist/lib/lanes/reconcile.js +155 -43
- package/dist/lib/lanes/spawn.js +4 -0
- package/dist/lib/lanes/store.js +48 -1
- package/dist/lib/lanes/types.d.ts +8 -1
- package/dist/lib/lanes/types.js +15 -4
- package/package.json +1 -1
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { Command } from "commander";
|
|
2
|
-
import { buildReportTarball, deployOrgContext, type ChannelReport, type DeployAccessPolicy, type ReportDeployment, type ReportPatch } from "../lib/report-publish.js";
|
|
2
|
+
import { buildReportTarball, deployOrgContext, type ChannelReport, type DeployAccessPolicy, type ReportDeployment, type ReportPatch, ReportPublishError } from "../lib/report-publish.js";
|
|
3
3
|
type ChannelRow = {
|
|
4
4
|
channelId?: string;
|
|
5
5
|
scope?: string;
|
|
@@ -17,7 +17,21 @@ export declare function withChannelCompany(report: ChannelReport, channels: Chan
|
|
|
17
17
|
/** Explicit context has priority; discovery is the authenticated fallback for normal CLI use. */
|
|
18
18
|
export declare function resolveReport(token: string, reportId: string, requestedChannelId?: string): Promise<ChannelReport>;
|
|
19
19
|
export declare function patchReport(token: string, report: ChannelReport, patch: ReportPatchInput): Promise<ChannelReport>;
|
|
20
|
+
/** hq-deploy refused the subdomain: another app, not visible to this caller, holds it. */
|
|
21
|
+
export declare class SubdomainTakenError extends ReportPublishError {
|
|
22
|
+
constructor();
|
|
23
|
+
}
|
|
24
|
+
/**
|
|
25
|
+
* Subdomains to try, in order. The first is the report's own slug; the rest
|
|
26
|
+
* are used only when an app the caller cannot see already holds it (for
|
|
27
|
+
* example, one left by an earlier publish under a different account).
|
|
28
|
+
*/
|
|
29
|
+
export declare function candidateSlugs(slug: string, attempts?: number): string[];
|
|
20
30
|
export declare function resolveCompanySlug(token: string, companyUid: string): Promise<string>;
|
|
31
|
+
export declare function findOrCreateApp(token: string, slug: string, org: ReturnType<typeof deployOrgContext>): Promise<{
|
|
32
|
+
id: string;
|
|
33
|
+
subdomain: string;
|
|
34
|
+
}>;
|
|
21
35
|
export declare function deployReportStatic(token: string, input: {
|
|
22
36
|
slug: string;
|
|
23
37
|
files: Parameters<typeof buildReportTarball>[0];
|
package/dist/commands/report.js
CHANGED
|
@@ -232,10 +232,28 @@ async function deployJson(token, route, method, body, extraHeaders = {}) {
|
|
|
232
232
|
catch {
|
|
233
233
|
throw new ReportPublishError("Could not reach hq-deploy.");
|
|
234
234
|
}
|
|
235
|
-
if (!res.ok)
|
|
235
|
+
if (!res.ok) {
|
|
236
|
+
const failure = await res.json().catch(() => ({}));
|
|
237
|
+
if (failure?.error?.code === "SUBDOMAIN_DUPLICATE")
|
|
238
|
+
throw new SubdomainTakenError();
|
|
236
239
|
throw new ReportPublishError("hq-deploy rejected the report deployment.");
|
|
240
|
+
}
|
|
237
241
|
return await res.json();
|
|
238
242
|
}
|
|
243
|
+
/** hq-deploy refused the subdomain: another app, not visible to this caller, holds it. */
|
|
244
|
+
export class SubdomainTakenError extends ReportPublishError {
|
|
245
|
+
constructor() {
|
|
246
|
+
super("hq-deploy rejected the report deployment.");
|
|
247
|
+
}
|
|
248
|
+
}
|
|
249
|
+
/**
|
|
250
|
+
* Subdomains to try, in order. The first is the report's own slug; the rest
|
|
251
|
+
* are used only when an app the caller cannot see already holds it (for
|
|
252
|
+
* example, one left by an earlier publish under a different account).
|
|
253
|
+
*/
|
|
254
|
+
export function candidateSlugs(slug, attempts = 3) {
|
|
255
|
+
return [slug, ...Array.from({ length: attempts }, (_, i) => `${slug}-${i + 2}`)];
|
|
256
|
+
}
|
|
239
257
|
export async function resolveCompanySlug(token, companyUid) {
|
|
240
258
|
const res = await vaultApiFetch({ token, path: `/entity/${encodeURIComponent(companyUid)}` });
|
|
241
259
|
if (!res.ok)
|
|
@@ -250,12 +268,29 @@ async function orgFor(token, report) {
|
|
|
250
268
|
const slug = report.companyUid ? await resolveCompanySlug(token, report.companyUid) : undefined;
|
|
251
269
|
return deployOrgContext(report, slug);
|
|
252
270
|
}
|
|
253
|
-
async function findOrCreateApp(token, slug, org) {
|
|
271
|
+
export async function findOrCreateApp(token, slug, org) {
|
|
254
272
|
const listed = await deployJson(token, "/api/apps", "GET", undefined, org.headers);
|
|
255
|
-
const
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
const
|
|
273
|
+
const apps = Array.isArray(listed.apps) ? listed.apps : [];
|
|
274
|
+
const mine = (name) => apps.find((app) => !!app && typeof app === "object" && (app.subdomain === name || app.name === name));
|
|
275
|
+
let app;
|
|
276
|
+
for (const name of candidateSlugs(slug)) {
|
|
277
|
+
const existing = mine(name);
|
|
278
|
+
if (existing) {
|
|
279
|
+
app = existing;
|
|
280
|
+
break;
|
|
281
|
+
}
|
|
282
|
+
try {
|
|
283
|
+
app = await deployJson(token, "/api/apps", "POST", { name, type: "static" }, org.headers);
|
|
284
|
+
break;
|
|
285
|
+
}
|
|
286
|
+
catch (error) {
|
|
287
|
+
if (error instanceof SubdomainTakenError)
|
|
288
|
+
continue;
|
|
289
|
+
throw error;
|
|
290
|
+
}
|
|
291
|
+
}
|
|
292
|
+
if (!app)
|
|
293
|
+
throw new ReportPublishError("Every address for this report is taken on hq-deploy.");
|
|
259
294
|
const id = typeof app.id === "string" ? app.id : undefined;
|
|
260
295
|
const subdomain = typeof app.subdomain === "string" ? app.subdomain : undefined;
|
|
261
296
|
if (!id || !subdomain)
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
import type { LaneRecord, LanesConfig } from "./types.js";
|
|
2
|
+
export interface AdmissionHost {
|
|
3
|
+
hostname: string;
|
|
4
|
+
outpostId: string | null;
|
|
5
|
+
}
|
|
6
|
+
export type AdmissionCapReason = "provider_cap" | "account_cap" | "profile_cap";
|
|
7
|
+
export declare const ADMISSION_LOCK_STALE_MS = 30000;
|
|
8
|
+
export declare const ADMISSION_LOCK_TIMEOUT_MS = 5000;
|
|
9
|
+
export declare const ADMISSION_RESERVATION_GRACE_MS = 30000;
|
|
10
|
+
export declare function withLaneAdmissionLock<T>(hqRoot: string, provider: LaneRecord["provider"], fn: () => Promise<T>, sleep?: (ms: number) => Promise<void>): Promise<T>;
|
|
11
|
+
export declare function withAllLaneAdmissionLocks<T>(hqRoot: string, fn: () => Promise<T>, sleep?: (ms: number) => Promise<void>): Promise<T>;
|
|
12
|
+
export declare function laneRunsOnHost(lane: LaneRecord, host: AdmissionHost): boolean;
|
|
13
|
+
export declare function compareLaneQueueOrder(a: LaneRecord, b: LaneRecord): number;
|
|
14
|
+
export declare function hasActiveAdmissionReservation(lane: LaneRecord, nowMs: number): boolean;
|
|
15
|
+
export declare function hasActivePendingResume(lane: LaneRecord, nowMs: number): boolean;
|
|
16
|
+
/** Return the first capacity limit that currently prevents this lane from starting. */
|
|
17
|
+
export declare function laneAdmissionCapReason(lanes: LaneRecord[], candidate: LaneRecord, config: LanesConfig, host: AdmissionHost, profileMaxConcurrent?: number, nowMs?: number): AdmissionCapReason | undefined;
|
|
18
|
+
/**
|
|
19
|
+
* A prior queue entry owns a free slot only when its own provider, account,
|
|
20
|
+
* and profile caps allow it to use that slot. A blocked profile therefore
|
|
21
|
+
* cannot hold an unrelated profile's queue position.
|
|
22
|
+
*/
|
|
23
|
+
export declare function hasEarlierEligibleQueuedLane(lanes: LaneRecord[], candidate: LaneRecord, config: LanesConfig, host: AdmissionHost, profileMaxFor: (lane: LaneRecord) => number | undefined, nowMs?: number): boolean;
|
|
24
|
+
//# sourceMappingURL=admission.d.ts.map
|
|
@@ -0,0 +1,193 @@
|
|
|
1
|
+
import * as fs from "node:fs";
|
|
2
|
+
import * as path from "node:path";
|
|
3
|
+
import { accountMaxRunning } from "./config.js";
|
|
4
|
+
import { LanesError } from "./errors.js";
|
|
5
|
+
export const ADMISSION_LOCK_STALE_MS = 30_000;
|
|
6
|
+
export const ADMISSION_LOCK_TIMEOUT_MS = 5_000;
|
|
7
|
+
export const ADMISSION_RESERVATION_GRACE_MS = 30_000;
|
|
8
|
+
function defaultAdmissionSleep(ms) {
|
|
9
|
+
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
10
|
+
}
|
|
11
|
+
function admissionLockPath(hqRoot, provider) {
|
|
12
|
+
return path.join(hqRoot, "workspace", "lanes-runs", ".admission-locks", `${provider}-provider.lock`);
|
|
13
|
+
}
|
|
14
|
+
function admissionLockOwnerAlive(pid) {
|
|
15
|
+
try {
|
|
16
|
+
process.kill(pid, 0);
|
|
17
|
+
return true;
|
|
18
|
+
}
|
|
19
|
+
catch (err) {
|
|
20
|
+
return err.code !== "ESRCH";
|
|
21
|
+
}
|
|
22
|
+
}
|
|
23
|
+
async function withAdmissionLock(lockPath, fn, sleep) {
|
|
24
|
+
fs.mkdirSync(path.dirname(lockPath), { recursive: true });
|
|
25
|
+
const started = Date.now();
|
|
26
|
+
while (true) {
|
|
27
|
+
let fd;
|
|
28
|
+
try {
|
|
29
|
+
fd = fs.openSync(lockPath, "wx");
|
|
30
|
+
const ownedStat = fs.fstatSync(fd);
|
|
31
|
+
try {
|
|
32
|
+
fs.writeSync(fd, Buffer.from(`${process.pid}\n${Date.now()}\n`, "utf8"));
|
|
33
|
+
}
|
|
34
|
+
catch (err) {
|
|
35
|
+
fs.closeSync(fd);
|
|
36
|
+
fd = undefined;
|
|
37
|
+
throw err;
|
|
38
|
+
}
|
|
39
|
+
try {
|
|
40
|
+
await new Promise((resolve) => setImmediate(resolve));
|
|
41
|
+
return await fn();
|
|
42
|
+
}
|
|
43
|
+
finally {
|
|
44
|
+
try {
|
|
45
|
+
fs.closeSync(fd);
|
|
46
|
+
}
|
|
47
|
+
catch (err) {
|
|
48
|
+
process.stderr.write(`hq lanes: admission lock close failed for ${lockPath}: ${err instanceof Error ? err.message : String(err)}\n`);
|
|
49
|
+
}
|
|
50
|
+
try {
|
|
51
|
+
const currentStat = fs.statSync(lockPath);
|
|
52
|
+
if (currentStat.dev === ownedStat.dev && currentStat.ino === ownedStat.ino) {
|
|
53
|
+
fs.unlinkSync(lockPath);
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
catch (err) {
|
|
57
|
+
if (err.code !== "ENOENT") {
|
|
58
|
+
process.stderr.write(`hq lanes: admission lock unlink failed for ${lockPath}: ${err instanceof Error ? err.message : String(err)}\n`);
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
catch (err) {
|
|
64
|
+
if (err.code !== "EEXIST")
|
|
65
|
+
throw err;
|
|
66
|
+
if (Date.now() - started > ADMISSION_LOCK_TIMEOUT_MS) {
|
|
67
|
+
throw new LanesError("store_lock_timeout", `Timed out waiting for admission lock ${lockPath}`, { path: lockPath });
|
|
68
|
+
}
|
|
69
|
+
try {
|
|
70
|
+
const raw = fs.readFileSync(lockPath, "utf8");
|
|
71
|
+
const [ownerLine, timestampLine] = raw.split("\n");
|
|
72
|
+
const ownerPid = ownerLine ? Number(ownerLine) : NaN;
|
|
73
|
+
const timestamp = timestampLine ? Number(timestampLine) : NaN;
|
|
74
|
+
if (Number.isSafeInteger(ownerPid) &&
|
|
75
|
+
ownerPid > 0 &&
|
|
76
|
+
Number.isFinite(timestamp) &&
|
|
77
|
+
Date.now() - timestamp > ADMISSION_LOCK_STALE_MS &&
|
|
78
|
+
!admissionLockOwnerAlive(ownerPid)) {
|
|
79
|
+
if (fs.readFileSync(lockPath, "utf8") !== raw)
|
|
80
|
+
continue;
|
|
81
|
+
try {
|
|
82
|
+
fs.unlinkSync(lockPath);
|
|
83
|
+
}
|
|
84
|
+
catch (unlinkErr) {
|
|
85
|
+
if (unlinkErr.code !== "ENOENT")
|
|
86
|
+
throw unlinkErr;
|
|
87
|
+
}
|
|
88
|
+
continue;
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
catch (probeErr) {
|
|
92
|
+
if (probeErr.code !== "ENOENT") {
|
|
93
|
+
process.stderr.write(`hq lanes: stale admission-lock probe failed for ${lockPath}: ${probeErr instanceof Error ? probeErr.message : String(probeErr)}\n`);
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
await sleep(10);
|
|
97
|
+
await new Promise((resolve) => setImmediate(resolve));
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
export function withLaneAdmissionLock(hqRoot, provider, fn, sleep = defaultAdmissionSleep) {
|
|
102
|
+
return withAdmissionLock(admissionLockPath(hqRoot, provider), fn, sleep);
|
|
103
|
+
}
|
|
104
|
+
export async function withAllLaneAdmissionLocks(hqRoot, fn, sleep = defaultAdmissionSleep) {
|
|
105
|
+
const providers = ["claude", "codex", "grok"];
|
|
106
|
+
const acquire = (index) => {
|
|
107
|
+
const provider = providers[index];
|
|
108
|
+
if (provider === undefined)
|
|
109
|
+
return fn();
|
|
110
|
+
return withLaneAdmissionLock(hqRoot, provider, () => acquire(index + 1), sleep);
|
|
111
|
+
};
|
|
112
|
+
return acquire(0);
|
|
113
|
+
}
|
|
114
|
+
export function laneRunsOnHost(lane, host) {
|
|
115
|
+
if (host.outpostId && lane.host.kind === "outpost") {
|
|
116
|
+
return lane.host.id === host.outpostId;
|
|
117
|
+
}
|
|
118
|
+
return lane.host.kind === "local" && lane.host.id === host.hostname;
|
|
119
|
+
}
|
|
120
|
+
export function compareLaneQueueOrder(a, b) {
|
|
121
|
+
if (b.priority !== a.priority)
|
|
122
|
+
return b.priority - a.priority;
|
|
123
|
+
const byQueueAge = Date.parse(a.queued_at ?? a.created_at) - Date.parse(b.queued_at ?? b.created_at);
|
|
124
|
+
return byQueueAge || a.lane_id.localeCompare(b.lane_id);
|
|
125
|
+
}
|
|
126
|
+
export function hasActiveAdmissionReservation(lane, nowMs) {
|
|
127
|
+
if (lane.state !== "queued" || lane.queued_reason !== undefined || !lane.admission_reserved_at) {
|
|
128
|
+
return false;
|
|
129
|
+
}
|
|
130
|
+
const reservedAt = Date.parse(lane.admission_reserved_at);
|
|
131
|
+
const ageMs = nowMs - reservedAt;
|
|
132
|
+
return Number.isFinite(reservedAt) && ageMs >= 0 && ageMs < ADMISSION_RESERVATION_GRACE_MS;
|
|
133
|
+
}
|
|
134
|
+
export function hasActivePendingResume(lane, nowMs) {
|
|
135
|
+
const pending = lane.pending_resume;
|
|
136
|
+
if (!pending)
|
|
137
|
+
return false;
|
|
138
|
+
const reservedAt = Date.parse(pending.at);
|
|
139
|
+
const ageMs = nowMs - reservedAt;
|
|
140
|
+
return Number.isFinite(reservedAt) && ageMs >= 0 && ageMs < ADMISSION_RESERVATION_GRACE_MS;
|
|
141
|
+
}
|
|
142
|
+
/** Return the first capacity limit that currently prevents this lane from starting. */
|
|
143
|
+
export function laneAdmissionCapReason(lanes, candidate, config, host, profileMaxConcurrent, nowMs = Date.now()) {
|
|
144
|
+
const running = lanes.filter((lane) => lane.lane_id !== candidate.lane_id &&
|
|
145
|
+
(lane.state === "running" ||
|
|
146
|
+
hasActiveAdmissionReservation(lane, nowMs) ||
|
|
147
|
+
hasActivePendingResume(lane, nowMs)) &&
|
|
148
|
+
laneRunsOnHost(lane, host));
|
|
149
|
+
const providerCfg = config.providers[candidate.provider];
|
|
150
|
+
const providerRunning = running.filter((lane) => lane.provider === candidate.provider);
|
|
151
|
+
if (providerRunning.length >= providerCfg.max_running)
|
|
152
|
+
return "provider_cap";
|
|
153
|
+
if (candidate.account &&
|
|
154
|
+
providerRunning.filter((lane) => lane.account === candidate.account).length >=
|
|
155
|
+
accountMaxRunning(providerCfg, candidate.account)) {
|
|
156
|
+
return "account_cap";
|
|
157
|
+
}
|
|
158
|
+
if (candidate.worker_profile &&
|
|
159
|
+
profileMaxConcurrent !== undefined &&
|
|
160
|
+
running.filter((lane) => lane.worker_profile === candidate.worker_profile).length >=
|
|
161
|
+
profileMaxConcurrent) {
|
|
162
|
+
return "profile_cap";
|
|
163
|
+
}
|
|
164
|
+
return undefined;
|
|
165
|
+
}
|
|
166
|
+
/**
|
|
167
|
+
* A prior queue entry owns a free slot only when its own provider, account,
|
|
168
|
+
* and profile caps allow it to use that slot. A blocked profile therefore
|
|
169
|
+
* cannot hold an unrelated profile's queue position.
|
|
170
|
+
*/
|
|
171
|
+
export function hasEarlierEligibleQueuedLane(lanes, candidate, config, host, profileMaxFor, nowMs = Date.now()) {
|
|
172
|
+
return lanes
|
|
173
|
+
.filter((lane) => lane.lane_id !== candidate.lane_id &&
|
|
174
|
+
lane.state === "queued" &&
|
|
175
|
+
lane.provider === candidate.provider &&
|
|
176
|
+
laneRunsOnHost(lane, host) &&
|
|
177
|
+
compareLaneQueueOrder(lane, candidate) < 0)
|
|
178
|
+
.some((lane) => {
|
|
179
|
+
const profileMaxConcurrent = profileMaxFor(lane);
|
|
180
|
+
const capReason = laneAdmissionCapReason(lanes, lane, config, host, profileMaxConcurrent, nowMs);
|
|
181
|
+
if (capReason === undefined)
|
|
182
|
+
return true;
|
|
183
|
+
if (capReason !== "account_cap" || lane.admission_accounts === undefined)
|
|
184
|
+
return false;
|
|
185
|
+
return lane.admission_accounts.some((account) => {
|
|
186
|
+
if (account === lane.account || config.providers[lane.provider].accounts[account]?.enabled !== true) {
|
|
187
|
+
return false;
|
|
188
|
+
}
|
|
189
|
+
return laneAdmissionCapReason(lanes, { ...lane, account }, config, host, profileMaxConcurrent, nowMs) === undefined;
|
|
190
|
+
});
|
|
191
|
+
});
|
|
192
|
+
}
|
|
193
|
+
//# sourceMappingURL=admission.js.map
|
|
@@ -280,9 +280,11 @@ export function resumeCommandRefusal(lane) {
|
|
|
280
280
|
error: "lane_not_resumable",
|
|
281
281
|
message: `Lane is ${lane.state}. hq lanes resume continues a lane in ${RESUME_COMMAND_STATES.join(", ")}; ${lane.state === "awaiting_input"
|
|
282
282
|
? "use hq lanes questions answer <question-id>."
|
|
283
|
-
: lane.state === "queued"
|
|
284
|
-
? "use hq lanes
|
|
285
|
-
:
|
|
283
|
+
: lane.state === "queued"
|
|
284
|
+
? "use hq lanes admit <lane-id>."
|
|
285
|
+
: lane.state === "running"
|
|
286
|
+
? "use hq lanes message for a live lane."
|
|
287
|
+
: "a lane in this state is not resumed."}`,
|
|
286
288
|
};
|
|
287
289
|
}
|
|
288
290
|
if (!providerSupportsSessionResume(lane.provider)) {
|
|
@@ -3,6 +3,8 @@
|
|
|
3
3
|
*/
|
|
4
4
|
export { AccountResolutionError, LaneNotFoundError, LaneSchemaError, LaneVersionConflict, LanesConfigError, LanesError, WorkerResolutionError, isLaneVersionConflict, isLanesError, } from "./errors.js";
|
|
5
5
|
export { MAX_LANE_DEPTH } from "./constants.js";
|
|
6
|
+
export { compareLaneQueueOrder, hasActiveAdmissionReservation, hasActivePendingResume, hasEarlierEligibleQueuedLane, laneAdmissionCapReason, laneRunsOnHost, withAllLaneAdmissionLocks, withLaneAdmissionLock, } from "./admission.js";
|
|
7
|
+
export type { AdmissionCapReason, AdmissionHost } from "./admission.js";
|
|
6
8
|
export { CONFIG_SCHEMA_VERSION, ENVELOPE_DECISIONS, ENVELOPE_SCHEMA_VERSION, HISTORY_CAP, HOST_KINDS, LANE_PROVIDERS, LANE_SCHEMA_VERSION, LANE_STORY_STATUSES, LANE_STATES, LIVENESS_STATES, QUESTION_BODY_STORES, QUESTION_SCHEMA_VERSION, QUESTION_STATUSES, QUEUED_REASONS, RAISED_VIA, ROUTING_REASONS, SENIOR_KINDS, STORY_STATUSES, TERMINAL_LANE_STATES, isEnvelopeDecision, isHostKind, isLaneProvider, isLaneState, isLivenessState, isQuestionStatus, isSeniorKind, isTerminalLaneState, parseLaneEnvelope, parseLaneProvider, parseLaneState, parseLivenessState, parseMajorSchema, parseQuestionStatus, parseSeniorKind, } from "./types.js";
|
|
7
9
|
export type { AccountConfig, AttributionHistoryEntry, EnvelopeDecision, EnvelopeQuestion, HostKind, LaneCompanyRef, LaneEnvelope, LaneHost, LaneProvider, LaneRecord, LaneStory, LaneStoryStatus, PendingResume, AccountHistoryEntry, LaneState, LanesConfig, LastEnvelopeRef, LivenessState, LivenessWindows, ProviderConfig, QuestionAnswer, QuestionBodyRef, QuestionRecord, QuestionStatus, QueuedReason, RaisedVia, RoutingHistoryEntry, RoutingReason, SeniorHistoryEntry, SeniorKind, SeniorRef, StoryStatus, TerminalLaneState, ThrottleConfig, } from "./types.js";
|
|
8
10
|
export { ACTIVE_LANE_STORY_STATUSES, activeLaneStories, laneStories, laneStory, storyKey, storyStatusForEnvelope, storyWarning, withCurrentStory, withLaneStories, } from "./stories.js";
|
package/dist/lib/lanes/index.js
CHANGED
|
@@ -3,6 +3,7 @@
|
|
|
3
3
|
*/
|
|
4
4
|
export { AccountResolutionError, LaneNotFoundError, LaneSchemaError, LaneVersionConflict, LanesConfigError, LanesError, WorkerResolutionError, isLaneVersionConflict, isLanesError, } from "./errors.js";
|
|
5
5
|
export { MAX_LANE_DEPTH } from "./constants.js";
|
|
6
|
+
export { compareLaneQueueOrder, hasActiveAdmissionReservation, hasActivePendingResume, hasEarlierEligibleQueuedLane, laneAdmissionCapReason, laneRunsOnHost, withAllLaneAdmissionLocks, withLaneAdmissionLock, } from "./admission.js";
|
|
6
7
|
export { CONFIG_SCHEMA_VERSION, ENVELOPE_DECISIONS, ENVELOPE_SCHEMA_VERSION, HISTORY_CAP, HOST_KINDS, LANE_PROVIDERS, LANE_SCHEMA_VERSION, LANE_STORY_STATUSES, LANE_STATES, LIVENESS_STATES, QUESTION_BODY_STORES, QUESTION_SCHEMA_VERSION, QUESTION_STATUSES, QUEUED_REASONS, RAISED_VIA, ROUTING_REASONS, SENIOR_KINDS, STORY_STATUSES, TERMINAL_LANE_STATES, isEnvelopeDecision, isHostKind, isLaneProvider, isLaneState, isLivenessState, isQuestionStatus, isSeniorKind, isTerminalLaneState, parseLaneEnvelope, parseLaneProvider, parseLaneState, parseLivenessState, parseMajorSchema, parseQuestionStatus, parseSeniorKind, } from "./types.js";
|
|
7
8
|
export { ACTIVE_LANE_STORY_STATUSES, activeLaneStories, laneStories, laneStory, storyKey, storyStatusForEnvelope, storyWarning, withCurrentStory, withLaneStories, } from "./stories.js";
|
|
8
9
|
export { DEFAULT_ACCOUNT_HOMES, DEFAULT_AUTO_COMPACT_TOKENS, DEFAULT_COMPACT_AFTER_MS, DEFAULT_CPU_BUSY_PAUSE, DEFAULT_CPU_BUSY_RESUME, DEFAULT_LIVE_WITHIN_MS, DEFAULT_LIVENESS_WINDOWS, DEFAULT_MAX_QUEUED, DEFAULT_MIN_AVAILABLE_MEM_MB, DEFAULT_PROVIDER_MAX_RUNNING, DEFAULT_SILENT_LOST_MS, DEFAULT_TURN_START_LOST_MS, LANES_CONFIG_REL, accountMaxRunning, canonicalizeHome, documentedDefaultConfig, expandHome, lanesConfigPath, loadLanesConfig, parseLanesConfig, providerCompactAfterMs, providerAutoCompactTokens, providerQuotaDefaultHoldMs, DEFAULT_QUOTA_HOLD_MS, writeLanesConfig, } from "./config.js";
|
|
@@ -8,14 +8,16 @@
|
|
|
8
8
|
import * as fs from "node:fs";
|
|
9
9
|
import * as path from "node:path";
|
|
10
10
|
import { parseUserYaml } from "../../utils/user-yaml-error.js";
|
|
11
|
-
import {
|
|
11
|
+
import { compareLaneQueueOrder, hasActiveAdmissionReservation, hasActivePendingResume, laneAdmissionCapReason, laneRunsOnHost, withAllLaneAdmissionLocks, } from "./admission.js";
|
|
12
|
+
import { accountStatus, isCredentialProven, resolveAccount } from "./accounts.js";
|
|
13
|
+
import { loadLanesConfig, providerQuotaDefaultHoldMs } from "./config.js";
|
|
12
14
|
import { MAX_LANE_DEPTH } from "./constants.js";
|
|
13
|
-
import { LanesError, isLaneVersionConflict } from "./errors.js";
|
|
15
|
+
import { AccountResolutionError, LanesError, isLaneVersionConflict } from "./errors.js";
|
|
14
16
|
import { loadWorkerProfile } from "./profile.js";
|
|
15
17
|
import { classifySessionLiveness, defaultLivenessDeps, loadLivenessIndex, readDaemonPresenceConnected, } from "./liveness.js";
|
|
16
18
|
import { ANSWERED_QUESTIONS_FILE, armLaneWatcher, isPidRecordAlive, laneRunDir, readPidRecord, readSpawnManifest, markSpawnFailure, spawnEnvelopeRepair, spawnLaneWorker, } from "./spawn.js";
|
|
17
19
|
import { appendCappedHistory, lanesStoreRoot, parseLaneRecord, parseQuestionRecord, readLane, readQuestion, updateLane, updateQuestion, } from "./store.js";
|
|
18
|
-
import { applyThrottleDecision, defaultThrottleDeps, emptyThrottleState, evaluateThrottle, isCpuCheckEnabled, readAvailableMemMb, readThrottleState, resolveCpuBusyFraction, throttleStatePath, writeThrottleState, } from "./throttle.js";
|
|
20
|
+
import { applyThrottleDecision, decideAdmission, defaultThrottleDeps, emptyThrottleState, evaluateThrottle, isCpuCheckEnabled, readAvailableMemMb, readThrottleState, resolveCpuBusyFraction, throttleStatePath, writeThrottleState, } from "./throttle.js";
|
|
19
21
|
import { applyDefaultQuotaHold, readQuotaSignal, recordAccountQuota } from "./quota.js";
|
|
20
22
|
import { answeredQuestionEntries, raiseEnvelopeQuestions } from "./questions.js";
|
|
21
23
|
import { SILENCE_MS, classifyLaneExit, isPidAlive, readLatestEnvelope, readLatestEnvelopeFile, readLatestEnvelopeRecord, } from "./watcher.js";
|
|
@@ -88,10 +90,10 @@ export function timeoutPath(hqRoot, laneId) {
|
|
|
88
90
|
return path.join(laneRunDir(hqRoot, laneId), "timeout.json");
|
|
89
91
|
}
|
|
90
92
|
export function laneRunsHere(lane, host) {
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
return lane
|
|
93
|
+
return laneRunsOnHost(lane, host);
|
|
94
|
+
}
|
|
95
|
+
function hasFreshAdmissionReservation(lane, nowMs) {
|
|
96
|
+
return hasActiveAdmissionReservation(lane, nowMs);
|
|
95
97
|
}
|
|
96
98
|
export function agedAdvanceCount(question) {
|
|
97
99
|
return question.routing_history.filter((entry) => entry.reason === "unanswered_timeout").length;
|
|
@@ -163,11 +165,6 @@ export function isDeadlinePassed(deadlineAt, nowMs) {
|
|
|
163
165
|
return false;
|
|
164
166
|
return nowMs > at;
|
|
165
167
|
}
|
|
166
|
-
function compareQueueOrder(a, b) {
|
|
167
|
-
if (b.priority !== a.priority)
|
|
168
|
-
return b.priority - a.priority;
|
|
169
|
-
return a.created_at.localeCompare(b.created_at);
|
|
170
|
-
}
|
|
171
168
|
function readDirJsonIds(dir, label) {
|
|
172
169
|
let names;
|
|
173
170
|
try {
|
|
@@ -308,6 +305,53 @@ function casLaneIf(hqRoot, laneId, shouldWrite, updater, dryRun) {
|
|
|
308
305
|
}
|
|
309
306
|
throw new LanesError("version_conflict", `Exhausted CAS retries for lane ${laneId}`, { lane_id: laneId });
|
|
310
307
|
}
|
|
308
|
+
function writeQueueReason(hqRoot, lane, reason, dryRun) {
|
|
309
|
+
if (lane.queued_reason === reason)
|
|
310
|
+
return;
|
|
311
|
+
casLaneIf(hqRoot, lane.lane_id, (current) => current.state === "queued" && current.queued_reason !== reason, (current) => ({ ...current, queued_reason: reason, admission_reserved_at: undefined }), dryRun);
|
|
312
|
+
}
|
|
313
|
+
function availableBalancedAccount(hqRoot, lane, occupying, config, host, profileMaxConcurrent, nowMs) {
|
|
314
|
+
if (!lane.account || !lane.admission_accounts?.includes(lane.account))
|
|
315
|
+
return undefined;
|
|
316
|
+
if (laneAdmissionCapReason(occupying, lane, config, host, profileMaxConcurrent, nowMs) !== "account_cap") {
|
|
317
|
+
return undefined;
|
|
318
|
+
}
|
|
319
|
+
const pinnedModel = lane.model_source === undefined || lane.model_source === "cli"
|
|
320
|
+
? lane.model
|
|
321
|
+
: lane.model_source === "profile"
|
|
322
|
+
? lane.model
|
|
323
|
+
: undefined;
|
|
324
|
+
for (const account of lane.admission_accounts) {
|
|
325
|
+
if (account === lane.account)
|
|
326
|
+
continue;
|
|
327
|
+
if (config.providers[lane.provider].accounts[account]?.enabled !== true)
|
|
328
|
+
continue;
|
|
329
|
+
const status = accountStatus(lane.provider, account, config);
|
|
330
|
+
if (!isCredentialProven(status))
|
|
331
|
+
continue;
|
|
332
|
+
try {
|
|
333
|
+
resolveAccount(lane.provider, account, config, {
|
|
334
|
+
hqRoot,
|
|
335
|
+
...(pinnedModel !== undefined ? { model: pinnedModel } : {}),
|
|
336
|
+
});
|
|
337
|
+
}
|
|
338
|
+
catch (err) {
|
|
339
|
+
if (err instanceof AccountResolutionError)
|
|
340
|
+
continue;
|
|
341
|
+
throw err;
|
|
342
|
+
}
|
|
343
|
+
const candidate = { ...lane, account };
|
|
344
|
+
if (laneAdmissionCapReason(occupying, candidate, config, host, profileMaxConcurrent, nowMs) === undefined) {
|
|
345
|
+
return account;
|
|
346
|
+
}
|
|
347
|
+
}
|
|
348
|
+
return undefined;
|
|
349
|
+
}
|
|
350
|
+
function writeAdmissionAccount(hqRoot, lane, account, dryRun) {
|
|
351
|
+
return casLaneIf(hqRoot, lane.lane_id, (current) => current.state === "queued" &&
|
|
352
|
+
current.account === lane.account &&
|
|
353
|
+
current.admission_accounts?.includes(account) === true, (current) => ({ ...current, account }), dryRun).record;
|
|
354
|
+
}
|
|
311
355
|
function casQuestionIf(hqRoot, questionId, shouldWrite, updater, dryRun) {
|
|
312
356
|
for (let i = 0; i < 5; i += 1) {
|
|
313
357
|
const current = readQuestion(hqRoot, questionId);
|
|
@@ -707,10 +751,12 @@ export async function runPassBody(hqRoot, deps) {
|
|
|
707
751
|
});
|
|
708
752
|
}
|
|
709
753
|
}
|
|
710
|
-
|
|
711
|
-
|
|
712
|
-
|
|
713
|
-
|
|
754
|
+
await withAllLaneAdmissionLocks(hqRoot, async () => {
|
|
755
|
+
const freshLanes = loadLaneRecords(hqRoot, summary);
|
|
756
|
+
const freshQuestions = loadQuestionRecords(hqRoot, summary);
|
|
757
|
+
await admitQueued(hqRoot, freshLanes, summary, deps, now);
|
|
758
|
+
await resumeAwaiting(hqRoot, freshLanes, freshQuestions, summary, deps);
|
|
759
|
+
});
|
|
714
760
|
if (!deps.dryRun && summary.errors.length === 0) {
|
|
715
761
|
try {
|
|
716
762
|
writeHeartbeatFile(hqRoot, summary, nowIso);
|
|
@@ -1005,7 +1051,8 @@ async function admitQueued(hqRoot, lanes, summary, deps, now) {
|
|
|
1005
1051
|
.filter((lane) => lane.state === "queued" &&
|
|
1006
1052
|
lane.pickup_validation_pending !== true &&
|
|
1007
1053
|
laneRunsHere(lane, deps.host))
|
|
1008
|
-
.
|
|
1054
|
+
.filter((lane) => !hasFreshAdmissionReservation(lane, now.getTime()))
|
|
1055
|
+
.sort(compareLaneQueueOrder);
|
|
1009
1056
|
if (localQueued.length === 0)
|
|
1010
1057
|
return;
|
|
1011
1058
|
const config = loadLanesConfig(hqRoot);
|
|
@@ -1030,31 +1077,37 @@ async function admitQueued(hqRoot, lanes, summary, deps, now) {
|
|
|
1030
1077
|
});
|
|
1031
1078
|
const spawnFn = deps.spawn ??
|
|
1032
1079
|
((root, lane) => spawnLaneWorker(root, lane, deps.spawnLaneDeps));
|
|
1033
|
-
const running = lanes.filter((lane) => lane.state === "running"
|
|
1080
|
+
const running = lanes.filter((lane) => (lane.state === "running" ||
|
|
1081
|
+
hasFreshAdmissionReservation(lane, now.getTime()) ||
|
|
1082
|
+
hasActivePendingResume(lane, now.getTime())) &&
|
|
1083
|
+
laneRunsHere(lane, deps.host));
|
|
1034
1084
|
const starvation = decision.admit && decision.reason === "starvation_bound";
|
|
1035
|
-
const candidates = starvation ? localQueued.slice(0, 1) : localQueued;
|
|
1036
1085
|
const spawned = [];
|
|
1037
1086
|
if (!decision.admit) {
|
|
1087
|
+
for (const lane of localQueued) {
|
|
1088
|
+
let profileMaxConcurrent;
|
|
1089
|
+
if (lane.worker_profile !== undefined) {
|
|
1090
|
+
try {
|
|
1091
|
+
profileMaxConcurrent = loadWorkerProfile(hqRoot, lane.worker_profile).meta.max_concurrent;
|
|
1092
|
+
}
|
|
1093
|
+
catch (err) {
|
|
1094
|
+
markDirty(summary, {
|
|
1095
|
+
kind: "lane",
|
|
1096
|
+
id: lane.lane_id,
|
|
1097
|
+
why: `profile admission failed: ${err instanceof Error ? err.message : String(err)}`,
|
|
1098
|
+
});
|
|
1099
|
+
continue;
|
|
1100
|
+
}
|
|
1101
|
+
}
|
|
1102
|
+
const capReason = laneAdmissionCapReason(running, lane, config, deps.host, profileMaxConcurrent, now.getTime());
|
|
1103
|
+
writeQueueReason(hqRoot, lane, capReason ?? "throttle", deps.dryRun);
|
|
1104
|
+
}
|
|
1038
1105
|
if (!deps.dryRun && throttleState !== "unreadable") {
|
|
1039
1106
|
writeThrottleState(filePath, applyThrottleDecision(throttleState, decision, nowMs), throttleDeps);
|
|
1040
1107
|
}
|
|
1041
1108
|
return;
|
|
1042
1109
|
}
|
|
1043
|
-
for (const lane of
|
|
1044
|
-
const provider = lane.provider;
|
|
1045
|
-
const providerCfg = config.providers[provider];
|
|
1046
|
-
const providerRunning = running.filter((row) => row.provider === provider).length +
|
|
1047
|
-
spawned.filter((row) => row.provider === provider).length;
|
|
1048
|
-
if (providerRunning >= providerCfg.max_running && !starvation)
|
|
1049
|
-
continue;
|
|
1050
|
-
if (lane.account) {
|
|
1051
|
-
const accountRunning = running.filter((row) => row.provider === provider && row.account === lane.account).length +
|
|
1052
|
-
spawned.filter((row) => row.provider === provider && row.account === lane.account).length;
|
|
1053
|
-
if (accountRunning >= accountMaxRunning(providerCfg, lane.account) &&
|
|
1054
|
-
!starvation) {
|
|
1055
|
-
continue;
|
|
1056
|
-
}
|
|
1057
|
-
}
|
|
1110
|
+
for (const lane of localQueued) {
|
|
1058
1111
|
let profileMaxConcurrent;
|
|
1059
1112
|
if (lane.worker_profile !== undefined) {
|
|
1060
1113
|
try {
|
|
@@ -1069,11 +1122,22 @@ async function admitQueued(hqRoot, lanes, summary, deps, now) {
|
|
|
1069
1122
|
continue;
|
|
1070
1123
|
}
|
|
1071
1124
|
}
|
|
1072
|
-
|
|
1073
|
-
|
|
1074
|
-
|
|
1075
|
-
|
|
1076
|
-
|
|
1125
|
+
const projected = [...running, ...spawned];
|
|
1126
|
+
const alternateAccount = availableBalancedAccount(hqRoot, lane, projected, config, deps.host, profileMaxConcurrent, now.getTime());
|
|
1127
|
+
const admittedLane = alternateAccount !== undefined
|
|
1128
|
+
? writeAdmissionAccount(hqRoot, lane, alternateAccount, deps.dryRun)
|
|
1129
|
+
: lane;
|
|
1130
|
+
const baseCap = laneAdmissionCapReason(running, admittedLane, config, deps.host, profileMaxConcurrent, now.getTime());
|
|
1131
|
+
const currentCap = laneAdmissionCapReason([...running, ...spawned], admittedLane, config, deps.host, profileMaxConcurrent, now.getTime());
|
|
1132
|
+
let blockedBy;
|
|
1133
|
+
if (currentCap !== undefined) {
|
|
1134
|
+
blockedBy = baseCap ?? "queue_position";
|
|
1135
|
+
}
|
|
1136
|
+
if (!blockedBy && !decision.admit)
|
|
1137
|
+
blockedBy = "throttle";
|
|
1138
|
+
if (blockedBy) {
|
|
1139
|
+
writeQueueReason(hqRoot, admittedLane, blockedBy, deps.dryRun);
|
|
1140
|
+
continue;
|
|
1077
1141
|
}
|
|
1078
1142
|
try {
|
|
1079
1143
|
if (deps.dryRun) {
|
|
@@ -1083,12 +1147,12 @@ async function admitQueued(hqRoot, lanes, summary, deps, now) {
|
|
|
1083
1147
|
to: "running",
|
|
1084
1148
|
reason: decision.reason,
|
|
1085
1149
|
});
|
|
1086
|
-
spawned.push({ ...
|
|
1150
|
+
spawned.push({ ...admittedLane, state: "running" });
|
|
1087
1151
|
if (starvation)
|
|
1088
1152
|
break;
|
|
1089
1153
|
continue;
|
|
1090
1154
|
}
|
|
1091
|
-
const started = spawnFn(hqRoot,
|
|
1155
|
+
const started = spawnFn(hqRoot, admittedLane);
|
|
1092
1156
|
if (decision.reason === "starvation_bound") {
|
|
1093
1157
|
casLaneIf(hqRoot, lane.lane_id, (current) => current.admitted_reason !== "starvation_bound", (current) => ({ ...current, admitted_reason: "starvation_bound" }), false);
|
|
1094
1158
|
}
|
|
@@ -1127,25 +1191,73 @@ async function admitQueued(hqRoot, lanes, summary, deps, now) {
|
|
|
1127
1191
|
}
|
|
1128
1192
|
}
|
|
1129
1193
|
async function resumeAwaiting(hqRoot, lanes, questions, summary, deps) {
|
|
1130
|
-
const
|
|
1131
|
-
((root, lane) => spawnLaneWorker(root, lane, deps.spawnLaneDeps));
|
|
1194
|
+
const config = loadLanesConfig(hqRoot);
|
|
1132
1195
|
const awaiting = lanes.filter((lane) => lane.state === "awaiting_input" && laneRunsHere(lane, deps.host));
|
|
1196
|
+
if (awaiting.length === 0)
|
|
1197
|
+
return;
|
|
1198
|
+
const throttleDeps = deps.throttleDeps ?? defaultThrottleDeps(deps.env ?? process.env);
|
|
1199
|
+
let throttleDecision;
|
|
1200
|
+
if (deps.dryRun) {
|
|
1201
|
+
const loaded = readThrottleState(throttleStatePath(hqRoot), throttleDeps);
|
|
1202
|
+
const state = loaded === "missing" ? emptyThrottleState() : loaded;
|
|
1203
|
+
throttleDecision = evaluateThrottle({
|
|
1204
|
+
busy: await resolveCpuBusyFraction(throttleDeps),
|
|
1205
|
+
availableMemMb: readAvailableMemMb(throttleDeps),
|
|
1206
|
+
cpuCheckEnabled: isCpuCheckEnabled(throttleDeps.env),
|
|
1207
|
+
state,
|
|
1208
|
+
throttle: config.throttle,
|
|
1209
|
+
nowMs: deps.now().getTime(),
|
|
1210
|
+
});
|
|
1211
|
+
}
|
|
1212
|
+
else {
|
|
1213
|
+
const throttle = await decideAdmission(hqRoot, config.throttle, throttleDeps);
|
|
1214
|
+
throttleDecision = throttle.decision;
|
|
1215
|
+
}
|
|
1216
|
+
if (!throttleDecision.admit)
|
|
1217
|
+
return;
|
|
1218
|
+
const spawnFn = deps.spawn ?? ((root, lane) => spawnLaneWorker(root, lane, deps.spawnLaneDeps));
|
|
1219
|
+
const nowMs = deps.now().getTime();
|
|
1220
|
+
const capacityLanes = lanes.map((lane) => {
|
|
1221
|
+
try {
|
|
1222
|
+
return readLane(hqRoot, lane.lane_id);
|
|
1223
|
+
}
|
|
1224
|
+
catch {
|
|
1225
|
+
return lane;
|
|
1226
|
+
}
|
|
1227
|
+
});
|
|
1228
|
+
const running = capacityLanes.filter((lane) => (lane.state === "running" ||
|
|
1229
|
+
hasFreshAdmissionReservation(lane, nowMs) ||
|
|
1230
|
+
hasActivePendingResume(lane, nowMs)) &&
|
|
1231
|
+
laneRunsHere(lane, deps.host));
|
|
1133
1232
|
for (const lane of awaiting) {
|
|
1233
|
+
const current = readLane(hqRoot, lane.lane_id);
|
|
1234
|
+
if (current.state !== "awaiting_input")
|
|
1235
|
+
continue;
|
|
1134
1236
|
const pending = questions.filter((q) => q.origin.id === lane.lane_id && q.status === "pending");
|
|
1135
1237
|
const answered = questions.filter((q) => q.origin.id === lane.lane_id && q.status === "answered");
|
|
1136
1238
|
if (pending.length > 0 || answered.length === 0)
|
|
1137
1239
|
continue;
|
|
1138
1240
|
try {
|
|
1241
|
+
let profileMaxConcurrent;
|
|
1242
|
+
if (current.worker_profile !== undefined) {
|
|
1243
|
+
profileMaxConcurrent = loadWorkerProfile(hqRoot, current.worker_profile).meta.max_concurrent;
|
|
1244
|
+
}
|
|
1245
|
+
if (laneAdmissionCapReason(running, current, config, deps.host, profileMaxConcurrent, nowMs) !== undefined) {
|
|
1246
|
+
continue;
|
|
1247
|
+
}
|
|
1139
1248
|
if (deps.dryRun) {
|
|
1140
1249
|
summary.resumed.push({
|
|
1141
1250
|
id: lane.lane_id,
|
|
1142
1251
|
from: "awaiting_input",
|
|
1143
1252
|
to: "running",
|
|
1144
1253
|
});
|
|
1254
|
+
running.push({ ...current, state: "running" });
|
|
1145
1255
|
continue;
|
|
1146
1256
|
}
|
|
1147
1257
|
writeAnsweredQuestions(hqRoot, lane, questions);
|
|
1148
1258
|
const started = spawnFn(hqRoot, lane);
|
|
1259
|
+
if (started.state === "running")
|
|
1260
|
+
running.push(started);
|
|
1149
1261
|
summary.resumed.push({
|
|
1150
1262
|
id: started.lane_id,
|
|
1151
1263
|
from: "awaiting_input",
|