@indigoai-us/hq-cli 5.298.0 → 5.300.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +11 -0
- package/dist/command-catalog.generated.d.ts +27 -4
- package/dist/command-catalog.generated.js +34 -4
- package/dist/commands/cloud.js +12 -7
- package/dist/commands/daemon.js +1 -0
- package/dist/commands/files.d.ts +1 -0
- package/dist/commands/files.js +59 -0
- package/dist/commands/lanes-accounts.d.ts +3 -0
- package/dist/commands/lanes-accounts.js +25 -1
- package/dist/commands/lanes-message.js +68 -35
- package/dist/commands/lanes-resume.js +247 -210
- package/dist/commands/lanes-shared.d.ts +3 -1
- package/dist/commands/lanes.d.ts +2 -0
- package/dist/commands/lanes.js +16 -4
- package/dist/lib/daemon/legacy.d.ts +5 -2
- package/dist/lib/daemon/legacy.js +6 -4
- package/dist/lib/daemon/migrate.d.ts +10 -1
- package/dist/lib/daemon/migrate.js +22 -7
- package/dist/lib/daemon/run.js +19 -7
- package/dist/lib/daemon/supervisor.js +9 -1
- package/dist/lib/lanes/quota.d.ts +19 -0
- package/dist/lib/lanes/quota.js +106 -29
- package/dist/lib/lanes/reconcile.js +34 -5
- package/dist/lib/lanes/store.d.ts +2 -0
- package/dist/lib/lanes/store.js +4 -0
- package/dist/lib/lanes/story-switch.d.ts +21 -0
- package/dist/lib/lanes/story-switch.js +75 -0
- package/dist/lib/lanes/types.d.ts +25 -0
- package/dist/lib/lanes/types.js +1 -1
- package/dist/lib/lanes/watcher.js +25 -5
- package/dist/lib/mesh/live/daemon/install.d.ts +3 -0
- package/dist/lib/mesh/live/daemon/install.js +13 -2
- package/dist/utils/self-update.js +249 -2
- package/dist/utils/staged-install.js +71 -21
- package/package.json +1 -1
|
@@ -17,11 +17,13 @@
|
|
|
17
17
|
* folder, so nobody but root decides what gets installed there.
|
|
18
18
|
*/
|
|
19
19
|
import type { RunCommand } from "../service-manager/launchd.js";
|
|
20
|
-
import type
|
|
20
|
+
import { type LegacyUnit } from "./legacy.js";
|
|
21
21
|
export interface AdoptedUnit extends LegacyUnit {
|
|
22
22
|
/** Where the unit's definition (plist, unit file, task XML) was moved. */
|
|
23
23
|
movedTo: string;
|
|
24
24
|
adoptedAt: number;
|
|
25
|
+
/** Launchd job was loaded, but its source plist no longer existed. */
|
|
26
|
+
definitionMissing?: true;
|
|
25
27
|
}
|
|
26
28
|
export interface MigrateDeps {
|
|
27
29
|
run: RunCommand;
|
|
@@ -64,5 +66,12 @@ export interface RestoreResult {
|
|
|
64
66
|
}
|
|
65
67
|
export declare function readAdoptedUnits(recordPath: string): AdoptedUnit[];
|
|
66
68
|
export declare function adoptLegacyUnits(units: readonly LegacyUnit[], deps: MigrateDeps): AdoptResult;
|
|
69
|
+
/** Retire the migratable legacy mesh job and emit the daemon's operator-facing result lines. */
|
|
70
|
+
export declare function retireLegacyMeshUnits(units: readonly LegacyUnit[], options: {
|
|
71
|
+
platform: NodeJS.Platform;
|
|
72
|
+
managed: boolean;
|
|
73
|
+
hosted: boolean;
|
|
74
|
+
meshEnabled: boolean;
|
|
75
|
+
}, deps: MigrateDeps, log: (level: "info" | "warn", message: string) => void): AdoptResult;
|
|
67
76
|
export declare function restoreLegacyUnits(deps: MigrateDeps): RestoreResult;
|
|
68
77
|
//# sourceMappingURL=migrate.d.ts.map
|
|
@@ -18,6 +18,7 @@
|
|
|
18
18
|
*/
|
|
19
19
|
import * as fs from "node:fs";
|
|
20
20
|
import * as path from "node:path";
|
|
21
|
+
import { legacyMeshUnitsToRetire } from "./legacy.js";
|
|
21
22
|
import { ensurePrivateDir } from "./paths.js";
|
|
22
23
|
export function readAdoptedUnits(recordPath) {
|
|
23
24
|
let text;
|
|
@@ -99,8 +100,10 @@ function adoptOne(unit, deps) {
|
|
|
99
100
|
if (bootout.status !== 0 && deps.run("launchctl", ["print", target]).status === 0) {
|
|
100
101
|
must(deps.run, "launchctl", ["bootout", target]);
|
|
101
102
|
}
|
|
103
|
+
if (!fs.existsSync(unit.path))
|
|
104
|
+
return { movedTo: unit.path, definitionMissing: true };
|
|
102
105
|
moveFile(unit.path, to);
|
|
103
|
-
return to;
|
|
106
|
+
return { movedTo: to };
|
|
104
107
|
}
|
|
105
108
|
case "systemd-user":
|
|
106
109
|
case "systemd-system": {
|
|
@@ -109,7 +112,7 @@ function adoptOne(unit, deps) {
|
|
|
109
112
|
must(deps.run, ...systemctl(unit, deps, ["disable", "--now", unit.name]));
|
|
110
113
|
moveFile(unit.path, to);
|
|
111
114
|
deps.run(...systemctl(unit, deps, ["daemon-reload"]));
|
|
112
|
-
return to;
|
|
115
|
+
return { movedTo: to };
|
|
113
116
|
}
|
|
114
117
|
case "schtasks": {
|
|
115
118
|
const xml = must(deps.run, "schtasks.exe", ["/Query", "/TN", unit.name, "/XML"]);
|
|
@@ -117,7 +120,7 @@ function adoptOne(unit, deps) {
|
|
|
117
120
|
fs.writeFileSync(to, encodeTaskXml(xml), { mode: 0o600 });
|
|
118
121
|
deps.run("schtasks.exe", ["/End", "/TN", unit.name]);
|
|
119
122
|
must(deps.run, "schtasks.exe", ["/Delete", "/TN", unit.name, "/F"]);
|
|
120
|
-
return to;
|
|
123
|
+
return { movedTo: to };
|
|
121
124
|
}
|
|
122
125
|
}
|
|
123
126
|
}
|
|
@@ -141,15 +144,15 @@ export function adoptLegacyUnits(units, deps) {
|
|
|
141
144
|
continue;
|
|
142
145
|
}
|
|
143
146
|
const store = storeFor(unit.kind, deps);
|
|
144
|
-
let
|
|
147
|
+
let adoptedFields;
|
|
145
148
|
try {
|
|
146
|
-
|
|
149
|
+
adoptedFields = adoptOne(unit, deps);
|
|
147
150
|
}
|
|
148
151
|
catch (err) {
|
|
149
152
|
result.failed.push({ unit, error: err.message });
|
|
150
153
|
continue;
|
|
151
154
|
}
|
|
152
|
-
const adopted = { ...unit,
|
|
155
|
+
const adopted = { ...unit, ...adoptedFields, adoptedAt: now() };
|
|
153
156
|
const next = [...recordOf(store), adopted];
|
|
154
157
|
try {
|
|
155
158
|
// Record each unit as soon as it is moved, so a failure later in the
|
|
@@ -165,7 +168,7 @@ export function adoptLegacyUnits(units, deps) {
|
|
|
165
168
|
restoreOne(adopted, deps);
|
|
166
169
|
}
|
|
167
170
|
catch (restoreErr) {
|
|
168
|
-
rollback = `; putting it back also failed (${restoreErr.message}), its definition is at ${movedTo}`;
|
|
171
|
+
rollback = `; putting it back also failed (${restoreErr.message}), its definition is at ${adopted.movedTo}`;
|
|
169
172
|
}
|
|
170
173
|
result.failed.push({ unit, error: `could not record the takeover: ${err.message}${rollback}` });
|
|
171
174
|
}
|
|
@@ -192,6 +195,8 @@ function restoreOne(unit, deps) {
|
|
|
192
195
|
case "launchd": {
|
|
193
196
|
if (!unit.path)
|
|
194
197
|
throw new Error(`no plist path recorded for ${unit.name}`);
|
|
198
|
+
if (unit.definitionMissing)
|
|
199
|
+
throw new Error("the original LaunchAgent plist is missing; this unit cannot be restored");
|
|
195
200
|
moveBack(unit, unit.path);
|
|
196
201
|
must(deps.run, "launchctl", ["bootstrap", `gui/${deps.uid}`, unit.path]);
|
|
197
202
|
return;
|
|
@@ -216,6 +221,16 @@ function restoreOne(unit, deps) {
|
|
|
216
221
|
return;
|
|
217
222
|
}
|
|
218
223
|
}
|
|
224
|
+
/** Retire the migratable legacy mesh job and emit the daemon's operator-facing result lines. */
|
|
225
|
+
export function retireLegacyMeshUnits(units, options, deps, log) {
|
|
226
|
+
const selected = legacyMeshUnitsToRetire(units, options);
|
|
227
|
+
const result = adoptLegacyUnits(selected, deps);
|
|
228
|
+
for (const unit of result.adopted)
|
|
229
|
+
log("info", `retired legacy ${unit.unitId} unit ${unit.name}; hq daemon now owns mesh`);
|
|
230
|
+
for (const failure of result.failed)
|
|
231
|
+
log("warn", `could not retire legacy ${failure.unit.unitId} unit ${failure.unit.name}: ${failure.error}`);
|
|
232
|
+
return result;
|
|
233
|
+
}
|
|
219
234
|
function restoreFrom(record, deps, check, result) {
|
|
220
235
|
const units = readAdoptedUnits(record);
|
|
221
236
|
if (units.length === 0)
|
package/dist/lib/daemon/run.js
CHANGED
|
@@ -22,7 +22,7 @@ import { SUPERVISOR_HANDOFF_FLAG, readDaemonFlag } from "./flag.js";
|
|
|
22
22
|
import { findLegacyUnits, legacyMeshUnitsToRetire, legacyOwnerLookup } from "./legacy.js";
|
|
23
23
|
import { readLegacySyncEnv } from "./legacy-sync-env.js";
|
|
24
24
|
import { acquireDaemonLock, isProcessAlive, processCommandLine, releaseDaemonLock } from "./lock.js";
|
|
25
|
-
import {
|
|
25
|
+
import { retireLegacyMeshUnits } from "./migrate.js";
|
|
26
26
|
import { daemonDir as meshDaemonDir, daemonPidPath as meshDaemonPidPath } from "../mesh/live/daemon/paths.js";
|
|
27
27
|
import { readPidLock } from "../mesh/live/daemon/pid-lock.js";
|
|
28
28
|
import { daemonRootRefusal } from "./install.js";
|
|
@@ -72,6 +72,17 @@ export function probeLegacyUnits(platform, home) {
|
|
|
72
72
|
home,
|
|
73
73
|
existsSync: fs.existsSync,
|
|
74
74
|
listDir: (dir) => fs.readdirSync(dir),
|
|
75
|
+
launchdJobLoaded: (label) => {
|
|
76
|
+
if (platform !== "darwin")
|
|
77
|
+
return false;
|
|
78
|
+
const uid = process.getuid?.() ?? 0;
|
|
79
|
+
const result = spawnSync("launchctl", ["print", `gui/${uid}/${label}`], {
|
|
80
|
+
encoding: "utf8",
|
|
81
|
+
windowsHide: true,
|
|
82
|
+
timeout: 15_000,
|
|
83
|
+
});
|
|
84
|
+
return result.status === 0;
|
|
85
|
+
},
|
|
75
86
|
windowsTaskExists,
|
|
76
87
|
});
|
|
77
88
|
}
|
|
@@ -344,7 +355,12 @@ export async function runDaemon(opts) {
|
|
|
344
355
|
if (legacyMesh.length > 0) {
|
|
345
356
|
try {
|
|
346
357
|
const uid = process.getuid?.() ?? 0;
|
|
347
|
-
|
|
358
|
+
retireLegacyMeshUnits(legacyMesh, {
|
|
359
|
+
platform,
|
|
360
|
+
managed: opts.managed === true,
|
|
361
|
+
hosted: opts.host !== undefined,
|
|
362
|
+
meshEnabled: enabledServices(machineType, initial).includes("mesh"),
|
|
363
|
+
}, {
|
|
348
364
|
run: (command, args) => {
|
|
349
365
|
const r = spawnSync(command, args, { encoding: "utf8", windowsHide: true, timeout: 15_000 });
|
|
350
366
|
return { status: r.status, stdout: r.stdout ?? "", stderr: r.stderr ?? "" };
|
|
@@ -356,11 +372,7 @@ export async function runDaemon(opts) {
|
|
|
356
372
|
systemAdoptedDir: "/var/lib/hq/daemon/adopted",
|
|
357
373
|
systemRecordPath: "/var/lib/hq/daemon/adopted.json",
|
|
358
374
|
systemUnitDir: "/etc/systemd/system",
|
|
359
|
-
});
|
|
360
|
-
for (const unit of result.adopted)
|
|
361
|
-
log("info", `retired legacy ${unit.unitId} unit ${unit.name}; hq daemon now owns mesh`);
|
|
362
|
-
for (const failure of result.failed)
|
|
363
|
-
log("warn", `could not retire legacy ${failure.unit.unitId} unit ${failure.unit.name}: ${failure.error}`);
|
|
375
|
+
}, (level, message) => log(level, message));
|
|
364
376
|
}
|
|
365
377
|
catch (err) {
|
|
366
378
|
log("error", `could not retire legacy mesh units: ${err.message}`);
|
|
@@ -28,6 +28,7 @@ export const BACKOFF_MAX_MS = 60_000;
|
|
|
28
28
|
export const HEALTHY_RUN_MS = 60_000;
|
|
29
29
|
export const STOP_ESCALATION_MS = 10_000;
|
|
30
30
|
const MESH_OWNER_RECHECK_MS = 15_000;
|
|
31
|
+
const MESH_OWNER_LOG_INTERVAL_MS = 10 * 60_000;
|
|
31
32
|
/** After a handoff, how long a replaced unit must stay up before the next one is replaced. */
|
|
32
33
|
export const HANDOFF_STAGGER_MS = 15_000;
|
|
33
34
|
/** How long a draining unit may take to finish its work before it gets SIGTERM. */
|
|
@@ -297,11 +298,18 @@ export function createSupervisor(deps) {
|
|
|
297
298
|
}
|
|
298
299
|
const meshOwnerReason = slot.plan.service === "mesh" ? deps.meshOwnerReason?.() : undefined;
|
|
299
300
|
if (meshOwnerReason) {
|
|
301
|
+
let lastLoggedReason;
|
|
302
|
+
let lastLoggedAt = Number.NEGATIVE_INFINITY;
|
|
300
303
|
const waitForOwner = (reason) => {
|
|
301
304
|
slot.state.status = "waiting";
|
|
302
305
|
slot.state.reason = reason;
|
|
303
306
|
slot.state.nextRunAt = now() + MESH_OWNER_RECHECK_MS;
|
|
304
|
-
|
|
307
|
+
const at = now();
|
|
308
|
+
if (reason !== lastLoggedReason || at - lastLoggedAt >= MESH_OWNER_LOG_INTERVAL_MS) {
|
|
309
|
+
deps.log("warn", `${reason}; retrying mesh ownership check in ${MESH_OWNER_RECHECK_MS}ms`);
|
|
310
|
+
lastLoggedReason = reason;
|
|
311
|
+
lastLoggedAt = at;
|
|
312
|
+
}
|
|
305
313
|
slot.timer = setT(() => {
|
|
306
314
|
slot.timer = undefined;
|
|
307
315
|
const current = deps.meshOwnerReason?.();
|
|
@@ -89,6 +89,25 @@ export declare function quotaSignalFromFields(fields: {
|
|
|
89
89
|
}, model: string | undefined, now: Date): QuotaSignal | null;
|
|
90
90
|
export declare function readQuotaSignal(runDir: string, model: string | undefined, now: Date, workerOutOffset?: number): QuotaSignal | null;
|
|
91
91
|
export declare function recordAccountQuota(hqRoot: string, provider: LaneProvider, accountId: string, signal: QuotaSignal): QuotaState;
|
|
92
|
+
export interface ClearedModelQuotaReset {
|
|
93
|
+
model: string;
|
|
94
|
+
quota_reset_at: string;
|
|
95
|
+
}
|
|
96
|
+
export interface AccountQuotaClearResult {
|
|
97
|
+
all_models_quota_reset_at: string | null;
|
|
98
|
+
model_quota_resets: ClearedModelQuotaReset[];
|
|
99
|
+
}
|
|
100
|
+
export interface ClearAccountQuotaOptions {
|
|
101
|
+
/** Clear only this model's reset instead of every per-model reset. */
|
|
102
|
+
model?: string;
|
|
103
|
+
/** Clear only marks whose reset time is still ahead of now. */
|
|
104
|
+
futureOnly?: boolean;
|
|
105
|
+
now?: Date;
|
|
106
|
+
}
|
|
107
|
+
/** Clear quota reset marks and persist them with the quota file's atomic writer. */
|
|
108
|
+
export declare function clearAccountQuota(hqRoot: string, provider: LaneProvider, accountId: string, options?: ClearAccountQuotaOptions): AccountQuotaClearResult;
|
|
109
|
+
export declare function clearAccountQuotaAfterSuccessfulTurn(hqRoot: string, provider: LaneProvider, accountId: string, model: string, now: Date): AccountQuotaClearResult;
|
|
110
|
+
export declare function clearAccountQuotaAfterSuccessfulTurnBestEffort(hqRoot: string, provider: LaneProvider, accountId: string, model: string, now: Date): void;
|
|
92
111
|
export interface QuotaBlock {
|
|
93
112
|
blocked: boolean;
|
|
94
113
|
reason?: "account_quota_exhausted" | "account_model_unavailable";
|
package/dist/lib/lanes/quota.js
CHANGED
|
@@ -19,6 +19,7 @@
|
|
|
19
19
|
*/
|
|
20
20
|
import * as fs from "node:fs";
|
|
21
21
|
import * as path from "node:path";
|
|
22
|
+
import { withLaneRecordFileLock } from "./store.js";
|
|
22
23
|
export const QUOTA_STATE_REL = path.join("personal", "lanes", "quota.json");
|
|
23
24
|
export const QUOTA_SCHEMA_VERSION = "hq-lanes-quota.v1";
|
|
24
25
|
/**
|
|
@@ -278,40 +279,116 @@ export function readQuotaSignal(runDir, model, now, workerOutOffset = 0) {
|
|
|
278
279
|
return quotaSignalFromFields(fields, model, now);
|
|
279
280
|
}
|
|
280
281
|
export function recordAccountQuota(hqRoot, provider, accountId, signal) {
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
282
|
+
return withLaneRecordFileLock(quotaStatePath(hqRoot), () => {
|
|
283
|
+
const state = loadQuotaState(hqRoot);
|
|
284
|
+
const accounts = state.providers[provider] ?? {};
|
|
285
|
+
const current = accounts[accountId] ?? { models: {} };
|
|
286
|
+
const models = { ...current.models };
|
|
287
|
+
if (signal.model) {
|
|
288
|
+
const prior = models[signal.model] ?? {};
|
|
289
|
+
const next = { ...prior };
|
|
290
|
+
if (signal.needs_credits) {
|
|
291
|
+
next.needs_credits = true;
|
|
292
|
+
next.marker = signal.marker ?? `needs credits for ${signal.model}`;
|
|
293
|
+
delete next.quota_reset_at;
|
|
294
|
+
}
|
|
295
|
+
if (signal.quota_reset_at &&
|
|
296
|
+
!signal.needs_credits &&
|
|
297
|
+
shouldReplaceReset(next.quota_reset_at, signal.quota_reset_at)) {
|
|
298
|
+
next.quota_reset_at = signal.quota_reset_at;
|
|
299
|
+
}
|
|
300
|
+
models[signal.model] = next;
|
|
292
301
|
}
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
shouldReplaceReset(
|
|
296
|
-
|
|
302
|
+
const account = { ...current, models };
|
|
303
|
+
if (signal.account_wide && signal.quota_reset_at) {
|
|
304
|
+
if (shouldReplaceReset(current.all_models_quota_reset_at, signal.quota_reset_at)) {
|
|
305
|
+
account.all_models_quota_reset_at = signal.quota_reset_at;
|
|
306
|
+
account.reset_source = signal.reset_source ?? "provider";
|
|
307
|
+
}
|
|
308
|
+
else if (current.all_models_quota_reset_at) {
|
|
309
|
+
account.all_models_quota_reset_at = current.all_models_quota_reset_at;
|
|
310
|
+
account.reset_source = current.reset_source ?? "provider";
|
|
311
|
+
}
|
|
297
312
|
}
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
313
|
+
accounts[accountId] = account;
|
|
314
|
+
state.providers[provider] = accounts;
|
|
315
|
+
writeQuotaState(hqRoot, state);
|
|
316
|
+
return state;
|
|
317
|
+
});
|
|
318
|
+
}
|
|
319
|
+
/** Clear quota reset marks and persist them with the quota file's atomic writer. */
|
|
320
|
+
export function clearAccountQuota(hqRoot, provider, accountId, options = {}) {
|
|
321
|
+
return withLaneRecordFileLock(quotaStatePath(hqRoot), () => {
|
|
322
|
+
const cleared = {
|
|
323
|
+
all_models_quota_reset_at: null,
|
|
324
|
+
model_quota_resets: [],
|
|
325
|
+
};
|
|
326
|
+
const state = loadQuotaState(hqRoot);
|
|
327
|
+
const account = state.providers[provider]?.[accountId];
|
|
328
|
+
if (!account)
|
|
329
|
+
return cleared;
|
|
330
|
+
const now = options.now ?? new Date();
|
|
331
|
+
const shouldClear = (reset) => reset !== undefined &&
|
|
332
|
+
(!options.futureOnly || resetStillAhead(reset, now));
|
|
333
|
+
let changed = false;
|
|
334
|
+
const accountWideReset = account.all_models_quota_reset_at;
|
|
335
|
+
if (shouldClear(accountWideReset)) {
|
|
336
|
+
cleared.all_models_quota_reset_at = accountWideReset;
|
|
337
|
+
delete account.all_models_quota_reset_at;
|
|
338
|
+
delete account.reset_source;
|
|
339
|
+
changed = true;
|
|
340
|
+
}
|
|
341
|
+
const modelNames = options.model === undefined
|
|
342
|
+
? Object.keys(account.models).sort()
|
|
343
|
+
: [options.model];
|
|
344
|
+
for (const model of modelNames) {
|
|
345
|
+
const entry = account.models[model];
|
|
346
|
+
const modelReset = entry?.quota_reset_at;
|
|
347
|
+
if (!entry || !shouldClear(modelReset))
|
|
348
|
+
continue;
|
|
349
|
+
cleared.model_quota_resets.push({
|
|
350
|
+
model,
|
|
351
|
+
quota_reset_at: modelReset,
|
|
352
|
+
});
|
|
353
|
+
delete entry.quota_reset_at;
|
|
354
|
+
if (Object.keys(entry).length === 0)
|
|
355
|
+
delete account.models[model];
|
|
356
|
+
changed = true;
|
|
305
357
|
}
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
358
|
+
if (!changed)
|
|
359
|
+
return cleared;
|
|
360
|
+
if (account.all_models_quota_reset_at === undefined) {
|
|
361
|
+
delete account.reset_source;
|
|
309
362
|
}
|
|
363
|
+
const accounts = state.providers[provider];
|
|
364
|
+
if (account.all_models_quota_reset_at === undefined &&
|
|
365
|
+
Object.keys(account.models).length === 0) {
|
|
366
|
+
delete accounts[accountId];
|
|
367
|
+
if (Object.keys(accounts).length === 0)
|
|
368
|
+
delete state.providers[provider];
|
|
369
|
+
}
|
|
370
|
+
else {
|
|
371
|
+
accounts[accountId] = account;
|
|
372
|
+
}
|
|
373
|
+
writeQuotaState(hqRoot, state);
|
|
374
|
+
return cleared;
|
|
375
|
+
});
|
|
376
|
+
}
|
|
377
|
+
export function clearAccountQuotaAfterSuccessfulTurn(hqRoot, provider, accountId, model, now) {
|
|
378
|
+
return clearAccountQuota(hqRoot, provider, accountId, {
|
|
379
|
+
model,
|
|
380
|
+
futureOnly: true,
|
|
381
|
+
now,
|
|
382
|
+
});
|
|
383
|
+
}
|
|
384
|
+
export function clearAccountQuotaAfterSuccessfulTurnBestEffort(hqRoot, provider, accountId, model, now) {
|
|
385
|
+
try {
|
|
386
|
+
clearAccountQuotaAfterSuccessfulTurn(hqRoot, provider, accountId, model, now);
|
|
387
|
+
}
|
|
388
|
+
catch (err) {
|
|
389
|
+
const detail = err instanceof Error ? err.message : String(err);
|
|
390
|
+
process.stderr.write(`hq lanes: quota cleanup after successful turn failed for provider=${provider} account=${accountId} model=${model}: ${detail}\n`);
|
|
310
391
|
}
|
|
311
|
-
accounts[accountId] = account;
|
|
312
|
-
state.providers[provider] = accounts;
|
|
313
|
-
writeQuotaState(hqRoot, state);
|
|
314
|
-
return state;
|
|
315
392
|
}
|
|
316
393
|
function shouldReplaceReset(current, incoming) {
|
|
317
394
|
if (!current)
|
|
@@ -27,11 +27,12 @@ import { appendDropboxMessage } from "./dropbox.js";
|
|
|
27
27
|
import { DEFAULT_RUNAWAY_SEARCH_GUARD_CONFIG, listHostProcesses, runRunawayToolGuard, RUNAWAY_TOOL_GUARD_FLAG, } from "./runaway-tool-guard.js";
|
|
28
28
|
import { LEAD_LANE_KEEP_ALIVE_FLAG, LEAD_LANE_KEEP_ALIVE_RESUME_BLOCKED_FLAG, LEAD_LANE_KEEP_ALIVE_BLOCKED_GRACE_MS, LEAD_LANE_KEEP_ALIVE_BLOCKED_RESUME_TEXT, queueLaneTerminalResume, LEAD_LANE_KEEP_ALIVE_INTERRUPTED_RESUME_TEXT, LEAD_LANE_KEEP_ALIVE_MAX_RESUMES, LEAD_LANE_KEEP_ALIVE_RESUME_TEXT, LANE_SUPERSESSION_LINEAGE_AWARE_FLAG, doneKeepAliveVerdict, hasPendingKeepAliveWork, keepAliveExclusion, loadRetiredLaneIds, leadLaneKeepAliveBackoffMs, undeliveredAnsweredQuestions, } from "./keep-alive.js";
|
|
29
29
|
import { providerSupportsSessionResume } from "./delivery.js";
|
|
30
|
-
import { applyDefaultQuotaHold, readQuotaSignal, recordAccountQuota } from "./quota.js";
|
|
30
|
+
import { applyDefaultQuotaHold, clearAccountQuotaAfterSuccessfulTurnBestEffort, readQuotaSignal, recordAccountQuota, } from "./quota.js";
|
|
31
31
|
import { answeredQuestionEntries, questionRecordRefusalCode, raiseEnvelopeQuestions, resolveQuestionCompanyWithFallback, } from "./questions.js";
|
|
32
32
|
import { SILENCE_MS, appendWatcherEvent, classifyLaneExit, isPidAlive, missingExpectedEnvelopeFile, readLatestEnvelopeFile, readLatestEnvelopeRecord, watcherEventsPath, watcherOutLogPath, writeWatcherDecision, } from "./watcher.js";
|
|
33
33
|
import { createLaneMesh, emitLaneDecisionEvents, emitLaneTerminalStateEvent, laneMeshCompany, } from "./mesh-events.js";
|
|
34
34
|
import { activeLaneStories, laneStories, terminalEnvelopeStoryStatus } from "./stories.js";
|
|
35
|
+
import { applyStorySwitchEnvelope } from "./story-switch.js";
|
|
35
36
|
import { decideEnvelopeRepair, envelopeQuestionIds, routeProviderFailure } from "./watcher.js";
|
|
36
37
|
import { PROVIDER_REJECTED_REASON, providerRejectionDetails, readProviderRejection, } from "./provider-rejection.js";
|
|
37
38
|
/** Spec §9: unanswered at the current hop for 60 minutes → one hop. */
|
|
@@ -1271,11 +1272,28 @@ async function heartbeatAndDeadHosts(hqRoot, lanes, summary, deps, now) {
|
|
|
1271
1272
|
}
|
|
1272
1273
|
if (terminalEnvelope && latest) {
|
|
1273
1274
|
const envelope = terminalEnvelope;
|
|
1274
|
-
|
|
1275
|
+
next.last_envelope = {
|
|
1276
|
+
ref: latest.ref,
|
|
1277
|
+
decision: envelope.decision,
|
|
1278
|
+
at: nowIso,
|
|
1279
|
+
artifacts: envelope.artifacts,
|
|
1280
|
+
...(envelope.blocked_reason !== undefined ? { blocked_reason: envelope.blocked_reason } : {}),
|
|
1281
|
+
};
|
|
1282
|
+
const storySwitch = applyStorySwitchEnvelope(current, envelope, nowIso, latest.ref);
|
|
1283
|
+
if (storySwitch.clearPending) {
|
|
1284
|
+
next.story_switch_pending = undefined;
|
|
1285
|
+
}
|
|
1286
|
+
if (storySwitch.storySwitchUnacknowledged !== undefined) {
|
|
1287
|
+
next.story_switch_unacknowledged = storySwitch.storySwitchUnacknowledged;
|
|
1288
|
+
next.attribution_history = storySwitch.attributionHistory ?? current.attribution_history;
|
|
1289
|
+
}
|
|
1290
|
+
const storyStatus = storySwitch.suppressCompletion
|
|
1291
|
+
? "in_progress"
|
|
1292
|
+
: terminalEnvelopeStoryStatus(current, envelope.decision, envelope.story_status);
|
|
1275
1293
|
if (storyStatus !== undefined) {
|
|
1276
|
-
next.story_status = envelope.story_status;
|
|
1277
|
-
next.story_complete = envelope.story_complete;
|
|
1278
|
-
next.story_complete_reason = envelope.story_complete_reason;
|
|
1294
|
+
next.story_status = storySwitch.suppressCompletion ? "in_progress" : envelope.story_status;
|
|
1295
|
+
next.story_complete = storySwitch.suppressCompletion ? false : envelope.story_complete;
|
|
1296
|
+
next.story_complete_reason = storySwitch.suppressCompletion ? undefined : envelope.story_complete_reason;
|
|
1279
1297
|
next.stories = laneStories(current).map((story) => story.project_id === current.project_id && story.story_id === current.story_id
|
|
1280
1298
|
? {
|
|
1281
1299
|
...story,
|
|
@@ -1299,6 +1317,17 @@ async function heartbeatAndDeadHosts(hqRoot, lanes, summary, deps, now) {
|
|
|
1299
1317
|
}
|
|
1300
1318
|
return next;
|
|
1301
1319
|
}, deps.dryRun);
|
|
1320
|
+
if (!deps.dryRun &&
|
|
1321
|
+
result.changed &&
|
|
1322
|
+
result.record.state === "done" &&
|
|
1323
|
+
terminalClassification.state === "done" &&
|
|
1324
|
+
terminalEnvelope?.decision === "done" &&
|
|
1325
|
+
!quota &&
|
|
1326
|
+
!providerRejection &&
|
|
1327
|
+
result.record.account &&
|
|
1328
|
+
result.record.model) {
|
|
1329
|
+
clearAccountQuotaAfterSuccessfulTurnBestEffort(hqRoot, result.record.provider, result.record.account, result.record.model, new Date(nowIso));
|
|
1330
|
+
}
|
|
1302
1331
|
if (result.changed) {
|
|
1303
1332
|
summary.dead_host.push({
|
|
1304
1333
|
id: lane.lane_id,
|
|
@@ -43,6 +43,8 @@ export declare function sanitizeHostComponent(hostname: string): string;
|
|
|
43
43
|
export declare function mintLaneId(deps?: Pick<StoreDeps, "now" | "hostname"> & Partial<Pick<StoreDeps, "generateLaneUlid">>): string;
|
|
44
44
|
export declare function mintQuestionId(deps?: Pick<StoreDeps, "now" | "hostname">): string;
|
|
45
45
|
export declare function appendCappedHistory<T>(history: readonly T[], entry: T, cap?: number): T[];
|
|
46
|
+
/** Run a lane-owned file read/modify/write transaction under its sidecar lock. */
|
|
47
|
+
export declare function withLaneRecordFileLock<T>(recordPath: string, fn: () => T): T;
|
|
46
48
|
export declare function parseLaneRecord(raw: unknown, filePath?: string): LaneRecord;
|
|
47
49
|
export declare function parseQuestionRecord(raw: unknown, filePath?: string): QuestionRecord;
|
|
48
50
|
/**
|
package/dist/lib/lanes/store.js
CHANGED
|
@@ -222,6 +222,10 @@ function withRecordLock(recordPath, deps, fn) {
|
|
|
222
222
|
}
|
|
223
223
|
}
|
|
224
224
|
}
|
|
225
|
+
/** Run a lane-owned file read/modify/write transaction under its sidecar lock. */
|
|
226
|
+
export function withLaneRecordFileLock(recordPath, fn) {
|
|
227
|
+
return withRecordLock(recordPath, defaultStoreDeps(), fn);
|
|
228
|
+
}
|
|
225
229
|
function readJsonFile(filePath, deps) {
|
|
226
230
|
let text;
|
|
227
231
|
try {
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
import type { AttributionHistoryEntry, LaneEnvelope, LaneRecord, StorySwitchUnacknowledgedNote } from "./types.js";
|
|
2
|
+
export declare function storySwitchNotice(previousProjectId: string, previousStoryId: string, projectId: string, storyId: string): string | undefined;
|
|
3
|
+
export declare function artifactIdentityKeys(artifacts: readonly unknown[] | undefined): string[];
|
|
4
|
+
export interface StorySwitchRollbackSnapshot {
|
|
5
|
+
message_id: string;
|
|
6
|
+
project_id: string;
|
|
7
|
+
story_id: string;
|
|
8
|
+
no_story_reason?: string;
|
|
9
|
+
added_story?: NonNullable<LaneRecord["stories"]>[number];
|
|
10
|
+
story_switch_unacknowledged: StorySwitchUnacknowledgedNote | undefined;
|
|
11
|
+
}
|
|
12
|
+
/** Restore only the attribution write owned by this failed command attempt. */
|
|
13
|
+
export declare function rollbackStorySwitchAttribution(current: LaneRecord, snapshot: StorySwitchRollbackSnapshot): LaneRecord;
|
|
14
|
+
/** Apply and consume one attribution-switch marker when the next envelope arrives. */
|
|
15
|
+
export declare function applyStorySwitchEnvelope(lane: LaneRecord, envelope: LaneEnvelope, at: string, envelopeRef: string): {
|
|
16
|
+
suppressCompletion: boolean;
|
|
17
|
+
clearPending: boolean;
|
|
18
|
+
storySwitchUnacknowledged?: StorySwitchUnacknowledgedNote;
|
|
19
|
+
attributionHistory?: AttributionHistoryEntry[];
|
|
20
|
+
};
|
|
21
|
+
//# sourceMappingURL=story-switch.d.ts.map
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
export function storySwitchNotice(previousProjectId, previousStoryId, projectId, storyId) {
|
|
2
|
+
if (previousProjectId === projectId && previousStoryId === storyId)
|
|
3
|
+
return undefined;
|
|
4
|
+
const from = `${previousProjectId}/${previousStoryId || "(no story)"}`;
|
|
5
|
+
const to = `${projectId}/${storyId || "(no story)"}`;
|
|
6
|
+
return `Story changed: ${from} -> ${to}. The previous story is closed; report only on the new story in this turn's envelope.`;
|
|
7
|
+
}
|
|
8
|
+
export function artifactIdentityKeys(artifacts) {
|
|
9
|
+
if (!Array.isArray(artifacts))
|
|
10
|
+
return [];
|
|
11
|
+
return artifacts.flatMap((artifact) => {
|
|
12
|
+
if (typeof artifact === "string" && artifact.trim())
|
|
13
|
+
return [`value:${artifact.trim()}`];
|
|
14
|
+
if (artifact === null || typeof artifact !== "object" || Array.isArray(artifact))
|
|
15
|
+
return [];
|
|
16
|
+
const record = artifact;
|
|
17
|
+
const url = typeof record.url === "string" ? record.url.trim() : "";
|
|
18
|
+
const path = typeof record.path === "string" ? record.path.trim() : "";
|
|
19
|
+
const values = [url && `url:${url}`, path && `path:${path}`].filter((value) => Boolean(value));
|
|
20
|
+
return values;
|
|
21
|
+
}).sort();
|
|
22
|
+
}
|
|
23
|
+
/** Restore only the attribution write owned by this failed command attempt. */
|
|
24
|
+
export function rollbackStorySwitchAttribution(current, snapshot) {
|
|
25
|
+
if (current.story_switch_pending?.message_id !== snapshot.message_id)
|
|
26
|
+
return current;
|
|
27
|
+
const addedStory = snapshot.added_story;
|
|
28
|
+
const stories = (current.stories ?? []).filter((story) => !(addedStory && story.project_id === addedStory.project_id && story.story_id === addedStory.story_id &&
|
|
29
|
+
story.added_at === addedStory.added_at));
|
|
30
|
+
return {
|
|
31
|
+
...current,
|
|
32
|
+
project_id: snapshot.project_id,
|
|
33
|
+
story_id: snapshot.story_id,
|
|
34
|
+
no_story_reason: snapshot.no_story_reason,
|
|
35
|
+
stories,
|
|
36
|
+
attribution_history: current.attribution_history.filter((entry) => entry.message_id !== snapshot.message_id),
|
|
37
|
+
story_switch_pending: undefined,
|
|
38
|
+
story_switch_unacknowledged: snapshot.story_switch_unacknowledged,
|
|
39
|
+
};
|
|
40
|
+
}
|
|
41
|
+
function sameKeys(left, right) {
|
|
42
|
+
return left.length > 0 && left.length === right.length && left.every((key, index) => key === right[index]);
|
|
43
|
+
}
|
|
44
|
+
/** Apply and consume one attribution-switch marker when the next envelope arrives. */
|
|
45
|
+
export function applyStorySwitchEnvelope(lane, envelope, at, envelopeRef) {
|
|
46
|
+
const pending = lane.story_switch_pending;
|
|
47
|
+
if (!pending)
|
|
48
|
+
return { suppressCompletion: false, clearPending: false };
|
|
49
|
+
if (envelope.story_complete !== true)
|
|
50
|
+
return { suppressCompletion: false, clearPending: false };
|
|
51
|
+
const currentKeys = artifactIdentityKeys(envelope.artifacts);
|
|
52
|
+
const suppressCompletion = sameKeys(pending.previous_artifact_keys, currentKeys);
|
|
53
|
+
if (!suppressCompletion)
|
|
54
|
+
return { suppressCompletion: false, clearPending: true };
|
|
55
|
+
const note = {
|
|
56
|
+
event: "story_switch_unacknowledged",
|
|
57
|
+
previous_project_id: pending.previous_project_id,
|
|
58
|
+
previous_story_id: pending.previous_story_id,
|
|
59
|
+
project_id: pending.project_id,
|
|
60
|
+
story_id: pending.story_id,
|
|
61
|
+
message_id: pending.message_id,
|
|
62
|
+
envelope_ref: envelopeRef,
|
|
63
|
+
at,
|
|
64
|
+
artifact_keys: currentKeys,
|
|
65
|
+
};
|
|
66
|
+
return {
|
|
67
|
+
suppressCompletion: true,
|
|
68
|
+
clearPending: true,
|
|
69
|
+
storySwitchUnacknowledged: note,
|
|
70
|
+
attributionHistory: lane.attribution_history.map((entry) => entry.message_id === pending.message_id
|
|
71
|
+
? { ...entry, story_switch_unacknowledged: note }
|
|
72
|
+
: entry),
|
|
73
|
+
};
|
|
74
|
+
}
|
|
75
|
+
//# sourceMappingURL=story-switch.js.map
|
|
@@ -351,6 +351,27 @@ export interface AttributionHistoryEntry {
|
|
|
351
351
|
story_id: string;
|
|
352
352
|
message_id: string;
|
|
353
353
|
at: string;
|
|
354
|
+
story_switch_unacknowledged?: StorySwitchUnacknowledgedNote;
|
|
355
|
+
}
|
|
356
|
+
export interface StorySwitchPending {
|
|
357
|
+
previous_project_id: string;
|
|
358
|
+
previous_story_id: string;
|
|
359
|
+
project_id: string;
|
|
360
|
+
story_id: string;
|
|
361
|
+
message_id: string;
|
|
362
|
+
at: string;
|
|
363
|
+
previous_artifact_keys: string[];
|
|
364
|
+
}
|
|
365
|
+
export interface StorySwitchUnacknowledgedNote {
|
|
366
|
+
event: "story_switch_unacknowledged";
|
|
367
|
+
previous_project_id: string;
|
|
368
|
+
previous_story_id: string;
|
|
369
|
+
project_id: string;
|
|
370
|
+
story_id: string;
|
|
371
|
+
message_id: string;
|
|
372
|
+
envelope_ref: string;
|
|
373
|
+
at: string;
|
|
374
|
+
artifact_keys: string[];
|
|
354
375
|
}
|
|
355
376
|
export interface LastEnvelopeRef {
|
|
356
377
|
ref: string;
|
|
@@ -573,6 +594,10 @@ export interface LaneRecord {
|
|
|
573
594
|
envelope_summary?: string;
|
|
574
595
|
attribution_history: AttributionHistoryEntry[];
|
|
575
596
|
last_envelope?: LastEnvelopeRef;
|
|
597
|
+
/** Pending attribution change, consumed by the next terminal envelope. */
|
|
598
|
+
story_switch_pending?: StorySwitchPending;
|
|
599
|
+
/** Last completion suppressed because the new story repeated prior evidence. */
|
|
600
|
+
story_switch_unacknowledged?: StorySwitchUnacknowledgedNote;
|
|
576
601
|
/** Effective model id recorded at create/spawn. Omitted when the provider chooses. */
|
|
577
602
|
model?: string;
|
|
578
603
|
/** Where the effective model came from; repaired models stay pinned across the account move. */
|
package/dist/lib/lanes/types.js
CHANGED
|
@@ -227,7 +227,7 @@ export const LANE_FIELD_WRITERS = {
|
|
|
227
227
|
priority: ["command"], worker: ["creator", "command"], worker_path: ["creator", "command"], worker_pack: ["creator", "command"], worker_profile: ["creator", "command"], worker_profile_hash: ["creator", "command", "spawn"], registry_worker: ["creator", "command", "spawn"], registry_worker_approval_history: ["command", "watcher", "reconcile"], worker_profile_repin_history: ["command"], worker_profile_ignored: ["creator", "command", "spawn"], worker_policy_delivery: ["command", "spawn"],
|
|
228
228
|
keep_alive: ["creator", "command"], keep_alive_standing: ["command"], keep_alive_opted_in_at: ["creator", "command"], keep_alive_blocked_reason: ["command", "reconcile"], keep_alive_resume_count: ["command", "reconcile"], keep_alive_history: ["reconcile"], keep_alive_blocked_resume_streak: ["reconcile", "spawn", "watcher"], keep_alive_last_resume_at: ["command", "reconcile"], keep_alive_next_eligible_at: ["command", "reconcile"], keep_alive_exhausted_at: ["command", "reconcile"],
|
|
229
229
|
provider: ["creator"], account: ["creator", "spawn", "command", "watcher", "reconcile"], account_selection: ["creator", "command"], admission_accounts: ["creator", "command"], account_history: ["command", "spawn", "watcher", "reconcile"], failed_resume_history: ["command"], pending_resume: ["command", "spawn", "watcher", "reconcile"], placement_default: ["creator"], host: ["creator", "command"], origin_host: ["creator"], origin_host_source: ["creator"],
|
|
230
|
-
version: ["creator"], created_at: ["creator"], updated_at: ["creator"], ended_at: ["reconcile"], envelope_summary: ["reconcile"], attribution_history: ["command"],
|
|
230
|
+
version: ["creator"], created_at: ["creator"], updated_at: ["creator"], ended_at: ["reconcile"], envelope_summary: ["reconcile"], attribution_history: ["command", "watcher", "reconcile"], story_switch_pending: ["command", "watcher", "reconcile"], story_switch_unacknowledged: ["watcher", "reconcile"],
|
|
231
231
|
last_envelope: ["watcher", "reconcile"], model: ["creator", "spawn", "command"], model_source: ["creator", "spawn", "command"], effort: ["creator", "spawn", "command"], effort_source: ["creator", "spawn", "command"], fast_mode: ["spawn"],
|
|
232
232
|
last_message_at: ["command"], status_gate: ["spawn", "watcher"], status_gate_history: ["watcher", "reconcile"], blocker_freshness_state: ["watcher"], kr_freshness_state: ["watcher"], last_compaction: ["spawn"], story_complete: ["watcher", "reconcile"], story_complete_reason: ["watcher", "reconcile"], story_status: ["watcher", "reconcile"],
|
|
233
233
|
overdue: ["reconcile"], overdue_at: ["reconcile"], last_host_report_at: ["watcher", "spawn", "reconcile"], stalled: ["watcher"], stalled_at: ["watcher"], stalled_reason: ["watcher"],
|