agl 43.10.0 → 43.10.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/changelog.md +8 -0
- package/dist_serve/bundle.js +1 -1
- package/dist_serve/bundle.js.map +2 -2
- package/dist_ts/00_commitinfo_data.js +1 -1
- package/dist_ts/classes.authstore.d.ts +6 -0
- package/dist_ts/classes.authstore.js +8 -1
- package/dist_ts/classes.controller.d.ts +6 -0
- package/dist_ts/classes.controller.js +19 -1
- package/dist_ts/classes.mcpcallerregistry.d.ts +19 -1
- package/dist_ts/classes.mcpcallerregistry.js +33 -7
- package/dist_ts/classes.terminalcallers.d.ts +40 -0
- package/dist_ts/classes.terminalcallers.js +90 -0
- package/dist_ts/classes.terminalmanager.d.ts +20 -1
- package/dist_ts/classes.terminalmanager.js +36 -2
- package/dist_ts/classes.ticketorderspread.d.ts +25 -0
- package/dist_ts/classes.ticketorderspread.js +52 -0
- package/dist_ts/classes.ticketstore.d.ts +27 -0
- package/dist_ts/classes.ticketstore.js +68 -15
- package/dist_ts/classes.workercallers.d.ts +5 -1
- package/dist_ts/classes.workercallers.js +19 -4
- package/dist_ts/fleet/classes.claudeworkers.d.ts +18 -5
- package/dist_ts/fleet/classes.claudeworkers.js +105 -31
- package/dist_ts/fleet/classes.workers.d.ts +5 -3
- package/dist_ts/fleet/classes.workers.js +17 -11
- package/dist_ts/fleet/interfaces.workers.d.ts +16 -2
- package/dist_ts/fleet/interfaces.workers.js +8 -1
- package/dist_ts_interfaces/mcprequests.d.ts +2 -1
- package/dist_ts_interfaces/mcprequests.js +3 -2
- package/dist_ts_migration/classes.documentmigrationrunner.d.ts +8 -1
- package/dist_ts_migration/classes.documentmigrationrunner.js +11 -2
- package/dist_ts_migration/index.d.ts +1 -0
- package/dist_ts_migration/index.js +2 -1
- package/dist_ts_migration/v43_ticketorderstride.d.ts +54 -0
- package/dist_ts_migration/v43_ticketorderstride.js +161 -0
- package/package.json +1 -1
- package/readme.md +18 -2
- package/ts/00_commitinfo_data.ts +1 -1
- package/ts/classes.authstore.ts +9 -0
- package/ts/classes.controller.ts +19 -0
- package/ts/classes.mcpcallerregistry.ts +49 -7
- package/ts/classes.terminalcallers.ts +91 -0
- package/ts/classes.terminalmanager.ts +41 -1
- package/ts/classes.ticketorderspread.ts +50 -0
- package/ts/classes.ticketstore.ts +61 -11
- package/ts/classes.workercallers.ts +19 -3
- package/ts/fleet/classes.claudeworkers.ts +101 -29
- package/ts/fleet/classes.workers.ts +15 -10
- package/ts/fleet/interfaces.workers.ts +16 -2
- package/ts_interfaces/mcprequests.ts +2 -1
- package/ts_migration/classes.documentmigrationrunner.ts +12 -1
- package/ts_migration/index.ts +1 -0
- package/ts_migration/v43_ticketorderstride.ts +156 -0
- package/ts_web/00_commitinfo_data.ts +1 -1
|
@@ -5,7 +5,7 @@ import { dropClaudeHookSwitches } from '../functions.proxyharnessconfig.js';
|
|
|
5
5
|
import type { IFleetWorker, IFleetWorkerProfile, IFleetWorkerReport, IFleetWorkerTurnOutcome, IFleetRelaunchBlocker, IFleetWorkerAbsenceProof, IFleetWorkerSessionProof, IFleetWorkerRebindProof } from '../../ts_interfaces/fleetworkers.js';
|
|
6
6
|
import { controllerSessionFeedExcerpt, type IControllerMessageBundle, type IControllerNativeWait } from '../../ts_interfaces/index.js';
|
|
7
7
|
import { claudePage, claudeTranscript } from '../classes.claudeprojection.js';
|
|
8
|
-
import { FleetDispatchError, FleetNativeTurnError, boundedFleetReport, fleetClaudeNativeWait, fleetMessageDigest, fleetReportMarker, type IFleetWorkerBackend, type IFleetWorkerObservedTurn, type IFleetNativeEvent, type IFleetNativeAgent, type IFleetNativeSession, type IFleetNativePage, type IFleetNativeSendTurn, type IFleetNativeSendReceipt } from './interfaces.workers.js';
|
|
8
|
+
import { FleetDispatchError, FleetInventoryReadError, FleetNativeTurnError, boundedFleetReport, fleetClaudeNativeWait, fleetMessageDigest, fleetReportMarker, type IFleetWorkerBackend, type IFleetWorkerObservedTurn, type IFleetNativeEvent, type IFleetNativeAgent, type IFleetNativeSession, type IFleetNativePage, type IFleetNativeSendTurn, type IFleetNativeSendReceipt, type TFleetWorkerPollPass } from './interfaces.workers.js';
|
|
9
9
|
import { runFleetWorkerCommand, type IFleetWorkerCommand, type TFleetWorkerOutputWatch, type TFleetWorkerRunner } from './classes.workercommand.js';
|
|
10
10
|
import { sanitizeFailureText } from '../functions.failuredescription.js';
|
|
11
11
|
import type { FleetWorkerLaunches } from './classes.workerlaunches.js';
|
|
@@ -70,20 +70,38 @@ const status = (agent?: TClaudeAgent): IFleetWorkerReport['status'] => !agent ?
|
|
|
70
70
|
* Claude Code does not mark as its own (`isMeta`) and that holds text, not only tool results; named by its uuid, as
|
|
71
71
|
* turn endings are.
|
|
72
72
|
*/
|
|
73
|
+
const claudePromptOf = (entry: Record<string, unknown>): { id: string; startedAt?: string } | undefined => {
|
|
74
|
+
const id = entry.uuid ?? entry.timestamp;
|
|
75
|
+
if (entry.type !== 'user' || entry.isSidechain === true || entry.isMeta === true || typeof id !== 'string' || !id) return undefined;
|
|
76
|
+
const content = record.safeParse(entry.message).data?.content;
|
|
77
|
+
if (typeof content !== 'string' && !(Array.isArray(content) && content.some((part) => {
|
|
78
|
+
const type = record.safeParse(part).data?.type;
|
|
79
|
+
return typeof type === 'string' && type !== 'tool_result';
|
|
80
|
+
}))) return undefined;
|
|
81
|
+
const at = typeof entry.timestamp === 'string' ? Date.parse(entry.timestamp) : Number.NaN;
|
|
82
|
+
return { id, ...(Number.isFinite(at) ? { startedAt: new Date(at).toISOString() } : {}) };
|
|
83
|
+
};
|
|
73
84
|
const claudePrompt = (records: Record<string, unknown>[]): { id: string; startedAt?: string } | undefined => {
|
|
74
85
|
for (let index = records.length - 1; index >= 0; index -= 1) {
|
|
75
|
-
const
|
|
76
|
-
if (
|
|
77
|
-
const content = record.safeParse(entry.message).data?.content;
|
|
78
|
-
if (typeof content !== 'string' && !(Array.isArray(content) && content.some((part) => {
|
|
79
|
-
const type = record.safeParse(part).data?.type;
|
|
80
|
-
return typeof type === 'string' && type !== 'tool_result';
|
|
81
|
-
}))) continue;
|
|
82
|
-
const at = typeof entry.timestamp === 'string' ? Date.parse(entry.timestamp) : Number.NaN;
|
|
83
|
-
return { id, ...(Number.isFinite(at) ? { startedAt: new Date(at).toISOString() } : {}) };
|
|
86
|
+
const prompt = claudePromptOf(records[index]!);
|
|
87
|
+
if (prompt) return prompt;
|
|
84
88
|
}
|
|
85
89
|
return undefined;
|
|
86
90
|
};
|
|
91
|
+
/**
|
|
92
|
+
* The turn endings Claude Code recorded among transcript records, `system`/`turn_duration`, in order: each ends the turn
|
|
93
|
+
* the newest prompt before it opened, or the turn `previous` while no prompt precedes it among `records`. `record` names
|
|
94
|
+
* the ending record by its uuid.
|
|
95
|
+
*/
|
|
96
|
+
const claudeTurnEndings = (records: Record<string, unknown>[], previous: string): Array<{ turn: string; record: string }> => {
|
|
97
|
+
const endings: Array<{ turn: string; record: string }> = [];
|
|
98
|
+
let turn = previous;
|
|
99
|
+
for (const entry of records) {
|
|
100
|
+
turn = claudePromptOf(entry)?.id ?? turn;
|
|
101
|
+
if (entry.type === 'system' && entry.subtype === 'turn_duration') endings.push({ turn, record: String(entry.uuid ?? entry.timestamp) });
|
|
102
|
+
}
|
|
103
|
+
return endings;
|
|
104
|
+
};
|
|
87
105
|
/** How a Claude Code job's turn ended, as its inventory lists the job: failed, or done. */
|
|
88
106
|
const claudeOutcome = (agent: TClaudeAgent | undefined, turnId: string): IFleetWorkerTurnOutcome | undefined => (
|
|
89
107
|
agent?.state === 'failed' ? { turnId, state: 'failed' } : agent?.state === 'done' ? { turnId, state: 'completed' } : undefined);
|
|
@@ -278,17 +296,29 @@ export const fleetClaudeRelayFailure = (result: IFleetWorkerCommand): FleetDispa
|
|
|
278
296
|
return new FleetDispatchError(`Claude -p failed (exit ${result.code})${unsent ? ' before it sent any message' : ''}${output ? `: ${output}` : '.'}`, !unsent);
|
|
279
297
|
};
|
|
280
298
|
|
|
281
|
-
/**
|
|
299
|
+
/**
|
|
300
|
+
* `offset` is absent once the poll baseline was forgotten: the next poll reads afresh but keeps the observed state, turn
|
|
301
|
+
* and wait. `ended` is the newest turn end a poll published: the turn (`turn`'s id) and the transcript's ending record,
|
|
302
|
+
* absent while only the inventory showed it, so a turn's end is published once whichever shows it first. `since`, set by
|
|
303
|
+
* a start or wake, is the newest inventory read begun by then: that read and every older one cannot show the job it
|
|
304
|
+
* started or woke, so a poll never observes it from one (AGL-1970).
|
|
305
|
+
*/
|
|
282
306
|
interface IClaudeObservation {
|
|
283
307
|
offset?: number; state?: IFleetWorkerReport['status']; turnState?: string; path?: string; excerpt?: string; turn?: IFleetWorkerObservedTurn;
|
|
284
|
-
nativeWait?: IControllerNativeWait;
|
|
308
|
+
nativeWait?: IControllerNativeWait; ended?: { turn: string; record?: string }; since?: number;
|
|
285
309
|
}
|
|
310
|
+
/** One read of a profile home's inventory: when and as which read it began (`serial`, counting this backend's reads). */
|
|
311
|
+
interface IClaudeInventoryRead { at: number; serial: number; agents: Promise<TClaudeAgent[]> }
|
|
286
312
|
/** The SDK's transcript chain reader, which builds a conversation from transcript records. */
|
|
287
313
|
export type TFleetClaudeTranscriptReader = Pick<typeof plugins.claudeAgentSdk, 'getSessionMessages'>;
|
|
288
314
|
export class FleetClaudeWorkers implements IFleetWorkerBackend {
|
|
289
315
|
private readonly observations = new Map<string, IClaudeObservation>();
|
|
290
316
|
private readonly versions = new Map<string, string>();
|
|
291
|
-
private readonly inventories = new Map<string,
|
|
317
|
+
private readonly inventories = new Map<string, IClaudeInventoryRead>();
|
|
318
|
+
/** Per poll pass, the inventory read its workers share, per profile home (AGL-1970). */
|
|
319
|
+
private readonly passes = new WeakMap<TFleetWorkerPollPass, Map<string, IClaudeInventoryRead>>();
|
|
320
|
+
/** The inventory reads begun so far, which number them (`IClaudeInventoryRead.serial`). */
|
|
321
|
+
private reads = 0;
|
|
292
322
|
constructor(
|
|
293
323
|
private readonly signal: AbortSignal,
|
|
294
324
|
private readonly runner: TFleetWorkerRunner = runFleetWorkerCommand,
|
|
@@ -382,17 +412,41 @@ export class FleetClaudeWorkers implements IFleetWorkerBackend {
|
|
|
382
412
|
if (starting) await this.trustedWorkspace(profile, worker.cwd);
|
|
383
413
|
}
|
|
384
414
|
/**
|
|
385
|
-
* The profile home's inventory. The fleet reads it fresh
|
|
386
|
-
* old, the fleet's own included, and concurrent readers share one read.
|
|
415
|
+
* The profile home's inventory. The fleet reads it fresh, a poll pass once for all of its workers (`passAgents`); an
|
|
416
|
+
* outside observer accepts a read at most `maxAgeMs` old, the fleet's own included, and concurrent readers share one read.
|
|
387
417
|
*/
|
|
388
418
|
private async agents(worker: Pick<IFleetWorker, 'cwd'>, profile: Pick<IFleetWorkerProfile, 'home'>, maxAgeMs = 0): Promise<TClaudeAgent[]> {
|
|
389
|
-
const
|
|
419
|
+
const shared = this.inventories.get(profile.home ?? '');
|
|
390
420
|
if (shared && maxAgeMs > 0 && Date.now() - shared.at <= maxAgeMs) return await shared.agents;
|
|
391
|
-
|
|
421
|
+
return await this.read(worker, profile).agents;
|
|
422
|
+
}
|
|
423
|
+
/** A fresh read of the profile home's inventory, which outside observers then share for their age. */
|
|
424
|
+
private read(worker: Pick<IFleetWorker, 'cwd'>, profile: Pick<IFleetWorkerProfile, 'home'>): IClaudeInventoryRead {
|
|
425
|
+
const key = profile.home ?? '';
|
|
426
|
+
const read = { at: Date.now(), serial: ++this.reads, agents: this.command(profile, worker.cwd, ['agents', '--json', '--all'])
|
|
392
427
|
.then((output) => plugins.z.array(agentSchema).max(4096).parse(JSON.parse(output))) };
|
|
393
428
|
this.inventories.set(key, read);
|
|
394
429
|
read.agents.catch(() => { if (this.inventories.get(key) === read) this.inventories.delete(key); });
|
|
395
|
-
return
|
|
430
|
+
return read;
|
|
431
|
+
}
|
|
432
|
+
/**
|
|
433
|
+
* The inventory a poll observes from (AGL-1970): with `pass`, the read the pass's workers of the profile home share,
|
|
434
|
+
* begun once per home and pass, since it lists every job of the home; without, a fresh read. A failed read fails the
|
|
435
|
+
* pass's workers of its home alike, as a `FleetInventoryReadError`, which keeps their baselines, and is not read again
|
|
436
|
+
* in the pass; the next pass reads again.
|
|
437
|
+
*/
|
|
438
|
+
private async passAgents(pass: TFleetWorkerPollPass | undefined, worker: Pick<IFleetWorker, 'cwd'>, profile: Pick<IFleetWorkerProfile, 'home'>): Promise<{ serial: number; agents: TClaudeAgent[] }> {
|
|
439
|
+
let read: IClaudeInventoryRead;
|
|
440
|
+
if (!pass) read = this.read(worker, profile);
|
|
441
|
+
else {
|
|
442
|
+
const key = profile.home ?? '';
|
|
443
|
+
let byHome = this.passes.get(pass);
|
|
444
|
+
if (!byHome) { byHome = new Map(); this.passes.set(pass, byHome); }
|
|
445
|
+
read = byHome.get(key) ?? this.read(worker, profile);
|
|
446
|
+
byHome.set(key, read);
|
|
447
|
+
}
|
|
448
|
+
try { return { serial: read.serial, agents: await read.agents }; }
|
|
449
|
+
catch (error) { throw new FleetInventoryReadError(error instanceof Error ? error.message : 'Claude inventory read failed.', { cause: error }); }
|
|
396
450
|
}
|
|
397
451
|
public async inventory(home: string | undefined, cwd: string, maxAgeMs: number): Promise<IFleetNativeAgent[]> {
|
|
398
452
|
return await this.agents({ cwd }, { home }, maxAgeMs);
|
|
@@ -619,7 +673,7 @@ export class FleetClaudeWorkers implements IFleetWorkerBackend {
|
|
|
619
673
|
if (!observed) {
|
|
620
674
|
const observation = await options.observation(agent);
|
|
621
675
|
if (!observation) { await plugins.timersPromises.setTimeout(250, undefined, { signal: this.signal }); continue; }
|
|
622
|
-
this.observations.set(agent.sessionId, observation); observed = true;
|
|
676
|
+
this.observations.set(agent.sessionId, { ...observation, since: this.reads }); observed = true;
|
|
623
677
|
}
|
|
624
678
|
// The caller launch binds to the job's process; a job still spawning it shows no pid yet.
|
|
625
679
|
if (this.callers && !live(agent) && agent.state === 'working') { await plugins.timersPromises.setTimeout(250, undefined, { signal: this.signal }); continue; }
|
|
@@ -714,16 +768,20 @@ export class FleetClaudeWorkers implements IFleetWorkerBackend {
|
|
|
714
768
|
return undefined;
|
|
715
769
|
} finally { await file.close(); }
|
|
716
770
|
}
|
|
717
|
-
public async observe(worker: IFleetWorker, profile: IFleetWorkerProfile): Promise<IFleetWorkerReport> {
|
|
718
|
-
const agent = await this.findAgent(worker, profile), transcript = await this.transcript(worker, profile);
|
|
771
|
+
public async observe(worker: IFleetWorker, profile: IFleetWorkerProfile, pass?: TFleetWorkerPollPass): Promise<IFleetWorkerReport> {
|
|
772
|
+
const agent = await this.findAgent(worker, profile, pass ? (await this.passAgents(pass, worker, profile)).agents : undefined), transcript = await this.transcript(worker, profile);
|
|
719
773
|
// A turn may continue past its report; the newest report outranks later chatter.
|
|
720
774
|
const texts = fleetClaudeAssistantTexts(transcript.records), latest = texts.findLast((item) => fleetReportMarker.test(item.text)) ?? texts.at(-1);
|
|
721
775
|
const outcome = claudeObservedTurn(worker.nativeId!, agent, transcript.records, this.observations.get(worker.nativeId!)?.turn).outcome;
|
|
722
776
|
return { worker, status: status(agent), ...boundedFleetReport(latest?.text ?? ''), ...(latest ? { reportId: latest.id } : {}),
|
|
723
777
|
...(agent ? { turnStatus: agent.state ?? agent.status } : {}), ...(outcome ? { outcome } : {}) };
|
|
724
778
|
}
|
|
725
|
-
public async poll(worker: IFleetWorker, profile: IFleetWorkerProfile): Promise<IFleetNativeEvent[]> {
|
|
726
|
-
const
|
|
779
|
+
public async poll(worker: IFleetWorker, profile: IFleetWorkerProfile, pass?: TFleetWorkerPollPass): Promise<IFleetNativeEvent[]> {
|
|
780
|
+
const read = await this.passAgents(pass, worker, profile), observation = this.observations.get(worker.nativeId!);
|
|
781
|
+
// A start or wake set the observation after the read began, which then cannot show the job it started or woke and
|
|
782
|
+
// would end its new turn at once: the worker waits for the next pass's read, its observation untouched (AGL-1970).
|
|
783
|
+
if (observation?.since !== undefined && observation.since >= read.serial) return [];
|
|
784
|
+
const agent = await this.findAgent(worker, profile, read.agents);
|
|
727
785
|
if (!agent) throw new Error('Claude background worker is absent from its inventory; its reaped state is not proven.');
|
|
728
786
|
const current = status(agent), previous = observation?.offset === undefined ? undefined : observation;
|
|
729
787
|
const tail = await this.transcript(worker, profile, previous?.offset);
|
|
@@ -731,16 +789,28 @@ export class FleetClaudeWorkers implements IFleetWorkerBackend {
|
|
|
731
789
|
const texts = fleetClaudeAssistantTexts(tail.records);
|
|
732
790
|
const excerpt = controllerSessionFeedExcerpt(texts.at(-1)?.text) ?? observation?.excerpt;
|
|
733
791
|
const turn = claudeObservedTurn(worker.nativeId!, agent, tail.records, observation?.turn), nativeWait = fleetClaudeNativeWait(agent);
|
|
792
|
+
// A turn ends once (`ended`): by its transcript's ending record, each of which ends a turn of its own, or by the
|
|
793
|
+
// inventory as the job stops working while the transcript shows no ending, whichever a poll sees first. The record
|
|
794
|
+
// of an end the inventory showed first, and the inventory's sight of an end a record showed, publish nothing.
|
|
795
|
+
const endings = previous ? claudeTurnEndings(tail.records, observation?.turn?.id ?? worker.nativeId!) : [], ends: IFleetNativeEvent[] = [];
|
|
796
|
+
let ended = observation?.ended;
|
|
797
|
+
for (const ending of endings) {
|
|
798
|
+
const seen = ended !== undefined && ended.record === undefined && ended.turn === ending.turn;
|
|
799
|
+
ended = ending;
|
|
800
|
+
if (!seen) ends.push({ nativeId: worker.nativeId!, kind: 'worker.turn.ended', turnId: ending.record, status: current });
|
|
801
|
+
}
|
|
802
|
+
if (previous && !endings.length && previous.turnState === 'working' && agent.state !== 'working' && ended?.turn !== turn.id) {
|
|
803
|
+
ended = { turn: turn.id };
|
|
804
|
+
ends.push({ nativeId: worker.nativeId!, kind: 'worker.turn.ended', status: current });
|
|
805
|
+
}
|
|
734
806
|
this.observations.set(worker.nativeId!, { offset: tail.end, state: current, turnState: agent.state, path: tail.path, ...(excerpt ? { excerpt } : {}), turn,
|
|
735
|
-
...(nativeWait ? { nativeWait } : {}) });
|
|
807
|
+
...(nativeWait ? { nativeWait } : {}), ...(ended ? { ended } : {}) });
|
|
736
808
|
const events: IFleetNativeEvent[] = current === 'reaped' && observation?.state !== 'reaped'
|
|
737
809
|
? [{ nativeId: worker.nativeId!, kind: 'worker.reaped', status: 'reaped' }] : [];
|
|
738
810
|
if (!previous) return events;
|
|
739
811
|
events.push(...texts.filter((item) => fleetReportMarker.test(item.text)).map((item) => ({ nativeId: worker.nativeId!, kind: 'worker.report.mid_turn' as const, reportId: item.id, ...boundedFleetReport(item.text), status: current })));
|
|
740
812
|
events.push(...fleetClaudeCredentialFailures(tail.records).map((item) => ({ nativeId: worker.nativeId!, kind: 'worker.credential.failed' as const, reportId: item.id, error: item.error, ...boundedFleetReport(item.text), status: current })));
|
|
741
|
-
|
|
742
|
-
for (const entry of endings) events.push({ nativeId: worker.nativeId!, kind: 'worker.turn.ended', turnId: String(entry.uuid ?? entry.timestamp), status: current });
|
|
743
|
-
if (!endings.length && previous.turnState === 'working' && agent.state !== 'working') events.push({ nativeId: worker.nativeId!, kind: 'worker.turn.ended', status: current });
|
|
813
|
+
events.push(...ends);
|
|
744
814
|
return events;
|
|
745
815
|
}
|
|
746
816
|
/**
|
|
@@ -755,7 +825,8 @@ export class FleetClaudeWorkers implements IFleetWorkerBackend {
|
|
|
755
825
|
await this.callers?.bindClaude(worker, worker.nativeId!, agent);
|
|
756
826
|
const previous = this.observations.get(worker.nativeId!);
|
|
757
827
|
this.observations.set(worker.nativeId!, { offset: previous?.offset ?? baseline.end, path: previous?.path ?? baseline.path, state: 'busy', turnState: 'working',
|
|
758
|
-
...(previous?.excerpt ? { excerpt: previous.excerpt } : {}), ...(previous?.turn ? { turn: previous.turn } : {}) })
|
|
828
|
+
...(previous?.excerpt ? { excerpt: previous.excerpt } : {}), ...(previous?.turn ? { turn: previous.turn } : {}), ...(previous?.ended ? { ended: previous.ended } : {}),
|
|
829
|
+
since: this.reads });
|
|
759
830
|
} catch (error) {
|
|
760
831
|
await this.callers?.revoke(worker, 'failure');
|
|
761
832
|
throw new FleetDispatchError(`Claude revival was acknowledged but reconciliation failed: ${error instanceof Error ? error.message : 'unknown failure.'} Inspect before retrying.`, true, { cause: error });
|
|
@@ -975,7 +1046,8 @@ export class FleetClaudeWorkers implements IFleetWorkerBackend {
|
|
|
975
1046
|
public forget(worker: IFleetWorker): void {
|
|
976
1047
|
const observation = worker.nativeId ? this.observations.get(worker.nativeId) : undefined;
|
|
977
1048
|
if (observation) this.observations.set(worker.nativeId!, { state: observation.state, turnState: observation.turnState, ...(observation.excerpt ? { excerpt: observation.excerpt } : {}),
|
|
978
|
-
...(observation.turn ? { turn: observation.turn } : {}), ...(observation.nativeWait ? { nativeWait: observation.nativeWait } : {})
|
|
1049
|
+
...(observation.turn ? { turn: observation.turn } : {}), ...(observation.nativeWait ? { nativeWait: observation.nativeWait } : {}),
|
|
1050
|
+
...(observation.ended ? { ended: observation.ended } : {}), ...(observation.since !== undefined ? { since: observation.since } : {}) });
|
|
979
1051
|
}
|
|
980
1052
|
public observed(worker: IFleetWorker): IFleetWorkerReport['status'] | undefined {
|
|
981
1053
|
return worker.nativeId ? this.observations.get(worker.nativeId)?.state : undefined;
|
|
@@ -11,7 +11,7 @@ import { FleetEvents } from './classes.events.js';
|
|
|
11
11
|
import { FleetClaudeWorkers } from './classes.claudeworkers.js';
|
|
12
12
|
import { FleetCodexWorkers } from './classes.codexworkers.js';
|
|
13
13
|
import type { FleetWorkerLaunches } from './classes.workerlaunches.js';
|
|
14
|
-
import { FleetDispatchError, FleetWorkerError, fleetMessageDigest, fleetReportMarker, type IFleetWorkerBackend, type IFleetWorkerObservedTurn, type IFleetNativeAgent, type IFleetNativeThread, type IFleetNativeSession, type IFleetNativePendingAttention, type IFleetNativePage, type IFleetNativeSendTurn, type IFleetNativeSendReceipt, type TFleetNativeReply } from './interfaces.workers.js';
|
|
14
|
+
import { FleetDispatchError, FleetInventoryReadError, FleetWorkerError, fleetMessageDigest, fleetReportMarker, type IFleetWorkerBackend, type IFleetWorkerObservedTurn, type IFleetNativeAgent, type IFleetNativeThread, type IFleetNativeSession, type IFleetNativePendingAttention, type IFleetNativePage, type IFleetNativeSendTurn, type IFleetNativeSendReceipt, type TFleetNativeReply, type TFleetWorkerPollPass } from './interfaces.workers.js';
|
|
15
15
|
import { assertFleetWorkerRequest, validateFleetWorkerProfiles } from './functions.workervalidation.js';
|
|
16
16
|
import { readControllerProcessIdentity, controllerProcessIdentityIsLive } from '../classes.processinspection.js';
|
|
17
17
|
import { ticketActorSchema } from '../functions.ticketvalidation.js';
|
|
@@ -143,11 +143,13 @@ export class FleetWorkers {
|
|
|
143
143
|
const observable = documents.filter(({ worker }) => worker.nativeId && !worker.lostDispatch && worker.state !== 'archived' && (!worker.pending || worker.state === 'unknown'));
|
|
144
144
|
// A worker no longer observed, archived for example, keeps no remembered failure.
|
|
145
145
|
for (const id of this.observationFailures.keys()) if (!observable.some(({ worker }) => worker.id === id)) this.observationFailures.delete(id);
|
|
146
|
+
// A backend reads what all of the pass's workers share, Claude Code's inventory of a profile home, once (AGL-1970).
|
|
147
|
+
const pass: TFleetWorkerPollPass = {};
|
|
146
148
|
for (const { owner, worker } of observable) {
|
|
147
149
|
try {
|
|
148
150
|
const backend = this.backends[worker.harness], observed = backend.observed?.(worker), excerpt = backend.observedExcerpt?.(worker);
|
|
149
151
|
const turnAndWait = JSON.stringify([backend.observedTurn?.(worker) ?? null, backend.observedNativeWait?.(worker) ?? null]);
|
|
150
|
-
const events = await this.backends[worker.harness].poll(worker, worker.profile);
|
|
152
|
+
const events = await this.backends[worker.harness].poll(worker, worker.profile, pass);
|
|
151
153
|
const ticketed = this.options.ticketEffects && worker.ticketKey && worker.ticketActor;
|
|
152
154
|
for (const { nativeId: _nativeId, ...event } of events) {
|
|
153
155
|
const published = { ...event, workerId: worker.id, workerName: worker.name, ...(worker.ticketKey ? { ticketKey: worker.ticketKey } : {}) };
|
|
@@ -158,7 +160,7 @@ export class FleetWorkers {
|
|
|
158
160
|
if (ticketed && event.kind === 'worker.turn.ended') {
|
|
159
161
|
let report: IFleetWorkerReport;
|
|
160
162
|
// The turn's end is published whether or not its report can be observed.
|
|
161
|
-
try { report = await this.backends[worker.harness].observe(worker, worker.profile); }
|
|
163
|
+
try { report = await this.backends[worker.harness].observe(worker, worker.profile, pass); }
|
|
162
164
|
catch (error) { this.events.publish(published, owner); throw error; }
|
|
163
165
|
if (report.reportId && report.text.trim()) { await this.reportTicket(owner, worker, report, published); continue; }
|
|
164
166
|
}
|
|
@@ -168,7 +170,7 @@ export class FleetWorkers {
|
|
|
168
170
|
const baseline = `${worker.id}:${worker.nativeId}:${worker.ticketKey}`;
|
|
169
171
|
if (!this.reportBaselines.has(baseline)) {
|
|
170
172
|
if (!events.some(event => event.kind === 'worker.report.mid_turn' || event.kind === 'worker.turn.ended')) {
|
|
171
|
-
const report = await this.backends[worker.harness].observe(worker, worker.profile);
|
|
173
|
+
const report = await this.backends[worker.harness].observe(worker, worker.profile, pass);
|
|
172
174
|
if (report.reportId && report.text.trim() && (report.status !== 'busy' && report.status !== 'gone' || fleetReportMarker.test(report.text))) await this.reportTicket(owner, worker, report);
|
|
173
175
|
}
|
|
174
176
|
this.reportBaselines.add(baseline);
|
|
@@ -178,7 +180,7 @@ export class FleetWorkers {
|
|
|
178
180
|
if (backend.observed?.(worker) !== observed || backend.observedExcerpt?.(worker) !== excerpt
|
|
179
181
|
|| JSON.stringify([backend.observedTurn?.(worker) ?? null, backend.observedNativeWait?.(worker) ?? null]) !== turnAndWait) for (const listener of this.viewListeners) listener();
|
|
180
182
|
this.observationFailures.delete(worker.id);
|
|
181
|
-
} catch (error) { await this.observationFailed(owner, worker, error); }
|
|
183
|
+
} catch (error) { await this.observationFailed(owner, worker, error, pass); }
|
|
182
184
|
if (this.abort.signal.aborted) return;
|
|
183
185
|
}
|
|
184
186
|
if (this.options.sample && Date.now() - this.lastLoad >= 10_000) {
|
|
@@ -188,19 +190,22 @@ export class FleetWorkers {
|
|
|
188
190
|
}
|
|
189
191
|
}
|
|
190
192
|
/**
|
|
191
|
-
* A failed observation is published once, and the fleet then requests a
|
|
192
|
-
* baseline, so the next poll observes the worker afresh, and the newest report is published now. The
|
|
193
|
-
* later polls stays quiet until an observation succeeds or the failure changes.
|
|
193
|
+
* A failed observation is published once, and the fleet then requests a report itself, in the same poll pass: the
|
|
194
|
+
* backend drops its baseline, so the next poll observes the worker afresh, and the newest report is published now. The
|
|
195
|
+
* same failure on later polls stays quiet until an observation succeeds or the failure changes. A failed read of what
|
|
196
|
+
* the pass shares (`FleetInventoryReadError`) says nothing about the worker: its baseline stays, nothing is read again in
|
|
197
|
+
* the pass, and the next pass that reads observes everything since the baseline (AGL-1970).
|
|
194
198
|
*/
|
|
195
|
-
private async observationFailed(owner: string, worker: IFleetWorker, cause: unknown): Promise<void> {
|
|
199
|
+
private async observationFailed(owner: string, worker: IFleetWorker, cause: unknown, pass: TFleetWorkerPollPass): Promise<void> {
|
|
196
200
|
const error = this.error(cause), ticket = worker.ticketKey ? { ticketKey: worker.ticketKey } : {};
|
|
197
201
|
if (this.observationFailures.get(worker.id) === error) return;
|
|
198
202
|
this.observationFailures.set(worker.id, error);
|
|
199
203
|
this.events.publish({ kind: 'worker.observation.failed', workerId: worker.id, workerName: worker.name, ...ticket, error }, owner);
|
|
204
|
+
if (cause instanceof FleetInventoryReadError) return;
|
|
200
205
|
const backend = this.backends[worker.harness];
|
|
201
206
|
backend.forget?.(worker);
|
|
202
207
|
let report: IFleetWorkerReport;
|
|
203
|
-
try { report = await backend.observe(worker, worker.profile); } catch { return; }
|
|
208
|
+
try { report = await backend.observe(worker, worker.profile, pass); } catch { return; }
|
|
204
209
|
if (!report.reportId || !report.text.trim() || !fleetReportMarker.test(report.text)) return;
|
|
205
210
|
const event: Omit<IFleetEvent, 'id' | 'occurredAt'> = { kind: 'worker.report.mid_turn', workerId: worker.id, workerName: worker.name, ...ticket, reportId: report.reportId, text: report.text, truncated: report.truncated, status: report.status };
|
|
206
211
|
if (this.options.ticketEffects && worker.ticketKey && worker.ticketActor) await this.reportTicket(owner, worker, report, event);
|
|
@@ -24,6 +24,13 @@ export class FleetWorkerError extends Error {
|
|
|
24
24
|
export class FleetNativeTurnError extends Error {
|
|
25
25
|
constructor(public readonly code: 'concurrent_change' | 'turn_ended' | 'delivery_unsupported', message: string) { super(message); this.name = 'FleetNativeTurnError'; }
|
|
26
26
|
}
|
|
27
|
+
/**
|
|
28
|
+
* A poll failed because a read of native state its pass shares failed, a profile home's inventory (AGL-1970): it says
|
|
29
|
+
* nothing about the worker, so the fleet keeps the worker's baseline and the next pass observes everything since it.
|
|
30
|
+
*/
|
|
31
|
+
export class FleetInventoryReadError extends Error {
|
|
32
|
+
constructor(message: string, options?: ErrorOptions) { super(message, options); this.name = 'FleetInventoryReadError'; }
|
|
33
|
+
}
|
|
27
34
|
/**
|
|
28
35
|
* How a native send treats the turn the session runs. With `expectedTurnId` the text steers only that running turn. An
|
|
29
36
|
* `agentMessage` steers a turn another client runs only when `expectedTurnId` names it, as an agent message must.
|
|
@@ -80,6 +87,11 @@ export interface IFleetNativePendingAttention {
|
|
|
80
87
|
export interface IFleetNativeSession { nativeId: string; cwd: string; home?: string; jobDelivery?: TControllerNativeJobDelivery; peerHomes?: string[] }
|
|
81
88
|
/** A transcript page read-only: `nextCursor` pages to older bundles, and `earlier` says older bundles exist at all. */
|
|
82
89
|
export interface IFleetNativePage { bundles: IControllerMessageBundle[]; earlier: boolean; nextCursor?: string }
|
|
90
|
+
/**
|
|
91
|
+
* One pass of the fleet's poll over its observable workers, a new object for each pass. A backend may read native state
|
|
92
|
+
* that all of the pass's workers share once per pass, keyed by this object, and never keeps it beyond the pass (AGL-1970).
|
|
93
|
+
*/
|
|
94
|
+
export type TFleetWorkerPollPass = object;
|
|
83
95
|
export interface IFleetWorkerBackend {
|
|
84
96
|
/**
|
|
85
97
|
* Fresh native absence proof; failures and ambiguous state must throw. Sends nothing.
|
|
@@ -106,8 +118,10 @@ export interface IFleetWorkerBackend {
|
|
|
106
118
|
relaunch?(worker: IFleetWorker, profile: IFleetWorkerProfile, nativeId: string, prompt: string, bind: (nativeId: string) => Promise<void>): Promise<void>;
|
|
107
119
|
/** Why `relaunch` of `nativeId` as a worker in `cwd` would refuse now, by the same checks, read-only; `undefined` when none does. */
|
|
108
120
|
relaunchBlocker?(cwd: string, profile: IFleetWorkerProfile, nativeId: string): Promise<IFleetRelaunchBlocker | undefined>;
|
|
109
|
-
|
|
110
|
-
|
|
121
|
+
/** Observes the worker since its last poll; `pass` is the poll pass it is observed in, absent for a read of its own. */
|
|
122
|
+
poll(worker: IFleetWorker, profile: IFleetWorkerProfile, pass?: TFleetWorkerPollPass): Promise<IFleetNativeEvent[]>;
|
|
123
|
+
/** The worker's newest report; `pass` is the poll pass that asks for it, absent for a read of its own. */
|
|
124
|
+
observe(worker: IFleetWorker, profile: IFleetWorkerProfile, pass?: TFleetWorkerPollPass): Promise<IFleetWorkerReport>;
|
|
111
125
|
send(messages: Array<{ worker: IFleetWorker; profile: IFleetWorkerProfile; text: string }>): Promise<void>;
|
|
112
126
|
stop(worker: IFleetWorker, profile: IFleetWorkerProfile): Promise<void>;
|
|
113
127
|
archive(worker: IFleetWorker, profile: IFleetWorkerProfile): Promise<void>;
|
|
@@ -132,7 +132,8 @@ export const controllerMcpSessionDetailFields = {
|
|
|
132
132
|
* harness process it spawns. Every `agl mcp` server that process launches inherits it and presents
|
|
133
133
|
* it on the private endpoint, so the controller authorizes the calling chat instead of trusting a
|
|
134
134
|
* caller-supplied task id. It is never allowlisted for inheritance: only explicit per-spawn writes
|
|
135
|
-
* may set it.
|
|
135
|
+
* may set it. Everything a terminal runs inherits it, daemons that leave the terminal too, so a
|
|
136
|
+
* terminal's is honoured only for a process still running inside that terminal (AGL-1891).
|
|
136
137
|
*/
|
|
137
138
|
export const controllerMcpCallerCredentialEnvironmentVariable = 'AGL_MCP_CALLER_CREDENTIAL';
|
|
138
139
|
/** Optional browser-minted owner capability; when absent, the CLI mints one through same-user controller database access. */
|
|
@@ -15,7 +15,9 @@ import { TicketOwnerPrincipalV38Migration } from './v38_ticketownerprincipal.js'
|
|
|
15
15
|
import { TicketClosedColumnV41Migration } from './v41_ticketclosedcolumn.js';
|
|
16
16
|
import { TicketLinkReposV41Migration } from './v41_ticketlinkrepos.js';
|
|
17
17
|
import { TicketActorNamesV43Migration } from './v43_ticketactornames.js';
|
|
18
|
+
import { TicketOrderStrideV43Migration } from './v43_ticketorderstride.js';
|
|
18
19
|
import type { IControllerIssuerIdentity } from '../ts/interfaces.identity.js';
|
|
20
|
+
import type { TicketStore } from '../ts/classes.ticketstore.js';
|
|
19
21
|
|
|
20
22
|
interface IDocumentMigration {
|
|
21
23
|
run(): Promise<void>;
|
|
@@ -58,7 +60,8 @@ export class PostDatabaseDocumentMigrationRunner {
|
|
|
58
60
|
/**
|
|
59
61
|
* The ticket migrations of a controller start, in their order; a new one goes last. v38 gives owner assignments the
|
|
60
62
|
* store's principal, v41 moves tickets released by hand without a release link to closed, and v43 names ticket actors
|
|
61
|
-
* by their role (AGL-1686). Answers the keys v41 moved and those whose revision v43 moved.
|
|
63
|
+
* by their role (AGL-1686). Answers the keys v41 moved and those whose revision v43 moved. The order spread (AGL-1907)
|
|
64
|
+
* is not one of them: it runs after the controller is ready (`runTicketOrderStride`).
|
|
62
65
|
*/
|
|
63
66
|
public async runTicketStart(principalIdArg: string): Promise<{ closed: string[]; renamed: string[] }> {
|
|
64
67
|
await this.runTicketOwnerPrincipal(principalIdArg);
|
|
@@ -84,6 +87,14 @@ export class PostDatabaseDocumentMigrationRunner {
|
|
|
84
87
|
return await new TicketActorNamesV43Migration(this.database).run();
|
|
85
88
|
}
|
|
86
89
|
|
|
90
|
+
/**
|
|
91
|
+
* AGL-1907: once, every column's tickets take orders a stride apart in their board order, through `storeArg`'s steps
|
|
92
|
+
* on the live board, until `signalArg` ends it; answers the keys it moved.
|
|
93
|
+
*/
|
|
94
|
+
public async runTicketOrderStride(storeArg: TicketStore, signalArg: AbortSignal): Promise<string[]> {
|
|
95
|
+
return await new TicketOrderStrideV43Migration(this.database, storeArg).run(signalArg);
|
|
96
|
+
}
|
|
97
|
+
|
|
87
98
|
/**
|
|
88
99
|
* AGL-909: once, links and release plans name each repository `resolveArg` resolves as `org/name`; answers the keys it
|
|
89
100
|
* changed. With no repository discovered (`discoveredArg` 0) it waits for a later start.
|
package/ts_migration/index.ts
CHANGED
|
@@ -29,4 +29,5 @@ export * from './v41_ticketmigrationmarkers.js';
|
|
|
29
29
|
export * from './v41_ticketclosedcolumn.js';
|
|
30
30
|
export * from './v41_ticketlinkrepos.js';
|
|
31
31
|
export * from './v43_ticketactornames.js';
|
|
32
|
+
export * from './v43_ticketorderstride.js';
|
|
32
33
|
export * from './v43_flexpostharnessmigrationmarkers.js';
|
|
@@ -0,0 +1,156 @@
|
|
|
1
|
+
import * as plugins from './plugins.js';
|
|
2
|
+
import { TicketModel } from '../ts/classes.ticketmodels.js';
|
|
3
|
+
import { TicketError, ticketOrderStride, type TicketStore } from '../ts/classes.ticketstore.js';
|
|
4
|
+
import { AuthError } from '../ts/interfaces.auth.js';
|
|
5
|
+
import { ticketColumns, type TTicketColumn } from '../ts_interfaces/tickets.js';
|
|
6
|
+
import { TicketMigrationMarkerModel } from './v41_ticketmigrationmarkers.js';
|
|
7
|
+
|
|
8
|
+
export const ticketOrderStrideMigrationId = 'v43_ticketorderstride';
|
|
9
|
+
|
|
10
|
+
/** Ticket documents one query of a column reads. */
|
|
11
|
+
const pageRecords = 500;
|
|
12
|
+
/**
|
|
13
|
+
* The tickets one transaction moves at most. It runs on a live board and holds the board's fence while it writes them,
|
|
14
|
+
* so a mutation that meets it runs again once it commits: 64 keeps that wait short (on a fully loaded host, 200 tickets
|
|
15
|
+
* took 1.8 s in one transaction, and the engine's cost per write grows with its transaction).
|
|
16
|
+
*/
|
|
17
|
+
const chunkTickets = 64;
|
|
18
|
+
|
|
19
|
+
type TTicketStored = Awaited<ReturnType<typeof TicketModel.exact.findStored>>[number];
|
|
20
|
+
|
|
21
|
+
/** A transaction's outcome: the keys it moved, and whether its column holds nothing more to move. */
|
|
22
|
+
interface IChunkResult { moved: string[]; complete: boolean }
|
|
23
|
+
|
|
24
|
+
/**
|
|
25
|
+
* The deadlines in a row one transaction may reach after it lost a write conflict, 20 s each, before the run ends: 60 s
|
|
26
|
+
* without a commit is a board the spread does not get through now, or a transaction too slow for its deadline.
|
|
27
|
+
*/
|
|
28
|
+
const deadlinesInARow = 3;
|
|
29
|
+
|
|
30
|
+
/**
|
|
31
|
+
* Whether a transaction that the driver ran more than once, so that lost a write conflict, ended without committing at
|
|
32
|
+
* its deadline. `TicketStore.migrate` gives every transaction a deadline, and with one the client rejects with `timeout`
|
|
33
|
+
* when the driver's retries run out (never with `transaction_conflict`, which only a transaction without one meets).
|
|
34
|
+
*/
|
|
35
|
+
const lostConflicts = (errorArg: unknown): boolean => errorArg instanceof plugins.nosqldb.SmartdataPersistenceError
|
|
36
|
+
&& errorArg.code === 'timeout';
|
|
37
|
+
|
|
38
|
+
/**
|
|
39
|
+
* AGL-1907: releases before kept each column's orders dense (0, 1, 2, …), so a placement before a ticket finds no free
|
|
40
|
+
* integer and moves a run of tickets by one. Once, every column's tickets take `(rank + 1) × ticketOrderStride` in their
|
|
41
|
+
* board order (by order, then by id, as the board breaks ties), so a placement finds the gap a stride leaves and writes
|
|
42
|
+
* one ticket.
|
|
43
|
+
*
|
|
44
|
+
* It runs after the controller is ready, in the background (`TicketOrderSpread`): the store is correct on dense orders,
|
|
45
|
+
* so nothing waits for it. Column by column in `ticketColumns` order, in transactions of at most `chunkTickets`
|
|
46
|
+
* tickets, each reading the column again. A ticket whose order rises moves after every one above it that rises, from
|
|
47
|
+
* the column's end down, and one whose order falls after every one below it that falls, from the column's head up (the
|
|
48
|
+
* rising ones first): each takes an order between its neighbours' orders as they stand, so the column keeps its board
|
|
49
|
+
* order after every transaction, and the next one, ranking the column again, finds the same ranks and only the tickets
|
|
50
|
+
* not yet moved. A spread column is a fixed point, so a run cut short starts again and finishes the rest; the
|
|
51
|
+
* transaction that completes the last column (`closed`) writes the marker, alone when nothing is left to move, and a
|
|
52
|
+
* later start does nothing whatever the board holds.
|
|
53
|
+
*
|
|
54
|
+
* Each transaction is a step of the ticket store (`TicketStore.migrate`): it takes the board fence every mutation takes
|
|
55
|
+
* and moves the revision of every ticket whose order it changes, and once it commits the store tells clients, devices
|
|
56
|
+
* and the board pump of the new revision and those tickets, as after a mutation. A mutation that meets a step re-runs,
|
|
57
|
+
* and a step that meets a mutation re-runs, as the driver re-runs every transaction that loses a write conflict. A
|
|
58
|
+
* transaction's first attempt reads its column before it takes the fence, so mutations go on while it reads and wait
|
|
59
|
+
* only for its writes; but then any mutation that commits while it reads makes it lose, and a busy board would starve
|
|
60
|
+
* it. An attempt after a lost one therefore takes the fence first: it waits for the mutations in flight, and the
|
|
61
|
+
* mutations that come after it wait for its read and its writes. A transaction that lost to a mutation and then reached the
|
|
62
|
+
* store's 20 s deadline runs again (nothing of it committed), up to `deadlinesInARow` deadlines in a row: a lost
|
|
63
|
+
* conflict does not prove the conflicts made it late (an attempt that won the fence may simply be too slow), so the
|
|
64
|
+
* next deadline in a row after them ends the run as any other failure does, and the next controller start continues
|
|
65
|
+
* it; every commit starts the count again. It changes nothing else of a ticket and adds no history, as the v41 migration did when it closed the released
|
|
66
|
+
* column's gaps.
|
|
67
|
+
*/
|
|
68
|
+
export class TicketOrderStrideV43Migration {
|
|
69
|
+
constructor(private readonly database: plugins.nosqldb.SmartdataDb | undefined, private readonly store: TicketStore) {}
|
|
70
|
+
|
|
71
|
+
/**
|
|
72
|
+
* Runs the spread to its end, one transaction after another, and answers the keys of the tickets whose order it
|
|
73
|
+
* moved, column by column, in the order it first moved them (a mutation that places a ticket before tickets already
|
|
74
|
+
* spread changes their ranks, so the run moves them again). `signalArg` ends it: the run rejects with the signal's
|
|
75
|
+
* reason, and the transaction in flight rolls back unless its commit was already sent.
|
|
76
|
+
*/
|
|
77
|
+
public async run(signalArg: AbortSignal): Promise<string[]> {
|
|
78
|
+
if (!this.database) throw new AuthError('not_initialized', 'The authentication store database is unavailable.');
|
|
79
|
+
if (await TicketMigrationMarkerModel.exact.findStoredOne({ id: ticketOrderStrideMigrationId }, { signal: signalArg })) return [];
|
|
80
|
+
const spread = new Set<string>();
|
|
81
|
+
for (const [index, columnId] of ticketColumns.entries()) {
|
|
82
|
+
const last = index === ticketColumns.length - 1;
|
|
83
|
+
/** Whether the column's next transaction lost a write conflict before: its attempts then take the fence first. */
|
|
84
|
+
let lost = false;
|
|
85
|
+
/** The deadlines the column's next transaction reached in a row since the last commit. */
|
|
86
|
+
let deadlines = 0;
|
|
87
|
+
while (true) {
|
|
88
|
+
signalArg.throwIfAborted();
|
|
89
|
+
let attempts = 0;
|
|
90
|
+
let result: IChunkResult;
|
|
91
|
+
try {
|
|
92
|
+
result = await this.store.migrate((sessionArg, fenceArg) => {
|
|
93
|
+
attempts += 1;
|
|
94
|
+
return this.chunk(sessionArg, fenceArg, columnId, last ? spread : undefined, lost || attempts > 1);
|
|
95
|
+
}, signalArg);
|
|
96
|
+
} catch (errorArg) {
|
|
97
|
+
// The driver ran it again after lost write conflicts until the deadline: it runs again, nothing committed,
|
|
98
|
+
// unless it reached `deadlinesInARow` deadlines without a commit.
|
|
99
|
+
if (attempts > 1 && lostConflicts(errorArg) && ++deadlines < deadlinesInARow) { lost = true; continue; }
|
|
100
|
+
throw errorArg;
|
|
101
|
+
}
|
|
102
|
+
lost = false;
|
|
103
|
+
deadlines = 0;
|
|
104
|
+
for (const key of result.moved) spread.add(key);
|
|
105
|
+
if (result.complete) break;
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
return [...spread];
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
/**
|
|
112
|
+
* One attempt of a transaction: up to `chunkTickets` tickets of the column take their stride order, after the board
|
|
113
|
+
* fence, which an attempt after a lost one (`fenceFirstArg`) takes before it reads the column. With `markerArg` (the
|
|
114
|
+
* keys moved so far) the transaction that completes the column writes the marker too. Nothing outside the transaction
|
|
115
|
+
* changes here: the driver may run it again.
|
|
116
|
+
*/
|
|
117
|
+
private async chunk(sessionArg: plugins.nosqldb.SmartdataSession, fenceArg: () => Promise<void>, columnIdArg: TTicketColumn,
|
|
118
|
+
markerArg: ReadonlySet<string> | undefined, fenceFirstArg: boolean): Promise<{ ticketIds: string[]; result: IChunkResult }> {
|
|
119
|
+
const options = { session: sessionArg };
|
|
120
|
+
if (fenceFirstArg) await fenceArg();
|
|
121
|
+
const ranked = (await this.page(columnIdArg, sessionArg))
|
|
122
|
+
.sort((leftArg, rightArg) => leftArg.ticket.order - rightArg.ticket.order || (leftArg.id < rightArg.id ? -1 : 1))
|
|
123
|
+
.map((recordArg, rankArg) => ({ record: recordArg, order: (rankArg + 1) * ticketOrderStride }));
|
|
124
|
+
const moving = [...ranked.filter((entryArg) => entryArg.order > entryArg.record.ticket.order).reverse(),
|
|
125
|
+
...ranked.filter((entryArg) => entryArg.order < entryArg.record.ticket.order)];
|
|
126
|
+
const taken = moving.slice(0, chunkTickets);
|
|
127
|
+
if (taken.length) await fenceArg();
|
|
128
|
+
for (const { record, order } of taken) {
|
|
129
|
+
const moved = await TicketModel.exact.transition({ current: record, change: (modelArg) => {
|
|
130
|
+
modelArg.ticket = { ...modelArg.ticket, order, revision: (modelArg.ticket.revision ?? 0) + 1 };
|
|
131
|
+
} }, options);
|
|
132
|
+
if (moved.status !== 'transitioned') throw new TicketError('concurrent_change', 'A ticket changed during the order spread.');
|
|
133
|
+
}
|
|
134
|
+
const moved = taken.map((entryArg) => entryArg.record.id);
|
|
135
|
+
const complete = moving.length === taken.length;
|
|
136
|
+
if (markerArg && complete) {
|
|
137
|
+
const inserted = await TicketMigrationMarkerModel.exact.insert({ id: ticketOrderStrideMigrationId, completedAt: new Date().toISOString(),
|
|
138
|
+
tickets: [...new Set([...markerArg, ...moved])] }, options);
|
|
139
|
+
if (inserted.status !== 'inserted') throw new TicketError('concurrent_change', 'The order spread ran twice.');
|
|
140
|
+
}
|
|
141
|
+
return { ticketIds: moved, result: { moved, complete } };
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
/** Every ticket of a column, read by id a page at a time. */
|
|
145
|
+
private async page(columnIdArg: TTicketColumn, sessionArg: plugins.nosqldb.SmartdataSession): Promise<TTicketStored[]> {
|
|
146
|
+
const records: TTicketStored[] = [];
|
|
147
|
+
let after: string | undefined;
|
|
148
|
+
while (true) {
|
|
149
|
+
const page = await TicketModel.exact.findStored({ filter: { 'ticket.columnId': columnIdArg, ...(after ? { id: { $gt: after } } : {}) },
|
|
150
|
+
sort: { id: 1 }, limit: pageRecords, session: sessionArg });
|
|
151
|
+
records.push(...page);
|
|
152
|
+
if (page.length < pageRecords) return records;
|
|
153
|
+
after = page.at(-1)!.id;
|
|
154
|
+
}
|
|
155
|
+
}
|
|
156
|
+
}
|