@wjarka/cezarion 0.15.1-pr920.1498 → 0.16.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/contract/delegation.d.ts +45 -0
- package/dist/contract/index.js +13 -1
- package/dist/contract/runs.d.ts +25 -0
- package/dist/contract/skills.d.ts +5 -0
- package/dist/delegation/destroy-observation.d.ts +42 -0
- package/dist/delegation/destroy-observation.js +115 -0
- package/dist/delegation/destroy-observation.js.map +1 -0
- package/dist/delegation/process-liveness.d.ts +32 -11
- package/dist/delegation/process-liveness.js +71 -32
- package/dist/delegation/process-liveness.js.map +1 -1
- package/dist/delegation/retry-backoff.d.ts +23 -0
- package/dist/delegation/retry-backoff.js +18 -0
- package/dist/delegation/retry-backoff.js.map +1 -0
- package/dist/delegation/routes.d.ts +5 -0
- package/dist/delegation/scratch-cleanup.d.ts +17 -3
- package/dist/delegation/scratch-cleanup.js +77 -25
- package/dist/delegation/scratch-cleanup.js.map +1 -1
- package/dist/delegation/service.d.ts +20 -1
- package/dist/delegation/service.js +211 -89
- package/dist/delegation/service.js.map +1 -1
- package/dist/delegation/transport.d.ts +18 -13
- package/dist/delegation/workspace.d.ts +16 -2
- package/dist/delegation/workspace.js +46 -23
- package/dist/delegation/workspace.js.map +1 -1
- package/dist/git-worktree-release.d.ts +2 -1
- package/dist/git-worktree-release.js +2 -2
- package/dist/git-worktree-release.js.map +1 -1
- package/dist/runs/delegation-state.d.ts +5 -1
- package/dist/runs/delegation-state.js +9 -0
- package/dist/runs/delegation-state.js.map +1 -1
- package/dist/runs/store.d.ts +26 -6
- package/dist/runs/store.js +39 -8
- package/dist/runs/store.js.map +1 -1
- package/dist/server/server.d.ts +174 -34
- package/dist/server/server.js +2 -1
- package/dist/server/server.js.map +1 -1
- package/dist/workflows/run.d.ts +15 -8
- package/dist/workflows/run.js +60 -33
- package/dist/workflows/run.js.map +1 -1
- package/package.json +4 -3
- package/scripts/benchmark-stuck-destroys.ts +143 -0
- package/web/dist/assets/{agent-accounts-BwNOfMup.js → agent-accounts-CFOBMVHc.js} +1 -1
- package/web/dist/assets/{alert-dialog-ClF_lIOt.js → alert-dialog-BrpNUwMt.js} +1 -1
- package/web/dist/assets/{button-CQ23Xr-k.js → button-C_rd7CPp.js} +1 -1
- package/web/dist/assets/{compare-variants-BxF8VLD-.js → compare-variants-D6WXhvXA.js} +1 -1
- package/web/dist/assets/{diff-BwokYGhU.js → diff-DrEARXTV.js} +2 -2
- package/web/dist/assets/{diff-controls-s_b8r_e3.js → diff-controls-B1-j7nFg.js} +1 -1
- package/web/dist/assets/{diff-view-_66ejTWp.js → diff-view-DvGLjdTk.js} +3 -3
- package/web/dist/assets/{dropdown-menu-CKWeWKuz.js → dropdown-menu-BegOZFrj.js} +1 -1
- package/web/dist/assets/{format-B822NqEe.js → format-DpNe3S0A.js} +1 -1
- package/web/dist/assets/{git-toolbar-BSbJegbU.js → git-toolbar-Cw4ljmhB.js} +1 -1
- package/web/dist/assets/{github-DPwkwUL5.js → github-CfV5sqL0.js} +1 -1
- package/web/dist/assets/{image-preview-CDQ-velO.js → image-preview-FOEfqpZP.js} +1 -1
- package/web/dist/assets/{index-BByCJyfO.js → index-DgWWlOLy.js} +5 -5
- package/web/dist/assets/{nav-row-styles-sVR90hos.js → nav-row-styles-vp2JbhuJ.js} +1 -1
- package/web/dist/assets/{popover-CrF4zCuc.js → popover-BohP6WTx.js} +1 -1
- package/web/dist/assets/preview-pane-connected-qTA5KNoJ.js +3 -0
- package/web/dist/assets/{preview-state-CrAhSlly.js → preview-state-vuMviq6q.js} +1 -1
- package/web/dist/assets/{prompt-templates-ClT0mkMX.js → prompt-templates-Cubdut_C.js} +1 -1
- package/web/dist/assets/{repo-git-Dp4xOwEa.js → repo-git-Cf4t5PD0.js} +1 -1
- package/web/dist/assets/{run-diff-5FDZwDJj.js → run-diff-BpkUW37Z.js} +3 -3
- package/web/dist/assets/run-header-68WHf9wV.js +1 -0
- package/web/dist/assets/{skills-DmFJO1MH.js → skills-DiVIgKeU.js} +1 -1
- package/web/dist/assets/{status-dot-BHTkHOX9.js → status-dot-CqTJHSP9.js} +1 -1
- package/web/dist/assets/{tab-link-DU12efzE.js → tab-link-B07Hu4QX.js} +1 -1
- package/web/dist/assets/{task-changes-QpE_dFpk.js → task-changes--lQF6EYf.js} +1 -1
- package/web/dist/assets/{task-commits-CMM6sqOB.js → task-commits-Cx0qy4DN.js} +1 -1
- package/web/dist/assets/{task-files-CzMjux0L.js → task-files-D50s-gKW.js} +2 -2
- package/web/dist/assets/{task-github-item-CORhYMIT.js → task-github-item-DaqmSwYT.js} +1 -1
- package/web/dist/assets/task-thread-BPQ64lui.js +7 -0
- package/web/dist/assets/{textarea-BaloTp1M.js → textarea-D20O4Pw0.js} +1 -1
- package/web/dist/assets/{thread-state-DbBJEUfD.js → thread-state-DWcFMrvb.js} +1 -1
- package/web/dist/assets/{use-hand-to-agent-state-C6Zr_nMh.js → use-hand-to-agent-state-Cy9eyyfL.js} +1 -1
- package/web/dist/assets/{utils-BgFF8-pP.js → utils-JsL3ZsJf.js} +1 -1
- package/web/dist/assets/{workflows-C1fbjIBm.js → workflows-jwG7uxqi.js} +3 -3
- package/web/dist/assets/{zoomable-image-C15xvw-A.js → zoomable-image-BJJ64KGc.js} +1 -1
- package/web/dist/index.html +13 -13
- package/web/dist/assets/preview-pane-connected-D8CRMQbn.js +0 -3
- package/web/dist/assets/run-header-6S5uxH8I.js +0 -1
- package/web/dist/assets/task-thread-BuD0Yyh9.js +0 -7
|
@@ -92,6 +92,18 @@ export declare const workerWaitSchema: z.ZodObject<{
|
|
|
92
92
|
wakeId: z.ZodOptional<z.ZodUUID>;
|
|
93
93
|
}, z.core.$strict>;
|
|
94
94
|
export type WorkerWait = z.infer<typeof workerWaitSchema>;
|
|
95
|
+
/** Automatic retries of a pending destroy (hearsay-tools/cezarion#879): how many scheduled
|
|
96
|
+
* attempts ran since the last explicit request, when the next one is due, and whether they
|
|
97
|
+
* reached the hourly cap with nothing changing, which the cockpit shows as needing attention.
|
|
98
|
+
* Absent until the first automatic attempt; a manual destroy clears it. Stored on the run record
|
|
99
|
+
* (`destroyRetry`), never inside the strict worker delegation, so an older cezar strips it instead
|
|
100
|
+
* of quarantining the worker; the API shows it as the worker's `destroy.retry`. */
|
|
101
|
+
export declare const workerDestroyRetrySchema: z.ZodObject<{
|
|
102
|
+
attempts: z.ZodNumber;
|
|
103
|
+
nextAt: z.ZodISODateTime;
|
|
104
|
+
needsAttention: z.ZodOptional<z.ZodLiteral<true>>;
|
|
105
|
+
}, z.core.$strict>;
|
|
106
|
+
export type WorkerDestroyRetry = z.infer<typeof workerDestroyRetrySchema>;
|
|
95
107
|
export declare const workerDestroySchema: z.ZodObject<{
|
|
96
108
|
requestedAt: z.ZodISODateTime;
|
|
97
109
|
phase: z.ZodEnum<{
|
|
@@ -109,6 +121,29 @@ export declare const workerDestroySchema: z.ZodObject<{
|
|
|
109
121
|
error: z.ZodOptional<z.ZodString>;
|
|
110
122
|
}, z.core.$strict>;
|
|
111
123
|
export type WorkerDestroy = z.infer<typeof workerDestroySchema>;
|
|
124
|
+
/** A worker's destroy as inspection and relationships show it: the stored phase, plus its retry state. */
|
|
125
|
+
export declare const workerDestroyViewSchema: z.ZodObject<{
|
|
126
|
+
requestedAt: z.ZodISODateTime;
|
|
127
|
+
phase: z.ZodEnum<{
|
|
128
|
+
cleaning: "cleaning";
|
|
129
|
+
complete: "complete";
|
|
130
|
+
incomplete: "incomplete";
|
|
131
|
+
requested: "requested";
|
|
132
|
+
terminating: "terminating";
|
|
133
|
+
}>;
|
|
134
|
+
remaining: z.ZodArray<z.ZodEnum<{
|
|
135
|
+
branch: "branch";
|
|
136
|
+
process: "process";
|
|
137
|
+
worktree: "worktree";
|
|
138
|
+
}>>;
|
|
139
|
+
error: z.ZodOptional<z.ZodString>;
|
|
140
|
+
retry: z.ZodOptional<z.ZodObject<{
|
|
141
|
+
attempts: z.ZodNumber;
|
|
142
|
+
nextAt: z.ZodISODateTime;
|
|
143
|
+
needsAttention: z.ZodOptional<z.ZodLiteral<true>>;
|
|
144
|
+
}, z.core.$strict>>;
|
|
145
|
+
}, z.core.$strict>;
|
|
146
|
+
export type WorkerDestroyView = z.infer<typeof workerDestroyViewSchema>;
|
|
112
147
|
export declare const workerCreationReceiptSchema: z.ZodObject<{
|
|
113
148
|
requestId: z.ZodUUID;
|
|
114
149
|
workerId: z.ZodUUID;
|
|
@@ -1128,6 +1163,11 @@ export declare const workerInspectionSchema: z.ZodObject<{
|
|
|
1128
1163
|
worktree: "worktree";
|
|
1129
1164
|
}>>;
|
|
1130
1165
|
error: z.ZodOptional<z.ZodString>;
|
|
1166
|
+
retry: z.ZodOptional<z.ZodObject<{
|
|
1167
|
+
attempts: z.ZodNumber;
|
|
1168
|
+
nextAt: z.ZodISODateTime;
|
|
1169
|
+
needsAttention: z.ZodOptional<z.ZodLiteral<true>>;
|
|
1170
|
+
}, z.core.$strict>>;
|
|
1131
1171
|
}, z.core.$strict>>;
|
|
1132
1172
|
outcome: z.ZodOptional<z.ZodObject<{
|
|
1133
1173
|
workerId: z.ZodUUID;
|
|
@@ -1316,6 +1356,11 @@ export declare const runRelationshipsSchema: z.ZodObject<{
|
|
|
1316
1356
|
worktree: "worktree";
|
|
1317
1357
|
}>>;
|
|
1318
1358
|
error: z.ZodOptional<z.ZodString>;
|
|
1359
|
+
retry: z.ZodOptional<z.ZodObject<{
|
|
1360
|
+
attempts: z.ZodNumber;
|
|
1361
|
+
nextAt: z.ZodISODateTime;
|
|
1362
|
+
needsAttention: z.ZodOptional<z.ZodLiteral<true>>;
|
|
1363
|
+
}, z.core.$strict>>;
|
|
1319
1364
|
}, z.core.$strict>>;
|
|
1320
1365
|
outcome: z.ZodOptional<z.ZodObject<{
|
|
1321
1366
|
workerId: z.ZodUUID;
|
package/dist/contract/index.js
CHANGED
|
@@ -626,12 +626,18 @@ var workerWaitSchema = z8.object({
|
|
|
626
626
|
outcomes: z8.array(workerOutcomeSchema).max(32),
|
|
627
627
|
wakeId: z8.uuid().optional()
|
|
628
628
|
}).strict();
|
|
629
|
+
var workerDestroyRetrySchema = z8.object({
|
|
630
|
+
attempts: z8.number().int().positive(),
|
|
631
|
+
nextAt: z8.iso.datetime(),
|
|
632
|
+
needsAttention: z8.literal(true).optional()
|
|
633
|
+
}).strict();
|
|
629
634
|
var workerDestroySchema = z8.object({
|
|
630
635
|
requestedAt: z8.iso.datetime(),
|
|
631
636
|
phase: z8.enum(["requested", "terminating", "cleaning", "complete", "incomplete"]),
|
|
632
637
|
remaining: remainingSchema,
|
|
633
638
|
error: errorSchema.optional()
|
|
634
639
|
}).strict();
|
|
640
|
+
var workerDestroyViewSchema = workerDestroySchema.extend({ retry: workerDestroyRetrySchema.optional() }).strict();
|
|
635
641
|
var workerCreationReceiptSchema = z8.object({
|
|
636
642
|
requestId: z8.uuid(),
|
|
637
643
|
workerId: z8.uuid(),
|
|
@@ -816,7 +822,7 @@ var workerInspectionSchema = z8.object({
|
|
|
816
822
|
model: z8.string().max(512).optional(),
|
|
817
823
|
inputs: z8.array(workerInputSchema).max(32).optional(),
|
|
818
824
|
wait: workerWaitSchema.optional(),
|
|
819
|
-
destroy:
|
|
825
|
+
destroy: workerDestroyViewSchema.optional(),
|
|
820
826
|
outcome: workerOutcomeSchema.optional()
|
|
821
827
|
});
|
|
822
828
|
var workerDiffSchema = z8.object({
|
|
@@ -1767,6 +1773,10 @@ var runRecordSchema = z14.object({
|
|
|
1767
1773
|
/** The scheduled instant (UTC ISO); for `manual`, the launch time. */
|
|
1768
1774
|
occurrenceAt: z14.string()
|
|
1769
1775
|
}).optional(),
|
|
1776
|
+
/** A pending worker destroy's automatic retry state (hearsay-tools/cezarion#879). Here, not in
|
|
1777
|
+
* the strict worker `delegation`, so an older cezar strips it instead of quarantining the worker,
|
|
1778
|
+
* and keeps it across its own writes. The API shows it as the worker's `destroy.retry`. */
|
|
1779
|
+
destroyRetry: workerDestroyRetrySchema.optional(),
|
|
1770
1780
|
/** Task webhook opt-in (#589): the project's webhook receives this run's status changes.
|
|
1771
1781
|
* Absent = off. Changes after start through `POST /runs/:id/notify`, which is why it is not
|
|
1772
1782
|
* part of the idempotent-start hash. */
|
|
@@ -3603,7 +3613,9 @@ export {
|
|
|
3603
3613
|
workerContextSchema,
|
|
3604
3614
|
workerCreationReceiptSchema,
|
|
3605
3615
|
workerDestroyResultSchema,
|
|
3616
|
+
workerDestroyRetrySchema,
|
|
3606
3617
|
workerDestroySchema,
|
|
3618
|
+
workerDestroyViewSchema,
|
|
3607
3619
|
workerDiffSchema,
|
|
3608
3620
|
workerEmptyRequestSchema,
|
|
3609
3621
|
workerInputRecipeSchema,
|
package/dist/contract/runs.d.ts
CHANGED
|
@@ -735,6 +735,11 @@ export declare const runRecordSchema: z.ZodObject<{
|
|
|
735
735
|
}>;
|
|
736
736
|
occurrenceAt: z.ZodString;
|
|
737
737
|
}, z.core.$strip>>;
|
|
738
|
+
destroyRetry: z.ZodOptional<z.ZodObject<{
|
|
739
|
+
attempts: z.ZodNumber;
|
|
740
|
+
nextAt: z.ZodISODateTime;
|
|
741
|
+
needsAttention: z.ZodOptional<z.ZodLiteral<true>>;
|
|
742
|
+
}, z.core.$strict>>;
|
|
738
743
|
notify: z.ZodOptional<z.ZodBoolean>;
|
|
739
744
|
webhook: z.ZodOptional<z.ZodObject<{
|
|
740
745
|
lastDeliveredAt: z.ZodOptional<z.ZodString>;
|
|
@@ -1472,6 +1477,11 @@ export declare const apiRunSchema: z.ZodObject<{
|
|
|
1472
1477
|
}>;
|
|
1473
1478
|
occurrenceAt: z.ZodString;
|
|
1474
1479
|
}, z.core.$strip>>;
|
|
1480
|
+
destroyRetry: z.ZodOptional<z.ZodObject<{
|
|
1481
|
+
attempts: z.ZodNumber;
|
|
1482
|
+
nextAt: z.ZodISODateTime;
|
|
1483
|
+
needsAttention: z.ZodOptional<z.ZodLiteral<true>>;
|
|
1484
|
+
}, z.core.$strict>>;
|
|
1475
1485
|
notify: z.ZodOptional<z.ZodBoolean>;
|
|
1476
1486
|
webhook: z.ZodOptional<z.ZodObject<{
|
|
1477
1487
|
lastDeliveredAt: z.ZodOptional<z.ZodString>;
|
|
@@ -2921,6 +2931,11 @@ export declare const createRunResponseSchema: z.ZodUnion<readonly [z.ZodObject<{
|
|
|
2921
2931
|
}>;
|
|
2922
2932
|
occurrenceAt: z.ZodString;
|
|
2923
2933
|
}, z.core.$strip>>;
|
|
2934
|
+
destroyRetry: z.ZodOptional<z.ZodObject<{
|
|
2935
|
+
attempts: z.ZodNumber;
|
|
2936
|
+
nextAt: z.ZodISODateTime;
|
|
2937
|
+
needsAttention: z.ZodOptional<z.ZodLiteral<true>>;
|
|
2938
|
+
}, z.core.$strict>>;
|
|
2924
2939
|
notify: z.ZodOptional<z.ZodBoolean>;
|
|
2925
2940
|
webhook: z.ZodOptional<z.ZodObject<{
|
|
2926
2941
|
lastDeliveredAt: z.ZodOptional<z.ZodString>;
|
|
@@ -3651,6 +3666,11 @@ export declare const createRunResponseSchema: z.ZodUnion<readonly [z.ZodObject<{
|
|
|
3651
3666
|
}>;
|
|
3652
3667
|
occurrenceAt: z.ZodString;
|
|
3653
3668
|
}, z.core.$strip>>;
|
|
3669
|
+
destroyRetry: z.ZodOptional<z.ZodObject<{
|
|
3670
|
+
attempts: z.ZodNumber;
|
|
3671
|
+
nextAt: z.ZodISODateTime;
|
|
3672
|
+
needsAttention: z.ZodOptional<z.ZodLiteral<true>>;
|
|
3673
|
+
}, z.core.$strict>>;
|
|
3654
3674
|
notify: z.ZodOptional<z.ZodBoolean>;
|
|
3655
3675
|
webhook: z.ZodOptional<z.ZodObject<{
|
|
3656
3676
|
lastDeliveredAt: z.ZodOptional<z.ZodString>;
|
|
@@ -4607,6 +4627,11 @@ export declare const pickVariantResponseSchema: z.ZodObject<{
|
|
|
4607
4627
|
}>;
|
|
4608
4628
|
occurrenceAt: z.ZodString;
|
|
4609
4629
|
}, z.core.$strip>>;
|
|
4630
|
+
destroyRetry: z.ZodOptional<z.ZodObject<{
|
|
4631
|
+
attempts: z.ZodNumber;
|
|
4632
|
+
nextAt: z.ZodISODateTime;
|
|
4633
|
+
needsAttention: z.ZodOptional<z.ZodLiteral<true>>;
|
|
4634
|
+
}, z.core.$strict>>;
|
|
4610
4635
|
notify: z.ZodOptional<z.ZodBoolean>;
|
|
4611
4636
|
webhook: z.ZodOptional<z.ZodObject<{
|
|
4612
4637
|
lastDeliveredAt: z.ZodOptional<z.ZodString>;
|
|
@@ -644,6 +644,11 @@ export declare const startTodoResponseSchema: z.ZodObject<{
|
|
|
644
644
|
}>;
|
|
645
645
|
occurrenceAt: z.ZodString;
|
|
646
646
|
}, z.core.$strip>>;
|
|
647
|
+
destroyRetry: z.ZodOptional<z.ZodObject<{
|
|
648
|
+
attempts: z.ZodNumber;
|
|
649
|
+
nextAt: z.ZodISODateTime;
|
|
650
|
+
needsAttention: z.ZodOptional<z.ZodLiteral<true>>;
|
|
651
|
+
}, z.core.$strict>>;
|
|
647
652
|
notify: z.ZodOptional<z.ZodBoolean>;
|
|
648
653
|
webhook: z.ZodOptional<z.ZodObject<{
|
|
649
654
|
lastDeliveredAt: z.ZodOptional<z.ZodString>;
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
import type { RunStore } from '../runs/store.ts';
|
|
2
|
+
import { type RecordedProcess } from './process-liveness.ts';
|
|
3
|
+
/**
|
|
4
|
+
* What a full destroy attempt started from, and the processes it found holding the worker's
|
|
5
|
+
* resources (hearsay-tools/cezarion#879). A scheduled retry that would observe the same, with
|
|
6
|
+
* every holder still the same live process, has nothing new to try: it skips the transcript
|
|
7
|
+
* read, the git spawns, the `/proc` scans and the snapshot writes.
|
|
8
|
+
*/
|
|
9
|
+
export type DestroyObservation = {
|
|
10
|
+
key: string;
|
|
11
|
+
holders: RecordedProcess[];
|
|
12
|
+
};
|
|
13
|
+
/**
|
|
14
|
+
* Everything a destroy attempt's outcome depends on that a stat or a small file read can see:
|
|
15
|
+
* the worker's record (minus `destroy`, which every attempt rewrites), the parent's receipt, the
|
|
16
|
+
* execution checkpoint, the process record, the scratch locations, the worktree and its own admin
|
|
17
|
+
* dir, lock and ownership marker and, given the repository's common Git dir, the ownership receipt
|
|
18
|
+
* and its cleanup checkpoint, the branch ref, `packed-refs`, the branch reflog and the
|
|
19
|
+
* linked-worktree admin dirs. Synchronous, with no git and no `/proc` scan. The worktree mutation
|
|
20
|
+
* lock is not here: every full attempt takes it, so its stamp would change on every attempt.
|
|
21
|
+
*/
|
|
22
|
+
export declare function observeDestroy({ store, dataDir, commonDir, workerId }: {
|
|
23
|
+
store: RunStore;
|
|
24
|
+
dataDir: string;
|
|
25
|
+
commonDir?: string;
|
|
26
|
+
workerId: string;
|
|
27
|
+
}): string;
|
|
28
|
+
/** The holders a failed attempt named, pinned to their current incarnation. */
|
|
29
|
+
export declare function recordHolders(pids: readonly number[]): RecordedProcess[];
|
|
30
|
+
/**
|
|
31
|
+
* Every holder the last attempt named still holds: the same live process and, unless it belongs to
|
|
32
|
+
* the generation's own process record (which blocks wherever it works), still working under the
|
|
33
|
+
* worktree or a scratch location. A holder that exited, was replaced or moved away is a change.
|
|
34
|
+
* One `readlink` (Linux) or one-process `lsof` (darwin) per holder; on win32, whose scan names no
|
|
35
|
+
* holder, only recorded processes reach this, and liveness decides.
|
|
36
|
+
*/
|
|
37
|
+
export declare function holdersStillHold({ store, dataDir, workerId, holders }: {
|
|
38
|
+
store: RunStore;
|
|
39
|
+
dataDir: string;
|
|
40
|
+
workerId: string;
|
|
41
|
+
holders: readonly RecordedProcess[];
|
|
42
|
+
}): boolean;
|
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
import { createHash } from 'node:crypto';
|
|
2
|
+
import { lstatSync, readFileSync } from 'node:fs';
|
|
3
|
+
import { join, resolve } from 'node:path';
|
|
4
|
+
import { agentTmpDirLocations } from '../runs/agent-tmpdir.js';
|
|
5
|
+
import { processCwdUnder, processStartToken, recordedProcessLive } from './process-liveness.js';
|
|
6
|
+
/** A file's identity, content and permission stamp, or why it cannot be read. `ctime` and `mode`
|
|
7
|
+
* catch a repair that keeps the size and the second (an in-place rewrite, a `chmod`). */
|
|
8
|
+
function fileSignature(path) {
|
|
9
|
+
try {
|
|
10
|
+
const info = lstatSync(path);
|
|
11
|
+
return `${info.ino}:${info.size}:${info.mtimeMs}:${info.ctimeMs}:${info.mode}`;
|
|
12
|
+
}
|
|
13
|
+
catch (error) {
|
|
14
|
+
return error.code ?? 'error';
|
|
15
|
+
}
|
|
16
|
+
}
|
|
17
|
+
/** A small identity file's bytes and permissions. Each removal attempt rewrites its cleanup
|
|
18
|
+
* checkpoint atomically (a new inode, the same bytes), so only what such a file says, and who may
|
|
19
|
+
* read it, is a change; an in-place repair or a `chmod` still is. */
|
|
20
|
+
function contentSignature(path) {
|
|
21
|
+
try {
|
|
22
|
+
const info = lstatSync(path);
|
|
23
|
+
if (!info.isFile() || info.size > 65_536)
|
|
24
|
+
return `${info.ino}:${info.size}:${info.mtimeMs}:${info.ctimeMs}:${info.mode}`;
|
|
25
|
+
return `${info.mode}:${createHash('sha256').update(readFileSync(path)).digest('hex')}`;
|
|
26
|
+
}
|
|
27
|
+
catch (error) {
|
|
28
|
+
return error.code ?? 'error';
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
/** A directory's identity and permissions only: a holder working inside it changes its times,
|
|
32
|
+
* never whether the directory is there, which one it is, or who may read it. */
|
|
33
|
+
function dirSignature(path) {
|
|
34
|
+
try {
|
|
35
|
+
const info = lstatSync(path);
|
|
36
|
+
return `${info.ino}:${info.mode}`;
|
|
37
|
+
}
|
|
38
|
+
catch (error) {
|
|
39
|
+
return error.code ?? 'error';
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
/** The linked worktree's own admin dir, which its `.git` file names: its lock and its ownership
|
|
43
|
+
* marker live there. */
|
|
44
|
+
function adminDir(worktree) {
|
|
45
|
+
try {
|
|
46
|
+
const gitdir = /^gitdir: (.+)$/m.exec(readFileSync(join(worktree, '.git'), 'utf8'))?.[1]?.trim();
|
|
47
|
+
return gitdir ? resolve(worktree, gitdir) : undefined;
|
|
48
|
+
}
|
|
49
|
+
catch {
|
|
50
|
+
return undefined;
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
/**
|
|
54
|
+
* Everything a destroy attempt's outcome depends on that a stat or a small file read can see:
|
|
55
|
+
* the worker's record (minus `destroy`, which every attempt rewrites), the parent's receipt, the
|
|
56
|
+
* execution checkpoint, the process record, the scratch locations, the worktree and its own admin
|
|
57
|
+
* dir, lock and ownership marker and, given the repository's common Git dir, the ownership receipt
|
|
58
|
+
* and its cleanup checkpoint, the branch ref, `packed-refs`, the branch reflog and the
|
|
59
|
+
* linked-worktree admin dirs. Synchronous, with no git and no `/proc` scan. The worktree mutation
|
|
60
|
+
* lock is not here: every full attempt takes it, so its stamp would change on every attempt.
|
|
61
|
+
*/
|
|
62
|
+
export function observeDestroy({ store, dataDir, commonDir, workerId }) {
|
|
63
|
+
const run = store.getRun(workerId);
|
|
64
|
+
if (run?.delegation?.role !== 'worker')
|
|
65
|
+
return JSON.stringify({ status: run?.status, role: run?.delegation?.role });
|
|
66
|
+
const { destroy: _destroy, ...delegation } = run.delegation;
|
|
67
|
+
const parent = store.getRun(delegation.parentRunId);
|
|
68
|
+
const receipt = parent?.delegation?.role === 'root' ? parent.delegation.receipts.find(entry => entry.workerId === workerId) : undefined;
|
|
69
|
+
const execution = store.readWorkerExecution(workerId);
|
|
70
|
+
const { path, branch, resourceId } = delegation.workspace;
|
|
71
|
+
const admin = adminDir(path);
|
|
72
|
+
return JSON.stringify({
|
|
73
|
+
status: run.status, delegation,
|
|
74
|
+
receipt: receipt ? receipt.deletion ?? 'present' : 'absent',
|
|
75
|
+
execution: execution ?? 'absent',
|
|
76
|
+
processes: execution ? store.readWorkerProcesses(workerId, execution.generation) : 'absent',
|
|
77
|
+
scratch: agentTmpDirLocations(dataDir, workerId).map(dirSignature),
|
|
78
|
+
worktree: dirSignature(path),
|
|
79
|
+
admin: admin ? [fileSignature(admin), fileSignature(join(admin, 'locked')), contentSignature(join(admin, 'cezar-owned-resource'))] : 'absent',
|
|
80
|
+
...(commonDir ? { git: [
|
|
81
|
+
contentSignature(join(commonDir, 'cezar-owned-workspaces', `${resourceId}.json`)),
|
|
82
|
+
contentSignature(join(commonDir, 'cezar-owned-workspaces', `${resourceId}.cleanup.json`)),
|
|
83
|
+
fileSignature(join(commonDir, 'refs/heads', branch)),
|
|
84
|
+
fileSignature(join(commonDir, 'packed-refs')),
|
|
85
|
+
fileSignature(join(commonDir, 'logs/refs/heads', branch)),
|
|
86
|
+
fileSignature(join(commonDir, 'worktrees')),
|
|
87
|
+
] } : {}),
|
|
88
|
+
});
|
|
89
|
+
}
|
|
90
|
+
/** The holders a failed attempt named, pinned to their current incarnation. */
|
|
91
|
+
export function recordHolders(pids) {
|
|
92
|
+
return pids.map(pid => { const startToken = processStartToken(pid); return { pid, ...(startToken ? { startToken } : {}) }; });
|
|
93
|
+
}
|
|
94
|
+
/**
|
|
95
|
+
* Every holder the last attempt named still holds: the same live process and, unless it belongs to
|
|
96
|
+
* the generation's own process record (which blocks wherever it works), still working under the
|
|
97
|
+
* worktree or a scratch location. A holder that exited, was replaced or moved away is a change.
|
|
98
|
+
* One `readlink` (Linux) or one-process `lsof` (darwin) per holder; on win32, whose scan names no
|
|
99
|
+
* holder, only recorded processes reach this, and liveness decides.
|
|
100
|
+
*/
|
|
101
|
+
export function holdersStillHold({ store, dataDir, workerId, holders }) {
|
|
102
|
+
if (!holders.length)
|
|
103
|
+
return true;
|
|
104
|
+
const run = store.getRun(workerId);
|
|
105
|
+
if (run?.delegation?.role !== 'worker')
|
|
106
|
+
return false;
|
|
107
|
+
const execution = store.readWorkerExecution(workerId);
|
|
108
|
+
const record = execution ? store.readWorkerProcesses(workerId, execution.generation) : 'absent';
|
|
109
|
+
const recorded = typeof record === 'string' ? [] : [record.controller, ...record.processes];
|
|
110
|
+
const paths = [run.delegation.workspace.path, ...agentTmpDirLocations(dataDir, workerId)];
|
|
111
|
+
// A recorded entry speaks for a holder only while it is that same incarnation: a reused PID is not the generation's.
|
|
112
|
+
const inRecord = (holder) => recorded.some(entry => entry.pid === holder.pid && recordedProcessLive(entry));
|
|
113
|
+
return holders.every(holder => recordedProcessLive(holder) && (inRecord(holder) || processCwdUnder(holder.pid, paths) !== false));
|
|
114
|
+
}
|
|
115
|
+
//# sourceMappingURL=destroy-observation.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"destroy-observation.js","sourceRoot":"","sources":["../../src/delegation/destroy-observation.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,SAAS,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AAClD,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,EAAE,oBAAoB,EAAE,MAAM,yBAAyB,CAAC;AAE/D,OAAO,EAAE,eAAe,EAAE,iBAAiB,EAAE,mBAAmB,EAAwB,MAAM,uBAAuB,CAAC;AAUtH;yFACyF;AACzF,SAAS,aAAa,CAAC,IAAY;IACjC,IAAI,CAAC;QAAC,MAAM,IAAI,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC;QAAC,OAAO,GAAG,IAAI,CAAC,GAAG,IAAI,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;IAAC,CAAC;IACrH,OAAO,KAAK,EAAE,CAAC;QAAC,OAAQ,KAA+B,CAAC,IAAI,IAAI,OAAO,CAAC;IAAC,CAAC;AAC5E,CAAC;AACD;;qEAEqE;AACrE,SAAS,gBAAgB,CAAC,IAAY;IACpC,IAAI,CAAC;QACH,MAAM,IAAI,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC;QAC7B,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,IAAI,CAAC,IAAI,GAAG,MAAM;YAAE,OAAO,GAAG,IAAI,CAAC,GAAG,IAAI,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;QACzH,OAAO,GAAG,IAAI,CAAC,IAAI,IAAI,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC;IACzF,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QAAC,OAAQ,KAA+B,CAAC,IAAI,IAAI,OAAO,CAAC;IAAC,CAAC;AAC9E,CAAC;AACD;gFACgF;AAChF,SAAS,YAAY,CAAC,IAAY;IAChC,IAAI,CAAC;QAAC,MAAM,IAAI,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC;QAAC,OAAO,GAAG,IAAI,CAAC,GAAG,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;IAAC,CAAC;IACxE,OAAO,KAAK,EAAE,CAAC;QAAC,OAAQ,KAA+B,CAAC,IAAI,IAAI,OAAO,CAAC;IAAC,CAAC;AAC5E,CAAC;AAED;wBACwB;AACxB,SAAS,QAAQ,CAAC,QAAgB;IAChC,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,iBAAiB,CAAC,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,QAAQ,EAAE,MAAM,CAAC,EAAE,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC;QACjG,OAAO,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IACxD,CAAC;IAAC,MAAM,CAAC;QAAC,OAAO,SAAS,CAAC;IAAC,CAAC;AAC/B,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,UAAU,cAAc,CAAC,EAAE,KAAK,EAAE,OAAO,EAAE,SAAS,EAAE,QAAQ,EAA8E;IAChJ,MAAM,GAAG,GAAG,KAAK,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;IACnC,IAAI,GAAG,EAAE,UAAU,EAAE,IAAI,KAAK,QAAQ;QAAE,OAAO,IAAI,CAAC,SAAS,CAAC,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE,GAAG,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC,CAAC;IACpH,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,GAAG,UAAU,EAAE,GAAG,GAAG,CAAC,UAAU,CAAC;IAC5D,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC,UAAU,CAAC,WAAW,CAAC,CAAC;IACpD,MAAM,OAAO,GAAG,MAAM,EAAE,UAAU,EAAE,IAAI,KAAK,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,QAAQ,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IACxI,MAAM,SAAS,GAAG,KAAK,CAAC,mBAAmB,CAAC,QAAQ,CAAC,CAAC;IACtD,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,UAAU,EAAE,GAAG,UAAU,CAAC,SAAS,CAAC;IAC1D,MAAM,KAAK,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC;IAC7B,OAAO,IAAI,CAAC,SAAS,CAAC;QACpB,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,UAAU;QAC9B,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,QAAQ,IAAI,SAAS,CAAC,CAAC,CAAC,QAAQ;QAC3D,SAAS,EAAE,SAAS,IAAI,QAAQ;QAChC,SAAS,EAAE,SAAS,CAAC,CAAC,CAAC,KAAK,CAAC,mBAAmB,CAAC,QAAQ,EAAE,SAAS,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,QAAQ;QAC3F,OAAO,EAAE,oBAAoB,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC,GAAG,CAAC,YAAY,CAAC;QAClE,QAAQ,EAAE,YAAY,CAAC,IAAI,CAAC;QAC5B,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC,KAAK,CAAC,EAAE,aAAa,CAAC,IAAI,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC,EAAE,gBAAgB,CAAC,IAAI,CAAC,KAAK,EAAE,sBAAsB,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ;QAC7I,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE;gBACrB,gBAAgB,CAAC,IAAI,CAAC,SAAS,EAAE,wBAAwB,EAAE,GAAG,UAAU,OAAO,CAAC,CAAC;gBACjF,gBAAgB,CAAC,IAAI,CAAC,SAAS,EAAE,wBAAwB,EAAE,GAAG,UAAU,eAAe,CAAC,CAAC;gBACzF,aAAa,CAAC,IAAI,CAAC,SAAS,EAAE,YAAY,EAAE,MAAM,CAAC,CAAC;gBACpD,aAAa,CAAC,IAAI,CAAC,SAAS,EAAE,aAAa,CAAC,CAAC;gBAC7C,aAAa,CAAC,IAAI,CAAC,SAAS,EAAE,iBAAiB,EAAE,MAAM,CAAC,CAAC;gBACzD,aAAa,CAAC,IAAI,CAAC,SAAS,EAAE,WAAW,CAAC,CAAC;aAC5C,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KACV,CAAC,CAAC;AACL,CAAC;AAED,+EAA+E;AAC/E,MAAM,UAAU,aAAa,CAAC,IAAuB;IACnD,OAAO,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,GAAG,MAAM,UAAU,GAAG,iBAAiB,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,EAAE,GAAG,EAAE,GAAG,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;AAChI,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,gBAAgB,CAAC,EAAE,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,OAAO,EAA+F;IACjK,IAAI,CAAC,OAAO,CAAC,MAAM;QAAE,OAAO,IAAI,CAAC;IACjC,MAAM,GAAG,GAAG,KAAK,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;IACnC,IAAI,GAAG,EAAE,UAAU,EAAE,IAAI,KAAK,QAAQ;QAAE,OAAO,KAAK,CAAC;IACrD,MAAM,SAAS,GAAG,KAAK,CAAC,mBAAmB,CAAC,QAAQ,CAAC,CAAC;IACtD,MAAM,MAAM,GAAG,SAAS,CAAC,CAAC,CAAC,KAAK,CAAC,mBAAmB,CAAC,QAAQ,EAAE,SAAS,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC;IAChG,MAAM,QAAQ,GAAG,OAAO,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,UAAU,EAAE,GAAG,MAAM,CAAC,SAAS,CAAC,CAAC;IAC5F,MAAM,KAAK,GAAG,CAAC,GAAG,CAAC,UAAU,CAAC,SAAS,CAAC,IAAI,EAAE,GAAG,oBAAoB,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC,CAAC;IAC1F,qHAAqH;IACrH,MAAM,QAAQ,GAAG,CAAC,MAAuB,EAAE,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,GAAG,KAAK,MAAM,CAAC,GAAG,IAAI,mBAAmB,CAAC,KAAK,CAAC,CAAC,CAAC;IAC7H,OAAO,OAAO,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,mBAAmB,CAAC,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,eAAe,CAAC,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,KAAK,KAAK,CAAC,CAAC,CAAC;AACpI,CAAC"}
|
|
@@ -37,22 +37,40 @@ export type ProcReader = {
|
|
|
37
37
|
readdir: () => string[];
|
|
38
38
|
readlink: (pid: string) => string;
|
|
39
39
|
};
|
|
40
|
-
/** The darwin scan: `lsof` for working directories. */
|
|
40
|
+
/** The darwin scan: `lsof` for working directories, of every process or of one. */
|
|
41
41
|
export type DarwinReader = {
|
|
42
42
|
lsof: () => {
|
|
43
43
|
ok: boolean;
|
|
44
44
|
stdout: string;
|
|
45
45
|
};
|
|
46
|
+
lsofPid?: (pid: number) => {
|
|
47
|
+
ok: boolean;
|
|
48
|
+
stdout: string;
|
|
49
|
+
};
|
|
46
50
|
};
|
|
51
|
+
/** Each other process's readable working directory, by pid, or `unknown` when none can be listed. */
|
|
52
|
+
export type CwdSnapshot = ReadonlyMap<number, string> | 'unknown';
|
|
53
|
+
/** A snapshot taken on first use. Shared only inside one synchronous reprobe tick, never across
|
|
54
|
+
* a yield; destroy, admission and history deletion scan fresh (hearsay-tools/cezarion#879). */
|
|
55
|
+
export type CwdSource = () => CwdSnapshot;
|
|
56
|
+
/** Every other process's readable working directory, in one scan pass; `unknown` when the
|
|
57
|
+
* processes cannot be listed at all. An unreadable cwd is no evidence (hearsay-tools/cezarion#889):
|
|
58
|
+
* a process that vanished, another user's, or a non-dumpable one of ours (login `sshd`,
|
|
59
|
+
* `systemd --user`, `gpg-agent`, which every host has) is skipped, as `lsof` skips what it cannot
|
|
60
|
+
* read. win32 has no cwd scan and answers none: Windows refuses to delete a directory a process
|
|
61
|
+
* holds, so the checked removal is the proof there, beside the recorded processes. */
|
|
62
|
+
export declare function scanProcessCwds(platform?: NodeJS.Platform, proc?: ProcReader, darwin?: DarwinReader): CwdSnapshot;
|
|
63
|
+
/** One lazy `scan`, memoized for the life of the returned source. */
|
|
64
|
+
export declare function sharedCwdScan(scan?: () => CwdSnapshot): CwdSource;
|
|
47
65
|
/** PIDs (never this process) whose readable working directory is one of `dirs` or beneath it,
|
|
48
|
-
*
|
|
49
|
-
* git children in a worktree make this read "alive" briefly: conservative, and a retry self-heals.
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
*
|
|
53
|
-
*
|
|
54
|
-
*
|
|
55
|
-
export declare function
|
|
66
|
+
* from `cwds` when the caller shares a snapshot, else from a fresh scan. cezar's own short-lived
|
|
67
|
+
* git children in a worktree make this read "alive" briefly: conservative, and a retry self-heals. */
|
|
68
|
+
export declare function processesWithCwdUnder(dirs: string | readonly string[], platform?: NodeJS.Platform, proc?: ProcReader, darwin?: DarwinReader, cwds?: CwdSnapshot): number[] | 'unknown';
|
|
69
|
+
/** Whether one process's working directory is still one of `dirs` or beneath it, without a scan:
|
|
70
|
+
* one `readlink` on Linux, one `lsof` of that process on darwin. `false` when it moved away or
|
|
71
|
+
* cannot be read (no evidence, hearsay-tools/cezarion#889); `undefined` on win32, whose scan names
|
|
72
|
+
* no holder in the first place. */
|
|
73
|
+
export declare function processCwdUnder(pid: number, dirs: readonly string[], platform?: NodeJS.Platform, proc?: ProcReader, darwin?: DarwinReader): boolean | undefined;
|
|
56
74
|
export type GenerationProbe = {
|
|
57
75
|
liveness: GenerationLiveness;
|
|
58
76
|
controller?: number;
|
|
@@ -63,10 +81,12 @@ export type GenerationProbe = {
|
|
|
63
81
|
* blocks only when its readable cwd is under `paths`, when it is a live recorded process of the
|
|
64
82
|
* generation, or when it is a live foreign controller (hearsay-tools/cezarion#889). `paths` are
|
|
65
83
|
* the worktree and every scratch location; a missing record (legacy) relies on the scan alone.
|
|
66
|
-
* A live foreign controller short-circuits the scan; otherwise `pids` names every holder.
|
|
67
|
-
|
|
84
|
+
* A live foreign controller short-circuits the scan; otherwise `pids` names every holder.
|
|
85
|
+
* `cwds` is a reprobe tick's shared snapshot (hearsay-tools/cezarion#879); without it, it scans. */
|
|
86
|
+
export declare function inspectGeneration({ record, paths, cwds }: {
|
|
68
87
|
record?: WorkerProcessRecord;
|
|
69
88
|
paths: readonly string[];
|
|
89
|
+
cwds?: CwdSource;
|
|
70
90
|
}): GenerationProbe;
|
|
71
91
|
export declare function probeGeneration(input: {
|
|
72
92
|
record?: WorkerProcessRecord;
|
|
@@ -79,4 +99,5 @@ export declare function inspectExecutionGeneration(input: {
|
|
|
79
99
|
record?: WorkerProcessRecord;
|
|
80
100
|
paths: readonly string[];
|
|
81
101
|
pathsComplete?: boolean;
|
|
102
|
+
cwds?: CwdSource;
|
|
82
103
|
}): GenerationProbe;
|
|
@@ -98,36 +98,29 @@ export function isCurrentProcess(entry) {
|
|
|
98
98
|
const own = processStartToken(process.pid);
|
|
99
99
|
return entry.startToken === undefined || own === undefined || sameIncarnation(entry.startToken, own);
|
|
100
100
|
}
|
|
101
|
+
const lsofCwd = (args, timeout) => {
|
|
102
|
+
const lsof = spawnSync('lsof', [...args, '-d', 'cwd', '-Fpn'], { encoding: 'utf8', timeout, maxBuffer: 16 * 1024 * 1024 });
|
|
103
|
+
// Status 1 is ordinary: some process was unreadable, and an unreadable cwd is no evidence.
|
|
104
|
+
return { ok: !lsof.error && (lsof.status === 0 || lsof.status === 1) && !!lsof.stdout, stdout: lsof.stdout ?? '' };
|
|
105
|
+
};
|
|
101
106
|
const realDarwin = {
|
|
102
|
-
lsof: () =>
|
|
103
|
-
|
|
104
|
-
// Status 1 is ordinary: some process was unreadable, and an unreadable cwd is no evidence.
|
|
105
|
-
return { ok: !lsof.error && (lsof.status === 0 || lsof.status === 1) && !!lsof.stdout, stdout: lsof.stdout ?? '' };
|
|
106
|
-
},
|
|
107
|
+
lsof: () => lsofCwd(['-a'], 5_000),
|
|
108
|
+
lsofPid: pid => lsofCwd(['-a', '-p', String(pid)], 2_000),
|
|
107
109
|
};
|
|
108
110
|
const realProc = {
|
|
109
111
|
readdir: () => readdirSync('/proc'),
|
|
110
112
|
readlink: pid => readlinkSync(`/proc/${pid}/cwd`),
|
|
111
113
|
};
|
|
112
|
-
/**
|
|
113
|
-
*
|
|
114
|
-
*
|
|
115
|
-
*
|
|
116
|
-
*
|
|
117
|
-
*
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
export function processesWithCwdUnder(dirs, platform = process.platform, proc = realProc, darwin = realDarwin) {
|
|
114
|
+
/** Every other process's readable working directory, in one scan pass; `unknown` when the
|
|
115
|
+
* processes cannot be listed at all. An unreadable cwd is no evidence (hearsay-tools/cezarion#889):
|
|
116
|
+
* a process that vanished, another user's, or a non-dumpable one of ours (login `sshd`,
|
|
117
|
+
* `systemd --user`, `gpg-agent`, which every host has) is skipped, as `lsof` skips what it cannot
|
|
118
|
+
* read. win32 has no cwd scan and answers none: Windows refuses to delete a directory a process
|
|
119
|
+
* holds, so the checked removal is the proof there, beside the recorded processes. */
|
|
120
|
+
export function scanProcessCwds(platform = process.platform, proc = realProc, darwin = realDarwin) {
|
|
121
|
+
const found = new Map();
|
|
121
122
|
if (platform === 'win32')
|
|
122
|
-
return
|
|
123
|
-
const targets = (typeof dirs === 'string' ? [dirs] : dirs).map(dir => { try {
|
|
124
|
-
return realpathSync(dir);
|
|
125
|
-
}
|
|
126
|
-
catch {
|
|
127
|
-
return resolve(dir);
|
|
128
|
-
} });
|
|
129
|
-
const under = (cwd) => { const path = cwd.replace(/ \(deleted\)$/, ''); return targets.some(target => path === target || path.startsWith(target + sep)); };
|
|
130
|
-
const found = new Set();
|
|
123
|
+
return found;
|
|
131
124
|
if (platform === 'linux') {
|
|
132
125
|
let entries;
|
|
133
126
|
try {
|
|
@@ -140,12 +133,11 @@ export function processesWithCwdUnder(dirs, platform = process.platform, proc =
|
|
|
140
133
|
if (!/^\d+$/.test(entry) || Number(entry) === process.pid)
|
|
141
134
|
continue;
|
|
142
135
|
try {
|
|
143
|
-
|
|
144
|
-
found.add(Number(entry));
|
|
136
|
+
found.set(Number(entry), proc.readlink(entry));
|
|
145
137
|
}
|
|
146
138
|
catch { /* unreadable: no evidence */ }
|
|
147
139
|
}
|
|
148
|
-
return
|
|
140
|
+
return found;
|
|
149
141
|
}
|
|
150
142
|
if (platform !== 'darwin')
|
|
151
143
|
return 'unknown';
|
|
@@ -156,22 +148,69 @@ export function processesWithCwdUnder(dirs, platform = process.platform, proc =
|
|
|
156
148
|
for (const line of lsof.stdout.split('\n')) {
|
|
157
149
|
if (line.startsWith('p'))
|
|
158
150
|
pid = Number(line.slice(1));
|
|
159
|
-
else if (line.startsWith('n') && pid !== undefined && pid !== process.pid
|
|
160
|
-
found.
|
|
151
|
+
else if (line.startsWith('n') && pid !== undefined && pid !== process.pid)
|
|
152
|
+
found.set(pid, line.slice(1));
|
|
161
153
|
}
|
|
162
|
-
return
|
|
154
|
+
return found;
|
|
155
|
+
}
|
|
156
|
+
/** One lazy `scan`, memoized for the life of the returned source. */
|
|
157
|
+
export function sharedCwdScan(scan = scanProcessCwds) {
|
|
158
|
+
let snapshot;
|
|
159
|
+
return () => snapshot ??= scan();
|
|
160
|
+
}
|
|
161
|
+
/** PIDs (never this process) whose readable working directory is one of `dirs` or beneath it,
|
|
162
|
+
* from `cwds` when the caller shares a snapshot, else from a fresh scan. cezar's own short-lived
|
|
163
|
+
* git children in a worktree make this read "alive" briefly: conservative, and a retry self-heals. */
|
|
164
|
+
export function processesWithCwdUnder(dirs, platform = process.platform, proc = realProc, darwin = realDarwin, cwds) {
|
|
165
|
+
const snapshot = cwds ?? scanProcessCwds(platform, proc, darwin);
|
|
166
|
+
if (snapshot === 'unknown')
|
|
167
|
+
return 'unknown';
|
|
168
|
+
const targets = (typeof dirs === 'string' ? [dirs] : dirs).map(dir => { try {
|
|
169
|
+
return realpathSync(dir);
|
|
170
|
+
}
|
|
171
|
+
catch {
|
|
172
|
+
return resolve(dir);
|
|
173
|
+
} });
|
|
174
|
+
const under = (cwd) => { const path = cwd.replace(/ \(deleted\)$/, ''); return targets.some(target => path === target || path.startsWith(target + sep)); };
|
|
175
|
+
return [...snapshot].filter(([, cwd]) => under(cwd)).map(([pid]) => pid);
|
|
176
|
+
}
|
|
177
|
+
/** Whether one process's working directory is still one of `dirs` or beneath it, without a scan:
|
|
178
|
+
* one `readlink` on Linux, one `lsof` of that process on darwin. `false` when it moved away or
|
|
179
|
+
* cannot be read (no evidence, hearsay-tools/cezarion#889); `undefined` on win32, whose scan names
|
|
180
|
+
* no holder in the first place. */
|
|
181
|
+
export function processCwdUnder(pid, dirs, platform = process.platform, proc = realProc, darwin = realDarwin) {
|
|
182
|
+
let cwd;
|
|
183
|
+
if (platform === 'linux') {
|
|
184
|
+
try {
|
|
185
|
+
cwd = proc.readlink(String(pid));
|
|
186
|
+
}
|
|
187
|
+
catch {
|
|
188
|
+
return false;
|
|
189
|
+
}
|
|
190
|
+
}
|
|
191
|
+
else if (platform === 'darwin') {
|
|
192
|
+
const lsof = darwin.lsofPid?.(pid);
|
|
193
|
+
cwd = lsof?.ok ? lsof.stdout.split('\n').find(line => line.startsWith('n'))?.slice(1) : undefined;
|
|
194
|
+
if (cwd === undefined)
|
|
195
|
+
return false;
|
|
196
|
+
}
|
|
197
|
+
else
|
|
198
|
+
return undefined;
|
|
199
|
+
const found = processesWithCwdUnder(dirs, platform, proc, undefined, new Map([[pid, cwd]]));
|
|
200
|
+
return found !== 'unknown' && found.length > 0;
|
|
163
201
|
}
|
|
164
202
|
/** Fresh resource proof for destroy, reuse, admission, scratch cleanup and history deletion.
|
|
165
203
|
* Controller boot says nothing about who holds persistent paths, so it always scans. A process
|
|
166
204
|
* blocks only when its readable cwd is under `paths`, when it is a live recorded process of the
|
|
167
205
|
* generation, or when it is a live foreign controller (hearsay-tools/cezarion#889). `paths` are
|
|
168
206
|
* the worktree and every scratch location; a missing record (legacy) relies on the scan alone.
|
|
169
|
-
* A live foreign controller short-circuits the scan; otherwise `pids` names every holder.
|
|
170
|
-
|
|
207
|
+
* A live foreign controller short-circuits the scan; otherwise `pids` names every holder.
|
|
208
|
+
* `cwds` is a reprobe tick's shared snapshot (hearsay-tools/cezarion#879); without it, it scans. */
|
|
209
|
+
export function inspectGeneration({ record, paths, cwds }) {
|
|
171
210
|
if (record && !isCurrentProcess(record.controller) && recordedProcessLive(record.controller))
|
|
172
211
|
return { liveness: 'alive', controller: record.controller.pid, pids: [] };
|
|
173
212
|
const recorded = record?.processes.filter(recordedProcessLive).map(entry => entry.pid) ?? [];
|
|
174
|
-
const scan = processesWithCwdUnder(paths);
|
|
213
|
+
const scan = processesWithCwdUnder(paths, undefined, undefined, undefined, cwds?.());
|
|
175
214
|
const pids = [...new Set([...recorded, ...(scan === 'unknown' ? [] : scan)])];
|
|
176
215
|
return { liveness: pids.length ? 'alive' : scan === 'unknown' ? 'unknown' : 'gone', pids };
|
|
177
216
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"process-liveness.js","sourceRoot":"","sources":["../../src/delegation/process-liveness.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,oBAAoB,CAAC;AAC/C,OAAO,EAAE,WAAW,EAAE,YAAY,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AAChF,OAAO,EAAE,OAAO,EAAE,GAAG,EAAE,MAAM,WAAW,CAAC;AAazC,8GAA8G;AAC9G,MAAM,UAAU,aAAa,CAAC,IAAY;IACxC,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IACzE,OAAO,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,MAAM,CAAC,MAAM,IAAI,EAAE,IAAI,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAE,CAAC;QAC3E,CAAC,CAAC,EAAE,KAAK,EAAE,MAAM,CAAC,CAAC,CAAE,EAAE,UAAU,EAAE,MAAM,CAAC,EAAE,CAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;AACjE,CAAC;AAED,SAAS,QAAQ,CAAC,GAAW;IAC3B,IAAI,CAAC;QAAC,OAAO,aAAa,CAAC,YAAY,CAAC,SAAS,GAAG,OAAO,EAAE,MAAM,CAAC,CAAC,CAAC;IAAC,CAAC;IAAC,MAAM,CAAC;QAAC,OAAO,SAAS,CAAC;IAAC,CAAC;AACtG,CAAC;AAED,IAAI,MAAiC,CAAC;AACtC,gGAAgG;AAChG,SAAS,WAAW;IAClB,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;QAAC,IAAI,CAAC;YAAC,MAAM,GAAG,YAAY,CAAC,iCAAiC,EAAE,MAAM,CAAC,CAAC,IAAI,EAAE,IAAI,IAAI,CAAC;QAAC,CAAC;QAAC,MAAM,CAAC;YAAC,MAAM,GAAG,IAAI,CAAC;QAAC,CAAC;IAAC,CAAC;IAC/I,OAAO,MAAM,IAAI,SAAS,CAAC;AAC7B,CAAC;AAED,mGAAmG;AACnG,MAAM,UAAU,iBAAiB,CAAC,GAAW,EAAE,QAAQ,GAAoB,OAAO,CAAC,QAAQ;IACzF,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,GAAG,CAAC,IAAI,GAAG,IAAI,CAAC;QAAE,OAAO,SAAS,CAAC;IAC7D,IAAI,QAAQ,KAAK,OAAO,EAAE,CAAC;QACzB,MAAM,KAAK,GAAG,QAAQ,CAAC,GAAG,CAAC,EAAE,UAAU,CAAC;QAAC,MAAM,IAAI,GAAG,WAAW,EAAE,CAAC;QACpE,OAAO,KAAK,IAAI,IAAI,CAAC,CAAC,CAAC,GAAG,IAAI,IAAI,KAAK,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC;IACpD,CAAC;IACD,IAAI,QAAQ,KAAK,QAAQ;QAAE,OAAO,SAAS,CAAC;IAC5C,kGAAkG;IAClG,MAAM,EAAE,GAAG,SAAS,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE,GAAG,OAAO,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,EAAE,EAAE,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC;IACxJ,MAAM,KAAK,GAAG,EAAE,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;IACtD,OAAO,KAAK,IAAI,SAAS,CAAC;AAC5B,CAAC;AAED,SAAS,SAAS,CAAC,GAAW;IAC5B,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,GAAG,CAAC,IAAI,GAAG,IAAI,CAAC;QAAE,OAAO,KAAK,CAAC;IACzD,IAAI,CAAC;QAAC,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC;IAAC,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QAAC,OAAQ,KAA+B,CAAC,IAAI,KAAK,OAAO,CAAC;IAAC,CAAC;IACzG,uDAAuD;IACvD,OAAO,OAAO,CAAC,QAAQ,KAAK,OAAO,IAAI,QAAQ,CAAC,GAAG,CAAC,EAAE,KAAK,KAAK,GAAG,CAAC;AACtE,CAAC;AAED,MAAM,WAAW,GAAG,4BAA4B,CAAC;AACjD,MAAM,aAAa,GAAG,+CAA+C,CAAC;AACtE,2FAA2F;AAC3F,SAAS,sBAAsB,CAAC,KAAyB,EAAE,IAAwB;IACjF,MAAM,YAAY,GAAG,KAAK,EAAE,KAAK,CAAC,YAAY,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IACrD,OAAO,YAAY,KAAK,SAAS,IAAI,IAAI,KAAK,SAAS,IAAI,aAAa,CAAC,IAAI,CAAC,YAAY,CAAC,IAAI,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,YAAY,KAAK,IAAI,CAAC;AACnJ,CAAC;AACD;mFACmF;AACnF,SAAS,eAAe,CAAC,QAAgB,EAAE,OAAe;IACxD,IAAI,QAAQ,KAAK,OAAO;QAAE,OAAO,IAAI,CAAC;IACtC,MAAM,CAAC,GAAG,WAAW,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,GAAG,WAAW,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACpE,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,KAAK,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;AACzF,CAAC;AAED,sHAAsH;AACtH,MAAM,UAAU,mBAAmB,CAAC,KAAsB;IACxD,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,GAAG,CAAC;QAAE,OAAO,KAAK,CAAC;IACxC,IAAI,KAAK,CAAC,UAAU,KAAK,SAAS;QAAE,OAAO,IAAI,CAAC;IAChD,MAAM,OAAO,GAAG,iBAAiB,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IAC7C,OAAO,OAAO,KAAK,SAAS,IAAI,eAAe,CAAC,KAAK,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;AAC7E,CAAC;AAED;;;;4FAI4F;AAC5F,MAAM,UAAU,uBAAuB,CAAC,KAAsB,EAAE,YAAY,GAAsB,EAAE;IAClG,IAAI,KAAK,CAAC,IAAI,KAAK,SAAS,IAAI,KAAK,CAAC,IAAI,KAAK,KAAK,CAAC,GAAG;QAAE,OAAO,KAAK,CAAC;IACvE,IAAI,SAAS,CAAC,KAAK,CAAC,GAAG,CAAC;QAAE,OAAO,KAAK,CAAC,UAAU,KAAK,SAAS,IAAI,iBAAiB,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,KAAK,CAAC,UAAU,CAAC;IACrH,OAAO,YAAY,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;AAC3C,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,KAAsB;IACrD,IAAI,KAAK,CAAC,GAAG,KAAK,OAAO,CAAC,GAAG;QAAE,OAAO,KAAK,CAAC;IAC5C,MAAM,GAAG,GAAG,iBAAiB,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IAC3C,OAAO,KAAK,CAAC,UAAU,KAAK,SAAS,IAAI,GAAG,KAAK,SAAS,IAAI,eAAe,CAAC,KAAK,CAAC,UAAU,EAAE,GAAG,CAAC,CAAC;AACvG,CAAC;AAMD,MAAM,
|
|
1
|
+
{"version":3,"file":"process-liveness.js","sourceRoot":"","sources":["../../src/delegation/process-liveness.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,oBAAoB,CAAC;AAC/C,OAAO,EAAE,WAAW,EAAE,YAAY,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AAChF,OAAO,EAAE,OAAO,EAAE,GAAG,EAAE,MAAM,WAAW,CAAC;AAazC,8GAA8G;AAC9G,MAAM,UAAU,aAAa,CAAC,IAAY;IACxC,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IACzE,OAAO,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,MAAM,CAAC,MAAM,IAAI,EAAE,IAAI,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAE,CAAC;QAC3E,CAAC,CAAC,EAAE,KAAK,EAAE,MAAM,CAAC,CAAC,CAAE,EAAE,UAAU,EAAE,MAAM,CAAC,EAAE,CAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;AACjE,CAAC;AAED,SAAS,QAAQ,CAAC,GAAW;IAC3B,IAAI,CAAC;QAAC,OAAO,aAAa,CAAC,YAAY,CAAC,SAAS,GAAG,OAAO,EAAE,MAAM,CAAC,CAAC,CAAC;IAAC,CAAC;IAAC,MAAM,CAAC;QAAC,OAAO,SAAS,CAAC;IAAC,CAAC;AACtG,CAAC;AAED,IAAI,MAAiC,CAAC;AACtC,gGAAgG;AAChG,SAAS,WAAW;IAClB,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;QAAC,IAAI,CAAC;YAAC,MAAM,GAAG,YAAY,CAAC,iCAAiC,EAAE,MAAM,CAAC,CAAC,IAAI,EAAE,IAAI,IAAI,CAAC;QAAC,CAAC;QAAC,MAAM,CAAC;YAAC,MAAM,GAAG,IAAI,CAAC;QAAC,CAAC;IAAC,CAAC;IAC/I,OAAO,MAAM,IAAI,SAAS,CAAC;AAC7B,CAAC;AAED,mGAAmG;AACnG,MAAM,UAAU,iBAAiB,CAAC,GAAW,EAAE,QAAQ,GAAoB,OAAO,CAAC,QAAQ;IACzF,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,GAAG,CAAC,IAAI,GAAG,IAAI,CAAC;QAAE,OAAO,SAAS,CAAC;IAC7D,IAAI,QAAQ,KAAK,OAAO,EAAE,CAAC;QACzB,MAAM,KAAK,GAAG,QAAQ,CAAC,GAAG,CAAC,EAAE,UAAU,CAAC;QAAC,MAAM,IAAI,GAAG,WAAW,EAAE,CAAC;QACpE,OAAO,KAAK,IAAI,IAAI,CAAC,CAAC,CAAC,GAAG,IAAI,IAAI,KAAK,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC;IACpD,CAAC;IACD,IAAI,QAAQ,KAAK,QAAQ;QAAE,OAAO,SAAS,CAAC;IAC5C,kGAAkG;IAClG,MAAM,EAAE,GAAG,SAAS,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE,GAAG,OAAO,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,EAAE,EAAE,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC;IACxJ,MAAM,KAAK,GAAG,EAAE,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;IACtD,OAAO,KAAK,IAAI,SAAS,CAAC;AAC5B,CAAC;AAED,SAAS,SAAS,CAAC,GAAW;IAC5B,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,GAAG,CAAC,IAAI,GAAG,IAAI,CAAC;QAAE,OAAO,KAAK,CAAC;IACzD,IAAI,CAAC;QAAC,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC;IAAC,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QAAC,OAAQ,KAA+B,CAAC,IAAI,KAAK,OAAO,CAAC;IAAC,CAAC;IACzG,uDAAuD;IACvD,OAAO,OAAO,CAAC,QAAQ,KAAK,OAAO,IAAI,QAAQ,CAAC,GAAG,CAAC,EAAE,KAAK,KAAK,GAAG,CAAC;AACtE,CAAC;AAED,MAAM,WAAW,GAAG,4BAA4B,CAAC;AACjD,MAAM,aAAa,GAAG,+CAA+C,CAAC;AACtE,2FAA2F;AAC3F,SAAS,sBAAsB,CAAC,KAAyB,EAAE,IAAwB;IACjF,MAAM,YAAY,GAAG,KAAK,EAAE,KAAK,CAAC,YAAY,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IACrD,OAAO,YAAY,KAAK,SAAS,IAAI,IAAI,KAAK,SAAS,IAAI,aAAa,CAAC,IAAI,CAAC,YAAY,CAAC,IAAI,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,YAAY,KAAK,IAAI,CAAC;AACnJ,CAAC;AACD;mFACmF;AACnF,SAAS,eAAe,CAAC,QAAgB,EAAE,OAAe;IACxD,IAAI,QAAQ,KAAK,OAAO;QAAE,OAAO,IAAI,CAAC;IACtC,MAAM,CAAC,GAAG,WAAW,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,GAAG,WAAW,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACpE,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,KAAK,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;AACzF,CAAC;AAED,sHAAsH;AACtH,MAAM,UAAU,mBAAmB,CAAC,KAAsB;IACxD,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,GAAG,CAAC;QAAE,OAAO,KAAK,CAAC;IACxC,IAAI,KAAK,CAAC,UAAU,KAAK,SAAS;QAAE,OAAO,IAAI,CAAC;IAChD,MAAM,OAAO,GAAG,iBAAiB,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IAC7C,OAAO,OAAO,KAAK,SAAS,IAAI,eAAe,CAAC,KAAK,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;AAC7E,CAAC;AAED;;;;4FAI4F;AAC5F,MAAM,UAAU,uBAAuB,CAAC,KAAsB,EAAE,YAAY,GAAsB,EAAE;IAClG,IAAI,KAAK,CAAC,IAAI,KAAK,SAAS,IAAI,KAAK,CAAC,IAAI,KAAK,KAAK,CAAC,GAAG;QAAE,OAAO,KAAK,CAAC;IACvE,IAAI,SAAS,CAAC,KAAK,CAAC,GAAG,CAAC;QAAE,OAAO,KAAK,CAAC,UAAU,KAAK,SAAS,IAAI,iBAAiB,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,KAAK,CAAC,UAAU,CAAC;IACrH,OAAO,YAAY,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;AAC3C,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,KAAsB;IACrD,IAAI,KAAK,CAAC,GAAG,KAAK,OAAO,CAAC,GAAG;QAAE,OAAO,KAAK,CAAC;IAC5C,MAAM,GAAG,GAAG,iBAAiB,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IAC3C,OAAO,KAAK,CAAC,UAAU,KAAK,SAAS,IAAI,GAAG,KAAK,SAAS,IAAI,eAAe,CAAC,KAAK,CAAC,UAAU,EAAE,GAAG,CAAC,CAAC;AACvG,CAAC;AAMD,MAAM,OAAO,GAAG,CAAC,IAAc,EAAE,OAAe,EAAE,EAAE;IAClD,MAAM,IAAI,GAAG,SAAS,CAAC,MAAM,EAAE,CAAC,GAAG,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,CAAC,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,SAAS,EAAE,EAAE,GAAG,IAAI,GAAG,IAAI,EAAE,CAAC,CAAC;IAC3H,2FAA2F;IAC3F,OAAO,EAAE,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,IAAI,CAAC,IAAI,CAAC,MAAM,KAAK,CAAC,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,MAAM,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,IAAI,EAAE,EAAE,CAAC;AACrH,CAAC,CAAC;AACF,MAAM,UAAU,GAAiB;IAC/B,IAAI,EAAE,GAAG,EAAE,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,EAAE,KAAK,CAAC;IAClC,OAAO,EAAE,GAAG,CAAC,EAAE,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,IAAI,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC;CAC1D,CAAC;AAEF,MAAM,QAAQ,GAAe;IAC3B,OAAO,EAAE,GAAG,EAAE,CAAC,WAAW,CAAC,OAAO,CAAC;IACnC,QAAQ,EAAE,GAAG,CAAC,EAAE,CAAC,YAAY,CAAC,SAAS,GAAG,MAAM,CAAC;CAClD,CAAC;AAQF;;;;;sFAKsF;AACtF,MAAM,UAAU,eAAe,CAAC,QAAQ,GAAoB,OAAO,CAAC,QAAQ,EAAE,IAAI,GAAe,QAAQ,EAAE,MAAM,GAAiB,UAAU;IAC1I,MAAM,KAAK,GAAG,IAAI,GAAG,EAAkB,CAAC;IACxC,IAAI,QAAQ,KAAK,OAAO;QAAE,OAAO,KAAK,CAAC;IACvC,IAAI,QAAQ,KAAK,OAAO,EAAE,CAAC;QACzB,IAAI,OAAiB,CAAC;QACtB,IAAI,CAAC;YAAC,OAAO,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC;QAAC,CAAC;QAAC,MAAM,CAAC;YAAC,OAAO,SAAS,CAAC;QAAC,CAAC;QAC7D,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;YAC5B,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,MAAM,CAAC,KAAK,CAAC,KAAK,OAAO,CAAC,GAAG;gBAAE,SAAS;YACpE,IAAI,CAAC;gBAAC,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC;YAAC,CAAC;YAAC,MAAM,CAAC,CAAC,6BAA6B,CAAC,CAAC;QACjG,CAAC;QACD,OAAO,KAAK,CAAC;IACf,CAAC;IACD,IAAI,QAAQ,KAAK,QAAQ;QAAE,OAAO,SAAS,CAAC;IAC5C,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,EAAE,CAAC;IAC3B,IAAI,CAAC,IAAI,CAAC,EAAE;QAAE,OAAO,SAAS,CAAC;IAC/B,IAAI,GAAuB,CAAC;IAC5B,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;QAC3C,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC;YAAE,GAAG,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;aACjD,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,GAAG,KAAK,SAAS,IAAI,GAAG,KAAK,OAAO,CAAC,GAAG;YAAE,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;IAC3G,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,qEAAqE;AACrE,MAAM,UAAU,aAAa,CAAC,IAAI,GAAsB,eAAe;IACrE,IAAI,QAAiC,CAAC;IACtC,OAAO,GAAG,EAAE,CAAC,QAAQ,KAAK,IAAI,EAAE,CAAC;AACnC,CAAC;AAED;;sGAEsG;AACtG,MAAM,UAAU,qBAAqB,CAAC,IAAgC,EAAE,QAAQ,GAAoB,OAAO,CAAC,QAAQ,EAAE,IAAI,GAAe,QAAQ,EAC/I,MAAM,GAAiB,UAAU,EAAE,IAAkB;IACrD,MAAM,QAAQ,GAAG,IAAI,IAAI,eAAe,CAAC,QAAQ,EAAE,IAAI,EAAE,MAAM,CAAC,CAAC;IACjE,IAAI,QAAQ,KAAK,SAAS;QAAE,OAAO,SAAS,CAAC;IAC7C,MAAM,OAAO,GAAG,CAAC,OAAO,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,GAAG,IAAI,CAAC;QAAC,OAAO,YAAY,CAAC,GAAG,CAAC,CAAC;IAAC,CAAC;IAAC,MAAM,CAAC;QAAC,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC;IAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC5I,MAAM,KAAK,GAAG,CAAC,GAAW,EAAE,EAAE,GAAG,MAAM,IAAI,GAAG,GAAG,CAAC,OAAO,CAAC,eAAe,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,IAAI,KAAK,MAAM,IAAI,IAAI,CAAC,UAAU,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACnK,OAAO,CAAC,GAAG,QAAQ,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC;AAC3E,CAAC;AAED;;;mCAGmC;AACnC,MAAM,UAAU,eAAe,CAAC,GAAW,EAAE,IAAuB,EAAE,QAAQ,GAAoB,OAAO,CAAC,QAAQ,EAAE,IAAI,GAAe,QAAQ,EAC7I,MAAM,GAAiB,UAAU;IACjC,IAAI,GAAuB,CAAC;IAC5B,IAAI,QAAQ,KAAK,OAAO,EAAE,CAAC;QACzB,IAAI,CAAC;YAAC,GAAG,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;QAAC,CAAC;QAAC,MAAM,CAAC;YAAC,OAAO,KAAK,CAAC;QAAC,CAAC;IACnE,CAAC;SAAM,IAAI,QAAQ,KAAK,QAAQ,EAAE,CAAC;QACjC,MAAM,IAAI,GAAG,MAAM,CAAC,OAAO,EAAE,CAAC,GAAG,CAAC,CAAC;QACnC,GAAG,GAAG,IAAI,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;QAClG,IAAI,GAAG,KAAK,SAAS;YAAE,OAAO,KAAK,CAAC;IACtC,CAAC;;QAAM,OAAO,SAAS,CAAC;IACxB,MAAM,KAAK,GAAG,qBAAqB,CAAC,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;IAC5F,OAAO,KAAK,KAAK,SAAS,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC;AACjD,CAAC;AAID;;;;;;oGAMoG;AACpG,MAAM,UAAU,iBAAiB,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,IAAI,EAAgF;IACrI,IAAI,MAAM,IAAI,CAAC,gBAAgB,CAAC,MAAM,CAAC,UAAU,CAAC,IAAI,mBAAmB,CAAC,MAAM,CAAC,UAAU,CAAC;QAAE,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,UAAU,EAAE,MAAM,CAAC,UAAU,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC;IACxK,MAAM,QAAQ,GAAG,MAAM,EAAE,SAAS,CAAC,MAAM,CAAC,mBAAmB,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC;IAC7F,MAAM,IAAI,GAAG,qBAAqB,CAAC,KAAK,EAAE,SAAS,EAAE,SAAS,EAAE,SAAS,EAAE,IAAI,EAAE,EAAE,CAAC,CAAC;IACrF,MAAM,IAAI,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,QAAQ,EAAE,GAAG,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;IAC9E,OAAO,EAAE,QAAQ,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC;AAC7F,CAAC;AAED,MAAM,UAAU,eAAe,CAAC,KAAiE;IAC/F,OAAO,iBAAiB,CAAC,KAAK,CAAC,CAAC,QAAQ,CAAC;AAC3C,CAAC;AAED;;6GAE6G;AAC7G,MAAM,UAAU,0BAA0B,CAAC,KAA4G;IACrJ,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC;IAC5B,IAAI,MAAM,IAAI,mBAAmB,CAAC,MAAM,CAAC,UAAU,CAAC;QAAE,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,UAAU,EAAE,MAAM,CAAC,UAAU,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC;IAChI,MAAM,IAAI,GAAG,MAAM,EAAE,SAAS,CAAC,MAAM,CAAC,mBAAmB,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC;IACzF,IAAI,IAAI,CAAC,MAAM;QAAE,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC;IACpD,IAAI,OAAO,CAAC,QAAQ,KAAK,OAAO,IAAI,sBAAsB,CAAC,MAAM,EAAE,UAAU,CAAC,UAAU,EAAE,WAAW,EAAE,CAAC;QAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC;IAChJ,MAAM,KAAK,GAAG,iBAAiB,CAAC,KAAK,CAAC,CAAC;IACvC,uFAAuF;IACvF,6FAA6F;IAC7F,OAAO,KAAK,CAAC,QAAQ,KAAK,MAAM,IAAI,KAAK,CAAC,aAAa,KAAK,KAAK,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,SAAS,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC;AAChH,CAAC"}
|