@halcyonlabs/app 0.47.0-halcyon.106 → 0.47.0-halcyon.110
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +32 -32
- package/package.json +1 -1
- package/server/dist/builtin-plugins/orbit/dist/assets/orbit/bundle/scripts/orbit-job-supervisor +58 -40
- package/server/dist/builtin-plugins/orbit/dist/assets/orbit/recipes/base-linux/swap.sh +74 -0
- package/server/dist/builtin-plugins/orbit/dist/assets/orbit/recipes/base-linux.toml +3 -0
- package/server/dist/builtin-plugins/orbit/dist/assets/orbit/revision +1 -1
- package/server/dist/builtin-plugins/plugin-api-docs/dist/app.js +1 -1
- package/server/dist/builtin-plugins/plugin-api-docs/dist/server.js +1 -1
package/dist/index.d.ts
CHANGED
|
@@ -155,10 +155,10 @@ declare const uiPreferenceDefinitions: {
|
|
|
155
155
|
}>>;
|
|
156
156
|
readonly "sidebar.threadGrouping.environment": UiPreferenceDefinition<z$1.ZodUnion<readonly [z$1.ZodLiteral<"auto">, z$1.ZodBoolean]>>;
|
|
157
157
|
readonly "sidebar.chronologicalSort": UiPreferenceDefinition<z$1.ZodEnum<{
|
|
158
|
-
none: "none";
|
|
159
158
|
updated: "updated";
|
|
160
159
|
created: "created";
|
|
161
160
|
alpha: "alpha";
|
|
161
|
+
none: "none";
|
|
162
162
|
}>>;
|
|
163
163
|
readonly "sidebar.sortDirection": UiPreferenceDefinition<z$1.ZodEnum<{
|
|
164
164
|
default: "default";
|
|
@@ -280,8 +280,8 @@ declare const appKeybindingOverridesSchema: z$1.ZodArray<z$1.ZodObject<{
|
|
|
280
280
|
shift: z$1.ZodBoolean;
|
|
281
281
|
}, z$1.core.$strict>>;
|
|
282
282
|
platform: z$1.ZodOptional<z$1.ZodEnum<{
|
|
283
|
-
mac: "mac";
|
|
284
283
|
windows: "windows";
|
|
284
|
+
mac: "mac";
|
|
285
285
|
linux: "linux";
|
|
286
286
|
}>>;
|
|
287
287
|
}, z$1.core.$strict>>;
|
|
@@ -487,8 +487,8 @@ declare const environmentSchema: z$1.ZodObject<{
|
|
|
487
487
|
retireAt: z$1.ZodNullable<z$1.ZodNumber>;
|
|
488
488
|
teardown: z$1.ZodNullable<z$1.ZodObject<{
|
|
489
489
|
status: z$1.ZodEnum<{
|
|
490
|
-
running: "running";
|
|
491
490
|
failed: "failed";
|
|
491
|
+
running: "running";
|
|
492
492
|
removed: "removed";
|
|
493
493
|
}>;
|
|
494
494
|
attempt: z$1.ZodNumber;
|
|
@@ -583,8 +583,8 @@ declare const hostSchema: z$1.ZodObject<{
|
|
|
583
583
|
pendingLog: z$1.ZodString;
|
|
584
584
|
teardown: z$1.ZodNullable<z$1.ZodObject<{
|
|
585
585
|
status: z$1.ZodEnum<{
|
|
586
|
-
running: "running";
|
|
587
586
|
failed: "failed";
|
|
587
|
+
running: "running";
|
|
588
588
|
removed: "removed";
|
|
589
589
|
}>;
|
|
590
590
|
attempt: z$1.ZodNumber;
|
|
@@ -685,8 +685,8 @@ declare const threadEventSchema: z$1.ZodUnion<readonly [z$1.ZodIntersection<z$1.
|
|
|
685
685
|
providerThreadId: z$1.ZodString;
|
|
686
686
|
objective: z$1.ZodString;
|
|
687
687
|
status: z$1.ZodEnum<{
|
|
688
|
-
paused: "paused";
|
|
689
688
|
active: "active";
|
|
689
|
+
paused: "paused";
|
|
690
690
|
budgetLimited: "budgetLimited";
|
|
691
691
|
complete: "complete";
|
|
692
692
|
}>;
|
|
@@ -1257,9 +1257,9 @@ declare const threadEventSchema: z$1.ZodUnion<readonly [z$1.ZodIntersection<z$1.
|
|
|
1257
1257
|
step: z$1.ZodString;
|
|
1258
1258
|
status: z$1.ZodOptional<z$1.ZodEnum<{
|
|
1259
1259
|
pending: "pending";
|
|
1260
|
+
active: "active";
|
|
1260
1261
|
completed: "completed";
|
|
1261
1262
|
failed: "failed";
|
|
1262
|
-
active: "active";
|
|
1263
1263
|
}>>;
|
|
1264
1264
|
}, z$1.core.$strip>>;
|
|
1265
1265
|
explanation: z$1.ZodOptional<z$1.ZodString>;
|
|
@@ -1338,10 +1338,10 @@ declare const threadEventSchema: z$1.ZodUnion<readonly [z$1.ZodIntersection<z$1.
|
|
|
1338
1338
|
}>;
|
|
1339
1339
|
taskStatus: z$1.ZodEnum<{
|
|
1340
1340
|
pending: "pending";
|
|
1341
|
-
running: "running";
|
|
1342
|
-
paused: "paused";
|
|
1343
1341
|
completed: "completed";
|
|
1344
1342
|
failed: "failed";
|
|
1343
|
+
paused: "paused";
|
|
1344
|
+
running: "running";
|
|
1345
1345
|
killed: "killed";
|
|
1346
1346
|
stopped: "stopped";
|
|
1347
1347
|
}>;
|
|
@@ -1358,8 +1358,8 @@ declare const threadEventSchema: z$1.ZodUnion<readonly [z$1.ZodIntersection<z$1.
|
|
|
1358
1358
|
index: z$1.ZodNumber;
|
|
1359
1359
|
label: z$1.ZodString;
|
|
1360
1360
|
state: z$1.ZodEnum<{
|
|
1361
|
-
running: "running";
|
|
1362
1361
|
failed: "failed";
|
|
1362
|
+
running: "running";
|
|
1363
1363
|
queued: "queued";
|
|
1364
1364
|
done: "done";
|
|
1365
1365
|
skipped: "skipped";
|
|
@@ -2029,9 +2029,9 @@ declare const threadEventSchema: z$1.ZodUnion<readonly [z$1.ZodIntersection<z$1.
|
|
|
2029
2029
|
step: z$1.ZodString;
|
|
2030
2030
|
status: z$1.ZodOptional<z$1.ZodEnum<{
|
|
2031
2031
|
pending: "pending";
|
|
2032
|
+
active: "active";
|
|
2032
2033
|
completed: "completed";
|
|
2033
2034
|
failed: "failed";
|
|
2034
|
-
active: "active";
|
|
2035
2035
|
}>>;
|
|
2036
2036
|
}, z$1.core.$strip>>;
|
|
2037
2037
|
explanation: z$1.ZodOptional<z$1.ZodString>;
|
|
@@ -2110,10 +2110,10 @@ declare const threadEventSchema: z$1.ZodUnion<readonly [z$1.ZodIntersection<z$1.
|
|
|
2110
2110
|
}>;
|
|
2111
2111
|
taskStatus: z$1.ZodEnum<{
|
|
2112
2112
|
pending: "pending";
|
|
2113
|
-
running: "running";
|
|
2114
|
-
paused: "paused";
|
|
2115
2113
|
completed: "completed";
|
|
2116
2114
|
failed: "failed";
|
|
2115
|
+
paused: "paused";
|
|
2116
|
+
running: "running";
|
|
2117
2117
|
killed: "killed";
|
|
2118
2118
|
stopped: "stopped";
|
|
2119
2119
|
}>;
|
|
@@ -2130,8 +2130,8 @@ declare const threadEventSchema: z$1.ZodUnion<readonly [z$1.ZodIntersection<z$1.
|
|
|
2130
2130
|
index: z$1.ZodNumber;
|
|
2131
2131
|
label: z$1.ZodString;
|
|
2132
2132
|
state: z$1.ZodEnum<{
|
|
2133
|
-
running: "running";
|
|
2134
2133
|
failed: "failed";
|
|
2134
|
+
running: "running";
|
|
2135
2135
|
queued: "queued";
|
|
2136
2136
|
done: "done";
|
|
2137
2137
|
skipped: "skipped";
|
|
@@ -2342,10 +2342,10 @@ declare const threadEventSchema: z$1.ZodUnion<readonly [z$1.ZodIntersection<z$1.
|
|
|
2342
2342
|
}>;
|
|
2343
2343
|
taskStatus: z$1.ZodEnum<{
|
|
2344
2344
|
pending: "pending";
|
|
2345
|
-
running: "running";
|
|
2346
|
-
paused: "paused";
|
|
2347
2345
|
completed: "completed";
|
|
2348
2346
|
failed: "failed";
|
|
2347
|
+
paused: "paused";
|
|
2348
|
+
running: "running";
|
|
2349
2349
|
killed: "killed";
|
|
2350
2350
|
stopped: "stopped";
|
|
2351
2351
|
}>;
|
|
@@ -2362,8 +2362,8 @@ declare const threadEventSchema: z$1.ZodUnion<readonly [z$1.ZodIntersection<z$1.
|
|
|
2362
2362
|
index: z$1.ZodNumber;
|
|
2363
2363
|
label: z$1.ZodString;
|
|
2364
2364
|
state: z$1.ZodEnum<{
|
|
2365
|
-
running: "running";
|
|
2366
2365
|
failed: "failed";
|
|
2366
|
+
running: "running";
|
|
2367
2367
|
queued: "queued";
|
|
2368
2368
|
done: "done";
|
|
2369
2369
|
skipped: "skipped";
|
|
@@ -2441,10 +2441,10 @@ declare const threadEventSchema: z$1.ZodUnion<readonly [z$1.ZodIntersection<z$1.
|
|
|
2441
2441
|
}>;
|
|
2442
2442
|
taskStatus: z$1.ZodEnum<{
|
|
2443
2443
|
pending: "pending";
|
|
2444
|
-
running: "running";
|
|
2445
|
-
paused: "paused";
|
|
2446
2444
|
completed: "completed";
|
|
2447
2445
|
failed: "failed";
|
|
2446
|
+
paused: "paused";
|
|
2447
|
+
running: "running";
|
|
2448
2448
|
killed: "killed";
|
|
2449
2449
|
stopped: "stopped";
|
|
2450
2450
|
}>;
|
|
@@ -2461,8 +2461,8 @@ declare const threadEventSchema: z$1.ZodUnion<readonly [z$1.ZodIntersection<z$1.
|
|
|
2461
2461
|
index: z$1.ZodNumber;
|
|
2462
2462
|
label: z$1.ZodString;
|
|
2463
2463
|
state: z$1.ZodEnum<{
|
|
2464
|
-
running: "running";
|
|
2465
2464
|
failed: "failed";
|
|
2465
|
+
running: "running";
|
|
2466
2466
|
queued: "queued";
|
|
2467
2467
|
done: "done";
|
|
2468
2468
|
skipped: "skipped";
|
|
@@ -2657,9 +2657,9 @@ declare const threadEventSchema: z$1.ZodUnion<readonly [z$1.ZodIntersection<z$1.
|
|
|
2657
2657
|
step: z$1.ZodString;
|
|
2658
2658
|
status: z$1.ZodOptional<z$1.ZodEnum<{
|
|
2659
2659
|
pending: "pending";
|
|
2660
|
+
active: "active";
|
|
2660
2661
|
completed: "completed";
|
|
2661
2662
|
failed: "failed";
|
|
2662
|
-
active: "active";
|
|
2663
2663
|
}>>;
|
|
2664
2664
|
}, z$1.core.$strip>>;
|
|
2665
2665
|
explanation: z$1.ZodOptional<z$1.ZodString>;
|
|
@@ -2668,8 +2668,8 @@ declare const threadEventSchema: z$1.ZodUnion<readonly [z$1.ZodIntersection<z$1.
|
|
|
2668
2668
|
threadId: z$1.ZodString;
|
|
2669
2669
|
providerThreadId: z$1.ZodString;
|
|
2670
2670
|
state: z$1.ZodEnum<{
|
|
2671
|
-
running: "running";
|
|
2672
2671
|
idle: "idle";
|
|
2672
|
+
running: "running";
|
|
2673
2673
|
}>;
|
|
2674
2674
|
}, z$1.core.$strip>, z$1.ZodObject<{
|
|
2675
2675
|
type: z$1.ZodLiteral<"turn/diff/updated">;
|
|
@@ -3531,9 +3531,9 @@ declare const threadEventSchema: z$1.ZodUnion<readonly [z$1.ZodIntersection<z$1.
|
|
|
3531
3531
|
threadId: z$1.ZodString;
|
|
3532
3532
|
provisioningId: z$1.ZodString;
|
|
3533
3533
|
status: z$1.ZodEnum<{
|
|
3534
|
+
active: "active";
|
|
3534
3535
|
completed: "completed";
|
|
3535
3536
|
failed: "failed";
|
|
3536
|
-
active: "active";
|
|
3537
3537
|
cancelled: "cancelled";
|
|
3538
3538
|
}>;
|
|
3539
3539
|
environmentId: z$1.ZodNullable<z$1.ZodString>;
|
|
@@ -7456,8 +7456,8 @@ declare const hostDaemonCommandRegistry: {
|
|
|
7456
7456
|
skillFilePath: z$1.ZodString;
|
|
7457
7457
|
}, z$1.core.$strict>], "kind">>;
|
|
7458
7458
|
instructionMode: z$1.ZodEnum<{
|
|
7459
|
-
append: "append";
|
|
7460
7459
|
replace: "replace";
|
|
7460
|
+
append: "append";
|
|
7461
7461
|
}>;
|
|
7462
7462
|
type: z$1.ZodLiteral<"thread.rewind.prepare">;
|
|
7463
7463
|
leaseId: z$1.ZodString;
|
|
@@ -7605,8 +7605,8 @@ declare const hostDaemonCommandRegistry: {
|
|
|
7605
7605
|
skillFilePath: z$1.ZodString;
|
|
7606
7606
|
}, z$1.core.$strict>], "kind">>;
|
|
7607
7607
|
instructionMode: z$1.ZodEnum<{
|
|
7608
|
-
append: "append";
|
|
7609
7608
|
replace: "replace";
|
|
7609
|
+
append: "append";
|
|
7610
7610
|
}>;
|
|
7611
7611
|
type: z$1.ZodLiteral<"thread.start">;
|
|
7612
7612
|
requestId: z$1.ZodString;
|
|
@@ -8069,8 +8069,8 @@ declare const hostDaemonCommandRegistry: {
|
|
|
8069
8069
|
envPassthrough: z$1.ZodArray<z$1.ZodString>;
|
|
8070
8070
|
}, z$1.core.$strict>;
|
|
8071
8071
|
instructionMode: z$1.ZodEnum<{
|
|
8072
|
-
append: "append";
|
|
8073
8072
|
replace: "replace";
|
|
8073
|
+
append: "append";
|
|
8074
8074
|
}>;
|
|
8075
8075
|
projectId: z$1.ZodString;
|
|
8076
8076
|
providerId: z$1.ZodString;
|
|
@@ -8285,8 +8285,8 @@ declare const hostDaemonCommandRegistry: {
|
|
|
8285
8285
|
envPassthrough: z$1.ZodArray<z$1.ZodString>;
|
|
8286
8286
|
}, z$1.core.$strict>;
|
|
8287
8287
|
instructionMode: z$1.ZodEnum<{
|
|
8288
|
-
append: "append";
|
|
8289
8288
|
replace: "replace";
|
|
8289
|
+
append: "append";
|
|
8290
8290
|
}>;
|
|
8291
8291
|
projectId: z$1.ZodString;
|
|
8292
8292
|
providerId: z$1.ZodString;
|
|
@@ -9192,9 +9192,9 @@ declare const hostDaemonCommandRegistry: {
|
|
|
9192
9192
|
base64: "base64";
|
|
9193
9193
|
utf8: "utf8";
|
|
9194
9194
|
}>;
|
|
9195
|
-
mimeType: z$1.ZodOptional<z$1.ZodString>;
|
|
9196
|
-
sizeBytes: z$1.ZodNumber;
|
|
9197
9195
|
sha256: z$1.ZodString;
|
|
9196
|
+
sizeBytes: z$1.ZodNumber;
|
|
9197
|
+
mimeType: z$1.ZodOptional<z$1.ZodString>;
|
|
9198
9198
|
modifiedAtMs: z$1.ZodOptional<z$1.ZodNumber>;
|
|
9199
9199
|
notModified: z$1.ZodLiteral<true>;
|
|
9200
9200
|
}, z$1.core.$strip>]>, "onlineRpc", true>;
|
|
@@ -9459,10 +9459,10 @@ declare const hostDaemonCommandRegistry: {
|
|
|
9459
9459
|
health: z$1.ZodObject<{
|
|
9460
9460
|
status: z$1.ZodEnum<{
|
|
9461
9461
|
unknown: "unknown";
|
|
9462
|
+
ready: "ready";
|
|
9462
9463
|
not_installed: "not_installed";
|
|
9463
9464
|
unauthenticated: "unauthenticated";
|
|
9464
9465
|
expired: "expired";
|
|
9465
|
-
ready: "ready";
|
|
9466
9466
|
unsupported_version: "unsupported_version";
|
|
9467
9467
|
}>;
|
|
9468
9468
|
statusMessage: z$1.ZodNullable<z$1.ZodString>;
|
|
@@ -9510,9 +9510,9 @@ declare const hostDaemonCommandRegistry: {
|
|
|
9510
9510
|
executablePath: z$1.ZodNullable<z$1.ZodString>;
|
|
9511
9511
|
installed: z$1.ZodBoolean;
|
|
9512
9512
|
installSource: z$1.ZodEnum<{
|
|
9513
|
-
external: "external";
|
|
9514
9513
|
notInstalled: "notInstalled";
|
|
9515
9514
|
npmGlobal: "npmGlobal";
|
|
9515
|
+
external: "external";
|
|
9516
9516
|
}>;
|
|
9517
9517
|
currentVersion: z$1.ZodNullable<z$1.ZodString>;
|
|
9518
9518
|
latestVersion: z$1.ZodNullable<z$1.ZodString>;
|
|
@@ -9958,8 +9958,8 @@ declare const hostDaemonCommandRegistry: {
|
|
|
9958
9958
|
message: z$1.ZodNullable<z$1.ZodString>;
|
|
9959
9959
|
state: z$1.ZodNullable<z$1.ZodEnum<{
|
|
9960
9960
|
pending: "pending";
|
|
9961
|
-
ready: "ready";
|
|
9962
9961
|
activating: "activating";
|
|
9962
|
+
ready: "ready";
|
|
9963
9963
|
}>>;
|
|
9964
9964
|
}, z$1.core.$strict>, "onlineRpc", true>;
|
|
9965
9965
|
"server_move.prepare": HostDaemonCommandDescriptor<"server_move.prepare", z$1.ZodObject<{
|
|
@@ -10061,9 +10061,9 @@ declare const providerCliStatusResponseSchema: z$1.ZodRecord<z$1.ZodString, z$1.
|
|
|
10061
10061
|
executablePath: z$1.ZodNullable<z$1.ZodString>;
|
|
10062
10062
|
installed: z$1.ZodBoolean;
|
|
10063
10063
|
installSource: z$1.ZodEnum<{
|
|
10064
|
-
external: "external";
|
|
10065
10064
|
notInstalled: "notInstalled";
|
|
10066
10065
|
npmGlobal: "npmGlobal";
|
|
10066
|
+
external: "external";
|
|
10067
10067
|
}>;
|
|
10068
10068
|
currentVersion: z$1.ZodNullable<z$1.ZodString>;
|
|
10069
10069
|
latestVersion: z$1.ZodNullable<z$1.ZodString>;
|
package/package.json
CHANGED
package/server/dist/builtin-plugins/orbit/dist/assets/orbit/bundle/scripts/orbit-job-supervisor
CHANGED
|
@@ -14,8 +14,7 @@ import re
|
|
|
14
14
|
# A job has no size or file-count cap of its own. Work stops only when the disk
|
|
15
15
|
# holding job folders falls below MAIN_FREE_FLOOR, which keeps room for the OS,
|
|
16
16
|
# bb and Orbit's own state. A new job starts only with START_HEADROOM above it.
|
|
17
|
-
|
|
18
|
-
RETAINED_TOTAL_BYTES = 8_000_000_000
|
|
17
|
+
RESULT_SPACE_FLOOR = 1_000_000
|
|
19
18
|
MAIN_FREE_FLOOR = 15_000_000_000
|
|
20
19
|
START_HEADROOM = 5_000_000_000
|
|
21
20
|
# Many tiny files can exhaust inodes while bytes remain: the same rule applies
|
|
@@ -27,11 +26,11 @@ MAX_CONCURRENT_JOBS = 8
|
|
|
27
26
|
JOB_MEMORY_BYTES = 6 * 2**30
|
|
28
27
|
DESKTOP_MEMORY_BYTES = 6 * 2**30
|
|
29
28
|
CAPACITY_FULL_STATUS = 75
|
|
30
|
-
LIMITS =
|
|
29
|
+
LIMITS = {}
|
|
31
30
|
# Records written before the free-space rule carry the old fixed caps. They load
|
|
32
31
|
# as current records and are rewritten in that form at their next transition.
|
|
33
|
-
LEGACY_LIMITS = dict(bytes=6_000_000_000, files=250_000, retained_bytes=
|
|
34
|
-
|
|
32
|
+
LEGACY_LIMITS = dict(bytes=6_000_000_000, files=250_000, retained_bytes=1_000_000_000)
|
|
33
|
+
PREVIOUS_LIMITS = dict(retained_bytes=1_000_000_000)
|
|
35
34
|
STATES = ('reserved', 'running', 'parked', 'stopping', 'exporting', 'destroying', 'closed')
|
|
36
35
|
ENDS = ('finished', 'failed', 'cancelled', 'timed_out', 'crashed', 'storage_limit')
|
|
37
36
|
|
|
@@ -104,6 +103,10 @@ def new_manifest(job_id, owner, outputs, *, workspace=True):
|
|
|
104
103
|
return validate_manifest(manifest)
|
|
105
104
|
|
|
106
105
|
|
|
106
|
+
def result_metadata_space(manifest):
|
|
107
|
+
return RESULT_SPACE_FLOOR + sum(len(name.encode('utf-8')) * 6 + 512 for name in manifest['outputs'])
|
|
108
|
+
|
|
109
|
+
|
|
107
110
|
def has_workspace(manifest):
|
|
108
111
|
return manifest.get('workspace', True)
|
|
109
112
|
|
|
@@ -120,7 +123,7 @@ def validate_manifest(manifest):
|
|
|
120
123
|
or not isinstance(manifest['owner'], str)
|
|
121
124
|
or re.fullmatch('thr_[A-Za-z0-9_-]{1,128}', manifest['owner']) is None
|
|
122
125
|
or manifest['state'] not in STATES
|
|
123
|
-
or manifest['limits'] not in (LIMITS, LEGACY_LIMITS)):
|
|
126
|
+
or manifest['limits'] not in (LIMITS, LEGACY_LIMITS, PREVIOUS_LIMITS)):
|
|
124
127
|
raise ValueError('Invalid job record.')
|
|
125
128
|
ending = manifest['state'] in ('stopping', 'exporting', 'destroying', 'closed')
|
|
126
129
|
if (ending and manifest['end'] not in ENDS) or (not ending and manifest['end'] is not None):
|
|
@@ -324,7 +327,7 @@ class Workspaces:
|
|
|
324
327
|
# or thread state. A corrupt or unreadable entry blocks admission.
|
|
325
328
|
return [self.read(name[:-5]) for name in os.listdir(self.state) if name.endswith('.json')]
|
|
326
329
|
|
|
327
|
-
def create(self, owner, outputs, *, free, results_free=None, inodes=None, job_id=None, workspace=True):
|
|
330
|
+
def create(self, owner, outputs, *, free=None, results_free=None, inodes=None, job_id=None, workspace=True):
|
|
328
331
|
manifest = new_manifest(job_id if job_id is not None else secrets.token_hex(16), owner, outputs, workspace=workspace)
|
|
329
332
|
with self.locked():
|
|
330
333
|
records = self.records()
|
|
@@ -339,14 +342,17 @@ class Workspaces:
|
|
|
339
342
|
slots = [r for r in records if holds_slot(r)]
|
|
340
343
|
if len(slots) >= concurrent_jobs(self.state):
|
|
341
344
|
raise CapacityFull('Job capacity is reserved; wait for disposal.')
|
|
345
|
+
if free is None:
|
|
346
|
+
free = work_free(self)
|
|
347
|
+
if workspace:
|
|
348
|
+
results_free = available(self.results)
|
|
349
|
+
inodes = work_inodes(self)
|
|
342
350
|
if not workspace:
|
|
343
351
|
if type(free) is not int or free < MAIN_FREE_FLOOR:
|
|
344
352
|
raise CapacityFull(f'Station disk has {gigabytes(free)} GB free space; a start needs '
|
|
345
353
|
f'{gigabytes(MAIN_FREE_FLOOR)} GB. Archive handled work to free space.')
|
|
346
354
|
else:
|
|
347
|
-
retained = sum(self.result_reservation(r) for r in records)
|
|
348
|
-
if retained + RETAINED_JOB_BYTES > RETAINED_TOTAL_BYTES:
|
|
349
|
-
raise CapacityFull('Result capacity is reserved.')
|
|
355
|
+
retained = sum(self.result_reservation(r) for r in records) + result_metadata_space(manifest)
|
|
350
356
|
if type(free) is not int or free < MAIN_FREE_FLOOR + START_HEADROOM:
|
|
351
357
|
raise CapacityFull(f'Station disk has {gigabytes(free)} GB free space; a new job needs '
|
|
352
358
|
f'{gigabytes(MAIN_FREE_FLOOR + START_HEADROOM)} GB. Archive handled work to free space.')
|
|
@@ -354,9 +360,9 @@ class Workspaces:
|
|
|
354
360
|
raise CapacityFull(f'Station disk has {inodes} free file entries; a new job needs '
|
|
355
361
|
f'{INODE_FLOOR + START_INODE_HEADROOM}. Archive handled work to free space.')
|
|
356
362
|
results_free = free if results_free is None else results_free
|
|
357
|
-
if type(results_free) is not int or results_free < MAIN_FREE_FLOOR +
|
|
363
|
+
if type(results_free) is not int or results_free < MAIN_FREE_FLOOR + retained:
|
|
358
364
|
raise CapacityFull(f'The saved-results disk has {gigabytes(results_free)} GB free space; a new job needs '
|
|
359
|
-
f'{gigabytes(MAIN_FREE_FLOOR +
|
|
365
|
+
f'{gigabytes(MAIN_FREE_FLOOR + retained)} GB there. Free space on it first.')
|
|
360
366
|
# Persist ownership first. A crash before mkdir still leaves a reserved
|
|
361
367
|
# attempt which restart must explicitly finish, never silently forget.
|
|
362
368
|
write_record(self.state, manifest['id'] + '.json', manifest, exclusive=True)
|
|
@@ -519,19 +525,39 @@ class Workspaces:
|
|
|
519
525
|
low=low, bytes=size, files=files, complete=complete)
|
|
520
526
|
|
|
521
527
|
def result_reservation(self, manifest):
|
|
522
|
-
if not has_workspace(manifest):
|
|
528
|
+
if not has_workspace(manifest) or not holds_capacity(manifest):
|
|
523
529
|
return 0
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
530
|
+
manifest = self.declared_outputs(manifest, persist=False)
|
|
531
|
+
total = result_metadata_space(manifest)
|
|
532
|
+
try:
|
|
533
|
+
root = self.open_workspace(manifest['id'])
|
|
534
|
+
except FileNotFoundError:
|
|
535
|
+
return total
|
|
536
|
+
try:
|
|
537
|
+
for name in manifest['outputs']:
|
|
538
|
+
try:
|
|
539
|
+
fd = open_output(root, name)
|
|
540
|
+
except UnsafeOutput:
|
|
541
|
+
raise
|
|
542
|
+
except (FileNotFoundError, ValueError):
|
|
543
|
+
continue
|
|
544
|
+
except OSError as error:
|
|
545
|
+
if error.errno not in (errno.ELOOP, errno.ENOTDIR):
|
|
546
|
+
raise
|
|
547
|
+
continue
|
|
548
|
+
try:
|
|
549
|
+
total += os.fstat(fd).st_size
|
|
550
|
+
finally:
|
|
551
|
+
os.close(fd)
|
|
552
|
+
finally:
|
|
553
|
+
os.close(root)
|
|
554
|
+
return total
|
|
555
|
+
|
|
556
|
+
def require_result_space(self, required):
|
|
557
|
+
free = available(self.results)
|
|
558
|
+
if free < MAIN_FREE_FLOOR + required:
|
|
559
|
+
raise CapacityFull(f'The saved-results disk has {gigabytes(free)} GB free space; saving results needs '
|
|
560
|
+
f'{gigabytes(MAIN_FREE_FLOOR + required)} GB there. Free space on it first.')
|
|
535
561
|
|
|
536
562
|
def read_output_declaration(self, root):
|
|
537
563
|
fd = open_output(root, '.orbit-outputs.json')
|
|
@@ -546,7 +572,7 @@ class Workspaces:
|
|
|
546
572
|
finally:
|
|
547
573
|
os.close(fd)
|
|
548
574
|
|
|
549
|
-
def declared_outputs(self, manifest):
|
|
575
|
+
def declared_outputs(self, manifest, *, persist=True):
|
|
550
576
|
try:
|
|
551
577
|
root = self.open_workspace(manifest['id'])
|
|
552
578
|
except FileNotFoundError:
|
|
@@ -565,7 +591,8 @@ class Workspaces:
|
|
|
565
591
|
if warned:
|
|
566
592
|
updated['outputWarning'] = OUTPUT_WARNING
|
|
567
593
|
updated = validate_manifest(updated)
|
|
568
|
-
|
|
594
|
+
if persist:
|
|
595
|
+
write_record(self.state, manifest['id'] + '.json', updated)
|
|
569
596
|
return updated
|
|
570
597
|
|
|
571
598
|
def park(self, job_id):
|
|
@@ -677,8 +704,8 @@ class Workspaces:
|
|
|
677
704
|
if 'outputs.json' in os.listdir(destination):
|
|
678
705
|
self.verify_outputs(manifest)
|
|
679
706
|
return
|
|
707
|
+
self.require_result_space(self.result_reservation(manifest))
|
|
680
708
|
receipts = []
|
|
681
|
-
total = 0
|
|
682
709
|
for index, name in enumerate(manifest['outputs']):
|
|
683
710
|
receipt = dict(name=name, status='missing')
|
|
684
711
|
if source is None:
|
|
@@ -703,8 +730,7 @@ class Workspaces:
|
|
|
703
730
|
continue
|
|
704
731
|
try:
|
|
705
732
|
metadata = os.fstat(fd)
|
|
706
|
-
|
|
707
|
-
raise ValueError('Declared results exceed their allowance; preserve and increase the allowance.')
|
|
733
|
+
self.require_result_space(metadata.st_size + RESULT_SPACE_FLOOR)
|
|
708
734
|
number = format(index, '04d')
|
|
709
735
|
temporary = '.' + number
|
|
710
736
|
# Private partial exports can be replaced on restart. The
|
|
@@ -721,8 +747,7 @@ class Workspaces:
|
|
|
721
747
|
if not block:
|
|
722
748
|
break
|
|
723
749
|
count += len(block)
|
|
724
|
-
|
|
725
|
-
raise ValueError('Declared results exceed their allowance.')
|
|
750
|
+
self.require_result_space(len(block) + RESULT_SPACE_FLOOR)
|
|
726
751
|
digest.update(block)
|
|
727
752
|
stream.write(block)
|
|
728
753
|
if file_identity(os.fstat(fd)) != file_identity(metadata):
|
|
@@ -731,7 +756,6 @@ class Workspaces:
|
|
|
731
756
|
os.fchmod(stream.fileno(), 0o444)
|
|
732
757
|
os.fsync(stream.fileno())
|
|
733
758
|
os.rename(temporary, number, src_dir_fd=destination, dst_dir_fd=destination)
|
|
734
|
-
total += count
|
|
735
759
|
receipt.update(status='saved', file=number, bytes=count, sha256=digest.hexdigest())
|
|
736
760
|
receipts.append(receipt)
|
|
737
761
|
finally:
|
|
@@ -778,12 +802,8 @@ class Workspaces:
|
|
|
778
802
|
if not block:
|
|
779
803
|
break
|
|
780
804
|
count += len(block)
|
|
781
|
-
if count > RETAINED_JOB_BYTES:
|
|
782
|
-
raise ValueError('Retained result exceeds its allowance.')
|
|
783
805
|
digest.update(block)
|
|
784
806
|
total += count
|
|
785
|
-
if total > RETAINED_JOB_BYTES:
|
|
786
|
-
raise ValueError('Retained results exceed their allowance.')
|
|
787
807
|
if count != item['bytes'] or digest.hexdigest() != item['sha256']:
|
|
788
808
|
raise ValueError('Retained result bytes changed; workspace preserved.')
|
|
789
809
|
finally:
|
|
@@ -1304,12 +1324,10 @@ def managed_request(request):
|
|
|
1304
1324
|
|
|
1305
1325
|
def managed_operation(store, account, request):
|
|
1306
1326
|
if request['operation'] == 'reserve':
|
|
1307
|
-
store.create(request['owner'], [],
|
|
1327
|
+
store.create(request['owner'], [], job_id=request['id'], workspace=False)
|
|
1308
1328
|
elif request['operation'] == 'create':
|
|
1309
1329
|
declared_output_names(request['outputs'], strict=True)
|
|
1310
|
-
job = store.create(request['owner'], request['outputs'],
|
|
1311
|
-
results_free=available(store.results), inodes=work_inodes(store),
|
|
1312
|
-
job_id=request['id'])
|
|
1330
|
+
job = store.create(request['owner'], request['outputs'], job_id=request['id'])
|
|
1313
1331
|
root = store.open_workspace(job['id'])
|
|
1314
1332
|
try:
|
|
1315
1333
|
prepare_hooks(root)
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
#!/bin/bash
|
|
2
|
+
set -euo pipefail
|
|
3
|
+
umask 077
|
|
4
|
+
dir=/var/lib/orbitswap
|
|
5
|
+
file="$dir/swapfile"
|
|
6
|
+
size=4294967296
|
|
7
|
+
minimum_free=12884901888
|
|
8
|
+
if [ -L "$dir" ] || { [ -e "$dir" ] && { [ ! -d "$dir" ] || [ "$(stat -c %u "$dir")" != 0 ]; }; }; then
|
|
9
|
+
echo 'Swap directory must be a root-owned directory, not a link.' >&2
|
|
10
|
+
exit 1
|
|
11
|
+
fi
|
|
12
|
+
mkdir -p "$dir"
|
|
13
|
+
chown root:root "$dir"
|
|
14
|
+
chmod 700 "$dir"
|
|
15
|
+
if [ -L "$file" ] || { [ -e "$file" ] && { [ ! -f "$file" ] || [ "$(stat -c %h "$file")" != 1 ]; }; }; then
|
|
16
|
+
echo 'Swap file must be a regular file with one name, not a link.' >&2
|
|
17
|
+
exit 1
|
|
18
|
+
fi
|
|
19
|
+
active=$(swapon --show=NAME --noheadings --raw)
|
|
20
|
+
is_active=no
|
|
21
|
+
if grep -Fxq "$file" <<< "$active"; then is_active=yes; fi
|
|
22
|
+
current_size=0
|
|
23
|
+
if [ -f "$file" ]; then
|
|
24
|
+
current_size=$(stat -c %s "$file")
|
|
25
|
+
chown root:root "$file"
|
|
26
|
+
chmod 600 "$file"
|
|
27
|
+
fi
|
|
28
|
+
if [ "$is_active" = yes ] && [ "$current_size" != "$size" ]; then
|
|
29
|
+
echo 'Station swap is active with the wrong size. Stop builds, disable this swap when memory allows, then reapply the base recipe.' >&2
|
|
30
|
+
exit 1
|
|
31
|
+
fi
|
|
32
|
+
signature=
|
|
33
|
+
if [ "$is_active" = yes ]; then
|
|
34
|
+
signature=swap
|
|
35
|
+
elif [ -f "$file" ]; then
|
|
36
|
+
signature=$(blkid -p -s TYPE -o value "$file" || true)
|
|
37
|
+
fi
|
|
38
|
+
if [ "$current_size" != "$size" ] || [ "$signature" != swap ]; then
|
|
39
|
+
free=$(df -B1 --output=avail "$dir" | tail -n 1 | tr -d '[:space:]')
|
|
40
|
+
if ! [[ "$free" =~ ^[0-9]{1,18}$ ]] || (( free < minimum_free )); then
|
|
41
|
+
echo 'Swap needs at least 12 GiB free to leave 8 GiB after allocation. Free disk space, then reapply the base recipe.' >&2
|
|
42
|
+
exit 1
|
|
43
|
+
fi
|
|
44
|
+
if [ "$(stat -f -c %T "$dir")" = btrfs ]; then
|
|
45
|
+
echo 'Station swap needs a filesystem that supports ordinary swap files; this recipe does not configure Btrfs swap.' >&2
|
|
46
|
+
exit 1
|
|
47
|
+
fi
|
|
48
|
+
candidate=$(mktemp "$dir/.swapfile.XXXXXX")
|
|
49
|
+
trap 'rm -f "$candidate"' EXIT
|
|
50
|
+
chown root:root "$candidate"
|
|
51
|
+
chmod 600 "$candidate"
|
|
52
|
+
dd if=/dev/zero of="$candidate" bs=1M count=4096 conv=fsync status=none
|
|
53
|
+
mkswap "$candidate" >/dev/null
|
|
54
|
+
mv -f "$candidate" "$file"
|
|
55
|
+
trap - EXIT
|
|
56
|
+
fi
|
|
57
|
+
cat > /etc/systemd/system/var-lib-orbitswap-swapfile.swap <<'UNIT'
|
|
58
|
+
[Unit]
|
|
59
|
+
Description=Station swap for build memory spikes
|
|
60
|
+
|
|
61
|
+
[Swap]
|
|
62
|
+
What=/var/lib/orbitswap/swapfile
|
|
63
|
+
|
|
64
|
+
[Install]
|
|
65
|
+
WantedBy=swap.target
|
|
66
|
+
UNIT
|
|
67
|
+
chown root:root /etc/systemd/system/var-lib-orbitswap-swapfile.swap
|
|
68
|
+
chmod 644 /etc/systemd/system/var-lib-orbitswap-swapfile.swap
|
|
69
|
+
systemctl daemon-reload
|
|
70
|
+
systemctl enable --now var-lib-orbitswap-swapfile.swap
|
|
71
|
+
printf 'vm.swappiness=10\n' > /etc/sysctl.d/90-orbit-swap.conf
|
|
72
|
+
chown root:root /etc/sysctl.d/90-orbit-swap.conf
|
|
73
|
+
chmod 644 /etc/sysctl.d/90-orbit-swap.conf
|
|
74
|
+
sysctl -q -p /etc/sysctl.d/90-orbit-swap.conf
|
|
@@ -1 +1 @@
|
|
|
1
|
-
|
|
1
|
+
b123f3719d730c4807326038d8516c21c33e7969
|