@halcyonlabs/app 0.47.0-halcyon.106 → 0.47.0-halcyon.110

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.d.ts CHANGED
@@ -155,10 +155,10 @@ declare const uiPreferenceDefinitions: {
155
155
  }>>;
156
156
  readonly "sidebar.threadGrouping.environment": UiPreferenceDefinition<z$1.ZodUnion<readonly [z$1.ZodLiteral<"auto">, z$1.ZodBoolean]>>;
157
157
  readonly "sidebar.chronologicalSort": UiPreferenceDefinition<z$1.ZodEnum<{
158
- none: "none";
159
158
  updated: "updated";
160
159
  created: "created";
161
160
  alpha: "alpha";
161
+ none: "none";
162
162
  }>>;
163
163
  readonly "sidebar.sortDirection": UiPreferenceDefinition<z$1.ZodEnum<{
164
164
  default: "default";
@@ -280,8 +280,8 @@ declare const appKeybindingOverridesSchema: z$1.ZodArray<z$1.ZodObject<{
280
280
  shift: z$1.ZodBoolean;
281
281
  }, z$1.core.$strict>>;
282
282
  platform: z$1.ZodOptional<z$1.ZodEnum<{
283
- mac: "mac";
284
283
  windows: "windows";
284
+ mac: "mac";
285
285
  linux: "linux";
286
286
  }>>;
287
287
  }, z$1.core.$strict>>;
@@ -487,8 +487,8 @@ declare const environmentSchema: z$1.ZodObject<{
487
487
  retireAt: z$1.ZodNullable<z$1.ZodNumber>;
488
488
  teardown: z$1.ZodNullable<z$1.ZodObject<{
489
489
  status: z$1.ZodEnum<{
490
- running: "running";
491
490
  failed: "failed";
491
+ running: "running";
492
492
  removed: "removed";
493
493
  }>;
494
494
  attempt: z$1.ZodNumber;
@@ -583,8 +583,8 @@ declare const hostSchema: z$1.ZodObject<{
583
583
  pendingLog: z$1.ZodString;
584
584
  teardown: z$1.ZodNullable<z$1.ZodObject<{
585
585
  status: z$1.ZodEnum<{
586
- running: "running";
587
586
  failed: "failed";
587
+ running: "running";
588
588
  removed: "removed";
589
589
  }>;
590
590
  attempt: z$1.ZodNumber;
@@ -685,8 +685,8 @@ declare const threadEventSchema: z$1.ZodUnion<readonly [z$1.ZodIntersection<z$1.
685
685
  providerThreadId: z$1.ZodString;
686
686
  objective: z$1.ZodString;
687
687
  status: z$1.ZodEnum<{
688
- paused: "paused";
689
688
  active: "active";
689
+ paused: "paused";
690
690
  budgetLimited: "budgetLimited";
691
691
  complete: "complete";
692
692
  }>;
@@ -1257,9 +1257,9 @@ declare const threadEventSchema: z$1.ZodUnion<readonly [z$1.ZodIntersection<z$1.
1257
1257
  step: z$1.ZodString;
1258
1258
  status: z$1.ZodOptional<z$1.ZodEnum<{
1259
1259
  pending: "pending";
1260
+ active: "active";
1260
1261
  completed: "completed";
1261
1262
  failed: "failed";
1262
- active: "active";
1263
1263
  }>>;
1264
1264
  }, z$1.core.$strip>>;
1265
1265
  explanation: z$1.ZodOptional<z$1.ZodString>;
@@ -1338,10 +1338,10 @@ declare const threadEventSchema: z$1.ZodUnion<readonly [z$1.ZodIntersection<z$1.
1338
1338
  }>;
1339
1339
  taskStatus: z$1.ZodEnum<{
1340
1340
  pending: "pending";
1341
- running: "running";
1342
- paused: "paused";
1343
1341
  completed: "completed";
1344
1342
  failed: "failed";
1343
+ paused: "paused";
1344
+ running: "running";
1345
1345
  killed: "killed";
1346
1346
  stopped: "stopped";
1347
1347
  }>;
@@ -1358,8 +1358,8 @@ declare const threadEventSchema: z$1.ZodUnion<readonly [z$1.ZodIntersection<z$1.
1358
1358
  index: z$1.ZodNumber;
1359
1359
  label: z$1.ZodString;
1360
1360
  state: z$1.ZodEnum<{
1361
- running: "running";
1362
1361
  failed: "failed";
1362
+ running: "running";
1363
1363
  queued: "queued";
1364
1364
  done: "done";
1365
1365
  skipped: "skipped";
@@ -2029,9 +2029,9 @@ declare const threadEventSchema: z$1.ZodUnion<readonly [z$1.ZodIntersection<z$1.
2029
2029
  step: z$1.ZodString;
2030
2030
  status: z$1.ZodOptional<z$1.ZodEnum<{
2031
2031
  pending: "pending";
2032
+ active: "active";
2032
2033
  completed: "completed";
2033
2034
  failed: "failed";
2034
- active: "active";
2035
2035
  }>>;
2036
2036
  }, z$1.core.$strip>>;
2037
2037
  explanation: z$1.ZodOptional<z$1.ZodString>;
@@ -2110,10 +2110,10 @@ declare const threadEventSchema: z$1.ZodUnion<readonly [z$1.ZodIntersection<z$1.
2110
2110
  }>;
2111
2111
  taskStatus: z$1.ZodEnum<{
2112
2112
  pending: "pending";
2113
- running: "running";
2114
- paused: "paused";
2115
2113
  completed: "completed";
2116
2114
  failed: "failed";
2115
+ paused: "paused";
2116
+ running: "running";
2117
2117
  killed: "killed";
2118
2118
  stopped: "stopped";
2119
2119
  }>;
@@ -2130,8 +2130,8 @@ declare const threadEventSchema: z$1.ZodUnion<readonly [z$1.ZodIntersection<z$1.
2130
2130
  index: z$1.ZodNumber;
2131
2131
  label: z$1.ZodString;
2132
2132
  state: z$1.ZodEnum<{
2133
- running: "running";
2134
2133
  failed: "failed";
2134
+ running: "running";
2135
2135
  queued: "queued";
2136
2136
  done: "done";
2137
2137
  skipped: "skipped";
@@ -2342,10 +2342,10 @@ declare const threadEventSchema: z$1.ZodUnion<readonly [z$1.ZodIntersection<z$1.
2342
2342
  }>;
2343
2343
  taskStatus: z$1.ZodEnum<{
2344
2344
  pending: "pending";
2345
- running: "running";
2346
- paused: "paused";
2347
2345
  completed: "completed";
2348
2346
  failed: "failed";
2347
+ paused: "paused";
2348
+ running: "running";
2349
2349
  killed: "killed";
2350
2350
  stopped: "stopped";
2351
2351
  }>;
@@ -2362,8 +2362,8 @@ declare const threadEventSchema: z$1.ZodUnion<readonly [z$1.ZodIntersection<z$1.
2362
2362
  index: z$1.ZodNumber;
2363
2363
  label: z$1.ZodString;
2364
2364
  state: z$1.ZodEnum<{
2365
- running: "running";
2366
2365
  failed: "failed";
2366
+ running: "running";
2367
2367
  queued: "queued";
2368
2368
  done: "done";
2369
2369
  skipped: "skipped";
@@ -2441,10 +2441,10 @@ declare const threadEventSchema: z$1.ZodUnion<readonly [z$1.ZodIntersection<z$1.
2441
2441
  }>;
2442
2442
  taskStatus: z$1.ZodEnum<{
2443
2443
  pending: "pending";
2444
- running: "running";
2445
- paused: "paused";
2446
2444
  completed: "completed";
2447
2445
  failed: "failed";
2446
+ paused: "paused";
2447
+ running: "running";
2448
2448
  killed: "killed";
2449
2449
  stopped: "stopped";
2450
2450
  }>;
@@ -2461,8 +2461,8 @@ declare const threadEventSchema: z$1.ZodUnion<readonly [z$1.ZodIntersection<z$1.
2461
2461
  index: z$1.ZodNumber;
2462
2462
  label: z$1.ZodString;
2463
2463
  state: z$1.ZodEnum<{
2464
- running: "running";
2465
2464
  failed: "failed";
2465
+ running: "running";
2466
2466
  queued: "queued";
2467
2467
  done: "done";
2468
2468
  skipped: "skipped";
@@ -2657,9 +2657,9 @@ declare const threadEventSchema: z$1.ZodUnion<readonly [z$1.ZodIntersection<z$1.
2657
2657
  step: z$1.ZodString;
2658
2658
  status: z$1.ZodOptional<z$1.ZodEnum<{
2659
2659
  pending: "pending";
2660
+ active: "active";
2660
2661
  completed: "completed";
2661
2662
  failed: "failed";
2662
- active: "active";
2663
2663
  }>>;
2664
2664
  }, z$1.core.$strip>>;
2665
2665
  explanation: z$1.ZodOptional<z$1.ZodString>;
@@ -2668,8 +2668,8 @@ declare const threadEventSchema: z$1.ZodUnion<readonly [z$1.ZodIntersection<z$1.
2668
2668
  threadId: z$1.ZodString;
2669
2669
  providerThreadId: z$1.ZodString;
2670
2670
  state: z$1.ZodEnum<{
2671
- running: "running";
2672
2671
  idle: "idle";
2672
+ running: "running";
2673
2673
  }>;
2674
2674
  }, z$1.core.$strip>, z$1.ZodObject<{
2675
2675
  type: z$1.ZodLiteral<"turn/diff/updated">;
@@ -3531,9 +3531,9 @@ declare const threadEventSchema: z$1.ZodUnion<readonly [z$1.ZodIntersection<z$1.
3531
3531
  threadId: z$1.ZodString;
3532
3532
  provisioningId: z$1.ZodString;
3533
3533
  status: z$1.ZodEnum<{
3534
+ active: "active";
3534
3535
  completed: "completed";
3535
3536
  failed: "failed";
3536
- active: "active";
3537
3537
  cancelled: "cancelled";
3538
3538
  }>;
3539
3539
  environmentId: z$1.ZodNullable<z$1.ZodString>;
@@ -7456,8 +7456,8 @@ declare const hostDaemonCommandRegistry: {
7456
7456
  skillFilePath: z$1.ZodString;
7457
7457
  }, z$1.core.$strict>], "kind">>;
7458
7458
  instructionMode: z$1.ZodEnum<{
7459
- append: "append";
7460
7459
  replace: "replace";
7460
+ append: "append";
7461
7461
  }>;
7462
7462
  type: z$1.ZodLiteral<"thread.rewind.prepare">;
7463
7463
  leaseId: z$1.ZodString;
@@ -7605,8 +7605,8 @@ declare const hostDaemonCommandRegistry: {
7605
7605
  skillFilePath: z$1.ZodString;
7606
7606
  }, z$1.core.$strict>], "kind">>;
7607
7607
  instructionMode: z$1.ZodEnum<{
7608
- append: "append";
7609
7608
  replace: "replace";
7609
+ append: "append";
7610
7610
  }>;
7611
7611
  type: z$1.ZodLiteral<"thread.start">;
7612
7612
  requestId: z$1.ZodString;
@@ -8069,8 +8069,8 @@ declare const hostDaemonCommandRegistry: {
8069
8069
  envPassthrough: z$1.ZodArray<z$1.ZodString>;
8070
8070
  }, z$1.core.$strict>;
8071
8071
  instructionMode: z$1.ZodEnum<{
8072
- append: "append";
8073
8072
  replace: "replace";
8073
+ append: "append";
8074
8074
  }>;
8075
8075
  projectId: z$1.ZodString;
8076
8076
  providerId: z$1.ZodString;
@@ -8285,8 +8285,8 @@ declare const hostDaemonCommandRegistry: {
8285
8285
  envPassthrough: z$1.ZodArray<z$1.ZodString>;
8286
8286
  }, z$1.core.$strict>;
8287
8287
  instructionMode: z$1.ZodEnum<{
8288
- append: "append";
8289
8288
  replace: "replace";
8289
+ append: "append";
8290
8290
  }>;
8291
8291
  projectId: z$1.ZodString;
8292
8292
  providerId: z$1.ZodString;
@@ -9192,9 +9192,9 @@ declare const hostDaemonCommandRegistry: {
9192
9192
  base64: "base64";
9193
9193
  utf8: "utf8";
9194
9194
  }>;
9195
- mimeType: z$1.ZodOptional<z$1.ZodString>;
9196
- sizeBytes: z$1.ZodNumber;
9197
9195
  sha256: z$1.ZodString;
9196
+ sizeBytes: z$1.ZodNumber;
9197
+ mimeType: z$1.ZodOptional<z$1.ZodString>;
9198
9198
  modifiedAtMs: z$1.ZodOptional<z$1.ZodNumber>;
9199
9199
  notModified: z$1.ZodLiteral<true>;
9200
9200
  }, z$1.core.$strip>]>, "onlineRpc", true>;
@@ -9459,10 +9459,10 @@ declare const hostDaemonCommandRegistry: {
9459
9459
  health: z$1.ZodObject<{
9460
9460
  status: z$1.ZodEnum<{
9461
9461
  unknown: "unknown";
9462
+ ready: "ready";
9462
9463
  not_installed: "not_installed";
9463
9464
  unauthenticated: "unauthenticated";
9464
9465
  expired: "expired";
9465
- ready: "ready";
9466
9466
  unsupported_version: "unsupported_version";
9467
9467
  }>;
9468
9468
  statusMessage: z$1.ZodNullable<z$1.ZodString>;
@@ -9510,9 +9510,9 @@ declare const hostDaemonCommandRegistry: {
9510
9510
  executablePath: z$1.ZodNullable<z$1.ZodString>;
9511
9511
  installed: z$1.ZodBoolean;
9512
9512
  installSource: z$1.ZodEnum<{
9513
- external: "external";
9514
9513
  notInstalled: "notInstalled";
9515
9514
  npmGlobal: "npmGlobal";
9515
+ external: "external";
9516
9516
  }>;
9517
9517
  currentVersion: z$1.ZodNullable<z$1.ZodString>;
9518
9518
  latestVersion: z$1.ZodNullable<z$1.ZodString>;
@@ -9958,8 +9958,8 @@ declare const hostDaemonCommandRegistry: {
9958
9958
  message: z$1.ZodNullable<z$1.ZodString>;
9959
9959
  state: z$1.ZodNullable<z$1.ZodEnum<{
9960
9960
  pending: "pending";
9961
- ready: "ready";
9962
9961
  activating: "activating";
9962
+ ready: "ready";
9963
9963
  }>>;
9964
9964
  }, z$1.core.$strict>, "onlineRpc", true>;
9965
9965
  "server_move.prepare": HostDaemonCommandDescriptor<"server_move.prepare", z$1.ZodObject<{
@@ -10061,9 +10061,9 @@ declare const providerCliStatusResponseSchema: z$1.ZodRecord<z$1.ZodString, z$1.
10061
10061
  executablePath: z$1.ZodNullable<z$1.ZodString>;
10062
10062
  installed: z$1.ZodBoolean;
10063
10063
  installSource: z$1.ZodEnum<{
10064
- external: "external";
10065
10064
  notInstalled: "notInstalled";
10066
10065
  npmGlobal: "npmGlobal";
10066
+ external: "external";
10067
10067
  }>;
10068
10068
  currentVersion: z$1.ZodNullable<z$1.ZodString>;
10069
10069
  latestVersion: z$1.ZodNullable<z$1.ZodString>;
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@halcyonlabs/app",
3
- "version": "0.47.0-halcyon.106",
3
+ "version": "0.47.0-halcyon.110",
4
4
  "description": "bb app launcher for npx",
5
5
  "keywords": [
6
6
  "bb",
@@ -14,8 +14,7 @@ import re
14
14
  # A job has no size or file-count cap of its own. Work stops only when the disk
15
15
  # holding job folders falls below MAIN_FREE_FLOOR, which keeps room for the OS,
16
16
  # bb and Orbit's own state. A new job starts only with START_HEADROOM above it.
17
- RETAINED_JOB_BYTES = 1_000_000_000
18
- RETAINED_TOTAL_BYTES = 8_000_000_000
17
+ RESULT_SPACE_FLOOR = 1_000_000
19
18
  MAIN_FREE_FLOOR = 15_000_000_000
20
19
  START_HEADROOM = 5_000_000_000
21
20
  # Many tiny files can exhaust inodes while bytes remain: the same rule applies
@@ -27,11 +26,11 @@ MAX_CONCURRENT_JOBS = 8
27
26
  JOB_MEMORY_BYTES = 6 * 2**30
28
27
  DESKTOP_MEMORY_BYTES = 6 * 2**30
29
28
  CAPACITY_FULL_STATUS = 75
30
- LIMITS = dict(retained_bytes=RETAINED_JOB_BYTES)
29
+ LIMITS = {}
31
30
  # Records written before the free-space rule carry the old fixed caps. They load
32
31
  # as current records and are rewritten in that form at their next transition.
33
- LEGACY_LIMITS = dict(bytes=6_000_000_000, files=250_000, retained_bytes=RETAINED_JOB_BYTES)
34
- assert START_HEADROOM >= RETAINED_JOB_BYTES
32
+ LEGACY_LIMITS = dict(bytes=6_000_000_000, files=250_000, retained_bytes=1_000_000_000)
33
+ PREVIOUS_LIMITS = dict(retained_bytes=1_000_000_000)
35
34
  STATES = ('reserved', 'running', 'parked', 'stopping', 'exporting', 'destroying', 'closed')
36
35
  ENDS = ('finished', 'failed', 'cancelled', 'timed_out', 'crashed', 'storage_limit')
37
36
 
@@ -104,6 +103,10 @@ def new_manifest(job_id, owner, outputs, *, workspace=True):
104
103
  return validate_manifest(manifest)
105
104
 
106
105
 
106
+ def result_metadata_space(manifest):
107
+ return RESULT_SPACE_FLOOR + sum(len(name.encode('utf-8')) * 6 + 512 for name in manifest['outputs'])
108
+
109
+
107
110
  def has_workspace(manifest):
108
111
  return manifest.get('workspace', True)
109
112
 
@@ -120,7 +123,7 @@ def validate_manifest(manifest):
120
123
  or not isinstance(manifest['owner'], str)
121
124
  or re.fullmatch('thr_[A-Za-z0-9_-]{1,128}', manifest['owner']) is None
122
125
  or manifest['state'] not in STATES
123
- or manifest['limits'] not in (LIMITS, LEGACY_LIMITS)):
126
+ or manifest['limits'] not in (LIMITS, LEGACY_LIMITS, PREVIOUS_LIMITS)):
124
127
  raise ValueError('Invalid job record.')
125
128
  ending = manifest['state'] in ('stopping', 'exporting', 'destroying', 'closed')
126
129
  if (ending and manifest['end'] not in ENDS) or (not ending and manifest['end'] is not None):
@@ -324,7 +327,7 @@ class Workspaces:
324
327
  # or thread state. A corrupt or unreadable entry blocks admission.
325
328
  return [self.read(name[:-5]) for name in os.listdir(self.state) if name.endswith('.json')]
326
329
 
327
- def create(self, owner, outputs, *, free, results_free=None, inodes=None, job_id=None, workspace=True):
330
+ def create(self, owner, outputs, *, free=None, results_free=None, inodes=None, job_id=None, workspace=True):
328
331
  manifest = new_manifest(job_id if job_id is not None else secrets.token_hex(16), owner, outputs, workspace=workspace)
329
332
  with self.locked():
330
333
  records = self.records()
@@ -339,14 +342,17 @@ class Workspaces:
339
342
  slots = [r for r in records if holds_slot(r)]
340
343
  if len(slots) >= concurrent_jobs(self.state):
341
344
  raise CapacityFull('Job capacity is reserved; wait for disposal.')
345
+ if free is None:
346
+ free = work_free(self)
347
+ if workspace:
348
+ results_free = available(self.results)
349
+ inodes = work_inodes(self)
342
350
  if not workspace:
343
351
  if type(free) is not int or free < MAIN_FREE_FLOOR:
344
352
  raise CapacityFull(f'Station disk has {gigabytes(free)} GB free space; a start needs '
345
353
  f'{gigabytes(MAIN_FREE_FLOOR)} GB. Archive handled work to free space.')
346
354
  else:
347
- retained = sum(self.result_reservation(r) for r in records)
348
- if retained + RETAINED_JOB_BYTES > RETAINED_TOTAL_BYTES:
349
- raise CapacityFull('Result capacity is reserved.')
355
+ retained = sum(self.result_reservation(r) for r in records) + result_metadata_space(manifest)
350
356
  if type(free) is not int or free < MAIN_FREE_FLOOR + START_HEADROOM:
351
357
  raise CapacityFull(f'Station disk has {gigabytes(free)} GB free space; a new job needs '
352
358
  f'{gigabytes(MAIN_FREE_FLOOR + START_HEADROOM)} GB. Archive handled work to free space.')
@@ -354,9 +360,9 @@ class Workspaces:
354
360
  raise CapacityFull(f'Station disk has {inodes} free file entries; a new job needs '
355
361
  f'{INODE_FLOOR + START_INODE_HEADROOM}. Archive handled work to free space.')
356
362
  results_free = free if results_free is None else results_free
357
- if type(results_free) is not int or results_free < MAIN_FREE_FLOOR + RETAINED_JOB_BYTES:
363
+ if type(results_free) is not int or results_free < MAIN_FREE_FLOOR + retained:
358
364
  raise CapacityFull(f'The saved-results disk has {gigabytes(results_free)} GB free space; a new job needs '
359
- f'{gigabytes(MAIN_FREE_FLOOR + RETAINED_JOB_BYTES)} GB there. Free space on it first.')
365
+ f'{gigabytes(MAIN_FREE_FLOOR + retained)} GB there. Free space on it first.')
360
366
  # Persist ownership first. A crash before mkdir still leaves a reserved
361
367
  # attempt which restart must explicitly finish, never silently forget.
362
368
  write_record(self.state, manifest['id'] + '.json', manifest, exclusive=True)
@@ -519,19 +525,39 @@ class Workspaces:
519
525
  low=low, bytes=size, files=files, complete=complete)
520
526
 
521
527
  def result_reservation(self, manifest):
522
- if not has_workspace(manifest):
528
+ if not has_workspace(manifest) or not holds_capacity(manifest):
523
529
  return 0
524
- if holds_capacity(manifest):
525
- return RETAINED_JOB_BYTES
526
- fd = os.open(manifest['id'] + '.retained', os.O_RDONLY | os.O_NOFOLLOW | os.O_CLOEXEC, dir_fd=self.state)
527
- with os.fdopen(fd, 'rb') as stream:
528
- metadata = os.fstat(stream.fileno())
529
- if not stat.S_ISREG(metadata.st_mode) or metadata.st_size > 1024 or metadata.st_nlink != 1:
530
- raise ValueError('Invalid retained-size record.')
531
- saved = json.load(stream)
532
- if set(saved) != {'bytes'} or type(saved['bytes']) is not int or not 0 <= saved['bytes'] <= RETAINED_JOB_BYTES:
533
- raise ValueError('Invalid retained-size record.')
534
- return saved['bytes']
530
+ manifest = self.declared_outputs(manifest, persist=False)
531
+ total = result_metadata_space(manifest)
532
+ try:
533
+ root = self.open_workspace(manifest['id'])
534
+ except FileNotFoundError:
535
+ return total
536
+ try:
537
+ for name in manifest['outputs']:
538
+ try:
539
+ fd = open_output(root, name)
540
+ except UnsafeOutput:
541
+ raise
542
+ except (FileNotFoundError, ValueError):
543
+ continue
544
+ except OSError as error:
545
+ if error.errno not in (errno.ELOOP, errno.ENOTDIR):
546
+ raise
547
+ continue
548
+ try:
549
+ total += os.fstat(fd).st_size
550
+ finally:
551
+ os.close(fd)
552
+ finally:
553
+ os.close(root)
554
+ return total
555
+
556
+ def require_result_space(self, required):
557
+ free = available(self.results)
558
+ if free < MAIN_FREE_FLOOR + required:
559
+ raise CapacityFull(f'The saved-results disk has {gigabytes(free)} GB free space; saving results needs '
560
+ f'{gigabytes(MAIN_FREE_FLOOR + required)} GB there. Free space on it first.')
535
561
 
536
562
  def read_output_declaration(self, root):
537
563
  fd = open_output(root, '.orbit-outputs.json')
@@ -546,7 +572,7 @@ class Workspaces:
546
572
  finally:
547
573
  os.close(fd)
548
574
 
549
- def declared_outputs(self, manifest):
575
+ def declared_outputs(self, manifest, *, persist=True):
550
576
  try:
551
577
  root = self.open_workspace(manifest['id'])
552
578
  except FileNotFoundError:
@@ -565,7 +591,8 @@ class Workspaces:
565
591
  if warned:
566
592
  updated['outputWarning'] = OUTPUT_WARNING
567
593
  updated = validate_manifest(updated)
568
- write_record(self.state, manifest['id'] + '.json', updated)
594
+ if persist:
595
+ write_record(self.state, manifest['id'] + '.json', updated)
569
596
  return updated
570
597
 
571
598
  def park(self, job_id):
@@ -677,8 +704,8 @@ class Workspaces:
677
704
  if 'outputs.json' in os.listdir(destination):
678
705
  self.verify_outputs(manifest)
679
706
  return
707
+ self.require_result_space(self.result_reservation(manifest))
680
708
  receipts = []
681
- total = 0
682
709
  for index, name in enumerate(manifest['outputs']):
683
710
  receipt = dict(name=name, status='missing')
684
711
  if source is None:
@@ -703,8 +730,7 @@ class Workspaces:
703
730
  continue
704
731
  try:
705
732
  metadata = os.fstat(fd)
706
- if total + metadata.st_size > RETAINED_JOB_BYTES:
707
- raise ValueError('Declared results exceed their allowance; preserve and increase the allowance.')
733
+ self.require_result_space(metadata.st_size + RESULT_SPACE_FLOOR)
708
734
  number = format(index, '04d')
709
735
  temporary = '.' + number
710
736
  # Private partial exports can be replaced on restart. The
@@ -721,8 +747,7 @@ class Workspaces:
721
747
  if not block:
722
748
  break
723
749
  count += len(block)
724
- if total + count > RETAINED_JOB_BYTES:
725
- raise ValueError('Declared results exceed their allowance.')
750
+ self.require_result_space(len(block) + RESULT_SPACE_FLOOR)
726
751
  digest.update(block)
727
752
  stream.write(block)
728
753
  if file_identity(os.fstat(fd)) != file_identity(metadata):
@@ -731,7 +756,6 @@ class Workspaces:
731
756
  os.fchmod(stream.fileno(), 0o444)
732
757
  os.fsync(stream.fileno())
733
758
  os.rename(temporary, number, src_dir_fd=destination, dst_dir_fd=destination)
734
- total += count
735
759
  receipt.update(status='saved', file=number, bytes=count, sha256=digest.hexdigest())
736
760
  receipts.append(receipt)
737
761
  finally:
@@ -778,12 +802,8 @@ class Workspaces:
778
802
  if not block:
779
803
  break
780
804
  count += len(block)
781
- if count > RETAINED_JOB_BYTES:
782
- raise ValueError('Retained result exceeds its allowance.')
783
805
  digest.update(block)
784
806
  total += count
785
- if total > RETAINED_JOB_BYTES:
786
- raise ValueError('Retained results exceed their allowance.')
787
807
  if count != item['bytes'] or digest.hexdigest() != item['sha256']:
788
808
  raise ValueError('Retained result bytes changed; workspace preserved.')
789
809
  finally:
@@ -1304,12 +1324,10 @@ def managed_request(request):
1304
1324
 
1305
1325
  def managed_operation(store, account, request):
1306
1326
  if request['operation'] == 'reserve':
1307
- store.create(request['owner'], [], free=work_free(store), job_id=request['id'], workspace=False)
1327
+ store.create(request['owner'], [], job_id=request['id'], workspace=False)
1308
1328
  elif request['operation'] == 'create':
1309
1329
  declared_output_names(request['outputs'], strict=True)
1310
- job = store.create(request['owner'], request['outputs'], free=work_free(store),
1311
- results_free=available(store.results), inodes=work_inodes(store),
1312
- job_id=request['id'])
1330
+ job = store.create(request['owner'], request['outputs'], job_id=request['id'])
1313
1331
  root = store.open_workspace(job['id'])
1314
1332
  try:
1315
1333
  prepare_hooks(root)
@@ -0,0 +1,74 @@
1
+ #!/bin/bash
2
+ set -euo pipefail
3
+ umask 077
4
+ dir=/var/lib/orbitswap
5
+ file="$dir/swapfile"
6
+ size=4294967296
7
+ minimum_free=12884901888
8
+ if [ -L "$dir" ] || { [ -e "$dir" ] && { [ ! -d "$dir" ] || [ "$(stat -c %u "$dir")" != 0 ]; }; }; then
9
+ echo 'Swap directory must be a root-owned directory, not a link.' >&2
10
+ exit 1
11
+ fi
12
+ mkdir -p "$dir"
13
+ chown root:root "$dir"
14
+ chmod 700 "$dir"
15
+ if [ -L "$file" ] || { [ -e "$file" ] && { [ ! -f "$file" ] || [ "$(stat -c %h "$file")" != 1 ]; }; }; then
16
+ echo 'Swap file must be a regular file with one name, not a link.' >&2
17
+ exit 1
18
+ fi
19
+ active=$(swapon --show=NAME --noheadings --raw)
20
+ is_active=no
21
+ if grep -Fxq "$file" <<< "$active"; then is_active=yes; fi
22
+ current_size=0
23
+ if [ -f "$file" ]; then
24
+ current_size=$(stat -c %s "$file")
25
+ chown root:root "$file"
26
+ chmod 600 "$file"
27
+ fi
28
+ if [ "$is_active" = yes ] && [ "$current_size" != "$size" ]; then
29
+ echo 'Station swap is active with the wrong size. Stop builds, disable this swap when memory allows, then reapply the base recipe.' >&2
30
+ exit 1
31
+ fi
32
+ signature=
33
+ if [ "$is_active" = yes ]; then
34
+ signature=swap
35
+ elif [ -f "$file" ]; then
36
+ signature=$(blkid -p -s TYPE -o value "$file" || true)
37
+ fi
38
+ if [ "$current_size" != "$size" ] || [ "$signature" != swap ]; then
39
+ free=$(df -B1 --output=avail "$dir" | tail -n 1 | tr -d '[:space:]')
40
+ if ! [[ "$free" =~ ^[0-9]{1,18}$ ]] || (( free < minimum_free )); then
41
+ echo 'Swap needs at least 12 GiB free to leave 8 GiB after allocation. Free disk space, then reapply the base recipe.' >&2
42
+ exit 1
43
+ fi
44
+ if [ "$(stat -f -c %T "$dir")" = btrfs ]; then
45
+ echo 'Station swap needs a filesystem that supports ordinary swap files; this recipe does not configure Btrfs swap.' >&2
46
+ exit 1
47
+ fi
48
+ candidate=$(mktemp "$dir/.swapfile.XXXXXX")
49
+ trap 'rm -f "$candidate"' EXIT
50
+ chown root:root "$candidate"
51
+ chmod 600 "$candidate"
52
+ dd if=/dev/zero of="$candidate" bs=1M count=4096 conv=fsync status=none
53
+ mkswap "$candidate" >/dev/null
54
+ mv -f "$candidate" "$file"
55
+ trap - EXIT
56
+ fi
57
+ cat > /etc/systemd/system/var-lib-orbitswap-swapfile.swap <<'UNIT'
58
+ [Unit]
59
+ Description=Station swap for build memory spikes
60
+
61
+ [Swap]
62
+ What=/var/lib/orbitswap/swapfile
63
+
64
+ [Install]
65
+ WantedBy=swap.target
66
+ UNIT
67
+ chown root:root /etc/systemd/system/var-lib-orbitswap-swapfile.swap
68
+ chmod 644 /etc/systemd/system/var-lib-orbitswap-swapfile.swap
69
+ systemctl daemon-reload
70
+ systemctl enable --now var-lib-orbitswap-swapfile.swap
71
+ printf 'vm.swappiness=10\n' > /etc/sysctl.d/90-orbit-swap.conf
72
+ chown root:root /etc/sysctl.d/90-orbit-swap.conf
73
+ chmod 644 /etc/sysctl.d/90-orbit-swap.conf
74
+ sysctl -q -p /etc/sysctl.d/90-orbit-swap.conf
@@ -42,6 +42,9 @@ name = "orbit"
42
42
  [[script]]
43
43
  path = "base-linux/packages.sh"
44
44
 
45
+ [[script]]
46
+ path = "base-linux/swap.sh"
47
+
45
48
  [[script]]
46
49
  path = "base-linux/bwrap-userns.sh"
47
50
 
@@ -1 +1 @@
1
- 204aea94877ae0fdc63ccee1e6e19e67f309dc03
1
+ b123f3719d730c4807326038d8516c21c33e7969