@elaraai/e3-types 1.0.79 → 1.0.81

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (158) hide show
  1. package/README.md +1 -1
  2. package/dist/src/api.d.ts +829 -36
  3. package/dist/src/api.d.ts.map +1 -1
  4. package/dist/src/api.js +301 -32
  5. package/dist/src/api.js.map +1 -1
  6. package/dist/src/collection-manifest.d.ts +37 -0
  7. package/dist/src/collection-manifest.d.ts.map +1 -0
  8. package/dist/src/collection-manifest.js +37 -0
  9. package/dist/src/collection-manifest.js.map +1 -0
  10. package/dist/src/dataflow.d.ts +264 -19
  11. package/dist/src/dataflow.d.ts.map +1 -1
  12. package/dist/src/dataflow.js +156 -15
  13. package/dist/src/dataflow.js.map +1 -1
  14. package/dist/src/dataflow.spec.js +184 -27
  15. package/dist/src/dataflow.spec.js.map +1 -1
  16. package/dist/src/dataset-blob.d.ts +75 -15
  17. package/dist/src/dataset-blob.d.ts.map +1 -1
  18. package/dist/src/dataset-blob.js +95 -28
  19. package/dist/src/dataset-blob.js.map +1 -1
  20. package/dist/src/dataset-blob.spec.d.ts +6 -0
  21. package/dist/src/dataset-blob.spec.d.ts.map +1 -0
  22. package/dist/src/dataset-blob.spec.js +92 -0
  23. package/dist/src/dataset-blob.spec.js.map +1 -0
  24. package/dist/src/dataset-ref.d.ts +5 -4
  25. package/dist/src/dataset-ref.d.ts.map +1 -1
  26. package/dist/src/dataset-ref.js +5 -4
  27. package/dist/src/dataset-ref.js.map +1 -1
  28. package/dist/src/environment.d.ts +3 -2
  29. package/dist/src/environment.d.ts.map +1 -1
  30. package/dist/src/environment.js +3 -2
  31. package/dist/src/environment.js.map +1 -1
  32. package/dist/src/execution.d.ts +112 -18
  33. package/dist/src/execution.d.ts.map +1 -1
  34. package/dist/src/execution.js +174 -45
  35. package/dist/src/execution.js.map +1 -1
  36. package/dist/src/execution.spec.d.ts +6 -0
  37. package/dist/src/execution.spec.d.ts.map +1 -0
  38. package/dist/src/execution.spec.js +108 -0
  39. package/dist/src/execution.spec.js.map +1 -0
  40. package/dist/src/function.d.ts +14 -12
  41. package/dist/src/function.d.ts.map +1 -1
  42. package/dist/src/function.js +17 -32
  43. package/dist/src/function.js.map +1 -1
  44. package/dist/src/index.d.ts +17 -12
  45. package/dist/src/index.d.ts.map +1 -1
  46. package/dist/src/index.js +28 -20
  47. package/dist/src/index.js.map +1 -1
  48. package/dist/src/intake.d.ts +75 -0
  49. package/dist/src/intake.d.ts.map +1 -0
  50. package/dist/src/intake.js +46 -0
  51. package/dist/src/intake.js.map +1 -0
  52. package/dist/src/lock.d.ts +253 -21
  53. package/dist/src/lock.d.ts.map +1 -1
  54. package/dist/src/lock.js +55 -21
  55. package/dist/src/lock.js.map +1 -1
  56. package/dist/src/names.d.ts +21 -0
  57. package/dist/src/names.d.ts.map +1 -0
  58. package/dist/src/names.js +31 -0
  59. package/dist/src/names.js.map +1 -0
  60. package/dist/src/package.d.ts +11 -78
  61. package/dist/src/package.d.ts.map +1 -1
  62. package/dist/src/package.js +13 -66
  63. package/dist/src/package.js.map +1 -1
  64. package/dist/src/record.d.ts +582 -18
  65. package/dist/src/record.d.ts.map +1 -1
  66. package/dist/src/record.js +405 -7
  67. package/dist/src/record.js.map +1 -1
  68. package/dist/src/{stream.spec.d.ts → record.spec.d.ts} +1 -1
  69. package/dist/src/record.spec.d.ts.map +1 -0
  70. package/dist/src/record.spec.js +143 -0
  71. package/dist/src/record.spec.js.map +1 -0
  72. package/dist/src/release.d.ts +36 -0
  73. package/dist/src/release.d.ts.map +1 -0
  74. package/dist/src/release.js +71 -0
  75. package/dist/src/release.js.map +1 -0
  76. package/dist/src/release.spec.d.ts +6 -0
  77. package/dist/src/release.spec.d.ts.map +1 -0
  78. package/dist/src/release.spec.js +42 -0
  79. package/dist/src/release.spec.js.map +1 -0
  80. package/dist/src/repository.d.ts +64 -0
  81. package/dist/src/repository.d.ts.map +1 -0
  82. package/dist/src/repository.js +49 -0
  83. package/dist/src/repository.js.map +1 -0
  84. package/dist/src/runner.d.ts +16 -62
  85. package/dist/src/runner.d.ts.map +1 -1
  86. package/dist/src/runner.js +16 -79
  87. package/dist/src/runner.js.map +1 -1
  88. package/dist/src/runner.spec.js +11 -85
  89. package/dist/src/runner.spec.js.map +1 -1
  90. package/dist/src/structure.d.ts +0 -57
  91. package/dist/src/structure.d.ts.map +1 -1
  92. package/dist/src/structure.js +0 -3
  93. package/dist/src/structure.js.map +1 -1
  94. package/dist/src/task.d.ts +210 -318
  95. package/dist/src/task.d.ts.map +1 -1
  96. package/dist/src/task.js +117 -472
  97. package/dist/src/task.js.map +1 -1
  98. package/dist/src/task.spec.js +62 -154
  99. package/dist/src/task.spec.js.map +1 -1
  100. package/dist/src/transfer.d.ts +38 -26
  101. package/dist/src/transfer.d.ts.map +1 -1
  102. package/dist/src/transfer.js +22 -26
  103. package/dist/src/transfer.js.map +1 -1
  104. package/dist/src/transfer.spec.js +4 -34
  105. package/dist/src/transfer.spec.js.map +1 -1
  106. package/dist/src/unit-plan.d.ts +130 -0
  107. package/dist/src/unit-plan.d.ts.map +1 -0
  108. package/dist/src/unit-plan.js +107 -0
  109. package/dist/src/unit-plan.js.map +1 -0
  110. package/dist/src/unit-plan.spec.d.ts +6 -0
  111. package/dist/src/unit-plan.spec.d.ts.map +1 -0
  112. package/dist/src/unit-plan.spec.js +29 -0
  113. package/dist/src/unit-plan.spec.js.map +1 -0
  114. package/dist/src/workspace.d.ts +22 -11
  115. package/dist/src/workspace.d.ts.map +1 -1
  116. package/dist/src/workspace.js +10 -11
  117. package/dist/src/workspace.js.map +1 -1
  118. package/package.json +3 -3
  119. package/src/api.ts +326 -32
  120. package/src/collection-manifest.ts +53 -0
  121. package/src/dataflow.spec.ts +223 -31
  122. package/src/dataflow.ts +183 -26
  123. package/src/dataset-blob.spec.ts +117 -0
  124. package/src/dataset-blob.ts +157 -21
  125. package/src/dataset-ref.ts +5 -4
  126. package/src/environment.ts +3 -2
  127. package/src/execution.spec.ts +114 -0
  128. package/src/execution.ts +175 -51
  129. package/src/function.ts +19 -32
  130. package/src/index.ts +141 -39
  131. package/src/intake.ts +53 -0
  132. package/src/lock.ts +67 -21
  133. package/src/names.ts +39 -0
  134. package/src/package.ts +15 -71
  135. package/src/record.spec.ts +195 -0
  136. package/src/record.ts +489 -7
  137. package/src/release.spec.ts +47 -0
  138. package/src/release.ts +69 -0
  139. package/src/repository.ts +67 -0
  140. package/src/runner.spec.ts +11 -94
  141. package/src/runner.ts +16 -80
  142. package/src/structure.ts +0 -8
  143. package/src/task.spec.ts +67 -188
  144. package/src/task.ts +140 -522
  145. package/src/transfer.spec.ts +4 -51
  146. package/src/transfer.ts +22 -26
  147. package/src/unit-plan.spec.ts +31 -0
  148. package/src/unit-plan.ts +121 -0
  149. package/src/workspace.ts +13 -11
  150. package/dist/src/stream.d.ts +0 -97
  151. package/dist/src/stream.d.ts.map +0 -1
  152. package/dist/src/stream.js +0 -134
  153. package/dist/src/stream.js.map +0 -1
  154. package/dist/src/stream.spec.d.ts.map +0 -1
  155. package/dist/src/stream.spec.js +0 -60
  156. package/dist/src/stream.spec.js.map +0 -1
  157. package/src/stream.spec.ts +0 -88
  158. package/src/stream.ts +0 -170
package/src/api.ts CHANGED
@@ -36,7 +36,10 @@ import {
36
36
  } from '@elaraai/east';
37
37
 
38
38
  import { StructureType, TreePathType } from './structure.js';
39
+ import { IntakeFileType } from './intake.js';
39
40
  import { RunnerType } from './runner.js';
41
+ import { TaskBodyType, TaskInputType, TaskOutputType, TaskRoleType } from './task.js';
42
+ import { RequeueReasonType, StageUnitType } from './dataflow.js';
40
43
 
41
44
  // =============================================================================
42
45
  // Error Types
@@ -78,6 +81,13 @@ export const DatasetTypeMismatchErrorType = StructType({
78
81
  path: StringType,
79
82
  message: StringType,
80
83
  });
84
+ /** A name e3 would make a path of that cannot be one path segment: a
85
+ * repository's, a workspace's, or a package's name or version. */
86
+ export const InvalidNameErrorType = StructType({
87
+ kind: StringType,
88
+ name: StringType,
89
+ message: StringType,
90
+ });
81
91
 
82
92
  export const ErrorType = VariantType({
83
93
  repository_not_found: RepositoryNotFoundErrorType,
@@ -96,9 +106,8 @@ export const ErrorType = VariantType({
96
106
  dataflow_aborted: NullType,
97
107
  permission_denied: PermissionDeniedErrorType,
98
108
  internal: InternalErrorType,
99
- // Appended last: BEAST2 encodes a variant case by index, so a case added at
100
- // the end leaves every existing payload decodable.
101
109
  dataset_type_mismatch: DatasetTypeMismatchErrorType,
110
+ invalid_name: InvalidNameErrorType,
102
111
  });
103
112
 
104
113
  // =============================================================================
@@ -134,10 +143,16 @@ export const RepositoryStatusType = StructType({
134
143
  *
135
144
  * @property dryRun - If true, report what would be deleted without deleting
136
145
  * @property minAge - Minimum age in milliseconds for objects to be considered for deletion
146
+ * @property keepRuns - The runs of each workspace kept however old, the latest
147
+ * first; the server's default when `none`
148
+ * @property keepDays - The days of runs and executions kept however many; the
149
+ * server's default when `none`
137
150
  */
138
151
  export const GcRequestType = StructType({
139
152
  dryRun: BooleanType,
140
153
  minAge: OptionType(IntegerType),
154
+ keepRuns: OptionType(IntegerType),
155
+ keepDays: OptionType(IntegerType),
141
156
  });
142
157
 
143
158
  /**
@@ -148,6 +163,9 @@ export const GcRequestType = StructType({
148
163
  * @property retainedObjects - Number of objects still referenced
149
164
  * @property skippedYoung - Number of objects skipped due to minAge
150
165
  * @property bytesFreed - Total bytes freed by deletion
166
+ * @property deletedRuns - Number of dataflow run records deleted
167
+ * @property deletedExecutions - Number of execution attempts deleted, each with
168
+ * its owner record and logs
151
169
  */
152
170
  export const GcResultType = StructType({
153
171
  deletedObjects: IntegerType,
@@ -155,6 +173,8 @@ export const GcResultType = StructType({
155
173
  retainedObjects: IntegerType,
156
174
  skippedYoung: IntegerType,
157
175
  bytesFreed: IntegerType,
176
+ deletedRuns: IntegerType,
177
+ deletedExecutions: IntegerType,
158
178
  });
159
179
 
160
180
  // =============================================================================
@@ -269,13 +289,164 @@ export const WorkspaceInfoType = StructType({
269
289
  packageVersion: OptionType(StringType),
270
290
  });
271
291
 
292
+ /**
293
+ * What a deploy does with a record it cannot keep as it is.
294
+ *
295
+ * - `migrate`: run the migrations the workspace has not applied, and refuse a
296
+ * record no migration carries to the package's type
297
+ * - `fail`: run none, and refuse a record with migrations to run, for a
298
+ * workspace whose migrations go through their own change control
299
+ * - `reset`: reset a record it can neither keep nor migrate to the package's
300
+ * initial value, under a `$reset` commit, so the reset is in its history
301
+ */
302
+ export const SchemaPolicyType = VariantType({
303
+ migrate: NullType,
304
+ fail: NullType,
305
+ reset: NullType,
306
+ });
307
+
272
308
  /**
273
309
  * Request to deploy a package to a workspace.
274
310
  *
275
311
  * @property packageRef - Package reference in format "name" or "name@version"
312
+ * @property schema - What the deploy does with a record it cannot keep as it is
313
+ * @property allowDropRecords - Whether a record the package no longer declares
314
+ * may be dropped, with its state and history
315
+ * @property plan - Say what the deploy would do, and write nothing
276
316
  */
277
317
  export const WorkspaceDeployRequestType = StructType({
278
318
  packageRef: StringType,
319
+ schema: SchemaPolicyType,
320
+ allowDropRecords: BooleanType,
321
+ plan: BooleanType,
322
+ });
323
+
324
+ /**
325
+ * What a deploy decided for one record.
326
+ *
327
+ * @property record - The record's dataset ref path
328
+ * @property action - What the deploy does to it:
329
+ * - `mint`: the workspace does not hold it, so it is minted from the
330
+ * package's initial value
331
+ * - `keep`: kept as the workspace holds it; `deploy` when the package under
332
+ * it changed, which a `$deploy` commit records in its history
333
+ * - `migrate`: migrated by the steps the workspace has not applied, in order
334
+ * - `reset`: reset to the package's initial value under the `reset` policy,
335
+ * and why it could not be kept or migrated
336
+ * - `drop`: the package does not declare it, so it is dropped, with its
337
+ * state and history
338
+ * - `refused`: refused, and why, with the fix: the deploy writes nothing
339
+ */
340
+ export const RecordPlanType = StructType({
341
+ record: StringType,
342
+ action: VariantType({
343
+ mint: NullType,
344
+ keep: StructType({ deploy: BooleanType }),
345
+ migrate: StructType({ steps: ArrayType(StringType) }),
346
+ reset: StructType({ reason: StringType }),
347
+ drop: NullType,
348
+ refused: StructType({ reason: StringType }),
349
+ }),
350
+ });
351
+
352
+ /**
353
+ * What a deploy decided for one of a record's indexes.
354
+ *
355
+ * @property record - The record's dataset ref path
356
+ * @property index - The index's name
357
+ * @property action - `build` when the state names no index under the
358
+ * package's declaration, `drop` when it names one the package does not
359
+ * declare, and `keep` when the two agree and nothing runs
360
+ */
361
+ export const RecordIndexPlanType = StructType({
362
+ record: StringType,
363
+ index: StringType,
364
+ action: VariantType({
365
+ build: NullType,
366
+ drop: NullType,
367
+ keep: NullType,
368
+ }),
369
+ });
370
+
371
+ /**
372
+ * The step a deploy is at with one record.
373
+ *
374
+ * - `waiting`: its turn has not come, or it needs nothing but the deploy's
375
+ * commit
376
+ * - `migrating`: running migration step `step` of the `steps` it owes, `name`
377
+ * - `indexing`: building `index`, the `build`th of the `builds` it owes
378
+ * - `done`: its migrations and index builds have run
379
+ */
380
+ export const RecordDeployStepType = VariantType({
381
+ waiting: NullType,
382
+ migrating: StructType({ name: StringType, step: IntegerType, steps: IntegerType }),
383
+ indexing: StructType({ index: StringType, build: IntegerType, builds: IntegerType }),
384
+ done: NullType,
385
+ });
386
+
387
+ /**
388
+ * One record a deploy is deploying.
389
+ *
390
+ * @property plan - What the deploy decided for it
391
+ * @property indexes - The indexes the package declares for it
392
+ * @property step - How far the deploy has got with it
393
+ */
394
+ export const RecordDeployStateType = StructType({
395
+ plan: RecordPlanType,
396
+ indexes: ArrayType(StringType),
397
+ step: RecordDeployStepType,
398
+ });
399
+
400
+ /**
401
+ * How far a deploy has got, for a client to show while it runs.
402
+ *
403
+ * @property package - The package it deploys
404
+ * @property startedAt - When it began taking its files in
405
+ * @property files - Each file source it takes in, in the package's order
406
+ * @property records - Each record it deploys, in the package's order, and
407
+ * each it drops
408
+ */
409
+ export const DeployProgressType = StructType({
410
+ package: StructType({ name: StringType, version: StringType }),
411
+ startedAt: DateTimeType,
412
+ files: ArrayType(IntakeFileType),
413
+ records: ArrayType(RecordDeployStateType),
414
+ });
415
+
416
+ /**
417
+ * What a deploy did, or under `plan` would do.
418
+ *
419
+ * @property records - What it decided for each record
420
+ * @property indexes - What it decided for each index of the records it keeps
421
+ * @property warnings - The inputs it left unassigned, and why: a server never
422
+ * reads a `file` source, whose path is on the client's machine
423
+ */
424
+ export const WorkspaceDeployResultType = StructType({
425
+ records: ArrayType(RecordPlanType),
426
+ indexes: ArrayType(RecordIndexPlanType),
427
+ warnings: ArrayType(StringType),
428
+ });
429
+
430
+ /**
431
+ * A deploy job's progress: `pending` until it starts, then `deploying`, with
432
+ * how far it has got once it has said.
433
+ */
434
+ export const WorkspaceDeployProgressType = VariantType({
435
+ pending: NullType,
436
+ deploying: OptionType(DeployProgressType),
437
+ });
438
+
439
+ /**
440
+ * A deploy job's status.
441
+ *
442
+ * - `processing`: still running, and how far it has got
443
+ * - `completed`: what the deploy did
444
+ * - `failed`: why it did not
445
+ */
446
+ export const WorkspaceDeployStatusType = VariantType({
447
+ processing: WorkspaceDeployProgressType,
448
+ completed: WorkspaceDeployResultType,
449
+ failed: StructType({ message: StringType }),
279
450
  });
280
451
 
281
452
  /**
@@ -391,6 +562,9 @@ export const DatasetStatusInfoType = StructType({
391
562
  * @property inputs - Dataset paths this task reads from
392
563
  * @property output - Dataset path this task writes to
393
564
  * @property dependsOn - Names of tasks that must complete before this one
565
+ * @property peakBytes - The highest peak resident memory, in bytes, a runner
566
+ * of the execution the status comes from reached: the one the output came
567
+ * from, or the failure; `none` when it recorded none, or while it runs
394
568
  */
395
569
  export const TaskStatusInfoType = StructType({
396
570
  name: StringType,
@@ -399,6 +573,7 @@ export const TaskStatusInfoType = StructType({
399
573
  inputs: ArrayType(StringType),
400
574
  output: StringType,
401
575
  dependsOn: ArrayType(StringType),
576
+ peakBytes: OptionType(IntegerType),
402
577
  });
403
578
 
404
579
  /**
@@ -451,31 +626,34 @@ export const WorkspaceStatusResultType = StructType({
451
626
  *
452
627
  * @property name - Task name
453
628
  * @property hash - Task definition hash
454
- * @property kind - Task kind: "ui" (e3-ui `ui()`), other future kinds; none for plain tasks / old packages
629
+ * @property role - What the task's output is for: data, or a ui with what it
630
+ * binds
455
631
  */
456
632
  export const TaskListItemType = StructType({
457
633
  name: StringType,
458
634
  hash: StringType,
459
- kind: OptionType(StringType),
635
+ role: TaskRoleType,
460
636
  });
461
637
 
462
638
  /**
463
- * Detailed task information.
639
+ * Detailed task information: the task object's fields.
464
640
  *
465
641
  * @property name - Task name
466
642
  * @property hash - Task definition hash
467
- * @property commandIr - East IR for the task's command
468
- * @property inputs - Tree paths for task inputs
469
- * @property output - Tree path for task output
643
+ * @property body - What the task runs: an East program, or a command
644
+ * @property runner - The runtime it runs on
645
+ * @property inputs - The datasets it reads, each with its partition
646
+ * @property output - Where its output goes, and how it is made
647
+ * @property role - What its output is for: data, or a ui with what it binds
470
648
  */
471
649
  export const TaskDetailsType = StructType({
472
650
  name: StringType,
473
651
  hash: StringType,
474
- commandIr: StringType,
475
- inputs: ArrayType(TreePathType),
476
- output: TreePathType,
477
- kind: OptionType(StringType),
478
- metadata: OptionType(BlobType),
652
+ body: TaskBodyType,
653
+ runner: RunnerType,
654
+ inputs: ArrayType(TaskInputType),
655
+ output: TaskOutputType,
656
+ role: TaskRoleType,
479
657
  });
480
658
 
481
659
  // =============================================================================
@@ -483,14 +661,13 @@ export const TaskDetailsType = StructType({
483
661
  // =============================================================================
484
662
 
485
663
  /**
486
- * Request to start dataflow execution.
664
+ * Request to start dataflow execution. The run takes the server's budget of
665
+ * cores and memory, which it shares with everything else the server runs.
487
666
  *
488
- * @property concurrency - Maximum parallel tasks (default: 4)
489
667
  * @property force - Force re-execution of all tasks
490
668
  * @property filter - Filter to specific task names (glob pattern)
491
669
  */
492
670
  export const DataflowRequestType = StructType({
493
- concurrency: OptionType(IntegerType),
494
671
  force: BooleanType,
495
672
  filter: OptionType(StringType),
496
673
  });
@@ -561,11 +738,15 @@ export const DataflowResultType = StructType({
561
738
  * Dataflow event types for API polling.
562
739
  *
563
740
  * - `start`: Task started executing
564
- * - `complete`: Task executed and succeeded
741
+ * - `complete`: Task executed and succeeded, with the highest peak memory its
742
+ * runners reached when they reported one
565
743
  * - `cached`: Task result retrieved from cache (no execution)
566
744
  * - `failed`: Task exited with non-zero code
567
745
  * - `error`: Internal error during task execution
568
746
  * - `input_unavailable`: Task couldn't run because inputs not available
747
+ * - `requeued`: A unit of a split task was stopped — by the budget's guard, or
748
+ * its cgroup's cap — and runs again once the memory it reserves fits: the
749
+ * unit, why, the most it was measured using and what it reserves, in bytes
569
750
  */
570
751
  export const DataflowEventType = VariantType({
571
752
  start: StructType({
@@ -576,6 +757,7 @@ export const DataflowEventType = VariantType({
576
757
  task: StringType,
577
758
  timestamp: StringType,
578
759
  duration: FloatType,
760
+ peakBytes: OptionType(IntegerType),
579
761
  }),
580
762
  cached: StructType({
581
763
  task: StringType,
@@ -597,6 +779,14 @@ export const DataflowEventType = VariantType({
597
779
  timestamp: StringType,
598
780
  reason: StringType,
599
781
  }),
782
+ requeued: StructType({
783
+ task: StringType,
784
+ timestamp: StringType,
785
+ unit: StageUnitType,
786
+ reason: RequeueReasonType,
787
+ peak: IntegerType,
788
+ reserves: IntegerType,
789
+ }),
600
790
  });
601
791
 
602
792
  /**
@@ -628,6 +818,55 @@ export const DataflowExecutionSummaryType = StructType({
628
818
  duration: FloatType,
629
819
  });
630
820
 
821
+ /**
822
+ * A server's budget of cores and memory, which every runner it spawns takes
823
+ * from, and what its runners hold of it now.
824
+ *
825
+ * @property cores - Runner processes at once
826
+ * @property memory - Bytes of memory the runners may hold between them
827
+ * @property coresInUse - Runner processes holding the budget now
828
+ * @property memoryInUse - Bytes the runners hold now, each at the larger of
829
+ * its reservation and what it was last measured using
830
+ */
831
+ export const DataflowBudgetType = StructType({
832
+ cores: IntegerType,
833
+ memory: IntegerType,
834
+ coresInUse: IntegerType,
835
+ memoryInUse: IntegerType,
836
+ });
837
+
838
+ /**
839
+ * A task, or a unit of a split task, waiting for room in the server's budget.
840
+ *
841
+ * @property task - The task
842
+ * @property unit - The unit, for a split task's; `none` for a task run as one
843
+ * @property needs - The memory, in bytes, it waits to reserve; 0 when it
844
+ * waits for a core alone
845
+ * @property since - ISO timestamp when it began waiting
846
+ */
847
+ export const UnitWaitType = StructType({
848
+ task: StringType,
849
+ unit: OptionType(StageUnitType),
850
+ needs: IntegerType,
851
+ since: StringType,
852
+ });
853
+
854
+ /**
855
+ * A split task's progress through the stage it is in.
856
+ *
857
+ * @property task - The task
858
+ * @property merge - The merge level, from 1, and the levels the merges take;
859
+ * `none` while its pieces run
860
+ * @property done - The stage's units that have finished
861
+ * @property units - The stage's units
862
+ */
863
+ export const SplitProgressType = StructType({
864
+ task: StringType,
865
+ merge: OptionType(StructType({ level: IntegerType, levels: IntegerType })),
866
+ done: IntegerType,
867
+ units: IntegerType,
868
+ });
869
+
631
870
  /**
632
871
  * Dataflow execution state returned by API polling.
633
872
  *
@@ -643,6 +882,12 @@ export const DataflowExecutionSummaryType = StructType({
643
882
  * @property summary - Execution summary (available when complete)
644
883
  * @property events - Task events (may be paginated via offset/limit)
645
884
  * @property totalEvents - Total number of events (for pagination)
885
+ * @property budget - The server's budget now, where it has one: a server whose
886
+ * runners hold none, as a remote backend's, serves `none`
887
+ * @property waiting - The tasks and units of the run waiting for room, while it
888
+ * runs; nothing stores them
889
+ * @property splits - Each split task's progress, while the run runs; nothing
890
+ * stores it
646
891
  */
647
892
  export const ApiDataflowExecutionStateType = StructType({
648
893
  status: ApiExecutionStatusType,
@@ -651,6 +896,9 @@ export const ApiDataflowExecutionStateType = StructType({
651
896
  summary: OptionType(DataflowExecutionSummaryType),
652
897
  events: ArrayType(DataflowEventType),
653
898
  totalEvents: IntegerType,
899
+ budget: OptionType(DataflowBudgetType),
900
+ waiting: ArrayType(UnitWaitType),
901
+ splits: ArrayType(SplitProgressType),
654
902
  });
655
903
 
656
904
  // =============================================================================
@@ -658,13 +906,17 @@ export const ApiDataflowExecutionStateType = StructType({
658
906
  // =============================================================================
659
907
 
660
908
  /**
661
- * Execution status for history listing.
909
+ * Execution status for history listing: `cancelled` when e3 stopped the
910
+ * execution because its run was aborted, `interrupted` when the orchestrator
911
+ * that owned it exited before it finished.
662
912
  */
663
913
  export const ExecutionHistoryStatusType = VariantType({
664
914
  running: NullType,
665
915
  success: NullType,
666
916
  failed: NullType,
667
917
  error: NullType,
918
+ cancelled: NullType,
919
+ interrupted: NullType,
668
920
  });
669
921
 
670
922
  /**
@@ -677,6 +929,8 @@ export const ExecutionHistoryStatusType = VariantType({
677
929
  * @property completedAt - ISO timestamp when execution finished (if done)
678
930
  * @property duration - Execution duration in milliseconds (if done)
679
931
  * @property exitCode - Process exit code (if failed)
932
+ * @property peakBytes - The highest peak resident memory, in bytes, a runner
933
+ * of the execution reached, when it succeeded or failed and one reported it
680
934
  */
681
935
  export const ExecutionListItemType = StructType({
682
936
  inputsHash: StringType,
@@ -686,6 +940,7 @@ export const ExecutionListItemType = StructType({
686
940
  completedAt: OptionType(StringType),
687
941
  duration: OptionType(IntegerType),
688
942
  exitCode: OptionType(IntegerType),
943
+ peakBytes: OptionType(IntegerType),
689
944
  });
690
945
 
691
946
  // =============================================================================
@@ -703,7 +958,9 @@ export const TreeKindType = VariantType({ struct: NullType });
703
958
  * A list entry -- either a dataset leaf or a tree branch.
704
959
  *
705
960
  * Used by the `?list=true&status=true` endpoints to return both
706
- * tree structure entries and dataset leaves in a single flat list.
961
+ * tree structure entries and dataset leaves in a single flat list. A dataset's
962
+ * `size` is what its value weighs in the store, as its status reports it: for
963
+ * a collection, its segments and its manifest.
707
964
  */
708
965
  export const ListEntryType = VariantType({
709
966
  dataset: StructType({
@@ -729,7 +986,8 @@ export const ListEntryType = VariantType({
729
986
  * @property type - East type of the dataset
730
987
  * @property refType - Ref type: "unassigned", "null", or "value"
731
988
  * @property hash - Object hash (None if unassigned/null)
732
- * @property size - Size in bytes (None if unassigned)
989
+ * @property size - Bytes the value weighs in the store: for a collection, its
990
+ * segments and its manifest (None if unassigned)
733
991
  */
734
992
  export const DatasetStatusDetailType = StructType({
735
993
  path: StringType,
@@ -737,10 +995,9 @@ export const DatasetStatusDetailType = StructType({
737
995
  refType: StringType,
738
996
  hash: OptionType(StringType),
739
997
  size: OptionType(IntegerType),
740
- /** Segment and element counts of a stored collection, read from the blob's
741
- * trailing index — so a re-pointed input is inspectable without decoding
742
- * it. `none` for a non-collection or an unset dataset. Appended LAST, per
743
- * the positional struct rule. */
998
+ /** Segment and element counts of a stored collection, read from its
999
+ * manifest — so a re-pointed input is inspectable without decoding it.
1000
+ * `none` for a non-collection or an unset dataset. */
744
1001
  segments: OptionType(IntegerType),
745
1002
  rows: OptionType(IntegerType),
746
1003
  });
@@ -834,6 +1091,10 @@ export const OneShotRequestType = StructType({
834
1091
  /**
835
1092
  * A record mutation call. Positional args (after the implicit current state),
836
1093
  * one beast2-encoded value per declared parameter.
1094
+ *
1095
+ * Of the limits, `timeoutMs` bounds each run of the mutation's program and
1096
+ * `maxLogBytes` the stderr a failure returns. `maxResultBytes` does not apply:
1097
+ * a mutation's output is stored as segments and never read whole.
837
1098
  */
838
1099
  export const MutationCallRequestType = StructType({
839
1100
  args: ArrayType(BlobType),
@@ -846,32 +1107,49 @@ export const MutationCallRequestType = StructType({
846
1107
  *
847
1108
  * - `committed`: the new commit + state hashes
848
1109
  * - `invalid`: record/mutation lookup or arity error; nothing ran
849
- * - `failed`: the reducer process exited non-zero (incl. a reducer `$.error`; see stderr)
850
- * - `too_large`: the new state exceeded the result cap
851
- * - `timed_out`: the reducer exceeded its time budget
852
- * - `conflict`: the compare-and-swap lost the race `attempts` times
1110
+ * - `failed`: the mutation's program failed (incl. a body's `$.error`; see stderr)
1111
+ * - `timed_out`: the program exceeded its time budget
1112
+ * - `conflict`: the compare-and-swap lost the race `attempts` times, or the
1113
+ * write disagreed with the state it landed on — then `detail` names the key,
1114
+ * and resubmitting the same write cannot help
853
1115
  */
854
1116
  export const MutationResultType = StructType({
855
1117
  outcome: VariantType({
856
1118
  committed: StructType({ commitHash: StringType, stateHash: StringType }),
857
1119
  invalid: StructType({ message: StringType }),
858
1120
  failed: StructType({ exitCode: IntegerType, stderr: StringType }),
859
- too_large: StructType({ bytes: IntegerType, limit: IntegerType, stderr: StringType }),
860
1121
  timed_out: StructType({ ms: IntegerType, stderr: StringType }),
861
- conflict: StructType({ attempts: IntegerType }),
1122
+ conflict: StructType({ attempts: IntegerType, detail: OptionType(StringType) }),
862
1123
  }),
863
1124
  });
864
1125
 
865
1126
  /**
866
1127
  * A record's callable surface, returned by `describe` so dynamic callers can
867
- * encode mutation arguments. The mutation `argTypes` are the EXTRA parameters
868
- * after the implicit current state.
1128
+ * encode mutation arguments and read through its indexes. The mutation
1129
+ * `argTypes` are the EXTRA parameters after the implicit current state.
869
1130
  */
870
1131
  export const RecordSignatureType = StructType({
871
1132
  name: StringType,
872
1133
  mutations: ArrayType(StructType({
873
1134
  name: StringType,
874
1135
  argTypes: ArrayType(EastTypeType),
1136
+ /** The write form — `reduce`, `edit` or `patch` — which says what the
1137
+ * arguments MEAN: a `patch` mutation's one argument is a
1138
+ * `PatchType(State)`, not a value of the record's own type. */
1139
+ form: StringType,
1140
+ })),
1141
+ /** The indexes the record declares. A page read through one is decoded by
1142
+ * `indexWindowType(K, keyType, valueType, V)` over the record's
1143
+ * `Dict<K, V>`, and a key search through one takes a `keyType` key. */
1144
+ indexes: ArrayType(StructType({
1145
+ name: StringType,
1146
+ /** The index key's type. */
1147
+ keyType: EastTypeType,
1148
+ /** The covering projection's type: `Null` when the index projects
1149
+ * nothing, and a read joins the rows instead. */
1150
+ valueType: EastTypeType,
1151
+ /** Whether a row may hold several index keys. */
1152
+ multi: BooleanType,
875
1153
  })),
876
1154
  });
877
1155
 
@@ -883,6 +1161,9 @@ export const RecordCommitInfoType = StructType({
883
1161
  mutation: StringType,
884
1162
  actor: StringType,
885
1163
  at: DateTimeType,
1164
+ /** The delta this commit applied, when it wrote one — so history shows WHAT
1165
+ * changed without diffing two states. */
1166
+ delta: OptionType(StringType),
886
1167
  });
887
1168
 
888
1169
  /** A page of a record's commit history, newest first. */
@@ -907,6 +1188,16 @@ export type PackageDetails = ValueTypeOf<typeof PackageDetailsType>;
907
1188
  export type WorkspaceInfo = ValueTypeOf<typeof WorkspaceInfoType>;
908
1189
  export type WorkspaceCreateRequest = ValueTypeOf<typeof WorkspaceCreateRequestType>;
909
1190
  export type WorkspaceDeployRequest = ValueTypeOf<typeof WorkspaceDeployRequestType>;
1191
+ /** A {@link SchemaPolicyType} by its name, as a deploy's options take it. */
1192
+ export type SchemaPolicy = ValueTypeOf<typeof SchemaPolicyType>['type'];
1193
+ export type RecordPlan = ValueTypeOf<typeof RecordPlanType>;
1194
+ export type RecordIndexPlan = ValueTypeOf<typeof RecordIndexPlanType>;
1195
+ export type RecordDeployStep = ValueTypeOf<typeof RecordDeployStepType>;
1196
+ export type RecordDeployState = ValueTypeOf<typeof RecordDeployStateType>;
1197
+ export type DeployProgress = ValueTypeOf<typeof DeployProgressType>;
1198
+ export type WorkspaceDeployResult = ValueTypeOf<typeof WorkspaceDeployResultType>;
1199
+ export type WorkspaceDeployProgress = ValueTypeOf<typeof WorkspaceDeployProgressType>;
1200
+ export type WorkspaceDeployStatus = ValueTypeOf<typeof WorkspaceDeployStatusType>;
910
1201
  export type DatasetStatus = ValueTypeOf<typeof DatasetStatusType>;
911
1202
  export type TaskStatus = ValueTypeOf<typeof TaskStatusType>;
912
1203
  export type DatasetStatusInfo = ValueTypeOf<typeof DatasetStatusInfoType>;
@@ -922,6 +1213,9 @@ export type DataflowResult = ValueTypeOf<typeof DataflowResultType>;
922
1213
  export type DataflowEvent = ValueTypeOf<typeof DataflowEventType>;
923
1214
  export type ApiExecutionStatus = ValueTypeOf<typeof ApiExecutionStatusType>;
924
1215
  export type DataflowExecutionSummary = ValueTypeOf<typeof DataflowExecutionSummaryType>;
1216
+ export type DataflowBudget = ValueTypeOf<typeof DataflowBudgetType>;
1217
+ export type UnitWait = ValueTypeOf<typeof UnitWaitType>;
1218
+ export type SplitProgress = ValueTypeOf<typeof SplitProgressType>;
925
1219
  export type ApiDataflowExecutionState = ValueTypeOf<typeof ApiDataflowExecutionStateType>;
926
1220
  export type ExecutionHistoryStatus = ValueTypeOf<typeof ExecutionHistoryStatusType>;
927
1221
  export type ExecutionListItem = ValueTypeOf<typeof ExecutionListItemType>;
@@ -0,0 +1,53 @@
1
+ /**
2
+ * Copyright (c) 2025 Elara AI Pty Ltd
3
+ * Dual-licensed under AGPL-3.0 and commercial license. See LICENSE for details.
4
+ */
5
+
6
+ /**
7
+ * The segment-object layout: a collection dataset is a **manifest naming
8
+ * standalone segment objects**, not one segmented blob.
9
+ *
10
+ * The type itself lives in `east`, beside the container it describes, because
11
+ * every runtime must read it: a task input staged as a manifest is one file
12
+ * naming sibling segment files, and a body that iterates it pays for the
13
+ * segments it touches rather than for a splice of the whole value. This module
14
+ * re-exports it for the e3 packages and adds what only a store cares about.
15
+ *
16
+ * Three properties are why records at millions of entries are affordable, and
17
+ * every one of them is about successive states SHARING storage rather than
18
+ * about indexing, which the trailing index already gave:
19
+ *
20
+ * - **A one-row edit costs one segment.** The new state names the same objects
21
+ * as the old one bar the segment that changed, so a commit's new storage is
22
+ * the touched segments plus a manifest — not the whole value.
23
+ * - **Two states diff in O(changed segments)**, by comparing entry hashes. No
24
+ * value is decoded to find out what moved, which is what index maintenance
25
+ * and the mutation delta are built on.
26
+ * - **A segment can be checked against its own hash**, which a ranged read of
27
+ * one blob can never offer: no digest exists for a byte range. No reader
28
+ * checks it — a segment is trusted as stored, like any other object.
29
+ *
30
+ * The manifest is the envelope the garbage collector recognizes, which is why
31
+ * its `kind` is a stored field rather than something inferred: an object whose
32
+ * entries name other objects MUST be traversable by shape, or the first sweep
33
+ * deletes a record's segments.
34
+ *
35
+ * @packageDocumentation
36
+ */
37
+
38
+ export {
39
+ COLLECTION_MANIFEST_KIND,
40
+ CollectionManifestType,
41
+ CollectionManifestEntryType,
42
+ type CollectionManifest,
43
+ type CollectionManifestEntry,
44
+ type Beast2ManifestSource,
45
+ isCollectionManifestType,
46
+ isCollectionManifest,
47
+ isBeast2ManifestSource,
48
+ encodeCollectionManifest,
49
+ decodeCollectionManifest,
50
+ readBeast2Manifest,
51
+ manifestElementCount,
52
+ manifestByteSize,
53
+ } from '@elaraai/east';