@elaraai/e3-types 1.0.78 → 1.0.80

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (153) hide show
  1. package/README.md +1 -1
  2. package/dist/src/api.d.ts +569 -36
  3. package/dist/src/api.d.ts.map +1 -1
  4. package/dist/src/api.js +257 -32
  5. package/dist/src/api.js.map +1 -1
  6. package/dist/src/collection-manifest.d.ts +37 -0
  7. package/dist/src/collection-manifest.d.ts.map +1 -0
  8. package/dist/src/collection-manifest.js +37 -0
  9. package/dist/src/collection-manifest.js.map +1 -0
  10. package/dist/src/dataflow.d.ts +264 -19
  11. package/dist/src/dataflow.d.ts.map +1 -1
  12. package/dist/src/dataflow.js +156 -15
  13. package/dist/src/dataflow.js.map +1 -1
  14. package/dist/src/dataflow.spec.js +184 -27
  15. package/dist/src/dataflow.spec.js.map +1 -1
  16. package/dist/src/dataset-blob.d.ts +75 -15
  17. package/dist/src/dataset-blob.d.ts.map +1 -1
  18. package/dist/src/dataset-blob.js +95 -28
  19. package/dist/src/dataset-blob.js.map +1 -1
  20. package/dist/src/dataset-blob.spec.d.ts +6 -0
  21. package/dist/src/dataset-blob.spec.d.ts.map +1 -0
  22. package/dist/src/dataset-blob.spec.js +92 -0
  23. package/dist/src/dataset-blob.spec.js.map +1 -0
  24. package/dist/src/dataset-ref.d.ts +5 -4
  25. package/dist/src/dataset-ref.d.ts.map +1 -1
  26. package/dist/src/dataset-ref.js +5 -4
  27. package/dist/src/dataset-ref.js.map +1 -1
  28. package/dist/src/environment.d.ts +3 -2
  29. package/dist/src/environment.d.ts.map +1 -1
  30. package/dist/src/environment.js +3 -2
  31. package/dist/src/environment.js.map +1 -1
  32. package/dist/src/execution.d.ts +112 -18
  33. package/dist/src/execution.d.ts.map +1 -1
  34. package/dist/src/execution.js +174 -45
  35. package/dist/src/execution.js.map +1 -1
  36. package/dist/src/execution.spec.d.ts +6 -0
  37. package/dist/src/execution.spec.d.ts.map +1 -0
  38. package/dist/src/execution.spec.js +108 -0
  39. package/dist/src/execution.spec.js.map +1 -0
  40. package/dist/src/function.d.ts +14 -12
  41. package/dist/src/function.d.ts.map +1 -1
  42. package/dist/src/function.js +17 -32
  43. package/dist/src/function.js.map +1 -1
  44. package/dist/src/index.d.ts +16 -12
  45. package/dist/src/index.d.ts.map +1 -1
  46. package/dist/src/index.js +25 -20
  47. package/dist/src/index.js.map +1 -1
  48. package/dist/src/lock.d.ts +68 -22
  49. package/dist/src/lock.d.ts.map +1 -1
  50. package/dist/src/lock.js +33 -22
  51. package/dist/src/lock.js.map +1 -1
  52. package/dist/src/names.d.ts +21 -0
  53. package/dist/src/names.d.ts.map +1 -0
  54. package/dist/src/names.js +31 -0
  55. package/dist/src/names.js.map +1 -0
  56. package/dist/src/package.d.ts +11 -78
  57. package/dist/src/package.d.ts.map +1 -1
  58. package/dist/src/package.js +13 -66
  59. package/dist/src/package.js.map +1 -1
  60. package/dist/src/record.d.ts +582 -18
  61. package/dist/src/record.d.ts.map +1 -1
  62. package/dist/src/record.js +405 -7
  63. package/dist/src/record.js.map +1 -1
  64. package/dist/src/{stream.spec.d.ts → record.spec.d.ts} +1 -1
  65. package/dist/src/record.spec.d.ts.map +1 -0
  66. package/dist/src/record.spec.js +143 -0
  67. package/dist/src/record.spec.js.map +1 -0
  68. package/dist/src/release.d.ts +36 -0
  69. package/dist/src/release.d.ts.map +1 -0
  70. package/dist/src/release.js +71 -0
  71. package/dist/src/release.js.map +1 -0
  72. package/dist/src/release.spec.d.ts +6 -0
  73. package/dist/src/release.spec.d.ts.map +1 -0
  74. package/dist/src/release.spec.js +42 -0
  75. package/dist/src/release.spec.js.map +1 -0
  76. package/dist/src/repository.d.ts +64 -0
  77. package/dist/src/repository.d.ts.map +1 -0
  78. package/dist/src/repository.js +49 -0
  79. package/dist/src/repository.js.map +1 -0
  80. package/dist/src/runner.d.ts +16 -62
  81. package/dist/src/runner.d.ts.map +1 -1
  82. package/dist/src/runner.js +16 -79
  83. package/dist/src/runner.js.map +1 -1
  84. package/dist/src/runner.spec.js +11 -85
  85. package/dist/src/runner.spec.js.map +1 -1
  86. package/dist/src/structure.d.ts +0 -57
  87. package/dist/src/structure.d.ts.map +1 -1
  88. package/dist/src/structure.js +0 -3
  89. package/dist/src/structure.js.map +1 -1
  90. package/dist/src/task.d.ts +210 -318
  91. package/dist/src/task.d.ts.map +1 -1
  92. package/dist/src/task.js +117 -472
  93. package/dist/src/task.js.map +1 -1
  94. package/dist/src/task.spec.js +62 -154
  95. package/dist/src/task.spec.js.map +1 -1
  96. package/dist/src/transfer.d.ts +16 -22
  97. package/dist/src/transfer.d.ts.map +1 -1
  98. package/dist/src/transfer.js +16 -22
  99. package/dist/src/transfer.js.map +1 -1
  100. package/dist/src/transfer.spec.js +4 -34
  101. package/dist/src/transfer.spec.js.map +1 -1
  102. package/dist/src/unit-plan.d.ts +130 -0
  103. package/dist/src/unit-plan.d.ts.map +1 -0
  104. package/dist/src/unit-plan.js +107 -0
  105. package/dist/src/unit-plan.js.map +1 -0
  106. package/dist/src/unit-plan.spec.d.ts +6 -0
  107. package/dist/src/unit-plan.spec.d.ts.map +1 -0
  108. package/dist/src/unit-plan.spec.js +29 -0
  109. package/dist/src/unit-plan.spec.js.map +1 -0
  110. package/dist/src/workspace.d.ts +22 -11
  111. package/dist/src/workspace.d.ts.map +1 -1
  112. package/dist/src/workspace.js +10 -11
  113. package/dist/src/workspace.js.map +1 -1
  114. package/package.json +3 -3
  115. package/src/api.ts +276 -32
  116. package/src/collection-manifest.ts +53 -0
  117. package/src/dataflow.spec.ts +223 -31
  118. package/src/dataflow.ts +183 -26
  119. package/src/dataset-blob.spec.ts +117 -0
  120. package/src/dataset-blob.ts +157 -21
  121. package/src/dataset-ref.ts +5 -4
  122. package/src/environment.ts +3 -2
  123. package/src/execution.spec.ts +114 -0
  124. package/src/execution.ts +175 -51
  125. package/src/function.ts +19 -32
  126. package/src/index.ts +122 -39
  127. package/src/lock.ts +39 -22
  128. package/src/names.ts +39 -0
  129. package/src/package.ts +15 -71
  130. package/src/record.spec.ts +195 -0
  131. package/src/record.ts +489 -7
  132. package/src/release.spec.ts +47 -0
  133. package/src/release.ts +69 -0
  134. package/src/repository.ts +67 -0
  135. package/src/runner.spec.ts +11 -94
  136. package/src/runner.ts +16 -80
  137. package/src/structure.ts +0 -8
  138. package/src/task.spec.ts +67 -188
  139. package/src/task.ts +140 -522
  140. package/src/transfer.spec.ts +4 -51
  141. package/src/transfer.ts +16 -22
  142. package/src/unit-plan.spec.ts +31 -0
  143. package/src/unit-plan.ts +121 -0
  144. package/src/workspace.ts +13 -11
  145. package/dist/src/stream.d.ts +0 -97
  146. package/dist/src/stream.d.ts.map +0 -1
  147. package/dist/src/stream.js +0 -134
  148. package/dist/src/stream.js.map +0 -1
  149. package/dist/src/stream.spec.d.ts.map +0 -1
  150. package/dist/src/stream.spec.js +0 -60
  151. package/dist/src/stream.spec.js.map +0 -1
  152. package/src/stream.spec.ts +0 -88
  153. package/src/stream.ts +0 -170
package/src/api.ts CHANGED
@@ -37,6 +37,8 @@ import {
37
37
 
38
38
  import { StructureType, TreePathType } from './structure.js';
39
39
  import { RunnerType } from './runner.js';
40
+ import { TaskBodyType, TaskInputType, TaskOutputType, TaskRoleType } from './task.js';
41
+ import { RequeueReasonType, StageUnitType } from './dataflow.js';
40
42
 
41
43
  // =============================================================================
42
44
  // Error Types
@@ -78,6 +80,13 @@ export const DatasetTypeMismatchErrorType = StructType({
78
80
  path: StringType,
79
81
  message: StringType,
80
82
  });
83
+ /** A name e3 would make a path of that cannot be one path segment: a
84
+ * repository's, a workspace's, or a package's name or version. */
85
+ export const InvalidNameErrorType = StructType({
86
+ kind: StringType,
87
+ name: StringType,
88
+ message: StringType,
89
+ });
81
90
 
82
91
  export const ErrorType = VariantType({
83
92
  repository_not_found: RepositoryNotFoundErrorType,
@@ -96,9 +105,8 @@ export const ErrorType = VariantType({
96
105
  dataflow_aborted: NullType,
97
106
  permission_denied: PermissionDeniedErrorType,
98
107
  internal: InternalErrorType,
99
- // Appended last: BEAST2 encodes a variant case by index, so a case added at
100
- // the end leaves every existing payload decodable.
101
108
  dataset_type_mismatch: DatasetTypeMismatchErrorType,
109
+ invalid_name: InvalidNameErrorType,
102
110
  });
103
111
 
104
112
  // =============================================================================
@@ -134,10 +142,16 @@ export const RepositoryStatusType = StructType({
134
142
  *
135
143
  * @property dryRun - If true, report what would be deleted without deleting
136
144
  * @property minAge - Minimum age in milliseconds for objects to be considered for deletion
145
+ * @property keepRuns - The runs of each workspace kept however old, the latest
146
+ * first; the server's default when `none`
147
+ * @property keepDays - The days of runs and executions kept however many; the
148
+ * server's default when `none`
137
149
  */
138
150
  export const GcRequestType = StructType({
139
151
  dryRun: BooleanType,
140
152
  minAge: OptionType(IntegerType),
153
+ keepRuns: OptionType(IntegerType),
154
+ keepDays: OptionType(IntegerType),
141
155
  });
142
156
 
143
157
  /**
@@ -148,6 +162,9 @@ export const GcRequestType = StructType({
148
162
  * @property retainedObjects - Number of objects still referenced
149
163
  * @property skippedYoung - Number of objects skipped due to minAge
150
164
  * @property bytesFreed - Total bytes freed by deletion
165
+ * @property deletedRuns - Number of dataflow run records deleted
166
+ * @property deletedExecutions - Number of execution attempts deleted, each with
167
+ * its owner record and logs
151
168
  */
152
169
  export const GcResultType = StructType({
153
170
  deletedObjects: IntegerType,
@@ -155,6 +172,8 @@ export const GcResultType = StructType({
155
172
  retainedObjects: IntegerType,
156
173
  skippedYoung: IntegerType,
157
174
  bytesFreed: IntegerType,
175
+ deletedRuns: IntegerType,
176
+ deletedExecutions: IntegerType,
158
177
  });
159
178
 
160
179
  // =============================================================================
@@ -269,13 +288,118 @@ export const WorkspaceInfoType = StructType({
269
288
  packageVersion: OptionType(StringType),
270
289
  });
271
290
 
291
+ /**
292
+ * What a deploy does with a record it cannot keep as it is.
293
+ *
294
+ * - `migrate`: run the migrations the workspace has not applied, and refuse a
295
+ * record no migration carries to the package's type
296
+ * - `fail`: run none, and refuse a record with migrations to run, for a
297
+ * workspace whose migrations go through their own change control
298
+ * - `reset`: reset a record it can neither keep nor migrate to the package's
299
+ * initial value, under a `$reset` commit, so the reset is in its history
300
+ */
301
+ export const SchemaPolicyType = VariantType({
302
+ migrate: NullType,
303
+ fail: NullType,
304
+ reset: NullType,
305
+ });
306
+
272
307
  /**
273
308
  * Request to deploy a package to a workspace.
274
309
  *
275
310
  * @property packageRef - Package reference in format "name" or "name@version"
311
+ * @property schema - What the deploy does with a record it cannot keep as it is
312
+ * @property allowDropRecords - Whether a record the package no longer declares
313
+ * may be dropped, with its state and history
314
+ * @property plan - Say what the deploy would do, and write nothing
276
315
  */
277
316
  export const WorkspaceDeployRequestType = StructType({
278
317
  packageRef: StringType,
318
+ schema: SchemaPolicyType,
319
+ allowDropRecords: BooleanType,
320
+ plan: BooleanType,
321
+ });
322
+
323
+ /**
324
+ * What a deploy decided for one record.
325
+ *
326
+ * @property record - The record's dataset ref path
327
+ * @property action - What the deploy does to it:
328
+ * - `mint`: the workspace does not hold it, so it is minted from the
329
+ * package's initial value
330
+ * - `keep`: kept as the workspace holds it; `deploy` when the package under
331
+ * it changed, which a `$deploy` commit records in its history
332
+ * - `migrate`: migrated by the steps the workspace has not applied, in order
333
+ * - `reset`: reset to the package's initial value under the `reset` policy,
334
+ * and why it could not be kept or migrated
335
+ * - `drop`: the package does not declare it, so it is dropped, with its
336
+ * state and history
337
+ * - `refused`: refused, and why, with the fix: the deploy writes nothing
338
+ */
339
+ export const RecordPlanType = StructType({
340
+ record: StringType,
341
+ action: VariantType({
342
+ mint: NullType,
343
+ keep: StructType({ deploy: BooleanType }),
344
+ migrate: StructType({ steps: ArrayType(StringType) }),
345
+ reset: StructType({ reason: StringType }),
346
+ drop: NullType,
347
+ refused: StructType({ reason: StringType }),
348
+ }),
349
+ });
350
+
351
+ /**
352
+ * What a deploy decided for one of a record's indexes.
353
+ *
354
+ * @property record - The record's dataset ref path
355
+ * @property index - The index's name
356
+ * @property action - `build` when the state names no index under the
357
+ * package's declaration, `drop` when it names one the package does not
358
+ * declare, and `keep` when the two agree and nothing runs
359
+ */
360
+ export const RecordIndexPlanType = StructType({
361
+ record: StringType,
362
+ index: StringType,
363
+ action: VariantType({
364
+ build: NullType,
365
+ drop: NullType,
366
+ keep: NullType,
367
+ }),
368
+ });
369
+
370
+ /**
371
+ * What a deploy did, or under `plan` would do.
372
+ *
373
+ * @property records - What it decided for each record
374
+ * @property indexes - What it decided for each index of the records it keeps
375
+ * @property warnings - The inputs it left unassigned, and why: a server never
376
+ * reads a `file` source, whose path is on the client's machine
377
+ */
378
+ export const WorkspaceDeployResultType = StructType({
379
+ records: ArrayType(RecordPlanType),
380
+ indexes: ArrayType(RecordIndexPlanType),
381
+ warnings: ArrayType(StringType),
382
+ });
383
+
384
+ /**
385
+ * A deploy job's progress: `pending` until it starts, then `deploying`.
386
+ */
387
+ export const WorkspaceDeployProgressType = VariantType({
388
+ pending: NullType,
389
+ deploying: NullType,
390
+ });
391
+
392
+ /**
393
+ * A deploy job's status.
394
+ *
395
+ * - `processing`: still running, and how far it has got
396
+ * - `completed`: what the deploy did
397
+ * - `failed`: why it did not
398
+ */
399
+ export const WorkspaceDeployStatusType = VariantType({
400
+ processing: WorkspaceDeployProgressType,
401
+ completed: WorkspaceDeployResultType,
402
+ failed: StructType({ message: StringType }),
279
403
  });
280
404
 
281
405
  /**
@@ -391,6 +515,9 @@ export const DatasetStatusInfoType = StructType({
391
515
  * @property inputs - Dataset paths this task reads from
392
516
  * @property output - Dataset path this task writes to
393
517
  * @property dependsOn - Names of tasks that must complete before this one
518
+ * @property peakBytes - The highest peak resident memory, in bytes, a runner
519
+ * of the execution the status comes from reached: the one the output came
520
+ * from, or the failure; `none` when it recorded none, or while it runs
394
521
  */
395
522
  export const TaskStatusInfoType = StructType({
396
523
  name: StringType,
@@ -399,6 +526,7 @@ export const TaskStatusInfoType = StructType({
399
526
  inputs: ArrayType(StringType),
400
527
  output: StringType,
401
528
  dependsOn: ArrayType(StringType),
529
+ peakBytes: OptionType(IntegerType),
402
530
  });
403
531
 
404
532
  /**
@@ -451,31 +579,34 @@ export const WorkspaceStatusResultType = StructType({
451
579
  *
452
580
  * @property name - Task name
453
581
  * @property hash - Task definition hash
454
- * @property kind - Task kind: "ui" (e3-ui `ui()`), other future kinds; none for plain tasks / old packages
582
+ * @property role - What the task's output is for: data, or a ui with what it
583
+ * binds
455
584
  */
456
585
  export const TaskListItemType = StructType({
457
586
  name: StringType,
458
587
  hash: StringType,
459
- kind: OptionType(StringType),
588
+ role: TaskRoleType,
460
589
  });
461
590
 
462
591
  /**
463
- * Detailed task information.
592
+ * Detailed task information: the task object's fields.
464
593
  *
465
594
  * @property name - Task name
466
595
  * @property hash - Task definition hash
467
- * @property commandIr - East IR for the task's command
468
- * @property inputs - Tree paths for task inputs
469
- * @property output - Tree path for task output
596
+ * @property body - What the task runs: an East program, or a command
597
+ * @property runner - The runtime it runs on
598
+ * @property inputs - The datasets it reads, each with its partition
599
+ * @property output - Where its output goes, and how it is made
600
+ * @property role - What its output is for: data, or a ui with what it binds
470
601
  */
471
602
  export const TaskDetailsType = StructType({
472
603
  name: StringType,
473
604
  hash: StringType,
474
- commandIr: StringType,
475
- inputs: ArrayType(TreePathType),
476
- output: TreePathType,
477
- kind: OptionType(StringType),
478
- metadata: OptionType(BlobType),
605
+ body: TaskBodyType,
606
+ runner: RunnerType,
607
+ inputs: ArrayType(TaskInputType),
608
+ output: TaskOutputType,
609
+ role: TaskRoleType,
479
610
  });
480
611
 
481
612
  // =============================================================================
@@ -483,14 +614,13 @@ export const TaskDetailsType = StructType({
483
614
  // =============================================================================
484
615
 
485
616
  /**
486
- * Request to start dataflow execution.
617
+ * Request to start dataflow execution. The run takes the server's budget of
618
+ * cores and memory, which it shares with everything else the server runs.
487
619
  *
488
- * @property concurrency - Maximum parallel tasks (default: 4)
489
620
  * @property force - Force re-execution of all tasks
490
621
  * @property filter - Filter to specific task names (glob pattern)
491
622
  */
492
623
  export const DataflowRequestType = StructType({
493
- concurrency: OptionType(IntegerType),
494
624
  force: BooleanType,
495
625
  filter: OptionType(StringType),
496
626
  });
@@ -561,11 +691,15 @@ export const DataflowResultType = StructType({
561
691
  * Dataflow event types for API polling.
562
692
  *
563
693
  * - `start`: Task started executing
564
- * - `complete`: Task executed and succeeded
694
+ * - `complete`: Task executed and succeeded, with the highest peak memory its
695
+ * runners reached when they reported one
565
696
  * - `cached`: Task result retrieved from cache (no execution)
566
697
  * - `failed`: Task exited with non-zero code
567
698
  * - `error`: Internal error during task execution
568
699
  * - `input_unavailable`: Task couldn't run because inputs not available
700
+ * - `requeued`: A unit of a split task was stopped — by the budget's guard, or
701
+ * its cgroup's cap — and runs again once the memory it reserves fits: the
702
+ * unit, why, the most it was measured using and what it reserves, in bytes
569
703
  */
570
704
  export const DataflowEventType = VariantType({
571
705
  start: StructType({
@@ -576,6 +710,7 @@ export const DataflowEventType = VariantType({
576
710
  task: StringType,
577
711
  timestamp: StringType,
578
712
  duration: FloatType,
713
+ peakBytes: OptionType(IntegerType),
579
714
  }),
580
715
  cached: StructType({
581
716
  task: StringType,
@@ -597,6 +732,14 @@ export const DataflowEventType = VariantType({
597
732
  timestamp: StringType,
598
733
  reason: StringType,
599
734
  }),
735
+ requeued: StructType({
736
+ task: StringType,
737
+ timestamp: StringType,
738
+ unit: StageUnitType,
739
+ reason: RequeueReasonType,
740
+ peak: IntegerType,
741
+ reserves: IntegerType,
742
+ }),
600
743
  });
601
744
 
602
745
  /**
@@ -628,6 +771,55 @@ export const DataflowExecutionSummaryType = StructType({
628
771
  duration: FloatType,
629
772
  });
630
773
 
774
+ /**
775
+ * A server's budget of cores and memory, which every runner it spawns takes
776
+ * from, and what its runners hold of it now.
777
+ *
778
+ * @property cores - Runner processes at once
779
+ * @property memory - Bytes of memory the runners may hold between them
780
+ * @property coresInUse - Runner processes holding the budget now
781
+ * @property memoryInUse - Bytes the runners hold now, each at the larger of
782
+ * its reservation and what it was last measured using
783
+ */
784
+ export const DataflowBudgetType = StructType({
785
+ cores: IntegerType,
786
+ memory: IntegerType,
787
+ coresInUse: IntegerType,
788
+ memoryInUse: IntegerType,
789
+ });
790
+
791
+ /**
792
+ * A task, or a unit of a split task, waiting for room in the server's budget.
793
+ *
794
+ * @property task - The task
795
+ * @property unit - The unit, for a split task's; `none` for a task run as one
796
+ * @property needs - The memory, in bytes, it waits to reserve; 0 when it
797
+ * waits for a core alone
798
+ * @property since - ISO timestamp when it began waiting
799
+ */
800
+ export const UnitWaitType = StructType({
801
+ task: StringType,
802
+ unit: OptionType(StageUnitType),
803
+ needs: IntegerType,
804
+ since: StringType,
805
+ });
806
+
807
+ /**
808
+ * A split task's progress through the stage it is in.
809
+ *
810
+ * @property task - The task
811
+ * @property merge - The merge level, from 1, and the levels the merges take;
812
+ * `none` while its pieces run
813
+ * @property done - The stage's units that have finished
814
+ * @property units - The stage's units
815
+ */
816
+ export const SplitProgressType = StructType({
817
+ task: StringType,
818
+ merge: OptionType(StructType({ level: IntegerType, levels: IntegerType })),
819
+ done: IntegerType,
820
+ units: IntegerType,
821
+ });
822
+
631
823
  /**
632
824
  * Dataflow execution state returned by API polling.
633
825
  *
@@ -643,6 +835,12 @@ export const DataflowExecutionSummaryType = StructType({
643
835
  * @property summary - Execution summary (available when complete)
644
836
  * @property events - Task events (may be paginated via offset/limit)
645
837
  * @property totalEvents - Total number of events (for pagination)
838
+ * @property budget - The server's budget now, where it has one: a server whose
839
+ * runners hold none, as a remote backend's, serves `none`
840
+ * @property waiting - The tasks and units of the run waiting for room, while it
841
+ * runs; nothing stores them
842
+ * @property splits - Each split task's progress, while the run runs; nothing
843
+ * stores it
646
844
  */
647
845
  export const ApiDataflowExecutionStateType = StructType({
648
846
  status: ApiExecutionStatusType,
@@ -651,6 +849,9 @@ export const ApiDataflowExecutionStateType = StructType({
651
849
  summary: OptionType(DataflowExecutionSummaryType),
652
850
  events: ArrayType(DataflowEventType),
653
851
  totalEvents: IntegerType,
852
+ budget: OptionType(DataflowBudgetType),
853
+ waiting: ArrayType(UnitWaitType),
854
+ splits: ArrayType(SplitProgressType),
654
855
  });
655
856
 
656
857
  // =============================================================================
@@ -658,13 +859,17 @@ export const ApiDataflowExecutionStateType = StructType({
658
859
  // =============================================================================
659
860
 
660
861
  /**
661
- * Execution status for history listing.
862
+ * Execution status for history listing: `cancelled` when e3 stopped the
863
+ * execution because its run was aborted, `interrupted` when the orchestrator
864
+ * that owned it exited before it finished.
662
865
  */
663
866
  export const ExecutionHistoryStatusType = VariantType({
664
867
  running: NullType,
665
868
  success: NullType,
666
869
  failed: NullType,
667
870
  error: NullType,
871
+ cancelled: NullType,
872
+ interrupted: NullType,
668
873
  });
669
874
 
670
875
  /**
@@ -677,6 +882,8 @@ export const ExecutionHistoryStatusType = VariantType({
677
882
  * @property completedAt - ISO timestamp when execution finished (if done)
678
883
  * @property duration - Execution duration in milliseconds (if done)
679
884
  * @property exitCode - Process exit code (if failed)
885
+ * @property peakBytes - The highest peak resident memory, in bytes, a runner
886
+ * of the execution reached, when it succeeded or failed and one reported it
680
887
  */
681
888
  export const ExecutionListItemType = StructType({
682
889
  inputsHash: StringType,
@@ -686,6 +893,7 @@ export const ExecutionListItemType = StructType({
686
893
  completedAt: OptionType(StringType),
687
894
  duration: OptionType(IntegerType),
688
895
  exitCode: OptionType(IntegerType),
896
+ peakBytes: OptionType(IntegerType),
689
897
  });
690
898
 
691
899
  // =============================================================================
@@ -703,7 +911,9 @@ export const TreeKindType = VariantType({ struct: NullType });
703
911
  * A list entry -- either a dataset leaf or a tree branch.
704
912
  *
705
913
  * Used by the `?list=true&status=true` endpoints to return both
706
- * tree structure entries and dataset leaves in a single flat list.
914
+ * tree structure entries and dataset leaves in a single flat list. A dataset's
915
+ * `size` is what its value weighs in the store, as its status reports it: for
916
+ * a collection, its segments and its manifest.
707
917
  */
708
918
  export const ListEntryType = VariantType({
709
919
  dataset: StructType({
@@ -729,7 +939,8 @@ export const ListEntryType = VariantType({
729
939
  * @property type - East type of the dataset
730
940
  * @property refType - Ref type: "unassigned", "null", or "value"
731
941
  * @property hash - Object hash (None if unassigned/null)
732
- * @property size - Size in bytes (None if unassigned)
942
+ * @property size - Bytes the value weighs in the store: for a collection, its
943
+ * segments and its manifest (None if unassigned)
733
944
  */
734
945
  export const DatasetStatusDetailType = StructType({
735
946
  path: StringType,
@@ -737,10 +948,9 @@ export const DatasetStatusDetailType = StructType({
737
948
  refType: StringType,
738
949
  hash: OptionType(StringType),
739
950
  size: OptionType(IntegerType),
740
- /** Segment and element counts of a stored collection, read from the blob's
741
- * trailing index — so a re-pointed input is inspectable without decoding
742
- * it. `none` for a non-collection or an unset dataset. Appended LAST, per
743
- * the positional struct rule. */
951
+ /** Segment and element counts of a stored collection, read from its
952
+ * manifest — so a re-pointed input is inspectable without decoding it.
953
+ * `none` for a non-collection or an unset dataset. */
744
954
  segments: OptionType(IntegerType),
745
955
  rows: OptionType(IntegerType),
746
956
  });
@@ -834,6 +1044,10 @@ export const OneShotRequestType = StructType({
834
1044
  /**
835
1045
  * A record mutation call. Positional args (after the implicit current state),
836
1046
  * one beast2-encoded value per declared parameter.
1047
+ *
1048
+ * Of the limits, `timeoutMs` bounds each run of the mutation's program and
1049
+ * `maxLogBytes` the stderr a failure returns. `maxResultBytes` does not apply:
1050
+ * a mutation's output is stored as segments and never read whole.
837
1051
  */
838
1052
  export const MutationCallRequestType = StructType({
839
1053
  args: ArrayType(BlobType),
@@ -846,32 +1060,49 @@ export const MutationCallRequestType = StructType({
846
1060
  *
847
1061
  * - `committed`: the new commit + state hashes
848
1062
  * - `invalid`: record/mutation lookup or arity error; nothing ran
849
- * - `failed`: the reducer process exited non-zero (incl. a reducer `$.error`; see stderr)
850
- * - `too_large`: the new state exceeded the result cap
851
- * - `timed_out`: the reducer exceeded its time budget
852
- * - `conflict`: the compare-and-swap lost the race `attempts` times
1063
+ * - `failed`: the mutation's program failed (incl. a body's `$.error`; see stderr)
1064
+ * - `timed_out`: the program exceeded its time budget
1065
+ * - `conflict`: the compare-and-swap lost the race `attempts` times, or the
1066
+ * write disagreed with the state it landed on — then `detail` names the key,
1067
+ * and resubmitting the same write cannot help
853
1068
  */
854
1069
  export const MutationResultType = StructType({
855
1070
  outcome: VariantType({
856
1071
  committed: StructType({ commitHash: StringType, stateHash: StringType }),
857
1072
  invalid: StructType({ message: StringType }),
858
1073
  failed: StructType({ exitCode: IntegerType, stderr: StringType }),
859
- too_large: StructType({ bytes: IntegerType, limit: IntegerType, stderr: StringType }),
860
1074
  timed_out: StructType({ ms: IntegerType, stderr: StringType }),
861
- conflict: StructType({ attempts: IntegerType }),
1075
+ conflict: StructType({ attempts: IntegerType, detail: OptionType(StringType) }),
862
1076
  }),
863
1077
  });
864
1078
 
865
1079
  /**
866
1080
  * A record's callable surface, returned by `describe` so dynamic callers can
867
- * encode mutation arguments. The mutation `argTypes` are the EXTRA parameters
868
- * after the implicit current state.
1081
+ * encode mutation arguments and read through its indexes. The mutation
1082
+ * `argTypes` are the EXTRA parameters after the implicit current state.
869
1083
  */
870
1084
  export const RecordSignatureType = StructType({
871
1085
  name: StringType,
872
1086
  mutations: ArrayType(StructType({
873
1087
  name: StringType,
874
1088
  argTypes: ArrayType(EastTypeType),
1089
+ /** The write form — `reduce`, `edit` or `patch` — which says what the
1090
+ * arguments MEAN: a `patch` mutation's one argument is a
1091
+ * `PatchType(State)`, not a value of the record's own type. */
1092
+ form: StringType,
1093
+ })),
1094
+ /** The indexes the record declares. A page read through one is decoded by
1095
+ * `indexWindowType(K, keyType, valueType, V)` over the record's
1096
+ * `Dict<K, V>`, and a key search through one takes a `keyType` key. */
1097
+ indexes: ArrayType(StructType({
1098
+ name: StringType,
1099
+ /** The index key's type. */
1100
+ keyType: EastTypeType,
1101
+ /** The covering projection's type: `Null` when the index projects
1102
+ * nothing, and a read joins the rows instead. */
1103
+ valueType: EastTypeType,
1104
+ /** Whether a row may hold several index keys. */
1105
+ multi: BooleanType,
875
1106
  })),
876
1107
  });
877
1108
 
@@ -883,6 +1114,9 @@ export const RecordCommitInfoType = StructType({
883
1114
  mutation: StringType,
884
1115
  actor: StringType,
885
1116
  at: DateTimeType,
1117
+ /** The delta this commit applied, when it wrote one — so history shows WHAT
1118
+ * changed without diffing two states. */
1119
+ delta: OptionType(StringType),
886
1120
  });
887
1121
 
888
1122
  /** A page of a record's commit history, newest first. */
@@ -907,6 +1141,13 @@ export type PackageDetails = ValueTypeOf<typeof PackageDetailsType>;
907
1141
  export type WorkspaceInfo = ValueTypeOf<typeof WorkspaceInfoType>;
908
1142
  export type WorkspaceCreateRequest = ValueTypeOf<typeof WorkspaceCreateRequestType>;
909
1143
  export type WorkspaceDeployRequest = ValueTypeOf<typeof WorkspaceDeployRequestType>;
1144
+ /** A {@link SchemaPolicyType} by its name, as a deploy's options take it. */
1145
+ export type SchemaPolicy = ValueTypeOf<typeof SchemaPolicyType>['type'];
1146
+ export type RecordPlan = ValueTypeOf<typeof RecordPlanType>;
1147
+ export type RecordIndexPlan = ValueTypeOf<typeof RecordIndexPlanType>;
1148
+ export type WorkspaceDeployResult = ValueTypeOf<typeof WorkspaceDeployResultType>;
1149
+ export type WorkspaceDeployProgress = ValueTypeOf<typeof WorkspaceDeployProgressType>;
1150
+ export type WorkspaceDeployStatus = ValueTypeOf<typeof WorkspaceDeployStatusType>;
910
1151
  export type DatasetStatus = ValueTypeOf<typeof DatasetStatusType>;
911
1152
  export type TaskStatus = ValueTypeOf<typeof TaskStatusType>;
912
1153
  export type DatasetStatusInfo = ValueTypeOf<typeof DatasetStatusInfoType>;
@@ -922,6 +1163,9 @@ export type DataflowResult = ValueTypeOf<typeof DataflowResultType>;
922
1163
  export type DataflowEvent = ValueTypeOf<typeof DataflowEventType>;
923
1164
  export type ApiExecutionStatus = ValueTypeOf<typeof ApiExecutionStatusType>;
924
1165
  export type DataflowExecutionSummary = ValueTypeOf<typeof DataflowExecutionSummaryType>;
1166
+ export type DataflowBudget = ValueTypeOf<typeof DataflowBudgetType>;
1167
+ export type UnitWait = ValueTypeOf<typeof UnitWaitType>;
1168
+ export type SplitProgress = ValueTypeOf<typeof SplitProgressType>;
925
1169
  export type ApiDataflowExecutionState = ValueTypeOf<typeof ApiDataflowExecutionStateType>;
926
1170
  export type ExecutionHistoryStatus = ValueTypeOf<typeof ExecutionHistoryStatusType>;
927
1171
  export type ExecutionListItem = ValueTypeOf<typeof ExecutionListItemType>;
@@ -0,0 +1,53 @@
1
+ /**
2
+ * Copyright (c) 2025 Elara AI Pty Ltd
3
+ * Dual-licensed under AGPL-3.0 and commercial license. See LICENSE for details.
4
+ */
5
+
6
+ /**
7
+ * The segment-object layout: a collection dataset is a **manifest naming
8
+ * standalone segment objects**, not one segmented blob.
9
+ *
10
+ * The type itself lives in `east`, beside the container it describes, because
11
+ * every runtime must read it: a task input staged as a manifest is one file
12
+ * naming sibling segment files, and a body that iterates it pays for the
13
+ * segments it touches rather than for a splice of the whole value. This module
14
+ * re-exports it for the e3 packages and adds what only a store cares about.
15
+ *
16
+ * Three properties are why records at millions of entries are affordable, and
17
+ * every one of them is about successive states SHARING storage rather than
18
+ * about indexing, which the trailing index already gave:
19
+ *
20
+ * - **A one-row edit costs one segment.** The new state names the same objects
21
+ * as the old one bar the segment that changed, so a commit's new storage is
22
+ * the touched segments plus a manifest — not the whole value.
23
+ * - **Two states diff in O(changed segments)**, by comparing entry hashes. No
24
+ * value is decoded to find out what moved, which is what index maintenance
25
+ * and the mutation delta are built on.
26
+ * - **A segment can be checked against its own hash**, which a ranged read of
27
+ * one blob can never offer: no digest exists for a byte range. No reader
28
+ * checks it — a segment is trusted as stored, like any other object.
29
+ *
30
+ * The manifest is the envelope the garbage collector recognizes, which is why
31
+ * its `kind` is a stored field rather than something inferred: an object whose
32
+ * entries name other objects MUST be traversable by shape, or the first sweep
33
+ * deletes a record's segments.
34
+ *
35
+ * @packageDocumentation
36
+ */
37
+
38
+ export {
39
+ COLLECTION_MANIFEST_KIND,
40
+ CollectionManifestType,
41
+ CollectionManifestEntryType,
42
+ type CollectionManifest,
43
+ type CollectionManifestEntry,
44
+ type Beast2ManifestSource,
45
+ isCollectionManifestType,
46
+ isCollectionManifest,
47
+ isBeast2ManifestSource,
48
+ encodeCollectionManifest,
49
+ decodeCollectionManifest,
50
+ readBeast2Manifest,
51
+ manifestElementCount,
52
+ manifestByteSize,
53
+ } from '@elaraai/east';