@elaraai/e3-types 1.0.78 → 1.0.80
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/src/api.d.ts +569 -36
- package/dist/src/api.d.ts.map +1 -1
- package/dist/src/api.js +257 -32
- package/dist/src/api.js.map +1 -1
- package/dist/src/collection-manifest.d.ts +37 -0
- package/dist/src/collection-manifest.d.ts.map +1 -0
- package/dist/src/collection-manifest.js +37 -0
- package/dist/src/collection-manifest.js.map +1 -0
- package/dist/src/dataflow.d.ts +264 -19
- package/dist/src/dataflow.d.ts.map +1 -1
- package/dist/src/dataflow.js +156 -15
- package/dist/src/dataflow.js.map +1 -1
- package/dist/src/dataflow.spec.js +184 -27
- package/dist/src/dataflow.spec.js.map +1 -1
- package/dist/src/dataset-blob.d.ts +75 -15
- package/dist/src/dataset-blob.d.ts.map +1 -1
- package/dist/src/dataset-blob.js +95 -28
- package/dist/src/dataset-blob.js.map +1 -1
- package/dist/src/dataset-blob.spec.d.ts +6 -0
- package/dist/src/dataset-blob.spec.d.ts.map +1 -0
- package/dist/src/dataset-blob.spec.js +92 -0
- package/dist/src/dataset-blob.spec.js.map +1 -0
- package/dist/src/dataset-ref.d.ts +5 -4
- package/dist/src/dataset-ref.d.ts.map +1 -1
- package/dist/src/dataset-ref.js +5 -4
- package/dist/src/dataset-ref.js.map +1 -1
- package/dist/src/environment.d.ts +3 -2
- package/dist/src/environment.d.ts.map +1 -1
- package/dist/src/environment.js +3 -2
- package/dist/src/environment.js.map +1 -1
- package/dist/src/execution.d.ts +112 -18
- package/dist/src/execution.d.ts.map +1 -1
- package/dist/src/execution.js +174 -45
- package/dist/src/execution.js.map +1 -1
- package/dist/src/execution.spec.d.ts +6 -0
- package/dist/src/execution.spec.d.ts.map +1 -0
- package/dist/src/execution.spec.js +108 -0
- package/dist/src/execution.spec.js.map +1 -0
- package/dist/src/function.d.ts +14 -12
- package/dist/src/function.d.ts.map +1 -1
- package/dist/src/function.js +17 -32
- package/dist/src/function.js.map +1 -1
- package/dist/src/index.d.ts +16 -12
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/index.js +25 -20
- package/dist/src/index.js.map +1 -1
- package/dist/src/lock.d.ts +68 -22
- package/dist/src/lock.d.ts.map +1 -1
- package/dist/src/lock.js +33 -22
- package/dist/src/lock.js.map +1 -1
- package/dist/src/names.d.ts +21 -0
- package/dist/src/names.d.ts.map +1 -0
- package/dist/src/names.js +31 -0
- package/dist/src/names.js.map +1 -0
- package/dist/src/package.d.ts +11 -78
- package/dist/src/package.d.ts.map +1 -1
- package/dist/src/package.js +13 -66
- package/dist/src/package.js.map +1 -1
- package/dist/src/record.d.ts +582 -18
- package/dist/src/record.d.ts.map +1 -1
- package/dist/src/record.js +405 -7
- package/dist/src/record.js.map +1 -1
- package/dist/src/{stream.spec.d.ts → record.spec.d.ts} +1 -1
- package/dist/src/record.spec.d.ts.map +1 -0
- package/dist/src/record.spec.js +143 -0
- package/dist/src/record.spec.js.map +1 -0
- package/dist/src/release.d.ts +36 -0
- package/dist/src/release.d.ts.map +1 -0
- package/dist/src/release.js +71 -0
- package/dist/src/release.js.map +1 -0
- package/dist/src/release.spec.d.ts +6 -0
- package/dist/src/release.spec.d.ts.map +1 -0
- package/dist/src/release.spec.js +42 -0
- package/dist/src/release.spec.js.map +1 -0
- package/dist/src/repository.d.ts +64 -0
- package/dist/src/repository.d.ts.map +1 -0
- package/dist/src/repository.js +49 -0
- package/dist/src/repository.js.map +1 -0
- package/dist/src/runner.d.ts +16 -62
- package/dist/src/runner.d.ts.map +1 -1
- package/dist/src/runner.js +16 -79
- package/dist/src/runner.js.map +1 -1
- package/dist/src/runner.spec.js +11 -85
- package/dist/src/runner.spec.js.map +1 -1
- package/dist/src/structure.d.ts +0 -57
- package/dist/src/structure.d.ts.map +1 -1
- package/dist/src/structure.js +0 -3
- package/dist/src/structure.js.map +1 -1
- package/dist/src/task.d.ts +210 -318
- package/dist/src/task.d.ts.map +1 -1
- package/dist/src/task.js +117 -472
- package/dist/src/task.js.map +1 -1
- package/dist/src/task.spec.js +62 -154
- package/dist/src/task.spec.js.map +1 -1
- package/dist/src/transfer.d.ts +16 -22
- package/dist/src/transfer.d.ts.map +1 -1
- package/dist/src/transfer.js +16 -22
- package/dist/src/transfer.js.map +1 -1
- package/dist/src/transfer.spec.js +4 -34
- package/dist/src/transfer.spec.js.map +1 -1
- package/dist/src/unit-plan.d.ts +130 -0
- package/dist/src/unit-plan.d.ts.map +1 -0
- package/dist/src/unit-plan.js +107 -0
- package/dist/src/unit-plan.js.map +1 -0
- package/dist/src/unit-plan.spec.d.ts +6 -0
- package/dist/src/unit-plan.spec.d.ts.map +1 -0
- package/dist/src/unit-plan.spec.js +29 -0
- package/dist/src/unit-plan.spec.js.map +1 -0
- package/dist/src/workspace.d.ts +22 -11
- package/dist/src/workspace.d.ts.map +1 -1
- package/dist/src/workspace.js +10 -11
- package/dist/src/workspace.js.map +1 -1
- package/package.json +3 -3
- package/src/api.ts +276 -32
- package/src/collection-manifest.ts +53 -0
- package/src/dataflow.spec.ts +223 -31
- package/src/dataflow.ts +183 -26
- package/src/dataset-blob.spec.ts +117 -0
- package/src/dataset-blob.ts +157 -21
- package/src/dataset-ref.ts +5 -4
- package/src/environment.ts +3 -2
- package/src/execution.spec.ts +114 -0
- package/src/execution.ts +175 -51
- package/src/function.ts +19 -32
- package/src/index.ts +122 -39
- package/src/lock.ts +39 -22
- package/src/names.ts +39 -0
- package/src/package.ts +15 -71
- package/src/record.spec.ts +195 -0
- package/src/record.ts +489 -7
- package/src/release.spec.ts +47 -0
- package/src/release.ts +69 -0
- package/src/repository.ts +67 -0
- package/src/runner.spec.ts +11 -94
- package/src/runner.ts +16 -80
- package/src/structure.ts +0 -8
- package/src/task.spec.ts +67 -188
- package/src/task.ts +140 -522
- package/src/transfer.spec.ts +4 -51
- package/src/transfer.ts +16 -22
- package/src/unit-plan.spec.ts +31 -0
- package/src/unit-plan.ts +121 -0
- package/src/workspace.ts +13 -11
- package/dist/src/stream.d.ts +0 -97
- package/dist/src/stream.d.ts.map +0 -1
- package/dist/src/stream.js +0 -134
- package/dist/src/stream.js.map +0 -1
- package/dist/src/stream.spec.d.ts.map +0 -1
- package/dist/src/stream.spec.js +0 -60
- package/dist/src/stream.spec.js.map +0 -1
- package/src/stream.spec.ts +0 -88
- package/src/stream.ts +0 -170
package/src/api.ts
CHANGED
|
@@ -37,6 +37,8 @@ import {
|
|
|
37
37
|
|
|
38
38
|
import { StructureType, TreePathType } from './structure.js';
|
|
39
39
|
import { RunnerType } from './runner.js';
|
|
40
|
+
import { TaskBodyType, TaskInputType, TaskOutputType, TaskRoleType } from './task.js';
|
|
41
|
+
import { RequeueReasonType, StageUnitType } from './dataflow.js';
|
|
40
42
|
|
|
41
43
|
// =============================================================================
|
|
42
44
|
// Error Types
|
|
@@ -78,6 +80,13 @@ export const DatasetTypeMismatchErrorType = StructType({
|
|
|
78
80
|
path: StringType,
|
|
79
81
|
message: StringType,
|
|
80
82
|
});
|
|
83
|
+
/** A name e3 would make a path of that cannot be one path segment: a
|
|
84
|
+
* repository's, a workspace's, or a package's name or version. */
|
|
85
|
+
export const InvalidNameErrorType = StructType({
|
|
86
|
+
kind: StringType,
|
|
87
|
+
name: StringType,
|
|
88
|
+
message: StringType,
|
|
89
|
+
});
|
|
81
90
|
|
|
82
91
|
export const ErrorType = VariantType({
|
|
83
92
|
repository_not_found: RepositoryNotFoundErrorType,
|
|
@@ -96,9 +105,8 @@ export const ErrorType = VariantType({
|
|
|
96
105
|
dataflow_aborted: NullType,
|
|
97
106
|
permission_denied: PermissionDeniedErrorType,
|
|
98
107
|
internal: InternalErrorType,
|
|
99
|
-
// Appended last: BEAST2 encodes a variant case by index, so a case added at
|
|
100
|
-
// the end leaves every existing payload decodable.
|
|
101
108
|
dataset_type_mismatch: DatasetTypeMismatchErrorType,
|
|
109
|
+
invalid_name: InvalidNameErrorType,
|
|
102
110
|
});
|
|
103
111
|
|
|
104
112
|
// =============================================================================
|
|
@@ -134,10 +142,16 @@ export const RepositoryStatusType = StructType({
|
|
|
134
142
|
*
|
|
135
143
|
* @property dryRun - If true, report what would be deleted without deleting
|
|
136
144
|
* @property minAge - Minimum age in milliseconds for objects to be considered for deletion
|
|
145
|
+
* @property keepRuns - The runs of each workspace kept however old, the latest
|
|
146
|
+
* first; the server's default when `none`
|
|
147
|
+
* @property keepDays - The days of runs and executions kept however many; the
|
|
148
|
+
* server's default when `none`
|
|
137
149
|
*/
|
|
138
150
|
export const GcRequestType = StructType({
|
|
139
151
|
dryRun: BooleanType,
|
|
140
152
|
minAge: OptionType(IntegerType),
|
|
153
|
+
keepRuns: OptionType(IntegerType),
|
|
154
|
+
keepDays: OptionType(IntegerType),
|
|
141
155
|
});
|
|
142
156
|
|
|
143
157
|
/**
|
|
@@ -148,6 +162,9 @@ export const GcRequestType = StructType({
|
|
|
148
162
|
* @property retainedObjects - Number of objects still referenced
|
|
149
163
|
* @property skippedYoung - Number of objects skipped due to minAge
|
|
150
164
|
* @property bytesFreed - Total bytes freed by deletion
|
|
165
|
+
* @property deletedRuns - Number of dataflow run records deleted
|
|
166
|
+
* @property deletedExecutions - Number of execution attempts deleted, each with
|
|
167
|
+
* its owner record and logs
|
|
151
168
|
*/
|
|
152
169
|
export const GcResultType = StructType({
|
|
153
170
|
deletedObjects: IntegerType,
|
|
@@ -155,6 +172,8 @@ export const GcResultType = StructType({
|
|
|
155
172
|
retainedObjects: IntegerType,
|
|
156
173
|
skippedYoung: IntegerType,
|
|
157
174
|
bytesFreed: IntegerType,
|
|
175
|
+
deletedRuns: IntegerType,
|
|
176
|
+
deletedExecutions: IntegerType,
|
|
158
177
|
});
|
|
159
178
|
|
|
160
179
|
// =============================================================================
|
|
@@ -269,13 +288,118 @@ export const WorkspaceInfoType = StructType({
|
|
|
269
288
|
packageVersion: OptionType(StringType),
|
|
270
289
|
});
|
|
271
290
|
|
|
291
|
+
/**
|
|
292
|
+
* What a deploy does with a record it cannot keep as it is.
|
|
293
|
+
*
|
|
294
|
+
* - `migrate`: run the migrations the workspace has not applied, and refuse a
|
|
295
|
+
* record no migration carries to the package's type
|
|
296
|
+
* - `fail`: run none, and refuse a record with migrations to run, for a
|
|
297
|
+
* workspace whose migrations go through their own change control
|
|
298
|
+
* - `reset`: reset a record it can neither keep nor migrate to the package's
|
|
299
|
+
* initial value, under a `$reset` commit, so the reset is in its history
|
|
300
|
+
*/
|
|
301
|
+
export const SchemaPolicyType = VariantType({
|
|
302
|
+
migrate: NullType,
|
|
303
|
+
fail: NullType,
|
|
304
|
+
reset: NullType,
|
|
305
|
+
});
|
|
306
|
+
|
|
272
307
|
/**
|
|
273
308
|
* Request to deploy a package to a workspace.
|
|
274
309
|
*
|
|
275
310
|
* @property packageRef - Package reference in format "name" or "name@version"
|
|
311
|
+
* @property schema - What the deploy does with a record it cannot keep as it is
|
|
312
|
+
* @property allowDropRecords - Whether a record the package no longer declares
|
|
313
|
+
* may be dropped, with its state and history
|
|
314
|
+
* @property plan - Say what the deploy would do, and write nothing
|
|
276
315
|
*/
|
|
277
316
|
export const WorkspaceDeployRequestType = StructType({
|
|
278
317
|
packageRef: StringType,
|
|
318
|
+
schema: SchemaPolicyType,
|
|
319
|
+
allowDropRecords: BooleanType,
|
|
320
|
+
plan: BooleanType,
|
|
321
|
+
});
|
|
322
|
+
|
|
323
|
+
/**
|
|
324
|
+
* What a deploy decided for one record.
|
|
325
|
+
*
|
|
326
|
+
* @property record - The record's dataset ref path
|
|
327
|
+
* @property action - What the deploy does to it:
|
|
328
|
+
* - `mint`: the workspace does not hold it, so it is minted from the
|
|
329
|
+
* package's initial value
|
|
330
|
+
* - `keep`: kept as the workspace holds it; `deploy` when the package under
|
|
331
|
+
* it changed, which a `$deploy` commit records in its history
|
|
332
|
+
* - `migrate`: migrated by the steps the workspace has not applied, in order
|
|
333
|
+
* - `reset`: reset to the package's initial value under the `reset` policy,
|
|
334
|
+
* and why it could not be kept or migrated
|
|
335
|
+
* - `drop`: the package does not declare it, so it is dropped, with its
|
|
336
|
+
* state and history
|
|
337
|
+
* - `refused`: refused, and why, with the fix: the deploy writes nothing
|
|
338
|
+
*/
|
|
339
|
+
export const RecordPlanType = StructType({
|
|
340
|
+
record: StringType,
|
|
341
|
+
action: VariantType({
|
|
342
|
+
mint: NullType,
|
|
343
|
+
keep: StructType({ deploy: BooleanType }),
|
|
344
|
+
migrate: StructType({ steps: ArrayType(StringType) }),
|
|
345
|
+
reset: StructType({ reason: StringType }),
|
|
346
|
+
drop: NullType,
|
|
347
|
+
refused: StructType({ reason: StringType }),
|
|
348
|
+
}),
|
|
349
|
+
});
|
|
350
|
+
|
|
351
|
+
/**
|
|
352
|
+
* What a deploy decided for one of a record's indexes.
|
|
353
|
+
*
|
|
354
|
+
* @property record - The record's dataset ref path
|
|
355
|
+
* @property index - The index's name
|
|
356
|
+
* @property action - `build` when the state names no index under the
|
|
357
|
+
* package's declaration, `drop` when it names one the package does not
|
|
358
|
+
* declare, and `keep` when the two agree and nothing runs
|
|
359
|
+
*/
|
|
360
|
+
export const RecordIndexPlanType = StructType({
|
|
361
|
+
record: StringType,
|
|
362
|
+
index: StringType,
|
|
363
|
+
action: VariantType({
|
|
364
|
+
build: NullType,
|
|
365
|
+
drop: NullType,
|
|
366
|
+
keep: NullType,
|
|
367
|
+
}),
|
|
368
|
+
});
|
|
369
|
+
|
|
370
|
+
/**
|
|
371
|
+
* What a deploy did, or under `plan` would do.
|
|
372
|
+
*
|
|
373
|
+
* @property records - What it decided for each record
|
|
374
|
+
* @property indexes - What it decided for each index of the records it keeps
|
|
375
|
+
* @property warnings - The inputs it left unassigned, and why: a server never
|
|
376
|
+
* reads a `file` source, whose path is on the client's machine
|
|
377
|
+
*/
|
|
378
|
+
export const WorkspaceDeployResultType = StructType({
|
|
379
|
+
records: ArrayType(RecordPlanType),
|
|
380
|
+
indexes: ArrayType(RecordIndexPlanType),
|
|
381
|
+
warnings: ArrayType(StringType),
|
|
382
|
+
});
|
|
383
|
+
|
|
384
|
+
/**
|
|
385
|
+
* A deploy job's progress: `pending` until it starts, then `deploying`.
|
|
386
|
+
*/
|
|
387
|
+
export const WorkspaceDeployProgressType = VariantType({
|
|
388
|
+
pending: NullType,
|
|
389
|
+
deploying: NullType,
|
|
390
|
+
});
|
|
391
|
+
|
|
392
|
+
/**
|
|
393
|
+
* A deploy job's status.
|
|
394
|
+
*
|
|
395
|
+
* - `processing`: still running, and how far it has got
|
|
396
|
+
* - `completed`: what the deploy did
|
|
397
|
+
* - `failed`: why it did not
|
|
398
|
+
*/
|
|
399
|
+
export const WorkspaceDeployStatusType = VariantType({
|
|
400
|
+
processing: WorkspaceDeployProgressType,
|
|
401
|
+
completed: WorkspaceDeployResultType,
|
|
402
|
+
failed: StructType({ message: StringType }),
|
|
279
403
|
});
|
|
280
404
|
|
|
281
405
|
/**
|
|
@@ -391,6 +515,9 @@ export const DatasetStatusInfoType = StructType({
|
|
|
391
515
|
* @property inputs - Dataset paths this task reads from
|
|
392
516
|
* @property output - Dataset path this task writes to
|
|
393
517
|
* @property dependsOn - Names of tasks that must complete before this one
|
|
518
|
+
* @property peakBytes - The highest peak resident memory, in bytes, a runner
|
|
519
|
+
* of the execution the status comes from reached: the one the output came
|
|
520
|
+
* from, or the failure; `none` when it recorded none, or while it runs
|
|
394
521
|
*/
|
|
395
522
|
export const TaskStatusInfoType = StructType({
|
|
396
523
|
name: StringType,
|
|
@@ -399,6 +526,7 @@ export const TaskStatusInfoType = StructType({
|
|
|
399
526
|
inputs: ArrayType(StringType),
|
|
400
527
|
output: StringType,
|
|
401
528
|
dependsOn: ArrayType(StringType),
|
|
529
|
+
peakBytes: OptionType(IntegerType),
|
|
402
530
|
});
|
|
403
531
|
|
|
404
532
|
/**
|
|
@@ -451,31 +579,34 @@ export const WorkspaceStatusResultType = StructType({
|
|
|
451
579
|
*
|
|
452
580
|
* @property name - Task name
|
|
453
581
|
* @property hash - Task definition hash
|
|
454
|
-
* @property
|
|
582
|
+
* @property role - What the task's output is for: data, or a ui with what it
|
|
583
|
+
* binds
|
|
455
584
|
*/
|
|
456
585
|
export const TaskListItemType = StructType({
|
|
457
586
|
name: StringType,
|
|
458
587
|
hash: StringType,
|
|
459
|
-
|
|
588
|
+
role: TaskRoleType,
|
|
460
589
|
});
|
|
461
590
|
|
|
462
591
|
/**
|
|
463
|
-
* Detailed task information.
|
|
592
|
+
* Detailed task information: the task object's fields.
|
|
464
593
|
*
|
|
465
594
|
* @property name - Task name
|
|
466
595
|
* @property hash - Task definition hash
|
|
467
|
-
* @property
|
|
468
|
-
* @property
|
|
469
|
-
* @property
|
|
596
|
+
* @property body - What the task runs: an East program, or a command
|
|
597
|
+
* @property runner - The runtime it runs on
|
|
598
|
+
* @property inputs - The datasets it reads, each with its partition
|
|
599
|
+
* @property output - Where its output goes, and how it is made
|
|
600
|
+
* @property role - What its output is for: data, or a ui with what it binds
|
|
470
601
|
*/
|
|
471
602
|
export const TaskDetailsType = StructType({
|
|
472
603
|
name: StringType,
|
|
473
604
|
hash: StringType,
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
605
|
+
body: TaskBodyType,
|
|
606
|
+
runner: RunnerType,
|
|
607
|
+
inputs: ArrayType(TaskInputType),
|
|
608
|
+
output: TaskOutputType,
|
|
609
|
+
role: TaskRoleType,
|
|
479
610
|
});
|
|
480
611
|
|
|
481
612
|
// =============================================================================
|
|
@@ -483,14 +614,13 @@ export const TaskDetailsType = StructType({
|
|
|
483
614
|
// =============================================================================
|
|
484
615
|
|
|
485
616
|
/**
|
|
486
|
-
* Request to start dataflow execution.
|
|
617
|
+
* Request to start dataflow execution. The run takes the server's budget of
|
|
618
|
+
* cores and memory, which it shares with everything else the server runs.
|
|
487
619
|
*
|
|
488
|
-
* @property concurrency - Maximum parallel tasks (default: 4)
|
|
489
620
|
* @property force - Force re-execution of all tasks
|
|
490
621
|
* @property filter - Filter to specific task names (glob pattern)
|
|
491
622
|
*/
|
|
492
623
|
export const DataflowRequestType = StructType({
|
|
493
|
-
concurrency: OptionType(IntegerType),
|
|
494
624
|
force: BooleanType,
|
|
495
625
|
filter: OptionType(StringType),
|
|
496
626
|
});
|
|
@@ -561,11 +691,15 @@ export const DataflowResultType = StructType({
|
|
|
561
691
|
* Dataflow event types for API polling.
|
|
562
692
|
*
|
|
563
693
|
* - `start`: Task started executing
|
|
564
|
-
* - `complete`: Task executed and succeeded
|
|
694
|
+
* - `complete`: Task executed and succeeded, with the highest peak memory its
|
|
695
|
+
* runners reached when they reported one
|
|
565
696
|
* - `cached`: Task result retrieved from cache (no execution)
|
|
566
697
|
* - `failed`: Task exited with non-zero code
|
|
567
698
|
* - `error`: Internal error during task execution
|
|
568
699
|
* - `input_unavailable`: Task couldn't run because inputs not available
|
|
700
|
+
* - `requeued`: A unit of a split task was stopped — by the budget's guard, or
|
|
701
|
+
* its cgroup's cap — and runs again once the memory it reserves fits: the
|
|
702
|
+
* unit, why, the most it was measured using and what it reserves, in bytes
|
|
569
703
|
*/
|
|
570
704
|
export const DataflowEventType = VariantType({
|
|
571
705
|
start: StructType({
|
|
@@ -576,6 +710,7 @@ export const DataflowEventType = VariantType({
|
|
|
576
710
|
task: StringType,
|
|
577
711
|
timestamp: StringType,
|
|
578
712
|
duration: FloatType,
|
|
713
|
+
peakBytes: OptionType(IntegerType),
|
|
579
714
|
}),
|
|
580
715
|
cached: StructType({
|
|
581
716
|
task: StringType,
|
|
@@ -597,6 +732,14 @@ export const DataflowEventType = VariantType({
|
|
|
597
732
|
timestamp: StringType,
|
|
598
733
|
reason: StringType,
|
|
599
734
|
}),
|
|
735
|
+
requeued: StructType({
|
|
736
|
+
task: StringType,
|
|
737
|
+
timestamp: StringType,
|
|
738
|
+
unit: StageUnitType,
|
|
739
|
+
reason: RequeueReasonType,
|
|
740
|
+
peak: IntegerType,
|
|
741
|
+
reserves: IntegerType,
|
|
742
|
+
}),
|
|
600
743
|
});
|
|
601
744
|
|
|
602
745
|
/**
|
|
@@ -628,6 +771,55 @@ export const DataflowExecutionSummaryType = StructType({
|
|
|
628
771
|
duration: FloatType,
|
|
629
772
|
});
|
|
630
773
|
|
|
774
|
+
/**
|
|
775
|
+
* A server's budget of cores and memory, which every runner it spawns takes
|
|
776
|
+
* from, and what its runners hold of it now.
|
|
777
|
+
*
|
|
778
|
+
* @property cores - Runner processes at once
|
|
779
|
+
* @property memory - Bytes of memory the runners may hold between them
|
|
780
|
+
* @property coresInUse - Runner processes holding the budget now
|
|
781
|
+
* @property memoryInUse - Bytes the runners hold now, each at the larger of
|
|
782
|
+
* its reservation and what it was last measured using
|
|
783
|
+
*/
|
|
784
|
+
export const DataflowBudgetType = StructType({
|
|
785
|
+
cores: IntegerType,
|
|
786
|
+
memory: IntegerType,
|
|
787
|
+
coresInUse: IntegerType,
|
|
788
|
+
memoryInUse: IntegerType,
|
|
789
|
+
});
|
|
790
|
+
|
|
791
|
+
/**
|
|
792
|
+
* A task, or a unit of a split task, waiting for room in the server's budget.
|
|
793
|
+
*
|
|
794
|
+
* @property task - The task
|
|
795
|
+
* @property unit - The unit, for a split task's; `none` for a task run as one
|
|
796
|
+
* @property needs - The memory, in bytes, it waits to reserve; 0 when it
|
|
797
|
+
* waits for a core alone
|
|
798
|
+
* @property since - ISO timestamp when it began waiting
|
|
799
|
+
*/
|
|
800
|
+
export const UnitWaitType = StructType({
|
|
801
|
+
task: StringType,
|
|
802
|
+
unit: OptionType(StageUnitType),
|
|
803
|
+
needs: IntegerType,
|
|
804
|
+
since: StringType,
|
|
805
|
+
});
|
|
806
|
+
|
|
807
|
+
/**
|
|
808
|
+
* A split task's progress through the stage it is in.
|
|
809
|
+
*
|
|
810
|
+
* @property task - The task
|
|
811
|
+
* @property merge - The merge level, from 1, and the levels the merges take;
|
|
812
|
+
* `none` while its pieces run
|
|
813
|
+
* @property done - The stage's units that have finished
|
|
814
|
+
* @property units - The stage's units
|
|
815
|
+
*/
|
|
816
|
+
export const SplitProgressType = StructType({
|
|
817
|
+
task: StringType,
|
|
818
|
+
merge: OptionType(StructType({ level: IntegerType, levels: IntegerType })),
|
|
819
|
+
done: IntegerType,
|
|
820
|
+
units: IntegerType,
|
|
821
|
+
});
|
|
822
|
+
|
|
631
823
|
/**
|
|
632
824
|
* Dataflow execution state returned by API polling.
|
|
633
825
|
*
|
|
@@ -643,6 +835,12 @@ export const DataflowExecutionSummaryType = StructType({
|
|
|
643
835
|
* @property summary - Execution summary (available when complete)
|
|
644
836
|
* @property events - Task events (may be paginated via offset/limit)
|
|
645
837
|
* @property totalEvents - Total number of events (for pagination)
|
|
838
|
+
* @property budget - The server's budget now, where it has one: a server whose
|
|
839
|
+
* runners hold none, as a remote backend's, serves `none`
|
|
840
|
+
* @property waiting - The tasks and units of the run waiting for room, while it
|
|
841
|
+
* runs; nothing stores them
|
|
842
|
+
* @property splits - Each split task's progress, while the run runs; nothing
|
|
843
|
+
* stores it
|
|
646
844
|
*/
|
|
647
845
|
export const ApiDataflowExecutionStateType = StructType({
|
|
648
846
|
status: ApiExecutionStatusType,
|
|
@@ -651,6 +849,9 @@ export const ApiDataflowExecutionStateType = StructType({
|
|
|
651
849
|
summary: OptionType(DataflowExecutionSummaryType),
|
|
652
850
|
events: ArrayType(DataflowEventType),
|
|
653
851
|
totalEvents: IntegerType,
|
|
852
|
+
budget: OptionType(DataflowBudgetType),
|
|
853
|
+
waiting: ArrayType(UnitWaitType),
|
|
854
|
+
splits: ArrayType(SplitProgressType),
|
|
654
855
|
});
|
|
655
856
|
|
|
656
857
|
// =============================================================================
|
|
@@ -658,13 +859,17 @@ export const ApiDataflowExecutionStateType = StructType({
|
|
|
658
859
|
// =============================================================================
|
|
659
860
|
|
|
660
861
|
/**
|
|
661
|
-
* Execution status for history listing
|
|
862
|
+
* Execution status for history listing: `cancelled` when e3 stopped the
|
|
863
|
+
* execution because its run was aborted, `interrupted` when the orchestrator
|
|
864
|
+
* that owned it exited before it finished.
|
|
662
865
|
*/
|
|
663
866
|
export const ExecutionHistoryStatusType = VariantType({
|
|
664
867
|
running: NullType,
|
|
665
868
|
success: NullType,
|
|
666
869
|
failed: NullType,
|
|
667
870
|
error: NullType,
|
|
871
|
+
cancelled: NullType,
|
|
872
|
+
interrupted: NullType,
|
|
668
873
|
});
|
|
669
874
|
|
|
670
875
|
/**
|
|
@@ -677,6 +882,8 @@ export const ExecutionHistoryStatusType = VariantType({
|
|
|
677
882
|
* @property completedAt - ISO timestamp when execution finished (if done)
|
|
678
883
|
* @property duration - Execution duration in milliseconds (if done)
|
|
679
884
|
* @property exitCode - Process exit code (if failed)
|
|
885
|
+
* @property peakBytes - The highest peak resident memory, in bytes, a runner
|
|
886
|
+
* of the execution reached, when it succeeded or failed and one reported it
|
|
680
887
|
*/
|
|
681
888
|
export const ExecutionListItemType = StructType({
|
|
682
889
|
inputsHash: StringType,
|
|
@@ -686,6 +893,7 @@ export const ExecutionListItemType = StructType({
|
|
|
686
893
|
completedAt: OptionType(StringType),
|
|
687
894
|
duration: OptionType(IntegerType),
|
|
688
895
|
exitCode: OptionType(IntegerType),
|
|
896
|
+
peakBytes: OptionType(IntegerType),
|
|
689
897
|
});
|
|
690
898
|
|
|
691
899
|
// =============================================================================
|
|
@@ -703,7 +911,9 @@ export const TreeKindType = VariantType({ struct: NullType });
|
|
|
703
911
|
* A list entry -- either a dataset leaf or a tree branch.
|
|
704
912
|
*
|
|
705
913
|
* Used by the `?list=true&status=true` endpoints to return both
|
|
706
|
-
* tree structure entries and dataset leaves in a single flat list.
|
|
914
|
+
* tree structure entries and dataset leaves in a single flat list. A dataset's
|
|
915
|
+
* `size` is what its value weighs in the store, as its status reports it: for
|
|
916
|
+
* a collection, its segments and its manifest.
|
|
707
917
|
*/
|
|
708
918
|
export const ListEntryType = VariantType({
|
|
709
919
|
dataset: StructType({
|
|
@@ -729,7 +939,8 @@ export const ListEntryType = VariantType({
|
|
|
729
939
|
* @property type - East type of the dataset
|
|
730
940
|
* @property refType - Ref type: "unassigned", "null", or "value"
|
|
731
941
|
* @property hash - Object hash (None if unassigned/null)
|
|
732
|
-
* @property size -
|
|
942
|
+
* @property size - Bytes the value weighs in the store: for a collection, its
|
|
943
|
+
* segments and its manifest (None if unassigned)
|
|
733
944
|
*/
|
|
734
945
|
export const DatasetStatusDetailType = StructType({
|
|
735
946
|
path: StringType,
|
|
@@ -737,10 +948,9 @@ export const DatasetStatusDetailType = StructType({
|
|
|
737
948
|
refType: StringType,
|
|
738
949
|
hash: OptionType(StringType),
|
|
739
950
|
size: OptionType(IntegerType),
|
|
740
|
-
/** Segment and element counts of a stored collection, read from
|
|
741
|
-
*
|
|
742
|
-
*
|
|
743
|
-
* the positional struct rule. */
|
|
951
|
+
/** Segment and element counts of a stored collection, read from its
|
|
952
|
+
* manifest — so a re-pointed input is inspectable without decoding it.
|
|
953
|
+
* `none` for a non-collection or an unset dataset. */
|
|
744
954
|
segments: OptionType(IntegerType),
|
|
745
955
|
rows: OptionType(IntegerType),
|
|
746
956
|
});
|
|
@@ -834,6 +1044,10 @@ export const OneShotRequestType = StructType({
|
|
|
834
1044
|
/**
|
|
835
1045
|
* A record mutation call. Positional args (after the implicit current state),
|
|
836
1046
|
* one beast2-encoded value per declared parameter.
|
|
1047
|
+
*
|
|
1048
|
+
* Of the limits, `timeoutMs` bounds each run of the mutation's program and
|
|
1049
|
+
* `maxLogBytes` the stderr a failure returns. `maxResultBytes` does not apply:
|
|
1050
|
+
* a mutation's output is stored as segments and never read whole.
|
|
837
1051
|
*/
|
|
838
1052
|
export const MutationCallRequestType = StructType({
|
|
839
1053
|
args: ArrayType(BlobType),
|
|
@@ -846,32 +1060,49 @@ export const MutationCallRequestType = StructType({
|
|
|
846
1060
|
*
|
|
847
1061
|
* - `committed`: the new commit + state hashes
|
|
848
1062
|
* - `invalid`: record/mutation lookup or arity error; nothing ran
|
|
849
|
-
* - `failed`: the
|
|
850
|
-
* - `
|
|
851
|
-
* - `
|
|
852
|
-
*
|
|
1063
|
+
* - `failed`: the mutation's program failed (incl. a body's `$.error`; see stderr)
|
|
1064
|
+
* - `timed_out`: the program exceeded its time budget
|
|
1065
|
+
* - `conflict`: the compare-and-swap lost the race `attempts` times, or the
|
|
1066
|
+
* write disagreed with the state it landed on — then `detail` names the key,
|
|
1067
|
+
* and resubmitting the same write cannot help
|
|
853
1068
|
*/
|
|
854
1069
|
export const MutationResultType = StructType({
|
|
855
1070
|
outcome: VariantType({
|
|
856
1071
|
committed: StructType({ commitHash: StringType, stateHash: StringType }),
|
|
857
1072
|
invalid: StructType({ message: StringType }),
|
|
858
1073
|
failed: StructType({ exitCode: IntegerType, stderr: StringType }),
|
|
859
|
-
too_large: StructType({ bytes: IntegerType, limit: IntegerType, stderr: StringType }),
|
|
860
1074
|
timed_out: StructType({ ms: IntegerType, stderr: StringType }),
|
|
861
|
-
conflict: StructType({ attempts: IntegerType }),
|
|
1075
|
+
conflict: StructType({ attempts: IntegerType, detail: OptionType(StringType) }),
|
|
862
1076
|
}),
|
|
863
1077
|
});
|
|
864
1078
|
|
|
865
1079
|
/**
|
|
866
1080
|
* A record's callable surface, returned by `describe` so dynamic callers can
|
|
867
|
-
* encode mutation arguments
|
|
868
|
-
* after the implicit current state.
|
|
1081
|
+
* encode mutation arguments and read through its indexes. The mutation
|
|
1082
|
+
* `argTypes` are the EXTRA parameters after the implicit current state.
|
|
869
1083
|
*/
|
|
870
1084
|
export const RecordSignatureType = StructType({
|
|
871
1085
|
name: StringType,
|
|
872
1086
|
mutations: ArrayType(StructType({
|
|
873
1087
|
name: StringType,
|
|
874
1088
|
argTypes: ArrayType(EastTypeType),
|
|
1089
|
+
/** The write form — `reduce`, `edit` or `patch` — which says what the
|
|
1090
|
+
* arguments MEAN: a `patch` mutation's one argument is a
|
|
1091
|
+
* `PatchType(State)`, not a value of the record's own type. */
|
|
1092
|
+
form: StringType,
|
|
1093
|
+
})),
|
|
1094
|
+
/** The indexes the record declares. A page read through one is decoded by
|
|
1095
|
+
* `indexWindowType(K, keyType, valueType, V)` over the record's
|
|
1096
|
+
* `Dict<K, V>`, and a key search through one takes a `keyType` key. */
|
|
1097
|
+
indexes: ArrayType(StructType({
|
|
1098
|
+
name: StringType,
|
|
1099
|
+
/** The index key's type. */
|
|
1100
|
+
keyType: EastTypeType,
|
|
1101
|
+
/** The covering projection's type: `Null` when the index projects
|
|
1102
|
+
* nothing, and a read joins the rows instead. */
|
|
1103
|
+
valueType: EastTypeType,
|
|
1104
|
+
/** Whether a row may hold several index keys. */
|
|
1105
|
+
multi: BooleanType,
|
|
875
1106
|
})),
|
|
876
1107
|
});
|
|
877
1108
|
|
|
@@ -883,6 +1114,9 @@ export const RecordCommitInfoType = StructType({
|
|
|
883
1114
|
mutation: StringType,
|
|
884
1115
|
actor: StringType,
|
|
885
1116
|
at: DateTimeType,
|
|
1117
|
+
/** The delta this commit applied, when it wrote one — so history shows WHAT
|
|
1118
|
+
* changed without diffing two states. */
|
|
1119
|
+
delta: OptionType(StringType),
|
|
886
1120
|
});
|
|
887
1121
|
|
|
888
1122
|
/** A page of a record's commit history, newest first. */
|
|
@@ -907,6 +1141,13 @@ export type PackageDetails = ValueTypeOf<typeof PackageDetailsType>;
|
|
|
907
1141
|
export type WorkspaceInfo = ValueTypeOf<typeof WorkspaceInfoType>;
|
|
908
1142
|
export type WorkspaceCreateRequest = ValueTypeOf<typeof WorkspaceCreateRequestType>;
|
|
909
1143
|
export type WorkspaceDeployRequest = ValueTypeOf<typeof WorkspaceDeployRequestType>;
|
|
1144
|
+
/** A {@link SchemaPolicyType} by its name, as a deploy's options take it. */
|
|
1145
|
+
export type SchemaPolicy = ValueTypeOf<typeof SchemaPolicyType>['type'];
|
|
1146
|
+
export type RecordPlan = ValueTypeOf<typeof RecordPlanType>;
|
|
1147
|
+
export type RecordIndexPlan = ValueTypeOf<typeof RecordIndexPlanType>;
|
|
1148
|
+
export type WorkspaceDeployResult = ValueTypeOf<typeof WorkspaceDeployResultType>;
|
|
1149
|
+
export type WorkspaceDeployProgress = ValueTypeOf<typeof WorkspaceDeployProgressType>;
|
|
1150
|
+
export type WorkspaceDeployStatus = ValueTypeOf<typeof WorkspaceDeployStatusType>;
|
|
910
1151
|
export type DatasetStatus = ValueTypeOf<typeof DatasetStatusType>;
|
|
911
1152
|
export type TaskStatus = ValueTypeOf<typeof TaskStatusType>;
|
|
912
1153
|
export type DatasetStatusInfo = ValueTypeOf<typeof DatasetStatusInfoType>;
|
|
@@ -922,6 +1163,9 @@ export type DataflowResult = ValueTypeOf<typeof DataflowResultType>;
|
|
|
922
1163
|
export type DataflowEvent = ValueTypeOf<typeof DataflowEventType>;
|
|
923
1164
|
export type ApiExecutionStatus = ValueTypeOf<typeof ApiExecutionStatusType>;
|
|
924
1165
|
export type DataflowExecutionSummary = ValueTypeOf<typeof DataflowExecutionSummaryType>;
|
|
1166
|
+
export type DataflowBudget = ValueTypeOf<typeof DataflowBudgetType>;
|
|
1167
|
+
export type UnitWait = ValueTypeOf<typeof UnitWaitType>;
|
|
1168
|
+
export type SplitProgress = ValueTypeOf<typeof SplitProgressType>;
|
|
925
1169
|
export type ApiDataflowExecutionState = ValueTypeOf<typeof ApiDataflowExecutionStateType>;
|
|
926
1170
|
export type ExecutionHistoryStatus = ValueTypeOf<typeof ExecutionHistoryStatusType>;
|
|
927
1171
|
export type ExecutionListItem = ValueTypeOf<typeof ExecutionListItemType>;
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Copyright (c) 2025 Elara AI Pty Ltd
|
|
3
|
+
* Dual-licensed under AGPL-3.0 and commercial license. See LICENSE for details.
|
|
4
|
+
*/
|
|
5
|
+
|
|
6
|
+
/**
|
|
7
|
+
* The segment-object layout: a collection dataset is a **manifest naming
|
|
8
|
+
* standalone segment objects**, not one segmented blob.
|
|
9
|
+
*
|
|
10
|
+
* The type itself lives in `east`, beside the container it describes, because
|
|
11
|
+
* every runtime must read it: a task input staged as a manifest is one file
|
|
12
|
+
* naming sibling segment files, and a body that iterates it pays for the
|
|
13
|
+
* segments it touches rather than for a splice of the whole value. This module
|
|
14
|
+
* re-exports it for the e3 packages and adds what only a store cares about.
|
|
15
|
+
*
|
|
16
|
+
* Three properties are why records at millions of entries are affordable, and
|
|
17
|
+
* every one of them is about successive states SHARING storage rather than
|
|
18
|
+
* about indexing, which the trailing index already gave:
|
|
19
|
+
*
|
|
20
|
+
* - **A one-row edit costs one segment.** The new state names the same objects
|
|
21
|
+
* as the old one bar the segment that changed, so a commit's new storage is
|
|
22
|
+
* the touched segments plus a manifest — not the whole value.
|
|
23
|
+
* - **Two states diff in O(changed segments)**, by comparing entry hashes. No
|
|
24
|
+
* value is decoded to find out what moved, which is what index maintenance
|
|
25
|
+
* and the mutation delta are built on.
|
|
26
|
+
* - **A segment can be checked against its own hash**, which a ranged read of
|
|
27
|
+
* one blob can never offer: no digest exists for a byte range. No reader
|
|
28
|
+
* checks it — a segment is trusted as stored, like any other object.
|
|
29
|
+
*
|
|
30
|
+
* The manifest is the envelope the garbage collector recognizes, which is why
|
|
31
|
+
* its `kind` is a stored field rather than something inferred: an object whose
|
|
32
|
+
* entries name other objects MUST be traversable by shape, or the first sweep
|
|
33
|
+
* deletes a record's segments.
|
|
34
|
+
*
|
|
35
|
+
* @packageDocumentation
|
|
36
|
+
*/
|
|
37
|
+
|
|
38
|
+
export {
|
|
39
|
+
COLLECTION_MANIFEST_KIND,
|
|
40
|
+
CollectionManifestType,
|
|
41
|
+
CollectionManifestEntryType,
|
|
42
|
+
type CollectionManifest,
|
|
43
|
+
type CollectionManifestEntry,
|
|
44
|
+
type Beast2ManifestSource,
|
|
45
|
+
isCollectionManifestType,
|
|
46
|
+
isCollectionManifest,
|
|
47
|
+
isBeast2ManifestSource,
|
|
48
|
+
encodeCollectionManifest,
|
|
49
|
+
decodeCollectionManifest,
|
|
50
|
+
readBeast2Manifest,
|
|
51
|
+
manifestElementCount,
|
|
52
|
+
manifestByteSize,
|
|
53
|
+
} from '@elaraai/east';
|