@elaraai/e3-types 1.0.79 → 1.0.81
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/src/api.d.ts +829 -36
- package/dist/src/api.d.ts.map +1 -1
- package/dist/src/api.js +301 -32
- package/dist/src/api.js.map +1 -1
- package/dist/src/collection-manifest.d.ts +37 -0
- package/dist/src/collection-manifest.d.ts.map +1 -0
- package/dist/src/collection-manifest.js +37 -0
- package/dist/src/collection-manifest.js.map +1 -0
- package/dist/src/dataflow.d.ts +264 -19
- package/dist/src/dataflow.d.ts.map +1 -1
- package/dist/src/dataflow.js +156 -15
- package/dist/src/dataflow.js.map +1 -1
- package/dist/src/dataflow.spec.js +184 -27
- package/dist/src/dataflow.spec.js.map +1 -1
- package/dist/src/dataset-blob.d.ts +75 -15
- package/dist/src/dataset-blob.d.ts.map +1 -1
- package/dist/src/dataset-blob.js +95 -28
- package/dist/src/dataset-blob.js.map +1 -1
- package/dist/src/dataset-blob.spec.d.ts +6 -0
- package/dist/src/dataset-blob.spec.d.ts.map +1 -0
- package/dist/src/dataset-blob.spec.js +92 -0
- package/dist/src/dataset-blob.spec.js.map +1 -0
- package/dist/src/dataset-ref.d.ts +5 -4
- package/dist/src/dataset-ref.d.ts.map +1 -1
- package/dist/src/dataset-ref.js +5 -4
- package/dist/src/dataset-ref.js.map +1 -1
- package/dist/src/environment.d.ts +3 -2
- package/dist/src/environment.d.ts.map +1 -1
- package/dist/src/environment.js +3 -2
- package/dist/src/environment.js.map +1 -1
- package/dist/src/execution.d.ts +112 -18
- package/dist/src/execution.d.ts.map +1 -1
- package/dist/src/execution.js +174 -45
- package/dist/src/execution.js.map +1 -1
- package/dist/src/execution.spec.d.ts +6 -0
- package/dist/src/execution.spec.d.ts.map +1 -0
- package/dist/src/execution.spec.js +108 -0
- package/dist/src/execution.spec.js.map +1 -0
- package/dist/src/function.d.ts +14 -12
- package/dist/src/function.d.ts.map +1 -1
- package/dist/src/function.js +17 -32
- package/dist/src/function.js.map +1 -1
- package/dist/src/index.d.ts +17 -12
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/index.js +28 -20
- package/dist/src/index.js.map +1 -1
- package/dist/src/intake.d.ts +75 -0
- package/dist/src/intake.d.ts.map +1 -0
- package/dist/src/intake.js +46 -0
- package/dist/src/intake.js.map +1 -0
- package/dist/src/lock.d.ts +253 -21
- package/dist/src/lock.d.ts.map +1 -1
- package/dist/src/lock.js +55 -21
- package/dist/src/lock.js.map +1 -1
- package/dist/src/names.d.ts +21 -0
- package/dist/src/names.d.ts.map +1 -0
- package/dist/src/names.js +31 -0
- package/dist/src/names.js.map +1 -0
- package/dist/src/package.d.ts +11 -78
- package/dist/src/package.d.ts.map +1 -1
- package/dist/src/package.js +13 -66
- package/dist/src/package.js.map +1 -1
- package/dist/src/record.d.ts +582 -18
- package/dist/src/record.d.ts.map +1 -1
- package/dist/src/record.js +405 -7
- package/dist/src/record.js.map +1 -1
- package/dist/src/{stream.spec.d.ts → record.spec.d.ts} +1 -1
- package/dist/src/record.spec.d.ts.map +1 -0
- package/dist/src/record.spec.js +143 -0
- package/dist/src/record.spec.js.map +1 -0
- package/dist/src/release.d.ts +36 -0
- package/dist/src/release.d.ts.map +1 -0
- package/dist/src/release.js +71 -0
- package/dist/src/release.js.map +1 -0
- package/dist/src/release.spec.d.ts +6 -0
- package/dist/src/release.spec.d.ts.map +1 -0
- package/dist/src/release.spec.js +42 -0
- package/dist/src/release.spec.js.map +1 -0
- package/dist/src/repository.d.ts +64 -0
- package/dist/src/repository.d.ts.map +1 -0
- package/dist/src/repository.js +49 -0
- package/dist/src/repository.js.map +1 -0
- package/dist/src/runner.d.ts +16 -62
- package/dist/src/runner.d.ts.map +1 -1
- package/dist/src/runner.js +16 -79
- package/dist/src/runner.js.map +1 -1
- package/dist/src/runner.spec.js +11 -85
- package/dist/src/runner.spec.js.map +1 -1
- package/dist/src/structure.d.ts +0 -57
- package/dist/src/structure.d.ts.map +1 -1
- package/dist/src/structure.js +0 -3
- package/dist/src/structure.js.map +1 -1
- package/dist/src/task.d.ts +210 -318
- package/dist/src/task.d.ts.map +1 -1
- package/dist/src/task.js +117 -472
- package/dist/src/task.js.map +1 -1
- package/dist/src/task.spec.js +62 -154
- package/dist/src/task.spec.js.map +1 -1
- package/dist/src/transfer.d.ts +38 -26
- package/dist/src/transfer.d.ts.map +1 -1
- package/dist/src/transfer.js +22 -26
- package/dist/src/transfer.js.map +1 -1
- package/dist/src/transfer.spec.js +4 -34
- package/dist/src/transfer.spec.js.map +1 -1
- package/dist/src/unit-plan.d.ts +130 -0
- package/dist/src/unit-plan.d.ts.map +1 -0
- package/dist/src/unit-plan.js +107 -0
- package/dist/src/unit-plan.js.map +1 -0
- package/dist/src/unit-plan.spec.d.ts +6 -0
- package/dist/src/unit-plan.spec.d.ts.map +1 -0
- package/dist/src/unit-plan.spec.js +29 -0
- package/dist/src/unit-plan.spec.js.map +1 -0
- package/dist/src/workspace.d.ts +22 -11
- package/dist/src/workspace.d.ts.map +1 -1
- package/dist/src/workspace.js +10 -11
- package/dist/src/workspace.js.map +1 -1
- package/package.json +3 -3
- package/src/api.ts +326 -32
- package/src/collection-manifest.ts +53 -0
- package/src/dataflow.spec.ts +223 -31
- package/src/dataflow.ts +183 -26
- package/src/dataset-blob.spec.ts +117 -0
- package/src/dataset-blob.ts +157 -21
- package/src/dataset-ref.ts +5 -4
- package/src/environment.ts +3 -2
- package/src/execution.spec.ts +114 -0
- package/src/execution.ts +175 -51
- package/src/function.ts +19 -32
- package/src/index.ts +141 -39
- package/src/intake.ts +53 -0
- package/src/lock.ts +67 -21
- package/src/names.ts +39 -0
- package/src/package.ts +15 -71
- package/src/record.spec.ts +195 -0
- package/src/record.ts +489 -7
- package/src/release.spec.ts +47 -0
- package/src/release.ts +69 -0
- package/src/repository.ts +67 -0
- package/src/runner.spec.ts +11 -94
- package/src/runner.ts +16 -80
- package/src/structure.ts +0 -8
- package/src/task.spec.ts +67 -188
- package/src/task.ts +140 -522
- package/src/transfer.spec.ts +4 -51
- package/src/transfer.ts +22 -26
- package/src/unit-plan.spec.ts +31 -0
- package/src/unit-plan.ts +121 -0
- package/src/workspace.ts +13 -11
- package/dist/src/stream.d.ts +0 -97
- package/dist/src/stream.d.ts.map +0 -1
- package/dist/src/stream.js +0 -134
- package/dist/src/stream.js.map +0 -1
- package/dist/src/stream.spec.d.ts.map +0 -1
- package/dist/src/stream.spec.js +0 -60
- package/dist/src/stream.spec.js.map +0 -1
- package/src/stream.spec.ts +0 -88
- package/src/stream.ts +0 -170
package/src/api.ts
CHANGED
|
@@ -36,7 +36,10 @@ import {
|
|
|
36
36
|
} from '@elaraai/east';
|
|
37
37
|
|
|
38
38
|
import { StructureType, TreePathType } from './structure.js';
|
|
39
|
+
import { IntakeFileType } from './intake.js';
|
|
39
40
|
import { RunnerType } from './runner.js';
|
|
41
|
+
import { TaskBodyType, TaskInputType, TaskOutputType, TaskRoleType } from './task.js';
|
|
42
|
+
import { RequeueReasonType, StageUnitType } from './dataflow.js';
|
|
40
43
|
|
|
41
44
|
// =============================================================================
|
|
42
45
|
// Error Types
|
|
@@ -78,6 +81,13 @@ export const DatasetTypeMismatchErrorType = StructType({
|
|
|
78
81
|
path: StringType,
|
|
79
82
|
message: StringType,
|
|
80
83
|
});
|
|
84
|
+
/** A name e3 would make a path of that cannot be one path segment: a
|
|
85
|
+
* repository's, a workspace's, or a package's name or version. */
|
|
86
|
+
export const InvalidNameErrorType = StructType({
|
|
87
|
+
kind: StringType,
|
|
88
|
+
name: StringType,
|
|
89
|
+
message: StringType,
|
|
90
|
+
});
|
|
81
91
|
|
|
82
92
|
export const ErrorType = VariantType({
|
|
83
93
|
repository_not_found: RepositoryNotFoundErrorType,
|
|
@@ -96,9 +106,8 @@ export const ErrorType = VariantType({
|
|
|
96
106
|
dataflow_aborted: NullType,
|
|
97
107
|
permission_denied: PermissionDeniedErrorType,
|
|
98
108
|
internal: InternalErrorType,
|
|
99
|
-
// Appended last: BEAST2 encodes a variant case by index, so a case added at
|
|
100
|
-
// the end leaves every existing payload decodable.
|
|
101
109
|
dataset_type_mismatch: DatasetTypeMismatchErrorType,
|
|
110
|
+
invalid_name: InvalidNameErrorType,
|
|
102
111
|
});
|
|
103
112
|
|
|
104
113
|
// =============================================================================
|
|
@@ -134,10 +143,16 @@ export const RepositoryStatusType = StructType({
|
|
|
134
143
|
*
|
|
135
144
|
* @property dryRun - If true, report what would be deleted without deleting
|
|
136
145
|
* @property minAge - Minimum age in milliseconds for objects to be considered for deletion
|
|
146
|
+
* @property keepRuns - The runs of each workspace kept however old, the latest
|
|
147
|
+
* first; the server's default when `none`
|
|
148
|
+
* @property keepDays - The days of runs and executions kept however many; the
|
|
149
|
+
* server's default when `none`
|
|
137
150
|
*/
|
|
138
151
|
export const GcRequestType = StructType({
|
|
139
152
|
dryRun: BooleanType,
|
|
140
153
|
minAge: OptionType(IntegerType),
|
|
154
|
+
keepRuns: OptionType(IntegerType),
|
|
155
|
+
keepDays: OptionType(IntegerType),
|
|
141
156
|
});
|
|
142
157
|
|
|
143
158
|
/**
|
|
@@ -148,6 +163,9 @@ export const GcRequestType = StructType({
|
|
|
148
163
|
* @property retainedObjects - Number of objects still referenced
|
|
149
164
|
* @property skippedYoung - Number of objects skipped due to minAge
|
|
150
165
|
* @property bytesFreed - Total bytes freed by deletion
|
|
166
|
+
* @property deletedRuns - Number of dataflow run records deleted
|
|
167
|
+
* @property deletedExecutions - Number of execution attempts deleted, each with
|
|
168
|
+
* its owner record and logs
|
|
151
169
|
*/
|
|
152
170
|
export const GcResultType = StructType({
|
|
153
171
|
deletedObjects: IntegerType,
|
|
@@ -155,6 +173,8 @@ export const GcResultType = StructType({
|
|
|
155
173
|
retainedObjects: IntegerType,
|
|
156
174
|
skippedYoung: IntegerType,
|
|
157
175
|
bytesFreed: IntegerType,
|
|
176
|
+
deletedRuns: IntegerType,
|
|
177
|
+
deletedExecutions: IntegerType,
|
|
158
178
|
});
|
|
159
179
|
|
|
160
180
|
// =============================================================================
|
|
@@ -269,13 +289,164 @@ export const WorkspaceInfoType = StructType({
|
|
|
269
289
|
packageVersion: OptionType(StringType),
|
|
270
290
|
});
|
|
271
291
|
|
|
292
|
+
/**
|
|
293
|
+
* What a deploy does with a record it cannot keep as it is.
|
|
294
|
+
*
|
|
295
|
+
* - `migrate`: run the migrations the workspace has not applied, and refuse a
|
|
296
|
+
* record no migration carries to the package's type
|
|
297
|
+
* - `fail`: run none, and refuse a record with migrations to run, for a
|
|
298
|
+
* workspace whose migrations go through their own change control
|
|
299
|
+
* - `reset`: reset a record it can neither keep nor migrate to the package's
|
|
300
|
+
* initial value, under a `$reset` commit, so the reset is in its history
|
|
301
|
+
*/
|
|
302
|
+
export const SchemaPolicyType = VariantType({
|
|
303
|
+
migrate: NullType,
|
|
304
|
+
fail: NullType,
|
|
305
|
+
reset: NullType,
|
|
306
|
+
});
|
|
307
|
+
|
|
272
308
|
/**
|
|
273
309
|
* Request to deploy a package to a workspace.
|
|
274
310
|
*
|
|
275
311
|
* @property packageRef - Package reference in format "name" or "name@version"
|
|
312
|
+
* @property schema - What the deploy does with a record it cannot keep as it is
|
|
313
|
+
* @property allowDropRecords - Whether a record the package no longer declares
|
|
314
|
+
* may be dropped, with its state and history
|
|
315
|
+
* @property plan - Say what the deploy would do, and write nothing
|
|
276
316
|
*/
|
|
277
317
|
export const WorkspaceDeployRequestType = StructType({
|
|
278
318
|
packageRef: StringType,
|
|
319
|
+
schema: SchemaPolicyType,
|
|
320
|
+
allowDropRecords: BooleanType,
|
|
321
|
+
plan: BooleanType,
|
|
322
|
+
});
|
|
323
|
+
|
|
324
|
+
/**
|
|
325
|
+
* What a deploy decided for one record.
|
|
326
|
+
*
|
|
327
|
+
* @property record - The record's dataset ref path
|
|
328
|
+
* @property action - What the deploy does to it:
|
|
329
|
+
* - `mint`: the workspace does not hold it, so it is minted from the
|
|
330
|
+
* package's initial value
|
|
331
|
+
* - `keep`: kept as the workspace holds it; `deploy` when the package under
|
|
332
|
+
* it changed, which a `$deploy` commit records in its history
|
|
333
|
+
* - `migrate`: migrated by the steps the workspace has not applied, in order
|
|
334
|
+
* - `reset`: reset to the package's initial value under the `reset` policy,
|
|
335
|
+
* and why it could not be kept or migrated
|
|
336
|
+
* - `drop`: the package does not declare it, so it is dropped, with its
|
|
337
|
+
* state and history
|
|
338
|
+
* - `refused`: refused, and why, with the fix: the deploy writes nothing
|
|
339
|
+
*/
|
|
340
|
+
export const RecordPlanType = StructType({
|
|
341
|
+
record: StringType,
|
|
342
|
+
action: VariantType({
|
|
343
|
+
mint: NullType,
|
|
344
|
+
keep: StructType({ deploy: BooleanType }),
|
|
345
|
+
migrate: StructType({ steps: ArrayType(StringType) }),
|
|
346
|
+
reset: StructType({ reason: StringType }),
|
|
347
|
+
drop: NullType,
|
|
348
|
+
refused: StructType({ reason: StringType }),
|
|
349
|
+
}),
|
|
350
|
+
});
|
|
351
|
+
|
|
352
|
+
/**
|
|
353
|
+
* What a deploy decided for one of a record's indexes.
|
|
354
|
+
*
|
|
355
|
+
* @property record - The record's dataset ref path
|
|
356
|
+
* @property index - The index's name
|
|
357
|
+
* @property action - `build` when the state names no index under the
|
|
358
|
+
* package's declaration, `drop` when it names one the package does not
|
|
359
|
+
* declare, and `keep` when the two agree and nothing runs
|
|
360
|
+
*/
|
|
361
|
+
export const RecordIndexPlanType = StructType({
|
|
362
|
+
record: StringType,
|
|
363
|
+
index: StringType,
|
|
364
|
+
action: VariantType({
|
|
365
|
+
build: NullType,
|
|
366
|
+
drop: NullType,
|
|
367
|
+
keep: NullType,
|
|
368
|
+
}),
|
|
369
|
+
});
|
|
370
|
+
|
|
371
|
+
/**
|
|
372
|
+
* The step a deploy is at with one record.
|
|
373
|
+
*
|
|
374
|
+
* - `waiting`: its turn has not come, or it needs nothing but the deploy's
|
|
375
|
+
* commit
|
|
376
|
+
* - `migrating`: running migration step `step` of the `steps` it owes, `name`
|
|
377
|
+
* - `indexing`: building `index`, the `build`th of the `builds` it owes
|
|
378
|
+
* - `done`: its migrations and index builds have run
|
|
379
|
+
*/
|
|
380
|
+
export const RecordDeployStepType = VariantType({
|
|
381
|
+
waiting: NullType,
|
|
382
|
+
migrating: StructType({ name: StringType, step: IntegerType, steps: IntegerType }),
|
|
383
|
+
indexing: StructType({ index: StringType, build: IntegerType, builds: IntegerType }),
|
|
384
|
+
done: NullType,
|
|
385
|
+
});
|
|
386
|
+
|
|
387
|
+
/**
|
|
388
|
+
* One record a deploy is deploying.
|
|
389
|
+
*
|
|
390
|
+
* @property plan - What the deploy decided for it
|
|
391
|
+
* @property indexes - The indexes the package declares for it
|
|
392
|
+
* @property step - How far the deploy has got with it
|
|
393
|
+
*/
|
|
394
|
+
export const RecordDeployStateType = StructType({
|
|
395
|
+
plan: RecordPlanType,
|
|
396
|
+
indexes: ArrayType(StringType),
|
|
397
|
+
step: RecordDeployStepType,
|
|
398
|
+
});
|
|
399
|
+
|
|
400
|
+
/**
|
|
401
|
+
* How far a deploy has got, for a client to show while it runs.
|
|
402
|
+
*
|
|
403
|
+
* @property package - The package it deploys
|
|
404
|
+
* @property startedAt - When it began taking its files in
|
|
405
|
+
* @property files - Each file source it takes in, in the package's order
|
|
406
|
+
* @property records - Each record it deploys, in the package's order, and
|
|
407
|
+
* each it drops
|
|
408
|
+
*/
|
|
409
|
+
export const DeployProgressType = StructType({
|
|
410
|
+
package: StructType({ name: StringType, version: StringType }),
|
|
411
|
+
startedAt: DateTimeType,
|
|
412
|
+
files: ArrayType(IntakeFileType),
|
|
413
|
+
records: ArrayType(RecordDeployStateType),
|
|
414
|
+
});
|
|
415
|
+
|
|
416
|
+
/**
|
|
417
|
+
* What a deploy did, or under `plan` would do.
|
|
418
|
+
*
|
|
419
|
+
* @property records - What it decided for each record
|
|
420
|
+
* @property indexes - What it decided for each index of the records it keeps
|
|
421
|
+
* @property warnings - The inputs it left unassigned, and why: a server never
|
|
422
|
+
* reads a `file` source, whose path is on the client's machine
|
|
423
|
+
*/
|
|
424
|
+
export const WorkspaceDeployResultType = StructType({
|
|
425
|
+
records: ArrayType(RecordPlanType),
|
|
426
|
+
indexes: ArrayType(RecordIndexPlanType),
|
|
427
|
+
warnings: ArrayType(StringType),
|
|
428
|
+
});
|
|
429
|
+
|
|
430
|
+
/**
|
|
431
|
+
* A deploy job's progress: `pending` until it starts, then `deploying`, with
|
|
432
|
+
* how far it has got once it has said.
|
|
433
|
+
*/
|
|
434
|
+
export const WorkspaceDeployProgressType = VariantType({
|
|
435
|
+
pending: NullType,
|
|
436
|
+
deploying: OptionType(DeployProgressType),
|
|
437
|
+
});
|
|
438
|
+
|
|
439
|
+
/**
|
|
440
|
+
* A deploy job's status.
|
|
441
|
+
*
|
|
442
|
+
* - `processing`: still running, and how far it has got
|
|
443
|
+
* - `completed`: what the deploy did
|
|
444
|
+
* - `failed`: why it did not
|
|
445
|
+
*/
|
|
446
|
+
export const WorkspaceDeployStatusType = VariantType({
|
|
447
|
+
processing: WorkspaceDeployProgressType,
|
|
448
|
+
completed: WorkspaceDeployResultType,
|
|
449
|
+
failed: StructType({ message: StringType }),
|
|
279
450
|
});
|
|
280
451
|
|
|
281
452
|
/**
|
|
@@ -391,6 +562,9 @@ export const DatasetStatusInfoType = StructType({
|
|
|
391
562
|
* @property inputs - Dataset paths this task reads from
|
|
392
563
|
* @property output - Dataset path this task writes to
|
|
393
564
|
* @property dependsOn - Names of tasks that must complete before this one
|
|
565
|
+
* @property peakBytes - The highest peak resident memory, in bytes, a runner
|
|
566
|
+
* of the execution the status comes from reached: the one the output came
|
|
567
|
+
* from, or the failure; `none` when it recorded none, or while it runs
|
|
394
568
|
*/
|
|
395
569
|
export const TaskStatusInfoType = StructType({
|
|
396
570
|
name: StringType,
|
|
@@ -399,6 +573,7 @@ export const TaskStatusInfoType = StructType({
|
|
|
399
573
|
inputs: ArrayType(StringType),
|
|
400
574
|
output: StringType,
|
|
401
575
|
dependsOn: ArrayType(StringType),
|
|
576
|
+
peakBytes: OptionType(IntegerType),
|
|
402
577
|
});
|
|
403
578
|
|
|
404
579
|
/**
|
|
@@ -451,31 +626,34 @@ export const WorkspaceStatusResultType = StructType({
|
|
|
451
626
|
*
|
|
452
627
|
* @property name - Task name
|
|
453
628
|
* @property hash - Task definition hash
|
|
454
|
-
* @property
|
|
629
|
+
* @property role - What the task's output is for: data, or a ui with what it
|
|
630
|
+
* binds
|
|
455
631
|
*/
|
|
456
632
|
export const TaskListItemType = StructType({
|
|
457
633
|
name: StringType,
|
|
458
634
|
hash: StringType,
|
|
459
|
-
|
|
635
|
+
role: TaskRoleType,
|
|
460
636
|
});
|
|
461
637
|
|
|
462
638
|
/**
|
|
463
|
-
* Detailed task information.
|
|
639
|
+
* Detailed task information: the task object's fields.
|
|
464
640
|
*
|
|
465
641
|
* @property name - Task name
|
|
466
642
|
* @property hash - Task definition hash
|
|
467
|
-
* @property
|
|
468
|
-
* @property
|
|
469
|
-
* @property
|
|
643
|
+
* @property body - What the task runs: an East program, or a command
|
|
644
|
+
* @property runner - The runtime it runs on
|
|
645
|
+
* @property inputs - The datasets it reads, each with its partition
|
|
646
|
+
* @property output - Where its output goes, and how it is made
|
|
647
|
+
* @property role - What its output is for: data, or a ui with what it binds
|
|
470
648
|
*/
|
|
471
649
|
export const TaskDetailsType = StructType({
|
|
472
650
|
name: StringType,
|
|
473
651
|
hash: StringType,
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
652
|
+
body: TaskBodyType,
|
|
653
|
+
runner: RunnerType,
|
|
654
|
+
inputs: ArrayType(TaskInputType),
|
|
655
|
+
output: TaskOutputType,
|
|
656
|
+
role: TaskRoleType,
|
|
479
657
|
});
|
|
480
658
|
|
|
481
659
|
// =============================================================================
|
|
@@ -483,14 +661,13 @@ export const TaskDetailsType = StructType({
|
|
|
483
661
|
// =============================================================================
|
|
484
662
|
|
|
485
663
|
/**
|
|
486
|
-
* Request to start dataflow execution.
|
|
664
|
+
* Request to start dataflow execution. The run takes the server's budget of
|
|
665
|
+
* cores and memory, which it shares with everything else the server runs.
|
|
487
666
|
*
|
|
488
|
-
* @property concurrency - Maximum parallel tasks (default: 4)
|
|
489
667
|
* @property force - Force re-execution of all tasks
|
|
490
668
|
* @property filter - Filter to specific task names (glob pattern)
|
|
491
669
|
*/
|
|
492
670
|
export const DataflowRequestType = StructType({
|
|
493
|
-
concurrency: OptionType(IntegerType),
|
|
494
671
|
force: BooleanType,
|
|
495
672
|
filter: OptionType(StringType),
|
|
496
673
|
});
|
|
@@ -561,11 +738,15 @@ export const DataflowResultType = StructType({
|
|
|
561
738
|
* Dataflow event types for API polling.
|
|
562
739
|
*
|
|
563
740
|
* - `start`: Task started executing
|
|
564
|
-
* - `complete`: Task executed and succeeded
|
|
741
|
+
* - `complete`: Task executed and succeeded, with the highest peak memory its
|
|
742
|
+
* runners reached when they reported one
|
|
565
743
|
* - `cached`: Task result retrieved from cache (no execution)
|
|
566
744
|
* - `failed`: Task exited with non-zero code
|
|
567
745
|
* - `error`: Internal error during task execution
|
|
568
746
|
* - `input_unavailable`: Task couldn't run because inputs not available
|
|
747
|
+
* - `requeued`: A unit of a split task was stopped — by the budget's guard, or
|
|
748
|
+
* its cgroup's cap — and runs again once the memory it reserves fits: the
|
|
749
|
+
* unit, why, the most it was measured using and what it reserves, in bytes
|
|
569
750
|
*/
|
|
570
751
|
export const DataflowEventType = VariantType({
|
|
571
752
|
start: StructType({
|
|
@@ -576,6 +757,7 @@ export const DataflowEventType = VariantType({
|
|
|
576
757
|
task: StringType,
|
|
577
758
|
timestamp: StringType,
|
|
578
759
|
duration: FloatType,
|
|
760
|
+
peakBytes: OptionType(IntegerType),
|
|
579
761
|
}),
|
|
580
762
|
cached: StructType({
|
|
581
763
|
task: StringType,
|
|
@@ -597,6 +779,14 @@ export const DataflowEventType = VariantType({
|
|
|
597
779
|
timestamp: StringType,
|
|
598
780
|
reason: StringType,
|
|
599
781
|
}),
|
|
782
|
+
requeued: StructType({
|
|
783
|
+
task: StringType,
|
|
784
|
+
timestamp: StringType,
|
|
785
|
+
unit: StageUnitType,
|
|
786
|
+
reason: RequeueReasonType,
|
|
787
|
+
peak: IntegerType,
|
|
788
|
+
reserves: IntegerType,
|
|
789
|
+
}),
|
|
600
790
|
});
|
|
601
791
|
|
|
602
792
|
/**
|
|
@@ -628,6 +818,55 @@ export const DataflowExecutionSummaryType = StructType({
|
|
|
628
818
|
duration: FloatType,
|
|
629
819
|
});
|
|
630
820
|
|
|
821
|
+
/**
|
|
822
|
+
* A server's budget of cores and memory, which every runner it spawns takes
|
|
823
|
+
* from, and what its runners hold of it now.
|
|
824
|
+
*
|
|
825
|
+
* @property cores - Runner processes at once
|
|
826
|
+
* @property memory - Bytes of memory the runners may hold between them
|
|
827
|
+
* @property coresInUse - Runner processes holding the budget now
|
|
828
|
+
* @property memoryInUse - Bytes the runners hold now, each at the larger of
|
|
829
|
+
* its reservation and what it was last measured using
|
|
830
|
+
*/
|
|
831
|
+
export const DataflowBudgetType = StructType({
|
|
832
|
+
cores: IntegerType,
|
|
833
|
+
memory: IntegerType,
|
|
834
|
+
coresInUse: IntegerType,
|
|
835
|
+
memoryInUse: IntegerType,
|
|
836
|
+
});
|
|
837
|
+
|
|
838
|
+
/**
|
|
839
|
+
* A task, or a unit of a split task, waiting for room in the server's budget.
|
|
840
|
+
*
|
|
841
|
+
* @property task - The task
|
|
842
|
+
* @property unit - The unit, for a split task's; `none` for a task run as one
|
|
843
|
+
* @property needs - The memory, in bytes, it waits to reserve; 0 when it
|
|
844
|
+
* waits for a core alone
|
|
845
|
+
* @property since - ISO timestamp when it began waiting
|
|
846
|
+
*/
|
|
847
|
+
export const UnitWaitType = StructType({
|
|
848
|
+
task: StringType,
|
|
849
|
+
unit: OptionType(StageUnitType),
|
|
850
|
+
needs: IntegerType,
|
|
851
|
+
since: StringType,
|
|
852
|
+
});
|
|
853
|
+
|
|
854
|
+
/**
|
|
855
|
+
* A split task's progress through the stage it is in.
|
|
856
|
+
*
|
|
857
|
+
* @property task - The task
|
|
858
|
+
* @property merge - The merge level, from 1, and the levels the merges take;
|
|
859
|
+
* `none` while its pieces run
|
|
860
|
+
* @property done - The stage's units that have finished
|
|
861
|
+
* @property units - The stage's units
|
|
862
|
+
*/
|
|
863
|
+
export const SplitProgressType = StructType({
|
|
864
|
+
task: StringType,
|
|
865
|
+
merge: OptionType(StructType({ level: IntegerType, levels: IntegerType })),
|
|
866
|
+
done: IntegerType,
|
|
867
|
+
units: IntegerType,
|
|
868
|
+
});
|
|
869
|
+
|
|
631
870
|
/**
|
|
632
871
|
* Dataflow execution state returned by API polling.
|
|
633
872
|
*
|
|
@@ -643,6 +882,12 @@ export const DataflowExecutionSummaryType = StructType({
|
|
|
643
882
|
* @property summary - Execution summary (available when complete)
|
|
644
883
|
* @property events - Task events (may be paginated via offset/limit)
|
|
645
884
|
* @property totalEvents - Total number of events (for pagination)
|
|
885
|
+
* @property budget - The server's budget now, where it has one: a server whose
|
|
886
|
+
* runners hold none, as a remote backend's, serves `none`
|
|
887
|
+
* @property waiting - The tasks and units of the run waiting for room, while it
|
|
888
|
+
* runs; nothing stores them
|
|
889
|
+
* @property splits - Each split task's progress, while the run runs; nothing
|
|
890
|
+
* stores it
|
|
646
891
|
*/
|
|
647
892
|
export const ApiDataflowExecutionStateType = StructType({
|
|
648
893
|
status: ApiExecutionStatusType,
|
|
@@ -651,6 +896,9 @@ export const ApiDataflowExecutionStateType = StructType({
|
|
|
651
896
|
summary: OptionType(DataflowExecutionSummaryType),
|
|
652
897
|
events: ArrayType(DataflowEventType),
|
|
653
898
|
totalEvents: IntegerType,
|
|
899
|
+
budget: OptionType(DataflowBudgetType),
|
|
900
|
+
waiting: ArrayType(UnitWaitType),
|
|
901
|
+
splits: ArrayType(SplitProgressType),
|
|
654
902
|
});
|
|
655
903
|
|
|
656
904
|
// =============================================================================
|
|
@@ -658,13 +906,17 @@ export const ApiDataflowExecutionStateType = StructType({
|
|
|
658
906
|
// =============================================================================
|
|
659
907
|
|
|
660
908
|
/**
|
|
661
|
-
* Execution status for history listing
|
|
909
|
+
* Execution status for history listing: `cancelled` when e3 stopped the
|
|
910
|
+
* execution because its run was aborted, `interrupted` when the orchestrator
|
|
911
|
+
* that owned it exited before it finished.
|
|
662
912
|
*/
|
|
663
913
|
export const ExecutionHistoryStatusType = VariantType({
|
|
664
914
|
running: NullType,
|
|
665
915
|
success: NullType,
|
|
666
916
|
failed: NullType,
|
|
667
917
|
error: NullType,
|
|
918
|
+
cancelled: NullType,
|
|
919
|
+
interrupted: NullType,
|
|
668
920
|
});
|
|
669
921
|
|
|
670
922
|
/**
|
|
@@ -677,6 +929,8 @@ export const ExecutionHistoryStatusType = VariantType({
|
|
|
677
929
|
* @property completedAt - ISO timestamp when execution finished (if done)
|
|
678
930
|
* @property duration - Execution duration in milliseconds (if done)
|
|
679
931
|
* @property exitCode - Process exit code (if failed)
|
|
932
|
+
* @property peakBytes - The highest peak resident memory, in bytes, a runner
|
|
933
|
+
* of the execution reached, when it succeeded or failed and one reported it
|
|
680
934
|
*/
|
|
681
935
|
export const ExecutionListItemType = StructType({
|
|
682
936
|
inputsHash: StringType,
|
|
@@ -686,6 +940,7 @@ export const ExecutionListItemType = StructType({
|
|
|
686
940
|
completedAt: OptionType(StringType),
|
|
687
941
|
duration: OptionType(IntegerType),
|
|
688
942
|
exitCode: OptionType(IntegerType),
|
|
943
|
+
peakBytes: OptionType(IntegerType),
|
|
689
944
|
});
|
|
690
945
|
|
|
691
946
|
// =============================================================================
|
|
@@ -703,7 +958,9 @@ export const TreeKindType = VariantType({ struct: NullType });
|
|
|
703
958
|
* A list entry -- either a dataset leaf or a tree branch.
|
|
704
959
|
*
|
|
705
960
|
* Used by the `?list=true&status=true` endpoints to return both
|
|
706
|
-
* tree structure entries and dataset leaves in a single flat list.
|
|
961
|
+
* tree structure entries and dataset leaves in a single flat list. A dataset's
|
|
962
|
+
* `size` is what its value weighs in the store, as its status reports it: for
|
|
963
|
+
* a collection, its segments and its manifest.
|
|
707
964
|
*/
|
|
708
965
|
export const ListEntryType = VariantType({
|
|
709
966
|
dataset: StructType({
|
|
@@ -729,7 +986,8 @@ export const ListEntryType = VariantType({
|
|
|
729
986
|
* @property type - East type of the dataset
|
|
730
987
|
* @property refType - Ref type: "unassigned", "null", or "value"
|
|
731
988
|
* @property hash - Object hash (None if unassigned/null)
|
|
732
|
-
* @property size -
|
|
989
|
+
* @property size - Bytes the value weighs in the store: for a collection, its
|
|
990
|
+
* segments and its manifest (None if unassigned)
|
|
733
991
|
*/
|
|
734
992
|
export const DatasetStatusDetailType = StructType({
|
|
735
993
|
path: StringType,
|
|
@@ -737,10 +995,9 @@ export const DatasetStatusDetailType = StructType({
|
|
|
737
995
|
refType: StringType,
|
|
738
996
|
hash: OptionType(StringType),
|
|
739
997
|
size: OptionType(IntegerType),
|
|
740
|
-
/** Segment and element counts of a stored collection, read from
|
|
741
|
-
*
|
|
742
|
-
*
|
|
743
|
-
* the positional struct rule. */
|
|
998
|
+
/** Segment and element counts of a stored collection, read from its
|
|
999
|
+
* manifest — so a re-pointed input is inspectable without decoding it.
|
|
1000
|
+
* `none` for a non-collection or an unset dataset. */
|
|
744
1001
|
segments: OptionType(IntegerType),
|
|
745
1002
|
rows: OptionType(IntegerType),
|
|
746
1003
|
});
|
|
@@ -834,6 +1091,10 @@ export const OneShotRequestType = StructType({
|
|
|
834
1091
|
/**
|
|
835
1092
|
* A record mutation call. Positional args (after the implicit current state),
|
|
836
1093
|
* one beast2-encoded value per declared parameter.
|
|
1094
|
+
*
|
|
1095
|
+
* Of the limits, `timeoutMs` bounds each run of the mutation's program and
|
|
1096
|
+
* `maxLogBytes` the stderr a failure returns. `maxResultBytes` does not apply:
|
|
1097
|
+
* a mutation's output is stored as segments and never read whole.
|
|
837
1098
|
*/
|
|
838
1099
|
export const MutationCallRequestType = StructType({
|
|
839
1100
|
args: ArrayType(BlobType),
|
|
@@ -846,32 +1107,49 @@ export const MutationCallRequestType = StructType({
|
|
|
846
1107
|
*
|
|
847
1108
|
* - `committed`: the new commit + state hashes
|
|
848
1109
|
* - `invalid`: record/mutation lookup or arity error; nothing ran
|
|
849
|
-
* - `failed`: the
|
|
850
|
-
* - `
|
|
851
|
-
* - `
|
|
852
|
-
*
|
|
1110
|
+
* - `failed`: the mutation's program failed (incl. a body's `$.error`; see stderr)
|
|
1111
|
+
* - `timed_out`: the program exceeded its time budget
|
|
1112
|
+
* - `conflict`: the compare-and-swap lost the race `attempts` times, or the
|
|
1113
|
+
* write disagreed with the state it landed on — then `detail` names the key,
|
|
1114
|
+
* and resubmitting the same write cannot help
|
|
853
1115
|
*/
|
|
854
1116
|
export const MutationResultType = StructType({
|
|
855
1117
|
outcome: VariantType({
|
|
856
1118
|
committed: StructType({ commitHash: StringType, stateHash: StringType }),
|
|
857
1119
|
invalid: StructType({ message: StringType }),
|
|
858
1120
|
failed: StructType({ exitCode: IntegerType, stderr: StringType }),
|
|
859
|
-
too_large: StructType({ bytes: IntegerType, limit: IntegerType, stderr: StringType }),
|
|
860
1121
|
timed_out: StructType({ ms: IntegerType, stderr: StringType }),
|
|
861
|
-
conflict: StructType({ attempts: IntegerType }),
|
|
1122
|
+
conflict: StructType({ attempts: IntegerType, detail: OptionType(StringType) }),
|
|
862
1123
|
}),
|
|
863
1124
|
});
|
|
864
1125
|
|
|
865
1126
|
/**
|
|
866
1127
|
* A record's callable surface, returned by `describe` so dynamic callers can
|
|
867
|
-
* encode mutation arguments
|
|
868
|
-
* after the implicit current state.
|
|
1128
|
+
* encode mutation arguments and read through its indexes. The mutation
|
|
1129
|
+
* `argTypes` are the EXTRA parameters after the implicit current state.
|
|
869
1130
|
*/
|
|
870
1131
|
export const RecordSignatureType = StructType({
|
|
871
1132
|
name: StringType,
|
|
872
1133
|
mutations: ArrayType(StructType({
|
|
873
1134
|
name: StringType,
|
|
874
1135
|
argTypes: ArrayType(EastTypeType),
|
|
1136
|
+
/** The write form — `reduce`, `edit` or `patch` — which says what the
|
|
1137
|
+
* arguments MEAN: a `patch` mutation's one argument is a
|
|
1138
|
+
* `PatchType(State)`, not a value of the record's own type. */
|
|
1139
|
+
form: StringType,
|
|
1140
|
+
})),
|
|
1141
|
+
/** The indexes the record declares. A page read through one is decoded by
|
|
1142
|
+
* `indexWindowType(K, keyType, valueType, V)` over the record's
|
|
1143
|
+
* `Dict<K, V>`, and a key search through one takes a `keyType` key. */
|
|
1144
|
+
indexes: ArrayType(StructType({
|
|
1145
|
+
name: StringType,
|
|
1146
|
+
/** The index key's type. */
|
|
1147
|
+
keyType: EastTypeType,
|
|
1148
|
+
/** The covering projection's type: `Null` when the index projects
|
|
1149
|
+
* nothing, and a read joins the rows instead. */
|
|
1150
|
+
valueType: EastTypeType,
|
|
1151
|
+
/** Whether a row may hold several index keys. */
|
|
1152
|
+
multi: BooleanType,
|
|
875
1153
|
})),
|
|
876
1154
|
});
|
|
877
1155
|
|
|
@@ -883,6 +1161,9 @@ export const RecordCommitInfoType = StructType({
|
|
|
883
1161
|
mutation: StringType,
|
|
884
1162
|
actor: StringType,
|
|
885
1163
|
at: DateTimeType,
|
|
1164
|
+
/** The delta this commit applied, when it wrote one — so history shows WHAT
|
|
1165
|
+
* changed without diffing two states. */
|
|
1166
|
+
delta: OptionType(StringType),
|
|
886
1167
|
});
|
|
887
1168
|
|
|
888
1169
|
/** A page of a record's commit history, newest first. */
|
|
@@ -907,6 +1188,16 @@ export type PackageDetails = ValueTypeOf<typeof PackageDetailsType>;
|
|
|
907
1188
|
export type WorkspaceInfo = ValueTypeOf<typeof WorkspaceInfoType>;
|
|
908
1189
|
export type WorkspaceCreateRequest = ValueTypeOf<typeof WorkspaceCreateRequestType>;
|
|
909
1190
|
export type WorkspaceDeployRequest = ValueTypeOf<typeof WorkspaceDeployRequestType>;
|
|
1191
|
+
/** A {@link SchemaPolicyType} by its name, as a deploy's options take it. */
|
|
1192
|
+
export type SchemaPolicy = ValueTypeOf<typeof SchemaPolicyType>['type'];
|
|
1193
|
+
export type RecordPlan = ValueTypeOf<typeof RecordPlanType>;
|
|
1194
|
+
export type RecordIndexPlan = ValueTypeOf<typeof RecordIndexPlanType>;
|
|
1195
|
+
export type RecordDeployStep = ValueTypeOf<typeof RecordDeployStepType>;
|
|
1196
|
+
export type RecordDeployState = ValueTypeOf<typeof RecordDeployStateType>;
|
|
1197
|
+
export type DeployProgress = ValueTypeOf<typeof DeployProgressType>;
|
|
1198
|
+
export type WorkspaceDeployResult = ValueTypeOf<typeof WorkspaceDeployResultType>;
|
|
1199
|
+
export type WorkspaceDeployProgress = ValueTypeOf<typeof WorkspaceDeployProgressType>;
|
|
1200
|
+
export type WorkspaceDeployStatus = ValueTypeOf<typeof WorkspaceDeployStatusType>;
|
|
910
1201
|
export type DatasetStatus = ValueTypeOf<typeof DatasetStatusType>;
|
|
911
1202
|
export type TaskStatus = ValueTypeOf<typeof TaskStatusType>;
|
|
912
1203
|
export type DatasetStatusInfo = ValueTypeOf<typeof DatasetStatusInfoType>;
|
|
@@ -922,6 +1213,9 @@ export type DataflowResult = ValueTypeOf<typeof DataflowResultType>;
|
|
|
922
1213
|
export type DataflowEvent = ValueTypeOf<typeof DataflowEventType>;
|
|
923
1214
|
export type ApiExecutionStatus = ValueTypeOf<typeof ApiExecutionStatusType>;
|
|
924
1215
|
export type DataflowExecutionSummary = ValueTypeOf<typeof DataflowExecutionSummaryType>;
|
|
1216
|
+
export type DataflowBudget = ValueTypeOf<typeof DataflowBudgetType>;
|
|
1217
|
+
export type UnitWait = ValueTypeOf<typeof UnitWaitType>;
|
|
1218
|
+
export type SplitProgress = ValueTypeOf<typeof SplitProgressType>;
|
|
925
1219
|
export type ApiDataflowExecutionState = ValueTypeOf<typeof ApiDataflowExecutionStateType>;
|
|
926
1220
|
export type ExecutionHistoryStatus = ValueTypeOf<typeof ExecutionHistoryStatusType>;
|
|
927
1221
|
export type ExecutionListItem = ValueTypeOf<typeof ExecutionListItemType>;
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Copyright (c) 2025 Elara AI Pty Ltd
|
|
3
|
+
* Dual-licensed under AGPL-3.0 and commercial license. See LICENSE for details.
|
|
4
|
+
*/
|
|
5
|
+
|
|
6
|
+
/**
|
|
7
|
+
* The segment-object layout: a collection dataset is a **manifest naming
|
|
8
|
+
* standalone segment objects**, not one segmented blob.
|
|
9
|
+
*
|
|
10
|
+
* The type itself lives in `east`, beside the container it describes, because
|
|
11
|
+
* every runtime must read it: a task input staged as a manifest is one file
|
|
12
|
+
* naming sibling segment files, and a body that iterates it pays for the
|
|
13
|
+
* segments it touches rather than for a splice of the whole value. This module
|
|
14
|
+
* re-exports it for the e3 packages and adds what only a store cares about.
|
|
15
|
+
*
|
|
16
|
+
* Three properties are why records at millions of entries are affordable, and
|
|
17
|
+
* every one of them is about successive states SHARING storage rather than
|
|
18
|
+
* about indexing, which the trailing index already gave:
|
|
19
|
+
*
|
|
20
|
+
* - **A one-row edit costs one segment.** The new state names the same objects
|
|
21
|
+
* as the old one bar the segment that changed, so a commit's new storage is
|
|
22
|
+
* the touched segments plus a manifest — not the whole value.
|
|
23
|
+
* - **Two states diff in O(changed segments)**, by comparing entry hashes. No
|
|
24
|
+
* value is decoded to find out what moved, which is what index maintenance
|
|
25
|
+
* and the mutation delta are built on.
|
|
26
|
+
* - **A segment can be checked against its own hash**, which a ranged read of
|
|
27
|
+
* one blob can never offer: no digest exists for a byte range. No reader
|
|
28
|
+
* checks it — a segment is trusted as stored, like any other object.
|
|
29
|
+
*
|
|
30
|
+
* The manifest is the envelope the garbage collector recognizes, which is why
|
|
31
|
+
* its `kind` is a stored field rather than something inferred: an object whose
|
|
32
|
+
* entries name other objects MUST be traversable by shape, or the first sweep
|
|
33
|
+
* deletes a record's segments.
|
|
34
|
+
*
|
|
35
|
+
* @packageDocumentation
|
|
36
|
+
*/
|
|
37
|
+
|
|
38
|
+
export {
|
|
39
|
+
COLLECTION_MANIFEST_KIND,
|
|
40
|
+
CollectionManifestType,
|
|
41
|
+
CollectionManifestEntryType,
|
|
42
|
+
type CollectionManifest,
|
|
43
|
+
type CollectionManifestEntry,
|
|
44
|
+
type Beast2ManifestSource,
|
|
45
|
+
isCollectionManifestType,
|
|
46
|
+
isCollectionManifest,
|
|
47
|
+
isBeast2ManifestSource,
|
|
48
|
+
encodeCollectionManifest,
|
|
49
|
+
decodeCollectionManifest,
|
|
50
|
+
readBeast2Manifest,
|
|
51
|
+
manifestElementCount,
|
|
52
|
+
manifestByteSize,
|
|
53
|
+
} from '@elaraai/east';
|