@elaraai/e3-types 1.0.79 → 1.0.81
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/src/api.d.ts +829 -36
- package/dist/src/api.d.ts.map +1 -1
- package/dist/src/api.js +301 -32
- package/dist/src/api.js.map +1 -1
- package/dist/src/collection-manifest.d.ts +37 -0
- package/dist/src/collection-manifest.d.ts.map +1 -0
- package/dist/src/collection-manifest.js +37 -0
- package/dist/src/collection-manifest.js.map +1 -0
- package/dist/src/dataflow.d.ts +264 -19
- package/dist/src/dataflow.d.ts.map +1 -1
- package/dist/src/dataflow.js +156 -15
- package/dist/src/dataflow.js.map +1 -1
- package/dist/src/dataflow.spec.js +184 -27
- package/dist/src/dataflow.spec.js.map +1 -1
- package/dist/src/dataset-blob.d.ts +75 -15
- package/dist/src/dataset-blob.d.ts.map +1 -1
- package/dist/src/dataset-blob.js +95 -28
- package/dist/src/dataset-blob.js.map +1 -1
- package/dist/src/dataset-blob.spec.d.ts +6 -0
- package/dist/src/dataset-blob.spec.d.ts.map +1 -0
- package/dist/src/dataset-blob.spec.js +92 -0
- package/dist/src/dataset-blob.spec.js.map +1 -0
- package/dist/src/dataset-ref.d.ts +5 -4
- package/dist/src/dataset-ref.d.ts.map +1 -1
- package/dist/src/dataset-ref.js +5 -4
- package/dist/src/dataset-ref.js.map +1 -1
- package/dist/src/environment.d.ts +3 -2
- package/dist/src/environment.d.ts.map +1 -1
- package/dist/src/environment.js +3 -2
- package/dist/src/environment.js.map +1 -1
- package/dist/src/execution.d.ts +112 -18
- package/dist/src/execution.d.ts.map +1 -1
- package/dist/src/execution.js +174 -45
- package/dist/src/execution.js.map +1 -1
- package/dist/src/execution.spec.d.ts +6 -0
- package/dist/src/execution.spec.d.ts.map +1 -0
- package/dist/src/execution.spec.js +108 -0
- package/dist/src/execution.spec.js.map +1 -0
- package/dist/src/function.d.ts +14 -12
- package/dist/src/function.d.ts.map +1 -1
- package/dist/src/function.js +17 -32
- package/dist/src/function.js.map +1 -1
- package/dist/src/index.d.ts +17 -12
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/index.js +28 -20
- package/dist/src/index.js.map +1 -1
- package/dist/src/intake.d.ts +75 -0
- package/dist/src/intake.d.ts.map +1 -0
- package/dist/src/intake.js +46 -0
- package/dist/src/intake.js.map +1 -0
- package/dist/src/lock.d.ts +253 -21
- package/dist/src/lock.d.ts.map +1 -1
- package/dist/src/lock.js +55 -21
- package/dist/src/lock.js.map +1 -1
- package/dist/src/names.d.ts +21 -0
- package/dist/src/names.d.ts.map +1 -0
- package/dist/src/names.js +31 -0
- package/dist/src/names.js.map +1 -0
- package/dist/src/package.d.ts +11 -78
- package/dist/src/package.d.ts.map +1 -1
- package/dist/src/package.js +13 -66
- package/dist/src/package.js.map +1 -1
- package/dist/src/record.d.ts +582 -18
- package/dist/src/record.d.ts.map +1 -1
- package/dist/src/record.js +405 -7
- package/dist/src/record.js.map +1 -1
- package/dist/src/{stream.spec.d.ts → record.spec.d.ts} +1 -1
- package/dist/src/record.spec.d.ts.map +1 -0
- package/dist/src/record.spec.js +143 -0
- package/dist/src/record.spec.js.map +1 -0
- package/dist/src/release.d.ts +36 -0
- package/dist/src/release.d.ts.map +1 -0
- package/dist/src/release.js +71 -0
- package/dist/src/release.js.map +1 -0
- package/dist/src/release.spec.d.ts +6 -0
- package/dist/src/release.spec.d.ts.map +1 -0
- package/dist/src/release.spec.js +42 -0
- package/dist/src/release.spec.js.map +1 -0
- package/dist/src/repository.d.ts +64 -0
- package/dist/src/repository.d.ts.map +1 -0
- package/dist/src/repository.js +49 -0
- package/dist/src/repository.js.map +1 -0
- package/dist/src/runner.d.ts +16 -62
- package/dist/src/runner.d.ts.map +1 -1
- package/dist/src/runner.js +16 -79
- package/dist/src/runner.js.map +1 -1
- package/dist/src/runner.spec.js +11 -85
- package/dist/src/runner.spec.js.map +1 -1
- package/dist/src/structure.d.ts +0 -57
- package/dist/src/structure.d.ts.map +1 -1
- package/dist/src/structure.js +0 -3
- package/dist/src/structure.js.map +1 -1
- package/dist/src/task.d.ts +210 -318
- package/dist/src/task.d.ts.map +1 -1
- package/dist/src/task.js +117 -472
- package/dist/src/task.js.map +1 -1
- package/dist/src/task.spec.js +62 -154
- package/dist/src/task.spec.js.map +1 -1
- package/dist/src/transfer.d.ts +38 -26
- package/dist/src/transfer.d.ts.map +1 -1
- package/dist/src/transfer.js +22 -26
- package/dist/src/transfer.js.map +1 -1
- package/dist/src/transfer.spec.js +4 -34
- package/dist/src/transfer.spec.js.map +1 -1
- package/dist/src/unit-plan.d.ts +130 -0
- package/dist/src/unit-plan.d.ts.map +1 -0
- package/dist/src/unit-plan.js +107 -0
- package/dist/src/unit-plan.js.map +1 -0
- package/dist/src/unit-plan.spec.d.ts +6 -0
- package/dist/src/unit-plan.spec.d.ts.map +1 -0
- package/dist/src/unit-plan.spec.js +29 -0
- package/dist/src/unit-plan.spec.js.map +1 -0
- package/dist/src/workspace.d.ts +22 -11
- package/dist/src/workspace.d.ts.map +1 -1
- package/dist/src/workspace.js +10 -11
- package/dist/src/workspace.js.map +1 -1
- package/package.json +3 -3
- package/src/api.ts +326 -32
- package/src/collection-manifest.ts +53 -0
- package/src/dataflow.spec.ts +223 -31
- package/src/dataflow.ts +183 -26
- package/src/dataset-blob.spec.ts +117 -0
- package/src/dataset-blob.ts +157 -21
- package/src/dataset-ref.ts +5 -4
- package/src/environment.ts +3 -2
- package/src/execution.spec.ts +114 -0
- package/src/execution.ts +175 -51
- package/src/function.ts +19 -32
- package/src/index.ts +141 -39
- package/src/intake.ts +53 -0
- package/src/lock.ts +67 -21
- package/src/names.ts +39 -0
- package/src/package.ts +15 -71
- package/src/record.spec.ts +195 -0
- package/src/record.ts +489 -7
- package/src/release.spec.ts +47 -0
- package/src/release.ts +69 -0
- package/src/repository.ts +67 -0
- package/src/runner.spec.ts +11 -94
- package/src/runner.ts +16 -80
- package/src/structure.ts +0 -8
- package/src/task.spec.ts +67 -188
- package/src/task.ts +140 -522
- package/src/transfer.spec.ts +4 -51
- package/src/transfer.ts +22 -26
- package/src/unit-plan.spec.ts +31 -0
- package/src/unit-plan.ts +121 -0
- package/src/workspace.ts +13 -11
- package/dist/src/stream.d.ts +0 -97
- package/dist/src/stream.d.ts.map +0 -1
- package/dist/src/stream.js +0 -134
- package/dist/src/stream.js.map +0 -1
- package/dist/src/stream.spec.d.ts.map +0 -1
- package/dist/src/stream.spec.js +0 -60
- package/dist/src/stream.spec.js.map +0 -1
- package/src/stream.spec.ts +0 -88
- package/src/stream.ts +0 -170
package/src/dataflow.spec.ts
CHANGED
|
@@ -4,41 +4,233 @@
|
|
|
4
4
|
*/
|
|
5
5
|
|
|
6
6
|
/**
|
|
7
|
-
*
|
|
8
|
-
*
|
|
9
|
-
*
|
|
10
|
-
*
|
|
11
|
-
* list below IS the wire of every persisted `execution.beast2`: adding,
|
|
12
|
-
* removing or renaming a case shifts later indices and released readers
|
|
13
|
-
* mis-decode (or fail to decode) existing states. This spec exists to make
|
|
14
|
-
* that impossible to do silently — see the wire warning in `dataflow.ts`.
|
|
7
|
+
* The execution state's form. A stored state is decoded against the whole
|
|
8
|
+
* type it was written with, so each state here is one an e3 wrote: this
|
|
9
|
+
* release's form reads, and every other is refused, naming the release that
|
|
10
|
+
* wrote it — or, for a state from before states recorded one, an older e3.
|
|
15
11
|
*/
|
|
16
12
|
|
|
17
13
|
import { describe, it } from 'node:test';
|
|
18
14
|
import assert from 'node:assert/strict';
|
|
19
|
-
import {
|
|
20
|
-
import {
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
15
|
+
import { ArrayType, DictType, StringType, StructType, encodeBeast2For, none, some, variant } from '@elaraai/east';
|
|
16
|
+
import { decodeDataflowExecutionState } from './dataflow.js';
|
|
17
|
+
import { E3_RELEASE } from './release.js';
|
|
18
|
+
|
|
19
|
+
/** Version 1, written by the released e3: task `double` completed, `sum`
|
|
20
|
+
* ready, and four events. */
|
|
21
|
+
const STATE_V1 = Buffer.from(
|
|
22
|
+
'iUVhc3QNCgUAnwgeHwEFAgQACAIEbm9uZQQEc29tZQAKAAkFBG5hbWUABGhhc2gABmlucHV0cwYGb3V0cHV0AAlkZXBlbmRz' +
|
|
23
|
+
'T24GCgcJAQV0YXNrcwgIAgRub25lBARzb21lCQgCBG5vbmUEBHNvbWUDCAIEbm9uZQQEc29tZQIIAgRub25lBARzb21lAQkJ' +
|
|
24
|
+
'BG5hbWUABnN0YXR1cwAGY2FjaGVkCwpvdXRwdXRIYXNoBQVlcnJvcgUIZXhpdENvZGUMCXN0YXJ0ZWRBdA0LY29tcGxldGVk' +
|
|
25
|
+
'QXQNCGR1cmF0aW9uDAsADgsAAAsAEAkDA3NlcQIJdGltZXN0YW1wAQZyZWFzb24FCQgDc2VxAgl0aW1lc3RhbXABB3N1Y2Nl' +
|
|
26
|
+
'c3MDCGV4ZWN1dGVkAgZjYWNoZWQCBmZhaWxlZAIHc2tpcHBlZAIIZHVyYXRpb24CCQQDc2VxAgl0aW1lc3RhbXABC2V4ZWN1' +
|
|
27
|
+
'dGlvbklkAAp0b3RhbFRhc2tzAgkFA3NlcQIJdGltZXN0YW1wAQRwYXRoAAxwcmV2aW91c0hhc2gAB25ld0hhc2gACQYDc2Vx' +
|
|
28
|
+
'Agl0aW1lc3RhbXABBHRhc2sABmNhY2hlZAMKb3V0cHV0SGFzaAAIZHVyYXRpb24CCQQDc2VxAgl0aW1lc3RhbXABBHRhc2sA' +
|
|
29
|
+
'DGNvbmZsaWN0UGF0aAAJBgNzZXECCXRpbWVzdGFtcAEEdGFzawAFZXJyb3IFCGV4aXRDb2RlDAhkdXJhdGlvbgIJBANzZXEC' +
|
|
30
|
+
'CXRpbWVzdGFtcAEEdGFzawAGcmVhc29uAAkDA3NlcQIJdGltZXN0YW1wAQR0YXNrAAkEA3NlcQIJdGltZXN0YW1wAQR0YXNr' +
|
|
31
|
+
'AAVjYXVzZQAICxNleGVjdXRpb25fY2FuY2VsbGVkEhNleGVjdXRpb25fY29tcGxldGVkExFleGVjdXRpb25fc3RhcnRlZBQN' +
|
|
32
|
+
'aW5wdXRfY2hhbmdlZBUOdGFza19jb21wbGV0ZWQWDXRhc2tfZGVmZXJyZWQXC3Rhc2tfZmFpbGVkGBB0YXNrX2ludmFsaWRh' +
|
|
33
|
+
'dGVkGQp0YXNrX3JlYWR5Ggx0YXNrX3NraXBwZWQbDHRhc2tfc3RhcnRlZBoKHAkXAmlkAARyZXBvAAl3b3Jrc3BhY2UACXN0' +
|
|
34
|
+
'YXJ0ZWRBdAELY29uY3VycmVuY3kCBWZvcmNlAwZmaWx0ZXIFBWdyYXBoCglncmFwaEhhc2gFBXRhc2tzDwhleGVjdXRlZAIG' +
|
|
35
|
+
'Y2FjaGVkAgZmYWlsZWQCB3NraXBwZWQCBnN0YXR1cwALY29tcGxldGVkQXQNBWVycm9yBQ52ZXJzaW9uVmVjdG9ycxENaW5w' +
|
|
36
|
+
'dXRTbmFwc2hvdBAPdGFza091dHB1dFBhdGhzBgpyZWV4ZWN1dGVkAgZldmVudHMdCGV2ZW50U2VxAgEAAcoCtgFjNGcpSi3I' +
|
|
37
|
+
'ZyovnrDpzKn3aRwMDIwMTGwp+aVJOalMiYYMjJx6mXkFpSXFehUMvHolicXZxXoQWQYG5uLSXKYkIwZGNAkuKBcozcAINYsB' +
|
|
38
|
+
'CGDmQinO5PzcgpzUktQURgZGpmRjoNUQR8AoCZAFIMxalJqYUgkyAmwMELMXleblZealA5nILkRiM6WYoEqCBZhQnYrsUgYG' +
|
|
39
|
+
'FiYGqM3mLFxMECbUrSwsKFwGoHMlONggYiAXMnAAAA==',
|
|
40
|
+
'base64',
|
|
41
|
+
);
|
|
42
|
+
|
|
43
|
+
/** Version 2: task `sum` split into pieces, its `$plan` named, and the
|
|
44
|
+
* events of its stages. */
|
|
45
|
+
const STATE_V2 = Buffer.from(
|
|
46
|
+
'iUVhc3QNCgUA2QkhIgIBBQQACAIEbm9uZQQEc29tZQEKAQkFBG5hbWUBBGhhc2gBBmlucHV0cwYGb3V0cHV0AQlkZXBlbmRz' +
|
|
47
|
+
'T24GCgcJAQV0YXNrcwgIAgRub25lBARzb21lCQgCBG5vbmUEBHNvbWUDCAIEbm9uZQQEc29tZQAIAgRub25lBARzb21lAgkK' +
|
|
48
|
+
'BG5hbWUBBnN0YXR1cwEGY2FjaGVkCwpvdXRwdXRIYXNoBQVlcnJvcgUIZXhpdENvZGUMCXN0YXJ0ZWRBdA0LY29tcGxldGVk' +
|
|
49
|
+
'QXQNCGR1cmF0aW9uDARwbGFuBQsBDgsBAQsBEAkDA3NlcQAJdGltZXN0YW1wAgZyZWFzb24FCQgDc2VxAAl0aW1lc3RhbXAC' +
|
|
50
|
+
'B3N1Y2Nlc3MDCGV4ZWN1dGVkAAZjYWNoZWQABmZhaWxlZAAHc2tpcHBlZAAIZHVyYXRpb24ACQQDc2VxAAl0aW1lc3RhbXAC' +
|
|
51
|
+
'C2V4ZWN1dGlvbklkAQp0b3RhbFRhc2tzAAkFA3NlcQAJdGltZXN0YW1wAgRwYXRoAQxwcmV2aW91c0hhc2gBB25ld0hhc2gB' +
|
|
52
|
+
'CQYDc2VxAAl0aW1lc3RhbXACBHRhc2sBBmNhY2hlZAMKb3V0cHV0SGFzaAEIZHVyYXRpb24ACQQDc2VxAAl0aW1lc3RhbXAC' +
|
|
53
|
+
'BHRhc2sBDGNvbmZsaWN0UGF0aAEJBgNzZXEACXRpbWVzdGFtcAIEdGFzawEFZXJyb3IFCGV4aXRDb2RlDAhkdXJhdGlvbgAJ' +
|
|
54
|
+
'BANzZXEACXRpbWVzdGFtcAIEdGFzawEGcmVhc29uAQkFA3NlcQAJdGltZXN0YW1wAgR0YXNrAQVsZXZlbAAGbGV2ZWxzAAkG' +
|
|
55
|
+
'A3NlcQAJdGltZXN0YW1wAgR0YXNrAQVsZXZlbAAGbGV2ZWxzAAV1bml0cwAJAwNzZXEACXRpbWVzdGFtcAIEdGFzawEJBANz' +
|
|
56
|
+
'ZXEACXRpbWVzdGFtcAIEdGFzawEFY2F1c2UBCQQDc2VxAAl0aW1lc3RhbXACBHRhc2sBBnBpZWNlcwAIDhNleGVjdXRpb25f' +
|
|
57
|
+
'Y2FuY2VsbGVkEhNleGVjdXRpb25fY29tcGxldGVkExFleGVjdXRpb25fc3RhcnRlZBQNaW5wdXRfY2hhbmdlZBUOdGFza19j' +
|
|
58
|
+
'b21wbGV0ZWQWDXRhc2tfZGVmZXJyZWQXC3Rhc2tfZmFpbGVkGBB0YXNrX2ludmFsaWRhdGVkGRR0YXNrX21lcmdlX2NvbXBs' +
|
|
59
|
+
'ZXRlZBoSdGFza19tZXJnZV9zdGFydGVkGwp0YXNrX3JlYWR5HAx0YXNrX3NraXBwZWQdCnRhc2tfc3BsaXQeDHRhc2tfc3Rh' +
|
|
60
|
+
'cnRlZBwKHwkYB3ZlcnNpb24AAmlkAQRyZXBvAQl3b3Jrc3BhY2UBCXN0YXJ0ZWRBdAILY29uY3VycmVuY3kABWZvcmNlAwZm' +
|
|
61
|
+
'aWx0ZXIFBWdyYXBoCglncmFwaEhhc2gFBXRhc2tzDwhleGVjdXRlZAAGY2FjaGVkAAZmYWlsZWQAB3NraXBwZWQABnN0YXR1' +
|
|
62
|
+
'cwELY29tcGxldGVkQXQNBWVycm9yBQ52ZXJzaW9uVmVjdG9ycxENaW5wdXRTbmFwc2hvdBAPdGFza091dHB1dFBhdGhzBgpy' +
|
|
63
|
+
'ZWV4ZWN1dGVkAAZldmVudHMgCGV2ZW50U2VxAAEAAfEBjAFjYTRnKUotyGcqL56w6cyp92kcDAyMDIzMxaW5TElGDIycepl5' +
|
|
64
|
+
'BaUlxXoVDFx6JYnF2cV6QCkGEAArAmHuzLz4gqL89KLU4mKwBMQkIIMp1RSslL2oNC8vMy8dJIlkIhKbKcUEVRIswIhqJysT' +
|
|
65
|
+
'A8RoRnMmXiYIE+QAHhYEm42TDcFhYmLi4EDmMnABAA==',
|
|
66
|
+
'base64',
|
|
67
|
+
);
|
|
68
|
+
|
|
69
|
+
/** Version 3: the version 2 state, without its `concurrency`. */
|
|
70
|
+
const STATE_V3 = Buffer.from(
|
|
71
|
+
'iUVhc3QNCgUAzAkhIgIBBQQACAIEbm9uZQQEc29tZQEKAQkFBG5hbWUBBGhhc2gBBmlucHV0cwYGb3V0cHV0AQlkZXBlbmRz' +
|
|
72
|
+
'T24GCgcJAQV0YXNrcwgIAgRub25lBARzb21lCQgCBG5vbmUEBHNvbWUDCAIEbm9uZQQEc29tZQAIAgRub25lBARzb21lAgkK' +
|
|
73
|
+
'BG5hbWUBBnN0YXR1cwEGY2FjaGVkCwpvdXRwdXRIYXNoBQVlcnJvcgUIZXhpdENvZGUMCXN0YXJ0ZWRBdA0LY29tcGxldGVk' +
|
|
74
|
+
'QXQNCGR1cmF0aW9uDARwbGFuBQsBDgsBAQsBEAkDA3NlcQAJdGltZXN0YW1wAgZyZWFzb24FCQgDc2VxAAl0aW1lc3RhbXAC' +
|
|
75
|
+
'B3N1Y2Nlc3MDCGV4ZWN1dGVkAAZjYWNoZWQABmZhaWxlZAAHc2tpcHBlZAAIZHVyYXRpb24ACQQDc2VxAAl0aW1lc3RhbXAC' +
|
|
76
|
+
'C2V4ZWN1dGlvbklkAQp0b3RhbFRhc2tzAAkFA3NlcQAJdGltZXN0YW1wAgRwYXRoAQxwcmV2aW91c0hhc2gBB25ld0hhc2gB' +
|
|
77
|
+
'CQYDc2VxAAl0aW1lc3RhbXACBHRhc2sBBmNhY2hlZAMKb3V0cHV0SGFzaAEIZHVyYXRpb24ACQQDc2VxAAl0aW1lc3RhbXAC' +
|
|
78
|
+
'BHRhc2sBDGNvbmZsaWN0UGF0aAEJBgNzZXEACXRpbWVzdGFtcAIEdGFzawEFZXJyb3IFCGV4aXRDb2RlDAhkdXJhdGlvbgAJ' +
|
|
79
|
+
'BANzZXEACXRpbWVzdGFtcAIEdGFzawEGcmVhc29uAQkFA3NlcQAJdGltZXN0YW1wAgR0YXNrAQVsZXZlbAAGbGV2ZWxzAAkG' +
|
|
80
|
+
'A3NlcQAJdGltZXN0YW1wAgR0YXNrAQVsZXZlbAAGbGV2ZWxzAAV1bml0cwAJAwNzZXEACXRpbWVzdGFtcAIEdGFzawEJBANz' +
|
|
81
|
+
'ZXEACXRpbWVzdGFtcAIEdGFzawEFY2F1c2UBCQQDc2VxAAl0aW1lc3RhbXACBHRhc2sBBnBpZWNlcwAIDhNleGVjdXRpb25f' +
|
|
82
|
+
'Y2FuY2VsbGVkEhNleGVjdXRpb25fY29tcGxldGVkExFleGVjdXRpb25fc3RhcnRlZBQNaW5wdXRfY2hhbmdlZBUOdGFza19j' +
|
|
83
|
+
'b21wbGV0ZWQWDXRhc2tfZGVmZXJyZWQXC3Rhc2tfZmFpbGVkGBB0YXNrX2ludmFsaWRhdGVkGRR0YXNrX21lcmdlX2NvbXBs' +
|
|
84
|
+
'ZXRlZBoSdGFza19tZXJnZV9zdGFydGVkGwp0YXNrX3JlYWR5HAx0YXNrX3NraXBwZWQdCnRhc2tfc3BsaXQeDHRhc2tfc3Rh' +
|
|
85
|
+
'cnRlZBwKHwkXB3ZlcnNpb24AAmlkAQRyZXBvAQl3b3Jrc3BhY2UBCXN0YXJ0ZWRBdAIFZm9yY2UDBmZpbHRlcgUFZ3JhcGgK' +
|
|
86
|
+
'CWdyYXBoSGFzaAUFdGFza3MPCGV4ZWN1dGVkAAZjYWNoZWQABmZhaWxlZAAHc2tpcHBlZAAGc3RhdHVzAQtjb21wbGV0ZWRB' +
|
|
87
|
+
'dA0FZXJyb3IFDnZlcnNpb25WZWN0b3JzEQ1pbnB1dFNuYXBzaG90EA90YXNrT3V0cHV0UGF0aHMGCnJlZXhlY3V0ZWQABmV2' +
|
|
88
|
+
'ZW50cyAIZXZlbnRTZXEAAQAB8AGJAWNjNGcpSi3IZyovnrDpzKn3aQwMjAyMzMWluUxJRgyMnHqZeQWlJcV6FQxceiWJxdnF' +
|
|
89
|
+
'ekApBhAAKwJh7sy8+IKi/PSi1OJisATcIKZUU7BS9qLSvLzMvHSQJJKJSGymFBNUSbAAI6qdrEwMEKMZzZl4mSBMkAN4WBBs' +
|
|
90
|
+
'Nk42BIeJiYmDA5nLwAUA',
|
|
91
|
+
'base64',
|
|
92
|
+
);
|
|
93
|
+
|
|
94
|
+
/** Version 4: the version 3 state, each task with the execution it completed
|
|
95
|
+
* with — `double`, completed since, by its inputs hash and id. */
|
|
96
|
+
const STATE_V4 = Buffer.from(
|
|
97
|
+
'iUVhc3QNCgUAgAojJAIBBQQACAIEbm9uZQQEc29tZQEKAQkFBG5hbWUBBGhhc2gBBmlucHV0cwYGb3V0cHV0AQlkZXBlbmRz' +
|
|
98
|
+
'T24GCgcJAQV0YXNrcwgIAgRub25lBARzb21lCQgCBG5vbmUEBHNvbWUDCAIEbm9uZQQEc29tZQAIAgRub25lBARzb21lAgkC' +
|
|
99
|
+
'CmlucHV0c0hhc2gBC2V4ZWN1dGlvbklkAQgCBG5vbmUEBHNvbWUOCQsEbmFtZQEGc3RhdHVzAQZjYWNoZWQLCm91dHB1dEhh' +
|
|
100
|
+
'c2gFBWVycm9yBQhleGl0Q29kZQwJc3RhcnRlZEF0DQtjb21wbGV0ZWRBdA0IZHVyYXRpb24MBHBsYW4FCWV4ZWN1dGlvbg8L' +
|
|
101
|
+
'ARALAQELARIJAwNzZXEACXRpbWVzdGFtcAIGcmVhc29uBQkIA3NlcQAJdGltZXN0YW1wAgdzdWNjZXNzAwhleGVjdXRlZAAG' +
|
|
102
|
+
'Y2FjaGVkAAZmYWlsZWQAB3NraXBwZWQACGR1cmF0aW9uAAkEA3NlcQAJdGltZXN0YW1wAgtleGVjdXRpb25JZAEKdG90YWxU' +
|
|
103
|
+
'YXNrcwAJBQNzZXEACXRpbWVzdGFtcAIEcGF0aAEMcHJldmlvdXNIYXNoAQduZXdIYXNoAQkGA3NlcQAJdGltZXN0YW1wAgR0' +
|
|
104
|
+
'YXNrAQZjYWNoZWQDCm91dHB1dEhhc2gBCGR1cmF0aW9uAAkEA3NlcQAJdGltZXN0YW1wAgR0YXNrAQxjb25mbGljdFBhdGgB' +
|
|
105
|
+
'CQYDc2VxAAl0aW1lc3RhbXACBHRhc2sBBWVycm9yBQhleGl0Q29kZQwIZHVyYXRpb24ACQQDc2VxAAl0aW1lc3RhbXACBHRh' +
|
|
106
|
+
'c2sBBnJlYXNvbgEJBQNzZXEACXRpbWVzdGFtcAIEdGFzawEFbGV2ZWwABmxldmVscwAJBgNzZXEACXRpbWVzdGFtcAIEdGFz' +
|
|
107
|
+
'awEFbGV2ZWwABmxldmVscwAFdW5pdHMACQMDc2VxAAl0aW1lc3RhbXACBHRhc2sBCQQDc2VxAAl0aW1lc3RhbXACBHRhc2sB' +
|
|
108
|
+
'BWNhdXNlAQkEA3NlcQAJdGltZXN0YW1wAgR0YXNrAQZwaWVjZXMACA4TZXhlY3V0aW9uX2NhbmNlbGxlZBQTZXhlY3V0aW9u' +
|
|
109
|
+
'X2NvbXBsZXRlZBURZXhlY3V0aW9uX3N0YXJ0ZWQWDWlucHV0X2NoYW5nZWQXDnRhc2tfY29tcGxldGVkGA10YXNrX2RlZmVy' +
|
|
110
|
+
'cmVkGQt0YXNrX2ZhaWxlZBoQdGFza19pbnZhbGlkYXRlZBsUdGFza19tZXJnZV9jb21wbGV0ZWQcEnRhc2tfbWVyZ2Vfc3Rh' +
|
|
111
|
+
'cnRlZB0KdGFza19yZWFkeR4MdGFza19za2lwcGVkHwp0YXNrX3NwbGl0IAx0YXNrX3N0YXJ0ZWQeCiEJFwd2ZXJzaW9uAAJp' +
|
|
112
|
+
'ZAEEcmVwbwEJd29ya3NwYWNlAQlzdGFydGVkQXQCBWZvcmNlAwZmaWx0ZXIFBWdyYXBoCglncmFwaEhhc2gFBXRhc2tzEQhl' +
|
|
113
|
+
'eGVjdXRlZAAGY2FjaGVkAAZmYWlsZWQAB3NraXBwZWQABnN0YXR1cwELY29tcGxldGVkQXQNBWVycm9yBQ52ZXJzaW9uVmVj' +
|
|
114
|
+
'dG9ycxMNaW5wdXRTbmFwc2hvdBIPdGFza091dHB1dFBhdGhzBgpyZWV4ZWN1dGVkAAZldmVudHMiCGV2ZW50U2VxAAEAAYkD' +
|
|
115
|
+
'xwHjYDRnKUotyGcqL56w6cyp92kMDIwMjMzFpblMSUYMjJx6mXkFpSXFehUMXHolicXZxXpAKQYQYGJLyS9NykmFUpzJ+bkF' +
|
|
116
|
+
'OaklqSlA/UwpJkBzIAbCKC4GRockCoGKgaGlQaJBkoGuBRDomhsYAFkgAohhwBDkeBDmzsyLLyjKTy9KLS4GORjqECCDKdUU' +
|
|
117
|
+
'7AUG9qLSvLzMvHSQLJJXkdhgn2AIMKIGBisTA9ST5ky8TBAmyAU8LAg2GycbgsPExMTBgcxl4AIA',
|
|
118
|
+
'base64',
|
|
119
|
+
);
|
|
120
|
+
|
|
121
|
+
/** Version 5: task `double` completed with the peak its runners reached, and
|
|
122
|
+
* a piece of `sum` stopped past the budget and requeued. */
|
|
123
|
+
const STATE_V5 = Buffer.from(
|
|
124
|
+
'iUVhc3QNCgUAngsoKQIBBQQACAIEbm9uZQQEc29tZQEKAQkFBG5hbWUBBGhhc2gBBmlucHV0cwYGb3V0cHV0AQlkZXBlbmRz' +
|
|
125
|
+
'T24GCgcJAQV0YXNrcwgIAgRub25lBARzb21lCQgCBG5vbmUEBHNvbWUDCAIEbm9uZQQEc29tZQAIAgRub25lBARzb21lAgkC' +
|
|
126
|
+
'CmlucHV0c0hhc2gBC2V4ZWN1dGlvbklkAQgCBG5vbmUEBHNvbWUOCQsEbmFtZQEGc3RhdHVzAQZjYWNoZWQLCm91dHB1dEhh' +
|
|
127
|
+
'c2gFBWVycm9yBQhleGl0Q29kZQwJc3RhcnRlZEF0DQtjb21wbGV0ZWRBdA0IZHVyYXRpb24MBHBsYW4FCWV4ZWN1dGlvbg8L' +
|
|
128
|
+
'ARALAQELARIJAwNzZXEACXRpbWVzdGFtcAIGcmVhc29uBQkIA3NlcQAJdGltZXN0YW1wAgdzdWNjZXNzAwhleGVjdXRlZAAG' +
|
|
129
|
+
'Y2FjaGVkAAZmYWlsZWQAB3NraXBwZWQACGR1cmF0aW9uAAkEA3NlcQAJdGltZXN0YW1wAgtleGVjdXRpb25JZAEKdG90YWxU' +
|
|
130
|
+
'YXNrcwAJBQNzZXEACXRpbWVzdGFtcAIEcGF0aAEMcHJldmlvdXNIYXNoAQduZXdIYXNoAQkHA3NlcQAJdGltZXN0YW1wAgR0' +
|
|
131
|
+
'YXNrAQZjYWNoZWQDCm91dHB1dEhhc2gBCGR1cmF0aW9uAAlwZWFrQnl0ZXMMCQQDc2VxAAl0aW1lc3RhbXACBHRhc2sBDGNv' +
|
|
132
|
+
'bmZsaWN0UGF0aAEJBgNzZXEACXRpbWVzdGFtcAIEdGFzawEFZXJyb3IFCGV4aXRDb2RlDAhkdXJhdGlvbgAJBANzZXEACXRp' +
|
|
133
|
+
'bWVzdGFtcAIEdGFzawEGcmVhc29uAQkFA3NlcQAJdGltZXN0YW1wAgR0YXNrAQVsZXZlbAAGbGV2ZWxzAAkGA3NlcQAJdGlt' +
|
|
134
|
+
'ZXN0YW1wAgR0YXNrAQVsZXZlbAAGbGV2ZWxzAAV1bml0cwAJAwNzZXEACXRpbWVzdGFtcAIEdGFzawEJBANzZXEACXRpbWVz' +
|
|
135
|
+
'dGFtcAIEdGFzawEFY2F1c2UBCQQDc2VxAAl0aW1lc3RhbXACBHRhc2sBBnBpZWNlcwAJAgVsZXZlbAAGbGV2ZWxzAAgCBG5v' +
|
|
136
|
+
'bmUEBHNvbWUhCQMFbWVyZ2UiBWluZGV4AAV1bml0cwAIAwZidWRnZXQEA2NhcAQHbWFjaGluZQQJBwNzZXEACXRpbWVzdGFt' +
|
|
137
|
+
'cAIEdGFzawEEdW5pdCMGcmVhc29uJARwZWFrAAhyZXNlcnZlcwAIDxNleGVjdXRpb25fY2FuY2VsbGVkFBNleGVjdXRpb25f' +
|
|
138
|
+
'Y29tcGxldGVkFRFleGVjdXRpb25fc3RhcnRlZBYNaW5wdXRfY2hhbmdlZBcOdGFza19jb21wbGV0ZWQYDXRhc2tfZGVmZXJy' +
|
|
139
|
+
'ZWQZC3Rhc2tfZmFpbGVkGhB0YXNrX2ludmFsaWRhdGVkGxR0YXNrX21lcmdlX2NvbXBsZXRlZBwSdGFza19tZXJnZV9zdGFy' +
|
|
140
|
+
'dGVkHQp0YXNrX3JlYWR5Hgx0YXNrX3NraXBwZWQfCnRhc2tfc3BsaXQgDHRhc2tfc3RhcnRlZB4NdW5pdF9yZXF1ZXVlZCUK' +
|
|
141
|
+
'JgkXB3ZlcnNpb24AAmlkAQRyZXBvAQl3b3Jrc3BhY2UBCXN0YXJ0ZWRBdAIFZm9yY2UDBmZpbHRlcgUFZ3JhcGgKCWdyYXBo' +
|
|
142
|
+
'SGFzaAUFdGFza3MRCGV4ZWN1dGVkAAZjYWNoZWQABmZhaWxlZAAHc2tpcHBlZAAGc3RhdHVzAQtjb21wbGV0ZWRBdA0FZXJy' +
|
|
143
|
+
'b3IFDnZlcnNpb25WZWN0b3JzEw1pbnB1dFNuYXBzaG90Eg90YXNrT3V0cHV0UGF0aHMGCnJlZXhlY3V0ZWQABmV2ZW50cycI' +
|
|
144
|
+
'ZXZlbnRTZXEAAQAB3waUAuNSMTC0NEg0SDLQtQACXXMDAyALRAAxHLAUpRbkM5UXNzw5fep9GgMDIwMTW0p+aVJOqkMihYCB' +
|
|
145
|
+
'kVMvM6+gtKRYr4KBV68ksTi7WA9iNgMDc3FprkMKhYCBEc1YLigXaDgDI9QfDEAA8xOU4kzOzy3ISS1JTWFkYHRIphAAA+3C' +
|
|
146
|
+
'Z1DoMS5oPgOiJICGJlEIiIk8Q1AggjB3Zl58QVF+elFqcTHIu4wfJoEcAmQwpZoCfQ8UYi8qzcvLzEsHSSNHjEMahQBoPGok' +
|
|
147
|
+
'IMcB0F4mJkjSIi418rJAghIaUSxskCCFRSXFcSXB2NDQkMDLAQkgUNjxcCHYLHw8BxbBOAxMLAxAxdNAOqYxcvJN2ASTYWJi' +
|
|
148
|
+
'YuADAA==',
|
|
149
|
+
'base64',
|
|
150
|
+
);
|
|
151
|
+
|
|
152
|
+
/** The version 5 state, as release 1.0.79 writes it: the release in place of
|
|
153
|
+
* the version. */
|
|
154
|
+
const STATE = Buffer.from(
|
|
155
|
+
'iUVhc3QNCgUAngsoKQEFBAAIAgRub25lAwRzb21lAAoACQUEbmFtZQAEaGFzaAAGaW5wdXRzBQZvdXRwdXQACWRlcGVuZHNP' +
|
|
156
|
+
'bgUKBgkBBXRhc2tzBwgCBG5vbmUDBHNvbWUICAIEbm9uZQMEc29tZQICCAIEbm9uZQMEc29tZQsIAgRub25lAwRzb21lAQkC' +
|
|
157
|
+
'CmlucHV0c0hhc2gAC2V4ZWN1dGlvbklkAAgCBG5vbmUDBHNvbWUOCQsEbmFtZQAGc3RhdHVzAAZjYWNoZWQKCm91dHB1dEhh' +
|
|
158
|
+
'c2gEBWVycm9yBAhleGl0Q29kZQwJc3RhcnRlZEF0DQtjb21wbGV0ZWRBdA0IZHVyYXRpb24MBHBsYW4ECWV4ZWN1dGlvbg8L' +
|
|
159
|
+
'ABALAAALABIJAwNzZXELCXRpbWVzdGFtcAEGcmVhc29uBAkIA3NlcQsJdGltZXN0YW1wAQdzdWNjZXNzAghleGVjdXRlZAsG' +
|
|
160
|
+
'Y2FjaGVkCwZmYWlsZWQLB3NraXBwZWQLCGR1cmF0aW9uCwkEA3NlcQsJdGltZXN0YW1wAQtleGVjdXRpb25JZAAKdG90YWxU' +
|
|
161
|
+
'YXNrcwsJBQNzZXELCXRpbWVzdGFtcAEEcGF0aAAMcHJldmlvdXNIYXNoAAduZXdIYXNoAAkHA3NlcQsJdGltZXN0YW1wAQR0' +
|
|
162
|
+
'YXNrAAZjYWNoZWQCCm91dHB1dEhhc2gACGR1cmF0aW9uCwlwZWFrQnl0ZXMMCQQDc2VxCwl0aW1lc3RhbXABBHRhc2sADGNv' +
|
|
163
|
+
'bmZsaWN0UGF0aAAJBgNzZXELCXRpbWVzdGFtcAEEdGFzawAFZXJyb3IECGV4aXRDb2RlDAhkdXJhdGlvbgsJBANzZXELCXRp' +
|
|
164
|
+
'bWVzdGFtcAEEdGFzawAGcmVhc29uAAkFA3NlcQsJdGltZXN0YW1wAQR0YXNrAAVsZXZlbAsGbGV2ZWxzCwkGA3NlcQsJdGlt' +
|
|
165
|
+
'ZXN0YW1wAQR0YXNrAAVsZXZlbAsGbGV2ZWxzCwV1bml0cwsJAwNzZXELCXRpbWVzdGFtcAEEdGFzawAJBANzZXELCXRpbWVz' +
|
|
166
|
+
'dGFtcAEEdGFzawAFY2F1c2UACQQDc2VxCwl0aW1lc3RhbXABBHRhc2sABnBpZWNlcwsJAgVsZXZlbAsGbGV2ZWxzCwgCBG5v' +
|
|
167
|
+
'bmUDBHNvbWUhCQMFbWVyZ2UiBWluZGV4CwV1bml0cwsIAwZidWRnZXQDA2NhcAMHbWFjaGluZQMJBwNzZXELCXRpbWVzdGFt' +
|
|
168
|
+
'cAEEdGFzawAEdW5pdCMGcmVhc29uJARwZWFrCwhyZXNlcnZlcwsIDxNleGVjdXRpb25fY2FuY2VsbGVkFBNleGVjdXRpb25f' +
|
|
169
|
+
'Y29tcGxldGVkFRFleGVjdXRpb25fc3RhcnRlZBYNaW5wdXRfY2hhbmdlZBcOdGFza19jb21wbGV0ZWQYDXRhc2tfZGVmZXJy' +
|
|
170
|
+
'ZWQZC3Rhc2tfZmFpbGVkGhB0YXNrX2ludmFsaWRhdGVkGxR0YXNrX21lcmdlX2NvbXBsZXRlZBwSdGFza19tZXJnZV9zdGFy' +
|
|
171
|
+
'dGVkHQp0YXNrX3JlYWR5Hgx0YXNrX3NraXBwZWQfCnRhc2tfc3BsaXQgDHRhc2tfc3RhcnRlZB4NdW5pdF9yZXF1ZXVlZCUK' +
|
|
172
|
+
'JgkXB3JlbGVhc2UAAmlkAARyZXBvAAl3b3Jrc3BhY2UACXN0YXJ0ZWRBdAEFZm9yY2UCBmZpbHRlcgQFZ3JhcGgJCWdyYXBo' +
|
|
173
|
+
'SGFzaAQFdGFza3MRCGV4ZWN1dGVkCwZjYWNoZWQLBmZhaWxlZAsHc2tpcHBlZAsGc3RhdHVzAAtjb21wbGV0ZWRBdA0FZXJy' +
|
|
174
|
+
'b3IEDnZlcnNpb25WZWN0b3JzEw1pbnB1dFNuYXBzaG90Eg90YXNrT3V0cHV0UGF0aHMFCnJlZXhlY3V0ZWQLBmV2ZW50cycI' +
|
|
175
|
+
'ZXZlbnRTZXELAQAB5QaaAmMz1DPQM7dUMTC0NEg0SDLQtQACXXMDAyALRAAxHLAUpRbkM5UXNzw5fep9GgMDIwMTW0p+aVJO' +
|
|
176
|
+
'qkMihYCBkVMvM6+gtKRYr4KBV68ksTi7WA9iNgMDc3FprkMKhYCBEc1YLigXaDgDI9QfDEAA8xOU4kzOzy3ISS1JTWFkYHRI' +
|
|
177
|
+
'phAAA+3CZ1DoMS5oPgOiJICGJlEIiIk8Q1AggjB3Zl58QVF+elFqcTHIu4wfJoEcAmQwpZoCfQ8UYi8qzcvLzEsHSSNHjEMa' +
|
|
178
|
+
'hQBoPGokIMcB0F4mJkjSIi418rJAghIaUSxskCCFRSXFcSXB2NDQkMDLAQkgUNjxcCHYLHw8BxbBOAxMLAxAxdNAOqYxcvJN' +
|
|
179
|
+
'2ASTYWJiYuADAA==',
|
|
180
|
+
'base64',
|
|
181
|
+
);
|
|
182
|
+
|
|
183
|
+
const NEW_FORM = 'a change to the execution state\'s type is a new form: ship a repository upgrade step that carries a repository\'s states into it, move this state to the refusals, and add one the new form writes';
|
|
184
|
+
|
|
185
|
+
describe('decodeDataflowExecutionState', () => {
|
|
186
|
+
it('refuses a state from before states recorded their release, naming the fix', () => {
|
|
187
|
+
for (const state of [STATE_V1, STATE_V2, STATE_V3, STATE_V4, STATE_V5]) {
|
|
188
|
+
assert.throws(
|
|
189
|
+
() => decodeDataflowExecutionState(state),
|
|
190
|
+
{ message: `the execution state was written by an older e3, in a form this e3, ${E3_RELEASE}, does not read — re-create the repository: deploy again and import its data again` },
|
|
191
|
+
);
|
|
192
|
+
}
|
|
193
|
+
});
|
|
194
|
+
|
|
195
|
+
it('reads a state in this release\'s form', () => {
|
|
196
|
+
assert.doesNotThrow(() => decodeDataflowExecutionState(STATE), NEW_FORM);
|
|
197
|
+
const state = decodeDataflowExecutionState(STATE);
|
|
198
|
+
assert.equal(state.release, '1.0.79');
|
|
199
|
+
assert.deepEqual(state.tasks.get('sum')!.plan, some('e5'));
|
|
200
|
+
assert.deepEqual(state.tasks.get('sum')!.execution, none);
|
|
201
|
+
assert.deepEqual(state.tasks.get('double')!.execution, some({ inputsHash: 'b'.repeat(64), executionId: '0190a0b0-8888-7000-8000-000000000001' }));
|
|
202
|
+
assert.deepEqual(state.events.map((event) => event.type),
|
|
203
|
+
['execution_started', 'task_started', 'task_completed', 'task_started', 'task_split', 'unit_requeued', 'task_merge_started']);
|
|
204
|
+
assert.deepEqual(state.events[2]!.value, {
|
|
205
|
+
seq: 3n, timestamp: new Date('2026-01-02T03:04:02.000Z'), task: 'double', cached: false, outputHash: 'c'.repeat(64), duration: 12n,
|
|
206
|
+
peakBytes: some(96n * 1024n ** 2n),
|
|
207
|
+
});
|
|
208
|
+
assert.deepEqual(state.events[5]!.value, {
|
|
209
|
+
seq: 6n, timestamp: new Date('2026-01-02T03:04:04.000Z'), task: 'sum', unit: { merge: none, index: 1n, units: 2n },
|
|
210
|
+
reason: variant('budget', null), peak: 150n * 1024n ** 2n, reserves: 150n * 1024n ** 2n,
|
|
211
|
+
});
|
|
212
|
+
});
|
|
213
|
+
|
|
214
|
+
it('refuses a state a newer release wrote, naming it', () => {
|
|
215
|
+
const newer = encodeBeast2For(StructType({ release: StringType, id: StringType }))({ release: '999.0.0', id: '7' });
|
|
216
|
+
assert.throws(
|
|
217
|
+
() => decodeDataflowExecutionState(newer),
|
|
218
|
+
{ message: `the execution state was written by e3 999.0.0, in a form this e3, ${E3_RELEASE}, does not read — use e3 999.0.0 or a newer one` },
|
|
219
|
+
);
|
|
220
|
+
});
|
|
221
|
+
|
|
222
|
+
it('refuses a state an older release wrote in another form, naming it and the fix', () => {
|
|
223
|
+
const older = encodeBeast2For(StructType({
|
|
224
|
+
release: StringType, workspace: StringType, tasks: DictType(StringType, StringType), events: ArrayType(StringType),
|
|
225
|
+
}))({ release: '1.0.0', workspace: 'main', tasks: new Map(), events: [] });
|
|
226
|
+
assert.throws(
|
|
227
|
+
() => decodeDataflowExecutionState(older),
|
|
228
|
+
{ message: `the execution state was written by e3 1.0.0, in a form this e3, ${E3_RELEASE}, does not read — re-create the repository: deploy again and import its data again` },
|
|
42
229
|
);
|
|
43
230
|
});
|
|
231
|
+
|
|
232
|
+
it('refuses data that is not an execution state', () => {
|
|
233
|
+
const other = encodeBeast2For(StructType({ id: StringType }))({ id: '7' });
|
|
234
|
+
assert.throws(() => decodeDataflowExecutionState(other), /not an execution state/);
|
|
235
|
+
});
|
|
44
236
|
});
|
package/src/dataflow.ts
CHANGED
|
@@ -25,8 +25,15 @@ import {
|
|
|
25
25
|
BooleanType,
|
|
26
26
|
DateTimeType,
|
|
27
27
|
OptionType,
|
|
28
|
+
NullType,
|
|
28
29
|
ValueTypeOf,
|
|
30
|
+
decodeBeast2,
|
|
31
|
+
decodeBeast2For,
|
|
32
|
+
isTypeValueEqual,
|
|
33
|
+
readBeast2Type,
|
|
34
|
+
toEastTypeValue,
|
|
29
35
|
} from '@elaraai/east';
|
|
36
|
+
import { E3_RELEASE, compareReleases } from './release.js';
|
|
30
37
|
|
|
31
38
|
// =============================================================================
|
|
32
39
|
// Status Literals (TypeScript only - East uses StringType)
|
|
@@ -70,6 +77,18 @@ export const TaskStateType = StructType({
|
|
|
70
77
|
completedAt: OptionType(DateTimeType),
|
|
71
78
|
/** Duration in milliseconds */
|
|
72
79
|
duration: OptionType(IntegerType),
|
|
80
|
+
/** The `$plan` object of the stage a task split into pieces is in, while
|
|
81
|
+
* it runs, and across a yield, so a resumed run takes the stage up where
|
|
82
|
+
* it stopped. */
|
|
83
|
+
plan: OptionType(StringType),
|
|
84
|
+
/** The execution the task completed with — run, or served from the cache —
|
|
85
|
+
* by its inputs hash and id, which the run's record names. */
|
|
86
|
+
execution: OptionType(StructType({
|
|
87
|
+
/** The combined hash of the task's inputs */
|
|
88
|
+
inputsHash: StringType,
|
|
89
|
+
/** The execution's id (UUIDv7) */
|
|
90
|
+
executionId: StringType,
|
|
91
|
+
})),
|
|
73
92
|
});
|
|
74
93
|
export type TaskState = ValueTypeOf<typeof TaskStateType>;
|
|
75
94
|
|
|
@@ -107,11 +126,46 @@ export type DataflowGraph = ValueTypeOf<typeof DataflowGraphType>;
|
|
|
107
126
|
// Event Types
|
|
108
127
|
// =============================================================================
|
|
109
128
|
|
|
129
|
+
/**
|
|
130
|
+
* A unit of a split task, by its place in the task: its stage — the pieces,
|
|
131
|
+
* or a level of the merges that assemble their outputs — and its index among
|
|
132
|
+
* the stage's units.
|
|
133
|
+
*/
|
|
134
|
+
export const StageUnitType = StructType({
|
|
135
|
+
/** The merge level, from 1, and the levels the merges take; `none` for a
|
|
136
|
+
* piece */
|
|
137
|
+
merge: OptionType(StructType({ level: IntegerType, levels: IntegerType })),
|
|
138
|
+
/** The unit's index in its stage, from 0 */
|
|
139
|
+
index: IntegerType,
|
|
140
|
+
/** The units of its stage */
|
|
141
|
+
units: IntegerType,
|
|
142
|
+
});
|
|
143
|
+
export type StageUnit = ValueTypeOf<typeof StageUnitType>;
|
|
144
|
+
|
|
145
|
+
/**
|
|
146
|
+
* Why a unit's runner was stopped and the unit requeued: `budget`, the guard
|
|
147
|
+
* stopped it with the runners past the budget; `machine`, the guard stopped it
|
|
148
|
+
* with the machine nearly out of memory; `cap`, its cgroup's cap killed it.
|
|
149
|
+
*/
|
|
150
|
+
export const RequeueReasonType = VariantType({
|
|
151
|
+
budget: NullType,
|
|
152
|
+
machine: NullType,
|
|
153
|
+
cap: NullType,
|
|
154
|
+
});
|
|
155
|
+
export type RequeueReason = ValueTypeOf<typeof RequeueReasonType>;
|
|
156
|
+
|
|
110
157
|
/**
|
|
111
158
|
* Execution events (VariantType for discriminated union).
|
|
112
159
|
*
|
|
113
160
|
* Events track the progress of a dataflow execution and are stored
|
|
114
161
|
* inline in the execution state (not as a separate JSONL file).
|
|
162
|
+
*
|
|
163
|
+
* @remarks
|
|
164
|
+
* Part of the execution state's wire: a reader decodes a state against the
|
|
165
|
+
* whole type it was written with, so a new event is a new form of the state,
|
|
166
|
+
* which the release that makes it carries repositories into with an upgrade
|
|
167
|
+
* step. Each unit's progress is a callback, {@link PartitionProgress}, and is
|
|
168
|
+
* not persisted.
|
|
115
169
|
*/
|
|
116
170
|
export const ExecutionEventType = VariantType({
|
|
117
171
|
/** Execution started */
|
|
@@ -157,6 +211,10 @@ export const ExecutionEventType = VariantType({
|
|
|
157
211
|
outputHash: StringType,
|
|
158
212
|
/** Duration in milliseconds */
|
|
159
213
|
duration: IntegerType,
|
|
214
|
+
/** The highest peak resident memory, in bytes, a runner of the task's
|
|
215
|
+
* execution reached; `none` when none reported one, or the cache served
|
|
216
|
+
* the task */
|
|
217
|
+
peakBytes: OptionType(IntegerType),
|
|
160
218
|
}),
|
|
161
219
|
/** Task failed */
|
|
162
220
|
task_failed: StructType({
|
|
@@ -247,38 +305,87 @@ export const ExecutionEventType = VariantType({
|
|
|
247
305
|
/** Path where version conflict was detected */
|
|
248
306
|
conflictPath: StringType,
|
|
249
307
|
}),
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
308
|
+
/** A task was split into pieces: its units start as the pieces, and its
|
|
309
|
+
* `$plan` names them. */
|
|
310
|
+
task_split: StructType({
|
|
311
|
+
/** Event sequence number */
|
|
312
|
+
seq: IntegerType,
|
|
313
|
+
/** When the event occurred */
|
|
314
|
+
timestamp: DateTimeType,
|
|
315
|
+
/** Task name */
|
|
316
|
+
task: StringType,
|
|
317
|
+
/** The pieces the task was split into */
|
|
318
|
+
pieces: IntegerType,
|
|
319
|
+
}),
|
|
320
|
+
/** A level of the merges assembling a split task's pieces started. */
|
|
321
|
+
task_merge_started: StructType({
|
|
322
|
+
/** Event sequence number */
|
|
323
|
+
seq: IntegerType,
|
|
324
|
+
/** When the event occurred */
|
|
325
|
+
timestamp: DateTimeType,
|
|
326
|
+
/** Task name */
|
|
327
|
+
task: StringType,
|
|
328
|
+
/** The level, from 1 */
|
|
329
|
+
level: IntegerType,
|
|
330
|
+
/** The number of levels the merges take */
|
|
331
|
+
levels: IntegerType,
|
|
332
|
+
/** The merge units of the level */
|
|
333
|
+
units: IntegerType,
|
|
334
|
+
}),
|
|
335
|
+
/** A level of the merges assembling a split task's pieces finished. */
|
|
336
|
+
task_merge_completed: StructType({
|
|
337
|
+
/** Event sequence number */
|
|
338
|
+
seq: IntegerType,
|
|
339
|
+
/** When the event occurred */
|
|
340
|
+
timestamp: DateTimeType,
|
|
341
|
+
/** Task name */
|
|
342
|
+
task: StringType,
|
|
343
|
+
/** The level, from 1 */
|
|
344
|
+
level: IntegerType,
|
|
345
|
+
/** The number of levels the merges take */
|
|
346
|
+
levels: IntegerType,
|
|
347
|
+
}),
|
|
348
|
+
/** A unit of a split task was stopped and requeued: it runs again, under
|
|
349
|
+
* the same execution, once the memory it reserves fits. */
|
|
350
|
+
unit_requeued: StructType({
|
|
351
|
+
/** Event sequence number */
|
|
352
|
+
seq: IntegerType,
|
|
353
|
+
/** When the event occurred */
|
|
354
|
+
timestamp: DateTimeType,
|
|
355
|
+
/** Task name */
|
|
356
|
+
task: StringType,
|
|
357
|
+
/** The unit */
|
|
358
|
+
unit: StageUnitType,
|
|
359
|
+
/** Why its runner was stopped */
|
|
360
|
+
reason: RequeueReasonType,
|
|
361
|
+
/** The most its runner was measured using, in bytes: for a cap, the cap */
|
|
362
|
+
peak: IntegerType,
|
|
363
|
+
/** The memory, in bytes, it reserves when it runs again */
|
|
364
|
+
reserves: IntegerType,
|
|
365
|
+
}),
|
|
259
366
|
});
|
|
260
367
|
export type ExecutionEvent = ValueTypeOf<typeof ExecutionEventType>;
|
|
261
368
|
|
|
262
369
|
/**
|
|
263
|
-
* Progress notification for one unit of a
|
|
264
|
-
*
|
|
265
|
-
*
|
|
266
|
-
*
|
|
267
|
-
*
|
|
268
|
-
*
|
|
269
|
-
* this callback.
|
|
370
|
+
* Progress notification for one unit of a task split into pieces — a piece,
|
|
371
|
+
* or a unit merging or folding their outputs. Delivered through runner-layer
|
|
372
|
+
* callbacks while the task runs, and not persisted: the execution state records
|
|
373
|
+
* a split task's stages, not each unit, so it stays small however many pieces
|
|
374
|
+
* a task has. The local `[PART]`/`[MERGE]`/`[COMBINE]` lines come straight
|
|
375
|
+
* from this callback.
|
|
270
376
|
*/
|
|
271
377
|
export interface PartitionProgress {
|
|
272
|
-
/** Which phase the unit belongs to
|
|
378
|
+
/** Which phase the unit belongs to: a piece (`partition`), a merge of a set
|
|
379
|
+
* or dict output, or a fold of a fold output (`combine`). */
|
|
273
380
|
phase: 'partition' | 'combine' | 'merge';
|
|
274
381
|
/** Zero-based index of the unit within its phase. */
|
|
275
382
|
index: number;
|
|
276
|
-
/** Total units in the phase (
|
|
383
|
+
/** Total units in the phase (the pieces, or the units of a merge level). */
|
|
277
384
|
total: number;
|
|
278
|
-
/** Units of the phase
|
|
279
|
-
* is `completed`. */
|
|
385
|
+
/** Units of the phase that succeeded so far, including this one when
|
|
386
|
+
* `state` is `completed`. */
|
|
280
387
|
completed: number;
|
|
281
|
-
/** Whether the unit started or
|
|
388
|
+
/** Whether the unit started, or succeeded. */
|
|
282
389
|
state: 'started' | 'completed';
|
|
283
390
|
/** Whether the unit was served from the execution cache (completed only). */
|
|
284
391
|
cached?: boolean;
|
|
@@ -299,10 +406,17 @@ export interface PartitionProgress {
|
|
|
299
406
|
* - Tasks are stored as a Dict (serializes as object, not array of tuples)
|
|
300
407
|
* - Events are stored inline (not as separate JSONL file)
|
|
301
408
|
* - Dates are Date objects (via DateTimeType)
|
|
409
|
+
* - Read it back with {@link decodeDataflowExecutionState}, which reads this
|
|
410
|
+
* form alone. A release that changes it ships a repository upgrade step,
|
|
411
|
+
* which carries a repository's states into the new form when that release
|
|
412
|
+
* first opens it.
|
|
302
413
|
*/
|
|
303
414
|
export const DataflowExecutionStateType = StructType({
|
|
415
|
+
/** The release of e3 that wrote the state */
|
|
416
|
+
release: StringType,
|
|
417
|
+
|
|
304
418
|
// Identity
|
|
305
|
-
/**
|
|
419
|
+
/** The run's id, a UUIDv7, which its `DataflowRun` record carries as `runId` */
|
|
306
420
|
id: StringType,
|
|
307
421
|
/** Repository identifier */
|
|
308
422
|
repo: StringType,
|
|
@@ -312,8 +426,6 @@ export const DataflowExecutionStateType = StructType({
|
|
|
312
426
|
startedAt: DateTimeType,
|
|
313
427
|
|
|
314
428
|
// Config (immutable after initialization)
|
|
315
|
-
/** Maximum concurrent task executions */
|
|
316
|
-
concurrency: IntegerType,
|
|
317
429
|
/** Force re-execution even if cached */
|
|
318
430
|
force: BooleanType,
|
|
319
431
|
/** Filter to run only specific task(s) by exact name */
|
|
@@ -365,6 +477,44 @@ export const DataflowExecutionStateType = StructType({
|
|
|
365
477
|
});
|
|
366
478
|
export type DataflowExecutionState = ValueTypeOf<typeof DataflowExecutionStateType>;
|
|
367
479
|
|
|
480
|
+
const STATE_TYPE = toEastTypeValue(DataflowExecutionStateType);
|
|
481
|
+
const decodeState = decodeBeast2For(DataflowExecutionStateType);
|
|
482
|
+
|
|
483
|
+
/**
|
|
484
|
+
* Decodes a stored execution state.
|
|
485
|
+
*
|
|
486
|
+
* The form is told by the type the state's header declares, since a decoder
|
|
487
|
+
* built for another form's type would misread it rather than fail. A
|
|
488
|
+
* repository's upgrade steps carry its states into this form when this
|
|
489
|
+
* release first opens it, so a state of another form is refused, naming the
|
|
490
|
+
* release that wrote it: a newer one, whose e3 reads it, or an older one that
|
|
491
|
+
* no step carries forward, whose repository is re-created.
|
|
492
|
+
*
|
|
493
|
+
* @param data - the stored state
|
|
494
|
+
* @returns the state
|
|
495
|
+
* @throws {Error} When the state is of another form, naming the release that
|
|
496
|
+
* wrote it, or the data is not an execution state.
|
|
497
|
+
*/
|
|
498
|
+
export function decodeDataflowExecutionState(data: Uint8Array): DataflowExecutionState {
|
|
499
|
+
const type = readBeast2Type(data);
|
|
500
|
+
if (isTypeValueEqual(type, STATE_TYPE)) return decodeState(data);
|
|
501
|
+
const { value } = decodeBeast2(data);
|
|
502
|
+
const state = typeof value === 'object' && value !== null ? value as Record<string, unknown> : null;
|
|
503
|
+
const release = typeof state?.release === 'string' ? state.release : null;
|
|
504
|
+
if (release !== null && compareReleases(release, E3_RELEASE) > 0) {
|
|
505
|
+
throw new Error(
|
|
506
|
+
`the execution state was written by e3 ${release}, in a form this e3, ${E3_RELEASE}, does not read — use e3 ${release} or a newer one`,
|
|
507
|
+
);
|
|
508
|
+
}
|
|
509
|
+
if (state !== null && 'workspace' in state && 'tasks' in state && 'events' in state) {
|
|
510
|
+
throw new Error(
|
|
511
|
+
`the execution state was written by ${release === null ? 'an older e3' : `e3 ${release}`}, in a form this e3, ${E3_RELEASE}, does not read — ` +
|
|
512
|
+
're-create the repository: deploy again and import its data again',
|
|
513
|
+
);
|
|
514
|
+
}
|
|
515
|
+
throw new Error('the data is not an execution state: its type is not the execution state\'s');
|
|
516
|
+
}
|
|
517
|
+
|
|
368
518
|
// =============================================================================
|
|
369
519
|
// Dataflow Run History
|
|
370
520
|
// =============================================================================
|
|
@@ -390,11 +540,18 @@ export const DataflowRunStatusType = VariantType({
|
|
|
390
540
|
export type DataflowRunStatus = ValueTypeOf<typeof DataflowRunStatusType>;
|
|
391
541
|
|
|
392
542
|
/**
|
|
393
|
-
* Record of a task execution within a dataflow run
|
|
543
|
+
* Record of a task execution within a dataflow run: the execution the run
|
|
544
|
+
* used, which a local repository keeps at
|
|
545
|
+
* executions/<taskHash>/<inputsHash>/<executionId>/, whatever the workspace
|
|
546
|
+
* has held since.
|
|
394
547
|
*/
|
|
395
548
|
export const TaskExecutionRecordType = StructType({
|
|
396
|
-
/** Execution ID (UUIDv7) */
|
|
549
|
+
/** Execution ID (UUIDv7): the attempt that ran, or the one the cache served */
|
|
397
550
|
executionId: StringType,
|
|
551
|
+
/** Hash of the task object */
|
|
552
|
+
taskHash: StringType,
|
|
553
|
+
/** Combined hash of the task's inputs */
|
|
554
|
+
inputsHash: StringType,
|
|
398
555
|
/** Whether this was a cache hit */
|
|
399
556
|
cached: BooleanType,
|
|
400
557
|
/** Output version vector (which root input versions produced this output) */
|