@elaraai/e3-types 1.0.84 → 1.0.85
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/api.d.ts +523 -17
- package/dist/src/api.d.ts.map +1 -1
- package/dist/src/api.js +208 -22
- package/dist/src/api.js.map +1 -1
- package/dist/src/dataflow.d.ts +115 -12
- package/dist/src/dataflow.d.ts.map +1 -1
- package/dist/src/dataflow.js +151 -15
- package/dist/src/dataflow.js.map +1 -1
- package/dist/src/dataflow.spec.js +41 -7
- package/dist/src/dataflow.spec.js.map +1 -1
- package/dist/src/dataset-blob.d.ts +19 -2
- package/dist/src/dataset-blob.d.ts.map +1 -1
- package/dist/src/dataset-blob.js +100 -21
- package/dist/src/dataset-blob.js.map +1 -1
- package/dist/src/dataset-blob.spec.js +132 -0
- package/dist/src/dataset-blob.spec.js.map +1 -1
- package/dist/src/execution.d.ts +68 -8
- package/dist/src/execution.d.ts.map +1 -1
- package/dist/src/execution.js +49 -8
- package/dist/src/execution.js.map +1 -1
- package/dist/src/execution.spec.js +31 -9
- package/dist/src/execution.spec.js.map +1 -1
- package/dist/src/index.d.ts +8 -7
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/index.js +14 -11
- package/dist/src/index.js.map +1 -1
- package/dist/src/names.d.ts +9 -0
- package/dist/src/names.d.ts.map +1 -1
- package/dist/src/names.js.map +1 -1
- package/dist/src/pieces.d.ts +47 -0
- package/dist/src/pieces.d.ts.map +1 -0
- package/dist/src/pieces.js +17 -0
- package/dist/src/pieces.js.map +1 -0
- package/dist/src/pieces.spec.d.ts +6 -0
- package/dist/src/pieces.spec.d.ts.map +1 -0
- package/dist/src/pieces.spec.js +20 -0
- package/dist/src/pieces.spec.js.map +1 -0
- package/dist/src/release.d.ts +1 -1
- package/dist/src/release.js +1 -1
- package/dist/src/repository.d.ts +16 -0
- package/dist/src/repository.d.ts.map +1 -1
- package/dist/src/repository.js +15 -0
- package/dist/src/repository.js.map +1 -1
- package/dist/src/structure.d.ts +27 -0
- package/dist/src/structure.d.ts.map +1 -1
- package/dist/src/structure.js +59 -18
- package/dist/src/structure.js.map +1 -1
- package/dist/src/structure.spec.d.ts +6 -0
- package/dist/src/structure.spec.d.ts.map +1 -0
- package/dist/src/structure.spec.js +57 -0
- package/dist/src/structure.spec.js.map +1 -0
- package/package.json +3 -2
- package/src/api.ts +225 -21
- package/src/dataflow.spec.ts +49 -7
- package/src/dataflow.ts +160 -14
- package/src/dataset-blob.spec.ts +147 -0
- package/src/dataset-blob.ts +97 -20
- package/src/execution.spec.ts +31 -9
- package/src/execution.ts +52 -7
- package/src/index.ts +38 -2
- package/src/names.ts +12 -0
- package/src/pieces.spec.ts +23 -0
- package/src/pieces.ts +49 -0
- package/src/release.ts +1 -1
- package/src/repository.ts +18 -0
- package/src/structure.spec.ts +65 -0
- package/src/structure.ts +64 -21
package/src/execution.spec.ts
CHANGED
|
@@ -5,15 +5,16 @@
|
|
|
5
5
|
|
|
6
6
|
/**
|
|
7
7
|
* Execution status wire: the typed outcomes with their peaks, a split task's
|
|
8
|
-
* plan
|
|
9
|
-
* refusal of a record an older e3 wrote before
|
|
10
|
-
* were cases of it, before a runner's peak was
|
|
11
|
-
* plan was,
|
|
8
|
+
* plan, whether the execution is a unit and why a stopped one stopped, what
|
|
9
|
+
* each keeps from gc, and the refusal of a record an older e3 wrote before
|
|
10
|
+
* `cancelled` and `interrupted` were cases of it, before a runner's peak was
|
|
11
|
+
* recorded, before a split task's plan was, before a record said whether it
|
|
12
|
+
* is a unit, or before a stopped one said why.
|
|
12
13
|
*/
|
|
13
14
|
|
|
14
15
|
import { describe, it } from 'node:test';
|
|
15
16
|
import assert from 'node:assert/strict';
|
|
16
|
-
import { ArrayType, DateTimeType, IntegerType, OptionType, StringType, StructType, VariantType, encodeBeast2For, none, some, variant } from '@elaraai/east';
|
|
17
|
+
import { ArrayType, BooleanType, DateTimeType, IntegerType, OptionType, StringType, StructType, VariantType, encodeBeast2For, none, some, variant } from '@elaraai/east';
|
|
17
18
|
import { ExecutionStatusType, decodeExecutionStatus, executionStatusRoots, type ExecutionStatus } from './execution.js';
|
|
18
19
|
|
|
19
20
|
/** The four-case wire every record had before the typed outcomes. */
|
|
@@ -45,7 +46,7 @@ describe('ExecutionStatusType', () => {
|
|
|
45
46
|
};
|
|
46
47
|
const encodeOlder = encodeBeast2For(PreOutcomeExecutionStatusType);
|
|
47
48
|
|
|
48
|
-
it('round-trips the typed outcomes, with a runner\'s peak or without one, a task\'s own or a unit\'s', () => {
|
|
49
|
+
it('round-trips the typed outcomes, with a runner\'s peak or without one, a task\'s own or a unit\'s, and why a stopped one stopped', () => {
|
|
49
50
|
const own = { ...stopped, unit: false };
|
|
50
51
|
const statuses: ExecutionStatus[] = [
|
|
51
52
|
variant('success', { ...own, outputHash: 'b'.repeat(64), peakBytes: some(96n * 1024n ** 2n), plan: none }),
|
|
@@ -54,8 +55,11 @@ describe('ExecutionStatusType', () => {
|
|
|
54
55
|
variant('running', { executionId: '0199-b', inputHashes: ['a'.repeat(64)], startedAt: new Date(1000), pid: 4242n, pidStartTime: 7n, bootId: 'boot', unit: true }),
|
|
55
56
|
variant('failed', { ...own, exitCode: 1n, peakBytes: some(412n * 1024n ** 2n) }),
|
|
56
57
|
variant('failed', { ...own, exitCode: -1n, peakBytes: none }),
|
|
57
|
-
variant('cancelled', own),
|
|
58
|
-
variant('interrupted', { ...own, pid: 4242n }),
|
|
58
|
+
variant('cancelled', { ...own, reason: { kind: variant('aborted', null), message: 'cancelled: e3 stopped the runner because the run was aborted' } }),
|
|
59
|
+
variant('interrupted', { ...own, pid: 4242n, reason: { kind: variant('owner_gone', null), message: 'its runner and its owner are gone' } }),
|
|
60
|
+
variant('interrupted', { ...own, pid: 0n, unit: true, reason: { kind: variant('yielded', null), message: 'its run yielded' } }),
|
|
61
|
+
variant('interrupted', { ...own, pid: 0n, reason: { kind: variant('host', 'OutOfMemoryError'), message: 'its container ran out of memory' } }),
|
|
62
|
+
variant('interrupted', { ...own, pid: 0n, reason: { kind: variant('unrecorded', null), message: '' } }),
|
|
59
63
|
variant('error', { ...own, message: 'Failed to read output: no such file' }),
|
|
60
64
|
];
|
|
61
65
|
const encode = encodeBeast2For(ExecutionStatusType);
|
|
@@ -67,7 +71,7 @@ describe('ExecutionStatusType', () => {
|
|
|
67
71
|
assert.deepEqual(executionStatusRoots(variant('success', { ...success, plan: none })), ['b'.repeat(64)]);
|
|
68
72
|
assert.deepEqual(executionStatusRoots(variant('success', { ...success, plan: some('c'.repeat(64)) })), ['b'.repeat(64), 'c'.repeat(64)]);
|
|
69
73
|
assert.deepEqual(executionStatusRoots(variant('failed', { ...stopped, exitCode: 1n, peakBytes: none, unit: false })), []);
|
|
70
|
-
assert.deepEqual(executionStatusRoots(variant('cancelled', { ...stopped, unit: false })), []);
|
|
74
|
+
assert.deepEqual(executionStatusRoots(variant('cancelled', { ...stopped, unit: false, reason: { kind: variant('aborted', null), message: '' } })), []);
|
|
71
75
|
});
|
|
72
76
|
|
|
73
77
|
it('keeps from gc the inputs a running attempt reads, passing over a merge\'s tag', () => {
|
|
@@ -113,6 +117,24 @@ describe('ExecutionStatusType', () => {
|
|
|
113
117
|
}),
|
|
114
118
|
});
|
|
115
119
|
assert.throws(() => decodeExecutionStatus(encodeBeast2For(withoutUnits)(variant('success', { ...stopped, outputHash: 'b'.repeat(64), peakBytes: none, plan: none }))), refusal);
|
|
120
|
+
// With units, before a stopped record said why it stopped: only the
|
|
121
|
+
// repository upgrade that carries such a record forward reads it, and every
|
|
122
|
+
// case of it is refused, since its type names the cases that changed.
|
|
123
|
+
const withoutReasons = VariantType({
|
|
124
|
+
...ExecutionStatusType.cases,
|
|
125
|
+
cancelled: StructType({ executionId: StringType, inputHashes: ArrayType(StringType), startedAt: DateTimeType, completedAt: DateTimeType, unit: BooleanType }),
|
|
126
|
+
interrupted: StructType({
|
|
127
|
+
executionId: StringType, inputHashes: ArrayType(StringType), startedAt: DateTimeType, completedAt: DateTimeType, pid: IntegerType, unit: BooleanType,
|
|
128
|
+
}),
|
|
129
|
+
});
|
|
130
|
+
const own = { ...stopped, unit: false };
|
|
131
|
+
for (const record of [
|
|
132
|
+
variant('cancelled', own),
|
|
133
|
+
variant('interrupted', { ...own, pid: 4242n }),
|
|
134
|
+
variant('success', { ...own, outputHash: 'b'.repeat(64), peakBytes: none, plan: none }),
|
|
135
|
+
] as const) {
|
|
136
|
+
assert.throws(() => decodeExecutionStatus(encodeBeast2For(withoutReasons)(record)), refusal);
|
|
137
|
+
}
|
|
116
138
|
});
|
|
117
139
|
|
|
118
140
|
it('throws for bytes of no status shape', () => {
|
package/src/execution.ts
CHANGED
|
@@ -15,7 +15,7 @@
|
|
|
15
15
|
* - For success: output hash, timing, peak memory and a split task's last plan
|
|
16
16
|
* - For failed: exit code, timing and peak memory
|
|
17
17
|
* - For error: internal error message and timing
|
|
18
|
-
* - For cancelled and interrupted: how e3, not the task, ended it
|
|
18
|
+
* - For cancelled and interrupted: how e3, not the task, ended it, and why
|
|
19
19
|
*
|
|
20
20
|
* Every case says whether the execution is a unit of a split task — a piece,
|
|
21
21
|
* or a merge of the pieces' outputs — which is recorded under its task's hash
|
|
@@ -31,10 +31,42 @@ import {
|
|
|
31
31
|
IntegerType,
|
|
32
32
|
BooleanType,
|
|
33
33
|
DateTimeType,
|
|
34
|
+
NullType,
|
|
34
35
|
ValueTypeOf,
|
|
35
36
|
decodeBeast2For,
|
|
36
37
|
} from '@elaraai/east';
|
|
37
38
|
|
|
39
|
+
/**
|
|
40
|
+
* Why an execution stopped without finishing: what a `cancelled` or an
|
|
41
|
+
* `interrupted` record says, written in the same write as the record.
|
|
42
|
+
*
|
|
43
|
+
* - `aborted`: the signal it ran under was aborted — its run was cancelled,
|
|
44
|
+
* or the call it ran for ended
|
|
45
|
+
* - `yielded`: its run yielded mid-stage, and a run resumed takes the stage up
|
|
46
|
+
* again
|
|
47
|
+
* - `owner_gone`: its runner and the process, or the browser tab, that owned
|
|
48
|
+
* it are gone
|
|
49
|
+
* - `host`: a host's own reason, by its code — e3-cloud's for a container
|
|
50
|
+
* that stopped, say
|
|
51
|
+
* - `unrecorded`: it stopped before e3 recorded reasons; only the repository
|
|
52
|
+
* upgrade that carried its record forward writes it
|
|
53
|
+
*
|
|
54
|
+
* `message` is what e3 wrote as the attempt's log's last line, after its
|
|
55
|
+
* `e3: `, or the host's words; empty for `unrecorded`.
|
|
56
|
+
*/
|
|
57
|
+
export const StopReasonType = StructType({
|
|
58
|
+
kind: VariantType({
|
|
59
|
+
aborted: NullType,
|
|
60
|
+
yielded: NullType,
|
|
61
|
+
owner_gone: NullType,
|
|
62
|
+
host: StringType,
|
|
63
|
+
unrecorded: NullType,
|
|
64
|
+
}),
|
|
65
|
+
message: StringType,
|
|
66
|
+
});
|
|
67
|
+
|
|
68
|
+
export type StopReason = ValueTypeOf<typeof StopReasonType>;
|
|
69
|
+
|
|
38
70
|
/** A running execution's process identification. */
|
|
39
71
|
const RunningStatusType = StructType({
|
|
40
72
|
/** Unique execution ID (UUIDv7) */
|
|
@@ -126,10 +158,14 @@ const ErrorStatusType = StructType({
|
|
|
126
158
|
* - `success`: Task ran and returned exit code 0
|
|
127
159
|
* - `failed`: Task ran and returned non-zero exit code
|
|
128
160
|
* - `error`: e3 execution engine had an internal error (runner not found, output missing, etc.)
|
|
129
|
-
* - `cancelled`: e3 stopped the execution because the
|
|
130
|
-
* its runner started or while it ran — not the task's own
|
|
131
|
-
*
|
|
132
|
-
*
|
|
161
|
+
* - `cancelled`: e3 stopped the execution because the signal it ran under was
|
|
162
|
+
* aborted, before its runner started or while it ran — not the task's own
|
|
163
|
+
* failure
|
|
164
|
+
* - `interrupted`: the execution can no longer finish, and nothing will write
|
|
165
|
+
* its outcome: its runner and its owner are gone, its host says it cannot,
|
|
166
|
+
* or its run yielded mid-stage
|
|
167
|
+
*
|
|
168
|
+
* Each of the two says why it stopped ({@link StopReasonType}).
|
|
133
169
|
*
|
|
134
170
|
* The `running` state includes process identification fields (pid, pidStartTime, bootId)
|
|
135
171
|
* to enable detection of crashed executions. See design/e3-execution.md for details.
|
|
@@ -154,6 +190,8 @@ export const ExecutionStatusType = VariantType({
|
|
|
154
190
|
* the pieces' outputs — rather than a task's own execution, which the one
|
|
155
191
|
* unit of a task whose input closes no piece is */
|
|
156
192
|
unit: BooleanType,
|
|
193
|
+
/** Why it stopped: `aborted` */
|
|
194
|
+
reason: StopReasonType,
|
|
157
195
|
}),
|
|
158
196
|
interrupted: StructType({
|
|
159
197
|
/** Unique execution ID (UUIDv7) */
|
|
@@ -170,6 +208,9 @@ export const ExecutionStatusType = VariantType({
|
|
|
170
208
|
* the pieces' outputs — rather than a task's own execution, which the one
|
|
171
209
|
* unit of a task whose input closes no piece is */
|
|
172
210
|
unit: BooleanType,
|
|
211
|
+
/** Why it can no longer finish: `owner_gone`, `yielded`, or its host's
|
|
212
|
+
* reason */
|
|
213
|
+
reason: StopReasonType,
|
|
173
214
|
}),
|
|
174
215
|
});
|
|
175
216
|
|
|
@@ -209,8 +250,12 @@ const decodeCurrentStatus = decodeBeast2For(ExecutionStatusType);
|
|
|
209
250
|
* Decode an execution status.
|
|
210
251
|
*
|
|
211
252
|
* @remarks
|
|
212
|
-
* Stored state,
|
|
213
|
-
*
|
|
253
|
+
* Stored state, which changes as `docs/conventions/WIRE_MIGRATION.md` says:
|
|
254
|
+
* this reads the current form alone. A repository's upgrade steps carry its
|
|
255
|
+
* records into it when an e3 first opens the repository — the stop reasons'
|
|
256
|
+
* step those from before a stopped execution recorded why — so a record in an
|
|
257
|
+
* earlier form here is one from before repositories recorded their upgrades,
|
|
258
|
+
* whose repository is re-created, and this says so.
|
|
214
259
|
*
|
|
215
260
|
* @param data - the stored bytes
|
|
216
261
|
* @returns the status
|
package/src/index.ts
CHANGED
|
@@ -92,6 +92,7 @@ export {
|
|
|
92
92
|
treePath,
|
|
93
93
|
pathToString,
|
|
94
94
|
parsePath,
|
|
95
|
+
parseKeypath,
|
|
95
96
|
parseDatasetPath,
|
|
96
97
|
parsePackageRef,
|
|
97
98
|
urlPathToTreePath,
|
|
@@ -185,6 +186,8 @@ export {
|
|
|
185
186
|
|
|
186
187
|
// Execution status
|
|
187
188
|
export {
|
|
189
|
+
StopReasonType,
|
|
190
|
+
type StopReason,
|
|
188
191
|
ExecutionStatusType,
|
|
189
192
|
type ExecutionStatus,
|
|
190
193
|
decodeExecutionStatus,
|
|
@@ -224,6 +227,8 @@ export {
|
|
|
224
227
|
export {
|
|
225
228
|
RepositoryRecordType,
|
|
226
229
|
type RepositoryRecord,
|
|
230
|
+
RepositoryUpgradeProgressType,
|
|
231
|
+
type RepositoryUpgradeProgress,
|
|
227
232
|
RepoStatusType,
|
|
228
233
|
type RepoStatus,
|
|
229
234
|
RepoMetadataType,
|
|
@@ -233,8 +238,8 @@ export {
|
|
|
233
238
|
// The release of e3 this build is, which what it keeps and ships records
|
|
234
239
|
export { E3_RELEASE, compareReleases } from './release.js';
|
|
235
240
|
|
|
236
|
-
// The names e3 makes paths of
|
|
237
|
-
export { type NamedKind, nameProblem } from './names.js';
|
|
241
|
+
// The names e3 makes paths of, and the hashes and ids
|
|
242
|
+
export { type NamedKind, type HashKind, type IdKind, type IdentifierKind, nameProblem } from './names.js';
|
|
238
243
|
|
|
239
244
|
// Dataset transfer types
|
|
240
245
|
export {
|
|
@@ -257,6 +262,7 @@ export {
|
|
|
257
262
|
isCollectionRoot,
|
|
258
263
|
encodeDatasetBlob,
|
|
259
264
|
writeCollectionManifest,
|
|
265
|
+
SEGMENT_WRITES_IN_FLIGHT,
|
|
260
266
|
type CollectionSegmentRef,
|
|
261
267
|
type CollectionPiece,
|
|
262
268
|
type SegmentSink,
|
|
@@ -356,6 +362,7 @@ export {
|
|
|
356
362
|
// Execution
|
|
357
363
|
DataflowRequestType,
|
|
358
364
|
LogChunkType,
|
|
365
|
+
TaskLogChunkType,
|
|
359
366
|
TaskExecutionResultType,
|
|
360
367
|
DataflowResultType,
|
|
361
368
|
// Dataflow API polling
|
|
@@ -366,9 +373,12 @@ export {
|
|
|
366
373
|
UnitWaitType,
|
|
367
374
|
SplitProgressType,
|
|
368
375
|
ApiDataflowExecutionStateType,
|
|
376
|
+
DATAFLOW_POLL_EVENTS_MAX,
|
|
369
377
|
// Task Execution History
|
|
370
378
|
ExecutionHistoryStatusType,
|
|
371
379
|
ExecutionListItemType,
|
|
380
|
+
TASK_EXECUTIONS_PAGE_DEFAULT,
|
|
381
|
+
TASK_EXECUTIONS_PAGE_MAX,
|
|
372
382
|
// Dataset List
|
|
373
383
|
TreeKindType,
|
|
374
384
|
ListEntryType,
|
|
@@ -378,6 +388,7 @@ export {
|
|
|
378
388
|
ExecuteLimitsType,
|
|
379
389
|
DiagnosticType,
|
|
380
390
|
ExecuteResultType,
|
|
391
|
+
CallArgType,
|
|
381
392
|
FunctionCallRequestType,
|
|
382
393
|
FunctionSignatureType,
|
|
383
394
|
OneShotRequestType,
|
|
@@ -389,9 +400,16 @@ export {
|
|
|
389
400
|
SplitCallPlanType,
|
|
390
401
|
MutationCallRequestType,
|
|
391
402
|
MutationResultType,
|
|
403
|
+
RecordMigrationAppliedType,
|
|
392
404
|
RecordSignatureType,
|
|
393
405
|
RecordCommitInfoType,
|
|
406
|
+
RecordArgPreviewType,
|
|
407
|
+
RecordCommitArgsType,
|
|
408
|
+
RecordHistoryCommitType,
|
|
394
409
|
RecordHistoryResultType,
|
|
410
|
+
RECORD_ARG_TEXT_CHARS,
|
|
411
|
+
RECORD_ARG_TEXT_BYTES,
|
|
412
|
+
RECORD_ARGS_READ_BYTES,
|
|
395
413
|
// Type aliases
|
|
396
414
|
type Error,
|
|
397
415
|
type RepositoryStatus,
|
|
@@ -425,6 +443,7 @@ export {
|
|
|
425
443
|
type TaskDetails,
|
|
426
444
|
type DataflowRequest,
|
|
427
445
|
type LogChunk,
|
|
446
|
+
type TaskLogChunk,
|
|
428
447
|
type TaskExecutionResult,
|
|
429
448
|
type DataflowResult,
|
|
430
449
|
type DataflowEvent,
|
|
@@ -442,6 +461,7 @@ export {
|
|
|
442
461
|
type ExecuteLimits,
|
|
443
462
|
type Diagnostic,
|
|
444
463
|
type ExecuteResult,
|
|
464
|
+
type CallArg,
|
|
445
465
|
type FunctionCallRequest,
|
|
446
466
|
type FunctionSignature,
|
|
447
467
|
type OneShotRequest,
|
|
@@ -452,8 +472,12 @@ export {
|
|
|
452
472
|
type SplitCallPlan,
|
|
453
473
|
type MutationCallRequest,
|
|
454
474
|
type MutationResult,
|
|
475
|
+
type RecordMigrationApplied,
|
|
455
476
|
type RecordSignature,
|
|
456
477
|
type RecordCommitInfo,
|
|
478
|
+
type RecordArgPreview,
|
|
479
|
+
type RecordCommitArgs,
|
|
480
|
+
type RecordHistoryCommit,
|
|
457
481
|
type RecordHistoryResult,
|
|
458
482
|
} from './api.js';
|
|
459
483
|
|
|
@@ -474,9 +498,17 @@ export {
|
|
|
474
498
|
ExecutionEventType,
|
|
475
499
|
type ExecutionEvent,
|
|
476
500
|
type PartitionProgress,
|
|
501
|
+
DataflowForceType,
|
|
502
|
+
type DataflowForce,
|
|
503
|
+
dataflowForce,
|
|
504
|
+
dataflowForceOption,
|
|
477
505
|
DataflowExecutionStateType,
|
|
478
506
|
type DataflowExecutionState,
|
|
479
507
|
decodeDataflowExecutionState,
|
|
508
|
+
ExecutionStateSummaryType,
|
|
509
|
+
type ExecutionStateSummary,
|
|
510
|
+
executionStateSummary,
|
|
511
|
+
lastEventSeq,
|
|
480
512
|
// Dataflow run history
|
|
481
513
|
DataflowRunStatusType,
|
|
482
514
|
type DataflowRunStatus,
|
|
@@ -488,6 +520,10 @@ export {
|
|
|
488
520
|
type DataflowRun,
|
|
489
521
|
} from './dataflow.js';
|
|
490
522
|
|
|
523
|
+
// The sizes e3 cuts a split task's input into pieces at, which a caller
|
|
524
|
+
// planning a split call weighs a dataset against
|
|
525
|
+
export { PIECE_SIZES, type PieceSizes } from './pieces.js';
|
|
526
|
+
|
|
491
527
|
// A split task's unit graph, a stage at a time
|
|
492
528
|
export {
|
|
493
529
|
UNIT_PLAN_KIND,
|
package/src/names.ts
CHANGED
|
@@ -13,6 +13,18 @@
|
|
|
13
13
|
/** What a name names. */
|
|
14
14
|
export type NamedKind = 'repository' | 'workspace' | 'package' | 'package version' | 'lock';
|
|
15
15
|
|
|
16
|
+
/** What a hash e3 makes a path or a key of names: an object, or an
|
|
17
|
+
* execution's task or inputs. */
|
|
18
|
+
export type HashKind = 'object hash' | 'task hash' | 'inputs hash';
|
|
19
|
+
|
|
20
|
+
/** What an id e3 makes a path or a key of names: an execution attempt, a
|
|
21
|
+
* dataflow run, or a gc run in steps. */
|
|
22
|
+
export type IdKind = 'execution id' | 'run id' | 'gc run id';
|
|
23
|
+
|
|
24
|
+
/** What a hash or an id e3 makes a path or a key of names: not a name, so
|
|
25
|
+
* checked for its form rather than by {@link nameProblem}. */
|
|
26
|
+
export type IdentifierKind = HashKind | IdKind;
|
|
27
|
+
|
|
16
28
|
/** What no name may hold: a path separator, a character a Windows file name
|
|
17
29
|
* refuses, or a control character. */
|
|
18
30
|
const REFUSED = /[/\\:*?"<>|\u0000-\u001f]/;
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Copyright (c) 2025 Elara AI Pty Ltd
|
|
3
|
+
* Dual-licensed under AGPL-3.0 and commercial license. See LICENSE for details.
|
|
4
|
+
*/
|
|
5
|
+
|
|
6
|
+
/**
|
|
7
|
+
* The platform's piece sizes, which e3-core's piece rule closes pieces at and
|
|
8
|
+
* a caller planning a split call weighs a dataset against.
|
|
9
|
+
*/
|
|
10
|
+
|
|
11
|
+
import { describe, it } from 'node:test';
|
|
12
|
+
import assert from 'node:assert/strict';
|
|
13
|
+
import { PIECE_SIZES } from './index.js';
|
|
14
|
+
|
|
15
|
+
describe('PIECE_SIZES', () => {
|
|
16
|
+
it('is 16, 64 and 256 MiB, exported from the package', () => {
|
|
17
|
+
assert.deepEqual(PIECE_SIZES, { min: 16 * 2 ** 20, target: 64 * 2 ** 20, max: 256 * 2 ** 20 });
|
|
18
|
+
});
|
|
19
|
+
|
|
20
|
+
it('orders its sizes: the least a piece holds, its middle, and the most', () => {
|
|
21
|
+
assert.ok(PIECE_SIZES.min < PIECE_SIZES.target && PIECE_SIZES.target < PIECE_SIZES.max);
|
|
22
|
+
});
|
|
23
|
+
});
|
package/src/pieces.ts
ADDED
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Copyright (c) 2025 Elara AI Pty Ltd
|
|
3
|
+
* Dual-licensed under AGPL-3.0 and commercial license. See LICENSE for details.
|
|
4
|
+
*/
|
|
5
|
+
|
|
6
|
+
/**
|
|
7
|
+
* The sizes e3 cuts a split task's input into pieces at.
|
|
8
|
+
*
|
|
9
|
+
* They live here, beside the wire types, so a caller that plans a split call
|
|
10
|
+
* in a browser — the query builder — weighs a dataset against the smallest
|
|
11
|
+
* piece e3 cuts without importing e3-core. e3-core's piece rule closes pieces
|
|
12
|
+
* at these sizes, and its `pieceSizes()` gives the sizes a process plans with:
|
|
13
|
+
* these, or the sizes a test sets.
|
|
14
|
+
*
|
|
15
|
+
* @packageDocumentation
|
|
16
|
+
*/
|
|
17
|
+
|
|
18
|
+
/**
|
|
19
|
+
* The sizes the piece rule closes pieces at, in stored bytes: a piece closes
|
|
20
|
+
* only once it holds `min`, weighs its segments against `max` until it holds
|
|
21
|
+
* `target` and against `min` after, and always closes once it holds `max`.
|
|
22
|
+
*
|
|
23
|
+
* @remarks
|
|
24
|
+
* A dataset that weighs no more than `min` is one piece, which is why a
|
|
25
|
+
* caller compares a dataset's stored bytes with `min` to tell whether a split
|
|
26
|
+
* call would split it.
|
|
27
|
+
*/
|
|
28
|
+
export interface PieceSizes {
|
|
29
|
+
/** The least a piece holds, unless the input ends first. */
|
|
30
|
+
readonly min: number;
|
|
31
|
+
/** Where a piece starts closing readily: the pieces' middle size, which a
|
|
32
|
+
* merge range aims for too. */
|
|
33
|
+
readonly target: number;
|
|
34
|
+
/** The most a piece holds, but for the rest of a `by` group. */
|
|
35
|
+
readonly max: number;
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
/**
|
|
39
|
+
* The platform's piece sizes: 16, 64 and 256 MiB.
|
|
40
|
+
*
|
|
41
|
+
* @example
|
|
42
|
+
* ```ts
|
|
43
|
+
* import { PIECE_SIZES } from '@elaraai/e3-types';
|
|
44
|
+
*
|
|
45
|
+
* // A dataset within one piece is read in one call.
|
|
46
|
+
* const split = storedBytes > PIECE_SIZES.min;
|
|
47
|
+
* ```
|
|
48
|
+
*/
|
|
49
|
+
export const PIECE_SIZES: PieceSizes = { min: 16 * 2 ** 20, target: 64 * 2 ** 20, max: 256 * 2 ** 20 };
|
package/src/release.ts
CHANGED
|
@@ -17,7 +17,7 @@
|
|
|
17
17
|
*/
|
|
18
18
|
|
|
19
19
|
/** The release of e3 this build is. */
|
|
20
|
-
export const E3_RELEASE = '1.0.
|
|
20
|
+
export const E3_RELEASE = '1.0.85';
|
|
21
21
|
|
|
22
22
|
/** A semantic version: its core, a pre-release, and build metadata. */
|
|
23
23
|
const RELEASE = /^(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(?:-([0-9A-Za-z-]+(?:\.[0-9A-Za-z-]+)*))?(?:\+[0-9A-Za-z-]+(?:\.[0-9A-Za-z-]+)*)?$/;
|
package/src/repository.ts
CHANGED
|
@@ -34,6 +34,24 @@ export const RepositoryRecordType = StructType({
|
|
|
34
34
|
|
|
35
35
|
export type RepositoryRecord = ValueTypeOf<typeof RepositoryRecordType>;
|
|
36
36
|
|
|
37
|
+
/**
|
|
38
|
+
* A store upgrade under way in a repository, kept beside the repository
|
|
39
|
+
* record: the step, the release of e3 applying it, and where its last part
|
|
40
|
+
* stopped. A step applies in parts, and the next part takes it up from here,
|
|
41
|
+
* in this process or another; the record lists the step only once it is done.
|
|
42
|
+
*/
|
|
43
|
+
export const RepositoryUpgradeProgressType = StructType({
|
|
44
|
+
/** The step's name */
|
|
45
|
+
step: StringType,
|
|
46
|
+
/** The release of e3 applying it */
|
|
47
|
+
release: StringType,
|
|
48
|
+
/** Where its last part stopped: the step's own cursor, which the next part
|
|
49
|
+
* takes it up from */
|
|
50
|
+
cursor: StringType,
|
|
51
|
+
});
|
|
52
|
+
|
|
53
|
+
export type RepositoryUpgradeProgress = ValueTypeOf<typeof RepositoryUpgradeProgressType>;
|
|
54
|
+
|
|
37
55
|
/**
|
|
38
56
|
* Where a repository is in its lifecycle.
|
|
39
57
|
*/
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Copyright (c) 2025 Elara AI Pty Ltd
|
|
3
|
+
* Dual-licensed under AGPL-3.0 and commercial license. See LICENSE for details.
|
|
4
|
+
*/
|
|
5
|
+
|
|
6
|
+
/**
|
|
7
|
+
* Keypaths: `parseKeypath` reads back what `pathToString` writes, whatever a
|
|
8
|
+
* field is named, with no structure to check it against; `parsePath` reads the
|
|
9
|
+
* same syntax and checks each field against a structure as it reads it.
|
|
10
|
+
*/
|
|
11
|
+
|
|
12
|
+
import { describe, it } from 'node:test';
|
|
13
|
+
import assert from 'node:assert/strict';
|
|
14
|
+
import { IntegerType, equalFor, printFor, toEastTypeValue, variant } from '@elaraai/east';
|
|
15
|
+
import { TreePathType, parseKeypath, parsePath, pathToString, type Structure, type TreePath } from './structure.js';
|
|
16
|
+
|
|
17
|
+
const equalPath = equalFor(TreePathType);
|
|
18
|
+
const printPath = printFor(TreePathType);
|
|
19
|
+
|
|
20
|
+
describe('parseKeypath', () => {
|
|
21
|
+
it('reads back what pathToString writes, whatever a field is named', () => {
|
|
22
|
+
const names = ['inputs', 'my field', 'a.b', 'a/b', 'back`tick', 'back\\slash', 'café', '1st', '', '`'];
|
|
23
|
+
for (const name of names) {
|
|
24
|
+
const path: TreePath = [variant('field', 'inputs'), variant('field', name)];
|
|
25
|
+
const printed = pathToString(path);
|
|
26
|
+
if (name === '') {
|
|
27
|
+
// An empty field name prints as nothing a keypath can read.
|
|
28
|
+
assert.throws(() => parseKeypath(printed));
|
|
29
|
+
continue;
|
|
30
|
+
}
|
|
31
|
+
const read = parseKeypath(printed);
|
|
32
|
+
assert.ok(equalPath(read, path), `${JSON.stringify(name)}: ${printed} read back as ${printPath(read)}`);
|
|
33
|
+
}
|
|
34
|
+
});
|
|
35
|
+
|
|
36
|
+
it('reads the empty string as the root', () => {
|
|
37
|
+
assert.ok(equalPath(parseKeypath(''), []));
|
|
38
|
+
});
|
|
39
|
+
|
|
40
|
+
it('refuses a string that is not a keypath, naming where', () => {
|
|
41
|
+
assert.throws(() => parseKeypath('inputs.x'), { message: "parseKeypath: unexpected character at position 0: 'i'" });
|
|
42
|
+
assert.throws(() => parseKeypath('.inputs.'), { message: "parseKeypath: expected identifier after '.' at position 8" });
|
|
43
|
+
assert.throws(() => parseKeypath('.inputs!'), { message: "parseKeypath: unexpected character at position 7: '!'" });
|
|
44
|
+
});
|
|
45
|
+
});
|
|
46
|
+
|
|
47
|
+
describe('parsePath', () => {
|
|
48
|
+
const structure: Structure = variant('struct', new Map([
|
|
49
|
+
['inputs', variant('struct', new Map([
|
|
50
|
+
['sales', variant('value', { type: toEastTypeValue(IntegerType), writable: true })],
|
|
51
|
+
]))],
|
|
52
|
+
]));
|
|
53
|
+
|
|
54
|
+
it('reads a keypath, and the structure at it', () => {
|
|
55
|
+
const { path, structure: at } = parsePath('.inputs.sales', structure);
|
|
56
|
+
assert.ok(equalPath(path, [variant('field', 'inputs'), variant('field', 'sales')]));
|
|
57
|
+
assert.equal(at.type, 'value');
|
|
58
|
+
});
|
|
59
|
+
|
|
60
|
+
it('refuses a field the structure lacks before a malformed one after it, and a path into a dataset', () => {
|
|
61
|
+
assert.throws(() => parsePath('.nope!', structure), { message: "parsePath: field 'nope' not found at ''. Available: inputs" });
|
|
62
|
+
assert.throws(() => parsePath('.inputs!', structure), { message: "parsePath: unexpected character at position 7: '!'" });
|
|
63
|
+
assert.throws(() => parsePath('.inputs.sales.more', structure), { message: "parsePath: cannot descend into dataset at '.inputs.sales'" });
|
|
64
|
+
});
|
|
65
|
+
});
|
package/src/structure.ts
CHANGED
|
@@ -181,10 +181,69 @@ export interface ParsePathResult {
|
|
|
181
181
|
* ```
|
|
182
182
|
*/
|
|
183
183
|
export function parsePath(pathStr: string, structure: Structure): ParsePathResult {
|
|
184
|
-
if (pathStr === '') return { path: [], structure };
|
|
185
|
-
|
|
186
184
|
const segments: TreePath = [];
|
|
187
185
|
let currentStructure = structure;
|
|
186
|
+
|
|
187
|
+
// Each field is checked as it is read, so a field the structure lacks is
|
|
188
|
+
// refused before a malformed one after it.
|
|
189
|
+
for (const fieldName of keypathFields(pathStr, 'parsePath')) {
|
|
190
|
+
if (currentStructure.type === 'value') {
|
|
191
|
+
throw new Error(`parsePath: cannot descend into dataset at '${pathToString(segments)}'`);
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
// currentStructure.type === 'struct' (only other option after 'value' check)
|
|
195
|
+
const fields = currentStructure.value;
|
|
196
|
+
const childStructure = fields.get(fieldName);
|
|
197
|
+
if (childStructure === undefined) {
|
|
198
|
+
const available = [...fields.keys()].map(k => printIdentifier(k)).join(', ');
|
|
199
|
+
throw new Error(`parsePath: field '${fieldName}' not found at '${pathToString(segments)}'. Available: ${available}`);
|
|
200
|
+
}
|
|
201
|
+
segments.push(variant('field', fieldName));
|
|
202
|
+
currentStructure = childStructure;
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
return { path: segments, structure: currentStructure };
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
/**
|
|
209
|
+
* Parses an East keypath string into a path, with no structure to check it
|
|
210
|
+
* against: the inverse of {@link pathToString}.
|
|
211
|
+
*
|
|
212
|
+
* @remarks
|
|
213
|
+
* The syntax is {@link parsePath}'s: `.field` for a simple identifier, and
|
|
214
|
+
* `` .`field` `` for one that needs quoting, with `` \` `` for a literal
|
|
215
|
+
* backtick and `\\` for a backslash. Whether the path names anything is the
|
|
216
|
+
* caller's to judge, as a status asked for named datasets leaves out a path
|
|
217
|
+
* the workspace does not hold.
|
|
218
|
+
*
|
|
219
|
+
* @param pathStr - A keypath string (e.g., ".inputs.sales", or ".inputs.`my field`")
|
|
220
|
+
* @returns The path; the root's, `[]`, for the empty string
|
|
221
|
+
*
|
|
222
|
+
* @throws {Error} When the string is not a keypath, such as one that does not
|
|
223
|
+
* start with `.`
|
|
224
|
+
*
|
|
225
|
+
* @example
|
|
226
|
+
* ```ts
|
|
227
|
+
* parseKeypath('.inputs.sales');
|
|
228
|
+
* // [field('inputs'), field('sales')]
|
|
229
|
+
*
|
|
230
|
+
* parseKeypath(pathToString([variant('field', 'inputs'), variant('field', 'my field')]));
|
|
231
|
+
* // [field('inputs'), field('my field')]
|
|
232
|
+
* ```
|
|
233
|
+
*/
|
|
234
|
+
export function parseKeypath(pathStr: string): TreePath {
|
|
235
|
+
return [...keypathFields(pathStr, 'parseKeypath')].map((fieldName) => variant('field', fieldName));
|
|
236
|
+
}
|
|
237
|
+
|
|
238
|
+
/**
|
|
239
|
+
* Reads a keypath string's field names in order, refusing a malformed one when
|
|
240
|
+
* it reaches it.
|
|
241
|
+
*
|
|
242
|
+
* @param pathStr - The keypath string
|
|
243
|
+
* @param caller - The function a refusal names
|
|
244
|
+
* @internal
|
|
245
|
+
*/
|
|
246
|
+
function* keypathFields(pathStr: string, caller: string): Generator<string> {
|
|
188
247
|
let pos = 0;
|
|
189
248
|
|
|
190
249
|
while (pos < pathStr.length) {
|
|
@@ -221,29 +280,13 @@ export function parsePath(pathStr: string, structure: Structure): ParsePathResul
|
|
|
221
280
|
}
|
|
222
281
|
|
|
223
282
|
if (fieldName.length === 0) {
|
|
224
|
-
throw new Error(
|
|
225
|
-
}
|
|
226
|
-
|
|
227
|
-
// Validate against structure
|
|
228
|
-
if (currentStructure.type === 'value') {
|
|
229
|
-
throw new Error(`parsePath: cannot descend into dataset at '${pathToString(segments)}'`);
|
|
283
|
+
throw new Error(`${caller}: expected identifier after '.' at position ${pos}`);
|
|
230
284
|
}
|
|
231
|
-
|
|
232
|
-
// currentStructure.type === 'struct' (only other option after 'value' check)
|
|
233
|
-
const fields = currentStructure.value;
|
|
234
|
-
const childStructure = fields.get(fieldName);
|
|
235
|
-
if (childStructure === undefined) {
|
|
236
|
-
const available = [...fields.keys()].map(k => printIdentifier(k)).join(', ');
|
|
237
|
-
throw new Error(`parsePath: field '${fieldName}' not found at '${pathToString(segments)}'. Available: ${available}`);
|
|
238
|
-
}
|
|
239
|
-
segments.push(variant('field', fieldName));
|
|
240
|
-
currentStructure = childStructure;
|
|
285
|
+
yield fieldName;
|
|
241
286
|
} else {
|
|
242
|
-
throw new Error(
|
|
287
|
+
throw new Error(`${caller}: unexpected character at position ${pos}: '${pathStr[pos]}'`);
|
|
243
288
|
}
|
|
244
289
|
}
|
|
245
|
-
|
|
246
|
-
return { path: segments, structure: currentStructure };
|
|
247
290
|
}
|
|
248
291
|
|
|
249
292
|
/**
|