@elaraai/east-node-cli 1.0.79 → 1.0.80
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +35 -105
- package/dist/cli.d.ts.map +1 -1
- package/dist/cli.js +37 -135
- package/dist/cli.js.map +1 -1
- package/dist/exec.d.ts +36 -0
- package/dist/exec.d.ts.map +1 -0
- package/dist/exec.js +296 -0
- package/dist/exec.js.map +1 -0
- package/dist/exec.spec.d.ts +6 -0
- package/dist/exec.spec.d.ts.map +1 -0
- package/dist/exec.spec.js +91 -0
- package/dist/exec.spec.js.map +1 -0
- package/dist/loader.d.ts +23 -3
- package/dist/loader.d.ts.map +1 -1
- package/dist/loader.js +190 -36
- package/dist/loader.js.map +1 -1
- package/dist/loader.spec.js +188 -4
- package/dist/loader.spec.js.map +1 -1
- package/dist/merge.d.ts +8 -8
- package/dist/merge.d.ts.map +1 -1
- package/dist/merge.js +167 -221
- package/dist/merge.js.map +1 -1
- package/dist/runner.d.ts +25 -49
- package/dist/runner.d.ts.map +1 -1
- package/dist/runner.js +76 -214
- package/dist/runner.js.map +1 -1
- package/dist/runner.spec.js +191 -336
- package/dist/runner.spec.js.map +1 -1
- package/package.json +4 -6
- package/dist/emit-writer.d.ts +0 -74
- package/dist/emit-writer.d.ts.map +0 -1
- package/dist/emit-writer.js +0 -159
- package/dist/emit-writer.js.map +0 -1
- package/dist/snapshot.d.ts +0 -19
- package/dist/snapshot.d.ts.map +0 -1
- package/dist/snapshot.js +0 -116
- package/dist/snapshot.js.map +0 -1
package/dist/runner.spec.js
CHANGED
|
@@ -14,10 +14,9 @@ import { tmpdir } from 'node:os';
|
|
|
14
14
|
import { join } from 'node:path';
|
|
15
15
|
import { spawn, spawnSync } from 'node:child_process';
|
|
16
16
|
import { fileURLToPath } from 'node:url';
|
|
17
|
-
import { ArrayType, DictType, FloatType, FunctionType, IntegerType, NullType, OptionType, SetType, StringType, StructType, East, SortedMap, SortedSet, compareFor, decodeBeast2For, encodeBeast2For, encodeBeast2PagedFor, toEastTypeValue, encodeEastIR, none, openBeast2PagesFor, some, spliceBeast2, } from '@elaraai/east';
|
|
17
|
+
import { ArrayType, DictType, FloatType, FunctionType, IntegerType, NullType, OptionType, SetType, StringType, StructType, Beast2ManifestWriter, East, SortedMap, SortedSet, UnitType, compareFor, decodeBeast2For, decodeCollectionManifest, encodeBeast2For, encodeBeast2PagedFor, encodeBeast2SegmentsFor, toEastTypeValue, encodeEastIR, none, openBeast2PagesFor, some, spliceBeast2, variant, } from '@elaraai/east';
|
|
18
18
|
import { runProgram, lazyThreshold } from './runner.js';
|
|
19
19
|
import { mergeBlobs } from './merge.js';
|
|
20
|
-
import { EmitFileWriter } from './emit-writer.js';
|
|
21
20
|
/** Runs `run` with console.error captured; returns what it printed. */
|
|
22
21
|
async function stderrOf(run) {
|
|
23
22
|
const lines = [];
|
|
@@ -31,6 +30,20 @@ async function stderrOf(run) {
|
|
|
31
30
|
}
|
|
32
31
|
return lines.join('\n');
|
|
33
32
|
}
|
|
33
|
+
/** Runs `run` with the lazy-open threshold set to `bytes`. */
|
|
34
|
+
async function withLazyThreshold(bytes, run) {
|
|
35
|
+
const saved = process.env.EAST_LAZY_INPUT_BYTES;
|
|
36
|
+
process.env.EAST_LAZY_INPUT_BYTES = String(bytes);
|
|
37
|
+
try {
|
|
38
|
+
return await run();
|
|
39
|
+
}
|
|
40
|
+
finally {
|
|
41
|
+
if (saved === undefined)
|
|
42
|
+
delete process.env.EAST_LAZY_INPUT_BYTES;
|
|
43
|
+
else
|
|
44
|
+
process.env.EAST_LAZY_INPUT_BYTES = saved;
|
|
45
|
+
}
|
|
46
|
+
}
|
|
34
47
|
/** `promise`'s value, or `undefined` once `ms` pass first — a bounded wait. */
|
|
35
48
|
async function within(promise, ms) {
|
|
36
49
|
let timer;
|
|
@@ -59,14 +72,15 @@ describe('runner output encoding', () => {
|
|
|
59
72
|
return new Uint8Array(readFileSync(outputPath));
|
|
60
73
|
}
|
|
61
74
|
it('writes large collection outputs segmented and indexed', async () => {
|
|
62
|
-
const output = await runToOutput(
|
|
75
|
+
const output = await runToOutput(20000n);
|
|
63
76
|
const AT = ArrayType(IntegerType);
|
|
64
77
|
const pages = openBeast2PagesFor(AT)(output);
|
|
65
|
-
assert.
|
|
66
|
-
assert.equal(pages.elementCount,
|
|
78
|
+
assert.ok(pages.segmentCount > 1);
|
|
79
|
+
assert.equal(pages.elementCount, 20_000);
|
|
67
80
|
assert.ok(pages.selfContained);
|
|
68
|
-
const expected = Array.from({ length:
|
|
69
|
-
|
|
81
|
+
const expected = Array.from({ length: 20_000 }, (_, i) => BigInt(i));
|
|
82
|
+
const edge = pages.counts[0];
|
|
83
|
+
assert.deepEqual(pages.slice(edge - 100, 200), expected.slice(edge - 100, edge + 100), 'window spans segments');
|
|
70
84
|
assert.deepEqual(decodeBeast2For(AT)(output), expected, 'whole decode equals the result');
|
|
71
85
|
});
|
|
72
86
|
it('writes small collection outputs segmented and indexed too', async () => {
|
|
@@ -80,10 +94,10 @@ describe('runner output encoding', () => {
|
|
|
80
94
|
assert.deepEqual(decodeBeast2For(AT)(output), expected, 'whole decode equals the result');
|
|
81
95
|
});
|
|
82
96
|
});
|
|
83
|
-
describe('
|
|
97
|
+
describe('lazy inputs', () => {
|
|
84
98
|
let tempDir;
|
|
85
99
|
beforeEach(() => {
|
|
86
|
-
tempDir = mkdtempSync(join(tmpdir(), 'east-node-
|
|
100
|
+
tempDir = mkdtempSync(join(tmpdir(), 'east-node-lazy-'));
|
|
87
101
|
});
|
|
88
102
|
afterEach(() => {
|
|
89
103
|
rmSync(tempDir, { recursive: true, force: true });
|
|
@@ -93,86 +107,16 @@ describe('runner streaming execution', () => {
|
|
|
93
107
|
writeFileSync(irPath, encodeEastIR(fn.toIR()));
|
|
94
108
|
return irPath;
|
|
95
109
|
}
|
|
96
|
-
it('a producer body emits an indexed Array output incrementally', async () => {
|
|
97
|
-
const fn = East.function([FunctionType([IntegerType], NullType)], NullType, ($, emit) => {
|
|
98
|
-
$.for(East.Array.range(0n, 2500n), ($, i) => {
|
|
99
|
-
$(emit(i.multiply(2n)));
|
|
100
|
-
});
|
|
101
|
-
});
|
|
102
|
-
const outputPath = join(tempDir, 'output.beast2');
|
|
103
|
-
await runProgram(writeIr(fn), [], [], [], outputPath, { emit: 'array' });
|
|
104
|
-
const output = new Uint8Array(readFileSync(outputPath));
|
|
105
|
-
const pages = openBeast2PagesFor(ArrayType(IntegerType))(output);
|
|
106
|
-
assert.equal(pages.elementCount, 2500);
|
|
107
|
-
assert.ok(pages.selfContained);
|
|
108
|
-
assert.equal(pages.element(1234), 2468n);
|
|
109
|
-
const expected = Array.from({ length: 2500 }, (_, i) => BigInt(i * 2));
|
|
110
|
-
assert.deepEqual(decodeBeast2For(ArrayType(IntegerType))(output), expected);
|
|
111
|
-
});
|
|
112
|
-
it('a stream input folds through emit with the input opened lazily', async () => {
|
|
113
|
-
const AT = ArrayType(IntegerType);
|
|
114
|
-
const events = Array.from({ length: 2500 }, (_, i) => BigInt(i));
|
|
115
|
-
const inputPath = join(tempDir, 'events.beast2');
|
|
116
|
-
writeFileSync(inputPath, encodeBeast2PagedFor(AT, { batchSize: 500 })(events));
|
|
117
|
-
const fn = East.function([AT, FunctionType([IntegerType], NullType)], NullType, ($, events, emit) => {
|
|
118
|
-
const acc = $.let(0n);
|
|
119
|
-
$.for(events, ($, v) => {
|
|
120
|
-
$.assign(acc, acc.add(v));
|
|
121
|
-
$(emit(acc));
|
|
122
|
-
});
|
|
123
|
-
});
|
|
124
|
-
const outputPath = join(tempDir, 'output.beast2');
|
|
125
|
-
await runProgram(writeIr(fn), [], [], [inputPath], outputPath, { emit: 'array', streamInputs: [0] });
|
|
126
|
-
const decoded = decodeBeast2For(AT)(new Uint8Array(readFileSync(outputPath)));
|
|
127
|
-
assert.equal(decoded.length, 2500);
|
|
128
|
-
assert.equal(decoded[0], 0n);
|
|
129
|
-
assert.equal(decoded[2499], (2499n * 2500n) / 2n);
|
|
130
|
-
});
|
|
131
|
-
/** A dict producer emitting `row-${i}` for each key in the given order. */
|
|
132
|
-
function dictEmitter(order) {
|
|
133
|
-
const emitType = FunctionType([IntegerType, StringType], NullType);
|
|
134
|
-
return East.function([emitType], NullType, ($, emit) => {
|
|
135
|
-
$.for($.const(order, ArrayType(IntegerType)), ($, i) => {
|
|
136
|
-
$(emit(i, East.str `row-${i}`));
|
|
137
|
-
});
|
|
138
|
-
});
|
|
139
|
-
}
|
|
140
|
-
it('an ascending dict emit writes the canonical blob, segmented and indexed', async () => {
|
|
141
|
-
const outputPath = join(tempDir, 'ordered.beast2');
|
|
142
|
-
const ascending = Array.from({ length: 1000 }, (_, i) => BigInt(i));
|
|
143
|
-
await runProgram(writeIr(dictEmitter(ascending)), [], [], [], outputPath, { emit: 'dict' });
|
|
144
|
-
const output = new Uint8Array(readFileSync(outputPath));
|
|
145
|
-
const DT = DictType(IntegerType, StringType);
|
|
146
|
-
const decoded = decodeBeast2For(DT)(output);
|
|
147
|
-
assert.equal(decoded.size, 1000);
|
|
148
|
-
assert.equal(decoded.get(42n), 'row-42');
|
|
149
|
-
assert.ok(openBeast2PagesFor(DT)(output).selfContained);
|
|
150
|
-
});
|
|
151
|
-
it('a dict emit rejects an out-of-order key with the ascending contract\'s error (#770)', async () => {
|
|
152
|
-
// Set/Dict emissions must ascend in East order: the sink writes one pass
|
|
153
|
-
// and never buffers, so a key below the previous one is the error — in
|
|
154
|
-
// the same words on every runner — and nothing is reported on stderr on
|
|
155
|
-
// the way there.
|
|
156
|
-
const err = await stderrOf(() => assert.rejects(runProgram(writeIr(dictEmitter([2n, 1n])), [], [], [], join(tempDir, 'disorder.beast2'), { emit: 'dict' }), { message: 'beast2 v5: Dict key emitted out of order: 1 after 2 — Set/Dict emissions must ascend in East order' }));
|
|
157
|
-
assert.equal(err, '');
|
|
158
|
-
});
|
|
159
|
-
it('a duplicate dict key is a hard error', async () => {
|
|
160
|
-
const emitType = FunctionType([IntegerType, StringType], NullType);
|
|
161
|
-
const adjacent = East.function([emitType], NullType, ($, emit) => {
|
|
162
|
-
$(emit(1n, 'a'));
|
|
163
|
-
$(emit(1n, 'b'));
|
|
164
|
-
});
|
|
165
|
-
await assert.rejects(runProgram(writeIr(adjacent), [], [], [], join(tempDir, 'dup.beast2'), { emit: 'dict' }), { message: 'beast2 v5: duplicate Dict key emitted: 1 — Dict keys must be unique' });
|
|
166
|
-
});
|
|
167
110
|
it('a lazy dict input serves keyed reads without a whole decode', async () => {
|
|
168
111
|
const DT = DictType(IntegerType, StringType);
|
|
169
112
|
const table = new SortedMap(Array.from({ length: 2000 }, (_, i) => [BigInt(i), `row-${i}`]), compareFor(IntegerType));
|
|
170
113
|
const inputPath = join(tempDir, 'table.beast2');
|
|
171
|
-
|
|
114
|
+
const entries = [...table];
|
|
115
|
+
writeFileSync(inputPath, encodeBeast2SegmentsFor(DT)(Array.from({ length: 8 }, (_, i) => new Map(entries.slice(i * 250, (i + 1) * 250)))));
|
|
172
116
|
const fn = East.function([DT], StringType, ($, table) => table.get(1234n));
|
|
173
117
|
const outputPath = join(tempDir, 'output.beast2');
|
|
174
118
|
// Threshold of 1 byte forces the lazy open for any input size.
|
|
175
|
-
await runProgram(writeIr(fn), [], [], [inputPath], outputPath
|
|
119
|
+
await withLazyThreshold(1, () => runProgram(writeIr(fn), [], [], [inputPath], outputPath));
|
|
176
120
|
const result = decodeBeast2For(StringType)(new Uint8Array(readFileSync(outputPath)));
|
|
177
121
|
assert.equal(result, 'row-1234');
|
|
178
122
|
});
|
|
@@ -180,19 +124,12 @@ describe('runner streaming execution', () => {
|
|
|
180
124
|
const DT = DictType(IntegerType, StringType);
|
|
181
125
|
const table = new SortedMap(Array.from({ length: 500 }, (_, i) => [BigInt(i), `row-${i}`]), compareFor(IntegerType));
|
|
182
126
|
const inputPath = join(tempDir, 'table.beast2');
|
|
183
|
-
|
|
127
|
+
const entries = [...table];
|
|
128
|
+
writeFileSync(inputPath, encodeBeast2SegmentsFor(DT)(Array.from({ length: 5 }, (_, i) => new Map(entries.slice(i * 100, (i + 1) * 100)))));
|
|
184
129
|
const fn = East.function([DT], StringType, ($, table) => table.get(42n));
|
|
185
130
|
const outputPath = join(tempDir, 'output.beast2');
|
|
186
|
-
const
|
|
187
|
-
|
|
188
|
-
console.error = (...args) => { lines.push(args.map(String).join(' ')); };
|
|
189
|
-
try {
|
|
190
|
-
await runProgram(writeIr(fn), [], [], [inputPath], outputPath, { lazyInputBytes: 1, verbose: true });
|
|
191
|
-
}
|
|
192
|
-
finally {
|
|
193
|
-
console.error = original;
|
|
194
|
-
}
|
|
195
|
-
assert.ok(lines.some((l) => l.includes('input 0: opened lazily')), `verbose output names the lazy input:\n${lines.join('\n')}`);
|
|
131
|
+
const err = await stderrOf(() => withLazyThreshold(1, () => runProgram(writeIr(fn), [], [], [inputPath], outputPath, true)));
|
|
132
|
+
assert.ok(err.includes('input 0: opened lazily'), `verbose output names the lazy input:\n${err}`);
|
|
196
133
|
assert.equal(decodeBeast2For(StringType)(new Uint8Array(readFileSync(outputPath))), 'row-42');
|
|
197
134
|
});
|
|
198
135
|
it('a lazily opened wide input reads a fraction of the file and stays below the eager run\'s residency', { skip: process.platform === 'win32' ? 'no RSS comparison on Windows' : false }, () => {
|
|
@@ -242,40 +179,11 @@ describe('runner streaming execution', () => {
|
|
|
242
179
|
assert.ok(lazy !== null && eager !== null, 'the runner reports its RSS');
|
|
243
180
|
assert.ok(lazy < eager, `lazy peak ${lazy} MB not below eager peak ${eager} MB for a ${wireMb.toFixed(1)} MB input`);
|
|
244
181
|
});
|
|
245
|
-
it('refuses --emit with a non-.beast2 output, like east-c and east-py', async () => {
|
|
246
|
-
const fn = East.function([FunctionType([IntegerType], NullType)], NullType, ($, emit) => {
|
|
247
|
-
$(emit(1n));
|
|
248
|
-
});
|
|
249
|
-
await assert.rejects(runProgram(writeIr(fn), [], [], [], join(tempDir, 'out.json'), { emit: 'array' }), /--emit requires a \.beast2 output file \(-o\)/);
|
|
250
|
-
});
|
|
251
|
-
it('refuses --emit on a zero-parameter function with the emit-capability error', async () => {
|
|
252
|
-
// The one sentence east-c and east-py give for the same shape — a
|
|
253
|
-
// zero-parameter function has no trailing parameter to be the capability,
|
|
254
|
-
// and so does one whose trailing parameter is not a function.
|
|
255
|
-
const message = "--emit requires the function's trailing parameter to be the emit capability (a function type)";
|
|
256
|
-
await assert.rejects(runProgram(writeIr(East.function([], IntegerType, (_$) => 1n)), [], [], [], join(tempDir, 'out.beast2'), { emit: 'array' }), { message });
|
|
257
|
-
await assert.rejects(runProgram(writeIr(East.function([IntegerType], IntegerType, (_$, n) => n)), [], [], [], join(tempDir, 'out.beast2'), { emit: 'array' }), { message });
|
|
258
|
-
});
|
|
259
|
-
it('a set emit rejects an out-of-order element and names duplicates with the element noun', async () => {
|
|
260
|
-
const emitType = FunctionType([IntegerType], NullType);
|
|
261
|
-
const disordered = East.function([emitType], NullType, ($, emit) => {
|
|
262
|
-
$(emit(2n));
|
|
263
|
-
$(emit(1n));
|
|
264
|
-
$(emit(3n));
|
|
265
|
-
});
|
|
266
|
-
await assert.rejects(runProgram(writeIr(disordered), [], [], [], join(tempDir, 'set.beast2'), { emit: 'set' }), { message: 'beast2 v5: Set element emitted out of order: 1 after 2 — Set/Dict emissions must ascend in East order' });
|
|
267
|
-
const dup = East.function([emitType], NullType, ($, emit) => {
|
|
268
|
-
$(emit(1n));
|
|
269
|
-
$(emit(2n));
|
|
270
|
-
$(emit(2n));
|
|
271
|
-
});
|
|
272
|
-
await assert.rejects(runProgram(writeIr(dup), [], [], [], join(tempDir, 'dupset.beast2'), { emit: 'set' }), { message: 'beast2 v5: duplicate Set element emitted: 2 — Set elements must be unique' });
|
|
273
|
-
});
|
|
274
182
|
});
|
|
275
|
-
describe('
|
|
183
|
+
describe('the blob merge and the stdin lifeline (#770)', () => {
|
|
276
184
|
let tempDir;
|
|
277
185
|
beforeEach(() => {
|
|
278
|
-
tempDir = mkdtempSync(join(tmpdir(), 'east-node-
|
|
186
|
+
tempDir = mkdtempSync(join(tmpdir(), 'east-node-merge-'));
|
|
279
187
|
});
|
|
280
188
|
afterEach(() => {
|
|
281
189
|
rmSync(tempDir, { recursive: true, force: true });
|
|
@@ -285,119 +193,64 @@ describe('folding emit, the blob merge and the stdin lifeline (#770)', () => {
|
|
|
285
193
|
writeFileSync(irPath, encodeEastIR(fn.toIR()));
|
|
286
194
|
return irPath;
|
|
287
195
|
}
|
|
288
|
-
const PairType = StructType({ key: IntegerType, value: StringType });
|
|
289
|
-
const emitPairType = FunctionType([IntegerType, StringType], NullType);
|
|
290
196
|
const emitKeyType = FunctionType([IntegerType], NullType);
|
|
291
|
-
const ascendingBigints = (a, b) => (a < b ? -1 : a > b ? 1 : 0);
|
|
292
197
|
const DT = DictType(IntegerType, StringType);
|
|
293
198
|
const ST = SetType(IntegerType);
|
|
294
|
-
/** The fold contract's emission sequence, as keys — the sequence
|
|
295
|
-
* generate_fixtures.mjs writes for east-c and east-py: 0..1199 in order
|
|
296
|
-
* with adjacent duplicates, every third key twice, and key 999 — the last
|
|
297
|
-
* entry of a full 1000-element batch — four times. */
|
|
298
|
-
function foldSequence() {
|
|
299
|
-
const keys = [];
|
|
300
|
-
for (let k = 0; k < 1200; k++) {
|
|
301
|
-
const copies = 1 + (k % 3 === 0 ? 1 : 0) + (k % 1000 === 999 ? 2 : 0);
|
|
302
|
-
for (let c = 0; c < copies; c++)
|
|
303
|
-
keys.push(BigInt(k));
|
|
304
|
-
}
|
|
305
|
-
return keys;
|
|
306
|
-
}
|
|
307
|
-
/** A dict producer emitting the given (key, value) pairs in order. */
|
|
308
|
-
function pairEmitter(pairs) {
|
|
309
|
-
return East.function([emitPairType], NullType, ($, emit) => {
|
|
310
|
-
$.for($.const(pairs, ArrayType(PairType)), ($, pair) => {
|
|
311
|
-
$(emit(pair.key, pair.value));
|
|
312
|
-
});
|
|
313
|
-
});
|
|
314
|
-
}
|
|
315
|
-
/** A set producer emitting the given keys in order. */
|
|
316
|
-
function keyEmitter(keys) {
|
|
317
|
-
return East.function([emitKeyType], NullType, ($, emit) => {
|
|
318
|
-
$.for($.const(keys, ArrayType(IntegerType)), ($, key) => {
|
|
319
|
-
$(emit(key));
|
|
320
|
-
});
|
|
321
|
-
});
|
|
322
|
-
}
|
|
323
199
|
/** The concatenating `(Integer, String, String) -> String` fold. */
|
|
324
200
|
function writeConcat() {
|
|
325
201
|
return writeIr('merge.beast2', East.function([IntegerType, StringType, StringType], StringType, (_$, _key, acc, value) => acc.concat(value)));
|
|
326
202
|
}
|
|
327
|
-
it('--merge writes the folded ascending sequence\'s bytes', async () => {
|
|
328
|
-
// The fold is over ADJACENT equal keys, and lands in place — key 999
|
|
329
|
-
// into a full batch's last entry included — so the output is
|
|
330
|
-
// byte-identical to what the flag-less sink writes for the folded
|
|
331
|
-
// sequence.
|
|
332
|
-
const pairs = foldSequence().map((key, i) => ({ key, value: `${i};` }));
|
|
333
|
-
const folded = new Map();
|
|
334
|
-
for (const { key, value } of pairs)
|
|
335
|
-
folded.set(key, (folded.get(key) ?? '') + value);
|
|
336
|
-
const foldedPairs = [...folded].sort(([a], [b]) => ascendingBigints(a, b)).map(([key, value]) => ({ key, value }));
|
|
337
|
-
const expectedPath = join(tempDir, 'expected.beast2');
|
|
338
|
-
await runProgram(writeIr('folded.beast2', pairEmitter(foldedPairs)), [], [], [], expectedPath, { emit: 'dict' });
|
|
339
|
-
const outputPath = join(tempDir, 'output.beast2');
|
|
340
|
-
await runProgram(writeIr('program.beast2', pairEmitter(pairs)), [], [], [], outputPath, { emit: 'dict', merge: writeConcat() });
|
|
341
|
-
assert.deepEqual(new Uint8Array(readFileSync(outputPath)), new Uint8Array(readFileSync(expectedPath)), 'the folded output must be byte-identical to the flag-less sink\'s output for the folded sequence');
|
|
342
|
-
});
|
|
343
|
-
it('--union writes the ascending sequence\'s distinct keys\' bytes', async () => {
|
|
344
|
-
const keys = foldSequence();
|
|
345
|
-
const distinct = [...new Set(keys)].sort(ascendingBigints);
|
|
346
|
-
const expectedPath = join(tempDir, 'expected.beast2');
|
|
347
|
-
await runProgram(writeIr('folded.beast2', keyEmitter(distinct)), [], [], [], expectedPath, { emit: 'set' });
|
|
348
|
-
const outputPath = join(tempDir, 'output.beast2');
|
|
349
|
-
await runProgram(writeIr('program.beast2', keyEmitter(keys)), [], [], [], outputPath, { emit: 'set', union: true });
|
|
350
|
-
assert.deepEqual(new Uint8Array(readFileSync(outputPath)), new Uint8Array(readFileSync(expectedPath)), 'the union output must be byte-identical to the flag-less sink\'s output for the distinct keys');
|
|
351
|
-
});
|
|
352
|
-
it('a fold does not admit an out-of-order key', async () => {
|
|
353
|
-
const pairs = [{ key: 2n, value: 'a' }, { key: 1n, value: 'b' }];
|
|
354
|
-
await assert.rejects(runProgram(writeIr('program.beast2', pairEmitter(pairs)), [], [], [], join(tempDir, 'output.beast2'), { emit: 'dict', merge: writeConcat() }), { message: 'beast2 v5: Dict key emitted out of order: 1 after 2 — Set/Dict emissions must ascend in East order' });
|
|
355
|
-
});
|
|
356
203
|
// ---- The blob merge ---------------------------------------------------
|
|
357
204
|
/** Three sorted Dict inputs whose keys overlap — a = 0..19, b = 10..29,
|
|
358
|
-
* c = {5, 15, 25, 40} — each value naming its input, written
|
|
359
|
-
*
|
|
205
|
+
* c = {5, 15, 25, 40} — each value naming its input, written in four-entry
|
|
206
|
+
* segments. */
|
|
360
207
|
const mergeKeys = {
|
|
361
208
|
a: Array.from({ length: 20 }, (_, k) => BigInt(k)),
|
|
362
209
|
b: Array.from({ length: 20 }, (_, k) => BigInt(k + 10)),
|
|
363
210
|
c: [5n, 15n, 25n, 40n],
|
|
364
211
|
};
|
|
212
|
+
/** Ascending entries as a blob of four-entry segments: a geometry chosen
|
|
213
|
+
* here rather than by the cut rule, so a range bound can fall inside a
|
|
214
|
+
* segment, on a fence, and between segments. */
|
|
215
|
+
function inFours(type, entries) {
|
|
216
|
+
const batches = [];
|
|
217
|
+
for (let i = 0; i < entries.length; i += 4) {
|
|
218
|
+
const chunk = entries.slice(i, i + 4);
|
|
219
|
+
batches.push(type.type === 'Dict' ? new Map(chunk) : new Set(chunk));
|
|
220
|
+
}
|
|
221
|
+
return encodeBeast2SegmentsFor(toEastTypeValue(type))(batches);
|
|
222
|
+
}
|
|
365
223
|
function writeDictInput(name) {
|
|
366
224
|
const path = join(tempDir, `in_${name}.beast2`);
|
|
367
|
-
writeFileSync(path,
|
|
225
|
+
writeFileSync(path, inFours(DT, mergeKeys[name].map((k) => [k, `${name}${k}`])));
|
|
368
226
|
return path;
|
|
369
227
|
}
|
|
370
228
|
function writeSetInput(name) {
|
|
371
229
|
const path = join(tempDir, `set_${name}.beast2`);
|
|
372
|
-
writeFileSync(path,
|
|
230
|
+
writeFileSync(path, inFours(ST, mergeKeys[name]));
|
|
373
231
|
return path;
|
|
374
232
|
}
|
|
375
|
-
it('merge writes the fold of overlapping dict inputs
|
|
376
|
-
const folded = new
|
|
233
|
+
it('merge writes the fold of overlapping dict inputs as the Writer writes the folded value', () => {
|
|
234
|
+
const folded = new SortedMap([], compareFor(IntegerType));
|
|
377
235
|
for (const name of ['a', 'b', 'c']) {
|
|
378
236
|
for (const k of mergeKeys[name])
|
|
379
237
|
folded.set(k, (folded.get(k) ?? '') + `${name}${k}`);
|
|
380
238
|
}
|
|
381
|
-
const foldedPairs = [...folded].sort(([a], [b]) => ascendingBigints(a, b)).map(([key, value]) => ({ key, value }));
|
|
382
|
-
const expectedPath = join(tempDir, 'expected.beast2');
|
|
383
|
-
await runProgram(writeIr('folded.beast2', pairEmitter(foldedPairs)), [], [], [], expectedPath, { emit: 'dict' });
|
|
384
239
|
const outputPath = join(tempDir, 'merged.beast2');
|
|
385
240
|
const inputs = [writeDictInput('a'), writeDictInput('b'), writeDictInput('c')];
|
|
386
241
|
const stats = mergeBlobs(inputs, outputPath, { mergePath: writeConcat() });
|
|
387
242
|
assert.deepEqual(stats, { inputs: 3, entries: 31, folds: 13 });
|
|
388
|
-
assert.deepEqual(new Uint8Array(readFileSync(outputPath)),
|
|
243
|
+
assert.deepEqual(new Uint8Array(readFileSync(outputPath)), encodeBeast2PagedFor(DT)(folded), 'the merge must write exactly what the Writer writes for the folded value');
|
|
389
244
|
const table = decodeBeast2For(DT)(new Uint8Array(readFileSync(outputPath)));
|
|
390
245
|
assert.equal(table.get(15n), 'a15b15c15');
|
|
391
246
|
assert.equal(table.get(40n), 'c40');
|
|
392
247
|
});
|
|
393
|
-
it('
|
|
394
|
-
const distinct =
|
|
395
|
-
const expectedPath = join(tempDir, 'expected.beast2');
|
|
396
|
-
await runProgram(writeIr('folded.beast2', keyEmitter(distinct)), [], [], [], expectedPath, { emit: 'set' });
|
|
248
|
+
it('a union merge writes the distinct elements of set inputs as the Writer writes them', () => {
|
|
249
|
+
const distinct = new SortedSet([...mergeKeys.a, ...mergeKeys.b, ...mergeKeys.c], compareFor(IntegerType));
|
|
397
250
|
const outputPath = join(tempDir, 'union.beast2');
|
|
398
251
|
const stats = mergeBlobs([writeSetInput('a'), writeSetInput('b'), writeSetInput('c')], outputPath, { union: true });
|
|
399
252
|
assert.deepEqual(stats, { inputs: 3, entries: 31, folds: 13 });
|
|
400
|
-
assert.deepEqual(new Uint8Array(readFileSync(outputPath)),
|
|
253
|
+
assert.deepEqual(new Uint8Array(readFileSync(outputPath)), encodeBeast2PagedFor(ST)(distinct));
|
|
401
254
|
assert.equal(openBeast2PagesFor(ST)(new Uint8Array(readFileSync(outputPath))).elementCount, 31);
|
|
402
255
|
});
|
|
403
256
|
it('merge without a fold refuses a shared key and leaves the output unfinalised', () => {
|
|
@@ -416,8 +269,8 @@ describe('folding emit, the blob merge and the stdin lifeline (#770)', () => {
|
|
|
416
269
|
// error, prefixed with the input — the same sentence east-c and east-py give.
|
|
417
270
|
const descending = join(tempDir, 'descending.beast2');
|
|
418
271
|
writeFileSync(descending, spliceBeast2([
|
|
419
|
-
encodeBeast2PagedFor(DT
|
|
420
|
-
encodeBeast2PagedFor(DT
|
|
272
|
+
encodeBeast2PagedFor(DT)(new SortedMap([[1000n, 'x'], [1001n, 'y']], compareFor(IntegerType))),
|
|
273
|
+
encodeBeast2PagedFor(DT)(new SortedMap([[1n, 'a'], [2n, 'b']], compareFor(IntegerType))),
|
|
421
274
|
]));
|
|
422
275
|
assert.throws(() => mergeBlobs([descending], join(tempDir, 'desc.beast2')), { message: `merge: input 0 (${descending}): beast2 v5: Dict keys are not strictly ascending in East order — the wire must hold the canonical value (corrupt or pre-contract blob)` });
|
|
423
276
|
const missing = join(tempDir, 'missing.beast2');
|
|
@@ -441,10 +294,10 @@ describe('folding emit, the blob merge and the stdin lifeline (#770)', () => {
|
|
|
441
294
|
});
|
|
442
295
|
it('merge names a fold of the wrong signature and refuses a fold of the wrong kind', () => {
|
|
443
296
|
const wrong = writeIr('wrong.beast2', East.function([StringType, StringType, StringType], StringType, (_$, _key, acc, value) => acc.concat(value)));
|
|
444
|
-
assert.throws(() => mergeBlobs([writeDictInput('a')], join(tempDir, 'w.beast2'), { mergePath: wrong }), (err) => err.message.startsWith('
|
|
445
|
-
assert.throws(() => mergeBlobs([writeSetInput('a')], join(tempDir, 'ms.beast2'), { mergePath: writeConcat() }), { message: '
|
|
446
|
-
assert.throws(() => mergeBlobs([writeDictInput('a')], join(tempDir, 'ud.beast2'), { union: true }), { message: '
|
|
447
|
-
assert.throws(() => mergeBlobs([writeDictInput('a')], join(tempDir, 'both.beast2'), { mergePath: writeConcat(), union: true }), { message: 'merge:
|
|
297
|
+
assert.throws(() => mergeBlobs([writeDictInput('a')], join(tempDir, 'w.beast2'), { mergePath: wrong }), (err) => err.message.startsWith('merge function: expected a function (K, V, V) -> V matching the inputs (K = '));
|
|
298
|
+
assert.throws(() => mergeBlobs([writeSetInput('a')], join(tempDir, 'ms.beast2'), { mergePath: writeConcat() }), { message: 'merge: a merge function applies to Dict inputs only' });
|
|
299
|
+
assert.throws(() => mergeBlobs([writeDictInput('a')], join(tempDir, 'ud.beast2'), { union: true }), { message: 'merge: union applies to Set inputs only' });
|
|
300
|
+
assert.throws(() => mergeBlobs([writeDictInput('a')], join(tempDir, 'both.beast2'), { mergePath: writeConcat(), union: true }), { message: 'merge: a merge function and union are two folds — give one' });
|
|
448
301
|
});
|
|
449
302
|
it('an empty input contributes nothing, and a lone empty input yields the empty collection, indexed', () => {
|
|
450
303
|
const empty = join(tempDir, 'empty.beast2');
|
|
@@ -460,7 +313,7 @@ describe('folding emit, the blob merge and the stdin lifeline (#770)', () => {
|
|
|
460
313
|
assert.equal(openBeast2PagesFor(DT)(new Uint8Array(readFileSync(emptyOut))).elementCount, 0);
|
|
461
314
|
});
|
|
462
315
|
// ---- Key ranges ---------------------------------------------------------
|
|
463
|
-
/** A
|
|
316
|
+
/** A range blob over Integer keys: `[from, to)`, a `null` bound open. */
|
|
464
317
|
const RangeT = StructType({ from: OptionType(IntegerType), to: OptionType(IntegerType) });
|
|
465
318
|
function writeRange(name, from, to) {
|
|
466
319
|
const path = join(tempDir, `range_${name}.beast2`);
|
|
@@ -472,15 +325,15 @@ describe('folding emit, the blob merge and the stdin lifeline (#770)', () => {
|
|
|
472
325
|
/** An input holding only its keys in the range — the sliced twin a ranged merge must equal. */
|
|
473
326
|
function writeDictSlice(name, from, to) {
|
|
474
327
|
const path = join(tempDir, `in_${name}_${from}_${to}.beast2`);
|
|
475
|
-
writeFileSync(path,
|
|
328
|
+
writeFileSync(path, inFours(DT, mergeKeys[name].filter((k) => inRange(k, from, to)).map((k) => [k, `${name}${k}`])));
|
|
476
329
|
return path;
|
|
477
330
|
}
|
|
478
331
|
function writeSetSlice(name, from, to) {
|
|
479
332
|
const path = join(tempDir, `set_${name}_${from}_${to}.beast2`);
|
|
480
|
-
writeFileSync(path,
|
|
333
|
+
writeFileSync(path, inFours(ST, mergeKeys[name].filter((k) => inRange(k, from, to))));
|
|
481
334
|
return path;
|
|
482
335
|
}
|
|
483
|
-
it('merge
|
|
336
|
+
it('a ranged merge writes the keys in [from, to) byte-identical to merging inputs that hold only those keys', () => {
|
|
484
337
|
// The inputs are written in four-entry segments, so the bounds fall
|
|
485
338
|
// inside segments, on a fence, before the first key, past the last,
|
|
486
339
|
// and one range holds nothing.
|
|
@@ -512,134 +365,110 @@ describe('folding emit, the blob merge and the stdin lifeline (#770)', () => {
|
|
|
512
365
|
assert.equal(openBeast2PagesFor(DT)(new Uint8Array(readFileSync(join(tempDir, 'ranged_7_22.beast2')))).elementCount, 15);
|
|
513
366
|
assert.equal(openBeast2PagesFor(DT)(new Uint8Array(readFileSync(join(tempDir, 'ranged_30_39.beast2')))).elementCount, 0, 'a range holding nothing writes the empty collection, indexed');
|
|
514
367
|
});
|
|
515
|
-
it('merge
|
|
368
|
+
it('a ranged merge refuses bounds of another key type, a missing file, an empty file and a file that is not a blob', () => {
|
|
516
369
|
const inputs = [writeDictInput('a'), writeDictInput('b')];
|
|
517
370
|
const mismatch = join(tempDir, 'range_mismatch.beast2');
|
|
518
371
|
writeFileSync(mismatch, encodeBeast2For(StructType({ from: OptionType(StringType), to: OptionType(StringType) }))({ from: none, to: none }));
|
|
519
|
-
assert.throws(() => mergeBlobs(inputs, join(tempDir, 'rm.beast2'), { mergePath: writeConcat(), rangePath: mismatch }), (err) => err.message.startsWith(`merge:
|
|
372
|
+
assert.throws(() => mergeBlobs(inputs, join(tempDir, 'rm.beast2'), { mergePath: writeConcat(), rangePath: mismatch }), (err) => err.message.startsWith(`merge: range (${mismatch}) has type `) && err.message.endsWith(" (bounds over the inputs' key type)"));
|
|
520
373
|
const missing = join(tempDir, 'range_missing.beast2');
|
|
521
|
-
assert.throws(() => mergeBlobs(inputs, join(tempDir, 'rm2.beast2'), { mergePath: writeConcat(), rangePath: missing }), { message: `merge:
|
|
374
|
+
assert.throws(() => mergeBlobs(inputs, join(tempDir, 'rm2.beast2'), { mergePath: writeConcat(), rangePath: missing }), { message: `merge: range (${missing}): cannot open the file` });
|
|
522
375
|
// Not a blob: the reader's own words, as east-c and east-py say them.
|
|
523
376
|
const text = join(tempDir, 'range_text.beast2');
|
|
524
377
|
writeFileSync(text, 'not a blob');
|
|
525
|
-
assert.throws(() => mergeBlobs(inputs, join(tempDir, 'rm3.beast2'), { mergePath: writeConcat(), rangePath: text }), { message: `merge:
|
|
378
|
+
assert.throws(() => mergeBlobs(inputs, join(tempDir, 'rm3.beast2'), { mergePath: writeConcat(), rangePath: text }), { message: `merge: range (${text}): Invalid Beast2 magic at offset 0: expected 0x89, got 0x6e` });
|
|
526
379
|
const empty = join(tempDir, 'range_empty.beast2');
|
|
527
380
|
writeFileSync(empty, '');
|
|
528
|
-
assert.throws(() => mergeBlobs(inputs, join(tempDir, 'rm4.beast2'), { mergePath: writeConcat(), rangePath: empty }), { message: `merge:
|
|
529
|
-
});
|
|
530
|
-
it('the
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
for (let i = 0; i < chars; i++) {
|
|
562
|
-
seed = (seed * 48271) % 2147483647;
|
|
563
|
-
out += alphabet[(seed >> 8) & 63];
|
|
564
|
-
}
|
|
565
|
-
return out;
|
|
566
|
-
};
|
|
567
|
-
for (const width of [4_000, 200_000]) {
|
|
568
|
-
seed = 12345;
|
|
569
|
-
const rows = Array.from({ length: 60 }, (_, i) => [BigInt(i), noise(width)]);
|
|
570
|
-
const value = new SortedMap(rows, compareFor(IntegerType));
|
|
571
|
-
const paged = encodeBeast2PagedFor(DT)(value);
|
|
572
|
-
const path = join(tempDir, `segments-${width}.beast2`);
|
|
573
|
-
const writer = new EmitFileWriter('dict', toEastTypeValue(DT), path);
|
|
574
|
-
for (const [key, text] of rows)
|
|
575
|
-
writer.push([key, text]);
|
|
576
|
-
writer.finishClose();
|
|
577
|
-
assert.deepEqual(new Uint8Array(readFileSync(path)), paged, `a ${width}-char row segments the same emitted as returned`);
|
|
381
|
+
assert.throws(() => mergeBlobs(inputs, join(tempDir, 'rm4.beast2'), { mergePath: writeConcat(), rangePath: empty }), { message: `merge: range (${empty}): Data too short for Beast2 format: 0 bytes` });
|
|
382
|
+
});
|
|
383
|
+
it('merge reads manifest directories, the form e3 stages a stored collection in, as east-c and east-py do', () => {
|
|
384
|
+
// Each input twice over: a blob, and a manifest directory — the manifest's
|
|
385
|
+
// file and, beside it in `<file>.segments/`, each object named by its
|
|
386
|
+
// SHA-256. Enough keys for several segments, so a range seeks through the
|
|
387
|
+
// manifest's fences.
|
|
388
|
+
const inputs = [0, 2000].map((start) => Array.from({ length: 3000 }, (_, k) => [BigInt(start + k), `v${start + k}`]));
|
|
389
|
+
const blobs = inputs.map((entries, i) => {
|
|
390
|
+
const path = join(tempDir, `wide_${i}.beast2`);
|
|
391
|
+
writeFileSync(path, encodeBeast2PagedFor(DT)(new SortedMap(entries, compareFor(IntegerType))));
|
|
392
|
+
return path;
|
|
393
|
+
});
|
|
394
|
+
const manifests = inputs.map((entries, i) => {
|
|
395
|
+
const path = join(tempDir, `wide_${i}.manifest.beast2`);
|
|
396
|
+
const dir = `${path}.segments`;
|
|
397
|
+
mkdirSync(dir);
|
|
398
|
+
const writer = new Beast2ManifestWriter(DT, {
|
|
399
|
+
object: (hash, bytes) => writeFileSync(join(dir, `${hash}.beast2`), bytes),
|
|
400
|
+
manifest: (bytes) => writeFileSync(path, bytes),
|
|
401
|
+
});
|
|
402
|
+
for (const entry of entries)
|
|
403
|
+
writer.add(entry);
|
|
404
|
+
writer.finish();
|
|
405
|
+
return path;
|
|
406
|
+
});
|
|
407
|
+
assert.ok(decodeCollectionManifest(new Uint8Array(readFileSync(manifests[0]))).entries.length > 1, 'several segments');
|
|
408
|
+
for (const rangePath of [undefined, writeRange('manifest', 1500n, 2600n)]) {
|
|
409
|
+
const options = { mergePath: writeConcat(), ...(rangePath !== undefined && { rangePath }) };
|
|
410
|
+
const fromBlobs = join(tempDir, 'from-blobs.beast2');
|
|
411
|
+
const fromManifests = join(tempDir, 'from-manifests.beast2');
|
|
412
|
+
assert.deepEqual(mergeBlobs([manifests[0], blobs[1]], fromManifests, options), mergeBlobs(blobs, fromBlobs, options));
|
|
413
|
+
assert.deepEqual(new Uint8Array(readFileSync(fromManifests)), new Uint8Array(readFileSync(fromBlobs)));
|
|
578
414
|
}
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
//
|
|
587
|
-
//
|
|
588
|
-
//
|
|
589
|
-
|
|
415
|
+
// A segment gone from the directory is named when the merge reaches it,
|
|
416
|
+
// in east-c's words.
|
|
417
|
+
const gone = `${manifests[1]}.segments/${decodeCollectionManifest(new Uint8Array(readFileSync(manifests[1]))).entries[0].hash}.beast2`;
|
|
418
|
+
rmSync(gone);
|
|
419
|
+
assert.throws(() => mergeBlobs([blobs[0], manifests[1]], join(tempDir, 'broken.beast2'), { mergePath: writeConcat() }), { message: `merge: input 1 (${manifests[1]}): beast2 v5: manifest segment ${gone} cannot be read` });
|
|
420
|
+
});
|
|
421
|
+
it('a container two entries share is written for each, and an older blob merges to that form', () => {
|
|
422
|
+
// The writer scopes beast2 aliasing per root ELEMENT, so a container two
|
|
423
|
+
// entries share is written out in each: the bytes depend on the entries'
|
|
424
|
+
// content alone. A merge of such an input writes its bytes back exactly,
|
|
425
|
+
// because the merge writes through the canonical writer. A blob an older
|
|
426
|
+
// writer left — aliasing scoped per segment, the second entry a REF —
|
|
427
|
+
// still reads, and merges to the same canonical bytes.
|
|
428
|
+
const TagsT = DictType(IntegerType, StructType({ tags: ArrayType(StringType) }));
|
|
590
429
|
const shared = ['x', 'y', 'zzzzzzzzzzzzzzzzzzzz'];
|
|
430
|
+
const value = new SortedMap([[1n, { tags: shared }], [2n, { tags: shared }]], compareFor(IntegerType));
|
|
431
|
+
const canonical = encodeBeast2PagedFor(TagsT)(value);
|
|
591
432
|
const inputPath = join(tempDir, 'aliased.beast2');
|
|
592
|
-
writeFileSync(inputPath,
|
|
433
|
+
writeFileSync(inputPath, canonical);
|
|
593
434
|
const outputPath = join(tempDir, 'aliased-merged.beast2');
|
|
594
435
|
assert.deepEqual(mergeBlobs([inputPath], outputPath), { inputs: 1, entries: 2, folds: 0 });
|
|
595
|
-
assert.deepEqual(new Uint8Array(readFileSync(outputPath)),
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
//
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
const
|
|
604
|
-
|
|
605
|
-
const
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
[['-i', input], 'Error: merge requires -o FILE'],
|
|
609
|
-
[['-i', input, '-o', out, '--merge', writeConcat(), '--union'],
|
|
610
|
-
'Error: --merge and --union are two folds — give one'],
|
|
611
|
-
[['-i', input, '-o', join(tempDir, 'args.bin')],
|
|
612
|
-
'Error: merge requires a .beast2 output file (-o)'],
|
|
613
|
-
];
|
|
614
|
-
for (const [args, message] of cases) {
|
|
615
|
-
const result = spawnSync(process.execPath, [bin, 'merge', ...args], { encoding: 'utf8' });
|
|
616
|
-
assert.equal(result.status, 1, result.stderr);
|
|
617
|
-
assert.deepEqual(result.stderr.trimEnd().split('\n'), [message]);
|
|
618
|
-
}
|
|
436
|
+
assert.deepEqual(new Uint8Array(readFileSync(outputPath)), canonical);
|
|
437
|
+
// The older writer's bytes: one segment aliased across its entries — what
|
|
438
|
+
// a writer without self-containment writes — flagged self-contained, as
|
|
439
|
+
// that writer flagged every blob.
|
|
440
|
+
const legacy = encodeBeast2SegmentsFor(TagsT, { selfContained: false })([value]);
|
|
441
|
+
const view = new DataView(legacy.buffer, legacy.byteOffset, legacy.byteLength);
|
|
442
|
+
legacy[Number(view.getBigUint64(legacy.length - 16, true))] = 0x01;
|
|
443
|
+
assert.notDeepEqual(legacy, canonical, 'the older bytes hold a REF the canonical ones do not');
|
|
444
|
+
const legacyPath = join(tempDir, 'aliased-legacy.beast2');
|
|
445
|
+
writeFileSync(legacyPath, legacy);
|
|
446
|
+
const legacyOut = join(tempDir, 'aliased-legacy-merged.beast2');
|
|
447
|
+
assert.deepEqual(mergeBlobs([legacyPath], legacyOut), { inputs: 1, entries: 2, folds: 0 });
|
|
448
|
+
assert.deepEqual(new Uint8Array(readFileSync(legacyOut)), canonical);
|
|
619
449
|
});
|
|
620
450
|
it('a repeated flag takes one value, so it never swallows the ir file', () => {
|
|
621
|
-
//
|
|
622
|
-
//
|
|
623
|
-
//
|
|
624
|
-
//
|
|
625
|
-
// repeated form.
|
|
451
|
+
// A variadic flag ate the positional that followed it, so a run that
|
|
452
|
+
// named its IR file after one reported the file missing — while east-c
|
|
453
|
+
// and east-py, whose flags take one value and repeat, accepted the same
|
|
454
|
+
// line. Every runner README documents the repeated form.
|
|
626
455
|
const bin = fileURLToPath(new URL('../bin/east-node.mjs', import.meta.url));
|
|
627
|
-
const
|
|
456
|
+
const inputPath = join(tempDir, 'n.beast2');
|
|
457
|
+
writeFileSync(inputPath, encodeBeast2For(IntegerType)(20n));
|
|
458
|
+
const irPath = writeIr('double.beast2', East.function([IntegerType], IntegerType, ($, n) => n.multiply(2n)));
|
|
628
459
|
const stdDir = fileURLToPath(new URL('../../east-node-std', import.meta.url));
|
|
629
|
-
const
|
|
630
|
-
|
|
631
|
-
|
|
632
|
-
|
|
633
|
-
assert.ok(!result.stderr.includes('Missing <ir_file>'), result.stderr);
|
|
634
|
-
assert.deepEqual(result.stderr.trimEnd().split('\n'), ['Error: --stream index 0 out of range (0 inputs)']);
|
|
460
|
+
const outputPath = join(tempDir, 'double-out.beast2');
|
|
461
|
+
const result = spawnSync(process.execPath, [bin, 'run', '-i', inputPath, irPath, '-p', '@elaraai/east-node-std', '-o', outputPath], { env: { ...process.env, E3_RUNNER_SEARCH_DIRS: stdDir }, encoding: 'utf8' });
|
|
462
|
+
assert.equal(result.status, 0, result.stderr);
|
|
463
|
+
assert.equal(decodeBeast2For(IntegerType)(new Uint8Array(readFileSync(outputPath))), 40n);
|
|
635
464
|
});
|
|
636
465
|
// ---- The stdin lifeline -----------------------------------------------
|
|
637
466
|
it('a runner given the stdin lifeline exits once its stdin closes, mid-computation', async () => {
|
|
638
467
|
// Gate (c): with --exit-with-parent and a stdin pipe nobody writes, the
|
|
639
|
-
// runner exits once that pipe closes.
|
|
640
|
-
// before the body runs, so
|
|
641
|
-
//
|
|
642
|
-
//
|
|
468
|
+
// runner exits once that pipe closes. A unit's set output directory is
|
|
469
|
+
// made before the body runs, so its existence is the sign the runner is
|
|
470
|
+
// up and computing; the body loops forever after one emission — only the
|
|
471
|
+
// watcher stops it.
|
|
643
472
|
const spin = East.function([emitKeyType], NullType, ($, emit) => {
|
|
644
473
|
$(emit(1n));
|
|
645
474
|
const turns = $.let(0n);
|
|
@@ -647,11 +476,17 @@ describe('folding emit, the blob merge and the stdin lifeline (#770)', () => {
|
|
|
647
476
|
$.assign(turns, turns.add(1n));
|
|
648
477
|
});
|
|
649
478
|
});
|
|
479
|
+
writeIr('spin.beast2', spin);
|
|
480
|
+
const unitPath = join(tempDir, 'unit.beast2');
|
|
481
|
+
writeFileSync(unitPath, encodeBeast2For(UnitType)({
|
|
482
|
+
work: variant('run', { program: 'spin.beast2', inputs: [], output: variant('set', 'spin-output') }),
|
|
483
|
+
platforms: [],
|
|
484
|
+
threads: 1n,
|
|
485
|
+
result: 'result.beast2',
|
|
486
|
+
}));
|
|
650
487
|
const bin = fileURLToPath(new URL('../bin/east-node.mjs', import.meta.url));
|
|
651
|
-
|
|
652
|
-
const
|
|
653
|
-
const outputPath = join(tempDir, 'spin-output.beast2');
|
|
654
|
-
const child = spawn(process.execPath, [bin, 'run', '--exit-with-parent', writeIr('spin.beast2', spin), '-p', '@elaraai/east-node-std', '--emit', 'set', '-o', outputPath], { env: { ...process.env, E3_RUNNER_SEARCH_DIRS: stdDir }, stdio: ['pipe', 'ignore', 'pipe'] });
|
|
488
|
+
const outputPath = join(tempDir, 'spin-output');
|
|
489
|
+
const child = spawn(process.execPath, [bin, 'exec', '--exit-with-parent', unitPath], { stdio: ['pipe', 'ignore', 'pipe'] });
|
|
655
490
|
const exited = new Promise((resolve) => {
|
|
656
491
|
child.on('exit', (code, signal) => resolve({ code, signal }));
|
|
657
492
|
});
|
|
@@ -790,12 +625,27 @@ describe('frozen inputs', () => {
|
|
|
790
625
|
return irPath;
|
|
791
626
|
}
|
|
792
627
|
const NestedT = DictType(IntegerType, StructType({ xs: ArrayType(IntegerType) }));
|
|
628
|
+
const nestedRows = [[1n, { xs: [1n, 2n] }], [2n, { xs: [] }], [3n, { xs: [3n] }]];
|
|
793
629
|
function writeNestedInput() {
|
|
794
|
-
const table = new SortedMap([[1n, { xs: [1n, 2n] }], [2n, { xs: [] }], [3n, { xs: [3n] }]], compareFor(IntegerType));
|
|
795
630
|
const inputPath = join(tempDir, 'nested.beast2');
|
|
796
|
-
writeFileSync(inputPath,
|
|
631
|
+
writeFileSync(inputPath, encodeBeast2SegmentsFor(NestedT)([new Map(nestedRows.slice(0, 2)), new Map(nestedRows.slice(2))]));
|
|
797
632
|
return inputPath;
|
|
798
633
|
}
|
|
634
|
+
/** The same rows as a manifest directory — the form e3 stages a stored
|
|
635
|
+
* collection in. */
|
|
636
|
+
function writeNestedManifest() {
|
|
637
|
+
const path = join(tempDir, 'nested.manifest.beast2');
|
|
638
|
+
const dir = `${path}.segments`;
|
|
639
|
+
mkdirSync(dir);
|
|
640
|
+
const writer = new Beast2ManifestWriter(NestedT, {
|
|
641
|
+
object: (hash, bytes) => writeFileSync(join(dir, `${hash}.beast2`), bytes),
|
|
642
|
+
manifest: (bytes) => writeFileSync(path, bytes),
|
|
643
|
+
});
|
|
644
|
+
for (const row of nestedRows)
|
|
645
|
+
writer.add(row);
|
|
646
|
+
writer.finish();
|
|
647
|
+
return path;
|
|
648
|
+
}
|
|
799
649
|
it('an input refuses mutation with the uniform copy-first error', async () => {
|
|
800
650
|
const fn = East.function([NestedT], NullType, ($, d) => {
|
|
801
651
|
$(d.insert(99n, { xs: [] }));
|
|
@@ -812,19 +662,26 @@ describe('frozen inputs', () => {
|
|
|
812
662
|
await runProgram(writeIr(fn), [], [], [writeNestedInput()], outputPath);
|
|
813
663
|
assert.equal(decodeBeast2For(IntegerType)(new Uint8Array(readFileSync(outputPath))), 4n);
|
|
814
664
|
});
|
|
815
|
-
it('the frozen shape gate admits nested containers lazily: reads serve, writes refuse', async () => {
|
|
665
|
+
it('the frozen shape gate admits nested containers lazily, blob or manifest: reads serve, writes refuse', async () => {
|
|
816
666
|
// Frozen is what makes lazy service safe for nested element shapes, so
|
|
817
|
-
// with a 1-byte threshold this opens pager-backed AND immutable.
|
|
667
|
+
// with a 1-byte threshold this opens pager-backed AND immutable. A gate
|
|
668
|
+
// that refused the shape would fall back to the eager frozen decode, which
|
|
669
|
+
// answers both the same, so the runner's own account, under -v, is what
|
|
670
|
+
// says the input was paged.
|
|
818
671
|
const read = East.function([NestedT], IntegerType, ($, d) => d.get(1n).xs.size());
|
|
819
|
-
const outputPath = join(tempDir, 'out.beast2');
|
|
820
|
-
await runProgram(writeIr(read), [], [], [writeNestedInput()], outputPath, { lazyInputBytes: 1 });
|
|
821
|
-
assert.equal(decodeBeast2For(IntegerType)(new Uint8Array(readFileSync(outputPath))), 2n);
|
|
822
672
|
const write = East.function([NestedT], IntegerType, ($, d) => {
|
|
823
673
|
const row = $.let(d.get(1n));
|
|
824
674
|
$(row.xs.pushLast(42n));
|
|
825
675
|
return d.get(1n).xs.size();
|
|
826
676
|
});
|
|
827
|
-
|
|
677
|
+
for (const input of [writeNestedInput(), writeNestedManifest()]) {
|
|
678
|
+
const outputPath = join(tempDir, 'out.beast2');
|
|
679
|
+
const readErr = await stderrOf(() => withLazyThreshold(1, () => runProgram(writeIr(read), [], [], [input], outputPath, true)));
|
|
680
|
+
assert.ok(readErr.includes('input 0: opened lazily'), `${input} was paged:\n${readErr}`);
|
|
681
|
+
assert.equal(decodeBeast2For(IntegerType)(new Uint8Array(readFileSync(outputPath))), 2n);
|
|
682
|
+
const writeErr = await stderrOf(() => withLazyThreshold(1, () => assert.rejects(runProgram(writeIr(write), [], [], [input], join(tempDir, 'out2.beast2'), true), /cannot mutate a frozen value \(task inputs are immutable\) — copy first/)));
|
|
683
|
+
assert.ok(writeErr.includes('input 0: opened lazily'), `${input} was paged:\n${writeErr}`);
|
|
684
|
+
}
|
|
828
685
|
});
|
|
829
686
|
});
|
|
830
687
|
describe('lazy input threshold resolution', () => {
|
|
@@ -837,21 +694,19 @@ describe('lazy input threshold resolution', () => {
|
|
|
837
694
|
});
|
|
838
695
|
it('defaults to 64 MiB when the environment variable is unset or empty', () => {
|
|
839
696
|
delete process.env.EAST_LAZY_INPUT_BYTES;
|
|
840
|
-
assert.equal(lazyThreshold(
|
|
697
|
+
assert.equal(lazyThreshold(), 64 * 1024 * 1024);
|
|
841
698
|
process.env.EAST_LAZY_INPUT_BYTES = '';
|
|
842
|
-
assert.equal(lazyThreshold(
|
|
699
|
+
assert.equal(lazyThreshold(), 64 * 1024 * 1024, 'empty must fall through to the default, not disable lazy opening');
|
|
843
700
|
});
|
|
844
701
|
it('honours numeric overrides, 0 as the kill switch, and falls back on invalid values', () => {
|
|
845
702
|
process.env.EAST_LAZY_INPUT_BYTES = '0';
|
|
846
|
-
assert.equal(lazyThreshold(
|
|
703
|
+
assert.equal(lazyThreshold(), 0, '0 disables lazy opening');
|
|
847
704
|
process.env.EAST_LAZY_INPUT_BYTES = '123';
|
|
848
|
-
assert.equal(lazyThreshold(
|
|
705
|
+
assert.equal(lazyThreshold(), 123);
|
|
849
706
|
process.env.EAST_LAZY_INPUT_BYTES = '-5';
|
|
850
|
-
assert.equal(lazyThreshold(
|
|
707
|
+
assert.equal(lazyThreshold(), 64 * 1024 * 1024, 'negative values fall back to the default');
|
|
851
708
|
process.env.EAST_LAZY_INPUT_BYTES = 'not-a-number';
|
|
852
|
-
assert.equal(lazyThreshold(
|
|
853
|
-
process.env.EAST_LAZY_INPUT_BYTES = '999';
|
|
854
|
-
assert.equal(lazyThreshold({ lazyInputBytes: 7 }), 7, 'the explicit option wins over the environment');
|
|
709
|
+
assert.equal(lazyThreshold(), 64 * 1024 * 1024, 'garbage falls back to the default');
|
|
855
710
|
});
|
|
856
711
|
});
|
|
857
712
|
//# sourceMappingURL=runner.spec.js.map
|