@elaraai/east-node-cli 1.0.79 → 1.0.81

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -14,10 +14,9 @@ import { tmpdir } from 'node:os';
14
14
  import { join } from 'node:path';
15
15
  import { spawn, spawnSync } from 'node:child_process';
16
16
  import { fileURLToPath } from 'node:url';
17
- import { ArrayType, DictType, FloatType, FunctionType, IntegerType, NullType, OptionType, SetType, StringType, StructType, East, SortedMap, SortedSet, compareFor, decodeBeast2For, encodeBeast2For, encodeBeast2PagedFor, toEastTypeValue, encodeEastIR, none, openBeast2PagesFor, some, spliceBeast2, } from '@elaraai/east';
17
+ import { ArrayType, DictType, FloatType, FunctionType, IntegerType, NullType, OptionType, SetType, StringType, StructType, Beast2ManifestWriter, East, SortedMap, SortedSet, UnitType, compareFor, decodeBeast2For, decodeCollectionManifest, encodeBeast2For, encodeBeast2PagedFor, encodeBeast2SegmentsFor, toEastTypeValue, encodeEastIR, none, openBeast2PagesFor, some, spliceBeast2, variant, } from '@elaraai/east';
18
18
  import { runProgram, lazyThreshold } from './runner.js';
19
19
  import { mergeBlobs } from './merge.js';
20
- import { EmitFileWriter } from './emit-writer.js';
21
20
  /** Runs `run` with console.error captured; returns what it printed. */
22
21
  async function stderrOf(run) {
23
22
  const lines = [];
@@ -31,6 +30,20 @@ async function stderrOf(run) {
31
30
  }
32
31
  return lines.join('\n');
33
32
  }
33
+ /** Runs `run` with the lazy-open threshold set to `bytes`. */
34
+ async function withLazyThreshold(bytes, run) {
35
+ const saved = process.env.EAST_LAZY_INPUT_BYTES;
36
+ process.env.EAST_LAZY_INPUT_BYTES = String(bytes);
37
+ try {
38
+ return await run();
39
+ }
40
+ finally {
41
+ if (saved === undefined)
42
+ delete process.env.EAST_LAZY_INPUT_BYTES;
43
+ else
44
+ process.env.EAST_LAZY_INPUT_BYTES = saved;
45
+ }
46
+ }
34
47
  /** `promise`'s value, or `undefined` once `ms` pass first — a bounded wait. */
35
48
  async function within(promise, ms) {
36
49
  let timer;
@@ -59,14 +72,15 @@ describe('runner output encoding', () => {
59
72
  return new Uint8Array(readFileSync(outputPath));
60
73
  }
61
74
  it('writes large collection outputs segmented and indexed', async () => {
62
- const output = await runToOutput(2500n);
75
+ const output = await runToOutput(20000n);
63
76
  const AT = ArrayType(IntegerType);
64
77
  const pages = openBeast2PagesFor(AT)(output);
65
- assert.equal(pages.segmentCount, 3);
66
- assert.equal(pages.elementCount, 2500);
78
+ assert.ok(pages.segmentCount > 1);
79
+ assert.equal(pages.elementCount, 20_000);
67
80
  assert.ok(pages.selfContained);
68
- const expected = Array.from({ length: 2500 }, (_, i) => BigInt(i));
69
- assert.deepEqual(pages.slice(900, 200), expected.slice(900, 1100), 'window spans segments');
81
+ const expected = Array.from({ length: 20_000 }, (_, i) => BigInt(i));
82
+ const edge = pages.counts[0];
83
+ assert.deepEqual(pages.slice(edge - 100, 200), expected.slice(edge - 100, edge + 100), 'window spans segments');
70
84
  assert.deepEqual(decodeBeast2For(AT)(output), expected, 'whole decode equals the result');
71
85
  });
72
86
  it('writes small collection outputs segmented and indexed too', async () => {
@@ -80,10 +94,10 @@ describe('runner output encoding', () => {
80
94
  assert.deepEqual(decodeBeast2For(AT)(output), expected, 'whole decode equals the result');
81
95
  });
82
96
  });
83
- describe('runner streaming execution', () => {
97
+ describe('lazy inputs', () => {
84
98
  let tempDir;
85
99
  beforeEach(() => {
86
- tempDir = mkdtempSync(join(tmpdir(), 'east-node-stream-'));
100
+ tempDir = mkdtempSync(join(tmpdir(), 'east-node-lazy-'));
87
101
  });
88
102
  afterEach(() => {
89
103
  rmSync(tempDir, { recursive: true, force: true });
@@ -93,86 +107,16 @@ describe('runner streaming execution', () => {
93
107
  writeFileSync(irPath, encodeEastIR(fn.toIR()));
94
108
  return irPath;
95
109
  }
96
- it('a producer body emits an indexed Array output incrementally', async () => {
97
- const fn = East.function([FunctionType([IntegerType], NullType)], NullType, ($, emit) => {
98
- $.for(East.Array.range(0n, 2500n), ($, i) => {
99
- $(emit(i.multiply(2n)));
100
- });
101
- });
102
- const outputPath = join(tempDir, 'output.beast2');
103
- await runProgram(writeIr(fn), [], [], [], outputPath, { emit: 'array' });
104
- const output = new Uint8Array(readFileSync(outputPath));
105
- const pages = openBeast2PagesFor(ArrayType(IntegerType))(output);
106
- assert.equal(pages.elementCount, 2500);
107
- assert.ok(pages.selfContained);
108
- assert.equal(pages.element(1234), 2468n);
109
- const expected = Array.from({ length: 2500 }, (_, i) => BigInt(i * 2));
110
- assert.deepEqual(decodeBeast2For(ArrayType(IntegerType))(output), expected);
111
- });
112
- it('a stream input folds through emit with the input opened lazily', async () => {
113
- const AT = ArrayType(IntegerType);
114
- const events = Array.from({ length: 2500 }, (_, i) => BigInt(i));
115
- const inputPath = join(tempDir, 'events.beast2');
116
- writeFileSync(inputPath, encodeBeast2PagedFor(AT, { batchSize: 500 })(events));
117
- const fn = East.function([AT, FunctionType([IntegerType], NullType)], NullType, ($, events, emit) => {
118
- const acc = $.let(0n);
119
- $.for(events, ($, v) => {
120
- $.assign(acc, acc.add(v));
121
- $(emit(acc));
122
- });
123
- });
124
- const outputPath = join(tempDir, 'output.beast2');
125
- await runProgram(writeIr(fn), [], [], [inputPath], outputPath, { emit: 'array', streamInputs: [0] });
126
- const decoded = decodeBeast2For(AT)(new Uint8Array(readFileSync(outputPath)));
127
- assert.equal(decoded.length, 2500);
128
- assert.equal(decoded[0], 0n);
129
- assert.equal(decoded[2499], (2499n * 2500n) / 2n);
130
- });
131
- /** A dict producer emitting `row-${i}` for each key in the given order. */
132
- function dictEmitter(order) {
133
- const emitType = FunctionType([IntegerType, StringType], NullType);
134
- return East.function([emitType], NullType, ($, emit) => {
135
- $.for($.const(order, ArrayType(IntegerType)), ($, i) => {
136
- $(emit(i, East.str `row-${i}`));
137
- });
138
- });
139
- }
140
- it('an ascending dict emit writes the canonical blob, segmented and indexed', async () => {
141
- const outputPath = join(tempDir, 'ordered.beast2');
142
- const ascending = Array.from({ length: 1000 }, (_, i) => BigInt(i));
143
- await runProgram(writeIr(dictEmitter(ascending)), [], [], [], outputPath, { emit: 'dict' });
144
- const output = new Uint8Array(readFileSync(outputPath));
145
- const DT = DictType(IntegerType, StringType);
146
- const decoded = decodeBeast2For(DT)(output);
147
- assert.equal(decoded.size, 1000);
148
- assert.equal(decoded.get(42n), 'row-42');
149
- assert.ok(openBeast2PagesFor(DT)(output).selfContained);
150
- });
151
- it('a dict emit rejects an out-of-order key with the ascending contract\'s error (#770)', async () => {
152
- // Set/Dict emissions must ascend in East order: the sink writes one pass
153
- // and never buffers, so a key below the previous one is the error — in
154
- // the same words on every runner — and nothing is reported on stderr on
155
- // the way there.
156
- const err = await stderrOf(() => assert.rejects(runProgram(writeIr(dictEmitter([2n, 1n])), [], [], [], join(tempDir, 'disorder.beast2'), { emit: 'dict' }), { message: 'beast2 v5: Dict key emitted out of order: 1 after 2 — Set/Dict emissions must ascend in East order' }));
157
- assert.equal(err, '');
158
- });
159
- it('a duplicate dict key is a hard error', async () => {
160
- const emitType = FunctionType([IntegerType, StringType], NullType);
161
- const adjacent = East.function([emitType], NullType, ($, emit) => {
162
- $(emit(1n, 'a'));
163
- $(emit(1n, 'b'));
164
- });
165
- await assert.rejects(runProgram(writeIr(adjacent), [], [], [], join(tempDir, 'dup.beast2'), { emit: 'dict' }), { message: 'beast2 v5: duplicate Dict key emitted: 1 — Dict keys must be unique' });
166
- });
167
110
  it('a lazy dict input serves keyed reads without a whole decode', async () => {
168
111
  const DT = DictType(IntegerType, StringType);
169
112
  const table = new SortedMap(Array.from({ length: 2000 }, (_, i) => [BigInt(i), `row-${i}`]), compareFor(IntegerType));
170
113
  const inputPath = join(tempDir, 'table.beast2');
171
- writeFileSync(inputPath, encodeBeast2PagedFor(DT, { batchSize: 250 })(table));
114
+ const entries = [...table];
115
+ writeFileSync(inputPath, encodeBeast2SegmentsFor(DT)(Array.from({ length: 8 }, (_, i) => new Map(entries.slice(i * 250, (i + 1) * 250)))));
172
116
  const fn = East.function([DT], StringType, ($, table) => table.get(1234n));
173
117
  const outputPath = join(tempDir, 'output.beast2');
174
118
  // Threshold of 1 byte forces the lazy open for any input size.
175
- await runProgram(writeIr(fn), [], [], [inputPath], outputPath, { lazyInputBytes: 1 });
119
+ await withLazyThreshold(1, () => runProgram(writeIr(fn), [], [], [inputPath], outputPath));
176
120
  const result = decodeBeast2For(StringType)(new Uint8Array(readFileSync(outputPath)));
177
121
  assert.equal(result, 'row-1234');
178
122
  });
@@ -180,19 +124,12 @@ describe('runner streaming execution', () => {
180
124
  const DT = DictType(IntegerType, StringType);
181
125
  const table = new SortedMap(Array.from({ length: 500 }, (_, i) => [BigInt(i), `row-${i}`]), compareFor(IntegerType));
182
126
  const inputPath = join(tempDir, 'table.beast2');
183
- writeFileSync(inputPath, encodeBeast2PagedFor(DT, { batchSize: 100 })(table));
127
+ const entries = [...table];
128
+ writeFileSync(inputPath, encodeBeast2SegmentsFor(DT)(Array.from({ length: 5 }, (_, i) => new Map(entries.slice(i * 100, (i + 1) * 100)))));
184
129
  const fn = East.function([DT], StringType, ($, table) => table.get(42n));
185
130
  const outputPath = join(tempDir, 'output.beast2');
186
- const lines = [];
187
- const original = console.error;
188
- console.error = (...args) => { lines.push(args.map(String).join(' ')); };
189
- try {
190
- await runProgram(writeIr(fn), [], [], [inputPath], outputPath, { lazyInputBytes: 1, verbose: true });
191
- }
192
- finally {
193
- console.error = original;
194
- }
195
- assert.ok(lines.some((l) => l.includes('input 0: opened lazily')), `verbose output names the lazy input:\n${lines.join('\n')}`);
131
+ const err = await stderrOf(() => withLazyThreshold(1, () => runProgram(writeIr(fn), [], [], [inputPath], outputPath, true)));
132
+ assert.ok(err.includes('input 0: opened lazily'), `verbose output names the lazy input:\n${err}`);
196
133
  assert.equal(decodeBeast2For(StringType)(new Uint8Array(readFileSync(outputPath))), 'row-42');
197
134
  });
198
135
  it('a lazily opened wide input reads a fraction of the file and stays below the eager run\'s residency', { skip: process.platform === 'win32' ? 'no RSS comparison on Windows' : false }, () => {
@@ -242,40 +179,11 @@ describe('runner streaming execution', () => {
242
179
  assert.ok(lazy !== null && eager !== null, 'the runner reports its RSS');
243
180
  assert.ok(lazy < eager, `lazy peak ${lazy} MB not below eager peak ${eager} MB for a ${wireMb.toFixed(1)} MB input`);
244
181
  });
245
- it('refuses --emit with a non-.beast2 output, like east-c and east-py', async () => {
246
- const fn = East.function([FunctionType([IntegerType], NullType)], NullType, ($, emit) => {
247
- $(emit(1n));
248
- });
249
- await assert.rejects(runProgram(writeIr(fn), [], [], [], join(tempDir, 'out.json'), { emit: 'array' }), /--emit requires a \.beast2 output file \(-o\)/);
250
- });
251
- it('refuses --emit on a zero-parameter function with the emit-capability error', async () => {
252
- // The one sentence east-c and east-py give for the same shape — a
253
- // zero-parameter function has no trailing parameter to be the capability,
254
- // and so does one whose trailing parameter is not a function.
255
- const message = "--emit requires the function's trailing parameter to be the emit capability (a function type)";
256
- await assert.rejects(runProgram(writeIr(East.function([], IntegerType, (_$) => 1n)), [], [], [], join(tempDir, 'out.beast2'), { emit: 'array' }), { message });
257
- await assert.rejects(runProgram(writeIr(East.function([IntegerType], IntegerType, (_$, n) => n)), [], [], [], join(tempDir, 'out.beast2'), { emit: 'array' }), { message });
258
- });
259
- it('a set emit rejects an out-of-order element and names duplicates with the element noun', async () => {
260
- const emitType = FunctionType([IntegerType], NullType);
261
- const disordered = East.function([emitType], NullType, ($, emit) => {
262
- $(emit(2n));
263
- $(emit(1n));
264
- $(emit(3n));
265
- });
266
- await assert.rejects(runProgram(writeIr(disordered), [], [], [], join(tempDir, 'set.beast2'), { emit: 'set' }), { message: 'beast2 v5: Set element emitted out of order: 1 after 2 — Set/Dict emissions must ascend in East order' });
267
- const dup = East.function([emitType], NullType, ($, emit) => {
268
- $(emit(1n));
269
- $(emit(2n));
270
- $(emit(2n));
271
- });
272
- await assert.rejects(runProgram(writeIr(dup), [], [], [], join(tempDir, 'dupset.beast2'), { emit: 'set' }), { message: 'beast2 v5: duplicate Set element emitted: 2 — Set elements must be unique' });
273
- });
274
182
  });
275
- describe('folding emit, the blob merge and the stdin lifeline (#770)', () => {
183
+ describe('the blob merge and the stdin lifeline (#770)', () => {
276
184
  let tempDir;
277
185
  beforeEach(() => {
278
- tempDir = mkdtempSync(join(tmpdir(), 'east-node-fold-'));
186
+ tempDir = mkdtempSync(join(tmpdir(), 'east-node-merge-'));
279
187
  });
280
188
  afterEach(() => {
281
189
  rmSync(tempDir, { recursive: true, force: true });
@@ -285,119 +193,64 @@ describe('folding emit, the blob merge and the stdin lifeline (#770)', () => {
285
193
  writeFileSync(irPath, encodeEastIR(fn.toIR()));
286
194
  return irPath;
287
195
  }
288
- const PairType = StructType({ key: IntegerType, value: StringType });
289
- const emitPairType = FunctionType([IntegerType, StringType], NullType);
290
196
  const emitKeyType = FunctionType([IntegerType], NullType);
291
- const ascendingBigints = (a, b) => (a < b ? -1 : a > b ? 1 : 0);
292
197
  const DT = DictType(IntegerType, StringType);
293
198
  const ST = SetType(IntegerType);
294
- /** The fold contract's emission sequence, as keys — the sequence
295
- * generate_fixtures.mjs writes for east-c and east-py: 0..1199 in order
296
- * with adjacent duplicates, every third key twice, and key 999 — the last
297
- * entry of a full 1000-element batch — four times. */
298
- function foldSequence() {
299
- const keys = [];
300
- for (let k = 0; k < 1200; k++) {
301
- const copies = 1 + (k % 3 === 0 ? 1 : 0) + (k % 1000 === 999 ? 2 : 0);
302
- for (let c = 0; c < copies; c++)
303
- keys.push(BigInt(k));
304
- }
305
- return keys;
306
- }
307
- /** A dict producer emitting the given (key, value) pairs in order. */
308
- function pairEmitter(pairs) {
309
- return East.function([emitPairType], NullType, ($, emit) => {
310
- $.for($.const(pairs, ArrayType(PairType)), ($, pair) => {
311
- $(emit(pair.key, pair.value));
312
- });
313
- });
314
- }
315
- /** A set producer emitting the given keys in order. */
316
- function keyEmitter(keys) {
317
- return East.function([emitKeyType], NullType, ($, emit) => {
318
- $.for($.const(keys, ArrayType(IntegerType)), ($, key) => {
319
- $(emit(key));
320
- });
321
- });
322
- }
323
199
  /** The concatenating `(Integer, String, String) -> String` fold. */
324
200
  function writeConcat() {
325
201
  return writeIr('merge.beast2', East.function([IntegerType, StringType, StringType], StringType, (_$, _key, acc, value) => acc.concat(value)));
326
202
  }
327
- it('--merge writes the folded ascending sequence\'s bytes', async () => {
328
- // The fold is over ADJACENT equal keys, and lands in place — key 999
329
- // into a full batch's last entry included — so the output is
330
- // byte-identical to what the flag-less sink writes for the folded
331
- // sequence.
332
- const pairs = foldSequence().map((key, i) => ({ key, value: `${i};` }));
333
- const folded = new Map();
334
- for (const { key, value } of pairs)
335
- folded.set(key, (folded.get(key) ?? '') + value);
336
- const foldedPairs = [...folded].sort(([a], [b]) => ascendingBigints(a, b)).map(([key, value]) => ({ key, value }));
337
- const expectedPath = join(tempDir, 'expected.beast2');
338
- await runProgram(writeIr('folded.beast2', pairEmitter(foldedPairs)), [], [], [], expectedPath, { emit: 'dict' });
339
- const outputPath = join(tempDir, 'output.beast2');
340
- await runProgram(writeIr('program.beast2', pairEmitter(pairs)), [], [], [], outputPath, { emit: 'dict', merge: writeConcat() });
341
- assert.deepEqual(new Uint8Array(readFileSync(outputPath)), new Uint8Array(readFileSync(expectedPath)), 'the folded output must be byte-identical to the flag-less sink\'s output for the folded sequence');
342
- });
343
- it('--union writes the ascending sequence\'s distinct keys\' bytes', async () => {
344
- const keys = foldSequence();
345
- const distinct = [...new Set(keys)].sort(ascendingBigints);
346
- const expectedPath = join(tempDir, 'expected.beast2');
347
- await runProgram(writeIr('folded.beast2', keyEmitter(distinct)), [], [], [], expectedPath, { emit: 'set' });
348
- const outputPath = join(tempDir, 'output.beast2');
349
- await runProgram(writeIr('program.beast2', keyEmitter(keys)), [], [], [], outputPath, { emit: 'set', union: true });
350
- assert.deepEqual(new Uint8Array(readFileSync(outputPath)), new Uint8Array(readFileSync(expectedPath)), 'the union output must be byte-identical to the flag-less sink\'s output for the distinct keys');
351
- });
352
- it('a fold does not admit an out-of-order key', async () => {
353
- const pairs = [{ key: 2n, value: 'a' }, { key: 1n, value: 'b' }];
354
- await assert.rejects(runProgram(writeIr('program.beast2', pairEmitter(pairs)), [], [], [], join(tempDir, 'output.beast2'), { emit: 'dict', merge: writeConcat() }), { message: 'beast2 v5: Dict key emitted out of order: 1 after 2 — Set/Dict emissions must ascend in East order' });
355
- });
356
203
  // ---- The blob merge ---------------------------------------------------
357
204
  /** Three sorted Dict inputs whose keys overlap — a = 0..19, b = 10..29,
358
- * c = {5, 15, 25, 40} — each value naming its input, written by the paged
359
- * writer in four-entry segments; the fixtures east-c and east-py merge. */
205
+ * c = {5, 15, 25, 40} — each value naming its input, written in four-entry
206
+ * segments. */
360
207
  const mergeKeys = {
361
208
  a: Array.from({ length: 20 }, (_, k) => BigInt(k)),
362
209
  b: Array.from({ length: 20 }, (_, k) => BigInt(k + 10)),
363
210
  c: [5n, 15n, 25n, 40n],
364
211
  };
212
+ /** Ascending entries as a blob of four-entry segments: a geometry chosen
213
+ * here rather than by the cut rule, so a range bound can fall inside a
214
+ * segment, on a fence, and between segments. */
215
+ function inFours(type, entries) {
216
+ const batches = [];
217
+ for (let i = 0; i < entries.length; i += 4) {
218
+ const chunk = entries.slice(i, i + 4);
219
+ batches.push(type.type === 'Dict' ? new Map(chunk) : new Set(chunk));
220
+ }
221
+ return encodeBeast2SegmentsFor(toEastTypeValue(type))(batches);
222
+ }
365
223
  function writeDictInput(name) {
366
224
  const path = join(tempDir, `in_${name}.beast2`);
367
- writeFileSync(path, encodeBeast2PagedFor(DT, { batchSize: 4 })(new SortedMap(mergeKeys[name].map((k) => [k, `${name}${k}`]), compareFor(IntegerType))));
225
+ writeFileSync(path, inFours(DT, mergeKeys[name].map((k) => [k, `${name}${k}`])));
368
226
  return path;
369
227
  }
370
228
  function writeSetInput(name) {
371
229
  const path = join(tempDir, `set_${name}.beast2`);
372
- writeFileSync(path, encodeBeast2PagedFor(ST, { batchSize: 4 })(new SortedSet(mergeKeys[name], compareFor(IntegerType))));
230
+ writeFileSync(path, inFours(ST, mergeKeys[name]));
373
231
  return path;
374
232
  }
375
- it('merge writes the fold of overlapping dict inputs byte-identical to the ascending sink', async () => {
376
- const folded = new Map();
233
+ it('merge writes the fold of overlapping dict inputs as the Writer writes the folded value', () => {
234
+ const folded = new SortedMap([], compareFor(IntegerType));
377
235
  for (const name of ['a', 'b', 'c']) {
378
236
  for (const k of mergeKeys[name])
379
237
  folded.set(k, (folded.get(k) ?? '') + `${name}${k}`);
380
238
  }
381
- const foldedPairs = [...folded].sort(([a], [b]) => ascendingBigints(a, b)).map(([key, value]) => ({ key, value }));
382
- const expectedPath = join(tempDir, 'expected.beast2');
383
- await runProgram(writeIr('folded.beast2', pairEmitter(foldedPairs)), [], [], [], expectedPath, { emit: 'dict' });
384
239
  const outputPath = join(tempDir, 'merged.beast2');
385
240
  const inputs = [writeDictInput('a'), writeDictInput('b'), writeDictInput('c')];
386
241
  const stats = mergeBlobs(inputs, outputPath, { mergePath: writeConcat() });
387
242
  assert.deepEqual(stats, { inputs: 3, entries: 31, folds: 13 });
388
- assert.deepEqual(new Uint8Array(readFileSync(outputPath)), new Uint8Array(readFileSync(expectedPath)), 'the merge must write exactly what the sink writes for the folded sequence emitted ascending');
243
+ assert.deepEqual(new Uint8Array(readFileSync(outputPath)), encodeBeast2PagedFor(DT)(folded), 'the merge must write exactly what the Writer writes for the folded value');
389
244
  const table = decodeBeast2For(DT)(new Uint8Array(readFileSync(outputPath)));
390
245
  assert.equal(table.get(15n), 'a15b15c15');
391
246
  assert.equal(table.get(40n), 'c40');
392
247
  });
393
- it('merge --union writes the distinct elements of set inputs byte-identical to the ascending sink', async () => {
394
- const distinct = [...new Set([...mergeKeys.a, ...mergeKeys.b, ...mergeKeys.c])].sort(ascendingBigints);
395
- const expectedPath = join(tempDir, 'expected.beast2');
396
- await runProgram(writeIr('folded.beast2', keyEmitter(distinct)), [], [], [], expectedPath, { emit: 'set' });
248
+ it('a union merge writes the distinct elements of set inputs as the Writer writes them', () => {
249
+ const distinct = new SortedSet([...mergeKeys.a, ...mergeKeys.b, ...mergeKeys.c], compareFor(IntegerType));
397
250
  const outputPath = join(tempDir, 'union.beast2');
398
251
  const stats = mergeBlobs([writeSetInput('a'), writeSetInput('b'), writeSetInput('c')], outputPath, { union: true });
399
252
  assert.deepEqual(stats, { inputs: 3, entries: 31, folds: 13 });
400
- assert.deepEqual(new Uint8Array(readFileSync(outputPath)), new Uint8Array(readFileSync(expectedPath)));
253
+ assert.deepEqual(new Uint8Array(readFileSync(outputPath)), encodeBeast2PagedFor(ST)(distinct));
401
254
  assert.equal(openBeast2PagesFor(ST)(new Uint8Array(readFileSync(outputPath))).elementCount, 31);
402
255
  });
403
256
  it('merge without a fold refuses a shared key and leaves the output unfinalised', () => {
@@ -416,8 +269,8 @@ describe('folding emit, the blob merge and the stdin lifeline (#770)', () => {
416
269
  // error, prefixed with the input — the same sentence east-c and east-py give.
417
270
  const descending = join(tempDir, 'descending.beast2');
418
271
  writeFileSync(descending, spliceBeast2([
419
- encodeBeast2PagedFor(DT, { batchSize: 2 })(new SortedMap([[1000n, 'x'], [1001n, 'y']], compareFor(IntegerType))),
420
- encodeBeast2PagedFor(DT, { batchSize: 2 })(new SortedMap([[1n, 'a'], [2n, 'b']], compareFor(IntegerType))),
272
+ encodeBeast2PagedFor(DT)(new SortedMap([[1000n, 'x'], [1001n, 'y']], compareFor(IntegerType))),
273
+ encodeBeast2PagedFor(DT)(new SortedMap([[1n, 'a'], [2n, 'b']], compareFor(IntegerType))),
421
274
  ]));
422
275
  assert.throws(() => mergeBlobs([descending], join(tempDir, 'desc.beast2')), { message: `merge: input 0 (${descending}): beast2 v5: Dict keys are not strictly ascending in East order — the wire must hold the canonical value (corrupt or pre-contract blob)` });
423
276
  const missing = join(tempDir, 'missing.beast2');
@@ -441,10 +294,10 @@ describe('folding emit, the blob merge and the stdin lifeline (#770)', () => {
441
294
  });
442
295
  it('merge names a fold of the wrong signature and refuses a fold of the wrong kind', () => {
443
296
  const wrong = writeIr('wrong.beast2', East.function([StringType, StringType, StringType], StringType, (_$, _key, acc, value) => acc.concat(value)));
444
- assert.throws(() => mergeBlobs([writeDictInput('a')], join(tempDir, 'w.beast2'), { mergePath: wrong }), (err) => err.message.startsWith('--merge: expected a function (K, V, V) -> V matching the inputs (K = '));
445
- assert.throws(() => mergeBlobs([writeSetInput('a')], join(tempDir, 'ms.beast2'), { mergePath: writeConcat() }), { message: '--merge applies to Dict inputs only' });
446
- assert.throws(() => mergeBlobs([writeDictInput('a')], join(tempDir, 'ud.beast2'), { union: true }), { message: '--union applies to Set inputs only' });
447
- assert.throws(() => mergeBlobs([writeDictInput('a')], join(tempDir, 'both.beast2'), { mergePath: writeConcat(), union: true }), { message: 'merge: --merge and --union are two folds — give one' });
297
+ assert.throws(() => mergeBlobs([writeDictInput('a')], join(tempDir, 'w.beast2'), { mergePath: wrong }), (err) => err.message.startsWith('merge function: expected a function (K, V, V) -> V matching the inputs (K = '));
298
+ assert.throws(() => mergeBlobs([writeSetInput('a')], join(tempDir, 'ms.beast2'), { mergePath: writeConcat() }), { message: 'merge: a merge function applies to Dict inputs only' });
299
+ assert.throws(() => mergeBlobs([writeDictInput('a')], join(tempDir, 'ud.beast2'), { union: true }), { message: 'merge: union applies to Set inputs only' });
300
+ assert.throws(() => mergeBlobs([writeDictInput('a')], join(tempDir, 'both.beast2'), { mergePath: writeConcat(), union: true }), { message: 'merge: a merge function and union are two folds — give one' });
448
301
  });
449
302
  it('an empty input contributes nothing, and a lone empty input yields the empty collection, indexed', () => {
450
303
  const empty = join(tempDir, 'empty.beast2');
@@ -460,7 +313,7 @@ describe('folding emit, the blob merge and the stdin lifeline (#770)', () => {
460
313
  assert.equal(openBeast2PagesFor(DT)(new Uint8Array(readFileSync(emptyOut))).elementCount, 0);
461
314
  });
462
315
  // ---- Key ranges ---------------------------------------------------------
463
- /** A `--range` blob over Integer keys: `[from, to)`, a `null` bound open. */
316
+ /** A range blob over Integer keys: `[from, to)`, a `null` bound open. */
464
317
  const RangeT = StructType({ from: OptionType(IntegerType), to: OptionType(IntegerType) });
465
318
  function writeRange(name, from, to) {
466
319
  const path = join(tempDir, `range_${name}.beast2`);
@@ -472,15 +325,15 @@ describe('folding emit, the blob merge and the stdin lifeline (#770)', () => {
472
325
  /** An input holding only its keys in the range — the sliced twin a ranged merge must equal. */
473
326
  function writeDictSlice(name, from, to) {
474
327
  const path = join(tempDir, `in_${name}_${from}_${to}.beast2`);
475
- writeFileSync(path, encodeBeast2PagedFor(DT, { batchSize: 4 })(new SortedMap(mergeKeys[name].filter((k) => inRange(k, from, to)).map((k) => [k, `${name}${k}`]), compareFor(IntegerType))));
328
+ writeFileSync(path, inFours(DT, mergeKeys[name].filter((k) => inRange(k, from, to)).map((k) => [k, `${name}${k}`])));
476
329
  return path;
477
330
  }
478
331
  function writeSetSlice(name, from, to) {
479
332
  const path = join(tempDir, `set_${name}_${from}_${to}.beast2`);
480
- writeFileSync(path, encodeBeast2PagedFor(ST, { batchSize: 4 })(new SortedSet(mergeKeys[name].filter((k) => inRange(k, from, to)), compareFor(IntegerType))));
333
+ writeFileSync(path, inFours(ST, mergeKeys[name].filter((k) => inRange(k, from, to))));
481
334
  return path;
482
335
  }
483
- it('merge --range writes the keys in [from, to) byte-identical to merging inputs that hold only those keys', () => {
336
+ it('a ranged merge writes the keys in [from, to) byte-identical to merging inputs that hold only those keys', () => {
484
337
  // The inputs are written in four-entry segments, so the bounds fall
485
338
  // inside segments, on a fence, before the first key, past the last,
486
339
  // and one range holds nothing.
@@ -512,134 +365,110 @@ describe('folding emit, the blob merge and the stdin lifeline (#770)', () => {
512
365
  assert.equal(openBeast2PagesFor(DT)(new Uint8Array(readFileSync(join(tempDir, 'ranged_7_22.beast2')))).elementCount, 15);
513
366
  assert.equal(openBeast2PagesFor(DT)(new Uint8Array(readFileSync(join(tempDir, 'ranged_30_39.beast2')))).elementCount, 0, 'a range holding nothing writes the empty collection, indexed');
514
367
  });
515
- it('merge --range refuses bounds of another key type, a missing file, an empty file and a file that is not a blob', () => {
368
+ it('a ranged merge refuses bounds of another key type, a missing file, an empty file and a file that is not a blob', () => {
516
369
  const inputs = [writeDictInput('a'), writeDictInput('b')];
517
370
  const mismatch = join(tempDir, 'range_mismatch.beast2');
518
371
  writeFileSync(mismatch, encodeBeast2For(StructType({ from: OptionType(StringType), to: OptionType(StringType) }))({ from: none, to: none }));
519
- assert.throws(() => mergeBlobs(inputs, join(tempDir, 'rm.beast2'), { mergePath: writeConcat(), rangePath: mismatch }), (err) => err.message.startsWith(`merge: --range (${mismatch}) has type `) && err.message.endsWith(" (bounds over the inputs' key type)"));
372
+ assert.throws(() => mergeBlobs(inputs, join(tempDir, 'rm.beast2'), { mergePath: writeConcat(), rangePath: mismatch }), (err) => err.message.startsWith(`merge: range (${mismatch}) has type `) && err.message.endsWith(" (bounds over the inputs' key type)"));
520
373
  const missing = join(tempDir, 'range_missing.beast2');
521
- assert.throws(() => mergeBlobs(inputs, join(tempDir, 'rm2.beast2'), { mergePath: writeConcat(), rangePath: missing }), { message: `merge: --range (${missing}): cannot open the file` });
374
+ assert.throws(() => mergeBlobs(inputs, join(tempDir, 'rm2.beast2'), { mergePath: writeConcat(), rangePath: missing }), { message: `merge: range (${missing}): cannot open the file` });
522
375
  // Not a blob: the reader's own words, as east-c and east-py say them.
523
376
  const text = join(tempDir, 'range_text.beast2');
524
377
  writeFileSync(text, 'not a blob');
525
- assert.throws(() => mergeBlobs(inputs, join(tempDir, 'rm3.beast2'), { mergePath: writeConcat(), rangePath: text }), { message: `merge: --range (${text}): Invalid Beast2 magic at offset 0: expected 0x89, got 0x6e` });
378
+ assert.throws(() => mergeBlobs(inputs, join(tempDir, 'rm3.beast2'), { mergePath: writeConcat(), rangePath: text }), { message: `merge: range (${text}): Invalid Beast2 magic at offset 0: expected 0x89, got 0x6e` });
526
379
  const empty = join(tempDir, 'range_empty.beast2');
527
380
  writeFileSync(empty, '');
528
- assert.throws(() => mergeBlobs(inputs, join(tempDir, 'rm4.beast2'), { mergePath: writeConcat(), rangePath: empty }), { message: `merge: --range (${empty}): Data too short for Beast2 format: 0 bytes` });
529
- });
530
- it('the merge command prints its account, takes a range and refuses two folds', () => {
531
- const bin = fileURLToPath(new URL('../bin/east-node.mjs', import.meta.url));
532
- const outputPath = join(tempDir, 'cli-merged.beast2');
533
- const inputs = [writeDictInput('a'), writeDictInput('b'), writeDictInput('c')];
534
- const result = spawnSync(process.execPath, [bin, 'merge', '--merge', writeConcat(), ...inputs.flatMap((p) => ['-i', p]), '-o', outputPath, '-v'], { encoding: 'utf8' });
535
- assert.equal(result.status, 0, result.stderr);
536
- assert.ok(result.stderr.includes('merge: 3 input(s), 31 entries, 13 fold(s)'), result.stderr);
537
- assert.equal(openBeast2PagesFor(DT)(new Uint8Array(readFileSync(outputPath))).elementCount, 31);
538
- const rangedPath = join(tempDir, 'cli-ranged.beast2');
539
- const ranged = spawnSync(process.execPath, [bin, 'merge', '--merge', writeConcat(), '--range', writeRange('cli', 7n, 22n), ...inputs.flatMap((p) => ['-i', p]), '-o', rangedPath, '-v'], { encoding: 'utf8' });
540
- assert.equal(ranged.status, 0, ranged.stderr);
541
- assert.ok(ranged.stderr.includes('merge: 3 input(s), 15 entries, 11 fold(s)'), ranged.stderr);
542
- const both = spawnSync(process.execPath, [bin, 'merge', '--merge', writeConcat(), '--union', '-i', inputs[0], '-o', join(tempDir, 'both.beast2')], { encoding: 'utf8' });
543
- assert.equal(both.status, 1);
544
- assert.ok(both.stderr.includes('Error: --merge and --union are two folds — give one'), both.stderr);
545
- });
546
- it('the emit writer segments a value exactly as the paged encoder does', () => {
547
- // One value segments the same wherever it is written (#770): a task that
548
- // RETURNS a collection writes it through encodeBeast2PagedFor, one that
549
- // emits or merges writes it through this writer, and a content-addressed
550
- // store must see one blob. The writer opened at the element cap where the
551
- // encoder probes its first entries, so rows above a segment's share got
552
- // one oversized first segment and every later boundary shifted.
553
- //
554
- // Incompressible rows, from a deterministic LCG: deflate shrinks anything
555
- // patterned back under the byte target, and then every batch is the cap
556
- // and the two agree no matter what.
557
- let seed = 12345;
558
- const noise = (chars) => {
559
- const alphabet = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/';
560
- let out = '';
561
- for (let i = 0; i < chars; i++) {
562
- seed = (seed * 48271) % 2147483647;
563
- out += alphabet[(seed >> 8) & 63];
564
- }
565
- return out;
566
- };
567
- for (const width of [4_000, 200_000]) {
568
- seed = 12345;
569
- const rows = Array.from({ length: 60 }, (_, i) => [BigInt(i), noise(width)]);
570
- const value = new SortedMap(rows, compareFor(IntegerType));
571
- const paged = encodeBeast2PagedFor(DT)(value);
572
- const path = join(tempDir, `segments-${width}.beast2`);
573
- const writer = new EmitFileWriter('dict', toEastTypeValue(DT), path);
574
- for (const [key, text] of rows)
575
- writer.push([key, text]);
576
- writer.finishClose();
577
- assert.deepEqual(new Uint8Array(readFileSync(path)), paged, `a ${width}-char row segments the same emitted as returned`);
381
+ assert.throws(() => mergeBlobs(inputs, join(tempDir, 'rm4.beast2'), { mergePath: writeConcat(), rangePath: empty }), { message: `merge: range (${empty}): Data too short for Beast2 format: 0 bytes` });
382
+ });
383
+ it('merge reads manifest directories, the form e3 stages a stored collection in, as east-c and east-py do', () => {
384
+ // Each input twice over: a blob, and a manifest directory — the manifest's
385
+ // file and, beside it in `<file>.segments/`, each object named by its
386
+ // SHA-256. Enough keys for several segments, so a range seeks through the
387
+ // manifest's fences.
388
+ const inputs = [0, 2000].map((start) => Array.from({ length: 3000 }, (_, k) => [BigInt(start + k), `v${start + k}`]));
389
+ const blobs = inputs.map((entries, i) => {
390
+ const path = join(tempDir, `wide_${i}.beast2`);
391
+ writeFileSync(path, encodeBeast2PagedFor(DT)(new SortedMap(entries, compareFor(IntegerType))));
392
+ return path;
393
+ });
394
+ const manifests = inputs.map((entries, i) => {
395
+ const path = join(tempDir, `wide_${i}.manifest.beast2`);
396
+ const dir = `${path}.segments`;
397
+ mkdirSync(dir);
398
+ const writer = new Beast2ManifestWriter(DT, {
399
+ object: (hash, bytes) => writeFileSync(join(dir, `${hash}.beast2`), bytes),
400
+ manifest: (bytes) => writeFileSync(path, bytes),
401
+ });
402
+ for (const entry of entries)
403
+ writer.add(entry);
404
+ writer.finish();
405
+ return path;
406
+ });
407
+ assert.ok(decodeCollectionManifest(new Uint8Array(readFileSync(manifests[0]))).entries.length > 1, 'several segments');
408
+ for (const rangePath of [undefined, writeRange('manifest', 1500n, 2600n)]) {
409
+ const options = { mergePath: writeConcat(), ...(rangePath !== undefined && { rangePath }) };
410
+ const fromBlobs = join(tempDir, 'from-blobs.beast2');
411
+ const fromManifests = join(tempDir, 'from-manifests.beast2');
412
+ assert.deepEqual(mergeBlobs([manifests[0], blobs[1]], fromManifests, options), mergeBlobs(blobs, fromBlobs, options));
413
+ assert.deepEqual(new Uint8Array(readFileSync(fromManifests)), new Uint8Array(readFileSync(fromBlobs)));
578
414
  }
579
- });
580
- it('a container two entries share stays one NEW and one REF', () => {
581
- // The writer scopes beast2 aliasing per SEGMENT, so a container two
582
- // entries of one segment share is written once and referenced. A merge
583
- // of such an input alone must write its bytes back exactly, because the
584
- // merge writes through the same writer `run --emit` does. east-c encoded
585
- // each entry under its own scope and wrote the shared value twice — a
586
- // bigger blob and a different hash for one value, and the two runners
587
- // disagreeing; the same fixture is `merge_aliased.beast2` in east-c's
588
- // and east-py's suites.
589
- const VT = StructType({ tags: ArrayType(StringType) });
415
+ // A segment gone from the directory is named when the merge reaches it,
416
+ // in east-c's words.
417
+ const gone = `${manifests[1]}.segments/${decodeCollectionManifest(new Uint8Array(readFileSync(manifests[1]))).entries[0].hash}.beast2`;
418
+ rmSync(gone);
419
+ assert.throws(() => mergeBlobs([blobs[0], manifests[1]], join(tempDir, 'broken.beast2'), { mergePath: writeConcat() }), { message: `merge: input 1 (${manifests[1]}): beast2 v5: manifest segment ${gone} cannot be read` });
420
+ });
421
+ it('a container two entries share is written for each, and an older blob merges to that form', () => {
422
+ // The writer scopes beast2 aliasing per root ELEMENT, so a container two
423
+ // entries share is written out in each: the bytes depend on the entries'
424
+ // content alone. A merge of such an input writes its bytes back exactly,
425
+ // because the merge writes through the canonical writer. A blob an older
426
+ // writer left — aliasing scoped per segment, the second entry a REF —
427
+ // still reads, and merges to the same canonical bytes.
428
+ const TagsT = DictType(IntegerType, StructType({ tags: ArrayType(StringType) }));
590
429
  const shared = ['x', 'y', 'zzzzzzzzzzzzzzzzzzzz'];
430
+ const value = new SortedMap([[1n, { tags: shared }], [2n, { tags: shared }]], compareFor(IntegerType));
431
+ const canonical = encodeBeast2PagedFor(TagsT)(value);
591
432
  const inputPath = join(tempDir, 'aliased.beast2');
592
- writeFileSync(inputPath, encodeBeast2PagedFor(DictType(IntegerType, VT), { batchSize: 10 })(new SortedMap([[1n, { tags: shared }], [2n, { tags: shared }]], compareFor(IntegerType))));
433
+ writeFileSync(inputPath, canonical);
593
434
  const outputPath = join(tempDir, 'aliased-merged.beast2');
594
435
  assert.deepEqual(mergeBlobs([inputPath], outputPath), { inputs: 1, entries: 2, folds: 0 });
595
- assert.deepEqual(new Uint8Array(readFileSync(outputPath)), new Uint8Array(readFileSync(inputPath)));
596
- });
597
- it('the merge command refuses its arguments in the other runners\' words', () => {
598
- // One sentence per condition, identical on east-c, east-node and east-py:
599
- // the three runners are interchangeable, so a task that names one of them
600
- // must fail the same way on any of them. One line, no JS stack — a
601
- // refusal of the command line is the user's error, not the program's.
602
- const bin = fileURLToPath(new URL('../bin/east-node.mjs', import.meta.url));
603
- const out = join(tempDir, 'args.beast2');
604
- const input = writeDictInput('a');
605
- const cases = [
606
- [[], 'Error: merge requires at least one -i input'],
607
- [['-o', out], 'Error: merge requires at least one -i input'],
608
- [['-i', input], 'Error: merge requires -o FILE'],
609
- [['-i', input, '-o', out, '--merge', writeConcat(), '--union'],
610
- 'Error: --merge and --union are two folds — give one'],
611
- [['-i', input, '-o', join(tempDir, 'args.bin')],
612
- 'Error: merge requires a .beast2 output file (-o)'],
613
- ];
614
- for (const [args, message] of cases) {
615
- const result = spawnSync(process.execPath, [bin, 'merge', ...args], { encoding: 'utf8' });
616
- assert.equal(result.status, 1, result.stderr);
617
- assert.deepEqual(result.stderr.trimEnd().split('\n'), [message]);
618
- }
436
+ assert.deepEqual(new Uint8Array(readFileSync(outputPath)), canonical);
437
+ // The older writer's bytes: one segment aliased across its entries — what
438
+ // a writer without self-containment writes — flagged self-contained, as
439
+ // that writer flagged every blob.
440
+ const legacy = encodeBeast2SegmentsFor(TagsT, { selfContained: false })([value]);
441
+ const view = new DataView(legacy.buffer, legacy.byteOffset, legacy.byteLength);
442
+ legacy[Number(view.getBigUint64(legacy.length - 16, true))] = 0x01;
443
+ assert.notDeepEqual(legacy, canonical, 'the older bytes hold a REF the canonical ones do not');
444
+ const legacyPath = join(tempDir, 'aliased-legacy.beast2');
445
+ writeFileSync(legacyPath, legacy);
446
+ const legacyOut = join(tempDir, 'aliased-legacy-merged.beast2');
447
+ assert.deepEqual(mergeBlobs([legacyPath], legacyOut), { inputs: 1, entries: 2, folds: 0 });
448
+ assert.deepEqual(new Uint8Array(readFileSync(legacyOut)), canonical);
619
449
  });
620
450
  it('a repeated flag takes one value, so it never swallows the ir file', () => {
621
- // `--stream <index...>` (and `-i`/`-p` variadic) ate the positional that
622
- // followed, so `run --stream 0 program.beast2` reported its IR file
623
- // missing — while east-c and east-py, whose flags take one value and
624
- // repeat, accepted the same line. Every runner README documents the
625
- // repeated form.
451
+ // A variadic flag ate the positional that followed it, so a run that
452
+ // named its IR file after one reported the file missing — while east-c
453
+ // and east-py, whose flags take one value and repeat, accepted the same
454
+ // line. Every runner README documents the repeated form.
626
455
  const bin = fileURLToPath(new URL('../bin/east-node.mjs', import.meta.url));
627
- const irPath = writeIr('streamed.beast2', pairEmitter([{ key: 1n, value: 'a' }]));
456
+ const inputPath = join(tempDir, 'n.beast2');
457
+ writeFileSync(inputPath, encodeBeast2For(IntegerType)(20n));
458
+ const irPath = writeIr('double.beast2', East.function([IntegerType], IntegerType, ($, n) => n.multiply(2n)));
628
459
  const stdDir = fileURLToPath(new URL('../../east-node-std', import.meta.url));
629
- const result = spawnSync(process.execPath, [bin, 'run', '--stream', '0', irPath, '-p', '@elaraai/east-node-std',
630
- '--emit', 'dict', '-o', join(tempDir, 'streamed-out.beast2')], { env: { ...process.env, E3_RUNNER_SEARCH_DIRS: stdDir }, encoding: 'utf8' });
631
- // The IR file was parsed as the positional — the run gets as far as the
632
- // stream index, and refuses it in the words east-c uses.
633
- assert.ok(!result.stderr.includes('Missing <ir_file>'), result.stderr);
634
- assert.deepEqual(result.stderr.trimEnd().split('\n'), ['Error: --stream index 0 out of range (0 inputs)']);
460
+ const outputPath = join(tempDir, 'double-out.beast2');
461
+ const result = spawnSync(process.execPath, [bin, 'run', '-i', inputPath, irPath, '-p', '@elaraai/east-node-std', '-o', outputPath], { env: { ...process.env, E3_RUNNER_SEARCH_DIRS: stdDir }, encoding: 'utf8' });
462
+ assert.equal(result.status, 0, result.stderr);
463
+ assert.equal(decodeBeast2For(IntegerType)(new Uint8Array(readFileSync(outputPath))), 40n);
635
464
  });
636
465
  // ---- The stdin lifeline -----------------------------------------------
637
466
  it('a runner given the stdin lifeline exits once its stdin closes, mid-computation', async () => {
638
467
  // Gate (c): with --exit-with-parent and a stdin pipe nobody writes, the
639
- // runner exits once that pipe closes. The sink opens the output file
640
- // before the body runs, so the file's existence is the sign the runner
641
- // is up and computing; the body loops forever after one emission — only
642
- // the watcher stops it.
468
+ // runner exits once that pipe closes. A unit's set output directory is
469
+ // made before the body runs, so its existence is the sign the runner is
470
+ // up and computing; the body loops forever after one emission — only the
471
+ // watcher stops it.
643
472
  const spin = East.function([emitKeyType], NullType, ($, emit) => {
644
473
  $(emit(1n));
645
474
  const turns = $.let(0n);
@@ -647,11 +476,17 @@ describe('folding emit, the blob merge and the stdin lifeline (#770)', () => {
647
476
  $.assign(turns, turns.add(1n));
648
477
  });
649
478
  });
479
+ writeIr('spin.beast2', spin);
480
+ const unitPath = join(tempDir, 'unit.beast2');
481
+ writeFileSync(unitPath, encodeBeast2For(UnitType)({
482
+ work: variant('run', { program: 'spin.beast2', inputs: [], output: variant('set', 'spin-output') }),
483
+ platforms: [],
484
+ threads: 1n,
485
+ result: 'result.beast2',
486
+ }));
650
487
  const bin = fileURLToPath(new URL('../bin/east-node.mjs', import.meta.url));
651
- // As in the wide-input test: point the platform loader at east-node-std.
652
- const stdDir = fileURLToPath(new URL('../../east-node-std', import.meta.url));
653
- const outputPath = join(tempDir, 'spin-output.beast2');
654
- const child = spawn(process.execPath, [bin, 'run', '--exit-with-parent', writeIr('spin.beast2', spin), '-p', '@elaraai/east-node-std', '--emit', 'set', '-o', outputPath], { env: { ...process.env, E3_RUNNER_SEARCH_DIRS: stdDir }, stdio: ['pipe', 'ignore', 'pipe'] });
488
+ const outputPath = join(tempDir, 'spin-output');
489
+ const child = spawn(process.execPath, [bin, 'exec', '--exit-with-parent', unitPath], { stdio: ['pipe', 'ignore', 'pipe'] });
655
490
  const exited = new Promise((resolve) => {
656
491
  child.on('exit', (code, signal) => resolve({ code, signal }));
657
492
  });
@@ -790,12 +625,27 @@ describe('frozen inputs', () => {
790
625
  return irPath;
791
626
  }
792
627
  const NestedT = DictType(IntegerType, StructType({ xs: ArrayType(IntegerType) }));
628
+ const nestedRows = [[1n, { xs: [1n, 2n] }], [2n, { xs: [] }], [3n, { xs: [3n] }]];
793
629
  function writeNestedInput() {
794
- const table = new SortedMap([[1n, { xs: [1n, 2n] }], [2n, { xs: [] }], [3n, { xs: [3n] }]], compareFor(IntegerType));
795
630
  const inputPath = join(tempDir, 'nested.beast2');
796
- writeFileSync(inputPath, encodeBeast2PagedFor(NestedT, { batchSize: 2 })(table));
631
+ writeFileSync(inputPath, encodeBeast2SegmentsFor(NestedT)([new Map(nestedRows.slice(0, 2)), new Map(nestedRows.slice(2))]));
797
632
  return inputPath;
798
633
  }
634
+ /** The same rows as a manifest directory — the form e3 stages a stored
635
+ * collection in. */
636
+ function writeNestedManifest() {
637
+ const path = join(tempDir, 'nested.manifest.beast2');
638
+ const dir = `${path}.segments`;
639
+ mkdirSync(dir);
640
+ const writer = new Beast2ManifestWriter(NestedT, {
641
+ object: (hash, bytes) => writeFileSync(join(dir, `${hash}.beast2`), bytes),
642
+ manifest: (bytes) => writeFileSync(path, bytes),
643
+ });
644
+ for (const row of nestedRows)
645
+ writer.add(row);
646
+ writer.finish();
647
+ return path;
648
+ }
799
649
  it('an input refuses mutation with the uniform copy-first error', async () => {
800
650
  const fn = East.function([NestedT], NullType, ($, d) => {
801
651
  $(d.insert(99n, { xs: [] }));
@@ -812,19 +662,26 @@ describe('frozen inputs', () => {
812
662
  await runProgram(writeIr(fn), [], [], [writeNestedInput()], outputPath);
813
663
  assert.equal(decodeBeast2For(IntegerType)(new Uint8Array(readFileSync(outputPath))), 4n);
814
664
  });
815
- it('the frozen shape gate admits nested containers lazily: reads serve, writes refuse', async () => {
665
+ it('the frozen shape gate admits nested containers lazily, blob or manifest: reads serve, writes refuse', async () => {
816
666
  // Frozen is what makes lazy service safe for nested element shapes, so
817
- // with a 1-byte threshold this opens pager-backed AND immutable.
667
+ // with a 1-byte threshold this opens pager-backed AND immutable. A gate
668
+ // that refused the shape would fall back to the eager frozen decode, which
669
+ // answers both the same, so the runner's own account, under -v, is what
670
+ // says the input was paged.
818
671
  const read = East.function([NestedT], IntegerType, ($, d) => d.get(1n).xs.size());
819
- const outputPath = join(tempDir, 'out.beast2');
820
- await runProgram(writeIr(read), [], [], [writeNestedInput()], outputPath, { lazyInputBytes: 1 });
821
- assert.equal(decodeBeast2For(IntegerType)(new Uint8Array(readFileSync(outputPath))), 2n);
822
672
  const write = East.function([NestedT], IntegerType, ($, d) => {
823
673
  const row = $.let(d.get(1n));
824
674
  $(row.xs.pushLast(42n));
825
675
  return d.get(1n).xs.size();
826
676
  });
827
- await assert.rejects(runProgram(writeIr(write), [], [], [writeNestedInput()], join(tempDir, 'out2.beast2'), { lazyInputBytes: 1 }), /cannot mutate a frozen value \(task inputs are immutable\) — copy first/);
677
+ for (const input of [writeNestedInput(), writeNestedManifest()]) {
678
+ const outputPath = join(tempDir, 'out.beast2');
679
+ const readErr = await stderrOf(() => withLazyThreshold(1, () => runProgram(writeIr(read), [], [], [input], outputPath, true)));
680
+ assert.ok(readErr.includes('input 0: opened lazily'), `${input} was paged:\n${readErr}`);
681
+ assert.equal(decodeBeast2For(IntegerType)(new Uint8Array(readFileSync(outputPath))), 2n);
682
+ const writeErr = await stderrOf(() => withLazyThreshold(1, () => assert.rejects(runProgram(writeIr(write), [], [], [input], join(tempDir, 'out2.beast2'), true), /cannot mutate a frozen value \(task inputs are immutable\) — copy first/)));
683
+ assert.ok(writeErr.includes('input 0: opened lazily'), `${input} was paged:\n${writeErr}`);
684
+ }
828
685
  });
829
686
  });
830
687
  describe('lazy input threshold resolution', () => {
@@ -837,21 +694,19 @@ describe('lazy input threshold resolution', () => {
837
694
  });
838
695
  it('defaults to 64 MiB when the environment variable is unset or empty', () => {
839
696
  delete process.env.EAST_LAZY_INPUT_BYTES;
840
- assert.equal(lazyThreshold({}), 64 * 1024 * 1024);
697
+ assert.equal(lazyThreshold(), 64 * 1024 * 1024);
841
698
  process.env.EAST_LAZY_INPUT_BYTES = '';
842
- assert.equal(lazyThreshold({}), 64 * 1024 * 1024, 'empty must fall through to the default, not disable lazy opening');
699
+ assert.equal(lazyThreshold(), 64 * 1024 * 1024, 'empty must fall through to the default, not disable lazy opening');
843
700
  });
844
701
  it('honours numeric overrides, 0 as the kill switch, and falls back on invalid values', () => {
845
702
  process.env.EAST_LAZY_INPUT_BYTES = '0';
846
- assert.equal(lazyThreshold({}), 0, '0 disables lazy opening');
703
+ assert.equal(lazyThreshold(), 0, '0 disables lazy opening');
847
704
  process.env.EAST_LAZY_INPUT_BYTES = '123';
848
- assert.equal(lazyThreshold({}), 123);
705
+ assert.equal(lazyThreshold(), 123);
849
706
  process.env.EAST_LAZY_INPUT_BYTES = '-5';
850
- assert.equal(lazyThreshold({}), 64 * 1024 * 1024, 'negative values fall back to the default');
707
+ assert.equal(lazyThreshold(), 64 * 1024 * 1024, 'negative values fall back to the default');
851
708
  process.env.EAST_LAZY_INPUT_BYTES = 'not-a-number';
852
- assert.equal(lazyThreshold({}), 64 * 1024 * 1024, 'garbage falls back to the default');
853
- process.env.EAST_LAZY_INPUT_BYTES = '999';
854
- assert.equal(lazyThreshold({ lazyInputBytes: 7 }), 7, 'the explicit option wins over the environment');
709
+ assert.equal(lazyThreshold(), 64 * 1024 * 1024, 'garbage falls back to the default');
855
710
  });
856
711
  });
857
712
  //# sourceMappingURL=runner.spec.js.map