@elaraai/east-codex-plugin 1.0.98 → 1.0.100

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (37) hide show
  1. package/.build/daemon/lsp.js +40 -3
  2. package/.build/daemon/server.js +40 -3
  3. package/.build/mcp/server.js +3 -2
  4. package/.codex-plugin/plugin.json +1 -1
  5. package/index.json +13340 -482
  6. package/package.json +1 -1
  7. package/skills/e3/SKILL.md +108 -677
  8. package/skills/e3/reference/browser.md +93 -0
  9. package/skills/e3/reference/cli.md +120 -0
  10. package/skills/e3/reference/from-code.md +48 -0
  11. package/skills/e3/reference/rack.md +139 -0
  12. package/skills/e3/reference/record-indexes.md +326 -0
  13. package/skills/e3/reference/record-migrations.md +63 -0
  14. package/skills/e3/reference/repository.md +25 -0
  15. package/skills/e3-ui/SKILL.md +426 -259
  16. package/skills/east/SKILL.md +33 -395
  17. package/skills/east/reference/beast2.md +71 -0
  18. package/skills/east/reference/cross-language-and-json-schema.md +87 -0
  19. package/skills/east/reference/queries.md +46 -0
  20. package/skills/east/reference/runners-and-ir.md +148 -0
  21. package/skills/east/reference/trees-and-recursion.md +56 -0
  22. package/skills/east/reference/values-in-typescript.md +62 -0
  23. package/skills/east-contribute/SKILL.md +0 -1
  24. package/skills/east-py/SKILL.md +36 -1613
  25. package/skills/east-py/reference/beast2.md +108 -0
  26. package/skills/east-py/reference/cli.md +51 -0
  27. package/skills/east-py/reference/columns-and-platform-functions.md +87 -0
  28. package/skills/east-py/reference/control-flow-and-queries.md +147 -0
  29. package/skills/east-py/reference/cross-language.md +70 -0
  30. package/skills/east-py/reference/ir-and-diagnostics.md +150 -0
  31. package/skills/east-py/reference/patterns.md +369 -0
  32. package/skills/east-py/reference/performance.md +70 -0
  33. package/skills/east-py/reference/value-methods.md +306 -0
  34. package/skills/east-py/reference/values-construction-and-json.md +88 -0
  35. package/skills/east-py/reference/writing-east-functions.md +275 -0
  36. package/skills/east-py-datascience/SKILL.md +11 -28
  37. package/skills/east-ui/SKILL.md +5 -5
@@ -15,9 +15,10 @@ in-process, or on the east-node, east-c and east-py runners.
15
15
 
16
16
  ## Before writing code — search the example index
17
17
 
18
- Every East API has a tested example in the plugin's index — the index IS the
19
- API reference, printed from each example's IR in TypeScript or python. Before
20
- writing or changing East code:
18
+ Every East API has a tested example in the plugin's index — the index is the
19
+ API reference, printed from each example's IR in TypeScript or python, which
20
+ is why this skill lists methods without their signatures. Before writing or
21
+ changing East code:
21
22
 
22
23
  1. Call `search_east_examples` for each capability you
23
24
  are about to use — `language: "python"` for east-py, `"typescript"`
@@ -30,7 +31,10 @@ writing or changing East code:
30
31
  programs, exact and far cheaper, and the signatures omit the runtime rules
31
32
  that make East code correct.
32
33
 
33
- Nothing is injected for you; the search is the step.
34
+ Nothing is injected for you; the search is the step. (Python code working
35
+ with East values directly — eager methods, not an East function — is a
36
+ separate set, searched with `package: "east-values"`; the `east-py` skill
37
+ covers it.)
34
38
 
35
39
  ## Quick Start
36
40
 
@@ -112,7 +116,7 @@ Task → What do you need?
112
116
  │ ├─ Blob → .size(), .getUint8(), .decodeUtf8(), .decodeUtf16(), .decodeBeast(T),
113
117
  │ │ .decodeCsv(rowType, { nullStrings (default [] — an empty field is an empty string; opt in for none),
114
118
  │ │ defaults (per-column fallbacks), skipShortRows (drop ragged rows rather than fail), trimFields, … }),
115
- │ │ .openBeast(ArrayType(T) | SetType(K) | DictType(K, V)) — a huge collection, opened lazily (below)
119
+ │ │ .openBeast(ArrayType(T) | SetType(K) | DictType(K, V)) — a huge collection, opened lazily (reference/beast2.md)
116
120
  │ ├─ Array
117
121
  │ │ ├─ Read → .size(), .length(), .has(), .get(), .at(), .tryGet(), .getKeys()
118
122
  │ │ ├─ Mutate → .update(), .pushLast(), .popLast(), .pushFirst(), .popFirst(), .append(), .prepend(),
@@ -124,7 +128,8 @@ Task → What do you need?
124
128
  │ │ ├─ Convert → .stringJoin(), .toSet(), .toDict(), .flattenToSet(), .flattenToDict(), .encodeCsv()
125
129
  │ │ ├─ Group → .groupReduce(), .groupSize(), .groupSum(), .groupMean(), .groupMinimum(), .groupMaximum(),
126
130
  │ │ │ .groupToArrays(), .groupToSets(), .groupToDicts(), .groupEvery(), .groupSome()
127
- │ │ └─ Tree → .toTree(Node, key, parent, build) — flat parent-keyed rows to nested nodes (below)
131
+ │ │ └─ Tree → .toTree(Node, key, parent, build) — flat parent-keyed rows to nested nodes;
132
+ │ │ .flattenTree(children) — nested nodes back to { path, node } rows, depth first (reference/trees-and-recursion.md)
128
133
  │ ├─ Set
129
134
  │ │ ├─ Read and mutate → .size(), .has(), .insert(), .tryInsert(), .delete(), .tryDelete(), .clear(), .unionInPlace()
130
135
  │ │ ├─ Set ops → .copy(), .union(), .intersection(), .difference(), .symmetricDifference(), .isSubsetOf(),
@@ -133,6 +138,7 @@ Task → What do you need?
133
138
  │ │ reduce .reduce(), .scan(), .every(), .some(), .sum(), .mean();
134
139
  │ │ convert .toArray(), .toSet(), .toDict(), .flattenToArray(), .flattenToSet(), .flattenToDict(),
135
140
  │ │ .toTree(Node, parent, build) (an element / a key is its own key; Dict callbacks take (value, key));
141
+ │ │ Dict only: .flattenTree(children), the nodes back again (reference/trees-and-recursion.md);
136
142
  │ │ group .groupReduce(), .groupSize(), .groupSum(), .groupMean(), .groupToArrays(), .groupToSets(),
137
143
  │ │ .groupToDicts(), .groupEvery(), .groupSome()
138
144
  │ ├─ Dict → .size(), .has(), .get(), .tryGet(), .keys(), .getKeys(), .insert(), .insertOrUpdate(), .update(),
@@ -155,7 +161,7 @@ Task → What do you need?
155
161
  │ ├─ Integer → East.Integer.printCommaSeperated(), .roundNearest(), .printOrdinal()
156
162
  │ ├─ Float → East.Float.roundToDecimals(), .printCurrency(), .printCompact()
157
163
  │ ├─ DateTime → East.DateTime.fromComponents(), .roundDownDay(), .parseFormatted()
158
- │ │ (fromComponents NORMALISES an out-of-range component into the next — (2024, 2, 31) is 2024-03-02 —
164
+ │ │ (fromComponents normalises an out-of-range component into the next — (2024, 2, 31) is 2024-03-02 —
159
165
  │ │ so validate by round-tripping getMonth(), not by range-checking)
160
166
  │ ├─ Array → East.Array.range(), .linspace(), .generate() · Set → East.Set.generate() · Dict → East.Dict.generate()
161
167
  │ ├─ Blob → East.Blob.encodeBeast() · String → East.String.printJson(), East.String.printError()
@@ -165,21 +171,24 @@ Task → What do you need?
165
171
  │
166
172
  ├─ Compare and convert (East.*) → East.equal(), East.notEqual(), East.less(), East.greater(), East.lessEqual(),
167
173
  │ East.greaterEqual() (and their aliases), East.min(), East.max(), East.clamp(), East.print(expr);
168
- │ East.is() is identity for mutable collections — FROZEN ones (task inputs) compare by value
174
+ │ East.is() is identity for mutable collections — frozen ones (task inputs) compare by value
169
175
  ├─ Patches (East.*) → East.diff(before, after), East.applyPatch(value, patch),
170
176
  │ East.composePatch(first, second, type), East.invertPatch(patch, type)
177
+ ├─ Build East over types you are handed (a library, a generator) → East.field(v, name, T?), East.fields(v, names?),
178
+ │ East.struct(T, fields), East.withField(v, name, x), East.inCase(T, name, x), East.patchesFor(T): each name
179
+ │ checked as the program builds → reference/values-in-typescript.md
171
180
  │
172
181
  ├─ Ask a question of a value in jq → East.jq(input, program, resultType), checked when the program
173
- │ builds → "Queries (typed jq)"
174
- ├─ Work with East values from TypeScript → "Values in TypeScript" (comparators, SortedMap, codecs)
175
- ├─ Run a program from the shell → "Runners" (east-node / east-c / east-py run, exec, east-c ir)
182
+ │ builds → reference/queries.md
183
+ ├─ Work with East values from TypeScript → reference/values-in-typescript.md (comparators, SortedMap, codecs)
184
+ ├─ Run a program from the shell → reference/runners-and-ir.md (east-node / east-c / east-py run, exec, east-c ir)
176
185
  ├─ Run a runner-protocol unit in a host of your own (a browser's worker) → executeUnit(unit, io, { platforms })
177
- │ over an InMemoryUnitIO ("Units in a host of your own")
186
+ │ over an InMemoryUnitIO (reference/runners-and-ir.md)
178
187
  ├─ IR ↔ source → fn.toIR(), encodeEastIR, East.toSource(fn), east-node transpile
179
188
  ├─ A function written in python (or another package) → East.importFunction; yours for python → East.exportFunctions
180
- ├─ Check that a module BUILDS → east-node check (tsc cannot see the build's own errors)
189
+ ├─ Check that a module builds → east-node check (tsc cannot see the build's own errors)
181
190
  ├─ A JSON contract for a type → jsonSchemaFor(T) · back from a vendored schema → typeFromJsonSchema(schema)
182
- └─ Data larger than memory → "Binary serialization (beast2)"; inside a body blob.openBeast(T),
191
+ └─ Data larger than memory → reference/beast2.md; inside a body blob.openBeast(T),
183
192
  and a file on the runner's disk FileSystem.openBeast(T, path) (east-node-std / east-c-std / east-py-std)
184
193
  ```
185
194
 
@@ -253,33 +262,6 @@ $.try($ => {
253
262
  }).finally($ => { /* cleanup */ });
254
263
  ```
255
264
 
256
- ### Trees from flat rows, and recursion
257
-
258
- `toTree` builds nested data — any depth, your own node type — from rows that
259
- name their key and their parent's key. `build` runs once per row, children
260
- first; the result is the roots in source order.
261
-
262
- ```typescript
263
- const Row = StructType({ id: StringType, parent: OptionType(StringType), name: StringType });
264
- const Node = RecursiveType(self => StructType({ name: StringType, children: ArrayType(self) }));
265
-
266
- const nest = East.function([ArrayType(Row)], ArrayType(Node), ($, rows) =>
267
- rows.toTree(Node, ($, r) => r.id, ($, r) => r.parent, ($, r, _i, children) => ({ name: r.name, children })));
268
- ```
269
-
270
- - A row whose parent is `none` is a root, and so is an orphan (its parent key
271
- is not in the rows). A repeated key ❗ `toTree: duplicate key <key>`; a cycle
272
- ❗ `toTree: cycle through key <key>` — both before any `build`.
273
- - A non-recursive `Node` folds bottom-up: `IntegerType` with
274
- `children.sum().add(1n)` is each root's subtree size.
275
- - **Recursion** — a function calling itself through a captured variable
276
- (`$.let(fn, FunctionType(...))`, then `$.assign(fn, East.function(...))`) —
277
- is for shallow depths, in the hundreds. Deeper, a call ❗
278
- `call stack exhausted: East calls nested too deeply`, a catchable error on
279
- every runtime. Walk deep data with `$.while` or `toTree`, which is iterative;
280
- the C and Python runtimes still bound a recursive VALUE's depth to tens of
281
- thousands of levels.
282
-
283
265
  ### Platform functions
284
266
 
285
267
  An implementation is a plain TypeScript function over the `ValueTypeOf` of each
@@ -316,362 +298,18 @@ implementation, python's included), and list the package in the task's runner,
316
298
  wiring is **east-project**'s; the task runner is **e3**'s; python's
317
299
  `@platform_function` is **east-py**'s.
318
300
 
319
- ## Values in TypeScript
320
-
321
- Host-side functions for East values — build time, no expression, no IR. Each
322
- takes the type and returns a function (`compareFor(T)(a, b)`).
323
-
324
- | Signature | Description |
325
- |-----------|-------------|
326
- | **Comparing** |
327
- | `compareFor(T)(a, b): -1 \| 0 \| 1` | East's total order — the order sets, dicts and `.sort()` use (NaN last) |
328
- | `equalFor(T)`, `notEqualFor(T)`, `lessFor(T)`, `lessEqualFor(T)`, `greaterFor(T)`, `greaterEqualFor(T)` | The comparisons as East defines them: by value, NaN equal to itself and −0 apart from 0, variants by case name. Never `===`, `<` or a comparator of your own |
329
- | `isFor(T)(a, b)` | East's `Is`: value for immutable types, identity for mutable containers — but two frozen collections by value, and a `Ref` always by identity |
330
- | `new SortedMap(entries?, compareFor(K))` · `new SortedSet(values?, compareFor(K))` | `Map` / `Set` in East order — what decoders return |
331
- | **Types** |
332
- | `isValueOf(value, T)` | Whether a TypeScript value is a value of `T` |
333
- | `printType(T)` · `isTypeEqual(a, b)` · `isSubtype(a, b)` | Print a type in East text; compare types |
334
- | `diffTypes(actual, expected)` → `renderTypeDiff(diffs)` | The differences between two types, each with its path |
335
- | `defaultValue(T)` · `minimalValue(T)` | A value of `T`: its default, and its least |
336
- | **Codecs** — each `…For(T)` builds a codec for one type |
337
- | `printFor(T)(value)` · `parseFor(T)(text)` | East text, the literal syntax `.east` files hold. `parseFor` returns `{ success, value \| error, position }`; `parseInferred(text)` returns `[type, value]` |
338
- | `toJSONFor(T)(value)` · `fromJSONFor(T)(json)` | East JSON as a JavaScript value (`encodeJSONFor` / `decodeJSONFor` as bytes): lossless — an Integer a quoted decimal, a Blob hex, a flat Option (see JSON Schema) |
339
- | `encodeCsvFor(StructT, config?)(rows)` · `decodeCsvFor(StructT, config?)(blob)` | An array of structs ↔ CSV bytes |
340
- | `encodeBeast2For(T)(value)` · `decodeBeast2For(T)(blob)` | beast2, the binary form every runtime and e3 store — see below |
341
-
342
- ## Queries (typed jq)
343
-
344
- `East.jq` asks a question of a value in jq, inside an East body. The program is
345
- parsed, type-checked against the input's East type and translated to ordinary
346
- IR when the function is built, so it runs on every runner; a mistake is a
347
- `QueryError` at build time, with the checker's sentence, span and fixes. The
348
- language is jq 1.8 over East values: `libs/east/devdocs/QUERY.md` in the East
349
- repository says what each builtin does, and every place East differs from jq.
350
-
351
- ```typescript
352
- const Order = StructType({ id: IntegerType, total: FloatType });
353
-
354
- const bigOrders = East.function([ArrayType(Order)], ArrayType(IntegerType), ($, orders) =>
355
- East.jq(orders, "[.[] | select(.total > 1000) | .id]", ArrayType(IntegerType)));
356
-
357
- // An object of inputs is read as an e3 root: `.orders` is that input alone, so a lazy one stays lazy.
358
- const revenue = East.function([ArrayType(Order)], FloatType, ($, orders) =>
359
- East.jq({ orders }, ".orders | map(.total) | add", FloatType));
360
- ```
361
-
362
- | Signature | Description | Example |
363
- |-----------|-------------|---------|
364
- | **In an East body** |
365
- | `East.jq<T>(input: Expr \| { [name: string]: Expr }, program: string, resultType: T): ExprType<T>` **❗** | The query's result, an expression of `resultType`: the outputs' type `T` for a query that gives exactly one output, `Option<T>` for at most one, `Array<T>` for any number. Throws `QueryError` when the query does not check, or checks to another type | `East.jq(orders, "map(.total)", ArrayType(FloatType)).sum()` |
366
- | **Host-side** |
367
- | `checkJq(program: string, input: EastType, options?: { root?: boolean, tooling?: boolean }): CheckJqResult` | The checker: `program` (the program as written, a `JqType` tree, or `null` when it does not check), `inputType`, `diagnostics` (each a code, a span, one sentence, suggestions and fixes), `elementType`, `multiplicity` (`"one"`, `"maybe"` or `"many"`). A query's types are never stored: check the program against its data wherever they are needed | `checkJq(".orders[0].totl", Root).diagnostics[0].message` |
368
- | `completeJq(text: string, offset: number, input: EastType, options?): JqCompletions \| null` | The completions at a cursor, from the type there: fields, cases, variables, builtins | `completeJq(".orders[0].to", 13, Root)` |
369
- | `printJq(program: JqNode, options?: { layout?: "line" \| "pipeline" }): PrintedJq` | A program's canonical text, and its nodes' spans; `pipeline` puts each top-level stage on a line of its own | `printJq(checked.program!).text` |
370
- | `translateJq(checked: CheckJqResult, options?: { maxOutputs?: number, tooling?: boolean }): JqTranslation` **❗** | The translation: `fn()`, an East function of the inputs the query reads (`inputs`), and its `resultType` | `East.compile(translateJq(checked).fn(), [])` |
371
- | `evaluateJq(program: string \| CheckJqResult, input: unknown, options?: { inputType?, root?, tooling?, platform? }): unknown` **❗** | Checks, translates, compiles (cached) and runs a query over a TypeScript value. Throws `QueryError`: the checker's diagnostics, or one `runtime` diagnostic at the node that raised | `evaluateJq("[.[] \| .id]", orders, { inputType: ArrayType(Order) })` |
372
- | `splitJq(checked: CheckJqResult, options?: { maxOutputs?: number }): JqSplit` | A query checked as an e3 root, split over one dataset's pieces for an e3 split call: `piece()` each piece runs, the `output` kind its outputs combine by (`array`, `set`, `dict` with `merge()`, `fold` with `zero` and `combine()`), and `then()` over the result; or, `kind: "whole"`, why it runs as one unit. QUERY.md §17.1 | `splitJq(checkJq(".orders \| group_by(.region) \| map(length)", Root, { root: true })).output.kind` |
373
-
374
- - **The values are East's.** A Dict is an object keyed by its key type, so
375
- `.byId[1035]` looks up an Integer key; a variant reads as `{type, value}`,
376
- and `select(.status.type == "shipped")` narrows it; an Option is `null` or
377
- its value. `.name` on a struct that lacks the field is an error (`.name?`
378
- gives `null`), and so is comparing values of two types.
379
- - **A lookup can miss**, so `.[i]` and `.[k]` give an `Option`, and
380
- `first(f)` gives at most one output, so its result type is `Option<T>`.
381
- - **In code a query is a call of the `Query` builtin**, which carries the
382
- program as written beside its translation: every runtime runs it as any
383
- builtin, and `East.toSource` prints it back as `East.jq(…)`.
384
-
385
- ## Runners: running a program from the command line
386
-
387
- A program's IR runs on any of three runners, which take the same arguments.
388
- Write it with `writeFileSync("prog.beast2", encodeEastIR(fn.toIR()))` — which
389
- keeps its source map, so errors name your lines — or `fn.toIR().toJSON()`.
390
-
391
- ```bash
392
- east-node run prog.beast2 -p @elaraai/east-node-std -i a.beast2 -i b.beast2 -o out.beast2 # -v: timing and peak memory
393
- east-c run prog.beast2 -p east-c-std -i a.beast2 -o out.beast2 [--profile] # native; --profile: time per function
394
- east-py run prog.beast2 -p east-py-std -i a.beast2 -o out.beast2 [--profile] # python (the east-py skill)
395
- ```
396
-
397
- - `--profile` (east-c and east-py, `run` and `exec`) prints every East
398
- function and platform function the work called, by self time, with its
399
- calls and where it is defined and called. `EAST_PROFILE=1` profiles without
400
- the flag — e3 hands its environment to the runners, so a task's report
401
- lands in its log — and `EAST_PROFILE_INTERVAL=N` prints the profile so far
402
- every N seconds while the work runs.
403
- - Each `-i` is one argument, in parameter order, and each `-p` loads one
404
- platform package (east-node needs at least one); `-o` writes the result,
405
- else it is printed. IR and values may be `.beast2`, `.beast`, `.east` or
406
- `.json`.
407
- - `exec <unit>` is the runner protocol e3 speaks: it runs one unit file — a
408
- program over its inputs, a merge of an output's parts, or the intake of a
409
- delivered collection — writes the output by its kind and records a result;
410
- exit 0 when it succeeded, 1 when it failed.
411
- - `version [-p <package>…]` prints the runner's version and each package's.
412
- - east-c also has `convert <file> [-o <file>] [--type <east-type>]`, a value
413
- from one format to another, and the IR toolbox: `ir normalize` (the canonical
414
- form — the round-trip equality contract), `ir diff a b` (the first structural
415
- difference) and `ir convert` (JSON ↔ beast2, the source map kept).
416
- - east-node also has `transpile`, `check`, `lsp` and `export-functions`, below;
417
- east-py has the python twins, and `lint`.
418
-
419
- ### Units in a host of your own: `executeUnit` and `UnitIO`
420
-
421
- east-node's `exec` is a file wrapper around `executeUnit`, the one TypeScript
422
- unit runner, which does a unit's work over a `UnitIO` rather than a file
423
- system — so a host with none runs units too: e3-web runs them in a browser's
424
- Web Workers. east-c has the same shape (`east/unit.h`).
425
-
426
- | Signature | Description |
427
- |-----------|-------------|
428
- | `executeUnit(unit: Unit, io: UnitIO, options: ExecuteUnitOptions): Promise<UnitResult>` | Does the unit's work — a run, a merge or an intake — over `io`, writing its output there by its kind, and returns the result, which the host writes where `unit.result` says if it keeps it. A failure is the result's `outcome`, `failed` with its message and source locations, never a throw |
429
- | `options.platforms(name: string): readonly PlatformFunction[] \| Promise<…>` | Each package the unit lists, asked once, in the unit's order, before the work begins; a throw fails the unit with its message |
430
- | `options.resident?: () => number` · `options.peakBytes?: () => bigint` | The host's memory gauges. Without `resident`, what reading a lazy input whole weighs and what decoding an input whole added both come to 0; without `peakBytes`, the result's `peakBytes` is 0, as a browser's is |
431
- | `options.report?: UnitRunReport` | Hears how a run unit's inputs open and what reading them came to: the account `exec -v` prints |
432
- | `UnitIO` | The unit's files, by the paths it names: `read`, `size`, `readRange`, `segment(path, fetch)`, `list`, `makeDirectory`, `write` — every one synchronous, since compiled East reads a lazy input without awaiting |
433
- | `new InMemoryUnitIO(files?: Iterable<[path: string, bytes: Uint8Array]>)` | A `UnitIO` in memory, for a host with no file system: given every file the unit names, segments included; `.files` then holds them and every file the unit wrote |
434
- | `UnitType` · `UnitResultType` | The unit and its result as East types, for a host that keeps them as beast2 |
435
-
436
- - The unit's thread grant (`threads`) caps the frame pool for the rest of the
437
- process: a grant of one frames every output inline.
438
- - A unit whose host places segments as they are read (`fetch: true`) has
439
- `io.segment` asked for each segment of a staged manifest before it is read.
440
- `InMemoryUnitIO` holds every segment already, so its unit says `fetch: false`,
441
- and a segment it does not hold fails at once, as any absent file does.
442
-
443
- ```typescript
444
- import { East, IntegerType, InMemoryUnitIO, decodeBeast2For, encodeBeast2For, encodeEastIR, executeUnit, variant, type Unit } from "@elaraai/east";
445
-
446
- const double = East.function([IntegerType], IntegerType, ($, x) => x.multiply(2n));
447
-
448
- // A unit names its files by path: here, the program, its one input and its output.
449
- const unit: Unit = {
450
- work: variant("run", {
451
- program: "program.beast2",
452
- inputs: ["x.beast2"],
453
- output: variant("value", "out.beast2"),
454
- decode: variant("lazy", null),
455
- }),
456
- platforms: [], // the platform packages it needs, each resolved by `options.platforms`
457
- threads: 1n,
458
- fetch: false, // every segment it reads is in the IO already
459
- result: "result.beast2",
460
- };
461
- const io = new InMemoryUnitIO([
462
- ["program.beast2", encodeEastIR(double.toIR())],
463
- ["x.beast2", encodeBeast2For(IntegerType)(21n)],
464
- ]);
465
- const result = await executeUnit(unit, io, { platforms: () => [] });
466
- // result.outcome: ok, or failed with its message and source locations; a failure is never thrown
467
- decodeBeast2For(IntegerType)(io.read("out.beast2")); // 42n
468
- ```
469
-
470
- ## IR → source: `East.toSource` and `east-node transpile`
471
-
472
- Any IR — yours, or one exported from python — prints back as a module whose
473
- `East.function(...)` rebuilds the same IR (equal under `east-c ir normalize`).
474
-
475
- ```typescript
476
- const double = East.function([IntegerType], IntegerType, ($, x) => x.multiply(2n));
477
- console.log(East.toSource(double));
478
- // import { East, … IntegerType } from "@elaraai/east";
479
- // export const main = East.function([IntegerType], IntegerType, ($, x) => x.multiply(2n));
480
- ```
481
-
482
- ```bash
483
- east-node transpile double.beast2 -o double.ts # any runtime's IR
484
- east-node transpile ./ir/ -o ./ts/ --rebuild ./rebuilt/ # a directory, and the IR each module builds back
485
- east-py transpile double.beast2 -o double.py # the python twin
486
- ```
301
+ ## Reference files
487
302
 
488
- - Statements print as the `$`-forms, a bound value as `$.let(value, T)`, an
489
- Option as `some(v)` / `none`, other values as `East.value(v, T)`, builtins as
490
- their methods; one with no method spelling prints as
491
- `East.builtin(name, [T...], [args], out)`, which rebuilds. `East.as(v, T)` and
492
- `East.wrapRecursive(v, T)` spell the `As` and `WrapRecursive` nodes.
493
- - A platform call prints as its library spells it when you pass the library:
494
- `East.toSource(fn, { libraries: { "@elaraai/east-node-io": await import("@elaraai/east-node-io") } })`.
495
- Without it the call hoists to a declaration named after the function
496
- (`const tar_create = East.asyncPlatform("tar_create", …)`). The printer finds
497
- the handles in the module's exports — each handle `East.platform` returns
498
- carries its `name`, `inputs`, `output` and `async`, and
499
- `isPlatformDeclaration(x)` tells one from a function.
500
- - Variables keep the names you wrote, read by the TypeScript compiler (an
501
- optional peer); without it, or in a browser, they print as `_N`, as does a
502
- slot the body did not name (`($, x) =>` for a `(value, index)` callback).
503
- - The contract and the round-trip suites: `docs/conventions/EAST_CODEGEN.md`.
504
-
505
- ## Checking that a module builds: `east-node check`
506
-
507
- `tsc` type-checks the TypeScript around a program, not the program: East's own
508
- refusals — a body whose type differs from the declared output, a capture the
509
- builder refuses, an `IRAnalysisError` — happen when `East.function` runs, at
510
- import.
511
-
512
- ```bash
513
- east-node check dist/index.js # every broken function, at its line
514
- east-node check dist/index.js --format json # the records `east-py check --format json` emits
515
- east-node lsp # the diagnostics over stdio (needs @elaraai/east-diagnostics)
516
- ```
517
-
518
- It reports **every** broken function, not the first. Importing a module runs
519
- it, so `EAST_CHECK=1` is set meanwhile, and a module should skip import-time
520
- work when it sees it. Edit-time idiom checks are separate:
521
- `@elaraai/tsserver-plugin-east` in the editor, `@elaraai/eslint-plugin-east`'s
522
- `east/east-rules` in `make lint`.
523
-
524
- ## Cross-language functions: `East.importFunction` / `East.exportFunctions`
525
-
526
- A function authored in python (with its `East.function`) is called from a
527
- TypeScript body by name, and the other way round; the deployed program is pure
528
- IR. Functions travel as a **manifest** — their IR, declared types and platform
529
- dependencies. In an e3 project `e3.export` builds and links manifests itself
530
- (see **e3**); by hand:
531
-
532
- ```bash
533
- east-py export-functions pricing -o pricing.functions.beast2 -p east-py-std # python side
534
- east-node export-functions dist/maths.js -o maths.functions.beast2 # reads the module's `eastFunctions`
535
- ```
536
-
537
- ```typescript
538
- const score = East.importFunction("pricing", "score", FunctionType([RowType], FloatType)); // typed, callable
539
- const total = East.function([ArrayType(RowType)], FloatType, ($, rows) => rows.map(($, r) => score(r)).sum());
540
-
541
- // in-process: link, then compile — linkImports returns a sync or async function's IR
542
- const { ir } = East.linkImports(total, [East.decodeFunctionManifest(readFileSync("pricing.functions.beast2"))]);
543
- new EastIR(ir as FunctionIR).compile([])(rows);
544
-
545
- // export TypeScript functions for python to import
546
- writeFileSync("maths.functions.beast2", East.encodeFunctionManifest(East.exportFunctions("maths", "1.0.0", { double, halve })));
547
- ```
548
-
549
- - The declared type must equal the exported type **exactly** — a mismatch is a
550
- build error naming both. Unlinked, the reference is a `Platform` node named
551
- `east.importFunction`, so compiling it without linking fails loudly.
552
- - Only closed values export: no captures, and no unresolved imports of their own.
553
- - A manifest carries each function's source map: `linkImports` adds its
554
- locations to the importer's map, which it returns as `sourceMap`, so an
555
- error or a profile names the exporter's source — the python line included.
556
- - The imported function's platform calls must be provided by the consuming
557
- runner (stock families count across runtimes: `east-py-std` ≡
558
- `@elaraai/east-node-std` ≡ `east-c-std`). See `docs/conventions/EAST_CODEGEN.md`.
559
-
560
- ## A JSON contract for a type: `jsonSchemaFor` / `typeFromJsonSchema`
561
-
562
- `East.String.printJson` writes East JSON, lossless and self-describing.
563
- `jsonSchemaFor(T)` emits the JSON Schema of exactly that encoding, so a third
564
- party can validate a payload before sending it; `typeFromJsonSchema` goes the
565
- other way. Both are host-side, like `compareFor`.
566
-
567
- ```typescript
568
- writeFileSync("contract.schema.json", JSON.stringify(jsonSchemaFor(ArrayType(ReadingType), { draft: "draft-07" }), null, 2));
569
- const T = typeFromJsonSchema(JSON.parse(readFileSync("partner.schema.json", "utf-8")));
570
- ```
571
-
572
- | Signature | Description |
573
- |-----------|-------------|
574
- | `jsonSchemaFor(type, options?: { draft }): JsonSchema` | The schema of `type`'s East JSON. `draft`: `"2020-12"` (default), `"draft-07"` or `"openapi-3.0"`. Throws on `Never`, `Function`, `AsyncFunction` |
575
- | `typeFromJsonSchema(schema): EastType` | The type a schema describes. Throws `JsonSchemaUnsupportedError`, with the RFC 6901 pointer, on a keyword East cannot express (`allOf`, `not`, `if`/`then`/`else`, `anyOf`, an open record, …) |
576
- | `EAST_JSON_PATTERNS` | The lexical forms of East JSON's scalars (`integer`, `blob`, `floatSpecials`), for a reader enforcing exactly what the schema says |
577
-
578
- - **An `Option<T>` is `null` or `T`'s own encoding** wherever `T` can never
579
- encode as `null` (`none` → `null`, `some(7n)` → `"7"`; a struct field is
580
- `"note": null` or `"note": "x"`, and an absent key is still an error). Only
581
- `Option<Null>` and `Option<Option<T>>` keep the tagged
582
- `{"type": …, "value": …}` object. A `Recursive` payload is judged by what it
583
- wraps. The rule is a total function of the type, the same on every runtime,
584
- and the tagged object under a flat Option is refused by the payload's decoder
585
- (`expected string, got {"type":"none","value":null}`).
586
- - **A `DateTime` is `format: "date-time"`**: every decoder reads any RFC 3339
587
- date-time — `Z` or any offset (read as UTC), `t`/`z` in either case, any
588
- fractional digits (past the millisecond dropped, never rounded), a leap second
589
- as the Unix time its fields add up to — in years 0001–9999. `printJson`
590
- writes one form: UTC, three fractional digits, `+00:00`.
591
- - **Other scalars are described as the encoder emits them**: an `Integer` a
592
- quoted i64 decimal, a `Blob` lowercase hex (`parseJson` also takes uppercase,
593
- which the contract does not), so a producer that validates never sends what
594
- the strict reader rejects. The patterns spell digits `[0-9]`, so python and
595
- JavaScript validators accept the same strings.
596
- - **The document is deterministic and identical across languages** (east-py's
597
- `json_schema_for` emits the same bytes), and its `x-east-type` annotations
598
- make the inverse exact: `Set` vs `Array`, `Dict` vs an array of pairs, a flat
599
- Option vs any `oneOf`. A foreign schema without them converts by a
600
- documented structural mapping; `nullable: true`, `["string", "null"]` and a
601
- `oneOf` of null and one schema read as an Option, and a
602
- `format: "date-time"` string as a `DateTime`.
603
- - **Recursion binds one `RecursiveType` per cycle group**; definitions that
604
- need two binders are refused, naming them.
605
- - To READ a JSON document larger than memory under a contract, use `Json.open`
606
- / `Json.next` from **east-node-std** (and its east-py-std and east-c-std twins).
607
-
608
- ## Binary serialization (beast2)
609
-
610
- Inside a body the builtins are `East.Blob.encodeBeast(value, 'v2')` and
611
- `blob.decodeBeast(type, 'v2')`; the functions below are the **host-side**
612
- TypeScript API for the same format. Blobs are self-describing, and a decoder
613
- reads any container version its type was written in.
614
-
615
- | Signature | Description |
616
- |-----------|-------------|
617
- | **Whole values** |
618
- | `encodeBeast2For(T, opts?): (value) => Uint8Array` | `opts.version` (`5` default, `4` for legacy readers); `opts.codec` (`"deflate"` \| `"none"`) and `opts.index` for v5 |
619
- | `decodeBeast2For(T, opts?): (blob) => ValueTypeOf<T>` | Any container version, in every runtime and browser. The blob's header type must be `T`, or a subtype whose variant cases line up with `T`'s (a `none` reads as any `Option`); any other is refused — `beast2: cannot decode a blob of type … as …`, the words east-c and east-py use. `opts.platform` for decoded functions |
620
- | `decodeBeast2(blob)` → `{ type, value }` | Without knowing the type |
621
- | `decodeBeast2ForAsync(T)` | The same, decompressing through the platform's `DecompressionStream` — faster for large blobs in a browser |
622
- | `encodeEastIR(ir)` · `decodeEastIR(blob)` | A program with its source map |
623
- | **Collections larger than memory** (v5) |
624
- | `new Beast2ElementWriter(T, sink, opts?)` | The canonical writer: `.add(element)` one Array/Set element or Dict `[key, value]` — Set elements and Dict keys strictly ascending in East order — and the content-defined cut rule places the segments, so the bytes are a function of the value, identical from east-c and east-py, and a one-row edit re-cuts only the segments around it. Memory is one segment; `.finish()` writes the last, the terminator and the index. `sink` is `(bytes) => void`, or `{ segment(s) }` for one standalone segment blob at a time (`s`: `{ count, fence, logicalBytes, blob }`, under `writer.header`) |
625
- | `encodeBeast2PagedFor(T)(value)` | One value in memory through the element writer; plain `Map`/`Set` inputs are sorted first |
626
- | `new Beast2RunSorter(T, openRun, { merge } \| { union })` | Elements in ANY order in, sorted canonical runs out (a key added twice folds with `merge(key, acc, value)`, or `union` for a Set; without one it throws); then `mergeBeast2For(T, { merge } \| { union })(runs, sink)` writes the value. Memory is one run, then one segment per run. A run closes at `RUN_MAX_COUNT` elements or `RUN_MAX_BYTES`, platform constants, so every runtime writes the same runs |
627
- | `mergeBeast2For(T, opts?)(sources, sink)` | Sorted collections — blobs, range readers or manifests — into their union's blob or manifest, segment by segment; a key several hold folds in input order; `opts.from` / `opts.to` merge a key range |
628
- | `recutBeast2For(T)(pieces, sink)` | A collection in pieces, some already written, into its canonical segments: a segment the whole shares with its piece is carried unread (`sink.carried`), the rest re-cut (`sink.written`) |
629
- | `intakeBeast2For(T)(delivery, sink, opts?)` | A collection from outside — a `Beast2SyncRangeReader` over a v4 or v5 blob whose header names exactly `T` — through the Writer into a `Beast2ManifestSink`, a segment at a time: each row is walked by its type, the Writer's own bytes go on as they stand, and any other row is decoded and written again. `opts.segments` takes in the index's segments `[from, to)`, a piece of a large delivery. Refuses another type, a malformed segment or one over `RUN_MAX_BYTES`, a row that does not decode, and keys that do not ascend, with `Beast2IntakeError` in the words every runner uses. What e3's intake units run, on every runner |
630
- | `new Beast2Writer(T, sink)` · `encodeBeast2SegmentsFor(T)(batches)` | Segments of your own choosing: one per non-empty batch, Set/Dict batches in ascending order |
631
- | `iterBeast2SegmentsFor(T)(blob)` | One decoded segment at a time |
632
- | `openBeast2PagesFor(T)(source)` | Random access: `.elementCount` and `.segmentCount` in O(1); `.segment(i)`, `.element(row)`, `.slice(offset, limit)` and `.get(key)` decode only the segments they touch |
633
- | `openBeast2LazyFor(T, { frozen? })(source)` | An ordinary `SortedMap` / `SortedSet` / array served from the index, a segment at a time — what `blob.openBeast` and `FileSystem.openBeast` return, frozen. `isBeast2LazySafe(T, { frozen })`: only `Ref`- and function-bearing shapes decode whole |
634
- | `Beast2SyncRangeReader` — `{ size, read(offset, length) }` | Positioned reads, wherever a blob goes, so a file's bytes never sit on the heap whole; `readBeast2Extents` / `readBeast2ExtentsRanged` (HTTP, S3) read the geometry |
635
- | **Segment manifests** — one object per segment plus a manifest naming them: how e3 stores a collection |
636
- | `new Beast2ManifestWriter(T, { object(hash, bytes), manifest(bytes) })` | The canonical manifest directory: the header, each segment as it is cut (named by its SHA-256, `<manifest>.segments/<hash>.beast2` on disk), then the manifest; east-c and east-py write the same |
637
- | `readBeast2Manifest(source)` | The manifest a blob holds (`{ kind, level, type, rule, header, entries: [{ hash, fence, count, bytes }] }`), or `null`; `manifestElementCount` / `manifestByteSize` total it |
638
- | `{ manifest, segment(i) }` (`Beast2ManifestSource`) | A manifest-held collection wherever a blob goes; `segment(i)` is called only for a segment a read decodes |
639
- | `spliceBeast2Segments(head, segments)` | A manifest's segments spliced back into one blob, a chunk at a time, decoding nothing and holding only the segment in hand — how e3-api-client downloads a collection. Throws on a segment that is not a segmented, indexed v5 collection, or was written under another header |
640
- | `sha256Hex(bytes)` | SHA-256 in lowercase hex, pure TypeScript |
641
-
642
- ```typescript
643
- import { ArrayType, StructType, StringType, IntegerType,
644
- Beast2ElementWriter, iterBeast2SegmentsFor, openBeast2PagesFor, decodeBeast2For } from "@elaraai/east";
645
-
646
- const Rows = ArrayType(StructType({ id: IntegerType, name: StringType }));
647
-
648
- const out = createWriteStream("rows.beast2");
649
- const writer = new Beast2ElementWriter(Rows, bytes => out.write(bytes));
650
- for (const row of rows) writer.add(row); // never whole in memory
651
- writer.finish();
652
-
653
- const blob = new Uint8Array(readFileSync("rows.beast2"));
654
- decodeBeast2For(Rows)(blob); // whole: segments concatenate
655
- for (const segment of iterBeast2SegmentsFor(Rows)(blob)) consume(segment); // a segment at a time
656
- openBeast2PagesFor(Rows)(blob).element(4_000_000); // one row: decodes ONE segment
657
- ```
303
+ Read the one a task needs; each is complete on its subject.
658
304
 
659
- - Set and Dict segments hold the canonical value — strictly ascending,
660
- disjoint, no duplicate keys — so writers refuse out-of-order elements and
661
- decoders refuse a non-canonical blob as corrupt.
662
- - **One encoding per value**: the element writer and `encodeBeast2PagedFor` cut
663
- by the same rule in every runtime, so equal values are equal bytes and share
664
- segments in a content-addressed store. `Beast2Writer` writes the batches you
665
- give it, which every reader also reads.
666
- - Streaming and paging are v5 only. The writer's segments are self-contained,
667
- with an index, which paging and parallel decode need: pass
668
- `{ selfContained: false }` only when aliasing must span segments. `write()`
669
- skips an empty batch, so a segment count is never zero.
670
- - Inside a body, `blob.openBeast(T)` and `FileSystem.openBeast(T, path)` hand
671
- back the frozen lazy value a runner gives a large task input: keyed reads and
672
- `$.for` cost one segment, mutation is refused (`.copy()` first), and the
673
- header type is checked against `T` first. `East.Blob.encodeBeast` writes no
674
- index, so a paged blob comes from the writers above or a runner's output.
305
+ | File | Holds |
306
+ |---|---|
307
+ | [reference/values-in-typescript.md](reference/values-in-typescript.md) | Comparators (`compareFor`, `equalFor`, …), `SortedMap`/`SortedSet`, type utilities, the East-text, JSON, CSV and beast2 codecs; building programs over names and types held as values (`East.field`, `East.withField`, `East.patchesFor`) |
308
+ | [reference/trees-and-recursion.md](reference/trees-and-recursion.md) | `toTree`, `flattenTree`, rollups, and the depth limits of recursion |
309
+ | [reference/queries.md](reference/queries.md) | `East.jq` in a body, and `checkJq` / `evaluateJq` / `splitJq` on the host |
310
+ | [reference/runners-and-ir.md](reference/runners-and-ir.md) | `east-node` / `east-c` / `east-py` `run` and `exec`, `executeUnit` over a `UnitIO`, `East.toSource` and `east-node transpile`, `east-node check` |
311
+ | [reference/cross-language-and-json-schema.md](reference/cross-language-and-json-schema.md) | `East.importFunction` / `East.exportFunctions`, and `jsonSchemaFor` / `typeFromJsonSchema` |
312
+ | [reference/beast2.md](reference/beast2.md) | `encodeBeast2For` / `decodeBeast2For`, paged and streaming writers, `openBeast2PagesFor`, `blob.openBeast` |
675
313
 
676
314
  ## Related skills
677
315
 
@@ -0,0 +1,71 @@
1
+ # Binary serialization (beast2)
2
+
3
+ beast2, the binary form every runtime and e3 store, and collections larger than memory. Part of the `east` skill.
4
+
5
+ ## Binary serialization (beast2)
6
+
7
+ Inside a body the builtins are `East.Blob.encodeBeast(value, 'v2')` and
8
+ `blob.decodeBeast(type, 'v2')`; the functions below are the **host-side**
9
+ TypeScript API for the same format. Blobs are self-describing, and a decoder
10
+ reads any container version its type was written in.
11
+
12
+ | Signature | Description |
13
+ |-----------|-------------|
14
+ | **Whole values** |
15
+ | `encodeBeast2For(T, opts?): (value) => Uint8Array` | `opts.version` (`5` default, `4` for legacy readers); `opts.codec` (`"deflate"` \| `"none"`) and `opts.index` for v5 |
16
+ | `decodeBeast2For(T, opts?): (blob) => ValueTypeOf<T>` | Any container version, in every runtime and browser. The blob's header type must be `T`, or a subtype whose variant cases line up with `T`'s (a `none` reads as any `Option`); any other is refused — `beast2: cannot decode a blob of type … as …`, the words east-c and east-py use. `opts.platform` for decoded functions |
17
+ | `decodeBeast2(blob)` → `{ type, value }` | Without knowing the type |
18
+ | `decodeBeast2ForAsync(T)` | The same, decompressing through the platform's `DecompressionStream` — faster for large blobs in a browser |
19
+ | `encodeEastIR(ir)` · `decodeEastIR(blob)` | A program with its source map |
20
+ | **Collections larger than memory** (v5) |
21
+ | `new Beast2ElementWriter(T, sink, opts?)` | The canonical writer: `.add(element)` one Array/Set element or Dict `[key, value]` — Set elements and Dict keys strictly ascending in East order — and the content-defined cut rule places the segments, so the bytes are a function of the value, identical from east-c and east-py, and a one-row edit re-cuts only the segments around it. Memory is one segment; `.finish()` writes the last, the terminator and the index. `sink` is `(bytes) => void`, or `{ segment(s) }` for one standalone segment blob at a time (`s`: `{ count, fence, logicalBytes, blob }`, under `writer.header`) |
22
+ | `encodeBeast2PagedFor(T)(value)` | One value in memory through the element writer; plain `Map`/`Set` inputs are sorted first |
23
+ | `new Beast2RunSorter(T, openRun, { merge } \| { union })` | Elements in ANY order in, sorted canonical runs out (a key added twice folds with `merge(key, acc, value)`, or `union` for a Set; without one it throws); then `mergeBeast2For(T, { merge } \| { union })(runs, sink)` writes the value. Memory is one run, then one segment per run. A run closes at `RUN_MAX_COUNT` elements or `RUN_MAX_BYTES`, platform constants, so every runtime writes the same runs |
24
+ | `mergeBeast2For(T, opts?)(sources, sink)` | Sorted collections — blobs, range readers or manifests — into their union's blob or manifest, segment by segment; a key several hold folds in input order; `opts.from` / `opts.to` merge a key range |
25
+ | `recutBeast2For(T)(pieces, sink)` | A collection in pieces, some already written, into its canonical segments: a segment the whole shares with its piece is carried unread (`sink.carried`), the rest re-cut (`sink.written`) |
26
+ | `intakeBeast2For(T)(delivery, sink, opts?)` | A collection from outside — a `Beast2SyncRangeReader` over a v4 or v5 blob whose header names exactly `T` — through the Writer into a `Beast2ManifestSink`, a segment at a time: each row is walked by its type, the Writer's own bytes go on as they stand, and any other row is decoded and written again. `opts.segments` takes in the index's segments `[from, to)`, a piece of a large delivery. Refuses another type, a malformed segment or one over `RUN_MAX_BYTES`, a row that does not decode, and keys that do not ascend, with `Beast2IntakeError` in the words every runner uses. What e3's intake units run, on every runner |
27
+ | `new Beast2Writer(T, sink)` · `encodeBeast2SegmentsFor(T)(batches)` | Segments of your own choosing: one per non-empty batch, Set/Dict batches in ascending order |
28
+ | `iterBeast2SegmentsFor(T)(blob)` | One decoded segment at a time |
29
+ | `openBeast2PagesFor(T)(source)` | Random access: `.elementCount` and `.segmentCount` in O(1); `.segment(i)`, `.element(row)`, `.slice(offset, limit)` and `.get(key)` decode only the segments they touch |
30
+ | `openBeast2LazyFor(T, { frozen? })(source)` | An ordinary `SortedMap` / `SortedSet` / array served from the index, a segment at a time — what `blob.openBeast` and `FileSystem.openBeast` return, frozen. `isBeast2LazySafe(T, { frozen })`: only `Ref`- and function-bearing shapes decode whole |
31
+ | `Beast2SyncRangeReader` — `{ size, read(offset, length) }` | Positioned reads, wherever a blob goes, so a file's bytes never sit on the heap whole; `readBeast2Extents` / `readBeast2ExtentsRanged` (HTTP, S3) read the geometry |
32
+ | **Segment manifests** — one object per segment plus a manifest naming them: how e3 stores a collection |
33
+ | `new Beast2ManifestWriter(T, { object(hash, bytes), manifest(bytes) })` | The canonical manifest directory: the header, each segment as it is cut (named by its SHA-256, `<manifest>.segments/<hash>.beast2` on disk), then the manifest; east-c and east-py write the same |
34
+ | `readBeast2Manifest(source)` | The manifest a blob holds (`{ kind, level, type, rule, header, entries: [{ hash, fence, count, bytes }] }`), or `null`; `manifestElementCount` / `manifestByteSize` total it |
35
+ | `{ manifest, segment(i) }` (`Beast2ManifestSource`) | A manifest-held collection wherever a blob goes; `segment(i)` is called only for a segment a read decodes |
36
+ | `spliceBeast2Segments(head, segments)` | A manifest's segments spliced back into one blob, a chunk at a time, decoding nothing and holding only the segment in hand — how e3-api-client downloads a collection. Throws on a segment that is not a segmented, indexed v5 collection, or was written under another header |
37
+ | `sha256Hex(bytes)` | SHA-256 in lowercase hex, pure TypeScript |
38
+
39
+ ```typescript
40
+ import { ArrayType, StructType, StringType, IntegerType,
41
+ Beast2ElementWriter, iterBeast2SegmentsFor, openBeast2PagesFor, decodeBeast2For } from "@elaraai/east";
42
+
43
+ const Rows = ArrayType(StructType({ id: IntegerType, name: StringType }));
44
+
45
+ const out = createWriteStream("rows.beast2");
46
+ const writer = new Beast2ElementWriter(Rows, bytes => out.write(bytes));
47
+ for (const row of rows) writer.add(row); // never whole in memory
48
+ writer.finish();
49
+
50
+ const blob = new Uint8Array(readFileSync("rows.beast2"));
51
+ decodeBeast2For(Rows)(blob); // whole: segments concatenate
52
+ for (const segment of iterBeast2SegmentsFor(Rows)(blob)) consume(segment); // a segment at a time
53
+ openBeast2PagesFor(Rows)(blob).element(4_000_000); // one row: decodes ONE segment
54
+ ```
55
+
56
+ - Set and Dict segments hold the canonical value — strictly ascending,
57
+ disjoint, no duplicate keys — so writers refuse out-of-order elements and
58
+ decoders refuse a non-canonical blob as corrupt.
59
+ - **One encoding per value**: the element writer and `encodeBeast2PagedFor` cut
60
+ by the same rule in every runtime, so equal values are equal bytes and share
61
+ segments in a content-addressed store. `Beast2Writer` writes the batches you
62
+ give it, which every reader also reads.
63
+ - Streaming and paging are v5 only. The writer's segments are self-contained,
64
+ with an index, which paging and parallel decode need: pass
65
+ `{ selfContained: false }` only when aliasing must span segments. `write()`
66
+ skips an empty batch, so a segment count is never zero.
67
+ - Inside a body, `blob.openBeast(T)` and `FileSystem.openBeast(T, path)` hand
68
+ back the frozen lazy value a runner gives a large task input: keyed reads and
69
+ `$.for` cost one segment, mutation is refused (`.copy()` first), and the
70
+ header type is checked against `T` first. `East.Blob.encodeBeast` writes no
71
+ index, so a paged blob comes from the writers above or a runner's output.