databonk 0.0.4 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +491 -96
- package/dist/dataframe-Bz1B7bIr.d.cts +1001 -0
- package/dist/dataframe-Bz1B7bIr.d.ts +1001 -0
- package/dist/index.cjs +5 -0
- package/dist/index.cjs.map +1 -0
- package/dist/index.d.cts +366 -0
- package/dist/index.d.ts +354 -31
- package/dist/index.js +4 -40
- package/dist/index.js.map +1 -1
- package/dist/parallel-fv5h4BkA.d.cts +152 -0
- package/dist/parallel-fv5h4BkA.d.ts +152 -0
- package/dist/parquet.cjs +3 -0
- package/dist/parquet.cjs.map +1 -0
- package/dist/parquet.d.cts +57 -0
- package/dist/parquet.d.ts +57 -0
- package/dist/parquet.js +3 -0
- package/dist/parquet.js.map +1 -0
- package/dist/scalar.wasm +0 -0
- package/dist/simd-threads.wasm +0 -0
- package/dist/simd.wasm +0 -0
- package/dist/workers.cjs +102 -0
- package/dist/workers.cjs.map +1 -0
- package/dist/workers.d.cts +74 -0
- package/dist/workers.d.ts +74 -0
- package/dist/workers.js +102 -0
- package/dist/workers.js.map +1 -0
- package/package.json +93 -32
- package/build/release.d.ts +0 -719
- package/build/release.js +0 -774
- package/build/release.wasm +0 -0
- package/build/release.wasm.map +0 -1
- package/build/release.wat +0 -22633
- package/dist/dataframe.d.ts +0 -82
- package/dist/dataframe.d.ts.map +0 -1
- package/dist/dataframe.js +0 -318
- package/dist/dataframe.js.map +0 -1
- package/dist/index.d.ts.map +0 -1
- package/dist/loader.d.ts +0 -86
- package/dist/loader.d.ts.map +0 -1
- package/dist/loader.js +0 -147
- package/dist/loader.js.map +0 -1
- package/dist/shared-memory.d.ts +0 -64
- package/dist/shared-memory.d.ts.map +0 -1
- package/dist/shared-memory.js +0 -113
- package/dist/shared-memory.js.map +0 -1
|
@@ -0,0 +1,1001 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* WASM memory-core loader with SIMD feature detection (ADR-004).
|
|
3
|
+
*
|
|
4
|
+
* Feature-detects SIMD via `WebAssembly.validate` on a tiny inlined SIMD module,
|
|
5
|
+
* then instantiates the matching binary — `simd.wasm` or `scalar.wasm` — over
|
|
6
|
+
* either a Node filesystem path or a browser URL. The module exports the
|
|
7
|
+
* Phase-1 memory core (ABI §9): `memory`, `alloc`, `free`, `realloc`,
|
|
8
|
+
* `mem_generation`.
|
|
9
|
+
*
|
|
10
|
+
* View access is NOT done here — hold no `TypedArray` outside `viewOf`
|
|
11
|
+
* (ADR-001). See {@link createViewOf} in `./views.ts`.
|
|
12
|
+
*/
|
|
13
|
+
/** True iff the current runtime supports wasm SIMD128 (ADR-004). */
|
|
14
|
+
declare function detectSimd(): boolean;
|
|
15
|
+
/** Raw exports of the memory-core wasm module (ABI §9, Phase 1). */
|
|
16
|
+
interface WasmExports {
|
|
17
|
+
/** The module's single linear memory (ADR-001; all column bytes live here). */
|
|
18
|
+
readonly memory: WebAssembly.Memory;
|
|
19
|
+
/** `alloc(size) -> ptr`: 16-byte-aligned, `0` on OOM (ABI §3). */
|
|
20
|
+
alloc(size: number): number;
|
|
21
|
+
/** `free(ptr)`: `free(0)` is a no-op (ABI §3). */
|
|
22
|
+
free(ptr: number): void;
|
|
23
|
+
/** `realloc(ptr, newSize) -> ptr`: preserves contents; `0` on OOM (ABI §3). */
|
|
24
|
+
realloc(ptr: number, newSize: number): number;
|
|
25
|
+
/** `mem_generation()`: changes on every successful `memory.grow` (ABI §2). */
|
|
26
|
+
mem_generation(): number;
|
|
27
|
+
}
|
|
28
|
+
/** A loaded memory-core module plus which build was selected. */
|
|
29
|
+
interface WasmMemoryModule extends WasmExports {
|
|
30
|
+
/** `true` if the SIMD build was loaded, `false` for the scalar build. */
|
|
31
|
+
readonly simd: boolean;
|
|
32
|
+
}
|
|
33
|
+
/** Options for {@link loadWasmModule}. */
|
|
34
|
+
interface LoadOptions {
|
|
35
|
+
/** Force a build. Default: auto-detect via {@link detectSimd}. */
|
|
36
|
+
simd?: boolean;
|
|
37
|
+
/**
|
|
38
|
+
* Location of `scalar.wasm` / `simd.wasm`.
|
|
39
|
+
* - Node: a filesystem directory path, or a `file:` / directory `URL`.
|
|
40
|
+
* - Browser: a base `URL` (or URL string) the two files are fetched under.
|
|
41
|
+
*
|
|
42
|
+
* Default: resolved relative to this module (the built binaries are copied
|
|
43
|
+
* next to the JS bundle in `dist/`).
|
|
44
|
+
*/
|
|
45
|
+
wasmDir?: string | URL;
|
|
46
|
+
}
|
|
47
|
+
/**
|
|
48
|
+
* Feature-detect, load, and instantiate the memory-core wasm module.
|
|
49
|
+
* Runs once per page/process (ADR-004); callers cache the returned module.
|
|
50
|
+
*/
|
|
51
|
+
declare function loadWasmModule(opts?: LoadOptions): Promise<WasmMemoryModule>;
|
|
52
|
+
|
|
53
|
+
/**
|
|
54
|
+
* The single sanctioned `viewOf()` accessor (ADR-001, ABI §2).
|
|
55
|
+
*
|
|
56
|
+
* `memory.grow` detaches every `TypedArray` over the old `ArrayBuffer`. To stay
|
|
57
|
+
* correct without copying, ALL view access goes through one accessor that:
|
|
58
|
+
* 1. caches `(generation, view)` per registered column buffer,
|
|
59
|
+
* 2. checks `mem_generation()` before every use, and
|
|
60
|
+
* 3. on a mismatch, rebuilds ALL registered views over the current
|
|
61
|
+
* `memory.buffer`.
|
|
62
|
+
*
|
|
63
|
+
* No raw `TypedArray` may be cached anywhere else in the library.
|
|
64
|
+
*/
|
|
65
|
+
|
|
66
|
+
/** Storage dtypes whose data buffer maps to a numeric `TypedArray` (dtypes.md §1). `i64` maps to `BigInt64Array`. */
|
|
67
|
+
type ViewDType = 'f64' | 'f32' | 'i32' | 'u32' | 'u8' | 'bool' | 'i64';
|
|
68
|
+
/** Location + shape of a column buffer inside linear memory. */
|
|
69
|
+
interface ColumnBuffer {
|
|
70
|
+
/** Byte offset into `memory.buffer` (16-byte aligned; ABI §3). */
|
|
71
|
+
readonly ptr: number;
|
|
72
|
+
/** Element count (NOT bytes; ABI §4.3). */
|
|
73
|
+
readonly length: number;
|
|
74
|
+
/** Determines the `TypedArray` kind; `bool` is `u8` storage (dtypes.md §1). */
|
|
75
|
+
readonly dtype: ViewDType;
|
|
76
|
+
}
|
|
77
|
+
/** The concrete `TypedArray` kinds a column view can be. `BigInt64Array` is used for `i64` columns. */
|
|
78
|
+
type ColumnView = Float64Array | Float32Array | Int32Array | Uint32Array | Uint8Array | BigInt64Array;
|
|
79
|
+
/** The `viewOf` accessor returned by {@link createViewOf}. */
|
|
80
|
+
interface ViewOf {
|
|
81
|
+
/**
|
|
82
|
+
* Return a live `TypedArray` for `col` over the current `memory.buffer`,
|
|
83
|
+
* rebuilding every registered view first if memory has grown since last use.
|
|
84
|
+
* The returned view must not be cached by callers — call `viewOf` again.
|
|
85
|
+
*/
|
|
86
|
+
(col: ColumnBuffer): ColumnView;
|
|
87
|
+
/** The generation the cache is currently synced to. */
|
|
88
|
+
generation(): number;
|
|
89
|
+
/** Stop tracking `col` (drops it from the registry and cache). */
|
|
90
|
+
forget(col: ColumnBuffer): void;
|
|
91
|
+
/** Drop all tracked columns and cached views. */
|
|
92
|
+
clear(): void;
|
|
93
|
+
}
|
|
94
|
+
/**
|
|
95
|
+
* Build the memory context's single `viewOf` accessor over `mod`'s linear
|
|
96
|
+
* memory and generation counter.
|
|
97
|
+
*/
|
|
98
|
+
declare function createViewOf(mod: Pick<WasmMemoryModule, 'memory' | 'mem_generation'>): ViewOf;
|
|
99
|
+
|
|
100
|
+
/**
|
|
101
|
+
* A `MemoryContext` bundles the loaded wasm module (allocator + linear memory)
|
|
102
|
+
* with its single {@link ViewOf} accessor (ADR-001). Every column / dictionary
|
|
103
|
+
* operation in this layer takes a context: it allocates through `ctx.mod` and
|
|
104
|
+
* reaches bytes only through `ctx.viewOf` — the one sanctioned TypedArray path.
|
|
105
|
+
*
|
|
106
|
+
* There is exactly **one** `viewOf` per module (one linear memory, one generation
|
|
107
|
+
* counter), so there is exactly one context per module.
|
|
108
|
+
*/
|
|
109
|
+
|
|
110
|
+
/** The allocator + the single `viewOf` accessor over one module's linear memory. */
|
|
111
|
+
interface MemoryContext {
|
|
112
|
+
/** The loaded memory-core module (`alloc`/`free`/`realloc`/`memory`, ABI §3/§9). */
|
|
113
|
+
readonly mod: WasmMemoryModule;
|
|
114
|
+
/** The one sanctioned `TypedArray` accessor over `mod.memory` (ADR-001). */
|
|
115
|
+
readonly viewOf: ViewOf;
|
|
116
|
+
}
|
|
117
|
+
/** Build the single {@link MemoryContext} for a loaded module. */
|
|
118
|
+
declare function createMemoryContext(mod: WasmMemoryModule): MemoryContext;
|
|
119
|
+
|
|
120
|
+
/**
|
|
121
|
+
* Schema / dtype registry (Phase 1, deliverable P1.2 §3).
|
|
122
|
+
*
|
|
123
|
+
* One descriptor per v1 storage dtype (`contracts/dtypes.md` §1). The descriptor
|
|
124
|
+
* carries everything the column, dictionary, and (Phase 2) kernel/expr layers need
|
|
125
|
+
* to lay a dtype out in linear memory and name its kernels:
|
|
126
|
+
* - `size` — bytes per stored element (utf8 stores an `i32` dictionary index),
|
|
127
|
+
* - `view` — the {@link ViewDType} its data buffer maps to through `viewOf`,
|
|
128
|
+
* - `ctor` — the matching `TypedArray` constructor for a JS-side staging copy,
|
|
129
|
+
* - `wasm` — the kernel-name dtype token (ABI §6 `[family_]op_dtype[_variant]`),
|
|
130
|
+
* - `float` — whether `NaN`/`±inf` are *values* (floats), never nulls (dtypes.md §4).
|
|
131
|
+
*/
|
|
132
|
+
|
|
133
|
+
/**
|
|
134
|
+
* The v2 column dtypes (`contracts/dtypes.md` §1/§6). `utf8` is dict-encoded; `i64` uses
|
|
135
|
+
* BigInt64Array. `date32` and `timestamp` are logical dtypes that dispatch to i32/i64 physical
|
|
136
|
+
* kernels respectively (ADR-010; `wasm` field carries the physical token).
|
|
137
|
+
*/
|
|
138
|
+
type DType = 'f64' | 'f32' | 'i32' | 'u32' | 'bool' | 'utf8' | 'i64' | 'date32' | 'timestamp';
|
|
139
|
+
/** `TypedArray` constructors a column data / auxiliary buffer can map to. */
|
|
140
|
+
type TypedArrayCtor = Float64ArrayConstructor | Float32ArrayConstructor | Int32ArrayConstructor | Uint32ArrayConstructor | Uint8ArrayConstructor | BigInt64ArrayConstructor;
|
|
141
|
+
/** Static description of one v1 dtype (see module doc). */
|
|
142
|
+
interface DTypeInfo {
|
|
143
|
+
/** The dtype this describes. */
|
|
144
|
+
readonly name: DType;
|
|
145
|
+
/** Bytes per stored element. `utf8` = 4 (the `i32` index into its dictionary). */
|
|
146
|
+
readonly size: number;
|
|
147
|
+
/** `viewOf` dtype for this column's *data* buffer (`utf8` → its `i32` indices). */
|
|
148
|
+
readonly view: ViewDType;
|
|
149
|
+
/** `TypedArray` constructor matching {@link view} (for staging copies). */
|
|
150
|
+
readonly ctor: TypedArrayCtor;
|
|
151
|
+
/** Kernel-name dtype token (ABI §6). Same as {@link name} for every v1 dtype. */
|
|
152
|
+
readonly wasm: string;
|
|
153
|
+
/** True for `f64`/`f32`: a `NaN`/`±inf` is a valid *value*, never a null (dtypes.md §4). */
|
|
154
|
+
readonly float: boolean;
|
|
155
|
+
}
|
|
156
|
+
/** The dtype registry: `DTYPES[dtype]` → its {@link DTypeInfo}. */
|
|
157
|
+
declare const DTYPES: Record<DType, DTypeInfo>;
|
|
158
|
+
/** Descriptor for `dtype`. Throws on an unknown dtype (helpful for API callers). */
|
|
159
|
+
declare function dtypeInfo(dtype: DType): DTypeInfo;
|
|
160
|
+
|
|
161
|
+
/**
|
|
162
|
+
* Dictionary string store (Phase 1, deliverable P1.2 §2; ABI §4.4, ADR-002).
|
|
163
|
+
*
|
|
164
|
+
* A `utf8` column's strings are dictionary-encoded into three wasm buffers:
|
|
165
|
+
* - the column's own `i32[len]` **indices** (built in `column.ts`), plus this
|
|
166
|
+
* dictionary's shared:
|
|
167
|
+
* - **offsets** — `i32[count + 1]`, Arrow-style monotonic byte offsets,
|
|
168
|
+
* `offsets[0] == 0`, string `k` = `bytes[offsets[k] .. offsets[k+1])`;
|
|
169
|
+
* - **bytes** — `u8[offsets[count]]`, UTF-8 concatenation of the unique strings.
|
|
170
|
+
*
|
|
171
|
+
* Decode is **memoized per slot** on the JS side (ADR-002): each unique string
|
|
172
|
+
* crosses the wasm→JS boundary at most once. The cache is keyed by *dictionary
|
|
173
|
+
* identity* (a `WeakMap` on the `Dictionary` object) + slot index, so distinct
|
|
174
|
+
* dictionaries never share decoded strings and the cache is GC'd with the dict.
|
|
175
|
+
*
|
|
176
|
+
* Unification (`unifyDictionaries`) is JS-side for Phase 1 — the wasm `unify_dict`
|
|
177
|
+
* kernel arrives in Phase 2 (ABI §9, Agent D). It produces a merged unique list
|
|
178
|
+
* plus per-slot index remaps so two columns can be compared under one dictionary.
|
|
179
|
+
*/
|
|
180
|
+
|
|
181
|
+
/**
|
|
182
|
+
* The shared dictionary buffers of a `utf8` column (ABI §4.4). Immutable: build
|
|
183
|
+
* once, decode many. `count == 0` is legal (all-null / empty column): `offsets`
|
|
184
|
+
* is still `[0]` and `bytesLen == 0`.
|
|
185
|
+
*/
|
|
186
|
+
interface Dictionary {
|
|
187
|
+
/** Number of unique strings. */
|
|
188
|
+
readonly count: number;
|
|
189
|
+
/** Byte offset of the `i32[count + 1]` offsets buffer. */
|
|
190
|
+
readonly offsetsPtr: number;
|
|
191
|
+
/** Byte offset of the `u8[bytesLen]` UTF-8 bytes buffer. */
|
|
192
|
+
readonly bytesPtr: number;
|
|
193
|
+
/** Total UTF-8 byte length (`offsets[count]`). */
|
|
194
|
+
readonly bytesLen: number;
|
|
195
|
+
}
|
|
196
|
+
/**
|
|
197
|
+
* Encode `uniques` (already-deduplicated strings) into fresh offsets + bytes
|
|
198
|
+
* buffers in linear memory and return the {@link Dictionary} describing them.
|
|
199
|
+
* Allocates all buffers *before* taking any view (a grow between alloc and write
|
|
200
|
+
* would detach it, ADR-001).
|
|
201
|
+
*/
|
|
202
|
+
declare function writeDictionary(ctx: MemoryContext, uniques: readonly string[]): Dictionary;
|
|
203
|
+
/**
|
|
204
|
+
* Write a dictionary directly from raw UTF-8 bytes and Arrow-style offsets —
|
|
205
|
+
* no JS string decode or encode (ABI §12 ingest note, ADR-002).
|
|
206
|
+
*
|
|
207
|
+
* `rawOffsets[k]..rawOffsets[k+1]` is the byte range of string `k` in
|
|
208
|
+
* `rawBytes`; both arrays are already in our dictionaries' layout. This is a
|
|
209
|
+
* pure bulk-copy: O(count + bytesLen), zero TextDecoder/TextEncoder calls.
|
|
210
|
+
*
|
|
211
|
+
* Allocates both wasm buffers before taking any view so a grow between the two
|
|
212
|
+
* allocs cannot detach a live TypedArray (ADR-001).
|
|
213
|
+
*/
|
|
214
|
+
declare function writeDictionaryFromRawBytes(ctx: MemoryContext, rawBytes: Uint8Array, rawOffsets: Int32Array, count: number): Dictionary;
|
|
215
|
+
/**
|
|
216
|
+
* Decode dictionary slot `slot` to a string, memoized (ADR-002). First call for a
|
|
217
|
+
* slot reads its UTF-8 bytes across the boundary (a *miss*); later calls reuse the
|
|
218
|
+
* cached string (a *hit*). `slot` must be in `[0, dict.count)`.
|
|
219
|
+
*/
|
|
220
|
+
declare function decodeSlot(ctx: MemoryContext, dict: Dictionary, slot: number): string;
|
|
221
|
+
/** Decode every slot of `dict` into a `string[]` (memoized per slot). */
|
|
222
|
+
declare function decodeDictionary(ctx: MemoryContext, dict: Dictionary): string[];
|
|
223
|
+
/** Decode-cache accounting for `dict` (`{hits:0,misses:0}` if never decoded). */
|
|
224
|
+
declare function decodeStats(dict: Dictionary): {
|
|
225
|
+
hits: number;
|
|
226
|
+
misses: number;
|
|
227
|
+
};
|
|
228
|
+
/** Result of unifying two dictionaries into a single merged one (JS-side). */
|
|
229
|
+
interface DictUnifyResult {
|
|
230
|
+
/** Merged unique strings; index `j` is a slot in the merged dictionary. */
|
|
231
|
+
readonly merged: string[];
|
|
232
|
+
/** `remapA[i]` = merged slot of dictionary-A slot `i` (`Int32Array`, ABI indices). */
|
|
233
|
+
readonly remapA: Int32Array;
|
|
234
|
+
/** `remapB[i]` = merged slot of dictionary-B slot `i`. */
|
|
235
|
+
readonly remapB: Int32Array;
|
|
236
|
+
}
|
|
237
|
+
/**
|
|
238
|
+
* Unify two dictionaries into one merged unique list plus per-slot index remaps
|
|
239
|
+
* (JS-side; the wasm `unify_dict` kernel is Phase 2). Value-preserving by
|
|
240
|
+
* construction: `merged[remapA[i]] === decode(dictA, i)` for every `i`. The
|
|
241
|
+
* merged order is A's slots in order, then B's not-yet-seen slots.
|
|
242
|
+
*/
|
|
243
|
+
declare function unifyDictionaries(ctx: MemoryContext, dictA: Dictionary, dictB: Dictionary): DictUnifyResult;
|
|
244
|
+
/** Free a dictionary's buffers and drop them from the view registry. */
|
|
245
|
+
declare function freeDictionary(ctx: MemoryContext, dict: Dictionary): void;
|
|
246
|
+
|
|
247
|
+
/**
|
|
248
|
+
* Column representation (Phase 1, deliverable P1.2 §1/§4; ABI §4).
|
|
249
|
+
*
|
|
250
|
+
* A {@link Column} is a small JS descriptor over buffers living in wasm linear
|
|
251
|
+
* memory (ADR-001); JS never owns the bytes. Per ABI §4 it carries `dtype`,
|
|
252
|
+
* `length`, a `dataPtr`, a `validityPtr` (`0` = all-valid), and — for `utf8` —
|
|
253
|
+
* the shared {@link Dictionary}.
|
|
254
|
+
*
|
|
255
|
+
* ## Zero-copy slice & the offset convention (deliverable §4)
|
|
256
|
+
* `sliceColumn` shares the parent's buffers; no bytes move. Two offset kinds:
|
|
257
|
+
*
|
|
258
|
+
* - **data** is byte-addressable, so a slice bakes its start into `dataPtr`:
|
|
259
|
+
* `dataPtr = parent.dataPtr + start * dtype.size`. Element `i`'s data byte
|
|
260
|
+
* offset is `dataPtr + i * size`. **`dataPtr` is exactly the base pointer a
|
|
261
|
+
* kernel receives** — the byte-offset math happens here, in the memory layer,
|
|
262
|
+
* so Phase-2 kernels take `(dataPtr, len)` unchanged. Alignment is preserved:
|
|
263
|
+
* offsetting by whole elements keeps the natural alignment `alloc` guarantees.
|
|
264
|
+
*
|
|
265
|
+
* - **validity** is *bit*-addressed, so a slice cannot move the pointer; it keeps
|
|
266
|
+
* the parent's `validityPtr` and records `validityBitOffset` (the bit index of
|
|
267
|
+
* element 0). Element `i`'s validity bit is `validityBitOffset + i` in the
|
|
268
|
+
* bitmap at `validityPtr`. For a root column `validityBitOffset == 0`. A
|
|
269
|
+
* Phase-2 kernel consuming a sliced column's validity must honor this bit
|
|
270
|
+
* offset (or the memory layer realigns first); root columns — the common case —
|
|
271
|
+
* pass `validityPtr` directly. `validityPtr == 0` always means all-valid.
|
|
272
|
+
*/
|
|
273
|
+
|
|
274
|
+
/** A column descriptor over wasm buffers (see module doc; ABI §4). */
|
|
275
|
+
interface Column {
|
|
276
|
+
/** Storage dtype (`contracts/dtypes.md` §1/§6). */
|
|
277
|
+
readonly dtype: DType;
|
|
278
|
+
/** Element count of this (possibly sliced) column. */
|
|
279
|
+
readonly length: number;
|
|
280
|
+
/** Base byte offset of element 0's data (slice start already baked in). */
|
|
281
|
+
readonly dataPtr: number;
|
|
282
|
+
/** Validity bitmap pointer, or `0` for an all-valid column (ABI §4.1). */
|
|
283
|
+
readonly validityPtr: number;
|
|
284
|
+
/** Bit index of element 0 within the bitmap at `validityPtr` (0 for roots). */
|
|
285
|
+
readonly validityBitOffset: number;
|
|
286
|
+
/** The shared dictionary for a `utf8` column; `null` for every other dtype. */
|
|
287
|
+
readonly dict: Dictionary | null;
|
|
288
|
+
/** True if this column owns its buffers (a root); slices share and own nothing. */
|
|
289
|
+
readonly owned: boolean;
|
|
290
|
+
/**
|
|
291
|
+
* IANA timezone string for `timestamp` columns (ADR-010 §10 display/accessor metadata only).
|
|
292
|
+
* `undefined` = UTC. Not present for any other dtype. The stored value is always UTC epoch ms;
|
|
293
|
+
* tz only affects how dt accessors and the printer interpret the instant.
|
|
294
|
+
*/
|
|
295
|
+
readonly tz?: string;
|
|
296
|
+
}
|
|
297
|
+
/** JS value shapes accepted by {@link createColumn}, per dtype. */
|
|
298
|
+
type ColumnInput = ArrayLike<number | null | undefined> | ArrayLike<bigint | number | null | undefined> | ArrayLike<boolean | null | undefined> | ArrayLike<string | null | undefined> | ArrayLike<Date | null | undefined>;
|
|
299
|
+
/** One decoded column cell: a value, or `null` for a null slot. */
|
|
300
|
+
type Cell = number | bigint | boolean | string | null;
|
|
301
|
+
/**
|
|
302
|
+
* Build a column from JS `values` for `dtype`. Two paths:
|
|
303
|
+
* - **fast path** — a matching `TypedArray` (no nulls possible): bulk-copy into
|
|
304
|
+
* linear memory, `validityPtr = 0`;
|
|
305
|
+
* - **slow path** — a plain array: detect `null`/`undefined` (only these are
|
|
306
|
+
* nulls — a `NaN` is a *value*, dtypes.md §4) and build the validity bitmap.
|
|
307
|
+
*
|
|
308
|
+
* Temporal dtypes (ADR-010):
|
|
309
|
+
* - `date32`: accepts `number` (days since epoch), `Int32Array` (fast path), or `Date` (→ day).
|
|
310
|
+
* - `timestamp`: accepts `bigint`/safe-int number (ms since epoch), `BigInt64Array`, `Date` (→ ms),
|
|
311
|
+
* or ISO-8601 string (only via explicit dtype). `toArray` returns `bigint` ms per dtypes.md §11.
|
|
312
|
+
* Optional `tz` attached as metadata for `timestamp` (display/accessor only, ADR-010).
|
|
313
|
+
*/
|
|
314
|
+
declare function createColumn(ctx: MemoryContext, dtype: DType, values: ColumnInput, tz?: string): Column;
|
|
315
|
+
/**
|
|
316
|
+
* Export a column back to a JS array, with null slots as `null` (deliverable §1).
|
|
317
|
+
* `f64`/`f32` `NaN`s round-trip as values; `bool` yields `boolean`; `utf8` yields
|
|
318
|
+
* memoized-decoded strings.
|
|
319
|
+
*
|
|
320
|
+
* Temporal boundary per dtypes.md §11:
|
|
321
|
+
* - `date32` → `number` (days since epoch), matching the physical i32 storage.
|
|
322
|
+
* - `timestamp` → `bigint` (ms since epoch), matching the physical i64 storage.
|
|
323
|
+
* Use `Series.toDates()` for convenient `Date[]` conversion.
|
|
324
|
+
*/
|
|
325
|
+
declare function columnToArray(ctx: MemoryContext, col: Column): Cell[];
|
|
326
|
+
/**
|
|
327
|
+
* Zero-copy slice `[start, end)` sharing the parent's buffers (deliverable §4).
|
|
328
|
+
* `start`/`end` are clamped to `[0, length]`; `end < start` yields an empty slice.
|
|
329
|
+
* See the module doc for the data/validity offset convention. Slice-of-slice
|
|
330
|
+
* composes: offsets accumulate, buffers stay shared.
|
|
331
|
+
*/
|
|
332
|
+
declare function sliceColumn(col: Column, start: number, end: number): Column;
|
|
333
|
+
/**
|
|
334
|
+
* Free a column's owned buffers and drop them from the view registry. A no-op for
|
|
335
|
+
* slices (`owned === false`) — they share buffers owned by the root. Frees the
|
|
336
|
+
* dictionary too for an owned `utf8` column.
|
|
337
|
+
*/
|
|
338
|
+
declare function freeColumn(ctx: MemoryContext, col: Column): void;
|
|
339
|
+
|
|
340
|
+
/**
|
|
341
|
+
* Expression AST (Phase 3, P3.1 deliverable §1).
|
|
342
|
+
*
|
|
343
|
+
* The public expression surface from spec §4. `col('a')` / `lit(v)` are the two
|
|
344
|
+
* leaf builders; every operator is a chainable method returning a **new**
|
|
345
|
+
* {@link Expr}. Nodes are immutable (deep-frozen), so an `Expr` can be shared and
|
|
346
|
+
* re-compiled without aliasing hazards.
|
|
347
|
+
*
|
|
348
|
+
* col('a').gt(5).and(col('b').eq('x'))
|
|
349
|
+
* col('a').add(col('b')).mul(2)
|
|
350
|
+
* col('a').cast('f32').sum()
|
|
351
|
+
*
|
|
352
|
+
* Type resolution (result dtype, the single int→float widening rule, cast-insertion
|
|
353
|
+
* points, unsupported-mix errors) lives in `./dtypes.ts`; lowering to kernel calls
|
|
354
|
+
* lives in `./compile.ts`. This file is pure data + ergonomics — no wasm, no memory.
|
|
355
|
+
*/
|
|
356
|
+
|
|
357
|
+
/** Binary arithmetic operators (dtypes.md §3.1/§3.2). */
|
|
358
|
+
type ArithOp = 'add' | 'sub' | 'mul' | 'div' | 'mod';
|
|
359
|
+
/** dt accessor field names (dtypes.md §10, ADR-010). */
|
|
360
|
+
type DtComponent = 'year' | 'month' | 'day' | 'hour' | 'minute' | 'second' | 'millisecond' | 'weekday' | 'dayOfYear' | 'quarter';
|
|
361
|
+
/** str namespace op names (dtypes.md §13). v1 has only 'slice'. */
|
|
362
|
+
type StrOp = 'slice';
|
|
363
|
+
/** Comparison operators → boolean/mask (dtypes.md §4.1). */
|
|
364
|
+
type CompareOp = 'gt' | 'ge' | 'lt' | 'le' | 'eq' | 'ne';
|
|
365
|
+
/** Short-circuit-free three-valued boolean operators (dtypes.md §4.2). */
|
|
366
|
+
type BoolOp = 'and' | 'or';
|
|
367
|
+
/** Reduction operators (dtypes.md §4.3). */
|
|
368
|
+
type AggOp = 'sum' | 'mean' | 'min' | 'max' | 'count' | 'nunique' | 'std' | 'var' | 'first' | 'last';
|
|
369
|
+
/** A raw JS scalar that a literal / fill value can hold. */
|
|
370
|
+
type ScalarValue = number | bigint | string | boolean;
|
|
371
|
+
/** The immutable AST node inside every {@link Expr}. Discriminated on `kind`. */
|
|
372
|
+
type ExprNode = Readonly<{
|
|
373
|
+
kind: 'col';
|
|
374
|
+
name: string;
|
|
375
|
+
}> | Readonly<{
|
|
376
|
+
kind: 'lit';
|
|
377
|
+
value: ScalarValue;
|
|
378
|
+
dtype: DType | null;
|
|
379
|
+
}> | Readonly<{
|
|
380
|
+
kind: 'arith';
|
|
381
|
+
op: ArithOp;
|
|
382
|
+
left: Expr;
|
|
383
|
+
right: Expr;
|
|
384
|
+
}> | Readonly<{
|
|
385
|
+
kind: 'neg';
|
|
386
|
+
operand: Expr;
|
|
387
|
+
}> | Readonly<{
|
|
388
|
+
kind: 'compare';
|
|
389
|
+
op: CompareOp;
|
|
390
|
+
left: Expr;
|
|
391
|
+
right: Expr;
|
|
392
|
+
}> | Readonly<{
|
|
393
|
+
kind: 'bool';
|
|
394
|
+
op: BoolOp;
|
|
395
|
+
left: Expr;
|
|
396
|
+
right: Expr;
|
|
397
|
+
}> | Readonly<{
|
|
398
|
+
kind: 'not';
|
|
399
|
+
operand: Expr;
|
|
400
|
+
}> | Readonly<{
|
|
401
|
+
kind: 'isNull';
|
|
402
|
+
operand: Expr;
|
|
403
|
+
}> | Readonly<{
|
|
404
|
+
kind: 'fillNull';
|
|
405
|
+
operand: Expr;
|
|
406
|
+
value: ScalarValue;
|
|
407
|
+
}> | Readonly<{
|
|
408
|
+
kind: 'cast';
|
|
409
|
+
operand: Expr;
|
|
410
|
+
to: DType;
|
|
411
|
+
}> | Readonly<{
|
|
412
|
+
kind: 'agg';
|
|
413
|
+
op: AggOp;
|
|
414
|
+
operand: Expr;
|
|
415
|
+
}>
|
|
416
|
+
/** dt accessor: extract a calendar field from a date32 or timestamp column. */
|
|
417
|
+
| Readonly<{
|
|
418
|
+
kind: 'dt';
|
|
419
|
+
component: DtComponent;
|
|
420
|
+
operand: Expr;
|
|
421
|
+
}>
|
|
422
|
+
/**
|
|
423
|
+
* str.slice: substring via JS String.prototype.slice semantics (dtypes.md §13).
|
|
424
|
+
* Applied to dictionary values once (O(unique)), then indices remapped (O(rows)).
|
|
425
|
+
*/
|
|
426
|
+
| Readonly<{
|
|
427
|
+
kind: 'strSlice';
|
|
428
|
+
operand: Expr;
|
|
429
|
+
start: number;
|
|
430
|
+
end: number | undefined;
|
|
431
|
+
}>;
|
|
432
|
+
/** Anything accepted where an expression operand is expected. Raw scalars wrap to `lit`. */
|
|
433
|
+
type ExprLike = Expr | ScalarValue;
|
|
434
|
+
/** Wrap an {@link ExprLike} into an {@link Expr} (raw scalar → `lit`). */
|
|
435
|
+
declare function toExpr(x: ExprLike): Expr;
|
|
436
|
+
/**
|
|
437
|
+
* An immutable expression tree node with a chainable, pandas-familiar surface
|
|
438
|
+
* (spec §4). Every method returns a new `Expr`; the receiver is never mutated.
|
|
439
|
+
*/
|
|
440
|
+
declare class Expr {
|
|
441
|
+
/** The frozen AST node this expression wraps. */
|
|
442
|
+
readonly node: ExprNode;
|
|
443
|
+
/** @internal — construct via {@link col}/{@link lit} or a chained method. */
|
|
444
|
+
constructor(node: ExprNode);
|
|
445
|
+
add(other: ExprLike): Expr;
|
|
446
|
+
sub(other: ExprLike): Expr;
|
|
447
|
+
mul(other: ExprLike): Expr;
|
|
448
|
+
div(other: ExprLike): Expr;
|
|
449
|
+
mod(other: ExprLike): Expr;
|
|
450
|
+
neg(): Expr;
|
|
451
|
+
gt(other: ExprLike): Expr;
|
|
452
|
+
ge(other: ExprLike): Expr;
|
|
453
|
+
lt(other: ExprLike): Expr;
|
|
454
|
+
le(other: ExprLike): Expr;
|
|
455
|
+
eq(other: ExprLike): Expr;
|
|
456
|
+
ne(other: ExprLike): Expr;
|
|
457
|
+
and(other: ExprLike): Expr;
|
|
458
|
+
or(other: ExprLike): Expr;
|
|
459
|
+
not(): Expr;
|
|
460
|
+
isNull(): Expr;
|
|
461
|
+
/** `notNull` = `not(isNull)` (dtypes.md §4.5). */
|
|
462
|
+
notNull(): Expr;
|
|
463
|
+
fillNull(value: ScalarValue): Expr;
|
|
464
|
+
cast(to: DType): Expr;
|
|
465
|
+
sum(): Expr;
|
|
466
|
+
mean(): Expr;
|
|
467
|
+
min(): Expr;
|
|
468
|
+
max(): Expr;
|
|
469
|
+
count(): Expr;
|
|
470
|
+
nunique(): Expr;
|
|
471
|
+
std(): Expr;
|
|
472
|
+
var(): Expr;
|
|
473
|
+
first(): Expr;
|
|
474
|
+
last(): Expr;
|
|
475
|
+
/**
|
|
476
|
+
* dt accessor namespace for date32 / timestamp columns (dtypes.md §10, ADR-010).
|
|
477
|
+
* Returns a {@link DtProxy} with `.year()`, `.month()`, `.day()`, `.hour()`,
|
|
478
|
+
* `.minute()`, `.second()`, `.millisecond()`, `.weekday()`, `.dayOfYear()`, `.quarter()`.
|
|
479
|
+
* Each method returns an `i32` Expr.
|
|
480
|
+
*/
|
|
481
|
+
get dt(): DtProxy;
|
|
482
|
+
/**
|
|
483
|
+
* str namespace for `utf8` columns (dtypes.md §13).
|
|
484
|
+
* Returns a {@link StrProxy} with `.slice(start, end?)`.
|
|
485
|
+
* Throws a dtype error at compile time if the column is not `utf8`.
|
|
486
|
+
*/
|
|
487
|
+
get str(): StrProxy;
|
|
488
|
+
/** Readable, unambiguous rendering for `console.log` / error messages. */
|
|
489
|
+
toString(): string;
|
|
490
|
+
}
|
|
491
|
+
/**
|
|
492
|
+
* dt accessor proxy returned by `Expr.dt`. Every method produces an `i32` Expr
|
|
493
|
+
* extracting the named calendar field from the parent date32 / timestamp column.
|
|
494
|
+
*/
|
|
495
|
+
declare class DtProxy {
|
|
496
|
+
private readonly operand;
|
|
497
|
+
constructor(operand: Expr);
|
|
498
|
+
year(): Expr;
|
|
499
|
+
month(): Expr;
|
|
500
|
+
day(): Expr;
|
|
501
|
+
hour(): Expr;
|
|
502
|
+
minute(): Expr;
|
|
503
|
+
second(): Expr;
|
|
504
|
+
millisecond(): Expr;
|
|
505
|
+
weekday(): Expr;
|
|
506
|
+
dayOfYear(): Expr;
|
|
507
|
+
quarter(): Expr;
|
|
508
|
+
}
|
|
509
|
+
/**
|
|
510
|
+
* str accessor proxy returned by `Expr.str`. Provides string operations over
|
|
511
|
+
* `utf8` columns; dtypes.md §13. A dtype error is raised at compile time (not
|
|
512
|
+
* here) if the parent expression is not `utf8`.
|
|
513
|
+
*/
|
|
514
|
+
declare class StrProxy {
|
|
515
|
+
private readonly operand;
|
|
516
|
+
constructor(operand: Expr);
|
|
517
|
+
/**
|
|
518
|
+
* Substring via `JS String.prototype.slice` semantics (dtypes.md §13).
|
|
519
|
+
*
|
|
520
|
+
* - Negative `start`/`end`: count from the end of the string.
|
|
521
|
+
* - `end` omitted: slice to the end of the string.
|
|
522
|
+
* - Out-of-range indices clamp (same as JS).
|
|
523
|
+
* - Null rows propagate null.
|
|
524
|
+
* - UTF-16 code-unit indexing (same as every JS string API).
|
|
525
|
+
* **Surrogate-pair caveat:** a supplementary character (emoji, CJK extension, etc.)
|
|
526
|
+
* occupies two code units; `slice` may split a surrogate pair, producing an
|
|
527
|
+
* unpaired surrogate (well-defined but unusual in JS, not valid UTF-8).
|
|
528
|
+
* If you need grapheme-cluster or codepoint semantics, pre-process with JS before
|
|
529
|
+
* loading into the frame.
|
|
530
|
+
*/
|
|
531
|
+
slice(start: number, end?: number): Expr;
|
|
532
|
+
}
|
|
533
|
+
/** Reference the frame column named `name`. */
|
|
534
|
+
declare function col(name: string): Expr;
|
|
535
|
+
/**
|
|
536
|
+
* A scalar literal. Its dtype is normally inferred from the operand it is combined
|
|
537
|
+
* with (an integer numeric literal adopts an integer column's dtype, a fractional
|
|
538
|
+
* one triggers int→float widening — see `./dtypes.ts`). Pass `dtype` to pin it.
|
|
539
|
+
*/
|
|
540
|
+
declare function lit(value: ScalarValue, dtype?: DType): Expr;
|
|
541
|
+
|
|
542
|
+
/**
|
|
543
|
+
* Type & validity resolution (Phase 3, P3.1 deliverable §2; dtypes.md §3.1 + §5).
|
|
544
|
+
*
|
|
545
|
+
* `resolve(expr, schema)` type-checks an {@link Expr} and lowers it to a **typed IR**
|
|
546
|
+
* ({@link TExpr}) the compiler executes directly. It encodes:
|
|
547
|
+
*
|
|
548
|
+
* - result-dtype inference (arith lattice dtypes.md §3.1, comparison/boolean → `bool`,
|
|
549
|
+
* aggregation result matrix dtypes.md §4.3),
|
|
550
|
+
* - the single implicit conversion — **int→float widening in mixed arithmetic** —
|
|
551
|
+
* materialised as an explicit {@link TCast} node so the compiler just emits a cast
|
|
552
|
+
* kernel (dtypes.md §5 "cast-insertion points"),
|
|
553
|
+
* - literal typing: a bare numeric literal adopts the dtype of the operand it is
|
|
554
|
+
* combined with; a *fractional* literal against an integer column triggers the same
|
|
555
|
+
* widening (documented extension of the §3.1 rule to literal operands),
|
|
556
|
+
* - unsupported-mix errors naming **both** dtypes and the op (spec §4 ergonomics).
|
|
557
|
+
*
|
|
558
|
+
* Identity casts survive as `TCast{from===to}`; the compiler elides the kernel
|
|
559
|
+
* (dtypes.md §2). Range/validity nulling is a runtime concern handled by the cast /
|
|
560
|
+
* div / mod kernels, not here.
|
|
561
|
+
*/
|
|
562
|
+
|
|
563
|
+
/** The dtype view of a frame the type checker needs (a subset of `FrameView`). */
|
|
564
|
+
interface Schema {
|
|
565
|
+
/** Dtype of column `name`, or `undefined` if absent. */
|
|
566
|
+
dtypeOf(name: string): DType | undefined;
|
|
567
|
+
/** All column names (for nearest-match suggestions). */
|
|
568
|
+
columnNames(): readonly string[];
|
|
569
|
+
}
|
|
570
|
+
/** Build a {@link Schema} from a plain `name → dtype` record (handy in tests). */
|
|
571
|
+
declare function schemaOf(record: Readonly<Record<string, DType>>): Schema;
|
|
572
|
+
/** A type-resolved expression node. `dtype` is the node's result dtype. */
|
|
573
|
+
type TExpr = Readonly<{
|
|
574
|
+
kind: 'col';
|
|
575
|
+
name: string;
|
|
576
|
+
dtype: DType;
|
|
577
|
+
}> | Readonly<{
|
|
578
|
+
kind: 'lit';
|
|
579
|
+
value: ScalarValue;
|
|
580
|
+
dtype: DType;
|
|
581
|
+
}> | Readonly<{
|
|
582
|
+
kind: 'arith';
|
|
583
|
+
op: ArithOp;
|
|
584
|
+
dtype: DType;
|
|
585
|
+
left: TExpr;
|
|
586
|
+
right: TExpr;
|
|
587
|
+
}> | Readonly<{
|
|
588
|
+
kind: 'neg';
|
|
589
|
+
dtype: DType;
|
|
590
|
+
operand: TExpr;
|
|
591
|
+
}> | Readonly<{
|
|
592
|
+
kind: 'compare';
|
|
593
|
+
op: CompareOp;
|
|
594
|
+
dtype: 'bool';
|
|
595
|
+
operandDtype: DType;
|
|
596
|
+
left: TExpr;
|
|
597
|
+
right: TExpr;
|
|
598
|
+
}> | Readonly<{
|
|
599
|
+
kind: 'bool';
|
|
600
|
+
op: BoolOp;
|
|
601
|
+
dtype: 'bool';
|
|
602
|
+
left: TExpr;
|
|
603
|
+
right: TExpr;
|
|
604
|
+
}> | Readonly<{
|
|
605
|
+
kind: 'not';
|
|
606
|
+
dtype: 'bool';
|
|
607
|
+
operand: TExpr;
|
|
608
|
+
}> | Readonly<{
|
|
609
|
+
kind: 'isNull';
|
|
610
|
+
dtype: 'bool';
|
|
611
|
+
operand: TExpr;
|
|
612
|
+
}> | Readonly<{
|
|
613
|
+
kind: 'fillNull';
|
|
614
|
+
dtype: DType;
|
|
615
|
+
operand: TExpr;
|
|
616
|
+
value: ScalarValue;
|
|
617
|
+
}> | Readonly<{
|
|
618
|
+
kind: 'cast';
|
|
619
|
+
dtype: DType;
|
|
620
|
+
from: DType;
|
|
621
|
+
operand: TExpr;
|
|
622
|
+
}> | Readonly<{
|
|
623
|
+
kind: 'agg';
|
|
624
|
+
op: AggOp;
|
|
625
|
+
dtype: DType;
|
|
626
|
+
operandDtype: DType;
|
|
627
|
+
operand: TExpr;
|
|
628
|
+
}>
|
|
629
|
+
/** dt accessor: extract a calendar field (result dtype = 'i32', dtypes.md §10). */
|
|
630
|
+
| Readonly<{
|
|
631
|
+
kind: 'dt';
|
|
632
|
+
component: DtComponent;
|
|
633
|
+
dtype: 'i32';
|
|
634
|
+
operand: TExpr;
|
|
635
|
+
}>
|
|
636
|
+
/** str.slice: substring on dictionary values (result dtype = 'utf8', dtypes.md §13). */
|
|
637
|
+
| Readonly<{
|
|
638
|
+
kind: 'strSlice';
|
|
639
|
+
dtype: 'utf8';
|
|
640
|
+
operand: TExpr;
|
|
641
|
+
start: number;
|
|
642
|
+
end: number | undefined;
|
|
643
|
+
}>;
|
|
644
|
+
/** Resolve `expr` against `schema`, returning the typed IR. Throws {@link ExprError}. */
|
|
645
|
+
declare function resolve(expr: Expr, schema: Schema): TExpr;
|
|
646
|
+
/** Convenience: the top-level result dtype of `expr` (`'bool'` for predicates). */
|
|
647
|
+
declare function inferType(expr: Expr, schema: Schema): DType;
|
|
648
|
+
/** Result dtype of aggregation `op` over operand dtype `d`; throws if unsupported. */
|
|
649
|
+
declare function aggResult(op: AggOp, d: DType): DType;
|
|
650
|
+
|
|
651
|
+
/**
|
|
652
|
+
* The surface the expression compiler consumes from a frame (Phase 3, P3.1).
|
|
653
|
+
*
|
|
654
|
+
* The DataFrame layer (P3.2) implements {@link FrameView} and hands it to
|
|
655
|
+
* {@link compile}/{@link compileFilter}. It is deliberately tiny: column lookup by
|
|
656
|
+
* name, a row count, and the memory + kernel handles the compiler needs to allocate
|
|
657
|
+
* temporaries and dispatch kernels. Keeping it small keeps the compiler decoupled
|
|
658
|
+
* from the concrete DataFrame implementation.
|
|
659
|
+
*/
|
|
660
|
+
|
|
661
|
+
/**
|
|
662
|
+
* The loaded wasm instance's kernel exports. Every Phase-2 kernel (ABI §9) lives on
|
|
663
|
+
* the **same** instance as the Phase-1 memory core, so `memory`/`alloc`/`free` here
|
|
664
|
+
* are the identical bindings behind {@link MemoryContext.mod} — allocations made
|
|
665
|
+
* through the context are visible to these kernels (same linear memory).
|
|
666
|
+
*
|
|
667
|
+
* Kernel functions are reached by their ABI name; {@link callKernel} does the typed
|
|
668
|
+
* lookup so the rest of the compiler never touches the index signature directly.
|
|
669
|
+
*/
|
|
670
|
+
interface KernelWasm {
|
|
671
|
+
readonly memory: WebAssembly.Memory;
|
|
672
|
+
alloc(size: number): number;
|
|
673
|
+
free(ptr: number): void;
|
|
674
|
+
realloc(ptr: number, newSize: number): number;
|
|
675
|
+
mem_generation(): number;
|
|
676
|
+
}
|
|
677
|
+
/** A kernel export: flat C ABI, wasm value types in/out (ABI §5). */
|
|
678
|
+
type KernelFn = (...args: number[]) => number;
|
|
679
|
+
/** Look up and invoke kernel `name` on `wasm` (ABI §6 naming). */
|
|
680
|
+
declare function callKernel(wasm: KernelWasm, name: string, args: readonly number[]): number;
|
|
681
|
+
/**
|
|
682
|
+
* Everything the compiler needs from a frame to type-check, allocate, and run a plan.
|
|
683
|
+
* Implements {@link Schema} (via `dtypeOf`/`columnNames`) so it doubles as the type
|
|
684
|
+
* checker's schema.
|
|
685
|
+
*/
|
|
686
|
+
interface FrameView extends Schema {
|
|
687
|
+
/** Row count (`size`, dtypes.md §4.4). Every column has this length. */
|
|
688
|
+
readonly length: number;
|
|
689
|
+
/** Allocator + the single `viewOf` accessor over the wasm linear memory. */
|
|
690
|
+
readonly ctx: MemoryContext;
|
|
691
|
+
/** The kernel exports (same instance as `ctx.mod`, see {@link KernelWasm}). */
|
|
692
|
+
readonly wasm: KernelWasm;
|
|
693
|
+
/** Resolve column `name` to its {@link Column}, or `undefined` if absent. */
|
|
694
|
+
getColumn(name: string): Column | undefined;
|
|
695
|
+
}
|
|
696
|
+
|
|
697
|
+
/**
|
|
698
|
+
* JS-side dispatch stubs for the hash kernel family (Phase 2, Agent D).
|
|
699
|
+
*
|
|
700
|
+
* Thin wrappers over the raw WASM exports that implement the two retry
|
|
701
|
+
* protocols defined in wasm-abi.md §9 D (this is Phase 3's contract):
|
|
702
|
+
*
|
|
703
|
+
* 1. **HT-grow protocol** (`group_build`, `join_*`): if the kernel returns
|
|
704
|
+
* `-1` (hash table full), double `htCap`, re-zero the table, and re-call.
|
|
705
|
+
*
|
|
706
|
+
* 2. **Out-grow protocol** (`join_*`): if the kernel returns `n > outCap`,
|
|
707
|
+
* the caller should re-allocate `out_l_idx` and `out_r_idx` to `n` and
|
|
708
|
+
* re-call. The JS stubs here implement that loop transparently.
|
|
709
|
+
*
|
|
710
|
+
* Both retry paths are tested in `tests/kernels/hash/` (including forced tiny
|
|
711
|
+
* `htCap` / `outCap` variants).
|
|
712
|
+
*/
|
|
713
|
+
/**
|
|
714
|
+
* The subset of WASM exports consumed by the hash family stubs.
|
|
715
|
+
* The actual `WebAssembly.Instance.exports` object is cast to this at
|
|
716
|
+
* load time; all hash_dt, group_build, and join exports live here.
|
|
717
|
+
*/
|
|
718
|
+
interface HashExports {
|
|
719
|
+
/** `alloc(size) -> ptr` — 16-byte aligned, 0 on OOM (ABI §3). */
|
|
720
|
+
alloc(size: number): number;
|
|
721
|
+
/** `free(ptr)` — free(0) is a no-op (ABI §3). */
|
|
722
|
+
free(ptr: number): void;
|
|
723
|
+
/** Single linear memory shared by all kernels (ABI §2). */
|
|
724
|
+
readonly memory: WebAssembly.Memory;
|
|
725
|
+
/** `hash_i32(data, vp, out_hash, len) -> ()` */
|
|
726
|
+
hash_i32(data: number, vp: number, outHash: number, len: number): void;
|
|
727
|
+
/** `hash_u32(data, vp, out_hash, len) -> ()` */
|
|
728
|
+
hash_u32(data: number, vp: number, outHash: number, len: number): void;
|
|
729
|
+
/** `hash_f64(data, vp, out_hash, len) -> ()` */
|
|
730
|
+
hash_f64(data: number, vp: number, outHash: number, len: number): void;
|
|
731
|
+
/** `hash_f32(data, vp, out_hash, len) -> ()` */
|
|
732
|
+
hash_f32(data: number, vp: number, outHash: number, len: number): void;
|
|
733
|
+
/** `hash_i64(data, vp, out_hash, len) -> ()` — v2.3 i64 column hash. */
|
|
734
|
+
hash_i64(data: number, vp: number, outHash: number, len: number): void;
|
|
735
|
+
/**
|
|
736
|
+
* `hash_utf8_dict(offsets_ptr, bytes_ptr, dict_count, out_hash_ptr) -> ()` — ABI §12.
|
|
737
|
+
*
|
|
738
|
+
* For each dictionary slot `k in 0..dict_count`, hashes the raw UTF-8 bytes
|
|
739
|
+
* `bytes[offsets[k]..offsets[k+1])` into `out_hash[k]` (i64). No row validity
|
|
740
|
+
* is involved; row nullness is handled separately by the join validity bitmaps.
|
|
741
|
+
*/
|
|
742
|
+
hash_utf8_dict(offsetsPtr: number, bytesPtr: number, dictCount: number, outHashPtr: number): void;
|
|
743
|
+
/** `hash_combine(acc_hash, add_hash, len) -> ()` — in-place multi-key mix. */
|
|
744
|
+
hash_combine(accHash: number, addHash: number, len: number): void;
|
|
745
|
+
/**
|
|
746
|
+
* `group_build(hash_ptr, len, ht_ptr, ht_cap, out_group_ids) -> i32`
|
|
747
|
+
*
|
|
748
|
+
* Returns `group_count` (≥ 0) or `-1` if the HT is too small.
|
|
749
|
+
*/
|
|
750
|
+
group_build(hashPtr: number, len: number, htPtr: number, htCap: number, outGroupIds: number): number;
|
|
751
|
+
/**
|
|
752
|
+
* `join_hash_inner / join_hash_left(...) -> i32`
|
|
753
|
+
*
|
|
754
|
+
* Returns total pair count, or `-1` if HT too small.
|
|
755
|
+
*/
|
|
756
|
+
join_hash_inner(lhPtr: number, lVp: number, lLen: number, rhPtr: number, rVp: number, rLen: number, htPtr: number, htCap: number, outLIdx: number, outRIdx: number, outCap: number): number;
|
|
757
|
+
join_hash_left(lhPtr: number, lVp: number, lLen: number, rhPtr: number, rVp: number, rLen: number, htPtr: number, htCap: number, outLIdx: number, outRIdx: number, outCap: number): number;
|
|
758
|
+
}
|
|
759
|
+
|
|
760
|
+
/**
|
|
761
|
+
* DataFrame runtime: the wasm instance (allocator + memory + full kernel surface)
|
|
762
|
+
* bundled with its MemoryContext. Instantiates the binary directly (loadWasmModule only
|
|
763
|
+
* exposes the memory core) so the frame layer can reach every Phase-2 kernel by ABI name.
|
|
764
|
+
* `init()` installs a process-wide default so `DataFrame.fromColumns({...})` needs no plumbing.
|
|
765
|
+
*/
|
|
766
|
+
|
|
767
|
+
type FrameWasm = KernelWasm & HashExports;
|
|
768
|
+
interface DfRuntime {
|
|
769
|
+
readonly ctx: MemoryContext;
|
|
770
|
+
readonly wasm: FrameWasm;
|
|
771
|
+
}
|
|
772
|
+
declare function runtimeFromExports(exports: WebAssembly.Exports, simd: boolean): DfRuntime;
|
|
773
|
+
declare function loadRuntime(opts?: LoadOptions): Promise<DfRuntime>;
|
|
774
|
+
declare function init(opts?: LoadOptions): Promise<DfRuntime>;
|
|
775
|
+
declare function useRuntime(rt: DfRuntime): void;
|
|
776
|
+
declare function defaultRuntime(): DfRuntime;
|
|
777
|
+
|
|
778
|
+
/** Series — one named column, zero-copy over a frame's buffers (spec §4 `df.col('a')`).
|
|
779
|
+
* Borrows (does not own) its parent's column; valid while that frame lives. Read-only. */
|
|
780
|
+
|
|
781
|
+
/**
|
|
782
|
+
* Series.dt accessor proxy. Returned by `series.dt`; provides per-field
|
|
783
|
+
* methods that return a new Series (dtype=i32) with the extracted values.
|
|
784
|
+
*
|
|
785
|
+
* Rules (dtypes.md §10, ADR-010, ADR-012):
|
|
786
|
+
* - timestamp columns: all 9 fields are valid.
|
|
787
|
+
* - date32 columns: year/month/day/weekday/dayOfYear/quarter are valid;
|
|
788
|
+
* hour/minute/second/millisecond throw a TypeError naming the op.
|
|
789
|
+
* - tz metadata on a timestamp column gives local-time components (ADR-010).
|
|
790
|
+
*/
|
|
791
|
+
declare class SeriesDtProxy {
|
|
792
|
+
private readonly series;
|
|
793
|
+
constructor(series: Series);
|
|
794
|
+
private extract;
|
|
795
|
+
year(): Series;
|
|
796
|
+
month(): Series;
|
|
797
|
+
day(): Series;
|
|
798
|
+
hour(): Series;
|
|
799
|
+
minute(): Series;
|
|
800
|
+
second(): Series;
|
|
801
|
+
millisecond(): Series;
|
|
802
|
+
weekday(): Series;
|
|
803
|
+
dayOfYear(): Series;
|
|
804
|
+
quarter(): Series;
|
|
805
|
+
}
|
|
806
|
+
/**
|
|
807
|
+
* Series.str accessor proxy. Returned by `series.str`; provides string operations
|
|
808
|
+
* over `utf8` columns (dtypes.md §13). Throws TypeError if the column is not utf8.
|
|
809
|
+
*/
|
|
810
|
+
declare class SeriesStrProxy {
|
|
811
|
+
private readonly series;
|
|
812
|
+
constructor(series: Series);
|
|
813
|
+
/**
|
|
814
|
+
* Substring via `JS String.prototype.slice` semantics (dtypes.md §13).
|
|
815
|
+
*
|
|
816
|
+
* - Negative `start`/`end`: count from the end of the string.
|
|
817
|
+
* - `end` omitted: slice to the end of the string.
|
|
818
|
+
* - Out-of-range indices clamp (same as JS).
|
|
819
|
+
* - Null rows propagate null; the op never inspects null-row string values.
|
|
820
|
+
* - UTF-16 code-unit indexing.
|
|
821
|
+
* **Surrogate-pair caveat:** a supplementary character occupies two code units;
|
|
822
|
+
* `slice` may split a surrogate pair (well-defined in JS, unusual in UTF-8).
|
|
823
|
+
*
|
|
824
|
+
* Implementation: applied to dictionary values once (O(unique)), then indices
|
|
825
|
+
* remapped (O(rows)) — no per-row string work.
|
|
826
|
+
*
|
|
827
|
+
* Returns a new `utf8` Series.
|
|
828
|
+
*/
|
|
829
|
+
slice(start: number, end?: number): Series;
|
|
830
|
+
}
|
|
831
|
+
declare class Series {
|
|
832
|
+
readonly name: string;
|
|
833
|
+
readonly dtype: DType;
|
|
834
|
+
readonly length: number;
|
|
835
|
+
private readonly ctx;
|
|
836
|
+
private readonly column;
|
|
837
|
+
constructor(ctx: MemoryContext, name: string, column: Column);
|
|
838
|
+
get col(): Column;
|
|
839
|
+
toArray(): Cell[];
|
|
840
|
+
get(i: number): Cell;
|
|
841
|
+
values(): ColumnView;
|
|
842
|
+
/**
|
|
843
|
+
* dt accessor namespace for date32 / timestamp columns (dtypes.md §10, ADR-010, ADR-012).
|
|
844
|
+
* Returns a {@link SeriesDtProxy} with accessor methods for each calendar field.
|
|
845
|
+
* Throws TypeError for disallowed fields (e.g. hour on date32).
|
|
846
|
+
*/
|
|
847
|
+
get dt(): SeriesDtProxy;
|
|
848
|
+
/**
|
|
849
|
+
* str accessor namespace for `utf8` columns (dtypes.md §13).
|
|
850
|
+
* Returns a {@link SeriesStrProxy} with string methods.
|
|
851
|
+
* Throws TypeError if the column is not `utf8`.
|
|
852
|
+
*/
|
|
853
|
+
get str(): SeriesStrProxy;
|
|
854
|
+
[Symbol.iterator](): IterableIterator<Cell>;
|
|
855
|
+
toString(): string;
|
|
856
|
+
}
|
|
857
|
+
|
|
858
|
+
/**
|
|
859
|
+
* Row proxy for the lambda escape hatch (ADR-003): one reusable proxy object moved across
|
|
860
|
+
* rows (no per-row allocation), reading live viewOf buffers with memoized utf8 decode. This
|
|
861
|
+
* is the documented SLOW PATH — the expression API (filter/withColumn) is the fast path.
|
|
862
|
+
*/
|
|
863
|
+
|
|
864
|
+
type Row = Record<string, Cell>;
|
|
865
|
+
interface RowCursor {
|
|
866
|
+
at(i: number): Row;
|
|
867
|
+
}
|
|
868
|
+
|
|
869
|
+
/**
|
|
870
|
+
* GroupBy + agg (spec §4; ADR-005 hash grouping). Group: hash key columns (`hash_dt` +
|
|
871
|
+
* `hash_combine`) and assign first-occurrence ids via `group_build`; string keys hash their
|
|
872
|
+
* dictionary indices (no unification needed within one frame); null keys form one group
|
|
873
|
+
* (dtypes.md §4.5). Aggregate: each operand is materialised once over the frame then reduced
|
|
874
|
+
* per group by one JS scatter pass (no per-group boundary chatter). skipna + result-dtype per §4.3.
|
|
875
|
+
*/
|
|
876
|
+
|
|
877
|
+
type AggName = AggOp | 'size';
|
|
878
|
+
type AggRequest = AggName | AggName[] | Expr;
|
|
879
|
+
type AggSpec = Record<string, AggRequest>;
|
|
880
|
+
interface NamedColumn {
|
|
881
|
+
readonly name: string;
|
|
882
|
+
readonly col: Column;
|
|
883
|
+
}
|
|
884
|
+
interface GroupBySource extends FrameView {
|
|
885
|
+
readonly rt: DfRuntime;
|
|
886
|
+
buildResult(named: NamedColumn[]): DataFrame;
|
|
887
|
+
}
|
|
888
|
+
declare class GroupBy {
|
|
889
|
+
private readonly src;
|
|
890
|
+
private readonly keys;
|
|
891
|
+
constructor(src: GroupBySource, keys: string[]);
|
|
892
|
+
agg(spec: AggSpec): DataFrame;
|
|
893
|
+
private buildPlan;
|
|
894
|
+
private columnPlan;
|
|
895
|
+
private exprPlan;
|
|
896
|
+
}
|
|
897
|
+
|
|
898
|
+
/**
|
|
899
|
+
* Hash join (spec §4; ADR-005). Builds on the right, probes left, via `join_hash_inner`/
|
|
900
|
+
* `join_hash_left` (ABI §9 D; left emits r_idx=-1 → nulls). utf8 keys use the
|
|
901
|
+
* unification-free path (ABI §12): each side's dictionary bytes are hashed once via
|
|
902
|
+
* `hash_utf8_dict`, then row hashes are gathered via `gather_i64`; no JS dict unification
|
|
903
|
+
* is performed. bool widens to i32; a null in any key excludes the row. Output = all left
|
|
904
|
+
* columns + right non-key columns (colliding right names suffixed _right). Output utf8 key
|
|
905
|
+
* column reuses the left dictionary (ABI §12).
|
|
906
|
+
*/
|
|
907
|
+
|
|
908
|
+
type JoinHow = 'inner' | 'left';
|
|
909
|
+
interface JoinOptions {
|
|
910
|
+
on: string | string[];
|
|
911
|
+
how?: JoinHow;
|
|
912
|
+
}
|
|
913
|
+
type JoinSource = GroupBySource;
|
|
914
|
+
|
|
915
|
+
/**
|
|
916
|
+
* DataFrame — the public columnar frame (spec §4). Every op returns a NEW frame; buffers are
|
|
917
|
+
* shared zero-copy where no data changes (select/drop/untouched withColumn, head/tail/slice)
|
|
918
|
+
* and reference-counted ({@link OwnedColumn}) so `dispose()` is order-independent. Expression
|
|
919
|
+
* ops (filter/withColumn) run the P3.1 compiler (fast path); filterFn/mapFn are the ADR-003
|
|
920
|
+
* slow path. Use `scope(fn)` to dispose a batch of intermediates.
|
|
921
|
+
*/
|
|
922
|
+
|
|
923
|
+
interface FrameOptions {
|
|
924
|
+
readonly dtypes?: Readonly<Record<string, DType>>;
|
|
925
|
+
readonly runtime?: DfRuntime;
|
|
926
|
+
/**
|
|
927
|
+
* IANA timezone strings for `timestamp` columns (ADR-010 §10 tz metadata).
|
|
928
|
+
* Keys are column names; values are IANA tz strings (e.g. `"America/New_York"`).
|
|
929
|
+
* Applied only to columns whose dtype is `'timestamp'`; ignored for other dtypes.
|
|
930
|
+
* Used by Arrow/CSV IO layers to propagate tz metadata from the source format.
|
|
931
|
+
*/
|
|
932
|
+
readonly tzs?: Readonly<Record<string, string>>;
|
|
933
|
+
}
|
|
934
|
+
interface SortOptions {
|
|
935
|
+
readonly descending?: boolean | readonly boolean[];
|
|
936
|
+
}
|
|
937
|
+
interface WithColumnOptions {
|
|
938
|
+
readonly dtype?: DType;
|
|
939
|
+
}
|
|
940
|
+
declare class DataFrame implements FrameView, GroupBySource {
|
|
941
|
+
readonly length: number;
|
|
942
|
+
private readonly _rt;
|
|
943
|
+
private readonly entries;
|
|
944
|
+
private readonly byName;
|
|
945
|
+
private disposed;
|
|
946
|
+
private constructor();
|
|
947
|
+
get ctx(): MemoryContext;
|
|
948
|
+
get wasm(): KernelWasm;
|
|
949
|
+
get rt(): DfRuntime;
|
|
950
|
+
getColumn(name: string): Column | undefined;
|
|
951
|
+
dtypeOf(name: string): DType | undefined;
|
|
952
|
+
columnNames(): readonly string[];
|
|
953
|
+
buildResult(named: NamedColumn[]): DataFrame;
|
|
954
|
+
static fromColumns(cols: Readonly<Record<string, ColumnInput>>, opts?: FrameOptions): DataFrame;
|
|
955
|
+
static fromRecords(records: ReadonlyArray<Readonly<Record<string, Cell>>>, opts?: FrameOptions): DataFrame;
|
|
956
|
+
private static fromRoots;
|
|
957
|
+
/**
|
|
958
|
+
* @internal Adopt pre-built wasm Column objects directly into a DataFrame,
|
|
959
|
+
* bypassing the JS-array round-trip of {@link fromColumns}. Used by
|
|
960
|
+
* `fromArrow` for zero-copy column adoption (CP.1 ingest fast path).
|
|
961
|
+
*
|
|
962
|
+
* Every column in `named` must have `owned === true` and correct wasm pointers;
|
|
963
|
+
* ownership transfers to the returned DataFrame (and its OwnedColumn wrappers).
|
|
964
|
+
*/
|
|
965
|
+
static _adoptColumns(rt: DfRuntime, named: ReadonlyArray<{
|
|
966
|
+
name: string;
|
|
967
|
+
col: Column;
|
|
968
|
+
}>, length: number): DataFrame;
|
|
969
|
+
get shape(): readonly [number, number];
|
|
970
|
+
get columns(): readonly string[];
|
|
971
|
+
get dtypes(): Readonly<Record<string, DType>>;
|
|
972
|
+
col(name: string): Series;
|
|
973
|
+
select(names: readonly string[]): DataFrame;
|
|
974
|
+
drop(names: readonly string[]): DataFrame;
|
|
975
|
+
withColumn(name: string, value: Expr | ColumnInput, opts?: WithColumnOptions): DataFrame;
|
|
976
|
+
assign(name: string, value: Expr | ColumnInput, opts?: WithColumnOptions): DataFrame;
|
|
977
|
+
filter(predicate: Expr): DataFrame;
|
|
978
|
+
filterFn(fn: (row: Row) => boolean): DataFrame;
|
|
979
|
+
mapFn<T>(fn: (row: Row) => T): T[];
|
|
980
|
+
sortValues(by: string | readonly string[], opts?: SortOptions): DataFrame;
|
|
981
|
+
groupby(keys: string | readonly string[]): GroupBy;
|
|
982
|
+
join(other: DataFrame, opts: JoinOptions): DataFrame;
|
|
983
|
+
head(n?: number): DataFrame;
|
|
984
|
+
tail(n?: number): DataFrame;
|
|
985
|
+
slice(start: number, end?: number): DataFrame;
|
|
986
|
+
private sliceRange;
|
|
987
|
+
toColumns(): Record<string, Cell[]>;
|
|
988
|
+
toRecords(): Array<Record<string, Cell>>;
|
|
989
|
+
describe(): DataFrame;
|
|
990
|
+
dispose(): void;
|
|
991
|
+
toString(): string;
|
|
992
|
+
private entryOf;
|
|
993
|
+
private retainEntry;
|
|
994
|
+
private shareEntries;
|
|
995
|
+
private materializeColumn;
|
|
996
|
+
private gatherRows;
|
|
997
|
+
private render;
|
|
998
|
+
}
|
|
999
|
+
declare function scope<T>(fn: (track: <F extends DataFrame>(df: F) => F) => T): T;
|
|
1000
|
+
|
|
1001
|
+
export { callKernel as $, type AggName as A, type BoolOp as B, type Column as C, type DType as D, Expr as E, type FrameView as F, GroupBy as G, Series as H, SeriesStrProxy as I, type JoinHow as J, type KernelFn as K, type LoadOptions as L, type MemoryContext as M, type NamedColumn as N, type SortOptions as O, type StrOp as P, StrProxy as Q, type Row as R, type ScalarValue as S, type TExpr as T, type TypedArrayCtor as U, type ViewDType as V, type ViewOf as W, type WasmExports as X, type WasmMemoryModule as Y, type WithColumnOptions as Z, aggResult as _, type Cell as a, col as a0, columnToArray as a1, createColumn as a2, createMemoryContext as a3, createViewOf as a4, decodeDictionary as a5, decodeSlot as a6, decodeStats as a7, defaultRuntime as a8, detectSimd as a9, dtypeInfo as aa, freeColumn as ab, freeDictionary as ac, inferType as ad, init as ae, lit as af, loadRuntime as ag, loadWasmModule as ah, resolve as ai, runtimeFromExports as aj, schemaOf as ak, scope as al, sliceColumn as am, toExpr as an, unifyDictionaries as ao, useRuntime as ap, writeDictionary as aq, writeDictionaryFromRawBytes as ar, type DfRuntime as b, DataFrame as c, type FrameOptions as d, type AggOp as e, type AggRequest as f, type AggSpec as g, type ArithOp as h, type ColumnBuffer as i, type ColumnInput as j, type ColumnView as k, type CompareOp as l, DTYPES as m, type DTypeInfo as n, type DictUnifyResult as o, type Dictionary as p, DtProxy as q, type ExprLike as r, type ExprNode as s, type FrameWasm as t, type GroupBySource as u, type JoinOptions as v, type JoinSource as w, type KernelWasm as x, type RowCursor as y, type Schema as z };
|