@columnar-js/core 0.0.0-stage → 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/LICENSE ADDED
@@ -0,0 +1,17 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Columnar contributors
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
package/README.md CHANGED
@@ -1,3 +1,211 @@
1
- # Temporary Holding Version
1
+ # Why Columnar?
2
2
 
3
- This version is a temporary placeholder for this package. An operational version to replace this has been submitted for review and is awaiting a staged release.
3
+ I built Columnar because I’ve always been a little uneasy about what happens underneath seemingly simple data operations. Whenever I see a pipeline chaining filter, map and reduce, I find myself thinking about how many times we’re traversing the same data and how many intermediate arrays we’re creating along the way. For small datasets, that’s rarely a problem. The code is readable, it works, and there’s no reason to overthink it. But scale changes things. When you’re processing millions of values, particularly in a browser where you still want to maintain a smooth 60 FPS experience, those repeated passes and allocations start to matter.
4
+
5
+ I wanted a way to keep the convenience of working with composable data operations without making every developer worry about how they’re executed.
6
+
7
+ # What is Columnar?
8
+
9
+ **A typed-array-native data processing engine for JavaScript and TypeScript.**
10
+
11
+ Columnar keeps tabular data in contiguous typed-array buffers, compiles predicates once per execution, and runs filtered aggregations in a fused traversal. It is designed for application workloads that need fast in-process analytics without converting every row into an intermediate object.
12
+
13
+ ```ts
14
+ import { col, count, mean, sum, table } from '@columnar/core';
15
+
16
+ const vehicles = table({
17
+ timestamp: new Float64Array([0, 10, 20, 30]),
18
+ speed: new Float32Array([120, 80, 140, 110]),
19
+ rpm: new Uint16Array([3_100, 2_400, 4_200, 3_600]),
20
+ });
21
+
22
+ const result = vehicles
23
+ .query()
24
+ .filter(col('speed').gte(100).and(col('rpm').lt(4_000)))
25
+ .aggregate({
26
+ observations: count(),
27
+ totalSpeed: sum('speed'),
28
+ averageSpeed: mean('speed'),
29
+ })
30
+ .execute();
31
+
32
+ console.log(result);
33
+ // { observations: 2, totalSpeed: 230, averageSpeed: 115 }
34
+ ```
35
+
36
+ ## What it provides
37
+
38
+ - Typed-array-backed numeric columns with zero-copy typed-array ingestion.
39
+ - Dictionary-encoded string columns.
40
+ - Bit-packed validity maps for nullable values.
41
+ - Boolean columns stored as bytes.
42
+ - Composable predicates with `and`, `or`, and `not`.
43
+ - Fused filtering and aggregation without intermediate row objects or filtered tables.
44
+ - Neumaier compensated summation, online mean, and Welford population/sample variance.
45
+ - Chunk-at-a-time asynchronous aggregation with upstream backpressure.
46
+ - ESM output and TypeScript declarations, with no runtime dependencies.
47
+
48
+ ## Why Columnar
49
+
50
+ A hand-written loop is a good choice when an operation is fixed and the application needs direct control over every step. It can be very lean, but the application also owns the predicate logic, null handling, aggregation rules, composition and ongoing maintenance.
51
+
52
+ Columnar gives that work a reusable home. Describe the operation with predicates and aggregate expressions; the engine binds the columns, chooses an execution path and runs the work over the underlying buffers. Your application stays focused on what the data means instead of rebuilding the mechanics for every query.
53
+
54
+ The goal isn't to replace hand-written loops. It's to make efficient, composable data operations practical without making every caller write a small execution engine.
55
+
56
+ ## Install
57
+
58
+ ```sh
59
+ pnpm add @columnar/core
60
+ ```
61
+
62
+ ## Input
63
+
64
+ ### Column-oriented input
65
+
66
+ Use typed arrays for the direct columnar path. Typed-array inputs are retained by reference; the table and caller share their backing buffers.
67
+
68
+ ```ts
69
+ import { table } from '@columnar/core';
70
+
71
+ const dataset = table({
72
+ timestamp: new Float64Array([0, 10, 20]),
73
+ temperature: new Float32Array([21.5, 22.1, 21.9]),
74
+ sample: new Uint32Array([1, 2, 3]),
75
+ });
76
+ ```
77
+
78
+ ### Row-oriented input
79
+
80
+ Ordinary records are accepted and converted to column storage once at ingestion.
81
+
82
+ ```ts
83
+ const dataset = table([
84
+ { city: 'Pune', temperature: 28.5, active: true },
85
+ { city: 'Chennai', temperature: 31.2, active: false },
86
+ ]);
87
+ ```
88
+
89
+ Numeric arrays become `Float64Array` columns, booleans become byte columns, and strings become dictionary-encoded columns. `null` and `undefined` values are represented by a validity bitmap. A column containing only null values is represented as a nullable numeric column.
90
+
91
+ ## Queries
92
+
93
+ ### Filter and project
94
+
95
+ ```ts
96
+ const fastVehicles = dataset
97
+ .query()
98
+ .filter(col('temperature').gt(25))
99
+ .select('city', 'temperature')
100
+ .execute();
101
+ ```
102
+
103
+ `select` returns a table containing references to the selected columns. Filtering creates compact output buffers for the selected columns.
104
+
105
+ ### Aggregation
106
+
107
+ ```ts
108
+ import { count, max, mean, min, sum, variance } from '@columnar/core';
109
+
110
+ const summary = dataset
111
+ .query()
112
+ .filter(col('temperature').gte(20))
113
+ .aggregate({
114
+ samples: count(),
115
+ validTemperatures: count('temperature'),
116
+ total: sum('temperature'),
117
+ average: mean('temperature'),
118
+ minimum: min('temperature'),
119
+ maximum: max('temperature'),
120
+ populationVariance: variance('temperature'),
121
+ sampleVariance: variance('temperature', true),
122
+ })
123
+ .execute();
124
+ ```
125
+
126
+ Supported aggregates are `count`, `sum`, `mean`, `min`, `max`, and `variance`. Numeric aggregates skip nulls. `count()` counts rows; `count(column)` counts non-null values. Empty numeric aggregates return `null`. Variance is population variance by default; pass `true` for sample variance.
127
+
128
+ ### Predicates
129
+
130
+ ```ts
131
+ const predicate = col('temperature')
132
+ .gte(20)
133
+ .and(col('city').ne('Pune'))
134
+ .or(col('temperature').isNull());
135
+ ```
136
+
137
+ Available comparisons: `eq`, `ne`, `gt`, `gte`, `lt`, `lte`, `isNull`, and `isNotNull`. Comparisons involving null evaluate to false; use the explicit null predicates to test missing values.
138
+
139
+ ## Streaming aggregation
140
+
141
+ `aggregateStream` consumes an `AsyncIterable<Table>` one chunk at a time. It retains aggregate state, not prior chunks, and requests the next chunk only after processing the current one.
142
+
143
+ ```ts
144
+ import { aggregateStream, count, mean, sum, table } from '@columnar/core';
145
+
146
+ async function* readBatches() {
147
+ yield table({ temperature: new Float32Array([21.5, 27.2, 29.1]) });
148
+ yield table({ temperature: new Float32Array([24.8, 31.4, 26.3]) });
149
+ }
150
+
151
+ const result = await aggregateStream(readBatches(), {
152
+ rows: count(),
153
+ total: sum('temperature'),
154
+ average: mean('temperature'),
155
+ });
156
+ ```
157
+
158
+ An optional `where` predicate filters rows per chunk, and an optional `AbortSignal` cancels processing:
159
+
160
+ ```ts
161
+ const abortController = new AbortController();
162
+
163
+ const result = await aggregateStream(
164
+ readBatches(),
165
+ { average: mean('temperature') },
166
+ {
167
+ where: col('temperature').gt(25),
168
+ signal: abortController.signal,
169
+ },
170
+ );
171
+ ```
172
+
173
+ ## Memory model
174
+
175
+ | Input | Representation |
176
+ | ------------------- | --------------------------------------------------------- |
177
+ | Numeric typed array | Original typed array, retained by reference |
178
+ | Numeric JS array | `Float64Array` |
179
+ | Boolean values | `Uint8Array` |
180
+ | Strings | `Uint32Array` dictionary indices plus a string dictionary |
181
+ | Nullability | Optional `Uint32Array` validity bitmap (one bit per row) |
182
+
183
+ The direct typed-array path is zero-copy. Mutations to a caller-owned typed array are visible to the table. Do not mutate shared input buffers while a query is executing.
184
+
185
+ A fused aggregate query allocates aggregate state proportional to the number of requested aggregates. It does not allocate an intermediate result table. A filtered table query allocates a selection index buffer and compact output buffers. Streaming aggregation retains only the current input chunk and aggregate state.
186
+
187
+ ## Under the hood
188
+
189
+ - [Architecture](docs/architecture.md) — how tables, predicates and execution fit together.
190
+ - [Execution model](docs/execution-model.md) — predicate compilation, aggregation and streaming.
191
+ - [Memory model](docs/memory-model.md) — column buffers, validity maps and allocation boundaries.
192
+ - [Benchmark methodology](docs/benchmark-methodology.md) and [latest results](docs/benchmark-results.md).
193
+
194
+ ## Development
195
+
196
+ Requires Node.js 26.13.0 and pnpm 12.0.0.
197
+
198
+ ```sh
199
+ pnpm install
200
+ pnpm check
201
+ pnpm bench
202
+ pnpm bench -- --large
203
+ ```
204
+
205
+ `pnpm check` runs type checking, ESLint, Prettier validation, tests, and the package build. The benchmark compares Columnar's fused query with a hand-written typed-array loop and materializing JavaScript pipelines. The loop is a useful workload-specific reference; Columnar provides the reusable query model around filtering, null handling, aggregation and streaming. Run comparisons on the same machine and Node.js version.
206
+
207
+ Each example can be opened as a StackBlitz project after `@columnar/core` is published to npm.
208
+
209
+ ## License
210
+
211
+ MIT
@@ -0,0 +1,217 @@
1
+ type Scalar = number | string | boolean | null;
2
+ type NumericArray = Int8Array | Uint8Array | Int16Array | Uint16Array | Int32Array | Uint32Array | Float32Array | Float64Array;
3
+ type ColumnInput = NumericArray | readonly (Scalar | undefined)[];
4
+ type Row = Record<string, Scalar | undefined>;
5
+
6
+ declare const ColumnKindIntent: {
7
+ readonly Int8: "int8";
8
+ readonly Uint8: "uint8";
9
+ readonly Int16: "int16";
10
+ readonly Uint16: "uint16";
11
+ readonly Int32: "int32";
12
+ readonly Uint32: "uint32";
13
+ readonly Float32: "float32";
14
+ readonly Float64: "float64";
15
+ readonly Boolean: "boolean";
16
+ readonly Dictionary: "dictionary";
17
+ };
18
+ type ColumnKindIntent = (typeof ColumnKindIntent)[keyof typeof ColumnKindIntent];
19
+ declare const PredicateNodeIntent: {
20
+ readonly Comparison: "comparison";
21
+ readonly Null: "null";
22
+ readonly And: "and";
23
+ readonly Or: "or";
24
+ readonly Not: "not";
25
+ };
26
+ declare const ComparisonIntent: {
27
+ readonly Equal: "eq";
28
+ readonly NotEqual: "ne";
29
+ readonly GreaterThan: "gt";
30
+ readonly GreaterThanOrEqual: "gte";
31
+ readonly LessThan: "lt";
32
+ readonly LessThanOrEqual: "lte";
33
+ };
34
+ type ComparisonIntent = (typeof ComparisonIntent)[keyof typeof ComparisonIntent];
35
+ declare const AggregateIntent: {
36
+ readonly Sum: "sum";
37
+ readonly Mean: "mean";
38
+ readonly Min: "min";
39
+ readonly Max: "max";
40
+ readonly Count: "count";
41
+ readonly Variance: "variance";
42
+ };
43
+ type AggregateIntent = (typeof AggregateIntent)[keyof typeof AggregateIntent];
44
+
45
+ type ColumnKind = ColumnKindIntent;
46
+ declare abstract class Column {
47
+ abstract readonly kind: ColumnKind;
48
+ abstract readonly length: number;
49
+ abstract getUnchecked(index: number): Scalar;
50
+ abstract take(indices: Uint32Array): Column;
51
+ /** @internal Validates and copies a selection prefix. */
52
+ takeValidated(indices: Uint32Array, length: number): Column;
53
+ /** @internal Copies a previously validated selection into a compact column. */
54
+ takeUnchecked(indices: Uint32Array, length: number): Column;
55
+ abstract getNullCount(): number;
56
+ get(index: number): Scalar;
57
+ protected assertIndex(index: number): void;
58
+ }
59
+
60
+ declare class NumericColumn extends Column {
61
+ readonly kind: ColumnKind;
62
+ readonly length: number;
63
+ readonly values: NumericArray;
64
+ readonly validity: Uint32Array | undefined;
65
+ constructor(values: NumericArray, validity?: Uint32Array);
66
+ get(index: number): number | null;
67
+ getUnchecked(index: number): number | null;
68
+ take(indices: Uint32Array): NumericColumn;
69
+ takeValidated(indices: Uint32Array, length: number): NumericColumn;
70
+ takeUnchecked(indices: Uint32Array, length: number): NumericColumn;
71
+ getNullCount(): number;
72
+ }
73
+
74
+ declare class BooleanColumn extends Column {
75
+ readonly kind: "boolean";
76
+ readonly length: number;
77
+ readonly values: Uint8Array;
78
+ readonly validity: Uint32Array | undefined;
79
+ constructor(values: Uint8Array, validity?: Uint32Array);
80
+ get(index: number): boolean | null;
81
+ getUnchecked(index: number): boolean | null;
82
+ take(indices: Uint32Array): BooleanColumn;
83
+ takeValidated(indices: Uint32Array, length: number): BooleanColumn;
84
+ takeUnchecked(indices: Uint32Array, length: number): BooleanColumn;
85
+ getNullCount(): number;
86
+ }
87
+
88
+ declare class DictionaryColumn extends Column {
89
+ readonly kind: "dictionary";
90
+ readonly length: number;
91
+ readonly indices: Uint32Array;
92
+ readonly dictionary: readonly string[];
93
+ readonly validity: Uint32Array | undefined;
94
+ constructor(indices: Uint32Array, dictionary: readonly string[], validity?: Uint32Array);
95
+ get(index: number): string | null;
96
+ getUnchecked(index: number): string | null;
97
+ take(indices: Uint32Array): DictionaryColumn;
98
+ takeValidated(indices: Uint32Array, length: number): DictionaryColumn;
99
+ takeUnchecked(indices: Uint32Array, length: number): DictionaryColumn;
100
+ getNullCount(): number;
101
+ }
102
+
103
+ declare class ColumnarError extends Error {
104
+ constructor(message: string, options?: ErrorOptions);
105
+ }
106
+ declare class SchemaError extends ColumnarError {
107
+ }
108
+ declare class ExecutionError extends ColumnarError {
109
+ }
110
+ declare class AbortExecutionError extends ColumnarError {
111
+ constructor();
112
+ }
113
+
114
+ declare class QueryBuilder {
115
+ #private;
116
+ constructor(table: Table);
117
+ filter(predicate: Predicate): this;
118
+ select(...names: string[]): this;
119
+ aggregate(spec: AggregateSpec, options?: AggregateOptions): AggregateQuery;
120
+ execute(): Table;
121
+ }
122
+ declare class AggregateQuery {
123
+ #private;
124
+ constructor(table: Table, spec: AggregateSpec, predicates: readonly Predicate[], options: AggregateOptions);
125
+ execute(): Record<string, number | null>;
126
+ }
127
+
128
+ declare class Table {
129
+ #private;
130
+ readonly rowCount: number;
131
+ readonly columnNames: readonly string[];
132
+ private constructor();
133
+ static fromColumns(columns: Record<string, ColumnInput>): Table;
134
+ static fromRows(rows: readonly Row[]): Table;
135
+ column(name: string): Column;
136
+ hasColumn(name: string): boolean;
137
+ query(): QueryBuilder;
138
+ take(indices: Uint32Array): Table;
139
+ /** @internal Validates and materializes a prefix of selected rows. */
140
+ takeSelection(indices: Uint32Array, selectedCount: number, projection?: readonly string[]): Table;
141
+ /** @internal Materializes indices produced by this table's query executor. */
142
+ takeSelectionUnchecked(indices: Uint32Array, selectedCount: number, projection?: readonly string[]): Table;
143
+ private assertSelectionLength;
144
+ private materializeSelection;
145
+ selectColumns(names: readonly string[]): Table;
146
+ toRows(): Row[];
147
+ getColumnEntries(): IterableIterator<[string, Column]>;
148
+ }
149
+ declare function table(rows: readonly Row[]): Table;
150
+ declare function table(columns: Record<string, ColumnInput>): Table;
151
+
152
+ type ComparisonOperator = ComparisonIntent;
153
+ type PredicateNode = {
154
+ readonly type: typeof PredicateNodeIntent.Comparison;
155
+ readonly column: string;
156
+ readonly operator: ComparisonOperator;
157
+ readonly value: Scalar;
158
+ } | {
159
+ readonly type: typeof PredicateNodeIntent.Null;
160
+ readonly column: string;
161
+ readonly negate: boolean;
162
+ } | {
163
+ readonly type: typeof PredicateNodeIntent.And | typeof PredicateNodeIntent.Or;
164
+ readonly left: PredicateNode;
165
+ readonly right: PredicateNode;
166
+ } | {
167
+ readonly type: typeof PredicateNodeIntent.Not;
168
+ readonly expression: PredicateNode;
169
+ };
170
+
171
+ declare class Predicate {
172
+ readonly node: PredicateNode;
173
+ constructor(node: PredicateNode);
174
+ and(other: Predicate): Predicate;
175
+ or(other: Predicate): Predicate;
176
+ not(): Predicate;
177
+ bind(table: Table): (rowIndex: number) => boolean;
178
+ matches(table: Table, rowIndex: number): boolean;
179
+ }
180
+ declare class ColumnExpression {
181
+ readonly name: string;
182
+ constructor(name: string);
183
+ eq(value: Scalar): Predicate;
184
+ ne(value: Scalar): Predicate;
185
+ gt(value: number): Predicate;
186
+ gte(value: number): Predicate;
187
+ lt(value: number): Predicate;
188
+ lte(value: number): Predicate;
189
+ isNull(): Predicate;
190
+ isNotNull(): Predicate;
191
+ private compare;
192
+ }
193
+ declare function col(name: string): ColumnExpression;
194
+
195
+ type AggregateKind = AggregateIntent;
196
+ interface AggregateExpression {
197
+ readonly kind: AggregateKind;
198
+ readonly column?: string;
199
+ readonly sample?: boolean;
200
+ }
201
+ type AggregateSpec = Readonly<Record<string, AggregateExpression>>;
202
+ interface AggregateOptions {
203
+ readonly signal?: AbortSignal;
204
+ }
205
+
206
+ declare function aggregateStream(chunks: AsyncIterable<Table>, spec: AggregateSpec, options?: AggregateOptions & {
207
+ readonly where?: Predicate;
208
+ }): Promise<Record<string, number | null>>;
209
+
210
+ declare function sum(column: string): AggregateExpression;
211
+ declare function mean(column: string): AggregateExpression;
212
+ declare function min(column: string): AggregateExpression;
213
+ declare function max(column: string): AggregateExpression;
214
+ declare function count(column?: string): AggregateExpression;
215
+ declare function variance(column: string, sample?: boolean): AggregateExpression;
216
+
217
+ export { AbortExecutionError, type AggregateExpression, type AggregateOptions, type AggregateSpec, BooleanColumn, Column, ColumnExpression, type ColumnInput, type ColumnKind, ColumnarError, DictionaryColumn, ExecutionError, type NumericArray, NumericColumn, Predicate, type Row, type Scalar, SchemaError, Table, aggregateStream, col, count, max, mean, min, sum, table, variance };