@query-farm/vgi 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +134 -0
- package/README.md +494 -0
- package/dist/__dbg.test-d.d.ts +1 -0
- package/dist/arguments/argument-spec.d.ts +51 -0
- package/dist/arguments/arguments.d.ts +13 -0
- package/dist/arrow/codec/branded.d.ts +83 -0
- package/dist/arrow/codec/registry.d.ts +29 -0
- package/dist/arrow/codec/repr.d.ts +23 -0
- package/dist/arrow/codec/type-descriptors.d.ts +109 -0
- package/dist/arrow/impl-arrowjs/build.d.ts +19 -0
- package/dist/arrow/impl-arrowjs/canonical.d.ts +15 -0
- package/dist/arrow/impl-arrowjs/empty.d.ts +7 -0
- package/dist/arrow/impl-arrowjs/filter.d.ts +11 -0
- package/dist/arrow/impl-arrowjs/index.d.ts +11 -0
- package/dist/arrow/impl-arrowjs/ipc.d.ts +24 -0
- package/dist/arrow/impl-arrowjs/iterate.d.ts +39 -0
- package/dist/arrow/impl-arrowjs/project.d.ts +10 -0
- package/dist/arrow/impl-arrowjs/schema.d.ts +62 -0
- package/dist/arrow/impl-arrowjs/statistics.d.ts +13 -0
- package/dist/arrow/impl-flechette/build.d.ts +14 -0
- package/dist/arrow/impl-flechette/canonical.d.ts +18 -0
- package/dist/arrow/impl-flechette/empty.d.ts +9 -0
- package/dist/arrow/impl-flechette/filter.d.ts +10 -0
- package/dist/arrow/impl-flechette/index.d.ts +11 -0
- package/dist/arrow/impl-flechette/ipc.d.ts +33 -0
- package/dist/arrow/impl-flechette/iterate.d.ts +29 -0
- package/dist/arrow/impl-flechette/message-meta.d.ts +10 -0
- package/dist/arrow/impl-flechette/normalize-type.d.ts +11 -0
- package/dist/arrow/impl-flechette/project.d.ts +3 -0
- package/dist/arrow/impl-flechette/schema.d.ts +47 -0
- package/dist/arrow/impl-flechette/statistics.d.ts +13 -0
- package/dist/arrow/index.d.ts +11 -0
- package/dist/arrow/predicates.d.ts +47 -0
- package/dist/arrow/repr.test-d.d.ts +52 -0
- package/dist/arrow/schema-types.d.ts +71 -0
- package/dist/arrow/types.d.ts +57 -0
- package/dist/catalog/attach-option.d.ts +33 -0
- package/dist/catalog/attach-options.d.ts +30 -0
- package/dist/catalog/composite.d.ts +33 -0
- package/dist/catalog/descriptors.d.ts +154 -0
- package/dist/catalog/interface.d.ts +239 -0
- package/dist/catalog/read-only.d.ts +33 -0
- package/dist/client/client.d.ts +189 -0
- package/dist/client/errors.d.ts +26 -0
- package/dist/client/helpers.d.ts +12 -0
- package/dist/client/protocol.d.ts +16 -0
- package/dist/client/types.d.ts +125 -0
- package/dist/client-entry.d.ts +11 -0
- package/dist/client-entry.js +48 -0
- package/dist/client-entry.js.map +9 -0
- package/dist/codec/asd.d.ts +25 -0
- package/dist/crypto.d.ts +36 -0
- package/dist/errors.d.ts +24 -0
- package/dist/filter-pushdown/collector.d.ts +32 -0
- package/dist/filter-pushdown/deserialize.d.ts +16 -0
- package/dist/filter-pushdown/evaluate.d.ts +51 -0
- package/dist/filter-pushdown/index.d.ts +4 -0
- package/dist/filter-pushdown/types.d.ts +84 -0
- package/dist/functions/aggregate.d.ts +129 -0
- package/dist/functions/copy-from.d.ts +81 -0
- package/dist/functions/copy-to.d.ts +121 -0
- package/dist/functions/registry.d.ts +16 -0
- package/dist/functions/scalar.d.ts +98 -0
- package/dist/functions/storage-cf-do.d.ts +47 -0
- package/dist/functions/storage.d.ts +86 -0
- package/dist/functions/table-buffering.d.ts +99 -0
- package/dist/functions/table-in-out.d.ts +64 -0
- package/dist/functions/table.d.ts +118 -0
- package/dist/functions/types.d.ts +157 -0
- package/dist/generated/vgi-client.d.ts +466 -0
- package/dist/generated/vgi-protocol-schemas.d.ts +115 -0
- package/dist/index.core.d.ts +42 -0
- package/dist/index.d.ts +2 -0
- package/dist/index.js +10888 -0
- package/dist/index.js.map +85 -0
- package/dist/metadata/resolve.d.ts +3 -0
- package/dist/metadata/serialize.d.ts +12 -0
- package/dist/metadata/types.d.ts +47 -0
- package/dist/protocol/dispatch.d.ts +23 -0
- package/dist/protocol/handlers/aggregate.d.ts +3 -0
- package/dist/protocol/handlers/catalog/admin.d.ts +3 -0
- package/dist/protocol/handlers/catalog/index.d.ts +3 -0
- package/dist/protocol/handlers/catalog/index_methods.d.ts +3 -0
- package/dist/protocol/handlers/catalog/macro.d.ts +3 -0
- package/dist/protocol/handlers/catalog/shared.d.ts +70 -0
- package/dist/protocol/handlers/catalog/table.d.ts +3 -0
- package/dist/protocol/handlers/catalog/view.d.ts +3 -0
- package/dist/protocol/handlers/function.d.ts +8 -0
- package/dist/protocol/handlers/shared.d.ts +31 -0
- package/dist/protocol/handlers/table-buffering.d.ts +3 -0
- package/dist/protocol/serialize.d.ts +4 -0
- package/dist/protocol/serializers/arguments.d.ts +3 -0
- package/dist/protocol/serializers/bind.d.ts +9 -0
- package/dist/protocol/serializers/cardinality.d.ts +7 -0
- package/dist/protocol/serializers/init.d.ts +9 -0
- package/dist/protocol/serializers/shared.d.ts +12 -0
- package/dist/protocol/state-serializer.d.ts +23 -0
- package/dist/protocol/types.d.ts +129 -0
- package/dist/secrets/helpers.d.ts +15 -0
- package/dist/types.d.ts +52 -0
- package/dist/util/arrow/index.d.ts +1 -0
- package/dist/util/bytes.d.ts +2 -0
- package/dist/util/statistics.d.ts +7 -0
- package/dist/worker-cf-entry.d.ts +30 -0
- package/dist/worker-cf-entry.js +8767 -0
- package/dist/worker-cf-entry.js.map +74 -0
- package/dist/worker.d.ts +36 -0
- package/package.json +96 -0
- package/src/arguments/argument-spec.ts +200 -0
- package/src/arguments/arguments.ts +81 -0
- package/src/arrow/codec/branded.ts +206 -0
- package/src/arrow/codec/registry.ts +674 -0
- package/src/arrow/codec/repr.ts +181 -0
- package/src/arrow/codec/type-descriptors.ts +137 -0
- package/src/arrow/impl-arrowjs/build.ts +88 -0
- package/src/arrow/impl-arrowjs/canonical.ts +509 -0
- package/src/arrow/impl-arrowjs/empty.ts +72 -0
- package/src/arrow/impl-arrowjs/filter.ts +53 -0
- package/src/arrow/impl-arrowjs/index.ts +54 -0
- package/src/arrow/impl-arrowjs/ipc.ts +67 -0
- package/src/arrow/impl-arrowjs/iterate.ts +159 -0
- package/src/arrow/impl-arrowjs/project.ts +73 -0
- package/src/arrow/impl-arrowjs/schema.ts +256 -0
- package/src/arrow/impl-arrowjs/statistics.ts +185 -0
- package/src/arrow/impl-flechette/build.ts +65 -0
- package/src/arrow/impl-flechette/canonical.ts +289 -0
- package/src/arrow/impl-flechette/empty.ts +20 -0
- package/src/arrow/impl-flechette/filter.ts +42 -0
- package/src/arrow/impl-flechette/index.ts +54 -0
- package/src/arrow/impl-flechette/ipc.ts +110 -0
- package/src/arrow/impl-flechette/iterate.ts +121 -0
- package/src/arrow/impl-flechette/message-meta.ts +175 -0
- package/src/arrow/impl-flechette/normalize-type.ts +99 -0
- package/src/arrow/impl-flechette/project.ts +64 -0
- package/src/arrow/impl-flechette/schema.ts +202 -0
- package/src/arrow/impl-flechette/statistics.ts +198 -0
- package/src/arrow/index.ts +129 -0
- package/src/arrow/predicates.ts +66 -0
- package/src/arrow/repr.test-d.ts +189 -0
- package/src/arrow/schema-types.ts +213 -0
- package/src/arrow/types.ts +74 -0
- package/src/catalog/attach-option.ts +107 -0
- package/src/catalog/attach-options.ts +79 -0
- package/src/catalog/composite.ts +180 -0
- package/src/catalog/descriptors.ts +164 -0
- package/src/catalog/interface.ts +721 -0
- package/src/catalog/read-only.ts +909 -0
- package/src/client/client.ts +1348 -0
- package/src/client/errors.ts +69 -0
- package/src/client/helpers.ts +56 -0
- package/src/client/protocol.ts +33 -0
- package/src/client/types.ts +140 -0
- package/src/client-entry.ts +112 -0
- package/src/codec/asd.ts +147 -0
- package/src/crypto.ts +143 -0
- package/src/errors.ts +61 -0
- package/src/filter-pushdown/collector.ts +147 -0
- package/src/filter-pushdown/deserialize.ts +253 -0
- package/src/filter-pushdown/evaluate.ts +591 -0
- package/src/filter-pushdown/index.ts +28 -0
- package/src/filter-pushdown/types.ts +91 -0
- package/src/functions/aggregate.ts +384 -0
- package/src/functions/copy-from.ts +282 -0
- package/src/functions/copy-to.ts +319 -0
- package/src/functions/registry.ts +258 -0
- package/src/functions/scalar.ts +356 -0
- package/src/functions/storage-cf-do.ts +303 -0
- package/src/functions/storage.ts +521 -0
- package/src/functions/table-buffering.ts +562 -0
- package/src/functions/table-in-out.ts +373 -0
- package/src/functions/table.ts +521 -0
- package/src/functions/types.ts +192 -0
- package/src/generated/vgi-client.ts +560 -0
- package/src/generated/vgi-protocol-schemas.ts +864 -0
- package/src/index.core.ts +267 -0
- package/src/index.ts +11 -0
- package/src/metadata/resolve.ts +109 -0
- package/src/metadata/serialize.ts +242 -0
- package/src/metadata/types.ts +60 -0
- package/src/protocol/dispatch.ts +52 -0
- package/src/protocol/handlers/aggregate.ts +355 -0
- package/src/protocol/handlers/catalog/admin.ts +368 -0
- package/src/protocol/handlers/catalog/index.ts +25 -0
- package/src/protocol/handlers/catalog/index_methods.ts +65 -0
- package/src/protocol/handlers/catalog/macro.ts +114 -0
- package/src/protocol/handlers/catalog/shared.ts +291 -0
- package/src/protocol/handlers/catalog/table.ts +401 -0
- package/src/protocol/handlers/catalog/view.ts +119 -0
- package/src/protocol/handlers/function.ts +302 -0
- package/src/protocol/handlers/shared.ts +80 -0
- package/src/protocol/handlers/table-buffering.ts +207 -0
- package/src/protocol/serialize.ts +23 -0
- package/src/protocol/serializers/arguments.ts +160 -0
- package/src/protocol/serializers/bind.ts +212 -0
- package/src/protocol/serializers/cardinality.ts +45 -0
- package/src/protocol/serializers/init.ts +247 -0
- package/src/protocol/serializers/shared.ts +46 -0
- package/src/protocol/state-serializer.ts +181 -0
- package/src/protocol/types.ts +153 -0
- package/src/secrets/helpers.ts +70 -0
- package/src/types.ts +71 -0
- package/src/util/arrow/index.ts +23 -0
- package/src/util/bytes.ts +14 -0
- package/src/util/statistics.ts +40 -0
- package/src/worker-cf-entry.ts +172 -0
- package/src/worker.ts +212 -0
|
@@ -0,0 +1,185 @@
|
|
|
1
|
+
// Copyright 2025, 2026 Query Farm LLC - https://query.farm
|
|
2
|
+
// arrow-js impl of column-statistics serialization. Builds a sparse-union
|
|
3
|
+
// column manually with `makeData` + `typeIds` buffer + per-type children.
|
|
4
|
+
// This is the only spot in vgi-typescript that touches makeData directly,
|
|
5
|
+
// because flechette's union-column API takes a fundamentally different
|
|
6
|
+
// shape (typeIdForValue closure).
|
|
7
|
+
|
|
8
|
+
import {
|
|
9
|
+
Field,
|
|
10
|
+
Null,
|
|
11
|
+
makeData,
|
|
12
|
+
RecordBatch,
|
|
13
|
+
Struct,
|
|
14
|
+
SparseUnion,
|
|
15
|
+
Schema,
|
|
16
|
+
Utf8,
|
|
17
|
+
Bool,
|
|
18
|
+
Int64,
|
|
19
|
+
Uint64,
|
|
20
|
+
type DataType,
|
|
21
|
+
} from "@query-farm/apache-arrow";
|
|
22
|
+
import type {
|
|
23
|
+
VgiBatch,
|
|
24
|
+
VgiDataType,
|
|
25
|
+
} from "../types.js";
|
|
26
|
+
import { codecFor } from "../codec/registry.js";
|
|
27
|
+
import { writeCanonicalColumn } from "./canonical.js";
|
|
28
|
+
|
|
29
|
+
export interface ColumnStatistics {
|
|
30
|
+
columnName: string;
|
|
31
|
+
arrowType: VgiDataType;
|
|
32
|
+
min: any;
|
|
33
|
+
max: any;
|
|
34
|
+
hasNull: boolean;
|
|
35
|
+
hasNotNull: boolean;
|
|
36
|
+
distinctCount: bigint | number | null;
|
|
37
|
+
containsUnicode: boolean | null;
|
|
38
|
+
maxStringLength: bigint | number | null;
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
// Stable, backend-agnostic type key. Dispatch on the structural fields
|
|
42
|
+
// (typeId + unit + bitWidth + precision/scale + byteWidth) rather than
|
|
43
|
+
// `.toString()`, so the union-child grouping matches the flechette backend
|
|
44
|
+
// exactly and never collapses distinct types into one child.
|
|
45
|
+
function typeKey(t: VgiDataType): string {
|
|
46
|
+
const a = t as any;
|
|
47
|
+
return [
|
|
48
|
+
t.typeId,
|
|
49
|
+
a.unit ?? "",
|
|
50
|
+
a.bitWidth ?? "",
|
|
51
|
+
a.precision ?? "",
|
|
52
|
+
a.scale ?? "",
|
|
53
|
+
a.byteWidth ?? "",
|
|
54
|
+
].join(":");
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
/**
|
|
58
|
+
* Build an arrow-js column `Data` node for `type` from RICH values, routed
|
|
59
|
+
* through the codec (rich -> canonical) and the canonical writer — the same
|
|
60
|
+
* path the public column-build uses. This guarantees a date/timestamp/decimal
|
|
61
|
+
* min/max is represented identically to ordinary column data and identically
|
|
62
|
+
* across both backends.
|
|
63
|
+
*/
|
|
64
|
+
function buildTypedColumnData(values: any[], type: VgiDataType): any {
|
|
65
|
+
const a = type as DataType;
|
|
66
|
+
if (a.typeId === 1 /* Null */) {
|
|
67
|
+
return makeData({ type: a, length: values.length, nullCount: values.length });
|
|
68
|
+
}
|
|
69
|
+
const codec = codecFor(type);
|
|
70
|
+
const canonical = values.map((v) => codec.richToCanonical(v));
|
|
71
|
+
return writeCanonicalColumn(type, canonical);
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
export function buildStatisticsBatch(stats: ColumnStatistics[]): VgiBatch {
|
|
75
|
+
if (stats.length === 0) {
|
|
76
|
+
// Minimal empty batch: sparse_union with a single null child. Matches
|
|
77
|
+
// Python's empty serialization — DuckDB tolerates this and reads zero
|
|
78
|
+
// rows of stats.
|
|
79
|
+
const nullField = new Field("0", new Null(), true);
|
|
80
|
+
const unionType = new SparseUnion([0], [nullField]);
|
|
81
|
+
const minData = makeData({
|
|
82
|
+
type: unionType,
|
|
83
|
+
length: 0,
|
|
84
|
+
typeIds: new Int8Array(0),
|
|
85
|
+
children: [makeData({ type: new Null(), length: 0, nullCount: 0 })],
|
|
86
|
+
} as any);
|
|
87
|
+
const maxData = makeData({
|
|
88
|
+
type: unionType,
|
|
89
|
+
length: 0,
|
|
90
|
+
typeIds: new Int8Array(0),
|
|
91
|
+
children: [makeData({ type: new Null(), length: 0, nullCount: 0 })],
|
|
92
|
+
} as any);
|
|
93
|
+
return assemble([], minData, maxData, [], [], [], [], [], unionType) as unknown as VgiBatch;
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
// Group distinct Arrow types in stable insertion order to assign type codes.
|
|
97
|
+
const typeOrder: VgiDataType[] = [];
|
|
98
|
+
const typeIdMap = new Map<string, number>();
|
|
99
|
+
const rowTypeCodes: number[] = [];
|
|
100
|
+
for (const s of stats) {
|
|
101
|
+
const key = typeKey(s.arrowType);
|
|
102
|
+
let code = typeIdMap.get(key);
|
|
103
|
+
if (code === undefined) {
|
|
104
|
+
code = typeOrder.length;
|
|
105
|
+
typeIdMap.set(key, code);
|
|
106
|
+
typeOrder.push(s.arrowType);
|
|
107
|
+
}
|
|
108
|
+
rowTypeCodes.push(code);
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
const unionFields: Field[] = [];
|
|
112
|
+
const minChildren: any[] = [];
|
|
113
|
+
const maxChildren: any[] = [];
|
|
114
|
+
const typeIds: number[] = [];
|
|
115
|
+
|
|
116
|
+
for (const [code, arrowType] of typeOrder.entries()) {
|
|
117
|
+
unionFields.push(new Field(String(code), arrowType as DataType, true));
|
|
118
|
+
typeIds.push(code);
|
|
119
|
+
const minVals = stats.map((s, i) => (rowTypeCodes[i] === code ? s.min : null));
|
|
120
|
+
const maxVals = stats.map((s, i) => (rowTypeCodes[i] === code ? s.max : null));
|
|
121
|
+
minChildren.push(buildTypedColumnData(minVals, arrowType));
|
|
122
|
+
maxChildren.push(buildTypedColumnData(maxVals, arrowType));
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
const unionType = new SparseUnion(typeIds, unionFields);
|
|
126
|
+
const typeIdsBuf = Int8Array.from(rowTypeCodes);
|
|
127
|
+
|
|
128
|
+
const minData = makeData({
|
|
129
|
+
type: unionType, length: stats.length, typeIds: typeIdsBuf, children: minChildren,
|
|
130
|
+
} as any);
|
|
131
|
+
const maxData = makeData({
|
|
132
|
+
type: unionType, length: stats.length, typeIds: typeIdsBuf, children: maxChildren,
|
|
133
|
+
} as any);
|
|
134
|
+
|
|
135
|
+
return assemble(
|
|
136
|
+
stats.map((s) => s.columnName),
|
|
137
|
+
minData,
|
|
138
|
+
maxData,
|
|
139
|
+
stats.map((s) => s.hasNull),
|
|
140
|
+
stats.map((s) => s.hasNotNull),
|
|
141
|
+
stats.map((s) => (s.distinctCount == null ? null : BigInt(s.distinctCount))),
|
|
142
|
+
stats.map((s) => s.containsUnicode),
|
|
143
|
+
stats.map((s) => (s.maxStringLength == null ? null : BigInt(s.maxStringLength))),
|
|
144
|
+
unionType,
|
|
145
|
+
) as unknown as VgiBatch;
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
function assemble(
|
|
149
|
+
columnNames: string[],
|
|
150
|
+
minData: any,
|
|
151
|
+
maxData: any,
|
|
152
|
+
hasNull: boolean[],
|
|
153
|
+
hasNotNull: boolean[],
|
|
154
|
+
distinctCount: (bigint | null)[],
|
|
155
|
+
containsUnicode: (boolean | null)[],
|
|
156
|
+
maxStringLength: (bigint | null)[],
|
|
157
|
+
unionType: SparseUnion,
|
|
158
|
+
): RecordBatch {
|
|
159
|
+
const schema = new Schema([
|
|
160
|
+
new Field("column_name", new Utf8(), false),
|
|
161
|
+
new Field("min", unionType, true),
|
|
162
|
+
new Field("max", unionType, true),
|
|
163
|
+
new Field("has_null", new Bool(), false),
|
|
164
|
+
new Field("has_not_null", new Bool(), false),
|
|
165
|
+
new Field("distinct_count", new Int64(), true),
|
|
166
|
+
new Field("contains_unicode", new Bool(), true),
|
|
167
|
+
new Field("max_string_length", new Uint64(), true),
|
|
168
|
+
]);
|
|
169
|
+
|
|
170
|
+
const n = columnNames.length;
|
|
171
|
+
const children = [
|
|
172
|
+
buildTypedColumnData(columnNames, new Utf8() as unknown as VgiDataType),
|
|
173
|
+
minData,
|
|
174
|
+
maxData,
|
|
175
|
+
buildTypedColumnData(hasNull, new Bool() as unknown as VgiDataType),
|
|
176
|
+
buildTypedColumnData(hasNotNull, new Bool() as unknown as VgiDataType),
|
|
177
|
+
buildTypedColumnData(distinctCount, new Int64() as unknown as VgiDataType),
|
|
178
|
+
buildTypedColumnData(containsUnicode, new Bool() as unknown as VgiDataType),
|
|
179
|
+
buildTypedColumnData(maxStringLength, new Uint64() as unknown as VgiDataType),
|
|
180
|
+
];
|
|
181
|
+
|
|
182
|
+
const structType = new Struct(schema.fields);
|
|
183
|
+
const data = makeData({ type: structType, length: n, children, nullCount: 0 });
|
|
184
|
+
return new RecordBatch(schema, data);
|
|
185
|
+
}
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
// Copyright 2025, 2026 Query Farm LLC - https://query.farm
|
|
2
|
+
// Build batches from row objects or column arrays for the flechette backend.
|
|
3
|
+
//
|
|
4
|
+
// The public build path routes RICH author values through the codec registry
|
|
5
|
+
// (rich -> canonical) and then through the per-backend canonical writer
|
|
6
|
+
// (canonical -> flechette Column). All flechette-specific assembly now lives in
|
|
7
|
+
// ./canonical.ts; this file is just the public shape.
|
|
8
|
+
|
|
9
|
+
import {
|
|
10
|
+
columnFromArray as f_columnFromArray,
|
|
11
|
+
} from "@query-farm/flechette";
|
|
12
|
+
import type { VgiSchema, VgiBatch, VgiDataType, VgiColumnData } from "../types.js";
|
|
13
|
+
import { emptyBatch } from "./empty.js";
|
|
14
|
+
import { codecFor } from "../codec/registry.js";
|
|
15
|
+
import { writeCanonicalBatch } from "./canonical.js";
|
|
16
|
+
|
|
17
|
+
/**
|
|
18
|
+
* Build an opaque column-data handle from a JS array. Low-level pass-through
|
|
19
|
+
* for callers that already hold backend-native values; does NOT run the codec.
|
|
20
|
+
*/
|
|
21
|
+
export function columnFromArray(values: any[], type: VgiDataType): VgiColumnData {
|
|
22
|
+
return f_columnFromArray(values, type as any, {
|
|
23
|
+
useBigInt: true,
|
|
24
|
+
useBigIntTimestamp: true,
|
|
25
|
+
useDecimalInt: true,
|
|
26
|
+
}) as VgiColumnData;
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
/** Build a batch from row objects (values are RICH). */
|
|
30
|
+
export function batchFromRows(
|
|
31
|
+
rows: Record<string, any>[],
|
|
32
|
+
schema: VgiSchema,
|
|
33
|
+
): VgiBatch {
|
|
34
|
+
if (rows.length === 0) return emptyBatch(schema);
|
|
35
|
+
const columns: Record<string, any[]> = {};
|
|
36
|
+
for (const f of schema.fields) {
|
|
37
|
+
columns[f.name] = rows.map((r) => r[f.name] ?? null);
|
|
38
|
+
}
|
|
39
|
+
return batchFromColumns(columns, schema);
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
/**
|
|
43
|
+
* Build a batch from column arrays (values are RICH). Each column is converted
|
|
44
|
+
* rich -> canonical via the codec, then the canonical batch writer assembles
|
|
45
|
+
* the flechette Table (preserving per-field nullable + metadata).
|
|
46
|
+
*/
|
|
47
|
+
export function batchFromColumns(
|
|
48
|
+
columns: Record<string, any[]>,
|
|
49
|
+
schema: VgiSchema,
|
|
50
|
+
repr: "rich" | "raw" = "rich",
|
|
51
|
+
): VgiBatch {
|
|
52
|
+
let numRows = 0;
|
|
53
|
+
for (const f of schema.fields) {
|
|
54
|
+
if (columns[f.name]) { numRows = columns[f.name].length; break; }
|
|
55
|
+
}
|
|
56
|
+
const canonicalColumns: Record<string, unknown[]> = {};
|
|
57
|
+
for (const f of schema.fields) {
|
|
58
|
+
const values = columns[f.name];
|
|
59
|
+
if (!values) continue;
|
|
60
|
+
const codec = codecFor(f.type as VgiDataType);
|
|
61
|
+
const toCanon = repr === "raw" ? codec.rawToCanonical : codec.richToCanonical;
|
|
62
|
+
canonicalColumns[f.name] = values.map((v) => toCanon.call(codec, v));
|
|
63
|
+
}
|
|
64
|
+
return writeCanonicalBatch(canonicalColumns, schema, numRows);
|
|
65
|
+
}
|
|
@@ -0,0 +1,289 @@
|
|
|
1
|
+
// Copyright 2025, 2026 Query Farm LLC - https://query.farm
|
|
2
|
+
//
|
|
3
|
+
// Canonical Arrow I/O for the flechette backend.
|
|
4
|
+
//
|
|
5
|
+
// The ONLY place in the flechette backend that knows flechette-specific
|
|
6
|
+
// build/read details. Produces and consumes the SAME canonical representation
|
|
7
|
+
// as impl-arrowjs/canonical.ts (see ../codec/registry.ts), so both backends
|
|
8
|
+
// agree byte-for-byte.
|
|
9
|
+
//
|
|
10
|
+
// flechette's `columnFromArray` already handles the complex cases (List, Map,
|
|
11
|
+
// Struct, Decimal, Timestamp[ns]) when given the right options, so the WRITE
|
|
12
|
+
// side is mostly delegation. The one wrinkle is Date: flechette stores a Date
|
|
13
|
+
// column correctly only when fed JS `Date` objects, but our canonical date32 is
|
|
14
|
+
// a day-number and date64 is ms-bigint — so we convert canonical -> Date here.
|
|
15
|
+
//
|
|
16
|
+
// On the READ side, flechette's Column/Data `.at()` already decodes every
|
|
17
|
+
// scalar to its canonical JS value LOSSLESSLY (timestamp/duration/time64 ->
|
|
18
|
+
// bigint, decimal -> unscaled bigint with useDecimalInt) — UNLIKE arrow-js,
|
|
19
|
+
// whose Vector.get() is lossy for timestamp. The only mismatch is Date, which
|
|
20
|
+
// `.at()` returns as epoch-ms; we convert that to days (date32) / ms-bigint
|
|
21
|
+
// (date64). Composites recurse so nested Date children are converted too.
|
|
22
|
+
|
|
23
|
+
import {
|
|
24
|
+
columnFromArray as f_columnFromArray,
|
|
25
|
+
field as f_field,
|
|
26
|
+
Table,
|
|
27
|
+
type Column,
|
|
28
|
+
} from "@query-farm/flechette";
|
|
29
|
+
import type { VgiSchema, VgiBatch, VgiDataType, VgiColumnData, TaggedUnion } from "../types.js";
|
|
30
|
+
import { toFlechetteType } from "./normalize-type.js";
|
|
31
|
+
|
|
32
|
+
const MS_PER_DAY = 86_400_000;
|
|
33
|
+
|
|
34
|
+
const COLUMN_OPTS = {
|
|
35
|
+
useBigInt: true,
|
|
36
|
+
useBigIntTimestamp: true,
|
|
37
|
+
useDecimalInt: true,
|
|
38
|
+
} as const;
|
|
39
|
+
|
|
40
|
+
// ===========================================================================
|
|
41
|
+
// WRITE: canonical[] -> flechette Column
|
|
42
|
+
// ===========================================================================
|
|
43
|
+
|
|
44
|
+
/** Build a flechette `Column` from an array of CANONICAL values for `type`. */
|
|
45
|
+
export function writeCanonicalColumn(
|
|
46
|
+
type: VgiDataType,
|
|
47
|
+
canonical: unknown[],
|
|
48
|
+
): VgiColumnData {
|
|
49
|
+
const ftype = toFlechetteType(type);
|
|
50
|
+
return f_columnFromArray(prepareForFlechette(type, canonical), ftype, COLUMN_OPTS) as VgiColumnData;
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
/**
|
|
54
|
+
* Build a whole batch (Table) from canonical column arrays. Kept here so the
|
|
55
|
+
* per-field `nullable` flag and metadata are preserved on the output schema
|
|
56
|
+
* (tableFromColumns would drop them, which the C++ extension rejects).
|
|
57
|
+
*/
|
|
58
|
+
export function writeCanonicalBatch(
|
|
59
|
+
columns: Record<string, unknown[]>,
|
|
60
|
+
schema: VgiSchema,
|
|
61
|
+
numRows: number,
|
|
62
|
+
): VgiBatch {
|
|
63
|
+
const childCols: Column<any>[] = [];
|
|
64
|
+
const fields: any[] = [];
|
|
65
|
+
for (const f of schema.fields) {
|
|
66
|
+
const ftype = toFlechetteType(f.type);
|
|
67
|
+
const values = columns[f.name];
|
|
68
|
+
const col: Column<any> = values
|
|
69
|
+
? f_columnFromArray(prepareForFlechette(f.type, values), ftype, COLUMN_OPTS)
|
|
70
|
+
: f_columnFromArray(new Array(numRows).fill(null), ftype, COLUMN_OPTS);
|
|
71
|
+
childCols.push(col);
|
|
72
|
+
const nullable = (f as any).nullable ?? true;
|
|
73
|
+
const metadata = (f as any).metadata ?? null;
|
|
74
|
+
fields.push(f_field(f.name, col.type as any, nullable, metadata));
|
|
75
|
+
}
|
|
76
|
+
const flechSchema = {
|
|
77
|
+
version: 5,
|
|
78
|
+
endianness: 0,
|
|
79
|
+
fields,
|
|
80
|
+
metadata: (schema as any).metadata ?? null,
|
|
81
|
+
};
|
|
82
|
+
return new Table(flechSchema as any, childCols) as unknown as VgiBatch;
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
/**
|
|
86
|
+
* Convert canonical values into the shape flechette's columnFromArray needs.
|
|
87
|
+
* Date columns: canonical day-number / ms-bigint -> JS Date. Map columns:
|
|
88
|
+
* canonical Array<[k,v]> -> Map. Struct/List recurse. Everything else passes
|
|
89
|
+
* through.
|
|
90
|
+
*/
|
|
91
|
+
export function prepareForFlechette(type: VgiDataType, values: unknown[]): unknown[] {
|
|
92
|
+
const tid = type.typeId;
|
|
93
|
+
if (tid === 8) {
|
|
94
|
+
const isDay = (type as any).unit === 0;
|
|
95
|
+
return values.map((v) => {
|
|
96
|
+
if (v == null) return null;
|
|
97
|
+
if (v instanceof Date) return v;
|
|
98
|
+
const n = typeof v === "bigint" ? Number(v) : (v as number);
|
|
99
|
+
return new Date(isDay ? n * MS_PER_DAY : n);
|
|
100
|
+
});
|
|
101
|
+
}
|
|
102
|
+
if (tid === 17) {
|
|
103
|
+
return values.map((v) => {
|
|
104
|
+
if (v == null) return null;
|
|
105
|
+
if (v instanceof Map) return v;
|
|
106
|
+
if (Array.isArray(v)) return new Map(v as Array<[unknown, unknown]>);
|
|
107
|
+
if (typeof v === "object") return new Map(Object.entries(v as any));
|
|
108
|
+
return v;
|
|
109
|
+
});
|
|
110
|
+
}
|
|
111
|
+
if (tid === 13) {
|
|
112
|
+
const children = (type as any).children as Array<{ name: string; type: VgiDataType }>;
|
|
113
|
+
if (children.some((c) => needsPrepare(c.type))) {
|
|
114
|
+
return values.map((row) => {
|
|
115
|
+
if (row == null) return null;
|
|
116
|
+
const out: Record<string, unknown> = {};
|
|
117
|
+
for (const c of children) {
|
|
118
|
+
out[c.name] = prepareForFlechette(c.type, [(row as any)[c.name] ?? null])[0];
|
|
119
|
+
}
|
|
120
|
+
return out;
|
|
121
|
+
});
|
|
122
|
+
}
|
|
123
|
+
return values;
|
|
124
|
+
}
|
|
125
|
+
if (tid === 12 || tid === 16) {
|
|
126
|
+
const childType = (type as any).children?.[0]?.type as VgiDataType;
|
|
127
|
+
if (childType && needsPrepare(childType)) {
|
|
128
|
+
return values.map((row) => {
|
|
129
|
+
if (row == null) return null;
|
|
130
|
+
const items = Array.isArray(row) ? row : Array.from(row as Iterable<unknown>);
|
|
131
|
+
return prepareForFlechette(childType, items);
|
|
132
|
+
});
|
|
133
|
+
}
|
|
134
|
+
return values;
|
|
135
|
+
}
|
|
136
|
+
return values;
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
function needsPrepare(type: VgiDataType): boolean {
|
|
140
|
+
const tid = type.typeId;
|
|
141
|
+
if (tid === 8 || tid === 17) return true;
|
|
142
|
+
if (tid === 13) return ((type as any).children ?? []).some((c: any) => needsPrepare(c.type));
|
|
143
|
+
if (tid === 12 || tid === 16) {
|
|
144
|
+
const ct = (type as any).children?.[0]?.type;
|
|
145
|
+
return ct ? needsPrepare(ct) : false;
|
|
146
|
+
}
|
|
147
|
+
return false;
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
// ===========================================================================
|
|
151
|
+
// READ: flechette column + index -> canonical value
|
|
152
|
+
// ===========================================================================
|
|
153
|
+
|
|
154
|
+
/** Read a single CANONICAL value at `index` from a flechette Column. */
|
|
155
|
+
export function readCanonicalValue(
|
|
156
|
+
type: VgiDataType,
|
|
157
|
+
column: unknown,
|
|
158
|
+
index: number,
|
|
159
|
+
): unknown {
|
|
160
|
+
const tid = type.typeId;
|
|
161
|
+
// Struct and Union must be read column-based: flechette's `.at()` collapses a
|
|
162
|
+
// union to only its member value (dropping the discriminator), and a struct
|
|
163
|
+
// wrapping a union would inherit that loss. Reading the child Data nodes
|
|
164
|
+
// directly preserves the union tag for any nesting depth.
|
|
165
|
+
if (tid === 13) return readStructColumn(type, column, index);
|
|
166
|
+
if (tid === 14) return readUnionColumn(type, column, index);
|
|
167
|
+
const at = (column as any)?.at?.(index);
|
|
168
|
+
return canonicalize(type, at);
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
/** Child Data nodes for a flechette Column/Data. A top-level Column exposes
|
|
172
|
+
* `.data[0]`; a nested child is already a Data node carrying `.children`. */
|
|
173
|
+
function childData(column: any): any {
|
|
174
|
+
return column?.children ? column : column?.data?.[0];
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
/** True when row `index` is null per the Data node's validity bitmap. */
|
|
178
|
+
function isNullAt(data: any, index: number): boolean {
|
|
179
|
+
if ((data?.nullCount ?? 0) === 0) return false;
|
|
180
|
+
const validity: Uint8Array | undefined = data?.validity;
|
|
181
|
+
if (!validity || validity.length === 0) return false;
|
|
182
|
+
return ((validity[index >> 3] >> (index & 7)) & 1) === 0;
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
/** Read a struct column-based so nested union children keep their tags. */
|
|
186
|
+
function readStructColumn(
|
|
187
|
+
type: VgiDataType,
|
|
188
|
+
column: unknown,
|
|
189
|
+
index: number,
|
|
190
|
+
): Record<string, unknown> | null {
|
|
191
|
+
const col: any = column;
|
|
192
|
+
const data = childData(col);
|
|
193
|
+
if (!data?.children) {
|
|
194
|
+
// No structural access: fall back to value-based canonicalization.
|
|
195
|
+
return canonicalize(type, col?.at?.(index)) as Record<string, unknown> | null;
|
|
196
|
+
}
|
|
197
|
+
// A null struct row has no member values to read.
|
|
198
|
+
if (isNullAt(data, index)) return null;
|
|
199
|
+
const children = (type as any).children as Array<{ name: string; type: VgiDataType }>;
|
|
200
|
+
const out: Record<string, unknown> = {};
|
|
201
|
+
for (let c = 0; c < children.length; c++) {
|
|
202
|
+
out[children[c].name] = readCanonicalValue(children[c].type, data.children[c], index);
|
|
203
|
+
}
|
|
204
|
+
return out;
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
/** Read a union column-based into a TaggedUnion, recovering the active tag. */
|
|
208
|
+
function readUnionColumn(
|
|
209
|
+
type: VgiDataType,
|
|
210
|
+
column: unknown,
|
|
211
|
+
index: number,
|
|
212
|
+
): TaggedUnion {
|
|
213
|
+
const children = (type as any).children as Array<{ name: string; type: VgiDataType }>;
|
|
214
|
+
const data = childData(column);
|
|
215
|
+
if (data && isNullAt(data, index)) return { tag: null, value: null };
|
|
216
|
+
// Per-row type code -> child index. DuckDB emits SPARSE unions: each child is
|
|
217
|
+
// full-length and aligned with the parent row.
|
|
218
|
+
const typeIds: ArrayLike<number> | undefined = data?.typeIds;
|
|
219
|
+
const tcode = typeIds ? typeIds[index] : children.length > 0 ? 0 : -1;
|
|
220
|
+
const map = (type as any).typeMap as Record<number, number> | undefined;
|
|
221
|
+
const childIdx = map && map[tcode] !== undefined ? map[tcode] : tcode;
|
|
222
|
+
const childField = children[childIdx];
|
|
223
|
+
if (!childField || !data?.children) return { tag: null, value: null };
|
|
224
|
+
// Dense unions remap the row via offsets; sparse unions share the row index.
|
|
225
|
+
const offsets: ArrayLike<number> | undefined = data?.offsets;
|
|
226
|
+
const isDense = (type as any).mode === 1;
|
|
227
|
+
const childRow = isDense && offsets ? offsets[index] : index;
|
|
228
|
+
const value = readCanonicalValue(childField.type, data.children[childIdx], childRow);
|
|
229
|
+
return { tag: childField.name, value };
|
|
230
|
+
}
|
|
231
|
+
|
|
232
|
+
/**
|
|
233
|
+
* Normalize a flechette-decoded value (from Column/Data `.at()`) into the
|
|
234
|
+
* canonical representation. `.at()` already produces canonical for every
|
|
235
|
+
* scalar except Date; composites recurse so nested Dates are handled.
|
|
236
|
+
*/
|
|
237
|
+
function canonicalize(type: VgiDataType, value: unknown): unknown {
|
|
238
|
+
if (value === null || value === undefined) return null;
|
|
239
|
+
const tid = type.typeId;
|
|
240
|
+
switch (tid) {
|
|
241
|
+
case 8: { // Date: .at() returns epoch-ms number
|
|
242
|
+
const ms = value instanceof Date ? value.getTime()
|
|
243
|
+
: typeof value === "bigint" ? Number(value) : (value as number);
|
|
244
|
+
if ((type as any).unit === 0) return Math.round(ms / MS_PER_DAY); // date32 -> days
|
|
245
|
+
return BigInt(ms); // date64 -> ms bigint
|
|
246
|
+
}
|
|
247
|
+
case 4: // Binary
|
|
248
|
+
case 19: // LargeBinary
|
|
249
|
+
case 15: // FixedSizeBinary
|
|
250
|
+
return toUint8(value);
|
|
251
|
+
case 13: { // Struct -> { field: canonical }
|
|
252
|
+
const children = (type as any).children as Array<{ name: string; type: VgiDataType }>;
|
|
253
|
+
const obj = value as any;
|
|
254
|
+
const out: Record<string, unknown> = {};
|
|
255
|
+
for (const c of children) out[c.name] = canonicalize(c.type, obj[c.name] ?? null);
|
|
256
|
+
return out;
|
|
257
|
+
}
|
|
258
|
+
case 12: // List
|
|
259
|
+
case 16: { // FixedSizeList -> canonical[]
|
|
260
|
+
const childType = (type as any).children[0].type as VgiDataType;
|
|
261
|
+
const arr = Array.isArray(value) ? value : Array.from(value as Iterable<unknown>);
|
|
262
|
+
return arr.map((v) => canonicalize(childType, v));
|
|
263
|
+
}
|
|
264
|
+
case 17: { // Map -> Array<[k,v]>
|
|
265
|
+
const entries = (type as any).children[0].type.children;
|
|
266
|
+
const keyType = entries[0].type as VgiDataType;
|
|
267
|
+
const valueType = entries[1].type as VgiDataType;
|
|
268
|
+
const pairs: Array<[unknown, unknown]> = value instanceof Map ? Array.from(value.entries())
|
|
269
|
+
: Array.isArray(value) ? (value as Array<[unknown, unknown]>)
|
|
270
|
+
: Object.entries(value as Record<string, unknown>);
|
|
271
|
+
return pairs.map(([k, v]) => [canonicalize(keyType, k), canonicalize(valueType, v)]);
|
|
272
|
+
}
|
|
273
|
+
case -1: // Dictionary -> decoded value's canonical
|
|
274
|
+
return canonicalize((type as any).dictionary as VgiDataType, value);
|
|
275
|
+
default:
|
|
276
|
+
// bool/int/float/utf8/decimal/time/timestamp/duration: already canonical.
|
|
277
|
+
return value;
|
|
278
|
+
}
|
|
279
|
+
}
|
|
280
|
+
|
|
281
|
+
function toUint8(v: any): Uint8Array | null {
|
|
282
|
+
if (v == null) return null;
|
|
283
|
+
if (v instanceof Uint8Array) return v;
|
|
284
|
+
if (ArrayBuffer.isView(v)) {
|
|
285
|
+
const view = v as ArrayBufferView;
|
|
286
|
+
return new Uint8Array(view.buffer, view.byteOffset, view.byteLength);
|
|
287
|
+
}
|
|
288
|
+
return v;
|
|
289
|
+
}
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
// Copyright 2025, 2026 Query Farm LLC - https://query.farm
|
|
2
|
+
// Empty (0-row) batch construction for the flechette backend.
|
|
3
|
+
|
|
4
|
+
import { columnFromArray, tableFromColumns } from "@query-farm/flechette";
|
|
5
|
+
import type { VgiSchema, VgiBatch } from "../types.js";
|
|
6
|
+
|
|
7
|
+
/**
|
|
8
|
+
* Create an empty (0-row) batch with the given schema.
|
|
9
|
+
*
|
|
10
|
+
* For each field we build a 0-length column of the matching type. flechette
|
|
11
|
+
* handles complex types (List, Map, Struct, Decimal) without the manual
|
|
12
|
+
* makeData scaffolding the arrow-js backend needs.
|
|
13
|
+
*/
|
|
14
|
+
export function emptyBatch(schema: VgiSchema): VgiBatch {
|
|
15
|
+
const cols: Record<string, ReturnType<typeof columnFromArray>> = {};
|
|
16
|
+
for (const f of schema.fields) {
|
|
17
|
+
cols[f.name] = columnFromArray([], f.type as any);
|
|
18
|
+
}
|
|
19
|
+
return tableFromColumns(cols) as unknown as VgiBatch;
|
|
20
|
+
}
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
// Copyright 2025, 2026 Query Farm LLC - https://query.farm
|
|
2
|
+
// Mask-based row filtering for flechette tables.
|
|
3
|
+
|
|
4
|
+
import type { VgiBatch, VgiDataType } from "../types.js";
|
|
5
|
+
import { emptyBatch } from "./empty.js";
|
|
6
|
+
import { batchFromColumns } from "./build.js";
|
|
7
|
+
import { codecFor } from "../codec/registry.js";
|
|
8
|
+
import { readCanonicalValue } from "./canonical.js";
|
|
9
|
+
|
|
10
|
+
/**
|
|
11
|
+
* Filter a batch using a Uint8Array mask (0=exclude, nonzero=include).
|
|
12
|
+
*
|
|
13
|
+
* Rows are read in canonical form then mapped back to RICH so the rebuild goes
|
|
14
|
+
* through the same codec/canonical path as every other column build — lossless,
|
|
15
|
+
* Date-correct (flechette `.at()` returns epoch-ms for dates), and identical to
|
|
16
|
+
* the arrow-js backend.
|
|
17
|
+
*/
|
|
18
|
+
export function filterBatch(batch: VgiBatch, mask: Uint8Array): VgiBatch {
|
|
19
|
+
const t = batch as any;
|
|
20
|
+
const n = t.numRows;
|
|
21
|
+
|
|
22
|
+
let passCount = 0;
|
|
23
|
+
for (let i = 0; i < n; i++) if (mask[i]) passCount++;
|
|
24
|
+
|
|
25
|
+
if (passCount === n) return batch;
|
|
26
|
+
if (passCount === 0) return emptyBatch(t.schema);
|
|
27
|
+
|
|
28
|
+
const indices: number[] = [];
|
|
29
|
+
for (let i = 0; i < n; i++) if (mask[i]) indices.push(i);
|
|
30
|
+
|
|
31
|
+
const columns: Record<string, any[]> = {};
|
|
32
|
+
for (const f of t.schema.fields) {
|
|
33
|
+
const col = t.getChild(f.name);
|
|
34
|
+
if (!col) continue;
|
|
35
|
+
const type = f.type as VgiDataType;
|
|
36
|
+
const codec = codecFor(type);
|
|
37
|
+
columns[f.name] = indices.map((i) =>
|
|
38
|
+
codec.canonicalToRich(readCanonicalValue(type, col, i)),
|
|
39
|
+
);
|
|
40
|
+
}
|
|
41
|
+
return batchFromColumns(columns, t.schema);
|
|
42
|
+
}
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
// Copyright 2025, 2026 Query Farm LLC - https://query.farm
|
|
2
|
+
// flechette backend for the vgi-typescript Arrow facade.
|
|
3
|
+
// Selected via `#arrow-impl` resolution under the workerd / worker / browser
|
|
4
|
+
// conditions. The Bun worker keeps using the arrow-js backend (default).
|
|
5
|
+
|
|
6
|
+
import type { VgiBackendInfo } from "../types.js";
|
|
7
|
+
|
|
8
|
+
export const backend: VgiBackendInfo = { name: "flechette" };
|
|
9
|
+
|
|
10
|
+
// Schema / Field / DataType factories
|
|
11
|
+
export {
|
|
12
|
+
schema,
|
|
13
|
+
field,
|
|
14
|
+
nullType,
|
|
15
|
+
bool,
|
|
16
|
+
int,
|
|
17
|
+
int8, int16, int32, int64,
|
|
18
|
+
uint8, uint16, uint32, uint64,
|
|
19
|
+
float16, float32, float64,
|
|
20
|
+
utf8, binary, fixedSizeBinary,
|
|
21
|
+
decimal, decimal128, decimal256,
|
|
22
|
+
date, dateDay, dateMillisecond,
|
|
23
|
+
time, timeSecond, timeMillisecond, timeMicrosecond, timeNanosecond,
|
|
24
|
+
timestamp, duration, interval,
|
|
25
|
+
list, fixedSizeList, struct, map,
|
|
26
|
+
dictionary,
|
|
27
|
+
union, sparseUnion, denseUnion,
|
|
28
|
+
TimeUnit, DateUnit, IntervalUnit, UnionMode,
|
|
29
|
+
} from "./schema.js";
|
|
30
|
+
|
|
31
|
+
// Batch construction / iteration / utilities
|
|
32
|
+
export { emptyBatch } from "./empty.js";
|
|
33
|
+
export { batchFromRows, batchFromColumns, columnFromArray } from "./build.js";
|
|
34
|
+
export {
|
|
35
|
+
iterRows,
|
|
36
|
+
batchToScalarDict,
|
|
37
|
+
batchToSecretDict,
|
|
38
|
+
safeNumber,
|
|
39
|
+
decodeDictValue,
|
|
40
|
+
} from "./iterate.js";
|
|
41
|
+
export { filterBatch } from "./filter.js";
|
|
42
|
+
export { projectSchema, projectBatch } from "./project.js";
|
|
43
|
+
export { readCanonicalValue } from "./canonical.js";
|
|
44
|
+
|
|
45
|
+
// IPC
|
|
46
|
+
export {
|
|
47
|
+
serializeSchema,
|
|
48
|
+
deserializeSchema,
|
|
49
|
+
serializeBatch,
|
|
50
|
+
deserializeBatch,
|
|
51
|
+
} from "./ipc.js";
|
|
52
|
+
|
|
53
|
+
// Column statistics — sparse-union min/max via the codec/canonical path.
|
|
54
|
+
export { buildStatisticsBatch, type ColumnStatistics } from "./statistics.js";
|