lakeql 0.1.8 → 0.1.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +18 -5
- package/dist/bin.js +816 -23
- package/dist/{chunk-5K5JMJ2M.js → chunk-2XZJK7EF.js} +3 -2
- package/dist/{chunk-TFD5RFKB.js → chunk-4VJQ56HF.js} +424 -4
- package/dist/{chunk-MXGEAVHL.js → chunk-6XXXYVXT.js} +3843 -3483
- package/dist/chunk-ZVHJD6R3.js +1175 -0
- package/dist/cloudflare.d.ts +1 -1
- package/dist/cloudflare.js +5 -4
- package/dist/{geo-backend-TSQJWAAB.js → geo-backend-MY6MET3L.js} +1 -1
- package/dist/index.d.ts +420 -220
- package/dist/index.js +4 -3
- package/dist/node.d.ts +1 -1
- package/dist/node.js +48 -9
- package/dist/window-backend-DIMZN7EZ.js +905 -0
- package/package.json +9 -9
package/dist/index.d.ts
CHANGED
|
@@ -17,7 +17,19 @@ declare function timestampToIsoString(value: TimestampValue): string;
|
|
|
17
17
|
declare function timestampEpochForUnit(value: TimestampValue, unit: TimestampUnit): bigint;
|
|
18
18
|
declare function compareTimestampValues(left: TimestampValue, right: TimestampValue): number;
|
|
19
19
|
|
|
20
|
-
|
|
20
|
+
interface IntervalValue {
|
|
21
|
+
kind: "interval";
|
|
22
|
+
months: number;
|
|
23
|
+
days: number;
|
|
24
|
+
nanoseconds: string;
|
|
25
|
+
}
|
|
26
|
+
declare function intervalValue(input: string): IntervalValue;
|
|
27
|
+
declare function isIntervalValue(value: unknown): value is IntervalValue;
|
|
28
|
+
declare function isNonNegativeInterval(value: IntervalValue): boolean;
|
|
29
|
+
declare function intervalToString(value: IntervalValue): string;
|
|
30
|
+
declare function applyIntervalToTimestamp(value: TimestampValue, interval: IntervalValue, direction: -1 | 1): TimestampValue;
|
|
31
|
+
|
|
32
|
+
type Scalar = string | number | boolean | bigint | Uint8Array | TimestampValue | IntervalValue | null;
|
|
21
33
|
type CompareOp = "eq" | "ne" | "lt" | "lte" | "gt" | "gte";
|
|
22
34
|
interface LiteralExpr {
|
|
23
35
|
kind: "literal";
|
|
@@ -116,223 +128,6 @@ declare function mul(left: ValueInput, right: ValueInput): ArithmeticExpr;
|
|
|
116
128
|
declare function div(left: ValueInput, right: ValueInput): ArithmeticExpr;
|
|
117
129
|
declare function mod(left: ValueInput, right: ValueInput): ArithmeticExpr;
|
|
118
130
|
|
|
119
|
-
type Row = Record<string, unknown>;
|
|
120
|
-
interface BookmarkQuery {
|
|
121
|
-
source: string;
|
|
122
|
-
select?: string[];
|
|
123
|
-
projections?: Record<string, Expr>;
|
|
124
|
-
where?: Expr;
|
|
125
|
-
distinct?: boolean;
|
|
126
|
-
orderBy?: {
|
|
127
|
-
column: string;
|
|
128
|
-
direction?: "asc" | "desc";
|
|
129
|
-
nulls?: "first" | "last";
|
|
130
|
-
}[];
|
|
131
|
-
limit?: number;
|
|
132
|
-
offset?: number;
|
|
133
|
-
batchSize?: number;
|
|
134
|
-
hive?: boolean;
|
|
135
|
-
}
|
|
136
|
-
/**
|
|
137
|
-
* A serializable position in a running query. Bookmarks are plain values:
|
|
138
|
-
* the engine only produces and consumes them, and the caller moves them
|
|
139
|
-
* over whatever transport it likes (queue, KV, URL, cron state).
|
|
140
|
-
*/
|
|
141
|
-
interface Bookmark {
|
|
142
|
-
version: 1;
|
|
143
|
-
planFingerprint: string;
|
|
144
|
-
snapshot: string;
|
|
145
|
-
query?: BookmarkQuery;
|
|
146
|
-
position: {
|
|
147
|
-
fileIndex: number;
|
|
148
|
-
rowGroup: number;
|
|
149
|
-
rowOffset: number;
|
|
150
|
-
taskId?: string;
|
|
151
|
-
outputManifestCursor?: number;
|
|
152
|
-
};
|
|
153
|
-
writeState?: {
|
|
154
|
-
taskState?: "planned" | "running" | "output-written" | "manifest-recorded" | "complete";
|
|
155
|
-
idempotencyKey?: string;
|
|
156
|
-
multipart?: {
|
|
157
|
-
uploadId: string;
|
|
158
|
-
path: string;
|
|
159
|
-
parts: {
|
|
160
|
-
partNumber: number;
|
|
161
|
-
etag: string;
|
|
162
|
-
byteSize: number;
|
|
163
|
-
}[];
|
|
164
|
-
};
|
|
165
|
-
};
|
|
166
|
-
operatorState?: {
|
|
167
|
-
limitEmitted?: number;
|
|
168
|
-
groupBy?: Uint8Array | {
|
|
169
|
-
spillRef: string;
|
|
170
|
-
};
|
|
171
|
-
topK?: Uint8Array | {
|
|
172
|
-
spillRef: string;
|
|
173
|
-
};
|
|
174
|
-
sort?: Uint8Array | {
|
|
175
|
-
spillRef: string;
|
|
176
|
-
};
|
|
177
|
-
sketches?: Record<string, Uint8Array>;
|
|
178
|
-
};
|
|
179
|
-
}
|
|
180
|
-
interface SliceResult {
|
|
181
|
-
rows: Row[];
|
|
182
|
-
/** Absent when the query completed. */
|
|
183
|
-
bookmark?: Bookmark;
|
|
184
|
-
}
|
|
185
|
-
interface QueryStats {
|
|
186
|
-
queryId: string;
|
|
187
|
-
elapsedMs: number;
|
|
188
|
-
manifestsRead: number;
|
|
189
|
-
manifestsSkipped: number;
|
|
190
|
-
filesPlanned: number;
|
|
191
|
-
filesRead: number;
|
|
192
|
-
filesSkipped: number;
|
|
193
|
-
rowGroupsRead: number;
|
|
194
|
-
rowGroupsSkipped: number;
|
|
195
|
-
columnsRead: string[];
|
|
196
|
-
bytesRequested: number;
|
|
197
|
-
rangeRequests: number;
|
|
198
|
-
rowsDecoded: number;
|
|
199
|
-
rowsMatched: number;
|
|
200
|
-
rowsReturned: number;
|
|
201
|
-
cacheHits: number;
|
|
202
|
-
cacheMisses: number;
|
|
203
|
-
}
|
|
204
|
-
|
|
205
|
-
type Vector = {
|
|
206
|
-
type: "null";
|
|
207
|
-
length: number;
|
|
208
|
-
} | {
|
|
209
|
-
type: "f64";
|
|
210
|
-
values: Float64Array;
|
|
211
|
-
valid?: Uint8Array;
|
|
212
|
-
} | {
|
|
213
|
-
type: "i64";
|
|
214
|
-
values: BigInt64Array;
|
|
215
|
-
valid?: Uint8Array;
|
|
216
|
-
} | {
|
|
217
|
-
type: "timestamp";
|
|
218
|
-
values: BigInt64Array;
|
|
219
|
-
unit: TimestampUnit;
|
|
220
|
-
isAdjustedToUTC: boolean;
|
|
221
|
-
valid?: Uint8Array;
|
|
222
|
-
} | {
|
|
223
|
-
type: "bool";
|
|
224
|
-
values: Uint8Array;
|
|
225
|
-
valid?: Uint8Array;
|
|
226
|
-
} | {
|
|
227
|
-
type: "utf8";
|
|
228
|
-
values: string[];
|
|
229
|
-
valid?: Uint8Array;
|
|
230
|
-
} | {
|
|
231
|
-
type: "dict";
|
|
232
|
-
indices: Uint32Array;
|
|
233
|
-
dictionary: Vector;
|
|
234
|
-
valid?: Uint8Array;
|
|
235
|
-
} | {
|
|
236
|
-
type: "list";
|
|
237
|
-
offsets: Int32Array;
|
|
238
|
-
child: Vector;
|
|
239
|
-
valid?: Uint8Array;
|
|
240
|
-
} | {
|
|
241
|
-
type: "struct";
|
|
242
|
-
fields: Record<string, Vector>;
|
|
243
|
-
length: number;
|
|
244
|
-
valid?: Uint8Array;
|
|
245
|
-
} | {
|
|
246
|
-
type: "map";
|
|
247
|
-
offsets: Int32Array;
|
|
248
|
-
keys: Vector;
|
|
249
|
-
values: Vector;
|
|
250
|
-
valid?: Uint8Array;
|
|
251
|
-
};
|
|
252
|
-
interface Batch {
|
|
253
|
-
rowCount: number;
|
|
254
|
-
columns: Record<string, Vector>;
|
|
255
|
-
}
|
|
256
|
-
type Selection = Uint8Array;
|
|
257
|
-
type VectorValue = unknown;
|
|
258
|
-
declare function batchFromColumns(columns: Record<string, ArrayLike<VectorValue>>): Batch;
|
|
259
|
-
declare function batchFromVectors(columns: Record<string, Vector>): Batch;
|
|
260
|
-
declare function vectorFromValues(values: ArrayLike<VectorValue>): Vector;
|
|
261
|
-
declare function materializeBatchRows(batch: Batch): Row[];
|
|
262
|
-
declare function materializeSelectedBatchRows(batch: Batch, selection?: Selection): Row[];
|
|
263
|
-
declare function selectedRowCount(rowCount: number, selection?: Selection): number;
|
|
264
|
-
declare function selectedRowIndices(rowCount: number, selection?: Selection): Iterable<number>;
|
|
265
|
-
declare function predicateSelection(batch: Batch, expr: Expr | undefined): Selection;
|
|
266
|
-
declare function tryPredicateSelection(batch: Batch, expr: Expr | undefined): Selection | undefined;
|
|
267
|
-
declare function vectorValue(vector: Vector, index: number): string | number | bigint | boolean | TimestampValue | unknown[] | Record<string, unknown> | null;
|
|
268
|
-
declare function vectorLength(vector: Vector): number;
|
|
269
|
-
interface BatchExprValues {
|
|
270
|
-
rowCount: number;
|
|
271
|
-
vector?: Vector;
|
|
272
|
-
literal?: Scalar;
|
|
273
|
-
valueAt(index: number): Scalar;
|
|
274
|
-
}
|
|
275
|
-
declare function batchExprValues(batch: Batch, expr: Expr): BatchExprValues;
|
|
276
|
-
declare function scalarVectorValue(vector: Vector, index: number): Scalar;
|
|
277
|
-
|
|
278
|
-
declare const ERROR_CODES: readonly ["LAKEQL_PARSE_ERROR", "LAKEQL_SQL_UNSUPPORTED", "LAKEQL_TYPE_ERROR", "LAKEQL_UNKNOWN_TABLE", "LAKEQL_UNKNOWN_COLUMN", "LAKEQL_UNSUPPORTED_PUSHDOWN", "LAKEQL_BUDGET_EXCEEDED", "LAKEQL_GROUP_LIMIT_EXCEEDED", "LAKEQL_OBJECT_NOT_FOUND", "LAKEQL_CATALOG_ERROR", "LAKEQL_UNSUPPORTED_ICEBERG_FEATURE", "LAKEQL_UNSUPPORTED_PARQUET_FEATURE", "LAKEQL_ICEBERG_COMMIT_CONFLICT", "LAKEQL_UNSUPPORTED_DELETE_FILES", "LAKEQL_PARQUET_READ_ERROR", "LAKEQL_PARQUET_WRITE_ERROR", "LAKEQL_VALIDATION_ERROR", "LAKEQL_BOOKMARK_STALE", "LAKEQL_BOOKMARK_INVALID", "LAKEQL_ABORTED", "LAKEQL_GEO_BACKEND_MISSING"];
|
|
279
|
-
type LakeqlErrorCode = (typeof ERROR_CODES)[number];
|
|
280
|
-
type ErrorDetails = Record<string, unknown>;
|
|
281
|
-
declare class LakeqlError extends Error {
|
|
282
|
-
readonly code: LakeqlErrorCode;
|
|
283
|
-
readonly details: ErrorDetails;
|
|
284
|
-
constructor(code: LakeqlErrorCode, message: string, details?: ErrorDetails);
|
|
285
|
-
}
|
|
286
|
-
declare function isLakeqlError(value: unknown): value is LakeqlError;
|
|
287
|
-
|
|
288
|
-
type SqlBoolean = boolean | null;
|
|
289
|
-
/** External geometry/H3 primitives the evaluator can't compute from bbox math alone. */
|
|
290
|
-
interface GeoBackend {
|
|
291
|
-
booleanContains(a: GeoJsonGeometry, b: GeoJsonGeometry): boolean;
|
|
292
|
-
booleanIntersects(a: GeoJsonGeometry, b: GeoJsonGeometry): boolean;
|
|
293
|
-
cellToParent(cell: string, res: number): string;
|
|
294
|
-
gridDisk(origin: string, k: number): string[];
|
|
295
|
-
isValidCell(cell: string): boolean;
|
|
296
|
-
latLngToCell(lat: number, lon: number, res: number): string;
|
|
297
|
-
}
|
|
298
|
-
/** Install the backend. Called by `./geo-backend.ts` once its libraries load. */
|
|
299
|
-
declare function setGeoBackend(backend: GeoBackend): void;
|
|
300
|
-
declare function requireGeoBackend(): GeoBackend;
|
|
301
|
-
/** Force-load the geo backend (idempotent). Exposed for direct evaluate() callers. */
|
|
302
|
-
declare function loadGeoBackend(): Promise<void>;
|
|
303
|
-
/**
|
|
304
|
-
* Load the geo backend iff any of `exprs` uses a backend-requiring spatial
|
|
305
|
-
* function. Cheap no-op once the backend is installed or when no geo is present,
|
|
306
|
-
* so query execution can call it unconditionally.
|
|
307
|
-
*/
|
|
308
|
-
declare function ensureGeoBackendForExprs(exprs: Iterable<Expr | undefined>): Promise<void>;
|
|
309
|
-
type EvalValue = Scalar;
|
|
310
|
-
interface BBox {
|
|
311
|
-
minx: number;
|
|
312
|
-
miny: number;
|
|
313
|
-
maxx: number;
|
|
314
|
-
maxy: number;
|
|
315
|
-
}
|
|
316
|
-
declare function evaluate(expr: Expr, row: Row): EvalValue;
|
|
317
|
-
declare function matches(expr: Expr | undefined, row: Row): boolean;
|
|
318
|
-
declare function jsonSafeValue(value: unknown): unknown;
|
|
319
|
-
declare function encodeJsonLine(row: Row): Uint8Array;
|
|
320
|
-
declare function envelopeOf(geometry: GeoJsonGeometry): BBox;
|
|
321
|
-
type GeoJsonGeometry = {
|
|
322
|
-
type: "Point";
|
|
323
|
-
coordinates: [number, number];
|
|
324
|
-
} | {
|
|
325
|
-
type: "LineString";
|
|
326
|
-
coordinates: [number, number][];
|
|
327
|
-
} | {
|
|
328
|
-
type: "Polygon";
|
|
329
|
-
coordinates: [number, number][][];
|
|
330
|
-
};
|
|
331
|
-
declare function toGeometry(parsed: Record<string, unknown>, name: string): GeoJsonGeometry;
|
|
332
|
-
declare function parseGeometry(value: EvalValue, name: string): Record<string, unknown>;
|
|
333
|
-
declare function envelopeFromGeometry(parsed: Record<string, unknown>, name: string): BBox;
|
|
334
|
-
declare function bboxIntersects(left: BBox, right: BBox): boolean;
|
|
335
|
-
|
|
336
131
|
interface PutOptions {
|
|
337
132
|
contentType?: string;
|
|
338
133
|
metadata?: Record<string, string>;
|
|
@@ -650,6 +445,82 @@ declare function buildBBoxIndex<T extends Record<string, unknown>>(rows: T[], co
|
|
|
650
445
|
maxy: string;
|
|
651
446
|
}): BBoxIndex;
|
|
652
447
|
|
|
448
|
+
interface WindowReadColumnInput {
|
|
449
|
+
select?: string[];
|
|
450
|
+
orderBy?: OrderByTerm[];
|
|
451
|
+
projections?: Record<string, Expr>;
|
|
452
|
+
windows?: Record<string, WindowExpr>;
|
|
453
|
+
where?: Expr;
|
|
454
|
+
qualify?: Expr;
|
|
455
|
+
}
|
|
456
|
+
interface CompatibleWindowSortItem {
|
|
457
|
+
expr: WindowExpr;
|
|
458
|
+
}
|
|
459
|
+
interface CompatibleWindowSortGroup<T extends CompatibleWindowSortItem> {
|
|
460
|
+
sortExpr: WindowExpr;
|
|
461
|
+
items: T[];
|
|
462
|
+
}
|
|
463
|
+
type WindowTaskPlan = {
|
|
464
|
+
topology: "window-partition-fanout";
|
|
465
|
+
available: true;
|
|
466
|
+
bucketCount: number;
|
|
467
|
+
partitionBy: Expr[];
|
|
468
|
+
} | {
|
|
469
|
+
topology: "window-partition-fanout";
|
|
470
|
+
available: false;
|
|
471
|
+
bucketCount: 1;
|
|
472
|
+
reason: string;
|
|
473
|
+
};
|
|
474
|
+
interface WindowExplainPlan {
|
|
475
|
+
expressions: number;
|
|
476
|
+
sortGroups: number;
|
|
477
|
+
qualify: boolean;
|
|
478
|
+
execution: WindowExecutionMode;
|
|
479
|
+
workUnits: WindowTaskPlan;
|
|
480
|
+
}
|
|
481
|
+
type WindowExecutionMode = "row-materialized" | "vector-window" | "parquet-work-unit-fanout";
|
|
482
|
+
declare function windowReadColumns(input: WindowReadColumnInput): string[] | undefined;
|
|
483
|
+
declare function windowExpressions(windows: Record<string, WindowExpr> | undefined): Expr[];
|
|
484
|
+
declare function collectWindowColumns(expr: WindowExpr, columns: Set<string>): void;
|
|
485
|
+
declare function compatibleWindowSortGroups<T extends CompatibleWindowSortItem>(items: readonly T[]): CompatibleWindowSortGroup<T>[];
|
|
486
|
+
declare function windowSortGroupCount(windows: Record<string, WindowExpr>): number;
|
|
487
|
+
declare function windowTaskPlanForWindows(windows: Record<string, WindowExpr>, plannedBucketCount: number): WindowTaskPlan;
|
|
488
|
+
declare function windowSortSpecsCompatible(left: WindowExpr, right: WindowExpr): boolean;
|
|
489
|
+
|
|
490
|
+
interface WindowOptions {
|
|
491
|
+
operatorState?: Uint8Array | WindowOperatorState | {
|
|
492
|
+
spillRef: string;
|
|
493
|
+
};
|
|
494
|
+
spill?: SpillAdapter;
|
|
495
|
+
spillId?: string;
|
|
496
|
+
}
|
|
497
|
+
interface WindowResult {
|
|
498
|
+
rows: Row[];
|
|
499
|
+
operatorState: Uint8Array;
|
|
500
|
+
operatorSpill?: SpillRef;
|
|
501
|
+
}
|
|
502
|
+
interface WindowOperatorState {
|
|
503
|
+
version: 1;
|
|
504
|
+
windows: Record<string, WindowExpr>;
|
|
505
|
+
rows?: Record<string, WindowSnapshotValue>[];
|
|
506
|
+
runs?: WindowRunState[];
|
|
507
|
+
}
|
|
508
|
+
type WindowSnapshotValue = string | number | boolean | null;
|
|
509
|
+
type WindowRunState = {
|
|
510
|
+
rows: Record<string, WindowSnapshotValue>[];
|
|
511
|
+
} | {
|
|
512
|
+
spillRef: string;
|
|
513
|
+
rowCount: number;
|
|
514
|
+
byteSize: number;
|
|
515
|
+
};
|
|
516
|
+
declare function serializeWindowOperatorState(state: WindowOperatorState): Uint8Array;
|
|
517
|
+
declare function deserializeWindowOperatorState(bytes: Uint8Array | WindowOperatorState): WindowOperatorState;
|
|
518
|
+
declare function windowRowsFromState(windows: Record<string, WindowExpr>, options: WindowOptions): Promise<Row[]>;
|
|
519
|
+
declare function windowOperatorState(windows: Record<string, WindowExpr>, rows: Row[]): WindowOperatorState;
|
|
520
|
+
declare function windowOperatorStateFromRuns(windows: Record<string, WindowExpr>, runs: readonly Row[][], spill: SpillAdapter | undefined, spillId: string): Promise<WindowOperatorState>;
|
|
521
|
+
|
|
522
|
+
declare function parseJsonQuery(input: unknown): PathQueryInit;
|
|
523
|
+
|
|
653
524
|
interface QueryBudget {
|
|
654
525
|
maxFiles?: number;
|
|
655
526
|
maxBytes?: number;
|
|
@@ -659,6 +530,8 @@ interface QueryBudget {
|
|
|
659
530
|
maxElapsedMs?: number;
|
|
660
531
|
/** Maximum rows an operator may buffer in memory for orderBy/top-k work. */
|
|
661
532
|
maxBufferedRows?: number;
|
|
533
|
+
/** Maximum rows a single window partition may retain during window evaluation. */
|
|
534
|
+
maxWindowPartitionRows?: number;
|
|
662
535
|
/** Maximum deterministic serialized bytes an in-memory operator may retain. */
|
|
663
536
|
maxMemoryBytes?: number;
|
|
664
537
|
/** Maximum object-store reads allowed to be in flight at once. */
|
|
@@ -705,6 +578,16 @@ interface ScanAdapter {
|
|
|
705
578
|
scanVectorBatches?(path: string, options: ScanOptions): AsyncIterable<ScanVectorBatch>;
|
|
706
579
|
scanColumnBatches?(path: string, options: ScanOptions): AsyncIterable<ScanColumnBatch>;
|
|
707
580
|
planTask?(path: string, options: ScanTaskPlanOptions): Promise<ScanTaskPlan>;
|
|
581
|
+
executeWindowTasks?(tasks: TaskInput[], windows: Record<string, WindowExpr>, options: WindowTaskExecutionOptions): Promise<Row[]>;
|
|
582
|
+
}
|
|
583
|
+
interface WindowTaskExecutionOptions {
|
|
584
|
+
budget: QueryBudget;
|
|
585
|
+
stats: QueryStats;
|
|
586
|
+
now: () => number;
|
|
587
|
+
startedAt: number;
|
|
588
|
+
batchSize: number;
|
|
589
|
+
maxConcurrentTasks?: number;
|
|
590
|
+
maxBufferedPartials?: number;
|
|
708
591
|
}
|
|
709
592
|
interface ScanVectorBatch {
|
|
710
593
|
rowOffset: number;
|
|
@@ -733,6 +616,8 @@ interface PathQueryInit {
|
|
|
733
616
|
projections?: Record<string, Expr>;
|
|
734
617
|
where?: Expr;
|
|
735
618
|
distinct?: boolean;
|
|
619
|
+
windows?: Record<string, WindowExpr>;
|
|
620
|
+
qualify?: Expr;
|
|
736
621
|
orderBy?: OrderByTerm[];
|
|
737
622
|
limit?: number;
|
|
738
623
|
offset?: number;
|
|
@@ -894,6 +779,7 @@ interface TaskInput {
|
|
|
894
779
|
projectedColumns?: string[];
|
|
895
780
|
residualPredicate?: Expr;
|
|
896
781
|
partitionValues: Record<string, string>;
|
|
782
|
+
window?: WindowTaskPlan;
|
|
897
783
|
}
|
|
898
784
|
interface ExplainJson {
|
|
899
785
|
queryId: string;
|
|
@@ -901,6 +787,7 @@ interface ExplainJson {
|
|
|
901
787
|
filesSkipped: number;
|
|
902
788
|
projectedColumns: string[];
|
|
903
789
|
predicatePlan: PredicatePlan;
|
|
790
|
+
windowPlan?: WindowExplainPlan;
|
|
904
791
|
tasks: TaskInput[];
|
|
905
792
|
}
|
|
906
793
|
interface ExplainResult {
|
|
@@ -984,6 +871,8 @@ declare class QueryBuilder {
|
|
|
984
871
|
constructor(lake: Lake, init: PathQueryInit);
|
|
985
872
|
select(columns: string[]): QueryBuilder;
|
|
986
873
|
project(projections: Record<string, Expr>): QueryBuilder;
|
|
874
|
+
window(alias: string, expr: WindowExpr): QueryBuilder;
|
|
875
|
+
qualify(expr: Expr): QueryBuilder;
|
|
987
876
|
where(expr: Expr): QueryBuilder;
|
|
988
877
|
distinct(enabled?: boolean): QueryBuilder;
|
|
989
878
|
orderBy(terms: OrderByTerm[]): QueryBuilder;
|
|
@@ -1000,6 +889,7 @@ declare class QueryBuilder {
|
|
|
1000
889
|
toArray(): Promise<Row[]>;
|
|
1001
890
|
topKWithState(options?: TopKOptions): Promise<TopKResult>;
|
|
1002
891
|
sortWithState(options?: SortOptions): Promise<SortResult>;
|
|
892
|
+
windowWithState(options?: WindowOptions): Promise<WindowResult>;
|
|
1003
893
|
first(): Promise<Row | undefined>;
|
|
1004
894
|
count(): Promise<number>;
|
|
1005
895
|
streamNdjson(): ReadableStream<Uint8Array>;
|
|
@@ -1033,12 +923,15 @@ declare class QueryResult {
|
|
|
1033
923
|
rows(): AsyncIterable<Row>;
|
|
1034
924
|
private matchedRows;
|
|
1035
925
|
batches(): AsyncIterable<Row[]>;
|
|
926
|
+
private windowedBatches;
|
|
927
|
+
private windowRunnerOptions;
|
|
1036
928
|
private tryColumnarProjectedBatches;
|
|
1037
929
|
private columnarProjectedBatches;
|
|
1038
930
|
private lateMaterializedLimitRows;
|
|
1039
931
|
toArray(): Promise<Row[]>;
|
|
1040
932
|
topKWithState(options?: TopKOptions): Promise<TopKResult>;
|
|
1041
933
|
sortWithState(options?: SortOptions): Promise<SortResult>;
|
|
934
|
+
windowWithState(options?: WindowOptions): Promise<WindowResult>;
|
|
1042
935
|
first(): Promise<Row | undefined>;
|
|
1043
936
|
count(): Promise<number>;
|
|
1044
937
|
planTasks(): Promise<TaskInput[]>;
|
|
@@ -1070,9 +963,270 @@ declare function serializeTopKOperatorState(state: TopKOperatorState): Uint8Arra
|
|
|
1070
963
|
declare function serializeSortOperatorState(state: SortOperatorState): Uint8Array;
|
|
1071
964
|
declare function deserializeTopKOperatorState(bytes: Uint8Array | TopKOperatorState): TopKOperatorState;
|
|
1072
965
|
declare function deserializeSortOperatorState(bytes: Uint8Array | SortOperatorState): SortOperatorState;
|
|
1073
|
-
declare function parseJsonQuery(input: unknown): PathQueryInit;
|
|
1074
966
|
declare function parseHivePartitions(path: string): Record<string, string>;
|
|
1075
967
|
|
|
968
|
+
interface WindowOrderTerm {
|
|
969
|
+
expr: Expr;
|
|
970
|
+
direction?: "asc" | "desc";
|
|
971
|
+
nulls?: "first" | "last";
|
|
972
|
+
}
|
|
973
|
+
interface WindowFrameBound {
|
|
974
|
+
kind: "unbounded-preceding" | "preceding" | "current-row" | "following" | "unbounded-following";
|
|
975
|
+
offset?: Expr;
|
|
976
|
+
}
|
|
977
|
+
interface WindowFrame {
|
|
978
|
+
mode: "rows" | "range" | "groups";
|
|
979
|
+
start: WindowFrameBound;
|
|
980
|
+
end: WindowFrameBound;
|
|
981
|
+
exclude: "no-others" | "current-row" | "group" | "ties";
|
|
982
|
+
}
|
|
983
|
+
type WindowRankingFn = "row_number" | "rank" | "dense_rank" | "percent_rank" | "cume_dist" | "ntile";
|
|
984
|
+
type WindowValueFn = "lag" | "lead" | "first_value" | "last_value" | "nth_value";
|
|
985
|
+
type WindowFn = WindowRankingFn | WindowValueFn | {
|
|
986
|
+
aggregate: AggregateOp;
|
|
987
|
+
};
|
|
988
|
+
interface WindowSpec {
|
|
989
|
+
partitionBy: Expr[];
|
|
990
|
+
orderBy: WindowOrderTerm[];
|
|
991
|
+
frame?: WindowFrame;
|
|
992
|
+
}
|
|
993
|
+
interface WindowExpr {
|
|
994
|
+
fn: WindowFn;
|
|
995
|
+
args: Expr[];
|
|
996
|
+
over: WindowSpec;
|
|
997
|
+
filter?: Expr;
|
|
998
|
+
ignoreNulls?: boolean;
|
|
999
|
+
distinct?: boolean;
|
|
1000
|
+
}
|
|
1001
|
+
|
|
1002
|
+
type Row = Record<string, unknown>;
|
|
1003
|
+
interface BookmarkQuery {
|
|
1004
|
+
source: string;
|
|
1005
|
+
select?: string[];
|
|
1006
|
+
projections?: Record<string, Expr>;
|
|
1007
|
+
where?: Expr;
|
|
1008
|
+
distinct?: boolean;
|
|
1009
|
+
windows?: Record<string, WindowExpr>;
|
|
1010
|
+
qualify?: Expr;
|
|
1011
|
+
orderBy?: {
|
|
1012
|
+
column: string;
|
|
1013
|
+
direction?: "asc" | "desc";
|
|
1014
|
+
nulls?: "first" | "last";
|
|
1015
|
+
}[];
|
|
1016
|
+
limit?: number;
|
|
1017
|
+
offset?: number;
|
|
1018
|
+
batchSize?: number;
|
|
1019
|
+
hive?: boolean;
|
|
1020
|
+
}
|
|
1021
|
+
/**
|
|
1022
|
+
* A serializable position in a running query. Bookmarks are plain values:
|
|
1023
|
+
* the engine only produces and consumes them, and the caller moves them
|
|
1024
|
+
* over whatever transport it likes (queue, KV, URL, cron state).
|
|
1025
|
+
*/
|
|
1026
|
+
interface Bookmark {
|
|
1027
|
+
version: 1;
|
|
1028
|
+
planFingerprint: string;
|
|
1029
|
+
snapshot: string;
|
|
1030
|
+
query?: BookmarkQuery;
|
|
1031
|
+
position: {
|
|
1032
|
+
fileIndex: number;
|
|
1033
|
+
rowGroup: number;
|
|
1034
|
+
rowOffset: number;
|
|
1035
|
+
taskId?: string;
|
|
1036
|
+
outputManifestCursor?: number;
|
|
1037
|
+
};
|
|
1038
|
+
writeState?: {
|
|
1039
|
+
taskState?: "planned" | "running" | "output-written" | "manifest-recorded" | "complete";
|
|
1040
|
+
idempotencyKey?: string;
|
|
1041
|
+
multipart?: {
|
|
1042
|
+
uploadId: string;
|
|
1043
|
+
path: string;
|
|
1044
|
+
parts: {
|
|
1045
|
+
partNumber: number;
|
|
1046
|
+
etag: string;
|
|
1047
|
+
byteSize: number;
|
|
1048
|
+
}[];
|
|
1049
|
+
};
|
|
1050
|
+
};
|
|
1051
|
+
operatorState?: {
|
|
1052
|
+
limitEmitted?: number;
|
|
1053
|
+
groupBy?: Uint8Array | {
|
|
1054
|
+
spillRef: string;
|
|
1055
|
+
};
|
|
1056
|
+
topK?: Uint8Array | {
|
|
1057
|
+
spillRef: string;
|
|
1058
|
+
};
|
|
1059
|
+
sort?: Uint8Array | {
|
|
1060
|
+
spillRef: string;
|
|
1061
|
+
};
|
|
1062
|
+
sketches?: Record<string, Uint8Array>;
|
|
1063
|
+
};
|
|
1064
|
+
}
|
|
1065
|
+
interface SliceResult {
|
|
1066
|
+
rows: Row[];
|
|
1067
|
+
/** Absent when the query completed. */
|
|
1068
|
+
bookmark?: Bookmark;
|
|
1069
|
+
}
|
|
1070
|
+
interface QueryStats {
|
|
1071
|
+
queryId: string;
|
|
1072
|
+
elapsedMs: number;
|
|
1073
|
+
manifestsRead: number;
|
|
1074
|
+
manifestsSkipped: number;
|
|
1075
|
+
filesPlanned: number;
|
|
1076
|
+
filesRead: number;
|
|
1077
|
+
filesSkipped: number;
|
|
1078
|
+
rowGroupsRead: number;
|
|
1079
|
+
rowGroupsSkipped: number;
|
|
1080
|
+
columnsRead: string[];
|
|
1081
|
+
bytesRequested: number;
|
|
1082
|
+
rangeRequests: number;
|
|
1083
|
+
rowsDecoded: number;
|
|
1084
|
+
rowsMatched: number;
|
|
1085
|
+
rowsReturned: number;
|
|
1086
|
+
cacheHits: number;
|
|
1087
|
+
cacheMisses: number;
|
|
1088
|
+
}
|
|
1089
|
+
|
|
1090
|
+
type Vector = {
|
|
1091
|
+
type: "null";
|
|
1092
|
+
length: number;
|
|
1093
|
+
} | {
|
|
1094
|
+
type: "f64";
|
|
1095
|
+
values: Float64Array;
|
|
1096
|
+
valid?: Uint8Array;
|
|
1097
|
+
} | {
|
|
1098
|
+
type: "i64";
|
|
1099
|
+
values: BigInt64Array;
|
|
1100
|
+
valid?: Uint8Array;
|
|
1101
|
+
} | {
|
|
1102
|
+
type: "timestamp";
|
|
1103
|
+
values: BigInt64Array;
|
|
1104
|
+
unit: TimestampUnit;
|
|
1105
|
+
isAdjustedToUTC: boolean;
|
|
1106
|
+
valid?: Uint8Array;
|
|
1107
|
+
} | {
|
|
1108
|
+
type: "bool";
|
|
1109
|
+
values: Uint8Array;
|
|
1110
|
+
valid?: Uint8Array;
|
|
1111
|
+
} | {
|
|
1112
|
+
type: "utf8";
|
|
1113
|
+
values: string[];
|
|
1114
|
+
valid?: Uint8Array;
|
|
1115
|
+
} | {
|
|
1116
|
+
type: "binary";
|
|
1117
|
+
values: Uint8Array[];
|
|
1118
|
+
valid?: Uint8Array;
|
|
1119
|
+
} | {
|
|
1120
|
+
type: "dict";
|
|
1121
|
+
indices: Uint32Array;
|
|
1122
|
+
dictionary: Vector;
|
|
1123
|
+
valid?: Uint8Array;
|
|
1124
|
+
} | {
|
|
1125
|
+
type: "list";
|
|
1126
|
+
offsets: Int32Array;
|
|
1127
|
+
child: Vector;
|
|
1128
|
+
valid?: Uint8Array;
|
|
1129
|
+
} | {
|
|
1130
|
+
type: "struct";
|
|
1131
|
+
fields: Record<string, Vector>;
|
|
1132
|
+
length: number;
|
|
1133
|
+
valid?: Uint8Array;
|
|
1134
|
+
} | {
|
|
1135
|
+
type: "map";
|
|
1136
|
+
offsets: Int32Array;
|
|
1137
|
+
keys: Vector;
|
|
1138
|
+
values: Vector;
|
|
1139
|
+
valid?: Uint8Array;
|
|
1140
|
+
};
|
|
1141
|
+
interface Batch {
|
|
1142
|
+
rowCount: number;
|
|
1143
|
+
columns: Record<string, Vector>;
|
|
1144
|
+
}
|
|
1145
|
+
type Selection = Uint8Array;
|
|
1146
|
+
type VectorValue = unknown;
|
|
1147
|
+
declare function batchFromColumns(columns: Record<string, ArrayLike<VectorValue>>): Batch;
|
|
1148
|
+
declare function batchFromVectors(columns: Record<string, Vector>): Batch;
|
|
1149
|
+
declare function vectorFromValues(values: ArrayLike<VectorValue>): Vector;
|
|
1150
|
+
declare function materializeBatchRows(batch: Batch): Row[];
|
|
1151
|
+
declare function materializeBatchRow(batch: Batch, index: number): Row;
|
|
1152
|
+
declare function materializeSelectedBatchRows(batch: Batch, selection?: Selection): Row[];
|
|
1153
|
+
declare function selectedRowCount(rowCount: number, selection?: Selection): number;
|
|
1154
|
+
declare function selectedRowIndices(rowCount: number, selection?: Selection): Iterable<number>;
|
|
1155
|
+
declare function predicateSelection(batch: Batch, expr: Expr | undefined): Selection;
|
|
1156
|
+
declare function tryPredicateSelection(batch: Batch, expr: Expr | undefined): Selection | undefined;
|
|
1157
|
+
declare function vectorValue(vector: Vector, index: number): string | number | bigint | boolean | Uint8Array | TimestampValue | unknown[] | Record<string, unknown> | null;
|
|
1158
|
+
declare function vectorLength(vector: Vector): number;
|
|
1159
|
+
interface BatchExprValues {
|
|
1160
|
+
rowCount: number;
|
|
1161
|
+
vector?: Vector;
|
|
1162
|
+
literal?: Scalar;
|
|
1163
|
+
valueAt(index: number): Scalar;
|
|
1164
|
+
}
|
|
1165
|
+
declare function batchExprValues(batch: Batch, expr: Expr): BatchExprValues;
|
|
1166
|
+
declare function scalarVectorValue(vector: Vector, index: number): Scalar;
|
|
1167
|
+
|
|
1168
|
+
declare const ERROR_CODES: readonly ["LAKEQL_PARSE_ERROR", "LAKEQL_SQL_UNSUPPORTED", "LAKEQL_TYPE_ERROR", "LAKEQL_UNKNOWN_TABLE", "LAKEQL_UNKNOWN_COLUMN", "LAKEQL_UNSUPPORTED_PUSHDOWN", "LAKEQL_BUDGET_EXCEEDED", "LAKEQL_GROUP_LIMIT_EXCEEDED", "LAKEQL_OBJECT_NOT_FOUND", "LAKEQL_CATALOG_ERROR", "LAKEQL_UNSUPPORTED_ICEBERG_FEATURE", "LAKEQL_UNSUPPORTED_PARQUET_FEATURE", "LAKEQL_ICEBERG_COMMIT_CONFLICT", "LAKEQL_UNSUPPORTED_DELETE_FILES", "LAKEQL_PARQUET_READ_ERROR", "LAKEQL_PARQUET_WRITE_ERROR", "LAKEQL_VALIDATION_ERROR", "LAKEQL_BOOKMARK_STALE", "LAKEQL_BOOKMARK_INVALID", "LAKEQL_ABORTED", "LAKEQL_GEO_BACKEND_MISSING"];
|
|
1169
|
+
type LakeqlErrorCode = (typeof ERROR_CODES)[number];
|
|
1170
|
+
type ErrorDetails = Record<string, unknown>;
|
|
1171
|
+
declare class LakeqlError extends Error {
|
|
1172
|
+
readonly code: LakeqlErrorCode;
|
|
1173
|
+
readonly details: ErrorDetails;
|
|
1174
|
+
constructor(code: LakeqlErrorCode, message: string, details?: ErrorDetails);
|
|
1175
|
+
}
|
|
1176
|
+
declare function isLakeqlError(value: unknown): value is LakeqlError;
|
|
1177
|
+
|
|
1178
|
+
type SqlBoolean = boolean | null;
|
|
1179
|
+
/** External geometry/H3 primitives the evaluator can't compute from bbox math alone. */
|
|
1180
|
+
interface GeoBackend {
|
|
1181
|
+
booleanContains(a: GeoJsonGeometry, b: GeoJsonGeometry): boolean;
|
|
1182
|
+
booleanIntersects(a: GeoJsonGeometry, b: GeoJsonGeometry): boolean;
|
|
1183
|
+
cellToParent(cell: string, res: number): string;
|
|
1184
|
+
gridDisk(origin: string, k: number): string[];
|
|
1185
|
+
isValidCell(cell: string): boolean;
|
|
1186
|
+
latLngToCell(lat: number, lon: number, res: number): string;
|
|
1187
|
+
}
|
|
1188
|
+
/** Install the backend. Called by `./geo-backend.ts` once its libraries load. */
|
|
1189
|
+
declare function setGeoBackend(backend: GeoBackend): void;
|
|
1190
|
+
declare function requireGeoBackend(): GeoBackend;
|
|
1191
|
+
/** Force-load the geo backend (idempotent). Exposed for direct evaluate() callers. */
|
|
1192
|
+
declare function loadGeoBackend(): Promise<void>;
|
|
1193
|
+
/**
|
|
1194
|
+
* Load the geo backend iff any of `exprs` uses a backend-requiring spatial
|
|
1195
|
+
* function. Cheap no-op once the backend is installed or when no geo is present,
|
|
1196
|
+
* so query execution can call it unconditionally.
|
|
1197
|
+
*/
|
|
1198
|
+
declare function ensureGeoBackendForExprs(exprs: Iterable<Expr | undefined>): Promise<void>;
|
|
1199
|
+
type EvalValue = Scalar;
|
|
1200
|
+
interface BBox {
|
|
1201
|
+
minx: number;
|
|
1202
|
+
miny: number;
|
|
1203
|
+
maxx: number;
|
|
1204
|
+
maxy: number;
|
|
1205
|
+
}
|
|
1206
|
+
declare function evaluate(expr: Expr, row: Row): EvalValue;
|
|
1207
|
+
declare function matches(expr: Expr | undefined, row: Row): boolean;
|
|
1208
|
+
declare function jsonSafeValue(value: unknown): unknown;
|
|
1209
|
+
declare function encodeJsonLine(row: Row): Uint8Array;
|
|
1210
|
+
declare function envelopeOf(geometry: GeoJsonGeometry): BBox;
|
|
1211
|
+
type GeoJsonGeometry = {
|
|
1212
|
+
type: "Point";
|
|
1213
|
+
coordinates: [number, number];
|
|
1214
|
+
} | {
|
|
1215
|
+
type: "LineString";
|
|
1216
|
+
coordinates: [number, number][];
|
|
1217
|
+
} | {
|
|
1218
|
+
type: "Polygon";
|
|
1219
|
+
coordinates: [number, number][][];
|
|
1220
|
+
};
|
|
1221
|
+
declare function toGeometry(parsed: Record<string, unknown>, name: string): GeoJsonGeometry;
|
|
1222
|
+
declare function parseGeometry(value: EvalValue, name: string): Record<string, unknown>;
|
|
1223
|
+
declare function envelopeFromGeometry(parsed: Record<string, unknown>, name: string): BBox;
|
|
1224
|
+
declare function bboxIntersects(left: BBox, right: BBox): boolean;
|
|
1225
|
+
declare function bboxDistance(left: BBox, right: BBox): number;
|
|
1226
|
+
declare function geometryDistance(left: GeoJsonGeometry, right: GeoJsonGeometry): number;
|
|
1227
|
+
|
|
1228
|
+
declare function collectExprColumns(expr: Expr | undefined, columns: Set<string>): void;
|
|
1229
|
+
|
|
1076
1230
|
interface InMemoryTableOptions {
|
|
1077
1231
|
maxRows?: number;
|
|
1078
1232
|
maxBytes?: number;
|
|
@@ -1147,6 +1301,8 @@ declare class MemoryObjectStore implements ObjectStore {
|
|
|
1147
1301
|
}
|
|
1148
1302
|
declare function memoryStore(): MemoryObjectStore;
|
|
1149
1303
|
|
|
1304
|
+
declare function isPrototypeMutationKey(key: string): boolean;
|
|
1305
|
+
|
|
1150
1306
|
interface ObjectStoreCacheOptions {
|
|
1151
1307
|
/** Maximum cached object/range bytes retained in memory. Defaults to 64 MiB. */
|
|
1152
1308
|
maxBytes?: number;
|
|
@@ -1391,6 +1547,36 @@ declare function vectorTopKIndices(batch: Batch, orderBy: readonly OrderByTerm[]
|
|
|
1391
1547
|
declare function gatherBatch(batch: Batch, indices: readonly number[]): Batch;
|
|
1392
1548
|
declare function concatBatches(batches: readonly Batch[]): Batch;
|
|
1393
1549
|
|
|
1550
|
+
interface WindowExecutionOptions {
|
|
1551
|
+
estimateMemoryBytes?: (value: unknown) => number;
|
|
1552
|
+
enforceBufferedRows?: (rows: number) => void;
|
|
1553
|
+
enforceWindowPartitionRows?: (rows: number) => void;
|
|
1554
|
+
enforceMemoryBytes?: (bytes: number) => void;
|
|
1555
|
+
}
|
|
1556
|
+
|
|
1557
|
+
interface WindowWorkUnitRow {
|
|
1558
|
+
ordinal: WindowWorkUnitOrdinal;
|
|
1559
|
+
row: Row;
|
|
1560
|
+
}
|
|
1561
|
+
interface WindowWorkUnitOrdinal {
|
|
1562
|
+
task: number;
|
|
1563
|
+
row: number;
|
|
1564
|
+
}
|
|
1565
|
+
interface WindowWorkUnitBucket {
|
|
1566
|
+
bucket: number;
|
|
1567
|
+
rows: WindowWorkUnitRow[];
|
|
1568
|
+
}
|
|
1569
|
+
interface WindowWorkUnitPartial {
|
|
1570
|
+
bucketCount: number;
|
|
1571
|
+
buckets: WindowWorkUnitBucket[];
|
|
1572
|
+
}
|
|
1573
|
+
interface WindowWorkUnitEvaluationOptions extends WindowExecutionOptions {
|
|
1574
|
+
bucketCount?: number;
|
|
1575
|
+
}
|
|
1576
|
+
declare function partitionWindowWorkUnitRows(rows: readonly WindowWorkUnitRow[], windows: Record<string, WindowExpr>, bucketCount: number): WindowWorkUnitPartial;
|
|
1577
|
+
declare function evaluateWindowWorkUnitPartials(partials: readonly WindowWorkUnitPartial[], windows: Record<string, WindowExpr>, options?: WindowWorkUnitEvaluationOptions): Promise<Row[]>;
|
|
1578
|
+
declare function windowPartitionBucket(row: Row, windows: Record<string, WindowExpr>, bucketCount: number): number;
|
|
1579
|
+
|
|
1394
1580
|
interface WorkUnitFanInOptions<Input, Partial, Accumulator> {
|
|
1395
1581
|
inputs: readonly Input[];
|
|
1396
1582
|
initial: Accumulator;
|
|
@@ -1791,6 +1977,9 @@ interface ScanParquetTaskOptions {
|
|
|
1791
1977
|
batchSize?: number;
|
|
1792
1978
|
stats?: QueryStats;
|
|
1793
1979
|
metadataCache?: CacheAdapter<ParquetMetadata>;
|
|
1980
|
+
budget?: QueryBudget;
|
|
1981
|
+
now?: () => number;
|
|
1982
|
+
startedAt?: number;
|
|
1794
1983
|
}
|
|
1795
1984
|
interface PlanParquetTaskWorkUnitsOptions {
|
|
1796
1985
|
maxRowGroupsPerTask?: number;
|
|
@@ -1847,6 +2036,7 @@ declare class ParquetScanAdapter implements ScanAdapter {
|
|
|
1847
2036
|
scanColumnBatches(path: string, options: ScanOptions): AsyncIterable<ScanColumnBatch>;
|
|
1848
2037
|
scanVectorBatches(path: string, options: ScanOptions): AsyncIterable<ScanVectorBatch>;
|
|
1849
2038
|
planTask(path: string, options: ScanTaskPlanOptions): Promise<ScanTaskPlan>;
|
|
2039
|
+
executeWindowTasks(tasks: TaskInput[], windows: Record<string, WindowExpr>, options: WindowTaskExecutionOptions): Promise<Row[]>;
|
|
1850
2040
|
private metadata;
|
|
1851
2041
|
private scanBuffer;
|
|
1852
2042
|
}
|
|
@@ -1867,6 +2057,16 @@ declare function planParquetTaskWorkUnits(store: ObjectStore, task: TaskInput, o
|
|
|
1867
2057
|
/** Read Parquet footer metadata (row groups, schema, stats). */
|
|
1868
2058
|
declare function readParquetMetadata(store: ObjectStore, path: string, metadataCache?: ScanParquetTaskOptions["metadataCache"]): Promise<ParquetMetadata>;
|
|
1869
2059
|
|
|
2060
|
+
interface WindowParquetTasksOptions extends ScanParquetTaskOptions {
|
|
2061
|
+
bucketCount?: number;
|
|
2062
|
+
budget?: QueryBudget;
|
|
2063
|
+
maxConcurrentTasks?: number;
|
|
2064
|
+
maxBufferedPartials?: number;
|
|
2065
|
+
partialBoundary?(partial: WindowWorkUnitPartial, task: TaskInput, index: number): WindowWorkUnitPartial | Promise<WindowWorkUnitPartial>;
|
|
2066
|
+
}
|
|
2067
|
+
declare function windowParquetTask(store: ObjectStore, task: TaskInput, windows: Record<string, WindowExpr>, index: number, options?: WindowParquetTasksOptions): Promise<WindowWorkUnitPartial>;
|
|
2068
|
+
declare function windowParquetTasks(store: ObjectStore, tasks: TaskInput[], windows: Record<string, WindowExpr>, options?: WindowParquetTasksOptions): Promise<Row[]>;
|
|
2069
|
+
|
|
1870
2070
|
interface PlanParquetRowGroupsOptions {
|
|
1871
2071
|
where?: Expr;
|
|
1872
2072
|
}
|
|
@@ -2026,4 +2226,4 @@ declare function planFiles(table: EngineTable, options?: PlanIcebergFilesOptions
|
|
|
2026
2226
|
declare function scanBatches(plan: EngineFilePlan, options?: ScanEngineOptions): AsyncIterable<ScanBatch>;
|
|
2027
2227
|
declare function scanRows(plan: EngineFilePlan, options?: ScanEngineOptions): AsyncIterable<Row>;
|
|
2028
2228
|
|
|
2029
|
-
export { type AggregateExpr, type AggregateGroupSnapshot, type AggregateOp, type AggregateOperatorState, type AggregateOptions, type AggregateParquetGroupTaskOptions, type AggregateParquetGroupTasksOptions, type AggregateParquetTaskOptions, type AggregateParquetTasksOptions, type AggregateResult, type AggregateSnapshotValue, type AggregateSpec, type AggregateStateSnapshot, AggregationBuilder, type ApplyIcebergDeletesOptions, type ArithmeticExpr, type BBox, type BBoxIndex, type Batch, type BatchExprValues, type BetweenExpr, type Bookmark, type BookmarkInit, type BookmarkPosition, type BookmarkQuery, type BroadcastJoinOptions, type CacheAdapter, CacheApiCache, type CacheApiCacheOptions, type CacheEntry, type CachePolicy, type CallExpr, type CaseExpr, type CaseWhenExpr, type CheckpointAdapter, type CheckpointStore, type Clock, type ColumnExpr, type ColumnInput, type CompareExpr, type CompareOp, type ConditionalObjectStore, type ConditionalPutOptions, type CreateParquetTableAsOptions, type CreateParquetTableAsQuery, type CreateParquetTableAsResult, type CsvStreamOptions, type DecodedIcebergDeletes, type DecodedIcebergParquetDeletes, ERROR_CODES, type EngineFilePlan, type EngineTable, type ErrorDetails, type ExplainJson, type ExplainResult, type Expr, type GeoBackend, type GeoJsonGeometry, type IcebergAppendFile, type IcebergAppendOptions, type IcebergAppendOutputManifestOptions, type IcebergAppendResult, type IcebergCatalog, type IcebergCommitCatalog, type IcebergCommitInput, type IcebergCommitResult, type IcebergDeleteFile, type IcebergDeletionVector, type IcebergEnginePlan, type IcebergEngineTable, type IcebergEqualityDelete, type IcebergField, type IcebergGlueCatalogOptions, type IcebergLoadTableOptions, type IcebergNessieCatalogOptions, type IcebergParquetDeleteFile, type IcebergParquetDeleteFileContent, type IcebergPartitionField, type IcebergPartitionSpec, type IcebergPlan, type IcebergPositionDelete, type IcebergReadMode, type IcebergRestAccessDelegation, IcebergRestCatalog, type IcebergRestCatalogConfig, type IcebergRestCatalogOptions, type IcebergRestLoadContext, type IcebergRestLoadTableOptions, type IcebergRestLoadTableResult, type IcebergRestStorageCredential, type IcebergRowBatch, type IcebergSortOrder, IcebergTable, type IcebergTableIdentifier, IcebergUnsupportedCatalog, type IdGenerator, type InExpr, type InMemoryLakeOptions, InMemoryRowsScanner, InMemoryRowsStore, type InMemoryTableOptions, type IndexPruneResult, type IndexValue, type InsertValidationRules, type JoinKey, type JoinType, type JsonExpr, type JsonOrderByTerm, type JsonQueryV1, Lake, type LakeConfig, LakeqlError, type LakeqlErrorCode, type LikeExpr, type ListOptions, type LiteralExpr, type LoadIcebergEngineTableOptions, type LoadIcebergTableFromObjectStoreOptions, type LoadIcebergTableFromRestOptions, type LoadIcebergTableOptions, type LoadParquetEngineTableOptions, type LoadTableOptions, type LockAdapter, type LogHook, type LogicalExpr, type LookupJoinFunction, type LookupJoinOptions, type Manifest, type ManifestDeleteFile, type ManifestFile, MemoryCache, MemoryCheckpointAdapter, MemoryObjectStore, MemorySpillAdapter, type MemorySpillAdapterOptions, type MetadataFile, type MetricsHook, type MinMaxColumnIndex, type NotExpr, type NullCheckExpr, type ObjectHead, type ObjectInfo, type ObjectStore, type ObjectStoreCacheOptions, ObjectStoreIcebergCommitCatalog, ObjectStoreJsonCache, type ObjectStoreJsonCacheOptions, type ObjectStoreReadControls, type ObjectStoreUriAuthority, type OperatorSnapshotValue, type OrderByTerm, type OutputManifest, type OutputManifestEntry, PACKAGE, type ParquetColumnBatch, type ParquetEnginePlan, type ParquetEngineTable, type ParquetLakeConfig, type ParquetMetadata, type ParquetRowBatch, type ParquetRowGroupPlan, ParquetScanAdapter, type PartitionedParquetOutputEntryOptions, type PathQueryInit, type PlanIcebergFilesOptions, type PlanParquetRowGroupsOptions, type PlanParquetTaskWorkUnitsOptions, type PlannedIcebergFile, type PlannedParquetRowGroup, type PredicatePlan, type PredicatePlanOptions, type ProjectIcebergRowOptions, type PutOptions, type QueryBudget, QueryBuilder, type QueryPolicy, type QueryPolicyContext, QueryResult, type QueryRunOptions, type QueryStats, type QueueAdapter, type ReadParquetBatchOptions, type ReadParquetOptions, type ResumableBatchOptions, ResumedQuery, type Row, type RuntimeSubstrate, type Scalar, type ScanAdapter, type ScanBatch, type ScanColumnBatch, type ScanEngineOptions, type ScanOptions, type ScanParquetTaskOptions, type ScanPlannedIcebergRowsOptions, type ScanTaskPlan, type ScanTaskPlanOptions, type ScanVectorBatch, type Selection, type SharedCacheEntry, type SharedCacheSetOptions, SharedMemoryCache, type SidecarFileIndex, type SliceOptions, type SliceResult, type Snapshot, type SortOperatorState, type SortOptions, type SortResult, type SortRunState, type SpillAdapter, type SpillRef, type SpillUsage, type SqlBoolean, type TaskCheckpoint, type TaskInput, type TaskManifest, type TaskManifestTask, type TaskState, type TaskTransitionInput, type TimestampUnit, TimestampValue, type TopKOperatorState, type TopKOptions, type TopKResult, type ValueInput, type Vector, type VectorAggregateOptions, type VectorAggregateSnapshotValue, type VectorAggregateState, type VectorAggregateStateSnapshot, type VectorAggregateStateSnapshots, type VectorAggregateStates, type VectorAggregateValue, type VectorGroup, type VectorGroupByGroupSnapshot, type VectorGroupByOptions, type VectorGroupBySnapshotValue, type VectorGroupByState, type VectorGroupByStateSnapshot, type VectorHashJoinOptions, type VectorProjectionSpec, type VectorTopKOptions, type WorkUnitFanInOptions, type WriteParquetOptions, type WriteParquetRowsOptions, type WritePartitionedParquetFile, type WritePartitionedParquetResult, type WritePartitionedParquetTaskOptions, type WritePartitionedParquetTaskResult, add, advanceTaskCheckpoint, aggregateParquetGroupTask, aggregateParquetGroupTasks, aggregateParquetGroupTasksBatch, aggregateParquetTask, aggregateParquetTasks, and, applyIcebergDeletes, assertBookmarkMatches, asyncBufferFromStore, batchExprValues, batchFromColumns, batchFromVectors, bboxIntersects, bboxMayIntersect, between, broadcastJoin, buildBBoxIndex, buildMinMaxIndex, cacheApiCache, cachedObjectStore, classifyPredicate, col, compareTimestampValues, concatBatches, createBookmark, createInMemoryLake, createParquetLake as createLake, createOutputManifest, createOutputManifestFromCheckpoints, createParquetLake, createParquetTableAs, createTaskManifest, createVectorAggregateStates, createVectorGroupByState, deserializeAggregateOperatorState, deserializeSortOperatorState, deserializeTopKOperatorState, div, encodeJsonLine, enforceVectorGroupByBudget, ensureGeoBackendForExprs, envelopeFromGeometry, envelopeOf, eq, evaluate, fanInWorkUnits, finalizeVectorAggregateStates, finalizeVectorGroupByBatch, finalizeVectorGroupByRows, fingerprint, fn, gatherBatch, getOrCreateVectorGroup, gt, gte, icebergGlueCatalog, icebergNessieCatalog, icebergRestCatalog, ilike, inMemoryRowsScanner, isIn, isLakeqlError, isNotNull, isNull, isTimestampValue, jsonSafeValue, jsonWorkUnitBoundary, like, lit, loadGeoBackend, loadIcebergTable, loadIcebergTableFromObjectStore, loadIcebergTableFromRest, loadTable, lookupJoin, lt, lte, matches, materializeBatchRows, materializeSelectedBatchRows, memoryCache, memoryCheckpointAdapter, memorySpillAdapter, memoryStore, mergeVectorAggregateStateSnapshots, mergeVectorAggregateStates, mergeVectorGroupByStates, mod, mul, ne, not, notIn, objectStoreJsonCache, or, parquetScanner, parseGeometry, parseHivePartitions, parseJsonQuery, partitionedParquetOutputEntries, planFiles, planParquetTaskWorkUnits, planRowGroups, planRowGroupsFromMetadata, predicateSelection, pruneFilesWithIndex, readControlSignal, readIcebergParquetDeletes, readOutputManifest, readParquetColumnBatches, readParquetMetadata, readParquetObjectBatches, readParquetObjects, rejectUnsupportedParquetSchema, requireGeoBackend, restoreVectorAggregateStates, restoreVectorGroupByState, rowGroupMayMatch, rowGroupMustMatch, scalarVectorValue, scanBatches, scanParquetTaskBatches, scanParquetTaskColumnBatches, scanPlannedIcebergRows, scanRows, selectedRowCount, selectedRowIndices, serializeAggregateOperatorState, serializeSortOperatorState, serializeTopKOperatorState, setGeoBackend, signPaginationToken, snapshotVectorAggregateStates, snapshotVectorGroupByState, stableStringify, sub, throwIfAborted, timestampEpochForUnit, timestampFromEpoch, timestampToIsoString, timestampValueFromIso, toGeometry, transitionTaskCheckpoint, tryPredicateSelection, updateVectorAggregateStateValue, updateVectorAggregateStates, updateVectorGroupAggregateValue, updateVectorGroupByState, uriObjectStore, vectorAggregateBatch, vectorFromValues, vectorGroupByBatch, vectorHashJoin, vectorLength, vectorOrderByBatch, vectorProjectBatch, vectorSortIndices, vectorTopKBatch, vectorTopKIndices, vectorValue, verifyPaginationToken, withObjectStoreReadControls, writeOutputManifest, writeParquet, writePartitionedParquet, writePartitionedParquetTask };
|
|
2229
|
+
export { type AggregateExpr, type AggregateGroupSnapshot, type AggregateOp, type AggregateOperatorState, type AggregateOptions, type AggregateParquetGroupTaskOptions, type AggregateParquetGroupTasksOptions, type AggregateParquetTaskOptions, type AggregateParquetTasksOptions, type AggregateResult, type AggregateSnapshotValue, type AggregateSpec, type AggregateStateSnapshot, AggregationBuilder, type ApplyIcebergDeletesOptions, type ArithmeticExpr, type BBox, type BBoxIndex, type Batch, type BatchExprValues, type BetweenExpr, type Bookmark, type BookmarkInit, type BookmarkPosition, type BookmarkQuery, type BroadcastJoinOptions, type CacheAdapter, CacheApiCache, type CacheApiCacheOptions, type CacheEntry, type CachePolicy, type CallExpr, type CaseExpr, type CaseWhenExpr, type CheckpointAdapter, type CheckpointStore, type Clock, type ColumnExpr, type ColumnInput, type CompareExpr, type CompareOp, type CompatibleWindowSortGroup, type CompatibleWindowSortItem, type ConditionalObjectStore, type ConditionalPutOptions, type CreateParquetTableAsOptions, type CreateParquetTableAsQuery, type CreateParquetTableAsResult, type CsvStreamOptions, type DecodedIcebergDeletes, type DecodedIcebergParquetDeletes, ERROR_CODES, type EngineFilePlan, type EngineTable, type ErrorDetails, type ExplainJson, type ExplainResult, type Expr, type GeoBackend, type GeoJsonGeometry, type IcebergAppendFile, type IcebergAppendOptions, type IcebergAppendOutputManifestOptions, type IcebergAppendResult, type IcebergCatalog, type IcebergCommitCatalog, type IcebergCommitInput, type IcebergCommitResult, type IcebergDeleteFile, type IcebergDeletionVector, type IcebergEnginePlan, type IcebergEngineTable, type IcebergEqualityDelete, type IcebergField, type IcebergGlueCatalogOptions, type IcebergLoadTableOptions, type IcebergNessieCatalogOptions, type IcebergParquetDeleteFile, type IcebergParquetDeleteFileContent, type IcebergPartitionField, type IcebergPartitionSpec, type IcebergPlan, type IcebergPositionDelete, type IcebergReadMode, type IcebergRestAccessDelegation, IcebergRestCatalog, type IcebergRestCatalogConfig, type IcebergRestCatalogOptions, type IcebergRestLoadContext, type IcebergRestLoadTableOptions, type IcebergRestLoadTableResult, type IcebergRestStorageCredential, type IcebergRowBatch, type IcebergSortOrder, IcebergTable, type IcebergTableIdentifier, IcebergUnsupportedCatalog, type IdGenerator, type InExpr, type InMemoryLakeOptions, InMemoryRowsScanner, InMemoryRowsStore, type InMemoryTableOptions, type IndexPruneResult, type IndexValue, type InsertValidationRules, type IntervalValue, type JoinKey, type JoinType, type JsonExpr, type JsonOrderByTerm, type JsonQueryV1, Lake, type LakeConfig, LakeqlError, type LakeqlErrorCode, type LikeExpr, type ListOptions, type LiteralExpr, type LoadIcebergEngineTableOptions, type LoadIcebergTableFromObjectStoreOptions, type LoadIcebergTableFromRestOptions, type LoadIcebergTableOptions, type LoadParquetEngineTableOptions, type LoadTableOptions, type LockAdapter, type LogHook, type LogicalExpr, type LookupJoinFunction, type LookupJoinOptions, type Manifest, type ManifestDeleteFile, type ManifestFile, MemoryCache, MemoryCheckpointAdapter, MemoryObjectStore, MemorySpillAdapter, type MemorySpillAdapterOptions, type MetadataFile, type MetricsHook, type MinMaxColumnIndex, type NotExpr, type NullCheckExpr, type ObjectHead, type ObjectInfo, type ObjectStore, type ObjectStoreCacheOptions, ObjectStoreIcebergCommitCatalog, ObjectStoreJsonCache, type ObjectStoreJsonCacheOptions, type ObjectStoreReadControls, type ObjectStoreUriAuthority, type OperatorSnapshotValue, type OrderByTerm, type OutputManifest, type OutputManifestEntry, PACKAGE, type ParquetColumnBatch, type ParquetEnginePlan, type ParquetEngineTable, type ParquetLakeConfig, type ParquetMetadata, type ParquetRowBatch, type ParquetRowGroupPlan, ParquetScanAdapter, type PartitionedParquetOutputEntryOptions, type PathQueryInit, type PlanIcebergFilesOptions, type PlanParquetRowGroupsOptions, type PlanParquetTaskWorkUnitsOptions, type PlannedIcebergFile, type PlannedParquetRowGroup, type PredicatePlan, type PredicatePlanOptions, type ProjectIcebergRowOptions, type PutOptions, type QueryBudget, QueryBuilder, type QueryPolicy, type QueryPolicyContext, QueryResult, type QueryRunOptions, type QueryStats, type QueueAdapter, type ReadParquetBatchOptions, type ReadParquetOptions, type ResumableBatchOptions, ResumedQuery, type Row, type RuntimeSubstrate, type Scalar, type ScanAdapter, type ScanBatch, type ScanColumnBatch, type ScanEngineOptions, type ScanOptions, type ScanParquetTaskOptions, type ScanPlannedIcebergRowsOptions, type ScanTaskPlan, type ScanTaskPlanOptions, type ScanVectorBatch, type Selection, type SharedCacheEntry, type SharedCacheSetOptions, SharedMemoryCache, type SidecarFileIndex, type SliceOptions, type SliceResult, type Snapshot, type SortOperatorState, type SortOptions, type SortResult, type SortRunState, type SpillAdapter, type SpillRef, type SpillUsage, type SqlBoolean, type TaskCheckpoint, type TaskInput, type TaskManifest, type TaskManifestTask, type TaskState, type TaskTransitionInput, type TimestampUnit, TimestampValue, type TopKOperatorState, type TopKOptions, type TopKResult, type ValueInput, type Vector, type VectorAggregateOptions, type VectorAggregateSnapshotValue, type VectorAggregateState, type VectorAggregateStateSnapshot, type VectorAggregateStateSnapshots, type VectorAggregateStates, type VectorAggregateValue, type VectorGroup, type VectorGroupByGroupSnapshot, type VectorGroupByOptions, type VectorGroupBySnapshotValue, type VectorGroupByState, type VectorGroupByStateSnapshot, type VectorHashJoinOptions, type VectorProjectionSpec, type VectorTopKOptions, type WindowExecutionMode, type WindowExplainPlan, type WindowExpr, type WindowFn, type WindowFrame, type WindowFrameBound, type WindowOperatorState, type WindowOptions, type WindowOrderTerm, type WindowParquetTasksOptions, type WindowRankingFn, type WindowReadColumnInput, type WindowResult, type WindowRunState, type WindowSnapshotValue, type WindowSpec, type WindowTaskExecutionOptions, type WindowTaskPlan, type WindowValueFn, type WindowWorkUnitBucket, type WindowWorkUnitEvaluationOptions, type WindowWorkUnitOrdinal, type WindowWorkUnitPartial, type WindowWorkUnitRow, type WorkUnitFanInOptions, type WriteParquetOptions, type WriteParquetRowsOptions, type WritePartitionedParquetFile, type WritePartitionedParquetResult, type WritePartitionedParquetTaskOptions, type WritePartitionedParquetTaskResult, add, advanceTaskCheckpoint, aggregateParquetGroupTask, aggregateParquetGroupTasks, aggregateParquetGroupTasksBatch, aggregateParquetTask, aggregateParquetTasks, and, applyIcebergDeletes, applyIntervalToTimestamp, assertBookmarkMatches, asyncBufferFromStore, batchExprValues, batchFromColumns, batchFromVectors, bboxDistance, bboxIntersects, bboxMayIntersect, between, broadcastJoin, buildBBoxIndex, buildMinMaxIndex, cacheApiCache, cachedObjectStore, classifyPredicate, col, collectExprColumns, collectWindowColumns, compareTimestampValues, compatibleWindowSortGroups, concatBatches, createBookmark, createInMemoryLake, createParquetLake as createLake, createOutputManifest, createOutputManifestFromCheckpoints, createParquetLake, createParquetTableAs, createTaskManifest, createVectorAggregateStates, createVectorGroupByState, deserializeAggregateOperatorState, deserializeSortOperatorState, deserializeTopKOperatorState, deserializeWindowOperatorState, div, encodeJsonLine, enforceVectorGroupByBudget, ensureGeoBackendForExprs, envelopeFromGeometry, envelopeOf, eq, evaluate, evaluateWindowWorkUnitPartials, fanInWorkUnits, finalizeVectorAggregateStates, finalizeVectorGroupByBatch, finalizeVectorGroupByRows, fingerprint, fn, gatherBatch, geometryDistance, getOrCreateVectorGroup, gt, gte, icebergGlueCatalog, icebergNessieCatalog, icebergRestCatalog, ilike, inMemoryRowsScanner, intervalToString, intervalValue, isIn, isIntervalValue, isLakeqlError, isNonNegativeInterval, isNotNull, isNull, isPrototypeMutationKey, isTimestampValue, jsonSafeValue, jsonWorkUnitBoundary, like, lit, loadGeoBackend, loadIcebergTable, loadIcebergTableFromObjectStore, loadIcebergTableFromRest, loadTable, lookupJoin, lt, lte, matches, materializeBatchRow, materializeBatchRows, materializeSelectedBatchRows, memoryCache, memoryCheckpointAdapter, memorySpillAdapter, memoryStore, mergeVectorAggregateStateSnapshots, mergeVectorAggregateStates, mergeVectorGroupByStates, mod, mul, ne, not, notIn, objectStoreJsonCache, or, parquetScanner, parseGeometry, parseHivePartitions, parseJsonQuery, partitionWindowWorkUnitRows, partitionedParquetOutputEntries, planFiles, planParquetTaskWorkUnits, planRowGroups, planRowGroupsFromMetadata, predicateSelection, pruneFilesWithIndex, readControlSignal, readIcebergParquetDeletes, readOutputManifest, readParquetColumnBatches, readParquetMetadata, readParquetObjectBatches, readParquetObjects, rejectUnsupportedParquetSchema, requireGeoBackend, restoreVectorAggregateStates, restoreVectorGroupByState, rowGroupMayMatch, rowGroupMustMatch, scalarVectorValue, scanBatches, scanParquetTaskBatches, scanParquetTaskColumnBatches, scanPlannedIcebergRows, scanRows, selectedRowCount, selectedRowIndices, serializeAggregateOperatorState, serializeSortOperatorState, serializeTopKOperatorState, serializeWindowOperatorState, setGeoBackend, signPaginationToken, snapshotVectorAggregateStates, snapshotVectorGroupByState, stableStringify, sub, throwIfAborted, timestampEpochForUnit, timestampFromEpoch, timestampToIsoString, timestampValueFromIso, toGeometry, transitionTaskCheckpoint, tryPredicateSelection, updateVectorAggregateStateValue, updateVectorAggregateStates, updateVectorGroupAggregateValue, updateVectorGroupByState, uriObjectStore, vectorAggregateBatch, vectorFromValues, vectorGroupByBatch, vectorHashJoin, vectorLength, vectorOrderByBatch, vectorProjectBatch, vectorSortIndices, vectorTopKBatch, vectorTopKIndices, vectorValue, verifyPaginationToken, windowExpressions, windowOperatorState, windowOperatorStateFromRuns, windowParquetTask, windowParquetTasks, windowPartitionBucket, windowReadColumns, windowRowsFromState, windowSortGroupCount, windowSortSpecsCompatible, windowTaskPlanForWindows, withObjectStoreReadControls, writeOutputManifest, writeParquet, writePartitionedParquet, writePartitionedParquetTask };
|