lakeql 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +25 -0
- package/dist/chunk-VRJ72DGA.js +6824 -0
- package/dist/cloudflare.d.ts +44 -0
- package/dist/cloudflare.js +79 -0
- package/dist/index.d.ts +1386 -0
- package/dist/index.js +1 -0
- package/dist/node.d.ts +27 -0
- package/dist/node.js +422 -0
- package/package.json +72 -0
package/dist/index.d.ts
ADDED
|
@@ -0,0 +1,1386 @@
|
|
|
1
|
+
import * as hyparquet_src_types_js from 'hyparquet/src/types.js';
|
|
2
|
+
import { parquetMetadataAsync, RowGroup } from 'hyparquet';
|
|
3
|
+
import { ParquetWriteOptions, ColumnSource, BasicType } from 'hyparquet-writer';
|
|
4
|
+
|
|
5
|
+
declare const ERROR_CODES: readonly ["LAQL_PARSE_ERROR", "LAQL_TYPE_ERROR", "LAQL_UNKNOWN_TABLE", "LAQL_UNKNOWN_COLUMN", "LAQL_UNSUPPORTED_PUSHDOWN", "LAQL_BUDGET_EXCEEDED", "LAQL_GROUP_LIMIT_EXCEEDED", "LAQL_OBJECT_NOT_FOUND", "LAQL_CATALOG_ERROR", "LAQL_UNSUPPORTED_ICEBERG_FEATURE", "LAQL_UNSUPPORTED_PARQUET_FEATURE", "LAQL_ICEBERG_COMMIT_CONFLICT", "LAQL_UNSUPPORTED_DELETE_FILES", "LAQL_PARQUET_READ_ERROR", "LAQL_PARQUET_WRITE_ERROR", "LAQL_VALIDATION_ERROR", "LAQL_BOOKMARK_STALE", "LAQL_BOOKMARK_INVALID", "LAQL_ABORTED"];
|
|
6
|
+
type LaQLErrorCode = (typeof ERROR_CODES)[number];
|
|
7
|
+
type ErrorDetails = Record<string, unknown>;
|
|
8
|
+
declare class LaQLError extends Error {
|
|
9
|
+
readonly code: LaQLErrorCode;
|
|
10
|
+
readonly details: ErrorDetails;
|
|
11
|
+
constructor(code: LaQLErrorCode, message: string, details?: ErrorDetails);
|
|
12
|
+
}
|
|
13
|
+
declare function isLaQLError(value: unknown): value is LaQLError;
|
|
14
|
+
|
|
15
|
+
type Scalar = string | number | boolean | bigint | null;
|
|
16
|
+
type CompareOp = "eq" | "ne" | "lt" | "lte" | "gt" | "gte";
|
|
17
|
+
interface LiteralExpr {
|
|
18
|
+
kind: "literal";
|
|
19
|
+
value: Scalar;
|
|
20
|
+
}
|
|
21
|
+
interface ColumnExpr {
|
|
22
|
+
kind: "column";
|
|
23
|
+
name: string;
|
|
24
|
+
}
|
|
25
|
+
interface CompareExpr {
|
|
26
|
+
kind: "compare";
|
|
27
|
+
op: CompareOp;
|
|
28
|
+
left: Expr;
|
|
29
|
+
right: Expr;
|
|
30
|
+
}
|
|
31
|
+
interface InExpr {
|
|
32
|
+
kind: "in";
|
|
33
|
+
negated: boolean;
|
|
34
|
+
target: Expr;
|
|
35
|
+
values: Expr[];
|
|
36
|
+
}
|
|
37
|
+
interface BetweenExpr {
|
|
38
|
+
kind: "between";
|
|
39
|
+
target: Expr;
|
|
40
|
+
low: Expr;
|
|
41
|
+
high: Expr;
|
|
42
|
+
}
|
|
43
|
+
interface NullCheckExpr {
|
|
44
|
+
kind: "null-check";
|
|
45
|
+
negated: boolean;
|
|
46
|
+
target: Expr;
|
|
47
|
+
}
|
|
48
|
+
interface LogicalExpr {
|
|
49
|
+
kind: "logical";
|
|
50
|
+
op: "and" | "or";
|
|
51
|
+
operands: Expr[];
|
|
52
|
+
}
|
|
53
|
+
interface NotExpr {
|
|
54
|
+
kind: "not";
|
|
55
|
+
operand: Expr;
|
|
56
|
+
}
|
|
57
|
+
interface LikeExpr {
|
|
58
|
+
kind: "like";
|
|
59
|
+
caseInsensitive: boolean;
|
|
60
|
+
target: Expr;
|
|
61
|
+
pattern: string;
|
|
62
|
+
}
|
|
63
|
+
interface CallExpr {
|
|
64
|
+
kind: "call";
|
|
65
|
+
fn: string;
|
|
66
|
+
args: Expr[];
|
|
67
|
+
}
|
|
68
|
+
type Expr = LiteralExpr | ColumnExpr | CompareExpr | InExpr | BetweenExpr | NullCheckExpr | LogicalExpr | NotExpr | LikeExpr | CallExpr;
|
|
69
|
+
/** A value accepted where a column is expected: a column name or any expression. */
|
|
70
|
+
type ColumnInput = string | Expr;
|
|
71
|
+
/** A value accepted where a literal is expected: a scalar or any expression. */
|
|
72
|
+
type ValueInput = Scalar | Expr;
|
|
73
|
+
declare function col(name: string): ColumnExpr;
|
|
74
|
+
declare function lit(value: Scalar): LiteralExpr;
|
|
75
|
+
declare function eq(column: ColumnInput, value: ValueInput): CompareExpr;
|
|
76
|
+
declare function ne(column: ColumnInput, value: ValueInput): CompareExpr;
|
|
77
|
+
declare function lt(column: ColumnInput, value: ValueInput): CompareExpr;
|
|
78
|
+
declare function lte(column: ColumnInput, value: ValueInput): CompareExpr;
|
|
79
|
+
declare function gt(column: ColumnInput, value: ValueInput): CompareExpr;
|
|
80
|
+
declare function gte(column: ColumnInput, value: ValueInput): CompareExpr;
|
|
81
|
+
declare function isIn(column: ColumnInput, values: ValueInput[]): InExpr;
|
|
82
|
+
declare function notIn(column: ColumnInput, values: ValueInput[]): InExpr;
|
|
83
|
+
declare function between(column: ColumnInput, low: ValueInput, high: ValueInput): BetweenExpr;
|
|
84
|
+
declare function isNull(column: ColumnInput): NullCheckExpr;
|
|
85
|
+
declare function isNotNull(column: ColumnInput): NullCheckExpr;
|
|
86
|
+
declare function and(...operands: Expr[]): LogicalExpr;
|
|
87
|
+
declare function or(...operands: Expr[]): LogicalExpr;
|
|
88
|
+
declare function not(operand: Expr): NotExpr;
|
|
89
|
+
declare function like(column: ColumnInput, pattern: string): LikeExpr;
|
|
90
|
+
declare function ilike(column: ColumnInput, pattern: string): LikeExpr;
|
|
91
|
+
/** Generic function-call expression; named functions (h3_*, st_*) build on this. */
|
|
92
|
+
declare function fn(name: string, ...args: ValueInput[]): CallExpr;
|
|
93
|
+
|
|
94
|
+
type Row = Record<string, unknown>;
|
|
95
|
+
interface BookmarkQuery {
|
|
96
|
+
source: string;
|
|
97
|
+
select?: string[];
|
|
98
|
+
where?: Expr;
|
|
99
|
+
orderBy?: {
|
|
100
|
+
column: string;
|
|
101
|
+
direction?: "asc" | "desc";
|
|
102
|
+
nulls?: "first" | "last";
|
|
103
|
+
}[];
|
|
104
|
+
limit?: number;
|
|
105
|
+
offset?: number;
|
|
106
|
+
batchSize?: number;
|
|
107
|
+
hive?: boolean;
|
|
108
|
+
}
|
|
109
|
+
/**
|
|
110
|
+
* A serializable position in a running query. Bookmarks are plain values:
|
|
111
|
+
* the engine only produces and consumes them, and the caller moves them
|
|
112
|
+
* over whatever transport it likes (queue, KV, URL, cron state).
|
|
113
|
+
*/
|
|
114
|
+
interface Bookmark {
|
|
115
|
+
version: 1;
|
|
116
|
+
planFingerprint: string;
|
|
117
|
+
snapshot: string;
|
|
118
|
+
query?: BookmarkQuery;
|
|
119
|
+
position: {
|
|
120
|
+
fileIndex: number;
|
|
121
|
+
rowGroup: number;
|
|
122
|
+
rowOffset: number;
|
|
123
|
+
taskId?: string;
|
|
124
|
+
outputManifestCursor?: number;
|
|
125
|
+
};
|
|
126
|
+
writeState?: {
|
|
127
|
+
taskState?: "planned" | "running" | "output-written" | "manifest-recorded" | "complete";
|
|
128
|
+
idempotencyKey?: string;
|
|
129
|
+
multipart?: {
|
|
130
|
+
uploadId: string;
|
|
131
|
+
path: string;
|
|
132
|
+
parts: {
|
|
133
|
+
partNumber: number;
|
|
134
|
+
etag: string;
|
|
135
|
+
byteSize: number;
|
|
136
|
+
}[];
|
|
137
|
+
};
|
|
138
|
+
};
|
|
139
|
+
operatorState?: {
|
|
140
|
+
limitEmitted?: number;
|
|
141
|
+
groupBy?: Uint8Array | {
|
|
142
|
+
spillRef: string;
|
|
143
|
+
};
|
|
144
|
+
topK?: Uint8Array | {
|
|
145
|
+
spillRef: string;
|
|
146
|
+
};
|
|
147
|
+
sort?: Uint8Array | {
|
|
148
|
+
spillRef: string;
|
|
149
|
+
};
|
|
150
|
+
sketches?: Record<string, Uint8Array>;
|
|
151
|
+
};
|
|
152
|
+
}
|
|
153
|
+
interface SliceResult {
|
|
154
|
+
rows: Row[];
|
|
155
|
+
/** Absent when the query completed. */
|
|
156
|
+
bookmark?: Bookmark;
|
|
157
|
+
}
|
|
158
|
+
interface QueryStats {
|
|
159
|
+
queryId: string;
|
|
160
|
+
elapsedMs: number;
|
|
161
|
+
manifestsRead: number;
|
|
162
|
+
manifestsSkipped: number;
|
|
163
|
+
filesPlanned: number;
|
|
164
|
+
filesRead: number;
|
|
165
|
+
filesSkipped: number;
|
|
166
|
+
rowGroupsRead: number;
|
|
167
|
+
rowGroupsSkipped: number;
|
|
168
|
+
columnsRead: string[];
|
|
169
|
+
bytesRequested: number;
|
|
170
|
+
rangeRequests: number;
|
|
171
|
+
rowsDecoded: number;
|
|
172
|
+
rowsMatched: number;
|
|
173
|
+
rowsReturned: number;
|
|
174
|
+
cacheHits: number;
|
|
175
|
+
cacheMisses: number;
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
type SqlBoolean = boolean | null;
|
|
179
|
+
type EvalValue = Scalar;
|
|
180
|
+
declare function evaluate(expr: Expr, row: Row): EvalValue;
|
|
181
|
+
declare function matches(expr: Expr | undefined, row: Row): boolean;
|
|
182
|
+
declare function jsonSafeValue(value: unknown): unknown;
|
|
183
|
+
declare function encodeJsonLine(row: Row): Uint8Array;
|
|
184
|
+
|
|
185
|
+
type JoinType = "inner" | "left" | "semi" | "anti";
|
|
186
|
+
interface BroadcastJoinOptions {
|
|
187
|
+
leftKey: string;
|
|
188
|
+
rightKey: string;
|
|
189
|
+
maxRightRows: number;
|
|
190
|
+
type?: JoinType;
|
|
191
|
+
rightPrefix?: string;
|
|
192
|
+
}
|
|
193
|
+
interface LookupJoinOptions {
|
|
194
|
+
leftKey: string;
|
|
195
|
+
rightKey: string;
|
|
196
|
+
maxRightRows: number;
|
|
197
|
+
type?: JoinType;
|
|
198
|
+
rightPrefix?: string;
|
|
199
|
+
}
|
|
200
|
+
type LookupJoinFunction = (key: string | number | boolean | bigint | null, leftRow: Row) => AsyncIterable<Row> | Iterable<Row> | Promise<AsyncIterable<Row> | Iterable<Row>>;
|
|
201
|
+
declare function broadcastJoin(left: AsyncIterable<Row> | Iterable<Row>, right: AsyncIterable<Row> | Iterable<Row>, options: BroadcastJoinOptions): Promise<Row[]>;
|
|
202
|
+
declare function lookupJoin(left: AsyncIterable<Row> | Iterable<Row>, lookup: LookupJoinFunction, options: LookupJoinOptions): Promise<Row[]>;
|
|
203
|
+
|
|
204
|
+
interface PredicatePlan {
|
|
205
|
+
partition: Expr[];
|
|
206
|
+
fileStats: Expr[];
|
|
207
|
+
rowGroupStats: Expr[];
|
|
208
|
+
residual: Expr[];
|
|
209
|
+
}
|
|
210
|
+
interface PredicatePlanOptions {
|
|
211
|
+
partitionColumns?: Iterable<string>;
|
|
212
|
+
fileStatsColumns?: Iterable<string>;
|
|
213
|
+
rowGroupStatsColumns?: Iterable<string>;
|
|
214
|
+
}
|
|
215
|
+
declare function classifyPredicate(expr: Expr | undefined, options?: PredicatePlanOptions): PredicatePlan;
|
|
216
|
+
|
|
217
|
+
interface CacheEntry<T> {
|
|
218
|
+
value: T;
|
|
219
|
+
expiresAt?: number;
|
|
220
|
+
}
|
|
221
|
+
interface CacheAdapter<T = Uint8Array> {
|
|
222
|
+
get(key: string): Promise<CacheEntry<T> | undefined>;
|
|
223
|
+
set(key: string, entry: CacheEntry<T>): Promise<void>;
|
|
224
|
+
delete(key: string): Promise<void>;
|
|
225
|
+
}
|
|
226
|
+
declare class MemoryCache<T = Uint8Array> implements CacheAdapter<T> {
|
|
227
|
+
private readonly entries;
|
|
228
|
+
get(key: string): Promise<CacheEntry<T> | undefined>;
|
|
229
|
+
set(key: string, entry: CacheEntry<T>): Promise<void>;
|
|
230
|
+
delete(key: string): Promise<void>;
|
|
231
|
+
}
|
|
232
|
+
declare function memoryCache<T = Uint8Array>(): CacheAdapter<T>;
|
|
233
|
+
interface CacheApiCacheOptions {
|
|
234
|
+
namespace?: string;
|
|
235
|
+
now?: () => number;
|
|
236
|
+
}
|
|
237
|
+
declare class CacheApiCache implements CacheAdapter<Uint8Array> {
|
|
238
|
+
private readonly cache;
|
|
239
|
+
private readonly namespace;
|
|
240
|
+
private readonly now;
|
|
241
|
+
constructor(cache: Cache, options?: CacheApiCacheOptions);
|
|
242
|
+
get(key: string): Promise<CacheEntry<Uint8Array> | undefined>;
|
|
243
|
+
set(key: string, entry: CacheEntry<Uint8Array>): Promise<void>;
|
|
244
|
+
delete(key: string): Promise<void>;
|
|
245
|
+
private request;
|
|
246
|
+
}
|
|
247
|
+
declare function cacheApiCache(cache: Cache, options?: CacheApiCacheOptions): CacheAdapter<Uint8Array>;
|
|
248
|
+
interface CheckpointStore {
|
|
249
|
+
get(jobId: string): Promise<Bookmark | undefined>;
|
|
250
|
+
put(jobId: string, bookmark: Bookmark): Promise<void>;
|
|
251
|
+
delete(jobId: string): Promise<void>;
|
|
252
|
+
}
|
|
253
|
+
interface SpillRef {
|
|
254
|
+
id: string;
|
|
255
|
+
byteSize: number;
|
|
256
|
+
}
|
|
257
|
+
interface SpillUsage {
|
|
258
|
+
entries: number;
|
|
259
|
+
byteSize: number;
|
|
260
|
+
}
|
|
261
|
+
interface SpillAdapter {
|
|
262
|
+
write(id: string, data: Uint8Array): Promise<SpillRef>;
|
|
263
|
+
read(ref: SpillRef | string): Promise<Uint8Array>;
|
|
264
|
+
delete(ref: SpillRef | string): Promise<void>;
|
|
265
|
+
usage(): Promise<SpillUsage>;
|
|
266
|
+
}
|
|
267
|
+
interface MemorySpillAdapterOptions {
|
|
268
|
+
maxBytes?: number;
|
|
269
|
+
}
|
|
270
|
+
declare class MemorySpillAdapter implements SpillAdapter {
|
|
271
|
+
private readonly entries;
|
|
272
|
+
private readonly maxBytes;
|
|
273
|
+
constructor(options?: MemorySpillAdapterOptions);
|
|
274
|
+
write(id: string, data: Uint8Array): Promise<SpillRef>;
|
|
275
|
+
read(ref: SpillRef | string): Promise<Uint8Array>;
|
|
276
|
+
delete(ref: SpillRef | string): Promise<void>;
|
|
277
|
+
usage(): Promise<SpillUsage>;
|
|
278
|
+
private currentBytes;
|
|
279
|
+
}
|
|
280
|
+
declare function memorySpillAdapter(options?: MemorySpillAdapterOptions): SpillAdapter;
|
|
281
|
+
interface QueueAdapter<T> {
|
|
282
|
+
send(message: T, options?: {
|
|
283
|
+
delayMs?: number;
|
|
284
|
+
}): Promise<void>;
|
|
285
|
+
}
|
|
286
|
+
interface LockAdapter {
|
|
287
|
+
withLock<T>(key: string, fn: () => Promise<T>): Promise<T>;
|
|
288
|
+
}
|
|
289
|
+
interface Clock {
|
|
290
|
+
now(): number;
|
|
291
|
+
}
|
|
292
|
+
interface IdGenerator {
|
|
293
|
+
id(prefix?: string): string;
|
|
294
|
+
}
|
|
295
|
+
interface MetricsHook {
|
|
296
|
+
count(name: string, value?: number, tags?: Record<string, string>): void;
|
|
297
|
+
timing(name: string, ms: number, tags?: Record<string, string>): void;
|
|
298
|
+
}
|
|
299
|
+
interface LogHook {
|
|
300
|
+
debug(message: string, fields?: Record<string, unknown>): void;
|
|
301
|
+
info(message: string, fields?: Record<string, unknown>): void;
|
|
302
|
+
warn(message: string, fields?: Record<string, unknown>): void;
|
|
303
|
+
error(message: string, fields?: Record<string, unknown>): void;
|
|
304
|
+
}
|
|
305
|
+
interface RuntimeSubstrate {
|
|
306
|
+
checkpointStore?: CheckpointStore;
|
|
307
|
+
spill?: SpillAdapter;
|
|
308
|
+
queue?: QueueAdapter<Bookmark>;
|
|
309
|
+
lock?: LockAdapter;
|
|
310
|
+
clock?: Clock;
|
|
311
|
+
ids?: IdGenerator;
|
|
312
|
+
metrics?: MetricsHook;
|
|
313
|
+
log?: LogHook;
|
|
314
|
+
}
|
|
315
|
+
|
|
316
|
+
type IndexValue = string | number | bigint | boolean;
|
|
317
|
+
interface MinMaxColumnIndex {
|
|
318
|
+
min: IndexValue;
|
|
319
|
+
max: IndexValue;
|
|
320
|
+
nullCount?: number;
|
|
321
|
+
}
|
|
322
|
+
interface BBoxIndex {
|
|
323
|
+
minx: number;
|
|
324
|
+
miny: number;
|
|
325
|
+
maxx: number;
|
|
326
|
+
maxy: number;
|
|
327
|
+
}
|
|
328
|
+
interface SidecarFileIndex {
|
|
329
|
+
path: string;
|
|
330
|
+
columns?: Record<string, MinMaxColumnIndex>;
|
|
331
|
+
bbox?: Record<string, BBoxIndex>;
|
|
332
|
+
h3?: Record<string, string[]>;
|
|
333
|
+
}
|
|
334
|
+
interface IndexPruneResult<T extends SidecarFileIndex = SidecarFileIndex> {
|
|
335
|
+
planned: T[];
|
|
336
|
+
skipped: T[];
|
|
337
|
+
}
|
|
338
|
+
declare function pruneFilesWithIndex<T extends SidecarFileIndex>(files: T[], where: Expr | undefined): IndexPruneResult<T>;
|
|
339
|
+
declare function bboxMayIntersect(index: BBoxIndex, query: BBoxIndex): boolean;
|
|
340
|
+
declare function buildMinMaxIndex<T extends Record<string, unknown>>(rows: T[], columns: string[]): Record<string, MinMaxColumnIndex>;
|
|
341
|
+
declare function buildBBoxIndex<T extends Record<string, unknown>>(rows: T[], columns: {
|
|
342
|
+
minx: string;
|
|
343
|
+
miny: string;
|
|
344
|
+
maxx: string;
|
|
345
|
+
maxy: string;
|
|
346
|
+
}): BBoxIndex;
|
|
347
|
+
|
|
348
|
+
interface PutOptions {
|
|
349
|
+
contentType?: string;
|
|
350
|
+
metadata?: Record<string, string>;
|
|
351
|
+
}
|
|
352
|
+
interface ConditionalPutOptions extends PutOptions {
|
|
353
|
+
/**
|
|
354
|
+
* ETag that must still identify the existing object. Use null to require
|
|
355
|
+
* the object to be absent.
|
|
356
|
+
*/
|
|
357
|
+
expectedEtag: string | null;
|
|
358
|
+
}
|
|
359
|
+
interface ListOptions {
|
|
360
|
+
/** Stop listing after this many objects. */
|
|
361
|
+
limit?: number;
|
|
362
|
+
/** List "directories" by stopping at this delimiter. */
|
|
363
|
+
delimiter?: string;
|
|
364
|
+
}
|
|
365
|
+
interface ObjectInfo {
|
|
366
|
+
path: string;
|
|
367
|
+
size: number;
|
|
368
|
+
etag?: string;
|
|
369
|
+
lastModified?: Date;
|
|
370
|
+
}
|
|
371
|
+
interface ObjectHead {
|
|
372
|
+
size: number;
|
|
373
|
+
etag?: string;
|
|
374
|
+
lastModified?: Date;
|
|
375
|
+
contentType?: string;
|
|
376
|
+
}
|
|
377
|
+
/**
|
|
378
|
+
* The single environmental contract for storage. Every runtime driver
|
|
379
|
+
* (R2, S3, HTTP, filesystem) supplies one of these; core never touches
|
|
380
|
+
* a runtime API directly.
|
|
381
|
+
*/
|
|
382
|
+
interface ObjectStore {
|
|
383
|
+
get(path: string): Promise<Uint8Array | null>;
|
|
384
|
+
getRange(path: string, range: {
|
|
385
|
+
offset: number;
|
|
386
|
+
length: number;
|
|
387
|
+
}): Promise<Uint8Array>;
|
|
388
|
+
put(path: string, body: Uint8Array | ReadableStream<Uint8Array>, options?: PutOptions): Promise<void>;
|
|
389
|
+
delete(path: string): Promise<void>;
|
|
390
|
+
list(prefix: string, options?: ListOptions): AsyncIterable<ObjectInfo>;
|
|
391
|
+
head(path: string): Promise<ObjectHead | null>;
|
|
392
|
+
}
|
|
393
|
+
interface ConditionalObjectStore extends ObjectStore {
|
|
394
|
+
conditionalPut(path: string, body: Uint8Array | ReadableStream<Uint8Array>, options: ConditionalPutOptions): Promise<boolean>;
|
|
395
|
+
}
|
|
396
|
+
interface ObjectStoreReadControls {
|
|
397
|
+
maxConcurrentReads?: number;
|
|
398
|
+
signal?: AbortSignal;
|
|
399
|
+
maxElapsedMs?: number;
|
|
400
|
+
}
|
|
401
|
+
declare function withObjectStoreReadControls(store: ObjectStore, controls: ObjectStoreReadControls): ObjectStore;
|
|
402
|
+
declare function readControlSignal(controls: ObjectStoreReadControls): AbortSignal | undefined;
|
|
403
|
+
declare function throwIfAborted(signal: AbortSignal | undefined): void;
|
|
404
|
+
|
|
405
|
+
interface QueryBudget {
|
|
406
|
+
maxFiles?: number;
|
|
407
|
+
maxBytes?: number;
|
|
408
|
+
maxRowsDecoded?: number;
|
|
409
|
+
maxOutputRows?: number;
|
|
410
|
+
maxRangeRequests?: number;
|
|
411
|
+
maxElapsedMs?: number;
|
|
412
|
+
/** Maximum rows an operator may buffer in memory for orderBy/top-k work. */
|
|
413
|
+
maxBufferedRows?: number;
|
|
414
|
+
/** Maximum deterministic serialized bytes an in-memory operator may retain. */
|
|
415
|
+
maxMemoryBytes?: number;
|
|
416
|
+
/** Maximum object-store reads allowed to be in flight at once. */
|
|
417
|
+
maxConcurrentReads?: number;
|
|
418
|
+
/** Abort query planning or scanning at await boundaries. */
|
|
419
|
+
signal?: AbortSignal;
|
|
420
|
+
}
|
|
421
|
+
type QueryPolicyContext = Record<string, unknown>;
|
|
422
|
+
interface QueryPolicy {
|
|
423
|
+
/** Restrict visible and predicate columns. When no select is supplied, this becomes the projection. */
|
|
424
|
+
allowedColumns?: string[];
|
|
425
|
+
/** Hard cap on rows returned by a query, applied even when the caller omits limit. */
|
|
426
|
+
maxLimit?: number;
|
|
427
|
+
/** Additional caller-owned predicate applied to every query. */
|
|
428
|
+
rowFilter?: Expr | ((context: QueryPolicyContext) => Expr | undefined);
|
|
429
|
+
context?: QueryPolicyContext;
|
|
430
|
+
}
|
|
431
|
+
interface LakeConfig {
|
|
432
|
+
store: ObjectStore;
|
|
433
|
+
scanner: ScanAdapter;
|
|
434
|
+
sidecarIndex?: SidecarFileIndex[];
|
|
435
|
+
budget?: QueryBudget;
|
|
436
|
+
policy?: QueryPolicy;
|
|
437
|
+
substrate?: RuntimeSubstrate;
|
|
438
|
+
now?: () => number;
|
|
439
|
+
queryId?: () => string;
|
|
440
|
+
}
|
|
441
|
+
interface ScanOptions {
|
|
442
|
+
columns?: string[];
|
|
443
|
+
where?: Expr;
|
|
444
|
+
batchSize: number;
|
|
445
|
+
stats: QueryStats;
|
|
446
|
+
budget: QueryBudget;
|
|
447
|
+
now: () => number;
|
|
448
|
+
startedAt: number;
|
|
449
|
+
}
|
|
450
|
+
interface ScanAdapter {
|
|
451
|
+
scan(path: string, options: ScanOptions): AsyncIterable<Row[]>;
|
|
452
|
+
planTask?(path: string, options: ScanTaskPlanOptions): Promise<ScanTaskPlan>;
|
|
453
|
+
}
|
|
454
|
+
interface ScanTaskPlanOptions {
|
|
455
|
+
columns?: string[];
|
|
456
|
+
where?: Expr;
|
|
457
|
+
partitionValues: Record<string, string>;
|
|
458
|
+
}
|
|
459
|
+
interface ScanTaskPlan {
|
|
460
|
+
rowGroupRanges: {
|
|
461
|
+
start: number;
|
|
462
|
+
end: number;
|
|
463
|
+
}[];
|
|
464
|
+
}
|
|
465
|
+
interface PathQueryInit {
|
|
466
|
+
source: string;
|
|
467
|
+
select?: string[];
|
|
468
|
+
where?: Expr;
|
|
469
|
+
orderBy?: OrderByTerm[];
|
|
470
|
+
limit?: number;
|
|
471
|
+
offset?: number;
|
|
472
|
+
batchSize?: number;
|
|
473
|
+
hive?: boolean;
|
|
474
|
+
}
|
|
475
|
+
interface OrderByTerm {
|
|
476
|
+
column: string;
|
|
477
|
+
direction?: "asc" | "desc";
|
|
478
|
+
nulls?: "first" | "last";
|
|
479
|
+
}
|
|
480
|
+
interface SliceOptions {
|
|
481
|
+
maxRows: number;
|
|
482
|
+
bookmark?: Bookmark;
|
|
483
|
+
}
|
|
484
|
+
interface QueryRunOptions {
|
|
485
|
+
slice: SliceOptions;
|
|
486
|
+
}
|
|
487
|
+
interface ResumableBatchOptions {
|
|
488
|
+
bookmarkEvery: number;
|
|
489
|
+
bookmark?: Bookmark;
|
|
490
|
+
}
|
|
491
|
+
interface CsvStreamOptions {
|
|
492
|
+
/** Emit the header row. Defaults to true. */
|
|
493
|
+
header?: boolean;
|
|
494
|
+
/** Column order for CSV output. Defaults to select columns or first-row keys. */
|
|
495
|
+
columns?: string[];
|
|
496
|
+
}
|
|
497
|
+
type AggregateOp = "count" | "sum" | "avg" | "min" | "max" | "count_distinct" | "approx_count_distinct" | "first" | "last" | "any";
|
|
498
|
+
interface AggregateExpr {
|
|
499
|
+
op: AggregateOp;
|
|
500
|
+
column?: string;
|
|
501
|
+
}
|
|
502
|
+
type AggregateSpec = Record<string, AggregateExpr>;
|
|
503
|
+
interface AggregateOptions {
|
|
504
|
+
maxGroups?: number;
|
|
505
|
+
operatorState?: Uint8Array | AggregateOperatorState | {
|
|
506
|
+
spillRef: string;
|
|
507
|
+
};
|
|
508
|
+
spill?: SpillAdapter;
|
|
509
|
+
spillId?: string;
|
|
510
|
+
}
|
|
511
|
+
interface AggregateResult {
|
|
512
|
+
rows: Row[];
|
|
513
|
+
operatorState: Uint8Array;
|
|
514
|
+
operatorSpill?: SpillRef;
|
|
515
|
+
}
|
|
516
|
+
interface TopKOptions {
|
|
517
|
+
operatorState?: Uint8Array | TopKOperatorState | {
|
|
518
|
+
spillRef: string;
|
|
519
|
+
};
|
|
520
|
+
spill?: SpillAdapter;
|
|
521
|
+
spillId?: string;
|
|
522
|
+
}
|
|
523
|
+
interface TopKResult {
|
|
524
|
+
rows: Row[];
|
|
525
|
+
operatorState: Uint8Array;
|
|
526
|
+
operatorSpill?: SpillRef;
|
|
527
|
+
}
|
|
528
|
+
interface SortOptions {
|
|
529
|
+
operatorState?: Uint8Array | SortOperatorState | {
|
|
530
|
+
spillRef: string;
|
|
531
|
+
};
|
|
532
|
+
spill?: SpillAdapter;
|
|
533
|
+
spillId?: string;
|
|
534
|
+
}
|
|
535
|
+
interface SortResult {
|
|
536
|
+
rows: Row[];
|
|
537
|
+
operatorState: Uint8Array;
|
|
538
|
+
operatorSpill?: SpillRef;
|
|
539
|
+
}
|
|
540
|
+
interface SortOperatorState {
|
|
541
|
+
version: 1;
|
|
542
|
+
orderBy: OrderByTerm[];
|
|
543
|
+
runs: SortRunState[];
|
|
544
|
+
}
|
|
545
|
+
type SortRunState = {
|
|
546
|
+
rows: Record<string, OperatorSnapshotValue>[];
|
|
547
|
+
} | {
|
|
548
|
+
spillRef: string;
|
|
549
|
+
rowCount: number;
|
|
550
|
+
byteSize: number;
|
|
551
|
+
};
|
|
552
|
+
interface TopKOperatorState {
|
|
553
|
+
version: 1;
|
|
554
|
+
orderBy: OrderByTerm[];
|
|
555
|
+
offset: number;
|
|
556
|
+
limit: number;
|
|
557
|
+
rows: Record<string, OperatorSnapshotValue>[];
|
|
558
|
+
}
|
|
559
|
+
type OperatorSnapshotValue = string | number | boolean | null;
|
|
560
|
+
type AggregateSnapshotValue = OperatorSnapshotValue;
|
|
561
|
+
interface AggregateOperatorState {
|
|
562
|
+
version: 1;
|
|
563
|
+
groupColumns: string[];
|
|
564
|
+
spec: AggregateSpec;
|
|
565
|
+
groups: AggregateGroupSnapshot[];
|
|
566
|
+
}
|
|
567
|
+
interface AggregateGroupSnapshot {
|
|
568
|
+
key: string;
|
|
569
|
+
keys: Record<string, AggregateSnapshotValue>;
|
|
570
|
+
states: Record<string, AggregateStateSnapshot>;
|
|
571
|
+
}
|
|
572
|
+
type AggregateStateSnapshot = {
|
|
573
|
+
op: "count";
|
|
574
|
+
count: number;
|
|
575
|
+
} | {
|
|
576
|
+
op: "sum";
|
|
577
|
+
sum: number;
|
|
578
|
+
} | {
|
|
579
|
+
op: "avg";
|
|
580
|
+
sum: number;
|
|
581
|
+
count: number;
|
|
582
|
+
} | {
|
|
583
|
+
op: "min" | "max";
|
|
584
|
+
value: AggregateSnapshotValue;
|
|
585
|
+
} | {
|
|
586
|
+
op: "count_distinct" | "approx_count_distinct";
|
|
587
|
+
values: string[];
|
|
588
|
+
} | {
|
|
589
|
+
op: "first" | "last" | "any";
|
|
590
|
+
seen: boolean;
|
|
591
|
+
value: AggregateSnapshotValue;
|
|
592
|
+
};
|
|
593
|
+
interface TaskInput {
|
|
594
|
+
path: string;
|
|
595
|
+
etag?: string;
|
|
596
|
+
size?: number;
|
|
597
|
+
rowGroupRanges: {
|
|
598
|
+
start: number;
|
|
599
|
+
end: number;
|
|
600
|
+
}[];
|
|
601
|
+
projectedColumns?: string[];
|
|
602
|
+
residualPredicate?: Expr;
|
|
603
|
+
partitionValues: Record<string, string>;
|
|
604
|
+
}
|
|
605
|
+
interface ExplainJson {
|
|
606
|
+
queryId: string;
|
|
607
|
+
filesPlanned: number;
|
|
608
|
+
filesSkipped: number;
|
|
609
|
+
projectedColumns: string[];
|
|
610
|
+
predicatePlan: PredicatePlan;
|
|
611
|
+
tasks: TaskInput[];
|
|
612
|
+
}
|
|
613
|
+
interface ExplainResult {
|
|
614
|
+
text: string;
|
|
615
|
+
json: ExplainJson;
|
|
616
|
+
}
|
|
617
|
+
interface JsonQueryV1 {
|
|
618
|
+
version: 1;
|
|
619
|
+
from: string;
|
|
620
|
+
select?: string[];
|
|
621
|
+
where?: JsonExpr;
|
|
622
|
+
orderBy?: JsonOrderByTerm[];
|
|
623
|
+
limit?: number;
|
|
624
|
+
offset?: number;
|
|
625
|
+
}
|
|
626
|
+
interface JsonOrderByTerm {
|
|
627
|
+
column: string;
|
|
628
|
+
direction?: "asc" | "desc";
|
|
629
|
+
nulls?: "first" | "last";
|
|
630
|
+
}
|
|
631
|
+
type JsonExpr = {
|
|
632
|
+
eq: [string, unknown];
|
|
633
|
+
} | {
|
|
634
|
+
ne: [string, unknown];
|
|
635
|
+
} | {
|
|
636
|
+
lt: [string, unknown];
|
|
637
|
+
} | {
|
|
638
|
+
lte: [string, unknown];
|
|
639
|
+
} | {
|
|
640
|
+
gt: [string, unknown];
|
|
641
|
+
} | {
|
|
642
|
+
gte: [string, unknown];
|
|
643
|
+
} | {
|
|
644
|
+
in: [string, unknown[]];
|
|
645
|
+
} | {
|
|
646
|
+
notIn: [string, unknown[]];
|
|
647
|
+
} | {
|
|
648
|
+
between: [string, unknown, unknown];
|
|
649
|
+
} | {
|
|
650
|
+
isNull: string;
|
|
651
|
+
} | {
|
|
652
|
+
isNotNull: string;
|
|
653
|
+
} | {
|
|
654
|
+
like: [string, string];
|
|
655
|
+
} | {
|
|
656
|
+
ilike: [string, string];
|
|
657
|
+
} | {
|
|
658
|
+
and: JsonExpr[];
|
|
659
|
+
} | {
|
|
660
|
+
or: JsonExpr[];
|
|
661
|
+
} | {
|
|
662
|
+
not: JsonExpr;
|
|
663
|
+
};
|
|
664
|
+
declare class Lake {
|
|
665
|
+
readonly store: ObjectStore;
|
|
666
|
+
private readonly scanner;
|
|
667
|
+
private readonly budget;
|
|
668
|
+
private readonly policy;
|
|
669
|
+
private readonly now;
|
|
670
|
+
private readonly queryId;
|
|
671
|
+
private readonly substrate;
|
|
672
|
+
private readonly sidecarIndex;
|
|
673
|
+
constructor(config: LakeConfig);
|
|
674
|
+
path(source: string): QueryBuilder;
|
|
675
|
+
hive(source: string): QueryBuilder;
|
|
676
|
+
query(input: JsonQueryV1): QueryBuilder;
|
|
677
|
+
resume(bookmark: Bookmark): ResumedQuery;
|
|
678
|
+
createResult(init: PathQueryInit): QueryResult;
|
|
679
|
+
}
|
|
680
|
+
declare class ResumedQuery {
|
|
681
|
+
private readonly lake;
|
|
682
|
+
private readonly bookmark;
|
|
683
|
+
constructor(lake: Lake, bookmark: Bookmark);
|
|
684
|
+
run(options: QueryRunOptions): Promise<SliceResult>;
|
|
685
|
+
}
|
|
686
|
+
declare class QueryBuilder {
|
|
687
|
+
private readonly lake;
|
|
688
|
+
private readonly init;
|
|
689
|
+
constructor(lake: Lake, init: PathQueryInit);
|
|
690
|
+
select(columns: string[]): QueryBuilder;
|
|
691
|
+
where(expr: Expr): QueryBuilder;
|
|
692
|
+
orderBy(terms: OrderByTerm[]): QueryBuilder;
|
|
693
|
+
limit(limit: number): QueryBuilder;
|
|
694
|
+
offset(offset: number): QueryBuilder;
|
|
695
|
+
batchSize(batchSize: number): QueryBuilder;
|
|
696
|
+
explain(): Promise<ExplainResult>;
|
|
697
|
+
planTasks(): Promise<TaskInput[]>;
|
|
698
|
+
taskManifest(jobId?: string): Promise<TaskManifest>;
|
|
699
|
+
run(): QueryResult;
|
|
700
|
+
run(options: QueryRunOptions): Promise<SliceResult>;
|
|
701
|
+
rows(): AsyncIterable<Row>;
|
|
702
|
+
batches(): AsyncIterable<Row[]>;
|
|
703
|
+
toArray(): Promise<Row[]>;
|
|
704
|
+
topKWithState(options?: TopKOptions): Promise<TopKResult>;
|
|
705
|
+
sortWithState(options?: SortOptions): Promise<SortResult>;
|
|
706
|
+
first(): Promise<Row | undefined>;
|
|
707
|
+
count(): Promise<number>;
|
|
708
|
+
streamNdjson(): ReadableStream<Uint8Array>;
|
|
709
|
+
streamJson(): ReadableStream<Uint8Array>;
|
|
710
|
+
streamCsv(options?: CsvStreamOptions): ReadableStream<Uint8Array>;
|
|
711
|
+
resumableBatches(options: ResumableBatchOptions): AsyncIterable<SliceResult>;
|
|
712
|
+
groupBy(columns: string[]): AggregationBuilder;
|
|
713
|
+
}
|
|
714
|
+
declare class AggregationBuilder {
|
|
715
|
+
private readonly result;
|
|
716
|
+
private readonly columns;
|
|
717
|
+
constructor(result: QueryResult, columns: string[]);
|
|
718
|
+
aggregate(spec: AggregateSpec, options?: AggregateOptions): Promise<Row[]>;
|
|
719
|
+
aggregateWithState(spec: AggregateSpec, options?: AggregateOptions): Promise<AggregateResult>;
|
|
720
|
+
}
|
|
721
|
+
interface QueryResultConfig extends PathQueryInit {
|
|
722
|
+
lake: Lake;
|
|
723
|
+
bookmarkQuery: BookmarkQuery;
|
|
724
|
+
scanner: ScanAdapter;
|
|
725
|
+
budget: QueryBudget;
|
|
726
|
+
now: () => number;
|
|
727
|
+
queryId: string;
|
|
728
|
+
metrics?: MetricsHook;
|
|
729
|
+
sidecarIndex?: SidecarFileIndex[];
|
|
730
|
+
}
|
|
731
|
+
declare class QueryResult {
|
|
732
|
+
readonly stats: QueryStats;
|
|
733
|
+
private readonly config;
|
|
734
|
+
constructor(config: QueryResultConfig);
|
|
735
|
+
rows(): AsyncIterable<Row>;
|
|
736
|
+
private matchedRows;
|
|
737
|
+
batches(): AsyncIterable<Row[]>;
|
|
738
|
+
toArray(): Promise<Row[]>;
|
|
739
|
+
topKWithState(options?: TopKOptions): Promise<TopKResult>;
|
|
740
|
+
sortWithState(options?: SortOptions): Promise<SortResult>;
|
|
741
|
+
first(): Promise<Row | undefined>;
|
|
742
|
+
count(): Promise<number>;
|
|
743
|
+
planTasks(): Promise<TaskInput[]>;
|
|
744
|
+
taskManifest(jobId?: string): Promise<TaskManifest>;
|
|
745
|
+
slice(options: SliceOptions): Promise<SliceResult>;
|
|
746
|
+
resumableBatches(options: ResumableBatchOptions): AsyncIterable<SliceResult>;
|
|
747
|
+
aggregate(groupColumns: string[], spec: AggregateSpec, options?: AggregateOptions): Promise<Row[]>;
|
|
748
|
+
aggregateWithState(groupColumns: string[], spec: AggregateSpec, options?: AggregateOptions): Promise<AggregateResult>;
|
|
749
|
+
private orderedBatches;
|
|
750
|
+
private collectOrderedMatches;
|
|
751
|
+
private collectSortRuns;
|
|
752
|
+
explain(): Promise<ExplainResult>;
|
|
753
|
+
private tasksFromObjects;
|
|
754
|
+
private planObjects;
|
|
755
|
+
streamNdjson(): ReadableStream<Uint8Array>;
|
|
756
|
+
streamJson(): ReadableStream<Uint8Array>;
|
|
757
|
+
streamCsv(options?: CsvStreamOptions): ReadableStream<Uint8Array>;
|
|
758
|
+
}
|
|
759
|
+
declare function serializeAggregateOperatorState(state: AggregateOperatorState): Uint8Array;
|
|
760
|
+
declare function deserializeAggregateOperatorState(bytes: Uint8Array | AggregateOperatorState): AggregateOperatorState;
|
|
761
|
+
declare function serializeTopKOperatorState(state: TopKOperatorState): Uint8Array;
|
|
762
|
+
declare function serializeSortOperatorState(state: SortOperatorState): Uint8Array;
|
|
763
|
+
declare function deserializeTopKOperatorState(bytes: Uint8Array | TopKOperatorState): TopKOperatorState;
|
|
764
|
+
declare function deserializeSortOperatorState(bytes: Uint8Array | SortOperatorState): SortOperatorState;
|
|
765
|
+
declare function parseJsonQuery(input: unknown): PathQueryInit;
|
|
766
|
+
declare function parseHivePartitions(path: string): Record<string, string>;
|
|
767
|
+
|
|
768
|
+
interface TaskManifestTask {
|
|
769
|
+
id: string;
|
|
770
|
+
input: TaskInput;
|
|
771
|
+
outputRole: "rows" | "data-file" | "manifest";
|
|
772
|
+
}
|
|
773
|
+
interface TaskManifest {
|
|
774
|
+
version: 1;
|
|
775
|
+
jobId: string;
|
|
776
|
+
planFingerprint: string;
|
|
777
|
+
snapshot: string;
|
|
778
|
+
tasks: TaskManifestTask[];
|
|
779
|
+
}
|
|
780
|
+
interface OutputManifestEntry {
|
|
781
|
+
taskId: string;
|
|
782
|
+
outputPath: string;
|
|
783
|
+
partitionValues: Record<string, string>;
|
|
784
|
+
rowCount: number;
|
|
785
|
+
byteSize: number;
|
|
786
|
+
contentHash?: string;
|
|
787
|
+
etag?: string;
|
|
788
|
+
iceberg?: {
|
|
789
|
+
recordCount: number;
|
|
790
|
+
fileSizeInBytes: number;
|
|
791
|
+
partitionValues: Record<string, string>;
|
|
792
|
+
};
|
|
793
|
+
}
|
|
794
|
+
interface OutputManifest {
|
|
795
|
+
version: 1;
|
|
796
|
+
jobId: string;
|
|
797
|
+
planFingerprint: string;
|
|
798
|
+
entries: OutputManifestEntry[];
|
|
799
|
+
}
|
|
800
|
+
type TaskState = "planned" | "running" | "output-written" | "manifest-recorded" | "complete";
|
|
801
|
+
interface TaskCheckpoint {
|
|
802
|
+
taskId: string;
|
|
803
|
+
state: TaskState;
|
|
804
|
+
idempotencyKey: string;
|
|
805
|
+
updatedAtMs: number;
|
|
806
|
+
output?: OutputManifestEntry;
|
|
807
|
+
outputs?: OutputManifestEntry[];
|
|
808
|
+
}
|
|
809
|
+
interface CheckpointAdapter {
|
|
810
|
+
get(taskId: string): Promise<TaskCheckpoint | undefined>;
|
|
811
|
+
put(checkpoint: TaskCheckpoint): Promise<void>;
|
|
812
|
+
list(jobId?: string): AsyncIterable<TaskCheckpoint>;
|
|
813
|
+
}
|
|
814
|
+
interface TaskTransitionInput {
|
|
815
|
+
taskId: string;
|
|
816
|
+
nextState: TaskState;
|
|
817
|
+
idempotencyKey: string;
|
|
818
|
+
nowMs: number;
|
|
819
|
+
staleTimeoutMs?: number;
|
|
820
|
+
output?: OutputManifestEntry;
|
|
821
|
+
outputs?: OutputManifestEntry[];
|
|
822
|
+
}
|
|
823
|
+
interface BookmarkPosition {
|
|
824
|
+
fileIndex: number;
|
|
825
|
+
rowGroup: number;
|
|
826
|
+
rowOffset: number;
|
|
827
|
+
taskId?: string;
|
|
828
|
+
outputManifestCursor?: number;
|
|
829
|
+
}
|
|
830
|
+
interface BookmarkInit {
|
|
831
|
+
planFingerprint: string;
|
|
832
|
+
snapshot: string;
|
|
833
|
+
query?: BookmarkQuery;
|
|
834
|
+
position: BookmarkPosition;
|
|
835
|
+
writeState?: Bookmark["writeState"];
|
|
836
|
+
operatorState?: Bookmark["operatorState"];
|
|
837
|
+
}
|
|
838
|
+
declare function createTaskManifest(input: {
|
|
839
|
+
jobId: string;
|
|
840
|
+
snapshot?: string;
|
|
841
|
+
tasks: TaskInput[];
|
|
842
|
+
outputRole?: TaskManifestTask["outputRole"];
|
|
843
|
+
}): TaskManifest;
|
|
844
|
+
declare function createOutputManifest(input: {
|
|
845
|
+
jobId: string;
|
|
846
|
+
planFingerprint: string;
|
|
847
|
+
entries: OutputManifestEntry[];
|
|
848
|
+
}): OutputManifest;
|
|
849
|
+
declare function createOutputManifestFromCheckpoints(input: {
|
|
850
|
+
jobId: string;
|
|
851
|
+
planFingerprint: string;
|
|
852
|
+
checkpoints: CheckpointAdapter;
|
|
853
|
+
}): Promise<OutputManifest>;
|
|
854
|
+
declare function writeOutputManifest(store: ObjectStore, path: string, manifest: OutputManifest): Promise<{
|
|
855
|
+
path: string;
|
|
856
|
+
byteSize: number;
|
|
857
|
+
etag?: string;
|
|
858
|
+
}>;
|
|
859
|
+
declare function readOutputManifest(store: ObjectStore, path: string): Promise<OutputManifest>;
|
|
860
|
+
declare function createBookmark(init: BookmarkInit): Bookmark;
|
|
861
|
+
declare function assertBookmarkMatches(bookmark: Bookmark, planFingerprint: string): void;
|
|
862
|
+
declare function signPaginationToken(bookmark: Bookmark, secret: string | Uint8Array): Promise<string>;
|
|
863
|
+
declare function verifyPaginationToken(token: string, secret: string | Uint8Array): Promise<Bookmark>;
|
|
864
|
+
declare class MemoryCheckpointAdapter implements CheckpointAdapter {
|
|
865
|
+
private readonly checkpoints;
|
|
866
|
+
private readonly taskJobs;
|
|
867
|
+
get(taskId: string): Promise<TaskCheckpoint | undefined>;
|
|
868
|
+
put(checkpoint: TaskCheckpoint): Promise<void>;
|
|
869
|
+
list(jobId?: string): AsyncIterable<TaskCheckpoint>;
|
|
870
|
+
}
|
|
871
|
+
declare function memoryCheckpointAdapter(): MemoryCheckpointAdapter;
|
|
872
|
+
declare function advanceTaskCheckpoint(checkpoints: CheckpointAdapter, input: TaskTransitionInput): Promise<TaskCheckpoint>;
|
|
873
|
+
declare function transitionTaskCheckpoint(existing: TaskCheckpoint | undefined, input: TaskTransitionInput): TaskCheckpoint;
|
|
874
|
+
declare function stableStringify(value: unknown): string;
|
|
875
|
+
declare function fingerprint(value: unknown): string;
|
|
876
|
+
|
|
877
|
+
/**
|
|
878
|
+
* In-memory ObjectStore. The reference implementation of the store
|
|
879
|
+
* contract, used by tests and usable as a small read-through cache.
|
|
880
|
+
*/
|
|
881
|
+
declare class MemoryObjectStore implements ObjectStore {
|
|
882
|
+
private readonly objects;
|
|
883
|
+
private version;
|
|
884
|
+
get(path: string): Promise<Uint8Array | null>;
|
|
885
|
+
getRange(path: string, range: {
|
|
886
|
+
offset: number;
|
|
887
|
+
length: number;
|
|
888
|
+
}): Promise<Uint8Array>;
|
|
889
|
+
put(path: string, body: Uint8Array | ReadableStream<Uint8Array>, options?: PutOptions): Promise<void>;
|
|
890
|
+
conditionalPut(path: string, body: Uint8Array | ReadableStream<Uint8Array>, options: ConditionalPutOptions): Promise<boolean>;
|
|
891
|
+
private writeObject;
|
|
892
|
+
delete(path: string): Promise<void>;
|
|
893
|
+
list(prefix: string, options?: ListOptions): AsyncIterable<ObjectInfo>;
|
|
894
|
+
head(path: string): Promise<ObjectHead | null>;
|
|
895
|
+
}
|
|
896
|
+
declare function memoryStore(): MemoryObjectStore;
|
|
897
|
+
|
|
898
|
+
declare const PACKAGE: "@laql/iceberg";
|
|
899
|
+
type IcebergReadMode = "strict" | "ignore-deletes" | "ignore-unsupported-deletes";
|
|
900
|
+
interface LoadIcebergTableOptions extends ObjectStoreReadControls {
|
|
901
|
+
store: ObjectStore;
|
|
902
|
+
metadataPath: string;
|
|
903
|
+
}
|
|
904
|
+
interface LoadIcebergTableFromObjectStoreOptions extends ObjectStoreReadControls {
|
|
905
|
+
store: ObjectStore;
|
|
906
|
+
tableLocation: string;
|
|
907
|
+
}
|
|
908
|
+
interface IcebergRestCatalogOptions {
|
|
909
|
+
url: string;
|
|
910
|
+
namespace: string | string[];
|
|
911
|
+
table: string;
|
|
912
|
+
prefix?: string;
|
|
913
|
+
token?: string;
|
|
914
|
+
fetch?: typeof fetch;
|
|
915
|
+
}
|
|
916
|
+
interface IcebergGlueCatalogOptions {
|
|
917
|
+
region: string;
|
|
918
|
+
namespace: string | string[];
|
|
919
|
+
table: string;
|
|
920
|
+
catalogId?: string;
|
|
921
|
+
}
|
|
922
|
+
interface IcebergNessieCatalogOptions {
|
|
923
|
+
url: string;
|
|
924
|
+
namespace: string | string[];
|
|
925
|
+
table: string;
|
|
926
|
+
ref?: string;
|
|
927
|
+
token?: string;
|
|
928
|
+
}
|
|
929
|
+
interface LoadIcebergTableFromRestOptions extends IcebergRestCatalogOptions, ObjectStoreReadControls {
|
|
930
|
+
store: ObjectStore;
|
|
931
|
+
}
|
|
932
|
+
interface PlanIcebergFilesOptions {
|
|
933
|
+
snapshotId?: number;
|
|
934
|
+
asOfTimestampMs?: number;
|
|
935
|
+
ref?: string;
|
|
936
|
+
where?: Expr;
|
|
937
|
+
select?: string[];
|
|
938
|
+
readMode?: IcebergReadMode;
|
|
939
|
+
}
|
|
940
|
+
interface ProjectIcebergRowOptions {
|
|
941
|
+
snapshotId?: number;
|
|
942
|
+
asOfTimestampMs?: number;
|
|
943
|
+
ref?: string;
|
|
944
|
+
select?: string[];
|
|
945
|
+
}
|
|
946
|
+
interface IcebergAppendFile {
|
|
947
|
+
path: string;
|
|
948
|
+
partition?: Record<string, string>;
|
|
949
|
+
recordCount: number;
|
|
950
|
+
fileSizeInBytes: number;
|
|
951
|
+
}
|
|
952
|
+
interface IcebergAppendOptions {
|
|
953
|
+
files: IcebergAppendFile[];
|
|
954
|
+
jobId?: string;
|
|
955
|
+
nowMs?: number;
|
|
956
|
+
nextSnapshotId?: number;
|
|
957
|
+
catalog?: IcebergCommitCatalog;
|
|
958
|
+
}
|
|
959
|
+
interface IcebergAppendOutputManifestOptions extends Omit<IcebergAppendOptions, "files" | "jobId"> {
|
|
960
|
+
manifest: OutputManifest;
|
|
961
|
+
}
|
|
962
|
+
interface IcebergCommitCatalog {
|
|
963
|
+
commitAppend(input: IcebergCommitInput): Promise<boolean | IcebergCommitResult>;
|
|
964
|
+
}
|
|
965
|
+
interface IcebergCatalog extends IcebergCommitCatalog {
|
|
966
|
+
loadTable(store: ObjectStore): Promise<IcebergTable>;
|
|
967
|
+
listTables(): Promise<IcebergTableIdentifier[]>;
|
|
968
|
+
}
|
|
969
|
+
interface IcebergTableIdentifier {
|
|
970
|
+
namespace: string[];
|
|
971
|
+
name: string;
|
|
972
|
+
}
|
|
973
|
+
interface IcebergCommitResult {
|
|
974
|
+
committed: boolean;
|
|
975
|
+
metadataPath?: string;
|
|
976
|
+
}
|
|
977
|
+
interface IcebergCommitInput {
|
|
978
|
+
store: ObjectStore;
|
|
979
|
+
currentMetadataPath: string;
|
|
980
|
+
nextMetadataPath: string;
|
|
981
|
+
expectedSnapshotId: number;
|
|
982
|
+
nextSnapshotId: number;
|
|
983
|
+
manifestPath: string;
|
|
984
|
+
manifest: Manifest;
|
|
985
|
+
metadata: MetadataFile;
|
|
986
|
+
}
|
|
987
|
+
interface IcebergAppendResult {
|
|
988
|
+
snapshotId: number;
|
|
989
|
+
previousSnapshotId: number;
|
|
990
|
+
metadataPath: string;
|
|
991
|
+
manifestPath: string;
|
|
992
|
+
files: PlannedIcebergFile[];
|
|
993
|
+
}
|
|
994
|
+
interface IcebergField {
|
|
995
|
+
id: number;
|
|
996
|
+
name: string;
|
|
997
|
+
sourceId?: number;
|
|
998
|
+
type: string;
|
|
999
|
+
required: boolean;
|
|
1000
|
+
}
|
|
1001
|
+
interface PlannedIcebergFile {
|
|
1002
|
+
path: string;
|
|
1003
|
+
sequenceNumber: number;
|
|
1004
|
+
partition: Record<string, string>;
|
|
1005
|
+
recordCount: number;
|
|
1006
|
+
fileSizeInBytes?: number;
|
|
1007
|
+
projectedFieldIds: number[];
|
|
1008
|
+
snapshotId: number;
|
|
1009
|
+
deleteFiles?: IcebergDeleteFile[];
|
|
1010
|
+
}
|
|
1011
|
+
interface IcebergDeleteFile {
|
|
1012
|
+
content: "position-delete" | "equality-delete" | "deletion-vector";
|
|
1013
|
+
path: string;
|
|
1014
|
+
}
|
|
1015
|
+
interface IcebergPlan {
|
|
1016
|
+
snapshotId: number;
|
|
1017
|
+
schemaId: number;
|
|
1018
|
+
manifestsRead: number;
|
|
1019
|
+
manifestsSkipped: number;
|
|
1020
|
+
filesPlanned: number;
|
|
1021
|
+
filesSkipped: number;
|
|
1022
|
+
deleteFilesPlanned: number;
|
|
1023
|
+
deleteFilesIgnored: number;
|
|
1024
|
+
files: PlannedIcebergFile[];
|
|
1025
|
+
}
|
|
1026
|
+
interface IcebergPositionDelete {
|
|
1027
|
+
path: string;
|
|
1028
|
+
position: number;
|
|
1029
|
+
}
|
|
1030
|
+
interface IcebergEqualityDelete {
|
|
1031
|
+
columns: string[];
|
|
1032
|
+
row: Row;
|
|
1033
|
+
}
|
|
1034
|
+
interface IcebergDeletionVector {
|
|
1035
|
+
path: string;
|
|
1036
|
+
positions: number[];
|
|
1037
|
+
}
|
|
1038
|
+
interface ApplyIcebergDeletesOptions {
|
|
1039
|
+
dataFilePath: string;
|
|
1040
|
+
rows: Row[];
|
|
1041
|
+
rowOffset?: number;
|
|
1042
|
+
positionDeletes?: IcebergPositionDelete[];
|
|
1043
|
+
equalityDeletes?: IcebergEqualityDelete[];
|
|
1044
|
+
deletionVectors?: IcebergDeletionVector[];
|
|
1045
|
+
}
|
|
1046
|
+
interface DecodedIcebergDeletes {
|
|
1047
|
+
positionDeletes?: IcebergPositionDelete[];
|
|
1048
|
+
equalityDeletes?: IcebergEqualityDelete[];
|
|
1049
|
+
deletionVectors?: IcebergDeletionVector[];
|
|
1050
|
+
}
|
|
1051
|
+
interface IcebergRowBatch {
|
|
1052
|
+
rowOffset: number;
|
|
1053
|
+
rows: Row[];
|
|
1054
|
+
}
|
|
1055
|
+
interface ScanPlannedIcebergRowsOptions extends ObjectStoreReadControls {
|
|
1056
|
+
plan: IcebergPlan | PlannedIcebergFile[];
|
|
1057
|
+
readDataFile(file: PlannedIcebergFile): Promise<Row[] | AsyncIterable<Row[] | IcebergRowBatch>>;
|
|
1058
|
+
readDeleteFile(deleteFile: IcebergDeleteFile, dataFile: PlannedIcebergFile): Promise<DecodedIcebergDeletes>;
|
|
1059
|
+
}
|
|
1060
|
+
interface MetadataFile {
|
|
1061
|
+
"format-version": number;
|
|
1062
|
+
"table-uuid": string;
|
|
1063
|
+
location: string;
|
|
1064
|
+
"current-snapshot-id": number;
|
|
1065
|
+
refs?: Record<string, {
|
|
1066
|
+
type: "branch" | "tag";
|
|
1067
|
+
"snapshot-id": number;
|
|
1068
|
+
}>;
|
|
1069
|
+
schemas: {
|
|
1070
|
+
"schema-id": number;
|
|
1071
|
+
fields: IcebergField[];
|
|
1072
|
+
}[];
|
|
1073
|
+
snapshots: Snapshot[];
|
|
1074
|
+
"partition-specs"?: IcebergPartitionSpec[];
|
|
1075
|
+
"sort-orders"?: IcebergSortOrder[];
|
|
1076
|
+
}
|
|
1077
|
+
interface IcebergPartitionSpec {
|
|
1078
|
+
"spec-id": number;
|
|
1079
|
+
fields: IcebergPartitionField[];
|
|
1080
|
+
}
|
|
1081
|
+
interface IcebergPartitionField {
|
|
1082
|
+
"source-id": number;
|
|
1083
|
+
"field-id": number;
|
|
1084
|
+
name: string;
|
|
1085
|
+
transform: string;
|
|
1086
|
+
}
|
|
1087
|
+
interface IcebergSortOrder {
|
|
1088
|
+
"order-id": number;
|
|
1089
|
+
fields: unknown[];
|
|
1090
|
+
}
|
|
1091
|
+
interface Snapshot {
|
|
1092
|
+
"snapshot-id": number;
|
|
1093
|
+
"timestamp-ms": number;
|
|
1094
|
+
"schema-id": number;
|
|
1095
|
+
"manifest-list"?: string;
|
|
1096
|
+
manifests?: Manifest[];
|
|
1097
|
+
}
|
|
1098
|
+
interface Manifest {
|
|
1099
|
+
path: string;
|
|
1100
|
+
files: ManifestFile[];
|
|
1101
|
+
deleteFiles?: ManifestDeleteFile[];
|
|
1102
|
+
}
|
|
1103
|
+
interface ManifestFile {
|
|
1104
|
+
path: string;
|
|
1105
|
+
sequenceNumber: number;
|
|
1106
|
+
partition?: Record<string, string>;
|
|
1107
|
+
recordCount: number;
|
|
1108
|
+
fileSizeInBytes?: number;
|
|
1109
|
+
deleteFiles?: ManifestDeleteFile[];
|
|
1110
|
+
}
|
|
1111
|
+
interface ManifestDeleteFile {
|
|
1112
|
+
content: string;
|
|
1113
|
+
path: string;
|
|
1114
|
+
partition?: Record<string, string>;
|
|
1115
|
+
}
|
|
1116
|
+
declare class IcebergTable {
|
|
1117
|
+
private readonly store;
|
|
1118
|
+
readonly metadataPath: string;
|
|
1119
|
+
readonly metadata: MetadataFile;
|
|
1120
|
+
constructor(store: ObjectStore, metadataPath: string, metadata: MetadataFile);
|
|
1121
|
+
snapshot(options?: PlanIcebergFilesOptions): Snapshot;
|
|
1122
|
+
schema(schemaId: number): IcebergField[];
|
|
1123
|
+
projectRow(row: Row, options?: ProjectIcebergRowOptions): Row;
|
|
1124
|
+
planFiles(options?: PlanIcebergFilesOptions): IcebergPlan;
|
|
1125
|
+
appendFiles(options: IcebergAppendOptions): Promise<IcebergAppendResult>;
|
|
1126
|
+
appendOutputManifest(options: IcebergAppendOutputManifestOptions): Promise<IcebergAppendResult>;
|
|
1127
|
+
private snapshotById;
|
|
1128
|
+
private fieldNameById;
|
|
1129
|
+
}
|
|
1130
|
+
declare class ObjectStoreIcebergCommitCatalog implements IcebergCommitCatalog {
|
|
1131
|
+
commitAppend(input: IcebergCommitInput): Promise<IcebergCommitResult>;
|
|
1132
|
+
}
|
|
1133
|
+
declare class IcebergRestCatalog implements IcebergCatalog {
|
|
1134
|
+
private readonly url;
|
|
1135
|
+
private readonly namespace;
|
|
1136
|
+
private readonly table;
|
|
1137
|
+
private readonly prefix;
|
|
1138
|
+
private readonly token;
|
|
1139
|
+
private readonly fetchFn;
|
|
1140
|
+
constructor(options: IcebergRestCatalogOptions);
|
|
1141
|
+
loadTable(store: ObjectStore): Promise<IcebergTable>;
|
|
1142
|
+
listTables(): Promise<IcebergTableIdentifier[]>;
|
|
1143
|
+
commitAppend(input: IcebergCommitInput): Promise<IcebergCommitResult>;
|
|
1144
|
+
private requestJson;
|
|
1145
|
+
private headers;
|
|
1146
|
+
private tableUrl;
|
|
1147
|
+
private namespaceTablesUrl;
|
|
1148
|
+
private namespaceTableSegments;
|
|
1149
|
+
}
|
|
1150
|
+
declare function icebergRestCatalog(options: IcebergRestCatalogOptions): IcebergRestCatalog;
|
|
1151
|
+
declare class IcebergUnsupportedCatalog implements IcebergCatalog {
|
|
1152
|
+
private readonly catalog;
|
|
1153
|
+
private readonly namespace;
|
|
1154
|
+
private readonly table;
|
|
1155
|
+
constructor(catalog: string, namespace: string | string[], table: string);
|
|
1156
|
+
loadTable(_store: ObjectStore): Promise<IcebergTable>;
|
|
1157
|
+
listTables(): Promise<IcebergTableIdentifier[]>;
|
|
1158
|
+
commitAppend(_input: IcebergCommitInput): Promise<IcebergCommitResult>;
|
|
1159
|
+
private unsupported;
|
|
1160
|
+
}
|
|
1161
|
+
declare function icebergGlueCatalog(options: IcebergGlueCatalogOptions): IcebergCatalog;
|
|
1162
|
+
declare function icebergNessieCatalog(options: IcebergNessieCatalogOptions): IcebergCatalog;
|
|
1163
|
+
declare function loadIcebergTable(options: LoadIcebergTableOptions): Promise<IcebergTable>;
|
|
1164
|
+
declare function loadIcebergTableFromObjectStore(options: LoadIcebergTableFromObjectStoreOptions): Promise<IcebergTable>;
|
|
1165
|
+
declare function loadIcebergTableFromRest(options: LoadIcebergTableFromRestOptions): Promise<IcebergTable>;
|
|
1166
|
+
declare function applyIcebergDeletes(options: ApplyIcebergDeletesOptions): Row[];
|
|
1167
|
+
declare function scanPlannedIcebergRows(options: ScanPlannedIcebergRowsOptions): AsyncIterable<Row[]>;
|
|
1168
|
+
|
|
1169
|
+
interface ReadParquetOptions {
|
|
1170
|
+
/** Columns to project; all columns when omitted. */
|
|
1171
|
+
columns?: string[];
|
|
1172
|
+
rowStart?: number;
|
|
1173
|
+
rowEnd?: number;
|
|
1174
|
+
}
|
|
1175
|
+
interface ReadParquetBatchOptions extends ReadParquetOptions {
|
|
1176
|
+
batchSize?: number;
|
|
1177
|
+
where?: Expr;
|
|
1178
|
+
}
|
|
1179
|
+
interface ParquetRowBatch {
|
|
1180
|
+
rowOffset: number;
|
|
1181
|
+
rows: Row[];
|
|
1182
|
+
}
|
|
1183
|
+
interface PlanParquetRowGroupsOptions {
|
|
1184
|
+
where?: Expr;
|
|
1185
|
+
}
|
|
1186
|
+
interface PlannedParquetRowGroup {
|
|
1187
|
+
index: number;
|
|
1188
|
+
rowStart: number;
|
|
1189
|
+
rowCount: number;
|
|
1190
|
+
byteRange?: {
|
|
1191
|
+
offset: number;
|
|
1192
|
+
length: number;
|
|
1193
|
+
};
|
|
1194
|
+
}
|
|
1195
|
+
interface ParquetRowGroupPlan {
|
|
1196
|
+
rowGroups: PlannedParquetRowGroup[];
|
|
1197
|
+
rowGroupRanges: {
|
|
1198
|
+
start: number;
|
|
1199
|
+
end: number;
|
|
1200
|
+
}[];
|
|
1201
|
+
}
|
|
1202
|
+
type IcebergParquetDeleteFileContent = "position-delete" | "equality-delete" | "deletion-vector";
|
|
1203
|
+
interface IcebergParquetDeleteFile {
|
|
1204
|
+
content: IcebergParquetDeleteFileContent;
|
|
1205
|
+
path: string;
|
|
1206
|
+
}
|
|
1207
|
+
interface DecodedIcebergParquetDeletes {
|
|
1208
|
+
positionDeletes?: {
|
|
1209
|
+
path: string;
|
|
1210
|
+
position: number;
|
|
1211
|
+
}[];
|
|
1212
|
+
equalityDeletes?: {
|
|
1213
|
+
columns: string[];
|
|
1214
|
+
row: Row;
|
|
1215
|
+
}[];
|
|
1216
|
+
}
|
|
1217
|
+
interface WriteParquetOptions extends Omit<ParquetWriteOptions, "writer" | "columnData"> {
|
|
1218
|
+
columnData: ColumnSource[];
|
|
1219
|
+
contentType?: string;
|
|
1220
|
+
writeMode?: "overwrite" | "create";
|
|
1221
|
+
}
|
|
1222
|
+
interface WriteParquetRowsOptions extends Omit<WriteParquetOptions, "columnData" | "schema"> {
|
|
1223
|
+
rows: Row[];
|
|
1224
|
+
partitionBy?: string[];
|
|
1225
|
+
maxRowsPerFile?: number;
|
|
1226
|
+
maxBytesPerFile?: number;
|
|
1227
|
+
jobId?: string;
|
|
1228
|
+
taskId?: string;
|
|
1229
|
+
idempotencyKey?: string;
|
|
1230
|
+
columnTypes?: Record<string, BasicType>;
|
|
1231
|
+
validation?: InsertValidationRules;
|
|
1232
|
+
}
|
|
1233
|
+
interface InsertValidationRules {
|
|
1234
|
+
required?: string[];
|
|
1235
|
+
unique?: string[][];
|
|
1236
|
+
ranges?: Record<string, {
|
|
1237
|
+
min?: ComparableInsertValue;
|
|
1238
|
+
max?: ComparableInsertValue;
|
|
1239
|
+
}>;
|
|
1240
|
+
enums?: Record<string, InsertValue[]>;
|
|
1241
|
+
}
|
|
1242
|
+
interface WritePartitionedParquetFile {
|
|
1243
|
+
path: string;
|
|
1244
|
+
byteSize: number;
|
|
1245
|
+
contentHash: string;
|
|
1246
|
+
etag?: string;
|
|
1247
|
+
rowCount: number;
|
|
1248
|
+
partitionValues: Record<string, string>;
|
|
1249
|
+
}
|
|
1250
|
+
interface WritePartitionedParquetResult {
|
|
1251
|
+
files: WritePartitionedParquetFile[];
|
|
1252
|
+
}
|
|
1253
|
+
interface WritePartitionedParquetTaskOptions extends WriteParquetRowsOptions {
|
|
1254
|
+
checkpoints: CheckpointAdapter;
|
|
1255
|
+
taskId: string;
|
|
1256
|
+
idempotencyKey: string;
|
|
1257
|
+
nowMs?: number;
|
|
1258
|
+
staleTimeoutMs?: number;
|
|
1259
|
+
iceberg?: boolean;
|
|
1260
|
+
}
|
|
1261
|
+
interface WritePartitionedParquetTaskResult {
|
|
1262
|
+
result: WritePartitionedParquetResult;
|
|
1263
|
+
entries: OutputManifestEntry[];
|
|
1264
|
+
}
|
|
1265
|
+
interface CreateParquetTableAsQuery {
|
|
1266
|
+
toArray(): Promise<Row[]>;
|
|
1267
|
+
}
|
|
1268
|
+
interface CreateParquetTableAsOptions extends Omit<WritePartitionedParquetTaskOptions, "rows" | "jobId" | "taskId" | "idempotencyKey"> {
|
|
1269
|
+
query: CreateParquetTableAsQuery;
|
|
1270
|
+
jobId: string;
|
|
1271
|
+
planFingerprint: string;
|
|
1272
|
+
taskId?: string;
|
|
1273
|
+
idempotencyKey: string;
|
|
1274
|
+
}
|
|
1275
|
+
interface CreateParquetTableAsResult extends WritePartitionedParquetTaskResult {
|
|
1276
|
+
manifest: OutputManifest;
|
|
1277
|
+
rowsRead: number;
|
|
1278
|
+
}
|
|
1279
|
+
interface PartitionedParquetOutputEntryOptions {
|
|
1280
|
+
taskId: string | ((file: WritePartitionedParquetFile, index: number) => string);
|
|
1281
|
+
iceberg?: boolean;
|
|
1282
|
+
}
|
|
1283
|
+
interface ParquetLakeConfig extends Omit<LakeConfig, "scanner"> {
|
|
1284
|
+
batchSize?: number;
|
|
1285
|
+
metadataCache?: CacheAdapter<ParquetMetadata>;
|
|
1286
|
+
}
|
|
1287
|
+
type ParquetMetadata = Awaited<ReturnType<typeof parquetMetadataAsync>>;
|
|
1288
|
+
interface StoreAsyncBuffer {
|
|
1289
|
+
byteLength: number;
|
|
1290
|
+
etag?: string;
|
|
1291
|
+
slice(start: number, end?: number): Promise<ArrayBuffer>;
|
|
1292
|
+
}
|
|
1293
|
+
/**
|
|
1294
|
+
* Bridge an ObjectStore path to hyparquet's AsyncBuffer (length + ranged slice).
|
|
1295
|
+
*/
|
|
1296
|
+
declare function asyncBufferFromStore(store: ObjectStore, path: string, options?: ScanOptions | undefined): Promise<StoreAsyncBuffer>;
|
|
1297
|
+
/**
|
|
1298
|
+
* Read rows from a Parquet object. Early scaffold: full planner-driven
|
|
1299
|
+
* row-group pruning and batch streaming land in phase 1-2 (see BUILD_PLAN.md).
|
|
1300
|
+
*/
|
|
1301
|
+
declare function readParquetObjects(store: ObjectStore, path: string, options?: ReadParquetOptions): Promise<Record<string, unknown>[]>;
|
|
1302
|
+
declare function readParquetObjectBatches(store: ObjectStore, path: string, options?: ReadParquetBatchOptions): AsyncIterable<ParquetRowBatch>;
|
|
1303
|
+
/** Read Parquet footer metadata (row groups, schema, stats). */
|
|
1304
|
+
declare function readParquetMetadata(store: ObjectStore, path: string): Promise<hyparquet_src_types_js.FileMetaData>;
|
|
1305
|
+
declare function planRowGroups(store: ObjectStore, path: string, options?: PlanParquetRowGroupsOptions): Promise<ParquetRowGroupPlan>;
|
|
1306
|
+
declare function readIcebergParquetDeletes(store: ObjectStore, deleteFile: IcebergParquetDeleteFile): Promise<DecodedIcebergParquetDeletes>;
|
|
1307
|
+
declare function writeParquet(store: ObjectStore, path: string, options: WriteParquetOptions): Promise<{
|
|
1308
|
+
path: string;
|
|
1309
|
+
byteSize: number;
|
|
1310
|
+
etag?: string;
|
|
1311
|
+
}>;
|
|
1312
|
+
declare function writePartitionedParquet(store: ObjectStore, prefix: string, options: WriteParquetRowsOptions): Promise<WritePartitionedParquetResult>;
|
|
1313
|
+
declare function writePartitionedParquetTask(store: ObjectStore, prefix: string, options: WritePartitionedParquetTaskOptions): Promise<WritePartitionedParquetTaskResult>;
|
|
1314
|
+
declare function createParquetTableAs(store: ObjectStore, prefix: string, options: CreateParquetTableAsOptions): Promise<CreateParquetTableAsResult>;
|
|
1315
|
+
declare function partitionedParquetOutputEntries(result: WritePartitionedParquetResult, options: PartitionedParquetOutputEntryOptions): OutputManifestEntry[];
|
|
1316
|
+
declare class ParquetScanAdapter implements ScanAdapter {
|
|
1317
|
+
private readonly store;
|
|
1318
|
+
private readonly defaultBatchSize;
|
|
1319
|
+
private readonly metadataCache;
|
|
1320
|
+
constructor(store: ObjectStore, options?: {
|
|
1321
|
+
batchSize?: number;
|
|
1322
|
+
metadataCache?: CacheAdapter<ParquetMetadata>;
|
|
1323
|
+
});
|
|
1324
|
+
scan(path: string, options: ScanOptions): AsyncIterable<Row[]>;
|
|
1325
|
+
planTask(path: string, options: ScanTaskPlanOptions): Promise<ScanTaskPlan>;
|
|
1326
|
+
private metadata;
|
|
1327
|
+
}
|
|
1328
|
+
declare function planRowGroupsFromMetadata(metadata: ParquetMetadata, where: Expr | undefined): ParquetRowGroupPlan;
|
|
1329
|
+
declare function rejectUnsupportedParquetSchema(metadata: ParquetMetadata): void;
|
|
1330
|
+
type InsertValue = string | number | boolean | bigint | null;
|
|
1331
|
+
type ComparableInsertValue = string | number | bigint;
|
|
1332
|
+
/** @internal Exposed for pruning tests; not part of the stable public API. */
|
|
1333
|
+
declare function rowGroupMayMatch(rowGroup: RowGroup, expr: Expr | undefined): boolean;
|
|
1334
|
+
declare function parquetScanner(store: ObjectStore, options?: {
|
|
1335
|
+
batchSize?: number;
|
|
1336
|
+
metadataCache?: CacheAdapter<ParquetMetadata>;
|
|
1337
|
+
}): ScanAdapter;
|
|
1338
|
+
declare function createParquetLake(config: ParquetLakeConfig): Lake;
|
|
1339
|
+
|
|
1340
|
+
interface LoadIcebergEngineTableOptions extends LoadIcebergTableOptions {
|
|
1341
|
+
format: "iceberg";
|
|
1342
|
+
}
|
|
1343
|
+
interface LoadParquetEngineTableOptions {
|
|
1344
|
+
format: "parquet";
|
|
1345
|
+
store: ObjectStore;
|
|
1346
|
+
path: string;
|
|
1347
|
+
}
|
|
1348
|
+
type LoadTableOptions = LoadIcebergEngineTableOptions | LoadParquetEngineTableOptions;
|
|
1349
|
+
interface IcebergEngineTable {
|
|
1350
|
+
format: "iceberg";
|
|
1351
|
+
store: ObjectStore;
|
|
1352
|
+
table: IcebergTable;
|
|
1353
|
+
}
|
|
1354
|
+
interface ParquetEngineTable {
|
|
1355
|
+
format: "parquet";
|
|
1356
|
+
store: ObjectStore;
|
|
1357
|
+
path: string;
|
|
1358
|
+
}
|
|
1359
|
+
type EngineTable = IcebergEngineTable | ParquetEngineTable;
|
|
1360
|
+
interface IcebergEnginePlan {
|
|
1361
|
+
format: "iceberg";
|
|
1362
|
+
store: ObjectStore;
|
|
1363
|
+
table: IcebergTable;
|
|
1364
|
+
plan: IcebergPlan;
|
|
1365
|
+
options: PlanIcebergFilesOptions;
|
|
1366
|
+
}
|
|
1367
|
+
interface ParquetEnginePlan {
|
|
1368
|
+
format: "parquet";
|
|
1369
|
+
store: ObjectStore;
|
|
1370
|
+
files: {
|
|
1371
|
+
path: string;
|
|
1372
|
+
}[];
|
|
1373
|
+
}
|
|
1374
|
+
type EngineFilePlan = IcebergEnginePlan | ParquetEnginePlan;
|
|
1375
|
+
interface ScanEngineOptions extends ObjectStoreReadControls {
|
|
1376
|
+
batchSize?: number;
|
|
1377
|
+
}
|
|
1378
|
+
type ScanBatch = Row[];
|
|
1379
|
+
declare function loadTable(options: LoadTableOptions): Promise<EngineTable>;
|
|
1380
|
+
declare function planFiles(table: IcebergEngineTable, options?: PlanIcebergFilesOptions): IcebergEnginePlan;
|
|
1381
|
+
declare function planFiles(table: ParquetEngineTable): ParquetEnginePlan;
|
|
1382
|
+
declare function planFiles(table: EngineTable, options?: PlanIcebergFilesOptions): EngineFilePlan;
|
|
1383
|
+
declare function scanBatches(plan: EngineFilePlan, options?: ScanEngineOptions): AsyncIterable<ScanBatch>;
|
|
1384
|
+
declare function scanRows(plan: EngineFilePlan, options?: ScanEngineOptions): AsyncIterable<Row>;
|
|
1385
|
+
|
|
1386
|
+
export { type AggregateExpr, type AggregateGroupSnapshot, type AggregateOp, type AggregateOperatorState, type AggregateOptions, type AggregateResult, type AggregateSnapshotValue, type AggregateSpec, type AggregateStateSnapshot, AggregationBuilder, type ApplyIcebergDeletesOptions, type BBoxIndex, type BetweenExpr, type Bookmark, type BookmarkInit, type BookmarkPosition, type BookmarkQuery, type BroadcastJoinOptions, type CacheAdapter, CacheApiCache, type CacheApiCacheOptions, type CacheEntry, type CallExpr, type CheckpointAdapter, type CheckpointStore, type Clock, type ColumnExpr, type ColumnInput, type CompareExpr, type CompareOp, type ConditionalObjectStore, type ConditionalPutOptions, type CreateParquetTableAsOptions, type CreateParquetTableAsQuery, type CreateParquetTableAsResult, type CsvStreamOptions, type DecodedIcebergDeletes, type DecodedIcebergParquetDeletes, ERROR_CODES, type EngineFilePlan, type EngineTable, type ErrorDetails, type ExplainJson, type ExplainResult, type Expr, type IcebergAppendFile, type IcebergAppendOptions, type IcebergAppendOutputManifestOptions, type IcebergAppendResult, type IcebergCatalog, type IcebergCommitCatalog, type IcebergCommitInput, type IcebergCommitResult, type IcebergDeleteFile, type IcebergDeletionVector, type IcebergEnginePlan, type IcebergEngineTable, type IcebergEqualityDelete, type IcebergField, type IcebergGlueCatalogOptions, type IcebergNessieCatalogOptions, type IcebergParquetDeleteFile, type IcebergParquetDeleteFileContent, type IcebergPartitionField, type IcebergPartitionSpec, type IcebergPlan, type IcebergPositionDelete, type IcebergReadMode, IcebergRestCatalog, type IcebergRestCatalogOptions, type IcebergRowBatch, type IcebergSortOrder, IcebergTable, type IcebergTableIdentifier, IcebergUnsupportedCatalog, type IdGenerator, type InExpr, type IndexPruneResult, type IndexValue, type InsertValidationRules, type JoinType, type JsonExpr, type JsonOrderByTerm, type JsonQueryV1, LaQLError, type LaQLErrorCode, Lake, type LakeConfig, type LikeExpr, type ListOptions, type LiteralExpr, type LoadIcebergEngineTableOptions, type LoadIcebergTableFromObjectStoreOptions, type LoadIcebergTableFromRestOptions, type LoadIcebergTableOptions, type LoadParquetEngineTableOptions, type LoadTableOptions, type LockAdapter, type LogHook, type LogicalExpr, type LookupJoinFunction, type LookupJoinOptions, type Manifest, type ManifestDeleteFile, type ManifestFile, MemoryCache, MemoryCheckpointAdapter, MemoryObjectStore, MemorySpillAdapter, type MemorySpillAdapterOptions, type MetadataFile, type MetricsHook, type MinMaxColumnIndex, type NotExpr, type NullCheckExpr, type ObjectHead, type ObjectInfo, type ObjectStore, ObjectStoreIcebergCommitCatalog, type ObjectStoreReadControls, type OperatorSnapshotValue, type OrderByTerm, type OutputManifest, type OutputManifestEntry, PACKAGE, type ParquetEnginePlan, type ParquetEngineTable, type ParquetLakeConfig, type ParquetMetadata, type ParquetRowBatch, type ParquetRowGroupPlan, ParquetScanAdapter, type PartitionedParquetOutputEntryOptions, type PathQueryInit, type PlanIcebergFilesOptions, type PlanParquetRowGroupsOptions, type PlannedIcebergFile, type PlannedParquetRowGroup, type PredicatePlan, type PredicatePlanOptions, type ProjectIcebergRowOptions, type PutOptions, type QueryBudget, QueryBuilder, type QueryPolicy, type QueryPolicyContext, QueryResult, type QueryRunOptions, type QueryStats, type QueueAdapter, type ReadParquetBatchOptions, type ReadParquetOptions, type ResumableBatchOptions, ResumedQuery, type Row, type RuntimeSubstrate, type Scalar, type ScanAdapter, type ScanBatch, type ScanEngineOptions, type ScanOptions, type ScanPlannedIcebergRowsOptions, type ScanTaskPlan, type ScanTaskPlanOptions, type SidecarFileIndex, type SliceOptions, type SliceResult, type Snapshot, type SortOperatorState, type SortOptions, type SortResult, type SortRunState, type SpillAdapter, type SpillRef, type SpillUsage, type SqlBoolean, type TaskCheckpoint, type TaskInput, type TaskManifest, type TaskManifestTask, type TaskState, type TaskTransitionInput, type TopKOperatorState, type TopKOptions, type TopKResult, type ValueInput, type WriteParquetOptions, type WriteParquetRowsOptions, type WritePartitionedParquetFile, type WritePartitionedParquetResult, type WritePartitionedParquetTaskOptions, type WritePartitionedParquetTaskResult, advanceTaskCheckpoint, and, applyIcebergDeletes, assertBookmarkMatches, asyncBufferFromStore, bboxMayIntersect, between, broadcastJoin, buildBBoxIndex, buildMinMaxIndex, cacheApiCache, classifyPredicate, col, createBookmark, createParquetLake as createLake, createOutputManifest, createOutputManifestFromCheckpoints, createParquetLake, createParquetTableAs, createTaskManifest, deserializeAggregateOperatorState, deserializeSortOperatorState, deserializeTopKOperatorState, encodeJsonLine, eq, evaluate, fingerprint, fn, gt, gte, icebergGlueCatalog, icebergNessieCatalog, icebergRestCatalog, ilike, isIn, isLaQLError, isNotNull, isNull, jsonSafeValue, like, lit, loadIcebergTable, loadIcebergTableFromObjectStore, loadIcebergTableFromRest, loadTable, lookupJoin, lt, lte, matches, memoryCache, memoryCheckpointAdapter, memorySpillAdapter, memoryStore, ne, not, notIn, or, parquetScanner, parseHivePartitions, parseJsonQuery, partitionedParquetOutputEntries, planFiles, planRowGroups, planRowGroupsFromMetadata, pruneFilesWithIndex, readControlSignal, readIcebergParquetDeletes, readOutputManifest, readParquetMetadata, readParquetObjectBatches, readParquetObjects, rejectUnsupportedParquetSchema, rowGroupMayMatch, scanBatches, scanPlannedIcebergRows, scanRows, serializeAggregateOperatorState, serializeSortOperatorState, serializeTopKOperatorState, signPaginationToken, stableStringify, throwIfAborted, transitionTaskCheckpoint, verifyPaginationToken, withObjectStoreReadControls, writeOutputManifest, writeParquet, writePartitionedParquet, writePartitionedParquetTask };
|