@sixb/ducklake 0.1.6 → 0.1.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +25 -27
- package/dist/ducklake-storage.d.ts +2 -1
- package/dist/ducklake-storage.d.ts.map +1 -1
- package/dist/index.js +736 -244
- package/dist/index.js.map +14 -12
- package/dist/internal/duckdb-reader.d.ts +28 -0
- package/dist/internal/duckdb-reader.d.ts.map +1 -0
- package/dist/internal/duckdb-runtime.d.ts +5 -3
- package/dist/internal/duckdb-runtime.d.ts.map +1 -1
- package/dist/internal/ducklake-connection-manager.d.ts +19 -3
- package/dist/internal/ducklake-connection-manager.d.ts.map +1 -1
- package/dist/internal/ducklake-read-gate.d.ts +11 -0
- package/dist/internal/ducklake-read-gate.d.ts.map +1 -0
- package/dist/internal/ducklake-row-reader.d.ts.map +1 -1
- package/dist/internal/ducklake-snapshot-reader.d.ts +26 -1
- package/dist/internal/ducklake-snapshot-reader.d.ts.map +1 -1
- package/dist/internal/ducklake-write-coordinator.d.ts +2 -1
- package/dist/internal/ducklake-write-coordinator.d.ts.map +1 -1
- package/dist/internal/sql.d.ts +11 -2
- package/dist/internal/sql.d.ts.map +1 -1
- package/dist/types.d.ts +2 -0
- package/dist/types.d.ts.map +1 -1
- package/package.json +4 -4
- package/src/ducklake-storage.ts +8 -0
- package/src/internal/duckdb-reader.ts +129 -0
- package/src/internal/duckdb-runtime.ts +75 -51
- package/src/internal/ducklake-connection-manager.ts +117 -33
- package/src/internal/ducklake-maintenance.ts +1 -1
- package/src/internal/ducklake-read-gate.ts +58 -0
- package/src/internal/ducklake-row-reader.ts +84 -63
- package/src/internal/ducklake-snapshot-reader.ts +331 -79
- package/src/internal/ducklake-write-coordinator.ts +56 -15
- package/src/internal/ducklake-write-session.ts +1 -0
- package/src/internal/sql.ts +35 -6
- package/src/types.ts +2 -0
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
import type { DuckDBConnection } from "@duckdb/node-api";
|
|
2
|
+
/** One native query, on a connection belonging to the runtime's shared instance. */
|
|
3
|
+
export interface DuckDbReader {
|
|
4
|
+
rows(): AsyncIterable<Record<string, unknown>>;
|
|
5
|
+
close(): Promise<void>;
|
|
6
|
+
}
|
|
7
|
+
export interface DuckDbReaderOptions {
|
|
8
|
+
readonly signal?: AbortSignal;
|
|
9
|
+
readonly onClose?: () => Promise<void>;
|
|
10
|
+
}
|
|
11
|
+
export declare class NodeDuckDbReader implements DuckDbReader {
|
|
12
|
+
private readonly connection;
|
|
13
|
+
private readonly options;
|
|
14
|
+
private resultPromise;
|
|
15
|
+
private pending;
|
|
16
|
+
private closePromise;
|
|
17
|
+
private finished;
|
|
18
|
+
private consumed;
|
|
19
|
+
private readonly onAbort;
|
|
20
|
+
constructor(connection: DuckDBConnection, options: DuckDbReaderOptions);
|
|
21
|
+
start(sql: string): Promise<void>;
|
|
22
|
+
rows(): AsyncIterable<Record<string, unknown>>;
|
|
23
|
+
close(): Promise<void>;
|
|
24
|
+
private finish;
|
|
25
|
+
private track;
|
|
26
|
+
private assertOpen;
|
|
27
|
+
}
|
|
28
|
+
//# sourceMappingURL=duckdb-reader.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"duckdb-reader.d.ts","sourceRoot":"","sources":["../../src/internal/duckdb-reader.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,gBAAgB,EAAgB,MAAM,kBAAkB,CAAA;AAItE,oFAAoF;AACpF,MAAM,WAAW,YAAY;IAC3B,IAAI,IAAI,aAAa,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAA;IAC9C,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAAA;CACvB;AAED,MAAM,WAAW,mBAAmB;IAClC,QAAQ,CAAC,MAAM,CAAC,EAAE,WAAW,CAAA;IAC7B,QAAQ,CAAC,OAAO,CAAC,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAA;CACvC;AAED,qBAAa,gBAAiB,YAAW,YAAY;IAajD,OAAO,CAAC,QAAQ,CAAC,UAAU;IAC3B,OAAO,CAAC,QAAQ,CAAC,OAAO;IAb1B,OAAO,CAAC,aAAa,CAAmC;IACxD,OAAO,CAAC,OAAO,CAA8B;IAC7C,OAAO,CAAC,YAAY,CAA2B;IAC/C,OAAO,CAAC,QAAQ,CAAQ;IACxB,OAAO,CAAC,QAAQ,CAAQ;IACxB,OAAO,CAAC,QAAQ,CAAC,OAAO,CAIvB;gBAGkB,UAAU,EAAE,gBAAgB,EAC5B,OAAO,EAAE,mBAAmB;IAGzC,KAAK,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAwBhC,IAAI,IAAI,aAAa,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IA6BrD,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;YASR,MAAM;YAkBN,KAAK;IAYnB,OAAO,CAAC,UAAU;CAInB"}
|
|
@@ -1,11 +1,12 @@
|
|
|
1
1
|
import { type DuckDBValue } from "@duckdb/node-api";
|
|
2
2
|
import type { DuckDbRuntimeOptions, DuckLakeStorageOptions } from "../types";
|
|
3
|
+
import { type DuckDbReader, type DuckDbReaderOptions } from "./duckdb-reader";
|
|
3
4
|
/**
|
|
4
5
|
* Narrow adapter around `@duckdb/node-api`.
|
|
5
6
|
*
|
|
6
7
|
* Keeping the public provider behind this interface confines the driver types to
|
|
7
|
-
* this module instead of leaking them into `DuckLakeStorage`.
|
|
8
|
-
*
|
|
8
|
+
* this module instead of leaking them into `DuckLakeStorage`. Main-connection calls
|
|
9
|
+
* share one queue; readers use separate connections to the same DuckDB instance.
|
|
9
10
|
*/
|
|
10
11
|
export interface DuckDbQueryRuntime {
|
|
11
12
|
run(sql: string, values?: readonly DuckDBValue[]): Promise<void>;
|
|
@@ -14,7 +15,8 @@ export interface DuckDbQueryRuntime {
|
|
|
14
15
|
withAppender<T>(tableName: string, useAppender: (appender: DuckDbAppender) => T | Promise<T>): Promise<T>;
|
|
15
16
|
}
|
|
16
17
|
export interface DuckDbRuntime extends DuckDbQueryRuntime {
|
|
17
|
-
|
|
18
|
+
openReader(sql: string, options?: DuckDbReaderOptions): Promise<DuckDbReader>;
|
|
19
|
+
stopReaders(): Promise<void>;
|
|
18
20
|
/**
|
|
19
21
|
* Reserve the runtime queue for a multi-statement critical section.
|
|
20
22
|
*
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"duckdb-runtime.d.ts","sourceRoot":"","sources":["../../src/internal/duckdb-runtime.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"duckdb-runtime.d.ts","sourceRoot":"","sources":["../../src/internal/duckdb-runtime.ts"],"names":[],"mappings":"AAEA,OAAO,EAGL,KAAK,WAAW,EAEjB,MAAM,kBAAkB,CAAA;AAEzB,OAAO,KAAK,EAAE,oBAAoB,EAAE,sBAAsB,EAAE,MAAM,UAAU,CAAA;AAC5E,OAAO,EAAE,KAAK,YAAY,EAAE,KAAK,mBAAmB,EAAoB,MAAM,iBAAiB,CAAA;AAW/F;;;;;;GAMG;AACH,MAAM,WAAW,kBAAkB;IACjC,GAAG,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,SAAS,WAAW,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAAA;IAChE,aAAa,CAAC,UAAU,EAAE,SAAS,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAAA;IAC3D,KAAK,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,SAAS,WAAW,EAAE,GAAG,OAAO,CAAC,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,CAAC,CAAA;IAChG,YAAY,CAAC,CAAC,EACZ,SAAS,EAAE,MAAM,EACjB,WAAW,EAAE,CAAC,QAAQ,EAAE,cAAc,KAAK,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC,GACxD,OAAO,CAAC,CAAC,CAAC,CAAA;CACd;AAED,MAAM,WAAW,aAAc,SAAQ,kBAAkB;IACvD,UAAU,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,mBAAmB,GAAG,OAAO,CAAC,YAAY,CAAC,CAAA;IAC7E,WAAW,IAAI,OAAO,CAAC,IAAI,CAAC,CAAA;IAC5B;;;;;OAKG;IACH,aAAa,CAAC,CAAC,EAAE,UAAU,EAAE,CAAC,OAAO,EAAE,sBAAsB,KAAK,OAAO,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC,CAAA;IACzF,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAAA;CACvB;AAED;;;;;GAKG;AACH,MAAM,MAAM,sBAAsB,GAAG,kBAAkB,CAAA;AAEvD,MAAM,WAAW,cAAc;IAC7B,UAAU,IAAI,IAAI,CAAA;IAClB,aAAa,CAAC,KAAK,EAAE,OAAO,GAAG,IAAI,CAAA;IACnC,aAAa,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI,CAAA;IAClC,YAAY,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI,CAAA;IACjC,YAAY,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI,CAAA;IACjC,YAAY,CAAC,KAAK,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC,GAAG,IAAI,CAAA;IAChE,MAAM,IAAI,IAAI,CAAA;CACf;AAED,UAAU,oBAAoB;IAC5B,QAAQ,CAAC,iBAAiB,CAAC,EAAE,OAAO,CAAA;IACpC,QAAQ,CAAC,MAAM,CAAC,EAAE,OAAO,CAAA;CAC1B;AAsND;;GAEG;AACH,wBAAsB,mBAAmB,CACvC,OAAO,GAAE,oBAAyB,GACjC,OAAO,CAAC,aAAa,CAAC,CAuBxB;AAED;;;;GAIG;AACH,wBAAsB,yBAAyB,CAC7C,OAAO,EAAE,aAAa,EACtB,OAAO,EAAE,sBAAsB,GAC9B,OAAO,CAAC,IAAI,CAAC,CAMf;AAED;;;GAGG;AACH,wBAAsB,aAAa,CACjC,OAAO,EAAE,aAAa,EACtB,OAAO,EAAE,sBAAsB,EAC/B,YAAY,GAAE,oBAAyB,GACtC,OAAO,CAAC,IAAI,CAAC,CA8Bf"}
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import type { DuckLakeStorageOptions } from "../types";
|
|
2
|
+
import type { DuckDbReader } from "./duckdb-reader";
|
|
2
3
|
import { type DuckDbExclusiveRuntime, type DuckDbQueryRuntime, type DuckDbRuntime } from "./duckdb-runtime";
|
|
3
4
|
export interface DuckLakeAttachedRuntimeLease {
|
|
4
5
|
readonly runtime: DuckDbRuntime;
|
|
@@ -18,12 +19,26 @@ export declare class DuckLakeConnectionManager {
|
|
|
18
19
|
private installExtensionsPromise;
|
|
19
20
|
private attachPromise;
|
|
20
21
|
private refreshPromise;
|
|
21
|
-
private
|
|
22
|
+
private attachmentLock;
|
|
22
23
|
private observedLocalCatalogGeneration;
|
|
23
24
|
private attached;
|
|
24
25
|
private closed;
|
|
26
|
+
private closePromise;
|
|
25
27
|
private runtimePoisoned;
|
|
28
|
+
private pinnedReaders;
|
|
29
|
+
private refreshRequested;
|
|
30
|
+
private readonly shutdown;
|
|
31
|
+
private readonly reads;
|
|
32
|
+
private readonly maxStreamingReads;
|
|
26
33
|
constructor(options: DuckLakeStorageOptions);
|
|
34
|
+
acquireRead(signal?: AbortSignal): Promise<{
|
|
35
|
+
readonly signal: AbortSignal;
|
|
36
|
+
readonly release: () => void;
|
|
37
|
+
}>;
|
|
38
|
+
private nativeReaderLimit;
|
|
39
|
+
/** Caller holds an attachment lease until this query has started. */
|
|
40
|
+
openReader(runtime: DuckDbRuntime, sql: string, signal: AbortSignal, releaseRead: () => void): Promise<DuckDbReader | null>;
|
|
41
|
+
withMaintenanceRuntime<T>(run: (runtime: DuckDbExclusiveRuntime) => Promise<T>): Promise<T>;
|
|
27
42
|
assertOpen(): void;
|
|
28
43
|
runtime(): Promise<DuckDbRuntime>;
|
|
29
44
|
attachedRuntime(): Promise<DuckDbRuntime>;
|
|
@@ -45,6 +60,7 @@ export declare class DuckLakeConnectionManager {
|
|
|
45
60
|
private refreshForExternalChangesUnlocked;
|
|
46
61
|
private needsLocalCatalogRefresh;
|
|
47
62
|
close(): Promise<void>;
|
|
63
|
+
private finishClose;
|
|
48
64
|
private ensureAttached;
|
|
49
65
|
private ensureExtensionsInstalled;
|
|
50
66
|
private attachRuntime;
|
|
@@ -59,8 +75,8 @@ export declare class DuckLakeConnectionManager {
|
|
|
59
75
|
private detachDuckDbCatalogAfterLease;
|
|
60
76
|
private discardRuntimeUnlocked;
|
|
61
77
|
private detachRuntimeUnlocked;
|
|
62
|
-
private
|
|
63
|
-
private
|
|
78
|
+
private withAttachmentLock;
|
|
79
|
+
private acquireAttachmentLock;
|
|
64
80
|
private closeRuntimePromise;
|
|
65
81
|
private closeRuntime;
|
|
66
82
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"ducklake-connection-manager.d.ts","sourceRoot":"","sources":["../../src/internal/ducklake-connection-manager.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,UAAU,CAAA;AAEtD,OAAO,EAEL,KAAK,sBAAsB,EAC3B,KAAK,kBAAkB,EACvB,KAAK,aAAa,EAGnB,MAAM,kBAAkB,CAAA;
|
|
1
|
+
{"version":3,"file":"ducklake-connection-manager.d.ts","sourceRoot":"","sources":["../../src/internal/ducklake-connection-manager.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,UAAU,CAAA;AAEtD,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAA;AACnD,OAAO,EAEL,KAAK,sBAAsB,EAC3B,KAAK,kBAAkB,EACvB,KAAK,aAAa,EAGnB,MAAM,kBAAkB,CAAA;AASzB,MAAM,WAAW,4BAA4B;IAC3C,QAAQ,CAAC,OAAO,EAAE,aAAa,CAAA;IAC/B,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,CAAA;CACzB;AAED;;;;;;;GAOG;AACH,qBAAa,yBAAyB;IAoBxB,OAAO,CAAC,QAAQ,CAAC,OAAO;IAnBpC,OAAO,CAAC,cAAc,CAAoC;IAC1D,OAAO,CAAC,wBAAwB,CAA2B;IAC3D,OAAO,CAAC,aAAa,CAA2B;IAChD,OAAO,CAAC,cAAc,CAA2B;IAIjD,OAAO,CAAC,cAAc,CAAmC;IACzD,OAAO,CAAC,8BAA8B,CAAQ;IAC9C,OAAO,CAAC,QAAQ,CAAQ;IACxB,OAAO,CAAC,MAAM,CAAQ;IACtB,OAAO,CAAC,YAAY,CAA2B;IAC/C,OAAO,CAAC,eAAe,CAAQ;IAC/B,OAAO,CAAC,aAAa,CAAI;IACzB,OAAO,CAAC,gBAAgB,CAAQ;IAChC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAwB;IACjD,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAyB;IAC/C,OAAO,CAAC,QAAQ,CAAC,iBAAiB,CAAQ;gBAEb,OAAO,EAAE,sBAAsB;IAQtD,WAAW,CAAC,MAAM,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC;QAC/C,QAAQ,CAAC,MAAM,EAAE,WAAW,CAAA;QAC5B,QAAQ,CAAC,OAAO,EAAE,MAAM,IAAI,CAAA;KAC7B,CAAC;YAMY,iBAAiB;IAe/B,qEAAqE;IAC/D,UAAU,CACd,OAAO,EAAE,aAAa,EACtB,GAAG,EAAE,MAAM,EACX,MAAM,EAAE,WAAW,EACnB,WAAW,EAAE,MAAM,IAAI,GACtB,OAAO,CAAC,YAAY,GAAG,IAAI,CAAC;IA6BzB,sBAAsB,CAAC,CAAC,EAC5B,GAAG,EAAE,CAAC,OAAO,EAAE,sBAAsB,KAAK,OAAO,CAAC,CAAC,CAAC,GACnD,OAAO,CAAC,CAAC,CAAC;IAIb,UAAU,IAAI,IAAI;IAMZ,OAAO,IAAI,OAAO,CAAC,aAAa,CAAC;IAgBjC,eAAe,IAAI,OAAO,CAAC,aAAa,CAAC;IAMzC,mBAAmB,CAAC,CAAC,EAAE,GAAG,EAAE,CAAC,OAAO,EAAE,aAAa,KAAK,OAAO,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC;IAS/E,sBAAsB,IAAI,OAAO,CAAC,4BAA4B,CAAC;IAmC/D,qBAAqB,CAAC,CAAC,EAAE,GAAG,EAAE,CAAC,OAAO,EAAE,sBAAsB,KAAK,OAAO,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC;IAS1F,cAAc,IAAI,OAAO,CAAC,aAAa,CAAC;IAa9C;;;;OAIG;IACG,aAAa,IAAI,OAAO,CAAC,aAAa,CAAC;IAoBvC,YAAY,IAAI,OAAO,CAAC,IAAI,CAAC;YASrB,oBAAoB;IA4C5B,yBAAyB,IAAI,OAAO,CAAC,IAAI,CAAC;IAwBhD,uBAAuB,IAAI,IAAI;IAQzB,6BAA6B,CAAC,OAAO,EAAE,kBAAkB,GAAG,OAAO,CAAC,IAAI,CAAC;YAkBjE,iCAAiC;IAW/C,OAAO,CAAC,wBAAwB;IAIhC,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;YAKR,WAAW;YAeX,cAAc;YAgCd,yBAAyB;YAgBzB,aAAa;YASb,sBAAsB;YAUtB,aAAa;IAI3B;;;;OAIG;IACH,aAAa,IAAI,IAAI;YAIP,6BAA6B;YAmB7B,sBAAsB;YAOtB,qBAAqB;YAerB,kBAAkB;YASlB,qBAAqB;YAqBrB,mBAAmB;YAqBnB,YAAY;CAiB3B"}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
/** Keeps destructive maintenance outside reads, including paged overflow readers. */
|
|
2
|
+
export declare class DuckLakeReadGate {
|
|
3
|
+
private active;
|
|
4
|
+
private maintenance;
|
|
5
|
+
private readonly waiters;
|
|
6
|
+
acquire(signal: AbortSignal): Promise<() => void>;
|
|
7
|
+
withMaintenance<T>(signal: AbortSignal, run: () => Promise<T>): Promise<T>;
|
|
8
|
+
private wait;
|
|
9
|
+
private wake;
|
|
10
|
+
}
|
|
11
|
+
//# sourceMappingURL=ducklake-read-gate.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"ducklake-read-gate.d.ts","sourceRoot":"","sources":["../../src/internal/ducklake-read-gate.ts"],"names":[],"mappings":"AAAA,qFAAqF;AACrF,qBAAa,gBAAgB;IAC3B,OAAO,CAAC,MAAM,CAAI;IAClB,OAAO,CAAC,WAAW,CAAQ;IAC3B,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAwB;IAE1C,OAAO,CAAC,MAAM,EAAE,WAAW,GAAG,OAAO,CAAC,MAAM,IAAI,CAAC;IAiBjD,eAAe,CAAC,CAAC,EAAE,MAAM,EAAE,WAAW,EAAE,GAAG,EAAE,MAAM,OAAO,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC;IAchF,OAAO,CAAC,IAAI;IAiBZ,OAAO,CAAC,IAAI;CAGb"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"ducklake-row-reader.d.ts","sourceRoot":"","sources":["../../src/internal/ducklake-row-reader.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAA2B,UAAU,EAAE,MAAM,YAAY,CAAA;AACrE,OAAO,KAAK,EAAkB,oBAAoB,EAAE,MAAM,yBAAyB,CAAA;AAEnF,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,UAAU,CAAA;
|
|
1
|
+
{"version":3,"file":"ducklake-row-reader.d.ts","sourceRoot":"","sources":["../../src/internal/ducklake-row-reader.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAA2B,UAAU,EAAE,MAAM,YAAY,CAAA;AACrE,OAAO,KAAK,EAAkB,oBAAoB,EAAE,MAAM,yBAAyB,CAAA;AAEnF,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,UAAU,CAAA;AAItD,OAAO,KAAK,EAAE,yBAAyB,EAAE,MAAM,+BAA+B,CAAA;AAE9E,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,4BAA4B,CAAA;AASxE;;;;;;GAMG;AACH,qBAAa,iBAAiB;IAE1B,OAAO,CAAC,QAAQ,CAAC,OAAO;IACxB,OAAO,CAAC,QAAQ,CAAC,WAAW;IAC5B,OAAO,CAAC,QAAQ,CAAC,SAAS;gBAFT,OAAO,EAAE,sBAAsB,EAC/B,WAAW,EAAE,yBAAyB,EACtC,SAAS,EAAE,sBAAsB;IAG7C,QAAQ,CAAC,KAAK,EAAE,oBAAoB,GAAG,aAAa,CAAC,UAAU,CAAC;YA4ExD,aAAa;YAed,cAAc;IAuB5B,OAAO,CAAC,kBAAkB;IAuB1B,OAAO,CAAC,uBAAuB;CAKhC"}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type { DatasetDefinition } from "@sixb/core";
|
|
2
|
-
import type { DatasetCatalogState, DatasetVersion, DatasetVersionRef } from "@sixb/core/lake-storage";
|
|
2
|
+
import type { DatasetCatalogState, DatasetVersion, DatasetVersionRef, LatestVersionsSince, ListLatestVersionsSinceInput } from "@sixb/core/lake-storage";
|
|
3
3
|
import type { DuckLakeStorageOptions } from "../types";
|
|
4
4
|
import type { DuckDbQueryRuntime } from "./duckdb-runtime";
|
|
5
5
|
import type { DuckLakeConnectionManager } from "./ducklake-connection-manager";
|
|
@@ -41,6 +41,21 @@ export declare class DuckLakeSnapshotReader {
|
|
|
41
41
|
*/
|
|
42
42
|
listDatasetCatalogState(datasetIds: readonly string[]): Promise<readonly DatasetCatalogState[]>;
|
|
43
43
|
private collectDatasetCatalogState;
|
|
44
|
+
/**
|
|
45
|
+
* Change feed over DuckLake's snapshot table, which is an append-only commit log: snapshot ids
|
|
46
|
+
* only increase, and a commit becomes visible only after every lower id has (concurrent commits
|
|
47
|
+
* that pick the same id conflict, and the loser retries with the next one). The cursor is the
|
|
48
|
+
* last snapshot id read. Every query is bounded by a snapshot-id range, so an idle read is one
|
|
49
|
+
* indexed query that returns the cursor's own snapshot, whatever the history length.
|
|
50
|
+
*/
|
|
51
|
+
listLatestVersionsSince(input: ListLatestVersionsSinceInput): Promise<LatestVersionsSince | null>;
|
|
52
|
+
private queryHeadSnapshotId;
|
|
53
|
+
/** The cursor's snapshot and up to one feed page of snapshots after it, oldest first. */
|
|
54
|
+
private querySnapshotsFrom;
|
|
55
|
+
/** Current dataset tables, with the first snapshot each table id existed in. */
|
|
56
|
+
private resolveFeedTables;
|
|
57
|
+
/** Folds a page of snapshots, oldest first, into each dataset's newest data version. */
|
|
58
|
+
private collectFeedPage;
|
|
44
59
|
private snapshotRowToSummary;
|
|
45
60
|
private resolveDatasetTableIds;
|
|
46
61
|
/**
|
|
@@ -67,6 +82,16 @@ export declare class DuckLakeSnapshotReader {
|
|
|
67
82
|
private getSnapshotRowForTableRef;
|
|
68
83
|
private getSnapshotRowsForTableRef;
|
|
69
84
|
private collectVisibleSnapshotRows;
|
|
85
|
+
/**
|
|
86
|
+
* Hydrates, newest first, the snapshots that may hold a version of one dataset table: those
|
|
87
|
+
* that added or removed its data or delete files, that DuckLake recorded as changing it inline,
|
|
88
|
+
* or whose Sixb commit metadata names the dataset. The text matches are a superset;
|
|
89
|
+
* candidateToSnapshotRow decides visibility. Only this table's snapshots are read, so the cost
|
|
90
|
+
* follows the dataset's own history rather than every commit in the catalog.
|
|
91
|
+
*
|
|
92
|
+
* Returns one row per candidate id, including ids whose snapshot no longer exists, so the caller
|
|
93
|
+
* can tell a short batch from the end of the history.
|
|
94
|
+
*/
|
|
70
95
|
private querySnapshotCandidates;
|
|
71
96
|
private candidateToSnapshotRow;
|
|
72
97
|
private snapshotToVersion;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"ducklake-snapshot-reader.d.ts","sourceRoot":"","sources":["../../src/internal/ducklake-snapshot-reader.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,YAAY,CAAA;AACnD,OAAO,KAAK,EACV,mBAAmB,EAEnB,cAAc,EAEd,iBAAiB,
|
|
1
|
+
{"version":3,"file":"ducklake-snapshot-reader.d.ts","sourceRoot":"","sources":["../../src/internal/ducklake-snapshot-reader.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,YAAY,CAAA;AACnD,OAAO,KAAK,EACV,mBAAmB,EAEnB,cAAc,EAEd,iBAAiB,EACjB,mBAAmB,EACnB,4BAA4B,EAC7B,MAAM,yBAAyB,CAAA;AAEhC,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,UAAU,CAAA;AAEtD,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAA;AAC1D,OAAO,KAAK,EAAE,yBAAyB,EAAE,MAAM,+BAA+B,CAAA;AAC9E,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,4BAA4B,CAAA;AACxE,OAAO,EAAE,KAAK,eAAe,EAA0B,MAAM,8BAA8B,CAAA;AAyD3F,MAAM,WAAW,sBAAsB;IACrC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAA;IAC1B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAA;IAC1B,QAAQ,CAAC,MAAM,CAAC,EAAE,SAAS,iBAAiB,EAAE,CAAA;IAC9C,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;IAC1B,QAAQ,CAAC,0BAA0B,CAAC,EAAE,SAAS,MAAM,EAAE,CAAA;CACxD;AA2BD;;;;;;;;;;;GAWG;AACH,qBAAa,sBAAsB;IAE/B,OAAO,CAAC,QAAQ,CAAC,OAAO;IACxB,OAAO,CAAC,QAAQ,CAAC,WAAW;IAC5B,OAAO,CAAC,QAAQ,CAAC,QAAQ;gBAFR,OAAO,EAAE,sBAAsB,EAC/B,WAAW,EAAE,yBAAyB,EACtC,QAAQ,EAAE,sBAAsB;IAG7C,YAAY,CAAC,SAAS,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,SAAS,cAAc,EAAE,CAAC;IAanF,gBAAgB,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,cAAc,GAAG,IAAI,CAAC;IAanE,UAAU,CAAC,SAAS,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,cAAc,GAAG,IAAI,CAAC;IActF;;;;;;OAMG;IACG,uBAAuB,CAC3B,UAAU,EAAE,SAAS,MAAM,EAAE,GAC5B,OAAO,CAAC,SAAS,mBAAmB,EAAE,CAAC;YAY5B,0BAA0B;IAsBxC;;;;;;OAMG;IACG,uBAAuB,CAC3B,KAAK,EAAE,4BAA4B,GAClC,OAAO,CAAC,mBAAmB,GAAG,IAAI,CAAC;YA8BxB,mBAAmB;IAajC,yFAAyF;YAC3E,kBAAkB;IAsBhC,gFAAgF;YAClE,iBAAiB;IAkC/B,wFAAwF;YAC1E,eAAe;IA2E7B,OAAO,CAAC,oBAAoB;YAad,sBAAsB;IAiCpC;;;;;;OAMG;YACW,yBAAyB;YA8DzB,yBAAyB;YAyCzB,oBAAoB;IAuDlC,OAAO,CAAC,wBAAwB;IAY1B,6BAA6B,CACjC,OAAO,EAAE,kBAAkB,EAC3B,OAAO,EAAE,iBAAiB,GACzB,OAAO,CAAC,cAAc,GAAG,IAAI,CAAC;IAK3B,gCAAgC,CACpC,OAAO,EAAE,kBAAkB,EAC3B,OAAO,EAAE,iBAAiB,GACzB,OAAO,CAAC,iBAAiB,GAAG,IAAI,CAAC;IAK9B,oCAAoC,CACxC,OAAO,EAAE,kBAAkB,EAC3B,OAAO,EAAE,iBAAiB,GACzB,OAAO,CAAC,sBAAsB,GAAG,IAAI,CAAC;IAsBnC,qBAAqB,CACzB,OAAO,EAAE,kBAAkB,EAC3B,OAAO,EAAE,iBAAiB,EAC1B,UAAU,EAAE,MAAM,GACjB,OAAO,CAAC,cAAc,GAAG,IAAI,CAAC;IAO3B,qBAAqB,CACzB,OAAO,EAAE,kBAAkB,EAC3B,QAAQ,EAAE,eAAe,EACzB,UAAU,EAAE,MAAM,GACjB,OAAO,CAAC,cAAc,GAAG,IAAI,CAAC;IAO3B,wBAAwB,CAC5B,OAAO,EAAE,kBAAkB,EAC3B,OAAO,EAAE,iBAAiB,EAC1B,UAAU,EAAE,MAAM,GACjB,OAAO,CAAC,iBAAiB,GAAG,IAAI,CAAC;IAc9B,yBAAyB,CAC7B,OAAO,EAAE,kBAAkB,EAC3B,OAAO,EAAE,iBAAiB,EAC1B,KAAK,CAAC,EAAE,MAAM,GACb,OAAO,CAAC,SAAS,cAAc,EAAE,CAAC;YAKvB,uBAAuB;IAe/B,2BAA2B,CAC/B,OAAO,EAAE,kBAAkB,EAC3B,QAAQ,EAAE,eAAe,GACxB,OAAO,CAAC,cAAc,GAAG,IAAI,CAAC;YAKnB,+BAA+B;YAY/B,yBAAyB;YAmCzB,0BAA0B;YAmB1B,0BAA0B;IA+CxC;;;;;;;;;OASG;YACW,uBAAuB;IAyIrC,OAAO,CAAC,sBAAsB;YAyChB,iBAAiB;IAgC/B,OAAO,CAAC,qBAAqB;CAa9B"}
|
|
@@ -49,7 +49,8 @@ export declare class DuckLakeWriteCoordinator {
|
|
|
49
49
|
private ensureNoOpHasSnapshot;
|
|
50
50
|
private versionForNoOpCommit;
|
|
51
51
|
private rollbackTransaction;
|
|
52
|
-
private
|
|
52
|
+
private beginVersionTransaction;
|
|
53
|
+
private currentSnapshotId;
|
|
53
54
|
private lastCommittedSnapshotId;
|
|
54
55
|
private findSnapshotIdByCommitId;
|
|
55
56
|
private findOwnSnapshotId;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"ducklake-write-coordinator.d.ts","sourceRoot":"","sources":["../../src/internal/ducklake-write-coordinator.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,YAAY,CAAA;AACnD,OAAO,KAAK,EACV,sBAAsB,EACtB,sBAAsB,EAGtB,wBAAwB,EACxB,gBAAgB,EAChB,gBAAgB,EAChB,gBAAgB,EACjB,MAAM,yBAAyB,CAAA;AAShC,OAAO,KAAK,
|
|
1
|
+
{"version":3,"file":"ducklake-write-coordinator.d.ts","sourceRoot":"","sources":["../../src/internal/ducklake-write-coordinator.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,YAAY,CAAA;AACnD,OAAO,KAAK,EACV,sBAAsB,EACtB,sBAAsB,EAGtB,wBAAwB,EACxB,gBAAgB,EAChB,gBAAgB,EAChB,gBAAgB,EACjB,MAAM,yBAAyB,CAAA;AAShC,OAAO,KAAK,EAA0B,sBAAsB,EAAE,MAAM,UAAU,CAAA;AAE9E,OAAO,EACL,KAAK,sBAAsB,EAI5B,MAAM,sBAAsB,CAAA;AAG7B,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAA;AAC1D,OAAO,KAAK,EAAE,yBAAyB,EAAE,MAAM,+BAA+B,CAAA;AAC9E,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,4BAA4B,CAAA;AAExE,OAAO,KAAK,EAAE,sBAAsB,EAA0B,MAAM,4BAA4B,CAAA;AAYhG,MAAM,WAAW,iCAAiC;IAChD,QAAQ,CAAC,OAAO,EAAE,iBAAiB,CAAA;IACnC,QAAQ,CAAC,IAAI,EAAE,gBAAgB,CAAA;IAC/B,QAAQ,CAAC,uBAAuB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAA;IAChD,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAA;IAC9B,QAAQ,CAAC,QAAQ,CAAC,EAAE,sBAAsB,CAAC,UAAU,CAAC,CAAA;IACtD,QAAQ,CAAC,MAAM,CAAC,EAAE,sBAAsB,CAAC,QAAQ,CAAC,CAAA;IAClD,YAAY,CACV,OAAO,EAAE,kBAAkB,EAC3B,OAAO,EAAE,2BAA2B,GACnC,OAAO,CAAC,sBAAsB,CAAC,CAAA;CACnC;AAgBD,MAAM,WAAW,2BAA2B;IAC1C,QAAQ,CAAC,gBAAgB,CAAC,EAAE,MAAM,CAAA;IAClC,QAAQ,CAAC,0BAA0B,CAAC,EAAE,SAAS,MAAM,EAAE,CAAA;CACxD;AAED;;;;;;;GAOG;AACH,qBAAa,wBAAwB;IAEjC,OAAO,CAAC,QAAQ,CAAC,OAAO;IACxB,OAAO,CAAC,QAAQ,CAAC,WAAW;IAC5B,OAAO,CAAC,QAAQ,CAAC,QAAQ;IACzB,OAAO,CAAC,QAAQ,CAAC,SAAS;gBAHT,OAAO,EAAE,sBAAsB,EAC/B,WAAW,EAAE,yBAAyB,EACtC,QAAQ,EAAE,sBAAsB,EAChC,SAAS,EAAE,sBAAsB;IAG9C,UAAU,CAAC,KAAK,EAAE,sBAAsB,GAAG,OAAO,CAAC,gBAAgB,CAAC;IA6BpE,UAAU,CAAC,KAAK,EAAE,sBAAsB,GAAG,OAAO,CAAC,gBAAgB,CAAC;YAkC5D,WAAW;YAyBX,WAAW;IAoDnB,oBAAoB,CACxB,KAAK,EAAE,iCAAiC,GACvC,OAAO,CAAC,wBAAwB,CAAC;IAM9B,iBAAiB,CAAC,CAAC,EAAE,GAAG,EAAE,CAAC,OAAO,EAAE,kBAAkB,KAAK,OAAO,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC;IAQlF,sCAAsC,CAC1C,OAAO,EAAE,kBAAkB,EAC3B,KAAK,EAAE,iCAAiC,GACvC,OAAO,CAAC,wBAAwB,CAAC;YAkBtB,sCAAsC;YAUtC,uBAAuB;YA2CvB,4BAA4B;YA6E5B,wBAAwB;YAiExB,eAAe;YAyBf,iBAAiB;YAkCjB,qBAAqB;YA+BrB,oBAAoB;YAmBpB,mBAAmB;YASnB,uBAAuB;YAmCvB,iBAAiB;YAUjB,uBAAuB;YAYvB,wBAAwB;YAiCxB,iBAAiB;IA0B/B,OAAO,CAAC,2BAA2B;CAgBpC"}
|
package/dist/internal/sql.d.ts
CHANGED
|
@@ -19,10 +19,19 @@ export declare function buildSetPostgresPoolSql(options: Pick<DuckLakeStorageOpt
|
|
|
19
19
|
* Render a fully-qualified DuckLake metadata table path.
|
|
20
20
|
*
|
|
21
21
|
* PostgreSQL catalogs expose DuckLake metadata tables under the configured
|
|
22
|
-
* metadata schema
|
|
23
|
-
* DuckDB's default `main` schema.
|
|
22
|
+
* metadata schema (PostgreSQL's `public` by default) inside the attached
|
|
23
|
+
* metadata catalog. Local catalogs use DuckDB's default `main` schema.
|
|
24
24
|
*/
|
|
25
25
|
export declare function duckLakeMetadataTableName(options: Pick<DuckLakeStorageOptions, "alias" | "catalog">, tableName: string): string;
|
|
26
|
+
/**
|
|
27
|
+
* Render a metadata query that runs where the catalog lives.
|
|
28
|
+
*
|
|
29
|
+
* DuckDB's PostgreSQL scanner pushes comparisons and constant IN lists down to PostgreSQL, but not
|
|
30
|
+
* string matching, so a query that filters on text would copy the whole column into DuckDB. On a
|
|
31
|
+
* PostgreSQL catalog the query therefore runs server-side through `postgres_query`; elsewhere the
|
|
32
|
+
* catalog is local and DuckDB runs it. `build` must produce SQL that both dialects accept.
|
|
33
|
+
*/
|
|
34
|
+
export declare function buildDuckLakeMetadataQuery(options: Pick<DuckLakeStorageOptions, "alias" | "catalog">, build: (table: (tableName: string) => string) => string): string;
|
|
26
35
|
/**
|
|
27
36
|
* Quote a SQL identifier after validating it is already identifier-shaped.
|
|
28
37
|
*
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"sql.d.ts","sourceRoot":"","sources":["../../src/internal/sql.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAEV,mBAAmB,EACnB,sBAAsB,EACtB,sBAAsB,EAEtB,8BAA8B,EAG/B,MAAM,UAAU,CAAA;AAIjB;;GAEG;AACH,wBAAgB,aAAa,CAAC,OAAO,EAAE,IAAI,CAAC,sBAAsB,EAAE,OAAO,CAAC,GAAG,MAAM,CAEpF;AAUD;;;;;GAKG;AACH,wBAAgB,qCAAqC,CACnD,OAAO,EAAE,IAAI,CAAC,sBAAsB,EAAE,OAAO,GAAG,cAAc,CAAC,GAC9D,MAAM,CAOR;AAED;;;GAGG;AACH,wBAAgB,uBAAuB,CACrC,OAAO,EAAE,IAAI,CAAC,sBAAsB,EAAE,cAAc,CAAC,GACpD,SAAS,MAAM,EAAE,CAInB;AAED;;;;;;GAMG;AACH,wBAAgB,yBAAyB,CACvC,OAAO,EAAE,IAAI,CAAC,sBAAsB,EAAE,OAAO,GAAG,SAAS,CAAC,EAC1D,SAAS,EAAE,MAAM,GAChB,MAAM,
|
|
1
|
+
{"version":3,"file":"sql.d.ts","sourceRoot":"","sources":["../../src/internal/sql.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAEV,mBAAmB,EACnB,sBAAsB,EACtB,sBAAsB,EAEtB,8BAA8B,EAG/B,MAAM,UAAU,CAAA;AAIjB;;GAEG;AACH,wBAAgB,aAAa,CAAC,OAAO,EAAE,IAAI,CAAC,sBAAsB,EAAE,OAAO,CAAC,GAAG,MAAM,CAEpF;AAUD;;;;;GAKG;AACH,wBAAgB,qCAAqC,CACnD,OAAO,EAAE,IAAI,CAAC,sBAAsB,EAAE,OAAO,GAAG,cAAc,CAAC,GAC9D,MAAM,CAOR;AAED;;;GAGG;AACH,wBAAgB,uBAAuB,CACrC,OAAO,EAAE,IAAI,CAAC,sBAAsB,EAAE,cAAc,CAAC,GACpD,SAAS,MAAM,EAAE,CAInB;AAED;;;;;;GAMG;AACH,wBAAgB,yBAAyB,CACvC,OAAO,EAAE,IAAI,CAAC,sBAAsB,EAAE,OAAO,GAAG,SAAS,CAAC,EAC1D,SAAS,EAAE,MAAM,GAChB,MAAM,CAIR;AAED;;;;;;;GAOG;AACH,wBAAgB,0BAA0B,CACxC,OAAO,EAAE,IAAI,CAAC,sBAAsB,EAAE,OAAO,GAAG,SAAS,CAAC,EAC1D,KAAK,EAAE,CAAC,KAAK,EAAE,CAAC,SAAS,EAAE,MAAM,KAAK,MAAM,KAAK,MAAM,GACtD,MAAM,CAaR;AAQD;;;;;GAKG;AACH,wBAAgB,eAAe,CAAC,UAAU,EAAE,MAAM,GAAG,MAAM,CAM1D;AAED;;GAEG;AACH,wBAAgB,cAAc,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAEpD;AAED,wBAAgB,kBAAkB,CAAC,MAAM,EAAE,SAAS,MAAM,EAAE,GAAG,MAAM,CAEpE;AAED;;GAEG;AACH,wBAAgB,kBAAkB,CAChC,OAAO,EAAE,IAAI,CAAC,sBAAsB,EAAE,OAAO,CAAC,EAC9C,SAAS,EAAE,MAAM,GAChB,MAAM,CAER;AAED;;GAEG;AACH,wBAAgB,UAAU,CAAC,OAAO,EAAE,sBAAsB,GAAG,MAAM,CAWlE;AAED;;GAEG;AACH,wBAAgB,wBAAwB,CAAC,OAAO,EAAE,8BAA8B,GAAG,MAAM,CAuBxF;AAkBD;;GAEG;AACH,wBAAgB,kBAAkB,CAAC,OAAO,EAAE,sBAAsB,GAAG,SAAS,MAAM,EAAE,CAcrF;AAED;;GAEG;AACH,wBAAgB,oBAAoB,CAAC,MAAM,EAAE,mBAAmB,GAAG,MAAM,CAIxE;AAED;;GAEG;AACH,wBAAgB,cAAc,CAAC,OAAO,EAAE,sBAAsB,GAAG,MAAM,CA+BtE"}
|
package/dist/types.d.ts
CHANGED
|
@@ -137,6 +137,8 @@ export interface DuckLakeStorageOptions {
|
|
|
137
137
|
readonly alias?: string;
|
|
138
138
|
readonly duckdb?: DuckDbRuntimeOptions;
|
|
139
139
|
readonly postgresPool?: DuckLakePostgresPoolOptions;
|
|
140
|
+
/** Maximum native streaming readers per provider; extra reads use pages. Default: 4. */
|
|
141
|
+
readonly maxStreamingReads?: number;
|
|
140
142
|
readonly secrets?: readonly DuckDbSecretOptions[];
|
|
141
143
|
/** Runs after DuckLake/catalog extensions load and before ATTACH. */
|
|
142
144
|
readonly setupSql?: readonly string[];
|
package/dist/types.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AACH,MAAM,MAAM,sBAAsB,GAC9B;IAAE,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC;IAAC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;CAAE,GAClD;IAAE,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC;IAAC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;CAAE,GAClD,8BAA8B;AAChC,2EAA2E;GACzE;IACE,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAA;IACvB,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAA;IACpB,QAAQ,CAAC,UAAU,CAAC,EAAE,SAAS,MAAM,EAAE,CAAA;IACvC,QAAQ,CAAC,kBAAkB,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAA;CAC/D,CAAA;AAEL;;;;;GAKG;AACH,MAAM,WAAW,8BAA8B;IAC7C,QAAQ,CAAC,IAAI,EAAE,UAAU,CAAA;IACzB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IACrB,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAA;IACzB,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,CAAA;IACtB,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,CAAA;IACtB,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;IAC1B,QAAQ,CAAC,OAAO,CAAC,EAAE,SAAS,GAAG,OAAO,GAAG,QAAQ,GAAG,SAAS,GAAG,WAAW,GAAG,aAAa,CAAA;IAC3F,QAAQ,CAAC,eAAe,CAAC,EAAE,MAAM,CAAA;IACjC,QAAQ,CAAC,qBAAqB,CAAC,EAAE,MAAM,CAAA;IACvC,QAAQ,CAAC,cAAc,CAAC,EAAE,MAAM,CAAA;IAChC,QAAQ,CAAC,UAAU,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,GAAG,OAAO,CAAC,CAAC,CAAA;CAC1E;AAED;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACnC,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,CAAA;IACtB,QAAQ,CAAC,MAAM,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAA;CACnD;AAED;;;GAGG;AACH,MAAM,WAAW,2BAA2B;IAC1C,QAAQ,CAAC,cAAc,CAAC,EAAE,MAAM,CAAA;IAChC,QAAQ,CAAC,iBAAiB,CAAC,EAAE,MAAM,CAAA;IACnC,QAAQ,CAAC,iBAAiB,CAAC,EAAE,MAAM,CAAA;IACnC,QAAQ,CAAC,iBAAiB,CAAC,EAAE,MAAM,CAAA;IACnC,QAAQ,CAAC,sBAAsB,CAAC,EAAE,OAAO,CAAA;IACzC,QAAQ,CAAC,kBAAkB,CAAC,EAAE,OAAO,CAAA;IACrC,QAAQ,CAAC,gBAAgB,CAAC,EAAE,MAAM,CAAA;CACnC;AAED;;GAEG;AACH,UAAU,iBAAiB;IACzB,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,CAAA;IACtB,QAAQ,CAAC,UAAU,CAAC,EAAE,OAAO,CAAA;IAC7B,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,CAAA;CACxB;AAED;;;;;GAKG;AACH,UAAU,kBAAmB,SAAQ,iBAAiB;IACpD,QAAQ,CAAC,QAAQ,CAAC,EAAE,QAAQ,GAAG,kBAAkB,CAAA;IACjD,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,CAAA;IACvB,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAA;IACxB,QAAQ,CAAC,YAAY,CAAC,EAAE,MAAM,CAAA;IAC9B,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAA;IACxB,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;IAC1B,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,OAAO,CAAA;IACpC,QAAQ,CAAC,MAAM,CAAC,EAAE,OAAO,CAAA;IACzB,QAAQ,CAAC,SAAS,CAAC,EAAE,OAAO,CAAA;IAC5B,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,CAAA;IACvB,QAAQ,CAAC,OAAO,CAAC,EAAE,MAAM,CAAA;CAC1B;AAED,MAAM,WAAW,eAAgB,SAAQ,kBAAkB;IACzD,QAAQ,CAAC,IAAI,EAAE,IAAI,CAAA;IACnB,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;IAC1B,QAAQ,CAAC,aAAa,CAAC,EAAE,OAAO,CAAA;CACjC;AAED,MAAM,WAAW,eAAgB,SAAQ,kBAAkB;IACzD,QAAQ,CAAC,IAAI,EAAE,IAAI,CAAA;IACnB,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAA;CAC5B;AAED,MAAM,WAAW,gBAAiB,SAAQ,kBAAkB;IAC1D,QAAQ,CAAC,IAAI,EAAE,KAAK,CAAA;CACrB;AAED,MAAM,WAAW,kBAAmB,SAAQ,iBAAiB;IAC3D,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAA;IACtB,QAAQ,CAAC,gBAAgB,CAAC,EAAE,MAAM,CAAA;IAClC,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAA;IAC7B,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,CAAA;IAC5B,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;IAC1B,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;IAC1B,QAAQ,CAAC,YAAY,CAAC,EAAE,MAAM,CAAA;CAC/B;AAED,MAAM,MAAM,mBAAmB,GAC3B,eAAe,GACf,eAAe,GACf,gBAAgB,GAChB,kBAAkB,CAAA;AAEtB,MAAM,WAAW,0BAA0B;IACzC,4EAA4E;IAC5E,QAAQ,CAAC,eAAe,CAAC,EAAE,MAAM,CAAA;IACjC,wFAAwF;IACxF,QAAQ,CAAC,eAAe,CAAC,EAAE,MAAM,CAAA;IACjC,sFAAsF;IACtF,QAAQ,CAAC,MAAM,CAAC,EAAE,OAAO,CAAA;CAC1B;AAED,MAAM,WAAW,yBAAyB;IACxC,QAAQ,CAAC,MAAM,EAAE,OAAO,CAAA;IACxB,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAA;IAChC,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAA;IAChC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAA;IAC1B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAA;IACzB,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAA;CAC/B;AAED;;;;;;;;GAQG;AACH,MAAM,WAAW,sBAAsB;IACrC,QAAQ,CAAC,OAAO,EAAE,sBAAsB,CAAA;IACxC,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;IAC1B,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,CAAA;IACvB,QAAQ,CAAC,MAAM,CAAC,EAAE,oBAAoB,CAAA;IACtC,QAAQ,CAAC,YAAY,CAAC,EAAE,2BAA2B,CAAA;IACnD,QAAQ,CAAC,OAAO,CAAC,EAAE,SAAS,mBAAmB,EAAE,CAAA;IACjD,qEAAqE;IACrE,QAAQ,CAAC,QAAQ,CAAC,EAAE,SAAS,MAAM,EAAE,CAAA;IACrC,QAAQ,CAAC,iBAAiB,CAAC,EAAE,OAAO,CAAA;IACpC,QAAQ,CAAC,QAAQ,CAAC,EAAE,OAAO,CAAA;CAC5B"}
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AACH,MAAM,MAAM,sBAAsB,GAC9B;IAAE,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC;IAAC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;CAAE,GAClD;IAAE,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC;IAAC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;CAAE,GAClD,8BAA8B;AAChC,2EAA2E;GACzE;IACE,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAA;IACvB,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAA;IACpB,QAAQ,CAAC,UAAU,CAAC,EAAE,SAAS,MAAM,EAAE,CAAA;IACvC,QAAQ,CAAC,kBAAkB,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAA;CAC/D,CAAA;AAEL;;;;;GAKG;AACH,MAAM,WAAW,8BAA8B;IAC7C,QAAQ,CAAC,IAAI,EAAE,UAAU,CAAA;IACzB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IACrB,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAA;IACzB,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,CAAA;IACtB,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,CAAA;IACtB,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;IAC1B,QAAQ,CAAC,OAAO,CAAC,EAAE,SAAS,GAAG,OAAO,GAAG,QAAQ,GAAG,SAAS,GAAG,WAAW,GAAG,aAAa,CAAA;IAC3F,QAAQ,CAAC,eAAe,CAAC,EAAE,MAAM,CAAA;IACjC,QAAQ,CAAC,qBAAqB,CAAC,EAAE,MAAM,CAAA;IACvC,QAAQ,CAAC,cAAc,CAAC,EAAE,MAAM,CAAA;IAChC,QAAQ,CAAC,UAAU,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,GAAG,OAAO,CAAC,CAAC,CAAA;CAC1E;AAED;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACnC,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,CAAA;IACtB,QAAQ,CAAC,MAAM,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAA;CACnD;AAED;;;GAGG;AACH,MAAM,WAAW,2BAA2B;IAC1C,QAAQ,CAAC,cAAc,CAAC,EAAE,MAAM,CAAA;IAChC,QAAQ,CAAC,iBAAiB,CAAC,EAAE,MAAM,CAAA;IACnC,QAAQ,CAAC,iBAAiB,CAAC,EAAE,MAAM,CAAA;IACnC,QAAQ,CAAC,iBAAiB,CAAC,EAAE,MAAM,CAAA;IACnC,QAAQ,CAAC,sBAAsB,CAAC,EAAE,OAAO,CAAA;IACzC,QAAQ,CAAC,kBAAkB,CAAC,EAAE,OAAO,CAAA;IACrC,QAAQ,CAAC,gBAAgB,CAAC,EAAE,MAAM,CAAA;CACnC;AAED;;GAEG;AACH,UAAU,iBAAiB;IACzB,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,CAAA;IACtB,QAAQ,CAAC,UAAU,CAAC,EAAE,OAAO,CAAA;IAC7B,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,CAAA;CACxB;AAED;;;;;GAKG;AACH,UAAU,kBAAmB,SAAQ,iBAAiB;IACpD,QAAQ,CAAC,QAAQ,CAAC,EAAE,QAAQ,GAAG,kBAAkB,CAAA;IACjD,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,CAAA;IACvB,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAA;IACxB,QAAQ,CAAC,YAAY,CAAC,EAAE,MAAM,CAAA;IAC9B,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAA;IACxB,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;IAC1B,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,OAAO,CAAA;IACpC,QAAQ,CAAC,MAAM,CAAC,EAAE,OAAO,CAAA;IACzB,QAAQ,CAAC,SAAS,CAAC,EAAE,OAAO,CAAA;IAC5B,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,CAAA;IACvB,QAAQ,CAAC,OAAO,CAAC,EAAE,MAAM,CAAA;CAC1B;AAED,MAAM,WAAW,eAAgB,SAAQ,kBAAkB;IACzD,QAAQ,CAAC,IAAI,EAAE,IAAI,CAAA;IACnB,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;IAC1B,QAAQ,CAAC,aAAa,CAAC,EAAE,OAAO,CAAA;CACjC;AAED,MAAM,WAAW,eAAgB,SAAQ,kBAAkB;IACzD,QAAQ,CAAC,IAAI,EAAE,IAAI,CAAA;IACnB,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAA;CAC5B;AAED,MAAM,WAAW,gBAAiB,SAAQ,kBAAkB;IAC1D,QAAQ,CAAC,IAAI,EAAE,KAAK,CAAA;CACrB;AAED,MAAM,WAAW,kBAAmB,SAAQ,iBAAiB;IAC3D,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAA;IACtB,QAAQ,CAAC,gBAAgB,CAAC,EAAE,MAAM,CAAA;IAClC,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAA;IAC7B,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,CAAA;IAC5B,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;IAC1B,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;IAC1B,QAAQ,CAAC,YAAY,CAAC,EAAE,MAAM,CAAA;CAC/B;AAED,MAAM,MAAM,mBAAmB,GAC3B,eAAe,GACf,eAAe,GACf,gBAAgB,GAChB,kBAAkB,CAAA;AAEtB,MAAM,WAAW,0BAA0B;IACzC,4EAA4E;IAC5E,QAAQ,CAAC,eAAe,CAAC,EAAE,MAAM,CAAA;IACjC,wFAAwF;IACxF,QAAQ,CAAC,eAAe,CAAC,EAAE,MAAM,CAAA;IACjC,sFAAsF;IACtF,QAAQ,CAAC,MAAM,CAAC,EAAE,OAAO,CAAA;CAC1B;AAED,MAAM,WAAW,yBAAyB;IACxC,QAAQ,CAAC,MAAM,EAAE,OAAO,CAAA;IACxB,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAA;IAChC,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAA;IAChC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAA;IAC1B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAA;IACzB,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAA;CAC/B;AAED;;;;;;;;GAQG;AACH,MAAM,WAAW,sBAAsB;IACrC,QAAQ,CAAC,OAAO,EAAE,sBAAsB,CAAA;IACxC,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;IAC1B,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,CAAA;IACvB,QAAQ,CAAC,MAAM,CAAC,EAAE,oBAAoB,CAAA;IACtC,QAAQ,CAAC,YAAY,CAAC,EAAE,2BAA2B,CAAA;IACnD,wFAAwF;IACxF,QAAQ,CAAC,iBAAiB,CAAC,EAAE,MAAM,CAAA;IACnC,QAAQ,CAAC,OAAO,CAAC,EAAE,SAAS,mBAAmB,EAAE,CAAA;IACjD,qEAAqE;IACrE,QAAQ,CAAC,QAAQ,CAAC,EAAE,SAAS,MAAM,EAAE,CAAA;IACrC,QAAQ,CAAC,iBAAiB,CAAC,EAAE,OAAO,CAAA;IACpC,QAAQ,CAAC,QAAQ,CAAC,EAAE,OAAO,CAAA;CAC5B"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@sixb/ducklake",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.8",
|
|
4
4
|
"description": "DuckDB/DuckLake lake storage provider for Sixb",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"sixb",
|
|
@@ -36,7 +36,7 @@
|
|
|
36
36
|
"typecheck": "tsc --noEmit",
|
|
37
37
|
"test": "bun test",
|
|
38
38
|
"test:e2e": "bun run test:e2e:local && bun run test:e2e:remote",
|
|
39
|
-
"test:e2e:local": "bun test ./tests/ducklake-changes.e2e.ts ./tests/ducklake-contract.e2e.ts ./tests/ducklake-file-refs.e2e.ts ./tests/ducklake-catalog-state.e2e.ts ./tests/ducklake-catalogs.e2e.ts ./tests/ducklake-concurrency.e2e.ts ./tests/ducklake-datasets.e2e.ts ./tests/ducklake-driver.e2e.ts ./tests/ducklake-maintenance.e2e.ts ./tests/ducklake-merges.e2e.ts ./tests/ducklake-sql-transforms.e2e.ts ./tests/ducklake-versions.e2e.ts ./tests/ducklake-writes.e2e.ts",
|
|
39
|
+
"test:e2e:local": "bun test ./tests/ducklake-reader.e2e.ts ./tests/ducklake-changes.e2e.ts ./tests/ducklake-contract.e2e.ts ./tests/ducklake-file-refs.e2e.ts ./tests/ducklake-catalog-state.e2e.ts ./tests/ducklake-catalogs.e2e.ts ./tests/ducklake-concurrency.e2e.ts ./tests/ducklake-datasets.e2e.ts ./tests/ducklake-driver.e2e.ts ./tests/ducklake-maintenance.e2e.ts ./tests/ducklake-merges.e2e.ts ./tests/ducklake-sql-transforms.e2e.ts ./tests/ducklake-versions.e2e.ts ./tests/ducklake-long-history.e2e.ts ./tests/ducklake-writes.e2e.ts ./tests/ducklake-sqlite-writes.e2e.ts",
|
|
40
40
|
"test:e2e:remote": "bun test --preload ./tests/setup.ts ./tests/ducklake-remote.e2e.ts",
|
|
41
41
|
"prepack": "bun run build"
|
|
42
42
|
},
|
|
@@ -44,10 +44,10 @@
|
|
|
44
44
|
"@duckdb/node-api": "^1.5.2-r.1"
|
|
45
45
|
},
|
|
46
46
|
"peerDependencies": {
|
|
47
|
-
"@sixb/core": "^0.1.
|
|
47
|
+
"@sixb/core": "^0.1.17"
|
|
48
48
|
},
|
|
49
49
|
"devDependencies": {
|
|
50
|
-
"@sixb/core": "0.1.
|
|
50
|
+
"@sixb/core": "0.1.17",
|
|
51
51
|
"@types/bun": "latest",
|
|
52
52
|
"typescript": "^5.7.0"
|
|
53
53
|
},
|
package/src/ducklake-storage.ts
CHANGED
|
@@ -7,6 +7,8 @@ import type {
|
|
|
7
7
|
DatasetVersion,
|
|
8
8
|
LakeMergeSession,
|
|
9
9
|
LakeWriteSession,
|
|
10
|
+
LatestVersionsSince,
|
|
11
|
+
ListLatestVersionsSinceInput,
|
|
10
12
|
ReadDatasetChangesInput,
|
|
11
13
|
ReadDatasetRowsInput,
|
|
12
14
|
} from "@sixb/core/lake-storage"
|
|
@@ -127,6 +129,12 @@ export class DuckLakeStorage implements LakeStorageWithSql<"duckdb"> {
|
|
|
127
129
|
return this.snapshotReader.getVersion(datasetId, versionId)
|
|
128
130
|
}
|
|
129
131
|
|
|
132
|
+
async listLatestVersionsSince(
|
|
133
|
+
input: ListLatestVersionsSinceInput
|
|
134
|
+
): Promise<LatestVersionsSince | null> {
|
|
135
|
+
return this.snapshotReader.listLatestVersionsSince(input)
|
|
136
|
+
}
|
|
137
|
+
|
|
130
138
|
readRows(input: ReadDatasetRowsInput): AsyncIterable<DatasetRow> {
|
|
131
139
|
return this.rows.readRows(input)
|
|
132
140
|
}
|
|
@@ -0,0 +1,129 @@
|
|
|
1
|
+
import type { DuckDBConnection, DuckDBResult } from "@duckdb/node-api"
|
|
2
|
+
import { LakeStorageError } from "@sixb/core/lake-storage"
|
|
3
|
+
import { sixbDuckDbValueConverter } from "./duckdb-value-converter"
|
|
4
|
+
|
|
5
|
+
/** One native query, on a connection belonging to the runtime's shared instance. */
|
|
6
|
+
export interface DuckDbReader {
|
|
7
|
+
rows(): AsyncIterable<Record<string, unknown>>
|
|
8
|
+
close(): Promise<void>
|
|
9
|
+
}
|
|
10
|
+
|
|
11
|
+
export interface DuckDbReaderOptions {
|
|
12
|
+
readonly signal?: AbortSignal
|
|
13
|
+
readonly onClose?: () => Promise<void>
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
export class NodeDuckDbReader implements DuckDbReader {
|
|
17
|
+
private resultPromise: Promise<DuckDBResult> | undefined
|
|
18
|
+
private pending: Promise<unknown> | undefined
|
|
19
|
+
private closePromise: Promise<void> | undefined
|
|
20
|
+
private finished = false
|
|
21
|
+
private consumed = false
|
|
22
|
+
private readonly onAbort = () => {
|
|
23
|
+
// Keep the rejected cleanup observable through close()/rows(), including when the consumer
|
|
24
|
+
// is paused. The event listener itself cannot return an awaited promise.
|
|
25
|
+
void this.close().catch(() => {})
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
constructor(
|
|
29
|
+
private readonly connection: DuckDBConnection,
|
|
30
|
+
private readonly options: DuckDbReaderOptions
|
|
31
|
+
) {}
|
|
32
|
+
|
|
33
|
+
async start(sql: string): Promise<void> {
|
|
34
|
+
this.options.signal?.throwIfAborted()
|
|
35
|
+
this.options.signal?.addEventListener("abort", this.onAbort, { once: true })
|
|
36
|
+
// Connection-local settings are not inherited by a new connection. Physical offsets must
|
|
37
|
+
// remain stable even when a dataset shadows DuckDB's virtual rowid column.
|
|
38
|
+
await this.track(this.connection.run("SET preserve_insertion_order = true"))
|
|
39
|
+
this.assertOpen()
|
|
40
|
+
const prepared = await this.track(this.connection.prepare(sql))
|
|
41
|
+
try {
|
|
42
|
+
this.assertOpen()
|
|
43
|
+
// Starting the pending query establishes its interruptible context synchronously. Avoid
|
|
44
|
+
// connection.stream() starting execution after an abort that happened during preparation.
|
|
45
|
+
this.resultPromise = this.track(prepared.startStream().getResult()).finally(() => {
|
|
46
|
+
prepared.destroySync()
|
|
47
|
+
})
|
|
48
|
+
// Execute independently after preparation, so a large sort does not occupy the main queue
|
|
49
|
+
// or attachment lease. rows() observes failures even if execution ends before consumption.
|
|
50
|
+
void this.resultPromise.catch(() => {})
|
|
51
|
+
} catch (error) {
|
|
52
|
+
prepared.destroySync()
|
|
53
|
+
throw error
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
async *rows(): AsyncIterable<Record<string, unknown>> {
|
|
58
|
+
if (this.consumed)
|
|
59
|
+
throw new LakeStorageError("[SixbDuckLake] Reader can only be consumed once.")
|
|
60
|
+
this.consumed = true
|
|
61
|
+
try {
|
|
62
|
+
this.assertOpen()
|
|
63
|
+
const result = await this.resultPromise!
|
|
64
|
+
this.assertOpen()
|
|
65
|
+
const batches = result
|
|
66
|
+
.yieldConvertedRowObjects(sixbDuckDbValueConverter)
|
|
67
|
+
[Symbol.asyncIterator]()
|
|
68
|
+
while (true) {
|
|
69
|
+
this.assertOpen()
|
|
70
|
+
const batch = await this.track(batches.next())
|
|
71
|
+
this.assertOpen()
|
|
72
|
+
if (batch.done) {
|
|
73
|
+
this.finished = true
|
|
74
|
+
return
|
|
75
|
+
}
|
|
76
|
+
for (const row of batch.value) {
|
|
77
|
+
this.assertOpen()
|
|
78
|
+
yield row as Record<string, unknown>
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
} finally {
|
|
82
|
+
await this.close()
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
close(): Promise<void> {
|
|
87
|
+
if (!this.closePromise) {
|
|
88
|
+
this.options.signal?.removeEventListener("abort", this.onAbort)
|
|
89
|
+
if (this.pending) this.connection.interrupt()
|
|
90
|
+
this.closePromise = this.finish()
|
|
91
|
+
}
|
|
92
|
+
return this.closePromise
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
private async finish(): Promise<void> {
|
|
96
|
+
try {
|
|
97
|
+
await this.pending?.catch(() => {})
|
|
98
|
+
if (!this.finished) {
|
|
99
|
+
// node-api 1.5.2 exposes no result.close(). Disconnect alone leaves a paused query's
|
|
100
|
+
// transaction alive until GC. A new statement ends it without draining the whole result.
|
|
101
|
+
await this.connection.run("SELECT 1")
|
|
102
|
+
}
|
|
103
|
+
} finally {
|
|
104
|
+
this.resultPromise = undefined
|
|
105
|
+
try {
|
|
106
|
+
this.connection.closeSync()
|
|
107
|
+
} finally {
|
|
108
|
+
await this.options.onClose?.()
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
private async track<T>(operation: Promise<T>): Promise<T> {
|
|
114
|
+
this.pending = operation
|
|
115
|
+
try {
|
|
116
|
+
return await operation
|
|
117
|
+
} catch (error) {
|
|
118
|
+
this.assertOpen()
|
|
119
|
+
throw error
|
|
120
|
+
} finally {
|
|
121
|
+
if (this.pending === operation) this.pending = undefined
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
private assertOpen(): void {
|
|
126
|
+
this.options.signal?.throwIfAborted()
|
|
127
|
+
if (this.closePromise) throw new LakeStorageError("[SixbDuckLake] Reader is closed.")
|
|
128
|
+
}
|
|
129
|
+
}
|