@sixb/ducklake 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +490 -0
- package/dist/ducklake-storage.d.ts +42 -0
- package/dist/ducklake-storage.d.ts.map +1 -0
- package/dist/index.d.ts +3 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +3112 -0
- package/dist/index.js.map +34 -0
- package/dist/internal/catalog-key.d.ts +11 -0
- package/dist/internal/catalog-key.d.ts.map +1 -0
- package/dist/internal/dataset-row-commit.d.ts +26 -0
- package/dist/internal/dataset-row-commit.d.ts.map +1 -0
- package/dist/internal/duckdb-column-types.d.ts +45 -0
- package/dist/internal/duckdb-column-types.d.ts.map +1 -0
- package/dist/internal/duckdb-decimal.d.ts +6 -0
- package/dist/internal/duckdb-decimal.d.ts.map +1 -0
- package/dist/internal/duckdb-row.d.ts +8 -0
- package/dist/internal/duckdb-row.d.ts.map +1 -0
- package/dist/internal/duckdb-runtime.d.ts +63 -0
- package/dist/internal/duckdb-runtime.d.ts.map +1 -0
- package/dist/internal/duckdb-value-converter.d.ts +7 -0
- package/dist/internal/duckdb-value-converter.d.ts.map +1 -0
- package/dist/internal/ducklake-catalog-schema.d.ts +12 -0
- package/dist/internal/ducklake-catalog-schema.d.ts.map +1 -0
- package/dist/internal/ducklake-connection-manager.d.ts +67 -0
- package/dist/internal/ducklake-connection-manager.d.ts.map +1 -0
- package/dist/internal/ducklake-current-dataset-definitions.d.ts +10 -0
- package/dist/internal/ducklake-current-dataset-definitions.d.ts.map +1 -0
- package/dist/internal/ducklake-dataset-catalog.d.ts +33 -0
- package/dist/internal/ducklake-dataset-catalog.d.ts.map +1 -0
- package/dist/internal/ducklake-dataset-table-ref.d.ts +9 -0
- package/dist/internal/ducklake-dataset-table-ref.d.ts.map +1 -0
- package/dist/internal/ducklake-maintenance.d.ts +9 -0
- package/dist/internal/ducklake-maintenance.d.ts.map +1 -0
- package/dist/internal/ducklake-row-reader.d.ts +24 -0
- package/dist/internal/ducklake-row-reader.d.ts.map +1 -0
- package/dist/internal/ducklake-snapshot-reader.d.ts +73 -0
- package/dist/internal/ducklake-snapshot-reader.d.ts.map +1 -0
- package/dist/internal/ducklake-sql-executor.d.ts +33 -0
- package/dist/internal/ducklake-sql-executor.d.ts.map +1 -0
- package/dist/internal/ducklake-write-coordinator.d.ts +53 -0
- package/dist/internal/ducklake-write-coordinator.d.ts.map +1 -0
- package/dist/internal/ducklake-write-session.d.ts +17 -0
- package/dist/internal/ducklake-write-session.d.ts.map +1 -0
- package/dist/internal/names.d.ts +13 -0
- package/dist/internal/names.d.ts.map +1 -0
- package/dist/internal/row-appender.d.ts +7 -0
- package/dist/internal/row-appender.d.ts.map +1 -0
- package/dist/internal/schema.d.ts +35 -0
- package/dist/internal/schema.d.ts.map +1 -0
- package/dist/internal/sql-transform-relations.d.ts +13 -0
- package/dist/internal/sql-transform-relations.d.ts.map +1 -0
- package/dist/internal/sql.d.ts +62 -0
- package/dist/internal/sql.d.ts.map +1 -0
- package/dist/internal/versions.d.ts +22 -0
- package/dist/internal/versions.d.ts.map +1 -0
- package/dist/tsconfig.json +5 -0
- package/dist/types.d.ts +147 -0
- package/dist/types.d.ts.map +1 -0
- package/package.json +64 -0
- package/src/ducklake-storage.ts +130 -0
- package/src/index.ts +14 -0
- package/src/internal/catalog-key.ts +28 -0
- package/src/internal/dataset-row-commit.ts +128 -0
- package/src/internal/duckdb-column-types.ts +68 -0
- package/src/internal/duckdb-decimal.ts +36 -0
- package/src/internal/duckdb-row.ts +83 -0
- package/src/internal/duckdb-runtime.ts +347 -0
- package/src/internal/duckdb-value-converter.ts +27 -0
- package/src/internal/ducklake-catalog-schema.ts +116 -0
- package/src/internal/ducklake-connection-manager.ts +510 -0
- package/src/internal/ducklake-current-dataset-definitions.ts +228 -0
- package/src/internal/ducklake-dataset-catalog.ts +363 -0
- package/src/internal/ducklake-dataset-table-ref.ts +35 -0
- package/src/internal/ducklake-maintenance.ts +116 -0
- package/src/internal/ducklake-row-reader.ts +143 -0
- package/src/internal/ducklake-snapshot-reader.ts +826 -0
- package/src/internal/ducklake-sql-executor.ts +334 -0
- package/src/internal/ducklake-write-coordinator.ts +532 -0
- package/src/internal/ducklake-write-session.ts +170 -0
- package/src/internal/names.ts +111 -0
- package/src/internal/row-appender.ts +73 -0
- package/src/internal/schema.ts +149 -0
- package/src/internal/sql-transform-relations.ts +99 -0
- package/src/internal/sql.ts +440 -0
- package/src/internal/versions.ts +140 -0
- package/src/types.ts +159 -0
package/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Sixb contributors
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
package/README.md
ADDED
|
@@ -0,0 +1,490 @@
|
|
|
1
|
+
# @sixb/ducklake
|
|
2
|
+
|
|
3
|
+
Durable Sixb `LakeStorage` provider backed by DuckDB and the DuckLake
|
|
4
|
+
extension.
|
|
5
|
+
|
|
6
|
+
DuckLake owns physical tables, Parquet files, transactions, snapshots, and time
|
|
7
|
+
travel. Sixb keeps ownership of dataset ids, dataset-definition compatibility,
|
|
8
|
+
version shape, producer metadata, and `fileRef` values.
|
|
9
|
+
|
|
10
|
+
## Usage
|
|
11
|
+
|
|
12
|
+
```ts
|
|
13
|
+
import { DuckLakeStorage } from "@sixb/ducklake"
|
|
14
|
+
|
|
15
|
+
const lakeStorage = new DuckLakeStorage({
|
|
16
|
+
catalog: {
|
|
17
|
+
type: "duckdb",
|
|
18
|
+
path: ".sixb/lake/metadata.ducklake",
|
|
19
|
+
},
|
|
20
|
+
dataPath: ".sixb/lake/data",
|
|
21
|
+
})
|
|
22
|
+
|
|
23
|
+
await lakeStorage.close()
|
|
24
|
+
```
|
|
25
|
+
|
|
26
|
+
## Runtime and Connections
|
|
27
|
+
|
|
28
|
+
`DuckLakeStorage` creates one embedded DuckDB runtime for each provider
|
|
29
|
+
instance. DuckLake attaches lazily on the first lake operation, so constructing
|
|
30
|
+
the provider does not immediately open catalog connections.
|
|
31
|
+
|
|
32
|
+
Within one provider instance, DuckDB work is queued on that runtime. This keeps
|
|
33
|
+
writes, SQL transforms, and metadata reads predictable and keeps PostgreSQL
|
|
34
|
+
connection use bounded.
|
|
35
|
+
|
|
36
|
+
```txt
|
|
37
|
+
one DuckLakeStorage instance
|
|
38
|
+
-> one DuckDB runtime
|
|
39
|
+
-> one DuckLake attachment
|
|
40
|
+
-> one bounded PostgreSQL extension pool, when catalog.type is "postgres"
|
|
41
|
+
```
|
|
42
|
+
|
|
43
|
+
For PostgreSQL catalogs, `postgresPool.maxConnections` is a per-instance budget.
|
|
44
|
+
If four production processes each create a `DuckLakeStorage` with
|
|
45
|
+
`maxConnections: 4`, the deployment can use about sixteen PostgreSQL catalog
|
|
46
|
+
connections for DuckLake metadata.
|
|
47
|
+
|
|
48
|
+
```ts
|
|
49
|
+
const lakeStorage = new DuckLakeStorage({
|
|
50
|
+
catalog: { type: "postgres", host, database, user, password },
|
|
51
|
+
dataPath: "s3://sixb-lake/data",
|
|
52
|
+
duckdb: {
|
|
53
|
+
config: {
|
|
54
|
+
threads: "1",
|
|
55
|
+
},
|
|
56
|
+
},
|
|
57
|
+
postgresPool: {
|
|
58
|
+
maxConnections: 4,
|
|
59
|
+
idleTimeoutMillis: 1_000,
|
|
60
|
+
enableThreadLocalCache: false,
|
|
61
|
+
},
|
|
62
|
+
})
|
|
63
|
+
```
|
|
64
|
+
|
|
65
|
+
Use `close()` during service shutdown so DuckDB can close its runtime and release
|
|
66
|
+
catalog connections cleanly.
|
|
67
|
+
|
|
68
|
+
### Maintenance
|
|
69
|
+
|
|
70
|
+
DuckLake keeps historical snapshots and deleted files until maintenance runs.
|
|
71
|
+
Maintenance is operator-driven: run it on demand from the CLI or call
|
|
72
|
+
`runMaintenance()` directly. It expires snapshots, reclaims deleted file blocks,
|
|
73
|
+
and removes orphaned files through DuckLake's explicit maintenance functions,
|
|
74
|
+
keeping seven days of snapshots/files by default.
|
|
75
|
+
|
|
76
|
+
Run it from the CLI (preview with `--dry-run`):
|
|
77
|
+
|
|
78
|
+
```bash
|
|
79
|
+
sixb lake cleanup --dry-run
|
|
80
|
+
sixb lake cleanup --expire-older-than "14 days" --delete-older-than "14 days"
|
|
81
|
+
```
|
|
82
|
+
|
|
83
|
+
Or run it directly against the provider:
|
|
84
|
+
|
|
85
|
+
```ts
|
|
86
|
+
const report = await lakeStorage.runMaintenance({
|
|
87
|
+
dryRun: true,
|
|
88
|
+
expireOlderThan: "7 days",
|
|
89
|
+
})
|
|
90
|
+
```
|
|
91
|
+
|
|
92
|
+
### Read and Write Concurrency
|
|
93
|
+
|
|
94
|
+
Reads, writes, SQL previews, and SQL transforms use the same DuckDB runtime and
|
|
95
|
+
the same DuckLake PostgreSQL metadata pool. Work is serialized inside one
|
|
96
|
+
`DuckLakeStorage` instance, so a queued DuckDB operation makes later operations
|
|
97
|
+
wait. Only actual DuckDB work holds the runtime: batch appends, commits, SQL
|
|
98
|
+
transforms, and metadata reads. A large streaming read can also make a later
|
|
99
|
+
write wait until the stream is consumed or closed.
|
|
100
|
+
|
|
101
|
+
A write does not hold the runtime while its source iterable is producing rows.
|
|
102
|
+
`writeRows(...)` validates and stages rows in bounded in-memory batches, then
|
|
103
|
+
takes a queue slot only to flush each batch into the staging table. Slow external
|
|
104
|
+
reads (pagination, APIs, SFTP, retries) run outside the queue, so other reads and
|
|
105
|
+
write batches can interleave between a write's batches.
|
|
106
|
+
|
|
107
|
+
```txt
|
|
108
|
+
same provider instance:
|
|
109
|
+
read -> write -> read
|
|
110
|
+
|
|
111
|
+
the operations run in order on one DuckDB runtime
|
|
112
|
+
```
|
|
113
|
+
|
|
114
|
+
This is the simplest and most predictable default, especially for small
|
|
115
|
+
PostgreSQL plans. A future provider option could add a bounded read-runtime pool
|
|
116
|
+
for deployments that have enough PostgreSQL capacity and need more concurrent
|
|
117
|
+
read throughput.
|
|
118
|
+
|
|
119
|
+
Datasets must declare a schema before they can be stored in DuckLake:
|
|
120
|
+
|
|
121
|
+
```ts
|
|
122
|
+
import { col, defineDataset, defineSync } from "@sixb/core"
|
|
123
|
+
|
|
124
|
+
const rawOrdersDataset = defineDataset("raw.erp.orders", {
|
|
125
|
+
schema: [
|
|
126
|
+
col("orderId", "string"),
|
|
127
|
+
col("customerId", "string", { nullable: true }),
|
|
128
|
+
col("total", "float64", { nullable: true }),
|
|
129
|
+
col("createdAt", "timestamp", { nullable: true }),
|
|
130
|
+
col("raw", "json", { nullable: true }),
|
|
131
|
+
],
|
|
132
|
+
partitionBy: ["createdAt"],
|
|
133
|
+
description: "Raw ERP order rows",
|
|
134
|
+
})
|
|
135
|
+
|
|
136
|
+
export const syncOrders = defineSync("sync-orders")
|
|
137
|
+
.from(erp)
|
|
138
|
+
.read(readOrders)
|
|
139
|
+
.intoDataset(rawOrdersDataset)
|
|
140
|
+
```
|
|
141
|
+
|
|
142
|
+
`intoDataset(...)` takes the dataset definition, not a dataset id and options.
|
|
143
|
+
Export the definition from `datasets/` or pass it to
|
|
144
|
+
`createSixb({ datasets: [rawOrdersDataset], ... })` so the runtime and worker
|
|
145
|
+
use the same schema that DuckLake materializes.
|
|
146
|
+
|
|
147
|
+
DuckLake stores Sixb `decimal` columns as `DECIMAL(38, 9)` and returns them as exact strings, including from SQL previews. Writes that need more than 29 integer digits or 9 fractional digits are rejected before DuckDB can round or overflow them.
|
|
148
|
+
|
|
149
|
+
## DuckLake SQL Transforms
|
|
150
|
+
|
|
151
|
+
DuckLake SQL transforms let the provider run dataset-to-dataset transforms in
|
|
152
|
+
DuckDB without materializing source rows in JavaScript. The capability is exposed
|
|
153
|
+
on `DuckLakeStorage`:
|
|
154
|
+
|
|
155
|
+
```ts
|
|
156
|
+
lakeStorage.standard.id // "ducklake"
|
|
157
|
+
lakeStorage.standard.version // "1.0"
|
|
158
|
+
lakeStorage.sql.dialect // "duckdb"
|
|
159
|
+
```
|
|
160
|
+
|
|
161
|
+
The SQL callback receives provider-owned relation handles. Interpolate those
|
|
162
|
+
handles where a relation belongs; do not use physical table names. The provider
|
|
163
|
+
renders each handle with DuckLake time travel and rejects unresolved or fabricated
|
|
164
|
+
placeholders.
|
|
165
|
+
|
|
166
|
+
```ts
|
|
167
|
+
const previewRows = []
|
|
168
|
+
|
|
169
|
+
for await (const row of lakeStorage.sql.preview({
|
|
170
|
+
sources: {
|
|
171
|
+
orders: { dataset: rawOrdersDataset },
|
|
172
|
+
},
|
|
173
|
+
limit: 25,
|
|
174
|
+
sql: ({ orders }) => `
|
|
175
|
+
select
|
|
176
|
+
customerId,
|
|
177
|
+
count(*)::BIGINT as orders,
|
|
178
|
+
sum(total)::DOUBLE as revenue
|
|
179
|
+
from ${orders}
|
|
180
|
+
group by customerId
|
|
181
|
+
order by customerId
|
|
182
|
+
`,
|
|
183
|
+
})) {
|
|
184
|
+
previewRows.push(row)
|
|
185
|
+
}
|
|
186
|
+
```
|
|
187
|
+
|
|
188
|
+
`preview(...)` is for development and UI inspection. It always applies a bounded
|
|
189
|
+
provider limit; omitted limits default to 100 rows and requested limits are capped
|
|
190
|
+
at 1000 rows.
|
|
191
|
+
|
|
192
|
+
`execute(...)` writes the transform result as a normal `DatasetVersion`. Target
|
|
193
|
+
datasets must already exist, result columns must match the target schema by name,
|
|
194
|
+
order, and compatible DuckDB type, and resolved source versions are recorded in
|
|
195
|
+
`DatasetVersion.inputs`.
|
|
196
|
+
|
|
197
|
+
```ts
|
|
198
|
+
const customerInsightsDataset = defineDataset("analytics.customer_insights", {
|
|
199
|
+
schema: [
|
|
200
|
+
col("customerId", "string"),
|
|
201
|
+
col("orders", "int64"),
|
|
202
|
+
col("revenue", "float64", { nullable: true }),
|
|
203
|
+
],
|
|
204
|
+
})
|
|
205
|
+
|
|
206
|
+
const version = await lakeStorage.sql.execute({
|
|
207
|
+
sources: {
|
|
208
|
+
orders: { dataset: rawOrdersDataset, versionId: "ducklake:42" },
|
|
209
|
+
},
|
|
210
|
+
target: customerInsightsDataset,
|
|
211
|
+
mode: "snapshot",
|
|
212
|
+
producer: { kind: "pipeline", id: "customer-insights", runId: "run_123" },
|
|
213
|
+
sql: ({ orders }) => `
|
|
214
|
+
select
|
|
215
|
+
customerId,
|
|
216
|
+
count(*)::BIGINT as orders,
|
|
217
|
+
sum(total)::DOUBLE as revenue
|
|
218
|
+
from ${orders}
|
|
219
|
+
group by customerId
|
|
220
|
+
`,
|
|
221
|
+
})
|
|
222
|
+
```
|
|
223
|
+
|
|
224
|
+
Use `mode: "snapshot"` to replace the target contents, or `mode: "append"` to add
|
|
225
|
+
new rows to the existing target. Omitting a source `versionId` resolves the latest
|
|
226
|
+
committed version at execution time; providing a `versionId` pins that source for
|
|
227
|
+
repeatable transforms.
|
|
228
|
+
|
|
229
|
+
## Definition Updates
|
|
230
|
+
|
|
231
|
+
DuckLake supports schema evolution, and this provider exposes a small safe
|
|
232
|
+
subset of dataset definition updates through repeated `createDataset(...)`
|
|
233
|
+
calls.
|
|
234
|
+
|
|
235
|
+
Schema changes:
|
|
236
|
+
|
|
237
|
+
- add nullable top-level columns
|
|
238
|
+
- reorder existing column declarations without changing the stored table
|
|
239
|
+
|
|
240
|
+
Metadata changes:
|
|
241
|
+
|
|
242
|
+
- add compatible metadata such as `description` or `partitionBy`
|
|
243
|
+
|
|
244
|
+
Rejected changes:
|
|
245
|
+
|
|
246
|
+
- add required columns
|
|
247
|
+
- drop columns
|
|
248
|
+
- rename columns
|
|
249
|
+
- change column types
|
|
250
|
+
- change column nullability
|
|
251
|
+
- change existing `description` or `partitionBy` metadata
|
|
252
|
+
|
|
253
|
+
Sixb matches existing columns by name, not by declaration position. If a
|
|
254
|
+
developer declares a new nullable column between existing columns, DuckLake still
|
|
255
|
+
stores that new physical column at the end of the table. Later `getDataset(...)`
|
|
256
|
+
calls return DuckLake's stored order.
|
|
257
|
+
|
|
258
|
+
Schema-only DuckLake commits count as Sixb dataset versions. After a schema
|
|
259
|
+
change, `getLatestVersion(...)` and `listVersions(...)` include the schema
|
|
260
|
+
version with `DatasetVersion.mode: "schema"` even if no rows were written in
|
|
261
|
+
that commit. Historical `DatasetVersion.schema` values come from DuckLake
|
|
262
|
+
metadata at that snapshot, so versioned reads validate projected columns against
|
|
263
|
+
the schema that existed at the requested version.
|
|
264
|
+
|
|
265
|
+
## Catalogs
|
|
266
|
+
|
|
267
|
+
Supported catalogs:
|
|
268
|
+
|
|
269
|
+
- `duckdb`: local DuckDB metadata file.
|
|
270
|
+
- `sqlite`: local SQLite metadata file.
|
|
271
|
+
- `postgres`: PostgreSQL metadata catalog, with optional `metadataSchema`.
|
|
272
|
+
- `custom`: escape hatch for DuckLake catalog URI forms Sixb does not model
|
|
273
|
+
yet, including DuckLake secrets.
|
|
274
|
+
|
|
275
|
+
```ts
|
|
276
|
+
const lakeStorage = new DuckLakeStorage({
|
|
277
|
+
catalog: {
|
|
278
|
+
type: "postgres",
|
|
279
|
+
host: "127.0.0.1",
|
|
280
|
+
port: 5432,
|
|
281
|
+
database: "ducklake",
|
|
282
|
+
user: "postgres",
|
|
283
|
+
password: process.env.POSTGRES_PASSWORD,
|
|
284
|
+
metadataSchema: "sixb_lake",
|
|
285
|
+
},
|
|
286
|
+
dataPath: "s3://sixb-lake/data",
|
|
287
|
+
secrets: [
|
|
288
|
+
{
|
|
289
|
+
type: "s3",
|
|
290
|
+
keyId: process.env.AWS_ACCESS_KEY_ID,
|
|
291
|
+
secret: process.env.AWS_SECRET_ACCESS_KEY,
|
|
292
|
+
region: "us-east-1",
|
|
293
|
+
scope: "s3://sixb-lake",
|
|
294
|
+
},
|
|
295
|
+
],
|
|
296
|
+
})
|
|
297
|
+
```
|
|
298
|
+
|
|
299
|
+
## PostgreSQL Configuration
|
|
300
|
+
|
|
301
|
+
Use a PostgreSQL catalog for production or any deployment where multiple Sixb
|
|
302
|
+
processes need to see the same DuckLake snapshots. Local DuckDB or SQLite
|
|
303
|
+
catalogs are better suited to development and tests.
|
|
304
|
+
|
|
305
|
+
PostgreSQL stores only DuckLake metadata. Dataset table data still lives under
|
|
306
|
+
`dataPath`, usually in object storage such as S3, R2, GCS, or Azure Blob.
|
|
307
|
+
|
|
308
|
+
```txt
|
|
309
|
+
PostgreSQL catalog -> snapshots, schemas, transactions, metadata
|
|
310
|
+
dataPath -> physical DuckLake table data and files
|
|
311
|
+
BlobStorage -> Sixb fileRef payload bytes
|
|
312
|
+
```
|
|
313
|
+
|
|
314
|
+
Recommended production shape:
|
|
315
|
+
|
|
316
|
+
```ts
|
|
317
|
+
const lakeStorage = new DuckLakeStorage({
|
|
318
|
+
catalog: {
|
|
319
|
+
type: "postgres",
|
|
320
|
+
host: process.env.PGHOST!,
|
|
321
|
+
port: Number(process.env.PGPORT ?? 5432),
|
|
322
|
+
database: process.env.PGDATABASE!,
|
|
323
|
+
user: process.env.PGUSER!,
|
|
324
|
+
password: process.env.PGPASSWORD!,
|
|
325
|
+
sslMode: "require",
|
|
326
|
+
applicationName: "sixb-api",
|
|
327
|
+
connectTimeoutSeconds: 10,
|
|
328
|
+
metadataSchema: "sixb_lake",
|
|
329
|
+
},
|
|
330
|
+
dataPath: "s3://sixb-lake/data",
|
|
331
|
+
duckdb: {
|
|
332
|
+
config: {
|
|
333
|
+
threads: "2",
|
|
334
|
+
},
|
|
335
|
+
},
|
|
336
|
+
postgresPool: {
|
|
337
|
+
maxConnections: 8,
|
|
338
|
+
waitTimeoutMillis: 30_000,
|
|
339
|
+
idleTimeoutMillis: 1_000,
|
|
340
|
+
maxLifetimeMillis: 300_000,
|
|
341
|
+
enableThreadLocalCache: false,
|
|
342
|
+
},
|
|
343
|
+
secrets: [
|
|
344
|
+
{
|
|
345
|
+
type: "s3",
|
|
346
|
+
keyId: process.env.AWS_ACCESS_KEY_ID,
|
|
347
|
+
secret: process.env.AWS_SECRET_ACCESS_KEY,
|
|
348
|
+
region: "us-east-1",
|
|
349
|
+
scope: "s3://sixb-lake",
|
|
350
|
+
},
|
|
351
|
+
],
|
|
352
|
+
})
|
|
353
|
+
```
|
|
354
|
+
|
|
355
|
+
### What Each Setting Controls
|
|
356
|
+
|
|
357
|
+
| Setting | Controls |
|
|
358
|
+
| --- | --- |
|
|
359
|
+
| `catalog` | The PostgreSQL database DuckLake uses for metadata. |
|
|
360
|
+
| `catalog.metadataSchema` | The schema that holds DuckLake metadata tables. |
|
|
361
|
+
| `dataPath` | Where DuckLake stores physical table data. |
|
|
362
|
+
| `duckdb.config.threads` | Local DuckDB worker threads inside this process. |
|
|
363
|
+
| `postgresPool` | DuckDB PostgreSQL extension connections for DuckLake metadata. |
|
|
364
|
+
|
|
365
|
+
This pool is separate from any normal Sixb `PostgresStorage` pool. Count both
|
|
366
|
+
when sizing a small database.
|
|
367
|
+
|
|
368
|
+
### Connection Sizing
|
|
369
|
+
|
|
370
|
+
Start with the smallest budget that supports your workload, then raise it after
|
|
371
|
+
observing real latency and PostgreSQL connection use.
|
|
372
|
+
|
|
373
|
+
| Deployment | `duckdb.config.threads` | `postgresPool.maxConnections` | Notes |
|
|
374
|
+
| --- | ---: | ---: | --- |
|
|
375
|
+
| Small managed PostgreSQL, about 20 connections | `"1"` | `4` | Run only the services that need the lake provider. |
|
|
376
|
+
| Normal production service | `"2"` to `"4"` | `8` to `16` | Good default range for API and worker processes. |
|
|
377
|
+
| Larger read-heavy deployment | `"4"` or more | `16` or more | Scale only with PostgreSQL headroom and monitoring. |
|
|
378
|
+
|
|
379
|
+
The budget multiplies by process count:
|
|
380
|
+
|
|
381
|
+
```txt
|
|
382
|
+
estimated DuckLake catalog connections =
|
|
383
|
+
number of processes with DuckLakeStorage * postgresPool.maxConnections
|
|
384
|
+
```
|
|
385
|
+
|
|
386
|
+
If the PostgreSQL role has a `CONNECTION LIMIT`, size it for the whole
|
|
387
|
+
deployment:
|
|
388
|
+
|
|
389
|
+
```txt
|
|
390
|
+
role connection limit >=
|
|
391
|
+
DuckLake catalog connections
|
|
392
|
+
+ normal application PostgreSQL pool connections
|
|
393
|
+
+ migration, admin, and monitoring headroom
|
|
394
|
+
```
|
|
395
|
+
|
|
396
|
+
Leave headroom for the normal Sixb PostgreSQL storage provider, migrations,
|
|
397
|
+
admin sessions, monitoring, and PostgreSQL reserved connections.
|
|
398
|
+
|
|
399
|
+
### Pool Options
|
|
400
|
+
|
|
401
|
+
`postgresPool` maps to DuckDB's PostgreSQL extension pool settings for the
|
|
402
|
+
DuckLake metadata catalog.
|
|
403
|
+
|
|
404
|
+
| Option | Use it for |
|
|
405
|
+
| --- | --- |
|
|
406
|
+
| `maxConnections` | Hard cap for DuckDB PostgreSQL extension connections. |
|
|
407
|
+
| `waitTimeoutMillis` | How long a request can wait for a pool connection. |
|
|
408
|
+
| `idleTimeoutMillis` | How quickly unused pool connections are released. |
|
|
409
|
+
| `maxLifetimeMillis` | Recycling long-lived PostgreSQL connections. |
|
|
410
|
+
| `enableThreadLocalCache` | Keep `false` for constrained databases. |
|
|
411
|
+
| `healthCheckQuery` | Custom pool connection health checks, rarely needed. |
|
|
412
|
+
|
|
413
|
+
Prefer `postgresPool` over raw `setupSql` for PostgreSQL pool tuning. The
|
|
414
|
+
provider applies the settings before and after DuckLake attachment so the
|
|
415
|
+
metadata catalog uses the intended pool.
|
|
416
|
+
|
|
417
|
+
### Service Layout
|
|
418
|
+
|
|
419
|
+
Each service process that constructs `DuckLakeStorage` owns its own DuckDB
|
|
420
|
+
runtime and PostgreSQL pool. Avoid creating the lake provider in services that
|
|
421
|
+
do not need it.
|
|
422
|
+
|
|
423
|
+
```txt
|
|
424
|
+
needs DuckLakeStorage:
|
|
425
|
+
api reads, sync worker, pipeline worker, projection worker
|
|
426
|
+
|
|
427
|
+
does not need DuckLakeStorage:
|
|
428
|
+
static app server, health-only process, services that only use BlobStorage
|
|
429
|
+
```
|
|
430
|
+
|
|
431
|
+
On very small PostgreSQL plans, run fewer lake-owning processes or group workers
|
|
432
|
+
into one process before lowering `maxConnections` too far. A pool budget below
|
|
433
|
+
`4` can become fragile because one runtime may need several metadata operations
|
|
434
|
+
during attach, commit, and snapshot hydration.
|
|
435
|
+
|
|
436
|
+
Use `custom` when you need to pass a DuckLake catalog URI that Sixb does not
|
|
437
|
+
model directly:
|
|
438
|
+
|
|
439
|
+
```ts
|
|
440
|
+
const lakeStorage = new DuckLakeStorage({
|
|
441
|
+
catalog: {
|
|
442
|
+
type: "custom",
|
|
443
|
+
uri: "postgres:dbname=ducklake host=127.0.0.1 user=postgres",
|
|
444
|
+
extensions: ["postgres"],
|
|
445
|
+
},
|
|
446
|
+
dataPath: "s3://sixb-lake/data",
|
|
447
|
+
})
|
|
448
|
+
```
|
|
449
|
+
|
|
450
|
+
`setupSql` runs after required DuckDB extensions load and before DuckLake
|
|
451
|
+
`ATTACH`, so callers can install extra extensions, set DuckDB options, or create
|
|
452
|
+
secrets manually when the typed options are not enough.
|
|
453
|
+
|
|
454
|
+
## Blob Storage
|
|
455
|
+
|
|
456
|
+
DuckLake stores `fileRef` values as dataset row metadata only. Blob payload
|
|
457
|
+
bytes are owned by a separate Sixb `BlobStorage` provider, such as
|
|
458
|
+
`@sixb/blob-local`, and should be composed beside this lake provider when
|
|
459
|
+
creating a runtime.
|
|
460
|
+
|
|
461
|
+
## Tests
|
|
462
|
+
|
|
463
|
+
Fast unit tests:
|
|
464
|
+
|
|
465
|
+
```bash
|
|
466
|
+
bun --filter @sixb/ducklake test
|
|
467
|
+
```
|
|
468
|
+
|
|
469
|
+
Local DuckLake e2e tests:
|
|
470
|
+
|
|
471
|
+
```bash
|
|
472
|
+
bun --filter @sixb/ducklake test:e2e:local
|
|
473
|
+
```
|
|
474
|
+
|
|
475
|
+
Docker-backed PostgreSQL and MinIO/S3-compatible e2e tests:
|
|
476
|
+
|
|
477
|
+
```bash
|
|
478
|
+
bun --filter @sixb/ducklake test:e2e:remote
|
|
479
|
+
```
|
|
480
|
+
|
|
481
|
+
`test:e2e` runs the local suite first, then the remote suite. Remote e2e
|
|
482
|
+
container setup and teardown live in `tests/setup.ts`, following the same
|
|
483
|
+
preload pattern as the other Docker-backed packages in this repo.
|
|
484
|
+
|
|
485
|
+
Relevant upstream docs:
|
|
486
|
+
|
|
487
|
+
- [DuckLake connecting](https://ducklake.select/docs/stable/duckdb/usage/connecting)
|
|
488
|
+
- [DuckLake catalog choices](https://ducklake.select/docs/stable/duckdb/usage/choosing_a_catalog_database)
|
|
489
|
+
- [DuckDB S3 API and secrets](https://duckdb.org/docs/current/core_extensions/httpfs/s3api)
|
|
490
|
+
- [DuckDB Azure extension](https://duckdb.org/docs/current/core_extensions/azure)
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
import type { DatasetDefinition, DatasetRow, LakeStorageWithSql } from "@sixb/core";
|
|
2
|
+
import type { BeginDatasetWriteInput, DatasetCatalogState, DatasetVersion, LakeWriteSession, ReadDatasetRowsInput } from "@sixb/core/lake-storage";
|
|
3
|
+
import { DuckLakeSqlExecutor } from "./internal/ducklake-sql-executor";
|
|
4
|
+
import type { DuckLakeMaintenanceOptions, DuckLakeMaintenanceReport, DuckLakeStorageOptions } from "./types";
|
|
5
|
+
/**
|
|
6
|
+
* DuckLake-backed implementation of Sixb's LakeStorage API.
|
|
7
|
+
*
|
|
8
|
+
* The public surface stays aligned with core LakeStorage while DuckLake owns
|
|
9
|
+
* physical tables, schemas, snapshots, and transactions behind the provider.
|
|
10
|
+
* Sixb-specific metadata is only used to reconstruct DatasetVersion shape.
|
|
11
|
+
*/
|
|
12
|
+
export declare class DuckLakeStorage implements LakeStorageWithSql<"duckdb"> {
|
|
13
|
+
readonly standard: {
|
|
14
|
+
readonly id: "ducklake";
|
|
15
|
+
readonly version: "1.0";
|
|
16
|
+
};
|
|
17
|
+
readonly sql: DuckLakeSqlExecutor;
|
|
18
|
+
private readonly connections;
|
|
19
|
+
private readonly datasets;
|
|
20
|
+
private readonly maintenance;
|
|
21
|
+
private readonly rows;
|
|
22
|
+
private readonly snapshotReader;
|
|
23
|
+
private readonly writes;
|
|
24
|
+
constructor(options: DuckLakeStorageOptions);
|
|
25
|
+
createDataset(definition: DatasetDefinition): Promise<DatasetDefinition>;
|
|
26
|
+
assertDatasetDefinitionsCompatible(definitions: readonly DatasetDefinition[]): Promise<void>;
|
|
27
|
+
getDataset(datasetId: string): Promise<DatasetDefinition | null>;
|
|
28
|
+
listDatasets(): Promise<readonly DatasetDefinition[]>;
|
|
29
|
+
listDatasetCatalogState(datasetIds: readonly string[]): Promise<readonly DatasetCatalogState[]>;
|
|
30
|
+
listVersions(datasetId: string, limit?: number): Promise<readonly DatasetVersion[]>;
|
|
31
|
+
beginWrite(input: BeginDatasetWriteInput): Promise<LakeWriteSession>;
|
|
32
|
+
getLatestVersion(datasetId: string): Promise<DatasetVersion | null>;
|
|
33
|
+
getVersion(datasetId: string, versionId: string): Promise<DatasetVersion | null>;
|
|
34
|
+
readRows(input: ReadDatasetRowsInput): AsyncIterable<DatasetRow>;
|
|
35
|
+
runMaintenance(options?: DuckLakeMaintenanceOptions): Promise<DuckLakeMaintenanceReport>;
|
|
36
|
+
/**
|
|
37
|
+
* `close()` is provider-specific and intentionally not part of the core
|
|
38
|
+
* LakeStorage interface.
|
|
39
|
+
*/
|
|
40
|
+
close(): Promise<void>;
|
|
41
|
+
}
|
|
42
|
+
//# sourceMappingURL=ducklake-storage.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"ducklake-storage.d.ts","sourceRoot":"","sources":["../src/ducklake-storage.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,iBAAiB,EAAE,UAAU,EAAE,kBAAkB,EAAE,MAAM,YAAY,CAAA;AACnF,OAAO,KAAK,EACV,sBAAsB,EACtB,mBAAmB,EACnB,cAAc,EACd,gBAAgB,EAChB,oBAAoB,EACrB,MAAM,yBAAyB,CAAA;AAMhC,OAAO,EAAE,mBAAmB,EAAE,MAAM,kCAAkC,CAAA;AAEtE,OAAO,KAAK,EACV,0BAA0B,EAC1B,yBAAyB,EACzB,sBAAsB,EACvB,MAAM,SAAS,CAAA;AAEhB;;;;;;GAMG;AACH,qBAAa,eAAgB,YAAW,kBAAkB,CAAC,QAAQ,CAAC;IAClE,QAAQ,CAAC,QAAQ;;;MAA8C;IAC/D,QAAQ,CAAC,GAAG,EAAE,mBAAmB,CAAA;IAEjC,OAAO,CAAC,QAAQ,CAAC,WAAW,CAA2B;IACvD,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAwB;IACjD,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAqB;IACjD,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAmB;IACxC,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAwB;IACvD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAA0B;gBAErC,OAAO,EAAE,sBAAsB;IAmCrC,aAAa,CAAC,UAAU,EAAE,iBAAiB,GAAG,OAAO,CAAC,iBAAiB,CAAC;IAIxE,kCAAkC,CACtC,WAAW,EAAE,SAAS,iBAAiB,EAAE,GACxC,OAAO,CAAC,IAAI,CAAC;IAIV,UAAU,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,iBAAiB,GAAG,IAAI,CAAC;IAIhE,YAAY,IAAI,OAAO,CAAC,SAAS,iBAAiB,EAAE,CAAC;IAIrD,uBAAuB,CAC3B,UAAU,EAAE,SAAS,MAAM,EAAE,GAC5B,OAAO,CAAC,SAAS,mBAAmB,EAAE,CAAC;IAIpC,YAAY,CAAC,SAAS,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,SAAS,cAAc,EAAE,CAAC;IAInF,UAAU,CAAC,KAAK,EAAE,sBAAsB,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAIpE,gBAAgB,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,cAAc,GAAG,IAAI,CAAC;IAInE,UAAU,CAAC,SAAS,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,cAAc,GAAG,IAAI,CAAC;IAItF,QAAQ,CAAC,KAAK,EAAE,oBAAoB,GAAG,aAAa,CAAC,UAAU,CAAC;IAI1D,cAAc,CAAC,OAAO,CAAC,EAAE,0BAA0B,GAAG,OAAO,CAAC,yBAAyB,CAAC;IAI9F;;;OAGG;IACG,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;CAG7B"}
|
package/dist/index.d.ts
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
1
|
+
export { DuckLakeStorage } from "./ducklake-storage";
|
|
2
|
+
export type { AzureSecretOptions, DuckDbRuntimeOptions, DuckDbSecretOptions, DuckLakeCatalogOptions, DuckLakeMaintenanceOptions, DuckLakeMaintenanceReport, DuckLakePostgresPoolOptions, DuckLakeStorageOptions, GcsSecretOptions, R2SecretOptions, S3SecretOptions, } from "./types";
|
|
3
|
+
//# sourceMappingURL=index.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAA;AACpD,YAAY,EACV,kBAAkB,EAClB,oBAAoB,EACpB,mBAAmB,EACnB,sBAAsB,EACtB,0BAA0B,EAC1B,yBAAyB,EACzB,2BAA2B,EAC3B,sBAAsB,EACtB,gBAAgB,EAChB,eAAe,EACf,eAAe,GAChB,MAAM,SAAS,CAAA"}
|