@sixb/ducklake 0.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (87) hide show
  1. package/LICENSE +21 -0
  2. package/README.md +490 -0
  3. package/dist/ducklake-storage.d.ts +42 -0
  4. package/dist/ducklake-storage.d.ts.map +1 -0
  5. package/dist/index.d.ts +3 -0
  6. package/dist/index.d.ts.map +1 -0
  7. package/dist/index.js +3112 -0
  8. package/dist/index.js.map +34 -0
  9. package/dist/internal/catalog-key.d.ts +11 -0
  10. package/dist/internal/catalog-key.d.ts.map +1 -0
  11. package/dist/internal/dataset-row-commit.d.ts +26 -0
  12. package/dist/internal/dataset-row-commit.d.ts.map +1 -0
  13. package/dist/internal/duckdb-column-types.d.ts +45 -0
  14. package/dist/internal/duckdb-column-types.d.ts.map +1 -0
  15. package/dist/internal/duckdb-decimal.d.ts +6 -0
  16. package/dist/internal/duckdb-decimal.d.ts.map +1 -0
  17. package/dist/internal/duckdb-row.d.ts +8 -0
  18. package/dist/internal/duckdb-row.d.ts.map +1 -0
  19. package/dist/internal/duckdb-runtime.d.ts +63 -0
  20. package/dist/internal/duckdb-runtime.d.ts.map +1 -0
  21. package/dist/internal/duckdb-value-converter.d.ts +7 -0
  22. package/dist/internal/duckdb-value-converter.d.ts.map +1 -0
  23. package/dist/internal/ducklake-catalog-schema.d.ts +12 -0
  24. package/dist/internal/ducklake-catalog-schema.d.ts.map +1 -0
  25. package/dist/internal/ducklake-connection-manager.d.ts +67 -0
  26. package/dist/internal/ducklake-connection-manager.d.ts.map +1 -0
  27. package/dist/internal/ducklake-current-dataset-definitions.d.ts +10 -0
  28. package/dist/internal/ducklake-current-dataset-definitions.d.ts.map +1 -0
  29. package/dist/internal/ducklake-dataset-catalog.d.ts +33 -0
  30. package/dist/internal/ducklake-dataset-catalog.d.ts.map +1 -0
  31. package/dist/internal/ducklake-dataset-table-ref.d.ts +9 -0
  32. package/dist/internal/ducklake-dataset-table-ref.d.ts.map +1 -0
  33. package/dist/internal/ducklake-maintenance.d.ts +9 -0
  34. package/dist/internal/ducklake-maintenance.d.ts.map +1 -0
  35. package/dist/internal/ducklake-row-reader.d.ts +24 -0
  36. package/dist/internal/ducklake-row-reader.d.ts.map +1 -0
  37. package/dist/internal/ducklake-snapshot-reader.d.ts +73 -0
  38. package/dist/internal/ducklake-snapshot-reader.d.ts.map +1 -0
  39. package/dist/internal/ducklake-sql-executor.d.ts +33 -0
  40. package/dist/internal/ducklake-sql-executor.d.ts.map +1 -0
  41. package/dist/internal/ducklake-write-coordinator.d.ts +53 -0
  42. package/dist/internal/ducklake-write-coordinator.d.ts.map +1 -0
  43. package/dist/internal/ducklake-write-session.d.ts +17 -0
  44. package/dist/internal/ducklake-write-session.d.ts.map +1 -0
  45. package/dist/internal/names.d.ts +13 -0
  46. package/dist/internal/names.d.ts.map +1 -0
  47. package/dist/internal/row-appender.d.ts +7 -0
  48. package/dist/internal/row-appender.d.ts.map +1 -0
  49. package/dist/internal/schema.d.ts +35 -0
  50. package/dist/internal/schema.d.ts.map +1 -0
  51. package/dist/internal/sql-transform-relations.d.ts +13 -0
  52. package/dist/internal/sql-transform-relations.d.ts.map +1 -0
  53. package/dist/internal/sql.d.ts +62 -0
  54. package/dist/internal/sql.d.ts.map +1 -0
  55. package/dist/internal/versions.d.ts +22 -0
  56. package/dist/internal/versions.d.ts.map +1 -0
  57. package/dist/tsconfig.json +5 -0
  58. package/dist/types.d.ts +147 -0
  59. package/dist/types.d.ts.map +1 -0
  60. package/package.json +64 -0
  61. package/src/ducklake-storage.ts +130 -0
  62. package/src/index.ts +14 -0
  63. package/src/internal/catalog-key.ts +28 -0
  64. package/src/internal/dataset-row-commit.ts +128 -0
  65. package/src/internal/duckdb-column-types.ts +68 -0
  66. package/src/internal/duckdb-decimal.ts +36 -0
  67. package/src/internal/duckdb-row.ts +83 -0
  68. package/src/internal/duckdb-runtime.ts +347 -0
  69. package/src/internal/duckdb-value-converter.ts +27 -0
  70. package/src/internal/ducklake-catalog-schema.ts +116 -0
  71. package/src/internal/ducklake-connection-manager.ts +510 -0
  72. package/src/internal/ducklake-current-dataset-definitions.ts +228 -0
  73. package/src/internal/ducklake-dataset-catalog.ts +363 -0
  74. package/src/internal/ducklake-dataset-table-ref.ts +35 -0
  75. package/src/internal/ducklake-maintenance.ts +116 -0
  76. package/src/internal/ducklake-row-reader.ts +143 -0
  77. package/src/internal/ducklake-snapshot-reader.ts +826 -0
  78. package/src/internal/ducklake-sql-executor.ts +334 -0
  79. package/src/internal/ducklake-write-coordinator.ts +532 -0
  80. package/src/internal/ducklake-write-session.ts +170 -0
  81. package/src/internal/names.ts +111 -0
  82. package/src/internal/row-appender.ts +73 -0
  83. package/src/internal/schema.ts +149 -0
  84. package/src/internal/sql-transform-relations.ts +99 -0
  85. package/src/internal/sql.ts +440 -0
  86. package/src/internal/versions.ts +140 -0
  87. package/src/types.ts +159 -0
package/LICENSE ADDED
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Sixb contributors
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
package/README.md ADDED
@@ -0,0 +1,490 @@
1
+ # @sixb/ducklake
2
+
3
+ Durable Sixb `LakeStorage` provider backed by DuckDB and the DuckLake
4
+ extension.
5
+
6
+ DuckLake owns physical tables, Parquet files, transactions, snapshots, and time
7
+ travel. Sixb keeps ownership of dataset ids, dataset-definition compatibility,
8
+ version shape, producer metadata, and `fileRef` values.
9
+
10
+ ## Usage
11
+
12
+ ```ts
13
+ import { DuckLakeStorage } from "@sixb/ducklake"
14
+
15
+ const lakeStorage = new DuckLakeStorage({
16
+ catalog: {
17
+ type: "duckdb",
18
+ path: ".sixb/lake/metadata.ducklake",
19
+ },
20
+ dataPath: ".sixb/lake/data",
21
+ })
22
+
23
+ await lakeStorage.close()
24
+ ```
25
+
26
+ ## Runtime and Connections
27
+
28
+ `DuckLakeStorage` creates one embedded DuckDB runtime for each provider
29
+ instance. DuckLake attaches lazily on the first lake operation, so constructing
30
+ the provider does not immediately open catalog connections.
31
+
32
+ Within one provider instance, DuckDB work is queued on that runtime. This keeps
33
+ writes, SQL transforms, and metadata reads predictable and keeps PostgreSQL
34
+ connection use bounded.
35
+
36
+ ```txt
37
+ one DuckLakeStorage instance
38
+ -> one DuckDB runtime
39
+ -> one DuckLake attachment
40
+ -> one bounded PostgreSQL extension pool, when catalog.type is "postgres"
41
+ ```
42
+
43
+ For PostgreSQL catalogs, `postgresPool.maxConnections` is a per-instance budget.
44
+ If four production processes each create a `DuckLakeStorage` with
45
+ `maxConnections: 4`, the deployment can use about sixteen PostgreSQL catalog
46
+ connections for DuckLake metadata.
47
+
48
+ ```ts
49
+ const lakeStorage = new DuckLakeStorage({
50
+ catalog: { type: "postgres", host, database, user, password },
51
+ dataPath: "s3://sixb-lake/data",
52
+ duckdb: {
53
+ config: {
54
+ threads: "1",
55
+ },
56
+ },
57
+ postgresPool: {
58
+ maxConnections: 4,
59
+ idleTimeoutMillis: 1_000,
60
+ enableThreadLocalCache: false,
61
+ },
62
+ })
63
+ ```
64
+
65
+ Use `close()` during service shutdown so DuckDB can close its runtime and release
66
+ catalog connections cleanly.
67
+
68
+ ### Maintenance
69
+
70
+ DuckLake keeps historical snapshots and deleted files until maintenance runs.
71
+ Maintenance is operator-driven: run it on demand from the CLI or call
72
+ `runMaintenance()` directly. It expires snapshots, reclaims deleted file blocks,
73
+ and removes orphaned files through DuckLake's explicit maintenance functions,
74
+ keeping seven days of snapshots/files by default.
75
+
76
+ Run it from the CLI (preview with `--dry-run`):
77
+
78
+ ```bash
79
+ sixb lake cleanup --dry-run
80
+ sixb lake cleanup --expire-older-than "14 days" --delete-older-than "14 days"
81
+ ```
82
+
83
+ Or run it directly against the provider:
84
+
85
+ ```ts
86
+ const report = await lakeStorage.runMaintenance({
87
+ dryRun: true,
88
+ expireOlderThan: "7 days",
89
+ })
90
+ ```
91
+
92
+ ### Read and Write Concurrency
93
+
94
+ Reads, writes, SQL previews, and SQL transforms use the same DuckDB runtime and
95
+ the same DuckLake PostgreSQL metadata pool. Work is serialized inside one
96
+ `DuckLakeStorage` instance, so a queued DuckDB operation makes later operations
97
+ wait. Only actual DuckDB work holds the runtime: batch appends, commits, SQL
98
+ transforms, and metadata reads. A large streaming read can also make a later
99
+ write wait until the stream is consumed or closed.
100
+
101
+ A write does not hold the runtime while its source iterable is producing rows.
102
+ `writeRows(...)` validates and stages rows in bounded in-memory batches, then
103
+ takes a queue slot only to flush each batch into the staging table. Slow external
104
+ reads (pagination, APIs, SFTP, retries) run outside the queue, so other reads and
105
+ write batches can interleave between a write's batches.
106
+
107
+ ```txt
108
+ same provider instance:
109
+ read -> write -> read
110
+
111
+ the operations run in order on one DuckDB runtime
112
+ ```
113
+
114
+ This is the simplest and most predictable default, especially for small
115
+ PostgreSQL plans. A future provider option could add a bounded read-runtime pool
116
+ for deployments that have enough PostgreSQL capacity and need more concurrent
117
+ read throughput.
118
+
119
+ Datasets must declare a schema before they can be stored in DuckLake:
120
+
121
+ ```ts
122
+ import { col, defineDataset, defineSync } from "@sixb/core"
123
+
124
+ const rawOrdersDataset = defineDataset("raw.erp.orders", {
125
+ schema: [
126
+ col("orderId", "string"),
127
+ col("customerId", "string", { nullable: true }),
128
+ col("total", "float64", { nullable: true }),
129
+ col("createdAt", "timestamp", { nullable: true }),
130
+ col("raw", "json", { nullable: true }),
131
+ ],
132
+ partitionBy: ["createdAt"],
133
+ description: "Raw ERP order rows",
134
+ })
135
+
136
+ export const syncOrders = defineSync("sync-orders")
137
+ .from(erp)
138
+ .read(readOrders)
139
+ .intoDataset(rawOrdersDataset)
140
+ ```
141
+
142
+ `intoDataset(...)` takes the dataset definition, not a dataset id and options.
143
+ Export the definition from `datasets/` or pass it to
144
+ `createSixb({ datasets: [rawOrdersDataset], ... })` so the runtime and worker
145
+ use the same schema that DuckLake materializes.
146
+
147
+ DuckLake stores Sixb `decimal` columns as `DECIMAL(38, 9)` and returns them as exact strings, including from SQL previews. Writes that need more than 29 integer digits or 9 fractional digits are rejected before DuckDB can round or overflow them.
148
+
149
+ ## DuckLake SQL Transforms
150
+
151
+ DuckLake SQL transforms let the provider run dataset-to-dataset transforms in
152
+ DuckDB without materializing source rows in JavaScript. The capability is exposed
153
+ on `DuckLakeStorage`:
154
+
155
+ ```ts
156
+ lakeStorage.standard.id // "ducklake"
157
+ lakeStorage.standard.version // "1.0"
158
+ lakeStorage.sql.dialect // "duckdb"
159
+ ```
160
+
161
+ The SQL callback receives provider-owned relation handles. Interpolate those
162
+ handles where a relation belongs; do not use physical table names. The provider
163
+ renders each handle with DuckLake time travel and rejects unresolved or fabricated
164
+ placeholders.
165
+
166
+ ```ts
167
+ const previewRows = []
168
+
169
+ for await (const row of lakeStorage.sql.preview({
170
+ sources: {
171
+ orders: { dataset: rawOrdersDataset },
172
+ },
173
+ limit: 25,
174
+ sql: ({ orders }) => `
175
+ select
176
+ customerId,
177
+ count(*)::BIGINT as orders,
178
+ sum(total)::DOUBLE as revenue
179
+ from ${orders}
180
+ group by customerId
181
+ order by customerId
182
+ `,
183
+ })) {
184
+ previewRows.push(row)
185
+ }
186
+ ```
187
+
188
+ `preview(...)` is for development and UI inspection. It always applies a bounded
189
+ provider limit; omitted limits default to 100 rows and requested limits are capped
190
+ at 1000 rows.
191
+
192
+ `execute(...)` writes the transform result as a normal `DatasetVersion`. Target
193
+ datasets must already exist, result columns must match the target schema by name,
194
+ order, and compatible DuckDB type, and resolved source versions are recorded in
195
+ `DatasetVersion.inputs`.
196
+
197
+ ```ts
198
+ const customerInsightsDataset = defineDataset("analytics.customer_insights", {
199
+ schema: [
200
+ col("customerId", "string"),
201
+ col("orders", "int64"),
202
+ col("revenue", "float64", { nullable: true }),
203
+ ],
204
+ })
205
+
206
+ const version = await lakeStorage.sql.execute({
207
+ sources: {
208
+ orders: { dataset: rawOrdersDataset, versionId: "ducklake:42" },
209
+ },
210
+ target: customerInsightsDataset,
211
+ mode: "snapshot",
212
+ producer: { kind: "pipeline", id: "customer-insights", runId: "run_123" },
213
+ sql: ({ orders }) => `
214
+ select
215
+ customerId,
216
+ count(*)::BIGINT as orders,
217
+ sum(total)::DOUBLE as revenue
218
+ from ${orders}
219
+ group by customerId
220
+ `,
221
+ })
222
+ ```
223
+
224
+ Use `mode: "snapshot"` to replace the target contents, or `mode: "append"` to add
225
+ new rows to the existing target. Omitting a source `versionId` resolves the latest
226
+ committed version at execution time; providing a `versionId` pins that source for
227
+ repeatable transforms.
228
+
229
+ ## Definition Updates
230
+
231
+ DuckLake supports schema evolution, and this provider exposes a small safe
232
+ subset of dataset definition updates through repeated `createDataset(...)`
233
+ calls.
234
+
235
+ Schema changes:
236
+
237
+ - add nullable top-level columns
238
+ - reorder existing column declarations without changing the stored table
239
+
240
+ Metadata changes:
241
+
242
+ - add compatible metadata such as `description` or `partitionBy`
243
+
244
+ Rejected changes:
245
+
246
+ - add required columns
247
+ - drop columns
248
+ - rename columns
249
+ - change column types
250
+ - change column nullability
251
+ - change existing `description` or `partitionBy` metadata
252
+
253
+ Sixb matches existing columns by name, not by declaration position. If a
254
+ developer declares a new nullable column between existing columns, DuckLake still
255
+ stores that new physical column at the end of the table. Later `getDataset(...)`
256
+ calls return DuckLake's stored order.
257
+
258
+ Schema-only DuckLake commits count as Sixb dataset versions. After a schema
259
+ change, `getLatestVersion(...)` and `listVersions(...)` include the schema
260
+ version with `DatasetVersion.mode: "schema"` even if no rows were written in
261
+ that commit. Historical `DatasetVersion.schema` values come from DuckLake
262
+ metadata at that snapshot, so versioned reads validate projected columns against
263
+ the schema that existed at the requested version.
264
+
265
+ ## Catalogs
266
+
267
+ Supported catalogs:
268
+
269
+ - `duckdb`: local DuckDB metadata file.
270
+ - `sqlite`: local SQLite metadata file.
271
+ - `postgres`: PostgreSQL metadata catalog, with optional `metadataSchema`.
272
+ - `custom`: escape hatch for DuckLake catalog URI forms Sixb does not model
273
+ yet, including DuckLake secrets.
274
+
275
+ ```ts
276
+ const lakeStorage = new DuckLakeStorage({
277
+ catalog: {
278
+ type: "postgres",
279
+ host: "127.0.0.1",
280
+ port: 5432,
281
+ database: "ducklake",
282
+ user: "postgres",
283
+ password: process.env.POSTGRES_PASSWORD,
284
+ metadataSchema: "sixb_lake",
285
+ },
286
+ dataPath: "s3://sixb-lake/data",
287
+ secrets: [
288
+ {
289
+ type: "s3",
290
+ keyId: process.env.AWS_ACCESS_KEY_ID,
291
+ secret: process.env.AWS_SECRET_ACCESS_KEY,
292
+ region: "us-east-1",
293
+ scope: "s3://sixb-lake",
294
+ },
295
+ ],
296
+ })
297
+ ```
298
+
299
+ ## PostgreSQL Configuration
300
+
301
+ Use a PostgreSQL catalog for production or any deployment where multiple Sixb
302
+ processes need to see the same DuckLake snapshots. Local DuckDB or SQLite
303
+ catalogs are better suited to development and tests.
304
+
305
+ PostgreSQL stores only DuckLake metadata. Dataset table data still lives under
306
+ `dataPath`, usually in object storage such as S3, R2, GCS, or Azure Blob.
307
+
308
+ ```txt
309
+ PostgreSQL catalog -> snapshots, schemas, transactions, metadata
310
+ dataPath -> physical DuckLake table data and files
311
+ BlobStorage -> Sixb fileRef payload bytes
312
+ ```
313
+
314
+ Recommended production shape:
315
+
316
+ ```ts
317
+ const lakeStorage = new DuckLakeStorage({
318
+ catalog: {
319
+ type: "postgres",
320
+ host: process.env.PGHOST!,
321
+ port: Number(process.env.PGPORT ?? 5432),
322
+ database: process.env.PGDATABASE!,
323
+ user: process.env.PGUSER!,
324
+ password: process.env.PGPASSWORD!,
325
+ sslMode: "require",
326
+ applicationName: "sixb-api",
327
+ connectTimeoutSeconds: 10,
328
+ metadataSchema: "sixb_lake",
329
+ },
330
+ dataPath: "s3://sixb-lake/data",
331
+ duckdb: {
332
+ config: {
333
+ threads: "2",
334
+ },
335
+ },
336
+ postgresPool: {
337
+ maxConnections: 8,
338
+ waitTimeoutMillis: 30_000,
339
+ idleTimeoutMillis: 1_000,
340
+ maxLifetimeMillis: 300_000,
341
+ enableThreadLocalCache: false,
342
+ },
343
+ secrets: [
344
+ {
345
+ type: "s3",
346
+ keyId: process.env.AWS_ACCESS_KEY_ID,
347
+ secret: process.env.AWS_SECRET_ACCESS_KEY,
348
+ region: "us-east-1",
349
+ scope: "s3://sixb-lake",
350
+ },
351
+ ],
352
+ })
353
+ ```
354
+
355
+ ### What Each Setting Controls
356
+
357
+ | Setting | Controls |
358
+ | --- | --- |
359
+ | `catalog` | The PostgreSQL database DuckLake uses for metadata. |
360
+ | `catalog.metadataSchema` | The schema that holds DuckLake metadata tables. |
361
+ | `dataPath` | Where DuckLake stores physical table data. |
362
+ | `duckdb.config.threads` | Local DuckDB worker threads inside this process. |
363
+ | `postgresPool` | DuckDB PostgreSQL extension connections for DuckLake metadata. |
364
+
365
+ This pool is separate from any normal Sixb `PostgresStorage` pool. Count both
366
+ when sizing a small database.
367
+
368
+ ### Connection Sizing
369
+
370
+ Start with the smallest budget that supports your workload, then raise it after
371
+ observing real latency and PostgreSQL connection use.
372
+
373
+ | Deployment | `duckdb.config.threads` | `postgresPool.maxConnections` | Notes |
374
+ | --- | ---: | ---: | --- |
375
+ | Small managed PostgreSQL, about 20 connections | `"1"` | `4` | Run only the services that need the lake provider. |
376
+ | Normal production service | `"2"` to `"4"` | `8` to `16` | Good default range for API and worker processes. |
377
+ | Larger read-heavy deployment | `"4"` or more | `16` or more | Scale only with PostgreSQL headroom and monitoring. |
378
+
379
+ The budget multiplies by process count:
380
+
381
+ ```txt
382
+ estimated DuckLake catalog connections =
383
+ number of processes with DuckLakeStorage * postgresPool.maxConnections
384
+ ```
385
+
386
+ If the PostgreSQL role has a `CONNECTION LIMIT`, size it for the whole
387
+ deployment:
388
+
389
+ ```txt
390
+ role connection limit >=
391
+ DuckLake catalog connections
392
+ + normal application PostgreSQL pool connections
393
+ + migration, admin, and monitoring headroom
394
+ ```
395
+
396
+ Leave headroom for the normal Sixb PostgreSQL storage provider, migrations,
397
+ admin sessions, monitoring, and PostgreSQL reserved connections.
398
+
399
+ ### Pool Options
400
+
401
+ `postgresPool` maps to DuckDB's PostgreSQL extension pool settings for the
402
+ DuckLake metadata catalog.
403
+
404
+ | Option | Use it for |
405
+ | --- | --- |
406
+ | `maxConnections` | Hard cap for DuckDB PostgreSQL extension connections. |
407
+ | `waitTimeoutMillis` | How long a request can wait for a pool connection. |
408
+ | `idleTimeoutMillis` | How quickly unused pool connections are released. |
409
+ | `maxLifetimeMillis` | Recycling long-lived PostgreSQL connections. |
410
+ | `enableThreadLocalCache` | Keep `false` for constrained databases. |
411
+ | `healthCheckQuery` | Custom pool connection health checks, rarely needed. |
412
+
413
+ Prefer `postgresPool` over raw `setupSql` for PostgreSQL pool tuning. The
414
+ provider applies the settings before and after DuckLake attachment so the
415
+ metadata catalog uses the intended pool.
416
+
417
+ ### Service Layout
418
+
419
+ Each service process that constructs `DuckLakeStorage` owns its own DuckDB
420
+ runtime and PostgreSQL pool. Avoid creating the lake provider in services that
421
+ do not need it.
422
+
423
+ ```txt
424
+ needs DuckLakeStorage:
425
+ api reads, sync worker, pipeline worker, projection worker
426
+
427
+ does not need DuckLakeStorage:
428
+ static app server, health-only process, services that only use BlobStorage
429
+ ```
430
+
431
+ On very small PostgreSQL plans, run fewer lake-owning processes or group workers
432
+ into one process before lowering `maxConnections` too far. A pool budget below
433
+ `4` can become fragile because one runtime may need several metadata operations
434
+ during attach, commit, and snapshot hydration.
435
+
436
+ Use `custom` when you need to pass a DuckLake catalog URI that Sixb does not
437
+ model directly:
438
+
439
+ ```ts
440
+ const lakeStorage = new DuckLakeStorage({
441
+ catalog: {
442
+ type: "custom",
443
+ uri: "postgres:dbname=ducklake host=127.0.0.1 user=postgres",
444
+ extensions: ["postgres"],
445
+ },
446
+ dataPath: "s3://sixb-lake/data",
447
+ })
448
+ ```
449
+
450
+ `setupSql` runs after required DuckDB extensions load and before DuckLake
451
+ `ATTACH`, so callers can install extra extensions, set DuckDB options, or create
452
+ secrets manually when the typed options are not enough.
453
+
454
+ ## Blob Storage
455
+
456
+ DuckLake stores `fileRef` values as dataset row metadata only. Blob payload
457
+ bytes are owned by a separate Sixb `BlobStorage` provider, such as
458
+ `@sixb/blob-local`, and should be composed beside this lake provider when
459
+ creating a runtime.
460
+
461
+ ## Tests
462
+
463
+ Fast unit tests:
464
+
465
+ ```bash
466
+ bun --filter @sixb/ducklake test
467
+ ```
468
+
469
+ Local DuckLake e2e tests:
470
+
471
+ ```bash
472
+ bun --filter @sixb/ducklake test:e2e:local
473
+ ```
474
+
475
+ Docker-backed PostgreSQL and MinIO/S3-compatible e2e tests:
476
+
477
+ ```bash
478
+ bun --filter @sixb/ducklake test:e2e:remote
479
+ ```
480
+
481
+ `test:e2e` runs the local suite first, then the remote suite. Remote e2e
482
+ container setup and teardown live in `tests/setup.ts`, following the same
483
+ preload pattern as the other Docker-backed packages in this repo.
484
+
485
+ Relevant upstream docs:
486
+
487
+ - [DuckLake connecting](https://ducklake.select/docs/stable/duckdb/usage/connecting)
488
+ - [DuckLake catalog choices](https://ducklake.select/docs/stable/duckdb/usage/choosing_a_catalog_database)
489
+ - [DuckDB S3 API and secrets](https://duckdb.org/docs/current/core_extensions/httpfs/s3api)
490
+ - [DuckDB Azure extension](https://duckdb.org/docs/current/core_extensions/azure)
@@ -0,0 +1,42 @@
1
+ import type { DatasetDefinition, DatasetRow, LakeStorageWithSql } from "@sixb/core";
2
+ import type { BeginDatasetWriteInput, DatasetCatalogState, DatasetVersion, LakeWriteSession, ReadDatasetRowsInput } from "@sixb/core/lake-storage";
3
+ import { DuckLakeSqlExecutor } from "./internal/ducklake-sql-executor";
4
+ import type { DuckLakeMaintenanceOptions, DuckLakeMaintenanceReport, DuckLakeStorageOptions } from "./types";
5
+ /**
6
+ * DuckLake-backed implementation of Sixb's LakeStorage API.
7
+ *
8
+ * The public surface stays aligned with core LakeStorage while DuckLake owns
9
+ * physical tables, schemas, snapshots, and transactions behind the provider.
10
+ * Sixb-specific metadata is only used to reconstruct DatasetVersion shape.
11
+ */
12
+ export declare class DuckLakeStorage implements LakeStorageWithSql<"duckdb"> {
13
+ readonly standard: {
14
+ readonly id: "ducklake";
15
+ readonly version: "1.0";
16
+ };
17
+ readonly sql: DuckLakeSqlExecutor;
18
+ private readonly connections;
19
+ private readonly datasets;
20
+ private readonly maintenance;
21
+ private readonly rows;
22
+ private readonly snapshotReader;
23
+ private readonly writes;
24
+ constructor(options: DuckLakeStorageOptions);
25
+ createDataset(definition: DatasetDefinition): Promise<DatasetDefinition>;
26
+ assertDatasetDefinitionsCompatible(definitions: readonly DatasetDefinition[]): Promise<void>;
27
+ getDataset(datasetId: string): Promise<DatasetDefinition | null>;
28
+ listDatasets(): Promise<readonly DatasetDefinition[]>;
29
+ listDatasetCatalogState(datasetIds: readonly string[]): Promise<readonly DatasetCatalogState[]>;
30
+ listVersions(datasetId: string, limit?: number): Promise<readonly DatasetVersion[]>;
31
+ beginWrite(input: BeginDatasetWriteInput): Promise<LakeWriteSession>;
32
+ getLatestVersion(datasetId: string): Promise<DatasetVersion | null>;
33
+ getVersion(datasetId: string, versionId: string): Promise<DatasetVersion | null>;
34
+ readRows(input: ReadDatasetRowsInput): AsyncIterable<DatasetRow>;
35
+ runMaintenance(options?: DuckLakeMaintenanceOptions): Promise<DuckLakeMaintenanceReport>;
36
+ /**
37
+ * `close()` is provider-specific and intentionally not part of the core
38
+ * LakeStorage interface.
39
+ */
40
+ close(): Promise<void>;
41
+ }
42
+ //# sourceMappingURL=ducklake-storage.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"ducklake-storage.d.ts","sourceRoot":"","sources":["../src/ducklake-storage.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,iBAAiB,EAAE,UAAU,EAAE,kBAAkB,EAAE,MAAM,YAAY,CAAA;AACnF,OAAO,KAAK,EACV,sBAAsB,EACtB,mBAAmB,EACnB,cAAc,EACd,gBAAgB,EAChB,oBAAoB,EACrB,MAAM,yBAAyB,CAAA;AAMhC,OAAO,EAAE,mBAAmB,EAAE,MAAM,kCAAkC,CAAA;AAEtE,OAAO,KAAK,EACV,0BAA0B,EAC1B,yBAAyB,EACzB,sBAAsB,EACvB,MAAM,SAAS,CAAA;AAEhB;;;;;;GAMG;AACH,qBAAa,eAAgB,YAAW,kBAAkB,CAAC,QAAQ,CAAC;IAClE,QAAQ,CAAC,QAAQ;;;MAA8C;IAC/D,QAAQ,CAAC,GAAG,EAAE,mBAAmB,CAAA;IAEjC,OAAO,CAAC,QAAQ,CAAC,WAAW,CAA2B;IACvD,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAwB;IACjD,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAqB;IACjD,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAmB;IACxC,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAwB;IACvD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAA0B;gBAErC,OAAO,EAAE,sBAAsB;IAmCrC,aAAa,CAAC,UAAU,EAAE,iBAAiB,GAAG,OAAO,CAAC,iBAAiB,CAAC;IAIxE,kCAAkC,CACtC,WAAW,EAAE,SAAS,iBAAiB,EAAE,GACxC,OAAO,CAAC,IAAI,CAAC;IAIV,UAAU,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,iBAAiB,GAAG,IAAI,CAAC;IAIhE,YAAY,IAAI,OAAO,CAAC,SAAS,iBAAiB,EAAE,CAAC;IAIrD,uBAAuB,CAC3B,UAAU,EAAE,SAAS,MAAM,EAAE,GAC5B,OAAO,CAAC,SAAS,mBAAmB,EAAE,CAAC;IAIpC,YAAY,CAAC,SAAS,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,SAAS,cAAc,EAAE,CAAC;IAInF,UAAU,CAAC,KAAK,EAAE,sBAAsB,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAIpE,gBAAgB,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,cAAc,GAAG,IAAI,CAAC;IAInE,UAAU,CAAC,SAAS,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,cAAc,GAAG,IAAI,CAAC;IAItF,QAAQ,CAAC,KAAK,EAAE,oBAAoB,GAAG,aAAa,CAAC,UAAU,CAAC;IAI1D,cAAc,CAAC,OAAO,CAAC,EAAE,0BAA0B,GAAG,OAAO,CAAC,yBAAyB,CAAC;IAI9F;;;OAGG;IACG,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;CAG7B"}
@@ -0,0 +1,3 @@
1
+ export { DuckLakeStorage } from "./ducklake-storage";
2
+ export type { AzureSecretOptions, DuckDbRuntimeOptions, DuckDbSecretOptions, DuckLakeCatalogOptions, DuckLakeMaintenanceOptions, DuckLakeMaintenanceReport, DuckLakePostgresPoolOptions, DuckLakeStorageOptions, GcsSecretOptions, R2SecretOptions, S3SecretOptions, } from "./types";
3
+ //# sourceMappingURL=index.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAA;AACpD,YAAY,EACV,kBAAkB,EAClB,oBAAoB,EACpB,mBAAmB,EACnB,sBAAsB,EACtB,0BAA0B,EAC1B,yBAAyB,EACzB,2BAA2B,EAC3B,sBAAsB,EACtB,gBAAgB,EAChB,eAAe,EACf,eAAe,GAChB,MAAM,SAAS,CAAA"}