@sixb/ducklake 0.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (87) hide show
  1. package/LICENSE +21 -0
  2. package/README.md +490 -0
  3. package/dist/ducklake-storage.d.ts +42 -0
  4. package/dist/ducklake-storage.d.ts.map +1 -0
  5. package/dist/index.d.ts +3 -0
  6. package/dist/index.d.ts.map +1 -0
  7. package/dist/index.js +3112 -0
  8. package/dist/index.js.map +34 -0
  9. package/dist/internal/catalog-key.d.ts +11 -0
  10. package/dist/internal/catalog-key.d.ts.map +1 -0
  11. package/dist/internal/dataset-row-commit.d.ts +26 -0
  12. package/dist/internal/dataset-row-commit.d.ts.map +1 -0
  13. package/dist/internal/duckdb-column-types.d.ts +45 -0
  14. package/dist/internal/duckdb-column-types.d.ts.map +1 -0
  15. package/dist/internal/duckdb-decimal.d.ts +6 -0
  16. package/dist/internal/duckdb-decimal.d.ts.map +1 -0
  17. package/dist/internal/duckdb-row.d.ts +8 -0
  18. package/dist/internal/duckdb-row.d.ts.map +1 -0
  19. package/dist/internal/duckdb-runtime.d.ts +63 -0
  20. package/dist/internal/duckdb-runtime.d.ts.map +1 -0
  21. package/dist/internal/duckdb-value-converter.d.ts +7 -0
  22. package/dist/internal/duckdb-value-converter.d.ts.map +1 -0
  23. package/dist/internal/ducklake-catalog-schema.d.ts +12 -0
  24. package/dist/internal/ducklake-catalog-schema.d.ts.map +1 -0
  25. package/dist/internal/ducklake-connection-manager.d.ts +67 -0
  26. package/dist/internal/ducklake-connection-manager.d.ts.map +1 -0
  27. package/dist/internal/ducklake-current-dataset-definitions.d.ts +10 -0
  28. package/dist/internal/ducklake-current-dataset-definitions.d.ts.map +1 -0
  29. package/dist/internal/ducklake-dataset-catalog.d.ts +33 -0
  30. package/dist/internal/ducklake-dataset-catalog.d.ts.map +1 -0
  31. package/dist/internal/ducklake-dataset-table-ref.d.ts +9 -0
  32. package/dist/internal/ducklake-dataset-table-ref.d.ts.map +1 -0
  33. package/dist/internal/ducklake-maintenance.d.ts +9 -0
  34. package/dist/internal/ducklake-maintenance.d.ts.map +1 -0
  35. package/dist/internal/ducklake-row-reader.d.ts +24 -0
  36. package/dist/internal/ducklake-row-reader.d.ts.map +1 -0
  37. package/dist/internal/ducklake-snapshot-reader.d.ts +73 -0
  38. package/dist/internal/ducklake-snapshot-reader.d.ts.map +1 -0
  39. package/dist/internal/ducklake-sql-executor.d.ts +33 -0
  40. package/dist/internal/ducklake-sql-executor.d.ts.map +1 -0
  41. package/dist/internal/ducklake-write-coordinator.d.ts +53 -0
  42. package/dist/internal/ducklake-write-coordinator.d.ts.map +1 -0
  43. package/dist/internal/ducklake-write-session.d.ts +17 -0
  44. package/dist/internal/ducklake-write-session.d.ts.map +1 -0
  45. package/dist/internal/names.d.ts +13 -0
  46. package/dist/internal/names.d.ts.map +1 -0
  47. package/dist/internal/row-appender.d.ts +7 -0
  48. package/dist/internal/row-appender.d.ts.map +1 -0
  49. package/dist/internal/schema.d.ts +35 -0
  50. package/dist/internal/schema.d.ts.map +1 -0
  51. package/dist/internal/sql-transform-relations.d.ts +13 -0
  52. package/dist/internal/sql-transform-relations.d.ts.map +1 -0
  53. package/dist/internal/sql.d.ts +62 -0
  54. package/dist/internal/sql.d.ts.map +1 -0
  55. package/dist/internal/versions.d.ts +22 -0
  56. package/dist/internal/versions.d.ts.map +1 -0
  57. package/dist/tsconfig.json +5 -0
  58. package/dist/types.d.ts +147 -0
  59. package/dist/types.d.ts.map +1 -0
  60. package/package.json +64 -0
  61. package/src/ducklake-storage.ts +130 -0
  62. package/src/index.ts +14 -0
  63. package/src/internal/catalog-key.ts +28 -0
  64. package/src/internal/dataset-row-commit.ts +128 -0
  65. package/src/internal/duckdb-column-types.ts +68 -0
  66. package/src/internal/duckdb-decimal.ts +36 -0
  67. package/src/internal/duckdb-row.ts +83 -0
  68. package/src/internal/duckdb-runtime.ts +347 -0
  69. package/src/internal/duckdb-value-converter.ts +27 -0
  70. package/src/internal/ducklake-catalog-schema.ts +116 -0
  71. package/src/internal/ducklake-connection-manager.ts +510 -0
  72. package/src/internal/ducklake-current-dataset-definitions.ts +228 -0
  73. package/src/internal/ducklake-dataset-catalog.ts +363 -0
  74. package/src/internal/ducklake-dataset-table-ref.ts +35 -0
  75. package/src/internal/ducklake-maintenance.ts +116 -0
  76. package/src/internal/ducklake-row-reader.ts +143 -0
  77. package/src/internal/ducklake-snapshot-reader.ts +826 -0
  78. package/src/internal/ducklake-sql-executor.ts +334 -0
  79. package/src/internal/ducklake-write-coordinator.ts +532 -0
  80. package/src/internal/ducklake-write-session.ts +170 -0
  81. package/src/internal/names.ts +111 -0
  82. package/src/internal/row-appender.ts +73 -0
  83. package/src/internal/schema.ts +149 -0
  84. package/src/internal/sql-transform-relations.ts +99 -0
  85. package/src/internal/sql.ts +440 -0
  86. package/src/internal/versions.ts +140 -0
  87. package/src/types.ts +159 -0
package/dist/index.js ADDED
@@ -0,0 +1,3112 @@
1
+ // @bun
2
+ // src/internal/ducklake-connection-manager.ts
3
+ import { LakeStorageError as LakeStorageError4 } from "@sixb/core/lake-storage";
4
+
5
+ // src/internal/catalog-key.ts
6
+ import { resolve } from "path";
7
+ function localCatalogCoordinationKey(options) {
8
+ switch (options.catalog.type) {
9
+ case "duckdb":
10
+ case "sqlite":
11
+ if (options.catalog.path === ":memory:") {
12
+ return;
13
+ }
14
+ return `${options.catalog.type}:${resolve(options.catalog.path)}`;
15
+ case "custom":
16
+ return `custom:${options.catalog.uri}`;
17
+ case "postgres":
18
+ return;
19
+ }
20
+ }
21
+
22
+ // src/internal/duckdb-runtime.ts
23
+ import {
24
+ DuckDBInstance
25
+ } from "@duckdb/node-api";
26
+ import { LakeStorageError as LakeStorageError3 } from "@sixb/core/lake-storage";
27
+
28
+ // src/internal/duckdb-value-converter.ts
29
+ import {
30
+ DuckDBDecimalValue,
31
+ DuckDBTypeId,
32
+ JSDuckDBValueConverter
33
+ } from "@duckdb/node-api";
34
+ import { LakeStorageError } from "@sixb/core/lake-storage";
35
+ var sixbDuckDbValueConverter = (value, type, converter) => {
36
+ if (value === null) {
37
+ return null;
38
+ }
39
+ if (type.typeId === DuckDBTypeId.DECIMAL) {
40
+ if (!(value instanceof DuckDBDecimalValue)) {
41
+ throw new LakeStorageError("[SixbDuckLake] DuckDB returned an invalid DECIMAL value.");
42
+ }
43
+ return value.toString();
44
+ }
45
+ return JSDuckDBValueConverter(value, type, converter);
46
+ };
47
+
48
+ // src/internal/sql.ts
49
+ import { LakeStorageError as LakeStorageError2 } from "@sixb/core/lake-storage";
50
+ var DEFAULT_ALIAS = "sixb_lake";
51
+ function duckLakeAlias(options) {
52
+ return options.alias ?? DEFAULT_ALIAS;
53
+ }
54
+ function duckLakeMetadataCatalog(options) {
55
+ return `__ducklake_metadata_${duckLakeAlias(options)}`;
56
+ }
57
+ function buildConfigurePostgresMetadataPoolSql(options) {
58
+ const parameters = [
59
+ `catalog_name=${quoteSqlString(duckLakeMetadataCatalog(options))}`,
60
+ ...postgresPoolParameters(options.postgresPool).map(({ name, value }) => `${name}=${value}`)
61
+ ];
62
+ return `FROM postgres_configure_pool(${parameters.join(", ")})`;
63
+ }
64
+ function buildSetPostgresPoolSql(options) {
65
+ return postgresPoolParameters(options.postgresPool).map(({ name, value }) => `SET pg_pool_${name} = ${value}`);
66
+ }
67
+ function duckLakeMetadataTableName(options, tableName) {
68
+ const schemaName = options.catalog.type === "postgres" ? options.catalog.metadataSchema ?? "main" : "main";
69
+ return `${quoteIdentifier(duckLakeMetadataCatalog(options))}.${quoteIdentifier(schemaName)}.${quoteIdentifier(tableName)}`;
70
+ }
71
+ function quoteIdentifier(identifier) {
72
+ if (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(identifier)) {
73
+ throw new LakeStorageError2(`[SixbDuckLake] Invalid SQL identifier '${identifier}'.`);
74
+ }
75
+ return `"${identifier}"`;
76
+ }
77
+ function quoteSqlString(value) {
78
+ return `'${value.replaceAll("'", "''")}'`;
79
+ }
80
+ function quoteSqlStringList(values) {
81
+ return values.map((value) => quoteSqlString(value)).join(", ");
82
+ }
83
+ function qualifiedTableName(options, tableName) {
84
+ return `${quoteIdentifier(duckLakeAlias(options))}.main.${quoteIdentifier(tableName)}`;
85
+ }
86
+ function catalogUri(catalog) {
87
+ switch (catalog.type) {
88
+ case "duckdb":
89
+ return `ducklake:${catalog.path}`;
90
+ case "sqlite":
91
+ return `ducklake:sqlite:${catalog.path}`;
92
+ case "postgres":
93
+ return `ducklake:postgres:${postgresConnectionString(catalog)}`;
94
+ case "custom":
95
+ return `ducklake:${catalog.uri}`;
96
+ }
97
+ }
98
+ function postgresConnectionString(catalog) {
99
+ const parameters = [];
100
+ const names = new Set;
101
+ appendPostgresConnectionParameter(parameters, names, "dbname", catalog.database);
102
+ appendPostgresConnectionParameter(parameters, names, "host", catalog.host);
103
+ appendPostgresConnectionParameter(parameters, names, "port", catalog.port);
104
+ appendPostgresConnectionParameter(parameters, names, "user", catalog.user);
105
+ appendPostgresConnectionParameter(parameters, names, "password", catalog.password);
106
+ appendPostgresConnectionParameter(parameters, names, "sslmode", catalog.sslMode);
107
+ appendPostgresConnectionParameter(parameters, names, "application_name", catalog.applicationName);
108
+ appendPostgresConnectionParameter(parameters, names, "connect_timeout", catalog.connectTimeoutSeconds);
109
+ for (const [name, value] of Object.entries(catalog.parameters ?? {})) {
110
+ appendPostgresConnectionParameter(parameters, names, name, value);
111
+ }
112
+ return parameters.join(" ");
113
+ }
114
+ function catalogExtensions(catalog) {
115
+ switch (catalog.type) {
116
+ case "duckdb":
117
+ return [];
118
+ case "sqlite":
119
+ return ["sqlite"];
120
+ case "postgres":
121
+ return ["postgres"];
122
+ case "custom":
123
+ return catalog.extensions ?? [];
124
+ }
125
+ }
126
+ function requiredExtensions(options) {
127
+ const extensions = new Set(catalogExtensions(options.catalog));
128
+ for (const extension of storageExtensions(options.dataPath)) {
129
+ extensions.add(extension);
130
+ }
131
+ for (const secret of options.secrets ?? []) {
132
+ for (const extension of secretExtensions(secret)) {
133
+ extensions.add(extension);
134
+ }
135
+ }
136
+ return [...extensions];
137
+ }
138
+ function buildCreateSecretSql(secret) {
139
+ const persistence = secret.persistent ? "PERSISTENT " : "TEMPORARY ";
140
+ const nameSql = secret.name === undefined ? "" : `${quoteIdentifier(secret.name)} `;
141
+ return `CREATE OR REPLACE ${persistence}SECRET ${nameSql}(${secretParameters(secret).join(", ")})`;
142
+ }
143
+ function buildAttachSql(options) {
144
+ const parameters = [];
145
+ if (options.dataPath !== undefined) {
146
+ parameters.push(`DATA_PATH ${quoteSqlString(options.dataPath)}`);
147
+ }
148
+ parameters.push(`CREATE_IF_NOT_EXISTS ${options.createIfNotExists ?? true}`);
149
+ if (options.readOnly) {
150
+ parameters.push("READ_ONLY");
151
+ }
152
+ if (options.catalog.type === "postgres" && options.catalog.metadataSchema !== undefined) {
153
+ parameters.push(`METADATA_SCHEMA ${quoteSqlString(options.catalog.metadataSchema)}`);
154
+ }
155
+ if (options.catalog.type === "custom" && options.catalog.metadataParameters !== undefined && Object.keys(options.catalog.metadataParameters).length > 0) {
156
+ const entries = Object.entries(options.catalog.metadataParameters).map(([key, value]) => `${quoteSqlString(key)}: ${quoteSqlString(value)}`);
157
+ parameters.push(`METADATA_PARAMETERS MAP {${entries.join(", ")}}`);
158
+ }
159
+ return `ATTACH ${quoteSqlString(catalogUri(options.catalog))} AS ${quoteIdentifier(duckLakeAlias(options))} (${parameters.join(", ")})`;
160
+ }
161
+ function storageExtensions(path) {
162
+ if (path === undefined) {
163
+ return [];
164
+ }
165
+ switch (pathScheme(path)) {
166
+ case "s3":
167
+ case "r2":
168
+ case "gcs":
169
+ case "gs":
170
+ return ["httpfs"];
171
+ case "az":
172
+ case "azure":
173
+ case "abfss":
174
+ return ["azure"];
175
+ default:
176
+ return [];
177
+ }
178
+ }
179
+ function postgresPoolParameters(pool) {
180
+ const parameters = [
181
+ { name: "enable_thread_local_cache", value: String(pool?.enableThreadLocalCache ?? false) }
182
+ ];
183
+ appendPoolNumber(parameters, "max_connections", pool?.maxConnections);
184
+ appendPoolNumber(parameters, "wait_timeout_millis", pool?.waitTimeoutMillis);
185
+ appendPoolNumber(parameters, "max_lifetime_millis", pool?.maxLifetimeMillis);
186
+ appendPoolNumber(parameters, "idle_timeout_millis", pool?.idleTimeoutMillis);
187
+ if (pool?.enableReaperThread !== undefined) {
188
+ parameters.push({ name: "enable_reaper_thread", value: String(pool.enableReaperThread) });
189
+ }
190
+ if (pool?.healthCheckQuery !== undefined) {
191
+ parameters.push({ name: "health_check_query", value: quoteSqlString(pool.healthCheckQuery) });
192
+ }
193
+ return parameters;
194
+ }
195
+ function appendPoolNumber(parameters, name, value) {
196
+ if (value === undefined) {
197
+ return;
198
+ }
199
+ parameters.push({ name, value: String(value) });
200
+ }
201
+ function secretExtensions(secret) {
202
+ switch (secret.type) {
203
+ case "s3":
204
+ case "r2":
205
+ case "gcs":
206
+ return ["httpfs"];
207
+ case "azure":
208
+ return ["azure"];
209
+ }
210
+ }
211
+ function secretParameters(secret) {
212
+ switch (secret.type) {
213
+ case "s3":
214
+ return withCommonSecretParameters(secret, [
215
+ "TYPE s3",
216
+ ...s3ApiSecretParameters(secret),
217
+ ...optionalStringParameter("KMS_KEY_ID", secret.kmsKeyId),
218
+ ...optionalBooleanParameter("REQUESTER_PAYS", secret.requesterPays)
219
+ ]);
220
+ case "r2":
221
+ return withCommonSecretParameters(secret, [
222
+ "TYPE r2",
223
+ ...s3ApiSecretParameters(secret),
224
+ ...optionalStringParameter("ACCOUNT_ID", secret.accountId)
225
+ ]);
226
+ case "gcs":
227
+ return withCommonSecretParameters(secret, ["TYPE gcs", ...s3ApiSecretParameters(secret)]);
228
+ case "azure":
229
+ return withCommonSecretParameters(secret, azureSecretParameters(secret));
230
+ }
231
+ }
232
+ function s3ApiSecretParameters(secret) {
233
+ return [
234
+ ...optionalIdentifierParameter("PROVIDER", secret.provider),
235
+ ...optionalStringParameter("KEY_ID", secret.keyId),
236
+ ...optionalStringParameter("SECRET", secret.secret),
237
+ ...optionalStringParameter("SESSION_TOKEN", secret.sessionToken),
238
+ ...optionalStringParameter("REGION", secret.region),
239
+ ...optionalStringParameter("ENDPOINT", secret.endpoint),
240
+ ...optionalStringParameter("URL_STYLE", secret.urlStyle),
241
+ ...optionalBooleanParameter("USE_SSL", secret.useSsl),
242
+ ...optionalBooleanParameter("VERIFY_SSL", secret.verifySsl),
243
+ ...optionalStringParameter("CHAIN", secret.chain),
244
+ ...optionalStringParameter("PROFILE", secret.profile)
245
+ ];
246
+ }
247
+ function azureSecretParameters(secret) {
248
+ if (secret.tenantId !== undefined || secret.clientId !== undefined || secret.clientSecret !== undefined) {
249
+ return [
250
+ "TYPE azure",
251
+ "PROVIDER service_principal",
252
+ ...optionalStringParameter("TENANT_ID", secret.tenantId),
253
+ ...optionalStringParameter("CLIENT_ID", secret.clientId),
254
+ ...optionalStringParameter("CLIENT_SECRET", secret.clientSecret),
255
+ ...optionalStringParameter("ACCOUNT_NAME", secret.accountName)
256
+ ];
257
+ }
258
+ const connectionString = secret.connectionString ?? (secret.accountName !== undefined && secret.accountKey !== undefined ? `DefaultEndpointsProtocol=https;AccountName=${secret.accountName};AccountKey=${secret.accountKey};EndpointSuffix=core.windows.net` : undefined);
259
+ if (connectionString !== undefined) {
260
+ return ["TYPE azure", ...optionalStringParameter("CONNECTION_STRING", connectionString)];
261
+ }
262
+ return [
263
+ "TYPE azure",
264
+ "PROVIDER credential_chain",
265
+ ...optionalStringParameter("ACCOUNT_NAME", secret.accountName)
266
+ ];
267
+ }
268
+ function withCommonSecretParameters(secret, parameters) {
269
+ return [...parameters, ...optionalStringParameter("SCOPE", secret.scope)];
270
+ }
271
+ function optionalStringParameter(name, value) {
272
+ return value === undefined ? [] : [`${name} ${quoteSqlString(value)}`];
273
+ }
274
+ function optionalBooleanParameter(name, value) {
275
+ return value === undefined ? [] : [`${name} ${value}`];
276
+ }
277
+ function optionalIdentifierParameter(name, value) {
278
+ if (value === undefined) {
279
+ return [];
280
+ }
281
+ if (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(value)) {
282
+ throw new LakeStorageError2(`[SixbDuckLake] Invalid SQL identifier '${value}'.`);
283
+ }
284
+ return [`${name} ${value}`];
285
+ }
286
+ function appendPostgresConnectionParameter(parameters, names, name, value) {
287
+ if (value === undefined) {
288
+ return;
289
+ }
290
+ if (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(name)) {
291
+ throw new LakeStorageError2(`[SixbDuckLake] Invalid PostgreSQL catalog parameter '${name}'.`);
292
+ }
293
+ const normalizedName = name.toLowerCase();
294
+ if (names.has(normalizedName)) {
295
+ throw new LakeStorageError2(`[SixbDuckLake] Duplicate PostgreSQL catalog parameter '${name}'.`);
296
+ }
297
+ names.add(normalizedName);
298
+ parameters.push(`${name}=${quotePostgresConnectionValue(String(value))}`);
299
+ }
300
+ function quotePostgresConnectionValue(value) {
301
+ return `'${value.replaceAll("\\", "\\\\").replaceAll("'", "\\'")}'`;
302
+ }
303
+ function pathScheme(path) {
304
+ const match = /^([a-z][a-z0-9+.-]*):\/\//i.exec(path);
305
+ return match ? match[1].toLowerCase() : null;
306
+ }
307
+
308
+ // src/internal/duckdb-runtime.ts
309
+ class NodeDuckDbRuntime {
310
+ instance;
311
+ connection;
312
+ closing = false;
313
+ closed = false;
314
+ operations = Promise.resolve();
315
+ constructor(instance, connection) {
316
+ this.instance = instance;
317
+ this.connection = connection;
318
+ }
319
+ async run(sql, values) {
320
+ await this.enqueue(async () => {
321
+ await this.connection.run(sql, values === undefined ? undefined : [...values]);
322
+ });
323
+ }
324
+ async runStatements(statements) {
325
+ await this.enqueue(async () => {
326
+ for (const sql of statements) {
327
+ await this.connection.run(sql);
328
+ }
329
+ });
330
+ }
331
+ async query(sql, values) {
332
+ return this.enqueue(async () => {
333
+ const reader = await this.connection.runAndReadAll(sql, values === undefined ? undefined : [...values]);
334
+ return reader.convertRowObjects(sixbDuckDbValueConverter);
335
+ });
336
+ }
337
+ async* streamRows(sql, values) {
338
+ this.assertAcceptingOperations();
339
+ let releaseOperation;
340
+ const operationFinished = new Promise((resolve2) => {
341
+ releaseOperation = resolve2;
342
+ });
343
+ const waitForTurn = this.operations.then(() => {
344
+ this.assertNotClosed();
345
+ });
346
+ this.operations = waitForTurn.then(() => operationFinished).then(() => {}, () => {});
347
+ try {
348
+ await waitForTurn;
349
+ const result = await this.connection.stream(sql, values === undefined ? undefined : [...values]);
350
+ for await (const batch of result.yieldConvertedRowObjects(sixbDuckDbValueConverter)) {
351
+ for (const row of batch) {
352
+ yield row;
353
+ }
354
+ }
355
+ } finally {
356
+ releaseOperation?.();
357
+ }
358
+ }
359
+ async withAppender(tableName, useAppender) {
360
+ return this.enqueue(async () => {
361
+ const appender = await this.connection.createAppender(tableName);
362
+ try {
363
+ return await useAppender(new NodeDuckDbAppenderAdapter(appender));
364
+ } finally {
365
+ appender.closeSync();
366
+ }
367
+ });
368
+ }
369
+ async withExclusive(useRuntime) {
370
+ return this.enqueue(async () => {
371
+ return useRuntime(new NodeDuckDbExclusiveRuntime(this.connection));
372
+ });
373
+ }
374
+ async close() {
375
+ if (this.closed) {
376
+ return;
377
+ }
378
+ this.closing = true;
379
+ await this.operations.catch(() => {});
380
+ this.closed = true;
381
+ this.connection.closeSync();
382
+ this.instance.closeSync();
383
+ }
384
+ enqueue(operation) {
385
+ this.assertAcceptingOperations();
386
+ const result = this.operations.then(async () => {
387
+ this.assertNotClosed();
388
+ return operation();
389
+ });
390
+ this.operations = result.then(() => {}, () => {});
391
+ return result;
392
+ }
393
+ assertAcceptingOperations() {
394
+ if (this.closing || this.closed) {
395
+ throw new LakeStorageError3("[SixbDuckLake] DuckDB runtime is closed.");
396
+ }
397
+ }
398
+ assertNotClosed() {
399
+ if (this.closed) {
400
+ throw new LakeStorageError3("[SixbDuckLake] DuckDB runtime is closed.");
401
+ }
402
+ }
403
+ }
404
+
405
+ class NodeDuckDbExclusiveRuntime {
406
+ connection;
407
+ constructor(connection) {
408
+ this.connection = connection;
409
+ }
410
+ async run(sql, values) {
411
+ await this.connection.run(sql, values === undefined ? undefined : [...values]);
412
+ }
413
+ async runStatements(statements) {
414
+ for (const sql of statements) {
415
+ await this.connection.run(sql);
416
+ }
417
+ }
418
+ async query(sql, values) {
419
+ const reader = await this.connection.runAndReadAll(sql, values === undefined ? undefined : [...values]);
420
+ return reader.convertRowObjects(sixbDuckDbValueConverter);
421
+ }
422
+ async withAppender(tableName, useAppender) {
423
+ const appender = await this.connection.createAppender(tableName);
424
+ try {
425
+ return await useAppender(new NodeDuckDbAppenderAdapter(appender));
426
+ } finally {
427
+ appender.closeSync();
428
+ }
429
+ }
430
+ }
431
+
432
+ class NodeDuckDbAppenderAdapter {
433
+ appender;
434
+ constructor(appender) {
435
+ this.appender = appender;
436
+ }
437
+ appendNull() {
438
+ this.appender.appendNull();
439
+ }
440
+ appendBoolean(value) {
441
+ this.appender.appendBoolean(value);
442
+ }
443
+ appendVarchar(value) {
444
+ this.appender.appendVarchar(value);
445
+ }
446
+ appendBigInt(value) {
447
+ this.appender.appendBigInt(value);
448
+ }
449
+ appendDouble(value) {
450
+ this.appender.appendDouble(value);
451
+ }
452
+ appendStruct(value) {
453
+ this.appender.appendStruct(value);
454
+ }
455
+ endRow() {
456
+ this.appender.endRow();
457
+ }
458
+ }
459
+ async function createDuckDbRuntime(options = {}) {
460
+ const instance = await DuckDBInstance.create(options.path ?? ":memory:", options.config ?? {});
461
+ return new NodeDuckDbRuntime(instance, await instance.connect());
462
+ }
463
+ async function installDuckLakeExtensions(runtime, options) {
464
+ await runtime.run("INSTALL ducklake");
465
+ for (const extension of requiredExtensions(options)) {
466
+ await runtime.run(`INSTALL ${quoteIdentifier(extension)}`);
467
+ }
468
+ }
469
+ async function setupDuckLake(runtime, options, setupOptions = {}) {
470
+ if (setupOptions.installExtensions ?? true) {
471
+ await installDuckLakeExtensions(runtime, options);
472
+ }
473
+ await runtime.run("LOAD ducklake");
474
+ for (const extension of requiredExtensions(options)) {
475
+ await runtime.run(`LOAD ${quoteIdentifier(extension)}`);
476
+ }
477
+ for (const sql of options.setupSql ?? []) {
478
+ await runtime.run(sql);
479
+ }
480
+ if (options.catalog.type === "postgres") {
481
+ await runtime.runStatements(buildSetPostgresPoolSql(options));
482
+ }
483
+ for (const secret of options.secrets ?? []) {
484
+ await runtime.run(buildCreateSecretSql(secret));
485
+ }
486
+ if (setupOptions.attach ?? true) {
487
+ await runtime.run(buildAttachSql(options));
488
+ if (options.catalog.type === "postgres") {
489
+ await runtime.run(buildConfigurePostgresMetadataPoolSql(options));
490
+ }
491
+ }
492
+ }
493
+
494
+ // src/internal/ducklake-connection-manager.ts
495
+ class DuckLakeConnectionManager {
496
+ options;
497
+ runtimePromise;
498
+ installExtensionsPromise;
499
+ attachPromise;
500
+ refreshPromise;
501
+ localAttachmentLock = Promise.resolve();
502
+ observedLocalCatalogGeneration;
503
+ attached = false;
504
+ closed = false;
505
+ runtimePoisoned = false;
506
+ constructor(options) {
507
+ this.options = options;
508
+ this.observedLocalCatalogGeneration = currentLocalCatalogGeneration(options);
509
+ }
510
+ assertOpen() {
511
+ if (this.closed) {
512
+ throw new LakeStorageError4("[SixbDuckLake] DuckLakeStorage is closed.");
513
+ }
514
+ }
515
+ async runtime() {
516
+ this.assertOpen();
517
+ if (this.runtimePromise === undefined) {
518
+ const runtimePromise = this.createRuntime().catch((error) => {
519
+ if (this.runtimePromise === runtimePromise) {
520
+ this.runtimePromise = undefined;
521
+ }
522
+ throw error;
523
+ });
524
+ this.runtimePromise = runtimePromise;
525
+ }
526
+ return this.runtimePromise;
527
+ }
528
+ async attachedRuntime() {
529
+ const runtime = await this.runtime();
530
+ await this.ensureAttached(runtime);
531
+ return runtime;
532
+ }
533
+ async withAttachedRuntime(run) {
534
+ const lease = await this.acquireAttachedRuntime();
535
+ try {
536
+ return await run(lease.runtime);
537
+ } finally {
538
+ await lease.release();
539
+ }
540
+ }
541
+ async acquireAttachedRuntime() {
542
+ this.assertOpen();
543
+ const releaseLock = await this.acquireLocalAttachmentLock();
544
+ let released = false;
545
+ const release = async () => {
546
+ if (released) {
547
+ return;
548
+ }
549
+ released = true;
550
+ try {
551
+ if (this.runtimePoisoned) {
552
+ this.runtimePoisoned = false;
553
+ await this.discardRuntimeUnlocked();
554
+ } else {
555
+ await this.detachDuckDbCatalogAfterLease();
556
+ }
557
+ } finally {
558
+ releaseLock();
559
+ }
560
+ };
561
+ try {
562
+ await this.refreshForExternalChangesUnlocked();
563
+ const runtime = await this.attachedRuntime();
564
+ return { runtime, release };
565
+ } catch (error) {
566
+ await release();
567
+ throw error;
568
+ }
569
+ }
570
+ async withExclusiveAttached(run) {
571
+ const lease = await this.acquireAttachedRuntime();
572
+ try {
573
+ return await lease.runtime.withExclusive(run);
574
+ } finally {
575
+ await lease.release();
576
+ }
577
+ }
578
+ async stagingRuntime() {
579
+ const runtime = await this.runtime();
580
+ if (this.options.catalog.type !== "postgres") {
581
+ await this.detachRuntime();
582
+ }
583
+ return runtime;
584
+ }
585
+ async createRuntime() {
586
+ this.assertOpen();
587
+ const runtime = await createDuckDbRuntime(this.options.duckdb);
588
+ try {
589
+ await this.ensureExtensionsInstalled(runtime);
590
+ await setupDuckLake(runtime, this.options, {
591
+ attach: false,
592
+ installExtensions: false
593
+ });
594
+ await runtime.run("SET preserve_insertion_order = true");
595
+ return runtime;
596
+ } catch (error) {
597
+ await this.closeRuntime(runtime, { detach: false });
598
+ throw error;
599
+ }
600
+ }
601
+ async resetRuntime() {
602
+ const releaseLock = await this.acquireLocalAttachmentLock();
603
+ try {
604
+ await this.resetRuntimeUnlocked();
605
+ } finally {
606
+ releaseLock();
607
+ }
608
+ }
609
+ async resetRuntimeUnlocked() {
610
+ const runtimePromise = this.runtimePromise;
611
+ if (runtimePromise === undefined) {
612
+ return;
613
+ }
614
+ if (this.attachPromise !== undefined) {
615
+ await this.attachPromise;
616
+ }
617
+ if (!this.attached) {
618
+ return;
619
+ }
620
+ let refreshedPromise;
621
+ refreshedPromise = runtimePromise.then(async (runtime) => {
622
+ try {
623
+ await this.refreshAttachedRuntime(runtime);
624
+ this.attached = true;
625
+ return runtime;
626
+ } catch (error) {
627
+ this.attached = false;
628
+ await this.closeRuntime(runtime, { detach: false });
629
+ throw error;
630
+ }
631
+ }).catch((error) => {
632
+ if (this.runtimePromise === refreshedPromise) {
633
+ this.runtimePromise = undefined;
634
+ }
635
+ throw error;
636
+ });
637
+ this.attached = false;
638
+ this.runtimePromise = refreshedPromise;
639
+ await refreshedPromise;
640
+ }
641
+ async refreshForExternalChanges() {
642
+ if (this.options.catalog.type === "postgres") {
643
+ return;
644
+ }
645
+ if (!this.needsLocalCatalogRefresh()) {
646
+ return;
647
+ }
648
+ if (this.refreshPromise === undefined) {
649
+ let refreshPromise;
650
+ refreshPromise = this.withLocalAttachmentLock(() => this.refreshForExternalChangesUnlocked()).finally(() => {
651
+ if (this.refreshPromise === refreshPromise) {
652
+ this.refreshPromise = undefined;
653
+ }
654
+ });
655
+ this.refreshPromise = refreshPromise;
656
+ }
657
+ await this.refreshPromise;
658
+ }
659
+ markLocalCatalogChanged() {
660
+ if (this.options.catalog.type === "postgres") {
661
+ return;
662
+ }
663
+ this.observedLocalCatalogGeneration = incrementLocalCatalogGeneration(this.options);
664
+ }
665
+ async detachLocalCatalogAfterCommit(runtime) {
666
+ if (this.options.catalog.type === "postgres") {
667
+ return;
668
+ }
669
+ try {
670
+ await runtime.run(`DETACH ${quoteIdentifier(duckLakeAlias(this.options))}`);
671
+ this.attached = false;
672
+ } catch {} finally {
673
+ this.markLocalCatalogChanged();
674
+ }
675
+ }
676
+ async refreshForExternalChangesUnlocked() {
677
+ const generation = currentLocalCatalogGeneration(this.options);
678
+ if (this.observedLocalCatalogGeneration === generation) {
679
+ return;
680
+ }
681
+ await this.resetRuntimeUnlocked();
682
+ this.observedLocalCatalogGeneration = generation;
683
+ }
684
+ needsLocalCatalogRefresh() {
685
+ return this.observedLocalCatalogGeneration !== currentLocalCatalogGeneration(this.options);
686
+ }
687
+ async close() {
688
+ if (this.closed) {
689
+ return;
690
+ }
691
+ this.closed = true;
692
+ const runtimePromise = this.runtimePromise;
693
+ this.runtimePromise = undefined;
694
+ await this.closeRuntimePromise(runtimePromise, {
695
+ detach: this.attached && this.options.catalog.type !== "postgres"
696
+ });
697
+ this.attached = false;
698
+ }
699
+ async ensureAttached(runtime) {
700
+ this.assertOpen();
701
+ if (this.attached) {
702
+ return;
703
+ }
704
+ if (this.attachPromise === undefined) {
705
+ let attachPromise;
706
+ attachPromise = this.attachRuntime(runtime).then(() => {
707
+ this.attached = true;
708
+ }).catch(async (error) => {
709
+ if (this.attachPromise === attachPromise) {
710
+ this.runtimePromise = undefined;
711
+ this.attached = false;
712
+ }
713
+ await this.closeRuntime(runtime, { detach: false });
714
+ throw error;
715
+ }).finally(() => {
716
+ if (this.attachPromise === attachPromise) {
717
+ this.attachPromise = undefined;
718
+ }
719
+ });
720
+ this.attachPromise = attachPromise;
721
+ }
722
+ await this.attachPromise;
723
+ }
724
+ async ensureExtensionsInstalled(runtime) {
725
+ if (this.installExtensionsPromise === undefined) {
726
+ const installExtensionsPromise = installDuckLakeExtensions(runtime, this.options).catch((error) => {
727
+ if (this.installExtensionsPromise === installExtensionsPromise) {
728
+ this.installExtensionsPromise = undefined;
729
+ }
730
+ throw error;
731
+ });
732
+ this.installExtensionsPromise = installExtensionsPromise;
733
+ }
734
+ await this.installExtensionsPromise;
735
+ }
736
+ async attachRuntime(runtime) {
737
+ await runtime.runStatements([
738
+ buildAttachSql(this.options),
739
+ ...this.options.catalog.type === "postgres" ? [buildConfigurePostgresMetadataPoolSql(this.options)] : []
740
+ ]);
741
+ }
742
+ async refreshAttachedRuntime(runtime) {
743
+ await runtime.runStatements([
744
+ `DETACH ${quoteIdentifier(duckLakeAlias(this.options))}`,
745
+ buildAttachSql(this.options),
746
+ ...this.options.catalog.type === "postgres" ? [buildConfigurePostgresMetadataPoolSql(this.options)] : []
747
+ ]);
748
+ }
749
+ async detachRuntime() {
750
+ await this.withLocalAttachmentLock(() => this.detachRuntimeUnlocked());
751
+ }
752
+ poisonRuntime() {
753
+ this.runtimePoisoned = true;
754
+ }
755
+ async detachDuckDbCatalogAfterLease() {
756
+ if (this.options.catalog.type !== "duckdb" || !this.attached) {
757
+ return;
758
+ }
759
+ try {
760
+ await this.detachRuntimeUnlocked();
761
+ } catch {
762
+ await this.discardRuntimeUnlocked();
763
+ }
764
+ }
765
+ async discardRuntimeUnlocked() {
766
+ const runtimePromise = this.runtimePromise;
767
+ this.runtimePromise = undefined;
768
+ this.attached = false;
769
+ await this.closeRuntimePromise(runtimePromise, { detach: false }).catch(() => {});
770
+ }
771
+ async detachRuntimeUnlocked() {
772
+ if (this.attachPromise !== undefined) {
773
+ await this.attachPromise;
774
+ }
775
+ if (!this.attached) {
776
+ return;
777
+ }
778
+ const runtime = await this.runtime();
779
+ await runtime.run(`DETACH ${quoteIdentifier(duckLakeAlias(this.options))}`);
780
+ this.attached = false;
781
+ }
782
+ async withLocalAttachmentLock(run) {
783
+ const releaseLock = await this.acquireLocalAttachmentLock();
784
+ try {
785
+ return await run();
786
+ } finally {
787
+ releaseLock();
788
+ }
789
+ }
790
+ async acquireLocalAttachmentLock() {
791
+ if (this.options.catalog.type === "postgres") {
792
+ return () => {};
793
+ }
794
+ const previous = this.localAttachmentLock.catch(() => {});
795
+ let release;
796
+ const current = new Promise((resolve2) => {
797
+ release = resolve2;
798
+ });
799
+ this.localAttachmentLock = previous.then(() => current);
800
+ await previous;
801
+ let released = false;
802
+ return () => {
803
+ if (released) {
804
+ return;
805
+ }
806
+ released = true;
807
+ release();
808
+ };
809
+ }
810
+ async closeRuntimePromise(runtimePromise, options) {
811
+ if (runtimePromise === undefined) {
812
+ return;
813
+ }
814
+ let runtime;
815
+ try {
816
+ runtime = await runtimePromise;
817
+ } catch {
818
+ return;
819
+ }
820
+ await this.closeRuntime(runtime, options);
821
+ }
822
+ async closeRuntime(runtime, options) {
823
+ if (options.detach) {
824
+ try {
825
+ await runtime.run(`DETACH ${quoteIdentifier(duckLakeAlias(this.options))}`);
826
+ } catch {}
827
+ }
828
+ await runtime.close();
829
+ }
830
+ }
831
+ var localCatalogGenerations = new Map;
832
+ function currentLocalCatalogGeneration(options) {
833
+ const key = localCatalogCoordinationKey(options);
834
+ return key === undefined ? 0 : localCatalogGenerations.get(key) ?? 0;
835
+ }
836
+ function incrementLocalCatalogGeneration(options) {
837
+ const key = localCatalogCoordinationKey(options);
838
+ if (key === undefined) {
839
+ return 0;
840
+ }
841
+ const nextGeneration = (localCatalogGenerations.get(key) ?? 0) + 1;
842
+ localCatalogGenerations.set(key, nextGeneration);
843
+ return nextGeneration;
844
+ }
845
+
846
+ // src/internal/ducklake-dataset-catalog.ts
847
+ import { randomUUID } from "crypto";
848
+ import { LakeStorageError as LakeStorageError11, planDatasetDefinitionUpdate } from "@sixb/core/lake-storage";
849
+
850
+ // src/internal/duckdb-row.ts
851
+ import { LakeStorageError as LakeStorageError5 } from "@sixb/core/lake-storage";
852
+ function getString(row, key) {
853
+ const value = row[key];
854
+ if (typeof value !== "string") {
855
+ throw new LakeStorageError5(`[SixbDuckLake] Expected DuckDB column '${key}' to be a string, got ${typeof value}.`);
856
+ }
857
+ return value;
858
+ }
859
+ function getOptionalString(row, key) {
860
+ const value = row[key];
861
+ if (value === null || value === undefined) {
862
+ return;
863
+ }
864
+ if (typeof value !== "string") {
865
+ throw new LakeStorageError5(`[SixbDuckLake] Expected DuckDB column '${key}' to be a string, got ${typeof value}.`);
866
+ }
867
+ return value;
868
+ }
869
+ function getBoolean(row, key) {
870
+ const value = row[key];
871
+ if (typeof value !== "boolean") {
872
+ throw new LakeStorageError5(`[SixbDuckLake] Expected DuckDB column '${key}' to be a boolean, got ${typeof value}.`);
873
+ }
874
+ return value;
875
+ }
876
+ function getBigIntLike(row, key) {
877
+ const value = row[key];
878
+ if (typeof value === "bigint") {
879
+ return value;
880
+ }
881
+ if (typeof value === "number" && Number.isInteger(value)) {
882
+ return BigInt(value);
883
+ }
884
+ if (typeof value === "string" && /^-?\d+$/.test(value)) {
885
+ return BigInt(value);
886
+ }
887
+ throw new LakeStorageError5(`[SixbDuckLake] Expected DuckDB column '${key}' to be an integer, got ${typeof value}.`);
888
+ }
889
+ function getOptionalBigIntLike(row, key) {
890
+ const value = row[key];
891
+ if (value === null || value === undefined) {
892
+ return;
893
+ }
894
+ return getBigIntLike(row, key);
895
+ }
896
+ function getDate(row, key) {
897
+ const value = row[key];
898
+ if (value instanceof Date) {
899
+ return value;
900
+ }
901
+ if (typeof value === "string") {
902
+ return new Date(value);
903
+ }
904
+ throw new LakeStorageError5(`[SixbDuckLake] Expected DuckDB column '${key}' to be a date, got ${typeof value}.`);
905
+ }
906
+
907
+ // src/internal/ducklake-catalog-schema.ts
908
+ import { LakeStorageError as LakeStorageError8 } from "@sixb/core/lake-storage";
909
+
910
+ // src/internal/schema.ts
911
+ import { LakeStorageError as LakeStorageError7 } from "@sixb/core/lake-storage";
912
+
913
+ // src/internal/duckdb-column-types.ts
914
+ function normalizeDuckDbTypeSql(typeSql) {
915
+ return typeSql.trim().toUpperCase().replace(/\s+/g, " ").replace(/\s*([(),])\s*/g, "$1");
916
+ }
917
+ function defineDuckDbColumnType(sql, aliases = []) {
918
+ const acceptedTypes = new Set([sql, ...aliases].map(normalizeDuckDbTypeSql));
919
+ return {
920
+ sql,
921
+ matches: (typeSql) => acceptedTypes.has(normalizeDuckDbTypeSql(typeSql))
922
+ };
923
+ }
924
+ function defineDuckDbDecimalType(precision, scale) {
925
+ return {
926
+ ...defineDuckDbColumnType(`DECIMAL(${precision}, ${scale})`),
927
+ precision,
928
+ scale
929
+ };
930
+ }
931
+ var DUCKDB_COLUMN_TYPES = {
932
+ string: defineDuckDbColumnType("VARCHAR"),
933
+ boolean: defineDuckDbColumnType("BOOLEAN"),
934
+ int64: defineDuckDbColumnType("BIGINT", ["INT64"]),
935
+ float64: defineDuckDbColumnType("DOUBLE", ["FLOAT64"]),
936
+ decimal: defineDuckDbDecimalType(38, 9),
937
+ date: defineDuckDbColumnType("DATE"),
938
+ timestamp: defineDuckDbColumnType("TIMESTAMPTZ", ["TIMESTAMP WITH TIME ZONE"]),
939
+ json: defineDuckDbColumnType("JSON"),
940
+ fileRef: defineDuckDbColumnType("STRUCT(blobId VARCHAR, digest VARCHAR, sizeBytes BIGINT, fileName VARCHAR, mediaType VARCHAR, logicalPath VARCHAR)")
941
+ };
942
+ function findDatasetColumnTypeForDuckDbSql(typeSql) {
943
+ for (const type of Object.keys(DUCKDB_COLUMN_TYPES)) {
944
+ if (DUCKDB_COLUMN_TYPES[type].matches(typeSql)) {
945
+ return type;
946
+ }
947
+ }
948
+ return;
949
+ }
950
+
951
+ // src/internal/duckdb-decimal.ts
952
+ import { normalizeDecimalValue } from "@sixb/core";
953
+ import { LakeStorageError as LakeStorageError6 } from "@sixb/core/lake-storage";
954
+ var decimalType = DUCKDB_COLUMN_TYPES.decimal;
955
+ var maxIntegerDigits = decimalType.precision - decimalType.scale;
956
+ function normalizeDuckDbDecimalValue(value, columnName) {
957
+ if (typeof value !== "string") {
958
+ throw new LakeStorageError6(`[SixbDuckLake] Decimal column '${columnName}' must be an exact decimal string.`);
959
+ }
960
+ let normalized;
961
+ try {
962
+ normalized = normalizeDecimalValue(value);
963
+ } catch {
964
+ throw new LakeStorageError6(`[SixbDuckLake] Decimal column '${columnName}' must be an exact decimal string.`);
965
+ }
966
+ const unsigned = normalized.startsWith("-") ? normalized.slice(1) : normalized;
967
+ const [integer, fraction = ""] = unsigned.split(".");
968
+ if (integer.length > maxIntegerDigits || fraction.length > decimalType.scale) {
969
+ throw new LakeStorageError6(`[SixbDuckLake] Decimal column '${columnName}' cannot be represented exactly as ${decimalType.sql}; expected at most ${maxIntegerDigits} integer digits and ${decimalType.scale} fractional digits.`);
970
+ }
971
+ return normalized;
972
+ }
973
+
974
+ // src/internal/schema.ts
975
+ function datasetColumnTypeToDuckDbSql(type) {
976
+ return DUCKDB_COLUMN_TYPES[type].sql;
977
+ }
978
+ function duckDbTypeToDatasetColumnType(typeSql) {
979
+ const type = findDatasetColumnTypeForDuckDbSql(typeSql);
980
+ if (type) {
981
+ return type;
982
+ }
983
+ throw new LakeStorageError7(`[SixbDuckLake] DuckDB column type '${typeSql}' cannot be mapped to a Sixb dataset column type.`);
984
+ }
985
+ function datasetColumnToDuckDbSql(column) {
986
+ const nullableSql = column.nullable ? "" : " NOT NULL";
987
+ return `${quoteIdentifier(column.name)} ${datasetColumnTypeToDuckDbSql(column.type)}${nullableSql}`;
988
+ }
989
+ function datasetSchemaToDuckDbColumnsSql(schema) {
990
+ return schema.columns.map((column) => datasetColumnToDuckDbSql(column)).join(", ");
991
+ }
992
+ function datasetSchemaColumnNamesSql(schema) {
993
+ return schema.columns.map((column) => quoteIdentifier(column.name)).join(", ");
994
+ }
995
+ function normalizeReadValue(value, column) {
996
+ if (value === null || value === undefined) {
997
+ return null;
998
+ }
999
+ switch (column.type) {
1000
+ case "int64":
1001
+ return typeof value === "bigint" ? value.toString() : value;
1002
+ case "decimal":
1003
+ return normalizeDuckDbDecimalValue(value, column.name);
1004
+ case "date":
1005
+ return value instanceof Date ? value.toISOString().slice(0, 10) : String(value);
1006
+ case "timestamp":
1007
+ return value instanceof Date ? value.toISOString() : String(value);
1008
+ case "json":
1009
+ return typeof value === "string" ? JSON.parse(value) : value;
1010
+ case "fileRef":
1011
+ return normalizeFileRef(value);
1012
+ default:
1013
+ return value;
1014
+ }
1015
+ }
1016
+ function normalizeFileRef(value) {
1017
+ if (typeof value !== "object" || value === null || Array.isArray(value)) {
1018
+ throw new LakeStorageError7("[SixbDuckLake] DuckDB fileRef value must be a struct.");
1019
+ }
1020
+ const record = value;
1021
+ return {
1022
+ blobId: getStringField(record, "blobId"),
1023
+ digest: getStringField(record, "digest"),
1024
+ sizeBytes: getIntegerField(record, "sizeBytes"),
1025
+ fileName: getOptionalStringField(record, "fileName"),
1026
+ mediaType: getOptionalStringField(record, "mediaType"),
1027
+ logicalPath: getOptionalStringField(record, "logicalPath")
1028
+ };
1029
+ }
1030
+ function getStringField(record, key) {
1031
+ const value = record[key];
1032
+ if (typeof value !== "string") {
1033
+ throw new LakeStorageError7(`[SixbDuckLake] DuckDB fileRef field '${key}' must be a string.`);
1034
+ }
1035
+ return value;
1036
+ }
1037
+ function getOptionalStringField(record, key) {
1038
+ const value = record[key];
1039
+ if (value === null || value === undefined) {
1040
+ return;
1041
+ }
1042
+ if (typeof value !== "string") {
1043
+ throw new LakeStorageError7(`[SixbDuckLake] DuckDB fileRef field '${key}' must be a string.`);
1044
+ }
1045
+ return value;
1046
+ }
1047
+ function getIntegerField(record, key) {
1048
+ const value = record[key];
1049
+ if (typeof value === "number" && Number.isInteger(value)) {
1050
+ return value;
1051
+ }
1052
+ if (typeof value === "bigint") {
1053
+ return Number(value);
1054
+ }
1055
+ throw new LakeStorageError7(`[SixbDuckLake] DuckDB fileRef field '${key}' must be an integer.`);
1056
+ }
1057
+
1058
+ // src/internal/ducklake-catalog-schema.ts
1059
+ var FILE_REF_STRUCT_CHILDREN = [
1060
+ { name: "blobId", type: "string" },
1061
+ { name: "digest", type: "string" },
1062
+ { name: "sizeBytes", type: "int64" },
1063
+ { name: "fileName", type: "string" },
1064
+ { name: "mediaType", type: "string" },
1065
+ { name: "logicalPath", type: "string" }
1066
+ ];
1067
+ function duckLakeCatalogColumnsToDatasetSchema(tableName, rows) {
1068
+ const childrenByParentId = new Map;
1069
+ const topLevelColumns = [];
1070
+ for (const row of rows) {
1071
+ if (row.parentColumnId === undefined) {
1072
+ topLevelColumns.push(row);
1073
+ continue;
1074
+ }
1075
+ const children = childrenByParentId.get(row.parentColumnId) ?? [];
1076
+ children.push(row);
1077
+ childrenByParentId.set(row.parentColumnId, children);
1078
+ }
1079
+ return {
1080
+ columns: topLevelColumns.sort(compareDuckLakeCatalogColumnOrder).map((column) => duckLakeCatalogColumnToDefinition(tableName, column, childrenByParentId))
1081
+ };
1082
+ }
1083
+ function duckLakeCatalogColumnToDefinition(tableName, column, childrenByParentId) {
1084
+ const children = [...childrenByParentId.get(column.columnId) ?? []].sort(compareDuckLakeCatalogColumnOrder);
1085
+ return {
1086
+ name: column.columnName,
1087
+ type: duckLakeCatalogColumnTypeToDatasetType(tableName, column, children),
1088
+ ...column.nullsAllowed ? { nullable: true } : {}
1089
+ };
1090
+ }
1091
+ function duckLakeCatalogColumnTypeToDatasetType(tableName, column, children) {
1092
+ if (children.length === 0 && column.columnType.toLowerCase() !== "struct") {
1093
+ return duckDbTypeToDatasetColumnType(column.columnType);
1094
+ }
1095
+ if (isFileRefStruct(children)) {
1096
+ return "fileRef";
1097
+ }
1098
+ throw new LakeStorageError8(`[SixbDuckLake] DuckDB column type '${formatDuckLakeCatalogColumnType(column, children)}' in dataset table '${tableName}' cannot be mapped to a Sixb dataset column type.`);
1099
+ }
1100
+ function compareDuckLakeCatalogColumnOrder(left, right) {
1101
+ return Number(left.columnOrder - right.columnOrder);
1102
+ }
1103
+ function isFileRefStruct(children) {
1104
+ if (children.length !== FILE_REF_STRUCT_CHILDREN.length) {
1105
+ return false;
1106
+ }
1107
+ return FILE_REF_STRUCT_CHILDREN.every((expected, index) => {
1108
+ const child = children[index];
1109
+ return child !== undefined && child.columnName === expected.name && duckDbTypeToDatasetColumnType(child.columnType) === expected.type;
1110
+ });
1111
+ }
1112
+ function formatDuckLakeCatalogColumnType(column, children) {
1113
+ if (children.length === 0) {
1114
+ return column.columnType;
1115
+ }
1116
+ return `STRUCT(${children.map((child) => `${child.columnName} ${child.columnType}`).join(", ")})`;
1117
+ }
1118
+
1119
+ // src/internal/ducklake-current-dataset-definitions.ts
1120
+ import { LakeStorageError as LakeStorageError10 } from "@sixb/core/lake-storage";
1121
+
1122
+ // src/internal/names.ts
1123
+ import { LakeStorageError as LakeStorageError9 } from "@sixb/core/lake-storage";
1124
+ var DATASET_TABLE_PREFIX = "sixb__ds__";
1125
+ var textEncoder = new TextEncoder;
1126
+ var textDecoder = new TextDecoder("utf-8", { fatal: true });
1127
+ function isInlineByte(byte) {
1128
+ return byte >= 97 && byte <= 122 || byte >= 48 && byte <= 57;
1129
+ }
1130
+ function isHexPair(value) {
1131
+ return /^[0-9a-f]{2}$/.test(value);
1132
+ }
1133
+ function encodeDatasetId(datasetId) {
1134
+ if (datasetId.trim().length === 0) {
1135
+ throw new LakeStorageError9("[SixbDuckLake] Dataset id must not be empty.");
1136
+ }
1137
+ let encoded = "";
1138
+ for (const byte of textEncoder.encode(datasetId)) {
1139
+ if (isInlineByte(byte)) {
1140
+ encoded += String.fromCharCode(byte);
1141
+ } else if (byte === 46) {
1142
+ encoded += "__";
1143
+ } else {
1144
+ encoded += `_${byte.toString(16).padStart(2, "0")}`;
1145
+ }
1146
+ }
1147
+ return encoded;
1148
+ }
1149
+ function decodeDatasetId(encoded) {
1150
+ if (encoded.length === 0) {
1151
+ return null;
1152
+ }
1153
+ const bytes = [];
1154
+ let index = 0;
1155
+ while (index < encoded.length) {
1156
+ const char = encoded[index];
1157
+ const byte = char.charCodeAt(0);
1158
+ if (isInlineByte(byte)) {
1159
+ bytes.push(byte);
1160
+ index += 1;
1161
+ continue;
1162
+ }
1163
+ if (char !== "_") {
1164
+ return null;
1165
+ }
1166
+ if (encoded[index + 1] === "_") {
1167
+ bytes.push(46);
1168
+ index += 2;
1169
+ continue;
1170
+ }
1171
+ const hex = encoded.slice(index + 1, index + 3);
1172
+ if (!isHexPair(hex)) {
1173
+ return null;
1174
+ }
1175
+ bytes.push(Number.parseInt(hex, 16));
1176
+ index += 3;
1177
+ }
1178
+ try {
1179
+ return textDecoder.decode(new Uint8Array(bytes));
1180
+ } catch {
1181
+ return null;
1182
+ }
1183
+ }
1184
+ function encodeDatasetTableName(datasetId) {
1185
+ return `${DATASET_TABLE_PREFIX}${encodeDatasetId(datasetId)}`;
1186
+ }
1187
+ function decodeDatasetTableName(tableName) {
1188
+ if (!tableName.startsWith(DATASET_TABLE_PREFIX)) {
1189
+ return null;
1190
+ }
1191
+ return decodeDatasetId(tableName.slice(DATASET_TABLE_PREFIX.length));
1192
+ }
1193
+
1194
+ // src/internal/ducklake-current-dataset-definitions.ts
1195
+ async function readCurrentDatasetDefinitions(input) {
1196
+ if (input.datasetIds?.length === 0) {
1197
+ return new Map;
1198
+ }
1199
+ const tables = await readCurrentDatasetTables(input, input.datasetIds);
1200
+ if (tables.length === 0) {
1201
+ return new Map;
1202
+ }
1203
+ const tableNames = tables.map((table) => table.tableName);
1204
+ const columnsByTableName = groupColumnsByTableName(await readCurrentDatasetColumns(input, tableNames));
1205
+ const partitionByTableName = groupPartitionsByTableName(await readCurrentDatasetPartitions(input, tableNames));
1206
+ const definitions = new Map;
1207
+ for (const table of tables) {
1208
+ const partitionBy = partitionByTableName.get(table.tableName) ?? [];
1209
+ definitions.set(table.datasetId, {
1210
+ kind: "dataset",
1211
+ id: table.datasetId,
1212
+ schema: duckLakeCatalogColumnsToDatasetSchema(table.tableName, columnsByTableName.get(table.tableName) ?? []),
1213
+ ...partitionBy.length > 0 ? { partitionBy } : {},
1214
+ ...table.comment !== undefined ? { description: table.comment } : {}
1215
+ });
1216
+ }
1217
+ return definitions;
1218
+ }
1219
+ async function readCurrentDatasetTables(input, datasetIds) {
1220
+ if (datasetIds !== undefined && datasetIds.length === 0) {
1221
+ return [];
1222
+ }
1223
+ const tableNameFilter = datasetIds === undefined ? "" : `AND table_name IN (${quoteSqlStringList([...new Set(datasetIds)].map((datasetId) => encodeDatasetTableName(datasetId)))})`;
1224
+ const rows = await input.runtime.query(`SELECT table_name, comment FROM duckdb_tables() WHERE database_name = ${quoteSqlString(duckLakeAlias(input.options))} AND schema_name = 'main' ${tableNameFilter} AND NOT internal ORDER BY table_name`);
1225
+ const tables = [];
1226
+ for (const row of rows) {
1227
+ const tableName = getString(row, "table_name");
1228
+ const datasetId = decodeDatasetTableName(tableName);
1229
+ if (datasetId === null) {
1230
+ continue;
1231
+ }
1232
+ tables.push({
1233
+ datasetId,
1234
+ tableName,
1235
+ comment: getOptionalString(row, "comment")
1236
+ });
1237
+ }
1238
+ return tables;
1239
+ }
1240
+ async function readCurrentDatasetColumns(input, tableNames) {
1241
+ if (tableNames.length === 0) {
1242
+ return [];
1243
+ }
1244
+ const ducklakeTable = duckLakeMetadataTableName(input.options, "ducklake_table");
1245
+ const ducklakeColumn = duckLakeMetadataTableName(input.options, "ducklake_column");
1246
+ const rows = await input.runtime.query(`
1247
+ SELECT
1248
+ table_meta.table_name,
1249
+ column_meta.column_id,
1250
+ column_meta.column_order,
1251
+ column_meta.column_name,
1252
+ column_meta.column_type,
1253
+ CAST(column_meta.nulls_allowed AS BOOLEAN) AS nulls_allowed,
1254
+ column_meta.parent_column
1255
+ FROM ${ducklakeTable} table_meta
1256
+ JOIN ${ducklakeColumn} column_meta
1257
+ ON column_meta.table_id = table_meta.table_id
1258
+ AND column_meta.end_snapshot IS NULL
1259
+ WHERE table_meta.end_snapshot IS NULL
1260
+ AND table_meta.table_name IN (${quoteSqlStringList(tableNames)})
1261
+ ORDER BY table_meta.table_name, column_meta.column_order
1262
+ `);
1263
+ return rows.map((row) => ({
1264
+ tableName: getString(row, "table_name"),
1265
+ columnId: getBigIntLike(row, "column_id"),
1266
+ columnOrder: getBigIntLike(row, "column_order"),
1267
+ columnName: getString(row, "column_name"),
1268
+ columnType: getString(row, "column_type"),
1269
+ nullsAllowed: getBoolean(row, "nulls_allowed"),
1270
+ parentColumnId: getOptionalBigIntLike(row, "parent_column")
1271
+ }));
1272
+ }
1273
+ async function readCurrentDatasetPartitions(input, tableNames) {
1274
+ if (tableNames.length === 0) {
1275
+ return [];
1276
+ }
1277
+ const ducklakeTable = duckLakeMetadataTableName(input.options, "ducklake_table");
1278
+ const ducklakePartitionInfo = duckLakeMetadataTableName(input.options, "ducklake_partition_info");
1279
+ const ducklakePartitionColumn = duckLakeMetadataTableName(input.options, "ducklake_partition_column");
1280
+ const ducklakeColumn = duckLakeMetadataTableName(input.options, "ducklake_column");
1281
+ const rows = await input.runtime.query(`
1282
+ SELECT
1283
+ table_meta.table_name,
1284
+ column_meta.column_name,
1285
+ partition_column.transform
1286
+ FROM ${ducklakeTable} table_meta
1287
+ JOIN ${ducklakePartitionInfo} partition_info
1288
+ ON partition_info.table_id = table_meta.table_id
1289
+ AND partition_info.end_snapshot IS NULL
1290
+ JOIN ${ducklakePartitionColumn} partition_column
1291
+ ON partition_column.table_id = table_meta.table_id
1292
+ AND partition_column.partition_id = partition_info.partition_id
1293
+ JOIN ${ducklakeColumn} column_meta
1294
+ ON column_meta.table_id = table_meta.table_id
1295
+ AND column_meta.column_id = partition_column.column_id
1296
+ AND column_meta.end_snapshot IS NULL
1297
+ WHERE table_meta.end_snapshot IS NULL
1298
+ AND table_meta.table_name IN (${quoteSqlStringList(tableNames)})
1299
+ ORDER BY table_meta.table_name, partition_column.partition_key_index
1300
+ `);
1301
+ return rows.map((row) => {
1302
+ const partition = {
1303
+ tableName: getString(row, "table_name"),
1304
+ columnName: getString(row, "column_name"),
1305
+ transform: getString(row, "transform")
1306
+ };
1307
+ if (partition.transform !== "identity") {
1308
+ throw new LakeStorageError10(`[SixbDuckLake] Dataset table '${partition.tableName}' uses unsupported DuckLake partition transform '${partition.transform}'.`);
1309
+ }
1310
+ return partition;
1311
+ });
1312
+ }
1313
+ function groupColumnsByTableName(rows) {
1314
+ const result = new Map;
1315
+ for (const row of rows) {
1316
+ const tableRows = result.get(row.tableName) ?? [];
1317
+ tableRows.push(row);
1318
+ result.set(row.tableName, tableRows);
1319
+ }
1320
+ return result;
1321
+ }
1322
+ function groupPartitionsByTableName(rows) {
1323
+ const result = new Map;
1324
+ for (const row of rows) {
1325
+ const tablePartitions = result.get(row.tableName) ?? [];
1326
+ tablePartitions.push(row.columnName);
1327
+ result.set(row.tableName, tablePartitions);
1328
+ }
1329
+ return result;
1330
+ }
1331
+
1332
+ // src/internal/ducklake-dataset-catalog.ts
1333
+ class DuckLakeDatasetCatalog {
1334
+ options;
1335
+ connections;
1336
+ constructor(options, connections) {
1337
+ this.options = options;
1338
+ this.connections = connections;
1339
+ }
1340
+ async createDataset(definition) {
1341
+ this.assertSchema(definition);
1342
+ const existing = await this.getDataset(definition.id);
1343
+ if (existing) {
1344
+ const plan = planDatasetDefinitionUpdate(existing, definition);
1345
+ if (plan.changed) {
1346
+ await this.applyExistingDatasetUpdate(plan);
1347
+ this.connections.markLocalCatalogChanged();
1348
+ await this.connections.resetRuntime();
1349
+ }
1350
+ return await this.getDataset(definition.id) ?? plan.definition;
1351
+ }
1352
+ const tableName = encodeDatasetTableName(definition.id);
1353
+ const qualifiedName = qualifiedTableName(this.options, tableName);
1354
+ await this.connections.withExclusiveAttached(async (runtime) => {
1355
+ await runtime.run(`CREATE TABLE ${qualifiedName} (${datasetSchemaToDuckDbColumnsSql(definition.schema)})`);
1356
+ if (definition.description !== undefined) {
1357
+ await runtime.run(`COMMENT ON TABLE ${qualifiedName} IS ${quoteSqlString(definition.description)}`);
1358
+ }
1359
+ await this.applyPartitionBy(runtime, qualifiedName, definition);
1360
+ });
1361
+ this.connections.markLocalCatalogChanged();
1362
+ return structuredClone(definition);
1363
+ }
1364
+ async assertDatasetDefinitionsCompatible(definitions) {
1365
+ const failures = [];
1366
+ const checkedDefinitions = [];
1367
+ for (const definition of definitions) {
1368
+ try {
1369
+ this.assertSchema(definition);
1370
+ checkedDefinitions.push(definition);
1371
+ } catch (error) {
1372
+ const message = error instanceof Error ? error.message : String(error);
1373
+ failures.push(`- ${definition.id}: ${message}`);
1374
+ }
1375
+ }
1376
+ const existingById = await this.readCurrentDatasetDefinitions({
1377
+ datasetIds: checkedDefinitions.map((definition) => definition.id)
1378
+ });
1379
+ for (const definition of checkedDefinitions) {
1380
+ const existing = existingById.get(definition.id);
1381
+ if (!existing) {
1382
+ continue;
1383
+ }
1384
+ try {
1385
+ planDatasetDefinitionUpdate(existing, definition);
1386
+ } catch (error) {
1387
+ const message = error instanceof Error ? error.message : String(error);
1388
+ failures.push(`- ${definition.id}: ${message}`);
1389
+ }
1390
+ }
1391
+ if (failures.length > 0) {
1392
+ const details = failures.join(`
1393
+ `);
1394
+ throw new LakeStorageError11(`[SixbLake] Lake dataset definition check failed for ${failures.length} dataset(s).
1395
+ ${details}`);
1396
+ }
1397
+ }
1398
+ async getDataset(datasetId) {
1399
+ this.connections.assertOpen();
1400
+ return this.connections.withAttachedRuntime((runtime) => this.getDatasetOnRuntime(runtime, datasetId));
1401
+ }
1402
+ async getDatasetOnRuntime(runtime, datasetId) {
1403
+ return (await this.readCurrentDatasetDefinitionsOnRuntime(runtime, {
1404
+ datasetIds: [datasetId]
1405
+ })).get(datasetId) ?? null;
1406
+ }
1407
+ async listDatasets() {
1408
+ this.connections.assertOpen();
1409
+ return this.connections.withAttachedRuntime(async (runtime) => {
1410
+ return [...(await this.readCurrentDatasetDefinitionsOnRuntime(runtime)).values()].sort((left, right) => left.id.localeCompare(right.id));
1411
+ });
1412
+ }
1413
+ async readCurrentDatasetDefinitions(options) {
1414
+ if (options?.datasetIds?.length === 0) {
1415
+ return new Map;
1416
+ }
1417
+ return this.connections.withAttachedRuntime((runtime) => this.readCurrentDatasetDefinitionsOnRuntime(runtime, options));
1418
+ }
1419
+ async readCurrentDatasetDefinitionsOnRuntime(runtime, options) {
1420
+ return readCurrentDatasetDefinitions({
1421
+ options: this.options,
1422
+ runtime,
1423
+ datasetIds: options?.datasetIds
1424
+ });
1425
+ }
1426
+ async getDatasetSchemaAtSnapshot(runtime, datasetId, tableName, tableId, snapshotId) {
1427
+ assertDuckLakeSnapshotId(snapshotId);
1428
+ const columns = await this.readDatasetColumnsAtSnapshot(runtime, tableName, tableId, snapshotId);
1429
+ if (columns.length === 0) {
1430
+ throw new LakeStorageError11(`[SixbDuckLake] Could not reconstruct schema for dataset '${datasetId}' at DuckLake snapshot '${snapshotId}'.`);
1431
+ }
1432
+ return duckLakeCatalogColumnsToDatasetSchema(tableName, columns);
1433
+ }
1434
+ assertSchema(definition) {
1435
+ if (!definition.schema) {
1436
+ throw new LakeStorageError11(`[SixbDuckLake] Dataset '${definition.id}' requires a schema for DuckLake storage.`);
1437
+ }
1438
+ }
1439
+ async applyExistingDatasetUpdate(plan) {
1440
+ const tableName = encodeDatasetTableName(plan.definition.id);
1441
+ const qualifiedName = qualifiedTableName(this.options, tableName);
1442
+ await this.connections.withExclusiveAttached(async (runtime) => {
1443
+ if (plan.schema.kind === "none") {
1444
+ await this.applyMetadataDelta(runtime, qualifiedName, plan);
1445
+ return;
1446
+ }
1447
+ await runtime.run("BEGIN TRANSACTION");
1448
+ let committed = false;
1449
+ try {
1450
+ await this.applySchemaEvolutionDdl(runtime, qualifiedName, plan.schema);
1451
+ await this.applyMetadataDelta(runtime, qualifiedName, plan);
1452
+ await this.setSchemaEvolutionCommitMetadata(runtime, plan.definition.id, plan.schema);
1453
+ await runtime.run("COMMIT");
1454
+ committed = true;
1455
+ } catch (error) {
1456
+ if (!committed) {
1457
+ await this.rollbackTransaction(runtime);
1458
+ }
1459
+ throw error;
1460
+ }
1461
+ });
1462
+ }
1463
+ async applySchemaEvolutionDdl(runtime, qualifiedName, plan) {
1464
+ for (const column of plan.columns) {
1465
+ await runtime.run(`ALTER TABLE ${qualifiedName} ADD COLUMN ${datasetColumnToDuckDbSql(column)}`);
1466
+ }
1467
+ }
1468
+ async applyMetadataDelta(runtime, qualifiedName, plan) {
1469
+ if (plan.metadata.descriptionChanged && plan.definition.description !== undefined) {
1470
+ await runtime.run(`COMMENT ON TABLE ${qualifiedName} IS ${quoteSqlString(plan.definition.description)}`);
1471
+ }
1472
+ if (plan.metadata.partitionByChanged) {
1473
+ await this.applyPartitionBy(runtime, qualifiedName, plan.definition);
1474
+ }
1475
+ }
1476
+ async applyPartitionBy(runtime, qualifiedName, definition) {
1477
+ if (definition.partitionBy === undefined || definition.partitionBy.length === 0) {
1478
+ return;
1479
+ }
1480
+ try {
1481
+ await runtime.run(`ALTER TABLE ${qualifiedName} SET PARTITIONED BY (${definition.partitionBy.map((columnName) => quoteIdentifier(columnName)).join(", ")})`);
1482
+ } catch (error) {
1483
+ const message = error instanceof Error ? error.message : String(error);
1484
+ throw new LakeStorageError11(`[SixbDuckLake] Dataset '${definition.id}' cannot apply partitionBy because DuckLake rejected ALTER TABLE SET PARTITIONED BY: ${message}`);
1485
+ }
1486
+ }
1487
+ async setSchemaEvolutionCommitMetadata(runtime, datasetId, plan) {
1488
+ await runtime.run(`CALL ${quoteIdentifier(duckLakeAlias(this.options))}.set_commit_message(${quoteSqlString("Sixb")}, ${quoteSqlString(`evolve dataset ${datasetId} schema`)}, extra_info => ${quoteSqlString(JSON.stringify({
1489
+ sixb: {
1490
+ kind: "datasetVersion",
1491
+ datasetId,
1492
+ commitId: randomUUID(),
1493
+ mode: "schema",
1494
+ schemaChange: {
1495
+ addColumns: plan.columns.map((column) => column.name)
1496
+ }
1497
+ }
1498
+ }))})`);
1499
+ }
1500
+ async rollbackTransaction(runtime) {
1501
+ try {
1502
+ await runtime.run("ROLLBACK");
1503
+ } catch {}
1504
+ }
1505
+ async readDatasetColumnsAtSnapshot(runtime, tableName, tableId, snapshotId) {
1506
+ const ducklakeColumn = duckLakeMetadataTableName(this.options, "ducklake_column");
1507
+ const rows = await runtime.query(`
1508
+ SELECT
1509
+ column_id,
1510
+ column_order,
1511
+ column_name,
1512
+ column_type,
1513
+ CAST(nulls_allowed AS BOOLEAN) AS nulls_allowed,
1514
+ parent_column
1515
+ FROM ${ducklakeColumn}
1516
+ WHERE table_id = ${tableId}
1517
+ AND begin_snapshot <= ${snapshotId}
1518
+ AND (end_snapshot IS NULL OR end_snapshot > ${snapshotId})
1519
+ ORDER BY column_order
1520
+ `);
1521
+ return rows.map((row) => ({
1522
+ tableName,
1523
+ columnId: getBigIntLike(row, "column_id"),
1524
+ columnOrder: getBigIntLike(row, "column_order"),
1525
+ columnName: getString(row, "column_name"),
1526
+ columnType: getString(row, "column_type"),
1527
+ nullsAllowed: getBoolean(row, "nulls_allowed"),
1528
+ parentColumnId: getOptionalBigIntLike(row, "parent_column")
1529
+ }));
1530
+ }
1531
+ }
1532
+ function assertDuckLakeSnapshotId(snapshotId) {
1533
+ if (!/^\d+$/.test(snapshotId)) {
1534
+ throw new LakeStorageError11(`[SixbDuckLake] Invalid DuckLake snapshot id '${snapshotId}'.`);
1535
+ }
1536
+ }
1537
+
1538
+ // src/internal/ducklake-maintenance.ts
1539
+ import { LakeStorageError as LakeStorageError12 } from "@sixb/core/lake-storage";
1540
+ var DEFAULT_RETENTION = "7 days";
1541
+
1542
+ class DuckLakeMaintenance {
1543
+ options;
1544
+ connections;
1545
+ constructor(options, connections) {
1546
+ this.options = options;
1547
+ this.connections = connections;
1548
+ }
1549
+ async runMaintenance(options = {}) {
1550
+ this.connections.assertOpen();
1551
+ const expireOlderThan = options.expireOlderThan ?? DEFAULT_RETENTION;
1552
+ const deleteOlderThan = options.deleteOlderThan ?? expireOlderThan;
1553
+ const dryRun = options.dryRun ?? false;
1554
+ if (this.options.readOnly && !dryRun) {
1555
+ throw new LakeStorageError12("[SixbDuckLake] DuckLake maintenance cannot run in read-only mode.");
1556
+ }
1557
+ return this.connections.withExclusiveAttached(async (runtime) => {
1558
+ const counts = await runMaintenanceProcedures(runtime, this.options, {
1559
+ expireOlderThan,
1560
+ deleteOlderThan
1561
+ }, dryRun);
1562
+ if (!dryRun) {
1563
+ this.connections.markLocalCatalogChanged();
1564
+ }
1565
+ return {
1566
+ dryRun,
1567
+ expireOlderThan,
1568
+ deleteOlderThan,
1569
+ ...counts
1570
+ };
1571
+ });
1572
+ }
1573
+ }
1574
+ async function runMaintenanceProcedures(runtime, options, retention, dryRun) {
1575
+ const aliasSql = quoteSqlString(duckLakeAlias(options));
1576
+ const snapshots = await runtime.query(maintenanceProcedureSql("ducklake_expire_snapshots", aliasSql, retention.expireOlderThan, dryRun));
1577
+ const oldFiles = await runtime.query(maintenanceProcedureSql("ducklake_cleanup_old_files", aliasSql, retention.deleteOlderThan, dryRun));
1578
+ const orphanedFiles = await runtime.query(maintenanceProcedureSql("ducklake_delete_orphaned_files", aliasSql, retention.deleteOlderThan, dryRun));
1579
+ return {
1580
+ snapshots: snapshots.length,
1581
+ oldFiles: oldFiles.length,
1582
+ orphanedFiles: orphanedFiles.length
1583
+ };
1584
+ }
1585
+ function maintenanceProcedureSql(procedure, aliasSql, olderThan, dryRun) {
1586
+ const olderThanSql = quoteSqlString(olderThan);
1587
+ const dryRunSql = dryRun ? "dry_run => true, " : "";
1588
+ return `CALL ${procedure}(${aliasSql}, ${dryRunSql}` + `older_than => now() - INTERVAL ${olderThanSql})`;
1589
+ }
1590
+
1591
+ // src/internal/ducklake-row-reader.ts
1592
+ import { LakeStorageError as LakeStorageError14 } from "@sixb/core/lake-storage";
1593
+
1594
+ // src/internal/ducklake-dataset-table-ref.ts
1595
+ async function resolveDatasetTableRef(options, runtime, datasetId) {
1596
+ const tableName = encodeDatasetTableName(datasetId);
1597
+ const ducklakeTable = duckLakeMetadataTableName(options, "ducklake_table");
1598
+ const [row] = await runtime.query(`
1599
+ SELECT table_id
1600
+ FROM ${ducklakeTable}
1601
+ WHERE table_name = ${quoteSqlString(tableName)}
1602
+ AND end_snapshot IS NULL
1603
+ LIMIT 1
1604
+ `);
1605
+ return row === undefined ? null : {
1606
+ datasetId,
1607
+ tableName,
1608
+ tableId: getBigIntLike(row, "table_id")
1609
+ };
1610
+ }
1611
+
1612
+ // src/internal/versions.ts
1613
+ import { LakeStorageError as LakeStorageError13 } from "@sixb/core/lake-storage";
1614
+ function parseVersionId(versionId) {
1615
+ if (!versionId.startsWith("ducklake:")) {
1616
+ throw new LakeStorageError13(`[SixbDuckLake] Invalid DuckLake version id '${versionId}'.`);
1617
+ }
1618
+ const snapshotId = versionId.slice("ducklake:".length);
1619
+ if (!/^\d+$/.test(snapshotId)) {
1620
+ throw new LakeStorageError13(`[SixbDuckLake] Invalid DuckLake version id '${versionId}'.`);
1621
+ }
1622
+ return snapshotId;
1623
+ }
1624
+ function toVersionId(snapshotId) {
1625
+ return `ducklake:${snapshotId}`;
1626
+ }
1627
+ function parseCommitMetadata(value) {
1628
+ if (typeof value !== "string") {
1629
+ return;
1630
+ }
1631
+ let parsed;
1632
+ try {
1633
+ parsed = JSON.parse(value);
1634
+ } catch {
1635
+ return;
1636
+ }
1637
+ if (!isRecord(parsed) || !isRecord(parsed.sixb) || parsed.sixb.kind !== "datasetVersion" || typeof parsed.sixb.datasetId !== "string") {
1638
+ return;
1639
+ }
1640
+ const mode = parsed.sixb.mode;
1641
+ const commitId = parsed.sixb.commitId;
1642
+ const rowCount = parsed.sixb.rowCount;
1643
+ return {
1644
+ kind: "datasetVersion",
1645
+ datasetId: parsed.sixb.datasetId,
1646
+ ...typeof commitId === "string" ? { commitId } : {},
1647
+ ...mode === "snapshot" || mode === "append" || mode === "schema" ? { mode } : {},
1648
+ ...isDatasetProducer(parsed.sixb.producer) ? { producer: parsed.sixb.producer } : {},
1649
+ ...isDatasetVersionRefs(parsed.sixb.inputs) ? { inputs: parsed.sixb.inputs } : {},
1650
+ ...typeof commitId === "string" && isRowCount(rowCount) ? { rowCount } : {},
1651
+ ...isSchemaChangeMetadata(parsed.sixb.schemaChange) ? { schemaChange: parsed.sixb.schemaChange } : {}
1652
+ };
1653
+ }
1654
+ function parseInlineDataChange(changesMade, tableId) {
1655
+ const tableIdText = tableId.toString();
1656
+ let hasDataChange = false;
1657
+ let hasDeleteChange = false;
1658
+ for (const change of changesMade.split(",")) {
1659
+ const [kind, changedTableId] = change.split(":");
1660
+ if (changedTableId !== tableIdText) {
1661
+ continue;
1662
+ }
1663
+ const normalizedKind = kind.toLowerCase();
1664
+ if (normalizedKind.includes("insert") || normalizedKind.includes("delete")) {
1665
+ hasDataChange = true;
1666
+ }
1667
+ if (normalizedKind.includes("delete")) {
1668
+ hasDeleteChange = true;
1669
+ }
1670
+ }
1671
+ return { hasDataChange, hasDeleteChange };
1672
+ }
1673
+ function isRecord(value) {
1674
+ return typeof value === "object" && value !== null && !Array.isArray(value);
1675
+ }
1676
+ function isDatasetProducer(value) {
1677
+ return isRecord(value) && (value.kind === "sync" || value.kind === "pipeline") && optionalString(value.id) && optionalString(value.runId);
1678
+ }
1679
+ function isDatasetVersionRefs(value) {
1680
+ return Array.isArray(value) && value.every((item) => isRecord(item) && typeof item.datasetId === "string" && typeof item.versionId === "string");
1681
+ }
1682
+ function isSchemaChangeMetadata(value) {
1683
+ return isRecord(value) && (value.addColumns === undefined || Array.isArray(value.addColumns) && value.addColumns.every((columnName) => typeof columnName === "string"));
1684
+ }
1685
+ function optionalString(value) {
1686
+ return value === undefined || typeof value === "string";
1687
+ }
1688
+ function isRowCount(value) {
1689
+ return typeof value === "number" && Number.isSafeInteger(value) && value >= 0;
1690
+ }
1691
+
1692
+ // src/internal/ducklake-row-reader.ts
1693
+ class DuckLakeRowReader {
1694
+ options;
1695
+ connections;
1696
+ snapshots;
1697
+ constructor(options, connections, snapshots) {
1698
+ this.options = options;
1699
+ this.connections = connections;
1700
+ this.snapshots = snapshots;
1701
+ }
1702
+ async* readRows(input) {
1703
+ this.connections.assertOpen();
1704
+ const lease = await this.connections.acquireAttachedRuntime();
1705
+ try {
1706
+ const runtime = lease.runtime;
1707
+ const tableRef = await resolveDatasetTableRef(this.options, runtime, input.datasetId);
1708
+ if (!tableRef) {
1709
+ throw new LakeStorageError14(`[SixbDuckLake] Unknown dataset '${input.datasetId}'.`);
1710
+ }
1711
+ const version = await this.resolveVersion(runtime, tableRef, input.versionId);
1712
+ const snapshotId = parseVersionId(version.versionId);
1713
+ const selectedColumns = this.resolveReadColumns(tableRef.datasetId, version.schema, input.columns);
1714
+ const columnsSql = selectedColumns.map((column) => quoteIdentifier(column.name)).join(", ");
1715
+ const tableSql = qualifiedTableName(this.options, tableRef.tableName);
1716
+ const orderSql = version.schema.columns.some((column) => column.name.toLowerCase() === "rowid") ? "" : " ORDER BY rowid";
1717
+ const limitSql = input.limit === undefined ? "" : ` LIMIT ${Math.max(0, Math.trunc(input.limit))}`;
1718
+ const offsetSql = input.offset === undefined ? "" : ` OFFSET ${Math.max(0, Math.trunc(input.offset))}`;
1719
+ const sql = `SELECT ${columnsSql} FROM ${tableSql} AT (VERSION => ${snapshotId})${orderSql}${limitSql}${offsetSql}`;
1720
+ const rows = input.limit === undefined ? runtime.streamRows(sql) : await runtime.query(sql);
1721
+ yield* this.normalizeRows(rows, selectedColumns);
1722
+ } finally {
1723
+ await lease.release();
1724
+ }
1725
+ }
1726
+ async* normalizeRows(rows, selectedColumns) {
1727
+ for await (const row of rows) {
1728
+ const output = {};
1729
+ for (const column of selectedColumns) {
1730
+ output[column.name] = normalizeReadValue(row[column.name], column);
1731
+ }
1732
+ yield output;
1733
+ }
1734
+ }
1735
+ async resolveVersion(runtime, tableRef, versionId) {
1736
+ if (versionId !== undefined) {
1737
+ const snapshotId = parseVersionId(versionId);
1738
+ const version = await this.snapshots.getVersionForTableRef(runtime, tableRef, snapshotId);
1739
+ if (!version) {
1740
+ this.throwNoCommittedVersion(tableRef.datasetId);
1741
+ }
1742
+ return version;
1743
+ }
1744
+ const latestVersion = await this.snapshots.getLatestVersionForTableRef(runtime, tableRef);
1745
+ if (!latestVersion) {
1746
+ this.throwNoCommittedVersion(tableRef.datasetId);
1747
+ }
1748
+ return latestVersion;
1749
+ }
1750
+ resolveReadColumns(datasetId, schema, columns) {
1751
+ if (columns === undefined || columns.length === 0) {
1752
+ return schema.columns;
1753
+ }
1754
+ const columnsByName = new Map(schema.columns.map((column) => [column.name, column]));
1755
+ return columns.map((columnName) => {
1756
+ const column = columnsByName.get(columnName);
1757
+ if (!column) {
1758
+ throw new LakeStorageError14(`[SixbDuckLake] Dataset '${datasetId}' does not have column '${columnName}' at the requested version.`);
1759
+ }
1760
+ return column;
1761
+ });
1762
+ }
1763
+ throwNoCommittedVersion(datasetId) {
1764
+ throw new LakeStorageError14(`[SixbDuckLake] No committed version found for dataset '${datasetId}'.`);
1765
+ }
1766
+ }
1767
+
1768
+ // src/internal/ducklake-snapshot-reader.ts
1769
+ import { LakeStorageError as LakeStorageError15 } from "@sixb/core/lake-storage";
1770
+ var SNAPSHOT_ROW_BATCH_SIZE = 128;
1771
+ var CATALOG_SNAPSHOT_SCAN_LIMIT = 512;
1772
+
1773
+ class DuckLakeSnapshotReader {
1774
+ options;
1775
+ connections;
1776
+ datasets;
1777
+ constructor(options, connections, datasets) {
1778
+ this.options = options;
1779
+ this.connections = connections;
1780
+ this.datasets = datasets;
1781
+ }
1782
+ async listVersions(datasetId, limit) {
1783
+ this.connections.assertOpen();
1784
+ return this.connections.withAttachedRuntime(async (runtime) => {
1785
+ const tableRef = await resolveDatasetTableRef(this.options, runtime, datasetId);
1786
+ if (!tableRef) {
1787
+ return [];
1788
+ }
1789
+ return this.listVersionsForTableRef(runtime, tableRef, limit);
1790
+ });
1791
+ }
1792
+ async getLatestVersion(datasetId) {
1793
+ this.connections.assertOpen();
1794
+ return this.connections.withAttachedRuntime(async (runtime) => {
1795
+ const tableRef = await resolveDatasetTableRef(this.options, runtime, datasetId);
1796
+ if (!tableRef) {
1797
+ return null;
1798
+ }
1799
+ return this.getLatestVersionForTableRef(runtime, tableRef);
1800
+ });
1801
+ }
1802
+ async getVersion(datasetId, versionId) {
1803
+ const snapshotId = parseVersionId(versionId);
1804
+ this.connections.assertOpen();
1805
+ return this.connections.withAttachedRuntime(async (runtime) => {
1806
+ const tableRef = await resolveDatasetTableRef(this.options, runtime, datasetId);
1807
+ if (!tableRef) {
1808
+ return null;
1809
+ }
1810
+ return this.getVersionForTableRef(runtime, tableRef, snapshotId);
1811
+ });
1812
+ }
1813
+ async listDatasetCatalogState(datasetIds) {
1814
+ this.connections.assertOpen();
1815
+ if (datasetIds.length === 0) {
1816
+ return [];
1817
+ }
1818
+ return this.connections.withAttachedRuntime((runtime) => this.collectDatasetCatalogState(runtime, datasetIds));
1819
+ }
1820
+ async collectDatasetCatalogState(runtime, datasetIds) {
1821
+ const uniqueIds = [...new Set(datasetIds)];
1822
+ const tableIdsByDatasetId = await this.resolveDatasetTableIds(runtime, uniqueIds);
1823
+ const latestRowByDatasetId = await this.resolveLatestSnapshotRows(runtime, tableIdsByDatasetId);
1824
+ return uniqueIds.map((datasetId) => {
1825
+ if (!tableIdsByDatasetId.has(datasetId)) {
1826
+ return { datasetId, materialized: false, latestVersion: null };
1827
+ }
1828
+ const row = latestRowByDatasetId.get(datasetId);
1829
+ return {
1830
+ datasetId,
1831
+ materialized: true,
1832
+ latestVersion: row ? this.snapshotRowToSummary(datasetId, row) : null
1833
+ };
1834
+ });
1835
+ }
1836
+ snapshotRowToSummary(datasetId, row) {
1837
+ return {
1838
+ datasetId,
1839
+ versionId: toVersionId(row.snapshotId),
1840
+ mode: row.mode,
1841
+ createdAt: row.createdAt,
1842
+ ...row.metadata?.rowCount !== undefined ? { rowCount: row.metadata.rowCount } : {}
1843
+ };
1844
+ }
1845
+ async resolveDatasetTableIds(runtime, datasetIds) {
1846
+ const datasetIdByTableName = new Map;
1847
+ for (const datasetId of datasetIds) {
1848
+ datasetIdByTableName.set(encodeDatasetTableName(datasetId), datasetId);
1849
+ }
1850
+ const ducklakeTable = duckLakeMetadataTableName(this.options, "ducklake_table");
1851
+ const tableNameList = [...datasetIdByTableName.keys()].map((name) => quoteSqlString(name)).join(", ");
1852
+ const rows = await runtime.query(`
1853
+ SELECT table_id, table_name
1854
+ FROM ${ducklakeTable}
1855
+ WHERE table_name IN (${tableNameList})
1856
+ AND end_snapshot IS NULL
1857
+ `);
1858
+ const tableIdsByDatasetId = new Map;
1859
+ for (const row of rows) {
1860
+ const datasetId = datasetIdByTableName.get(getString(row, "table_name"));
1861
+ if (datasetId !== undefined) {
1862
+ tableIdsByDatasetId.set(datasetId, getBigIntLike(row, "table_id"));
1863
+ }
1864
+ }
1865
+ return tableIdsByDatasetId;
1866
+ }
1867
+ async resolveLatestSnapshotRows(runtime, tableIdsByDatasetId) {
1868
+ const result = new Map;
1869
+ if (tableIdsByDatasetId.size === 0) {
1870
+ return result;
1871
+ }
1872
+ const unresolved = new Map(tableIdsByDatasetId);
1873
+ const tableIds = [...new Set(tableIdsByDatasetId.values())];
1874
+ let beforeSnapshotId;
1875
+ let scanned = 0;
1876
+ while (unresolved.size > 0 && scanned < CATALOG_SNAPSHOT_SCAN_LIMIT) {
1877
+ const candidates = await this.queryCatalogSnapshotBatch(runtime, beforeSnapshotId, SNAPSHOT_ROW_BATCH_SIZE);
1878
+ if (candidates.length === 0) {
1879
+ break;
1880
+ }
1881
+ const fileFlags = await this.queryFileChangeFlags(runtime, candidates.map((candidate) => candidate.snapshotId), tableIds);
1882
+ for (const candidate of candidates) {
1883
+ scanned += 1;
1884
+ for (const [datasetId, tableId] of [...unresolved]) {
1885
+ const flags = fileFlags.get(fileChangeKey(candidate.snapshotId, tableId));
1886
+ const row = this.candidateToSnapshotRow(datasetId, tableId, {
1887
+ snapshotId: candidate.snapshotId,
1888
+ createdAt: candidate.createdAt,
1889
+ changesMade: candidate.changesMade,
1890
+ hasFileChange: flags?.hasFileChange ?? false,
1891
+ hasFileDeleteChange: flags?.hasFileDeleteChange ?? false,
1892
+ ...candidate.metadata !== undefined ? { metadata: candidate.metadata } : {}
1893
+ });
1894
+ if (row) {
1895
+ result.set(datasetId, row);
1896
+ unresolved.delete(datasetId);
1897
+ }
1898
+ }
1899
+ if (unresolved.size === 0) {
1900
+ break;
1901
+ }
1902
+ }
1903
+ if (candidates.length < SNAPSHOT_ROW_BATCH_SIZE) {
1904
+ break;
1905
+ }
1906
+ beforeSnapshotId = candidates[candidates.length - 1]?.snapshotId;
1907
+ }
1908
+ return result;
1909
+ }
1910
+ async queryCatalogSnapshotBatch(runtime, beforeSnapshotId, limit) {
1911
+ const ducklakeSnapshot = duckLakeMetadataTableName(this.options, "ducklake_snapshot");
1912
+ const ducklakeSnapshotChanges = duckLakeMetadataTableName(this.options, "ducklake_snapshot_changes");
1913
+ let whereSql = "";
1914
+ if (beforeSnapshotId !== undefined) {
1915
+ assertDuckLakeSnapshotId2(beforeSnapshotId);
1916
+ whereSql = `WHERE snapshot.snapshot_id < ${beforeSnapshotId}`;
1917
+ }
1918
+ const rows = await runtime.query(`
1919
+ SELECT
1920
+ snapshot.snapshot_id,
1921
+ snapshot.snapshot_time,
1922
+ changes.changes_made,
1923
+ changes.commit_extra_info
1924
+ FROM ${ducklakeSnapshot} snapshot
1925
+ JOIN ${ducklakeSnapshotChanges} changes ON changes.snapshot_id = snapshot.snapshot_id
1926
+ ${whereSql}
1927
+ ORDER BY snapshot.snapshot_id DESC
1928
+ LIMIT ${Math.max(0, Math.trunc(limit))}
1929
+ `);
1930
+ return rows.map((row) => {
1931
+ const metadata = parseCommitMetadata(getOptionalString(row, "commit_extra_info"));
1932
+ return {
1933
+ snapshotId: String(getBigIntLike(row, "snapshot_id")),
1934
+ createdAt: getDate(row, "snapshot_time"),
1935
+ changesMade: getString(row, "changes_made"),
1936
+ ...metadata !== undefined ? { metadata } : {}
1937
+ };
1938
+ });
1939
+ }
1940
+ async queryFileChangeFlags(runtime, snapshotIds, tableIds) {
1941
+ const flags = new Map;
1942
+ if (snapshotIds.length === 0 || tableIds.length === 0) {
1943
+ return flags;
1944
+ }
1945
+ for (const snapshotId of snapshotIds) {
1946
+ assertDuckLakeSnapshotId2(snapshotId);
1947
+ }
1948
+ const snapshotIdList = snapshotIds.join(", ");
1949
+ const tableIdList = tableIds.map((tableId) => tableId.toString()).join(", ");
1950
+ const ducklakeDataFile = duckLakeMetadataTableName(this.options, "ducklake_data_file");
1951
+ const ducklakeDeleteFile = duckLakeMetadataTableName(this.options, "ducklake_delete_file");
1952
+ const rows = await runtime.query(`
1953
+ WITH file_changes AS (
1954
+ SELECT begin_snapshot AS snapshot_id, table_id, false AS is_delete
1955
+ FROM ${ducklakeDataFile}
1956
+ WHERE table_id IN (${tableIdList}) AND begin_snapshot IN (${snapshotIdList})
1957
+ UNION ALL
1958
+ SELECT end_snapshot, table_id, true
1959
+ FROM ${ducklakeDataFile}
1960
+ WHERE table_id IN (${tableIdList}) AND end_snapshot IN (${snapshotIdList})
1961
+ UNION ALL
1962
+ SELECT begin_snapshot, table_id, true
1963
+ FROM ${ducklakeDeleteFile}
1964
+ WHERE table_id IN (${tableIdList}) AND begin_snapshot IN (${snapshotIdList})
1965
+ UNION ALL
1966
+ SELECT end_snapshot, table_id, true
1967
+ FROM ${ducklakeDeleteFile}
1968
+ WHERE table_id IN (${tableIdList}) AND end_snapshot IN (${snapshotIdList})
1969
+ )
1970
+ SELECT snapshot_id, table_id, bool_or(is_delete) AS has_delete
1971
+ FROM file_changes
1972
+ GROUP BY snapshot_id, table_id
1973
+ `);
1974
+ for (const row of rows) {
1975
+ const key = fileChangeKey(String(getBigIntLike(row, "snapshot_id")), getBigIntLike(row, "table_id"));
1976
+ flags.set(key, { hasFileChange: true, hasFileDeleteChange: getBoolean(row, "has_delete") });
1977
+ }
1978
+ return flags;
1979
+ }
1980
+ assertNoMetadataConflict(snapshotId, datasetId, metadata) {
1981
+ if (metadata !== undefined && metadata.datasetId !== datasetId) {
1982
+ throw new LakeStorageError15(`[SixbDuckLake] DuckLake snapshot '${snapshotId}' changed dataset '${datasetId}' but Sixb commit metadata references dataset '${metadata.datasetId}'.`);
1983
+ }
1984
+ }
1985
+ async getLatestVersionForDefinition(runtime, dataset) {
1986
+ const tableRef = await resolveDatasetTableRef(this.options, runtime, dataset.id);
1987
+ return tableRef ? this.getLatestVersionForTableRef(runtime, tableRef) : null;
1988
+ }
1989
+ async getLatestVersionRefForDefinition(runtime, dataset) {
1990
+ const summary = await this.getLatestVersionSummaryForDefinition(runtime, dataset);
1991
+ return summary ? { datasetId: summary.datasetId, versionId: summary.versionId } : null;
1992
+ }
1993
+ async getLatestVersionSummaryForDefinition(runtime, dataset) {
1994
+ const tableRef = await resolveDatasetTableRef(this.options, runtime, dataset.id);
1995
+ if (!tableRef) {
1996
+ return null;
1997
+ }
1998
+ const row = await this.getLatestSnapshotRowForTableRef(runtime, tableRef);
1999
+ if (!row) {
2000
+ return null;
2001
+ }
2002
+ return {
2003
+ datasetId: tableRef.datasetId,
2004
+ versionId: toVersionId(row.snapshotId),
2005
+ ...row.metadata?.rowCount !== undefined ? { rowCount: row.metadata.rowCount } : {}
2006
+ };
2007
+ }
2008
+ async getVersionForSnapshot(runtime, dataset, snapshotId) {
2009
+ assertDuckLakeSnapshotId2(snapshotId);
2010
+ const tableRef = await resolveDatasetTableRef(this.options, runtime, dataset.id);
2011
+ return tableRef ? this.getVersionForTableRef(runtime, tableRef, snapshotId) : null;
2012
+ }
2013
+ async getVersionForTableRef(runtime, tableRef, snapshotId) {
2014
+ const match = await this.getSnapshotRowForTableRef(runtime, tableRef, snapshotId, {
2015
+ includeParent: true
2016
+ });
2017
+ return match ? this.snapshotToVersion(runtime, tableRef, match) : null;
2018
+ }
2019
+ async getVersionRefForSnapshot(runtime, dataset, snapshotId) {
2020
+ assertDuckLakeSnapshotId2(snapshotId);
2021
+ const tableRef = await resolveDatasetTableRef(this.options, runtime, dataset.id);
2022
+ if (!tableRef) {
2023
+ return null;
2024
+ }
2025
+ const row = await this.getSnapshotRowForTableRef(runtime, tableRef, snapshotId, {
2026
+ includeParent: false
2027
+ });
2028
+ return row ? { datasetId: tableRef.datasetId, versionId: toVersionId(row.snapshotId) } : null;
2029
+ }
2030
+ async listVersionsForDefinition(runtime, dataset, limit) {
2031
+ const tableRef = await resolveDatasetTableRef(this.options, runtime, dataset.id);
2032
+ return tableRef ? this.listVersionsForTableRef(runtime, tableRef, limit) : [];
2033
+ }
2034
+ async listVersionsForTableRef(runtime, tableRef, limit) {
2035
+ const limitedRows = await this.getSnapshotRowsForTableRef(runtime, tableRef, limit);
2036
+ const versions = [];
2037
+ for (const row of limitedRows) {
2038
+ versions.push(await this.snapshotToVersion(runtime, tableRef, row));
2039
+ }
2040
+ return versions;
2041
+ }
2042
+ async getLatestVersionForTableRef(runtime, tableRef) {
2043
+ const [latest] = await this.listVersionsForTableRef(runtime, tableRef, 1);
2044
+ return latest ?? null;
2045
+ }
2046
+ async getLatestSnapshotRowForTableRef(runtime, tableRef) {
2047
+ const [row] = await this.collectVisibleSnapshotRows(runtime, {
2048
+ datasetId: tableRef.datasetId,
2049
+ tableId: tableRef.tableId,
2050
+ visibleRowLimit: 1
2051
+ });
2052
+ return row ?? null;
2053
+ }
2054
+ async getSnapshotRowForTableRef(runtime, tableRef, snapshotId, options) {
2055
+ assertDuckLakeSnapshotId2(snapshotId);
2056
+ const [row] = await this.collectVisibleSnapshotRows(runtime, {
2057
+ datasetId: tableRef.datasetId,
2058
+ tableId: tableRef.tableId,
2059
+ exactSnapshotId: snapshotId,
2060
+ visibleRowLimit: 1
2061
+ });
2062
+ if (!row) {
2063
+ return null;
2064
+ }
2065
+ if (!options.includeParent || row.mode !== "append" && row.mode !== "schema") {
2066
+ return row;
2067
+ }
2068
+ const [parent] = await this.collectVisibleSnapshotRows(runtime, {
2069
+ datasetId: tableRef.datasetId,
2070
+ tableId: tableRef.tableId,
2071
+ beforeSnapshotId: snapshotId,
2072
+ visibleRowLimit: 1
2073
+ });
2074
+ return parent ? { ...row, parentSnapshotId: parent.snapshotId } : row;
2075
+ }
2076
+ async getSnapshotRowsForTableRef(runtime, tableRef, limit) {
2077
+ const visibleLimit = limit === undefined ? undefined : Math.max(0, limit);
2078
+ if (visibleLimit === 0) {
2079
+ return [];
2080
+ }
2081
+ const rows = await this.collectVisibleSnapshotRows(runtime, {
2082
+ datasetId: tableRef.datasetId,
2083
+ tableId: tableRef.tableId,
2084
+ visibleRowLimit: visibleLimit === undefined ? undefined : visibleLimit + 1
2085
+ });
2086
+ const rowsWithParents = this.withParentSnapshotIds(rows);
2087
+ return visibleLimit === undefined ? rowsWithParents : rowsWithParents.slice(0, visibleLimit);
2088
+ }
2089
+ async collectVisibleSnapshotRows(runtime, input) {
2090
+ if (input.visibleRowLimit !== undefined && input.visibleRowLimit <= 0) {
2091
+ return [];
2092
+ }
2093
+ const snapshots = [];
2094
+ let beforeSnapshotId = input.beforeSnapshotId;
2095
+ while (input.visibleRowLimit === undefined || snapshots.length < input.visibleRowLimit) {
2096
+ const candidates = await this.querySnapshotCandidates(runtime, {
2097
+ tableId: input.tableId,
2098
+ exactSnapshotId: input.exactSnapshotId,
2099
+ beforeSnapshotId,
2100
+ limit: input.exactSnapshotId === undefined ? SNAPSHOT_ROW_BATCH_SIZE : 1
2101
+ });
2102
+ if (candidates.length === 0) {
2103
+ break;
2104
+ }
2105
+ for (const candidate of candidates) {
2106
+ const snapshot = this.candidateToSnapshotRow(input.datasetId, input.tableId, candidate);
2107
+ if (snapshot) {
2108
+ snapshots.push(snapshot);
2109
+ }
2110
+ if (input.visibleRowLimit !== undefined && snapshots.length >= input.visibleRowLimit) {
2111
+ break;
2112
+ }
2113
+ }
2114
+ if (input.exactSnapshotId !== undefined || candidates.length < SNAPSHOT_ROW_BATCH_SIZE) {
2115
+ break;
2116
+ }
2117
+ beforeSnapshotId = candidates[candidates.length - 1]?.snapshotId;
2118
+ }
2119
+ return snapshots;
2120
+ }
2121
+ async querySnapshotCandidates(runtime, input) {
2122
+ if (input.limit !== undefined && input.limit <= 0) {
2123
+ return [];
2124
+ }
2125
+ const where = [];
2126
+ if (input.exactSnapshotId !== undefined) {
2127
+ assertDuckLakeSnapshotId2(input.exactSnapshotId);
2128
+ where.push(`snapshot.snapshot_id = ${input.exactSnapshotId}`);
2129
+ }
2130
+ if (input.beforeSnapshotId !== undefined) {
2131
+ assertDuckLakeSnapshotId2(input.beforeSnapshotId);
2132
+ where.push(`snapshot.snapshot_id < ${input.beforeSnapshotId}`);
2133
+ }
2134
+ const whereSql = where.length === 0 ? "" : `WHERE ${where.join(" AND ")}`;
2135
+ const limitSql = input.limit === undefined ? "" : `LIMIT ${Math.max(0, Math.trunc(input.limit))}`;
2136
+ const ducklakeSnapshot = duckLakeMetadataTableName(this.options, "ducklake_snapshot");
2137
+ const ducklakeSnapshotChanges = duckLakeMetadataTableName(this.options, "ducklake_snapshot_changes");
2138
+ const ducklakeDataFile = duckLakeMetadataTableName(this.options, "ducklake_data_file");
2139
+ const ducklakeDeleteFile = duckLakeMetadataTableName(this.options, "ducklake_delete_file");
2140
+ const rows = await runtime.query(`
2141
+ WITH candidate_snapshots AS (
2142
+ -- Recent DuckLake snapshots to inspect; Sixb visibility is filtered later.
2143
+ SELECT
2144
+ snapshot.snapshot_id,
2145
+ snapshot.snapshot_time,
2146
+ changes.changes_made,
2147
+ changes.commit_extra_info
2148
+ FROM ${ducklakeSnapshot} snapshot
2149
+ JOIN ${ducklakeSnapshotChanges} changes
2150
+ ON changes.snapshot_id = snapshot.snapshot_id
2151
+ ${whereSql}
2152
+ ORDER BY snapshot.snapshot_id DESC
2153
+ ${limitSql}
2154
+ ),
2155
+ file_changes AS (
2156
+ -- File metadata tells whether this table changed in each candidate snapshot.
2157
+ SELECT begin_snapshot AS snapshot_id, false AS is_delete_change
2158
+ FROM ${ducklakeDataFile}
2159
+ WHERE table_id = ${input.tableId}
2160
+ AND begin_snapshot IN (SELECT snapshot_id FROM candidate_snapshots)
2161
+ UNION ALL
2162
+ SELECT end_snapshot AS snapshot_id, true AS is_delete_change
2163
+ FROM ${ducklakeDataFile}
2164
+ WHERE table_id = ${input.tableId}
2165
+ AND end_snapshot IN (SELECT snapshot_id FROM candidate_snapshots)
2166
+ UNION ALL
2167
+ SELECT begin_snapshot AS snapshot_id, true AS is_delete_change
2168
+ FROM ${ducklakeDeleteFile}
2169
+ WHERE table_id = ${input.tableId}
2170
+ AND begin_snapshot IN (SELECT snapshot_id FROM candidate_snapshots)
2171
+ UNION ALL
2172
+ SELECT end_snapshot AS snapshot_id, true AS is_delete_change
2173
+ FROM ${ducklakeDeleteFile}
2174
+ WHERE table_id = ${input.tableId}
2175
+ AND end_snapshot IN (SELECT snapshot_id FROM candidate_snapshots)
2176
+ ),
2177
+ file_changes_by_snapshot AS (
2178
+ -- Collapse file-level changes into one row per snapshot.
2179
+ SELECT
2180
+ snapshot_id,
2181
+ count(*) > 0 AS has_file_change,
2182
+ count(*) FILTER (WHERE is_delete_change) > 0 AS has_file_delete_change
2183
+ FROM file_changes
2184
+ GROUP BY snapshot_id
2185
+ )
2186
+ SELECT
2187
+ -- Keep metadata-only candidates; Sixb filters them with commit_extra_info.
2188
+ candidate.snapshot_id,
2189
+ candidate.snapshot_time,
2190
+ candidate.changes_made,
2191
+ candidate.commit_extra_info,
2192
+ coalesce(file_changes.has_file_change, false) AS has_file_change,
2193
+ coalesce(file_changes.has_file_delete_change, false) AS has_file_delete_change
2194
+ FROM candidate_snapshots candidate
2195
+ LEFT JOIN file_changes_by_snapshot file_changes
2196
+ ON file_changes.snapshot_id = candidate.snapshot_id
2197
+ ORDER BY candidate.snapshot_id DESC
2198
+ `);
2199
+ return rows.map((row) => {
2200
+ const metadata = parseCommitMetadata(row.commit_extra_info);
2201
+ return {
2202
+ snapshotId: String(getBigIntLike(row, "snapshot_id")),
2203
+ createdAt: getDate(row, "snapshot_time"),
2204
+ changesMade: getString(row, "changes_made"),
2205
+ hasFileChange: getBoolean(row, "has_file_change"),
2206
+ hasFileDeleteChange: getBoolean(row, "has_file_delete_change"),
2207
+ ...metadata !== undefined ? { metadata } : {}
2208
+ };
2209
+ });
2210
+ }
2211
+ candidateToSnapshotRow(datasetId, tableId, candidate) {
2212
+ const inlineChange = parseInlineDataChange(candidate.changesMade, tableId);
2213
+ const hasDataChange = inlineChange.hasDataChange || candidate.hasFileChange;
2214
+ const hasDeleteChange = inlineChange.hasDeleteChange || candidate.hasFileDeleteChange;
2215
+ if (!hasDataChange) {
2216
+ const metadata = candidate.metadata;
2217
+ if (!metadata || metadata.datasetId !== datasetId) {
2218
+ return null;
2219
+ }
2220
+ return {
2221
+ snapshotId: candidate.snapshotId,
2222
+ tableId,
2223
+ createdAt: candidate.createdAt,
2224
+ mode: metadata.mode ?? "schema",
2225
+ metadata
2226
+ };
2227
+ }
2228
+ this.assertNoMetadataConflict(candidate.snapshotId, datasetId, candidate.metadata);
2229
+ return {
2230
+ snapshotId: candidate.snapshotId,
2231
+ tableId,
2232
+ createdAt: candidate.createdAt,
2233
+ mode: candidate.metadata?.mode ?? (hasDeleteChange ? "snapshot" : "append"),
2234
+ ...candidate.metadata !== undefined ? { metadata: candidate.metadata } : {}
2235
+ };
2236
+ }
2237
+ async snapshotToVersion(runtime, tableRef, row) {
2238
+ const schemaAtSnapshot = await this.datasets.getDatasetSchemaAtSnapshot(runtime, tableRef.datasetId, tableRef.tableName, tableRef.tableId, row.snapshotId);
2239
+ return {
2240
+ datasetId: tableRef.datasetId,
2241
+ versionId: toVersionId(row.snapshotId),
2242
+ parentVersionId: row.parentSnapshotId === undefined ? undefined : toVersionId(row.parentSnapshotId),
2243
+ mode: row.mode,
2244
+ createdAt: row.createdAt,
2245
+ schema: schemaAtSnapshot,
2246
+ producer: row.metadata?.producer,
2247
+ inputs: row.metadata?.inputs,
2248
+ ...row.metadata?.rowCount !== undefined ? { rowCount: row.metadata.rowCount } : {}
2249
+ };
2250
+ }
2251
+ withParentSnapshotIds(snapshots) {
2252
+ return snapshots.map((snapshot, index) => {
2253
+ const parentSnapshotId = snapshot.mode === "append" || snapshot.mode === "schema" ? snapshots[index + 1]?.snapshotId : undefined;
2254
+ return parentSnapshotId === undefined ? snapshot : { ...snapshot, parentSnapshotId };
2255
+ });
2256
+ }
2257
+ }
2258
+ function assertDuckLakeSnapshotId2(snapshotId) {
2259
+ if (!/^\d+$/.test(snapshotId)) {
2260
+ throw new LakeStorageError15(`[SixbDuckLake] Invalid DuckLake snapshot id '${snapshotId}'.`);
2261
+ }
2262
+ }
2263
+ function fileChangeKey(snapshotId, tableId) {
2264
+ return `${snapshotId}|${tableId}`;
2265
+ }
2266
+
2267
+ // src/internal/ducklake-sql-executor.ts
2268
+ import { randomUUID as randomUUID3 } from "crypto";
2269
+ import {
2270
+ LakeStorageError as LakeStorageError18
2271
+ } from "@sixb/core/lake-storage";
2272
+
2273
+ // src/internal/dataset-row-commit.ts
2274
+ import { LakeStorageError as LakeStorageError16 } from "@sixb/core/lake-storage";
2275
+ function assertDatasetWriteMode(mode, operation) {
2276
+ if (mode !== "snapshot" && mode !== "append") {
2277
+ throw new LakeStorageError16(`[SixbDuckLake] Invalid ${operation} mode '${String(mode)}'. Expected 'snapshot' or 'append'.`);
2278
+ }
2279
+ }
2280
+ async function applyDatasetRowsFromRelation(input) {
2281
+ const tableName = encodeDatasetTableName(input.dataset.id);
2282
+ const table = qualifiedTableName(input.options, tableName);
2283
+ const columnsSql = datasetSchemaColumnNamesSql(input.dataset.schema);
2284
+ if (input.mode === "snapshot") {
2285
+ const source = await fingerprintRelation(input.runtime, input.sourceRelationSql, columnsSql);
2286
+ const target = await fingerprintRelation(input.runtime, table, columnsSql);
2287
+ if (sameFingerprint(source, target)) {
2288
+ return {
2289
+ dataChangeExpected: false,
2290
+ sourceRowCount: source.rowCount,
2291
+ resultingRowCount: { kind: "exact", value: target.rowCount }
2292
+ };
2293
+ }
2294
+ await input.runtime.run(`DELETE FROM ${table}`);
2295
+ await input.runtime.run(`INSERT INTO ${table} (${columnsSql}) SELECT ${columnsSql} FROM ${input.sourceRelationSql}`);
2296
+ return {
2297
+ dataChangeExpected: source.rowCount > 0 || target.rowCount > 0,
2298
+ sourceRowCount: source.rowCount,
2299
+ resultingRowCount: { kind: "exact", value: source.rowCount }
2300
+ };
2301
+ }
2302
+ const sourceRowCount = await countRows(input.runtime, input.sourceRelationSql);
2303
+ await input.runtime.run(`INSERT INTO ${table} (${columnsSql}) SELECT ${columnsSql} FROM ${input.sourceRelationSql}`);
2304
+ return {
2305
+ dataChangeExpected: sourceRowCount > 0,
2306
+ sourceRowCount,
2307
+ resultingRowCount: input.previousRowCount === undefined ? { kind: "unknown" } : { kind: "exact", value: input.previousRowCount + sourceRowCount }
2308
+ };
2309
+ }
2310
+ async function fingerprintRelation(runtime, relationSql, columnsSql) {
2311
+ const [row] = await runtime.query(`
2312
+ SELECT
2313
+ count(*) AS row_count,
2314
+ coalesce(sum(hash(${columnsSql})::HUGEINT), 0::HUGEINT) AS hash_sum,
2315
+ coalesce(bit_xor(hash('sixb:snapshot:v1', ${columnsSql})), 0::UBIGINT) AS hash_xor
2316
+ FROM ${relationSql}
2317
+ `);
2318
+ if (row === undefined) {
2319
+ return { rowCount: 0, hashSum: "0", hashXor: "0" };
2320
+ }
2321
+ return {
2322
+ rowCount: Number(getBigIntLike(row, "row_count")),
2323
+ hashSum: getBigIntLike(row, "hash_sum").toString(),
2324
+ hashXor: getBigIntLike(row, "hash_xor").toString()
2325
+ };
2326
+ }
2327
+ function sameFingerprint(left, right) {
2328
+ return left.rowCount === right.rowCount && left.hashSum === right.hashSum && left.hashXor === right.hashXor;
2329
+ }
2330
+ async function countRows(runtime, relationSql) {
2331
+ const [row] = await runtime.query(`SELECT count(*) AS row_count FROM ${relationSql}`);
2332
+ if (row === undefined) {
2333
+ return 0;
2334
+ }
2335
+ return Number(getBigIntLike(row, "row_count"));
2336
+ }
2337
+
2338
+ // src/internal/sql-transform-relations.ts
2339
+ import { randomUUID as randomUUID2 } from "crypto";
2340
+ import {
2341
+ LakeStorageError as LakeStorageError17
2342
+ } from "@sixb/core/lake-storage";
2343
+ var RELATION_PLACEHOLDER_PREFIX = "__sixb_sql_transform_relation_";
2344
+ var RELATION_PLACEHOLDER_PATTERN = /__sixb_sql_transform_relation_[0-9a-f]{32}_[0-9]+__/g;
2345
+
2346
+ class DuckLakeSqlTransformRelation {
2347
+ placeholder;
2348
+ constructor(placeholder) {
2349
+ this.placeholder = placeholder;
2350
+ }
2351
+ toString() {
2352
+ return this.placeholder;
2353
+ }
2354
+ }
2355
+ function renderDuckLakeSqlTransformSql(input) {
2356
+ const context = createDuckLakeSqlTransformRelationContext(input);
2357
+ const sql = input.sql(context.relations);
2358
+ if (typeof sql !== "string") {
2359
+ throw new LakeStorageError17("[SixbDuckLake] SQL transform body must return a SQL string.");
2360
+ }
2361
+ return context.render(sql);
2362
+ }
2363
+ function createDuckLakeSqlTransformRelationContext(input) {
2364
+ const token = randomUUID2().replaceAll("-", "");
2365
+ const relations = Object.create(null);
2366
+ const replacements = new Map;
2367
+ let index = 0;
2368
+ for (const [sourceName, source] of Object.entries(input.sources)) {
2369
+ const placeholder = `${RELATION_PLACEHOLDER_PREFIX}${token}_${index}__`;
2370
+ relations[sourceName] = Object.freeze(new DuckLakeSqlTransformRelation(placeholder));
2371
+ replacements.set(placeholder, renderDuckLakeSourceRelation(input.options, source));
2372
+ index += 1;
2373
+ }
2374
+ return {
2375
+ relations: Object.freeze(relations),
2376
+ render(sql) {
2377
+ const rendered = sql.replace(RELATION_PLACEHOLDER_PATTERN, (placeholder) => {
2378
+ const replacement = replacements.get(placeholder);
2379
+ if (replacement === undefined) {
2380
+ throwUnresolvedPlaceholder();
2381
+ }
2382
+ return replacement;
2383
+ });
2384
+ if (rendered.includes(RELATION_PLACEHOLDER_PREFIX)) {
2385
+ throwUnresolvedPlaceholder();
2386
+ }
2387
+ return rendered;
2388
+ }
2389
+ };
2390
+ }
2391
+ function renderDuckLakeSourceRelation(options, source) {
2392
+ const snapshotId = parseVersionId(source.versionId);
2393
+ const tableName = encodeDatasetTableName(source.datasetId);
2394
+ return `(SELECT * FROM ${qualifiedTableName(options, tableName)} AT (VERSION => ${snapshotId}))`;
2395
+ }
2396
+ function throwUnresolvedPlaceholder() {
2397
+ throw new LakeStorageError17("[SixbDuckLake] SQL transform contains an unresolved relation placeholder.");
2398
+ }
2399
+
2400
+ // src/internal/ducklake-sql-executor.ts
2401
+ var DEFAULT_PREVIEW_LIMIT = 100;
2402
+ var MAX_PREVIEW_LIMIT = 1000;
2403
+
2404
+ class DuckLakeSqlExecutor {
2405
+ options;
2406
+ connections;
2407
+ datasets;
2408
+ snapshots;
2409
+ writes;
2410
+ dialect = "duckdb";
2411
+ capabilities = {
2412
+ preview: true,
2413
+ supportsAppend: true,
2414
+ supportsSnapshot: true
2415
+ };
2416
+ constructor(options, connections, datasets, snapshots, writes) {
2417
+ this.options = options;
2418
+ this.connections = connections;
2419
+ this.datasets = datasets;
2420
+ this.snapshots = snapshots;
2421
+ this.writes = writes;
2422
+ }
2423
+ async* preview(input) {
2424
+ this.connections.assertOpen();
2425
+ const rows = await this.connections.withAttachedRuntime(async (runtime) => {
2426
+ const sources = await this.resolveSources(runtime, input.sources);
2427
+ const sql = renderDuckLakeSqlTransformSql({
2428
+ options: this.options,
2429
+ sources: sources.relations,
2430
+ sql: input.sql
2431
+ });
2432
+ return runtime.query(buildPreviewSql(sql, previewLimit(input.limit)));
2433
+ });
2434
+ for (const row of rows) {
2435
+ yield normalizePreviewRow(row);
2436
+ }
2437
+ }
2438
+ async execute(input) {
2439
+ this.connections.assertOpen();
2440
+ assertDatasetWriteMode(input.mode, "SQL transform");
2441
+ return this.writes.withCommitRuntime(async (runtime) => {
2442
+ const target = await this.resolveTargetDataset(runtime, input.target);
2443
+ const sources = await this.resolveSources(runtime, input.sources);
2444
+ const sql = renderDuckLakeSqlTransformSql({
2445
+ options: this.options,
2446
+ sources: sources.relations,
2447
+ sql: input.sql
2448
+ });
2449
+ await this.assertResultSchemaMatchesTarget(runtime, target, sql);
2450
+ return this.writes.commitDatasetVersionOnExclusiveRuntime(runtime, {
2451
+ dataset: target,
2452
+ mode: input.mode,
2453
+ expectedLatestVersionId: input.expectedLatestVersionId,
2454
+ commitMessage: input.commitMessage ?? `execute SQL transform for dataset ${target.id}`,
2455
+ producer: input.producer,
2456
+ inputs: sources.inputs,
2457
+ applyChanges: (runtime2, context) => this.applySqlTransform({
2458
+ runtime: runtime2,
2459
+ target,
2460
+ mode: input.mode,
2461
+ sql,
2462
+ previousRowCount: context.previousRowCount
2463
+ })
2464
+ });
2465
+ });
2466
+ }
2467
+ async resolveSources(runtime, sources) {
2468
+ const relations = Object.create(null);
2469
+ const inputs = [];
2470
+ for (const [sourceName, source] of Object.entries(sources)) {
2471
+ const definition = await this.resolveSourceDataset(runtime, source.dataset);
2472
+ const versionRef = await this.resolveSourceVersion(runtime, definition, source.versionId);
2473
+ relations[sourceName] = {
2474
+ datasetId: definition.id,
2475
+ versionId: versionRef.versionId
2476
+ };
2477
+ inputs.push({
2478
+ datasetId: definition.id,
2479
+ versionId: versionRef.versionId
2480
+ });
2481
+ }
2482
+ return {
2483
+ relations: Object.freeze(relations),
2484
+ inputs: Object.freeze(inputs)
2485
+ };
2486
+ }
2487
+ async resolveSourceDataset(runtime, dataset) {
2488
+ const definition = await this.datasets.getDatasetOnRuntime(runtime, dataset.id);
2489
+ if (!definition) {
2490
+ throw new LakeStorageError18(`[SixbDuckLake] Unknown SQL transform source dataset '${dataset.id}'.`);
2491
+ }
2492
+ return definition;
2493
+ }
2494
+ async resolveTargetDataset(runtime, dataset) {
2495
+ const definition = await this.datasets.getDatasetOnRuntime(runtime, dataset.id);
2496
+ if (!definition) {
2497
+ throw new LakeStorageError18(`[SixbDuckLake] Unknown SQL transform target dataset '${dataset.id}'.`);
2498
+ }
2499
+ this.datasets.assertSchema(definition);
2500
+ return definition;
2501
+ }
2502
+ async resolveSourceVersion(runtime, dataset, versionId) {
2503
+ if (versionId !== undefined) {
2504
+ const snapshotId = parseVersionId(versionId);
2505
+ const versionRef = await this.snapshots.getVersionRefForSnapshot(runtime, dataset, snapshotId);
2506
+ if (versionRef) {
2507
+ return versionRef;
2508
+ }
2509
+ throwNoCommittedSourceVersion(dataset.id);
2510
+ }
2511
+ const latestVersionRef = await this.snapshots.getLatestVersionRefForDefinition(runtime, dataset);
2512
+ if (latestVersionRef) {
2513
+ return latestVersionRef;
2514
+ }
2515
+ throwNoCommittedSourceVersion(dataset.id);
2516
+ }
2517
+ async assertResultSchemaMatchesTarget(runtime, target, sql) {
2518
+ const rows = await runtime.query(`DESCRIBE SELECT * FROM (${sql}) AS sixb_sql_transform_result_schema`);
2519
+ const actualColumns = rows.map((row) => resultColumnFromDescribeRow(row));
2520
+ const expectedColumns = target.schema.columns;
2521
+ if (actualColumns.length !== expectedColumns.length) {
2522
+ throwResultSchemaMismatch(target.id, `expected ${expectedColumns.length} columns, got ${actualColumns.length}`);
2523
+ }
2524
+ for (let index = 0;index < expectedColumns.length; index += 1) {
2525
+ const expected = expectedColumns[index];
2526
+ const actual = actualColumns[index];
2527
+ if (actual.name !== expected.name) {
2528
+ throwResultSchemaMismatch(target.id, `column ${index + 1} should be '${expected.name}', got '${actual.name}'`);
2529
+ }
2530
+ if (actual.type !== expected.type) {
2531
+ throwResultSchemaMismatch(target.id, `column '${expected.name}' should have type '${expected.type}', got '${actual.type}'`);
2532
+ }
2533
+ }
2534
+ }
2535
+ async applySqlTransform(input) {
2536
+ const tempTableName = `sixb_sql_transform_${randomUUID3().replaceAll("-", "")}`;
2537
+ const tempTable = quoteIdentifier(tempTableName);
2538
+ await input.runtime.run(`CREATE TEMP TABLE ${tempTable} AS SELECT * FROM (${input.sql}) AS sixb_sql_transform_result`);
2539
+ let result;
2540
+ try {
2541
+ result = await applyDatasetRowsFromRelation({
2542
+ options: this.options,
2543
+ runtime: input.runtime,
2544
+ dataset: input.target,
2545
+ mode: input.mode,
2546
+ sourceRelationSql: tempTable,
2547
+ previousRowCount: input.previousRowCount
2548
+ });
2549
+ } catch (error) {
2550
+ try {
2551
+ await input.runtime.run(`DROP TABLE IF EXISTS ${tempTable}`);
2552
+ } catch {}
2553
+ throw error;
2554
+ }
2555
+ await input.runtime.run(`DROP TABLE IF EXISTS ${tempTable}`);
2556
+ return result;
2557
+ }
2558
+ }
2559
+ function previewLimit(limit) {
2560
+ if (limit === undefined || !Number.isFinite(limit)) {
2561
+ return DEFAULT_PREVIEW_LIMIT;
2562
+ }
2563
+ return Math.min(Math.max(0, Math.trunc(limit)), MAX_PREVIEW_LIMIT);
2564
+ }
2565
+ function buildPreviewSql(sql, limit) {
2566
+ return `SELECT * FROM (${sql}) AS sixb_sql_transform_preview LIMIT ${limit}`;
2567
+ }
2568
+ function resultColumnFromDescribeRow(row) {
2569
+ const name = getString(row, "column_name");
2570
+ const type = duckDbTypeToDatasetColumnType(getString(row, "column_type"));
2571
+ return { name, type };
2572
+ }
2573
+ function normalizePreviewRow(row) {
2574
+ const normalized = {};
2575
+ for (const [key, value] of Object.entries(row)) {
2576
+ normalized[key] = normalizePreviewValue(value);
2577
+ }
2578
+ return normalized;
2579
+ }
2580
+ function normalizePreviewValue(value) {
2581
+ if (typeof value === "bigint") {
2582
+ return value.toString();
2583
+ }
2584
+ if (value instanceof Date) {
2585
+ return value.toISOString();
2586
+ }
2587
+ if (Array.isArray(value)) {
2588
+ return value.map((item) => normalizePreviewValue(item));
2589
+ }
2590
+ if (typeof value === "object" && value !== null) {
2591
+ const normalized = {};
2592
+ for (const [key, child] of Object.entries(value)) {
2593
+ normalized[key] = normalizePreviewValue(child);
2594
+ }
2595
+ return normalized;
2596
+ }
2597
+ return value;
2598
+ }
2599
+ function throwNoCommittedSourceVersion(datasetId) {
2600
+ throw new LakeStorageError18(`[SixbDuckLake] No committed version found for SQL transform source dataset '${datasetId}'.`);
2601
+ }
2602
+ function throwResultSchemaMismatch(datasetId, detail) {
2603
+ throw new LakeStorageError18(`[SixbDuckLake] SQL transform result schema does not match target dataset '${datasetId}': ${detail}.`);
2604
+ }
2605
+
2606
+ // src/internal/ducklake-write-coordinator.ts
2607
+ import { randomUUID as randomUUID5 } from "crypto";
2608
+ import { LakeStorageError as LakeStorageError20 } from "@sixb/core/lake-storage";
2609
+
2610
+ // src/internal/ducklake-write-session.ts
2611
+ import { randomUUID as randomUUID4 } from "crypto";
2612
+ import { getDatasetRowValidationError } from "@sixb/core";
2613
+ import { LakeStorageError as LakeStorageError19 } from "@sixb/core/lake-storage";
2614
+
2615
+ // src/internal/row-appender.ts
2616
+ function appendDatasetRow(appender, schema, row) {
2617
+ for (const column of schema.columns) {
2618
+ appendColumnValue(appender, column, row[column.name] ?? null);
2619
+ }
2620
+ appender.endRow();
2621
+ }
2622
+ function appendColumnValue(appender, column, value) {
2623
+ if (value === null || value === undefined) {
2624
+ appender.appendNull();
2625
+ return;
2626
+ }
2627
+ switch (column.type) {
2628
+ case "string":
2629
+ appender.appendVarchar(value);
2630
+ return;
2631
+ case "boolean":
2632
+ appender.appendBoolean(value);
2633
+ return;
2634
+ case "int64":
2635
+ appender.appendBigInt(typeof value === "bigint" ? value : BigInt(value));
2636
+ return;
2637
+ case "float64":
2638
+ appender.appendDouble(value);
2639
+ return;
2640
+ case "decimal":
2641
+ appender.appendVarchar(normalizeDuckDbDecimalValue(value, column.name));
2642
+ return;
2643
+ case "date":
2644
+ appender.appendVarchar(value instanceof Date ? value.toISOString().slice(0, 10) : String(value));
2645
+ return;
2646
+ case "timestamp":
2647
+ appender.appendVarchar(value instanceof Date ? value.toISOString() : String(value));
2648
+ return;
2649
+ case "json":
2650
+ appender.appendVarchar(JSON.stringify(value));
2651
+ return;
2652
+ case "fileRef":
2653
+ appender.appendStruct(fileRefToDuckDbStruct(value));
2654
+ return;
2655
+ }
2656
+ }
2657
+ function fileRefToDuckDbStruct(fileRef) {
2658
+ return {
2659
+ blobId: fileRef.blobId,
2660
+ digest: fileRef.digest,
2661
+ sizeBytes: BigInt(fileRef.sizeBytes),
2662
+ fileName: fileRef.fileName ?? null,
2663
+ mediaType: fileRef.mediaType ?? null,
2664
+ logicalPath: fileRef.logicalPath ?? null
2665
+ };
2666
+ }
2667
+
2668
+ // src/internal/ducklake-write-session.ts
2669
+ var WRITE_ROW_BATCH_SIZE = 1000;
2670
+ async function createDuckLakeWriteSession(input) {
2671
+ const stagingTableName = `sixb_write_${randomUUID4().replaceAll("-", "")}`;
2672
+ await input.runtime.run(`CREATE TEMP TABLE ${quoteIdentifier(stagingTableName)} (${datasetSchemaToDuckDbColumnsSql(input.write.dataset.schema)})`);
2673
+ return new DuckLakeWriteSession(input.commitWrite, input.runtime, input.write, stagingTableName);
2674
+ }
2675
+
2676
+ class DuckLakeWriteSession {
2677
+ commitWrite;
2678
+ runtime;
2679
+ input;
2680
+ stagingTableName;
2681
+ closed = false;
2682
+ cleanedUp = false;
2683
+ rowsWritten = 0;
2684
+ constructor(commitWrite, runtime, input, stagingTableName) {
2685
+ this.commitWrite = commitWrite;
2686
+ this.runtime = runtime;
2687
+ this.input = input;
2688
+ this.stagingTableName = stagingTableName;
2689
+ }
2690
+ async writeRows(rows) {
2691
+ this.assertOpen();
2692
+ let batch = [];
2693
+ for await (const row of rows) {
2694
+ const validationError = getDatasetRowValidationError(row, this.input.dataset);
2695
+ if (validationError) {
2696
+ throw new LakeStorageError19(`[SixbDuckLake] ${validationError}`);
2697
+ }
2698
+ batch.push(structuredClone(row));
2699
+ if (batch.length >= WRITE_ROW_BATCH_SIZE) {
2700
+ await this.appendBatchToStagingTable(batch);
2701
+ batch = [];
2702
+ }
2703
+ }
2704
+ if (batch.length > 0) {
2705
+ await this.appendBatchToStagingTable(batch);
2706
+ }
2707
+ }
2708
+ async appendBatchToStagingTable(batch) {
2709
+ await this.runtime.withAppender(this.stagingTableName, (appender) => {
2710
+ for (const row of batch) {
2711
+ appendDatasetRow(appender, this.input.dataset.schema, row);
2712
+ }
2713
+ });
2714
+ this.rowsWritten += batch.length;
2715
+ }
2716
+ async commit(input) {
2717
+ this.assertOpen();
2718
+ this.closed = true;
2719
+ let committed = false;
2720
+ try {
2721
+ const result = await this.commitWrite({
2722
+ write: this.input,
2723
+ commit: input,
2724
+ rowsWritten: this.rowsWritten,
2725
+ stagingTableName: this.stagingTableName
2726
+ });
2727
+ committed = true;
2728
+ await this.cleanup();
2729
+ return result;
2730
+ } catch (error) {
2731
+ if (!committed) {
2732
+ await this.cleanup();
2733
+ }
2734
+ throw error;
2735
+ }
2736
+ }
2737
+ async abort() {
2738
+ if (this.closed) {
2739
+ return;
2740
+ }
2741
+ this.closed = true;
2742
+ await this.cleanup();
2743
+ }
2744
+ assertOpen() {
2745
+ if (this.closed) {
2746
+ throw new LakeStorageError19("[SixbDuckLake] Write session is already closed.");
2747
+ }
2748
+ }
2749
+ async cleanup() {
2750
+ if (this.cleanedUp) {
2751
+ return;
2752
+ }
2753
+ this.cleanedUp = true;
2754
+ await this.dropStagingTable();
2755
+ }
2756
+ async dropStagingTable() {
2757
+ try {
2758
+ await this.runtime.run(`DROP TABLE IF EXISTS ${quoteIdentifier(this.stagingTableName)}`);
2759
+ } catch {}
2760
+ }
2761
+ }
2762
+
2763
+ // src/internal/ducklake-write-coordinator.ts
2764
+ class DuckLakeWriteCoordinator {
2765
+ options;
2766
+ connections;
2767
+ datasets;
2768
+ snapshots;
2769
+ constructor(options, connections, datasets, snapshots) {
2770
+ this.options = options;
2771
+ this.connections = connections;
2772
+ this.datasets = datasets;
2773
+ this.snapshots = snapshots;
2774
+ }
2775
+ async beginWrite(input) {
2776
+ this.connections.assertOpen();
2777
+ const definition = await this.datasets.getDataset(input.dataset.id);
2778
+ if (!definition) {
2779
+ throw new LakeStorageError20(`[SixbDuckLake] Unknown dataset '${input.dataset.id}'.`);
2780
+ }
2781
+ this.datasets.assertSchema(definition);
2782
+ const mode = input.mode ?? "snapshot";
2783
+ assertDatasetWriteMode(mode, "write");
2784
+ const runtime = await this.connections.stagingRuntime();
2785
+ return createDuckLakeWriteSession({
2786
+ commitWrite: (options) => this.commitWrite(options),
2787
+ runtime,
2788
+ write: {
2789
+ ...input,
2790
+ dataset: definition,
2791
+ mode
2792
+ }
2793
+ });
2794
+ }
2795
+ async commitWrite(input) {
2796
+ const definition = await this.datasets.getDataset(input.write.dataset.id);
2797
+ if (!definition) {
2798
+ throw new LakeStorageError20(`[SixbDuckLake] Unknown dataset '${input.write.dataset.id}'.`);
2799
+ }
2800
+ const mode = input.write.mode ?? "snapshot";
2801
+ assertDatasetWriteMode(mode, "write");
2802
+ return this.commitDatasetVersion({
2803
+ dataset: definition,
2804
+ mode,
2805
+ expectedLatestVersionId: input.commit?.expectedLatestVersionId,
2806
+ commitMessage: input.commit?.commitMessage ?? `write dataset ${input.write.dataset.id}`,
2807
+ producer: input.write.producer,
2808
+ inputs: input.write.inputs,
2809
+ applyChanges: (runtime, context) => this.applyStagedRows({
2810
+ ...input,
2811
+ runtime,
2812
+ previousRowCount: context.previousRowCount
2813
+ })
2814
+ });
2815
+ }
2816
+ async commitDatasetVersion(input) {
2817
+ return this.withCommitRuntime((runtime) => this.commitDatasetVersionOnExclusiveRuntime(runtime, input));
2818
+ }
2819
+ async withCommitRuntime(run) {
2820
+ return withCatalogCommitLock(this.options, async () => {
2821
+ return this.connections.withExclusiveAttached(run);
2822
+ });
2823
+ }
2824
+ async commitDatasetVersionOnExclusiveRuntime(runtime, input) {
2825
+ const runtimeInput = { ...input, runtime };
2826
+ return this.withGuardedRetrySetting(runtimeInput, () => this.commitDatasetVersionUnlocked(runtimeInput));
2827
+ }
2828
+ async withGuardedRetrySetting(input, run) {
2829
+ if (input.expectedLatestVersionId === undefined) {
2830
+ return run();
2831
+ }
2832
+ await input.runtime.run("SET ducklake_max_retry_count = 0");
2833
+ let outcome;
2834
+ try {
2835
+ outcome = { kind: "success", value: await run() };
2836
+ } catch (error) {
2837
+ outcome = { kind: "error", error };
2838
+ }
2839
+ try {
2840
+ await input.runtime.run("RESET ducklake_max_retry_count");
2841
+ } catch {
2842
+ this.connections.poisonRuntime();
2843
+ }
2844
+ if (outcome.kind === "error") {
2845
+ throw outcome.error;
2846
+ }
2847
+ return outcome.value;
2848
+ }
2849
+ async commitDatasetVersionUnlocked(input) {
2850
+ const previousWriteSnapshotId = await this.lastCommittedSnapshotId(input.runtime);
2851
+ await input.runtime.run("BEGIN TRANSACTION");
2852
+ let committed = false;
2853
+ try {
2854
+ const latestVersion = await this.latestVersionForCommit(input);
2855
+ if (input.expectedLatestVersionId !== undefined) {
2856
+ this.assertExpectedLatestVersion({
2857
+ datasetId: input.dataset.id,
2858
+ expectedLatestVersionId: input.expectedLatestVersionId,
2859
+ actualLatestVersionId: latestVersion?.versionId ?? null
2860
+ });
2861
+ }
2862
+ const changeResult = await input.applyChanges(input.runtime, {
2863
+ previousRowCount: latestVersion?.rowCount
2864
+ });
2865
+ const commitId = randomUUID5();
2866
+ await this.setCommitMetadata(input, commitId, commitRowCount(input, changeResult, latestVersion));
2867
+ await input.runtime.run("COMMIT");
2868
+ committed = true;
2869
+ const committedWriteSnapshotId = await this.lastCommittedSnapshotId(input.runtime);
2870
+ try {
2871
+ return await this.versionForCommittedWrite({
2872
+ ...input,
2873
+ commitId,
2874
+ previousWriteSnapshotId,
2875
+ committedWriteSnapshotId,
2876
+ changeResult
2877
+ });
2878
+ } finally {
2879
+ await this.connections.detachLocalCatalogAfterCommit(input.runtime);
2880
+ }
2881
+ } catch (error) {
2882
+ if (!committed) {
2883
+ await this.rollbackTransaction(input.runtime);
2884
+ }
2885
+ throw error;
2886
+ }
2887
+ }
2888
+ async versionForCommittedWrite(input) {
2889
+ const ownSnapshotId = await this.findOwnSnapshotId(input.runtime, input.dataset.id, input.commitId, input.previousWriteSnapshotId, input.committedWriteSnapshotId);
2890
+ if (ownSnapshotId !== null) {
2891
+ const version = await this.snapshots.getVersionForSnapshot(input.runtime, input.dataset, ownSnapshotId);
2892
+ if (version) {
2893
+ return { ...version, outcome: "created" };
2894
+ }
2895
+ throw new LakeStorageError20(`[SixbDuckLake] DuckLake committed snapshot '${ownSnapshotId}' for dataset '${input.dataset.id}', but Sixb could not hydrate it as a dataset version.`);
2896
+ }
2897
+ if (!input.changeResult.dataChangeExpected) {
2898
+ return {
2899
+ ...await this.versionForNoOpCommit(input.runtime, input.dataset),
2900
+ outcome: "unchanged"
2901
+ };
2902
+ }
2903
+ if (input.committedWriteSnapshotId === input.previousWriteSnapshotId) {
2904
+ throw new LakeStorageError20(`[SixbDuckLake] DuckLake commit for dataset '${input.dataset.id}' completed without producing a snapshot for this non-empty write.`);
2905
+ }
2906
+ throw new LakeStorageError20(`[SixbDuckLake] DuckLake commit for dataset '${input.dataset.id}' completed, but Sixb could not find a matching snapshot for this write.`);
2907
+ }
2908
+ async applyStagedRows(input) {
2909
+ const mode = input.write.mode ?? "snapshot";
2910
+ const result = await applyDatasetRowsFromRelation({
2911
+ options: this.options,
2912
+ runtime: input.runtime,
2913
+ dataset: input.write.dataset,
2914
+ mode,
2915
+ sourceRelationSql: quoteIdentifier(input.stagingTableName),
2916
+ previousRowCount: input.previousRowCount
2917
+ });
2918
+ if (result.sourceRowCount !== input.rowsWritten) {
2919
+ throw new LakeStorageError20(`[SixbDuckLake] Staged write for dataset '${input.write.dataset.id}' accepted ${input.rowsWritten} row(s), but DuckLake saw ${result.sourceRowCount} source row(s) at commit time.`);
2920
+ }
2921
+ return result;
2922
+ }
2923
+ async setCommitMetadata(input, commitId, rowCount) {
2924
+ const metadata = {
2925
+ kind: "datasetVersion",
2926
+ datasetId: input.dataset.id,
2927
+ commitId,
2928
+ mode: input.mode,
2929
+ producer: input.producer ? structuredClone(input.producer) : undefined,
2930
+ inputs: input.inputs ? structuredClone(input.inputs) : undefined,
2931
+ ...rowCount.kind === "exact" ? { rowCount: rowCount.value } : {}
2932
+ };
2933
+ await input.runtime.run(`CALL ${quoteIdentifier(duckLakeAlias(this.options))}.set_commit_message(${quoteSqlString("Sixb")}, ${quoteSqlString(input.commitMessage)}, extra_info => ${quoteSqlString(JSON.stringify({ sixb: metadata }))})`);
2934
+ }
2935
+ async versionForNoOpCommit(runtime, definition) {
2936
+ const latestVersion = await this.snapshots.getLatestVersionForDefinition(runtime, definition);
2937
+ if (latestVersion) {
2938
+ return latestVersion;
2939
+ }
2940
+ throw new LakeStorageError20(`[SixbDuckLake] No DuckLake changes were committed for dataset '${definition.id}', and no previous version exists.`);
2941
+ }
2942
+ async rollbackTransaction(runtime) {
2943
+ try {
2944
+ await runtime.run("ROLLBACK");
2945
+ } catch {}
2946
+ }
2947
+ async latestVersionForCommit(input) {
2948
+ if (input.expectedLatestVersionId === undefined && input.mode !== "append") {
2949
+ return null;
2950
+ }
2951
+ return this.snapshots.getLatestVersionSummaryForDefinition(input.runtime, input.dataset);
2952
+ }
2953
+ async lastCommittedSnapshotId(runtime) {
2954
+ const [row] = await runtime.query(`SELECT id FROM ${quoteIdentifier(duckLakeAlias(this.options))}.last_committed_snapshot()`);
2955
+ if (row === undefined || row.id === null || row.id === undefined) {
2956
+ return null;
2957
+ }
2958
+ return String(getBigIntLike(row, "id"));
2959
+ }
2960
+ async findSnapshotIdByCommitId(runtime, datasetId, commitId, snapshotId) {
2961
+ if (snapshotId !== undefined) {
2962
+ assertDuckLakeSnapshotId3(snapshotId);
2963
+ }
2964
+ const ducklakeSnapshotChanges = duckLakeMetadataTableName(this.options, "ducklake_snapshot_changes");
2965
+ const whereSql = snapshotId === undefined ? "" : `WHERE snapshot_id = ${snapshotId}`;
2966
+ const orderSql = snapshotId === undefined ? "ORDER BY snapshot_id DESC" : "";
2967
+ const rows = await runtime.query(`
2968
+ SELECT snapshot_id, commit_extra_info
2969
+ FROM ${ducklakeSnapshotChanges}
2970
+ ${whereSql}
2971
+ ${orderSql}
2972
+ `);
2973
+ for (const row of rows) {
2974
+ const metadata = parseCommitMetadata(row.commit_extra_info);
2975
+ if (metadata?.datasetId === datasetId && metadata.commitId === commitId) {
2976
+ return String(getBigIntLike(row, "snapshot_id"));
2977
+ }
2978
+ }
2979
+ return null;
2980
+ }
2981
+ async findOwnSnapshotId(runtime, datasetId, commitId, previousWriteSnapshotId, committedWriteSnapshotId) {
2982
+ if (committedWriteSnapshotId === null || committedWriteSnapshotId === previousWriteSnapshotId) {
2983
+ return null;
2984
+ }
2985
+ const exactMatch = await this.findSnapshotIdByCommitId(runtime, datasetId, commitId, committedWriteSnapshotId);
2986
+ if (exactMatch !== null) {
2987
+ return exactMatch;
2988
+ }
2989
+ return this.findSnapshotIdByCommitId(runtime, datasetId, commitId);
2990
+ }
2991
+ assertExpectedLatestVersion(input) {
2992
+ if (input.expectedLatestVersionId === undefined) {
2993
+ return;
2994
+ }
2995
+ if (input.actualLatestVersionId !== input.expectedLatestVersionId) {
2996
+ throw new LakeStorageError20(`[SixbDuckLake] Optimistic commit failed for dataset '${input.datasetId}': expected latest version '${input.expectedLatestVersionId}', found '${input.actualLatestVersionId ?? "none"}'.`);
2997
+ }
2998
+ }
2999
+ }
3000
+ function commitRowCount(input, changeResult, latestVersion) {
3001
+ if (input.mode === "append" && input.expectedLatestVersionId === undefined) {
3002
+ return { kind: "unknown" };
3003
+ }
3004
+ if (changeResult.resultingRowCount.kind === "exact") {
3005
+ return changeResult.resultingRowCount;
3006
+ }
3007
+ if (input.mode === "append" && latestVersion === null) {
3008
+ return { kind: "exact", value: changeResult.sourceRowCount };
3009
+ }
3010
+ return { kind: "unknown" };
3011
+ }
3012
+ function assertDuckLakeSnapshotId3(snapshotId) {
3013
+ if (!/^\d+$/.test(snapshotId)) {
3014
+ throw new LakeStorageError20(`[SixbDuckLake] Invalid DuckLake snapshot id '${snapshotId}'.`);
3015
+ }
3016
+ }
3017
+ var localCatalogCommitLocks = new Map;
3018
+ async function withCatalogCommitLock(options, run) {
3019
+ const key = localCatalogCoordinationKey(options);
3020
+ if (key === undefined) {
3021
+ return run();
3022
+ }
3023
+ const previous = localCatalogCommitLocks.get(key) ?? Promise.resolve();
3024
+ const ready = previous.catch(() => {});
3025
+ let release;
3026
+ const next = new Promise((resolve2) => {
3027
+ release = resolve2;
3028
+ });
3029
+ const current = ready.then(() => next);
3030
+ localCatalogCommitLocks.set(key, current);
3031
+ await ready;
3032
+ try {
3033
+ return await run();
3034
+ } finally {
3035
+ release();
3036
+ if (localCatalogCommitLocks.get(key) === current) {
3037
+ localCatalogCommitLocks.delete(key);
3038
+ }
3039
+ }
3040
+ }
3041
+
3042
+ // src/ducklake-storage.ts
3043
+ class DuckLakeStorage {
3044
+ standard = { id: "ducklake", version: "1.0" };
3045
+ sql;
3046
+ connections;
3047
+ datasets;
3048
+ maintenance;
3049
+ rows;
3050
+ snapshotReader;
3051
+ writes;
3052
+ constructor(options) {
3053
+ const normalizedOptions = {
3054
+ ...options,
3055
+ alias: options.alias ?? "sixb_lake",
3056
+ createIfNotExists: options.createIfNotExists ?? true,
3057
+ readOnly: options.readOnly ?? false,
3058
+ duckdb: {
3059
+ ...options.duckdb,
3060
+ path: options.duckdb?.path ?? ":memory:"
3061
+ }
3062
+ };
3063
+ this.connections = new DuckLakeConnectionManager(normalizedOptions);
3064
+ this.datasets = new DuckLakeDatasetCatalog(normalizedOptions, this.connections);
3065
+ this.maintenance = new DuckLakeMaintenance(normalizedOptions, this.connections);
3066
+ this.snapshotReader = new DuckLakeSnapshotReader(normalizedOptions, this.connections, this.datasets);
3067
+ this.rows = new DuckLakeRowReader(normalizedOptions, this.connections, this.snapshotReader);
3068
+ this.writes = new DuckLakeWriteCoordinator(normalizedOptions, this.connections, this.datasets, this.snapshotReader);
3069
+ this.sql = new DuckLakeSqlExecutor(normalizedOptions, this.connections, this.datasets, this.snapshotReader, this.writes);
3070
+ }
3071
+ async createDataset(definition) {
3072
+ return this.datasets.createDataset(definition);
3073
+ }
3074
+ async assertDatasetDefinitionsCompatible(definitions) {
3075
+ await this.datasets.assertDatasetDefinitionsCompatible(definitions);
3076
+ }
3077
+ async getDataset(datasetId) {
3078
+ return this.datasets.getDataset(datasetId);
3079
+ }
3080
+ async listDatasets() {
3081
+ return this.datasets.listDatasets();
3082
+ }
3083
+ async listDatasetCatalogState(datasetIds) {
3084
+ return this.snapshotReader.listDatasetCatalogState(datasetIds);
3085
+ }
3086
+ async listVersions(datasetId, limit) {
3087
+ return this.snapshotReader.listVersions(datasetId, limit);
3088
+ }
3089
+ async beginWrite(input) {
3090
+ return this.writes.beginWrite(input);
3091
+ }
3092
+ async getLatestVersion(datasetId) {
3093
+ return this.snapshotReader.getLatestVersion(datasetId);
3094
+ }
3095
+ async getVersion(datasetId, versionId) {
3096
+ return this.snapshotReader.getVersion(datasetId, versionId);
3097
+ }
3098
+ readRows(input) {
3099
+ return this.rows.readRows(input);
3100
+ }
3101
+ async runMaintenance(options) {
3102
+ return this.maintenance.runMaintenance(options);
3103
+ }
3104
+ async close() {
3105
+ await this.connections.close();
3106
+ }
3107
+ }
3108
+ export {
3109
+ DuckLakeStorage
3110
+ };
3111
+
3112
+ //# debugId=0B56C838FAF193D864756E2164756E21