@sixb/ducklake 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +490 -0
- package/dist/ducklake-storage.d.ts +42 -0
- package/dist/ducklake-storage.d.ts.map +1 -0
- package/dist/index.d.ts +3 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +3112 -0
- package/dist/index.js.map +34 -0
- package/dist/internal/catalog-key.d.ts +11 -0
- package/dist/internal/catalog-key.d.ts.map +1 -0
- package/dist/internal/dataset-row-commit.d.ts +26 -0
- package/dist/internal/dataset-row-commit.d.ts.map +1 -0
- package/dist/internal/duckdb-column-types.d.ts +45 -0
- package/dist/internal/duckdb-column-types.d.ts.map +1 -0
- package/dist/internal/duckdb-decimal.d.ts +6 -0
- package/dist/internal/duckdb-decimal.d.ts.map +1 -0
- package/dist/internal/duckdb-row.d.ts +8 -0
- package/dist/internal/duckdb-row.d.ts.map +1 -0
- package/dist/internal/duckdb-runtime.d.ts +63 -0
- package/dist/internal/duckdb-runtime.d.ts.map +1 -0
- package/dist/internal/duckdb-value-converter.d.ts +7 -0
- package/dist/internal/duckdb-value-converter.d.ts.map +1 -0
- package/dist/internal/ducklake-catalog-schema.d.ts +12 -0
- package/dist/internal/ducklake-catalog-schema.d.ts.map +1 -0
- package/dist/internal/ducklake-connection-manager.d.ts +67 -0
- package/dist/internal/ducklake-connection-manager.d.ts.map +1 -0
- package/dist/internal/ducklake-current-dataset-definitions.d.ts +10 -0
- package/dist/internal/ducklake-current-dataset-definitions.d.ts.map +1 -0
- package/dist/internal/ducklake-dataset-catalog.d.ts +33 -0
- package/dist/internal/ducklake-dataset-catalog.d.ts.map +1 -0
- package/dist/internal/ducklake-dataset-table-ref.d.ts +9 -0
- package/dist/internal/ducklake-dataset-table-ref.d.ts.map +1 -0
- package/dist/internal/ducklake-maintenance.d.ts +9 -0
- package/dist/internal/ducklake-maintenance.d.ts.map +1 -0
- package/dist/internal/ducklake-row-reader.d.ts +24 -0
- package/dist/internal/ducklake-row-reader.d.ts.map +1 -0
- package/dist/internal/ducklake-snapshot-reader.d.ts +73 -0
- package/dist/internal/ducklake-snapshot-reader.d.ts.map +1 -0
- package/dist/internal/ducklake-sql-executor.d.ts +33 -0
- package/dist/internal/ducklake-sql-executor.d.ts.map +1 -0
- package/dist/internal/ducklake-write-coordinator.d.ts +53 -0
- package/dist/internal/ducklake-write-coordinator.d.ts.map +1 -0
- package/dist/internal/ducklake-write-session.d.ts +17 -0
- package/dist/internal/ducklake-write-session.d.ts.map +1 -0
- package/dist/internal/names.d.ts +13 -0
- package/dist/internal/names.d.ts.map +1 -0
- package/dist/internal/row-appender.d.ts +7 -0
- package/dist/internal/row-appender.d.ts.map +1 -0
- package/dist/internal/schema.d.ts +35 -0
- package/dist/internal/schema.d.ts.map +1 -0
- package/dist/internal/sql-transform-relations.d.ts +13 -0
- package/dist/internal/sql-transform-relations.d.ts.map +1 -0
- package/dist/internal/sql.d.ts +62 -0
- package/dist/internal/sql.d.ts.map +1 -0
- package/dist/internal/versions.d.ts +22 -0
- package/dist/internal/versions.d.ts.map +1 -0
- package/dist/tsconfig.json +5 -0
- package/dist/types.d.ts +147 -0
- package/dist/types.d.ts.map +1 -0
- package/package.json +64 -0
- package/src/ducklake-storage.ts +130 -0
- package/src/index.ts +14 -0
- package/src/internal/catalog-key.ts +28 -0
- package/src/internal/dataset-row-commit.ts +128 -0
- package/src/internal/duckdb-column-types.ts +68 -0
- package/src/internal/duckdb-decimal.ts +36 -0
- package/src/internal/duckdb-row.ts +83 -0
- package/src/internal/duckdb-runtime.ts +347 -0
- package/src/internal/duckdb-value-converter.ts +27 -0
- package/src/internal/ducklake-catalog-schema.ts +116 -0
- package/src/internal/ducklake-connection-manager.ts +510 -0
- package/src/internal/ducklake-current-dataset-definitions.ts +228 -0
- package/src/internal/ducklake-dataset-catalog.ts +363 -0
- package/src/internal/ducklake-dataset-table-ref.ts +35 -0
- package/src/internal/ducklake-maintenance.ts +116 -0
- package/src/internal/ducklake-row-reader.ts +143 -0
- package/src/internal/ducklake-snapshot-reader.ts +826 -0
- package/src/internal/ducklake-sql-executor.ts +334 -0
- package/src/internal/ducklake-write-coordinator.ts +532 -0
- package/src/internal/ducklake-write-session.ts +170 -0
- package/src/internal/names.ts +111 -0
- package/src/internal/row-appender.ts +73 -0
- package/src/internal/schema.ts +149 -0
- package/src/internal/sql-transform-relations.ts +99 -0
- package/src/internal/sql.ts +440 -0
- package/src/internal/versions.ts +140 -0
- package/src/types.ts +159 -0
package/dist/index.js
ADDED
|
@@ -0,0 +1,3112 @@
|
|
|
1
|
+
// @bun
|
|
2
|
+
// src/internal/ducklake-connection-manager.ts
|
|
3
|
+
import { LakeStorageError as LakeStorageError4 } from "@sixb/core/lake-storage";
|
|
4
|
+
|
|
5
|
+
// src/internal/catalog-key.ts
|
|
6
|
+
import { resolve } from "path";
|
|
7
|
+
function localCatalogCoordinationKey(options) {
|
|
8
|
+
switch (options.catalog.type) {
|
|
9
|
+
case "duckdb":
|
|
10
|
+
case "sqlite":
|
|
11
|
+
if (options.catalog.path === ":memory:") {
|
|
12
|
+
return;
|
|
13
|
+
}
|
|
14
|
+
return `${options.catalog.type}:${resolve(options.catalog.path)}`;
|
|
15
|
+
case "custom":
|
|
16
|
+
return `custom:${options.catalog.uri}`;
|
|
17
|
+
case "postgres":
|
|
18
|
+
return;
|
|
19
|
+
}
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
// src/internal/duckdb-runtime.ts
|
|
23
|
+
import {
|
|
24
|
+
DuckDBInstance
|
|
25
|
+
} from "@duckdb/node-api";
|
|
26
|
+
import { LakeStorageError as LakeStorageError3 } from "@sixb/core/lake-storage";
|
|
27
|
+
|
|
28
|
+
// src/internal/duckdb-value-converter.ts
|
|
29
|
+
import {
|
|
30
|
+
DuckDBDecimalValue,
|
|
31
|
+
DuckDBTypeId,
|
|
32
|
+
JSDuckDBValueConverter
|
|
33
|
+
} from "@duckdb/node-api";
|
|
34
|
+
import { LakeStorageError } from "@sixb/core/lake-storage";
|
|
35
|
+
var sixbDuckDbValueConverter = (value, type, converter) => {
|
|
36
|
+
if (value === null) {
|
|
37
|
+
return null;
|
|
38
|
+
}
|
|
39
|
+
if (type.typeId === DuckDBTypeId.DECIMAL) {
|
|
40
|
+
if (!(value instanceof DuckDBDecimalValue)) {
|
|
41
|
+
throw new LakeStorageError("[SixbDuckLake] DuckDB returned an invalid DECIMAL value.");
|
|
42
|
+
}
|
|
43
|
+
return value.toString();
|
|
44
|
+
}
|
|
45
|
+
return JSDuckDBValueConverter(value, type, converter);
|
|
46
|
+
};
|
|
47
|
+
|
|
48
|
+
// src/internal/sql.ts
|
|
49
|
+
import { LakeStorageError as LakeStorageError2 } from "@sixb/core/lake-storage";
|
|
50
|
+
var DEFAULT_ALIAS = "sixb_lake";
|
|
51
|
+
function duckLakeAlias(options) {
|
|
52
|
+
return options.alias ?? DEFAULT_ALIAS;
|
|
53
|
+
}
|
|
54
|
+
function duckLakeMetadataCatalog(options) {
|
|
55
|
+
return `__ducklake_metadata_${duckLakeAlias(options)}`;
|
|
56
|
+
}
|
|
57
|
+
function buildConfigurePostgresMetadataPoolSql(options) {
|
|
58
|
+
const parameters = [
|
|
59
|
+
`catalog_name=${quoteSqlString(duckLakeMetadataCatalog(options))}`,
|
|
60
|
+
...postgresPoolParameters(options.postgresPool).map(({ name, value }) => `${name}=${value}`)
|
|
61
|
+
];
|
|
62
|
+
return `FROM postgres_configure_pool(${parameters.join(", ")})`;
|
|
63
|
+
}
|
|
64
|
+
function buildSetPostgresPoolSql(options) {
|
|
65
|
+
return postgresPoolParameters(options.postgresPool).map(({ name, value }) => `SET pg_pool_${name} = ${value}`);
|
|
66
|
+
}
|
|
67
|
+
function duckLakeMetadataTableName(options, tableName) {
|
|
68
|
+
const schemaName = options.catalog.type === "postgres" ? options.catalog.metadataSchema ?? "main" : "main";
|
|
69
|
+
return `${quoteIdentifier(duckLakeMetadataCatalog(options))}.${quoteIdentifier(schemaName)}.${quoteIdentifier(tableName)}`;
|
|
70
|
+
}
|
|
71
|
+
function quoteIdentifier(identifier) {
|
|
72
|
+
if (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(identifier)) {
|
|
73
|
+
throw new LakeStorageError2(`[SixbDuckLake] Invalid SQL identifier '${identifier}'.`);
|
|
74
|
+
}
|
|
75
|
+
return `"${identifier}"`;
|
|
76
|
+
}
|
|
77
|
+
function quoteSqlString(value) {
|
|
78
|
+
return `'${value.replaceAll("'", "''")}'`;
|
|
79
|
+
}
|
|
80
|
+
function quoteSqlStringList(values) {
|
|
81
|
+
return values.map((value) => quoteSqlString(value)).join(", ");
|
|
82
|
+
}
|
|
83
|
+
function qualifiedTableName(options, tableName) {
|
|
84
|
+
return `${quoteIdentifier(duckLakeAlias(options))}.main.${quoteIdentifier(tableName)}`;
|
|
85
|
+
}
|
|
86
|
+
function catalogUri(catalog) {
|
|
87
|
+
switch (catalog.type) {
|
|
88
|
+
case "duckdb":
|
|
89
|
+
return `ducklake:${catalog.path}`;
|
|
90
|
+
case "sqlite":
|
|
91
|
+
return `ducklake:sqlite:${catalog.path}`;
|
|
92
|
+
case "postgres":
|
|
93
|
+
return `ducklake:postgres:${postgresConnectionString(catalog)}`;
|
|
94
|
+
case "custom":
|
|
95
|
+
return `ducklake:${catalog.uri}`;
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
function postgresConnectionString(catalog) {
|
|
99
|
+
const parameters = [];
|
|
100
|
+
const names = new Set;
|
|
101
|
+
appendPostgresConnectionParameter(parameters, names, "dbname", catalog.database);
|
|
102
|
+
appendPostgresConnectionParameter(parameters, names, "host", catalog.host);
|
|
103
|
+
appendPostgresConnectionParameter(parameters, names, "port", catalog.port);
|
|
104
|
+
appendPostgresConnectionParameter(parameters, names, "user", catalog.user);
|
|
105
|
+
appendPostgresConnectionParameter(parameters, names, "password", catalog.password);
|
|
106
|
+
appendPostgresConnectionParameter(parameters, names, "sslmode", catalog.sslMode);
|
|
107
|
+
appendPostgresConnectionParameter(parameters, names, "application_name", catalog.applicationName);
|
|
108
|
+
appendPostgresConnectionParameter(parameters, names, "connect_timeout", catalog.connectTimeoutSeconds);
|
|
109
|
+
for (const [name, value] of Object.entries(catalog.parameters ?? {})) {
|
|
110
|
+
appendPostgresConnectionParameter(parameters, names, name, value);
|
|
111
|
+
}
|
|
112
|
+
return parameters.join(" ");
|
|
113
|
+
}
|
|
114
|
+
function catalogExtensions(catalog) {
|
|
115
|
+
switch (catalog.type) {
|
|
116
|
+
case "duckdb":
|
|
117
|
+
return [];
|
|
118
|
+
case "sqlite":
|
|
119
|
+
return ["sqlite"];
|
|
120
|
+
case "postgres":
|
|
121
|
+
return ["postgres"];
|
|
122
|
+
case "custom":
|
|
123
|
+
return catalog.extensions ?? [];
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
function requiredExtensions(options) {
|
|
127
|
+
const extensions = new Set(catalogExtensions(options.catalog));
|
|
128
|
+
for (const extension of storageExtensions(options.dataPath)) {
|
|
129
|
+
extensions.add(extension);
|
|
130
|
+
}
|
|
131
|
+
for (const secret of options.secrets ?? []) {
|
|
132
|
+
for (const extension of secretExtensions(secret)) {
|
|
133
|
+
extensions.add(extension);
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
return [...extensions];
|
|
137
|
+
}
|
|
138
|
+
function buildCreateSecretSql(secret) {
|
|
139
|
+
const persistence = secret.persistent ? "PERSISTENT " : "TEMPORARY ";
|
|
140
|
+
const nameSql = secret.name === undefined ? "" : `${quoteIdentifier(secret.name)} `;
|
|
141
|
+
return `CREATE OR REPLACE ${persistence}SECRET ${nameSql}(${secretParameters(secret).join(", ")})`;
|
|
142
|
+
}
|
|
143
|
+
function buildAttachSql(options) {
|
|
144
|
+
const parameters = [];
|
|
145
|
+
if (options.dataPath !== undefined) {
|
|
146
|
+
parameters.push(`DATA_PATH ${quoteSqlString(options.dataPath)}`);
|
|
147
|
+
}
|
|
148
|
+
parameters.push(`CREATE_IF_NOT_EXISTS ${options.createIfNotExists ?? true}`);
|
|
149
|
+
if (options.readOnly) {
|
|
150
|
+
parameters.push("READ_ONLY");
|
|
151
|
+
}
|
|
152
|
+
if (options.catalog.type === "postgres" && options.catalog.metadataSchema !== undefined) {
|
|
153
|
+
parameters.push(`METADATA_SCHEMA ${quoteSqlString(options.catalog.metadataSchema)}`);
|
|
154
|
+
}
|
|
155
|
+
if (options.catalog.type === "custom" && options.catalog.metadataParameters !== undefined && Object.keys(options.catalog.metadataParameters).length > 0) {
|
|
156
|
+
const entries = Object.entries(options.catalog.metadataParameters).map(([key, value]) => `${quoteSqlString(key)}: ${quoteSqlString(value)}`);
|
|
157
|
+
parameters.push(`METADATA_PARAMETERS MAP {${entries.join(", ")}}`);
|
|
158
|
+
}
|
|
159
|
+
return `ATTACH ${quoteSqlString(catalogUri(options.catalog))} AS ${quoteIdentifier(duckLakeAlias(options))} (${parameters.join(", ")})`;
|
|
160
|
+
}
|
|
161
|
+
function storageExtensions(path) {
|
|
162
|
+
if (path === undefined) {
|
|
163
|
+
return [];
|
|
164
|
+
}
|
|
165
|
+
switch (pathScheme(path)) {
|
|
166
|
+
case "s3":
|
|
167
|
+
case "r2":
|
|
168
|
+
case "gcs":
|
|
169
|
+
case "gs":
|
|
170
|
+
return ["httpfs"];
|
|
171
|
+
case "az":
|
|
172
|
+
case "azure":
|
|
173
|
+
case "abfss":
|
|
174
|
+
return ["azure"];
|
|
175
|
+
default:
|
|
176
|
+
return [];
|
|
177
|
+
}
|
|
178
|
+
}
|
|
179
|
+
function postgresPoolParameters(pool) {
|
|
180
|
+
const parameters = [
|
|
181
|
+
{ name: "enable_thread_local_cache", value: String(pool?.enableThreadLocalCache ?? false) }
|
|
182
|
+
];
|
|
183
|
+
appendPoolNumber(parameters, "max_connections", pool?.maxConnections);
|
|
184
|
+
appendPoolNumber(parameters, "wait_timeout_millis", pool?.waitTimeoutMillis);
|
|
185
|
+
appendPoolNumber(parameters, "max_lifetime_millis", pool?.maxLifetimeMillis);
|
|
186
|
+
appendPoolNumber(parameters, "idle_timeout_millis", pool?.idleTimeoutMillis);
|
|
187
|
+
if (pool?.enableReaperThread !== undefined) {
|
|
188
|
+
parameters.push({ name: "enable_reaper_thread", value: String(pool.enableReaperThread) });
|
|
189
|
+
}
|
|
190
|
+
if (pool?.healthCheckQuery !== undefined) {
|
|
191
|
+
parameters.push({ name: "health_check_query", value: quoteSqlString(pool.healthCheckQuery) });
|
|
192
|
+
}
|
|
193
|
+
return parameters;
|
|
194
|
+
}
|
|
195
|
+
function appendPoolNumber(parameters, name, value) {
|
|
196
|
+
if (value === undefined) {
|
|
197
|
+
return;
|
|
198
|
+
}
|
|
199
|
+
parameters.push({ name, value: String(value) });
|
|
200
|
+
}
|
|
201
|
+
function secretExtensions(secret) {
|
|
202
|
+
switch (secret.type) {
|
|
203
|
+
case "s3":
|
|
204
|
+
case "r2":
|
|
205
|
+
case "gcs":
|
|
206
|
+
return ["httpfs"];
|
|
207
|
+
case "azure":
|
|
208
|
+
return ["azure"];
|
|
209
|
+
}
|
|
210
|
+
}
|
|
211
|
+
function secretParameters(secret) {
|
|
212
|
+
switch (secret.type) {
|
|
213
|
+
case "s3":
|
|
214
|
+
return withCommonSecretParameters(secret, [
|
|
215
|
+
"TYPE s3",
|
|
216
|
+
...s3ApiSecretParameters(secret),
|
|
217
|
+
...optionalStringParameter("KMS_KEY_ID", secret.kmsKeyId),
|
|
218
|
+
...optionalBooleanParameter("REQUESTER_PAYS", secret.requesterPays)
|
|
219
|
+
]);
|
|
220
|
+
case "r2":
|
|
221
|
+
return withCommonSecretParameters(secret, [
|
|
222
|
+
"TYPE r2",
|
|
223
|
+
...s3ApiSecretParameters(secret),
|
|
224
|
+
...optionalStringParameter("ACCOUNT_ID", secret.accountId)
|
|
225
|
+
]);
|
|
226
|
+
case "gcs":
|
|
227
|
+
return withCommonSecretParameters(secret, ["TYPE gcs", ...s3ApiSecretParameters(secret)]);
|
|
228
|
+
case "azure":
|
|
229
|
+
return withCommonSecretParameters(secret, azureSecretParameters(secret));
|
|
230
|
+
}
|
|
231
|
+
}
|
|
232
|
+
function s3ApiSecretParameters(secret) {
|
|
233
|
+
return [
|
|
234
|
+
...optionalIdentifierParameter("PROVIDER", secret.provider),
|
|
235
|
+
...optionalStringParameter("KEY_ID", secret.keyId),
|
|
236
|
+
...optionalStringParameter("SECRET", secret.secret),
|
|
237
|
+
...optionalStringParameter("SESSION_TOKEN", secret.sessionToken),
|
|
238
|
+
...optionalStringParameter("REGION", secret.region),
|
|
239
|
+
...optionalStringParameter("ENDPOINT", secret.endpoint),
|
|
240
|
+
...optionalStringParameter("URL_STYLE", secret.urlStyle),
|
|
241
|
+
...optionalBooleanParameter("USE_SSL", secret.useSsl),
|
|
242
|
+
...optionalBooleanParameter("VERIFY_SSL", secret.verifySsl),
|
|
243
|
+
...optionalStringParameter("CHAIN", secret.chain),
|
|
244
|
+
...optionalStringParameter("PROFILE", secret.profile)
|
|
245
|
+
];
|
|
246
|
+
}
|
|
247
|
+
function azureSecretParameters(secret) {
|
|
248
|
+
if (secret.tenantId !== undefined || secret.clientId !== undefined || secret.clientSecret !== undefined) {
|
|
249
|
+
return [
|
|
250
|
+
"TYPE azure",
|
|
251
|
+
"PROVIDER service_principal",
|
|
252
|
+
...optionalStringParameter("TENANT_ID", secret.tenantId),
|
|
253
|
+
...optionalStringParameter("CLIENT_ID", secret.clientId),
|
|
254
|
+
...optionalStringParameter("CLIENT_SECRET", secret.clientSecret),
|
|
255
|
+
...optionalStringParameter("ACCOUNT_NAME", secret.accountName)
|
|
256
|
+
];
|
|
257
|
+
}
|
|
258
|
+
const connectionString = secret.connectionString ?? (secret.accountName !== undefined && secret.accountKey !== undefined ? `DefaultEndpointsProtocol=https;AccountName=${secret.accountName};AccountKey=${secret.accountKey};EndpointSuffix=core.windows.net` : undefined);
|
|
259
|
+
if (connectionString !== undefined) {
|
|
260
|
+
return ["TYPE azure", ...optionalStringParameter("CONNECTION_STRING", connectionString)];
|
|
261
|
+
}
|
|
262
|
+
return [
|
|
263
|
+
"TYPE azure",
|
|
264
|
+
"PROVIDER credential_chain",
|
|
265
|
+
...optionalStringParameter("ACCOUNT_NAME", secret.accountName)
|
|
266
|
+
];
|
|
267
|
+
}
|
|
268
|
+
function withCommonSecretParameters(secret, parameters) {
|
|
269
|
+
return [...parameters, ...optionalStringParameter("SCOPE", secret.scope)];
|
|
270
|
+
}
|
|
271
|
+
function optionalStringParameter(name, value) {
|
|
272
|
+
return value === undefined ? [] : [`${name} ${quoteSqlString(value)}`];
|
|
273
|
+
}
|
|
274
|
+
function optionalBooleanParameter(name, value) {
|
|
275
|
+
return value === undefined ? [] : [`${name} ${value}`];
|
|
276
|
+
}
|
|
277
|
+
function optionalIdentifierParameter(name, value) {
|
|
278
|
+
if (value === undefined) {
|
|
279
|
+
return [];
|
|
280
|
+
}
|
|
281
|
+
if (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(value)) {
|
|
282
|
+
throw new LakeStorageError2(`[SixbDuckLake] Invalid SQL identifier '${value}'.`);
|
|
283
|
+
}
|
|
284
|
+
return [`${name} ${value}`];
|
|
285
|
+
}
|
|
286
|
+
function appendPostgresConnectionParameter(parameters, names, name, value) {
|
|
287
|
+
if (value === undefined) {
|
|
288
|
+
return;
|
|
289
|
+
}
|
|
290
|
+
if (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(name)) {
|
|
291
|
+
throw new LakeStorageError2(`[SixbDuckLake] Invalid PostgreSQL catalog parameter '${name}'.`);
|
|
292
|
+
}
|
|
293
|
+
const normalizedName = name.toLowerCase();
|
|
294
|
+
if (names.has(normalizedName)) {
|
|
295
|
+
throw new LakeStorageError2(`[SixbDuckLake] Duplicate PostgreSQL catalog parameter '${name}'.`);
|
|
296
|
+
}
|
|
297
|
+
names.add(normalizedName);
|
|
298
|
+
parameters.push(`${name}=${quotePostgresConnectionValue(String(value))}`);
|
|
299
|
+
}
|
|
300
|
+
function quotePostgresConnectionValue(value) {
|
|
301
|
+
return `'${value.replaceAll("\\", "\\\\").replaceAll("'", "\\'")}'`;
|
|
302
|
+
}
|
|
303
|
+
function pathScheme(path) {
|
|
304
|
+
const match = /^([a-z][a-z0-9+.-]*):\/\//i.exec(path);
|
|
305
|
+
return match ? match[1].toLowerCase() : null;
|
|
306
|
+
}
|
|
307
|
+
|
|
308
|
+
// src/internal/duckdb-runtime.ts
|
|
309
|
+
class NodeDuckDbRuntime {
|
|
310
|
+
instance;
|
|
311
|
+
connection;
|
|
312
|
+
closing = false;
|
|
313
|
+
closed = false;
|
|
314
|
+
operations = Promise.resolve();
|
|
315
|
+
constructor(instance, connection) {
|
|
316
|
+
this.instance = instance;
|
|
317
|
+
this.connection = connection;
|
|
318
|
+
}
|
|
319
|
+
async run(sql, values) {
|
|
320
|
+
await this.enqueue(async () => {
|
|
321
|
+
await this.connection.run(sql, values === undefined ? undefined : [...values]);
|
|
322
|
+
});
|
|
323
|
+
}
|
|
324
|
+
async runStatements(statements) {
|
|
325
|
+
await this.enqueue(async () => {
|
|
326
|
+
for (const sql of statements) {
|
|
327
|
+
await this.connection.run(sql);
|
|
328
|
+
}
|
|
329
|
+
});
|
|
330
|
+
}
|
|
331
|
+
async query(sql, values) {
|
|
332
|
+
return this.enqueue(async () => {
|
|
333
|
+
const reader = await this.connection.runAndReadAll(sql, values === undefined ? undefined : [...values]);
|
|
334
|
+
return reader.convertRowObjects(sixbDuckDbValueConverter);
|
|
335
|
+
});
|
|
336
|
+
}
|
|
337
|
+
async* streamRows(sql, values) {
|
|
338
|
+
this.assertAcceptingOperations();
|
|
339
|
+
let releaseOperation;
|
|
340
|
+
const operationFinished = new Promise((resolve2) => {
|
|
341
|
+
releaseOperation = resolve2;
|
|
342
|
+
});
|
|
343
|
+
const waitForTurn = this.operations.then(() => {
|
|
344
|
+
this.assertNotClosed();
|
|
345
|
+
});
|
|
346
|
+
this.operations = waitForTurn.then(() => operationFinished).then(() => {}, () => {});
|
|
347
|
+
try {
|
|
348
|
+
await waitForTurn;
|
|
349
|
+
const result = await this.connection.stream(sql, values === undefined ? undefined : [...values]);
|
|
350
|
+
for await (const batch of result.yieldConvertedRowObjects(sixbDuckDbValueConverter)) {
|
|
351
|
+
for (const row of batch) {
|
|
352
|
+
yield row;
|
|
353
|
+
}
|
|
354
|
+
}
|
|
355
|
+
} finally {
|
|
356
|
+
releaseOperation?.();
|
|
357
|
+
}
|
|
358
|
+
}
|
|
359
|
+
async withAppender(tableName, useAppender) {
|
|
360
|
+
return this.enqueue(async () => {
|
|
361
|
+
const appender = await this.connection.createAppender(tableName);
|
|
362
|
+
try {
|
|
363
|
+
return await useAppender(new NodeDuckDbAppenderAdapter(appender));
|
|
364
|
+
} finally {
|
|
365
|
+
appender.closeSync();
|
|
366
|
+
}
|
|
367
|
+
});
|
|
368
|
+
}
|
|
369
|
+
async withExclusive(useRuntime) {
|
|
370
|
+
return this.enqueue(async () => {
|
|
371
|
+
return useRuntime(new NodeDuckDbExclusiveRuntime(this.connection));
|
|
372
|
+
});
|
|
373
|
+
}
|
|
374
|
+
async close() {
|
|
375
|
+
if (this.closed) {
|
|
376
|
+
return;
|
|
377
|
+
}
|
|
378
|
+
this.closing = true;
|
|
379
|
+
await this.operations.catch(() => {});
|
|
380
|
+
this.closed = true;
|
|
381
|
+
this.connection.closeSync();
|
|
382
|
+
this.instance.closeSync();
|
|
383
|
+
}
|
|
384
|
+
enqueue(operation) {
|
|
385
|
+
this.assertAcceptingOperations();
|
|
386
|
+
const result = this.operations.then(async () => {
|
|
387
|
+
this.assertNotClosed();
|
|
388
|
+
return operation();
|
|
389
|
+
});
|
|
390
|
+
this.operations = result.then(() => {}, () => {});
|
|
391
|
+
return result;
|
|
392
|
+
}
|
|
393
|
+
assertAcceptingOperations() {
|
|
394
|
+
if (this.closing || this.closed) {
|
|
395
|
+
throw new LakeStorageError3("[SixbDuckLake] DuckDB runtime is closed.");
|
|
396
|
+
}
|
|
397
|
+
}
|
|
398
|
+
assertNotClosed() {
|
|
399
|
+
if (this.closed) {
|
|
400
|
+
throw new LakeStorageError3("[SixbDuckLake] DuckDB runtime is closed.");
|
|
401
|
+
}
|
|
402
|
+
}
|
|
403
|
+
}
|
|
404
|
+
|
|
405
|
+
class NodeDuckDbExclusiveRuntime {
|
|
406
|
+
connection;
|
|
407
|
+
constructor(connection) {
|
|
408
|
+
this.connection = connection;
|
|
409
|
+
}
|
|
410
|
+
async run(sql, values) {
|
|
411
|
+
await this.connection.run(sql, values === undefined ? undefined : [...values]);
|
|
412
|
+
}
|
|
413
|
+
async runStatements(statements) {
|
|
414
|
+
for (const sql of statements) {
|
|
415
|
+
await this.connection.run(sql);
|
|
416
|
+
}
|
|
417
|
+
}
|
|
418
|
+
async query(sql, values) {
|
|
419
|
+
const reader = await this.connection.runAndReadAll(sql, values === undefined ? undefined : [...values]);
|
|
420
|
+
return reader.convertRowObjects(sixbDuckDbValueConverter);
|
|
421
|
+
}
|
|
422
|
+
async withAppender(tableName, useAppender) {
|
|
423
|
+
const appender = await this.connection.createAppender(tableName);
|
|
424
|
+
try {
|
|
425
|
+
return await useAppender(new NodeDuckDbAppenderAdapter(appender));
|
|
426
|
+
} finally {
|
|
427
|
+
appender.closeSync();
|
|
428
|
+
}
|
|
429
|
+
}
|
|
430
|
+
}
|
|
431
|
+
|
|
432
|
+
class NodeDuckDbAppenderAdapter {
|
|
433
|
+
appender;
|
|
434
|
+
constructor(appender) {
|
|
435
|
+
this.appender = appender;
|
|
436
|
+
}
|
|
437
|
+
appendNull() {
|
|
438
|
+
this.appender.appendNull();
|
|
439
|
+
}
|
|
440
|
+
appendBoolean(value) {
|
|
441
|
+
this.appender.appendBoolean(value);
|
|
442
|
+
}
|
|
443
|
+
appendVarchar(value) {
|
|
444
|
+
this.appender.appendVarchar(value);
|
|
445
|
+
}
|
|
446
|
+
appendBigInt(value) {
|
|
447
|
+
this.appender.appendBigInt(value);
|
|
448
|
+
}
|
|
449
|
+
appendDouble(value) {
|
|
450
|
+
this.appender.appendDouble(value);
|
|
451
|
+
}
|
|
452
|
+
appendStruct(value) {
|
|
453
|
+
this.appender.appendStruct(value);
|
|
454
|
+
}
|
|
455
|
+
endRow() {
|
|
456
|
+
this.appender.endRow();
|
|
457
|
+
}
|
|
458
|
+
}
|
|
459
|
+
async function createDuckDbRuntime(options = {}) {
|
|
460
|
+
const instance = await DuckDBInstance.create(options.path ?? ":memory:", options.config ?? {});
|
|
461
|
+
return new NodeDuckDbRuntime(instance, await instance.connect());
|
|
462
|
+
}
|
|
463
|
+
async function installDuckLakeExtensions(runtime, options) {
|
|
464
|
+
await runtime.run("INSTALL ducklake");
|
|
465
|
+
for (const extension of requiredExtensions(options)) {
|
|
466
|
+
await runtime.run(`INSTALL ${quoteIdentifier(extension)}`);
|
|
467
|
+
}
|
|
468
|
+
}
|
|
469
|
+
async function setupDuckLake(runtime, options, setupOptions = {}) {
|
|
470
|
+
if (setupOptions.installExtensions ?? true) {
|
|
471
|
+
await installDuckLakeExtensions(runtime, options);
|
|
472
|
+
}
|
|
473
|
+
await runtime.run("LOAD ducklake");
|
|
474
|
+
for (const extension of requiredExtensions(options)) {
|
|
475
|
+
await runtime.run(`LOAD ${quoteIdentifier(extension)}`);
|
|
476
|
+
}
|
|
477
|
+
for (const sql of options.setupSql ?? []) {
|
|
478
|
+
await runtime.run(sql);
|
|
479
|
+
}
|
|
480
|
+
if (options.catalog.type === "postgres") {
|
|
481
|
+
await runtime.runStatements(buildSetPostgresPoolSql(options));
|
|
482
|
+
}
|
|
483
|
+
for (const secret of options.secrets ?? []) {
|
|
484
|
+
await runtime.run(buildCreateSecretSql(secret));
|
|
485
|
+
}
|
|
486
|
+
if (setupOptions.attach ?? true) {
|
|
487
|
+
await runtime.run(buildAttachSql(options));
|
|
488
|
+
if (options.catalog.type === "postgres") {
|
|
489
|
+
await runtime.run(buildConfigurePostgresMetadataPoolSql(options));
|
|
490
|
+
}
|
|
491
|
+
}
|
|
492
|
+
}
|
|
493
|
+
|
|
494
|
+
// src/internal/ducklake-connection-manager.ts
|
|
495
|
+
class DuckLakeConnectionManager {
|
|
496
|
+
options;
|
|
497
|
+
runtimePromise;
|
|
498
|
+
installExtensionsPromise;
|
|
499
|
+
attachPromise;
|
|
500
|
+
refreshPromise;
|
|
501
|
+
localAttachmentLock = Promise.resolve();
|
|
502
|
+
observedLocalCatalogGeneration;
|
|
503
|
+
attached = false;
|
|
504
|
+
closed = false;
|
|
505
|
+
runtimePoisoned = false;
|
|
506
|
+
constructor(options) {
|
|
507
|
+
this.options = options;
|
|
508
|
+
this.observedLocalCatalogGeneration = currentLocalCatalogGeneration(options);
|
|
509
|
+
}
|
|
510
|
+
assertOpen() {
|
|
511
|
+
if (this.closed) {
|
|
512
|
+
throw new LakeStorageError4("[SixbDuckLake] DuckLakeStorage is closed.");
|
|
513
|
+
}
|
|
514
|
+
}
|
|
515
|
+
async runtime() {
|
|
516
|
+
this.assertOpen();
|
|
517
|
+
if (this.runtimePromise === undefined) {
|
|
518
|
+
const runtimePromise = this.createRuntime().catch((error) => {
|
|
519
|
+
if (this.runtimePromise === runtimePromise) {
|
|
520
|
+
this.runtimePromise = undefined;
|
|
521
|
+
}
|
|
522
|
+
throw error;
|
|
523
|
+
});
|
|
524
|
+
this.runtimePromise = runtimePromise;
|
|
525
|
+
}
|
|
526
|
+
return this.runtimePromise;
|
|
527
|
+
}
|
|
528
|
+
async attachedRuntime() {
|
|
529
|
+
const runtime = await this.runtime();
|
|
530
|
+
await this.ensureAttached(runtime);
|
|
531
|
+
return runtime;
|
|
532
|
+
}
|
|
533
|
+
async withAttachedRuntime(run) {
|
|
534
|
+
const lease = await this.acquireAttachedRuntime();
|
|
535
|
+
try {
|
|
536
|
+
return await run(lease.runtime);
|
|
537
|
+
} finally {
|
|
538
|
+
await lease.release();
|
|
539
|
+
}
|
|
540
|
+
}
|
|
541
|
+
async acquireAttachedRuntime() {
|
|
542
|
+
this.assertOpen();
|
|
543
|
+
const releaseLock = await this.acquireLocalAttachmentLock();
|
|
544
|
+
let released = false;
|
|
545
|
+
const release = async () => {
|
|
546
|
+
if (released) {
|
|
547
|
+
return;
|
|
548
|
+
}
|
|
549
|
+
released = true;
|
|
550
|
+
try {
|
|
551
|
+
if (this.runtimePoisoned) {
|
|
552
|
+
this.runtimePoisoned = false;
|
|
553
|
+
await this.discardRuntimeUnlocked();
|
|
554
|
+
} else {
|
|
555
|
+
await this.detachDuckDbCatalogAfterLease();
|
|
556
|
+
}
|
|
557
|
+
} finally {
|
|
558
|
+
releaseLock();
|
|
559
|
+
}
|
|
560
|
+
};
|
|
561
|
+
try {
|
|
562
|
+
await this.refreshForExternalChangesUnlocked();
|
|
563
|
+
const runtime = await this.attachedRuntime();
|
|
564
|
+
return { runtime, release };
|
|
565
|
+
} catch (error) {
|
|
566
|
+
await release();
|
|
567
|
+
throw error;
|
|
568
|
+
}
|
|
569
|
+
}
|
|
570
|
+
async withExclusiveAttached(run) {
|
|
571
|
+
const lease = await this.acquireAttachedRuntime();
|
|
572
|
+
try {
|
|
573
|
+
return await lease.runtime.withExclusive(run);
|
|
574
|
+
} finally {
|
|
575
|
+
await lease.release();
|
|
576
|
+
}
|
|
577
|
+
}
|
|
578
|
+
async stagingRuntime() {
|
|
579
|
+
const runtime = await this.runtime();
|
|
580
|
+
if (this.options.catalog.type !== "postgres") {
|
|
581
|
+
await this.detachRuntime();
|
|
582
|
+
}
|
|
583
|
+
return runtime;
|
|
584
|
+
}
|
|
585
|
+
async createRuntime() {
|
|
586
|
+
this.assertOpen();
|
|
587
|
+
const runtime = await createDuckDbRuntime(this.options.duckdb);
|
|
588
|
+
try {
|
|
589
|
+
await this.ensureExtensionsInstalled(runtime);
|
|
590
|
+
await setupDuckLake(runtime, this.options, {
|
|
591
|
+
attach: false,
|
|
592
|
+
installExtensions: false
|
|
593
|
+
});
|
|
594
|
+
await runtime.run("SET preserve_insertion_order = true");
|
|
595
|
+
return runtime;
|
|
596
|
+
} catch (error) {
|
|
597
|
+
await this.closeRuntime(runtime, { detach: false });
|
|
598
|
+
throw error;
|
|
599
|
+
}
|
|
600
|
+
}
|
|
601
|
+
async resetRuntime() {
|
|
602
|
+
const releaseLock = await this.acquireLocalAttachmentLock();
|
|
603
|
+
try {
|
|
604
|
+
await this.resetRuntimeUnlocked();
|
|
605
|
+
} finally {
|
|
606
|
+
releaseLock();
|
|
607
|
+
}
|
|
608
|
+
}
|
|
609
|
+
async resetRuntimeUnlocked() {
|
|
610
|
+
const runtimePromise = this.runtimePromise;
|
|
611
|
+
if (runtimePromise === undefined) {
|
|
612
|
+
return;
|
|
613
|
+
}
|
|
614
|
+
if (this.attachPromise !== undefined) {
|
|
615
|
+
await this.attachPromise;
|
|
616
|
+
}
|
|
617
|
+
if (!this.attached) {
|
|
618
|
+
return;
|
|
619
|
+
}
|
|
620
|
+
let refreshedPromise;
|
|
621
|
+
refreshedPromise = runtimePromise.then(async (runtime) => {
|
|
622
|
+
try {
|
|
623
|
+
await this.refreshAttachedRuntime(runtime);
|
|
624
|
+
this.attached = true;
|
|
625
|
+
return runtime;
|
|
626
|
+
} catch (error) {
|
|
627
|
+
this.attached = false;
|
|
628
|
+
await this.closeRuntime(runtime, { detach: false });
|
|
629
|
+
throw error;
|
|
630
|
+
}
|
|
631
|
+
}).catch((error) => {
|
|
632
|
+
if (this.runtimePromise === refreshedPromise) {
|
|
633
|
+
this.runtimePromise = undefined;
|
|
634
|
+
}
|
|
635
|
+
throw error;
|
|
636
|
+
});
|
|
637
|
+
this.attached = false;
|
|
638
|
+
this.runtimePromise = refreshedPromise;
|
|
639
|
+
await refreshedPromise;
|
|
640
|
+
}
|
|
641
|
+
async refreshForExternalChanges() {
|
|
642
|
+
if (this.options.catalog.type === "postgres") {
|
|
643
|
+
return;
|
|
644
|
+
}
|
|
645
|
+
if (!this.needsLocalCatalogRefresh()) {
|
|
646
|
+
return;
|
|
647
|
+
}
|
|
648
|
+
if (this.refreshPromise === undefined) {
|
|
649
|
+
let refreshPromise;
|
|
650
|
+
refreshPromise = this.withLocalAttachmentLock(() => this.refreshForExternalChangesUnlocked()).finally(() => {
|
|
651
|
+
if (this.refreshPromise === refreshPromise) {
|
|
652
|
+
this.refreshPromise = undefined;
|
|
653
|
+
}
|
|
654
|
+
});
|
|
655
|
+
this.refreshPromise = refreshPromise;
|
|
656
|
+
}
|
|
657
|
+
await this.refreshPromise;
|
|
658
|
+
}
|
|
659
|
+
markLocalCatalogChanged() {
|
|
660
|
+
if (this.options.catalog.type === "postgres") {
|
|
661
|
+
return;
|
|
662
|
+
}
|
|
663
|
+
this.observedLocalCatalogGeneration = incrementLocalCatalogGeneration(this.options);
|
|
664
|
+
}
|
|
665
|
+
async detachLocalCatalogAfterCommit(runtime) {
|
|
666
|
+
if (this.options.catalog.type === "postgres") {
|
|
667
|
+
return;
|
|
668
|
+
}
|
|
669
|
+
try {
|
|
670
|
+
await runtime.run(`DETACH ${quoteIdentifier(duckLakeAlias(this.options))}`);
|
|
671
|
+
this.attached = false;
|
|
672
|
+
} catch {} finally {
|
|
673
|
+
this.markLocalCatalogChanged();
|
|
674
|
+
}
|
|
675
|
+
}
|
|
676
|
+
async refreshForExternalChangesUnlocked() {
|
|
677
|
+
const generation = currentLocalCatalogGeneration(this.options);
|
|
678
|
+
if (this.observedLocalCatalogGeneration === generation) {
|
|
679
|
+
return;
|
|
680
|
+
}
|
|
681
|
+
await this.resetRuntimeUnlocked();
|
|
682
|
+
this.observedLocalCatalogGeneration = generation;
|
|
683
|
+
}
|
|
684
|
+
needsLocalCatalogRefresh() {
|
|
685
|
+
return this.observedLocalCatalogGeneration !== currentLocalCatalogGeneration(this.options);
|
|
686
|
+
}
|
|
687
|
+
async close() {
|
|
688
|
+
if (this.closed) {
|
|
689
|
+
return;
|
|
690
|
+
}
|
|
691
|
+
this.closed = true;
|
|
692
|
+
const runtimePromise = this.runtimePromise;
|
|
693
|
+
this.runtimePromise = undefined;
|
|
694
|
+
await this.closeRuntimePromise(runtimePromise, {
|
|
695
|
+
detach: this.attached && this.options.catalog.type !== "postgres"
|
|
696
|
+
});
|
|
697
|
+
this.attached = false;
|
|
698
|
+
}
|
|
699
|
+
async ensureAttached(runtime) {
|
|
700
|
+
this.assertOpen();
|
|
701
|
+
if (this.attached) {
|
|
702
|
+
return;
|
|
703
|
+
}
|
|
704
|
+
if (this.attachPromise === undefined) {
|
|
705
|
+
let attachPromise;
|
|
706
|
+
attachPromise = this.attachRuntime(runtime).then(() => {
|
|
707
|
+
this.attached = true;
|
|
708
|
+
}).catch(async (error) => {
|
|
709
|
+
if (this.attachPromise === attachPromise) {
|
|
710
|
+
this.runtimePromise = undefined;
|
|
711
|
+
this.attached = false;
|
|
712
|
+
}
|
|
713
|
+
await this.closeRuntime(runtime, { detach: false });
|
|
714
|
+
throw error;
|
|
715
|
+
}).finally(() => {
|
|
716
|
+
if (this.attachPromise === attachPromise) {
|
|
717
|
+
this.attachPromise = undefined;
|
|
718
|
+
}
|
|
719
|
+
});
|
|
720
|
+
this.attachPromise = attachPromise;
|
|
721
|
+
}
|
|
722
|
+
await this.attachPromise;
|
|
723
|
+
}
|
|
724
|
+
async ensureExtensionsInstalled(runtime) {
|
|
725
|
+
if (this.installExtensionsPromise === undefined) {
|
|
726
|
+
const installExtensionsPromise = installDuckLakeExtensions(runtime, this.options).catch((error) => {
|
|
727
|
+
if (this.installExtensionsPromise === installExtensionsPromise) {
|
|
728
|
+
this.installExtensionsPromise = undefined;
|
|
729
|
+
}
|
|
730
|
+
throw error;
|
|
731
|
+
});
|
|
732
|
+
this.installExtensionsPromise = installExtensionsPromise;
|
|
733
|
+
}
|
|
734
|
+
await this.installExtensionsPromise;
|
|
735
|
+
}
|
|
736
|
+
async attachRuntime(runtime) {
|
|
737
|
+
await runtime.runStatements([
|
|
738
|
+
buildAttachSql(this.options),
|
|
739
|
+
...this.options.catalog.type === "postgres" ? [buildConfigurePostgresMetadataPoolSql(this.options)] : []
|
|
740
|
+
]);
|
|
741
|
+
}
|
|
742
|
+
async refreshAttachedRuntime(runtime) {
|
|
743
|
+
await runtime.runStatements([
|
|
744
|
+
`DETACH ${quoteIdentifier(duckLakeAlias(this.options))}`,
|
|
745
|
+
buildAttachSql(this.options),
|
|
746
|
+
...this.options.catalog.type === "postgres" ? [buildConfigurePostgresMetadataPoolSql(this.options)] : []
|
|
747
|
+
]);
|
|
748
|
+
}
|
|
749
|
+
async detachRuntime() {
|
|
750
|
+
await this.withLocalAttachmentLock(() => this.detachRuntimeUnlocked());
|
|
751
|
+
}
|
|
752
|
+
poisonRuntime() {
|
|
753
|
+
this.runtimePoisoned = true;
|
|
754
|
+
}
|
|
755
|
+
async detachDuckDbCatalogAfterLease() {
|
|
756
|
+
if (this.options.catalog.type !== "duckdb" || !this.attached) {
|
|
757
|
+
return;
|
|
758
|
+
}
|
|
759
|
+
try {
|
|
760
|
+
await this.detachRuntimeUnlocked();
|
|
761
|
+
} catch {
|
|
762
|
+
await this.discardRuntimeUnlocked();
|
|
763
|
+
}
|
|
764
|
+
}
|
|
765
|
+
async discardRuntimeUnlocked() {
|
|
766
|
+
const runtimePromise = this.runtimePromise;
|
|
767
|
+
this.runtimePromise = undefined;
|
|
768
|
+
this.attached = false;
|
|
769
|
+
await this.closeRuntimePromise(runtimePromise, { detach: false }).catch(() => {});
|
|
770
|
+
}
|
|
771
|
+
async detachRuntimeUnlocked() {
|
|
772
|
+
if (this.attachPromise !== undefined) {
|
|
773
|
+
await this.attachPromise;
|
|
774
|
+
}
|
|
775
|
+
if (!this.attached) {
|
|
776
|
+
return;
|
|
777
|
+
}
|
|
778
|
+
const runtime = await this.runtime();
|
|
779
|
+
await runtime.run(`DETACH ${quoteIdentifier(duckLakeAlias(this.options))}`);
|
|
780
|
+
this.attached = false;
|
|
781
|
+
}
|
|
782
|
+
async withLocalAttachmentLock(run) {
|
|
783
|
+
const releaseLock = await this.acquireLocalAttachmentLock();
|
|
784
|
+
try {
|
|
785
|
+
return await run();
|
|
786
|
+
} finally {
|
|
787
|
+
releaseLock();
|
|
788
|
+
}
|
|
789
|
+
}
|
|
790
|
+
async acquireLocalAttachmentLock() {
|
|
791
|
+
if (this.options.catalog.type === "postgres") {
|
|
792
|
+
return () => {};
|
|
793
|
+
}
|
|
794
|
+
const previous = this.localAttachmentLock.catch(() => {});
|
|
795
|
+
let release;
|
|
796
|
+
const current = new Promise((resolve2) => {
|
|
797
|
+
release = resolve2;
|
|
798
|
+
});
|
|
799
|
+
this.localAttachmentLock = previous.then(() => current);
|
|
800
|
+
await previous;
|
|
801
|
+
let released = false;
|
|
802
|
+
return () => {
|
|
803
|
+
if (released) {
|
|
804
|
+
return;
|
|
805
|
+
}
|
|
806
|
+
released = true;
|
|
807
|
+
release();
|
|
808
|
+
};
|
|
809
|
+
}
|
|
810
|
+
async closeRuntimePromise(runtimePromise, options) {
|
|
811
|
+
if (runtimePromise === undefined) {
|
|
812
|
+
return;
|
|
813
|
+
}
|
|
814
|
+
let runtime;
|
|
815
|
+
try {
|
|
816
|
+
runtime = await runtimePromise;
|
|
817
|
+
} catch {
|
|
818
|
+
return;
|
|
819
|
+
}
|
|
820
|
+
await this.closeRuntime(runtime, options);
|
|
821
|
+
}
|
|
822
|
+
async closeRuntime(runtime, options) {
|
|
823
|
+
if (options.detach) {
|
|
824
|
+
try {
|
|
825
|
+
await runtime.run(`DETACH ${quoteIdentifier(duckLakeAlias(this.options))}`);
|
|
826
|
+
} catch {}
|
|
827
|
+
}
|
|
828
|
+
await runtime.close();
|
|
829
|
+
}
|
|
830
|
+
}
|
|
831
|
+
var localCatalogGenerations = new Map;
|
|
832
|
+
function currentLocalCatalogGeneration(options) {
|
|
833
|
+
const key = localCatalogCoordinationKey(options);
|
|
834
|
+
return key === undefined ? 0 : localCatalogGenerations.get(key) ?? 0;
|
|
835
|
+
}
|
|
836
|
+
function incrementLocalCatalogGeneration(options) {
|
|
837
|
+
const key = localCatalogCoordinationKey(options);
|
|
838
|
+
if (key === undefined) {
|
|
839
|
+
return 0;
|
|
840
|
+
}
|
|
841
|
+
const nextGeneration = (localCatalogGenerations.get(key) ?? 0) + 1;
|
|
842
|
+
localCatalogGenerations.set(key, nextGeneration);
|
|
843
|
+
return nextGeneration;
|
|
844
|
+
}
|
|
845
|
+
|
|
846
|
+
// src/internal/ducklake-dataset-catalog.ts
|
|
847
|
+
import { randomUUID } from "crypto";
|
|
848
|
+
import { LakeStorageError as LakeStorageError11, planDatasetDefinitionUpdate } from "@sixb/core/lake-storage";
|
|
849
|
+
|
|
850
|
+
// src/internal/duckdb-row.ts
|
|
851
|
+
import { LakeStorageError as LakeStorageError5 } from "@sixb/core/lake-storage";
|
|
852
|
+
function getString(row, key) {
|
|
853
|
+
const value = row[key];
|
|
854
|
+
if (typeof value !== "string") {
|
|
855
|
+
throw new LakeStorageError5(`[SixbDuckLake] Expected DuckDB column '${key}' to be a string, got ${typeof value}.`);
|
|
856
|
+
}
|
|
857
|
+
return value;
|
|
858
|
+
}
|
|
859
|
+
function getOptionalString(row, key) {
|
|
860
|
+
const value = row[key];
|
|
861
|
+
if (value === null || value === undefined) {
|
|
862
|
+
return;
|
|
863
|
+
}
|
|
864
|
+
if (typeof value !== "string") {
|
|
865
|
+
throw new LakeStorageError5(`[SixbDuckLake] Expected DuckDB column '${key}' to be a string, got ${typeof value}.`);
|
|
866
|
+
}
|
|
867
|
+
return value;
|
|
868
|
+
}
|
|
869
|
+
function getBoolean(row, key) {
|
|
870
|
+
const value = row[key];
|
|
871
|
+
if (typeof value !== "boolean") {
|
|
872
|
+
throw new LakeStorageError5(`[SixbDuckLake] Expected DuckDB column '${key}' to be a boolean, got ${typeof value}.`);
|
|
873
|
+
}
|
|
874
|
+
return value;
|
|
875
|
+
}
|
|
876
|
+
function getBigIntLike(row, key) {
|
|
877
|
+
const value = row[key];
|
|
878
|
+
if (typeof value === "bigint") {
|
|
879
|
+
return value;
|
|
880
|
+
}
|
|
881
|
+
if (typeof value === "number" && Number.isInteger(value)) {
|
|
882
|
+
return BigInt(value);
|
|
883
|
+
}
|
|
884
|
+
if (typeof value === "string" && /^-?\d+$/.test(value)) {
|
|
885
|
+
return BigInt(value);
|
|
886
|
+
}
|
|
887
|
+
throw new LakeStorageError5(`[SixbDuckLake] Expected DuckDB column '${key}' to be an integer, got ${typeof value}.`);
|
|
888
|
+
}
|
|
889
|
+
function getOptionalBigIntLike(row, key) {
|
|
890
|
+
const value = row[key];
|
|
891
|
+
if (value === null || value === undefined) {
|
|
892
|
+
return;
|
|
893
|
+
}
|
|
894
|
+
return getBigIntLike(row, key);
|
|
895
|
+
}
|
|
896
|
+
function getDate(row, key) {
|
|
897
|
+
const value = row[key];
|
|
898
|
+
if (value instanceof Date) {
|
|
899
|
+
return value;
|
|
900
|
+
}
|
|
901
|
+
if (typeof value === "string") {
|
|
902
|
+
return new Date(value);
|
|
903
|
+
}
|
|
904
|
+
throw new LakeStorageError5(`[SixbDuckLake] Expected DuckDB column '${key}' to be a date, got ${typeof value}.`);
|
|
905
|
+
}
|
|
906
|
+
|
|
907
|
+
// src/internal/ducklake-catalog-schema.ts
|
|
908
|
+
import { LakeStorageError as LakeStorageError8 } from "@sixb/core/lake-storage";
|
|
909
|
+
|
|
910
|
+
// src/internal/schema.ts
|
|
911
|
+
import { LakeStorageError as LakeStorageError7 } from "@sixb/core/lake-storage";
|
|
912
|
+
|
|
913
|
+
// src/internal/duckdb-column-types.ts
|
|
914
|
+
function normalizeDuckDbTypeSql(typeSql) {
|
|
915
|
+
return typeSql.trim().toUpperCase().replace(/\s+/g, " ").replace(/\s*([(),])\s*/g, "$1");
|
|
916
|
+
}
|
|
917
|
+
function defineDuckDbColumnType(sql, aliases = []) {
|
|
918
|
+
const acceptedTypes = new Set([sql, ...aliases].map(normalizeDuckDbTypeSql));
|
|
919
|
+
return {
|
|
920
|
+
sql,
|
|
921
|
+
matches: (typeSql) => acceptedTypes.has(normalizeDuckDbTypeSql(typeSql))
|
|
922
|
+
};
|
|
923
|
+
}
|
|
924
|
+
function defineDuckDbDecimalType(precision, scale) {
|
|
925
|
+
return {
|
|
926
|
+
...defineDuckDbColumnType(`DECIMAL(${precision}, ${scale})`),
|
|
927
|
+
precision,
|
|
928
|
+
scale
|
|
929
|
+
};
|
|
930
|
+
}
|
|
931
|
+
var DUCKDB_COLUMN_TYPES = {
|
|
932
|
+
string: defineDuckDbColumnType("VARCHAR"),
|
|
933
|
+
boolean: defineDuckDbColumnType("BOOLEAN"),
|
|
934
|
+
int64: defineDuckDbColumnType("BIGINT", ["INT64"]),
|
|
935
|
+
float64: defineDuckDbColumnType("DOUBLE", ["FLOAT64"]),
|
|
936
|
+
decimal: defineDuckDbDecimalType(38, 9),
|
|
937
|
+
date: defineDuckDbColumnType("DATE"),
|
|
938
|
+
timestamp: defineDuckDbColumnType("TIMESTAMPTZ", ["TIMESTAMP WITH TIME ZONE"]),
|
|
939
|
+
json: defineDuckDbColumnType("JSON"),
|
|
940
|
+
fileRef: defineDuckDbColumnType("STRUCT(blobId VARCHAR, digest VARCHAR, sizeBytes BIGINT, fileName VARCHAR, mediaType VARCHAR, logicalPath VARCHAR)")
|
|
941
|
+
};
|
|
942
|
+
function findDatasetColumnTypeForDuckDbSql(typeSql) {
|
|
943
|
+
for (const type of Object.keys(DUCKDB_COLUMN_TYPES)) {
|
|
944
|
+
if (DUCKDB_COLUMN_TYPES[type].matches(typeSql)) {
|
|
945
|
+
return type;
|
|
946
|
+
}
|
|
947
|
+
}
|
|
948
|
+
return;
|
|
949
|
+
}
|
|
950
|
+
|
|
951
|
+
// src/internal/duckdb-decimal.ts
|
|
952
|
+
import { normalizeDecimalValue } from "@sixb/core";
|
|
953
|
+
import { LakeStorageError as LakeStorageError6 } from "@sixb/core/lake-storage";
|
|
954
|
+
var decimalType = DUCKDB_COLUMN_TYPES.decimal;
|
|
955
|
+
var maxIntegerDigits = decimalType.precision - decimalType.scale;
|
|
956
|
+
function normalizeDuckDbDecimalValue(value, columnName) {
|
|
957
|
+
if (typeof value !== "string") {
|
|
958
|
+
throw new LakeStorageError6(`[SixbDuckLake] Decimal column '${columnName}' must be an exact decimal string.`);
|
|
959
|
+
}
|
|
960
|
+
let normalized;
|
|
961
|
+
try {
|
|
962
|
+
normalized = normalizeDecimalValue(value);
|
|
963
|
+
} catch {
|
|
964
|
+
throw new LakeStorageError6(`[SixbDuckLake] Decimal column '${columnName}' must be an exact decimal string.`);
|
|
965
|
+
}
|
|
966
|
+
const unsigned = normalized.startsWith("-") ? normalized.slice(1) : normalized;
|
|
967
|
+
const [integer, fraction = ""] = unsigned.split(".");
|
|
968
|
+
if (integer.length > maxIntegerDigits || fraction.length > decimalType.scale) {
|
|
969
|
+
throw new LakeStorageError6(`[SixbDuckLake] Decimal column '${columnName}' cannot be represented exactly as ${decimalType.sql}; expected at most ${maxIntegerDigits} integer digits and ${decimalType.scale} fractional digits.`);
|
|
970
|
+
}
|
|
971
|
+
return normalized;
|
|
972
|
+
}
|
|
973
|
+
|
|
974
|
+
// src/internal/schema.ts
|
|
975
|
+
function datasetColumnTypeToDuckDbSql(type) {
|
|
976
|
+
return DUCKDB_COLUMN_TYPES[type].sql;
|
|
977
|
+
}
|
|
978
|
+
function duckDbTypeToDatasetColumnType(typeSql) {
|
|
979
|
+
const type = findDatasetColumnTypeForDuckDbSql(typeSql);
|
|
980
|
+
if (type) {
|
|
981
|
+
return type;
|
|
982
|
+
}
|
|
983
|
+
throw new LakeStorageError7(`[SixbDuckLake] DuckDB column type '${typeSql}' cannot be mapped to a Sixb dataset column type.`);
|
|
984
|
+
}
|
|
985
|
+
function datasetColumnToDuckDbSql(column) {
|
|
986
|
+
const nullableSql = column.nullable ? "" : " NOT NULL";
|
|
987
|
+
return `${quoteIdentifier(column.name)} ${datasetColumnTypeToDuckDbSql(column.type)}${nullableSql}`;
|
|
988
|
+
}
|
|
989
|
+
function datasetSchemaToDuckDbColumnsSql(schema) {
|
|
990
|
+
return schema.columns.map((column) => datasetColumnToDuckDbSql(column)).join(", ");
|
|
991
|
+
}
|
|
992
|
+
function datasetSchemaColumnNamesSql(schema) {
|
|
993
|
+
return schema.columns.map((column) => quoteIdentifier(column.name)).join(", ");
|
|
994
|
+
}
|
|
995
|
+
function normalizeReadValue(value, column) {
|
|
996
|
+
if (value === null || value === undefined) {
|
|
997
|
+
return null;
|
|
998
|
+
}
|
|
999
|
+
switch (column.type) {
|
|
1000
|
+
case "int64":
|
|
1001
|
+
return typeof value === "bigint" ? value.toString() : value;
|
|
1002
|
+
case "decimal":
|
|
1003
|
+
return normalizeDuckDbDecimalValue(value, column.name);
|
|
1004
|
+
case "date":
|
|
1005
|
+
return value instanceof Date ? value.toISOString().slice(0, 10) : String(value);
|
|
1006
|
+
case "timestamp":
|
|
1007
|
+
return value instanceof Date ? value.toISOString() : String(value);
|
|
1008
|
+
case "json":
|
|
1009
|
+
return typeof value === "string" ? JSON.parse(value) : value;
|
|
1010
|
+
case "fileRef":
|
|
1011
|
+
return normalizeFileRef(value);
|
|
1012
|
+
default:
|
|
1013
|
+
return value;
|
|
1014
|
+
}
|
|
1015
|
+
}
|
|
1016
|
+
function normalizeFileRef(value) {
|
|
1017
|
+
if (typeof value !== "object" || value === null || Array.isArray(value)) {
|
|
1018
|
+
throw new LakeStorageError7("[SixbDuckLake] DuckDB fileRef value must be a struct.");
|
|
1019
|
+
}
|
|
1020
|
+
const record = value;
|
|
1021
|
+
return {
|
|
1022
|
+
blobId: getStringField(record, "blobId"),
|
|
1023
|
+
digest: getStringField(record, "digest"),
|
|
1024
|
+
sizeBytes: getIntegerField(record, "sizeBytes"),
|
|
1025
|
+
fileName: getOptionalStringField(record, "fileName"),
|
|
1026
|
+
mediaType: getOptionalStringField(record, "mediaType"),
|
|
1027
|
+
logicalPath: getOptionalStringField(record, "logicalPath")
|
|
1028
|
+
};
|
|
1029
|
+
}
|
|
1030
|
+
function getStringField(record, key) {
|
|
1031
|
+
const value = record[key];
|
|
1032
|
+
if (typeof value !== "string") {
|
|
1033
|
+
throw new LakeStorageError7(`[SixbDuckLake] DuckDB fileRef field '${key}' must be a string.`);
|
|
1034
|
+
}
|
|
1035
|
+
return value;
|
|
1036
|
+
}
|
|
1037
|
+
function getOptionalStringField(record, key) {
|
|
1038
|
+
const value = record[key];
|
|
1039
|
+
if (value === null || value === undefined) {
|
|
1040
|
+
return;
|
|
1041
|
+
}
|
|
1042
|
+
if (typeof value !== "string") {
|
|
1043
|
+
throw new LakeStorageError7(`[SixbDuckLake] DuckDB fileRef field '${key}' must be a string.`);
|
|
1044
|
+
}
|
|
1045
|
+
return value;
|
|
1046
|
+
}
|
|
1047
|
+
function getIntegerField(record, key) {
|
|
1048
|
+
const value = record[key];
|
|
1049
|
+
if (typeof value === "number" && Number.isInteger(value)) {
|
|
1050
|
+
return value;
|
|
1051
|
+
}
|
|
1052
|
+
if (typeof value === "bigint") {
|
|
1053
|
+
return Number(value);
|
|
1054
|
+
}
|
|
1055
|
+
throw new LakeStorageError7(`[SixbDuckLake] DuckDB fileRef field '${key}' must be an integer.`);
|
|
1056
|
+
}
|
|
1057
|
+
|
|
1058
|
+
// src/internal/ducklake-catalog-schema.ts
|
|
1059
|
+
var FILE_REF_STRUCT_CHILDREN = [
|
|
1060
|
+
{ name: "blobId", type: "string" },
|
|
1061
|
+
{ name: "digest", type: "string" },
|
|
1062
|
+
{ name: "sizeBytes", type: "int64" },
|
|
1063
|
+
{ name: "fileName", type: "string" },
|
|
1064
|
+
{ name: "mediaType", type: "string" },
|
|
1065
|
+
{ name: "logicalPath", type: "string" }
|
|
1066
|
+
];
|
|
1067
|
+
function duckLakeCatalogColumnsToDatasetSchema(tableName, rows) {
|
|
1068
|
+
const childrenByParentId = new Map;
|
|
1069
|
+
const topLevelColumns = [];
|
|
1070
|
+
for (const row of rows) {
|
|
1071
|
+
if (row.parentColumnId === undefined) {
|
|
1072
|
+
topLevelColumns.push(row);
|
|
1073
|
+
continue;
|
|
1074
|
+
}
|
|
1075
|
+
const children = childrenByParentId.get(row.parentColumnId) ?? [];
|
|
1076
|
+
children.push(row);
|
|
1077
|
+
childrenByParentId.set(row.parentColumnId, children);
|
|
1078
|
+
}
|
|
1079
|
+
return {
|
|
1080
|
+
columns: topLevelColumns.sort(compareDuckLakeCatalogColumnOrder).map((column) => duckLakeCatalogColumnToDefinition(tableName, column, childrenByParentId))
|
|
1081
|
+
};
|
|
1082
|
+
}
|
|
1083
|
+
function duckLakeCatalogColumnToDefinition(tableName, column, childrenByParentId) {
|
|
1084
|
+
const children = [...childrenByParentId.get(column.columnId) ?? []].sort(compareDuckLakeCatalogColumnOrder);
|
|
1085
|
+
return {
|
|
1086
|
+
name: column.columnName,
|
|
1087
|
+
type: duckLakeCatalogColumnTypeToDatasetType(tableName, column, children),
|
|
1088
|
+
...column.nullsAllowed ? { nullable: true } : {}
|
|
1089
|
+
};
|
|
1090
|
+
}
|
|
1091
|
+
function duckLakeCatalogColumnTypeToDatasetType(tableName, column, children) {
|
|
1092
|
+
if (children.length === 0 && column.columnType.toLowerCase() !== "struct") {
|
|
1093
|
+
return duckDbTypeToDatasetColumnType(column.columnType);
|
|
1094
|
+
}
|
|
1095
|
+
if (isFileRefStruct(children)) {
|
|
1096
|
+
return "fileRef";
|
|
1097
|
+
}
|
|
1098
|
+
throw new LakeStorageError8(`[SixbDuckLake] DuckDB column type '${formatDuckLakeCatalogColumnType(column, children)}' in dataset table '${tableName}' cannot be mapped to a Sixb dataset column type.`);
|
|
1099
|
+
}
|
|
1100
|
+
function compareDuckLakeCatalogColumnOrder(left, right) {
|
|
1101
|
+
return Number(left.columnOrder - right.columnOrder);
|
|
1102
|
+
}
|
|
1103
|
+
function isFileRefStruct(children) {
|
|
1104
|
+
if (children.length !== FILE_REF_STRUCT_CHILDREN.length) {
|
|
1105
|
+
return false;
|
|
1106
|
+
}
|
|
1107
|
+
return FILE_REF_STRUCT_CHILDREN.every((expected, index) => {
|
|
1108
|
+
const child = children[index];
|
|
1109
|
+
return child !== undefined && child.columnName === expected.name && duckDbTypeToDatasetColumnType(child.columnType) === expected.type;
|
|
1110
|
+
});
|
|
1111
|
+
}
|
|
1112
|
+
function formatDuckLakeCatalogColumnType(column, children) {
|
|
1113
|
+
if (children.length === 0) {
|
|
1114
|
+
return column.columnType;
|
|
1115
|
+
}
|
|
1116
|
+
return `STRUCT(${children.map((child) => `${child.columnName} ${child.columnType}`).join(", ")})`;
|
|
1117
|
+
}
|
|
1118
|
+
|
|
1119
|
+
// src/internal/ducklake-current-dataset-definitions.ts
|
|
1120
|
+
import { LakeStorageError as LakeStorageError10 } from "@sixb/core/lake-storage";
|
|
1121
|
+
|
|
1122
|
+
// src/internal/names.ts
|
|
1123
|
+
import { LakeStorageError as LakeStorageError9 } from "@sixb/core/lake-storage";
|
|
1124
|
+
var DATASET_TABLE_PREFIX = "sixb__ds__";
|
|
1125
|
+
var textEncoder = new TextEncoder;
|
|
1126
|
+
var textDecoder = new TextDecoder("utf-8", { fatal: true });
|
|
1127
|
+
function isInlineByte(byte) {
|
|
1128
|
+
return byte >= 97 && byte <= 122 || byte >= 48 && byte <= 57;
|
|
1129
|
+
}
|
|
1130
|
+
function isHexPair(value) {
|
|
1131
|
+
return /^[0-9a-f]{2}$/.test(value);
|
|
1132
|
+
}
|
|
1133
|
+
function encodeDatasetId(datasetId) {
|
|
1134
|
+
if (datasetId.trim().length === 0) {
|
|
1135
|
+
throw new LakeStorageError9("[SixbDuckLake] Dataset id must not be empty.");
|
|
1136
|
+
}
|
|
1137
|
+
let encoded = "";
|
|
1138
|
+
for (const byte of textEncoder.encode(datasetId)) {
|
|
1139
|
+
if (isInlineByte(byte)) {
|
|
1140
|
+
encoded += String.fromCharCode(byte);
|
|
1141
|
+
} else if (byte === 46) {
|
|
1142
|
+
encoded += "__";
|
|
1143
|
+
} else {
|
|
1144
|
+
encoded += `_${byte.toString(16).padStart(2, "0")}`;
|
|
1145
|
+
}
|
|
1146
|
+
}
|
|
1147
|
+
return encoded;
|
|
1148
|
+
}
|
|
1149
|
+
function decodeDatasetId(encoded) {
|
|
1150
|
+
if (encoded.length === 0) {
|
|
1151
|
+
return null;
|
|
1152
|
+
}
|
|
1153
|
+
const bytes = [];
|
|
1154
|
+
let index = 0;
|
|
1155
|
+
while (index < encoded.length) {
|
|
1156
|
+
const char = encoded[index];
|
|
1157
|
+
const byte = char.charCodeAt(0);
|
|
1158
|
+
if (isInlineByte(byte)) {
|
|
1159
|
+
bytes.push(byte);
|
|
1160
|
+
index += 1;
|
|
1161
|
+
continue;
|
|
1162
|
+
}
|
|
1163
|
+
if (char !== "_") {
|
|
1164
|
+
return null;
|
|
1165
|
+
}
|
|
1166
|
+
if (encoded[index + 1] === "_") {
|
|
1167
|
+
bytes.push(46);
|
|
1168
|
+
index += 2;
|
|
1169
|
+
continue;
|
|
1170
|
+
}
|
|
1171
|
+
const hex = encoded.slice(index + 1, index + 3);
|
|
1172
|
+
if (!isHexPair(hex)) {
|
|
1173
|
+
return null;
|
|
1174
|
+
}
|
|
1175
|
+
bytes.push(Number.parseInt(hex, 16));
|
|
1176
|
+
index += 3;
|
|
1177
|
+
}
|
|
1178
|
+
try {
|
|
1179
|
+
return textDecoder.decode(new Uint8Array(bytes));
|
|
1180
|
+
} catch {
|
|
1181
|
+
return null;
|
|
1182
|
+
}
|
|
1183
|
+
}
|
|
1184
|
+
function encodeDatasetTableName(datasetId) {
|
|
1185
|
+
return `${DATASET_TABLE_PREFIX}${encodeDatasetId(datasetId)}`;
|
|
1186
|
+
}
|
|
1187
|
+
function decodeDatasetTableName(tableName) {
|
|
1188
|
+
if (!tableName.startsWith(DATASET_TABLE_PREFIX)) {
|
|
1189
|
+
return null;
|
|
1190
|
+
}
|
|
1191
|
+
return decodeDatasetId(tableName.slice(DATASET_TABLE_PREFIX.length));
|
|
1192
|
+
}
|
|
1193
|
+
|
|
1194
|
+
// src/internal/ducklake-current-dataset-definitions.ts
|
|
1195
|
+
async function readCurrentDatasetDefinitions(input) {
|
|
1196
|
+
if (input.datasetIds?.length === 0) {
|
|
1197
|
+
return new Map;
|
|
1198
|
+
}
|
|
1199
|
+
const tables = await readCurrentDatasetTables(input, input.datasetIds);
|
|
1200
|
+
if (tables.length === 0) {
|
|
1201
|
+
return new Map;
|
|
1202
|
+
}
|
|
1203
|
+
const tableNames = tables.map((table) => table.tableName);
|
|
1204
|
+
const columnsByTableName = groupColumnsByTableName(await readCurrentDatasetColumns(input, tableNames));
|
|
1205
|
+
const partitionByTableName = groupPartitionsByTableName(await readCurrentDatasetPartitions(input, tableNames));
|
|
1206
|
+
const definitions = new Map;
|
|
1207
|
+
for (const table of tables) {
|
|
1208
|
+
const partitionBy = partitionByTableName.get(table.tableName) ?? [];
|
|
1209
|
+
definitions.set(table.datasetId, {
|
|
1210
|
+
kind: "dataset",
|
|
1211
|
+
id: table.datasetId,
|
|
1212
|
+
schema: duckLakeCatalogColumnsToDatasetSchema(table.tableName, columnsByTableName.get(table.tableName) ?? []),
|
|
1213
|
+
...partitionBy.length > 0 ? { partitionBy } : {},
|
|
1214
|
+
...table.comment !== undefined ? { description: table.comment } : {}
|
|
1215
|
+
});
|
|
1216
|
+
}
|
|
1217
|
+
return definitions;
|
|
1218
|
+
}
|
|
1219
|
+
async function readCurrentDatasetTables(input, datasetIds) {
|
|
1220
|
+
if (datasetIds !== undefined && datasetIds.length === 0) {
|
|
1221
|
+
return [];
|
|
1222
|
+
}
|
|
1223
|
+
const tableNameFilter = datasetIds === undefined ? "" : `AND table_name IN (${quoteSqlStringList([...new Set(datasetIds)].map((datasetId) => encodeDatasetTableName(datasetId)))})`;
|
|
1224
|
+
const rows = await input.runtime.query(`SELECT table_name, comment FROM duckdb_tables() WHERE database_name = ${quoteSqlString(duckLakeAlias(input.options))} AND schema_name = 'main' ${tableNameFilter} AND NOT internal ORDER BY table_name`);
|
|
1225
|
+
const tables = [];
|
|
1226
|
+
for (const row of rows) {
|
|
1227
|
+
const tableName = getString(row, "table_name");
|
|
1228
|
+
const datasetId = decodeDatasetTableName(tableName);
|
|
1229
|
+
if (datasetId === null) {
|
|
1230
|
+
continue;
|
|
1231
|
+
}
|
|
1232
|
+
tables.push({
|
|
1233
|
+
datasetId,
|
|
1234
|
+
tableName,
|
|
1235
|
+
comment: getOptionalString(row, "comment")
|
|
1236
|
+
});
|
|
1237
|
+
}
|
|
1238
|
+
return tables;
|
|
1239
|
+
}
|
|
1240
|
+
async function readCurrentDatasetColumns(input, tableNames) {
|
|
1241
|
+
if (tableNames.length === 0) {
|
|
1242
|
+
return [];
|
|
1243
|
+
}
|
|
1244
|
+
const ducklakeTable = duckLakeMetadataTableName(input.options, "ducklake_table");
|
|
1245
|
+
const ducklakeColumn = duckLakeMetadataTableName(input.options, "ducklake_column");
|
|
1246
|
+
const rows = await input.runtime.query(`
|
|
1247
|
+
SELECT
|
|
1248
|
+
table_meta.table_name,
|
|
1249
|
+
column_meta.column_id,
|
|
1250
|
+
column_meta.column_order,
|
|
1251
|
+
column_meta.column_name,
|
|
1252
|
+
column_meta.column_type,
|
|
1253
|
+
CAST(column_meta.nulls_allowed AS BOOLEAN) AS nulls_allowed,
|
|
1254
|
+
column_meta.parent_column
|
|
1255
|
+
FROM ${ducklakeTable} table_meta
|
|
1256
|
+
JOIN ${ducklakeColumn} column_meta
|
|
1257
|
+
ON column_meta.table_id = table_meta.table_id
|
|
1258
|
+
AND column_meta.end_snapshot IS NULL
|
|
1259
|
+
WHERE table_meta.end_snapshot IS NULL
|
|
1260
|
+
AND table_meta.table_name IN (${quoteSqlStringList(tableNames)})
|
|
1261
|
+
ORDER BY table_meta.table_name, column_meta.column_order
|
|
1262
|
+
`);
|
|
1263
|
+
return rows.map((row) => ({
|
|
1264
|
+
tableName: getString(row, "table_name"),
|
|
1265
|
+
columnId: getBigIntLike(row, "column_id"),
|
|
1266
|
+
columnOrder: getBigIntLike(row, "column_order"),
|
|
1267
|
+
columnName: getString(row, "column_name"),
|
|
1268
|
+
columnType: getString(row, "column_type"),
|
|
1269
|
+
nullsAllowed: getBoolean(row, "nulls_allowed"),
|
|
1270
|
+
parentColumnId: getOptionalBigIntLike(row, "parent_column")
|
|
1271
|
+
}));
|
|
1272
|
+
}
|
|
1273
|
+
async function readCurrentDatasetPartitions(input, tableNames) {
|
|
1274
|
+
if (tableNames.length === 0) {
|
|
1275
|
+
return [];
|
|
1276
|
+
}
|
|
1277
|
+
const ducklakeTable = duckLakeMetadataTableName(input.options, "ducklake_table");
|
|
1278
|
+
const ducklakePartitionInfo = duckLakeMetadataTableName(input.options, "ducklake_partition_info");
|
|
1279
|
+
const ducklakePartitionColumn = duckLakeMetadataTableName(input.options, "ducklake_partition_column");
|
|
1280
|
+
const ducklakeColumn = duckLakeMetadataTableName(input.options, "ducklake_column");
|
|
1281
|
+
const rows = await input.runtime.query(`
|
|
1282
|
+
SELECT
|
|
1283
|
+
table_meta.table_name,
|
|
1284
|
+
column_meta.column_name,
|
|
1285
|
+
partition_column.transform
|
|
1286
|
+
FROM ${ducklakeTable} table_meta
|
|
1287
|
+
JOIN ${ducklakePartitionInfo} partition_info
|
|
1288
|
+
ON partition_info.table_id = table_meta.table_id
|
|
1289
|
+
AND partition_info.end_snapshot IS NULL
|
|
1290
|
+
JOIN ${ducklakePartitionColumn} partition_column
|
|
1291
|
+
ON partition_column.table_id = table_meta.table_id
|
|
1292
|
+
AND partition_column.partition_id = partition_info.partition_id
|
|
1293
|
+
JOIN ${ducklakeColumn} column_meta
|
|
1294
|
+
ON column_meta.table_id = table_meta.table_id
|
|
1295
|
+
AND column_meta.column_id = partition_column.column_id
|
|
1296
|
+
AND column_meta.end_snapshot IS NULL
|
|
1297
|
+
WHERE table_meta.end_snapshot IS NULL
|
|
1298
|
+
AND table_meta.table_name IN (${quoteSqlStringList(tableNames)})
|
|
1299
|
+
ORDER BY table_meta.table_name, partition_column.partition_key_index
|
|
1300
|
+
`);
|
|
1301
|
+
return rows.map((row) => {
|
|
1302
|
+
const partition = {
|
|
1303
|
+
tableName: getString(row, "table_name"),
|
|
1304
|
+
columnName: getString(row, "column_name"),
|
|
1305
|
+
transform: getString(row, "transform")
|
|
1306
|
+
};
|
|
1307
|
+
if (partition.transform !== "identity") {
|
|
1308
|
+
throw new LakeStorageError10(`[SixbDuckLake] Dataset table '${partition.tableName}' uses unsupported DuckLake partition transform '${partition.transform}'.`);
|
|
1309
|
+
}
|
|
1310
|
+
return partition;
|
|
1311
|
+
});
|
|
1312
|
+
}
|
|
1313
|
+
function groupColumnsByTableName(rows) {
|
|
1314
|
+
const result = new Map;
|
|
1315
|
+
for (const row of rows) {
|
|
1316
|
+
const tableRows = result.get(row.tableName) ?? [];
|
|
1317
|
+
tableRows.push(row);
|
|
1318
|
+
result.set(row.tableName, tableRows);
|
|
1319
|
+
}
|
|
1320
|
+
return result;
|
|
1321
|
+
}
|
|
1322
|
+
function groupPartitionsByTableName(rows) {
|
|
1323
|
+
const result = new Map;
|
|
1324
|
+
for (const row of rows) {
|
|
1325
|
+
const tablePartitions = result.get(row.tableName) ?? [];
|
|
1326
|
+
tablePartitions.push(row.columnName);
|
|
1327
|
+
result.set(row.tableName, tablePartitions);
|
|
1328
|
+
}
|
|
1329
|
+
return result;
|
|
1330
|
+
}
|
|
1331
|
+
|
|
1332
|
+
// src/internal/ducklake-dataset-catalog.ts
|
|
1333
|
+
class DuckLakeDatasetCatalog {
|
|
1334
|
+
options;
|
|
1335
|
+
connections;
|
|
1336
|
+
constructor(options, connections) {
|
|
1337
|
+
this.options = options;
|
|
1338
|
+
this.connections = connections;
|
|
1339
|
+
}
|
|
1340
|
+
async createDataset(definition) {
|
|
1341
|
+
this.assertSchema(definition);
|
|
1342
|
+
const existing = await this.getDataset(definition.id);
|
|
1343
|
+
if (existing) {
|
|
1344
|
+
const plan = planDatasetDefinitionUpdate(existing, definition);
|
|
1345
|
+
if (plan.changed) {
|
|
1346
|
+
await this.applyExistingDatasetUpdate(plan);
|
|
1347
|
+
this.connections.markLocalCatalogChanged();
|
|
1348
|
+
await this.connections.resetRuntime();
|
|
1349
|
+
}
|
|
1350
|
+
return await this.getDataset(definition.id) ?? plan.definition;
|
|
1351
|
+
}
|
|
1352
|
+
const tableName = encodeDatasetTableName(definition.id);
|
|
1353
|
+
const qualifiedName = qualifiedTableName(this.options, tableName);
|
|
1354
|
+
await this.connections.withExclusiveAttached(async (runtime) => {
|
|
1355
|
+
await runtime.run(`CREATE TABLE ${qualifiedName} (${datasetSchemaToDuckDbColumnsSql(definition.schema)})`);
|
|
1356
|
+
if (definition.description !== undefined) {
|
|
1357
|
+
await runtime.run(`COMMENT ON TABLE ${qualifiedName} IS ${quoteSqlString(definition.description)}`);
|
|
1358
|
+
}
|
|
1359
|
+
await this.applyPartitionBy(runtime, qualifiedName, definition);
|
|
1360
|
+
});
|
|
1361
|
+
this.connections.markLocalCatalogChanged();
|
|
1362
|
+
return structuredClone(definition);
|
|
1363
|
+
}
|
|
1364
|
+
async assertDatasetDefinitionsCompatible(definitions) {
|
|
1365
|
+
const failures = [];
|
|
1366
|
+
const checkedDefinitions = [];
|
|
1367
|
+
for (const definition of definitions) {
|
|
1368
|
+
try {
|
|
1369
|
+
this.assertSchema(definition);
|
|
1370
|
+
checkedDefinitions.push(definition);
|
|
1371
|
+
} catch (error) {
|
|
1372
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
1373
|
+
failures.push(`- ${definition.id}: ${message}`);
|
|
1374
|
+
}
|
|
1375
|
+
}
|
|
1376
|
+
const existingById = await this.readCurrentDatasetDefinitions({
|
|
1377
|
+
datasetIds: checkedDefinitions.map((definition) => definition.id)
|
|
1378
|
+
});
|
|
1379
|
+
for (const definition of checkedDefinitions) {
|
|
1380
|
+
const existing = existingById.get(definition.id);
|
|
1381
|
+
if (!existing) {
|
|
1382
|
+
continue;
|
|
1383
|
+
}
|
|
1384
|
+
try {
|
|
1385
|
+
planDatasetDefinitionUpdate(existing, definition);
|
|
1386
|
+
} catch (error) {
|
|
1387
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
1388
|
+
failures.push(`- ${definition.id}: ${message}`);
|
|
1389
|
+
}
|
|
1390
|
+
}
|
|
1391
|
+
if (failures.length > 0) {
|
|
1392
|
+
const details = failures.join(`
|
|
1393
|
+
`);
|
|
1394
|
+
throw new LakeStorageError11(`[SixbLake] Lake dataset definition check failed for ${failures.length} dataset(s).
|
|
1395
|
+
${details}`);
|
|
1396
|
+
}
|
|
1397
|
+
}
|
|
1398
|
+
async getDataset(datasetId) {
|
|
1399
|
+
this.connections.assertOpen();
|
|
1400
|
+
return this.connections.withAttachedRuntime((runtime) => this.getDatasetOnRuntime(runtime, datasetId));
|
|
1401
|
+
}
|
|
1402
|
+
async getDatasetOnRuntime(runtime, datasetId) {
|
|
1403
|
+
return (await this.readCurrentDatasetDefinitionsOnRuntime(runtime, {
|
|
1404
|
+
datasetIds: [datasetId]
|
|
1405
|
+
})).get(datasetId) ?? null;
|
|
1406
|
+
}
|
|
1407
|
+
async listDatasets() {
|
|
1408
|
+
this.connections.assertOpen();
|
|
1409
|
+
return this.connections.withAttachedRuntime(async (runtime) => {
|
|
1410
|
+
return [...(await this.readCurrentDatasetDefinitionsOnRuntime(runtime)).values()].sort((left, right) => left.id.localeCompare(right.id));
|
|
1411
|
+
});
|
|
1412
|
+
}
|
|
1413
|
+
async readCurrentDatasetDefinitions(options) {
|
|
1414
|
+
if (options?.datasetIds?.length === 0) {
|
|
1415
|
+
return new Map;
|
|
1416
|
+
}
|
|
1417
|
+
return this.connections.withAttachedRuntime((runtime) => this.readCurrentDatasetDefinitionsOnRuntime(runtime, options));
|
|
1418
|
+
}
|
|
1419
|
+
async readCurrentDatasetDefinitionsOnRuntime(runtime, options) {
|
|
1420
|
+
return readCurrentDatasetDefinitions({
|
|
1421
|
+
options: this.options,
|
|
1422
|
+
runtime,
|
|
1423
|
+
datasetIds: options?.datasetIds
|
|
1424
|
+
});
|
|
1425
|
+
}
|
|
1426
|
+
async getDatasetSchemaAtSnapshot(runtime, datasetId, tableName, tableId, snapshotId) {
|
|
1427
|
+
assertDuckLakeSnapshotId(snapshotId);
|
|
1428
|
+
const columns = await this.readDatasetColumnsAtSnapshot(runtime, tableName, tableId, snapshotId);
|
|
1429
|
+
if (columns.length === 0) {
|
|
1430
|
+
throw new LakeStorageError11(`[SixbDuckLake] Could not reconstruct schema for dataset '${datasetId}' at DuckLake snapshot '${snapshotId}'.`);
|
|
1431
|
+
}
|
|
1432
|
+
return duckLakeCatalogColumnsToDatasetSchema(tableName, columns);
|
|
1433
|
+
}
|
|
1434
|
+
assertSchema(definition) {
|
|
1435
|
+
if (!definition.schema) {
|
|
1436
|
+
throw new LakeStorageError11(`[SixbDuckLake] Dataset '${definition.id}' requires a schema for DuckLake storage.`);
|
|
1437
|
+
}
|
|
1438
|
+
}
|
|
1439
|
+
async applyExistingDatasetUpdate(plan) {
|
|
1440
|
+
const tableName = encodeDatasetTableName(plan.definition.id);
|
|
1441
|
+
const qualifiedName = qualifiedTableName(this.options, tableName);
|
|
1442
|
+
await this.connections.withExclusiveAttached(async (runtime) => {
|
|
1443
|
+
if (plan.schema.kind === "none") {
|
|
1444
|
+
await this.applyMetadataDelta(runtime, qualifiedName, plan);
|
|
1445
|
+
return;
|
|
1446
|
+
}
|
|
1447
|
+
await runtime.run("BEGIN TRANSACTION");
|
|
1448
|
+
let committed = false;
|
|
1449
|
+
try {
|
|
1450
|
+
await this.applySchemaEvolutionDdl(runtime, qualifiedName, plan.schema);
|
|
1451
|
+
await this.applyMetadataDelta(runtime, qualifiedName, plan);
|
|
1452
|
+
await this.setSchemaEvolutionCommitMetadata(runtime, plan.definition.id, plan.schema);
|
|
1453
|
+
await runtime.run("COMMIT");
|
|
1454
|
+
committed = true;
|
|
1455
|
+
} catch (error) {
|
|
1456
|
+
if (!committed) {
|
|
1457
|
+
await this.rollbackTransaction(runtime);
|
|
1458
|
+
}
|
|
1459
|
+
throw error;
|
|
1460
|
+
}
|
|
1461
|
+
});
|
|
1462
|
+
}
|
|
1463
|
+
async applySchemaEvolutionDdl(runtime, qualifiedName, plan) {
|
|
1464
|
+
for (const column of plan.columns) {
|
|
1465
|
+
await runtime.run(`ALTER TABLE ${qualifiedName} ADD COLUMN ${datasetColumnToDuckDbSql(column)}`);
|
|
1466
|
+
}
|
|
1467
|
+
}
|
|
1468
|
+
async applyMetadataDelta(runtime, qualifiedName, plan) {
|
|
1469
|
+
if (plan.metadata.descriptionChanged && plan.definition.description !== undefined) {
|
|
1470
|
+
await runtime.run(`COMMENT ON TABLE ${qualifiedName} IS ${quoteSqlString(plan.definition.description)}`);
|
|
1471
|
+
}
|
|
1472
|
+
if (plan.metadata.partitionByChanged) {
|
|
1473
|
+
await this.applyPartitionBy(runtime, qualifiedName, plan.definition);
|
|
1474
|
+
}
|
|
1475
|
+
}
|
|
1476
|
+
async applyPartitionBy(runtime, qualifiedName, definition) {
|
|
1477
|
+
if (definition.partitionBy === undefined || definition.partitionBy.length === 0) {
|
|
1478
|
+
return;
|
|
1479
|
+
}
|
|
1480
|
+
try {
|
|
1481
|
+
await runtime.run(`ALTER TABLE ${qualifiedName} SET PARTITIONED BY (${definition.partitionBy.map((columnName) => quoteIdentifier(columnName)).join(", ")})`);
|
|
1482
|
+
} catch (error) {
|
|
1483
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
1484
|
+
throw new LakeStorageError11(`[SixbDuckLake] Dataset '${definition.id}' cannot apply partitionBy because DuckLake rejected ALTER TABLE SET PARTITIONED BY: ${message}`);
|
|
1485
|
+
}
|
|
1486
|
+
}
|
|
1487
|
+
async setSchemaEvolutionCommitMetadata(runtime, datasetId, plan) {
|
|
1488
|
+
await runtime.run(`CALL ${quoteIdentifier(duckLakeAlias(this.options))}.set_commit_message(${quoteSqlString("Sixb")}, ${quoteSqlString(`evolve dataset ${datasetId} schema`)}, extra_info => ${quoteSqlString(JSON.stringify({
|
|
1489
|
+
sixb: {
|
|
1490
|
+
kind: "datasetVersion",
|
|
1491
|
+
datasetId,
|
|
1492
|
+
commitId: randomUUID(),
|
|
1493
|
+
mode: "schema",
|
|
1494
|
+
schemaChange: {
|
|
1495
|
+
addColumns: plan.columns.map((column) => column.name)
|
|
1496
|
+
}
|
|
1497
|
+
}
|
|
1498
|
+
}))})`);
|
|
1499
|
+
}
|
|
1500
|
+
async rollbackTransaction(runtime) {
|
|
1501
|
+
try {
|
|
1502
|
+
await runtime.run("ROLLBACK");
|
|
1503
|
+
} catch {}
|
|
1504
|
+
}
|
|
1505
|
+
async readDatasetColumnsAtSnapshot(runtime, tableName, tableId, snapshotId) {
|
|
1506
|
+
const ducklakeColumn = duckLakeMetadataTableName(this.options, "ducklake_column");
|
|
1507
|
+
const rows = await runtime.query(`
|
|
1508
|
+
SELECT
|
|
1509
|
+
column_id,
|
|
1510
|
+
column_order,
|
|
1511
|
+
column_name,
|
|
1512
|
+
column_type,
|
|
1513
|
+
CAST(nulls_allowed AS BOOLEAN) AS nulls_allowed,
|
|
1514
|
+
parent_column
|
|
1515
|
+
FROM ${ducklakeColumn}
|
|
1516
|
+
WHERE table_id = ${tableId}
|
|
1517
|
+
AND begin_snapshot <= ${snapshotId}
|
|
1518
|
+
AND (end_snapshot IS NULL OR end_snapshot > ${snapshotId})
|
|
1519
|
+
ORDER BY column_order
|
|
1520
|
+
`);
|
|
1521
|
+
return rows.map((row) => ({
|
|
1522
|
+
tableName,
|
|
1523
|
+
columnId: getBigIntLike(row, "column_id"),
|
|
1524
|
+
columnOrder: getBigIntLike(row, "column_order"),
|
|
1525
|
+
columnName: getString(row, "column_name"),
|
|
1526
|
+
columnType: getString(row, "column_type"),
|
|
1527
|
+
nullsAllowed: getBoolean(row, "nulls_allowed"),
|
|
1528
|
+
parentColumnId: getOptionalBigIntLike(row, "parent_column")
|
|
1529
|
+
}));
|
|
1530
|
+
}
|
|
1531
|
+
}
|
|
1532
|
+
function assertDuckLakeSnapshotId(snapshotId) {
|
|
1533
|
+
if (!/^\d+$/.test(snapshotId)) {
|
|
1534
|
+
throw new LakeStorageError11(`[SixbDuckLake] Invalid DuckLake snapshot id '${snapshotId}'.`);
|
|
1535
|
+
}
|
|
1536
|
+
}
|
|
1537
|
+
|
|
1538
|
+
// src/internal/ducklake-maintenance.ts
|
|
1539
|
+
import { LakeStorageError as LakeStorageError12 } from "@sixb/core/lake-storage";
|
|
1540
|
+
var DEFAULT_RETENTION = "7 days";
|
|
1541
|
+
|
|
1542
|
+
class DuckLakeMaintenance {
|
|
1543
|
+
options;
|
|
1544
|
+
connections;
|
|
1545
|
+
constructor(options, connections) {
|
|
1546
|
+
this.options = options;
|
|
1547
|
+
this.connections = connections;
|
|
1548
|
+
}
|
|
1549
|
+
async runMaintenance(options = {}) {
|
|
1550
|
+
this.connections.assertOpen();
|
|
1551
|
+
const expireOlderThan = options.expireOlderThan ?? DEFAULT_RETENTION;
|
|
1552
|
+
const deleteOlderThan = options.deleteOlderThan ?? expireOlderThan;
|
|
1553
|
+
const dryRun = options.dryRun ?? false;
|
|
1554
|
+
if (this.options.readOnly && !dryRun) {
|
|
1555
|
+
throw new LakeStorageError12("[SixbDuckLake] DuckLake maintenance cannot run in read-only mode.");
|
|
1556
|
+
}
|
|
1557
|
+
return this.connections.withExclusiveAttached(async (runtime) => {
|
|
1558
|
+
const counts = await runMaintenanceProcedures(runtime, this.options, {
|
|
1559
|
+
expireOlderThan,
|
|
1560
|
+
deleteOlderThan
|
|
1561
|
+
}, dryRun);
|
|
1562
|
+
if (!dryRun) {
|
|
1563
|
+
this.connections.markLocalCatalogChanged();
|
|
1564
|
+
}
|
|
1565
|
+
return {
|
|
1566
|
+
dryRun,
|
|
1567
|
+
expireOlderThan,
|
|
1568
|
+
deleteOlderThan,
|
|
1569
|
+
...counts
|
|
1570
|
+
};
|
|
1571
|
+
});
|
|
1572
|
+
}
|
|
1573
|
+
}
|
|
1574
|
+
async function runMaintenanceProcedures(runtime, options, retention, dryRun) {
|
|
1575
|
+
const aliasSql = quoteSqlString(duckLakeAlias(options));
|
|
1576
|
+
const snapshots = await runtime.query(maintenanceProcedureSql("ducklake_expire_snapshots", aliasSql, retention.expireOlderThan, dryRun));
|
|
1577
|
+
const oldFiles = await runtime.query(maintenanceProcedureSql("ducklake_cleanup_old_files", aliasSql, retention.deleteOlderThan, dryRun));
|
|
1578
|
+
const orphanedFiles = await runtime.query(maintenanceProcedureSql("ducklake_delete_orphaned_files", aliasSql, retention.deleteOlderThan, dryRun));
|
|
1579
|
+
return {
|
|
1580
|
+
snapshots: snapshots.length,
|
|
1581
|
+
oldFiles: oldFiles.length,
|
|
1582
|
+
orphanedFiles: orphanedFiles.length
|
|
1583
|
+
};
|
|
1584
|
+
}
|
|
1585
|
+
function maintenanceProcedureSql(procedure, aliasSql, olderThan, dryRun) {
|
|
1586
|
+
const olderThanSql = quoteSqlString(olderThan);
|
|
1587
|
+
const dryRunSql = dryRun ? "dry_run => true, " : "";
|
|
1588
|
+
return `CALL ${procedure}(${aliasSql}, ${dryRunSql}` + `older_than => now() - INTERVAL ${olderThanSql})`;
|
|
1589
|
+
}
|
|
1590
|
+
|
|
1591
|
+
// src/internal/ducklake-row-reader.ts
|
|
1592
|
+
import { LakeStorageError as LakeStorageError14 } from "@sixb/core/lake-storage";
|
|
1593
|
+
|
|
1594
|
+
// src/internal/ducklake-dataset-table-ref.ts
|
|
1595
|
+
async function resolveDatasetTableRef(options, runtime, datasetId) {
|
|
1596
|
+
const tableName = encodeDatasetTableName(datasetId);
|
|
1597
|
+
const ducklakeTable = duckLakeMetadataTableName(options, "ducklake_table");
|
|
1598
|
+
const [row] = await runtime.query(`
|
|
1599
|
+
SELECT table_id
|
|
1600
|
+
FROM ${ducklakeTable}
|
|
1601
|
+
WHERE table_name = ${quoteSqlString(tableName)}
|
|
1602
|
+
AND end_snapshot IS NULL
|
|
1603
|
+
LIMIT 1
|
|
1604
|
+
`);
|
|
1605
|
+
return row === undefined ? null : {
|
|
1606
|
+
datasetId,
|
|
1607
|
+
tableName,
|
|
1608
|
+
tableId: getBigIntLike(row, "table_id")
|
|
1609
|
+
};
|
|
1610
|
+
}
|
|
1611
|
+
|
|
1612
|
+
// src/internal/versions.ts
|
|
1613
|
+
import { LakeStorageError as LakeStorageError13 } from "@sixb/core/lake-storage";
|
|
1614
|
+
function parseVersionId(versionId) {
|
|
1615
|
+
if (!versionId.startsWith("ducklake:")) {
|
|
1616
|
+
throw new LakeStorageError13(`[SixbDuckLake] Invalid DuckLake version id '${versionId}'.`);
|
|
1617
|
+
}
|
|
1618
|
+
const snapshotId = versionId.slice("ducklake:".length);
|
|
1619
|
+
if (!/^\d+$/.test(snapshotId)) {
|
|
1620
|
+
throw new LakeStorageError13(`[SixbDuckLake] Invalid DuckLake version id '${versionId}'.`);
|
|
1621
|
+
}
|
|
1622
|
+
return snapshotId;
|
|
1623
|
+
}
|
|
1624
|
+
function toVersionId(snapshotId) {
|
|
1625
|
+
return `ducklake:${snapshotId}`;
|
|
1626
|
+
}
|
|
1627
|
+
function parseCommitMetadata(value) {
|
|
1628
|
+
if (typeof value !== "string") {
|
|
1629
|
+
return;
|
|
1630
|
+
}
|
|
1631
|
+
let parsed;
|
|
1632
|
+
try {
|
|
1633
|
+
parsed = JSON.parse(value);
|
|
1634
|
+
} catch {
|
|
1635
|
+
return;
|
|
1636
|
+
}
|
|
1637
|
+
if (!isRecord(parsed) || !isRecord(parsed.sixb) || parsed.sixb.kind !== "datasetVersion" || typeof parsed.sixb.datasetId !== "string") {
|
|
1638
|
+
return;
|
|
1639
|
+
}
|
|
1640
|
+
const mode = parsed.sixb.mode;
|
|
1641
|
+
const commitId = parsed.sixb.commitId;
|
|
1642
|
+
const rowCount = parsed.sixb.rowCount;
|
|
1643
|
+
return {
|
|
1644
|
+
kind: "datasetVersion",
|
|
1645
|
+
datasetId: parsed.sixb.datasetId,
|
|
1646
|
+
...typeof commitId === "string" ? { commitId } : {},
|
|
1647
|
+
...mode === "snapshot" || mode === "append" || mode === "schema" ? { mode } : {},
|
|
1648
|
+
...isDatasetProducer(parsed.sixb.producer) ? { producer: parsed.sixb.producer } : {},
|
|
1649
|
+
...isDatasetVersionRefs(parsed.sixb.inputs) ? { inputs: parsed.sixb.inputs } : {},
|
|
1650
|
+
...typeof commitId === "string" && isRowCount(rowCount) ? { rowCount } : {},
|
|
1651
|
+
...isSchemaChangeMetadata(parsed.sixb.schemaChange) ? { schemaChange: parsed.sixb.schemaChange } : {}
|
|
1652
|
+
};
|
|
1653
|
+
}
|
|
1654
|
+
function parseInlineDataChange(changesMade, tableId) {
|
|
1655
|
+
const tableIdText = tableId.toString();
|
|
1656
|
+
let hasDataChange = false;
|
|
1657
|
+
let hasDeleteChange = false;
|
|
1658
|
+
for (const change of changesMade.split(",")) {
|
|
1659
|
+
const [kind, changedTableId] = change.split(":");
|
|
1660
|
+
if (changedTableId !== tableIdText) {
|
|
1661
|
+
continue;
|
|
1662
|
+
}
|
|
1663
|
+
const normalizedKind = kind.toLowerCase();
|
|
1664
|
+
if (normalizedKind.includes("insert") || normalizedKind.includes("delete")) {
|
|
1665
|
+
hasDataChange = true;
|
|
1666
|
+
}
|
|
1667
|
+
if (normalizedKind.includes("delete")) {
|
|
1668
|
+
hasDeleteChange = true;
|
|
1669
|
+
}
|
|
1670
|
+
}
|
|
1671
|
+
return { hasDataChange, hasDeleteChange };
|
|
1672
|
+
}
|
|
1673
|
+
function isRecord(value) {
|
|
1674
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
1675
|
+
}
|
|
1676
|
+
function isDatasetProducer(value) {
|
|
1677
|
+
return isRecord(value) && (value.kind === "sync" || value.kind === "pipeline") && optionalString(value.id) && optionalString(value.runId);
|
|
1678
|
+
}
|
|
1679
|
+
function isDatasetVersionRefs(value) {
|
|
1680
|
+
return Array.isArray(value) && value.every((item) => isRecord(item) && typeof item.datasetId === "string" && typeof item.versionId === "string");
|
|
1681
|
+
}
|
|
1682
|
+
function isSchemaChangeMetadata(value) {
|
|
1683
|
+
return isRecord(value) && (value.addColumns === undefined || Array.isArray(value.addColumns) && value.addColumns.every((columnName) => typeof columnName === "string"));
|
|
1684
|
+
}
|
|
1685
|
+
function optionalString(value) {
|
|
1686
|
+
return value === undefined || typeof value === "string";
|
|
1687
|
+
}
|
|
1688
|
+
function isRowCount(value) {
|
|
1689
|
+
return typeof value === "number" && Number.isSafeInteger(value) && value >= 0;
|
|
1690
|
+
}
|
|
1691
|
+
|
|
1692
|
+
// src/internal/ducklake-row-reader.ts
|
|
1693
|
+
class DuckLakeRowReader {
|
|
1694
|
+
options;
|
|
1695
|
+
connections;
|
|
1696
|
+
snapshots;
|
|
1697
|
+
constructor(options, connections, snapshots) {
|
|
1698
|
+
this.options = options;
|
|
1699
|
+
this.connections = connections;
|
|
1700
|
+
this.snapshots = snapshots;
|
|
1701
|
+
}
|
|
1702
|
+
async* readRows(input) {
|
|
1703
|
+
this.connections.assertOpen();
|
|
1704
|
+
const lease = await this.connections.acquireAttachedRuntime();
|
|
1705
|
+
try {
|
|
1706
|
+
const runtime = lease.runtime;
|
|
1707
|
+
const tableRef = await resolveDatasetTableRef(this.options, runtime, input.datasetId);
|
|
1708
|
+
if (!tableRef) {
|
|
1709
|
+
throw new LakeStorageError14(`[SixbDuckLake] Unknown dataset '${input.datasetId}'.`);
|
|
1710
|
+
}
|
|
1711
|
+
const version = await this.resolveVersion(runtime, tableRef, input.versionId);
|
|
1712
|
+
const snapshotId = parseVersionId(version.versionId);
|
|
1713
|
+
const selectedColumns = this.resolveReadColumns(tableRef.datasetId, version.schema, input.columns);
|
|
1714
|
+
const columnsSql = selectedColumns.map((column) => quoteIdentifier(column.name)).join(", ");
|
|
1715
|
+
const tableSql = qualifiedTableName(this.options, tableRef.tableName);
|
|
1716
|
+
const orderSql = version.schema.columns.some((column) => column.name.toLowerCase() === "rowid") ? "" : " ORDER BY rowid";
|
|
1717
|
+
const limitSql = input.limit === undefined ? "" : ` LIMIT ${Math.max(0, Math.trunc(input.limit))}`;
|
|
1718
|
+
const offsetSql = input.offset === undefined ? "" : ` OFFSET ${Math.max(0, Math.trunc(input.offset))}`;
|
|
1719
|
+
const sql = `SELECT ${columnsSql} FROM ${tableSql} AT (VERSION => ${snapshotId})${orderSql}${limitSql}${offsetSql}`;
|
|
1720
|
+
const rows = input.limit === undefined ? runtime.streamRows(sql) : await runtime.query(sql);
|
|
1721
|
+
yield* this.normalizeRows(rows, selectedColumns);
|
|
1722
|
+
} finally {
|
|
1723
|
+
await lease.release();
|
|
1724
|
+
}
|
|
1725
|
+
}
|
|
1726
|
+
async* normalizeRows(rows, selectedColumns) {
|
|
1727
|
+
for await (const row of rows) {
|
|
1728
|
+
const output = {};
|
|
1729
|
+
for (const column of selectedColumns) {
|
|
1730
|
+
output[column.name] = normalizeReadValue(row[column.name], column);
|
|
1731
|
+
}
|
|
1732
|
+
yield output;
|
|
1733
|
+
}
|
|
1734
|
+
}
|
|
1735
|
+
async resolveVersion(runtime, tableRef, versionId) {
|
|
1736
|
+
if (versionId !== undefined) {
|
|
1737
|
+
const snapshotId = parseVersionId(versionId);
|
|
1738
|
+
const version = await this.snapshots.getVersionForTableRef(runtime, tableRef, snapshotId);
|
|
1739
|
+
if (!version) {
|
|
1740
|
+
this.throwNoCommittedVersion(tableRef.datasetId);
|
|
1741
|
+
}
|
|
1742
|
+
return version;
|
|
1743
|
+
}
|
|
1744
|
+
const latestVersion = await this.snapshots.getLatestVersionForTableRef(runtime, tableRef);
|
|
1745
|
+
if (!latestVersion) {
|
|
1746
|
+
this.throwNoCommittedVersion(tableRef.datasetId);
|
|
1747
|
+
}
|
|
1748
|
+
return latestVersion;
|
|
1749
|
+
}
|
|
1750
|
+
resolveReadColumns(datasetId, schema, columns) {
|
|
1751
|
+
if (columns === undefined || columns.length === 0) {
|
|
1752
|
+
return schema.columns;
|
|
1753
|
+
}
|
|
1754
|
+
const columnsByName = new Map(schema.columns.map((column) => [column.name, column]));
|
|
1755
|
+
return columns.map((columnName) => {
|
|
1756
|
+
const column = columnsByName.get(columnName);
|
|
1757
|
+
if (!column) {
|
|
1758
|
+
throw new LakeStorageError14(`[SixbDuckLake] Dataset '${datasetId}' does not have column '${columnName}' at the requested version.`);
|
|
1759
|
+
}
|
|
1760
|
+
return column;
|
|
1761
|
+
});
|
|
1762
|
+
}
|
|
1763
|
+
throwNoCommittedVersion(datasetId) {
|
|
1764
|
+
throw new LakeStorageError14(`[SixbDuckLake] No committed version found for dataset '${datasetId}'.`);
|
|
1765
|
+
}
|
|
1766
|
+
}
|
|
1767
|
+
|
|
1768
|
+
// src/internal/ducklake-snapshot-reader.ts
|
|
1769
|
+
import { LakeStorageError as LakeStorageError15 } from "@sixb/core/lake-storage";
|
|
1770
|
+
var SNAPSHOT_ROW_BATCH_SIZE = 128;
|
|
1771
|
+
var CATALOG_SNAPSHOT_SCAN_LIMIT = 512;
|
|
1772
|
+
|
|
1773
|
+
class DuckLakeSnapshotReader {
|
|
1774
|
+
options;
|
|
1775
|
+
connections;
|
|
1776
|
+
datasets;
|
|
1777
|
+
constructor(options, connections, datasets) {
|
|
1778
|
+
this.options = options;
|
|
1779
|
+
this.connections = connections;
|
|
1780
|
+
this.datasets = datasets;
|
|
1781
|
+
}
|
|
1782
|
+
async listVersions(datasetId, limit) {
|
|
1783
|
+
this.connections.assertOpen();
|
|
1784
|
+
return this.connections.withAttachedRuntime(async (runtime) => {
|
|
1785
|
+
const tableRef = await resolveDatasetTableRef(this.options, runtime, datasetId);
|
|
1786
|
+
if (!tableRef) {
|
|
1787
|
+
return [];
|
|
1788
|
+
}
|
|
1789
|
+
return this.listVersionsForTableRef(runtime, tableRef, limit);
|
|
1790
|
+
});
|
|
1791
|
+
}
|
|
1792
|
+
async getLatestVersion(datasetId) {
|
|
1793
|
+
this.connections.assertOpen();
|
|
1794
|
+
return this.connections.withAttachedRuntime(async (runtime) => {
|
|
1795
|
+
const tableRef = await resolveDatasetTableRef(this.options, runtime, datasetId);
|
|
1796
|
+
if (!tableRef) {
|
|
1797
|
+
return null;
|
|
1798
|
+
}
|
|
1799
|
+
return this.getLatestVersionForTableRef(runtime, tableRef);
|
|
1800
|
+
});
|
|
1801
|
+
}
|
|
1802
|
+
async getVersion(datasetId, versionId) {
|
|
1803
|
+
const snapshotId = parseVersionId(versionId);
|
|
1804
|
+
this.connections.assertOpen();
|
|
1805
|
+
return this.connections.withAttachedRuntime(async (runtime) => {
|
|
1806
|
+
const tableRef = await resolveDatasetTableRef(this.options, runtime, datasetId);
|
|
1807
|
+
if (!tableRef) {
|
|
1808
|
+
return null;
|
|
1809
|
+
}
|
|
1810
|
+
return this.getVersionForTableRef(runtime, tableRef, snapshotId);
|
|
1811
|
+
});
|
|
1812
|
+
}
|
|
1813
|
+
async listDatasetCatalogState(datasetIds) {
|
|
1814
|
+
this.connections.assertOpen();
|
|
1815
|
+
if (datasetIds.length === 0) {
|
|
1816
|
+
return [];
|
|
1817
|
+
}
|
|
1818
|
+
return this.connections.withAttachedRuntime((runtime) => this.collectDatasetCatalogState(runtime, datasetIds));
|
|
1819
|
+
}
|
|
1820
|
+
async collectDatasetCatalogState(runtime, datasetIds) {
|
|
1821
|
+
const uniqueIds = [...new Set(datasetIds)];
|
|
1822
|
+
const tableIdsByDatasetId = await this.resolveDatasetTableIds(runtime, uniqueIds);
|
|
1823
|
+
const latestRowByDatasetId = await this.resolveLatestSnapshotRows(runtime, tableIdsByDatasetId);
|
|
1824
|
+
return uniqueIds.map((datasetId) => {
|
|
1825
|
+
if (!tableIdsByDatasetId.has(datasetId)) {
|
|
1826
|
+
return { datasetId, materialized: false, latestVersion: null };
|
|
1827
|
+
}
|
|
1828
|
+
const row = latestRowByDatasetId.get(datasetId);
|
|
1829
|
+
return {
|
|
1830
|
+
datasetId,
|
|
1831
|
+
materialized: true,
|
|
1832
|
+
latestVersion: row ? this.snapshotRowToSummary(datasetId, row) : null
|
|
1833
|
+
};
|
|
1834
|
+
});
|
|
1835
|
+
}
|
|
1836
|
+
snapshotRowToSummary(datasetId, row) {
|
|
1837
|
+
return {
|
|
1838
|
+
datasetId,
|
|
1839
|
+
versionId: toVersionId(row.snapshotId),
|
|
1840
|
+
mode: row.mode,
|
|
1841
|
+
createdAt: row.createdAt,
|
|
1842
|
+
...row.metadata?.rowCount !== undefined ? { rowCount: row.metadata.rowCount } : {}
|
|
1843
|
+
};
|
|
1844
|
+
}
|
|
1845
|
+
async resolveDatasetTableIds(runtime, datasetIds) {
|
|
1846
|
+
const datasetIdByTableName = new Map;
|
|
1847
|
+
for (const datasetId of datasetIds) {
|
|
1848
|
+
datasetIdByTableName.set(encodeDatasetTableName(datasetId), datasetId);
|
|
1849
|
+
}
|
|
1850
|
+
const ducklakeTable = duckLakeMetadataTableName(this.options, "ducklake_table");
|
|
1851
|
+
const tableNameList = [...datasetIdByTableName.keys()].map((name) => quoteSqlString(name)).join(", ");
|
|
1852
|
+
const rows = await runtime.query(`
|
|
1853
|
+
SELECT table_id, table_name
|
|
1854
|
+
FROM ${ducklakeTable}
|
|
1855
|
+
WHERE table_name IN (${tableNameList})
|
|
1856
|
+
AND end_snapshot IS NULL
|
|
1857
|
+
`);
|
|
1858
|
+
const tableIdsByDatasetId = new Map;
|
|
1859
|
+
for (const row of rows) {
|
|
1860
|
+
const datasetId = datasetIdByTableName.get(getString(row, "table_name"));
|
|
1861
|
+
if (datasetId !== undefined) {
|
|
1862
|
+
tableIdsByDatasetId.set(datasetId, getBigIntLike(row, "table_id"));
|
|
1863
|
+
}
|
|
1864
|
+
}
|
|
1865
|
+
return tableIdsByDatasetId;
|
|
1866
|
+
}
|
|
1867
|
+
async resolveLatestSnapshotRows(runtime, tableIdsByDatasetId) {
|
|
1868
|
+
const result = new Map;
|
|
1869
|
+
if (tableIdsByDatasetId.size === 0) {
|
|
1870
|
+
return result;
|
|
1871
|
+
}
|
|
1872
|
+
const unresolved = new Map(tableIdsByDatasetId);
|
|
1873
|
+
const tableIds = [...new Set(tableIdsByDatasetId.values())];
|
|
1874
|
+
let beforeSnapshotId;
|
|
1875
|
+
let scanned = 0;
|
|
1876
|
+
while (unresolved.size > 0 && scanned < CATALOG_SNAPSHOT_SCAN_LIMIT) {
|
|
1877
|
+
const candidates = await this.queryCatalogSnapshotBatch(runtime, beforeSnapshotId, SNAPSHOT_ROW_BATCH_SIZE);
|
|
1878
|
+
if (candidates.length === 0) {
|
|
1879
|
+
break;
|
|
1880
|
+
}
|
|
1881
|
+
const fileFlags = await this.queryFileChangeFlags(runtime, candidates.map((candidate) => candidate.snapshotId), tableIds);
|
|
1882
|
+
for (const candidate of candidates) {
|
|
1883
|
+
scanned += 1;
|
|
1884
|
+
for (const [datasetId, tableId] of [...unresolved]) {
|
|
1885
|
+
const flags = fileFlags.get(fileChangeKey(candidate.snapshotId, tableId));
|
|
1886
|
+
const row = this.candidateToSnapshotRow(datasetId, tableId, {
|
|
1887
|
+
snapshotId: candidate.snapshotId,
|
|
1888
|
+
createdAt: candidate.createdAt,
|
|
1889
|
+
changesMade: candidate.changesMade,
|
|
1890
|
+
hasFileChange: flags?.hasFileChange ?? false,
|
|
1891
|
+
hasFileDeleteChange: flags?.hasFileDeleteChange ?? false,
|
|
1892
|
+
...candidate.metadata !== undefined ? { metadata: candidate.metadata } : {}
|
|
1893
|
+
});
|
|
1894
|
+
if (row) {
|
|
1895
|
+
result.set(datasetId, row);
|
|
1896
|
+
unresolved.delete(datasetId);
|
|
1897
|
+
}
|
|
1898
|
+
}
|
|
1899
|
+
if (unresolved.size === 0) {
|
|
1900
|
+
break;
|
|
1901
|
+
}
|
|
1902
|
+
}
|
|
1903
|
+
if (candidates.length < SNAPSHOT_ROW_BATCH_SIZE) {
|
|
1904
|
+
break;
|
|
1905
|
+
}
|
|
1906
|
+
beforeSnapshotId = candidates[candidates.length - 1]?.snapshotId;
|
|
1907
|
+
}
|
|
1908
|
+
return result;
|
|
1909
|
+
}
|
|
1910
|
+
async queryCatalogSnapshotBatch(runtime, beforeSnapshotId, limit) {
|
|
1911
|
+
const ducklakeSnapshot = duckLakeMetadataTableName(this.options, "ducklake_snapshot");
|
|
1912
|
+
const ducklakeSnapshotChanges = duckLakeMetadataTableName(this.options, "ducklake_snapshot_changes");
|
|
1913
|
+
let whereSql = "";
|
|
1914
|
+
if (beforeSnapshotId !== undefined) {
|
|
1915
|
+
assertDuckLakeSnapshotId2(beforeSnapshotId);
|
|
1916
|
+
whereSql = `WHERE snapshot.snapshot_id < ${beforeSnapshotId}`;
|
|
1917
|
+
}
|
|
1918
|
+
const rows = await runtime.query(`
|
|
1919
|
+
SELECT
|
|
1920
|
+
snapshot.snapshot_id,
|
|
1921
|
+
snapshot.snapshot_time,
|
|
1922
|
+
changes.changes_made,
|
|
1923
|
+
changes.commit_extra_info
|
|
1924
|
+
FROM ${ducklakeSnapshot} snapshot
|
|
1925
|
+
JOIN ${ducklakeSnapshotChanges} changes ON changes.snapshot_id = snapshot.snapshot_id
|
|
1926
|
+
${whereSql}
|
|
1927
|
+
ORDER BY snapshot.snapshot_id DESC
|
|
1928
|
+
LIMIT ${Math.max(0, Math.trunc(limit))}
|
|
1929
|
+
`);
|
|
1930
|
+
return rows.map((row) => {
|
|
1931
|
+
const metadata = parseCommitMetadata(getOptionalString(row, "commit_extra_info"));
|
|
1932
|
+
return {
|
|
1933
|
+
snapshotId: String(getBigIntLike(row, "snapshot_id")),
|
|
1934
|
+
createdAt: getDate(row, "snapshot_time"),
|
|
1935
|
+
changesMade: getString(row, "changes_made"),
|
|
1936
|
+
...metadata !== undefined ? { metadata } : {}
|
|
1937
|
+
};
|
|
1938
|
+
});
|
|
1939
|
+
}
|
|
1940
|
+
async queryFileChangeFlags(runtime, snapshotIds, tableIds) {
|
|
1941
|
+
const flags = new Map;
|
|
1942
|
+
if (snapshotIds.length === 0 || tableIds.length === 0) {
|
|
1943
|
+
return flags;
|
|
1944
|
+
}
|
|
1945
|
+
for (const snapshotId of snapshotIds) {
|
|
1946
|
+
assertDuckLakeSnapshotId2(snapshotId);
|
|
1947
|
+
}
|
|
1948
|
+
const snapshotIdList = snapshotIds.join(", ");
|
|
1949
|
+
const tableIdList = tableIds.map((tableId) => tableId.toString()).join(", ");
|
|
1950
|
+
const ducklakeDataFile = duckLakeMetadataTableName(this.options, "ducklake_data_file");
|
|
1951
|
+
const ducklakeDeleteFile = duckLakeMetadataTableName(this.options, "ducklake_delete_file");
|
|
1952
|
+
const rows = await runtime.query(`
|
|
1953
|
+
WITH file_changes AS (
|
|
1954
|
+
SELECT begin_snapshot AS snapshot_id, table_id, false AS is_delete
|
|
1955
|
+
FROM ${ducklakeDataFile}
|
|
1956
|
+
WHERE table_id IN (${tableIdList}) AND begin_snapshot IN (${snapshotIdList})
|
|
1957
|
+
UNION ALL
|
|
1958
|
+
SELECT end_snapshot, table_id, true
|
|
1959
|
+
FROM ${ducklakeDataFile}
|
|
1960
|
+
WHERE table_id IN (${tableIdList}) AND end_snapshot IN (${snapshotIdList})
|
|
1961
|
+
UNION ALL
|
|
1962
|
+
SELECT begin_snapshot, table_id, true
|
|
1963
|
+
FROM ${ducklakeDeleteFile}
|
|
1964
|
+
WHERE table_id IN (${tableIdList}) AND begin_snapshot IN (${snapshotIdList})
|
|
1965
|
+
UNION ALL
|
|
1966
|
+
SELECT end_snapshot, table_id, true
|
|
1967
|
+
FROM ${ducklakeDeleteFile}
|
|
1968
|
+
WHERE table_id IN (${tableIdList}) AND end_snapshot IN (${snapshotIdList})
|
|
1969
|
+
)
|
|
1970
|
+
SELECT snapshot_id, table_id, bool_or(is_delete) AS has_delete
|
|
1971
|
+
FROM file_changes
|
|
1972
|
+
GROUP BY snapshot_id, table_id
|
|
1973
|
+
`);
|
|
1974
|
+
for (const row of rows) {
|
|
1975
|
+
const key = fileChangeKey(String(getBigIntLike(row, "snapshot_id")), getBigIntLike(row, "table_id"));
|
|
1976
|
+
flags.set(key, { hasFileChange: true, hasFileDeleteChange: getBoolean(row, "has_delete") });
|
|
1977
|
+
}
|
|
1978
|
+
return flags;
|
|
1979
|
+
}
|
|
1980
|
+
assertNoMetadataConflict(snapshotId, datasetId, metadata) {
|
|
1981
|
+
if (metadata !== undefined && metadata.datasetId !== datasetId) {
|
|
1982
|
+
throw new LakeStorageError15(`[SixbDuckLake] DuckLake snapshot '${snapshotId}' changed dataset '${datasetId}' but Sixb commit metadata references dataset '${metadata.datasetId}'.`);
|
|
1983
|
+
}
|
|
1984
|
+
}
|
|
1985
|
+
async getLatestVersionForDefinition(runtime, dataset) {
|
|
1986
|
+
const tableRef = await resolveDatasetTableRef(this.options, runtime, dataset.id);
|
|
1987
|
+
return tableRef ? this.getLatestVersionForTableRef(runtime, tableRef) : null;
|
|
1988
|
+
}
|
|
1989
|
+
async getLatestVersionRefForDefinition(runtime, dataset) {
|
|
1990
|
+
const summary = await this.getLatestVersionSummaryForDefinition(runtime, dataset);
|
|
1991
|
+
return summary ? { datasetId: summary.datasetId, versionId: summary.versionId } : null;
|
|
1992
|
+
}
|
|
1993
|
+
async getLatestVersionSummaryForDefinition(runtime, dataset) {
|
|
1994
|
+
const tableRef = await resolveDatasetTableRef(this.options, runtime, dataset.id);
|
|
1995
|
+
if (!tableRef) {
|
|
1996
|
+
return null;
|
|
1997
|
+
}
|
|
1998
|
+
const row = await this.getLatestSnapshotRowForTableRef(runtime, tableRef);
|
|
1999
|
+
if (!row) {
|
|
2000
|
+
return null;
|
|
2001
|
+
}
|
|
2002
|
+
return {
|
|
2003
|
+
datasetId: tableRef.datasetId,
|
|
2004
|
+
versionId: toVersionId(row.snapshotId),
|
|
2005
|
+
...row.metadata?.rowCount !== undefined ? { rowCount: row.metadata.rowCount } : {}
|
|
2006
|
+
};
|
|
2007
|
+
}
|
|
2008
|
+
async getVersionForSnapshot(runtime, dataset, snapshotId) {
|
|
2009
|
+
assertDuckLakeSnapshotId2(snapshotId);
|
|
2010
|
+
const tableRef = await resolveDatasetTableRef(this.options, runtime, dataset.id);
|
|
2011
|
+
return tableRef ? this.getVersionForTableRef(runtime, tableRef, snapshotId) : null;
|
|
2012
|
+
}
|
|
2013
|
+
async getVersionForTableRef(runtime, tableRef, snapshotId) {
|
|
2014
|
+
const match = await this.getSnapshotRowForTableRef(runtime, tableRef, snapshotId, {
|
|
2015
|
+
includeParent: true
|
|
2016
|
+
});
|
|
2017
|
+
return match ? this.snapshotToVersion(runtime, tableRef, match) : null;
|
|
2018
|
+
}
|
|
2019
|
+
async getVersionRefForSnapshot(runtime, dataset, snapshotId) {
|
|
2020
|
+
assertDuckLakeSnapshotId2(snapshotId);
|
|
2021
|
+
const tableRef = await resolveDatasetTableRef(this.options, runtime, dataset.id);
|
|
2022
|
+
if (!tableRef) {
|
|
2023
|
+
return null;
|
|
2024
|
+
}
|
|
2025
|
+
const row = await this.getSnapshotRowForTableRef(runtime, tableRef, snapshotId, {
|
|
2026
|
+
includeParent: false
|
|
2027
|
+
});
|
|
2028
|
+
return row ? { datasetId: tableRef.datasetId, versionId: toVersionId(row.snapshotId) } : null;
|
|
2029
|
+
}
|
|
2030
|
+
async listVersionsForDefinition(runtime, dataset, limit) {
|
|
2031
|
+
const tableRef = await resolveDatasetTableRef(this.options, runtime, dataset.id);
|
|
2032
|
+
return tableRef ? this.listVersionsForTableRef(runtime, tableRef, limit) : [];
|
|
2033
|
+
}
|
|
2034
|
+
async listVersionsForTableRef(runtime, tableRef, limit) {
|
|
2035
|
+
const limitedRows = await this.getSnapshotRowsForTableRef(runtime, tableRef, limit);
|
|
2036
|
+
const versions = [];
|
|
2037
|
+
for (const row of limitedRows) {
|
|
2038
|
+
versions.push(await this.snapshotToVersion(runtime, tableRef, row));
|
|
2039
|
+
}
|
|
2040
|
+
return versions;
|
|
2041
|
+
}
|
|
2042
|
+
async getLatestVersionForTableRef(runtime, tableRef) {
|
|
2043
|
+
const [latest] = await this.listVersionsForTableRef(runtime, tableRef, 1);
|
|
2044
|
+
return latest ?? null;
|
|
2045
|
+
}
|
|
2046
|
+
async getLatestSnapshotRowForTableRef(runtime, tableRef) {
|
|
2047
|
+
const [row] = await this.collectVisibleSnapshotRows(runtime, {
|
|
2048
|
+
datasetId: tableRef.datasetId,
|
|
2049
|
+
tableId: tableRef.tableId,
|
|
2050
|
+
visibleRowLimit: 1
|
|
2051
|
+
});
|
|
2052
|
+
return row ?? null;
|
|
2053
|
+
}
|
|
2054
|
+
async getSnapshotRowForTableRef(runtime, tableRef, snapshotId, options) {
|
|
2055
|
+
assertDuckLakeSnapshotId2(snapshotId);
|
|
2056
|
+
const [row] = await this.collectVisibleSnapshotRows(runtime, {
|
|
2057
|
+
datasetId: tableRef.datasetId,
|
|
2058
|
+
tableId: tableRef.tableId,
|
|
2059
|
+
exactSnapshotId: snapshotId,
|
|
2060
|
+
visibleRowLimit: 1
|
|
2061
|
+
});
|
|
2062
|
+
if (!row) {
|
|
2063
|
+
return null;
|
|
2064
|
+
}
|
|
2065
|
+
if (!options.includeParent || row.mode !== "append" && row.mode !== "schema") {
|
|
2066
|
+
return row;
|
|
2067
|
+
}
|
|
2068
|
+
const [parent] = await this.collectVisibleSnapshotRows(runtime, {
|
|
2069
|
+
datasetId: tableRef.datasetId,
|
|
2070
|
+
tableId: tableRef.tableId,
|
|
2071
|
+
beforeSnapshotId: snapshotId,
|
|
2072
|
+
visibleRowLimit: 1
|
|
2073
|
+
});
|
|
2074
|
+
return parent ? { ...row, parentSnapshotId: parent.snapshotId } : row;
|
|
2075
|
+
}
|
|
2076
|
+
async getSnapshotRowsForTableRef(runtime, tableRef, limit) {
|
|
2077
|
+
const visibleLimit = limit === undefined ? undefined : Math.max(0, limit);
|
|
2078
|
+
if (visibleLimit === 0) {
|
|
2079
|
+
return [];
|
|
2080
|
+
}
|
|
2081
|
+
const rows = await this.collectVisibleSnapshotRows(runtime, {
|
|
2082
|
+
datasetId: tableRef.datasetId,
|
|
2083
|
+
tableId: tableRef.tableId,
|
|
2084
|
+
visibleRowLimit: visibleLimit === undefined ? undefined : visibleLimit + 1
|
|
2085
|
+
});
|
|
2086
|
+
const rowsWithParents = this.withParentSnapshotIds(rows);
|
|
2087
|
+
return visibleLimit === undefined ? rowsWithParents : rowsWithParents.slice(0, visibleLimit);
|
|
2088
|
+
}
|
|
2089
|
+
async collectVisibleSnapshotRows(runtime, input) {
|
|
2090
|
+
if (input.visibleRowLimit !== undefined && input.visibleRowLimit <= 0) {
|
|
2091
|
+
return [];
|
|
2092
|
+
}
|
|
2093
|
+
const snapshots = [];
|
|
2094
|
+
let beforeSnapshotId = input.beforeSnapshotId;
|
|
2095
|
+
while (input.visibleRowLimit === undefined || snapshots.length < input.visibleRowLimit) {
|
|
2096
|
+
const candidates = await this.querySnapshotCandidates(runtime, {
|
|
2097
|
+
tableId: input.tableId,
|
|
2098
|
+
exactSnapshotId: input.exactSnapshotId,
|
|
2099
|
+
beforeSnapshotId,
|
|
2100
|
+
limit: input.exactSnapshotId === undefined ? SNAPSHOT_ROW_BATCH_SIZE : 1
|
|
2101
|
+
});
|
|
2102
|
+
if (candidates.length === 0) {
|
|
2103
|
+
break;
|
|
2104
|
+
}
|
|
2105
|
+
for (const candidate of candidates) {
|
|
2106
|
+
const snapshot = this.candidateToSnapshotRow(input.datasetId, input.tableId, candidate);
|
|
2107
|
+
if (snapshot) {
|
|
2108
|
+
snapshots.push(snapshot);
|
|
2109
|
+
}
|
|
2110
|
+
if (input.visibleRowLimit !== undefined && snapshots.length >= input.visibleRowLimit) {
|
|
2111
|
+
break;
|
|
2112
|
+
}
|
|
2113
|
+
}
|
|
2114
|
+
if (input.exactSnapshotId !== undefined || candidates.length < SNAPSHOT_ROW_BATCH_SIZE) {
|
|
2115
|
+
break;
|
|
2116
|
+
}
|
|
2117
|
+
beforeSnapshotId = candidates[candidates.length - 1]?.snapshotId;
|
|
2118
|
+
}
|
|
2119
|
+
return snapshots;
|
|
2120
|
+
}
|
|
2121
|
+
async querySnapshotCandidates(runtime, input) {
|
|
2122
|
+
if (input.limit !== undefined && input.limit <= 0) {
|
|
2123
|
+
return [];
|
|
2124
|
+
}
|
|
2125
|
+
const where = [];
|
|
2126
|
+
if (input.exactSnapshotId !== undefined) {
|
|
2127
|
+
assertDuckLakeSnapshotId2(input.exactSnapshotId);
|
|
2128
|
+
where.push(`snapshot.snapshot_id = ${input.exactSnapshotId}`);
|
|
2129
|
+
}
|
|
2130
|
+
if (input.beforeSnapshotId !== undefined) {
|
|
2131
|
+
assertDuckLakeSnapshotId2(input.beforeSnapshotId);
|
|
2132
|
+
where.push(`snapshot.snapshot_id < ${input.beforeSnapshotId}`);
|
|
2133
|
+
}
|
|
2134
|
+
const whereSql = where.length === 0 ? "" : `WHERE ${where.join(" AND ")}`;
|
|
2135
|
+
const limitSql = input.limit === undefined ? "" : `LIMIT ${Math.max(0, Math.trunc(input.limit))}`;
|
|
2136
|
+
const ducklakeSnapshot = duckLakeMetadataTableName(this.options, "ducklake_snapshot");
|
|
2137
|
+
const ducklakeSnapshotChanges = duckLakeMetadataTableName(this.options, "ducklake_snapshot_changes");
|
|
2138
|
+
const ducklakeDataFile = duckLakeMetadataTableName(this.options, "ducklake_data_file");
|
|
2139
|
+
const ducklakeDeleteFile = duckLakeMetadataTableName(this.options, "ducklake_delete_file");
|
|
2140
|
+
const rows = await runtime.query(`
|
|
2141
|
+
WITH candidate_snapshots AS (
|
|
2142
|
+
-- Recent DuckLake snapshots to inspect; Sixb visibility is filtered later.
|
|
2143
|
+
SELECT
|
|
2144
|
+
snapshot.snapshot_id,
|
|
2145
|
+
snapshot.snapshot_time,
|
|
2146
|
+
changes.changes_made,
|
|
2147
|
+
changes.commit_extra_info
|
|
2148
|
+
FROM ${ducklakeSnapshot} snapshot
|
|
2149
|
+
JOIN ${ducklakeSnapshotChanges} changes
|
|
2150
|
+
ON changes.snapshot_id = snapshot.snapshot_id
|
|
2151
|
+
${whereSql}
|
|
2152
|
+
ORDER BY snapshot.snapshot_id DESC
|
|
2153
|
+
${limitSql}
|
|
2154
|
+
),
|
|
2155
|
+
file_changes AS (
|
|
2156
|
+
-- File metadata tells whether this table changed in each candidate snapshot.
|
|
2157
|
+
SELECT begin_snapshot AS snapshot_id, false AS is_delete_change
|
|
2158
|
+
FROM ${ducklakeDataFile}
|
|
2159
|
+
WHERE table_id = ${input.tableId}
|
|
2160
|
+
AND begin_snapshot IN (SELECT snapshot_id FROM candidate_snapshots)
|
|
2161
|
+
UNION ALL
|
|
2162
|
+
SELECT end_snapshot AS snapshot_id, true AS is_delete_change
|
|
2163
|
+
FROM ${ducklakeDataFile}
|
|
2164
|
+
WHERE table_id = ${input.tableId}
|
|
2165
|
+
AND end_snapshot IN (SELECT snapshot_id FROM candidate_snapshots)
|
|
2166
|
+
UNION ALL
|
|
2167
|
+
SELECT begin_snapshot AS snapshot_id, true AS is_delete_change
|
|
2168
|
+
FROM ${ducklakeDeleteFile}
|
|
2169
|
+
WHERE table_id = ${input.tableId}
|
|
2170
|
+
AND begin_snapshot IN (SELECT snapshot_id FROM candidate_snapshots)
|
|
2171
|
+
UNION ALL
|
|
2172
|
+
SELECT end_snapshot AS snapshot_id, true AS is_delete_change
|
|
2173
|
+
FROM ${ducklakeDeleteFile}
|
|
2174
|
+
WHERE table_id = ${input.tableId}
|
|
2175
|
+
AND end_snapshot IN (SELECT snapshot_id FROM candidate_snapshots)
|
|
2176
|
+
),
|
|
2177
|
+
file_changes_by_snapshot AS (
|
|
2178
|
+
-- Collapse file-level changes into one row per snapshot.
|
|
2179
|
+
SELECT
|
|
2180
|
+
snapshot_id,
|
|
2181
|
+
count(*) > 0 AS has_file_change,
|
|
2182
|
+
count(*) FILTER (WHERE is_delete_change) > 0 AS has_file_delete_change
|
|
2183
|
+
FROM file_changes
|
|
2184
|
+
GROUP BY snapshot_id
|
|
2185
|
+
)
|
|
2186
|
+
SELECT
|
|
2187
|
+
-- Keep metadata-only candidates; Sixb filters them with commit_extra_info.
|
|
2188
|
+
candidate.snapshot_id,
|
|
2189
|
+
candidate.snapshot_time,
|
|
2190
|
+
candidate.changes_made,
|
|
2191
|
+
candidate.commit_extra_info,
|
|
2192
|
+
coalesce(file_changes.has_file_change, false) AS has_file_change,
|
|
2193
|
+
coalesce(file_changes.has_file_delete_change, false) AS has_file_delete_change
|
|
2194
|
+
FROM candidate_snapshots candidate
|
|
2195
|
+
LEFT JOIN file_changes_by_snapshot file_changes
|
|
2196
|
+
ON file_changes.snapshot_id = candidate.snapshot_id
|
|
2197
|
+
ORDER BY candidate.snapshot_id DESC
|
|
2198
|
+
`);
|
|
2199
|
+
return rows.map((row) => {
|
|
2200
|
+
const metadata = parseCommitMetadata(row.commit_extra_info);
|
|
2201
|
+
return {
|
|
2202
|
+
snapshotId: String(getBigIntLike(row, "snapshot_id")),
|
|
2203
|
+
createdAt: getDate(row, "snapshot_time"),
|
|
2204
|
+
changesMade: getString(row, "changes_made"),
|
|
2205
|
+
hasFileChange: getBoolean(row, "has_file_change"),
|
|
2206
|
+
hasFileDeleteChange: getBoolean(row, "has_file_delete_change"),
|
|
2207
|
+
...metadata !== undefined ? { metadata } : {}
|
|
2208
|
+
};
|
|
2209
|
+
});
|
|
2210
|
+
}
|
|
2211
|
+
candidateToSnapshotRow(datasetId, tableId, candidate) {
|
|
2212
|
+
const inlineChange = parseInlineDataChange(candidate.changesMade, tableId);
|
|
2213
|
+
const hasDataChange = inlineChange.hasDataChange || candidate.hasFileChange;
|
|
2214
|
+
const hasDeleteChange = inlineChange.hasDeleteChange || candidate.hasFileDeleteChange;
|
|
2215
|
+
if (!hasDataChange) {
|
|
2216
|
+
const metadata = candidate.metadata;
|
|
2217
|
+
if (!metadata || metadata.datasetId !== datasetId) {
|
|
2218
|
+
return null;
|
|
2219
|
+
}
|
|
2220
|
+
return {
|
|
2221
|
+
snapshotId: candidate.snapshotId,
|
|
2222
|
+
tableId,
|
|
2223
|
+
createdAt: candidate.createdAt,
|
|
2224
|
+
mode: metadata.mode ?? "schema",
|
|
2225
|
+
metadata
|
|
2226
|
+
};
|
|
2227
|
+
}
|
|
2228
|
+
this.assertNoMetadataConflict(candidate.snapshotId, datasetId, candidate.metadata);
|
|
2229
|
+
return {
|
|
2230
|
+
snapshotId: candidate.snapshotId,
|
|
2231
|
+
tableId,
|
|
2232
|
+
createdAt: candidate.createdAt,
|
|
2233
|
+
mode: candidate.metadata?.mode ?? (hasDeleteChange ? "snapshot" : "append"),
|
|
2234
|
+
...candidate.metadata !== undefined ? { metadata: candidate.metadata } : {}
|
|
2235
|
+
};
|
|
2236
|
+
}
|
|
2237
|
+
async snapshotToVersion(runtime, tableRef, row) {
|
|
2238
|
+
const schemaAtSnapshot = await this.datasets.getDatasetSchemaAtSnapshot(runtime, tableRef.datasetId, tableRef.tableName, tableRef.tableId, row.snapshotId);
|
|
2239
|
+
return {
|
|
2240
|
+
datasetId: tableRef.datasetId,
|
|
2241
|
+
versionId: toVersionId(row.snapshotId),
|
|
2242
|
+
parentVersionId: row.parentSnapshotId === undefined ? undefined : toVersionId(row.parentSnapshotId),
|
|
2243
|
+
mode: row.mode,
|
|
2244
|
+
createdAt: row.createdAt,
|
|
2245
|
+
schema: schemaAtSnapshot,
|
|
2246
|
+
producer: row.metadata?.producer,
|
|
2247
|
+
inputs: row.metadata?.inputs,
|
|
2248
|
+
...row.metadata?.rowCount !== undefined ? { rowCount: row.metadata.rowCount } : {}
|
|
2249
|
+
};
|
|
2250
|
+
}
|
|
2251
|
+
withParentSnapshotIds(snapshots) {
|
|
2252
|
+
return snapshots.map((snapshot, index) => {
|
|
2253
|
+
const parentSnapshotId = snapshot.mode === "append" || snapshot.mode === "schema" ? snapshots[index + 1]?.snapshotId : undefined;
|
|
2254
|
+
return parentSnapshotId === undefined ? snapshot : { ...snapshot, parentSnapshotId };
|
|
2255
|
+
});
|
|
2256
|
+
}
|
|
2257
|
+
}
|
|
2258
|
+
function assertDuckLakeSnapshotId2(snapshotId) {
|
|
2259
|
+
if (!/^\d+$/.test(snapshotId)) {
|
|
2260
|
+
throw new LakeStorageError15(`[SixbDuckLake] Invalid DuckLake snapshot id '${snapshotId}'.`);
|
|
2261
|
+
}
|
|
2262
|
+
}
|
|
2263
|
+
function fileChangeKey(snapshotId, tableId) {
|
|
2264
|
+
return `${snapshotId}|${tableId}`;
|
|
2265
|
+
}
|
|
2266
|
+
|
|
2267
|
+
// src/internal/ducklake-sql-executor.ts
|
|
2268
|
+
import { randomUUID as randomUUID3 } from "crypto";
|
|
2269
|
+
import {
|
|
2270
|
+
LakeStorageError as LakeStorageError18
|
|
2271
|
+
} from "@sixb/core/lake-storage";
|
|
2272
|
+
|
|
2273
|
+
// src/internal/dataset-row-commit.ts
|
|
2274
|
+
import { LakeStorageError as LakeStorageError16 } from "@sixb/core/lake-storage";
|
|
2275
|
+
function assertDatasetWriteMode(mode, operation) {
|
|
2276
|
+
if (mode !== "snapshot" && mode !== "append") {
|
|
2277
|
+
throw new LakeStorageError16(`[SixbDuckLake] Invalid ${operation} mode '${String(mode)}'. Expected 'snapshot' or 'append'.`);
|
|
2278
|
+
}
|
|
2279
|
+
}
|
|
2280
|
+
async function applyDatasetRowsFromRelation(input) {
|
|
2281
|
+
const tableName = encodeDatasetTableName(input.dataset.id);
|
|
2282
|
+
const table = qualifiedTableName(input.options, tableName);
|
|
2283
|
+
const columnsSql = datasetSchemaColumnNamesSql(input.dataset.schema);
|
|
2284
|
+
if (input.mode === "snapshot") {
|
|
2285
|
+
const source = await fingerprintRelation(input.runtime, input.sourceRelationSql, columnsSql);
|
|
2286
|
+
const target = await fingerprintRelation(input.runtime, table, columnsSql);
|
|
2287
|
+
if (sameFingerprint(source, target)) {
|
|
2288
|
+
return {
|
|
2289
|
+
dataChangeExpected: false,
|
|
2290
|
+
sourceRowCount: source.rowCount,
|
|
2291
|
+
resultingRowCount: { kind: "exact", value: target.rowCount }
|
|
2292
|
+
};
|
|
2293
|
+
}
|
|
2294
|
+
await input.runtime.run(`DELETE FROM ${table}`);
|
|
2295
|
+
await input.runtime.run(`INSERT INTO ${table} (${columnsSql}) SELECT ${columnsSql} FROM ${input.sourceRelationSql}`);
|
|
2296
|
+
return {
|
|
2297
|
+
dataChangeExpected: source.rowCount > 0 || target.rowCount > 0,
|
|
2298
|
+
sourceRowCount: source.rowCount,
|
|
2299
|
+
resultingRowCount: { kind: "exact", value: source.rowCount }
|
|
2300
|
+
};
|
|
2301
|
+
}
|
|
2302
|
+
const sourceRowCount = await countRows(input.runtime, input.sourceRelationSql);
|
|
2303
|
+
await input.runtime.run(`INSERT INTO ${table} (${columnsSql}) SELECT ${columnsSql} FROM ${input.sourceRelationSql}`);
|
|
2304
|
+
return {
|
|
2305
|
+
dataChangeExpected: sourceRowCount > 0,
|
|
2306
|
+
sourceRowCount,
|
|
2307
|
+
resultingRowCount: input.previousRowCount === undefined ? { kind: "unknown" } : { kind: "exact", value: input.previousRowCount + sourceRowCount }
|
|
2308
|
+
};
|
|
2309
|
+
}
|
|
2310
|
+
async function fingerprintRelation(runtime, relationSql, columnsSql) {
|
|
2311
|
+
const [row] = await runtime.query(`
|
|
2312
|
+
SELECT
|
|
2313
|
+
count(*) AS row_count,
|
|
2314
|
+
coalesce(sum(hash(${columnsSql})::HUGEINT), 0::HUGEINT) AS hash_sum,
|
|
2315
|
+
coalesce(bit_xor(hash('sixb:snapshot:v1', ${columnsSql})), 0::UBIGINT) AS hash_xor
|
|
2316
|
+
FROM ${relationSql}
|
|
2317
|
+
`);
|
|
2318
|
+
if (row === undefined) {
|
|
2319
|
+
return { rowCount: 0, hashSum: "0", hashXor: "0" };
|
|
2320
|
+
}
|
|
2321
|
+
return {
|
|
2322
|
+
rowCount: Number(getBigIntLike(row, "row_count")),
|
|
2323
|
+
hashSum: getBigIntLike(row, "hash_sum").toString(),
|
|
2324
|
+
hashXor: getBigIntLike(row, "hash_xor").toString()
|
|
2325
|
+
};
|
|
2326
|
+
}
|
|
2327
|
+
function sameFingerprint(left, right) {
|
|
2328
|
+
return left.rowCount === right.rowCount && left.hashSum === right.hashSum && left.hashXor === right.hashXor;
|
|
2329
|
+
}
|
|
2330
|
+
async function countRows(runtime, relationSql) {
|
|
2331
|
+
const [row] = await runtime.query(`SELECT count(*) AS row_count FROM ${relationSql}`);
|
|
2332
|
+
if (row === undefined) {
|
|
2333
|
+
return 0;
|
|
2334
|
+
}
|
|
2335
|
+
return Number(getBigIntLike(row, "row_count"));
|
|
2336
|
+
}
|
|
2337
|
+
|
|
2338
|
+
// src/internal/sql-transform-relations.ts
|
|
2339
|
+
import { randomUUID as randomUUID2 } from "crypto";
|
|
2340
|
+
import {
|
|
2341
|
+
LakeStorageError as LakeStorageError17
|
|
2342
|
+
} from "@sixb/core/lake-storage";
|
|
2343
|
+
var RELATION_PLACEHOLDER_PREFIX = "__sixb_sql_transform_relation_";
|
|
2344
|
+
var RELATION_PLACEHOLDER_PATTERN = /__sixb_sql_transform_relation_[0-9a-f]{32}_[0-9]+__/g;
|
|
2345
|
+
|
|
2346
|
+
class DuckLakeSqlTransformRelation {
|
|
2347
|
+
placeholder;
|
|
2348
|
+
constructor(placeholder) {
|
|
2349
|
+
this.placeholder = placeholder;
|
|
2350
|
+
}
|
|
2351
|
+
toString() {
|
|
2352
|
+
return this.placeholder;
|
|
2353
|
+
}
|
|
2354
|
+
}
|
|
2355
|
+
function renderDuckLakeSqlTransformSql(input) {
|
|
2356
|
+
const context = createDuckLakeSqlTransformRelationContext(input);
|
|
2357
|
+
const sql = input.sql(context.relations);
|
|
2358
|
+
if (typeof sql !== "string") {
|
|
2359
|
+
throw new LakeStorageError17("[SixbDuckLake] SQL transform body must return a SQL string.");
|
|
2360
|
+
}
|
|
2361
|
+
return context.render(sql);
|
|
2362
|
+
}
|
|
2363
|
+
function createDuckLakeSqlTransformRelationContext(input) {
|
|
2364
|
+
const token = randomUUID2().replaceAll("-", "");
|
|
2365
|
+
const relations = Object.create(null);
|
|
2366
|
+
const replacements = new Map;
|
|
2367
|
+
let index = 0;
|
|
2368
|
+
for (const [sourceName, source] of Object.entries(input.sources)) {
|
|
2369
|
+
const placeholder = `${RELATION_PLACEHOLDER_PREFIX}${token}_${index}__`;
|
|
2370
|
+
relations[sourceName] = Object.freeze(new DuckLakeSqlTransformRelation(placeholder));
|
|
2371
|
+
replacements.set(placeholder, renderDuckLakeSourceRelation(input.options, source));
|
|
2372
|
+
index += 1;
|
|
2373
|
+
}
|
|
2374
|
+
return {
|
|
2375
|
+
relations: Object.freeze(relations),
|
|
2376
|
+
render(sql) {
|
|
2377
|
+
const rendered = sql.replace(RELATION_PLACEHOLDER_PATTERN, (placeholder) => {
|
|
2378
|
+
const replacement = replacements.get(placeholder);
|
|
2379
|
+
if (replacement === undefined) {
|
|
2380
|
+
throwUnresolvedPlaceholder();
|
|
2381
|
+
}
|
|
2382
|
+
return replacement;
|
|
2383
|
+
});
|
|
2384
|
+
if (rendered.includes(RELATION_PLACEHOLDER_PREFIX)) {
|
|
2385
|
+
throwUnresolvedPlaceholder();
|
|
2386
|
+
}
|
|
2387
|
+
return rendered;
|
|
2388
|
+
}
|
|
2389
|
+
};
|
|
2390
|
+
}
|
|
2391
|
+
function renderDuckLakeSourceRelation(options, source) {
|
|
2392
|
+
const snapshotId = parseVersionId(source.versionId);
|
|
2393
|
+
const tableName = encodeDatasetTableName(source.datasetId);
|
|
2394
|
+
return `(SELECT * FROM ${qualifiedTableName(options, tableName)} AT (VERSION => ${snapshotId}))`;
|
|
2395
|
+
}
|
|
2396
|
+
function throwUnresolvedPlaceholder() {
|
|
2397
|
+
throw new LakeStorageError17("[SixbDuckLake] SQL transform contains an unresolved relation placeholder.");
|
|
2398
|
+
}
|
|
2399
|
+
|
|
2400
|
+
// src/internal/ducklake-sql-executor.ts
|
|
2401
|
+
var DEFAULT_PREVIEW_LIMIT = 100;
|
|
2402
|
+
var MAX_PREVIEW_LIMIT = 1000;
|
|
2403
|
+
|
|
2404
|
+
class DuckLakeSqlExecutor {
|
|
2405
|
+
options;
|
|
2406
|
+
connections;
|
|
2407
|
+
datasets;
|
|
2408
|
+
snapshots;
|
|
2409
|
+
writes;
|
|
2410
|
+
dialect = "duckdb";
|
|
2411
|
+
capabilities = {
|
|
2412
|
+
preview: true,
|
|
2413
|
+
supportsAppend: true,
|
|
2414
|
+
supportsSnapshot: true
|
|
2415
|
+
};
|
|
2416
|
+
constructor(options, connections, datasets, snapshots, writes) {
|
|
2417
|
+
this.options = options;
|
|
2418
|
+
this.connections = connections;
|
|
2419
|
+
this.datasets = datasets;
|
|
2420
|
+
this.snapshots = snapshots;
|
|
2421
|
+
this.writes = writes;
|
|
2422
|
+
}
|
|
2423
|
+
async* preview(input) {
|
|
2424
|
+
this.connections.assertOpen();
|
|
2425
|
+
const rows = await this.connections.withAttachedRuntime(async (runtime) => {
|
|
2426
|
+
const sources = await this.resolveSources(runtime, input.sources);
|
|
2427
|
+
const sql = renderDuckLakeSqlTransformSql({
|
|
2428
|
+
options: this.options,
|
|
2429
|
+
sources: sources.relations,
|
|
2430
|
+
sql: input.sql
|
|
2431
|
+
});
|
|
2432
|
+
return runtime.query(buildPreviewSql(sql, previewLimit(input.limit)));
|
|
2433
|
+
});
|
|
2434
|
+
for (const row of rows) {
|
|
2435
|
+
yield normalizePreviewRow(row);
|
|
2436
|
+
}
|
|
2437
|
+
}
|
|
2438
|
+
async execute(input) {
|
|
2439
|
+
this.connections.assertOpen();
|
|
2440
|
+
assertDatasetWriteMode(input.mode, "SQL transform");
|
|
2441
|
+
return this.writes.withCommitRuntime(async (runtime) => {
|
|
2442
|
+
const target = await this.resolveTargetDataset(runtime, input.target);
|
|
2443
|
+
const sources = await this.resolveSources(runtime, input.sources);
|
|
2444
|
+
const sql = renderDuckLakeSqlTransformSql({
|
|
2445
|
+
options: this.options,
|
|
2446
|
+
sources: sources.relations,
|
|
2447
|
+
sql: input.sql
|
|
2448
|
+
});
|
|
2449
|
+
await this.assertResultSchemaMatchesTarget(runtime, target, sql);
|
|
2450
|
+
return this.writes.commitDatasetVersionOnExclusiveRuntime(runtime, {
|
|
2451
|
+
dataset: target,
|
|
2452
|
+
mode: input.mode,
|
|
2453
|
+
expectedLatestVersionId: input.expectedLatestVersionId,
|
|
2454
|
+
commitMessage: input.commitMessage ?? `execute SQL transform for dataset ${target.id}`,
|
|
2455
|
+
producer: input.producer,
|
|
2456
|
+
inputs: sources.inputs,
|
|
2457
|
+
applyChanges: (runtime2, context) => this.applySqlTransform({
|
|
2458
|
+
runtime: runtime2,
|
|
2459
|
+
target,
|
|
2460
|
+
mode: input.mode,
|
|
2461
|
+
sql,
|
|
2462
|
+
previousRowCount: context.previousRowCount
|
|
2463
|
+
})
|
|
2464
|
+
});
|
|
2465
|
+
});
|
|
2466
|
+
}
|
|
2467
|
+
async resolveSources(runtime, sources) {
|
|
2468
|
+
const relations = Object.create(null);
|
|
2469
|
+
const inputs = [];
|
|
2470
|
+
for (const [sourceName, source] of Object.entries(sources)) {
|
|
2471
|
+
const definition = await this.resolveSourceDataset(runtime, source.dataset);
|
|
2472
|
+
const versionRef = await this.resolveSourceVersion(runtime, definition, source.versionId);
|
|
2473
|
+
relations[sourceName] = {
|
|
2474
|
+
datasetId: definition.id,
|
|
2475
|
+
versionId: versionRef.versionId
|
|
2476
|
+
};
|
|
2477
|
+
inputs.push({
|
|
2478
|
+
datasetId: definition.id,
|
|
2479
|
+
versionId: versionRef.versionId
|
|
2480
|
+
});
|
|
2481
|
+
}
|
|
2482
|
+
return {
|
|
2483
|
+
relations: Object.freeze(relations),
|
|
2484
|
+
inputs: Object.freeze(inputs)
|
|
2485
|
+
};
|
|
2486
|
+
}
|
|
2487
|
+
async resolveSourceDataset(runtime, dataset) {
|
|
2488
|
+
const definition = await this.datasets.getDatasetOnRuntime(runtime, dataset.id);
|
|
2489
|
+
if (!definition) {
|
|
2490
|
+
throw new LakeStorageError18(`[SixbDuckLake] Unknown SQL transform source dataset '${dataset.id}'.`);
|
|
2491
|
+
}
|
|
2492
|
+
return definition;
|
|
2493
|
+
}
|
|
2494
|
+
async resolveTargetDataset(runtime, dataset) {
|
|
2495
|
+
const definition = await this.datasets.getDatasetOnRuntime(runtime, dataset.id);
|
|
2496
|
+
if (!definition) {
|
|
2497
|
+
throw new LakeStorageError18(`[SixbDuckLake] Unknown SQL transform target dataset '${dataset.id}'.`);
|
|
2498
|
+
}
|
|
2499
|
+
this.datasets.assertSchema(definition);
|
|
2500
|
+
return definition;
|
|
2501
|
+
}
|
|
2502
|
+
async resolveSourceVersion(runtime, dataset, versionId) {
|
|
2503
|
+
if (versionId !== undefined) {
|
|
2504
|
+
const snapshotId = parseVersionId(versionId);
|
|
2505
|
+
const versionRef = await this.snapshots.getVersionRefForSnapshot(runtime, dataset, snapshotId);
|
|
2506
|
+
if (versionRef) {
|
|
2507
|
+
return versionRef;
|
|
2508
|
+
}
|
|
2509
|
+
throwNoCommittedSourceVersion(dataset.id);
|
|
2510
|
+
}
|
|
2511
|
+
const latestVersionRef = await this.snapshots.getLatestVersionRefForDefinition(runtime, dataset);
|
|
2512
|
+
if (latestVersionRef) {
|
|
2513
|
+
return latestVersionRef;
|
|
2514
|
+
}
|
|
2515
|
+
throwNoCommittedSourceVersion(dataset.id);
|
|
2516
|
+
}
|
|
2517
|
+
async assertResultSchemaMatchesTarget(runtime, target, sql) {
|
|
2518
|
+
const rows = await runtime.query(`DESCRIBE SELECT * FROM (${sql}) AS sixb_sql_transform_result_schema`);
|
|
2519
|
+
const actualColumns = rows.map((row) => resultColumnFromDescribeRow(row));
|
|
2520
|
+
const expectedColumns = target.schema.columns;
|
|
2521
|
+
if (actualColumns.length !== expectedColumns.length) {
|
|
2522
|
+
throwResultSchemaMismatch(target.id, `expected ${expectedColumns.length} columns, got ${actualColumns.length}`);
|
|
2523
|
+
}
|
|
2524
|
+
for (let index = 0;index < expectedColumns.length; index += 1) {
|
|
2525
|
+
const expected = expectedColumns[index];
|
|
2526
|
+
const actual = actualColumns[index];
|
|
2527
|
+
if (actual.name !== expected.name) {
|
|
2528
|
+
throwResultSchemaMismatch(target.id, `column ${index + 1} should be '${expected.name}', got '${actual.name}'`);
|
|
2529
|
+
}
|
|
2530
|
+
if (actual.type !== expected.type) {
|
|
2531
|
+
throwResultSchemaMismatch(target.id, `column '${expected.name}' should have type '${expected.type}', got '${actual.type}'`);
|
|
2532
|
+
}
|
|
2533
|
+
}
|
|
2534
|
+
}
|
|
2535
|
+
async applySqlTransform(input) {
|
|
2536
|
+
const tempTableName = `sixb_sql_transform_${randomUUID3().replaceAll("-", "")}`;
|
|
2537
|
+
const tempTable = quoteIdentifier(tempTableName);
|
|
2538
|
+
await input.runtime.run(`CREATE TEMP TABLE ${tempTable} AS SELECT * FROM (${input.sql}) AS sixb_sql_transform_result`);
|
|
2539
|
+
let result;
|
|
2540
|
+
try {
|
|
2541
|
+
result = await applyDatasetRowsFromRelation({
|
|
2542
|
+
options: this.options,
|
|
2543
|
+
runtime: input.runtime,
|
|
2544
|
+
dataset: input.target,
|
|
2545
|
+
mode: input.mode,
|
|
2546
|
+
sourceRelationSql: tempTable,
|
|
2547
|
+
previousRowCount: input.previousRowCount
|
|
2548
|
+
});
|
|
2549
|
+
} catch (error) {
|
|
2550
|
+
try {
|
|
2551
|
+
await input.runtime.run(`DROP TABLE IF EXISTS ${tempTable}`);
|
|
2552
|
+
} catch {}
|
|
2553
|
+
throw error;
|
|
2554
|
+
}
|
|
2555
|
+
await input.runtime.run(`DROP TABLE IF EXISTS ${tempTable}`);
|
|
2556
|
+
return result;
|
|
2557
|
+
}
|
|
2558
|
+
}
|
|
2559
|
+
function previewLimit(limit) {
|
|
2560
|
+
if (limit === undefined || !Number.isFinite(limit)) {
|
|
2561
|
+
return DEFAULT_PREVIEW_LIMIT;
|
|
2562
|
+
}
|
|
2563
|
+
return Math.min(Math.max(0, Math.trunc(limit)), MAX_PREVIEW_LIMIT);
|
|
2564
|
+
}
|
|
2565
|
+
function buildPreviewSql(sql, limit) {
|
|
2566
|
+
return `SELECT * FROM (${sql}) AS sixb_sql_transform_preview LIMIT ${limit}`;
|
|
2567
|
+
}
|
|
2568
|
+
function resultColumnFromDescribeRow(row) {
|
|
2569
|
+
const name = getString(row, "column_name");
|
|
2570
|
+
const type = duckDbTypeToDatasetColumnType(getString(row, "column_type"));
|
|
2571
|
+
return { name, type };
|
|
2572
|
+
}
|
|
2573
|
+
function normalizePreviewRow(row) {
|
|
2574
|
+
const normalized = {};
|
|
2575
|
+
for (const [key, value] of Object.entries(row)) {
|
|
2576
|
+
normalized[key] = normalizePreviewValue(value);
|
|
2577
|
+
}
|
|
2578
|
+
return normalized;
|
|
2579
|
+
}
|
|
2580
|
+
function normalizePreviewValue(value) {
|
|
2581
|
+
if (typeof value === "bigint") {
|
|
2582
|
+
return value.toString();
|
|
2583
|
+
}
|
|
2584
|
+
if (value instanceof Date) {
|
|
2585
|
+
return value.toISOString();
|
|
2586
|
+
}
|
|
2587
|
+
if (Array.isArray(value)) {
|
|
2588
|
+
return value.map((item) => normalizePreviewValue(item));
|
|
2589
|
+
}
|
|
2590
|
+
if (typeof value === "object" && value !== null) {
|
|
2591
|
+
const normalized = {};
|
|
2592
|
+
for (const [key, child] of Object.entries(value)) {
|
|
2593
|
+
normalized[key] = normalizePreviewValue(child);
|
|
2594
|
+
}
|
|
2595
|
+
return normalized;
|
|
2596
|
+
}
|
|
2597
|
+
return value;
|
|
2598
|
+
}
|
|
2599
|
+
function throwNoCommittedSourceVersion(datasetId) {
|
|
2600
|
+
throw new LakeStorageError18(`[SixbDuckLake] No committed version found for SQL transform source dataset '${datasetId}'.`);
|
|
2601
|
+
}
|
|
2602
|
+
function throwResultSchemaMismatch(datasetId, detail) {
|
|
2603
|
+
throw new LakeStorageError18(`[SixbDuckLake] SQL transform result schema does not match target dataset '${datasetId}': ${detail}.`);
|
|
2604
|
+
}
|
|
2605
|
+
|
|
2606
|
+
// src/internal/ducklake-write-coordinator.ts
|
|
2607
|
+
import { randomUUID as randomUUID5 } from "crypto";
|
|
2608
|
+
import { LakeStorageError as LakeStorageError20 } from "@sixb/core/lake-storage";
|
|
2609
|
+
|
|
2610
|
+
// src/internal/ducklake-write-session.ts
|
|
2611
|
+
import { randomUUID as randomUUID4 } from "crypto";
|
|
2612
|
+
import { getDatasetRowValidationError } from "@sixb/core";
|
|
2613
|
+
import { LakeStorageError as LakeStorageError19 } from "@sixb/core/lake-storage";
|
|
2614
|
+
|
|
2615
|
+
// src/internal/row-appender.ts
|
|
2616
|
+
function appendDatasetRow(appender, schema, row) {
|
|
2617
|
+
for (const column of schema.columns) {
|
|
2618
|
+
appendColumnValue(appender, column, row[column.name] ?? null);
|
|
2619
|
+
}
|
|
2620
|
+
appender.endRow();
|
|
2621
|
+
}
|
|
2622
|
+
function appendColumnValue(appender, column, value) {
|
|
2623
|
+
if (value === null || value === undefined) {
|
|
2624
|
+
appender.appendNull();
|
|
2625
|
+
return;
|
|
2626
|
+
}
|
|
2627
|
+
switch (column.type) {
|
|
2628
|
+
case "string":
|
|
2629
|
+
appender.appendVarchar(value);
|
|
2630
|
+
return;
|
|
2631
|
+
case "boolean":
|
|
2632
|
+
appender.appendBoolean(value);
|
|
2633
|
+
return;
|
|
2634
|
+
case "int64":
|
|
2635
|
+
appender.appendBigInt(typeof value === "bigint" ? value : BigInt(value));
|
|
2636
|
+
return;
|
|
2637
|
+
case "float64":
|
|
2638
|
+
appender.appendDouble(value);
|
|
2639
|
+
return;
|
|
2640
|
+
case "decimal":
|
|
2641
|
+
appender.appendVarchar(normalizeDuckDbDecimalValue(value, column.name));
|
|
2642
|
+
return;
|
|
2643
|
+
case "date":
|
|
2644
|
+
appender.appendVarchar(value instanceof Date ? value.toISOString().slice(0, 10) : String(value));
|
|
2645
|
+
return;
|
|
2646
|
+
case "timestamp":
|
|
2647
|
+
appender.appendVarchar(value instanceof Date ? value.toISOString() : String(value));
|
|
2648
|
+
return;
|
|
2649
|
+
case "json":
|
|
2650
|
+
appender.appendVarchar(JSON.stringify(value));
|
|
2651
|
+
return;
|
|
2652
|
+
case "fileRef":
|
|
2653
|
+
appender.appendStruct(fileRefToDuckDbStruct(value));
|
|
2654
|
+
return;
|
|
2655
|
+
}
|
|
2656
|
+
}
|
|
2657
|
+
function fileRefToDuckDbStruct(fileRef) {
|
|
2658
|
+
return {
|
|
2659
|
+
blobId: fileRef.blobId,
|
|
2660
|
+
digest: fileRef.digest,
|
|
2661
|
+
sizeBytes: BigInt(fileRef.sizeBytes),
|
|
2662
|
+
fileName: fileRef.fileName ?? null,
|
|
2663
|
+
mediaType: fileRef.mediaType ?? null,
|
|
2664
|
+
logicalPath: fileRef.logicalPath ?? null
|
|
2665
|
+
};
|
|
2666
|
+
}
|
|
2667
|
+
|
|
2668
|
+
// src/internal/ducklake-write-session.ts
|
|
2669
|
+
var WRITE_ROW_BATCH_SIZE = 1000;
|
|
2670
|
+
async function createDuckLakeWriteSession(input) {
|
|
2671
|
+
const stagingTableName = `sixb_write_${randomUUID4().replaceAll("-", "")}`;
|
|
2672
|
+
await input.runtime.run(`CREATE TEMP TABLE ${quoteIdentifier(stagingTableName)} (${datasetSchemaToDuckDbColumnsSql(input.write.dataset.schema)})`);
|
|
2673
|
+
return new DuckLakeWriteSession(input.commitWrite, input.runtime, input.write, stagingTableName);
|
|
2674
|
+
}
|
|
2675
|
+
|
|
2676
|
+
class DuckLakeWriteSession {
|
|
2677
|
+
commitWrite;
|
|
2678
|
+
runtime;
|
|
2679
|
+
input;
|
|
2680
|
+
stagingTableName;
|
|
2681
|
+
closed = false;
|
|
2682
|
+
cleanedUp = false;
|
|
2683
|
+
rowsWritten = 0;
|
|
2684
|
+
constructor(commitWrite, runtime, input, stagingTableName) {
|
|
2685
|
+
this.commitWrite = commitWrite;
|
|
2686
|
+
this.runtime = runtime;
|
|
2687
|
+
this.input = input;
|
|
2688
|
+
this.stagingTableName = stagingTableName;
|
|
2689
|
+
}
|
|
2690
|
+
async writeRows(rows) {
|
|
2691
|
+
this.assertOpen();
|
|
2692
|
+
let batch = [];
|
|
2693
|
+
for await (const row of rows) {
|
|
2694
|
+
const validationError = getDatasetRowValidationError(row, this.input.dataset);
|
|
2695
|
+
if (validationError) {
|
|
2696
|
+
throw new LakeStorageError19(`[SixbDuckLake] ${validationError}`);
|
|
2697
|
+
}
|
|
2698
|
+
batch.push(structuredClone(row));
|
|
2699
|
+
if (batch.length >= WRITE_ROW_BATCH_SIZE) {
|
|
2700
|
+
await this.appendBatchToStagingTable(batch);
|
|
2701
|
+
batch = [];
|
|
2702
|
+
}
|
|
2703
|
+
}
|
|
2704
|
+
if (batch.length > 0) {
|
|
2705
|
+
await this.appendBatchToStagingTable(batch);
|
|
2706
|
+
}
|
|
2707
|
+
}
|
|
2708
|
+
async appendBatchToStagingTable(batch) {
|
|
2709
|
+
await this.runtime.withAppender(this.stagingTableName, (appender) => {
|
|
2710
|
+
for (const row of batch) {
|
|
2711
|
+
appendDatasetRow(appender, this.input.dataset.schema, row);
|
|
2712
|
+
}
|
|
2713
|
+
});
|
|
2714
|
+
this.rowsWritten += batch.length;
|
|
2715
|
+
}
|
|
2716
|
+
async commit(input) {
|
|
2717
|
+
this.assertOpen();
|
|
2718
|
+
this.closed = true;
|
|
2719
|
+
let committed = false;
|
|
2720
|
+
try {
|
|
2721
|
+
const result = await this.commitWrite({
|
|
2722
|
+
write: this.input,
|
|
2723
|
+
commit: input,
|
|
2724
|
+
rowsWritten: this.rowsWritten,
|
|
2725
|
+
stagingTableName: this.stagingTableName
|
|
2726
|
+
});
|
|
2727
|
+
committed = true;
|
|
2728
|
+
await this.cleanup();
|
|
2729
|
+
return result;
|
|
2730
|
+
} catch (error) {
|
|
2731
|
+
if (!committed) {
|
|
2732
|
+
await this.cleanup();
|
|
2733
|
+
}
|
|
2734
|
+
throw error;
|
|
2735
|
+
}
|
|
2736
|
+
}
|
|
2737
|
+
async abort() {
|
|
2738
|
+
if (this.closed) {
|
|
2739
|
+
return;
|
|
2740
|
+
}
|
|
2741
|
+
this.closed = true;
|
|
2742
|
+
await this.cleanup();
|
|
2743
|
+
}
|
|
2744
|
+
assertOpen() {
|
|
2745
|
+
if (this.closed) {
|
|
2746
|
+
throw new LakeStorageError19("[SixbDuckLake] Write session is already closed.");
|
|
2747
|
+
}
|
|
2748
|
+
}
|
|
2749
|
+
async cleanup() {
|
|
2750
|
+
if (this.cleanedUp) {
|
|
2751
|
+
return;
|
|
2752
|
+
}
|
|
2753
|
+
this.cleanedUp = true;
|
|
2754
|
+
await this.dropStagingTable();
|
|
2755
|
+
}
|
|
2756
|
+
async dropStagingTable() {
|
|
2757
|
+
try {
|
|
2758
|
+
await this.runtime.run(`DROP TABLE IF EXISTS ${quoteIdentifier(this.stagingTableName)}`);
|
|
2759
|
+
} catch {}
|
|
2760
|
+
}
|
|
2761
|
+
}
|
|
2762
|
+
|
|
2763
|
+
// src/internal/ducklake-write-coordinator.ts
|
|
2764
|
+
class DuckLakeWriteCoordinator {
|
|
2765
|
+
options;
|
|
2766
|
+
connections;
|
|
2767
|
+
datasets;
|
|
2768
|
+
snapshots;
|
|
2769
|
+
constructor(options, connections, datasets, snapshots) {
|
|
2770
|
+
this.options = options;
|
|
2771
|
+
this.connections = connections;
|
|
2772
|
+
this.datasets = datasets;
|
|
2773
|
+
this.snapshots = snapshots;
|
|
2774
|
+
}
|
|
2775
|
+
async beginWrite(input) {
|
|
2776
|
+
this.connections.assertOpen();
|
|
2777
|
+
const definition = await this.datasets.getDataset(input.dataset.id);
|
|
2778
|
+
if (!definition) {
|
|
2779
|
+
throw new LakeStorageError20(`[SixbDuckLake] Unknown dataset '${input.dataset.id}'.`);
|
|
2780
|
+
}
|
|
2781
|
+
this.datasets.assertSchema(definition);
|
|
2782
|
+
const mode = input.mode ?? "snapshot";
|
|
2783
|
+
assertDatasetWriteMode(mode, "write");
|
|
2784
|
+
const runtime = await this.connections.stagingRuntime();
|
|
2785
|
+
return createDuckLakeWriteSession({
|
|
2786
|
+
commitWrite: (options) => this.commitWrite(options),
|
|
2787
|
+
runtime,
|
|
2788
|
+
write: {
|
|
2789
|
+
...input,
|
|
2790
|
+
dataset: definition,
|
|
2791
|
+
mode
|
|
2792
|
+
}
|
|
2793
|
+
});
|
|
2794
|
+
}
|
|
2795
|
+
async commitWrite(input) {
|
|
2796
|
+
const definition = await this.datasets.getDataset(input.write.dataset.id);
|
|
2797
|
+
if (!definition) {
|
|
2798
|
+
throw new LakeStorageError20(`[SixbDuckLake] Unknown dataset '${input.write.dataset.id}'.`);
|
|
2799
|
+
}
|
|
2800
|
+
const mode = input.write.mode ?? "snapshot";
|
|
2801
|
+
assertDatasetWriteMode(mode, "write");
|
|
2802
|
+
return this.commitDatasetVersion({
|
|
2803
|
+
dataset: definition,
|
|
2804
|
+
mode,
|
|
2805
|
+
expectedLatestVersionId: input.commit?.expectedLatestVersionId,
|
|
2806
|
+
commitMessage: input.commit?.commitMessage ?? `write dataset ${input.write.dataset.id}`,
|
|
2807
|
+
producer: input.write.producer,
|
|
2808
|
+
inputs: input.write.inputs,
|
|
2809
|
+
applyChanges: (runtime, context) => this.applyStagedRows({
|
|
2810
|
+
...input,
|
|
2811
|
+
runtime,
|
|
2812
|
+
previousRowCount: context.previousRowCount
|
|
2813
|
+
})
|
|
2814
|
+
});
|
|
2815
|
+
}
|
|
2816
|
+
async commitDatasetVersion(input) {
|
|
2817
|
+
return this.withCommitRuntime((runtime) => this.commitDatasetVersionOnExclusiveRuntime(runtime, input));
|
|
2818
|
+
}
|
|
2819
|
+
async withCommitRuntime(run) {
|
|
2820
|
+
return withCatalogCommitLock(this.options, async () => {
|
|
2821
|
+
return this.connections.withExclusiveAttached(run);
|
|
2822
|
+
});
|
|
2823
|
+
}
|
|
2824
|
+
async commitDatasetVersionOnExclusiveRuntime(runtime, input) {
|
|
2825
|
+
const runtimeInput = { ...input, runtime };
|
|
2826
|
+
return this.withGuardedRetrySetting(runtimeInput, () => this.commitDatasetVersionUnlocked(runtimeInput));
|
|
2827
|
+
}
|
|
2828
|
+
async withGuardedRetrySetting(input, run) {
|
|
2829
|
+
if (input.expectedLatestVersionId === undefined) {
|
|
2830
|
+
return run();
|
|
2831
|
+
}
|
|
2832
|
+
await input.runtime.run("SET ducklake_max_retry_count = 0");
|
|
2833
|
+
let outcome;
|
|
2834
|
+
try {
|
|
2835
|
+
outcome = { kind: "success", value: await run() };
|
|
2836
|
+
} catch (error) {
|
|
2837
|
+
outcome = { kind: "error", error };
|
|
2838
|
+
}
|
|
2839
|
+
try {
|
|
2840
|
+
await input.runtime.run("RESET ducklake_max_retry_count");
|
|
2841
|
+
} catch {
|
|
2842
|
+
this.connections.poisonRuntime();
|
|
2843
|
+
}
|
|
2844
|
+
if (outcome.kind === "error") {
|
|
2845
|
+
throw outcome.error;
|
|
2846
|
+
}
|
|
2847
|
+
return outcome.value;
|
|
2848
|
+
}
|
|
2849
|
+
async commitDatasetVersionUnlocked(input) {
|
|
2850
|
+
const previousWriteSnapshotId = await this.lastCommittedSnapshotId(input.runtime);
|
|
2851
|
+
await input.runtime.run("BEGIN TRANSACTION");
|
|
2852
|
+
let committed = false;
|
|
2853
|
+
try {
|
|
2854
|
+
const latestVersion = await this.latestVersionForCommit(input);
|
|
2855
|
+
if (input.expectedLatestVersionId !== undefined) {
|
|
2856
|
+
this.assertExpectedLatestVersion({
|
|
2857
|
+
datasetId: input.dataset.id,
|
|
2858
|
+
expectedLatestVersionId: input.expectedLatestVersionId,
|
|
2859
|
+
actualLatestVersionId: latestVersion?.versionId ?? null
|
|
2860
|
+
});
|
|
2861
|
+
}
|
|
2862
|
+
const changeResult = await input.applyChanges(input.runtime, {
|
|
2863
|
+
previousRowCount: latestVersion?.rowCount
|
|
2864
|
+
});
|
|
2865
|
+
const commitId = randomUUID5();
|
|
2866
|
+
await this.setCommitMetadata(input, commitId, commitRowCount(input, changeResult, latestVersion));
|
|
2867
|
+
await input.runtime.run("COMMIT");
|
|
2868
|
+
committed = true;
|
|
2869
|
+
const committedWriteSnapshotId = await this.lastCommittedSnapshotId(input.runtime);
|
|
2870
|
+
try {
|
|
2871
|
+
return await this.versionForCommittedWrite({
|
|
2872
|
+
...input,
|
|
2873
|
+
commitId,
|
|
2874
|
+
previousWriteSnapshotId,
|
|
2875
|
+
committedWriteSnapshotId,
|
|
2876
|
+
changeResult
|
|
2877
|
+
});
|
|
2878
|
+
} finally {
|
|
2879
|
+
await this.connections.detachLocalCatalogAfterCommit(input.runtime);
|
|
2880
|
+
}
|
|
2881
|
+
} catch (error) {
|
|
2882
|
+
if (!committed) {
|
|
2883
|
+
await this.rollbackTransaction(input.runtime);
|
|
2884
|
+
}
|
|
2885
|
+
throw error;
|
|
2886
|
+
}
|
|
2887
|
+
}
|
|
2888
|
+
async versionForCommittedWrite(input) {
|
|
2889
|
+
const ownSnapshotId = await this.findOwnSnapshotId(input.runtime, input.dataset.id, input.commitId, input.previousWriteSnapshotId, input.committedWriteSnapshotId);
|
|
2890
|
+
if (ownSnapshotId !== null) {
|
|
2891
|
+
const version = await this.snapshots.getVersionForSnapshot(input.runtime, input.dataset, ownSnapshotId);
|
|
2892
|
+
if (version) {
|
|
2893
|
+
return { ...version, outcome: "created" };
|
|
2894
|
+
}
|
|
2895
|
+
throw new LakeStorageError20(`[SixbDuckLake] DuckLake committed snapshot '${ownSnapshotId}' for dataset '${input.dataset.id}', but Sixb could not hydrate it as a dataset version.`);
|
|
2896
|
+
}
|
|
2897
|
+
if (!input.changeResult.dataChangeExpected) {
|
|
2898
|
+
return {
|
|
2899
|
+
...await this.versionForNoOpCommit(input.runtime, input.dataset),
|
|
2900
|
+
outcome: "unchanged"
|
|
2901
|
+
};
|
|
2902
|
+
}
|
|
2903
|
+
if (input.committedWriteSnapshotId === input.previousWriteSnapshotId) {
|
|
2904
|
+
throw new LakeStorageError20(`[SixbDuckLake] DuckLake commit for dataset '${input.dataset.id}' completed without producing a snapshot for this non-empty write.`);
|
|
2905
|
+
}
|
|
2906
|
+
throw new LakeStorageError20(`[SixbDuckLake] DuckLake commit for dataset '${input.dataset.id}' completed, but Sixb could not find a matching snapshot for this write.`);
|
|
2907
|
+
}
|
|
2908
|
+
async applyStagedRows(input) {
|
|
2909
|
+
const mode = input.write.mode ?? "snapshot";
|
|
2910
|
+
const result = await applyDatasetRowsFromRelation({
|
|
2911
|
+
options: this.options,
|
|
2912
|
+
runtime: input.runtime,
|
|
2913
|
+
dataset: input.write.dataset,
|
|
2914
|
+
mode,
|
|
2915
|
+
sourceRelationSql: quoteIdentifier(input.stagingTableName),
|
|
2916
|
+
previousRowCount: input.previousRowCount
|
|
2917
|
+
});
|
|
2918
|
+
if (result.sourceRowCount !== input.rowsWritten) {
|
|
2919
|
+
throw new LakeStorageError20(`[SixbDuckLake] Staged write for dataset '${input.write.dataset.id}' accepted ${input.rowsWritten} row(s), but DuckLake saw ${result.sourceRowCount} source row(s) at commit time.`);
|
|
2920
|
+
}
|
|
2921
|
+
return result;
|
|
2922
|
+
}
|
|
2923
|
+
async setCommitMetadata(input, commitId, rowCount) {
|
|
2924
|
+
const metadata = {
|
|
2925
|
+
kind: "datasetVersion",
|
|
2926
|
+
datasetId: input.dataset.id,
|
|
2927
|
+
commitId,
|
|
2928
|
+
mode: input.mode,
|
|
2929
|
+
producer: input.producer ? structuredClone(input.producer) : undefined,
|
|
2930
|
+
inputs: input.inputs ? structuredClone(input.inputs) : undefined,
|
|
2931
|
+
...rowCount.kind === "exact" ? { rowCount: rowCount.value } : {}
|
|
2932
|
+
};
|
|
2933
|
+
await input.runtime.run(`CALL ${quoteIdentifier(duckLakeAlias(this.options))}.set_commit_message(${quoteSqlString("Sixb")}, ${quoteSqlString(input.commitMessage)}, extra_info => ${quoteSqlString(JSON.stringify({ sixb: metadata }))})`);
|
|
2934
|
+
}
|
|
2935
|
+
async versionForNoOpCommit(runtime, definition) {
|
|
2936
|
+
const latestVersion = await this.snapshots.getLatestVersionForDefinition(runtime, definition);
|
|
2937
|
+
if (latestVersion) {
|
|
2938
|
+
return latestVersion;
|
|
2939
|
+
}
|
|
2940
|
+
throw new LakeStorageError20(`[SixbDuckLake] No DuckLake changes were committed for dataset '${definition.id}', and no previous version exists.`);
|
|
2941
|
+
}
|
|
2942
|
+
async rollbackTransaction(runtime) {
|
|
2943
|
+
try {
|
|
2944
|
+
await runtime.run("ROLLBACK");
|
|
2945
|
+
} catch {}
|
|
2946
|
+
}
|
|
2947
|
+
async latestVersionForCommit(input) {
|
|
2948
|
+
if (input.expectedLatestVersionId === undefined && input.mode !== "append") {
|
|
2949
|
+
return null;
|
|
2950
|
+
}
|
|
2951
|
+
return this.snapshots.getLatestVersionSummaryForDefinition(input.runtime, input.dataset);
|
|
2952
|
+
}
|
|
2953
|
+
async lastCommittedSnapshotId(runtime) {
|
|
2954
|
+
const [row] = await runtime.query(`SELECT id FROM ${quoteIdentifier(duckLakeAlias(this.options))}.last_committed_snapshot()`);
|
|
2955
|
+
if (row === undefined || row.id === null || row.id === undefined) {
|
|
2956
|
+
return null;
|
|
2957
|
+
}
|
|
2958
|
+
return String(getBigIntLike(row, "id"));
|
|
2959
|
+
}
|
|
2960
|
+
async findSnapshotIdByCommitId(runtime, datasetId, commitId, snapshotId) {
|
|
2961
|
+
if (snapshotId !== undefined) {
|
|
2962
|
+
assertDuckLakeSnapshotId3(snapshotId);
|
|
2963
|
+
}
|
|
2964
|
+
const ducklakeSnapshotChanges = duckLakeMetadataTableName(this.options, "ducklake_snapshot_changes");
|
|
2965
|
+
const whereSql = snapshotId === undefined ? "" : `WHERE snapshot_id = ${snapshotId}`;
|
|
2966
|
+
const orderSql = snapshotId === undefined ? "ORDER BY snapshot_id DESC" : "";
|
|
2967
|
+
const rows = await runtime.query(`
|
|
2968
|
+
SELECT snapshot_id, commit_extra_info
|
|
2969
|
+
FROM ${ducklakeSnapshotChanges}
|
|
2970
|
+
${whereSql}
|
|
2971
|
+
${orderSql}
|
|
2972
|
+
`);
|
|
2973
|
+
for (const row of rows) {
|
|
2974
|
+
const metadata = parseCommitMetadata(row.commit_extra_info);
|
|
2975
|
+
if (metadata?.datasetId === datasetId && metadata.commitId === commitId) {
|
|
2976
|
+
return String(getBigIntLike(row, "snapshot_id"));
|
|
2977
|
+
}
|
|
2978
|
+
}
|
|
2979
|
+
return null;
|
|
2980
|
+
}
|
|
2981
|
+
async findOwnSnapshotId(runtime, datasetId, commitId, previousWriteSnapshotId, committedWriteSnapshotId) {
|
|
2982
|
+
if (committedWriteSnapshotId === null || committedWriteSnapshotId === previousWriteSnapshotId) {
|
|
2983
|
+
return null;
|
|
2984
|
+
}
|
|
2985
|
+
const exactMatch = await this.findSnapshotIdByCommitId(runtime, datasetId, commitId, committedWriteSnapshotId);
|
|
2986
|
+
if (exactMatch !== null) {
|
|
2987
|
+
return exactMatch;
|
|
2988
|
+
}
|
|
2989
|
+
return this.findSnapshotIdByCommitId(runtime, datasetId, commitId);
|
|
2990
|
+
}
|
|
2991
|
+
assertExpectedLatestVersion(input) {
|
|
2992
|
+
if (input.expectedLatestVersionId === undefined) {
|
|
2993
|
+
return;
|
|
2994
|
+
}
|
|
2995
|
+
if (input.actualLatestVersionId !== input.expectedLatestVersionId) {
|
|
2996
|
+
throw new LakeStorageError20(`[SixbDuckLake] Optimistic commit failed for dataset '${input.datasetId}': expected latest version '${input.expectedLatestVersionId}', found '${input.actualLatestVersionId ?? "none"}'.`);
|
|
2997
|
+
}
|
|
2998
|
+
}
|
|
2999
|
+
}
|
|
3000
|
+
function commitRowCount(input, changeResult, latestVersion) {
|
|
3001
|
+
if (input.mode === "append" && input.expectedLatestVersionId === undefined) {
|
|
3002
|
+
return { kind: "unknown" };
|
|
3003
|
+
}
|
|
3004
|
+
if (changeResult.resultingRowCount.kind === "exact") {
|
|
3005
|
+
return changeResult.resultingRowCount;
|
|
3006
|
+
}
|
|
3007
|
+
if (input.mode === "append" && latestVersion === null) {
|
|
3008
|
+
return { kind: "exact", value: changeResult.sourceRowCount };
|
|
3009
|
+
}
|
|
3010
|
+
return { kind: "unknown" };
|
|
3011
|
+
}
|
|
3012
|
+
function assertDuckLakeSnapshotId3(snapshotId) {
|
|
3013
|
+
if (!/^\d+$/.test(snapshotId)) {
|
|
3014
|
+
throw new LakeStorageError20(`[SixbDuckLake] Invalid DuckLake snapshot id '${snapshotId}'.`);
|
|
3015
|
+
}
|
|
3016
|
+
}
|
|
3017
|
+
var localCatalogCommitLocks = new Map;
|
|
3018
|
+
async function withCatalogCommitLock(options, run) {
|
|
3019
|
+
const key = localCatalogCoordinationKey(options);
|
|
3020
|
+
if (key === undefined) {
|
|
3021
|
+
return run();
|
|
3022
|
+
}
|
|
3023
|
+
const previous = localCatalogCommitLocks.get(key) ?? Promise.resolve();
|
|
3024
|
+
const ready = previous.catch(() => {});
|
|
3025
|
+
let release;
|
|
3026
|
+
const next = new Promise((resolve2) => {
|
|
3027
|
+
release = resolve2;
|
|
3028
|
+
});
|
|
3029
|
+
const current = ready.then(() => next);
|
|
3030
|
+
localCatalogCommitLocks.set(key, current);
|
|
3031
|
+
await ready;
|
|
3032
|
+
try {
|
|
3033
|
+
return await run();
|
|
3034
|
+
} finally {
|
|
3035
|
+
release();
|
|
3036
|
+
if (localCatalogCommitLocks.get(key) === current) {
|
|
3037
|
+
localCatalogCommitLocks.delete(key);
|
|
3038
|
+
}
|
|
3039
|
+
}
|
|
3040
|
+
}
|
|
3041
|
+
|
|
3042
|
+
// src/ducklake-storage.ts
|
|
3043
|
+
class DuckLakeStorage {
|
|
3044
|
+
standard = { id: "ducklake", version: "1.0" };
|
|
3045
|
+
sql;
|
|
3046
|
+
connections;
|
|
3047
|
+
datasets;
|
|
3048
|
+
maintenance;
|
|
3049
|
+
rows;
|
|
3050
|
+
snapshotReader;
|
|
3051
|
+
writes;
|
|
3052
|
+
constructor(options) {
|
|
3053
|
+
const normalizedOptions = {
|
|
3054
|
+
...options,
|
|
3055
|
+
alias: options.alias ?? "sixb_lake",
|
|
3056
|
+
createIfNotExists: options.createIfNotExists ?? true,
|
|
3057
|
+
readOnly: options.readOnly ?? false,
|
|
3058
|
+
duckdb: {
|
|
3059
|
+
...options.duckdb,
|
|
3060
|
+
path: options.duckdb?.path ?? ":memory:"
|
|
3061
|
+
}
|
|
3062
|
+
};
|
|
3063
|
+
this.connections = new DuckLakeConnectionManager(normalizedOptions);
|
|
3064
|
+
this.datasets = new DuckLakeDatasetCatalog(normalizedOptions, this.connections);
|
|
3065
|
+
this.maintenance = new DuckLakeMaintenance(normalizedOptions, this.connections);
|
|
3066
|
+
this.snapshotReader = new DuckLakeSnapshotReader(normalizedOptions, this.connections, this.datasets);
|
|
3067
|
+
this.rows = new DuckLakeRowReader(normalizedOptions, this.connections, this.snapshotReader);
|
|
3068
|
+
this.writes = new DuckLakeWriteCoordinator(normalizedOptions, this.connections, this.datasets, this.snapshotReader);
|
|
3069
|
+
this.sql = new DuckLakeSqlExecutor(normalizedOptions, this.connections, this.datasets, this.snapshotReader, this.writes);
|
|
3070
|
+
}
|
|
3071
|
+
async createDataset(definition) {
|
|
3072
|
+
return this.datasets.createDataset(definition);
|
|
3073
|
+
}
|
|
3074
|
+
async assertDatasetDefinitionsCompatible(definitions) {
|
|
3075
|
+
await this.datasets.assertDatasetDefinitionsCompatible(definitions);
|
|
3076
|
+
}
|
|
3077
|
+
async getDataset(datasetId) {
|
|
3078
|
+
return this.datasets.getDataset(datasetId);
|
|
3079
|
+
}
|
|
3080
|
+
async listDatasets() {
|
|
3081
|
+
return this.datasets.listDatasets();
|
|
3082
|
+
}
|
|
3083
|
+
async listDatasetCatalogState(datasetIds) {
|
|
3084
|
+
return this.snapshotReader.listDatasetCatalogState(datasetIds);
|
|
3085
|
+
}
|
|
3086
|
+
async listVersions(datasetId, limit) {
|
|
3087
|
+
return this.snapshotReader.listVersions(datasetId, limit);
|
|
3088
|
+
}
|
|
3089
|
+
async beginWrite(input) {
|
|
3090
|
+
return this.writes.beginWrite(input);
|
|
3091
|
+
}
|
|
3092
|
+
async getLatestVersion(datasetId) {
|
|
3093
|
+
return this.snapshotReader.getLatestVersion(datasetId);
|
|
3094
|
+
}
|
|
3095
|
+
async getVersion(datasetId, versionId) {
|
|
3096
|
+
return this.snapshotReader.getVersion(datasetId, versionId);
|
|
3097
|
+
}
|
|
3098
|
+
readRows(input) {
|
|
3099
|
+
return this.rows.readRows(input);
|
|
3100
|
+
}
|
|
3101
|
+
async runMaintenance(options) {
|
|
3102
|
+
return this.maintenance.runMaintenance(options);
|
|
3103
|
+
}
|
|
3104
|
+
async close() {
|
|
3105
|
+
await this.connections.close();
|
|
3106
|
+
}
|
|
3107
|
+
}
|
|
3108
|
+
export {
|
|
3109
|
+
DuckLakeStorage
|
|
3110
|
+
};
|
|
3111
|
+
|
|
3112
|
+
//# debugId=0B56C838FAF193D864756E2164756E21
|