@sixb/ducklake 0.1.8 → 0.1.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (69) hide show
  1. package/README.md +59 -4
  2. package/dist/ducklake-storage.d.ts +1 -0
  3. package/dist/ducklake-storage.d.ts.map +1 -1
  4. package/dist/index.js +1817 -511
  5. package/dist/index.js.map +24 -14
  6. package/dist/internal/dataset-row-commit.d.ts +10 -0
  7. package/dist/internal/dataset-row-commit.d.ts.map +1 -1
  8. package/dist/internal/dataset-row-merge.d.ts.map +1 -1
  9. package/dist/internal/ducklake-dataset-catalog.d.ts +20 -3
  10. package/dist/internal/ducklake-dataset-catalog.d.ts.map +1 -1
  11. package/dist/internal/ducklake-dataset-table-ref.d.ts +13 -0
  12. package/dist/internal/ducklake-dataset-table-ref.d.ts.map +1 -1
  13. package/dist/internal/ducklake-maintenance.d.ts +21 -1
  14. package/dist/internal/ducklake-maintenance.d.ts.map +1 -1
  15. package/dist/internal/ducklake-snapshot-reader.d.ts +63 -16
  16. package/dist/internal/ducklake-snapshot-reader.d.ts.map +1 -1
  17. package/dist/internal/ducklake-sql-executor.d.ts +9 -1
  18. package/dist/internal/ducklake-sql-executor.d.ts.map +1 -1
  19. package/dist/internal/ducklake-version-transaction.d.ts +54 -0
  20. package/dist/internal/ducklake-version-transaction.d.ts.map +1 -0
  21. package/dist/internal/ducklake-write-coordinator.d.ts +30 -3
  22. package/dist/internal/ducklake-write-coordinator.d.ts.map +1 -1
  23. package/dist/internal/lineage-encoding.d.ts +8 -0
  24. package/dist/internal/lineage-encoding.d.ts.map +1 -0
  25. package/dist/internal/sql-lineage/analysis.d.ts +32 -0
  26. package/dist/internal/sql-lineage/analysis.d.ts.map +1 -0
  27. package/dist/internal/sql-lineage/expressions.d.ts +9 -0
  28. package/dist/internal/sql-lineage/expressions.d.ts.map +1 -0
  29. package/dist/internal/sql-lineage/plan-json.d.ts +34 -0
  30. package/dist/internal/sql-lineage/plan-json.d.ts.map +1 -0
  31. package/dist/internal/sql-lineage/plan-relation.d.ts +28 -0
  32. package/dist/internal/sql-lineage/plan-relation.d.ts.map +1 -0
  33. package/dist/internal/sql-lineage/plan-walker.d.ts +18 -0
  34. package/dist/internal/sql-lineage/plan-walker.d.ts.map +1 -0
  35. package/dist/internal/sql-lineage/rejection.d.ts +11 -0
  36. package/dist/internal/sql-lineage/rejection.d.ts.map +1 -0
  37. package/dist/internal/sql-lineage/statement-gate.d.ts +21 -0
  38. package/dist/internal/sql-lineage/statement-gate.d.ts.map +1 -0
  39. package/dist/internal/sql-lineage/table-functions.d.ts +6 -0
  40. package/dist/internal/sql-lineage/table-functions.d.ts.map +1 -0
  41. package/dist/internal/sql-transform-relations.d.ts +20 -5
  42. package/dist/internal/sql-transform-relations.d.ts.map +1 -1
  43. package/dist/internal/versions.d.ts +10 -1
  44. package/dist/internal/versions.d.ts.map +1 -1
  45. package/dist/types.d.ts +10 -1
  46. package/dist/types.d.ts.map +1 -1
  47. package/package.json +4 -4
  48. package/src/ducklake-storage.ts +9 -2
  49. package/src/internal/dataset-row-commit.ts +28 -2
  50. package/src/internal/dataset-row-merge.ts +4 -0
  51. package/src/internal/ducklake-dataset-catalog.ts +162 -49
  52. package/src/internal/ducklake-dataset-table-ref.ts +75 -17
  53. package/src/internal/ducklake-maintenance.ts +110 -48
  54. package/src/internal/ducklake-snapshot-reader.ts +352 -304
  55. package/src/internal/ducklake-sql-executor.ts +214 -14
  56. package/src/internal/ducklake-version-transaction.ts +224 -0
  57. package/src/internal/ducklake-write-coordinator.ts +226 -148
  58. package/src/internal/lineage-encoding.ts +136 -0
  59. package/src/internal/sql-lineage/analysis.ts +257 -0
  60. package/src/internal/sql-lineage/expressions.ts +208 -0
  61. package/src/internal/sql-lineage/plan-json.ts +130 -0
  62. package/src/internal/sql-lineage/plan-relation.ts +59 -0
  63. package/src/internal/sql-lineage/plan-walker.ts +727 -0
  64. package/src/internal/sql-lineage/rejection.ts +16 -0
  65. package/src/internal/sql-lineage/statement-gate.ts +164 -0
  66. package/src/internal/sql-lineage/table-functions.ts +11 -0
  67. package/src/internal/sql-transform-relations.ts +90 -40
  68. package/src/internal/versions.ts +37 -14
  69. package/src/types.ts +11 -1
@@ -1,9 +1,14 @@
1
1
  import { randomUUID } from "node:crypto"
2
2
  import type { DatasetDefinition, DatasetSchema } from "@sixb/core"
3
- import type { DatasetDefinitionUpdatePlan, DatasetSchemaUpdatePlan } from "@sixb/core/lake-storage"
3
+ import type {
4
+ DatasetDefinitionUpdatePlan,
5
+ DatasetSchemaUpdatePlan,
6
+ DatasetVersionLineage,
7
+ } from "@sixb/core/lake-storage"
4
8
  import {
5
9
  assertDatasetDefinition,
6
10
  LakeStorageError,
11
+ lineageAtCommit,
7
12
  planDatasetDefinitionUpdate,
8
13
  } from "@sixb/core/lake-storage"
9
14
  import type { DuckLakeStorageOptions } from "../types"
@@ -17,6 +22,15 @@ import {
17
22
  } from "./ducklake-catalog-schema"
18
23
  import type { DuckLakeConnectionManager } from "./ducklake-connection-manager"
19
24
  import { readCurrentDatasetDefinitions } from "./ducklake-current-dataset-definitions"
25
+ import type { DatasetTableRef } from "./ducklake-dataset-table-ref"
26
+ import type { DuckLakeVersionSummary } from "./ducklake-snapshot-reader"
27
+ import {
28
+ beginTransactionOnLatestVersion,
29
+ commitVersionTransaction,
30
+ type LatestVersionAtSnapshot,
31
+ rerunWhenCatalogMoves,
32
+ } from "./ducklake-version-transaction"
33
+ import { encodeLineage } from "./lineage-encoding"
20
34
  import { encodeDatasetSequenceTableName, encodeDatasetTableName } from "./names"
21
35
  import { datasetColumnToDuckDbSql, datasetSchemaToDuckDbColumnsSql } from "./schema"
22
36
  import {
@@ -27,6 +41,28 @@ import {
27
41
  quoteSqlString,
28
42
  } from "./sql"
29
43
 
44
+ /**
45
+ * Reads the latest version of a dataset, which a schema change keeps the rows of, as
46
+ * `BeginVersionTransactionInput.readLatestVersion`.
47
+ */
48
+ export type LatestDatasetVersionReader = (
49
+ runtime: DuckDbQueryRuntime,
50
+ definition: DatasetDefinition
51
+ ) => Promise<LatestVersionAtSnapshot<DuckLakeVersionSummary>>
52
+
53
+ export interface DatasetSchemaAtSnapshotRequest {
54
+ readonly table: DatasetTableRef
55
+ readonly snapshotId: string
56
+ }
57
+
58
+ /** A row of `ducklake_column`: one column of one table, alive over a snapshot range. */
59
+ interface DuckLakeColumnVersion {
60
+ readonly tableId: bigint
61
+ readonly beginSnapshot: bigint
62
+ readonly endSnapshot?: bigint
63
+ readonly column: Omit<DuckLakeCatalogColumn, "tableName">
64
+ }
65
+
30
66
  /**
31
67
  * Translates between Sixb dataset definitions and DuckLake table metadata.
32
68
  *
@@ -40,7 +76,10 @@ export class DuckLakeDatasetCatalog {
40
76
  private readonly connections: DuckLakeConnectionManager
41
77
  ) {}
42
78
 
43
- async createDataset(definition: DatasetDefinition): Promise<DatasetDefinition> {
79
+ async createDataset(
80
+ definition: DatasetDefinition,
81
+ readLatestVersion: LatestDatasetVersionReader
82
+ ): Promise<DatasetDefinition> {
44
83
  this.assertSchema(definition)
45
84
 
46
85
  // Step 1: read the current DuckLake table back into Sixb shape. The
@@ -49,9 +88,25 @@ export class DuckLakeDatasetCatalog {
49
88
  const existing = await this.getDataset(definition.id)
50
89
 
51
90
  if (existing) {
52
- const plan = planDatasetDefinitionUpdate(existing, definition)
91
+ // A schema change the catalog moved under, such as one another host applied concurrently,
92
+ // reruns from the stored definition: it may have nothing left to change.
93
+ const plan = await rerunWhenCatalogMoves(definition.id, async (attempt) => {
94
+ const current = attempt === 1 ? existing : await this.getDataset(definition.id)
95
+ if (!current) {
96
+ throw new LakeStorageError(
97
+ `[SixbDuckLake] Dataset '${definition.id}' was removed while its definition was updated.`
98
+ )
99
+ }
100
+
101
+ const plan = planDatasetDefinitionUpdate(current, definition)
102
+ if (plan.changed) {
103
+ await this.applyExistingDatasetUpdate(plan, readLatestVersion)
104
+ }
105
+
106
+ return plan
107
+ })
108
+
53
109
  if (plan.changed) {
54
- await this.applyExistingDatasetUpdate(plan)
55
110
  this.connections.markLocalCatalogChanged()
56
111
  await this.connections.resetRuntime()
57
112
  }
@@ -201,23 +256,50 @@ export class DuckLakeDatasetCatalog {
201
256
  })
202
257
  }
203
258
 
204
- async getDatasetSchemaAtSnapshot(
259
+ /**
260
+ * Dataset schemas at DuckLake snapshots, in request order, from one catalog read: a column row
261
+ * is visible from its begin_snapshot until its end_snapshot, so the rows alive anywhere between
262
+ * the oldest and the newest requested snapshot answer every request.
263
+ */
264
+ async getDatasetSchemasAtSnapshots(
205
265
  runtime: DuckDbQueryRuntime,
206
- datasetId: string,
207
- tableName: string,
208
- tableId: bigint,
209
- snapshotId: string
210
- ): Promise<DatasetSchema> {
211
- assertDuckLakeSnapshotId(snapshotId)
212
-
213
- const columns = await this.readDatasetColumnsAtSnapshot(runtime, tableName, tableId, snapshotId)
214
- if (columns.length === 0) {
215
- throw new LakeStorageError(
216
- `[SixbDuckLake] Could not reconstruct schema for dataset '${datasetId}' at DuckLake snapshot '${snapshotId}'.`
217
- )
266
+ requests: readonly DatasetSchemaAtSnapshotRequest[]
267
+ ): Promise<readonly DatasetSchema[]> {
268
+ if (requests.length === 0) {
269
+ return []
218
270
  }
219
271
 
220
- return duckLakeCatalogColumnsToDatasetSchema(tableName, columns)
272
+ for (const request of requests) {
273
+ assertDuckLakeSnapshotId(request.snapshotId)
274
+ }
275
+
276
+ const snapshots = requests.map((request) => BigInt(request.snapshotId))
277
+ const rows = await this.readDatasetColumnsBetweenSnapshots(
278
+ runtime,
279
+ requests.map((request) => request.table),
280
+ snapshots.reduce((oldest, snapshot) => (snapshot < oldest ? snapshot : oldest)),
281
+ snapshots.reduce((newest, snapshot) => (snapshot > newest ? snapshot : newest))
282
+ )
283
+
284
+ return requests.map(({ table, snapshotId }, index) => {
285
+ const snapshot = snapshots[index]
286
+ const columns = rows
287
+ .filter(
288
+ (row) =>
289
+ row.tableId === table.tableId &&
290
+ row.beginSnapshot <= snapshot &&
291
+ (row.endSnapshot === undefined || row.endSnapshot > snapshot)
292
+ )
293
+ .map((row) => ({ tableName: table.tableName, ...row.column }))
294
+
295
+ if (columns.length === 0) {
296
+ throw new LakeStorageError(
297
+ `[SixbDuckLake] Could not reconstruct schema for dataset '${table.datasetId}' at DuckLake snapshot '${snapshotId}'.`
298
+ )
299
+ }
300
+
301
+ return duckLakeCatalogColumnsToDatasetSchema(table.tableName, columns)
302
+ })
221
303
  }
222
304
 
223
305
  assertSchema(definition: DatasetDefinition): void {
@@ -227,27 +309,48 @@ export class DuckLakeDatasetCatalog {
227
309
  )
228
310
  }
229
311
 
230
- private async applyExistingDatasetUpdate(plan: DatasetDefinitionUpdatePlan): Promise<void> {
312
+ private async applyExistingDatasetUpdate(
313
+ plan: DatasetDefinitionUpdatePlan,
314
+ readLatestVersion: LatestDatasetVersionReader
315
+ ): Promise<void> {
231
316
  const tableName = encodeDatasetTableName(plan.definition.id)
232
317
  const qualifiedName = qualifiedTableName(this.options, tableName)
318
+ const { schema } = plan
233
319
 
234
- await this.connections.withExclusiveAttached(async (runtime) => {
235
- if (plan.schema.kind === "none") {
236
- await this.applyMetadataDelta(runtime, qualifiedName, plan)
237
- return
238
- }
320
+ if (schema.kind === "none") {
321
+ await this.connections.withExclusiveAttached((runtime) =>
322
+ this.applyMetadataDelta(runtime, qualifiedName, plan)
323
+ )
324
+ return
325
+ }
239
326
 
240
- // Schema-only DuckLake snapshots still matter to Sixb. Apply schema and
241
- // compatible metadata DDL atomically, then tag the commit so versions can
242
- // treat it as this dataset's schema-change version.
243
- await runtime.run("BEGIN TRANSACTION")
327
+ // Schema-only DuckLake snapshots still matter to Sixb. Apply schema and
328
+ // compatible metadata DDL atomically, then tag the commit so versions can
329
+ // treat it as this dataset's schema-change version. It keeps the rows of
330
+ // the latest version, and so its lineage. ALTER TABLE claims the table, and
331
+ // changes no rows: it commits on the snapshot it read, or fails and reruns.
332
+ await this.connections.withExclusiveAttached(async (runtime) => {
244
333
  let committed = false
245
334
  try {
246
- await this.applySchemaEvolutionDdl(runtime, qualifiedName, plan.schema)
335
+ const parent = await beginTransactionOnLatestVersion({
336
+ options: this.options,
337
+ runtime,
338
+ datasetId: plan.definition.id,
339
+ readLatestVersion: () => readLatestVersion(runtime, plan.definition),
340
+ })
341
+
342
+ await this.applySchemaEvolutionDdl(runtime, qualifiedName, schema)
247
343
  await this.applyMetadataDelta(runtime, qualifiedName, plan)
248
- await this.setSchemaEvolutionCommitMetadata(runtime, plan.definition.id, plan.schema)
344
+ await this.setSchemaEvolutionCommitMetadata(
345
+ runtime,
346
+ plan.definition.id,
347
+ schema,
348
+ lineageAtCommit({ mode: "schema", schema: plan.definition.schema, parent })
349
+ )
249
350
 
250
- await runtime.run("COMMIT")
351
+ await commitVersionTransaction(runtime, plan.definition.id, {
352
+ poisonRuntime: () => this.connections.poisonRuntime(),
353
+ })
251
354
  committed = true
252
355
  } catch (error) {
253
356
  if (!committed) {
@@ -338,7 +441,8 @@ export class DuckLakeDatasetCatalog {
338
441
  private async setSchemaEvolutionCommitMetadata(
339
442
  runtime: DuckDbQueryRuntime,
340
443
  datasetId: string,
341
- plan: Extract<DatasetSchemaUpdatePlan, { readonly kind: "addNullableColumns" }>
444
+ plan: Extract<DatasetSchemaUpdatePlan, { readonly kind: "addNullableColumns" }>,
445
+ lineage: DatasetVersionLineage | undefined
342
446
  ): Promise<void> {
343
447
  await runtime.run(
344
448
  `CALL ${quoteIdentifier(duckLakeAlias(this.options))}.set_commit_message(${quoteSqlString(
@@ -353,6 +457,7 @@ export class DuckLakeDatasetCatalog {
353
457
  schemaChange: {
354
458
  addColumns: plan.columns.map((column) => column.name),
355
459
  },
460
+ ...(lineage === undefined ? {} : { lineage: encodeLineage(lineage) }),
356
461
  },
357
462
  })
358
463
  )})`
@@ -368,36 +473,44 @@ export class DuckLakeDatasetCatalog {
368
473
  }
369
474
  }
370
475
 
371
- private async readDatasetColumnsAtSnapshot(
476
+ private async readDatasetColumnsBetweenSnapshots(
372
477
  runtime: DuckDbQueryRuntime,
373
- tableName: string,
374
- tableId: bigint,
375
- snapshotId: string
376
- ): Promise<readonly DuckLakeCatalogColumn[]> {
478
+ tables: readonly DatasetTableRef[],
479
+ oldestSnapshot: bigint,
480
+ newestSnapshot: bigint
481
+ ): Promise<readonly DuckLakeColumnVersion[]> {
482
+ const tableIds = new Set(tables.map((table) => table.tableId))
377
483
  const ducklakeColumn = duckLakeMetadataTableName(this.options, "ducklake_column")
378
484
  const rows = await runtime.query(`
379
485
  SELECT
486
+ table_id,
380
487
  column_id,
381
488
  column_order,
382
489
  column_name,
383
490
  column_type,
384
491
  CAST(nulls_allowed AS BOOLEAN) AS nulls_allowed,
385
- parent_column
492
+ parent_column,
493
+ begin_snapshot,
494
+ end_snapshot
386
495
  FROM ${ducklakeColumn}
387
- WHERE table_id = ${tableId}
388
- AND begin_snapshot <= ${snapshotId}
389
- AND (end_snapshot IS NULL OR end_snapshot > ${snapshotId})
390
- ORDER BY column_order
496
+ WHERE table_id IN (${[...tableIds].join(", ")})
497
+ AND begin_snapshot <= ${newestSnapshot}
498
+ AND (end_snapshot IS NULL OR end_snapshot > ${oldestSnapshot})
499
+ ORDER BY column_order, column_id
391
500
  `)
392
501
 
393
502
  return rows.map((row) => ({
394
- tableName,
395
- columnId: getBigIntLike(row, "column_id"),
396
- columnOrder: getBigIntLike(row, "column_order"),
397
- columnName: getString(row, "column_name"),
398
- columnType: getString(row, "column_type"),
399
- nullsAllowed: getBoolean(row, "nulls_allowed"),
400
- parentColumnId: getOptionalBigIntLike(row, "parent_column"),
503
+ tableId: getBigIntLike(row, "table_id"),
504
+ beginSnapshot: getBigIntLike(row, "begin_snapshot"),
505
+ endSnapshot: getOptionalBigIntLike(row, "end_snapshot"),
506
+ column: {
507
+ columnId: getBigIntLike(row, "column_id"),
508
+ columnOrder: getBigIntLike(row, "column_order"),
509
+ columnName: getString(row, "column_name"),
510
+ columnType: getString(row, "column_type"),
511
+ nullsAllowed: getBoolean(row, "nulls_allowed"),
512
+ parentColumnId: getOptionalBigIntLike(row, "parent_column"),
513
+ },
401
514
  }))
402
515
  }
403
516
  }
@@ -1,13 +1,16 @@
1
+ import { LakeStorageError } from "@sixb/core/lake-storage"
1
2
  import type { DuckLakeStorageOptions } from "../types"
2
3
  import { getBigIntLike } from "./duckdb-row"
3
4
  import type { DuckDbQueryRuntime } from "./duckdb-runtime"
4
- import { encodeDatasetTableName } from "./names"
5
- import { duckLakeMetadataTableName, quoteSqlString } from "./sql"
5
+ import { encodeDatasetSequenceTableName, encodeDatasetTableName } from "./names"
6
+ import { duckLakeAlias, duckLakeMetadataTableName, quoteIdentifier, quoteSqlString } from "./sql"
6
7
 
7
8
  export interface DatasetTableRef {
8
9
  readonly datasetId: string
9
10
  readonly tableName: string
10
11
  readonly tableId: bigint
12
+ /** The table a sequenced dataset keeps its source ordering in, which merges also change. */
13
+ readonly sequenceTableId?: bigint
11
14
  }
12
15
 
13
16
  export async function resolveDatasetTableRef(
@@ -15,21 +18,76 @@ export async function resolveDatasetTableRef(
15
18
  runtime: DuckDbQueryRuntime,
16
19
  datasetId: string
17
20
  ): Promise<DatasetTableRef | null> {
18
- const tableName = encodeDatasetTableName(datasetId)
19
- const ducklakeTable = duckLakeMetadataTableName(options, "ducklake_table")
20
- const [row] = await runtime.query(`
21
- SELECT table_id
22
- FROM ${ducklakeTable}
23
- WHERE table_name = ${quoteSqlString(tableName)}
24
- AND end_snapshot IS NULL
25
- LIMIT 1
21
+ return tableRefFromRows(datasetId, await runtime.query(datasetTablesSql(options, datasetId)))
22
+ }
23
+
24
+ /**
25
+ * The dataset's tables and the lake's current snapshot, in one catalog read. Inside a
26
+ * transaction, reading the current snapshot also fixes the transaction's DuckLake snapshot, which
27
+ * DuckLake otherwise fixes on its first read of the lake.
28
+ */
29
+ export async function resolveDatasetTableRefAtCurrentSnapshot(
30
+ options: DuckLakeStorageOptions,
31
+ runtime: DuckDbQueryRuntime,
32
+ datasetId: string
33
+ ): Promise<{ readonly snapshotId: string; readonly table: DatasetTableRef | null }> {
34
+ // Joined on true, the current snapshot still reads when the dataset has no table.
35
+ const rows = await runtime.query(`
36
+ SELECT current.id AS current_snapshot, tables.table_id, tables.table_name
37
+ FROM ${quoteIdentifier(duckLakeAlias(options))}.current_snapshot() current
38
+ LEFT JOIN (${datasetTablesSql(options, datasetId)}) tables ON true
26
39
  `)
40
+ const [first] = rows
41
+ if (first === undefined) {
42
+ throw new LakeStorageError("[SixbDuckLake] DuckLake did not return a current snapshot.")
43
+ }
44
+
45
+ return {
46
+ snapshotId: String(getBigIntLike(first, "current_snapshot")),
47
+ table: tableRefFromRows(
48
+ datasetId,
49
+ rows.filter((row) => row.table_id !== null && row.table_id !== undefined)
50
+ ),
51
+ }
52
+ }
53
+
54
+ /** Every table a dataset's versions change. */
55
+ export function datasetTableIds(table: Omit<DatasetTableRef, "tableName">): readonly bigint[] {
56
+ return table.sequenceTableId === undefined
57
+ ? [table.tableId]
58
+ : [table.tableId, table.sequenceTableId]
59
+ }
60
+
61
+ function datasetTablesSql(options: DuckLakeStorageOptions, datasetId: string): string {
62
+ const tableNames = [encodeDatasetTableName(datasetId), encodeDatasetSequenceTableName(datasetId)]
63
+ return `
64
+ SELECT table_id, table_name
65
+ FROM ${duckLakeMetadataTableName(options, "ducklake_table")}
66
+ WHERE table_name IN (${tableNames.map((name) => quoteSqlString(name)).join(", ")})
67
+ AND end_snapshot IS NULL
68
+ `
69
+ }
70
+
71
+ function tableRefFromRows(
72
+ datasetId: string,
73
+ rows: readonly Readonly<Record<string, unknown>>[]
74
+ ): DatasetTableRef | null {
75
+ const tableName = encodeDatasetTableName(datasetId)
76
+ const tableIdOf = (name: string) => {
77
+ const row = rows.find((candidate) => candidate.table_name === name)
78
+ return row === undefined ? undefined : getBigIntLike(row, "table_id")
79
+ }
80
+
81
+ const tableId = tableIdOf(tableName)
82
+ if (tableId === undefined) {
83
+ return null
84
+ }
27
85
 
28
- return row === undefined
29
- ? null
30
- : {
31
- datasetId,
32
- tableName,
33
- tableId: getBigIntLike(row, "table_id"),
34
- }
86
+ const sequenceTableId = tableIdOf(encodeDatasetSequenceTableName(datasetId))
87
+ return {
88
+ datasetId,
89
+ tableName,
90
+ tableId,
91
+ ...(sequenceTableId === undefined ? {} : { sequenceTableId }),
92
+ }
35
93
  }
@@ -1,25 +1,40 @@
1
+ import type { DatasetVersionRef } from "@sixb/core/lake-storage"
1
2
  import { LakeStorageError } from "@sixb/core/lake-storage"
2
3
  import type {
3
4
  DuckLakeMaintenanceOptions,
4
5
  DuckLakeMaintenanceReport,
5
6
  DuckLakeStorageOptions,
6
7
  } from "../types"
8
+ import { getBigIntLike } from "./duckdb-row"
7
9
  import type { DuckDbQueryRuntime } from "./duckdb-runtime"
8
10
  import type { DuckLakeConnectionManager } from "./ducklake-connection-manager"
9
- import { duckLakeAlias, quoteSqlString } from "./sql"
11
+ import type { DuckLakeSnapshotReader } from "./ducklake-snapshot-reader"
12
+ import { duckLakeAlias, duckLakeMetadataTableName, quoteSqlString } from "./sql"
13
+ import { parseVersionId } from "./versions"
10
14
 
11
15
  const DEFAULT_RETENTION = "7 days"
12
16
 
13
17
  interface MaintenanceCounts {
14
18
  readonly snapshots: number
19
+ readonly keptSnapshots: number
15
20
  readonly oldFiles: number
16
21
  readonly orphanedFiles: number
17
22
  }
18
23
 
24
+ /**
25
+ * Expires old snapshots, except those holding a version that is still needed, then reclaims files.
26
+ *
27
+ * DuckLake's own `older_than` would expire the latest version of a dataset idle longer than the
28
+ * retention, leaving its rows without a version to read them through. Maintenance lists the
29
+ * snapshots older than the retention, keeps those holding each dataset's latest version or a
30
+ * protected version, and expires the rest by id. DuckLake then schedules the files only the
31
+ * expired snapshots referenced, as it does with `older_than`.
32
+ */
19
33
  export class DuckLakeMaintenance {
20
34
  constructor(
21
35
  private readonly options: DuckLakeStorageOptions,
22
- private readonly connections: DuckLakeConnectionManager
36
+ private readonly connections: DuckLakeConnectionManager,
37
+ private readonly snapshots: DuckLakeSnapshotReader
23
38
  ) {}
24
39
 
25
40
  async runMaintenance(
@@ -30,6 +45,7 @@ export class DuckLakeMaintenance {
30
45
  const expireOlderThan = options.expireOlderThan ?? DEFAULT_RETENTION
31
46
  const deleteOlderThan = options.deleteOlderThan ?? expireOlderThan
32
47
  const dryRun = options.dryRun ?? false
48
+ const protect = (options.protect ?? []).map(protectedSnapshotId)
33
49
 
34
50
  if (this.options.readOnly && !dryRun) {
35
51
  throw new LakeStorageError(
@@ -38,13 +54,10 @@ export class DuckLakeMaintenance {
38
54
  }
39
55
 
40
56
  return this.connections.withMaintenanceRuntime(async (runtime) => {
41
- const counts = await runMaintenanceProcedures(
57
+ const counts = await this.runMaintenanceProcedures(
42
58
  runtime,
43
- this.options,
44
- {
45
- expireOlderThan,
46
- deleteOlderThan,
47
- },
59
+ { expireOlderThan, deleteOlderThan },
60
+ protect,
48
61
  dryRun
49
62
  )
50
63
 
@@ -60,57 +73,106 @@ export class DuckLakeMaintenance {
60
73
  }
61
74
  })
62
75
  }
63
- }
64
76
 
65
- async function runMaintenanceProcedures(
66
- runtime: DuckDbQueryRuntime,
67
- options: DuckLakeStorageOptions,
68
- retention: Pick<DuckLakeMaintenanceReport, "expireOlderThan" | "deleteOlderThan">,
69
- dryRun: boolean
70
- ): Promise<MaintenanceCounts> {
71
- const aliasSql = quoteSqlString(duckLakeAlias(options))
72
- const snapshots = await runtime.query(
73
- maintenanceProcedureSql(
74
- "ducklake_expire_snapshots",
75
- aliasSql,
76
- retention.expireOlderThan,
77
- dryRun
78
- )
79
- )
80
- const oldFiles = await runtime.query(
81
- maintenanceProcedureSql(
82
- "ducklake_cleanup_old_files",
83
- aliasSql,
84
- retention.deleteOlderThan,
85
- dryRun
77
+ private async runMaintenanceProcedures(
78
+ runtime: DuckDbQueryRuntime,
79
+ retention: Pick<DuckLakeMaintenanceReport, "expireOlderThan" | "deleteOlderThan">,
80
+ protect: readonly string[],
81
+ dryRun: boolean
82
+ ): Promise<MaintenanceCounts> {
83
+ const aliasSql = quoteSqlString(duckLakeAlias(this.options))
84
+
85
+ const oldSnapshotIds = await this.snapshotsOlderThan(runtime, retention.expireOlderThan)
86
+ const needed = new Set([
87
+ ...(await this.snapshots.latestVersionSnapshotIds(runtime)),
88
+ ...protect,
89
+ ])
90
+ const expirable = oldSnapshotIds.filter((snapshotId) => !needed.has(snapshotId))
91
+
92
+ const snapshots =
93
+ expirable.length === 0
94
+ ? []
95
+ : await runtime.query(
96
+ maintenanceProcedureSql(
97
+ "ducklake_expire_snapshots",
98
+ aliasSql,
99
+ `versions => [${expirable.join(", ")}]`,
100
+ dryRun
101
+ )
102
+ )
103
+
104
+ const oldFiles = await runtime.query(
105
+ maintenanceProcedureSql(
106
+ "ducklake_cleanup_old_files",
107
+ aliasSql,
108
+ `older_than => ${olderThanSql(retention.deleteOlderThan)}`,
109
+ dryRun
110
+ )
86
111
  )
87
- )
88
- const orphanedFiles = await runtime.query(
89
- maintenanceProcedureSql(
90
- "ducklake_delete_orphaned_files",
91
- aliasSql,
92
- retention.deleteOlderThan,
93
- dryRun
112
+ const orphanedFiles = await runtime.query(
113
+ maintenanceProcedureSql(
114
+ "ducklake_delete_orphaned_files",
115
+ aliasSql,
116
+ `older_than => ${olderThanSql(retention.deleteOlderThan)}`,
117
+ dryRun
118
+ )
94
119
  )
95
- )
96
120
 
97
- return {
98
- snapshots: snapshots.length,
99
- oldFiles: oldFiles.length,
100
- orphanedFiles: orphanedFiles.length,
121
+ return {
122
+ snapshots: snapshots.length,
123
+ keptSnapshots: oldSnapshotIds.length - expirable.length,
124
+ oldFiles: oldFiles.length,
125
+ orphanedFiles: orphanedFiles.length,
126
+ }
127
+ }
128
+
129
+ /**
130
+ * Snapshots older than the retention, except the current one, which DuckLake never expires.
131
+ *
132
+ * Not DuckLake's own `older_than` selection: a SQLite catalog stores snapshot times as text with
133
+ * their UTC offset, and with DuckLake 415a9ebd that selection then also returned snapshots
134
+ * written a moment ago outside UTC. Casting the stored text compares instants on every catalog.
135
+ */
136
+ private async snapshotsOlderThan(
137
+ runtime: DuckDbQueryRuntime,
138
+ olderThan: string
139
+ ): Promise<readonly string[]> {
140
+ const ducklakeSnapshot = duckLakeMetadataTableName(this.options, "ducklake_snapshot")
141
+ const rows = await runtime.query(`
142
+ SELECT snapshot_id
143
+ FROM ${ducklakeSnapshot}
144
+ WHERE CAST(snapshot_time AS TIMESTAMPTZ) < ${olderThanSql(olderThan)}
145
+ AND snapshot_id < (
146
+ SELECT snapshot_id FROM ${ducklakeSnapshot} ORDER BY snapshot_id DESC LIMIT 1
147
+ )
148
+ ORDER BY snapshot_id
149
+ `)
150
+
151
+ return rows.map((row) => String(getBigIntLike(row, "snapshot_id")))
152
+ }
153
+ }
154
+
155
+ function protectedSnapshotId(ref: DatasetVersionRef): string {
156
+ try {
157
+ return parseVersionId(ref.versionId)
158
+ } catch (error) {
159
+ throw new LakeStorageError(
160
+ `[SixbDuckLake] Cannot protect version '${ref.versionId}' of dataset '${ref.datasetId}' from maintenance: it is not a DuckLake version id.`,
161
+ { cause: error }
162
+ )
101
163
  }
102
164
  }
103
165
 
166
+ function olderThanSql(olderThan: string): string {
167
+ return `now() - INTERVAL ${quoteSqlString(olderThan)}`
168
+ }
169
+
104
170
  function maintenanceProcedureSql(
105
171
  procedure: string,
106
172
  aliasSql: string,
107
- olderThan: string,
173
+ selection: string,
108
174
  dryRun: boolean
109
175
  ): string {
110
- const olderThanSql = quoteSqlString(olderThan)
111
176
  const dryRunSql = dryRun ? "dry_run => true, " : ""
112
- return (
113
- `CALL ${procedure}(${aliasSql}, ${dryRunSql}` +
114
- `older_than => now() - INTERVAL ${olderThanSql})`
115
- )
177
+ return `CALL ${procedure}(${aliasSql}, ${dryRunSql}${selection})`
116
178
  }