@sixb/ducklake 0.0.1

package/src/internal/names.ts

@@ -0,0 +1,111 @@
+import { LakeStorageError } from "@sixb/core/lake-storage"
+
+/**
+ * Prefix used for all Sixb dataset tables inside the DuckLake schema.
+ */
+export const DATASET_TABLE_PREFIX = "sixb__ds__"
+
+const textEncoder = new TextEncoder()
+const textDecoder = new TextDecoder("utf-8", { fatal: true })
+
+function isInlineByte(byte: number): boolean {
+  return (byte >= 0x61 && byte <= 0x7a) || (byte >= 0x30 && byte <= 0x39)
+}
+
+function isHexPair(value: string): boolean {
+  return /^[0-9a-f]{2}$/.test(value)
+}
+
+/**
+ * Encode a Sixb dataset id into a SQL identifier-safe table suffix.
+ *
+ * Common lowercase dotted ids stay readable by mapping "." to "__". Literal
+ * underscores and all other non-inline UTF-8 bytes are encoded as "_xx" hex
+ * escapes so decoding is collision-free.
+ */
+function encodeDatasetId(datasetId: string): string {
+  if (datasetId.trim().length === 0) {
+    throw new LakeStorageError("[SixbDuckLake] Dataset id must not be empty.")
+  }
+
+  let encoded = ""
+  for (const byte of textEncoder.encode(datasetId)) {
+    if (isInlineByte(byte)) {
+      encoded += String.fromCharCode(byte)
+    } else if (byte === 0x2e) {
+      encoded += "__"
+    } else {
+      encoded += `_${byte.toString(16).padStart(2, "0")}`
+    }
+  }
+
+  return encoded
+}
+
+/**
+ * Decode a table suffix created by `encodeDatasetId`.
+ *
+ * Returns null for malformed provider table names so metadata scans can filter
+ * defensively instead of throwing on unrelated tables.
+ */
+function decodeDatasetId(encoded: string): string | null {
+  if (encoded.length === 0) {
+    return null
+  }
+
+  const bytes: number[] = []
+  let index = 0
+
+  while (index < encoded.length) {
+    const char = encoded[index]
+    const byte = char.charCodeAt(0)
+
+    if (isInlineByte(byte)) {
+      bytes.push(byte)
+      index += 1
+      continue
+    }
+
+    if (char !== "_") {
+      return null
+    }
+
+    if (encoded[index + 1] === "_") {
+      bytes.push(0x2e)
+      index += 2
+      continue
+    }
+
+    const hex = encoded.slice(index + 1, index + 3)
+    if (!isHexPair(hex)) {
+      return null
+    }
+
+    bytes.push(Number.parseInt(hex, 16))
+    index += 3
+  }
+
+  try {
+    return textDecoder.decode(new Uint8Array(bytes))
+  } catch {
+    return null
+  }
+}
+
+/**
+ * Convert a Sixb dataset id to its physical DuckLake table name.
+ */
+export function encodeDatasetTableName(datasetId: string): string {
+  return `${DATASET_TABLE_PREFIX}${encodeDatasetId(datasetId)}`
+}
+
+/**
+ * Decode a physical DuckLake table name back to a Sixb dataset id.
+ */
+export function decodeDatasetTableName(tableName: string): string | null {
+  if (!tableName.startsWith(DATASET_TABLE_PREFIX)) {
+    return null
+  }
+
+  return decodeDatasetId(tableName.slice(DATASET_TABLE_PREFIX.length))
+}

package/src/internal/row-appender.ts

@@ -0,0 +1,73 @@
+import type { DuckDBValue } from "@duckdb/node-api"
+import type { DatasetColumnDefinition, DatasetRow, DatasetSchema, FileRef } from "@sixb/core"
+import { normalizeDuckDbDecimalValue } from "./duckdb-decimal"
+import type { DuckDbAppender } from "./duckdb-runtime"
+
+/**
+ * Append one validated dataset row into the staging table's DuckDB appender.
+ */
+export function appendDatasetRow(
+  appender: DuckDbAppender,
+  schema: DatasetSchema,
+  row: DatasetRow
+): void {
+  for (const column of schema.columns) {
+    appendColumnValue(appender, column, row[column.name] ?? null)
+  }
+
+  appender.endRow()
+}
+
+function appendColumnValue(
+  appender: DuckDbAppender,
+  column: DatasetColumnDefinition,
+  value: unknown
+): void {
+  if (value === null || value === undefined) {
+    appender.appendNull()
+    return
+  }
+
+  switch (column.type) {
+    case "string":
+      appender.appendVarchar(value as string)
+      return
+    case "boolean":
+      appender.appendBoolean(value as boolean)
+      return
+    case "int64":
+      appender.appendBigInt(typeof value === "bigint" ? value : BigInt(value as string | number))
+      return
+    case "float64":
+      appender.appendDouble(value as number)
+      return
+    case "decimal":
+      appender.appendVarchar(normalizeDuckDbDecimalValue(value, column.name))
+      return
+    case "date":
+      appender.appendVarchar(
+        value instanceof Date ? value.toISOString().slice(0, 10) : String(value)
+      )
+      return
+    case "timestamp":
+      appender.appendVarchar(value instanceof Date ? value.toISOString() : String(value))
+      return
+    case "json":
+      appender.appendVarchar(JSON.stringify(value))
+      return
+    case "fileRef":
+      appender.appendStruct(fileRefToDuckDbStruct(value as FileRef))
+      return
+  }
+}
+
+function fileRefToDuckDbStruct(fileRef: FileRef): Readonly<Record<string, DuckDBValue>> {
+  return {
+    blobId: fileRef.blobId,
+    digest: fileRef.digest,
+    sizeBytes: BigInt(fileRef.sizeBytes),
+    fileName: fileRef.fileName ?? null,
+    mediaType: fileRef.mediaType ?? null,
+    logicalPath: fileRef.logicalPath ?? null,
+  }
+}

package/src/internal/schema.ts

@@ -0,0 +1,149 @@
+import type { DatasetColumnDefinition, DatasetColumnType, DatasetSchema, FileRef } from "@sixb/core"
+import { LakeStorageError } from "@sixb/core/lake-storage"
+import { DUCKDB_COLUMN_TYPES, findDatasetColumnTypeForDuckDbSql } from "./duckdb-column-types"
+import { normalizeDuckDbDecimalValue } from "./duckdb-decimal"
+import { quoteIdentifier } from "./sql"
+
+/**
+ * Normalized column metadata read from DuckDB/DuckLake introspection queries.
+ */
+export interface DuckDbColumnMetadata {
+  readonly name: string
+  readonly type: string
+  readonly nullable: boolean
+}
+
+/**
+ * Map a Sixb dataset column type to the DuckDB type used in CREATE TABLE.
+ */
+export function datasetColumnTypeToDuckDbSql(type: DatasetColumnType): string {
+  return DUCKDB_COLUMN_TYPES[type].sql
+}
+
+/**
+ * Reconstruct a Sixb dataset column type from DuckDB column metadata.
+ */
+export function duckDbTypeToDatasetColumnType(typeSql: string): DatasetColumnType {
+  const type = findDatasetColumnTypeForDuckDbSql(typeSql)
+  if (type) {
+    return type
+  }
+
+  throw new LakeStorageError(
+    `[SixbDuckLake] DuckDB column type '${typeSql}' cannot be mapped to a Sixb dataset column type.`
+  )
+}
+
+/**
+ * Render one Sixb dataset column as DuckDB DDL.
+ */
+export function datasetColumnToDuckDbSql(column: DatasetColumnDefinition): string {
+  const nullableSql = column.nullable ? "" : " NOT NULL"
+  return `${quoteIdentifier(column.name)} ${datasetColumnTypeToDuckDbSql(column.type)}${nullableSql}`
+}
+
+/**
+ * Render the column list used by CREATE TABLE and staging table DDL.
+ */
+export function datasetSchemaToDuckDbColumnsSql(schema: DatasetSchema): string {
+  return schema.columns.map((column) => datasetColumnToDuckDbSql(column)).join(", ")
+}
+
+/**
+ * Render the column list used by INSERT ... SELECT operations.
+ */
+export function datasetSchemaColumnNamesSql(schema: DatasetSchema): string {
+  return schema.columns.map((column) => quoteIdentifier(column.name)).join(", ")
+}
+
+/**
+ * Convert DuckDB column metadata back into a Sixb dataset schema.
+ */
+export function duckDbColumnsToDatasetSchema(
+  columns: readonly DuckDbColumnMetadata[]
+): DatasetSchema {
+  return {
+    columns: columns.map((column) => ({
+      name: column.name,
+      type: duckDbTypeToDatasetColumnType(column.type),
+      ...(column.nullable ? { nullable: true } : {}),
+    })),
+  }
+}
+
+export function normalizeReadValue(value: unknown, column: DatasetColumnDefinition): unknown {
+  if (value === null || value === undefined) {
+    return null
+  }
+
+  switch (column.type) {
+    case "int64":
+      return typeof value === "bigint" ? value.toString() : value
+    case "decimal":
+      return normalizeDuckDbDecimalValue(value, column.name)
+    case "date":
+      return value instanceof Date ? value.toISOString().slice(0, 10) : String(value)
+    case "timestamp":
+      return value instanceof Date ? value.toISOString() : String(value)
+    case "json":
+      return typeof value === "string" ? JSON.parse(value) : value
+    case "fileRef":
+      return normalizeFileRef(value)
+    default:
+      return value
+  }
+}
+
+function normalizeFileRef(value: unknown): FileRef {
+  if (typeof value !== "object" || value === null || Array.isArray(value)) {
+    throw new LakeStorageError("[SixbDuckLake] DuckDB fileRef value must be a struct.")
+  }
+
+  const record = value as Record<string, unknown>
+  return {
+    blobId: getStringField(record, "blobId"),
+    digest: getStringField(record, "digest") as FileRef["digest"],
+    sizeBytes: getIntegerField(record, "sizeBytes"),
+    fileName: getOptionalStringField(record, "fileName"),
+    mediaType: getOptionalStringField(record, "mediaType"),
+    logicalPath: getOptionalStringField(record, "logicalPath"),
+  }
+}
+
+function getStringField(record: Readonly<Record<string, unknown>>, key: string): string {
+  const value = record[key]
+  if (typeof value !== "string") {
+    throw new LakeStorageError(`[SixbDuckLake] DuckDB fileRef field '${key}' must be a string.`)
+  }
+
+  return value
+}
+
+function getOptionalStringField(
+  record: Readonly<Record<string, unknown>>,
+  key: string
+): string | undefined {
+  const value = record[key]
+  if (value === null || value === undefined) {
+    return undefined
+  }
+
+  if (typeof value !== "string") {
+    throw new LakeStorageError(`[SixbDuckLake] DuckDB fileRef field '${key}' must be a string.`)
+  }
+
+  return value
+}
+
+function getIntegerField(record: Readonly<Record<string, unknown>>, key: string): number {
+  const value = record[key]
+  if (typeof value === "number" && Number.isInteger(value)) {
+    return value
+  }
+
+  if (typeof value === "bigint") {
+    return Number(value)
+  }
+
+  throw new LakeStorageError(`[SixbDuckLake] DuckDB fileRef field '${key}' must be an integer.`)
+}

package/src/internal/sql-transform-relations.ts

@@ -0,0 +1,99 @@
+import { randomUUID } from "node:crypto"
+import {
+  LakeStorageError,
+  type SqlTransformBody,
+  type SqlTransformRelation,
+} from "@sixb/core/lake-storage"
+import type { DuckLakeStorageOptions } from "../types"
+import { encodeDatasetTableName } from "./names"
+import { qualifiedTableName } from "./sql"
+import { parseVersionId } from "./versions"
+
+const RELATION_PLACEHOLDER_PREFIX = "__sixb_sql_transform_relation_"
+const RELATION_PLACEHOLDER_PATTERN = /__sixb_sql_transform_relation_[0-9a-f]{32}_[0-9]+__/g
+
+export interface DuckLakeSqlTransformSourceRelation {
+  readonly datasetId: string
+  readonly versionId: string
+}
+
+export interface RenderDuckLakeSqlTransformSqlInput {
+  readonly options: Pick<DuckLakeStorageOptions, "alias">
+  readonly sources: Readonly<Record<string, DuckLakeSqlTransformSourceRelation>>
+  readonly sql: SqlTransformBody<"duckdb">
+}
+
+interface DuckLakeSqlTransformRelationContext {
+  readonly relations: Readonly<Record<string, SqlTransformRelation>>
+  render(sql: string): string
+}
+
+class DuckLakeSqlTransformRelation implements SqlTransformRelation {
+  constructor(private readonly placeholder: string) {}
+
+  toString(): string {
+    return this.placeholder
+  }
+}
+
+export function renderDuckLakeSqlTransformSql(input: RenderDuckLakeSqlTransformSqlInput): string {
+  const context = createDuckLakeSqlTransformRelationContext(input)
+  const sql = input.sql(context.relations)
+
+  if (typeof sql !== "string") {
+    throw new LakeStorageError("[SixbDuckLake] SQL transform body must return a SQL string.")
+  }
+
+  return context.render(sql)
+}
+
+function createDuckLakeSqlTransformRelationContext(
+  input: Omit<RenderDuckLakeSqlTransformSqlInput, "sql">
+): DuckLakeSqlTransformRelationContext {
+  const token = randomUUID().replaceAll("-", "")
+  const relations: Record<string, SqlTransformRelation> = Object.create(null)
+  const replacements = new Map<string, string>()
+
+  let index = 0
+  for (const [sourceName, source] of Object.entries(input.sources)) {
+    const placeholder = `${RELATION_PLACEHOLDER_PREFIX}${token}_${index}__`
+    relations[sourceName] = Object.freeze(new DuckLakeSqlTransformRelation(placeholder))
+    replacements.set(placeholder, renderDuckLakeSourceRelation(input.options, source))
+    index += 1
+  }
+
+  return {
+    relations: Object.freeze(relations),
+    render(sql) {
+      const rendered = sql.replace(RELATION_PLACEHOLDER_PATTERN, (placeholder) => {
+        const replacement = replacements.get(placeholder)
+        if (replacement === undefined) {
+          throwUnresolvedPlaceholder()
+        }
+
+        return replacement
+      })
+
+      if (rendered.includes(RELATION_PLACEHOLDER_PREFIX)) {
+        throwUnresolvedPlaceholder()
+      }
+
+      return rendered
+    },
+  }
+}
+
+function renderDuckLakeSourceRelation(
+  options: Pick<DuckLakeStorageOptions, "alias">,
+  source: DuckLakeSqlTransformSourceRelation
+): string {
+  const snapshotId = parseVersionId(source.versionId)
+  const tableName = encodeDatasetTableName(source.datasetId)
+  return `(SELECT * FROM ${qualifiedTableName(options, tableName)} AT (VERSION => ${snapshotId}))`
+}
+
+function throwUnresolvedPlaceholder(): never {
+  throw new LakeStorageError(
+    "[SixbDuckLake] SQL transform contains an unresolved relation placeholder."
+  )
+}