@sixb/core 0.0.1

package/src/lake-storage/in-memory.ts

@@ -0,0 +1,348 @@
+import { randomUUID } from "node:crypto"
+import type { DatasetDefinition, DatasetSchema } from "../datasets"
+import { getDatasetRowValidationError } from "../datasets/validation"
+import { mergeStrictDatasetDefinition } from "./definition-updates"
+import { LakeStorageError } from "./errors"
+import type {
+  BeginDatasetWriteInput,
+  CommitDatasetWriteInput,
+  DatasetCatalogState,
+  DatasetRow,
+  DatasetVersion,
+  DatasetWriteCommitResult,
+  DatasetWriteMode,
+  LakeStorage,
+  LakeWriteSession,
+  ReadDatasetRowsInput,
+} from "./types"
+
+function cloneDatasetDefinition(definition: DatasetDefinition): DatasetDefinition {
+  return structuredClone(definition)
+}
+
+function cloneDatasetVersion(version: DatasetVersion): DatasetVersion {
+  return {
+    ...structuredClone(version),
+    createdAt: new Date(version.createdAt),
+  }
+}
+
+function cloneRow(row: DatasetRow): DatasetRow {
+  return structuredClone(row)
+}
+
+function selectColumns(row: DatasetRow, columns?: readonly string[]): DatasetRow {
+  if (!columns || columns.length === 0) {
+    return cloneRow(row)
+  }
+
+  const selected: Record<string, unknown> = {}
+  for (const column of columns) {
+    selected[column] = row[column]
+  }
+  return selected
+}
+
+// Canonical content key for unchanged-write detection. JSON.stringify
+// serializes Date values via toJSON, matching how rows persist as JSON;
+// missing and undefined nullable values normalize to null. The unchanged-write
+// semantics are pinned by the shared lake-storage contract suite.
+function rowContentKey(row: DatasetRow, schema: DatasetSchema): string {
+  return JSON.stringify(schema.columns.map((column) => row[column.name] ?? null))
+}
+
+// Order-insensitive multiset equality over canonical row keys.
+function sameRowContent(left: readonly string[], right: readonly string[]): boolean {
+  if (left.length !== right.length) {
+    return false
+  }
+
+  const sortedLeft = [...left].sort()
+  const sortedRight = [...right].sort()
+  return sortedLeft.every((key, index) => key === sortedRight[index])
+}
+
+function assertDatasetId(datasetId: string): void {
+  if (datasetId.trim().length === 0) {
+    throw new LakeStorageError("[LakeStorage] Dataset id must not be empty")
+  }
+}
+
+function errorMessage(error: unknown): string {
+  return error instanceof Error ? error.message : String(error)
+}
+
+class InMemoryLakeWriteSession implements LakeWriteSession {
+  private readonly rows: DatasetRow[] = []
+  private closed = false
+
+  constructor(
+    private readonly storage: InMemoryLakeStorage,
+    private readonly input: BeginDatasetWriteInput
+  ) {}
+
+  async writeRows(rows: Iterable<DatasetRow> | AsyncIterable<DatasetRow>): Promise<void> {
+    this.assertOpen()
+
+    for await (const row of rows) {
+      const validationError = getDatasetRowValidationError(row, this.input.dataset)
+      if (validationError) {
+        throw new LakeStorageError(`[LakeStorage] ${validationError}`)
+      }
+
+      this.rows.push(cloneRow(row))
+    }
+  }
+
+  async commit(input?: CommitDatasetWriteInput): Promise<DatasetWriteCommitResult> {
+    this.assertOpen()
+    this.closed = true
+    return this.storage.commitWrite({
+      write: this.input,
+      rows: this.rows,
+      commit: input,
+    })
+  }
+
+  async abort(): Promise<void> {
+    this.closed = true
+    this.rows.length = 0
+  }
+
+  private assertOpen(): void {
+    if (this.closed) {
+      throw new LakeStorageError("[LakeStorage] Write session is already closed")
+    }
+  }
+}
+
+export class InMemoryLakeStorage implements LakeStorage {
+  private readonly datasets = new Map<string, DatasetDefinition>()
+  private readonly versionsByDataset = new Map<string, DatasetVersion[]>()
+  private readonly rowsByVersionId = new Map<string, readonly DatasetRow[]>()
+  private readonly latestVersionIdByDataset = new Map<string, string>()
+
+  async createDataset(definition: DatasetDefinition): Promise<DatasetDefinition> {
+    assertDatasetId(definition.id)
+    const existing = this.datasets.get(definition.id)
+    const stored = mergeStrictDatasetDefinition({
+      existing,
+      next: definition,
+    })
+    this.datasets.set(definition.id, stored)
+    return cloneDatasetDefinition(stored)
+  }
+
+  async assertDatasetDefinitionsCompatible(
+    definitions: readonly DatasetDefinition[]
+  ): Promise<void> {
+    const failures: string[] = []
+
+    for (const definition of definitions) {
+      try {
+        assertDatasetId(definition.id)
+        const existing = this.datasets.get(definition.id)
+        if (!existing) {
+          continue
+        }
+
+        mergeStrictDatasetDefinition({
+          existing,
+          next: definition,
+        })
+      } catch (error) {
+        failures.push(`- ${definition.id}: ${errorMessage(error)}`)
+      }
+    }
+
+    if (failures.length > 0) {
+      const details = failures.join("\n")
+      throw new LakeStorageError(
+        `[SixbLake] Lake dataset definition check failed for ${failures.length} dataset(s).\n${details}`
+      )
+    }
+  }
+
+  async getDataset(datasetId: string): Promise<DatasetDefinition | null> {
+    const definition = this.datasets.get(datasetId)
+    return definition ? cloneDatasetDefinition(definition) : null
+  }
+
+  async listDatasets(): Promise<readonly DatasetDefinition[]> {
+    return [...this.datasets.values()].map((definition) => cloneDatasetDefinition(definition))
+  }
+
+  async listDatasetCatalogState(
+    datasetIds: readonly string[]
+  ): Promise<readonly DatasetCatalogState[]> {
+    return Promise.all(
+      datasetIds.map(async (datasetId) => {
+        if (!this.datasets.has(datasetId)) {
+          return { datasetId, materialized: false, latestVersion: null }
+        }
+
+        const latest = await this.getLatestVersion(datasetId)
+        return {
+          datasetId,
+          materialized: true,
+          latestVersion: latest
+            ? {
+                datasetId: latest.datasetId,
+                versionId: latest.versionId,
+                mode: latest.mode,
+                createdAt: latest.createdAt,
+                ...(latest.rowCount !== undefined ? { rowCount: latest.rowCount } : {}),
+              }
+            : null,
+        }
+      })
+    )
+  }
+
+  async listVersions(datasetId: string, limit?: number): Promise<readonly DatasetVersion[]> {
+    const versions = [...(this.versionsByDataset.get(datasetId) ?? [])]
+      .sort((left, right) => right.createdAt.getTime() - left.createdAt.getTime())
+      .map((version) => cloneDatasetVersion(version))
+
+    return limit === undefined ? versions : versions.slice(0, Math.max(0, limit))
+  }
+
+  async beginWrite(input: BeginDatasetWriteInput): Promise<LakeWriteSession> {
+    const definition = await this.getDataset(input.dataset.id)
+    if (!definition) {
+      throw new LakeStorageError(`[LakeStorage] Unknown dataset '${input.dataset.id}'`)
+    }
+
+    return new InMemoryLakeWriteSession(this, {
+      ...input,
+      dataset: definition,
+      mode: input.mode ?? "snapshot",
+    })
+  }
+
+  async getLatestVersion(datasetId: string): Promise<DatasetVersion | null> {
+    const latestVersionId = this.latestVersionIdByDataset.get(datasetId)
+    if (!latestVersionId) {
+      return null
+    }
+
+    return this.getVersion(datasetId, latestVersionId)
+  }
+
+  async getVersion(datasetId: string, versionId: string): Promise<DatasetVersion | null> {
+    const versions = this.versionsByDataset.get(datasetId) ?? []
+    const version = versions.find((candidate) => candidate.versionId === versionId)
+    return version ? cloneDatasetVersion(version) : null
+  }
+
+  async *readRows(input: ReadDatasetRowsInput): AsyncIterable<DatasetRow> {
+    const definition = this.datasets.get(input.datasetId)
+    if (!definition) {
+      throw new LakeStorageError(`[LakeStorage] Unknown dataset '${input.datasetId}'`)
+    }
+
+    const version =
+      input.versionId === undefined
+        ? await this.getLatestVersion(input.datasetId)
+        : await this.getVersion(input.datasetId, input.versionId)
+
+    if (!version) {
+      throw new LakeStorageError(
+        `[LakeStorage] No committed version found for dataset '${input.datasetId}'`
+      )
+    }
+
+    const offset = Math.max(0, input.offset ?? 0)
+    const limit = input.limit === undefined ? undefined : Math.max(0, input.limit)
+    const rows = this.rowsByVersionId
+      .get(version.versionId)
+      ?.slice(offset, limit === undefined ? undefined : offset + limit)
+
+    for (const row of rows ?? []) {
+      yield selectColumns(row, input.columns)
+    }
+  }
+
+  async commitWrite(options: {
+    write: BeginDatasetWriteInput
+    rows: readonly DatasetRow[]
+    commit?: CommitDatasetWriteInput
+  }): Promise<DatasetWriteCommitResult> {
+    const definition = this.datasets.get(options.write.dataset.id)
+    if (!definition) {
+      throw new LakeStorageError(`[LakeStorage] Unknown dataset '${options.write.dataset.id}'`)
+    }
+
+    const mode = options.write.mode ?? "snapshot"
+    const latestVersion = await this.getLatestVersion(options.write.dataset.id)
+
+    if (options.commit?.expectedLatestVersionId !== undefined) {
+      const actual = latestVersion?.versionId
+      if (actual !== options.commit.expectedLatestVersionId) {
+        throw new LakeStorageError(
+          `[LakeStorage] Optimistic commit failed for dataset '${options.write.dataset.id}': expected latest version '${options.commit.expectedLatestVersionId}', found '${actual ?? "none"}'`
+        )
+      }
+    }
+
+    // Content-identical snapshots and empty appends reuse the latest version
+    // instead of creating a new one.
+    if (
+      latestVersion &&
+      this.isUnchangedWrite(mode, latestVersion, options.rows, definition.schema)
+    ) {
+      return { ...latestVersion, outcome: "unchanged" }
+    }
+
+    const versionId = `ver_${randomUUID()}`
+    const visibleRows =
+      mode === "append"
+        ? [...(this.rowsByVersionId.get(latestVersion?.versionId ?? "") ?? []), ...options.rows]
+        : [...options.rows]
+
+    const sizeBytes = new TextEncoder().encode(
+      visibleRows.map((row) => JSON.stringify(row)).join("\n")
+    ).byteLength
+
+    const version: DatasetVersion = {
+      datasetId: options.write.dataset.id,
+      versionId,
+      parentVersionId: mode === "append" ? latestVersion?.versionId : undefined,
+      mode,
+      createdAt: new Date(),
+      schema: definition.schema,
+      producer: options.write.producer ? structuredClone(options.write.producer) : undefined,
+      inputs: options.write.inputs ? structuredClone(options.write.inputs) : undefined,
+      rowCount: visibleRows.length,
+      sizeBytes,
+    }
+
+    const versions = this.versionsByDataset.get(options.write.dataset.id) ?? []
+    versions.push(version)
+    this.versionsByDataset.set(options.write.dataset.id, versions)
+    this.rowsByVersionId.set(
+      versionId,
+      visibleRows.map((row) => cloneRow(row))
+    )
+    this.latestVersionIdByDataset.set(options.write.dataset.id, versionId)
+
+    return { ...cloneDatasetVersion(version), outcome: "created" }
+  }
+
+  private isUnchangedWrite(
+    mode: DatasetWriteMode,
+    latestVersion: DatasetVersion,
+    rows: readonly DatasetRow[],
+    schema: DatasetSchema
+  ): boolean {
+    if (mode === "append") {
+      return rows.length === 0
+    }
+
+    const previousRows = this.rowsByVersionId.get(latestVersion.versionId) ?? []
+    return sameRowContent(
+      rows.map((row) => rowContentKey(row, schema)),
+      previousRows.map((row) => rowContentKey(row, schema))
+    )
+  }
+}

package/src/lake-storage/index.ts

@@ -0,0 +1,48 @@
+export type {
+  DatasetColumnDefinition,
+  DatasetColumnType,
+  DatasetDefinition,
+  DatasetSchema,
+} from "../datasets"
+export type { JsonValue } from "../json"
+export type {
+  DatasetDefinitionUpdatePlan,
+  DatasetMetadataUpdatePlan,
+  DatasetSchemaUpdatePlan,
+} from "./definition-updates"
+export {
+  assertLakeDatasetDefinitionsCompatible,
+  mergeStrictDatasetDefinition,
+  planDatasetDefinitionUpdate,
+} from "./definition-updates"
+export { LakeStorageError } from "./errors"
+export { InMemoryLakeStorage } from "./in-memory"
+export type {
+  ExecuteSqlTransformInput,
+  LakeSqlExecutor,
+  LakeSqlTransformCapabilities,
+  LakeStorageWithSql,
+  PreviewSqlTransformInput,
+  SqlDialect,
+  SqlTransformBody,
+  SqlTransformRelation,
+  SqlTransformSource,
+} from "./sql-transforms"
+export type {
+  BeginDatasetWriteInput,
+  CommitDatasetWriteInput,
+  DatasetCatalogState,
+  DatasetLatestVersionSummary,
+  DatasetProducer,
+  DatasetRow,
+  DatasetVersion,
+  DatasetVersionMode,
+  DatasetVersionRef,
+  DatasetWriteCommitResult,
+  DatasetWriteMode,
+  LakeStandardDescriptor,
+  LakeStandardId,
+  LakeStorage,
+  LakeWriteSession,
+  ReadDatasetRowsInput,
+} from "./types"

package/src/lake-storage/sql-transforms.ts

@@ -0,0 +1,67 @@
+import type { DatasetDefinition } from "../datasets"
+import type {
+  DatasetProducer,
+  DatasetRow,
+  DatasetWriteCommitResult,
+  DatasetWriteMode,
+  LakeStorage,
+} from "./types"
+
+export const SQL_DIALECT = {
+  duckdb: "duckdb",
+} as const
+
+export const SQL_DIALECTS = Object.values(SQL_DIALECT)
+export type SqlDialect = (typeof SQL_DIALECT)[keyof typeof SQL_DIALECT]
+
+export interface LakeStorageWithSql<TSqlDialect extends SqlDialect = SqlDialect>
+  extends LakeStorage {
+  readonly sql: LakeSqlExecutor<TSqlDialect>
+}
+
+export interface LakeSqlExecutor<TSqlDialect extends SqlDialect = SqlDialect> {
+  readonly dialect: TSqlDialect
+  readonly capabilities: LakeSqlTransformCapabilities
+
+  preview(input: PreviewSqlTransformInput<TSqlDialect>): AsyncIterable<DatasetRow>
+  execute(input: ExecuteSqlTransformInput<TSqlDialect>): Promise<DatasetWriteCommitResult>
+}
+
+export interface LakeSqlTransformCapabilities {
+  readonly preview: boolean
+  readonly supportsAppend: boolean
+  readonly supportsSnapshot: boolean
+}
+
+export interface ExecuteSqlTransformInput<TSqlDialect extends SqlDialect = SqlDialect> {
+  readonly sources: Readonly<Record<string, SqlTransformSource>>
+  readonly sql: SqlTransformBody<TSqlDialect>
+  readonly target: DatasetDefinition
+  readonly mode: DatasetWriteMode
+  readonly producer?: DatasetProducer
+  readonly expectedLatestVersionId?: string
+  readonly commitMessage?: string
+}
+
+export interface PreviewSqlTransformInput<TSqlDialect extends SqlDialect = SqlDialect> {
+  readonly sources: Readonly<Record<string, SqlTransformSource>>
+  readonly sql: SqlTransformBody<TSqlDialect>
+  readonly limit?: number
+}
+
+export interface SqlTransformSource {
+  readonly dataset: DatasetDefinition
+  readonly versionId?: string
+}
+
+export interface SqlTransformRelation {
+  toString(): string
+}
+
+export type SqlTransformBody<_TSqlDialect extends SqlDialect = SqlDialect> = (
+  relations: Readonly<Record<string, SqlTransformRelation>>
+) => string
+
+export function isSqlDialect(value: unknown): value is SqlDialect {
+  return typeof value === "string" && SQL_DIALECTS.includes(value as SqlDialect)
+}

package/src/lake-storage/types.ts

@@ -0,0 +1,135 @@
+import type { DatasetDefinition, DatasetSchema } from "../datasets"
+
+export type DatasetWriteMode = "snapshot" | "append"
+export type DatasetVersionMode = DatasetWriteMode | "schema"
+
+export type DatasetRow = Readonly<Record<string, unknown>>
+
+export type LakeStandardId = "ducklake"
+
+export interface LakeStandardDescriptor<TStandard extends LakeStandardId = LakeStandardId> {
+  readonly id: TStandard
+  readonly version?: string
+}
+
+export interface DatasetVersionRef {
+  readonly datasetId: string
+  readonly versionId: string
+}
+
+export interface DatasetProducer {
+  readonly kind: "sync" | "pipeline"
+  readonly id?: string
+  readonly runId?: string
+  readonly stepId?: string
+}
+
+export interface DatasetVersion {
+  readonly datasetId: string
+  readonly versionId: string
+  readonly parentVersionId?: string
+  readonly mode: DatasetVersionMode
+  readonly createdAt: Date
+  readonly schema: DatasetSchema
+  readonly producer?: DatasetProducer
+  readonly inputs?: readonly DatasetVersionRef[]
+  readonly rowCount?: number
+  readonly sizeBytes?: number
+}
+
+export interface DatasetLatestVersionSummary {
+  readonly datasetId: string
+  readonly versionId: string
+  readonly mode: DatasetVersionMode
+  readonly createdAt: Date
+  readonly rowCount?: number
+}
+
+export interface DatasetCatalogState {
+  readonly datasetId: string
+  readonly materialized: boolean
+  readonly latestVersion?: DatasetLatestVersionSummary | null
+}
+
+export interface BeginDatasetWriteInput {
+  readonly dataset: DatasetDefinition
+  readonly mode?: DatasetWriteMode
+  readonly producer?: DatasetProducer
+  readonly inputs?: readonly DatasetVersionRef[]
+}
+
+export interface ReadDatasetRowsInput {
+  readonly datasetId: string
+  /**
+   * Pins an immutable physical row sequence. For an explicit version, providers must return the
+   * same order across reads and reopen; `offset: N` is exactly the suffix starting at row N.
+   */
+  readonly versionId?: string
+  /** Column projection must not alter the physical row order. */
+  readonly columns?: readonly string[]
+  readonly limit?: number
+  readonly offset?: number
+}
+
+export interface CommitDatasetWriteInput {
+  readonly expectedLatestVersionId?: string
+  readonly commitMessage?: string
+}
+
+export interface DatasetWriteCommitResult extends DatasetVersion {
+  /**
+   * Whether this operation created the returned version or reused the
+   * unchanged latest version. When a latest version exists, providers must
+   * report `unchanged` and reuse it for commits that cannot change visible
+   * dataset content: a snapshot whose rows equal the latest visible rows
+   * (order-insensitive, with equal duplicate counts), or an append of zero
+   * rows.
+   */
+  readonly outcome: "created" | "unchanged"
+}
+
+export interface LakeWriteSession {
+  writeRows(rows: Iterable<DatasetRow> | AsyncIterable<DatasetRow>): Promise<void>
+  commit(input?: CommitDatasetWriteInput): Promise<DatasetWriteCommitResult>
+  abort(): Promise<void>
+}
+
+export interface LakeStorage {
+  readonly standard?: LakeStandardDescriptor
+
+  /**
+   * Read-only preflight for hosts that want to fail before serving traffic.
+   *
+   * Implementations must not create datasets, apply DDL, write rows, or commit
+   * versions from this method. Missing persisted datasets should be treated as
+   * compatible because createDataset remains the materialization path.
+   */
+  assertDatasetDefinitionsCompatible(definitions: readonly DatasetDefinition[]): Promise<void>
+
+  createDataset(definition: DatasetDefinition): Promise<DatasetDefinition>
+  getDataset(datasetId: string): Promise<DatasetDefinition | null>
+  listDatasets(): Promise<readonly DatasetDefinition[]>
+
+  /**
+   * Bulk catalog-summary read for the dataset list view.
+   *
+   * Returns lightweight materialized + latest-version state for the requested
+   * dataset ids using a bounded number of catalog calls, never a per-dataset
+   * full version hydration or a count(*) over dataset contents. Row counts are
+   * only populated when storage already knows them cheaply (such as Sixb commit
+   * metadata); otherwise `rowCount` is omitted.
+   */
+  listDatasetCatalogState(datasetIds: readonly string[]): Promise<readonly DatasetCatalogState[]>
+
+  listVersions(datasetId: string, limit?: number): Promise<readonly DatasetVersion[]>
+
+  beginWrite(input: BeginDatasetWriteInput): Promise<LakeWriteSession>
+
+  getLatestVersion(datasetId: string): Promise<DatasetVersion | null>
+  getVersion(datasetId: string, versionId: string): Promise<DatasetVersion | null>
+  /** Explicit immutable-version reads have stable physical order and stable offset semantics. */
+  readRows(input: ReadDatasetRowsInput): AsyncIterable<DatasetRow>
+
+  /** Release external resources. Optional: a provider that owns none omits it. */
+  close?(): void | Promise<void>
+}

package/src/logging/console-logger.ts

@@ -0,0 +1,63 @@
+import {
+  isLevelEnabled,
+  type LogEntry,
+  type Logger,
+  type LoggerProvider,
+  type LogLevel,
+} from "./types"
+
+/** Options for {@link ConsoleLogger}. */
+export interface ConsoleLoggerOptions {
+  /** Entries below this level are not printed. Defaults to `"info"`. */
+  readonly level?: LogLevel
+}
+
+const CONSOLE_METHOD: Record<LogLevel, "debug" | "info" | "warn" | "error"> = {
+  debug: "debug",
+  info: "info",
+  warn: "warn",
+  error: "error",
+}
+
+/** Opt-in console output provider. Broker capture is configured independently. */
+export class ConsoleLogger implements LoggerProvider {
+  readonly level: LogLevel
+
+  constructor(options: ConsoleLoggerOptions = {}) {
+    this.level = options.level ?? "info"
+  }
+
+  write(entry: LogEntry): void {
+    if (!isLevelEnabled(entry.level, this.level)) {
+      return
+    }
+
+    let suffix = ""
+    try {
+      suffix = ` ${JSON.stringify({
+        ...(entry.fields ?? {}),
+        sixb: entry.context,
+      })}`
+    } catch {
+      suffix = ` ${JSON.stringify({ sixb: entry.context, sixb_unloggableFields: true })}`
+    }
+
+    console[CONSOLE_METHOD[entry.level]](`${entry.level.toUpperCase()} ${entry.message}${suffix}`)
+  }
+}
+
+/** Output provider used by minimal worker contexts with no configured runtime. */
+export const noopLoggerProvider: LoggerProvider = {
+  write() {},
+}
+
+/** Handler façade used by tests and contexts where logging is intentionally absent. */
+export const noopLogger: Logger = {
+  debug() {},
+  info() {},
+  warn() {},
+  error() {},
+  child() {
+    return noopLogger
+  },
+}

package/src/logging/index.ts

@@ -0,0 +1,43 @@
+export {
+  ConsoleLogger,
+  type ConsoleLoggerOptions,
+  noopLogger,
+  noopLoggerProvider,
+} from "./console-logger"
+export {
+  type LogsObservabilityOptions,
+  type LogsPage,
+  type LogsReadInput,
+  LogsRuntime,
+  type LogsRuntimeOptions,
+  type LogsSubscribeInput,
+  type LogsTailInput,
+  type ObservabilityOptions,
+  resolveLogsRuntime,
+} from "./runtime"
+export {
+  DEFAULT_LOGS_MAX_BYTES,
+  DEFAULT_LOGS_RETENTION,
+  LOGS_STREAM,
+} from "./stream"
+export {
+  isLevelEnabled,
+  isLogLevel,
+  isLogRecord,
+  isSixbRunKind,
+  isStoredLogLine,
+  LOG_LEVELS,
+  type LogContext,
+  type LogEntry,
+  type LogFields,
+  type Logger,
+  type LoggerProvider,
+  type LogLevel,
+  type LogRecord,
+  type LogRunRef,
+  logLevelsAtOrAbove,
+  normalizeLogError,
+  SIXB_RUN_KINDS,
+  type SixbRunKind,
+  type StoredLogLine,
+} from "./types"