@sixb/core 0.0.1

package/src/connectors/runtime.ts

@@ -0,0 +1,100 @@
+import { ConnectorError, ConnectorNotFoundError } from "./errors"
+import type { ConnectorAdapter, ConnectorClient, ConnectorDefinition } from "./types"
+
+type ConnectorConnectionState = {
+  controller: AbortController
+  clientPromise: Promise<unknown>
+}
+
+export class ConnectorRuntime {
+  private readonly definitionsById = new Map<string, ConnectorDefinition>()
+  private readonly connectionStates = new Map<ConnectorDefinition, ConnectorConnectionState>()
+
+  constructor(
+    private readonly projectId: string,
+    definitions: readonly ConnectorDefinition[]
+  ) {
+    for (const definition of definitions) {
+      if (this.definitionsById.has(definition.id)) {
+        throw new ConnectorError(`Duplicate connector id: ${definition.id}`)
+      }
+
+      this.definitionsById.set(definition.id, definition)
+    }
+  }
+
+  list(): readonly ConnectorDefinition[] {
+    return [...this.definitionsById.values()]
+  }
+
+  getById(id: string): ConnectorDefinition | null {
+    return this.definitionsById.get(id) ?? null
+  }
+
+  async connect<TAdapter extends ConnectorAdapter>(
+    definition: ConnectorDefinition<string, TAdapter>
+  ): Promise<ConnectorClient<TAdapter>> {
+    const registeredDefinition = this.definitionsById.get(definition.id)
+
+    if (!registeredDefinition) {
+      throw new ConnectorNotFoundError(definition.id)
+    }
+
+    if (registeredDefinition !== definition) {
+      throw new ConnectorError(
+        `Connector '${definition.id}' is not the registered definition instance.`
+      )
+    }
+
+    let state = this.connectionStates.get(definition)
+    if (!state) {
+      const controller = new AbortController()
+      const clientPromise = Promise.resolve(
+        definition.adapter.connect({
+          projectId: this.projectId,
+          connectorId: definition.id,
+          signal: controller.signal,
+        })
+      )
+
+      state = { controller, clientPromise }
+      this.connectionStates.set(definition, state)
+    }
+
+    try {
+      return (await state.clientPromise) as ConnectorClient<TAdapter>
+    } catch (error) {
+      if (this.connectionStates.get(definition) === state) {
+        this.connectionStates.delete(definition)
+      }
+      throw error
+    }
+  }
+
+  async disconnectAll(): Promise<void> {
+    const activeConnections = [...this.connectionStates.entries()]
+    this.connectionStates.clear()
+
+    for (const [definition, state] of activeConnections) {
+      await this.disconnectState(definition, state)
+    }
+  }
+
+  private async disconnectState(
+    definition: ConnectorDefinition,
+    state: ConnectorConnectionState
+  ): Promise<void> {
+    state.controller.abort()
+
+    let client: unknown
+    try {
+      client = await state.clientPromise
+    } catch {
+      return
+    }
+
+    if (typeof definition.adapter.disconnect === "function") {
+      await definition.adapter.disconnect(client as never)
+    }
+  }
+}

package/src/connectors/types.ts

@@ -0,0 +1,64 @@
+import type { WebhookDefinition } from "../webhooks/types"
+
+/**
+ * Runtime context passed to connector adapters when Sixb establishes a connection.
+ *
+ * Connectors can use this to scope logs, build cache keys, or attach cancellation
+ * to long-running startup work.
+ */
+export interface ConnectorContext {
+  readonly projectId: string
+  readonly connectorId: string
+  readonly signal: AbortSignal
+}
+
+/**
+ * Minimal contract for a Sixb connector adapter.
+ *
+ * Adapters are responsible for creating and optionally tearing down a client for an
+ * external system. They should return the client shape that feels natural for that system.
+ */
+export interface ConnectorAdapter<TType extends string = string, TClient = unknown> {
+  readonly type: TType
+  readonly webhooks?: readonly WebhookDefinition<unknown, TClient>[]
+  connect(context: ConnectorContext): Promise<TClient> | TClient
+  disconnect?(client: TClient): Promise<void> | void
+}
+
+/**
+ * Inert connector definition registered with Sixb.
+ *
+ * Definitions are safe to export from `connectors/` modules. The runtime turns them into
+ * live clients when `sixb.connector(...)` is called.
+ */
+export interface ConnectorDefinition<
+  TId extends string = string,
+  TAdapter extends ConnectorAdapter = ConnectorAdapter,
+> {
+  readonly kind: "connector"
+  readonly id: TId
+  readonly adapter: TAdapter
+}
+
+/** Infer the connected client type returned by a connector adapter. */
+export type ConnectorClient<TAdapter extends ConnectorAdapter> = Awaited<
+  ReturnType<TAdapter["connect"]>
+>
+
+export function isConnectorDefinition(value: unknown): value is ConnectorDefinition {
+  if (!isRecord(value)) {
+    return false
+  }
+
+  return (
+    value.kind === "connector" &&
+    typeof value.id === "string" &&
+    isRecord(value.adapter) &&
+    typeof value.adapter.type === "string" &&
+    typeof value.adapter.connect === "function"
+  )
+}
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return typeof value === "object" && value !== null && !Array.isArray(value)
+}

package/src/datasets/builders.ts

@@ -0,0 +1,233 @@
+import { DatasetValidationError } from "./errors"
+import type {
+  DatasetColumnDefinition,
+  DatasetColumnNameOf,
+  DatasetColumnType,
+  DatasetDefinition,
+} from "./types"
+import { assertDatasetColumnDefinition, assertDatasetDefinition } from "./validation"
+
+type DatasetColumnOptions = {
+  readonly nullable?: boolean
+}
+
+type DefineDatasetOptions = {
+  readonly schema: readonly DatasetColumnDefinition[]
+  readonly partitionBy?: readonly string[]
+  readonly description?: string
+}
+
+type DeriveDatasetOptions<TParent extends DatasetDefinition> = {
+  readonly pick?: readonly DatasetColumnNameOf<TParent>[]
+  readonly add?: readonly DatasetColumnDefinition[]
+  readonly partitionBy?: readonly string[]
+  readonly description?: string
+}
+
+type FieldFromOptions<TOptions, TKey extends string, TFallback> = TOptions extends {
+  readonly [K in TKey]: infer TValue
+}
+  ? TValue extends TFallback
+    ? { readonly [K in TKey]: TValue }
+    : { readonly [K in TKey]?: TFallback }
+  : { readonly [K in TKey]?: TFallback }
+
+type ColumnNameOf<TColumns extends readonly DatasetColumnDefinition[]> = TColumns[number]["name"]
+
+type ColumnDefinitionOf<
+  TColumns extends readonly DatasetColumnDefinition[],
+  TName extends ColumnNameOf<TColumns>,
+> = Extract<TColumns[number], { readonly name: TName }>
+
+type PickColumns<
+  TColumns extends readonly DatasetColumnDefinition[],
+  TNames extends readonly ColumnNameOf<TColumns>[],
+> = TNames extends readonly [infer THead, ...infer TTail]
+  ? THead extends ColumnNameOf<TColumns>
+    ? TTail extends readonly ColumnNameOf<TColumns>[]
+      ? readonly [ColumnDefinitionOf<TColumns, THead>, ...PickColumns<TColumns, TTail>]
+      : never
+    : never
+  : TNames extends readonly []
+    ? readonly []
+    : readonly ColumnDefinitionOf<TColumns, TNames[number]>[]
+
+type AddColumns<
+  TColumns extends readonly DatasetColumnDefinition[],
+  TAdd extends readonly DatasetColumnDefinition[],
+> = readonly [...TColumns, ...TAdd]
+
+type DerivedBaseColumns<
+  TParent extends DatasetDefinition,
+  TOptions extends DeriveDatasetOptions<TParent> | undefined,
+> = TOptions extends { readonly pick: infer TPick }
+  ? TPick extends readonly DatasetColumnNameOf<TParent>[]
+    ? PickColumns<TParent["schema"]["columns"], TPick>
+    : TParent["schema"]["columns"]
+  : TParent["schema"]["columns"]
+
+type DerivedColumns<
+  TParent extends DatasetDefinition,
+  TOptions extends DeriveDatasetOptions<TParent> | undefined,
+> = TOptions extends { readonly add: infer TAdd }
+  ? TAdd extends readonly DatasetColumnDefinition[]
+    ? AddColumns<DerivedBaseColumns<TParent, TOptions>, TAdd>
+    : DerivedBaseColumns<TParent, TOptions>
+  : DerivedBaseColumns<TParent, TOptions>
+
+type DatasetColumnResult<
+  TName extends string,
+  TType extends DatasetColumnType,
+  TOptions extends DatasetColumnOptions | undefined,
+> = {
+  readonly name: TName
+  readonly type: TType
+} & FieldFromOptions<TOptions, "nullable", boolean>
+
+type DatasetDefinitionResult<TId extends string, TOptions extends DefineDatasetOptions> = Omit<
+  DatasetDefinition<TId, TOptions["schema"]>,
+  "partitionBy" | "description"
+> &
+  FieldFromOptions<TOptions, "partitionBy", readonly string[]> &
+  FieldFromOptions<TOptions, "description", string>
+
+type DerivedDatasetDefinitionResult<
+  TId extends string,
+  TParent extends DatasetDefinition,
+  TOptions extends DeriveDatasetOptions<TParent> | undefined,
+> = Omit<DatasetDefinition<TId, DerivedColumns<TParent, TOptions>>, "partitionBy" | "description"> &
+  FieldFromOptions<TOptions, "partitionBy", readonly string[]> &
+  FieldFromOptions<TOptions, "description", string>
+
+export function col<const TName extends string, const TType extends DatasetColumnType>(
+  name: TName,
+  type: TType
+): DatasetColumnResult<TName, TType, undefined>
+export function col<
+  const TName extends string,
+  const TType extends DatasetColumnType,
+  const TOptions extends DatasetColumnOptions,
+>(name: TName, type: TType, options: TOptions): DatasetColumnResult<TName, TType, TOptions>
+export function col(
+  name: string,
+  type: DatasetColumnType,
+  options?: DatasetColumnOptions
+): DatasetColumnDefinition {
+  const column: DatasetColumnDefinition = {
+    name,
+    type,
+    ...(options?.nullable !== undefined ? { nullable: options.nullable } : {}),
+  }
+
+  assertDatasetColumnDefinition(column, (message) => new DatasetValidationError(message))
+  return column
+}
+
+function assertDatasetId(id: string): void {
+  if (!id.trim()) {
+    throw new DatasetValidationError("Dataset id must not be empty.")
+  }
+}
+
+function createDatasetDefinition(id: string, options: DefineDatasetOptions): DatasetDefinition {
+  const definition: DatasetDefinition = {
+    kind: "dataset",
+    id,
+    schema: {
+      columns: [...options.schema],
+    },
+    ...(options.partitionBy !== undefined ? { partitionBy: [...options.partitionBy] } : {}),
+    ...(options.description !== undefined ? { description: options.description } : {}),
+  }
+
+  assertDatasetDefinition(definition, (message) => new DatasetValidationError(message))
+  return definition
+}
+
+function assertStringArray(value: unknown, field: string): asserts value is readonly string[] {
+  if (!Array.isArray(value)) {
+    throw new DatasetValidationError(`${field} must be an array of column names.`)
+  }
+
+  for (const item of value) {
+    if (typeof item !== "string" || !item.trim()) {
+      throw new DatasetValidationError(`${field} must contain only non-empty column names.`)
+    }
+  }
+}
+
+function deriveDatasetDefinition(
+  id: string,
+  parent: DatasetDefinition,
+  options: DeriveDatasetOptions<DatasetDefinition> = {}
+): DatasetDefinition {
+  assertDatasetDefinition(parent, (message) => new DatasetValidationError(message))
+
+  if (options.pick !== undefined) {
+    assertStringArray(options.pick, "Dataset derive pick")
+  }
+
+  if (options.add !== undefined && !Array.isArray(options.add)) {
+    throw new DatasetValidationError("Dataset derive add must be an array of columns.")
+  }
+
+  const parentColumnsByName = new Map(
+    parent.schema.columns.map((column) => [column.name, column] as const)
+  )
+  const pickedColumns =
+    options.pick === undefined
+      ? parent.schema.columns
+      : options.pick.map((columnName) => {
+          const column = parentColumnsByName.get(columnName)
+          if (column === undefined) {
+            throw new DatasetValidationError(
+              `Dataset derive pick column '${columnName}' is not declared on parent dataset '${parent.id}'.`
+            )
+          }
+          return column
+        })
+
+  return createDatasetDefinition(id, {
+    schema: [...pickedColumns, ...(options.add ?? [])],
+    ...(options.partitionBy !== undefined ? { partitionBy: options.partitionBy } : {}),
+    ...(options.description !== undefined ? { description: options.description } : {}),
+  })
+}
+
+function createDatasetDeriveBuilder<TId extends string>(id: TId) {
+  assertDatasetId(id)
+
+  function derive<const TParent extends DatasetDefinition>(
+    parent: TParent
+  ): DerivedDatasetDefinitionResult<TId, TParent, undefined>
+  function derive<
+    const TParent extends DatasetDefinition,
+    const TOptions extends DeriveDatasetOptions<TParent>,
+  >(parent: TParent, options: TOptions): DerivedDatasetDefinitionResult<TId, TParent, TOptions>
+  function derive(
+    parent: DatasetDefinition,
+    options?: DeriveDatasetOptions<DatasetDefinition>
+  ): DatasetDefinition {
+    return deriveDatasetDefinition(id, parent, options)
+  }
+
+  return { derive }
+}
+
+export function defineDataset<const TId extends string>(
+  id: TId
+): ReturnType<typeof createDatasetDeriveBuilder<TId>>
+export function defineDataset<
+  const TId extends string,
+  const TOptions extends DefineDatasetOptions,
+>(id: TId, options: TOptions): DatasetDefinitionResult<TId, TOptions>
+export function defineDataset(
+  id: string,
+  options?: DefineDatasetOptions
+): DatasetDefinition | ReturnType<typeof createDatasetDeriveBuilder<string>> {
+  if (options === undefined) {
+    return createDatasetDeriveBuilder(id)
+  }
+
+  return createDatasetDefinition(id, options)
+}

package/src/datasets/errors.ts

@@ -0,0 +1,3 @@
+export class DatasetValidationError extends Error {
+  readonly name = "DatasetValidationError"
+}

package/src/datasets/index.ts

@@ -0,0 +1,13 @@
+export { col, defineDataset } from "./builders"
+export { DatasetValidationError } from "./errors"
+export type {
+  DatasetColumnDefinition,
+  DatasetColumnDefinitionOf,
+  DatasetColumnNameOf,
+  DatasetColumnType,
+  DatasetColumnTypeOf,
+  DatasetColumnUnionOf,
+  DatasetDefinition,
+  DatasetSchema,
+} from "./types"
+export { getDatasetRowValidationError, isDatasetDefinition } from "./validation"

package/src/datasets/types.ts

@@ -0,0 +1,52 @@
+export type DatasetColumnType =
+  | "string"
+  | "boolean"
+  | "int64"
+  | "float64"
+  | "decimal"
+  | "date"
+  | "timestamp"
+  | "json"
+  | "fileRef"
+
+export interface DatasetColumnDefinition<
+  TName extends string = string,
+  TType extends DatasetColumnType = DatasetColumnType,
+> {
+  readonly name: TName
+  readonly type: TType
+  readonly nullable?: boolean
+}
+
+export interface DatasetSchema<
+  TColumns extends readonly DatasetColumnDefinition[] = readonly DatasetColumnDefinition[],
+> {
+  readonly columns: TColumns
+}
+
+export interface DatasetDefinition<
+  TId extends string = string,
+  TColumns extends readonly DatasetColumnDefinition[] = readonly DatasetColumnDefinition[],
+> {
+  readonly kind: "dataset"
+  readonly id: TId
+  readonly schema: DatasetSchema<TColumns>
+  readonly partitionBy?: readonly string[]
+  readonly description?: string
+}
+
+export type DatasetColumnUnionOf<TDataset extends DatasetDefinition> =
+  TDataset["schema"]["columns"][number]
+
+export type DatasetColumnNameOf<TDataset extends DatasetDefinition> =
+  DatasetColumnUnionOf<TDataset>["name"]
+
+export type DatasetColumnDefinitionOf<
+  TDataset extends DatasetDefinition,
+  TName extends DatasetColumnNameOf<TDataset>,
+> = Extract<DatasetColumnUnionOf<TDataset>, { readonly name: TName }>
+
+export type DatasetColumnTypeOf<
+  TDataset extends DatasetDefinition,
+  TName extends DatasetColumnNameOf<TDataset>,
+> = DatasetColumnDefinitionOf<TDataset, TName>["type"]