@sixb/core 0.0.1

package/src/pipelines/index.ts

@@ -0,0 +1,26 @@
+export { definePipeline, definePipelineStep } from "./builders"
+export { PipelineError } from "./errors"
+export type {
+  PipelineRunRequestOptions,
+  PipelineRunRequestResult,
+  RequestPipelineRunInput,
+} from "./request"
+export { requestPipelineRun } from "./request"
+export type {
+  PipelineBuilder,
+  PipelineDefinition,
+  PipelineGraph,
+  PipelineSequenceGraph,
+  PipelineStepDefinition,
+  PipelineStepExecutor,
+  PipelineStepExecutorBuilder,
+  PipelineStepInput,
+  PipelineStepInputBuilder,
+  PipelineStepNode,
+  PipelineStepOutput,
+  PipelineStepOutputBuilder,
+  PipelineStepOutputOptions,
+  PipelineStepRunContext,
+  PipelineStepRunHandler,
+} from "./types"
+export { isPipelineDefinition, isPipelineStepDefinition } from "./types"

package/src/pipelines/request.ts

@@ -0,0 +1,71 @@
+import { randomUUID } from "node:crypto"
+import { assertAuthorized } from "../authorization"
+import type { SixbRuntimeContext } from "../runtime/types"
+import { PipelineError } from "./errors"
+import type { PipelineDefinition } from "./types"
+
+export interface PipelineRunRequestOptions {
+  /**
+   * Identity of the run to create. Deduplicated when a worker claims the job, not here: two requests
+   * carrying the same id enqueue two jobs, and the second loses to the unique run id. So one run
+   * happens, but this call cannot tell you whether it was yours.
+   */
+  readonly runId?: string
+}
+
+export interface RequestPipelineRunInput extends PipelineRunRequestOptions {
+  readonly pipelineId: string
+}
+
+export interface PipelineRunRequestResult {
+  readonly pipelineId: string
+  readonly runId: string
+  /** When the job was enqueued. Not persisted — the run row records `startedAt` instead. */
+  readonly queuedAt: string
+  readonly jobId?: string
+}
+
+/**
+ * Authorize and queue a pipeline run request.
+ *
+ * Operates on an already-resolved definition and the shared runtime context, so the HTTP route and
+ * the runtime verb cannot drift apart.
+ *
+ * Like `requestSyncRun` and unlike `requestWorkflowRun`, the run row is created by the worker that
+ * claims the job — see the note there for why the two cannot be aligned by adding a field.
+ */
+export async function requestPipelineRun(
+  runtime: SixbRuntimeContext,
+  pipeline: PipelineDefinition,
+  options: PipelineRunRequestOptions = {}
+): Promise<PipelineRunRequestResult> {
+  assertAuthorized(runtime, { kind: "pipeline.run", pipelineId: pipeline.id })
+  if (!runtime.storage.pipelineRuns) {
+    throw new PipelineError("[Sixb] Pipeline run storage is not configured.")
+  }
+
+  const queue = runtime.queues.pipelines
+  if (!queue) {
+    throw new PipelineError("[Sixb] Pipeline run queue is not configured.")
+  }
+
+  const runId = createPipelineRunId(options.runId)
+  const queuedAt = new Date().toISOString()
+  const [job] = await queue.enqueue({
+    projectId: runtime.projectId,
+    jobs: [{ type: "pipeline.run.requested", payload: { pipelineId: pipeline.id, runId } }],
+  })
+
+  return { pipelineId: pipeline.id, runId, queuedAt, jobId: job?.id }
+}
+
+function createPipelineRunId(runId: string | undefined): string {
+  if (runId !== undefined) {
+    if (!runId.trim()) {
+      throw new PipelineError("[Sixb] Pipeline run id must not be empty")
+    }
+    return runId
+  }
+
+  return `run_${randomUUID()}`
+}

package/src/pipelines/types.ts

@@ -0,0 +1,199 @@
+import type { DatasetDefinition } from "../datasets"
+import { isDatasetDefinition } from "../datasets"
+import type {
+  DatasetRow,
+  DatasetVersion,
+  DatasetWriteMode,
+  ReadDatasetRowsInput,
+} from "../lake-storage"
+import {
+  isSqlDialect,
+  type SqlDialect,
+  type SqlTransformBody,
+} from "../lake-storage/sql-transforms"
+import type { Logger } from "../logging"
+import type { ScheduleDefinition, ScheduleReference } from "../schedules"
+import { isScheduleReference } from "../schedules"
+
+// ── Step execution ─────────────────────────────────────────
+
+export type PipelineStepExecutor =
+  | {
+      readonly kind: "sql"
+      readonly dialect: SqlDialect
+      readonly sql: SqlTransformBody<SqlDialect>
+    }
+  | {
+      readonly kind: "run"
+      readonly handler: PipelineStepRunHandler
+    }
+
+export interface PipelineStepInput {
+  readonly dataset: DatasetDefinition
+  readonly version: DatasetVersion
+  readRows(input?: Omit<ReadDatasetRowsInput, "datasetId" | "versionId">): AsyncIterable<DatasetRow>
+}
+
+export interface PipelineStepOutput {
+  writeRows(rows: Iterable<DatasetRow> | AsyncIterable<DatasetRow>): Promise<void>
+}
+
+export interface PipelineStepRunContext<
+  TInputs extends Record<string, DatasetDefinition> = Record<string, DatasetDefinition>,
+> {
+  readonly projectId: string
+  readonly pipelineId: string
+  readonly stepId: string
+  readonly runId: string
+  readonly signal: AbortSignal
+  readonly inputs: { readonly [K in keyof TInputs]: PipelineStepInput }
+  readonly output: PipelineStepOutput
+  readonly logger: Logger
+}
+
+export type PipelineStepRunHandler<
+  TInputs extends Record<string, DatasetDefinition> = Record<string, DatasetDefinition>,
+> = (ctx: PipelineStepRunContext<TInputs>) => Promise<void> | void
+
+// ── Step definition ────────────────────────────────────────
+
+export interface PipelineStepDefinition<
+  TId extends string = string,
+  TInputs extends Record<string, DatasetDefinition> = Record<string, DatasetDefinition>,
+> {
+  readonly kind: "pipeline.step"
+  readonly id: TId
+  readonly inputs: Readonly<TInputs>
+  readonly output: DatasetDefinition
+  readonly mode: DatasetWriteMode
+  readonly executor: PipelineStepExecutor
+}
+
+// ── Pipeline definition ────────────────────────────────────
+
+export interface PipelineStepNode {
+  readonly kind: "step"
+  readonly step: PipelineStepDefinition
+}
+
+export interface PipelineSequenceGraph {
+  readonly kind: "sequence"
+  readonly nodes: readonly PipelineStepNode[]
+}
+
+export type PipelineGraph = PipelineSequenceGraph
+
+/**
+ * Inert pipeline definition registered with Sixb.
+ *
+ * V1 executes a single sequential graph. The graph shape leaves room for
+ * parallel nodes later without changing step definitions.
+ */
+export interface PipelineDefinition<TId extends string = string> {
+  readonly kind: "pipeline"
+  readonly id: TId
+  readonly triggers: readonly ScheduleReference[]
+  readonly graph: PipelineGraph
+}
+
+// ── Builder interfaces ─────────────────────────────────────
+
+export interface PipelineStepInputBuilder<TId extends string = string> {
+  inputs<TInputs extends Record<string, DatasetDefinition>>(
+    inputs: TInputs
+  ): PipelineStepOutputBuilder<TId, TInputs>
+}
+
+export interface PipelineStepOutputBuilder<
+  TId extends string = string,
+  TInputs extends Record<string, DatasetDefinition> = Record<string, DatasetDefinition>,
+> {
+  output(
+    dataset: DatasetDefinition,
+    options?: PipelineStepOutputOptions
+  ): PipelineStepExecutorBuilder<TId, TInputs>
+}
+
+export interface PipelineStepExecutorBuilder<
+  TId extends string = string,
+  TInputs extends Record<string, DatasetDefinition> = Record<string, DatasetDefinition>,
+> {
+  run(handler: PipelineStepRunHandler<TInputs>): PipelineStepDefinition<TId, TInputs>
+  sql(sql: SqlTransformBody<SqlDialect>): PipelineStepDefinition<TId, TInputs>
+}
+
+export interface PipelineStepOutputOptions {
+  readonly mode?: DatasetWriteMode
+}
+
+export interface PipelineBuilder<TId extends string = string> extends PipelineDefinition<TId> {
+  when(schedule: ScheduleDefinition): PipelineBuilder<TId>
+  then(step: PipelineStepDefinition): PipelineBuilder<TId>
+}
+
+// ── Type guards ────────────────────────────────────────────
+
+/** Runtime type guard for values discovered from `pipelines/` modules. */
+export function isPipelineDefinition(value: unknown): value is PipelineDefinition {
+  if (!isRecord(value)) return false
+  return (
+    value.kind === "pipeline" &&
+    typeof value.id === "string" &&
+    Array.isArray(value.triggers) &&
+    value.triggers.every(isScheduleReference) &&
+    isPipelineGraph(value.graph)
+  )
+}
+
+export function isPipelineStepDefinition(value: unknown): value is PipelineStepDefinition {
+  return (
+    isRecord(value) &&
+    value.kind === "pipeline.step" &&
+    typeof value.id === "string" &&
+    isInputsRecord(value.inputs) &&
+    isDatasetDefinition(value.output) &&
+    isDatasetWriteMode(value.mode) &&
+    isPipelineStepExecutor(value.executor)
+  )
+}
+
+function isPipelineGraph(value: unknown): value is PipelineGraph {
+  return (
+    isRecord(value) &&
+    value.kind === "sequence" &&
+    Array.isArray(value.nodes) &&
+    value.nodes.every(
+      (node) => isRecord(node) && node.kind === "step" && isPipelineStepDefinition(node.step)
+    )
+  )
+}
+
+function isPipelineStepExecutor(value: unknown): value is PipelineStepExecutor {
+  if (!isRecord(value)) return false
+
+  switch (value.kind) {
+    case "sql":
+      return isSqlDialect(value.dialect) && typeof value.sql === "function"
+    case "run":
+      return typeof value.handler === "function"
+    default:
+      return false
+  }
+}
+
+function isInputsRecord(value: unknown): value is Readonly<Record<string, DatasetDefinition>> {
+  return (
+    isRecord(value) &&
+    Object.entries(value).every(
+      ([key, dataset]) => key.trim().length > 0 && isDatasetDefinition(dataset)
+    )
+  )
+}
+
+function isDatasetWriteMode(value: unknown): value is DatasetWriteMode {
+  return value === "snapshot" || value === "append"
+}
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return typeof value === "object" && value !== null && !Array.isArray(value)
+}

package/src/predicates/builders.ts

@@ -0,0 +1,152 @@
+import type {
+  FieldPredicate,
+  LinkPredicate,
+  LinkPredicateBuilder,
+  Predicate,
+  PredicateValue,
+  PropertyPredicate,
+  PropertyPredicateOperator,
+} from "./types"
+import { isPredicateValue } from "./validation"
+
+export type RuntimePropertyPredicateBuilder<TResult = PropertyPredicate> = {
+  eq(value: PredicateValue): TResult
+  notEq(value: PredicateValue): TResult
+  gt(value: PredicateValue): TResult
+  gte(value: PredicateValue): TResult
+  lt(value: PredicateValue): TResult
+  lte(value: PredicateValue): TResult
+  isPresent(): TResult
+  isMissing(): TResult
+}
+
+export function createFieldPredicate(
+  field: string,
+  value: PredicateValue,
+  options: Pick<PropertyPredicateBuilderOptions<unknown>, "subject" | "createError"> = {}
+): FieldPredicate {
+  const subject = options.subject ?? "Predicate"
+  const createError = options.createError ?? defaultError
+  if (!field.trim()) {
+    throw createError(`${subject} predicate field must not be empty.`)
+  }
+  assertSerializablePredicateValue(value, subject, createError)
+
+  return { kind: "field", field, op: "eq", value }
+}
+
+export type PropertyPredicateBuilderOptions<TResult> = {
+  readonly subject?: string
+  readonly createError?: (message: string) => Error
+  readonly wrap?: (predicate: PropertyPredicate) => TResult
+}
+
+function defaultError(message: string): Error {
+  return new Error(message)
+}
+
+function assertSerializablePredicateValue(
+  value: PredicateValue,
+  subject: string,
+  createError: (message: string) => Error
+): void {
+  if (!isPredicateValue(value)) {
+    throw createError(`${subject} predicate values must be serializable scalar values.`)
+  }
+}
+
+export function createPropertyPredicate(
+  propertyId: string,
+  op: PropertyPredicateOperator,
+  value?: PredicateValue,
+  options: Pick<PropertyPredicateBuilderOptions<unknown>, "subject" | "createError"> = {}
+): PropertyPredicate {
+  const subject = options.subject ?? "Predicate"
+  const createError = options.createError ?? defaultError
+  if (value !== undefined) {
+    assertSerializablePredicateValue(value, subject, createError)
+  }
+
+  return {
+    kind: "property",
+    propertyId,
+    op,
+    ...(value !== undefined ? { value } : {}),
+  }
+}
+
+export function createPropertyPredicateBuilder<TResult = PropertyPredicate>(
+  propertyId: string,
+  options: PropertyPredicateBuilderOptions<TResult> = {}
+): RuntimePropertyPredicateBuilder<TResult> {
+  const wrap = options.wrap ?? ((predicate: PropertyPredicate) => predicate as TResult)
+  const createPredicate = (op: PropertyPredicateOperator, value?: PredicateValue) =>
+    wrap(createPropertyPredicate(propertyId, op, value, options))
+
+  return {
+    eq(value) {
+      return createPredicate("eq", value)
+    },
+    notEq(value) {
+      return createPredicate("notEq", value)
+    },
+    gt(value) {
+      return createPredicate("gt", value)
+    },
+    gte(value) {
+      return createPredicate("gte", value)
+    },
+    lt(value) {
+      return createPredicate("lt", value)
+    },
+    lte(value) {
+      return createPredicate("lte", value)
+    },
+    isPresent() {
+      return createPredicate("isPresent")
+    },
+    isMissing() {
+      return createPredicate("isMissing")
+    },
+  }
+}
+
+export function createLinkPredicateBuilder(linkId: string): LinkPredicateBuilder {
+  return {
+    exists(): LinkPredicate {
+      return {
+        kind: "link",
+        linkId,
+        op: "exists",
+      }
+    },
+    isMissing(): LinkPredicate {
+      return {
+        kind: "link",
+        linkId,
+        op: "isMissing",
+      }
+    },
+  }
+}
+
+export function allPredicates(predicates: readonly Predicate[]): Predicate {
+  return {
+    kind: "all",
+    predicates: [...predicates],
+  }
+}
+
+export function anyPredicates(predicates: readonly Predicate[]): Predicate {
+  return {
+    kind: "any",
+    predicates: [...predicates],
+  }
+}
+
+export function notPredicate(predicate: Predicate): Predicate {
+  return {
+    kind: "not",
+    predicate,
+  }
+}

package/src/predicates/evaluate.ts

@@ -0,0 +1,107 @@
+import { compareDecimalValues, isDecimalValue } from "../ontology/decimal"
+import type {
+  FieldPredicate,
+  LinkPredicate,
+  Predicate,
+  PredicateValue,
+  PropertyPredicate,
+} from "./types"
+
+export interface PredicateEvaluationSubject {
+  readonly properties?: Readonly<Record<string, unknown>>
+  readonly fields?: Readonly<Record<string, unknown>>
+  readonly links?: Readonly<Record<string, boolean>>
+}
+
+export function evaluatePredicate(
+  predicate: Predicate,
+  subject: PredicateEvaluationSubject
+): boolean {
+  switch (predicate.kind) {
+    case "all":
+      return predicate.predicates.every((child) => evaluatePredicate(child, subject))
+    case "any":
+      return predicate.predicates.some((child) => evaluatePredicate(child, subject))
+    case "not":
+      return !evaluatePredicate(predicate.predicate, subject)
+    case "property":
+      return evaluatePropertyPredicate(predicate, subject.properties ?? {})
+    case "field":
+      return evaluateFieldPredicate(predicate, subject.fields ?? {})
+    case "link":
+      return evaluateLinkPredicate(predicate, subject.links ?? {})
+  }
+}
+
+function evaluateFieldPredicate(
+  predicate: FieldPredicate,
+  fields: Readonly<Record<string, unknown>>
+): boolean {
+  return predicateValueEquals(fields[predicate.field], predicate.value)
+}
+
+function evaluatePropertyPredicate(
+  predicate: PropertyPredicate,
+  properties: Readonly<Record<string, unknown>>
+): boolean {
+  const value = properties[predicate.propertyId]
+  const isPresent = value !== undefined && value !== null
+
+  switch (predicate.op) {
+    case "eq":
+      return isPresent && predicateValueEquals(value, predicate.value)
+    case "notEq":
+      return !isPresent || !predicateValueEquals(value, predicate.value)
+    case "gt":
+      return compareOrderedValues(value, predicate.value, (comparison) => comparison > 0)
+    case "gte":
+      return compareOrderedValues(value, predicate.value, (comparison) => comparison >= 0)
+    case "lt":
+      return compareOrderedValues(value, predicate.value, (comparison) => comparison < 0)
+    case "lte":
+      return compareOrderedValues(value, predicate.value, (comparison) => comparison <= 0)
+    case "isPresent":
+      return isPresent
+    case "isMissing":
+      return !isPresent
+  }
+}
+
+function evaluateLinkPredicate(
+  predicate: LinkPredicate,
+  links: Readonly<Record<string, boolean>>
+): boolean {
+  const exists = links[predicate.linkId] === true
+  switch (predicate.op) {
+    case "exists":
+      return exists
+    case "isMissing":
+      return !exists
+  }
+}
+
+function predicateValueEquals(left: unknown, right: PredicateValue | undefined): boolean {
+  return isPredicateComparableValue(left) && right !== undefined && Object.is(left, right)
+}
+
+function isPredicateComparableValue(value: unknown): value is PredicateValue {
+  return (
+    value === null || typeof value === "string" || typeof value === "boolean" || isNumber(value)
+  )
+}
+
+function isNumber(value: unknown): value is number {
+  return typeof value === "number" && Number.isFinite(value)
+}
+
+function compareOrderedValues(
+  left: unknown,
+  right: unknown,
+  matches: (comparison: number) => boolean
+): boolean {
+  if (isNumber(left) && isNumber(right)) return matches(left - right)
+  if (isDecimalValue(left) && isDecimalValue(right)) {
+    return matches(compareDecimalValues(left, right))
+  }
+  return false
+}

package/src/predicates/index.ts

@@ -0,0 +1,28 @@
+export {
+  allPredicates,
+  anyPredicates,
+  createFieldPredicate,
+  createLinkPredicateBuilder,
+  createPropertyPredicate,
+  createPropertyPredicateBuilder,
+  notPredicate,
+  type RuntimePropertyPredicateBuilder,
+} from "./builders"
+export { evaluatePredicate, type PredicateEvaluationSubject } from "./evaluate"
+export type {
+  AllPredicate,
+  AnyPredicate,
+  FieldPredicate,
+  LinkPredicate,
+  LinkPredicateBuilder,
+  LinkPredicateOperator,
+  NotPredicate,
+  OrderedPredicateValueFor,
+  Predicate,
+  PredicateValue,
+  PredicateValueFor,
+  PropertyPredicate,
+  PropertyPredicateBuilder,
+  PropertyPredicateOperator,
+} from "./types"
+export { assertPredicateShape, isPredicateValue } from "./validation"