@sixb/core 0.0.1

package/src/ontology/validation/query.ts

@@ -0,0 +1,300 @@
+import type { ObjectType, Property, Schema, ValueType } from ".."
+import { OntologyValidationError } from "../errors"
+
+type QueryFeature = "filterable" | "sortable" | "text" | "exact" | "facet" | "vector"
+
+const queryFeatures: readonly QueryFeature[] = [
+  "filterable",
+  "sortable",
+  "text",
+  "exact",
+  "facet",
+  "vector",
+]
+
+export function validateQueryMetadata(
+  objectTypesById: ReadonlyMap<string, ObjectType>,
+  valueTypesById: ReadonlyMap<string, ValueType>
+): void {
+  for (const [typeId, objectType] of objectTypesById) {
+    validatePropertiesQueryMetadata(typeId, objectType.properties, valueTypesById)
+
+    for (const link of objectType.links) {
+      if (link.properties) {
+        validatePropertiesQueryMetadata(`${typeId}.${link.id}`, link.properties, valueTypesById)
+      }
+    }
+
+    validateObjectSearchMetadata(typeId, objectType, valueTypesById)
+  }
+}
+
+function validatePropertiesQueryMetadata(
+  ownerPath: string,
+  properties: readonly Property[],
+  valueTypesById: ReadonlyMap<string, ValueType>
+): void {
+  for (const property of properties) {
+    validatePropertyQueryMetadata(ownerPath, property, valueTypesById)
+  }
+}
+
+function validatePropertyQueryMetadata(
+  ownerPath: string,
+  property: Property,
+  valueTypesById: ReadonlyMap<string, ValueType>
+): void {
+  const query = property.query
+  if (!query) return
+
+  const enabledFeatures = queryFeatures.filter((feature) => query[feature] === true)
+  if ((enabledFeatures.length > 0 || query.weight !== undefined) && query.searchable !== true) {
+    throw new OntologyValidationError(
+      `[Sixb] Query metadata for property '${property.id}' on '${ownerPath}' must set query.searchable: true before enabling query features`
+    )
+  }
+
+  if (query.weight !== undefined) {
+    if (!Number.isFinite(query.weight) || query.weight <= 0) {
+      throw new OntologyValidationError(
+        `[Sixb] Query metadata for property '${property.id}' on '${ownerPath}' has invalid text weight. Expected a positive finite number.`
+      )
+    }
+    if (query.text !== true) {
+      throw new OntologyValidationError(
+        `[Sixb] Query metadata for property '${property.id}' on '${ownerPath}' can only set weight when query.text is true`
+      )
+    }
+  }
+
+  if (enabledFeatures.length === 0) return
+
+  const schema = resolveQueryableSchema(property, ownerPath, valueTypesById)
+
+  if (query.text && !isTextSearchableSchema(schema)) {
+    throw new OntologyValidationError(
+      `[Sixb] Query metadata for property '${property.id}' on '${ownerPath}' enables text search, but its schema is not string-like`
+    )
+  }
+
+  if (query.exact && !isExactSearchableSchema(schema)) {
+    throw new OntologyValidationError(
+      `[Sixb] Query metadata for property '${property.id}' on '${ownerPath}' enables exact search, but its schema cannot be exact-matched`
+    )
+  }
+
+  if (query.filterable && !isFilterableSchema(schema)) {
+    throw new OntologyValidationError(
+      `[Sixb] Query metadata for property '${property.id}' on '${ownerPath}' enables filtering, but its schema cannot be filtered`
+    )
+  }
+
+  if (query.sortable && !isSortableSchema(schema)) {
+    throw new OntologyValidationError(
+      `[Sixb] Query metadata for property '${property.id}' on '${ownerPath}' enables sorting, but its schema is not orderable`
+    )
+  }
+
+  if (query.facet && !isFacetSchema(schema)) {
+    throw new OntologyValidationError(
+      `[Sixb] Query metadata for property '${property.id}' on '${ownerPath}' enables faceting, but its schema cannot be faceted`
+    )
+  }
+
+  if (query.vector && !isVectorSchema(schema, valueTypesById)) {
+    throw new OntologyValidationError(
+      `[Sixb] Query metadata for property '${property.id}' on '${ownerPath}' enables vector search, but its schema is not a numeric array`
+    )
+  }
+}
+
+function validateObjectSearchMetadata(
+  typeId: string,
+  objectType: ObjectType,
+  valueTypesById: ReadonlyMap<string, ValueType>
+): void {
+  const search = objectType.search
+  if (!search) return
+
+  const primaryPropertyId = objectType.properties.find((property) => property.primary)?.id
+
+  if (search.title) {
+    const title = requireObjectProperty(typeId, objectType, search.title, "search.title")
+    assertStaticSearchProfileProperty(typeId, title, "search.title")
+    const schema = resolveQueryableSchema(title, typeId, valueTypesById)
+    if (!isTextSearchableSchema(schema)) {
+      throw new OntologyValidationError(
+        `[Sixb] Object type '${typeId}' search.title references '${title.id}', but title fields must be string-like`
+      )
+    }
+  }
+
+  for (const propertyId of search.defaultText ?? []) {
+    const property = requireObjectProperty(typeId, objectType, propertyId, "search.defaultText")
+    assertStaticSearchProfileProperty(typeId, property, "search.defaultText")
+    assertPropertyQueryFlag(typeId, property, "text", "search.defaultText")
+  }
+
+  for (const propertyId of search.exact ?? []) {
+    const property = requireObjectProperty(typeId, objectType, propertyId, "search.exact")
+    assertStaticSearchProfileProperty(typeId, property, "search.exact")
+    if (property.id !== primaryPropertyId) {
+      assertPropertyQueryFlag(typeId, property, "exact", "search.exact")
+    }
+  }
+
+  if (search.vector) {
+    const vectorProperty = requireObjectProperty(
+      typeId,
+      objectType,
+      search.vector.property,
+      "search.vector.property"
+    )
+    assertStaticSearchProfileProperty(typeId, vectorProperty, "search.vector.property")
+    assertPropertyQueryFlag(typeId, vectorProperty, "vector", "search.vector.property")
+
+    if (search.vector.source.length === 0) {
+      throw new OntologyValidationError(
+        `[Sixb] Object type '${typeId}' search.vector.source must include at least one source property`
+      )
+    }
+
+    for (const sourceId of search.vector.source) {
+      const sourceProperty = requireObjectProperty(
+        typeId,
+        objectType,
+        sourceId,
+        "search.vector.source"
+      )
+      assertStaticSearchProfileProperty(typeId, sourceProperty, "search.vector.source")
+      assertPropertyQueryFlag(typeId, sourceProperty, "text", "search.vector.source")
+    }
+  }
+}
+
+function requireObjectProperty(
+  typeId: string,
+  objectType: ObjectType,
+  propertyId: string,
+  metadataPath: string
+): Property {
+  const property = objectType.properties.find((candidate) => candidate.id === propertyId)
+  if (!property) {
+    throw new OntologyValidationError(
+      `[Sixb] Object type '${typeId}' ${metadataPath} references unknown property '${propertyId}'`
+    )
+  }
+  return property
+}
+
+function assertStaticSearchProfileProperty(
+  typeId: string,
+  property: Property,
+  metadataPath: string
+): void {
+  if (property.mode === "telemetry") {
+    throw new OntologyValidationError(
+      `[Sixb] Object type '${typeId}' ${metadataPath} references telemetry property '${property.id}'. Search profiles can only reference static properties because telemetry latest values are not object-query indexed.`
+    )
+  }
+}
+
+function assertPropertyQueryFlag(
+  typeId: string,
+  property: Property,
+  flag: QueryFeature,
+  metadataPath: string
+): void {
+  if (property.query?.searchable === true && property.query[flag] === true) {
+    return
+  }
+
+  throw new OntologyValidationError(
+    `[Sixb] Object type '${typeId}' ${metadataPath} references property '${property.id}', but that property must set query.searchable: true and query.${flag}: true`
+  )
+}
+
+function resolveQueryableSchema(
+  property: Property,
+  ownerPath: string,
+  valueTypesById: ReadonlyMap<string, ValueType>
+): Schema {
+  return resolveSchema(property.schema, valueTypesById, `${ownerPath}.${property.id}`)
+}
+
+function resolveSchema(
+  schema: Schema,
+  valueTypesById: ReadonlyMap<string, ValueType>,
+  path: string,
+  seenValueTypeIds = new Set<string>()
+): Schema {
+  if (typeof schema === "string") {
+    return schema
+  }
+
+  if (schema.type !== "valueTypeRef") {
+    return schema
+  }
+
+  if (seenValueTypeIds.has(schema.valueTypeId)) {
+    throw new OntologyValidationError(
+      `[Sixb] Circular valueTypeRef '${schema.valueTypeId}' in query metadata at ${path}`
+    )
+  }
+
+  const resolved = schema._resolved ?? valueTypesById.get(schema.valueTypeId)?.schema
+  if (!resolved) {
+    throw new OntologyValidationError(
+      `[Sixb] Query metadata for ${path} references unknown valueTypeRef '${schema.valueTypeId}'`
+    )
+  }
+
+  seenValueTypeIds.add(schema.valueTypeId)
+  return resolveSchema(resolved, valueTypesById, path, seenValueTypeIds)
+}
+
+function isTextSearchableSchema(schema: Schema): boolean {
+  if (schema === "string") return true
+  return typeof schema !== "string" && schema.type === "enum" && schema.valueType === "string"
+}
+
+function isExactSearchableSchema(schema: Schema): boolean {
+  if (typeof schema === "string") {
+    return schema !== "fileRef"
+  }
+  return schema.type === "enum"
+}
+
+function isFilterableSchema(schema: Schema): boolean {
+  return (
+    isExactSearchableSchema(schema) ||
+    (typeof schema !== "string" && (schema.type === "array" || schema.type === "map"))
+  )
+}
+
+function isSortableSchema(schema: Schema): boolean {
+  if (typeof schema === "string") {
+    return (
+      schema === "string" ||
+      schema === "uuid" ||
+      schema === "integer" ||
+      schema === "double" ||
+      schema === "decimal" ||
+      schema === "date" ||
+      schema === "timestamp"
+    )
+  }
+  return schema.type === "enum"
+}
+
+function isFacetSchema(schema: Schema): boolean {
+  return isExactSearchableSchema(schema)
+}
+
+function isVectorSchema(schema: Schema, valueTypesById: ReadonlyMap<string, ValueType>): boolean {
+  if (typeof schema === "string") return false
+  if (schema.type !== "array") return false
+
+  const itemSchema = resolveSchema(schema.items, valueTypesById, "vector item")
+  return itemSchema === "integer" || itemSchema === "double" || itemSchema === "decimal"
+}

package/src/ontology/validation/schema.ts

@@ -0,0 +1,174 @@
+import { isFileRef } from "../../blob-storage/validation"
+import type { ObjectFieldSchema, Schema, ValueType, ValueTypeRefSchema } from ".."
+import { isDecimalString } from "../decimal"
+import { OntologyValidationError } from "../errors"
+
+/** Recursive schema validator used by both object and link property validation. */
+export function validateSchemaValue(
+  schema: Schema,
+  value: unknown,
+  path: string,
+  valueTypesById: ReadonlyMap<string, ValueType>
+): void {
+  if (typeof schema === "string") {
+    switch (schema) {
+      case "string":
+      case "uuid": {
+        if (typeof value !== "string") {
+          throw new OntologyValidationError(`[Sixb] Property ${path} must be a string`)
+        }
+        return
+      }
+      case "boolean": {
+        if (typeof value !== "boolean") {
+          throw new OntologyValidationError(`[Sixb] Property ${path} must be a boolean`)
+        }
+        return
+      }
+      case "integer": {
+        if (typeof value !== "number" || !Number.isInteger(value)) {
+          throw new OntologyValidationError(`[Sixb] Property ${path} must be an integer`)
+        }
+        return
+      }
+      case "double": {
+        if (typeof value !== "number" || !Number.isFinite(value)) {
+          throw new OntologyValidationError(`[Sixb] Property ${path} must be numeric`)
+        }
+        return
+      }
+      case "decimal": {
+        if (!isDecimalString(value)) {
+          throw new OntologyValidationError(
+            `[Sixb] Property ${path} must be an exact decimal string`
+          )
+        }
+        return
+      }
+      case "date":
+      case "timestamp": {
+        if (!(value instanceof Date) && typeof value !== "string") {
+          throw new OntologyValidationError(`[Sixb] Property ${path} must be a Date or ISO string`)
+        }
+        return
+      }
+      case "fileRef": {
+        if (!isFileRef(value)) {
+          throw new OntologyValidationError(`[Sixb] Property ${path} must be a fileRef`)
+        }
+        return
+      }
+    }
+  }
+
+  if (schema.type === "enum") {
+    if (!schema.values.includes(value as never)) {
+      throw new OntologyValidationError(
+        `[Sixb] Property ${path} must be one of: ${schema.values.join(", ")}`
+      )
+    }
+    return
+  }
+
+  if (schema.type === "array") {
+    if (!Array.isArray(value)) {
+      throw new OntologyValidationError(`[Sixb] Property ${path} must be an array`)
+    }
+    for (let index = 0; index < value.length; index += 1) {
+      validateSchemaValue(schema.items, value[index], `${path}[${index}]`, valueTypesById)
+    }
+    return
+  }
+
+  if (schema.type === "map") {
+    if (!isRecord(value)) {
+      throw new OntologyValidationError(`[Sixb] Property ${path} must be an object map`)
+    }
+    for (const [key, entry] of Object.entries(value)) {
+      validateSchemaValue(schema.valueSchema, entry, `${path}.${key}`, valueTypesById)
+    }
+    return
+  }
+
+  if (schema.type === "object") {
+    if (!isRecord(value)) {
+      throw new OntologyValidationError(`[Sixb] Property ${path} must be an object`)
+    }
+
+    const fields = schema.properties
+    const fieldIds = new Set(Object.keys(fields))
+    for (const fieldId of Object.keys(value)) {
+      if (!fieldIds.has(fieldId)) {
+        throw new OntologyValidationError(`[Sixb] Unknown field '${path}.${fieldId}'`)
+      }
+    }
+
+    for (const [fieldId, field] of Object.entries(fields)) {
+      const fieldValue = value[fieldId]
+      if (fieldValue === undefined) {
+        if (field.required) {
+          throw new OntologyValidationError(`[Sixb] Missing required field '${path}.${fieldId}'`)
+        }
+        continue
+      }
+
+      validateFieldValue(field, fieldValue, `${path}.${fieldId}`, valueTypesById)
+    }
+    return
+  }
+
+  if (schema.type === "valueTypeRef") {
+    validateSchemaValue(
+      resolveValueTypeSchema(schema, valueTypesById, path),
+      value,
+      path,
+      valueTypesById
+    )
+  }
+}
+
+function validateFieldValue(
+  field: ObjectFieldSchema,
+  value: unknown,
+  path: string,
+  valueTypesById: ReadonlyMap<string, ValueType>
+): void {
+  if (value === null) {
+    if (field.nullable) {
+      return
+    }
+    throw new OntologyValidationError(`[Sixb] Field ${path} cannot be null`)
+  }
+
+  validateSchemaValue(field.schema, value, path, valueTypesById)
+}
+
+export function resolveValueTypeRef(schema: Schema): string | undefined {
+  if (typeof schema === "string") {
+    return undefined
+  }
+
+  if (schema.type === "valueTypeRef") {
+    return schema.valueTypeId
+  }
+
+  return undefined
+}
+
+export function resolveValueTypeSchema(
+  schema: ValueTypeRefSchema,
+  valueTypesById: ReadonlyMap<string, ValueType>,
+  path: string
+): Schema {
+  const resolved = schema._resolved ?? valueTypesById.get(schema.valueTypeId)?.schema
+  if (!resolved) {
+    throw new OntologyValidationError(
+      `[Sixb] Unknown valueTypeRef '${schema.valueTypeId}' at ${path}`
+    )
+  }
+  return resolved
+}
+
+export function isRecord(value: unknown): value is Record<string, unknown> {
+  return typeof value === "object" && value !== null && !Array.isArray(value)
+}

package/src/ontology/validation/telemetry.ts

@@ -0,0 +1,53 @@
+import type { Property, ValueType } from ".."
+import { OntologyValidationError } from "../errors"
+import type { QuantitativeTypeId } from "../units"
+import { isValidUnit } from "../units"
+import { resolveValueTypeRef } from "./schema"
+
+export function assertTelemetryProperty(property: Property): void {
+  if (property.mode !== "telemetry") {
+    throw new OntologyValidationError(`[Sixb] Property ${property.id} is not telemetry-enabled`)
+  }
+}
+
+export function resolveSemanticType(
+  property: Property,
+  valueTypesById: ReadonlyMap<string, ValueType>
+): QuantitativeTypeId | undefined {
+  if (property.semanticType) {
+    return property.semanticType
+  }
+
+  const valueTypeId = resolveValueTypeRef(property.schema)
+  if (!valueTypeId) {
+    return undefined
+  }
+
+  return valueTypesById.get(valueTypeId)?.semanticType
+}
+
+/** Validates that unit is present/absent based on semantic type, and that it's a valid unit. */
+export function validateTelemetryUnit(
+  property: Property,
+  propertyPath: string,
+  unit: string | undefined,
+  valueTypesById: ReadonlyMap<string, ValueType>
+): void {
+  const semanticType = resolveSemanticType(property, valueTypesById)
+  if (semanticType) {
+    if (!unit) {
+      throw new OntologyValidationError(
+        `[Sixb] Missing unit for telemetry property ${propertyPath}`
+      )
+    }
+    if (!isValidUnit(semanticType, unit)) {
+      throw new OntologyValidationError(
+        `[Sixb] Invalid unit '${unit}' for ${propertyPath} (${semanticType})`
+      )
+    }
+  } else if (unit) {
+    throw new OntologyValidationError(
+      `[Sixb] Property ${propertyPath} does not define semanticType and cannot accept a unit`
+    )
+  }
+}

package/src/pipelines/builders.ts

@@ -0,0 +1,155 @@
+import type { DatasetDefinition } from "../datasets"
+import type { DatasetWriteMode } from "../lake-storage"
+import { SQL_DIALECT } from "../lake-storage/sql-transforms"
+import type { ScheduleDefinition, ScheduleReference } from "../schedules"
+import { isScheduleDefinition } from "../schedules"
+import { PipelineError } from "./errors"
+import type {
+  PipelineBuilder,
+  PipelineDefinition,
+  PipelineGraph,
+  PipelineStepDefinition,
+  PipelineStepExecutorBuilder,
+  PipelineStepInputBuilder,
+  PipelineStepOutputBuilder,
+  PipelineStepOutputOptions,
+  PipelineStepRunHandler,
+} from "./types"
+
+function assertNonEmpty(value: string, field: string): void {
+  if (!value.trim()) {
+    throw new PipelineError(`Pipeline ${field} must not be empty.`)
+  }
+}
+
+function assertDataset(dataset: DatasetDefinition, field: string): void {
+  assertNonEmpty(dataset.id, field)
+}
+
+function assertDatasetWriteMode(mode: DatasetWriteMode): void {
+  if (mode !== "snapshot" && mode !== "append") {
+    throw new PipelineError(`Pipeline step output mode '${mode}' is not supported.`)
+  }
+}
+
+function assertInputs(inputs: Readonly<Record<string, DatasetDefinition>>): void {
+  const entries = Object.entries(inputs)
+  if (entries.length === 0) {
+    throw new PipelineError("Pipeline step input dataset must contain at least one entry.")
+  }
+
+  for (const [name, dataset] of entries) {
+    assertNonEmpty(name, "step input name")
+    assertDataset(dataset, "step input dataset")
+  }
+}
+
+function scheduleReference(schedule: ScheduleDefinition): ScheduleReference {
+  if (!isScheduleDefinition(schedule)) {
+    throw new PipelineError("Pipeline .when(...) only accepts schedules.")
+  }
+  return { type: "schedule", scheduleId: schedule.id }
+}
+
+function createPipelineBuilder<TId extends string>(
+  definition: PipelineDefinition<TId>
+): PipelineBuilder<TId> {
+  return {
+    ...definition,
+    when(schedule: ScheduleDefinition): PipelineBuilder<TId> {
+      return createPipelineBuilder({
+        ...definition,
+        triggers: [...definition.triggers, scheduleReference(schedule)],
+      })
+    },
+    // biome-ignore lint/suspicious/noThenProperty: Pipeline composition intentionally uses `.then(step)`.
+    then(step: PipelineStepDefinition): PipelineBuilder<TId> {
+      const graph: PipelineGraph = {
+        kind: "sequence",
+        nodes: [...definition.graph.nodes, { kind: "step", step }],
+      }
+
+      return createPipelineBuilder({
+        ...definition,
+        graph,
+      })
+    },
+  }
+}
+
+/**
+ * Define one dataset transform step.
+ *
+ * The returned definition is inert until referenced by a pipeline definition.
+ */
+export function definePipelineStep<TId extends string>(id: TId): PipelineStepInputBuilder<TId> {
+  assertNonEmpty(id, "step id")
+
+  return {
+    inputs<TInputs extends Record<string, DatasetDefinition>>(
+      inputs: TInputs
+    ): PipelineStepOutputBuilder<TId, TInputs> {
+      assertInputs(inputs)
+
+      return {
+        output(
+          dataset: DatasetDefinition,
+          options: PipelineStepOutputOptions = {}
+        ): PipelineStepExecutorBuilder<TId, TInputs> {
+          assertDataset(dataset, "step output dataset")
+          const mode = options.mode ?? "snapshot"
+          assertDatasetWriteMode(mode)
+
+          return {
+            run(handler): PipelineStepDefinition<TId, TInputs> {
+              return {
+                kind: "pipeline.step",
+                id,
+                inputs,
+                output: dataset,
+                mode,
+                executor: {
+                  kind: "run",
+                  handler: handler as unknown as PipelineStepRunHandler,
+                },
+              }
+            },
+            sql(sql): PipelineStepDefinition<TId, TInputs> {
+              return {
+                kind: "pipeline.step",
+                id,
+                inputs,
+                output: dataset,
+                mode,
+                executor: {
+                  kind: "sql",
+                  dialect: SQL_DIALECT.duckdb,
+                  sql,
+                },
+              }
+            },
+          }
+        },
+      }
+    },
+  }
+}
+
+/**
+ * Define a sequential pipeline that composes dataset transform steps.
+ *
+ * The returned definition is inert and safe to export from `pipelines/` modules.
+ */
+export function definePipeline<TId extends string>(id: TId): PipelineBuilder<TId> {
+  assertNonEmpty(id, "id")
+
+  return createPipelineBuilder({
+    kind: "pipeline",
+    id,
+    triggers: [],
+    graph: {
+      kind: "sequence",
+      nodes: [],
+    },
+  })
+}

package/src/pipelines/errors.ts

@@ -0,0 +1,3 @@
+export class PipelineError extends Error {
+  readonly name = "PipelineError"
+}