@sixb/core 0.1.20 → 0.1.21

package/src/security/marking-propagation.ts

@@ -0,0 +1,255 @@
+import type { DatasetDefinition } from "../datasets/types"
+import { lineageColumn } from "../lake-storage/lineage"
+import type { DatasetColumnLineage, DatasetColumnRef } from "../lake-storage/types"
+import { runStepLineage, stepReadColumns } from "../pipelines/lineage"
+import type { PipelineDefinition, PipelineStepDefinition } from "../pipelines/types"
+
+/** A step of a registered pipeline. A step reused by two pipelines is two registered steps. */
+export interface RegisteredStep {
+  readonly pipelineId: string
+  readonly step: PipelineStepDefinition
+}
+
+/** The column that declares a marking, and the steps it went through to reach a column. */
+export interface MarkingOrigin {
+  readonly column: DatasetColumnRef
+  /** `pipeline/step`, upstream first. Empty when the column declares the marking itself. */
+  readonly steps: readonly string[]
+}
+
+/** Each marking a column carries, with one path it arrived by. */
+export type ColumnMarkingOrigins = ReadonlyMap<string, MarkingOrigin>
+
+/** Markings of every column of every registered dataset, derived from the definitions. */
+export interface DatasetMarkings {
+  /** Empty for a column without markings or that is not registered. */
+  of(column: DatasetColumnRef): ColumnMarkingOrigins
+}
+
+export function registeredSteps(
+  pipelines: readonly PipelineDefinition[]
+): readonly RegisteredStep[] {
+  return pipelines.flatMap((pipeline) =>
+    pipeline.graph.nodes.map(({ step }) => ({ pipelineId: pipeline.id, step }))
+  )
+}
+
+/** Lineage of each `run` step, built once per step object; undefined for other steps. */
+export function runStepLineages(
+  datasetsById: ReadonlyMap<string, DatasetDefinition>
+): (entry: RegisteredStep) => DatasetColumnLineage | undefined {
+  const lineages = new Map<PipelineStepDefinition, DatasetColumnLineage>()
+
+  return ({ step }) => {
+    if (step.executor.kind !== "run") return undefined
+
+    let lineage = lineages.get(step)
+    if (!lineage) {
+      lineage = runStepLineage(step, datasetsById)
+      lineages.set(step, lineage)
+    }
+
+    return lineage
+  }
+}
+
+export function stepLabel(entry: RegisteredStep): string {
+  return `${entry.pipelineId}/${entry.step.id}`
+}
+
+type MutableOrigins = Map<string, MarkingOrigin>
+
+const NO_MARKINGS: ColumnMarkingOrigins = new Map()
+
+interface KnownLineage {
+  readonly lineage: DatasetColumnLineage
+  /** What a column the lineage does not list depends on. */
+  readonly reads: readonly DatasetColumnRef[]
+}
+
+/**
+ * Markings each column carries according to the registered definitions: the ones it declares, and
+ * the ones each step derives from the columns it depends on, once that step's lineage is known.
+ *
+ * What decided the rows of a step's inputs decides the rows of its output, whether or not the step
+ * reads a column of that input: `SELECT count(*) FROM rich` reveals which rows `rich` holds. So
+ * each dataset also keeps the markings of its row dependencies, and every output column of a step
+ * carries those of every input it reads, as the lineage each version records does.
+ *
+ * A step may read its own output and a dataset may have several producers, so a step whose output
+ * gains a marking queues the steps that read that output, until nothing changes. Lineage can be
+ * added in rounds: each round only revisits what the new lineage reaches.
+ */
+export class MarkingPropagation implements DatasetMarkings {
+  private readonly datasetsById: ReadonlyMap<string, DatasetDefinition>
+  /** Marked columns only, by dataset, built when a dataset is first looked at. */
+  private readonly markings = new Map<string, Map<string, MutableOrigins>>()
+  /** Markings of what decides each dataset's rows, for datasets a step writes. */
+  private readonly rowMarkings = new Map<string, MutableOrigins>()
+  private readonly ordered: readonly RegisteredStep[]
+  private readonly readers = new Map<string, RegisteredStep[]>()
+  private readonly known = new Map<RegisteredStep, KnownLineage>()
+  private readonly queue = new Set<RegisteredStep>()
+
+  constructor(
+    datasetsById: ReadonlyMap<string, DatasetDefinition>,
+    steps: readonly RegisteredStep[]
+  ) {
+    this.datasetsById = datasetsById
+    this.ordered = inDependencyOrder(steps)
+
+    for (const entry of steps) {
+      for (const input of new Set(Object.values(entry.step.inputs).map((dataset) => dataset.id))) {
+        appendTo(this.readers, input, entry)
+      }
+    }
+  }
+
+  of(ref: DatasetColumnRef): ColumnMarkingOrigins {
+    return this.markedColumns(ref.datasetId).get(ref.column) ?? NO_MARKINGS
+  }
+
+  /**
+   * Derive markings through every step `lineageOf` knows the lineage of. A step it returns null
+   * for derives nothing until a later call knows it.
+   */
+  derive(lineageOf: (entry: RegisteredStep) => DatasetColumnLineage | null): this {
+    for (const entry of this.ordered) {
+      if (this.known.has(entry)) continue
+
+      const lineage = lineageOf(entry)
+      if (!lineage) continue
+
+      this.known.set(entry, { lineage, reads: stepReadColumns(entry.step) })
+      this.queue.add(entry)
+    }
+
+    for (const entry of this.queue) {
+      this.queue.delete(entry)
+      const known = this.known.get(entry)
+      if (!known || !this.apply(entry, known)) continue
+
+      // Entries added while iterating a Set are visited, so the queue drains in this loop.
+      for (const reader of this.readers.get(entry.step.output.id) ?? []) {
+        if (this.known.has(reader)) this.queue.add(reader)
+      }
+    }
+
+    return this
+  }
+
+  /** Whether the output's rows or any output column gained a marking. */
+  private apply(entry: RegisteredStep, { lineage, reads }: KnownLineage): boolean {
+    const label = stepLabel(entry)
+    const rows = this.union(lineage.rows)
+
+    for (const input of new Set(Object.values(entry.step.inputs).map((dataset) => dataset.id))) {
+      for (const [markingId, origin] of this.rowMarkings.get(input) ?? []) {
+        if (!rows.has(markingId)) rows.set(markingId, origin)
+      }
+    }
+
+    let everyRead: ReadonlyMap<string, MarkingOrigin> | undefined
+    let changed = false
+
+    const outputId = entry.step.output.id
+    let outputRows = this.rowMarkings.get(outputId)
+
+    for (const [markingId, origin] of rows) {
+      if (outputRows?.has(markingId)) continue
+      if (!outputRows) {
+        outputRows = new Map()
+        this.rowMarkings.set(outputId, outputRows)
+      }
+
+      outputRows.set(markingId, { column: origin.column, steps: [...origin.steps, label] })
+      changed = true
+    }
+
+    const marked = this.markedColumns(outputId)
+    for (const { name: column } of this.datasetsById.get(outputId)?.schema.columns ?? []) {
+      const values = lineageColumn(lineage, column)
+      const valueSource = values ? this.union(values) : (everyRead ?? this.union(reads))
+      if (!values) everyRead = valueSource
+
+      for (const source of [rows, valueSource]) {
+        for (const [markingId, origin] of source) {
+          let origins = marked.get(column)
+          if (origins?.has(markingId)) continue
+          if (!origins) {
+            origins = new Map()
+            marked.set(column, origins)
+          }
+
+          origins.set(markingId, { column: origin.column, steps: [...origin.steps, label] })
+          changed = true
+        }
+      }
+    }
+
+    return changed
+  }
+
+  private markedColumns(datasetId: string): Map<string, MutableOrigins> {
+    let columns = this.markings.get(datasetId)
+    if (!columns) {
+      columns = new Map()
+      for (const column of this.datasetsById.get(datasetId)?.schema.columns ?? []) {
+        if (!column.markings?.length) continue
+
+        const origin = { column: { datasetId, column: column.name }, steps: [] }
+        columns.set(column.name, new Map(column.markings.map((id) => [id, origin])))
+      }
+
+      this.markings.set(datasetId, columns)
+    }
+
+    return columns
+  }
+
+  /** Every marking of these columns, each with the first origin found. */
+  private union(columns: readonly DatasetColumnRef[]): MutableOrigins {
+    const union: MutableOrigins = new Map()
+    for (const column of columns) {
+      for (const [markingId, origin] of this.of(column)) {
+        if (!union.has(markingId)) union.set(markingId, origin)
+      }
+    }
+    return union
+  }
+}
+
+/** Producers before consumers, so most markings arrive by their shortest path. */
+function inDependencyOrder(steps: readonly RegisteredStep[]): readonly RegisteredStep[] {
+  const producers = producersByDataset(steps)
+  const ordered: RegisteredStep[] = []
+  const visited = new Set<RegisteredStep>()
+
+  const visit = (entry: RegisteredStep): void => {
+    if (visited.has(entry)) return
+
+    visited.add(entry)
+    for (const input of Object.values(entry.step.inputs)) {
+      for (const producer of producers.get(input.id) ?? []) visit(producer)
+    }
+
+    ordered.push(entry)
+  }
+
+  for (const entry of steps) visit(entry)
+  return ordered
+}
+
+function producersByDataset(
+  steps: readonly RegisteredStep[]
+): ReadonlyMap<string, readonly RegisteredStep[]> {
+  const producers = new Map<string, RegisteredStep[]>()
+  for (const entry of steps) appendTo(producers, entry.step.output.id, entry)
+  return producers
+}
+
+function appendTo<TValue>(map: Map<string, TValue[]>, key: string, value: TValue): void {
+  const values = map.get(key)
+  if (values) values.push(value)
+  else map.set(key, [value])
+}

package/src/security/marking-sets.ts

@@ -0,0 +1,42 @@
+/**
+ * Shared, immutable sets of marking ids.
+ *
+ * Every column of every resolved version holds one of a handful of distinct sets, so each distinct
+ * set is created once and compared by identity.
+ */
+export class MarkingSets {
+  readonly empty: ReadonlySet<string> = new Set()
+  private readonly sets = new Map<string, ReadonlySet<string>>([["", this.empty]])
+
+  of(markingIds: Iterable<string> | undefined): ReadonlySet<string> {
+    // Most columns carry no marking.
+    if (markingIds === undefined || (Array.isArray(markingIds) && markingIds.length === 0)) {
+      return this.empty
+    }
+
+    const sorted = [...new Set(markingIds)].sort()
+    const key = sorted.join("\u0000")
+
+    let set = this.sets.get(key)
+    if (!set) {
+      set = new Set(sorted)
+      this.sets.set(key, set)
+    }
+
+    return set
+  }
+
+  /** The union of sets this instance created. */
+  union(sets: Iterable<ReadonlySet<string> | undefined>): ReadonlySet<string> {
+    let single: ReadonlySet<string> | undefined
+    const ids: string[] = []
+
+    for (const set of sets) {
+      if (!set || set.size === 0) continue
+      single = single === undefined && ids.length === 0 ? set : undefined
+      ids.push(...set)
+    }
+
+    return single ?? this.of(ids)
+  }
+}

package/src/security/markings.ts

@@ -3,15 +3,19 @@ import type { OntologyDefinitionCatalog } from "../ontology/registry"
 import type { ObjectType, Property } from "../ontology/types"
 import type { PipelineDefinition } from "../pipelines/types"
 import type { ProjectionDefinitionCatalog } from "../projections/registry"
-import type { ObjectProjectionDefinition } from "../projections/types"
 import { SecurityValidationError } from "./errors"
+import { MarkingPropagation, registeredSteps, runStepLineages } from "./marking-propagation"
+import { validateProjectionMarkings } from "./projection-markings"
 
 /**
  * Check every marking once at startup.
  *
- * A marking is enforced at read time from the property or column definition alone, so each rule
- * here closes a path that would otherwise return the value unredacted or make enforcement
- * ambiguous.
+ * A marking is enforced at read time from the property or column definition and the lineage of
+ * the version read, so each rule here closes a path that would otherwise return the value
+ * unredacted or make enforcement ambiguous.
+ *
+ * Projections are checked against the markings columns declare or inherit through `run` steps.
+ * What SQL steps derive needs the lake provider's analysis, which `checkPipelineMarkings` adds.
  */
 export function validateMarkingsAtStartup(input: {
   readonly markingIds: ReadonlySet<string>
@@ -22,8 +26,14 @@ export function validateMarkingsAtStartup(input: {
 }): void {
   validatePropertyMarkings(input.ontology, input.markingIds)
   validateColumnMarkings(input.datasetsById, input.markingIds)
-  rejectPipelineStepsOverMarkedColumns(input.pipelines, input.datasetsById)
-  validateProjectionMarkings(input.projections, input.ontology, input.datasetsById)
+
+  const runLineageOf = runStepLineages(input.datasetsById)
+  const markings = new MarkingPropagation(
+    input.datasetsById,
+    registeredSteps(input.pipelines)
+  ).derive((entry) => runLineageOf(entry) ?? null)
+
+  validateProjectionMarkings(input.projections, input.ontology, markings)
 }
 
 // ── Declarations ────────────────────────────────────────────
@@ -85,9 +95,20 @@ function assertSubtypeKeepsParentMarkings(
   const parent = ontology.getObjectTypeById(objectType.extends)
   if (!parent) return
 
+  // Indexed on the first marked parent property: most parents mark nothing.
+  let propertiesById: Map<string, Property> | undefined
   for (const parentProperty of parent.properties) {
-    const property = objectType.properties.find((candidate) => candidate.id === parentProperty.id)
-    const missing = missingMarkings(parentProperty.markings, property?.markings)
+    const parentMarkings = parentProperty.markings
+    if (!parentMarkings) continue
+
+    if (!propertiesById) {
+      propertiesById = new Map()
+      for (const property of objectType.properties) {
+        if (!propertiesById.has(property.id)) propertiesById.set(property.id, property)
+      }
+    }
+
+    const missing = missingMarkings(parentMarkings, propertiesById.get(parentProperty.id)?.markings)
     if (missing.length > 0) {
       throw new SecurityValidationError(
         `[Sixb] Property '${objectType.id}.${parentProperty.id}' must keep the markings of '${parent.id}.${parentProperty.id}': add ${missing.map((markingId) => `'${markingId}'`).join(", ")}.`
@@ -134,172 +155,6 @@ function assertMarkingList(
   }
 }
 
-// ── Propagation ─────────────────────────────────────────────
-
-/**
- * A step's output inherits nothing from its inputs until Sixb can derive each output column's
- * markings from the transform. Until then, reading a marked column would declassify it.
- */
-function rejectPipelineStepsOverMarkedColumns(
-  pipelines: readonly PipelineDefinition[],
-  datasetsById: ReadonlyMap<string, DatasetDefinition>
-): void {
-  for (const pipeline of pipelines) {
-    for (const { step } of pipeline.graph.nodes) {
-      for (const input of Object.values(step.inputs)) {
-        const dataset = datasetsById.get(input.id)
-        const column = dataset?.schema.columns.find((candidate) => candidate.markings)
-        if (!dataset || !column) continue
-        throw new SecurityValidationError(
-          `[Sixb] Pipeline '${pipeline.id}' step '${step.id}' reads column '${dataset.id}.${column.name}', which carries ${markingList(column.markings)}. Pipeline steps cannot read marked columns yet.`
-        )
-      }
-    }
-  }
-}
-
-/**
- * Objects are read through the ontology, so each projected property declares the markings of its
- * column: the ontology stays the reviewed contract of what each reader receives. Values that
- * identify objects or key links are returned wherever the object or link is, and telemetry does
- * not enforce markings yet, so those columns cannot carry markings.
- */
-function validateProjectionMarkings(
-  projections: ProjectionDefinitionCatalog,
-  ontology: OntologyDefinitionCatalog,
-  datasetsById: ReadonlyMap<string, DatasetDefinition>
-): void {
-  for (const projection of projections.listObjects()) {
-    const objectType = ontology.getObjectTypeById(projection.objectTypeId)
-    if (objectType) validateObjectProjection(projection, objectType, datasetsById)
-  }
-
-  for (const projection of projections.listLinks()) {
-    const role = `keys link '${projection.sourceObjectTypeId}.${projection.linkId}'`
-    for (const columnName of [projection.sourceField, projection.targetField]) {
-      assertUnmarkedColumn(projection, sourceColumn(projection, columnName, datasetsById), role)
-    }
-  }
-
-  for (const projection of projections.listTelemetry()) {
-    const columnNames = [
-      projection.objectIdField,
-      projection.atField,
-      ...Object.values(projection.properties).flatMap((mapping) =>
-        mapping.unitField === undefined
-          ? [mapping.valueField]
-          : [mapping.valueField, mapping.unitField]
-      ),
-    ]
-    for (const columnName of columnNames) {
-      const column = sourceColumn(projection, columnName, datasetsById)
-      if (!column.markings) continue
-      throw new SecurityValidationError(
-        `[Sixb] Projection '${projection.id}' reads column '${column.path}', which carries ${markingList(column.markings)}. Telemetry projections cannot read marked columns yet.`
-      )
-    }
-  }
-}
-
-function validateObjectProjection(
-  projection: ObjectProjectionDefinition,
-  objectType: ObjectType,
-  datasetsById: ReadonlyMap<string, DatasetDefinition>
-): void {
-  const keyRoles = objectProjectionKeyRoles(projection, objectType)
-
-  for (const [propertyId, columnName] of Object.entries(projection.properties)) {
-    const column = sourceColumn(projection, columnName, datasetsById)
-    const property = objectType.properties.find((candidate) => candidate.id === propertyId)
-    const path = `${objectType.id}.${propertyId}`
-    const keyRole = keyRoles.get(propertyId)
-    if (keyRole) {
-      assertUnmarkedColumn(projection, column, keyRole)
-      if (property?.markings) {
-        throw new SecurityValidationError(
-          `[Sixb] Projection '${projection.id}': property '${path}' ${keyRole} and cannot carry markings.`
-        )
-      }
-      continue
-    }
-    const missing = missingMarkings(column.markings, property?.markings)
-    if (missing.length > 0) {
-      throw new SecurityValidationError(
-        `[Sixb] Projection '${projection.id}': property '${path}' is projected from column '${column.path}', which carries ${markingList(column.markings)}. Add markings: ${markingList(missing)} to the property.`
-      )
-    }
-  }
-
-  for (const [linkId, link] of Object.entries(projection.links)) {
-    if (link.sourceField === undefined) continue
-    assertUnmarkedColumn(
-      projection,
-      sourceColumn(projection, link.sourceField, datasetsById),
-      `keys link '${objectType.id}.${linkId}'`
-    )
-  }
-
-  // The timestamp decides whether the source value or an app edit wins, which reveals how they
-  // compare.
-  const conflictResolution = projection.conflictResolution
-  if (conflictResolution?.strategy === "mostRecent") {
-    assertUnmarkedColumn(
-      projection,
-      sourceColumn(projection, conflictResolution.sourceTimestamp, datasetsById),
-      "decides which value wins under 'mostRecent'"
-    )
-  }
-}
-
-/** Properties whose value becomes an object id or a link key, with the role they play. */
-function objectProjectionKeyRoles(
-  projection: ObjectProjectionDefinition,
-  objectType: ObjectType
-): ReadonlyMap<string, string> {
-  const roles = new Map<string, string>()
-  for (const property of objectType.properties) {
-    if (property.primary) roles.set(property.id, `becomes the id of '${objectType.id}'`)
-  }
-  for (const [linkId, link] of Object.entries(projection.links)) {
-    if (link.sourcePropertyId === undefined) continue
-    roles.set(link.sourcePropertyId, `keys link '${objectType.id}.${linkId}'`)
-  }
-  return roles
-}
-
-interface SourceColumn {
-  readonly path: string
-  readonly markings?: readonly string[]
-}
-
-function sourceColumn(
-  projection: { readonly id: string; readonly datasetId: string },
-  columnName: string,
-  datasetsById: ReadonlyMap<string, DatasetDefinition>
-): SourceColumn {
-  const column = datasetsById
-    .get(projection.datasetId)
-    ?.schema.columns.find((candidate) => candidate.name === columnName)
-  // Projection validation runs first and resolves every mapped column.
-  if (!column) {
-    throw new Error(`[Sixb] Projection '${projection.id}' maps unknown column '${columnName}'.`)
-  }
-  return { path: `${projection.datasetId}.${columnName}`, markings: column.markings }
-}
-
-function assertUnmarkedColumn(
-  projection: { readonly id: string },
-  column: SourceColumn,
-  role: string
-): void {
-  if (!column.markings) return
-  throw new SecurityValidationError(
-    `[Sixb] Projection '${projection.id}': column '${column.path}' ${role} and cannot carry markings.`
-  )
-}
-
-// ── Helpers ─────────────────────────────────────────────────
-
 function missingMarkings(
   required: readonly string[] | undefined,
   declared: readonly string[] | undefined
@@ -307,7 +162,3 @@ function missingMarkings(
   const present = new Set(declared ?? [])
   return (required ?? []).filter((markingId) => !present.has(markingId))
 }
-
-function markingList(markings: readonly string[] | undefined): string {
-  return `[${(markings ?? []).join(", ")}]`
-}