@sixb/core 0.1.19 → 0.1.21

package/src/security/markings.ts

@@ -3,15 +3,19 @@ import type { OntologyDefinitionCatalog } from "../ontology/registry"
 import type { ObjectType, Property } from "../ontology/types"
 import type { PipelineDefinition } from "../pipelines/types"
 import type { ProjectionDefinitionCatalog } from "../projections/registry"
-import type { ObjectProjectionDefinition } from "../projections/types"
 import { SecurityValidationError } from "./errors"
+import { MarkingPropagation, registeredSteps, runStepLineages } from "./marking-propagation"
+import { validateProjectionMarkings } from "./projection-markings"
 
 /**
  * Check every marking once at startup.
  *
- * A marking is enforced at read time from the property or column definition alone, so each rule
- * here closes a path that would otherwise return the value unredacted or make enforcement
- * ambiguous.
+ * A marking is enforced at read time from the property or column definition and the lineage of
+ * the version read, so each rule here closes a path that would otherwise return the value
+ * unredacted or make enforcement ambiguous.
+ *
+ * Projections are checked against the markings columns declare or inherit through `run` steps.
+ * What SQL steps derive needs the lake provider's analysis, which `checkPipelineMarkings` adds.
  */
 export function validateMarkingsAtStartup(input: {
   readonly markingIds: ReadonlySet<string>
@@ -22,8 +26,14 @@ export function validateMarkingsAtStartup(input: {
 }): void {
   validatePropertyMarkings(input.ontology, input.markingIds)
   validateColumnMarkings(input.datasetsById, input.markingIds)
-  rejectPipelineStepsOverMarkedColumns(input.pipelines, input.datasetsById)
-  validateProjectionMarkings(input.projections, input.ontology, input.datasetsById)
+
+  const runLineageOf = runStepLineages(input.datasetsById)
+  const markings = new MarkingPropagation(
+    input.datasetsById,
+    registeredSteps(input.pipelines)
+  ).derive((entry) => runLineageOf(entry) ?? null)
+
+  validateProjectionMarkings(input.projections, input.ontology, markings)
 }
 
 // ── Declarations ────────────────────────────────────────────
@@ -85,9 +95,20 @@ function assertSubtypeKeepsParentMarkings(
   const parent = ontology.getObjectTypeById(objectType.extends)
   if (!parent) return
 
+  // Indexed on the first marked parent property: most parents mark nothing.
+  let propertiesById: Map<string, Property> | undefined
   for (const parentProperty of parent.properties) {
-    const property = objectType.properties.find((candidate) => candidate.id === parentProperty.id)
-    const missing = missingMarkings(parentProperty.markings, property?.markings)
+    const parentMarkings = parentProperty.markings
+    if (!parentMarkings) continue
+
+    if (!propertiesById) {
+      propertiesById = new Map()
+      for (const property of objectType.properties) {
+        if (!propertiesById.has(property.id)) propertiesById.set(property.id, property)
+      }
+    }
+
+    const missing = missingMarkings(parentMarkings, propertiesById.get(parentProperty.id)?.markings)
     if (missing.length > 0) {
       throw new SecurityValidationError(
         `[Sixb] Property '${objectType.id}.${parentProperty.id}' must keep the markings of '${parent.id}.${parentProperty.id}': add ${missing.map((markingId) => `'${markingId}'`).join(", ")}.`
@@ -134,172 +155,6 @@ function assertMarkingList(
   }
 }
 
-// ── Propagation ─────────────────────────────────────────────
-
-/**
- * A step's output inherits nothing from its inputs until Sixb can derive each output column's
- * markings from the transform. Until then, reading a marked column would declassify it.
- */
-function rejectPipelineStepsOverMarkedColumns(
-  pipelines: readonly PipelineDefinition[],
-  datasetsById: ReadonlyMap<string, DatasetDefinition>
-): void {
-  for (const pipeline of pipelines) {
-    for (const { step } of pipeline.graph.nodes) {
-      for (const input of Object.values(step.inputs)) {
-        const dataset = datasetsById.get(input.id)
-        const column = dataset?.schema.columns.find((candidate) => candidate.markings)
-        if (!dataset || !column) continue
-        throw new SecurityValidationError(
-          `[Sixb] Pipeline '${pipeline.id}' step '${step.id}' reads column '${dataset.id}.${column.name}', which carries ${markingList(column.markings)}. Pipeline steps cannot read marked columns yet.`
-        )
-      }
-    }
-  }
-}
-
-/**
- * Objects are read through the ontology, so each projected property declares the markings of its
- * column: the ontology stays the reviewed contract of what each reader receives. Values that
- * identify objects or key links are returned wherever the object or link is, and telemetry does
- * not enforce markings yet, so those columns cannot carry markings.
- */
-function validateProjectionMarkings(
-  projections: ProjectionDefinitionCatalog,
-  ontology: OntologyDefinitionCatalog,
-  datasetsById: ReadonlyMap<string, DatasetDefinition>
-): void {
-  for (const projection of projections.listObjects()) {
-    const objectType = ontology.getObjectTypeById(projection.objectTypeId)
-    if (objectType) validateObjectProjection(projection, objectType, datasetsById)
-  }
-
-  for (const projection of projections.listLinks()) {
-    const role = `keys link '${projection.sourceObjectTypeId}.${projection.linkId}'`
-    for (const columnName of [projection.sourceField, projection.targetField]) {
-      assertUnmarkedColumn(projection, sourceColumn(projection, columnName, datasetsById), role)
-    }
-  }
-
-  for (const projection of projections.listTelemetry()) {
-    const columnNames = [
-      projection.objectIdField,
-      projection.atField,
-      ...Object.values(projection.properties).flatMap((mapping) =>
-        mapping.unitField === undefined
-          ? [mapping.valueField]
-          : [mapping.valueField, mapping.unitField]
-      ),
-    ]
-    for (const columnName of columnNames) {
-      const column = sourceColumn(projection, columnName, datasetsById)
-      if (!column.markings) continue
-      throw new SecurityValidationError(
-        `[Sixb] Projection '${projection.id}' reads column '${column.path}', which carries ${markingList(column.markings)}. Telemetry projections cannot read marked columns yet.`
-      )
-    }
-  }
-}
-
-function validateObjectProjection(
-  projection: ObjectProjectionDefinition,
-  objectType: ObjectType,
-  datasetsById: ReadonlyMap<string, DatasetDefinition>
-): void {
-  const keyRoles = objectProjectionKeyRoles(projection, objectType)
-
-  for (const [propertyId, columnName] of Object.entries(projection.properties)) {
-    const column = sourceColumn(projection, columnName, datasetsById)
-    const property = objectType.properties.find((candidate) => candidate.id === propertyId)
-    const path = `${objectType.id}.${propertyId}`
-    const keyRole = keyRoles.get(propertyId)
-    if (keyRole) {
-      assertUnmarkedColumn(projection, column, keyRole)
-      if (property?.markings) {
-        throw new SecurityValidationError(
-          `[Sixb] Projection '${projection.id}': property '${path}' ${keyRole} and cannot carry markings.`
-        )
-      }
-      continue
-    }
-    const missing = missingMarkings(column.markings, property?.markings)
-    if (missing.length > 0) {
-      throw new SecurityValidationError(
-        `[Sixb] Projection '${projection.id}': property '${path}' is projected from column '${column.path}', which carries ${markingList(column.markings)}. Add markings: ${markingList(missing)} to the property.`
-      )
-    }
-  }
-
-  for (const [linkId, link] of Object.entries(projection.links)) {
-    if (link.sourceField === undefined) continue
-    assertUnmarkedColumn(
-      projection,
-      sourceColumn(projection, link.sourceField, datasetsById),
-      `keys link '${objectType.id}.${linkId}'`
-    )
-  }
-
-  // The timestamp decides whether the source value or an app edit wins, which reveals how they
-  // compare.
-  const conflictResolution = projection.conflictResolution
-  if (conflictResolution?.strategy === "mostRecent") {
-    assertUnmarkedColumn(
-      projection,
-      sourceColumn(projection, conflictResolution.sourceTimestamp, datasetsById),
-      "decides which value wins under 'mostRecent'"
-    )
-  }
-}
-
-/** Properties whose value becomes an object id or a link key, with the role they play. */
-function objectProjectionKeyRoles(
-  projection: ObjectProjectionDefinition,
-  objectType: ObjectType
-): ReadonlyMap<string, string> {
-  const roles = new Map<string, string>()
-  for (const property of objectType.properties) {
-    if (property.primary) roles.set(property.id, `becomes the id of '${objectType.id}'`)
-  }
-  for (const [linkId, link] of Object.entries(projection.links)) {
-    if (link.sourcePropertyId === undefined) continue
-    roles.set(link.sourcePropertyId, `keys link '${objectType.id}.${linkId}'`)
-  }
-  return roles
-}
-
-interface SourceColumn {
-  readonly path: string
-  readonly markings?: readonly string[]
-}
-
-function sourceColumn(
-  projection: { readonly id: string; readonly datasetId: string },
-  columnName: string,
-  datasetsById: ReadonlyMap<string, DatasetDefinition>
-): SourceColumn {
-  const column = datasetsById
-    .get(projection.datasetId)
-    ?.schema.columns.find((candidate) => candidate.name === columnName)
-  // Projection validation runs first and resolves every mapped column.
-  if (!column) {
-    throw new Error(`[Sixb] Projection '${projection.id}' maps unknown column '${columnName}'.`)
-  }
-  return { path: `${projection.datasetId}.${columnName}`, markings: column.markings }
-}
-
-function assertUnmarkedColumn(
-  projection: { readonly id: string },
-  column: SourceColumn,
-  role: string
-): void {
-  if (!column.markings) return
-  throw new SecurityValidationError(
-    `[Sixb] Projection '${projection.id}': column '${column.path}' ${role} and cannot carry markings.`
-  )
-}
-
-// ── Helpers ─────────────────────────────────────────────────
-
 function missingMarkings(
   required: readonly string[] | undefined,
   declared: readonly string[] | undefined
@@ -307,7 +162,3 @@ function missingMarkings(
   const present = new Set(declared ?? [])
   return (required ?? []).filter((markingId) => !present.has(markingId))
 }
-
-function markingList(markings: readonly string[] | undefined): string {
-  return `[${(markings ?? []).join(", ")}]`
-}

package/src/security/membership-policies.ts

@@ -10,7 +10,7 @@ export interface MembershipPolicyScope {
   readonly operations: Readonly<Record<MembershipOperation, MembershipOperationScope>>
 }
 
-const MEMBERSHIP_OPERATIONS = ["invite", "assignGroups", "suspend"] as const
+const MEMBERSHIP_OPERATIONS = ["invite", "assignGroups", "suspend", "serviceAccounts"] as const
 
 export function resolveMembershipPolicyScope(input: {
   readonly membershipPolicies: readonly MembershipPolicyDefinition[]
@@ -40,6 +40,7 @@ export function resolveMembershipPolicyScope(input: {
       invite: freezeOperationScope(operations.invite),
       assignGroups: freezeOperationScope(operations.assignGroups),
       suspend: freezeOperationScope(operations.suspend),
+      serviceAccounts: freezeOperationScope(operations.serviceAccounts),
     },
   }
 }
@@ -83,6 +84,7 @@ function createEmptyOperationScopes(): Record<
     invite: { policyIds: [], groupIds: new Set<string>() },
     assignGroups: { policyIds: [], groupIds: new Set<string>() },
     suspend: { policyIds: [], groupIds: new Set<string>() },
+    serviceAccounts: { policyIds: [], groupIds: new Set<string>() },
   }
 }
 

package/src/security/pipeline-markings.ts

@@ -0,0 +1,175 @@
+import type { DatasetDefinition } from "../datasets/types"
+import type {
+  LakeStorageWithSql,
+  SqlTransformAnalysis,
+  SqlTransformColumnRef,
+} from "../lake-storage/sql-transforms"
+import type { DatasetColumnLineage, DatasetColumnRef, LakeStorage } from "../lake-storage/types"
+import { PipelineError } from "../pipelines/errors"
+import { stepInputColumns, stepReadColumns } from "../pipelines/lineage"
+import type { PipelineStepDefinition } from "../pipelines/types"
+import type { SixbDefinitions } from "../runtime/definitions"
+import { SecurityValidationError } from "./errors"
+import {
+  type DatasetMarkings,
+  MarkingPropagation,
+  type RegisteredStep,
+  registeredSteps,
+  runStepLineages,
+} from "./marking-propagation"
+import { describeOrigins, validateProjectionMarkings } from "./projection-markings"
+
+/**
+ * Derive the markings SQL steps give their outputs and check every projection against them.
+ *
+ * Startup cannot see through SQL, so this preflight completes it before a host serves traffic. It
+ * is a diagnostic: each version records the lineage of the SQL that actually ran, and reads and
+ * projections enforce markings from that record whether or not this ran.
+ *
+ * Only steps that read a marked column are analyzed, each step once. Analyzing a step can mark
+ * the inputs of others, so steps are analyzed in waves, a few at a time within a wave. A step whose
+ * SQL cannot be analyzed, or a lake provider that cannot analyze SQL, is reported with the step
+ * and a marked column it reads.
+ */
+export async function checkPipelineMarkings(input: {
+  readonly definitions: Pick<SixbDefinitions, "ontology" | "datasets" | "pipelines" | "projections">
+  readonly lakeStorage: LakeStorage | LakeStorageWithSql
+}): Promise<void> {
+  const { definitions, lakeStorage } = input
+  const datasetsById = new Map(definitions.datasets.list().map((dataset) => [dataset.id, dataset]))
+  const steps = registeredSteps(definitions.pipelines.list())
+  const runLineageOf = runStepLineages(datasetsById)
+  const analyzed = new Map<PipelineStepDefinition, DatasetColumnLineage>()
+  const lineageOf = (entry: RegisteredStep) =>
+    runLineageOf(entry) ?? analyzed.get(entry.step) ?? null
+
+  const markings = new MarkingPropagation(datasetsById, steps).derive(lineageOf)
+
+  for (;;) {
+    const wave = new Map<PipelineStepDefinition, RegisteredStep>()
+    for (const entry of steps) {
+      if (entry.step.executor.kind !== "sql" || analyzed.has(entry.step)) continue
+      if (stepReadColumns(entry.step).some((ref) => markings.of(ref).size > 0)) {
+        if (!wave.has(entry.step)) wave.set(entry.step, entry)
+      }
+    }
+
+    if (wave.size === 0) break
+
+    await forEachConcurrently([...wave], ANALYSIS_CONCURRENCY, async ([step, entry]) => {
+      analyzed.set(step, await analyzeSqlStep(entry, markings, datasetsById, lakeStorage))
+    })
+    markings.derive(lineageOf)
+  }
+
+  validateProjectionMarkings(definitions.projections, definitions.ontology, markings)
+}
+
+/** Analyses run at once. A provider may open a database per analysis, so a wave is not unbounded. */
+const ANALYSIS_CONCURRENCY = 8
+
+/** Run `task` on each item, `limit` at a time. Stops starting new tasks once one fails. */
+async function forEachConcurrently<TItem>(
+  items: readonly TItem[],
+  limit: number,
+  task: (item: TItem) => Promise<void>
+): Promise<void> {
+  const pending = items.values()
+  let failed = false
+
+  const worker = async () => {
+    for (const item of pending) {
+      if (failed) return
+
+      try {
+        await task(item)
+      } catch (error) {
+        failed = true
+        throw error
+      }
+    }
+  }
+
+  await Promise.all(Array.from({ length: Math.min(limit, items.length) }, worker))
+}
+
+async function analyzeSqlStep(
+  entry: RegisteredStep,
+  markings: DatasetMarkings,
+  datasetsById: ReadonlyMap<string, DatasetDefinition>,
+  lakeStorage: LakeStorage | LakeStorageWithSql
+): Promise<DatasetColumnLineage> {
+  const { step } = entry
+  const subject = `[Sixb] Pipeline '${entry.pipelineId}' step '${step.id}'`
+
+  if (step.executor.kind !== "sql") throw new PipelineError(`${subject} is not a SQL step.`)
+
+  const sql = "sql" in lakeStorage ? lakeStorage.sql : undefined
+  if (!sql?.analyze) {
+    throw new SecurityValidationError(
+      `${subject} reads ${describeMarkedRead(step, markings)}, but the lake provider cannot analyze SQL to derive the markings of its output. Use a lake provider that analyzes SQL, or pick only unmarked columns.`
+    )
+  }
+
+  const inputs = Object.entries(step.inputs)
+  let analysis: SqlTransformAnalysis
+  try {
+    analysis = await sql.analyze({
+      sql: step.executor.sql,
+      sources: Object.fromEntries(
+        inputs.map(([name, dataset]) => [
+          name,
+          {
+            dataset: registeredDataset(datasetsById, dataset.id, subject),
+            columns: stepInputColumns(dataset),
+          },
+        ])
+      ),
+    })
+  } catch (error) {
+    const reason = error instanceof Error ? error.message : String(error)
+    throw new PipelineError(`${subject} could not be analyzed: ${reason}`, { cause: error })
+  }
+
+  if (!analysis.supported) {
+    throw new SecurityValidationError(
+      `${subject} reads ${describeMarkedRead(step, markings)}, but its SQL cannot be analyzed: ${analysis.reason}. Rewrite that part of the query, or pick only unmarked columns.`
+    )
+  }
+
+  const datasetIdBySource = new Map(inputs.map(([name, dataset]) => [name, dataset.id]))
+  const toRef = (ref: SqlTransformColumnRef): DatasetColumnRef => {
+    const datasetId = datasetIdBySource.get(ref.source)
+    if (datasetId === undefined) {
+      throw new PipelineError(`${subject} analysis references unknown source '${ref.source}'.`)
+    }
+
+    return { datasetId, column: ref.column }
+  }
+
+  return {
+    rows: analysis.lineage.rows.map(toRef),
+    columns: Object.fromEntries(
+      Object.entries(analysis.lineage.columns).map(([column, refs]) => [column, refs.map(toRef)])
+    ),
+  }
+}
+
+/** A marked column the step reads, for messages. */
+function describeMarkedRead(step: PipelineStepDefinition, markings: DatasetMarkings): string {
+  const ref = stepReadColumns(step).find((candidate) => markings.of(candidate).size > 0)
+  if (!ref) return "no marked column"
+
+  const path = `${ref.datasetId}.${ref.column}`
+  return `column '${path}', which carries ${describeOrigins(markings.of(ref), path)}`
+}
+
+function registeredDataset(
+  datasetsById: ReadonlyMap<string, DatasetDefinition>,
+  datasetId: string,
+  subject: string
+): DatasetDefinition {
+  const dataset = datasetsById.get(datasetId)
+  if (!dataset) throw new PipelineError(`${subject} reads unregistered dataset '${datasetId}'.`)
+  return dataset
+}

package/src/security/project.ts

@@ -0,0 +1,8 @@
+import type { ProjectReference } from "./types"
+
+/** Project-wide surfaces used in security grants such as `can.observe(project.logs)`. */
+export const project: ProjectReference = Object.freeze({
+  kind: "project",
+  logs: Object.freeze({ kind: "logs" }),
+  activity: Object.freeze({ kind: "activity" }),
+})