@sixb/core 0.1.17 → 0.1.19

package/src/security/markings.ts

@@ -0,0 +1,313 @@
+import type { DatasetDefinition } from "../datasets/types"
+import type { OntologyDefinitionCatalog } from "../ontology/registry"
+import type { ObjectType, Property } from "../ontology/types"
+import type { PipelineDefinition } from "../pipelines/types"
+import type { ProjectionDefinitionCatalog } from "../projections/registry"
+import type { ObjectProjectionDefinition } from "../projections/types"
+import { SecurityValidationError } from "./errors"
+
+/**
+ * Check every marking once at startup.
+ *
+ * A marking is enforced at read time from the property or column definition alone, so each rule
+ * here closes a path that would otherwise return the value unredacted or make enforcement
+ * ambiguous.
+ */
+export function validateMarkingsAtStartup(input: {
+  readonly markingIds: ReadonlySet<string>
+  readonly ontology: OntologyDefinitionCatalog
+  readonly datasetsById: ReadonlyMap<string, DatasetDefinition>
+  readonly pipelines: readonly PipelineDefinition[]
+  readonly projections: ProjectionDefinitionCatalog
+}): void {
+  validatePropertyMarkings(input.ontology, input.markingIds)
+  validateColumnMarkings(input.datasetsById, input.markingIds)
+  rejectPipelineStepsOverMarkedColumns(input.pipelines, input.datasetsById)
+  validateProjectionMarkings(input.projections, input.ontology, input.datasetsById)
+}
+
+// ── Declarations ────────────────────────────────────────────
+
+function validatePropertyMarkings(
+  ontology: OntologyDefinitionCatalog,
+  markingIds: ReadonlySet<string>
+): void {
+  for (const objectType of ontology.listObjectTypes()) {
+    for (const property of objectType.properties) {
+      validateProperty(objectType, property, markingIds)
+    }
+    for (const link of objectType.links) {
+      for (const property of link.properties ?? []) {
+        if (property.markings !== undefined) {
+          throw new SecurityValidationError(
+            `[Sixb] Link property '${objectType.id}.${link.id}.${property.id}' cannot carry markings. Mark object properties instead.`
+          )
+        }
+      }
+    }
+    assertSubtypeKeepsParentMarkings(objectType, ontology)
+  }
+}
+
+function validateProperty(
+  objectType: ObjectType,
+  property: Property,
+  markingIds: ReadonlySet<string>
+): void {
+  if (property.markings === undefined) return
+  const path = `${objectType.id}.${property.id}`
+  assertMarkingList(property.markings, `Property '${path}'`, markingIds)
+
+  // The primary id identifies the object in every response, link, and event.
+  if (property.primary) {
+    throw new SecurityValidationError(
+      `[Sixb] Primary property '${path}' cannot carry markings: it identifies the object.`
+    )
+  }
+  // Telemetry values also flow through history reads and telemetry events, which do not
+  // enforce markings yet.
+  if (property.mode === "telemetry") {
+    throw new SecurityValidationError(
+      `[Sixb] Telemetry property '${path}' cannot carry markings yet.`
+    )
+  }
+}
+
+/**
+ * A subtype inherits its parent's properties and may redefine them. Subtype rows are returned by
+ * parent-type reads, so a redefinition that dropped a marking would declassify the parent's data.
+ */
+function assertSubtypeKeepsParentMarkings(
+  objectType: ObjectType,
+  ontology: OntologyDefinitionCatalog
+): void {
+  if (!objectType.extends) return
+  const parent = ontology.getObjectTypeById(objectType.extends)
+  if (!parent) return
+
+  for (const parentProperty of parent.properties) {
+    const property = objectType.properties.find((candidate) => candidate.id === parentProperty.id)
+    const missing = missingMarkings(parentProperty.markings, property?.markings)
+    if (missing.length > 0) {
+      throw new SecurityValidationError(
+        `[Sixb] Property '${objectType.id}.${parentProperty.id}' must keep the markings of '${parent.id}.${parentProperty.id}': add ${missing.map((markingId) => `'${markingId}'`).join(", ")}.`
+      )
+    }
+  }
+}
+
+function validateColumnMarkings(
+  datasetsById: ReadonlyMap<string, DatasetDefinition>,
+  markingIds: ReadonlySet<string>
+): void {
+  for (const dataset of datasetsById.values()) {
+    for (const column of dataset.schema.columns) {
+      if (column.markings === undefined) continue
+      assertMarkingList(
+        column.markings,
+        `Dataset column '${dataset.id}.${column.name}'`,
+        markingIds
+      )
+    }
+  }
+}
+
+function assertMarkingList(
+  markings: unknown,
+  subject: string,
+  markingIds: ReadonlySet<string>
+): void {
+  if (!Array.isArray(markings) || markings.length === 0) {
+    throw new SecurityValidationError(
+      `[Sixb] ${subject} markings must be a non-empty list of marking ids.`
+    )
+  }
+  if (new Set(markings).size !== markings.length) {
+    throw new SecurityValidationError(`[Sixb] ${subject} lists the same marking twice.`)
+  }
+  for (const markingId of markings) {
+    if (typeof markingId !== "string" || !markingIds.has(markingId)) {
+      throw new SecurityValidationError(
+        `[Sixb] ${subject} references unknown marking '${String(markingId)}'. Add it to 'security/markings/' or pass it to createSixb({ markings }).`
+      )
+    }
+  }
+}
+
+// ── Propagation ─────────────────────────────────────────────
+
+/**
+ * A step's output inherits nothing from its inputs until Sixb can derive each output column's
+ * markings from the transform. Until then, reading a marked column would declassify it.
+ */
+function rejectPipelineStepsOverMarkedColumns(
+  pipelines: readonly PipelineDefinition[],
+  datasetsById: ReadonlyMap<string, DatasetDefinition>
+): void {
+  for (const pipeline of pipelines) {
+    for (const { step } of pipeline.graph.nodes) {
+      for (const input of Object.values(step.inputs)) {
+        const dataset = datasetsById.get(input.id)
+        const column = dataset?.schema.columns.find((candidate) => candidate.markings)
+        if (!dataset || !column) continue
+        throw new SecurityValidationError(
+          `[Sixb] Pipeline '${pipeline.id}' step '${step.id}' reads column '${dataset.id}.${column.name}', which carries ${markingList(column.markings)}. Pipeline steps cannot read marked columns yet.`
+        )
+      }
+    }
+  }
+}
+
+/**
+ * Objects are read through the ontology, so each projected property declares the markings of its
+ * column: the ontology stays the reviewed contract of what each reader receives. Values that
+ * identify objects or key links are returned wherever the object or link is, and telemetry does
+ * not enforce markings yet, so those columns cannot carry markings.
+ */
+function validateProjectionMarkings(
+  projections: ProjectionDefinitionCatalog,
+  ontology: OntologyDefinitionCatalog,
+  datasetsById: ReadonlyMap<string, DatasetDefinition>
+): void {
+  for (const projection of projections.listObjects()) {
+    const objectType = ontology.getObjectTypeById(projection.objectTypeId)
+    if (objectType) validateObjectProjection(projection, objectType, datasetsById)
+  }
+
+  for (const projection of projections.listLinks()) {
+    const role = `keys link '${projection.sourceObjectTypeId}.${projection.linkId}'`
+    for (const columnName of [projection.sourceField, projection.targetField]) {
+      assertUnmarkedColumn(projection, sourceColumn(projection, columnName, datasetsById), role)
+    }
+  }
+
+  for (const projection of projections.listTelemetry()) {
+    const columnNames = [
+      projection.objectIdField,
+      projection.atField,
+      ...Object.values(projection.properties).flatMap((mapping) =>
+        mapping.unitField === undefined
+          ? [mapping.valueField]
+          : [mapping.valueField, mapping.unitField]
+      ),
+    ]
+    for (const columnName of columnNames) {
+      const column = sourceColumn(projection, columnName, datasetsById)
+      if (!column.markings) continue
+      throw new SecurityValidationError(
+        `[Sixb] Projection '${projection.id}' reads column '${column.path}', which carries ${markingList(column.markings)}. Telemetry projections cannot read marked columns yet.`
+      )
+    }
+  }
+}
+
+function validateObjectProjection(
+  projection: ObjectProjectionDefinition,
+  objectType: ObjectType,
+  datasetsById: ReadonlyMap<string, DatasetDefinition>
+): void {
+  const keyRoles = objectProjectionKeyRoles(projection, objectType)
+
+  for (const [propertyId, columnName] of Object.entries(projection.properties)) {
+    const column = sourceColumn(projection, columnName, datasetsById)
+    const property = objectType.properties.find((candidate) => candidate.id === propertyId)
+    const path = `${objectType.id}.${propertyId}`
+    const keyRole = keyRoles.get(propertyId)
+    if (keyRole) {
+      assertUnmarkedColumn(projection, column, keyRole)
+      if (property?.markings) {
+        throw new SecurityValidationError(
+          `[Sixb] Projection '${projection.id}': property '${path}' ${keyRole} and cannot carry markings.`
+        )
+      }
+      continue
+    }
+    const missing = missingMarkings(column.markings, property?.markings)
+    if (missing.length > 0) {
+      throw new SecurityValidationError(
+        `[Sixb] Projection '${projection.id}': property '${path}' is projected from column '${column.path}', which carries ${markingList(column.markings)}. Add markings: ${markingList(missing)} to the property.`
+      )
+    }
+  }
+
+  for (const [linkId, link] of Object.entries(projection.links)) {
+    if (link.sourceField === undefined) continue
+    assertUnmarkedColumn(
+      projection,
+      sourceColumn(projection, link.sourceField, datasetsById),
+      `keys link '${objectType.id}.${linkId}'`
+    )
+  }
+
+  // The timestamp decides whether the source value or an app edit wins, which reveals how they
+  // compare.
+  const conflictResolution = projection.conflictResolution
+  if (conflictResolution?.strategy === "mostRecent") {
+    assertUnmarkedColumn(
+      projection,
+      sourceColumn(projection, conflictResolution.sourceTimestamp, datasetsById),
+      "decides which value wins under 'mostRecent'"
+    )
+  }
+}
+
+/** Properties whose value becomes an object id or a link key, with the role they play. */
+function objectProjectionKeyRoles(
+  projection: ObjectProjectionDefinition,
+  objectType: ObjectType
+): ReadonlyMap<string, string> {
+  const roles = new Map<string, string>()
+  for (const property of objectType.properties) {
+    if (property.primary) roles.set(property.id, `becomes the id of '${objectType.id}'`)
+  }
+  for (const [linkId, link] of Object.entries(projection.links)) {
+    if (link.sourcePropertyId === undefined) continue
+    roles.set(link.sourcePropertyId, `keys link '${objectType.id}.${linkId}'`)
+  }
+  return roles
+}
+
+interface SourceColumn {
+  readonly path: string
+  readonly markings?: readonly string[]
+}
+
+function sourceColumn(
+  projection: { readonly id: string; readonly datasetId: string },
+  columnName: string,
+  datasetsById: ReadonlyMap<string, DatasetDefinition>
+): SourceColumn {
+  const column = datasetsById
+    .get(projection.datasetId)
+    ?.schema.columns.find((candidate) => candidate.name === columnName)
+  // Projection validation runs first and resolves every mapped column.
+  if (!column) {
+    throw new Error(`[Sixb] Projection '${projection.id}' maps unknown column '${columnName}'.`)
+  }
+  return { path: `${projection.datasetId}.${columnName}`, markings: column.markings }
+}
+
+function assertUnmarkedColumn(
+  projection: { readonly id: string },
+  column: SourceColumn,
+  role: string
+): void {
+  if (!column.markings) return
+  throw new SecurityValidationError(
+    `[Sixb] Projection '${projection.id}': column '${column.path}' ${role} and cannot carry markings.`
+  )
+}
+
+// ── Helpers ─────────────────────────────────────────────────
+
+function missingMarkings(
+  required: readonly string[] | undefined,
+  declared: readonly string[] | undefined
+): readonly string[] {
+  const present = new Set(declared ?? [])
+  return (required ?? []).filter((markingId) => !present.has(markingId))
+}
+
+function markingList(markings: readonly string[] | undefined): string {
+  return `[${(markings ?? []).join(", ")}]`
+}

package/src/security/redactions.ts

@@ -0,0 +1,6 @@
+import type { Redactions } from "./types"
+
+/** List the values hidden from a reader that lacks a clearance for their markings. */
+export function missingClearanceRedactions(ids: Iterable<string>): Redactions {
+  return Object.fromEntries([...ids].map((id) => [id, { reason: "missing_clearance" }] as const))
+}

package/src/security/registry.ts

@@ -1,10 +1,17 @@
 import { type ResolvedRole, resolveRoleGrants } from "../authorization"
 import { APPLICATION_IDS } from "./applications"
-import type { GroupDefinition, MembershipPolicyDefinition, RoleDefinition } from "./types"
+import type {
+  GroupDefinition,
+  MarkingDefinition,
+  MembershipPolicyDefinition,
+  RoleDefinition,
+} from "./types"
 import { validateSecurityDefinitionsAtStartup } from "./validation"
 
 /** Read-only access to validated security definitions and resolved role grants. */
 export interface SecurityDefinitionCatalog {
+  listMarkings(): readonly MarkingDefinition[]
+  getMarkingById(markingId: string): MarkingDefinition | null
   listGroups(): readonly GroupDefinition[]
   getGroupById(groupId: string): GroupDefinition | null
   listRoles(): readonly RoleDefinition[]
@@ -15,6 +22,7 @@ export interface SecurityDefinitionCatalog {
 }
 
 export interface SecurityRegistryOptions {
+  readonly markings?: readonly MarkingDefinition[]
   readonly groups?: readonly GroupDefinition[]
   readonly roles?: readonly RoleDefinition[]
   readonly membershipPolicies?: readonly MembershipPolicyDefinition[]
@@ -32,6 +40,7 @@ export interface SecurityRegistryOptions {
 }
 
 export class SecurityRegistry implements SecurityDefinitionCatalog {
+  private readonly markingsById: ReadonlyMap<string, MarkingDefinition>
   private readonly groupsById: ReadonlyMap<string, GroupDefinition>
   private readonly rolesById: ReadonlyMap<string, RoleDefinition>
   private readonly membershipPoliciesById: ReadonlyMap<string, MembershipPolicyDefinition>
@@ -39,6 +48,7 @@ export class SecurityRegistry implements SecurityDefinitionCatalog {
 
   constructor(input: SecurityRegistryOptions) {
     const definitions = validateSecurityDefinitionsAtStartup({
+      markings: input.markings ?? [],
       groups: input.groups ?? [],
       roles: input.roles ?? [],
       membershipPolicies: input.membershipPolicies ?? [],
@@ -71,6 +81,7 @@ export class SecurityRegistry implements SecurityDefinitionCatalog {
       getSubTypes: input.getSubTypes ?? (() => []),
     }
 
+    this.markingsById = definitions.markingsById
     this.groupsById = definitions.groupsById
     this.rolesById = definitions.rolesById
     this.membershipPoliciesById = definitions.membershipPoliciesById
@@ -78,9 +89,18 @@ export class SecurityRegistry implements SecurityDefinitionCatalog {
       id: role.id,
       grantedToGroupIds: role.grantedToGroupIds,
       grants: resolveRoleGrants(role, universe),
+      clearances: new Set(role.clearances ?? []),
     }))
   }
 
+  listMarkings(): readonly MarkingDefinition[] {
+    return [...this.markingsById.values()]
+  }
+
+  getMarkingById(markingId: string): MarkingDefinition | null {
+    return this.markingsById.get(markingId) ?? null
+  }
+
   listGroups(): readonly GroupDefinition[] {
     return [...this.groupsById.values()]
   }

package/src/security/types.ts

@@ -7,6 +7,28 @@ export interface GroupDefinition<TId extends string = string> {
   readonly description?: string
 }
 
+/**
+ * A classification carried by sensitive properties and dataset columns.
+ *
+ * A principal reads a marked value only when one of its roles lists the marking in `clearances`.
+ * No grant bypasses a marking.
+ */
+export interface MarkingDefinition<TId extends string = string> {
+  readonly kind: "marking"
+  readonly id: TId
+  readonly label?: string
+  readonly description?: string
+}
+
+/** Why a value was omitted from a read. */
+export type RedactionReason = "missing_clearance"
+
+/**
+ * Values omitted from a read, keyed by property id or column name. A redacted value is absent,
+ * never `null`.
+ */
+export type Redactions = Readonly<Record<string, { readonly reason: RedactionReason }>>
+
 /**
  * A group named either by its definition or by its id.
  *
@@ -149,6 +171,8 @@ export interface RoleDefinition<TId extends string = string> {
   readonly description?: string
   readonly grantedToGroupIds: readonly string[]
   readonly grants: readonly GrantDefinition[]
+  /** Ids of the markings this role's members may read. Absent means none. */
+  readonly clearances?: readonly string[]
 }
 
 export type MembershipOperation = "invite" | "assignGroups" | "suspend"
@@ -162,6 +186,8 @@ export interface MembershipPolicyDefinition<TId extends string = string> {
 }
 
 export interface RegisteredSecurityDefinitions {
+  readonly markings: readonly MarkingDefinition[]
+  readonly markingsById: ReadonlyMap<string, MarkingDefinition>
   readonly groups: readonly GroupDefinition[]
   readonly groupsById: ReadonlyMap<string, GroupDefinition>
   readonly roles: readonly RoleDefinition[]

package/src/security/validation.ts

@@ -5,6 +5,7 @@ import type {
   GrantCapability,
   GrantDefinition,
   GroupDefinition,
+  MarkingDefinition,
   MembershipPolicyDefinition,
   RegisteredSecurityDefinitions,
   RoleDefinition,
@@ -132,6 +133,32 @@ function assertNoDuplicateOperations(
   }
 }
 
+export function assertMarkingDefinition(
+  value: unknown,
+  createError: CreateSecurityError = (message) => new SecurityValidationError(message)
+): asserts value is MarkingDefinition {
+  if (!isRecord(value)) {
+    throw createError("[Sixb] Marking definition must be an object.")
+  }
+
+  if (value.kind !== "marking") {
+    throw createError("[Sixb] Marking definition kind must be 'marking'.")
+  }
+
+  assertNonEmptyString(value.id, "Marking id", createError)
+  assertOptionalString(value.label, `Marking '${value.id}' label`, createError)
+  assertOptionalString(value.description, `Marking '${value.id}' description`, createError)
+}
+
+export function isMarkingDefinition(value: unknown): value is MarkingDefinition {
+  try {
+    assertMarkingDefinition(value, (message) => new Error(message))
+    return true
+  } catch {
+    return false
+  }
+}
+
 export function assertGroupDefinition(
   value: unknown,
   createError: CreateSecurityError = (message) => new SecurityValidationError(message)
@@ -285,6 +312,10 @@ export function assertRoleDefinition(
   for (const grant of value.grants) {
     assertGrantDefinition(grant, `Role '${value.id}' grants`, createError)
   }
+
+  if (value.clearances !== undefined) {
+    assertStringArray(value.clearances, `Role '${value.id}' clearances`, createError)
+  }
 }
 
 export function isRoleDefinition(value: unknown): value is RoleDefinition {
@@ -297,6 +328,7 @@ export function isRoleDefinition(value: unknown): value is RoleDefinition {
 }
 
 export function validateSecurityDefinitionsAtStartup(input: {
+  readonly markings?: readonly MarkingDefinition[]
   readonly groups: readonly GroupDefinition[]
   readonly membershipPolicies: readonly MembershipPolicyDefinition[]
   readonly roles?: readonly RoleDefinition[]
@@ -325,10 +357,21 @@ export function validateSecurityDefinitionsAtStartup(input: {
   /** Registered Share ids — when provided, share grants must reference them. */
   readonly shareIds?: ReadonlySet<string>
 }): RegisteredSecurityDefinitions {
+  const markingsById = new Map<string, MarkingDefinition>()
   const groupsById = new Map<string, GroupDefinition>()
   const rolesById = new Map<string, RoleDefinition>()
   const membershipPoliciesById = new Map<string, MembershipPolicyDefinition>()
 
+  for (const marking of input.markings ?? []) {
+    assertMarkingDefinition(marking)
+
+    if (markingsById.has(marking.id)) {
+      throw new SecurityValidationError(`[Sixb] Duplicate marking id: ${marking.id}`)
+    }
+
+    markingsById.set(marking.id, marking)
+  }
+
   for (const group of input.groups) {
     assertGroupDefinition(group)
 
@@ -352,11 +395,23 @@ export function validateSecurityDefinitionsAtStartup(input: {
       )
     }
 
-    if (role.grants.length === 0) {
-      throw new SecurityValidationError(`[Sixb] Role '${role.id}' must declare at least one grant.`)
+    const clearances = role.clearances ?? []
+    if (role.grants.length === 0 && clearances.length === 0) {
+      throw new SecurityValidationError(
+        `[Sixb] Role '${role.id}' must declare at least one grant or clearance.`
+      )
     }
 
     assertNoDuplicateIds(role.grantedToGroupIds, `Role '${role.id}' grantedTo`)
+    assertNoDuplicateIds(clearances, `Role '${role.id}' clearances`)
+
+    for (const markingId of clearances) {
+      if (!markingsById.has(markingId)) {
+        throw new SecurityValidationError(
+          `[Sixb] Role '${role.id}' clearances reference unknown marking '${markingId}'. Add it to 'security/markings/' or pass it to createSixb({ markings }).`
+        )
+      }
+    }
 
     for (const groupId of role.grantedToGroupIds) {
       if (!groupsById.has(groupId)) {
@@ -416,6 +471,8 @@ export function validateSecurityDefinitionsAtStartup(input: {
   }
 
   return {
+    markings: [...markingsById.values()],
+    markingsById,
     groups: [...groupsById.values()],
     groupsById,
     roles: [...rolesById.values()],

package/src/storage/action-runs/idempotency.ts

@@ -1,13 +1,6 @@
 import type { ActionSubject } from "../../actions"
 import { stableJsonStringify } from "../../json"
-import type {
-  ActionRunEffectsRecord,
-  ActionRunPhase,
-  ActionRunRecord,
-  ActionRunWritebackRecord,
-  FinishActionRunInput,
-  QueueActionRunInput,
-} from "./types"
+import type { ActionRunEffectsRecord, ActionRunRecord, ActionRunWritebackRecord } from "./types"
 
 export function actionSubjectsEqual(left: ActionSubject, right: ActionSubject): boolean {
   if (left.kind !== right.kind) return false
@@ -20,6 +13,18 @@ export function actionRunParamsEqual(left: unknown, right: unknown): boolean {
   return stableJsonStringify(left) === stableJsonStringify(right)
 }
 
+/** Whether two runs carry the same request: the same Action, subject, and params. */
+export function actionRunRequestsEqual(
+  left: Pick<ActionRunRecord, "actionId" | "subject" | "params">,
+  right: Pick<ActionRunRecord, "actionId" | "subject" | "params">
+): boolean {
+  return (
+    left.actionId === right.actionId &&
+    actionSubjectsEqual(left.subject, right.subject) &&
+    actionRunParamsEqual(left.params, right.params)
+  )
+}
+
 export type ActionRunPhaseRecord = ActionRunWritebackRecord | ActionRunEffectsRecord
 
 export function actionRunPhaseRecordsEqual(
@@ -29,39 +34,6 @@ export function actionRunPhaseRecordsEqual(
   return actionRunParamsEqual(stripPhaseRecordCompletedAt(left), stripPhaseRecordCompletedAt(right))
 }
 
-export function finishActionRunPhase(
-  input: FinishActionRunInput,
-  current: ActionRunPhase | null | undefined
-): ActionRunPhase {
-  if (input.status === "succeeded") {
-    return input.phase ?? current ?? "validation"
-  }
-
-  return input.error.details.phase
-}
-
-export function canRequeueActionRunAfterEnqueueFailure(
-  existing: ActionRunRecord,
-  input: QueueActionRunInput
-): boolean {
-  return (
-    existing.status === "failed" &&
-    existing.phase === "enqueue" &&
-    existing.executionId === input.executionId &&
-    existing.error?.retryable === true &&
-    existing.actionId === input.actionId &&
-    existing.idempotencyKey === input.idempotencyKey &&
-    actionSubjectsEqual(existing.subject, input.subject) &&
-    actionRunParamsEqual(existing.params, input.params)
-  )
-}
-
-export function isTerminalActionRun(record: Pick<ActionRunRecord, "status">): boolean {
-  return (
-    record.status === "succeeded" || record.status === "failed" || record.status === "cancelled"
-  )
-}
-
 function stripPhaseRecordCompletedAt(
   record: ActionRunPhaseRecord
 ): Omit<ActionRunPhaseRecord, "completedAt"> {