@sixb/core 0.1.19 → 0.1.21

package/src/security/applied-markings-snapshot.ts

@@ -0,0 +1,235 @@
+import type { ProjectionKeyTarget } from "./projection-markings"
+
+/** Why readers lacking some markings cannot read an object type or a link. */
+export interface AppliedSeal {
+  readonly projectionId: string
+  readonly markings: ReadonlySet<string>
+  /** What carries the marking, completed by "carries marking 'm'" in messages. */
+  readonly reason: string
+}
+
+export interface AppliedLinkSeals {
+  readonly objectTypeId: string
+  readonly linkId: string
+  readonly targetObjectTypeId: string
+  readonly seals: readonly AppliedSeal[]
+}
+
+export interface AppliedMarkingsNotice {
+  readonly projectionId: string
+  /** Whether the projection makes an object type or a link unreadable, not only properties. */
+  readonly sealed: boolean
+  readonly message: string
+}
+
+/**
+ * What the applied versions add to the ontology's markings.
+ *
+ * Every reader of a host shares one snapshot and the clearances resolved for it, so a snapshot is
+ * immutable all the way down: its maps and sets refuse writes.
+ */
+export interface AppliedMarkingsSnapshot {
+  /** Surplus markings by object type, then property. */
+  readonly properties: ReadonlyMap<string, ReadonlyMap<string, ReadonlySet<string>>>
+  /** Object types readers lacking a seal's markings cannot read. */
+  readonly types: ReadonlyMap<string, readonly AppliedSeal[]>
+  /** Links, by {@link appliedLinkKey}, readers lacking a seal's markings cannot read. */
+  readonly links: ReadonlyMap<string, AppliedLinkSeals>
+  /** Every marking above: a reader cleared for all of them reads what the code allows. */
+  readonly markings: ReadonlySet<string>
+  /** One per projection whose applied version adds markings, for diagnostics. */
+  readonly notices: readonly AppliedMarkingsNotice[]
+}
+
+export function appliedLinkKey(objectTypeId: string, linkId: string): string {
+  return `${objectTypeId}\0${linkId}`
+}
+
+/** What one applied version adds, as plain data so that equal results serialize equally. */
+export interface AppliedContribution {
+  readonly properties: readonly {
+    readonly objectTypeId: string
+    readonly propertyId: string
+    readonly markings: readonly string[]
+  }[]
+  readonly seals: readonly {
+    readonly target: ProjectionKeyTarget
+    readonly seal: {
+      readonly projectionId: string
+      readonly markings: readonly string[]
+      readonly reason: string
+    }
+  }[]
+  readonly notice?: AppliedMarkingsNotice
+}
+
+/** An applied version that adds nothing. */
+export const NO_CONTRIBUTION: AppliedContribution = Object.freeze({
+  properties: Object.freeze([]),
+  seals: Object.freeze([]),
+})
+
+export const NO_APPLIED_MARKINGS: AppliedMarkingsSnapshot = Object.freeze({
+  properties: readonlyMap(new Map()),
+  types: readonlyMap(new Map()),
+  links: readonlyMap(new Map()),
+  markings: readonlySet(new Set<string>()),
+  notices: Object.freeze([]),
+})
+
+/** One snapshot from the contributions of every applied version, in projection order. */
+export function assembleAppliedMarkings(
+  contributions: readonly AppliedContribution[]
+): AppliedMarkingsSnapshot {
+  if (contributions.length === 0) return NO_APPLIED_MARKINGS
+
+  const properties = new Map<string, Map<string, Set<string>>>()
+  const types = new Map<string, AppliedSeal[]>()
+  const links = new Map<string, AppliedLinkSeals & { seals: AppliedSeal[] }>()
+  const markings = new Set<string>()
+  const notices: AppliedMarkingsNotice[] = []
+
+  for (const contribution of contributions) {
+    for (const { objectTypeId, propertyId, markings: added } of contribution.properties) {
+      const byProperty = properties.get(objectTypeId) ?? new Map<string, Set<string>>()
+      properties.set(objectTypeId, byProperty)
+
+      const set = byProperty.get(propertyId) ?? new Set<string>()
+      byProperty.set(propertyId, set)
+
+      for (const markingId of added) {
+        set.add(markingId)
+        markings.add(markingId)
+      }
+    }
+
+    for (const { target, seal: data } of contribution.seals) {
+      const seal: AppliedSeal = Object.freeze({
+        ...data,
+        markings: readonlySet(new Set(data.markings)),
+      })
+
+      for (const markingId of data.markings) markings.add(markingId)
+
+      if (target.kind === "type") {
+        types.set(target.objectTypeId, [...(types.get(target.objectTypeId) ?? []), seal])
+        continue
+      }
+
+      const key = appliedLinkKey(target.objectTypeId, target.linkId)
+      const existing = links.get(key)
+      if (existing) existing.seals.push(seal)
+      else {
+        links.set(key, {
+          objectTypeId: target.objectTypeId,
+          linkId: target.linkId,
+          targetObjectTypeId: target.targetObjectTypeId,
+          seals: [seal],
+        })
+      }
+    }
+
+    if (contribution.notice) notices.push(Object.freeze({ ...contribution.notice }))
+  }
+
+  return Object.freeze({
+    properties: readonlyMap(
+      new Map(
+        [...properties].map(([objectTypeId, byProperty]) => [
+          objectTypeId,
+          readonlyMap(
+            new Map([...byProperty].map(([propertyId, set]) => [propertyId, readonlySet(set)]))
+          ),
+        ])
+      )
+    ),
+    types: readonlyMap(
+      new Map([...types].map(([objectTypeId, seals]) => [objectTypeId, Object.freeze(seals)]))
+    ),
+    links: readonlyMap(
+      new Map(
+        [...links].map(([key, link]) => [
+          key,
+          Object.freeze({ ...link, seals: Object.freeze(link.seals) }),
+        ])
+      )
+    ),
+    markings: readonlySet(markings),
+    notices: Object.freeze(notices),
+  })
+}
+
+/**
+ * The snapshot with these properties' markings added: what an object event reads under, given the
+ * markings its origin version carries on them.
+ */
+export function withPropertyMarkings(
+  snapshot: AppliedMarkingsSnapshot,
+  added: AppliedContribution["properties"]
+): AppliedMarkingsSnapshot {
+  if (added.length === 0) return snapshot
+
+  const properties = new Map<string, Map<string, Set<string>>>(
+    [...snapshot.properties].map(([objectTypeId, byProperty]) => [
+      objectTypeId,
+      new Map([...byProperty].map(([propertyId, set]) => [propertyId, new Set(set)])),
+    ])
+  )
+
+  const markings = new Set(snapshot.markings)
+  for (const { objectTypeId, propertyId, markings: ids } of added) {
+    const byProperty = properties.get(objectTypeId) ?? new Map<string, Set<string>>()
+    properties.set(objectTypeId, byProperty)
+
+    const set = byProperty.get(propertyId) ?? new Set<string>()
+    byProperty.set(propertyId, set)
+
+    for (const markingId of ids) {
+      set.add(markingId)
+      markings.add(markingId)
+    }
+  }
+
+  return Object.freeze({
+    ...snapshot,
+    properties: readonlyMap(
+      new Map(
+        [...properties].map(([objectTypeId, byProperty]) => [
+          objectTypeId,
+          readonlyMap(
+            new Map([...byProperty].map(([propertyId, set]) => [propertyId, readonlySet(set)]))
+          ),
+        ])
+      )
+    ),
+    markings: readonlySet(markings),
+  })
+}
+
+/**
+ * A map that refuses writes. `Object.freeze` alone leaves a map's entries writable through its
+ * methods, so they are shadowed by ones that throw, and then frozen.
+ */
+function readonlyMap<TKey, TValue>(map: Map<TKey, TValue>): ReadonlyMap<TKey, TValue> {
+  return Object.freeze(
+    Object.defineProperties(map, {
+      set: { value: refuseWrite },
+      delete: { value: refuseWrite },
+      clear: { value: refuseWrite },
+    })
+  )
+}
+
+function readonlySet<TValue>(set: Set<TValue>): ReadonlySet<TValue> {
+  return Object.freeze(
+    Object.defineProperties(set, {
+      add: { value: refuseWrite },
+      delete: { value: refuseWrite },
+      clear: { value: refuseWrite },
+    })
+  )
+}
+
+function refuseWrite(): never {
+  throw new TypeError("[Sixb] Applied markings are immutable.")
+}

package/src/security/applied-markings.ts

@@ -0,0 +1,405 @@
+import type { LakeStorage } from "../lake-storage/types"
+import type { OntologyDefinitionCatalog } from "../ontology/registry"
+import type { PipelineDefinition } from "../pipelines/types"
+import type { ProjectionDefinitionCatalog } from "../projections/registry"
+import type { OntologySourceStorage } from "../storage/ontology"
+import {
+  type AppliedVersionRead,
+  appliedContribution,
+  type EventOriginRef,
+  errorMessage,
+  eventOriginMarkings,
+  producingPipelines,
+  type SourceProjection,
+  sentence,
+} from "./applied-markings-contribution"
+import {
+  type AppliedContribution,
+  type AppliedMarkingsSnapshot,
+  assembleAppliedMarkings,
+  NO_APPLIED_MARKINGS,
+  NO_CONTRIBUTION,
+  withPropertyMarkings,
+} from "./applied-markings-snapshot"
+import type { DatasetVersionMarkings } from "./dataset-version-markings"
+
+/**
+ * Objects keep the markings of the dataset version they were projected from.
+ *
+ * Startup checks the code: each projected property declares the markings its column carries. The
+ * objects a projection applied may predate that code. When a property stops declaring a marking
+ * its applied version still carries on the column, the objects still hold those values. Until the
+ * projection applies a version the code allows, each such property keeps the marking (its
+ * surplus), and an object type or link whose key column carries markings is unreadable to readers
+ * lacking them, like a type they have no view grant for.
+ *
+ * Lifecycle: the snapshot is read when a read needs it, with no background work.
+ * - A read reads it with one storage query for the projections' active sources; only projections
+ *   whose applied version or definition changed are classified again. A project whose columns
+ *   carry no marking reads nothing.
+ * - A read uses a snapshot younger than {@link APPLIED_MARKINGS_FRESH_MS}. An older one, up to
+ *   {@link APPLIED_MARKINGS_MAX_AGE_MS}, is used while one refresh runs behind the read. Past that
+ *   age, or before any snapshot, the read waits for a refresh and fails if it fails or hangs past
+ *   the same bound. Other servers so see a newly applied version within seconds.
+ * - A host that applies a projection invalidates the snapshot, so its next read refreshes first:
+ *   `sixb dev` sees a version at once.
+ */
+export interface AppliedMarkings {
+  /** Whether readers with these clearances can be denied anything the code alone allows. */
+  affects(clearances: ReadonlySet<string>): boolean
+  /** The current snapshot, refreshed on demand. */
+  current(): Promise<AppliedMarkingsSnapshot>
+  /** A projection applied a version in this process: the next read refreshes first. */
+  invalidate(): void
+  /**
+   * The snapshot with the markings an object event's origin version carries on the properties
+   * its projection maps: the event holds that version's values, whatever version the projection
+   * applied since. Reads each version once, keeping the last few.
+   */
+  forEventOrigin(
+    snapshot: AppliedMarkingsSnapshot,
+    origin: EventOriginRef
+  ): Promise<AppliedMarkingsSnapshot>
+  /**
+   * The snapshot with every marking a column declares on each property a projection maps to this
+   * type: what an object event's earlier values read under. They are the values an update or a
+   * deletion replaced, and the projected values an action or a direct write recorded. They came
+   * from versions projections applied before, which no event records, so they read like a version
+   * that cannot be read. Like the objects they wrote, a projection the code no longer declares
+   * marks nothing.
+   */
+  forEarlierValues(snapshot: AppliedMarkingsSnapshot, objectTypeId: string): AppliedMarkingsSnapshot
+}
+
+export interface AppliedMarkingsOptions {
+  readonly projectId: string
+  readonly ontology: Pick<OntologyDefinitionCatalog, "getObjectTypeById">
+  readonly projections: Pick<ProjectionDefinitionCatalog, "listObjects" | "listLinks">
+  /** The revision of a projection's current definition. */
+  readonly revisionOf: (projectionId: string) => string
+  readonly pipelines: readonly PipelineDefinition[]
+  readonly versionMarkings: DatasetVersionMarkings
+  readonly lakeStorage: Pick<LakeStorage, "getVersion">
+  readonly sources: Pick<OntologySourceStorage, "listActive">
+  /** A monotonic clock in milliseconds. */
+  readonly now?: () => number
+  /** For this repo's tests: the bound past which a read waits, and a refresh fails. */
+  readonly maxAgeMs?: number
+}
+
+/** Origin versions, and snapshots combined with them, kept for the events read next. */
+const EVENT_ORIGINS_KEPT = 64
+
+/** A read uses a snapshot younger than this as it is. */
+export const APPLIED_MARKINGS_FRESH_MS = 5_000
+/** A read waits for a snapshot older than this, and a refresh that takes longer fails. */
+export const APPLIED_MARKINGS_MAX_AGE_MS = 10_000
+
+interface ProjectionState {
+  /** The applied version and revision the contribution was computed for. */
+  readonly key: string
+  /** The version could not be read: seal now, read it again on the next refresh. */
+  readonly retry: boolean
+  readonly contribution: AppliedContribution
+}
+
+export function createAppliedMarkings(options: AppliedMarkingsOptions): AppliedMarkings {
+  const maxAgeMs = options.maxAgeMs ?? APPLIED_MARKINGS_MAX_AGE_MS
+  const freshMs = Math.min(APPLIED_MARKINGS_FRESH_MS, maxAgeMs / 2)
+  const now = options.now ?? (() => performance.now())
+  const versionMarkings = options.versionMarkings
+  let pipelineOf: ((datasetId: string) => string | undefined) | undefined
+
+  let snapshot: AppliedMarkingsSnapshot | undefined
+  let fingerprint: string | undefined
+  let refreshedAt = 0
+  let states = new Map<string, ProjectionState>()
+  let generation = 0
+  let refreshedGeneration = -1
+  let inflight: Promise<void> | undefined
+
+  // Only the latest refresh may publish: one past its deadline is abandoned.
+  let latestRun = 0
+  let failing = false
+  const warned = new Set<string>()
+
+  const refresh = (): Promise<void> => {
+    if (inflight) return inflight
+
+    latestRun += 1
+    const run = latestRun
+
+    let timer: ReturnType<typeof setTimeout> | undefined
+    const deadline = new Promise<never>((_resolve, reject) => {
+      timer = setTimeout(() => {
+        latestRun += 1
+        reject(
+          new Error(
+            `[Sixb] Reading the dataset versions projections applied took longer than ${maxAgeMs / 1000}s.`
+          )
+        )
+      }, maxAgeMs)
+      timer.unref?.()
+    })
+
+    const work = read(run)
+    // The deadline may settle first; the abandoned read must not report an unhandled rejection.
+    work.catch(() => {})
+
+    inflight = Promise.race([work, deadline]).finally(() => {
+      clearTimeout(timer)
+      inflight = undefined
+    })
+
+    return inflight
+  }
+
+  const read = async (run: number) => {
+    const startedAt = now()
+    const startedGeneration = generation
+
+    const active = await options.sources.listActive({ projectId: options.projectId })
+    const applied = new Map(active.map((source) => [source.source.projectionId, source]))
+    const projections: readonly SourceProjection[] = [
+      ...options.projections.listObjects(),
+      ...options.projections.listLinks(),
+    ]
+
+    const next = new Map<string, ProjectionState>()
+    await Promise.all(
+      projections.map(async (projection) => {
+        const projectionId = projection.id
+
+        // A source the code no longer declares is read under the code's markings.
+        const source = applied.get(projectionId)
+        if (!source) return
+
+        const key = JSON.stringify([
+          source.datasetVersion.datasetId,
+          source.datasetVersion.versionId,
+          source.projectionRevision,
+        ])
+
+        const previous = states.get(projectionId)
+        if (previous && previous.key === key && !previous.retry) {
+          next.set(projectionId, previous)
+          return
+        }
+
+        const version = await readVersion(source.datasetVersion)
+        pipelineOf ??= producingPipelines(options.pipelines)
+        const contribution = appliedContribution(projection, source, version, {
+          ontology: options.ontology,
+          revisionOf: options.revisionOf,
+          everyMarking: versionMarkings.everyMarking(),
+          pipelineOf,
+        })
+
+        next.set(projectionId, { key, retry: version.kind !== "read", contribution })
+      })
+    )
+
+    if (run !== latestRun) return
+
+    states = next
+    failing = false
+    publish(next)
+    refreshedAt = startedAt
+    refreshedGeneration = startedGeneration
+  }
+
+  /** Read and classify a version; a version that cannot be read is sealed, then read again. */
+  const readVersion = async (ref: {
+    readonly datasetId: string
+    readonly versionId: string
+  }): Promise<AppliedVersionRead> => {
+    try {
+      const version = await options.lakeStorage.getVersion(ref.datasetId, ref.versionId)
+      if (!version) return { kind: "missing" }
+      return { kind: "read", version, classification: versionMarkings.classify(version) }
+    } catch (error) {
+      return { kind: "failed", error }
+    }
+  }
+
+  const publish = (next: ReadonlyMap<string, ProjectionState>) => {
+    const contributions = [...next]
+      .sort(([left], [right]) => (left < right ? -1 : left > right ? 1 : 0))
+      .flatMap(([, state]) => (state.contribution === NO_CONTRIBUTION ? [] : [state.contribution]))
+    const nextFingerprint = JSON.stringify(contributions)
+
+    // Equal content keeps the same object, so clearances resolved for it stay cached.
+    if (snapshot && nextFingerprint === fingerprint) return
+
+    snapshot = assembleAppliedMarkings(contributions)
+    fingerprint = nextFingerprint
+
+    const sealed = new Set(
+      snapshot.notices.filter((notice) => notice.sealed).map((notice) => notice.projectionId)
+    )
+
+    // A projection sealed again after it was cleared warns again.
+    for (const projectionId of warned) if (!sealed.has(projectionId)) warned.delete(projectionId)
+
+    for (const notice of snapshot.notices) {
+      if (!notice.sealed || warned.has(notice.projectionId)) continue
+      warned.add(notice.projectionId)
+      console.warn(notice.message)
+    }
+  }
+
+  /** One refresh behind reads that keep using the snapshot; a failure is reported once a streak. */
+  const refreshBehind = () => {
+    if (inflight) return
+    refresh().catch((error: unknown) => {
+      if (failing) return
+      failing = true
+      console.warn(
+        `[Sixb] Could not read the dataset versions projections applied: ${sentence(errorMessage(error))} Reads keep the markings read before for up to ${maxAgeMs / 1000}s, then fail until it succeeds.`
+      )
+    })
+  }
+
+  const current = async (): Promise<AppliedMarkingsSnapshot> => {
+    if (versionMarkings.everyMarking().size === 0) return NO_APPLIED_MARKINGS
+
+    // An invalidation during a refresh is only covered by a refresh that starts after it.
+    for (let attempt = 0; attempt < 3; attempt += 1) {
+      if (snapshot && refreshedGeneration === generation) {
+        const age = now() - refreshedAt
+        if (age < freshMs) return snapshot
+        if (age <= maxAgeMs) {
+          refreshBehind()
+          return snapshot
+        }
+      }
+
+      try {
+        await refresh()
+      } catch (error) {
+        if (snapshot && now() - refreshedAt <= maxAgeMs) return snapshot
+        throw new Error(
+          `[Sixb] Cannot read the dataset versions projections applied, so markings cannot be enforced: ${errorMessage(error)}`,
+          { cause: error }
+        )
+      }
+    }
+
+    if (!snapshot) throw new Error("[Sixb] Applied markings were not read.")
+    return snapshot
+  }
+
+  // A version's classification never changes. One that could not be read is read again.
+  const originVersions = new Map<string, Promise<AppliedVersionRead>>()
+  const readOriginVersion = (origin: EventOriginRef): Promise<AppliedVersionRead> => {
+    const key = JSON.stringify([origin.datasetId, origin.datasetVersionId])
+    let version = originVersions.get(key)
+
+    if (version) {
+      originVersions.delete(key)
+    } else {
+      const read = readVersion({ datasetId: origin.datasetId, versionId: origin.datasetVersionId })
+      void read.then((result) => {
+        if (result.kind === "failed" && originVersions.get(key) === read) originVersions.delete(key)
+      })
+      version = read
+    }
+
+    keep(originVersions, key, version)
+    return version
+  }
+
+  const combined = new WeakMap<AppliedMarkingsSnapshot, Map<string, AppliedMarkingsSnapshot>>()
+  const forEventOrigin = async (snapshot: AppliedMarkingsSnapshot, origin: EventOriginRef) => {
+    const projection = options.projections
+      .listObjects()
+      .find((candidate) => candidate.id === origin.projectionId)
+
+    // Like the objects it wrote, an event of a projection the code no longer declares is read
+    // under the markings the code declares.
+    if (!projection) return snapshot
+
+    const read = await readOriginVersion(origin)
+    const key = JSON.stringify([
+      origin.projectionId,
+      origin.datasetId,
+      origin.datasetVersionId,
+      origin.objectTypeId,
+      read.kind,
+    ])
+
+    let bySnapshot = combined.get(snapshot)
+    const cached = bySnapshot?.get(key)
+    if (cached) return cached
+
+    const result = withPropertyMarkings(
+      snapshot,
+      eventOriginMarkings(projection, origin, read, {
+        ontology: options.ontology,
+        everyMarking: versionMarkings.everyMarking(),
+      })
+    )
+
+    // Equal snapshots share resolved clearances; a failed read is not kept.
+    if (read.kind !== "failed") {
+      bySnapshot ??= new Map()
+      combined.set(snapshot, bySnapshot)
+      keep(bySnapshot, key, result)
+    }
+
+    return result
+  }
+
+  const earlier = new WeakMap<AppliedMarkingsSnapshot, Map<string, AppliedMarkingsSnapshot>>()
+  const forEarlierValues = (snapshot: AppliedMarkingsSnapshot, objectTypeId: string) => {
+    let byType = earlier.get(snapshot)
+    const cached = byType?.get(objectTypeId)
+    if (cached) return cached
+
+    const primary = options.ontology
+      .getObjectTypeById(objectTypeId)
+      ?.properties.find((property) => property.primary)?.id
+    const projected = options.projections
+      .listObjects()
+      .filter((projection) => projection.objectTypeId === objectTypeId)
+      .flatMap((projection) => Object.keys(projection.properties))
+
+    const markings = [...versionMarkings.everyMarking()].sort()
+    const result = withPropertyMarkings(
+      snapshot,
+      [...new Set(projected)]
+        .filter((propertyId) => propertyId !== primary)
+        .map((propertyId) => ({ objectTypeId, propertyId, markings }))
+    )
+
+    byType ??= new Map()
+    earlier.set(snapshot, byType)
+    keep(byType, objectTypeId, result)
+    return result
+  }
+
+  return {
+    forEventOrigin,
+    forEarlierValues,
+    affects: (clearances) => {
+      for (const markingId of versionMarkings.everyMarking()) {
+        if (!clearances.has(markingId)) return true
+      }
+      return false
+    },
+    current,
+    // Also before the first snapshot: a read in flight may have listed the sources before it.
+    invalidate: () => {
+      generation += 1
+    },
+  }
+}
+
+/** Insert as the newest entry, dropping the oldest past the bound. */
+function keep<TValue>(map: Map<string, TValue>, key: string, value: TValue): void {
+  map.set(key, value)
+  if (map.size <= EVENT_ORIGINS_KEPT) return
+
+  const oldest = map.keys().next().value
+  if (oldest !== undefined) map.delete(oldest)
+}

package/src/security/builders.ts

@@ -39,7 +39,12 @@ export interface DefineMembershipPolicyOptions {
   readonly can: readonly MembershipOperation[]
 }
 
-const MEMBERSHIP_OPERATIONS = new Set<MembershipOperation>(["invite", "assignGroups", "suspend"])
+const MEMBERSHIP_OPERATIONS = new Set<MembershipOperation>([
+  "invite",
+  "assignGroups",
+  "suspend",
+  "serviceAccounts",
+])
 
 function assertNonEmptyArray<T>(value: readonly T[], field: string): void {
   if (value.length === 0) {
@@ -65,7 +70,7 @@ function membershipOperationsFrom(
   return operations.map((operation) => {
     if (!MEMBERSHIP_OPERATIONS.has(operation)) {
       throw new SecurityValidationError(
-        `[Sixb] ${field} must contain only membership operations: invite, assignGroups, suspend.`
+        `[Sixb] ${field} must contain only membership operations: invite, assignGroups, suspend, serviceAccounts.`
       )
     }