@sixb/core 0.1.20 → 0.1.22

package/src/security/applied-markings.ts

@@ -0,0 +1,405 @@
+import type { LakeStorage } from "../lake-storage/types"
+import type { OntologyDefinitionCatalog } from "../ontology/registry"
+import type { PipelineDefinition } from "../pipelines/types"
+import type { ProjectionDefinitionCatalog } from "../projections/registry"
+import type { OntologySourceStorage } from "../storage/ontology"
+import {
+  type AppliedVersionRead,
+  appliedContribution,
+  type EventOriginRef,
+  errorMessage,
+  eventOriginMarkings,
+  producingPipelines,
+  type SourceProjection,
+  sentence,
+} from "./applied-markings-contribution"
+import {
+  type AppliedContribution,
+  type AppliedMarkingsSnapshot,
+  assembleAppliedMarkings,
+  NO_APPLIED_MARKINGS,
+  NO_CONTRIBUTION,
+  withPropertyMarkings,
+} from "./applied-markings-snapshot"
+import type { DatasetVersionMarkings } from "./dataset-version-markings"
+
+/**
+ * Objects keep the markings of the dataset version they were projected from.
+ *
+ * Startup checks the code: each projected property declares the markings its column carries. The
+ * objects a projection applied may predate that code. When a property stops declaring a marking
+ * its applied version still carries on the column, the objects still hold those values. Until the
+ * projection applies a version the code allows, each such property keeps the marking (its
+ * surplus), and an object type or link whose key column carries markings is unreadable to readers
+ * lacking them, like a type they have no view grant for.
+ *
+ * Lifecycle: the snapshot is read when a read needs it, with no background work.
+ * - A read reads it with one storage query for the projections' active sources; only projections
+ *   whose applied version or definition changed are classified again. A project whose columns
+ *   carry no marking reads nothing.
+ * - A read uses a snapshot younger than {@link APPLIED_MARKINGS_FRESH_MS}. An older one, up to
+ *   {@link APPLIED_MARKINGS_MAX_AGE_MS}, is used while one refresh runs behind the read. Past that
+ *   age, or before any snapshot, the read waits for a refresh and fails if it fails or hangs past
+ *   the same bound. Other servers so see a newly applied version within seconds.
+ * - A host that applies a projection invalidates the snapshot, so its next read refreshes first:
+ *   `sixb dev` sees a version at once.
+ */
+export interface AppliedMarkings {
+  /** Whether readers with these clearances can be denied anything the code alone allows. */
+  affects(clearances: ReadonlySet<string>): boolean
+  /** The current snapshot, refreshed on demand. */
+  current(): Promise<AppliedMarkingsSnapshot>
+  /** A projection applied a version in this process: the next read refreshes first. */
+  invalidate(): void
+  /**
+   * The snapshot with the markings an object event's origin version carries on the properties
+   * its projection maps: the event holds that version's values, whatever version the projection
+   * applied since. Reads each version once, keeping the last few.
+   */
+  forEventOrigin(
+    snapshot: AppliedMarkingsSnapshot,
+    origin: EventOriginRef
+  ): Promise<AppliedMarkingsSnapshot>
+  /**
+   * The snapshot with every marking a column declares on each property a projection maps to this
+   * type: what an object event's earlier values read under. They are the values an update or a
+   * deletion replaced, and the projected values an action or a direct write recorded. They came
+   * from versions projections applied before, which no event records, so they read like a version
+   * that cannot be read. Like the objects they wrote, a projection the code no longer declares
+   * marks nothing.
+   */
+  forEarlierValues(snapshot: AppliedMarkingsSnapshot, objectTypeId: string): AppliedMarkingsSnapshot
+}
+
+export interface AppliedMarkingsOptions {
+  readonly projectId: string
+  readonly ontology: Pick<OntologyDefinitionCatalog, "getObjectTypeById">
+  readonly projections: Pick<ProjectionDefinitionCatalog, "listObjects" | "listLinks">
+  /** The revision of a projection's current definition. */
+  readonly revisionOf: (projectionId: string) => string
+  readonly pipelines: readonly PipelineDefinition[]
+  readonly versionMarkings: DatasetVersionMarkings
+  readonly lakeStorage: Pick<LakeStorage, "getVersion">
+  readonly sources: Pick<OntologySourceStorage, "listActive">
+  /** A monotonic clock in milliseconds. */
+  readonly now?: () => number
+  /** For this repo's tests: the bound past which a read waits, and a refresh fails. */
+  readonly maxAgeMs?: number
+}
+
+/** Origin versions, and snapshots combined with them, kept for the events read next. */
+const EVENT_ORIGINS_KEPT = 64
+
+/** A read uses a snapshot younger than this as it is. */
+export const APPLIED_MARKINGS_FRESH_MS = 5_000
+/** A read waits for a snapshot older than this, and a refresh that takes longer fails. */
+export const APPLIED_MARKINGS_MAX_AGE_MS = 10_000
+
+interface ProjectionState {
+  /** The applied version and revision the contribution was computed for. */
+  readonly key: string
+  /** The version could not be read: seal now, read it again on the next refresh. */
+  readonly retry: boolean
+  readonly contribution: AppliedContribution
+}
+
+export function createAppliedMarkings(options: AppliedMarkingsOptions): AppliedMarkings {
+  const maxAgeMs = options.maxAgeMs ?? APPLIED_MARKINGS_MAX_AGE_MS
+  const freshMs = Math.min(APPLIED_MARKINGS_FRESH_MS, maxAgeMs / 2)
+  const now = options.now ?? (() => performance.now())
+  const versionMarkings = options.versionMarkings
+  let pipelineOf: ((datasetId: string) => string | undefined) | undefined
+
+  let snapshot: AppliedMarkingsSnapshot | undefined
+  let fingerprint: string | undefined
+  let refreshedAt = 0
+  let states = new Map<string, ProjectionState>()
+  let generation = 0
+  let refreshedGeneration = -1
+  let inflight: Promise<void> | undefined
+
+  // Only the latest refresh may publish: one past its deadline is abandoned.
+  let latestRun = 0
+  let failing = false
+  const warned = new Set<string>()
+
+  const refresh = (): Promise<void> => {
+    if (inflight) return inflight
+
+    latestRun += 1
+    const run = latestRun
+
+    let timer: ReturnType<typeof setTimeout> | undefined
+    const deadline = new Promise<never>((_resolve, reject) => {
+      timer = setTimeout(() => {
+        latestRun += 1
+        reject(
+          new Error(
+            `[Sixb] Reading the dataset versions projections applied took longer than ${maxAgeMs / 1000}s.`
+          )
+        )
+      }, maxAgeMs)
+      timer.unref?.()
+    })
+
+    const work = read(run)
+    // The deadline may settle first; the abandoned read must not report an unhandled rejection.
+    work.catch(() => {})
+
+    inflight = Promise.race([work, deadline]).finally(() => {
+      clearTimeout(timer)
+      inflight = undefined
+    })
+
+    return inflight
+  }
+
+  const read = async (run: number) => {
+    const startedAt = now()
+    const startedGeneration = generation
+
+    const active = await options.sources.listActive({ projectId: options.projectId })
+    const applied = new Map(active.map((source) => [source.source.projectionId, source]))
+    const projections: readonly SourceProjection[] = [
+      ...options.projections.listObjects(),
+      ...options.projections.listLinks(),
+    ]
+
+    const next = new Map<string, ProjectionState>()
+    await Promise.all(
+      projections.map(async (projection) => {
+        const projectionId = projection.id
+
+        // A source the code no longer declares is read under the code's markings.
+        const source = applied.get(projectionId)
+        if (!source) return
+
+        const key = JSON.stringify([
+          source.datasetVersion.datasetId,
+          source.datasetVersion.versionId,
+          source.projectionRevision,
+        ])
+
+        const previous = states.get(projectionId)
+        if (previous && previous.key === key && !previous.retry) {
+          next.set(projectionId, previous)
+          return
+        }
+
+        const version = await readVersion(source.datasetVersion)
+        pipelineOf ??= producingPipelines(options.pipelines)
+        const contribution = appliedContribution(projection, source, version, {
+          ontology: options.ontology,
+          revisionOf: options.revisionOf,
+          everyMarking: versionMarkings.everyMarking(),
+          pipelineOf,
+        })
+
+        next.set(projectionId, { key, retry: version.kind !== "read", contribution })
+      })
+    )
+
+    if (run !== latestRun) return
+
+    states = next
+    failing = false
+    publish(next)
+    refreshedAt = startedAt
+    refreshedGeneration = startedGeneration
+  }
+
+  /** Read and classify a version; a version that cannot be read is sealed, then read again. */
+  const readVersion = async (ref: {
+    readonly datasetId: string
+    readonly versionId: string
+  }): Promise<AppliedVersionRead> => {
+    try {
+      const version = await options.lakeStorage.getVersion(ref.datasetId, ref.versionId)
+      if (!version) return { kind: "missing" }
+      return { kind: "read", version, classification: versionMarkings.classify(version) }
+    } catch (error) {
+      return { kind: "failed", error }
+    }
+  }
+
+  const publish = (next: ReadonlyMap<string, ProjectionState>) => {
+    const contributions = [...next]
+      .sort(([left], [right]) => (left < right ? -1 : left > right ? 1 : 0))
+      .flatMap(([, state]) => (state.contribution === NO_CONTRIBUTION ? [] : [state.contribution]))
+    const nextFingerprint = JSON.stringify(contributions)
+
+    // Equal content keeps the same object, so clearances resolved for it stay cached.
+    if (snapshot && nextFingerprint === fingerprint) return
+
+    snapshot = assembleAppliedMarkings(contributions)
+    fingerprint = nextFingerprint
+
+    const sealed = new Set(
+      snapshot.notices.filter((notice) => notice.sealed).map((notice) => notice.projectionId)
+    )
+
+    // A projection sealed again after it was cleared warns again.
+    for (const projectionId of warned) if (!sealed.has(projectionId)) warned.delete(projectionId)
+
+    for (const notice of snapshot.notices) {
+      if (!notice.sealed || warned.has(notice.projectionId)) continue
+      warned.add(notice.projectionId)
+      console.warn(notice.message)
+    }
+  }
+
+  /** One refresh behind reads that keep using the snapshot; a failure is reported once a streak. */
+  const refreshBehind = () => {
+    if (inflight) return
+    refresh().catch((error: unknown) => {
+      if (failing) return
+      failing = true
+      console.warn(
+        `[Sixb] Could not read the dataset versions projections applied: ${sentence(errorMessage(error))} Reads keep the markings read before for up to ${maxAgeMs / 1000}s, then fail until it succeeds.`
+      )
+    })
+  }
+
+  const current = async (): Promise<AppliedMarkingsSnapshot> => {
+    if (versionMarkings.everyMarking().size === 0) return NO_APPLIED_MARKINGS
+
+    // An invalidation during a refresh is only covered by a refresh that starts after it.
+    for (let attempt = 0; attempt < 3; attempt += 1) {
+      if (snapshot && refreshedGeneration === generation) {
+        const age = now() - refreshedAt
+        if (age < freshMs) return snapshot
+        if (age <= maxAgeMs) {
+          refreshBehind()
+          return snapshot
+        }
+      }
+
+      try {
+        await refresh()
+      } catch (error) {
+        if (snapshot && now() - refreshedAt <= maxAgeMs) return snapshot
+        throw new Error(
+          `[Sixb] Cannot read the dataset versions projections applied, so markings cannot be enforced: ${errorMessage(error)}`,
+          { cause: error }
+        )
+      }
+    }
+
+    if (!snapshot) throw new Error("[Sixb] Applied markings were not read.")
+    return snapshot
+  }
+
+  // A version's classification never changes. One that could not be read is read again.
+  const originVersions = new Map<string, Promise<AppliedVersionRead>>()
+  const readOriginVersion = (origin: EventOriginRef): Promise<AppliedVersionRead> => {
+    const key = JSON.stringify([origin.datasetId, origin.datasetVersionId])
+    let version = originVersions.get(key)
+
+    if (version) {
+      originVersions.delete(key)
+    } else {
+      const read = readVersion({ datasetId: origin.datasetId, versionId: origin.datasetVersionId })
+      void read.then((result) => {
+        if (result.kind === "failed" && originVersions.get(key) === read) originVersions.delete(key)
+      })
+      version = read
+    }
+
+    keep(originVersions, key, version)
+    return version
+  }
+
+  const combined = new WeakMap<AppliedMarkingsSnapshot, Map<string, AppliedMarkingsSnapshot>>()
+  const forEventOrigin = async (snapshot: AppliedMarkingsSnapshot, origin: EventOriginRef) => {
+    const projection = options.projections
+      .listObjects()
+      .find((candidate) => candidate.id === origin.projectionId)
+
+    // Like the objects it wrote, an event of a projection the code no longer declares is read
+    // under the markings the code declares.
+    if (!projection) return snapshot
+
+    const read = await readOriginVersion(origin)
+    const key = JSON.stringify([
+      origin.projectionId,
+      origin.datasetId,
+      origin.datasetVersionId,
+      origin.objectTypeId,
+      read.kind,
+    ])
+
+    let bySnapshot = combined.get(snapshot)
+    const cached = bySnapshot?.get(key)
+    if (cached) return cached
+
+    const result = withPropertyMarkings(
+      snapshot,
+      eventOriginMarkings(projection, origin, read, {
+        ontology: options.ontology,
+        everyMarking: versionMarkings.everyMarking(),
+      })
+    )
+
+    // Equal snapshots share resolved clearances; a failed read is not kept.
+    if (read.kind !== "failed") {
+      bySnapshot ??= new Map()
+      combined.set(snapshot, bySnapshot)
+      keep(bySnapshot, key, result)
+    }
+
+    return result
+  }
+
+  const earlier = new WeakMap<AppliedMarkingsSnapshot, Map<string, AppliedMarkingsSnapshot>>()
+  const forEarlierValues = (snapshot: AppliedMarkingsSnapshot, objectTypeId: string) => {
+    let byType = earlier.get(snapshot)
+    const cached = byType?.get(objectTypeId)
+    if (cached) return cached
+
+    const primary = options.ontology
+      .getObjectTypeById(objectTypeId)
+      ?.properties.find((property) => property.primary)?.id
+    const projected = options.projections
+      .listObjects()
+      .filter((projection) => projection.objectTypeId === objectTypeId)
+      .flatMap((projection) => Object.keys(projection.properties))
+
+    const markings = [...versionMarkings.everyMarking()].sort()
+    const result = withPropertyMarkings(
+      snapshot,
+      [...new Set(projected)]
+        .filter((propertyId) => propertyId !== primary)
+        .map((propertyId) => ({ objectTypeId, propertyId, markings }))
+    )
+
+    byType ??= new Map()
+    earlier.set(snapshot, byType)
+    keep(byType, objectTypeId, result)
+    return result
+  }
+
+  return {
+    forEventOrigin,
+    forEarlierValues,
+    affects: (clearances) => {
+      for (const markingId of versionMarkings.everyMarking()) {
+        if (!clearances.has(markingId)) return true
+      }
+      return false
+    },
+    current,
+    // Also before the first snapshot: a read in flight may have listed the sources before it.
+    invalidate: () => {
+      generation += 1
+    },
+  }
+}
+
+/** Insert as the newest entry, dropping the oldest past the bound. */
+function keep<TValue>(map: Map<string, TValue>, key: string, value: TValue): void {
+  map.set(key, value)
+  if (map.size <= EVENT_ORIGINS_KEPT) return
+
+  const oldest = map.keys().next().value
+  if (oldest !== undefined) map.delete(oldest)
+}

package/src/security/dataset-version-markings.ts

@@ -0,0 +1,163 @@
+import type { DatasetDefinition } from "../datasets/types"
+import { isUnknownLineage, lineageColumn, versionLineage } from "../lake-storage/lineage"
+import type { DatasetColumnRef, DatasetVersion, LakeStorage } from "../lake-storage/types"
+import { MarkingSets } from "./marking-sets"
+
+/** Markings of each column of one dataset version. A column without markings has an empty set. */
+export type VersionColumnMarkings = ReadonlyMap<string, ReadonlySet<string>>
+
+export interface VersionClassification {
+  readonly columns: VersionColumnMarkings
+  /**
+   * Why part of the version was classified conservatively, with every marking it could hold.
+   * Absent when every column was classified from its lineage.
+   */
+  readonly conservative?: string
+}
+
+/**
+ * Markings each version of a dataset carries.
+ *
+ * Markings are never stored. A column carries the markings its registered dataset declares, and
+ * those every column its lineage lists declares. The version records that lineage resolved across
+ * every upstream step and every row it keeps (`DatasetVersion.lineage`), so a version keeps the
+ * classification of the code that produced it, and classifying it reads no other version.
+ *
+ * Wherever the record does not say what a column holds (no lineage on a pipeline version, or on
+ * any version of a provider that does not declare `recordsLineage`, an unknown history, a column
+ * it does not list, a dataset no longer registered) the column is classified with every marking
+ * the project declares.
+ */
+export interface DatasetVersionMarkings {
+  classify(version: DatasetVersion): VersionClassification
+  /**
+   * Every marking declared on a registered column. Every classification, conservative ones
+   * included, is drawn from it, so a reader cleared for all of it needs no classification. Empty
+   * when the project declares no column markings.
+   */
+  everyMarking(): ReadonlySet<string>
+}
+
+export interface DatasetVersionMarkingsOptions {
+  readonly datasets: { list(): readonly DatasetDefinition[] }
+  /** The lake the versions are read from: whether it records lineage on every version. */
+  readonly lakeStorage: Pick<LakeStorage, "recordsLineage">
+}
+
+export function createDatasetVersionMarkings(
+  options: DatasetVersionMarkingsOptions
+): DatasetVersionMarkings {
+  const sets = new MarkingSets()
+
+  // Built on first use: a host that never classifies a version pays nothing for it.
+  let declared: ReadonlyMap<string, ReadonlyMap<string, ReadonlySet<string>>> | undefined
+  const declaredByDataset = () =>
+    (declared ??= new Map(
+      options.datasets
+        .list()
+        .map((dataset) => [
+          dataset.id,
+          new Map(dataset.schema.columns.map((column) => [column.name, sets.of(column.markings)])),
+        ])
+    ))
+
+  /**
+   * Every marking a registered column declares: what a column may hold where its record does not
+   * say. The pipelines that wrote it may have read datasets the current code no longer reads, so
+   * nothing narrower is safe.
+   */
+  let every: ReadonlySet<string> | undefined
+  const everyMarking = () =>
+    (every ??= sets.union(
+      [...declaredByDataset().values()].flatMap((columns) => [...columns.values()])
+    ))
+
+  const classify = (version: DatasetVersion): VersionClassification => {
+    const subject = `version '${version.versionId}' of '${version.datasetId}'`
+    const own = declaredByDataset().get(version.datasetId)
+
+    // No definition says what its columns declare: they may hold anything a column declares.
+    if (!own) return conservatively(version, `dataset '${version.datasetId}' is not registered`)
+
+    const lineage = versionLineage(version, options.lakeStorage)
+    if (isUnknownLineage(lineage)) {
+      // A reason core derived names this version. A recorded one names where the lineage was
+      // lost: upstream, or, for a provider that could not decode it, this version's own record.
+      return conservatively(
+        version,
+        lineage === version.lineage
+          ? `the lineage of ${subject} is unknown: ${lineage.unknown}`
+          : lineage.unknown
+      )
+    }
+
+    let conservative: string | undefined
+    const markingsOf = (ref: DatasetColumnRef): ReadonlySet<string> | undefined => {
+      const columns = declaredByDataset().get(ref.datasetId)
+      if (columns) return columns.get(ref.column)
+      conservative ??= `dataset '${ref.datasetId}' is not registered`
+      return everyMarking()
+    }
+
+    const rows = sets.union(lineage.rows.map(markingsOf))
+    const columns = new Map(
+      version.schema.columns.map(({ name }) => {
+        const values = lineageColumn(lineage, name)
+        if (!values) {
+          conservative ??= `${subject} recorded no lineage for column '${name}'`
+          return [name, everyMarking()]
+        }
+
+        return [name, sets.union([own.get(name), rows, ...values.map(markingsOf)])]
+      })
+    )
+
+    return { columns, ...(conservative === undefined ? {} : { conservative }) }
+  }
+
+  const conservatively = (version: DatasetVersion, reason: string): VersionClassification => {
+    const markings = everyMarking()
+    return {
+      columns: new Map(version.schema.columns.map(({ name }) => [name, markings])),
+      conservative: reason,
+    }
+  }
+
+  return { classify, everyMarking }
+}
+
+// ── Host capability ─────────────────────────────────────────
+
+const datasetVersionMarkingsKey: unique symbol = Symbol("sixb.datasetVersionMarkings")
+
+interface DatasetVersionMarkingsOwner {
+  readonly [datasetVersionMarkingsKey]?: DatasetVersionMarkings
+}
+
+/** One classifier per host, so every reader shares the markings it reads from the definitions. */
+export function registerDatasetVersionMarkings(
+  owner: object,
+  markings: DatasetVersionMarkings
+): void {
+  const registered = (owner as DatasetVersionMarkingsOwner)[datasetVersionMarkingsKey]
+  if (registered && registered !== markings) {
+    throw new Error("[Sixb] Dataset version markings are already registered for this owner.")
+  }
+
+  Object.defineProperty(owner, datasetVersionMarkingsKey, {
+    configurable: false,
+    enumerable: false,
+    value: markings,
+    writable: false,
+  })
+}
+
+export function getDatasetVersionMarkings(owner: object): DatasetVersionMarkings {
+  const markings = (owner as DatasetVersionMarkingsOwner)[datasetVersionMarkingsKey]
+  if (markings) return markings
+  throw new Error("[Sixb] Dataset version markings are not registered for this runtime.")
+}
+
+export function shareDatasetVersionMarkings(source: object, target: object): void {
+  registerDatasetVersionMarkings(target, getDatasetVersionMarkings(source))
+}