@sixb/core 0.1.20 → 0.1.22

package/src/security/applied-markings-contribution.ts

@@ -0,0 +1,277 @@
+import type { ProjectionEventOrigin } from "../events/envelope"
+import type { DatasetVersion } from "../lake-storage/types"
+import type { OntologyDefinitionCatalog } from "../ontology/registry"
+import type { PipelineDefinition } from "../pipelines/types"
+import type { LinkProjectionDefinition, ObjectProjectionDefinition } from "../projections/types"
+import type { ActiveOntologySource } from "../storage/ontology"
+import { type AppliedContribution, NO_CONTRIBUTION } from "./applied-markings-snapshot"
+import type { VersionClassification } from "./dataset-version-markings"
+import { registeredSteps } from "./marking-propagation"
+import { type ProjectionKeyTarget, projectionSurplus } from "./projection-markings"
+
+export type SourceProjection = ObjectProjectionDefinition | LinkProjectionDefinition
+
+/** What reading a projection's applied version from the lake gave. */
+export type AppliedVersionRead =
+  | {
+      readonly kind: "read"
+      readonly version: DatasetVersion
+      readonly classification: VersionClassification
+    }
+  | { readonly kind: "missing" }
+  | { readonly kind: "failed"; readonly error: unknown }
+
+export interface AppliedContributionContext {
+  readonly ontology: Pick<OntologyDefinitionCatalog, "getObjectTypeById">
+  /** The revision of a projection's current definition. */
+  readonly revisionOf: (projectionId: string) => string
+  /** What a version that cannot be read may carry. */
+  readonly everyMarking: ReadonlySet<string>
+  /** The pipeline that writes a dataset in the current code, for messages. */
+  readonly pipelineOf: (datasetId: string) => string | undefined
+}
+
+/**
+ * What one projection's applied version adds to the code's markings, with the notice that explains
+ * it. It reads nothing: the caller reads and classifies the version.
+ *
+ * - A version that could not be read seals the projection's target with every marking.
+ * - Objects written by an earlier definition, which may have mapped any column, seal it with every
+ *   marking of the version.
+ * - Otherwise a property keeps the markings its column carries beyond its declaration, and a key
+ *   column that carries markings seals what it keys.
+ */
+export function appliedContribution(
+  projection: SourceProjection,
+  source: ActiveOntologySource,
+  read: AppliedVersionRead,
+  context: AppliedContributionContext
+): AppliedContribution {
+  const { datasetId, versionId } = source.datasetVersion
+  const subject = `version '${versionId}' of '${datasetId}'`
+  const target = projectionTarget(projection)
+  const hint = rerunHint(datasetId, read.kind === "read" ? read.version : undefined, context)
+  const every = [...context.everyMarking].sort()
+
+  if (read.kind === "failed") {
+    return sealed(projection.id, target, every, hint, {
+      reason: `the ${subject} it applied, which could not be read from the lake,`,
+      detail: `applied ${subject}, which could not be read from the lake: ${sentence(errorMessage(read.error))}`,
+    })
+  }
+
+  if (read.kind === "missing") {
+    return sealed(projection.id, target, every, hint, {
+      reason: `the ${subject} it applied, which the lake no longer has,`,
+      detail: `applied ${subject}, which the lake no longer has.`,
+    })
+  }
+
+  const { columns, conservative } = read.classification
+  const because =
+    conservative === undefined
+      ? ""
+      : ` The version was classified with every marking its sources could hold, because ${conservative}.`
+
+  if (
+    datasetId !== projection.datasetId ||
+    source.projectionRevision !== context.revisionOf(projection.id)
+  ) {
+    const markings = appliedVersionMarkings(read, context.everyMarking)
+    return sealed(projection.id, target, markings, hint, {
+      reason: `the ${subject} it applied under an earlier definition of the projection`,
+      detail: `applied ${subject} under an earlier definition of the projection, and that version carries ${markingList(markings)}.${because}`,
+    })
+  }
+
+  const markingsOf = (column: string) => [...(columns.get(column) ?? [])].sort()
+  const surplus = projectionSurplus(projection, context.ontology, markingsOf)
+  if (surplus.properties.length === 0 && surplus.keys.length === 0) return NO_CONTRIBUTION
+
+  const column = (name: string) => `'${datasetId}.${name}'`
+  const seals = surplus.keys.map((key) => ({
+    target: key.target,
+    seal: {
+      projectionId: projection.id,
+      markings: key.markings,
+      reason: `in the version '${versionId}' it applied, column ${column(key.column)}, which ${key.role},`,
+    },
+  }))
+
+  const details = [
+    ...surplus.keys.map(
+      (key) => `column ${column(key.column)} ${key.role} and carries ${markingList(key.markings)}`
+    ),
+    ...surplus.properties.map(
+      (property) =>
+        `property '${property.objectTypeId}.${property.propertyId}' is projected from column ${column(property.column)}, which carries ${markingList(property.markings)}`
+    ),
+  ]
+
+  const consequences: string[] = []
+  if (seals.length > 0) {
+    const markings = [...new Set(seals.flatMap(({ seal }) => seal.markings))].sort()
+    consequences.push(
+      `${describeTargets(seals.map(({ target }) => target))} cannot be read without clearance for ${markingList(markings)}`
+    )
+  }
+
+  if (surplus.properties.length > 0) {
+    const names = surplus.properties.map((p) => `'${p.objectTypeId}.${p.propertyId}'`)
+    const verb = surplus.properties.length === 1 ? "keeps" : "keep"
+    consequences.push(`${describeNames(names)} ${verb} these markings`)
+  }
+
+  return {
+    properties: surplus.properties.map(({ objectTypeId, propertyId, markings }) => ({
+      objectTypeId,
+      propertyId,
+      markings,
+    })),
+    seals,
+    notice: {
+      projectionId: projection.id,
+      sealed: seals.length > 0,
+      message: `[Sixb] Projection '${projection.id}' applied ${subject}, in which ${details.join("; ")}.${because} ${capitalize(consequences.join(", and "))} until the projection applies a version the code allows. ${hint}`,
+    },
+  }
+}
+
+/** Every marking the version carries in any column; every marking when it could not be read. */
+export function appliedVersionMarkings(
+  read: AppliedVersionRead,
+  everyMarking: ReadonlySet<string>
+): readonly string[] {
+  if (read.kind !== "read") return [...everyMarking].sort()
+
+  const { columns } = read.classification
+  return [...new Set([...columns.values()].flatMap((set) => [...set]))].sort()
+}
+
+/** An object event a projection emitted: the version its values came from, and its type. */
+export type EventOriginRef = Pick<
+  ProjectionEventOrigin,
+  "projectionId" | "datasetId" | "datasetVersionId"
+> & { readonly objectTypeId: string }
+
+/**
+ * The markings an object event's origin version carries on each property its projection maps.
+ * When the projection now maps another dataset or type, or the version cannot be read, every
+ * property but the primary one carries every marking it could, as objects written under an
+ * earlier definition do.
+ */
+export function eventOriginMarkings(
+  projection: ObjectProjectionDefinition,
+  origin: EventOriginRef,
+  read: AppliedVersionRead,
+  context: Pick<AppliedContributionContext, "ontology" | "everyMarking">
+): AppliedContribution["properties"] {
+  const { objectTypeId } = origin
+
+  if (
+    read.kind === "read" &&
+    projection.datasetId === origin.datasetId &&
+    projection.objectTypeId === objectTypeId
+  ) {
+    const { columns } = read.classification
+    return Object.entries(projection.properties).flatMap(([propertyId, column]) => {
+      const markings = [...(columns.get(column) ?? [])].sort()
+      return markings.length === 0 ? [] : [{ objectTypeId, propertyId, markings }]
+    })
+  }
+
+  const markings = appliedVersionMarkings(read, context.everyMarking)
+  if (markings.length === 0) return []
+
+  return (context.ontology.getObjectTypeById(objectTypeId)?.properties ?? [])
+    .filter((property) => !property.primary)
+    .map((property) => ({ objectTypeId, propertyId: property.id, markings }))
+}
+
+/** The pipeline that writes each dataset in the current code. */
+export function producingPipelines(
+  pipelines: readonly PipelineDefinition[]
+): (datasetId: string) => string | undefined {
+  const byDataset = new Map<string, string>()
+  for (const { pipelineId, step } of registeredSteps(pipelines)) {
+    if (!byDataset.has(step.output.id)) byDataset.set(step.output.id, pipelineId)
+  }
+
+  return (datasetId) => byDataset.get(datasetId)
+}
+
+/** Seal the projection's whole target: its object type, or its link. */
+function sealed(
+  projectionId: string,
+  target: ProjectionKeyTarget,
+  markings: readonly string[],
+  hint: string,
+  text: { readonly reason: string; readonly detail: string }
+): AppliedContribution {
+  if (markings.length === 0) return NO_CONTRIBUTION
+  return {
+    properties: [],
+    seals: [{ target, seal: { projectionId, markings, reason: text.reason } }],
+    notice: {
+      projectionId,
+      sealed: true,
+      message: `[Sixb] Projection '${projectionId}' ${text.detail} ${describeTargets([target])} cannot be read without clearance for ${markingList(markings)} until the projection applies a version the code allows. ${hint}`,
+    },
+  }
+}
+
+function projectionTarget(projection: SourceProjection): ProjectionKeyTarget {
+  return projection._tag === "ObjectProjectionDefinition"
+    ? { kind: "type", objectTypeId: projection.objectTypeId }
+    : {
+        kind: "link",
+        objectTypeId: projection.sourceObjectTypeId,
+        linkId: projection.linkId,
+        targetObjectTypeId: projection.targetObjectTypeId,
+      }
+}
+
+function rerunHint(
+  datasetId: string,
+  version: DatasetVersion | undefined,
+  context: AppliedContributionContext
+): string {
+  const pipelineId =
+    context.pipelineOf(datasetId) ??
+    (version?.producer?.kind === "pipeline" ? version.producer.id : undefined)
+  return pipelineId === undefined
+    ? `Write a new version of '${datasetId}' to replace it.`
+    : `Rerun pipeline '${pipelineId}' to replace it.`
+}
+
+function describeTargets(targets: readonly ProjectionKeyTarget[]): string {
+  return describeNames(
+    targets.map((target) =>
+      target.kind === "type"
+        ? `'${target.objectTypeId}'`
+        : `link '${target.objectTypeId}.${target.linkId}'`
+    )
+  )
+}
+
+function describeNames(names: readonly string[]): string {
+  const unique = [...new Set(names)]
+  const last = unique.pop() ?? ""
+  return unique.length === 0 ? last : `${unique.join(", ")} and ${last}`
+}
+
+function capitalize(text: string): string {
+  return text.charAt(0).toUpperCase() + text.slice(1)
+}
+
+export function sentence(text: string): string {
+  return text.endsWith(".") ? text : `${text}.`
+}
+
+function markingList(markings: readonly string[]): string {
+  return `[${markings.join(", ")}]`
+}
+
+export function errorMessage(error: unknown): string {
+  return error instanceof Error ? error.message : String(error)
+}

package/src/security/applied-markings-host.ts

@@ -0,0 +1,41 @@
+import type { AppliedMarkings } from "./applied-markings"
+import { errorMessage } from "./applied-markings-contribution"
+
+const appliedMarkingsKey: unique symbol = Symbol("sixb.appliedMarkings")
+
+interface AppliedMarkingsOwner {
+  readonly [appliedMarkingsKey]?: AppliedMarkings
+}
+
+/** One snapshot per host, so every reader shares it and its resolved clearances. */
+export function registerAppliedMarkings(owner: object, markings: AppliedMarkings): void {
+  const registered = (owner as AppliedMarkingsOwner)[appliedMarkingsKey]
+  if (registered && registered !== markings) {
+    throw new Error("[Sixb] Applied markings are already registered for this owner.")
+  }
+
+  Object.defineProperty(owner, appliedMarkingsKey, {
+    configurable: false,
+    enumerable: false,
+    value: markings,
+    writable: false,
+  })
+}
+
+export function getAppliedMarkings(owner: object): AppliedMarkings {
+  const markings = (owner as AppliedMarkingsOwner)[appliedMarkingsKey]
+  if (markings) return markings
+  throw new Error("[Sixb] Applied markings are not registered for this runtime.")
+}
+
+/**
+ * Read the applied versions ahead of the first request that needs them. A failure is only
+ * reported here: that request reads them again, and fails closed if it cannot.
+ */
+export function warmAppliedMarkings(owner: object): void {
+  getAppliedMarkings(owner)
+    .current()
+    .catch((error: unknown) => {
+      console.warn(errorMessage(error))
+    })
+}

package/src/security/applied-markings-snapshot.ts

@@ -0,0 +1,235 @@
+import type { ProjectionKeyTarget } from "./projection-markings"
+
+/** Why readers lacking some markings cannot read an object type or a link. */
+export interface AppliedSeal {
+  readonly projectionId: string
+  readonly markings: ReadonlySet<string>
+  /** What carries the marking, completed by "carries marking 'm'" in messages. */
+  readonly reason: string
+}
+
+export interface AppliedLinkSeals {
+  readonly objectTypeId: string
+  readonly linkId: string
+  readonly targetObjectTypeId: string
+  readonly seals: readonly AppliedSeal[]
+}
+
+export interface AppliedMarkingsNotice {
+  readonly projectionId: string
+  /** Whether the projection makes an object type or a link unreadable, not only properties. */
+  readonly sealed: boolean
+  readonly message: string
+}
+
+/**
+ * What the applied versions add to the ontology's markings.
+ *
+ * Every reader of a host shares one snapshot and the clearances resolved for it, so a snapshot is
+ * immutable all the way down: its maps and sets refuse writes.
+ */
+export interface AppliedMarkingsSnapshot {
+  /** Surplus markings by object type, then property. */
+  readonly properties: ReadonlyMap<string, ReadonlyMap<string, ReadonlySet<string>>>
+  /** Object types readers lacking a seal's markings cannot read. */
+  readonly types: ReadonlyMap<string, readonly AppliedSeal[]>
+  /** Links, by {@link appliedLinkKey}, readers lacking a seal's markings cannot read. */
+  readonly links: ReadonlyMap<string, AppliedLinkSeals>
+  /** Every marking above: a reader cleared for all of them reads what the code allows. */
+  readonly markings: ReadonlySet<string>
+  /** One per projection whose applied version adds markings, for diagnostics. */
+  readonly notices: readonly AppliedMarkingsNotice[]
+}
+
+export function appliedLinkKey(objectTypeId: string, linkId: string): string {
+  return `${objectTypeId}\0${linkId}`
+}
+
+/** What one applied version adds, as plain data so that equal results serialize equally. */
+export interface AppliedContribution {
+  readonly properties: readonly {
+    readonly objectTypeId: string
+    readonly propertyId: string
+    readonly markings: readonly string[]
+  }[]
+  readonly seals: readonly {
+    readonly target: ProjectionKeyTarget
+    readonly seal: {
+      readonly projectionId: string
+      readonly markings: readonly string[]
+      readonly reason: string
+    }
+  }[]
+  readonly notice?: AppliedMarkingsNotice
+}
+
+/** An applied version that adds nothing. */
+export const NO_CONTRIBUTION: AppliedContribution = Object.freeze({
+  properties: Object.freeze([]),
+  seals: Object.freeze([]),
+})
+
+export const NO_APPLIED_MARKINGS: AppliedMarkingsSnapshot = Object.freeze({
+  properties: readonlyMap(new Map()),
+  types: readonlyMap(new Map()),
+  links: readonlyMap(new Map()),
+  markings: readonlySet(new Set<string>()),
+  notices: Object.freeze([]),
+})
+
+/** One snapshot from the contributions of every applied version, in projection order. */
+export function assembleAppliedMarkings(
+  contributions: readonly AppliedContribution[]
+): AppliedMarkingsSnapshot {
+  if (contributions.length === 0) return NO_APPLIED_MARKINGS
+
+  const properties = new Map<string, Map<string, Set<string>>>()
+  const types = new Map<string, AppliedSeal[]>()
+  const links = new Map<string, AppliedLinkSeals & { seals: AppliedSeal[] }>()
+  const markings = new Set<string>()
+  const notices: AppliedMarkingsNotice[] = []
+
+  for (const contribution of contributions) {
+    for (const { objectTypeId, propertyId, markings: added } of contribution.properties) {
+      const byProperty = properties.get(objectTypeId) ?? new Map<string, Set<string>>()
+      properties.set(objectTypeId, byProperty)
+
+      const set = byProperty.get(propertyId) ?? new Set<string>()
+      byProperty.set(propertyId, set)
+
+      for (const markingId of added) {
+        set.add(markingId)
+        markings.add(markingId)
+      }
+    }
+
+    for (const { target, seal: data } of contribution.seals) {
+      const seal: AppliedSeal = Object.freeze({
+        ...data,
+        markings: readonlySet(new Set(data.markings)),
+      })
+
+      for (const markingId of data.markings) markings.add(markingId)
+
+      if (target.kind === "type") {
+        types.set(target.objectTypeId, [...(types.get(target.objectTypeId) ?? []), seal])
+        continue
+      }
+
+      const key = appliedLinkKey(target.objectTypeId, target.linkId)
+      const existing = links.get(key)
+      if (existing) existing.seals.push(seal)
+      else {
+        links.set(key, {
+          objectTypeId: target.objectTypeId,
+          linkId: target.linkId,
+          targetObjectTypeId: target.targetObjectTypeId,
+          seals: [seal],
+        })
+      }
+    }
+
+    if (contribution.notice) notices.push(Object.freeze({ ...contribution.notice }))
+  }
+
+  return Object.freeze({
+    properties: readonlyMap(
+      new Map(
+        [...properties].map(([objectTypeId, byProperty]) => [
+          objectTypeId,
+          readonlyMap(
+            new Map([...byProperty].map(([propertyId, set]) => [propertyId, readonlySet(set)]))
+          ),
+        ])
+      )
+    ),
+    types: readonlyMap(
+      new Map([...types].map(([objectTypeId, seals]) => [objectTypeId, Object.freeze(seals)]))
+    ),
+    links: readonlyMap(
+      new Map(
+        [...links].map(([key, link]) => [
+          key,
+          Object.freeze({ ...link, seals: Object.freeze(link.seals) }),
+        ])
+      )
+    ),
+    markings: readonlySet(markings),
+    notices: Object.freeze(notices),
+  })
+}
+
+/**
+ * The snapshot with these properties' markings added: what an object event reads under, given the
+ * markings its origin version carries on them.
+ */
+export function withPropertyMarkings(
+  snapshot: AppliedMarkingsSnapshot,
+  added: AppliedContribution["properties"]
+): AppliedMarkingsSnapshot {
+  if (added.length === 0) return snapshot
+
+  const properties = new Map<string, Map<string, Set<string>>>(
+    [...snapshot.properties].map(([objectTypeId, byProperty]) => [
+      objectTypeId,
+      new Map([...byProperty].map(([propertyId, set]) => [propertyId, new Set(set)])),
+    ])
+  )
+
+  const markings = new Set(snapshot.markings)
+  for (const { objectTypeId, propertyId, markings: ids } of added) {
+    const byProperty = properties.get(objectTypeId) ?? new Map<string, Set<string>>()
+    properties.set(objectTypeId, byProperty)
+
+    const set = byProperty.get(propertyId) ?? new Set<string>()
+    byProperty.set(propertyId, set)
+
+    for (const markingId of ids) {
+      set.add(markingId)
+      markings.add(markingId)
+    }
+  }
+
+  return Object.freeze({
+    ...snapshot,
+    properties: readonlyMap(
+      new Map(
+        [...properties].map(([objectTypeId, byProperty]) => [
+          objectTypeId,
+          readonlyMap(
+            new Map([...byProperty].map(([propertyId, set]) => [propertyId, readonlySet(set)]))
+          ),
+        ])
+      )
+    ),
+    markings: readonlySet(markings),
+  })
+}
+
+/**
+ * A map that refuses writes. `Object.freeze` alone leaves a map's entries writable through its
+ * methods, so they are shadowed by ones that throw, and then frozen.
+ */
+function readonlyMap<TKey, TValue>(map: Map<TKey, TValue>): ReadonlyMap<TKey, TValue> {
+  return Object.freeze(
+    Object.defineProperties(map, {
+      set: { value: refuseWrite },
+      delete: { value: refuseWrite },
+      clear: { value: refuseWrite },
+    })
+  )
+}
+
+function readonlySet<TValue>(set: Set<TValue>): ReadonlySet<TValue> {
+  return Object.freeze(
+    Object.defineProperties(set, {
+      add: { value: refuseWrite },
+      delete: { value: refuseWrite },
+      clear: { value: refuseWrite },
+    })
+  )
+}
+
+function refuseWrite(): never {
+  throw new TypeError("[Sixb] Applied markings are immutable.")
+}