@sixb/core 0.1.19 → 0.1.21

package/src/datasets/execution.ts

@@ -11,6 +11,7 @@ import type { DatasetMergeCommitResult } from "../lake-storage/merge"
 import { getDatasetPrimaryKeyColumns } from "../lake-storage/merge-validation"
 import type { DatasetRow, DatasetVersion, LakeStorage } from "../lake-storage/types"
 import type { SixbRuntimeContext } from "../runtime/types"
+import type { DatasetVersionMarkings } from "../security/dataset-version-markings"
 import { missingClearanceRedactions } from "../security/redactions"
 import type { Redactions } from "../security/types"
 import type { MergeChange } from "./changes"
@@ -42,7 +43,7 @@ export interface DatasetRows {
   /** Columns each row holds. Redacted columns are left out. */
   readonly columns: readonly string[]
   /**
-   * Every column of the dataset the reader cannot read, whether or not it was requested. Absent
+   * Every column of the version the reader cannot read, whether or not it was requested. Absent
    * when the reader can read every column.
    */
   readonly redactions?: Redactions
@@ -65,7 +66,8 @@ export function createDatasetsRuntime(
   execution: ExecutionContext,
   source: Pick<DatasetsRuntime, "list" | "getById">,
   lakeStorage: LakeStorage,
-  blobStorage: Pick<BlobStorage, "stat">
+  blobStorage: Pick<BlobStorage, "stat">,
+  versionMarkings: DatasetVersionMarkings
 ): DatasetsRuntime {
   const authority = resolveRuntimeAuthorizationForProject(runtime)
   const allowed = (datasetId: string) =>
@@ -91,7 +93,11 @@ export function createDatasetsRuntime(
           : await lakeStorage.getVersion(registered.id, input.versionId)
       if (!version) return null
 
-      const redacted = redactedColumns(registered, assertRuntimeAuthorizationBound(runtime))
+      const redacted = redactedColumns(
+        version,
+        versionMarkings,
+        assertRuntimeAuthorizationBound(runtime)
+      )
       const selected = selectColumns(version, input.columns)
       const columns = selected.filter((column) => !redacted.has(column))
       const read = (readColumns: readonly string[] | undefined) =>
@@ -181,20 +187,28 @@ function selectColumns(version: DatasetVersion, requested: readonly string[] | u
 }
 
 /**
- * Columns the reader cannot read. Trusted code reads every column, a principal reads what its roles
- * clear, and any other authority is cleared for nothing.
+ * Columns of the version the reader cannot read. Trusted code reads every column, a principal reads
+ * what its roles clear, and any other authority is cleared for nothing.
+ *
+ * No version carries a marking the project does not declare on a column, so a reader cleared for
+ * every declared marking (all readers, when none is declared) reads every column without
+ * classifying the version.
  */
 function redactedColumns(
-  dataset: DatasetDefinition,
+  version: DatasetVersion,
+  versionMarkings: DatasetVersionMarkings,
   authority: ReturnType<typeof assertRuntimeAuthorizationBound>
 ): ReadonlySet<string> {
   if (authority.type === "unrestricted") return new Set()
   const clearances =
     authority.type === "principal" ? (authority.context.clearances ?? new Set()) : new Set()
+  if ([...versionMarkings.everyMarking()].every((id) => clearances.has(id))) return new Set()
+
+  const { columns } = versionMarkings.classify(version)
   return new Set(
-    dataset.schema.columns
-      .filter((column) => column.markings?.some((markingId) => !clearances.has(markingId)))
-      .map((column) => column.name)
+    [...columns]
+      .filter(([, columnMarkings]) => [...columnMarkings].some((id) => !clearances.has(id)))
+      .map(([column]) => column)
   )
 }
 

package/src/deploy/services.ts

@@ -10,7 +10,6 @@ import type {
 
 const SERVICES = {
   api: "http",
-  atlas: "http",
   app: "http",
   orchestrator: "singleton",
   scheduler: "singleton",

package/src/deploy/types.ts

@@ -29,7 +29,7 @@ export interface DeployConfig {
 export type DeployEnv = Readonly<Record<string, string>>
 
 /** The services a browser or client reaches through a public domain. */
-export type DeployHttpServiceName = "api" | "atlas" | "app"
+export type DeployHttpServiceName = "api" | "app"
 
 /** The services that must run as exactly one process: a second would duplicate their work. */
 export type DeploySingletonServiceName = "orchestrator" | "scheduler" | "rules"
@@ -40,7 +40,6 @@ export type DeployWorkerType = "sync" | "agent" | "pipeline" | "projection" | "w
 
 export interface DeployServicesConfig {
   readonly api?: boolean | DeployHttpServiceConfig
-  readonly atlas?: boolean | DeployHttpServiceConfig
   readonly app?: boolean | DeployHttpServiceConfig
   readonly orchestrator?: boolean | DeploySingletonServiceConfig
   readonly scheduler?: boolean | DeploySingletonServiceConfig

package/src/deploy/validation.ts

@@ -55,7 +55,9 @@ function target(value: unknown): void {
 
 function services(value: unknown): void {
   const config = record(value, "services")
-  knownKeys(config, DEPLOY_SERVICES, "services")
+  knownKeys(config, DEPLOY_SERVICES, "services", {
+    atlas: "Atlas's pages are part of the app now; remove the setting and configure `app`.",
+  })
 
   for (const service of DEPLOY_SERVICES) {
     const override = config[service]

package/src/errors/internal.ts

@@ -13,6 +13,7 @@ const SIXB_ERROR_CODE_SET: ReadonlySet<string> = new Set(SIXB_ERROR_CODES)
 type SixbErrorCodeTuple = readonly [SixbErrorCode, ...SixbErrorCode[]]
 
 export { SIXB_ERROR_CODES }
+export { withFailureMessage } from "./failure-message"
 
 export interface SixbErrorOptions {
   readonly cause?: unknown

package/src/events/execution.ts

@@ -1,6 +1,13 @@
 import { assertPrivileged, canViewEvent } from "../authorization"
 import { resolveRuntimeAuthorizationForProject } from "../execution/authorization"
-import { redactDomainEvent, resolvePropertyClearance } from "../objects/property-clearance"
+import {
+  type AuthorizedObjectReader,
+  readerClearance,
+  readerEventClearance,
+} from "../execution/authorized-object-reader"
+import type { EventSources } from "../execution/object-reader-clearance"
+import { redactDomainEvent } from "../objects/clearance-redaction"
+import type { PropertyClearance } from "../objects/property-clearance"
 import type { SixbRuntimeContext } from "../runtime/types"
 import type {
   EventsAppendInput,
@@ -11,12 +18,22 @@ import type {
 import type { StoredDomainEvent } from "./types"
 
 export interface EventsRuntime {
-  /** The event as this execution may read it: hidden events are `undefined`, others redacted. */
-  readable(event: StoredDomainEvent): StoredDomainEvent | undefined
+  /**
+   * The events this execution may read, in order: hidden events are dropped, the others redacted.
+   * Markings apply as they are now, with those of the versions an object event's values came from:
+   * the version a projection's event applied, and every marking a column declares for projected
+   * values that came from earlier versions.
+   */
+  visible(events: readonly StoredDomainEvent[]): Promise<readonly StoredDomainEvent[]>
   append(input: EventsAppendInput): Promise<readonly StoredDomainEvent[]>
   emit(input: EventsAppendInput, options: EventsEmitOptions): Promise<void>
   read(input?: EventsReadInput): Promise<readonly StoredDomainEvent[]>
   latestCursor(): Promise<string | undefined>
+  /**
+   * Deliver the events this execution may read as they are appended. A batch whose markings cannot
+   * be enforced, because the versions projections applied cannot be read, is dropped with a
+   * warning: read it again with `read({ afterCursor })`.
+   */
   subscribe(
     input: EventsSubscribeInput,
     handler: (events: readonly StoredDomainEvent[]) => unknown
@@ -25,27 +42,34 @@ export interface EventsRuntime {
 
 export function createEventsRuntime(runtime: SixbRuntimeContext): EventsRuntime {
   const authority = resolveRuntimeAuthorizationForProject(runtime)
-  const clearance =
-    authority.type === "principal"
-      ? resolvePropertyClearance(runtime.ontology, authority.context.clearances ?? new Set())
-      : undefined
-  const readable = (event: StoredDomainEvent): StoredDomainEvent | undefined => {
+  const visible = async (
+    events: readonly StoredDomainEvent[]
+  ): Promise<readonly StoredDomainEvent[]> => {
     switch (authority.type) {
       case "denied":
       case "delegated":
-        return undefined
-      case "principal":
-        if (!canViewEvent(authority.context, event)) return undefined
-        return clearance ? redactDomainEvent(event, clearance) : event
+        return []
       case "unrestricted":
-        return event
+        return events
+      case "principal": {
+        const viewable = events.filter((event) => canViewEvent(authority.context, event))
+        if (viewable.length === 0) return viewable
+
+        const clearance = await readerClearance(runtime.objectReader)
+        const clearanceFor = await eventClearances(runtime.objectReader, viewable, clearance)
+
+        return viewable.flatMap((event) => {
+          if (clearance && readsUnreadable(event, clearance)) return []
+
+          const redacting = clearanceFor(event)
+          return [redacting ? redactDomainEvent(event, redacting) : event]
+        })
+      }
     }
   }
-  const visibleEvents = (events: readonly StoredDomainEvent[]): readonly StoredDomainEvent[] =>
-    events.flatMap((event) => readable(event) ?? [])
 
   return {
-    readable,
+    visible,
     append: (input) => {
       assertPrivileged(runtime, "events.append")
       return runtime.events.append(input)
@@ -63,8 +87,7 @@ export function createEventsRuntime(runtime: SixbRuntimeContext): EventsRuntime
         case "unrestricted":
           break
       }
-      const events = await runtime.events.read(input)
-      return visibleEvents(events)
+      return visible(await runtime.events.read(input))
     },
     latestCursor: () => {
       switch (authority.type) {
@@ -85,10 +108,100 @@ export function createEventsRuntime(runtime: SixbRuntimeContext): EventsRuntime
         case "unrestricted":
           break
       }
-      return runtime.events.subscribe(input, (events) => {
-        const visible = visibleEvents(events)
-        if (visible.length > 0) return handler(visible)
+      return runtime.events.subscribe(input, async (events) => {
+        let shown: readonly StoredDomainEvent[]
+        try {
+          shown = await visible(events)
+        } catch (error) {
+          // Fail closed. Delivery is live and keeps no progress, and a handler error is not a
+          // negative acknowledgement, so the batch is dropped and reported rather than retried.
+          const message = error instanceof Error ? error.message : String(error)
+          console.warn(
+            `[Sixb] Dropped ${events.length} event(s) from cursor '${events[0]?.cursor}' of a subscription, whose markings could not be enforced: ${message}`
+          )
+          return
+        }
+
+        if (shown.length > 0) return handler(shown)
       })
     },
   }
 }
+
+/**
+ * The clearance each event is redacted with, resolved once per kind of event for the whole batch.
+ * An event that holds only values the current markings describe uses the reader's clearance.
+ */
+async function eventClearances(
+  reader: AuthorizedObjectReader,
+  events: readonly StoredDomainEvent[],
+  clearance: PropertyClearance | undefined
+): Promise<(event: StoredDomainEvent) => PropertyClearance | undefined> {
+  const kinds = new Map<string, EventSources>()
+  for (const event of events) {
+    const kind = eventSources(event)
+    if (kind) kinds.set(kind.key, kind.sources)
+  }
+
+  const resolved = await Promise.all(
+    [...kinds].map(
+      async ([key, sources]) => [key, await readerEventClearance(reader, sources)] as const
+    )
+  )
+  const byKind = new Map(resolved)
+
+  return (event) => {
+    const kind = eventSources(event)
+    return kind ? byKind.get(kind.key) : clearance
+  }
+}
+
+/**
+ * Where an object event's values came from, when the current markings do not describe them all.
+ *
+ * - A projection's event holds the values of the version it applied.
+ * - An update or a deletion also holds the values it replaced. An action's or a direct write's
+ *   record of an object holds the values projections had applied then. Both came from versions
+ *   applied before the event, which it does not name: they read like a version that cannot be read.
+ * - Only an object created by an action or a direct write holds nothing but its own values.
+ */
+function eventSources(
+  event: StoredDomainEvent
+): { readonly key: string; readonly sources: EventSources } | undefined {
+  if (event.topic !== "objects") return undefined
+
+  const objectTypeId = event.payload.objectTypeId
+  const earlier = event.type !== "object.created"
+
+  if (event.origin?.kind !== "projection") {
+    if (!earlier) return undefined
+    return { key: JSON.stringify([objectTypeId]), sources: { objectTypeId, earlier } }
+  }
+
+  const { projectionId, datasetId, datasetVersionId } = event.origin
+  const origin = { projectionId, datasetId, datasetVersionId, objectTypeId }
+  return {
+    key: JSON.stringify([objectTypeId, projectionId, datasetId, datasetVersionId, earlier]),
+    sources: { objectTypeId, origin, earlier },
+  }
+}
+
+/**
+ * Whether the event is about an object type or a link the reader cannot read: it reveals ids and
+ * keys, so it is not delivered, as for a type the reader has no view grant for.
+ */
+function readsUnreadable(event: StoredDomainEvent, clearance: PropertyClearance): boolean {
+  switch (event.topic) {
+    case "objects":
+    case "telemetry":
+      return clearance.unreadableType(event.payload.objectTypeId) !== undefined
+    case "links":
+      return (
+        clearance.unreadableType(event.payload.sourceTypeId) !== undefined ||
+        clearance.unreadableType(event.payload.targetTypeId) !== undefined ||
+        clearance.unreadableLink(event.payload.sourceTypeId, event.payload.linkId) !== undefined
+      )
+    default:
+      return false
+  }
+}

package/src/execution/authorization.ts

@@ -1,7 +1,12 @@
 import { isDeepStrictEqual } from "node:util"
 import { agentServiceAccountId } from "../agents/identity"
 import type { Principal } from "../auth"
-import { TARGETED_GRANT_KIND_KEYS, type TargetedGrantKind } from "../authorization/grant-kinds"
+import {
+  SINGLETON_GRANT_KIND_KEYS,
+  type SingletonGrantKind,
+  TARGETED_GRANT_KIND_KEYS,
+  type TargetedGrantKind,
+} from "../authorization/grant-kinds"
 import type { AuthorizationContext, GrantIndex } from "../authorization/types"
 import { createSixbError } from "../errors/internal"
 import type { ObjectRef } from "../ontology"
@@ -586,10 +591,11 @@ function snapshotAuthorizationContext(
   for (const kind of TARGETED_GRANT_KIND_KEYS) {
     grants[kind] = new ImmutableGrantSet(context.grants[kind])
   }
-  const frozenGrants: GrantIndex = Object.freeze({
-    ...grants,
-    "run:agent": context.grants["run:agent"],
-  })
+  const singletons = {} as Record<SingletonGrantKind, boolean>
+  for (const kind of SINGLETON_GRANT_KIND_KEYS) {
+    singletons[kind] = context.grants[kind]
+  }
+  const frozenGrants: GrantIndex = Object.freeze({ ...grants, ...singletons })
   return Object.freeze({
     principal,
     ...(context.sessionId === undefined ? {} : { sessionId: context.sessionId }),