@sixb/core 0.1.20 → 0.1.22

package/src/security/grants.ts

@@ -9,10 +9,16 @@
 
 import type { ActionDefinition, ObjectActionDefinition } from "../actions/types"
 import { isObjectActionDefinition } from "../actions/validation"
-import type { AgentReference, AgentUsageReference } from "../agents/types"
+import type {
+  AgentMemoryReference,
+  AgentReference,
+  AgentThreadsReference,
+  AgentUsageReference,
+} from "../agents/types"
 import type { ConnectorDefinition } from "../connectors"
 import type { DatasetDefinition } from "../datasets"
-import type { ObjectType } from "../ontology"
+import { resolveWhere } from "../objects/sdk/where"
+import type { ObjectType, ObjectTypeWithPropertyTokens } from "../ontology"
 import type { PipelineDefinition } from "../pipelines"
 import { createShareActionGrant, createShareViewGrant, isShareTarget } from "../shares/builders"
 import type {
@@ -30,16 +36,21 @@ import {
   definitionIdOf,
   isBreadthSelector,
 } from "./every"
+import { project } from "./project"
 import type {
   AgentRunGrant,
   AppendGrant,
   ApplyGrant,
   EditGrant,
   ManageGrant,
+  ModelsRunGrant,
+  ObjectViewGrantBuilder,
   ObserveGrant,
+  ProjectReference,
   RunGrant,
   Selection,
   ShareGrant,
+  UsersViewGrant,
   ViewGrant,
 } from "./types"
 
@@ -155,17 +166,112 @@ function resolveGrant<TDefinition extends { readonly id?: unknown }, TTarget ext
   return { target, selection: { all: false, ids } }
 }
 
+/** `project` itself and the surfaces of it that are not viewed, which `can.view` rejects. */
+const UNVIEWABLE_PROJECT_REFERENCES: ReadonlySet<unknown> = new Set([
+  project,
+  project.logs,
+  project.activity,
+])
+
 function view<const TObjectTypeId extends string>(
   input: ShareTarget<TObjectTypeId>
 ): ShareViewGrantBuilder<TObjectTypeId>
+function view(input: ProjectReference["users"]): UsersViewGrant
+function view<const TObjectType extends ObjectTypeWithPropertyTokens>(
+  input: TObjectType
+): ObjectViewGrantBuilder<TObjectType>
 function view(input: GrantInput<ObjectType, "object">): ViewGrant<"object">
 function view(input: GrantInput<DatasetDefinition, "dataset">): ViewGrant<"dataset">
 function view(
-  input: ShareTarget | GrantInput<ObjectType | DatasetDefinition, "object" | "dataset">
-): ViewGrant | ShareViewGrantBuilder {
+  input:
+    | ShareTarget
+    | ProjectReference["users"]
+    | GrantInput<ObjectType | DatasetDefinition, "object" | "dataset">
+): ViewGrant | UsersViewGrant | ShareViewGrantBuilder {
   if (isShareTarget(input)) return createShareViewGrant(input)
+
+  if (isUsersReference(input)) {
+    return { kind: "grant", capability: "view", target: "users" }
+  }
+
+  // `project` and its other surfaces would otherwise read as an object type with no id.
+  if (UNVIEWABLE_PROJECT_REFERENCES.has(input)) {
+    throw new SecurityValidationError(
+      "[Sixb] can.view accepts object types, datasets, shares, or project.users."
+    )
+  }
+
   const { target, selection } = resolveGrant(input, "can.view", VIEW_TARGETS)
-  return { kind: "grant", capability: "view", target, selection }
+  const grant: ViewGrant = { kind: "grant", capability: "view", target, selection }
+
+  // Only a grant on one object type takes a condition, since a condition reads that type's
+  // properties. As in `apply`, the cast restores the branch `Array.isArray` cannot narrow.
+  const one: ObjectType | undefined =
+    target !== "object" || Array.isArray(input) || isBreadthSelector(input)
+      ? undefined
+      : (input as ObjectType)
+
+  if (one) {
+    Object.defineProperty(grant, "where", {
+      configurable: false,
+      enumerable: false,
+      writable: false,
+      value: (callback: unknown) => conditionalViewGrant(grant, one, callback),
+    })
+  }
+
+  return grant
+}
+
+/**
+ * The condition is built once, here, with the same builder and the same conversion as an object
+ * query's `.where()`. Startup validates it against the ontology.
+ */
+function conditionalViewGrant(
+  grant: ViewGrant,
+  objectType: ObjectType,
+  callback: unknown
+): ViewGrant<"object"> {
+  const subject = `can.view(${objectType.id}).where(...)`
+
+  if (typeof callback !== "function") {
+    throw new SecurityValidationError(
+      `[Sixb] ${subject} expects a function, such as (object) => object.p.status.eq("open").`
+    )
+  }
+
+  const condition = resolveWhere(callback as Parameters<typeof resolveWhere>[0])
+  if (condition === undefined) {
+    throw new SecurityValidationError(
+      `[Sixb] ${subject} returned no condition. Return a predicate, or drop .where() to grant every object.`
+    )
+  }
+
+  return deepFreeze({
+    kind: "grant",
+    capability: "view",
+    target: "object",
+    selection: structuredClone(grant.selection),
+    condition: structuredClone(condition),
+  })
+}
+
+function deepFreeze<T>(value: T): T {
+  if (typeof value !== "object" || value === null || Object.isFrozen(value)) return value
+
+  for (const child of Object.values(value)) deepFreeze(child)
+  return Object.freeze(value)
+}
+
+function isUsersReference(input: unknown): input is ProjectReference["users"] {
+  return (
+    typeof input === "object" &&
+    input !== null &&
+    !Array.isArray(input) &&
+    !isBreadthSelector(input) &&
+    "kind" in input &&
+    input.kind === "users"
+  )
 }
 
 function edit(input: GrantInput<ObjectType, "object">): EditGrant {
@@ -221,17 +327,22 @@ function run(input: GrantInput<WorkflowDefinition, "workflow">): RunGrant<"workf
 function run(input: GrantInput<SyncDefinition, "sync">): RunGrant<"sync">
 function run(input: GrantInput<PipelineDefinition, "pipeline">): RunGrant<"pipeline">
 function run(input: AgentReference): AgentRunGrant
+function run(input: ProjectReference["models"]): ModelsRunGrant
 function run(
   input:
     | AgentReference
+    | ProjectReference["models"]
     | GrantInput<
         WorkflowDefinition | SyncDefinition | PipelineDefinition,
         "workflow" | "sync" | "pipeline"
       >
-): RunGrant | AgentRunGrant {
+): RunGrant | AgentRunGrant | ModelsRunGrant {
   if (isAgentReference(input)) {
     return { kind: "grant", capability: "run", target: "agent" }
   }
+  if (isModelsReference(input)) {
+    return { kind: "grant", capability: "run", target: "models" }
+  }
   const { target, selection } = resolveGrant(input, "can.run", RUN_TARGETS)
   return { kind: "grant", capability: "run", target, selection }
 }
@@ -247,6 +358,17 @@ function isAgentReference(input: unknown): input is AgentReference {
   )
 }
 
+function isModelsReference(input: unknown): input is ProjectReference["models"] {
+  return (
+    typeof input === "object" &&
+    input !== null &&
+    !Array.isArray(input) &&
+    !isBreadthSelector(input) &&
+    "kind" in input &&
+    input.kind === "models"
+  )
+}
+
 function isAgentUsageReference(input: unknown): input is AgentUsageReference {
   return (
     typeof input === "object" &&
@@ -258,21 +380,79 @@ function isAgentUsageReference(input: unknown): input is AgentUsageReference {
   )
 }
 
-function observe(input: "logs" | AgentUsageReference): ObserveGrant {
-  if (input !== "logs" && !isAgentUsageReference(input)) {
-    throw new SecurityValidationError('[Sixb] can.observe accepts "logs" or agent.usage.')
+/**
+ * Listed member by member: a `project` surface is not observable just by existing.
+ * `project.users` is read with `can.view`.
+ */
+type ObservableReference =
+  | ProjectReference["logs"]
+  | ProjectReference["activity"]
+  | AgentUsageReference
+  | AgentThreadsReference
+
+/**
+ * Every reference `can.observe` accepts, keyed by its `kind`, which is also its grant target. The
+ * full `Record` is what stops a new observable reference from compiling until it is listed here.
+ */
+const OBSERVABLE_REFERENCES: Readonly<Record<ObservableReference["kind"], string>> = {
+  logs: "project.logs",
+  activity: "project.activity",
+  aiUsage: "agent.usage",
+  agentThreads: "agent.threads",
+}
+
+function isObservableReference(input: unknown): input is ObservableReference {
+  return (
+    typeof input === "object" &&
+    input !== null &&
+    "kind" in input &&
+    typeof input.kind === "string" &&
+    Object.hasOwn(OBSERVABLE_REFERENCES, input.kind)
+  )
+}
+
+function observe(input: ObservableReference): ObserveGrant {
+  const value: unknown = input
+
+  // Older role files pass the string form. Point them at the reference instead of only rejecting
+  // the input.
+  if (value === "logs") {
+    throw new SecurityValidationError(
+      '[Sixb] can.observe("logs") is now can.observe(project.logs), imported from "@sixb/core".'
+    )
+  }
+
+  if (!isObservableReference(value)) {
+    const accepted = Object.values(OBSERVABLE_REFERENCES).join(" or ")
+    throw new SecurityValidationError(`[Sixb] can.observe accepts ${accepted}.`)
   }
-  return { kind: "grant", capability: "observe", target: input === "logs" ? "logs" : "aiUsage" }
+
+  return { kind: "grant", capability: "observe", target: value.kind }
+}
+
+function isAgentMemoryReference(input: unknown): input is AgentMemoryReference {
+  return (
+    typeof input === "object" &&
+    input !== null &&
+    !Array.isArray(input) &&
+    !isBreadthSelector(input) &&
+    "kind" in input &&
+    input.kind === "agentMemory"
+  )
 }
 
 function manage(input: GrantInput<ConnectorDefinition, "connector">): ManageGrant<"connector">
 function manage(input: AgentUsageReference): ManageGrant<"aiUsage">
+function manage(input: AgentMemoryReference): ManageGrant<"agentMemory">
 function manage(
-  input: GrantInput<ConnectorDefinition, "connector"> | AgentUsageReference
+  input: GrantInput<ConnectorDefinition, "connector"> | AgentUsageReference | AgentMemoryReference
 ): ManageGrant {
   if (isAgentUsageReference(input)) {
     return { kind: "grant", capability: "manage", target: "aiUsage" }
   }
+  if (isAgentMemoryReference(input)) {
+    return { kind: "grant", capability: "manage", target: "agentMemory" }
+  }
   return {
     kind: "grant",
     capability: "manage",

package/src/security/index.ts

@@ -16,6 +16,7 @@ export {
   missingMembershipGroupIds,
   resolveMembershipPolicyScope,
 } from "./membership-policies"
+export { project } from "./project"
 export type { SecurityDefinitionCatalog, SecurityRegistryOptions } from "./registry"
 export { SecurityRegistry } from "./registry"
 export type {
@@ -32,8 +33,11 @@ export type {
   MarkingDefinition,
   MembershipOperation,
   MembershipPolicyDefinition,
+  ModelsRunGrant,
+  ObjectViewGrantBuilder,
   ObserveGrant,
   ObserveGrantTarget,
+  ProjectReference,
   RedactionReason,
   Redactions,
   RegisteredSecurityDefinitions,
@@ -42,6 +46,7 @@ export type {
   RunGrantTarget,
   Selection,
   ShareGrant,
+  UsersViewGrant,
   ViewGrant,
   ViewGrantTarget,
 } from "./types"

package/src/security/internal.ts

@@ -0,0 +1,18 @@
+/**
+ * Marking plumbing for this repo's hosts and workers, not for app authors.
+ *
+ * `checkPipelineMarkings` is the SQL preflight the API server and CLI run before serving. The
+ * projection worker shares the host's version classifier with `shareDatasetVersionMarkings`, then
+ * classifies each pinned version and refuses one with `assertProjectionVersionMarkings`. The API
+ * server reads the versions projections applied ahead of its first request with
+ * `warmAppliedMarkings`, and the CLI reports what they add with `getAppliedMarkings`.
+ */
+export { getAppliedMarkings, warmAppliedMarkings } from "./applied-markings-host"
+export type {
+  DatasetVersionMarkings,
+  VersionClassification,
+  VersionColumnMarkings,
+} from "./dataset-version-markings"
+export { getDatasetVersionMarkings, shareDatasetVersionMarkings } from "./dataset-version-markings"
+export { checkPipelineMarkings } from "./pipeline-markings"
+export { assertProjectionVersionMarkings } from "./projection-markings"

package/src/security/marking-propagation.ts

@@ -0,0 +1,255 @@
+import type { DatasetDefinition } from "../datasets/types"
+import { lineageColumn } from "../lake-storage/lineage"
+import type { DatasetColumnLineage, DatasetColumnRef } from "../lake-storage/types"
+import { runStepLineage, stepReadColumns } from "../pipelines/lineage"
+import type { PipelineDefinition, PipelineStepDefinition } from "../pipelines/types"
+
+/** A step of a registered pipeline. A step reused by two pipelines is two registered steps. */
+export interface RegisteredStep {
+  readonly pipelineId: string
+  readonly step: PipelineStepDefinition
+}
+
+/** The column that declares a marking, and the steps it went through to reach a column. */
+export interface MarkingOrigin {
+  readonly column: DatasetColumnRef
+  /** `pipeline/step`, upstream first. Empty when the column declares the marking itself. */
+  readonly steps: readonly string[]
+}
+
+/** Each marking a column carries, with one path it arrived by. */
+export type ColumnMarkingOrigins = ReadonlyMap<string, MarkingOrigin>
+
+/** Markings of every column of every registered dataset, derived from the definitions. */
+export interface DatasetMarkings {
+  /** Empty for a column without markings or that is not registered. */
+  of(column: DatasetColumnRef): ColumnMarkingOrigins
+}
+
+export function registeredSteps(
+  pipelines: readonly PipelineDefinition[]
+): readonly RegisteredStep[] {
+  return pipelines.flatMap((pipeline) =>
+    pipeline.graph.nodes.map(({ step }) => ({ pipelineId: pipeline.id, step }))
+  )
+}
+
+/** Lineage of each `run` step, built once per step object; undefined for other steps. */
+export function runStepLineages(
+  datasetsById: ReadonlyMap<string, DatasetDefinition>
+): (entry: RegisteredStep) => DatasetColumnLineage | undefined {
+  const lineages = new Map<PipelineStepDefinition, DatasetColumnLineage>()
+
+  return ({ step }) => {
+    if (step.executor.kind !== "run") return undefined
+
+    let lineage = lineages.get(step)
+    if (!lineage) {
+      lineage = runStepLineage(step, datasetsById)
+      lineages.set(step, lineage)
+    }
+
+    return lineage
+  }
+}
+
+export function stepLabel(entry: RegisteredStep): string {
+  return `${entry.pipelineId}/${entry.step.id}`
+}
+
+type MutableOrigins = Map<string, MarkingOrigin>
+
+const NO_MARKINGS: ColumnMarkingOrigins = new Map()
+
+interface KnownLineage {
+  readonly lineage: DatasetColumnLineage
+  /** What a column the lineage does not list depends on. */
+  readonly reads: readonly DatasetColumnRef[]
+}
+
+/**
+ * Markings each column carries according to the registered definitions: the ones it declares, and
+ * the ones each step derives from the columns it depends on, once that step's lineage is known.
+ *
+ * What decided the rows of a step's inputs decides the rows of its output, whether or not the step
+ * reads a column of that input: `SELECT count(*) FROM rich` reveals which rows `rich` holds. So
+ * each dataset also keeps the markings of its row dependencies, and every output column of a step
+ * carries those of every input it reads, as the lineage each version records does.
+ *
+ * A step may read its own output and a dataset may have several producers, so a step whose output
+ * gains a marking queues the steps that read that output, until nothing changes. Lineage can be
+ * added in rounds: each round only revisits what the new lineage reaches.
+ */
+export class MarkingPropagation implements DatasetMarkings {
+  private readonly datasetsById: ReadonlyMap<string, DatasetDefinition>
+  /** Marked columns only, by dataset, built when a dataset is first looked at. */
+  private readonly markings = new Map<string, Map<string, MutableOrigins>>()
+  /** Markings of what decides each dataset's rows, for datasets a step writes. */
+  private readonly rowMarkings = new Map<string, MutableOrigins>()
+  private readonly ordered: readonly RegisteredStep[]
+  private readonly readers = new Map<string, RegisteredStep[]>()
+  private readonly known = new Map<RegisteredStep, KnownLineage>()
+  private readonly queue = new Set<RegisteredStep>()
+
+  constructor(
+    datasetsById: ReadonlyMap<string, DatasetDefinition>,
+    steps: readonly RegisteredStep[]
+  ) {
+    this.datasetsById = datasetsById
+    this.ordered = inDependencyOrder(steps)
+
+    for (const entry of steps) {
+      for (const input of new Set(Object.values(entry.step.inputs).map((dataset) => dataset.id))) {
+        appendTo(this.readers, input, entry)
+      }
+    }
+  }
+
+  of(ref: DatasetColumnRef): ColumnMarkingOrigins {
+    return this.markedColumns(ref.datasetId).get(ref.column) ?? NO_MARKINGS
+  }
+
+  /**
+   * Derive markings through every step `lineageOf` knows the lineage of. A step it returns null
+   * for derives nothing until a later call knows it.
+   */
+  derive(lineageOf: (entry: RegisteredStep) => DatasetColumnLineage | null): this {
+    for (const entry of this.ordered) {
+      if (this.known.has(entry)) continue
+
+      const lineage = lineageOf(entry)
+      if (!lineage) continue
+
+      this.known.set(entry, { lineage, reads: stepReadColumns(entry.step) })
+      this.queue.add(entry)
+    }
+
+    for (const entry of this.queue) {
+      this.queue.delete(entry)
+      const known = this.known.get(entry)
+      if (!known || !this.apply(entry, known)) continue
+
+      // Entries added while iterating a Set are visited, so the queue drains in this loop.
+      for (const reader of this.readers.get(entry.step.output.id) ?? []) {
+        if (this.known.has(reader)) this.queue.add(reader)
+      }
+    }
+
+    return this
+  }
+
+  /** Whether the output's rows or any output column gained a marking. */
+  private apply(entry: RegisteredStep, { lineage, reads }: KnownLineage): boolean {
+    const label = stepLabel(entry)
+    const rows = this.union(lineage.rows)
+
+    for (const input of new Set(Object.values(entry.step.inputs).map((dataset) => dataset.id))) {
+      for (const [markingId, origin] of this.rowMarkings.get(input) ?? []) {
+        if (!rows.has(markingId)) rows.set(markingId, origin)
+      }
+    }
+
+    let everyRead: ReadonlyMap<string, MarkingOrigin> | undefined
+    let changed = false
+
+    const outputId = entry.step.output.id
+    let outputRows = this.rowMarkings.get(outputId)
+
+    for (const [markingId, origin] of rows) {
+      if (outputRows?.has(markingId)) continue
+      if (!outputRows) {
+        outputRows = new Map()
+        this.rowMarkings.set(outputId, outputRows)
+      }
+
+      outputRows.set(markingId, { column: origin.column, steps: [...origin.steps, label] })
+      changed = true
+    }
+
+    const marked = this.markedColumns(outputId)
+    for (const { name: column } of this.datasetsById.get(outputId)?.schema.columns ?? []) {
+      const values = lineageColumn(lineage, column)
+      const valueSource = values ? this.union(values) : (everyRead ?? this.union(reads))
+      if (!values) everyRead = valueSource
+
+      for (const source of [rows, valueSource]) {
+        for (const [markingId, origin] of source) {
+          let origins = marked.get(column)
+          if (origins?.has(markingId)) continue
+          if (!origins) {
+            origins = new Map()
+            marked.set(column, origins)
+          }
+
+          origins.set(markingId, { column: origin.column, steps: [...origin.steps, label] })
+          changed = true
+        }
+      }
+    }
+
+    return changed
+  }
+
+  private markedColumns(datasetId: string): Map<string, MutableOrigins> {
+    let columns = this.markings.get(datasetId)
+    if (!columns) {
+      columns = new Map()
+      for (const column of this.datasetsById.get(datasetId)?.schema.columns ?? []) {
+        if (!column.markings?.length) continue
+
+        const origin = { column: { datasetId, column: column.name }, steps: [] }
+        columns.set(column.name, new Map(column.markings.map((id) => [id, origin])))
+      }
+
+      this.markings.set(datasetId, columns)
+    }
+
+    return columns
+  }
+
+  /** Every marking of these columns, each with the first origin found. */
+  private union(columns: readonly DatasetColumnRef[]): MutableOrigins {
+    const union: MutableOrigins = new Map()
+    for (const column of columns) {
+      for (const [markingId, origin] of this.of(column)) {
+        if (!union.has(markingId)) union.set(markingId, origin)
+      }
+    }
+    return union
+  }
+}
+
+/** Producers before consumers, so most markings arrive by their shortest path. */
+function inDependencyOrder(steps: readonly RegisteredStep[]): readonly RegisteredStep[] {
+  const producers = producersByDataset(steps)
+  const ordered: RegisteredStep[] = []
+  const visited = new Set<RegisteredStep>()
+
+  const visit = (entry: RegisteredStep): void => {
+    if (visited.has(entry)) return
+
+    visited.add(entry)
+    for (const input of Object.values(entry.step.inputs)) {
+      for (const producer of producers.get(input.id) ?? []) visit(producer)
+    }
+
+    ordered.push(entry)
+  }
+
+  for (const entry of steps) visit(entry)
+  return ordered
+}
+
+function producersByDataset(
+  steps: readonly RegisteredStep[]
+): ReadonlyMap<string, readonly RegisteredStep[]> {
+  const producers = new Map<string, RegisteredStep[]>()
+  for (const entry of steps) appendTo(producers, entry.step.output.id, entry)
+  return producers
+}
+
+function appendTo<TValue>(map: Map<string, TValue[]>, key: string, value: TValue): void {
+  const values = map.get(key)
+  if (values) values.push(value)
+  else map.set(key, [value])
+}

package/src/security/marking-sets.ts

@@ -0,0 +1,42 @@
+/**
+ * Shared, immutable sets of marking ids.
+ *
+ * Every column of every resolved version holds one of a handful of distinct sets, so each distinct
+ * set is created once and compared by identity.
+ */
+export class MarkingSets {
+  readonly empty: ReadonlySet<string> = new Set()
+  private readonly sets = new Map<string, ReadonlySet<string>>([["", this.empty]])
+
+  of(markingIds: Iterable<string> | undefined): ReadonlySet<string> {
+    // Most columns carry no marking.
+    if (markingIds === undefined || (Array.isArray(markingIds) && markingIds.length === 0)) {
+      return this.empty
+    }
+
+    const sorted = [...new Set(markingIds)].sort()
+    const key = sorted.join("\u0000")
+
+    let set = this.sets.get(key)
+    if (!set) {
+      set = new Set(sorted)
+      this.sets.set(key, set)
+    }
+
+    return set
+  }
+
+  /** The union of sets this instance created. */
+  union(sets: Iterable<ReadonlySet<string> | undefined>): ReadonlySet<string> {
+    let single: ReadonlySet<string> | undefined
+    const ids: string[] = []
+
+    for (const set of sets) {
+      if (!set || set.size === 0) continue
+      single = single === undefined && ids.length === 0 ? set : undefined
+      ids.push(...set)
+    }
+
+    return single ?? this.of(ids)
+  }
+}