@sixb/core 0.1.16 → 0.1.18

package/src/datasets/builders.ts

@@ -1,3 +1,5 @@
+import { markingIdsFrom } from "../security/builders"
+import type { MarkingDefinition } from "../security/types"
 import { DatasetValidationError } from "./errors"
 import type {
   DatasetColumnDefinition,
@@ -10,6 +12,8 @@ import { assertDatasetColumnDefinition, assertDatasetDefinition } from "./valida
 
 type DatasetColumnOptions = {
   readonly nullable?: boolean
+  /** Principals need a clearance for every marking to read this column. */
+  readonly markings?: readonly MarkingDefinition[]
 }
 
 type DefineDatasetOptions = {
@@ -112,6 +116,13 @@ type DerivedColumns<
     : DerivedBaseColumns<TParent, TOptions>
   : DerivedBaseColumns<TParent, TOptions>
 
+/** The column records marking ids; only a declared, non-empty list marks it. */
+type MarkingsFromOptions<TOptions> = TOptions extends {
+  readonly markings: readonly [MarkingDefinition, ...MarkingDefinition[]]
+}
+  ? { readonly markings: readonly string[] }
+  : { readonly markings?: readonly string[] }
+
 type DatasetColumnResult<
   TName extends string,
   TType extends DatasetColumnType,
@@ -119,7 +130,8 @@ type DatasetColumnResult<
 > = {
   readonly name: TName
   readonly type: TType
-} & FieldFromOptions<TOptions, "nullable", boolean>
+} & FieldFromOptions<TOptions, "nullable", boolean> &
+  MarkingsFromOptions<TOptions>
 
 type DatasetDefinitionResult<TId extends string, TOptions extends DefineDatasetOptions> = Omit<
   DatasetDefinition<TId, TOptions["schema"]>,
@@ -165,10 +177,14 @@ export function col(
   type: DatasetColumnType,
   options?: DatasetColumnOptions
 ): DatasetColumnDefinition {
+  const markings = options?.markings
   const column: DatasetColumnDefinition = {
     name,
     type,
     ...(options?.nullable !== undefined ? { nullable: options.nullable } : {}),
+    ...(markings && markings.length > 0
+      ? { markings: markingIdsFrom(markings, `Dataset column '${name}' markings`) }
+      : {}),
   }
 
   assertDatasetColumnDefinition(column, (message) => new DatasetValidationError(message))

package/src/datasets/execution.ts

@@ -1,11 +1,18 @@
-import { assertPrivileged, isRuntimeAllowed } from "../authorization"
+import {
+  assertAuthorized,
+  assertPrivileged,
+  assertRuntimeAuthorizationBound,
+  isRuntimeAllowed,
+} from "../authorization"
 import type { BlobStorage } from "../blob-storage"
 import type { ExecutionContext } from "../execution"
 import { resolveRuntimeAuthorizationForProject } from "../execution/authorization"
 import type { DatasetMergeCommitResult } from "../lake-storage/merge"
 import { getDatasetPrimaryKeyColumns } from "../lake-storage/merge-validation"
-import type { DatasetRow, LakeStorage } from "../lake-storage/types"
+import type { DatasetRow, DatasetVersion, LakeStorage } from "../lake-storage/types"
 import type { SixbRuntimeContext } from "../runtime/types"
+import { missingClearanceRedactions } from "../security/redactions"
+import type { Redactions } from "../security/types"
 import type { MergeChange } from "./changes"
 import type { DatasetDefinition } from "./types"
 import { writeDataset } from "./write"
@@ -20,9 +27,36 @@ export interface DatasetIngestInput {
 /** Source commit result; downstream pipelines and projections run asynchronously. */
 export type DatasetIngestResult = DatasetMergeCommitResult & { readonly rowsRead: number }
 
+export interface DatasetReadRowsInput {
+  /** Defaults to the latest version. */
+  readonly versionId?: string
+  /** Defaults to every column of the version, in schema order. */
+  readonly columns?: readonly string[]
+  readonly limit?: number
+  readonly offset?: number
+  readonly signal?: AbortSignal
+}
+
+export interface DatasetRows {
+  readonly version: DatasetVersion
+  /** Columns each row holds. Redacted columns are left out. */
+  readonly columns: readonly string[]
+  /**
+   * Every column of the dataset the reader cannot read, whether or not it was requested. Absent
+   * when the reader can read every column.
+   */
+  readonly redactions?: Redactions
+  readonly rows: AsyncIterable<DatasetRow>
+}
+
 export interface DatasetsRuntime {
   list(): readonly DatasetDefinition[]
   getById(datasetId: string): DatasetDefinition | null
+  /**
+   * Read rows of one version. Returns null when the dataset has no version yet or the requested
+   * version does not exist.
+   */
+  readRows(dataset: DatasetDefinition, input?: DatasetReadRowsInput): Promise<DatasetRows | null>
   ingest(dataset: DatasetDefinition, input: DatasetIngestInput): Promise<DatasetIngestResult>
 }
 
@@ -47,6 +81,44 @@ export function createDatasetsRuntime(
       const dataset = source.getById(datasetId)
       return dataset && allowed(datasetId) ? dataset : null
     },
+    async readRows(dataset, input = {}) {
+      assertAuthorized(runtime, { kind: "dataset.view", datasetId: dataset.id })
+      const registered = source.getById(dataset.id)
+      if (!registered) throw new Error(`[Sixb] Dataset '${dataset.id}' is not registered.`)
+      const version =
+        input.versionId === undefined
+          ? await lakeStorage.getLatestVersion(registered.id)
+          : await lakeStorage.getVersion(registered.id, input.versionId)
+      if (!version) return null
+
+      const redacted = redactedColumns(registered, assertRuntimeAuthorizationBound(runtime))
+      const selected = selectColumns(version, input.columns)
+      const columns = selected.filter((column) => !redacted.has(column))
+      const read = (readColumns: readonly string[] | undefined) =>
+        lakeStorage.readRows({
+          datasetId: registered.id,
+          versionId: version.versionId,
+          columns: readColumns,
+          limit: input.limit,
+          offset: input.offset,
+          signal: input.signal,
+        })
+
+      // Redacted columns are not read. A provider reads every column when given none, so a page
+      // made only of redacted columns reads one of them to count its rows and returns them empty.
+      const rows =
+        columns.length === selected.length
+          ? read(input.columns)
+          : columns.length > 0
+            ? read(columns)
+            : emptyRows(read(selected.slice(0, 1)))
+      return {
+        version,
+        columns,
+        ...(redacted.size > 0 ? { redactions: missingClearanceRedactions(redacted) } : {}),
+        rows,
+      }
+    },
     async ingest(dataset, input) {
       assertPrivileged(runtime, "datasets.ingest")
       const registered = source.getById(dataset.id)
@@ -93,3 +165,39 @@ export function createDatasetsRuntime(
     },
   }
 }
+
+/** Requested columns, or every column of the version. Like the lake, an empty list means all. */
+function selectColumns(version: DatasetVersion, requested: readonly string[] | undefined) {
+  const available = version.schema.columns.map((column) => column.name)
+  if (requested === undefined || requested.length === 0) return available
+  for (const column of requested) {
+    if (!available.includes(column)) {
+      throw new Error(
+        `[Sixb] Dataset '${version.datasetId}' does not have column '${column}' at version '${version.versionId}'.`
+      )
+    }
+  }
+  return [...requested]
+}
+
+/**
+ * Columns the reader cannot read. Trusted code reads every column, a principal reads what its roles
+ * clear, and any other authority is cleared for nothing.
+ */
+function redactedColumns(
+  dataset: DatasetDefinition,
+  authority: ReturnType<typeof assertRuntimeAuthorizationBound>
+): ReadonlySet<string> {
+  if (authority.type === "unrestricted") return new Set()
+  const clearances =
+    authority.type === "principal" ? (authority.context.clearances ?? new Set()) : new Set()
+  return new Set(
+    dataset.schema.columns
+      .filter((column) => column.markings?.some((markingId) => !clearances.has(markingId)))
+      .map((column) => column.name)
+  )
+}
+
+async function* emptyRows(rows: AsyncIterable<DatasetRow>): AsyncIterable<DatasetRow> {
+  for await (const _row of rows) yield {}
+}

package/src/datasets/index.ts

@@ -2,7 +2,13 @@ export { col, defineDataset } from "./builders"
 export type { MergeChange } from "./changes"
 export { change } from "./changes"
 export { DatasetValidationError } from "./errors"
-export type { DatasetIngestInput, DatasetIngestResult, DatasetsRuntime } from "./execution"
+export type {
+  DatasetIngestInput,
+  DatasetIngestResult,
+  DatasetReadRowsInput,
+  DatasetRows,
+  DatasetsRuntime,
+} from "./execution"
 export type {
   DatasetColumnDefinition,
   DatasetColumnDefinitionOf,

package/src/datasets/types.ts

@@ -16,6 +16,8 @@ export interface DatasetColumnDefinition<
   readonly name: TName
   readonly type: TType
   readonly nullable?: boolean
+  /** Marking ids. Principals need a clearance for each one to read this column. */
+  readonly markings?: readonly string[]
 }
 
 export interface DatasetSchema<

package/src/datasets/write.ts

@@ -1,5 +1,6 @@
 import type { BlobStorage, FileRef } from "../blob-storage"
 import { isFileRef } from "../blob-storage"
+import { isValidFileRef } from "../blob-storage/validation"
 import type { DatasetMergeCommitResult } from "../lake-storage/merge"
 import { getDatasetMergeChangeValidationError } from "../lake-storage/merge-validation"
 import type {
@@ -37,6 +38,18 @@ export interface WriteDatasetInput<TValue extends DatasetWriteValue = DatasetWri
 
 export type WriteDatasetResult = DatasetMergeCommitResult & { readonly rowsRead: number }
 
+/**
+ * The definition the lake receives. Markings are enforced from the registered definition and never
+ * stored, so a stored schema cannot keep a classification the code has since changed.
+ */
+export function lakeDatasetDefinition(dataset: DatasetDefinition): DatasetDefinition {
+  if (!dataset.schema.columns.some((column) => column.markings)) return dataset
+  return {
+    ...dataset,
+    schema: { columns: dataset.schema.columns.map(({ markings: _markings, ...column }) => column) },
+  }
+}
+
 /**
  * Validate, stage, and commit one dataset write using existing lake sessions.
  *
@@ -47,7 +60,8 @@ export type WriteDatasetResult = DatasetMergeCommitResult & { readonly rowsRead:
 export async function writeDataset<TValue extends DatasetWriteValue>(
   input: WriteDatasetInput<TValue>
 ): Promise<WriteDatasetResult> {
-  const { lakeStorage, dataset, signal } = input
+  const { lakeStorage, signal } = input
+  const dataset = lakeDatasetDefinition(input.dataset)
   let abortWrite: (() => Promise<void>) | undefined
   let rowsRead = 0
   const onRead = () => {
@@ -207,8 +221,13 @@ async function verifyFileRef(
   columnName: string,
   itemIndex: number
 ): Promise<void> {
-  const blobInfo = await input.blobStorage.stat(fileRef.blobId)
   const context = `${sourceLabel(input)} returned row ${itemIndex} with dataset '${input.dataset.id}' column '${columnName}'`
+  if (!isValidFileRef(fileRef)) {
+    throw new Error(
+      `${context} with an invalid mediaType; use one media type, such as "application/pdf".`
+    )
+  }
+  const blobInfo = await input.blobStorage.stat(fileRef.blobId)
   if (!blobInfo) {
     throw new Error(`${context} referencing unknown blob '${fileRef.blobId}'.`)
   }

package/src/edits/properties.ts

@@ -3,6 +3,7 @@ import { OntologyValidationError } from "../ontology/errors"
 import type { ObjectTypeWithPropertyTokens } from "../ontology/tokens"
 import {
   assertKnownProperties,
+  assertValidFileRefs,
   normalizeObjectProperties,
   validateObjectProperties,
   validatePropertyValue,
@@ -20,6 +21,7 @@ export function normalizeObjectEditProperties(params: {
   assertKnownProperties(objectType, properties)
   assertNoTelemetryProperties(objectType.properties, properties, path)
   validateObjectProperties(objectType, properties, valueTypesById)
+  assertValidFileRefs(properties, objectType.id)
   return normalizeObjectProperties(objectType.properties, properties, valueTypesById, path)
 }
 
@@ -60,6 +62,7 @@ export function normalizeLinkEditProperties(params: {
       valueTypesById
     )
   }
+  assertValidFileRefs(properties, `${sourceObjectTypeId}.${linkId}`)
 
   return normalizeObjectProperties(
     linkProperties,

package/src/events/execution.ts

@@ -1,5 +1,6 @@
 import { assertPrivileged, canViewEvent } from "../authorization"
 import { resolveRuntimeAuthorizationForProject } from "../execution/authorization"
+import { redactDomainEvent, resolvePropertyClearance } from "../objects/property-clearance"
 import type { SixbRuntimeContext } from "../runtime/types"
 import type {
   EventsAppendInput,
@@ -10,7 +11,8 @@ import type {
 import type { StoredDomainEvent } from "./types"
 
 export interface EventsRuntime {
-  canRead(event: StoredDomainEvent): boolean
+  /** The event as this execution may read it: hidden events are `undefined`, others redacted. */
+  readable(event: StoredDomainEvent): StoredDomainEvent | undefined
   append(input: EventsAppendInput): Promise<readonly StoredDomainEvent[]>
   emit(input: EventsAppendInput, options: EventsEmitOptions): Promise<void>
   read(input?: EventsReadInput): Promise<readonly StoredDomainEvent[]>
@@ -23,30 +25,27 @@ export interface EventsRuntime {
 
 export function createEventsRuntime(runtime: SixbRuntimeContext): EventsRuntime {
   const authority = resolveRuntimeAuthorizationForProject(runtime)
-  const visibleEvents = (events: readonly StoredDomainEvent[]): readonly StoredDomainEvent[] => {
+  const clearance =
+    authority.type === "principal"
+      ? resolvePropertyClearance(runtime.ontology, authority.context.clearances ?? new Set())
+      : undefined
+  const readable = (event: StoredDomainEvent): StoredDomainEvent | undefined => {
     switch (authority.type) {
       case "denied":
       case "delegated":
-        return []
+        return undefined
       case "principal":
-        return events.filter((event) => canViewEvent(authority.context, event))
+        if (!canViewEvent(authority.context, event)) return undefined
+        return clearance ? redactDomainEvent(event, clearance) : event
       case "unrestricted":
-        return [...events]
+        return event
     }
   }
+  const visibleEvents = (events: readonly StoredDomainEvent[]): readonly StoredDomainEvent[] =>
+    events.flatMap((event) => readable(event) ?? [])
 
   return {
-    canRead: (event) => {
-      switch (authority.type) {
-        case "denied":
-        case "delegated":
-          return false
-        case "principal":
-          return canViewEvent(authority.context, event)
-        case "unrestricted":
-          return true
-      }
-    },
+    readable,
     append: (input) => {
       assertPrivileged(runtime, "events.append")
       return runtime.events.append(input)

package/src/events/types/objects.ts

@@ -1,3 +1,4 @@
+import type { Redactions } from "../../security/types"
 import type { EventEnvelope } from "../envelope"
 import type { PropertyChangeMap } from "../property-changes"
 
@@ -6,12 +7,16 @@ export interface ObjectMutationEventPayload<TValue = unknown> {
   primaryId: string
   properties: Record<string, TValue>
   propertyChanges: PropertyChangeMap<TValue>
+  /** Properties omitted for this reader, which lacks their clearance. Never stored. */
+  redactions?: Redactions
 }
 
 export interface ObjectDeletedEventPayload<TValue = unknown> {
   objectTypeId: string
   primaryId: string
   propertyChanges: PropertyChangeMap<TValue>
+  /** Properties omitted for this reader, which lacks their clearance. Never stored. */
+  redactions?: Redactions
 }
 
 export interface ObjectCreatedEvent extends EventEnvelope {

package/src/execution/agent.ts

@@ -1,4 +1,6 @@
 import { type AgentExecutionAuthorization, agentServiceAccountId } from "../agents/authority"
+import { bindModelExecutionAttempt } from "../models/execution/binding"
+import type { AiModelCallRecorder } from "../models/execution/model-call-recorder"
 import { isBoundSixb, type Sixb } from "../runtime/sixb"
 import type { ExecutionRecord } from "../storage/executions"
 import { ExecutionStorageError } from "../storage/executions"
@@ -107,6 +109,31 @@ export function bindDurableAgentExecution(
   return sixb
 }
 
+/**
+ * Account model calls made through a run's bound SDK — by its tools, or embeddings behind their
+ * searches — like the run's own: same execution attempt, limits, failure state and cancellation.
+ * The attempt is the recorder's, so the two cannot diverge.
+ */
+export function bindAgentModelExecution(
+  sixb: Sixb,
+  input: {
+    readonly signal: AbortSignal
+    readonly accounting: AiModelCallRecorder
+  }
+): void {
+  const { accounting } = input
+  if (accounting.executionId !== sixb.execution.id) {
+    throw new Error(
+      `[Sixb] Agent execution '${sixb.execution.id}' cannot account its model calls to execution '${accounting.executionId}'.`
+    )
+  }
+  bindModelExecutionAttempt(sixb.models, {
+    attempt: accounting.attempt,
+    signal: input.signal,
+    accounting,
+  })
+}
+
 /** Restore one provider-validated Agent execution with its currently resolved grants. */
 export function restoreAgentExecutionScope(input: {
   readonly execution: ExecutionRecord

package/src/execution/authorization.ts

@@ -596,6 +596,7 @@ function snapshotAuthorizationContext(
     groupIds: Object.freeze([...context.groupIds]),
     roleIds: Object.freeze([...context.roleIds]),
     grants: frozenGrants,
+    clearances: new ImmutableGrantSet(context.clearances ?? []),
   })
 }