@sixb/core 0.1.20 → 0.1.21

package/src/artifacts/workbook.ts

@@ -0,0 +1,315 @@
+import { ARTIFACT_LIMITS } from "./limits"
+
+/**
+ * A sheet artifact is `workbook.json` and one CSV per sheet under `sheets/`. The CSV holds the
+ * values; the workbook holds what CSV can't: sheet order, column types and formats, frozen rows
+ * and columns, widths, and charts drawn from the sheets. Formulas belong in an `.xlsx` file.
+ */
+export const WORKBOOK_FILE = "workbook.json"
+
+export const SHEET_COLUMN_TYPES = [
+  "text",
+  "number",
+  "currency",
+  "percent",
+  "date",
+  "link",
+  "image",
+] as const
+export type SheetColumnType = (typeof SHEET_COLUMN_TYPES)[number]
+
+/** Column types whose cells are numbers, so a chart can plot them and a range can sum them. */
+export const NUMERIC_COLUMN_TYPES: readonly SheetColumnType[] = ["number", "currency", "percent"]
+
+export const SHEET_CHART_TYPES = ["bar", "line", "area", "pie"] as const
+export type SheetChartType = (typeof SHEET_CHART_TYPES)[number]
+
+export interface SheetColumnFormat {
+  readonly type: SheetColumnType
+  /** An ISO 4217 code, for a currency column. */
+  readonly currency?: string
+  /** Digits after the decimal point, for numbers, currencies, and percents. */
+  readonly decimals?: number
+}
+
+export interface WorkbookSheet {
+  readonly name: string
+  /** Rows that stay in view as the sheet scrolls, the header row included. */
+  readonly frozenRows: number
+  /** Columns that stay in view as the sheet scrolls sideways. */
+  readonly frozenColumns: number
+  /** Formats by column name; a column not named here is text. */
+  readonly columns: Readonly<Record<string, SheetColumnFormat>>
+  /** Widths in pixels by column name. */
+  readonly widths: Readonly<Record<string, number>>
+}
+
+export interface WorkbookChart {
+  /** What `#chart=<name>` names in an address; lowercase letters, digits, and hyphens. */
+  readonly name: string
+  readonly type: SheetChartType
+  readonly sheet: string
+  /** The column along the x axis, or a pie's slice labels. */
+  readonly x: string
+  /** The plotted columns: one series each, or a pie's one value column. */
+  readonly y: readonly string[]
+  readonly title?: string
+  /** Stack a bar or area chart's series. */
+  readonly stacked: boolean
+}
+
+export interface Workbook {
+  readonly sheets: readonly WorkbookSheet[]
+  readonly charts: readonly WorkbookChart[]
+}
+
+/** Where a sheet's values are, in the artifact. */
+export function sheetFilePath(sheetName: string): string {
+  return `sheets/${sheetName}.csv`
+}
+
+/**
+ * Reads `workbook.json` into its full form: `"Revenue": "currency"` becomes
+ * `{ type: "currency" }`, a single `y` becomes a list, and defaults are filled. Returns what is
+ * wrong in words the author can act on. Whether the columns exist is checked against the CSVs.
+ */
+export function parseWorkbook(
+  text: string
+): { readonly workbook: Workbook } | { readonly error: string } {
+  let value: unknown
+  try {
+    value = JSON.parse(text)
+  } catch {
+    return { error: `${WORKBOOK_FILE} is not valid JSON.` }
+  }
+  try {
+    return { workbook: readWorkbook(value) }
+  } catch (error) {
+    if (error instanceof WorkbookError) return { error: `${WORKBOOK_FILE}: ${error.message}` }
+    throw error
+  }
+}
+
+/** A number as a sheet stores it: plain digits, an optional sign, decimals, and exponent. */
+export function parseSheetNumber(value: string): number | undefined {
+  const trimmed = value.trim()
+  if (!/^[-+]?(\d+\.?\d*|\.\d+)(e[-+]?\d+)?$/i.test(trimmed)) return undefined
+  const number = Number(trimmed)
+  return Number.isFinite(number) ? number : undefined
+}
+
+/** A date as a sheet stores it: `YYYY-MM-DD`, optionally with an ISO 8601 time. */
+export function parseSheetDate(value: string): Date | undefined {
+  const trimmed = value.trim()
+  if (!/^\d{4}-\d{2}-\d{2}(T\d{2}:\d{2}(:\d{2}(\.\d+)?)?(Z|[-+]\d{2}:\d{2})?)?$/.test(trimmed)) {
+    return undefined
+  }
+  const date = new Date(trimmed.length === 10 ? `${trimmed}T00:00:00Z` : trimmed)
+  return Number.isNaN(date.getTime()) ? undefined : date
+}
+
+class WorkbookError extends Error {}
+
+const SHEET_NAME_MAX = 60
+const CHART_NAME = /^[a-z0-9][a-z0-9-]{0,39}$/
+const CONTROL = /[\u0000-\u001f\u007f]/
+
+function readWorkbook(value: unknown): Workbook {
+  const input = record(value, "the workbook", ["sheets", "charts"])
+  if (!Array.isArray(input.sheets) || input.sheets.length === 0) {
+    throw new WorkbookError('"sheets" must list at least one sheet.')
+  }
+  if (input.sheets.length > ARTIFACT_LIMITS.maxSheets) {
+    throw new WorkbookError(
+      `a workbook holds at most ${ARTIFACT_LIMITS.maxSheets} sheets; this one has ${input.sheets.length}.`
+    )
+  }
+  const sheets = input.sheets.map((sheet, index) => readSheet(sheet, index))
+  const names = new Set<string>()
+  for (const sheet of sheets) {
+    const key = sheet.name.toLowerCase()
+    if (names.has(key)) throw new WorkbookError(`two sheets are named '${sheet.name}'.`)
+    names.add(key)
+  }
+
+  const chartsInput = input.charts ?? []
+  if (!Array.isArray(chartsInput)) throw new WorkbookError('"charts" must be a list.')
+  if (chartsInput.length > ARTIFACT_LIMITS.maxCharts) {
+    throw new WorkbookError(
+      `a workbook holds at most ${ARTIFACT_LIMITS.maxCharts} charts; this one has ${chartsInput.length}.`
+    )
+  }
+  const charts = chartsInput.map((chart, index) => readChart(chart, index, sheets))
+  const chartNames = new Set<string>()
+  for (const chart of charts) {
+    if (chartNames.has(chart.name)) throw new WorkbookError(`two charts are named '${chart.name}'.`)
+    chartNames.add(chart.name)
+  }
+  return { sheets, charts }
+}
+
+function readSheet(value: unknown, index: number): WorkbookSheet {
+  const at = `sheet ${index + 1}`
+  const input = record(value, at, ["name", "frozenRows", "frozenColumns", "columns", "widths"])
+  const name = typeof input.name === "string" ? input.name.trim() : ""
+  if (
+    !name ||
+    name.length > SHEET_NAME_MAX ||
+    CONTROL.test(name) ||
+    /[/\\]/.test(name) ||
+    name === "." ||
+    name === ".."
+  ) {
+    throw new WorkbookError(
+      `${at} needs a "name" of at most ${SHEET_NAME_MAX} characters without / or \\; its values go in sheets/<name>.csv.`
+    )
+  }
+  const sheetAt = `sheet '${name}'`
+  const columns: Record<string, SheetColumnFormat> = {}
+  for (const [column, format] of Object.entries(
+    record(input.columns ?? {}, `${sheetAt} "columns"`, null)
+  )) {
+    columns[column] = readColumnFormat(format, `${sheetAt} column '${column}'`)
+  }
+  const widths: Record<string, number> = {}
+  for (const [column, width] of Object.entries(
+    record(input.widths ?? {}, `${sheetAt} "widths"`, null)
+  )) {
+    if (typeof width !== "number" || !Number.isInteger(width) || width < 40 || width > 800) {
+      throw new WorkbookError(
+        `${sheetAt} gives column '${column}' a width of ${String(width)}; use 40 to 800 pixels.`
+      )
+    }
+    widths[column] = width
+  }
+  return {
+    name,
+    frozenRows: integer(input.frozenRows, 1, 0, 10, `${sheetAt} "frozenRows"`),
+    frozenColumns: integer(input.frozenColumns, 0, 0, 5, `${sheetAt} "frozenColumns"`),
+    columns,
+    widths,
+  }
+}
+
+function readColumnFormat(value: unknown, at: string): SheetColumnFormat {
+  if (typeof value === "string") return { type: columnType(value, at) }
+  const input = record(value, at, ["type", "currency", "decimals"])
+  const type = columnType(input.type, at)
+  if (input.currency !== undefined) {
+    if (type !== "currency")
+      throw new WorkbookError(`${at} sets "currency" but is not a currency column.`)
+    if (typeof input.currency !== "string" || !/^[A-Z]{3}$/.test(input.currency)) {
+      throw new WorkbookError(`${at} "currency" must be an ISO 4217 code such as "USD".`)
+    }
+  }
+  if (input.decimals !== undefined && !["number", "currency", "percent"].includes(type)) {
+    throw new WorkbookError(
+      `${at} sets "decimals" but is not a number, currency, or percent column.`
+    )
+  }
+  return {
+    type,
+    ...(typeof input.currency === "string" ? { currency: input.currency } : {}),
+    ...(input.decimals === undefined
+      ? {}
+      : { decimals: integer(input.decimals, 0, 0, 6, `${at} "decimals"`) }),
+  }
+}
+
+function columnType(value: unknown, at: string): SheetColumnType {
+  if (typeof value === "string" && (SHEET_COLUMN_TYPES as readonly string[]).includes(value)) {
+    return value as SheetColumnType
+  }
+  throw new WorkbookError(
+    `${at} has type ${JSON.stringify(value)}; use one of ${SHEET_COLUMN_TYPES.join(", ")}.`
+  )
+}
+
+function readChart(value: unknown, index: number, sheets: readonly WorkbookSheet[]): WorkbookChart {
+  const input = record(value, `chart ${index + 1}`, [
+    "name",
+    "type",
+    "sheet",
+    "x",
+    "y",
+    "title",
+    "stacked",
+  ])
+  if (typeof input.name !== "string" || !CHART_NAME.test(input.name)) {
+    throw new WorkbookError(
+      `chart ${index + 1} needs a "name" of lowercase letters, digits, and hyphens, such as "revenue".`
+    )
+  }
+  const at = `chart '${input.name}'`
+  if (!(SHEET_CHART_TYPES as readonly unknown[]).includes(input.type)) {
+    throw new WorkbookError(
+      `${at} has type ${JSON.stringify(input.type)}; use one of ${SHEET_CHART_TYPES.join(", ")}.`
+    )
+  }
+  const type = input.type as SheetChartType
+  if (typeof input.sheet !== "string" || !sheets.some((sheet) => sheet.name === input.sheet)) {
+    throw new WorkbookError(
+      `${at} names sheet ${JSON.stringify(input.sheet)}, which the workbook does not have.`
+    )
+  }
+  if (typeof input.x !== "string" || !input.x) throw new WorkbookError(`${at} needs an "x" column.`)
+  const y = typeof input.y === "string" ? [input.y] : input.y
+  if (
+    !Array.isArray(y) ||
+    y.length === 0 ||
+    !y.every((column) => typeof column === "string" && column)
+  ) {
+    throw new WorkbookError(`${at} needs a "y" column, or a list of them.`)
+  }
+  if (y.length > ARTIFACT_LIMITS.maxChartSeries) {
+    throw new WorkbookError(
+      `${at} plots ${y.length} columns; a chart reads well with at most ${ARTIFACT_LIMITS.maxChartSeries}. Split it, or chart the ones that matter.`
+    )
+  }
+  if (type === "pie" && y.length !== 1)
+    throw new WorkbookError(`${at} is a pie: give it one "y" column.`)
+  if (input.stacked !== undefined && typeof input.stacked !== "boolean") {
+    throw new WorkbookError(`${at} "stacked" must be true or false.`)
+  }
+  if (input.stacked === true && type !== "bar" && type !== "area") {
+    throw new WorkbookError(`${at} is a ${type} chart; only bar and area charts stack.`)
+  }
+  if (input.title !== undefined && (typeof input.title !== "string" || input.title.length > 120)) {
+    throw new WorkbookError(`${at} "title" must be text of at most 120 characters.`)
+  }
+  return {
+    name: input.name,
+    type,
+    sheet: input.sheet,
+    x: input.x,
+    y: y as string[],
+    ...(typeof input.title === "string" && input.title.trim() ? { title: input.title.trim() } : {}),
+    stacked: input.stacked === true,
+  }
+}
+
+function record(
+  value: unknown,
+  at: string,
+  keys: readonly string[] | null
+): Record<string, unknown> {
+  if (typeof value !== "object" || value === null || Array.isArray(value)) {
+    throw new WorkbookError(`${at} must be an object.`)
+  }
+  const input = value as Record<string, unknown>
+  const unknownKey = keys && Object.keys(input).find((key) => !keys.includes(key))
+  if (unknownKey) {
+    throw new WorkbookError(
+      `${at} has '${unknownKey}'; it takes ${keys.map((key) => `"${key}"`).join(", ")}.`
+    )
+  }
+  return input
+}
+
+function integer(value: unknown, fallback: number, min: number, max: number, at: string): number {
+  if (value === undefined) return fallback
+  if (typeof value !== "number" || !Number.isInteger(value) || value < min || value > max) {
+    throw new WorkbookError(`${at} must be a whole number from ${min} to ${max}.`)
+  }
+  return value
+}

package/src/authorization/decision.ts

@@ -34,6 +34,7 @@ export type AuthzRequest =
   | { readonly kind: "agent.run" }
   | { readonly kind: "logs.observe" }
   | { readonly kind: "aiUsage.observe" }
+  | { readonly kind: "activity.observe" }
   | { readonly kind: "aiUsage.manage" }
   | { readonly kind: "connector.manage"; readonly connectorId: string }
   | { readonly kind: "object.query"; readonly touchedObjectTypeIds: readonly string[] }
@@ -86,6 +87,8 @@ function atomsFor(request: AuthzRequest): readonly Atom[] {
       return [{ kind: "observe:logs" }]
     case "aiUsage.observe":
       return [{ kind: "observe:aiUsage" }]
+    case "activity.observe":
+      return [{ kind: "observe:activity" }]
     case "aiUsage.manage":
       return [{ kind: "manage:aiUsage" }]
     case "connector.manage":
@@ -278,6 +281,8 @@ function deniedMessage(
       return `[Sixb] Principal '${principalId}' is not allowed to observe project logs.`
     case "aiUsage.observe":
       return `[Sixb] Principal '${principalId}' is not allowed to observe project AI usage.`
+    case "activity.observe":
+      return `[Sixb] Principal '${principalId}' is not allowed to observe project activity.`
     case "aiUsage.manage":
       return `[Sixb] Principal '${principalId}' is not allowed to manage project AI usage limits.`
     case "connector.manage":

package/src/authorization/grant-kinds.ts

@@ -23,11 +23,17 @@ export type GrantKind =
   | "run:agent"
   | "observe:logs"
   | "observe:aiUsage"
+  | "observe:activity"
   | "manage:connector"
   | "manage:aiUsage"
 
 /** Capabilities over one project-wide surface: held or not, with no ids to select. */
-export type SingletonGrantKind = "run:agent" | "observe:logs" | "observe:aiUsage" | "manage:aiUsage"
+export type SingletonGrantKind =
+  | "run:agent"
+  | "observe:logs"
+  | "observe:aiUsage"
+  | "observe:activity"
+  | "manage:aiUsage"
 export type TargetedGrantKind = Exclude<GrantKind, SingletonGrantKind>
 
 /** Registered id universes a grant kind ranges over, plus subtype expansion. */
@@ -130,13 +136,18 @@ export const GRANT_KINDS: {
   "observe:logs": {
     singleton: true,
     subject: "logs",
-    fix: 'Use can.observe("logs").',
+    fix: "Use can.observe(project.logs).",
   },
   "observe:aiUsage": {
     singleton: true,
     subject: "AI usage",
     fix: "Use can.observe(agent.usage).",
   },
+  "observe:activity": {
+    singleton: true,
+    subject: "activity",
+    fix: "Use can.observe(project.activity).",
+  },
   "manage:connector": {
     universeKey: "connectorIds",
     subject: "connector",
@@ -202,6 +213,7 @@ export function emptyGrantSets(): MutableGrantIndex {
     "run:agent": false,
     "observe:logs": false,
     "observe:aiUsage": false,
+    "observe:activity": false,
     "manage:connector": new Set(),
     "manage:aiUsage": false,
   }

package/src/blob-storage/internal.ts

@@ -5,4 +5,4 @@ export {
   resolveFileRefAtPath,
   statFileRef,
 } from "./file-content"
-export { canonicalMediaType } from "./media-type"
+export { canonicalMediaType, inferFileMediaType, sniffFileMediaType } from "./media-type"

package/src/blob-storage/media-type.ts

@@ -27,3 +27,72 @@ export function canonicalMediaType(value: unknown): string | null {
   }
   return canonical
 }
+
+/**
+ * The media type a file's bytes declare by their leading signature, or `undefined` when they carry
+ * none this function knows. Only formats with an unambiguous signature are recognized; text formats
+ * have none and are left to the file name.
+ */
+export function sniffFileMediaType(bytes: Uint8Array): string | undefined {
+  if (startsWith(bytes, [0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a])) return "image/png"
+  if (startsWith(bytes, [0xff, 0xd8, 0xff])) return "image/jpeg"
+  if (asciiAt(bytes, 0, "GIF87a") || asciiAt(bytes, 0, "GIF89a")) return "image/gif"
+  if (asciiAt(bytes, 0, "RIFF") && asciiAt(bytes, 8, "WEBP")) return "image/webp"
+  if (asciiAt(bytes, 0, "%PDF-")) return "application/pdf"
+  return undefined
+}
+
+/**
+ * The media type of a file handed over without one: its bytes' signature when they have one, else
+ * its extension, else `application/octet-stream`. Every place that classifies a sandbox or tool file
+ * goes through this one table, so the same file gets the same type wherever it comes from.
+ */
+export function inferFileMediaType(fileName: string, bytes?: Uint8Array): string {
+  const sniffed = bytes === undefined ? undefined : sniffFileMediaType(bytes)
+  if (sniffed) return sniffed
+  const dot = fileName.lastIndexOf(".")
+  const extension = dot < 0 ? "" : fileName.slice(dot).toLowerCase()
+  return EXTENSION_MEDIA_TYPES[extension] ?? "application/octet-stream"
+}
+
+const EXTENSION_MEDIA_TYPES: Readonly<Record<string, string>> = {
+  ".bmp": "image/bmp",
+  ".css": "text/css",
+  ".csv": "text/csv",
+  ".docx": "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
+  ".gif": "image/gif",
+  ".htm": "text/html",
+  ".html": "text/html",
+  ".jpeg": "image/jpeg",
+  ".jpg": "image/jpeg",
+  ".js": "application/javascript",
+  ".json": "application/json",
+  ".jsonl": "application/x-ndjson",
+  ".markdown": "text/markdown",
+  ".md": "text/markdown",
+  ".mjs": "application/javascript",
+  ".pdf": "application/pdf",
+  ".png": "image/png",
+  ".pptx": "application/vnd.openxmlformats-officedocument.presentationml.presentation",
+  ".svg": "image/svg+xml",
+  ".tsv": "text/tab-separated-values",
+  ".txt": "text/plain",
+  ".webp": "image/webp",
+  ".xlsx": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
+  ".xml": "application/xml",
+  ".yaml": "application/yaml",
+  ".yml": "application/yaml",
+  ".zip": "application/zip",
+}
+
+function startsWith(bytes: Uint8Array, signature: readonly number[]): boolean {
+  return signature.every((byte, index) => bytes[index] === byte)
+}
+
+function asciiAt(bytes: Uint8Array, offset: number, value: string): boolean {
+  if (bytes.byteLength < offset + value.length) return false
+  for (let index = 0; index < value.length; index += 1) {
+    if (bytes[offset + index] !== value.charCodeAt(index)) return false
+  }
+  return true
+}

package/src/datasets/builders.ts

@@ -93,6 +93,17 @@ type PickColumns<
     ? readonly []
     : readonly ColumnDefinitionOf<TColumns, TNames[number]>[]
 
+/** A dataset read through some of its columns: `rawInvoices.pick("id", "title")`. */
+type PickableDataset<TId extends string, TColumns extends readonly DatasetColumnDefinition[]> = {
+  /**
+   * The same dataset with only these columns, in this order. A pipeline step given it as an input
+   * reads nothing else, so its output inherits no other column's markings.
+   */
+  pick<const TNames extends readonly [ColumnNameOf<TColumns>, ...ColumnNameOf<TColumns>[]]>(
+    ...columns: TNames
+  ): DatasetDefinition<TId, PickColumns<TColumns, TNames>>
+}
+
 type AddColumns<
   TColumns extends readonly DatasetColumnDefinition[],
   TAdd extends readonly DatasetColumnDefinition[],
@@ -140,7 +151,8 @@ type DatasetDefinitionResult<TId extends string, TOptions extends DefineDatasetO
   FieldFromOptions<TOptions, "sequenceBy", string> &
   FieldFromOptions<TOptions, "primaryKey", DatasetPrimaryKey> &
   FieldFromOptions<TOptions, "partitionBy", readonly string[]> &
-  FieldFromOptions<TOptions, "description", string>
+  FieldFromOptions<TOptions, "description", string> &
+  PickableDataset<TId, TOptions["schema"]>
 
 type DerivedDatasetDefinitionResult<
   TId extends string,
@@ -153,7 +165,8 @@ type DerivedDatasetDefinitionResult<
   FieldFromOptions<TOptions, "sequenceBy", string> &
   FieldFromOptions<TOptions, "primaryKey", DatasetPrimaryKey> &
   FieldFromOptions<TOptions, "partitionBy", readonly string[]> &
-  FieldFromOptions<TOptions, "description", string>
+  FieldFromOptions<TOptions, "description", string> &
+  PickableDataset<TId, DerivedColumns<TParent, TOptions>>
 
 type CheckedDeriveDatasetOptions<
   TParent extends DatasetDefinition,
@@ -217,9 +230,60 @@ function createDatasetDefinition(id: string, options: DefineDatasetOptions): Dat
   }
 
   assertDatasetDefinition(definition, (message) => new DatasetValidationError(message))
+
+  // Not enumerable: the definition stays plain data for cloning, spreading, and storage.
+  Object.defineProperty(definition, "pick", {
+    value: (...columns: string[]) => pickDatasetColumns(definition, columns),
+  })
+
   return definition
 }
 
+const datasetPick: unique symbol = Symbol("sixb.datasetPick")
+
+/**
+ * Whether a definition was made by `pick()`. Only a pick may hold some of a dataset's columns: any
+ * other definition with a registered id stands for the whole dataset.
+ */
+export function isDatasetPick(dataset: DatasetDefinition): boolean {
+  return (dataset as { readonly [datasetPick]?: true })[datasetPick] === true
+}
+
+function pickDatasetColumns(
+  dataset: DatasetDefinition,
+  columnNames: readonly string[]
+): DatasetDefinition {
+  if (columnNames.length === 0) {
+    throw new DatasetValidationError(
+      `[Sixb] Dataset '${dataset.id}' pick needs at least one column.`
+    )
+  }
+
+  const columnsByName = new Map(dataset.schema.columns.map((column) => [column.name, column]))
+  const picked = new Set<string>()
+  const columns = columnNames.map((columnName) => {
+    if (picked.has(columnName)) {
+      throw new DatasetValidationError(
+        `[Sixb] Dataset '${dataset.id}' pick lists column '${columnName}' twice.`
+      )
+    }
+    picked.add(columnName)
+
+    const column = columnsByName.get(columnName)
+    if (column === undefined) {
+      throw new DatasetValidationError(
+        `[Sixb] Dataset '${dataset.id}' has no column '${columnName}' to pick.`
+      )
+    }
+
+    return column
+  })
+
+  const pick: DatasetDefinition = { kind: "dataset", id: dataset.id, schema: { columns } }
+  Object.defineProperty(pick, datasetPick, { value: true })
+  return pick
+}
+
 function assertStringArray(value: unknown, field: string): asserts value is readonly string[] {
   if (!Array.isArray(value)) {
     throw new DatasetValidationError(`${field} must be an array of column names.`)

package/src/datasets/column-index.ts

@@ -0,0 +1,19 @@
+import type { DatasetColumnDefinition, DatasetDefinition } from "./types"
+
+const indexes = new WeakMap<DatasetDefinition, ReadonlyMap<string, DatasetColumnDefinition>>()
+
+/**
+ * The columns of a dataset definition by name, built once per definition object. Registered
+ * definitions are not changed after startup, so every check and resolver can share one index.
+ */
+export function columnsByName(
+  dataset: DatasetDefinition
+): ReadonlyMap<string, DatasetColumnDefinition> {
+  let index = indexes.get(dataset)
+  if (!index) {
+    index = new Map(dataset.schema.columns.map((column) => [column.name, column]))
+    indexes.set(dataset, index)
+  }
+
+  return index
+}

package/src/datasets/execution.ts

@@ -11,6 +11,7 @@ import type { DatasetMergeCommitResult } from "../lake-storage/merge"
 import { getDatasetPrimaryKeyColumns } from "../lake-storage/merge-validation"
 import type { DatasetRow, DatasetVersion, LakeStorage } from "../lake-storage/types"
 import type { SixbRuntimeContext } from "../runtime/types"
+import type { DatasetVersionMarkings } from "../security/dataset-version-markings"
 import { missingClearanceRedactions } from "../security/redactions"
 import type { Redactions } from "../security/types"
 import type { MergeChange } from "./changes"
@@ -42,7 +43,7 @@ export interface DatasetRows {
   /** Columns each row holds. Redacted columns are left out. */
   readonly columns: readonly string[]
   /**
-   * Every column of the dataset the reader cannot read, whether or not it was requested. Absent
+   * Every column of the version the reader cannot read, whether or not it was requested. Absent
    * when the reader can read every column.
    */
   readonly redactions?: Redactions
@@ -65,7 +66,8 @@ export function createDatasetsRuntime(
   execution: ExecutionContext,
   source: Pick<DatasetsRuntime, "list" | "getById">,
   lakeStorage: LakeStorage,
-  blobStorage: Pick<BlobStorage, "stat">
+  blobStorage: Pick<BlobStorage, "stat">,
+  versionMarkings: DatasetVersionMarkings
 ): DatasetsRuntime {
   const authority = resolveRuntimeAuthorizationForProject(runtime)
   const allowed = (datasetId: string) =>
@@ -91,7 +93,11 @@ export function createDatasetsRuntime(
           : await lakeStorage.getVersion(registered.id, input.versionId)
       if (!version) return null
 
-      const redacted = redactedColumns(registered, assertRuntimeAuthorizationBound(runtime))
+      const redacted = redactedColumns(
+        version,
+        versionMarkings,
+        assertRuntimeAuthorizationBound(runtime)
+      )
       const selected = selectColumns(version, input.columns)
       const columns = selected.filter((column) => !redacted.has(column))
       const read = (readColumns: readonly string[] | undefined) =>
@@ -181,20 +187,28 @@ function selectColumns(version: DatasetVersion, requested: readonly string[] | u
 }
 
 /**
- * Columns the reader cannot read. Trusted code reads every column, a principal reads what its roles
- * clear, and any other authority is cleared for nothing.
+ * Columns of the version the reader cannot read. Trusted code reads every column, a principal reads
+ * what its roles clear, and any other authority is cleared for nothing.
+ *
+ * No version carries a marking the project does not declare on a column, so a reader cleared for
+ * every declared marking (all readers, when none is declared) reads every column without
+ * classifying the version.
  */
 function redactedColumns(
-  dataset: DatasetDefinition,
+  version: DatasetVersion,
+  versionMarkings: DatasetVersionMarkings,
   authority: ReturnType<typeof assertRuntimeAuthorizationBound>
 ): ReadonlySet<string> {
   if (authority.type === "unrestricted") return new Set()
   const clearances =
     authority.type === "principal" ? (authority.context.clearances ?? new Set()) : new Set()
+  if ([...versionMarkings.everyMarking()].every((id) => clearances.has(id))) return new Set()
+
+  const { columns } = versionMarkings.classify(version)
   return new Set(
-    dataset.schema.columns
-      .filter((column) => column.markings?.some((markingId) => !clearances.has(markingId)))
-      .map((column) => column.name)
+    [...columns]
+      .filter(([, columnMarkings]) => [...columnMarkings].some((id) => !clearances.has(id)))
+      .map(([column]) => column)
   )
 }
 

package/src/errors/internal.ts

@@ -13,6 +13,7 @@ const SIXB_ERROR_CODE_SET: ReadonlySet<string> = new Set(SIXB_ERROR_CODES)
 type SixbErrorCodeTuple = readonly [SixbErrorCode, ...SixbErrorCode[]]
 
 export { SIXB_ERROR_CODES }
+export { withFailureMessage } from "./failure-message"
 
 export interface SixbErrorOptions {
   readonly cause?: unknown