@sixb/core 0.1.20 → 0.1.22

package/src/runtime/host.ts

@@ -8,7 +8,12 @@
 import { resolve } from "node:path"
 import { ActionRunExecutor, registerActionRunExecutor } from "../actions/run/executor"
 import type { ActionDefinition } from "../actions/types"
+import { type ActivityOptions, ActivityService } from "../activity/service"
 import type { AgentDefinitionsInput, AgentToolDefinition } from "../agents"
+import {
+  createAgentApprovalExecutor,
+  registerAgentApprovalExecutor,
+} from "../agents/approval-executor"
 import {
   AuthRuntime,
   AuthRuntimeError,
@@ -70,7 +75,9 @@ import {
   createSandboxDefinition,
   type InferSandboxParams,
   type SandboxDefinition,
+  sandboxCredentialIssue,
 } from "../sandboxes/configuration"
+import { SandboxError } from "../sandboxes/errors"
 import { type SchedulerController, SchedulerRuntime } from "../scheduler"
 import type { ScheduleDefinition } from "../schedules"
 import type {
@@ -80,6 +87,14 @@ import type {
   RoleDefinition,
   SecurityDefinitionCatalog,
 } from "../security"
+import { type AppliedMarkings, createAppliedMarkings } from "../security/applied-markings"
+import { registerAppliedMarkings } from "../security/applied-markings-host"
+import {
+  createDatasetVersionMarkings,
+  type DatasetVersionMarkings,
+  registerDatasetVersionMarkings,
+} from "../security/dataset-version-markings"
+import { assertViewConditionsReadable } from "../security/view-conditions"
 import type { ParamsConfig } from "../shared/params/types"
 import type { ShareDefinition } from "../shares"
 import type { Storage } from "../storage"
@@ -122,6 +137,8 @@ export interface SixbHostOptions<in out TParams extends ParamsConfig = ParamsCon
   logger?: LoggerProvider
   /** Broker capture controls, independent from the output provider. */
   observability?: ObservabilityOptions
+  /** Records visits and page views in the project's app. Off by default. */
+  activity?: ActivityOptions
   /** Observes runtime failures without changing their outcome. */
   onError?: SixbErrorHandler
   /** Recovery and retention settings. The runtime constructor never starts maintenance timers. */
@@ -170,11 +187,14 @@ export class SixbHost<in out TParams extends ParamsConfig = ParamsConfig> {
   private readonly storageReadiness: StorageReadiness
   private readonly connectorService: ConnectorService
   private readonly materializer: OntologyMaterializerContract
+  private readonly datasetVersionMarkings: DatasetVersionMarkings
+  private readonly appliedMarkings: AppliedMarkings
   readonly sandboxDefinition?: SandboxDefinition
   readonly definitions: SixbDefinitions
   readonly broker: Broker
   readonly events: DomainEventLog
   readonly logging: LoggingService
+  readonly activity: ActivityService
   readonly storage: Storage
   readonly lakeStorage: LakeStorage
   readonly blobStorage: BlobStorage
@@ -203,6 +223,7 @@ export class SixbHost<in out TParams extends ParamsConfig = ParamsConfig> {
       observability: options.observability?.logs,
     })
     this.storage = options.storage
+    this.activity = new ActivityService(this.projectId, this.storage.activity, options.activity)
     this.storageReadiness = new StorageReadiness(this.storage)
     this.lakeStorage = options.lakeStorage
     this.blobStorage = options.blobStorage
@@ -211,18 +232,45 @@ export class SixbHost<in out TParams extends ParamsConfig = ParamsConfig> {
     this.projectRoot = resolve(options.projectRoot ?? process.cwd())
 
     const definitions = resolveDefinitions(options)
+    assertViewConditionsReadable({
+      projectId: this.projectId,
+      conditions: definitions.security.listViewConditions(),
+      objectStorage: this.storage.objects,
+      getSubTypes: (objectTypeId) => definitions.ontology.listSubTypes(objectTypeId),
+    })
     this.definitions = definitions
     this.sandboxDefinition =
       options.sandboxes?.configuration === undefined
         ? undefined
         : createSandboxDefinition(options.sandboxes.configuration, definitions.ontology)
+    assertSandboxCredentials(options.sandboxes)
     registerProjectionRegistry(this, definitions.projections)
 
+    this.datasetVersionMarkings = createDatasetVersionMarkings({
+      datasets: definitions.datasets,
+      lakeStorage: this.lakeStorage,
+    })
+    registerDatasetVersionMarkings(this, this.datasetVersionMarkings)
+
+    this.appliedMarkings = createAppliedMarkings({
+      projectId: this.projectId,
+      ontology: definitions.ontology,
+      projections: definitions.projections,
+      revisionOf: (projectionId) =>
+        definitions.projections.resolveDispatch(projectionId).projectionRevision,
+      pipelines: definitions.pipelines.list(),
+      versionMarkings: this.datasetVersionMarkings,
+      lakeStorage: this.lakeStorage,
+      sources: this.storage.ontology.sources,
+    })
+    registerAppliedMarkings(this, this.appliedMarkings)
+
     this.auth = new AuthRuntime({
       projectId: this.projectId,
       storage: this.storage,
       security: definitions.security,
       config: options.auth,
+      events: this.eventService,
     })
     validateAuthStrategySecurityReferences(this.auth.getStrategy(), definitions.security)
     const connectors = definitions.connectors.list()
@@ -289,7 +337,7 @@ export class SixbHost<in out TParams extends ParamsConfig = ParamsConfig> {
       broker: this.broker,
       ontology: definitions.ontology,
       actionRegistry: definitions.actions,
-      events: this.events,
+      events: this.eventService,
       storage: this.storage,
       queues: this.queues,
     }
@@ -299,6 +347,10 @@ export class SixbHost<in out TParams extends ParamsConfig = ParamsConfig> {
     const executor = new ActionRunExecutor(this)
     registerActionRunExecutor(this, executor)
     registerActionRunExecutor(this.hostContext, executor)
+    // Deciding on a held Agent call runs it under another execution than the decider's.
+    const approvals = createAgentApprovalExecutor(this)
+    registerAgentApprovalExecutor(this, approvals)
+    registerAgentApprovalExecutor(this.hostContext, approvals)
     registerVectorIndexingRuntime(
       this,
       createVectorIndexingRuntime(this.hostContext, this.materializer)
@@ -330,6 +382,8 @@ export class SixbHost<in out TParams extends ParamsConfig = ParamsConfig> {
       objectStorage: this.storage.objects,
       embeddingModels,
       rerankingModels,
+      appliedMarkings: this.appliedMarkings,
+      users: this.storage.auth?.users,
     })
 
     const runtime: SixbRuntimeContext = {
@@ -349,6 +403,8 @@ export class SixbHost<in out TParams extends ParamsConfig = ParamsConfig> {
           this.committedFacts.notify()
           this.vectorIndexing?.notify()
         },
+        // A projection this process applies is visible to its next read.
+        onProjectionApplied: () => this.appliedMarkings.invalidate(),
       })
     )
     return createBoundSixb<InferSandboxParams<TParams>>(
@@ -417,6 +473,7 @@ export class SixbHost<in out TParams extends ParamsConfig = ParamsConfig> {
         supportsPersistence: typeof this.sandboxes?.resume === "function",
       },
       logging: this.logging,
+      activity: this.activity,
       connectorService: this.connectorService,
       ...(this.connectorService.connectionProcess === undefined
         ? {}
@@ -424,6 +481,7 @@ export class SixbHost<in out TParams extends ParamsConfig = ParamsConfig> {
       blobStorage: this.blobStorage,
       apiPublicOrigin: () => getApiPublicOrigin(this),
       lakeStorage: this.lakeStorage,
+      datasetVersionMarkings: this.datasetVersionMarkings,
     }
   }
 }
@@ -494,3 +552,17 @@ function assertWebhookRunStorage(
     }
   }
 }
+
+/** Reject source access the factory cannot provide: a run checks what `resolve` returns. */
+function assertSandboxCredentials<TParams extends ParamsConfig>(
+  sandboxes: SandboxFactory<TParams> | undefined
+): void {
+  if (!sandboxes) return
+
+  const issue = sandboxCredentialIssue({
+    supportsRequestCredentials: sandboxes.supportsRequestCredentials,
+    auth: sandboxes.configuration?.auth,
+    source: sandboxes.configuration?.source,
+  })
+  if (issue) throw new SandboxError(`[Sixb] ${issue}`)
+}

package/src/runtime/index.ts

@@ -61,4 +61,6 @@ export type {
   TelemetryHistoryInput,
   TelemetryPropertyToken,
   TwinObject,
+  UniquePropertyToken,
+  UniquePropertyValue,
 } from "./types"

package/src/runtime/internal.ts

@@ -1,3 +1,5 @@
+export type { ActivityCaptureRuntime } from "../activity/execution"
+export { getActivityCaptureRuntime } from "../activity/execution"
 export { getConnectorConnectionsRuntime } from "../connectors/connections/capability"
 export type {
   CompleteConnectorAuthorizationInput,

package/src/runtime/ontology-mutations.ts

@@ -56,17 +56,24 @@ export function shareOntologyMutationRuntime(source: object, target: object): vo
 export function createOntologyMutationRuntime(input: {
   readonly materializer: BoundOntologyMaterializer
   readonly notifyCommittedFacts: () => void
+  /** Called once a projection may have changed the dataset version its source applied. */
+  readonly onProjectionApplied?: () => void
 }): OntologyMutationRuntime {
+  const applied = <TResult>(work: Promise<TResult>): Promise<TResult> =>
+    work.finally(() => notifyWithoutAffecting(input.onProjectionApplied))
+
   const runtime: OntologyMutationRuntime = {
     getProjectionSource: (source) => input.materializer.projections.getActive(source),
     commitEdits: (command) =>
       commitAndNotify(() => input.materializer.edits.commit(command), input.notifyCommittedFacts),
     replaceProjection: (command) =>
-      commitAndNotify(
-        () => input.materializer.projections.replace(command),
-        input.notifyCommittedFacts
+      applied(
+        commitAndNotify(
+          () => input.materializer.projections.replace(command),
+          input.notifyCommittedFacts
+        )
       ),
-    finishProjection: (command) => input.materializer.projections.finishRun(command),
+    finishProjection: (command) => applied(input.materializer.projections.finishRun(command)),
     appendTelemetry: (command) =>
       commitAndNotify(
         () => input.materializer.telemetry.append(command),
@@ -94,3 +101,11 @@ function notifyWithoutAffectingCommit(notify: () => void): void {
     console.error("[Sixb] Failed to wake the ontology outbox dispatcher:", error)
   }
 }
+
+function notifyWithoutAffecting(notify: (() => void) | undefined): void {
+  try {
+    notify?.()
+  } catch (error) {
+    console.error("[Sixb] Failed to report an applied projection:", error)
+  }
+}

package/src/runtime/resolve-definitions.ts

@@ -1,10 +1,13 @@
 import { ActionRegistry } from "../actions"
 import type { ActionDefinition } from "../actions/types"
 import type { AgentDefinitionsInput, AgentToolDefinition } from "../agents"
+import { assertTurnModelRouting } from "../agents/model-routing"
 import { assertNoAgentDefinitions } from "../agents/retired-config"
 import { createAgentSkillCatalog } from "../agents/skills"
 import { createAgentToolCatalog } from "../agents/tool-catalog"
 import type { ConnectorDefinition } from "../connectors"
+import { isDatasetPick } from "../datasets/builders"
+import { columnsByName } from "../datasets/column-index"
 import type { DatasetDefinition } from "../datasets/types"
 import { assertDatasetDefinition } from "../datasets/validation"
 import { createModelCatalog, type ModelCatalogInput } from "../models"
@@ -77,6 +80,7 @@ export function resolveDefinitions(options: DefinitionOptions): ResolvedDefiniti
   })
 
   const models = options.models === undefined ? undefined : createModelCatalog(options.models)
+  assertTurnModelRouting(models)
   for (const type of ontology.listObjectTypes()) {
     for (const [name, profile] of Object.entries(type.search?.vectors ?? {})) {
       const registered = models?.embedding.getByRef({
@@ -116,12 +120,15 @@ export function resolveDefinitions(options: DefinitionOptions): ResolvedDefiniti
   for (const sync of options.syncs ?? []) {
     assertUniqueDefinitionId("sync", sync.id, syncsById)
 
-    if (!datasetsById.has(sync.target.dataset.id)) {
+    const target = datasetsById.get(sync.target.dataset.id)
+    if (!target) {
       throw new RuntimeError(
         `Sync '${sync.id}' targets unknown dataset '${sync.target.dataset.id}'. Add it to 'datasets' in createSixb() or export it from 'datasets/'.`
       )
     }
 
+    assertRegisteredColumns(`Sync '${sync.id}' target '${target.id}'`, sync.target.dataset, target)
+
     syncsById.set(sync.id, sync)
   }
 
@@ -181,6 +188,7 @@ export function resolveDefinitions(options: DefinitionOptions): ResolvedDefiniti
     connectorIds: new Set(connectorsById.keys()),
     shareIds: new Set(sharesById.keys()),
     getSubTypes: (objectTypeId) => ontology.listSubTypes(objectTypeId),
+    ontology,
   })
   validateWorkflowAgentStepGroupReferences(workflows, security)
 
@@ -265,18 +273,76 @@ function validatePipelineDatasets(
     stepIds.add(step.id)
 
     for (const [inputName, dataset] of Object.entries(step.inputs)) {
-      if (!datasetsById.has(dataset.id)) {
+      const registered = datasetsById.get(dataset.id)
+      if (!registered) {
         throw new RuntimeError(
           `Pipeline '${pipeline.id}' step '${step.id}' input '${inputName}' references unknown dataset '${dataset.id}'. Add it to 'datasets' in createSixb() or export it from 'datasets/'.`
         )
       }
+
+      assertRegisteredColumns(
+        `Pipeline '${pipeline.id}' step '${step.id}' input '${inputName}'`,
+        dataset,
+        registered,
+        { pick: "allowed" }
+      )
     }
 
-    if (!datasetsById.has(step.output.id)) {
+    const output = datasetsById.get(step.output.id)
+    if (!output) {
       throw new RuntimeError(
         `Pipeline '${pipeline.id}' step '${step.id}' outputs unknown dataset '${step.output.id}'. Add it to 'datasets' in createSixb() or export it from 'datasets/'.`
       )
     }
+
+    assertRegisteredColumns(
+      `Pipeline '${pipeline.id}' step '${step.id}' output '${output.id}'`,
+      step.output,
+      output
+    )
+  }
+}
+
+/**
+ * A definition with a registered id stands for the whole dataset unless `pick()` made it, and only
+ * a step input may be a pick. Comparing columns catches a stale copy of a definition, which would
+ * read or write other columns than the registered ones.
+ */
+function assertRegisteredColumns(
+  subject: string,
+  dataset: DatasetDefinition,
+  registered: DatasetDefinition,
+  options: { readonly pick?: "allowed" } = {}
+): void {
+  // The registered definition itself, as almost every step and sync uses.
+  if (dataset === registered) return
+
+  const pick = isDatasetPick(dataset)
+  if (pick && options.pick !== "allowed") {
+    throw new RuntimeError(`${subject} must be the whole dataset, not a pick.`)
+  }
+
+  const registeredColumns = columnsByName(registered)
+  for (const column of dataset.schema.columns) {
+    const declared = registeredColumns.get(column.name)
+    if (!declared) {
+      throw new RuntimeError(
+        `${subject} has column '${column.name}', which dataset '${registered.id}' does not declare. Use the registered definition.`
+      )
+    }
+
+    if (declared.type !== column.type) {
+      throw new RuntimeError(
+        `${subject} has column '${column.name}' as '${column.type}', but dataset '${registered.id}' declares it as '${declared.type}'. Use the registered definition.`
+      )
+    }
+  }
+
+  if (!pick && dataset.schema.columns.length !== registeredColumns.size) {
+    const hint = options.pick === "allowed" ? ", or .pick(...) to read some of its columns" : ""
+    throw new RuntimeError(
+      `${subject} lacks columns of dataset '${registered.id}'. Use the registered definition${hint}.`
+    )
   }
 }
 

package/src/runtime/sixb.ts

@@ -1,6 +1,14 @@
 import { type ActionsRuntime, createActionsRuntime } from "../actions/execution"
+import {
+  type ActivityRuntime,
+  createActivityCaptureRuntime,
+  createActivityRuntime,
+  registerActivityCaptureRuntime,
+} from "../activity/execution"
+import type { ActivityService } from "../activity/service"
 import { type AgentRuntime, createAgentRuntime } from "../agents/execution"
 import { type AiUsageRuntime, createAiUsageRuntime } from "../ai-usage"
+import { type ArtifactsRuntime, createArtifactsRuntime } from "../artifacts/execution"
 import { type BlobsRuntime, createBlobsRuntime } from "../blob-storage/execution"
 import type { BlobStorage } from "../blob-storage/types"
 import { registerConnectorConnectionsRuntime } from "../connectors/connections/capability"
@@ -13,10 +21,13 @@ import { createEventsRuntime, type EventsRuntime } from "../events/execution"
 import type { ExecutionContext } from "../execution"
 import { resolveExecutionScopeAuthorization } from "../execution/authorization"
 import { assertAuthorizedObjectReaderBinding } from "../execution/authorized-object-reader"
+import { registerRequestPreparation } from "../execution/preparation"
 import type { LakeStorage } from "../lake-storage/types"
 import { createLogsRuntime, type LogsRuntime } from "../logging/execution"
 import type { LoggingService } from "../logging/service"
+import { createDecisionRuntime } from "../models/decision/runtime"
 import { createModelsRuntime } from "../models/execution"
+import { ModelExecutionSession } from "../models/execution/session"
 import type { ModelsRuntime } from "../models/generation-types"
 import { createObjectsRuntime, type ObjectsRuntime } from "../objects/execution"
 import { createPipelinesRuntime, type PipelinesRuntime } from "../pipelines/execution"
@@ -24,8 +35,10 @@ import { createProjectionsRuntime, type ProjectionsRuntime } from "../projection
 import { createRulesRuntime, type RulesRuntime } from "../rules/execution"
 import type { SandboxDefinition } from "../sandboxes/configuration"
 import { createSchedulesRuntime, type SchedulesRuntime } from "../schedules/execution"
+import type { DatasetVersionMarkings } from "../security/dataset-version-markings"
 import { createSharesRuntime, type SharesRuntime } from "../shares/execution"
 import { createSyncsRuntime, type SyncsRuntime } from "../syncs/execution"
+import { createUsersRuntime, type UsersRuntime } from "../users"
 import { createWorkflowsRuntime, type WorkflowsRuntime } from "../workflows/execution"
 import type { SixbDefinitions } from "./definitions"
 import { shareOntologyMutationRuntime } from "./ontology-mutations"
@@ -43,12 +56,15 @@ export interface Sixb<TParams extends Record<string, unknown> = Record<string, u
   readonly projections: ProjectionsRuntime
   readonly rules: RulesRuntime
   readonly agent: AgentRuntime<TParams>
+  readonly artifacts: ArtifactsRuntime
   readonly models: ModelsRuntime
   readonly aiUsage: AiUsageRuntime
   readonly events: EventsRuntime
   readonly logs: LogsRuntime
+  readonly activity: ActivityRuntime
   readonly schedules: SchedulesRuntime
   readonly shares: SharesRuntime
+  readonly users: UsersRuntime
   readonly connector: ConnectorRuntime
   readonly blobs: BlobsRuntime
 }
@@ -62,11 +78,13 @@ export interface SixbDependencies {
   }
   readonly definitions: SixbDefinitions
   readonly logging: LoggingService
+  readonly activity: ActivityService
   readonly connectorService: ConnectorService
   readonly connectorConnections?: ConnectorConnectionProcess
   readonly blobStorage: BlobStorage
   readonly apiPublicOrigin: () => string | undefined
   readonly lakeStorage: LakeStorage
+  readonly datasetVersionMarkings: DatasetVersionMarkings
 }
 
 export function createBoundSixb<TParams extends Record<string, unknown> = Record<string, unknown>>(
@@ -87,6 +105,8 @@ export function createBoundSixb<TParams extends Record<string, unknown> = Record
     ...createExecutionFacades<TParams>(runtime, execution, dependencies),
   }
   shareOntologyMutationRuntime(runtime, sixb)
+  registerActivityCaptureRuntime(sixb, createActivityCaptureRuntime(runtime, dependencies.activity))
+  registerRequestPreparation(sixb, runtime, execution, dependencies.definitions.workflows)
   if (dependencies.connectorConnections) {
     registerConnectorConnectionsRuntime(
       sixb,
@@ -109,38 +129,62 @@ function createExecutionFacades<TParams extends Record<string, unknown>>(
   execution: ExecutionContext,
   dependencies: SixbDependencies
 ): Omit<Sixb<TParams>, "execution"> {
+  const actions = createActionsRuntime(runtime, execution)
+  const workflows = createWorkflowsRuntime(runtime, execution, dependencies.definitions.workflows)
+  // One session per execution: the facade's model calls and the Agent's decisions share its
+  // accounting, AI limits and cancellation.
+  const session = runtime.modelExecution ?? new ModelExecutionSession(runtime, execution)
+  const agent = createAgentRuntime<TParams>(
+    runtime,
+    execution,
+    dependencies.definitions.models,
+    createDecisionRuntime(
+      runtime,
+      execution,
+      dependencies.definitions.models,
+      session,
+      "agent.run"
+    ),
+    dependencies.sandbox,
+    dependencies.definitions.security
+  )
   return {
     objects: createObjectsRuntime(runtime, execution, dependencies.blobStorage),
-    actions: createActionsRuntime(runtime, execution),
+    actions,
     datasets: createDatasetsRuntime(
       runtime,
       execution,
       dependencies.definitions.datasets,
       dependencies.lakeStorage,
-      dependencies.blobStorage
+      dependencies.blobStorage,
+      dependencies.datasetVersionMarkings
     ),
-    workflows: createWorkflowsRuntime(runtime, execution, dependencies.definitions.workflows),
+    workflows,
     syncs: createSyncsRuntime(runtime, execution, dependencies.definitions.syncs),
     pipelines: createPipelinesRuntime(runtime, execution, dependencies.definitions.pipelines),
     projections: createProjectionsRuntime(runtime, dependencies.definitions.projections),
     rules: createRulesRuntime(runtime, dependencies.definitions.rules),
-    agent: createAgentRuntime<TParams>(
-      runtime,
-      execution,
-      dependencies.definitions.models,
-      dependencies.sandbox
-    ),
+    agent,
+    artifacts: createArtifactsRuntime(runtime, dependencies.blobStorage),
     models: createModelsRuntime(
       runtime,
       execution,
       dependencies.definitions.models,
+      session,
+      {
+        canViewActionRun: async (id) => (await actions.runs.getById(id)) !== null,
+        canViewWorkflowRun: async (id) => (await workflows.runs.getById(id)) !== null,
+        canViewAgentRun: async (id) => (await agent.runs.getById(id)) !== null,
+      },
       dependencies.blobStorage
     ),
     aiUsage: createAiUsageRuntime(runtime, dependencies.definitions.security),
     events: createEventsRuntime(runtime),
     logs: createLogsRuntime(runtime, dependencies.logging),
+    activity: createActivityRuntime(runtime, dependencies.activity),
     schedules: createSchedulesRuntime(runtime, dependencies.definitions.schedules),
     shares: createSharesRuntime(runtime, execution, dependencies.definitions.shares),
+    users: createUsersRuntime(runtime),
     connector: createConnectorRuntime(runtime, execution, dependencies.connectorService),
     blobs: createBlobsRuntime(
       runtime,