@sixb/core 0.1.15 → 0.1.17

package/src/storage/in-memory/index.ts

@@ -13,6 +13,10 @@ import {
 import { StorageTransactionError } from "../errors"
 import { InMemoryExecutionStorage } from "../executions/in-memory"
 import type { ExecutionStorage } from "../executions/types"
+import {
+  type FileDownloadGrantStorage,
+  InMemoryFileDownloadGrantStorage,
+} from "../file-download-grants"
 import { type FileUploadSessionStore, InMemoryFileUploadSessions } from "../file-upload-sessions"
 import type { ObjectStorage } from "../objects"
 import { InMemoryObjectStorage } from "../objects/in-memory"
@@ -116,6 +120,7 @@ export class InMemoryStorage implements Storage {
   private readonly webhookRunStorage = new InMemoryWebhookRunStorage(this.executionStorage)
   private readonly rulesStorage = new InMemoryRulesStorage()
   private readonly fileUploadSessionStorage = new InMemoryFileUploadSessions()
+  private readonly fileDownloadGrantStorage = new InMemoryFileDownloadGrantStorage()
   private readonly connectorConnectionStorage: InMemoryConnectorConnectionStorage
   readonly auth: AuthStorage
   readonly executions: ExecutionStorage
@@ -134,6 +139,7 @@ export class InMemoryStorage implements Storage {
   readonly shareGrants: ShareGrantStorage
   readonly shareSessions: ShareSessionStorage
   readonly fileUploadSessions: FileUploadSessionStore
+  readonly fileDownloadGrants: FileDownloadGrantStorage
   readonly connectorConnections: ConnectorConnectionStorage
 
   constructor(options: InMemoryStorageOptions = {}) {
@@ -166,6 +172,7 @@ export class InMemoryStorage implements Storage {
     this.shareGrants = createOperationScopedFacade(this.shareGrantStorage, scope)
     this.shareSessions = createOperationScopedFacade(this.shareSessionStorage, scope)
     this.fileUploadSessions = createOperationScopedFacade(this.fileUploadSessionStorage, scope)
+    this.fileDownloadGrants = createOperationScopedFacade(this.fileDownloadGrantStorage, scope)
     this.connectorConnections = createOperationScopedFacade(this.connectorConnectionStorage, scope)
     this.ontologyStorage = new InMemoryOntologyStorage(this.objectStorage, this.timeseriesStorage, {
       runRootOperation: async (run) => run(),
@@ -173,8 +180,8 @@ export class InMemoryStorage implements Storage {
       getMaterializationLifecycle: () => this.getActiveMaterializationLifecycle(),
       assertSourceMaterializationExecution: (input) =>
         this.projectionRunStorage.assertSourceMaterializationExecutionUnlocked(input),
-      executionExists: async (projectId, executionId) =>
-        (await this.executionStorage.getById({ projectId, id: executionId })) !== null,
+      readExecution: (projectId, executionId) =>
+        this.executionStorage.getById({ projectId, id: executionId }),
     })
     this.ontology = createOntologyOperationScope(this.ontologyStorage, scope)
     this.ontologyStorage.registerTestingAlias(this.ontology)
@@ -375,6 +382,7 @@ export class InMemoryStorage implements Storage {
       shareGrants: this.shareGrantStorage,
       shareSessions: this.shareSessionStorage,
       fileUploadSessions: this.fileUploadSessionStorage,
+      fileDownloadGrants: this.fileDownloadGrantStorage,
       connectorConnections: this.connectorConnectionStorage,
       ping: async () => undefined,
       transaction: async <T>(): Promise<T> => {
@@ -405,6 +413,7 @@ export class InMemoryStorage implements Storage {
       shareGrants: this.shareGrantStorage.snapshot(),
       shareSessions: this.shareSessionStorage.snapshot(),
       fileUploadSessions: this.fileUploadSessionStorage.snapshot(),
+      fileDownloadGrants: this.fileDownloadGrantStorage.snapshot(),
       connectorConnections: this.connectorConnectionStorage.snapshot(),
     }
   }
@@ -430,6 +439,7 @@ export class InMemoryStorage implements Storage {
     this.shareGrantStorage.restore(snapshot.shareGrants)
     this.shareSessionStorage.restore(snapshot.shareSessions)
     this.fileUploadSessionStorage.restore(snapshot.fileUploadSessions)
+    this.fileDownloadGrantStorage.restore(snapshot.fileDownloadGrants)
     this.connectorConnectionStorage.restore(snapshot.connectorConnections)
   }
 }
@@ -455,5 +465,6 @@ export interface InMemoryStorageSnapshot {
   readonly shareGrants: ReturnType<InMemoryShareGrantStorage["snapshot"]>
   readonly shareSessions: ReturnType<InMemoryShareSessionStorage["snapshot"]>
   readonly fileUploadSessions: ReturnType<InMemoryFileUploadSessions["snapshot"]>
+  readonly fileDownloadGrants: ReturnType<InMemoryFileDownloadGrantStorage["snapshot"]>
   readonly connectorConnections: ReturnType<InMemoryConnectorConnectionStorage["snapshot"]>
 }

package/src/storage/index.ts

@@ -12,9 +12,12 @@ export type { StoredRuleResolvedEvent, StoredRuleTriggeredEvent } from "../event
 export type { ReadonlyJsonObject } from "../json"
 export { stableJsonStringify } from "../json"
 export type {
+  EffectiveChangeCounts,
   PinnedDatasetVersion,
+  ProjectionEntityRef,
   ProjectionExecution,
   ProjectionMaterializationIdentity,
+  ProjectionSourceRef,
 } from "../materialization/model"
 export type { ModelProviderIds } from "../models/events"
 export type { ObjectRef } from "../ontology"
@@ -351,6 +354,13 @@ export type {
   TrustedPrimitiveRef,
 } from "./executions"
 export { ExecutionStorageError } from "./executions"
+export type {
+  DeleteExpiredFileDownloadGrantsInput,
+  FileDownloadGrantRecord,
+  FileDownloadGrantStorage,
+  FindFileDownloadGrantInput,
+  RevokeFileDownloadGrantInput,
+} from "./file-download-grants"
 export type {
   CreateFileUploadSessionInput,
   FileUploadSession,
@@ -380,6 +390,7 @@ export type {
   MigrationStep,
   MigrationStepInfo,
   MigrationStepOptions,
+  StorageMigrationOptions,
   StorageMigrationResult,
   StorageMigrator,
   StorageSchemaCheck,
@@ -448,13 +459,19 @@ export {
   objectLinkCursor,
   snapshotObjectReadExecutionLimits,
 } from "./objects"
+export type {
+  ObjectQueryPreparationPlan,
+  ObjectQueryPreparationResult,
+  QueryPreparationCapableStorage,
+} from "./objects/query-preparation"
 export type {
   AbandonRunSourceMaterializationInput,
   AbandonSourceMaterializationCandidateInput,
   AbandonSourceMaterializationInput,
   AdoptedSourceMaterialization,
   AdoptSourceMaterializationInput,
-  ApplyMaterializationChunkInput,
+  AppliedMaterialization,
+  ApplyMaterializationInput,
   ApplyMaterializationResult,
   AssertSourceMaterializationExecution,
   AssertSourceMaterializationExecutionInput,
@@ -469,16 +486,13 @@ export type {
   ExactEffectiveLinkWrite,
   ExactEffectiveObjectDelete,
   ExactEffectiveObjectWrite,
-  ExactEffectiveWrites,
   ExactLinkOverrideDelete,
   ExactLinkOverrideWrite,
   ExactLinkSlotOverrideDelete,
   ExactLinkSlotOverrideWrite,
   ExactObjectOverrideDelete,
   ExactObjectOverrideWrite,
-  ExactOverrideWrites,
   ExactTimeseriesPointWrite,
-  ExactTimeseriesWrites,
   ExpectedSourceRevision,
   ExpectedTimeseriesPointRevision,
   FinalizeMaterializationInput,
@@ -501,7 +515,6 @@ export type {
   MaterializationObjectExistence,
   MaterializationObjectExistenceWorkRecord,
   MaterializationObjectState,
-  MaterializationPlanChunk,
   MaterializationPlanFinalization,
   MaterializationPlanHeader,
   MaterializationPlanWorkItem,
@@ -509,8 +522,9 @@ export type {
   MaterializationSession,
   MaterializationStatePage,
   MaterializationStateRequestChunk,
+  MaterializationVectorChange,
+  MaterializationVectorChangePage,
   MaterializationWorkEntityKind,
-  MaterializationWorkPage,
   MaterializationWorkRecord,
   ObjectVectorState,
   OntologyCommitOriginSelector,
@@ -526,22 +540,28 @@ export type {
   OntologyOutboxRecord,
   OntologyOutboxStorage,
   OntologyOutboxSummary,
-  OntologyOutboxWrite,
+  OntologyReplacementPlanStorage,
   OntologySourceMaterializationStatus,
   OntologySourceRecord,
   OntologySourceStorage,
   OntologyStorage,
   OntologyVectorStorage,
+  OpenedReplacementPlan,
+  OpenReplacementPlanInput,
+  PlannedReplacementIdentity,
   ProjectionOntologyCommitIntent,
   PurgeAbandonedSourceMaterializationsInput,
   PurgePublishedOntologyOutboxInput,
-  ReadMaterializationObjectExistenceInput,
+  PurgeReplacementPlansInput,
+  ReplacementPlanRef,
+  ReplacementPlanStatePage,
+  ReplacementPlanStatus,
   RescheduleOntologyOutboxLeaseInput,
   SourceActivationWrite,
   SourceReplacementLinkState,
   SourceReplacementObjectState,
-  SourceReplacementStatePage,
   StageMaterializationWorkInput,
+  StageReplacementPlanInput,
   StageSourceAssertion,
   StageSourceRoot,
   StageSourceRowsInput,
@@ -555,12 +575,10 @@ export type {
   StoredSourceObjectAssertion,
   StoredTelemetryPoint,
   StreamMaterializationStateInput,
-  StreamMaterializationWorkInput,
-  StreamSourceReplacementStateInput,
+  StreamMaterializationVectorChangesInput,
+  StreamReplacementPlanStateInput,
   SummarizeOntologyOutboxInput,
-  SummarizeTerminalSourceMaterializationsInput,
   TelemetryOntologyCommitIntent,
-  TerminalSourceMaterializationSummary,
 } from "./ontology"
 export { ONTOLOGY_OUTBOX_FAILURE_CODES } from "./ontology"
 export type {

package/src/storage/internal.ts

@@ -9,6 +9,7 @@ export {
 } from "./action-runs/idempotency"
 export { assertAiUsageExecutionId, normalizeAiModelCallRecord } from "./ai-usage/record"
 export { normalizeAiModelCallUsage } from "./ai-usage/usage"
+export { parseFileDownloadGrantRow } from "./file-download-grants/record"
 export { createFileUploadId, createUploadExpiresAt } from "./file-upload-sessions/utils"
 export { canRequeuePipelineRunAfterEnqueueFailure } from "./pipeline-runs/idempotency"
 export { projectionRunObjectTypesVisible, zeroProjectionRunProgress } from "./projection-runs/types"

package/src/storage/migrations/index.ts

@@ -18,6 +18,7 @@ export type {
   MigrationStep,
   MigrationStepInfo,
   MigrationStepOptions,
+  StorageMigrationOptions,
   StorageMigrationResult,
   StorageMigrator,
 } from "./types"

package/src/storage/migrations/migrate-storage.ts

@@ -1,24 +1,48 @@
+import {
+  isQueryPreparationCapableStorage,
+  type ObjectQueryPreparationResult,
+  prepareObjectQueries,
+} from "../objects/query-preparation"
 import type { Storage } from "../types"
-import type { MigrationCapableStorage, MigrationReport, StorageMigrationResult } from "./types"
+import type {
+  MigrationCapableStorage,
+  MigrationReport,
+  StorageMigrationOptions,
+  StorageMigrationResult,
+} from "./types"
 
 export function isMigrationCapableStorage(storage: Storage): storage is MigrationCapableStorage {
   return Array.isArray((storage as { migrators?: unknown }).migrators)
 }
 
-/** Runs storage adapter migrators when the configured storage exposes them. */
-export async function migrateStorage(storage: Storage): Promise<StorageMigrationResult> {
-  if (!isMigrationCapableStorage(storage) || storage.migrators.length === 0) {
-    return { status: "skipped", reports: [] }
-  }
-
+/** Run schema migrations, then prepare query indexes when the provider supports it. */
+export async function migrateStorage(
+  storage: Storage,
+  options?: StorageMigrationOptions
+): Promise<StorageMigrationResult> {
   const reports: MigrationReport[] = []
+  if (isMigrationCapableStorage(storage)) {
+    for (const migrator of storage.migrators) {
+      reports.push(await migrator.migrate())
+    }
+  }
 
-  for (const migrator of storage.migrators) {
-    reports.push(await migrator.migrate())
+  let queries: ObjectQueryPreparationResult | undefined
+  if (options && isQueryPreparationCapableStorage(storage)) {
+    queries = await prepareObjectQueries({ ...options, storage })
+    for (const warning of queries.warnings) {
+      console.warn(`[Sixb] ${warning}`)
+    }
   }
 
-  return {
-    status: reports.some((report) => report.status === "migrated") ? "migrated" : "current",
-    reports,
+  let status: StorageMigrationResult["status"] = "skipped"
+  if (reports.length > 0 || queries) {
+    status = "current"
   }
+  if (reports.some((report) => report.status === "migrated") || queries?.status === "prepared") {
+    status = "migrated"
+  }
+
+  if (queries) return { status, reports, queries }
+  return { status, reports }
 }

package/src/storage/migrations/types.ts

@@ -1,3 +1,5 @@
+import type { OntologyDefinitionCatalog } from "../../ontology/registry"
+import type { ObjectQueryPreparationResult } from "../objects/query-preparation"
 import type { Storage } from "../types"
 
 export interface MigrationStepInfo {
@@ -123,7 +125,13 @@ export interface MigrationCapableStorage extends Storage {
   readonly migrators: readonly StorageMigrator[]
 }
 
+export interface StorageMigrationOptions {
+  readonly projectId: string
+  readonly ontology: OntologyDefinitionCatalog
+}
+
 export interface StorageMigrationResult {
+  readonly queries?: ObjectQueryPreparationResult
   readonly status: "migrated" | "current" | "skipped"
   readonly reports: readonly MigrationReport[]
 }

package/src/storage/objects/in-memory/query.ts

@@ -117,6 +117,8 @@ export function evaluateObjectQuery(
   source: InMemoryReadSource
 ): QueryEvaluation {
   switch (query.kind) {
+    case "rerank":
+      throw new Error("[Sixb] Reranking must execute in the model runtime, not storage.")
     case "start":
       return evaluateStart(query.objectTypeId, source)
     case "refs":

package/src/storage/objects/in-memory/selected-reader.ts

@@ -105,6 +105,7 @@ export function createInMemorySelectedReader(
       const result = evaluateObjectQuery(stripOuterRowShape(input.query), universe)
       return visible(
         structuredClone({
+          total: result.entries.length,
           facets: buildFacetResults(
             result.entries.map((entry) => entry.row),
             input.facets

package/src/storage/objects/in-memory/storage.ts

@@ -305,6 +305,7 @@ export class InMemoryObjectStorage implements ObjectStorage {
       this.readSource(params.projectId)
     )
     return {
+      total: result.entries.length,
       facets: buildFacetResults(
         result.entries.map((entry) => entry.row),
         params.facets

package/src/storage/objects/query-preparation.ts

@@ -0,0 +1,77 @@
+import type { QueryScalarKind } from "../../objects/query/ir"
+import {
+  queryScalarKindForSchema,
+  resolvePropertyQueryCapabilities,
+  resolveQuerySchema,
+} from "../../ontology/query-capabilities"
+import type { OntologyDefinitionCatalog } from "../../ontology/registry"
+import type { ObjectQueryIndexDefinition } from "../../ontology/types"
+import type { Storage } from "../types"
+
+export interface ObjectQueryPreparationPlan {
+  readonly projectId: string
+  readonly objectTypes: readonly {
+    readonly objectTypeId: string
+    readonly properties: readonly {
+      readonly propertyId: string
+      readonly scalarKind?: QueryScalarKind
+      readonly filterable: boolean
+      readonly sortable: boolean
+      readonly text: boolean
+    }[]
+    readonly indexes: readonly ObjectQueryIndexDefinition[]
+  }[]
+}
+
+export interface ObjectQueryPreparationResult {
+  readonly status: "prepared" | "current"
+  readonly objectTypes: number
+  readonly indexes: number
+  readonly warnings: readonly string[]
+}
+
+/** Optional maintenance capability, separate from object reads and writes. */
+export interface QueryPreparationCapableStorage extends Storage {
+  /** Coordinate concurrent callers and skip completed plans. Backfills may block writes. */
+  prepareObjectQueries(plan: ObjectQueryPreparationPlan): Promise<ObjectQueryPreparationResult>
+}
+
+export function isQueryPreparationCapableStorage(
+  storage: Storage
+): storage is QueryPreparationCapableStorage {
+  return "prepareObjectQueries" in storage && typeof storage.prepareObjectQueries === "function"
+}
+
+/** Prepare declared query access paths after schema migration. Providers own physical layout;
+ * applications describe their ontology, not PostgreSQL expressions or generated columns.
+ * This is maintenance: backfills may block writes and must not run in a request handler. */
+export async function prepareObjectQueries(input: {
+  projectId: string
+  ontology: OntologyDefinitionCatalog
+  storage: Storage
+}): Promise<ObjectQueryPreparationResult> {
+  if (!isQueryPreparationCapableStorage(input.storage)) {
+    throw new Error("[Sixb] This storage provider does not support object query preparation")
+  }
+
+  const values = input.ontology.getValueTypesById()
+  const objectTypes = input.ontology.listObjectTypes().map((type) => ({
+    objectTypeId: type.id,
+    properties: type.properties
+      .filter((property) => property.mode !== "telemetry" && !property.primary)
+      .map((property) => {
+        const capabilities = resolvePropertyQueryCapabilities(property, values)
+        const schema = resolveQuerySchema(property.schema, values)
+        return {
+          propertyId: property.id,
+          scalarKind: schema ? queryScalarKindForSchema(schema) : undefined,
+          filterable: capabilities.operators.includes("eq"),
+          sortable: capabilities.sortable,
+          text: capabilities.text,
+        }
+      })
+      .filter((property) => property.filterable || property.sortable || property.text),
+    indexes: type.query?.indexes ?? [],
+  }))
+  return input.storage.prepareObjectQueries({ projectId: input.projectId, objectTypes })
+}

package/src/storage/objects/types.ts

@@ -261,6 +261,8 @@ export interface FacetObjectsInput {
 }
 
 export interface FacetObjectsResult {
+  /** Exact matching rows, including absent facet values and buckets beyond the limit. */
+  total: number
   facets: readonly ObjectFacetResult[]
 }
 

package/src/storage/ontology/commits.ts

@@ -1,5 +1,3 @@
-import type { AuthorizablePrincipal } from "../../execution/types"
-import type { EventExecutor } from "../../materialization/events"
 import type {
   EditCommitResult,
   OntologyMaterializationOrigin,
@@ -56,12 +54,9 @@ interface OntologyCommitFields {
   readonly id: string
   readonly idempotencyKey: string
   readonly requestHash: string
+  /** The execution the commit ran under: who asked for it and what wrote it live there. */
   readonly executionId: string
   readonly origin: OntologyMaterializationOrigin
-  /** Principal on whose behalf the execution ran, copied from `ExecutionRecord.requestedBy`. */
-  readonly requestedBy?: AuthorizablePrincipal
-  /** Workload that wrote the commit, copied from its execution. */
-  readonly executor: EventExecutor
   readonly ontologyRevision: string
   readonly projectionRevision?: string
   readonly ownershipHash?: string

package/src/storage/ontology/in-memory/index.ts

@@ -7,14 +7,17 @@ import type { AssertSourceMaterializationExecution } from "../sources"
 import { InMemoryOntologyCommitStorage } from "./commits"
 import { InMemoryOntologyMaterializationStorage } from "./materializations"
 import { InMemoryOntologyOutboxStorage } from "./outbox"
+import { InMemoryOntologyReplacementPlanStorage } from "./replacement-plans"
 import {
   cloneOntologyState,
   createInMemoryOntologyState,
+  type InMemoryExecutionReader,
   type InMemoryOntologyState,
   type InMemoryOntologyStorageTestHooks,
   restoreOntologyState,
 } from "./shared-state"
 import { InMemoryOntologySourceStorage } from "./sources"
+import { InMemoryMaterializationStateReader } from "./state-reader"
 import {
   type InMemoryOntologyStorageTestingAdapter,
   registerInMemoryOntologyStorageTestingAdapter,
@@ -27,7 +30,7 @@ interface InMemoryOntologyStorageOptions {
   readonly getTransactionToken: () => object | null
   readonly getMaterializationLifecycle: () => ProviderMaterializationTransactionLifecycle | null
   readonly assertSourceMaterializationExecution: AssertSourceMaterializationExecution
-  readonly executionExists: (projectId: string, executionId: string) => Promise<boolean>
+  readonly readExecution: InMemoryExecutionReader
 }
 
 import { InMemoryVectorIndexingStorage } from "./vector-indexing"
@@ -40,6 +43,7 @@ export class InMemoryOntologyStorage implements OntologyStorage {
   readonly commits: InMemoryOntologyCommitStorage
   readonly sources: InMemoryOntologySourceStorage
   readonly materializations: InMemoryOntologyMaterializationStorage
+  readonly replacementPlans: InMemoryOntologyReplacementPlanStorage
   readonly outbox: InMemoryOntologyOutboxStorage
   private testHooks: InMemoryOntologyStorageTestHooks = {}
   private readonly testingAdapter: InMemoryOntologyStorageTestingAdapter = {
@@ -72,20 +76,35 @@ export class InMemoryOntologyStorage implements OntologyStorage {
       options.runRootOperation,
       options.assertSourceMaterializationExecution
     )
-    this.outbox = new InMemoryOntologyOutboxStorage(this.state, options.runRootOperation)
+    this.outbox = new InMemoryOntologyOutboxStorage(
+      this.state,
+      options.runRootOperation,
+      options.readExecution
+    )
+    const hooks: InMemoryOntologyStorageTestHooks = {
+      beforeRead: (boundary) => this.testHooks.beforeRead?.(boundary),
+      beforeWrite: (boundary, ordinal) => this.testHooks.beforeWrite?.(boundary, ordinal),
+      observeBuffer: (boundary, rows) => this.testHooks.observeBuffer?.(boundary, rows),
+      observeWork: (records) => this.testHooks.observeWork?.(records),
+    }
+    const reader = new InMemoryMaterializationStateReader(this.state, objects, timeseries, hooks)
     this.materializations = new InMemoryOntologyMaterializationStorage(
       this.state,
       objects,
       timeseries,
+      reader,
       options.getTransactionToken,
       options.getMaterializationLifecycle,
-      options.executionExists,
-      {
-        beforeRead: (boundary) => this.testHooks.beforeRead?.(boundary),
-        beforeWrite: (boundary, ordinal) => this.testHooks.beforeWrite?.(boundary, ordinal),
-        observeBuffer: (boundary, rows) => this.testHooks.observeBuffer?.(boundary, rows),
-        observeWork: (records) => this.testHooks.observeWork?.(records),
-      }
+      options.readExecution,
+      hooks
+    )
+    this.replacementPlans = new InMemoryOntologyReplacementPlanStorage(
+      this.state,
+      objects,
+      reader,
+      options.runRootOperation,
+      options.assertSourceMaterializationExecution,
+      hooks
     )
     registerInMemoryOntologyStorageTestingAdapter(this, this.testingAdapter)
   }