@sixb/core 0.1.16 → 0.1.17

package/src/blob-storage/download-urls.ts

@@ -0,0 +1,138 @@
+import { createHash, randomBytes, randomUUID } from "node:crypto"
+import type { FileDownloadGrantRecord, FileDownloadGrantStorage, Storage } from "../storage"
+import { BlobStorageError } from "./errors"
+import type { BlobStorage, FileRef } from "./types"
+import { isFileRef } from "./validation"
+
+/**
+ * Download URLs let a third party that cannot hold a Sixb session — a social network fetching
+ * media to publish, for instance — read one file for a bounded time.
+ *
+ * Each URL carries a random token. Storage keeps only its SHA-256, with the file, the expiry, and
+ * the execution that issued it. The API looks the token up on every request, so any process
+ * sharing the storage can issue a URL, and a URL can be revoked before it expires.
+ *
+ * The API serves the bytes itself rather than redirecting to the blob provider: fetchers such as
+ * TikTok refuse redirects and accept only hosts whose ownership the developer verified.
+ */
+
+/** Path prefix of the public route that serves download URLs. */
+export const FILE_DOWNLOAD_ROUTE_PREFIX = "/api/files/downloads/"
+
+const DEFAULT_EXPIRY_MS = 60 * 60 * 1000
+// SQLite compares ISO timestamps as text, which orders them correctly only for four-digit years.
+const LATEST_EXPIRY_MS = Date.UTC(9999, 11, 31, 23, 59, 59, 999)
+/** 32 random bytes in base64url. */
+const TOKEN_PATTERN = /^[A-Za-z0-9_-]{43}$/
+
+export interface CreateFileDownloadUrlOptions {
+  /** How long the URL stays valid. Defaults to one hour. */
+  readonly expiresInMs?: number
+}
+
+export interface FileDownloadUrl {
+  /** Pass to `blobs.revokeDownloadUrl()` to stop the URL before it expires. */
+  readonly id: string
+  readonly url: string
+  readonly expiresAt: Date
+}
+
+/** What issuing and revoking need from the execution that asks. */
+export interface FileDownloadUrlContext {
+  readonly projectId: string
+  readonly storage: Storage
+  readonly blobStorage: BlobStorage
+  readonly executionId: string
+  readonly apiPublicOrigin: string | undefined
+}
+
+export async function createFileDownloadUrl(
+  context: FileDownloadUrlContext,
+  file: FileRef,
+  options: CreateFileDownloadUrlOptions = {}
+): Promise<FileDownloadUrl> {
+  if (!isFileRef(file)) {
+    throw new Error("[Sixb] blobs.createDownloadUrl() requires a valid file reference.")
+  }
+  const createdAt = new Date()
+  const expiresAt = expiryFrom(createdAt, options.expiresInMs ?? DEFAULT_EXPIRY_MS)
+  const grants = requireGrantStorage(context.storage)
+  if (context.apiPublicOrigin === undefined) {
+    throw new Error(
+      "[Sixb] blobs.createDownloadUrl() needs the API's public origin. Set SIXB_API_PUBLIC_ORIGIN for the API and every worker."
+    )
+  }
+  // Fail here, not when a third party fetches the URL minutes later and gets a bare 404.
+  const stat = await context.blobStorage.stat(file.blobId)
+  if (!stat || stat.digest !== file.digest || stat.sizeBytes !== file.sizeBytes) {
+    throw new BlobStorageError(`[Sixb] No stored blob matches file '${file.blobId}'.`)
+  }
+
+  const token = randomBytes(32).toString("base64url")
+  const { logicalPath: _logicalPath, ...served } = file
+  const grant: FileDownloadGrantRecord = {
+    id: `filedownload_${randomUUID()}`,
+    projectId: context.projectId,
+    tokenHash: hashToken(token),
+    file: served,
+    executionId: context.executionId,
+    createdAt,
+    expiresAt,
+  }
+  await grants.create(grant)
+
+  return {
+    id: grant.id,
+    url: new URL(`${FILE_DOWNLOAD_ROUTE_PREFIX}${token}`, context.apiPublicOrigin).toString(),
+    expiresAt: grant.expiresAt,
+  }
+}
+
+export async function revokeFileDownloadUrl(
+  context: FileDownloadUrlContext,
+  id: string
+): Promise<void> {
+  const revoked = await requireGrantStorage(context.storage).revoke({
+    projectId: context.projectId,
+    id,
+    revokedAt: new Date(),
+  })
+  if (!revoked) {
+    throw new Error(`[Sixb] Download URL '${id}' does not exist.`)
+  }
+}
+
+/** The file a token serves, or `null` when the token is unknown, expired, or revoked. */
+export async function resolveFileDownload(
+  host: { readonly id: string; readonly storage: Storage },
+  token: string
+): Promise<FileRef | null> {
+  const grants = host.storage.fileDownloadGrants
+  if (!grants || !TOKEN_PATTERN.test(token)) return null
+
+  const grant = await grants.findByTokenHash({ projectId: host.id, tokenHash: hashToken(token) })
+  if (!grant || grant.revokedAt || grant.expiresAt.getTime() <= Date.now()) return null
+  return grant.file
+}
+
+function hashToken(token: string): string {
+  return createHash("sha256").update(token).digest("hex")
+}
+
+function requireGrantStorage(storage: Storage): FileDownloadGrantStorage {
+  if (!storage.fileDownloadGrants) {
+    throw new Error("[Sixb] Download URLs require storage.fileDownloadGrants.")
+  }
+  return storage.fileDownloadGrants
+}
+
+function expiryFrom(createdAt: Date, expiresInMs: number): Date {
+  if (!Number.isSafeInteger(expiresInMs) || expiresInMs <= 0) {
+    throw new Error("[Sixb] expiresInMs must be a positive integer.")
+  }
+  const expiresAt = createdAt.getTime() + expiresInMs
+  if (expiresAt > LATEST_EXPIRY_MS) {
+    throw new Error("[Sixb] expiresInMs must end before the year 10000.")
+  }
+  return new Date(expiresAt)
+}

package/src/blob-storage/execution.ts

@@ -1,6 +1,13 @@
 import { assertProviderAccess } from "../authorization"
 import type { ExecutionContext } from "../execution"
 import type { SixbRuntimeContext } from "../runtime/types"
+import {
+  type CreateFileDownloadUrlOptions,
+  createFileDownloadUrl,
+  type FileDownloadUrl,
+  type FileDownloadUrlContext,
+  revokeFileDownloadUrl,
+} from "./download-urls"
 import type {
   AbortBlobUploadInput,
   BlobByteRange,
@@ -21,6 +28,13 @@ export interface BlobsRuntime {
   put(input: PutBlobInput): Promise<FileRef>
   open(blobId: string): Promise<ReadableStream<Uint8Array>>
   stat(blobId: string): Promise<BlobInfo | null>
+  /**
+   * A URL anyone can use to read this file until it expires or is revoked, for services that
+   * fetch media themselves. Requires the API's public origin (`SIXB_API_PUBLIC_ORIGIN`).
+   */
+  createDownloadUrl(file: FileRef, options?: CreateFileDownloadUrlOptions): Promise<FileDownloadUrl>
+  /** Stop a URL from `createDownloadUrl()` before it expires. */
+  revokeDownloadUrl(id: string): Promise<void>
   openRange?(blobId: string, range: BlobByteRange): Promise<ReadableStream<Uint8Array>>
   createUpload?(input: CreateBlobUploadInput): Promise<BlobUploadSession>
   signUploadPart?(input: SignBlobUploadPartInput): Promise<SignedBlobUploadPart>
@@ -31,9 +45,18 @@ export interface BlobsRuntime {
 export function createBlobsRuntime(
   runtime: SixbRuntimeContext,
   execution: ExecutionContext,
-  blobStorage: BlobStorage
+  blobStorage: BlobStorage,
+  apiPublicOrigin: () => string | undefined
 ): BlobsRuntime {
   const assertAccess = () => assertProviderAccess(runtime, execution, "blobs.access")
+  // Read the origin per call: the server or CLI may record it after this SDK was bound.
+  const downloadUrlContext = (): FileDownloadUrlContext => ({
+    projectId: runtime.projectId,
+    storage: runtime.storage,
+    blobStorage,
+    executionId: execution.id,
+    apiPublicOrigin: apiPublicOrigin(),
+  })
   const executionBlobs: BlobsRuntime = {
     put: (input) => {
       assertAccess()
@@ -47,6 +70,14 @@ export function createBlobsRuntime(
       assertAccess()
       return blobStorage.stat(blobId)
     },
+    createDownloadUrl: async (file, options) => {
+      assertAccess()
+      return createFileDownloadUrl(downloadUrlContext(), file, options)
+    },
+    revokeDownloadUrl: async (id) => {
+      assertAccess()
+      return revokeFileDownloadUrl(downloadUrlContext(), id)
+    },
   }
 
   if (supportsRangeRead(blobStorage)) {

package/src/blob-storage/internal.ts

@@ -0,0 +1 @@
+export { FILE_DOWNLOAD_ROUTE_PREFIX, resolveFileDownload } from "./download-urls"

package/src/execution/authorized-object-reader.ts

@@ -2,7 +2,7 @@ import { assertNever } from "../assert-never"
 import { assertAuthorized, isAllowed } from "../authorization"
 import { AuthorizationError } from "../authorization/errors"
 import type { AuthorizationContext } from "../authorization/types"
-import type { EmbeddingModelCatalog } from "../models/catalog"
+import type { EmbeddingModelCatalog, RerankingModelCatalog } from "../models/catalog"
 import {
   type AuthorizedOntologySelection,
   type AuthorizedOntologyView,
@@ -87,6 +87,7 @@ class AuthorizedObjectReaderImpl {
   readonly #runtime: RuntimeReadAuthorization
   readonly #ontology: OntologyRegistry
   readonly #embeddingModels?: EmbeddingModelCatalog
+  readonly #rerankingModels?: RerankingModelCatalog
   #ontologyView?: AuthorizedOntologyView
   readonly #storage: ObjectReadStorage
   readonly #delegatedObjectTypeIds?: ReadonlySet<string>
@@ -99,6 +100,7 @@ class AuthorizedObjectReaderImpl {
       readonly scope: ExecutionScope
       readonly ontology: OntologyRegistry
       readonly embeddingModels?: EmbeddingModelCatalog
+      readonly rerankingModels?: RerankingModelCatalog
       readonly storage: ObjectReadStorage
       readonly authority: ResolvedExecutionAuthority
     }
@@ -111,6 +113,7 @@ class AuthorizedObjectReaderImpl {
     this.#authority = input.authority
     this.#ontology = input.ontology
     this.#embeddingModels = input.embeddingModels
+    this.#rerankingModels = input.rerankingModels
     this.#storage = input.storage
     this.#delegatedObjectTypeIds =
       input.authority.type === "delegated"
@@ -469,11 +472,13 @@ class AuthorizedObjectReaderImpl {
         ontology: this.#ontology,
         storage: this.#storage,
         embeddingModels: this.#embeddingModels,
+        rerankingModels: this.#rerankingModels,
       }
     }
     return {
       ontology: this.#ontology,
       embeddingModels: this.#embeddingModels,
+      rerankingModels: this.#rerankingModels,
       storage: this.#storage,
       runtimeAuthorization: this.#runtime.runtimeAuthorization,
       ...(this.#runtime.authorization === undefined
@@ -558,6 +563,7 @@ export function createAuthorizedObjectReader(input: {
   readonly scope: ExecutionScope
   readonly ontology: OntologyRegistry
   readonly embeddingModels?: EmbeddingModelCatalog
+  readonly rerankingModels?: RerankingModelCatalog
   readonly objectStorage: ObjectStorage
 }): AuthorizedObjectReader {
   const scope = captureExecutionScope(input.scope)
@@ -568,6 +574,7 @@ export function createAuthorizedObjectReader(input: {
     scope,
     ontology: input.ontology,
     embeddingModels: input.embeddingModels,
+    rerankingModels: input.rerankingModels,
     storage,
     authority,
   })

package/src/index.ts

@@ -21,10 +21,12 @@ export type {
   ObjectLinkTargetMetadata,
   ObjectLinkTargetType,
   ObjectPropertiesMetadata,
+  ObjectQueryIndexDefinition,
   ObjectRef,
   ObjectRefSchema,
   ObjectSchema,
   ObjectType,
+  ObjectTypeQueryMetadata,
   ObjectTypeSearchMetadata,
   ObjectTypeWithPropertyTokens,
   ObjectTypeWithTokens,
@@ -536,6 +538,7 @@ export type {
   MigrationState,
   MigrationStatus,
   Storage,
+  StorageMigrationOptions,
   StorageMigrationResult,
   StorageMigrator,
   StorageSchemaCheck,
@@ -886,6 +889,7 @@ export type {
   AgentThreadsRuntime,
   ExecutionAgentRunResult,
 } from "./agents/execution"
+export type { CreateFileDownloadUrlOptions, FileDownloadUrl } from "./blob-storage/download-urls"
 export type { BlobsRuntime } from "./blob-storage/execution"
 export type { ConnectorRuntime } from "./connectors/execution"
 export type { DatasetIngestInput, DatasetIngestResult, DatasetsRuntime } from "./datasets/execution"
@@ -949,6 +953,7 @@ export type {
   ObjectQueryFacetBucket,
   ObjectQueryFacetInput,
   ObjectQueryFacetResult,
+  ObjectQueryFacetsResult,
   ObjectQueryListOptions,
   ObjectQueryRow,
   ObjectReadByIdHandle,
@@ -1120,6 +1125,15 @@ export type {
   ModelReportedCost,
   ModelRoute,
   ModelUsage,
+  RerankingModel,
+  RerankingModelCatalog,
+  RerankingModelDefinition,
+  RerankingModelEntry,
+  RerankingModelRef,
+  RerankingModelRequest,
+  RerankingModelResponseMetadata,
+  RerankingModelResult,
+  RerankingResult,
 } from "./models"
 export type {
   InferLanguageModelOutput,
@@ -1142,6 +1156,12 @@ export type {
   ActionParamDescriptor,
   ActionPhaseDescriptor,
 } from "./actions/descriptor"
+export type {
+  AudioModelsRuntime,
+  AudioTranscribeInput,
+  AudioTranscribeResult,
+  TranscriptionOutput,
+} from "./models/audio/types"
 export type { DecisionAnswerSchema, DecisionOutputShape } from "./models/decision/output"
 export { decisionOutput } from "./models/decision/output"
 export { question } from "./models/decision/questions"
@@ -1183,3 +1203,8 @@ export {
   isProjectionDefinition,
   isTelemetryProjectionDefinition,
 } from "./projections"
+export type {
+  ObjectQueryPreparationPlan,
+  ObjectQueryPreparationResult,
+} from "./storage/objects/query-preparation"
+export { prepareObjectQueries } from "./storage/objects/query-preparation"

package/src/lake-storage/in-memory.ts

@@ -29,6 +29,7 @@ import type {
   CommitDatasetWriteInput,
   DatasetCatalogState,
   DatasetChanges,
+  DatasetLatestVersionSummary,
   DatasetRow,
   DatasetRowChange,
   DatasetVersion,
@@ -36,6 +37,8 @@ import type {
   DatasetWriteMode,
   LakeStorage,
   LakeWriteSession,
+  LatestVersionsSince,
+  ListLatestVersionsSinceInput,
   ReadDatasetChangesInput,
   ReadDatasetRowsInput,
 } from "./types"
@@ -52,6 +55,16 @@ function cloneDatasetVersion(version: DatasetVersion): DatasetVersion {
   }
 }
 
+function latestVersionSummary(version: DatasetVersion): DatasetLatestVersionSummary {
+  return {
+    datasetId: version.datasetId,
+    versionId: version.versionId,
+    mode: version.mode,
+    createdAt: version.createdAt,
+    ...(version.rowCount !== undefined ? { rowCount: version.rowCount } : {}),
+  }
+}
+
 function cloneRow(row: DatasetRow): DatasetRow {
   return structuredClone(row)
 }
@@ -223,6 +236,9 @@ export class InMemoryLakeStorage implements LakeStorage {
   private readonly sequencesByVersionId = new Map<string, Map<string, DatasetSequenceState>>()
   private readonly latestVersionIdByDataset = new Map<string, string>()
   private readonly commitLocks = new Map<string, Promise<void>>()
+  /** Commit counter for listLatestVersionsSince; each dataset keeps the position of its latest. */
+  private commitPosition = 0
+  private readonly commitPositionByDataset = new Map<string, number>()
 
   async createDataset(definition: DatasetDefinition): Promise<DatasetDefinition> {
     assertDatasetId(definition.id)
@@ -287,15 +303,7 @@ export class InMemoryLakeStorage implements LakeStorage {
         return {
           datasetId,
           materialized: true,
-          latestVersion: latest
-            ? {
-                datasetId: latest.datasetId,
-                versionId: latest.versionId,
-                mode: latest.mode,
-                createdAt: latest.createdAt,
-                ...(latest.rowCount !== undefined ? { rowCount: latest.rowCount } : {}),
-              }
-            : null,
+          latestVersion: latest ? latestVersionSummary(latest) : null,
         }
       })
     )
@@ -362,6 +370,25 @@ export class InMemoryLakeStorage implements LakeStorage {
     return this.getVersion(datasetId, latestVersionId)
   }
 
+  async listLatestVersionsSince(
+    input: ListLatestVersionsSinceInput
+  ): Promise<LatestVersionsSince | null> {
+    const cursor = String(this.commitPosition)
+    if (input.cursor === null) return { cursor, versions: [] }
+    if (!/^\d+$/.test(input.cursor) || Number(input.cursor) > this.commitPosition) return null
+
+    const after = Number(input.cursor)
+    const datasetIds = input.datasetIds ? new Set(input.datasetIds) : null
+    const versions: DatasetLatestVersionSummary[] = []
+    for (const [datasetId, position] of this.commitPositionByDataset) {
+      if (position <= after || (datasetIds && !datasetIds.has(datasetId))) continue
+      // This provider never commits schema-only versions, so the latest version carries data.
+      const latest = await this.getLatestVersion(datasetId)
+      if (latest) versions.push(latestVersionSummary(latest))
+    }
+    return { cursor, versions }
+  }
+
   async getVersion(datasetId: string, versionId: string): Promise<DatasetVersion | null> {
     const versions = this.versionsByDataset.get(datasetId) ?? []
     const version = versions.find((candidate) => candidate.versionId === versionId)
@@ -564,6 +591,7 @@ export class InMemoryLakeStorage implements LakeStorage {
         visibleRows.map((row) => cloneRow(row))
       )
       this.latestVersionIdByDataset.set(options.merge.dataset.id, versionId)
+      this.commitPositionByDataset.set(options.merge.dataset.id, ++this.commitPosition)
       if (ordered) this.sequencesByVersionId.set(versionId, ordered.states)
 
       return { outcome: "created", version: cloneDatasetVersion(version) }
@@ -636,6 +664,7 @@ export class InMemoryLakeStorage implements LakeStorage {
       visibleRows.map((row) => cloneRow(row))
     )
     this.latestVersionIdByDataset.set(options.write.dataset.id, versionId)
+    this.commitPositionByDataset.set(options.write.dataset.id, ++this.commitPosition)
 
     return { ...cloneDatasetVersion(version), outcome: "created" }
   }

package/src/lake-storage/index.ts

@@ -69,6 +69,8 @@ export type {
   LakeStandardId,
   LakeStorage,
   LakeWriteSession,
+  LatestVersionsSince,
+  ListLatestVersionsSinceInput,
   ReadDatasetChangesInput,
   ReadDatasetRowsInput,
 } from "./types"

package/src/lake-storage/types.ts

@@ -53,6 +53,24 @@ export interface DatasetCatalogState {
   readonly latestVersion?: DatasetLatestVersionSummary | null
 }
 
+export interface ListLatestVersionsSinceInput {
+  /** A cursor returned by an earlier call, or null to start at the lake's current position. */
+  readonly cursor: string | null
+  /** Report only these datasets. Omit to report every dataset. */
+  readonly datasetIds?: readonly string[]
+}
+
+export interface LatestVersionsSince {
+  /** Opaque position to pass to the next call. */
+  readonly cursor: string
+  /**
+   * The newest data-bearing (non-`schema`) version of each dataset that committed one after the
+   * input cursor and up to the returned one, in no particular order. A provider may stop short of
+   * its newest commit to bound one call; the next call continues. Empty for a null input cursor.
+   */
+  readonly versions: readonly DatasetLatestVersionSummary[]
+}
+
 export interface BeginDatasetWriteInput {
   readonly dataset: DatasetDefinition
   readonly mode?: DatasetWriteMode
@@ -170,6 +188,18 @@ export interface LakeStorage {
 
   getLatestVersion(datasetId: string): Promise<DatasetVersion | null>
   getVersion(datasetId: string, versionId: string): Promise<DatasetVersion | null>
+
+  /**
+   * Change feed: which datasets committed data since a cursor. Consumers that must not miss a
+   * commit, such as projection reconciliation, poll it instead of reading every dataset's latest
+   * version. Its cost must not grow with the lake's history: an idle read should be about as
+   * cheap as one metadata lookup.
+   *
+   * Null means the cursor is no longer a position in this lake (for example, the catalog was
+   * recreated or the cursor's commit expired): read current state, then restart from null.
+   */
+  listLatestVersionsSince(input: ListLatestVersionsSinceInput): Promise<LatestVersionsSince | null>
+
   /** Explicit immutable-version reads have stable physical order and stable offset semantics. */
   readRows(input: ReadDatasetRowsInput): AsyncIterable<DatasetRow>
 

package/src/maintenance/ontology-maintenance.ts

@@ -1,5 +1,6 @@
 import { type BlobStorage, supportsDirectUpload } from "../blob-storage"
 import type { OntologyOutboxDispatcher } from "../events"
+import type { VectorIndexingDispatcher } from "../objects/vectors/indexing-dispatch"
 import {
   type FileUploadSession,
   FileUploadSessionError,
@@ -27,11 +28,15 @@ const ABANDONED_UPLOAD_ABORT_GRACE_MS = 15 * 60_000
 // One abandoned candidate of a large projection holds hundreds of thousands of rows. Batches this
 // size keep each delete short while clearing such a candidate within a pass.
 const ABANDONED_SOURCE_PURGE_BATCH = 50_000
+// An expired download URL serves nothing, but its grant records which execution exposed which
+// file. It stays this long for that audit trail.
+const EXPIRED_FILE_DOWNLOAD_RETENTION_MS = 7 * 24 * 60 * 60_000
 
 interface OntologyMaintenanceDependencies {
   readonly projectId: string
   readonly storage: Storage
   readonly dispatcher: OntologyOutboxDispatcher
+  readonly vectorIndexing?: VectorIndexingDispatcher
   /** Aborts the provider side of abandoned staged uploads. */
   readonly blobStorage?: BlobStorage
   readonly options?: OntologyMaintenanceOptions
@@ -44,6 +49,7 @@ export class OntologyMaintenance {
   private readonly projectId: string
   private readonly storage: Storage
   private readonly dispatcher: OntologyOutboxDispatcher
+  private readonly vectorIndexing: VectorIndexingDispatcher | undefined
   private readonly blobStorage: BlobStorage | undefined
   private readonly intervalMs: number
   private readonly publishedOutboxRetentionMs: number
@@ -64,6 +70,7 @@ export class OntologyMaintenance {
     this.projectId = dependencies.projectId
     this.storage = dependencies.storage
     this.dispatcher = dependencies.dispatcher
+    this.vectorIndexing = dependencies.vectorIndexing
     this.blobStorage = dependencies.blobStorage
     this.intervalMs = positiveInteger(options.intervalMs ?? DEFAULT_INTERVAL_MS, "intervalMs")
     this.publishedOutboxRetentionMs = nonnegativeInteger(
@@ -158,9 +165,15 @@ export class OntologyMaintenance {
 
     const failures: unknown[] = []
     await captureFailure(() => this.dispatcher.drain(), failures)
+    const vectorIndexing = this.vectorIndexing
+    if (vectorIndexing) await captureFailure(() => vectorIndexing.drain(), failures)
     const cleanup = await this.cleanupExpiredRows(started, failures)
-    const { outbox, terminalSources } = await this.readOperationalSummaries(failures)
-    this.completePass({ started, failures, cleanup, outbox, terminalSources })
+    const outbox = await captureFailure(
+      () => this.storage.ontology.outbox.summarize({ projectId: this.projectId }),
+      failures,
+      this.snapshot.outbox
+    )
+    this.completePass({ started, failures, cleanup, outbox })
   }
 
   private async cleanupExpiredRows(
@@ -179,6 +192,8 @@ export class OntologyMaintenance {
       failures,
       0
     )
+    // Plans go first: a source version is cleaned only once its plan is gone.
+    const spentPlanRows = await captureFailure(() => this.purgeSpentPlans(started), failures, 0)
     const terminalSources = await captureFailure(
       () =>
         this.storage.ontology.sources.cleanupTerminal({
@@ -202,10 +217,24 @@ export class OntologyMaintenance {
       await captureFailure(() => this.retireAbandonedUploads(uploads, started, failures), failures)
       await captureFailure(() => uploads.cleanupExpired(started), failures, 0)
     }
+    const downloads = this.storage.fileDownloadGrants
+    if (downloads) {
+      await captureFailure(
+        () =>
+          downloads.deleteExpired({
+            projectId: this.projectId,
+            expiredBefore: new Date(started.getTime() - EXPIRED_FILE_DOWNLOAD_RETENTION_MS),
+            limit: this.cleanupLimit,
+          }),
+        failures,
+        0
+      )
+    }
 
     return {
       publishedOutboxRowsDeleted,
-      terminalSourceRowsDeleted: terminalSources.rowsDeleted + abandonedSources.rowsDeleted,
+      terminalSourceRowsDeleted:
+        terminalSources.rowsDeleted + abandonedSources.rowsDeleted + spentPlanRows,
       terminalSourceMaterializationsDeleted:
         terminalSources.materializationsDeleted + abandonedSources.materializationsDeleted,
     }
@@ -232,6 +261,24 @@ export class OntologyMaintenance {
     }
   }
 
+  /** Deletes the plans of candidates that left `ready`, batch after batch, like candidates. */
+  private async purgeSpentPlans(started: Date): Promise<number> {
+    let rowsDeleted = 0
+    while (true) {
+      const purged = await this.storage.ontology.replacementPlans.purge({
+        projectId: this.projectId,
+        limit: ABANDONED_SOURCE_PURGE_BATCH,
+      })
+      rowsDeleted += purged
+      if (
+        purged < ABANDONED_SOURCE_PURGE_BATCH ||
+        this.now().getTime() - started.getTime() >= this.intervalMs
+      ) {
+        return rowsDeleted
+      }
+    }
+  }
+
   private async retireAbandonedUploads(
     uploads: FileUploadSessionStore,
     now: Date,
@@ -288,29 +335,11 @@ export class OntologyMaintenance {
     if (abortFailure !== undefined) throw abortFailure
   }
 
-  private async readOperationalSummaries(failures: unknown[]): Promise<{
-    readonly outbox: OntologyMaintenanceSnapshot["outbox"]
-    readonly terminalSources: OntologyMaintenanceSnapshot["terminalSources"]
-  }> {
-    const outbox = await captureFailure(
-      () => this.storage.ontology.outbox.summarize({ projectId: this.projectId }),
-      failures,
-      this.snapshot.outbox
-    )
-    const terminalSummary = await captureFailure(
-      () => this.storage.ontology.sources.summarizeTerminal({ projectId: this.projectId }),
-      failures,
-      this.snapshot.terminalSources
-    )
-    return { outbox, terminalSources: terminalSummary }
-  }
-
   private completePass(input: {
     readonly started: Date
     readonly failures: readonly unknown[]
     readonly cleanup: OntologyMaintenanceCleanupSnapshot
     readonly outbox: OntologyMaintenanceSnapshot["outbox"]
-    readonly terminalSources: OntologyMaintenanceSnapshot["terminalSources"]
   }): void {
     const completed = this.now()
     const failure =
@@ -326,7 +355,6 @@ export class OntologyMaintenance {
       consecutiveFailures: failure ? this.snapshot.consecutiveFailures + 1 : 0,
       lastError: failure ? errorMessage(failure) : null,
       outbox: input.outbox,
-      terminalSources: input.terminalSources,
       cleanup: input.cleanup,
     }
     if (failure) this.reportError(failure)
@@ -386,7 +414,6 @@ function emptySnapshot(intervalMs: number): OntologyMaintenanceSnapshot {
     consecutiveFailures: 0,
     lastError: null,
     outbox: null,
-    terminalSources: null,
     cleanup: null,
   }
 }

package/src/maintenance/types.ts

@@ -19,6 +19,7 @@ export interface OntologyMaintenanceOptions {
 
 export interface OntologyMaintenanceCleanupSnapshot {
   readonly publishedOutboxRowsDeleted: number
+  /** Rows of candidates that will not be published again, their replacement plans included. */
   readonly terminalSourceRowsDeleted: number
   readonly terminalSourceMaterializationsDeleted: number
 }
@@ -37,10 +38,6 @@ export interface OntologyMaintenanceSnapshot {
     readonly retryingCount: number
     readonly maxAttempts: number
   } | null
-  readonly terminalSources: {
-    readonly count: number
-    readonly oldestTerminalAt: string | null
-  } | null
   readonly cleanup: OntologyMaintenanceCleanupSnapshot | null
 }