@sixb/core 0.0.1

package/src/storage/projection-runs/types.ts

@@ -0,0 +1,254 @@
+import type {
+  PinnedDatasetVersion,
+  ProjectionExecution,
+  ProjectionMaterializationIdentity,
+  ProjectionRunTerminalDecision,
+} from "../../materialization/model"
+import type {
+  LinkProjectionTarget,
+  ObjectProjectionTarget,
+  ProjectionKind,
+  ProjectionTarget,
+} from "../../projections/types"
+
+export type ProjectionRunStatus = "running" | "succeeded" | "failed" | "cancelled"
+
+export interface ProjectionRunProgress {
+  /** Physical immutable-version rows consumed, before normalization or semantic materialization. */
+  readonly sourceRowsRead: number
+  /** Physical rows intentionally skipped because a required mapped value was blank. */
+  readonly sourceRowsSkipped: number
+}
+
+const progressKeyFlags: Record<keyof ProjectionRunProgress, true> = {
+  sourceRowsRead: true,
+  sourceRowsSkipped: true,
+}
+
+export const PROJECTION_RUN_PROGRESS_KEYS = Object.keys(
+  progressKeyFlags
+) as readonly (keyof ProjectionRunProgress)[]
+
+export function zeroProjectionRunProgress(): ProjectionRunProgress {
+  const progress = {} as Record<keyof ProjectionRunProgress, number>
+  for (const key of PROJECTION_RUN_PROGRESS_KEYS) progress[key] = 0
+  return progress
+}
+
+export interface ProjectionTelemetryCheckpoint {
+  readonly fixedBatchSize: number
+  readonly nextBatchOrdinal: number
+  readonly nextRowOffset: number
+  readonly inputExhausted: boolean
+}
+
+/**
+ * The object a telemetry batch is waiting for, and when this run first found it missing.
+ *
+ * A projection's dataset can legitimately be materialized before the objects it references, so
+ * the run waits instead of failing. The wait needs a start, and the run has no other durable one:
+ * `startedAt` is when the run began, which on a batched projection is long before the batch that
+ * cannot commit, and `attempt` counts every reclaim including the ones an unreachable queue caused.
+ *
+ * Cleared by any committed batch, so a run that is making progress never carries a stale wait.
+ */
+export interface ProjectionMissingTarget {
+  readonly objectTypeId: string
+  readonly objectId: string
+  /** The batch that could not commit because of it. */
+  readonly batchOrdinal: number
+  readonly firstSeenAt: Date
+}
+
+interface ProjectionRunRecordBase {
+  readonly id: string
+  readonly projectId: string
+  readonly status: ProjectionRunStatus
+  readonly attempt: number
+  readonly progress: ProjectionRunProgress
+  readonly startedAt: Date
+  readonly finishedAt?: Date
+  readonly errorMessage?: string
+}
+
+type ProjectionIdentity<TKind extends ProjectionKind> = Extract<
+  ProjectionMaterializationIdentity,
+  { readonly projectionKind: TKind }
+>
+
+export type ObjectProjectionRunRecord = ProjectionRunRecordBase & {
+  readonly identity: ProjectionIdentity<"object">
+  readonly target: ObjectProjectionTarget
+  readonly telemetryCheckpoint?: never
+  readonly missingTarget?: never
+}
+
+export type LinkProjectionRunRecord = ProjectionRunRecordBase & {
+  readonly identity: ProjectionIdentity<"link">
+  readonly target: LinkProjectionTarget
+  readonly telemetryCheckpoint?: never
+  readonly missingTarget?: never
+}
+
+export type TelemetryProjectionRunRecord = ProjectionRunRecordBase & {
+  readonly identity: ProjectionIdentity<"telemetry">
+  readonly target: ObjectProjectionTarget
+  readonly telemetryCheckpoint: ProjectionTelemetryCheckpoint
+  /**
+   * Only telemetry carries this: `MaterializationObjectNotFoundError` is raised by telemetry
+   * validation alone, and only a batched protocol has a checkpoint to anchor the wait to.
+   */
+  readonly missingTarget?: ProjectionMissingTarget
+}
+
+/** Durable projection lifecycle state. The execution token is deliberately returned separately. */
+export type ProjectionRunRecord =
+  | ObjectProjectionRunRecord
+  | LinkProjectionRunRecord
+  | TelemetryProjectionRunRecord
+
+export function projectionRunObjectTypesVisible(
+  run: ProjectionRunRecord,
+  canView: (objectTypeId: string) => boolean
+): boolean {
+  const target: ProjectionTarget = run.target
+  if ("sourceObjectTypeId" in target) {
+    return canView(target.sourceObjectTypeId) && canView(target.targetObjectTypeId)
+  }
+  return canView(target.objectTypeId)
+}
+
+interface StartOrReclaimProjectionRunBase {
+  readonly id: string
+  readonly projectId: string
+  readonly startedAt?: Date
+}
+
+export type StartOrReclaimProjectionRunInput =
+  | (StartOrReclaimProjectionRunBase & {
+      readonly identity: ProjectionIdentity<"object">
+      readonly target: ObjectProjectionRunRecord["target"]
+      readonly fixedBatchSize?: never
+    })
+  | (StartOrReclaimProjectionRunBase & {
+      readonly identity: ProjectionIdentity<"link">
+      readonly target: LinkProjectionRunRecord["target"]
+      readonly fixedBatchSize?: never
+    })
+  | (StartOrReclaimProjectionRunBase & {
+      readonly identity: ProjectionIdentity<"telemetry">
+      readonly target: TelemetryProjectionRunRecord["target"]
+      readonly fixedBatchSize: number
+    })
+
+export interface ProjectionRunClaim {
+  readonly run: ProjectionRunRecord
+  readonly execution: ProjectionExecution
+}
+
+export interface LockProjectionRunForMaterializationInput {
+  readonly id: string
+  readonly projectId: string
+  readonly executionToken: string
+  readonly identity: ProjectionMaterializationIdentity
+}
+
+export interface UpdateProjectionRunInput extends LockProjectionRunForMaterializationInput {
+  readonly progress: Partial<ProjectionRunProgress>
+}
+
+export type FinishProjectionRunInput = ProjectionRunTerminalDecision &
+  LockProjectionRunForMaterializationInput & {
+    readonly finishedAt?: Date
+    readonly progress?: Partial<ProjectionRunProgress>
+  }
+
+export interface AdvanceProjectionTelemetryCheckpointInput
+  extends LockProjectionRunForMaterializationInput {
+  /** Must equal the run's current nextBatchOrdinal. */
+  readonly batchOrdinal: number
+  /** Physical dataset rows consumed by this batch, including skipped rows. */
+  readonly batchRowCount: number
+  /** Physical rows intentionally skipped inside this batch. */
+  readonly batchRowsSkipped: number
+  /** True when this batch consumed the final row of the immutable dataset version. */
+  readonly inputExhausted: boolean
+}
+
+export interface RecordProjectionMissingTargetInput
+  extends LockProjectionRunForMaterializationInput {
+  readonly missingTarget: ProjectionMissingTarget
+}
+
+export interface ListProjectionRunsInput {
+  readonly projectId: string
+  readonly projectionId?: string
+  readonly projectionKind?: ProjectionKind
+  readonly datasetId?: string
+  readonly datasetVersionId?: string
+  /**
+   * Viewable object type ids. A run is included only if every target type is present. Omit for a
+   * privileged caller; an empty set matches no runs.
+   */
+  readonly objectTypeIds?: readonly string[]
+  readonly statuses?: readonly ProjectionRunStatus[]
+  readonly startedAfter?: Date
+  readonly startedBefore?: Date
+  readonly limit?: number
+  readonly offset?: number
+  readonly order?: "asc" | "desc"
+}
+
+export interface ListProjectionRunsResult {
+  readonly runs: readonly ProjectionRunRecord[]
+  readonly hasMore: boolean
+  readonly total: number
+}
+
+export interface ListLatestProjectionRunsInput {
+  readonly projectId: string
+  readonly projectionIds: readonly string[]
+}
+
+export interface ListLatestProjectionRunsResult {
+  readonly runs: readonly ProjectionRunRecord[]
+}
+
+export interface ProjectionRunStorage {
+  startOrReclaim(input: StartOrReclaimProjectionRunInput): Promise<ProjectionRunClaim>
+  /** Transactionally fences the current execution before ontology materialization writes. */
+  lockForMaterialization(
+    input: LockProjectionRunForMaterializationInput
+  ): Promise<ProjectionRunRecord>
+  /** Updates physical source progress only. */
+  update(input: UpdateProjectionRunInput): Promise<ProjectionRunRecord>
+  /**
+   * Advances telemetry progress atomically with the corresponding ontology commit.
+   *
+   * Clears any recorded missing target: a batch that commits is a batch that was not waiting.
+   */
+  advanceTelemetryCheckpoint(
+    input: AdvanceProjectionTelemetryCheckpointInput
+  ): Promise<TelemetryProjectionRunRecord>
+  /** Records the object a telemetry batch is waiting for, so the wait has a durable start. */
+  recordMissingTarget(
+    input: RecordProjectionMissingTargetInput
+  ): Promise<TelemetryProjectionRunRecord>
+  /** Telemetry success also records the worker's EOF observation atomically. */
+  finish(input: FinishProjectionRunInput): Promise<ProjectionRunRecord>
+  getById(params: { projectId: string; id: string }): Promise<ProjectionRunRecord | null>
+  list(input: ListProjectionRunsInput): Promise<ListProjectionRunsResult>
+  listLatestByProjectionIds(
+    input: ListLatestProjectionRunsInput
+  ): Promise<ListLatestProjectionRunsResult>
+}
+
+export type {
+  LinkProjectionTarget,
+  ObjectProjectionTarget,
+  PinnedDatasetVersion,
+  ProjectionExecution,
+  ProjectionKind,
+  ProjectionMaterializationIdentity,
+  ProjectionTarget,
+}

package/src/storage/rules/in-memory.ts

@@ -0,0 +1,191 @@
+import type {
+  RuleEventSubject,
+  StoredRuleResolvedEvent,
+  StoredRuleTriggeredEvent,
+} from "../../events"
+import type {
+  ListActiveRuleStatesInput,
+  ListActiveRuleStatesResult,
+  ListRuleStatesReconciliationPageInput,
+  ListRuleStatesReconciliationPageResult,
+  RuleStateCursor,
+  RuleStateRecord,
+  RuleStateTransitionEvent,
+  RulesStorage,
+} from "./types"
+
+function stateKey(params: {
+  readonly projectId: string
+  readonly ruleId: string
+  readonly subject: RuleEventSubject
+}): string {
+  return JSON.stringify([
+    params.projectId,
+    params.ruleId,
+    params.subject.kind,
+    params.subject.objectTypeId,
+    params.subject.primaryId,
+  ])
+}
+
+function cloneRuleStateRecord(record: RuleStateRecord): RuleStateRecord {
+  return structuredClone(record)
+}
+
+function compareRuleStateRecords(
+  left: RuleStateRecord,
+  right: RuleStateRecord,
+  order: "asc" | "desc"
+): number {
+  const direction = order === "asc" ? 1 : -1
+  const fields = [
+    left.triggeredAt.localeCompare(right.triggeredAt),
+    left.ruleId.localeCompare(right.ruleId),
+    left.subject.objectTypeId.localeCompare(right.subject.objectTypeId),
+    left.subject.primaryId.localeCompare(right.subject.primaryId),
+  ]
+
+  for (const value of fields) {
+    if (value !== 0) {
+      return value * direction
+    }
+  }
+
+  return 0
+}
+
+function compareRuleStateIdentity(left: RuleStateRecord, right: RuleStateRecord): number {
+  return (
+    left.ruleId.localeCompare(right.ruleId) ||
+    left.subject.objectTypeId.localeCompare(right.subject.objectTypeId) ||
+    left.subject.primaryId.localeCompare(right.subject.primaryId)
+  )
+}
+
+function compareCursor(record: RuleStateRecord, cursor: RuleStateCursor): number {
+  return (
+    record.ruleId.localeCompare(cursor.ruleId) ||
+    record.subject.objectTypeId.localeCompare(cursor.objectTypeId) ||
+    record.subject.primaryId.localeCompare(cursor.primaryId)
+  )
+}
+
+export class InMemoryRulesStorage implements RulesStorage {
+  private readonly active = new Map<string, RuleStateRecord>()
+
+  snapshot(): InMemoryRulesStorageSnapshot {
+    return structuredClone(this.active)
+  }
+
+  restore(snapshot: InMemoryRulesStorageSnapshot): void {
+    this.active.clear()
+    for (const [key, record] of structuredClone(snapshot)) {
+      this.active.set(key, record)
+    }
+  }
+
+  async getActive(params: {
+    projectId: string
+    ruleId: string
+    subject: RuleEventSubject
+  }): Promise<RuleStateRecord | null> {
+    const record = this.active.get(stateKey(params))
+    return record ? cloneRuleStateRecord(record) : null
+  }
+
+  async getActiveBatch(params: {
+    projectId: string
+    items: readonly { readonly ruleId: string; readonly subject: RuleEventSubject }[]
+  }): Promise<readonly RuleStateRecord[]> {
+    return params.items.flatMap((item) => {
+      const record = this.active.get(stateKey({ projectId: params.projectId, ...item }))
+      return record ? [cloneRuleStateRecord(record)] : []
+    })
+  }
+
+  async listActive(input: ListActiveRuleStatesInput): Promise<ListActiveRuleStatesResult> {
+    if (input.objectTypeIds?.length === 0) {
+      return { states: [], hasMore: false, total: 0 }
+    }
+
+    const order = input.order ?? "desc"
+    const offset = input.offset ?? 0
+    const objectTypeIds = input.objectTypeIds ? new Set(input.objectTypeIds) : null
+    const states = [...this.active.values()]
+      .filter((record) => record.projectId === input.projectId)
+      .filter((record) => !input.ruleId || record.ruleId === input.ruleId)
+      .filter((record) => !input.objectTypeId || record.subject.objectTypeId === input.objectTypeId)
+      .filter((record) => (objectTypeIds ? objectTypeIds.has(record.subject.objectTypeId) : true))
+      .filter((record) => !input.primaryId || record.subject.primaryId === input.primaryId)
+      .sort((left, right) => compareRuleStateRecords(left, right, order))
+
+    const total = states.length
+    const windowed =
+      input.limit === undefined ? states.slice(offset) : states.slice(offset, offset + input.limit)
+
+    return {
+      states: windowed.map(cloneRuleStateRecord),
+      hasMore: offset + windowed.length < total,
+      total,
+    }
+  }
+
+  async listReconciliationPage(
+    input: ListRuleStatesReconciliationPageInput
+  ): Promise<ListRuleStatesReconciliationPageResult> {
+    const rows = [...this.active.values()]
+      .filter((record) => record.projectId === input.projectId)
+      .filter((record) => record.subject.kind === "object")
+      .filter((record) => !input.after || compareCursor(record, input.after) > 0)
+      .sort(compareRuleStateIdentity)
+      .slice(0, input.limit + 1)
+    const hasMore = rows.length > input.limit
+    const states = rows.slice(0, input.limit).map(cloneRuleStateRecord)
+    const last = states.at(-1)
+    return {
+      states,
+      ...(hasMore && last
+        ? {
+            next: {
+              ruleId: last.ruleId,
+              objectTypeId: last.subject.objectTypeId,
+              primaryId: last.subject.primaryId,
+            },
+          }
+        : {}),
+    }
+  }
+
+  async applyTriggered(event: StoredRuleTriggeredEvent): Promise<void> {
+    const record: RuleStateRecord = {
+      projectId: event.projectId,
+      ruleId: event.payload.ruleId,
+      subject: structuredClone(event.payload.subject),
+      triggeredAt: event.payload.triggeredAt,
+    }
+
+    this.active.set(stateKey(record), cloneRuleStateRecord(record))
+  }
+
+  async applyResolved(event: StoredRuleResolvedEvent): Promise<void> {
+    this.active.delete(
+      stateKey({
+        projectId: event.projectId,
+        ruleId: event.payload.ruleId,
+        subject: event.payload.subject,
+      })
+    )
+  }
+
+  async applyTransitions(events: readonly RuleStateTransitionEvent[]): Promise<void> {
+    for (const event of events) {
+      if (event.type === "rule.triggered") {
+        await this.applyTriggered(event)
+      } else {
+        await this.applyResolved(event)
+      }
+    }
+  }
+}
+
+export type InMemoryRulesStorageSnapshot = Map<string, RuleStateRecord>

package/src/storage/rules/index.ts

@@ -0,0 +1,12 @@
+export { InMemoryRulesStorage } from "./in-memory"
+export type {
+  ListActiveRuleStatesInput,
+  ListActiveRuleStatesResult,
+  ListRuleStatesReconciliationPageInput,
+  ListRuleStatesReconciliationPageResult,
+  RuleStateCursor,
+  RuleStateIdentity,
+  RuleStateRecord,
+  RuleStateTransitionEvent,
+  RulesStorage,
+} from "./types"

package/src/storage/rules/types.ts

@@ -0,0 +1,80 @@
+import type {
+  RuleEventSubject,
+  StoredRuleResolvedEvent,
+  StoredRuleTriggeredEvent,
+} from "../../events"
+
+export interface RuleStateRecord {
+  readonly projectId: string
+  readonly ruleId: string
+  readonly subject: RuleEventSubject
+  readonly triggeredAt: string
+}
+
+export interface ListActiveRuleStatesInput {
+  readonly projectId: string
+  readonly ruleId?: string
+  readonly objectTypeId?: string
+  readonly objectTypeIds?: readonly string[]
+  readonly primaryId?: string
+  readonly limit?: number
+  readonly offset?: number
+  readonly order?: "asc" | "desc"
+}
+
+export interface ListActiveRuleStatesResult {
+  readonly states: readonly RuleStateRecord[]
+  readonly hasMore: boolean
+  readonly total: number
+}
+
+export interface RuleStateIdentity {
+  readonly ruleId: string
+  readonly subject: RuleEventSubject
+}
+
+export interface RuleStateCursor {
+  readonly ruleId: string
+  readonly objectTypeId: string
+  readonly primaryId: string
+}
+
+export interface ListRuleStatesReconciliationPageInput {
+  readonly projectId: string
+  readonly after?: RuleStateCursor
+  readonly limit: number
+}
+
+export interface ListRuleStatesReconciliationPageResult {
+  readonly states: readonly RuleStateRecord[]
+  readonly next?: RuleStateCursor
+}
+
+export type RuleStateTransitionEvent = StoredRuleTriggeredEvent | StoredRuleResolvedEvent
+
+export interface RulesStorage {
+  getActive(params: {
+    projectId: string
+    ruleId: string
+    subject: RuleEventSubject
+  }): Promise<RuleStateRecord | null>
+
+  getActiveBatch(params: {
+    projectId: string
+    items: readonly RuleStateIdentity[]
+  }): Promise<readonly RuleStateRecord[]>
+
+  listActive(params: ListActiveRuleStatesInput): Promise<ListActiveRuleStatesResult>
+
+  /** Stable keyset page used to repair deleted or otherwise missed active subjects. */
+  listReconciliationPage(
+    params: ListRuleStatesReconciliationPageInput
+  ): Promise<ListRuleStatesReconciliationPageResult>
+
+  applyTriggered(event: StoredRuleTriggeredEvent): Promise<void>
+
+  applyResolved(event: StoredRuleResolvedEvent): Promise<void>
+
+  /** Applies one reconciliation page without one provider round trip per transition. */
+  applyTransitions(events: readonly RuleStateTransitionEvent[]): Promise<void>
+}

package/src/storage/run-listing.ts

@@ -0,0 +1,107 @@
+export type RunListOrder = "asc" | "desc"
+
+export type { ListPage as RunListPage, ListWindow as RunListWindow } from "./pagination"
+export { paginate } from "./pagination"
+
+export interface StartedAtRunRecord<TStatus extends string = string> {
+  readonly id: string
+  readonly status: TStatus
+  readonly startedAt: Date
+}
+
+export interface RunListDateFilters<TStatus extends string = string> {
+  readonly statuses?: readonly TStatus[]
+  readonly startedAfter?: Date
+  readonly startedBefore?: Date
+}
+
+export function storageKey(projectId: string, id: string): string {
+  return `${projectId}:${id}`
+}
+
+export function cloneRecord<T>(record: T): T {
+  return structuredClone(record)
+}
+
+export function hasEmptyStatuses(input: { readonly statuses?: readonly unknown[] }): boolean {
+  return input.statuses !== undefined && input.statuses.length === 0
+}
+
+export function toStatusSet<TStatus extends string>(
+  statuses: readonly TStatus[] | undefined
+): ReadonlySet<TStatus> | null {
+  return statuses ? new Set(statuses) : null
+}
+
+export function matchesRunListDateFilters<TStatus extends string>(
+  record: StartedAtRunRecord<TStatus>,
+  filters: {
+    readonly statuses: ReadonlySet<TStatus> | null
+    readonly startedAfter?: Date
+    readonly startedBefore?: Date
+  }
+): boolean {
+  if (filters.statuses && !filters.statuses.has(record.status)) {
+    return false
+  }
+
+  if (filters.startedAfter && record.startedAt < filters.startedAfter) {
+    return false
+  }
+
+  if (filters.startedBefore && record.startedAt > filters.startedBefore) {
+    return false
+  }
+
+  return true
+}
+
+export function compareStartedAt(
+  left: { readonly id: string; readonly startedAt: Date },
+  right: { readonly id: string; readonly startedAt: Date },
+  order: RunListOrder
+): number {
+  const delta = left.startedAt.getTime() - right.startedAt.getTime()
+  if (delta !== 0) {
+    return order === "asc" ? delta : -delta
+  }
+
+  if (left.id === right.id) {
+    return 0
+  }
+
+  return order === "asc" ? left.id.localeCompare(right.id) : right.id.localeCompare(left.id)
+}
+
+export function latestStartedAtByOwnerId<
+  TRecord extends { readonly id: string; readonly startedAt: Date },
+>(
+  records: readonly TRecord[],
+  ownerIds: readonly string[],
+  ownerIdFor: (record: TRecord) => string
+): readonly TRecord[] {
+  const uniqueOwnerIds = [...new Set(ownerIds)]
+  if (uniqueOwnerIds.length === 0) {
+    return []
+  }
+
+  const requested = new Set(uniqueOwnerIds)
+  const latestByOwnerId = new Map<string, TRecord>()
+
+  for (const record of records) {
+    const ownerId = ownerIdFor(record)
+    if (!requested.has(ownerId)) {
+      continue
+    }
+
+    const current = latestByOwnerId.get(ownerId)
+    if (!current || compareStartedAt(record, current, "desc") < 0) {
+      latestByOwnerId.set(ownerId, record)
+    }
+  }
+
+  return uniqueOwnerIds.flatMap((ownerId) => {
+    const record = latestByOwnerId.get(ownerId)
+    return record ? [record] : []
+  })
+}

package/src/storage/sync-runs/errors.ts

@@ -0,0 +1,7 @@
+/**
+ * Base error for sync-run storage operations. Callers can use this to catch
+ * domain-specific failures without relying on message text.
+ */
+export class SyncRunError extends Error {
+  readonly name = "SyncRunError"
+}