@sixb/core 0.1.9 → 0.1.10

package/src/materializer/projections/replace.ts

@@ -10,6 +10,8 @@ import type {
   ProjectionCommitResult,
   ProjectionExecution,
   ProjectionMaterializationIdentity,
+  ProjectionSourceBase,
+  ProjectionSourceDeletion,
   ProjectionSourceEntry,
   ProjectionSourceRef,
   ProjectionSourceReplacement,
@@ -19,6 +21,7 @@ import type {
   OntologyCommitRecord,
   OntologyCommitWrite,
   OntologyMaterializationStorage,
+  OntologySourceRecord,
   OntologyStorage,
 } from "../../storage/ontology"
 import type { MaterializerContext, MaterializerStorage } from "../context"
@@ -60,7 +63,8 @@ interface PreparedProjectionReplacement {
   readonly source: ProjectionSourceRef
   readonly datasetVersion: PinnedDatasetVersion
   readonly execution: ProjectionExecution
-  readonly entries: AsyncIterable<ProjectionSourceEntry>
+  readonly entries: AsyncIterable<ProjectionSourceEntry | ProjectionSourceDeletion>
+  readonly base?: ProjectionSourceBase
   readonly signal?: AbortSignal
   readonly resolved: ResolvedSourceProjection
   readonly projectionKind: "object" | "link"
@@ -128,6 +132,7 @@ function prepareProjectionReplacement(
     datasetVersion,
     execution,
     entries: raw.input.entries,
+    ...(raw.input.base ? { base: Object.freeze({ ...raw.input.base }) } : {}),
     resolved,
     projectionKind,
     runIdentity,
@@ -237,6 +242,7 @@ async function prepareProjectionCandidate(
     source: command.source,
   })
   validateProjectionWatermark(active, command.datasetVersion)
+  assertDeltaBase(active, command)
   return {
     materializationId: context.materializationId(),
     createdAt: context.clock().toISOString(),
@@ -263,6 +269,7 @@ async function stageProjectionCandidate(
     ownershipHash: command.resolved.ownershipHash,
     createdAt: candidate.createdAt,
     entries: command.entries,
+    ...(command.base ? { base: command.base } : {}),
     validateEntry: createProjectionEntryValidator(context.ontology, command.resolved),
   }
   const staged = await stageCandidateEntries(context, input, command.signal)
@@ -312,6 +319,14 @@ async function executeProjectionTransaction(
   )
   if (replay) return projectionReplayResult(replay, command)
 
+  if (command.base) {
+    const active = await storage.ontology.sources.getActive({
+      projectId: context.projectId,
+      source: command.source,
+    })
+    validateProjectionWatermark(active, command.datasetVersion)
+    assertDeltaBase(active, command)
+  }
   const origin = projectionOrigin(command)
   throwIfAborted(command.signal)
   const session = await storage.ontology.materializations.begin({
@@ -478,3 +493,23 @@ function validateProjectionWatermark(
   if (!active) return
   assertPinnedDatasetWatermark(active.datasetVersion, next, "Projection replacement")
 }
+
+function assertDeltaBase(
+  active: OntologySourceRecord | null,
+  command: PreparedProjectionReplacement
+): void {
+  if (
+    command.base &&
+    (!active ||
+      active.materializationId !== command.base.materializationId ||
+      active.lastCommitId !== command.base.lastCommitId ||
+      active.projectionRevision !== command.runIdentity.projectionRevision ||
+      active.ontologyRevision !== command.runIdentity.ontologyRevision ||
+      active.ownershipHash !== command.runIdentity.ownershipHash)
+  ) {
+    throw new MaterializationConflictError(
+      "source-materialization",
+      "Projection delta no longer matches the active source or its definition."
+    )
+  }
+}

package/src/materializer/projections/source-materialization.ts

@@ -1,16 +1,19 @@
+import { MaterializationValidationError } from "../../materialization/errors"
 import type {
   PinnedDatasetVersion,
   ProjectionExecution,
+  ProjectionSourceBase,
+  ProjectionSourceDeletion,
   ProjectionSourceEntry,
   ProjectionSourceRef,
 } from "../../materialization/model"
 import { utf8JsonByteLength } from "../../materialization/refs"
-import type { StageSourceAssertion } from "../../storage/ontology"
+import type { StageSourceAssertion, StageSourceRoot } from "../../storage/ontology"
 import type { MaterializerContext } from "../context"
 import { throwIfAborted } from "../shared/abort"
 import { chunkBySize } from "../shared/chunking"
-import { normalizeProjectionSourceEntry } from "../shared/normalize"
-import type { ProjectionEntryValidator } from "./entry-validator"
+import { normalizeProjectionEntity, normalizeProjectionSourceEntry } from "../shared/normalize"
+import { type ProjectionEntryValidator, validateProjectionRootRef } from "./entry-validator"
 
 export interface StagedProjectionMaterialization {
   readonly rootCount: number
@@ -37,7 +40,8 @@ export async function stageProjectionMaterialization(
     readonly projectionRevision: string
     readonly ownershipHash: string
     readonly createdAt: string
-    readonly entries: AsyncIterable<ProjectionSourceEntry>
+    readonly entries: AsyncIterable<ProjectionSourceEntry | ProjectionSourceDeletion>
+    readonly base?: ProjectionSourceBase
     readonly validateEntry: ProjectionEntryValidator
     readonly signal?: AbortSignal
   }
@@ -54,13 +58,28 @@ export async function stageProjectionMaterialization(
     ownershipHash: input.ownershipHash,
     ontologyRevision: context.projectionRegistry.ontologyRevision,
     createdAt: input.createdAt,
+    ...(input.base ? { base: input.base } : {}),
   })
 
   let rootCount = 0
   let assertionCount = 0
-  async function* assertions(): AsyncIterable<StageSourceAssertion> {
+  async function* assertions(): AsyncIterable<StageSourceAssertion | StageSourceRoot> {
     for await (const rawEntry of input.entries) {
       throwIfAborted(input.signal)
+      if ("deleted" in rawEntry) {
+        if (!input.base || rawEntry.deleted !== true) {
+          throw new MaterializationValidationError(
+            "Explicit root deletions require a pinned source base."
+          )
+        }
+        const root = normalizeProjectionEntity(rawEntry.root)
+        validateProjectionRootRef(
+          context.projectionRegistry.resolveSource(input.source.projectionId),
+          root
+        )
+        yield { root, stagingOrdinal: rootCount++ }
+        continue
+      }
       const entry = input.validateEntry(normalizeProjectionSourceEntry(rawEntry))
       const stagingOrdinal = rootCount
       for (const assertion of entry.assertions) {
@@ -82,7 +101,8 @@ export async function stageProjectionMaterialization(
       source: input.source,
       materializationId: input.materializationId,
       execution: input.execution,
-      rows,
+      rows: rows.filter((row): row is StageSourceAssertion => "assertion" in row),
+      deletions: rows.filter((row) => !("assertion" in row)),
     })
   }
 

package/src/materializer/shared/normalize.ts

@@ -344,7 +344,7 @@ export function normalizeProjectionExecution(input: ProjectionExecution): Projec
   })
 }
 
-function normalizeProjectionEntity(entity: ProjectionEntityRef): ProjectionEntityRef {
+export function normalizeProjectionEntity(entity: ProjectionEntityRef): ProjectionEntityRef {
   return entity.kind === "object"
     ? Object.freeze({ kind: "object", ref: normalizeObjectRef(entity.ref) })
     : Object.freeze({ kind: "link", ref: normalizeLinkRef(entity.ref) })

package/src/pipelines/run-dispatch.ts

@@ -101,7 +101,10 @@ export async function dispatchPipelineRun(
   input: DispatchPipelineRunInput
 ): Promise<PipelineRunRequestResult> {
   const runId = createPipelineRunId(input.runId)
-  const persisted = await persistPipelineRun(input, runId)
+  let persisted = await persistPipelineRun(input, runId)
+  if (!persisted.publish && input.validateExistingRun && persisted.run.status === "queued") {
+    persisted = await reusePipelineRun(input, persisted.run)
+  }
   if (!persisted.publish) return existingPipelineRunResult(persisted.run)
   return publishPipelineRun(input, persisted)
 }
@@ -155,6 +158,15 @@ async function reusePipelineRun(
 ): Promise<PersistedPipelineRun> {
   assertExistingRunMatchesRequest(existing, input)
   await input.validateExistingRun?.(existing, input.storage)
+  if (input.validateExistingRun && existing.status === "queued") {
+    const execution = await input.storage.executions.getById({
+      projectId: input.projectId,
+      id: existing.executionId,
+    })
+    if (!execution)
+      throw new PipelineRunError(`[Sixb] Execution '${existing.executionId}' was not found.`)
+    return { publish: true, run: existing, execution, queuedAt: existing.queuedAt, created: false }
+  }
   if (
     existing.status !== "failed" ||
     existing.error?.code !== "queue.enqueue_failed" ||
@@ -200,6 +212,11 @@ async function publishPipelineRun(
         },
       ],
     })
+    if (!job || job.id !== persisted.run.id) {
+      throw new PipelineRunError(
+        `[Sixb] Queue did not acknowledge Pipeline run '${persisted.run.id}'.`
+      )
+    }
     return {
       pipelineId: persisted.run.pipelineId,
       runId: persisted.run.id,
@@ -208,6 +225,9 @@ async function publishPipelineRun(
       created: persisted.created,
     }
   } catch (error) {
+    // Automatic consumers retain the batch and retry. Keep uncertain delivery queued:
+    // a lost reply does not prove that the queue rejected the job.
+    if (input.validateExistingRun) throw error
     const failedAt = new Date()
     const failure = toEnqueueFailure(error, persisted.run, failedAt)
     const failed = await requirePipelineRunStorage(input.storage).finish({

package/src/projections/run-dispatch.ts

@@ -1,7 +1,4 @@
 import { randomUUID } from "node:crypto"
-import { reportRunFailure } from "../error-reporting/capability"
-import { createSixbError, toSixbFailure } from "../errors/internal"
-import type { SixbFailure } from "../errors/types"
 import type { RunDispatcher } from "../execution/dispatch"
 import { createPrimitiveExecutionRecord } from "../execution/durable"
 import type { DatasetProducer, LakeStorage } from "../lake-storage"
@@ -17,11 +14,7 @@ import {
 import { getProjectionRegistry } from "./capability"
 import { ProjectionValidationError } from "./errors"
 import { createProjectionRunId } from "./run-id"
-import {
-  PROJECTION_RUN_FAILURE_CODES,
-  type ProjectionDefinition,
-  type ProjectionRunFailureCode,
-} from "./types"
+import type { ProjectionDefinition } from "./types"
 
 /** Target maximum mapped points per telemetry commit; a single-property projection reads 500 rows. */
 export const PROJECTION_TELEMETRY_BATCH_SIZE = 500
@@ -106,7 +99,7 @@ export class ProjectionRunDispatcher implements ProjectionRunDispatchPort {
     const runId = createProjectionRunId(this.dependencies.id, identity)
     const projectionRuns = requireProjectionRunStorage(this.dependencies.storage)
     const existing = await projectionRuns.getById({ projectId: this.dependencies.id, id: runId })
-    const persisted = existing
+    let persisted = existing
       ? await reuseProjectionRun(this.dependencies, existing, identity)
       : await persistProjectionRun({
           dependencies: this.dependencies,
@@ -120,6 +113,9 @@ export class ProjectionRunDispatcher implements ProjectionRunDispatchPort {
           ),
         })
 
+    if (!persisted.publish && persisted.run.status === "queued") {
+      persisted = await reuseProjectionRun(this.dependencies, persisted.run, identity)
+    }
     if (!persisted.publish) return existingResult(persisted.run)
     return publishProjectionRun(this.dependencies, input.metadata, persisted)
   }
@@ -195,6 +191,9 @@ async function reuseProjectionRun(
   identity: ProjectionRunRecord["identity"]
 ): Promise<PersistedProjectionRun> {
   await assertExistingRun(dependencies.storage, existing, identity)
+  if (existing.status === "queued") {
+    return { publish: true, run: existing, queuedAt: existing.queuedAt, created: false }
+  }
   if (
     existing.status !== "failed" ||
     existing.error?.code !== "queue.enqueue_failed" ||
@@ -228,45 +227,29 @@ async function publishProjectionRun(
   metadata: Readonly<Record<string, string>> | undefined,
   persisted: Extract<PersistedProjectionRun, { readonly publish: true }>
 ): Promise<ProjectionRunDispatchResult> {
-  try {
-    const [job] = await dependencies.queues.projections.enqueue({
-      projectId: dependencies.id,
-      jobs: [
-        {
-          id: persisted.run.id,
-          type: "projection.run.requested",
-          payload: { runId: persisted.run.id },
-          ...(metadata === undefined ? {} : { metadata }),
-        },
-      ],
-    })
-    return {
-      projectionId: persisted.run.identity.projectionId,
-      runId: persisted.run.id,
-      queuedAt: persisted.queuedAt.toISOString(),
-      ...(job?.id ? { jobId: job.id } : {}),
-      created: persisted.created,
-    }
-  } catch (error) {
-    const failedAt = new Date()
-    const failure = toEnqueueFailure(error, persisted.run, failedAt)
-    const failed = await requireProjectionRunStorage(dependencies.storage).failEnqueue({
-      projectId: dependencies.id,
-      id: persisted.run.id,
-      finishedAt: failedAt,
-      error: failure,
-    })
-    reportRunFailure(dependencies, error, {
-      projectId: dependencies.id,
-      runKind: "projection",
-      run: {
-        runId: failed.id,
-        projectionId: failed.identity.projectionId,
-        projectionKind: failed.identity.projectionKind,
+  // Keep the run queued on uncertain publication; the caller retries the same identity.
+  const [job] = await dependencies.queues.projections.enqueue({
+    projectId: dependencies.id,
+    jobs: [
+      {
+        id: persisted.run.id,
+        type: "projection.run.requested",
+        payload: { runId: persisted.run.id },
+        ...(metadata === undefined ? {} : { metadata }),
       },
-      failure,
-    })
-    throw error
+    ],
+  })
+  if (!job || job.id !== persisted.run.id) {
+    throw new ProjectionRunError(
+      `[Sixb] Queue did not acknowledge Projection run '${persisted.run.id}'.`
+    )
+  }
+  return {
+    projectionId: persisted.run.identity.projectionId,
+    runId: persisted.run.id,
+    queuedAt: persisted.queuedAt.toISOString(),
+    ...(job?.id ? { jobId: job.id } : {}),
+    created: persisted.created,
   }
 }
 
@@ -478,24 +461,3 @@ function assertDispatchInput(input: ProjectionRunDispatchInput): void {
     throw new ProjectionValidationError("[Sixb] Projection dataset version timestamp is invalid.")
   }
 }
-
-function toEnqueueFailure(
-  error: unknown,
-  run: ProjectionRunRecord,
-  at: Date
-): SixbFailure<ProjectionRunFailureCode> {
-  const enqueueError = createSixbError(
-    "queue.enqueue_failed",
-    `[Sixb] Could not enqueue Projection run '${run.id}'.`,
-    {
-      cause: error,
-      details: {
-        projectionId: run.identity.projectionId,
-        projectionKind: run.identity.projectionKind,
-        runId: run.id,
-        phase: "enqueue",
-      },
-    }
-  )
-  return toSixbFailure(enqueueError, { allowedCodes: PROJECTION_RUN_FAILURE_CODES, at })
-}

package/src/queues/in-memory.ts

@@ -1,6 +1,7 @@
 import { randomUUID } from "node:crypto"
 import type { SixbErrorCode } from "../errors/types"
 import { QueueError } from "./errors"
+import { IndexedHeap } from "./indexed-heap"
 import type {
   ActionQueueJobFailureCode,
   ActionRunRequestedQueueJob,
@@ -29,6 +30,9 @@ interface QueueRecord<TQueueJob extends QueueJob = QueueJob> {
   readonly sequence: number
   readonly queueId: string
   job: TQueueJob
+  availableAtMs: number
+  readonly createdAtMs: number
+  leaseExpiryMs: number | null
   // A leased job stays "queued" until a worker completes or fails it.
   state: QueueRecordState
   failure: QueueJobFailure | null
@@ -63,25 +67,15 @@ function clearLease(record: QueueRecord): void {
   record.leaseId = null
   record.claimedAt = null
   record.leaseExpiresAt = null
+  record.leaseExpiryMs = null
 }
 
 // Best-effort ordering is part of the contract; this keeps the in-memory provider deterministic.
 function compareQueueRecords(left: QueueRecord, right: QueueRecord): number {
-  const availableDifference =
-    parseTimestamp(left.job.availableAt, "job.availableAt") -
-    parseTimestamp(right.job.availableAt, "job.availableAt")
-
-  if (availableDifference !== 0) {
-    return availableDifference
-  }
-
-  const createdDifference =
-    parseTimestamp(left.job.createdAt, "job.createdAt") -
-    parseTimestamp(right.job.createdAt, "job.createdAt")
-
-  if (createdDifference !== 0) {
-    return createdDifference
-  }
+  const availableDifference = left.availableAtMs - right.availableAtMs
+  if (availableDifference !== 0) return availableDifference
+  const createdDifference = left.createdAtMs - right.createdAtMs
+  if (createdDifference !== 0) return createdDifference
 
   return left.sequence - right.sequence
 }
@@ -107,11 +101,18 @@ function createQueueJob<TQueueJob extends QueueJob>(
 }
 
 function leaseExpiresAtMs(record: QueueRecord): number | null {
-  if (!record.leaseId || !record.leaseExpiresAt) {
-    return null
-  }
+  return record.leaseExpiryMs
+}
 
-  return parseTimestamp(record.leaseExpiresAt, "leaseExpiresAt")
+function eligibleAt(record: QueueRecord): number {
+  return Math.max(record.availableAtMs, record.leaseExpiryMs ?? -Infinity)
+}
+
+interface QueueLane {
+  // Terminal records remain here for caller-id deduplication, never in the delivery indexes.
+  readonly records: Map<string, QueueRecord>
+  readonly ready: IndexedHeap<QueueRecord>
+  readonly waiting: IndexedHeap<QueueRecord>
 }
 
 function toClaimedQueueJob<TQueueJob extends QueueJob>(
@@ -130,27 +131,26 @@ function toClaimedQueueJob<TQueueJob extends QueueJob>(
 }
 
 class InMemoryQueueStore {
-  private readonly recordsByProject = new Map<string, QueueRecord[]>()
+  private readonly projects = new Map<string, Map<string, QueueLane>>()
   private nextSequence = 1
 
   add<TQueueJob extends QueueJob>(projectId: string, queueId: string, job: TQueueJob): void {
-    const records = this.getOrCreateRecords(projectId)
-    records.push({
-      sequence: this.nextSequence,
+    const lane = this.lane(projectId, queueId)
+    const record: QueueRecord = {
+      sequence: this.nextSequence++,
       queueId,
       job,
+      availableAtMs: parseTimestamp(job.availableAt, "job.availableAt"),
+      createdAtMs: parseTimestamp(job.createdAt, "job.createdAt"),
+      leaseExpiryMs: null,
       state: "queued",
       failure: null,
       leaseId: null,
       claimedAt: null,
       leaseExpiresAt: null,
-    })
-    this.nextSequence += 1
-  }
-
-  list<TQueueJob extends QueueJob>(projectId: string, queueId: string): QueueRecord<TQueueJob>[] {
-    const records = this.recordsByProject.get(projectId) ?? []
-    return records.filter((record) => record.queueId === queueId) as QueueRecord<TQueueJob>[]
+    }
+    lane.records.set(job.id, record)
+    this.schedule(projectId, record, Date.now())
   }
 
   find<TQueueJob extends QueueJob>(
@@ -158,20 +158,67 @@ class InMemoryQueueStore {
     queueId: string,
     jobId: string
   ): QueueRecord<TQueueJob> | null {
-    return (
-      this.list<TQueueJob>(projectId, queueId).find((record) => record.job.id === jobId) ?? null
-    )
+    return (this.projects.get(projectId)?.get(queueId)?.records.get(jobId) ??
+      null) as QueueRecord<TQueueJob> | null
   }
 
-  private getOrCreateRecords(projectId: string): QueueRecord[] {
-    const existing = this.recordsByProject.get(projectId)
-    if (existing) {
-      return existing
+  schedule(projectId: string, record: QueueRecord, now: number): void {
+    const lane = this.lane(projectId, record.queueId)
+    if (record.state !== "queued") {
+      lane.ready.delete(record)
+      lane.waiting.delete(record)
+    } else if (eligibleAt(record) <= now) {
+      lane.waiting.delete(record)
+      lane.ready.set(record)
+    } else {
+      lane.ready.delete(record)
+      lane.waiting.set(record)
+    }
+  }
+
+  takeReady<TQueueJob extends QueueJob>(
+    projectId: string,
+    queueId: string,
+    now: number,
+    limit: number
+  ): QueueRecord<TQueueJob>[] {
+    const lane = this.projects.get(projectId)?.get(queueId)
+    if (!lane) return []
+    // Wake delayed jobs and expired leases, then order all ready jobs by the original contract.
+    // A lease expiry controls eligibility, not the priority of a redelivered job.
+    while (lane.waiting.peek() && eligibleAt(lane.waiting.peek()!) <= now) {
+      lane.ready.set(lane.waiting.pop()!)
     }
+    const selected: QueueRecord[] = []
+    while (selected.length < limit) {
+      const record = lane.ready.peek()
+      if (!record || record.availableAtMs > now) break
+      lane.ready.pop()
+      // A backwards wall-clock adjustment can make a previously expired lease live again.
+      if (eligibleAt(record) > now) lane.waiting.set(record)
+      else selected.push(record)
+    }
+    return selected as QueueRecord<TQueueJob>[]
+  }
 
-    const records: QueueRecord[] = []
-    this.recordsByProject.set(projectId, records)
-    return records
+  private lane(projectId: string, queueId: string): QueueLane {
+    let queues = this.projects.get(projectId)
+    if (!queues) {
+      queues = new Map()
+      this.projects.set(projectId, queues)
+    }
+    let lane = queues.get(queueId)
+    if (!lane) {
+      lane = {
+        records: new Map(),
+        ready: new IndexedHeap(compareQueueRecords),
+        waiting: new IndexedHeap(
+          (left, right) => eligibleAt(left) - eligibleAt(right) || left.sequence - right.sequence
+        ),
+      }
+      queues.set(queueId, lane)
+    }
+    return lane
   }
 }
 
@@ -219,8 +266,8 @@ class InMemoryQueue<TQueueJob extends QueueJob, TFailureCode extends SixbErrorCo
     assertNonEmpty(params.projectId, "projectId")
     assertNonEmpty(params.workerId, "workerId")
 
-    const limit = params.limit ?? 1
-    if (limit <= 0) {
+    const limit = Math.trunc(params.limit ?? 1)
+    if (!(limit > 0)) {
       return []
     }
 
@@ -228,24 +275,11 @@ class InMemoryQueue<TQueueJob extends QueueJob, TFailureCode extends SixbErrorCo
     assertPositiveNumber(leaseMs, "leaseMs")
 
     const now = Date.now()
-    const claimable = this.store
-      .list<TQueueJob>(params.projectId, this.queueId)
-      .filter((record) => {
-        if (record.state !== "queued") {
-          return false
-        }
-
-        if (parseTimestamp(record.job.availableAt, "job.availableAt") > now) {
-          return false
-        }
-
-        const expiresAt = leaseExpiresAtMs(record)
-        return expiresAt === null || expiresAt <= now
-      })
-      .sort(compareQueueRecords)
-      .slice(0, limit)
-
     const claimedAt = new Date(now).toISOString()
+    // Validate before removing candidates, and cache the same millisecond precision we return.
+    const leaseExpiry = new Date(now + leaseMs)
+    const leaseExpiresAt = leaseExpiry.toISOString()
+    const claimable = this.store.takeReady<TQueueJob>(params.projectId, this.queueId, now, limit)
 
     return claimable.map((record) => {
       // Attempts count claims so redelivery after lease expiry or retry is visible to workers.
@@ -255,7 +289,9 @@ class InMemoryQueue<TQueueJob extends QueueJob, TFailureCode extends SixbErrorCo
       }
       record.leaseId = randomUUID()
       record.claimedAt = claimedAt
-      record.leaseExpiresAt = new Date(now + leaseMs).toISOString()
+      record.leaseExpiresAt = leaseExpiresAt
+      record.leaseExpiryMs = leaseExpiry.getTime()
+      this.store.schedule(params.projectId, record, now)
 
       return toClaimedQueueJob(record)
     })
@@ -265,6 +301,7 @@ class InMemoryQueue<TQueueJob extends QueueJob, TFailureCode extends SixbErrorCo
     const record = this.requireActiveLease(params)
     record.state = "completed"
     clearLease(record)
+    this.store.schedule(params.projectId, record, Date.now())
   }
 
   async retry(params: {
@@ -275,13 +312,15 @@ class InMemoryQueue<TQueueJob extends QueueJob, TFailureCode extends SixbErrorCo
   }): Promise<void> {
     const record = this.requireActiveLease(params)
     const availableAt = params.availableAt ?? new Date().toISOString()
-    parseTimestamp(availableAt, "availableAt")
+    const availableAtMs = parseTimestamp(availableAt, "availableAt")
 
+    record.availableAtMs = availableAtMs
     record.job = {
       ...record.job,
       availableAt,
     }
     clearLease(record)
+    this.store.schedule(params.projectId, record, Date.now())
   }
 
   async fail(params: {
@@ -294,6 +333,7 @@ class InMemoryQueue<TQueueJob extends QueueJob, TFailureCode extends SixbErrorCo
     record.state = "failed"
     record.failure = structuredClone(params.failure)
     clearLease(record)
+    this.store.schedule(params.projectId, record, Date.now())
   }
 
   async renewLease(params: {
@@ -319,7 +359,10 @@ class InMemoryQueue<TQueueJob extends QueueJob, TFailureCode extends SixbErrorCo
       return null
     }
 
-    record.leaseExpiresAt = new Date(now + params.leaseMs).toISOString()
+    const leaseExpiry = new Date(now + params.leaseMs)
+    record.leaseExpiresAt = leaseExpiry.toISOString()
+    record.leaseExpiryMs = leaseExpiry.getTime()
+    this.store.schedule(params.projectId, record, now)
     return toClaimedQueueJob(record)
   }