@lota-sdk/core 0.1.19 → 0.1.21

package/infrastructure/schema/09_queue_job.surql

@@ -0,0 +1,38 @@
+DEFINE TABLE IF NOT EXISTS queueJob SCHEMAFULL;
+DEFINE FIELD IF NOT EXISTS queueName ON TABLE queueJob TYPE string;
+DEFINE FIELD IF NOT EXISTS jobName ON TABLE queueJob TYPE string;
+DEFINE FIELD IF NOT EXISTS bullmqJobId ON TABLE queueJob TYPE string;
+DEFINE FIELD IF NOT EXISTS status ON TABLE queueJob TYPE string;
+DEFINE FIELD IF NOT EXISTS data ON TABLE queueJob TYPE option<object> FLEXIBLE;
+DEFINE FIELD IF NOT EXISTS options ON TABLE queueJob TYPE option<object> FLEXIBLE;
+DEFINE FIELD IF NOT EXISTS context ON TABLE queueJob TYPE option<object> FLEXIBLE;
+DEFINE FIELD IF NOT EXISTS deduplicationId ON TABLE queueJob TYPE option<string>;
+DEFINE FIELD IF NOT EXISTS schedulerId ON TABLE queueJob TYPE option<string>;
+DEFINE FIELD IF NOT EXISTS maxAttempts ON TABLE queueJob TYPE option<int>;
+DEFINE FIELD IF NOT EXISTS attemptCount ON TABLE queueJob TYPE int DEFAULT 0;
+DEFINE FIELD IF NOT EXISTS result ON TABLE queueJob TYPE option<object> FLEXIBLE;
+DEFINE FIELD IF NOT EXISTS lastError ON TABLE queueJob TYPE option<object> FLEXIBLE;
+DEFINE FIELD IF NOT EXISTS queuedAt ON TABLE queueJob TYPE datetime;
+DEFINE FIELD IF NOT EXISTS startedAt ON TABLE queueJob TYPE option<datetime>;
+DEFINE FIELD IF NOT EXISTS completedAt ON TABLE queueJob TYPE option<datetime>;
+DEFINE FIELD IF NOT EXISTS failedAt ON TABLE queueJob TYPE option<datetime>;
+DEFINE FIELD IF NOT EXISTS createdAt ON TABLE queueJob TYPE datetime DEFAULT time::now() READONLY;
+DEFINE FIELD IF NOT EXISTS updatedAt ON TABLE queueJob TYPE datetime VALUE time::now();
+
+DEFINE INDEX IF NOT EXISTS queueJobQueueStatusIdx ON TABLE queueJob COLUMNS queueName, status;
+DEFINE INDEX IF NOT EXISTS queueJobQueueBullmqIdx ON TABLE queueJob COLUMNS queueName, bullmqJobId UNIQUE;
+
+DEFINE TABLE IF NOT EXISTS queueJobAttempt SCHEMAFULL;
+DEFINE FIELD IF NOT EXISTS queueJobId ON TABLE queueJobAttempt TYPE record<queueJob> REFERENCE ON DELETE CASCADE;
+DEFINE FIELD IF NOT EXISTS attemptNumber ON TABLE queueJobAttempt TYPE int;
+DEFINE FIELD IF NOT EXISTS status ON TABLE queueJobAttempt TYPE string;
+DEFINE FIELD IF NOT EXISTS result ON TABLE queueJobAttempt TYPE option<object> FLEXIBLE;
+DEFINE FIELD IF NOT EXISTS error ON TABLE queueJobAttempt TYPE option<object> FLEXIBLE;
+DEFINE FIELD IF NOT EXISTS startedAt ON TABLE queueJobAttempt TYPE datetime;
+DEFINE FIELD IF NOT EXISTS completedAt ON TABLE queueJobAttempt TYPE option<datetime>;
+DEFINE FIELD IF NOT EXISTS durationMs ON TABLE queueJobAttempt TYPE option<int>;
+DEFINE FIELD IF NOT EXISTS createdAt ON TABLE queueJobAttempt TYPE datetime DEFAULT time::now() READONLY;
+DEFINE FIELD IF NOT EXISTS updatedAt ON TABLE queueJobAttempt TYPE datetime VALUE time::now();
+
+DEFINE INDEX IF NOT EXISTS queueJobAttemptQueueIdx ON TABLE queueJobAttempt COLUMNS queueJobId;
+DEFINE INDEX IF NOT EXISTS queueJobAttemptQueueNumberIdx ON TABLE queueJobAttempt COLUMNS queueJobId, attemptNumber UNIQUE;

package/infrastructure/schema/10_autonomous_job.surql

@@ -0,0 +1,44 @@
+DEFINE TABLE IF NOT EXISTS autonomousJob SCHEMAFULL;
+DEFINE FIELD IF NOT EXISTS organizationId ON TABLE autonomousJob TYPE record<organization>;
+DEFINE FIELD IF NOT EXISTS ownerUserId ON TABLE autonomousJob TYPE record<user>;
+DEFINE FIELD IF NOT EXISTS ownerUserName ON TABLE autonomousJob TYPE option<string>;
+DEFINE FIELD IF NOT EXISTS workstreamId ON TABLE autonomousJob TYPE record<workstream> REFERENCE ON DELETE CASCADE;
+DEFINE FIELD IF NOT EXISTS agentId ON TABLE autonomousJob TYPE string;
+DEFINE FIELD IF NOT EXISTS title ON TABLE autonomousJob TYPE string;
+DEFINE FIELD IF NOT EXISTS prompt ON TABLE autonomousJob TYPE string;
+DEFINE FIELD IF NOT EXISTS schedule ON TABLE autonomousJob TYPE object FLEXIBLE;
+DEFINE FIELD IF NOT EXISTS status ON TABLE autonomousJob TYPE string;
+DEFINE FIELD IF NOT EXISTS autoPauseThreshold ON TABLE autonomousJob TYPE int DEFAULT 3;
+DEFINE FIELD IF NOT EXISTS consecutiveErrorCount ON TABLE autonomousJob TYPE int DEFAULT 0;
+DEFINE FIELD IF NOT EXISTS lastRunStatus ON TABLE autonomousJob TYPE option<string>;
+DEFINE FIELD IF NOT EXISTS lastRunAt ON TABLE autonomousJob TYPE option<datetime>;
+DEFINE FIELD IF NOT EXISTS nextRunAt ON TABLE autonomousJob TYPE option<datetime>;
+DEFINE FIELD IF NOT EXISTS linkedPlanSpecId ON TABLE autonomousJob TYPE option<record<planSpec>>;
+DEFINE FIELD IF NOT EXISTS linkedPlanRunId ON TABLE autonomousJob TYPE option<record<planRun>>;
+DEFINE FIELD IF NOT EXISTS lastError ON TABLE autonomousJob TYPE option<object> FLEXIBLE;
+DEFINE FIELD IF NOT EXISTS createdAt ON TABLE autonomousJob TYPE datetime DEFAULT time::now() READONLY;
+DEFINE FIELD IF NOT EXISTS updatedAt ON TABLE autonomousJob TYPE datetime VALUE time::now();
+
+DEFINE INDEX IF NOT EXISTS autonomousJobOrgStatusIdx ON TABLE autonomousJob COLUMNS organizationId, status;
+DEFINE INDEX IF NOT EXISTS autonomousJobWorkstreamIdx ON TABLE autonomousJob COLUMNS workstreamId;
+DEFINE INDEX IF NOT EXISTS autonomousJobOwnerIdx ON TABLE autonomousJob COLUMNS ownerUserId;
+
+DEFINE TABLE IF NOT EXISTS autonomousJobRun SCHEMAFULL;
+DEFINE FIELD IF NOT EXISTS autonomousJobId ON TABLE autonomousJobRun TYPE record<autonomousJob> REFERENCE ON DELETE CASCADE;
+DEFINE FIELD IF NOT EXISTS workstreamId ON TABLE autonomousJobRun TYPE record<workstream> REFERENCE ON DELETE CASCADE;
+DEFINE FIELD IF NOT EXISTS queueJobId ON TABLE autonomousJobRun TYPE option<record<queueJob>>;
+DEFINE FIELD IF NOT EXISTS status ON TABLE autonomousJobRun TYPE string;
+DEFINE FIELD IF NOT EXISTS inputMessageId ON TABLE autonomousJobRun TYPE option<string>;
+DEFINE FIELD IF NOT EXISTS assistantMessageIds ON TABLE autonomousJobRun TYPE array<string> DEFAULT [];
+DEFINE FIELD IF NOT EXISTS summary ON TABLE autonomousJobRun TYPE option<string>;
+DEFINE FIELD IF NOT EXISTS error ON TABLE autonomousJobRun TYPE option<object> FLEXIBLE;
+DEFINE FIELD IF NOT EXISTS linkedPlanSpecId ON TABLE autonomousJobRun TYPE option<record<planSpec>>;
+DEFINE FIELD IF NOT EXISTS linkedPlanRunId ON TABLE autonomousJobRun TYPE option<record<planRun>>;
+DEFINE FIELD IF NOT EXISTS startedAt ON TABLE autonomousJobRun TYPE option<datetime>;
+DEFINE FIELD IF NOT EXISTS completedAt ON TABLE autonomousJobRun TYPE option<datetime>;
+DEFINE FIELD IF NOT EXISTS createdAt ON TABLE autonomousJobRun TYPE datetime DEFAULT time::now() READONLY;
+DEFINE FIELD IF NOT EXISTS updatedAt ON TABLE autonomousJobRun TYPE datetime VALUE time::now();
+
+DEFINE INDEX IF NOT EXISTS autonomousJobRunJobIdx ON TABLE autonomousJobRun COLUMNS autonomousJobId;
+DEFINE INDEX IF NOT EXISTS autonomousJobRunQueueJobIdx ON TABLE autonomousJobRun COLUMNS queueJobId;
+DEFINE INDEX IF NOT EXISTS autonomousJobRunStatusIdx ON TABLE autonomousJobRun COLUMNS status;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lota-sdk/core",
-  "version": "0.1.19",
+  "version": "0.1.21",
   "type": "module",
   "main": "./src/index.ts",
   "types": "./src/index.ts",
@@ -32,11 +32,11 @@
     "@chat-adapter/slack": "^4.23.0",
     "@chat-adapter/state-ioredis": "^4.23.0",
     "@logtape/logtape": "^2.0.5",
-    "@lota-sdk/shared": "0.1.19",
-    "@mendable/firecrawl-js": "^4.17.0",
+    "@lota-sdk/shared": "0.1.21",
+    "@mendable/firecrawl-js": "^4.18.0",
     "@surrealdb/node": "^3.0.3",
-    "ai": "^6.0.137",
-    "bullmq": "^5.71.0",
+    "ai": "^6.0.141",
+    "bullmq": "^5.71.1",
     "chat": "^4.23.0",
     "cron-parser": "^5.5.0",
     "hono": "^4.12.9",

package/src/ai-gateway/ai-gateway.ts

@@ -24,6 +24,7 @@ const EXPECTED_GATEWAY_KEY_PREFIX = 'sk-bf-'
 const AI_GATEWAY_VIRTUAL_KEY_HEADER = 'x-bf-vk'
 const AI_GATEWAY_EXTRA_PARAMS_HEADER = 'x-bf-passthrough-extra-params'
 const DEFAULT_AI_GATEWAY_URL = 'https://ai-gateway.gobrainy.ai' as const
+const OPENAI_PROMPT_CACHE_RETENTION = '24h' as const
 const OPENROUTER_RESPONSE_HEALING_EXTRA_PARAMS = {
   plugins: [{ id: 'response-healing' }],
 } as const satisfies AiGatewayExtraParams
@@ -50,6 +51,29 @@ function mergeAiGatewayHeaders(
   return Object.fromEntries(merged.entries())
 }
 
+function parseAiGatewayJsonRequestBody(body: BodyInit | null | undefined): Record<string, unknown> | null {
+  if (typeof body !== 'string') return null
+
+  let parsed: unknown
+  try {
+    parsed = JSON.parse(body)
+  } catch {
+    return null
+  }
+
+  return isRecord(parsed) ? parsed : null
+}
+
+function isAiGatewayOpenAIModelRequest(body: BodyInit | null | undefined): boolean {
+  const parsed = parseAiGatewayJsonRequestBody(body)
+  return readString(parsed?.model)?.startsWith('openai/') ?? false
+}
+
+function hasAiGatewayPromptCacheRetention(body: BodyInit | null | undefined): boolean {
+  const parsed = parseAiGatewayJsonRequestBody(body)
+  return readString(parsed?.prompt_cache_retention) !== null
+}
+
 function withDefaultAiGatewayCacheHeaders(params: AiGatewayCallOptions, modelId: string): AiGatewayCallOptions {
   return {
     ...params,
@@ -296,16 +320,8 @@ export function injectAiGatewayExtraParamsRequestBody(
   body: BodyInit | null | undefined,
   extraParams: AiGatewayExtraParams,
 ): BodyInit | null | undefined {
-  if (typeof body !== 'string') return body
-
-  let parsed: unknown
-  try {
-    parsed = JSON.parse(body)
-  } catch {
-    return body
-  }
-
-  if (!isRecord(parsed)) return body
+  const parsed = parseAiGatewayJsonRequestBody(body)
+  if (!parsed) return body
 
   const mergedExtraParams = isRecord(parsed.extra_params)
     ? { ...parsed.extra_params, ...extraParams }
@@ -314,11 +330,35 @@ export function injectAiGatewayExtraParamsRequestBody(
   return JSON.stringify({ ...parsed, extra_params: mergedExtraParams })
 }
 
-function createAiGatewayFetchWithExtraParams(extraParams: AiGatewayExtraParams): typeof fetch {
-  const fetchWithExtraParams = (input: RequestInfo | URL, init?: RequestInit | BunFetchRequestInit) =>
-    globalThis.fetch(input, { ...init, body: injectAiGatewayExtraParamsRequestBody(init?.body, extraParams) })
+export function injectAiGatewayOpenAIPromptCacheRetentionRequestBody(
+  body: BodyInit | null | undefined,
+): BodyInit | null | undefined {
+  const parsed = parseAiGatewayJsonRequestBody(body)
+  if (!parsed) return body
+  if (!readString(parsed.model)?.startsWith('openai/')) return body
+  if (readString(parsed.prompt_cache_retention) !== null) return body
+
+  return JSON.stringify({ ...parsed, prompt_cache_retention: OPENAI_PROMPT_CACHE_RETENTION })
+}
+
+function createAiGatewayFetch(extraParams?: AiGatewayExtraParams): typeof fetch {
+  const fetchWithMutations = (input: RequestInfo | URL, init?: RequestInit | BunFetchRequestInit) => {
+    const bodyWithPromptCacheRetention = injectAiGatewayOpenAIPromptCacheRetentionRequestBody(init?.body)
+    const body =
+      extraParams !== undefined
+        ? injectAiGatewayExtraParamsRequestBody(bodyWithPromptCacheRetention, extraParams)
+        : bodyWithPromptCacheRetention
+
+    const headers = new Headers(init?.headers)
+    if (extraParams !== undefined || (isAiGatewayOpenAIModelRequest(body) && hasAiGatewayPromptCacheRetention(body))) {
+      // Bifrost only forwards provider-specific extra params when passthrough is enabled.
+      headers.set(AI_GATEWAY_EXTRA_PARAMS_HEADER, 'true')
+    }
 
-  return Object.assign(fetchWithExtraParams, { preconnect: globalThis.fetch.preconnect.bind(globalThis.fetch) })
+    return globalThis.fetch(input, { ...init, headers, body })
+  }
+
+  return Object.assign(fetchWithMutations, { preconnect: globalThis.fetch.preconnect.bind(globalThis.fetch) })
 }
 
 function createAiGatewayProvider(extraParams?: AiGatewayExtraParams) {
@@ -330,11 +370,8 @@ function createAiGatewayProvider(extraParams?: AiGatewayExtraParams) {
   return createOpenAI({
     baseURL,
     apiKey,
-    headers: {
-      [AI_GATEWAY_VIRTUAL_KEY_HEADER]: apiKey,
-      ...(extraParams ? { [AI_GATEWAY_EXTRA_PARAMS_HEADER]: 'true' } : {}),
-    },
-    ...(extraParams ? { fetch: createAiGatewayFetchWithExtraParams(extraParams) } : {}),
+    headers: { [AI_GATEWAY_VIRTUAL_KEY_HEADER]: apiKey },
+    fetch: createAiGatewayFetch(extraParams),
   })
 }
 

package/src/create-runtime.ts

@@ -29,6 +29,8 @@ import type { LotaRuntimeSocialChat } from './runtime/social-chat'
 import { createSocialChatRuntime } from './runtime/social-chat'
 import type { attachmentService } from './services/attachment.service'
 import { attachmentService as attachmentServiceSingleton } from './services/attachment.service'
+import type { autonomousJobService } from './services/autonomous-job.service'
+import { autonomousJobService as autonomousJobServiceSingleton } from './services/autonomous-job.service'
 import { coordinationRegistryService as coordinationRegistryServiceSingleton } from './services/coordination-registry.service'
 import type { documentChunkService } from './services/document-chunk.service'
 import { documentChunkService as documentChunkServiceSingleton } from './services/document-chunk.service'
@@ -112,6 +114,7 @@ export interface LotaRuntime {
     redis: RedisConnectionManager
     closeRedisConnection: () => Promise<void>
     attachmentService: typeof attachmentService
+    autonomousJobService: typeof autonomousJobService
     documentChunkService: typeof documentChunkService
     generatedDocumentStorageService: typeof generatedDocumentStorageService
     memoryService: typeof memoryService
@@ -382,6 +385,7 @@ export async function createLotaRuntime(config: LotaRuntimeConfig): Promise<Lota
         redis: redisManager,
         closeRedisConnection: async () => await redisManager.closeConnection(),
         attachmentService: attachmentServiceSingleton,
+        autonomousJobService: autonomousJobServiceSingleton,
         documentChunkService: documentChunkServiceSingleton,
         generatedDocumentStorageService: generatedDocumentStorageServiceSingleton,
         memoryService: memoryServiceSingleton,
@@ -458,8 +462,13 @@ function getBuiltInSchemaFiles(): URL[] {
     new URL('../infrastructure/schema/01_memory.surql', import.meta.url),
     new URL('../infrastructure/schema/02_execution_plan.surql', import.meta.url),
     new URL('../infrastructure/schema/03_learned_skill.surql', import.meta.url),
-    new URL('../infrastructure/schema/05_recent_activity.surql', import.meta.url),
     new URL('../infrastructure/schema/04_runtime_bootstrap.surql', import.meta.url),
+    new URL('../infrastructure/schema/05_recent_activity.surql', import.meta.url),
+    new URL('../infrastructure/schema/06_playbook.surql', import.meta.url),
+    new URL('../infrastructure/schema/07_institutional_memory.surql', import.meta.url),
+    new URL('../infrastructure/schema/08_quality_metrics.surql', import.meta.url),
+    new URL('../infrastructure/schema/09_queue_job.surql', import.meta.url),
+    new URL('../infrastructure/schema/10_autonomous_job.surql', import.meta.url),
   ]
 }
 

package/src/db/base.service.ts

@@ -1,6 +1,7 @@
 import type { z } from 'zod'
 
 import { NotFoundError } from '../utils/errors'
+import { ensureRecordId } from './record-id'
 import { databaseService as defaultDatabaseService } from './service'
 import type { SurrealDBService } from './service'
 import type { DatabaseTable } from './tables'
@@ -16,7 +17,11 @@ export abstract class BaseService<T extends z.ZodType> {
   }
 
   async findById(id: unknown): Promise<z.infer<T> | null> {
-    return this.databaseService.findOne(this.table, { id }, this.schema)
+    return this.databaseService.findOne(
+      this.table,
+      { id: ensureRecordId(id as Parameters<typeof ensureRecordId>[0], this.table) },
+      this.schema,
+    )
   }
 
   async getById(id: unknown): Promise<z.infer<T>> {

package/src/db/tables.ts

@@ -29,6 +29,10 @@ export const TABLES = {
   PLAYBOOK_VERSION: 'playbookVersion',
   INSTITUTIONAL_MEMORY: 'institutionalMemory',
   QUALITY_METRIC: 'qualityMetric',
+  QUEUE_JOB: 'queueJob',
+  QUEUE_JOB_ATTEMPT: 'queueJobAttempt',
+  AUTONOMOUS_JOB: 'autonomousJob',
+  AUTONOMOUS_JOB_RUN: 'autonomousJobRun',
 } as const
 
 export type DatabaseTable = (typeof TABLES)[keyof typeof TABLES] | (string & {})

package/src/queues/autonomous-job.queue.ts

@@ -0,0 +1,134 @@
+import type { AutonomousJobSchedule } from '@lota-sdk/shared'
+import type { Job } from 'bullmq'
+
+import { serverLogger } from '../config/logger'
+import { databaseService } from '../db/service'
+import { autonomousJobService } from '../services/autonomous-job.service'
+import { queueJobService } from '../services/queue-job.service'
+import type { WorkerHandle } from '../workers/worker-utils'
+import { DEFAULT_JOB_RETENTION } from '../workers/worker-utils'
+import { createQueueFactory } from './queue-factory'
+
+export interface AutonomousJobQueuePayload {
+  autonomousJobId: string
+  autonomousJobRunId?: string
+  trigger: 'scheduled' | 'manual'
+}
+
+export const AUTONOMOUS_JOB_QUEUE = 'autonomous-job'
+
+const DEFAULT_AUTONOMOUS_JOB_OPTIONS = {
+  ...DEFAULT_JOB_RETENTION,
+  attempts: 3,
+  backoff: { type: 'exponential', delay: 5_000 },
+} as const
+
+async function processAutonomousJob(
+  job: Job<AutonomousJobQueuePayload>,
+): Promise<{ status: string; summary?: string }> {
+  await databaseService.connect()
+  return autonomousJobService.executeQueuedRun(job)
+}
+
+const autonomousJobQueue = createQueueFactory<AutonomousJobQueuePayload>({
+  name: AUTONOMOUS_JOB_QUEUE,
+  displayName: 'Autonomous job',
+  jobName: 'run-autonomous-job',
+  concurrency: 2,
+  defaultJobOptions: DEFAULT_AUTONOMOUS_JOB_OPTIONS,
+  processor: processAutonomousJob,
+})
+
+function buildAutonomousSchedulerId(autonomousJobId: string): string {
+  return `autonomous:${autonomousJobId}`
+}
+
+function encodeBullmqId(raw: string): string {
+  return Buffer.from(raw).toString('base64url')
+}
+
+export function buildAutonomousAtJobId(autonomousJobId: string): string {
+  return `autonomous-at-${encodeBullmqId(autonomousJobId)}`
+}
+
+export async function enqueueAutonomousJobRun(params: {
+  payload: AutonomousJobQueuePayload
+  delayMs?: number
+  jobId?: string
+}): Promise<{ bullmqJobId: string; queueJobId: string }> {
+  const queuedJob = await autonomousJobQueue
+    .getQueue()
+    .add('run-autonomous-job', params.payload, {
+      ...(typeof params.delayMs === 'number' ? { delay: Math.max(0, params.delayMs) } : {}),
+      ...(params.jobId ? { jobId: params.jobId } : {}),
+    })
+
+  const queueJobId = await queueJobService.recordEnqueued({
+    queueName: AUTONOMOUS_JOB_QUEUE,
+    id: queuedJob.id,
+    name: queuedJob.name,
+    data: queuedJob.data,
+    opts: queuedJob.opts,
+    attemptsMade: queuedJob.attemptsMade,
+    timestamp: queuedJob.timestamp,
+  })
+
+  return { bullmqJobId: String(queuedJob.id), queueJobId }
+}
+
+export async function upsertAutonomousJobScheduler(params: {
+  autonomousJobId: string
+  schedule: Extract<AutonomousJobSchedule, { kind: 'cron' | 'every' }>
+}): Promise<void> {
+  const repeatOpts =
+    params.schedule.kind === 'cron' ? { pattern: params.schedule.cron } : { every: params.schedule.intervalMs }
+  const queuedJob = await autonomousJobQueue
+    .getQueue()
+    .upsertJobScheduler(buildAutonomousSchedulerId(params.autonomousJobId), repeatOpts, {
+      name: 'run-autonomous-job',
+      data: { autonomousJobId: params.autonomousJobId, trigger: 'scheduled' },
+      opts: DEFAULT_AUTONOMOUS_JOB_OPTIONS,
+    })
+
+  await queueJobService.recordEnqueued({
+    queueName: AUTONOMOUS_JOB_QUEUE,
+    id: queuedJob.id,
+    name: queuedJob.name,
+    data: queuedJob.data,
+    opts: queuedJob.opts,
+    attemptsMade: queuedJob.attemptsMade,
+    timestamp: queuedJob.timestamp,
+  })
+}
+
+export async function removeAutonomousJobScheduler(autonomousJobId: string): Promise<void> {
+  await autonomousJobQueue.getQueue().removeJobScheduler(buildAutonomousSchedulerId(autonomousJobId))
+}
+
+export async function removeAutonomousAtJob(autonomousJobId: string): Promise<void> {
+  try {
+    await autonomousJobQueue.getQueue().remove(buildAutonomousAtJobId(autonomousJobId))
+  } catch {
+    // The delayed job may have already fired or never existed.
+  }
+}
+
+type AutonomousJobWorkerOptions = Parameters<typeof autonomousJobQueue.startWorker>[0]
+
+export function startAutonomousJobWorker(options: AutonomousJobWorkerOptions = {}): WorkerHandle {
+  const handle = autonomousJobQueue.startWorker(options)
+
+  autonomousJobService.recoverActiveJobs().catch((error: unknown) => {
+    serverLogger.error`Autonomous job startup recovery failed: ${error}`
+  })
+
+  return handle
+}
+
+export function getAutonomousJobQueueHandle(): WorkerHandle {
+  return startAutonomousJobWorker()
+}
+
+if (import.meta.main) {
+  startAutonomousJobWorker()
+}

package/src/queues/document-processor.queue.ts

@@ -1,7 +1,9 @@
 import { Queue, Worker } from 'bullmq'
+import type { QueueOptions } from 'bullmq'
 import type IORedis from 'ioredis'
 
 import type { chatLogger } from '../config/logger'
+import { queueJobService } from '../services/queue-job.service'
 import {
   attachWorkerEvents,
   createWorkerShutdown,
@@ -69,22 +71,24 @@ export function createDocumentProcessorQueueRuntime<TJob extends DocumentProcess
   enqueue: (job: TJob) => Promise<unknown>
   startWorker: (options?: { registerSignals?: boolean }) => WorkerHandle
 } {
+  type QueueShape = Queue<TJob, unknown, string, TJob, unknown, string>
+
   const queueName = params.queueName ?? DEFAULT_DOCUMENT_PROCESSOR_QUEUE
   const workerName = params.workerName ?? DEFAULT_WORKER_NAME
   const concurrency = params.concurrency ?? 10
   const lockDuration = params.lockDuration ?? 300_000
-  const jobName = 'process-document' as Parameters<Queue<TJob, unknown, string>['add']>[0]
-  const toQueueData = (job: TJob): Parameters<Queue<TJob, unknown, string>['add']>[1] =>
-    job as Parameters<Queue<TJob, unknown, string>['add']>[1]
-  let queue: Queue<TJob, unknown, string> | null = null
+  const jobName = 'process-document' as Parameters<QueueShape['add']>[0]
+  const toQueueData = (job: TJob): Parameters<QueueShape['add']>[1] => job
+  let queue: QueueShape | null = null
+  const getConnection = (): IORedis => params.getConnectionForBullMQ()
 
-  const getQueue = (): Queue<TJob, unknown, string> => {
+  const getQueue = (): QueueShape => {
     if (queue) {
       return queue
     }
 
-    queue = new Queue<TJob, unknown, string>(queueName, {
-      connection: params.getConnectionForBullMQ(),
+    queue = new Queue<TJob, unknown, string, TJob, unknown, string>(queueName, {
+      connection: getConnection() as QueueOptions['connection'],
       defaultJobOptions: { ...DEFAULT_JOB_RETENTION, attempts: 3, backoff: { type: 'exponential', delay: 1000 } },
     })
 
@@ -92,12 +96,22 @@ export function createDocumentProcessorQueueRuntime<TJob extends DocumentProcess
   }
 
   return {
-    enqueue: async (job) =>
-      await getQueue().add(jobName, toQueueData(job), { jobId: buildDocumentProcessorJobId(job) }),
+    enqueue: async (job) => {
+      const queuedJob = await getQueue().add(jobName, toQueueData(job), { jobId: buildDocumentProcessorJobId(job) })
+      await queueJobService.recordEnqueued({
+        queueName,
+        id: queuedJob.id,
+        name: queuedJob.name,
+        data: queuedJob.data,
+        opts: queuedJob.opts,
+        attemptsMade: queuedJob.attemptsMade,
+        timestamp: queuedJob.timestamp,
+      })
+    },
     startWorker: (options = {}) => {
       const { registerSignals = import.meta.main } = options
       const worker = new Worker(queueName, params.getWorkerPath(), {
-        connection: params.getConnectionForBullMQ(),
+        connection: getConnection() as QueueOptions['connection'],
         concurrency,
         lockDuration,
       })

package/src/queues/index.ts

@@ -1,4 +1,5 @@
 export * from './queue-factory'
+export * from './autonomous-job.queue'
 export * from './context-compaction.queue'
 export * from './delayed-node-promotion.queue'
 export * from './document-processor.queue'

package/src/queues/memory-consolidation.queue.ts

@@ -1,3 +1,4 @@
+import { queueJobService } from '../services/queue-job.service'
 import { getWorkerPath, LONG_JOB_LOCK_DURATION_MS, LOW_JOB_RETENTION } from '../workers/worker-utils'
 import { createQueueFactory } from './queue-factory'
 
@@ -6,7 +7,7 @@ export interface MemoryConsolidationJob {
 }
 
 const MEMORY_CONSOLIDATION_INTERVAL_MS = 24 * 60 * 60 * 1000
-const MEMORY_CONSOLIDATION_JOB_ID = 'memory-consolidation-recurring'
+const MEMORY_CONSOLIDATION_SCHEDULER_ID = 'memory-consolidation-recurring'
 
 const memoryConsolidation = createQueueFactory<MemoryConsolidationJob>({
   name: 'memory-consolidation',
@@ -23,9 +24,27 @@ export async function enqueueMemoryConsolidation(job: MemoryConsolidationJob = {
 }
 
 export async function scheduleRecurringConsolidation() {
-  await memoryConsolidation
+  const queuedJob = await memoryConsolidation
     .getQueue()
-    .add('consolidate', {}, { repeat: { every: MEMORY_CONSOLIDATION_INTERVAL_MS }, jobId: MEMORY_CONSOLIDATION_JOB_ID })
+    .upsertJobScheduler(
+      MEMORY_CONSOLIDATION_SCHEDULER_ID,
+      { every: MEMORY_CONSOLIDATION_INTERVAL_MS },
+      {
+        name: 'consolidate',
+        data: {},
+        opts: { ...LOW_JOB_RETENTION, attempts: 2, backoff: { type: 'exponential', delay: 5000 } },
+      },
+    )
+
+  await queueJobService.recordEnqueued({
+    queueName: 'memory-consolidation',
+    id: queuedJob.id,
+    name: queuedJob.name,
+    data: queuedJob.data,
+    opts: queuedJob.opts,
+    attemptsMade: queuedJob.attemptsMade,
+    timestamp: queuedJob.timestamp,
+  })
 }
 
 export const startMemoryConsolidationWorker = memoryConsolidation.startWorker

package/src/queues/queue-factory.ts

@@ -1,9 +1,10 @@
 import { Queue, Worker } from 'bullmq'
-import type { Job, JobsOptions, WorkerOptions } from 'bullmq'
+import type { Job, JobsOptions, QueueOptions, WorkerOptions } from 'bullmq'
 import type IORedis from 'ioredis'
 
 import { serverLogger } from '../config/logger'
 import { getRedisConnectionForBullMQ } from '../redis'
+import { queueJobService } from '../services/queue-job.service'
 import {
   attachWorkerEvents,
   createTracedWorkerProcessor,
@@ -26,7 +27,7 @@ interface QueueFactoryConfigBase {
 }
 
 interface QueueFactoryConfigInline<TJob> extends QueueFactoryConfigBase {
-  processor: (job: Job<TJob>) => Promise<void>
+  processor: (job: Job<TJob>) => Promise<unknown>
   processorPath?: never
 }
 
@@ -38,39 +39,68 @@ interface QueueFactoryConfigFile extends QueueFactoryConfigBase {
 export type QueueFactoryConfig<TJob> = QueueFactoryConfigInline<TJob> | QueueFactoryConfigFile
 
 export interface QueueFactory<TJob> {
-  getQueue: () => Queue<TJob, unknown, string>
+  getQueue: () => Queue<TJob, unknown, string, TJob, unknown, string>
   enqueue: (job: TJob, options?: JobsOptions) => Promise<void>
   startWorker: (options?: { registerSignals?: boolean }) => WorkerHandle
 }
 
 export function createQueueFactory<TJob>(config: QueueFactoryConfig<TJob>): QueueFactory<TJob> {
-  let _queue: Queue<TJob, unknown, string> | null = null
-
-  const getConnection = () => config.connectionProvider?.() ?? getRedisConnectionForBullMQ()
-
-  const getQueue = (): Queue<TJob, unknown, string> => {
-    if (!_queue) {
-      _queue = new Queue<TJob, unknown, string>(config.name, {
-        connection: getConnection(),
+  type QueueShape = Queue<TJob, unknown, string, TJob, unknown, string>
+
+  let _queue: QueueShape | null = null
+  let _queueConnection: IORedis | null = null
+
+  const getConnection = (): IORedis => config.connectionProvider?.() ?? getRedisConnectionForBullMQ()
+
+  const getQueue = (): QueueShape => {
+    const connection = getConnection()
+    const shouldRecreateQueue =
+      _queue === null ||
+      _queueConnection === null ||
+      _queueConnection !== connection ||
+      _queueConnection.status === 'close' ||
+      _queueConnection.status === 'end'
+
+    if (shouldRecreateQueue) {
+      if (_queue) {
+        void _queue.close().catch((error: unknown) => {
+          serverLogger.warn`Failed to close stale ${config.displayName} queue: ${error}`
+        })
+      }
+
+      _queue = new Queue<TJob, unknown, string, TJob, unknown, string>(config.name, {
+        connection: connection as QueueOptions['connection'],
         defaultJobOptions: { ...DEFAULT_JOB_RETENTION, ...config.defaultJobOptions },
       })
+      _queueConnection = connection
+    }
+    if (_queue === null) {
+      throw new Error(`Failed to initialize queue: ${config.name}`)
     }
     return _queue
   }
 
-  type QueueAdd = Queue<TJob, unknown, string>['add']
-  const jobName = config.jobName as Parameters<QueueAdd>[0]
-  const toData = (job: TJob) => job as Parameters<QueueAdd>[1]
+  const jobName = config.jobName
+  const toData = (job: TJob) => job
 
   const enqueue = async (job: TJob, options?: JobsOptions): Promise<void> => {
-    await getQueue().add(jobName, toData(job), options)
+    const queuedJob = await getQueue().add(jobName, toData(job), options)
+    await queueJobService.recordEnqueued({
+      queueName: config.name,
+      id: queuedJob.id,
+      name: queuedJob.name,
+      data: queuedJob.data,
+      opts: queuedJob.opts,
+      attemptsMade: queuedJob.attemptsMade,
+      timestamp: queuedJob.timestamp,
+    })
   }
 
   const startWorker = (options: { registerSignals?: boolean } = {}): WorkerHandle => {
     const { registerSignals = import.meta.main } = options
 
     const workerOptions: WorkerOptions = {
-      connection: getConnection(),
+      connection: getConnection() as QueueOptions['connection'],
       concurrency: config.concurrency,
       ...(config.lockDuration !== undefined ? { lockDuration: config.lockDuration } : {}),
       ...(config.stalledInterval !== undefined ? { stalledInterval: config.stalledInterval } : {}),