@lota-sdk/core 0.1.14 → 0.1.16

package/src/workers/bootstrap.ts

@@ -1,4 +1,4 @@
-import { configureLogger, serverLogger } from '../config/logger'
+import { configureLotaLogger, serverLogger } from '../config/logger'
 import { LOTA_SDK_DATABASE_NAME } from '../db/sdk-database'
 import { SurrealDBService, databaseService, setDatabaseService } from '../db/service'
 import { connectWithStartupRetry, waitForDatabaseBootstrap } from '../db/startup'
@@ -27,7 +27,7 @@ export async function initializeSandboxedWorkerRuntime(): Promise<void> {
 
   sandboxedWorkerRuntimePromise = (async () => {
     const env = parseWorkerBootstrapEnv(process.env)
-    await configureLogger()
+    await configureLotaLogger()
 
     ensureDatabaseServiceConfigured()
 

package/src/workers/index.ts

@@ -5,3 +5,4 @@ export * from './utils/file-section-chunker'
 export * from './utils/repomix-file-sections'
 export * from './utils/repo-structure-extractor'
 export * from './utils/repomix-process-concurrency'
+export * from './utils/sandbox-error'

package/src/workers/memory-consolidation.worker.ts

@@ -1,6 +1,7 @@
 import type { SandboxedJob } from 'bullmq'
 import { BoundQuery, eq, inside } from 'surrealdb'
 
+import { MEMORY } from '../config/constants'
 import { serverLogger } from '../config/logger'
 import { ensureRecordId, recordIdToString } from '../db/record-id'
 import type { RecordIdInput } from '../db/record-id'
@@ -16,6 +17,7 @@ await initializeSandboxedWorkerRuntime()
 const MEMORY_TABLE = TABLES.MEMORY
 const MEMORY_RELATION_TABLE = TABLES.MEMORY_RELATION
 const MEMORY_HISTORY_TABLE = TABLES.MEMORY_HISTORY
+const RELATION_SUPERSEDES = 'supersedes' as const
 const HARD_SIMILARITY_THRESHOLD = 0.95
 const SOFT_SIMILARITY_THRESHOLD = 0.9
 const MAX_MEMORIES_PER_SCOPE = 500
@@ -130,7 +132,7 @@ async function deduplicateScope(scopeId: string): Promise<number> {
         ensureRecordId(winner.id, TABLES.MEMORY),
         MEMORY_RELATION_TABLE,
         ensureRecordId(loser.id, TABLES.MEMORY),
-        { relationType: 'supersedes', confidence: 1.0 },
+        { relationType: RELATION_SUPERSEDES, confidence: 1.0 },
       )
 
       await databaseService.query(
@@ -187,13 +189,13 @@ async function collapseSupersedeCh(): Promise<number> {
     new BoundQuery(
       `SELECT
        id AS middleId,
-       <-${MEMORY_RELATION_TABLE}[WHERE relationType = 'supersedes']<-${MEMORY_TABLE}.id AS predecessors,
-       ->${MEMORY_RELATION_TABLE}[WHERE relationType = 'supersedes']->${MEMORY_TABLE}.id AS successors
+       <-${MEMORY_RELATION_TABLE}[WHERE relationType = '${RELATION_SUPERSEDES}']<-${MEMORY_TABLE}.id AS predecessors,
+       ->${MEMORY_RELATION_TABLE}[WHERE relationType = '${RELATION_SUPERSEDES}']->${MEMORY_TABLE}.id AS successors
      FROM ${MEMORY_TABLE}
      WHERE archivedAt IS NONE
-       AND count(->${MEMORY_RELATION_TABLE}[WHERE relationType = 'supersedes']) > 0
-       AND count(<-${MEMORY_RELATION_TABLE}[WHERE relationType = 'supersedes']) > 0
-     LIMIT 100`,
+       AND count(->${MEMORY_RELATION_TABLE}[WHERE relationType = '${RELATION_SUPERSEDES}']) > 0
+       AND count(<-${MEMORY_RELATION_TABLE}[WHERE relationType = '${RELATION_SUPERSEDES}']) > 0
+     LIMIT ${MEMORY.MAX_KNN_LIMIT}`,
     ),
   )
 
@@ -210,7 +212,7 @@ async function collapseSupersedeCh(): Promise<number> {
         const existing = await databaseService.query<{ id: RecordIdInput }>(
           new BoundQuery(
             `SELECT id FROM ${MEMORY_RELATION_TABLE}
-           WHERE in = $predId AND out = $succId AND relationType = 'supersedes'
+           WHERE in = $predId AND out = $succId AND relationType = '${RELATION_SUPERSEDES}'
            LIMIT 1`,
             { predId: predRef, succId: succRef },
           ),
@@ -218,7 +220,7 @@ async function collapseSupersedeCh(): Promise<number> {
 
         if (existing.length === 0) {
           await databaseService.relate(predRef, MEMORY_RELATION_TABLE, succRef, {
-            relationType: 'supersedes',
+            relationType: RELATION_SUPERSEDES,
             confidence: 1.0,
           })
         }
@@ -288,10 +290,8 @@ const handler = async (job: SandboxedJob<MemoryConsolidationJob>) => {
         new BoundQuery(`SELECT VALUE scopeId FROM ${MEMORY_TABLE} WHERE archivedAt IS NONE GROUP BY scopeId`),
       )
 
-      for (const scopeId of scopeIds) {
-        const merged = await deduplicateScope(scopeId)
-        totalMerged += merged
-      }
+      const results = await Promise.all(scopeIds.map(deduplicateScope))
+      totalMerged = results.reduce((a, b) => a + b, 0)
     }
 
     const pruned = await pruneStaleMemories()

package/src/workers/regular-chat-memory-digest.helpers.ts

@@ -1,4 +1,5 @@
 import { isAgentName } from '../config/agent-defaults'
+import { compactWhitespace } from '../utils/string'
 
 interface DigestMessageForTranscript {
   source: 'workstream'
@@ -8,10 +9,6 @@ interface DigestMessageForTranscript {
   metadata?: Record<string, unknown>
 }
 
-function normalizeWhitespace(value: string): string {
-  return value.replace(/\s+/g, ' ').trim()
-}
-
 function normalizeFilePartMetadata(part: Record<string, unknown>): string | null {
   if (part.type !== 'file') return null
 
@@ -57,9 +54,7 @@ export function buildDigestTranscript(params: { messages: DigestMessageForTransc
 
     const sourcePrefix = `[${message.source}:${message.sourceId}]`
     const textParts = message.parts
-      .flatMap((part) =>
-        part.type === 'text' && typeof part.text === 'string' ? [normalizeWhitespace(part.text)] : [],
-      )
+      .flatMap((part) => (part.type === 'text' && typeof part.text === 'string' ? [compactWhitespace(part.text)] : []))
       .filter((value) => value.length > 0)
     const fileParts = message.parts
       .map((part) => normalizeFilePartMetadata(part))

package/src/workers/regular-chat-memory-digest.runner.ts

@@ -1,4 +1,3 @@
-import { toTimestamp } from '@lota-sdk/shared'
 import { BoundQuery } from 'surrealdb'
 import { z } from 'zod'
 
@@ -16,26 +15,20 @@ import { createHelperModelRuntime } from '../runtime/helper-model'
 import { getRuntimeAdapters, withConfiguredWorkspaceMemoryLock } from '../runtime/runtime-extensions'
 import { memoryService } from '../services/memory.service'
 import { createRegularChatMemoryDigestAgent } from '../system-agents/regular-chat-memory-digest.agent'
+import { compactWhitespace } from '../utils/string'
 import { buildDigestTranscript, resolveWorkspaceBootstrapCutoff } from './regular-chat-memory-digest.helpers'
+import {
+  compareDigestMessageOrder,
+  listEligibleWorkstreamMessages,
+  listWorkstreamIdsForOrg,
+  normalizeBlock,
+} from './utils/workstream-message-query'
+import type { DigestCursor, DigestMessage } from './utils/workstream-message-query'
 
 const StructuredProfilePatchSchema = z.record(z.string(), z.unknown()).default({})
 
 const REGULAR_CHAT_MEMORY_DIGEST_TIMEOUT_MS = 10 * 60 * 1000
 const WorkspaceMemoryRowSchema = z.object({ content: z.string() })
-const EntityIdRowSchema = z.string().trim().min(1)
-const RecordTimestampSchema = z.union([z.date(), z.string(), z.number()])
-const MessageRoleSchema = z.enum(['system', 'user', 'assistant'])
-const MessagePartSchema = z.record(z.string(), z.unknown())
-const MessageMetadataSchema = z.record(z.string(), z.unknown()).nullish()
-
-const WorkstreamDigestMessageRowSchema = z.object({
-  id: z.string(),
-  workstreamId: z.string(),
-  role: MessageRoleSchema,
-  parts: z.array(MessagePartSchema).optional(),
-  metadata: MessageMetadataSchema,
-  createdAt: RecordTimestampSchema,
-})
 
 const ExtractedFactSchema = z.object({
   content: z.string().trim().min(1),
@@ -52,40 +45,18 @@ const RegularChatMemoryDigestOutputSchema = z.object({
 
 const helperModelRuntime = createHelperModelRuntime()
 
-interface DigestCursor {
-  createdAt: Date
-  id: string
-}
-
-interface DigestMessage {
-  source: 'workstream'
-  sourceId: string
-  role: 'system' | 'user' | 'assistant'
-  parts: Array<Record<string, unknown>>
-  metadata?: Record<string, unknown>
-  cursor: DigestCursor
-}
-
 interface RegularChatDigestRunResult {
   skipped: boolean
   processedWorkstreamMessages: number
   followUpScheduled: boolean
 }
 
-function normalizeWhitespace(value: string): string {
-  return value.replace(/\s+/g, ' ').trim()
-}
-
-function normalizeBlock(value: string): string {
-  return value.replaceAll(String.fromCharCode(0), '').replace(/\r/g, '').trim()
-}
-
 function buildMemoryContext(memories: Array<{ content: string }>): string {
   if (memories.length === 0) return 'No existing memories.'
 
   return memories
     .map((memory, index) => {
-      const content = normalizeWhitespace(memory.content)
+      const content = compactWhitespace(memory.content)
       if (!content) return ''
       return `${index + 1}. ${content}`
     })
@@ -117,75 +88,10 @@ function buildPrompt(params: {
   ].join('\n')
 }
 
-function mapWorkstreamDigestRow(row: z.infer<typeof WorkstreamDigestMessageRowSchema>): DigestMessage {
-  return {
-    source: 'workstream',
-    sourceId: row.workstreamId,
-    role: row.role,
-    parts: row.parts ?? [],
-    metadata: row.metadata ?? undefined,
-    cursor: { createdAt: new Date(toTimestamp(row.createdAt)), id: row.id },
-  }
-}
-
-function compareDigestMessageOrder(left: DigestMessage, right: DigestMessage): number {
-  const timeDiff = left.cursor.createdAt.getTime() - right.cursor.createdAt.getTime()
-  if (timeDiff !== 0) return timeDiff
-  return left.cursor.id.localeCompare(right.cursor.id)
-}
-
 function getLastCursor(messages: DigestMessage[]): DigestCursor | null {
   return messages.length > 0 ? messages[messages.length - 1].cursor : null
 }
 
-async function listWorkstreamIdsForOrg(orgRef: RecordIdRef): Promise<RecordIdRef[]> {
-  const ids = await databaseService.query<unknown>(
-    new BoundQuery(
-      `SELECT VALUE type::string(id) FROM ${TABLES.WORKSTREAM}
-       WHERE organizationId = $organizationId`,
-      { organizationId: orgRef },
-    ),
-  )
-
-  return ids.map((value) => ensureRecordId(EntityIdRowSchema.parse(value), TABLES.WORKSTREAM))
-}
-
-async function listEligibleWorkstreamMessages(params: {
-  workstreamIds: RecordIdRef[]
-  cursor: DigestCursor | null
-  onboardingCutoff: Date | null
-}): Promise<DigestMessage[]> {
-  if (params.workstreamIds.length === 0) return []
-
-  let query: BoundQuery | null = null
-  if (params.cursor) {
-    const cursorRowId = ensureRecordId(params.cursor.id, TABLES.WORKSTREAM_MESSAGE)
-    query = new BoundQuery(
-      `SELECT type::string(id) AS id, type::string(workstreamId) AS workstreamId, role, parts, metadata, createdAt FROM ${TABLES.WORKSTREAM_MESSAGE}
-       WHERE workstreamId IN $workstreamIds
-         AND (
-           createdAt > $cursorCreatedAt
-           OR (createdAt = $cursorCreatedAt AND id > $cursorRowId)
-         )
-       ORDER BY createdAt ASC, id ASC`,
-      { workstreamIds: params.workstreamIds, cursorCreatedAt: params.cursor.createdAt, cursorRowId },
-    )
-  } else if (params.onboardingCutoff) {
-    query = new BoundQuery(
-      `SELECT type::string(id) AS id, type::string(workstreamId) AS workstreamId, role, parts, metadata, createdAt FROM ${TABLES.WORKSTREAM_MESSAGE}
-       WHERE workstreamId IN $workstreamIds
-         AND createdAt > $onboardingCutoff
-       ORDER BY createdAt ASC, id ASC`,
-      { workstreamIds: params.workstreamIds, onboardingCutoff: params.onboardingCutoff },
-    )
-  }
-
-  if (!query) return []
-
-  const rows = await databaseService.query<unknown>(query)
-  return rows.map((row) => mapWorkstreamDigestRow(WorkstreamDigestMessageRowSchema.parse(row)))
-}
-
 async function hasNewEligibleWorkstreamMessages(params: {
   workstreamIds: RecordIdRef[]
   cursor: DigestCursor | null
@@ -224,7 +130,7 @@ async function hasNewEligibleWorkstreamMessages(params: {
 }
 
 async function loadExistingOrganizationMemories(orgId: string): Promise<Array<{ content: string }>> {
-  return await databaseService.queryMany(
+  return databaseService.queryMany(
     new BoundQuery(
       `SELECT content, createdAt, id FROM ${TABLES.MEMORY}
        WHERE metadata.orgId = $orgId
@@ -249,7 +155,7 @@ export async function runRegularChatMemoryDigest(
     return { skipped: true, processedWorkstreamMessages: 0, followUpScheduled: false }
   }
 
-  return await withConfiguredWorkspaceMemoryLock(orgId, async () => {
+  return withConfiguredWorkspaceMemoryLock(orgId, async () => {
     if (
       !workspaceProvider.getBackgroundCursor ||
       !workspaceProvider.setBackgroundCursor ||

package/src/workers/skill-extraction.runner.ts

@@ -1,11 +1,5 @@
-import { toTimestamp } from '@lota-sdk/shared'
-import { BoundQuery } from 'surrealdb'
-import { z } from 'zod'
-
 import { serverLogger } from '../config/logger'
 import { ensureRecordId, recordIdToString } from '../db/record-id'
-import type { RecordIdRef } from '../db/record-id'
-import { databaseService } from '../db/service'
 import { TABLES } from '../db/tables'
 import { getDefaultEmbeddings } from '../embeddings/provider'
 import type { SkillExtractionJob } from '../queues/skill-extraction.queue'
@@ -16,38 +10,15 @@ import { createSkillExtractorAgent, SkillExtractionOutputSchema } from '../syste
 import type { SkillCandidate } from '../system-agents/skill-extractor.agent'
 import { createSkillManagerAgent, SkillManagerOutputSchema } from '../system-agents/skill-manager.agent'
 import { buildDigestTranscript, resolveWorkspaceBootstrapCutoff } from './regular-chat-memory-digest.helpers'
+import {
+  compareDigestMessageOrder,
+  listEligibleWorkstreamMessages,
+  listWorkstreamIdsForOrg,
+} from './utils/workstream-message-query'
 
 const SKILL_EXTRACTION_TIMEOUT_MS = 10 * 60 * 1000
 const MIN_MESSAGE_THRESHOLD = 10
 
-const RecordTimestampSchema = z.union([z.date(), z.string(), z.number()])
-const MessageRoleSchema = z.enum(['system', 'user', 'assistant'])
-const MessagePartSchema = z.record(z.string(), z.unknown())
-const MessageMetadataSchema = z.record(z.string(), z.unknown()).nullish()
-
-const WorkstreamMessageRowSchema = z.object({
-  id: z.string(),
-  workstreamId: z.string(),
-  role: MessageRoleSchema,
-  parts: z.array(MessagePartSchema).optional(),
-  metadata: MessageMetadataSchema,
-  createdAt: RecordTimestampSchema,
-})
-
-interface DigestCursor {
-  createdAt: Date
-  id: string
-}
-
-interface DigestMessage {
-  source: 'workstream'
-  sourceId: string
-  role: 'system' | 'user' | 'assistant'
-  parts: Array<Record<string, unknown>>
-  metadata?: Record<string, unknown>
-  cursor: DigestCursor
-}
-
 interface SkillExtractionRunResult {
   skipped: boolean
   processedMessages: number
@@ -58,71 +29,6 @@ const embeddings = getDefaultEmbeddings()
 
 const helperModelRuntime = createHelperModelRuntime()
 
-function mapWorkstreamRow(row: z.infer<typeof WorkstreamMessageRowSchema>): DigestMessage {
-  return {
-    source: 'workstream',
-    sourceId: row.workstreamId,
-    role: row.role,
-    parts: row.parts ?? [],
-    metadata: row.metadata ?? undefined,
-    cursor: { createdAt: new Date(toTimestamp(row.createdAt)), id: row.id },
-  }
-}
-
-function compareMessageOrder(left: DigestMessage, right: DigestMessage): number {
-  const timeDiff = left.cursor.createdAt.getTime() - right.cursor.createdAt.getTime()
-  if (timeDiff !== 0) return timeDiff
-  return left.cursor.id.localeCompare(right.cursor.id)
-}
-
-async function listWorkstreamIdsForOrg(orgRef: RecordIdRef): Promise<RecordIdRef[]> {
-  const EntityIdRowSchema = z.string().trim().min(1)
-  const ids = await databaseService.query<unknown>(
-    new BoundQuery(
-      `SELECT VALUE type::string(id) FROM ${TABLES.WORKSTREAM}
-       WHERE organizationId = $organizationId`,
-      { organizationId: orgRef },
-    ),
-  )
-  return ids.map((value) => ensureRecordId(EntityIdRowSchema.parse(value), TABLES.WORKSTREAM))
-}
-
-async function listEligibleMessages(params: {
-  workstreamIds: RecordIdRef[]
-  cursor: DigestCursor | null
-  onboardingCutoff: Date | null
-}): Promise<DigestMessage[]> {
-  if (params.workstreamIds.length === 0) return []
-
-  let query: BoundQuery | null = null
-  if (params.cursor) {
-    const cursorRowId = ensureRecordId(params.cursor.id, TABLES.WORKSTREAM_MESSAGE)
-    query = new BoundQuery(
-      `SELECT type::string(id) AS id, type::string(workstreamId) AS workstreamId, role, parts, metadata, createdAt FROM ${TABLES.WORKSTREAM_MESSAGE}
-       WHERE workstreamId IN $workstreamIds
-         AND (
-           createdAt > $cursorCreatedAt
-           OR (createdAt = $cursorCreatedAt AND id > $cursorRowId)
-         )
-       ORDER BY createdAt ASC, id ASC`,
-      { workstreamIds: params.workstreamIds, cursorCreatedAt: params.cursor.createdAt, cursorRowId },
-    )
-  } else if (params.onboardingCutoff) {
-    query = new BoundQuery(
-      `SELECT type::string(id) AS id, type::string(workstreamId) AS workstreamId, role, parts, metadata, createdAt FROM ${TABLES.WORKSTREAM_MESSAGE}
-       WHERE workstreamId IN $workstreamIds
-         AND createdAt > $onboardingCutoff
-       ORDER BY createdAt ASC, id ASC`,
-      { workstreamIds: params.workstreamIds, onboardingCutoff: params.onboardingCutoff },
-    )
-  }
-
-  if (!query) return []
-
-  const rows = await databaseService.query<unknown>(query)
-  return rows.map((row) => mapWorkstreamRow(WorkstreamMessageRowSchema.parse(row)))
-}
-
 function buildExtractionPrompt(params: { workspaceName: string; transcript: string; existingSkills: string }): string {
   return [
     `Workspace name: ${params.workspaceName}`,
@@ -180,7 +86,7 @@ export async function runSkillExtraction(data: SkillExtractionJob): Promise<Skil
     return { skipped: true, processedMessages: 0, extractedSkills: 0 }
   }
 
-  return await withConfiguredWorkspaceMemoryLock(orgId, async () => {
+  return withConfiguredWorkspaceMemoryLock(orgId, async () => {
     const workspace = await cursorAwareWorkspaceProvider.getWorkspace(orgRef)
     const lifecycleState = await cursorAwareWorkspaceProvider.getLifecycleState?.(workspace)
     if (lifecycleState?.bootstrapActive ?? false) {
@@ -196,14 +102,14 @@ export async function runSkillExtraction(data: SkillExtractionJob): Promise<Skil
     })
 
     const workstreamIds = await listWorkstreamIdsForOrg(orgRef)
-    const messages = await listEligibleMessages({ workstreamIds, cursor: existingCursor, onboardingCutoff })
+    const messages = await listEligibleWorkstreamMessages({ workstreamIds, cursor: existingCursor, onboardingCutoff })
 
     if (messages.length < MIN_MESSAGE_THRESHOLD) {
       serverLogger.info`Skipping skill extraction for ${orgId}: only ${messages.length} messages (threshold: ${MIN_MESSAGE_THRESHOLD})`
       return { skipped: true, processedMessages: messages.length, extractedSkills: 0 }
     }
 
-    const sortedMessages = [...messages].sort(compareMessageOrder)
+    const sortedMessages = [...messages].sort(compareDigestMessageOrder)
     const { transcript } = buildDigestTranscript({ messages: sortedMessages })
 
     const existingSkills = await learnedSkillService.listForOrg(orgId)

package/src/workers/utils/file-section-chunker.ts

@@ -1,7 +1,10 @@
+import { CHARS_PER_TOKEN_ESTIMATE } from '../../utils/string'
+
 export const DEFAULT_FILE_SECTION_CHUNK_MAX_CHARS = 250_000
-const MIN_FILE_SECTION_CHUNK_MAX_CHARS = 4_000
+export const MIN_FILE_SECTION_CHUNK_MAX_CHARS = 4_000
 export const DEFAULT_FILE_SECTION_CHUNK_MIN_CHARS = 10_000
 const SECTION_SEPARATOR_LENGTH = 2
+const MIN_CHUNK_CHARS_FLOOR = 512
 
 export interface FileSection {
   kind: 'preamble' | 'file'
@@ -29,7 +32,7 @@ export interface FileSectionChunkOptions {
 
 function estimateTokenCountFromChars(text: string): number {
   if (!text) return 0
-  return Math.ceil(text.length / 3)
+  return Math.ceil(text.length / CHARS_PER_TOKEN_ESTIMATE)
 }
 
 function normalizeMaxChars(value?: number): number {
@@ -43,7 +46,7 @@ function normalizeMinChunkChars(value: number | undefined, maxChars: number): nu
   if (typeof value !== 'number' || !Number.isFinite(value)) {
     return Math.min(DEFAULT_FILE_SECTION_CHUNK_MIN_CHARS, Math.floor(maxChars * 0.35))
   }
-  const normalized = Math.max(512, Math.floor(value))
+  const normalized = Math.max(MIN_CHUNK_CHARS_FLOOR, Math.floor(value))
   return Math.min(normalized, Math.floor(maxChars * 0.6))
 }
 

package/src/workers/utils/repomix-file-sections.ts

@@ -27,7 +27,7 @@ export function parseRepomixFileSections(repomixOutput: string): FileSection[] {
     const nextStart = matches[index + 1]?.index ?? source.length
     const content = source.slice(start, nextStart).trim()
     if (!content) continue
-    const filePath = (match[1] ?? '').trim()
+    const filePath = match[1].trim()
     sections.push({ kind: 'file', content, filePath: filePath || undefined })
   }
 
@@ -38,5 +38,5 @@ export async function chunkRepomixFileSections(
   repomixOutput: string,
   options: FileSectionChunkOptions = {},
 ): Promise<FileSectionChunk[]> {
-  return await chunkFileSections(parseRepomixFileSections(repomixOutput), options)
+  return chunkFileSections(parseRepomixFileSections(repomixOutput), options)
 }

package/src/workers/utils/sandbox-error.ts

@@ -1,5 +1,14 @@
-export function toSandboxedWorkerError(error: unknown, context?: string): Error {
-  const base = error instanceof Error ? error : new Error(String(error))
+export interface SandboxedWorkerError {
+  name: string
+  message: string
+  stack?: string
+}
+
+export function toSandboxedWorkerError(error: unknown, context?: string): SandboxedWorkerError {
+  const base =
+    error instanceof Error
+      ? { name: error.name || 'Error', message: error.message, stack: error.stack }
+      : { name: 'Error', message: String(error) }
   if (context) base.message = `${context}: ${base.message}`
   return base
 }

package/src/workers/utils/workstream-message-query.ts

@@ -0,0 +1,97 @@
+import { requireTimestamp } from '@lota-sdk/shared'
+import { BoundQuery } from 'surrealdb'
+import { z } from 'zod'
+
+import { ensureRecordId, recordIdToString } from '../../db/record-id'
+import type { RecordIdRef } from '../../db/record-id'
+import { databaseService } from '../../db/service'
+import { TABLES } from '../../db/tables'
+import { WorkstreamMessageRowSchema } from '../../db/workstream-message-row'
+import type { WorkstreamMessageRow } from '../../db/workstream-message-row'
+import { normalizeTextBody } from '../../document/parsing'
+
+export interface DigestCursor {
+  createdAt: Date
+  id: string
+}
+
+export interface DigestMessage {
+  source: 'workstream'
+  sourceId: string
+  role: 'system' | 'user' | 'assistant'
+  parts: Array<Record<string, unknown>>
+  metadata?: Record<string, unknown>
+  cursor: DigestCursor
+}
+
+function mapWorkstreamRow(row: WorkstreamMessageRow): DigestMessage {
+  return {
+    source: 'workstream',
+    sourceId: recordIdToString(row.workstreamId, TABLES.WORKSTREAM),
+    role: row.role,
+    parts: row.parts as Array<Record<string, unknown>>,
+    metadata: row.metadata ?? undefined,
+    cursor: {
+      createdAt: new Date(requireTimestamp(row.createdAt)),
+      id: recordIdToString(row.id, TABLES.WORKSTREAM_MESSAGE),
+    },
+  }
+}
+
+export function compareDigestMessageOrder(left: DigestMessage, right: DigestMessage): number {
+  const timeDiff = left.cursor.createdAt.getTime() - right.cursor.createdAt.getTime()
+  if (timeDiff !== 0) return timeDiff
+  return left.cursor.id.localeCompare(right.cursor.id)
+}
+
+export async function listWorkstreamIdsForOrg(orgRef: RecordIdRef): Promise<RecordIdRef[]> {
+  const EntityIdRowSchema = z.string().trim().min(1)
+  const ids = await databaseService.query<unknown>(
+    new BoundQuery(
+      `SELECT VALUE type::string(id) FROM ${TABLES.WORKSTREAM}
+       WHERE organizationId = $organizationId`,
+      { organizationId: orgRef },
+    ),
+  )
+  return ids.map((value) => ensureRecordId(EntityIdRowSchema.parse(value), TABLES.WORKSTREAM))
+}
+
+export async function listEligibleWorkstreamMessages(params: {
+  workstreamIds: RecordIdRef[]
+  cursor: DigestCursor | null
+  onboardingCutoff: Date | null
+}): Promise<DigestMessage[]> {
+  if (params.workstreamIds.length === 0) return []
+
+  let query: BoundQuery | null = null
+  if (params.cursor) {
+    const cursorRowId = ensureRecordId(params.cursor.id, TABLES.WORKSTREAM_MESSAGE)
+    query = new BoundQuery(
+      `SELECT type::string(id) AS id, type::string(workstreamId) AS workstreamId, role, parts, metadata, createdAt FROM ${TABLES.WORKSTREAM_MESSAGE}
+       WHERE workstreamId IN $workstreamIds
+         AND (
+           createdAt > $cursorCreatedAt
+           OR (createdAt = $cursorCreatedAt AND id > $cursorRowId)
+         )
+       ORDER BY createdAt ASC, id ASC`,
+      { workstreamIds: params.workstreamIds, cursorCreatedAt: params.cursor.createdAt, cursorRowId },
+    )
+  } else if (params.onboardingCutoff) {
+    query = new BoundQuery(
+      `SELECT type::string(id) AS id, type::string(workstreamId) AS workstreamId, role, parts, metadata, createdAt FROM ${TABLES.WORKSTREAM_MESSAGE}
+       WHERE workstreamId IN $workstreamIds
+         AND createdAt > $onboardingCutoff
+       ORDER BY createdAt ASC, id ASC`,
+      { workstreamIds: params.workstreamIds, onboardingCutoff: params.onboardingCutoff },
+    )
+  }
+
+  if (!query) return []
+
+  const rows = await databaseService.query<unknown>(query)
+  return rows.map((row) => mapWorkstreamRow(WorkstreamMessageRowSchema.parse(row)))
+}
+
+export function normalizeBlock(value: string): string {
+  return normalizeTextBody(value)
+}

package/src/workers/worker-utils.ts

@@ -4,6 +4,11 @@ import { fileURLToPath } from 'node:url'
 import type { Job, Worker } from 'bullmq'
 
 import { chatLogger } from '../config/logger'
+import { truncateText } from '../utils/string'
+
+export const DEFAULT_JOB_RETENTION = { removeOnComplete: 200, removeOnFail: 200 }
+export const LOW_JOB_RETENTION = { removeOnComplete: 50, removeOnFail: 50 }
+export const LONG_JOB_LOCK_DURATION_MS = 600_000
 
 const DEFAULT_SHUTDOWN_TIMEOUT_MS = 10_000
 const MAX_TRACE_STRING_CHARS = 2_000
@@ -28,8 +33,7 @@ interface TracedWorkerJobLike {
 }
 
 function truncateTraceString(value: string, maxChars = MAX_TRACE_STRING_CHARS): string {
-  if (value.length <= maxChars) return value
-  return `${value.slice(0, maxChars - 3)}...`
+  return truncateText(value, maxChars)
 }
 
 function normalizeTraceValue(value: unknown, depth = 0): unknown {

package/src/runtime/retrieval-pipeline.ts

@@ -1,3 +0,0 @@
-export function resolveCandidateLimit(params: { limit: number; multiplier: number; minimum: number }): number {
-  return Math.max(params.limit * params.multiplier, params.minimum)
-}