@lota-sdk/core 0.1.13 → 0.1.15

package/src/utils/index.ts

@@ -1,6 +1,14 @@
 export * from './async'
 export * from './date-time'
-export * from './error'
 export * from './errors'
 export * from './hono-error-handler'
-export * from './string'
+export * from './sse-keepalive'
+export {
+  CHARS_PER_TOKEN_ESTIMATE,
+  compactWhitespace,
+  isRecord,
+  readString,
+  readStringField,
+  truncateOptionalText,
+  truncateText,
+} from './string'

package/src/utils/string.ts

@@ -49,3 +49,17 @@ export function truncateOptionalText(value: string | undefined, maxChars: number
 export function compactWhitespace(value: string): string {
   return value.trim().replace(/\s+/g, ' ')
 }
+
+/**
+ * Returns the value as a plain record if it is a non-null, non-array object,
+ * or null otherwise.
+ */
+export function readRecord(value: unknown): Record<string, unknown> | null {
+  if (!value || typeof value !== 'object' || Array.isArray(value)) return null
+  return value as Record<string, unknown>
+}
+
+/**
+ * Rough character-to-token estimate used for context budget calculations.
+ */
+export const CHARS_PER_TOKEN_ESTIMATE = 3

package/src/workers/bootstrap.ts

@@ -1,4 +1,4 @@
-import { configureLogger, serverLogger } from '../config/logger'
+import { configureLotaLogger, serverLogger } from '../config/logger'
 import { LOTA_SDK_DATABASE_NAME } from '../db/sdk-database'
 import { SurrealDBService, databaseService, setDatabaseService } from '../db/service'
 import { connectWithStartupRetry, waitForDatabaseBootstrap } from '../db/startup'
@@ -27,7 +27,7 @@ export async function initializeSandboxedWorkerRuntime(): Promise<void> {
 
   sandboxedWorkerRuntimePromise = (async () => {
     const env = parseWorkerBootstrapEnv(process.env)
-    await configureLogger()
+    await configureLotaLogger()
 
     ensureDatabaseServiceConfigured()
 

package/src/workers/memory-consolidation.worker.ts

@@ -1,6 +1,7 @@
 import type { SandboxedJob } from 'bullmq'
 import { BoundQuery, eq, inside } from 'surrealdb'
 
+import { MEMORY } from '../config/constants'
 import { serverLogger } from '../config/logger'
 import { ensureRecordId, recordIdToString } from '../db/record-id'
 import type { RecordIdInput } from '../db/record-id'
@@ -16,6 +17,7 @@ await initializeSandboxedWorkerRuntime()
 const MEMORY_TABLE = TABLES.MEMORY
 const MEMORY_RELATION_TABLE = TABLES.MEMORY_RELATION
 const MEMORY_HISTORY_TABLE = TABLES.MEMORY_HISTORY
+const RELATION_SUPERSEDES = 'supersedes' as const
 const HARD_SIMILARITY_THRESHOLD = 0.95
 const SOFT_SIMILARITY_THRESHOLD = 0.9
 const MAX_MEMORIES_PER_SCOPE = 500
@@ -130,7 +132,7 @@ async function deduplicateScope(scopeId: string): Promise<number> {
         ensureRecordId(winner.id, TABLES.MEMORY),
         MEMORY_RELATION_TABLE,
         ensureRecordId(loser.id, TABLES.MEMORY),
-        { relationType: 'supersedes', confidence: 1.0 },
+        { relationType: RELATION_SUPERSEDES, confidence: 1.0 },
       )
 
       await databaseService.query(
@@ -187,13 +189,13 @@ async function collapseSupersedeCh(): Promise<number> {
     new BoundQuery(
       `SELECT
        id AS middleId,
-       <-${MEMORY_RELATION_TABLE}[WHERE relationType = 'supersedes']<-${MEMORY_TABLE}.id AS predecessors,
-       ->${MEMORY_RELATION_TABLE}[WHERE relationType = 'supersedes']->${MEMORY_TABLE}.id AS successors
+       <-${MEMORY_RELATION_TABLE}[WHERE relationType = '${RELATION_SUPERSEDES}']<-${MEMORY_TABLE}.id AS predecessors,
+       ->${MEMORY_RELATION_TABLE}[WHERE relationType = '${RELATION_SUPERSEDES}']->${MEMORY_TABLE}.id AS successors
      FROM ${MEMORY_TABLE}
      WHERE archivedAt IS NONE
-       AND count(->${MEMORY_RELATION_TABLE}[WHERE relationType = 'supersedes']) > 0
-       AND count(<-${MEMORY_RELATION_TABLE}[WHERE relationType = 'supersedes']) > 0
-     LIMIT 100`,
+       AND count(->${MEMORY_RELATION_TABLE}[WHERE relationType = '${RELATION_SUPERSEDES}']) > 0
+       AND count(<-${MEMORY_RELATION_TABLE}[WHERE relationType = '${RELATION_SUPERSEDES}']) > 0
+     LIMIT ${MEMORY.MAX_KNN_LIMIT}`,
     ),
   )
 
@@ -210,7 +212,7 @@ async function collapseSupersedeCh(): Promise<number> {
         const existing = await databaseService.query<{ id: RecordIdInput }>(
           new BoundQuery(
             `SELECT id FROM ${MEMORY_RELATION_TABLE}
-           WHERE in = $predId AND out = $succId AND relationType = 'supersedes'
+           WHERE in = $predId AND out = $succId AND relationType = '${RELATION_SUPERSEDES}'
            LIMIT 1`,
             { predId: predRef, succId: succRef },
           ),
@@ -218,7 +220,7 @@ async function collapseSupersedeCh(): Promise<number> {
 
         if (existing.length === 0) {
           await databaseService.relate(predRef, MEMORY_RELATION_TABLE, succRef, {
-            relationType: 'supersedes',
+            relationType: RELATION_SUPERSEDES,
             confidence: 1.0,
           })
         }
@@ -288,10 +290,8 @@ const handler = async (job: SandboxedJob<MemoryConsolidationJob>) => {
         new BoundQuery(`SELECT VALUE scopeId FROM ${MEMORY_TABLE} WHERE archivedAt IS NONE GROUP BY scopeId`),
       )
 
-      for (const scopeId of scopeIds) {
-        const merged = await deduplicateScope(scopeId)
-        totalMerged += merged
-      }
+      const results = await Promise.all(scopeIds.map(deduplicateScope))
+      totalMerged = results.reduce((a, b) => a + b, 0)
     }
 
     const pruned = await pruneStaleMemories()

package/src/workers/regular-chat-memory-digest.helpers.ts

@@ -1,4 +1,5 @@
 import { isAgentName } from '../config/agent-defaults'
+import { compactWhitespace } from '../utils/string'
 
 interface DigestMessageForTranscript {
   source: 'workstream'
@@ -8,10 +9,6 @@ interface DigestMessageForTranscript {
   metadata?: Record<string, unknown>
 }
 
-function normalizeWhitespace(value: string): string {
-  return value.replace(/\s+/g, ' ').trim()
-}
-
 function normalizeFilePartMetadata(part: Record<string, unknown>): string | null {
   if (part.type !== 'file') return null
 
@@ -57,9 +54,7 @@ export function buildDigestTranscript(params: { messages: DigestMessageForTransc
 
     const sourcePrefix = `[${message.source}:${message.sourceId}]`
     const textParts = message.parts
-      .flatMap((part) =>
-        part.type === 'text' && typeof part.text === 'string' ? [normalizeWhitespace(part.text)] : [],
-      )
+      .flatMap((part) => (part.type === 'text' && typeof part.text === 'string' ? [compactWhitespace(part.text)] : []))
       .filter((value) => value.length > 0)
     const fileParts = message.parts
       .map((part) => normalizeFilePartMetadata(part))

package/src/workers/regular-chat-memory-digest.runner.ts

@@ -1,4 +1,3 @@
-import { toTimestamp } from '@lota-sdk/shared'
 import { BoundQuery } from 'surrealdb'
 import { z } from 'zod'
 
@@ -16,26 +15,20 @@ import { createHelperModelRuntime } from '../runtime/helper-model'
 import { getRuntimeAdapters, withConfiguredWorkspaceMemoryLock } from '../runtime/runtime-extensions'
 import { memoryService } from '../services/memory.service'
 import { createRegularChatMemoryDigestAgent } from '../system-agents/regular-chat-memory-digest.agent'
+import { compactWhitespace } from '../utils/string'
 import { buildDigestTranscript, resolveWorkspaceBootstrapCutoff } from './regular-chat-memory-digest.helpers'
+import {
+  compareDigestMessageOrder,
+  listEligibleWorkstreamMessages,
+  listWorkstreamIdsForOrg,
+  normalizeBlock,
+} from './utils/workstream-message-query'
+import type { DigestCursor, DigestMessage } from './utils/workstream-message-query'
 
 const StructuredProfilePatchSchema = z.record(z.string(), z.unknown()).default({})
 
 const REGULAR_CHAT_MEMORY_DIGEST_TIMEOUT_MS = 10 * 60 * 1000
 const WorkspaceMemoryRowSchema = z.object({ content: z.string() })
-const EntityIdRowSchema = z.string().trim().min(1)
-const RecordTimestampSchema = z.union([z.date(), z.string(), z.number()])
-const MessageRoleSchema = z.enum(['system', 'user', 'assistant'])
-const MessagePartSchema = z.record(z.string(), z.unknown())
-const MessageMetadataSchema = z.record(z.string(), z.unknown()).nullish()
-
-const WorkstreamDigestMessageRowSchema = z.object({
-  id: z.string(),
-  workstreamId: z.string(),
-  role: MessageRoleSchema,
-  parts: z.array(MessagePartSchema).optional(),
-  metadata: MessageMetadataSchema,
-  createdAt: RecordTimestampSchema,
-})
 
 const ExtractedFactSchema = z.object({
   content: z.string().trim().min(1),
@@ -52,40 +45,18 @@ const RegularChatMemoryDigestOutputSchema = z.object({
 
 const helperModelRuntime = createHelperModelRuntime()
 
-interface DigestCursor {
-  createdAt: Date
-  id: string
-}
-
-interface DigestMessage {
-  source: 'workstream'
-  sourceId: string
-  role: 'system' | 'user' | 'assistant'
-  parts: Array<Record<string, unknown>>
-  metadata?: Record<string, unknown>
-  cursor: DigestCursor
-}
-
 interface RegularChatDigestRunResult {
   skipped: boolean
   processedWorkstreamMessages: number
   followUpScheduled: boolean
 }
 
-function normalizeWhitespace(value: string): string {
-  return value.replace(/\s+/g, ' ').trim()
-}
-
-function normalizeBlock(value: string): string {
-  return value.replaceAll(String.fromCharCode(0), '').replace(/\r/g, '').trim()
-}
-
 function buildMemoryContext(memories: Array<{ content: string }>): string {
   if (memories.length === 0) return 'No existing memories.'
 
   return memories
     .map((memory, index) => {
-      const content = normalizeWhitespace(memory.content)
+      const content = compactWhitespace(memory.content)
       if (!content) return ''
       return `${index + 1}. ${content}`
     })
@@ -117,75 +88,10 @@ function buildPrompt(params: {
   ].join('\n')
 }
 
-function mapWorkstreamDigestRow(row: z.infer<typeof WorkstreamDigestMessageRowSchema>): DigestMessage {
-  return {
-    source: 'workstream',
-    sourceId: row.workstreamId,
-    role: row.role,
-    parts: row.parts ?? [],
-    metadata: row.metadata ?? undefined,
-    cursor: { createdAt: new Date(toTimestamp(row.createdAt)), id: row.id },
-  }
-}
-
-function compareDigestMessageOrder(left: DigestMessage, right: DigestMessage): number {
-  const timeDiff = left.cursor.createdAt.getTime() - right.cursor.createdAt.getTime()
-  if (timeDiff !== 0) return timeDiff
-  return left.cursor.id.localeCompare(right.cursor.id)
-}
-
 function getLastCursor(messages: DigestMessage[]): DigestCursor | null {
   return messages.length > 0 ? messages[messages.length - 1].cursor : null
 }
 
-async function listWorkstreamIdsForOrg(orgRef: RecordIdRef): Promise<RecordIdRef[]> {
-  const ids = await databaseService.query<unknown>(
-    new BoundQuery(
-      `SELECT VALUE type::string(id) FROM ${TABLES.WORKSTREAM}
-       WHERE organizationId = $organizationId`,
-      { organizationId: orgRef },
-    ),
-  )
-
-  return ids.map((value) => ensureRecordId(EntityIdRowSchema.parse(value), TABLES.WORKSTREAM))
-}
-
-async function listEligibleWorkstreamMessages(params: {
-  workstreamIds: RecordIdRef[]
-  cursor: DigestCursor | null
-  onboardingCutoff: Date | null
-}): Promise<DigestMessage[]> {
-  if (params.workstreamIds.length === 0) return []
-
-  let query: BoundQuery | null = null
-  if (params.cursor) {
-    const cursorRowId = ensureRecordId(params.cursor.id, TABLES.WORKSTREAM_MESSAGE)
-    query = new BoundQuery(
-      `SELECT type::string(id) AS id, type::string(workstreamId) AS workstreamId, role, parts, metadata, createdAt FROM ${TABLES.WORKSTREAM_MESSAGE}
-       WHERE workstreamId IN $workstreamIds
-         AND (
-           createdAt > $cursorCreatedAt
-           OR (createdAt = $cursorCreatedAt AND id > $cursorRowId)
-         )
-       ORDER BY createdAt ASC, id ASC`,
-      { workstreamIds: params.workstreamIds, cursorCreatedAt: params.cursor.createdAt, cursorRowId },
-    )
-  } else if (params.onboardingCutoff) {
-    query = new BoundQuery(
-      `SELECT type::string(id) AS id, type::string(workstreamId) AS workstreamId, role, parts, metadata, createdAt FROM ${TABLES.WORKSTREAM_MESSAGE}
-       WHERE workstreamId IN $workstreamIds
-         AND createdAt > $onboardingCutoff
-       ORDER BY createdAt ASC, id ASC`,
-      { workstreamIds: params.workstreamIds, onboardingCutoff: params.onboardingCutoff },
-    )
-  }
-
-  if (!query) return []
-
-  const rows = await databaseService.query<unknown>(query)
-  return rows.map((row) => mapWorkstreamDigestRow(WorkstreamDigestMessageRowSchema.parse(row)))
-}
-
 async function hasNewEligibleWorkstreamMessages(params: {
   workstreamIds: RecordIdRef[]
   cursor: DigestCursor | null

package/src/workers/skill-extraction.runner.ts

@@ -1,11 +1,5 @@
-import { toTimestamp } from '@lota-sdk/shared'
-import { BoundQuery } from 'surrealdb'
-import { z } from 'zod'
-
 import { serverLogger } from '../config/logger'
 import { ensureRecordId, recordIdToString } from '../db/record-id'
-import type { RecordIdRef } from '../db/record-id'
-import { databaseService } from '../db/service'
 import { TABLES } from '../db/tables'
 import { getDefaultEmbeddings } from '../embeddings/provider'
 import type { SkillExtractionJob } from '../queues/skill-extraction.queue'
@@ -16,38 +10,15 @@ import { createSkillExtractorAgent, SkillExtractionOutputSchema } from '../syste
 import type { SkillCandidate } from '../system-agents/skill-extractor.agent'
 import { createSkillManagerAgent, SkillManagerOutputSchema } from '../system-agents/skill-manager.agent'
 import { buildDigestTranscript, resolveWorkspaceBootstrapCutoff } from './regular-chat-memory-digest.helpers'
+import {
+  compareDigestMessageOrder,
+  listEligibleWorkstreamMessages,
+  listWorkstreamIdsForOrg,
+} from './utils/workstream-message-query'
 
 const SKILL_EXTRACTION_TIMEOUT_MS = 10 * 60 * 1000
 const MIN_MESSAGE_THRESHOLD = 10
 
-const RecordTimestampSchema = z.union([z.date(), z.string(), z.number()])
-const MessageRoleSchema = z.enum(['system', 'user', 'assistant'])
-const MessagePartSchema = z.record(z.string(), z.unknown())
-const MessageMetadataSchema = z.record(z.string(), z.unknown()).nullish()
-
-const WorkstreamMessageRowSchema = z.object({
-  id: z.string(),
-  workstreamId: z.string(),
-  role: MessageRoleSchema,
-  parts: z.array(MessagePartSchema).optional(),
-  metadata: MessageMetadataSchema,
-  createdAt: RecordTimestampSchema,
-})
-
-interface DigestCursor {
-  createdAt: Date
-  id: string
-}
-
-interface DigestMessage {
-  source: 'workstream'
-  sourceId: string
-  role: 'system' | 'user' | 'assistant'
-  parts: Array<Record<string, unknown>>
-  metadata?: Record<string, unknown>
-  cursor: DigestCursor
-}
-
 interface SkillExtractionRunResult {
   skipped: boolean
   processedMessages: number
@@ -58,71 +29,6 @@ const embeddings = getDefaultEmbeddings()
 
 const helperModelRuntime = createHelperModelRuntime()
 
-function mapWorkstreamRow(row: z.infer<typeof WorkstreamMessageRowSchema>): DigestMessage {
-  return {
-    source: 'workstream',
-    sourceId: row.workstreamId,
-    role: row.role,
-    parts: row.parts ?? [],
-    metadata: row.metadata ?? undefined,
-    cursor: { createdAt: new Date(toTimestamp(row.createdAt)), id: row.id },
-  }
-}
-
-function compareMessageOrder(left: DigestMessage, right: DigestMessage): number {
-  const timeDiff = left.cursor.createdAt.getTime() - right.cursor.createdAt.getTime()
-  if (timeDiff !== 0) return timeDiff
-  return left.cursor.id.localeCompare(right.cursor.id)
-}
-
-async function listWorkstreamIdsForOrg(orgRef: RecordIdRef): Promise<RecordIdRef[]> {
-  const EntityIdRowSchema = z.string().trim().min(1)
-  const ids = await databaseService.query<unknown>(
-    new BoundQuery(
-      `SELECT VALUE type::string(id) FROM ${TABLES.WORKSTREAM}
-       WHERE organizationId = $organizationId`,
-      { organizationId: orgRef },
-    ),
-  )
-  return ids.map((value) => ensureRecordId(EntityIdRowSchema.parse(value), TABLES.WORKSTREAM))
-}
-
-async function listEligibleMessages(params: {
-  workstreamIds: RecordIdRef[]
-  cursor: DigestCursor | null
-  onboardingCutoff: Date | null
-}): Promise<DigestMessage[]> {
-  if (params.workstreamIds.length === 0) return []
-
-  let query: BoundQuery | null = null
-  if (params.cursor) {
-    const cursorRowId = ensureRecordId(params.cursor.id, TABLES.WORKSTREAM_MESSAGE)
-    query = new BoundQuery(
-      `SELECT type::string(id) AS id, type::string(workstreamId) AS workstreamId, role, parts, metadata, createdAt FROM ${TABLES.WORKSTREAM_MESSAGE}
-       WHERE workstreamId IN $workstreamIds
-         AND (
-           createdAt > $cursorCreatedAt
-           OR (createdAt = $cursorCreatedAt AND id > $cursorRowId)
-         )
-       ORDER BY createdAt ASC, id ASC`,
-      { workstreamIds: params.workstreamIds, cursorCreatedAt: params.cursor.createdAt, cursorRowId },
-    )
-  } else if (params.onboardingCutoff) {
-    query = new BoundQuery(
-      `SELECT type::string(id) AS id, type::string(workstreamId) AS workstreamId, role, parts, metadata, createdAt FROM ${TABLES.WORKSTREAM_MESSAGE}
-       WHERE workstreamId IN $workstreamIds
-         AND createdAt > $onboardingCutoff
-       ORDER BY createdAt ASC, id ASC`,
-      { workstreamIds: params.workstreamIds, onboardingCutoff: params.onboardingCutoff },
-    )
-  }
-
-  if (!query) return []
-
-  const rows = await databaseService.query<unknown>(query)
-  return rows.map((row) => mapWorkstreamRow(WorkstreamMessageRowSchema.parse(row)))
-}
-
 function buildExtractionPrompt(params: { workspaceName: string; transcript: string; existingSkills: string }): string {
   return [
     `Workspace name: ${params.workspaceName}`,
@@ -196,14 +102,14 @@ export async function runSkillExtraction(data: SkillExtractionJob): Promise<Skil
     })
 
     const workstreamIds = await listWorkstreamIdsForOrg(orgRef)
-    const messages = await listEligibleMessages({ workstreamIds, cursor: existingCursor, onboardingCutoff })
+    const messages = await listEligibleWorkstreamMessages({ workstreamIds, cursor: existingCursor, onboardingCutoff })
 
     if (messages.length < MIN_MESSAGE_THRESHOLD) {
       serverLogger.info`Skipping skill extraction for ${orgId}: only ${messages.length} messages (threshold: ${MIN_MESSAGE_THRESHOLD})`
       return { skipped: true, processedMessages: messages.length, extractedSkills: 0 }
     }
 
-    const sortedMessages = [...messages].sort(compareMessageOrder)
+    const sortedMessages = [...messages].sort(compareDigestMessageOrder)
     const { transcript } = buildDigestTranscript({ messages: sortedMessages })
 
     const existingSkills = await learnedSkillService.listForOrg(orgId)

package/src/workers/utils/file-section-chunker.ts

@@ -1,7 +1,9 @@
 export const DEFAULT_FILE_SECTION_CHUNK_MAX_CHARS = 250_000
-const MIN_FILE_SECTION_CHUNK_MAX_CHARS = 4_000
+export const MIN_FILE_SECTION_CHUNK_MAX_CHARS = 4_000
 export const DEFAULT_FILE_SECTION_CHUNK_MIN_CHARS = 10_000
 const SECTION_SEPARATOR_LENGTH = 2
+const CHARS_PER_TOKEN_ESTIMATE = 3
+const MIN_CHUNK_CHARS_FLOOR = 512
 
 export interface FileSection {
   kind: 'preamble' | 'file'
@@ -29,7 +31,7 @@ export interface FileSectionChunkOptions {
 
 function estimateTokenCountFromChars(text: string): number {
   if (!text) return 0
-  return Math.ceil(text.length / 3)
+  return Math.ceil(text.length / CHARS_PER_TOKEN_ESTIMATE)
 }
 
 function normalizeMaxChars(value?: number): number {
@@ -43,7 +45,7 @@ function normalizeMinChunkChars(value: number | undefined, maxChars: number): nu
   if (typeof value !== 'number' || !Number.isFinite(value)) {
     return Math.min(DEFAULT_FILE_SECTION_CHUNK_MIN_CHARS, Math.floor(maxChars * 0.35))
   }
-  const normalized = Math.max(512, Math.floor(value))
+  const normalized = Math.max(MIN_CHUNK_CHARS_FLOOR, Math.floor(value))
   return Math.min(normalized, Math.floor(maxChars * 0.6))
 }
 

package/src/workers/utils/workstream-message-query.ts

@@ -0,0 +1,106 @@
+import { toTimestamp } from '@lota-sdk/shared'
+import { BoundQuery } from 'surrealdb'
+import { z } from 'zod'
+
+import { ensureRecordId } from '../../db/record-id'
+import type { RecordIdRef } from '../../db/record-id'
+import { databaseService } from '../../db/service'
+import { TABLES } from '../../db/tables'
+import { normalizeTextBody } from '../../document/parsing'
+
+const RecordTimestampSchema = z.union([z.date(), z.string(), z.number()])
+const MessageRoleSchema = z.enum(['system', 'user', 'assistant'])
+const MessagePartSchema = z.record(z.string(), z.unknown())
+const MessageMetadataSchema = z.record(z.string(), z.unknown()).nullish()
+
+const WorkstreamMessageRowSchema = z.object({
+  id: z.string(),
+  workstreamId: z.string(),
+  role: MessageRoleSchema,
+  parts: z.array(MessagePartSchema).optional(),
+  metadata: MessageMetadataSchema,
+  createdAt: RecordTimestampSchema,
+})
+
+export interface DigestCursor {
+  createdAt: Date
+  id: string
+}
+
+export interface DigestMessage {
+  source: 'workstream'
+  sourceId: string
+  role: 'system' | 'user' | 'assistant'
+  parts: Array<Record<string, unknown>>
+  metadata?: Record<string, unknown>
+  cursor: DigestCursor
+}
+
+function mapWorkstreamRow(row: z.infer<typeof WorkstreamMessageRowSchema>): DigestMessage {
+  return {
+    source: 'workstream',
+    sourceId: row.workstreamId,
+    role: row.role,
+    parts: row.parts ?? [],
+    metadata: row.metadata ?? undefined,
+    cursor: { createdAt: new Date(toTimestamp(row.createdAt) ?? Date.now()), id: row.id },
+  }
+}
+
+export function compareDigestMessageOrder(left: DigestMessage, right: DigestMessage): number {
+  const timeDiff = left.cursor.createdAt.getTime() - right.cursor.createdAt.getTime()
+  if (timeDiff !== 0) return timeDiff
+  return left.cursor.id.localeCompare(right.cursor.id)
+}
+
+export async function listWorkstreamIdsForOrg(orgRef: RecordIdRef): Promise<RecordIdRef[]> {
+  const EntityIdRowSchema = z.string().trim().min(1)
+  const ids = await databaseService.query<unknown>(
+    new BoundQuery(
+      `SELECT VALUE type::string(id) FROM ${TABLES.WORKSTREAM}
+       WHERE organizationId = $organizationId`,
+      { organizationId: orgRef },
+    ),
+  )
+  return ids.map((value) => ensureRecordId(EntityIdRowSchema.parse(value), TABLES.WORKSTREAM))
+}
+
+export async function listEligibleWorkstreamMessages(params: {
+  workstreamIds: RecordIdRef[]
+  cursor: DigestCursor | null
+  onboardingCutoff: Date | null
+}): Promise<DigestMessage[]> {
+  if (params.workstreamIds.length === 0) return []
+
+  let query: BoundQuery | null = null
+  if (params.cursor) {
+    const cursorRowId = ensureRecordId(params.cursor.id, TABLES.WORKSTREAM_MESSAGE)
+    query = new BoundQuery(
+      `SELECT type::string(id) AS id, type::string(workstreamId) AS workstreamId, role, parts, metadata, createdAt FROM ${TABLES.WORKSTREAM_MESSAGE}
+       WHERE workstreamId IN $workstreamIds
+         AND (
+           createdAt > $cursorCreatedAt
+           OR (createdAt = $cursorCreatedAt AND id > $cursorRowId)
+         )
+       ORDER BY createdAt ASC, id ASC`,
+      { workstreamIds: params.workstreamIds, cursorCreatedAt: params.cursor.createdAt, cursorRowId },
+    )
+  } else if (params.onboardingCutoff) {
+    query = new BoundQuery(
+      `SELECT type::string(id) AS id, type::string(workstreamId) AS workstreamId, role, parts, metadata, createdAt FROM ${TABLES.WORKSTREAM_MESSAGE}
+       WHERE workstreamId IN $workstreamIds
+         AND createdAt > $onboardingCutoff
+       ORDER BY createdAt ASC, id ASC`,
+      { workstreamIds: params.workstreamIds, onboardingCutoff: params.onboardingCutoff },
+    )
+  }
+
+  if (!query) return []
+
+  const rows = await databaseService.query<unknown>(query)
+  return rows.map((row) => mapWorkstreamRow(WorkstreamMessageRowSchema.parse(row)))
+}
+
+export function normalizeBlock(value: string): string {
+  return normalizeTextBody(value)
+}

package/src/workers/worker-utils.ts

@@ -5,6 +5,10 @@ import type { Job, Worker } from 'bullmq'
 
 import { chatLogger } from '../config/logger'
 
+export const DEFAULT_JOB_RETENTION = { removeOnComplete: 200, removeOnFail: 200 }
+export const LOW_JOB_RETENTION = { removeOnComplete: 50, removeOnFail: 50 }
+export const LONG_JOB_LOCK_DURATION_MS = 600_000
+
 const DEFAULT_SHUTDOWN_TIMEOUT_MS = 10_000
 const MAX_TRACE_STRING_CHARS = 2_000
 const MAX_TRACE_ARRAY_ITEMS = 12

package/src/runtime/retrieval-pipeline.ts

@@ -1,3 +0,0 @@
-export function resolveCandidateLimit(params: { limit: number; multiplier: number; minimum: number }): number {
-  return Math.max(params.limit * params.multiplier, params.minimum)
-}

package/src/utils/error.ts

@@ -1,10 +0,0 @@
-export function getErrorMessage(error: unknown): string {
-  if (error instanceof Error) return error.message
-  if (typeof error === 'string') return error
-
-  try {
-    return JSON.stringify(error)
-  } catch {
-    return String(error)
-  }
-}