@lota-sdk/core 0.1.5

package/src/workers/regular-chat-memory-digest.runner.ts

@@ -0,0 +1,363 @@
+import { toTimestamp } from '@lota-sdk/shared/runtime/chat-message-metadata'
+import { BoundQuery } from 'surrealdb'
+import { z } from 'zod'
+
+import { serverLogger } from '../config/logger'
+import { ensureRecordId, recordIdToString } from '../db/record-id'
+import type { RecordIdRef } from '../db/record-id'
+import { databaseService } from '../db/service'
+import { TABLES } from '../db/tables'
+import {
+  clearRegularChatMemoryDigestDeduplicationKey,
+  enqueueRegularChatMemoryDigest,
+} from '../queues/regular-chat-memory-digest.queue'
+import type { RegularChatMemoryDigestJob } from '../queues/regular-chat-memory-digest.queue'
+import { createHelperModelRuntime } from '../runtime/helper-model'
+import { getRuntimeAdapters, withConfiguredWorkspaceMemoryLock } from '../runtime/runtime-extensions'
+import { memoryService } from '../services/memory.service'
+import { createRegularChatMemoryDigestAgent } from '../system-agents/regular-chat-memory-digest.agent'
+import { buildDigestTranscript, resolveWorkspaceBootstrapCutoff } from './regular-chat-memory-digest.helpers'
+
+const StructuredProfilePatchSchema = z.record(z.string(), z.unknown()).default({})
+
+const REGULAR_CHAT_MEMORY_DIGEST_TIMEOUT_MS = 10 * 60 * 1000
+const WorkspaceMemoryRowSchema = z.object({ content: z.string() })
+const EntityIdRowSchema = z.string().trim().min(1)
+const RecordTimestampSchema = z.union([z.date(), z.string(), z.number()])
+const MessageRoleSchema = z.enum(['system', 'user', 'assistant'])
+const MessagePartSchema = z.record(z.string(), z.unknown())
+const MessageMetadataSchema = z.record(z.string(), z.unknown()).nullish()
+
+const WorkstreamDigestMessageRowSchema = z.object({
+  id: z.string(),
+  workstreamId: z.string(),
+  role: MessageRoleSchema,
+  parts: z.array(MessagePartSchema).optional(),
+  metadata: MessageMetadataSchema,
+  createdAt: RecordTimestampSchema,
+})
+
+const ExtractedFactSchema = z.object({
+  content: z.string().trim().min(1),
+  type: z.enum(['fact', 'preference', 'decision']),
+  confidence: z.number().min(0).max(1),
+  durability: z.enum(['core', 'standard', 'ephemeral']),
+})
+
+const RegularChatMemoryDigestOutputSchema = z.object({
+  summaryBlock: z.string().trim().min(1),
+  structuredProfilePatch: StructuredProfilePatchSchema,
+  facts: z.array(ExtractedFactSchema).default([]),
+})
+
+const helperModelRuntime = createHelperModelRuntime()
+
+interface DigestCursor {
+  createdAt: Date
+  id: string
+}
+
+interface DigestMessage {
+  source: 'workstream'
+  sourceId: string
+  role: 'system' | 'user' | 'assistant'
+  parts: Array<Record<string, unknown>>
+  metadata?: Record<string, unknown>
+  cursor: DigestCursor
+}
+
+interface RegularChatDigestRunResult {
+  skipped: boolean
+  processedWorkstreamMessages: number
+  followUpScheduled: boolean
+}
+
+function normalizeWhitespace(value: string): string {
+  return value.replace(/\s+/g, ' ').trim()
+}
+
+function normalizeBlock(value: string): string {
+  return value.replaceAll(String.fromCharCode(0), '').replace(/\r/g, '').trim()
+}
+
+function buildMemoryContext(memories: Array<{ content: string }>): string {
+  if (memories.length === 0) return 'No existing memories.'
+
+  return memories
+    .map((memory, index) => {
+      const content = normalizeWhitespace(memory.content)
+      if (!content) return ''
+      return `${index + 1}. ${content}`
+    })
+    .filter((line) => line.length > 0)
+    .join('\n')
+}
+
+function buildPrompt(params: {
+  workspaceName: string
+  currentSummaryBlock: string
+  currentStructuredProfile: string
+  existingMemories: string
+  transcript: string
+}): string {
+  return [
+    `Workspace name: ${params.workspaceName}`,
+    '',
+    'Current workspace profile summary block:',
+    params.currentSummaryBlock || 'No current workspace profile summary block.',
+    '',
+    'Current structured workspace profile JSON:',
+    params.currentStructuredProfile,
+    '',
+    'Existing durable memories:',
+    params.existingMemories,
+    '',
+    'Newly added regular-chat transcript:',
+    params.transcript || 'No new transcript lines.',
+  ].join('\n')
+}
+
+function mapWorkstreamDigestRow(row: z.infer<typeof WorkstreamDigestMessageRowSchema>): DigestMessage {
+  return {
+    source: 'workstream',
+    sourceId: row.workstreamId,
+    role: row.role,
+    parts: row.parts ?? [],
+    metadata: row.metadata ?? undefined,
+    cursor: { createdAt: new Date(toTimestamp(row.createdAt)), id: row.id },
+  }
+}
+
+function compareDigestMessageOrder(left: DigestMessage, right: DigestMessage): number {
+  const timeDiff = left.cursor.createdAt.getTime() - right.cursor.createdAt.getTime()
+  if (timeDiff !== 0) return timeDiff
+  return left.cursor.id.localeCompare(right.cursor.id)
+}
+
+function getLastCursor(messages: DigestMessage[]): DigestCursor | null {
+  return messages.length > 0 ? messages[messages.length - 1].cursor : null
+}
+
+async function listWorkstreamIdsForOrg(orgRef: RecordIdRef): Promise<RecordIdRef[]> {
+  const ids = await databaseService.query<unknown>(
+    new BoundQuery(
+      `SELECT VALUE type::string(id) FROM ${TABLES.WORKSTREAM}
+       WHERE organizationId = $organizationId`,
+      { organizationId: orgRef },
+    ),
+  )
+
+  return ids.map((value) => ensureRecordId(EntityIdRowSchema.parse(value), TABLES.WORKSTREAM))
+}
+
+async function listEligibleWorkstreamMessages(params: {
+  workstreamIds: RecordIdRef[]
+  cursor: DigestCursor | null
+  onboardingCutoff: Date | null
+}): Promise<DigestMessage[]> {
+  if (params.workstreamIds.length === 0) return []
+
+  let query: BoundQuery | null = null
+  if (params.cursor) {
+    const cursorRowId = ensureRecordId(params.cursor.id, TABLES.WORKSTREAM_MESSAGE)
+    query = new BoundQuery(
+      `SELECT type::string(id) AS id, type::string(workstreamId) AS workstreamId, role, parts, metadata, createdAt FROM ${TABLES.WORKSTREAM_MESSAGE}
+       WHERE workstreamId IN $workstreamIds
+         AND (
+           createdAt > $cursorCreatedAt
+           OR (createdAt = $cursorCreatedAt AND id > $cursorRowId)
+         )
+       ORDER BY createdAt ASC, id ASC`,
+      { workstreamIds: params.workstreamIds, cursorCreatedAt: params.cursor.createdAt, cursorRowId },
+    )
+  } else if (params.onboardingCutoff) {
+    query = new BoundQuery(
+      `SELECT type::string(id) AS id, type::string(workstreamId) AS workstreamId, role, parts, metadata, createdAt FROM ${TABLES.WORKSTREAM_MESSAGE}
+       WHERE workstreamId IN $workstreamIds
+         AND createdAt > $onboardingCutoff
+       ORDER BY createdAt ASC, id ASC`,
+      { workstreamIds: params.workstreamIds, onboardingCutoff: params.onboardingCutoff },
+    )
+  }
+
+  if (!query) return []
+
+  const rows = await databaseService.query<unknown>(query)
+  return rows.map((row) => mapWorkstreamDigestRow(WorkstreamDigestMessageRowSchema.parse(row)))
+}
+
+async function hasNewEligibleWorkstreamMessages(params: {
+  workstreamIds: RecordIdRef[]
+  cursor: DigestCursor | null
+  onboardingCutoff: Date | null
+}): Promise<boolean> {
+  if (params.workstreamIds.length === 0) return false
+
+  let query: BoundQuery | null = null
+  if (params.cursor) {
+    const cursorRowId = ensureRecordId(params.cursor.id, TABLES.WORKSTREAM_MESSAGE)
+    query = new BoundQuery(
+      `SELECT id, createdAt FROM ${TABLES.WORKSTREAM_MESSAGE}
+       WHERE workstreamId IN $workstreamIds
+         AND (
+           createdAt > $cursorCreatedAt
+           OR (createdAt = $cursorCreatedAt AND id > $cursorRowId)
+         )
+       ORDER BY createdAt ASC, id ASC
+       LIMIT 1`,
+      { workstreamIds: params.workstreamIds, cursorCreatedAt: params.cursor.createdAt, cursorRowId },
+    )
+  } else if (params.onboardingCutoff) {
+    query = new BoundQuery(
+      `SELECT id, createdAt FROM ${TABLES.WORKSTREAM_MESSAGE}
+       WHERE workstreamId IN $workstreamIds
+         AND createdAt > $onboardingCutoff
+       ORDER BY createdAt ASC, id ASC
+       LIMIT 1`,
+      { workstreamIds: params.workstreamIds, onboardingCutoff: params.onboardingCutoff },
+    )
+  }
+
+  if (!query) return false
+  const rows = await databaseService.query<unknown>(query)
+  return rows.length > 0
+}
+
+async function loadExistingOrganizationMemories(orgId: string): Promise<Array<{ content: string }>> {
+  return await databaseService.queryMany(
+    new BoundQuery(
+      `SELECT content, createdAt, id FROM ${TABLES.MEMORY}
+       WHERE metadata.orgId = $orgId
+         AND archivedAt IS NONE
+         AND (validUntil IS NONE OR validUntil > time::now())
+       ORDER BY createdAt DESC, id DESC
+      LIMIT 250`,
+      { orgId },
+    ),
+    WorkspaceMemoryRowSchema,
+  )
+}
+
+export async function runRegularChatMemoryDigest(
+  data: RegularChatMemoryDigestJob,
+): Promise<RegularChatDigestRunResult> {
+  const orgRef = ensureRecordId(data.orgId, TABLES.ORGANIZATION)
+  const orgId = recordIdToString(orgRef, TABLES.ORGANIZATION)
+  const workspaceProvider = getRuntimeAdapters().services?.workspaceProvider
+  if (!workspaceProvider) {
+    serverLogger.info`Skipping regular chat memory digest for ${orgId}: workspaceProvider is not configured`
+    return { skipped: true, processedWorkstreamMessages: 0, followUpScheduled: false }
+  }
+
+  return await withConfiguredWorkspaceMemoryLock(orgId, async () => {
+    if (
+      !workspaceProvider.getBackgroundCursor ||
+      !workspaceProvider.setBackgroundCursor ||
+      !workspaceProvider.applyProfileProjection
+    ) {
+      serverLogger.info`Skipping regular chat memory digest for ${orgId}: workspaceProvider background/profile methods are incomplete`
+      return { skipped: true, processedWorkstreamMessages: 0, followUpScheduled: false }
+    }
+
+    const workspace = await workspaceProvider.getWorkspace(orgRef)
+    const lifecycleState = await workspaceProvider.getLifecycleState?.(workspace)
+    if (lifecycleState?.bootstrapActive ?? false) {
+      serverLogger.info`Skipping regular chat memory digest for ${orgId}: onboarding is not completed`
+      return { skipped: true, processedWorkstreamMessages: 0, followUpScheduled: false }
+    }
+    const projectionState = await workspaceProvider.readProfileProjectionState?.(workspace)
+
+    const existingWorkstreamCursor = await workspaceProvider.getBackgroundCursor('regular-chat-digest', orgRef)
+    const workstreamOnboardingCutoff = resolveWorkspaceBootstrapCutoff({
+      hasExistingCursor: existingWorkstreamCursor !== null,
+      bootstrapCompletedAt: lifecycleState?.bootstrapCompletedAt,
+    })
+
+    const workstreamIds = await listWorkstreamIdsForOrg(orgRef)
+    const workstreamMessages = await listEligibleWorkstreamMessages({
+      workstreamIds,
+      cursor: existingWorkstreamCursor,
+      onboardingCutoff: workstreamOnboardingCutoff,
+    })
+
+    if (workstreamMessages.length === 0) {
+      serverLogger.info`Skipping regular chat memory digest for ${orgId}: no eligible messages`
+      return { skipped: true, processedWorkstreamMessages: 0, followUpScheduled: false }
+    }
+
+    const combinedMessages = [...workstreamMessages].sort(compareDigestMessageOrder)
+    const { transcript, involvedAgentNames } = buildDigestTranscript({ messages: combinedMessages })
+    const existingMemories = await loadExistingOrganizationMemories(orgId)
+
+    const synthesis = await helperModelRuntime.generateHelperStructured({
+      tag: 'regular-chat-memory-digest',
+      createAgent: createRegularChatMemoryDigestAgent,
+      timeoutMs: REGULAR_CHAT_MEMORY_DIGEST_TIMEOUT_MS,
+      messages: [
+        {
+          role: 'user',
+          content: buildPrompt({
+            workspaceName: projectionState?.workspaceName || 'Workspace',
+            currentSummaryBlock: projectionState?.summaryBlock ?? '',
+            currentStructuredProfile: JSON.stringify(projectionState?.structuredProfile ?? {}, null, 2),
+            existingMemories: buildMemoryContext(existingMemories),
+            transcript,
+          }),
+        },
+      ],
+      schema: RegularChatMemoryDigestOutputSchema,
+    })
+
+    const summaryBlock = normalizeBlock(synthesis.summaryBlock)
+    if (!summaryBlock) {
+      throw new Error('Regular chat memory digest returned an empty summaryBlock')
+    }
+
+    const processedWorkstreamCursor = getLastCursor(combinedMessages)
+    const digestRunAt = new Date().toISOString()
+
+    if (synthesis.facts.length > 0) {
+      await memoryService.addExtractedFactsToScopes({
+        orgId,
+        facts: synthesis.facts,
+        source: 'regular_chat_digest',
+        sourceMetadata: {
+          digestRunAt,
+          ...(processedWorkstreamCursor
+            ? {
+                digestWorkstreamCursorCreatedAt: processedWorkstreamCursor.createdAt.toISOString(),
+                digestWorkstreamCursorId: processedWorkstreamCursor.id,
+              }
+            : {}),
+        },
+        agentNames: involvedAgentNames,
+        acquireLock: false,
+      })
+    }
+
+    await workspaceProvider.applyProfileProjection(orgRef, {
+      summaryBlock,
+      structuredPatch: synthesis.structuredProfilePatch,
+    })
+    if (processedWorkstreamCursor) {
+      await workspaceProvider.setBackgroundCursor('regular-chat-digest', orgRef, processedWorkstreamCursor)
+    }
+
+    const workstreamBoundaryCursor = processedWorkstreamCursor ?? existingWorkstreamCursor
+    const hasMoreWorkstreamMessages = await hasNewEligibleWorkstreamMessages({
+      workstreamIds,
+      cursor: workstreamBoundaryCursor,
+      onboardingCutoff: workstreamBoundaryCursor ? null : workstreamOnboardingCutoff,
+    })
+
+    const followUpScheduled = hasMoreWorkstreamMessages
+    if (followUpScheduled) {
+      await clearRegularChatMemoryDigestDeduplicationKey(orgId)
+      await enqueueRegularChatMemoryDigest({ orgId })
+    }
+
+    serverLogger.info`Regular chat memory digest completed for ${orgId}: workstreamMessages=${workstreamMessages.length}, facts=${synthesis.facts.length}, followUpScheduled=${followUpScheduled}`
+
+    return { skipped: false, processedWorkstreamMessages: workstreamMessages.length, followUpScheduled }
+  })
+}

package/src/workers/regular-chat-memory-digest.worker.ts

@@ -0,0 +1,22 @@
+import type { SandboxedJob } from 'bullmq'
+
+import { serverLogger } from '../config/logger'
+import type { RegularChatMemoryDigestJob } from '../queues/regular-chat-memory-digest.queue'
+import { initializeSandboxedWorkerRuntime } from './bootstrap'
+import { runRegularChatMemoryDigest } from './regular-chat-memory-digest.runner'
+import { toSandboxedWorkerError } from './utils/sandbox-error'
+import { createTracedWorkerProcessor } from './worker-utils'
+
+await initializeSandboxedWorkerRuntime()
+
+const handler = async (job: SandboxedJob<RegularChatMemoryDigestJob>) => {
+  try {
+    await runRegularChatMemoryDigest(job.data)
+  } catch (error) {
+    const serialized = toSandboxedWorkerError(error, 'Regular chat memory digest failed')
+    serverLogger.error`${serialized.message}`
+    throw serialized
+  }
+}
+
+export default createTracedWorkerProcessor('regular-chat-memory-digest', handler)