@lota-sdk/core 0.1.5

package/src/runtime/workstream-state.ts

@@ -0,0 +1,261 @@
+import { z } from 'zod'
+
+const DecisionConfidenceSchema = z.enum(['high', 'medium', 'low'])
+const StateSourceSchema = z.enum(['user', 'agent'])
+
+const WorkstreamPlanSchema = z.object({
+  id: z.string(),
+  text: z.string(),
+  source: StateSourceSchema,
+  approved: z.boolean(),
+  timestamp: z.number(),
+  sourceMessageIds: z.array(z.string()).default([]),
+})
+
+const WorkstreamConstraintSchema = z.object({
+  id: z.string(),
+  text: z.string(),
+  source: StateSourceSchema,
+  approved: z.boolean(),
+  timestamp: z.number(),
+  sourceMessageIds: z.array(z.string()).default([]),
+})
+
+const WorkstreamDecisionSchema = z.object({
+  id: z.string(),
+  decision: z.string(),
+  rationale: z.string(),
+  agent: z.string(),
+  sourceMessageIds: z.array(z.string()).default([]),
+  confidence: DecisionConfidenceSchema,
+  timestamp: z.number(),
+})
+
+const WorkstreamTaskSchema = z.object({
+  id: z.string(),
+  title: z.string(),
+  status: z.enum(['open', 'in-progress', 'done', 'blocked']),
+  owner: z.string(),
+  externalId: z.string().optional(),
+  source: StateSourceSchema,
+  timestamp: z.number(),
+  sourceMessageIds: z.array(z.string()).default([]),
+})
+
+const WorkstreamQuestionSchema = z.object({
+  id: z.string(),
+  text: z.string(),
+  source: StateSourceSchema,
+  timestamp: z.number(),
+  sourceMessageIds: z.array(z.string()).default([]),
+})
+
+const WorkstreamArtifactSchema = z.object({
+  id: z.string(),
+  name: z.string(),
+  type: z.string(),
+  pointer: z.string(),
+  timestamp: z.number(),
+})
+
+const WorkstreamAgentNoteSchema = z.object({
+  id: z.string(),
+  agent: z.string(),
+  summary: z.string(),
+  timestamp: z.number(),
+})
+
+export const WorkstreamStateSchema = z.object({
+  currentPlan: WorkstreamPlanSchema.nullable(),
+  activeConstraints: z.array(WorkstreamConstraintSchema),
+  keyDecisions: z.array(WorkstreamDecisionSchema),
+  tasks: z.array(WorkstreamTaskSchema),
+  openQuestions: z.array(WorkstreamQuestionSchema),
+  risks: z.array(z.string()),
+  artifacts: z.array(WorkstreamArtifactSchema),
+  agentContributions: z.array(WorkstreamAgentNoteSchema),
+  approvedBy: z.string().optional(),
+  approvedAt: z.number().int().optional(),
+  approvalMessageId: z.string().optional(),
+  approvalNote: z.string().optional(),
+  lastUpdated: z.number(),
+})
+
+export type WorkstreamState = z.infer<typeof WorkstreamStateSchema>
+
+export const WorkstreamStateDeltaSchema = z.object({
+  currentPlan: z.string().nullable().optional(),
+  newDecisions: z
+    .array(
+      z.object({
+        decision: z.string(),
+        rationale: z.string(),
+        agent: z.string(),
+        sourceMessageIds: z.array(z.string()),
+        confidence: DecisionConfidenceSchema,
+      }),
+    )
+    .optional(),
+  resolvedQuestions: z.array(z.string()).optional(),
+  newQuestions: z.array(z.string()).optional(),
+  newConstraints: z.array(z.string()).optional(),
+  newRisks: z.array(z.string()).optional(),
+  taskUpdates: z
+    .array(
+      z.object({
+        title: z.string(),
+        status: WorkstreamTaskSchema.shape.status,
+        owner: z.string(),
+        externalId: z.string().nullable(),
+        sourceMessageIds: z.array(z.string()),
+      }),
+    )
+    .optional(),
+  artifacts: z.array(z.object({ name: z.string(), type: z.string(), pointer: z.string() })).optional(),
+  agentNote: z.object({ agent: z.string(), summary: z.string() }).optional(),
+  conflicts: z
+    .array(z.object({ newFact: z.string(), conflictsWith: z.string(), recommendation: z.string() }))
+    .optional(),
+  approvedBy: z.string().optional(),
+  approvedAt: z.number().int().optional(),
+  approvalMessageId: z.string().optional(),
+  approvalNote: z.string().optional(),
+})
+
+export type WorkstreamStateDelta = z.infer<typeof WorkstreamStateDeltaSchema>
+
+const StructuredWorkstreamPlanDeltaSchema = z.object({
+  action: z.enum(['unchanged', 'clear', 'set']),
+  text: z.string().nullable(),
+})
+
+const StructuredWorkstreamDecisionDeltaSchema = z.object({
+  decision: z.string(),
+  rationale: z.string(),
+  agent: z.string(),
+  sourceMessageIds: z.array(z.string()),
+  confidence: DecisionConfidenceSchema,
+})
+
+const StructuredWorkstreamTaskDeltaSchema = z.object({
+  title: z.string(),
+  status: WorkstreamTaskSchema.shape.status,
+  owner: z.string(),
+  externalId: z.string().nullable(),
+  sourceMessageIds: z.array(z.string()),
+})
+
+const StructuredWorkstreamArtifactDeltaSchema = z.object({ name: z.string(), type: z.string(), pointer: z.string() })
+
+const StructuredWorkstreamAgentNoteDeltaSchema = z.object({ agent: z.string(), summary: z.string() })
+
+const StructuredWorkstreamConflictDeltaSchema = z.object({
+  newFact: z.string(),
+  conflictsWith: z.string(),
+  recommendation: z.string(),
+})
+
+export const StructuredWorkstreamStateDeltaSchema = z.object({
+  currentPlan: StructuredWorkstreamPlanDeltaSchema,
+  newDecisions: z.array(StructuredWorkstreamDecisionDeltaSchema),
+  resolvedQuestions: z.array(z.string()),
+  newQuestions: z.array(z.string()),
+  newConstraints: z.array(z.string()),
+  newRisks: z.array(z.string()),
+  taskUpdates: z.array(StructuredWorkstreamTaskDeltaSchema),
+  artifacts: z.array(StructuredWorkstreamArtifactDeltaSchema),
+  agentNote: StructuredWorkstreamAgentNoteDeltaSchema.nullable(),
+  conflicts: z.array(StructuredWorkstreamConflictDeltaSchema),
+  approvedBy: z.string().nullable(),
+  approvedAt: z.number().int().nullable(),
+  approvalMessageId: z.string().nullable(),
+  approvalNote: z.string().nullable(),
+})
+
+export type StructuredWorkstreamStateDelta = z.infer<typeof StructuredWorkstreamStateDeltaSchema>
+
+export function createEmptyStructuredWorkstreamStateDelta(): StructuredWorkstreamStateDelta {
+  return {
+    currentPlan: { action: 'unchanged', text: null },
+    newDecisions: [],
+    resolvedQuestions: [],
+    newQuestions: [],
+    newConstraints: [],
+    newRisks: [],
+    taskUpdates: [],
+    artifacts: [],
+    agentNote: null,
+    conflicts: [],
+    approvedBy: null,
+    approvedAt: null,
+    approvalMessageId: null,
+    approvalNote: null,
+  }
+}
+
+export function parseStructuredWorkstreamStateDelta(value: unknown): WorkstreamStateDelta {
+  const parsed = StructuredWorkstreamStateDeltaSchema.parse(value)
+
+  return {
+    ...(parsed.currentPlan.action === 'clear'
+      ? { currentPlan: null }
+      : parsed.currentPlan.action === 'set' && typeof parsed.currentPlan.text === 'string'
+        ? { currentPlan: parsed.currentPlan.text }
+        : {}),
+    ...(parsed.newDecisions.length > 0 ? { newDecisions: parsed.newDecisions } : {}),
+    ...(parsed.resolvedQuestions.length > 0 ? { resolvedQuestions: parsed.resolvedQuestions } : {}),
+    ...(parsed.newQuestions.length > 0 ? { newQuestions: parsed.newQuestions } : {}),
+    ...(parsed.newConstraints.length > 0 ? { newConstraints: parsed.newConstraints } : {}),
+    ...(parsed.newRisks.length > 0 ? { newRisks: parsed.newRisks } : {}),
+    ...(parsed.taskUpdates.length > 0
+      ? {
+          taskUpdates: parsed.taskUpdates.map((taskUpdate) => ({
+            title: taskUpdate.title,
+            status: taskUpdate.status,
+            owner: taskUpdate.owner,
+            externalId: taskUpdate.externalId,
+            sourceMessageIds: taskUpdate.sourceMessageIds,
+          })),
+        }
+      : {}),
+    ...(parsed.artifacts.length > 0 ? { artifacts: parsed.artifacts } : {}),
+    ...(parsed.agentNote ? { agentNote: parsed.agentNote } : {}),
+    ...(parsed.conflicts.length > 0 ? { conflicts: parsed.conflicts } : {}),
+    ...(parsed.approvedBy !== null ? { approvedBy: parsed.approvedBy } : {}),
+    ...(parsed.approvedAt !== null ? { approvedAt: parsed.approvedAt } : {}),
+    ...(parsed.approvalMessageId !== null ? { approvalMessageId: parsed.approvalMessageId } : {}),
+    ...(parsed.approvalNote !== null ? { approvalNote: parsed.approvalNote } : {}),
+  }
+}
+
+export const StructuredCompactionOutputSchema = z.object({
+  summary: z.string(),
+  stateDelta: StructuredWorkstreamStateDeltaSchema,
+})
+
+export interface CompactionOutput {
+  summary: string
+  stateDelta: WorkstreamStateDelta
+}
+
+export const WORKSTREAM_STATE_MAX_KEY_DECISIONS = 8
+export const WORKSTREAM_STATE_MAX_ACTIVE_CONSTRAINTS = 6
+export const WORKSTREAM_STATE_MAX_TASKS = 10
+export const WORKSTREAM_STATE_MAX_OPEN_QUESTIONS = 5
+export const WORKSTREAM_STATE_MAX_RISKS = 5
+export const WORKSTREAM_STATE_MAX_ARTIFACTS = 10
+export const WORKSTREAM_STATE_MAX_AGENT_CONTRIBUTIONS = 6
+
+export function createEmptyWorkstreamState(now = Date.now()): WorkstreamState {
+  return {
+    currentPlan: null,
+    activeConstraints: [],
+    keyDecisions: [],
+    tasks: [],
+    openQuestions: [],
+    risks: [],
+    artifacts: [],
+    agentContributions: [],
+    lastUpdated: now,
+  }
+}

package/src/services/attachment.service.ts

@@ -0,0 +1,159 @@
+import type { RecordIdRef } from '../db/record-id'
+import { recordIdToString } from '../db/record-id'
+import { TABLES } from '../db/tables'
+import { attachmentStorageService } from '../storage/attachment-storage.service'
+import type { UploadedWorkstreamAttachment as SdkUploadedWorkstreamAttachment } from '../storage/attachment-storage.service'
+import type { MessagePartLike, ReadableUploadMetadata as SdkReadableUploadMetadata } from '../storage/attachments.types'
+
+export type ReadableUploadMetadata = SdkReadableUploadMetadata
+
+function toOrgId(orgId: RecordIdRef): string {
+  return recordIdToString(orgId, TABLES.ORGANIZATION)
+}
+
+function toUserId(userId: RecordIdRef): string {
+  return recordIdToString(userId, TABLES.USER)
+}
+
+class AttachmentService {
+  hydrateSignedFileUrlsInMessageParts({
+    parts,
+    orgId,
+    userId,
+  }: {
+    parts: readonly MessagePartLike[]
+    orgId: RecordIdRef
+    userId: RecordIdRef
+  }): MessagePartLike[] {
+    return attachmentStorageService.hydrateSignedFileUrlsInMessageParts({
+      parts,
+      orgId: toOrgId(orgId),
+      userId: toUserId(userId),
+    })
+  }
+
+  listReadableUploadsFromMessages({
+    messages,
+    orgId,
+    userId,
+  }: {
+    messages: ReadonlyArray<{ parts: readonly MessagePartLike[] }>
+    orgId: RecordIdRef
+    userId: RecordIdRef
+  }): ReadableUploadMetadata[] {
+    return attachmentStorageService.listReadableUploadsFromMessages({
+      messages,
+      orgId: toOrgId(orgId),
+      userId: toUserId(userId),
+    })
+  }
+
+  async extractStoredAttachmentText({
+    storageKey,
+    name,
+    contentType,
+  }: {
+    storageKey: string
+    name: string
+    contentType: string
+  }): Promise<string> {
+    return await attachmentStorageService.extractStoredAttachmentText({ storageKey, name, contentType })
+  }
+
+  async extractStoredAttachmentPages({
+    storageKey,
+    name,
+    contentType,
+  }: {
+    storageKey: string
+    name: string
+    contentType: string
+  }): Promise<{ pageMode: 'logical' | 'pdf'; pages: string[] }> {
+    return await attachmentStorageService.extractStoredAttachmentPages({ storageKey, name, contentType })
+  }
+
+  async readFilePartsFromUpload({
+    upload,
+    orgId,
+    userId,
+    part,
+    pagesPerPart,
+  }: {
+    upload: ReadableUploadMetadata
+    orgId: RecordIdRef
+    userId: RecordIdRef
+    part?: number
+    pagesPerPart?: number
+  }) {
+    return await attachmentStorageService.readFilePartsFromUpload({
+      upload,
+      orgId: toOrgId(orgId),
+      userId: toUserId(userId),
+      part,
+      pagesPerPart,
+    })
+  }
+
+  getAttachmentUrl(storageKey: string): string {
+    return attachmentStorageService.getAttachmentUrl(storageKey)
+  }
+
+  async writeOrganizationDocument({
+    orgId,
+    namespace,
+    relativePath,
+    content,
+    contentType,
+  }: {
+    orgId: RecordIdRef
+    namespace: string
+    relativePath: string
+    content: string
+    contentType: string
+  }): Promise<{ storageKey: string; sizeBytes: number }> {
+    return await attachmentStorageService.writeOrganizationDocument({
+      orgId: toOrgId(orgId),
+      namespace,
+      relativePath,
+      content,
+      contentType,
+    })
+  }
+
+  async uploadOrganizationDocument({
+    file,
+    orgId,
+    namespace,
+    relativePath,
+  }: {
+    file: File
+    orgId: RecordIdRef
+    namespace: string
+    relativePath: string
+  }) {
+    return await attachmentStorageService.uploadOrganizationDocument({
+      file,
+      orgId: toOrgId(orgId),
+      namespace,
+      relativePath,
+    })
+  }
+
+  async uploadWorkstreamAttachment({
+    file,
+    orgId,
+    userId,
+  }: {
+    file: File
+    orgId: RecordIdRef
+    userId: RecordIdRef
+  }): Promise<SdkUploadedWorkstreamAttachment> {
+    return await attachmentStorageService.uploadWorkstreamAttachment({
+      file,
+      orgId: toOrgId(orgId),
+      userId: toUserId(userId),
+    })
+  }
+}
+
+export const attachmentService = new AttachmentService()

package/src/services/chat-attachments.service.ts

@@ -0,0 +1,17 @@
+import type { ChatMessage } from '@lota-sdk/shared/schemas/chat-message'
+
+import type { RecordIdRef } from '../db/record-id'
+import { attachmentService } from './attachment.service'
+import type { ReadableUploadMetadata } from './attachment.service'
+
+export function listReadableUploadsFromChatMessages(params: {
+  messages: ChatMessage[]
+  orgId: RecordIdRef
+  userId: RecordIdRef
+}): ReadableUploadMetadata[] {
+  return attachmentService.listReadableUploadsFromMessages({
+    messages: params.messages.map((message) => ({ parts: message.parts as Array<Record<string, unknown>> })),
+    orgId: params.orgId,
+    userId: params.userId,
+  })
+}

package/src/services/chat-run-registry.service.ts

@@ -0,0 +1,3 @@
+import { ChatRunRegistry } from '../runtime/chat-run-registry'
+
+export const chatRunRegistry = new ChatRunRegistry()

package/src/services/context-compaction-runtime.ts

@@ -0,0 +1,13 @@
+import { createWiredContextCompactionRuntime } from '../runtime/context-compaction-runtime'
+import { createHelperModelRuntime } from '../runtime/helper-model'
+
+const helperModelRuntime = createHelperModelRuntime()
+
+const wiredRuntime = createWiredContextCompactionRuntime({
+  helperModelRuntime,
+  now: () => Date.now(),
+  randomId: () => Bun.randomUUIDv7(),
+})
+
+export const contextCompactionRuntime = wiredRuntime
+export const compactMemoryBlockSummary = wiredRuntime.compactMemoryBlockSummary

package/src/services/context-compaction.service.ts

@@ -0,0 +1,115 @@
+import type { ChatMessage } from '@lota-sdk/shared/schemas/chat-message'
+
+import { chatLogger } from '../config/logger'
+import type { RecordIdRef } from '../db/record-id'
+import { recordIdToString } from '../db/record-id'
+import { databaseService } from '../db/service'
+import { TABLES } from '../db/tables'
+import { parseWorkstreamState, toStateFieldsUpdated } from '../runtime/context-compaction'
+import { CONTEXT_SIZE, WORKSTREAM_RAW_TAIL_MESSAGES } from '../runtime/context-compaction-constants'
+import type { WorkstreamState } from '../runtime/workstream-state'
+import { compactMemoryBlockSummary, contextCompactionRuntime } from './context-compaction-runtime'
+import { workstreamMessageService } from './workstream-message.service'
+import { WorkstreamSchema } from './workstream.types'
+
+interface PersistedCompactionMetrics {
+  domain: 'workstream'
+  entityId: string
+  inputChars: number
+  outputChars: number
+  savedChars: number
+  summaryLength: number
+  compactedMessageCount: number
+  remainingMessageCount: number
+  estimatedTokens: number
+  stateFieldsUpdated: string[]
+  conflictsDetected: number
+}
+
+class ContextCompactionService {
+  createSummaryMessage(summaryText: string) {
+    return contextCompactionRuntime.createSummaryMessage(summaryText)
+  }
+
+  formatWorkstreamStateForPrompt(state: WorkstreamState | null | undefined) {
+    return contextCompactionRuntime.formatWorkstreamStateForPrompt(state)
+  }
+
+  estimateThreshold(contextSize = CONTEXT_SIZE): number {
+    return contextCompactionRuntime.estimateThreshold(contextSize)
+  }
+
+  shouldCompactHistory(params: { summaryText: string; liveMessages: ChatMessage[]; contextSize?: number }) {
+    return contextCompactionRuntime.shouldCompactHistory(params)
+  }
+
+  async compactWorkstreamHistory(params: {
+    workstreamId: RecordIdRef
+    contextSize?: number
+  }): Promise<{ compacted: boolean; state: WorkstreamState | null }> {
+    const workstream = await databaseService.findOne(TABLES.WORKSTREAM, { id: params.workstreamId }, WorkstreamSchema)
+    if (!workstream) {
+      throw new Error(
+        `Workstream not found for compaction: ${recordIdToString(params.workstreamId, TABLES.WORKSTREAM)}`,
+      )
+    }
+
+    const currentState = parseWorkstreamState(workstream.state)
+    const liveMessages = await workstreamMessageService.listMessagesAfterCursor(
+      params.workstreamId,
+      typeof workstream.lastCompactedMessageId === 'string' ? workstream.lastCompactedMessageId : undefined,
+    )
+
+    const result = await contextCompactionRuntime.compactHistory({
+      summaryText: typeof workstream.chatSummary === 'string' ? workstream.chatSummary : '',
+      liveMessages,
+      tailMessageCount: WORKSTREAM_RAW_TAIL_MESSAGES,
+      contextSize: params.contextSize,
+      existingState: currentState,
+    })
+
+    if (!result.compacted || !result.lastCompactedMessageId) {
+      return { compacted: false, state: currentState }
+    }
+
+    if (result.compactedMessages.length > 0) {
+      await workstreamMessageService.upsertMessages({
+        workstreamId: params.workstreamId,
+        messages: result.compactedMessages,
+      })
+    }
+
+    await databaseService.update(
+      TABLES.WORKSTREAM,
+      params.workstreamId,
+      { chatSummary: result.summaryText, lastCompactedMessageId: result.lastCompactedMessageId, state: result.state },
+      WorkstreamSchema,
+    )
+
+    this.logCompactionMetrics({
+      domain: 'workstream',
+      entityId: recordIdToString(params.workstreamId, TABLES.WORKSTREAM),
+      inputChars: result.inputChars,
+      outputChars: result.outputChars,
+      savedChars: Math.max(0, result.inputChars - result.outputChars),
+      summaryLength: result.summaryText.length,
+      compactedMessageCount: result.compactedMessageCount,
+      remainingMessageCount: result.remainingMessageCount,
+      estimatedTokens: result.estimatedTokens,
+      stateFieldsUpdated: toStateFieldsUpdated(result.stateDelta),
+      conflictsDetected: result.stateDelta.conflicts?.length ?? 0,
+    })
+
+    return { compacted: true, state: result.state }
+  }
+
+  async compactMemoryBlock(params: { previousSummary: string; newEntriesText: string }): Promise<string> {
+    return await compactMemoryBlockSummary(params)
+  }
+
+  private logCompactionMetrics(metrics: PersistedCompactionMetrics): void {
+    chatLogger.info`Persisted chat compaction applied metrics=${JSON.stringify(metrics)}`
+  }
+}
+
+export const contextCompactionService = new ContextCompactionService()