@lota-sdk/core 0.4.44 → 0.4.46

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lota-sdk/core",
-  "version": "0.4.44",
+  "version": "0.4.46",
   "files": [
     "src",
     "infrastructure/schema"
@@ -32,7 +32,7 @@
     "@ai-sdk/provider": "^3.0.9",
     "@chat-adapter/slack": "^4.26.0",
     "@chat-adapter/state-ioredis": "^4.26.0",
-    "@lota-sdk/shared": "0.4.44",
+    "@lota-sdk/shared": "0.4.46",
     "@mendable/firecrawl-js": "^4.20.0",
     "@surrealdb/node": "^3.0.3",
     "ai": "^6.0.170",

package/src/runtime/context-compaction/context-compaction.ts

@@ -11,6 +11,7 @@ import {
   readString,
   stringifyUnknown,
 } from '../../utils/string'
+import { trimOversizedTextForRequestBudget } from '../model-input-budget'
 import {
   COMPACTION_CHUNK_MAX_CHARS,
   CONTEXT_COMPACTION_INCLUDED_TOOL_NAMES,
@@ -71,6 +72,8 @@ class CompactionParseError extends Schema.TaggedErrorClass<CompactionParseError>
 }) {}
 
 const COMPACTION_RUNNER_RETRY_OPTIONS = { times: 2, schedule: Schedule.exponential(Duration.millis(500), 2) } as const
+const COMPACTION_CONTEXT_MESSAGE_TEXT_MAX_CHARS = 24_000
+const COMPACTION_MIN_SOURCE_CHARS = 2_000
 
 export interface ContextCompactionPromptParams {
   previousSummary: string
@@ -294,11 +297,14 @@ export function createContextCompactionRuntime(
     return chunks.join('\n').trim()
   }
 
-  const toContextMessageFromChatMessage = (message: ChatMessage): ContextMessage => ({
-    role: message.role,
-    text: toContextTextFromChatMessage(message),
-    sourceMessageId: message.id,
-  })
+  const toContextMessageFromChatMessage = (message: ChatMessage): ContextMessage => {
+    const text = toContextTextFromChatMessage(message)
+    return {
+      role: message.role,
+      text: trimOversizedTextForRequestBudget(text, COMPACTION_CONTEXT_MESSAGE_TEXT_MAX_CHARS),
+      sourceMessageId: message.id,
+    }
+  }
 
   const createSummaryMessage = (summaryText: string): ChatMessage | null => {
     const summary = normalizeSummary(summaryText)
@@ -458,6 +464,10 @@ export function createContextCompactionRuntime(
                 return { ...state, estimatedTokens: assessment.estimatedTokens, done: true }
               }
 
+              if (sourceText.length < COMPACTION_MIN_SOURCE_CHARS) {
+                return { ...state, estimatedTokens: assessment.estimatedTokens, done: true }
+              }
+
               const compacted = yield* compactContextMessagesEffect({
                 previousSummary: state.summaryText,
                 newMessages: contextMessages,
@@ -465,9 +475,7 @@ export function createContextCompactionRuntime(
               const rolledSummary = yield* rollupSummaryIfOversizedEffect(normalizeSummary(compacted.summary))
 
               if (rolledSummary.length >= sourceText.length) {
-                return yield* new CompactionError({
-                  message: 'Compaction summary is not shorter than compacted source',
-                })
+                return { ...state, estimatedTokens: assessment.estimatedTokens, done: true }
               }
 
               return yield* stepHistory({

package/src/runtime/model-input-budget.ts

@@ -0,0 +1,189 @@
+import type { ChatMessage } from '@lota-sdk/shared'
+
+import { CONTEXT_WINDOW_TOKENS } from './context-compaction/context-compaction-constants'
+
+const MODEL_INPUT_MAX_TOTAL_CHARS = 240_000
+const MODEL_INPUT_LATEST_USER_TEXT_MAX_CHARS = 120_000
+const MODEL_INPUT_TEXT_PART_MAX_CHARS = 24_000
+const MODEL_INPUT_TOOL_PAYLOAD_MAX_CHARS = 12_000
+const MODEL_INPUT_DATA_PAYLOAD_MAX_CHARS = 8_000
+const MODEL_INPUT_NOTICE_ID = 'model-input-budget-notice'
+
+interface ShapeModelInputMessagesOptions {
+  latestUserMessageId?: string
+  maxTotalChars?: number
+}
+
+function serializedLength(value: unknown): number {
+  try {
+    const serialized = JSON.stringify(value)
+    return typeof serialized === 'string' ? serialized.length : 0
+  } catch {
+    return String(value).length
+  }
+}
+
+function stringifyPayload(value: unknown): string | null {
+  if (value === null || value === undefined) return null
+  if (typeof value === 'string') return value
+
+  try {
+    return JSON.stringify(value)
+  } catch {
+    return '[unserializable payload]'
+  }
+}
+
+function trimMiddle(value: string, maxChars: number, notice: string): string {
+  if (value.length <= maxChars) return value
+
+  const marker = `\n\n[${notice}]\n\n`
+  const available = Math.max(0, maxChars - marker.length)
+  if (available <= 0) return marker.slice(0, maxChars)
+
+  const headChars = Math.ceil(available * 0.7)
+  const tailChars = available - headChars
+  const head = value.slice(0, headChars).trimEnd()
+  const tail = tailChars > 0 ? value.slice(-tailChars).trimStart() : ''
+
+  return `${head}${marker}${tail}`
+}
+
+function trimTextPart(text: string, maxChars: number): string {
+  return trimMiddle(
+    text,
+    maxChars,
+    `Model input truncated this text from ${text.length} characters to fit the request budget. ` +
+      'Only the beginning and end are visible to the model.',
+  )
+}
+
+export function trimOversizedTextForRequestBudget(text: string, maxChars: number): string {
+  return trimTextPart(text, maxChars)
+}
+
+function trimPayload(value: unknown, maxChars: number, label: string): unknown {
+  const payload = stringifyPayload(value)
+  if (!payload || payload.length <= maxChars) return value
+
+  return trimMiddle(
+    payload,
+    maxChars,
+    `Model input truncated ${label} from ${payload.length} serialized characters to fit the request budget.`,
+  )
+}
+
+function shapeMessageParts(message: ChatMessage, maxTotalChars: number, latestUserMessageId?: string): ChatMessage {
+  const textLimit =
+    latestUserMessageId && message.id === latestUserMessageId
+      ? Math.min(MODEL_INPUT_LATEST_USER_TEXT_MAX_CHARS, Math.floor(maxTotalChars * 0.7))
+      : MODEL_INPUT_TEXT_PART_MAX_CHARS
+
+  const parts = message.parts.map((part) => {
+    if (part.type === 'text' && typeof part.text === 'string' && part.text.length > textLimit) {
+      return { ...part, text: trimTextPart(part.text, textLimit) }
+    }
+
+    if (part.type.startsWith('tool-')) {
+      const nextPart = { ...part } as Record<string, unknown>
+      const originalInput = nextPart.input
+      const originalOutput = nextPart.output
+      const originalData = nextPart.data
+      if (originalInput !== undefined)
+        nextPart.input = trimPayload(originalInput, MODEL_INPUT_TOOL_PAYLOAD_MAX_CHARS, 'tool input')
+      if (originalOutput !== undefined) {
+        nextPart.output = trimPayload(originalOutput, MODEL_INPUT_TOOL_PAYLOAD_MAX_CHARS, 'tool output')
+      }
+      if (originalData !== undefined)
+        nextPart.data = trimPayload(originalData, MODEL_INPUT_DATA_PAYLOAD_MAX_CHARS, 'tool data')
+      if (nextPart.input !== originalInput || nextPart.output !== originalOutput || nextPart.data !== originalData) {
+        return nextPart as ChatMessage['parts'][number]
+      }
+    }
+
+    if (part.type.startsWith('data-') && 'data' in part) {
+      const nextPart = { ...part } as Record<string, unknown>
+      const originalData = nextPart.data
+      nextPart.data = trimPayload(originalData, MODEL_INPUT_DATA_PAYLOAD_MAX_CHARS, 'data payload')
+      if (nextPart.data !== originalData) {
+        return nextPart as ChatMessage['parts'][number]
+      }
+    }
+
+    return part
+  })
+
+  return parts.some((part, index) => part !== message.parts[index]) ? { ...message, parts } : message
+}
+
+function createBudgetNotice(omittedCount: number): ChatMessage {
+  return {
+    id: MODEL_INPUT_NOTICE_ID,
+    role: 'system',
+    parts: [
+      {
+        type: 'text',
+        text:
+          `${omittedCount} older message${omittedCount === 1 ? '' : 's'} were omitted from this model call ` +
+          'because the live thread exceeded the model input budget. Use the compacted summary and latest user request as source of truth.',
+      },
+    ],
+  }
+}
+
+function fitMessagesToBudget(
+  messages: ChatMessage[],
+  maxTotalChars: number,
+  latestUserMessageId?: string,
+): ChatMessage[] {
+  const kept: ChatMessage[] = []
+  let currentChars = 2
+  let omittedCount = 0
+
+  for (const message of [...messages].reverse()) {
+    const messageChars = serializedLength(message) + 1
+    const isLatestUserMessage = latestUserMessageId !== undefined && message.id === latestUserMessageId
+
+    if (currentChars + messageChars <= maxTotalChars || isLatestUserMessage) {
+      kept.push(message)
+      currentChars += messageChars
+      continue
+    }
+
+    omittedCount += 1
+  }
+
+  const ordered = kept.reverse()
+  if (omittedCount === 0) return ordered
+
+  while (
+    ordered.length > 0 &&
+    serializedLength(ordered) + serializedLength(createBudgetNotice(omittedCount)) + 1 > maxTotalChars
+  ) {
+    const latestUserIndex = ordered.findIndex(
+      (message) => latestUserMessageId !== undefined && message.id === latestUserMessageId,
+    )
+    const removableIndex = ordered.findIndex((_, index) => index !== latestUserIndex)
+    if (removableIndex === -1) break
+    ordered.splice(removableIndex, 1)
+    omittedCount += 1
+  }
+
+  return [createBudgetNotice(omittedCount), ...ordered]
+}
+
+export function shapeModelInputMessagesForBudget(
+  messages: ChatMessage[],
+  options: ShapeModelInputMessagesOptions = {},
+): ChatMessage[] {
+  const maxTotalChars = options.maxTotalChars ?? Math.min(MODEL_INPUT_MAX_TOTAL_CHARS, CONTEXT_WINDOW_TOKENS * 2)
+  const shapedMessages = messages.map((message) =>
+    shapeMessageParts(message, maxTotalChars, options.latestUserMessageId),
+  )
+
+  if (serializedLength(shapedMessages) <= maxTotalChars) {
+    return shapedMessages
+  }
+
+  return fitMessagesToBudget(shapedMessages, maxTotalChars, options.latestUserMessageId)
+}

package/src/services/thread/thread-turn-message-context.ts

@@ -7,6 +7,7 @@ import {
 } from '../../runtime/chat-attachments'
 import type { ReadableUploadMetadataLike } from '../../runtime/chat-types'
 import type { ContextCompactionRuntime } from '../../runtime/context-compaction/context-compaction'
+import { shapeModelInputMessagesForBudget } from '../../runtime/model-input-budget'
 
 export function upsertChatHistoryMessage(messages: ChatMessage[], nextMessage: ChatMessage): ChatMessage[] {
   const existingIndex = messages.findIndex((message) => message.id === nextMessage.id)
@@ -52,11 +53,14 @@ export function createThreadTurnMessageContext(params: {
     })
 
   const buildRunInputMessages = (extraMessages: ChatMessage[] = []): ChatMessage[] =>
-    buildModelInputMessagesWithUploadMetadata({
-      messages: [...currentMessages, ...extraMessages],
-      latestUserMessageId: params.latestUserMessageId,
-      uploadMetadataText: buildReadableUploadMetadataText(listReadableUploads(extraMessages)),
-    })
+    shapeModelInputMessagesForBudget(
+      buildModelInputMessagesWithUploadMetadata({
+        messages: [...currentMessages, ...extraMessages],
+        latestUserMessageId: params.latestUserMessageId,
+        uploadMetadataText: buildReadableUploadMetadataText(listReadableUploads(extraMessages)),
+      }),
+      { latestUserMessageId: params.latestUserMessageId },
+    )
 
   return {
     get currentMessages() {

package/src/services/thread/thread-turn-preparation.service.ts

@@ -43,6 +43,7 @@ import { createWiredContextCompactionRuntime } from '../../runtime/context-compa
 import { createExecutionPlanInstructionSectionCache, toExecutionPlanCacheError } from '../../runtime/execution-plan'
 import type { HelperModelRuntime } from '../../runtime/helper-model'
 import { HelperModelTag } from '../../runtime/helper-model'
+import { trimOversizedTextForRequestBudget } from '../../runtime/model-input-budget'
 import { runPostTurnSideEffects } from '../../runtime/post-turn-side-effects'
 import { extractMessageText, toOptionalTrimmedString } from '../../runtime/thread-chat-helpers'
 import {
@@ -118,6 +119,8 @@ type ThreadTurnPreparationTaggedError =
 
 type ThreadTurnRunEffect = Effect.Effect<PreparedThreadTurnResult, ThreadTurnPreparationSurfaceError, never>
 
+const THREAD_TURN_AUXILIARY_MESSAGE_TEXT_MAX_CHARS = 24_000
+
 type ExecuteThreadTurnRunEffect = Effect.Effect<
   PreparedThreadTurnResult | void,
   ThreadTurnPreparationSurfaceError,
@@ -406,12 +409,13 @@ function deriveTurnMessageInputs(params: {
       ? undefined
       : (userMessage ?? [...liveHistory].reverse().find((message) => message.role === 'user'))
 
-  const messageText =
+  const rawMessageText =
     turnParams.kind === 'planTurn'
       ? `${turnParams.planTurn.nodeSpec.label}\n${turnParams.planTurn.nodeSpec.objective}\n${turnParams.planTurn.nodeSpec.instructions}`
       : referenceUserMessage
         ? extractMessageText(referenceUserMessage).trim()
         : ''
+  const messageText = trimOversizedTextForRequestBudget(rawMessageText, THREAD_TURN_AUXILIARY_MESSAGE_TEXT_MAX_CHARS)
 
   return { userMessage, originalMessages, referenceUserMessage, messageText }
 }