@things-factory/ai-client-base 10.1.81 → 10.1.129

package/server/service/agentic-loop/agentic-loop.test.ts

@@ -644,8 +644,8 @@ describe('runAgenticLoop — max iterations 표면화', () => {
 
     expect(r.lastText).toBe('알아낸 것으로 답한다')
     expect(seen[seen.length - 1]).toBe('none')
-    /* 사유는 남는다 — 살려냈다고 한도에 닿은 사실이 사라지지 않는다. */
-    expect(r.abortReason).toEqual({ type: 'max_iterations', iter: 2 })
+    /* 사유는 남는다 — 살려냈다고 한도에 닿은 사실이 사라지지 않는다. 살려냈다는 것도 함께 남는다. */
+    expect(r.abortReason).toEqual({ type: 'max_iterations', iter: 2, salvaged: true })
   })
 
   test('★ 살려내기가 실패해도 삼키지 않는다 — 원래대로 한도 사유가 남는다', async () => {
@@ -917,3 +917,126 @@ describe('runAgenticLoop — 같은 조치는 한 번만 제안된다', () => {
     expect(r.proposals).toHaveLength(2)
   })
 })
+
+describe('a read tool that hands back a picture (design/ai/shared-dock.md §9-4)', () => {
+  const png = Buffer.from([0x89, 0x50, 0x4e, 0x47, 1, 2, 3])
+
+  test('★ the picture rides as an image part after the results - never in the JSON, the grounding or the usage', async () => {
+    const sent: any[][] = []
+    let turn = 0
+    const input = mkInput(
+      {
+        chat: async (conversation: any) => {
+          sent.push(conversation)
+          return turn++ === 0 ? okResult(undefined, [TC('readChatAttachment', { attachmentId: 'a1' })]) : okResult('영수증 금액은 12,400원입니다')
+        }
+      },
+      {
+        isReadTool: name => name === 'readChatAttachment',
+        executeReadTool: () => ({ attachmentId: 'a1', type: 'image', images: [{ data: png, mediaType: 'image/png' }] })
+      }
+    )
+    const r = await runAgenticLoop(input)
+    const last = sent[1][sent[1].length - 1]
+    expect(last.role).toBe('user')
+    expect(last.content.map((part: any) => part.type)).toEqual(['tool_result', 'image'])
+    expect(last.content[1].source).toEqual({ kind: 'base64', data: png, mediaType: 'image/png' })
+    expect(JSON.parse(last.content[0].content)).toEqual({ attachmentId: 'a1', type: 'image', imagesAttached: 1 })
+    expect(JSON.stringify(r.groundedTexts)).not.toContain('"images":')
+    expect(JSON.stringify(r.groundedTexts)).not.toContain('mediaType')
+    expect(JSON.stringify(r.toolUsages)).not.toContain('"images":')
+  })
+
+  test('a picture of a type every provider does not take, or with no bytes, is left out and said', async () => {
+    const { takeToolImages } = await import('./agentic-loop')
+    const { value, images } = takeToolImages({ images: [{ data: png, mediaType: 'image/tiff' }, { data: Buffer.alloc(0), mediaType: 'image/png' }, { data: png.toString('base64'), mediaType: 'IMAGE/JPEG' }] })
+    expect(images.map(one => (one.source as any).mediaType)).toEqual(['image/jpeg'])
+    expect(value).toEqual({ imagesAttached: 1, imagesLeftOut: 2 })
+    expect(takeToolImages({ text: 'a' })).toEqual({ value: { text: 'a' }, images: [] })
+  })
+
+  test('★ OpenAI - the tool messages first, then the pictures as one user message (a tool message holds text only)', async () => {
+    const { toSDKMessages } = await import('../providers/openai')
+    const messages = toSDKMessages({
+      role: 'user',
+      content: [
+        { type: 'tool_result', toolUseId: 'c1', content: '{"imagesAttached":1}' },
+        { type: 'image', source: { kind: 'base64', data: png, mediaType: 'image/png' } }
+      ]
+    } as any)
+    expect(messages.map((one: any) => one.role)).toEqual(['tool', 'user'])
+    expect(messages[1].content[0]).toEqual({ type: 'image_url', image_url: { url: `data:image/png;base64,${png.toString('base64')}` } })
+  })
+})
+
+describe('what only the card needs (design/ai/read-targets.md §4)', () => {
+  test('★ a proposal\'s forScreen reaches the card, never the model', async () => {
+    const sent: any[][] = []
+    let turn = 0
+    const rows = Array.from({ length: 300 }, (_, i) => ({ amount: i }))
+    const input = mkInput(
+      {
+        chat: async (conversation: any) => {
+          sent.push(conversation)
+          return turn++ === 0 ? okResult(undefined, [TC('queryOfficeCashTransaction')]) : okResult('300건입니다')
+        }
+      },
+      {
+        isReadTool: () => true,
+        executeReadTool: () => ({ proposed: true, kind: 'workspace', count: 300, forScreen: { workspace: { name: 'w', title: 't', blocks: [{ kind: 'table', rows }] } } })
+      }
+    )
+    const r = await runAgenticLoop(input)
+    const toModel = JSON.parse(sent[1][sent[1].length - 1].content[0].content)
+    expect(toModel).toEqual({ proposed: true, kind: 'workspace', count: 300 })
+    expect(r.proposals[0].workspace.blocks[0].rows).toHaveLength(300)
+    expect(JSON.stringify(r.toolUsages)).not.toContain('"forScreen"')
+  })
+
+  test('only a proposal carries it - a plain result keeps the field as it is', async () => {
+    const { takeForScreen } = await import('./agentic-loop')
+    expect(takeForScreen({ forScreen: { a: 1 } })).toEqual({ value: { forScreen: { a: 1 } }, forScreen: {} })
+  })
+})
+
+/*
+ * A tool that names its proposal keeps that name (2026-10-02, Figure and AI lanes): figure-service gives a UUID the
+ * modeller creates the new figure under, and the effect key written over it refused the run. Folding is by the effect
+ * key, kept apart as `effectKey`.
+ */
+describe('runAgenticLoop — a proposal keeps the name its tool gave it', () => {
+  const twice = (results: any[]) => {
+    let call = 0
+    return mkInput(
+      {
+        chat: async () => (++call <= results.length ? okResult(undefined, [TC('proposeFigureCandidate', {}, `t${call}`)]) : okResult('done'))
+      },
+      { isReadTool: () => true, executeReadTool: () => results[call - 1] }
+    )
+  }
+  const uuid = '3f1c2a9e-7b4d-4c1e-9a2f-5d6e7f8a9b0c'
+
+  test('the tool\'s key stays the key; the effect key goes beside it', async () => {
+    const r = await runAgenticLoop(twice([{ proposed: true, key: uuid, steps: [{ kind: 'rename', name: 'Bin' }] }]))
+    expect(r.proposals).toHaveLength(1)
+    expect(r.proposals[0].key).toBe(uuid)
+    expect(r.proposals[0].effectKey).toMatch(/^proposeFigureCandidate#/)
+  })
+
+  test('the same proposal twice, worded differently, is still one card', async () => {
+    const r = await runAgenticLoop(
+      twice([
+        { proposed: true, key: uuid, label: 'one', steps: [{ kind: 'rename', name: 'Bin' }] },
+        { proposed: true, key: uuid, label: 'two', steps: [{ kind: 'rename', name: 'Bin' }] }
+      ])
+    )
+    expect(r.proposals).toHaveLength(1)
+    expect(r.toolUsages.map((u: any) => u.outcome)).toEqual(['proposed', 'folded'])
+  })
+
+  test('a tool that gives no key gets the effect key, as before', async () => {
+    const r = await runAgenticLoop(twice([{ proposed: true, values: { capacity: 10 } }]))
+    expect(r.proposals[0].key).toBe(r.proposals[0].effectKey)
+    expect(r.proposals[0].key).toMatch(/^proposeFigureCandidate#/)
+  })
+})

package/server/service/agentic-loop/agentic-loop.ts

@@ -39,7 +39,7 @@
 import { createHash } from 'node:crypto'
 
 import debug from 'debug'
-import type { AIMessage, AIToolCall, AIToolChatResult } from '../types.js'
+import type { AIImageMediaType, AIImagePart, AIMessage, AIToolCall, AIToolChatResult } from '../types.js'
 import { retryTransient } from '../utils/index.js'
 
 const log = debug('things-factory:ai:agentic-loop')
@@ -200,7 +200,7 @@ export interface AgenticLoopResult<TOp = unknown> {
   abortReason?:
     | { type: 'provider_error'; message: string; iter: number }
     | { type: 'aborted'; iter: number }
-    | { type: 'max_iterations'; iter: number }
+    | { type: 'max_iterations'; iter: number; salvaged?: boolean }
     | { type: 'repeated_validation_failure'; toolName: string; consecutive: number; iter: number }
 }
 
@@ -403,6 +403,8 @@ export async function runAgenticLoop<TCtx = unknown, TOp = unknown>(
         })
         if (salvaged?.text) {
           lastText = salvaged.text
+          /* The wrap-up answered from what was found and says what is missing; no notice need follow it. */
+          abortReason = { type: 'max_iterations', iter, salvaged: true }
           stopReason = salvaged.stopReason ?? stopReason
           log('iter %d: salvaged a final answer without tools', iter)
         }
@@ -534,13 +536,18 @@ async function processToolCalls<TCtx, TOp>(
   iter: number
 ): Promise<any[]> {
   const toolResultParts: any[] = []
+  const toolResultImages: AIImagePart[] = []
 
   for (const tc of toolCalls) {
     if (dispatch.isReadTool(tc.name)) {
       /* await 필수 — registry 에 등록된 read/external builder 는 async 일 수 있다(DB·외부 호출).
        * await 없이 JSON.stringify 하면 Promise 가 '{}' 로 직렬화되어 LLM 이 **빈 결과**를 받는다
        * (문서 가져오기·트윈 구조 조회 등 — 무엇인지는 부르는 쪽이 안다). */
-      const value = await dispatch.executeReadTool(tc, context)
+      /* Pictures a read tool hands back ride as image parts after the results - never inside the JSON. */
+      const { value: given, images } = takeToolImages(await dispatch.executeReadTool(tc, context))
+      toolResultImages.push(...images)
+      /* What only the card needs (a table of a thousand rows) goes to the screen, never to the model. */
+      const { value, forScreen } = takeForScreen(given)
       /* 제안 규약 — 결과가 `proposed: true` 면 조치 제안이다(도메인은 모른 채 그대로 실어 올린다).
        * 이 채널이 없으면 대화면은 도구 추적(trace)을 뒤져야 하고, 실행 버튼을 만들 근거가 없다. */
       /* 판정(outcome) — 화면이 단계별 배지로 보여줄 값. 조용한 무동작을 남기지 않는다. */
@@ -549,13 +556,22 @@ async function processToolCalls<TCtx, TOp>(
         /* **같은 조치는 한 번만** — 모델이 같은 것을 두 번 제안하는 일이 실제로 있었다(하나는 인자가
          * 비어 있었다). 카드가 둘이면 사용자가 같은 명령을 두 번 실행할 수 있고, 그건 되돌릴 수 없다.
          * 판정은 **효과**(도구·명령·대상·인자)로 한다 — 설명 문구(label·reason)가 달라도 같은 조치다. */
-        const candidate = { tool: tc.name, ...(value as any) }
-        const key = proposalEffectKey(candidate)
-        if (proposals.some(p => p.key === key)) {
+        const candidate = { tool: tc.name, ...(value as any), ...forScreen }
+        const effectKey = proposalEffectKey(candidate)
+        if (proposals.some(p => p.effectKey === effectKey)) {
           outcome = 'folded' // 접힌 사실을 남긴다 — "왜 카드가 하나뿐인가" 가 추적 가능해야 한다
         } else {
-          /* 키를 함께 실어 보낸다 — 카드가 같은 규칙으로 다시 세지 않게(원본은 하나). */
-          proposals.push({ ...candidate, key })
+          /*
+           * 키를 함께 실어 보낸다 — 카드가 같은 규칙으로 다시 세지 않게(원본은 하나).
+           *
+           * **도구가 이름을 준 제안은 그 이름을 쓴다**(2026-10-02, Figure 레인 · AI 레인 합의). figure-service 의
+           * 도구는 제안에 UUID `key` 를 붙이고, 모델러는 그 UUID 로 새 도형을 만든다(ADR-0097 5.6절 5항). 효과 키로
+           * 덮어쓰면 「제안 식별자가 올바르지 않습니다」로 실행이 막혔다. 같은 조치인지는 효과 키(`effectKey`)로
+           * 따로 판정하므로 접기는 그대로다. 도구 `key` 는 **한 대화 안에서 유일해야 한다** — 겹치면 입력안의 「새
+           * 카드가 옛 카드를 대신함」 판정과 결말 줄이 섞인다. 이름을 주지 않는 도구(ai-assistant 의 제안 도구들)는
+           * 지금처럼 효과 키를 받는다.
+           */
+          proposals.push({ ...candidate, key: typeof (candidate as any).key === 'string' && (candidate as any).key ? (candidate as any).key : effectKey, effectKey })
           outcome = 'proposed'
         }
       } else if (value && typeof value === 'object' && ((value as any).rejected === true || (value as any).error)) {
@@ -656,5 +672,49 @@ async function processToolCalls<TCtx, TOp>(
     }
   }
 
-  return toolResultParts
+  /* Every provider takes a picture after the results of the same turn (design/ai/shared-dock.md §9-4). */
+  return [...toolResultParts, ...toolResultImages]
+}
+
+/**
+ * A proposal's reserved `forScreen` field, taken out of what the model reads and kept for the card.
+ *
+ * A read tool that has queried many rows puts them on the card (the AI 작업판 table) but tells the model only the
+ * count, the totals and a few rows - the model does not copy a table it never received, so the card's numbers are
+ * the query's own (operato-application/design/ai/read-targets.md §4). Only a proposal (`proposed: true`) carries it.
+ */
+export function takeForScreen(result: any): { value: any; forScreen: Record<string, unknown> } {
+  if (!result || typeof result !== 'object' || Array.isArray(result) || result.proposed !== true || !result.forScreen || typeof result.forScreen !== 'object') {
+    return { value: result, forScreen: {} }
+  }
+  const { forScreen, ...value } = result
+  return { value, forScreen }
+}
+
+const TOOL_IMAGE_TYPES = ['image/png', 'image/jpeg', 'image/gif', 'image/webp']
+
+/**
+ * The pictures a read tool returned in its reserved `images` field, taken out of the result.
+ *
+ * A tool result reaches the model as JSON text; a picture cannot. So a read tool that must show one - a picture
+ * sent in an earlier turn of the conversation - returns `images: [{ data, mediaType }]`, and the loop sends each as an
+ * image part of the same turn. The bytes never stay in the JSON, the grounding texts or the usage record. A picture
+ * of a type the providers do not all take is left out and said in the result.
+ */
+export function takeToolImages(result: any): { value: any; images: AIImagePart[] } {
+  if (!result || typeof result !== 'object' || Array.isArray(result) || !Array.isArray(result.images)) return { value: result, images: [] }
+  const { images: given, ...value } = result
+  const images: AIImagePart[] = []
+  let left = 0
+  for (const one of given) {
+    const data = typeof one?.data === 'string' ? Buffer.from(one.data, 'base64') : one?.data
+    const mediaType = String(one?.mediaType ?? '').toLowerCase()
+    if ((Buffer.isBuffer(data) || data instanceof Uint8Array) && data.length && TOOL_IMAGE_TYPES.includes(mediaType)) {
+      images.push({ type: 'image', source: { kind: 'base64', data, mediaType: mediaType as AIImageMediaType } })
+    } else left++
+  }
+  return {
+    value: { ...value, imagesAttached: images.length, ...(left ? { imagesLeftOut: left } : {}) },
+    images
+  }
 }

package/server/service/assistant/abort-notice.ts

@@ -27,7 +27,7 @@ const NOTICE_DEFAULTS: Record<string, string> = {
   'ai-assistant.notice.stopped': 'the request stopped partway, so only part of it was done.',
   'ai-assistant.notice.provider-busy': 'the ai service did not answer just now. try again in a moment.',
   'ai-assistant.notice.provider-error': 'something went wrong while handling this. tell an administrator if it keeps happening.',
-  'ai-assistant.notice.too-long': 'this ran long and stopped at step {iter}. try asking in smaller pieces.',
+  'ai-assistant.notice.too-long': 'this stopped after {iter} lookups without an answer.',
   'ai-assistant.notice.tool-kept-failing': '{tool} failed {count} times in a row, so it stopped. try saying it differently.'
 }
 
@@ -59,7 +59,8 @@ export function formatAbortNotice(
         ? line('ai-assistant.notice.provider-busy', {}, t)
         : line('ai-assistant.notice.provider-error', {}, t)
     case 'max_iterations':
-      return line('ai-assistant.notice.too-long', { iter: abortReason.iter }, t)
+      /* A salvaged answer already says what it could not find. */
+      return abortReason.salvaged ? '' : line('ai-assistant.notice.too-long', { iter: abortReason.iter }, t)
     case 'repeated_validation_failure':
       return line('ai-assistant.notice.tool-kept-failing', { tool: abortReason.toolName, count: abortReason.consecutive }, t)
   }

package/server/service/assistant/assistant-chat.test.ts

@@ -95,7 +95,7 @@ describe('assistantChat — ① 중단 이유가 답에 닿는다', () => {
     )
     expect(r.loop.abortReason?.type).toBe('max_iterations')
     /* No translator was handed in, so the line falls back to English. */
-    expect(r.reply).toMatch(/stopped at step 2/)
+    expect(r.reply).toMatch(/stopped after 2 lookups/)
   })
 
   it('★ 통지의 말은 읽는 사람의 것이다 — 번역기를 주면 그것이 쓴 문장이 붙는다', async () => {
@@ -115,7 +115,7 @@ describe('assistantChat — ① 중단 이유가 답에 닿는다', () => {
     )
     expect(asked[0].key).toBe('ai-assistant.notice.too-long')
     expect(asked[0].params.iter).toBe(2)
-    expect(asked[0].params.defaultValue).toMatch(/stopped at step/)
+    expect(asked[0].params.defaultValue).toMatch(/stopped after/)
     expect(r.reply).toContain('멈췄습니다(2단계)')
   })
 

package/server/service/assistant/registry-assistant.test.ts

@@ -11,7 +11,7 @@
  * apply them, so offering them would put a name in front of the model that leads nowhere. These
  * tests fix that: offered and dispatchable are the same set, and it is the answering kinds only.
  */
-import { registryAssistantChat } from './registry-assistant'
+import { registryAssistantChat, REGISTRY_MAX_ITERATIONS } from './registry-assistant'
 import { clearToolRegistry, registerToolCategory } from '../tool-registry/index.js'
 import { toolCallContextFixture } from '../testing/tool-call-context.js'
 import type { AIToolCall, AIToolChatResult } from '../types.js'
@@ -357,3 +357,22 @@ describe('the tool context carries the conversation', () => {
     expect(ran[0].context).not.toHaveProperty('mode')
   })
 })
+
+describe('a question that needs many lookups', () => {
+  it('★ at the limit the model answers from what it read — no notice tells the person to split the question', async () => {
+    const seen: any[] = []
+    const client = {
+      chatWithTools: async (_msgs: any, _tools: any[], options: any) => {
+        seen.push(options.toolChoice)
+        return options.toolChoice === 'none' ? okResult('found two; the third is missing') : okResult(undefined, [TC('listFigures')])
+      }
+    } as any
+
+    const result = await registryAssistantChat(input({ client, toolCategories: ['figure-authoring'] }))
+
+    expect(ran).toHaveLength(REGISTRY_MAX_ITERATIONS)
+    expect(REGISTRY_MAX_ITERATIONS).toBeGreaterThan(8)
+    expect(seen[seen.length - 1]).toBe('none')
+    expect(result.reply).toBe('found two; the third is missing')
+  })
+})

package/server/service/assistant/registry-assistant.ts

@@ -28,6 +28,9 @@ import type { GroundingSources } from '../utils/index.js'
 import type { ToolCallContext } from '../tool-registry/types.js'
 
 
+/** Lookups a registry conversation may take before it answers from what it has (operato-twin's assistant takes 16 too). */
+export const REGISTRY_MAX_ITERATIONS = 16
+
 export interface RegistryAssistantInput {
   /** Provider client. Must implement `chatWithTools`. */
   client: AIClient
@@ -128,7 +131,13 @@ export async function registryAssistantChat(input: RegistryAssistantInput): Prom
       model: input.model,
       temperature: input.temperature,
       maxTokens: input.maxTokens,
-      requireToolOnFirstTurn: input.requireToolOnFirstTurn
+      requireToolOnFirstTurn: input.requireToolOnFirstTurn,
+      /*
+       * A question across a product's records takes many lookups. At the limit the model answers from
+       * what it read and says what is missing — the person is never told to split the question.
+       */
+      maxIterations: REGISTRY_MAX_ITERATIONS,
+      salvageOnMaxIterations: true
     },
     chat: (msgs, toolDefs, opts) => client.chatWithTools!(msgs, toolDefs, opts),
     context: toolContext,

package/server/service/providers/openai.ts

@@ -186,7 +186,7 @@ function buildSDKMessages(messages: AIMessage[], externalSystem?: string): any[]
  * 한 AIMessage 가 다중 tool_result 를 가지면 OpenAI 측에서 여러 role:'tool' 메시지로
  * 분리해야 함 (OpenAI 는 메시지당 tool_call_id 하나만 매핑). 따라서 array 를 반환.
  */
-function toSDKMessages(msg: AIMessage): any[] {
+export function toSDKMessages(msg: AIMessage): any[] {
   if (typeof msg.content === 'string') {
     return [{ role: msg.role, content: msg.content }]
   }
@@ -198,11 +198,14 @@ function toSDKMessages(msg: AIMessage): any[] {
 
   // tool_result 가 있으면 각각을 role='tool' 메시지로 분리 (OpenAI 요구사항).
   if (toolResultParts.length > 0) {
-    return toolResultParts.map((r: any) => ({
+    const results = toolResultParts.map((r: any) => ({
       role: 'tool',
       tool_call_id: r.toolUseId,
       content: r.content
     }))
+    /* A tool message holds text only - pictures a read tool returned follow as one user message (agentic-loop takeToolImages). */
+    const images = otherParts.filter(p => p.type === 'image')
+    return images.length ? [...results, { role: 'user', content: images.map(toSDKContentPart) }] : results
   }
 
   // assistant 측 tool_use 가 있으면 message.tool_calls 로