@things-factory/ai-assistant 10.1.167 → 10.1.168

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@things-factory/ai-assistant",
-  "version": "10.1.167",
+  "version": "10.1.168",
   "main": "dist-server/index.js",
   "things-factory": true,
   "author": "heartyoh",
@@ -34,7 +34,7 @@
     "@operato/popup": "^10.16.0",
     "@operato/shell": "^10.0.0",
     "@operato/styles": "^10.0.0",
-    "@things-factory/ai-client-base": "^10.1.167",
+    "@things-factory/ai-client-base": "^10.1.168",
     "@things-factory/apptool-base": "^10.1.165",
     "@things-factory/attachment-base": "^10.1.167",
     "@things-factory/auth-base": "^10.1.167",
@@ -54,5 +54,5 @@
     "copyfiles": "^2.4.1",
     "rimraf": "^5.0.0"
   },
-  "gitHead": "86283c5d35042880be6827f777067fc5e6417eed"
+  "gitHead": "fe067ff11d90effffc2523374453d9b8313b489f"
 }

package/server/service/assistant-chat-resolver.ts

@@ -50,6 +50,7 @@ import { isSessionParticipant } from './chat-session-participant/participant-loo
 import { assertMayWriteSession } from './chat-session/session-write-gate.js'
 import { attachmentHistoryLine, refuseAttachments, textCharsOf } from './chat-session/attachment-policy.js'
 import { attachmentForModel, describeAttachment, findChatAttachment } from './chat-session/session-attachments.js'
+import { readTurnAttachments } from './chat-session/turn-attachments.js'
 import { getAssistantMode } from './assistant-mode/registry.js'
 import { resolveModeSettings } from './assistant-mode/resolve.js'
 import { splitNextRequest } from './assistant-mode/screen-rules.js'
@@ -468,22 +469,20 @@ export class AssistantChatResolver {
         )
       : input.messages.map(message => ({ role: message.role as any, content: message.content }))
 
+    /*
+     * The files checked above, read once: the model reads them on this turn's question, and the tools get the same
+     * files as `context.attachments` (ADR-0061, 2026-10-16 decision 2) - figure's step generator draws from the
+     * picture the person sent. Never mixed into `hostContext`, which the client fills.
+     */
+    const turnFiles = await readTurnAttachments(
+      attachmentRows,
+      textCharsOf(getAssistantMode(input.mode?.kind ?? '')?.attachments),
+      attachmentForModel
+    )
     if (attachmentRows.length && conversation.length > 0) {
       const last = conversation[conversation.length - 1]
       if (last.role === 'user' && typeof last.content === 'string') {
-        const parts: any[] = [{ type: 'text', text: last.content }]
-        for (const row of attachmentRows) {
-          const file = await attachmentForModel(row, textCharsOf(getAssistantMode(input.mode?.kind ?? '')?.attachments))
-          if (file.type === 'image') {
-            parts.push({ type: 'image', source: { kind: 'base64', data: file.data, mediaType: file.mediaType } })
-          } else {
-            parts.push({
-              type: 'text',
-              text: `[attachment ${row.id}: ${file.name}${file.truncated ? ' - cut, the file is longer' : ''}]\n${file.text}`
-            })
-          }
-        }
-        last.content = parts as any
+        last.content = [{ type: 'text', text: last.content }, ...turnFiles.parts] as any
       }
     } else if (input.hostContext?.image?.data && conversation.length > 0) {
       /* The screen-image path (figure's rendered view) - not a file of the conversation. */
@@ -510,6 +509,7 @@ export class AssistantChatResolver {
       /* The stop notice is a sentence a person reads — the server writes it in their language (ADR-0054). */
       t,
       hostContext: input.hostContext,
+      attachments: turnFiles.forTools,
       /*
        * 근거는 도구 결과만이 아니다 — 시스템 프롬프트 · 이 대화의 말 · 화면이 보낸 스냅숏도 근거다
        * (`ai-client-base/utils/grounding`). 이 문은 그 셋을 넘기지 않아, **사람이 방금 타이핑한

package/server/service/chat-session/turn-attachments.test.ts

@@ -0,0 +1,106 @@
+/*
+ * The files of this turn, read once for the model and for the tools (ADR-0061, 2026-10-16 decision 2).
+ *
+ * Before this the door put the files on the model's last message only and handed the tools the client's
+ * `hostContext`. A tool that needed the picture - figure's inner step generator - never got it: the dock uploads by
+ * `attachmentIds` and leaves `hostContext.image` empty.
+ */
+import { clearToolRegistry, registerToolCategory, registryAssistantChat } from '@things-factory/ai-client-base'
+
+import { readTurnAttachments } from './turn-attachments'
+
+const PNG_BYTES = Buffer.from([0x89, 0x50, 0x4e, 0x47])
+const JPG_BYTES = Buffer.from([0xff, 0xd8, 0xff])
+
+const rows = [
+  { id: 'att-1', name: 'front.png', mimetype: 'image/png', size: '4' },
+  { id: 'att-2', name: 'notes.txt', mimetype: 'text/plain', size: 12 },
+  { id: 'att-3', name: 'side.jpg', mimetype: 'image/jpeg', size: 3 }
+]
+
+/* Storage stand-in: what attachmentForModel answers for each row. */
+const read = async (row: any, maxChars: number) => {
+  if (row.id === 'att-1') return { type: 'image' as const, data: PNG_BYTES, mediaType: 'image/png', name: row.name }
+  if (row.id === 'att-3') return { type: 'image' as const, data: JPG_BYTES, mediaType: 'image/jpeg', name: row.name }
+  return { type: 'text' as const, text: 'twelve chars'.slice(0, maxChars), name: row.name, truncated: maxChars < 12 }
+}
+
+describe('the files of this turn', () => {
+  it('★ the tools get every file as the contract shapes it, in the order the person attached them', async () => {
+    const { forTools } = await readTurnAttachments(rows, 20_000, read)
+
+    expect(forTools).toEqual([
+      { id: 'att-1', name: 'front.png', kind: 'image', mediaType: 'image/png', size: 4, data: PNG_BYTES },
+      { id: 'att-2', name: 'notes.txt', kind: 'text', mediaType: 'text/plain', size: 12, text: 'twelve chars', truncated: false },
+      { id: 'att-3', name: 'side.jpg', kind: 'image', mediaType: 'image/jpeg', size: 3, data: JPG_BYTES }
+    ])
+  })
+
+  it('the model reads the same files - an image as bytes, a text file as its text with the cut said', async () => {
+    const { parts } = await readTurnAttachments(rows, 6, read)
+
+    expect(parts).toEqual([
+      { type: 'image', source: { kind: 'base64', data: PNG_BYTES, mediaType: 'image/png' } },
+      { type: 'text', text: '[attachment att-2: notes.txt - cut, the file is longer]\ntwelve' },
+      { type: 'image', source: { kind: 'base64', data: JPG_BYTES, mediaType: 'image/jpeg' } }
+    ])
+  })
+
+  it('a text file cut for the model is cut for the tools too, and says so', async () => {
+    const { forTools } = await readTurnAttachments([rows[1]], 6, read)
+
+    expect(forTools).toEqual([expect.objectContaining({ kind: 'text', text: 'twelve', truncated: true })])
+  })
+
+  it('no files: nothing for either', async () => {
+    expect(await readTurnAttachments([], 20_000, read)).toEqual({ parts: [], forTools: [] })
+  })
+})
+
+describe('the files reach a registered tool through the door', () => {
+  afterEach(() => clearToolRegistry())
+
+  it('★ what the door read is what the tool is handed - in its own field, not in host', async () => {
+    const handed: any[] = []
+    registerToolCategory({
+      name: 'drawing',
+      guidance: '- Call proposeDrawing for a drawing.',
+      specs: [
+        {
+          kind: 'read',
+          name: 'proposeDrawing',
+          description: 'proposes a drawing',
+          schema: { type: 'object', properties: {} },
+          doors: [],
+          builder: (_args: any, context: any) => {
+            handed.push(context)
+            return { ok: true }
+          }
+        } as any
+      ]
+    })
+    const { forTools } = await readTurnAttachments(rows, 20_000, read)
+    let turn = 0
+    const client: any = {
+      chatWithTools: async () =>
+        turn++ === 0
+          ? { toolCalls: [{ id: 'c1', name: 'proposeDrawing', arguments: {}, type: 'tool_use' }], stopReason: 'tool_use' }
+          : { text: 'done', toolCalls: [], stopReason: 'end_turn' }
+    }
+
+    await registryAssistantChat({
+      client,
+      messages: [{ role: 'user', content: 'make it like the pictures' }],
+      systemPrompt: 'draw',
+      toolCategories: ['drawing'],
+      toolAccess: async () => true,
+      state: { domain: { id: 'dom-1' }, user: { id: 'u-1' } },
+      hostContext: { figureId: 'fig-1' },
+      attachments: forTools
+    })
+
+    expect(handed).toHaveLength(1)
+    expect(handed[0].attachments.filter((file: any) => file.kind === 'image').map((file: any) => file.data)).toEqual([PNG_BYTES, JPG_BYTES])
+    expect(handed[0].host).toEqual({ figureId: 'fig-1' })
+  })
+})

package/server/service/chat-session/turn-attachments.ts

@@ -0,0 +1,45 @@
+/*
+ * The files sent with this turn, read once: as the parts the model reads after the question, and as the tools'
+ * `context.attachments` (ADR-0061, 2026-10-16 decision 2).
+ *
+ * The door checked them before this - each is stored with this conversation, the caller may write to it, the mode's
+ * policy took it. Only the door builds the tools' list; what a client puts in `hostContext` never becomes one.
+ *
+ * Storage is the caller's: it passes `attachmentForModel`, so the rule is tested without a file store.
+ */
+import type { ToolAttachment } from '@things-factory/ai-client-base'
+
+export type ReadAttachment = (
+  row: any,
+  maxChars: number
+) => Promise<
+  | { type: 'image'; data: Buffer; mediaType: string; name: string }
+  | { type: 'text'; text: string; name: string; truncated: boolean }
+>
+
+export interface TurnAttachments {
+  /** What the model reads after the question's text, one part per file. */
+  parts: any[]
+  /** The same files for the tools, in the order the person attached them. */
+  forTools: ToolAttachment[]
+}
+
+export async function readTurnAttachments(rows: any[], maxChars: number, read: ReadAttachment): Promise<TurnAttachments> {
+  const parts: any[] = []
+  const forTools: ToolAttachment[] = []
+  for (const row of rows) {
+    const file = await read(row, maxChars)
+    const stored = { id: row.id, name: row.name, mediaType: row.mimetype, size: Number(row.size) || 0 }
+    if (file.type === 'image') {
+      parts.push({ type: 'image', source: { kind: 'base64', data: file.data, mediaType: file.mediaType } })
+      forTools.push({ ...stored, kind: 'image', data: file.data })
+    } else {
+      parts.push({
+        type: 'text',
+        text: `[attachment ${row.id}: ${file.name}${file.truncated ? ' - cut, the file is longer' : ''}]\n${file.text}`
+      })
+      forTools.push({ ...stored, kind: 'text', text: file.text, truncated: file.truncated })
+    }
+  }
+  return { parts, forTools }
+}