@things-factory/ai-client-base 10.1.260 → 10.1.261

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@things-factory/ai-client-base",
-  "version": "10.1.260",
+  "version": "10.1.261",
   "main": "dist-server/index.js",
   "things-factory": true,
   "author": "heartyoh",
@@ -24,11 +24,11 @@
   "dependencies": {
     "@anthropic-ai/sdk": "^0.40.0",
     "@google/generative-ai": "^0.24.0",
-    "@things-factory/auth-base": "^10.1.260",
+    "@things-factory/auth-base": "^10.1.261",
     "@things-factory/env": "^10.1.149",
-    "@things-factory/setting-base": "^10.1.260",
-    "@things-factory/shell": "^10.1.260",
+    "@things-factory/setting-base": "^10.1.261",
+    "@things-factory/shell": "^10.1.261",
     "openai": "^4.0.0"
   },
-  "gitHead": "72de583ba0394dcf8195ef723b0ee4217507e0e5"
+  "gitHead": "06537757f8708e6a511c6232e8792f18fdd6a986"
 }

package/server/service/agent/ai-client-for.test.ts

@@ -48,6 +48,7 @@ registerAIProvider('fake', config => ({
     options.onUsage?.({ model: 'fake-1-0607', inputTokens: 300, outputTokens: 5 })
     return { toolCalls: [], stopReason: 'end_turn' }
   },
+  transcribe: async (_audio, options: any) => { options.onUsage?.({ model: 'fake-speech' }); return { text: 'spoken', segments: [{ start: 0, end: 1, text: 'spoken' }] } },
   listModels: async () => ['fake-1']
 }))
 
@@ -157,3 +158,13 @@ it('a failure is named by what the provider said', () => {
 it('a call is grouped by the UTC hour it started in', () => {
   expect(hourOf(new Date('2026-10-12T05:47:13.120Z')).toISOString()).toBe('2026-10-12T05:00:00.000Z')
 })
+
+it('speech transcription uses the same tenant usage boundary without fabricated token counts', async () => {
+  ;(readAiAgentDirectory as jest.Mock).mockResolvedValue({ entries: [entry('speech')], defaultAgentId: 'speech' })
+  const client = await aiClientFor({ domain: { id: 'office' } as any, user: { id: 'manager' }, modeKind: 'learning.material-transcription' })
+  expect(await client.transcribe!(Buffer.from('audio'))).toEqual({ text: 'spoken', segments: [{ start: 0, end: 1, text: 'spoken' }] })
+  await aiUsageRecorded()
+  expect(saved).toHaveLength(1)
+  expect(saved[0]).toMatchObject({ operation: 'transcribe', model: 'fake-speech', domain: { id: 'office' }, creator: { id: 'manager' } })
+  expect(saved[0].inputTokens).toBeUndefined()
+})

package/server/service/agent/ai-client-for.ts

@@ -156,6 +156,7 @@ function meter(client: AIClient, entry: AiAgentEntry, request: AiClientRequest):
     agent: { id: spec.id, name: spec.name, provider: spec.provider, model: spec.model },
     chat: (messages, options) => measured('chat', options, o => client.chat(messages, o)),
     generateJSON: (prompt, options) => measured('generateJSON', options, o => client.generateJSON(prompt, o)),
+    ...(client.transcribe && { transcribe: (audio, options) => measured('transcribe', options, o => client.transcribe!(audio, o)) }),
     ...(client.vision && {
       vision: (image, prompt, options) => measured('vision', options, o => client.vision!(image, prompt, o))
     }),

package/server/service/agent/ai-usage.ts

@@ -52,7 +52,7 @@ export class AiUsage {
   modeKind?: string
 
   @Column()
-  @Field(type => String, { description: 'chat, chatWithTools, generateJSON or vision.' })
+  @Field(type => String, { description: 'chat, chatWithTools, generateJSON, vision or transcribe.' })
   operation: string
 
   @Column({ nullable: true })

package/server/service/providers/openai.ts

@@ -7,6 +7,7 @@
  * organization 필드 지원.
  */
 import OpenAI from 'openai'
+import { toFile } from 'openai/uploads'
 import type {
   AIChatOptions,
   AIClient,
@@ -42,6 +43,17 @@ export class OpenAIAIClient implements AIClient {
     this.id = `openai:${this.defaultModel}`
   }
 
+  /** Whisper accepts the small MP3 prepared by the material reader and returns speech timestamps. */
+  async transcribe(audio: Buffer, options: AIChatOptions = {}) {
+    const model = 'whisper-1'
+    const answer = await this.sdk.audio.transcriptions.create({
+      file: await toFile(audio, 'speech.mp3', { type: 'audio/mpeg' }),
+      model, response_format: 'verbose_json', timestamp_granularities: ['segment']
+    }, requestOptionsOf(options))
+    options.onUsage?.({ model }) // Speech billing does not report text token counts.
+    return { text: answer.text, segments: (answer.segments ?? []).map(s => ({ start: s.start, end: s.end, text: s.text })) }
+  }
+
   async chat(messages: AIMessage[], options: AIChatOptions = {}): Promise<string> {
     const sdkMessages = buildSDKMessages(messages, options.systemPrompt)
     const response = await this.sdk.chat.completions.create({

package/server/service/providers/transcribe.test.ts

@@ -0,0 +1,18 @@
+import { OpenAIAIClient } from './openai'
+
+it('transcribes speech with timestamps, cancellation and usage reporting through the configured SDK', async () => {
+  const client: any = new OpenAIAIClient({ provider: 'openai', apiKey: 'test-only' })
+  const calls: any[] = []
+  client.sdk = { audio: { transcriptions: { create: async (body: any, request: any) => {
+    calls.push({ body, request })
+    return { text: 'Hello', segments: [{ start: 0, end: 1.5, text: 'Hello' }] }
+  } } } }
+  const usage: any[] = []
+  const { signal } = new AbortController()
+  const answer = await client.transcribe(Buffer.from('test audio'), { signal, onUsage: (u: any) => usage.push(u) })
+  expect(answer).toEqual({ text: 'Hello', segments: [{ start: 0, end: 1.5, text: 'Hello' }] })
+  expect(calls[0].body.model).toBe('whisper-1')
+  expect(calls[0].body.response_format).toBe('verbose_json')
+  expect(calls[0].request.signal).toBe(signal)
+  expect(usage).toEqual([{ model: 'whisper-1' }])
+})

package/server/service/types.ts

@@ -198,6 +198,9 @@ export interface AIClient {
     options?: AIToolChatOptions
   ): Promise<AIToolChatResult>
 
+  /** Optional speech recognition. The provider returns timestamped speech, never inferred video visuals. */
+  transcribe?(audio: Buffer, options?: AIChatOptions): Promise<{ text: string; segments: { start: number; end: number; text: string }[] }>
+
   /** The ids of the conversation models this key may call, sorted. Reading it also proves the key is accepted. */
   listModels(): Promise<string[]>
 }