@tanstack-ai-browser/afm 0.0.0-stage → 0.2.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/Package.swift +33 -0
- package/README.md +75 -2
- package/dist/adapter.d.ts +107 -0
- package/dist/adapter.d.ts.map +1 -0
- package/dist/adapter.js +522 -0
- package/dist/electron.d.ts +29 -0
- package/dist/electron.d.ts.map +1 -0
- package/dist/electron.js +136 -0
- package/dist/index.d.ts +17 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +12 -0
- package/dist/json-value.d.ts +9 -0
- package/dist/json-value.d.ts.map +1 -0
- package/dist/json-value.js +19 -0
- package/dist/messages.d.ts +12 -0
- package/dist/messages.d.ts.map +1 -0
- package/dist/messages.js +29 -0
- package/dist/model-meta.d.ts +23 -0
- package/dist/model-meta.d.ts.map +1 -0
- package/dist/model-meta.js +20 -0
- package/dist/native.d.ts +175 -0
- package/dist/native.d.ts.map +1 -0
- package/dist/native.js +21 -0
- package/dist/schema.d.ts +17 -0
- package/dist/schema.d.ts.map +1 -0
- package/dist/schema.js +133 -0
- package/dist/validate.d.ts +2 -0
- package/dist/validate.d.ts.map +1 -0
- package/dist/validate.js +124 -0
- package/electron/AfmNode.swift +81 -0
- package/electron/afm.c +94 -0
- package/electron/afm.node +0 -0
- package/electron/afm.node.sha256 +1 -0
- package/electron/build.mjs +54 -0
- package/electron/preload.cjs +35 -0
- package/electron/sources-hash.mjs +40 -0
- package/ios/Sources/AfmCore/Bridge.swift +354 -0
- package/ios/Sources/AfmCore/Conversation.swift +49 -0
- package/ios/Sources/AfmCore/Schema.swift +90 -0
- package/ios/Sources/AfmPlugin/AfmPlugin.swift +89 -0
- package/package.json +97 -3
- package/src/adapter.test.ts +428 -0
- package/src/adapter.ts +616 -0
- package/src/electron.ts +193 -0
- package/src/index.ts +28 -0
- package/src/json-value.test.ts +26 -0
- package/src/json-value.ts +17 -0
- package/src/messages.ts +37 -0
- package/src/model-meta.ts +37 -0
- package/src/native-build.test.ts +13 -0
- package/src/native.ts +128 -0
- package/src/schema.test.ts +146 -0
- package/src/schema.ts +153 -0
- package/src/validate.test.ts +38 -0
- package/src/validate.ts +112 -0
package/src/adapter.ts
ADDED
|
@@ -0,0 +1,616 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* A TanStack AI text adapter for Apple's on-device model, through the
|
|
3
|
+
* Foundation Models framework itself: the Capacitor plugin on iOS, or any
|
|
4
|
+
* host that implements `AfmNative`. Hand it to `chat()` like any
|
|
5
|
+
* other adapter: `chat()` runs the tools and the agent loop, and each
|
|
6
|
+
* `chatStream` call here is one step of the model, streamed as AG-UI events.
|
|
7
|
+
*
|
|
8
|
+
* Tools are the framework's own, so the model's arguments always fit their
|
|
9
|
+
* parameters, and it calls several at once. The framework runs a tool in the
|
|
10
|
+
* middle of a response; here the call is reported to `chat()` (the step ends
|
|
11
|
+
* with `tool_calls`) while the native response waits, and the next step hands
|
|
12
|
+
* it the results: the response goes on where it was, without the session
|
|
13
|
+
* reading the conversation again.
|
|
14
|
+
*
|
|
15
|
+
* Each thread (`threadId`) keeps a session: a step prompts it with only the
|
|
16
|
+
* new message. A conversation that changed (an edit, other tools, a new
|
|
17
|
+
* system prompt) starts a new session from the messages.
|
|
18
|
+
*/
|
|
19
|
+
import { buildBaseUsage, EventType, normalizeSystemPrompts } from "@tanstack/ai"
|
|
20
|
+
import type { AdapterYieldChunk, ModelMessage, TextOptions, TokenUsage, Tool } from "@tanstack/ai"
|
|
21
|
+
import { BaseTextAdapter } from "@tanstack/ai/adapters"
|
|
22
|
+
import type { StructuredOutputOptions, StructuredOutputResult } from "@tanstack/ai/adapters"
|
|
23
|
+
|
|
24
|
+
import { contentText, toNativeMessages } from "./messages.js"
|
|
25
|
+
import type { AfmChatModel } from "./model-meta.js"
|
|
26
|
+
import { Afm } from "./native.js"
|
|
27
|
+
import type { AfmGuardrails, AfmNative, NativeEvent, NativeMessage, NativeUsage, UnavailableReason } from "./native.js"
|
|
28
|
+
import { toNativeSchema } from "./schema.js"
|
|
29
|
+
import { parseJsonValue } from "./json-value.js"
|
|
30
|
+
import { schemaErrors } from "./validate.js"
|
|
31
|
+
|
|
32
|
+
export type AfmConfig = {
|
|
33
|
+
/**
|
|
34
|
+
* The native side. By default `window.afm` where a host put it there
|
|
35
|
+
* (Electron, with `serveAfm()` in the main process), else the Capacitor
|
|
36
|
+
* plugin (`Afm`).
|
|
37
|
+
*/
|
|
38
|
+
native?: AfmNative
|
|
39
|
+
/**
|
|
40
|
+
* Apple's guardrails: "default" (the default), or "permissiveContentTransformations",
|
|
41
|
+
* for text the default ones block out of context, such as a card game's card
|
|
42
|
+
* names. A request's `modelOptions.guardrails` overrides it.
|
|
43
|
+
*/
|
|
44
|
+
guardrails?: AfmGuardrails
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
/** Per-call `modelOptions`: the response's sampling and length, and the guardrails. */
|
|
48
|
+
export type AfmTextModelOptions = {
|
|
49
|
+
temperature?: number
|
|
50
|
+
/** Most tokens to generate. */
|
|
51
|
+
maximumResponseTokens?: number
|
|
52
|
+
/** Apple's guardrails for this request's session, over the adapter's `guardrails`. */
|
|
53
|
+
guardrails?: AfmGuardrails
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
export type Availability = "unavailable" | "downloadable" | "downloading" | "available"
|
|
57
|
+
|
|
58
|
+
type MetadataByModality = {
|
|
59
|
+
text: unknown
|
|
60
|
+
image: unknown
|
|
61
|
+
audio: unknown
|
|
62
|
+
video: unknown
|
|
63
|
+
document: unknown
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
/** Events of one native response, as they come. */
|
|
67
|
+
class EventQueue {
|
|
68
|
+
private readonly events: Array<NativeEvent> = []
|
|
69
|
+
private wake: (() => void) | undefined
|
|
70
|
+
|
|
71
|
+
push(event: NativeEvent) {
|
|
72
|
+
this.events.push(event)
|
|
73
|
+
this.wake?.()
|
|
74
|
+
this.wake = undefined
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
async next(signal: AbortSignal | undefined): Promise<NativeEvent> {
|
|
78
|
+
// Stopped: not even an event already here (a buffered done) goes out.
|
|
79
|
+
signal?.throwIfAborted()
|
|
80
|
+
while (this.events.length === 0) {
|
|
81
|
+
signal?.throwIfAborted()
|
|
82
|
+
await new Promise<void>((resolve, reject) => {
|
|
83
|
+
const abort = () => reject(signal?.reason ?? new DOMException("Aborted", "AbortError"))
|
|
84
|
+
this.wake = () => {
|
|
85
|
+
signal?.removeEventListener("abort", abort)
|
|
86
|
+
resolve()
|
|
87
|
+
}
|
|
88
|
+
signal?.addEventListener("abort", abort, { once: true })
|
|
89
|
+
})
|
|
90
|
+
}
|
|
91
|
+
return this.events.shift() as NativeEvent
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
/** A thread's native session, and its response in progress (waiting at tool calls). */
|
|
96
|
+
type Thread = {
|
|
97
|
+
sessionId: string
|
|
98
|
+
settings: string
|
|
99
|
+
/** The conversation the session holds, by message key. */
|
|
100
|
+
seen: Array<string>
|
|
101
|
+
turn: number
|
|
102
|
+
/** The response in progress: set from `respond` until `done` or an error. */
|
|
103
|
+
queue: EventQueue | undefined
|
|
104
|
+
/** The response's text so far, over all its steps. */
|
|
105
|
+
written: string
|
|
106
|
+
/** Tool calls reported to `chat()` that wait for results. */
|
|
107
|
+
waiting: Set<string>
|
|
108
|
+
busy: boolean
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
const KEPT_SESSIONS = 4
|
|
112
|
+
|
|
113
|
+
/** A message, as the session holds it: an assistant's text and its tool calls (ids, names, arguments). */
|
|
114
|
+
const messageKey = (m: ModelMessage) =>
|
|
115
|
+
m.role === "assistant"
|
|
116
|
+
? JSON.stringify([
|
|
117
|
+
"assistant",
|
|
118
|
+
contentText(m.content).trim(),
|
|
119
|
+
(m.toolCalls ?? []).map((call) => [call.id, call.function.name, call.function.arguments])
|
|
120
|
+
])
|
|
121
|
+
: JSON.stringify([m.role, contentText(m.content), m.toolCallId ?? null])
|
|
122
|
+
|
|
123
|
+
function offeredTools(options: TextOptions<AfmTextModelOptions>): ReadonlyArray<Tool> {
|
|
124
|
+
const { toolChoice, tools = [] } = options
|
|
125
|
+
if (toolChoice === "none") return []
|
|
126
|
+
if (typeof toolChoice === "object") return tools.filter((tool) => tool.name === toolChoice.name)
|
|
127
|
+
return tools
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
/**
|
|
131
|
+
* When to call tools, added to the instructions of a session that has some.
|
|
132
|
+
* Given tools and no word on when to use them, Apple's model on an iPhone
|
|
133
|
+
* (iOS 27.2) calls one for every prompt, "Hi" and "What is 2 + 2?" included;
|
|
134
|
+
* on a Mac it doesn't. With this, the iPhone's answered greetings, small talk,
|
|
135
|
+
* math, explanations and writing itself, and called the weather, location and
|
|
136
|
+
* web search tools for the weather, "here", current facts and the news. Told
|
|
137
|
+
* only to answer such requests "directly", it refused to write poems ("I
|
|
138
|
+
* cannot write a haiku"); naming poems, stories and emails as its own work
|
|
139
|
+
* keeps it writing.
|
|
140
|
+
*/
|
|
141
|
+
const TOOL_USE =
|
|
142
|
+
"Most requests need no tool: greetings and small talk, math, writing (poems, stories, emails), explanations and general knowledge. Answer those yourself, without calling a tool. Call a tool only for what you can't do yourself: information that changes or that you don't have, or something only a tool can do."
|
|
143
|
+
|
|
144
|
+
/** The framework's tool-calling mode for a step: "required" when a tool must be called ("required", or the one tool named). */
|
|
145
|
+
const toolCallingModeOf = (options: TextOptions<AfmTextModelOptions>) =>
|
|
146
|
+
options.toolChoice === "required" || typeof options.toolChoice === "object" ? ("required" as const) : undefined
|
|
147
|
+
|
|
148
|
+
/** The system prompts, and when to call tools if the session has some. */
|
|
149
|
+
const instructionsOf = (options: TextOptions<AfmTextModelOptions>, tools: ReadonlyArray<Tool>) =>
|
|
150
|
+
[...normalizeSystemPrompts(options.systemPrompts).map((p) => p.content), ...(tools.length > 0 ? [TOOL_USE] : [])]
|
|
151
|
+
.filter(Boolean)
|
|
152
|
+
.join("\n\n")
|
|
153
|
+
|
|
154
|
+
const AVAILABILITY: Record<UnavailableReason, Availability> = {
|
|
155
|
+
deviceNotEligible: "unavailable",
|
|
156
|
+
appleIntelligenceNotEnabled: "unavailable",
|
|
157
|
+
unsupportedOS: "unavailable",
|
|
158
|
+
unavailable: "unavailable",
|
|
159
|
+
// Apple Intelligence is on, and its model is still downloading.
|
|
160
|
+
modelNotReady: "downloading"
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
const usageOf = (usage: NativeUsage | undefined): TokenUsage | undefined =>
|
|
164
|
+
usage && {
|
|
165
|
+
...buildBaseUsage({
|
|
166
|
+
promptTokens: usage.inputTokens,
|
|
167
|
+
completionTokens: usage.outputTokens,
|
|
168
|
+
totalTokens: usage.inputTokens + usage.outputTokens
|
|
169
|
+
}),
|
|
170
|
+
// 0 too: nothing was cached. Left out where the framework can't say.
|
|
171
|
+
...(usage.cachedTokens !== undefined && { promptTokensDetails: { cachedTokens: usage.cachedTokens } })
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
let ids = 0
|
|
175
|
+
const nextId = (prefix: string) => `${prefix}_${Date.now().toString(36)}_${(ids++).toString(36)}`
|
|
176
|
+
|
|
177
|
+
const signalOf = (options: TextOptions<AfmTextModelOptions>) =>
|
|
178
|
+
options.abortController?.signal ?? (options.request as RequestInit | undefined)?.signal ?? undefined
|
|
179
|
+
|
|
180
|
+
/**
|
|
181
|
+
* A structured answer the model wrote as text (permissive guardrails' fallback,
|
|
182
|
+
* which the framework didn't hold to the schema): its JSON value, if it fits the
|
|
183
|
+
* schema. chat() validates only schemas with a Standard Schema validator, not
|
|
184
|
+
* plain JSON Schema, so the adapter checks it here.
|
|
185
|
+
*/
|
|
186
|
+
function checkedAnswer(text: string, schema: unknown): unknown {
|
|
187
|
+
let data: unknown
|
|
188
|
+
try {
|
|
189
|
+
data = parseJsonValue(text)
|
|
190
|
+
} catch (error) {
|
|
191
|
+
throw Object.assign(new Error(`Apple's model wrote no JSON for the schema (its answer, written as text past its guardrails, was ${JSON.stringify(text.slice(0, 120))})`, { cause: error }), { code: "invalidJson" })
|
|
192
|
+
}
|
|
193
|
+
const problems = schemaErrors(schema, data)
|
|
194
|
+
if (problems.length > 0) {
|
|
195
|
+
throw Object.assign(new Error(`Apple's model's answer, written as text past its guardrails, doesn't fit the schema: ${problems.slice(0, 5).join("; ")}`), { code: "schemaMismatch" })
|
|
196
|
+
}
|
|
197
|
+
return data
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
const parse = (json: string): unknown => {
|
|
201
|
+
try {
|
|
202
|
+
return JSON.parse(json)
|
|
203
|
+
} catch {
|
|
204
|
+
return {}
|
|
205
|
+
}
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
export class AfmTextAdapter extends BaseTextAdapter<
|
|
209
|
+
AfmChatModel,
|
|
210
|
+
AfmTextModelOptions,
|
|
211
|
+
readonly ["text"],
|
|
212
|
+
MetadataByModality
|
|
213
|
+
> {
|
|
214
|
+
override readonly kind = "text" as const
|
|
215
|
+
readonly name = "afm" as const
|
|
216
|
+
override readonly provider = "afm" as const
|
|
217
|
+
override readonly api = "afm" as const
|
|
218
|
+
override readonly inputModalities = ["text"] as const
|
|
219
|
+
private readonly native: AfmNative
|
|
220
|
+
private readonly threads = new Map<string, Thread>()
|
|
221
|
+
private readonly queues = new Map<string, Thread>()
|
|
222
|
+
/**
|
|
223
|
+
* Responses waiting for tool results in sessions no thread keeps (prompt
|
|
224
|
+
* cache retention "none"), by thread: the next step resumes them.
|
|
225
|
+
*/
|
|
226
|
+
private readonly paused = new Map<string, Thread>()
|
|
227
|
+
private listening: Promise<unknown> | undefined
|
|
228
|
+
|
|
229
|
+
constructor(config: AfmConfig, model: AfmChatModel) {
|
|
230
|
+
super({}, model)
|
|
231
|
+
this.native = config.native ?? (globalThis as { afm?: AfmNative }).afm ?? Afm
|
|
232
|
+
this.guardrails = config.guardrails
|
|
233
|
+
}
|
|
234
|
+
|
|
235
|
+
private readonly guardrails: AfmGuardrails | undefined
|
|
236
|
+
|
|
237
|
+
/** A request's guardrails: its own, else the adapter's (left out: the host's, else Apple's default). */
|
|
238
|
+
private guardrailsOf(options: TextOptions<AfmTextModelOptions>) {
|
|
239
|
+
return options.modelOptions?.guardrails ?? this.guardrails
|
|
240
|
+
}
|
|
241
|
+
|
|
242
|
+
/** Whether Apple's model can run here: "downloading" while Apple Intelligence gets it. */
|
|
243
|
+
async availability(): Promise<Availability> {
|
|
244
|
+
try {
|
|
245
|
+
const answer = await this.native.availability(this.guardrails ? { guardrails: this.guardrails } : undefined)
|
|
246
|
+
return answer.available ? "available" : AVAILABILITY[answer.reason ?? "unavailable"]
|
|
247
|
+
} catch {
|
|
248
|
+
return "unavailable"
|
|
249
|
+
}
|
|
250
|
+
}
|
|
251
|
+
|
|
252
|
+
/** Tokens a session holds, where the OS says (26.4 and later). */
|
|
253
|
+
async contextSize(): Promise<number | undefined> {
|
|
254
|
+
try {
|
|
255
|
+
return (await this.native.availability(this.guardrails ? { guardrails: this.guardrails } : undefined)).contextSize
|
|
256
|
+
} catch {
|
|
257
|
+
return undefined
|
|
258
|
+
}
|
|
259
|
+
}
|
|
260
|
+
|
|
261
|
+
/** Resolves to this adapter if the model can run here, else rejects. */
|
|
262
|
+
async create(options: { signal?: AbortSignal } = {}): Promise<this> {
|
|
263
|
+
options.signal?.throwIfAborted()
|
|
264
|
+
const answer = await this.native.availability()
|
|
265
|
+
options.signal?.throwIfAborted()
|
|
266
|
+
if (!answer.available) throw new Error(`Apple's on-device model is unavailable here (${answer.reason ?? "unavailable"})`)
|
|
267
|
+
return this
|
|
268
|
+
}
|
|
269
|
+
|
|
270
|
+
/**
|
|
271
|
+
* Ends every session: the threads', responses waiting for tool results, and
|
|
272
|
+
* responses streaming now (structured output's, and those of sessions no
|
|
273
|
+
* thread keeps), each once.
|
|
274
|
+
*/
|
|
275
|
+
async destroy(): Promise<void> {
|
|
276
|
+
const all = new Set([...this.threads.values(), ...this.paused.values(), ...this.queues.values()])
|
|
277
|
+
for (const thread of all) this.close(thread)
|
|
278
|
+
this.threads.clear()
|
|
279
|
+
this.paused.clear()
|
|
280
|
+
}
|
|
281
|
+
|
|
282
|
+
/** Routes the native events to their thread's response. Listens once. */
|
|
283
|
+
private listen() {
|
|
284
|
+
this.listening ??= this.native.addListener("event", (event) => {
|
|
285
|
+
const thread = this.queues.get(event.sessionId)
|
|
286
|
+
if (thread && event.turn === thread.turn) thread.queue?.push(event)
|
|
287
|
+
})
|
|
288
|
+
return this.listening
|
|
289
|
+
}
|
|
290
|
+
|
|
291
|
+
/** Ends a session. A response still streaming from it ends with an error. */
|
|
292
|
+
private close(thread: Thread) {
|
|
293
|
+
this.queues.delete(thread.sessionId)
|
|
294
|
+
thread.queue?.push({ type: "error", code: "closed", message: "The session was closed", sessionId: thread.sessionId, turn: thread.turn })
|
|
295
|
+
thread.queue = undefined
|
|
296
|
+
void this.native.deleteSession({ sessionId: thread.sessionId }).catch(() => undefined)
|
|
297
|
+
}
|
|
298
|
+
|
|
299
|
+
/** Keeps a thread's session, ending the least recently used past the limit. */
|
|
300
|
+
private keep(key: string, thread: Thread) {
|
|
301
|
+
this.threads.delete(key)
|
|
302
|
+
this.threads.set(key, thread)
|
|
303
|
+
for (const [id, old] of this.threads) {
|
|
304
|
+
if (this.threads.size <= KEPT_SESSIONS) break
|
|
305
|
+
if (old.busy) continue
|
|
306
|
+
this.close(old)
|
|
307
|
+
this.threads.delete(id)
|
|
308
|
+
}
|
|
309
|
+
}
|
|
310
|
+
|
|
311
|
+
/** A new native session for a thread, holding `messages`. */
|
|
312
|
+
private async open(
|
|
313
|
+
options: TextOptions<AfmTextModelOptions>,
|
|
314
|
+
tools: ReadonlyArray<Tool>,
|
|
315
|
+
settings: string,
|
|
316
|
+
messages: Array<NativeMessage>
|
|
317
|
+
): Promise<Thread> {
|
|
318
|
+
const thread: Thread = {
|
|
319
|
+
sessionId: nextId("session"),
|
|
320
|
+
settings,
|
|
321
|
+
seen: [],
|
|
322
|
+
turn: 0,
|
|
323
|
+
queue: undefined,
|
|
324
|
+
written: "",
|
|
325
|
+
waiting: new Set(),
|
|
326
|
+
busy: true
|
|
327
|
+
}
|
|
328
|
+
const instructions = instructionsOf(options, tools)
|
|
329
|
+
await this.native.createSession({
|
|
330
|
+
sessionId: thread.sessionId,
|
|
331
|
+
...(instructions && { instructions }),
|
|
332
|
+
tools: tools.map((tool) => ({
|
|
333
|
+
name: tool.name,
|
|
334
|
+
description: tool.description ?? "",
|
|
335
|
+
parameters: toNativeSchema(tool.inputSchema ?? { type: "object", properties: {} }, tool.name)
|
|
336
|
+
})),
|
|
337
|
+
messages,
|
|
338
|
+
...(this.guardrailsOf(options) && { guardrails: this.guardrailsOf(options) })
|
|
339
|
+
})
|
|
340
|
+
return thread
|
|
341
|
+
}
|
|
342
|
+
|
|
343
|
+
/**
|
|
344
|
+
* The step's thread, and what it sends: the results of the tool calls its
|
|
345
|
+
* response waits on, a new prompt to its session, or (for a conversation the
|
|
346
|
+
* session doesn't hold) a new session and the last message as its prompt.
|
|
347
|
+
*/
|
|
348
|
+
private async begin(options: TextOptions<AfmTextModelOptions>) {
|
|
349
|
+
await this.listen()
|
|
350
|
+
const key = options.promptCache?.key ?? options.threadId
|
|
351
|
+
const retain = options.promptCache?.retention !== "none"
|
|
352
|
+
const tools = offeredTools(options)
|
|
353
|
+
const { temperature, maximumResponseTokens } = options.modelOptions ?? {}
|
|
354
|
+
const settings = JSON.stringify([
|
|
355
|
+
instructionsOf(options, tools),
|
|
356
|
+
tools.map((tool) => [tool.name, tool.description, tool.inputSchema]),
|
|
357
|
+
temperature,
|
|
358
|
+
maximumResponseTokens,
|
|
359
|
+
// A response keeps the mode it started with: a step with another mode
|
|
360
|
+
// (required, then auto) can't resume it, and starts a new session.
|
|
361
|
+
toolCallingModeOf(options),
|
|
362
|
+
// A session's guardrails are its model's: other guardrails, another session.
|
|
363
|
+
// None named isn't "default": the host's (serveAfm's) may be permissive.
|
|
364
|
+
this.guardrailsOf(options) ?? "inherited"
|
|
365
|
+
])
|
|
366
|
+
const keys = options.messages.map(messageKey)
|
|
367
|
+
// A kept session, or one whose response waits for tool results.
|
|
368
|
+
const held = (retain ? this.threads.get(key) : undefined) ?? this.paused.get(key)
|
|
369
|
+
const holds = !!held && !held.busy && held.settings === settings && held.seen.every((k, i) => k === keys[i])
|
|
370
|
+
|
|
371
|
+
// The results of the tool calls the session's response waits on.
|
|
372
|
+
if (holds && held.queue && held.waiting.size > 0) {
|
|
373
|
+
const results = options.messages
|
|
374
|
+
.slice(held.seen.length)
|
|
375
|
+
.filter((m) => m.role === "tool" && !!m.toolCallId && held.waiting.has(m.toolCallId))
|
|
376
|
+
if (results.length === held.waiting.size) {
|
|
377
|
+
held.busy = true
|
|
378
|
+
this.paused.delete(key)
|
|
379
|
+
return { key, thread: held, kept: this.threads.get(key) === held, keys, results }
|
|
380
|
+
}
|
|
381
|
+
}
|
|
382
|
+
// A new message to the session's finished response.
|
|
383
|
+
const last = options.messages.at(-1)
|
|
384
|
+
if (holds && !held.queue && keys.length === held.seen.length + 1 && last?.role === "user") {
|
|
385
|
+
held.busy = true
|
|
386
|
+
return { key, thread: held, kept: true, keys, prompt: contentText(last.content) }
|
|
387
|
+
}
|
|
388
|
+
// A new session for the conversation, prompted with its last message.
|
|
389
|
+
if (held && !held.busy) {
|
|
390
|
+
this.close(held)
|
|
391
|
+
if (this.threads.get(key) === held) this.threads.delete(key)
|
|
392
|
+
if (this.paused.get(key) === held) this.paused.delete(key)
|
|
393
|
+
}
|
|
394
|
+
const prompted = last?.role === "user"
|
|
395
|
+
const before = prompted ? options.messages.slice(0, -1) : options.messages
|
|
396
|
+
const thread = await this.open(options, tools, settings, toNativeMessages(before))
|
|
397
|
+
const kept = retain && !this.threads.get(key)?.busy
|
|
398
|
+
if (kept) this.keep(key, thread)
|
|
399
|
+
return { key, thread, kept, keys, prompt: prompted ? contentText(last.content) : "" }
|
|
400
|
+
}
|
|
401
|
+
|
|
402
|
+
async *chatStream(options: TextOptions<AfmTextModelOptions>): AsyncIterable<AdapterYieldChunk> {
|
|
403
|
+
const runId = options.runId ?? nextId("run")
|
|
404
|
+
const { threadId } = options
|
|
405
|
+
const messageId = nextId("msg")
|
|
406
|
+
const model = this.model
|
|
407
|
+
const signal = signalOf(options)
|
|
408
|
+
const timestamp = () => Date.now()
|
|
409
|
+
yield {
|
|
410
|
+
type: EventType.RUN_STARTED,
|
|
411
|
+
runId,
|
|
412
|
+
threadId,
|
|
413
|
+
model,
|
|
414
|
+
timestamp: timestamp(),
|
|
415
|
+
...(options.parentRunId && { parentRunId: options.parentRunId })
|
|
416
|
+
}
|
|
417
|
+
|
|
418
|
+
let step: Awaited<ReturnType<typeof this.begin>> | undefined
|
|
419
|
+
let paused = false
|
|
420
|
+
try {
|
|
421
|
+
// A stopped request starts no native work (and one stopped meanwhile ends here).
|
|
422
|
+
signal?.throwIfAborted()
|
|
423
|
+
step = await this.begin(options)
|
|
424
|
+
signal?.throwIfAborted()
|
|
425
|
+
const { thread, keys } = step
|
|
426
|
+
if (step.results) {
|
|
427
|
+
for (const result of step.results) {
|
|
428
|
+
const callId = result.toolCallId as string
|
|
429
|
+
thread.waiting.delete(callId)
|
|
430
|
+
await this.native.submitToolResult({ sessionId: thread.sessionId, callId, output: contentText(result.content) })
|
|
431
|
+
}
|
|
432
|
+
} else {
|
|
433
|
+
const { temperature, maximumResponseTokens } = options.modelOptions ?? {}
|
|
434
|
+
thread.turn += 1
|
|
435
|
+
thread.written = ""
|
|
436
|
+
thread.queue = new EventQueue()
|
|
437
|
+
this.queues.set(thread.sessionId, thread)
|
|
438
|
+
await this.native.respond({
|
|
439
|
+
sessionId: thread.sessionId,
|
|
440
|
+
turn: thread.turn,
|
|
441
|
+
prompt: step.prompt ?? "",
|
|
442
|
+
options: {
|
|
443
|
+
...(temperature !== undefined && { temperature }),
|
|
444
|
+
...(maximumResponseTokens !== undefined && { maximumResponseTokens }),
|
|
445
|
+
...(toolCallingModeOf(options) && { toolCallingMode: toolCallingModeOf(options) })
|
|
446
|
+
}
|
|
447
|
+
})
|
|
448
|
+
}
|
|
449
|
+
|
|
450
|
+
// This step's text (its assistant message), and the events that write it.
|
|
451
|
+
let text = ""
|
|
452
|
+
const textEvents = function* (delta: string): Generator<AdapterYieldChunk> {
|
|
453
|
+
if (!delta) return
|
|
454
|
+
if (!text) yield { type: EventType.TEXT_MESSAGE_START, messageId, role: "assistant", model, timestamp: timestamp() }
|
|
455
|
+
text += delta
|
|
456
|
+
thread.written += delta
|
|
457
|
+
yield { type: EventType.TEXT_MESSAGE_CONTENT, messageId, delta, content: text, model, timestamp: timestamp() }
|
|
458
|
+
}
|
|
459
|
+
const endText = function* (): Generator<AdapterYieldChunk> {
|
|
460
|
+
if (text) yield { type: EventType.TEXT_MESSAGE_END, messageId, model, timestamp: timestamp() }
|
|
461
|
+
}
|
|
462
|
+
const queue = thread.queue as EventQueue
|
|
463
|
+
for (;;) {
|
|
464
|
+
const event = await queue.next(signal)
|
|
465
|
+
if (event.type === "delta") {
|
|
466
|
+
yield* textEvents(event.text)
|
|
467
|
+
} else if (event.type === "toolCalls") {
|
|
468
|
+
yield* endText()
|
|
469
|
+
for (const [index, call] of event.calls.entries()) {
|
|
470
|
+
thread.waiting.add(call.id)
|
|
471
|
+
yield {
|
|
472
|
+
type: EventType.TOOL_CALL_START,
|
|
473
|
+
toolCallId: call.id,
|
|
474
|
+
toolCallName: call.name,
|
|
475
|
+
parentMessageId: messageId,
|
|
476
|
+
index,
|
|
477
|
+
model,
|
|
478
|
+
timestamp: timestamp()
|
|
479
|
+
}
|
|
480
|
+
yield { type: EventType.TOOL_CALL_ARGS, toolCallId: call.id, delta: call.arguments, args: call.arguments, model, timestamp: timestamp() }
|
|
481
|
+
yield {
|
|
482
|
+
type: EventType.TOOL_CALL_END,
|
|
483
|
+
toolCallId: call.id,
|
|
484
|
+
toolCallName: call.name,
|
|
485
|
+
args: call.arguments,
|
|
486
|
+
input: parse(call.arguments),
|
|
487
|
+
model,
|
|
488
|
+
timestamp: timestamp()
|
|
489
|
+
}
|
|
490
|
+
}
|
|
491
|
+
// The response waits for the results, which the next step sends.
|
|
492
|
+
thread.seen = keys
|
|
493
|
+
paused = true
|
|
494
|
+
// A session no thread keeps is still found by the step with the results.
|
|
495
|
+
if (!step.kept) this.paused.set(step.key, thread)
|
|
496
|
+
yield { type: EventType.RUN_FINISHED, runId, threadId, model, timestamp: timestamp(), finishReason: "tool_calls" }
|
|
497
|
+
return
|
|
498
|
+
} else if (event.type === "done") {
|
|
499
|
+
// Any of the response not sent as deltas.
|
|
500
|
+
if (event.text.startsWith(thread.written)) yield* textEvents(event.text.slice(thread.written.length))
|
|
501
|
+
yield* endText()
|
|
502
|
+
thread.queue = undefined
|
|
503
|
+
this.queues.delete(thread.sessionId)
|
|
504
|
+
thread.seen = [...keys, messageKey({ role: "assistant", content: text })]
|
|
505
|
+
const usage = usageOf(event.usage)
|
|
506
|
+
yield {
|
|
507
|
+
type: EventType.RUN_FINISHED,
|
|
508
|
+
runId,
|
|
509
|
+
threadId,
|
|
510
|
+
model,
|
|
511
|
+
timestamp: timestamp(),
|
|
512
|
+
finishReason: "stop",
|
|
513
|
+
...(usage && { usage })
|
|
514
|
+
}
|
|
515
|
+
return
|
|
516
|
+
} else {
|
|
517
|
+
throw Object.assign(new Error(event.message), { code: event.code })
|
|
518
|
+
}
|
|
519
|
+
}
|
|
520
|
+
} catch (error) {
|
|
521
|
+
const aborted = !!signal?.aborted
|
|
522
|
+
if (step) {
|
|
523
|
+
if (aborted) void this.native.cancel({ sessionId: step.thread.sessionId }).catch(() => undefined)
|
|
524
|
+
// A stopped or failed response leaves the session behind: the next step starts a new one.
|
|
525
|
+
this.close(step.thread)
|
|
526
|
+
if (this.threads.get(step.key) === step.thread) this.threads.delete(step.key)
|
|
527
|
+
if (this.paused.get(step.key) === step.thread) this.paused.delete(step.key)
|
|
528
|
+
}
|
|
529
|
+
options.logger.errors("afm chatStream failed", { error, source: "afm.chatStream" })
|
|
530
|
+
const message = error instanceof Error ? error.message : String(error)
|
|
531
|
+
const code = aborted ? "aborted" : (error as { code?: unknown })?.code
|
|
532
|
+
yield {
|
|
533
|
+
type: EventType.RUN_ERROR,
|
|
534
|
+
runId,
|
|
535
|
+
model,
|
|
536
|
+
timestamp: timestamp(),
|
|
537
|
+
message,
|
|
538
|
+
error: { message, ...(typeof code === "string" && { code }) }
|
|
539
|
+
}
|
|
540
|
+
} finally {
|
|
541
|
+
if (step) {
|
|
542
|
+
step.thread.busy = false
|
|
543
|
+
// A session no thread keeps ends with its step, unless its response waits for tools.
|
|
544
|
+
if (!step.kept && !paused) this.close(step.thread)
|
|
545
|
+
}
|
|
546
|
+
}
|
|
547
|
+
}
|
|
548
|
+
|
|
549
|
+
/**
|
|
550
|
+
* A JSON answer for `chat({ outputSchema })`: a session holding the
|
|
551
|
+
* conversation answers its last message, held to the schema by the
|
|
552
|
+
* framework. `chat()` validates the result.
|
|
553
|
+
*/
|
|
554
|
+
async structuredOutput(
|
|
555
|
+
options: StructuredOutputOptions<AfmTextModelOptions>
|
|
556
|
+
): Promise<StructuredOutputResult<unknown>> {
|
|
557
|
+
const { chatOptions, outputSchema } = options
|
|
558
|
+
const signal = signalOf(chatOptions)
|
|
559
|
+
signal?.throwIfAborted()
|
|
560
|
+
await this.listen()
|
|
561
|
+
const last = chatOptions.messages.at(-1)
|
|
562
|
+
const prompted = last?.role === "user"
|
|
563
|
+
const before = prompted ? chatOptions.messages.slice(0, -1) : chatOptions.messages
|
|
564
|
+
const thread = await this.open(chatOptions, [], "", toNativeMessages(before))
|
|
565
|
+
try {
|
|
566
|
+
thread.turn = 1
|
|
567
|
+
thread.queue = new EventQueue()
|
|
568
|
+
this.queues.set(thread.sessionId, thread)
|
|
569
|
+
const { temperature, maximumResponseTokens } = chatOptions.modelOptions ?? {}
|
|
570
|
+
await this.native.respond({
|
|
571
|
+
sessionId: thread.sessionId,
|
|
572
|
+
turn: 1,
|
|
573
|
+
prompt: prompted ? contentText(last.content) : "",
|
|
574
|
+
schema: toNativeSchema(outputSchema, "Answer"),
|
|
575
|
+
options: {
|
|
576
|
+
...(temperature !== undefined && { temperature }),
|
|
577
|
+
...(maximumResponseTokens !== undefined && { maximumResponseTokens })
|
|
578
|
+
}
|
|
579
|
+
})
|
|
580
|
+
for (;;) {
|
|
581
|
+
const event = await thread.queue.next(signal)
|
|
582
|
+
if (event.type === "done") {
|
|
583
|
+
const usage = usageOf(event.usage)
|
|
584
|
+
return { data: event.asText ? checkedAnswer(event.text, outputSchema) : (JSON.parse(event.text) as unknown), rawText: event.text, model: this.model, ...(usage && { usage }) }
|
|
585
|
+
}
|
|
586
|
+
if (event.type === "error") throw Object.assign(new Error(event.message), { code: event.code })
|
|
587
|
+
}
|
|
588
|
+
} finally {
|
|
589
|
+
this.close(thread)
|
|
590
|
+
}
|
|
591
|
+
}
|
|
592
|
+
}
|
|
593
|
+
|
|
594
|
+
/**
|
|
595
|
+
* A TanStack AI text adapter for Apple's on-device model, through the
|
|
596
|
+
* Foundation Models framework (the Capacitor plugin on iOS, by default).
|
|
597
|
+
* The model is Apple's system model ("system"), so naming it is optional.
|
|
598
|
+
*
|
|
599
|
+
* @example
|
|
600
|
+
* ```ts
|
|
601
|
+
* const adapter = afmText()
|
|
602
|
+
* if ((await adapter.availability()) === "available") {
|
|
603
|
+
* chat({ adapter, messages, tools })
|
|
604
|
+
* }
|
|
605
|
+
* ```
|
|
606
|
+
*/
|
|
607
|
+
export function afmText(config?: AfmConfig): AfmTextAdapter
|
|
608
|
+
export function afmText(model: AfmChatModel, config?: AfmConfig): AfmTextAdapter
|
|
609
|
+
export function afmText(
|
|
610
|
+
modelOrConfig?: AfmChatModel | AfmConfig,
|
|
611
|
+
config: AfmConfig = {}
|
|
612
|
+
): AfmTextAdapter {
|
|
613
|
+
return typeof modelOrConfig === "string"
|
|
614
|
+
? new AfmTextAdapter(config, modelOrConfig)
|
|
615
|
+
: new AfmTextAdapter(modelOrConfig ?? {}, "system")
|
|
616
|
+
}
|