@tanstack-ai-browser/afm 0.0.0-stage → 0.2.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/Package.swift +33 -0
- package/README.md +75 -2
- package/dist/adapter.d.ts +107 -0
- package/dist/adapter.d.ts.map +1 -0
- package/dist/adapter.js +522 -0
- package/dist/electron.d.ts +29 -0
- package/dist/electron.d.ts.map +1 -0
- package/dist/electron.js +136 -0
- package/dist/index.d.ts +17 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +12 -0
- package/dist/json-value.d.ts +9 -0
- package/dist/json-value.d.ts.map +1 -0
- package/dist/json-value.js +19 -0
- package/dist/messages.d.ts +12 -0
- package/dist/messages.d.ts.map +1 -0
- package/dist/messages.js +29 -0
- package/dist/model-meta.d.ts +23 -0
- package/dist/model-meta.d.ts.map +1 -0
- package/dist/model-meta.js +20 -0
- package/dist/native.d.ts +175 -0
- package/dist/native.d.ts.map +1 -0
- package/dist/native.js +21 -0
- package/dist/schema.d.ts +17 -0
- package/dist/schema.d.ts.map +1 -0
- package/dist/schema.js +133 -0
- package/dist/validate.d.ts +2 -0
- package/dist/validate.d.ts.map +1 -0
- package/dist/validate.js +124 -0
- package/electron/AfmNode.swift +81 -0
- package/electron/afm.c +94 -0
- package/electron/afm.node +0 -0
- package/electron/afm.node.sha256 +1 -0
- package/electron/build.mjs +54 -0
- package/electron/preload.cjs +35 -0
- package/electron/sources-hash.mjs +40 -0
- package/ios/Sources/AfmCore/Bridge.swift +354 -0
- package/ios/Sources/AfmCore/Conversation.swift +49 -0
- package/ios/Sources/AfmCore/Schema.swift +90 -0
- package/ios/Sources/AfmPlugin/AfmPlugin.swift +89 -0
- package/package.json +97 -3
- package/src/adapter.test.ts +428 -0
- package/src/adapter.ts +616 -0
- package/src/electron.ts +193 -0
- package/src/index.ts +28 -0
- package/src/json-value.test.ts +26 -0
- package/src/json-value.ts +17 -0
- package/src/messages.ts +37 -0
- package/src/model-meta.ts +37 -0
- package/src/native-build.test.ts +13 -0
- package/src/native.ts +128 -0
- package/src/schema.test.ts +146 -0
- package/src/schema.ts +153 -0
- package/src/validate.test.ts +38 -0
- package/src/validate.ts +112 -0
package/src/electron.ts
ADDED
|
@@ -0,0 +1,193 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Apple's on-device model in an Electron app on macOS: `serveAfm()` in the
|
|
3
|
+
* main process runs it, through the package's Node-API addon
|
|
4
|
+
* (electron/afm.node, built from the same Swift as the Capacitor plugin), and
|
|
5
|
+
* adds the package's preload (electron/preload.cjs) to the app's pages, which
|
|
6
|
+
* puts it at `window.afm` for `afmText()`.
|
|
7
|
+
*
|
|
8
|
+
* Each frame's sessions are its own: their native ids are the frame's id and
|
|
9
|
+
* the page's, a frame's calls reach only its sessions, and their events go to
|
|
10
|
+
* that frame only, while `allow()` still allows its page. A frame's sessions
|
|
11
|
+
* end when it navigates or closes.
|
|
12
|
+
*
|
|
13
|
+
* Electron is required when `serveAfm()` runs, not imported: bundlers of the
|
|
14
|
+
* app's pages (web, iOS) never see it, and drop this code.
|
|
15
|
+
*/
|
|
16
|
+
import type { App, IpcMain, Session, WebContents, WebFrameMain } from "electron"
|
|
17
|
+
|
|
18
|
+
import type { AfmGuardrails } from "./native.js"
|
|
19
|
+
|
|
20
|
+
const PACKAGE = "@tanstack-ai-browser/afm"
|
|
21
|
+
const CALL = "afm:call"
|
|
22
|
+
const EVENT = "afm:event"
|
|
23
|
+
|
|
24
|
+
/** The addon (electron/afm.c): AfmBridge's calls, and its events, in JSON. */
|
|
25
|
+
type Addon = {
|
|
26
|
+
call(method: string, options: string): string
|
|
27
|
+
setListener(listener: ((json: string) => void) | null): void
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
type Electron = {
|
|
31
|
+
app: App
|
|
32
|
+
ipcMain: IpcMain
|
|
33
|
+
session: { defaultSession: Session }
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
/** A native session's frame, and the page's id for it. */
|
|
37
|
+
type Owner = { frame: WebFrameMain; frameId: string; contentsId: number; sessionId: string }
|
|
38
|
+
|
|
39
|
+
/** The bits of Node used here, typed here: this is a browser package otherwise. */
|
|
40
|
+
type NodeRequire = ((id: string) => unknown) & { resolve(id: string): string }
|
|
41
|
+
type NodeProcess = {
|
|
42
|
+
platform: string
|
|
43
|
+
execPath: string
|
|
44
|
+
getBuiltinModule(id: "node:module"): { createRequire(path: string): NodeRequire }
|
|
45
|
+
getBuiltinModule(id: "node:path"): { join(...parts: Array<string>): string; dirname(path: string): string }
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
export type ServeAfmOptions = {
|
|
49
|
+
/**
|
|
50
|
+
* The folder with the package's Electron files (afm.node and preload.cjs):
|
|
51
|
+
* by default the package's own `electron` folder, found from the app's
|
|
52
|
+
* node_modules. An app that leaves node_modules out of its package copies
|
|
53
|
+
* the two files in and passes their folder. Inside an asar archive, the
|
|
54
|
+
* unpacked afm.node beside it is loaded (native code can't load from an archive).
|
|
55
|
+
*/
|
|
56
|
+
directory?: string
|
|
57
|
+
/**
|
|
58
|
+
* Which pages may use the model, by URL. By default the app's own: any page
|
|
59
|
+
* but an http(s) one on another host than localhost.
|
|
60
|
+
*/
|
|
61
|
+
allow?: (url: string) => boolean
|
|
62
|
+
/**
|
|
63
|
+
* Apple's guardrails for every page's sessions that don't name their own
|
|
64
|
+
* (`afmText({ guardrails })` or a request's `modelOptions.guardrails`):
|
|
65
|
+
* "default" (the default), or "permissiveContentTransformations".
|
|
66
|
+
*/
|
|
67
|
+
guardrails?: AfmGuardrails
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
const appPages = (url: string) => {
|
|
71
|
+
try {
|
|
72
|
+
const { protocol, hostname } = new URL(url)
|
|
73
|
+
if (protocol !== "http:" && protocol !== "https:") return true
|
|
74
|
+
return hostname === "localhost" || hostname === "127.0.0.1" || hostname === "[::1]"
|
|
75
|
+
} catch {
|
|
76
|
+
return false
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
const unpacked = (file: string) => file.replace(/app\.asar(?=[\\/])/, "app.asar.unpacked")
|
|
81
|
+
|
|
82
|
+
let served = false
|
|
83
|
+
|
|
84
|
+
/**
|
|
85
|
+
* Runs Apple's on-device model for the app's pages, where `afmText()` finds it.
|
|
86
|
+
* Call it once in the main process, before opening windows. On a Mac before
|
|
87
|
+
* macOS 26, or elsewhere, the pages hear the model is unavailable.
|
|
88
|
+
*/
|
|
89
|
+
export function serveAfm(options: ServeAfmOptions = {}): void {
|
|
90
|
+
const appGuardrails = options.guardrails
|
|
91
|
+
if (served) return
|
|
92
|
+
served = true
|
|
93
|
+
const node = (globalThis as unknown as { process: NodeProcess }).process
|
|
94
|
+
const { createRequire } = node.getBuiltinModule("node:module")
|
|
95
|
+
const path = node.getBuiltinModule("node:path")
|
|
96
|
+
// Electron answers `require("electron")` from anywhere in the main process.
|
|
97
|
+
const electron = createRequire(node.execPath)("electron") as Electron
|
|
98
|
+
const { app } = electron
|
|
99
|
+
const allow = options.allow ?? appPages
|
|
100
|
+
|
|
101
|
+
let directory = options.directory
|
|
102
|
+
if (!directory) {
|
|
103
|
+
try {
|
|
104
|
+
const fromApp = createRequire(path.join(app.getAppPath(), "package.json"))
|
|
105
|
+
directory = path.join(path.dirname(fromApp.resolve(`${PACKAGE}/package.json`)), "electron")
|
|
106
|
+
} catch (error) {
|
|
107
|
+
console.error(`Apple's on-device model is unavailable: ${PACKAGE} isn't in the app's node_modules (pass serveAfm's directory)`, error)
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
let addon: Addon | undefined
|
|
112
|
+
if (directory && node.platform === "darwin") {
|
|
113
|
+
try {
|
|
114
|
+
addon = createRequire(node.execPath)(unpacked(path.join(directory, "afm.node"))) as Addon
|
|
115
|
+
} catch (error) {
|
|
116
|
+
console.error("Apple's on-device model is unavailable: its addon didn't load", error)
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
// The native sessions, by native id: `${frame id}/${the page's id}`.
|
|
121
|
+
const owners = new Map<string, Owner>()
|
|
122
|
+
const frameIdOf = (frame: WebFrameMain) => `${frame.processId}:${frame.frameToken}`
|
|
123
|
+
const endSessions = (ended: (owner: Owner) => boolean) => {
|
|
124
|
+
for (const [id, owner] of owners) {
|
|
125
|
+
if (!ended(owner)) continue
|
|
126
|
+
owners.delete(id)
|
|
127
|
+
addon?.call("deleteSession", JSON.stringify({ sessionId: id }))
|
|
128
|
+
}
|
|
129
|
+
}
|
|
130
|
+
// A page's sessions end with its document: a navigation, or its contents closing.
|
|
131
|
+
const watched = new WeakSet<WebContents>()
|
|
132
|
+
const watch = (contents: WebContents) => {
|
|
133
|
+
if (watched.has(contents)) return
|
|
134
|
+
watched.add(contents)
|
|
135
|
+
const contentsId = contents.id
|
|
136
|
+
contents.on("did-start-navigation", ({ isSameDocument, frame }) => {
|
|
137
|
+
if (isSameDocument || !frame) return
|
|
138
|
+
const frameId = frameIdOf(frame)
|
|
139
|
+
endSessions((owner) => owner.frameId === frameId)
|
|
140
|
+
})
|
|
141
|
+
contents.once("destroyed", () => endSessions((owner) => owner.contentsId === contentsId))
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
electron.ipcMain.handle(CALL, (event, method: string, args: unknown) => {
|
|
145
|
+
const frame = event.senderFrame
|
|
146
|
+
if (!frame || !allow(frame.url)) throw new Error("This page may not use Apple's on-device model")
|
|
147
|
+
if (!addon) {
|
|
148
|
+
if (method === "availability") return { available: false, reason: "unsupportedOS" }
|
|
149
|
+
throw new Error("Apple's on-device model needs macOS 26 on a Mac with Apple silicon")
|
|
150
|
+
}
|
|
151
|
+
const options = { ...(typeof args === "object" && args !== null ? args : {}) } as Record<string, unknown>
|
|
152
|
+
// The app's guardrails, where the page names none.
|
|
153
|
+
if ((method === "createSession" || method === "availability") && options.guardrails === undefined && appGuardrails) {
|
|
154
|
+
options.guardrails = appGuardrails
|
|
155
|
+
}
|
|
156
|
+
// The frame's own session: another frame's can't be reached.
|
|
157
|
+
if (typeof options.sessionId === "string") {
|
|
158
|
+
const frameId = frameIdOf(frame)
|
|
159
|
+
const id = `${frameId}/${options.sessionId}`
|
|
160
|
+
if (method === "createSession") {
|
|
161
|
+
owners.set(id, { frame, frameId, contentsId: event.sender.id, sessionId: options.sessionId })
|
|
162
|
+
watch(event.sender)
|
|
163
|
+
} else if (method === "deleteSession") {
|
|
164
|
+
owners.delete(id)
|
|
165
|
+
}
|
|
166
|
+
options.sessionId = id
|
|
167
|
+
}
|
|
168
|
+
const reply = JSON.parse(addon.call(method, JSON.stringify(options))) as { result?: unknown; error?: string }
|
|
169
|
+
if (reply.error) throw new Error(reply.error)
|
|
170
|
+
return reply.result ?? null
|
|
171
|
+
})
|
|
172
|
+
// Each event to its session's frame only, with the page's id, while its page is allowed.
|
|
173
|
+
addon?.setListener((json) => {
|
|
174
|
+
const event = JSON.parse(json) as { sessionId?: unknown }
|
|
175
|
+
const owner = typeof event.sessionId === "string" ? owners.get(event.sessionId) : undefined
|
|
176
|
+
if (!owner || owner.frame.isDestroyed() || owner.frame.detached || !allow(owner.frame.url)) return
|
|
177
|
+
owner.frame.send(EVENT, { ...event, sessionId: owner.sessionId })
|
|
178
|
+
})
|
|
179
|
+
|
|
180
|
+
// The preload, in every session's pages: window.afm.
|
|
181
|
+
if (!directory) return
|
|
182
|
+
const preload = path.join(directory, "preload.cjs")
|
|
183
|
+
const preloaded = new WeakSet<Session>()
|
|
184
|
+
const addPreload = (session: Session) => {
|
|
185
|
+
if (preloaded.has(session)) return
|
|
186
|
+
preloaded.add(session)
|
|
187
|
+
session.registerPreloadScript({ type: "frame", id: "tanstack-ai-browser-afm", filePath: preload })
|
|
188
|
+
}
|
|
189
|
+
// Now if the app is ready, so a window opened next has it.
|
|
190
|
+
if (app.isReady()) addPreload(electron.session.defaultSession)
|
|
191
|
+
else void app.whenReady().then(() => addPreload(electron.session.defaultSession))
|
|
192
|
+
app.on("session-created", addPreload)
|
|
193
|
+
}
|
package/src/index.ts
ADDED
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* TanStack AI adapter for Apple's on-device Foundation Model, through the
|
|
3
|
+
* Foundation Models framework, with its native tool calling: a Capacitor
|
|
4
|
+
* plugin for iOS (`Afm`), Electron's main process on macOS (`serveAfm()`), or
|
|
5
|
+
* any host implementing `AfmNative`. The model catalog (model-meta.ts) is
|
|
6
|
+
* exported here too.
|
|
7
|
+
*/
|
|
8
|
+
export { AfmTextAdapter, afmText } from "./adapter.js"
|
|
9
|
+
export type { Availability, AfmConfig, AfmTextModelOptions } from "./adapter.js"
|
|
10
|
+
export { Afm } from "./native.js"
|
|
11
|
+
export { serveAfm } from "./electron.js"
|
|
12
|
+
export type { ServeAfmOptions } from "./electron.js"
|
|
13
|
+
export type {
|
|
14
|
+
AfmGuardrails,
|
|
15
|
+
AfmNative,
|
|
16
|
+
NativeAvailability,
|
|
17
|
+
NativeEvent,
|
|
18
|
+
NativeGenerationSchema,
|
|
19
|
+
NativeMessage,
|
|
20
|
+
NativeProperty,
|
|
21
|
+
NativeSchema,
|
|
22
|
+
NativeTool,
|
|
23
|
+
NativeUsage,
|
|
24
|
+
UnavailableReason
|
|
25
|
+
} from "./native.js"
|
|
26
|
+
export { toNativeSchema } from "./schema.js"
|
|
27
|
+
export { AFM_CHAT_MODELS, AFM_MODEL_META } from "./model-meta.js"
|
|
28
|
+
export type { AfmChatModel, AfmModelMeta } from "./model-meta.js"
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
import assert from "node:assert/strict"
|
|
2
|
+
import { test } from "node:test"
|
|
3
|
+
|
|
4
|
+
import { parseJsonValue } from "./json-value.js"
|
|
5
|
+
|
|
6
|
+
test("a text answer's JSON value comes whole: arrays, primitives, and strings with braces in them", () => {
|
|
7
|
+
assert.deepEqual(parseJsonValue('[{"name":"Searing Blood"}]'), [{ name: "Searing Blood" }])
|
|
8
|
+
assert.deepEqual(parseJsonValue(' [{"a":1},{"b":2}]\n'), [{ a: 1 }, { b: 2 }])
|
|
9
|
+
assert.equal(parseJsonValue("42"), 42)
|
|
10
|
+
assert.equal(parseJsonValue("true"), true)
|
|
11
|
+
assert.equal(parseJsonValue("null"), null)
|
|
12
|
+
assert.equal(parseJsonValue('"a {b} c"'), "a {b} c")
|
|
13
|
+
assert.deepEqual(parseJsonValue('{"text":"} {"}'), { text: "} {" })
|
|
14
|
+
})
|
|
15
|
+
|
|
16
|
+
test("a fenced answer gives everything inside its fence", () => {
|
|
17
|
+
assert.deepEqual(parseJsonValue('```json\n[{"a":1},{"b":2}]\n```'), [{ a: 1 }, { b: 2 }])
|
|
18
|
+
assert.equal(parseJsonValue('```\n"a {b} c"\n```'), "a {b} c")
|
|
19
|
+
assert.deepEqual(parseJsonValue('\n```json\n{\n "x": [1, 2]\n}\n```\n'), { x: [1, 2] })
|
|
20
|
+
})
|
|
21
|
+
|
|
22
|
+
test("text that isn't a JSON value, or more than one, fails", () => {
|
|
23
|
+
assert.throws(() => parseJsonValue("Here you go: {\"a\":1}"))
|
|
24
|
+
assert.throws(() => parseJsonValue('{"a":1} {"b":2}'))
|
|
25
|
+
assert.throws(() => parseJsonValue("```json\nnot json\n```"))
|
|
26
|
+
})
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The JSON value in a model's text answer: the whole trimmed text, or, when the
|
|
3
|
+
* model wrapped it in a Markdown code fence, everything inside the fence. Any
|
|
4
|
+
* value: an object, an array, a string, a number, a boolean or null. Never by
|
|
5
|
+
* brace positions, which would cut an array down to one of its objects, or a
|
|
6
|
+
* string that contains braces out of its quotes.
|
|
7
|
+
*/
|
|
8
|
+
export function parseJsonValue(text: string): unknown {
|
|
9
|
+
const trimmed = text.trim()
|
|
10
|
+
try {
|
|
11
|
+
return JSON.parse(trimmed)
|
|
12
|
+
} catch (error) {
|
|
13
|
+
const fenced = /^```[\w-]*[^\S\n]*\n([\s\S]*?)\n?```$/.exec(trimmed)
|
|
14
|
+
if (fenced) return JSON.parse(fenced[1].trim())
|
|
15
|
+
throw error
|
|
16
|
+
}
|
|
17
|
+
}
|
package/src/messages.ts
ADDED
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* TanStack AI's messages as a native session's conversation (`NativeMessage`):
|
|
3
|
+
* text, the assistant's tool calls, and the tools' results. The framework
|
|
4
|
+
* reads text only here, so other parts are left out.
|
|
5
|
+
*/
|
|
6
|
+
import type { ModelMessage } from "@tanstack/ai"
|
|
7
|
+
|
|
8
|
+
import type { NativeMessage } from "./native.js"
|
|
9
|
+
|
|
10
|
+
/** A message's text: its string, or its text parts joined. */
|
|
11
|
+
export function contentText(content: ModelMessage["content"]): string {
|
|
12
|
+
if (typeof content === "string") return content
|
|
13
|
+
if (!Array.isArray(content)) return ""
|
|
14
|
+
return content
|
|
15
|
+
.map((part) => (part.type === "text" ? part.content : ""))
|
|
16
|
+
.filter(Boolean)
|
|
17
|
+
.join("\n")
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
/** The messages as a session's conversation, each tool result named by its call's tool. */
|
|
21
|
+
export function toNativeMessages(messages: ReadonlyArray<ModelMessage>): Array<NativeMessage> {
|
|
22
|
+
const toolNames = new Map<string, string>()
|
|
23
|
+
return messages.map((m): NativeMessage => {
|
|
24
|
+
if (m.role === "tool") {
|
|
25
|
+
const id = m.toolCallId ?? ""
|
|
26
|
+
return { role: "tool", toolCallId: id, name: m.name ?? toolNames.get(id) ?? "", text: contentText(m.content) }
|
|
27
|
+
}
|
|
28
|
+
if (m.role === "assistant") {
|
|
29
|
+
const toolCalls = (m.toolCalls ?? []).map((call) => {
|
|
30
|
+
toolNames.set(call.id, call.function.name)
|
|
31
|
+
return { id: call.id, name: call.function.name, arguments: call.function.arguments || "{}" }
|
|
32
|
+
})
|
|
33
|
+
return { role: "assistant", text: contentText(m.content), ...(toolCalls.length > 0 && { toolCalls }) }
|
|
34
|
+
}
|
|
35
|
+
return { role: "user", text: contentText(m.content) }
|
|
36
|
+
})
|
|
37
|
+
}
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The model the framework runs: Apple's system language model
|
|
3
|
+
* (`SystemLanguageModel.default`), the on-device model of Apple Intelligence.
|
|
4
|
+
* It calls tools natively, and holds structured answers to a schema. It doesn't
|
|
5
|
+
* reason, so `chat()` offers no `reasoning` option for it.
|
|
6
|
+
*/
|
|
7
|
+
|
|
8
|
+
/** What a Foundation Models model supports. */
|
|
9
|
+
export type AfmModelMeta = {
|
|
10
|
+
/** The model's name here. */
|
|
11
|
+
name: string
|
|
12
|
+
supports: {
|
|
13
|
+
input: ReadonlyArray<"text">
|
|
14
|
+
output: ReadonlyArray<"text">
|
|
15
|
+
capabilities: ReadonlyArray<"tools" | "structured-output">
|
|
16
|
+
}
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
const SYSTEM_MODEL = {
|
|
20
|
+
name: "system",
|
|
21
|
+
supports: {
|
|
22
|
+
input: ["text"],
|
|
23
|
+
output: ["text"],
|
|
24
|
+
capabilities: ["tools", "structured-output"]
|
|
25
|
+
}
|
|
26
|
+
} as const satisfies AfmModelMeta
|
|
27
|
+
|
|
28
|
+
/** The models the framework runs: Apple's system language model. */
|
|
29
|
+
export const AFM_CHAT_MODELS = [SYSTEM_MODEL.name] as const
|
|
30
|
+
|
|
31
|
+
/** A model the framework runs. */
|
|
32
|
+
export type AfmChatModel = (typeof AFM_CHAT_MODELS)[number]
|
|
33
|
+
|
|
34
|
+
/** Each model's metadata, by name. */
|
|
35
|
+
export const AFM_MODEL_META: Readonly<Record<AfmChatModel, AfmModelMeta>> = {
|
|
36
|
+
[SYSTEM_MODEL.name]: SYSTEM_MODEL
|
|
37
|
+
}
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
import assert from "node:assert/strict"
|
|
2
|
+
import { readFileSync } from "node:fs"
|
|
3
|
+
import { test } from "node:test"
|
|
4
|
+
|
|
5
|
+
import { hashFile, sourcesHash } from "../electron/sources-hash.mjs"
|
|
6
|
+
|
|
7
|
+
test("the committed Electron addon was built from the current Swift and C sources", () => {
|
|
8
|
+
assert.equal(
|
|
9
|
+
readFileSync(hashFile, "utf8").trim(),
|
|
10
|
+
sourcesHash(),
|
|
11
|
+
"electron/afm.node is stale: rebuild it (node electron/build.mjs, on a Mac with Xcode) and commit it with electron/afm.node.sha256"
|
|
12
|
+
)
|
|
13
|
+
})
|
package/src/native.ts
ADDED
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The native side of the adapter: Apple's Foundation Models framework, reached
|
|
3
|
+
* through the Capacitor plugin on iOS (ios/Sources), or the Electron main
|
|
4
|
+
* process (`serveAfm()`, whose preload puts it at `window.afm`). Any host can
|
|
5
|
+
* stand in for them by implementing `AfmNative`.
|
|
6
|
+
*/
|
|
7
|
+
import type { PluginListenerHandle } from "@capacitor/core"
|
|
8
|
+
|
|
9
|
+
/** Why the model can't run: Apple's reasons, or an OS before 26. */
|
|
10
|
+
export type UnavailableReason =
|
|
11
|
+
| "deviceNotEligible"
|
|
12
|
+
| "appleIntelligenceNotEnabled"
|
|
13
|
+
| "modelNotReady"
|
|
14
|
+
| "unsupportedOS"
|
|
15
|
+
| "unavailable"
|
|
16
|
+
|
|
17
|
+
export type NativeAvailability = {
|
|
18
|
+
available: boolean
|
|
19
|
+
reason?: UnavailableReason
|
|
20
|
+
/** Tokens a session holds (iOS/macOS 26.4 and later). */
|
|
21
|
+
contextSize?: number
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
/**
|
|
25
|
+
* A schema as the framework takes it: JSON Schema read into an ordered form
|
|
26
|
+
* (`schema.ts`), properties in the order given, and named definitions for
|
|
27
|
+
* `reference`s.
|
|
28
|
+
*/
|
|
29
|
+
export type NativeSchema =
|
|
30
|
+
| { kind: "object"; name: string; description?: string; properties: Array<NativeProperty> }
|
|
31
|
+
| { kind: "array"; items: NativeSchema; minimumElements?: number; maximumElements?: number }
|
|
32
|
+
| { kind: "string"; choices?: Array<string>; constant?: string }
|
|
33
|
+
| { kind: "integer" | "number"; minimum?: number; maximum?: number }
|
|
34
|
+
| { kind: "boolean" }
|
|
35
|
+
| { kind: "null" }
|
|
36
|
+
| { kind: "anyOf"; name: string; description?: string; choices: Array<NativeSchema> }
|
|
37
|
+
| { kind: "reference"; name: string }
|
|
38
|
+
|
|
39
|
+
export type NativeProperty = { name: string; description?: string; optional: boolean; schema: NativeSchema }
|
|
40
|
+
|
|
41
|
+
/** A root schema and the definitions its references name. */
|
|
42
|
+
export type NativeGenerationSchema = { root: NativeSchema; definitions: Array<NativeSchema> }
|
|
43
|
+
|
|
44
|
+
export type NativeTool = { name: string; description: string; parameters: NativeGenerationSchema }
|
|
45
|
+
|
|
46
|
+
/** A message of the conversation a session starts from. */
|
|
47
|
+
export type NativeMessage =
|
|
48
|
+
| { role: "user"; text: string }
|
|
49
|
+
| { role: "assistant"; text: string; toolCalls?: Array<{ id: string; name: string; arguments: string }> }
|
|
50
|
+
| { role: "tool"; toolCallId: string; name: string; text: string }
|
|
51
|
+
|
|
52
|
+
/**
|
|
53
|
+
* A response's tokens, over every pass of the model (a tool call's too).
|
|
54
|
+
* `cachedTokens` is there only where the framework counts it right: for a
|
|
55
|
+
* session without tools.
|
|
56
|
+
*/
|
|
57
|
+
export type NativeUsage = { inputTokens: number; cachedTokens?: number; outputTokens: number }
|
|
58
|
+
|
|
59
|
+
/** What a session sends while it responds, for the `turn` of the `respond` call. */
|
|
60
|
+
export type NativeEvent = { sessionId: string; turn: number } & (
|
|
61
|
+
| { type: "delta"; text: string }
|
|
62
|
+
| { type: "toolCalls"; calls: Array<{ id: string; name: string; arguments: string }> }
|
|
63
|
+
/** `asText`: a schema's answer written as text (permissive guardrails' fallback), not held to the schema. */
|
|
64
|
+
| { type: "done"; text: string; usage?: NativeUsage; asText?: boolean }
|
|
65
|
+
| { type: "error"; code: string; message: string }
|
|
66
|
+
)
|
|
67
|
+
|
|
68
|
+
/**
|
|
69
|
+
* Apple's guardrails for the model: "default", or "permissiveContentTransformations",
|
|
70
|
+
* which lets through text the default ones can block out of context (a card game's
|
|
71
|
+
* "Searing Blood" or "Play with Fire").
|
|
72
|
+
*/
|
|
73
|
+
export type AfmGuardrails = "default" | "permissiveContentTransformations"
|
|
74
|
+
|
|
75
|
+
export interface AfmNative {
|
|
76
|
+
availability(options?: { guardrails?: AfmGuardrails }): Promise<NativeAvailability>
|
|
77
|
+
createSession(options: {
|
|
78
|
+
sessionId: string
|
|
79
|
+
instructions?: string
|
|
80
|
+
tools: Array<NativeTool>
|
|
81
|
+
messages: Array<NativeMessage>
|
|
82
|
+
guardrails?: AfmGuardrails
|
|
83
|
+
}): Promise<void>
|
|
84
|
+
prewarm(options: { sessionId: string }): Promise<void>
|
|
85
|
+
/** Prompts a session; the response comes as events. A `schema` holds the answer to it (JSON in `done`). */
|
|
86
|
+
respond(options: {
|
|
87
|
+
sessionId: string
|
|
88
|
+
turn: number
|
|
89
|
+
prompt: string
|
|
90
|
+
options?: {
|
|
91
|
+
temperature?: number
|
|
92
|
+
maximumResponseTokens?: number
|
|
93
|
+
/** Whether the model may call the session's tools, must, or mustn't (iOS/macOS 27). Left out, the framework decides. */
|
|
94
|
+
toolCallingMode?: "allowed" | "required" | "disallowed"
|
|
95
|
+
}
|
|
96
|
+
schema?: NativeGenerationSchema
|
|
97
|
+
}): Promise<void>
|
|
98
|
+
/** A tool's result: the session's waiting call returns it, and the response goes on. */
|
|
99
|
+
submitToolResult(options: { sessionId: string; callId: string; output: string }): Promise<void>
|
|
100
|
+
/** Stops a session's response. */
|
|
101
|
+
cancel(options: { sessionId: string }): Promise<void>
|
|
102
|
+
deleteSession(options: { sessionId: string }): Promise<void>
|
|
103
|
+
addListener(eventName: "event", listener: (event: NativeEvent) => void): Promise<PluginListenerHandle>
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
let loading: Promise<{ plugin: AfmNative }> | undefined
|
|
107
|
+
/**
|
|
108
|
+
* The plugin, loading Capacitor on first use: pages elsewhere never load it.
|
|
109
|
+
* It's wrapped: a Capacitor plugin answers every property, `then` too, so a
|
|
110
|
+
* promise of the plugin itself would wait on it forever.
|
|
111
|
+
*/
|
|
112
|
+
const capacitor = () =>
|
|
113
|
+
(loading ??= import("@capacitor/core").then(({ registerPlugin }) => ({ plugin: registerPlugin<AfmNative>("Afm") })))
|
|
114
|
+
|
|
115
|
+
/**
|
|
116
|
+
* The Capacitor plugin (iOS), loading Capacitor on its first call. Elsewhere
|
|
117
|
+
* its calls reject: Capacitor has no web version of it.
|
|
118
|
+
*/
|
|
119
|
+
export const Afm: AfmNative = {
|
|
120
|
+
availability: async (options) => (await capacitor()).plugin.availability(options),
|
|
121
|
+
createSession: async (options) => (await capacitor()).plugin.createSession(options),
|
|
122
|
+
prewarm: async (options) => (await capacitor()).plugin.prewarm(options),
|
|
123
|
+
respond: async (options) => (await capacitor()).plugin.respond(options),
|
|
124
|
+
submitToolResult: async (options) => (await capacitor()).plugin.submitToolResult(options),
|
|
125
|
+
cancel: async (options) => (await capacitor()).plugin.cancel(options),
|
|
126
|
+
deleteSession: async (options) => (await capacitor()).plugin.deleteSession(options),
|
|
127
|
+
addListener: async (name, listener) => (await capacitor()).plugin.addListener(name, listener)
|
|
128
|
+
}
|
|
@@ -0,0 +1,146 @@
|
|
|
1
|
+
import assert from "node:assert/strict"
|
|
2
|
+
import { test } from "node:test"
|
|
3
|
+
|
|
4
|
+
import { toNativeSchema } from "./schema.js"
|
|
5
|
+
|
|
6
|
+
test("an object keeps its properties' order; required, nullable and descriptions carry over", () => {
|
|
7
|
+
const schema = toNativeSchema(
|
|
8
|
+
{
|
|
9
|
+
type: "object",
|
|
10
|
+
description: "Where",
|
|
11
|
+
properties: {
|
|
12
|
+
city: { type: "string", description: "The city" },
|
|
13
|
+
units: { type: "string", enum: ["celsius", "fahrenheit"] },
|
|
14
|
+
days: { type: ["integer", "null"], minimum: 1, maximum: 7 },
|
|
15
|
+
hourly: { type: "boolean" }
|
|
16
|
+
},
|
|
17
|
+
required: ["city", "days"]
|
|
18
|
+
},
|
|
19
|
+
"get_weather"
|
|
20
|
+
)
|
|
21
|
+
assert.deepEqual(schema, {
|
|
22
|
+
root: {
|
|
23
|
+
kind: "object",
|
|
24
|
+
name: "get_weather",
|
|
25
|
+
description: "Where",
|
|
26
|
+
properties: [
|
|
27
|
+
{ name: "city", description: "The city", optional: false, schema: { kind: "string" } },
|
|
28
|
+
{ name: "units", optional: true, schema: { kind: "string", choices: ["celsius", "fahrenheit"] } },
|
|
29
|
+
{ name: "days", optional: true, schema: { kind: "integer", minimum: 1, maximum: 7 } },
|
|
30
|
+
{ name: "hourly", optional: true, schema: { kind: "boolean" } }
|
|
31
|
+
]
|
|
32
|
+
},
|
|
33
|
+
definitions: []
|
|
34
|
+
})
|
|
35
|
+
})
|
|
36
|
+
|
|
37
|
+
test("nested objects and arrays are named by their path", () => {
|
|
38
|
+
const { root } = toNativeSchema(
|
|
39
|
+
{
|
|
40
|
+
type: "object",
|
|
41
|
+
properties: {
|
|
42
|
+
stops: { type: "array", minItems: 1, maxItems: 3, items: { type: "object", properties: { name: { type: "string" } } } }
|
|
43
|
+
}
|
|
44
|
+
},
|
|
45
|
+
"plan-trip"
|
|
46
|
+
)
|
|
47
|
+
assert.deepEqual(root, {
|
|
48
|
+
kind: "object",
|
|
49
|
+
name: "plan_trip",
|
|
50
|
+
properties: [
|
|
51
|
+
{
|
|
52
|
+
name: "stops",
|
|
53
|
+
optional: true,
|
|
54
|
+
schema: {
|
|
55
|
+
kind: "array",
|
|
56
|
+
minimumElements: 1,
|
|
57
|
+
maximumElements: 3,
|
|
58
|
+
items: { kind: "object", name: "plan_trip_stops_item", properties: [{ name: "name", optional: true, schema: { kind: "string" } }] }
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
]
|
|
62
|
+
})
|
|
63
|
+
})
|
|
64
|
+
|
|
65
|
+
test("anyOf becomes a choice (null left out); one non-null choice is the schema itself; const is a constant", () => {
|
|
66
|
+
const { root } = toNativeSchema(
|
|
67
|
+
{
|
|
68
|
+
type: "object",
|
|
69
|
+
properties: {
|
|
70
|
+
target: { anyOf: [{ type: "string" }, { type: "number" }] },
|
|
71
|
+
note: { anyOf: [{ type: "string" }, { type: "null" }] },
|
|
72
|
+
kind: { const: "search" }
|
|
73
|
+
}
|
|
74
|
+
},
|
|
75
|
+
"tool"
|
|
76
|
+
)
|
|
77
|
+
assert.equal(root.kind, "object")
|
|
78
|
+
const [target, note, kind] = root.kind === "object" ? root.properties : []
|
|
79
|
+
assert.deepEqual(target.schema, { kind: "anyOf", name: "tool_target", choices: [{ kind: "string" }, { kind: "number" }] })
|
|
80
|
+
assert.deepEqual(note, { name: "note", optional: true, schema: { kind: "string" } })
|
|
81
|
+
assert.deepEqual(kind.schema, { kind: "string", constant: "search" })
|
|
82
|
+
})
|
|
83
|
+
|
|
84
|
+
test("$refs to object definitions are references; to plain values, written in place", () => {
|
|
85
|
+
const schema = toNativeSchema(
|
|
86
|
+
{
|
|
87
|
+
type: "object",
|
|
88
|
+
properties: { home: { $ref: "#/$defs/Place" }, mood: { $ref: "#/$defs/Mood" } },
|
|
89
|
+
$defs: {
|
|
90
|
+
Place: { type: "object", properties: { city: { type: "string" } } },
|
|
91
|
+
Mood: { type: "string", enum: ["happy", "sad"] }
|
|
92
|
+
}
|
|
93
|
+
},
|
|
94
|
+
"profile"
|
|
95
|
+
)
|
|
96
|
+
assert.deepEqual(schema.root.kind === "object" && schema.root.properties.map((p) => p.schema), [
|
|
97
|
+
{ kind: "reference", name: "Place" },
|
|
98
|
+
{ kind: "string", choices: ["happy", "sad"] }
|
|
99
|
+
])
|
|
100
|
+
assert.deepEqual(schema.definitions, [
|
|
101
|
+
{ kind: "object", name: "Place", properties: [{ name: "city", optional: true, schema: { kind: "string" } }] }
|
|
102
|
+
])
|
|
103
|
+
})
|
|
104
|
+
|
|
105
|
+
test("enums hold to their exact members: strings as choices, integers (with strings) as a choice of exact values; what the framework can't hold is refused", () => {
|
|
106
|
+
const { root } = toNativeSchema(
|
|
107
|
+
{
|
|
108
|
+
type: "object",
|
|
109
|
+
properties: {
|
|
110
|
+
unit: { type: "string", enum: ["c", "f"] },
|
|
111
|
+
stars: { type: "integer", enum: [1, 10], description: "A rating." },
|
|
112
|
+
size: { enum: ["auto", 8] },
|
|
113
|
+
on: { type: "boolean", enum: [true, false] }
|
|
114
|
+
},
|
|
115
|
+
required: ["unit", "stars", "size", "on"]
|
|
116
|
+
},
|
|
117
|
+
"t"
|
|
118
|
+
)
|
|
119
|
+
assert.equal(root.kind, "object")
|
|
120
|
+
const property = (name: string) => (root.kind === "object" ? root.properties.find((p) => p.name === name) : undefined)
|
|
121
|
+
assert.deepEqual(property("unit")?.schema, { kind: "string", choices: ["c", "f"] })
|
|
122
|
+
assert.deepEqual(property("stars")?.schema, {
|
|
123
|
+
kind: "anyOf",
|
|
124
|
+
name: "t_stars",
|
|
125
|
+
description: "A rating.",
|
|
126
|
+
choices: [
|
|
127
|
+
{ kind: "integer", minimum: 1, maximum: 1 },
|
|
128
|
+
{ kind: "integer", minimum: 10, maximum: 10 }
|
|
129
|
+
]
|
|
130
|
+
})
|
|
131
|
+
assert.equal(property("stars")?.description, "A rating.")
|
|
132
|
+
assert.deepEqual(property("size")?.schema, {
|
|
133
|
+
kind: "anyOf",
|
|
134
|
+
name: "t_size",
|
|
135
|
+
choices: [
|
|
136
|
+
{ kind: "string", constant: "auto" },
|
|
137
|
+
{ kind: "integer", minimum: 8, maximum: 8 }
|
|
138
|
+
]
|
|
139
|
+
})
|
|
140
|
+
assert.deepEqual(property("on")?.schema, { kind: "boolean" })
|
|
141
|
+
// One integer is that integer.
|
|
142
|
+
assert.deepEqual(toNativeSchema({ type: "integer", enum: [3] }, "n").root, { kind: "integer", minimum: 3, maximum: 3 })
|
|
143
|
+
assert.throws(() => toNativeSchema({ type: "boolean", enum: [true] }, "flag"), /enum of only true isn't supported/)
|
|
144
|
+
assert.throws(() => toNativeSchema({ type: "number", enum: [0.5, 1.5] }, "ratio"), /decimal numbers isn't supported/)
|
|
145
|
+
assert.throws(() => toNativeSchema({ enum: ["a", true] }, "mixed"), /mixing string and boolean/)
|
|
146
|
+
})
|