expo-local-llm 0.1.2 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +91 -5
- package/android/src/main/java/expo/modules/localllm/Exceptions.kt +3 -0
- package/android/src/main/java/expo/modules/localllm/ExpoLocalLlmModule.kt +21 -0
- package/android/src/main/java/expo/modules/localllm/Records.kt +23 -0
- package/build/ExpoLocalLlm.types.d.ts +32 -0
- package/build/ExpoLocalLlm.types.d.ts.map +1 -1
- package/build/ExpoLocalLlm.types.js.map +1 -1
- package/build/LLMSession.d.ts +5 -1
- package/build/LLMSession.d.ts.map +1 -1
- package/build/LLMSession.js +10 -1
- package/build/LLMSession.js.map +1 -1
- package/build/useLocalLLM.d.ts +2 -1
- package/build/useLocalLLM.d.ts.map +1 -1
- package/build/useLocalLLM.js +81 -3
- package/build/useLocalLLM.js.map +1 -1
- package/ios/DynamicTool.swift +154 -0
- package/ios/Exceptions.swift +24 -0
- package/ios/ExpoLocalLlmModule.swift +18 -0
- package/ios/FoundationModelBridge.swift +12 -5
- package/ios/LLMSession.swift +142 -7
- package/ios/Records.swift +16 -0
- package/package.json +1 -1
- package/src/ExpoLocalLlm.types.ts +43 -0
- package/src/LLMSession.ts +16 -2
- package/src/useLocalLLM.ts +112 -3
package/README.md
CHANGED
|
@@ -6,7 +6,7 @@ Expo module for on-device LLM inference. Wraps Apple Foundation Models (iOS 26+)
|
|
|
6
6
|
|
|
7
7
|
| Platform | Requirement |
|
|
8
8
|
|----------|-------------|
|
|
9
|
-
| iOS | iOS 26+ with Apple Intelligence enabled. Loads on iOS 16+ without crashing (returns `notEligible`). |
|
|
9
|
+
| iOS | iOS 26+ with Apple Intelligence enabled. Loads on iOS 16+ without crashing (returns `notEligible`). Host project must target iOS 16.0+. |
|
|
10
10
|
| Android | Device with Gemini Nano support (Pixel 8+, Galaxy S25+). Model may require download. |
|
|
11
11
|
|
|
12
12
|
## Installation
|
|
@@ -58,6 +58,55 @@ function Chat() {
|
|
|
58
58
|
}
|
|
59
59
|
```
|
|
60
60
|
|
|
61
|
+
### Tool Calling (iOS 26+)
|
|
62
|
+
|
|
63
|
+
```tsx
|
|
64
|
+
import { useLocalLLM } from 'expo-local-llm';
|
|
65
|
+
|
|
66
|
+
function WeatherChat() {
|
|
67
|
+
const { streamResponse, streamedText, activeToolCalls, error } = useLocalLLM({
|
|
68
|
+
instructions: 'You are a helpful weather assistant.',
|
|
69
|
+
tools: [
|
|
70
|
+
{
|
|
71
|
+
name: 'getWeather',
|
|
72
|
+
description: 'Get the current weather for a city',
|
|
73
|
+
parameters: {
|
|
74
|
+
city: { type: 'string', description: 'The city name' },
|
|
75
|
+
},
|
|
76
|
+
handler: async (args) => {
|
|
77
|
+
const res = await fetch(`https://api.example.com/weather?city=${args.city}`);
|
|
78
|
+
return JSON.stringify(await res.json());
|
|
79
|
+
},
|
|
80
|
+
},
|
|
81
|
+
],
|
|
82
|
+
});
|
|
83
|
+
|
|
84
|
+
return (
|
|
85
|
+
<View>
|
|
86
|
+
{activeToolCalls.length > 0 && (
|
|
87
|
+
<Text>Using {activeToolCalls.map((c) => c.toolName).join(', ')}...</Text>
|
|
88
|
+
)}
|
|
89
|
+
<Text>{streamedText}</Text>
|
|
90
|
+
</View>
|
|
91
|
+
);
|
|
92
|
+
}
|
|
93
|
+
```
|
|
94
|
+
|
|
95
|
+
### Structured Output
|
|
96
|
+
|
|
97
|
+
```tsx
|
|
98
|
+
const { respond } = useLocalLLM({
|
|
99
|
+
responseFormat: 'json',
|
|
100
|
+
schema: {
|
|
101
|
+
name: { type: 'string', description: 'Recipe name' },
|
|
102
|
+
ingredients: { type: 'array', description: 'List of ingredients' },
|
|
103
|
+
steps: { type: 'array', description: 'Cooking steps' },
|
|
104
|
+
},
|
|
105
|
+
});
|
|
106
|
+
|
|
107
|
+
const recipe = JSON.parse(await respond('Give me a pasta recipe'));
|
|
108
|
+
```
|
|
109
|
+
|
|
61
110
|
## API
|
|
62
111
|
|
|
63
112
|
### `useLocalLLM(options?)`
|
|
@@ -70,6 +119,10 @@ function Chat() {
|
|
|
70
119
|
| `options.temperature` | `number` | Sampling temperature |
|
|
71
120
|
| `options.maxTokens` | `number` | Max output tokens (capped at 256 on Android) |
|
|
72
121
|
| `options.topK` | `number` | Top-K sampling |
|
|
122
|
+
| `tools` | `ToolDefinition[]` | Tools the model can invoke (iOS 26+ only) |
|
|
123
|
+
| `toolTimeout` | `number` | Seconds before an unresolved tool call times out (default 30) |
|
|
124
|
+
| `responseFormat` | `"text" \| "json"` | Set to `"json"` for structured JSON output |
|
|
125
|
+
| `schema` | `Record<string, any>` | JSON schema for structured output (used with `responseFormat: "json"`) |
|
|
73
126
|
|
|
74
127
|
#### Returns
|
|
75
128
|
|
|
@@ -80,8 +133,10 @@ function Chat() {
|
|
|
80
133
|
| `streamedText` | `string` | Accumulated text from current stream |
|
|
81
134
|
| `downloadProgress` | `number \| null` | Download progress 0-1 (Android only) |
|
|
82
135
|
| `error` | `string \| null` | Last error message |
|
|
136
|
+
| `activeToolCalls` | `ActiveToolCall[]` | In-flight tool calls (`{ callId, toolName }`) |
|
|
83
137
|
| `respond` | `(prompt: string) => Promise<string>` | Non-streaming generation. `undefined` when unavailable. |
|
|
84
|
-
| `
|
|
138
|
+
| `session` | `LLMSession \| null` | The native session object. `null` when unavailable. |
|
|
139
|
+
| `streamResponse` | `(prompt: string) => Promise<void>` | Streaming generation (updates `streamedText`). `undefined` when model not `available`. |
|
|
85
140
|
| `cancelStream` | `() => Promise<void>` | Cancel active stream. `undefined` when unavailable. |
|
|
86
141
|
| `downloadModel` | `() => Promise<void>` | Trigger model download (Android). `undefined` when unavailable. |
|
|
87
142
|
|
|
@@ -101,9 +156,40 @@ function Chat() {
|
|
|
101
156
|
|
|
102
157
|
## Known Limitations
|
|
103
158
|
|
|
104
|
-
-
|
|
105
|
-
- **Android**: Gemini Nano SDK is in beta. API surface may change.
|
|
106
|
-
-
|
|
159
|
+
- Only one active stream at a time. Events are not scoped to session ID.
|
|
160
|
+
- **Android**: Gemini Nano SDK is in beta. API surface may change — not yet validated on device.
|
|
161
|
+
- **iOS**: Apple's Foundation Model may refuse certain categories of prompts (e.g. personal health data interpretation) due to built-in safety guardrails.
|
|
162
|
+
- Methods (`respond`, `streamResponse`, `cancelStream`) are only defined when `availability === 'available'`.
|
|
163
|
+
- Tool calling is iOS 26+ only. Android will throw at session creation if tools are passed.
|
|
164
|
+
- Structured output uses instruction-based JSON guidance, not Apple's constrained decoding.
|
|
165
|
+
|
|
166
|
+
## How is this different from @callstack/ai? (as of March 2026)
|
|
167
|
+
|
|
168
|
+
[@callstack/ai](https://github.com/callstackincubator/ai) is a full-featured
|
|
169
|
+
framework that supports multiple on-device backends (Apple Foundation Models,
|
|
170
|
+
Llama/GGUF, MLC) with Vercel AI SDK integration, generative UI, and DevTools.
|
|
171
|
+
|
|
172
|
+
`expo-local-llm` takes a different approach:
|
|
173
|
+
|
|
174
|
+
| | `expo-local-llm` | `@callstack/ai` |
|
|
175
|
+
|---|---|---|
|
|
176
|
+
| **Philosophy** | Thin bridge to the OS-provided model | Multi-backend AI framework |
|
|
177
|
+
| **iOS model** | Apple Foundation Models (system-provided) | Apple Foundation Models, Llama, MLC |
|
|
178
|
+
| **Android model** | Gemini Nano (system-provided) | No Android support |
|
|
179
|
+
| **Model management** | None — the OS handles it | You configure/download models |
|
|
180
|
+
| **Bundle size impact** | Near zero | Depends on backend + model weights |
|
|
181
|
+
| **Tool calling** | Yes (iOS 26+) | Yes |
|
|
182
|
+
| **Structured output** | Yes (instruction-based) | Yes |
|
|
183
|
+
| **Expo native module** | Yes — `expo install` and go | Bare React Native setup |
|
|
184
|
+
| **Dependencies** | None | Vercel AI SDK, Jotai, backend runtimes |
|
|
185
|
+
|
|
186
|
+
**Choose `expo-local-llm` if** you want the simplest path to on-device LLM in
|
|
187
|
+
an Expo app and are happy using whatever model the OS provides. There's nothing
|
|
188
|
+
to configure, no weights to bundle, and it works on both iOS and Android.
|
|
189
|
+
|
|
190
|
+
**Choose `@callstack/ai` if** you need to pick specific models (Llama 3.2,
|
|
191
|
+
Phi-3, Mistral), want Vercel AI SDK compatibility, or need the generative UI
|
|
192
|
+
and DevTools ecosystem.
|
|
107
193
|
|
|
108
194
|
## License
|
|
109
195
|
|
|
@@ -32,6 +32,9 @@ class ExpoLocalLlmModule : Module() {
|
|
|
32
32
|
|
|
33
33
|
Class(LLMSession::class) {
|
|
34
34
|
Constructor { config: SessionConfig ->
|
|
35
|
+
if (!config.tools.isNullOrEmpty()) {
|
|
36
|
+
throw ToolNotSupportedException()
|
|
37
|
+
}
|
|
35
38
|
LLMSession(appContext.reactContext!!, config)
|
|
36
39
|
}
|
|
37
40
|
|
|
@@ -47,6 +50,24 @@ class ExpoLocalLlmModule : Module() {
|
|
|
47
50
|
session.cancelStream()
|
|
48
51
|
}
|
|
49
52
|
|
|
53
|
+
// Tool calling stubs — not yet supported on Android
|
|
54
|
+
Function("registerTool") { _: LLMSession, _: ToolConfig ->
|
|
55
|
+
throw ToolNotSupportedException()
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
Function("unregisterTool") { _: LLMSession, _: String ->
|
|
59
|
+
throw ToolNotSupportedException()
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
Function("resolveToolCall") { _: LLMSession, _: String, _: String ->
|
|
63
|
+
throw ToolNotSupportedException()
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
Function("rejectToolCall") { _: LLMSession, _: String, _: String ->
|
|
67
|
+
throw ToolNotSupportedException()
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
Events("token", "streamComplete", "streamError", "toolCall")
|
|
50
71
|
}
|
|
51
72
|
|
|
52
73
|
OnActivityDestroys {
|
|
@@ -9,6 +9,18 @@ class SessionConfig : Record {
|
|
|
9
9
|
|
|
10
10
|
@Field
|
|
11
11
|
var options: GenerationOptionsRecord? = null
|
|
12
|
+
|
|
13
|
+
@Field
|
|
14
|
+
var tools: List<ToolConfig>? = null
|
|
15
|
+
|
|
16
|
+
@Field
|
|
17
|
+
var toolTimeout: Double? = null
|
|
18
|
+
|
|
19
|
+
@Field
|
|
20
|
+
var responseFormat: String? = null
|
|
21
|
+
|
|
22
|
+
@Field
|
|
23
|
+
var schema: Map<String, Any>? = null
|
|
12
24
|
}
|
|
13
25
|
|
|
14
26
|
class GenerationOptionsRecord : Record {
|
|
@@ -21,3 +33,14 @@ class GenerationOptionsRecord : Record {
|
|
|
21
33
|
@Field
|
|
22
34
|
var topK: Int? = null
|
|
23
35
|
}
|
|
36
|
+
|
|
37
|
+
class ToolConfig : Record {
|
|
38
|
+
@Field
|
|
39
|
+
var name: String = ""
|
|
40
|
+
|
|
41
|
+
@Field
|
|
42
|
+
var description: String = ""
|
|
43
|
+
|
|
44
|
+
@Field
|
|
45
|
+
var parameters: Map<String, Any> = emptyMap()
|
|
46
|
+
}
|
|
@@ -4,9 +4,40 @@ export type GenerationOptions = {
|
|
|
4
4
|
maxTokens?: number;
|
|
5
5
|
topK?: number;
|
|
6
6
|
};
|
|
7
|
+
export type ToolParameterType = "string" | "number" | "boolean";
|
|
8
|
+
export type ToolParameter = {
|
|
9
|
+
type: ToolParameterType;
|
|
10
|
+
description: string;
|
|
11
|
+
enum?: string[];
|
|
12
|
+
};
|
|
13
|
+
export type ToolDefinition = {
|
|
14
|
+
name: string;
|
|
15
|
+
description: string;
|
|
16
|
+
parameters: Record<string, ToolParameter>;
|
|
17
|
+
handler: (args: Record<string, any>) => Promise<string>;
|
|
18
|
+
};
|
|
19
|
+
export type ToolConfig = {
|
|
20
|
+
name: string;
|
|
21
|
+
description: string;
|
|
22
|
+
parameters: Record<string, ToolParameter>;
|
|
23
|
+
};
|
|
24
|
+
export type ToolCallEvent = {
|
|
25
|
+
callId: string;
|
|
26
|
+
toolName: string;
|
|
27
|
+
arguments: Record<string, any>;
|
|
28
|
+
};
|
|
29
|
+
export type ActiveToolCall = {
|
|
30
|
+
callId: string;
|
|
31
|
+
toolName: string;
|
|
32
|
+
};
|
|
33
|
+
export type ResponseFormat = "text" | "json";
|
|
7
34
|
export type SessionConfig = {
|
|
8
35
|
instructions?: string;
|
|
9
36
|
options?: GenerationOptions;
|
|
37
|
+
tools?: ToolDefinition[];
|
|
38
|
+
toolTimeout?: number;
|
|
39
|
+
responseFormat?: ResponseFormat;
|
|
40
|
+
schema?: Record<string, any>;
|
|
10
41
|
};
|
|
11
42
|
export type TokenEvent = {
|
|
12
43
|
token: string;
|
|
@@ -25,6 +56,7 @@ export type LLMSessionEvents = {
|
|
|
25
56
|
token: (event: TokenEvent) => void;
|
|
26
57
|
streamComplete: (event: StreamCompleteEvent) => void;
|
|
27
58
|
streamError: (event: StreamErrorEvent) => void;
|
|
59
|
+
toolCall: (event: ToolCallEvent) => void;
|
|
28
60
|
};
|
|
29
61
|
export type ExpoLocalLlmModuleEvents = {
|
|
30
62
|
downloadProgress: (event: DownloadProgress) => void;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"ExpoLocalLlm.types.d.ts","sourceRoot":"","sources":["../src/ExpoLocalLlm.types.ts"],"names":[],"mappings":"AAAA,MAAM,MAAM,iBAAiB,GACzB,WAAW,GACX,YAAY,GACZ,UAAU,GACV,aAAa,GACb,kBAAkB,GAClB,aAAa,GACb,SAAS,CAAC;AAEd,MAAM,MAAM,iBAAiB,GAAG;IAC9B,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,IAAI,CAAC,EAAE,MAAM,CAAC;CACf,CAAC;AAEF,MAAM,MAAM,aAAa,GAAG;IAC1B,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,OAAO,CAAC,EAAE,iBAAiB,CAAC;
|
|
1
|
+
{"version":3,"file":"ExpoLocalLlm.types.d.ts","sourceRoot":"","sources":["../src/ExpoLocalLlm.types.ts"],"names":[],"mappings":"AAAA,MAAM,MAAM,iBAAiB,GACzB,WAAW,GACX,YAAY,GACZ,UAAU,GACV,aAAa,GACb,kBAAkB,GAClB,aAAa,GACb,SAAS,CAAC;AAEd,MAAM,MAAM,iBAAiB,GAAG;IAC9B,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,IAAI,CAAC,EAAE,MAAM,CAAC;CACf,CAAC;AAIF,MAAM,MAAM,iBAAiB,GAAG,QAAQ,GAAG,QAAQ,GAAG,SAAS,CAAC;AAEhE,MAAM,MAAM,aAAa,GAAG;IAC1B,IAAI,EAAE,iBAAiB,CAAC;IACxB,WAAW,EAAE,MAAM,CAAC;IACpB,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;CACjB,CAAC;AAEF,MAAM,MAAM,cAAc,GAAG;IAC3B,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,MAAM,CAAC;IACpB,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,aAAa,CAAC,CAAC;IAC1C,OAAO,EAAE,CAAC,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,KAAK,OAAO,CAAC,MAAM,CAAC,CAAC;CACzD,CAAC;AAEF,MAAM,MAAM,UAAU,GAAG;IACvB,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,MAAM,CAAC;IACpB,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,aAAa,CAAC,CAAC;CAC3C,CAAC;AAEF,MAAM,MAAM,aAAa,GAAG;IAC1B,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;CAChC,CAAC;AAEF,MAAM,MAAM,cAAc,GAAG;IAC3B,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;CAClB,CAAC;AAIF,MAAM,MAAM,cAAc,GAAG,MAAM,GAAG,MAAM,CAAC;AAE7C,MAAM,MAAM,aAAa,GAAG;IAC1B,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,OAAO,CAAC,EAAE,iBAAiB,CAAC;IAC5B,KAAK,CAAC,EAAE,cAAc,EAAE,CAAC;IACzB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,cAAc,CAAC,EAAE,cAAc,CAAC;IAChC,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;CAC9B,CAAC;AAEF,MAAM,MAAM,UAAU,GAAG;IACvB,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,MAAM,CAAC;CACrB,CAAC;AAEF,MAAM,MAAM,mBAAmB,GAAG;IAChC,IAAI,EAAE,MAAM,CAAC;CACd,CAAC;AAEF,MAAM,MAAM,gBAAgB,GAAG;IAC7B,KAAK,EAAE,MAAM,CAAC;CACf,CAAC;AAEF,MAAM,MAAM,gBAAgB,GAAG;IAC7B,QAAQ,EAAE,MAAM,CAAC;CAClB,CAAC;AAEF,MAAM,MAAM,gBAAgB,GAAG;IAC7B,KAAK,EAAE,CAAC,KAAK,EAAE,UAAU,KAAK,IAAI,CAAC;IACnC,cAAc,EAAE,CAAC,KAAK,EAAE,mBAAmB,KAAK,IAAI,CAAC;IACrD,WAAW,EAAE,CAAC,KAAK,EAAE,gBAAgB,KAAK,IAAI,CAAC;IAC/C,QAAQ,EAAE,CAAC,KAAK,EAAE,aAAa,KAAK,IAAI,CAAC;CAC1C,CAAC;AAEF,MAAM,MAAM,wBAAwB,GAAG;IACrC,gBAAgB,EAAE,CAAC,KAAK,EAAE,gBAAgB,KAAK,IAAI,CAAC;IACpD,kBAAkB,EAAE,CAAC,KAAK,EAAE;QAAE,YAAY,EAAE,iBAAiB,CAAA;KAAE,KAAK,IAAI,CAAC;CAC1E,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"ExpoLocalLlm.types.js","sourceRoot":"","sources":["../src/ExpoLocalLlm.types.ts"],"names":[],"mappings":"","sourcesContent":["export type ModelAvailability =\n | \"available\"\n | \"notEnabled\"\n | \"notReady\"\n | \"notEligible\"\n | \"downloadRequired\"\n | \"downloading\"\n | \"unknown\";\n\nexport type GenerationOptions = {\n temperature?: number;\n maxTokens?: number;\n topK?: number;\n};\n\nexport type SessionConfig = {\n instructions?: string;\n options?: GenerationOptions;\n};\n\nexport type TokenEvent = {\n token: string;\n accumulated: string;\n};\n\nexport type StreamCompleteEvent = {\n text: string;\n};\n\nexport type StreamErrorEvent = {\n error: string;\n};\n\nexport type DownloadProgress = {\n progress: number;\n};\n\nexport type LLMSessionEvents = {\n token: (event: TokenEvent) => void;\n streamComplete: (event: StreamCompleteEvent) => void;\n streamError: (event: StreamErrorEvent) => void;\n};\n\nexport type ExpoLocalLlmModuleEvents = {\n downloadProgress: (event: DownloadProgress) => void;\n availabilityChange: (event: { availability: ModelAvailability }) => void;\n};\n"]}
|
|
1
|
+
{"version":3,"file":"ExpoLocalLlm.types.js","sourceRoot":"","sources":["../src/ExpoLocalLlm.types.ts"],"names":[],"mappings":"","sourcesContent":["export type ModelAvailability =\n | \"available\"\n | \"notEnabled\"\n | \"notReady\"\n | \"notEligible\"\n | \"downloadRequired\"\n | \"downloading\"\n | \"unknown\";\n\nexport type GenerationOptions = {\n temperature?: number;\n maxTokens?: number;\n topK?: number;\n};\n\n// Tool calling types\n\nexport type ToolParameterType = \"string\" | \"number\" | \"boolean\";\n\nexport type ToolParameter = {\n type: ToolParameterType;\n description: string;\n enum?: string[];\n};\n\nexport type ToolDefinition = {\n name: string;\n description: string;\n parameters: Record<string, ToolParameter>;\n handler: (args: Record<string, any>) => Promise<string>;\n};\n\nexport type ToolConfig = {\n name: string;\n description: string;\n parameters: Record<string, ToolParameter>;\n};\n\nexport type ToolCallEvent = {\n callId: string;\n toolName: string;\n arguments: Record<string, any>;\n};\n\nexport type ActiveToolCall = {\n callId: string;\n toolName: string;\n};\n\n// Structured output types\n\nexport type ResponseFormat = \"text\" | \"json\";\n\nexport type SessionConfig = {\n instructions?: string;\n options?: GenerationOptions;\n tools?: ToolDefinition[];\n toolTimeout?: number;\n responseFormat?: ResponseFormat;\n schema?: Record<string, any>;\n};\n\nexport type TokenEvent = {\n token: string;\n accumulated: string;\n};\n\nexport type StreamCompleteEvent = {\n text: string;\n};\n\nexport type StreamErrorEvent = {\n error: string;\n};\n\nexport type DownloadProgress = {\n progress: number;\n};\n\nexport type LLMSessionEvents = {\n token: (event: TokenEvent) => void;\n streamComplete: (event: StreamCompleteEvent) => void;\n streamError: (event: StreamErrorEvent) => void;\n toolCall: (event: ToolCallEvent) => void;\n};\n\nexport type ExpoLocalLlmModuleEvents = {\n downloadProgress: (event: DownloadProgress) => void;\n availabilityChange: (event: { availability: ModelAvailability }) => void;\n};\n"]}
|
package/build/LLMSession.d.ts
CHANGED
|
@@ -1,9 +1,13 @@
|
|
|
1
1
|
import { SharedObject } from "expo-modules-core";
|
|
2
|
-
import type { SessionConfig, LLMSessionEvents } from "./ExpoLocalLlm.types";
|
|
2
|
+
import type { SessionConfig, LLMSessionEvents, ToolConfig } from "./ExpoLocalLlm.types";
|
|
3
3
|
export declare class LLMSession extends SharedObject<LLMSessionEvents> {
|
|
4
4
|
respond: (prompt: string) => Promise<string>;
|
|
5
5
|
streamResponse: (prompt: string) => Promise<void>;
|
|
6
6
|
cancelStream: () => Promise<void>;
|
|
7
|
+
registerTool: (config: ToolConfig) => void;
|
|
8
|
+
unregisterTool: (name: string) => void;
|
|
9
|
+
resolveToolCall: (callId: string, result: string) => void;
|
|
10
|
+
rejectToolCall: (callId: string, error: string) => void;
|
|
7
11
|
}
|
|
8
12
|
export declare function createLLMSession(config?: SessionConfig): LLMSession;
|
|
9
13
|
//# sourceMappingURL=LLMSession.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"LLMSession.d.ts","sourceRoot":"","sources":["../src/LLMSession.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAuB,MAAM,mBAAmB,CAAC;AAEtE,OAAO,KAAK,
|
|
1
|
+
{"version":3,"file":"LLMSession.d.ts","sourceRoot":"","sources":["../src/LLMSession.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAuB,MAAM,mBAAmB,CAAC;AAEtE,OAAO,KAAK,EACV,aAAa,EACb,gBAAgB,EAChB,UAAU,EACX,MAAM,sBAAsB,CAAC;AAG9B,qBAAa,UAAW,SAAQ,YAAY,CAAC,gBAAgB,CAAC;IAG5D,OAAO,EAAG,CAAC,MAAM,EAAE,MAAM,KAAK,OAAO,CAAC,MAAM,CAAC,CAAC;IAC9C,cAAc,EAAG,CAAC,MAAM,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IACnD,YAAY,EAAG,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;IACnC,YAAY,EAAG,CAAC,MAAM,EAAE,UAAU,KAAK,IAAI,CAAC;IAC5C,cAAc,EAAG,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,CAAC;IACxC,eAAe,EAAG,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,KAAK,IAAI,CAAC;IAC3D,cAAc,EAAG,CAAC,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,KAAK,IAAI,CAAC;CAC1D;AAED,wBAAgB,gBAAgB,CAAC,MAAM,GAAE,aAAkB,GAAG,UAAU,CAWvE"}
|
package/build/LLMSession.js
CHANGED
|
@@ -6,11 +6,20 @@ export class LLMSession extends SharedObject {
|
|
|
6
6
|
respond;
|
|
7
7
|
streamResponse;
|
|
8
8
|
cancelStream;
|
|
9
|
+
registerTool;
|
|
10
|
+
unregisterTool;
|
|
11
|
+
resolveToolCall;
|
|
12
|
+
rejectToolCall;
|
|
9
13
|
}
|
|
10
14
|
export function createLLMSession(config = {}) {
|
|
11
15
|
if (!ExpoLocalLlmModule) {
|
|
12
16
|
throw new UnavailabilityError("ExpoLocalLlm", "createLLMSession");
|
|
13
17
|
}
|
|
14
|
-
|
|
18
|
+
// Strip handler functions before passing to native — native doesn't need them
|
|
19
|
+
const nativeConfig = {
|
|
20
|
+
...config,
|
|
21
|
+
tools: config.tools?.map(({ handler, ...rest }) => rest),
|
|
22
|
+
};
|
|
23
|
+
return new ExpoLocalLlmModule.LLMSession(nativeConfig);
|
|
15
24
|
}
|
|
16
25
|
//# sourceMappingURL=LLMSession.js.map
|
package/build/LLMSession.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"LLMSession.js","sourceRoot":"","sources":["../src/LLMSession.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,mBAAmB,EAAE,MAAM,mBAAmB,CAAC;
|
|
1
|
+
{"version":3,"file":"LLMSession.js","sourceRoot":"","sources":["../src/LLMSession.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,mBAAmB,EAAE,MAAM,mBAAmB,CAAC;AAOtE,OAAO,kBAAkB,MAAM,sBAAsB,CAAC;AAEtD,MAAM,OAAO,UAAW,SAAQ,YAA8B;IAC5D,wDAAwD;IACxD,oDAAoD;IACpD,OAAO,CAAuC;IAC9C,cAAc,CAAqC;IACnD,YAAY,CAAuB;IACnC,YAAY,CAAgC;IAC5C,cAAc,CAA0B;IACxC,eAAe,CAA4C;IAC3D,cAAc,CAA2C;CAC1D;AAED,MAAM,UAAU,gBAAgB,CAAC,SAAwB,EAAE;IACzD,IAAI,CAAC,kBAAkB,EAAE,CAAC;QACxB,MAAM,IAAI,mBAAmB,CAAC,cAAc,EAAE,kBAAkB,CAAC,CAAC;IACpE,CAAC;IAED,8EAA8E;IAC9E,MAAM,YAAY,GAAG;QACnB,GAAG,MAAM;QACT,KAAK,EAAE,MAAM,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,EAAE,OAAO,EAAE,GAAG,IAAI,EAAE,EAAE,EAAE,CAAC,IAAI,CAAC;KACzD,CAAC;IACF,OAAO,IAAI,kBAAkB,CAAC,UAAU,CAAC,YAAY,CAAe,CAAC;AACvE,CAAC","sourcesContent":["import { SharedObject, UnavailabilityError } from \"expo-modules-core\";\n\nimport type {\n SessionConfig,\n LLMSessionEvents,\n ToolConfig,\n} from \"./ExpoLocalLlm.types\";\nimport ExpoLocalLlmModule from \"./ExpoLocalLlmModule\";\n\nexport class LLMSession extends SharedObject<LLMSessionEvents> {\n // Methods are implemented natively via the Class() DSL.\n // TypeScript declarations provide type safety only.\n respond!: (prompt: string) => Promise<string>;\n streamResponse!: (prompt: string) => Promise<void>;\n cancelStream!: () => Promise<void>;\n registerTool!: (config: ToolConfig) => void;\n unregisterTool!: (name: string) => void;\n resolveToolCall!: (callId: string, result: string) => void;\n rejectToolCall!: (callId: string, error: string) => void;\n}\n\nexport function createLLMSession(config: SessionConfig = {}): LLMSession {\n if (!ExpoLocalLlmModule) {\n throw new UnavailabilityError(\"ExpoLocalLlm\", \"createLLMSession\");\n }\n\n // Strip handler functions before passing to native — native doesn't need them\n const nativeConfig = {\n ...config,\n tools: config.tools?.map(({ handler, ...rest }) => rest),\n };\n return new ExpoLocalLlmModule.LLMSession(nativeConfig) as LLMSession;\n}\n"]}
|
package/build/useLocalLLM.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type { ModelAvailability, SessionConfig } from "./ExpoLocalLlm.types";
|
|
1
|
+
import type { ModelAvailability, SessionConfig, ActiveToolCall } from "./ExpoLocalLlm.types";
|
|
2
2
|
import { LLMSession } from "./LLMSession";
|
|
3
3
|
type UseLocalLLMOptions = SessionConfig;
|
|
4
4
|
type UseLocalLLMResult = {
|
|
@@ -8,6 +8,7 @@ type UseLocalLLMResult = {
|
|
|
8
8
|
streamedText: string;
|
|
9
9
|
downloadProgress: number | null;
|
|
10
10
|
error: string | null;
|
|
11
|
+
activeToolCalls: ActiveToolCall[];
|
|
11
12
|
respond?: (prompt: string) => Promise<string>;
|
|
12
13
|
streamResponse?: (prompt: string) => Promise<void>;
|
|
13
14
|
cancelStream?: () => Promise<void>;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"useLocalLLM.d.ts","sourceRoot":"","sources":["../src/useLocalLLM.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EACV,iBAAiB,EACjB,aAAa,
|
|
1
|
+
{"version":3,"file":"useLocalLLM.d.ts","sourceRoot":"","sources":["../src/useLocalLLM.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EACV,iBAAiB,EACjB,aAAa,EAOb,cAAc,EACf,MAAM,sBAAsB,CAAC;AAE9B,OAAO,EAAoB,UAAU,EAAE,MAAM,cAAc,CAAC;AAE5D,KAAK,kBAAkB,GAAG,aAAa,CAAC;AAExC,KAAK,iBAAiB,GAAG;IACvB,YAAY,EAAE,iBAAiB,CAAC;IAChC,OAAO,EAAE,UAAU,GAAG,IAAI,CAAC;IAC3B,YAAY,EAAE,OAAO,CAAC;IACtB,YAAY,EAAE,MAAM,CAAC;IACrB,gBAAgB,EAAE,MAAM,GAAG,IAAI,CAAC;IAChC,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,eAAe,EAAE,cAAc,EAAE,CAAC;IAClC,OAAO,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,OAAO,CAAC,MAAM,CAAC,CAAC;IAC9C,cAAc,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IACnD,YAAY,CAAC,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;IACnC,aAAa,CAAC,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;CACrC,CAAC;AAEF,wBAAgB,WAAW,CACzB,OAAO,GAAE,kBAAuB,GAC/B,iBAAiB,CA0PnB"}
|
package/build/useLocalLLM.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { useEffect, useState, useCallback, useMemo } from "react";
|
|
1
|
+
import { useEffect, useState, useCallback, useMemo, useRef } from "react";
|
|
2
2
|
import ExpoLocalLlmModule from "./ExpoLocalLlmModule";
|
|
3
3
|
import { createLLMSession } from "./LLMSession";
|
|
4
4
|
export function useLocalLLM(options = {}) {
|
|
@@ -7,20 +7,47 @@ export function useLocalLLM(options = {}) {
|
|
|
7
7
|
const [streamedText, setStreamedText] = useState("");
|
|
8
8
|
const [downloadProgress, setDownloadProgress] = useState(null);
|
|
9
9
|
const [error, setError] = useState(null);
|
|
10
|
+
const [activeToolCalls, setActiveToolCalls] = useState([]);
|
|
11
|
+
// Keep a ref to the current tool handlers so the event listener
|
|
12
|
+
// always sees the latest handlers without needing to recreate the session.
|
|
13
|
+
const toolHandlersRef = useRef(new Map());
|
|
14
|
+
// Update handler ref whenever tools change
|
|
15
|
+
useEffect(() => {
|
|
16
|
+
const handlers = new Map();
|
|
17
|
+
if (options.tools) {
|
|
18
|
+
for (const tool of options.tools) {
|
|
19
|
+
handlers.set(tool.name, tool.handler);
|
|
20
|
+
}
|
|
21
|
+
}
|
|
22
|
+
toolHandlersRef.current = handlers;
|
|
23
|
+
}, [options.tools]);
|
|
10
24
|
// Stabilize options to avoid unnecessary session recreation
|
|
25
|
+
// Include tool names/descriptions so session recreates when tools change structurally
|
|
26
|
+
const toolsFingerprint = useMemo(() => options.tools
|
|
27
|
+
?.map((t) => `${t.name}:${t.description}:${JSON.stringify(t.parameters)}`)
|
|
28
|
+
.join("|") ?? "", [options.tools]);
|
|
29
|
+
// Fingerprint the schema so session recreates when it changes
|
|
30
|
+
const schemaFingerprint = useMemo(() => (options.schema ? JSON.stringify(options.schema) : ""), [options.schema]);
|
|
11
31
|
const stableConfig = useMemo(() => options, [
|
|
12
32
|
options.instructions,
|
|
13
33
|
options.options?.temperature,
|
|
14
34
|
options.options?.maxTokens,
|
|
15
35
|
options.options?.topK,
|
|
36
|
+
options.toolTimeout,
|
|
37
|
+
options.responseFormat,
|
|
38
|
+
schemaFingerprint,
|
|
39
|
+
toolsFingerprint,
|
|
16
40
|
]);
|
|
17
41
|
const session = useMemo(() => {
|
|
18
42
|
if (!ExpoLocalLlmModule)
|
|
19
43
|
return null;
|
|
20
44
|
try {
|
|
21
|
-
|
|
45
|
+
const s = createLLMSession(stableConfig);
|
|
46
|
+
setError(null);
|
|
47
|
+
return s;
|
|
22
48
|
}
|
|
23
49
|
catch (e) {
|
|
50
|
+
setError(e.message ?? "Failed to create LLM session");
|
|
24
51
|
return null;
|
|
25
52
|
}
|
|
26
53
|
}, [stableConfig]);
|
|
@@ -39,13 +66,62 @@ export function useLocalLLM(options = {}) {
|
|
|
39
66
|
subs.push(session.addListener("token", (event) => {
|
|
40
67
|
setStreamedText(event.accumulated);
|
|
41
68
|
}));
|
|
42
|
-
subs.push(session.addListener("streamComplete", (
|
|
69
|
+
subs.push(session.addListener("streamComplete", (event) => {
|
|
70
|
+
setStreamedText(event.text);
|
|
43
71
|
setIsGenerating(false);
|
|
44
72
|
}));
|
|
45
73
|
subs.push(session.addListener("streamError", (event) => {
|
|
46
74
|
setError(event.error);
|
|
47
75
|
setIsGenerating(false);
|
|
48
76
|
}));
|
|
77
|
+
// Tool call event listener
|
|
78
|
+
subs.push(session.addListener("toolCall", (event) => {
|
|
79
|
+
const activeCall = { callId: event.callId, toolName: event.toolName };
|
|
80
|
+
setActiveToolCalls((prev) => [...prev, activeCall]);
|
|
81
|
+
const removeActiveCall = () => {
|
|
82
|
+
setActiveToolCalls((prev) => prev.filter((c) => c.callId !== event.callId));
|
|
83
|
+
};
|
|
84
|
+
const handler = toolHandlersRef.current.get(event.toolName);
|
|
85
|
+
if (!handler) {
|
|
86
|
+
removeActiveCall();
|
|
87
|
+
try {
|
|
88
|
+
session.rejectToolCall(event.callId, `No handler registered for tool: ${event.toolName}`);
|
|
89
|
+
}
|
|
90
|
+
catch {
|
|
91
|
+
// Ignore if session is already torn down
|
|
92
|
+
}
|
|
93
|
+
return;
|
|
94
|
+
}
|
|
95
|
+
let result;
|
|
96
|
+
try {
|
|
97
|
+
result = handler(event.arguments);
|
|
98
|
+
}
|
|
99
|
+
catch (e) {
|
|
100
|
+
// Handler threw synchronously before returning a promise
|
|
101
|
+
removeActiveCall();
|
|
102
|
+
try {
|
|
103
|
+
session.rejectToolCall(event.callId, e.message || "Tool handler failed");
|
|
104
|
+
}
|
|
105
|
+
catch {
|
|
106
|
+
// Ignore if session is already torn down
|
|
107
|
+
}
|
|
108
|
+
return;
|
|
109
|
+
}
|
|
110
|
+
result
|
|
111
|
+
.then((value) => {
|
|
112
|
+
removeActiveCall();
|
|
113
|
+
session.resolveToolCall(event.callId, value);
|
|
114
|
+
})
|
|
115
|
+
.catch((e) => {
|
|
116
|
+
removeActiveCall();
|
|
117
|
+
try {
|
|
118
|
+
session.rejectToolCall(event.callId, e.message || "Tool handler failed");
|
|
119
|
+
}
|
|
120
|
+
catch {
|
|
121
|
+
// Ignore if session is already torn down
|
|
122
|
+
}
|
|
123
|
+
});
|
|
124
|
+
}));
|
|
49
125
|
}
|
|
50
126
|
return () => {
|
|
51
127
|
subs.forEach((s) => s.remove());
|
|
@@ -103,6 +179,7 @@ export function useLocalLLM(options = {}) {
|
|
|
103
179
|
streamedText: "",
|
|
104
180
|
downloadProgress: null,
|
|
105
181
|
error: null,
|
|
182
|
+
activeToolCalls: [],
|
|
106
183
|
};
|
|
107
184
|
}
|
|
108
185
|
return {
|
|
@@ -112,6 +189,7 @@ export function useLocalLLM(options = {}) {
|
|
|
112
189
|
streamedText,
|
|
113
190
|
downloadProgress,
|
|
114
191
|
error,
|
|
192
|
+
activeToolCalls,
|
|
115
193
|
respond: session && availability === "available" ? respond : undefined,
|
|
116
194
|
streamResponse: session && availability === "available" ? streamResponse : undefined,
|
|
117
195
|
cancelStream: session && availability === "available" ? cancelStream : undefined,
|
package/build/useLocalLLM.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"useLocalLLM.js","sourceRoot":"","sources":["../src/useLocalLLM.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,QAAQ,EAAE,WAAW,EAAE,OAAO,EAAE,MAAM,OAAO,CAAC;AAUlE,OAAO,kBAAkB,MAAM,sBAAsB,CAAC;AACtD,OAAO,EAAE,gBAAgB,EAAc,MAAM,cAAc,CAAC;AAiB5D,MAAM,UAAU,WAAW,CACzB,UAA8B,EAAE;IAEhC,MAAM,CAAC,YAAY,EAAE,eAAe,CAAC,GACnC,QAAQ,CAAoB,SAAS,CAAC,CAAC;IACzC,MAAM,CAAC,YAAY,EAAE,eAAe,CAAC,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC;IACxD,MAAM,CAAC,YAAY,EAAE,eAAe,CAAC,GAAG,QAAQ,CAAC,EAAE,CAAC,CAAC;IACrD,MAAM,CAAC,gBAAgB,EAAE,mBAAmB,CAAC,GAAG,QAAQ,CAAgB,IAAI,CAAC,CAAC;IAC9E,MAAM,CAAC,KAAK,EAAE,QAAQ,CAAC,GAAG,QAAQ,CAAgB,IAAI,CAAC,CAAC;IAExD,4DAA4D;IAC5D,MAAM,YAAY,GAAG,OAAO,CAC1B,GAAG,EAAE,CAAC,OAAO,EACb;QACE,OAAO,CAAC,YAAY;QACpB,OAAO,CAAC,OAAO,EAAE,WAAW;QAC5B,OAAO,CAAC,OAAO,EAAE,SAAS;QAC1B,OAAO,CAAC,OAAO,EAAE,IAAI;KACtB,CACF,CAAC;IAEF,MAAM,OAAO,GAAG,OAAO,CAAC,GAAG,EAAE;QAC3B,IAAI,CAAC,kBAAkB;YAAE,OAAO,IAAI,CAAC;QACrC,IAAI,CAAC;YACH,OAAO,gBAAgB,CAAC,YAAY,CAAC,CAAC;QACxC,CAAC;QAAC,OAAO,CAAM,EAAE,CAAC;YAChB,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC,EAAE,CAAC,YAAY,CAAC,CAAC,CAAC;IAEnB,SAAS,CAAC,GAAG,EAAE;QACb,IAAI,CAAC,kBAAkB;YAAE,OAAO;QAEhC,eAAe,CAAC,kBAAkB,CAAC,eAAe,EAAuB,CAAC,CAAC;QAE3E,MAAM,IAAI,GAAyB,EAAE,CAAC;QAEtC,IAAI,CAAC,IAAI,CACP,kBAAkB,CAAC,WAAW,CAC5B,kBAAkB,EAClB,CAAC,KAAuB,EAAE,EAAE;YAC1B,mBAAmB,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;QACtC,CAAC,CACF,CACF,CAAC;QACF,IAAI,CAAC,IAAI,CACP,kBAAkB,CAAC,WAAW,CAAC,oBAAoB,EAAE,CAAC,KAAK,EAAE,EAAE;YAC7D,eAAe,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC;QACtC,CAAC,CAAC,CACH,CAAC;QAEF,IAAI,OAAO,EAAE,CAAC;YACZ,IAAI,CAAC,IAAI,CACP,OAAO,CAAC,WAAW,CAAC,OAAO,EAAE,CAAC,KAAiB,EAAE,EAAE;gBACjD,eAAe,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;YACrC,CAAC,CAAC,CACH,CAAC;YACF,IAAI,CAAC,IAAI,CACP,OAAO,CAAC,WAAW,CAAC,gBAAgB,EAAE,CAAC,MAA2B,EAAE,EAAE;gBACpE,eAAe,CAAC,KAAK,CAAC,CAAC;YACzB,CAAC,CAAC,CACH,CAAC;YACF,IAAI,CAAC,IAAI,CACP,OAAO,CAAC,WAAW,CAAC,aAAa,EAAE,CAAC,KAAuB,EAAE,EAAE;gBAC7D,QAAQ,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;gBACtB,eAAe,CAAC,KAAK,CAAC,CAAC;YACzB,CAAC,CAAC,CACH,CAAC;QACJ,CAAC;QAED,OAAO,GAAG,EAAE;YACV,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC;YAChC,OAAO,EAAE,OAAO,EAAE,CAAC;QACrB,CAAC,CAAC;IACJ,CAAC,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC;IAEd,MAAM,OAAO,GAAG,WAAW,CACzB,KAAK,EAAE,MAAc,EAAmB,EAAE;QACxC,IAAI,CAAC,OAAO;YAAE,MAAM,IAAI,KAAK,CAAC,uBAAuB,CAAC,CAAC;QACvD,QAAQ,CAAC,IAAI,CAAC,CAAC;QACf,eAAe,CAAC,IAAI,CAAC,CAAC;QACtB,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;YAC7C,OAAO,MAAM,CAAC;QAChB,CAAC;QAAC,OAAO,CAAM,EAAE,CAAC;YAChB,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;YACpB,MAAM,CAAC,CAAC;QACV,CAAC;gBAAS,CAAC;YACT,eAAe,CAAC,KAAK,CAAC,CAAC;QACzB,CAAC;IACH,CAAC,EACD,CAAC,OAAO,CAAC,CACV,CAAC;IAEF,MAAM,cAAc,GAAG,WAAW,CAChC,KAAK,EAAE,MAAc,EAAiB,EAAE;QACtC,IAAI,CAAC,OAAO;YAAE,MAAM,IAAI,KAAK,CAAC,uBAAuB,CAAC,CAAC;QACvD,QAAQ,CAAC,IAAI,CAAC,CAAC;QACf,eAAe,CAAC,EAAE,CAAC,CAAC;QACpB,eAAe,CAAC,IAAI,CAAC,CAAC;QACtB,IAAI,CAAC;YACH,MAAM,OAAO,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC;QACvC,CAAC;QAAC,OAAO,CAAM,EAAE,CAAC;YAChB,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;YACpB,eAAe,CAAC,KAAK,CAAC,CAAC;YACvB,MAAM,CAAC,CAAC;QACV,CAAC;IACH,CAAC,EACD,CAAC,OAAO,CAAC,CACV,CAAC;IAEF,MAAM,YAAY,GAAG,WAAW,CAAC,KAAK,IAAmB,EAAE;QACzD,IAAI,CAAC,OAAO;YAAE,OAAO;QACrB,MAAM,OAAO,CAAC,YAAY,EAAE,CAAC;QAC7B,eAAe,CAAC,KAAK,CAAC,CAAC;IACzB,CAAC,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC;IAEd,MAAM,aAAa,GAAG,WAAW,CAAC,KAAK,IAAmB,EAAE;QAC1D,IAAI,CAAC,kBAAkB;YAAE,OAAO;QAChC,MAAM,kBAAkB,CAAC,aAAa,EAAE,CAAC;IAC3C,CAAC,EAAE,EAAE,CAAC,CAAC;IAEP,IAAI,CAAC,kBAAkB,EAAE,CAAC;QACxB,OAAO;YACL,YAAY,EAAE,aAAa;YAC3B,OAAO,EAAE,IAAI;YACb,YAAY,EAAE,KAAK;YACnB,YAAY,EAAE,EAAE;YAChB,gBAAgB,EAAE,IAAI;YACtB,KAAK,EAAE,IAAI;SACZ,CAAC;IACJ,CAAC;IAED,OAAO;QACL,YAAY;QACZ,OAAO;QACP,YAAY;QACZ,YAAY;QACZ,gBAAgB;QAChB,KAAK;QACL,OAAO,EAAE,OAAO,IAAI,YAAY,KAAK,WAAW,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS;QACtE,cAAc,EACZ,OAAO,IAAI,YAAY,KAAK,WAAW,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,SAAS;QACtE,YAAY,EACV,OAAO,IAAI,YAAY,KAAK,WAAW,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,SAAS;QACpE,aAAa;KACd,CAAC;AACJ,CAAC","sourcesContent":["import { useEffect, useState, useCallback, useMemo } from \"react\";\n\nimport type {\n ModelAvailability,\n SessionConfig,\n TokenEvent,\n StreamCompleteEvent,\n StreamErrorEvent,\n DownloadProgress,\n} from \"./ExpoLocalLlm.types\";\nimport ExpoLocalLlmModule from \"./ExpoLocalLlmModule\";\nimport { createLLMSession, LLMSession } from \"./LLMSession\";\n\ntype UseLocalLLMOptions = SessionConfig;\n\ntype UseLocalLLMResult = {\n availability: ModelAvailability;\n session: LLMSession | null;\n isGenerating: boolean;\n streamedText: string;\n downloadProgress: number | null;\n error: string | null;\n respond?: (prompt: string) => Promise<string>;\n streamResponse?: (prompt: string) => Promise<void>;\n cancelStream?: () => Promise<void>;\n downloadModel?: () => Promise<void>;\n};\n\nexport function useLocalLLM(\n options: UseLocalLLMOptions = {}\n): UseLocalLLMResult {\n const [availability, setAvailability] =\n useState<ModelAvailability>(\"unknown\");\n const [isGenerating, setIsGenerating] = useState(false);\n const [streamedText, setStreamedText] = useState(\"\");\n const [downloadProgress, setDownloadProgress] = useState<number | null>(null);\n const [error, setError] = useState<string | null>(null);\n\n // Stabilize options to avoid unnecessary session recreation\n const stableConfig = useMemo(\n () => options,\n [\n options.instructions,\n options.options?.temperature,\n options.options?.maxTokens,\n options.options?.topK,\n ]\n );\n\n const session = useMemo(() => {\n if (!ExpoLocalLlmModule) return null;\n try {\n return createLLMSession(stableConfig);\n } catch (e: any) {\n return null;\n }\n }, [stableConfig]);\n\n useEffect(() => {\n if (!ExpoLocalLlmModule) return;\n\n setAvailability(ExpoLocalLlmModule.getAvailability() as ModelAvailability);\n\n const subs: { remove(): void }[] = [];\n\n subs.push(\n ExpoLocalLlmModule.addListener(\n \"downloadProgress\",\n (event: DownloadProgress) => {\n setDownloadProgress(event.progress);\n }\n )\n );\n subs.push(\n ExpoLocalLlmModule.addListener(\"availabilityChange\", (event) => {\n setAvailability(event.availability);\n })\n );\n\n if (session) {\n subs.push(\n session.addListener(\"token\", (event: TokenEvent) => {\n setStreamedText(event.accumulated);\n })\n );\n subs.push(\n session.addListener(\"streamComplete\", (_event: StreamCompleteEvent) => {\n setIsGenerating(false);\n })\n );\n subs.push(\n session.addListener(\"streamError\", (event: StreamErrorEvent) => {\n setError(event.error);\n setIsGenerating(false);\n })\n );\n }\n\n return () => {\n subs.forEach((s) => s.remove());\n session?.release();\n };\n }, [session]);\n\n const respond = useCallback(\n async (prompt: string): Promise<string> => {\n if (!session) throw new Error(\"Session not available\");\n setError(null);\n setIsGenerating(true);\n try {\n const result = await session.respond(prompt);\n return result;\n } catch (e: any) {\n setError(e.message);\n throw e;\n } finally {\n setIsGenerating(false);\n }\n },\n [session]\n );\n\n const streamResponse = useCallback(\n async (prompt: string): Promise<void> => {\n if (!session) throw new Error(\"Session not available\");\n setError(null);\n setStreamedText(\"\");\n setIsGenerating(true);\n try {\n await session.streamResponse(prompt);\n } catch (e: any) {\n setError(e.message);\n setIsGenerating(false);\n throw e;\n }\n },\n [session]\n );\n\n const cancelStream = useCallback(async (): Promise<void> => {\n if (!session) return;\n await session.cancelStream();\n setIsGenerating(false);\n }, [session]);\n\n const downloadModel = useCallback(async (): Promise<void> => {\n if (!ExpoLocalLlmModule) return;\n await ExpoLocalLlmModule.downloadModel();\n }, []);\n\n if (!ExpoLocalLlmModule) {\n return {\n availability: \"notEligible\",\n session: null,\n isGenerating: false,\n streamedText: \"\",\n downloadProgress: null,\n error: null,\n };\n }\n\n return {\n availability,\n session,\n isGenerating,\n streamedText,\n downloadProgress,\n error,\n respond: session && availability === \"available\" ? respond : undefined,\n streamResponse:\n session && availability === \"available\" ? streamResponse : undefined,\n cancelStream:\n session && availability === \"available\" ? cancelStream : undefined,\n downloadModel,\n };\n}\n"]}
|
|
1
|
+
{"version":3,"file":"useLocalLLM.js","sourceRoot":"","sources":["../src/useLocalLLM.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,QAAQ,EAAE,WAAW,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,OAAO,CAAC;AAa1E,OAAO,kBAAkB,MAAM,sBAAsB,CAAC;AACtD,OAAO,EAAE,gBAAgB,EAAc,MAAM,cAAc,CAAC;AAkB5D,MAAM,UAAU,WAAW,CACzB,UAA8B,EAAE;IAEhC,MAAM,CAAC,YAAY,EAAE,eAAe,CAAC,GACnC,QAAQ,CAAoB,SAAS,CAAC,CAAC;IACzC,MAAM,CAAC,YAAY,EAAE,eAAe,CAAC,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC;IACxD,MAAM,CAAC,YAAY,EAAE,eAAe,CAAC,GAAG,QAAQ,CAAC,EAAE,CAAC,CAAC;IACrD,MAAM,CAAC,gBAAgB,EAAE,mBAAmB,CAAC,GAAG,QAAQ,CAAgB,IAAI,CAAC,CAAC;IAC9E,MAAM,CAAC,KAAK,EAAE,QAAQ,CAAC,GAAG,QAAQ,CAAgB,IAAI,CAAC,CAAC;IACxD,MAAM,CAAC,eAAe,EAAE,kBAAkB,CAAC,GAAG,QAAQ,CAAmB,EAAE,CAAC,CAAC;IAE7E,gEAAgE;IAChE,2EAA2E;IAC3E,MAAM,eAAe,GAAG,MAAM,CAC5B,IAAI,GAAG,EAAE,CACV,CAAC;IAEF,2CAA2C;IAC3C,SAAS,CAAC,GAAG,EAAE;QACb,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAqC,CAAC;QAC9D,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;YAClB,KAAK,MAAM,IAAI,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;gBACjC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;YACxC,CAAC;QACH,CAAC;QACD,eAAe,CAAC,OAAO,GAAG,QAAQ,CAAC;IACrC,CAAC,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC;IAEpB,4DAA4D;IAC5D,sFAAsF;IACtF,MAAM,gBAAgB,GAAG,OAAO,CAC9B,GAAG,EAAE,CACH,OAAO,CAAC,KAAK;QACX,EAAE,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,WAAW,IAAI,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,UAAU,CAAC,EAAE,CAAC;SACzE,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,EACpB,CAAC,OAAO,CAAC,KAAK,CAAC,CAChB,CAAC;IAEF,8DAA8D;IAC9D,MAAM,iBAAiB,GAAG,OAAO,CAC/B,GAAG,EAAE,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,EAC5D,CAAC,OAAO,CAAC,MAAM,CAAC,CACjB,CAAC;IAEF,MAAM,YAAY,GAAG,OAAO,CAC1B,GAAG,EAAE,CAAC,OAAO,EACb;QACE,OAAO,CAAC,YAAY;QACpB,OAAO,CAAC,OAAO,EAAE,WAAW;QAC5B,OAAO,CAAC,OAAO,EAAE,SAAS;QAC1B,OAAO,CAAC,OAAO,EAAE,IAAI;QACrB,OAAO,CAAC,WAAW;QACnB,OAAO,CAAC,cAAc;QACtB,iBAAiB;QACjB,gBAAgB;KACjB,CACF,CAAC;IAEF,MAAM,OAAO,GAAG,OAAO,CAAC,GAAG,EAAE;QAC3B,IAAI,CAAC,kBAAkB;YAAE,OAAO,IAAI,CAAC;QACrC,IAAI,CAAC;YACH,MAAM,CAAC,GAAG,gBAAgB,CAAC,YAAY,CAAC,CAAC;YACzC,QAAQ,CAAC,IAAI,CAAC,CAAC;YACf,OAAO,CAAC,CAAC;QACX,CAAC;QAAC,OAAO,CAAM,EAAE,CAAC;YAChB,QAAQ,CAAC,CAAC,CAAC,OAAO,IAAI,8BAA8B,CAAC,CAAC;YACtD,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC,EAAE,CAAC,YAAY,CAAC,CAAC,CAAC;IAEnB,SAAS,CAAC,GAAG,EAAE;QACb,IAAI,CAAC,kBAAkB;YAAE,OAAO;QAEhC,eAAe,CAAC,kBAAkB,CAAC,eAAe,EAAuB,CAAC,CAAC;QAE3E,MAAM,IAAI,GAAyB,EAAE,CAAC;QAEtC,IAAI,CAAC,IAAI,CACP,kBAAkB,CAAC,WAAW,CAC5B,kBAAkB,EAClB,CAAC,KAAuB,EAAE,EAAE;YAC1B,mBAAmB,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;QACtC,CAAC,CACF,CACF,CAAC;QACF,IAAI,CAAC,IAAI,CACP,kBAAkB,CAAC,WAAW,CAAC,oBAAoB,EAAE,CAAC,KAAK,EAAE,EAAE;YAC7D,eAAe,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC;QACtC,CAAC,CAAC,CACH,CAAC;QAEF,IAAI,OAAO,EAAE,CAAC;YACZ,IAAI,CAAC,IAAI,CACP,OAAO,CAAC,WAAW,CAAC,OAAO,EAAE,CAAC,KAAiB,EAAE,EAAE;gBACjD,eAAe,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;YACrC,CAAC,CAAC,CACH,CAAC;YACF,IAAI,CAAC,IAAI,CACP,OAAO,CAAC,WAAW,CAAC,gBAAgB,EAAE,CAAC,KAA0B,EAAE,EAAE;gBACnE,eAAe,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;gBAC5B,eAAe,CAAC,KAAK,CAAC,CAAC;YACzB,CAAC,CAAC,CACH,CAAC;YACF,IAAI,CAAC,IAAI,CACP,OAAO,CAAC,WAAW,CAAC,aAAa,EAAE,CAAC,KAAuB,EAAE,EAAE;gBAC7D,QAAQ,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;gBACtB,eAAe,CAAC,KAAK,CAAC,CAAC;YACzB,CAAC,CAAC,CACH,CAAC;YAEF,2BAA2B;YAC3B,IAAI,CAAC,IAAI,CACP,OAAO,CAAC,WAAW,CAAC,UAAU,EAAE,CAAC,KAAoB,EAAE,EAAE;gBACvD,MAAM,UAAU,GAAG,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,EAAE,CAAC;gBACtE,kBAAkB,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,GAAG,IAAI,EAAE,UAAU,CAAC,CAAC,CAAC;gBAEpD,MAAM,gBAAgB,GAAG,GAAG,EAAE;oBAC5B,kBAAkB,CAAC,CAAC,IAAI,EAAE,EAAE,CAC1B,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,KAAK,CAAC,MAAM,CAAC,CAC9C,CAAC;gBACJ,CAAC,CAAC;gBAEF,MAAM,OAAO,GAAG,eAAe,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;gBAC5D,IAAI,CAAC,OAAO,EAAE,CAAC;oBACb,gBAAgB,EAAE,CAAC;oBACnB,IAAI,CAAC;wBACH,OAAO,CAAC,cAAc,CACpB,KAAK,CAAC,MAAM,EACZ,mCAAmC,KAAK,CAAC,QAAQ,EAAE,CACpD,CAAC;oBACJ,CAAC;oBAAC,MAAM,CAAC;wBACP,yCAAyC;oBAC3C,CAAC;oBACD,OAAO;gBACT,CAAC;gBAED,IAAI,MAAuB,CAAC;gBAC5B,IAAI,CAAC;oBACH,MAAM,GAAG,OAAO,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;gBACpC,CAAC;gBAAC,OAAO,CAAM,EAAE,CAAC;oBAChB,yDAAyD;oBACzD,gBAAgB,EAAE,CAAC;oBACnB,IAAI,CAAC;wBACH,OAAO,CAAC,cAAc,CACpB,KAAK,CAAC,MAAM,EACZ,CAAC,CAAC,OAAO,IAAI,qBAAqB,CACnC,CAAC;oBACJ,CAAC;oBAAC,MAAM,CAAC;wBACP,yCAAyC;oBAC3C,CAAC;oBACD,OAAO;gBACT,CAAC;gBAED,MAAM;qBACH,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE;oBACd,gBAAgB,EAAE,CAAC;oBACnB,OAAO,CAAC,eAAe,CAAC,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;gBAC/C,CAAC,CAAC;qBACD,KAAK,CAAC,CAAC,CAAM,EAAE,EAAE;oBAChB,gBAAgB,EAAE,CAAC;oBACnB,IAAI,CAAC;wBACH,OAAO,CAAC,cAAc,CACpB,KAAK,CAAC,MAAM,EACZ,CAAC,CAAC,OAAO,IAAI,qBAAqB,CACnC,CAAC;oBACJ,CAAC;oBAAC,MAAM,CAAC;wBACP,yCAAyC;oBAC3C,CAAC;gBACH,CAAC,CAAC,CAAC;YACP,CAAC,CAAC,CACH,CAAC;QACJ,CAAC;QAED,OAAO,GAAG,EAAE;YACV,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC;YAChC,OAAO,EAAE,OAAO,EAAE,CAAC;QACrB,CAAC,CAAC;IACJ,CAAC,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC;IAEd,MAAM,OAAO,GAAG,WAAW,CACzB,KAAK,EAAE,MAAc,EAAmB,EAAE;QACxC,IAAI,CAAC,OAAO;YAAE,MAAM,IAAI,KAAK,CAAC,uBAAuB,CAAC,CAAC;QACvD,QAAQ,CAAC,IAAI,CAAC,CAAC;QACf,eAAe,CAAC,IAAI,CAAC,CAAC;QACtB,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;YAC7C,OAAO,MAAM,CAAC;QAChB,CAAC;QAAC,OAAO,CAAM,EAAE,CAAC;YAChB,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;YACpB,MAAM,CAAC,CAAC;QACV,CAAC;gBAAS,CAAC;YACT,eAAe,CAAC,KAAK,CAAC,CAAC;QACzB,CAAC;IACH,CAAC,EACD,CAAC,OAAO,CAAC,CACV,CAAC;IAEF,MAAM,cAAc,GAAG,WAAW,CAChC,KAAK,EAAE,MAAc,EAAiB,EAAE;QACtC,IAAI,CAAC,OAAO;YAAE,MAAM,IAAI,KAAK,CAAC,uBAAuB,CAAC,CAAC;QACvD,QAAQ,CAAC,IAAI,CAAC,CAAC;QACf,eAAe,CAAC,EAAE,CAAC,CAAC;QACpB,eAAe,CAAC,IAAI,CAAC,CAAC;QACtB,IAAI,CAAC;YACH,MAAM,OAAO,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC;QACvC,CAAC;QAAC,OAAO,CAAM,EAAE,CAAC;YAChB,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;YACpB,eAAe,CAAC,KAAK,CAAC,CAAC;YACvB,MAAM,CAAC,CAAC;QACV,CAAC;IACH,CAAC,EACD,CAAC,OAAO,CAAC,CACV,CAAC;IAEF,MAAM,YAAY,GAAG,WAAW,CAAC,KAAK,IAAmB,EAAE;QACzD,IAAI,CAAC,OAAO;YAAE,OAAO;QACrB,MAAM,OAAO,CAAC,YAAY,EAAE,CAAC;QAC7B,eAAe,CAAC,KAAK,CAAC,CAAC;IACzB,CAAC,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC;IAEd,MAAM,aAAa,GAAG,WAAW,CAAC,KAAK,IAAmB,EAAE;QAC1D,IAAI,CAAC,kBAAkB;YAAE,OAAO;QAChC,MAAM,kBAAkB,CAAC,aAAa,EAAE,CAAC;IAC3C,CAAC,EAAE,EAAE,CAAC,CAAC;IAEP,IAAI,CAAC,kBAAkB,EAAE,CAAC;QACxB,OAAO;YACL,YAAY,EAAE,aAAa;YAC3B,OAAO,EAAE,IAAI;YACb,YAAY,EAAE,KAAK;YACnB,YAAY,EAAE,EAAE;YAChB,gBAAgB,EAAE,IAAI;YACtB,KAAK,EAAE,IAAI;YACX,eAAe,EAAE,EAAE;SACpB,CAAC;IACJ,CAAC;IAED,OAAO;QACL,YAAY;QACZ,OAAO;QACP,YAAY;QACZ,YAAY;QACZ,gBAAgB;QAChB,KAAK;QACL,eAAe;QACf,OAAO,EAAE,OAAO,IAAI,YAAY,KAAK,WAAW,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS;QACtE,cAAc,EACZ,OAAO,IAAI,YAAY,KAAK,WAAW,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,SAAS;QACtE,YAAY,EACV,OAAO,IAAI,YAAY,KAAK,WAAW,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,SAAS;QACpE,aAAa;KACd,CAAC;AACJ,CAAC","sourcesContent":["import { useEffect, useState, useCallback, useMemo, useRef } from \"react\";\n\nimport type {\n ModelAvailability,\n SessionConfig,\n TokenEvent,\n StreamCompleteEvent,\n StreamErrorEvent,\n DownloadProgress,\n ToolDefinition,\n ToolCallEvent,\n ActiveToolCall,\n} from \"./ExpoLocalLlm.types\";\nimport ExpoLocalLlmModule from \"./ExpoLocalLlmModule\";\nimport { createLLMSession, LLMSession } from \"./LLMSession\";\n\ntype UseLocalLLMOptions = SessionConfig;\n\ntype UseLocalLLMResult = {\n availability: ModelAvailability;\n session: LLMSession | null;\n isGenerating: boolean;\n streamedText: string;\n downloadProgress: number | null;\n error: string | null;\n activeToolCalls: ActiveToolCall[];\n respond?: (prompt: string) => Promise<string>;\n streamResponse?: (prompt: string) => Promise<void>;\n cancelStream?: () => Promise<void>;\n downloadModel?: () => Promise<void>;\n};\n\nexport function useLocalLLM(\n options: UseLocalLLMOptions = {}\n): UseLocalLLMResult {\n const [availability, setAvailability] =\n useState<ModelAvailability>(\"unknown\");\n const [isGenerating, setIsGenerating] = useState(false);\n const [streamedText, setStreamedText] = useState(\"\");\n const [downloadProgress, setDownloadProgress] = useState<number | null>(null);\n const [error, setError] = useState<string | null>(null);\n const [activeToolCalls, setActiveToolCalls] = useState<ActiveToolCall[]>([]);\n\n // Keep a ref to the current tool handlers so the event listener\n // always sees the latest handlers without needing to recreate the session.\n const toolHandlersRef = useRef<Map<string, ToolDefinition[\"handler\"]>>(\n new Map()\n );\n\n // Update handler ref whenever tools change\n useEffect(() => {\n const handlers = new Map<string, ToolDefinition[\"handler\"]>();\n if (options.tools) {\n for (const tool of options.tools) {\n handlers.set(tool.name, tool.handler);\n }\n }\n toolHandlersRef.current = handlers;\n }, [options.tools]);\n\n // Stabilize options to avoid unnecessary session recreation\n // Include tool names/descriptions so session recreates when tools change structurally\n const toolsFingerprint = useMemo(\n () =>\n options.tools\n ?.map((t) => `${t.name}:${t.description}:${JSON.stringify(t.parameters)}`)\n .join(\"|\") ?? \"\",\n [options.tools]\n );\n\n // Fingerprint the schema so session recreates when it changes\n const schemaFingerprint = useMemo(\n () => (options.schema ? JSON.stringify(options.schema) : \"\"),\n [options.schema]\n );\n\n const stableConfig = useMemo(\n () => options,\n [\n options.instructions,\n options.options?.temperature,\n options.options?.maxTokens,\n options.options?.topK,\n options.toolTimeout,\n options.responseFormat,\n schemaFingerprint,\n toolsFingerprint,\n ]\n );\n\n const session = useMemo(() => {\n if (!ExpoLocalLlmModule) return null;\n try {\n const s = createLLMSession(stableConfig);\n setError(null);\n return s;\n } catch (e: any) {\n setError(e.message ?? \"Failed to create LLM session\");\n return null;\n }\n }, [stableConfig]);\n\n useEffect(() => {\n if (!ExpoLocalLlmModule) return;\n\n setAvailability(ExpoLocalLlmModule.getAvailability() as ModelAvailability);\n\n const subs: { remove(): void }[] = [];\n\n subs.push(\n ExpoLocalLlmModule.addListener(\n \"downloadProgress\",\n (event: DownloadProgress) => {\n setDownloadProgress(event.progress);\n }\n )\n );\n subs.push(\n ExpoLocalLlmModule.addListener(\"availabilityChange\", (event) => {\n setAvailability(event.availability);\n })\n );\n\n if (session) {\n subs.push(\n session.addListener(\"token\", (event: TokenEvent) => {\n setStreamedText(event.accumulated);\n })\n );\n subs.push(\n session.addListener(\"streamComplete\", (event: StreamCompleteEvent) => {\n setStreamedText(event.text);\n setIsGenerating(false);\n })\n );\n subs.push(\n session.addListener(\"streamError\", (event: StreamErrorEvent) => {\n setError(event.error);\n setIsGenerating(false);\n })\n );\n\n // Tool call event listener\n subs.push(\n session.addListener(\"toolCall\", (event: ToolCallEvent) => {\n const activeCall = { callId: event.callId, toolName: event.toolName };\n setActiveToolCalls((prev) => [...prev, activeCall]);\n\n const removeActiveCall = () => {\n setActiveToolCalls((prev) =>\n prev.filter((c) => c.callId !== event.callId)\n );\n };\n\n const handler = toolHandlersRef.current.get(event.toolName);\n if (!handler) {\n removeActiveCall();\n try {\n session.rejectToolCall(\n event.callId,\n `No handler registered for tool: ${event.toolName}`\n );\n } catch {\n // Ignore if session is already torn down\n }\n return;\n }\n\n let result: Promise<string>;\n try {\n result = handler(event.arguments);\n } catch (e: any) {\n // Handler threw synchronously before returning a promise\n removeActiveCall();\n try {\n session.rejectToolCall(\n event.callId,\n e.message || \"Tool handler failed\"\n );\n } catch {\n // Ignore if session is already torn down\n }\n return;\n }\n\n result\n .then((value) => {\n removeActiveCall();\n session.resolveToolCall(event.callId, value);\n })\n .catch((e: any) => {\n removeActiveCall();\n try {\n session.rejectToolCall(\n event.callId,\n e.message || \"Tool handler failed\"\n );\n } catch {\n // Ignore if session is already torn down\n }\n });\n })\n );\n }\n\n return () => {\n subs.forEach((s) => s.remove());\n session?.release();\n };\n }, [session]);\n\n const respond = useCallback(\n async (prompt: string): Promise<string> => {\n if (!session) throw new Error(\"Session not available\");\n setError(null);\n setIsGenerating(true);\n try {\n const result = await session.respond(prompt);\n return result;\n } catch (e: any) {\n setError(e.message);\n throw e;\n } finally {\n setIsGenerating(false);\n }\n },\n [session]\n );\n\n const streamResponse = useCallback(\n async (prompt: string): Promise<void> => {\n if (!session) throw new Error(\"Session not available\");\n setError(null);\n setStreamedText(\"\");\n setIsGenerating(true);\n try {\n await session.streamResponse(prompt);\n } catch (e: any) {\n setError(e.message);\n setIsGenerating(false);\n throw e;\n }\n },\n [session]\n );\n\n const cancelStream = useCallback(async (): Promise<void> => {\n if (!session) return;\n await session.cancelStream();\n setIsGenerating(false);\n }, [session]);\n\n const downloadModel = useCallback(async (): Promise<void> => {\n if (!ExpoLocalLlmModule) return;\n await ExpoLocalLlmModule.downloadModel();\n }, []);\n\n if (!ExpoLocalLlmModule) {\n return {\n availability: \"notEligible\",\n session: null,\n isGenerating: false,\n streamedText: \"\",\n downloadProgress: null,\n error: null,\n activeToolCalls: [],\n };\n }\n\n return {\n availability,\n session,\n isGenerating,\n streamedText,\n downloadProgress,\n error,\n activeToolCalls,\n respond: session && availability === \"available\" ? respond : undefined,\n streamResponse:\n session && availability === \"available\" ? streamResponse : undefined,\n cancelStream:\n session && availability === \"available\" ? cancelStream : undefined,\n downloadModel,\n };\n}\n"]}
|
|
@@ -0,0 +1,154 @@
|
|
|
1
|
+
import Foundation
|
|
2
|
+
import FoundationModels
|
|
3
|
+
|
|
4
|
+
// MARK: - Tool errors
|
|
5
|
+
|
|
6
|
+
enum DynamicToolError: LocalizedError {
|
|
7
|
+
case timeout
|
|
8
|
+
case handlerFailed(String)
|
|
9
|
+
|
|
10
|
+
var errorDescription: String? {
|
|
11
|
+
switch self {
|
|
12
|
+
case .timeout:
|
|
13
|
+
return "Tool call timed out waiting for JavaScript response"
|
|
14
|
+
case .handlerFailed(let message):
|
|
15
|
+
return "Tool handler error: \(message)"
|
|
16
|
+
}
|
|
17
|
+
}
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
// MARK: - Generable arguments wrapper
|
|
21
|
+
|
|
22
|
+
/// A fixed @Generable struct that wraps tool parameters as a JSON string.
|
|
23
|
+
/// The tool's description tells the model what JSON shape to produce.
|
|
24
|
+
@available(iOS 26, *)
|
|
25
|
+
@Generable
|
|
26
|
+
struct DynamicArguments {
|
|
27
|
+
@Guide(description: "A JSON object string containing the tool's parameters")
|
|
28
|
+
var parametersJSON: String
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
// MARK: - DynamicTool
|
|
32
|
+
|
|
33
|
+
/// A generic Tool conformance that delegates execution back to JavaScript.
|
|
34
|
+
/// When the model invokes this tool, it serializes arguments and sends an event
|
|
35
|
+
/// to JS, then awaits a result via Swift async continuations.
|
|
36
|
+
@available(iOS 26, *)
|
|
37
|
+
final class DynamicTool: Tool {
|
|
38
|
+
let name: String
|
|
39
|
+
let description: String
|
|
40
|
+
|
|
41
|
+
typealias Arguments = DynamicArguments
|
|
42
|
+
|
|
43
|
+
/// Callback invoked when the model calls this tool.
|
|
44
|
+
/// Parameters: (callId, toolName, argumentsJSON) -> Void
|
|
45
|
+
private let onToolCall: @Sendable (String, String, String) -> Void
|
|
46
|
+
|
|
47
|
+
/// Pending continuations keyed by callId, awaiting JS results.
|
|
48
|
+
private let continuationStore: ToolContinuationStore
|
|
49
|
+
|
|
50
|
+
/// Timeout in seconds for JS to respond.
|
|
51
|
+
private let timeoutSeconds: TimeInterval
|
|
52
|
+
|
|
53
|
+
init(
|
|
54
|
+
name: String,
|
|
55
|
+
description: String,
|
|
56
|
+
parameterSchema: [String: Any],
|
|
57
|
+
timeoutSeconds: TimeInterval = 30,
|
|
58
|
+
continuationStore: ToolContinuationStore,
|
|
59
|
+
onToolCall: @Sendable @escaping (String, String, String) -> Void
|
|
60
|
+
) {
|
|
61
|
+
self.name = name
|
|
62
|
+
// Embed the parameter schema in the description so the model knows
|
|
63
|
+
// what JSON shape to produce for parametersJSON.
|
|
64
|
+
let schemaDesc = DynamicTool.buildSchemaDescription(from: parameterSchema)
|
|
65
|
+
self.description = "\(description)\n\nThe parametersJSON must be a JSON object with these fields:\n\(schemaDesc)"
|
|
66
|
+
self.onToolCall = onToolCall
|
|
67
|
+
self.continuationStore = continuationStore
|
|
68
|
+
self.timeoutSeconds = timeoutSeconds
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
func call(arguments: DynamicArguments) async throws -> String {
|
|
72
|
+
let callId = UUID().uuidString
|
|
73
|
+
|
|
74
|
+
// Send event to JS
|
|
75
|
+
onToolCall(callId, name, arguments.parametersJSON)
|
|
76
|
+
|
|
77
|
+
// Wait for JS to resolve this call
|
|
78
|
+
return try await withCheckedThrowingContinuation { continuation in
|
|
79
|
+
continuationStore.store(callId: callId, continuation: continuation)
|
|
80
|
+
|
|
81
|
+
// Schedule timeout
|
|
82
|
+
Task {
|
|
83
|
+
try? await Task.sleep(for: .seconds(timeoutSeconds))
|
|
84
|
+
if let timedOut = continuationStore.remove(callId: callId) {
|
|
85
|
+
timedOut.resume(throwing: DynamicToolError.timeout)
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
/// Builds a human-readable schema description from the parameter dictionary.
|
|
92
|
+
private static func buildSchemaDescription(from schema: [String: Any]) -> String {
|
|
93
|
+
var lines: [String] = []
|
|
94
|
+
for (key, value) in schema {
|
|
95
|
+
if let param = value as? [String: Any] {
|
|
96
|
+
let type = param["type"] as? String ?? "string"
|
|
97
|
+
let desc = param["description"] as? String ?? ""
|
|
98
|
+
var line = "- \(key) (\(type)): \(desc)"
|
|
99
|
+
if let enumValues = param["enum"] as? [String] {
|
|
100
|
+
line += " [possible values: \(enumValues.joined(separator: ", "))]"
|
|
101
|
+
}
|
|
102
|
+
lines.append(line)
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
return lines.joined(separator: "\n")
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
// MARK: - ToolContinuationStore
|
|
110
|
+
|
|
111
|
+
/// Thread-safe store for pending tool call continuations.
|
|
112
|
+
@available(iOS 26, *)
|
|
113
|
+
final class ToolContinuationStore: @unchecked Sendable {
|
|
114
|
+
private var continuations: [String: CheckedContinuation<String, Error>] = [:]
|
|
115
|
+
private let lock = NSLock()
|
|
116
|
+
|
|
117
|
+
func store(callId: String, continuation: CheckedContinuation<String, Error>) {
|
|
118
|
+
lock.lock()
|
|
119
|
+
continuations[callId] = continuation
|
|
120
|
+
lock.unlock()
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
func remove(callId: String) -> CheckedContinuation<String, Error>? {
|
|
124
|
+
lock.lock()
|
|
125
|
+
let cont = continuations.removeValue(forKey: callId)
|
|
126
|
+
lock.unlock()
|
|
127
|
+
return cont
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
/// Resolve a pending continuation with a result string.
|
|
131
|
+
func resolve(callId: String, result: String) -> Bool {
|
|
132
|
+
guard let continuation = remove(callId: callId) else { return false }
|
|
133
|
+
continuation.resume(returning: result)
|
|
134
|
+
return true
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
/// Reject a pending continuation with an error.
|
|
138
|
+
func reject(callId: String, error: Error) -> Bool {
|
|
139
|
+
guard let continuation = remove(callId: callId) else { return false }
|
|
140
|
+
continuation.resume(throwing: error)
|
|
141
|
+
return true
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
/// Cancel all pending continuations (e.g., on session teardown).
|
|
145
|
+
func cancelAll() {
|
|
146
|
+
lock.lock()
|
|
147
|
+
let pending = continuations
|
|
148
|
+
continuations.removeAll()
|
|
149
|
+
lock.unlock()
|
|
150
|
+
for (_, continuation) in pending {
|
|
151
|
+
continuation.resume(throwing: CancellationError())
|
|
152
|
+
}
|
|
153
|
+
}
|
|
154
|
+
}
|
package/ios/Exceptions.swift
CHANGED
|
@@ -17,3 +17,27 @@ class StreamException: GenericException<String> {
|
|
|
17
17
|
"Stream error: \(param ?? "Unknown")"
|
|
18
18
|
}
|
|
19
19
|
}
|
|
20
|
+
|
|
21
|
+
class ToolTimeoutException: Exception {
|
|
22
|
+
override var reason: String {
|
|
23
|
+
"Tool call timed out waiting for JavaScript response"
|
|
24
|
+
}
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
class ToolCallException: GenericException<String> {
|
|
28
|
+
override var reason: String {
|
|
29
|
+
"Tool call error: \(param ?? "Unknown")"
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
class ToolNotFoundException: GenericException<String> {
|
|
34
|
+
override var reason: String {
|
|
35
|
+
"No pending tool call found with ID: \(param ?? "Unknown")"
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
class ToolNotSupportedException: Exception {
|
|
40
|
+
override var reason: String {
|
|
41
|
+
"Tool calling is not supported on this platform"
|
|
42
|
+
}
|
|
43
|
+
}
|
|
@@ -36,6 +36,24 @@ public class ExpoLocalLlmModule: Module {
|
|
|
36
36
|
session.cancelStream()
|
|
37
37
|
}
|
|
38
38
|
|
|
39
|
+
// Tool calling support
|
|
40
|
+
Function("registerTool") { (session: LLMSession, config: ToolConfig) in
|
|
41
|
+
session.registerTool(config: config)
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
Function("unregisterTool") { (session: LLMSession, name: String) in
|
|
45
|
+
session.unregisterTool(name: name)
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
Function("resolveToolCall") { (session: LLMSession, callId: String, result: String) in
|
|
49
|
+
try session.resolveToolCall(callId: callId, result: result)
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
Function("rejectToolCall") { (session: LLMSession, callId: String, error: String) in
|
|
53
|
+
try session.rejectToolCall(callId: callId, error: error)
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
Events("token", "streamComplete", "streamError", "toolCall")
|
|
39
57
|
}
|
|
40
58
|
|
|
41
59
|
OnAppEntersForeground {
|
|
@@ -7,7 +7,7 @@ enum FoundationModelBridge {
|
|
|
7
7
|
switch availability {
|
|
8
8
|
case .available:
|
|
9
9
|
return .available
|
|
10
|
-
case .
|
|
10
|
+
case .unavailable(let reason):
|
|
11
11
|
switch reason {
|
|
12
12
|
case .deviceNotEligible:
|
|
13
13
|
return .notEligible
|
|
@@ -23,11 +23,18 @@ enum FoundationModelBridge {
|
|
|
23
23
|
}
|
|
24
24
|
}
|
|
25
25
|
|
|
26
|
-
static func createSession(instructions: String
|
|
27
|
-
if
|
|
28
|
-
|
|
26
|
+
static func createSession(instructions: String?, tools: [any Tool] = []) -> LanguageModelSession {
|
|
27
|
+
if tools.isEmpty {
|
|
28
|
+
if let instructions {
|
|
29
|
+
return LanguageModelSession(instructions: instructions)
|
|
30
|
+
}
|
|
31
|
+
return LanguageModelSession()
|
|
32
|
+
} else {
|
|
33
|
+
if let instructions {
|
|
34
|
+
return LanguageModelSession(tools: tools, instructions: instructions)
|
|
35
|
+
}
|
|
36
|
+
return LanguageModelSession(tools: tools)
|
|
29
37
|
}
|
|
30
|
-
return LanguageModelSession()
|
|
31
38
|
}
|
|
32
39
|
|
|
33
40
|
static func respond(session: Any, prompt: String) async throws -> String {
|
package/ios/LLMSession.swift
CHANGED
|
@@ -4,6 +4,11 @@ import Foundation
|
|
|
4
4
|
class LLMSession: SharedObject {
|
|
5
5
|
private var nativeSession: Any?
|
|
6
6
|
private var streamTask: Task<Void, Never>?
|
|
7
|
+
private var registeredTools: [String: Any] = [:] // name -> DynamicTool (type-erased)
|
|
8
|
+
private var continuationStore: Any? // ToolContinuationStore (type-erased for availability)
|
|
9
|
+
private var toolConfigs: [ToolConfig] = []
|
|
10
|
+
private var sessionInstructions: String?
|
|
11
|
+
private var toolTimeout: TimeInterval = 30
|
|
7
12
|
|
|
8
13
|
static func checkAvailability() -> ModelAvailability {
|
|
9
14
|
guard #available(iOS 26, *) else {
|
|
@@ -16,9 +21,134 @@ class LLMSession: SharedObject {
|
|
|
16
21
|
guard #available(iOS 26, *) else {
|
|
17
22
|
throw NotSupportedException()
|
|
18
23
|
}
|
|
19
|
-
|
|
24
|
+
|
|
25
|
+
// Build instructions, appending JSON schema guidance if structured output requested
|
|
26
|
+
var instructions = config.instructions ?? ""
|
|
27
|
+
if config.responseFormat == "json" {
|
|
28
|
+
let schemaClause: String
|
|
29
|
+
if let schema = config.schema,
|
|
30
|
+
let data = try? JSONSerialization.data(withJSONObject: schema, options: .prettyPrinted),
|
|
31
|
+
let schemaStr = String(data: data, encoding: .utf8) {
|
|
32
|
+
schemaClause = "Respond ONLY with a valid JSON object matching this schema:\n\(schemaStr)"
|
|
33
|
+
} else {
|
|
34
|
+
schemaClause = "Respond ONLY with a valid JSON object."
|
|
35
|
+
}
|
|
36
|
+
if instructions.isEmpty {
|
|
37
|
+
instructions = schemaClause
|
|
38
|
+
} else {
|
|
39
|
+
instructions += "\n\n\(schemaClause)"
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
sessionInstructions = instructions.isEmpty ? nil : instructions
|
|
43
|
+
toolTimeout = config.toolTimeout ?? 30
|
|
44
|
+
|
|
45
|
+
if let tools = config.tools, !tools.isEmpty {
|
|
46
|
+
toolConfigs = tools
|
|
47
|
+
let store = ToolContinuationStore()
|
|
48
|
+
continuationStore = store
|
|
49
|
+
rebuildTools(store: store)
|
|
50
|
+
} else {
|
|
51
|
+
nativeSession = FoundationModelBridge.createSession(instructions: sessionInstructions)
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
// MARK: - Tool Management
|
|
56
|
+
|
|
57
|
+
func registerTool(config: ToolConfig) {
|
|
58
|
+
guard #available(iOS 26, *) else { return }
|
|
59
|
+
|
|
60
|
+
toolConfigs.removeAll { $0.name == config.name }
|
|
61
|
+
toolConfigs.append(config)
|
|
62
|
+
|
|
63
|
+
let store: ToolContinuationStore
|
|
64
|
+
if let existing = continuationStore as? ToolContinuationStore {
|
|
65
|
+
store = existing
|
|
66
|
+
} else {
|
|
67
|
+
store = ToolContinuationStore()
|
|
68
|
+
continuationStore = store
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
rebuildTools(store: store)
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
func unregisterTool(name: String) {
|
|
75
|
+
guard #available(iOS 26, *) else { return }
|
|
76
|
+
|
|
77
|
+
toolConfigs.removeAll { $0.name == name }
|
|
78
|
+
registeredTools.removeValue(forKey: name)
|
|
79
|
+
|
|
80
|
+
if let store = continuationStore as? ToolContinuationStore {
|
|
81
|
+
rebuildTools(store: store)
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
func resolveToolCall(callId: String, result: String) throws {
|
|
86
|
+
guard #available(iOS 26, *) else {
|
|
87
|
+
throw NotSupportedException()
|
|
88
|
+
}
|
|
89
|
+
guard let store = continuationStore as? ToolContinuationStore else {
|
|
90
|
+
throw ToolNotFoundException(callId)
|
|
91
|
+
}
|
|
92
|
+
if !store.resolve(callId: callId, result: result) {
|
|
93
|
+
throw ToolNotFoundException(callId)
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
func rejectToolCall(callId: String, error: String) throws {
|
|
98
|
+
guard #available(iOS 26, *) else {
|
|
99
|
+
throw NotSupportedException()
|
|
100
|
+
}
|
|
101
|
+
guard let store = continuationStore as? ToolContinuationStore else {
|
|
102
|
+
throw ToolNotFoundException(callId)
|
|
103
|
+
}
|
|
104
|
+
if !store.reject(callId: callId, error: DynamicToolError.handlerFailed(error)) {
|
|
105
|
+
throw ToolNotFoundException(callId)
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
// MARK: - Session Rebuild
|
|
110
|
+
|
|
111
|
+
@available(iOS 26, *)
|
|
112
|
+
private func rebuildTools(store: ToolContinuationStore) {
|
|
113
|
+
// Cancel any pending tool calls from the old session
|
|
114
|
+
store.cancelAll()
|
|
115
|
+
|
|
116
|
+
var tools: [any Tool] = []
|
|
117
|
+
registeredTools.removeAll()
|
|
118
|
+
|
|
119
|
+
for config in toolConfigs {
|
|
120
|
+
let tool = DynamicTool(
|
|
121
|
+
name: config.name,
|
|
122
|
+
description: config.description,
|
|
123
|
+
parameterSchema: config.parameters,
|
|
124
|
+
timeoutSeconds: toolTimeout,
|
|
125
|
+
continuationStore: store
|
|
126
|
+
) { [weak self] callId, toolName, argumentsJSON in
|
|
127
|
+
guard let self else { return }
|
|
128
|
+
// Parse the JSON string into a dictionary for the JS event
|
|
129
|
+
var args: [String: Any] = [:]
|
|
130
|
+
if let data = argumentsJSON.data(using: .utf8),
|
|
131
|
+
let parsed = try? JSONSerialization.jsonObject(with: data) as? [String: Any] {
|
|
132
|
+
args = parsed
|
|
133
|
+
}
|
|
134
|
+
self.emit(event: "toolCall", arguments: [
|
|
135
|
+
"callId": callId,
|
|
136
|
+
"toolName": toolName,
|
|
137
|
+
"arguments": args,
|
|
138
|
+
])
|
|
139
|
+
}
|
|
140
|
+
registeredTools[config.name] = tool
|
|
141
|
+
tools.append(tool)
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
nativeSession = FoundationModelBridge.createSession(
|
|
145
|
+
instructions: sessionInstructions,
|
|
146
|
+
tools: tools
|
|
147
|
+
)
|
|
20
148
|
}
|
|
21
149
|
|
|
150
|
+
// MARK: - Generation
|
|
151
|
+
|
|
22
152
|
func respond(to prompt: String) async throws -> String {
|
|
23
153
|
guard #available(iOS 26, *), let session = nativeSession else {
|
|
24
154
|
throw NotSupportedException()
|
|
@@ -35,18 +165,20 @@ class LLMSession: SharedObject {
|
|
|
35
165
|
|
|
36
166
|
let weakSelf = self
|
|
37
167
|
streamTask = Task {
|
|
38
|
-
var
|
|
168
|
+
var lastContent = ""
|
|
39
169
|
do {
|
|
40
170
|
let stream = FoundationModelBridge.stream(session: session, prompt: prompt)
|
|
41
|
-
for try await
|
|
42
|
-
accumulated
|
|
171
|
+
for try await content in stream {
|
|
172
|
+
// content is already the full accumulated text from Foundation Models
|
|
173
|
+
let newToken = String(content.dropFirst(lastContent.count))
|
|
174
|
+
lastContent = content
|
|
43
175
|
weakSelf.emit(event: "token", arguments: [
|
|
44
|
-
"token":
|
|
45
|
-
"accumulated":
|
|
176
|
+
"token": newToken,
|
|
177
|
+
"accumulated": content
|
|
46
178
|
])
|
|
47
179
|
}
|
|
48
180
|
weakSelf.emit(event: "streamComplete", arguments: [
|
|
49
|
-
"text":
|
|
181
|
+
"text": lastContent
|
|
50
182
|
])
|
|
51
183
|
} catch {
|
|
52
184
|
if Task.isCancelled { return }
|
|
@@ -64,5 +196,8 @@ class LLMSession: SharedObject {
|
|
|
64
196
|
|
|
65
197
|
deinit {
|
|
66
198
|
streamTask?.cancel()
|
|
199
|
+
if #available(iOS 26, *), let store = continuationStore as? ToolContinuationStore {
|
|
200
|
+
store.cancelAll()
|
|
201
|
+
}
|
|
67
202
|
}
|
|
68
203
|
}
|
package/ios/Records.swift
CHANGED
|
@@ -3,6 +3,10 @@ import ExpoModulesCore
|
|
|
3
3
|
struct SessionConfig: Record {
|
|
4
4
|
@Field var instructions: String?
|
|
5
5
|
@Field var options: GenerationOptions?
|
|
6
|
+
@Field var tools: [ToolConfig]?
|
|
7
|
+
@Field var toolTimeout: Double?
|
|
8
|
+
@Field var responseFormat: String?
|
|
9
|
+
@Field var schema: [String: Any]?
|
|
6
10
|
}
|
|
7
11
|
|
|
8
12
|
struct GenerationOptions: Record {
|
|
@@ -10,3 +14,15 @@ struct GenerationOptions: Record {
|
|
|
10
14
|
@Field var maxTokens: Int?
|
|
11
15
|
@Field var topK: Int?
|
|
12
16
|
}
|
|
17
|
+
|
|
18
|
+
struct ToolConfig: Record {
|
|
19
|
+
@Field var name: String = ""
|
|
20
|
+
@Field var description: String = ""
|
|
21
|
+
@Field var parameters: [String: Any] = [:]
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
struct ToolCallEvent: Record {
|
|
25
|
+
@Field var callId: String = ""
|
|
26
|
+
@Field var toolName: String = ""
|
|
27
|
+
@Field var arguments: [String: Any] = [:]
|
|
28
|
+
}
|
package/package.json
CHANGED
|
@@ -13,9 +13,51 @@ export type GenerationOptions = {
|
|
|
13
13
|
topK?: number;
|
|
14
14
|
};
|
|
15
15
|
|
|
16
|
+
// Tool calling types
|
|
17
|
+
|
|
18
|
+
export type ToolParameterType = "string" | "number" | "boolean";
|
|
19
|
+
|
|
20
|
+
export type ToolParameter = {
|
|
21
|
+
type: ToolParameterType;
|
|
22
|
+
description: string;
|
|
23
|
+
enum?: string[];
|
|
24
|
+
};
|
|
25
|
+
|
|
26
|
+
export type ToolDefinition = {
|
|
27
|
+
name: string;
|
|
28
|
+
description: string;
|
|
29
|
+
parameters: Record<string, ToolParameter>;
|
|
30
|
+
handler: (args: Record<string, any>) => Promise<string>;
|
|
31
|
+
};
|
|
32
|
+
|
|
33
|
+
export type ToolConfig = {
|
|
34
|
+
name: string;
|
|
35
|
+
description: string;
|
|
36
|
+
parameters: Record<string, ToolParameter>;
|
|
37
|
+
};
|
|
38
|
+
|
|
39
|
+
export type ToolCallEvent = {
|
|
40
|
+
callId: string;
|
|
41
|
+
toolName: string;
|
|
42
|
+
arguments: Record<string, any>;
|
|
43
|
+
};
|
|
44
|
+
|
|
45
|
+
export type ActiveToolCall = {
|
|
46
|
+
callId: string;
|
|
47
|
+
toolName: string;
|
|
48
|
+
};
|
|
49
|
+
|
|
50
|
+
// Structured output types
|
|
51
|
+
|
|
52
|
+
export type ResponseFormat = "text" | "json";
|
|
53
|
+
|
|
16
54
|
export type SessionConfig = {
|
|
17
55
|
instructions?: string;
|
|
18
56
|
options?: GenerationOptions;
|
|
57
|
+
tools?: ToolDefinition[];
|
|
58
|
+
toolTimeout?: number;
|
|
59
|
+
responseFormat?: ResponseFormat;
|
|
60
|
+
schema?: Record<string, any>;
|
|
19
61
|
};
|
|
20
62
|
|
|
21
63
|
export type TokenEvent = {
|
|
@@ -39,6 +81,7 @@ export type LLMSessionEvents = {
|
|
|
39
81
|
token: (event: TokenEvent) => void;
|
|
40
82
|
streamComplete: (event: StreamCompleteEvent) => void;
|
|
41
83
|
streamError: (event: StreamErrorEvent) => void;
|
|
84
|
+
toolCall: (event: ToolCallEvent) => void;
|
|
42
85
|
};
|
|
43
86
|
|
|
44
87
|
export type ExpoLocalLlmModuleEvents = {
|
package/src/LLMSession.ts
CHANGED
|
@@ -1,6 +1,10 @@
|
|
|
1
1
|
import { SharedObject, UnavailabilityError } from "expo-modules-core";
|
|
2
2
|
|
|
3
|
-
import type {
|
|
3
|
+
import type {
|
|
4
|
+
SessionConfig,
|
|
5
|
+
LLMSessionEvents,
|
|
6
|
+
ToolConfig,
|
|
7
|
+
} from "./ExpoLocalLlm.types";
|
|
4
8
|
import ExpoLocalLlmModule from "./ExpoLocalLlmModule";
|
|
5
9
|
|
|
6
10
|
export class LLMSession extends SharedObject<LLMSessionEvents> {
|
|
@@ -9,11 +13,21 @@ export class LLMSession extends SharedObject<LLMSessionEvents> {
|
|
|
9
13
|
respond!: (prompt: string) => Promise<string>;
|
|
10
14
|
streamResponse!: (prompt: string) => Promise<void>;
|
|
11
15
|
cancelStream!: () => Promise<void>;
|
|
16
|
+
registerTool!: (config: ToolConfig) => void;
|
|
17
|
+
unregisterTool!: (name: string) => void;
|
|
18
|
+
resolveToolCall!: (callId: string, result: string) => void;
|
|
19
|
+
rejectToolCall!: (callId: string, error: string) => void;
|
|
12
20
|
}
|
|
13
21
|
|
|
14
22
|
export function createLLMSession(config: SessionConfig = {}): LLMSession {
|
|
15
23
|
if (!ExpoLocalLlmModule) {
|
|
16
24
|
throw new UnavailabilityError("ExpoLocalLlm", "createLLMSession");
|
|
17
25
|
}
|
|
18
|
-
|
|
26
|
+
|
|
27
|
+
// Strip handler functions before passing to native — native doesn't need them
|
|
28
|
+
const nativeConfig = {
|
|
29
|
+
...config,
|
|
30
|
+
tools: config.tools?.map(({ handler, ...rest }) => rest),
|
|
31
|
+
};
|
|
32
|
+
return new ExpoLocalLlmModule.LLMSession(nativeConfig) as LLMSession;
|
|
19
33
|
}
|
package/src/useLocalLLM.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { useEffect, useState, useCallback, useMemo } from "react";
|
|
1
|
+
import { useEffect, useState, useCallback, useMemo, useRef } from "react";
|
|
2
2
|
|
|
3
3
|
import type {
|
|
4
4
|
ModelAvailability,
|
|
@@ -7,6 +7,9 @@ import type {
|
|
|
7
7
|
StreamCompleteEvent,
|
|
8
8
|
StreamErrorEvent,
|
|
9
9
|
DownloadProgress,
|
|
10
|
+
ToolDefinition,
|
|
11
|
+
ToolCallEvent,
|
|
12
|
+
ActiveToolCall,
|
|
10
13
|
} from "./ExpoLocalLlm.types";
|
|
11
14
|
import ExpoLocalLlmModule from "./ExpoLocalLlmModule";
|
|
12
15
|
import { createLLMSession, LLMSession } from "./LLMSession";
|
|
@@ -20,6 +23,7 @@ type UseLocalLLMResult = {
|
|
|
20
23
|
streamedText: string;
|
|
21
24
|
downloadProgress: number | null;
|
|
22
25
|
error: string | null;
|
|
26
|
+
activeToolCalls: ActiveToolCall[];
|
|
23
27
|
respond?: (prompt: string) => Promise<string>;
|
|
24
28
|
streamResponse?: (prompt: string) => Promise<void>;
|
|
25
29
|
cancelStream?: () => Promise<void>;
|
|
@@ -35,8 +39,41 @@ export function useLocalLLM(
|
|
|
35
39
|
const [streamedText, setStreamedText] = useState("");
|
|
36
40
|
const [downloadProgress, setDownloadProgress] = useState<number | null>(null);
|
|
37
41
|
const [error, setError] = useState<string | null>(null);
|
|
42
|
+
const [activeToolCalls, setActiveToolCalls] = useState<ActiveToolCall[]>([]);
|
|
43
|
+
|
|
44
|
+
// Keep a ref to the current tool handlers so the event listener
|
|
45
|
+
// always sees the latest handlers without needing to recreate the session.
|
|
46
|
+
const toolHandlersRef = useRef<Map<string, ToolDefinition["handler"]>>(
|
|
47
|
+
new Map()
|
|
48
|
+
);
|
|
49
|
+
|
|
50
|
+
// Update handler ref whenever tools change
|
|
51
|
+
useEffect(() => {
|
|
52
|
+
const handlers = new Map<string, ToolDefinition["handler"]>();
|
|
53
|
+
if (options.tools) {
|
|
54
|
+
for (const tool of options.tools) {
|
|
55
|
+
handlers.set(tool.name, tool.handler);
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
toolHandlersRef.current = handlers;
|
|
59
|
+
}, [options.tools]);
|
|
38
60
|
|
|
39
61
|
// Stabilize options to avoid unnecessary session recreation
|
|
62
|
+
// Include tool names/descriptions so session recreates when tools change structurally
|
|
63
|
+
const toolsFingerprint = useMemo(
|
|
64
|
+
() =>
|
|
65
|
+
options.tools
|
|
66
|
+
?.map((t) => `${t.name}:${t.description}:${JSON.stringify(t.parameters)}`)
|
|
67
|
+
.join("|") ?? "",
|
|
68
|
+
[options.tools]
|
|
69
|
+
);
|
|
70
|
+
|
|
71
|
+
// Fingerprint the schema so session recreates when it changes
|
|
72
|
+
const schemaFingerprint = useMemo(
|
|
73
|
+
() => (options.schema ? JSON.stringify(options.schema) : ""),
|
|
74
|
+
[options.schema]
|
|
75
|
+
);
|
|
76
|
+
|
|
40
77
|
const stableConfig = useMemo(
|
|
41
78
|
() => options,
|
|
42
79
|
[
|
|
@@ -44,14 +81,21 @@ export function useLocalLLM(
|
|
|
44
81
|
options.options?.temperature,
|
|
45
82
|
options.options?.maxTokens,
|
|
46
83
|
options.options?.topK,
|
|
84
|
+
options.toolTimeout,
|
|
85
|
+
options.responseFormat,
|
|
86
|
+
schemaFingerprint,
|
|
87
|
+
toolsFingerprint,
|
|
47
88
|
]
|
|
48
89
|
);
|
|
49
90
|
|
|
50
91
|
const session = useMemo(() => {
|
|
51
92
|
if (!ExpoLocalLlmModule) return null;
|
|
52
93
|
try {
|
|
53
|
-
|
|
94
|
+
const s = createLLMSession(stableConfig);
|
|
95
|
+
setError(null);
|
|
96
|
+
return s;
|
|
54
97
|
} catch (e: any) {
|
|
98
|
+
setError(e.message ?? "Failed to create LLM session");
|
|
55
99
|
return null;
|
|
56
100
|
}
|
|
57
101
|
}, [stableConfig]);
|
|
@@ -84,7 +128,8 @@ export function useLocalLLM(
|
|
|
84
128
|
})
|
|
85
129
|
);
|
|
86
130
|
subs.push(
|
|
87
|
-
session.addListener("streamComplete", (
|
|
131
|
+
session.addListener("streamComplete", (event: StreamCompleteEvent) => {
|
|
132
|
+
setStreamedText(event.text);
|
|
88
133
|
setIsGenerating(false);
|
|
89
134
|
})
|
|
90
135
|
);
|
|
@@ -94,6 +139,68 @@ export function useLocalLLM(
|
|
|
94
139
|
setIsGenerating(false);
|
|
95
140
|
})
|
|
96
141
|
);
|
|
142
|
+
|
|
143
|
+
// Tool call event listener
|
|
144
|
+
subs.push(
|
|
145
|
+
session.addListener("toolCall", (event: ToolCallEvent) => {
|
|
146
|
+
const activeCall = { callId: event.callId, toolName: event.toolName };
|
|
147
|
+
setActiveToolCalls((prev) => [...prev, activeCall]);
|
|
148
|
+
|
|
149
|
+
const removeActiveCall = () => {
|
|
150
|
+
setActiveToolCalls((prev) =>
|
|
151
|
+
prev.filter((c) => c.callId !== event.callId)
|
|
152
|
+
);
|
|
153
|
+
};
|
|
154
|
+
|
|
155
|
+
const handler = toolHandlersRef.current.get(event.toolName);
|
|
156
|
+
if (!handler) {
|
|
157
|
+
removeActiveCall();
|
|
158
|
+
try {
|
|
159
|
+
session.rejectToolCall(
|
|
160
|
+
event.callId,
|
|
161
|
+
`No handler registered for tool: ${event.toolName}`
|
|
162
|
+
);
|
|
163
|
+
} catch {
|
|
164
|
+
// Ignore if session is already torn down
|
|
165
|
+
}
|
|
166
|
+
return;
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
let result: Promise<string>;
|
|
170
|
+
try {
|
|
171
|
+
result = handler(event.arguments);
|
|
172
|
+
} catch (e: any) {
|
|
173
|
+
// Handler threw synchronously before returning a promise
|
|
174
|
+
removeActiveCall();
|
|
175
|
+
try {
|
|
176
|
+
session.rejectToolCall(
|
|
177
|
+
event.callId,
|
|
178
|
+
e.message || "Tool handler failed"
|
|
179
|
+
);
|
|
180
|
+
} catch {
|
|
181
|
+
// Ignore if session is already torn down
|
|
182
|
+
}
|
|
183
|
+
return;
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
result
|
|
187
|
+
.then((value) => {
|
|
188
|
+
removeActiveCall();
|
|
189
|
+
session.resolveToolCall(event.callId, value);
|
|
190
|
+
})
|
|
191
|
+
.catch((e: any) => {
|
|
192
|
+
removeActiveCall();
|
|
193
|
+
try {
|
|
194
|
+
session.rejectToolCall(
|
|
195
|
+
event.callId,
|
|
196
|
+
e.message || "Tool handler failed"
|
|
197
|
+
);
|
|
198
|
+
} catch {
|
|
199
|
+
// Ignore if session is already torn down
|
|
200
|
+
}
|
|
201
|
+
});
|
|
202
|
+
})
|
|
203
|
+
);
|
|
97
204
|
}
|
|
98
205
|
|
|
99
206
|
return () => {
|
|
@@ -156,6 +263,7 @@ export function useLocalLLM(
|
|
|
156
263
|
streamedText: "",
|
|
157
264
|
downloadProgress: null,
|
|
158
265
|
error: null,
|
|
266
|
+
activeToolCalls: [],
|
|
159
267
|
};
|
|
160
268
|
}
|
|
161
269
|
|
|
@@ -166,6 +274,7 @@ export function useLocalLLM(
|
|
|
166
274
|
streamedText,
|
|
167
275
|
downloadProgress,
|
|
168
276
|
error,
|
|
277
|
+
activeToolCalls,
|
|
169
278
|
respond: session && availability === "available" ? respond : undefined,
|
|
170
279
|
streamResponse:
|
|
171
280
|
session && availability === "available" ? streamResponse : undefined,
|