@kenkaiiii/gg-ai 5.92.0 → 5.93.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +1 -1
- package/README.md +177 -144
- package/package.json +20 -3
package/NOTICE
CHANGED
package/README.md
CHANGED
|
@@ -1,108 +1,144 @@
|
|
|
1
|
-
|
|
1
|
+
<h1 align="center">@kenkaiiii/gg-ai</h1>
|
|
2
2
|
|
|
3
3
|
<p align="center">
|
|
4
|
-
<strong>
|
|
4
|
+
<strong>One <code>stream()</code> call for 12 LLM providers and your local models.</strong>
|
|
5
5
|
</p>
|
|
6
6
|
|
|
7
7
|
<p align="center">
|
|
8
|
-
<a href="https://www.npmjs.com/package/@kenkaiiii/gg-ai"><img src="https://img.shields.io/npm/v/@kenkaiiii/gg-ai?style=for-the-badge" alt="npm version"></a>
|
|
9
|
-
<a href="
|
|
8
|
+
<a href="https://www.npmjs.com/package/@kenkaiiii/gg-ai"><img src="https://img.shields.io/npm/v/@kenkaiiii/gg-ai?style=for-the-badge&color=blue" alt="npm version"></a>
|
|
9
|
+
<a href="https://www.npmjs.com/package/@kenkaiiii/gg-ai"><img src="https://img.shields.io/npm/dm/@kenkaiiii/gg-ai?style=for-the-badge&color=2ecc71" alt="npm downloads per month"></a>
|
|
10
|
+
<a href="https://github.com/KenKaiii/gg-framework/blob/main/LICENSE"><img src="https://img.shields.io/badge/License-AGPL--3.0-blue.svg?style=for-the-badge" alt="AGPL-3.0 License"></a>
|
|
10
11
|
</p>
|
|
11
12
|
|
|
12
|
-
|
|
13
|
+
gg-ai streams chat, tool calls and reasoning from Anthropic, OpenAI, Gemini, xAI, Moonshot, GLM, MiniMax, Xiaomi, DeepSeek, Sakana, OpenRouter, Hugging Face and local servers. To switch provider, change one string. There are no adapters or wrapper classes. It also does embeddings and reranking.
|
|
13
14
|
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
---
|
|
15
|
+
It's the provider layer under [GG Coder](https://github.com/KenKaiii/gg-framework), a desktop coding agent.
|
|
17
16
|
|
|
18
17
|
## Install
|
|
19
18
|
|
|
20
19
|
```bash
|
|
21
|
-
npm
|
|
20
|
+
npm install @kenkaiiii/gg-ai zod
|
|
22
21
|
```
|
|
23
22
|
|
|
24
|
-
|
|
23
|
+
Needs Node.js 22 or newer. Ships ESM and CommonJS builds with TypeScript types.
|
|
24
|
+
|
|
25
|
+
## Quick start
|
|
26
|
+
|
|
27
|
+
```ts
|
|
28
|
+
import { stream } from "@kenkaiiii/gg-ai";
|
|
29
|
+
|
|
30
|
+
const result = stream({
|
|
31
|
+
provider: "anthropic",
|
|
32
|
+
model: "claude-sonnet-5-5",
|
|
33
|
+
apiKey: process.env.ANTHROPIC_API_KEY,
|
|
34
|
+
messages: [{ role: "user", content: "Explain closures in one paragraph." }],
|
|
35
|
+
});
|
|
36
|
+
|
|
37
|
+
// Iterate for live events...
|
|
38
|
+
for await (const event of result) {
|
|
39
|
+
if (event.type === "text_delta") process.stdout.write(event.text);
|
|
40
|
+
}
|
|
25
41
|
|
|
26
|
-
|
|
42
|
+
// ...and await for the final message, stop reason and token usage.
|
|
43
|
+
const { message, stopReason, usage } = await result;
|
|
44
|
+
```
|
|
27
45
|
|
|
28
|
-
|
|
46
|
+
`stream()` returns one object that you can iterate (`for await`) and await. You can do either, or both.
|
|
29
47
|
|
|
30
|
-
|
|
31
|
-
- **`await`** gives you the final response (`message`, `stopReason`, `usage`)
|
|
48
|
+
## Tools
|
|
32
49
|
|
|
33
|
-
|
|
50
|
+
Tool parameters are [Zod](https://zod.dev) schemas. gg-ai converts them to each provider's JSON Schema format for you.
|
|
34
51
|
|
|
35
|
-
|
|
52
|
+
```ts
|
|
53
|
+
import { stream, type Tool } from "@kenkaiiii/gg-ai";
|
|
54
|
+
import { z } from "zod";
|
|
36
55
|
|
|
37
|
-
|
|
56
|
+
const getWeather: Tool = {
|
|
57
|
+
name: "get_weather",
|
|
58
|
+
description: "Current weather for a city",
|
|
59
|
+
parameters: z.object({ city: z.string() }),
|
|
60
|
+
};
|
|
61
|
+
|
|
62
|
+
const result = stream({
|
|
63
|
+
provider: "openai",
|
|
64
|
+
model: "gpt-6-luna",
|
|
65
|
+
apiKey,
|
|
66
|
+
messages,
|
|
67
|
+
tools: [getWeather],
|
|
68
|
+
});
|
|
69
|
+
|
|
70
|
+
for await (const event of result) {
|
|
71
|
+
if (event.type === "toolcall_done") console.log(event.name, event.args); // get_weather { city: "Paris" }
|
|
72
|
+
}
|
|
73
|
+
```
|
|
74
|
+
|
|
75
|
+
gg-ai makes one model call. To run the tools and send results back until the model is done, use [`@kenkaiiii/gg-agent`](https://www.npmjs.com/package/@kenkaiiii/gg-agent).
|
|
38
76
|
|
|
39
77
|
## Providers
|
|
40
78
|
|
|
41
|
-
| Provider |
|
|
42
|
-
| ------------- |
|
|
43
|
-
| `anthropic` | Claude Fable 5.1, Opus 5.5, Sonnet 5.5, Haiku 5.5
|
|
44
|
-
| `openai` | GPT-6 Astra, GPT-6.1 Sol, GPT-6 Luna
|
|
45
|
-
| `gemini` | Gemini 3.1 Pro (Preview), 3.8 / 3.
|
|
46
|
-
| `xai` | Grok 4.7
|
|
47
|
-
| `moonshot` | Kimi K3, K2.8 Preview, K2.7 Code, K2.7 Code HighSpeed
|
|
48
|
-
| `glm` | GLM-5.3, GLM-5.3-Flash
|
|
49
|
-
| `minimax` | MiniMax M3
|
|
50
|
-
| `xiaomi` | MiMo-V2.6-Pro, MiMo-V2.6-Flash, MiMo-V2.6-Pro-UltraSpeed
|
|
51
|
-
| `deepseek` | DeepSeek V4 Pro, V4.1 Flash
|
|
52
|
-
| `sakana` | Fugu, Fugu Max, Fugu Ultra
|
|
53
|
-
| `openrouter` | Qwen3.8 Max, or any OpenRouter model id
|
|
54
|
-
| `huggingface` | Kimi K2.7 Code, DeepSeek V4.1 Flash, GPT-OSS 120B
|
|
55
|
-
| `local` |
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
---
|
|
79
|
+
| Provider | Built-in models | Notes |
|
|
80
|
+
| ------------- | --------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------- |
|
|
81
|
+
| `anthropic` | Claude Fable 5.1, Opus 5.5, Sonnet 5.5, Haiku 5.5 | Extended thinking, prompt caching, server-side compaction |
|
|
82
|
+
| `openai` | GPT-6 Astra, GPT-6.1 Sol, GPT-6 Luna | API key, or a ChatGPT sign-in token plus `accountId` (Codex endpoint) |
|
|
83
|
+
| `gemini` | Gemini 3.1 Pro (Preview), 3.8 / 3.5 Flash, 3.5 / 3.1 Flash Lite | Code Assist sign-in by default; for an AI Studio key, set `baseUrl` to the Gemini API. Native video input |
|
|
84
|
+
| `xai` | Grok 4.7 | API key or Grok sign-in |
|
|
85
|
+
| `moonshot` | Kimi K3, K2.8 Preview, K2.7 Code, K2.7 Code HighSpeed | API key or Kimi sign-in. Native video input |
|
|
86
|
+
| `glm` | GLM-5.3, GLM-5.3-Flash | Z.AI coding endpoint |
|
|
87
|
+
| `minimax` | MiniMax M3.1 Flash, M3 | Anthropic-compatible endpoint. Native video input |
|
|
88
|
+
| `xiaomi` | MiMo-V2.6-Pro, MiMo-V2.6-Flash, MiMo-V2.6-Pro-UltraSpeed | Native video input |
|
|
89
|
+
| `deepseek` | DeepSeek V4 Pro, V4.1 Flash | |
|
|
90
|
+
| `sakana` | Fugu, Fugu Max, Fugu Ultra | |
|
|
91
|
+
| `openrouter` | Qwen3.8 Max, or any OpenRouter model id | Gateway to hundreds of models |
|
|
92
|
+
| `huggingface` | Kimi K2.7 Code, DeepSeek V4.1 Flash, GPT-OSS 120B | Inference Providers router |
|
|
93
|
+
| `local` | Whatever your server runs | Ollama, LM Studio, llama.cpp, vLLM or any OpenAI-compatible server. `baseUrl` is required |
|
|
94
|
+
|
|
95
|
+
"Built-in models" are the ones GG Coder lists (from [`@kenkaiiii/gg-core`](https://www.npmjs.com/package/@kenkaiiii/gg-core)). `model` is passed straight to the provider, so any model id the provider accepts works. Every default endpoint can be changed with `baseUrl`, and `providerRegistry.register()` adds your own provider.
|
|
60
96
|
|
|
61
97
|
## Stream events
|
|
62
98
|
|
|
63
|
-
| Event |
|
|
64
|
-
| ------------------- |
|
|
65
|
-
| `text_delta` |
|
|
66
|
-
| `thinking_delta` |
|
|
67
|
-
| `toolcall_delta` |
|
|
68
|
-
| `toolcall_done` |
|
|
69
|
-
| `server_toolcall` |
|
|
70
|
-
| `server_toolresult` |
|
|
71
|
-
| `keepalive` |
|
|
72
|
-
| `done` |
|
|
73
|
-
| `error` |
|
|
74
|
-
|
|
75
|
-
---
|
|
99
|
+
| Event | When |
|
|
100
|
+
| ------------------- | ------------------------------------------------------------ |
|
|
101
|
+
| `text_delta` | New text (`event.text`) |
|
|
102
|
+
| `thinking_delta` | New reasoning text, from providers that stream it |
|
|
103
|
+
| `toolcall_delta` | Tool call arguments as they stream in |
|
|
104
|
+
| `toolcall_done` | A complete tool call with parsed `args` |
|
|
105
|
+
| `server_toolcall` | The provider ran a server-side tool (for example web search) |
|
|
106
|
+
| `server_toolresult` | Result of that server-side tool |
|
|
107
|
+
| `keepalive` | The stream is alive but has nothing new yet |
|
|
108
|
+
| `done` | Finished, with `stopReason` |
|
|
109
|
+
| `error` | Something failed |
|
|
76
110
|
|
|
77
111
|
## Options
|
|
78
112
|
|
|
79
|
-
| Option | Type |
|
|
80
|
-
| ----------------- | ------------------------------------------------------------ |
|
|
81
|
-
| `provider` | `Provider`
|
|
82
|
-
| `model` | `string` | Required
|
|
83
|
-
| `messages` | `Message[]` | Required
|
|
84
|
-
| `
|
|
85
|
-
| `
|
|
86
|
-
| `
|
|
87
|
-
| `
|
|
88
|
-
| `
|
|
89
|
-
| `
|
|
90
|
-
| `
|
|
91
|
-
| `
|
|
92
|
-
| `
|
|
93
|
-
| `
|
|
94
|
-
| `
|
|
95
|
-
| `
|
|
96
|
-
| `
|
|
97
|
-
| `
|
|
98
|
-
| `compaction` | `boolean` | Server-side compaction (Anthropic only)
|
|
99
|
-
| `clearToolUses` | `boolean` | Server-side clearing of old tool results (Anthropic only)
|
|
100
|
-
| `fetch` | `typeof fetch` |
|
|
101
|
-
| `providerOptions` | `Record<string, unknown>` |
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
113
|
+
| Option | Type | What it does |
|
|
114
|
+
| ----------------- | ------------------------------------------------------------ | ------------------------------------------------------------------- |
|
|
115
|
+
| `provider` | `Provider` | **Required.** Any id from the providers table |
|
|
116
|
+
| `model` | `string` | **Required.** |
|
|
117
|
+
| `messages` | `Message[]` | **Required.** `system`, `user`, `assistant` and `tool` messages |
|
|
118
|
+
| `apiKey` | `string` | API key or sign-in token |
|
|
119
|
+
| `baseUrl` | `string` | Use a different endpoint |
|
|
120
|
+
| `tools` | `Tool[]` | Tools with Zod parameter schemas |
|
|
121
|
+
| `toolChoice` | `"auto" \| "none" \| "required" \| { name }` | Whether the model may, must or must not call a tool |
|
|
122
|
+
| `thinking` | `"low" \| "medium" \| "high" \| "xhigh" \| "max" \| "ultra"` | Reasoning effort. Each provider maps it to what the model supports |
|
|
123
|
+
| `maxTokens` | `number` | Output token limit |
|
|
124
|
+
| `temperature` | `number` | Sampling temperature |
|
|
125
|
+
| `topP` | `number` | Nucleus sampling |
|
|
126
|
+
| `stop` | `string[]` | Stop sequences |
|
|
127
|
+
| `signal` | `AbortSignal` | Cancel the request |
|
|
128
|
+
| `cacheRetention` | `"none" \| "short" \| "long"` | Prompt cache preference |
|
|
129
|
+
| `promptCacheKey` | `string` | Stable cache routing key (OpenAI, Moonshot, Gemini) |
|
|
130
|
+
| `webSearch` | `boolean` | The provider's own web search, where it has one |
|
|
131
|
+
| `serverTools` | `ServerToolDefinition[]` | Other server-side tools |
|
|
132
|
+
| `compaction` | `boolean` | Server-side context compaction (Anthropic only) |
|
|
133
|
+
| `clearToolUses` | `boolean` | Server-side clearing of old tool results (Anthropic only) |
|
|
134
|
+
| `fetch` | `typeof fetch` | Your own `fetch`, for example in React Native |
|
|
135
|
+
| `providerOptions` | `Record<string, unknown>` | Extra provider-specific request fields (see Provider options below) |
|
|
136
|
+
|
|
137
|
+
The full list, including less common transport settings, is the `StreamOptions` type.
|
|
138
|
+
|
|
139
|
+
### Provider options
|
|
140
|
+
|
|
141
|
+
`providerOptions` is merged into the JSON body of `stream()`, `embed()` and `rerank()` requests. It only adds fields gg-ai hasn't set itself. Core fields (`model`, `messages`, `input`, `query`, `documents`, `stream`, `tools`, `system`, `contents`, `requests`, `dimensions`, `encoding_format`, `top_n`) can't be set this way. The main use is OpenRouter's [provider routing](https://openrouter.ai/docs/features/provider-routing):
|
|
106
142
|
|
|
107
143
|
```ts
|
|
108
144
|
import { stream, type OpenRouterProviderPreferences } from "@kenkaiiii/gg-ai";
|
|
@@ -110,33 +146,41 @@ import { stream, type OpenRouterProviderPreferences } from "@kenkaiiii/gg-ai";
|
|
|
110
146
|
const result = stream({
|
|
111
147
|
provider: "openrouter",
|
|
112
148
|
model: "anthropic/claude-sonnet-4.5",
|
|
113
|
-
messages,
|
|
114
149
|
apiKey,
|
|
150
|
+
messages,
|
|
115
151
|
providerOptions: {
|
|
116
152
|
provider: { zdr: true, data_collection: "deny" } satisfies OpenRouterProviderPreferences,
|
|
117
153
|
},
|
|
118
154
|
});
|
|
119
155
|
```
|
|
120
156
|
|
|
121
|
-
|
|
157
|
+
## Errors
|
|
122
158
|
|
|
123
|
-
|
|
159
|
+
| Failure | Error |
|
|
160
|
+
| ------------------------------------------------------ | --------------------------------------------------------------------------- |
|
|
161
|
+
| HTTP 401 / 403 | `ProviderError`, `source: "auth"` |
|
|
162
|
+
| HTTP 402, or billing text such as `insufficient_quota` | `ProviderError` where `isUsageLimitError(err)` is true |
|
|
163
|
+
| HTTP 429 | `ProviderError`, `statusCode: 429`, `resetsAt` from `Retry-After` when sent |
|
|
164
|
+
| Other HTTP errors | `ProviderError` with `statusCode` and `requestId` |
|
|
165
|
+
| Connection failure | `GGAIError`, `source: "network"` |
|
|
166
|
+
| `signal` aborted | `AbortError` (`DOMException`) |
|
|
167
|
+
|
|
168
|
+
Provider error bodies are trimmed to their message, redacted and truncated, so API keys never end up in an error. `formatError(err)` turns any of these into a plain-English `headline`, `message` and `guidance` you can show a user.
|
|
169
|
+
|
|
170
|
+
gg-ai doesn't retry; that's up to the caller (`@kenkaiiii/gg-agent` does it for you). If you retry yourself, back off on 429 and 5xx, and never retry auth errors or errors where `isUsageLimitError(err)` is true.
|
|
124
171
|
|
|
125
172
|
## Embeddings and reranking
|
|
126
173
|
|
|
127
|
-
`embed()` and `rerank()` use the same
|
|
174
|
+
`embed()` and `rerank()` use the same providers as `stream()`.
|
|
128
175
|
|
|
129
176
|
```ts
|
|
130
177
|
import { embed, rerank } from "@kenkaiiii/gg-ai";
|
|
131
178
|
|
|
132
|
-
const { embeddings
|
|
133
|
-
provider: "
|
|
134
|
-
model: "
|
|
135
|
-
input: chunks, // any length;
|
|
136
|
-
dimensions: 1536,
|
|
137
|
-
normalize: true,
|
|
179
|
+
const { embeddings } = await embed({
|
|
180
|
+
provider: "openai",
|
|
181
|
+
model: "text-embedding-3-small",
|
|
182
|
+
input: chunks, // any length; batched for you
|
|
138
183
|
apiKey,
|
|
139
|
-
providerOptions: { provider: { zdr: true, data_collection: "deny", allow_fallbacks: true } },
|
|
140
184
|
});
|
|
141
185
|
// embeddings[i] is the vector for chunks[i]
|
|
142
186
|
|
|
@@ -148,60 +192,31 @@ const { results } = await rerank({
|
|
|
148
192
|
topN: 5,
|
|
149
193
|
apiKey,
|
|
150
194
|
});
|
|
151
|
-
// results: [{ index, score }], best first
|
|
195
|
+
// results: [{ index, score }], best first
|
|
152
196
|
```
|
|
153
197
|
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
- Vectors come back in input order, even when the provider returns them out of order (gg-ai sorts by `index`).
|
|
157
|
-
- Long inputs are split at the provider's per-request limit (table below; override with `batchSize`). Batches run one at a time, in order.
|
|
158
|
-
- The response is checked: exactly one vector per input, all numbers, all the same length, and `dimensions` long when you asked for that. If any check fails it throws `ProviderError`. Nothing is padded or truncated.
|
|
159
|
-
- `normalize: true` L2-normalizes every vector. It's off by default. OpenAI embeddings come back unit-length already, and so does Gemini at full 3072 dimensions. `gemini-embedding-2` also normalizes shortened vectors, but `gemini-embedding-001` doesn't: at reduced `dimensions` its vectors are not unit-length, so pass `normalize: true`. Through OpenRouter or a local server, it depends on the upstream model. If you compare with cosine similarity or a dot product and aren't sure, normalize.
|
|
160
|
-
- `inputType: "query" | "document"` maps to the provider's native task parameter: Gemini `taskType` (`RETRIEVAL_QUERY` / `RETRIEVAL_DOCUMENT`) and OpenRouter `input_type` (`search_query` / `search_document`). It's ignored by OpenAI and local servers, which have no such parameter. It's also ignored for `gemini-embedding-2`, where Google wants the task written into the text instead (`task: search result | query: …`).
|
|
161
|
-
- An empty `input` returns `{ embeddings: [] }` without making a request.
|
|
162
|
-
|
|
163
|
-
**What `rerank()` guarantees**: results are sorted by score, highest first, and each `index` points into your `documents`. At most `topN` results come back, and every index is checked. An empty `documents` returns `{ results: [] }` without making a request.
|
|
164
|
-
|
|
165
|
-
### Provider support
|
|
166
|
-
|
|
167
|
-
| Provider | `embed` | Max inputs per request | `inputType` | `rerank` |
|
|
168
|
-
| ----------------------------------------------------------------------------------------------- | ------------------------------------------------------------- | ------------------------------------------------- | ------------ | ------------------------------------------------------------------------------------------------ |
|
|
169
|
-
| `openai` | ✓ `/v1/embeddings` | 2048 | ignored | — |
|
|
170
|
-
| `openrouter` | ✓ `/api/v1/embeddings` | 64 (gg-ai default; OpenRouter documents no limit) | `input_type` | ✓ `/api/v1/rerank` |
|
|
171
|
-
| `gemini` | ✓ native `batchEmbedContents` | 100 | `taskType` | — |
|
|
172
|
-
| `local` | ✓ `{baseUrl}/embeddings` (Ollama, LM Studio, llama.cpp, vLLM) | 64 (gg-ai default) | ignored | ✓ `{baseUrl}/rerank` (llama.cpp `--rerank`, vLLM; Ollama and LM Studio have none and return 404) |
|
|
173
|
-
| `palsu` | ✓ deterministic fake | all | ignored | ✓ deterministic fake |
|
|
174
|
-
| `anthropic`, `xiaomi`, `glm`, `moonshot`, `minimax`, `deepseek`, `sakana`, `xai`, `huggingface` | — | | | — |
|
|
198
|
+
What you can rely on:
|
|
175
199
|
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
- **
|
|
181
|
-
- **
|
|
182
|
-
-
|
|
183
|
-
- Document parsing and OCR are out of scope for now. That would be a separate API (file in, pages out), and providers disagree on it much more than on embeddings.
|
|
184
|
-
|
|
185
|
-
### Errors, retries and cancellation
|
|
186
|
-
|
|
187
|
-
Failures use the same error model as `stream()`:
|
|
188
|
-
|
|
189
|
-
| Failure | Error |
|
|
190
|
-
| ------------------------------------------------------ | --------------------------------------------------------------------------- |
|
|
191
|
-
| HTTP 401 / 403 | `ProviderError`, `source: "auth"` |
|
|
192
|
-
| HTTP 402, or billing text such as `insufficient_quota` | `ProviderError` with "usage limit reached" (`isUsageLimitError()` is true) |
|
|
193
|
-
| HTTP 429 | `ProviderError`, `statusCode: 429`, `resetsAt` from `Retry-After` when sent |
|
|
194
|
-
| Other HTTP errors | `ProviderError` with `statusCode` and `requestId` |
|
|
195
|
-
| Connection failure | `GGAIError`, `source: "network"` |
|
|
196
|
-
| `signal` aborted | `AbortError` (`DOMException`); remaining batches are skipped |
|
|
200
|
+
- **Order.** Vectors come back in input order, even if the provider returns them out of order.
|
|
201
|
+
- **Batching.** Long inputs are split at the provider's limit (override with `batchSize`). Batches run one at a time.
|
|
202
|
+
- **Validation.** Exactly one vector per input, all numbers, all the same length, and `dimensions` long if you asked for that. Otherwise it throws `ProviderError`. Nothing is padded or cut.
|
|
203
|
+
- **Normalizing.** `normalize: true` scales every vector to length 1. It's off by default. If you compare vectors with cosine similarity or a dot product and aren't sure your model already returns unit vectors, turn it on.
|
|
204
|
+
- **Query vs document.** `inputType: "query" | "document"` maps to Gemini's `taskType` and OpenRouter's `input_type`. Other providers ignore it.
|
|
205
|
+
- **Reranking.** Results are sorted best first, at most `topN` come back, and every `index` points into your `documents`.
|
|
206
|
+
- **Empty input** returns an empty result without a request.
|
|
197
207
|
|
|
198
|
-
Provider
|
|
208
|
+
| Provider | `embed` | Inputs per request | `rerank` |
|
|
209
|
+
| ------------ | ------------------------------------ | ------------------ | --------------------------------- |
|
|
210
|
+
| `openai` | ✓ | 2048 | |
|
|
211
|
+
| `openrouter` | ✓ | 64 | ✓ |
|
|
212
|
+
| `gemini` | ✓ (AI Studio key, not sign-in) | 100 | |
|
|
213
|
+
| `local` | ✓ Ollama, LM Studio, llama.cpp, vLLM | 64 | ✓ llama.cpp (`--rerank`) and vLLM |
|
|
199
214
|
|
|
200
|
-
|
|
215
|
+
Other providers don't offer these endpoints. Calling them throws `GGAIError` with `source: "capability"`; check first with `providerRegistry.supports(provider, "embed")`. OpenAI embeddings need a platform API key; ChatGPT sign-in tokens can't call them.
|
|
201
216
|
|
|
202
|
-
###
|
|
217
|
+
### Your own provider
|
|
203
218
|
|
|
204
|
-
A
|
|
219
|
+
A registered provider can offer `embed` and `rerank` alongside `stream`. Your functions handle one request; gg-ai does the batching, checks, ordering and normalizing.
|
|
205
220
|
|
|
206
221
|
```ts
|
|
207
222
|
import { providerRegistry, embed, type Provider, type ProviderEntry } from "@kenkaiiii/gg-ai";
|
|
@@ -211,31 +226,49 @@ providerRegistry.register("my-search", {
|
|
|
211
226
|
throw new Error("chat not supported");
|
|
212
227
|
},
|
|
213
228
|
embedBatchSize: 128,
|
|
214
|
-
embed: async ({ model, input,
|
|
229
|
+
embed: async ({ model, input, apiKey, signal, fetch = globalThis.fetch }) => {
|
|
215
230
|
const res = await fetch("https://example.com/embed", {
|
|
216
231
|
method: "POST",
|
|
217
232
|
headers: { Authorization: `Bearer ${apiKey}`, "Content-Type": "application/json" },
|
|
218
|
-
body: JSON.stringify({ model, texts: input
|
|
233
|
+
body: JSON.stringify({ model, texts: input }),
|
|
219
234
|
signal,
|
|
220
235
|
});
|
|
221
236
|
const json = await res.json();
|
|
222
237
|
return { embeddings: json.vectors, usage: { inputTokens: json.tokens } };
|
|
223
238
|
},
|
|
224
|
-
rerank: async ({ query, documents }) => ({
|
|
225
|
-
results: documents.map((doc, index) => ({ index, score: doc.includes(query) ? 1 : 0 })),
|
|
226
|
-
}),
|
|
227
239
|
} satisfies ProviderEntry);
|
|
228
240
|
|
|
229
|
-
// `Provider` lists the built-ins, so custom names need a cast
|
|
241
|
+
// `Provider` lists the built-ins, so custom names need a cast.
|
|
230
242
|
await embed({ provider: "my-search" as Provider, model: "v1", input: ["hello"] });
|
|
231
243
|
```
|
|
232
244
|
|
|
233
|
-
|
|
245
|
+
## Testing without a network
|
|
246
|
+
|
|
247
|
+
`registerPalsuProvider()` adds a fake `palsu` provider that replays responses you queue, so tests run offline and give the same result every time.
|
|
248
|
+
|
|
249
|
+
```ts
|
|
250
|
+
import { registerPalsuProvider, palsuText, stream } from "@kenkaiiii/gg-ai";
|
|
251
|
+
|
|
252
|
+
const palsu = registerPalsuProvider();
|
|
253
|
+
palsu.appendResponses(palsuText("Hello!"));
|
|
254
|
+
|
|
255
|
+
const { message } = await stream({ provider: "palsu", model: "test", messages });
|
|
256
|
+
palsu.unregister();
|
|
257
|
+
```
|
|
258
|
+
|
|
259
|
+
`palsuToolCall()`, `palsuThinking()` and friends script other responses. `palsu` also has deterministic `embed` and `rerank`.
|
|
260
|
+
|
|
261
|
+
## The GG Framework
|
|
234
262
|
|
|
235
|
-
|
|
263
|
+
| Package | What it does |
|
|
264
|
+
| ------------------------------------------------------------------------ | ----------------------------------------------------------------- |
|
|
265
|
+
| **@kenkaiiii/gg-ai** | One streaming API for every provider (this package) |
|
|
266
|
+
| [@kenkaiiii/gg-agent](https://www.npmjs.com/package/@kenkaiiii/gg-agent) | The agent loop: runs tools and feeds results back to the model |
|
|
267
|
+
| [@kenkaiiii/gg-core](https://www.npmjs.com/package/@kenkaiiii/gg-core) | Model list, sign-in and stored credentials, local model discovery |
|
|
268
|
+
| [@kenkaiiii/ggcoder](https://www.npmjs.com/package/@kenkaiiii/ggcoder) | The GG Coder coding agent: CLI, tools and sessions |
|
|
236
269
|
|
|
237
|
-
|
|
270
|
+
Source, issues and the desktop app: [github.com/KenKaiii/gg-framework](https://github.com/KenKaiii/gg-framework).
|
|
238
271
|
|
|
239
272
|
## License
|
|
240
273
|
|
|
241
|
-
AGPL-3.0-only
|
|
274
|
+
[AGPL-3.0-only](https://github.com/KenKaiii/gg-framework/blob/main/LICENSE). You can build on it; if you distribute your changes or run them as a network service, share their source under the same license.
|
package/package.json
CHANGED
|
@@ -1,17 +1,34 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@kenkaiiii/gg-ai",
|
|
3
|
-
"version": "5.
|
|
3
|
+
"version": "5.93.0",
|
|
4
4
|
"type": "module",
|
|
5
|
-
"description": "
|
|
5
|
+
"description": "One streaming API for 12 LLM providers and local models: Anthropic, OpenAI, Gemini, xAI, Moonshot, GLM, DeepSeek, OpenRouter and more. Tool calls, reasoning, embeddings and reranking.",
|
|
6
|
+
"author": "Ken Kai",
|
|
7
|
+
"keywords": [
|
|
8
|
+
"llm",
|
|
9
|
+
"ai",
|
|
10
|
+
"streaming",
|
|
11
|
+
"anthropic",
|
|
12
|
+
"openai",
|
|
13
|
+
"gemini",
|
|
14
|
+
"openrouter",
|
|
15
|
+
"ollama",
|
|
16
|
+
"tool-calling",
|
|
17
|
+
"embeddings"
|
|
18
|
+
],
|
|
6
19
|
"license": "AGPL-3.0-only",
|
|
7
20
|
"engines": {
|
|
8
21
|
"node": ">=22"
|
|
9
22
|
},
|
|
10
23
|
"repository": {
|
|
11
24
|
"type": "git",
|
|
12
|
-
"url": "git+https://github.com/
|
|
25
|
+
"url": "git+https://github.com/KenKaiii/gg-framework.git",
|
|
13
26
|
"directory": "packages/gg-ai"
|
|
14
27
|
},
|
|
28
|
+
"homepage": "https://github.com/KenKaiii/gg-framework/tree/main/packages/gg-ai#readme",
|
|
29
|
+
"bugs": {
|
|
30
|
+
"url": "https://github.com/KenKaiii/gg-framework/issues"
|
|
31
|
+
},
|
|
15
32
|
"exports": {
|
|
16
33
|
".": {
|
|
17
34
|
"types": "./dist/index.d.ts",
|