@zhexin/pi-agy 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/api.ts +127 -167
- package/src/ccpa.ts +17 -13
package/package.json
CHANGED
package/src/api.ts
CHANGED
|
@@ -1,17 +1,11 @@
|
|
|
1
|
-
import { randomBytes, timingSafeEqual } from "node:crypto";
|
|
2
|
-
import { once } from "node:events";
|
|
3
|
-
import { createServer, type IncomingMessage, type ServerResponse } from "node:http";
|
|
4
|
-
import { createInterface } from "node:readline";
|
|
5
|
-
import { Readable } from "node:stream";
|
|
6
|
-
|
|
7
1
|
import {
|
|
8
2
|
clampThinkingLevel,
|
|
9
3
|
lazyStream,
|
|
4
|
+
type FetchFunction,
|
|
10
5
|
type GoogleOptions,
|
|
11
6
|
type Model,
|
|
12
7
|
type ModelThinkingLevel,
|
|
13
8
|
type ProviderStreams,
|
|
14
|
-
type SimpleStreamOptions,
|
|
15
9
|
type StreamOptions,
|
|
16
10
|
type TranscriptContext,
|
|
17
11
|
} from "@earendil-works/pi-ai";
|
|
@@ -19,159 +13,128 @@ import { googleGenerativeAIApi } from "@earendil-works/pi-ai/compat";
|
|
|
19
13
|
|
|
20
14
|
import { endpointUrl, PROJECT_HEADER, USER_AGENT } from "./ccpa.ts";
|
|
21
15
|
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
let size = 0;
|
|
25
|
-
for await (const chunk of request) {
|
|
26
|
-
size += chunk.length;
|
|
27
|
-
if (size > 64 * 1024 * 1024) throw new Error("AGY request exceeds 64 MiB.");
|
|
28
|
-
chunks.push(chunk);
|
|
29
|
-
}
|
|
30
|
-
return JSON.parse(Buffer.concat(chunks).toString("utf8"));
|
|
31
|
-
}
|
|
32
|
-
|
|
33
|
-
async function unwrap(
|
|
34
|
-
upstream: Response,
|
|
35
|
-
response: ServerResponse,
|
|
36
|
-
signal: AbortSignal,
|
|
37
|
-
): Promise<void> {
|
|
38
|
-
if (!upstream.body) throw new Error("AGY returned an empty response body.");
|
|
39
|
-
const input = Readable.fromWeb(upstream.body).setEncoding("utf8");
|
|
40
|
-
const lines = createInterface({ input, crlfDelay: Infinity });
|
|
16
|
+
function createSseUnwrapTransform(): TransformStream<string, string> {
|
|
17
|
+
let lineBuffer = "";
|
|
41
18
|
let data: string[] = [];
|
|
42
|
-
|
|
19
|
+
|
|
20
|
+
const flushEvent = (controller: TransformStreamDefaultController<string>) => {
|
|
43
21
|
if (!data.length) return;
|
|
44
22
|
const text = data.join("\n");
|
|
45
23
|
data = [];
|
|
46
24
|
if (text === "[DONE]") return;
|
|
47
25
|
const event = JSON.parse(text) as { response?: unknown };
|
|
48
|
-
|
|
49
|
-
await once(response, "drain", { signal });
|
|
50
|
-
}
|
|
26
|
+
controller.enqueue(`data: ${JSON.stringify(event.response ?? event)}\n\n`);
|
|
51
27
|
};
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
28
|
+
|
|
29
|
+
const processLine = (line: string, controller: TransformStreamDefaultController<string>) => {
|
|
30
|
+
const normalized = line.endsWith("\r") ? line.slice(0, -1) : line;
|
|
31
|
+
if (!normalized) flushEvent(controller);
|
|
32
|
+
else if (normalized.startsWith("data:")) data.push(normalized.slice(5).replace(/^ /, ""));
|
|
33
|
+
};
|
|
34
|
+
|
|
35
|
+
return new TransformStream<string, string>({
|
|
36
|
+
transform(chunk, controller) {
|
|
37
|
+
lineBuffer += chunk;
|
|
38
|
+
const lines = lineBuffer.split("\n");
|
|
39
|
+
lineBuffer = lines.pop() ?? "";
|
|
40
|
+
for (const line of lines) processLine(line, controller);
|
|
41
|
+
},
|
|
42
|
+
flush(controller) {
|
|
43
|
+
if (lineBuffer) processLine(lineBuffer, controller);
|
|
44
|
+
flushEvent(controller);
|
|
45
|
+
},
|
|
46
|
+
});
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
async function parseRequestBody(
|
|
50
|
+
input: Parameters<FetchFunction>[0],
|
|
51
|
+
init?: RequestInit,
|
|
52
|
+
): Promise<unknown> {
|
|
53
|
+
if (typeof init?.body === "string") return JSON.parse(init.body);
|
|
54
|
+
if (init?.body != null) return new Response(init.body).json();
|
|
55
|
+
if (input instanceof Request) return input.json();
|
|
56
|
+
throw new Error("AGY request is missing a JSON body.");
|
|
63
57
|
}
|
|
64
58
|
|
|
65
|
-
|
|
59
|
+
function createAgyFetch(
|
|
66
60
|
model: Model<string>,
|
|
67
61
|
backendId: string,
|
|
68
62
|
access: string,
|
|
69
63
|
project: string,
|
|
70
64
|
endpoint: string,
|
|
71
65
|
options: StreamOptions,
|
|
72
|
-
) {
|
|
73
|
-
const
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
typeof
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
)
|
|
82
|
-
|
|
83
|
-
return;
|
|
84
|
-
}
|
|
85
|
-
const url = new URL(request.url ?? "/", "http://127.0.0.1");
|
|
86
|
-
const match = /^\/models\/([^/]+):streamGenerateContent$/.exec(url.pathname);
|
|
87
|
-
if (request.method !== "POST" || !match?.[1]) {
|
|
88
|
-
response.writeHead(404).end();
|
|
89
|
-
return;
|
|
66
|
+
): FetchFunction {
|
|
67
|
+
const baseFetch = options.fetch ?? fetch;
|
|
68
|
+
|
|
69
|
+
return async (input, init) => {
|
|
70
|
+
const rawUrl =
|
|
71
|
+
typeof input === "string" ? input : input instanceof URL ? input.toString() : input.url;
|
|
72
|
+
const url = new URL(rawUrl);
|
|
73
|
+
const method = (
|
|
74
|
+
init?.method ?? (input instanceof Request ? input.method : "GET")
|
|
75
|
+
).toUpperCase();
|
|
76
|
+
if (method !== "POST" || !/^\/models\/([^/]+):streamGenerateContent$/.test(url.pathname)) {
|
|
77
|
+
return new Response(null, { status: 404 });
|
|
90
78
|
}
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
const
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
Authorization: `Bearer ${access}`,
|
|
105
|
-
"Content-Type": "application/json",
|
|
106
|
-
"User-Agent": USER_AGENT,
|
|
107
|
-
},
|
|
108
|
-
body: JSON.stringify({ project, model: backendId, request: body }),
|
|
109
|
-
signal,
|
|
110
|
-
redirect: "error",
|
|
79
|
+
|
|
80
|
+
const signals: AbortSignal[] = [AbortSignal.timeout(options.timeoutMs ?? 300_000)];
|
|
81
|
+
if (init?.signal) signals.push(init.signal);
|
|
82
|
+
if (options.signal && options.signal !== init?.signal) signals.push(options.signal);
|
|
83
|
+
const signal = AbortSignal.any(signals);
|
|
84
|
+
|
|
85
|
+
const body = await parseRequestBody(input, init);
|
|
86
|
+
const upstream = await baseFetch(`${endpoint}/v1internal:streamGenerateContent?alt=sse`, {
|
|
87
|
+
method: "POST",
|
|
88
|
+
headers: {
|
|
89
|
+
Authorization: `Bearer ${access}`,
|
|
90
|
+
"Content-Type": "application/json",
|
|
91
|
+
"User-Agent": USER_AGENT,
|
|
111
92
|
},
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
93
|
+
body: JSON.stringify({ project, model: backendId, request: body }),
|
|
94
|
+
signal,
|
|
95
|
+
redirect: "error",
|
|
96
|
+
});
|
|
97
|
+
|
|
98
|
+
await options
|
|
99
|
+
.onResponse?.(
|
|
115
100
|
{ status: upstream.status, headers: Object.fromEntries(upstream.headers) },
|
|
116
101
|
model,
|
|
117
|
-
)
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
(upstream.ok ? "text/event-stream" : "application/json"),
|
|
122
|
-
...(upstream.headers.has("retry-after")
|
|
123
|
-
? { "Retry-After": upstream.headers.get("retry-after")! }
|
|
124
|
-
: {}),
|
|
102
|
+
)
|
|
103
|
+
?.catch(async (error) => {
|
|
104
|
+
await upstream.body?.cancel().catch(() => {});
|
|
105
|
+
throw error;
|
|
125
106
|
});
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
code: 502,
|
|
142
|
-
message: error instanceof Error ? error.message : "AGY forwarding failed",
|
|
143
|
-
},
|
|
144
|
-
}),
|
|
145
|
-
);
|
|
107
|
+
|
|
108
|
+
if (!upstream.ok) return upstream;
|
|
109
|
+
if (!upstream.body) throw new Error("AGY returned an empty response body.");
|
|
110
|
+
|
|
111
|
+
const unwrapped = upstream.body
|
|
112
|
+
.pipeThrough(new TextDecoderStream())
|
|
113
|
+
.pipeThrough(createSseUnwrapTransform())
|
|
114
|
+
.pipeThrough(new TextEncoderStream());
|
|
115
|
+
|
|
116
|
+
return new Response(unwrapped, {
|
|
117
|
+
status: upstream.status,
|
|
118
|
+
statusText: upstream.statusText,
|
|
119
|
+
headers: {
|
|
120
|
+
"Content-Type": upstream.headers.get("content-type") ?? "text/event-stream",
|
|
121
|
+
},
|
|
146
122
|
});
|
|
147
|
-
});
|
|
148
|
-
await new Promise<void>((resolve, reject) => {
|
|
149
|
-
server.once("error", reject);
|
|
150
|
-
server.listen(0, "127.0.0.1", resolve);
|
|
151
|
-
});
|
|
152
|
-
server.unref();
|
|
153
|
-
const address = server.address();
|
|
154
|
-
if (!address || typeof address === "string") {
|
|
155
|
-
server.close();
|
|
156
|
-
throw new Error("Could not start the AGY request adapter.");
|
|
157
|
-
}
|
|
158
|
-
return {
|
|
159
|
-
key,
|
|
160
|
-
baseUrl: `http://127.0.0.1:${address.port}`,
|
|
161
|
-
close() {
|
|
162
|
-
server.close();
|
|
163
|
-
server.closeAllConnections();
|
|
164
|
-
},
|
|
165
123
|
};
|
|
166
124
|
}
|
|
167
125
|
|
|
168
126
|
const google = googleGenerativeAIApi();
|
|
169
127
|
|
|
170
|
-
function
|
|
171
|
-
|
|
128
|
+
function readGoogleThinkingLevel({ thinking }: GoogleOptions): ModelThinkingLevel {
|
|
129
|
+
if (!thinking?.enabled) return "off";
|
|
130
|
+
return (thinking.level?.toLowerCase() as ModelThinkingLevel | undefined) ?? "medium";
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
function streamAgy(
|
|
172
134
|
model: Model<string>,
|
|
173
135
|
context: TranscriptContext,
|
|
174
|
-
options:
|
|
136
|
+
options: GoogleOptions,
|
|
137
|
+
requestedLevel: ModelThinkingLevel,
|
|
175
138
|
) {
|
|
176
139
|
return lazyStream(
|
|
177
140
|
model,
|
|
@@ -180,47 +143,39 @@ function stream(
|
|
|
180
143
|
const headers = new Headers();
|
|
181
144
|
for (const [name, value] of Object.entries({ ...model.headers, ...options.headers }))
|
|
182
145
|
if (value !== null) headers.set(name, value);
|
|
146
|
+
|
|
183
147
|
const project = headers.get(PROJECT_HEADER);
|
|
184
148
|
if (!options.apiKey || !project) throw new Error("Run /login agy to authenticate.");
|
|
185
|
-
|
|
186
|
-
method === "streamSimple"
|
|
187
|
-
? (options.reasoning ?? "off")
|
|
188
|
-
: options.thinking?.enabled
|
|
189
|
-
? ((options.thinking.level?.toLowerCase() as ModelThinkingLevel | undefined) ??
|
|
190
|
-
"medium")
|
|
191
|
-
: "off";
|
|
149
|
+
|
|
192
150
|
const level = clampThinkingLevel(model, requestedLevel);
|
|
193
151
|
const backendId = model.thinkingLevelMap?.[level];
|
|
194
|
-
|
|
195
|
-
model
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
options,
|
|
201
|
-
);
|
|
152
|
+
if (!backendId) {
|
|
153
|
+
throw new Error(`AGY model "${model.id}" has no backend ID for thinking level "${level}".`);
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
const endpoint = endpointUrl(model.baseUrl);
|
|
157
|
+
const agyFetch = createAgyFetch(model, backendId, options.apiKey, project, endpoint, options);
|
|
202
158
|
const { fetch: _fetch, onResponse: _onResponse, ...innerOptions } = options;
|
|
159
|
+
|
|
203
160
|
headers.delete(PROJECT_HEADER);
|
|
204
161
|
headers.delete("authorization");
|
|
205
162
|
headers.delete("x-goog-api-key");
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
}
|
|
223
|
-
})();
|
|
163
|
+
const cleanHeaders = Object.fromEntries(headers);
|
|
164
|
+
|
|
165
|
+
const streamOptions: GoogleOptions = {
|
|
166
|
+
...innerOptions,
|
|
167
|
+
maxTokens: innerOptions.maxTokens ?? model.maxTokens,
|
|
168
|
+
// AGY selects thinking effort through the backend ID, not Gemini's thinking config.
|
|
169
|
+
thinking: { enabled: true },
|
|
170
|
+
apiKey: "agy",
|
|
171
|
+
headers: cleanHeaders,
|
|
172
|
+
fetch: agyFetch,
|
|
173
|
+
};
|
|
174
|
+
return google.stream(
|
|
175
|
+
{ ...model, baseUrl: endpoint, headers: cleanHeaders },
|
|
176
|
+
context,
|
|
177
|
+
streamOptions,
|
|
178
|
+
);
|
|
224
179
|
},
|
|
225
180
|
undefined,
|
|
226
181
|
options.signal,
|
|
@@ -228,6 +183,11 @@ function stream(
|
|
|
228
183
|
}
|
|
229
184
|
|
|
230
185
|
export const api: ProviderStreams = {
|
|
231
|
-
stream
|
|
232
|
-
|
|
186
|
+
stream(model, context, options = {}) {
|
|
187
|
+
return streamAgy(model, context, options, readGoogleThinkingLevel(options));
|
|
188
|
+
},
|
|
189
|
+
streamSimple(model, context, options = {}) {
|
|
190
|
+
const { reasoning = "off", ...googleOptions } = options;
|
|
191
|
+
return streamAgy(model, context, googleOptions, reasoning);
|
|
192
|
+
},
|
|
233
193
|
};
|
package/src/ccpa.ts
CHANGED
|
@@ -178,24 +178,28 @@ export async function fetchModels(
|
|
|
178
178
|
const chatIds = catalog.agentModelSorts.flatMap((sort) =>
|
|
179
179
|
sort.groups.flatMap((group) => group.modelIds),
|
|
180
180
|
);
|
|
181
|
-
const variants = chatIds.
|
|
181
|
+
const variants = chatIds.flatMap((id) => {
|
|
182
|
+
const match = /^(.+)-(off|minimal|low|medium|high|xhigh|max)$/.exec(id);
|
|
183
|
+
if (!match) return [];
|
|
184
|
+
|
|
182
185
|
const info = catalog.models[id]!;
|
|
183
|
-
const
|
|
184
|
-
const
|
|
185
|
-
const level = match?.[2] as ModelThinkingLevel | undefined;
|
|
186
|
+
const modelId = match[1]!;
|
|
187
|
+
const level = match[2] as ModelThinkingLevel;
|
|
186
188
|
const name = info.displayName || modelId;
|
|
187
|
-
return
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
189
|
+
return [
|
|
190
|
+
{
|
|
191
|
+
id,
|
|
192
|
+
modelId,
|
|
193
|
+
level,
|
|
194
|
+
name: name.replace(new RegExp(`\\s*\\(${level}\\)\\s*$`, "i"), ""),
|
|
195
|
+
contextWindow: info.maxTokens,
|
|
196
|
+
maxOutputTokens: info.maxOutputTokens,
|
|
197
|
+
},
|
|
198
|
+
];
|
|
195
199
|
});
|
|
196
200
|
const chatModels = [...Map.groupBy(variants, ({ modelId }) => modelId)].map(
|
|
197
201
|
([id, variants]): Model<"google-generative-ai"> => {
|
|
198
|
-
const levels = variants.
|
|
202
|
+
const levels = variants.map(({ level, id }) => [level, id]);
|
|
199
203
|
const thinkingLevelMap: ThinkingLevelMap = {
|
|
200
204
|
off: null,
|
|
201
205
|
minimal: null,
|