@charmland/pi-hyper-provider 0.4.3 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +6 -6
- package/src/compression.ts +104 -0
- package/src/index.ts +2 -1
- package/src/models.ts +3 -0
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@charmland/pi-hyper-provider",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.5.0",
|
|
4
4
|
"description": "Pi extension for Charm's Hyper inference provider",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"keywords": [
|
|
@@ -29,12 +29,12 @@
|
|
|
29
29
|
"typebox": "*"
|
|
30
30
|
},
|
|
31
31
|
"devDependencies": {
|
|
32
|
-
"@biomejs/biome": "^2.
|
|
33
|
-
"@earendil-works/pi-ai": "^1.
|
|
34
|
-
"@earendil-works/pi-coding-agent": "^1.
|
|
35
|
-
"@earendil-works/pi-tui": "^1.
|
|
32
|
+
"@biomejs/biome": "^2.6.8",
|
|
33
|
+
"@earendil-works/pi-ai": "^1.6.0",
|
|
34
|
+
"@earendil-works/pi-coding-agent": "^1.6.0",
|
|
35
|
+
"@earendil-works/pi-tui": "^1.6.0",
|
|
36
36
|
"tsx": "^4.23.15",
|
|
37
|
-
"typebox": "^1.
|
|
37
|
+
"typebox": "^1.4.7",
|
|
38
38
|
"typescript": "^7.0.2"
|
|
39
39
|
},
|
|
40
40
|
"type": "module",
|
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
import { promisify } from "node:util";
|
|
2
|
+
import { brotliCompress, constants, gzip } from "node:zlib";
|
|
3
|
+
import type { FetchFunction, ProviderStreams } from "@earendil-works/pi-ai";
|
|
4
|
+
|
|
5
|
+
// The async codecs run on libuv's threadpool, so Pi's TUI stays responsive.
|
|
6
|
+
const brotliCompressAsync = promisify(brotliCompress);
|
|
7
|
+
const gzipAsync = promisify(gzip);
|
|
8
|
+
|
|
9
|
+
// Pi re-sends the whole conversation every turn, and Hyper inflates gzip and
|
|
10
|
+
// brotli request bodies before it reads them. On real Pi requests brotli
|
|
11
|
+
// shrinks a long session about 5x and gets it to Hyper tens of milliseconds
|
|
12
|
+
// sooner. These are Crush's levels: brotli quality 5 and up doubles the CPU
|
|
13
|
+
// for a few percent fewer bytes.
|
|
14
|
+
const ENCODINGS = [
|
|
15
|
+
{
|
|
16
|
+
name: "br",
|
|
17
|
+
encode: (body: Buffer) =>
|
|
18
|
+
brotliCompressAsync(body, {
|
|
19
|
+
params: {
|
|
20
|
+
[constants.BROTLI_PARAM_MODE]: constants.BROTLI_MODE_TEXT,
|
|
21
|
+
[constants.BROTLI_PARAM_QUALITY]: 4,
|
|
22
|
+
[constants.BROTLI_PARAM_SIZE_HINT]: body.length,
|
|
23
|
+
},
|
|
24
|
+
}),
|
|
25
|
+
},
|
|
26
|
+
{ name: "gzip", encode: (body: Buffer) => gzipAsync(body, { level: 6 }) },
|
|
27
|
+
] as const;
|
|
28
|
+
|
|
29
|
+
// Compressing a few hundred bytes saves nothing and costs a header.
|
|
30
|
+
const COMPRESSED_BODY_BYTES_MIN = 1024;
|
|
31
|
+
|
|
32
|
+
/**
|
|
33
|
+
* Wraps an API implementation so its JSON request bodies reach Hyper
|
|
34
|
+
* compressed, most preferred encoding first. A 400 or 415 answer means Hyper
|
|
35
|
+
* could not read that encoding, so the request is replayed with the next one
|
|
36
|
+
* and finally uncompressed. Other statuses are never replayed: they can follow
|
|
37
|
+
* work that was already billed. The first encoding Hyper accepts is
|
|
38
|
+
* remembered, so a refusal costs one request per process, not one per turn.
|
|
39
|
+
*/
|
|
40
|
+
export function compressRequests(api: ProviderStreams): ProviderStreams {
|
|
41
|
+
// Index into ENCODINGS where requests start; ENCODINGS.length means plain.
|
|
42
|
+
let accepted = 0;
|
|
43
|
+
|
|
44
|
+
function compressing(fetch: FetchFunction | undefined): FetchFunction {
|
|
45
|
+
return async (input, init) => {
|
|
46
|
+
// Resolve at call time so Pi's dispatcher and any later override apply.
|
|
47
|
+
const send = fetch ?? globalThis.fetch;
|
|
48
|
+
const body = compressibleBody(init);
|
|
49
|
+
if (init === undefined || body === undefined) return send(input, init);
|
|
50
|
+
|
|
51
|
+
let attempted = false;
|
|
52
|
+
for (let index = accepted; index < ENCODINGS.length; index += 1) {
|
|
53
|
+
const encoding = ENCODINGS[index];
|
|
54
|
+
if (encoding === undefined) throw new Error(`no request encoding at ${index}`);
|
|
55
|
+
const encoded = await encoding.encode(body);
|
|
56
|
+
// Incompressible JSON would spend the header and Hyper's inflation for nothing.
|
|
57
|
+
if (encoded.length >= body.length) continue;
|
|
58
|
+
attempted = true;
|
|
59
|
+
const response = await send(input, withEncodedBody(init, encoded, encoding.name));
|
|
60
|
+
if (!bodyRefused(response)) {
|
|
61
|
+
accepted = index;
|
|
62
|
+
return response;
|
|
63
|
+
}
|
|
64
|
+
await response.body?.cancel();
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
// init still holds the original string body, which can be sent again.
|
|
68
|
+
const response = await send(input, init);
|
|
69
|
+
// A plain body that is refused too says nothing about encodings.
|
|
70
|
+
if (attempted && !bodyRefused(response)) accepted = ENCODINGS.length;
|
|
71
|
+
return response;
|
|
72
|
+
};
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
return {
|
|
76
|
+
...api,
|
|
77
|
+
stream: (model, context, options) => api.stream(model, context, { ...options, fetch: compressing(options?.fetch) }),
|
|
78
|
+
streamSimple: (model, context, options) =>
|
|
79
|
+
api.streamSimple(model, context, { ...options, fetch: compressing(options?.fetch) }),
|
|
80
|
+
};
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
function compressibleBody(init: RequestInit | undefined): Buffer | undefined {
|
|
84
|
+
// The OpenAI SDK serializes JSON bodies to strings; anything else is left alone.
|
|
85
|
+
if (typeof init?.body !== "string") return undefined;
|
|
86
|
+
const headers = new Headers(init.headers);
|
|
87
|
+
// Stacking a second encoding would hand Hyper layers to undo in order.
|
|
88
|
+
if (headers.has("Content-Encoding")) return undefined;
|
|
89
|
+
if (!headers.get("Content-Type")?.startsWith("application/json")) return undefined;
|
|
90
|
+
const body = Buffer.from(init.body);
|
|
91
|
+
return body.length >= COMPRESSED_BODY_BYTES_MIN ? body : undefined;
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
function withEncodedBody(init: RequestInit, body: Buffer, encoding: string): RequestInit {
|
|
95
|
+
const headers = new Headers(init.headers);
|
|
96
|
+
headers.set("Content-Encoding", encoding);
|
|
97
|
+
// fetch derives the length from the new body; a stale one would truncate it.
|
|
98
|
+
headers.delete("Content-Length");
|
|
99
|
+
return { ...init, headers, body: new Uint8Array(body) };
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
function bodyRefused(response: Response): boolean {
|
|
103
|
+
return response.status === 400 || response.status === 415;
|
|
104
|
+
}
|
package/src/index.ts
CHANGED
|
@@ -3,6 +3,7 @@ import { createProvider, envApiKeyAuth, lazyOAuth, type OAuthAuth } from "@earen
|
|
|
3
3
|
import { openAICompletionsApi } from "@earendil-works/pi-ai/compat";
|
|
4
4
|
import type { ExtensionAPI, ExtensionContext } from "@earendil-works/pi-coding-agent";
|
|
5
5
|
import { Text } from "@earendil-works/pi-tui";
|
|
6
|
+
import { compressRequests } from "./compression.js";
|
|
6
7
|
import type { CreditStatusRuntime } from "./credits.js";
|
|
7
8
|
import { HYPER_API_BASE_URL, PROVIDER_DISPLAY_NAME, PROVIDER_NAME } from "./hyper.js";
|
|
8
9
|
import { createNotifier } from "./notify.js";
|
|
@@ -192,7 +193,7 @@ export default function (pi: ExtensionAPI) {
|
|
|
192
193
|
const { fetchHyperModels } = await import("./models.js");
|
|
193
194
|
return fetchHyperModels({ signal, token });
|
|
194
195
|
},
|
|
195
|
-
api: openAICompletionsApi(),
|
|
196
|
+
api: compressRequests(openAICompletionsApi()),
|
|
196
197
|
}),
|
|
197
198
|
);
|
|
198
199
|
|
package/src/models.ts
CHANGED
|
@@ -90,6 +90,9 @@ function toProviderModel(model: ProviderModel): Model<"openai-completions"> {
|
|
|
90
90
|
supportsReasoningEffort,
|
|
91
91
|
thinkingFormat: "deepseek",
|
|
92
92
|
maxTokensField: "max_tokens",
|
|
93
|
+
// Hyper keys provider pinning and upstream prompt caches on x-session-affinity.
|
|
94
|
+
sendSessionAffinityHeaders: true,
|
|
95
|
+
sessionAffinityFormat: "openai-nosession",
|
|
93
96
|
},
|
|
94
97
|
};
|
|
95
98
|
}
|