@goodandready/dsh-ollama-cloud-provider 0.1.39 → 0.1.41
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +26 -3
- package/README.md +64 -23
- package/README.ru.md +78 -18
- package/README.zh.md +86 -17
- package/lib/adapter.js +9 -2
- package/lib/catalog-cache.js +8 -2
- package/lib/catalog-scheduler.js +34 -0
- package/lib/catalog.js +23 -14
- package/lib/client.js +331 -191
- package/lib/config.js +19 -6
- package/lib/credentials.d.ts +1 -3
- package/lib/credentials.js +6 -6
- package/lib/discovery.js +74 -6
- package/lib/index.d.ts +2 -2
- package/lib/index.js +18 -14
- package/lib/quota-model.js +5 -6
- package/lib/request-log.js +12 -3
- package/lib/routes-values.js +10 -83
- package/lib/routes.js +13 -7
- package/lib/status.js +1 -0
- package/lib/transports/ndjson.js +10 -26
- package/lib/transports/openai-chat.js +32 -3
- package/lib/transports/sse.js +10 -25
- package/lib/transports/stream-reader.js +31 -0
- package/lib/usage.js +17 -10
- package/package.json +3 -3
package/lib/transports/ndjson.js
CHANGED
|
@@ -1,30 +1,8 @@
|
|
|
1
1
|
import { bestEffortFailed } from "../best-effort.js";
|
|
2
|
+
import { readWithIdleTimeout } from "./stream-reader.js";
|
|
2
3
|
import { LlmError } from "@deepseek-ai/dsh-llm";
|
|
3
4
|
// src/transports/ndjson.ts
|
|
4
|
-
|
|
5
|
-
if (signal?.aborted === true) throw signal.reason;
|
|
6
|
-
return await new Promise((resolve, reject) => {
|
|
7
|
-
let settled = false;
|
|
8
|
-
const finish = (action) => {
|
|
9
|
-
if (settled) return;
|
|
10
|
-
settled = true;
|
|
11
|
-
clearTimeout(timer);
|
|
12
|
-
signal?.removeEventListener("abort", onAbort);
|
|
13
|
-
action();
|
|
14
|
-
};
|
|
15
|
-
const timer = setTimeout(() => {
|
|
16
|
-
reader.cancel().catch((err) => bestEffortFailed("ndjson:cancel", err));
|
|
17
|
-
finish(() => reject(new LlmError(`Ollama Cloud stream went idle for ${idleTimeoutMs}ms`, "TIMEOUT")));
|
|
18
|
-
}, idleTimeoutMs);
|
|
19
|
-
timer.unref?.();
|
|
20
|
-
const onAbort = () => {
|
|
21
|
-
reader.cancel().catch((err) => bestEffortFailed("ndjson:cancel", err));
|
|
22
|
-
finish(() => reject(signal?.reason ?? new DOMException("aborted", "AbortError")));
|
|
23
|
-
};
|
|
24
|
-
signal?.addEventListener("abort", onAbort, { once: true });
|
|
25
|
-
reader.read().then((value) => finish(() => resolve(value)), (error) => finish(() => reject(error)));
|
|
26
|
-
});
|
|
27
|
-
}
|
|
5
|
+
const MAX_FRAME_BYTES = 1e6;
|
|
28
6
|
/**
|
|
29
7
|
* Yield parsed NDJSON objects from a `/api/chat` stream. Ollama uses one JSON
|
|
30
8
|
* object per line, with no `data:` prefix, and a final `{done:true}` line.
|
|
@@ -35,9 +13,10 @@ async function* ndjsonObjects(body, idleTimeoutMs, signal) {
|
|
|
35
13
|
let buffer = "";
|
|
36
14
|
try {
|
|
37
15
|
for (;;) {
|
|
38
|
-
const result = await readWithIdleTimeout(reader, idleTimeoutMs, signal);
|
|
16
|
+
const result = await readWithIdleTimeout(reader, idleTimeoutMs, signal, "ndjson");
|
|
39
17
|
if (result.done) break;
|
|
40
18
|
buffer += decoder.decode(result.value, { stream: true }).replace(/\r\n/g, "\n");
|
|
19
|
+
if (Buffer.byteLength(buffer, "utf8") > MAX_FRAME_BYTES) throw new LlmError("Ollama Cloud sent an oversized stream frame", "PROVIDER_HTTP_ERROR");
|
|
41
20
|
for (;;) {
|
|
42
21
|
const boundary = buffer.indexOf("\n");
|
|
43
22
|
if (boundary < 0) break;
|
|
@@ -65,7 +44,12 @@ async function* ndjsonObjects(body, idleTimeoutMs, signal) {
|
|
|
65
44
|
if (typeof parsed === "object" && parsed !== null && !Array.isArray(parsed)) yield parsed;
|
|
66
45
|
}
|
|
67
46
|
} finally {
|
|
68
|
-
reader.
|
|
47
|
+
await reader.cancel().catch((err) => bestEffortFailed("ndjson:cancel", err));
|
|
48
|
+
try {
|
|
49
|
+
reader.releaseLock();
|
|
50
|
+
} catch (error) {
|
|
51
|
+
bestEffortFailed("ndjson:release", error);
|
|
52
|
+
}
|
|
69
53
|
}
|
|
70
54
|
}
|
|
71
55
|
export { ndjsonObjects };
|
|
@@ -108,24 +108,36 @@ async function* parseChatStream(response, idleTimeoutMs, signal) {
|
|
|
108
108
|
entry = {
|
|
109
109
|
dshIndex: nextIndex++,
|
|
110
110
|
id: uniqueToolCallId(toolCall.id),
|
|
111
|
+
upstreamId: typeof toolCall.id === "string" ? toolCall.id.trim() : "",
|
|
111
112
|
name: "",
|
|
112
113
|
args: "",
|
|
113
114
|
started: false
|
|
114
115
|
};
|
|
115
116
|
toolCalls.set(key, entry);
|
|
116
117
|
}
|
|
117
|
-
if (typeof toolCall.id === "string" && toolCall.id.trim() !== "") entry.
|
|
118
|
+
if (entry.upstreamId === "" && typeof toolCall.id === "string" && toolCall.id.trim() !== "") entry.upstreamId = toolCall.id.trim();
|
|
118
119
|
if (toolCall.function?.name !== void 0 && toolCall.function.name !== "") entry.name = toolCall.function.name;
|
|
120
|
+
const argumentsDelta = toolCall.function?.arguments ?? "";
|
|
121
|
+
if (argumentsDelta !== "") entry.args += argumentsDelta;
|
|
122
|
+
if (entry.upstreamId === "" && !entry.started) continue;
|
|
119
123
|
if (!entry.started) {
|
|
124
|
+
entry.id = ToolCallId(entry.upstreamId);
|
|
120
125
|
entry.started = true;
|
|
126
|
+
sawContent = true;
|
|
121
127
|
yield {
|
|
122
128
|
type: "block-start",
|
|
123
129
|
index: entry.dshIndex,
|
|
124
130
|
blockType: "tool-call"
|
|
125
131
|
};
|
|
132
|
+
yield {
|
|
133
|
+
type: "tool-call-delta",
|
|
134
|
+
index: entry.dshIndex,
|
|
135
|
+
id: entry.id,
|
|
136
|
+
...entry.name === "" ? {} : { name: entry.name },
|
|
137
|
+
argumentsDelta: entry.args
|
|
138
|
+
};
|
|
139
|
+
continue;
|
|
126
140
|
}
|
|
127
|
-
const argumentsDelta = toolCall.function?.arguments ?? "";
|
|
128
|
-
if (argumentsDelta !== "") entry.args += argumentsDelta;
|
|
129
141
|
sawContent = true;
|
|
130
142
|
yield {
|
|
131
143
|
type: "tool-call-delta",
|
|
@@ -141,6 +153,23 @@ async function* parseChatStream(response, idleTimeoutMs, signal) {
|
|
|
141
153
|
}
|
|
142
154
|
}
|
|
143
155
|
}
|
|
156
|
+
for (const entry of toolCalls.values()) {
|
|
157
|
+
if (entry.started) continue;
|
|
158
|
+
entry.started = true;
|
|
159
|
+
sawContent = true;
|
|
160
|
+
yield {
|
|
161
|
+
type: "block-start",
|
|
162
|
+
index: entry.dshIndex,
|
|
163
|
+
blockType: "tool-call"
|
|
164
|
+
};
|
|
165
|
+
yield {
|
|
166
|
+
type: "tool-call-delta",
|
|
167
|
+
index: entry.dshIndex,
|
|
168
|
+
id: ToolCallId(entry.id),
|
|
169
|
+
...entry.name === "" ? {} : { name: entry.name },
|
|
170
|
+
argumentsDelta: entry.args
|
|
171
|
+
};
|
|
172
|
+
}
|
|
144
173
|
if (!sawContent) throw new LlmError("Ollama Cloud completed the request without content", "EMPTY_RESPONSE");
|
|
145
174
|
if (!terminated) throw new LlmError("Ollama Cloud stream ended before [DONE] or finish_reason", "TRANSPORT");
|
|
146
175
|
if (textIndex !== void 0) yield {
|
package/lib/transports/sse.js
CHANGED
|
@@ -1,29 +1,8 @@
|
|
|
1
1
|
import { bestEffortFailed } from "../best-effort.js";
|
|
2
|
+
import { readWithIdleTimeout } from "./stream-reader.js";
|
|
2
3
|
import { LlmError } from "@deepseek-ai/dsh-llm";
|
|
3
4
|
// src/transports/sse.ts
|
|
4
|
-
|
|
5
|
-
if (signal?.aborted === true) throw signal.reason;
|
|
6
|
-
return await new Promise((resolve, reject) => {
|
|
7
|
-
let settled = false;
|
|
8
|
-
const finish = (action) => {
|
|
9
|
-
if (settled) return;
|
|
10
|
-
settled = true;
|
|
11
|
-
clearTimeout(timer);
|
|
12
|
-
signal?.removeEventListener("abort", onAbort);
|
|
13
|
-
action();
|
|
14
|
-
};
|
|
15
|
-
const timer = setTimeout(() => {
|
|
16
|
-
reader.cancel().catch((err) => bestEffortFailed("sse:cancel", err));
|
|
17
|
-
finish(() => reject(new LlmError(`Ollama Cloud stream went idle for ${idleTimeoutMs}ms`, "TIMEOUT")));
|
|
18
|
-
}, idleTimeoutMs);
|
|
19
|
-
const onAbort = () => {
|
|
20
|
-
reader.cancel().catch((err) => bestEffortFailed("sse:cancel", err));
|
|
21
|
-
finish(() => reject(signal?.reason ?? new DOMException("aborted", "AbortError")));
|
|
22
|
-
};
|
|
23
|
-
signal?.addEventListener("abort", onAbort, { once: true });
|
|
24
|
-
reader.read().then((value) => finish(() => resolve(value)), (error) => finish(() => reject(error)));
|
|
25
|
-
});
|
|
26
|
-
}
|
|
5
|
+
const MAX_FRAME_BYTES = 1e6;
|
|
27
6
|
/** Yield the `data:` payload of every SSE event, honoring cancellation and idle timeouts. */
|
|
28
7
|
async function* sseData(body, idleTimeoutMs, signal) {
|
|
29
8
|
const reader = body.getReader();
|
|
@@ -31,9 +10,10 @@ async function* sseData(body, idleTimeoutMs, signal) {
|
|
|
31
10
|
let buffer = "";
|
|
32
11
|
try {
|
|
33
12
|
for (;;) {
|
|
34
|
-
const result = await readWithIdleTimeout(reader, idleTimeoutMs, signal);
|
|
13
|
+
const result = await readWithIdleTimeout(reader, idleTimeoutMs, signal, "sse");
|
|
35
14
|
if (result.done) break;
|
|
36
15
|
buffer = (buffer + decoder.decode(result.value, { stream: true })).replace(/\r\n/g, "\n");
|
|
16
|
+
if (Buffer.byteLength(buffer, "utf8") > MAX_FRAME_BYTES) throw new LlmError("Ollama Cloud sent an oversized stream event", "PROVIDER_HTTP_ERROR");
|
|
37
17
|
for (;;) {
|
|
38
18
|
const boundary = buffer.indexOf("\n\n");
|
|
39
19
|
if (boundary < 0) break;
|
|
@@ -49,7 +29,12 @@ async function* sseData(body, idleTimeoutMs, signal) {
|
|
|
49
29
|
if (data !== "") yield data;
|
|
50
30
|
}
|
|
51
31
|
} finally {
|
|
52
|
-
reader.
|
|
32
|
+
await reader.cancel().catch((err) => bestEffortFailed("sse:cancel", err));
|
|
33
|
+
try {
|
|
34
|
+
reader.releaseLock();
|
|
35
|
+
} catch (error) {
|
|
36
|
+
bestEffortFailed("sse:release", error);
|
|
37
|
+
}
|
|
53
38
|
}
|
|
54
39
|
}
|
|
55
40
|
export { sseData };
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
import { bestEffortFailed } from "../best-effort.js";
|
|
2
|
+
import { LlmError } from "@deepseek-ai/dsh-llm";
|
|
3
|
+
// src/transports/stream-reader.ts
|
|
4
|
+
/** Read one stream chunk, rejecting with TIMEOUT or AbortError instead of hanging. */
|
|
5
|
+
async function readWithIdleTimeout(reader, idleTimeoutMs, signal, label) {
|
|
6
|
+
if (signal?.aborted === true) throw signal.reason;
|
|
7
|
+
return await new Promise((resolve, reject) => {
|
|
8
|
+
let settled = false;
|
|
9
|
+
const finish = (action) => {
|
|
10
|
+
if (settled) return;
|
|
11
|
+
settled = true;
|
|
12
|
+
clearTimeout(timer);
|
|
13
|
+
signal?.removeEventListener("abort", onAbort);
|
|
14
|
+
action();
|
|
15
|
+
};
|
|
16
|
+
const timer = setTimeout(() => {
|
|
17
|
+
finish(() => reject(new LlmError(`Ollama Cloud stream went idle for ${idleTimeoutMs}ms`, "TIMEOUT")));
|
|
18
|
+
reader.cancel().catch((err) => bestEffortFailed(`${label}:cancel`, err));
|
|
19
|
+
}, idleTimeoutMs);
|
|
20
|
+
timer.unref?.();
|
|
21
|
+
const onAbort = () => {
|
|
22
|
+
finish(() => reject(signal?.reason ?? new DOMException("aborted", "AbortError")));
|
|
23
|
+
reader.cancel().catch((err) => bestEffortFailed(`${label}:cancel`, err));
|
|
24
|
+
};
|
|
25
|
+
signal?.addEventListener("abort", onAbort, { once: true });
|
|
26
|
+
reader.read().then((value) => finish(() => resolve(value)), (error) => finish(() => reject(error)));
|
|
27
|
+
});
|
|
28
|
+
}
|
|
29
|
+
export { readWithIdleTimeout };
|
|
30
|
+
|
|
31
|
+
//# sourceMappingURL=stream-reader.js.map
|
package/lib/usage.js
CHANGED
|
@@ -111,13 +111,21 @@ function createUsageReader(config, resolveKey, fetchImpl = fetch) {
|
|
|
111
111
|
const ttl = hit.report === void 0 ? FAILURE_TTL_MS : USAGE_TTL_MS;
|
|
112
112
|
return Date.now() - hit.checkedAt < ttl ? hit : void 0;
|
|
113
113
|
};
|
|
114
|
-
const
|
|
114
|
+
const noteUsageCall = () => {
|
|
115
|
+
const now = Date.now();
|
|
116
|
+
const recent = usageCalls.filter((at) => now - at < USAGE_WINDOW_MS);
|
|
117
|
+
recent.push(now);
|
|
118
|
+
usageCalls.length = 0;
|
|
119
|
+
usageCalls.push(...recent);
|
|
120
|
+
};
|
|
121
|
+
const request = async (url, key, countUsage = false) => {
|
|
115
122
|
const current = config();
|
|
116
123
|
const policy = retryPolicy({
|
|
117
124
|
...current,
|
|
118
125
|
retryMaxRetries: Math.max(2, current.retryMaxRetries)
|
|
119
126
|
});
|
|
120
127
|
for (let attempt = 0;; attempt += 1) {
|
|
128
|
+
if (countUsage) noteUsageCall();
|
|
121
129
|
let response;
|
|
122
130
|
try {
|
|
123
131
|
response = await fetchImpl(url, {
|
|
@@ -126,6 +134,7 @@ function createUsageReader(config, resolveKey, fetchImpl = fetch) {
|
|
|
126
134
|
accept: "application/json",
|
|
127
135
|
"user-agent": current.userAgent
|
|
128
136
|
},
|
|
137
|
+
redirect: "error",
|
|
129
138
|
signal: AbortSignal.timeout(USAGE_TIMEOUT_MS)
|
|
130
139
|
});
|
|
131
140
|
} catch (error) {
|
|
@@ -168,16 +177,12 @@ function createUsageReader(config, resolveKey, fetchImpl = fetch) {
|
|
|
168
177
|
};
|
|
169
178
|
}
|
|
170
179
|
const now = Date.now();
|
|
171
|
-
|
|
172
|
-
if (recent.length >= USAGE_WINDOW_LIMIT) return void 0;
|
|
173
|
-
recent.push(now);
|
|
174
|
-
usageCalls.length = 0;
|
|
175
|
-
usageCalls.push(...recent);
|
|
180
|
+
if (usageCalls.filter((at) => now - at < USAGE_WINDOW_MS).length >= USAGE_WINDOW_LIMIT) return { rateLimited: true };
|
|
176
181
|
try {
|
|
177
182
|
const url = new URL(current.usageUrl);
|
|
178
183
|
url.searchParams.set("range", range);
|
|
179
184
|
url.searchParams.set("group_by", "model");
|
|
180
|
-
const response = await request(url.toString(), key);
|
|
185
|
+
const response = await request(url.toString(), key, true);
|
|
181
186
|
if (!response.ok) {
|
|
182
187
|
if (epoch === generation) cache.set(cacheKey, {
|
|
183
188
|
error: `usage endpoint returned HTTP ${response.status}`,
|
|
@@ -195,15 +200,17 @@ function createUsageReader(config, resolveKey, fetchImpl = fetch) {
|
|
|
195
200
|
}
|
|
196
201
|
};
|
|
197
202
|
const compose = async (key, range, epoch) => {
|
|
198
|
-
const [balance,
|
|
203
|
+
const [balance, usageRead] = await Promise.all([readBalance(key), readUsage(key, range, epoch)]);
|
|
204
|
+
const rateLimited = usageRead !== void 0 && "rateLimited" in usageRead;
|
|
205
|
+
const usage = usageRead !== void 0 && !("rateLimited" in usageRead) ? usageRead : void 0;
|
|
199
206
|
const reportKey = `${fingerprint(key)}:report:${range}`;
|
|
200
207
|
if (balance === void 0 && usage === void 0) {
|
|
201
208
|
const result = {
|
|
202
|
-
error: "usage and balance are unavailable",
|
|
209
|
+
error: rateLimited ? "usage rate limit reached; retry shortly" : "usage and balance are unavailable",
|
|
203
210
|
checkedAt: Date.now()
|
|
204
211
|
};
|
|
205
212
|
const previous = lastGood.get(reportKey);
|
|
206
|
-
if (epoch === generation) cache.set(reportKey, result);
|
|
213
|
+
if (!rateLimited && epoch === generation) cache.set(reportKey, result);
|
|
207
214
|
return previous === void 0 ? result : {
|
|
208
215
|
...previous,
|
|
209
216
|
stale: true,
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@goodandready/dsh-ollama-cloud-provider",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.41",
|
|
4
4
|
"description": "Ollama Cloud LLM provider plugin for DeepSeek Harness (dsh).",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"type": "module",
|
|
@@ -73,8 +73,8 @@
|
|
|
73
73
|
"build": "tsdown",
|
|
74
74
|
"prepare": "tsdown 1>&2",
|
|
75
75
|
"typecheck": "tsc --noEmit",
|
|
76
|
-
"test": "node --import tsx --test --test-concurrency=1 tests/core.test.ts tests/usage.test.ts tests/adapter.test.ts tests/routes.test.ts tests/native.test.ts tests/openai-chat.test.ts tests/commands.test.ts tests/tui-settings.test.ts tests/analytics.test.ts tests/apply.test.ts tests/account-rules.test.ts tests/request-log.test.ts tests/image-budget.test.ts tests/tool-schema.test.ts tests/tools.test.ts tests/plugin-updater.test.ts tests/compat.test.ts tests/status.test.ts tests/settings-draft.test.ts tests/update-state.test.ts tests/ui-render.test.tsx tests/update-card-view.test.tsx tests/web.test.ts tests/mock-ollama.test.ts tests/reasoning.test.ts tests/catalog-cache.test.ts tests/attempt.test.ts tests/model-groups.test.tsx tests/discovery.test.ts tests/sanitize.test.ts tests/retry.test.ts tests/provider-card.test.tsx tests/host-verification.test.ts tests/ui-actions.test.tsx tests/tool-call-id.test.ts tests/deploy-script.test.ts tests/endpoints.test.ts tests/route-guard.test.ts",
|
|
77
|
-
"check": "npm run typecheck && npm test && npm run build && npm run pack:check",
|
|
76
|
+
"test": "node --import tsx --test --test-concurrency=1 tests/core.test.ts tests/usage.test.ts tests/adapter.test.ts tests/routes.test.ts tests/native.test.ts tests/openai-chat.test.ts tests/commands.test.ts tests/tui-settings.test.ts tests/analytics.test.ts tests/apply.test.ts tests/account-rules.test.ts tests/request-log.test.ts tests/image-budget.test.ts tests/tool-schema.test.ts tests/tools.test.ts tests/plugin-updater.test.ts tests/compat.test.ts tests/status.test.ts tests/settings-draft.test.ts tests/update-state.test.ts tests/ui-render.test.tsx tests/update-card-view.test.tsx tests/web.test.ts tests/mock-ollama.test.ts tests/reasoning.test.ts tests/catalog-cache.test.ts tests/attempt.test.ts tests/model-groups.test.tsx tests/discovery.test.ts tests/sanitize.test.ts tests/retry.test.ts tests/provider-card.test.tsx tests/host-verification.test.ts tests/ui-actions.test.tsx tests/tool-call-id.test.ts tests/deploy-script.test.ts tests/endpoints.test.ts tests/route-guard.test.ts tests/catalog-scheduler.test.ts tests/format.test.ts",
|
|
77
|
+
"check": "npm run typecheck && npm test && npm run build && node scripts/smoke-load.mjs && npm run pack:check",
|
|
78
78
|
"release": "npm run check && mkdir -p dist && npm pack --pack-destination dist",
|
|
79
79
|
"sync:models": "node scripts/sync-models.mjs",
|
|
80
80
|
"pack:check": "node scripts/pack-check.mjs"
|