@goodandready/dsh-ollama-cloud-provider 0.1.38 → 0.1.40
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +23 -0
- package/README.md +13 -6
- package/README.ru.md +13 -6
- package/README.zh.md +13 -6
- package/lib/adapter.js +10 -3
- package/lib/catalog-cache.js +8 -2
- package/lib/catalog-scheduler.js +34 -0
- package/lib/catalog.js +23 -14
- package/lib/client.js +171 -94
- package/lib/config.js +23 -9
- package/lib/discovery.js +75 -7
- package/lib/endpoints.js +58 -0
- package/lib/index.js +14 -10
- package/lib/local-host.js +17 -8
- package/lib/plugin-updater.js +35 -8
- package/lib/request-log.js +22 -4
- package/lib/route-guard.js +25 -0
- package/lib/routes-values.js +17 -4
- package/lib/routes.js +138 -77
- package/lib/status.js +20 -1
- package/lib/transports/ndjson.js +10 -3
- package/lib/transports/openai-chat.js +32 -3
- package/lib/transports/sse.js +11 -3
- package/lib/usage.js +17 -10
- package/package.json +2 -2
package/lib/routes.js
CHANGED
|
@@ -3,13 +3,15 @@ import { readModelAnalytics } from "./analytics.js";
|
|
|
3
3
|
import { clearCatalogCache, effectiveModel, modelAvailable, modelVisible, refreshCatalog } from "./catalog.js";
|
|
4
4
|
import { NS } from "./config.js";
|
|
5
5
|
import { discoverModels } from "./discovery.js";
|
|
6
|
+
import { checkEndpoint } from "./endpoints.js";
|
|
6
7
|
import { priceFor } from "./pricing.js";
|
|
7
8
|
import { authorityHostname, isLocalHostname } from "./local-host.js";
|
|
8
9
|
import { PROVIDER } from "./provider-ids.js";
|
|
10
|
+
import { createRouteGuard } from "./route-guard.js";
|
|
9
11
|
import { publicConfig } from "./routes-public.js";
|
|
10
|
-
import {
|
|
12
|
+
import { clearStatusCache, probeServiceCached } from "./status.js";
|
|
11
13
|
import { cleanAccounts, cleanBooleanRecord, cleanManualModels, cleanModelAccountRules, cleanModelOverrides, cleanNumberRecord, cleanPrices, cleanStringArray, cleanTransports, isCredentialRef, normalizeCredentialRef } from "./routes-values.js";
|
|
12
|
-
import { safeErrorMessage } from "./sanitize.js";
|
|
14
|
+
import { redactText, safeErrorMessage } from "./sanitize.js";
|
|
13
15
|
import { ReasoningEffortId } from "@deepseek-ai/dsh-llm";
|
|
14
16
|
import { credentialRef } from "@deepseek-ai/dsh-credentials";
|
|
15
17
|
// src/routes.ts
|
|
@@ -213,10 +215,22 @@ function configOps(patch) {
|
|
|
213
215
|
set("apiKeyEnv", ref);
|
|
214
216
|
}
|
|
215
217
|
}
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
218
|
+
for (const field of [
|
|
219
|
+
"apiBase",
|
|
220
|
+
"openAiBase",
|
|
221
|
+
"usageUrl",
|
|
222
|
+
"balanceUrl"
|
|
223
|
+
]) {
|
|
224
|
+
if (!(field in patch)) continue;
|
|
225
|
+
const raw = patch[field];
|
|
226
|
+
if (raw === null || raw === void 0 || typeof raw === "string" && raw.trim() === "") {
|
|
227
|
+
set(field, void 0);
|
|
228
|
+
continue;
|
|
229
|
+
}
|
|
230
|
+
const checked = checkEndpoint(field, raw);
|
|
231
|
+
if (!checked.ok) throw new Error(checked.error);
|
|
232
|
+
set(field, checked.value);
|
|
233
|
+
}
|
|
220
234
|
if ("transport" in patch) set("transport", patch.transport);
|
|
221
235
|
if ("modelTransports" in patch) set("modelTransports", cleanTransports(patch.modelTransports));
|
|
222
236
|
if ("requestTimeoutMs" in patch) set("requestTimeoutMs", patch.requestTimeoutMs);
|
|
@@ -274,7 +288,7 @@ async function buildState(ctx, deps) {
|
|
|
274
288
|
keyInfo = {
|
|
275
289
|
configured: info.configured,
|
|
276
290
|
writable: info.writable,
|
|
277
|
-
...info.source === void 0 ? {} : { source: info.source }
|
|
291
|
+
...info.source === void 0 ? {} : { source: redactText(info.source) }
|
|
278
292
|
};
|
|
279
293
|
} catch (error) {
|
|
280
294
|
bestEffortFailed("primary credential describe failed", error);
|
|
@@ -328,7 +342,7 @@ async function buildState(ctx, deps) {
|
|
|
328
342
|
accounts.push({
|
|
329
343
|
...slot,
|
|
330
344
|
configured,
|
|
331
|
-
source,
|
|
345
|
+
source: redactText(source),
|
|
332
346
|
active: activeLabel === slot.label,
|
|
333
347
|
coolingUntil: health?.coolingUntil ?? 0,
|
|
334
348
|
mark: health?.reason ?? "",
|
|
@@ -336,6 +350,7 @@ async function buildState(ctx, deps) {
|
|
|
336
350
|
});
|
|
337
351
|
}
|
|
338
352
|
let models = [];
|
|
353
|
+
let providerError;
|
|
339
354
|
try {
|
|
340
355
|
models = (await deps.adapter.listModels(PROVIDER, { unfiltered: true })).map((model) => {
|
|
341
356
|
const meta = effectiveModel(model.id, config);
|
|
@@ -356,7 +371,9 @@ async function buildState(ctx, deps) {
|
|
|
356
371
|
visible: modelVisible(model.id, config)
|
|
357
372
|
};
|
|
358
373
|
});
|
|
359
|
-
} catch {
|
|
374
|
+
} catch (error) {
|
|
375
|
+
bestEffortFailed("state:listModels", error);
|
|
376
|
+
providerError = redactText(error instanceof Error ? error.message : String(error));
|
|
360
377
|
models = [];
|
|
361
378
|
}
|
|
362
379
|
const displayEntry = resolved.find((entry) => (entry.label ?? "default") === activeLabel) ?? resolved[0];
|
|
@@ -368,7 +385,9 @@ async function buildState(ctx, deps) {
|
|
|
368
385
|
models,
|
|
369
386
|
usage: usage ?? null,
|
|
370
387
|
routing: deps.adapter.routingTrace(),
|
|
371
|
-
revision: settingsRevision(ctx)
|
|
388
|
+
revision: settingsRevision(ctx),
|
|
389
|
+
configWritable: typeof getSettings(ctx)?.mutate === "function" && getSettings(ctx)?.writable !== false,
|
|
390
|
+
...providerError === void 0 ? {} : { providerError }
|
|
372
391
|
};
|
|
373
392
|
}
|
|
374
393
|
async function refreshAndState(ctx, deps, res) {
|
|
@@ -389,6 +408,10 @@ function applyRoutes(ctx, web, deps) {
|
|
|
389
408
|
handler
|
|
390
409
|
}));
|
|
391
410
|
};
|
|
411
|
+
const refreshGuard = createRouteGuard(2e3);
|
|
412
|
+
const modelsGuard = createRouteGuard(2e3);
|
|
413
|
+
const discoverGuard = createRouteGuard(1e3);
|
|
414
|
+
const testGuard = createRouteGuard(2e3);
|
|
392
415
|
const readOnly = (req, res) => {
|
|
393
416
|
if (req.method !== "GET") {
|
|
394
417
|
json(res, 405, { error: "GET required" });
|
|
@@ -422,7 +445,7 @@ function applyRoutes(ctx, web, deps) {
|
|
|
422
445
|
const config = deps.config();
|
|
423
446
|
let providerError;
|
|
424
447
|
const models = await deps.adapter.listModels(PROVIDER, { unfiltered: true }).catch((err) => {
|
|
425
|
-
providerError = err instanceof Error ? err.message : String(err);
|
|
448
|
+
providerError = redactText(err instanceof Error ? err.message : String(err));
|
|
426
449
|
bestEffortFailed("health:listModels", err);
|
|
427
450
|
return [];
|
|
428
451
|
});
|
|
@@ -440,7 +463,7 @@ function applyRoutes(ctx, web, deps) {
|
|
|
440
463
|
route("/dsh-ollama/status", async (req, res) => {
|
|
441
464
|
if (!readOnly(req, res)) return;
|
|
442
465
|
try {
|
|
443
|
-
json(res, 200, await
|
|
466
|
+
json(res, 200, await probeServiceCached(deps.config(), fetch));
|
|
444
467
|
} catch (error) {
|
|
445
468
|
json(res, 200, {
|
|
446
469
|
ok: false,
|
|
@@ -582,90 +605,128 @@ function applyRoutes(ctx, web, deps) {
|
|
|
582
605
|
});
|
|
583
606
|
route("/dsh-ollama/refresh", async (req, res) => {
|
|
584
607
|
if (!await requirePost(req, res, verifyHost)) return;
|
|
585
|
-
|
|
586
|
-
|
|
608
|
+
const gate = refreshGuard.begin();
|
|
609
|
+
if (gate !== "ok") {
|
|
610
|
+
json(res, 429, { error: gate === "busy" ? "request already in progress" : "rate limited; retry shortly" });
|
|
611
|
+
return;
|
|
612
|
+
}
|
|
613
|
+
try {
|
|
614
|
+
clearStatusCache();
|
|
615
|
+
deps.usage.clear();
|
|
616
|
+
await refreshAndState(ctx, deps, res);
|
|
617
|
+
} finally {
|
|
618
|
+
refreshGuard.end();
|
|
619
|
+
}
|
|
587
620
|
});
|
|
588
621
|
route("/dsh-ollama/models/refresh", async (req, res) => {
|
|
589
622
|
if (!await requirePost(req, res, verifyHost)) return;
|
|
590
|
-
|
|
591
|
-
|
|
623
|
+
const gate = modelsGuard.begin();
|
|
624
|
+
if (gate !== "ok") {
|
|
625
|
+
json(res, 429, { error: gate === "busy" ? "request already in progress" : "rate limited; retry shortly" });
|
|
626
|
+
return;
|
|
627
|
+
}
|
|
592
628
|
try {
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
|
|
629
|
+
clearCatalogCache();
|
|
630
|
+
clearStatusCache();
|
|
631
|
+
deps.usage.clear();
|
|
632
|
+
try {
|
|
633
|
+
await refreshCatalog(deps.config(), fetch, true);
|
|
634
|
+
} catch (error) {
|
|
635
|
+
bestEffortFailed("catalog refresh failed", error);
|
|
636
|
+
}
|
|
637
|
+
await refreshAndState(ctx, deps, res);
|
|
638
|
+
} finally {
|
|
639
|
+
modelsGuard.end();
|
|
596
640
|
}
|
|
597
|
-
await refreshAndState(ctx, deps, res);
|
|
598
641
|
});
|
|
599
642
|
route("/dsh-ollama/discover", async (req, res) => {
|
|
600
643
|
if (!await requirePost(req, res, verifyHost)) return;
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
} catch (error) {
|
|
605
|
-
json(res, 400, { error: error instanceof Error ? error.message : "invalid body" });
|
|
644
|
+
const gate = discoverGuard.begin();
|
|
645
|
+
if (gate !== "ok") {
|
|
646
|
+
json(res, 429, { error: gate === "busy" ? "request already in progress" : "rate limited; retry shortly" });
|
|
606
647
|
return;
|
|
607
648
|
}
|
|
608
|
-
const apiBase = typeof body.apiBase === "string" ? body.apiBase : "";
|
|
609
|
-
const apiKey = typeof body.apiKey === "string" ? body.apiKey : "";
|
|
610
649
|
try {
|
|
611
|
-
|
|
612
|
-
|
|
613
|
-
|
|
650
|
+
let body;
|
|
651
|
+
try {
|
|
652
|
+
body = await readJson(req);
|
|
653
|
+
} catch (error) {
|
|
654
|
+
json(res, 400, { error: error instanceof Error ? error.message : "invalid body" });
|
|
655
|
+
return;
|
|
656
|
+
}
|
|
657
|
+
const apiBase = typeof body.apiBase === "string" ? body.apiBase : "";
|
|
658
|
+
const apiKey = typeof body.apiKey === "string" ? body.apiKey : "";
|
|
659
|
+
try {
|
|
660
|
+
json(res, 200, await discoverModels(apiBase, apiKey, deps.discoveryFetch));
|
|
661
|
+
} catch (error) {
|
|
662
|
+
json(res, 400, { error: error instanceof Error ? error.message : "discovery failed" });
|
|
663
|
+
}
|
|
664
|
+
} finally {
|
|
665
|
+
discoverGuard.end();
|
|
614
666
|
}
|
|
615
667
|
});
|
|
616
668
|
route("/dsh-ollama/test", async (req, res) => {
|
|
617
669
|
if (!await requirePost(req, res, verifyHost)) return;
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
} catch (error) {
|
|
622
|
-
json(res, 400, { error: error instanceof Error ? error.message : "invalid body" });
|
|
670
|
+
const gate = testGuard.begin();
|
|
671
|
+
if (gate !== "ok") {
|
|
672
|
+
json(res, 429, { error: gate === "busy" ? "request already in progress" : "rate limited; retry shortly" });
|
|
623
673
|
return;
|
|
624
674
|
}
|
|
625
|
-
const model = typeof body.model === "string" && body.model !== "" ? body.model : "gpt-oss:120b";
|
|
626
|
-
const started = Date.now();
|
|
627
|
-
const controller = new AbortController();
|
|
628
|
-
const timer = setTimeout(() => controller.abort(), 6e4);
|
|
629
|
-
let text = "";
|
|
630
|
-
let finish;
|
|
631
675
|
try {
|
|
632
|
-
|
|
633
|
-
|
|
634
|
-
|
|
635
|
-
|
|
636
|
-
|
|
637
|
-
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
|
|
641
|
-
|
|
642
|
-
|
|
643
|
-
|
|
644
|
-
|
|
645
|
-
|
|
646
|
-
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
|
|
650
|
-
|
|
651
|
-
|
|
652
|
-
|
|
653
|
-
|
|
654
|
-
|
|
655
|
-
|
|
656
|
-
|
|
657
|
-
|
|
658
|
-
|
|
659
|
-
|
|
660
|
-
|
|
661
|
-
|
|
662
|
-
|
|
663
|
-
|
|
664
|
-
|
|
665
|
-
|
|
666
|
-
|
|
676
|
+
let body;
|
|
677
|
+
try {
|
|
678
|
+
body = await readJson(req);
|
|
679
|
+
} catch (error) {
|
|
680
|
+
json(res, 400, { error: error instanceof Error ? error.message : "invalid body" });
|
|
681
|
+
return;
|
|
682
|
+
}
|
|
683
|
+
const model = typeof body.model === "string" && body.model !== "" ? body.model : "gpt-oss:120b";
|
|
684
|
+
const started = Date.now();
|
|
685
|
+
const controller = new AbortController();
|
|
686
|
+
const timer = setTimeout(() => controller.abort(), 6e4);
|
|
687
|
+
let text = "";
|
|
688
|
+
let finish;
|
|
689
|
+
try {
|
|
690
|
+
const options = {
|
|
691
|
+
provider: PROVIDER,
|
|
692
|
+
model,
|
|
693
|
+
messages: [{
|
|
694
|
+
role: "user",
|
|
695
|
+
content: [{
|
|
696
|
+
type: "text",
|
|
697
|
+
text: "Reply with exactly: OK"
|
|
698
|
+
}]
|
|
699
|
+
}],
|
|
700
|
+
reasoningEffort: ReasoningEffortId("off"),
|
|
701
|
+
maxTokens: 64,
|
|
702
|
+
signal: controller.signal
|
|
703
|
+
};
|
|
704
|
+
let reasoning = "";
|
|
705
|
+
for await (const chunk of deps.adapter.stream(options)) if (chunk.type === "text-delta") text += chunk.text;
|
|
706
|
+
else if (chunk.type === "reasoning-delta") reasoning += chunk.text;
|
|
707
|
+
else if (chunk.type === "finish") finish = chunk.reason;
|
|
708
|
+
const reply = text === "" ? reasoning : text;
|
|
709
|
+
json(res, 200, {
|
|
710
|
+
ok: true,
|
|
711
|
+
model,
|
|
712
|
+
provider: PROVIDER,
|
|
713
|
+
latencyMs: Date.now() - started,
|
|
714
|
+
text: reply.slice(0, 200),
|
|
715
|
+
finish
|
|
716
|
+
});
|
|
717
|
+
} catch (error) {
|
|
718
|
+
json(res, 200, {
|
|
719
|
+
ok: false,
|
|
720
|
+
model,
|
|
721
|
+
provider: PROVIDER,
|
|
722
|
+
latencyMs: Date.now() - started,
|
|
723
|
+
error: controller.signal.aborted ? "timeout" : error instanceof Error ? error.message : "test failed"
|
|
724
|
+
});
|
|
725
|
+
} finally {
|
|
726
|
+
clearTimeout(timer);
|
|
727
|
+
}
|
|
667
728
|
} finally {
|
|
668
|
-
|
|
729
|
+
testGuard.end();
|
|
669
730
|
}
|
|
670
731
|
});
|
|
671
732
|
ctx.effect(() => () => {
|
package/lib/status.js
CHANGED
|
@@ -14,6 +14,7 @@ async function probeService(config, fetchImpl = fetch) {
|
|
|
14
14
|
try {
|
|
15
15
|
const response = await fetchImpl(tagsUrlFor(config), {
|
|
16
16
|
method: "GET",
|
|
17
|
+
redirect: "error",
|
|
17
18
|
signal: controller.signal,
|
|
18
19
|
headers: { accept: "application/json" }
|
|
19
20
|
});
|
|
@@ -37,6 +38,24 @@ async function probeService(config, fetchImpl = fetch) {
|
|
|
37
38
|
clearTimeout(timer);
|
|
38
39
|
}
|
|
39
40
|
}
|
|
40
|
-
|
|
41
|
+
const STATUS_CACHE_MS = 1e4;
|
|
42
|
+
let cached;
|
|
43
|
+
function clearStatusCache() {
|
|
44
|
+
cached = void 0;
|
|
45
|
+
}
|
|
46
|
+
/** Cached probe: repeated UI polls must not fan out external requests. */
|
|
47
|
+
async function probeServiceCached(config, fetchImpl = fetch, now = Date.now) {
|
|
48
|
+
const key = tagsUrlFor(config);
|
|
49
|
+
const at = now();
|
|
50
|
+
if (cached !== void 0 && cached.key === key && at - cached.at < STATUS_CACHE_MS) return cached.value;
|
|
51
|
+
const value = await probeService(config, fetchImpl);
|
|
52
|
+
cached = {
|
|
53
|
+
key,
|
|
54
|
+
at,
|
|
55
|
+
value
|
|
56
|
+
};
|
|
57
|
+
return value;
|
|
58
|
+
}
|
|
59
|
+
export { clearStatusCache, probeService, probeServiceCached };
|
|
41
60
|
|
|
42
61
|
//# sourceMappingURL=status.js.map
|
package/lib/transports/ndjson.js
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { bestEffortFailed } from "../best-effort.js";
|
|
2
2
|
import { LlmError } from "@deepseek-ai/dsh-llm";
|
|
3
3
|
// src/transports/ndjson.ts
|
|
4
|
+
const MAX_FRAME_BYTES = 1e6;
|
|
4
5
|
async function readWithIdleTimeout(reader, idleTimeoutMs, signal) {
|
|
5
6
|
if (signal?.aborted === true) throw signal.reason;
|
|
6
7
|
return await new Promise((resolve, reject) => {
|
|
@@ -13,13 +14,13 @@ async function readWithIdleTimeout(reader, idleTimeoutMs, signal) {
|
|
|
13
14
|
action();
|
|
14
15
|
};
|
|
15
16
|
const timer = setTimeout(() => {
|
|
16
|
-
reader.cancel().catch((err) => bestEffortFailed("ndjson:cancel", err));
|
|
17
17
|
finish(() => reject(new LlmError(`Ollama Cloud stream went idle for ${idleTimeoutMs}ms`, "TIMEOUT")));
|
|
18
|
+
reader.cancel().catch((err) => bestEffortFailed("ndjson:cancel", err));
|
|
18
19
|
}, idleTimeoutMs);
|
|
19
20
|
timer.unref?.();
|
|
20
21
|
const onAbort = () => {
|
|
21
|
-
reader.cancel().catch((err) => bestEffortFailed("ndjson:cancel", err));
|
|
22
22
|
finish(() => reject(signal?.reason ?? new DOMException("aborted", "AbortError")));
|
|
23
|
+
reader.cancel().catch((err) => bestEffortFailed("ndjson:cancel", err));
|
|
23
24
|
};
|
|
24
25
|
signal?.addEventListener("abort", onAbort, { once: true });
|
|
25
26
|
reader.read().then((value) => finish(() => resolve(value)), (error) => finish(() => reject(error)));
|
|
@@ -38,6 +39,7 @@ async function* ndjsonObjects(body, idleTimeoutMs, signal) {
|
|
|
38
39
|
const result = await readWithIdleTimeout(reader, idleTimeoutMs, signal);
|
|
39
40
|
if (result.done) break;
|
|
40
41
|
buffer += decoder.decode(result.value, { stream: true }).replace(/\r\n/g, "\n");
|
|
42
|
+
if (Buffer.byteLength(buffer, "utf8") > MAX_FRAME_BYTES) throw new LlmError("Ollama Cloud sent an oversized stream frame", "PROVIDER_HTTP_ERROR");
|
|
41
43
|
for (;;) {
|
|
42
44
|
const boundary = buffer.indexOf("\n");
|
|
43
45
|
if (boundary < 0) break;
|
|
@@ -65,7 +67,12 @@ async function* ndjsonObjects(body, idleTimeoutMs, signal) {
|
|
|
65
67
|
if (typeof parsed === "object" && parsed !== null && !Array.isArray(parsed)) yield parsed;
|
|
66
68
|
}
|
|
67
69
|
} finally {
|
|
68
|
-
reader.
|
|
70
|
+
await reader.cancel().catch((err) => bestEffortFailed("ndjson:cancel", err));
|
|
71
|
+
try {
|
|
72
|
+
reader.releaseLock();
|
|
73
|
+
} catch (error) {
|
|
74
|
+
bestEffortFailed("ndjson:release", error);
|
|
75
|
+
}
|
|
69
76
|
}
|
|
70
77
|
}
|
|
71
78
|
export { ndjsonObjects };
|
|
@@ -108,24 +108,36 @@ async function* parseChatStream(response, idleTimeoutMs, signal) {
|
|
|
108
108
|
entry = {
|
|
109
109
|
dshIndex: nextIndex++,
|
|
110
110
|
id: uniqueToolCallId(toolCall.id),
|
|
111
|
+
upstreamId: typeof toolCall.id === "string" ? toolCall.id.trim() : "",
|
|
111
112
|
name: "",
|
|
112
113
|
args: "",
|
|
113
114
|
started: false
|
|
114
115
|
};
|
|
115
116
|
toolCalls.set(key, entry);
|
|
116
117
|
}
|
|
117
|
-
if (typeof toolCall.id === "string" && toolCall.id.trim() !== "") entry.
|
|
118
|
+
if (entry.upstreamId === "" && typeof toolCall.id === "string" && toolCall.id.trim() !== "") entry.upstreamId = toolCall.id.trim();
|
|
118
119
|
if (toolCall.function?.name !== void 0 && toolCall.function.name !== "") entry.name = toolCall.function.name;
|
|
120
|
+
const argumentsDelta = toolCall.function?.arguments ?? "";
|
|
121
|
+
if (argumentsDelta !== "") entry.args += argumentsDelta;
|
|
122
|
+
if (entry.upstreamId === "" && !entry.started) continue;
|
|
119
123
|
if (!entry.started) {
|
|
124
|
+
entry.id = ToolCallId(entry.upstreamId);
|
|
120
125
|
entry.started = true;
|
|
126
|
+
sawContent = true;
|
|
121
127
|
yield {
|
|
122
128
|
type: "block-start",
|
|
123
129
|
index: entry.dshIndex,
|
|
124
130
|
blockType: "tool-call"
|
|
125
131
|
};
|
|
132
|
+
yield {
|
|
133
|
+
type: "tool-call-delta",
|
|
134
|
+
index: entry.dshIndex,
|
|
135
|
+
id: entry.id,
|
|
136
|
+
...entry.name === "" ? {} : { name: entry.name },
|
|
137
|
+
argumentsDelta: entry.args
|
|
138
|
+
};
|
|
139
|
+
continue;
|
|
126
140
|
}
|
|
127
|
-
const argumentsDelta = toolCall.function?.arguments ?? "";
|
|
128
|
-
if (argumentsDelta !== "") entry.args += argumentsDelta;
|
|
129
141
|
sawContent = true;
|
|
130
142
|
yield {
|
|
131
143
|
type: "tool-call-delta",
|
|
@@ -141,6 +153,23 @@ async function* parseChatStream(response, idleTimeoutMs, signal) {
|
|
|
141
153
|
}
|
|
142
154
|
}
|
|
143
155
|
}
|
|
156
|
+
for (const entry of toolCalls.values()) {
|
|
157
|
+
if (entry.started) continue;
|
|
158
|
+
entry.started = true;
|
|
159
|
+
sawContent = true;
|
|
160
|
+
yield {
|
|
161
|
+
type: "block-start",
|
|
162
|
+
index: entry.dshIndex,
|
|
163
|
+
blockType: "tool-call"
|
|
164
|
+
};
|
|
165
|
+
yield {
|
|
166
|
+
type: "tool-call-delta",
|
|
167
|
+
index: entry.dshIndex,
|
|
168
|
+
id: ToolCallId(entry.id),
|
|
169
|
+
...entry.name === "" ? {} : { name: entry.name },
|
|
170
|
+
argumentsDelta: entry.args
|
|
171
|
+
};
|
|
172
|
+
}
|
|
144
173
|
if (!sawContent) throw new LlmError("Ollama Cloud completed the request without content", "EMPTY_RESPONSE");
|
|
145
174
|
if (!terminated) throw new LlmError("Ollama Cloud stream ended before [DONE] or finish_reason", "TRANSPORT");
|
|
146
175
|
if (textIndex !== void 0) yield {
|
package/lib/transports/sse.js
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { bestEffortFailed } from "../best-effort.js";
|
|
2
2
|
import { LlmError } from "@deepseek-ai/dsh-llm";
|
|
3
3
|
// src/transports/sse.ts
|
|
4
|
+
const MAX_FRAME_BYTES = 1e6;
|
|
4
5
|
async function readWithIdleTimeout(reader, idleTimeoutMs, signal) {
|
|
5
6
|
if (signal?.aborted === true) throw signal.reason;
|
|
6
7
|
return await new Promise((resolve, reject) => {
|
|
@@ -13,12 +14,13 @@ async function readWithIdleTimeout(reader, idleTimeoutMs, signal) {
|
|
|
13
14
|
action();
|
|
14
15
|
};
|
|
15
16
|
const timer = setTimeout(() => {
|
|
16
|
-
reader.cancel().catch((err) => bestEffortFailed("sse:cancel", err));
|
|
17
17
|
finish(() => reject(new LlmError(`Ollama Cloud stream went idle for ${idleTimeoutMs}ms`, "TIMEOUT")));
|
|
18
|
+
reader.cancel().catch((err) => bestEffortFailed("sse:cancel", err));
|
|
18
19
|
}, idleTimeoutMs);
|
|
20
|
+
timer.unref?.();
|
|
19
21
|
const onAbort = () => {
|
|
20
|
-
reader.cancel().catch((err) => bestEffortFailed("sse:cancel", err));
|
|
21
22
|
finish(() => reject(signal?.reason ?? new DOMException("aborted", "AbortError")));
|
|
23
|
+
reader.cancel().catch((err) => bestEffortFailed("sse:cancel", err));
|
|
22
24
|
};
|
|
23
25
|
signal?.addEventListener("abort", onAbort, { once: true });
|
|
24
26
|
reader.read().then((value) => finish(() => resolve(value)), (error) => finish(() => reject(error)));
|
|
@@ -34,6 +36,7 @@ async function* sseData(body, idleTimeoutMs, signal) {
|
|
|
34
36
|
const result = await readWithIdleTimeout(reader, idleTimeoutMs, signal);
|
|
35
37
|
if (result.done) break;
|
|
36
38
|
buffer = (buffer + decoder.decode(result.value, { stream: true })).replace(/\r\n/g, "\n");
|
|
39
|
+
if (Buffer.byteLength(buffer, "utf8") > MAX_FRAME_BYTES) throw new LlmError("Ollama Cloud sent an oversized stream event", "PROVIDER_HTTP_ERROR");
|
|
37
40
|
for (;;) {
|
|
38
41
|
const boundary = buffer.indexOf("\n\n");
|
|
39
42
|
if (boundary < 0) break;
|
|
@@ -49,7 +52,12 @@ async function* sseData(body, idleTimeoutMs, signal) {
|
|
|
49
52
|
if (data !== "") yield data;
|
|
50
53
|
}
|
|
51
54
|
} finally {
|
|
52
|
-
reader.
|
|
55
|
+
await reader.cancel().catch((err) => bestEffortFailed("sse:cancel", err));
|
|
56
|
+
try {
|
|
57
|
+
reader.releaseLock();
|
|
58
|
+
} catch (error) {
|
|
59
|
+
bestEffortFailed("sse:release", error);
|
|
60
|
+
}
|
|
53
61
|
}
|
|
54
62
|
}
|
|
55
63
|
export { sseData };
|
package/lib/usage.js
CHANGED
|
@@ -111,13 +111,21 @@ function createUsageReader(config, resolveKey, fetchImpl = fetch) {
|
|
|
111
111
|
const ttl = hit.report === void 0 ? FAILURE_TTL_MS : USAGE_TTL_MS;
|
|
112
112
|
return Date.now() - hit.checkedAt < ttl ? hit : void 0;
|
|
113
113
|
};
|
|
114
|
-
const
|
|
114
|
+
const noteUsageCall = () => {
|
|
115
|
+
const now = Date.now();
|
|
116
|
+
const recent = usageCalls.filter((at) => now - at < USAGE_WINDOW_MS);
|
|
117
|
+
recent.push(now);
|
|
118
|
+
usageCalls.length = 0;
|
|
119
|
+
usageCalls.push(...recent);
|
|
120
|
+
};
|
|
121
|
+
const request = async (url, key, countUsage = false) => {
|
|
115
122
|
const current = config();
|
|
116
123
|
const policy = retryPolicy({
|
|
117
124
|
...current,
|
|
118
125
|
retryMaxRetries: Math.max(2, current.retryMaxRetries)
|
|
119
126
|
});
|
|
120
127
|
for (let attempt = 0;; attempt += 1) {
|
|
128
|
+
if (countUsage) noteUsageCall();
|
|
121
129
|
let response;
|
|
122
130
|
try {
|
|
123
131
|
response = await fetchImpl(url, {
|
|
@@ -126,6 +134,7 @@ function createUsageReader(config, resolveKey, fetchImpl = fetch) {
|
|
|
126
134
|
accept: "application/json",
|
|
127
135
|
"user-agent": current.userAgent
|
|
128
136
|
},
|
|
137
|
+
redirect: "error",
|
|
129
138
|
signal: AbortSignal.timeout(USAGE_TIMEOUT_MS)
|
|
130
139
|
});
|
|
131
140
|
} catch (error) {
|
|
@@ -168,16 +177,12 @@ function createUsageReader(config, resolveKey, fetchImpl = fetch) {
|
|
|
168
177
|
};
|
|
169
178
|
}
|
|
170
179
|
const now = Date.now();
|
|
171
|
-
|
|
172
|
-
if (recent.length >= USAGE_WINDOW_LIMIT) return void 0;
|
|
173
|
-
recent.push(now);
|
|
174
|
-
usageCalls.length = 0;
|
|
175
|
-
usageCalls.push(...recent);
|
|
180
|
+
if (usageCalls.filter((at) => now - at < USAGE_WINDOW_MS).length >= USAGE_WINDOW_LIMIT) return { rateLimited: true };
|
|
176
181
|
try {
|
|
177
182
|
const url = new URL(current.usageUrl);
|
|
178
183
|
url.searchParams.set("range", range);
|
|
179
184
|
url.searchParams.set("group_by", "model");
|
|
180
|
-
const response = await request(url.toString(), key);
|
|
185
|
+
const response = await request(url.toString(), key, true);
|
|
181
186
|
if (!response.ok) {
|
|
182
187
|
if (epoch === generation) cache.set(cacheKey, {
|
|
183
188
|
error: `usage endpoint returned HTTP ${response.status}`,
|
|
@@ -195,15 +200,17 @@ function createUsageReader(config, resolveKey, fetchImpl = fetch) {
|
|
|
195
200
|
}
|
|
196
201
|
};
|
|
197
202
|
const compose = async (key, range, epoch) => {
|
|
198
|
-
const [balance,
|
|
203
|
+
const [balance, usageRead] = await Promise.all([readBalance(key), readUsage(key, range, epoch)]);
|
|
204
|
+
const rateLimited = usageRead !== void 0 && "rateLimited" in usageRead;
|
|
205
|
+
const usage = usageRead !== void 0 && !("rateLimited" in usageRead) ? usageRead : void 0;
|
|
199
206
|
const reportKey = `${fingerprint(key)}:report:${range}`;
|
|
200
207
|
if (balance === void 0 && usage === void 0) {
|
|
201
208
|
const result = {
|
|
202
|
-
error: "usage and balance are unavailable",
|
|
209
|
+
error: rateLimited ? "usage rate limit reached; retry shortly" : "usage and balance are unavailable",
|
|
203
210
|
checkedAt: Date.now()
|
|
204
211
|
};
|
|
205
212
|
const previous = lastGood.get(reportKey);
|
|
206
|
-
if (epoch === generation) cache.set(reportKey, result);
|
|
213
|
+
if (!rateLimited && epoch === generation) cache.set(reportKey, result);
|
|
207
214
|
return previous === void 0 ? result : {
|
|
208
215
|
...previous,
|
|
209
216
|
stale: true,
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@goodandready/dsh-ollama-cloud-provider",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.40",
|
|
4
4
|
"description": "Ollama Cloud LLM provider plugin for DeepSeek Harness (dsh).",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"type": "module",
|
|
@@ -73,7 +73,7 @@
|
|
|
73
73
|
"build": "tsdown",
|
|
74
74
|
"prepare": "tsdown 1>&2",
|
|
75
75
|
"typecheck": "tsc --noEmit",
|
|
76
|
-
"test": "node --import tsx --test --test-concurrency=1 tests/core.test.ts tests/usage.test.ts tests/adapter.test.ts tests/routes.test.ts tests/native.test.ts tests/openai-chat.test.ts tests/commands.test.ts tests/tui-settings.test.ts tests/analytics.test.ts tests/apply.test.ts tests/account-rules.test.ts tests/request-log.test.ts tests/image-budget.test.ts tests/tool-schema.test.ts tests/tools.test.ts tests/plugin-updater.test.ts tests/compat.test.ts tests/status.test.ts tests/settings-draft.test.ts tests/update-state.test.ts tests/ui-render.test.tsx tests/update-card-view.test.tsx tests/web.test.ts tests/mock-ollama.test.ts tests/reasoning.test.ts tests/catalog-cache.test.ts tests/attempt.test.ts tests/model-groups.test.tsx tests/discovery.test.ts tests/sanitize.test.ts tests/retry.test.ts tests/provider-card.test.tsx tests/host-verification.test.ts tests/ui-actions.test.tsx tests/tool-call-id.test.ts",
|
|
76
|
+
"test": "node --import tsx --test --test-concurrency=1 tests/core.test.ts tests/usage.test.ts tests/adapter.test.ts tests/routes.test.ts tests/native.test.ts tests/openai-chat.test.ts tests/commands.test.ts tests/tui-settings.test.ts tests/analytics.test.ts tests/apply.test.ts tests/account-rules.test.ts tests/request-log.test.ts tests/image-budget.test.ts tests/tool-schema.test.ts tests/tools.test.ts tests/plugin-updater.test.ts tests/compat.test.ts tests/status.test.ts tests/settings-draft.test.ts tests/update-state.test.ts tests/ui-render.test.tsx tests/update-card-view.test.tsx tests/web.test.ts tests/mock-ollama.test.ts tests/reasoning.test.ts tests/catalog-cache.test.ts tests/attempt.test.ts tests/model-groups.test.tsx tests/discovery.test.ts tests/sanitize.test.ts tests/retry.test.ts tests/provider-card.test.tsx tests/host-verification.test.ts tests/ui-actions.test.tsx tests/tool-call-id.test.ts tests/deploy-script.test.ts tests/endpoints.test.ts tests/route-guard.test.ts tests/catalog-scheduler.test.ts",
|
|
77
77
|
"check": "npm run typecheck && npm test && npm run build && npm run pack:check",
|
|
78
78
|
"release": "npm run check && mkdir -p dist && npm pack --pack-destination dist",
|
|
79
79
|
"sync:models": "node scripts/sync-models.mjs",
|