dsh-connect-workbuddy 3.1.0 → 3.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +186 -0
- package/README.en.md +49 -20
- package/README.md +56 -26
- package/THIRD_PARTY_NOTICES.md +4 -2
- package/docs/WINDOWS.md +1 -0
- package/docs/assets/account-pool-demo.html +207 -262
- package/lib/bin.js +1 -1
- package/lib/client.js +457 -725
- package/lib/{host-heartbeat-DZnPggVT.js → host-heartbeat-C2_Zmuqq.js} +96 -3
- package/lib/index.d.ts +126 -32
- package/lib/index.js +1441 -278
- package/package.json +1 -1
package/lib/index.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
import { $ as
|
|
1
|
+
import { $ as readAtRestKey, A as CREDENTIAL_REJECTED_CODE, B as prepareChatBody, C as defaultDesktopAuthPath, D as parseWorkBuddyAuth, E as legacyWorkbuddyOwnAuthPath, F as declaredTools, G as deriveAtRestKeyId, H as WORKBUDDY_APP_EXECUTABLE_ENV, I as isCredentialRejectedError, J as isEncryptedFieldWrapper, K as fetchAtRestKeyPayload, L as parseCreditMultiplier, M as WorkBuddyCredentialRejectedError, N as WorkBuddyUpstreamClient, O as workbuddyAccountId, P as classifyUpstreamError, Q as openEncryptedField, R as parseReasoning, S as defaultDesktopAuthDirs, T as isEncryptedCredentialError, U as clearAtRestKeyCache, V as regionOf, W as deriveAtRestKey, X as macosBundleExecutable, Y as isWorkbuddyBundle, Z as macosNestedAppBundles, _ as WORKBUDDY_AUTH_FILE_ENV, a as readHostHeartbeat, b as authFileName, c as WORKBUDDY_CONNECT_VERSION, d as FALLBACK_WORKBUDDY_MODELS_GLOBAL, et as workbuddyAppExecutableCandidates, f as WorkBuddyCatalog, g as WORKBUDDY_AUTH_FILENAME, h as ENCRYPTED_CREDENTIAL_CODE, i as processStartTimeMs, j as WORKBUDDY_FALLBACK_SYSTEM_PROMPT, k as workbuddyOwnAuthPath, l as piAiRuntimeInfo, m as fallbackModelsFor, n as clearHostHeartbeat, o as workbuddyHostHeartbeatPath, p as deriveCatalog, q as findWorkbuddyAppExecutable, r as isHeartbeatProcessAlive, s as writeHostHeartbeat, t as WORKBUDDY_HOST_HEARTBEAT_FILENAME, u as FALLBACK_WORKBUDDY_MODELS, v as WorkBuddyCredentialStore, w as hasEncryptedCredentialFields, x as defaultDesktopAuthCandidates, y as WorkBuddyEncryptedCredentialError, z as parseUpstreamModel } from "./host-heartbeat-C2_Zmuqq.js";
|
|
2
2
|
import z from "@deepseek-ai/schemastery";
|
|
3
3
|
import { readFile } from "node:fs/promises";
|
|
4
|
-
import { randomBytes, timingSafeEqual } from "node:crypto";
|
|
4
|
+
import { randomBytes, randomUUID, timingSafeEqual } from "node:crypto";
|
|
5
5
|
import { join } from "node:path";
|
|
6
6
|
import { withFileLock, writeFileAtomic } from "@deepseek-ai/dsh-atomic-write";
|
|
7
7
|
import { resolveDshHome } from "@deepseek-ai/dsh-home-paths";
|
|
@@ -414,6 +414,692 @@ function createWorkBuddyAdapter(options) {
|
|
|
414
414
|
}
|
|
415
415
|
};
|
|
416
416
|
}
|
|
417
|
+
/** One or two full-width bars — the count is a spelling variable. */
|
|
418
|
+
const BARS = `||?`;
|
|
419
|
+
new RegExp(`${BARS}DSML${BARS}`, "g");
|
|
420
|
+
/**
|
|
421
|
+
* Spellings a tag may carry before its name, longest first so the doubled form
|
|
422
|
+
* is consumed before the single one can match its prefix.
|
|
423
|
+
*
|
|
424
|
+
* The ASCII spellings are accepted because the model mixes them: the marker is
|
|
425
|
+
* normalised to ASCII by {@link normalizedPrefixLength}, which is also the only
|
|
426
|
+
* way to catch a block that spells the two sides differently.
|
|
427
|
+
*/
|
|
428
|
+
const MARKUP_PREFIXES = [
|
|
429
|
+
`||DSML||`,
|
|
430
|
+
"||DSML||",
|
|
431
|
+
"|DSML|"
|
|
432
|
+
];
|
|
433
|
+
/** What the full-width marker looks like after {@link normalizedPrefixLength}. */
|
|
434
|
+
const ASCII_PREFIXES = ["||DSML||", "|DSML|"];
|
|
435
|
+
/**
|
|
436
|
+
* Tag names that can OPEN a block worth recovering.
|
|
437
|
+
*
|
|
438
|
+
* Deliberately tiny. `parameter` and the model's ad-hoc tag names are NOT here:
|
|
439
|
+
* they only ever appear inside a block, so treating one as a block start would
|
|
440
|
+
* hold a response hostage waiting for a close that naming never promised.
|
|
441
|
+
*/
|
|
442
|
+
const BLOCK_TAGS = ["tool_calls", "invoke"];
|
|
443
|
+
/**
|
|
444
|
+
* Tag names whose content belongs to the markup rather than to the answer.
|
|
445
|
+
*
|
|
446
|
+
* Used by {@link stripMarkup} to decide whether removing a marker-bearing tag
|
|
447
|
+
* should also remove everything up to its matching close. An INVENTED name is
|
|
448
|
+
* not here on purpose: `<|DSML| validate>` brackets nothing, so removing it
|
|
449
|
+
* means removing the tag and nothing else.
|
|
450
|
+
*/
|
|
451
|
+
const KEYWORD_TAGS = [
|
|
452
|
+
"tool_calls",
|
|
453
|
+
"tool-calls",
|
|
454
|
+
"toolcalls",
|
|
455
|
+
"invoke",
|
|
456
|
+
"parameter",
|
|
457
|
+
"calls",
|
|
458
|
+
"tool"
|
|
459
|
+
];
|
|
460
|
+
/** Tag names normalised to their canonical spelling. */
|
|
461
|
+
const CANONICAL_TAG_NAMES = /* @__PURE__ */ new Map([
|
|
462
|
+
["tool_calls", "tool_calls"],
|
|
463
|
+
["tool-calls", "tool_calls"],
|
|
464
|
+
["toolcalls", "tool_calls"],
|
|
465
|
+
["invoke", "invoke"],
|
|
466
|
+
["parameter", "parameter"]
|
|
467
|
+
]);
|
|
468
|
+
/**
|
|
469
|
+
* Canonical names accepted ONLY in a marker form.
|
|
470
|
+
*
|
|
471
|
+
* `<toolcalls>` without the marker is somebody's HTML, not the model's markup;
|
|
472
|
+
* the marker is what makes the misspelling meaningful.
|
|
473
|
+
*/
|
|
474
|
+
const MARKER_ONLY_TAG_NAMES = /* @__PURE__ */ new Set(["tool-calls", "toolcalls"]);
|
|
475
|
+
/** Fence markers at the start of a line. */
|
|
476
|
+
const FENCE_MARKERS = ["```", "~~~"];
|
|
477
|
+
/**
|
|
478
|
+
* Find every span that must not be scanned for markup.
|
|
479
|
+
*
|
|
480
|
+
* One pass, not a predicate call per character: the reference asks
|
|
481
|
+
* `is_inside_markdown_fence(text, i)` inside a per-character loop, which is
|
|
482
|
+
* quadratic in the size of the buffer. The buffer holds a whole block, and a
|
|
483
|
+
* block can be large (a file's worth of arguments), so the port computes the
|
|
484
|
+
* spans once and walks them.
|
|
485
|
+
*
|
|
486
|
+
* Semantics match the reference: fences only count at the start of a line,
|
|
487
|
+
* code spans are runs of one or two backticks, and an unterminated span runs to
|
|
488
|
+
* the end of the text (the closer may still be in flight).
|
|
489
|
+
*/
|
|
490
|
+
function ignoredSpans(text) {
|
|
491
|
+
const spans = [];
|
|
492
|
+
let i = 0;
|
|
493
|
+
let fence;
|
|
494
|
+
let fenceStart = 0;
|
|
495
|
+
while (i < text.length) {
|
|
496
|
+
const atLineStart = i === 0 || text.charAt(i - 1) === "\n";
|
|
497
|
+
if (fence !== void 0) {
|
|
498
|
+
if (atLineStart && text.startsWith(fence, i)) {
|
|
499
|
+
const lineEnd = text.indexOf("\n", i);
|
|
500
|
+
const end = lineEnd === -1 ? text.length : lineEnd + 1;
|
|
501
|
+
spans.push({
|
|
502
|
+
start: fenceStart,
|
|
503
|
+
end,
|
|
504
|
+
open: false
|
|
505
|
+
});
|
|
506
|
+
fence = void 0;
|
|
507
|
+
i = end;
|
|
508
|
+
continue;
|
|
509
|
+
}
|
|
510
|
+
i += 1;
|
|
511
|
+
continue;
|
|
512
|
+
}
|
|
513
|
+
if (atLineStart) {
|
|
514
|
+
const marker = FENCE_MARKERS.find((candidate) => text.startsWith(candidate, i));
|
|
515
|
+
if (marker !== void 0) {
|
|
516
|
+
fence = marker;
|
|
517
|
+
fenceStart = i;
|
|
518
|
+
i += marker.length;
|
|
519
|
+
continue;
|
|
520
|
+
}
|
|
521
|
+
}
|
|
522
|
+
if (text.charAt(i) === "`") {
|
|
523
|
+
const end = codeSpanEnd(text, i);
|
|
524
|
+
if (end === -1) {
|
|
525
|
+
spans.push({
|
|
526
|
+
start: i,
|
|
527
|
+
end: text.length,
|
|
528
|
+
open: true
|
|
529
|
+
});
|
|
530
|
+
break;
|
|
531
|
+
}
|
|
532
|
+
spans.push({
|
|
533
|
+
start: i,
|
|
534
|
+
end,
|
|
535
|
+
open: false
|
|
536
|
+
});
|
|
537
|
+
i = end;
|
|
538
|
+
continue;
|
|
539
|
+
}
|
|
540
|
+
const xml = xmlIgnoredEnd(text, i);
|
|
541
|
+
if (xml !== void 0) {
|
|
542
|
+
spans.push(xml);
|
|
543
|
+
if (xml.end >= text.length && xml.open) break;
|
|
544
|
+
i = xml.end;
|
|
545
|
+
continue;
|
|
546
|
+
}
|
|
547
|
+
i += 1;
|
|
548
|
+
}
|
|
549
|
+
if (fence !== void 0) spans.push({
|
|
550
|
+
start: fenceStart,
|
|
551
|
+
end: text.length,
|
|
552
|
+
open: true
|
|
553
|
+
});
|
|
554
|
+
return spans;
|
|
555
|
+
}
|
|
556
|
+
/**
|
|
557
|
+
* End of the inline code span starting at `start`, or -1 when there is none.
|
|
558
|
+
*
|
|
559
|
+
* Three or more backticks are a fence, not a code span — {@link ignoredSpans}
|
|
560
|
+
* handles those, and conflating the two is how a fence's opening ticks get
|
|
561
|
+
* mistaken for an unclosed inline span.
|
|
562
|
+
*/
|
|
563
|
+
function codeSpanEnd(text, start) {
|
|
564
|
+
let ticks = 0;
|
|
565
|
+
let i = start;
|
|
566
|
+
while (i < text.length && text.charAt(i) === "`") {
|
|
567
|
+
ticks += 1;
|
|
568
|
+
i += 1;
|
|
569
|
+
}
|
|
570
|
+
if (ticks === 0 || ticks >= 3) return -1;
|
|
571
|
+
let end = i;
|
|
572
|
+
while (end < text.length) {
|
|
573
|
+
if (text.charAt(end) !== "`") {
|
|
574
|
+
end += 1;
|
|
575
|
+
continue;
|
|
576
|
+
}
|
|
577
|
+
let closingTicks = 0;
|
|
578
|
+
let j = end;
|
|
579
|
+
while (j < text.length && text.charAt(j) === "`") {
|
|
580
|
+
closingTicks += 1;
|
|
581
|
+
j += 1;
|
|
582
|
+
}
|
|
583
|
+
if (closingTicks === ticks) return j;
|
|
584
|
+
end = j;
|
|
585
|
+
}
|
|
586
|
+
return -1;
|
|
587
|
+
}
|
|
588
|
+
/** CDATA / comment / processing instruction at `index`, if one starts there. */
|
|
589
|
+
function xmlIgnoredEnd(text, index) {
|
|
590
|
+
for (const [open, close] of [
|
|
591
|
+
["<![CDATA[", "]]>"],
|
|
592
|
+
["<!--", "-->"],
|
|
593
|
+
["<?", "?>"]
|
|
594
|
+
]) {
|
|
595
|
+
if (!text.startsWith(open, index)) continue;
|
|
596
|
+
const found = text.indexOf(close, index + open.length);
|
|
597
|
+
if (found === -1) return {
|
|
598
|
+
start: index,
|
|
599
|
+
end: text.length,
|
|
600
|
+
open: true
|
|
601
|
+
};
|
|
602
|
+
return {
|
|
603
|
+
start: index,
|
|
604
|
+
end: found + close.length,
|
|
605
|
+
open: false
|
|
606
|
+
};
|
|
607
|
+
}
|
|
608
|
+
}
|
|
609
|
+
/**
|
|
610
|
+
* Index just past the ignored span containing `index`, or `index` unchanged.
|
|
611
|
+
*
|
|
612
|
+
* `cursor` is a hint into the sorted span list; callers that walk positions in
|
|
613
|
+
* order get O(1) amortised lookup, which is why the parameter exists.
|
|
614
|
+
*/
|
|
615
|
+
function skipIgnored(spans, index, cursor) {
|
|
616
|
+
while (cursor.at < spans.length) {
|
|
617
|
+
const span = spans[cursor.at];
|
|
618
|
+
if (span === void 0 || index < span.start) return index;
|
|
619
|
+
if (index < span.end) return span.end;
|
|
620
|
+
cursor.at += 1;
|
|
621
|
+
}
|
|
622
|
+
return index;
|
|
623
|
+
}
|
|
624
|
+
/**
|
|
625
|
+
* Length of the marker prefix at `index`, or 0 when none is there.
|
|
626
|
+
*
|
|
627
|
+
* Two passes on purpose. The literal spellings are tried first; then the text
|
|
628
|
+
* is normalised character by character, which is the only way to recognise a
|
|
629
|
+
* block that mixes the two bar styles — and mixing is normal: one captured
|
|
630
|
+
* emission spelled the opening tags with single bars and the closer with double.
|
|
631
|
+
*/
|
|
632
|
+
function prefixLengthAt(text, index) {
|
|
633
|
+
for (const prefix of MARKUP_PREFIXES) if (text.startsWith(prefix, index)) return prefix.length;
|
|
634
|
+
const normalized = normalizeFullwidth(text.slice(index, index + 8));
|
|
635
|
+
for (const prefix of ASCII_PREFIXES) if (normalized.startsWith(prefix)) return prefix.length;
|
|
636
|
+
return 0;
|
|
637
|
+
}
|
|
638
|
+
/** Is a single character usable inside a tag name? Mirrors Python's `isalnum()`. */
|
|
639
|
+
function isNameCharacter(text, index) {
|
|
640
|
+
return /[\p{L}\p{N}_-]/u.test(text.charAt(index));
|
|
641
|
+
}
|
|
642
|
+
/** Read a tag name at `index` (after any marker prefix). */
|
|
643
|
+
function readName(text, index) {
|
|
644
|
+
const prefixLength = prefixLengthAt(text, index);
|
|
645
|
+
const marked = prefixLength > 0;
|
|
646
|
+
let position = index + prefixLength;
|
|
647
|
+
while (position < text.length && /\s/.test(text.charAt(position))) position += 1;
|
|
648
|
+
const start = position;
|
|
649
|
+
while (position < text.length && isNameCharacter(text, position)) position += 1;
|
|
650
|
+
if (position === start) return {
|
|
651
|
+
name: "",
|
|
652
|
+
end: index,
|
|
653
|
+
marked
|
|
654
|
+
};
|
|
655
|
+
return {
|
|
656
|
+
name: text.slice(start, position).toLowerCase(),
|
|
657
|
+
end: position,
|
|
658
|
+
marked
|
|
659
|
+
};
|
|
660
|
+
}
|
|
661
|
+
/**
|
|
662
|
+
* Scan a tag at `index`, or nothing when there is not one.
|
|
663
|
+
*
|
|
664
|
+
* Any name is accepted (the model invents parameter tags freely); only the ones
|
|
665
|
+
* in {@link CANONICAL_TAG_NAMES} are normalised, and the misspellings are
|
|
666
|
+
* normalised only in a marker form — `<toolcalls>` alone is somebody's HTML.
|
|
667
|
+
*/
|
|
668
|
+
function scanTag(text, index) {
|
|
669
|
+
if (text.charAt(index) !== "<") return void 0;
|
|
670
|
+
let position = index + 1;
|
|
671
|
+
let closing = false;
|
|
672
|
+
if (text.charAt(position) === "/") {
|
|
673
|
+
closing = true;
|
|
674
|
+
position += 1;
|
|
675
|
+
}
|
|
676
|
+
while (position < text.length && /\s/.test(text.charAt(position))) position += 1;
|
|
677
|
+
const read = readName(text, position);
|
|
678
|
+
if (read.name === "") return void 0;
|
|
679
|
+
const canonical = CANONICAL_TAG_NAMES.get(read.name);
|
|
680
|
+
const name = canonical !== void 0 && !(MARKER_ONLY_TAG_NAMES.has(read.name) && !read.marked) ? canonical : read.name;
|
|
681
|
+
const attributesStart = read.end;
|
|
682
|
+
let selfClosing = false;
|
|
683
|
+
let cursor = read.end;
|
|
684
|
+
while (cursor < text.length) {
|
|
685
|
+
const character = text.charAt(cursor);
|
|
686
|
+
if (character === ">") return {
|
|
687
|
+
start: index,
|
|
688
|
+
end: cursor,
|
|
689
|
+
name,
|
|
690
|
+
closing,
|
|
691
|
+
selfClosing,
|
|
692
|
+
marked: read.marked,
|
|
693
|
+
attributes: text.slice(attributesStart, cursor).trim()
|
|
694
|
+
};
|
|
695
|
+
if (character === "/" && text.charAt(cursor + 1) === ">") {
|
|
696
|
+
selfClosing = true;
|
|
697
|
+
return {
|
|
698
|
+
start: index,
|
|
699
|
+
end: cursor + 1,
|
|
700
|
+
name,
|
|
701
|
+
closing,
|
|
702
|
+
selfClosing,
|
|
703
|
+
marked: read.marked,
|
|
704
|
+
attributes: text.slice(attributesStart, cursor).trim()
|
|
705
|
+
};
|
|
706
|
+
}
|
|
707
|
+
cursor += 1;
|
|
708
|
+
}
|
|
709
|
+
}
|
|
710
|
+
/** The next markup tag outside every ignored span, at or after `from`. */
|
|
711
|
+
function findTag(text, spans, from) {
|
|
712
|
+
const cursor = { at: 0 };
|
|
713
|
+
let index = Math.max(from, 0);
|
|
714
|
+
while (index < text.length) {
|
|
715
|
+
const skipped = skipIgnored(spans, index, cursor);
|
|
716
|
+
if (skipped !== index) {
|
|
717
|
+
index = skipped;
|
|
718
|
+
continue;
|
|
719
|
+
}
|
|
720
|
+
if (text.charAt(index) === "<") {
|
|
721
|
+
const tag = scanTag(text, index);
|
|
722
|
+
if (tag !== void 0) return tag;
|
|
723
|
+
}
|
|
724
|
+
index += 1;
|
|
725
|
+
}
|
|
726
|
+
}
|
|
727
|
+
/**
|
|
728
|
+
* The close tag matching `open`, or nothing when it has not arrived yet.
|
|
729
|
+
*
|
|
730
|
+
* Depth counting so a nested block of the same name cannot terminate its parent
|
|
731
|
+
* early. This is gate 1's teeth: no close, no call.
|
|
732
|
+
*/
|
|
733
|
+
function findMatchingClose(text, spans, open) {
|
|
734
|
+
let depth = 1;
|
|
735
|
+
let index = open.end + 1;
|
|
736
|
+
while (index < text.length) {
|
|
737
|
+
const tag = findTag(text, spans, index);
|
|
738
|
+
if (tag === void 0) return void 0;
|
|
739
|
+
if (tag.name === open.name) {
|
|
740
|
+
if (tag.closing) {
|
|
741
|
+
depth -= 1;
|
|
742
|
+
if (depth === 0) return tag;
|
|
743
|
+
} else if (!tag.selfClosing) depth += 1;
|
|
744
|
+
}
|
|
745
|
+
index = tag.end + 1;
|
|
746
|
+
}
|
|
747
|
+
}
|
|
748
|
+
/**
|
|
749
|
+
* The first block-opening tag in `text`, with its close when one has arrived.
|
|
750
|
+
*
|
|
751
|
+
* The close is OPTIONAL on purpose, and the two callers need it differently:
|
|
752
|
+
* the buffer holds from `open` while it is missing (gate 1 — the block may
|
|
753
|
+
* still complete), whereas parsing already-complete text stops rather than
|
|
754
|
+
* correcting a broken block and then trusting what follows it.
|
|
755
|
+
*/
|
|
756
|
+
function findFirstOpener(text, spans) {
|
|
757
|
+
let index = 0;
|
|
758
|
+
while (index < text.length) {
|
|
759
|
+
const tag = findTag(text, spans, index);
|
|
760
|
+
if (tag === void 0) return void 0;
|
|
761
|
+
if (!tag.closing && BLOCK_TAGS.includes(tag.name)) {
|
|
762
|
+
const close = findMatchingClose(text, spans, tag);
|
|
763
|
+
return close === void 0 ? { open: tag } : {
|
|
764
|
+
open: tag,
|
|
765
|
+
close
|
|
766
|
+
};
|
|
767
|
+
}
|
|
768
|
+
index = tag.end + 1;
|
|
769
|
+
}
|
|
770
|
+
}
|
|
771
|
+
/** Decode the five predefined XML entities plus numeric references. */
|
|
772
|
+
function decodeEntities(text) {
|
|
773
|
+
if (!text.includes("&")) return text;
|
|
774
|
+
const named = /* @__PURE__ */ new Map([
|
|
775
|
+
["amp", "&"],
|
|
776
|
+
["lt", "<"],
|
|
777
|
+
["gt", ">"],
|
|
778
|
+
["quot", "\""],
|
|
779
|
+
["apos", "'"]
|
|
780
|
+
]);
|
|
781
|
+
return text.replace(/&(#[0-9]+|#x[0-9a-f]+|[a-z]+);/gi, (whole, body) => {
|
|
782
|
+
if (body.startsWith("#")) {
|
|
783
|
+
const hex = body.charAt(1).toLowerCase() === "x";
|
|
784
|
+
const digits = hex ? body.slice(2) : body.slice(1);
|
|
785
|
+
const code = Number.parseInt(digits, hex ? 16 : 10);
|
|
786
|
+
if (!Number.isFinite(code) || code <= 0 || code > 1114111) return whole;
|
|
787
|
+
return String.fromCodePoint(code);
|
|
788
|
+
}
|
|
789
|
+
return named.get(body.toLowerCase()) ?? whole;
|
|
790
|
+
});
|
|
791
|
+
}
|
|
792
|
+
/** `name="value"` / `name='value'` pairs, entities decoded. */
|
|
793
|
+
function parseAttributes(text) {
|
|
794
|
+
const attributes = {};
|
|
795
|
+
const pattern = /([a-z0-9_:-]+)\s*=\s*(?:"([^"]*)"|'([^']*)')/gi;
|
|
796
|
+
let match = pattern.exec(text);
|
|
797
|
+
while (match !== null) {
|
|
798
|
+
const key = match[1];
|
|
799
|
+
const value = match[2] ?? match[3];
|
|
800
|
+
if (key !== void 0 && value !== void 0) attributes[key] = decodeEntities(value);
|
|
801
|
+
match = pattern.exec(text);
|
|
802
|
+
}
|
|
803
|
+
return attributes;
|
|
804
|
+
}
|
|
805
|
+
/**
|
|
806
|
+
* Parse the parameters inside one block body.
|
|
807
|
+
*
|
|
808
|
+
* Two shapes, in the order the model prefers them:
|
|
809
|
+
* - `<|DSML|parameter name="cmd">value</…parameter>` (the explicit form);
|
|
810
|
+
* - `<cmd>value</cmd>` (the tag name IS the parameter name), used only when
|
|
811
|
+
* the explicit form has not already claimed that name — otherwise a nested
|
|
812
|
+
* tag echoing it would overwrite the real value.
|
|
813
|
+
*
|
|
814
|
+
* A `CDATA` body is taken verbatim; everything else is entity-decoded.
|
|
815
|
+
*
|
|
816
|
+
* Every scanner here computes its ignored spans from the string it is about to
|
|
817
|
+
* scan. Span offsets are only meaningful against the text they were measured
|
|
818
|
+
* on, and a body is a substring of its parent — reusing the parent's offsets
|
|
819
|
+
* would misplace every fence and silently turn "quoted markup" into "called
|
|
820
|
+
* markup", which is gate 4's worst failure. Each of these strings is small, so
|
|
821
|
+
* recomputing is cheaper than passing a coordinate system around.
|
|
822
|
+
*/
|
|
823
|
+
function parseParameters(body) {
|
|
824
|
+
const parameters = {};
|
|
825
|
+
const spans = ignoredSpans(body);
|
|
826
|
+
let index = 0;
|
|
827
|
+
while (index < body.length) {
|
|
828
|
+
const tag = findTag(body, spans, index);
|
|
829
|
+
if (tag === void 0) break;
|
|
830
|
+
if (tag.closing || tag.selfClosing) {
|
|
831
|
+
index = tag.end + 1;
|
|
832
|
+
continue;
|
|
833
|
+
}
|
|
834
|
+
const close = findMatchingClose(body, spans, tag);
|
|
835
|
+
if (close === void 0) {
|
|
836
|
+
index = tag.end + 1;
|
|
837
|
+
continue;
|
|
838
|
+
}
|
|
839
|
+
const value = body.slice(tag.end + 1, close.start).trim();
|
|
840
|
+
const attributes = parseAttributes(tag.attributes);
|
|
841
|
+
const explicitName = tag.name === "parameter" ? attributes["name"] : void 0;
|
|
842
|
+
const name = explicitName ?? tag.name;
|
|
843
|
+
if (name !== void 0 && name !== "" && (explicitName !== void 0 || parameters[name] === void 0)) parameters[name] = value.startsWith("<![CDATA[") && value.endsWith("]]>") ? value.slice(9, -3) : nestedOrText(value);
|
|
844
|
+
index = close.end + 1;
|
|
845
|
+
}
|
|
846
|
+
return parameters;
|
|
847
|
+
}
|
|
848
|
+
/** A value that itself contains a complete tag pair is a nested object. */
|
|
849
|
+
function nestedOrText(value) {
|
|
850
|
+
if (!value.includes("<") || !value.includes(">")) return decodeEntities(value);
|
|
851
|
+
const nested = parseParameters(value);
|
|
852
|
+
return Object.keys(nested).length > 0 ? nested : decodeEntities(value);
|
|
853
|
+
}
|
|
854
|
+
/**
|
|
855
|
+
* One block's worth of text → the calls it declares, or nothing.
|
|
856
|
+
*
|
|
857
|
+
* This is where gates 2 and 4 live. Returning nothing is the COMMON case for
|
|
858
|
+
* mistaken input and is not an error: callers must show the block verbatim.
|
|
859
|
+
*/
|
|
860
|
+
function parseBlock(block, gate) {
|
|
861
|
+
const calls = [];
|
|
862
|
+
const spans = ignoredSpans(block);
|
|
863
|
+
let index = 0;
|
|
864
|
+
while (index < block.length) {
|
|
865
|
+
const tag = findTag(block, spans, index);
|
|
866
|
+
if (tag === void 0) break;
|
|
867
|
+
if (!tag.closing && tag.name === "invoke") {
|
|
868
|
+
const close = findMatchingClose(block, spans, tag);
|
|
869
|
+
if (close === void 0) break;
|
|
870
|
+
const call = parseInvoke(tag, close, block, gate);
|
|
871
|
+
if (call !== void 0) calls.push(call);
|
|
872
|
+
index = close.end + 1;
|
|
873
|
+
continue;
|
|
874
|
+
}
|
|
875
|
+
index = tag.end + 1;
|
|
876
|
+
}
|
|
877
|
+
return calls;
|
|
878
|
+
}
|
|
879
|
+
/** One `<invoke>` element → a call, or nothing when a gate refuses it. */
|
|
880
|
+
function parseInvoke(open, close, block, gate) {
|
|
881
|
+
const name = parseAttributes(open.attributes)["name"];
|
|
882
|
+
if (name === void 0 || name === "") return void 0;
|
|
883
|
+
if (!gate.declaredNames.has(name)) return void 0;
|
|
884
|
+
const pinned = gate.pinnedToolName;
|
|
885
|
+
if (pinned !== void 0 && pinned !== "auto" && pinned !== "required" && pinned !== "none" && pinned !== name) return;
|
|
886
|
+
const parameters = parseParameters(block.slice(open.end + 1, close.start));
|
|
887
|
+
const required = gate.requiredParameters?.get(name);
|
|
888
|
+
if (required !== void 0 && required.some((parameter) => parameters[parameter] === void 0)) return void 0;
|
|
889
|
+
return {
|
|
890
|
+
id: `call_${randomUUID().replace(/-/g, "").slice(0, 24)}`,
|
|
891
|
+
name,
|
|
892
|
+
arguments: JSON.stringify(parameters)
|
|
893
|
+
};
|
|
894
|
+
}
|
|
895
|
+
/**
|
|
896
|
+
* `text` with every markup block removed — the check behind {@link hasProse}.
|
|
897
|
+
*
|
|
898
|
+
* ANY marker-bearing tag counts as markup, whatever it is called. Keying on the
|
|
899
|
+
* known names alone was the first attempt and it got the one shape this project
|
|
900
|
+
* has actually captured wrong: `<|DSML| validate>` carries an invented name
|
|
901
|
+
* (`validate` is the TOOL name, written where `invoke name="…"` belongs), so a
|
|
902
|
+
* keyword-only rule left it standing, the response was classified as prose, and
|
|
903
|
+
* the retry-when-the-turn-produced-nothing rule could never fire for the very
|
|
904
|
+
* leak it was written for — a check that never fires, which is the trap this
|
|
905
|
+
* project has already paid for twice.
|
|
906
|
+
*
|
|
907
|
+
* Removal reaches the matching close only for names that bracket content
|
|
908
|
+
* ({@link KEYWORD_TAGS}); an invented name removes just its own tag, because
|
|
909
|
+
* nothing says it opens anything.
|
|
910
|
+
*
|
|
911
|
+
* Ignored spans are KEPT: a fenced or back-ticked quotation of the marker is
|
|
912
|
+
* prose, and the rule is explicitly about a turn that produced nothing usable.
|
|
913
|
+
*/
|
|
914
|
+
function stripMarkup(text) {
|
|
915
|
+
const spans = ignoredSpans(text);
|
|
916
|
+
const cursor = { at: 0 };
|
|
917
|
+
let result = "";
|
|
918
|
+
let index = 0;
|
|
919
|
+
while (index < text.length) {
|
|
920
|
+
const skipped = skipIgnored(spans, index, cursor);
|
|
921
|
+
if (skipped !== index) {
|
|
922
|
+
result += text.slice(index, skipped);
|
|
923
|
+
index = skipped;
|
|
924
|
+
continue;
|
|
925
|
+
}
|
|
926
|
+
const tag = text.charAt(index) === "<" ? scanTag(text, index) : void 0;
|
|
927
|
+
if (tag === void 0 || !tag.marked) {
|
|
928
|
+
result += text.charAt(index);
|
|
929
|
+
index += 1;
|
|
930
|
+
continue;
|
|
931
|
+
}
|
|
932
|
+
const close = !tag.closing && !tag.selfClosing && KEYWORD_TAGS.includes(tag.name) ? findMatchingClose(text, spans, tag) : void 0;
|
|
933
|
+
index = close === void 0 ? tag.end + 1 : close.end + 1;
|
|
934
|
+
}
|
|
935
|
+
return result;
|
|
936
|
+
}
|
|
937
|
+
/**
|
|
938
|
+
* Does `text` carry content outside markup blocks?
|
|
939
|
+
*
|
|
940
|
+
* This is what makes "the turn produced nothing usable" a checkable fact rather
|
|
941
|
+
* than a guess: a response whose only content is markup residue returns false,
|
|
942
|
+
* and the caller may then consider a single retry. Text inside a fence or code
|
|
943
|
+
* span counts as content — a documented quotation of the marker is prose.
|
|
944
|
+
*/
|
|
945
|
+
function hasProse(text) {
|
|
946
|
+
return stripMarkup(text).trim() !== "";
|
|
947
|
+
}
|
|
948
|
+
/**
|
|
949
|
+
* Which of `text`'s tail must wait for more bytes.
|
|
950
|
+
*
|
|
951
|
+
* Two reasons to hold, and only these two:
|
|
952
|
+
* - the tail starts a tag that is not closed yet, and could still turn out to
|
|
953
|
+
* be a block opener (`…<|DSML|inv`), so committing it as text would make
|
|
954
|
+
* the block unparseable;
|
|
955
|
+
* - the tail opens a code span whose closer is still in flight, because
|
|
956
|
+
* whether the following markup is *quoted* depends on that span.
|
|
957
|
+
*
|
|
958
|
+
* Anything else is emitted immediately. In particular a COMPLETE tag that is
|
|
959
|
+
* not a block opener — `<|DSML| validate>`, the shape this project actually
|
|
960
|
+
* captured — is not held: it is prose as far as this layer is concerned, and
|
|
961
|
+
* holding it would delay every ordinary answer that mentions the markup.
|
|
962
|
+
*/
|
|
963
|
+
function holdFrom(text, spans) {
|
|
964
|
+
let hold = text.length;
|
|
965
|
+
const lastLt = text.lastIndexOf("<");
|
|
966
|
+
if (lastLt !== -1 && looksLikeTagStart(text.slice(lastLt))) hold = Math.min(hold, lastLt);
|
|
967
|
+
for (const span of spans) if (span.open) hold = Math.min(hold, span.start);
|
|
968
|
+
return hold;
|
|
969
|
+
}
|
|
970
|
+
/**
|
|
971
|
+
* Could this fragment be the beginning of a block-opening tag?
|
|
972
|
+
*
|
|
973
|
+
* Conservative by construction: it requires an unterminated `<`, and after
|
|
974
|
+
* stripping an optional `/`, whitespace and marker, the remainder must be a
|
|
975
|
+
* prefix of (or extend) a known opener. That is what keeps `a < b`, `<50%` and
|
|
976
|
+
* `i < n` in ordinary prose from stalling a stream — each of them either
|
|
977
|
+
* contains a `>` or fails the opener test.
|
|
978
|
+
*
|
|
979
|
+
* A marker that is STILL ARRIVING also holds (`<|`, `<|DSML`, `||D`). Without
|
|
980
|
+
* that case the leading `<|` is emitted as text the moment it arrives, and the
|
|
981
|
+
* block can never be recovered afterwards because its opening tag has already
|
|
982
|
+
* been split in two — a byte-by-byte feed is how this was found, and the
|
|
983
|
+
* reference implementation has the same hole.
|
|
984
|
+
*/
|
|
985
|
+
function looksLikeTagStart(tail) {
|
|
986
|
+
if (!tail.startsWith("<")) return false;
|
|
987
|
+
if (tail.includes(">")) return false;
|
|
988
|
+
let body = tail.slice(1);
|
|
989
|
+
if (body.startsWith("/")) body = body.slice(1);
|
|
990
|
+
if (body === "" || isPartialMarker(body)) return true;
|
|
991
|
+
const prefixLength = prefixLengthAt(body, 0);
|
|
992
|
+
body = body.slice(prefixLength).trim().toLowerCase();
|
|
993
|
+
if (body === "" || isPartialMarker(body)) return true;
|
|
994
|
+
return [
|
|
995
|
+
"tool",
|
|
996
|
+
"invoke",
|
|
997
|
+
"tool_calls",
|
|
998
|
+
"tool-calls",
|
|
999
|
+
"toolcalls",
|
|
1000
|
+
"tool_call",
|
|
1001
|
+
"tool-call"
|
|
1002
|
+
].some((opener) => opener.startsWith(body) || body.startsWith(opener));
|
|
1003
|
+
}
|
|
1004
|
+
/**
|
|
1005
|
+
* Is this the start of a marker whose remainder has not arrived yet?
|
|
1006
|
+
*
|
|
1007
|
+
* Compared both literally and full-width-normalised, because the model mixes
|
|
1008
|
+
* the two bar styles and a mixture splits at an unpredictable character.
|
|
1009
|
+
*/
|
|
1010
|
+
function isPartialMarker(body) {
|
|
1011
|
+
const trimmed = body.trimStart();
|
|
1012
|
+
if (trimmed === "") return true;
|
|
1013
|
+
const forms = [...MARKUP_PREFIXES, ...ASCII_PREFIXES];
|
|
1014
|
+
return [trimmed, normalizeFullwidth(trimmed.slice(0, 8))].some((candidate) => candidate !== "" && forms.some((form) => form.startsWith(candidate) && candidate.length < form.length));
|
|
1015
|
+
}
|
|
1016
|
+
/** Full-width ASCII (U+FF01..U+FF5E) mapped to its ASCII counterpart. */
|
|
1017
|
+
function normalizeFullwidth(text) {
|
|
1018
|
+
let normalized = "";
|
|
1019
|
+
for (const character of text) {
|
|
1020
|
+
const code = character.codePointAt(0) ?? 0;
|
|
1021
|
+
normalized += code >= 65281 && code <= 65374 ? String.fromCodePoint(code - 65248) : character;
|
|
1022
|
+
}
|
|
1023
|
+
return normalized;
|
|
1024
|
+
}
|
|
1025
|
+
/**
|
|
1026
|
+
* Buffers `delta.content` and commits a call only when the block that declares
|
|
1027
|
+
* it is complete.
|
|
1028
|
+
*
|
|
1029
|
+
* Ported from the reference's `ToolCallStreamBuffer` with its hard-won
|
|
1030
|
+
* behaviour intact:
|
|
1031
|
+
*
|
|
1032
|
+
* - a block with no close tag yet holds the ENTIRE block (not a partial
|
|
1033
|
+
* guess), and only the tail that could still become a tag is held back
|
|
1034
|
+
* from otherwise-finished text;
|
|
1035
|
+
* - a block that fails the gates is emitted VERBATIM (gate 4), not dropped;
|
|
1036
|
+
* - {@link flush} hands back whatever is still held, so a truncated block
|
|
1037
|
+
* loses none of its text.
|
|
1038
|
+
*
|
|
1039
|
+
* The reference's own comment explains why the tail rule is not "hold
|
|
1040
|
+
* everything from the first `<`": an architecture document that shows
|
|
1041
|
+
* `<tool_calls><invoke>` as an EXAMPLE would otherwise starve the rest of the
|
|
1042
|
+
* response until the stream ended, and then lose it.
|
|
1043
|
+
*/
|
|
1044
|
+
var DsmlStreamBuffer = class {
|
|
1045
|
+
pending = "";
|
|
1046
|
+
gate;
|
|
1047
|
+
constructor(gate) {
|
|
1048
|
+
this.gate = gate;
|
|
1049
|
+
}
|
|
1050
|
+
/** Feed one `delta.content` chunk; returns text and any call it completed. */
|
|
1051
|
+
add(chunk) {
|
|
1052
|
+
this.pending += chunk;
|
|
1053
|
+
const parts = [];
|
|
1054
|
+
const calls = [];
|
|
1055
|
+
if (this.gate.declaredNames.size === 0) {
|
|
1056
|
+
const text = this.pending;
|
|
1057
|
+
this.pending = "";
|
|
1058
|
+
return {
|
|
1059
|
+
text,
|
|
1060
|
+
prose: hasProse(text)
|
|
1061
|
+
};
|
|
1062
|
+
}
|
|
1063
|
+
for (;;) {
|
|
1064
|
+
const found = findFirstOpener(this.pending, ignoredSpans(this.pending));
|
|
1065
|
+
if (found === void 0) break;
|
|
1066
|
+
const head = this.pending.slice(0, found.open.start);
|
|
1067
|
+
if (head !== "") parts.push(head);
|
|
1068
|
+
if (found.close === void 0) {
|
|
1069
|
+
this.pending = this.pending.slice(found.open.start);
|
|
1070
|
+
return finish(parts, calls);
|
|
1071
|
+
}
|
|
1072
|
+
const body = this.pending.slice(found.open.start, found.close.end + 1);
|
|
1073
|
+
const recovered = parseBlock(body, this.gate);
|
|
1074
|
+
if (recovered.length > 0) calls.push(...recovered);
|
|
1075
|
+
else parts.push(body);
|
|
1076
|
+
this.pending = this.pending.slice(found.close.end + 1);
|
|
1077
|
+
}
|
|
1078
|
+
const hold = holdFrom(this.pending, ignoredSpans(this.pending));
|
|
1079
|
+
if (hold > 0) parts.push(this.pending.slice(0, hold));
|
|
1080
|
+
this.pending = this.pending.slice(hold);
|
|
1081
|
+
return finish(parts, calls);
|
|
1082
|
+
}
|
|
1083
|
+
/** End of stream: hand back everything still held, verbatim. */
|
|
1084
|
+
flush() {
|
|
1085
|
+
const text = this.pending;
|
|
1086
|
+
this.pending = "";
|
|
1087
|
+
return {
|
|
1088
|
+
text,
|
|
1089
|
+
prose: hasProse(text)
|
|
1090
|
+
};
|
|
1091
|
+
}
|
|
1092
|
+
};
|
|
1093
|
+
/** Assemble one `add()` outcome without ever emitting a `calls: undefined` key. */
|
|
1094
|
+
function finish(parts, calls) {
|
|
1095
|
+
const text = parts.join("");
|
|
1096
|
+
const outcome = {
|
|
1097
|
+
text,
|
|
1098
|
+
prose: hasProse(text)
|
|
1099
|
+
};
|
|
1100
|
+
if (calls.length > 0) outcome.calls = [...calls];
|
|
1101
|
+
return outcome;
|
|
1102
|
+
}
|
|
417
1103
|
//#endregion
|
|
418
1104
|
//#region src/shim.ts
|
|
419
1105
|
/**
|
|
@@ -519,12 +1205,352 @@ function readBody(req) {
|
|
|
519
1205
|
});
|
|
520
1206
|
}
|
|
521
1207
|
/**
|
|
1208
|
+
* Rewrites the upstream SSE stream, converting markup the model wrote into the
|
|
1209
|
+
* text back into a real `delta.tool_calls` — and, when a whole turn produced
|
|
1210
|
+
* nothing but markup, holding the bytes long enough to retry ONCE.
|
|
1211
|
+
*
|
|
1212
|
+
* ============================================================================
|
|
1213
|
+
* Why this class exists rather than a `.pipe()`
|
|
1214
|
+
* ============================================================================
|
|
1215
|
+
*
|
|
1216
|
+
* Recovering a call means inspecting `delta.content` before the client sees it,
|
|
1217
|
+
* which the old `body.pipe(res)` could not do. The parsing rules live in
|
|
1218
|
+
* `src/dsml-recovery.ts`; everything here is about the STREAM's obligations:
|
|
1219
|
+
*
|
|
1220
|
+
* 1. FRAME ORDER IS NEVER CHANGED. Frames are forwarded in arrival order.
|
|
1221
|
+
* The only frame this class invents is the trailing one that carries a
|
|
1222
|
+
* block still unfinished at end-of-stream.
|
|
1223
|
+
*
|
|
1224
|
+
* 2. HOLDS ARE TEMPORARY AND PAY FOR THEMSELVES. Nothing is held unless the
|
|
1225
|
+
* answer so far consists only of markup: the moment real prose or a real
|
|
1226
|
+
* call appears, everything held is flushed in order and the stream returns
|
|
1227
|
+
* to plain pass-through. An ordinary answer therefore pays no latency at
|
|
1228
|
+
* all, and no byte is ever discarded while a retry is still possible.
|
|
1229
|
+
*
|
|
1230
|
+
* 3. A RETRY MUST NOT SPLICE TWO ANSWERS TOGETHER. That constraint is why the
|
|
1231
|
+
* hold exists: because no content frame has been written yet, the caller
|
|
1232
|
+
* can discard this attempt and re-send the same body. The existing
|
|
1233
|
+
* failover loop makes the same promise for failures BEFORE the stream
|
|
1234
|
+
* starts (see its comment in `chatCompletions`); this extends it to the
|
|
1235
|
+
* one case that can only be recognised after the fact.
|
|
1236
|
+
*
|
|
1237
|
+
* 4. NOTHING IS INVENTED WHEN THE UPSTREAM ALREADY SPEAKS STRUCTURED CALLS.
|
|
1238
|
+
* Once a native `delta.tool_calls` appears, recovery switches off for the
|
|
1239
|
+
* rest of the response — two live call channels for one answer is worse
|
|
1240
|
+
* than either one alone.
|
|
1241
|
+
*/
|
|
1242
|
+
var RecoveryStream = class {
|
|
1243
|
+
res;
|
|
1244
|
+
logger;
|
|
1245
|
+
carry = "";
|
|
1246
|
+
decoder = new TextDecoder("utf-8");
|
|
1247
|
+
/** Frames received after the hold began, in arrival order. */
|
|
1248
|
+
held = [];
|
|
1249
|
+
/**
|
|
1250
|
+
* The hold window, as two separate facts.
|
|
1251
|
+
*
|
|
1252
|
+
* `windowOpened` becomes true at the FIRST content-bearing frame and stays
|
|
1253
|
+
* true: before it, frames carry no answer (a role delta, a keep-alive) and go
|
|
1254
|
+
* straight out; from it on, they are held until the answer is known to be
|
|
1255
|
+
* real. `windowClosed` means "an answer was delivered", after which every
|
|
1256
|
+
* frame passes through again — for the rest of the response, because a turn
|
|
1257
|
+
* that has already produced prose can no longer be replaced by a retry.
|
|
1258
|
+
*/
|
|
1259
|
+
windowOpened = false;
|
|
1260
|
+
windowClosed = false;
|
|
1261
|
+
wroteContent = false;
|
|
1262
|
+
sawResidue = false;
|
|
1263
|
+
sawDone = false;
|
|
1264
|
+
/**
|
|
1265
|
+
* Set once the upstream speaks structured calls.
|
|
1266
|
+
*
|
|
1267
|
+
* Sticky for the whole response on purpose: recovering markup into a second
|
|
1268
|
+
* call channel after the upstream already produced a real one would hand the
|
|
1269
|
+
* client two answers to the same question.
|
|
1270
|
+
*/
|
|
1271
|
+
nativeCallsSeen = false;
|
|
1272
|
+
buffer;
|
|
1273
|
+
constructor(res, gate, logger) {
|
|
1274
|
+
this.res = res;
|
|
1275
|
+
this.logger = logger;
|
|
1276
|
+
this.buffer = gate === void 0 ? void 0 : new DsmlStreamBuffer(gate);
|
|
1277
|
+
}
|
|
1278
|
+
/** True when a call or real prose has reached the client. */
|
|
1279
|
+
get delivered() {
|
|
1280
|
+
return this.wroteContent;
|
|
1281
|
+
}
|
|
1282
|
+
/** True when the held bytes contain markup the gates refused. */
|
|
1283
|
+
get residueSeen() {
|
|
1284
|
+
return this.sawResidue;
|
|
1285
|
+
}
|
|
1286
|
+
/** Record that text the gates refused was seen (markup residue, not prose). */
|
|
1287
|
+
noteResidue(seen) {
|
|
1288
|
+
this.sawResidue ||= seen;
|
|
1289
|
+
}
|
|
1290
|
+
/** Drain one upstream body into the client. */
|
|
1291
|
+
async consume(body) {
|
|
1292
|
+
if (body === null) return;
|
|
1293
|
+
const source = Readable.fromWeb(body);
|
|
1294
|
+
try {
|
|
1295
|
+
for await (const chunk of source) this.push(this.decoder.decode(chunk, { stream: true }));
|
|
1296
|
+
} catch (error) {
|
|
1297
|
+
this.logger?.warn("dsh-connect-workbuddy: upstream stream failed mid-flight", error);
|
|
1298
|
+
}
|
|
1299
|
+
const tail = this.carry;
|
|
1300
|
+
this.carry = "";
|
|
1301
|
+
if (tail !== "") this.handleFrame(tail);
|
|
1302
|
+
}
|
|
1303
|
+
/**
|
|
1304
|
+
* Close the response: emit the block still unfinished at end-of-stream, then
|
|
1305
|
+
* everything held, then the terminator.
|
|
1306
|
+
*/
|
|
1307
|
+
finish() {
|
|
1308
|
+
const flushed = this.buffer?.flush();
|
|
1309
|
+
if (flushed !== void 0 && flushed.text !== "") {
|
|
1310
|
+
this.noteResidue(!flushed.prose);
|
|
1311
|
+
this.emitContent(flushed.text);
|
|
1312
|
+
}
|
|
1313
|
+
this.closeWindow();
|
|
1314
|
+
if (!this.sawDone && this.res.writable) this.res.write("data: [DONE]\n\n");
|
|
1315
|
+
if (this.res.writable) this.res.end();
|
|
1316
|
+
}
|
|
1317
|
+
/** Drop held bytes without writing them (used when a retry replaced them). */
|
|
1318
|
+
discard() {
|
|
1319
|
+
this.held = [];
|
|
1320
|
+
this.windowClosed = true;
|
|
1321
|
+
}
|
|
1322
|
+
/** Split incoming bytes into whole SSE frames, handling chunk boundaries. */
|
|
1323
|
+
push(text) {
|
|
1324
|
+
this.carry += text;
|
|
1325
|
+
let start = 0;
|
|
1326
|
+
for (;;) {
|
|
1327
|
+
const newline = this.carry.indexOf("\n", start);
|
|
1328
|
+
if (newline === -1) break;
|
|
1329
|
+
if (this.carry.slice(start, newline).trim() === "") {
|
|
1330
|
+
const frame = this.carry.slice(0, newline + 1);
|
|
1331
|
+
this.carry = this.carry.slice(newline + 1);
|
|
1332
|
+
start = 0;
|
|
1333
|
+
this.handleFrame(frame);
|
|
1334
|
+
continue;
|
|
1335
|
+
}
|
|
1336
|
+
start = newline + 1;
|
|
1337
|
+
}
|
|
1338
|
+
}
|
|
1339
|
+
/** Handle one complete frame, rewriting it only when recovery produced something. */
|
|
1340
|
+
handleFrame(frame) {
|
|
1341
|
+
const payload = dataPayload(frame);
|
|
1342
|
+
if (payload === void 0) {
|
|
1343
|
+
this.send(frame, false);
|
|
1344
|
+
return;
|
|
1345
|
+
}
|
|
1346
|
+
if (payload === "[DONE]") {
|
|
1347
|
+
this.sawDone = true;
|
|
1348
|
+
this.send(frame, false);
|
|
1349
|
+
return;
|
|
1350
|
+
}
|
|
1351
|
+
let chunk;
|
|
1352
|
+
try {
|
|
1353
|
+
chunk = JSON.parse(payload);
|
|
1354
|
+
} catch {
|
|
1355
|
+
this.send(frame, false);
|
|
1356
|
+
return;
|
|
1357
|
+
}
|
|
1358
|
+
const choices = chunk["choices"];
|
|
1359
|
+
const choice = Array.isArray(choices) ? choices[0] : void 0;
|
|
1360
|
+
if (typeof choice !== "object" || choice === null || Array.isArray(choice)) {
|
|
1361
|
+
this.send(frame, false);
|
|
1362
|
+
return;
|
|
1363
|
+
}
|
|
1364
|
+
const choiceRecord = choice;
|
|
1365
|
+
const deltaValue = choiceRecord["delta"];
|
|
1366
|
+
const delta = typeof deltaValue === "object" && deltaValue !== null && !Array.isArray(deltaValue) ? deltaValue : void 0;
|
|
1367
|
+
if (delta === void 0) {
|
|
1368
|
+
this.send(frame, false);
|
|
1369
|
+
return;
|
|
1370
|
+
}
|
|
1371
|
+
const nativeCalls = delta["tool_calls"];
|
|
1372
|
+
if (Array.isArray(nativeCalls) && nativeCalls.length > 0) {
|
|
1373
|
+
this.nativeCallsSeen = true;
|
|
1374
|
+
this.windowOpened = true;
|
|
1375
|
+
this.send(frame, false);
|
|
1376
|
+
this.wroteContent = true;
|
|
1377
|
+
this.closeWindow();
|
|
1378
|
+
return;
|
|
1379
|
+
}
|
|
1380
|
+
const content = delta["content"];
|
|
1381
|
+
if (typeof content !== "string" || content === "") {
|
|
1382
|
+
const reason = finishReasonOf(choiceRecord);
|
|
1383
|
+
if (reason !== "") this.drainBuffer();
|
|
1384
|
+
this.send(frame, reason !== "");
|
|
1385
|
+
return;
|
|
1386
|
+
}
|
|
1387
|
+
if (this.buffer === void 0 || this.nativeCallsSeen) {
|
|
1388
|
+
this.windowOpened = true;
|
|
1389
|
+
this.closeWindow();
|
|
1390
|
+
this.res.write(frame);
|
|
1391
|
+
this.wroteContent = true;
|
|
1392
|
+
return;
|
|
1393
|
+
}
|
|
1394
|
+
this.windowOpened = true;
|
|
1395
|
+
const outcome = this.buffer.add(content);
|
|
1396
|
+
if (outcome.calls !== void 0 && outcome.calls.length > 0) {
|
|
1397
|
+
this.send(callsFrame(chunk, choiceRecord, outcome.calls), false);
|
|
1398
|
+
this.wroteContent = true;
|
|
1399
|
+
this.closeWindow();
|
|
1400
|
+
if (outcome.text !== "") this.res.write(contentFrame(chunk, choiceRecord, outcome.text));
|
|
1401
|
+
return;
|
|
1402
|
+
}
|
|
1403
|
+
this.noteResidue(!outcome.prose && outcome.text !== "");
|
|
1404
|
+
if (outcome.text === "") {
|
|
1405
|
+
this.send(contentFrame(chunk, choiceRecord, ""), false);
|
|
1406
|
+
return;
|
|
1407
|
+
}
|
|
1408
|
+
if (outcome.prose) {
|
|
1409
|
+
this.send(contentFrame(chunk, choiceRecord, outcome.text), false);
|
|
1410
|
+
this.wroteContent = true;
|
|
1411
|
+
this.closeWindow();
|
|
1412
|
+
return;
|
|
1413
|
+
}
|
|
1414
|
+
this.send(contentFrame(chunk, choiceRecord, outcome.text), false);
|
|
1415
|
+
}
|
|
1416
|
+
/**
|
|
1417
|
+
* Hand back whatever the buffer is still holding, right now.
|
|
1418
|
+
*
|
|
1419
|
+
* Called before a terminator frame is forwarded. `flush()` ends the buffer's
|
|
1420
|
+
* attempt — if content somehow keeps arriving afterwards it starts a fresh
|
|
1421
|
+
* one, which is the correct reading of a stream that declared itself finished.
|
|
1422
|
+
*/
|
|
1423
|
+
drainBuffer() {
|
|
1424
|
+
const flushed = this.buffer?.flush();
|
|
1425
|
+
if (flushed === void 0 || flushed.text === "") return;
|
|
1426
|
+
this.noteResidue(!flushed.prose);
|
|
1427
|
+
this.emitContent(flushed.text);
|
|
1428
|
+
}
|
|
1429
|
+
/**
|
|
1430
|
+
* Emit text the buffer was still holding when the stream ended.
|
|
1431
|
+
*
|
|
1432
|
+
* It goes through `send` + `closeWindow` rather than straight to the socket on
|
|
1433
|
+
* purpose: a `finish_reason` frame may already be queued, and this text is
|
|
1434
|
+
* content that logically precedes it. Writing directly would put the answer
|
|
1435
|
+
* after its own terminator — a client that stops accumulating at
|
|
1436
|
+
* `finish_reason` would silently lose the tail.
|
|
1437
|
+
*/
|
|
1438
|
+
emitContent(text) {
|
|
1439
|
+
const chunk = { choices: [{
|
|
1440
|
+
index: 0,
|
|
1441
|
+
delta: {},
|
|
1442
|
+
finish_reason: ""
|
|
1443
|
+
}] };
|
|
1444
|
+
const choiceRecord = {
|
|
1445
|
+
index: 0,
|
|
1446
|
+
delta: {},
|
|
1447
|
+
finish_reason: ""
|
|
1448
|
+
};
|
|
1449
|
+
this.windowOpened = true;
|
|
1450
|
+
const frame = contentFrame(chunk, choiceRecord, text);
|
|
1451
|
+
if (text.trim() !== "") {
|
|
1452
|
+
this.send(frame, false);
|
|
1453
|
+
this.wroteContent = true;
|
|
1454
|
+
this.closeWindow();
|
|
1455
|
+
return;
|
|
1456
|
+
}
|
|
1457
|
+
this.send(frame, false);
|
|
1458
|
+
}
|
|
1459
|
+
/** Queue a frame while the window is open, or write it once it has closed. */
|
|
1460
|
+
send(frame, final) {
|
|
1461
|
+
if (this.windowClosed || !this.windowOpened) {
|
|
1462
|
+
this.res.write(frame);
|
|
1463
|
+
return;
|
|
1464
|
+
}
|
|
1465
|
+
const entry = {
|
|
1466
|
+
frame,
|
|
1467
|
+
final
|
|
1468
|
+
};
|
|
1469
|
+
if (!final) {
|
|
1470
|
+
const at = this.held.findIndex((item) => item.final);
|
|
1471
|
+
if (at !== -1) {
|
|
1472
|
+
this.held.splice(at, 0, entry);
|
|
1473
|
+
return;
|
|
1474
|
+
}
|
|
1475
|
+
}
|
|
1476
|
+
this.held.push(entry);
|
|
1477
|
+
}
|
|
1478
|
+
/**
|
|
1479
|
+
* An answer has been delivered: flush everything held, in order, and never
|
|
1480
|
+
* hold again for this response.
|
|
1481
|
+
*/
|
|
1482
|
+
closeWindow() {
|
|
1483
|
+
this.windowClosed = true;
|
|
1484
|
+
const pending = this.held;
|
|
1485
|
+
this.held = [];
|
|
1486
|
+
for (const entry of pending) this.res.write(entry.frame);
|
|
1487
|
+
}
|
|
1488
|
+
};
|
|
1489
|
+
/** The SSE `data:` payload of one frame, or nothing when it carries none. */
|
|
1490
|
+
function dataPayload(frame) {
|
|
1491
|
+
for (const line of frame.split("\n")) {
|
|
1492
|
+
const trimmed = line.trimEnd();
|
|
1493
|
+
if (!trimmed.startsWith("data:")) continue;
|
|
1494
|
+
return trimmed.slice(5).trim();
|
|
1495
|
+
}
|
|
1496
|
+
}
|
|
1497
|
+
/** A frame's `finish_reason`, or '' when it carries none. */
|
|
1498
|
+
function finishReasonOf(choice) {
|
|
1499
|
+
const reason = choice["finish_reason"];
|
|
1500
|
+
return typeof reason === "string" ? reason : "";
|
|
1501
|
+
}
|
|
1502
|
+
/**
|
|
1503
|
+
* One frame carrying recovered calls.
|
|
1504
|
+
*
|
|
1505
|
+
* The shape is OpenAI's: `index` per call, `type: "function"`, arguments as a
|
|
1506
|
+
* JSON STRING. `finish_reason` becomes `tool_calls` — without it a client that
|
|
1507
|
+
* waits for the reason before executing an assembled call would never run it.
|
|
1508
|
+
*/
|
|
1509
|
+
function callsFrame(chunk, choice, calls) {
|
|
1510
|
+
const toolCalls = calls.map((call, index) => ({
|
|
1511
|
+
index,
|
|
1512
|
+
id: call.id,
|
|
1513
|
+
type: "function",
|
|
1514
|
+
function: {
|
|
1515
|
+
name: call.name,
|
|
1516
|
+
arguments: call.arguments
|
|
1517
|
+
}
|
|
1518
|
+
}));
|
|
1519
|
+
const next = {
|
|
1520
|
+
...chunk,
|
|
1521
|
+
choices: [{
|
|
1522
|
+
...choice,
|
|
1523
|
+
delta: { tool_calls: toolCalls },
|
|
1524
|
+
finish_reason: "tool_calls"
|
|
1525
|
+
}]
|
|
1526
|
+
};
|
|
1527
|
+
return `data: ${JSON.stringify(next)}\n\n`;
|
|
1528
|
+
}
|
|
1529
|
+
/** One frame carrying replacement text for a frame whose content was consumed. */
|
|
1530
|
+
function contentFrame(chunk, choice, content) {
|
|
1531
|
+
const delta = {
|
|
1532
|
+
...choice["delta"],
|
|
1533
|
+
content
|
|
1534
|
+
};
|
|
1535
|
+
const next = {
|
|
1536
|
+
...chunk,
|
|
1537
|
+
choices: [{
|
|
1538
|
+
...choice,
|
|
1539
|
+
delta
|
|
1540
|
+
}]
|
|
1541
|
+
};
|
|
1542
|
+
return `data: ${JSON.stringify(next)}\n\n`;
|
|
1543
|
+
}
|
|
1544
|
+
/**
|
|
522
1545
|
* Start the loopback endpoint. Requests must carry the shim's shared secret;
|
|
523
1546
|
* the loopback bind alone is not a trust boundary.
|
|
524
1547
|
*/
|
|
525
1548
|
function createWorkBuddyShim(options) {
|
|
526
1549
|
const { store, client, catalog } = options;
|
|
527
1550
|
const logger = options.logger;
|
|
1551
|
+
const failoverAccount = options.failoverAccount;
|
|
1552
|
+
const prepareAccount = options.prepareAccount;
|
|
1553
|
+
const onAccountFailure = options.onAccountFailure;
|
|
528
1554
|
const SHARED_SECRET = randomBytes(32).toString("base64url");
|
|
529
1555
|
/** Constant-time bearer check; absent or mismatched bearers are rejected. */
|
|
530
1556
|
function bearerOk(req) {
|
|
@@ -593,11 +1619,27 @@ function createWorkBuddyShim(options) {
|
|
|
593
1619
|
else res.end();
|
|
594
1620
|
}
|
|
595
1621
|
}
|
|
1622
|
+
/**
|
|
1623
|
+
* Failure classes worth retrying against another account.
|
|
1624
|
+
*
|
|
1625
|
+
* `client` is deliberately NOT one of them: the upstream rejected the
|
|
1626
|
+
* REQUEST (malformed body, unsupported field), so every account answers the
|
|
1627
|
+
* same 400 and walking the rest of the pool only multiplies the wait before
|
|
1628
|
+
* the user sees an error they must act on anyway. Everything else describes a
|
|
1629
|
+
* PER-ACCOUNT condition — a rate limit, exhausted credits, a dead session, a
|
|
1630
|
+
* gateway that failed this one call — which another account may well survive.
|
|
1631
|
+
*/
|
|
1632
|
+
function isFailoverWorthy(kind) {
|
|
1633
|
+
return kind !== "client";
|
|
1634
|
+
}
|
|
596
1635
|
async function chatCompletions(req, res) {
|
|
597
1636
|
if (!isJsonContentType(req)) {
|
|
598
1637
|
writeOpenAIError(res, 415, "unsupported_media_type", "Content-Type must be application/json");
|
|
599
1638
|
return;
|
|
600
1639
|
}
|
|
1640
|
+
await prepareAccount?.().catch((error) => {
|
|
1641
|
+
logger?.warn("dsh-connect-workbuddy: pool account selection failed", error);
|
|
1642
|
+
});
|
|
601
1643
|
let credential;
|
|
602
1644
|
try {
|
|
603
1645
|
credential = await store.resolve();
|
|
@@ -608,10 +1650,63 @@ function createWorkBuddyShim(options) {
|
|
|
608
1650
|
const raw = (await readBody(req)).toString("utf8");
|
|
609
1651
|
const prepared = prepareChatBody(raw);
|
|
610
1652
|
const controller = new AbortController();
|
|
611
|
-
|
|
612
|
-
|
|
1653
|
+
res.on("close", () => controller.abort());
|
|
1654
|
+
let result = await client.chatStream(credential, prepared, controller.signal);
|
|
1655
|
+
const triedAccountIds = [];
|
|
1656
|
+
/**
|
|
1657
|
+
* Tell the plugin about a failed attempt so it can store the measurement.
|
|
1658
|
+
*
|
|
1659
|
+
* Every failed attempt is reported exactly once, INCLUDING the last one — an
|
|
1660
|
+
* account that failed with nobody left to try it is precisely the one the
|
|
1661
|
+
* next request must avoid, so reporting only the attempts that had a
|
|
1662
|
+
* successor would miss the case that matters most.
|
|
1663
|
+
*
|
|
1664
|
+
* Reported after the loop rather than inside it, so the account that broke
|
|
1665
|
+
* out of the loop is not also reported by the post-loop call. Not awaited: the
|
|
1666
|
+
* user is waiting on the retry, and the write only affects later requests.
|
|
1667
|
+
*/
|
|
1668
|
+
const failures = [];
|
|
1669
|
+
const recordAttempt = (credential, failure) => {
|
|
1670
|
+
if (failure.ok) return;
|
|
1671
|
+
failures.push({
|
|
1672
|
+
accountId: workbuddyAccountId(credential),
|
|
1673
|
+
failure
|
|
1674
|
+
});
|
|
1675
|
+
};
|
|
1676
|
+
const reportFailures = () => {
|
|
1677
|
+
const batch = failures.splice(0, failures.length);
|
|
1678
|
+
if (onAccountFailure === void 0) return;
|
|
1679
|
+
if (controller.signal.aborted) return;
|
|
1680
|
+
for (const entry of batch) {
|
|
1681
|
+
if (entry.failure.ok) continue;
|
|
1682
|
+
try {
|
|
1683
|
+
onAccountFailure(entry.accountId, {
|
|
1684
|
+
status: entry.failure.status,
|
|
1685
|
+
kind: entry.failure.kind,
|
|
1686
|
+
message: entry.failure.message
|
|
1687
|
+
});
|
|
1688
|
+
} catch (error) {
|
|
1689
|
+
logger?.warn("dsh-connect-workbuddy: recording an account failure failed", error);
|
|
1690
|
+
}
|
|
1691
|
+
}
|
|
1692
|
+
};
|
|
1693
|
+
while (!result.ok && isFailoverWorthy(result.kind) && failoverAccount !== void 0) {
|
|
1694
|
+
if (controller.signal.aborted) break;
|
|
1695
|
+
recordAttempt(credential, result);
|
|
1696
|
+
triedAccountIds.push(workbuddyAccountId(credential));
|
|
1697
|
+
const next = await failoverAccount(triedAccountIds).catch(() => void 0);
|
|
1698
|
+
if (next === void 0) break;
|
|
1699
|
+
if (triedAccountIds.includes(workbuddyAccountId(next))) break;
|
|
1700
|
+
logger?.warn(`dsh-connect-workbuddy: retrying chat on another pool account after ${result.kind} (http ${result.status})`);
|
|
1701
|
+
credential = next;
|
|
1702
|
+
result = await client.chatStream(credential, prepared, controller.signal);
|
|
1703
|
+
}
|
|
1704
|
+
if (failures.length === 0 || failures[failures.length - 1]?.accountId !== workbuddyAccountId(credential)) recordAttempt(credential, result);
|
|
1705
|
+
reportFailures();
|
|
613
1706
|
if (!result.ok) {
|
|
614
|
-
|
|
1707
|
+
const attempts = triedAccountIds.length;
|
|
1708
|
+
const note = attempts > 1 ? ` (after trying ${attempts} accounts)` : "";
|
|
1709
|
+
writeOpenAIError(res, KIND_STATUS[result.kind], result.kind, `workbuddy upstream ${result.kind} (http ${result.status})${note}: ${result.message.slice(0, 400)}`);
|
|
615
1710
|
return;
|
|
616
1711
|
}
|
|
617
1712
|
res.writeHead(200, {
|
|
@@ -620,16 +1715,41 @@ function createWorkBuddyShim(options) {
|
|
|
620
1715
|
"Connection": "keep-alive",
|
|
621
1716
|
"X-Accel-Buffering": "no"
|
|
622
1717
|
});
|
|
623
|
-
|
|
624
|
-
const
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
|
|
631
|
-
|
|
632
|
-
|
|
1718
|
+
const declared = declaredTools(prepared);
|
|
1719
|
+
const gate = declared === void 0 ? void 0 : {
|
|
1720
|
+
declaredNames: declared.names,
|
|
1721
|
+
requiredParameters: declared.requiredParameters,
|
|
1722
|
+
...declared.pinnedToolName === void 0 ? {} : { pinnedToolName: declared.pinnedToolName }
|
|
1723
|
+
};
|
|
1724
|
+
const writer = new RecoveryStream(res, gate, logger);
|
|
1725
|
+
await writer.consume(result.response.body);
|
|
1726
|
+
if (gate !== void 0 && !writer.delivered && writer.residueSeen && !controller.signal.aborted) {
|
|
1727
|
+
logger?.warn("dsh-connect-workbuddy: the turn produced DSML markup only; retrying it once");
|
|
1728
|
+
const retryTried = [...triedAccountIds, workbuddyAccountId(credential)];
|
|
1729
|
+
let retryCredential = credential;
|
|
1730
|
+
for (;;) {
|
|
1731
|
+
const attempt = await client.chatStream(retryCredential, prepared, controller.signal);
|
|
1732
|
+
if (attempt.ok) {
|
|
1733
|
+
writer.discard();
|
|
1734
|
+
const retried = new RecoveryStream(res, gate, logger);
|
|
1735
|
+
await retried.consume(attempt.response.body);
|
|
1736
|
+
retried.finish();
|
|
1737
|
+
reportFailures();
|
|
1738
|
+
return;
|
|
1739
|
+
}
|
|
1740
|
+
recordAttempt(retryCredential, attempt);
|
|
1741
|
+
if (!isFailoverWorthy(attempt.kind) || failoverAccount === void 0) break;
|
|
1742
|
+
if (controller.signal.aborted) break;
|
|
1743
|
+
const next = await failoverAccount(retryTried).catch(() => void 0);
|
|
1744
|
+
if (next === void 0) break;
|
|
1745
|
+
if (retryTried.includes(workbuddyAccountId(next))) break;
|
|
1746
|
+
retryTried.push(workbuddyAccountId(next));
|
|
1747
|
+
retryCredential = next;
|
|
1748
|
+
}
|
|
1749
|
+
logger?.warn("dsh-connect-workbuddy: markup-only retry did not start; showing the original text");
|
|
1750
|
+
}
|
|
1751
|
+
writer.finish();
|
|
1752
|
+
reportFailures();
|
|
633
1753
|
}
|
|
634
1754
|
return {
|
|
635
1755
|
ready,
|
|
@@ -872,12 +1992,18 @@ const PROBE_MESSAGE_LIMIT = 300;
|
|
|
872
1992
|
/**
|
|
873
1993
|
* Redact token-shaped content out of upstream text before it is stored or sent.
|
|
874
1994
|
*
|
|
875
|
-
*
|
|
876
|
-
*
|
|
877
|
-
*
|
|
878
|
-
*
|
|
1995
|
+
* Lives here rather than in the route layer because this module is the one that
|
|
1996
|
+
* reads raw failure bodies, so the redaction belongs at the point of capture. A
|
|
1997
|
+
* probe failure body is the one place a raw upstream string from an arbitrary
|
|
1998
|
+
* endpoint enters the plugin's data flow.
|
|
1999
|
+
*
|
|
2000
|
+
* EXPORTED because a failed CHAT request now persists its reason into the pool
|
|
2001
|
+
* store too (`recordAccountFailure` in `src/index.ts`), and that text is a raw
|
|
2002
|
+
* upstream body as well. One implementation, so the two writers cannot drift
|
|
2003
|
+
* into redacting differently — the same "one definition" rule this codebase
|
|
2004
|
+
* applies to `effectiveMembersOf`.
|
|
879
2005
|
*/
|
|
880
|
-
function
|
|
2006
|
+
function redactUpstreamText(text) {
|
|
881
2007
|
return text.replace(/\beyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\b/gu, "[redacted token]").replace(/(\b(?:code|token|refresh_token|access_token)=)[^&\s]+/giu, "$1[redacted]").slice(0, PROBE_MESSAGE_LIMIT);
|
|
882
2008
|
}
|
|
883
2009
|
/**
|
|
@@ -900,7 +2026,7 @@ async function probeModel(input) {
|
|
|
900
2026
|
return {
|
|
901
2027
|
modelId: input.modelId,
|
|
902
2028
|
outcome: "unavailable",
|
|
903
|
-
message:
|
|
2029
|
+
message: redactUpstreamText(error instanceof Error ? error.message : String(error))
|
|
904
2030
|
};
|
|
905
2031
|
}
|
|
906
2032
|
const elapsedMs = Date.now() - startedAt;
|
|
@@ -930,7 +2056,7 @@ async function probeModel(input) {
|
|
|
930
2056
|
outcome,
|
|
931
2057
|
elapsedMs,
|
|
932
2058
|
status: answer.status,
|
|
933
|
-
...body === "" ? {} : { message:
|
|
2059
|
+
...body === "" ? {} : { message: redactUpstreamText(body) },
|
|
934
2060
|
...cooldownOf({
|
|
935
2061
|
outcome,
|
|
936
2062
|
retryAfter: answer.retryAfter,
|
|
@@ -1014,12 +2140,21 @@ async function checkinOne(target, deps) {
|
|
|
1014
2140
|
* exactly the silent spend the plan forbids.
|
|
1015
2141
|
*
|
|
1016
2142
|
* Never throws: a failed probe is a ROW, not an aborted batch.
|
|
2143
|
+
*
|
|
2144
|
+
* `onRow` fires the moment one account's row exists, which is what lets a caller
|
|
2145
|
+
* report progress. The loop is SERIAL by design (one account at a time — hitting
|
|
2146
|
+
* the whole pool at once is the fastest way to trip the upstream's volume limit),
|
|
2147
|
+
* so without this a caller cannot say anything until the SLOWEST member answers:
|
|
2148
|
+
* one stuck account made a working batch look like a dead button. The return
|
|
2149
|
+
* value is unchanged, so every existing caller keeps all rows at the end.
|
|
1017
2150
|
*/
|
|
1018
|
-
async function testAllAccounts(targets, modelId, deps) {
|
|
2151
|
+
async function testAllAccounts(targets, modelId, deps, onRow) {
|
|
1019
2152
|
const rows = [];
|
|
1020
2153
|
for (const [index, target] of targets.entries()) {
|
|
1021
2154
|
if (index > 0) await (deps.wait ?? defaultWait)(400);
|
|
1022
|
-
|
|
2155
|
+
const row = await testOne(target, modelId, deps);
|
|
2156
|
+
rows.push(row);
|
|
2157
|
+
onRow?.(row);
|
|
1023
2158
|
}
|
|
1024
2159
|
return rows;
|
|
1025
2160
|
}
|
|
@@ -1065,6 +2200,11 @@ async function testOne(target, modelId, deps) {
|
|
|
1065
2200
|
*
|
|
1066
2201
|
* `atMs` is stamped here, once for the batch, so the card's "tested N minutes
|
|
1067
2202
|
* ago" reflects when the batch finished rather than drifting per row.
|
|
2203
|
+
*
|
|
2204
|
+
* `message` is carried through as well. `probeModel` already redacted it before
|
|
2205
|
+
* returning, so persisting it cannot leak a token, and without it the pool table
|
|
2206
|
+
* could only repeat one generic sentence for a DNS failure, a gateway 502 and a
|
|
2207
|
+
* cancelled request alike.
|
|
1068
2208
|
*/
|
|
1069
2209
|
function probeUpdatesOf(rows) {
|
|
1070
2210
|
const atMs = Date.now();
|
|
@@ -1074,7 +2214,8 @@ function probeUpdatesOf(rows) {
|
|
|
1074
2214
|
updates[row.accountId] = {
|
|
1075
2215
|
outcome: row.result.outcome,
|
|
1076
2216
|
atMs,
|
|
1077
|
-
...row.result.retryAtMs === void 0 ? {} : { retryAtMs: row.result.retryAtMs }
|
|
2217
|
+
...row.result.retryAtMs === void 0 ? {} : { retryAtMs: row.result.retryAtMs },
|
|
2218
|
+
...row.result.message === void 0 ? {} : { message: row.result.message }
|
|
1078
2219
|
};
|
|
1079
2220
|
}
|
|
1080
2221
|
return updates;
|
|
@@ -1106,18 +2247,37 @@ function messageOf(error) {
|
|
|
1106
2247
|
*
|
|
1107
2248
|
* - A `credential-rejected` account is excluded regardless of any cooldown: its
|
|
1108
2249
|
* token is not accepted, so waiting cannot help and the user must re-auth.
|
|
1109
|
-
* - A limited outcome is checked against its stated (or
|
|
1110
|
-
* an account whose limit has already reset
|
|
1111
|
-
* needing another test
|
|
2250
|
+
* - A limited, unreachable or failed outcome is checked against its stated (or
|
|
2251
|
+
* fallback) cooldown, so an account whose limit has already reset — or whose
|
|
2252
|
+
* gateway blipped once — is back in the pool without needing another test
|
|
2253
|
+
* first.
|
|
1112
2254
|
* - An `ok` measurement, and no measurement at all, are both candidates. The
|
|
1113
2255
|
* latter matters: a freshly discovered account has never been tested and must
|
|
1114
2256
|
* not be invisible.
|
|
1115
2257
|
*
|
|
2258
|
+
* WHY TRANSIENT OUTCOMES GET A COOLDOWN. They are not statements about the
|
|
2259
|
+
* ACCOUNT. A transport failure can be a DNS blip, and — before the shim learned
|
|
2260
|
+
* to skip reporting an aborted request — it was routinely the user closing the
|
|
2261
|
+
* panel mid-flight. A 5xx is the gateway's afternoon, not this account's health.
|
|
2262
|
+
* Excluding them permanently meant one transient event idled a perfectly good
|
|
2263
|
+
* account until the user happened to press "test" again, and with every member
|
|
2264
|
+
* so idled the pool had no candidate at all: failover then had nowhere to go and
|
|
2265
|
+
* the raw upstream error was reported as-is. The cooldown is what makes the pool
|
|
2266
|
+
* RECOVER instead of staying broken.
|
|
2267
|
+
*
|
|
2268
|
+
* `not-found` and `credential-rejected` deliberately get none: one describes a
|
|
2269
|
+
* model the region no longer offers (a catalog fact, not a transient one) and
|
|
2270
|
+
* the other needs the user to sign in again. Neither is fixed by waiting, and
|
|
2271
|
+
* pretending otherwise would re-bill a token the upstream has already refused.
|
|
2272
|
+
*
|
|
1116
2273
|
* Each unusable outcome keeps its OWN value rather than being folded into
|
|
1117
2274
|
* `credential-rejected`. The fold made one row state two contradictory things —
|
|
1118
2275
|
* the name column said "被拒绝" (sign in again) while the probe column, reading
|
|
1119
2276
|
* the same measurement, said "连不上上游——这是网络问题,不是模型问题" — and it
|
|
1120
2277
|
* told a user whose upstream had merely blipped that their sign-in was bad.
|
|
2278
|
+
* (That label has since been made cause-NEUTRAL: it used to assert "not the
|
|
2279
|
+
* model" for a bucket that also holds gateway 5xx and request timeouts, where
|
|
2280
|
+
* the claim is simply unfounded. The recorded `message` now carries the cause.)
|
|
1121
2281
|
* The distinctions are already computed upstream in `outcomeOfFailure`
|
|
1122
2282
|
* (`src/probe.ts:290-299`); this only stops discarding them.
|
|
1123
2283
|
*/
|
|
@@ -1127,14 +2287,22 @@ function exclusionOf(probe, nowMs) {
|
|
|
1127
2287
|
case "ok": return;
|
|
1128
2288
|
case "credential-rejected": return "credential-rejected";
|
|
1129
2289
|
case "not-found": return "not-found";
|
|
1130
|
-
case "unavailable": return "unavailable";
|
|
1131
|
-
case "failed": return "failed";
|
|
2290
|
+
case "unavailable": return retryDue(probe, nowMs) ? void 0 : "unavailable";
|
|
2291
|
+
case "failed": return retryDue(probe, nowMs) ? void 0 : "failed";
|
|
1132
2292
|
case "rate-limited": return retryDue(probe, nowMs) ? void 0 : "rate-limited";
|
|
1133
2293
|
case "out-of-credit": return retryDue(probe, nowMs) ? void 0 : "out-of-credit";
|
|
1134
2294
|
default: return "unusable";
|
|
1135
2295
|
}
|
|
1136
2296
|
}
|
|
1137
|
-
/**
|
|
2297
|
+
/**
|
|
2298
|
+
* Whether an account's cooldown has elapsed (or was never stated).
|
|
2299
|
+
*
|
|
2300
|
+
* The fallback is the ONLY thing a transient outcome relies on: `cooldownOf`
|
|
2301
|
+
* attaches `retryAtMs` for a rate limit or a drained quota, never for a
|
|
2302
|
+
* transport failure, so `atMs + POOL_UNKNOWN_COOLDOWN_MS` is what brings those
|
|
2303
|
+
* back. That is deliberate — the alternative is a permanent exclusion decided by
|
|
2304
|
+
* an event that said nothing about the account.
|
|
2305
|
+
*/
|
|
1138
2306
|
function retryDue(probe, nowMs) {
|
|
1139
2307
|
return nowMs >= (probe.retryAtMs ?? probe.atMs + 18e5);
|
|
1140
2308
|
}
|
|
@@ -1213,15 +2381,6 @@ function effectiveMembersOf(orderedIds, listedIds) {
|
|
|
1213
2381
|
return orderedIds.filter((id) => listedIds.has(id));
|
|
1214
2382
|
}
|
|
1215
2383
|
/**
|
|
1216
|
-
* The account to bill, or undefined when the pool has no usable member.
|
|
1217
|
-
*
|
|
1218
|
-
* A thin selector over {@link rankPool} so the winner is decided in exactly one
|
|
1219
|
-
* place: every caller that needs "who serves now" gets the same answer.
|
|
1220
|
-
*/
|
|
1221
|
-
function pickAccount(members, nowMs) {
|
|
1222
|
-
return rankPool(members, nowMs).find((row) => row.excludedBy === void 0)?.account;
|
|
1223
|
-
}
|
|
1224
|
-
/**
|
|
1225
2384
|
* The credit figure used for ordering.
|
|
1226
2385
|
*
|
|
1227
2386
|
* Absent credits rank as zero. Deliberately NOT "unknown ranks first": an
|
|
@@ -1255,94 +2414,6 @@ function pickFreeModel(catalog) {
|
|
|
1255
2414
|
return best;
|
|
1256
2415
|
}
|
|
1257
2416
|
/**
|
|
1258
|
-
* A latest-wins guard for one region's asynchronous work.
|
|
1259
|
-
*
|
|
1260
|
-
* Exists because `applyRotation` awaits (it fetches credits per member), so two
|
|
1261
|
-
* overlapping calls can interleave: a slow call that read "rotation is ON" can
|
|
1262
|
-
* resume AFTER a newer call cleared the override for "rotation is OFF" and write
|
|
1263
|
-
* the stale account back — leaving the card showing the pool switched off while
|
|
1264
|
-
* a rotated account is still billed.
|
|
1265
|
-
*
|
|
1266
|
-
* `begin()` claims a new generation and returns a probe; the caller must check
|
|
1267
|
-
* it after EVERY await and bail when it reports stale. Extracted as a pure
|
|
1268
|
-
* factory rather than inlined so the rule is unit-testable: an inlined token is
|
|
1269
|
-
* only observable by reproducing a real interleaving, which is slow and easy to
|
|
1270
|
-
* get subtly wrong (a harness that never actually overlaps passes either way).
|
|
1271
|
-
*/
|
|
1272
|
-
function createLatestWins() {
|
|
1273
|
-
const generations = {};
|
|
1274
|
-
return { begin(key) {
|
|
1275
|
-
const generation = (generations[key] ?? 0) + 1;
|
|
1276
|
-
generations[key] = generation;
|
|
1277
|
-
return () => generations[key] !== generation;
|
|
1278
|
-
} };
|
|
1279
|
-
}
|
|
1280
|
-
/**
|
|
1281
|
-
* How often the pool scheduler wakes to check whether a region is due.
|
|
1282
|
-
*
|
|
1283
|
-
* A fixed, coarse heartbeat rather than one timer per region: the heartbeat
|
|
1284
|
-
* itself does no network work (it only reads config and compares timestamps),
|
|
1285
|
-
* so it is free, and it makes re-arming on a config change unnecessary — the
|
|
1286
|
-
* next tick simply reads the new interval.
|
|
1287
|
-
*/
|
|
1288
|
-
const POOL_TICK_MS = 6e4;
|
|
1289
|
-
/**
|
|
1290
|
-
* Whether an automatic test is due for one region.
|
|
1291
|
-
*
|
|
1292
|
-
* `lastRunMs === undefined` means "never armed", and returns FALSE: the clock
|
|
1293
|
-
* starts when the scheduler first sees the region, so enabling the pool does
|
|
1294
|
-
* not immediately spend credits at startup or after a restart. Waiting one
|
|
1295
|
-
* interval is the least surprising reading of "test every N minutes" — the
|
|
1296
|
-
* alternative (fire immediately) would bill the user on every launch.
|
|
1297
|
-
*
|
|
1298
|
-
* Pure and total so the scheduling rule can be pinned without real timers.
|
|
1299
|
-
*/
|
|
1300
|
-
function poolDueAt(input) {
|
|
1301
|
-
if (input.lastRunMs === void 0) return false;
|
|
1302
|
-
const minutes = Math.max(5, Math.round(input.intervalMinutes));
|
|
1303
|
-
return input.nowMs - input.lastRunMs >= minutes * 6e4;
|
|
1304
|
-
}
|
|
1305
|
-
/**
|
|
1306
|
-
* Decide which regions a scheduler tick should run.
|
|
1307
|
-
*
|
|
1308
|
-
* The FIRST sighting of an enabled region ARMS its clock instead of running it.
|
|
1309
|
-
* That split is the whole reason this is a function rather than an `if`:
|
|
1310
|
-
* {@link poolDueAt} answers "has enough time passed", which is FALSE for a
|
|
1311
|
-
* region that has never run — so a scheduler that only armed the clock when it
|
|
1312
|
-
* decided to run would never arm it at all, and the scheduled test would
|
|
1313
|
-
* silently never fire. Returning the clock state explicitly makes that
|
|
1314
|
-
* impossible to get wrong, and testable without real timers.
|
|
1315
|
-
*
|
|
1316
|
-
* A region that is switched off keeps NO clock: enabling it later must wait a
|
|
1317
|
-
* full interval rather than inheriting a stale timestamp and firing at once.
|
|
1318
|
-
*/
|
|
1319
|
-
function duePoolRegions(input) {
|
|
1320
|
-
const due = [];
|
|
1321
|
-
const next = {};
|
|
1322
|
-
for (const region of input.regions) {
|
|
1323
|
-
if (!input.enabledOf(region)) continue;
|
|
1324
|
-
const last = input.state[region];
|
|
1325
|
-
if (last === void 0) {
|
|
1326
|
-
next[region] = input.nowMs;
|
|
1327
|
-
continue;
|
|
1328
|
-
}
|
|
1329
|
-
if (!poolDueAt({
|
|
1330
|
-
lastRunMs: last,
|
|
1331
|
-
intervalMinutes: input.intervalMinutesOf(region),
|
|
1332
|
-
nowMs: input.nowMs
|
|
1333
|
-
})) {
|
|
1334
|
-
next[region] = last;
|
|
1335
|
-
continue;
|
|
1336
|
-
}
|
|
1337
|
-
next[region] = input.nowMs;
|
|
1338
|
-
due.push(region);
|
|
1339
|
-
}
|
|
1340
|
-
return {
|
|
1341
|
-
due,
|
|
1342
|
-
next
|
|
1343
|
-
};
|
|
1344
|
-
}
|
|
1345
|
-
/**
|
|
1346
2417
|
* Resolve the model the pool should test against.
|
|
1347
2418
|
*
|
|
1348
2419
|
* A user-specified id always wins (it is a preference, not a computed default),
|
|
@@ -1406,7 +2477,7 @@ const POOL_STORE_PREFIX = ".workbuddy-pool";
|
|
|
1406
2477
|
function isProbe(value) {
|
|
1407
2478
|
if (typeof value !== "object" || value === null) return false;
|
|
1408
2479
|
const candidate = value;
|
|
1409
|
-
return typeof candidate["outcome"] === "string" && typeof candidate["atMs"] === "number" && (candidate["retryAtMs"] === void 0 || typeof candidate["retryAtMs"] === "number");
|
|
2480
|
+
return typeof candidate["outcome"] === "string" && typeof candidate["atMs"] === "number" && (candidate["retryAtMs"] === void 0 || typeof candidate["retryAtMs"] === "number") && (candidate["message"] === void 0 || typeof candidate["message"] === "string");
|
|
1410
2481
|
}
|
|
1411
2482
|
/**
|
|
1412
2483
|
* One region's path for measured pool facts.
|
|
@@ -1420,9 +2491,11 @@ function workbuddyPoolStorePath(region) {
|
|
|
1420
2491
|
/**
|
|
1421
2492
|
* Read one region's measured facts.
|
|
1422
2493
|
*
|
|
1423
|
-
* Contains NO credentials
|
|
1424
|
-
*
|
|
1425
|
-
*
|
|
2494
|
+
* Contains NO credentials. The one free-text field is `message`, and it is
|
|
2495
|
+
* REDACTED BY THE WRITER before it ever reaches this file (`probe.ts`'s
|
|
2496
|
+
* `redactUpstreamText`), so a token-shaped string cannot survive into storage.
|
|
2497
|
+
* Because of that the file is not written with the credential store's stricter
|
|
2498
|
+
* secrecy requirements, though it still uses the same atomic-write helpers.
|
|
1426
2499
|
*
|
|
1427
2500
|
* Returns an empty map for a missing, unreadable, malformed, or
|
|
1428
2501
|
* version-mismatched file: all four mean "nothing measured yet", which is a
|
|
@@ -1451,7 +2524,8 @@ async function readPoolProbes(region) {
|
|
|
1451
2524
|
probes[accountId] = {
|
|
1452
2525
|
outcome: probe.outcome,
|
|
1453
2526
|
atMs: probe.atMs,
|
|
1454
|
-
...probe.retryAtMs === void 0 ? {} : { retryAtMs: probe.retryAtMs }
|
|
2527
|
+
...probe.retryAtMs === void 0 ? {} : { retryAtMs: probe.retryAtMs },
|
|
2528
|
+
...probe.message === void 0 ? {} : { message: probe.message }
|
|
1455
2529
|
};
|
|
1456
2530
|
}
|
|
1457
2531
|
return probes;
|
|
@@ -1935,7 +3009,6 @@ async function workBuddyWebPool(deps, region) {
|
|
|
1935
3009
|
const byId = new Map(members.map((member) => [member.account.id, member]));
|
|
1936
3010
|
const rankedIds = new Set(ranked.map((row) => row.account.id));
|
|
1937
3011
|
const others = await pool.otherAccounts?.(region) ?? [];
|
|
1938
|
-
const rotatedTo = await pool.rotatedToAccountId?.(region).catch(() => void 0);
|
|
1939
3012
|
const checkedIn = await pool.checkedInToday?.(region).catch(() => ({})) ?? {};
|
|
1940
3013
|
const accounts = [...ranked.map((row) => ({
|
|
1941
3014
|
account: row.account,
|
|
@@ -1957,7 +3030,8 @@ async function workBuddyWebPool(deps, region) {
|
|
|
1957
3030
|
...measured?.probe === void 0 ? {} : { probe: {
|
|
1958
3031
|
outcome: measured.probe.outcome,
|
|
1959
3032
|
atMs: measured.probe.atMs,
|
|
1960
|
-
...measured.probe.retryAtMs === void 0 ? {} : { retryAtMs: measured.probe.retryAtMs }
|
|
3033
|
+
...measured.probe.retryAtMs === void 0 ? {} : { retryAtMs: measured.probe.retryAtMs },
|
|
3034
|
+
...measured.probe.message === void 0 ? {} : { message: measured.probe.message }
|
|
1961
3035
|
} },
|
|
1962
3036
|
...row?.excludedBy === void 0 ? {} : { excludedBy: row.excludedBy },
|
|
1963
3037
|
current: account.id === currentAccountId,
|
|
@@ -1967,14 +3041,12 @@ async function workBuddyWebPool(deps, region) {
|
|
|
1967
3041
|
});
|
|
1968
3042
|
return {
|
|
1969
3043
|
enabled: preferences.enabled,
|
|
1970
|
-
|
|
1971
|
-
autoTestIntervalMinutes: preferences.autoTestIntervalMinutes,
|
|
3044
|
+
checkinSupported: pool.checkinSupported?.(region) === true,
|
|
1972
3045
|
...target.modelId === void 0 ? {} : { targetModelId: target.modelId },
|
|
1973
3046
|
...target.staleModelId === void 0 ? {} : { staleTargetModelId: target.staleModelId },
|
|
1974
3047
|
targetModelSource: target.source,
|
|
1975
3048
|
memberAccountIds: preferences.memberAccountIds,
|
|
1976
3049
|
effectiveMemberAccountIds: effectiveMembersOf(accounts.map((account) => account.accountId), new Set(preferences.memberAccountIds)),
|
|
1977
|
-
...rotatedTo === void 0 ? {} : { rotatedToAccountId: rotatedTo },
|
|
1978
3050
|
accounts,
|
|
1979
3051
|
catalog: deps.displayModels(region).map((model) => ({
|
|
1980
3052
|
id: model.id,
|
|
@@ -2120,10 +3192,6 @@ function registerWorkBuddyStatusRoute(ctx, deps) {
|
|
|
2120
3192
|
error: "account pool unavailable"
|
|
2121
3193
|
});
|
|
2122
3194
|
try {
|
|
2123
|
-
if (!pool.preferences(region).enabled) return json(res, 409, {
|
|
2124
|
-
reason: "pool-disabled",
|
|
2125
|
-
error: "account pool is disabled for this region"
|
|
2126
|
-
});
|
|
2127
3195
|
const savedMembers = pool.preferences(region).memberAccountIds;
|
|
2128
3196
|
if ((await pool.effectiveMemberAccountIds?.(region).catch(() => void 0) ?? savedMembers).length === 0) {
|
|
2129
3197
|
const noneChecked = savedMembers.length === 0;
|
|
@@ -2133,6 +3201,10 @@ function registerWorkBuddyStatusRoute(ctx, deps) {
|
|
|
2133
3201
|
});
|
|
2134
3202
|
}
|
|
2135
3203
|
if (action === "checkin") {
|
|
3204
|
+
if (pool.checkinSupported?.(region) !== true) return json(res, 409, {
|
|
3205
|
+
reason: "checkin-unsupported",
|
|
3206
|
+
error: "this region does not offer a daily check-in"
|
|
3207
|
+
});
|
|
2136
3208
|
if (pool.checkin === void 0) return json(res, 503, {
|
|
2137
3209
|
reason: "pool-unavailable",
|
|
2138
3210
|
error: "pool check-in unavailable"
|
|
@@ -2154,12 +3226,32 @@ function registerWorkBuddyStatusRoute(ctx, deps) {
|
|
|
2154
3226
|
reason: target.source === "stale" ? "target-model-stale" : "no-free-model",
|
|
2155
3227
|
error: target.source === "stale" ? "the saved target model is no longer offered by this region; pick another or switch back to automatic" : "no zero-multiplier model in this region; refresh the catalog or set a target model"
|
|
2156
3228
|
});
|
|
2157
|
-
|
|
2158
|
-
|
|
3229
|
+
res.writeHead(200, {
|
|
3230
|
+
"Content-Type": "application/x-ndjson",
|
|
3231
|
+
"Cache-Control": "no-cache",
|
|
3232
|
+
"Connection": "keep-alive",
|
|
3233
|
+
"X-Accel-Buffering": "no"
|
|
3234
|
+
});
|
|
3235
|
+
const writeLine = (payload) => {
|
|
3236
|
+
res.write(`${JSON.stringify(payload)}\n`);
|
|
3237
|
+
};
|
|
3238
|
+
writeLine({
|
|
2159
3239
|
action: "test",
|
|
2160
|
-
modelId: target.modelId
|
|
2161
|
-
rows
|
|
3240
|
+
modelId: target.modelId
|
|
2162
3241
|
});
|
|
3242
|
+
try {
|
|
3243
|
+
await pool.test(region, target.modelId, (row) => {
|
|
3244
|
+
writeLine({ row });
|
|
3245
|
+
});
|
|
3246
|
+
writeLine({ done: true });
|
|
3247
|
+
} catch (error) {
|
|
3248
|
+
writeLine({
|
|
3249
|
+
reason: "pool-failed",
|
|
3250
|
+
error: safeMessage(error)
|
|
3251
|
+
});
|
|
3252
|
+
}
|
|
3253
|
+
res.end();
|
|
3254
|
+
return;
|
|
2163
3255
|
} catch (error) {
|
|
2164
3256
|
json(res, 500, {
|
|
2165
3257
|
reason: "pool-failed",
|
|
@@ -2331,9 +3423,7 @@ const modelConfig = z.object({
|
|
|
2331
3423
|
* one "Save" press overwrite results the timer had just written.
|
|
2332
3424
|
*/
|
|
2333
3425
|
const poolConfig = z.object({
|
|
2334
|
-
enabled: z.boolean().default(false).description("Whether
|
|
2335
|
-
rotateByCredits: z.boolean().default(false).description("Whether the plugin may switch the billed account by credits (opt-in; disables manual selection)"),
|
|
2336
|
-
autoTestIntervalMinutes: z.number().step(1).min(5).max(1440).default(30).description("Minutes between automatic pool tests (5..1440)"),
|
|
3426
|
+
enabled: z.boolean().default(false).description("Whether this region's account pool is active (opt-in). While on, a failed chat request is retried against the other usable pool members before the error is reported."),
|
|
2337
3427
|
targetModelId: z.string().default("").description("Model id to test; empty means pick a zero-multiplier model from this region's catalog"),
|
|
2338
3428
|
memberAccountIds: z.array(z.string()).default([]).description("Account ids checked into this pool (opt-in; empty means the pool covers nothing)")
|
|
2339
3429
|
});
|
|
@@ -2521,14 +3611,6 @@ function apply(ctx, config) {
|
|
|
2521
3611
|
* immediately instead of waiting out the TTL.
|
|
2522
3612
|
*/
|
|
2523
3613
|
const checkinStateCache = {};
|
|
2524
|
-
/**
|
|
2525
|
-
* Latest-wins guard for `applyRotation`, keyed by region.
|
|
2526
|
-
*
|
|
2527
|
-
* Declared here (near the top of `apply`) rather than beside `applyRotation`
|
|
2528
|
-
* itself: that function is CALLED from the startup path above its definition,
|
|
2529
|
-
* so a `const` declared down there would be in its temporal dead zone.
|
|
2530
|
-
*/
|
|
2531
|
-
const rotationGuard = createLatestWins();
|
|
2532
3614
|
const settingsNs = settingsNamespaceOf(ctx);
|
|
2533
3615
|
const stacks = {};
|
|
2534
3616
|
for (const region of REGION_KEYS) {
|
|
@@ -2546,7 +3628,27 @@ function apply(ctx, config) {
|
|
|
2546
3628
|
store,
|
|
2547
3629
|
client,
|
|
2548
3630
|
catalog,
|
|
2549
|
-
logger: ctx.logger
|
|
3631
|
+
logger: ctx.logger,
|
|
3632
|
+
/**
|
|
3633
|
+
* Pool routing, bound to THIS region.
|
|
3634
|
+
*
|
|
3635
|
+
* The region is captured in the closure rather than passed per call: each
|
|
3636
|
+
* region owns its own store, pool and shim, and routing that consulted the
|
|
3637
|
+
* other region's members would bill an account from a different account
|
|
3638
|
+
* pool than the one in play.
|
|
3639
|
+
*
|
|
3640
|
+
* `prepareAccount` runs before every request (so the ranking decides who
|
|
3641
|
+
* serves), `failoverAccount` runs after a failure (so the ranking decides
|
|
3642
|
+
* who to try next). Both are resolved lazily — they read pool plumbing
|
|
3643
|
+
* defined below — but never called before it exists: the shim only takes
|
|
3644
|
+
* traffic once the provider is registered, which happens after this whole
|
|
3645
|
+
* setup completes.
|
|
3646
|
+
*/
|
|
3647
|
+
prepareAccount: () => applyPoolSelection(region),
|
|
3648
|
+
failoverAccount: (triedAccountIds) => failoverAccountFor(region, triedAccountIds),
|
|
3649
|
+
onAccountFailure: (accountId, failure) => {
|
|
3650
|
+
recordAccountFailure(region, accountId, failure);
|
|
3651
|
+
}
|
|
2550
3652
|
})
|
|
2551
3653
|
};
|
|
2552
3654
|
}
|
|
@@ -2708,11 +3810,11 @@ function apply(ctx, config) {
|
|
|
2708
3810
|
* stacks of accounts, so one region's pool can never rank or bill the
|
|
2709
3811
|
* other's accounts.
|
|
2710
3812
|
*
|
|
2711
|
-
*
|
|
2712
|
-
*
|
|
2713
|
-
*
|
|
2714
|
-
*
|
|
2715
|
-
*
|
|
3813
|
+
* The pool decides who serves a FAILED request (see `failoverAccountFor`),
|
|
3814
|
+
* and it does so without writing `config.accounts[region]`: a retry borrows
|
|
3815
|
+
* another member's credential for that one request, leaving the record of
|
|
3816
|
+
* what the user picked — and the account the next request starts from —
|
|
3817
|
+
* untouched.
|
|
2716
3818
|
*/
|
|
2717
3819
|
pool: {
|
|
2718
3820
|
preferences: (region) => poolPreferencesOf(current(), region),
|
|
@@ -2773,17 +3875,18 @@ function apply(ctx, config) {
|
|
|
2773
3875
|
}));
|
|
2774
3876
|
return Object.fromEntries(entries.filter((entry) => entry !== void 0));
|
|
2775
3877
|
},
|
|
2776
|
-
/**
|
|
2777
|
-
* The account rotation last moved to, read from the store's runtime
|
|
2778
|
-
* override. Absent means rotation is not currently redirecting billing.
|
|
2779
|
-
*/
|
|
2780
|
-
async rotatedToAccountId(region) {
|
|
2781
|
-
return stacks[region].store.rotatedAccount();
|
|
2782
|
-
},
|
|
2783
3878
|
async currentAccountId(region) {
|
|
3879
|
+
await applyPoolSelection(region).catch(() => void 0);
|
|
2784
3880
|
const credential = await stacks[region].store.current().catch(() => void 0);
|
|
2785
|
-
|
|
3881
|
+
if (credential === void 0) return void 0;
|
|
3882
|
+
if (await servingAccountIsExcluded(region).catch(() => false)) return void 0;
|
|
3883
|
+
return workbuddyAccountId(credential);
|
|
2786
3884
|
},
|
|
3885
|
+
/**
|
|
3886
|
+
* Only the CN app rewards a daily check-in; the international region has
|
|
3887
|
+
* no equivalent here, so its card must not offer the action.
|
|
3888
|
+
*/
|
|
3889
|
+
checkinSupported: (region) => region === "cn",
|
|
2787
3890
|
checkin: async (region) => {
|
|
2788
3891
|
const rows = await checkinAllAccounts(await poolTargets(region), poolRunnerDeps(region));
|
|
2789
3892
|
for (const row of rows) {
|
|
@@ -2793,10 +3896,9 @@ function apply(ctx, config) {
|
|
|
2793
3896
|
}
|
|
2794
3897
|
return rows;
|
|
2795
3898
|
},
|
|
2796
|
-
test: async (region, modelId) => {
|
|
2797
|
-
const rows = await testAllAccounts(await poolTargets(region), modelId, poolRunnerDeps(region));
|
|
3899
|
+
test: async (region, modelId, onRow) => {
|
|
3900
|
+
const rows = await testAllAccounts(await poolTargets(region), modelId, poolRunnerDeps(region), onRow);
|
|
2798
3901
|
await writePoolProbes(region, probeUpdatesOf(rows));
|
|
2799
|
-
await applyRotation(region);
|
|
2800
3902
|
return rows;
|
|
2801
3903
|
},
|
|
2802
3904
|
catalog: (region) => displayModels(current(), region)
|
|
@@ -2820,33 +3922,174 @@ function apply(ctx, config) {
|
|
|
2820
3922
|
}
|
|
2821
3923
|
};
|
|
2822
3924
|
/**
|
|
2823
|
-
*
|
|
3925
|
+
* The next account to try for a request whose current account just failed,
|
|
3926
|
+
* or `undefined` when this region's pool has nobody left.
|
|
2824
3927
|
*
|
|
2825
|
-
*
|
|
2826
|
-
*
|
|
2827
|
-
*
|
|
2828
|
-
*
|
|
2829
|
-
*
|
|
3928
|
+
* Reads the pool's EXISTING measurements instead of probing: a request that
|
|
3929
|
+
* just failed must not spend further requests deciding who serves next (and a
|
|
3930
|
+
* probe of the target model would itself be billed). Candidates are this
|
|
3931
|
+
* region's checked-in members in rank order, minus everyone this request has
|
|
3932
|
+
* already tried, minus anyone a measurement says cannot serve right now — a
|
|
3933
|
+
* limited account whose stated cooldown has not elapsed, or one whose
|
|
3934
|
+
* credential the upstream rejected.
|
|
2830
3935
|
*
|
|
2831
|
-
*
|
|
3936
|
+
* The pool switch gates this: off means the user's account serves every
|
|
3937
|
+
* request and a failure is reported as-is, exactly as before the pool existed.
|
|
3938
|
+
*
|
|
3939
|
+
* Reads the candidate through `credentialFor()`, never `resolve()`: the saved
|
|
3940
|
+
* selection and the store's runtime state stay untouched, so this is
|
|
3941
|
+
* per-request borrowing rather than a silent change of who pays.
|
|
3942
|
+
*
|
|
3943
|
+
* Uses `localPoolMembers`, NOT `poolMembersOf`. This runs on the failure path
|
|
3944
|
+
* of a request the user is already waiting on, and `poolMembersOf` fetches
|
|
3945
|
+
* credits per member — so the expensive version turned one failed request into
|
|
3946
|
+
* N extra upstream calls before the retry even left. The doc above always said
|
|
3947
|
+
* "must not spend further requests"; the code used to contradict it. Credits
|
|
3948
|
+
* still steer the ranking when a recent snapshot exists, because
|
|
3949
|
+
* `localPoolMembers` reads the one the card's poll already paid for.
|
|
2832
3950
|
*/
|
|
2833
|
-
const
|
|
2834
|
-
|
|
2835
|
-
const
|
|
2836
|
-
const
|
|
2837
|
-
|
|
2838
|
-
|
|
2839
|
-
|
|
3951
|
+
const failoverAccountFor = async (region, triedAccountIds) => {
|
|
3952
|
+
if (!poolPreferencesOf(current(), region).enabled) return void 0;
|
|
3953
|
+
const tried = new Set(triedAccountIds);
|
|
3954
|
+
for (const row of rankPool(await localPoolMembers(region), Date.now())) {
|
|
3955
|
+
if (row.excludedBy !== void 0) continue;
|
|
3956
|
+
if (tried.has(row.account.id)) continue;
|
|
3957
|
+
const credential = await stacks[region].store.credentialFor(row.account.id);
|
|
3958
|
+
if (credential !== void 0) return credential;
|
|
2840
3959
|
}
|
|
2841
|
-
|
|
2842
|
-
|
|
2843
|
-
|
|
2844
|
-
|
|
2845
|
-
|
|
3960
|
+
};
|
|
3961
|
+
/**
|
|
3962
|
+
* The last CREDITS reading per region, so a request can rank by balance
|
|
3963
|
+
* without fetching one.
|
|
3964
|
+
*
|
|
3965
|
+
* Credits are the ranking's second key, and they cost an upstream call per
|
|
3966
|
+
* member. A chat request must never pay that: it happens on the hot path, and
|
|
3967
|
+
* `fetchCredits` per member would turn one page of conversation into N extra
|
|
3968
|
+
* requests. So the request path uses whatever the pool last read (the card's
|
|
3969
|
+
* 60-second poll, a batch test, or a save) and simply skips the key when the
|
|
3970
|
+
* snapshot is too old to speak for today's balances.
|
|
3971
|
+
*/
|
|
3972
|
+
const creditsSnapshot = {};
|
|
3973
|
+
/** How long a credits reading may steer routing. */
|
|
3974
|
+
const CREDITS_SNAPSHOT_MS = 6e5;
|
|
3975
|
+
/** When each region's snapshot was taken, for the freshness check above. */
|
|
3976
|
+
const creditsSnapshotAt = {};
|
|
3977
|
+
/**
|
|
3978
|
+
* One region's pool members from LOCAL sources only: the credential store and
|
|
3979
|
+
* the probe store, plus the credits snapshot when it is fresh.
|
|
3980
|
+
*
|
|
3981
|
+
* The request path calls this on every chat completion, so it must not touch
|
|
3982
|
+
* the network. `poolMembersOf` is the honest-but-expensive version (it fetches
|
|
3983
|
+
* credits per member) and stays on the card's route, which is where those
|
|
3984
|
+
* readings come from in the first place.
|
|
3985
|
+
*/
|
|
3986
|
+
async function localPoolMembers(region) {
|
|
3987
|
+
const accounts = await poolMemberAccounts(region);
|
|
3988
|
+
const probes = await readPoolProbes(region);
|
|
3989
|
+
const credits = Date.now() - (creditsSnapshotAt[region] ?? 0) <= CREDITS_SNAPSHOT_MS ? creditsSnapshot[region] : void 0;
|
|
3990
|
+
return accounts.map((account) => {
|
|
3991
|
+
const reading = credits?.get(account.id);
|
|
3992
|
+
return {
|
|
3993
|
+
account: {
|
|
3994
|
+
id: account.id,
|
|
3995
|
+
accountName: account.accountName
|
|
3996
|
+
},
|
|
3997
|
+
...reading === void 0 ? {} : { credits: reading },
|
|
3998
|
+
...probes[account.id] === void 0 ? {} : { probe: probes[account.id] },
|
|
3999
|
+
tokenExpiresAtMs: account.tokenExpiresAtMs
|
|
4000
|
+
};
|
|
4001
|
+
});
|
|
4002
|
+
}
|
|
4003
|
+
/**
|
|
4004
|
+
* Record a failure a LIVE request hit, so the NEXT request starts from a
|
|
4005
|
+
* usable account.
|
|
4006
|
+
*
|
|
4007
|
+
* Without this the ranking only knew what the manual batch test had measured:
|
|
4008
|
+
* an account that had just answered 429 carried no measurement, so
|
|
4009
|
+
* `exclusionOf` returned "candidate" and the ranking kept picking it — every
|
|
4010
|
+
* request paid one failed round trip before failing over. Keeping traffic off
|
|
4011
|
+
* an account the upstream just refused is the pool's whole point
|
|
4012
|
+
* (`account-pool.ts` says exactly that about `rate-limited`), and it only
|
|
4013
|
+
* works if live failures count as measurements.
|
|
4014
|
+
*
|
|
4015
|
+
* Deliberately NOT recorded for `client` (HTTP 400): the upstream rejected the
|
|
4016
|
+
* REQUEST, which says nothing about the account, and recording it would
|
|
4017
|
+
* sideline a good account over a bad body.
|
|
4018
|
+
*
|
|
4019
|
+
* The reset time comes from the upstream's own words — the 429 body carries
|
|
4020
|
+
* 「将在 … 重置」 — which `cooldownOf` parses. When it states no time, the store's
|
|
4021
|
+
* own short window applies rather than a guess at a long one.
|
|
4022
|
+
*
|
|
4023
|
+
* The upstream's text is persisted alongside the outcome so the card can name
|
|
4024
|
+
* the reason. It is REDACTED first because this message is a raw upstream body:
|
|
4025
|
+
* without that, a failure containing a token-shaped string would write it into
|
|
4026
|
+
* a file on disk.
|
|
4027
|
+
*/
|
|
4028
|
+
const recordAccountFailure = async (region, accountId, failure) => {
|
|
4029
|
+
if (!poolPreferencesOf(current(), region).enabled) return;
|
|
4030
|
+
if (failure.kind === "client") return;
|
|
4031
|
+
const members = poolPreferencesOf(current(), region).memberAccountIds;
|
|
4032
|
+
if (members.length > 0 && !members.includes(accountId)) return;
|
|
4033
|
+
const outcome = outcomeOfFailure(failure.status, failure.message);
|
|
4034
|
+
const { retryAtMs } = cooldownOf({
|
|
4035
|
+
outcome,
|
|
4036
|
+
retryAfter: null,
|
|
4037
|
+
nowMs: Date.now(),
|
|
4038
|
+
body: failure.message
|
|
4039
|
+
});
|
|
4040
|
+
const message = failure.message === "" ? "" : redactUpstreamText(failure.message);
|
|
4041
|
+
await writePoolProbes(region, { [accountId]: {
|
|
4042
|
+
outcome,
|
|
4043
|
+
atMs: Date.now(),
|
|
4044
|
+
...retryAtMs === void 0 ? {} : { retryAtMs },
|
|
4045
|
+
...message === "" ? {} : { message }
|
|
4046
|
+
} });
|
|
4047
|
+
};
|
|
4048
|
+
/**
|
|
4049
|
+
* Point this region's store at whoever the pool's ranking says should serve.
|
|
4050
|
+
*
|
|
4051
|
+
* Called once per chat request. With the pool ON the ranking decides the
|
|
4052
|
+
* serving account — the whole point of the switch, and the reason the card can
|
|
4053
|
+
* show a "current account" that is not simply the saved selection. With the
|
|
4054
|
+
* pool OFF the override is CLEARED, which restores the user's own choice
|
|
4055
|
+
* immediately: leaving a stale override in place would keep billing under a
|
|
4056
|
+
* switch that reads as off.
|
|
4057
|
+
*
|
|
4058
|
+
* The override is runtime-only and never written to settings, so the user's
|
|
4059
|
+
* recorded choice survives untouched either way — turning the pool off gives
|
|
4060
|
+
* it back verbatim.
|
|
4061
|
+
*
|
|
4062
|
+
* Best-effort: a failure here leaves the previous state in effect, and the
|
|
4063
|
+
* request still goes out (to the user's account when the pool is off, which is
|
|
4064
|
+
* the safe default).
|
|
4065
|
+
*/
|
|
4066
|
+
const applyPoolSelection = async (region) => {
|
|
4067
|
+
const store = stacks[region].store;
|
|
4068
|
+
if (!poolPreferencesOf(current(), region).enabled) {
|
|
2846
4069
|
store.setRotatedAccount(void 0);
|
|
2847
4070
|
return;
|
|
2848
4071
|
}
|
|
2849
|
-
|
|
4072
|
+
const winner = rankPool(await localPoolMembers(region), Date.now()).find((row) => row.excludedBy === void 0);
|
|
4073
|
+
store.setRotatedAccount(winner?.account.id);
|
|
4074
|
+
};
|
|
4075
|
+
/**
|
|
4076
|
+
* Whether the account the store would bill is one the pool has measured as
|
|
4077
|
+
* unusable.
|
|
4078
|
+
*
|
|
4079
|
+
* The two questions differ exactly when the pool runs out of usable members: a
|
|
4080
|
+
* REQUEST still has to go somewhere (the saved selection), but the CARD must
|
|
4081
|
+
* not label that account "in use" while the same table says it is rate-limited.
|
|
4082
|
+
* One screen contradicting itself is how a user learns to trust neither half —
|
|
4083
|
+
* and "the pool moved to an available account" is unverifiable if the display
|
|
4084
|
+
* keeps naming an account the table calls excluded.
|
|
4085
|
+
*/
|
|
4086
|
+
const servingAccountIsExcluded = async (region) => {
|
|
4087
|
+
if (!poolPreferencesOf(current(), region).enabled) return false;
|
|
4088
|
+
const credential = await stacks[region].store.current().catch(() => void 0);
|
|
4089
|
+
if (credential === void 0) return false;
|
|
4090
|
+
const servingId = workbuddyAccountId(credential);
|
|
4091
|
+
const row = rankPool(await localPoolMembers(region), Date.now()).find((candidate) => candidate.account.id === servingId);
|
|
4092
|
+
return row !== void 0 && row.excludedBy !== void 0;
|
|
2850
4093
|
};
|
|
2851
4094
|
/**
|
|
2852
4095
|
* One region's POOL members, each with its credits and last measurement.
|
|
@@ -2864,7 +4107,7 @@ function apply(ctx, config) {
|
|
|
2864
4107
|
const store = stacks[region].store;
|
|
2865
4108
|
const accounts = await poolMemberAccounts(region);
|
|
2866
4109
|
const probes = await readPoolProbes(region);
|
|
2867
|
-
|
|
4110
|
+
const rows = await Promise.all(accounts.map(async (account) => {
|
|
2868
4111
|
const credential = await store.credentialFor(account.id).catch(() => void 0);
|
|
2869
4112
|
const credits = credential === void 0 ? void 0 : await client.fetchCredits(credential).catch(() => void 0);
|
|
2870
4113
|
return {
|
|
@@ -2881,14 +4124,15 @@ function apply(ctx, config) {
|
|
|
2881
4124
|
tokenExpiresAtMs: account.tokenExpiresAtMs
|
|
2882
4125
|
};
|
|
2883
4126
|
}));
|
|
4127
|
+
creditsSnapshot[region] = new Map(rows.filter((row) => row.credits !== void 0).map((row) => [row.account.id, row.credits]));
|
|
4128
|
+
creditsSnapshotAt[region] = Date.now();
|
|
4129
|
+
return rows;
|
|
2884
4130
|
}
|
|
2885
4131
|
/** One region's pool preferences, with the schema's defaults applied. */
|
|
2886
4132
|
const poolPreferencesOf = (config, region) => {
|
|
2887
4133
|
const pool = regionStateOf(config, region).pool;
|
|
2888
4134
|
return {
|
|
2889
4135
|
enabled: pool?.enabled === true,
|
|
2890
|
-
rotateByCredits: pool?.rotateByCredits === true,
|
|
2891
|
-
autoTestIntervalMinutes: pool?.autoTestIntervalMinutes ?? 30,
|
|
2892
4136
|
targetModelId: pool?.targetModelId ?? "",
|
|
2893
4137
|
memberAccountIds: pool?.memberAccountIds ?? []
|
|
2894
4138
|
};
|
|
@@ -2951,10 +4195,8 @@ function apply(ctx, config) {
|
|
|
2951
4195
|
});
|
|
2952
4196
|
ctx.on("loader/volatile-update", () => {
|
|
2953
4197
|
applySelection(current());
|
|
2954
|
-
for (const region of REGION_KEYS) applyRotation(region).catch(() => {});
|
|
2955
4198
|
});
|
|
2956
4199
|
applySelection(config);
|
|
2957
|
-
for (const region of REGION_KEYS) applyRotation(region).catch(() => {});
|
|
2958
4200
|
(async () => {
|
|
2959
4201
|
try {
|
|
2960
4202
|
const region = await legacyAttributionRegion(current(), async (candidate) => stacks[candidate].store.accounts());
|
|
@@ -2964,85 +4206,6 @@ function apply(ctx, config) {
|
|
|
2964
4206
|
} catch {}
|
|
2965
4207
|
})();
|
|
2966
4208
|
let stopped = false;
|
|
2967
|
-
/**
|
|
2968
|
-
* When each region's automatic test last ran, in epoch ms.
|
|
2969
|
-
*
|
|
2970
|
-
* In-memory only, and deliberately NOT persisted: it exists to space out
|
|
2971
|
-
* runs within one session. Persisting it would make a restart skip a test the
|
|
2972
|
-
* user was owed, and losing it only costs one interval.
|
|
2973
|
-
*/
|
|
2974
|
-
const poolLastRunMs = {};
|
|
2975
|
-
/**
|
|
2976
|
-
* The pool scheduler: ONE heartbeat for both regions.
|
|
2977
|
-
*
|
|
2978
|
-
* A single timer rather than one per region, because the tick does no network
|
|
2979
|
-
* work of its own — it reads the config, compares timestamps, and only then
|
|
2980
|
-
* runs a batch. That also means a settings change needs no re-arming: the next
|
|
2981
|
-
* tick reads whatever interval is now saved.
|
|
2982
|
-
*
|
|
2983
|
-
* This is what makes the "自动测试间隔" setting real. Before it existed the
|
|
2984
|
-
* interval was stored and never consumed, so the card promised a behaviour the
|
|
2985
|
-
* plugin did not have.
|
|
2986
|
-
*/
|
|
2987
|
-
ctx.effect(() => {
|
|
2988
|
-
const timer = setInterval(() => {
|
|
2989
|
-
(async () => {
|
|
2990
|
-
if (stopped) return;
|
|
2991
|
-
const schedule = duePoolRegions({
|
|
2992
|
-
state: poolLastRunMs,
|
|
2993
|
-
regions: REGION_KEYS,
|
|
2994
|
-
enabledOf: (region) => poolPreferencesOf(current(), region).enabled && regionEnabled(current(), region),
|
|
2995
|
-
intervalMinutesOf: (region) => poolPreferencesOf(current(), region).autoTestIntervalMinutes,
|
|
2996
|
-
nowMs: Date.now()
|
|
2997
|
-
});
|
|
2998
|
-
for (const region of REGION_KEYS) {
|
|
2999
|
-
const next = schedule.next[region];
|
|
3000
|
-
if (next === void 0) delete poolLastRunMs[region];
|
|
3001
|
-
else poolLastRunMs[region] = next;
|
|
3002
|
-
}
|
|
3003
|
-
for (const region of schedule.due) await runScheduledPoolTest(region).catch(() => {});
|
|
3004
|
-
})();
|
|
3005
|
-
}, POOL_TICK_MS);
|
|
3006
|
-
return () => {
|
|
3007
|
-
clearInterval(timer);
|
|
3008
|
-
};
|
|
3009
|
-
}, "dsh-connect-workbuddy: account pool scheduler");
|
|
3010
|
-
/**
|
|
3011
|
-
* One scheduled pass for a region: test the pool's members, then re-rank.
|
|
3012
|
-
*
|
|
3013
|
-
* Never bills a paid model on a timer: the target is the user's saved model
|
|
3014
|
-
* when it resolves, and the region's FREE model when the saved one has left
|
|
3015
|
-
* the catalog. An unresolvable target with no free model available stops the
|
|
3016
|
-
* pass — the free-target rule is what stops a timer from spending credits.
|
|
3017
|
-
*
|
|
3018
|
-
* The free FALLBACK for a stale saved model is D3's fix, not a convenience.
|
|
3019
|
-
* The scheduler's job is to keep MEASUREMENTS fresh, and a measurement is
|
|
3020
|
-
* what keeps rotation off a dead account. It used to return early whenever
|
|
3021
|
-
* `resolveTargetModel` produced no id — which is exactly what a saved model
|
|
3022
|
-
* that left the catalog produces — so a stale *display preference* froze the
|
|
3023
|
-
* measurement loop for the whole region. The two exclusions then behaved
|
|
3024
|
-
* asymmetrically: a `rate-limited` member clears by the clock alone
|
|
3025
|
-
* (`exclusionOf` consults `retryAtMs`), while an `unavailable` member can
|
|
3026
|
-
* only be cleared by a NEW probe — and no probe was ever written. The member
|
|
3027
|
-
* hung in the pool's "excluded" state indefinitely, until the user noticed
|
|
3028
|
-
* the stale model and fixed it by hand.
|
|
3029
|
-
*
|
|
3030
|
-
* The manual batch route keeps refusing a stale target (`reason:
|
|
3031
|
-
* 'target-model-stale'`): there the user asked to test THAT model, so
|
|
3032
|
-
* substituting another one would answer a question they did not ask. Here
|
|
3033
|
-
* nobody asked for anything; the pass exists to refresh the pool's own
|
|
3034
|
-
* picture, so it must not be starved by a configuration error it can work
|
|
3035
|
-
* around without spending anything.
|
|
3036
|
-
*/
|
|
3037
|
-
async function runScheduledPoolTest(region) {
|
|
3038
|
-
const targets = await poolTargets(region);
|
|
3039
|
-
if (targets.length === 0) return;
|
|
3040
|
-
const catalog = displayModels(current(), region);
|
|
3041
|
-
const modelId = resolveTargetModel(catalog, poolPreferencesOf(current(), region).targetModelId).modelId ?? pickFreeModel(catalog)?.id;
|
|
3042
|
-
if (modelId === void 0) return;
|
|
3043
|
-
await writePoolProbes(region, probeUpdatesOf(await testAllAccounts(targets, modelId, poolRunnerDeps(region))));
|
|
3044
|
-
await applyRotation(region);
|
|
3045
|
-
}
|
|
3046
4209
|
ctx.effect(() => () => {
|
|
3047
4210
|
stopped = true;
|
|
3048
4211
|
for (const region of REGION_KEYS) stacks[region].shim.close();
|