@tokenbuddy/tokenbuddy 1.0.26 → 1.0.27
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/tb-clawtip-proof.js +2 -0
- package/dist/src/clawtip-bootstrap.d.ts +1 -0
- package/dist/src/clawtip-bootstrap.d.ts.map +1 -1
- package/dist/src/clawtip-bootstrap.js +1 -0
- package/dist/src/clawtip-bootstrap.js.map +1 -1
- package/dist/src/cli.d.ts +1 -0
- package/dist/src/cli.d.ts.map +1 -1
- package/dist/src/cli.js +172 -51
- package/dist/src/cli.js.map +1 -1
- package/dist/src/daemon.d.ts +6 -0
- package/dist/src/daemon.d.ts.map +1 -1
- package/dist/src/daemon.js +562 -292
- package/dist/src/daemon.js.map +1 -1
- package/dist/src/init-clawtip-activation.d.ts +5 -0
- package/dist/src/init-clawtip-activation.d.ts.map +1 -1
- package/dist/src/init-clawtip-activation.js +61 -1
- package/dist/src/init-clawtip-activation.js.map +1 -1
- package/dist/src/package-update.d.ts +60 -0
- package/dist/src/package-update.d.ts.map +1 -0
- package/dist/src/package-update.js +220 -0
- package/dist/src/package-update.js.map +1 -0
- package/dist/src/registry-trust.d.ts +7 -0
- package/dist/src/registry-trust.d.ts.map +1 -0
- package/dist/src/registry-trust.js +37 -0
- package/dist/src/registry-trust.js.map +1 -0
- package/dist/src/route-failover.d.ts +2 -2
- package/dist/src/route-failover.d.ts.map +1 -1
- package/dist/src/route-failover.js +11 -0
- package/dist/src/route-failover.js.map +1 -1
- package/dist/src/seller-catalog.d.ts +20 -0
- package/dist/src/seller-catalog.d.ts.map +1 -1
- package/dist/src/seller-catalog.js +41 -4
- package/dist/src/seller-catalog.js.map +1 -1
- package/dist/src/seller-concurrency-limiter.d.ts +36 -0
- package/dist/src/seller-concurrency-limiter.d.ts.map +1 -0
- package/dist/src/seller-concurrency-limiter.js +126 -0
- package/dist/src/seller-concurrency-limiter.js.map +1 -0
- package/dist/src/seller-pool.d.ts +7 -1
- package/dist/src/seller-pool.d.ts.map +1 -1
- package/dist/src/seller-pool.js +18 -0
- package/dist/src/seller-pool.js.map +1 -1
- package/dist/src/seller-route-planner.d.ts +21 -0
- package/dist/src/seller-route-planner.d.ts.map +1 -1
- package/dist/src/seller-route-planner.js +98 -20
- package/dist/src/seller-route-planner.js.map +1 -1
- package/dist/src/tb-clawtip-proof.d.ts +3 -0
- package/dist/src/tb-clawtip-proof.d.ts.map +1 -0
- package/dist/src/tb-clawtip-proof.js +24 -0
- package/dist/src/tb-clawtip-proof.js.map +1 -0
- package/dist/src/tb-proxyd.js +45 -3
- package/dist/src/tb-proxyd.js.map +1 -1
- package/package.json +3 -2
- package/src/clawtip-bootstrap.ts +1 -0
- package/src/cli.ts +200 -47
- package/src/daemon.ts +347 -50
- package/src/init-clawtip-activation.ts +77 -1
- package/src/package-update.ts +313 -0
- package/src/registry-trust.ts +51 -0
- package/src/route-failover.ts +14 -2
- package/src/seller-catalog.ts +67 -4
- package/src/seller-concurrency-limiter.ts +161 -0
- package/src/seller-pool.ts +20 -0
- package/src/seller-route-planner.ts +142 -20
- package/src/tb-clawtip-proof.ts +28 -0
- package/src/tb-proxyd.ts +48 -3
- package/static/ui/assets/index-Bzbrp7Qe.css +1 -0
- package/static/ui/assets/index-UAfOhbwC.js +236 -0
- package/static/ui/assets/index-UAfOhbwC.js.map +1 -0
- package/static/ui/index.html +2 -2
- package/tests/cli-routing.test.ts +37 -4
- package/tests/control-plane-ui-endpoints.test.ts +7 -7
- package/tests/daemon-trusted-registry-cache.test.ts +132 -0
- package/tests/e2e.test.ts +14 -1
- package/tests/package-update.test.ts +132 -0
- package/tests/registry-trust.test.ts +28 -0
- package/tests/route-failover.test.ts +13 -0
- package/tests/seller-catalog-413.test.ts +60 -1
- package/tests/seller-concurrency-limiter.test.ts +83 -0
- package/tests/seller-pool.test.ts +23 -0
- package/tests/seller-route-planner.test.ts +78 -0
- package/tests/tokenbuddy.test.ts +316 -34
- package/static/ui/assets/index-1uuyCCzj.css +0 -1
- package/static/ui/assets/index-cm_EgQZ-.js +0 -236
- package/static/ui/assets/index-cm_EgQZ-.js.map +0 -1
package/src/daemon.ts
CHANGED
|
@@ -7,7 +7,7 @@ import { AddressInfo } from "net";
|
|
|
7
7
|
import { ErrorCode } from "@tokenbuddy/contracts";
|
|
8
8
|
import { createModuleLogger } from "@tokenbuddy/logging";
|
|
9
9
|
import { BuyerStore, type PaymentConfig } from "./buyer-store.js";
|
|
10
|
-
import { fetchClawtipBootstrap } from "./clawtip-bootstrap.js";
|
|
10
|
+
import { DEFAULT_CLAWTIP_BOOTSTRAP_URL, fetchClawtipBootstrap } from "./clawtip-bootstrap.js";
|
|
11
11
|
import type { ClawtipBootstrapResponse } from "./clawtip-bootstrap.js";
|
|
12
12
|
import {
|
|
13
13
|
inspectOpenClawWalletConfig,
|
|
@@ -29,7 +29,7 @@ import {
|
|
|
29
29
|
} from "./provider-install.js";
|
|
30
30
|
import {
|
|
31
31
|
discoverSellerBackedModels,
|
|
32
|
-
|
|
32
|
+
fetchSellerRegistryWithTrust,
|
|
33
33
|
manifestModelIds,
|
|
34
34
|
manifestPaymentMethods,
|
|
35
35
|
manifestProtocols,
|
|
@@ -39,13 +39,16 @@ import {
|
|
|
39
39
|
type RegistrySeller,
|
|
40
40
|
type SellerManifest,
|
|
41
41
|
type SellerRegistryDocument,
|
|
42
|
+
type SellerRegistryTrustMetadata,
|
|
42
43
|
} from "./seller-catalog.js";
|
|
44
|
+
import { shouldVerifyRegistry, verifyTrustedRegistrySignature } from "./registry-trust.js";
|
|
43
45
|
import { ModelIndex } from "./model-index.js";
|
|
44
46
|
import { PrewarmCache, prewarmKey } from "./prewarm-cache.js";
|
|
45
47
|
import { CreditTracker } from "./credit-tracker.js";
|
|
46
48
|
import { SellerPool, type FailureKind } from "./seller-pool.js";
|
|
47
49
|
import { RouteFailover, type FailoverDecision, type RouteCandidate } from "./route-failover.js";
|
|
48
50
|
import { PrewarmScheduler, type PrewarmReason, type SellerProber } from "./prewarm-scheduler.js";
|
|
51
|
+
import { SellerConcurrencyLimiter, type SellerConcurrencyLimiterOptions } from "./seller-concurrency-limiter.js";
|
|
49
52
|
import type { PoolEntry } from "./seller-pool.js";
|
|
50
53
|
import { planSellerRouteSet } from "./seller-route-planner.js";
|
|
51
54
|
import type { SellerRoutePlan } from "./seller-route-planner.js";
|
|
@@ -67,9 +70,15 @@ import {
|
|
|
67
70
|
normalizeInitSetupMarker,
|
|
68
71
|
resolveInitRecommendedModels,
|
|
69
72
|
} from "./init-setup.js";
|
|
73
|
+
import {
|
|
74
|
+
checkPackageUpdate,
|
|
75
|
+
runPackageUpdate,
|
|
76
|
+
scheduleLaunchAgentRestart,
|
|
77
|
+
} from "./package-update.js";
|
|
70
78
|
|
|
71
79
|
const logger = createModuleLogger("tb-proxyd");
|
|
72
80
|
const FOCUS_SET_CONFIG_KEY = "focus-set";
|
|
81
|
+
const TRUSTED_REGISTRY_CACHE_CONFIG_KEY = "trusted-registry-snapshot";
|
|
73
82
|
const PROXY_JSON_BODY_LIMIT = "10mb";
|
|
74
83
|
const SELLER_CAPACITY_BLOCK_MS = 2_000;
|
|
75
84
|
const CLAWTIP_STATIC_ROUTE = "/static/clawtip";
|
|
@@ -135,6 +144,17 @@ interface InitDoctorCatalogSnapshot {
|
|
|
135
144
|
errorMessage?: string;
|
|
136
145
|
}
|
|
137
146
|
|
|
147
|
+
interface TrustedRegistryCacheRecord {
|
|
148
|
+
schemaVersion: 1;
|
|
149
|
+
registryUrl: string;
|
|
150
|
+
version?: number;
|
|
151
|
+
registrySha256: string;
|
|
152
|
+
cachedAt: string;
|
|
153
|
+
registryJson: string;
|
|
154
|
+
registry: SellerRegistryDocument;
|
|
155
|
+
trust: SellerRegistryTrustMetadata;
|
|
156
|
+
}
|
|
157
|
+
|
|
138
158
|
function clientToolStatusFromProvider(provider: ProviderCandidate): ClientToolStatus {
|
|
139
159
|
return {
|
|
140
160
|
id: provider.id,
|
|
@@ -234,6 +254,8 @@ export interface DaemonConfig {
|
|
|
234
254
|
warmupRefreshIntervalSecs?: number;
|
|
235
255
|
/** 预热探测超时(毫秒) */
|
|
236
256
|
warmupProbeTimeoutMs?: number;
|
|
257
|
+
/** buyer 端本地 seller 并发 lease 限制;默认关闭。 */
|
|
258
|
+
sellerConcurrency?: SellerConcurrencyLimiterOptions;
|
|
237
259
|
}
|
|
238
260
|
|
|
239
261
|
interface SellerRoute {
|
|
@@ -242,6 +264,9 @@ interface SellerRoute {
|
|
|
242
264
|
protocol: string;
|
|
243
265
|
modelId: string;
|
|
244
266
|
paymentMethod: string;
|
|
267
|
+
planSource: SellerRoutePlan["source"];
|
|
268
|
+
planReason: string;
|
|
269
|
+
planSellerCount: number;
|
|
245
270
|
poolEntry?: PoolEntry;
|
|
246
271
|
}
|
|
247
272
|
|
|
@@ -547,6 +572,7 @@ export class TokenbuddyDaemon {
|
|
|
547
572
|
// config-derived knobs. The `!` opts out of strict-initialization so the
|
|
548
573
|
// rest of the class can treat it as non-nullable.
|
|
549
574
|
private readonly prewarmScheduler!: PrewarmScheduler;
|
|
575
|
+
private readonly sellerConcurrencyLimiter: SellerConcurrencyLimiter;
|
|
550
576
|
|
|
551
577
|
constructor(config: DaemonConfig) {
|
|
552
578
|
this.tokenStore = new BuyerStore({ dbPath: config.dbPath });
|
|
@@ -559,6 +585,7 @@ export class TokenbuddyDaemon {
|
|
|
559
585
|
storedRouting,
|
|
560
586
|
config.sellerRouting
|
|
561
587
|
);
|
|
588
|
+
this.sellerConcurrencyLimiter = new SellerConcurrencyLimiter(config.sellerConcurrency);
|
|
562
589
|
this.selectionMode = selectionModeForRouting(this.sellerRouting);
|
|
563
590
|
this.selectedSellerId = selectedSellerIdForRouting(this.sellerRouting);
|
|
564
591
|
// tb-ui v1: explicit focus set 优先于 env / historical
|
|
@@ -685,7 +712,7 @@ export class TokenbuddyDaemon {
|
|
|
685
712
|
}
|
|
686
713
|
|
|
687
714
|
private async startClawtipActivationQr(): Promise<ClawtipQrResponse> {
|
|
688
|
-
const bootstrapUrl = process.env.TOKENBUDDY_BOOTSTRAP_URL ||
|
|
715
|
+
const bootstrapUrl = process.env.TOKENBUDDY_BOOTSTRAP_URL || DEFAULT_CLAWTIP_BOOTSTRAP_URL;
|
|
689
716
|
const fetchBootstrap = this.config.clawtipBootstrapFetcher || fetchClawtipBootstrap;
|
|
690
717
|
const bootstrap = await fetchBootstrap(bootstrapUrl);
|
|
691
718
|
const payment = normalizeClawtipActivationPayment(bootstrap);
|
|
@@ -833,13 +860,15 @@ export class TokenbuddyDaemon {
|
|
|
833
860
|
|
|
834
861
|
private async fetchRegistry(): Promise<SellerRegistryDocument> {
|
|
835
862
|
try {
|
|
836
|
-
const
|
|
863
|
+
const fetched = await fetchSellerRegistryWithTrust(this.config.sellerRegistryUrl);
|
|
864
|
+
const registry = fetched.registry;
|
|
837
865
|
this.modelIndex.rebuild(registry.sellers, {
|
|
838
866
|
registryVersion: registry.version,
|
|
839
867
|
defaultSellerId: registry.defaultSeller
|
|
840
868
|
});
|
|
841
869
|
this.sellerPool.sync();
|
|
842
870
|
this.lastRegistrySnapshot = registry;
|
|
871
|
+
this.saveTrustedRegistryCache(fetched);
|
|
843
872
|
return registry;
|
|
844
873
|
} catch (err) {
|
|
845
874
|
// v1.2 §18.9: if the bootstrap returns 413, fall back to the
|
|
@@ -859,10 +888,105 @@ export class TokenbuddyDaemon {
|
|
|
859
888
|
this.sellerPool.sync();
|
|
860
889
|
return stale;
|
|
861
890
|
}
|
|
891
|
+
const cached = this.loadTrustedRegistryCache(err);
|
|
892
|
+
if (cached) {
|
|
893
|
+
this.modelIndex.rebuild(cached.sellers, {
|
|
894
|
+
registryVersion: cached.version,
|
|
895
|
+
defaultSellerId: cached.defaultSeller
|
|
896
|
+
});
|
|
897
|
+
this.sellerPool.sync();
|
|
898
|
+
this.lastRegistrySnapshot = cached;
|
|
899
|
+
return cached;
|
|
900
|
+
}
|
|
862
901
|
throw err;
|
|
863
902
|
}
|
|
864
903
|
}
|
|
865
904
|
|
|
905
|
+
private saveTrustedRegistryCache(fetched: { registry: SellerRegistryDocument; registryJson: string; trust: SellerRegistryTrustMetadata }): void {
|
|
906
|
+
const cache: TrustedRegistryCacheRecord = {
|
|
907
|
+
schemaVersion: 1,
|
|
908
|
+
registryUrl: this.config.sellerRegistryUrl,
|
|
909
|
+
version: fetched.registry.version,
|
|
910
|
+
registrySha256: fetched.trust.registrySha256,
|
|
911
|
+
cachedAt: new Date().toISOString(),
|
|
912
|
+
registryJson: fetched.registryJson,
|
|
913
|
+
registry: fetched.registry,
|
|
914
|
+
trust: fetched.trust
|
|
915
|
+
};
|
|
916
|
+
this.tokenStore.saveDaemonRuntimeConfig(TRUSTED_REGISTRY_CACHE_CONFIG_KEY, cache);
|
|
917
|
+
logger.info("registry.trusted_cache.saved", "trusted registry snapshot cached", {
|
|
918
|
+
registryUrl: cache.registryUrl,
|
|
919
|
+
registryVersion: cache.version ?? null,
|
|
920
|
+
registrySha256: cache.registrySha256,
|
|
921
|
+
verified: cache.trust.verified,
|
|
922
|
+
signingKeyId: cache.trust.signingKeyId ?? null
|
|
923
|
+
});
|
|
924
|
+
}
|
|
925
|
+
|
|
926
|
+
private loadTrustedRegistryCache(error: unknown): SellerRegistryDocument | undefined {
|
|
927
|
+
const errorMessage = error instanceof Error ? error.message : String(error);
|
|
928
|
+
const record = this.tokenStore.getDaemonRuntimeConfig<unknown>(TRUSTED_REGISTRY_CACHE_CONFIG_KEY)?.config;
|
|
929
|
+
const cache = normalizeTrustedRegistryCache(record);
|
|
930
|
+
if (!cache) {
|
|
931
|
+
logger.warn("registry.trusted_cache.missing", "trusted registry cache is unavailable", {
|
|
932
|
+
registryUrl: this.config.sellerRegistryUrl,
|
|
933
|
+
errorMessage
|
|
934
|
+
});
|
|
935
|
+
return undefined;
|
|
936
|
+
}
|
|
937
|
+
if (cache.registryUrl !== this.config.sellerRegistryUrl) {
|
|
938
|
+
logger.warn("registry.trusted_cache.missing", "trusted registry cache URL mismatch", {
|
|
939
|
+
registryUrl: this.config.sellerRegistryUrl,
|
|
940
|
+
cachedRegistryUrl: cache.registryUrl,
|
|
941
|
+
errorMessage
|
|
942
|
+
});
|
|
943
|
+
return undefined;
|
|
944
|
+
}
|
|
945
|
+
const actualHash = crypto.createHash("sha256").update(cache.registryJson).digest("hex");
|
|
946
|
+
if (actualHash !== cache.registrySha256 || actualHash !== cache.trust.registrySha256) {
|
|
947
|
+
logger.warn("registry.trusted_cache.missing", "trusted registry cache hash mismatch", {
|
|
948
|
+
registryUrl: this.config.sellerRegistryUrl,
|
|
949
|
+
cachedVersion: cache.version ?? null,
|
|
950
|
+
errorMessage
|
|
951
|
+
});
|
|
952
|
+
return undefined;
|
|
953
|
+
}
|
|
954
|
+
if (shouldVerifyRegistry(this.config.sellerRegistryUrl)) {
|
|
955
|
+
if (!cache.trust.signature) {
|
|
956
|
+
logger.warn("registry.trusted_cache.missing", "trusted registry cache has no signature", {
|
|
957
|
+
registryUrl: this.config.sellerRegistryUrl,
|
|
958
|
+
cachedVersion: cache.version ?? null,
|
|
959
|
+
errorMessage
|
|
960
|
+
});
|
|
961
|
+
return undefined;
|
|
962
|
+
}
|
|
963
|
+
try {
|
|
964
|
+
const signingKeyId = verifyTrustedRegistrySignature(cache.registryJson, cache.trust.signature);
|
|
965
|
+
if (cache.trust.signingKeyId && cache.trust.signingKeyId !== signingKeyId) {
|
|
966
|
+
throw new Error("registry cache signing key mismatch");
|
|
967
|
+
}
|
|
968
|
+
} catch (verifyError) {
|
|
969
|
+
logger.warn("registry.trusted_cache.missing", "trusted registry cache signature verification failed", {
|
|
970
|
+
registryUrl: this.config.sellerRegistryUrl,
|
|
971
|
+
cachedVersion: cache.version ?? null,
|
|
972
|
+
errorMessage: verifyError instanceof Error ? verifyError.message : String(verifyError)
|
|
973
|
+
});
|
|
974
|
+
return undefined;
|
|
975
|
+
}
|
|
976
|
+
}
|
|
977
|
+
logger.warn("registry.trusted_cache.used", "using trusted registry cache after refresh failure", {
|
|
978
|
+
registryUrl: this.config.sellerRegistryUrl,
|
|
979
|
+
cachedVersion: cache.version ?? null,
|
|
980
|
+
cachedSellers: cache.registry.sellers.length,
|
|
981
|
+
cachedAt: cache.cachedAt,
|
|
982
|
+
registrySha256: cache.registrySha256,
|
|
983
|
+
verified: cache.trust.verified,
|
|
984
|
+
signingKeyId: cache.trust.signingKeyId ?? null,
|
|
985
|
+
errorMessage
|
|
986
|
+
});
|
|
987
|
+
return cache.registry;
|
|
988
|
+
}
|
|
989
|
+
|
|
866
990
|
private runtimeSummary() {
|
|
867
991
|
this.refreshSellerRoutingConfig();
|
|
868
992
|
return {
|
|
@@ -877,6 +1001,7 @@ export class TokenbuddyDaemon {
|
|
|
877
1001
|
selectedSellerId: this.selectedSellerId,
|
|
878
1002
|
dbPath: this.config.dbPath,
|
|
879
1003
|
sellerRegistryUrl: this.config.sellerRegistryUrl,
|
|
1004
|
+
sellerConcurrency: this.sellerConcurrencyLimiter.snapshot(),
|
|
880
1005
|
store: this.tokenStore.summary()
|
|
881
1006
|
};
|
|
882
1007
|
}
|
|
@@ -1193,7 +1318,7 @@ export class TokenbuddyDaemon {
|
|
|
1193
1318
|
return payments.find((payment) => payment.isDefault)?.method || payments.find((payment) => payment.method === "mock")?.method;
|
|
1194
1319
|
}
|
|
1195
1320
|
|
|
1196
|
-
private async selectSellerRoutes(endpoint: string, modelId: string): Promise<{ routes: SellerRoute[]; plan: SellerRoutePlan; paymentMethod: string }> {
|
|
1321
|
+
private async selectSellerRoutes(endpoint: string, modelId: string, requestId?: string): Promise<{ routes: SellerRoute[]; plan: SellerRoutePlan; paymentMethod: string }> {
|
|
1197
1322
|
const protocol = this.endpointProtocol(endpoint);
|
|
1198
1323
|
if (!protocol) {
|
|
1199
1324
|
throw new Error(`unsupported proxy endpoint: ${endpoint}`);
|
|
@@ -1214,7 +1339,10 @@ export class TokenbuddyDaemon {
|
|
|
1214
1339
|
const registrySellers = reorderDefaultSellerFirst(registry.sellers, registry.defaultSeller);
|
|
1215
1340
|
this.sellerPool.ensureRegistrySellers(registrySellers);
|
|
1216
1341
|
this.scheduleLazyPrewarmIfNeeded(modelId, protocol, paymentMethod);
|
|
1342
|
+
this.sellerPool.recycleOpenCircuits();
|
|
1217
1343
|
const poolById = new Map(this.sellerPool.snapshot().map((entry) => [entry.sellerId, entry]));
|
|
1344
|
+
const concurrencySnapshot = this.sellerConcurrencyLimiter.snapshot();
|
|
1345
|
+
const localConcurrencyBySellerId = new Map(concurrencySnapshot.active.map((entry) => [entry.sellerId, entry.activeCount]));
|
|
1218
1346
|
const planned = planSellerRouteSet({
|
|
1219
1347
|
modelId,
|
|
1220
1348
|
protocol,
|
|
@@ -1229,25 +1357,19 @@ export class TokenbuddyDaemon {
|
|
|
1229
1357
|
ttftMs: entry.ttftMs,
|
|
1230
1358
|
avgInferenceMs: entry.avgInferenceMs,
|
|
1231
1359
|
circuit: entry.circuit,
|
|
1232
|
-
capacityBlockedUntil: entry.capacityBlockedUntil
|
|
1360
|
+
capacityBlockedUntil: entry.capacityBlockedUntil,
|
|
1361
|
+
...(concurrencySnapshot.enabled
|
|
1362
|
+
? {
|
|
1363
|
+
localConcurrencyActive: localConcurrencyBySellerId.get(entry.sellerId) ?? 0,
|
|
1364
|
+
localConcurrencyLimit: concurrencySnapshot.maxInFlightPerSeller
|
|
1365
|
+
}
|
|
1366
|
+
: {})
|
|
1233
1367
|
})),
|
|
1234
1368
|
now: Date.now()
|
|
1235
1369
|
});
|
|
1236
1370
|
|
|
1237
|
-
if (planned.routes.length === 0) {
|
|
1238
|
-
throw new Error(`no compatible seller for ${endpoint} model ${modelId}`);
|
|
1239
|
-
}
|
|
1240
|
-
|
|
1241
|
-
const routes: SellerRoute[] = planned.routes.map((route) => ({
|
|
1242
|
-
seller: route.seller,
|
|
1243
|
-
manifest: null,
|
|
1244
|
-
protocol,
|
|
1245
|
-
modelId,
|
|
1246
|
-
paymentMethod,
|
|
1247
|
-
poolEntry: poolById.get(route.seller.id)
|
|
1248
|
-
}));
|
|
1249
|
-
|
|
1250
1371
|
logger.info("route.candidates.prewarmed", "seller route candidates prewarmed", {
|
|
1372
|
+
requestId,
|
|
1251
1373
|
model: modelId,
|
|
1252
1374
|
endpoint,
|
|
1253
1375
|
protocol,
|
|
@@ -1258,9 +1380,26 @@ export class TokenbuddyDaemon {
|
|
|
1258
1380
|
routeSource: planned.source,
|
|
1259
1381
|
routeSourceReason: planned.sourceReason,
|
|
1260
1382
|
routeReason: planned.reason,
|
|
1261
|
-
|
|
1262
|
-
|
|
1383
|
+
candidateDiagnostics: planned.diagnostics,
|
|
1384
|
+
sellerCount: planned.routes.length,
|
|
1385
|
+
sellers: planned.routes.map((route) => route.seller.id)
|
|
1263
1386
|
});
|
|
1387
|
+
|
|
1388
|
+
if (planned.routes.length === 0) {
|
|
1389
|
+
throw new Error(`no compatible seller for ${endpoint} model ${modelId}`);
|
|
1390
|
+
}
|
|
1391
|
+
|
|
1392
|
+
const routes: SellerRoute[] = planned.routes.map((route) => ({
|
|
1393
|
+
seller: route.seller,
|
|
1394
|
+
manifest: null,
|
|
1395
|
+
protocol,
|
|
1396
|
+
modelId,
|
|
1397
|
+
paymentMethod,
|
|
1398
|
+
planSource: planned.source,
|
|
1399
|
+
planReason: planned.reason,
|
|
1400
|
+
planSellerCount: planned.routes.length,
|
|
1401
|
+
poolEntry: poolById.get(route.seller.id)
|
|
1402
|
+
}));
|
|
1264
1403
|
return { routes, plan: planned, paymentMethod };
|
|
1265
1404
|
}
|
|
1266
1405
|
|
|
@@ -1427,6 +1566,8 @@ export class TokenbuddyDaemon {
|
|
|
1427
1566
|
reason?: string;
|
|
1428
1567
|
}
|
|
1429
1568
|
): void {
|
|
1569
|
+
const routesRemainingAfterCurrent = Math.max(0, context.routesRemaining - 1);
|
|
1570
|
+
const hasNextRoute = routesRemainingAfterCurrent > 0;
|
|
1430
1571
|
if (decision.action === "retry_same_seller") {
|
|
1431
1572
|
logger.warn("route.failover.retry_scheduled", "seller route retry scheduled", {
|
|
1432
1573
|
requestId: context.requestId,
|
|
@@ -1435,8 +1576,12 @@ export class TokenbuddyDaemon {
|
|
|
1435
1576
|
endpoint: context.endpoint,
|
|
1436
1577
|
routeIndex: context.routeIndex,
|
|
1437
1578
|
routesRemaining: context.routesRemaining,
|
|
1579
|
+
routesRemainingAfterCurrent,
|
|
1580
|
+
hasNextRoute,
|
|
1438
1581
|
attempt: context.attempt,
|
|
1582
|
+
attemptNumber: context.attempt + 1,
|
|
1439
1583
|
nextAttempt: context.attempt + 1,
|
|
1584
|
+
nextAttemptNumber: context.attempt + 2,
|
|
1440
1585
|
reason: decision.reason,
|
|
1441
1586
|
status: context.status,
|
|
1442
1587
|
retryDelayMs: decision.retryDelayMs
|
|
@@ -1444,7 +1589,7 @@ export class TokenbuddyDaemon {
|
|
|
1444
1589
|
return;
|
|
1445
1590
|
}
|
|
1446
1591
|
if (decision.action === "failover_next") {
|
|
1447
|
-
logger.warn("route.failover.triggered", "seller route failed over
|
|
1592
|
+
logger.warn("route.failover.triggered", "seller route failed over after seller failure", {
|
|
1448
1593
|
requestId: context.requestId,
|
|
1449
1594
|
sellerKey: context.sellerKey,
|
|
1450
1595
|
model: context.model,
|
|
@@ -1452,7 +1597,10 @@ export class TokenbuddyDaemon {
|
|
|
1452
1597
|
routeIndex: context.routeIndex,
|
|
1453
1598
|
nextRouteIndex: context.routeIndex + 1,
|
|
1454
1599
|
routesRemaining: context.routesRemaining,
|
|
1600
|
+
routesRemainingAfterCurrent,
|
|
1601
|
+
hasNextRoute,
|
|
1455
1602
|
attempt: context.attempt,
|
|
1603
|
+
attemptNumber: context.attempt + 1,
|
|
1456
1604
|
reason: decision.reason,
|
|
1457
1605
|
status: context.status,
|
|
1458
1606
|
wastedCreditMicros: decision.wastedCreditMicros,
|
|
@@ -1468,7 +1616,10 @@ export class TokenbuddyDaemon {
|
|
|
1468
1616
|
endpoint: context.endpoint,
|
|
1469
1617
|
routeIndex: context.routeIndex,
|
|
1470
1618
|
routesRemaining: context.routesRemaining,
|
|
1619
|
+
routesRemainingAfterCurrent,
|
|
1620
|
+
hasNextRoute,
|
|
1471
1621
|
attempt: context.attempt,
|
|
1622
|
+
attemptNumber: context.attempt + 1,
|
|
1472
1623
|
action: decision.action,
|
|
1473
1624
|
reason: decision.reason,
|
|
1474
1625
|
status: context.status
|
|
@@ -1533,12 +1684,31 @@ export class TokenbuddyDaemon {
|
|
|
1533
1684
|
models: Array<{ id: string; sellerId: string; sellerName?: string; sellerUrl: string; supportedProtocols: string[]; paymentMethods: string[] }>;
|
|
1534
1685
|
sellers: Array<{ id: string; name?: string; url: string; status: string; manifestSellerId?: string; errorMessage?: string }>;
|
|
1535
1686
|
}> {
|
|
1536
|
-
|
|
1537
|
-
|
|
1538
|
-
|
|
1539
|
-
|
|
1540
|
-
|
|
1541
|
-
|
|
1687
|
+
try {
|
|
1688
|
+
const catalog = await discoverSellerBackedModels(this.config.sellerRegistryUrl);
|
|
1689
|
+
this.lastRegistrySnapshot = catalog.registry ?? sellerCatalogResultToRegistrySnapshot(catalog);
|
|
1690
|
+
if (catalog.registry && catalog.registryJson && catalog.registryTrust) {
|
|
1691
|
+
this.saveTrustedRegistryCache({
|
|
1692
|
+
registry: catalog.registry,
|
|
1693
|
+
registryJson: catalog.registryJson,
|
|
1694
|
+
trust: catalog.registryTrust
|
|
1695
|
+
});
|
|
1696
|
+
}
|
|
1697
|
+
return {
|
|
1698
|
+
models: catalog.models,
|
|
1699
|
+
sellers: catalog.sellers
|
|
1700
|
+
};
|
|
1701
|
+
} catch (error) {
|
|
1702
|
+
const cached = this.loadTrustedRegistryCache(error);
|
|
1703
|
+
if (!cached) {
|
|
1704
|
+
throw error;
|
|
1705
|
+
}
|
|
1706
|
+
const snapshot = catalogSnapshotFromRegistry(cached);
|
|
1707
|
+
return {
|
|
1708
|
+
models: snapshot.models,
|
|
1709
|
+
sellers: snapshot.sellers
|
|
1710
|
+
};
|
|
1711
|
+
}
|
|
1542
1712
|
}
|
|
1543
1713
|
|
|
1544
1714
|
private readUsage(bodyText: string): UsageSummary {
|
|
@@ -2171,7 +2341,7 @@ export class TokenbuddyDaemon {
|
|
|
2171
2341
|
}
|
|
2172
2342
|
|
|
2173
2343
|
try {
|
|
2174
|
-
const { routes, plan, paymentMethod } = await this.selectSellerRoutes(endpoint, modelId);
|
|
2344
|
+
const { routes, plan, paymentMethod } = await this.selectSellerRoutes(endpoint, modelId, requestId);
|
|
2175
2345
|
const upstreamStatusFromHeaders = (h: Headers): string | undefined => {
|
|
2176
2346
|
const raw = h.get("x-tokenbuddy-upstream-status") || h.get("x-upstream-status");
|
|
2177
2347
|
if (!raw) return undefined;
|
|
@@ -2182,24 +2352,57 @@ export class TokenbuddyDaemon {
|
|
|
2182
2352
|
for (let routeIndex = 0; routeIndex < routes.length; routeIndex += 1) {
|
|
2183
2353
|
const route = routes[routeIndex];
|
|
2184
2354
|
const sellerKey = route.seller.id;
|
|
2185
|
-
|
|
2186
|
-
|
|
2187
|
-
|
|
2188
|
-
|
|
2189
|
-
|
|
2190
|
-
|
|
2191
|
-
|
|
2192
|
-
|
|
2193
|
-
|
|
2194
|
-
|
|
2195
|
-
|
|
2196
|
-
|
|
2197
|
-
|
|
2198
|
-
|
|
2199
|
-
|
|
2200
|
-
|
|
2201
|
-
|
|
2202
|
-
|
|
2355
|
+
const lease = this.sellerConcurrencyLimiter.tryAcquire(sellerKey, { requestId, modelId, endpoint });
|
|
2356
|
+
if (!lease) {
|
|
2357
|
+
const routesRemaining = routes.length - routeIndex;
|
|
2358
|
+
const routesRemainingAfterCurrent = Math.max(0, routesRemaining - 1);
|
|
2359
|
+
lastError = new Error(`seller ${sellerKey} reached local concurrency limit`);
|
|
2360
|
+
logger.info("route.local_capacity.skipped", "seller route skipped by local concurrency limit", {
|
|
2361
|
+
requestId,
|
|
2362
|
+
sellerKey,
|
|
2363
|
+
sellerId: sellerKey,
|
|
2364
|
+
model: modelId,
|
|
2365
|
+
endpoint,
|
|
2366
|
+
protocol: route.protocol,
|
|
2367
|
+
paymentMethod: route.paymentMethod,
|
|
2368
|
+
routeIndex,
|
|
2369
|
+
routePlanSource: route.planSource,
|
|
2370
|
+
routePlanReason: route.planReason,
|
|
2371
|
+
routePlanSellerCount: route.planSellerCount,
|
|
2372
|
+
routesRemaining,
|
|
2373
|
+
routesRemainingAfterCurrent,
|
|
2374
|
+
hasNextRoute: routesRemainingAfterCurrent > 0,
|
|
2375
|
+
localConcurrencyEnabled: this.sellerConcurrencyLimiter.isEnabled()
|
|
2376
|
+
});
|
|
2377
|
+
continue;
|
|
2378
|
+
}
|
|
2379
|
+
|
|
2380
|
+
try {
|
|
2381
|
+
logger.info("route.selected", "seller route selected", {
|
|
2382
|
+
requestId,
|
|
2383
|
+
sellerKey,
|
|
2384
|
+
sellerId: sellerKey,
|
|
2385
|
+
model: modelId,
|
|
2386
|
+
endpoint,
|
|
2387
|
+
protocol: route.protocol,
|
|
2388
|
+
paymentMethod: route.paymentMethod,
|
|
2389
|
+
routeIndex,
|
|
2390
|
+
backup: routeIndex > 0,
|
|
2391
|
+
routePlanSource: route.planSource,
|
|
2392
|
+
routePlanReason: route.planReason,
|
|
2393
|
+
routePlanSellerCount: route.planSellerCount,
|
|
2394
|
+
localConcurrencyEnabled: this.sellerConcurrencyLimiter.isEnabled(),
|
|
2395
|
+
localConcurrencyActive: lease.activeCount,
|
|
2396
|
+
localConcurrencyLimit: lease.maxInFlight
|
|
2397
|
+
});
|
|
2398
|
+
let attempt = 0;
|
|
2399
|
+
// Soft-failure retry budget; the route-failover controller decides
|
|
2400
|
+
// whether the same seller should be retried or we move on. The
|
|
2401
|
+
// v1 "1 retry for 4xx fallback" loop is replaced with a
|
|
2402
|
+
// stateful decision per attempt.
|
|
2403
|
+
// eslint-disable-next-line no-constant-condition
|
|
2404
|
+
while (true) {
|
|
2405
|
+
try {
|
|
2203
2406
|
logger.info("proxy.request.started", "proxy request started", {
|
|
2204
2407
|
requestId,
|
|
2205
2408
|
sellerKey,
|
|
@@ -2239,7 +2442,7 @@ export class TokenbuddyDaemon {
|
|
|
2239
2442
|
}
|
|
2240
2443
|
// v1.1: a purchase failure means the seller is unreachable for
|
|
2241
2444
|
// payment, not "transiently flapping". Do not retry the same
|
|
2242
|
-
// seller;
|
|
2445
|
+
// seller; fail over immediately without marking old credit wasted.
|
|
2243
2446
|
let token: string;
|
|
2244
2447
|
try {
|
|
2245
2448
|
token = await this.getOrPurchaseToken(route, requestId);
|
|
@@ -2254,7 +2457,7 @@ export class TokenbuddyDaemon {
|
|
|
2254
2457
|
const decision = this.routeFailover.decide(
|
|
2255
2458
|
{
|
|
2256
2459
|
sellerId: sellerKey,
|
|
2257
|
-
errorKind: "
|
|
2460
|
+
errorKind: "purchase_failed",
|
|
2258
2461
|
errorMessage: this.failoverErrorMessage(purchaseError),
|
|
2259
2462
|
attempt
|
|
2260
2463
|
},
|
|
@@ -2268,7 +2471,10 @@ export class TokenbuddyDaemon {
|
|
|
2268
2471
|
routeIndex,
|
|
2269
2472
|
nextRouteIndex: routeIndex + 1,
|
|
2270
2473
|
routesRemaining: routes.length - routeIndex,
|
|
2474
|
+
routesRemainingAfterCurrent: Math.max(0, routes.length - routeIndex - 1),
|
|
2475
|
+
hasNextRoute: routeIndex + 1 < routes.length,
|
|
2271
2476
|
attempt,
|
|
2477
|
+
attemptNumber: attempt + 1,
|
|
2272
2478
|
reason: "purchase_failed",
|
|
2273
2479
|
controllerReason: decision.reason,
|
|
2274
2480
|
controllerAction: decision.action
|
|
@@ -2304,12 +2510,15 @@ export class TokenbuddyDaemon {
|
|
|
2304
2510
|
}
|
|
2305
2511
|
};
|
|
2306
2512
|
let upstreamResponse = await sendSellerRequest(token);
|
|
2513
|
+
lease.refresh();
|
|
2307
2514
|
|
|
2308
2515
|
if (!upstreamResponse.ok) {
|
|
2309
2516
|
const errorBody = await upstreamResponse.text();
|
|
2517
|
+
lease.refresh();
|
|
2310
2518
|
if (this.isInsufficientFundsResponse(upstreamResponse.status, errorBody)) {
|
|
2311
2519
|
token = await this.recoverFromInsufficientFunds(route, token, requestId);
|
|
2312
2520
|
upstreamResponse = await sendSellerRequest(token);
|
|
2521
|
+
lease.refresh();
|
|
2313
2522
|
if (upstreamResponse.ok) {
|
|
2314
2523
|
logger.info("proxy.retry_after_402.succeeded", "seller request succeeded after one-shot auto purchase retry", {
|
|
2315
2524
|
requestId,
|
|
@@ -2406,6 +2615,7 @@ export class TokenbuddyDaemon {
|
|
|
2406
2615
|
const settlementExtractor = new SellerSettlementStreamExtractor();
|
|
2407
2616
|
while (true) {
|
|
2408
2617
|
const { done, value } = await reader.read();
|
|
2618
|
+
lease.refresh();
|
|
2409
2619
|
if (done) {
|
|
2410
2620
|
break;
|
|
2411
2621
|
}
|
|
@@ -2461,6 +2671,7 @@ export class TokenbuddyDaemon {
|
|
|
2461
2671
|
}
|
|
2462
2672
|
|
|
2463
2673
|
const responseBody = await upstreamResponse.text();
|
|
2674
|
+
lease.refresh();
|
|
2464
2675
|
markFirstByte();
|
|
2465
2676
|
res.send(responseBody);
|
|
2466
2677
|
const usage = this.readUsage(responseBody);
|
|
@@ -2526,6 +2737,9 @@ export class TokenbuddyDaemon {
|
|
|
2526
2737
|
// failover_next
|
|
2527
2738
|
break;
|
|
2528
2739
|
}
|
|
2740
|
+
}
|
|
2741
|
+
} finally {
|
|
2742
|
+
lease.release();
|
|
2529
2743
|
}
|
|
2530
2744
|
}
|
|
2531
2745
|
|
|
@@ -2651,6 +2865,55 @@ export class TokenbuddyDaemon {
|
|
|
2651
2865
|
}
|
|
2652
2866
|
});
|
|
2653
2867
|
|
|
2868
|
+
controlApp.get("/update/status", async (_req, res) => {
|
|
2869
|
+
try {
|
|
2870
|
+
const update = await checkPackageUpdate();
|
|
2871
|
+
logger.info("control.update.status.requested", "package update status requested", {
|
|
2872
|
+
currentVersion: update.currentVersion,
|
|
2873
|
+
latestVersion: update.latestVersion,
|
|
2874
|
+
updateAvailable: update.updateAvailable
|
|
2875
|
+
});
|
|
2876
|
+
res.status(200).json(update);
|
|
2877
|
+
} catch (error: unknown) {
|
|
2878
|
+
const errorMessage = error instanceof Error ? error.message : String(error);
|
|
2879
|
+
logger.warn("control.update.status.failed", "package update status failed", { errorMessage });
|
|
2880
|
+
res.status(502).json({
|
|
2881
|
+
error: {
|
|
2882
|
+
code: "update_status_failed",
|
|
2883
|
+
message: errorMessage
|
|
2884
|
+
}
|
|
2885
|
+
});
|
|
2886
|
+
}
|
|
2887
|
+
});
|
|
2888
|
+
|
|
2889
|
+
controlApp.post("/update/run", async (_req, res) => {
|
|
2890
|
+
try {
|
|
2891
|
+
const result = await runPackageUpdate(
|
|
2892
|
+
{ apply: true, controlPort: this.activeControlPort() },
|
|
2893
|
+
{
|
|
2894
|
+
restartProxyd: async () => scheduleLaunchAgentRestart()
|
|
2895
|
+
},
|
|
2896
|
+
);
|
|
2897
|
+
logger.info("control.update.run.completed", "package update run completed", {
|
|
2898
|
+
currentVersion: result.check.currentVersion,
|
|
2899
|
+
latestVersion: result.check.latestVersion,
|
|
2900
|
+
updateAvailable: result.check.updateAvailable,
|
|
2901
|
+
installSucceeded: result.install.succeeded,
|
|
2902
|
+
restartScheduled: result.restart.scheduled === true
|
|
2903
|
+
});
|
|
2904
|
+
res.status(result.install.error || result.restart.error ? 500 : 200).json(result);
|
|
2905
|
+
} catch (error: unknown) {
|
|
2906
|
+
const errorMessage = error instanceof Error ? error.message : String(error);
|
|
2907
|
+
logger.warn("control.update.run.failed", "package update run failed", { errorMessage });
|
|
2908
|
+
res.status(500).json({
|
|
2909
|
+
error: {
|
|
2910
|
+
code: "update_run_failed",
|
|
2911
|
+
message: errorMessage
|
|
2912
|
+
}
|
|
2913
|
+
});
|
|
2914
|
+
}
|
|
2915
|
+
});
|
|
2916
|
+
|
|
2654
2917
|
controlApp.post("/payments/clawtip/activate", async (req, res) => {
|
|
2655
2918
|
try {
|
|
2656
2919
|
const qr = await this.startClawtipActivationQr();
|
|
@@ -3077,7 +3340,7 @@ export class TokenbuddyDaemon {
|
|
|
3077
3340
|
// SPA fallback: React Router 用 BrowserRouter(history mode),客户端路径如
|
|
3078
3341
|
// /overview /routing /ledger 不对应任何文件,必须回 index.html 让 React Router 接管。
|
|
3079
3342
|
// Express 已经按注册顺序匹配了所有 17+ 个 API 路由,这里只接住"什么都没匹配"的 GET。
|
|
3080
|
-
controlApp.get(/^(?!\/(?:health|status|payments|sellers|models|ledger|routing|prewarm|v1\.2|daemon|providers|static)\/).*/, (_req, res) => {
|
|
3343
|
+
controlApp.get(/^(?!\/(?:health|status|payments|sellers|models|ledger|routing|prewarm|v1\.2|daemon|providers|update|static)\/).*/, (_req, res) => {
|
|
3081
3344
|
res.sendFile(path.join(uiDir, "index.html"));
|
|
3082
3345
|
});
|
|
3083
3346
|
logger.info("ui.static.attached", "tb-ui dist attached to control plane SPA fallback", { uiDir });
|
|
@@ -3463,6 +3726,40 @@ function catalogSnapshotFromRegistry(registry: SellerRegistryDocument): InitDoct
|
|
|
3463
3726
|
};
|
|
3464
3727
|
}
|
|
3465
3728
|
|
|
3729
|
+
function normalizeTrustedRegistryCache(value: unknown): TrustedRegistryCacheRecord | undefined {
|
|
3730
|
+
if (!value || typeof value !== "object") {
|
|
3731
|
+
return undefined;
|
|
3732
|
+
}
|
|
3733
|
+
const data = value as Partial<TrustedRegistryCacheRecord>;
|
|
3734
|
+
if (
|
|
3735
|
+
data.schemaVersion !== 1 ||
|
|
3736
|
+
typeof data.registryUrl !== "string" ||
|
|
3737
|
+
typeof data.registrySha256 !== "string" ||
|
|
3738
|
+
typeof data.cachedAt !== "string" ||
|
|
3739
|
+
typeof data.registryJson !== "string" ||
|
|
3740
|
+
!data.registry ||
|
|
3741
|
+
typeof data.registry !== "object" ||
|
|
3742
|
+
!Array.isArray(data.registry.sellers) ||
|
|
3743
|
+
!data.trust ||
|
|
3744
|
+
typeof data.trust !== "object" ||
|
|
3745
|
+
typeof data.trust.registryUrl !== "string" ||
|
|
3746
|
+
typeof data.trust.registrySha256 !== "string" ||
|
|
3747
|
+
typeof data.trust.verified !== "boolean"
|
|
3748
|
+
) {
|
|
3749
|
+
return undefined;
|
|
3750
|
+
}
|
|
3751
|
+
return {
|
|
3752
|
+
schemaVersion: 1,
|
|
3753
|
+
registryUrl: data.registryUrl,
|
|
3754
|
+
version: typeof data.version === "number" ? data.version : data.registry.version,
|
|
3755
|
+
registrySha256: data.registrySha256,
|
|
3756
|
+
cachedAt: data.cachedAt,
|
|
3757
|
+
registryJson: data.registryJson,
|
|
3758
|
+
registry: data.registry,
|
|
3759
|
+
trust: data.trust
|
|
3760
|
+
};
|
|
3761
|
+
}
|
|
3762
|
+
|
|
3466
3763
|
/**
|
|
3467
3764
|
* 从 query string 构造 `BuyerSellerRoutingConfig` override(用于 GET /routing/preview)。
|
|
3468
3765
|
* 接受 `mode` / `scorer` / `sellerId` / `sellerIds`(逗号分隔)。
|