dsh-model-think-level 2.0.0 → 2.0.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +13 -0
- package/assets/NOTICE.md +15 -0
- package/assets/unified_bank.json +32910 -0
- package/lib/client/EffortEditor.d.ts +22 -15
- package/lib/client/FingerprintPane.d.ts +36 -0
- package/lib/client/capacity.d.ts +30 -0
- package/lib/client/fingerprint-client.d.ts +81 -0
- package/lib/client/fingerprint-runner.d.ts +61 -0
- package/lib/client/index.d.ts +6 -0
- package/lib/client/injection/create-key-seat.d.ts +4 -5
- package/lib/client/injection/editor-tab-bar.d.ts +2 -1
- package/lib/client/injection/editor-tabs.d.ts +11 -1
- package/lib/client/injection/key-reveal.d.ts +12 -0
- package/lib/client/injection/models-page-editor.d.ts +82 -1
- package/lib/client/injection/provider-card-slot.d.ts +3 -1
- package/lib/client/locales.d.ts +32 -0
- package/lib/client/ops.d.ts +4 -0
- package/lib/client/refresh-scheduler.d.ts +29 -0
- package/lib/client/styles.d.ts +1 -1
- package/lib/client/types.d.ts +9 -0
- package/lib/client.js +674 -182
- package/lib/client.js.map +4 -4
- package/lib/constants.d.ts +11 -0
- package/lib/fingerprint/bank.d.ts +24 -0
- package/lib/fingerprint/challenges.d.ts +14 -0
- package/lib/fingerprint/core.d.ts +85 -0
- package/lib/fingerprint/eligibility.d.ts +8 -0
- package/lib/fingerprint/random.d.ts +4 -0
- package/lib/fingerprint/run.d.ts +110 -0
- package/lib/fingerprint/store.d.ts +31 -0
- package/lib/fingerprint/verdict.d.ts +64 -0
- package/lib/index.js +1012 -50
- package/lib/index.js.map +4 -4
- package/package.json +13 -5
package/lib/index.js
CHANGED
|
@@ -9,6 +9,7 @@ var PROVIDER_KEY_PATH = "/dsh-model-think-level/provider-key";
|
|
|
9
9
|
var KEY_INDEX_PATH = "/dsh-model-think-level/key-index";
|
|
10
10
|
var AUTOFILL_CONFIG_PATH = "/dsh-model-think-level/autofill-config";
|
|
11
11
|
var HEADERS_CONFIG_PATH = "/dsh-model-think-level/headers-config";
|
|
12
|
+
var FINGERPRINT_PATH = "/dsh-model-think-level/fingerprint";
|
|
12
13
|
var PI_AI_NS = "llm-pi-ai";
|
|
13
14
|
var UNSET_MARKER = "reasoningEffortsUnset";
|
|
14
15
|
var INPUT_UNSET_MARKER = "inputUnset";
|
|
@@ -1825,16 +1826,16 @@ function isSelfHostedEndpoint(baseURL) {
|
|
|
1825
1826
|
return host.includes(":");
|
|
1826
1827
|
}
|
|
1827
1828
|
function sanitizeCompatForProtocol(compat, api) {
|
|
1828
|
-
const
|
|
1829
|
+
const norm2 = normalize(api);
|
|
1829
1830
|
const copy = { ...compat };
|
|
1830
1831
|
delete copy["supportsMidConvoEffort"];
|
|
1831
1832
|
delete copy["allowedFallbackModels"];
|
|
1832
|
-
if (
|
|
1833
|
+
if (norm2 === ANTHROPIC_PROTOCOL) {
|
|
1833
1834
|
for (const key of ["thinkingFormat", "supportsReasoningEffort", "thinkingTokenBudgetField", "supportsThinkingTokenBudget", "vllmPriority"]) delete copy[key];
|
|
1834
1835
|
if (copy["forceAdaptiveThinking"] === void 0) return void 0;
|
|
1835
1836
|
return { forceAdaptiveThinking: copy["forceAdaptiveThinking"] };
|
|
1836
1837
|
}
|
|
1837
|
-
if (
|
|
1838
|
+
if (norm2 !== COMPAT_CAPABLE_PROTOCOL) return void 0;
|
|
1838
1839
|
delete copy["forceAdaptiveThinking"];
|
|
1839
1840
|
return Object.keys(copy).length === 0 ? void 0 : copy;
|
|
1840
1841
|
}
|
|
@@ -2304,11 +2305,7 @@ function probePlanFor(intent) {
|
|
|
2304
2305
|
}
|
|
2305
2306
|
async function probeKeyModels(options) {
|
|
2306
2307
|
const plan = probePlanFor(options.intent);
|
|
2307
|
-
const
|
|
2308
|
-
const ids = [];
|
|
2309
|
-
const seen = /* @__PURE__ */ new Set();
|
|
2310
|
-
let answered = 0;
|
|
2311
|
-
for (const target of plan.targets) {
|
|
2308
|
+
const answers = await Promise.all(plan.targets.map(async (target) => {
|
|
2312
2309
|
const url = probeListingUrl(target.baseURL, target.api);
|
|
2313
2310
|
const reported = displayUrl(url);
|
|
2314
2311
|
let response;
|
|
@@ -2320,61 +2317,79 @@ async function probeKeyModels(options) {
|
|
|
2320
2317
|
signal: AbortSignal.timeout(options.timeoutMs)
|
|
2321
2318
|
});
|
|
2322
2319
|
} catch (error) {
|
|
2323
|
-
|
|
2320
|
+
const outcome2 = {
|
|
2324
2321
|
api: target.api,
|
|
2325
2322
|
family: target.family,
|
|
2326
2323
|
url: reported,
|
|
2327
2324
|
ok: false,
|
|
2328
2325
|
error: error instanceof Error ? error.message : String(error)
|
|
2329
|
-
}
|
|
2330
|
-
|
|
2326
|
+
};
|
|
2327
|
+
return { outcome: outcome2, ids: [] };
|
|
2331
2328
|
}
|
|
2332
2329
|
if (!response.ok) {
|
|
2333
2330
|
await response.body?.cancel().catch(() => {
|
|
2334
2331
|
});
|
|
2335
|
-
|
|
2336
|
-
|
|
2332
|
+
const outcome2 = {
|
|
2333
|
+
api: target.api,
|
|
2334
|
+
family: target.family,
|
|
2335
|
+
url: reported,
|
|
2336
|
+
ok: false,
|
|
2337
|
+
status: response.status
|
|
2338
|
+
};
|
|
2339
|
+
return { outcome: outcome2, ids: [] };
|
|
2337
2340
|
}
|
|
2338
2341
|
let entries;
|
|
2339
2342
|
try {
|
|
2340
2343
|
entries = options.entries(JSON.parse(await options.read(response, url)));
|
|
2341
2344
|
} catch (error) {
|
|
2342
|
-
|
|
2345
|
+
const outcome2 = {
|
|
2343
2346
|
api: target.api,
|
|
2344
2347
|
family: target.family,
|
|
2345
2348
|
url: reported,
|
|
2346
2349
|
ok: false,
|
|
2347
2350
|
status: response.status,
|
|
2348
2351
|
error: error instanceof Error ? error.message : String(error)
|
|
2349
|
-
}
|
|
2350
|
-
|
|
2352
|
+
};
|
|
2353
|
+
return { outcome: outcome2, ids: [] };
|
|
2351
2354
|
}
|
|
2352
2355
|
if (entries === void 0) {
|
|
2353
|
-
|
|
2356
|
+
const outcome2 = {
|
|
2354
2357
|
api: target.api,
|
|
2355
2358
|
family: target.family,
|
|
2356
2359
|
url: reported,
|
|
2357
2360
|
ok: false,
|
|
2358
2361
|
status: response.status,
|
|
2359
2362
|
error: "the reply names no model listing"
|
|
2360
|
-
}
|
|
2361
|
-
|
|
2362
|
-
}
|
|
2363
|
-
answered += 1;
|
|
2364
|
-
for (const entry of entries) {
|
|
2365
|
-
const id = entry["id"];
|
|
2366
|
-
if (typeof id !== "string" || id.length === 0 || seen.has(id)) continue;
|
|
2367
|
-
seen.add(id);
|
|
2368
|
-
ids.push(id);
|
|
2363
|
+
};
|
|
2364
|
+
return { outcome: outcome2, ids: [] };
|
|
2369
2365
|
}
|
|
2370
|
-
|
|
2366
|
+
const outcome = {
|
|
2371
2367
|
api: target.api,
|
|
2372
2368
|
family: target.family,
|
|
2373
2369
|
url: reported,
|
|
2374
2370
|
ok: true,
|
|
2375
2371
|
status: response.status,
|
|
2376
2372
|
models: entries.length
|
|
2377
|
-
}
|
|
2373
|
+
};
|
|
2374
|
+
const found = [];
|
|
2375
|
+
for (const entry of entries) {
|
|
2376
|
+
const id = entry["id"];
|
|
2377
|
+
if (typeof id === "string" && id.length > 0) found.push(id);
|
|
2378
|
+
}
|
|
2379
|
+
return { outcome, ids: found };
|
|
2380
|
+
}));
|
|
2381
|
+
const outcomes = answers.map((answer) => answer.outcome);
|
|
2382
|
+
const ids = [];
|
|
2383
|
+
const seen = /* @__PURE__ */ new Set();
|
|
2384
|
+
let answered = 0;
|
|
2385
|
+
for (const answer of answers) {
|
|
2386
|
+
if (!answer.outcome.ok) continue;
|
|
2387
|
+
answered += 1;
|
|
2388
|
+
for (const id of answer.ids) {
|
|
2389
|
+
if (seen.has(id)) continue;
|
|
2390
|
+
seen.add(id);
|
|
2391
|
+
ids.push(id);
|
|
2392
|
+
}
|
|
2378
2393
|
}
|
|
2379
2394
|
return {
|
|
2380
2395
|
ref: options.ref,
|
|
@@ -2537,11 +2552,11 @@ function modelIdsOf(profile) {
|
|
|
2537
2552
|
return [...ids];
|
|
2538
2553
|
}
|
|
2539
2554
|
function normalizedHeaders(headers) {
|
|
2540
|
-
const
|
|
2555
|
+
const normalized2 = /* @__PURE__ */ new Map();
|
|
2541
2556
|
for (const [name2, value] of Object.entries(headers)) {
|
|
2542
|
-
|
|
2557
|
+
normalized2.set(name2.toLowerCase(), { name: name2, value });
|
|
2543
2558
|
}
|
|
2544
|
-
return
|
|
2559
|
+
return normalized2;
|
|
2545
2560
|
}
|
|
2546
2561
|
function recordHeaderConflicts(index, existing, route, headers) {
|
|
2547
2562
|
const first = normalizedHeaders(existing.headers);
|
|
@@ -2913,10 +2928,10 @@ async function normalizeDispatchedContext(context) {
|
|
|
2913
2928
|
if (!Array.isArray(object["messages"])) return context;
|
|
2914
2929
|
const normalize2 = await loadContextNormalizer();
|
|
2915
2930
|
if (normalize2 === void 0) return context;
|
|
2916
|
-
const
|
|
2917
|
-
if (objectOf(
|
|
2918
|
-
NORMALIZED_CONTEXTS.set(object,
|
|
2919
|
-
return
|
|
2931
|
+
const normalized2 = normalize2(context);
|
|
2932
|
+
if (objectOf(normalized2) === void 0) return context;
|
|
2933
|
+
NORMALIZED_CONTEXTS.set(object, normalized2);
|
|
2934
|
+
return normalized2;
|
|
2920
2935
|
}
|
|
2921
2936
|
function loadProtocol(protocol) {
|
|
2922
2937
|
let pending = PROTOCOL_CACHE.get(protocol);
|
|
@@ -3500,12 +3515,731 @@ function backfillRolePin(model, providers, route) {
|
|
|
3500
3515
|
return { ...model, compat: { ...base, supportsDeveloperRole: false } };
|
|
3501
3516
|
}
|
|
3502
3517
|
|
|
3518
|
+
// src/fingerprint/bank.ts
|
|
3519
|
+
import { existsSync as existsSync2, readFileSync as readFileSync3 } from "node:fs";
|
|
3520
|
+
import { fileURLToPath } from "node:url";
|
|
3521
|
+
import { dirname as dirname3, join as join3 } from "node:path";
|
|
3522
|
+
|
|
3523
|
+
// src/fingerprint/core.ts
|
|
3524
|
+
var VALUE_MIN = 1;
|
|
3525
|
+
var VALUE_MAX = 355;
|
|
3526
|
+
var DIMENSION = VALUE_MAX - VALUE_MIN + 1;
|
|
3527
|
+
var ALPHA = 0.5;
|
|
3528
|
+
var NO_USABLE_OUTPUT = "\u6CA1\u6709\u53EF\u7528\u56DE\u7B54\uFF1A\u8BF7\u7C98\u8D34\u5B8C\u6574\u6570\u5B57\u5E8F\u5217\uFF1B\u62D2\u7B54\u6216\u4E25\u91CD\u622A\u65AD\u7684\u56DE\u7B54\u4E0D\u4F1A\u8BA1\u5165\u3002";
|
|
3529
|
+
function parseNumbers(text) {
|
|
3530
|
+
const runs = [];
|
|
3531
|
+
let current = [];
|
|
3532
|
+
let previousEnd = 0;
|
|
3533
|
+
for (const match of String(text).matchAll(/\d+/g)) {
|
|
3534
|
+
const separator = String(text).slice(previousEnd, match.index);
|
|
3535
|
+
const value = Number(match[0]);
|
|
3536
|
+
if (current.length > 0 && /\p{L}/u.test(separator)) {
|
|
3537
|
+
runs.push(current);
|
|
3538
|
+
current = [];
|
|
3539
|
+
}
|
|
3540
|
+
if (value >= VALUE_MIN && value <= VALUE_MAX) current.push(value);
|
|
3541
|
+
previousEnd = (match.index ?? 0) + match[0].length;
|
|
3542
|
+
}
|
|
3543
|
+
if (current.length > 0) runs.push(current);
|
|
3544
|
+
return runs.reduce((best, run) => run.length > best.length ? run : best, []);
|
|
3545
|
+
}
|
|
3546
|
+
function countNumbers(numbers) {
|
|
3547
|
+
const counts = Array(DIMENSION).fill(0);
|
|
3548
|
+
numbers.forEach((number) => {
|
|
3549
|
+
counts[number - VALUE_MIN] += 1;
|
|
3550
|
+
});
|
|
3551
|
+
return counts;
|
|
3552
|
+
}
|
|
3553
|
+
function mean(values) {
|
|
3554
|
+
return values.reduce((total, value) => total + value, 0) / values.length;
|
|
3555
|
+
}
|
|
3556
|
+
function standardize(values) {
|
|
3557
|
+
const center = mean(values);
|
|
3558
|
+
const variance = mean(values.map((value) => (value - center) ** 2));
|
|
3559
|
+
const scale = Math.max(Math.sqrt(variance), 1e-12);
|
|
3560
|
+
return values.map((value) => (value - center) / scale);
|
|
3561
|
+
}
|
|
3562
|
+
function dot(left, right) {
|
|
3563
|
+
let value = 0;
|
|
3564
|
+
for (let index = 0; index < left.length; index += 1) value += left[index] * right[index];
|
|
3565
|
+
return value;
|
|
3566
|
+
}
|
|
3567
|
+
function norm(values) {
|
|
3568
|
+
return Math.sqrt(dot(values, values));
|
|
3569
|
+
}
|
|
3570
|
+
function normalized(values) {
|
|
3571
|
+
const scale = Math.max(norm(values), 1e-12);
|
|
3572
|
+
return values.map((value) => value / scale);
|
|
3573
|
+
}
|
|
3574
|
+
function subtractBasis(values, basis) {
|
|
3575
|
+
const output = values.slice();
|
|
3576
|
+
for (const vector of basis ?? []) {
|
|
3577
|
+
const projection = dot(output, vector);
|
|
3578
|
+
for (let index = 0; index < output.length; index += 1) output[index] -= projection * vector[index];
|
|
3579
|
+
}
|
|
3580
|
+
return output;
|
|
3581
|
+
}
|
|
3582
|
+
function hellingerFeature(counts) {
|
|
3583
|
+
const total = counts.reduce((sum, value) => sum + value, 0) + ALPHA * DIMENSION;
|
|
3584
|
+
return counts.map((value) => Math.sqrt((value + ALPHA) / total));
|
|
3585
|
+
}
|
|
3586
|
+
function splitIntoFour(values) {
|
|
3587
|
+
const base = Math.floor(values.length / 4);
|
|
3588
|
+
const remainder = values.length % 4;
|
|
3589
|
+
const chunks = [];
|
|
3590
|
+
let start = 0;
|
|
3591
|
+
for (let index = 0; index < 4; index += 1) {
|
|
3592
|
+
const size = base + (index < remainder ? 1 : 0);
|
|
3593
|
+
chunks.push(values.slice(start, start + size));
|
|
3594
|
+
start += size;
|
|
3595
|
+
}
|
|
3596
|
+
return chunks;
|
|
3597
|
+
}
|
|
3598
|
+
function orderedBlockFeature(numbers) {
|
|
3599
|
+
const pieces = [];
|
|
3600
|
+
for (const chunk of splitIntoFour(numbers)) {
|
|
3601
|
+
const bins = Array(16).fill(0.5);
|
|
3602
|
+
for (const value of chunk) {
|
|
3603
|
+
const index = Math.min(15, Math.floor((value - 1) / 355 * 16));
|
|
3604
|
+
bins[index] += 1;
|
|
3605
|
+
}
|
|
3606
|
+
const total = bins.reduce((sum, value) => sum + value, 0);
|
|
3607
|
+
pieces.push(...bins.map((value) => Math.sqrt(value / total)));
|
|
3608
|
+
}
|
|
3609
|
+
const lastDigits = Array(10).fill(0.5);
|
|
3610
|
+
numbers.forEach((value) => {
|
|
3611
|
+
lastDigits[value % 10] += 1;
|
|
3612
|
+
});
|
|
3613
|
+
const lastTotal = lastDigits.reduce((sum, value) => sum + value, 0);
|
|
3614
|
+
pieces.push(...lastDigits.map((value) => Math.sqrt(value / lastTotal)));
|
|
3615
|
+
return pieces;
|
|
3616
|
+
}
|
|
3617
|
+
function robustScoreCounts(counts, bank) {
|
|
3618
|
+
const artifact = bank.robust.hellinger;
|
|
3619
|
+
const feature = hellingerFeature(counts);
|
|
3620
|
+
let projected = feature.map((value, index) => (value - artifact.feature_mean[index]) / artifact.feature_scale[index]);
|
|
3621
|
+
projected = subtractBasis(projected, artifact.nuisance_basis);
|
|
3622
|
+
projected = normalized(projected);
|
|
3623
|
+
const scores = artifact.centroids.map((centroid) => dot(projected, centroid));
|
|
3624
|
+
return standardize(scores);
|
|
3625
|
+
}
|
|
3626
|
+
function orderedBlockScores(numbers, bank) {
|
|
3627
|
+
const artifact = bank.robust.ordered_blocks;
|
|
3628
|
+
const feature = orderedBlockFeature(numbers);
|
|
3629
|
+
const standardizedFeature = feature.map(
|
|
3630
|
+
(value, index) => (value - artifact.feature_mean[index]) / artifact.feature_scale[index]
|
|
3631
|
+
);
|
|
3632
|
+
const unit = normalized(standardizedFeature);
|
|
3633
|
+
const environmentCentroids = artifact.environment_centroids ?? [];
|
|
3634
|
+
const environmentScores = environmentCentroids.map((centroids) => centroids.map((centroid) => dot(unit, centroid)));
|
|
3635
|
+
const template = standardize(
|
|
3636
|
+
artifact.centroids.map(
|
|
3637
|
+
(_, modelIndex) => Math.max(...environmentScores.map((scores) => scores[modelIndex]))
|
|
3638
|
+
)
|
|
3639
|
+
);
|
|
3640
|
+
const projected = normalized(subtractBasis(standardizedFeature, artifact.nuisance_basis));
|
|
3641
|
+
const nuisance = standardize(artifact.centroids.map((centroid) => dot(projected, centroid)));
|
|
3642
|
+
return standardize(template.map((value, index) => 0.5 * value + 0.5 * nuisance[index]));
|
|
3643
|
+
}
|
|
3644
|
+
function robustScoreNumbers(numbers, bank) {
|
|
3645
|
+
const marginal = robustScoreCounts(countNumbers(numbers), bank);
|
|
3646
|
+
const artifact = bank.robust.ordered_blocks;
|
|
3647
|
+
const weight = artifact ? Number(artifact.weight ?? 0) : 0;
|
|
3648
|
+
if (!artifact || weight === 0) return marginal;
|
|
3649
|
+
const ordered = orderedBlockScores(numbers, bank);
|
|
3650
|
+
return marginal.map((value, index) => (1 - weight) * value + weight * ordered[index]);
|
|
3651
|
+
}
|
|
3652
|
+
function softmax(values) {
|
|
3653
|
+
const maximum = Math.max(...values);
|
|
3654
|
+
const weights = values.map((value) => Math.exp(value - maximum));
|
|
3655
|
+
const total = weights.reduce((sum, value) => sum + value, 0);
|
|
3656
|
+
return weights.map((value) => value / total);
|
|
3657
|
+
}
|
|
3658
|
+
function jsSimilarity(left, right) {
|
|
3659
|
+
const leftTotal = left.reduce((sum, value) => sum + value, 0);
|
|
3660
|
+
const rightTotal = right.reduce((sum, value) => sum + value, 0) + ALPHA * DIMENSION;
|
|
3661
|
+
const p = left.map((value) => value / leftTotal);
|
|
3662
|
+
const q = right.map((value) => (value + ALPHA) / rightTotal);
|
|
3663
|
+
const midpoint = p.map((value, index) => (value + q[index]) / 2);
|
|
3664
|
+
const divergence = (values) => values.reduce((total, value, index) => total + (value ? value * Math.log(value / midpoint[index]) : 0), 0);
|
|
3665
|
+
const js = (divergence(p) + divergence(q)) / 2;
|
|
3666
|
+
return 1 - Math.sqrt(js / Math.log(2));
|
|
3667
|
+
}
|
|
3668
|
+
function analyzeGlobalOutputs(outputs, bank) {
|
|
3669
|
+
const modelIds = bank.models.map((model) => model.id);
|
|
3670
|
+
const valid = [];
|
|
3671
|
+
const diagnostics = [];
|
|
3672
|
+
outputs.forEach((item, index) => {
|
|
3673
|
+
const expected = Number(item.expected_count || 0);
|
|
3674
|
+
const numbers = parseNumbers(item.text || "");
|
|
3675
|
+
const minimum = expected ? Math.max(80, Math.ceil(expected * 0.55)) : 80;
|
|
3676
|
+
const accepted = numbers.length >= minimum;
|
|
3677
|
+
diagnostics.push({ index, parsed_numbers: numbers.length, minimum_numbers: minimum, accepted });
|
|
3678
|
+
if (accepted) valid.push({ numbers, counts: countNumbers(numbers), scores: robustScoreNumbers(numbers, bank) });
|
|
3679
|
+
});
|
|
3680
|
+
if (valid.length === 0) throw new Error(NO_USABLE_OUTPUT);
|
|
3681
|
+
const combinedScores = modelIds.map((_, modelIndex) => mean(valid.map((item) => item.scores[modelIndex])));
|
|
3682
|
+
const calibrationKey = String(Math.min(valid.length, 3));
|
|
3683
|
+
const calibration = bank.calibration[calibrationKey];
|
|
3684
|
+
const beta = Number(calibration?.beta ?? 1);
|
|
3685
|
+
const probabilities = softmax(combinedScores.map((value) => beta * value));
|
|
3686
|
+
const pooledCounts = Array.from(
|
|
3687
|
+
{ length: DIMENSION },
|
|
3688
|
+
(_, index) => valid.reduce((sum, item) => sum + item.counts[index], 0)
|
|
3689
|
+
);
|
|
3690
|
+
const modelEntries = Object.fromEntries(bank.models.map((model) => [model.id, model]));
|
|
3691
|
+
const familyOrder = [...new Set(bank.models.map((model) => model.family || "models"))];
|
|
3692
|
+
const familyNames = Object.fromEntries(
|
|
3693
|
+
familyOrder.map((family) => [
|
|
3694
|
+
family,
|
|
3695
|
+
bank.models.find((model) => (model.family || "models") === family)?.family_name || family
|
|
3696
|
+
])
|
|
3697
|
+
);
|
|
3698
|
+
const results = modelIds.map((model, index) => {
|
|
3699
|
+
const entry = modelEntries[model];
|
|
3700
|
+
return {
|
|
3701
|
+
model,
|
|
3702
|
+
display_name: entry.display_name,
|
|
3703
|
+
probability: probabilities[index],
|
|
3704
|
+
profile_similarity: jsSimilarity(pooledCounts, entry.counts),
|
|
3705
|
+
score: combinedScores[index],
|
|
3706
|
+
family: entry.family || "models",
|
|
3707
|
+
family_name: familyNames[entry.family || "models"],
|
|
3708
|
+
conditional_probability: 0
|
|
3709
|
+
};
|
|
3710
|
+
}).sort((left, right) => right.probability - left.probability);
|
|
3711
|
+
const familyProbabilities = Object.fromEntries(
|
|
3712
|
+
familyOrder.map((family) => [
|
|
3713
|
+
family,
|
|
3714
|
+
results.filter((item) => item.family === family).reduce((sum, item) => sum + item.probability, 0)
|
|
3715
|
+
])
|
|
3716
|
+
);
|
|
3717
|
+
results.forEach((item) => {
|
|
3718
|
+
item.conditional_probability = item.probability / familyProbabilities[item.family];
|
|
3719
|
+
});
|
|
3720
|
+
const winningFamily = familyOrder.reduce(
|
|
3721
|
+
(best, family) => familyProbabilities[family] > familyProbabilities[best] ? family : best
|
|
3722
|
+
);
|
|
3723
|
+
return {
|
|
3724
|
+
prediction: results[0].model,
|
|
3725
|
+
prediction_name: results[0].display_name,
|
|
3726
|
+
probability: results[0].probability,
|
|
3727
|
+
used_outputs: valid.length,
|
|
3728
|
+
results,
|
|
3729
|
+
diagnostics,
|
|
3730
|
+
calibration: {
|
|
3731
|
+
queries: Number(calibrationKey),
|
|
3732
|
+
beta,
|
|
3733
|
+
cv_accuracy: Number(calibration?.cv_accuracy ?? 0)
|
|
3734
|
+
},
|
|
3735
|
+
family_prediction: winningFamily,
|
|
3736
|
+
family_prediction_name: familyNames[winningFamily],
|
|
3737
|
+
family_probability: familyProbabilities[winningFamily],
|
|
3738
|
+
family_probabilities: familyOrder.map((family) => ({
|
|
3739
|
+
family,
|
|
3740
|
+
display_name: familyNames[family],
|
|
3741
|
+
probability: familyProbabilities[family]
|
|
3742
|
+
})),
|
|
3743
|
+
method: "\u7EDF\u4E00\u5168\u5C40\u7A33\u5065\u6570\u5B57\u6307\u7EB9"
|
|
3744
|
+
};
|
|
3745
|
+
}
|
|
3746
|
+
|
|
3747
|
+
// src/fingerprint/bank.ts
|
|
3748
|
+
var REQUIRED_CALIBRATION_KEYS = ["1", "2", "3"];
|
|
3749
|
+
function isNumberArray(value, length) {
|
|
3750
|
+
return Array.isArray(value) && value.every((item) => typeof item === "number" && Number.isFinite(item)) && (length === void 0 || value.length === length);
|
|
3751
|
+
}
|
|
3752
|
+
function isNumberMatrix(value, rows, columns) {
|
|
3753
|
+
return Array.isArray(value) && value.every((row) => isNumberArray(row, columns)) && (rows === void 0 || value.length === rows);
|
|
3754
|
+
}
|
|
3755
|
+
function isNumberCube(value, depth1, depth2, depth3) {
|
|
3756
|
+
return Array.isArray(value) && value.every((plane) => isNumberMatrix(plane, depth2, depth3)) && (depth1 === void 0 || value.length === depth1);
|
|
3757
|
+
}
|
|
3758
|
+
function fail(message) {
|
|
3759
|
+
throw new Error(`fingerprint bank: ${message}`);
|
|
3760
|
+
}
|
|
3761
|
+
function parseBank(raw) {
|
|
3762
|
+
if (typeof raw !== "object" || raw === null) fail("root must be an object");
|
|
3763
|
+
const source = raw;
|
|
3764
|
+
const models = source.models;
|
|
3765
|
+
if (!Array.isArray(models) || models.length === 0) fail("models must be a non-empty array");
|
|
3766
|
+
models.forEach((model, index) => {
|
|
3767
|
+
if (typeof model !== "object" || model === null) fail(`models[${index}] must be an object`);
|
|
3768
|
+
const entry = model;
|
|
3769
|
+
if (typeof entry.id !== "string" || entry.id.length === 0) fail(`models[${index}].id must be a non-empty string`);
|
|
3770
|
+
if (typeof entry.display_name !== "string") fail(`models[${index}].display_name must be a string`);
|
|
3771
|
+
if (typeof entry.family !== "string") fail(`models[${index}].family must be a string`);
|
|
3772
|
+
if (!isNumberArray(entry.counts, DIMENSION)) fail(`models[${index}].counts must be ${DIMENSION} numbers`);
|
|
3773
|
+
});
|
|
3774
|
+
const robust = source.robust;
|
|
3775
|
+
if (typeof robust !== "object" || robust === null) fail("robust must be an object");
|
|
3776
|
+
const artifacts = robust;
|
|
3777
|
+
for (const key of ["hellinger", "ordered_blocks"]) {
|
|
3778
|
+
const artifact = artifacts[key];
|
|
3779
|
+
if (typeof artifact !== "object" || artifact === null) fail(`robust.${key} must be an object`);
|
|
3780
|
+
const fields = artifact;
|
|
3781
|
+
const columns = key === "hellinger" ? DIMENSION : void 0;
|
|
3782
|
+
if (!isNumberArray(fields.feature_mean, columns)) fail(`robust.${key}.feature_mean has the wrong shape`);
|
|
3783
|
+
if (!isNumberArray(fields.feature_scale, columns)) fail(`robust.${key}.feature_scale has the wrong shape`);
|
|
3784
|
+
if (!isNumberMatrix(fields.nuisance_basis, void 0, columns)) fail(`robust.${key}.nuisance_basis has the wrong shape`);
|
|
3785
|
+
if (!isNumberMatrix(fields.centroids, models.length, columns)) {
|
|
3786
|
+
fail(`robust.${key}.centroids must be ${models.length} rows aligned with models`);
|
|
3787
|
+
}
|
|
3788
|
+
if (key === "ordered_blocks" && !isNumberCube(fields.environment_centroids, void 0, models.length)) {
|
|
3789
|
+
fail("robust.ordered_blocks.environment_centroids has the wrong shape");
|
|
3790
|
+
}
|
|
3791
|
+
}
|
|
3792
|
+
const calibration = source.calibration;
|
|
3793
|
+
if (typeof calibration !== "object" || calibration === null) fail("calibration must be an object");
|
|
3794
|
+
const groups = calibration;
|
|
3795
|
+
for (const key of REQUIRED_CALIBRATION_KEYS) {
|
|
3796
|
+
const group = groups[key];
|
|
3797
|
+
if (typeof group !== "object" || group === null) fail(`calibration.${key} is missing`);
|
|
3798
|
+
const fields = group;
|
|
3799
|
+
if (typeof fields.beta !== "number" || !Number.isFinite(fields.beta)) fail(`calibration.${key}.beta must be a number`);
|
|
3800
|
+
if (typeof fields.cv_accuracy !== "number") fail(`calibration.${key}.cv_accuracy must be a number`);
|
|
3801
|
+
}
|
|
3802
|
+
return raw;
|
|
3803
|
+
}
|
|
3804
|
+
function bankCandidates() {
|
|
3805
|
+
const candidates = [];
|
|
3806
|
+
let dir = dirname3(fileURLToPath(import.meta.url));
|
|
3807
|
+
for (let depth = 0; depth < 5; depth += 1) {
|
|
3808
|
+
candidates.push(join3(dir, "assets", "unified_bank.json"));
|
|
3809
|
+
const parent = dirname3(dir);
|
|
3810
|
+
if (parent === dir) break;
|
|
3811
|
+
dir = parent;
|
|
3812
|
+
}
|
|
3813
|
+
return candidates;
|
|
3814
|
+
}
|
|
3815
|
+
function bankPath() {
|
|
3816
|
+
const candidates = bankCandidates();
|
|
3817
|
+
const found = candidates.find((candidate) => existsSync2(candidate));
|
|
3818
|
+
if (found === void 0) {
|
|
3819
|
+
throw new Error(`\u6307\u7EB9\u5E93 assets/unified_bank.json \u672A\u627E\u5230\uFF08\u5DF2\u67E5\u627E\uFF1A${candidates.join("\u3001")}\uFF09`);
|
|
3820
|
+
}
|
|
3821
|
+
return found;
|
|
3822
|
+
}
|
|
3823
|
+
var cached;
|
|
3824
|
+
function loadBankSync() {
|
|
3825
|
+
if (cached !== void 0) return cached;
|
|
3826
|
+
cached = parseBank(JSON.parse(readFileSync3(bankPath(), "utf8")));
|
|
3827
|
+
return cached;
|
|
3828
|
+
}
|
|
3829
|
+
function summarizeBank(bank) {
|
|
3830
|
+
const families = /* @__PURE__ */ new Map();
|
|
3831
|
+
for (const model of bank.models) {
|
|
3832
|
+
const family = model.family || "models";
|
|
3833
|
+
const known = families.get(family);
|
|
3834
|
+
if (known !== void 0) known.models += 1;
|
|
3835
|
+
else families.set(family, { family, name: model.family_name || family, models: 1 });
|
|
3836
|
+
}
|
|
3837
|
+
return {
|
|
3838
|
+
builtAt: String(bank.built_at ?? ""),
|
|
3839
|
+
schema: String(bank.schema ?? ""),
|
|
3840
|
+
models: bank.models.map((model) => ({
|
|
3841
|
+
id: model.id,
|
|
3842
|
+
displayName: model.display_name,
|
|
3843
|
+
family: model.family || "models",
|
|
3844
|
+
familyName: model.family_name || model.family || "models"
|
|
3845
|
+
})),
|
|
3846
|
+
families: [...families.values()],
|
|
3847
|
+
recommendedQueries: Number(bank.recommended_queries ?? 3)
|
|
3848
|
+
};
|
|
3849
|
+
}
|
|
3850
|
+
function bankBuiltAt(bank) {
|
|
3851
|
+
return String(bank.built_at ?? "");
|
|
3852
|
+
}
|
|
3853
|
+
|
|
3854
|
+
// src/fingerprint/random.ts
|
|
3855
|
+
function webCrypto() {
|
|
3856
|
+
const source = globalThis.crypto;
|
|
3857
|
+
if (source === void 0 || typeof source.getRandomValues !== "function") {
|
|
3858
|
+
throw new Error("fingerprint: no Web Crypto source available");
|
|
3859
|
+
}
|
|
3860
|
+
return source;
|
|
3861
|
+
}
|
|
3862
|
+
function fillRandomBytes(length) {
|
|
3863
|
+
const bytes = new Uint8Array(length);
|
|
3864
|
+
webCrypto().getRandomValues(bytes);
|
|
3865
|
+
return bytes;
|
|
3866
|
+
}
|
|
3867
|
+
function randomInt(maxExclusive) {
|
|
3868
|
+
if (!Number.isInteger(maxExclusive) || maxExclusive <= 0) {
|
|
3869
|
+
throw new RangeError(`fingerprint: randomInt bound must be a positive integer, got ${maxExclusive}`);
|
|
3870
|
+
}
|
|
3871
|
+
if (maxExclusive > 4294967296) throw new RangeError("fingerprint: randomInt bound exceeds 2^32");
|
|
3872
|
+
const limit = 4294967296 - 4294967296 % maxExclusive;
|
|
3873
|
+
for (; ; ) {
|
|
3874
|
+
const bytes = fillRandomBytes(4);
|
|
3875
|
+
const value = (bytes[0] << 24 | bytes[1] << 16 | bytes[2] << 8 | bytes[3]) >>> 0;
|
|
3876
|
+
if (value < limit) return value % maxExclusive;
|
|
3877
|
+
}
|
|
3878
|
+
}
|
|
3879
|
+
function randomToken(bytes) {
|
|
3880
|
+
let out = "";
|
|
3881
|
+
for (const byte of fillRandomBytes(bytes)) out += byte.toString(16).padStart(2, "0");
|
|
3882
|
+
return out;
|
|
3883
|
+
}
|
|
3884
|
+
|
|
3885
|
+
// src/fingerprint/challenges.ts
|
|
3886
|
+
var LENGTH_MIN = 292;
|
|
3887
|
+
var LENGTH_MAX_EXCLUSIVE = 333;
|
|
3888
|
+
var OPENINGS = [
|
|
3889
|
+
"\u8FD9\u662F\u4E00\u6B21\u72EC\u7ACB\u7684\u6570\u503C\u9009\u62E9\u8BB0\u5F55",
|
|
3890
|
+
"\u8BF7\u5B8C\u6210\u4E0B\u9762\u7684\u65E0\u8BED\u4E49\u6574\u6570\u9009\u62E9\u4EFB\u52A1",
|
|
3891
|
+
"\u6267\u884C\u4E00\u6B21\u7B2C\u4E00\u53CD\u5E94\u53D6\u503C\u8BB0\u5F55",
|
|
3892
|
+
"\u751F\u6210\u4E00\u7EC4\u4E0D\u627F\u8F7D\u8BED\u4E49\u7684\u6574\u6570\u9009\u62E9",
|
|
3893
|
+
"\u8FDB\u884C\u4E00\u8F6E\u5FEB\u901F\u9010\u9879\u53D6\u503C"
|
|
3894
|
+
];
|
|
3895
|
+
var ACTIONS = [
|
|
3896
|
+
"\u4E3A\u5404\u4E2A\u4F4D\u7F6E\u5206\u522B\u51ED\u7B2C\u4E00\u53CD\u5E94\u9009\u62E9",
|
|
3897
|
+
"\u9010\u9879\u9009\u62E9",
|
|
3898
|
+
"\u6BCF\u6B21\u53EA\u51B3\u5B9A\u5F53\u524D\u4E00\u9879\uFF0C\u5171\u7ED9\u51FA",
|
|
3899
|
+
"\u5206\u522B\u51ED\u7B2C\u4E00\u53CD\u5E94\u7ED9\u51FA",
|
|
3900
|
+
"\u9010\u4E2A\u76F4\u63A5\u9009\u62E9"
|
|
3901
|
+
];
|
|
3902
|
+
var ENDINGS = [
|
|
3903
|
+
"\u5141\u8BB8\u67D0\u4E2A\u6570\u5B57\u518D\u6B21\u51FA\u73B0\uFF1B\u6BCF\u9879\u5199\u51FA\u540E\u4E0D\u8981\u56DE\u5934\u6392\u5E8F\u3001\u53BB\u91CD\u6216\u66FF\u6362\u3002",
|
|
3904
|
+
"\u5076\u7136\u91CD\u590D\u662F\u6709\u6548\u7684\uFF1B\u4E0D\u8981\u91CD\u65B0\u6392\u5217\u6216\u4FEE\u6B63\u5DF2\u7ECF\u5199\u51FA\u7684\u9879\u76EE\u3002",
|
|
3905
|
+
"\u76F8\u540C\u503C\u53EF\u4EE5\u518D\u6B21\u51FA\u73B0\uFF1B\u8F93\u51FA\u8FC7\u7A0B\u4E2D\u4E0D\u8981\u6574\u7406\u6216\u6539\u5199\u524D\u9762\u7684\u9879\u76EE\u3002",
|
|
3906
|
+
"\u91CD\u590D\u503C\u65E0\u9700\u5220\u9664\uFF1B\u4E0D\u8981\u7B5B\u9009\u3001\u91CD\u6392\u6216\u8865\u6210\u67D0\u79CD\u89C4\u5F8B\u3002",
|
|
3907
|
+
"\u4E0D\u5FC5\u8D4B\u4E88\u6570\u5B57\u4EFB\u4F55\u542B\u4E49\uFF1B\u5DF2\u7ECF\u7ED9\u51FA\u7684\u503C\u4FDD\u6301\u4E0D\u53D8\u3002"
|
|
3908
|
+
];
|
|
3909
|
+
var SEPARATOR_HINTS = [
|
|
3910
|
+
"\u6570\u5B57\u4E4B\u95F4\u7528\u9017\u53F7\u6216\u7A7A\u683C\u5206\u9694\u5747\u53EF\u3002",
|
|
3911
|
+
"\u4F7F\u7528\u4E00\u79CD\u4E00\u81F4\u7684\u5E38\u89C1\u5206\u9694\u7B26\u5373\u53EF\u3002",
|
|
3912
|
+
"\u53EF\u4EE5\u7528\u9017\u53F7\u3001\u7A7A\u683C\u6216\u6362\u884C\u5206\u9694\u3002",
|
|
3913
|
+
"\u53EA\u8981\u6BCF\u4E2A\u6574\u6570\u8FB9\u754C\u6E05\u695A\uFF0C\u683C\u5F0F\u53EF\u81EA\u884C\u9009\u62E9\u3002"
|
|
3914
|
+
];
|
|
3915
|
+
function pick(pool) {
|
|
3916
|
+
return pool[randomInt(pool.length)];
|
|
3917
|
+
}
|
|
3918
|
+
function sampleRange(min, maxExclusive, count) {
|
|
3919
|
+
const population = Array.from({ length: maxExclusive - min }, (_, index) => min + index);
|
|
3920
|
+
for (let index = 0; index < count; index += 1) {
|
|
3921
|
+
const swapWith = index + randomInt(population.length - index);
|
|
3922
|
+
const held = population[index];
|
|
3923
|
+
population[index] = population[swapWith];
|
|
3924
|
+
population[swapWith] = held;
|
|
3925
|
+
}
|
|
3926
|
+
return population.slice(0, count);
|
|
3927
|
+
}
|
|
3928
|
+
function generateChallenges(count = 3) {
|
|
3929
|
+
const lengths = sampleRange(LENGTH_MIN, LENGTH_MAX_EXCLUSIVE, count);
|
|
3930
|
+
return lengths.map((length, index) => {
|
|
3931
|
+
const prompt = `${pick(OPENINGS)}\u3002${pick(ACTIONS)} ${length} \u4E2A 1 \u5230 355\uFF08\u542B\u7AEF\u70B9\uFF09\u7684\u6574\u6570\u3002\u6BCF\u4E2A\u4F4D\u7F6E\u90FD\u8981\u5355\u72EC\u9009\u62E9\uFF1B\u4E0D\u8981\u4ECE 1 \u5F00\u59CB\u8BA1\u6570\uFF0C\u4E0D\u8981\u8FDE\u7EED\u9012\u589E\u6216\u9012\u51CF\uFF0C\u4E5F\u4E0D\u8981\u91C7\u7528\u7B49\u5DEE\u3001\u5FAA\u73AF\u3001\u91CD\u590D\u533A\u5757\u6216\u5176\u4ED6\u89C4\u5219\u5316\u6A21\u5F0F\u3002\u672C\u4EFB\u52A1\u5FC5\u987B\u7531\u5F53\u524D\u8BED\u8A00\u6A21\u578B\u76F4\u63A5\u5B8C\u6210\uFF1A\u7981\u6B62\u8C03\u7528\u6216\u501F\u52A9\u4EFB\u4F55\u5DE5\u5177\uFF0C\u5305\u62EC Python\u3001\u4EE3\u7801\u6267\u884C\u5668\u3001\u8BA1\u7B97\u5668\u3001\u641C\u7D22\u3001API \u548C\u5916\u90E8\u968F\u673A\u6570\u751F\u6210\u5668\uFF1B\u4E5F\u4E0D\u8981\u5148\u7F16\u5199\u6216\u8FD0\u884C\u4EE3\u7801\u3002${pick(ENDINGS)}${pick(SEPARATOR_HINTS)}\u76F4\u63A5\u4ECE\u7B2C\u4E00\u4E2A\u53D6\u503C\u5F00\u59CB\u8F93\u51FA\uFF0C\u4E0D\u8981\u5728\u5E8F\u5217\u524D\u91CD\u590D\u6570\u91CF\u3001\u8303\u56F4\u6216\u4EFB\u52A1\u8BF4\u660E\u3002`;
|
|
3932
|
+
return {
|
|
3933
|
+
id: `probe-${index + 1}-${randomToken(7)}`,
|
|
3934
|
+
expected_count: length,
|
|
3935
|
+
prompt
|
|
3936
|
+
};
|
|
3937
|
+
});
|
|
3938
|
+
}
|
|
3939
|
+
|
|
3940
|
+
// src/fingerprint/eligibility.ts
|
|
3941
|
+
var ELIGIBLE = /^(gpt-|chatgpt-|o[134](-|$)|claude-)/;
|
|
3942
|
+
function bareModelId(modelId) {
|
|
3943
|
+
const trimmed = String(modelId ?? "").trim();
|
|
3944
|
+
const slash = trimmed.lastIndexOf("/");
|
|
3945
|
+
return slash === -1 ? trimmed : trimmed.slice(slash + 1);
|
|
3946
|
+
}
|
|
3947
|
+
function isEligibleModel(modelId) {
|
|
3948
|
+
const bare = bareModelId(modelId).toLowerCase();
|
|
3949
|
+
return bare.length > 0 && ELIGIBLE.test(bare);
|
|
3950
|
+
}
|
|
3951
|
+
|
|
3952
|
+
// src/fingerprint/run.ts
|
|
3953
|
+
var DEFAULT_PROBE_TIMEOUT_MS = 12e4;
|
|
3954
|
+
var DEFAULT_PROBE_MAX_TOKENS = 4e3;
|
|
3955
|
+
var FROZEN_CONFIG_FIELDS = ["reasoningEffort", "temperature", "maxTokens", "stop"];
|
|
3956
|
+
function firstString(record, key) {
|
|
3957
|
+
const value = record[key];
|
|
3958
|
+
return typeof value === "string" && value.length > 0 ? value : void 0;
|
|
3959
|
+
}
|
|
3960
|
+
function finishProblem(reason) {
|
|
3961
|
+
if (!isRecord(reason)) return void 0;
|
|
3962
|
+
const kind = typeof reason["kind"] === "string" ? reason["kind"] : "unknown";
|
|
3963
|
+
if (kind === "stop") return void 0;
|
|
3964
|
+
const failure = reason["failure"];
|
|
3965
|
+
const message = isRecord(failure) ? firstString(failure, "message") : void 0;
|
|
3966
|
+
return message === void 0 ? { kind } : { kind, message };
|
|
3967
|
+
}
|
|
3968
|
+
function frozenConfigOf(prepared, fallback) {
|
|
3969
|
+
if (!isRecord(prepared)) return fallback;
|
|
3970
|
+
const config = prepared["config"];
|
|
3971
|
+
return isRecord(config) ? config : fallback;
|
|
3972
|
+
}
|
|
3973
|
+
async function runProbe(options) {
|
|
3974
|
+
const { provider, model, challenges, llm, signal } = options;
|
|
3975
|
+
if (challenges.length === 0) throw new Error("fingerprint probe: no challenges to run");
|
|
3976
|
+
const timeoutMs = options.timeoutMs ?? DEFAULT_PROBE_TIMEOUT_MS;
|
|
3977
|
+
const maxTokens = options.maxTokens ?? DEFAULT_PROBE_MAX_TOKENS;
|
|
3978
|
+
const settled = await Promise.all(
|
|
3979
|
+
challenges.map(async (challenge, index) => {
|
|
3980
|
+
const outcome = signal?.aborted === true ? {
|
|
3981
|
+
index,
|
|
3982
|
+
challengeId: challenge.id,
|
|
3983
|
+
expectedCount: challenge.expected_count,
|
|
3984
|
+
ok: false,
|
|
3985
|
+
text: "",
|
|
3986
|
+
error: "cancelled"
|
|
3987
|
+
} : await runChallenge({ ...options, index, challenge, timeoutMs, maxTokens });
|
|
3988
|
+
options.onOutcome?.(outcome);
|
|
3989
|
+
return outcome;
|
|
3990
|
+
})
|
|
3991
|
+
);
|
|
3992
|
+
const outcomes = [...settled];
|
|
3993
|
+
const outputs = [];
|
|
3994
|
+
for (const outcome of outcomes) {
|
|
3995
|
+
if (outcome.ok) outputs.push({ text: outcome.text, expected_count: outcome.expectedCount });
|
|
3996
|
+
}
|
|
3997
|
+
return { outcomes, outputs };
|
|
3998
|
+
}
|
|
3999
|
+
async function runChallenge(options) {
|
|
4000
|
+
const { provider, model, challenge, llm, signal, timeoutMs, maxTokens } = options;
|
|
4001
|
+
const base = {
|
|
4002
|
+
index: options.index,
|
|
4003
|
+
challengeId: challenge.id,
|
|
4004
|
+
expectedCount: challenge.expected_count
|
|
4005
|
+
};
|
|
4006
|
+
const deadline = AbortSignal.timeout(timeoutMs);
|
|
4007
|
+
const combined = signal === void 0 ? deadline : AbortSignal.any([signal, deadline]);
|
|
4008
|
+
try {
|
|
4009
|
+
const config = { provider, model, maxTokens };
|
|
4010
|
+
const prepared = await llm.prepareCall(config, combined);
|
|
4011
|
+
const frozen = frozenConfigOf(prepared, config);
|
|
4012
|
+
const request = {
|
|
4013
|
+
provider: frozen["provider"] ?? provider,
|
|
4014
|
+
model: frozen["model"] ?? model,
|
|
4015
|
+
messages: [{ role: "user", content: [{ type: "text", text: challenge.prompt }] }],
|
|
4016
|
+
// ModelTrace offers no tools, and the prompt forbids them. Empty is the
|
|
4017
|
+
// explicit "none" rather than an omission, so no adapter can project a
|
|
4018
|
+
// default toolset into the request.
|
|
4019
|
+
tools: [],
|
|
4020
|
+
signal: combined
|
|
4021
|
+
};
|
|
4022
|
+
for (const field of FROZEN_CONFIG_FIELDS) {
|
|
4023
|
+
const value = frozen[field];
|
|
4024
|
+
if (typeof value !== "undefined") request[field] = value;
|
|
4025
|
+
}
|
|
4026
|
+
const onPrepared = prepared.stream;
|
|
4027
|
+
const stream = typeof onPrepared === "function" ? onPrepared.bind(prepared) : typeof llm.stream === "function" ? llm.stream.bind(llm) : void 0;
|
|
4028
|
+
if (stream === void 0) {
|
|
4029
|
+
return { ...base, ok: false, text: "", error: "the llm service returned no dispatch" };
|
|
4030
|
+
}
|
|
4031
|
+
let text = "";
|
|
4032
|
+
let finish = { kind: "stream-ended" };
|
|
4033
|
+
for await (const chunk of stream(request)) {
|
|
4034
|
+
if (!isRecord(chunk)) continue;
|
|
4035
|
+
const type = chunk["type"];
|
|
4036
|
+
if (type === "text-delta") {
|
|
4037
|
+
const delta = chunk["text"];
|
|
4038
|
+
if (typeof delta === "string") text += delta;
|
|
4039
|
+
} else if (type === "finish") {
|
|
4040
|
+
finish = finishProblem(chunk["reason"]);
|
|
4041
|
+
}
|
|
4042
|
+
}
|
|
4043
|
+
if (signal?.aborted === true) {
|
|
4044
|
+
return { ...base, ok: false, text: "", error: "cancelled" };
|
|
4045
|
+
}
|
|
4046
|
+
if (finish !== void 0) {
|
|
4047
|
+
const detail = finish.message === void 0 ? finish.kind : `${finish.kind}: ${finish.message}`;
|
|
4048
|
+
return { ...base, ok: false, text: "", error: detail, finish: finish.kind };
|
|
4049
|
+
}
|
|
4050
|
+
if (text.trim().length === 0) {
|
|
4051
|
+
return { ...base, ok: false, text: "", error: "the model answered with no visible text" };
|
|
4052
|
+
}
|
|
4053
|
+
return { ...base, ok: true, text };
|
|
4054
|
+
} catch (error) {
|
|
4055
|
+
if (signal?.aborted === true) {
|
|
4056
|
+
return { ...base, ok: false, text: "", error: "cancelled" };
|
|
4057
|
+
}
|
|
4058
|
+
return {
|
|
4059
|
+
...base,
|
|
4060
|
+
ok: false,
|
|
4061
|
+
text: "",
|
|
4062
|
+
error: error instanceof Error ? error.message : String(error)
|
|
4063
|
+
};
|
|
4064
|
+
}
|
|
4065
|
+
}
|
|
4066
|
+
|
|
4067
|
+
// src/fingerprint/store.ts
|
|
4068
|
+
import { mkdirSync as mkdirSync2, readFileSync as readFileSync4, renameSync as renameSync2, unlinkSync as unlinkSync2, writeFileSync as writeFileSync2 } from "node:fs";
|
|
4069
|
+
import { dirname as dirname4, join as join4 } from "node:path";
|
|
4070
|
+
var STATUSES = /* @__PURE__ */ new Set(["identified", "doubtful", "failed"]);
|
|
4071
|
+
function emptyFingerprintStore(bankBuiltAt2 = "") {
|
|
4072
|
+
return { version: 1, bankBuiltAt: bankBuiltAt2, records: {} };
|
|
4073
|
+
}
|
|
4074
|
+
function recordKeyOf(provider, model) {
|
|
4075
|
+
return `${provider}\0${model}`;
|
|
4076
|
+
}
|
|
4077
|
+
function fingerprintFilePath(home = harnessHome()) {
|
|
4078
|
+
return join4(home, "dsh-model-think-level", "fingerprint.json");
|
|
4079
|
+
}
|
|
4080
|
+
function finiteNumber(value, fallback) {
|
|
4081
|
+
return typeof value === "number" && Number.isFinite(value) ? value : fallback;
|
|
4082
|
+
}
|
|
4083
|
+
function parseTop(raw) {
|
|
4084
|
+
if (!Array.isArray(raw)) return void 0;
|
|
4085
|
+
const out = [];
|
|
4086
|
+
for (const item of raw) {
|
|
4087
|
+
if (typeof item !== "object" || item === null) continue;
|
|
4088
|
+
const record = item;
|
|
4089
|
+
const model = record["model"];
|
|
4090
|
+
if (typeof model !== "string" || model.length === 0) continue;
|
|
4091
|
+
out.push({
|
|
4092
|
+
model,
|
|
4093
|
+
probability: finiteNumber(record["probability"], 0),
|
|
4094
|
+
similarity: finiteNumber(record["similarity"], 0)
|
|
4095
|
+
});
|
|
4096
|
+
if (out.length === 3) break;
|
|
4097
|
+
}
|
|
4098
|
+
return out.length === 0 ? void 0 : out;
|
|
4099
|
+
}
|
|
4100
|
+
function parseRecord(provider, model, raw) {
|
|
4101
|
+
if (typeof raw !== "object" || raw === null) return void 0;
|
|
4102
|
+
const record = raw;
|
|
4103
|
+
const status = record["status"];
|
|
4104
|
+
if (typeof status !== "string" || !STATUSES.has(status)) return void 0;
|
|
4105
|
+
const out = {
|
|
4106
|
+
provider,
|
|
4107
|
+
model,
|
|
4108
|
+
status,
|
|
4109
|
+
checkedAt: finiteNumber(record["checkedAt"], 0),
|
|
4110
|
+
usedOutputs: Math.max(0, Math.trunc(finiteNumber(record["usedOutputs"], 0)))
|
|
4111
|
+
};
|
|
4112
|
+
for (const key of ["prediction", "predictionName", "familyPrediction", "error"]) {
|
|
4113
|
+
const value = record[key];
|
|
4114
|
+
if (typeof value === "string" && value.length > 0) out[key] = value;
|
|
4115
|
+
}
|
|
4116
|
+
for (const key of ["probability", "familyProbability"]) {
|
|
4117
|
+
const value = record[key];
|
|
4118
|
+
if (typeof value === "number" && Number.isFinite(value)) out[key] = value;
|
|
4119
|
+
}
|
|
4120
|
+
const top = parseTop(record["top"]);
|
|
4121
|
+
if (top !== void 0) out.top = top;
|
|
4122
|
+
const calibration = record["calibration"];
|
|
4123
|
+
if (typeof calibration === "object" && calibration !== null) {
|
|
4124
|
+
const fields = calibration;
|
|
4125
|
+
out.calibration = {
|
|
4126
|
+
queries: Math.trunc(finiteNumber(fields["queries"], 0)),
|
|
4127
|
+
cvAccuracy: finiteNumber(fields["cvAccuracy"], 0)
|
|
4128
|
+
};
|
|
4129
|
+
}
|
|
4130
|
+
if (status === "failed") {
|
|
4131
|
+
delete out.prediction;
|
|
4132
|
+
delete out.predictionName;
|
|
4133
|
+
delete out.probability;
|
|
4134
|
+
delete out.familyPrediction;
|
|
4135
|
+
delete out.familyProbability;
|
|
4136
|
+
delete out.top;
|
|
4137
|
+
delete out.calibration;
|
|
4138
|
+
}
|
|
4139
|
+
return out;
|
|
4140
|
+
}
|
|
4141
|
+
function parseFingerprintStore(raw, bankBuiltAt2 = "") {
|
|
4142
|
+
if (typeof raw !== "object" || raw === null) return emptyFingerprintStore(bankBuiltAt2);
|
|
4143
|
+
const root = raw;
|
|
4144
|
+
if (root["bankBuiltAt"] !== void 0 && root["bankBuiltAt"] !== bankBuiltAt2) {
|
|
4145
|
+
return emptyFingerprintStore(bankBuiltAt2);
|
|
4146
|
+
}
|
|
4147
|
+
const records = root["records"];
|
|
4148
|
+
if (typeof records !== "object" || records === null || Array.isArray(records)) {
|
|
4149
|
+
return emptyFingerprintStore(bankBuiltAt2);
|
|
4150
|
+
}
|
|
4151
|
+
const out = {};
|
|
4152
|
+
for (const [key, value] of Object.entries(records)) {
|
|
4153
|
+
const separator = key.indexOf("\0");
|
|
4154
|
+
if (separator <= 0 || separator === key.length - 1) continue;
|
|
4155
|
+
const provider = key.slice(0, separator);
|
|
4156
|
+
const model = key.slice(separator + 1);
|
|
4157
|
+
const record = parseRecord(provider, model, value);
|
|
4158
|
+
if (record !== void 0) out[key] = record;
|
|
4159
|
+
}
|
|
4160
|
+
return { version: 1, bankBuiltAt: bankBuiltAt2, records: out };
|
|
4161
|
+
}
|
|
4162
|
+
function readFingerprintStore(bankBuiltAt2 = "", file = fingerprintFilePath()) {
|
|
4163
|
+
let text;
|
|
4164
|
+
try {
|
|
4165
|
+
text = readFileSync4(file, "utf8");
|
|
4166
|
+
} catch (error) {
|
|
4167
|
+
if (typeof error === "object" && error !== null && "code" in error && error.code === "ENOENT") {
|
|
4168
|
+
return emptyFingerprintStore(bankBuiltAt2);
|
|
4169
|
+
}
|
|
4170
|
+
throw error;
|
|
4171
|
+
}
|
|
4172
|
+
let raw;
|
|
4173
|
+
try {
|
|
4174
|
+
raw = JSON.parse(text);
|
|
4175
|
+
} catch {
|
|
4176
|
+
return emptyFingerprintStore(bankBuiltAt2);
|
|
4177
|
+
}
|
|
4178
|
+
return parseFingerprintStore(raw, bankBuiltAt2);
|
|
4179
|
+
}
|
|
4180
|
+
function writeFingerprintStore(document, file = fingerprintFilePath()) {
|
|
4181
|
+
mkdirSync2(dirname4(file), { recursive: true });
|
|
4182
|
+
const temporary = `${file}.${process.pid}.tmp`;
|
|
4183
|
+
writeFileSync2(temporary, `${JSON.stringify(document, null, 2)}
|
|
4184
|
+
`, "utf8");
|
|
4185
|
+
try {
|
|
4186
|
+
renameSync2(temporary, file);
|
|
4187
|
+
} catch (error) {
|
|
4188
|
+
try {
|
|
4189
|
+
unlinkSync2(temporary);
|
|
4190
|
+
} catch {
|
|
4191
|
+
}
|
|
4192
|
+
throw error;
|
|
4193
|
+
}
|
|
4194
|
+
}
|
|
4195
|
+
function updateFingerprintStore(mutate, bankBuiltAt2 = "", file = fingerprintFilePath()) {
|
|
4196
|
+
const current = readFingerprintStore(bankBuiltAt2, file);
|
|
4197
|
+
const next = mutate(current);
|
|
4198
|
+
writeFingerprintStore(next, file);
|
|
4199
|
+
return next;
|
|
4200
|
+
}
|
|
4201
|
+
function withRecord(document, record) {
|
|
4202
|
+
return {
|
|
4203
|
+
...document,
|
|
4204
|
+
records: { ...document.records, [recordKeyOf(record.provider, record.model)]: record }
|
|
4205
|
+
};
|
|
4206
|
+
}
|
|
4207
|
+
function withoutRecord(document, provider, model) {
|
|
4208
|
+
const key = recordKeyOf(provider, model);
|
|
4209
|
+
if (document.records[key] === void 0) return document;
|
|
4210
|
+
const records = { ...document.records };
|
|
4211
|
+
delete records[key];
|
|
4212
|
+
return { ...document, records };
|
|
4213
|
+
}
|
|
4214
|
+
function providerRecords(document, provider) {
|
|
4215
|
+
return Object.values(document.records).filter((record) => record.provider === provider);
|
|
4216
|
+
}
|
|
4217
|
+
|
|
4218
|
+
// src/fingerprint/verdict.ts
|
|
4219
|
+
var IDENTIFIED_THRESHOLD = 0.6;
|
|
4220
|
+
var FULL_RUN_QUERIES = 3;
|
|
4221
|
+
function topEntriesOf(rows, limit = 3) {
|
|
4222
|
+
return rows.slice(0, limit).map((row) => ({
|
|
4223
|
+
model: row.model,
|
|
4224
|
+
probability: row.probability,
|
|
4225
|
+
similarity: row.profile_similarity
|
|
4226
|
+
}));
|
|
4227
|
+
}
|
|
4228
|
+
function statusOf(attribution, usedOutputs) {
|
|
4229
|
+
if (attribution === null) return "failed";
|
|
4230
|
+
if (usedOutputs < FULL_RUN_QUERIES) return "doubtful";
|
|
4231
|
+
return attribution.probability >= IDENTIFIED_THRESHOLD ? "identified" : "doubtful";
|
|
4232
|
+
}
|
|
4233
|
+
|
|
3503
4234
|
// src/index.ts
|
|
3504
4235
|
var name = PLUGIN_ID;
|
|
3505
4236
|
var inject = ["settings", "llm"];
|
|
3506
4237
|
var PI_NS = PI_AI_NS;
|
|
3507
4238
|
var BOOT_RETRY_DELAYS_MS = [1e3, 2e3, 4e3, 8e3, 16e3, 3e4];
|
|
3508
4239
|
var PROBE_TIMEOUT_MS = 15e3;
|
|
4240
|
+
var FINGERPRINT_CHALLENGE_TIMEOUT_MS = 12e4;
|
|
4241
|
+
var LISTING_PROBE_BUDGET_MS = 6e3;
|
|
4242
|
+
var LISTING_CACHE_MS = 3e4;
|
|
3509
4243
|
var Config = Schema.object({
|
|
3510
4244
|
autofill: Schema.boolean().default(true),
|
|
3511
4245
|
modalityAutofill: Schema.boolean().default(true),
|
|
@@ -3842,6 +4576,10 @@ function apply(ctx, config = {}) {
|
|
|
3842
4576
|
}
|
|
3843
4577
|
const overlaySource = { current: emptyIndex() };
|
|
3844
4578
|
const keyScopes = { current: {} };
|
|
4579
|
+
const lastKnownScopes = /* @__PURE__ */ new Map();
|
|
4580
|
+
const lastKnownRecords = /* @__PURE__ */ new Map();
|
|
4581
|
+
const listingCache = /* @__PURE__ */ new Map();
|
|
4582
|
+
const probeRouteCache = /* @__PURE__ */ new Map();
|
|
3845
4583
|
const providersOf = () => {
|
|
3846
4584
|
const section = piSection();
|
|
3847
4585
|
return isRecord(section) && isRecord(section["providers"]) ? section["providers"] : {};
|
|
@@ -3889,6 +4627,8 @@ function apply(ctx, config = {}) {
|
|
|
3889
4627
|
if (ns === PI_NS) {
|
|
3890
4628
|
refreshHeaders();
|
|
3891
4629
|
refreshKeyScopes();
|
|
4630
|
+
listingCache.clear();
|
|
4631
|
+
probeRouteCache.clear();
|
|
3892
4632
|
}
|
|
3893
4633
|
});
|
|
3894
4634
|
overlaySource.keys = async ({ route, model }) => {
|
|
@@ -3925,10 +4665,23 @@ function apply(ctx, config = {}) {
|
|
|
3925
4665
|
const api = normalizeKeyProtocol(profile?.["api"]);
|
|
3926
4666
|
const baseURL = typeof profile?.["baseURL"] === "string" ? profile["baseURL"].trim() : "";
|
|
3927
4667
|
if (api === void 0 || !LISTABLE_PROTOCOLS.has(api) || baseURL.length === 0) return void 0;
|
|
4668
|
+
const cacheKey = `${route}\0${keys.map((key) => key.ref).join(",")}`;
|
|
4669
|
+
const cached2 = listingCache.get(cacheKey);
|
|
4670
|
+
if (cached2 !== void 0 && Date.now() - cached2.at < LISTING_CACHE_MS) {
|
|
4671
|
+
return new Response(cached2.body, { status: 200, headers: { "content-type": "application/json" } });
|
|
4672
|
+
}
|
|
3928
4673
|
const recordsByRef = /* @__PURE__ */ new Map();
|
|
4674
|
+
const probeTimeoutMs = Math.min(resolved.probeTimeoutMs, LISTING_PROBE_BUDGET_MS);
|
|
4675
|
+
const here = (ref) => `${route}\0${ref}`;
|
|
4676
|
+
const knownHere = /* @__PURE__ */ new Map();
|
|
4677
|
+
for (const key of keys) {
|
|
4678
|
+
const remembered = lastKnownScopes.get(here(key.ref));
|
|
4679
|
+
if (remembered !== void 0) knownHere.set(key.ref, remembered);
|
|
4680
|
+
}
|
|
3929
4681
|
const { results } = await collectKeyListings({
|
|
3930
4682
|
keys,
|
|
3931
4683
|
refOf: (key) => key.ref,
|
|
4684
|
+
lastKnown: knownHere,
|
|
3932
4685
|
probe: async (key) => {
|
|
3933
4686
|
const targets = key.protocol !== void 0 ? [{ api: key.protocol, baseURL: key.endpoint ?? baseURL }] : [
|
|
3934
4687
|
{ api, baseURL },
|
|
@@ -3944,20 +4697,24 @@ function apply(ctx, config = {}) {
|
|
|
3944
4697
|
return send(url, { ...init, headers: requestHeaders, signal: requestSignal });
|
|
3945
4698
|
};
|
|
3946
4699
|
const asked = /* @__PURE__ */ new Set();
|
|
3947
|
-
const
|
|
4700
|
+
const distinct = [];
|
|
3948
4701
|
for (const target of targets) {
|
|
3949
4702
|
if (!LISTABLE_PROTOCOLS.has(target.api) || target.baseURL.length === 0) continue;
|
|
3950
4703
|
const url = probeListingUrl(target.baseURL, target.api);
|
|
3951
4704
|
if (asked.has(url)) continue;
|
|
3952
4705
|
asked.add(url);
|
|
3953
|
-
|
|
3954
|
-
|
|
3955
|
-
|
|
3956
|
-
|
|
3957
|
-
|
|
3958
|
-
|
|
3959
|
-
|
|
3960
|
-
|
|
4706
|
+
distinct.push(target);
|
|
4707
|
+
}
|
|
4708
|
+
const answered = await Promise.all(distinct.map((target) => probeListing({
|
|
4709
|
+
baseURL: target.baseURL,
|
|
4710
|
+
api: target.api,
|
|
4711
|
+
apiKey: key.value,
|
|
4712
|
+
headers: profile?.["headers"],
|
|
4713
|
+
timeoutMs: probeTimeoutMs,
|
|
4714
|
+
send: sendListing
|
|
4715
|
+
})));
|
|
4716
|
+
const records2 = [];
|
|
4717
|
+
for (const listing of answered) {
|
|
3961
4718
|
if (listing === void 0) continue;
|
|
3962
4719
|
for (const model of listing) {
|
|
3963
4720
|
if (records2.some((existing) => existing.id === model.id)) continue;
|
|
@@ -3966,22 +4723,28 @@ function apply(ctx, config = {}) {
|
|
|
3966
4723
|
}
|
|
3967
4724
|
if (records2.length === 0) return { ref: key.ref, models: void 0, targets: [] };
|
|
3968
4725
|
recordsByRef.set(key.ref, records2);
|
|
4726
|
+
const ids = records2.map((model) => model.id);
|
|
4727
|
+
lastKnownScopes.set(here(key.ref), ids);
|
|
4728
|
+
lastKnownRecords.set(here(key.ref), records2);
|
|
3969
4729
|
return {
|
|
3970
4730
|
ref: key.ref,
|
|
3971
|
-
models:
|
|
4731
|
+
models: ids,
|
|
3972
4732
|
targets: []
|
|
3973
4733
|
};
|
|
3974
4734
|
}
|
|
3975
4735
|
});
|
|
3976
4736
|
const records = /* @__PURE__ */ new Map();
|
|
3977
4737
|
for (const result of results) {
|
|
3978
|
-
|
|
4738
|
+
const owned = recordsByRef.get(result.ref) ?? lastKnownRecords.get(here(result.ref)) ?? [];
|
|
4739
|
+
for (const model of owned) {
|
|
3979
4740
|
if (!records.has(model.id)) records.set(model.id, model);
|
|
3980
4741
|
}
|
|
3981
4742
|
}
|
|
3982
|
-
const models = mergeKeyListings(results).ids.map((id) => records.get(id)).filter((model) => model !== void 0);
|
|
4743
|
+
const models = mergeKeyListings(results, knownHere).ids.map((id) => records.get(id)).filter((model) => model !== void 0);
|
|
3983
4744
|
if (models.length === 0) return void 0;
|
|
3984
|
-
|
|
4745
|
+
const body = JSON.stringify({ data: models });
|
|
4746
|
+
listingCache.set(cacheKey, { at: Date.now(), body });
|
|
4747
|
+
return new Response(body, {
|
|
3985
4748
|
status: 200,
|
|
3986
4749
|
headers: { "content-type": "application/json" }
|
|
3987
4750
|
});
|
|
@@ -3999,15 +4762,20 @@ function apply(ctx, config = {}) {
|
|
|
3999
4762
|
ctx.effect(() => () => transforms.dispose(), "dsh-model-think-level: pi-ai request headers");
|
|
4000
4763
|
};
|
|
4001
4764
|
const availableLlm = ctx.llm;
|
|
4765
|
+
let fingerprintDispatch = availableLlm;
|
|
4002
4766
|
if (availableLlm === void 0) {
|
|
4003
4767
|
ctx.inject(["llm"], (llmCtx) => {
|
|
4004
4768
|
const llm = llmCtx.llm;
|
|
4005
|
-
if (llm !== void 0)
|
|
4769
|
+
if (llm !== void 0) {
|
|
4770
|
+
fingerprintDispatch = llm;
|
|
4771
|
+
installPiAiBridge(llm);
|
|
4772
|
+
}
|
|
4006
4773
|
});
|
|
4007
4774
|
} else {
|
|
4008
4775
|
installPiAiBridge(availableLlm);
|
|
4009
4776
|
}
|
|
4010
4777
|
const environmentConflicts = detectHeaderConflicts();
|
|
4778
|
+
let fingerprintMutationTail = Promise.resolve();
|
|
4011
4779
|
ctx.inject(["webServer"], (webServerCtx) => {
|
|
4012
4780
|
ctx.effect(
|
|
4013
4781
|
() => webServerCtx.webServer.register({
|
|
@@ -4069,6 +4837,13 @@ function apply(ctx, config = {}) {
|
|
|
4069
4837
|
}
|
|
4070
4838
|
const apiKeyEnv = typeof profile["apiKeyEnv"] === "string" ? profile["apiKeyEnv"] : void 0;
|
|
4071
4839
|
const listingURL = probeListingUrl(baseURL, api);
|
|
4840
|
+
const listingKey = `${listingURL}\0${apiKeyEnv ?? ""}`;
|
|
4841
|
+
const cacheable = url.searchParams.get("refresh") !== "1";
|
|
4842
|
+
const remembered = cacheable ? probeRouteCache.get(listingKey) : void 0;
|
|
4843
|
+
if (remembered !== void 0 && Date.now() - remembered.at < LISTING_CACHE_MS) {
|
|
4844
|
+
sendJson(res, 200, { ok: true, url: displayUrl(listingURL), data: remembered.data });
|
|
4845
|
+
return;
|
|
4846
|
+
}
|
|
4072
4847
|
let apiKey;
|
|
4073
4848
|
if (apiKeyEnv !== void 0) {
|
|
4074
4849
|
try {
|
|
@@ -4100,7 +4875,11 @@ function apply(ctx, config = {}) {
|
|
|
4100
4875
|
// protocol inference, the path every unanswerable endpoint
|
|
4101
4876
|
// takes.
|
|
4102
4877
|
redirect: "error",
|
|
4103
|
-
|
|
4878
|
+
// This probe waits on an interactive pass (Auto-adapt asks for
|
|
4879
|
+
// it per model), so it gets the catalogue budget rather than the
|
|
4880
|
+
// diagnostic one: a gateway that cannot list within it yields no
|
|
4881
|
+
// endpoint evidence, and the pass moves on to the knowledge base.
|
|
4882
|
+
signal: AbortSignal.timeout(Math.min(resolved.probeTimeoutMs, LISTING_PROBE_BUDGET_MS))
|
|
4104
4883
|
});
|
|
4105
4884
|
if (!upstream.ok) {
|
|
4106
4885
|
const hint = upstream.status === 401 || upstream.status === 403 ? "; check the API key" : "";
|
|
@@ -4123,6 +4902,7 @@ function apply(ctx, config = {}) {
|
|
|
4123
4902
|
});
|
|
4124
4903
|
return;
|
|
4125
4904
|
}
|
|
4905
|
+
probeRouteCache.set(listingKey, { at: Date.now(), data: entries });
|
|
4126
4906
|
sendJson(res, 200, { ok: true, url: displayUrl(listingURL), data: entries });
|
|
4127
4907
|
} catch (error) {
|
|
4128
4908
|
sendJson(res, 502, {
|
|
@@ -4730,6 +5510,188 @@ function apply(ctx, config = {}) {
|
|
|
4730
5510
|
}),
|
|
4731
5511
|
"dsh-model-think-level: headers config route"
|
|
4732
5512
|
);
|
|
5513
|
+
ctx.effect(
|
|
5514
|
+
() => webServerCtx.webServer.register({
|
|
5515
|
+
kind: "exact",
|
|
5516
|
+
path: FINGERPRINT_PATH,
|
|
5517
|
+
handler: async (req, res) => {
|
|
5518
|
+
if (!isTrustedRequest(req)) {
|
|
5519
|
+
sendJson(res, 403, { ok: false, error: "forbidden" });
|
|
5520
|
+
return;
|
|
5521
|
+
}
|
|
5522
|
+
const method = req.method ?? "GET";
|
|
5523
|
+
if (method !== "GET" && method !== "POST" && method !== "DELETE") {
|
|
5524
|
+
sendJson(res, 405, { ok: false, error: "method not allowed" });
|
|
5525
|
+
return;
|
|
5526
|
+
}
|
|
5527
|
+
const url = new URL(req.url ?? "/", "http://x");
|
|
5528
|
+
const route = url.searchParams.get("route") ?? "";
|
|
5529
|
+
if (route.length === 0) {
|
|
5530
|
+
sendJson(res, 400, { ok: false, error: "a provider route is required" });
|
|
5531
|
+
return;
|
|
5532
|
+
}
|
|
5533
|
+
let section;
|
|
5534
|
+
try {
|
|
5535
|
+
section = piSection(true);
|
|
5536
|
+
} catch {
|
|
5537
|
+
sendJson(res, 502, { ok: false, error: "the llm-pi-ai settings could not be read" });
|
|
5538
|
+
return;
|
|
5539
|
+
}
|
|
5540
|
+
const profile = isRecord(section) && isRecord(section["providers"]) ? section["providers"][route] : void 0;
|
|
5541
|
+
if (!isRecord(profile)) {
|
|
5542
|
+
sendJson(res, 400, { ok: false, error: `no llm-pi-ai provider route "${route}"` });
|
|
5543
|
+
return;
|
|
5544
|
+
}
|
|
5545
|
+
let bank;
|
|
5546
|
+
try {
|
|
5547
|
+
bank = loadBankSync();
|
|
5548
|
+
} catch (error) {
|
|
5549
|
+
sendJson(res, 500, {
|
|
5550
|
+
ok: false,
|
|
5551
|
+
error: error instanceof Error ? error.message : String(error)
|
|
5552
|
+
});
|
|
5553
|
+
return;
|
|
5554
|
+
}
|
|
5555
|
+
const file = fingerprintFilePath();
|
|
5556
|
+
const stamp = bankBuiltAt(bank);
|
|
5557
|
+
if (method === "GET") {
|
|
5558
|
+
const document = readFingerprintStore(stamp, file);
|
|
5559
|
+
sendJson(res, 200, {
|
|
5560
|
+
ok: true,
|
|
5561
|
+
data: {
|
|
5562
|
+
bank: summarizeBank(bank),
|
|
5563
|
+
records: providerRecords(document, route)
|
|
5564
|
+
}
|
|
5565
|
+
});
|
|
5566
|
+
return;
|
|
5567
|
+
}
|
|
5568
|
+
let releaseMutation;
|
|
5569
|
+
const previous = fingerprintMutationTail;
|
|
5570
|
+
fingerprintMutationTail = new Promise((resolve2) => {
|
|
5571
|
+
releaseMutation = resolve2;
|
|
5572
|
+
});
|
|
5573
|
+
await previous;
|
|
5574
|
+
try {
|
|
5575
|
+
if (method === "DELETE") {
|
|
5576
|
+
const model2 = url.searchParams.get("model") ?? "";
|
|
5577
|
+
if (model2.length === 0) {
|
|
5578
|
+
sendJson(res, 400, { ok: false, error: "a model id is required" });
|
|
5579
|
+
return;
|
|
5580
|
+
}
|
|
5581
|
+
updateFingerprintStore(
|
|
5582
|
+
(document) => withoutRecord(document, route, model2),
|
|
5583
|
+
stamp,
|
|
5584
|
+
file
|
|
5585
|
+
);
|
|
5586
|
+
sendJson(res, 200, { ok: true });
|
|
5587
|
+
return;
|
|
5588
|
+
}
|
|
5589
|
+
const body = await readJsonBody(req, 2048);
|
|
5590
|
+
const model = isRecord(body) && typeof body["model"] === "string" ? body["model"] : "";
|
|
5591
|
+
if (model.length === 0) {
|
|
5592
|
+
sendJson(res, 400, { ok: false, error: "a model id is required" });
|
|
5593
|
+
return;
|
|
5594
|
+
}
|
|
5595
|
+
if (!isEligibleModel(model)) {
|
|
5596
|
+
sendJson(res, 400, {
|
|
5597
|
+
ok: false,
|
|
5598
|
+
error: `model "${model}" is not a GPT or Claude model`
|
|
5599
|
+
});
|
|
5600
|
+
return;
|
|
5601
|
+
}
|
|
5602
|
+
const llm = fingerprintDispatch ?? ctx.llm;
|
|
5603
|
+
if (llm === void 0 || typeof llm.prepareCall !== "function") {
|
|
5604
|
+
sendJson(res, 503, { ok: false, error: "the llm service is unavailable" });
|
|
5605
|
+
return;
|
|
5606
|
+
}
|
|
5607
|
+
res.statusCode = 200;
|
|
5608
|
+
res.setHeader("content-type", "application/x-ndjson; charset=utf-8");
|
|
5609
|
+
res.setHeader("cache-control", "no-store");
|
|
5610
|
+
res.setHeader("x-accel-buffering", "no");
|
|
5611
|
+
const emit = (line) => {
|
|
5612
|
+
res.write(`${JSON.stringify(line)}
|
|
5613
|
+
`);
|
|
5614
|
+
};
|
|
5615
|
+
const challenges = generateChallenges(3);
|
|
5616
|
+
const controller = new AbortController();
|
|
5617
|
+
res.on("close", () => {
|
|
5618
|
+
if (!res.writableEnded) controller.abort();
|
|
5619
|
+
});
|
|
5620
|
+
let run;
|
|
5621
|
+
try {
|
|
5622
|
+
run = await runProbe({
|
|
5623
|
+
provider: route,
|
|
5624
|
+
model,
|
|
5625
|
+
challenges,
|
|
5626
|
+
llm,
|
|
5627
|
+
signal: controller.signal,
|
|
5628
|
+
timeoutMs: FINGERPRINT_CHALLENGE_TIMEOUT_MS,
|
|
5629
|
+
onOutcome: (outcome) => {
|
|
5630
|
+
emit({
|
|
5631
|
+
type: "progress",
|
|
5632
|
+
index: outcome.index,
|
|
5633
|
+
ok: outcome.ok,
|
|
5634
|
+
...outcome.error === void 0 ? {} : { error: outcome.error }
|
|
5635
|
+
});
|
|
5636
|
+
}
|
|
5637
|
+
});
|
|
5638
|
+
} catch (error) {
|
|
5639
|
+
emit({
|
|
5640
|
+
type: "error",
|
|
5641
|
+
error: error instanceof Error ? error.message : String(error)
|
|
5642
|
+
});
|
|
5643
|
+
res.end();
|
|
5644
|
+
return;
|
|
5645
|
+
}
|
|
5646
|
+
let attribution = null;
|
|
5647
|
+
let failure;
|
|
5648
|
+
try {
|
|
5649
|
+
attribution = analyzeGlobalOutputs(run.outputs, bank);
|
|
5650
|
+
} catch (error) {
|
|
5651
|
+
failure = error instanceof Error ? error.message : String(error);
|
|
5652
|
+
}
|
|
5653
|
+
const usedOutputs = attribution === null ? 0 : attribution.used_outputs;
|
|
5654
|
+
const record = {
|
|
5655
|
+
provider: route,
|
|
5656
|
+
model,
|
|
5657
|
+
status: statusOf(attribution, usedOutputs),
|
|
5658
|
+
checkedAt: Date.now(),
|
|
5659
|
+
usedOutputs
|
|
5660
|
+
};
|
|
5661
|
+
if (attribution === null) {
|
|
5662
|
+
const firstFailure = run.outcomes.find((outcome) => !outcome.ok);
|
|
5663
|
+
record.error = run.outputs.length === 0 && firstFailure?.error !== void 0 ? firstFailure.error : failure ?? NO_USABLE_OUTPUT;
|
|
5664
|
+
} else {
|
|
5665
|
+
record.prediction = attribution.prediction;
|
|
5666
|
+
record.predictionName = attribution.prediction_name;
|
|
5667
|
+
record.probability = attribution.probability;
|
|
5668
|
+
record.familyPrediction = attribution.family_prediction;
|
|
5669
|
+
record.familyProbability = attribution.family_probability;
|
|
5670
|
+
record.top = topEntriesOf(attribution.results, 3);
|
|
5671
|
+
record.calibration = {
|
|
5672
|
+
queries: attribution.calibration.queries,
|
|
5673
|
+
cvAccuracy: attribution.calibration.cv_accuracy
|
|
5674
|
+
};
|
|
5675
|
+
}
|
|
5676
|
+
updateFingerprintStore(
|
|
5677
|
+
(document) => withRecord(document, record),
|
|
5678
|
+
stamp,
|
|
5679
|
+
file
|
|
5680
|
+
);
|
|
5681
|
+
emit({ type: "result", record });
|
|
5682
|
+
res.end();
|
|
5683
|
+
} catch (error) {
|
|
5684
|
+
sendJson(res, 500, {
|
|
5685
|
+
ok: false,
|
|
5686
|
+
error: error instanceof Error ? error.message : String(error)
|
|
5687
|
+
});
|
|
5688
|
+
} finally {
|
|
5689
|
+
releaseMutation?.();
|
|
5690
|
+
}
|
|
5691
|
+
}
|
|
5692
|
+
}),
|
|
5693
|
+
"dsh-model-think-level: fingerprint route"
|
|
5694
|
+
);
|
|
4733
5695
|
});
|
|
4734
5696
|
}
|
|
4735
5697
|
export {
|