dsh-fingerprint 0.4.0 → 0.5.0

package/README.md

@@ -50,12 +50,13 @@ dsh plugin --profile web add github:Jerry2262/dsh-fingerprint
 ## 使用
 
 1. 选择服务商和模型。不能关闭 reasoning 的模型会置灰、不可选，并标注原因。
-2. 设置每个 cell 的重复次数（默认 30，UI 最低 10），点“开始采集”。
-3. 运行中的记录显示进度；完成后显示有效率、分半 JSD、纳入 cell 数。插件被卸载或进程重启时，未完成的记录会标为失败（保留已抓到的响应），可以直接删除，不会自动续跑。
-4. 在“对比记录”里选两条记录，点“对比记录”。比较方式只有一种，结果会标注两边是否同一目标。
-5. 结果包含差异分数、判定线、结论、关系、cell 级差异和警告。低于判定线只说明差异小，不等于 100% 是同一个模型。
-6. 行内垃圾桶可以删除记录；删除前会二次确认。运行中的记录不能删除。
-7. 面板底部的“设置”里有 Agent 工具开关，默认关闭。开启后把 6 个 `llm_fingerprint_*` 工具暴露给 agent（每个请求多约 4.3 KB 工具 schema），关闭后立即从后续请求里移除；选择保存在数据目录的 `settings.json`，删掉这个文件即回到默认关闭。
+2. 点“检测可行性”。它用 2 个 cell × 3 次（6 个请求）实际发一次，确认这条路由能给出单词答案且不带 reasoning trace；通过后“开始采集”才可用，换服务商或模型会要求重新检测。模型元数据说支持 `off` 不代表服务商真的照做，这一步是唯一能提前发现的地方。
+3. 设置每个 cell 的重复次数（默认 30，UI 最低 10），点“开始采集”。
+4. 运行中的记录显示进度；完成后显示有效率、分半 JSD、纳入 cell 数。插件被卸载或进程重启时，未完成的记录会标为失败（保留已抓到的响应），可以直接删除，不会自动续跑。
+5. 在“对比记录”里选两条记录，点“对比记录”。比较方式只有一种，结果会标注两边是否同一目标。
+6. 结果包含差异分数、判定线、结论、关系、cell 级差异和警告。低于判定线只说明差异小，不等于 100% 是同一个模型。
+7. 行内垃圾桶可以删除记录；删除前会二次确认。运行中的记录不能删除。
+8. 面板底部的“设置”里有 Agent 工具开关，默认关闭。开启后把 6 个 `llm_fingerprint_*` 工具暴露给 agent（每个请求多约 4.3 KB 工具 schema），关闭后立即从后续请求里移除；选择保存在数据目录的 `settings.json`，删掉这个文件即回到默认关闭。
 
 ## 工作原理
 

package/client.js

@@ -64,6 +64,12 @@ window.__ModuleLoader__.load({
       noCompatibleModels: 'This provider has no model that can disable reasoning.',
       noCompatibleModelsShort: 'No models eligible for capture',
       offAvailable: 'Reasoning will be disabled (off) for capture.',
+      checkTitle: 'Check feasibility',
+      checking: 'Checking…',
+      checkHint: 'Sends 2 cells × 3 samples (6 requests) to confirm this route answers in one word without a reasoning trace.',
+      checkRequired: 'Run the feasibility check first: it confirms the route satisfies the protocol in 6 requests instead of discovering it after the full battery.',
+      checkPassed: 'Feasibility check passed: {valid}/{responses} one-word answers, {leak} with a reasoning trace.',
+      checkFailed: 'Feasibility check failed: {valid}/{responses} one-word answers, {leak} with a reasoning trace, {errors} failed requests.',
       effort: 'Reasoning effort',
       started: 'Capture started:',
       error: 'Error',
@@ -183,6 +189,12 @@ window.__ModuleLoader__.load({
       noCompatibleModels: '该服务商没有支持关闭推理的模型，不能用于论文协议。',
       noCompatibleModelsShort: '无符合采集条件的模型',
       offAvailable: '采集时将关闭推理（off）。',
+      checkTitle: '检测可行性',
+      checking: '检测中…',
+      checkHint: '先用 2 个 cell × 3 次（6 个请求）确认这条路由能给出单词答案，且不带 reasoning trace。',
+      checkRequired: '请先检测可行性：用 6 个请求确认这条路由满足论文协议，避免跑完 1200 个请求才发现结果无效。',
+      checkPassed: '可行性检测通过：{valid}/{responses} 个单词答案，{leak} 个带 reasoning trace。',
+      checkFailed: '可行性检测未通过：{valid}/{responses} 个单词答案，{leak} 个带 reasoning trace，{errors} 个请求失败。',
       effort: '推理档位',
       started: '指纹采集已开始：',
       error: '错误',
@@ -309,6 +321,16 @@ window.__ModuleLoader__.load({
       return text;
     }
 
+    /** One line describing the last feasibility check for the selected target. */
+    function checkMessage(check, passed, t) {
+      const fill = (text) => text
+        .replace('{valid}', String(check.valid))
+        .replace('{responses}', String(check.responses))
+        .replace('{leak}', String(check.reasoningLeak))
+        .replace('{errors}', String(check.errors));
+      return check.error ? `${fill(t('checkFailed'))} ${check.error}` : fill(t(passed ? 'checkPassed' : 'checkFailed'));
+    }
+
     function friendlyError(message, t) {
       const text = String(message ?? '');
       if (/cannot disable reasoning/i.test(text)) return t('errorNoReasoning');
@@ -347,7 +369,7 @@ window.__ModuleLoader__.load({
     function visibleWarnings(comparison) {
       return (comparison.warnings || []).filter((item) => {
         if (comparison.verdictReason === 'partial_battery' && /full battery/i.test(item)) return false;
-        if (comparison.verdictReason === 'repetitions' && /threshold needs 30/i.test(item)) return false;
+        if (comparison.verdictReason === 'repetitions' && /cutoff needs 30/i.test(item)) return false;
         return true;
       });
     }
@@ -663,6 +685,48 @@ window.__ModuleLoader__.load({
       );
     }
 
+    /**
+     * Sidebar glyph: the fingerprint mark drawn to the same optical spec as the
+     * built-in panel icons — 16 viewBox, 1.33 stroke, currentColor, artwork in
+     * the 1.3–14.7 box — so it matches their visual size. The sidebar row owns
+     * the edge and passes it as `size`.
+     */
+    function PanelIcon({ size }) {
+      return h('svg', {
+        viewBox: '0 0 16 16',
+        width: size,
+        height: size,
+        fill: 'none',
+        'aria-hidden': true,
+        strokeWidth: 1.33,
+        style: { display: 'block', flex: 'none' },
+      },
+        h('rect', {
+          x: 1.97,
+          y: 1.97,
+          width: 12.06,
+          height: 12.06,
+          rx: 3.1,
+          stroke: 'currentColor',
+        }),
+        h('path', {
+          d: 'M4.84 9.57c0-1.83 1.15-3.15 3.16-3.15s3.16 1.32 3.16 3.15',
+          stroke: 'currentColor',
+          strokeLinecap: 'round',
+        }),
+        h('path', {
+          d: 'M6.27 10.35c0-1.05.58-1.83 1.73-1.83s1.73.78 1.73 1.83',
+          stroke: 'currentColor',
+          strokeLinecap: 'round',
+        }),
+        h('path', {
+          d: 'M8 11.4v-1.3',
+          stroke: 'currentColor',
+          strokeLinecap: 'round',
+        }),
+      );
+    }
+
     function TrashIcon({ size = 14 }) {
       return h('svg', {
         viewBox: '0 0 24 24',
@@ -737,6 +801,8 @@ window.__ModuleLoader__.load({
       const [modelId, setModelId] = useState('');
       const [reps, setReps] = useState(30);
       const [starting, setStarting] = useState(false);
+      const [checking, setChecking] = useState(false);
+      const [check, setCheck] = useState(null);
       const [comparing, setComparing] = useState(false);
       const [deleting, setDeleting] = useState(false);
       const [pendingDelete, setPendingDelete] = useState(null);
@@ -839,6 +905,9 @@ window.__ModuleLoader__.load({
       const hasIncompatible = models.some((entry) => entry.paperCompatible === false);
       const noCompatibleModels = models.length > 0 && compatibleModels.length === 0;
       const modelCompatible = model ? model.paperCompatible !== false : false;
+      /** A feasibility check only counts for the target it ran against. */
+      const checkMatches = Boolean(check && check.provider === providerId && check.model === modelId);
+      const checkPassed = checkMatches && check.passed === true;
       const completed = checkpoints.filter((entry) => entry.status === 'completed' || entry.status === 'completed_with_failures');
       const bOptions = completed.filter((entry) => entry.id !== a);
 
@@ -850,10 +919,54 @@ window.__ModuleLoader__.load({
         setModelId(nextModel ? nextModel.id : '');
         setWarning('');
         setNotice('');
+        setCheck(null);
+      }
+
+      async function runCheck() {
+        if (!providerId || !modelId || !modelCompatible) return;
+        setChecking(true);
+        setError('');
+        setWarning('');
+        setNotice('');
+        try {
+          const result = await api('/probe', {
+            method: 'POST',
+            body: JSON.stringify({ provider: providerId, model: modelId }),
+          });
+          const summary = result.summary || {};
+          setCheck({
+            provider: providerId,
+            model: modelId,
+            passed: summary.protocolCompliant === true,
+            responses: Number(summary.responses ?? 0),
+            valid: Number(summary.valid ?? 0),
+            reasoningLeak: Number(summary.reasoningLeak ?? 0),
+            errors: Number(summary.errors ?? 0),
+            probeId: result.probeId ?? null,
+          });
+        } catch (cause) {
+          setCheck({
+            provider: providerId,
+            model: modelId,
+            passed: false,
+            responses: 0,
+            valid: 0,
+            reasoningLeak: 0,
+            errors: 1,
+            error: friendlyError(cause.message || cause, t),
+            probeId: null,
+          });
+        } finally {
+          setChecking(false);
+        }
       }
 
       async function start() {
         if (!providerId || !modelId) return;
+        if (!checkPassed) {
+          setError(t('checkRequired'));
+          return;
+        }
         if (!modelCompatible) {
           setError(noCompatibleModels ? t('noCompatibleModels') : t('noOff'));
           return;
@@ -865,7 +978,12 @@ window.__ModuleLoader__.load({
         try {
           const created = await api('/checkpoints', {
             method: 'POST',
-            body: JSON.stringify({ provider: providerId, model: modelId, reps: Number(reps) }),
+            body: JSON.stringify({
+              provider: providerId,
+              model: modelId,
+              reps: Number(reps),
+              ...(checkPassed && check.probeId ? { probeId: check.probeId } : {}),
+            }),
           });
           const recordId = created.recordId ?? created.checkpointId;
           setNotice(`${t('started')} ${recordId}`);
@@ -1004,7 +1122,10 @@ window.__ModuleLoader__.load({
                 h('select', {
                   style: S.input,
                   value: modelId,
-                  onChange: (event) => setModelId(event.target.value),
+                  onChange: (event) => {
+                    setModelId(event.target.value);
+                    setCheck(null);
+                  },
                   disabled: models.length === 0 || noCompatibleModels,
                 },
                   h('option', { value: '' }, models.length === 0 ? t('noModels') : noCompatibleModels ? t('noCompatibleModelsShort') : '—'),
@@ -1028,11 +1149,19 @@ window.__ModuleLoader__.load({
                   onChange: (event) => setReps(Math.max(10, Math.min(100, Number(event.target.value) || 30))),
                 }),
               ),
-              h('button', {
-                style: { ...S.button, ...(starting || !modelCompatible ? S.disabled : {}) },
-                disabled: starting || !providerId || !modelId || !modelCompatible,
-                onClick: start,
-              }, starting ? t('starting') : t('start')),
+              h('div', { style: { display: 'flex', gap: '8px', flex: '1 1 auto', justifyContent: 'flex-end' } },
+                h('button', {
+                  style: { ...S.buttonSecondary, ...(checking || !modelCompatible || !providerId || !modelId ? S.disabled : {}) },
+                  disabled: checking || starting || !providerId || !modelId || !modelCompatible,
+                  onClick: runCheck,
+                }, checking ? t('checking') : t('checkTitle')),
+                h('button', {
+                  style: { ...S.button, ...(starting || !modelCompatible || !checkPassed ? S.disabled : {}) },
+                  disabled: starting || !providerId || !modelId || !modelCompatible || !checkPassed,
+                  title: checkPassed ? undefined : t('checkRequired'),
+                  onClick: start,
+                }, starting ? t('starting') : t('start')),
+              ),
             ),
             h('div', { style: { ...S.hint, marginTop: '10px' } },
               `${t('estimated')}: ${estimate} ${t('requests')} (40 ${t('cellsUnit')} × ${reps})`,
@@ -1052,6 +1181,16 @@ window.__ModuleLoader__.load({
                 ? h('div', { style: { ...S.hint, color: 'var(--dsw-alias-state-warn-primary)' } }, t('noCompatibleModels'))
                 : null,
             hasIncompatible ? h('div', { style: S.hint }, t('incompatibleHint')) : null,
+            model
+              ? h('div', {
+                style: {
+                  ...S.hint,
+                  color: checkMatches
+                    ? (checkPassed ? 'var(--dsw-alias-state-success-primary)' : 'var(--dsw-alias-state-error-primary)')
+                    : 'var(--dsw-alias-label-secondary)',
+                },
+              }, checkMatches ? checkMessage(check, checkPassed, t) : t('checkHint'))
+              : null,
           ),
 
           h(Section, {
@@ -1376,7 +1515,7 @@ window.__ModuleLoader__.load({
           order: 30,
           locale: NS,
           label: () => t('title'),
-        }, ({ label }) => h(Mark, { size: 18 })));
+        }, PanelIcon));
       },
     };
   },

package/docs/DESIGN.md

@@ -78,6 +78,14 @@ llm-fingerprint/
 
 遇到 `MissingSessionID` 时停止调度后续采样，等待已经发出的并发请求结束后将记录标为失败，保留原始错误，不继续发完整批次。
 
+### 采集前的可行性检测
+
+面板选完服务商和模型后必须先点“检测可行性”：它调用 `POST /probe`（2 个 cell × 3 次，共 6 个请求），只有返回的 `summary.protocolCompliant` 为真时“开始采集”才可用；换服务商或模型会清掉结果，要求重新检测。
+
+原因是元数据里的 `off` 只说明适配器愿意请求关闭档位，不保证服务商真的照做。路由把 `off` 翻成“不发送 reasoning 参数”时，服务商可能仍按默认推理，`max_tokens=16` 全被推理吃光，整批响应都会变成 `post_reasoning`：`command-goat` 的 `openai-completions` 路由就这样跑完 1200 个请求才暴露 0 有效。这种失败只有在真的发一次请求后才看得见，所以检测放在采集之前，用 6 个请求代替 1200 个。
+
+检测只拦面板入口：HTTP API 和 agent 工具仍然可以直接 `POST /checkpoints`（`startCheckpoint` 只做元数据层面的 `supportsPaperProtocol` 检查），这条路径留给诊断和自动化。通过检测后再采集时，面板把 `probeId` 一并提交，记录到 `meta.probeId`，事后可以把记录和当次检测对上。检测日志按时间保留最近 `PROBE_LOG_RETENTION`（20）条，其余在写入时删除。
+
 ## 5. 兼容性硬拦截
 
 `supportsPaperProtocol(info)` 的规则：

package/index.js

@@ -47,6 +47,8 @@ const MAX_TOKENS = 16;
 const TEMPERATURE = 1.0;
 const DEFAULT_REPS = 30;
 const API_PREFIX = '/llm-fingerprint/api';
+/** Feasibility checks are cheap and disposable: keep the newest logs only. */
+const PROBE_LOG_RETENTION = 20;
 
 function resolveDataDir(config) {
   const configured = config && typeof config.dataDir === 'string' && config.dataDir.trim();
@@ -476,6 +478,18 @@ function recoverInterruptedRecords(root) {
   return recovered;
 }
 
+/** Keep the probe directory bounded: probe ids start with a timestamp, so name order is age order. */
+function pruneProbeLogs(dir) {
+  const names = readdirSync(dir).filter((name) => /^probe-.+\.json$/.test(name)).sort();
+  for (const name of names.slice(0, Math.max(0, names.length - PROBE_LOG_RETENTION))) {
+    try {
+      rmSync(path.join(dir, name), { force: true });
+    } catch {
+      // A log that cannot be removed is not worth failing a diagnostic check over.
+    }
+  }
+}
+
 function checkpointSummary(meta) {
   return {
     id: meta.id,
@@ -650,6 +664,7 @@ function createService(ctx, root, lifetimeSignal) {
       summary,
       responses,
     }, null, 2)}\n`);
+    pruneProbeLogs(probeDir);
     return {
       probeId,
       provider: args.provider,
@@ -681,6 +696,10 @@ function createService(ctx, root, lifetimeSignal) {
       if (!cellById(cellId)) throw new Error(`unknown cell "${cellId}"`);
     }
     const reps = Math.max(1, Math.min(100, Number(args.reps ?? DEFAULT_REPS)));
+    const probeId = args.probeId == null ? null : String(args.probeId);
+    if (probeId !== null && !/^probe-[0-9A-Za-z-]+$/.test(probeId)) {
+      throw new Error('invalid probe id');
+    }
     const id = `cp-${nowUtc().replace(/[-:.]/g, '').slice(0, 15)}-${randomUUID().slice(0, 8)}`;
     const provider = await providerInfo(ctx, args.provider);
     const warnings = [];
@@ -702,6 +721,7 @@ function createService(ctx, root, lifetimeSignal) {
       reasoningEffort: effort,
       reasoningPolicy: effort == null ? 'none' : effortIsDisabled(effort) ? 'disabled' : 'least-available',
       reasoningLeakResponses: 0,
+      ...probeId === null ? {} : { probeId },
       modelInfo: adapterMetadata(info),
       request: { temperature: TEMPERATURE, maxTokens: MAX_TOKENS, systemPrompt: 'per-cell', sessionPolicy: 'new-per-sample-reuse-on-retry' },
       usage: { inputTokens: 0, outputTokens: 0, cacheReadTokens: 0, cacheWriteTokens: 0, reasoningTokens: 0 },

package/package.json

@@ -1,12 +1,12 @@
 {
   "name": "dsh-fingerprint",
-  "version": "0.4.0",
+  "version": "0.5.0",
   "type": "module",
   "description": "Paper-protocol fingerprint records and comparison panel for configured DSH provider/model routes.",
   "license": "MIT",
   "repository": {
     "type": "git",
-    "url": "https://github.com/Jerry2262/dsh-fingerprint.git"
+    "url": "git+https://github.com/Jerry2262/dsh-fingerprint.git"
   },
   "homepage": "https://github.com/Jerry2262/dsh-fingerprint",
   "bugs": {