mycomesh-consumer 0.7.0 → 0.8.0

package/networks/mycomesh-v11-sepolia.json

@@ -6,7 +6,7 @@
   "stablecoin": "0xbce27efad4191277167fe76819581a97959048cf",
   "registry": "0x0ff3210e8e8abf5f5bfa73a39e60db7416b6c8c7",
   "relay_directory": "0xa0aaad9393f51cb5786c69f152133a174588bdab",
-  "probe_ledger": "0x30055eae6645cce8754f834a17689d20a3795ca3",
+  "probe_ledger": "0x3bbc6fc971a6c4e528af9228b95d4b83ef188795",
   "emission": "0x85ee0e8f63400041f0c76010c590c330bc135a8d",
   "token": "0x620d026ad854fb17c7b16534a0f28ef5499e2d4a",
   "emission_block": 11815544,
@@ -15,7 +15,8 @@
       "name": "OpenAI frontier",
       "models": [
         "gpt-5.5"
-      ]
+      ],
+      "capability_floor": 0.85
     },
     "2": {
       "name": "Anthropic Claude",

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mycomesh-consumer",
-  "version": "0.7.0",
+  "version": "0.8.0",
   "description": "MycoMesh V11 Consumer: a custodied deposit, relay-blind encrypted requests to any Provider, and a local OpenAI-compatible endpoint.",
   "repository": {
     "type": "git",

package/src/probes.mjs

@@ -4,9 +4,108 @@ import { verifySignedReceipt } from "./eip712.mjs";
 
 export const PROBE_EVIDENCE_SCHEMA = "mycomesh.v11.probe-evidence.v1";
 const WEEKDAYS = ["Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday", "Sunday"];
-const numbers = (text) => (text.match(/\d{1,3}(?:[,\s_]\d{3})+(?!\d)|\d+/g) || []).map((match) => match.replace(/[,\s_]/g, ""));
+const numbers = (text) => (text.match(/[0-9]{1,3}(?:[,\s_][0-9]{3})+(?![0-9])|[0-9]+/g) || []).map((match) => match.replace(/[,\s_]/g, ""));
+
+// ---------------- capability probes (mirrors mycomesh/capability.py) ----------------
+
+export const CAPABILITY_KINDS = ["trace", "crt", "long_multiply", "path", "kth", "letters", "calendar"];
+/** Default per-tier pass-rate floors; a manifest tier's capability_floor overrides them. */
+export const CAPABILITY_FLOORS = { 1: 0.85 }; // tier 2 (Claude) has none until calibrated
+
+export function ordinal(n) {
+  const suffix = n % 100 >= 10 && n % 100 <= 20 ? "th" : ({ 1: "st", 2: "nd", 3: "rd" })[n % 10] || "th";
+  return `${n}${suffix}`;
+}
+
+const LINE_BREAKS = /\r\n|[\n\r\v\f\x1c\x1d\x1e\x85\u2028\u2029]/;
+const weekdays = (text) => WEEKDAYS.filter((day) => text.toLowerCase().includes(day.toLowerCase()));
+
+function weekdayAfter(start, offset) {
+  const day = new Date(`${start}T00:00:00Z`);
+  day.setUTCDate(day.getUTCDate() + Number(offset));
+  return WEEKDAYS[(day.getUTCDay() + 6) % 7];
+}
+
+function capabilityTask(kind, p) {
+  if (kind === "trace") {
+    let [x, y] = [Number(p.x), Number(p.y)];
+    const [a, m, n] = [Number(p.a), Number(p.m), Number(p.n)];
+    const question = `Start with x = ${x} and y = ${y}. Repeat the following ${n} times: first set x to (x * ${a} + y) mod ${m}, `
+      + `then set y to (y + 2 * x) mod ${m}. What is x + y at the end?`;
+    for (let i = 0; i < n; i += 1) { x = (x * a + y) % m; y = (y + 2 * x) % m; }
+    return { question, reference: String(x + y) };
+  }
+  if (kind === "crt") {
+    const moduli = p.moduli.map(Number); const residues = p.residues.map(Number);
+    const parts = moduli.map((m, i) => `remainder ${residues[i]} when divided by ${m}`);
+    const question = `What is the smallest positive integer that leaves ${parts.slice(0, -1).join(", ")}, and ${parts[parts.length - 1]}?`;
+    // Constructive CRT (the moduli are distinct primes): the least positive solution.
+    const big = moduli.map(BigInt);
+    const product = big.reduce((a, b) => a * b, 1n);
+    const power = (base, exponent, m) => { let r = 1n; base %= m; for (; exponent > 0n; exponent >>= 1n, base = base * base % m) if (exponent & 1n) r = r * base % m; return r; };
+    const x = big.reduce((sum, m, i) => sum + BigInt(residues[i]) * (product / m) * power(product / m, m - 2n, m), 0n) % product;
+    return { question, reference: String(x || product) };
+  }
+  if (kind === "long_multiply") return { question: `What is ${p.a} times ${p.b}?`, reference: String(BigInt(p.a) * BigInt(p.b)) };
+  if (kind === "path") {
+    const edges = p.edges.map(([u, v, w]) => [String(u), String(v), Number(w)]);
+    const question = `In a road network the two-way roads and their lengths are: ${edges.map(([u, v, w]) => `${u}-${v} ${w}`).join(", ")}. `
+      + `What is the length of the shortest route from ${p.start} to ${p.end}?`;
+    const distance = new Map([[String(p.start), 0]]); const done = new Set();
+    for (;;) {
+      const open = [...distance].filter(([node]) => !done.has(node)).sort((a, b) => a[1] - b[1] || (a[0] < b[0] ? -1 : 1));
+      if (!open.length) break;
+      const [node, d] = open[0];
+      done.add(node);
+      for (const [u, v, w] of edges) {
+        for (const [here, there] of [[u, v], [v, u]]) {
+          if (here === node && d + w < (distance.get(there) ?? Infinity)) distance.set(there, d + w);
+        }
+      }
+    }
+    return { question, reference: String(distance.get(String(p.end))) };
+  }
+  if (kind === "kth") {
+    const values = p.values.map(Number); const k = Number(p.k);
+    return { question: `What is the ${ordinal(k)} smallest number in this list: ${values.join(", ")}?`,
+      reference: String([...values].sort((a, b) => a - b)[k - 1]) };
+  }
+  if (kind === "letters") {
+    return { question: `How many times does the letter "${p.letter}" appear in the following text: "${p.text}"?`,
+      reference: String(String(p.text).split(String(p.letter)).length - 1) };
+  }
+  if (kind === "calendar") {
+    return { question: `What day of the week will it be ${p.offset} days after ${p.start}?`, reference: weekdayAfter(p.start, p.offset) };
+  }
+  throw new Error(`unknown capability kind ${kind}`);
+}
+
+function gradeCapability(task, answer) {
+  // The answer finally stated decides: the last \boxed{...}, else the last line that states a value.
+  // A handful of values at most: listing candidates is not answering.
+  const text = String(answer);
+  const pick = task.kind === "calendar" ? weekdays : numbers;
+  const boxed = [...text.matchAll(/\\boxed\{([^{}]*)\}/g)];
+  let stated = [];
+  if (boxed.length) stated = pick(boxed[boxed.length - 1][1]);
+  else for (const line of text.split(LINE_BREAKS).reverse()) { const found = pick(line); if (found.length) { stated = found; break; } }
+  return stated.length <= (task.kind === "calendar" ? 1 : 3) && stated.includes(task.reference) ? "pass" : "wrong";
+}
+
+/** Upper bound of a pass rate at 99% one-sided confidence (Wilson). */
+export function wilsonUpper(passes, total, z = 2.326) {
+  if (!total) return 1;
+  const p = passes / total;
+  const centre = p + (z * z) / (2 * total);
+  const spread = z * Math.sqrt((p * (1 - p)) / total + (z * z) / (4 * total * total));
+  return Math.min(1, (centre + spread) / (1 + (z * z) / total));
+}
+
+/** 99% confident the pass rate is below the floor, after enough probes. */
+export const capabilityFlagged = (passes, total, floor, minimum = 20) => total >= minimum && wilsonUpper(passes, total) < floor;
 
 export function buildTask(kind, params) {
+  if (CAPABILITY_KINDS.includes(kind)) return { kind, capability: true, ...capabilityTask(kind, params) };
   if (kind === "multiply") return { kind, question: `What is ${params.a} multiplied by ${params.b}?`, reference: String(BigInt(params.a) * BigInt(params.b)), numeric: true };
   if (kind === "sum") {
     return { kind, question: `What is the sum of ${params.values.join(", ")}?`, reference: String(params.values.reduce((a, b) => a + Number(b), 0)), numeric: true };
@@ -26,6 +125,7 @@ export function buildTask(kind, params) {
 }
 
 export function grade(task, answer) {
+  if (task.capability) return gradeCapability(task, answer);
   const text = String(answer).trim();
   if (!text) return "unrelated";
   if (task.numeric) {
@@ -66,5 +166,5 @@ export function verifyProbeEvidence(evidence, deployment) {
   }
   const verdict = grade(task, outputText(JSON.parse(response.toString("utf8")).output));
   if (verdict !== evidence.verdict || verdict === "unrelated") throw new Error(`re-grading gives ${verdict}`);
-  return { provider: signed.authorization.provider_signer, verdict };
+  return { provider: signed.authorization.provider_signer, verdict, capability: Boolean(task.capability) };
 }

package/src/reputation.mjs

@@ -2,7 +2,7 @@
 import { encodeCall, keccak, hex } from "./eip712.mjs";
 import { rpcCall } from "./chain.mjs";
 import { evidenceHash } from "./disputes.mjs";
-import { verifyProbeEvidence } from "./probes.mjs";
+import { CAPABILITY_FLOORS, capabilityFlagged, verifyProbeEvidence } from "./probes.mjs";
 
 const PROBE_RECORDED = hex(keccak(Buffer.from("ProbeRecorded(address,address,bytes32,bytes32,uint8)")));
 const LOOKBACK_BLOCKS = 50_400n; // about 7 days of 12-second blocks
@@ -10,6 +10,9 @@ const LOG_CHUNK = 10_000n;
 const FRAUD_COOLDOWN = 30 * 86_400;
 const MAX_VERIFIED = 20; // evidence downloads per refresh
 
+// Ledger codes: basic probes 1/2, capability probes (is the advertised model really served?) 3/4.
+const VERDICT_CODES = { 1: { verdict: "pass", capability: false }, 2: { verdict: "wrong", capability: false },
+  3: { verdict: "pass", capability: true }, 4: { verdict: "wrong", capability: true } };
 const word = (raw, index) => BigInt(`0x${raw.slice(2 + index * 64, 2 + (index + 1) * 64)}`);
 const address = (topic) => `0x${topic.slice(26)}`.toLowerCase();
 
@@ -31,7 +34,7 @@ async function ledgerEvents(network) {
     for (const log of await rpcCall(network.rpc_urls, "eth_getLogs", [{ address: network.probe_ledger, topics: [PROBE_RECORDED],
       fromBlock: `0x${from.toString(16)}`, toBlock: `0x${to.toString(16)}` }])) {
       events.push({ provider: address(log.topics[1]), relay: address(log.topics[2]), key: log.topics[3],
-        evidence: `0x${log.data.slice(2, 66)}`, verdict: Number(BigInt(`0x${log.data.slice(66, 130)}`)) === 1 ? "pass" : "wrong" });
+        evidence: `0x${log.data.slice(2, 66)}`, ...VERDICT_CODES[Number(BigInt(`0x${log.data.slice(66, 130)}`))] });
     }
   }
   return events;
@@ -50,34 +53,50 @@ export async function reputation(consumer, descriptors, relaysByOwner) {
   let events = [];
   try { events = await ledgerEvents(network); } catch {}
   const byOwner = {};
-  for (const record of Object.values(records)) byOwner[record.owner] = { ...record, probes_passed: 0, probes_failed_verified: 0 };
+  for (const record of Object.values(records)) {
+    byOwner[record.owner] = { ...record, probes_passed: 0, probes_failed_verified: 0, capability_passed: 0, capability_failed_verified: 0 };
+  }
   let verified = 0;
   for (const event of events) {
     const entry = byOwner[event.provider];
-    if (!entry) continue;
-    if (event.verdict === "pass") { entry.probes_passed += 1; continue; }
+    if (!entry || !event.verdict) continue;
+    if (event.verdict === "pass") { entry[event.capability ? "capability_passed" : "probes_passed"] += 1; continue; }
     const relay = relaysByOwner[event.relay];
     if (!relay || verified >= MAX_VERIFIED) continue;
     verified += 1;
     try {
       const { status, body } = await consumer.fetchJson(`${relay.url}/v11/evidence/${event.evidence}`,
         { ca: network.tls_ca, pin: relay.pin, timeoutMs: 8_000 });
-      if (status === 200 && evidenceHash(body) === event.evidence && verifyProbeEvidence(body, consumer.deployment).verdict === "wrong") {
-        entry.probes_failed_verified += 1;
+      const checked = status === 200 && evidenceHash(body) === event.evidence ? verifyProbeEvidence(body, consumer.deployment) : null;
+      if (checked?.verdict === "wrong" && checked.capability === event.capability) {
+        entry[event.capability ? "capability_failed_verified" : "probes_failed_verified"] += 1;
       }
     } catch {}
   }
-  return Object.fromEntries(Object.entries(records).map(([signer, record]) => [signer, byOwner[record.owner]]));
+  return Object.fromEntries(descriptors.filter((d) => records[d.provider_signer]).map((descriptor) => {
+    const entry = { ...byOwner[records[descriptor.provider_signer].owner] };
+    const tier = Number(descriptor.tier || 0);
+    const floor = network.tiers?.[tier]?.capability_floor ?? CAPABILITY_FLOORS[tier];
+    const graded = entry.capability_passed + entry.capability_failed_verified;
+    // Passes are counted as reported; failures only once re-graded here, so no Relay can frame a Provider.
+    entry.downgrade_suspected = floor !== undefined && capabilityFlagged(entry.capability_passed, graded, floor);
+    entry.capability_floor = floor ?? null;
+    return [descriptor.provider_signer, entry];
+  }));
 }
 
-/** Lower is better: recent confirmed fraud last, then verified probe failure rate, then reputation, then price. */
+/**
+ * Lower is better: recent confirmed fraud last, then a suspected model downgrade, then the verified probe
+ * failure rate, then reputation, then price.
+ */
 export function rankKey(descriptor, record, now = Math.floor(Date.now() / 1000)) {
   const fraud = record && record.last_fraud_at && now < record.last_fraud_at + FRAUD_COOLDOWN ? 1 : 0;
+  const downgrade = record?.downgrade_suspected ? 1 : 0;
   const probes = record ? record.probes_passed + record.probes_failed_verified : 0;
   const failure = record && probes ? Math.round((10 * record.probes_failed_verified) / probes) : 0;
   const volume = record ? -Number(BigInt(record.counted_volume) / 1000n) : 0;
   const price = descriptor.prices.output_per_1k * 1e6 + descriptor.prices.input_per_1k;
-  return [fraud, failure, volume, price];
+  return [fraud, downgrade, failure, volume, price];
 }
 
 export function compareRank(a, b) {

package/src/web/console.html

@@ -381,13 +381,21 @@ async function refreshNetwork() {
     const card = document.createElement("div"); card.className = "card"; card.style.marginBottom = "12px";
     const reputation = (r) => r ? `${fmt(r.counted_volume)} · ${r.counterparties} 个付款方` : "—";
     const probes = (r) => r ? `通过 ${r.probes_passed} · 验证失败 ${r.probes_failed_verified}` : "—";
+    // Capability probes: hard tasks the advertised model passes and a cheaper substitute tends to miss.
+    const capability = (r) => {
+      if (!r) return "—";
+      const graded = r.capability_passed + r.capability_failed_verified;
+      if (!graded) return `<span class="muted">暂无</span>`;
+      const text = `${r.capability_passed}/${graded}`;
+      return r.downgrade_suspected ? `<span class="chip disputed" title="通过率在 99% 置信度下低于该档位下限 ${Math.round(r.capability_floor * 100)}%">疑似降级 ${text}</span>` : text;
+    };
     const fraud = (r) => r && r.last_fraud_at && Date.now() / 1000 < r.last_fraud_at + 30 * 86400 ? `<span class="chip disputed">近期欺诈</span>` : "无";
     card.innerHTML = `<div class="row" style="margin:0"><strong>${relay.url}</strong><span class="chip ${relay.healthy ? "released" : "disputed"}">${relay.healthy ? "在线" : "离线"}</span>
       <span class="muted mono">签名 ${short(relay.signer)}</span>${relay.pinned ? `<span class="chip">证书指纹已上链</span>` : ""}</div>
-      <table style="margin-top:8px"><thead><tr><th>Provider</th><th>模型</th><th>输入</th><th>输出</th><th>最低</th><th>信誉（计入成交）</th><th>探针</th><th>欺诈</th></tr></thead><tbody>${relay.providers.map((p) =>
+      <table style="margin-top:8px"><thead><tr><th>Provider</th><th>模型</th><th>输入</th><th>输出</th><th>最低</th><th>信誉（计入成交）</th><th>探针</th><th>能力探针</th><th>欺诈</th></tr></thead><tbody>${relay.providers.map((p) =>
         `<tr><td class="mono">${short(p.signer)}</td><td>${p.models.join(", ")}</td><td>${fmt(p.prices.input_per_1k)}</td><td>${fmt(p.prices.output_per_1k)}</td><td>${fmt(p.prices.minimum_fee)}</td>
-         <td>${reputation(p.reputation)}</td><td>${probes(p.reputation)}</td><td>${fraud(p.reputation)}</td></tr>`).join("")
-        || `<tr><td colspan="8" class="muted">没有已连接的 Provider</td></tr>`}</tbody></table>`;
+         <td>${reputation(p.reputation)}</td><td>${probes(p.reputation)}</td><td>${capability(p.reputation)}</td><td>${fraud(p.reputation)}</td></tr>`).join("")
+        || `<tr><td colspan="9" class="muted">没有已连接的 Provider</td></tr>`}</tbody></table>`;
     $("network").append(card);
   }
 }