@coinrithm/mcp-trading 0.3.0 → 0.4.0

package/CHANGELOG.md

@@ -5,6 +5,23 @@ ships two binaries — `coinrithm-mcp` (the MCP server) and `coinrithm-agent` (t
 self-host agent runner) — versioned together. The CoinRithm **API contract** is
 versioned separately (see `openapi.yaml` `info.version`, currently `1.4.0`).
 
+## 0.4.0
+
+- **Deterministic scorecard engine (`computeScorecard`).** The reproducible-
+  evaluation engine for `coinrithm.agent.scorecard.v1` — pure math over an
+  agent's realized track record (no network, no model): realized PnL, win rate,
+  expectancy, profit factor, reward-to-risk, Sharpe, Sortino, deflated /
+  probabilistic Sharpe (Bailey & López de Prado — skill vs luck with a multiple-
+  testing penalty), max drawdown, and Brier + ECE calibration for probabilistic
+  calls. Same inputs → identical metrics **and** a sha256 `contentHash` of the
+  canonicalized result, so a scorecard whose hash doesn't reproduce isn't
+  trusted. Metrics are computed AFTER the run from immutable evidence (leakage-
+  separation), so tuning-to-the-metric is structurally impossible. Returns
+  `null` for thin records — never a fabricated number.
+- **Resolver: committable file metadata + functionality pin.** The OKF resolver
+  now carries per-file metadata and pins functionality through resolution, so a
+  bundle's behavior is reproducible from its committed files.
+
 ## 0.3.0
 
 - **Agent risk config: coin deny-list (`blocklist`).** `risk.blocklist` lets an

package/dist/agent/resolve.js

@@ -48,6 +48,7 @@ const JOURNAL_MAX_LINES = 200;
 const JOURNAL_MAX_BYTES = 8_000;
 // Optional prose files (markdown the LLM reads), in assembly order.
 const PROSE_FILES = ["character/thesis.md", "character/persona.md"];
+const FUNCTIONALITY_PIN = "functionality/coinrithm.yaml";
 // Enforced cap field names. sizing.yaml is SOFT guidance and must NOT contain
 // any of these (or a user could think a limit binds when it does not).
 const ENFORCED_FIELD_NAMES = new Set([
@@ -58,6 +59,7 @@ const ENFORCED_FIELD_NAMES = new Set([
     "maxConsecutiveModelFailures",
     "onRateLimitPressure",
 ]);
+const SKILL_METADATA_KEYS = new Set(["type", "title", "description", "tags"]);
 // A $ref must be a LOCAL, RELATIVE path inside the agent folder — never a URL,
 // an absolute path, a home/drive path, or a Windows backslash path.
 function refSyntaxIssue(ref) {
@@ -391,6 +393,13 @@ function resolveDirectory(dir) {
             // a skill file may be pure prose (no frontmatter) — treat whole as body.
             body = readFileSync(abs, "utf8");
         }
+        for (const f of scanForSecrets(patch)) {
+            ctx.issues.push({
+                code: "secret_in_frontmatter",
+                path: refPath,
+                message: `${f} (skill frontmatter is committable metadata — remove secrets)`,
+            });
+        }
         includeOrder.push(name);
         skillProse.push({ source: refPath, text: body });
         applySkillPatch(ctx, rawFrontmatter, patch, refPath);
@@ -420,6 +429,23 @@ function resolveDirectory(dir) {
             });
         }
     }
+    // Optional API/tool contract pin. It is locked for reproducibility and stale
+    // warnings, but it is not part of AgentSpec and is never sent to the model.
+    const functionalityPath = join(dir, FUNCTIONALITY_PIN);
+    if (existsSync(functionalityPath)) {
+        const abs = safePath(ctx, FUNCTIONALITY_PIN, "functionality pin");
+        if (abs) {
+            const parsed = parseYamlSafe(ctx, readHashed(ctx, abs), FUNCTIONALITY_PIN);
+            for (const f of scanForSecrets(parsed)) {
+                ctx.issues.push({
+                    code: "secret_in_functionality",
+                    path: FUNCTIONALITY_PIN,
+                    message: `${f} (the functionality pin is committable metadata — remove secrets)`,
+                });
+            }
+            sources.functionality = FUNCTIONALITY_PIN;
+        }
+    }
     const mergedProse = mergeProseParts(proseParts);
     checkSizing(ctx, rawFrontmatter);
     scanSecrets(ctx, rawFrontmatter, mergedProse);
@@ -444,6 +470,8 @@ function resolveDirectory(dir) {
 // permitted, tighten-only; anything else is rejected (no permission expansion).
 function applySkillPatch(ctx, rawFrontmatter, patch, sourceLabel) {
     for (const key of Object.keys(patch)) {
+        if (SKILL_METADATA_KEYS.has(key))
+            continue;
         if (key === "risk" || key === "limits") {
             const caps = key === "risk" ? RISK_CAPS : LIMIT_CAPS;
             const base = rawFrontmatter[key] ?? {};

package/dist/agent/scorecard.d.ts

@@ -0,0 +1,24 @@
+export interface ScorecardInput {
+    realizedPnls: number[];
+    cumulative?: number[];
+    returns?: number[];
+    annualizationFactor?: number;
+    trials?: number;
+    predictions?: Array<{
+        p: number;
+        outcome: 0 | 1;
+    }>;
+    gates?: {
+        stopCoverage?: number;
+        evidenceCoverage?: number;
+        leakageClean?: boolean;
+    };
+}
+export interface Scorecard {
+    schema: "coinrithm.agent.scorecard.v1";
+    sampleSize: number;
+    returnsBasis: "returns" | "realized_pnl";
+    metrics: Record<string, number | null>;
+    contentHash: string;
+}
+export declare function computeScorecard(input: ScorecardInput): Scorecard;

package/dist/agent/scorecard.js

@@ -0,0 +1,177 @@
+// Deterministic scorecard engine — pure math over an agent's realized track
+// record. The reproducible-evaluation half of coinrithm.agent.scorecard.v1
+// (see examples/agents/_shared/scorecard.metrics.yaml + DECISIONS D17).
+//
+// DETERMINISM CONTRACT: the same inputs always yield the same metrics AND the
+// same contentHash (sha256 of the canonicalized result), mirroring
+// meta/manifest.lock.json. The engine NEVER calls the network or the model — it
+// reads the run-evidence ledger export + realized equity curve (fetched by the
+// caller) and computes, so tuning-to-the-metric is structurally impossible
+// (leakage separation, arXiv 2512.02227). Every function returns null when there
+// is too little data, so a thin record reports "n/a" rather than a fake number.
+//
+// SCIENTIFIC BASIS: risk-adjusted ratios (Sharpe/Sortino), skill-vs-luck
+// deflation (probabilistic + deflated Sharpe, Bailey & Lopez de Prado), and
+// calibration (Brier/ECE) for probabilistic calls — the reproducible-evaluation
+// layer the field lacks (arXiv 2605.19337).
+import { createHash } from "node:crypto";
+const round = (n, d = 6) => {
+    const f = 10 ** d;
+    return Math.round(n * f) / f;
+};
+const sum = (xs) => xs.reduce((a, b) => a + b, 0);
+const mean = (xs) => (xs.length ? sum(xs) / xs.length : 0);
+// Sample standard deviation (n-1). null for < 2 points (undefined dispersion).
+function sampleStd(xs) {
+    if (xs.length < 2)
+        return null;
+    const m = mean(xs);
+    const v = sum(xs.map((x) => (x - m) ** 2)) / (xs.length - 1);
+    return Math.sqrt(v);
+}
+// Downside deviation about a 0 minimum-acceptable-return (Sortino denominator).
+function downsideDev(xs) {
+    if (xs.length < 2)
+        return null;
+    const sq = xs.map((x) => (x < 0 ? x * x : 0));
+    return Math.sqrt(sum(sq) / xs.length);
+}
+// Population moments used by the (probabilistic) Sharpe formula.
+function moment(xs, k) {
+    const m = mean(xs);
+    return sum(xs.map((x) => (x - m) ** k)) / xs.length;
+}
+// Standard normal CDF via an Abramowitz & Stegun erf approximation (max err ~1e-7).
+function normalCdf(z) {
+    const t = 1 / (1 + 0.2316419 * Math.abs(z));
+    const d = 0.3989422804014327 * Math.exp(-(z * z) / 2);
+    const p = d *
+        t *
+        (0.31938153 +
+            t * (-0.356563782 + t * (1.781477937 + t * (-1.821255978 + t * 1.330274429))));
+    return z >= 0 ? 1 - p : p;
+}
+// Max peak-to-trough drawdown (mUSD, >= 0) on a cumulative series.
+function maxDrawdown(cumulative) {
+    let peak = -Infinity;
+    let maxDd = 0;
+    for (const c of cumulative) {
+        if (!Number.isFinite(c))
+            continue;
+        peak = Math.max(peak, c);
+        maxDd = Math.max(maxDd, peak - c);
+    }
+    return Number.isFinite(maxDd) ? maxDd : 0;
+}
+// Per-observation Sharpe (mean/std), the basis for the probabilistic SR test.
+function rawSharpe(rs) {
+    const sd = sampleStd(rs);
+    if (sd == null || sd === 0)
+        return null;
+    return mean(rs) / sd;
+}
+// Probabilistic / deflated Sharpe (Bailey & Lopez de Prado, approximated).
+// PSR(SR0) = Phi( (SR - SR0) * sqrt(n-1) / sqrt(1 - skew*SR + ((kurt-1)/4)*SR^2) ).
+// Deflation: SR0 = sqrt(2*ln(trials)) / sqrt(n) — the expected max per-obs Sharpe
+// of `trials` random strategies (extreme-value heuristic). trials=1 -> SR0=0, so
+// it reduces to the probabilistic Sharpe (already penalizing short, skewed tracks).
+function deflatedSharpe(rs, trials) {
+    const n = rs.length;
+    if (n < 3)
+        return null;
+    const sr = rawSharpe(rs);
+    if (sr == null)
+        return null;
+    const m2 = moment(rs, 2);
+    if (m2 === 0)
+        return null;
+    const skew = moment(rs, 3) / m2 ** 1.5;
+    const kurt = moment(rs, 4) / m2 ** 2; // 3 for a normal distribution
+    const denom = Math.sqrt(Math.max(1e-9, 1 - skew * sr + ((kurt - 1) / 4) * sr * sr));
+    const sr0 = Math.sqrt(2 * Math.log(Math.max(1, trials))) / Math.sqrt(n);
+    const z = ((sr - sr0) * Math.sqrt(n - 1)) / denom;
+    return normalCdf(z);
+}
+// Brier score: mean((p - outcome)^2). Lower = better-calibrated.
+function brier(preds) {
+    if (preds.length === 0)
+        return null;
+    return mean(preds.map((x) => (x.p - x.outcome) ** 2));
+}
+// Expected calibration error over 10 equal-width probability buckets.
+function ece(preds) {
+    if (preds.length === 0)
+        return null;
+    const buckets = 10;
+    let total = 0;
+    for (let b = 0; b < buckets; b += 1) {
+        const lo = b / buckets;
+        const hi = (b + 1) / buckets;
+        const inB = preds.filter((x) => (b === buckets - 1 ? x.p >= lo && x.p <= hi : x.p >= lo && x.p < hi));
+        if (inB.length === 0)
+            continue;
+        const avgP = mean(inB.map((x) => x.p));
+        const avgO = mean(inB.map((x) => x.outcome));
+        total += (inB.length / preds.length) * Math.abs(avgP - avgO);
+    }
+    return total;
+}
+export function computeScorecard(input) {
+    const pnls = input.realizedPnls.filter(Number.isFinite);
+    const n = pnls.length;
+    const wins = pnls.filter((x) => x > 0);
+    const losses = pnls.filter((x) => x < 0);
+    const decided = wins.length + losses.length;
+    const grossWin = sum(wins);
+    const grossLoss = Math.abs(sum(losses));
+    const avgWin = wins.length ? grossWin / wins.length : 0;
+    const avgLoss = losses.length ? grossLoss / losses.length : 0;
+    const pWin = decided ? wins.length / decided : null;
+    const rs = input.returns && input.returns.length ? input.returns.filter(Number.isFinite) : pnls;
+    const returnsBasis = input.returns && input.returns.length ? "returns" : "realized_pnl";
+    const ann = input.annualizationFactor ?? 1;
+    const cumulative = input.cumulative && input.cumulative.length
+        ? input.cumulative
+        : pnls.reduce((acc, x) => {
+            acc.push((acc.length ? acc[acc.length - 1] : 0) + x);
+            return acc;
+        }, []);
+    const sd = sampleStd(rs);
+    const dd = downsideDev(rs);
+    const sharpe = sd && sd !== 0 ? (mean(rs) / sd) * ann : null;
+    const sortino = dd && dd !== 0 ? (mean(rs) / dd) * ann : null;
+    const metrics = {
+        realized_pnl_musd: round(sum(pnls)),
+        trade_count: n,
+        decided_count: decided,
+        win_rate: pWin == null ? null : round(pWin),
+        expectancy_musd: pWin == null ? null : round(pWin * avgWin - (1 - pWin) * avgLoss),
+        profit_factor: grossLoss > 0 ? round(grossWin / grossLoss) : grossWin > 0 ? null : 0, // null = ∞ (no losses)
+        reward_to_risk: avgLoss > 0 ? round(avgWin / avgLoss) : null,
+        sharpe: sharpe == null ? null : round(sharpe),
+        sortino: sortino == null ? null : round(sortino),
+        deflated_sharpe: round0(deflatedSharpe(rs, input.trials ?? 1)),
+        max_drawdown_musd: round(maxDrawdown(cumulative)),
+        brier_score: round0(brier(input.predictions ?? [])),
+        calibration_error: round0(ece(input.predictions ?? [])),
+        stop_coverage: input.gates?.stopCoverage ?? null,
+        evidence_coverage: input.gates?.evidenceCoverage ?? null,
+        leakage_clean: input.gates?.leakageClean == null ? null : input.gates.leakageClean ? 1 : 0,
+    };
+    // Canonicalize (sorted keys) and hash, so the report card carries a stable,
+    // verifiable fingerprint — a scorecard whose hash does not reproduce is not trusted.
+    const canonical = JSON.stringify(Object.keys(metrics)
+        .sort()
+        .map((k) => [k, metrics[k]]));
+    const contentHash = createHash("sha256").update(canonical).digest("hex");
+    return {
+        schema: "coinrithm.agent.scorecard.v1",
+        sampleSize: n,
+        returnsBasis,
+        metrics,
+        contentHash,
+    };
+}
+function round0(n) {
+    return n == null ? null : round(n);
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@coinrithm/mcp-trading",
-  "version": "0.3.0",
+  "version": "0.4.0",
   "mcpName": "io.github.CoinRithm/mcp-trading",
   "description": "CoinRithm paper-trading toolkit: an MCP server (coinrithm-mcp) AND a self-host agent runner (coinrithm-agent) for spot, futures, and prediction markets with a user-minted API key.",
   "type": "module",