canli-validation-mcp 0.14.1 → 0.16.0

package/README.md

@@ -116,18 +116,35 @@ and one column per variant, as `matrix` or as `matrix_file` (a CSV or JSON on yo
 optional `benchmark` series; with none, variants are tested against zero. It runs three tests on one
 seeded stationary bootstrap:
 
-- Hansen's SPA: the chance that the best variant's studentized excess return is this good if no
-  variant has an edge, with lower and upper bounds (`spa.p_value`, the consistent p-value, is the
-  headline, and `monte_carlo_se` its sampling error);
+- Hansen's SPA: the chance that the best variant's studentized excess return is at least this good
+  if no variant has an edge, with lower and upper bounds (`spa.p_value`, the consistent p-value, is
+  the headline, and `monte_carlo_se` its sampling error);
 - White's Reality Check: the same question without studentizing, so one volatile variant can
   dominate it;
 - Romano and Wolf's StepM: which variants beat the benchmark, with the familywise error held at
   `alpha`.
 
-A result reproduces exactly from its `seed`. On three fixed-seed cases the p-values agree with
-Python's `arch` 8.0 and with a numpy transcription of Hansen's formulas within Monte Carlo error, and
-the StepM sets match `arch`'s (`js/snooping-core.test.js`). Only the variants sent are counted: a
-search that tried more than it sends makes luck look smaller than it was.
+Studentizing divides a variant's mean excess return by its standard deviation. Each bootstrap
+resample is divided by its own standard deviation, worked out again from that resample, as Romano
+and Wolf (2005) recommend, and SPA and StepM both use these figures. Earlier versions divided every
+resample by one figure estimated once from your data. That ignores how noisy the estimate is, and
+the test rejected too often: on Null Zoo v1's searches with no edge (20 variants, 504 days, 1,000
+resamples) it rejected 7.5% at a nominal 5% with normal returns and 12.5% with negatively skewed
+returns. On 4,000 new searches of each kind the new version rejected 5.1% and 5.4% (6.9% with
+autocorrelated returns, where the Reality Check also rejected 6.6%). Studentizing helps find a
+skilled variant that is less volatile than the rest and costs power when the skilled variant is the
+most volatile; the Reality Check, which does not studentize, is reported beside it for that case.
+
+A p-value is the share of resamples at least as extreme as your result, ties included. When no
+variant beats the benchmark, the SPA statistic is 0, so its p-values are 1 and StepM names no
+variant. Earlier versions counted only resamples strictly above 0 and could report a p-value near 0,
+a rejection, in exactly that case.
+
+A result reproduces exactly from its `seed`. On four fixed-seed cases the p-values agree within
+Monte Carlo error with Python's `arch` 8.0 (run without studentizing) and with a separate numpy
+implementation of the per-resample studentization, and the StepM sets match both
+(`js/snooping-core.test.js`). Only the variants sent are counted: a search that tried more than it
+sends makes luck look smaller than it was.
 
 `company_financial_history` is different from the other tools: it reads the public company
 reference at canlicapital.com, not the validation API. Give it a CIK (1 to 10 digits) to list a

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "canli-validation-mcp",
-  "version": "0.14.1",
+  "version": "0.16.0",
   "description": "Check whether a backtest is real: deflated Sharpe, CSCV probability of backtest overfitting, minimum track record and backtest length, haircut Sharpe and luck-equivalent trials, as an MCP server. Local mode, hosted endpoint, signed receipts.",
   "keywords": [
     "mcp",

package/src/local/js/backtest-core.js

@@ -456,6 +456,9 @@ export function runBacktest({ prices, family, grid = {}, periods_per_year: ppy =
       top: shown.map((r) => [...spec.params.map((k) => r.params[k]), r.metrics.sharpe, r.metrics.cagr, r.metrics.max_drawdown, r.metrics.turnover_per_year]),
     },
     selected: { params: best.params, metrics: best.metrics },
+    // Every variant's annualized Sharpe, best first as in top, so a server that counts trials across a session
+    // (canli-mcp's luck) counts each variant run and not one, as with canli-quant-mcp's strategy_sweep.
+    every_trial: { sharpes: ranked.filter((r) => r.metrics.sharpe !== null).map((r) => r.metrics.sharpe), periods: evaluated, periods_per_year: Number(ppy) },
     buy_and_hold: performance(buyHold.returns, ppy),
     validation,
     plain_reading: reading(family, runs.length, best, validation, performance(buyHold.returns, ppy)),

package/src/local/js/snooping-core.js

@@ -7,18 +7,29 @@
 //   White's Reality Check (2000)     the best mean excess return against the bootstrap distribution
 //                                    of the best recentred mean. Not studentized, so one volatile
 //                                    variant can dominate it.
-//   Hansen's SPA (2005)              the best t-statistic, each variant studentized by the
-//                                    Politis-Romano variance of its mean. Three recentrings give a
-//                                    lower bound, a consistent p-value and an upper bound: poor
-//                                    variants are not allowed to inflate the p-value.
+//   Hansen's SPA (2005)              the best t-statistic. Three recentrings give a lower bound, a
+//                                    consistent p-value and an upper bound: poor variants are not
+//                                    allowed to inflate the p-value.
 //   Romano and Wolf's StepM (2005)   step-down: which variants beat the benchmark, controlling the
 //                                    familywise error at alpha.
 //
+// Studentizing is a bootstrap-t. The observed t-statistic divides each variant's mean by its sample
+// standard deviation, and every resample divides its recentred means by that resample's own
+// standard deviations, as in Romano and Wolf's (2005) studentized StepM. Dividing every resample by
+// one variance estimated once from the sample ignores that estimate's noise and its downward bias,
+// and the test rejected too often (Null Zoo v1: 7.5% of skill-less searches at a nominal 5% with
+// i.i.d. normal returns, 12.5% with negative skew). The Politis-Romano variance still sets Hansen's
+// consistent-recentring threshold.
+//
+// p-values count ties, #(T* >= T) / reps. The SPA statistic is floored at zero, so when no variant
+// beats the benchmark T = 0 and p = 1; a strict inequality counted only the few positive resamples
+// there and rejected. StepM's critical value is floored at zero for the same reason.
+//
 // d[t][k] is variant k's return minus the benchmark's in period t; higher is better. Every draw
-// comes from makeRandom(seed), so a result reproduces exactly from its inputs. The reference
-// implementation it is checked against (Python's arch 8.0, and a separate numpy transcription of
-// Hansen's formulas for the studentized SPA) draws with numpy, so agreement is within Monte Carlo
-// error, not to the digit; scripts/research/reality-check/ holds the comparison.
+// comes from makeRandom(seed), so a result reproduces exactly from its inputs. The implementations
+// it is checked against (Python's arch 8.0 without studentizing, and a separate numpy transcription
+// of the bootstrap-t) draw with numpy, so agreement is within Monte Carlo error, not to the digit;
+// scripts/research/reality-check/ holds the comparison.
 // =============================================================================
 
 import { makeRandom } from "./selection-risk-core.js";
@@ -77,8 +88,9 @@ export function bootstrapVariance(d, n, k, block, means) {
 
 /**
  * All three tests on one bootstrap. `d` is a row-major Float64Array of n periods by k variants.
- * Returns means, the variance estimates, the bootstrap maxima under each recentring and the
- * StepM rejections; p-values are shares of `reps` draws strictly above the observed statistic.
+ * Returns means, the Politis-Romano variances, the sample standard deviations, the p-values and the
+ * StepM rejections. A p-value is the share of `reps` draws at least as large as the observed
+ * statistic (the Reality Check's, which is not floored, strictly above it, as in arch).
  */
 export function snoopingTests(d, n, k, { block = defaultBlock(n), reps = 1000, seed = 42, alpha = 0.05, studentize = true } = {}) {
   if (!(n >= 2 && k >= 1)) throw new RangeError("need at least 2 periods and 1 variant");
@@ -89,58 +101,72 @@ export function snoopingTests(d, n, k, { block = defaultBlock(n), reps = 1000, s
   for (let t = 0; t < n; t += 1) for (let j = 0; j < k; j += 1) means[j] += d[t * k + j];
   for (let j = 0; j < k; j += 1) means[j] /= n;
   const variance = bootstrapVariance(d, n, k, block, means);
-  const sd = new Float64Array(k);
-  for (let j = 0; j < k; j += 1) sd[j] = Math.sqrt(Math.max(variance[j], 0));
+  // Sample variances (divisor n - 1), the observed statistic's studentizer.
+  const sampleVar = new Float64Array(k);
+  for (let t = 0; t < n; t += 1) for (let j = 0; j < k; j += 1) sampleVar[j] += (d[t * k + j] - means[j]) ** 2;
+  for (let j = 0; j < k; j += 1) sampleVar[j] /= n - 1;
+  const sd = Float64Array.from(sampleVar, Math.sqrt);
   // A variant whose excess return never varies has no sampling distribution to studentize by.
-  const usable = Array.from(sd, (s) => s > 0);
+  const usable = Array.from(sd, (s, j) => s > 0 && variance[j] > 0);
   if (!usable.some(Boolean)) throw new RangeError("every variant's excess return is constant; there is nothing to test");
   const root = Math.sqrt(n);
-  const scale = (j) => (studentize ? root / sd[j] : 1);
   // Hansen's recentrings: lower keeps poor variants at zero, consistent drops only those far below
-  // zero, upper recentres everything (White's null, the least favourable configuration).
+  // zero, upper recentres everything (White's null, the least favourable configuration). A resample
+  // mean recentred is (resample mean - sample mean) + offset, with offset = mean - centre.
   const threshold = (j) => -Math.sqrt((variance[j] / n) * 2 * Math.log(Math.log(n)));
-  const centre = [
-    Float64Array.from(means, (m) => Math.max(m, 0)),
-    Float64Array.from(means, (m, j) => (m >= threshold(j) ? m : 0)),
-    Float64Array.from(means),
+  const offset = [
+    Float64Array.from(means, (m) => Math.min(m, 0)),
+    Float64Array.from(means, (m, j) => (m >= threshold(j) ? 0 : m)),
+    new Float64Array(k),
   ];
+  const statistic = (j) => (studentize ? (root * means[j]) / sd[j] : means[j]);
   let observed = Number.NEGATIVE_INFINITY;
   let best = -1;
   for (let j = 0; j < k; j += 1) {
     if (!usable[j]) continue;
-    const s = means[j] * scale(j);
+    const s = statistic(j);
     if (s > observed) { observed = s; best = j; }
   }
   let rcObserved = Number.NEGATIVE_INFINITY;
   for (let j = 0; j < k; j += 1) if (usable[j]) rcObserved = Math.max(rcObserved, means[j]);
 
+  const spaStat = Math.max(observed, 0);
+  const exceed = [0, 0, 0];
+  let rcExceed = 0;
   const next = makeRandom(seed);
   const idx = new Int32Array(n);
-  const star = new Float64Array(k);
-  const boot = new Float64Array(reps * k);   // bootstrap mean of each variant, per draw
+  const sum = new Float64Array(k);    // a resample's sum of deviations from the sample mean
+  const sumSq = new Float64Array(k);  // and of their squares
+  const stepT = new Float64Array(reps * k);   // each variant's consistent-recentred statistic, per draw
   for (let r = 0; r < reps; r += 1) {
     stationaryIndices(n, block, next, idx);
-    star.fill(0);
+    sum.fill(0);
+    sumSq.fill(0);
     for (let t = 0; t < n; t += 1) {
       const row = idx[t] * k;
-      for (let j = 0; j < k; j += 1) star[j] += d[row + j];
+      for (let j = 0; j < k; j += 1) {
+        const x = d[row + j] - means[j];
+        sum[j] += x;
+        sumSq[j] += x * x;
+      }
     }
-    for (let j = 0; j < k; j += 1) boot[r * k + j] = star[j] / n;
-  }
-
-  const spaStat = Math.max(observed, 0);
-  const exceed = [0, 0, 0];
-  let rcExceed = 0;
-  for (let r = 0; r < reps; r += 1) {
     const maxima = [Number.NEGATIVE_INFINITY, Number.NEGATIVE_INFINITY, Number.NEGATIVE_INFINITY];
     let rcMax = Number.NEGATIVE_INFINITY;
     for (let j = 0; j < k; j += 1) {
       if (!usable[j]) continue;
-      const m = boot[r * k + j];
-      for (let c = 0; c < 3; c += 1) maxima[c] = Math.max(maxima[c], (m - centre[c][j]) * scale(j));
-      rcMax = Math.max(rcMax, m - means[j]);
+      const shift = sum[j] / n;
+      let scale = 1;
+      if (studentize) {
+        // The resample's own standard deviation. A variant constant in this resample (it differs
+        // from the benchmark in few periods and the resample missed them) keeps the sample's.
+        const v = (sumSq[j] - sum[j] * shift) / (n - 1);
+        scale = root / (v > Number.EPSILON * sampleVar[j] ? Math.sqrt(v) : sd[j]);
+      }
+      for (let c = 0; c < 3; c += 1) maxima[c] = Math.max(maxima[c], (shift + offset[c][j]) * scale);
+      stepT[r * k + j] = (shift + offset[1][j]) * scale;
+      rcMax = Math.max(rcMax, shift);
     }
-    for (let c = 0; c < 3; c += 1) if (Math.max(maxima[c], 0) > spaStat) exceed[c] += 1;
+    for (let c = 0; c < 3; c += 1) if (Math.max(maxima[c], 0) >= spaStat) exceed[c] += 1;
     if (rcMax > rcObserved) rcExceed += 1;
   }
 
@@ -155,12 +181,14 @@ export function snoopingTests(d, n, k, { block = defaultBlock(n), reps = 1000, s
     const maxima = new Float64Array(reps);
     for (let r = 0; r < reps; r += 1) {
       let m = Number.NEGATIVE_INFINITY;
-      for (const j of live) m = Math.max(m, (boot[r * k + j] - centre[1][j]) * scale(j));
+      for (const j of live) m = Math.max(m, stepT[r * k + j]);
       maxima[r] = m;
     }
     maxima.sort();
-    const critical = quantile(maxima, 1 - alpha);
-    const now = live.filter((j) => means[j] * scale(j) > critical);
+    // At least zero, as the SPA statistic is: a variant that does not beat the benchmark in sample
+    // is never declared superior. It binds only when every live variant is far below zero.
+    const critical = Math.max(quantile(maxima, 1 - alpha), 0);
+    const now = live.filter((j) => statistic(j) > critical);
     if (!now.length) break;
     for (const j of now) rejected.add(j);
     rounds.push({ critical, rejected: now });
@@ -168,7 +196,7 @@ export function snoopingTests(d, n, k, { block = defaultBlock(n), reps = 1000, s
 
   return {
     n, k, block, reps, seed, alpha, studentize,
-    means, variance, usable,
+    means, variance, sd, usable,
     best, observed,
     spa: { statistic: spaStat, p_lower: exceed[0] / reps, p_consistent: exceed[1] / reps, p_upper: exceed[2] / reps },
     reality_check: { statistic: rcObserved, p_value: rcExceed / reps },