canli-validation-mcp 0.4.0 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +80 -12
- package/package.json +6 -3
- package/src/local/api/_lib/limits.js +30 -0
- package/src/local/js/breadth-core.js +96 -0
- package/src/local/js/dsr-core.js +302 -0
- package/src/local/js/haircut-core.js +96 -0
- package/src/local/js/luck-core.js +97 -0
- package/src/local/js/moments-core.js +54 -0
- package/src/local/js/paper-evidence-core.js +167 -0
- package/src/local/js/pbo-core.js +112 -0
- package/src/local/js/receipt-statement.js +54 -0
- package/src/local/js/selection-risk-core.js +197 -0
- package/src/local/js/student-t.js +139 -0
- package/src/local/js/validate/backtest-length.js +61 -0
- package/src/local/js/validate/breadth.js +24 -0
- package/src/local/js/validate/deflated-sharpe.js +28 -0
- package/src/local/js/validate/haircut-sharpe.js +54 -0
- package/src/local/js/validate/luck-trials.js +63 -0
- package/src/local/js/validate/overfitting.js +29 -0
- package/src/local/js/validate/paper-evidence.js +11 -0
- package/src/local/js/validate/track-record.js +50 -0
- package/src/local/scripts/canonical-json.mjs +176 -0
- package/src/local/standards/paper-evidence/schema.json +429 -0
- package/src/local.mjs +45 -0
- package/src/receipt-keys.json +13 -0
- package/src/schemas.mjs +195 -51
- package/src/series-file.mjs +124 -0
- package/src/server.mjs +349 -10
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
// The haircut Sharpe ratio: Harvey and Liu, "Backtesting", Journal of Portfolio Management, 2015.
|
|
2
|
+
// A Sharpe ratio found among several tests is converted to a t-statistic, its p-value is adjusted
|
|
3
|
+
// for the number of tests, and the adjusted p-value is converted back to the Sharpe ratio a single
|
|
4
|
+
// test would have needed: the haircut Sharpe ratio. Checked against the authors' own Haircut_SR.m
|
|
5
|
+
// (run in GNU Octave) and against R's p.adjust in js/haircut-paper-vectors.test.js.
|
|
6
|
+
import { studentTQuantileUpper, studentTUpper } from "./student-t.js";
|
|
7
|
+
|
|
8
|
+
// Lo (2002) for returns with first-order autocorrelation rho sampled q times a year: the annualized
|
|
9
|
+
// Sharpe ratio is scaled by [1 + (2 rho / (1 - rho)) (1 - (1 - rho^q) / (q (1 - rho)))]^(-1/2), the
|
|
10
|
+
// form Haircut_SR.m applies to an annualized input.
|
|
11
|
+
export function autocorrelationFactor(rho, periodsPerYear) {
|
|
12
|
+
const r = Number(rho);
|
|
13
|
+
const q = Number(periodsPerYear);
|
|
14
|
+
if (!(r > -1 && r < 1)) throw new RangeError("autocorrelation must be strictly between -1 and 1");
|
|
15
|
+
if (r === 0) return 1;
|
|
16
|
+
const inner = 1 + ((2 * r) / (1 - r)) * (1 - (1 - r ** q) / (q * (1 - r)));
|
|
17
|
+
if (!(inner > 0)) throw new RangeError("This autocorrelation gives no valid annualization factor");
|
|
18
|
+
return inner ** -0.5;
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
function requireCount(name, value, min) {
|
|
22
|
+
const n = Number(value);
|
|
23
|
+
if (!Number.isInteger(n) || n < min) throw new RangeError(`${name} must be an integer of at least ${min}`);
|
|
24
|
+
return n;
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
// Holm (step-down) and Benjamini-Hochberg-Yekutieli adjusted p-values for one member of a family,
|
|
28
|
+
// computed exactly as R's p.adjust computes them (stable order, ties by position).
|
|
29
|
+
export function familyAdjusted(pValues, index) {
|
|
30
|
+
const n = pValues.length;
|
|
31
|
+
const ascending = pValues.map((p, i) => [p, i]).sort((a, b) => a[0] - b[0] || a[1] - b[1]);
|
|
32
|
+
const holm = new Array(n);
|
|
33
|
+
let runningMax = 0;
|
|
34
|
+
ascending.forEach(([p, i], k) => {
|
|
35
|
+
runningMax = Math.max(runningMax, (n - k) * p);
|
|
36
|
+
holm[i] = Math.min(1, runningMax);
|
|
37
|
+
});
|
|
38
|
+
const harmonic = pValues.reduce((sum, _, k) => sum + 1 / (k + 1), 0);
|
|
39
|
+
const descending = pValues.map((p, i) => [p, i]).sort((a, b) => b[0] - a[0] || b[1] - a[1]);
|
|
40
|
+
const bhy = new Array(n);
|
|
41
|
+
let runningMin = Infinity;
|
|
42
|
+
descending.forEach(([p, i], k) => {
|
|
43
|
+
const rank = n - k;
|
|
44
|
+
runningMin = Math.min(runningMin, ((harmonic * n) / rank) * p);
|
|
45
|
+
bhy[i] = Math.min(1, runningMin);
|
|
46
|
+
});
|
|
47
|
+
return { holm: holm[index], bhy: bhy[index] };
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
export function haircutSharpe({ sharpeAnnualized, periodsPerYear, observations, tests, autocorrelation = 0, otherSharpesAnnualized }) {
|
|
51
|
+
const sharpe = Number(sharpeAnnualized);
|
|
52
|
+
const q = Number(periodsPerYear);
|
|
53
|
+
if (!(sharpe > 0 && Number.isFinite(sharpe))) throw new RangeError("The haircut applies to a positive, finite Sharpe ratio");
|
|
54
|
+
if (!(q > 0 && q <= 10000)) throw new RangeError("periods_per_year must be greater than 0 and at most 10000");
|
|
55
|
+
const T = requireCount("observations", observations, 3);
|
|
56
|
+
const others = otherSharpesAnnualized === undefined ? null : otherSharpesAnnualized.map(Number);
|
|
57
|
+
if (others && !others.every(Number.isFinite)) throw new RangeError("Every other Sharpe ratio must be a finite number");
|
|
58
|
+
const m = others ? others.length + 1 : requireCount("tests", tests, 1);
|
|
59
|
+
if (others && tests !== undefined && Number(tests) !== m) throw new RangeError("tests must equal the number of other Sharpe ratios plus one, or be left out");
|
|
60
|
+
|
|
61
|
+
const factor = autocorrelationFactor(autocorrelation, q);
|
|
62
|
+
const df = T - 1;
|
|
63
|
+
const tOf = (annual) => ((annual * factor) / Math.sqrt(q)) * Math.sqrt(T);
|
|
64
|
+
// Two-sided, as in Haircut_SR.m, but from the upper tail directly so that a large t keeps its p-value.
|
|
65
|
+
const pOf = (annual) => {
|
|
66
|
+
const t = tOf(annual);
|
|
67
|
+
return 2 * (t >= 0 ? studentTUpper(t, df) : studentTUpper(-t, df));
|
|
68
|
+
};
|
|
69
|
+
const srCorrected = sharpe * factor;
|
|
70
|
+
const tStat = tOf(sharpe);
|
|
71
|
+
const pSingle = pOf(sharpe);
|
|
72
|
+
const invert = (adjustedP) => {
|
|
73
|
+
const p = Math.min(1, adjustedP);
|
|
74
|
+
const tAdjusted = p >= 1 ? 0 : studentTQuantileUpper(p / 2, df);
|
|
75
|
+
const haircutSharpe = (tAdjusted / Math.sqrt(T)) * Math.sqrt(q);
|
|
76
|
+
return { adjusted_p: p, haircut_sharpe_annualized: haircutSharpe, haircut: (srCorrected - haircutSharpe) / srCorrected };
|
|
77
|
+
};
|
|
78
|
+
const result = {
|
|
79
|
+
sharpe_annualized_corrected: srCorrected,
|
|
80
|
+
autocorrelation_factor: factor,
|
|
81
|
+
t_statistic: tStat,
|
|
82
|
+
degrees_of_freedom: df,
|
|
83
|
+
p_value_single: pSingle,
|
|
84
|
+
tests: m,
|
|
85
|
+
bonferroni: invert(m * pSingle),
|
|
86
|
+
// Harvey and Liu's Eq. 4 for independent tests: 1 - (1 - p)^M.
|
|
87
|
+
independent: invert(-Math.expm1(m * Math.log1p(-pSingle))),
|
|
88
|
+
};
|
|
89
|
+
if (others) {
|
|
90
|
+
const family = [pSingle, ...others.map(pOf)];
|
|
91
|
+
const adjusted = familyAdjusted(family, 0);
|
|
92
|
+
result.holm = invert(adjusted.holm);
|
|
93
|
+
result.bhy = invert(adjusted.bhy);
|
|
94
|
+
}
|
|
95
|
+
return result;
|
|
96
|
+
}
|
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
// Luck-equivalent trials: how many skill-less strategies a search would have had to try for the best
|
|
2
|
+
// of them to reach the observed Sharpe ratio by luck alone. A reviewer's statistic, stated in the
|
|
3
|
+
// unit a research log records (trials), assembled from known results:
|
|
4
|
+
//
|
|
5
|
+
// - Under the null of no skill and normal returns, the Sharpe ratio's t-statistic, SR * sqrt(T) with
|
|
6
|
+
// the Sharpe per period, is exactly Student t with T - 1 degrees of freedom, so one trial reaches
|
|
7
|
+
// the observed Sharpe with probability p1 = P(t_{T-1} >= SR sqrt(T)).
|
|
8
|
+
// - The best of N independent skill-less trials reaches it with probability 1 - (1 - p1)^N (Sidak),
|
|
9
|
+
// so the N at which that probability equals q is N_q = ln(1 - q) / ln(1 - p1).
|
|
10
|
+
// - The expected maximum of N standard normals (Bailey, Borwein, Lopez de Prado and Zhu 2014), the
|
|
11
|
+
// deflated Sharpe ratio's benchmark, gives the N whose best is expected to reach it.
|
|
12
|
+
//
|
|
13
|
+
// Calibrated by Monte Carlo in js/luck-core.test.js; the full size study, with fixed seeds, is
|
|
14
|
+
// scripts/research/luck-trials-size-study.mjs and its output config/research/luck-trials-size-study.json.
|
|
15
|
+
// The size is correct for normal and for symmetric fat-tailed (Student t4) returns, conservative for
|
|
16
|
+
// positively skewed returns, and too small a count, so too kind to the strategy, for negatively
|
|
17
|
+
// skewed returns: with 252 observations a nominal 5% test rejected 10.8% of skill-less searches at
|
|
18
|
+
// skew -1.3 and 20.8% at skew -3.7.
|
|
19
|
+
import { expectedMaxStandardNormal } from "./dsr-core.js";
|
|
20
|
+
import { autocorrelationFactor } from "./haircut-core.js";
|
|
21
|
+
import { studentTUpper } from "./student-t.js";
|
|
22
|
+
|
|
23
|
+
export const TRIAL_CAP = 1e15;
|
|
24
|
+
|
|
25
|
+
function requirePositiveInteger(name, value, min) {
|
|
26
|
+
const n = Number(value);
|
|
27
|
+
if (!Number.isInteger(n) || n < min) throw new RangeError(`${name} must be an integer of at least ${min}`);
|
|
28
|
+
return n;
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
// P(one skill-less trial shows a Sharpe at least this high): the Student t upper tail.
|
|
32
|
+
export function singleTrialProbability({ sharpe, observations, periodsPerYear }) {
|
|
33
|
+
const sr = Number(sharpe);
|
|
34
|
+
const periods = Number(periodsPerYear);
|
|
35
|
+
const t = requirePositiveInteger("observations", observations, 3);
|
|
36
|
+
if (!Number.isFinite(sr)) throw new RangeError("The Sharpe ratio must be a finite number");
|
|
37
|
+
if (!(periods > 0 && Number.isFinite(periods))) throw new RangeError("periods per year must be a positive number");
|
|
38
|
+
const tStatistic = (sr / Math.sqrt(periods)) * Math.sqrt(t);
|
|
39
|
+
return { t_statistic: tStatistic, probability: studentTUpper(tStatistic, t - 1) };
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
// The N at which the best of N skill-less trials reaches the Sharpe with probability q, as a real
|
|
43
|
+
// number. Below 1 means a single trial already reaches it with probability above q; the count is
|
|
44
|
+
// capped at TRIAL_CAP, beyond which the tail probability is below what double precision resolves.
|
|
45
|
+
export function trialsAtProbability(probability, q) {
|
|
46
|
+
const p = Number(probability);
|
|
47
|
+
const level = Number(q);
|
|
48
|
+
if (!(level > 0 && level < 1)) throw new RangeError("The probability level must be strictly between 0 and 1");
|
|
49
|
+
if (!(p >= 0 && p <= 1)) throw new RangeError("The single-trial probability must be between 0 and 1");
|
|
50
|
+
if (p === 0) return TRIAL_CAP;
|
|
51
|
+
return Math.min(TRIAL_CAP, Math.log1p(-level) / Math.log1p(-p));
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
// P(the best of N skill-less trials shows a Sharpe at least this high): 1 - (1 - p1)^N.
|
|
55
|
+
export function bestOfTrialsProbability(probability, trials) {
|
|
56
|
+
const n = requirePositiveInteger("trials", trials, 1);
|
|
57
|
+
return -Math.expm1(n * Math.log1p(-Number(probability)));
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
// The largest N whose best skill-less trial is expected (the deflated Sharpe ratio's approximation)
|
|
61
|
+
// to stay at or below the t-statistic; 1 when even two are expected to exceed it.
|
|
62
|
+
export function expectedMaximumTrials(tStatistic) {
|
|
63
|
+
const ceiling = Number(tStatistic);
|
|
64
|
+
const fits = (n) => expectedMaxStandardNormal(n) <= ceiling;
|
|
65
|
+
if (!fits(2)) return 1;
|
|
66
|
+
let lo = 2;
|
|
67
|
+
let hi = 4;
|
|
68
|
+
while (fits(hi)) {
|
|
69
|
+
lo = hi;
|
|
70
|
+
if (hi >= TRIAL_CAP) return TRIAL_CAP;
|
|
71
|
+
hi = Math.min(hi * 2, TRIAL_CAP);
|
|
72
|
+
}
|
|
73
|
+
while (hi - lo > 1) {
|
|
74
|
+
const mid = Math.floor((lo + hi) / 2);
|
|
75
|
+
if (fits(mid)) lo = mid;
|
|
76
|
+
else hi = mid;
|
|
77
|
+
}
|
|
78
|
+
return lo;
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
// With the returns' lag-1 autocorrelation, the Sharpe is first corrected as Lo (2002), the form
|
|
82
|
+
// js/haircut-core.js applies; the Null Zoo measured that without it, positively autocorrelated
|
|
83
|
+
// returns (0.2) make every best-of-N test reject about four times as often as its level.
|
|
84
|
+
export function luckEquivalentTrials({ sharpe, observations, periodsPerYear, trials, autocorrelation = 0 }) {
|
|
85
|
+
const factor = autocorrelationFactor(autocorrelation, periodsPerYear);
|
|
86
|
+
const single = singleTrialProbability({ sharpe: Number(sharpe) * factor, observations, periodsPerYear });
|
|
87
|
+
const result = {
|
|
88
|
+
autocorrelation_factor: factor,
|
|
89
|
+
t_statistic: single.t_statistic,
|
|
90
|
+
single_trial_probability: single.probability,
|
|
91
|
+
trials_for_even_odds: trialsAtProbability(single.probability, 0.5),
|
|
92
|
+
trials_for_five_percent: trialsAtProbability(single.probability, 0.05),
|
|
93
|
+
trials_expected_to_match: expectedMaximumTrials(single.t_statistic),
|
|
94
|
+
};
|
|
95
|
+
if (trials !== undefined) result.best_of_trials_probability = bestOfTrialsProbability(single.probability, trials);
|
|
96
|
+
return result;
|
|
97
|
+
}
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
// js/moments-core.js
|
|
2
|
+
// Per-period moments of a return series, in the exact conventions of
|
|
3
|
+
// alphaforge.validation.dsr._per_period_moments: Sharpe with the SAMPLE standard
|
|
4
|
+
// deviation (ddof=1); skewness and kurtosis as the BIASED population moment
|
|
5
|
+
// estimators (scipy bias=True), kurtosis NON-excess (fisher=False). These are the
|
|
6
|
+
// Bailey-Lopez de Prado conventions the PSR/DSR formulas assume. Pinned to
|
|
7
|
+
// standards/validation-api/vectors.json.
|
|
8
|
+
import { calculateDsr } from "./dsr-core.js";
|
|
9
|
+
|
|
10
|
+
export function perPeriodMoments(returns) {
|
|
11
|
+
if (!Array.isArray(returns)) throw new RangeError("returns must be an array of numbers");
|
|
12
|
+
const x = returns.map(Number).filter((v) => Number.isFinite(v));
|
|
13
|
+
const n = x.length;
|
|
14
|
+
if (n < 2) throw new RangeError("need at least 2 finite return observations");
|
|
15
|
+
let min = Infinity;
|
|
16
|
+
let max = -Infinity;
|
|
17
|
+
let sum = 0;
|
|
18
|
+
for (const v of x) { sum += v; if (v < min) min = v; if (v > max) max = v; }
|
|
19
|
+
if (max - min === 0) throw new RangeError("return series has zero variance; Sharpe is undefined");
|
|
20
|
+
const mean = sum / n;
|
|
21
|
+
let m2 = 0;
|
|
22
|
+
let m3 = 0;
|
|
23
|
+
let m4 = 0;
|
|
24
|
+
for (const v of x) {
|
|
25
|
+
const d = v - mean;
|
|
26
|
+
const d2 = d * d;
|
|
27
|
+
m2 += d2; m3 += d2 * d; m4 += d2 * d2;
|
|
28
|
+
}
|
|
29
|
+
const sampleStd = Math.sqrt(m2 / (n - 1));
|
|
30
|
+
if (sampleStd === 0) throw new RangeError("return series has zero variance; Sharpe is undefined");
|
|
31
|
+
m2 /= n; m3 /= n; m4 /= n;
|
|
32
|
+
return {
|
|
33
|
+
sharpe_per_period: mean / sampleStd,
|
|
34
|
+
skew: m3 / Math.pow(m2, 1.5),
|
|
35
|
+
non_excess_kurtosis: m4 / (m2 * m2),
|
|
36
|
+
observations: n,
|
|
37
|
+
};
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
export function dsrFromReturns({ returns, periods_per_year, effective_independent_trials, cross_trial_sharpe_sd_annualized }) {
|
|
41
|
+
const ppy = Number(periods_per_year);
|
|
42
|
+
if (!(ppy > 0)) throw new RangeError("periods_per_year must be greater than zero");
|
|
43
|
+
const m = perPeriodMoments(returns);
|
|
44
|
+
const derived_inputs = {
|
|
45
|
+
observed_sharpe_annualized: m.sharpe_per_period * Math.sqrt(ppy),
|
|
46
|
+
observations: m.observations,
|
|
47
|
+
periods_per_year: ppy,
|
|
48
|
+
skew: m.skew,
|
|
49
|
+
non_excess_kurtosis: m.non_excess_kurtosis,
|
|
50
|
+
effective_independent_trials: Number(effective_independent_trials),
|
|
51
|
+
cross_trial_sharpe_sd_annualized: Number(cross_trial_sharpe_sd_annualized),
|
|
52
|
+
};
|
|
53
|
+
return { derived_inputs, result: calculateDsr(derived_inputs) };
|
|
54
|
+
}
|
|
@@ -0,0 +1,167 @@
|
|
|
1
|
+
// =============================================================================
|
|
2
|
+
// paper-evidence-core.js
|
|
3
|
+
// -----------------------------------------------------------------------------
|
|
4
|
+
// A validator for canli.paper-evidence.v0, with no dependencies.
|
|
5
|
+
//
|
|
6
|
+
// It implements the subset of JSON Schema the standard actually uses: type,
|
|
7
|
+
// const, enum, required, additionalProperties, properties, items, minItems,
|
|
8
|
+
// minLength, minimum, maximum, pattern, and the date and date-time formats.
|
|
9
|
+
//
|
|
10
|
+
// WHY NOT A LIBRARY. A standard is only adoptable if checking it is cheap. A
|
|
11
|
+
// validator that pulls a dependency tree makes conformance a project; one that
|
|
12
|
+
// runs in any browser and any Node without installing anything makes it a
|
|
13
|
+
// paste. The subset is small enough to read in one sitting, which is the other
|
|
14
|
+
// half of adoptability.
|
|
15
|
+
//
|
|
16
|
+
// Every error carries a JSON Pointer to the offending location, because "invalid"
|
|
17
|
+
// without a path is a validator that makes you do its work.
|
|
18
|
+
// =============================================================================
|
|
19
|
+
|
|
20
|
+
const ISO_DATE = /^\d{4}-\d{2}-\d{2}$/;
|
|
21
|
+
const ISO_DATE_TIME = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(\.\d+)?(Z|[+-]\d{2}:\d{2})$/;
|
|
22
|
+
|
|
23
|
+
const typeOf = (value) => {
|
|
24
|
+
if (value === null) return "null";
|
|
25
|
+
if (Array.isArray(value)) return "array";
|
|
26
|
+
if (Number.isInteger(value)) return "integer";
|
|
27
|
+
return typeof value;
|
|
28
|
+
};
|
|
29
|
+
|
|
30
|
+
const matchesType = (value, expected) => {
|
|
31
|
+
const actual = typeOf(value);
|
|
32
|
+
if (expected === "number") return actual === "number" || actual === "integer";
|
|
33
|
+
return actual === expected;
|
|
34
|
+
};
|
|
35
|
+
|
|
36
|
+
function isValidDate(text) {
|
|
37
|
+
if (!ISO_DATE.test(text)) return false;
|
|
38
|
+
const [y, m, d] = text.split("-").map(Number);
|
|
39
|
+
const date = new Date(Date.UTC(y, m - 1, d));
|
|
40
|
+
// Round-trip: rejects 2026-02-30, which a regex alone accepts.
|
|
41
|
+
return date.getUTCFullYear() === y && date.getUTCMonth() === m - 1 && date.getUTCDate() === d;
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
/**
|
|
45
|
+
* Validate `value` against `schema`. Returns every error found, not just the
|
|
46
|
+
* first: a conformance report that stops at the first problem turns adoption
|
|
47
|
+
* into a guessing game.
|
|
48
|
+
*/
|
|
49
|
+
export function validate(value, schema, pointer = "") {
|
|
50
|
+
const errors = [];
|
|
51
|
+
const fail = (message, at = pointer) => errors.push({ pointer: at || "/", message });
|
|
52
|
+
|
|
53
|
+
if ("const" in schema && value !== schema.const) {
|
|
54
|
+
fail(`must equal ${JSON.stringify(schema.const)}`);
|
|
55
|
+
return errors;
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
if (schema.enum && !schema.enum.includes(value)) {
|
|
59
|
+
fail(`must be one of ${schema.enum.map((v) => JSON.stringify(v)).join(", ")}`);
|
|
60
|
+
return errors;
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
if (schema.type) {
|
|
64
|
+
const allowed = Array.isArray(schema.type) ? schema.type : [schema.type];
|
|
65
|
+
if (!allowed.some((t) => matchesType(value, t))) {
|
|
66
|
+
fail(`expected ${allowed.join(" or ")}, got ${typeOf(value)}`);
|
|
67
|
+
return errors;
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
if (typeof value === "string") {
|
|
72
|
+
if (schema.minLength !== undefined && value.length < schema.minLength) {
|
|
73
|
+
fail(`must be at least ${schema.minLength} character(s)`);
|
|
74
|
+
}
|
|
75
|
+
if (schema.pattern && !new RegExp(schema.pattern).test(value)) {
|
|
76
|
+
fail(`must match ${schema.pattern}`);
|
|
77
|
+
}
|
|
78
|
+
if (schema.format === "date" && !isValidDate(value)) fail("must be a valid ISO date");
|
|
79
|
+
if (schema.format === "date-time" && !ISO_DATE_TIME.test(value)) {
|
|
80
|
+
fail("must be an ISO date-time");
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
if (typeof value === "number") {
|
|
85
|
+
if (schema.minimum !== undefined && value < schema.minimum) fail(`must be >= ${schema.minimum}`);
|
|
86
|
+
if (schema.maximum !== undefined && value > schema.maximum) fail(`must be <= ${schema.maximum}`);
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
if (Array.isArray(value)) {
|
|
90
|
+
if (schema.minItems !== undefined && value.length < schema.minItems) {
|
|
91
|
+
fail(`must contain at least ${schema.minItems} item(s)`);
|
|
92
|
+
}
|
|
93
|
+
if (schema.items) {
|
|
94
|
+
value.forEach((item, i) => errors.push(...validate(item, schema.items, `${pointer}/${i}`)));
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
if (value && typeof value === "object" && !Array.isArray(value)) {
|
|
99
|
+
for (const key of schema.required ?? []) {
|
|
100
|
+
if (!(key in value)) fail(`missing required property "${key}"`);
|
|
101
|
+
}
|
|
102
|
+
if (schema.additionalProperties === false) {
|
|
103
|
+
for (const key of Object.keys(value)) {
|
|
104
|
+
if (!(schema.properties && key in schema.properties)) {
|
|
105
|
+
fail(`unexpected property "${key}"`, `${pointer}/${key}`);
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
for (const [key, sub] of Object.entries(schema.properties ?? {})) {
|
|
110
|
+
if (key in value) errors.push(...validate(value[key], sub, `${pointer}/${key}`));
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
return errors;
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
/**
|
|
118
|
+
* Checks the standard cannot express as shape, only as a relationship between
|
|
119
|
+
* fields. A record can satisfy every type in the schema and still make a claim
|
|
120
|
+
* it has disqualified itself from making, and these are exactly those cases.
|
|
121
|
+
*/
|
|
122
|
+
export function semanticChecks(record) {
|
|
123
|
+
const errors = [];
|
|
124
|
+
const fail = (pointer, message) => errors.push({ pointer, message });
|
|
125
|
+
|
|
126
|
+
if (record?.returns?.sharpe_reportable === false && record?.returns?.sharpe_annualised !== null) {
|
|
127
|
+
fail(
|
|
128
|
+
"/returns/sharpe_annualised",
|
|
129
|
+
"sharpe_reportable is false, so sharpe_annualised must be null. Declaring a sample " +
|
|
130
|
+
"cannot support a figure and then publishing the figure is the defect this field exists to prevent.",
|
|
131
|
+
);
|
|
132
|
+
}
|
|
133
|
+
if (record?.selection?.trials_counted === true && record?.selection?.trial_count === null) {
|
|
134
|
+
fail("/selection/trial_count", "trials_counted is true, so trial_count may not be null");
|
|
135
|
+
}
|
|
136
|
+
if (record?.selection?.deflation_applied === true && record?.selection?.deflated_sharpe_ratio === null) {
|
|
137
|
+
fail("/selection/deflated_sharpe_ratio", "deflation_applied is true, so the deflated ratio must be present");
|
|
138
|
+
}
|
|
139
|
+
if (record?.capital?.kind === "PAPER" && record?.returns?.basis === "NET_OF_REALISED_COSTS") {
|
|
140
|
+
fail(
|
|
141
|
+
"/returns/basis",
|
|
142
|
+
"a PAPER record cannot be net of REALISED costs: realised costs require funded execution. " +
|
|
143
|
+
"Use NET_OF_MODELLED_COSTS.",
|
|
144
|
+
);
|
|
145
|
+
}
|
|
146
|
+
if (record?.risk?.drawdown_basis === "NOT_ESTABLISHED" && record?.risk?.max_drawdown_realised !== null) {
|
|
147
|
+
fail(
|
|
148
|
+
"/risk/max_drawdown_realised",
|
|
149
|
+
"drawdown_basis is NOT_ESTABLISHED, so no realised drawdown may be stated",
|
|
150
|
+
);
|
|
151
|
+
}
|
|
152
|
+
const first = record?.period?.first_observation;
|
|
153
|
+
const last = record?.period?.last_observation;
|
|
154
|
+
if (first && last && first > last) {
|
|
155
|
+
fail("/period", "first_observation is after last_observation");
|
|
156
|
+
}
|
|
157
|
+
return errors;
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
/** Full conformance: shape, then the relationships shape cannot express. */
|
|
161
|
+
export function conformance(record, schema) {
|
|
162
|
+
const structural = validate(record, schema);
|
|
163
|
+
// Semantic checks read fields; running them on a structurally broken record
|
|
164
|
+
// produces noise, so they are gated on the shape being right first.
|
|
165
|
+
const semantic = structural.length === 0 ? semanticChecks(record) : [];
|
|
166
|
+
return { valid: structural.length === 0 && semantic.length === 0, structural, semantic };
|
|
167
|
+
}
|
|
@@ -0,0 +1,112 @@
|
|
|
1
|
+
// js/pbo-core.js
|
|
2
|
+
// Probability of Backtest Overfitting by Combinatorially Symmetric Cross-Validation.
|
|
3
|
+
// A line-for-line port of alphaforge.validation.pbo.pbo_cscv. Where the Python samples
|
|
4
|
+
// combinations with numpy's generator this port uses the repo's mulberry32 and SAYS SO
|
|
5
|
+
// in the result (`exhaustive: false`): the exhaustive case is bit-comparable, the sampled
|
|
6
|
+
// case is comparable within sampling noise. Pinned to standards/validation-api/vectors.json.
|
|
7
|
+
import { makeRandom } from "./selection-risk-core.js";
|
|
8
|
+
|
|
9
|
+
function blockSharpe(rows, nConfigs) {
|
|
10
|
+
// Population std (ddof=0); zero variance -> -Infinity so it is never IS-best and ranks worst OOS.
|
|
11
|
+
const out = new Array(nConfigs);
|
|
12
|
+
for (let j = 0; j < nConfigs; j++) {
|
|
13
|
+
let sum = 0;
|
|
14
|
+
for (const row of rows) sum += row[j];
|
|
15
|
+
const mean = sum / rows.length;
|
|
16
|
+
let ss = 0;
|
|
17
|
+
for (const row of rows) { const d = row[j] - mean; ss += d * d; }
|
|
18
|
+
const std = Math.sqrt(ss / rows.length);
|
|
19
|
+
out[j] = std > 0 ? mean / std : Number.NEGATIVE_INFINITY;
|
|
20
|
+
}
|
|
21
|
+
return out;
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
function averageRanks(values) {
|
|
25
|
+
// scipy.stats.rankdata(method="average"): 1-based, ties share the mean rank.
|
|
26
|
+
const order = values.map((v, i) => [v, i]).sort((a, b) => (a[0] < b[0] ? -1 : a[0] > b[0] ? 1 : 0));
|
|
27
|
+
const ranks = new Array(values.length);
|
|
28
|
+
let i = 0;
|
|
29
|
+
while (i < order.length) {
|
|
30
|
+
let j = i;
|
|
31
|
+
while (j + 1 < order.length && order[j + 1][0] === order[i][0]) j++;
|
|
32
|
+
const rank = (i + 1 + j + 1) / 2;
|
|
33
|
+
for (let k = i; k <= j; k++) ranks[order[k][1]] = rank;
|
|
34
|
+
i = j + 1;
|
|
35
|
+
}
|
|
36
|
+
return ranks;
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
function binomial(n, k) {
|
|
40
|
+
let r = 1;
|
|
41
|
+
for (let i = 1; i <= k; i++) r = (r * (n - k + i)) / i;
|
|
42
|
+
return Math.round(r);
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
function* combinations(n, k) {
|
|
46
|
+
const idx = Array.from({ length: k }, (_, i) => i);
|
|
47
|
+
while (true) {
|
|
48
|
+
yield idx.slice();
|
|
49
|
+
let i = k - 1;
|
|
50
|
+
while (i >= 0 && idx[i] === n - k + i) i--;
|
|
51
|
+
if (i < 0) return;
|
|
52
|
+
idx[i]++;
|
|
53
|
+
for (let j = i + 1; j < k; j++) idx[j] = idx[j - 1] + 1;
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
function sampleCombinations(n, k, count, seed) {
|
|
58
|
+
const next = makeRandom(seed);
|
|
59
|
+
const seen = new Set();
|
|
60
|
+
const out = [];
|
|
61
|
+
while (out.length < count) {
|
|
62
|
+
const pool = Array.from({ length: n }, (_, i) => i);
|
|
63
|
+
const pick = [];
|
|
64
|
+
for (let d = 0; d < k; d++) {
|
|
65
|
+
const at = Math.floor(next() * pool.length);
|
|
66
|
+
pick.push(pool[at]);
|
|
67
|
+
pool.splice(at, 1);
|
|
68
|
+
}
|
|
69
|
+
pick.sort((a, b) => a - b);
|
|
70
|
+
const key = pick.join(",");
|
|
71
|
+
if (!seen.has(key)) { seen.add(key); out.push(pick); }
|
|
72
|
+
}
|
|
73
|
+
return out.sort((a, b) => a.join(",").localeCompare(b.join(","), undefined, { numeric: true }));
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
export function pboCscv(matrix, { nSplits = 16, maxCombinations = 2000, seed = 42 } = {}) {
|
|
77
|
+
if (!Number.isInteger(nSplits) || nSplits < 2 || nSplits % 2 !== 0) throw new RangeError(`n_splits must be an even integer >= 2; got ${nSplits}`);
|
|
78
|
+
if (!Number.isInteger(maxCombinations) || maxCombinations < 1) throw new RangeError(`max_combinations must be >= 1; got ${maxCombinations}`);
|
|
79
|
+
if (!Array.isArray(matrix) || matrix.length === 0 || !Array.isArray(matrix[0])) throw new RangeError("matrix must be a non-empty array of rows");
|
|
80
|
+
const nConfigs = matrix[0].length;
|
|
81
|
+
if (nConfigs < 2) throw new RangeError(`matrix needs >= 2 config columns to rank cross-sectionally; got ${nConfigs}`);
|
|
82
|
+
for (const row of matrix) {
|
|
83
|
+
if (row.length !== nConfigs) throw new RangeError("every row must have the same number of columns");
|
|
84
|
+
for (const v of row) if (!Number.isFinite(Number(v))) throw new RangeError("matrix values must be finite numbers");
|
|
85
|
+
}
|
|
86
|
+
const nRows = matrix.length;
|
|
87
|
+
if (nRows < nSplits) throw new RangeError(`matrix has ${nRows} rows < n_splits=${nSplits}; cannot form that many non-empty contiguous blocks`);
|
|
88
|
+
const blockLen = Math.floor(nRows / nSplits);
|
|
89
|
+
const blocks = Array.from({ length: nSplits }, (_, b) => matrix.slice(b * blockLen, (b + 1) * blockLen).map((r) => r.map(Number)));
|
|
90
|
+
const half = nSplits / 2;
|
|
91
|
+
const total = binomial(nSplits, half);
|
|
92
|
+
const exhaustive = total <= maxCombinations;
|
|
93
|
+
const combos = exhaustive ? [...combinations(nSplits, half)] : sampleCombinations(nSplits, half, maxCombinations, seed);
|
|
94
|
+
const lambdas = [];
|
|
95
|
+
const isOos = [];
|
|
96
|
+
for (const isBlocks of combos) {
|
|
97
|
+
const isSet = new Set(isBlocks);
|
|
98
|
+
const isRows = isBlocks.flatMap((b) => blocks[b]);
|
|
99
|
+
const oosRows = [];
|
|
100
|
+
for (let b = 0; b < nSplits; b++) if (!isSet.has(b)) oosRows.push(...blocks[b]);
|
|
101
|
+
const srIs = blockSharpe(isRows, nConfigs);
|
|
102
|
+
let best = 0;
|
|
103
|
+
for (let j = 1; j < nConfigs; j++) if (srIs[j] > srIs[best]) best = j;
|
|
104
|
+
const srOos = blockSharpe(oosRows, nConfigs);
|
|
105
|
+
const ranks = averageRanks(srOos);
|
|
106
|
+
const omega = ranks[best] / (nConfigs + 1);
|
|
107
|
+
lambdas.push(Math.log(omega / (1 - omega)));
|
|
108
|
+
isOos.push([srIs[best], srOos[best]]);
|
|
109
|
+
}
|
|
110
|
+
const pbo = lambdas.filter((l) => l <= 0).length / lambdas.length;
|
|
111
|
+
return { pbo, n_combinations: combos.length, lambdas, is_oos_pairs: isOos, exhaustive, block_length: blockLen };
|
|
112
|
+
}
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
// js/receipt-statement.js
|
|
2
|
+
// What a canlicapital.com receipt signature covers, and how to check one. Shared by the API, which
|
|
3
|
+
// signs (api/_lib/receipt-signature.js), and by the MCP package, which verifies offline
|
|
4
|
+
// (mcp/src/local mirrors this file byte for byte), so the two cannot disagree about the bytes signed.
|
|
5
|
+
//
|
|
6
|
+
// The statement is the canonical JSON of the receipt's content: its id, the validator's endpoint,
|
|
7
|
+
// the sha256 of the input, the sha256 of the output, and the sha256 of every source file that
|
|
8
|
+
// computed it. The id is itself the first 24 hex characters of sha256 over the canonical JSON of
|
|
9
|
+
// {endpoint, input_sha256, output, bindings}, so the signature ties the result to the exact code.
|
|
10
|
+
import { createHash, createPublicKey, verify } from "node:crypto";
|
|
11
|
+
|
|
12
|
+
import { canonicalJson } from "../scripts/canonical-json.mjs";
|
|
13
|
+
|
|
14
|
+
export const SIGNATURE_SCHEMA = "canli.receipt-signature.v1";
|
|
15
|
+
export const KEYS_URL = "https://canlicapital.com/.well-known/canli-receipt-keys.json";
|
|
16
|
+
|
|
17
|
+
const sha256Hex = (text) => createHash("sha256").update(text).digest("hex");
|
|
18
|
+
|
|
19
|
+
export function outputSha256(output) {
|
|
20
|
+
return `sha256:${sha256Hex(canonicalJson(output))}`;
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
export function receiptId({ endpoint, input_sha256, output, bindings }) {
|
|
24
|
+
return sha256Hex(canonicalJson({ endpoint, input_sha256, output, bindings })).slice(0, 24);
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
export function receiptStatement({ id, endpoint, input_sha256, output_sha256, bindings }) {
|
|
28
|
+
return canonicalJson({ schema: SIGNATURE_SCHEMA, id, endpoint, input_sha256, output_sha256, bindings });
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
// The key id is derived from the key: the first 16 hex characters of sha256 over its 32 raw bytes.
|
|
32
|
+
export function keyIdFor(rawPublicKey) {
|
|
33
|
+
return sha256Hex(rawPublicKey).slice(0, 16);
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
// Checks a stored receipt end to end: its output hashes to output_sha256, its content hashes to its
|
|
37
|
+
// id, its signature verifies over the statement, and the signing key is one canlicapital.com
|
|
38
|
+
// publishes. Returns every check so a caller can see which one failed.
|
|
39
|
+
export function verifyReceipt({ id, endpoint, input_sha256, output, bindings, signature }, keys) {
|
|
40
|
+
const checks = { id_matches_content: false, signature_valid: false, key_published: false };
|
|
41
|
+
const computedOutputSha = outputSha256(output);
|
|
42
|
+
checks.id_matches_content = receiptId({ endpoint, input_sha256, output, bindings }) === id;
|
|
43
|
+
const key = signature ? (keys ?? []).find((k) => k.key_id === signature.key_id && k.alg === "Ed25519") : undefined;
|
|
44
|
+
checks.key_published = Boolean(key);
|
|
45
|
+
if (key && signature?.value && signature.schema === SIGNATURE_SCHEMA) {
|
|
46
|
+
const raw = Buffer.from(key.x, "base64url");
|
|
47
|
+
if (keyIdFor(raw) === key.key_id) {
|
|
48
|
+
const publicKey = createPublicKey({ key: { kty: "OKP", crv: "Ed25519", x: key.x }, format: "jwk" });
|
|
49
|
+
const statement = receiptStatement({ id, endpoint, input_sha256, output_sha256: computedOutputSha, bindings });
|
|
50
|
+
checks.signature_valid = verify(null, Buffer.from(statement, "utf8"), publicKey, Buffer.from(signature.value, "base64url"));
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
return { valid: checks.id_matches_content && checks.signature_valid && checks.key_published, checks, output_sha256: computedOutputSha, key_id: signature?.key_id ?? null };
|
|
54
|
+
}
|