@jinn-network/client 0.1.3-canary.8e61ba74 → 0.1.3-canary.941614bc
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -8
- package/deployments/deployment-task-coordinator-router-v3-baseSepolia-fast.json +38 -0
- package/dist/agent/agent-ws.d.ts +2 -1
- package/dist/agent/agent-ws.js +2 -1
- package/dist/agent/agent-ws.js.map +1 -1
- package/dist/api/agent-binding-endpoint.d.ts +26 -0
- package/dist/api/agent-binding-endpoint.js +28 -0
- package/dist/api/agent-binding-endpoint.js.map +1 -0
- package/dist/api/bootstrap-endpoint.d.ts +8 -0
- package/dist/api/bootstrap-endpoint.js +27 -2
- package/dist/api/bootstrap-endpoint.js.map +1 -1
- package/dist/api/gather-status.d.ts +4 -0
- package/dist/api/gather-status.js +99 -0
- package/dist/api/gather-status.js.map +1 -1
- package/dist/api/prediction-v1-build.d.ts +47 -0
- package/dist/api/prediction-v1-build.js +73 -0
- package/dist/api/prediction-v1-build.js.map +1 -0
- package/dist/api/server.d.ts +16 -7
- package/dist/api/server.js +59 -12
- package/dist/api/server.js.map +1 -1
- package/dist/api/setup-endpoints.d.ts +8 -0
- package/dist/api/setup-endpoints.js +193 -3
- package/dist/api/setup-endpoints.js.map +1 -1
- package/dist/api/solvernets-catalog-build.d.ts +37 -0
- package/dist/api/solvernets-catalog-build.js +19 -0
- package/dist/api/solvernets-catalog-build.js.map +1 -0
- package/dist/api/solvernets-endpoint.d.ts +15 -0
- package/dist/api/solvernets-endpoint.js +5 -0
- package/dist/api/solvernets-endpoint.js.map +1 -0
- package/dist/api/status-build.d.ts +5 -0
- package/dist/api/status-build.js +1 -0
- package/dist/api/status-build.js.map +1 -1
- package/dist/build-info.json +4 -4
- package/dist/build-meta.json +1 -1
- package/dist/cli/commands/doctor.d.ts +15 -0
- package/dist/cli/commands/doctor.js +21 -0
- package/dist/cli/commands/doctor.js.map +1 -1
- package/dist/cli/commands/prediction-scoreboard.d.ts +10 -0
- package/dist/cli/commands/prediction-scoreboard.js +136 -0
- package/dist/cli/commands/prediction-scoreboard.js.map +1 -0
- package/dist/cli/commands/run.js +10 -0
- package/dist/cli/commands/run.js.map +1 -1
- package/dist/cli/commands/solver-nets.js +27 -5
- package/dist/cli/commands/solver-nets.js.map +1 -1
- package/dist/cli/commands/status.d.ts +3 -0
- package/dist/cli/commands/status.js +11 -3
- package/dist/cli/commands/status.js.map +1 -1
- package/dist/cli/commands/update.d.ts +8 -1
- package/dist/cli/commands/update.js +232 -7
- package/dist/cli/commands/update.js.map +1 -1
- package/dist/cli/index.js +2 -0
- package/dist/cli/index.js.map +1 -1
- package/dist/cli/task-native-readiness.d.ts +27 -0
- package/dist/cli/task-native-readiness.js +170 -0
- package/dist/cli/task-native-readiness.js.map +1 -0
- package/dist/config.d.ts +92 -48
- package/dist/config.js +91 -124
- package/dist/config.js.map +1 -1
- package/dist/corpus/envelope-projection.d.ts +12 -0
- package/dist/corpus/envelope-projection.js +110 -0
- package/dist/corpus/envelope-projection.js.map +1 -0
- package/dist/corpus/index.d.ts +3 -0
- package/dist/corpus/index.js +3 -0
- package/dist/corpus/index.js.map +1 -1
- package/dist/corpus/prediction-brier-scoreboard-report.d.ts +17 -0
- package/dist/corpus/prediction-brier-scoreboard-report.js +148 -0
- package/dist/corpus/prediction-brier-scoreboard-report.js.map +1 -0
- package/dist/corpus/prediction-brier-scoreboard.d.ts +56 -0
- package/dist/corpus/prediction-brier-scoreboard.js +269 -0
- package/dist/corpus/prediction-brier-scoreboard.js.map +1 -0
- package/dist/corpus/prediction-scoreable-verdicts.d.ts +4 -0
- package/dist/corpus/prediction-scoreable-verdicts.js +23 -0
- package/dist/corpus/prediction-scoreable-verdicts.js.map +1 -0
- package/dist/corpus/types.d.ts +43 -1
- package/dist/corpus/types.js.map +1 -1
- package/dist/daemon/daemon.js +1 -1
- package/dist/daemon/daemon.js.map +1 -1
- package/dist/dashboard/assets/index-5ArwpmyK.js +76 -0
- package/dist/dashboard/assets/{index-DQ3u_vP5.css → index-Bx5KH3YV.css} +1 -1
- package/dist/dashboard/index.html +2 -2
- package/dist/earning/bootstrap.d.ts +24 -0
- package/dist/earning/bootstrap.js +84 -23
- package/dist/earning/bootstrap.js.map +1 -1
- package/dist/earning/contracts.d.ts +1 -0
- package/dist/earning/contracts.js +16 -0
- package/dist/earning/contracts.js.map +1 -1
- package/dist/earning/faucet.d.ts +31 -0
- package/dist/earning/faucet.js +37 -0
- package/dist/earning/faucet.js.map +1 -1
- package/dist/earning/safe-adapter.js +45 -3
- package/dist/earning/safe-adapter.js.map +1 -1
- package/dist/earning/store.d.ts +8 -0
- package/dist/earning/store.js +41 -0
- package/dist/earning/store.js.map +1 -1
- package/dist/earning/types.d.ts +4 -0
- package/dist/events/types.d.ts +2 -2
- package/dist/harnesses/engine/engine.d.ts +5 -3
- package/dist/harnesses/engine/engine.js +33 -11
- package/dist/harnesses/engine/engine.js.map +1 -1
- package/dist/harnesses/engine/registry.d.ts +2 -0
- package/dist/harnesses/engine/registry.js +3 -0
- package/dist/harnesses/engine/registry.js.map +1 -1
- package/dist/harnesses/impls/claude-code-learner/adapters/claude-code.d.ts +19 -0
- package/dist/harnesses/impls/claude-code-learner/adapters/claude-code.js +37 -2
- package/dist/harnesses/impls/claude-code-learner/adapters/claude-code.js.map +1 -1
- package/dist/harnesses/impls/claude-code-learner/harness.js +2 -0
- package/dist/harnesses/impls/claude-code-learner/harness.js.map +1 -1
- package/dist/harnesses/impls/claude-code-learner/harvest.js +132 -2
- package/dist/harnesses/impls/claude-code-learner/harvest.js.map +1 -1
- package/dist/harnesses/impls/claude-code-learner/test-utils/fake-plugin-outputs.d.ts +20 -0
- package/dist/harnesses/impls/claude-code-learner/test-utils/fake-plugin-outputs.js +65 -0
- package/dist/harnesses/impls/claude-code-learner/test-utils/fake-plugin-outputs.js.map +1 -1
- package/dist/harnesses/impls/claude-code-learner/types.d.ts +5 -0
- package/dist/harnesses/impls/claude-mcp-hyperliquid/index.js +4 -3
- package/dist/harnesses/impls/claude-mcp-hyperliquid/index.js.map +1 -1
- package/dist/harnesses/impls/claude-mcp-prediction/index.js +6 -5
- package/dist/harnesses/impls/claude-mcp-prediction/index.js.map +1 -1
- package/dist/harnesses/impls/claude-mcp-prediction-apy/index.js +6 -5
- package/dist/harnesses/impls/claude-mcp-prediction-apy/index.js.map +1 -1
- package/dist/harnesses/impls/index.d.ts +2 -2
- package/dist/harnesses/impls/index.js +8 -9
- package/dist/harnesses/impls/index.js.map +1 -1
- package/dist/harnesses/impls/legacy-claude/index.d.ts +2 -2
- package/dist/harnesses/impls/prediction-v1-evaluator/index.js +33 -12
- package/dist/harnesses/impls/prediction-v1-evaluator/index.js.map +1 -1
- package/dist/harnesses/types.d.ts +6 -1
- package/dist/harnesses/types.js.map +1 -1
- package/dist/main.d.ts +13 -1
- package/dist/main.js +178 -18
- package/dist/main.js.map +1 -1
- package/dist/mcp/search-records.d.ts +92 -0
- package/dist/mcp/search-records.js +393 -0
- package/dist/mcp/search-records.js.map +1 -0
- package/dist/mcp/server.js +56 -15
- package/dist/mcp/server.js.map +1 -1
- package/dist/operator-errors.d.ts +14 -0
- package/dist/operator-errors.js +35 -5
- package/dist/operator-errors.js.map +1 -1
- package/dist/plugins/resolvers.js +45 -7
- package/dist/plugins/resolvers.js.map +1 -1
- package/dist/plugins/validator.js +8 -0
- package/dist/plugins/validator.js.map +1 -1
- package/dist/runner/runner.d.ts +6 -6
- package/dist/solver-nets/prediction-operator-ux.d.ts +105 -0
- package/dist/solver-nets/prediction-operator-ux.js +438 -0
- package/dist/solver-nets/prediction-operator-ux.js.map +1 -0
- package/dist/solver-nets/registry.d.ts +13 -6
- package/dist/solver-nets/registry.js +50 -16
- package/dist/solver-nets/registry.js.map +1 -1
- package/dist/solver-types/index.d.ts +11 -0
- package/dist/solver-types/index.js +8 -1
- package/dist/solver-types/index.js.map +1 -1
- package/dist/solver-types/prediction-v0.d.ts +1 -1
- package/dist/solver-types/prediction-v0.js +1 -1
- package/dist/solver-types/prediction-v0.js.map +1 -1
- package/dist/solver-types/prediction-v1-auto.d.ts +2 -0
- package/dist/solver-types/prediction-v1-auto.js +56 -7
- package/dist/solver-types/prediction-v1-auto.js.map +1 -1
- package/dist/solver-types/prediction-v1.js +25 -1
- package/dist/solver-types/prediction-v1.js.map +1 -1
- package/dist/solver-types/solver-type.d.ts +18 -5
- package/dist/store/store.d.ts +44 -2
- package/dist/store/store.js +387 -8
- package/dist/store/store.js.map +1 -1
- package/dist/tx-retry.js +11 -0
- package/dist/tx-retry.js.map +1 -1
- package/dist/vendor/@jinn-network/sdk/dist/contracts.d.ts +1 -1
- package/dist/vendor/@jinn-network/sdk/dist/contracts.js +1 -1
- package/package.json +1 -1
- package/plugins/jinn-prediction-plugin/.claude-plugin/plugin.json +4 -1
- package/plugins/jinn-prediction-plugin/GEMINI.md +5 -1
- package/plugins/jinn-prediction-plugin/jinn.plugin.json +5 -2
- package/plugins/jinn-prediction-plugin/mcp/polymarket-server.mjs +321 -1
- package/plugins/jinn-prediction-plugin/skills/common-biases/SKILL.md +3 -0
- package/plugins/jinn-prediction-plugin/skills/polymarket-task-handling/SKILL.md +5 -0
- package/plugins/jinn-prediction-plugin/skills/prediction-corpus-retrieval/SKILL.md +35 -0
- package/plugins/network-tools/.claude-plugin/plugin.json +12 -0
- package/plugins/network-tools/jinn.plugin.json +29 -0
- package/plugins/network-tools/mcp/jinn-client-server.mjs +110 -0
- package/dist/dashboard/assets/index-CaUOdCs6.js +0 -68
- package/dist/mcp/search-artifacts.d.ts +0 -31
- package/dist/mcp/search-artifacts.js +0 -40
- package/dist/mcp/search-artifacts.js.map +0 -1
|
@@ -0,0 +1,148 @@
|
|
|
1
|
+
import { DEFAULT_PREDICTION_BRIER_SCOREBOARD_WINDOW_DAYS, aggregatePredictionBrierScoreboard, } from './prediction-brier-scoreboard.js';
|
|
2
|
+
export const DEFAULT_PREDICTION_SCOREBOARD_REPORT_PATH = 'docs/superpowers/reports/prediction-solvernet-scoreboard.md';
|
|
3
|
+
export function queryPredictionBrierScoreboardProjections(store, query = {}) {
|
|
4
|
+
const verdicts = store.queryEnvelopeProjections({
|
|
5
|
+
solverType: 'prediction.v1',
|
|
6
|
+
role: 'verdict',
|
|
7
|
+
generatedAfter: query.generatedAfter,
|
|
8
|
+
generatedBefore: query.generatedBefore,
|
|
9
|
+
limit: query.limit ?? 1000,
|
|
10
|
+
});
|
|
11
|
+
const solutionRefs = compactRefs(verdicts.flatMap(verdictSolutionRefs));
|
|
12
|
+
if (solutionRefs.length === 0)
|
|
13
|
+
return verdicts;
|
|
14
|
+
const solutions = store.queryEnvelopeProjections({
|
|
15
|
+
envelopeRefs: solutionRefs,
|
|
16
|
+
solverType: 'prediction.v1',
|
|
17
|
+
role: 'restoration',
|
|
18
|
+
limit: solutionRefs.length,
|
|
19
|
+
});
|
|
20
|
+
return dedupeProjections([...verdicts, ...solutions]);
|
|
21
|
+
}
|
|
22
|
+
export function buildPredictionBrierScoreboard(store, options = {}) {
|
|
23
|
+
const projectionQuery = windowedProjectionQuery(options);
|
|
24
|
+
return aggregatePredictionBrierScoreboard(queryPredictionBrierScoreboardProjections(store, projectionQuery), options);
|
|
25
|
+
}
|
|
26
|
+
export function renderPredictionBrierScoreboardMarkdown(scoreboard, options = {}) {
|
|
27
|
+
const title = options.title ?? 'Prediction SolverNet Scoreboard';
|
|
28
|
+
const generatedAtIso = options.generatedAtIso ?? new Date().toISOString();
|
|
29
|
+
const status = scoreboardStatus(scoreboard, generatedAtIso, options.staleAfterDays ?? 14);
|
|
30
|
+
const lines = [
|
|
31
|
+
`# ${title}`,
|
|
32
|
+
'',
|
|
33
|
+
`Generated: ${generatedAtIso}`,
|
|
34
|
+
`Status: ${status}`,
|
|
35
|
+
`Window: trailing ${scoreboard.trailingWindowDays} days ending ${formatGeneratedAt(scoreboard.asOfGeneratedAt)}`,
|
|
36
|
+
'',
|
|
37
|
+
'## Headline',
|
|
38
|
+
'',
|
|
39
|
+
];
|
|
40
|
+
if (scoreboard.overall.scoredVerdictCount === 0) {
|
|
41
|
+
lines.push('Insufficient scoreable Verdict data. The report will populate after Prediction SolverNet evaluators emit `SCORED` Verdict envelopes with Brier metadata.', '');
|
|
42
|
+
}
|
|
43
|
+
lines.push('| Metric | Value |', '|---|---:|', `| Mean Brier spread | ${formatNumber(scoreboard.overall.meanBrierSpread)} |`, `| Mean solver Brier | ${formatNumber(scoreboard.overall.meanSolverBrier)} |`, `| Mean consensus Brier | ${formatNumber(scoreboard.overall.meanConsensusBrier)} |`, `| Scored Verdicts | ${scoreboard.overall.scoredVerdictCount} |`, `| Shared Tasks | ${scoreboard.overall.distinctTaskCount} |`, `| Active operators | ${scoreboard.overall.activeOperatorCount} |`, '', '## Weekly Trend', '', metricTable(['Week', 'Scored Verdicts', 'Mean Spread', 'Solver Brier', 'Consensus Brier', 'Tasks', 'Operators'], scoreboard.weeklyTrend.map((summary) => [
|
|
44
|
+
summary.weekStartIso.slice(0, 10),
|
|
45
|
+
String(summary.scoredVerdictCount),
|
|
46
|
+
formatNumber(summary.meanBrierSpread),
|
|
47
|
+
formatNumber(summary.meanSolverBrier),
|
|
48
|
+
formatNumber(summary.meanConsensusBrier),
|
|
49
|
+
String(summary.distinctTaskCount),
|
|
50
|
+
String(summary.activeOperatorCount),
|
|
51
|
+
]), 'No weekly trend data yet.'), '', '## Operators', '', metricTable(['Operator', 'Scored Verdicts', 'Mean Spread', 'Solver Brier', 'Consensus Brier', 'Tasks'], scoreboard.perOperator.map((summary) => [
|
|
52
|
+
summary.participantSafeAddress ?? summary.participantAgentEoa ?? summary.key,
|
|
53
|
+
String(summary.scoredVerdictCount),
|
|
54
|
+
formatNumber(summary.meanBrierSpread),
|
|
55
|
+
formatNumber(summary.meanSolverBrier),
|
|
56
|
+
formatNumber(summary.meanConsensusBrier),
|
|
57
|
+
String(summary.distinctTaskCount),
|
|
58
|
+
]), 'No operator rows yet.'), '', '## Harnesses', '', metricTable(['Harness', 'Runtime Digest', 'Scored Verdicts', 'Mean Spread', 'Tasks'], scoreboard.perHarness.map((summary) => [
|
|
59
|
+
[summary.implName ?? 'unknown', summary.implVersion ?? 'unknown'].join('@'),
|
|
60
|
+
summary.runtimeBundleDigest ?? 'unknown',
|
|
61
|
+
String(summary.scoredVerdictCount),
|
|
62
|
+
formatNumber(summary.meanBrierSpread),
|
|
63
|
+
String(summary.distinctTaskCount),
|
|
64
|
+
]), 'No Harness rows yet.'), '', '## Plugins', '', metricTable(['Plugin', 'Scored Verdicts', 'Mean Spread', 'Tasks'], scoreboard.perPlugin.map((summary) => [
|
|
65
|
+
summary.plugin,
|
|
66
|
+
String(summary.scoredVerdictCount),
|
|
67
|
+
formatNumber(summary.meanBrierSpread),
|
|
68
|
+
String(summary.distinctTaskCount),
|
|
69
|
+
]), 'No plugin rows yet.'), '', '## Exclusions', '', '| Reason | Count |', '|---|---:|', `| Total input rows | ${scoreboard.excluded.totalInputRows} |`, `| Included score rows | ${scoreboard.excluded.includedRows} |`, `| Non-prediction/non-Verdict rows | ${scoreboard.excluded.nonPredictionVerdictRows} |`, `| Non-SCORED Verdict rows | ${scoreboard.excluded.nonScoredVerdictRows} |`, `| Missing score rows | ${scoreboard.excluded.missingScoreRows} |`, `| Non-numeric score rows | ${scoreboard.excluded.nonNumericScoreRows} |`, `| Outside trailing window | ${scoreboard.excluded.outsideWindowRows} |`);
|
|
70
|
+
return `${lines.join('\n')}\n`;
|
|
71
|
+
}
|
|
72
|
+
function windowedProjectionQuery(options) {
|
|
73
|
+
const projectionQuery = options.projectionQuery ?? {};
|
|
74
|
+
if (options.asOfGeneratedAt === undefined)
|
|
75
|
+
return projectionQuery;
|
|
76
|
+
const trailingWindowDays = options.trailingWindowDays
|
|
77
|
+
?? DEFAULT_PREDICTION_BRIER_SCOREBOARD_WINDOW_DAYS;
|
|
78
|
+
return {
|
|
79
|
+
...projectionQuery,
|
|
80
|
+
generatedAfter: projectionQuery.generatedAfter
|
|
81
|
+
?? options.asOfGeneratedAt - generatedAtSpanForDays(options.asOfGeneratedAt, trailingWindowDays),
|
|
82
|
+
generatedBefore: projectionQuery.generatedBefore ?? options.asOfGeneratedAt,
|
|
83
|
+
};
|
|
84
|
+
}
|
|
85
|
+
function generatedAtSpanForDays(asOfGeneratedAt, days) {
|
|
86
|
+
const daySpan = Math.abs(asOfGeneratedAt) >= 10_000_000_000 ? 86_400_000 : 86_400;
|
|
87
|
+
return days * daySpan;
|
|
88
|
+
}
|
|
89
|
+
function verdictSolutionRefs(projection) {
|
|
90
|
+
return [
|
|
91
|
+
projection.solutionEnvelopeCid,
|
|
92
|
+
projection.solutionEnvelopeSha256,
|
|
93
|
+
projection.solutionEnvelopeRef,
|
|
94
|
+
projection.metadata['solutionEnvelope.cid'],
|
|
95
|
+
projection.metadata['solutionEnvelope.sha256'],
|
|
96
|
+
].filter((value) => typeof value === 'string' && value.length > 0);
|
|
97
|
+
}
|
|
98
|
+
function compactRefs(refs) {
|
|
99
|
+
return [...new Set(refs)];
|
|
100
|
+
}
|
|
101
|
+
function dedupeProjections(projections) {
|
|
102
|
+
const byId = new Map();
|
|
103
|
+
for (const projection of projections)
|
|
104
|
+
byId.set(projection.envelopeId, projection);
|
|
105
|
+
return [...byId.values()];
|
|
106
|
+
}
|
|
107
|
+
function metricTable(headers, rows, emptyText) {
|
|
108
|
+
if (rows.length === 0)
|
|
109
|
+
return emptyText;
|
|
110
|
+
const alignment = headers.map((_, index) => (index === 0 ? '---' : '---:')).join('|');
|
|
111
|
+
const rendered = [
|
|
112
|
+
`| ${headers.map(escapeTableCell).join(' | ')} |`,
|
|
113
|
+
`|${alignment}|`,
|
|
114
|
+
];
|
|
115
|
+
for (const row of rows) {
|
|
116
|
+
rendered.push(`| ${row.map(escapeTableCell).join(' | ')} |`);
|
|
117
|
+
}
|
|
118
|
+
return rendered.join('\n');
|
|
119
|
+
}
|
|
120
|
+
function scoreboardStatus(scoreboard, generatedAtIso, staleAfterDays) {
|
|
121
|
+
if (scoreboard.overall.scoredVerdictCount === 0)
|
|
122
|
+
return 'INSUFFICIENT_DATA';
|
|
123
|
+
if (scoreboard.asOfGeneratedAt === null)
|
|
124
|
+
return 'INSUFFICIENT_DATA';
|
|
125
|
+
const generatedAtMs = Date.parse(generatedAtIso);
|
|
126
|
+
if (!Number.isFinite(generatedAtMs))
|
|
127
|
+
return 'OK';
|
|
128
|
+
const asOfMs = generatedAtToMs(scoreboard.asOfGeneratedAt);
|
|
129
|
+
const staleAfterMs = staleAfterDays * 86_400_000;
|
|
130
|
+
if (generatedAtMs - asOfMs > staleAfterMs)
|
|
131
|
+
return `STALE (latest scored Verdict older than ${staleAfterDays} days)`;
|
|
132
|
+
return 'OK';
|
|
133
|
+
}
|
|
134
|
+
function formatGeneratedAt(generatedAt) {
|
|
135
|
+
if (generatedAt === null)
|
|
136
|
+
return '-';
|
|
137
|
+
return new Date(generatedAtToMs(generatedAt)).toISOString();
|
|
138
|
+
}
|
|
139
|
+
function generatedAtToMs(generatedAt) {
|
|
140
|
+
return Math.abs(generatedAt) >= 10_000_000_000 ? generatedAt : generatedAt * 1000;
|
|
141
|
+
}
|
|
142
|
+
function formatNumber(value) {
|
|
143
|
+
return value === null ? '-' : value.toFixed(6);
|
|
144
|
+
}
|
|
145
|
+
function escapeTableCell(value) {
|
|
146
|
+
return value.replace(/\|/g, '\\|');
|
|
147
|
+
}
|
|
148
|
+
//# sourceMappingURL=prediction-brier-scoreboard-report.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"prediction-brier-scoreboard-report.js","sourceRoot":"","sources":["../../src/corpus/prediction-brier-scoreboard-report.ts"],"names":[],"mappings":"AAEA,OAAO,EACL,+CAA+C,EAC/C,kCAAkC,GAInC,MAAM,kCAAkC,CAAC;AAE1C,MAAM,CAAC,MAAM,yCAAyC,GACpD,6DAA6D,CAAC;AAiBhE,MAAM,UAAU,yCAAyC,CACvD,KAA8C,EAC9C,QAAkD,EAAE;IAEpD,MAAM,QAAQ,GAAG,KAAK,CAAC,wBAAwB,CAAC;QAC9C,UAAU,EAAE,eAAe;QAC3B,IAAI,EAAE,SAAS;QACf,cAAc,EAAE,KAAK,CAAC,cAAc;QACpC,eAAe,EAAE,KAAK,CAAC,eAAe;QACtC,KAAK,EAAE,KAAK,CAAC,KAAK,IAAI,IAAI;KAC3B,CAAC,CAAC;IACH,MAAM,YAAY,GAAG,WAAW,CAAC,QAAQ,CAAC,OAAO,CAAC,mBAAmB,CAAC,CAAC,CAAC;IACxE,IAAI,YAAY,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,QAAQ,CAAC;IAE/C,MAAM,SAAS,GAAG,KAAK,CAAC,wBAAwB,CAAC;QAC/C,YAAY,EAAE,YAAY;QAC1B,UAAU,EAAE,eAAe;QAC3B,IAAI,EAAE,aAAa;QACnB,KAAK,EAAE,YAAY,CAAC,MAAM;KAC3B,CAAC,CAAC;IACH,OAAO,iBAAiB,CAAC,CAAC,GAAG,QAAQ,EAAE,GAAG,SAAS,CAAC,CAAC,CAAC;AACxD,CAAC;AAED,MAAM,UAAU,8BAA8B,CAC5C,KAA8C,EAC9C,UAAiD,EAAE;IAEnD,MAAM,eAAe,GAAG,uBAAuB,CAAC,OAAO,CAAC,CAAC;IACzD,OAAO,kCAAkC,CACvC,yCAAyC,CAAC,KAAK,EAAE,eAAe,CAAC,EACjE,OAAO,CACR,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,uCAAuC,CACrD,UAAqC,EACrC,UAAoD,EAAE;IAEtD,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,IAAI,iCAAiC,CAAC;IACjE,MAAM,cAAc,GAAG,OAAO,CAAC,cAAc,IAAI,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IAC1E,MAAM,MAAM,GAAG,gBAAgB,CAAC,UAAU,EAAE,cAAc,EAAE,OAAO,CAAC,cAAc,IAAI,EAAE,CAAC,CAAC;IAC1F,MAAM,KAAK,GAAa;QACtB,KAAK,KAAK,EAAE;QACZ,EAAE;QACF,cAAc,cAAc,EAAE;QAC9B,WAAW,MAAM,EAAE;QACnB,oBAAoB,UAAU,CAAC,kBAAkB,gBAAgB,iBAAiB,CAAC,UAAU,CAAC,eAAe,CAAC,EAAE;QAChH,EAAE;QACF,aAAa;QACb,EAAE;KACH,CAAC;IAEF,IAAI,UAAU,CAAC,OAAO,CAAC,kBAAkB,KAAK,CAAC,EAAE,CAAC;QAChD,KAAK,CAAC,IAAI,CACR,0JAA0J,EAC1J,EAAE,CACH,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,IAAI,CACR,oBAAoB,EACpB,YAAY,EACZ,yBAAyB,YAAY,CAAC,UAAU,CAAC,OAAO,CAAC,eAAe,CAAC,IAAI,EAC7E,yBAAyB,YAAY,CAAC,UAAU,CAAC,OAAO,CAAC,eAAe,CAAC,IAAI,EAC7E,4BAA4B,YAAY,CAAC,UAAU,CAAC,OAAO,CAAC,kBAAkB,CAAC,IAAI,EACnF,uBAAuB,UAAU,CAAC,OAAO,CAAC,kBAAkB,IAAI,EAChE,oBAAoB,UAAU,CAAC,OAAO,CAAC,iBAAiB,IAAI,EAC5D,wBAAwB,UAAU,CAAC,OAAO,CAAC,mBAAmB,IAAI,EAClE,EAAE,EACF,iBAAiB,EACjB,EAAE,EACF,WAAW,CACT,CAAC,MAAM,EAAE,iBAAiB,EAAE,aAAa,EAAE,cAAc,EAAE,iBAAiB,EAAE,OAAO,EAAE,WAAW,CAAC,EACnG,UAAU,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC;QACtC,OAAO,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC;QACjC,MAAM,CAAC,OAAO,CAAC,kBAAkB,CAAC;QAClC,YAAY,CAAC,OAAO,CAAC,eAAe,CAAC;QACrC,YAAY,CAAC,OAAO,CAAC,eAAe,CAAC;QACrC,YAAY,CAAC,OAAO,CAAC,kBAAkB,CAAC;QACxC,MAAM,CAAC,OAAO,CAAC,iBAAiB,CAAC;QACjC,MAAM,CAAC,OAAO,CAAC,mBAAmB,CAAC;KACpC,CAAC,EACF,2BAA2B,CAC5B,EACD,EAAE,EACF,cAAc,EACd,EAAE,EACF,WAAW,CACT,CAAC,UAAU,EAAE,iBAAiB,EAAE,aAAa,EAAE,cAAc,EAAE,iBAAiB,EAAE,OAAO,CAAC,EAC1F,UAAU,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC;QACtC,OAAO,CAAC,sBAAsB,IAAI,OAAO,CAAC,mBAAmB,IAAI,OAAO,CAAC,GAAG;QAC5E,MAAM,CAAC,OAAO,CAAC,kBAAkB,CAAC;QAClC,YAAY,CAAC,OAAO,CAAC,eAAe,CAAC;QACrC,YAAY,CAAC,OAAO,CAAC,eAAe,CAAC;QACrC,YAAY,CAAC,OAAO,CAAC,kBAAkB,CAAC;QACxC,MAAM,CAAC,OAAO,CAAC,iBAAiB,CAAC;KAClC,CAAC,EACF,uBAAuB,CACxB,EACD,EAAE,EACF,cAAc,EACd,EAAE,EACF,WAAW,CACT,CAAC,SAAS,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,aAAa,EAAE,OAAO,CAAC,EACxE,UAAU,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC;QACrC,CAAC,OAAO,CAAC,QAAQ,IAAI,SAAS,EAAE,OAAO,CAAC,WAAW,IAAI,SAAS,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC;QAC3E,OAAO,CAAC,mBAAmB,IAAI,SAAS;QACxC,MAAM,CAAC,OAAO,CAAC,kBAAkB,CAAC;QAClC,YAAY,CAAC,OAAO,CAAC,eAAe,CAAC;QACrC,MAAM,CAAC,OAAO,CAAC,iBAAiB,CAAC;KAClC,CAAC,EACF,sBAAsB,CACvB,EACD,EAAE,EACF,YAAY,EACZ,EAAE,EACF,WAAW,CACT,CAAC,QAAQ,EAAE,iBAAiB,EAAE,aAAa,EAAE,OAAO,CAAC,EACrD,UAAU,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC;QACpC,OAAO,CAAC,MAAM;QACd,MAAM,CAAC,OAAO,CAAC,kBAAkB,CAAC;QAClC,YAAY,CAAC,OAAO,CAAC,eAAe,CAAC;QACrC,MAAM,CAAC,OAAO,CAAC,iBAAiB,CAAC;KAClC,CAAC,EACF,qBAAqB,CACtB,EACD,EAAE,EACF,eAAe,EACf,EAAE,EACF,oBAAoB,EACpB,YAAY,EACZ,wBAAwB,UAAU,CAAC,QAAQ,CAAC,cAAc,IAAI,EAC9D,2BAA2B,UAAU,CAAC,QAAQ,CAAC,YAAY,IAAI,EAC/D,uCAAuC,UAAU,CAAC,QAAQ,CAAC,wBAAwB,IAAI,EACvF,+BAA+B,UAAU,CAAC,QAAQ,CAAC,oBAAoB,IAAI,EAC3E,0BAA0B,UAAU,CAAC,QAAQ,CAAC,gBAAgB,IAAI,EAClE,8BAA8B,UAAU,CAAC,QAAQ,CAAC,mBAAmB,IAAI,EACzE,+BAA+B,UAAU,CAAC,QAAQ,CAAC,iBAAiB,IAAI,CACzE,CAAC;IAEF,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC;AACjC,CAAC;AAED,SAAS,uBAAuB,CAC9B,OAA8C;IAE9C,MAAM,eAAe,GAAG,OAAO,CAAC,eAAe,IAAI,EAAE,CAAC;IACtD,IAAI,OAAO,CAAC,eAAe,KAAK,SAAS;QAAE,OAAO,eAAe,CAAC;IAElE,MAAM,kBAAkB,GAAG,OAAO,CAAC,kBAAkB;WAChD,+CAA+C,CAAC;IACrD,OAAO;QACL,GAAG,eAAe;QAClB,cAAc,EAAE,eAAe,CAAC,cAAc;eACzC,OAAO,CAAC,eAAe,GAAG,sBAAsB,CAAC,OAAO,CAAC,eAAe,EAAE,kBAAkB,CAAC;QAClG,eAAe,EAAE,eAAe,CAAC,eAAe,IAAI,OAAO,CAAC,eAAe;KAC5E,CAAC;AACJ,CAAC;AAED,SAAS,sBAAsB,CAAC,eAAuB,EAAE,IAAY;IACnE,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,eAAe,CAAC,IAAI,cAAc,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,MAAM,CAAC;IAClF,OAAO,IAAI,GAAG,OAAO,CAAC;AACxB,CAAC;AAED,SAAS,mBAAmB,CAAC,UAA8B;IACzD,OAAO;QACL,UAAU,CAAC,mBAAmB;QAC9B,UAAU,CAAC,sBAAsB;QACjC,UAAU,CAAC,mBAAmB;QAC9B,UAAU,CAAC,QAAQ,CAAC,sBAAsB,CAAC;QAC3C,UAAU,CAAC,QAAQ,CAAC,yBAAyB,CAAC;KAC/C,CAAC,MAAM,CAAC,CAAC,KAAK,EAAmB,EAAE,CAAC,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;AACtF,CAAC;AAED,SAAS,WAAW,CAAC,IAAuB;IAC1C,OAAO,CAAC,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC;AAC5B,CAAC;AAED,SAAS,iBAAiB,CAAC,WAA0C;IACnE,MAAM,IAAI,GAAG,IAAI,GAAG,EAA8B,CAAC;IACnD,KAAK,MAAM,UAAU,IAAI,WAAW;QAAE,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,UAAU,EAAE,UAAU,CAAC,CAAC;IAClF,OAAO,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;AAC5B,CAAC;AAED,SAAS,WAAW,CAAC,OAAiB,EAAE,IAAgB,EAAE,SAAiB;IACzE,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,SAAS,CAAC;IACxC,MAAM,SAAS,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACtF,MAAM,QAAQ,GAAG;QACf,KAAK,OAAO,CAAC,GAAG,CAAC,eAAe,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI;QACjD,IAAI,SAAS,GAAG;KACjB,CAAC;IACF,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,QAAQ,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,GAAG,CAAC,eAAe,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAC/D,CAAC;IACD,OAAO,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC7B,CAAC;AAED,SAAS,gBAAgB,CACvB,UAAqC,EACrC,cAAsB,EACtB,cAAsB;IAEtB,IAAI,UAAU,CAAC,OAAO,CAAC,kBAAkB,KAAK,CAAC;QAAE,OAAO,mBAAmB,CAAC;IAC5E,IAAI,UAAU,CAAC,eAAe,KAAK,IAAI;QAAE,OAAO,mBAAmB,CAAC;IAEpE,MAAM,aAAa,GAAG,IAAI,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC;IACjD,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,aAAa,CAAC;QAAE,OAAO,IAAI,CAAC;IACjD,MAAM,MAAM,GAAG,eAAe,CAAC,UAAU,CAAC,eAAe,CAAC,CAAC;IAC3D,MAAM,YAAY,GAAG,cAAc,GAAG,UAAU,CAAC;IACjD,IAAI,aAAa,GAAG,MAAM,GAAG,YAAY;QAAE,OAAO,2CAA2C,cAAc,QAAQ,CAAC;IACpH,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,iBAAiB,CAAC,WAA0B;IACnD,IAAI,WAAW,KAAK,IAAI;QAAE,OAAO,GAAG,CAAC;IACrC,OAAO,IAAI,IAAI,CAAC,eAAe,CAAC,WAAW,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC;AAC9D,CAAC;AAED,SAAS,eAAe,CAAC,WAAmB;IAC1C,OAAO,IAAI,CAAC,GAAG,CAAC,WAAW,CAAC,IAAI,cAAc,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,WAAW,GAAG,IAAI,CAAC;AACpF,CAAC;AAED,SAAS,YAAY,CAAC,KAAsD;IAC1E,OAAO,KAAK,KAAK,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;AACjD,CAAC;AAED,SAAS,eAAe,CAAC,KAAa;IACpC,OAAO,KAAK,CAAC,OAAO,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;AACrC,CAAC"}
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
import type { EnvelopeProjection } from './types.js';
|
|
2
|
+
export declare const DEFAULT_PREDICTION_BRIER_SCOREBOARD_WINDOW_DAYS = 84;
|
|
3
|
+
export interface PredictionBrierScoreboardOptions {
|
|
4
|
+
asOfGeneratedAt?: number;
|
|
5
|
+
trailingWindowDays?: number;
|
|
6
|
+
}
|
|
7
|
+
export interface PredictionBrierScoreboard {
|
|
8
|
+
asOfGeneratedAt: number | null;
|
|
9
|
+
windowStartGeneratedAt: number | null;
|
|
10
|
+
trailingWindowDays: number;
|
|
11
|
+
overall: PredictionBrierOverallSummary;
|
|
12
|
+
weeklyTrend: PredictionBrierWeeklySummary[];
|
|
13
|
+
perOperator: PredictionBrierOperatorSummary[];
|
|
14
|
+
perHarness: PredictionBrierHarnessSummary[];
|
|
15
|
+
perPlugin: PredictionBrierPluginSummary[];
|
|
16
|
+
excluded: PredictionBrierExclusionCounts;
|
|
17
|
+
}
|
|
18
|
+
export interface PredictionBrierOverallSummary extends PredictionBrierMetricSummary {
|
|
19
|
+
activeOperatorCount: number;
|
|
20
|
+
}
|
|
21
|
+
export interface PredictionBrierMetricSummary {
|
|
22
|
+
scoredVerdictCount: number;
|
|
23
|
+
distinctTaskCount: number;
|
|
24
|
+
meanBrierSpread: number | null;
|
|
25
|
+
meanSolverBrier: number | null;
|
|
26
|
+
meanConsensusBrier: number | null;
|
|
27
|
+
}
|
|
28
|
+
export interface PredictionBrierOperatorSummary extends PredictionBrierMetricSummary {
|
|
29
|
+
key: string;
|
|
30
|
+
participantSafeAddress: string | null;
|
|
31
|
+
participantAgentEoa: string | null;
|
|
32
|
+
}
|
|
33
|
+
export interface PredictionBrierHarnessSummary extends PredictionBrierMetricSummary {
|
|
34
|
+
key: string;
|
|
35
|
+
implName: string | null;
|
|
36
|
+
implVersion: string | null;
|
|
37
|
+
runtimeBundleDigest: string | null;
|
|
38
|
+
}
|
|
39
|
+
export interface PredictionBrierPluginSummary extends PredictionBrierMetricSummary {
|
|
40
|
+
key: string;
|
|
41
|
+
plugin: string;
|
|
42
|
+
}
|
|
43
|
+
export interface PredictionBrierWeeklySummary extends PredictionBrierOverallSummary {
|
|
44
|
+
weekStartGeneratedAt: number;
|
|
45
|
+
weekStartIso: string;
|
|
46
|
+
}
|
|
47
|
+
export interface PredictionBrierExclusionCounts {
|
|
48
|
+
totalInputRows: number;
|
|
49
|
+
includedRows: number;
|
|
50
|
+
nonPredictionVerdictRows: number;
|
|
51
|
+
nonScoredVerdictRows: number;
|
|
52
|
+
missingScoreRows: number;
|
|
53
|
+
nonNumericScoreRows: number;
|
|
54
|
+
outsideWindowRows: number;
|
|
55
|
+
}
|
|
56
|
+
export declare function aggregatePredictionBrierScoreboard(projections: readonly EnvelopeProjection[], options?: PredictionBrierScoreboardOptions): PredictionBrierScoreboard;
|
|
@@ -0,0 +1,269 @@
|
|
|
1
|
+
export const DEFAULT_PREDICTION_BRIER_SCOREBOARD_WINDOW_DAYS = 84;
|
|
2
|
+
export function aggregatePredictionBrierScoreboard(projections, options = {}) {
|
|
3
|
+
const trailingWindowDays = options.trailingWindowDays
|
|
4
|
+
?? DEFAULT_PREDICTION_BRIER_SCOREBOARD_WINDOW_DAYS;
|
|
5
|
+
const excluded = {
|
|
6
|
+
totalInputRows: projections.length,
|
|
7
|
+
includedRows: 0,
|
|
8
|
+
nonPredictionVerdictRows: 0,
|
|
9
|
+
nonScoredVerdictRows: 0,
|
|
10
|
+
missingScoreRows: 0,
|
|
11
|
+
nonNumericScoreRows: 0,
|
|
12
|
+
outsideWindowRows: 0,
|
|
13
|
+
};
|
|
14
|
+
const candidates = [];
|
|
15
|
+
const solutionProjectionByRef = indexSolutionProjections(projections);
|
|
16
|
+
for (const projection of projections) {
|
|
17
|
+
const parsed = parseScoreableProjection(projection, solutionProjectionByRef, excluded);
|
|
18
|
+
if (parsed)
|
|
19
|
+
candidates.push(parsed);
|
|
20
|
+
}
|
|
21
|
+
const asOfGeneratedAt = options.asOfGeneratedAt ?? maxGeneratedAt(candidates.map((row) => row.projection));
|
|
22
|
+
const windowStartGeneratedAt = asOfGeneratedAt === null
|
|
23
|
+
? null
|
|
24
|
+
: asOfGeneratedAt - generatedAtSpanForDays(asOfGeneratedAt, trailingWindowDays);
|
|
25
|
+
const included = windowStartGeneratedAt === null || asOfGeneratedAt === null
|
|
26
|
+
? []
|
|
27
|
+
: candidates.filter((row) => {
|
|
28
|
+
const insideWindow = row.projection.generatedAt >= windowStartGeneratedAt
|
|
29
|
+
&& row.projection.generatedAt <= asOfGeneratedAt;
|
|
30
|
+
if (!insideWindow)
|
|
31
|
+
excluded.outsideWindowRows += 1;
|
|
32
|
+
return insideWindow;
|
|
33
|
+
});
|
|
34
|
+
excluded.includedRows = included.length;
|
|
35
|
+
return {
|
|
36
|
+
asOfGeneratedAt,
|
|
37
|
+
windowStartGeneratedAt,
|
|
38
|
+
trailingWindowDays,
|
|
39
|
+
overall: summarizeOverall(included),
|
|
40
|
+
weeklyTrend: summarizeWeeklyTrend(included),
|
|
41
|
+
perOperator: summarizeByOperator(included),
|
|
42
|
+
perHarness: summarizeByHarness(included),
|
|
43
|
+
perPlugin: summarizeByPlugin(included),
|
|
44
|
+
excluded,
|
|
45
|
+
};
|
|
46
|
+
}
|
|
47
|
+
function parseScoreableProjection(projection, solutionProjectionByRef, excluded) {
|
|
48
|
+
if (projection.solverType !== 'prediction.v1' || projection.role !== 'verdict') {
|
|
49
|
+
excluded.nonPredictionVerdictRows += 1;
|
|
50
|
+
return null;
|
|
51
|
+
}
|
|
52
|
+
if (projection.metadata['verdict'] !== 'SCORED') {
|
|
53
|
+
excluded.nonScoredVerdictRows += 1;
|
|
54
|
+
return null;
|
|
55
|
+
}
|
|
56
|
+
const solverBrier = parseRequiredFiniteNumber(projection.metadata['scores.solverBrier']);
|
|
57
|
+
const consensusBrier = parseRequiredFiniteNumber(projection.metadata['scores.consensusBrier']);
|
|
58
|
+
const brierSpread = parseRequiredFiniteNumber(projection.metadata['scores.brierSpread']);
|
|
59
|
+
if (solverBrier.kind === 'missing' || consensusBrier.kind === 'missing' || brierSpread.kind === 'missing') {
|
|
60
|
+
excluded.missingScoreRows += 1;
|
|
61
|
+
return null;
|
|
62
|
+
}
|
|
63
|
+
if (solverBrier.kind === 'non-numeric' || consensusBrier.kind === 'non-numeric' || brierSpread.kind === 'non-numeric') {
|
|
64
|
+
excluded.nonNumericScoreRows += 1;
|
|
65
|
+
return null;
|
|
66
|
+
}
|
|
67
|
+
const attributionProjection = solutionAttributionProjection(projection, solutionProjectionByRef);
|
|
68
|
+
return {
|
|
69
|
+
projection,
|
|
70
|
+
attributionProjection,
|
|
71
|
+
solverBrier: solverBrier.value,
|
|
72
|
+
consensusBrier: consensusBrier.value,
|
|
73
|
+
brierSpread: brierSpread.value,
|
|
74
|
+
taskKey: taskKey(projection),
|
|
75
|
+
operatorKey: operatorKey(attributionProjection),
|
|
76
|
+
harnessKey: harnessKey(attributionProjection),
|
|
77
|
+
};
|
|
78
|
+
}
|
|
79
|
+
function summarizeOverall(rows) {
|
|
80
|
+
return {
|
|
81
|
+
...summarizeMetrics(rows),
|
|
82
|
+
activeOperatorCount: new Set(rows.map((row) => row.operatorKey)).size,
|
|
83
|
+
};
|
|
84
|
+
}
|
|
85
|
+
function summarizeMetrics(rows) {
|
|
86
|
+
return {
|
|
87
|
+
scoredVerdictCount: rows.length,
|
|
88
|
+
distinctTaskCount: new Set(rows.map((row) => row.taskKey)).size,
|
|
89
|
+
meanBrierSpread: mean(rows.map((row) => row.brierSpread)),
|
|
90
|
+
meanSolverBrier: mean(rows.map((row) => row.solverBrier)),
|
|
91
|
+
meanConsensusBrier: mean(rows.map((row) => row.consensusBrier)),
|
|
92
|
+
};
|
|
93
|
+
}
|
|
94
|
+
function summarizeWeeklyTrend(rows) {
|
|
95
|
+
const byWeek = new Map();
|
|
96
|
+
for (const row of rows) {
|
|
97
|
+
const bucket = weekBucket(row.projection.generatedAt);
|
|
98
|
+
const existing = byWeek.get(bucket.key) ?? [];
|
|
99
|
+
existing.push(row);
|
|
100
|
+
byWeek.set(bucket.key, existing);
|
|
101
|
+
}
|
|
102
|
+
return [...byWeek.entries()]
|
|
103
|
+
.map(([key, bucketRows]) => {
|
|
104
|
+
const generatedAt = Number(key);
|
|
105
|
+
return {
|
|
106
|
+
weekStartGeneratedAt: generatedAt,
|
|
107
|
+
weekStartIso: generatedAtIso(generatedAt),
|
|
108
|
+
...summarizeOverall(bucketRows),
|
|
109
|
+
};
|
|
110
|
+
})
|
|
111
|
+
.sort((a, b) => a.weekStartGeneratedAt - b.weekStartGeneratedAt);
|
|
112
|
+
}
|
|
113
|
+
function summarizeByOperator(rows) {
|
|
114
|
+
return summarizeGroups(rows, (row) => row.operatorKey, (key, groupRows) => ({
|
|
115
|
+
key,
|
|
116
|
+
participantSafeAddress: firstPresent(groupRows.map((row) => row.attributionProjection.participantSafeAddress)),
|
|
117
|
+
participantAgentEoa: firstPresent(groupRows.map((row) => row.attributionProjection.participantAgentEoa)),
|
|
118
|
+
}));
|
|
119
|
+
}
|
|
120
|
+
function summarizeByHarness(rows) {
|
|
121
|
+
return summarizeGroups(rows, (row) => row.harnessKey, (key, groupRows) => ({
|
|
122
|
+
key,
|
|
123
|
+
implName: firstPresent(groupRows.map((row) => row.attributionProjection.executorImplName)),
|
|
124
|
+
implVersion: firstPresent(groupRows.map((row) => row.attributionProjection.executorImplVersion)),
|
|
125
|
+
runtimeBundleDigest: firstPresent(groupRows.map((row) => row.attributionProjection.executorRuntimeBundleDigest)),
|
|
126
|
+
}));
|
|
127
|
+
}
|
|
128
|
+
function summarizeByPlugin(rows) {
|
|
129
|
+
const expanded = [];
|
|
130
|
+
for (const row of rows) {
|
|
131
|
+
for (const plugin of new Set(row.attributionProjection.executorPlugins)) {
|
|
132
|
+
expanded.push({ row, plugin });
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
return summarizeGroups(expanded, (entry) => entry.plugin, (key) => ({ key, plugin: key }), (entry) => entry.row);
|
|
136
|
+
}
|
|
137
|
+
function summarizeGroups(entries, keyOf, metadataOf, rowOf = (entry) => entry) {
|
|
138
|
+
const groups = new Map();
|
|
139
|
+
for (const entry of entries) {
|
|
140
|
+
const key = keyOf(entry);
|
|
141
|
+
const existing = groups.get(key) ?? [];
|
|
142
|
+
existing.push(rowOf(entry));
|
|
143
|
+
groups.set(key, existing);
|
|
144
|
+
}
|
|
145
|
+
return [...groups.entries()]
|
|
146
|
+
.map(([key, groupRows]) => ({
|
|
147
|
+
key,
|
|
148
|
+
...metadataOf(key, groupRows),
|
|
149
|
+
...summarizeMetrics(groupRows),
|
|
150
|
+
}))
|
|
151
|
+
.sort(groupSummarySort);
|
|
152
|
+
}
|
|
153
|
+
function groupSummarySort(a, b) {
|
|
154
|
+
if (b.scoredVerdictCount !== a.scoredVerdictCount)
|
|
155
|
+
return b.scoredVerdictCount - a.scoredVerdictCount;
|
|
156
|
+
const aSpread = a.meanBrierSpread ?? Number.POSITIVE_INFINITY;
|
|
157
|
+
const bSpread = b.meanBrierSpread ?? Number.POSITIVE_INFINITY;
|
|
158
|
+
if (aSpread !== bSpread)
|
|
159
|
+
return aSpread - bSpread;
|
|
160
|
+
return a.key.localeCompare(b.key);
|
|
161
|
+
}
|
|
162
|
+
function parseRequiredFiniteNumber(value) {
|
|
163
|
+
if (value === undefined || value === null || value === '')
|
|
164
|
+
return { kind: 'missing' };
|
|
165
|
+
const parsed = typeof value === 'number' ? value : Number(value);
|
|
166
|
+
if (!Number.isFinite(parsed))
|
|
167
|
+
return { kind: 'non-numeric' };
|
|
168
|
+
return { kind: 'ok', value: parsed };
|
|
169
|
+
}
|
|
170
|
+
function indexSolutionProjections(projections) {
|
|
171
|
+
const byRef = new Map();
|
|
172
|
+
for (const projection of projections) {
|
|
173
|
+
if (projection.solverType !== 'prediction.v1' || projection.role !== 'restoration')
|
|
174
|
+
continue;
|
|
175
|
+
for (const ref of solutionProjectionRefs(projection)) {
|
|
176
|
+
if (!byRef.has(ref))
|
|
177
|
+
byRef.set(ref, projection);
|
|
178
|
+
}
|
|
179
|
+
}
|
|
180
|
+
return byRef;
|
|
181
|
+
}
|
|
182
|
+
function solutionAttributionProjection(verdictProjection, solutionProjectionByRef) {
|
|
183
|
+
for (const ref of verdictSolutionRefs(verdictProjection)) {
|
|
184
|
+
const solutionProjection = solutionProjectionByRef.get(ref);
|
|
185
|
+
if (solutionProjection)
|
|
186
|
+
return solutionProjection;
|
|
187
|
+
}
|
|
188
|
+
return verdictProjection;
|
|
189
|
+
}
|
|
190
|
+
function solutionProjectionRefs(projection) {
|
|
191
|
+
return compactRefs([
|
|
192
|
+
projection.envelopeCid,
|
|
193
|
+
projection.envelopeSha256,
|
|
194
|
+
projection.envelopeId,
|
|
195
|
+
projection.signatureHash,
|
|
196
|
+
]);
|
|
197
|
+
}
|
|
198
|
+
function verdictSolutionRefs(projection) {
|
|
199
|
+
return compactRefs([
|
|
200
|
+
projection.solutionEnvelopeCid,
|
|
201
|
+
projection.solutionEnvelopeSha256,
|
|
202
|
+
projection.solutionEnvelopeRef,
|
|
203
|
+
projection.metadata['solutionEnvelope.cid'],
|
|
204
|
+
projection.metadata['solutionEnvelope.sha256'],
|
|
205
|
+
]);
|
|
206
|
+
}
|
|
207
|
+
function compactRefs(values) {
|
|
208
|
+
return values.filter((value) => typeof value === 'string' && value.length > 0);
|
|
209
|
+
}
|
|
210
|
+
function mean(values) {
|
|
211
|
+
if (values.length === 0)
|
|
212
|
+
return null;
|
|
213
|
+
return values.reduce((sum, value) => sum + value, 0) / values.length;
|
|
214
|
+
}
|
|
215
|
+
function taskKey(projection) {
|
|
216
|
+
if (projection.taskCid)
|
|
217
|
+
return `taskCid:${projection.taskCid}`;
|
|
218
|
+
if (projection.taskId)
|
|
219
|
+
return `taskId:${projection.taskId}`;
|
|
220
|
+
return `unknown-task:${projection.envelopeId}`;
|
|
221
|
+
}
|
|
222
|
+
function operatorKey(projection) {
|
|
223
|
+
if (projection.participantSafeAddress)
|
|
224
|
+
return `safe:${projection.participantSafeAddress.toLowerCase()}`;
|
|
225
|
+
if (projection.participantAgentEoa)
|
|
226
|
+
return `agent:${projection.participantAgentEoa.toLowerCase()}`;
|
|
227
|
+
return 'unknown-operator';
|
|
228
|
+
}
|
|
229
|
+
function harnessKey(projection) {
|
|
230
|
+
return [
|
|
231
|
+
projection.executorImplName ?? 'unknown-harness',
|
|
232
|
+
projection.executorImplVersion ?? 'unknown-version',
|
|
233
|
+
projection.executorRuntimeBundleDigest ?? 'unknown-runtime',
|
|
234
|
+
].join('@');
|
|
235
|
+
}
|
|
236
|
+
function firstPresent(values) {
|
|
237
|
+
return values.find((value) => typeof value === 'string' && value.length > 0) ?? null;
|
|
238
|
+
}
|
|
239
|
+
function maxGeneratedAt(projections) {
|
|
240
|
+
if (projections.length === 0)
|
|
241
|
+
return null;
|
|
242
|
+
return Math.max(...projections.map((projection) => projection.generatedAt));
|
|
243
|
+
}
|
|
244
|
+
function generatedAtSpanForDays(asOfGeneratedAt, days) {
|
|
245
|
+
const unit = timestampUnit(asOfGeneratedAt);
|
|
246
|
+
const daySpan = unit === 'milliseconds' ? 86_400_000 : 86_400;
|
|
247
|
+
return days * daySpan;
|
|
248
|
+
}
|
|
249
|
+
function weekBucket(generatedAt) {
|
|
250
|
+
const unit = timestampUnit(generatedAt);
|
|
251
|
+
const date = generatedAtDate(generatedAt);
|
|
252
|
+
const daysSinceMonday = (date.getUTCDay() + 6) % 7;
|
|
253
|
+
date.setUTCDate(date.getUTCDate() - daysSinceMonday);
|
|
254
|
+
date.setUTCHours(0, 0, 0, 0);
|
|
255
|
+
const weekStart = unit === 'milliseconds' ? date.getTime() : Math.floor(date.getTime() / 1000);
|
|
256
|
+
return { key: String(weekStart) };
|
|
257
|
+
}
|
|
258
|
+
function generatedAtIso(generatedAt) {
|
|
259
|
+
return generatedAtDate(generatedAt).toISOString();
|
|
260
|
+
}
|
|
261
|
+
function generatedAtDate(generatedAt) {
|
|
262
|
+
return timestampUnit(generatedAt) === 'milliseconds'
|
|
263
|
+
? new Date(generatedAt)
|
|
264
|
+
: new Date(generatedAt * 1000);
|
|
265
|
+
}
|
|
266
|
+
function timestampUnit(generatedAt) {
|
|
267
|
+
return Math.abs(generatedAt) >= 10_000_000_000 ? 'milliseconds' : 'seconds';
|
|
268
|
+
}
|
|
269
|
+
//# sourceMappingURL=prediction-brier-scoreboard.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"prediction-brier-scoreboard.js","sourceRoot":"","sources":["../../src/corpus/prediction-brier-scoreboard.ts"],"names":[],"mappings":"AAEA,MAAM,CAAC,MAAM,+CAA+C,GAAG,EAAE,CAAC;AA6ElE,MAAM,UAAU,kCAAkC,CAChD,WAA0C,EAC1C,UAA4C,EAAE;IAE9C,MAAM,kBAAkB,GAAG,OAAO,CAAC,kBAAkB;WAChD,+CAA+C,CAAC;IACrD,MAAM,QAAQ,GAAmC;QAC/C,cAAc,EAAE,WAAW,CAAC,MAAM;QAClC,YAAY,EAAE,CAAC;QACf,wBAAwB,EAAE,CAAC;QAC3B,oBAAoB,EAAE,CAAC;QACvB,gBAAgB,EAAE,CAAC;QACnB,mBAAmB,EAAE,CAAC;QACtB,iBAAiB,EAAE,CAAC;KACrB,CAAC;IACF,MAAM,UAAU,GAAkC,EAAE,CAAC;IACrD,MAAM,uBAAuB,GAAG,wBAAwB,CAAC,WAAW,CAAC,CAAC;IAEtE,KAAK,MAAM,UAAU,IAAI,WAAW,EAAE,CAAC;QACrC,MAAM,MAAM,GAAG,wBAAwB,CAAC,UAAU,EAAE,uBAAuB,EAAE,QAAQ,CAAC,CAAC;QACvF,IAAI,MAAM;YAAE,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IACtC,CAAC;IAED,MAAM,eAAe,GAAG,OAAO,CAAC,eAAe,IAAI,cAAc,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC,CAAC;IAC3G,MAAM,sBAAsB,GAAG,eAAe,KAAK,IAAI;QACrD,CAAC,CAAC,IAAI;QACN,CAAC,CAAC,eAAe,GAAG,sBAAsB,CAAC,eAAe,EAAE,kBAAkB,CAAC,CAAC;IAElF,MAAM,QAAQ,GAAG,sBAAsB,KAAK,IAAI,IAAI,eAAe,KAAK,IAAI;QAC1E,CAAC,CAAC,EAAE;QACJ,CAAC,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE;YACxB,MAAM,YAAY,GAAG,GAAG,CAAC,UAAU,CAAC,WAAW,IAAI,sBAAsB;mBACpE,GAAG,CAAC,UAAU,CAAC,WAAW,IAAI,eAAe,CAAC;YACnD,IAAI,CAAC,YAAY;gBAAE,QAAQ,CAAC,iBAAiB,IAAI,CAAC,CAAC;YACnD,OAAO,YAAY,CAAC;QACtB,CAAC,CAAC,CAAC;IAEP,QAAQ,CAAC,YAAY,GAAG,QAAQ,CAAC,MAAM,CAAC;IAExC,OAAO;QACL,eAAe;QACf,sBAAsB;QACtB,kBAAkB;QAClB,OAAO,EAAE,gBAAgB,CAAC,QAAQ,CAAC;QACnC,WAAW,EAAE,oBAAoB,CAAC,QAAQ,CAAC;QAC3C,WAAW,EAAE,mBAAmB,CAAC,QAAQ,CAAC;QAC1C,UAAU,EAAE,kBAAkB,CAAC,QAAQ,CAAC;QACxC,SAAS,EAAE,iBAAiB,CAAC,QAAQ,CAAC;QACtC,QAAQ;KACT,CAAC;AACJ,CAAC;AAED,SAAS,wBAAwB,CAC/B,UAA8B,EAC9B,uBAAgE,EAChE,QAAwC;IAExC,IAAI,UAAU,CAAC,UAAU,KAAK,eAAe,IAAI,UAAU,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;QAC/E,QAAQ,CAAC,wBAAwB,IAAI,CAAC,CAAC;QACvC,OAAO,IAAI,CAAC;IACd,CAAC;IACD,IAAI,UAAU,CAAC,QAAQ,CAAC,SAAS,CAAC,KAAK,QAAQ,EAAE,CAAC;QAChD,QAAQ,CAAC,oBAAoB,IAAI,CAAC,CAAC;QACnC,OAAO,IAAI,CAAC;IACd,CAAC;IAED,MAAM,WAAW,GAAG,yBAAyB,CAAC,UAAU,CAAC,QAAQ,CAAC,oBAAoB,CAAC,CAAC,CAAC;IACzF,MAAM,cAAc,GAAG,yBAAyB,CAAC,UAAU,CAAC,QAAQ,CAAC,uBAAuB,CAAC,CAAC,CAAC;IAC/F,MAAM,WAAW,GAAG,yBAAyB,CAAC,UAAU,CAAC,QAAQ,CAAC,oBAAoB,CAAC,CAAC,CAAC;IACzF,IAAI,WAAW,CAAC,IAAI,KAAK,SAAS,IAAI,cAAc,CAAC,IAAI,KAAK,SAAS,IAAI,WAAW,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;QAC1G,QAAQ,CAAC,gBAAgB,IAAI,CAAC,CAAC;QAC/B,OAAO,IAAI,CAAC;IACd,CAAC;IACD,IAAI,WAAW,CAAC,IAAI,KAAK,aAAa,IAAI,cAAc,CAAC,IAAI,KAAK,aAAa,IAAI,WAAW,CAAC,IAAI,KAAK,aAAa,EAAE,CAAC;QACtH,QAAQ,CAAC,mBAAmB,IAAI,CAAC,CAAC;QAClC,OAAO,IAAI,CAAC;IACd,CAAC;IAED,MAAM,qBAAqB,GAAG,6BAA6B,CAAC,UAAU,EAAE,uBAAuB,CAAC,CAAC;IAEjG,OAAO;QACL,UAAU;QACV,qBAAqB;QACrB,WAAW,EAAE,WAAW,CAAC,KAAK;QAC9B,cAAc,EAAE,cAAc,CAAC,KAAK;QACpC,WAAW,EAAE,WAAW,CAAC,KAAK;QAC9B,OAAO,EAAE,OAAO,CAAC,UAAU,CAAC;QAC5B,WAAW,EAAE,WAAW,CAAC,qBAAqB,CAAC;QAC/C,UAAU,EAAE,UAAU,CAAC,qBAAqB,CAAC;KAC9C,CAAC;AACJ,CAAC;AAED,SAAS,gBAAgB,CAAC,IAA4C;IACpE,OAAO;QACL,GAAG,gBAAgB,CAAC,IAAI,CAAC;QACzB,mBAAmB,EAAE,IAAI,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC,CAAC,IAAI;KACtE,CAAC;AACJ,CAAC;AAED,SAAS,gBAAgB,CAAC,IAA4C;IACpE,OAAO;QACL,kBAAkB,EAAE,IAAI,CAAC,MAAM;QAC/B,iBAAiB,EAAE,IAAI,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI;QAC/D,eAAe,EAAE,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;QACzD,eAAe,EAAE,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;QACzD,kBAAkB,EAAE,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC;KAChE,CAAC;AACJ,CAAC;AAED,SAAS,oBAAoB,CAAC,IAA4C;IACxE,MAAM,MAAM,GAAG,IAAI,GAAG,EAAyC,CAAC;IAEhE,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,MAAM,MAAM,GAAG,UAAU,CAAC,GAAG,CAAC,UAAU,CAAC,WAAW,CAAC,CAAC;QACtD,MAAM,QAAQ,GAAG,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC;QAC9C,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACnB,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;IACnC,CAAC;IAED,OAAO,CAAC,GAAG,MAAM,CAAC,OAAO,EAAE,CAAC;SACzB,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,UAAU,CAAC,EAAE,EAAE;QACzB,MAAM,WAAW,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC;QAChC,OAAO;YACL,oBAAoB,EAAE,WAAW;YACjC,YAAY,EAAE,cAAc,CAAC,WAAW,CAAC;YACzC,GAAG,gBAAgB,CAAC,UAAU,CAAC;SAChC,CAAC;IACJ,CAAC,CAAC;SACD,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,oBAAoB,GAAG,CAAC,CAAC,oBAAoB,CAAC,CAAC;AACrE,CAAC;AAED,SAAS,mBAAmB,CAAC,IAA4C;IACvE,OAAO,eAAe,CACpB,IAAI,EACJ,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,WAAW,EACxB,CAAC,GAAG,EAAE,SAAS,EAAE,EAAE,CAAC,CAAC;QACnB,GAAG;QACH,sBAAsB,EAAE,YAAY,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,qBAAqB,CAAC,sBAAsB,CAAC,CAAC;QAC9G,mBAAmB,EAAE,YAAY,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,qBAAqB,CAAC,mBAAmB,CAAC,CAAC;KACzG,CAAC,CACH,CAAC;AACJ,CAAC;AAED,SAAS,kBAAkB,CAAC,IAA4C;IACtE,OAAO,eAAe,CACpB,IAAI,EACJ,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,UAAU,EACvB,CAAC,GAAG,EAAE,SAAS,EAAE,EAAE,CAAC,CAAC;QACnB,GAAG;QACH,QAAQ,EAAE,YAAY,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,qBAAqB,CAAC,gBAAgB,CAAC,CAAC;QAC1F,WAAW,EAAE,YAAY,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,qBAAqB,CAAC,mBAAmB,CAAC,CAAC;QAChG,mBAAmB,EAAE,YAAY,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,qBAAqB,CAAC,2BAA2B,CAAC,CAAC;KACjH,CAAC,CACH,CAAC;AACJ,CAAC;AAED,SAAS,iBAAiB,CAAC,IAA4C;IACrE,MAAM,QAAQ,GAAgE,EAAE,CAAC;IACjF,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,KAAK,MAAM,MAAM,IAAI,IAAI,GAAG,CAAC,GAAG,CAAC,qBAAqB,CAAC,eAAe,CAAC,EAAE,CAAC;YACxE,QAAQ,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,CAAC,CAAC;QACjC,CAAC;IACH,CAAC;IAED,OAAO,eAAe,CACpB,QAAQ,EACR,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,EACvB,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC,EAC/B,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,GAAG,CACrB,CAAC;AACJ,CAAC;AAED,SAAS,eAAe,CACtB,OAAqB,EACrB,KAA2B,EAC3B,UAAwE,EACxE,QAAmD,CAAC,KAAK,EAAE,EAAE,CAAC,KAAoC;IAElG,MAAM,MAAM,GAAG,IAAI,GAAG,EAAyC,CAAC;IAChE,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;QAC5B,MAAM,GAAG,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC;QACzB,MAAM,QAAQ,GAAG,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC;QACvC,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC;QAC5B,MAAM,CAAC,GAAG,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;IAC5B,CAAC;IAED,OAAO,CAAC,GAAG,MAAM,CAAC,OAAO,EAAE,CAAC;SACzB,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,SAAS,CAAC,EAAE,EAAE,CAAC,CAAC;QAC1B,GAAG;QACH,GAAG,UAAU,CAAC,GAAG,EAAE,SAAS,CAAC;QAC7B,GAAG,gBAAgB,CAAC,SAAS,CAAC;KAC/B,CAAC,CAAC;SACF,IAAI,CAAC,gBAAgB,CAAC,CAAC;AAC5B,CAAC;AAED,SAAS,gBAAgB,CAAC,CAAiD,EAAE,CAAiD;IAC5H,IAAI,CAAC,CAAC,kBAAkB,KAAK,CAAC,CAAC,kBAAkB;QAAE,OAAO,CAAC,CAAC,kBAAkB,GAAG,CAAC,CAAC,kBAAkB,CAAC;IACtG,MAAM,OAAO,GAAG,CAAC,CAAC,eAAe,IAAI,MAAM,CAAC,iBAAiB,CAAC;IAC9D,MAAM,OAAO,GAAG,CAAC,CAAC,eAAe,IAAI,MAAM,CAAC,iBAAiB,CAAC;IAC9D,IAAI,OAAO,KAAK,OAAO;QAAE,OAAO,OAAO,GAAG,OAAO,CAAC;IAClD,OAAO,CAAC,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;AACpC,CAAC;AAOD,SAAS,yBAAyB,CAAC,KAAc;IAC/C,IAAI,KAAK,KAAK,SAAS,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,KAAK,EAAE;QAAE,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,CAAC;IACtF,MAAM,MAAM,GAAG,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IACjE,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC;QAAE,OAAO,EAAE,IAAI,EAAE,aAAa,EAAE,CAAC;IAC7D,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC;AACvC,CAAC;AAED,SAAS,wBAAwB,CAAC,WAA0C;IAC1E,MAAM,KAAK,GAAG,IAAI,GAAG,EAA8B,CAAC;IAEpD,KAAK,MAAM,UAAU,IAAI,WAAW,EAAE,CAAC;QACrC,IAAI,UAAU,CAAC,UAAU,KAAK,eAAe,IAAI,UAAU,CAAC,IAAI,KAAK,aAAa;YAAE,SAAS;QAC7F,KAAK,MAAM,GAAG,IAAI,sBAAsB,CAAC,UAAU,CAAC,EAAE,CAAC;YACrD,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC;gBAAE,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,UAAU,CAAC,CAAC;QAClD,CAAC;IACH,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,6BAA6B,CACpC,iBAAqC,EACrC,uBAAgE;IAEhE,KAAK,MAAM,GAAG,IAAI,mBAAmB,CAAC,iBAAiB,CAAC,EAAE,CAAC;QACzD,MAAM,kBAAkB,GAAG,uBAAuB,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAC5D,IAAI,kBAAkB;YAAE,OAAO,kBAAkB,CAAC;IACpD,CAAC;IACD,OAAO,iBAAiB,CAAC;AAC3B,CAAC;AAED,SAAS,sBAAsB,CAAC,UAA8B;IAC5D,OAAO,WAAW,CAAC;QACjB,UAAU,CAAC,WAAW;QACtB,UAAU,CAAC,cAAc;QACzB,UAAU,CAAC,UAAU;QACrB,UAAU,CAAC,aAAa;KACzB,CAAC,CAAC;AACL,CAAC;AAED,SAAS,mBAAmB,CAAC,UAA8B;IACzD,OAAO,WAAW,CAAC;QACjB,UAAU,CAAC,mBAAmB;QAC9B,UAAU,CAAC,sBAAsB;QACjC,UAAU,CAAC,mBAAmB;QAC9B,UAAU,CAAC,QAAQ,CAAC,sBAAsB,CAAC;QAC3C,UAAU,CAAC,QAAQ,CAAC,yBAAyB,CAAC;KAC/C,CAAC,CAAC;AACL,CAAC;AAED,SAAS,WAAW,CAAC,MAA0B;IAC7C,OAAO,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,EAAmB,EAAE,CAAC,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;AAClG,CAAC;AAED,SAAS,IAAI,CAAC,MAAyB;IACrC,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IACrC,OAAO,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,GAAG,GAAG,KAAK,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC;AACvE,CAAC;AAED,SAAS,OAAO,CAAC,UAA8B;IAC7C,IAAI,UAAU,CAAC,OAAO;QAAE,OAAO,WAAW,UAAU,CAAC,OAAO,EAAE,CAAC;IAC/D,IAAI,UAAU,CAAC,MAAM;QAAE,OAAO,UAAU,UAAU,CAAC,MAAM,EAAE,CAAC;IAC5D,OAAO,gBAAgB,UAAU,CAAC,UAAU,EAAE,CAAC;AACjD,CAAC;AAED,SAAS,WAAW,CAAC,UAA8B;IACjD,IAAI,UAAU,CAAC,sBAAsB;QAAE,OAAO,QAAQ,UAAU,CAAC,sBAAsB,CAAC,WAAW,EAAE,EAAE,CAAC;IACxG,IAAI,UAAU,CAAC,mBAAmB;QAAE,OAAO,SAAS,UAAU,CAAC,mBAAmB,CAAC,WAAW,EAAE,EAAE,CAAC;IACnG,OAAO,kBAAkB,CAAC;AAC5B,CAAC;AAED,SAAS,UAAU,CAAC,UAA8B;IAChD,OAAO;QACL,UAAU,CAAC,gBAAgB,IAAI,iBAAiB;QAChD,UAAU,CAAC,mBAAmB,IAAI,iBAAiB;QACnD,UAAU,CAAC,2BAA2B,IAAI,iBAAiB;KAC5D,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AACd,CAAC;AAED,SAAS,YAAY,CAAC,MAAoC;IACxD,OAAO,MAAM,CAAC,IAAI,CAAC,CAAC,KAAK,EAAmB,EAAE,CAAC,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,IAAI,IAAI,CAAC;AACxG,CAAC;AAED,SAAS,cAAc,CAAC,WAA0C;IAChE,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IAC1C,OAAO,IAAI,CAAC,GAAG,CAAC,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC,UAAU,EAAE,EAAE,CAAC,UAAU,CAAC,WAAW,CAAC,CAAC,CAAC;AAC9E,CAAC;AAED,SAAS,sBAAsB,CAAC,eAAuB,EAAE,IAAY;IACnE,MAAM,IAAI,GAAG,aAAa,CAAC,eAAe,CAAC,CAAC;IAC5C,MAAM,OAAO,GAAG,IAAI,KAAK,cAAc,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,MAAM,CAAC;IAC9D,OAAO,IAAI,GAAG,OAAO,CAAC;AACxB,CAAC;AAED,SAAS,UAAU,CAAC,WAAmB;IACrC,MAAM,IAAI,GAAG,aAAa,CAAC,WAAW,CAAC,CAAC;IACxC,MAAM,IAAI,GAAG,eAAe,CAAC,WAAW,CAAC,CAAC;IAC1C,MAAM,eAAe,GAAG,CAAC,IAAI,CAAC,SAAS,EAAE,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC;IACnD,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,UAAU,EAAE,GAAG,eAAe,CAAC,CAAC;IACrD,IAAI,CAAC,WAAW,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;IAC7B,MAAM,SAAS,GAAG,IAAI,KAAK,cAAc,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,IAAI,CAAC,CAAC;IAC/F,OAAO,EAAE,GAAG,EAAE,MAAM,CAAC,SAAS,CAAC,EAAE,CAAC;AACpC,CAAC;AAED,SAAS,cAAc,CAAC,WAAmB;IACzC,OAAO,eAAe,CAAC,WAAW,CAAC,CAAC,WAAW,EAAE,CAAC;AACpD,CAAC;AAED,SAAS,eAAe,CAAC,WAAmB;IAC1C,OAAO,aAAa,CAAC,WAAW,CAAC,KAAK,cAAc;QAClD,CAAC,CAAC,IAAI,IAAI,CAAC,WAAW,CAAC;QACvB,CAAC,CAAC,IAAI,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,CAAC;AACnC,CAAC;AAED,SAAS,aAAa,CAAC,WAAmB;IACxC,OAAO,IAAI,CAAC,GAAG,CAAC,WAAW,CAAC,IAAI,cAAc,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,SAAS,CAAC;AAC9E,CAAC"}
|
|
@@ -0,0 +1,4 @@
|
|
|
1
|
+
import type { Store } from '../store/store.js';
|
|
2
|
+
import type { EnvelopeProjection, EnvelopeProjectionQuery } from './types.js';
|
|
3
|
+
export type ScoreablePredictionBrierVerdictQuery = Omit<EnvelopeProjectionQuery, 'solverType' | 'role' | 'metadata'>;
|
|
4
|
+
export declare function queryScoreablePredictionBrierVerdicts(store: Pick<Store, 'queryEnvelopeProjections'>, query?: ScoreablePredictionBrierVerdictQuery): EnvelopeProjection[];
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
export function queryScoreablePredictionBrierVerdicts(store, query = {}) {
|
|
2
|
+
const { limit, ...projectionQuery } = query;
|
|
3
|
+
const scoreable = store.queryEnvelopeProjections({
|
|
4
|
+
...projectionQuery,
|
|
5
|
+
limit: 1000,
|
|
6
|
+
solverType: 'prediction.v1',
|
|
7
|
+
role: 'verdict',
|
|
8
|
+
metadata: { verdict: 'SCORED' },
|
|
9
|
+
}).filter(hasBrierScoreMetadata);
|
|
10
|
+
if (limit === undefined)
|
|
11
|
+
return scoreable;
|
|
12
|
+
return scoreable.slice(0, Math.max(0, limit));
|
|
13
|
+
}
|
|
14
|
+
function hasBrierScoreMetadata(projection) {
|
|
15
|
+
return hasStringMetadata(projection, 'scores.solverBrier')
|
|
16
|
+
&& hasStringMetadata(projection, 'scores.consensusBrier')
|
|
17
|
+
&& hasStringMetadata(projection, 'scores.brierSpread');
|
|
18
|
+
}
|
|
19
|
+
function hasStringMetadata(projection, key) {
|
|
20
|
+
const value = projection.metadata[key];
|
|
21
|
+
return typeof value === 'string' && value.length > 0;
|
|
22
|
+
}
|
|
23
|
+
//# sourceMappingURL=prediction-scoreable-verdicts.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"prediction-scoreable-verdicts.js","sourceRoot":"","sources":["../../src/corpus/prediction-scoreable-verdicts.ts"],"names":[],"mappings":"AAQA,MAAM,UAAU,qCAAqC,CACnD,KAA8C,EAC9C,QAA8C,EAAE;IAEhD,MAAM,EAAE,KAAK,EAAE,GAAG,eAAe,EAAE,GAAG,KAAK,CAAC;IAC5C,MAAM,SAAS,GAAG,KAAK,CAAC,wBAAwB,CAAC;QAC/C,GAAG,eAAe;QAClB,KAAK,EAAE,IAAI;QACX,UAAU,EAAE,eAAe;QAC3B,IAAI,EAAE,SAAS;QACf,QAAQ,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE;KAChC,CAAC,CAAC,MAAM,CAAC,qBAAqB,CAAC,CAAC;IAEjC,IAAI,KAAK,KAAK,SAAS;QAAE,OAAO,SAAS,CAAC;IAC1C,OAAO,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC;AAChD,CAAC;AAED,SAAS,qBAAqB,CAAC,UAA8B;IAC3D,OAAO,iBAAiB,CAAC,UAAU,EAAE,oBAAoB,CAAC;WACrD,iBAAiB,CAAC,UAAU,EAAE,uBAAuB,CAAC;WACtD,iBAAiB,CAAC,UAAU,EAAE,oBAAoB,CAAC,CAAC;AAC3D,CAAC;AAED,SAAS,iBAAiB,CAAC,UAA8B,EAAE,GAAW;IACpE,MAAM,KAAK,GAAG,UAAU,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;IACvC,OAAO,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC;AACvD,CAAC"}
|
package/dist/corpus/types.d.ts
CHANGED
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
* narrative description of the read pipeline.
|
|
6
6
|
*/
|
|
7
7
|
import type { Store } from '../store/store.js';
|
|
8
|
-
import type { SignedEnvelope } from '../types/envelope.js';
|
|
8
|
+
import type { EvidenceTier, Role, SignedEnvelope } from '../types/envelope.js';
|
|
9
9
|
export interface CorpusOptions {
|
|
10
10
|
subgraphUrl: string;
|
|
11
11
|
ipfsGatewayUrl: string;
|
|
@@ -28,6 +28,48 @@ export interface CorpusQuery {
|
|
|
28
28
|
generatedBefore?: number;
|
|
29
29
|
limit?: number;
|
|
30
30
|
}
|
|
31
|
+
export type EnvelopeProjectionMetadataValue = string | number | boolean;
|
|
32
|
+
export type EnvelopeProjectionMetadata = Record<string, EnvelopeProjectionMetadataValue>;
|
|
33
|
+
export interface EnvelopeProjection {
|
|
34
|
+
envelopeId: string;
|
|
35
|
+
envelopeCid: string | null;
|
|
36
|
+
envelopeSha256: string | null;
|
|
37
|
+
signatureHash: string;
|
|
38
|
+
solverType: string;
|
|
39
|
+
role: Role;
|
|
40
|
+
taskCid: string | null;
|
|
41
|
+
taskId: string | null;
|
|
42
|
+
requestId: string | null;
|
|
43
|
+
generatedAt: number;
|
|
44
|
+
evidenceTier: EvidenceTier;
|
|
45
|
+
participantSafeAddress: string | null;
|
|
46
|
+
participantAgentEoa: string | null;
|
|
47
|
+
executorImplName: string | null;
|
|
48
|
+
executorImplVersion: string | null;
|
|
49
|
+
executorRuntimeBundleDigest: string | null;
|
|
50
|
+
executorPlugins: string[];
|
|
51
|
+
solutionEnvelopeCid: string | null;
|
|
52
|
+
solutionEnvelopeSha256: string | null;
|
|
53
|
+
solutionEnvelopeRef: string | null;
|
|
54
|
+
metadata: EnvelopeProjectionMetadata;
|
|
55
|
+
}
|
|
56
|
+
export interface EnvelopeProjectionQuery {
|
|
57
|
+
envelopeRefs?: readonly string[];
|
|
58
|
+
solverType?: string;
|
|
59
|
+
role?: Role;
|
|
60
|
+
taskCid?: string;
|
|
61
|
+
taskId?: string;
|
|
62
|
+
requestId?: string;
|
|
63
|
+
participant?: {
|
|
64
|
+
safeAddress?: string;
|
|
65
|
+
agentEoa?: string;
|
|
66
|
+
};
|
|
67
|
+
solutionEnvelopeRef?: string;
|
|
68
|
+
metadata?: EnvelopeProjectionMetadata;
|
|
69
|
+
generatedAfter?: number;
|
|
70
|
+
generatedBefore?: number;
|
|
71
|
+
limit?: number;
|
|
72
|
+
}
|
|
31
73
|
export interface ReadArgs {
|
|
32
74
|
query: CorpusQuery;
|
|
33
75
|
select?: (manifests: ManifestPreview[]) => ManifestPreview[];
|
package/dist/corpus/types.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/corpus/types.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;
|
|
1
|
+
{"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/corpus/types.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAuHH,0EAA0E;AAE1E,MAAM,OAAO,gBAAiB,SAAQ,KAAK;IACI;IAA7C,YAAY,OAAe,EAAkB,KAAe;QAC1D,KAAK,CAAC,OAAO,CAAC,CAAC;QAD4B,UAAK,GAAL,KAAK,CAAU;QAE1D,IAAI,CAAC,IAAI,GAAG,kBAAkB,CAAC;IACjC,CAAC;CACF;AAED,MAAM,OAAO,kBAAmB,SAAQ,KAAK;IACf;IAAsD;IAAlF,YAA4B,WAAmB,EAAE,OAAe,EAAkB,KAAe;QAC/F,KAAK,CAAC,YAAY,WAAW,KAAK,OAAO,EAAE,CAAC,CAAC;QADnB,gBAAW,GAAX,WAAW,CAAQ;QAAmC,UAAK,GAAL,KAAK,CAAU;QAE/F,IAAI,CAAC,IAAI,GAAG,oBAAoB,CAAC;IACnC,CAAC;CACF;AAED,MAAM,OAAO,YAAa,SAAQ,KAAK;IACT;IAAiD;IAA7E,YAA4B,MAAc,EAAE,OAAe,EAAkB,KAAe;QAC1F,KAAK,CAAC,WAAW,MAAM,KAAK,OAAO,EAAE,CAAC,CAAC;QADb,WAAM,GAAN,MAAM,CAAQ;QAAmC,UAAK,GAAL,KAAK,CAAU;QAE1F,IAAI,CAAC,IAAI,GAAG,cAAc,CAAC;IAC7B,CAAC;CACF;AAED,MAAM,OAAO,iBAAkB,SAAQ,KAAK;IAExB;IACA;IACA;IACA;IAJlB,YACkB,cAAsB,EACtB,YAAoB,EACpB,MAAc,EACd,cAAuB;QAEvC,KAAK,CAAC,2BAA2B,cAAc,SAAS,YAAY,SAAS,MAAM,EAAE,CAAC,CAAC;QALvE,mBAAc,GAAd,cAAc,CAAQ;QACtB,iBAAY,GAAZ,YAAY,CAAQ;QACpB,WAAM,GAAN,MAAM,CAAQ;QACd,mBAAc,GAAd,cAAc,CAAS;QAGvC,IAAI,CAAC,IAAI,GAAG,mBAAmB,CAAC;IAClC,CAAC;CACF"}
|
package/dist/daemon/daemon.js
CHANGED
|
@@ -278,7 +278,7 @@ export class Daemon {
|
|
|
278
278
|
continue;
|
|
279
279
|
const solverType = taskAnnouncement.task.solverType ?? undefined;
|
|
280
280
|
const taskRole = (taskAnnouncement.task.role ?? 'restoration');
|
|
281
|
-
const accept = await engine.canAcceptTask({ solverType, taskRole });
|
|
281
|
+
const accept = await engine.canAcceptTask({ solverType, taskRole, task: taskAnnouncement.task });
|
|
282
282
|
if (!accept.ok) {
|
|
283
283
|
console.log(`[daemon] skipping task ${taskAnnouncement.taskId} — ${accept.reason}`);
|
|
284
284
|
continue;
|