@rine-kit/pubg-intelligence 0.0.0-stage → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/dist/src/analytics.d.ts +176 -0
- package/dist/src/analytics.js +431 -0
- package/dist/src/analytics.js.map +1 -0
- package/dist/src/assistant.d.ts +87 -0
- package/dist/src/assistant.js +310 -0
- package/dist/src/assistant.js.map +1 -0
- package/dist/src/battle.d.ts +280 -0
- package/dist/src/battle.js +1249 -0
- package/dist/src/battle.js.map +1 -0
- package/dist/src/combat-long.d.ts +42 -0
- package/dist/src/combat-long.js +146 -0
- package/dist/src/combat-long.js.map +1 -0
- package/dist/src/combat-quality.d.ts +101 -0
- package/dist/src/combat-quality.js +253 -0
- package/dist/src/combat-quality.js.map +1 -0
- package/dist/src/evaluation-fixtures.d.ts +13 -0
- package/dist/src/evaluation-fixtures.js +144 -0
- package/dist/src/evaluation-fixtures.js.map +1 -0
- package/dist/src/evaluation-replay.d.ts +5 -0
- package/dist/src/evaluation-replay.js +107 -0
- package/dist/src/evaluation-replay.js.map +1 -0
- package/dist/src/evaluation.d.ts +140 -0
- package/dist/src/evaluation.js +180 -0
- package/dist/src/evaluation.js.map +1 -0
- package/dist/src/experiment-store.d.ts +56 -0
- package/dist/src/experiment-store.js +159 -0
- package/dist/src/experiment-store.js.map +1 -0
- package/dist/src/field.d.ts +103 -0
- package/dist/src/field.js +298 -0
- package/dist/src/field.js.map +1 -0
- package/dist/src/fights.d.ts +89 -0
- package/dist/src/fights.js +238 -0
- package/dist/src/fights.js.map +1 -0
- package/dist/src/highlights.d.ts +44 -0
- package/dist/src/highlights.js +153 -0
- package/dist/src/highlights.js.map +1 -0
- package/dist/src/index.d.ts +24 -0
- package/dist/src/index.js +25 -0
- package/dist/src/index.js.map +1 -0
- package/dist/src/insights.d.ts +44 -0
- package/dist/src/insights.js +343 -0
- package/dist/src/insights.js.map +1 -0
- package/dist/src/intent.d.ts +40 -0
- package/dist/src/intent.js +119 -0
- package/dist/src/intent.js.map +1 -0
- package/dist/src/landing.d.ts +74 -0
- package/dist/src/landing.js +195 -0
- package/dist/src/landing.js.map +1 -0
- package/dist/src/lobby.d.ts +48 -0
- package/dist/src/lobby.js +209 -0
- package/dist/src/lobby.js.map +1 -0
- package/dist/src/match-insight.d.ts +165 -0
- package/dist/src/match-insight.js +542 -0
- package/dist/src/match-insight.js.map +1 -0
- package/dist/src/match-relative.d.ts +73 -0
- package/dist/src/match-relative.js +145 -0
- package/dist/src/match-relative.js.map +1 -0
- package/dist/src/match-story.d.ts +155 -0
- package/dist/src/match-story.js +636 -0
- package/dist/src/match-story.js.map +1 -0
- package/dist/src/profiles.d.ts +41 -0
- package/dist/src/profiles.js +136 -0
- package/dist/src/profiles.js.map +1 -0
- package/dist/src/raw-telemetry-archive.d.ts +29 -0
- package/dist/src/raw-telemetry-archive.js +340 -0
- package/dist/src/raw-telemetry-archive.js.map +1 -0
- package/dist/src/raw-telemetry-backfill.d.ts +28 -0
- package/dist/src/raw-telemetry-backfill.js +144 -0
- package/dist/src/raw-telemetry-backfill.js.map +1 -0
- package/dist/src/research.d.ts +89 -0
- package/dist/src/research.js +376 -0
- package/dist/src/research.js.map +1 -0
- package/dist/src/rhythm.d.ts +104 -0
- package/dist/src/rhythm.js +260 -0
- package/dist/src/rhythm.js.map +1 -0
- package/dist/src/streaks.d.ts +117 -0
- package/dist/src/streaks.js +367 -0
- package/dist/src/streaks.js.map +1 -0
- package/dist/src/style.d.ts +304 -0
- package/dist/src/style.js +1461 -0
- package/dist/src/style.js.map +1 -0
- package/dist/src/tactics.d.ts +88 -0
- package/dist/src/tactics.js +1221 -0
- package/dist/src/tactics.js.map +1 -0
- package/package.json +49 -3
- package/README.md +0 -3
|
@@ -0,0 +1,140 @@
|
|
|
1
|
+
export declare const EVALUATION_MANIFEST_VERSION: 1;
|
|
2
|
+
export declare const EVALUATION_ENGINE_VERSION: "pubg-intelligence-evaluation-v1";
|
|
3
|
+
export type BenchmarkSplit = 'development' | 'holdout';
|
|
4
|
+
export type BenchmarkPrediction = boolean | null;
|
|
5
|
+
export type BenchmarkErrorKind = 'false_positive' | 'false_negative';
|
|
6
|
+
export interface BenchmarkEvidenceRef {
|
|
7
|
+
matchId: string;
|
|
8
|
+
eventId: string | null;
|
|
9
|
+
timeSeconds: number | null;
|
|
10
|
+
kind: string;
|
|
11
|
+
}
|
|
12
|
+
export interface BenchmarkCaseResult {
|
|
13
|
+
caseId: string;
|
|
14
|
+
split: BenchmarkSplit;
|
|
15
|
+
stratum: string;
|
|
16
|
+
expected: boolean | null;
|
|
17
|
+
baseline: BenchmarkPrediction;
|
|
18
|
+
candidate: BenchmarkPrediction;
|
|
19
|
+
baselineEvidence: BenchmarkEvidenceRef[];
|
|
20
|
+
candidateEvidence: BenchmarkEvidenceRef[];
|
|
21
|
+
baselineError: BenchmarkErrorKind | null;
|
|
22
|
+
candidateError: BenchmarkErrorKind | null;
|
|
23
|
+
comparison: 'improved' | 'regressed' | 'unchanged' | 'unresolved';
|
|
24
|
+
}
|
|
25
|
+
export interface BinaryBenchmarkMetrics {
|
|
26
|
+
labeledCases: number;
|
|
27
|
+
unresolvedLabels: number;
|
|
28
|
+
abstainedPredictions: number;
|
|
29
|
+
truePositive: number;
|
|
30
|
+
falsePositive: number;
|
|
31
|
+
falseNegative: number;
|
|
32
|
+
trueNegative: number;
|
|
33
|
+
precision: number | null;
|
|
34
|
+
recall: number | null;
|
|
35
|
+
f1: number | null;
|
|
36
|
+
}
|
|
37
|
+
export interface EvaluationExperimentReport {
|
|
38
|
+
manifestVersion: typeof EVALUATION_MANIFEST_VERSION;
|
|
39
|
+
evaluationEngineVersion: typeof EVALUATION_ENGINE_VERSION;
|
|
40
|
+
experimentName: string;
|
|
41
|
+
dataset: {
|
|
42
|
+
id: string;
|
|
43
|
+
fingerprint: string;
|
|
44
|
+
caseCount: number;
|
|
45
|
+
holdoutCaseCount: number;
|
|
46
|
+
strata: Array<{
|
|
47
|
+
name: string;
|
|
48
|
+
count: number;
|
|
49
|
+
}>;
|
|
50
|
+
};
|
|
51
|
+
algorithm: {
|
|
52
|
+
ruleId: string;
|
|
53
|
+
baselineVersion: string;
|
|
54
|
+
candidateVersion: string;
|
|
55
|
+
parserVersion: string;
|
|
56
|
+
thresholds: Record<string, number | string | boolean | null>;
|
|
57
|
+
};
|
|
58
|
+
provenance: {
|
|
59
|
+
source: 'sanitized-fixture';
|
|
60
|
+
annotation: 'fixture-ground-truth';
|
|
61
|
+
runtimeMs: number;
|
|
62
|
+
startedAt: string;
|
|
63
|
+
resourceUsage: {
|
|
64
|
+
heapUsedBytesBefore: number;
|
|
65
|
+
heapUsedBytesAfter: number;
|
|
66
|
+
heapUsedDeltaBytes: number;
|
|
67
|
+
rssBytesAfter: number;
|
|
68
|
+
};
|
|
69
|
+
};
|
|
70
|
+
baseline: {
|
|
71
|
+
all: BinaryBenchmarkMetrics;
|
|
72
|
+
development: BinaryBenchmarkMetrics;
|
|
73
|
+
holdout: BinaryBenchmarkMetrics;
|
|
74
|
+
};
|
|
75
|
+
candidate: {
|
|
76
|
+
all: BinaryBenchmarkMetrics;
|
|
77
|
+
development: BinaryBenchmarkMetrics;
|
|
78
|
+
holdout: BinaryBenchmarkMetrics;
|
|
79
|
+
};
|
|
80
|
+
templateEvaluation: {
|
|
81
|
+
templateCount: number;
|
|
82
|
+
numericFactParity: boolean;
|
|
83
|
+
uniqueWordingCount: number;
|
|
84
|
+
humanPreferenceLabels: number;
|
|
85
|
+
humanPreferenceStatus: 'not_collected';
|
|
86
|
+
};
|
|
87
|
+
comparison: {
|
|
88
|
+
holdoutFalsePositiveDelta: number;
|
|
89
|
+
holdoutFalseNegativeDelta: number;
|
|
90
|
+
baselineNotRegressed: boolean;
|
|
91
|
+
fixtureLabeledHoldoutCaseCount: number;
|
|
92
|
+
};
|
|
93
|
+
promotionGate: {
|
|
94
|
+
baselineNotRegressed: boolean;
|
|
95
|
+
fixtureLabeledHoldoutAvailable: boolean;
|
|
96
|
+
humanReviewedHoldoutAvailable: false;
|
|
97
|
+
historicalHoldoutAvailable: false;
|
|
98
|
+
ready: boolean;
|
|
99
|
+
reasons: string[];
|
|
100
|
+
};
|
|
101
|
+
caseOutcomes: BenchmarkCaseResult[];
|
|
102
|
+
counterexamples: BenchmarkCaseResult[];
|
|
103
|
+
}
|
|
104
|
+
export interface BenchmarkPredictionInput {
|
|
105
|
+
caseId: string;
|
|
106
|
+
split: BenchmarkSplit;
|
|
107
|
+
stratum: string;
|
|
108
|
+
expected: boolean | null;
|
|
109
|
+
baseline: BenchmarkPrediction;
|
|
110
|
+
candidate: BenchmarkPrediction;
|
|
111
|
+
baselineEvidence?: readonly BenchmarkEvidenceRef[];
|
|
112
|
+
candidateEvidence?: readonly BenchmarkEvidenceRef[];
|
|
113
|
+
}
|
|
114
|
+
export interface EvaluationExperimentInput {
|
|
115
|
+
experimentName: string;
|
|
116
|
+
datasetId: string;
|
|
117
|
+
datasetFingerprint: string;
|
|
118
|
+
ruleId: string;
|
|
119
|
+
baselineVersion: string;
|
|
120
|
+
candidateVersion: string;
|
|
121
|
+
parserVersion: string;
|
|
122
|
+
thresholds: Record<string, number | string | boolean | null>;
|
|
123
|
+
startedAt: string;
|
|
124
|
+
runtimeMs: number;
|
|
125
|
+
resourceUsage: {
|
|
126
|
+
heapUsedBytesBefore: number;
|
|
127
|
+
heapUsedBytesAfter: number;
|
|
128
|
+
heapUsedDeltaBytes: number;
|
|
129
|
+
rssBytesAfter: number;
|
|
130
|
+
};
|
|
131
|
+
predictions: readonly BenchmarkPredictionInput[];
|
|
132
|
+
templateEvaluation: {
|
|
133
|
+
templateCount: number;
|
|
134
|
+
numericFactParity: boolean;
|
|
135
|
+
uniqueWordingCount: number;
|
|
136
|
+
humanPreferenceLabels: number;
|
|
137
|
+
};
|
|
138
|
+
}
|
|
139
|
+
export declare function fingerprintBenchmarkDataset(dataset: unknown): string;
|
|
140
|
+
export declare function buildEvaluationExperiment(input: EvaluationExperimentInput): EvaluationExperimentReport;
|
|
@@ -0,0 +1,180 @@
|
|
|
1
|
+
import { createHash } from 'node:crypto';
|
|
2
|
+
export const EVALUATION_MANIFEST_VERSION = 1;
|
|
3
|
+
export const EVALUATION_ENGINE_VERSION = 'pubg-intelligence-evaluation-v1';
|
|
4
|
+
function errorFor(expected, prediction) {
|
|
5
|
+
if (expected === null)
|
|
6
|
+
return null;
|
|
7
|
+
if (expected && prediction !== true)
|
|
8
|
+
return 'false_negative';
|
|
9
|
+
if (!expected && prediction === true)
|
|
10
|
+
return 'false_positive';
|
|
11
|
+
return null;
|
|
12
|
+
}
|
|
13
|
+
function metricsFor(rows, which) {
|
|
14
|
+
let labeledCases = 0;
|
|
15
|
+
let unresolvedLabels = 0;
|
|
16
|
+
let abstainedPredictions = 0;
|
|
17
|
+
let truePositive = 0;
|
|
18
|
+
let falsePositive = 0;
|
|
19
|
+
let falseNegative = 0;
|
|
20
|
+
let trueNegative = 0;
|
|
21
|
+
for (const row of rows) {
|
|
22
|
+
if (row.expected === null) {
|
|
23
|
+
unresolvedLabels += 1;
|
|
24
|
+
continue;
|
|
25
|
+
}
|
|
26
|
+
labeledCases += 1;
|
|
27
|
+
const prediction = row[which];
|
|
28
|
+
if (prediction === null)
|
|
29
|
+
abstainedPredictions += 1;
|
|
30
|
+
if (row.expected && prediction === true)
|
|
31
|
+
truePositive += 1;
|
|
32
|
+
else if (row.expected)
|
|
33
|
+
falseNegative += 1;
|
|
34
|
+
else if (prediction === true)
|
|
35
|
+
falsePositive += 1;
|
|
36
|
+
else if (prediction === false)
|
|
37
|
+
trueNegative += 1;
|
|
38
|
+
}
|
|
39
|
+
const precision = truePositive + falsePositive ? truePositive / (truePositive + falsePositive) : null;
|
|
40
|
+
const recall = truePositive + falseNegative ? truePositive / (truePositive + falseNegative) : null;
|
|
41
|
+
const f1 = precision !== null && recall !== null && precision + recall > 0
|
|
42
|
+
? (2 * precision * recall) / (precision + recall)
|
|
43
|
+
: null;
|
|
44
|
+
return {
|
|
45
|
+
labeledCases,
|
|
46
|
+
unresolvedLabels,
|
|
47
|
+
abstainedPredictions,
|
|
48
|
+
truePositive,
|
|
49
|
+
falsePositive,
|
|
50
|
+
falseNegative,
|
|
51
|
+
trueNegative,
|
|
52
|
+
precision,
|
|
53
|
+
recall,
|
|
54
|
+
f1,
|
|
55
|
+
};
|
|
56
|
+
}
|
|
57
|
+
export function fingerprintBenchmarkDataset(dataset) {
|
|
58
|
+
return createHash('sha256').update(JSON.stringify(dataset)).digest('hex');
|
|
59
|
+
}
|
|
60
|
+
export function buildEvaluationExperiment(input) {
|
|
61
|
+
if (!input.predictions.length)
|
|
62
|
+
throw new Error('benchmark_corpus_empty');
|
|
63
|
+
const caseIds = new Set();
|
|
64
|
+
for (const row of input.predictions) {
|
|
65
|
+
if (caseIds.has(row.caseId))
|
|
66
|
+
throw new Error('benchmark_case_id_duplicate');
|
|
67
|
+
caseIds.add(row.caseId);
|
|
68
|
+
}
|
|
69
|
+
const caseOutcomes = input.predictions
|
|
70
|
+
.map((row) => {
|
|
71
|
+
const baselineError = errorFor(row.expected, row.baseline);
|
|
72
|
+
const candidateError = errorFor(row.expected, row.candidate);
|
|
73
|
+
const comparison = row.expected === null
|
|
74
|
+
? 'unresolved'
|
|
75
|
+
: candidateError === null && baselineError !== null
|
|
76
|
+
? 'improved'
|
|
77
|
+
: candidateError !== null && baselineError === null
|
|
78
|
+
? 'regressed'
|
|
79
|
+
: 'unchanged';
|
|
80
|
+
return {
|
|
81
|
+
caseId: row.caseId,
|
|
82
|
+
split: row.split,
|
|
83
|
+
stratum: row.stratum,
|
|
84
|
+
expected: row.expected,
|
|
85
|
+
baseline: row.baseline,
|
|
86
|
+
candidate: row.candidate,
|
|
87
|
+
baselineEvidence: [...(row.baselineEvidence ?? [])],
|
|
88
|
+
candidateEvidence: [...(row.candidateEvidence ?? [])],
|
|
89
|
+
baselineError,
|
|
90
|
+
candidateError,
|
|
91
|
+
comparison,
|
|
92
|
+
};
|
|
93
|
+
})
|
|
94
|
+
.sort((left, right) => left.split.localeCompare(right.split) || left.caseId.localeCompare(right.caseId));
|
|
95
|
+
const all = caseOutcomes;
|
|
96
|
+
const development = all.filter((row) => row.split === 'development');
|
|
97
|
+
const holdout = all.filter((row) => row.split === 'holdout');
|
|
98
|
+
if (!development.length || !holdout.length)
|
|
99
|
+
throw new Error('benchmark_requires_development_and_holdout');
|
|
100
|
+
const baselineHoldout = metricsFor(holdout, 'baseline');
|
|
101
|
+
const candidateHoldout = metricsFor(holdout, 'candidate');
|
|
102
|
+
const baselineNotRegressed = candidateHoldout.falsePositive <= baselineHoldout.falsePositive &&
|
|
103
|
+
candidateHoldout.falseNegative <= baselineHoldout.falseNegative;
|
|
104
|
+
const fixtureLabeledHoldoutCaseCount = holdout.filter((row) => row.expected !== null).length;
|
|
105
|
+
const reasons = [
|
|
106
|
+
...(!baselineNotRegressed
|
|
107
|
+
? ['Candidate increases a labeled holdout false positive or false negative.']
|
|
108
|
+
: []),
|
|
109
|
+
...(fixtureLabeledHoldoutCaseCount === 0
|
|
110
|
+
? ['Holdout has no fixture ground-truth labels.']
|
|
111
|
+
: []),
|
|
112
|
+
'No analyst review labels have been collected for this benchmark run.',
|
|
113
|
+
'This replay uses sanitized fixtures only; a historical holdout has not been independently labeled.',
|
|
114
|
+
];
|
|
115
|
+
const strata = new Map();
|
|
116
|
+
for (const row of caseOutcomes)
|
|
117
|
+
strata.set(row.stratum, (strata.get(row.stratum) ?? 0) + 1);
|
|
118
|
+
return {
|
|
119
|
+
manifestVersion: EVALUATION_MANIFEST_VERSION,
|
|
120
|
+
evaluationEngineVersion: EVALUATION_ENGINE_VERSION,
|
|
121
|
+
experimentName: input.experimentName,
|
|
122
|
+
dataset: {
|
|
123
|
+
id: input.datasetId,
|
|
124
|
+
fingerprint: input.datasetFingerprint ||
|
|
125
|
+
fingerprintBenchmarkDataset({
|
|
126
|
+
datasetId: input.datasetId,
|
|
127
|
+
cases: input.predictions
|
|
128
|
+
.map(({ caseId, split, stratum, expected }) => ({ caseId, split, stratum, expected }))
|
|
129
|
+
.sort((left, right) => left.caseId.localeCompare(right.caseId)),
|
|
130
|
+
}),
|
|
131
|
+
caseCount: caseOutcomes.length,
|
|
132
|
+
holdoutCaseCount: holdout.length,
|
|
133
|
+
strata: [...strata.entries()]
|
|
134
|
+
.sort(([left], [right]) => left.localeCompare(right))
|
|
135
|
+
.map(([name, count]) => ({ name, count })),
|
|
136
|
+
},
|
|
137
|
+
algorithm: {
|
|
138
|
+
ruleId: input.ruleId,
|
|
139
|
+
baselineVersion: input.baselineVersion,
|
|
140
|
+
candidateVersion: input.candidateVersion,
|
|
141
|
+
parserVersion: input.parserVersion,
|
|
142
|
+
thresholds: { ...input.thresholds },
|
|
143
|
+
},
|
|
144
|
+
provenance: {
|
|
145
|
+
source: 'sanitized-fixture',
|
|
146
|
+
annotation: 'fixture-ground-truth',
|
|
147
|
+
runtimeMs: Math.max(0, input.runtimeMs),
|
|
148
|
+
startedAt: input.startedAt,
|
|
149
|
+
resourceUsage: { ...input.resourceUsage },
|
|
150
|
+
},
|
|
151
|
+
baseline: {
|
|
152
|
+
all: metricsFor(all, 'baseline'),
|
|
153
|
+
development: metricsFor(development, 'baseline'),
|
|
154
|
+
holdout: baselineHoldout,
|
|
155
|
+
},
|
|
156
|
+
candidate: {
|
|
157
|
+
all: metricsFor(all, 'candidate'),
|
|
158
|
+
development: metricsFor(development, 'candidate'),
|
|
159
|
+
holdout: candidateHoldout,
|
|
160
|
+
},
|
|
161
|
+
templateEvaluation: { ...input.templateEvaluation, humanPreferenceStatus: 'not_collected' },
|
|
162
|
+
comparison: {
|
|
163
|
+
holdoutFalsePositiveDelta: candidateHoldout.falsePositive - baselineHoldout.falsePositive,
|
|
164
|
+
holdoutFalseNegativeDelta: candidateHoldout.falseNegative - baselineHoldout.falseNegative,
|
|
165
|
+
baselineNotRegressed,
|
|
166
|
+
fixtureLabeledHoldoutCaseCount,
|
|
167
|
+
},
|
|
168
|
+
promotionGate: {
|
|
169
|
+
baselineNotRegressed,
|
|
170
|
+
fixtureLabeledHoldoutAvailable: fixtureLabeledHoldoutCaseCount > 0,
|
|
171
|
+
humanReviewedHoldoutAvailable: false,
|
|
172
|
+
historicalHoldoutAvailable: false,
|
|
173
|
+
ready: false,
|
|
174
|
+
reasons,
|
|
175
|
+
},
|
|
176
|
+
caseOutcomes,
|
|
177
|
+
counterexamples: caseOutcomes.filter((row) => row.candidateError !== null || row.baselineError !== null || row.expected === null),
|
|
178
|
+
};
|
|
179
|
+
}
|
|
180
|
+
//# sourceMappingURL=evaluation.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"evaluation.js","sourceRoot":"","sources":["../../src/evaluation.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAEzC,MAAM,CAAC,MAAM,2BAA2B,GAAG,CAAU,CAAC;AACtD,MAAM,CAAC,MAAM,yBAAyB,GAAG,iCAA0C,CAAC;AA8IpF,SAAS,QAAQ,CACf,QAAwB,EACxB,UAA+B;IAE/B,IAAI,QAAQ,KAAK,IAAI;QAAE,OAAO,IAAI,CAAC;IACnC,IAAI,QAAQ,IAAI,UAAU,KAAK,IAAI;QAAE,OAAO,gBAAgB,CAAC;IAC7D,IAAI,CAAC,QAAQ,IAAI,UAAU,KAAK,IAAI;QAAE,OAAO,gBAAgB,CAAC;IAC9D,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,UAAU,CACjB,IAAoC,EACpC,KAA+B;IAE/B,IAAI,YAAY,GAAG,CAAC,CAAC;IACrB,IAAI,gBAAgB,GAAG,CAAC,CAAC;IACzB,IAAI,oBAAoB,GAAG,CAAC,CAAC;IAC7B,IAAI,YAAY,GAAG,CAAC,CAAC;IACrB,IAAI,aAAa,GAAG,CAAC,CAAC;IACtB,IAAI,aAAa,GAAG,CAAC,CAAC;IACtB,IAAI,YAAY,GAAG,CAAC,CAAC;IACrB,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,IAAI,GAAG,CAAC,QAAQ,KAAK,IAAI,EAAE,CAAC;YAC1B,gBAAgB,IAAI,CAAC,CAAC;YACtB,SAAS;QACX,CAAC;QACD,YAAY,IAAI,CAAC,CAAC;QAClB,MAAM,UAAU,GAAG,GAAG,CAAC,KAAK,CAAC,CAAC;QAC9B,IAAI,UAAU,KAAK,IAAI;YAAE,oBAAoB,IAAI,CAAC,CAAC;QACnD,IAAI,GAAG,CAAC,QAAQ,IAAI,UAAU,KAAK,IAAI;YAAE,YAAY,IAAI,CAAC,CAAC;aACtD,IAAI,GAAG,CAAC,QAAQ;YAAE,aAAa,IAAI,CAAC,CAAC;aACrC,IAAI,UAAU,KAAK,IAAI;YAAE,aAAa,IAAI,CAAC,CAAC;aAC5C,IAAI,UAAU,KAAK,KAAK;YAAE,YAAY,IAAI,CAAC,CAAC;IACnD,CAAC;IACD,MAAM,SAAS,GACb,YAAY,GAAG,aAAa,CAAC,CAAC,CAAC,YAAY,GAAG,CAAC,YAAY,GAAG,aAAa,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IACtF,MAAM,MAAM,GACV,YAAY,GAAG,aAAa,CAAC,CAAC,CAAC,YAAY,GAAG,CAAC,YAAY,GAAG,aAAa,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IACtF,MAAM,EAAE,GACN,SAAS,KAAK,IAAI,IAAI,MAAM,KAAK,IAAI,IAAI,SAAS,GAAG,MAAM,GAAG,CAAC;QAC7D,CAAC,CAAC,CAAC,CAAC,GAAG,SAAS,GAAG,MAAM,CAAC,GAAG,CAAC,SAAS,GAAG,MAAM,CAAC;QACjD,CAAC,CAAC,IAAI,CAAC;IACX,OAAO;QACL,YAAY;QACZ,gBAAgB;QAChB,oBAAoB;QACpB,YAAY;QACZ,aAAa;QACb,aAAa;QACb,YAAY;QACZ,SAAS;QACT,MAAM;QACN,EAAE;KACH,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,2BAA2B,CAAC,OAAgB;IAC1D,OAAO,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;AAC5E,CAAC;AAED,MAAM,UAAU,yBAAyB,CACvC,KAAgC;IAEhC,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,MAAM;QAAE,MAAM,IAAI,KAAK,CAAC,wBAAwB,CAAC,CAAC;IACzE,MAAM,OAAO,GAAG,IAAI,GAAG,EAAU,CAAC;IAClC,KAAK,MAAM,GAAG,IAAI,KAAK,CAAC,WAAW,EAAE,CAAC;QACpC,IAAI,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,6BAA6B,CAAC,CAAC;QAC5E,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;IAC1B,CAAC;IACD,MAAM,YAAY,GAA0B,KAAK,CAAC,WAAW;SAC1D,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE;QACX,MAAM,aAAa,GAAG,QAAQ,CAAC,GAAG,CAAC,QAAQ,EAAE,GAAG,CAAC,QAAQ,CAAC,CAAC;QAC3D,MAAM,cAAc,GAAG,QAAQ,CAAC,GAAG,CAAC,QAAQ,EAAE,GAAG,CAAC,SAAS,CAAC,CAAC;QAC7D,MAAM,UAAU,GACd,GAAG,CAAC,QAAQ,KAAK,IAAI;YACnB,CAAC,CAAC,YAAY;YACd,CAAC,CAAC,cAAc,KAAK,IAAI,IAAI,aAAa,KAAK,IAAI;gBACjD,CAAC,CAAC,UAAU;gBACZ,CAAC,CAAC,cAAc,KAAK,IAAI,IAAI,aAAa,KAAK,IAAI;oBACjD,CAAC,CAAC,WAAW;oBACb,CAAC,CAAC,WAAW,CAAC;QACtB,OAAO;YACL,MAAM,EAAE,GAAG,CAAC,MAAM;YAClB,KAAK,EAAE,GAAG,CAAC,KAAK;YAChB,OAAO,EAAE,GAAG,CAAC,OAAO;YACpB,QAAQ,EAAE,GAAG,CAAC,QAAQ;YACtB,QAAQ,EAAE,GAAG,CAAC,QAAQ;YACtB,SAAS,EAAE,GAAG,CAAC,SAAS;YACxB,gBAAgB,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,gBAAgB,IAAI,EAAE,CAAC,CAAC;YACnD,iBAAiB,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,iBAAiB,IAAI,EAAE,CAAC,CAAC;YACrD,aAAa;YACb,cAAc;YACd,UAAU;SACX,CAAC;IACJ,CAAC,CAAC;SACD,IAAI,CACH,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CACd,IAAI,CAAC,KAAK,CAAC,aAAa,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,KAAK,CAAC,MAAM,CAAC,CACnF,CAAC;IACJ,MAAM,GAAG,GAAG,YAAY,CAAC;IACzB,MAAM,WAAW,GAAG,GAAG,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,KAAK,KAAK,aAAa,CAAC,CAAC;IACrE,MAAM,OAAO,GAAG,GAAG,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC;IAC7D,IAAI,CAAC,WAAW,CAAC,MAAM,IAAI,CAAC,OAAO,CAAC,MAAM;QACxC,MAAM,IAAI,KAAK,CAAC,4CAA4C,CAAC,CAAC;IAChE,MAAM,eAAe,GAAG,UAAU,CAAC,OAAO,EAAE,UAAU,CAAC,CAAC;IACxD,MAAM,gBAAgB,GAAG,UAAU,CAAC,OAAO,EAAE,WAAW,CAAC,CAAC;IAC1D,MAAM,oBAAoB,GACxB,gBAAgB,CAAC,aAAa,IAAI,eAAe,CAAC,aAAa;QAC/D,gBAAgB,CAAC,aAAa,IAAI,eAAe,CAAC,aAAa,CAAC;IAClE,MAAM,8BAA8B,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,QAAQ,KAAK,IAAI,CAAC,CAAC,MAAM,CAAC;IAC7F,MAAM,OAAO,GAAG;QACd,GAAG,CAAC,CAAC,oBAAoB;YACvB,CAAC,CAAC,CAAC,yEAAyE,CAAC;YAC7E,CAAC,CAAC,EAAE,CAAC;QACP,GAAG,CAAC,8BAA8B,KAAK,CAAC;YACtC,CAAC,CAAC,CAAC,6CAA6C,CAAC;YACjD,CAAC,CAAC,EAAE,CAAC;QACP,sEAAsE;QACtE,oGAAoG;KACrG,CAAC;IACF,MAAM,MAAM,GAAG,IAAI,GAAG,EAAkB,CAAC;IACzC,KAAK,MAAM,GAAG,IAAI,YAAY;QAAE,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAC5F,OAAO;QACL,eAAe,EAAE,2BAA2B;QAC5C,uBAAuB,EAAE,yBAAyB;QAClD,cAAc,EAAE,KAAK,CAAC,cAAc;QACpC,OAAO,EAAE;YACP,EAAE,EAAE,KAAK,CAAC,SAAS;YACnB,WAAW,EACT,KAAK,CAAC,kBAAkB;gBACxB,2BAA2B,CAAC;oBAC1B,SAAS,EAAE,KAAK,CAAC,SAAS;oBAC1B,KAAK,EAAE,KAAK,CAAC,WAAW;yBACrB,GAAG,CAAC,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAC,CAAC;yBACrF,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;iBAClE,CAAC;YACJ,SAAS,EAAE,YAAY,CAAC,MAAM;YAC9B,gBAAgB,EAAE,OAAO,CAAC,MAAM;YAChC,MAAM,EAAE,CAAC,GAAG,MAAM,CAAC,OAAO,EAAE,CAAC;iBAC1B,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,KAAK,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC;iBACpD,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;SAC7C;QACD,SAAS,EAAE;YACT,MAAM,EAAE,KAAK,CAAC,MAAM;YACpB,eAAe,EAAE,KAAK,CAAC,eAAe;YACtC,gBAAgB,EAAE,KAAK,CAAC,gBAAgB;YACxC,aAAa,EAAE,KAAK,CAAC,aAAa;YAClC,UAAU,EAAE,EAAE,GAAG,KAAK,CAAC,UAAU,EAAE;SACpC;QACD,UAAU,EAAE;YACV,MAAM,EAAE,mBAAmB;YAC3B,UAAU,EAAE,sBAAsB;YAClC,SAAS,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,SAAS,CAAC;YACvC,SAAS,EAAE,KAAK,CAAC,SAAS;YAC1B,aAAa,EAAE,EAAE,GAAG,KAAK,CAAC,aAAa,EAAE;SAC1C;QACD,QAAQ,EAAE;YACR,GAAG,EAAE,UAAU,CAAC,GAAG,EAAE,UAAU,CAAC;YAChC,WAAW,EAAE,UAAU,CAAC,WAAW,EAAE,UAAU,CAAC;YAChD,OAAO,EAAE,eAAe;SACzB;QACD,SAAS,EAAE;YACT,GAAG,EAAE,UAAU,CAAC,GAAG,EAAE,WAAW,CAAC;YACjC,WAAW,EAAE,UAAU,CAAC,WAAW,EAAE,WAAW,CAAC;YACjD,OAAO,EAAE,gBAAgB;SAC1B;QACD,kBAAkB,EAAE,EAAE,GAAG,KAAK,CAAC,kBAAkB,EAAE,qBAAqB,EAAE,eAAe,EAAE;QAC3F,UAAU,EAAE;YACV,yBAAyB,EAAE,gBAAgB,CAAC,aAAa,GAAG,eAAe,CAAC,aAAa;YACzF,yBAAyB,EAAE,gBAAgB,CAAC,aAAa,GAAG,eAAe,CAAC,aAAa;YACzF,oBAAoB;YACpB,8BAA8B;SAC/B;QACD,aAAa,EAAE;YACb,oBAAoB;YACpB,8BAA8B,EAAE,8BAA8B,GAAG,CAAC;YAClE,6BAA6B,EAAE,KAAK;YACpC,0BAA0B,EAAE,KAAK;YACjC,KAAK,EAAE,KAAK;YACZ,OAAO;SACR;QACD,YAAY;QACZ,eAAe,EAAE,YAAY,CAAC,MAAM,CAClC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,cAAc,KAAK,IAAI,IAAI,GAAG,CAAC,aAAa,KAAK,IAAI,IAAI,GAAG,CAAC,QAAQ,KAAK,IAAI,CAC5F;KACF,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
import type { EvaluationExperimentReport } from './evaluation.js';
|
|
2
|
+
export interface StoredResearchRun {
|
|
3
|
+
question: string;
|
|
4
|
+
intent: unknown;
|
|
5
|
+
source: string;
|
|
6
|
+
report: unknown;
|
|
7
|
+
assistant?: unknown;
|
|
8
|
+
}
|
|
9
|
+
export type EvaluationReviewLabel = 'confirm' | 'reject' | 'unresolved' | 'wrong_evidence';
|
|
10
|
+
export type BattleReviewLabel = 'correct' | 'insufficient_inference' | 'wrong' | 'unknown';
|
|
11
|
+
export interface EvaluationRunSummary {
|
|
12
|
+
runId: string;
|
|
13
|
+
createdAt: string;
|
|
14
|
+
experimentName: string;
|
|
15
|
+
datasetFingerprint: string;
|
|
16
|
+
candidateVersion: string;
|
|
17
|
+
caseCount: number;
|
|
18
|
+
runtimeMs: number;
|
|
19
|
+
reviewedLabels: number;
|
|
20
|
+
}
|
|
21
|
+
export interface EvaluationReview {
|
|
22
|
+
caseId: string;
|
|
23
|
+
ruleId: string;
|
|
24
|
+
label: EvaluationReviewLabel;
|
|
25
|
+
createdAt: string;
|
|
26
|
+
}
|
|
27
|
+
export interface BattleReview {
|
|
28
|
+
matchId: string;
|
|
29
|
+
battleId: string;
|
|
30
|
+
label: BattleReviewLabel;
|
|
31
|
+
createdAt: string;
|
|
32
|
+
}
|
|
33
|
+
/** Persistent experiment output lives outside the production PUBG database. */
|
|
34
|
+
export declare class ExperimentRunStore {
|
|
35
|
+
readonly filename: string;
|
|
36
|
+
private readonly db;
|
|
37
|
+
constructor(filename: string);
|
|
38
|
+
save(run: StoredResearchRun): string;
|
|
39
|
+
saveEvaluation(report: EvaluationExperimentReport): string;
|
|
40
|
+
listEvaluations(limit?: number): EvaluationRunSummary[];
|
|
41
|
+
getEvaluation(runId: string): EvaluationExperimentReport | null;
|
|
42
|
+
saveEvaluationReview(input: {
|
|
43
|
+
runId: string;
|
|
44
|
+
caseId: string;
|
|
45
|
+
ruleId: string;
|
|
46
|
+
label: EvaluationReviewLabel;
|
|
47
|
+
}): EvaluationReview;
|
|
48
|
+
listEvaluationReviews(runId: string): EvaluationReview[];
|
|
49
|
+
saveBattleReview(input: {
|
|
50
|
+
matchId: string;
|
|
51
|
+
battleId: string;
|
|
52
|
+
label: BattleReviewLabel;
|
|
53
|
+
}): BattleReview;
|
|
54
|
+
listBattleReviews(matchId: string): BattleReview[];
|
|
55
|
+
close(): void;
|
|
56
|
+
}
|
|
@@ -0,0 +1,159 @@
|
|
|
1
|
+
import { chmodSync, mkdirSync } from 'node:fs';
|
|
2
|
+
import { createRequire } from 'node:module';
|
|
3
|
+
import { dirname } from 'node:path';
|
|
4
|
+
import { randomUUID } from 'node:crypto';
|
|
5
|
+
import { redactSensitiveText } from './intent.js';
|
|
6
|
+
const require = createRequire(import.meta.url);
|
|
7
|
+
const { DatabaseSync } = require('node:sqlite');
|
|
8
|
+
/** Persistent experiment output lives outside the production PUBG database. */
|
|
9
|
+
export class ExperimentRunStore {
|
|
10
|
+
filename;
|
|
11
|
+
db;
|
|
12
|
+
constructor(filename) {
|
|
13
|
+
this.filename = filename;
|
|
14
|
+
mkdirSync(dirname(filename), { recursive: true, mode: 0o700 });
|
|
15
|
+
this.db = new DatabaseSync(filename);
|
|
16
|
+
chmodSync(filename, 0o600);
|
|
17
|
+
this.db.exec(`
|
|
18
|
+
CREATE TABLE IF NOT EXISTS research_runs (
|
|
19
|
+
run_id TEXT PRIMARY KEY,
|
|
20
|
+
created_at TEXT NOT NULL,
|
|
21
|
+
question TEXT NOT NULL,
|
|
22
|
+
intent_json TEXT NOT NULL,
|
|
23
|
+
planner_source TEXT NOT NULL,
|
|
24
|
+
report_json TEXT NOT NULL
|
|
25
|
+
);
|
|
26
|
+
CREATE INDEX IF NOT EXISTS idx_research_runs_created ON research_runs(created_at DESC);
|
|
27
|
+
CREATE TABLE IF NOT EXISTS battle_reviews (
|
|
28
|
+
match_id TEXT NOT NULL,
|
|
29
|
+
battle_id TEXT NOT NULL,
|
|
30
|
+
label TEXT NOT NULL CHECK (label IN ('correct', 'insufficient_inference', 'wrong', 'unknown')),
|
|
31
|
+
created_at TEXT NOT NULL,
|
|
32
|
+
PRIMARY KEY (match_id, battle_id)
|
|
33
|
+
);
|
|
34
|
+
CREATE TABLE IF NOT EXISTS evaluation_experiments (
|
|
35
|
+
run_id TEXT PRIMARY KEY,
|
|
36
|
+
created_at TEXT NOT NULL,
|
|
37
|
+
experiment_name TEXT NOT NULL,
|
|
38
|
+
dataset_fingerprint TEXT NOT NULL,
|
|
39
|
+
candidate_version TEXT NOT NULL,
|
|
40
|
+
case_count INTEGER NOT NULL,
|
|
41
|
+
runtime_ms REAL NOT NULL,
|
|
42
|
+
report_json TEXT NOT NULL
|
|
43
|
+
);
|
|
44
|
+
CREATE INDEX IF NOT EXISTS idx_evaluation_experiments_created ON evaluation_experiments(created_at DESC);
|
|
45
|
+
CREATE TABLE IF NOT EXISTS evaluation_reviews (
|
|
46
|
+
run_id TEXT NOT NULL,
|
|
47
|
+
case_id TEXT NOT NULL,
|
|
48
|
+
rule_id TEXT NOT NULL,
|
|
49
|
+
label TEXT NOT NULL CHECK (label IN ('confirm', 'reject', 'unresolved', 'wrong_evidence')),
|
|
50
|
+
created_at TEXT NOT NULL,
|
|
51
|
+
PRIMARY KEY (run_id, case_id, rule_id),
|
|
52
|
+
FOREIGN KEY (run_id) REFERENCES evaluation_experiments(run_id) ON DELETE CASCADE
|
|
53
|
+
);
|
|
54
|
+
`);
|
|
55
|
+
}
|
|
56
|
+
save(run) {
|
|
57
|
+
const id = randomUUID();
|
|
58
|
+
this.db
|
|
59
|
+
.prepare(`
|
|
60
|
+
INSERT INTO research_runs(run_id, created_at, question, intent_json, planner_source, report_json)
|
|
61
|
+
VALUES (?, ?, ?, ?, ?, ?)
|
|
62
|
+
`)
|
|
63
|
+
.run(id, new Date().toISOString(), redactSensitiveText(run.question), JSON.stringify(run.intent), run.source, JSON.stringify({ report: run.report, assistant: run.assistant ?? null }));
|
|
64
|
+
return id;
|
|
65
|
+
}
|
|
66
|
+
saveEvaluation(report) {
|
|
67
|
+
const runId = randomUUID();
|
|
68
|
+
this.db
|
|
69
|
+
.prepare(`
|
|
70
|
+
INSERT INTO evaluation_experiments(run_id, created_at, experiment_name, dataset_fingerprint, candidate_version, case_count, runtime_ms, report_json)
|
|
71
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
|
72
|
+
`)
|
|
73
|
+
.run(runId, report.provenance.startedAt, report.experimentName, report.dataset.fingerprint, report.algorithm.candidateVersion, report.dataset.caseCount, report.provenance.runtimeMs, JSON.stringify(report));
|
|
74
|
+
return runId;
|
|
75
|
+
}
|
|
76
|
+
listEvaluations(limit = 20) {
|
|
77
|
+
const bounded = Math.max(1, Math.min(100, Math.floor(limit)));
|
|
78
|
+
const rows = this.db
|
|
79
|
+
.prepare(`
|
|
80
|
+
SELECT e.run_id, e.created_at, e.experiment_name, e.dataset_fingerprint, e.candidate_version, e.case_count, e.runtime_ms,
|
|
81
|
+
(SELECT COUNT(*) FROM evaluation_reviews r WHERE r.run_id = e.run_id) AS reviewed_labels
|
|
82
|
+
FROM evaluation_experiments e ORDER BY e.created_at DESC, e.run_id DESC LIMIT ?
|
|
83
|
+
`)
|
|
84
|
+
.all(bounded);
|
|
85
|
+
return rows.map((row) => ({
|
|
86
|
+
runId: row.run_id,
|
|
87
|
+
createdAt: row.created_at,
|
|
88
|
+
experimentName: row.experiment_name,
|
|
89
|
+
datasetFingerprint: row.dataset_fingerprint,
|
|
90
|
+
candidateVersion: row.candidate_version,
|
|
91
|
+
caseCount: row.case_count,
|
|
92
|
+
runtimeMs: row.runtime_ms,
|
|
93
|
+
reviewedLabels: row.reviewed_labels,
|
|
94
|
+
}));
|
|
95
|
+
}
|
|
96
|
+
getEvaluation(runId) {
|
|
97
|
+
const row = this.db
|
|
98
|
+
.prepare('SELECT report_json FROM evaluation_experiments WHERE run_id = ?')
|
|
99
|
+
.get(runId);
|
|
100
|
+
if (!row?.report_json)
|
|
101
|
+
return null;
|
|
102
|
+
return JSON.parse(row.report_json);
|
|
103
|
+
}
|
|
104
|
+
saveEvaluationReview(input) {
|
|
105
|
+
const report = this.getEvaluation(input.runId);
|
|
106
|
+
if (!report)
|
|
107
|
+
throw new Error('evaluation_run_not_found');
|
|
108
|
+
if (input.ruleId !== report.algorithm.ruleId ||
|
|
109
|
+
!report.caseOutcomes.some((row) => row.caseId === input.caseId)) {
|
|
110
|
+
throw new Error('evaluation_review_outside_run');
|
|
111
|
+
}
|
|
112
|
+
const createdAt = new Date().toISOString();
|
|
113
|
+
this.db
|
|
114
|
+
.prepare(`
|
|
115
|
+
INSERT INTO evaluation_reviews(run_id, case_id, rule_id, label, created_at) VALUES (?, ?, ?, ?, ?)
|
|
116
|
+
ON CONFLICT(run_id, case_id, rule_id) DO UPDATE SET label = excluded.label, created_at = excluded.created_at
|
|
117
|
+
`)
|
|
118
|
+
.run(input.runId, input.caseId, input.ruleId, input.label, createdAt);
|
|
119
|
+
return { caseId: input.caseId, ruleId: input.ruleId, label: input.label, createdAt };
|
|
120
|
+
}
|
|
121
|
+
listEvaluationReviews(runId) {
|
|
122
|
+
return this.db
|
|
123
|
+
.prepare(`
|
|
124
|
+
SELECT case_id, rule_id, label, created_at FROM evaluation_reviews WHERE run_id = ? ORDER BY case_id, rule_id
|
|
125
|
+
`)
|
|
126
|
+
.all(runId).map((row) => ({
|
|
127
|
+
caseId: row.case_id,
|
|
128
|
+
ruleId: row.rule_id,
|
|
129
|
+
label: row.label,
|
|
130
|
+
createdAt: row.created_at,
|
|
131
|
+
}));
|
|
132
|
+
}
|
|
133
|
+
saveBattleReview(input) {
|
|
134
|
+
const createdAt = new Date().toISOString();
|
|
135
|
+
this.db
|
|
136
|
+
.prepare(`
|
|
137
|
+
INSERT INTO battle_reviews(match_id, battle_id, label, created_at) VALUES (?, ?, ?, ?)
|
|
138
|
+
ON CONFLICT(match_id, battle_id) DO UPDATE SET label = excluded.label, created_at = excluded.created_at
|
|
139
|
+
`)
|
|
140
|
+
.run(input.matchId, input.battleId, input.label, createdAt);
|
|
141
|
+
return { ...input, createdAt };
|
|
142
|
+
}
|
|
143
|
+
listBattleReviews(matchId) {
|
|
144
|
+
return this.db
|
|
145
|
+
.prepare(`
|
|
146
|
+
SELECT match_id, battle_id, label, created_at FROM battle_reviews WHERE match_id = ? ORDER BY battle_id
|
|
147
|
+
`)
|
|
148
|
+
.all(matchId).map((row) => ({
|
|
149
|
+
matchId: row.match_id,
|
|
150
|
+
battleId: row.battle_id,
|
|
151
|
+
label: row.label,
|
|
152
|
+
createdAt: row.created_at,
|
|
153
|
+
}));
|
|
154
|
+
}
|
|
155
|
+
close() {
|
|
156
|
+
this.db.close();
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
//# sourceMappingURL=experiment-store.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"experiment-store.js","sourceRoot":"","sources":["../../src/experiment-store.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AAC/C,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAC5C,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,mBAAmB,EAAE,MAAM,aAAa,CAAC;AAelD,MAAM,OAAO,GAAG,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AAC/C,MAAM,EAAE,YAAY,EAAE,GAAG,OAAO,CAAC,aAAa,CAAqD,CAAC;AA6CpG,+EAA+E;AAC/E,MAAM,OAAO,kBAAkB;IAGR;IAFJ,EAAE,CAAW;IAE9B,YAAqB,QAAgB;QAAhB,aAAQ,GAAR,QAAQ,CAAQ;QACnC,SAAS,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;QAC/D,IAAI,CAAC,EAAE,GAAG,IAAI,YAAY,CAAC,QAAQ,CAAC,CAAC;QACrC,SAAS,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;QAC3B,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;KAqCZ,CAAC,CAAC;IACL,CAAC;IAED,IAAI,CAAC,GAAsB;QACzB,MAAM,EAAE,GAAG,UAAU,EAAE,CAAC;QACxB,IAAI,CAAC,EAAE;aACJ,OAAO,CACN;;;KAGH,CACE;aACA,GAAG,CACF,EAAE,EACF,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,EACxB,mBAAmB,CAAC,GAAG,CAAC,QAAQ,CAAC,EACjC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,EAC1B,GAAG,CAAC,MAAM,EACV,IAAI,CAAC,SAAS,CAAC,EAAE,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,SAAS,EAAE,GAAG,CAAC,SAAS,IAAI,IAAI,EAAE,CAAC,CACzE,CAAC;QACJ,OAAO,EAAE,CAAC;IACZ,CAAC;IAED,cAAc,CAAC,MAAkC;QAC/C,MAAM,KAAK,GAAG,UAAU,EAAE,CAAC;QAC3B,IAAI,CAAC,EAAE;aACJ,OAAO,CACN;;;KAGH,CACE;aACA,GAAG,CACF,KAAK,EACL,MAAM,CAAC,UAAU,CAAC,SAAS,EAC3B,MAAM,CAAC,cAAc,EACrB,MAAM,CAAC,OAAO,CAAC,WAAW,EAC1B,MAAM,CAAC,SAAS,CAAC,gBAAgB,EACjC,MAAM,CAAC,OAAO,CAAC,SAAS,EACxB,MAAM,CAAC,UAAU,CAAC,SAAS,EAC3B,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CACvB,CAAC;QACJ,OAAO,KAAK,CAAC;IACf,CAAC;IAED,eAAe,CAAC,KAAK,GAAG,EAAE;QACxB,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QAC9D,MAAM,IAAI,GAAG,IAAI,CAAC,EAAE;aACjB,OAAO,CACN;;;;KAIH,CACE;aACA,GAAG,CAAC,OAAO,CASZ,CAAC;QACH,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;YACxB,KAAK,EAAE,GAAG,CAAC,MAAM;YACjB,SAAS,EAAE,GAAG,CAAC,UAAU;YACzB,cAAc,EAAE,GAAG,CAAC,eAAe;YACnC,kBAAkB,EAAE,GAAG,CAAC,mBAAmB;YAC3C,gBAAgB,EAAE,GAAG,CAAC,iBAAiB;YACvC,SAAS,EAAE,GAAG,CAAC,UAAU;YACzB,SAAS,EAAE,GAAG,CAAC,UAAU;YACzB,cAAc,EAAE,GAAG,CAAC,eAAe;SACpC,CAAC,CAAC,CAAC;IACN,CAAC;IAED,aAAa,CAAC,KAAa;QACzB,MAAM,GAAG,GAAG,IAAI,CAAC,EAAE;aAChB,OAAO,CAAC,iEAAiE,CAAC;aAC1E,GAAG,CAAC,KAAK,CAAyC,CAAC;QACtD,IAAI,CAAC,GAAG,EAAE,WAAW;YAAE,OAAO,IAAI,CAAC;QACnC,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,WAAW,CAA+B,CAAC;IACnE,CAAC;IAED,oBAAoB,CAAC,KAKpB;QACC,MAAM,MAAM,GAAG,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAC/C,IAAI,CAAC,MAAM;YAAE,MAAM,IAAI,KAAK,CAAC,0BAA0B,CAAC,CAAC;QACzD,IACE,KAAK,CAAC,MAAM,KAAK,MAAM,CAAC,SAAS,CAAC,MAAM;YACxC,CAAC,MAAM,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,MAAM,KAAK,KAAK,CAAC,MAAM,CAAC,EAC/D,CAAC;YACD,MAAM,IAAI,KAAK,CAAC,+BAA+B,CAAC,CAAC;QACnD,CAAC;QACD,MAAM,SAAS,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;QAC3C,IAAI,CAAC,EAAE;aACJ,OAAO,CACN;;;KAGH,CACE;aACA,GAAG,CAAC,KAAK,CAAC,KAAK,EAAE,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC;QACxE,OAAO,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,SAAS,EAAE,CAAC;IACvF,CAAC;IAED,qBAAqB,CAAC,KAAa;QACjC,OACE,IAAI,CAAC,EAAE;aACJ,OAAO,CACN;;KAEL,CACI;aACA,GAAG,CAAC,KAAK,CAMb,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;YACd,MAAM,EAAE,GAAG,CAAC,OAAO;YACnB,MAAM,EAAE,GAAG,CAAC,OAAO;YACnB,KAAK,EAAE,GAAG,CAAC,KAAK;YAChB,SAAS,EAAE,GAAG,CAAC,UAAU;SAC1B,CAAC,CAAC,CAAC;IACN,CAAC;IAED,gBAAgB,CAAC,KAIhB;QACC,MAAM,SAAS,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;QAC3C,IAAI,CAAC,EAAE;aACJ,OAAO,CACN;;;KAGH,CACE;aACA,GAAG,CAAC,KAAK,CAAC,OAAO,EAAE,KAAK,CAAC,QAAQ,EAAE,KAAK,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC;QAC9D,OAAO,EAAE,GAAG,KAAK,EAAE,SAAS,EAAE,CAAC;IACjC,CAAC;IAED,iBAAiB,CAAC,OAAe;QAC/B,OACE,IAAI,CAAC,EAAE;aACJ,OAAO,CACN;;KAEL,CACI;aACA,GAAG,CAAC,OAAO,CAMf,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;YACd,OAAO,EAAE,GAAG,CAAC,QAAQ;YACrB,QAAQ,EAAE,GAAG,CAAC,SAAS;YACvB,KAAK,EAAE,GAAG,CAAC,KAAK;YAChB,SAAS,EAAE,GAAG,CAAC,UAAU;SAC1B,CAAC,CAAC,CAAC;IACN,CAAC;IAED,KAAK;QACH,IAAI,CAAC,EAAE,CAAC,KAAK,EAAE,CAAC;IAClB,CAAC;CACF"}
|
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
import { type RateEstimate } from './analytics.js';
|
|
2
|
+
export declare const FIELD_SCHEMA_VERSION: 1;
|
|
3
|
+
/** Engagement distance bands in metres; the last band is open ended. */
|
|
4
|
+
export declare const DISTANCE_BANDS: readonly [{
|
|
5
|
+
readonly id: "close";
|
|
6
|
+
readonly label: "近距 <25m";
|
|
7
|
+
readonly maxMeters: 25;
|
|
8
|
+
}, {
|
|
9
|
+
readonly id: "mid";
|
|
10
|
+
readonly label: "中近 25–75m";
|
|
11
|
+
readonly maxMeters: 75;
|
|
12
|
+
}, {
|
|
13
|
+
readonly id: "far";
|
|
14
|
+
readonly label: "中远 75–150m";
|
|
15
|
+
readonly maxMeters: 150;
|
|
16
|
+
}, {
|
|
17
|
+
readonly id: "long";
|
|
18
|
+
readonly label: "远距 ≥150m";
|
|
19
|
+
readonly maxMeters: number;
|
|
20
|
+
}];
|
|
21
|
+
export declare const DISTANCE_MIN_DAMAGE = 600;
|
|
22
|
+
export declare const DISTANCE_GAP_POINTS = 0.1;
|
|
23
|
+
export declare const ZONE_MAX_PHASE = 9;
|
|
24
|
+
export declare const ZONE_MIN_SAMPLES = 150;
|
|
25
|
+
export declare const ZONE_BIN_EDGES: readonly [0.5, 0.85, 1];
|
|
26
|
+
export declare const CONTEST_RADIUS_METERS = 500;
|
|
27
|
+
export declare const CONTEST_MIN_MATCHES = 8;
|
|
28
|
+
export declare function distanceBandOf(meters: number): number;
|
|
29
|
+
export interface ZonePhaseStats {
|
|
30
|
+
samples: number;
|
|
31
|
+
blueSamples: number;
|
|
32
|
+
/** Position samples by distance from the safe-zone centre divided by its radius: <0.5, 0.5-0.85, 0.85-1, outside. */
|
|
33
|
+
bins: [number, number, number, number];
|
|
34
|
+
blueDamage: number;
|
|
35
|
+
}
|
|
36
|
+
export interface FieldMatchStats {
|
|
37
|
+
distance: {
|
|
38
|
+
dealt: number[];
|
|
39
|
+
dealtHead: number[];
|
|
40
|
+
taken: number[];
|
|
41
|
+
takenHead: number[];
|
|
42
|
+
weapons: Record<string, number[]>;
|
|
43
|
+
};
|
|
44
|
+
zone: {
|
|
45
|
+
phases: Record<number, ZonePhaseStats>;
|
|
46
|
+
blueDamage: number;
|
|
47
|
+
damageTaken: number;
|
|
48
|
+
};
|
|
49
|
+
landing: {
|
|
50
|
+
/** Other teams whose nearest member landed within the contest radius of the squad centre; null without landings. */
|
|
51
|
+
contestedTeams: number | null;
|
|
52
|
+
nearestEnemyMeters: number | null;
|
|
53
|
+
/** Most common named map area reported after landing. */
|
|
54
|
+
region: string | null;
|
|
55
|
+
};
|
|
56
|
+
}
|
|
57
|
+
/**
|
|
58
|
+
* One pass over a raw telemetry payload for the tracked squad: damage by engagement distance, time spent
|
|
59
|
+
* outside the safe zone, blue-zone damage and how crowded the landing spot was.
|
|
60
|
+
*/
|
|
61
|
+
export declare function analyzeFieldTelemetry(payload: unknown, trackedIds: ReadonlySet<string>): FieldMatchStats;
|
|
62
|
+
export interface DistanceBandSummary {
|
|
63
|
+
id: string;
|
|
64
|
+
label: string;
|
|
65
|
+
dealt: number;
|
|
66
|
+
taken: number;
|
|
67
|
+
dealtShare: number | null;
|
|
68
|
+
takenShare: number | null;
|
|
69
|
+
/** Share of dealt damage that was headshot damage. */
|
|
70
|
+
headshotShare: number | null;
|
|
71
|
+
}
|
|
72
|
+
export interface WeaponDistanceSummary {
|
|
73
|
+
weapon: string;
|
|
74
|
+
damage: number;
|
|
75
|
+
shares: number[];
|
|
76
|
+
mainBand: string;
|
|
77
|
+
}
|
|
78
|
+
export interface DistanceSummary {
|
|
79
|
+
matches: number;
|
|
80
|
+
totalDealt: number;
|
|
81
|
+
totalTaken: number;
|
|
82
|
+
bands: DistanceBandSummary[];
|
|
83
|
+
weapons: WeaponDistanceSummary[];
|
|
84
|
+
/** Taken minus dealt share in the two long bands; positive means the squad absorbs more long-range damage than it deals. */
|
|
85
|
+
longRangeGap: number | null;
|
|
86
|
+
}
|
|
87
|
+
export declare function summarizeDistance(items: readonly FieldMatchStats[]): DistanceSummary;
|
|
88
|
+
export interface ZonePhaseSummary {
|
|
89
|
+
phase: number;
|
|
90
|
+
samples: number;
|
|
91
|
+
/** Share of position samples flagged as inside the blue zone. */
|
|
92
|
+
outsideRate: RateEstimate;
|
|
93
|
+
/** Share of position samples by distance from the safe-zone centre: inner, middle, edge, outside. */
|
|
94
|
+
binShares: [number, number, number, number] | null;
|
|
95
|
+
blueDamage: number;
|
|
96
|
+
}
|
|
97
|
+
export interface ZoneSummary {
|
|
98
|
+
matches: number;
|
|
99
|
+
blueDamagePerMatch: number | null;
|
|
100
|
+
blueDamageShare: number | null;
|
|
101
|
+
phases: ZonePhaseSummary[];
|
|
102
|
+
}
|
|
103
|
+
export declare function summarizeZone(items: readonly FieldMatchStats[]): ZoneSummary;
|