@mrsf/cli 0.6.0 → 0.7.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +12 -9
- package/dist/browser.cjs +1101 -24
- package/dist/browser.cjs.map +4 -4
- package/dist/browser.d.ts +11 -0
- package/dist/browser.js +8 -0
- package/dist/commands/add.js +12 -4
- package/dist/evaluation/generate-reanchor-cases.d.ts +10 -0
- package/dist/evaluation/generate-reanchor-cases.js +301 -0
- package/dist/evaluation/reanchor-eval.d.ts +111 -0
- package/dist/evaluation/reanchor-eval.js +266 -0
- package/dist/evaluation/reanchor-gates.d.ts +29 -0
- package/dist/evaluation/reanchor-gates.js +47 -0
- package/dist/evaluation/run-reanchor-eval.d.ts +3 -0
- package/dist/evaluation/run-reanchor-eval.js +103 -0
- package/dist/evaluation/run-reanchor-generator.d.ts +3 -0
- package/dist/evaluation/run-reanchor-generator.js +63 -0
- package/dist/evaluation/run-reanchor-profile.d.ts +3 -0
- package/dist/evaluation/run-reanchor-profile.js +129 -0
- package/dist/index.cjs +1226 -34
- package/dist/index.cjs.map +4 -4
- package/dist/index.d.ts +13 -0
- package/dist/index.js +10 -0
- package/dist/lib/anchor-context.d.ts +46 -0
- package/dist/lib/anchor-context.js +474 -0
- package/dist/lib/confidence-calibration.d.ts +16 -0
- package/dist/lib/confidence-calibration.js +120 -0
- package/dist/lib/fuzzy.d.ts +14 -1
- package/dist/lib/fuzzy.js +140 -19
- package/dist/lib/git.d.ts +9 -0
- package/dist/lib/git.js +30 -0
- package/dist/lib/global-reconciliation.d.ts +11 -0
- package/dist/lib/global-reconciliation.js +145 -0
- package/dist/lib/reanchor-core.d.ts +7 -0
- package/dist/lib/reanchor-core.js +46 -4
- package/dist/lib/reanchor.js +82 -11
- package/dist/lib/revision-projection.d.ts +21 -0
- package/dist/lib/revision-projection.js +203 -0
- package/package.json +5 -1
|
@@ -0,0 +1,266 @@
|
|
|
1
|
+
import Ajv2020Module from "ajv/dist/2020.js";
|
|
2
|
+
import { readFile, readdir } from "node:fs/promises";
|
|
3
|
+
import path from "node:path";
|
|
4
|
+
import { performance } from "node:perf_hooks";
|
|
5
|
+
import { reanchorComment, toReanchorLines } from "../lib/reanchor-core.js";
|
|
6
|
+
import { createRevisionProjection } from "../lib/revision-projection.js";
|
|
7
|
+
import { createAnchorContextIndex } from "../lib/anchor-context.js";
|
|
8
|
+
import { createFuzzySearchIndex, } from "../lib/fuzzy.js";
|
|
9
|
+
import { reconcileCommentAnchors } from "../lib/global-reconciliation.js";
|
|
10
|
+
const Ajv2020 = Ajv2020Module;
|
|
11
|
+
export async function loadEvaluationCases(casesPath, schemaPath) {
|
|
12
|
+
const schema = JSON.parse(await readFile(schemaPath, "utf8"));
|
|
13
|
+
const validator = new Ajv2020({
|
|
14
|
+
allErrors: true,
|
|
15
|
+
strict: true,
|
|
16
|
+
strictRequired: false,
|
|
17
|
+
}).compile(schema);
|
|
18
|
+
const files = await findJsonFiles(casesPath);
|
|
19
|
+
const loaded = [];
|
|
20
|
+
const caseIds = new Set();
|
|
21
|
+
for (const casePath of files) {
|
|
22
|
+
const parsed = JSON.parse(await readFile(casePath, "utf8"));
|
|
23
|
+
const values = Array.isArray(parsed) ? parsed : [parsed];
|
|
24
|
+
for (const [index, value] of values.entries()) {
|
|
25
|
+
const location = values.length === 1 ? casePath : `${casePath}[${index}]`;
|
|
26
|
+
assertValidCase(validator, value, location);
|
|
27
|
+
if (caseIds.has(value.id)) {
|
|
28
|
+
throw new Error(`Duplicate evaluation case id "${value.id}" in ${location}.`);
|
|
29
|
+
}
|
|
30
|
+
caseIds.add(value.id);
|
|
31
|
+
loaded.push({ casePath, value });
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
if (loaded.length === 0) {
|
|
35
|
+
throw new Error(`No evaluation cases found in ${casesPath}.`);
|
|
36
|
+
}
|
|
37
|
+
return loaded;
|
|
38
|
+
}
|
|
39
|
+
export async function evaluateCases(cases) {
|
|
40
|
+
const startedAt = performance.now();
|
|
41
|
+
const results = [];
|
|
42
|
+
const reconciliationTimings = [];
|
|
43
|
+
for (const loadedCase of cases) {
|
|
44
|
+
const sourceText = await readEvaluationDocument(loadedCase.value.source, loadedCase.casePath);
|
|
45
|
+
validateSourceSelections(loadedCase.value, sourceText);
|
|
46
|
+
const targetText = await readEvaluationDocument(loadedCase.value.target, loadedCase.casePath);
|
|
47
|
+
const sourceLines = toReanchorLines(sourceText);
|
|
48
|
+
const documentLines = toReanchorLines(targetText);
|
|
49
|
+
const revisionProjection = createRevisionProjection(sourceLines, documentLines);
|
|
50
|
+
const anchorContext = createAnchorContextIndex(sourceLines, documentLines);
|
|
51
|
+
let fuzzySearchIndex;
|
|
52
|
+
const getFuzzySearchIndex = () => {
|
|
53
|
+
fuzzySearchIndex ??= createFuzzySearchIndex(documentLines);
|
|
54
|
+
return fuzzySearchIndex;
|
|
55
|
+
};
|
|
56
|
+
const caseComments = [];
|
|
57
|
+
const caseResults = [];
|
|
58
|
+
for (const evaluationComment of loadedCase.value.comments) {
|
|
59
|
+
const comment = toComment(evaluationComment);
|
|
60
|
+
const commentStartedAt = performance.now();
|
|
61
|
+
const actual = reanchorComment(comment, documentLines, {
|
|
62
|
+
revisionProjection,
|
|
63
|
+
anchorContext,
|
|
64
|
+
getFuzzySearchIndex,
|
|
65
|
+
});
|
|
66
|
+
const durationMs = performance.now() - commentStartedAt;
|
|
67
|
+
caseComments.push(comment);
|
|
68
|
+
caseResults.push({
|
|
69
|
+
evaluationComment,
|
|
70
|
+
comment,
|
|
71
|
+
actual,
|
|
72
|
+
durationMs,
|
|
73
|
+
});
|
|
74
|
+
}
|
|
75
|
+
const reconciliationStartedAt = performance.now();
|
|
76
|
+
const reconciled = reconcileCommentAnchors(caseComments, caseResults.map((result) => result.actual), anchorContext);
|
|
77
|
+
reconciliationTimings.push(performance.now() - reconciliationStartedAt);
|
|
78
|
+
for (const [index, caseResult] of caseResults.entries()) {
|
|
79
|
+
const { evaluationComment, comment, durationMs, } = caseResult;
|
|
80
|
+
const actual = reconciled[index];
|
|
81
|
+
const statusCorrect = actual.status === evaluationComment.expected.status;
|
|
82
|
+
const rangeCorrect = matchesExpectedRange(actual, comment, evaluationComment.expected.ranges);
|
|
83
|
+
results.push({
|
|
84
|
+
caseId: loadedCase.value.id,
|
|
85
|
+
commentId: evaluationComment.id,
|
|
86
|
+
expected: evaluationComment.expected,
|
|
87
|
+
actual,
|
|
88
|
+
statusCorrect,
|
|
89
|
+
rangeCorrect,
|
|
90
|
+
passed: statusCorrect && rangeCorrect,
|
|
91
|
+
durationMs,
|
|
92
|
+
});
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
const durationMs = performance.now() - startedAt;
|
|
96
|
+
const timings = results
|
|
97
|
+
.map((result) => result.durationMs)
|
|
98
|
+
.sort((left, right) => left - right);
|
|
99
|
+
return {
|
|
100
|
+
algorithm: "baseline",
|
|
101
|
+
cases: cases.length,
|
|
102
|
+
comments: results.length,
|
|
103
|
+
passed: results.filter((result) => result.passed).length,
|
|
104
|
+
failed: results.filter((result) => !result.passed).length,
|
|
105
|
+
exactRangeMatches: results.filter((result) => result.rangeCorrect).length,
|
|
106
|
+
statusMatches: results.filter((result) => result.statusCorrect).length,
|
|
107
|
+
incorrectConfidentRelocations: results.filter((result) => (result.actual.status === "anchored" || result.actual.status === "shifted")
|
|
108
|
+
&& (result.expected.ranges?.length
|
|
109
|
+
? !result.rangeCorrect
|
|
110
|
+
: result.expected.status === "ambiguous"
|
|
111
|
+
|| result.expected.status === "orphaned")).length,
|
|
112
|
+
durationMs,
|
|
113
|
+
timingMs: {
|
|
114
|
+
median: percentile(timings, 0.5),
|
|
115
|
+
p95: percentile(timings, 0.95),
|
|
116
|
+
max: timings.at(-1) ?? 0,
|
|
117
|
+
perComment: results.length > 0 ? durationMs / results.length : 0,
|
|
118
|
+
reconciliationMax: Math.max(0, ...reconciliationTimings),
|
|
119
|
+
},
|
|
120
|
+
results,
|
|
121
|
+
};
|
|
122
|
+
}
|
|
123
|
+
function percentile(sortedValues, quantile) {
|
|
124
|
+
if (sortedValues.length === 0)
|
|
125
|
+
return 0;
|
|
126
|
+
const index = Math.max(0, Math.ceil(sortedValues.length * quantile) - 1);
|
|
127
|
+
return sortedValues[index];
|
|
128
|
+
}
|
|
129
|
+
export function createEvaluationBaseline(summary) {
|
|
130
|
+
return {
|
|
131
|
+
version: 1,
|
|
132
|
+
algorithm: summary.algorithm,
|
|
133
|
+
cases: summary.cases,
|
|
134
|
+
comments: summary.comments,
|
|
135
|
+
metrics: {
|
|
136
|
+
passed: summary.passed,
|
|
137
|
+
failed: summary.failed,
|
|
138
|
+
exactRangeMatches: summary.exactRangeMatches,
|
|
139
|
+
statusMatches: summary.statusMatches,
|
|
140
|
+
incorrectConfidentRelocations: summary.incorrectConfidentRelocations,
|
|
141
|
+
},
|
|
142
|
+
results: summary.results.map((result) => ({
|
|
143
|
+
caseId: result.caseId,
|
|
144
|
+
commentId: result.commentId,
|
|
145
|
+
expectedStatus: result.expected.status,
|
|
146
|
+
actual: result.actual,
|
|
147
|
+
statusCorrect: result.statusCorrect,
|
|
148
|
+
rangeCorrect: result.rangeCorrect,
|
|
149
|
+
passed: result.passed,
|
|
150
|
+
})),
|
|
151
|
+
};
|
|
152
|
+
}
|
|
153
|
+
export function findBaselineDifferences(expected, actual) {
|
|
154
|
+
const differences = [];
|
|
155
|
+
if (expected.version !== actual.version) {
|
|
156
|
+
differences.push(`Baseline version changed from ${expected.version} to ${actual.version}.`);
|
|
157
|
+
}
|
|
158
|
+
if (expected.algorithm !== actual.algorithm) {
|
|
159
|
+
differences.push(`Algorithm changed from ${expected.algorithm} to ${actual.algorithm}.`);
|
|
160
|
+
}
|
|
161
|
+
const expectedHeader = {
|
|
162
|
+
cases: expected.cases,
|
|
163
|
+
comments: expected.comments,
|
|
164
|
+
metrics: expected.metrics,
|
|
165
|
+
};
|
|
166
|
+
const actualHeader = {
|
|
167
|
+
cases: actual.cases,
|
|
168
|
+
comments: actual.comments,
|
|
169
|
+
metrics: actual.metrics,
|
|
170
|
+
};
|
|
171
|
+
if (JSON.stringify(expectedHeader) !== JSON.stringify(actualHeader)) {
|
|
172
|
+
differences.push(`Summary changed: expected ${JSON.stringify(expectedHeader)}, `
|
|
173
|
+
+ `received ${JSON.stringify(actualHeader)}.`);
|
|
174
|
+
}
|
|
175
|
+
const expectedResults = new Map(expected.results.map((result) => [baselineResultKey(result), result]));
|
|
176
|
+
const actualResults = new Map(actual.results.map((result) => [baselineResultKey(result), result]));
|
|
177
|
+
for (const [key, expectedResult] of expectedResults) {
|
|
178
|
+
const actualResult = actualResults.get(key);
|
|
179
|
+
if (!actualResult) {
|
|
180
|
+
differences.push(`Result ${key} is missing.`);
|
|
181
|
+
}
|
|
182
|
+
else if (JSON.stringify(expectedResult) !== JSON.stringify(actualResult)) {
|
|
183
|
+
differences.push(`Result ${key} changed.`);
|
|
184
|
+
}
|
|
185
|
+
}
|
|
186
|
+
for (const key of actualResults.keys()) {
|
|
187
|
+
if (!expectedResults.has(key)) {
|
|
188
|
+
differences.push(`Result ${key} is new.`);
|
|
189
|
+
}
|
|
190
|
+
}
|
|
191
|
+
return differences;
|
|
192
|
+
}
|
|
193
|
+
function baselineResultKey(result) {
|
|
194
|
+
return `${result.caseId}/${result.commentId}`;
|
|
195
|
+
}
|
|
196
|
+
async function findJsonFiles(rootPath) {
|
|
197
|
+
const entries = await readdir(rootPath, { withFileTypes: true });
|
|
198
|
+
const files = [];
|
|
199
|
+
for (const entry of entries.sort((left, right) => left.name.localeCompare(right.name))) {
|
|
200
|
+
const entryPath = path.join(rootPath, entry.name);
|
|
201
|
+
if (entry.isDirectory()) {
|
|
202
|
+
files.push(...await findJsonFiles(entryPath));
|
|
203
|
+
}
|
|
204
|
+
else if (entry.isFile() && entry.name.endsWith(".json")) {
|
|
205
|
+
files.push(entryPath);
|
|
206
|
+
}
|
|
207
|
+
}
|
|
208
|
+
return files;
|
|
209
|
+
}
|
|
210
|
+
function assertValidCase(validator, value, casePath) {
|
|
211
|
+
if (validator(value))
|
|
212
|
+
return;
|
|
213
|
+
const details = (validator.errors ?? [])
|
|
214
|
+
.map(formatValidationError)
|
|
215
|
+
.join("\n");
|
|
216
|
+
throw new Error(`Invalid evaluation case ${casePath}:\n${details}`);
|
|
217
|
+
}
|
|
218
|
+
function formatValidationError(error) {
|
|
219
|
+
return ` ${error.instancePath || "/"} ${error.message ?? "is invalid"}`;
|
|
220
|
+
}
|
|
221
|
+
async function readEvaluationDocument(document, casePath) {
|
|
222
|
+
if (document.text != null)
|
|
223
|
+
return document.text;
|
|
224
|
+
if (document.path != null) {
|
|
225
|
+
return readFile(path.resolve(path.dirname(casePath), document.path), "utf8");
|
|
226
|
+
}
|
|
227
|
+
throw new Error(`Evaluation document in ${casePath} has neither text nor path.`);
|
|
228
|
+
}
|
|
229
|
+
function validateSourceSelections(evaluationCase, sourceText) {
|
|
230
|
+
const normalizedSource = sourceText.replace(/\r\n/g, "\n");
|
|
231
|
+
for (const comment of evaluationCase.comments) {
|
|
232
|
+
const selectedText = comment.anchor.selected_text?.replace(/\r\n/g, "\n");
|
|
233
|
+
if (selectedText != null && !normalizedSource.includes(selectedText)) {
|
|
234
|
+
throw new Error(`Evaluation case ${evaluationCase.id}/${comment.id} has selected_text `
|
|
235
|
+
+ "that does not occur in its source document.");
|
|
236
|
+
}
|
|
237
|
+
}
|
|
238
|
+
}
|
|
239
|
+
function toComment(evaluationComment) {
|
|
240
|
+
return {
|
|
241
|
+
id: evaluationComment.id,
|
|
242
|
+
author: "MRSF evaluation",
|
|
243
|
+
timestamp: "1970-01-01T00:00:00Z",
|
|
244
|
+
text: "Reanchoring evaluation fixture",
|
|
245
|
+
resolved: false,
|
|
246
|
+
...evaluationComment.anchor,
|
|
247
|
+
};
|
|
248
|
+
}
|
|
249
|
+
function matchesExpectedRange(actual, original, expectedRanges) {
|
|
250
|
+
if (!expectedRanges?.length)
|
|
251
|
+
return true;
|
|
252
|
+
const actualRange = {
|
|
253
|
+
line: actual.newLine ?? original.line ?? 0,
|
|
254
|
+
end_line: actual.newEndLine ?? original.end_line ?? actual.newLine ?? original.line,
|
|
255
|
+
start_column: actual.newStartColumn ?? original.start_column,
|
|
256
|
+
end_column: actual.newEndColumn ?? original.end_column,
|
|
257
|
+
};
|
|
258
|
+
return expectedRanges.some((expected) => actualRange.line === expected.line
|
|
259
|
+
&& (expected.end_line == null
|
|
260
|
+
|| (actualRange.end_line ?? actualRange.line) === expected.end_line)
|
|
261
|
+
&& (expected.start_column == null
|
|
262
|
+
|| actualRange.start_column === expected.start_column)
|
|
263
|
+
&& (expected.end_column == null
|
|
264
|
+
|| actualRange.end_column === expected.end_column));
|
|
265
|
+
}
|
|
266
|
+
//# sourceMappingURL=reanchor-eval.js.map
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
export type ReanchorGateMode = "correctness" | "performance";
|
|
2
|
+
export interface ReanchorProfileReport {
|
|
3
|
+
profile: string;
|
|
4
|
+
metrics: {
|
|
5
|
+
passRate: number;
|
|
6
|
+
incorrectConfidentRelocations: number;
|
|
7
|
+
};
|
|
8
|
+
timingMs: {
|
|
9
|
+
perComment: number;
|
|
10
|
+
p95: number;
|
|
11
|
+
max: number;
|
|
12
|
+
reconciliationMax: number;
|
|
13
|
+
};
|
|
14
|
+
}
|
|
15
|
+
export interface ReanchorGatePolicy {
|
|
16
|
+
version: 1;
|
|
17
|
+
correctness: Record<string, {
|
|
18
|
+
minimum_pass_rate: number;
|
|
19
|
+
maximum_incorrect_confident_relocations: number;
|
|
20
|
+
}>;
|
|
21
|
+
performance: Record<string, {
|
|
22
|
+
maximum_ms_per_comment: number;
|
|
23
|
+
maximum_p95_comment_ms: number;
|
|
24
|
+
maximum_single_comment_ms: number;
|
|
25
|
+
maximum_reconciliation_ms: number;
|
|
26
|
+
}>;
|
|
27
|
+
}
|
|
28
|
+
export declare function evaluateReanchorGate(report: ReanchorProfileReport, policy: ReanchorGatePolicy, mode: ReanchorGateMode): string[];
|
|
29
|
+
//# sourceMappingURL=reanchor-gates.d.ts.map
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
export function evaluateReanchorGate(report, policy, mode) {
|
|
2
|
+
const failures = [];
|
|
3
|
+
const correctness = policy.correctness[report.profile];
|
|
4
|
+
if (!correctness) {
|
|
5
|
+
failures.push(`No correctness gate configured for profile "${report.profile}".`);
|
|
6
|
+
}
|
|
7
|
+
else {
|
|
8
|
+
if (report.metrics.passRate < correctness.minimum_pass_rate) {
|
|
9
|
+
failures.push(`Pass rate ${report.metrics.passRate.toFixed(4)} is below `
|
|
10
|
+
+ `${correctness.minimum_pass_rate.toFixed(4)}.`);
|
|
11
|
+
}
|
|
12
|
+
if (report.metrics.incorrectConfidentRelocations
|
|
13
|
+
> correctness.maximum_incorrect_confident_relocations) {
|
|
14
|
+
failures.push(`${report.metrics.incorrectConfidentRelocations} incorrect confident `
|
|
15
|
+
+ `relocations exceed the allowed `
|
|
16
|
+
+ `${correctness.maximum_incorrect_confident_relocations}.`);
|
|
17
|
+
}
|
|
18
|
+
}
|
|
19
|
+
if (mode === "performance") {
|
|
20
|
+
const performance = policy.performance[report.profile];
|
|
21
|
+
if (!performance) {
|
|
22
|
+
failures.push(`No performance gate configured for profile "${report.profile}".`);
|
|
23
|
+
}
|
|
24
|
+
else {
|
|
25
|
+
if (report.timingMs.perComment > performance.maximum_ms_per_comment) {
|
|
26
|
+
failures.push(`${report.timingMs.perComment.toFixed(3)} ms/comment exceeds `
|
|
27
|
+
+ `${performance.maximum_ms_per_comment.toFixed(3)} ms/comment.`);
|
|
28
|
+
}
|
|
29
|
+
if (report.timingMs.p95 > performance.maximum_p95_comment_ms) {
|
|
30
|
+
failures.push(`p95 ${report.timingMs.p95.toFixed(3)} ms exceeds `
|
|
31
|
+
+ `${performance.maximum_p95_comment_ms.toFixed(3)} ms.`);
|
|
32
|
+
}
|
|
33
|
+
if (report.timingMs.max > performance.maximum_single_comment_ms) {
|
|
34
|
+
failures.push(`maximum ${report.timingMs.max.toFixed(3)} ms exceeds `
|
|
35
|
+
+ `${performance.maximum_single_comment_ms.toFixed(3)} ms.`);
|
|
36
|
+
}
|
|
37
|
+
if (report.timingMs.reconciliationMax
|
|
38
|
+
> performance.maximum_reconciliation_ms) {
|
|
39
|
+
failures.push(`reconciliation maximum `
|
|
40
|
+
+ `${report.timingMs.reconciliationMax.toFixed(3)} ms exceeds `
|
|
41
|
+
+ `${performance.maximum_reconciliation_ms.toFixed(3)} ms.`);
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
return failures;
|
|
46
|
+
}
|
|
47
|
+
//# sourceMappingURL=reanchor-gates.js.map
|
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
import path from "node:path";
|
|
3
|
+
import process from "node:process";
|
|
4
|
+
import { readFile, writeFile } from "node:fs/promises";
|
|
5
|
+
import { createEvaluationBaseline, evaluateCases, findBaselineDifferences, loadEvaluationCases, } from "./reanchor-eval.js";
|
|
6
|
+
async function main() {
|
|
7
|
+
const options = parseArguments(process.argv.slice(2));
|
|
8
|
+
const casesPath = path.resolve(options.casesPath);
|
|
9
|
+
const schemaPath = path.resolve(options.schemaPath);
|
|
10
|
+
const cases = await loadEvaluationCases(casesPath, schemaPath);
|
|
11
|
+
const summary = await evaluateCases(cases);
|
|
12
|
+
const baseline = createEvaluationBaseline(summary);
|
|
13
|
+
if (options.json) {
|
|
14
|
+
process.stdout.write(`${JSON.stringify(summary, null, 2)}\n`);
|
|
15
|
+
}
|
|
16
|
+
else {
|
|
17
|
+
printSummary(summary);
|
|
18
|
+
}
|
|
19
|
+
if (options.writeBaselinePath) {
|
|
20
|
+
await writeFile(path.resolve(options.writeBaselinePath), `${JSON.stringify(baseline, null, 2)}\n`);
|
|
21
|
+
return;
|
|
22
|
+
}
|
|
23
|
+
if (options.baselinePath) {
|
|
24
|
+
const expected = JSON.parse(await readFile(path.resolve(options.baselinePath), "utf8"));
|
|
25
|
+
const differences = findBaselineDifferences(expected, baseline);
|
|
26
|
+
if (differences.length > 0) {
|
|
27
|
+
for (const difference of differences) {
|
|
28
|
+
console.error(`BASELINE ${difference}`);
|
|
29
|
+
}
|
|
30
|
+
process.exitCode = 1;
|
|
31
|
+
}
|
|
32
|
+
else if (!options.json) {
|
|
33
|
+
console.log("Baseline matches.");
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
else if (summary.failed > 0) {
|
|
37
|
+
process.exitCode = 1;
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
function parseArguments(args) {
|
|
41
|
+
let casesPath = "../evaluation/reanchor/cases";
|
|
42
|
+
let schemaPath = "../evaluation/reanchor/schema.json";
|
|
43
|
+
let json = false;
|
|
44
|
+
let baselinePath;
|
|
45
|
+
let writeBaselinePath;
|
|
46
|
+
for (let index = 0; index < args.length; index += 1) {
|
|
47
|
+
const argument = args[index];
|
|
48
|
+
if (argument === "--json") {
|
|
49
|
+
json = true;
|
|
50
|
+
}
|
|
51
|
+
else if (argument === "--cases" && args[index + 1]) {
|
|
52
|
+
casesPath = args[index += 1];
|
|
53
|
+
}
|
|
54
|
+
else if (argument === "--schema" && args[index + 1]) {
|
|
55
|
+
schemaPath = args[index += 1];
|
|
56
|
+
}
|
|
57
|
+
else if (argument === "--baseline" && args[index + 1]) {
|
|
58
|
+
baselinePath = args[index += 1];
|
|
59
|
+
}
|
|
60
|
+
else if (argument === "--write-baseline" && args[index + 1]) {
|
|
61
|
+
writeBaselinePath = args[index += 1];
|
|
62
|
+
}
|
|
63
|
+
else {
|
|
64
|
+
throw new Error(`Unknown or incomplete argument: ${argument}`);
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
if (baselinePath && writeBaselinePath) {
|
|
68
|
+
throw new Error("--baseline and --write-baseline cannot be used together.");
|
|
69
|
+
}
|
|
70
|
+
return {
|
|
71
|
+
casesPath,
|
|
72
|
+
schemaPath,
|
|
73
|
+
json,
|
|
74
|
+
baselinePath,
|
|
75
|
+
writeBaselinePath,
|
|
76
|
+
};
|
|
77
|
+
}
|
|
78
|
+
function printSummary(summary) {
|
|
79
|
+
const passRate = summary.comments === 0
|
|
80
|
+
? 0
|
|
81
|
+
: (summary.passed / summary.comments) * 100;
|
|
82
|
+
console.log(`Reanchoring evaluation (${summary.algorithm})`);
|
|
83
|
+
console.log(`Cases: ${summary.cases}`);
|
|
84
|
+
console.log(`Comments: ${summary.comments}`);
|
|
85
|
+
console.log(`Passed: ${summary.passed}/${summary.comments} (${passRate.toFixed(1)}%)`);
|
|
86
|
+
console.log(`Status matches: ${summary.statusMatches}/${summary.comments}`);
|
|
87
|
+
console.log(`Exact range matches: ${summary.exactRangeMatches}/${summary.comments}`);
|
|
88
|
+
console.log(`Incorrect confident relocations: ${summary.incorrectConfidentRelocations}`);
|
|
89
|
+
console.log(`Duration: ${summary.durationMs.toFixed(2)} ms`);
|
|
90
|
+
console.log(`Per-comment timing: median ${summary.timingMs.median.toFixed(3)} ms, `
|
|
91
|
+
+ `p95 ${summary.timingMs.p95.toFixed(3)} ms, `
|
|
92
|
+
+ `max ${summary.timingMs.max.toFixed(3)} ms`);
|
|
93
|
+
for (const result of summary.results.filter((item) => !item.passed)) {
|
|
94
|
+
console.log(`FAIL ${result.caseId}/${result.commentId}: expected ${result.expected.status}, `
|
|
95
|
+
+ `received ${result.actual.status} (${result.actual.reason})`);
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
main().catch((error) => {
|
|
99
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
100
|
+
console.error(message);
|
|
101
|
+
process.exitCode = 1;
|
|
102
|
+
});
|
|
103
|
+
//# sourceMappingURL=run-reanchor-eval.js.map
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
import { mkdir, writeFile } from "node:fs/promises";
|
|
3
|
+
import path from "node:path";
|
|
4
|
+
import process from "node:process";
|
|
5
|
+
import { generateEvaluationCases } from "./generate-reanchor-cases.js";
|
|
6
|
+
async function main() {
|
|
7
|
+
const options = parseArguments(process.argv.slice(2));
|
|
8
|
+
const outputPath = path.resolve(options.output);
|
|
9
|
+
const cases = generateEvaluationCases(options);
|
|
10
|
+
await mkdir(path.dirname(outputPath), { recursive: true });
|
|
11
|
+
await writeFile(outputPath, `${JSON.stringify(cases, null, 2)}\n`);
|
|
12
|
+
console.log(`Generated ${cases.length} reanchoring cases at ${outputPath} `
|
|
13
|
+
+ `(seeds ${options.seed}-${options.seed + cases.length - 1}).`);
|
|
14
|
+
}
|
|
15
|
+
function parseArguments(args) {
|
|
16
|
+
const values = new Map();
|
|
17
|
+
for (let index = 0; index < args.length; index += 2) {
|
|
18
|
+
const name = args[index];
|
|
19
|
+
const value = args[index + 1];
|
|
20
|
+
if (!name?.startsWith("--") || value == null) {
|
|
21
|
+
throw new Error(`Unknown or incomplete argument: ${name ?? ""}`);
|
|
22
|
+
}
|
|
23
|
+
values.set(name, value);
|
|
24
|
+
}
|
|
25
|
+
const supported = new Set([
|
|
26
|
+
"--seed",
|
|
27
|
+
"--cases",
|
|
28
|
+
"--blocks",
|
|
29
|
+
"--comments",
|
|
30
|
+
"--mutations",
|
|
31
|
+
"--output",
|
|
32
|
+
]);
|
|
33
|
+
for (const name of values.keys()) {
|
|
34
|
+
if (!supported.has(name))
|
|
35
|
+
throw new Error(`Unknown argument: ${name}`);
|
|
36
|
+
}
|
|
37
|
+
const seed = integerArgument(values, "--seed", 1);
|
|
38
|
+
return {
|
|
39
|
+
seed,
|
|
40
|
+
caseCount: integerArgument(values, "--cases", 10),
|
|
41
|
+
blocksPerCase: integerArgument(values, "--blocks", 12),
|
|
42
|
+
commentsPerCase: integerArgument(values, "--comments", 4),
|
|
43
|
+
mutationsPerCase: integerArgument(values, "--mutations", 5),
|
|
44
|
+
output: values.get("--output")
|
|
45
|
+
?? `../evaluation/reanchor/generated/seed-${seed}.json`,
|
|
46
|
+
};
|
|
47
|
+
}
|
|
48
|
+
function integerArgument(values, name, fallback) {
|
|
49
|
+
const raw = values.get(name);
|
|
50
|
+
if (raw == null)
|
|
51
|
+
return fallback;
|
|
52
|
+
const value = Number(raw);
|
|
53
|
+
if (!Number.isSafeInteger(value)) {
|
|
54
|
+
throw new Error(`${name} must be an integer.`);
|
|
55
|
+
}
|
|
56
|
+
return value;
|
|
57
|
+
}
|
|
58
|
+
main().catch((error) => {
|
|
59
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
60
|
+
console.error(message);
|
|
61
|
+
process.exitCode = 1;
|
|
62
|
+
});
|
|
63
|
+
//# sourceMappingURL=run-reanchor-generator.js.map
|
|
@@ -0,0 +1,129 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
import { readFile } from "node:fs/promises";
|
|
3
|
+
import path from "node:path";
|
|
4
|
+
import process from "node:process";
|
|
5
|
+
import { generateEvaluationCases } from "./generate-reanchor-cases.js";
|
|
6
|
+
import { evaluateCases } from "./reanchor-eval.js";
|
|
7
|
+
import { evaluateReanchorGate, } from "./reanchor-gates.js";
|
|
8
|
+
async function main() {
|
|
9
|
+
const options = parseArguments(process.argv.slice(2));
|
|
10
|
+
const workloads = JSON.parse(await readFile(path.resolve(options.workloadsPath), "utf8"));
|
|
11
|
+
assertWorkloadFile(workloads);
|
|
12
|
+
const config = workloads.profiles[options.profile];
|
|
13
|
+
if (!config) {
|
|
14
|
+
throw new Error(`Unknown workload profile "${options.profile}". Available profiles: `
|
|
15
|
+
+ `${Object.keys(workloads.profiles).sort().join(", ")}.`);
|
|
16
|
+
}
|
|
17
|
+
const generated = generateEvaluationCases({
|
|
18
|
+
seed: config.seed,
|
|
19
|
+
caseCount: config.case_count,
|
|
20
|
+
blocksPerCase: config.blocks_per_case,
|
|
21
|
+
commentsPerCase: config.comments_per_case,
|
|
22
|
+
mutationsPerCase: config.mutations_per_case,
|
|
23
|
+
});
|
|
24
|
+
const cases = generated.map((value) => ({
|
|
25
|
+
casePath: `<generated:${value.id}>`,
|
|
26
|
+
value,
|
|
27
|
+
}));
|
|
28
|
+
const summary = await evaluateCases(cases);
|
|
29
|
+
const report = {
|
|
30
|
+
profile: options.profile,
|
|
31
|
+
config,
|
|
32
|
+
metrics: {
|
|
33
|
+
cases: summary.cases,
|
|
34
|
+
comments: summary.comments,
|
|
35
|
+
passed: summary.passed,
|
|
36
|
+
failed: summary.failed,
|
|
37
|
+
passRate: summary.comments > 0
|
|
38
|
+
? summary.passed / summary.comments
|
|
39
|
+
: 0,
|
|
40
|
+
incorrectConfidentRelocations: summary.incorrectConfidentRelocations,
|
|
41
|
+
},
|
|
42
|
+
timingMs: {
|
|
43
|
+
total: summary.durationMs,
|
|
44
|
+
...summary.timingMs,
|
|
45
|
+
},
|
|
46
|
+
};
|
|
47
|
+
let gateFailures = [];
|
|
48
|
+
if (options.gatePath) {
|
|
49
|
+
const policy = JSON.parse(await readFile(path.resolve(options.gatePath), "utf8"));
|
|
50
|
+
gateFailures = evaluateReanchorGate(report, policy, options.gateMode);
|
|
51
|
+
}
|
|
52
|
+
if (options.json) {
|
|
53
|
+
console.log(JSON.stringify(report, null, 2));
|
|
54
|
+
}
|
|
55
|
+
else {
|
|
56
|
+
console.log(`Reanchoring workload: ${options.profile}`);
|
|
57
|
+
console.log(`${report.metrics.cases} cases, ${report.metrics.comments} comments, `
|
|
58
|
+
+ `${report.metrics.passed} passed, ${report.metrics.failed} failed`);
|
|
59
|
+
console.log(`${report.metrics.incorrectConfidentRelocations} incorrect confident relocations`);
|
|
60
|
+
console.log(`Timing: ${report.timingMs.total.toFixed(2)} ms total, `
|
|
61
|
+
+ `${report.timingMs.perComment.toFixed(3)} ms/comment, `
|
|
62
|
+
+ `${report.timingMs.p95.toFixed(3)} ms p95`);
|
|
63
|
+
}
|
|
64
|
+
if (gateFailures.length > 0) {
|
|
65
|
+
throw new Error(`Reanchoring ${options.gateMode} gate failed for ${options.profile}:\n`
|
|
66
|
+
+ gateFailures.map((failure) => `- ${failure}`).join("\n"));
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
function parseArguments(args) {
|
|
70
|
+
let profile = "small";
|
|
71
|
+
let workloadsPath = "../evaluation/reanchor/workloads.json";
|
|
72
|
+
let json = false;
|
|
73
|
+
let gatePath;
|
|
74
|
+
let gateMode = "correctness";
|
|
75
|
+
for (let index = 0; index < args.length; index += 1) {
|
|
76
|
+
const argument = args[index];
|
|
77
|
+
if (argument === "--json") {
|
|
78
|
+
json = true;
|
|
79
|
+
}
|
|
80
|
+
else if (argument === "--profile" && args[index + 1]) {
|
|
81
|
+
profile = args[index += 1];
|
|
82
|
+
}
|
|
83
|
+
else if (argument === "--workloads" && args[index + 1]) {
|
|
84
|
+
workloadsPath = args[index += 1];
|
|
85
|
+
}
|
|
86
|
+
else if (argument === "--gate" && args[index + 1]) {
|
|
87
|
+
gatePath = args[index += 1];
|
|
88
|
+
}
|
|
89
|
+
else if (argument === "--gate-mode"
|
|
90
|
+
&& (args[index + 1] === "correctness"
|
|
91
|
+
|| args[index + 1] === "performance")) {
|
|
92
|
+
gateMode = args[index += 1];
|
|
93
|
+
}
|
|
94
|
+
else {
|
|
95
|
+
throw new Error(`Unknown or incomplete argument: ${argument}`);
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
return { profile, workloadsPath, json, gatePath, gateMode };
|
|
99
|
+
}
|
|
100
|
+
function assertWorkloadFile(value) {
|
|
101
|
+
if (!isRecord(value) || value.version !== 1 || !isRecord(value.profiles)) {
|
|
102
|
+
throw new Error("Workload file must contain version 1 and a profiles object.");
|
|
103
|
+
}
|
|
104
|
+
for (const [name, profile] of Object.entries(value.profiles)) {
|
|
105
|
+
if (!isRecord(profile)
|
|
106
|
+
|| !isNonNegativeInteger(profile.seed)
|
|
107
|
+
|| !isPositiveInteger(profile.case_count)
|
|
108
|
+
|| !isPositiveInteger(profile.blocks_per_case)
|
|
109
|
+
|| !isPositiveInteger(profile.comments_per_case)
|
|
110
|
+
|| !isPositiveInteger(profile.mutations_per_case)) {
|
|
111
|
+
throw new Error(`Workload profile "${name}" is invalid.`);
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
function isRecord(value) {
|
|
116
|
+
return typeof value === "object" && value != null && !Array.isArray(value);
|
|
117
|
+
}
|
|
118
|
+
function isPositiveInteger(value) {
|
|
119
|
+
return Number.isSafeInteger(value) && Number(value) > 0;
|
|
120
|
+
}
|
|
121
|
+
function isNonNegativeInteger(value) {
|
|
122
|
+
return Number.isSafeInteger(value) && Number(value) >= 0;
|
|
123
|
+
}
|
|
124
|
+
main().catch((error) => {
|
|
125
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
126
|
+
console.error(message);
|
|
127
|
+
process.exitCode = 1;
|
|
128
|
+
});
|
|
129
|
+
//# sourceMappingURL=run-reanchor-profile.js.map
|