@mrsf/cli 0.6.0 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +10 -7
- package/dist/browser.cjs +1101 -24
- package/dist/browser.cjs.map +4 -4
- package/dist/browser.d.ts +11 -0
- package/dist/browser.js +8 -0
- package/dist/evaluation/generate-reanchor-cases.d.ts +10 -0
- package/dist/evaluation/generate-reanchor-cases.js +301 -0
- package/dist/evaluation/reanchor-eval.d.ts +111 -0
- package/dist/evaluation/reanchor-eval.js +266 -0
- package/dist/evaluation/reanchor-gates.d.ts +29 -0
- package/dist/evaluation/reanchor-gates.js +47 -0
- package/dist/evaluation/run-reanchor-eval.d.ts +3 -0
- package/dist/evaluation/run-reanchor-eval.js +103 -0
- package/dist/evaluation/run-reanchor-generator.d.ts +3 -0
- package/dist/evaluation/run-reanchor-generator.js +63 -0
- package/dist/evaluation/run-reanchor-profile.d.ts +3 -0
- package/dist/evaluation/run-reanchor-profile.js +129 -0
- package/dist/index.cjs +1209 -34
- package/dist/index.cjs.map +4 -4
- package/dist/index.d.ts +12 -0
- package/dist/index.js +9 -0
- package/dist/lib/anchor-context.d.ts +46 -0
- package/dist/lib/anchor-context.js +474 -0
- package/dist/lib/confidence-calibration.d.ts +16 -0
- package/dist/lib/confidence-calibration.js +120 -0
- package/dist/lib/fuzzy.d.ts +14 -1
- package/dist/lib/fuzzy.js +140 -19
- package/dist/lib/git.d.ts +2 -0
- package/dist/lib/git.js +12 -0
- package/dist/lib/global-reconciliation.d.ts +11 -0
- package/dist/lib/global-reconciliation.js +145 -0
- package/dist/lib/reanchor-core.d.ts +7 -0
- package/dist/lib/reanchor-core.js +46 -4
- package/dist/lib/reanchor.js +82 -11
- package/dist/lib/revision-projection.d.ts +21 -0
- package/dist/lib/revision-projection.js +203 -0
- package/package.json +5 -1
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import type { EvaluationCase } from "./reanchor-eval.js";
|
|
2
|
+
export interface GenerateCasesOptions {
|
|
3
|
+
seed: number;
|
|
4
|
+
caseCount?: number;
|
|
5
|
+
blocksPerCase?: number;
|
|
6
|
+
commentsPerCase?: number;
|
|
7
|
+
mutationsPerCase?: number;
|
|
8
|
+
}
|
|
9
|
+
export declare function generateEvaluationCases(options: GenerateCasesOptions): EvaluationCase[];
|
|
10
|
+
//# sourceMappingURL=generate-reanchor-cases.d.ts.map
|
|
@@ -0,0 +1,301 @@
|
|
|
1
|
+
const MUTATION_TYPES = [
|
|
2
|
+
"insert-block",
|
|
3
|
+
"delete-block",
|
|
4
|
+
"move-block",
|
|
5
|
+
"rewrite-block",
|
|
6
|
+
"duplicate-block",
|
|
7
|
+
"swap-blocks",
|
|
8
|
+
"whitespace-block",
|
|
9
|
+
"split-block",
|
|
10
|
+
"merge-blocks",
|
|
11
|
+
"rename-heading",
|
|
12
|
+
];
|
|
13
|
+
export function generateEvaluationCases(options) {
|
|
14
|
+
const caseCount = positiveInteger(options.caseCount ?? 10, "caseCount");
|
|
15
|
+
const blocksPerCase = positiveInteger(options.blocksPerCase ?? 12, "blocksPerCase");
|
|
16
|
+
const commentsPerCase = positiveInteger(options.commentsPerCase ?? 4, "commentsPerCase");
|
|
17
|
+
const mutationsPerCase = positiveInteger(options.mutationsPerCase ?? 5, "mutationsPerCase");
|
|
18
|
+
const seed = unsignedSeed(options.seed);
|
|
19
|
+
return Array.from({ length: caseCount }, (_, index) => generateCase({
|
|
20
|
+
seed: (seed + index) >>> 0,
|
|
21
|
+
blocksPerCase,
|
|
22
|
+
commentsPerCase,
|
|
23
|
+
mutationsPerCase,
|
|
24
|
+
}));
|
|
25
|
+
}
|
|
26
|
+
function generateCase(options) {
|
|
27
|
+
const random = new SeededRandom(options.seed);
|
|
28
|
+
const sourceBlocks = createSourceBlocks(options.seed, options.blocksPerCase);
|
|
29
|
+
const source = renderBlocks(sourceBlocks);
|
|
30
|
+
const trackedComments = selectComments(sourceBlocks, source, options.commentsPerCase, random);
|
|
31
|
+
const targetBlocks = sourceBlocks.map((block) => ({ ...block }));
|
|
32
|
+
const operations = [];
|
|
33
|
+
let nextBlockNumber = sourceBlocks.length + 1;
|
|
34
|
+
for (let index = 0; index < options.mutationsPerCase; index += 1) {
|
|
35
|
+
const preferredType = random.pick(MUTATION_TYPES);
|
|
36
|
+
const operation = applyMutation(preferredType, targetBlocks, trackedComments, random, () => `generated-${options.seed}-block-${nextBlockNumber++}`);
|
|
37
|
+
operations.push(operation);
|
|
38
|
+
}
|
|
39
|
+
const target = renderBlocks(targetBlocks);
|
|
40
|
+
const comments = trackedComments.map((comment) => createEvaluationComment(comment, targetBlocks, target));
|
|
41
|
+
const categories = [
|
|
42
|
+
"generated",
|
|
43
|
+
...new Set(operations.map((operation) => operation.type)),
|
|
44
|
+
];
|
|
45
|
+
return {
|
|
46
|
+
id: `generated-${options.seed}`,
|
|
47
|
+
description: `Seeded mutation case ${options.seed} with ${operations.length} operations.`,
|
|
48
|
+
categories,
|
|
49
|
+
source: { text: source.text },
|
|
50
|
+
target: { text: target.text },
|
|
51
|
+
generation: {
|
|
52
|
+
generator_version: 1,
|
|
53
|
+
seed: options.seed,
|
|
54
|
+
blocks_per_case: options.blocksPerCase,
|
|
55
|
+
comments_per_case: Math.min(options.commentsPerCase, options.blocksPerCase),
|
|
56
|
+
operations,
|
|
57
|
+
},
|
|
58
|
+
comments,
|
|
59
|
+
};
|
|
60
|
+
}
|
|
61
|
+
function createSourceBlocks(seed, count) {
|
|
62
|
+
return Array.from({ length: count }, (_, index) => {
|
|
63
|
+
const number = index + 1;
|
|
64
|
+
return {
|
|
65
|
+
id: `generated-${seed}-block-${number}`,
|
|
66
|
+
text: number % 4 === 1
|
|
67
|
+
? `## Section ${number} for seed ${seed}`
|
|
68
|
+
: `Paragraph ${number} describes behavior token-${seed}-${number}.`,
|
|
69
|
+
};
|
|
70
|
+
});
|
|
71
|
+
}
|
|
72
|
+
function selectComments(blocks, rendered, requestedCount, random) {
|
|
73
|
+
const available = blocks.map((_, index) => index);
|
|
74
|
+
random.shuffle(available);
|
|
75
|
+
return available
|
|
76
|
+
.slice(0, Math.min(requestedCount, blocks.length))
|
|
77
|
+
.sort((left, right) => left - right)
|
|
78
|
+
.map((blockIndex, commentIndex) => {
|
|
79
|
+
const block = blocks[blockIndex];
|
|
80
|
+
const range = rendered.ranges.get(block.id);
|
|
81
|
+
if (!range)
|
|
82
|
+
throw new Error(`Missing source range for ${block.id}.`);
|
|
83
|
+
return {
|
|
84
|
+
id: `generated-comment-${commentIndex + 1}`,
|
|
85
|
+
sourceLine: range.line,
|
|
86
|
+
selectedText: block.text,
|
|
87
|
+
targetBlockId: block.id,
|
|
88
|
+
};
|
|
89
|
+
});
|
|
90
|
+
}
|
|
91
|
+
function applyMutation(preferredType, blocks, comments, random, nextBlockId) {
|
|
92
|
+
if (preferredType === "insert-block") {
|
|
93
|
+
const id = nextBlockId();
|
|
94
|
+
const position = random.nextInt(blocks.length + 1);
|
|
95
|
+
blocks.splice(position, 0, {
|
|
96
|
+
id,
|
|
97
|
+
text: `Inserted paragraph ${id} describes new material.`,
|
|
98
|
+
});
|
|
99
|
+
return { type: "insert-block", block_ids: [id] };
|
|
100
|
+
}
|
|
101
|
+
if (preferredType === "delete-block" && blocks.length > 1) {
|
|
102
|
+
const index = random.nextInt(blocks.length);
|
|
103
|
+
const [removed] = blocks.splice(index, 1);
|
|
104
|
+
for (const comment of comments) {
|
|
105
|
+
if (comment.targetBlockId === removed.id) {
|
|
106
|
+
delete comment.targetBlockId;
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
return { type: "delete-block", block_ids: [removed.id] };
|
|
110
|
+
}
|
|
111
|
+
if (preferredType === "move-block" && blocks.length > 1) {
|
|
112
|
+
const from = random.nextInt(blocks.length);
|
|
113
|
+
let to = random.nextInt(blocks.length);
|
|
114
|
+
if (to === from)
|
|
115
|
+
to = (to + 1) % blocks.length;
|
|
116
|
+
const [moved] = blocks.splice(from, 1);
|
|
117
|
+
blocks.splice(to, 0, moved);
|
|
118
|
+
return { type: "move-block", block_ids: [moved.id] };
|
|
119
|
+
}
|
|
120
|
+
if (preferredType === "rewrite-block") {
|
|
121
|
+
const block = random.pick(blocks);
|
|
122
|
+
block.text = rewriteFirstWord(block.text);
|
|
123
|
+
return { type: "rewrite-block", block_ids: [block.id] };
|
|
124
|
+
}
|
|
125
|
+
if (preferredType === "duplicate-block") {
|
|
126
|
+
const source = random.pick(blocks);
|
|
127
|
+
const duplicateId = nextBlockId();
|
|
128
|
+
const sourceIndex = blocks.indexOf(source);
|
|
129
|
+
blocks.splice(sourceIndex + 1, 0, {
|
|
130
|
+
id: duplicateId,
|
|
131
|
+
text: source.text,
|
|
132
|
+
});
|
|
133
|
+
return {
|
|
134
|
+
type: "duplicate-block",
|
|
135
|
+
block_ids: [source.id, duplicateId],
|
|
136
|
+
};
|
|
137
|
+
}
|
|
138
|
+
if (preferredType === "swap-blocks" && blocks.length > 1) {
|
|
139
|
+
const first = random.nextInt(blocks.length);
|
|
140
|
+
let second = random.nextInt(blocks.length);
|
|
141
|
+
if (first === second)
|
|
142
|
+
second = (second + 1) % blocks.length;
|
|
143
|
+
[blocks[first], blocks[second]] = [blocks[second], blocks[first]];
|
|
144
|
+
return {
|
|
145
|
+
type: "swap-blocks",
|
|
146
|
+
block_ids: [blocks[second].id, blocks[first].id],
|
|
147
|
+
};
|
|
148
|
+
}
|
|
149
|
+
if (preferredType === "whitespace-block") {
|
|
150
|
+
const block = random.pick(blocks);
|
|
151
|
+
block.text = block.text.replace(/ /g, " ");
|
|
152
|
+
return { type: "whitespace-block", block_ids: [block.id] };
|
|
153
|
+
}
|
|
154
|
+
if (preferredType === "split-block") {
|
|
155
|
+
const candidates = blocks.filter((block) => block.text.includes(" "));
|
|
156
|
+
const block = random.pick(candidates.length > 0 ? candidates : blocks);
|
|
157
|
+
const splitAt = splitPosition(block.text);
|
|
158
|
+
block.text = `${block.text.slice(0, splitAt)}\n\n${block.text.slice(splitAt + 1)}`;
|
|
159
|
+
return { type: "split-block", block_ids: [block.id] };
|
|
160
|
+
}
|
|
161
|
+
if (preferredType === "merge-blocks" && blocks.length > 1) {
|
|
162
|
+
const firstIndex = random.nextInt(blocks.length - 1);
|
|
163
|
+
const first = blocks[firstIndex];
|
|
164
|
+
const [second] = blocks.splice(firstIndex + 1, 1);
|
|
165
|
+
first.text = `${first.text} ${second.text}`;
|
|
166
|
+
for (const comment of comments) {
|
|
167
|
+
if (comment.targetBlockId === second.id) {
|
|
168
|
+
comment.targetBlockId = first.id;
|
|
169
|
+
}
|
|
170
|
+
}
|
|
171
|
+
return { type: "merge-blocks", block_ids: [first.id, second.id] };
|
|
172
|
+
}
|
|
173
|
+
if (preferredType === "rename-heading") {
|
|
174
|
+
const headings = blocks.filter((block) => block.text.startsWith("#"));
|
|
175
|
+
if (headings.length > 0) {
|
|
176
|
+
const heading = random.pick(headings);
|
|
177
|
+
heading.text = heading.text.replace("Section", "Topic");
|
|
178
|
+
return { type: "rename-heading", block_ids: [heading.id] };
|
|
179
|
+
}
|
|
180
|
+
}
|
|
181
|
+
return applyMutation("insert-block", blocks, comments, random, nextBlockId);
|
|
182
|
+
}
|
|
183
|
+
function createEvaluationComment(tracked, targetBlocks, target) {
|
|
184
|
+
if (!tracked.targetBlockId) {
|
|
185
|
+
return {
|
|
186
|
+
id: tracked.id,
|
|
187
|
+
anchor: {
|
|
188
|
+
line: tracked.sourceLine,
|
|
189
|
+
selected_text: tracked.selectedText,
|
|
190
|
+
},
|
|
191
|
+
expected: {
|
|
192
|
+
status: "orphaned",
|
|
193
|
+
rationale: "The semantic source block was deleted.",
|
|
194
|
+
},
|
|
195
|
+
};
|
|
196
|
+
}
|
|
197
|
+
const targetBlock = targetBlocks.find((block) => block.id === tracked.targetBlockId);
|
|
198
|
+
const blockRange = target.ranges.get(tracked.targetBlockId);
|
|
199
|
+
if (!targetBlock || !blockRange) {
|
|
200
|
+
throw new Error(`Missing target block ${tracked.targetBlockId}.`);
|
|
201
|
+
}
|
|
202
|
+
const exactIndex = targetBlock.text.indexOf(tracked.selectedText);
|
|
203
|
+
const exact = exactIndex >= 0;
|
|
204
|
+
const range = exact
|
|
205
|
+
? exactRangeWithinBlock(blockRange.line, targetBlock.text, tracked.selectedText, exactIndex)
|
|
206
|
+
: blockRange;
|
|
207
|
+
return {
|
|
208
|
+
id: tracked.id,
|
|
209
|
+
anchor: {
|
|
210
|
+
line: tracked.sourceLine,
|
|
211
|
+
selected_text: tracked.selectedText,
|
|
212
|
+
},
|
|
213
|
+
expected: {
|
|
214
|
+
status: exact ? "anchored" : "fuzzy",
|
|
215
|
+
ranges: [range],
|
|
216
|
+
rationale: exact
|
|
217
|
+
? "The original selection remains inside its semantic block."
|
|
218
|
+
: "The semantic block remains but its selected text changed.",
|
|
219
|
+
},
|
|
220
|
+
};
|
|
221
|
+
}
|
|
222
|
+
function exactRangeWithinBlock(blockLine, blockText, selectedText, index) {
|
|
223
|
+
const linesBefore = blockText.slice(0, index).split("\n").length - 1;
|
|
224
|
+
const selectedLines = selectedText.split("\n").length;
|
|
225
|
+
const line = blockLine + linesBefore;
|
|
226
|
+
return {
|
|
227
|
+
line,
|
|
228
|
+
end_line: line + selectedLines - 1,
|
|
229
|
+
};
|
|
230
|
+
}
|
|
231
|
+
function renderBlocks(blocks) {
|
|
232
|
+
const ranges = new Map();
|
|
233
|
+
let line = 1;
|
|
234
|
+
for (const block of blocks) {
|
|
235
|
+
const lineCount = block.text.split("\n").length;
|
|
236
|
+
ranges.set(block.id, {
|
|
237
|
+
line,
|
|
238
|
+
end_line: line + lineCount - 1,
|
|
239
|
+
});
|
|
240
|
+
line += lineCount + 1;
|
|
241
|
+
}
|
|
242
|
+
return {
|
|
243
|
+
text: `${blocks.map((block) => block.text).join("\n\n")}\n`,
|
|
244
|
+
ranges,
|
|
245
|
+
};
|
|
246
|
+
}
|
|
247
|
+
function rewriteFirstWord(text) {
|
|
248
|
+
return text.replace(/^(\W*)([\p{L}\p{N}]+)/u, "$1$2-revised");
|
|
249
|
+
}
|
|
250
|
+
function splitPosition(text) {
|
|
251
|
+
const midpoint = Math.floor(text.length / 2);
|
|
252
|
+
const after = text.indexOf(" ", midpoint);
|
|
253
|
+
if (after >= 0)
|
|
254
|
+
return after;
|
|
255
|
+
const before = text.lastIndexOf(" ", midpoint);
|
|
256
|
+
return before >= 0 ? before : midpoint;
|
|
257
|
+
}
|
|
258
|
+
function positiveInteger(value, name) {
|
|
259
|
+
if (!Number.isSafeInteger(value) || value < 1) {
|
|
260
|
+
throw new Error(`${name} must be a positive integer.`);
|
|
261
|
+
}
|
|
262
|
+
return value;
|
|
263
|
+
}
|
|
264
|
+
function unsignedSeed(value) {
|
|
265
|
+
if (!Number.isSafeInteger(value) || value < 0 || value > 0xffffffff) {
|
|
266
|
+
throw new Error("seed must be an integer between 0 and 4294967295.");
|
|
267
|
+
}
|
|
268
|
+
return value >>> 0;
|
|
269
|
+
}
|
|
270
|
+
class SeededRandom {
|
|
271
|
+
state;
|
|
272
|
+
constructor(seed) {
|
|
273
|
+
this.state = seed === 0 ? 0x9e3779b9 : seed >>> 0;
|
|
274
|
+
}
|
|
275
|
+
nextInt(maxExclusive) {
|
|
276
|
+
if (!Number.isSafeInteger(maxExclusive) || maxExclusive < 1) {
|
|
277
|
+
throw new Error("Random upper bound must be a positive integer.");
|
|
278
|
+
}
|
|
279
|
+
return Math.floor(this.next() * maxExclusive);
|
|
280
|
+
}
|
|
281
|
+
pick(values) {
|
|
282
|
+
if (values.length === 0)
|
|
283
|
+
throw new Error("Cannot pick from an empty array.");
|
|
284
|
+
return values[this.nextInt(values.length)];
|
|
285
|
+
}
|
|
286
|
+
shuffle(values) {
|
|
287
|
+
for (let index = values.length - 1; index > 0; index -= 1) {
|
|
288
|
+
const other = this.nextInt(index + 1);
|
|
289
|
+
[values[index], values[other]] = [values[other], values[index]];
|
|
290
|
+
}
|
|
291
|
+
}
|
|
292
|
+
next() {
|
|
293
|
+
let value = this.state;
|
|
294
|
+
value ^= value << 13;
|
|
295
|
+
value ^= value >>> 17;
|
|
296
|
+
value ^= value << 5;
|
|
297
|
+
this.state = value >>> 0;
|
|
298
|
+
return this.state / 0x100000000;
|
|
299
|
+
}
|
|
300
|
+
}
|
|
301
|
+
//# sourceMappingURL=generate-reanchor-cases.js.map
|
|
@@ -0,0 +1,111 @@
|
|
|
1
|
+
import type { ReanchorResult, ReanchorStatus } from "../lib/types.js";
|
|
2
|
+
export interface EvaluationDocument {
|
|
3
|
+
text?: string;
|
|
4
|
+
path?: string;
|
|
5
|
+
}
|
|
6
|
+
export interface EvaluationAnchor {
|
|
7
|
+
line?: number;
|
|
8
|
+
end_line?: number;
|
|
9
|
+
start_column?: number;
|
|
10
|
+
end_column?: number;
|
|
11
|
+
selected_text?: string;
|
|
12
|
+
selected_text_hash?: string;
|
|
13
|
+
commit?: string;
|
|
14
|
+
}
|
|
15
|
+
export interface EvaluationRange {
|
|
16
|
+
line: number;
|
|
17
|
+
end_line?: number;
|
|
18
|
+
start_column?: number;
|
|
19
|
+
end_column?: number;
|
|
20
|
+
}
|
|
21
|
+
export interface EvaluationExpectation {
|
|
22
|
+
status: ReanchorStatus;
|
|
23
|
+
ranges?: EvaluationRange[];
|
|
24
|
+
rationale?: string;
|
|
25
|
+
}
|
|
26
|
+
export interface EvaluationComment {
|
|
27
|
+
id: string;
|
|
28
|
+
anchor: EvaluationAnchor;
|
|
29
|
+
expected: EvaluationExpectation;
|
|
30
|
+
}
|
|
31
|
+
export interface EvaluationCase {
|
|
32
|
+
id: string;
|
|
33
|
+
description?: string;
|
|
34
|
+
categories: string[];
|
|
35
|
+
source: EvaluationDocument;
|
|
36
|
+
target: EvaluationDocument;
|
|
37
|
+
generation?: EvaluationGeneration;
|
|
38
|
+
comments: EvaluationComment[];
|
|
39
|
+
}
|
|
40
|
+
export interface EvaluationGeneration {
|
|
41
|
+
generator_version: 1;
|
|
42
|
+
seed: number;
|
|
43
|
+
blocks_per_case: number;
|
|
44
|
+
comments_per_case: number;
|
|
45
|
+
operations: EvaluationMutation[];
|
|
46
|
+
}
|
|
47
|
+
export interface EvaluationMutation {
|
|
48
|
+
type: "insert-block" | "delete-block" | "move-block" | "rewrite-block" | "duplicate-block" | "swap-blocks" | "whitespace-block" | "split-block" | "merge-blocks" | "rename-heading";
|
|
49
|
+
block_ids: string[];
|
|
50
|
+
}
|
|
51
|
+
export interface EvaluationCommentResult {
|
|
52
|
+
caseId: string;
|
|
53
|
+
commentId: string;
|
|
54
|
+
expected: EvaluationExpectation;
|
|
55
|
+
actual: ReanchorResult;
|
|
56
|
+
statusCorrect: boolean;
|
|
57
|
+
rangeCorrect: boolean;
|
|
58
|
+
passed: boolean;
|
|
59
|
+
durationMs: number;
|
|
60
|
+
}
|
|
61
|
+
export interface EvaluationSummary {
|
|
62
|
+
algorithm: string;
|
|
63
|
+
cases: number;
|
|
64
|
+
comments: number;
|
|
65
|
+
passed: number;
|
|
66
|
+
failed: number;
|
|
67
|
+
exactRangeMatches: number;
|
|
68
|
+
statusMatches: number;
|
|
69
|
+
incorrectConfidentRelocations: number;
|
|
70
|
+
durationMs: number;
|
|
71
|
+
timingMs: {
|
|
72
|
+
median: number;
|
|
73
|
+
p95: number;
|
|
74
|
+
max: number;
|
|
75
|
+
perComment: number;
|
|
76
|
+
reconciliationMax: number;
|
|
77
|
+
};
|
|
78
|
+
results: EvaluationCommentResult[];
|
|
79
|
+
}
|
|
80
|
+
export interface EvaluationBaseline {
|
|
81
|
+
version: 1;
|
|
82
|
+
algorithm: string;
|
|
83
|
+
cases: number;
|
|
84
|
+
comments: number;
|
|
85
|
+
metrics: {
|
|
86
|
+
passed: number;
|
|
87
|
+
failed: number;
|
|
88
|
+
exactRangeMatches: number;
|
|
89
|
+
statusMatches: number;
|
|
90
|
+
incorrectConfidentRelocations: number;
|
|
91
|
+
};
|
|
92
|
+
results: EvaluationBaselineResult[];
|
|
93
|
+
}
|
|
94
|
+
export interface EvaluationBaselineResult {
|
|
95
|
+
caseId: string;
|
|
96
|
+
commentId: string;
|
|
97
|
+
expectedStatus: ReanchorStatus;
|
|
98
|
+
actual: ReanchorResult;
|
|
99
|
+
statusCorrect: boolean;
|
|
100
|
+
rangeCorrect: boolean;
|
|
101
|
+
passed: boolean;
|
|
102
|
+
}
|
|
103
|
+
export interface LoadedEvaluationCase {
|
|
104
|
+
casePath: string;
|
|
105
|
+
value: EvaluationCase;
|
|
106
|
+
}
|
|
107
|
+
export declare function loadEvaluationCases(casesPath: string, schemaPath: string): Promise<LoadedEvaluationCase[]>;
|
|
108
|
+
export declare function evaluateCases(cases: LoadedEvaluationCase[]): Promise<EvaluationSummary>;
|
|
109
|
+
export declare function createEvaluationBaseline(summary: EvaluationSummary): EvaluationBaseline;
|
|
110
|
+
export declare function findBaselineDifferences(expected: EvaluationBaseline, actual: EvaluationBaseline): string[];
|
|
111
|
+
//# sourceMappingURL=reanchor-eval.d.ts.map
|
|
@@ -0,0 +1,266 @@
|
|
|
1
|
+
import Ajv2020Module from "ajv/dist/2020.js";
|
|
2
|
+
import { readFile, readdir } from "node:fs/promises";
|
|
3
|
+
import path from "node:path";
|
|
4
|
+
import { performance } from "node:perf_hooks";
|
|
5
|
+
import { reanchorComment, toReanchorLines } from "../lib/reanchor-core.js";
|
|
6
|
+
import { createRevisionProjection } from "../lib/revision-projection.js";
|
|
7
|
+
import { createAnchorContextIndex } from "../lib/anchor-context.js";
|
|
8
|
+
import { createFuzzySearchIndex, } from "../lib/fuzzy.js";
|
|
9
|
+
import { reconcileCommentAnchors } from "../lib/global-reconciliation.js";
|
|
10
|
+
const Ajv2020 = Ajv2020Module;
|
|
11
|
+
export async function loadEvaluationCases(casesPath, schemaPath) {
|
|
12
|
+
const schema = JSON.parse(await readFile(schemaPath, "utf8"));
|
|
13
|
+
const validator = new Ajv2020({
|
|
14
|
+
allErrors: true,
|
|
15
|
+
strict: true,
|
|
16
|
+
strictRequired: false,
|
|
17
|
+
}).compile(schema);
|
|
18
|
+
const files = await findJsonFiles(casesPath);
|
|
19
|
+
const loaded = [];
|
|
20
|
+
const caseIds = new Set();
|
|
21
|
+
for (const casePath of files) {
|
|
22
|
+
const parsed = JSON.parse(await readFile(casePath, "utf8"));
|
|
23
|
+
const values = Array.isArray(parsed) ? parsed : [parsed];
|
|
24
|
+
for (const [index, value] of values.entries()) {
|
|
25
|
+
const location = values.length === 1 ? casePath : `${casePath}[${index}]`;
|
|
26
|
+
assertValidCase(validator, value, location);
|
|
27
|
+
if (caseIds.has(value.id)) {
|
|
28
|
+
throw new Error(`Duplicate evaluation case id "${value.id}" in ${location}.`);
|
|
29
|
+
}
|
|
30
|
+
caseIds.add(value.id);
|
|
31
|
+
loaded.push({ casePath, value });
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
if (loaded.length === 0) {
|
|
35
|
+
throw new Error(`No evaluation cases found in ${casesPath}.`);
|
|
36
|
+
}
|
|
37
|
+
return loaded;
|
|
38
|
+
}
|
|
39
|
+
export async function evaluateCases(cases) {
|
|
40
|
+
const startedAt = performance.now();
|
|
41
|
+
const results = [];
|
|
42
|
+
const reconciliationTimings = [];
|
|
43
|
+
for (const loadedCase of cases) {
|
|
44
|
+
const sourceText = await readEvaluationDocument(loadedCase.value.source, loadedCase.casePath);
|
|
45
|
+
validateSourceSelections(loadedCase.value, sourceText);
|
|
46
|
+
const targetText = await readEvaluationDocument(loadedCase.value.target, loadedCase.casePath);
|
|
47
|
+
const sourceLines = toReanchorLines(sourceText);
|
|
48
|
+
const documentLines = toReanchorLines(targetText);
|
|
49
|
+
const revisionProjection = createRevisionProjection(sourceLines, documentLines);
|
|
50
|
+
const anchorContext = createAnchorContextIndex(sourceLines, documentLines);
|
|
51
|
+
let fuzzySearchIndex;
|
|
52
|
+
const getFuzzySearchIndex = () => {
|
|
53
|
+
fuzzySearchIndex ??= createFuzzySearchIndex(documentLines);
|
|
54
|
+
return fuzzySearchIndex;
|
|
55
|
+
};
|
|
56
|
+
const caseComments = [];
|
|
57
|
+
const caseResults = [];
|
|
58
|
+
for (const evaluationComment of loadedCase.value.comments) {
|
|
59
|
+
const comment = toComment(evaluationComment);
|
|
60
|
+
const commentStartedAt = performance.now();
|
|
61
|
+
const actual = reanchorComment(comment, documentLines, {
|
|
62
|
+
revisionProjection,
|
|
63
|
+
anchorContext,
|
|
64
|
+
getFuzzySearchIndex,
|
|
65
|
+
});
|
|
66
|
+
const durationMs = performance.now() - commentStartedAt;
|
|
67
|
+
caseComments.push(comment);
|
|
68
|
+
caseResults.push({
|
|
69
|
+
evaluationComment,
|
|
70
|
+
comment,
|
|
71
|
+
actual,
|
|
72
|
+
durationMs,
|
|
73
|
+
});
|
|
74
|
+
}
|
|
75
|
+
const reconciliationStartedAt = performance.now();
|
|
76
|
+
const reconciled = reconcileCommentAnchors(caseComments, caseResults.map((result) => result.actual), anchorContext);
|
|
77
|
+
reconciliationTimings.push(performance.now() - reconciliationStartedAt);
|
|
78
|
+
for (const [index, caseResult] of caseResults.entries()) {
|
|
79
|
+
const { evaluationComment, comment, durationMs, } = caseResult;
|
|
80
|
+
const actual = reconciled[index];
|
|
81
|
+
const statusCorrect = actual.status === evaluationComment.expected.status;
|
|
82
|
+
const rangeCorrect = matchesExpectedRange(actual, comment, evaluationComment.expected.ranges);
|
|
83
|
+
results.push({
|
|
84
|
+
caseId: loadedCase.value.id,
|
|
85
|
+
commentId: evaluationComment.id,
|
|
86
|
+
expected: evaluationComment.expected,
|
|
87
|
+
actual,
|
|
88
|
+
statusCorrect,
|
|
89
|
+
rangeCorrect,
|
|
90
|
+
passed: statusCorrect && rangeCorrect,
|
|
91
|
+
durationMs,
|
|
92
|
+
});
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
const durationMs = performance.now() - startedAt;
|
|
96
|
+
const timings = results
|
|
97
|
+
.map((result) => result.durationMs)
|
|
98
|
+
.sort((left, right) => left - right);
|
|
99
|
+
return {
|
|
100
|
+
algorithm: "baseline",
|
|
101
|
+
cases: cases.length,
|
|
102
|
+
comments: results.length,
|
|
103
|
+
passed: results.filter((result) => result.passed).length,
|
|
104
|
+
failed: results.filter((result) => !result.passed).length,
|
|
105
|
+
exactRangeMatches: results.filter((result) => result.rangeCorrect).length,
|
|
106
|
+
statusMatches: results.filter((result) => result.statusCorrect).length,
|
|
107
|
+
incorrectConfidentRelocations: results.filter((result) => (result.actual.status === "anchored" || result.actual.status === "shifted")
|
|
108
|
+
&& (result.expected.ranges?.length
|
|
109
|
+
? !result.rangeCorrect
|
|
110
|
+
: result.expected.status === "ambiguous"
|
|
111
|
+
|| result.expected.status === "orphaned")).length,
|
|
112
|
+
durationMs,
|
|
113
|
+
timingMs: {
|
|
114
|
+
median: percentile(timings, 0.5),
|
|
115
|
+
p95: percentile(timings, 0.95),
|
|
116
|
+
max: timings.at(-1) ?? 0,
|
|
117
|
+
perComment: results.length > 0 ? durationMs / results.length : 0,
|
|
118
|
+
reconciliationMax: Math.max(0, ...reconciliationTimings),
|
|
119
|
+
},
|
|
120
|
+
results,
|
|
121
|
+
};
|
|
122
|
+
}
|
|
123
|
+
function percentile(sortedValues, quantile) {
|
|
124
|
+
if (sortedValues.length === 0)
|
|
125
|
+
return 0;
|
|
126
|
+
const index = Math.max(0, Math.ceil(sortedValues.length * quantile) - 1);
|
|
127
|
+
return sortedValues[index];
|
|
128
|
+
}
|
|
129
|
+
export function createEvaluationBaseline(summary) {
|
|
130
|
+
return {
|
|
131
|
+
version: 1,
|
|
132
|
+
algorithm: summary.algorithm,
|
|
133
|
+
cases: summary.cases,
|
|
134
|
+
comments: summary.comments,
|
|
135
|
+
metrics: {
|
|
136
|
+
passed: summary.passed,
|
|
137
|
+
failed: summary.failed,
|
|
138
|
+
exactRangeMatches: summary.exactRangeMatches,
|
|
139
|
+
statusMatches: summary.statusMatches,
|
|
140
|
+
incorrectConfidentRelocations: summary.incorrectConfidentRelocations,
|
|
141
|
+
},
|
|
142
|
+
results: summary.results.map((result) => ({
|
|
143
|
+
caseId: result.caseId,
|
|
144
|
+
commentId: result.commentId,
|
|
145
|
+
expectedStatus: result.expected.status,
|
|
146
|
+
actual: result.actual,
|
|
147
|
+
statusCorrect: result.statusCorrect,
|
|
148
|
+
rangeCorrect: result.rangeCorrect,
|
|
149
|
+
passed: result.passed,
|
|
150
|
+
})),
|
|
151
|
+
};
|
|
152
|
+
}
|
|
153
|
+
export function findBaselineDifferences(expected, actual) {
|
|
154
|
+
const differences = [];
|
|
155
|
+
if (expected.version !== actual.version) {
|
|
156
|
+
differences.push(`Baseline version changed from ${expected.version} to ${actual.version}.`);
|
|
157
|
+
}
|
|
158
|
+
if (expected.algorithm !== actual.algorithm) {
|
|
159
|
+
differences.push(`Algorithm changed from ${expected.algorithm} to ${actual.algorithm}.`);
|
|
160
|
+
}
|
|
161
|
+
const expectedHeader = {
|
|
162
|
+
cases: expected.cases,
|
|
163
|
+
comments: expected.comments,
|
|
164
|
+
metrics: expected.metrics,
|
|
165
|
+
};
|
|
166
|
+
const actualHeader = {
|
|
167
|
+
cases: actual.cases,
|
|
168
|
+
comments: actual.comments,
|
|
169
|
+
metrics: actual.metrics,
|
|
170
|
+
};
|
|
171
|
+
if (JSON.stringify(expectedHeader) !== JSON.stringify(actualHeader)) {
|
|
172
|
+
differences.push(`Summary changed: expected ${JSON.stringify(expectedHeader)}, `
|
|
173
|
+
+ `received ${JSON.stringify(actualHeader)}.`);
|
|
174
|
+
}
|
|
175
|
+
const expectedResults = new Map(expected.results.map((result) => [baselineResultKey(result), result]));
|
|
176
|
+
const actualResults = new Map(actual.results.map((result) => [baselineResultKey(result), result]));
|
|
177
|
+
for (const [key, expectedResult] of expectedResults) {
|
|
178
|
+
const actualResult = actualResults.get(key);
|
|
179
|
+
if (!actualResult) {
|
|
180
|
+
differences.push(`Result ${key} is missing.`);
|
|
181
|
+
}
|
|
182
|
+
else if (JSON.stringify(expectedResult) !== JSON.stringify(actualResult)) {
|
|
183
|
+
differences.push(`Result ${key} changed.`);
|
|
184
|
+
}
|
|
185
|
+
}
|
|
186
|
+
for (const key of actualResults.keys()) {
|
|
187
|
+
if (!expectedResults.has(key)) {
|
|
188
|
+
differences.push(`Result ${key} is new.`);
|
|
189
|
+
}
|
|
190
|
+
}
|
|
191
|
+
return differences;
|
|
192
|
+
}
|
|
193
|
+
function baselineResultKey(result) {
|
|
194
|
+
return `${result.caseId}/${result.commentId}`;
|
|
195
|
+
}
|
|
196
|
+
async function findJsonFiles(rootPath) {
|
|
197
|
+
const entries = await readdir(rootPath, { withFileTypes: true });
|
|
198
|
+
const files = [];
|
|
199
|
+
for (const entry of entries.sort((left, right) => left.name.localeCompare(right.name))) {
|
|
200
|
+
const entryPath = path.join(rootPath, entry.name);
|
|
201
|
+
if (entry.isDirectory()) {
|
|
202
|
+
files.push(...await findJsonFiles(entryPath));
|
|
203
|
+
}
|
|
204
|
+
else if (entry.isFile() && entry.name.endsWith(".json")) {
|
|
205
|
+
files.push(entryPath);
|
|
206
|
+
}
|
|
207
|
+
}
|
|
208
|
+
return files;
|
|
209
|
+
}
|
|
210
|
+
function assertValidCase(validator, value, casePath) {
|
|
211
|
+
if (validator(value))
|
|
212
|
+
return;
|
|
213
|
+
const details = (validator.errors ?? [])
|
|
214
|
+
.map(formatValidationError)
|
|
215
|
+
.join("\n");
|
|
216
|
+
throw new Error(`Invalid evaluation case ${casePath}:\n${details}`);
|
|
217
|
+
}
|
|
218
|
+
function formatValidationError(error) {
|
|
219
|
+
return ` ${error.instancePath || "/"} ${error.message ?? "is invalid"}`;
|
|
220
|
+
}
|
|
221
|
+
async function readEvaluationDocument(document, casePath) {
|
|
222
|
+
if (document.text != null)
|
|
223
|
+
return document.text;
|
|
224
|
+
if (document.path != null) {
|
|
225
|
+
return readFile(path.resolve(path.dirname(casePath), document.path), "utf8");
|
|
226
|
+
}
|
|
227
|
+
throw new Error(`Evaluation document in ${casePath} has neither text nor path.`);
|
|
228
|
+
}
|
|
229
|
+
function validateSourceSelections(evaluationCase, sourceText) {
|
|
230
|
+
const normalizedSource = sourceText.replace(/\r\n/g, "\n");
|
|
231
|
+
for (const comment of evaluationCase.comments) {
|
|
232
|
+
const selectedText = comment.anchor.selected_text?.replace(/\r\n/g, "\n");
|
|
233
|
+
if (selectedText != null && !normalizedSource.includes(selectedText)) {
|
|
234
|
+
throw new Error(`Evaluation case ${evaluationCase.id}/${comment.id} has selected_text `
|
|
235
|
+
+ "that does not occur in its source document.");
|
|
236
|
+
}
|
|
237
|
+
}
|
|
238
|
+
}
|
|
239
|
+
function toComment(evaluationComment) {
|
|
240
|
+
return {
|
|
241
|
+
id: evaluationComment.id,
|
|
242
|
+
author: "MRSF evaluation",
|
|
243
|
+
timestamp: "1970-01-01T00:00:00Z",
|
|
244
|
+
text: "Reanchoring evaluation fixture",
|
|
245
|
+
resolved: false,
|
|
246
|
+
...evaluationComment.anchor,
|
|
247
|
+
};
|
|
248
|
+
}
|
|
249
|
+
function matchesExpectedRange(actual, original, expectedRanges) {
|
|
250
|
+
if (!expectedRanges?.length)
|
|
251
|
+
return true;
|
|
252
|
+
const actualRange = {
|
|
253
|
+
line: actual.newLine ?? original.line ?? 0,
|
|
254
|
+
end_line: actual.newEndLine ?? original.end_line ?? actual.newLine ?? original.line,
|
|
255
|
+
start_column: actual.newStartColumn ?? original.start_column,
|
|
256
|
+
end_column: actual.newEndColumn ?? original.end_column,
|
|
257
|
+
};
|
|
258
|
+
return expectedRanges.some((expected) => actualRange.line === expected.line
|
|
259
|
+
&& (expected.end_line == null
|
|
260
|
+
|| (actualRange.end_line ?? actualRange.line) === expected.end_line)
|
|
261
|
+
&& (expected.start_column == null
|
|
262
|
+
|| actualRange.start_column === expected.start_column)
|
|
263
|
+
&& (expected.end_column == null
|
|
264
|
+
|| actualRange.end_column === expected.end_column));
|
|
265
|
+
}
|
|
266
|
+
//# sourceMappingURL=reanchor-eval.js.map
|