eklavya 1.14.0 → 1.15.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/assets/tutor/SKILL.md +180 -0
- package/dist/assets/tutor/references/focus-and-level.md +156 -0
- package/dist/assets/tutor/references/grading.md +113 -0
- package/dist/assets/tutor/references/writing-mcq.md +133 -0
- package/dist/cli.js +66 -17
- package/dist/cli.js.map +1 -1
- package/dist/eval/extract-json.js +81 -0
- package/dist/eval/extract-json.js.map +1 -0
- package/dist/eval/extraction-score.js +115 -0
- package/dist/eval/extraction-score.js.map +1 -0
- package/dist/eval/history-stats.js +230 -0
- package/dist/eval/history-stats.js.map +1 -0
- package/dist/eval/question-checks.js +220 -0
- package/dist/eval/question-checks.js.map +1 -0
- package/dist/hooks/checkpoint-quiz.js +1 -1
- package/dist/hooks/lib.js +38 -13
- package/dist/hooks/lib.js.map +1 -1
- package/dist/hooks/prompt-submit-nudge.js +155 -0
- package/dist/hooks/prompt-submit-nudge.js.map +1 -0
- package/dist/hooks/session-start.js +32 -14
- package/dist/hooks/session-start.js.map +1 -1
- package/dist/hooks/stop-quiz-check.js +4 -3
- package/dist/hooks/stop-quiz-check.js.map +1 -1
- package/dist/plugin/.claude-plugin/plugin.json +1 -1
- package/dist/plugin/agents/tutor.md +19 -5
- package/dist/plugin/hooks/CLAUDE.md +83 -2
- package/dist/plugin/hooks/hooks.json +12 -0
- package/dist/plugin/hooks/run.mjs +3 -3
- package/dist/plugin/skills/CLAUDE.md +107 -15
- package/dist/plugin/skills/setup/SKILL.md +1 -1
- package/dist/plugin/skills/tutor/SKILL.md +119 -304
- package/dist/plugin/skills/tutor/references/focus-and-level.md +156 -0
- package/dist/plugin/skills/tutor/references/grading.md +113 -0
- package/dist/plugin/skills/tutor/references/writing-mcq.md +133 -0
- package/dist/slug.js +63 -0
- package/dist/slug.js.map +1 -1
- package/dist/stdin.js +131 -0
- package/dist/stdin.js.map +1 -0
- package/dist/store.js +16 -2
- package/dist/store.js.map +1 -1
- package/dist/tools/get_session_quiz_plan.js +1 -1
- package/dist/tools/get_session_quiz_plan.js.map +1 -1
- package/dist/tools/record_attempt.js +15 -3
- package/dist/tools/record_attempt.js.map +1 -1
- package/package.json +1 -1
- package/dist/assets/tutor-skill.md +0 -365
package/dist/cli.js
CHANGED
|
@@ -9,6 +9,7 @@ import path from 'node:path';
|
|
|
9
9
|
import { fileURLToPath } from 'node:url';
|
|
10
10
|
import { openDb } from './db.js';
|
|
11
11
|
import { dbPath, eklavyaHome } from './paths.js';
|
|
12
|
+
import { readStdinBounded, stripBom, STATUSLINE_STDIN } from './stdin.js';
|
|
12
13
|
import { loadConfig, writeConfigFile, REPO_CONFIG_FILE, DEFAULT_CONFIG } from './config.js';
|
|
13
14
|
import { levelStanding } from './store.js';
|
|
14
15
|
import { statusLine } from './statusline.js';
|
|
@@ -45,24 +46,82 @@ function fail(message) {
|
|
|
45
46
|
process.stderr.write(`${message}\n`);
|
|
46
47
|
process.exit(1);
|
|
47
48
|
}
|
|
48
|
-
|
|
49
|
-
|
|
49
|
+
/**
|
|
50
|
+
* The pedagogy is one SKILL.md plus `references/*.md`, and Claude Code reads
|
|
51
|
+
* those on demand -- SKILL.md says "read references/grading.md before you
|
|
52
|
+
* grade" and the model opens the file.
|
|
53
|
+
*
|
|
54
|
+
* A Cursor rules file has no such mechanism. It is one document with
|
|
55
|
+
* `alwaysApply: true`, so a pointer to a sibling path is a pointer to nothing.
|
|
56
|
+
* The references are therefore inlined here, and the preamble says so: without
|
|
57
|
+
* that, the split would have quietly cut four fifths of the pedagogy out of
|
|
58
|
+
* Cursor while every test still passed.
|
|
59
|
+
*
|
|
60
|
+
* Alphabetical, deliberately. In an always-apply document the whole thing is in
|
|
61
|
+
* context at once so order carries no meaning, and a hand-maintained order is
|
|
62
|
+
* one more place a new reference file gets forgotten.
|
|
63
|
+
*/
|
|
64
|
+
const FRONTMATTER = /^---\n[\s\S]*?\n---\n/;
|
|
65
|
+
function tutorSections() {
|
|
66
|
+
const tutorDir = path.join(moduleDir, 'assets', 'tutor');
|
|
67
|
+
const skillPath = path.join(tutorDir, 'SKILL.md');
|
|
50
68
|
if (!fs.existsSync(skillPath)) {
|
|
51
69
|
fail('The bundled tutor skill is missing. Run `npm run build` in the mcp/ directory.');
|
|
52
70
|
}
|
|
53
|
-
const skill = fs.readFileSync(skillPath, 'utf8').replace(
|
|
71
|
+
const skill = fs.readFileSync(skillPath, 'utf8').replace(FRONTMATTER, '');
|
|
72
|
+
const refsDir = path.join(tutorDir, 'references');
|
|
73
|
+
// Which references are required is not a list kept here -- it is whatever
|
|
74
|
+
// SKILL.md tells the model to read. Anything it names has to be inlined, or
|
|
75
|
+
// the preamble below is a lie: it promises the material is further down the
|
|
76
|
+
// document, so a reader who cannot find it hunts instead of falling back.
|
|
77
|
+
//
|
|
78
|
+
// Hence a hard failure rather than a warning. `copy-assets.mjs` only warns
|
|
79
|
+
// when it cannot bundle the skill, so a half-copied directory is reachable,
|
|
80
|
+
// and a rules file carrying the dispatch logic and none of the craft is worse
|
|
81
|
+
// than no rules file at all -- the split's whole failure mode, re-created at
|
|
82
|
+
// the last step.
|
|
83
|
+
const cited = [...skill.matchAll(/references\/([a-z0-9-]+\.md)/g)].flatMap((m) => (m[1] ? [m[1]] : []));
|
|
84
|
+
const required = [...new Set(cited)];
|
|
85
|
+
const missing = required.filter((name) => !fs.existsSync(path.join(refsDir, name)));
|
|
86
|
+
if (missing.length > 0) {
|
|
87
|
+
fail(`The tutor skill's reference files are missing: ${missing.join(', ')}.\n` +
|
|
88
|
+
'Run `npm run build` in the mcp/ directory.');
|
|
89
|
+
}
|
|
90
|
+
const sections = [skill.trim()];
|
|
91
|
+
// Read from the directory rather than from `required`, so a reference that
|
|
92
|
+
// ships without being pointed at is still inlined. Frontmatter stripped from
|
|
93
|
+
// each: a stray YAML block mid-document renders as content in a rules file,
|
|
94
|
+
// and the natural instinct for a file inside a skill directory is to give it
|
|
95
|
+
// some.
|
|
96
|
+
if (fs.existsSync(refsDir)) {
|
|
97
|
+
for (const name of fs.readdirSync(refsDir).sort()) {
|
|
98
|
+
if (!name.endsWith('.md'))
|
|
99
|
+
continue;
|
|
100
|
+
const body = fs.readFileSync(path.join(refsDir, name), 'utf8').replace(FRONTMATTER, '');
|
|
101
|
+
sections.push(body.trim());
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
return sections;
|
|
105
|
+
}
|
|
106
|
+
function exportRules(args) {
|
|
54
107
|
const rules = `---
|
|
55
108
|
description: Eklavya tutor — teach the concepts behind the code being written
|
|
56
109
|
alwaysApply: true
|
|
57
110
|
---
|
|
58
111
|
|
|
59
112
|
<!-- Generated by \`eklavya export-rules\`. Do not edit by hand; edit the plugin's
|
|
60
|
-
skills/tutor/SKILL.md and regenerate, so Claude Code and
|
|
113
|
+
skills/tutor/SKILL.md and its references/ and regenerate, so Claude Code and
|
|
114
|
+
Cursor never drift. -->
|
|
61
115
|
|
|
62
116
|
The Eklavya MCP server is available in this editor. Its tools are the source of
|
|
63
117
|
truth for what this developer already knows.
|
|
64
118
|
|
|
65
|
-
|
|
119
|
+
The first section below is the skill itself; the ones after it are its
|
|
120
|
+
reference files, inlined, because this editor has no way to open one on demand.
|
|
121
|
+
Read them as part of the whole: wherever the first section says to read
|
|
122
|
+
\`references/something.md\`, that material is further down this same file.
|
|
123
|
+
|
|
124
|
+
${tutorSections().join('\n\n')}
|
|
66
125
|
`;
|
|
67
126
|
const outIndex = args.indexOf('--out');
|
|
68
127
|
if (outIndex >= 0) {
|
|
@@ -211,17 +270,7 @@ async function statuslineCommand(argv) {
|
|
|
211
270
|
// PowerShell block that swallows the pipe, so `end` never fires and a naive
|
|
212
271
|
// read waits forever -- which in a status bar means every refresh blocks.
|
|
213
272
|
// `unref` keeps the timer off the normal path, where `end` arrives first.
|
|
214
|
-
const raw = await
|
|
215
|
-
if (process.stdin.isTTY)
|
|
216
|
-
return resolve('');
|
|
217
|
-
let buf = '';
|
|
218
|
-
const done = () => resolve(buf);
|
|
219
|
-
process.stdin.setEncoding('utf8');
|
|
220
|
-
process.stdin.on('data', (c) => (buf += c));
|
|
221
|
-
process.stdin.on('end', done);
|
|
222
|
-
process.stdin.on('error', done);
|
|
223
|
-
setTimeout(done, 250).unref();
|
|
224
|
-
});
|
|
273
|
+
const raw = await readStdinBounded(STATUSLINE_STDIN);
|
|
225
274
|
let cwd = process.cwd();
|
|
226
275
|
// Parsed in its own try: input we cannot read is a reason to fall back to
|
|
227
276
|
// the working directory, not a reason to show the developer nothing. The
|
|
@@ -230,7 +279,7 @@ async function statuslineCommand(argv) {
|
|
|
230
279
|
if (raw.trim()) {
|
|
231
280
|
// Strip a BOM: some Windows shells prepend one, and JSON.parse throws
|
|
232
281
|
// on input that looks perfectly well-formed.
|
|
233
|
-
const parsed = JSON.parse(raw
|
|
282
|
+
const parsed = JSON.parse(stripBom(raw));
|
|
234
283
|
const input = (parsed ?? {});
|
|
235
284
|
cwd = input.workspace?.current_dir ?? input.cwd ?? cwd;
|
|
236
285
|
}
|
package/dist/cli.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"cli.js","sourceRoot":"","sources":["../src/cli.ts"],"names":[],"mappings":";AACA;;;;GAIG;AACH,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AACzC,OAAO,EAAE,MAAM,EAAE,MAAM,SAAS,CAAC;AACjC,OAAO,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AACjD,OAAO,EAAE,UAAU,EAAE,eAAe,EAAE,gBAAgB,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAC5F,OAAO,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAC3C,OAAO,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAC7C,OAAO,EAAE,WAAW,EAAc,MAAM,UAAU,CAAC;AACnD,OAAO,QAAQ,MAAM,gBAAgB,CAAC;AACtC,OAAO,EAAE,cAAc,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAC/D,OAAO,EAAE,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,cAAc,CAAC;AAE1D,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;AAE/D,MAAM,KAAK,GAAG;;;;;;;;;;;;;;;;;;;;;;;CAuBb,CAAC;AAEF,SAAS,IAAI,CAAC,OAAe;IAC3B,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,OAAO,IAAI,CAAC,CAAC;IACrC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC;AAED,SAAS,WAAW,CAAC,IAAc;IACjC,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,QAAQ,EAAE,gBAAgB,CAAC,CAAC;IACnE,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE,CAAC;QAC9B,IAAI,CAAC,gFAAgF,CAAC,CAAC;IACzF,CAAC;IAED,MAAM,KAAK,GAAG,EAAE,CAAC,YAAY,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC,OAAO,CAAC,uBAAuB,EAAE,EAAE,CAAC,CAAC;IAEtF,MAAM,KAAK,GAAG;;;;;;;;;;;EAWd,KAAK,CAAC,IAAI,EAAE;CACb,CAAC;IAEA,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;IACvC,IAAI,QAAQ,IAAI,CAAC,EAAE,CAAC;QAClB,MAAM,GAAG,GAAG,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC,CAAC;QAC/B,IAAI,CAAC,GAAG;YAAE,IAAI,CAAC,0BAA0B,CAAC,CAAC;QAC3C,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QACnE,EAAE,CAAC,aAAa,CAAC,GAAG,EAAE,KAAK,EAAE,MAAM,CAAC,CAAC;QACrC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,SAAS,GAAG,IAAI,CAAC,CAAC;QACvC,OAAO;IACT,CAAC;IAED,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;AAC9B,CAAC;AAED,SAAS,aAAa,CAAC,IAAc;IACnC,MAAM,CAAC,MAAM,EAAE,GAAG,EAAE,KAAK,CAAC,GAAG,IAAI,CAAC;IAClC,MAAM,SAAS,GAAG,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;IAC1C,MAAM,QAAQ,GAAG,UAAU,EAAE,CAAC;IAE9B,IAAI,CAAC,MAAM,IAAI,MAAM,KAAK,KAAK,EAAE,CAAC;QAChC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC;QACtE,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,aAAa,QAAQ,CAAC,UAAU,IAAI,CAAC,CAAC;QAC3D,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,WAAW,QAAQ,CAAC,QAAQ,IAAI,QAAQ,IAAI,CAAC,CAAC;QACnE,OAAO;IACT,CAAC;IAED,IAAI,MAAM,KAAK,KAAK;QAAE,IAAI,CAAC,0BAA0B,MAAM,IAAI,CAAC,CAAC;IACjE,IAAI,CAAC,GAAG,IAAI,KAAK,KAAK,SAAS;QAAE,IAAI,CAAC,yCAAyC,CAAC,CAAC;IACjF,IAAI,CAAC,CAAC,GAAG,IAAI,cAAc,CAAC,EAAE,CAAC;QAC7B,IAAI,CAAC,oBAAoB,GAAG,aAAa,MAAM,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACrF,CAAC;IAED,4EAA4E;IAC5E,gEAAgE;IAChE,MAAM,UAAU,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;IAC3C,MAAM,KAAK,GAAG,UAAU,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,UAAU,GAAG,CAAC,CAAC,CAAC;IACnE,IAAI,UAAU,KAAK,CAAC,CAAC,IAAI,CAAC,KAAK;QAAE,IAAI,CAAC,wBAAwB,CAAC,CAAC;IAChE,IAAI,KAAK,KAAK,SAAS,IAAI,GAAG,KAAK,OAAO,IAAI,GAAG,KAAK,aAAa,EAAE,CAAC;QACpE,IAAI,CAAC,0CAA0C,CAAC,CAAC;IACnD,CAAC;IACD,IAAI,GAAG,KAAK,OAAO,IAAI,KAAK,KAAK,OAAO,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;QAChE,IAAI,CAAC,6EAA6E,CAAC,CAAC;IACtF,CAAC;IAED,gFAAgF;IAChF,mEAAmE;IACnE,MAAM,UAAU,GAAG,GAAG,KAAK,aAAa,CAAC;IACzC,IAAI,MAAM,GAAY,KAAK,CAAC;IAC5B,IAAI,UAAU;QAAE,MAAM,GAAG,KAAK,CAAC;SAC1B,IAAI,KAAK,KAAK,MAAM,IAAI,KAAK,KAAK,OAAO;QAAE,MAAM,GAAG,KAAK,KAAK,MAAM,CAAC;SACrE,IAAI,KAAK,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QAAE,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;IAE9E,MAAM,KAAK,GAA4B,EAAE,CAAC,GAAG,CAAC,EAAE,MAAM,EAAE,CAAC;IACzD,IAAI,KAAK,KAAK,SAAS,IAAI,GAAG,KAAK,OAAO;QAAE,KAAK,CAAC,WAAW,GAAG,KAAK,CAAC;IAEtE,IAAI,MAAc,CAAC;IACnB,IAAI,SAAS,EAAE,CAAC;QACd,IAAI,CAAC,QAAQ,CAAC,QAAQ;YAAE,IAAI,CAAC,0EAA0E,CAAC,CAAC;QACzG,MAAM,GAAG,QAAQ,CAAC,QAAQ,IAAI,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,QAAQ,EAAE,gBAAgB,CAAC,CAAC;IAC/E,CAAC;SAAM,CAAC;QACN,MAAM,GAAG,QAAQ,CAAC,UAAU,CAAC;IAC/B,CAAC;IAED,eAAe,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;IAC/B,KAAK,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;QAC3C,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,MAAM,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS,MAAM,IAAI,CAAC,CAAC;IACvE,CAAC;AACH,CAAC;AAED,SAAS,MAAM;IACb,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC;IACtB,MAAM,QAAQ,GAAG,UAAU,EAAE,CAAC;IAC9B,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,IAAI,EAAE,GAAG,IAAI,CAAC;IAEd,KAAK,CAAC,IAAI,CAAC,aAAa,WAAW,EAAE,EAAE,CAAC,CAAC;IAEzC,6EAA6E;IAC7E,8EAA8E;IAC9E,yCAAyC;IACzC,MAAM,MAAM,GAAG,MAAM,EAAE,CAAC;IACxB,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,IAAI,CAAC,KAAK,CAAC,EAAE;YAAE,EAAE,GAAG,KAAK,CAAC;QAC1B,KAAK,CAAC,IAAI,CAAC,GAAG,GAAG,KAAK,CAAC,IAAI,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,WAAW,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC;IAC5F,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,aAAa,IAAI,GAAG,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,sBAAsB,EAAE,CAAC,CAAC;IAEpF,IAAI,CAAC;QACH,MAAM,EAAE,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC;QACxB,MAAM,QAAQ,GAAI,EAAE,CAAC,OAAO,CAAC,iCAAiC,CAAC,CAAC,GAAG,EAAoB,CAAC,CAAC,CAAC;QAC1F,MAAM,QAAQ,GAAI,EAAE,CAAC,OAAO,CAAC,iCAAiC,CAAC,CAAC,GAAG,EAAoB,CAAC,CAAC,CAAC;QAC1F,MAAM,KAAK,GACT,EAAE,CAAC,OAAO,CAAC,iEAAiE,CAAC,CAAC,GAAG,EAClF,CAAC,CAAC,CAAC;QACJ,KAAK,CAAC,IAAI,CAAC,aAAa,QAAQ,EAAE,CAAC,CAAC;QACpC,KAAK,CAAC,IAAI,CAAC,aAAa,QAAQ,EAAE,CAAC,CAAC;QACpC,KAAK,CAAC,IAAI,CAAC,aAAa,KAAK,EAAE,CAAC,CAAC;QACjC,KAAK,CAAC,IAAI,CAAC,aAAa,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,cAAc,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC;QAC/E,4EAA4E;QAC5E,iEAAiE;QACjE,MAAM,QAAQ,GAAG,aAAa,CAAC,EAAE,EAAE,QAAQ,CAAC,MAAM,EAAE,QAAQ,CAAC,QAAQ,CAAC,CAAC;QACvE,KAAK,CAAC,IAAI,CACR,aAAa,QAAQ,CAAC,KAAK,GACzB,QAAQ,CAAC,MAAM;YACb,CAAC,CAAC,sCAAsC;YACxC,CAAC,CAAC,KAAK,QAAQ,CAAC,MAAM,CAAC,MAAM,IAAI,QAAQ,CAAC,MAAM,CAAC,OAAO,uBAAuB,QAAQ,CAAC,IAAI,GAChG,EAAE,CACH,CAAC;QACF,EAAE,CAAC,KAAK,EAAE,CAAC;IACb,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,EAAE,GAAG,KAAK,CAAC;QACX,KAAK,CAAC,IAAI,CAAC,mBAAmB,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IACpF,CAAC;IAED,MAAM,QAAQ,GAAG,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC,mBAAmB,CAAC,CAAC,CAAC,EAAE,CAAC;IAC9D,KAAK,CAAC,IAAI,CAAC,aAAa,QAAQ,CAAC,MAAM,CAAC,IAAI,GAAG,QAAQ,EAAE,CAAC,CAAC;IAC3D,KAAK,CAAC,IAAI,CACR,aAAa,QAAQ,CAAC,MAAM,CAAC,KAAK,GAChC,QAAQ,CAAC,MAAM,CAAC,KAAK,KAAK,OAAO,CAAC,CAAC,CAAC,KAAK,QAAQ,CAAC,MAAM,CAAC,WAAW,IAAI,cAAc,GAAG,CAAC,CAAC,CAAC,EAC9F,GAAG,QAAQ,EAAE,CACd,CAAC;IACF,KAAK,CAAC,IAAI,CACR,aAAa,QAAQ,CAAC,MAAM,CAAC,OAAO,GAClC,QAAQ,CAAC,MAAM,CAAC,OAAO,KAAK,aAAa;QACvC,CAAC,CAAC,gCAAgC,QAAQ,CAAC,MAAM,CAAC,+BAA+B,UAAU;QAC3F,CAAC,CAAC,yCACN,GAAG,QAAQ,EAAE,CACd,CAAC;IACF,IAAI,QAAQ,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAClC,KAAK,CAAC,IAAI,CAAC,uBAAuB,QAAQ,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACrE,CAAC;IAED,4EAA4E;IAC5E,2EAA2E;IAC3E,6CAA6C;IAC7C,IAAI,CAAC,EAAE,EAAE,CAAC;QACR,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACf,KAAK,CAAC,IAAI,CAAC,2CAA2C,CAAC,CAAC;IAC1D,CAAC;IAED,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC9C,IAAI,CAAC,EAAE;QAAE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAC3B,CAAC;AAED;;;;;;;;;;;;GAYG;AACH,KAAK,UAAU,iBAAiB,CAAC,IAAc;IAC7C,IAAI,CAAC;QACH,yEAAyE;QACzE,qEAAqE;QACrE,EAAE;QACF,oEAAoE;QACpE,4EAA4E;QAC5E,0EAA0E;QAC1E,0EAA0E;QAC1E,MAAM,GAAG,GAAG,MAAM,IAAI,OAAO,CAAS,CAAC,OAAO,EAAE,EAAE;YAChD,IAAI,OAAO,CAAC,KAAK,CAAC,KAAK;gBAAE,OAAO,OAAO,CAAC,EAAE,CAAC,CAAC;YAC5C,IAAI,GAAG,GAAG,EAAE,CAAC;YACb,MAAM,IAAI,GAAG,GAAS,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;YACtC,OAAO,CAAC,KAAK,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;YAClC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,CAAS,EAAE,EAAE,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;YACpD,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;YAC9B,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC;YAChC,UAAU,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC;QAChC,CAAC,CAAC,CAAC;QAEH,IAAI,GAAG,GAAG,OAAO,CAAC,GAAG,EAAE,CAAC;QACxB,0EAA0E;QAC1E,yEAAyE;QACzE,uEAAuE;QACvE,IAAI,CAAC;YACH,IAAI,GAAG,CAAC,IAAI,EAAE,EAAE,CAAC;gBACf,sEAAsE;gBACtE,6CAA6C;gBAC7C,MAAM,MAAM,GAAY,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,CAAC;gBAC1D,MAAM,KAAK,GAAG,CAAC,MAAM,IAAI,EAAE,CAA2D,CAAC;gBACvF,GAAG,GAAG,KAAK,CAAC,SAAS,EAAE,WAAW,IAAI,KAAK,CAAC,GAAG,IAAI,GAAG,CAAC;YACzD,CAAC;QACH,CAAC;QAAC,MAAM,CAAC;YACP,4CAA4C;QAC9C,CAAC;QAED,MAAM,QAAQ,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC;QACjC,MAAM,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAC,UAAU,KAAK,MAAM,CAAC;QAErD,IAAI,KAAK,GAAU,MAAM,CAAC,CAAC,CAAE,QAAQ,CAAC,MAAM,CAAC,UAAoB,CAAC,CAAC,CAAC,WAAW,CAAC;QAChF,IAAI,CAAC,MAAM,IAAI,EAAE,CAAC,UAAU,CAAC,MAAM,EAAE,CAAC,EAAE,CAAC;YACvC,0EAA0E;YAC1E,0EAA0E;YAC1E,0CAA0C;YAC1C,MAAM,EAAE,GAAG,IAAI,QAAQ,CAAC,MAAM,EAAE,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC;YACtD,IAAI,CAAC;gBACH,KAAK,GAAG,aAAa,CAAC,EAAE,EAAE,QAAQ,CAAC,MAAM,EAAE,QAAQ,CAAC,QAAQ,CAAC,CAAC,KAAK,CAAC;YACtE,CAAC;oBAAS,CAAC;gBACT,EAAE,CAAC,KAAK,EAAE,CAAC;YACb,CAAC;QACH,CAAC;QAED,yEAAyE;QACzE,4CAA4C;QAC5C,MAAM,KAAK,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC;QACpE,MAAM,IAAI,GAAG,UAAU,CAAC,EAAE,MAAM,EAAE,QAAQ,CAAC,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC;QAC3E,IAAI,IAAI;YAAE,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,IAAI,IAAI,CAAC,CAAC;IAC9C,CAAC;IAAC,MAAM,CAAC;QACP,oDAAoD;IACtD,CAAC;AACH,CAAC;AAED,SAAS,gBAAgB,CAAC,IAAc;IACtC,MAAM,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;IACjC,MAAM,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;IACxD,IAAI,IAAI,KAAK,SAAS,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC;QAClD,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,4CAA4C,CAAC,CAAC;QACnE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IACD,wEAAwE;IACxE,8EAA8E;IAC9E,wCAAwC;IACxC,MAAM,IAAI,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC;IAEzC,cAAc,CAAC,MAAM,EAAE,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,CACrC,CAAC,EAAE,GAAG,EAAE,EAAE,EAAE;QACV,OAAO,CAAC,MAAM,CAAC,KAAK,CAClB,wBAAwB,GAAG,aAAa,MAAM,EAAE,4BAA4B,CAC7E,CAAC;QACF,IAAI,IAAI,EAAE,CAAC;YACT,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,+BAA+B,CAAC,CAAC;YACtD,aAAa,CAAC,GAAG,CAAC,CAAC;QACrB,CAAC;IACH,CAAC,EACD,CAAC,GAAU,EAAE,EAAE;QACb,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,sBAAsB,GAAG,CAAC,OAAO,IAAI,CAAC,CAAC;QAC5D,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC,CACF,CAAC;AACJ,CAAC;AAED,SAAS,IAAI;IACX,MAAM,CAAC,OAAO,EAAE,GAAG,IAAI,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IAEjD,QAAQ,OAAO,EAAE,CAAC;QAChB,KAAK,OAAO;YACV,0EAA0E;YAC1E,yEAAyE;YACzE,0BAA0B;YAC1B,KAAK,MAAM,CAAC,aAAa,CAAC,CAAC,KAAK,CAAC,CAAC,GAAY,EAAE,EAAE;gBAChD,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,kBAAkB,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;gBAC7F,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAClB,CAAC,CAAC,CAAC;YACH,OAAO;QACT,KAAK,SAAS;YACZ,OAAO,OAAO,CAAC,IAAI,CAAC,CAAC;QACvB,KAAK,WAAW;YACd,OAAO,SAAS,CAAC,IAAI,CAAC,CAAC;QACzB,KAAK,cAAc;YACjB,OAAO,WAAW,CAAC,IAAI,CAAC,CAAC;QAC3B,KAAK,QAAQ;YACX,OAAO,aAAa,CAAC,IAAI,CAAC,CAAC;QAC7B,KAAK,YAAY;YACf,KAAK,iBAAiB,CAAC,IAAI,CAAC,CAAC;YAC7B,OAAO;QACT,KAAK,WAAW;YACd,OAAO,gBAAgB,CAAC,IAAI,CAAC,CAAC;QAChC,KAAK,QAAQ;YACX,OAAO,MAAM,EAAE,CAAC;QAClB,KAAK,SAAS;YACZ,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,MAAM,EAAE,IAAI,CAAC,CAAC;YACtC,OAAO;QACT,KAAK,SAAS,CAAC;QACf,KAAK,IAAI,CAAC;QACV,KAAK,QAAQ;YACX,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;YAC5B,OAAO;QACT;YACE,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;YAC5B,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACpB,CAAC;AACH,CAAC;AAED,IAAI,EAAE,CAAC"}
|
|
1
|
+
{"version":3,"file":"cli.js","sourceRoot":"","sources":["../src/cli.ts"],"names":[],"mappings":";AACA;;;;GAIG;AACH,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AACzC,OAAO,EAAE,MAAM,EAAE,MAAM,SAAS,CAAC;AACjC,OAAO,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AACjD,OAAO,EAAE,gBAAgB,EAAE,QAAQ,EAAE,gBAAgB,EAAE,MAAM,YAAY,CAAC;AAC1E,OAAO,EAAE,UAAU,EAAE,eAAe,EAAE,gBAAgB,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAC5F,OAAO,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAC3C,OAAO,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAC7C,OAAO,EAAE,WAAW,EAAc,MAAM,UAAU,CAAC;AACnD,OAAO,QAAQ,MAAM,gBAAgB,CAAC;AACtC,OAAO,EAAE,cAAc,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAC/D,OAAO,EAAE,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,cAAc,CAAC;AAE1D,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;AAE/D,MAAM,KAAK,GAAG;;;;;;;;;;;;;;;;;;;;;;;CAuBb,CAAC;AAEF,SAAS,IAAI,CAAC,OAAe;IAC3B,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,OAAO,IAAI,CAAC,CAAC;IACrC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC;AAED;;;;;;;;;;;;;;GAcG;AACH,MAAM,WAAW,GAAG,uBAAuB,CAAC;AAE5C,SAAS,aAAa;IACpB,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC;IACzD,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;IAClD,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE,CAAC;QAC9B,IAAI,CAAC,gFAAgF,CAAC,CAAC;IACzF,CAAC;IAED,MAAM,KAAK,GAAG,EAAE,CAAC,YAAY,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC,OAAO,CAAC,WAAW,EAAE,EAAE,CAAC,CAAC;IAC1E,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,YAAY,CAAC,CAAC;IAElD,0EAA0E;IAC1E,4EAA4E;IAC5E,4EAA4E;IAC5E,0EAA0E;IAC1E,EAAE;IACF,2EAA2E;IAC3E,4EAA4E;IAC5E,8EAA8E;IAC9E,6EAA6E;IAC7E,iBAAiB;IACjB,MAAM,KAAK,GAAG,CAAC,GAAG,KAAK,CAAC,QAAQ,CAAC,+BAA+B,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IACxG,MAAM,QAAQ,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC;IACrC,MAAM,OAAO,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC;IACpF,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACvB,IAAI,CACF,kDAAkD,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK;YACvE,4CAA4C,CAC/C,CAAC;IACJ,CAAC;IAED,MAAM,QAAQ,GAAG,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC;IAChC,2EAA2E;IAC3E,6EAA6E;IAC7E,4EAA4E;IAC5E,6EAA6E;IAC7E,QAAQ;IACR,IAAI,EAAE,CAAC,UAAU,CAAC,OAAO,CAAC,EAAE,CAAC;QAC3B,KAAK,MAAM,IAAI,IAAI,EAAE,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC;YAClD,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC;gBAAE,SAAS;YACpC,MAAM,IAAI,GAAG,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,EAAE,MAAM,CAAC,CAAC,OAAO,CAAC,WAAW,EAAE,EAAE,CAAC,CAAC;YACxF,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;QAC7B,CAAC;IACH,CAAC;IAED,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,SAAS,WAAW,CAAC,IAAc;IACjC,MAAM,KAAK,GAAG;;;;;;;;;;;;;;;;;EAiBd,aAAa,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC;CAC7B,CAAC;IAEA,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;IACvC,IAAI,QAAQ,IAAI,CAAC,EAAE,CAAC;QAClB,MAAM,GAAG,GAAG,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC,CAAC;QAC/B,IAAI,CAAC,GAAG;YAAE,IAAI,CAAC,0BAA0B,CAAC,CAAC;QAC3C,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QACnE,EAAE,CAAC,aAAa,CAAC,GAAG,EAAE,KAAK,EAAE,MAAM,CAAC,CAAC;QACrC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,SAAS,GAAG,IAAI,CAAC,CAAC;QACvC,OAAO;IACT,CAAC;IAED,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;AAC9B,CAAC;AAED,SAAS,aAAa,CAAC,IAAc;IACnC,MAAM,CAAC,MAAM,EAAE,GAAG,EAAE,KAAK,CAAC,GAAG,IAAI,CAAC;IAClC,MAAM,SAAS,GAAG,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;IAC1C,MAAM,QAAQ,GAAG,UAAU,EAAE,CAAC;IAE9B,IAAI,CAAC,MAAM,IAAI,MAAM,KAAK,KAAK,EAAE,CAAC;QAChC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC;QACtE,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,aAAa,QAAQ,CAAC,UAAU,IAAI,CAAC,CAAC;QAC3D,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,WAAW,QAAQ,CAAC,QAAQ,IAAI,QAAQ,IAAI,CAAC,CAAC;QACnE,OAAO;IACT,CAAC;IAED,IAAI,MAAM,KAAK,KAAK;QAAE,IAAI,CAAC,0BAA0B,MAAM,IAAI,CAAC,CAAC;IACjE,IAAI,CAAC,GAAG,IAAI,KAAK,KAAK,SAAS;QAAE,IAAI,CAAC,yCAAyC,CAAC,CAAC;IACjF,IAAI,CAAC,CAAC,GAAG,IAAI,cAAc,CAAC,EAAE,CAAC;QAC7B,IAAI,CAAC,oBAAoB,GAAG,aAAa,MAAM,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACrF,CAAC;IAED,4EAA4E;IAC5E,gEAAgE;IAChE,MAAM,UAAU,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;IAC3C,MAAM,KAAK,GAAG,UAAU,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,UAAU,GAAG,CAAC,CAAC,CAAC;IACnE,IAAI,UAAU,KAAK,CAAC,CAAC,IAAI,CAAC,KAAK;QAAE,IAAI,CAAC,wBAAwB,CAAC,CAAC;IAChE,IAAI,KAAK,KAAK,SAAS,IAAI,GAAG,KAAK,OAAO,IAAI,GAAG,KAAK,aAAa,EAAE,CAAC;QACpE,IAAI,CAAC,0CAA0C,CAAC,CAAC;IACnD,CAAC;IACD,IAAI,GAAG,KAAK,OAAO,IAAI,KAAK,KAAK,OAAO,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;QAChE,IAAI,CAAC,6EAA6E,CAAC,CAAC;IACtF,CAAC;IAED,gFAAgF;IAChF,mEAAmE;IACnE,MAAM,UAAU,GAAG,GAAG,KAAK,aAAa,CAAC;IACzC,IAAI,MAAM,GAAY,KAAK,CAAC;IAC5B,IAAI,UAAU;QAAE,MAAM,GAAG,KAAK,CAAC;SAC1B,IAAI,KAAK,KAAK,MAAM,IAAI,KAAK,KAAK,OAAO;QAAE,MAAM,GAAG,KAAK,KAAK,MAAM,CAAC;SACrE,IAAI,KAAK,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QAAE,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;IAE9E,MAAM,KAAK,GAA4B,EAAE,CAAC,GAAG,CAAC,EAAE,MAAM,EAAE,CAAC;IACzD,IAAI,KAAK,KAAK,SAAS,IAAI,GAAG,KAAK,OAAO;QAAE,KAAK,CAAC,WAAW,GAAG,KAAK,CAAC;IAEtE,IAAI,MAAc,CAAC;IACnB,IAAI,SAAS,EAAE,CAAC;QACd,IAAI,CAAC,QAAQ,CAAC,QAAQ;YAAE,IAAI,CAAC,0EAA0E,CAAC,CAAC;QACzG,MAAM,GAAG,QAAQ,CAAC,QAAQ,IAAI,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,QAAQ,EAAE,gBAAgB,CAAC,CAAC;IAC/E,CAAC;SAAM,CAAC;QACN,MAAM,GAAG,QAAQ,CAAC,UAAU,CAAC;IAC/B,CAAC;IAED,eAAe,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;IAC/B,KAAK,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;QAC3C,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,MAAM,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS,MAAM,IAAI,CAAC,CAAC;IACvE,CAAC;AACH,CAAC;AAED,SAAS,MAAM;IACb,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC;IACtB,MAAM,QAAQ,GAAG,UAAU,EAAE,CAAC;IAC9B,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,IAAI,EAAE,GAAG,IAAI,CAAC;IAEd,KAAK,CAAC,IAAI,CAAC,aAAa,WAAW,EAAE,EAAE,CAAC,CAAC;IAEzC,6EAA6E;IAC7E,8EAA8E;IAC9E,yCAAyC;IACzC,MAAM,MAAM,GAAG,MAAM,EAAE,CAAC;IACxB,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,IAAI,CAAC,KAAK,CAAC,EAAE;YAAE,EAAE,GAAG,KAAK,CAAC;QAC1B,KAAK,CAAC,IAAI,CAAC,GAAG,GAAG,KAAK,CAAC,IAAI,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,WAAW,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC;IAC5F,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,aAAa,IAAI,GAAG,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,sBAAsB,EAAE,CAAC,CAAC;IAEpF,IAAI,CAAC;QACH,MAAM,EAAE,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC;QACxB,MAAM,QAAQ,GAAI,EAAE,CAAC,OAAO,CAAC,iCAAiC,CAAC,CAAC,GAAG,EAAoB,CAAC,CAAC,CAAC;QAC1F,MAAM,QAAQ,GAAI,EAAE,CAAC,OAAO,CAAC,iCAAiC,CAAC,CAAC,GAAG,EAAoB,CAAC,CAAC,CAAC;QAC1F,MAAM,KAAK,GACT,EAAE,CAAC,OAAO,CAAC,iEAAiE,CAAC,CAAC,GAAG,EAClF,CAAC,CAAC,CAAC;QACJ,KAAK,CAAC,IAAI,CAAC,aAAa,QAAQ,EAAE,CAAC,CAAC;QACpC,KAAK,CAAC,IAAI,CAAC,aAAa,QAAQ,EAAE,CAAC,CAAC;QACpC,KAAK,CAAC,IAAI,CAAC,aAAa,KAAK,EAAE,CAAC,CAAC;QACjC,KAAK,CAAC,IAAI,CAAC,aAAa,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,cAAc,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC;QAC/E,4EAA4E;QAC5E,iEAAiE;QACjE,MAAM,QAAQ,GAAG,aAAa,CAAC,EAAE,EAAE,QAAQ,CAAC,MAAM,EAAE,QAAQ,CAAC,QAAQ,CAAC,CAAC;QACvE,KAAK,CAAC,IAAI,CACR,aAAa,QAAQ,CAAC,KAAK,GACzB,QAAQ,CAAC,MAAM;YACb,CAAC,CAAC,sCAAsC;YACxC,CAAC,CAAC,KAAK,QAAQ,CAAC,MAAM,CAAC,MAAM,IAAI,QAAQ,CAAC,MAAM,CAAC,OAAO,uBAAuB,QAAQ,CAAC,IAAI,GAChG,EAAE,CACH,CAAC;QACF,EAAE,CAAC,KAAK,EAAE,CAAC;IACb,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,EAAE,GAAG,KAAK,CAAC;QACX,KAAK,CAAC,IAAI,CAAC,mBAAmB,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IACpF,CAAC;IAED,MAAM,QAAQ,GAAG,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC,mBAAmB,CAAC,CAAC,CAAC,EAAE,CAAC;IAC9D,KAAK,CAAC,IAAI,CAAC,aAAa,QAAQ,CAAC,MAAM,CAAC,IAAI,GAAG,QAAQ,EAAE,CAAC,CAAC;IAC3D,KAAK,CAAC,IAAI,CACR,aAAa,QAAQ,CAAC,MAAM,CAAC,KAAK,GAChC,QAAQ,CAAC,MAAM,CAAC,KAAK,KAAK,OAAO,CAAC,CAAC,CAAC,KAAK,QAAQ,CAAC,MAAM,CAAC,WAAW,IAAI,cAAc,GAAG,CAAC,CAAC,CAAC,EAC9F,GAAG,QAAQ,EAAE,CACd,CAAC;IACF,KAAK,CAAC,IAAI,CACR,aAAa,QAAQ,CAAC,MAAM,CAAC,OAAO,GAClC,QAAQ,CAAC,MAAM,CAAC,OAAO,KAAK,aAAa;QACvC,CAAC,CAAC,gCAAgC,QAAQ,CAAC,MAAM,CAAC,+BAA+B,UAAU;QAC3F,CAAC,CAAC,yCACN,GAAG,QAAQ,EAAE,CACd,CAAC;IACF,IAAI,QAAQ,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAClC,KAAK,CAAC,IAAI,CAAC,uBAAuB,QAAQ,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACrE,CAAC;IAED,4EAA4E;IAC5E,2EAA2E;IAC3E,6CAA6C;IAC7C,IAAI,CAAC,EAAE,EAAE,CAAC;QACR,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACf,KAAK,CAAC,IAAI,CAAC,2CAA2C,CAAC,CAAC;IAC1D,CAAC;IAED,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC9C,IAAI,CAAC,EAAE;QAAE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAC3B,CAAC;AAED;;;;;;;;;;;;GAYG;AACH,KAAK,UAAU,iBAAiB,CAAC,IAAc;IAC7C,IAAI,CAAC;QACH,yEAAyE;QACzE,qEAAqE;QACrE,EAAE;QACF,oEAAoE;QACpE,4EAA4E;QAC5E,0EAA0E;QAC1E,0EAA0E;QAC1E,MAAM,GAAG,GAAG,MAAM,gBAAgB,CAAC,gBAAgB,CAAC,CAAC;QAErD,IAAI,GAAG,GAAG,OAAO,CAAC,GAAG,EAAE,CAAC;QACxB,0EAA0E;QAC1E,yEAAyE;QACzE,uEAAuE;QACvE,IAAI,CAAC;YACH,IAAI,GAAG,CAAC,IAAI,EAAE,EAAE,CAAC;gBACf,sEAAsE;gBACtE,6CAA6C;gBAC7C,MAAM,MAAM,GAAY,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC;gBAClD,MAAM,KAAK,GAAG,CAAC,MAAM,IAAI,EAAE,CAA2D,CAAC;gBACvF,GAAG,GAAG,KAAK,CAAC,SAAS,EAAE,WAAW,IAAI,KAAK,CAAC,GAAG,IAAI,GAAG,CAAC;YACzD,CAAC;QACH,CAAC;QAAC,MAAM,CAAC;YACP,4CAA4C;QAC9C,CAAC;QAED,MAAM,QAAQ,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC;QACjC,MAAM,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAC,UAAU,KAAK,MAAM,CAAC;QAErD,IAAI,KAAK,GAAU,MAAM,CAAC,CAAC,CAAE,QAAQ,CAAC,MAAM,CAAC,UAAoB,CAAC,CAAC,CAAC,WAAW,CAAC;QAChF,IAAI,CAAC,MAAM,IAAI,EAAE,CAAC,UAAU,CAAC,MAAM,EAAE,CAAC,EAAE,CAAC;YACvC,0EAA0E;YAC1E,0EAA0E;YAC1E,0CAA0C;YAC1C,MAAM,EAAE,GAAG,IAAI,QAAQ,CAAC,MAAM,EAAE,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC;YACtD,IAAI,CAAC;gBACH,KAAK,GAAG,aAAa,CAAC,EAAE,EAAE,QAAQ,CAAC,MAAM,EAAE,QAAQ,CAAC,QAAQ,CAAC,CAAC,KAAK,CAAC;YACtE,CAAC;oBAAS,CAAC;gBACT,EAAE,CAAC,KAAK,EAAE,CAAC;YACb,CAAC;QACH,CAAC;QAED,yEAAyE;QACzE,4CAA4C;QAC5C,MAAM,KAAK,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC;QACpE,MAAM,IAAI,GAAG,UAAU,CAAC,EAAE,MAAM,EAAE,QAAQ,CAAC,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC;QAC3E,IAAI,IAAI;YAAE,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,IAAI,IAAI,CAAC,CAAC;IAC9C,CAAC;IAAC,MAAM,CAAC;QACP,oDAAoD;IACtD,CAAC;AACH,CAAC;AAED,SAAS,gBAAgB,CAAC,IAAc;IACtC,MAAM,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;IACjC,MAAM,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;IACxD,IAAI,IAAI,KAAK,SAAS,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC;QAClD,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,4CAA4C,CAAC,CAAC;QACnE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IACD,wEAAwE;IACxE,8EAA8E;IAC9E,wCAAwC;IACxC,MAAM,IAAI,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC;IAEzC,cAAc,CAAC,MAAM,EAAE,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,CACrC,CAAC,EAAE,GAAG,EAAE,EAAE,EAAE;QACV,OAAO,CAAC,MAAM,CAAC,KAAK,CAClB,wBAAwB,GAAG,aAAa,MAAM,EAAE,4BAA4B,CAC7E,CAAC;QACF,IAAI,IAAI,EAAE,CAAC;YACT,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,+BAA+B,CAAC,CAAC;YACtD,aAAa,CAAC,GAAG,CAAC,CAAC;QACrB,CAAC;IACH,CAAC,EACD,CAAC,GAAU,EAAE,EAAE;QACb,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,sBAAsB,GAAG,CAAC,OAAO,IAAI,CAAC,CAAC;QAC5D,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC,CACF,CAAC;AACJ,CAAC;AAED,SAAS,IAAI;IACX,MAAM,CAAC,OAAO,EAAE,GAAG,IAAI,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IAEjD,QAAQ,OAAO,EAAE,CAAC;QAChB,KAAK,OAAO;YACV,0EAA0E;YAC1E,yEAAyE;YACzE,0BAA0B;YAC1B,KAAK,MAAM,CAAC,aAAa,CAAC,CAAC,KAAK,CAAC,CAAC,GAAY,EAAE,EAAE;gBAChD,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,kBAAkB,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;gBAC7F,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAClB,CAAC,CAAC,CAAC;YACH,OAAO;QACT,KAAK,SAAS;YACZ,OAAO,OAAO,CAAC,IAAI,CAAC,CAAC;QACvB,KAAK,WAAW;YACd,OAAO,SAAS,CAAC,IAAI,CAAC,CAAC;QACzB,KAAK,cAAc;YACjB,OAAO,WAAW,CAAC,IAAI,CAAC,CAAC;QAC3B,KAAK,QAAQ;YACX,OAAO,aAAa,CAAC,IAAI,CAAC,CAAC;QAC7B,KAAK,YAAY;YACf,KAAK,iBAAiB,CAAC,IAAI,CAAC,CAAC;YAC7B,OAAO;QACT,KAAK,WAAW;YACd,OAAO,gBAAgB,CAAC,IAAI,CAAC,CAAC;QAChC,KAAK,QAAQ;YACX,OAAO,MAAM,EAAE,CAAC;QAClB,KAAK,SAAS;YACZ,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,MAAM,EAAE,IAAI,CAAC,CAAC;YACtC,OAAO;QACT,KAAK,SAAS,CAAC;QACf,KAAK,IAAI,CAAC;QACV,KAAK,QAAQ;YACX,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;YAC5B,OAAO;QACT;YACE,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;YAC5B,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACpB,CAAC;AACH,CAAC;AAED,IAAI,EAAE,CAAC"}
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Pull the JSON object out of a model's reply.
|
|
3
|
+
*
|
|
4
|
+
* Lives here rather than in the harness because it is the only part of the eval
|
|
5
|
+
* with parsing logic, and the harness is a script whose command dispatch runs on
|
|
6
|
+
* import -- so a copy there is a copy no test can reach. That is not
|
|
7
|
+
* hypothetical: the first version hung forever on a reply beginning with `{`,
|
|
8
|
+
* and nothing could have caught it.
|
|
9
|
+
*
|
|
10
|
+
* Two rules, both learned from the shape of the replies:
|
|
11
|
+
*
|
|
12
|
+
* - **Last object first.** A model asked for JSON often narrates before it, so
|
|
13
|
+
* scanning back from the end finds the answer rather than an example inside
|
|
14
|
+
* the preamble.
|
|
15
|
+
* - **Strings are not structure.** A question about code contains braces --
|
|
16
|
+
* `retry(() => {...})` in a stem, `${...}` in an option -- and counting those
|
|
17
|
+
* as depth is what made the first version fail to terminate. The scan tracks
|
|
18
|
+
* quoting and escaping, so a brace inside a string is just a character.
|
|
19
|
+
*/
|
|
20
|
+
/** Parse one candidate object starting at `start`, or `undefined` if it is not one. */
|
|
21
|
+
function objectAt(text, start) {
|
|
22
|
+
let depth = 0;
|
|
23
|
+
let inString = false;
|
|
24
|
+
let escaped = false;
|
|
25
|
+
for (let i = start; i < text.length; i++) {
|
|
26
|
+
const ch = text[i];
|
|
27
|
+
if (inString) {
|
|
28
|
+
if (escaped)
|
|
29
|
+
escaped = false;
|
|
30
|
+
else if (ch === '\\')
|
|
31
|
+
escaped = true;
|
|
32
|
+
else if (ch === '"')
|
|
33
|
+
inString = false;
|
|
34
|
+
continue;
|
|
35
|
+
}
|
|
36
|
+
if (ch === '"') {
|
|
37
|
+
inString = true;
|
|
38
|
+
}
|
|
39
|
+
else if (ch === '{') {
|
|
40
|
+
depth++;
|
|
41
|
+
}
|
|
42
|
+
else if (ch === '}') {
|
|
43
|
+
depth--;
|
|
44
|
+
if (depth === 0) {
|
|
45
|
+
try {
|
|
46
|
+
return { end: i, value: JSON.parse(text.slice(start, i + 1)) };
|
|
47
|
+
}
|
|
48
|
+
catch {
|
|
49
|
+
return undefined;
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
return undefined;
|
|
55
|
+
}
|
|
56
|
+
export function extractJson(text) {
|
|
57
|
+
// Every candidate is parsed, rather than walking back with lastIndexOf:
|
|
58
|
+
// `lastIndexOf('{', -1)` clamps to 0 and returns 0 again, so the obvious loop
|
|
59
|
+
// never advances past a leading brace and cannot terminate.
|
|
60
|
+
const found = [];
|
|
61
|
+
for (let i = 0; i < text.length; i++) {
|
|
62
|
+
if (text[i] !== '{')
|
|
63
|
+
continue;
|
|
64
|
+
const parsed = objectAt(text, i);
|
|
65
|
+
if (parsed)
|
|
66
|
+
found.push({ start: i, end: parsed.end, value: parsed.value });
|
|
67
|
+
}
|
|
68
|
+
if (found.length === 0)
|
|
69
|
+
return null;
|
|
70
|
+
// Outermost first, then latest. "Last object in the text" alone was wrong the
|
|
71
|
+
// moment a reply nested one: for `{"concepts":[{...},{...}]}` the last `{` is
|
|
72
|
+
// the final array element, and it parses perfectly -- so the caller got one
|
|
73
|
+
// concept instead of the list and read it as an empty extraction.
|
|
74
|
+
//
|
|
75
|
+
// So: discard any object contained inside another, which leaves the wrappers;
|
|
76
|
+
// then take the last of those, which is still the rule that skips an example
|
|
77
|
+
// in a model's preamble.
|
|
78
|
+
const outermost = found.filter((a) => !found.some((b) => b !== a && b.start <= a.start && a.end <= b.end));
|
|
79
|
+
return outermost[outermost.length - 1]?.value ?? null;
|
|
80
|
+
}
|
|
81
|
+
//# sourceMappingURL=extract-json.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"extract-json.js","sourceRoot":"","sources":["../../src/eval/extract-json.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;AAEH,uFAAuF;AACvF,SAAS,QAAQ,CAAC,IAAY,EAAE,KAAa;IAC3C,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,IAAI,QAAQ,GAAG,KAAK,CAAC;IACrB,IAAI,OAAO,GAAG,KAAK,CAAC;IAEpB,KAAK,IAAI,CAAC,GAAG,KAAK,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACzC,MAAM,EAAE,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;QAEnB,IAAI,QAAQ,EAAE,CAAC;YACb,IAAI,OAAO;gBAAE,OAAO,GAAG,KAAK,CAAC;iBACxB,IAAI,EAAE,KAAK,IAAI;gBAAE,OAAO,GAAG,IAAI,CAAC;iBAChC,IAAI,EAAE,KAAK,GAAG;gBAAE,QAAQ,GAAG,KAAK,CAAC;YACtC,SAAS;QACX,CAAC;QAED,IAAI,EAAE,KAAK,GAAG,EAAE,CAAC;YACf,QAAQ,GAAG,IAAI,CAAC;QAClB,CAAC;aAAM,IAAI,EAAE,KAAK,GAAG,EAAE,CAAC;YACtB,KAAK,EAAE,CAAC;QACV,CAAC;aAAM,IAAI,EAAE,KAAK,GAAG,EAAE,CAAC;YACtB,KAAK,EAAE,CAAC;YACR,IAAI,KAAK,KAAK,CAAC,EAAE,CAAC;gBAChB,IAAI,CAAC;oBACH,OAAO,EAAE,GAAG,EAAE,CAAC,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;gBACjE,CAAC;gBAAC,MAAM,CAAC;oBACP,OAAO,SAAS,CAAC;gBACnB,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,MAAM,UAAU,WAAW,CAAC,IAAY;IACtC,wEAAwE;IACxE,8EAA8E;IAC9E,4DAA4D;IAC5D,MAAM,KAAK,GAAqD,EAAE,CAAC;IACnE,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACrC,IAAI,IAAI,CAAC,CAAC,CAAC,KAAK,GAAG;YAAE,SAAS;QAC9B,MAAM,MAAM,GAAG,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;QACjC,IAAI,MAAM;YAAE,KAAK,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,MAAM,CAAC,GAAG,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC;IAC7E,CAAC;IACD,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IAEpC,8EAA8E;IAC9E,8EAA8E;IAC9E,4EAA4E;IAC5E,kEAAkE;IAClE,EAAE;IACF,8EAA8E;IAC9E,6EAA6E;IAC7E,yBAAyB;IACzB,MAAM,SAAS,GAAG,KAAK,CAAC,MAAM,CAC5B,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,GAAG,CAAC,CAC3E,CAAC;IACF,OAAO,SAAS,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,EAAE,KAAK,IAAI,IAAI,CAAC;AACxD,CAAC"}
|
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Does `log_session_concepts` name the concepts a diff actually exercises?
|
|
3
|
+
*
|
|
4
|
+
* This sits upstream of everything else the eval measures. If extraction picks
|
|
5
|
+
* the wrong concepts, every question after it is well-formed and about the
|
|
6
|
+
* wrong thing -- and `question-checks.ts` would score that run perfectly clean,
|
|
7
|
+
* because a good question about an irrelevant concept is still a good question.
|
|
8
|
+
*
|
|
9
|
+
* Scored with the product's own matcher. `findFuzzyMatch` and
|
|
10
|
+
* `FUZZY_MATCH_THRESHOLD` are what the server uses to decide two slugs mean the
|
|
11
|
+
* same concept, so a model that logs `wal-mode` against a label of
|
|
12
|
+
* `wal-journal-mode` is credited here exactly as the server would credit it.
|
|
13
|
+
* A second opinion invented for the eval would measure the eval.
|
|
14
|
+
*
|
|
15
|
+
* Pure: labels and extractions in, numbers out.
|
|
16
|
+
*/
|
|
17
|
+
import { findFuzzyMatch, isValidSlug, normalizeSlug } from '../slug.js';
|
|
18
|
+
/** The skill asks for 3-8 concepts the work genuinely exercises. */
|
|
19
|
+
export const MIN_CONCEPTS = 3;
|
|
20
|
+
export const MAX_CONCEPTS = 8;
|
|
21
|
+
function ratio(numerator, denominator) {
|
|
22
|
+
return denominator === 0 ? 0 : numerator / denominator;
|
|
23
|
+
}
|
|
24
|
+
export function scoreExtraction(fixture, labels, extracted) {
|
|
25
|
+
const candidates = labels.map((slug) => ({ slug }));
|
|
26
|
+
const matched = new Set();
|
|
27
|
+
const unlabelled = [];
|
|
28
|
+
for (const item of extracted) {
|
|
29
|
+
const slug = normalizeSlug(item.slug);
|
|
30
|
+
// Exact first, then the server's fuzzy rules -- the same order
|
|
31
|
+
// `log_session_concepts` resolves a slug in.
|
|
32
|
+
const hit = labels.includes(slug) ? { slug } : findFuzzyMatch(slug, candidates);
|
|
33
|
+
// A second logged slug matching an already-credited label is not a second
|
|
34
|
+
// hit. Counting it would let a model inflate recall by logging one concept
|
|
35
|
+
// five different ways.
|
|
36
|
+
if (hit && !matched.has(hit.slug))
|
|
37
|
+
matched.add(hit.slug);
|
|
38
|
+
else if (!hit)
|
|
39
|
+
unlabelled.push(slug);
|
|
40
|
+
}
|
|
41
|
+
const precision = ratio(matched.size, extracted.length);
|
|
42
|
+
const recall = ratio(matched.size, labels.length);
|
|
43
|
+
return {
|
|
44
|
+
fixture,
|
|
45
|
+
matched: [...matched],
|
|
46
|
+
missed: labels.filter((l) => !matched.has(l)),
|
|
47
|
+
unlabelled,
|
|
48
|
+
precision,
|
|
49
|
+
recall,
|
|
50
|
+
f1: precision + recall === 0 ? 0 : (2 * precision * recall) / (precision + recall),
|
|
51
|
+
};
|
|
52
|
+
}
|
|
53
|
+
/**
|
|
54
|
+
* What the skill asks for, checked without a model.
|
|
55
|
+
*
|
|
56
|
+
* The count and the context line are instructions with an exact form, so they
|
|
57
|
+
* are countable -- and the context line is the one that decays silently. A
|
|
58
|
+
* concept logged with `"used cookies"` still produces a question, just an
|
|
59
|
+
* ungrounded one, and nothing downstream notices.
|
|
60
|
+
*/
|
|
61
|
+
export function checkExtractionShape(extracted, diff) {
|
|
62
|
+
const checks = [];
|
|
63
|
+
const n = extracted.length;
|
|
64
|
+
checks.push({
|
|
65
|
+
id: 'count_in_range',
|
|
66
|
+
ok: n >= MIN_CONCEPTS && n <= MAX_CONCEPTS,
|
|
67
|
+
detail: `${n} concept(s) (asked for ${MIN_CONCEPTS}-${MAX_CONCEPTS})`,
|
|
68
|
+
});
|
|
69
|
+
const invalid = extracted.filter((e) => !isValidSlug(normalizeSlug(e.slug)));
|
|
70
|
+
checks.push({
|
|
71
|
+
id: 'slugs_valid',
|
|
72
|
+
ok: invalid.length === 0,
|
|
73
|
+
detail: invalid.length === 0 ? 'all slugs normalize cleanly' : `${invalid.length} unusable slug(s)`,
|
|
74
|
+
});
|
|
75
|
+
const contextless = extracted.filter((e) => !e.context || e.context.trim().length === 0);
|
|
76
|
+
checks.push({
|
|
77
|
+
id: 'every_concept_has_context',
|
|
78
|
+
ok: contextless.length === 0,
|
|
79
|
+
detail: contextless.length === 0 ? 'every concept carries a context line' : `${contextless.length} without context`,
|
|
80
|
+
});
|
|
81
|
+
// "One line naming the actual decision, in the actual file." A context that
|
|
82
|
+
// shares no identifier with the diff is the `"used cookies"` case the skill
|
|
83
|
+
// calls useless -- it reads fine and grounds nothing.
|
|
84
|
+
const identifiers = new Set((diff.match(/[A-Za-z_][A-Za-z0-9_]{3,}/g) ?? []).map((w) => w.toLowerCase()));
|
|
85
|
+
const ungrounded = extracted.filter((e) => {
|
|
86
|
+
const words = (e.context ?? '').toLowerCase().match(/[a-z_][a-z0-9_]{3,}/g) ?? [];
|
|
87
|
+
return !words.some((w) => identifiers.has(w));
|
|
88
|
+
});
|
|
89
|
+
checks.push({
|
|
90
|
+
id: 'context_names_the_code',
|
|
91
|
+
ok: ungrounded.length === 0,
|
|
92
|
+
detail: ungrounded.length === 0
|
|
93
|
+
? 'every context line names something from the diff'
|
|
94
|
+
: `${ungrounded.length} context line(s) name nothing in the diff`,
|
|
95
|
+
});
|
|
96
|
+
return checks;
|
|
97
|
+
}
|
|
98
|
+
export function summarizeExtraction(results) {
|
|
99
|
+
const matched = results.reduce((n, r) => n + r.matched.length, 0);
|
|
100
|
+
const labels = results.reduce((n, r) => n + r.matched.length + r.missed.length, 0);
|
|
101
|
+
const extracted = results.reduce((n, r) => n + r.matched.length + r.unlabelled.length, 0);
|
|
102
|
+
const precision = ratio(matched, extracted);
|
|
103
|
+
const recall = ratio(matched, labels);
|
|
104
|
+
return {
|
|
105
|
+
fixtures: results.length,
|
|
106
|
+
labels,
|
|
107
|
+
extracted,
|
|
108
|
+
matched,
|
|
109
|
+
precision,
|
|
110
|
+
recall,
|
|
111
|
+
f1: precision + recall === 0 ? 0 : (2 * precision * recall) / (precision + recall),
|
|
112
|
+
unlabelled: results.reduce((n, r) => n + r.unlabelled.length, 0),
|
|
113
|
+
};
|
|
114
|
+
}
|
|
115
|
+
//# sourceMappingURL=extraction-score.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"extraction-score.js","sourceRoot":"","sources":["../../src/eval/extraction-score.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AACH,OAAO,EAAE,cAAc,EAAE,WAAW,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAQxE,oEAAoE;AACpE,MAAM,CAAC,MAAM,YAAY,GAAG,CAAC,CAAC;AAC9B,MAAM,CAAC,MAAM,YAAY,GAAG,CAAC,CAAC;AAsB9B,SAAS,KAAK,CAAC,SAAiB,EAAE,WAAmB;IACnD,OAAO,WAAW,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,GAAG,WAAW,CAAC;AACzD,CAAC;AAED,MAAM,UAAU,eAAe,CAAC,OAAe,EAAE,MAAgB,EAAE,SAAsB;IACvF,MAAM,UAAU,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;IACpD,MAAM,OAAO,GAAG,IAAI,GAAG,EAAU,CAAC;IAClC,MAAM,UAAU,GAAa,EAAE,CAAC;IAEhC,KAAK,MAAM,IAAI,IAAI,SAAS,EAAE,CAAC;QAC7B,MAAM,IAAI,GAAG,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACtC,+DAA+D;QAC/D,6CAA6C;QAC7C,MAAM,GAAG,GAAG,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,cAAc,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC;QAChF,0EAA0E;QAC1E,2EAA2E;QAC3E,uBAAuB;QACvB,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC;YAAE,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;aACpD,IAAI,CAAC,GAAG;YAAE,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACvC,CAAC;IAED,MAAM,SAAS,GAAG,KAAK,CAAC,OAAO,CAAC,IAAI,EAAE,SAAS,CAAC,MAAM,CAAC,CAAC;IACxD,MAAM,MAAM,GAAG,KAAK,CAAC,OAAO,CAAC,IAAI,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC;IAElD,OAAO;QACL,OAAO;QACP,OAAO,EAAE,CAAC,GAAG,OAAO,CAAC;QACrB,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;QAC7C,UAAU;QACV,SAAS;QACT,MAAM;QACN,EAAE,EAAE,SAAS,GAAG,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,SAAS,GAAG,MAAM,CAAC,GAAG,CAAC,SAAS,GAAG,MAAM,CAAC;KACnF,CAAC;AACJ,CAAC;AAQD;;;;;;;GAOG;AACH,MAAM,UAAU,oBAAoB,CAAC,SAAsB,EAAE,IAAY;IACvE,MAAM,MAAM,GAAiB,EAAE,CAAC;IAChC,MAAM,CAAC,GAAG,SAAS,CAAC,MAAM,CAAC;IAE3B,MAAM,CAAC,IAAI,CAAC;QACV,EAAE,EAAE,gBAAgB;QACpB,EAAE,EAAE,CAAC,IAAI,YAAY,IAAI,CAAC,IAAI,YAAY;QAC1C,MAAM,EAAE,GAAG,CAAC,0BAA0B,YAAY,IAAI,YAAY,GAAG;KACtE,CAAC,CAAC;IAEH,MAAM,OAAO,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,WAAW,CAAC,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAC7E,MAAM,CAAC,IAAI,CAAC;QACV,EAAE,EAAE,aAAa;QACjB,EAAE,EAAE,OAAO,CAAC,MAAM,KAAK,CAAC;QACxB,MAAM,EAAE,OAAO,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,6BAA6B,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,MAAM,mBAAmB;KACpG,CAAC,CAAC;IAEH,MAAM,WAAW,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,OAAO,IAAI,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC;IACzF,MAAM,CAAC,IAAI,CAAC;QACV,EAAE,EAAE,2BAA2B;QAC/B,EAAE,EAAE,WAAW,CAAC,MAAM,KAAK,CAAC;QAC5B,MAAM,EAAE,WAAW,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,sCAAsC,CAAC,CAAC,CAAC,GAAG,WAAW,CAAC,MAAM,kBAAkB;KACpH,CAAC,CAAC;IAEH,4EAA4E;IAC5E,4EAA4E;IAC5E,sDAAsD;IACtD,MAAM,WAAW,GAAG,IAAI,GAAG,CACzB,CAAC,IAAI,CAAC,KAAK,CAAC,4BAA4B,CAAC,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,CAC7E,CAAC;IACF,MAAM,UAAU,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE;QACxC,MAAM,KAAK,GAAG,CAAC,CAAC,CAAC,OAAO,IAAI,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,sBAAsB,CAAC,IAAI,EAAE,CAAC;QAClF,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;IAChD,CAAC,CAAC,CAAC;IACH,MAAM,CAAC,IAAI,CAAC;QACV,EAAE,EAAE,wBAAwB;QAC5B,EAAE,EAAE,UAAU,CAAC,MAAM,KAAK,CAAC;QAC3B,MAAM,EACJ,UAAU,CAAC,MAAM,KAAK,CAAC;YACrB,CAAC,CAAC,kDAAkD;YACpD,CAAC,CAAC,GAAG,UAAU,CAAC,MAAM,2CAA2C;KACtE,CAAC,CAAC;IAEH,OAAO,MAAM,CAAC;AAChB,CAAC;AAcD,MAAM,UAAU,mBAAmB,CAAC,OAA2B;IAC7D,MAAM,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;IAClE,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;IACnF,MAAM,SAAS,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,UAAU,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;IAC1F,MAAM,SAAS,GAAG,KAAK,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC;IAC5C,MAAM,MAAM,GAAG,KAAK,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;IAEtC,OAAO;QACL,QAAQ,EAAE,OAAO,CAAC,MAAM;QACxB,MAAM;QACN,SAAS;QACT,OAAO;QACP,SAAS;QACT,MAAM;QACN,EAAE,EAAE,SAAS,GAAG,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,SAAS,GAAG,MAAM,CAAC,GAAG,CAAC,SAAS,GAAG,MAAM,CAAC;QAClF,UAAU,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,UAAU,CAAC,MAAM,EAAE,CAAC,CAAC;KACjE,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1,230 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* What a real answer history says about the promises the product makes.
|
|
3
|
+
*
|
|
4
|
+
* The question-quality eval next door measures questions before anyone answers
|
|
5
|
+
* them. This measures what happened after: it reads an actual `knowledge.db`
|
|
6
|
+
* and asks whether the claims hold on it.
|
|
7
|
+
*
|
|
8
|
+
* Three, in the order they matter:
|
|
9
|
+
*
|
|
10
|
+
* 1. **The repeat rate.** *Never the same question twice* is the promise the
|
|
11
|
+
* whole tool rests on, and it is the one claim measurable today with no new
|
|
12
|
+
* harness. If it is not kept, nothing else here is worth reading.
|
|
13
|
+
* 2. **Tier calibration.** Tiers 1-5 are supposed to be a difficulty ladder. If
|
|
14
|
+
* grades do not fall as tiers rise, the ladder is decoration.
|
|
15
|
+
* 3. **Whether anything survives a gap.** Not retention -- that needs a study
|
|
16
|
+
* with people in it -- but the weakest honest version: when a concept came
|
|
17
|
+
* back days later, did the answer hold.
|
|
18
|
+
*
|
|
19
|
+
* Pure, like `srs.ts`: rows in, numbers out, no database handle and no clock
|
|
20
|
+
* beyond what the rows carry. Aggregates only, and deliberately so -- nothing
|
|
21
|
+
* here returns question or answer text, because the output is written to a file
|
|
22
|
+
* that gets committed and the repo rule is that a learner's data never is.
|
|
23
|
+
*/
|
|
24
|
+
import { questionFingerprint } from '../store.js';
|
|
25
|
+
/**
|
|
26
|
+
* How far back the product itself looks for a repeat.
|
|
27
|
+
*
|
|
28
|
+
* `hasAskedQuestion` in `store.ts` fingerprints the last 20 attempts on a
|
|
29
|
+
* concept and no further. That is a deliberate bound, not an oversight -- but it
|
|
30
|
+
* means a repeat older than the window is a *design limit* rather than a broken
|
|
31
|
+
* promise, and reporting the two as one number would be dishonest in whichever
|
|
32
|
+
* direction happened to flatter.
|
|
33
|
+
*/
|
|
34
|
+
export const REPEAT_WINDOW = 20;
|
|
35
|
+
/**
|
|
36
|
+
* How many previous questions the *planner* puts in front of the tutor.
|
|
37
|
+
*
|
|
38
|
+
* `get_session_quiz_plan` fills each item's `asked_before` from
|
|
39
|
+
* `recentQuestions(db, concept.id, ASKED_HISTORY)` with ASKED_HISTORY = 3. That
|
|
40
|
+
* is the mechanism that actually prevents a repeat, because it acts while the
|
|
41
|
+
* question is being written.
|
|
42
|
+
*
|
|
43
|
+
* `REPEAT_WINDOW` above is a different thing and a weaker one: `hasAskedQuestion`
|
|
44
|
+
* is consulted by `record_attempt` *after* the question was asked, and it does
|
|
45
|
+
* not reject anything -- it records the attempt and returns `repeat_question:
|
|
46
|
+
* true`. Calling a repeat at distance 4-20 "one the product should have caught"
|
|
47
|
+
* credits a check that never saw it. Both numbers are reported, separately.
|
|
48
|
+
*/
|
|
49
|
+
export const PLANNER_HISTORY = 3;
|
|
50
|
+
export function repeatStats(rows) {
|
|
51
|
+
const byConcept = new Map();
|
|
52
|
+
for (const row of rows) {
|
|
53
|
+
const list = byConcept.get(row.concept_id);
|
|
54
|
+
if (list)
|
|
55
|
+
list.push(row);
|
|
56
|
+
else
|
|
57
|
+
byConcept.set(row.concept_id, [row]);
|
|
58
|
+
}
|
|
59
|
+
let repeats = 0;
|
|
60
|
+
let repeatable = 0;
|
|
61
|
+
let plannerSaw = 0;
|
|
62
|
+
let recorderOnly = 0;
|
|
63
|
+
let worstRun = 0;
|
|
64
|
+
for (const list of byConcept.values()) {
|
|
65
|
+
const ordered = [...list].sort((a, b) => a.id - b.id);
|
|
66
|
+
const seen = [];
|
|
67
|
+
const counts = new Map();
|
|
68
|
+
for (const [index, row] of ordered.entries()) {
|
|
69
|
+
const print = questionFingerprint(row.question);
|
|
70
|
+
// An empty fingerprint is not evidence of anything -- it means the stem
|
|
71
|
+
// was blank or punctuation only, which the product also refuses to match.
|
|
72
|
+
// It cannot count as a repeat, so it must not swell the denominator
|
|
73
|
+
// either; that error would run in the flattering direction.
|
|
74
|
+
if (print) {
|
|
75
|
+
if (index > 0)
|
|
76
|
+
repeatable++;
|
|
77
|
+
// lastIndexOf, not indexOf: what matters is how far back the NEAREST
|
|
78
|
+
// earlier copy was, since that is the one both windows would have held.
|
|
79
|
+
// Taking the oldest copy under-reports repeats the product could see.
|
|
80
|
+
const previous = seen.lastIndexOf(print);
|
|
81
|
+
if (previous >= 0) {
|
|
82
|
+
repeats++;
|
|
83
|
+
const distance = seen.length - previous;
|
|
84
|
+
if (distance <= PLANNER_HISTORY)
|
|
85
|
+
plannerSaw++;
|
|
86
|
+
else if (distance <= REPEAT_WINDOW)
|
|
87
|
+
recorderOnly++;
|
|
88
|
+
}
|
|
89
|
+
const n = (counts.get(print) ?? 0) + 1;
|
|
90
|
+
counts.set(print, n);
|
|
91
|
+
if (n > worstRun)
|
|
92
|
+
worstRun = n;
|
|
93
|
+
}
|
|
94
|
+
seen.push(print);
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
return {
|
|
98
|
+
attempts: rows.length,
|
|
99
|
+
concepts: byConcept.size,
|
|
100
|
+
repeatable,
|
|
101
|
+
repeats,
|
|
102
|
+
plannerSaw,
|
|
103
|
+
recorderOnly,
|
|
104
|
+
outsideBoth: repeats - plannerSaw - recorderOnly,
|
|
105
|
+
worstRun,
|
|
106
|
+
};
|
|
107
|
+
}
|
|
108
|
+
/**
|
|
109
|
+
* Both readings, because the data cannot choose between them.
|
|
110
|
+
*
|
|
111
|
+
* `migrations/004_attempt_outcome.sql` states the contract: NULL means
|
|
112
|
+
* "recorded before this column existed", and readers "must treat it as unknown,
|
|
113
|
+
* not as any particular outcome". Keeping those rows counts a grade 0 of unknown
|
|
114
|
+
* provenance as evidence the learner did not know; dropping them throws away
|
|
115
|
+
* real answers. On the first published history the choice moved tier 2 from
|
|
116
|
+
* mean 2.25 to 3.10 -- from below the pass threshold to above it -- and changed
|
|
117
|
+
* which tier scored worst.
|
|
118
|
+
*
|
|
119
|
+
* A single number there would have been a choice presented as a measurement, so
|
|
120
|
+
* both are reported and the result file has to show both.
|
|
121
|
+
*/
|
|
122
|
+
export function tierReadings(rows) {
|
|
123
|
+
return {
|
|
124
|
+
allGraded: tierStats(rows, true),
|
|
125
|
+
knownOutcome: tierStats(rows, false),
|
|
126
|
+
};
|
|
127
|
+
}
|
|
128
|
+
export function tierStats(rows, includeUnknownOutcome = true) {
|
|
129
|
+
// A decline is recorded as grade 0 and means "not now", not "did not know".
|
|
130
|
+
// Averaging it in would make any tier the developer happened to skip look
|
|
131
|
+
// harder than it is.
|
|
132
|
+
const notDeclined = rows.filter((r) => r.outcome !== 'declined');
|
|
133
|
+
const graded = includeUnknownOutcome ? notDeclined : notDeclined.filter((r) => r.outcome != null);
|
|
134
|
+
const byTier = new Map();
|
|
135
|
+
for (const row of graded) {
|
|
136
|
+
const list = byTier.get(row.difficulty);
|
|
137
|
+
if (list)
|
|
138
|
+
list.push(row);
|
|
139
|
+
else
|
|
140
|
+
byTier.set(row.difficulty, [row]);
|
|
141
|
+
}
|
|
142
|
+
const out = [...byTier.entries()]
|
|
143
|
+
.sort(([a], [b]) => a - b)
|
|
144
|
+
.map(([tier, list]) => ({
|
|
145
|
+
tier,
|
|
146
|
+
attempts: list.length,
|
|
147
|
+
meanGrade: list.reduce((n, r) => n + r.grade, 0) / list.length,
|
|
148
|
+
passRate: list.filter((r) => r.grade >= 3).length / list.length,
|
|
149
|
+
}));
|
|
150
|
+
let monotonic = true;
|
|
151
|
+
for (let i = 1; i < out.length; i++) {
|
|
152
|
+
if ((out[i]?.meanGrade ?? 0) > (out[i - 1]?.meanGrade ?? 0))
|
|
153
|
+
monotonic = false;
|
|
154
|
+
}
|
|
155
|
+
return {
|
|
156
|
+
basis: includeUnknownOutcome ? 'all-graded' : 'known-outcome',
|
|
157
|
+
rows: out,
|
|
158
|
+
monotonic,
|
|
159
|
+
excludedDeclines: rows.length - notDeclined.length,
|
|
160
|
+
excludedUnknown: notDeclined.length - graded.length,
|
|
161
|
+
};
|
|
162
|
+
}
|
|
163
|
+
const DAY_MS = 24 * 60 * 60 * 1000;
|
|
164
|
+
/**
|
|
165
|
+
* `ts` is written by SQLite's `datetime('now')`, which is UTC with no zone
|
|
166
|
+
* marker. `Date.parse('2026-09-01 10:00:00')` reads that as local time, so on a
|
|
167
|
+
* machine at UTC+5:30 every timestamp shifts and a gap near the threshold lands
|
|
168
|
+
* on the wrong side of it.
|
|
169
|
+
*/
|
|
170
|
+
function parseTs(ts) {
|
|
171
|
+
return Date.parse(`${ts.replace(' ', 'T')}Z`);
|
|
172
|
+
}
|
|
173
|
+
/**
|
|
174
|
+
* Consecutive attempts on the same concept separated by at least `minDays`.
|
|
175
|
+
*
|
|
176
|
+
* Consecutive, not all-pairs: counting every combination would weight a concept
|
|
177
|
+
* asked five times far above one asked twice, and the question being asked is
|
|
178
|
+
* "did it hold across a gap", once per gap.
|
|
179
|
+
*/
|
|
180
|
+
export function gapStats(rows, minDays = 1) {
|
|
181
|
+
const byConcept = new Map();
|
|
182
|
+
for (const row of rows) {
|
|
183
|
+
const list = byConcept.get(row.concept_id);
|
|
184
|
+
if (list)
|
|
185
|
+
list.push(row);
|
|
186
|
+
else
|
|
187
|
+
byConcept.set(row.concept_id, [row]);
|
|
188
|
+
}
|
|
189
|
+
let pairs = 0;
|
|
190
|
+
let held = 0;
|
|
191
|
+
let skippedNoPriorPass = 0;
|
|
192
|
+
for (const list of byConcept.values()) {
|
|
193
|
+
// Ordered over EVERY attempt, declines included. Filtering them out first
|
|
194
|
+
// makes two attempts either side of a skipped one look consecutive, so a
|
|
195
|
+
// concept the learner saw three times reports one long quiet gap it never
|
|
196
|
+
// had. Declines are skipped as the *later* member below instead.
|
|
197
|
+
const ordered = [...list].sort((a, b) => a.id - b.id);
|
|
198
|
+
for (let i = 1; i < ordered.length; i++) {
|
|
199
|
+
const before = ordered[i - 1];
|
|
200
|
+
const after = ordered[i];
|
|
201
|
+
if (!before || !after)
|
|
202
|
+
continue;
|
|
203
|
+
if (after.outcome === 'declined')
|
|
204
|
+
continue;
|
|
205
|
+
// `datetime('now')` writes UTC without a zone; treat it as UTC rather than
|
|
206
|
+
// letting Date.parse guess local, which shifts every gap by the offset.
|
|
207
|
+
const gap = parseTs(after.ts) - parseTs(before.ts);
|
|
208
|
+
if (!Number.isFinite(gap) || gap < minDays * DAY_MS)
|
|
209
|
+
continue;
|
|
210
|
+
if (before.grade < 3) {
|
|
211
|
+
skippedNoPriorPass++;
|
|
212
|
+
continue;
|
|
213
|
+
}
|
|
214
|
+
pairs++;
|
|
215
|
+
if (after.grade >= 3)
|
|
216
|
+
held++;
|
|
217
|
+
}
|
|
218
|
+
}
|
|
219
|
+
return { minDays, pairs, held, skippedNoPriorPass };
|
|
220
|
+
}
|
|
221
|
+
export function outcomeStats(rows) {
|
|
222
|
+
const count = (o) => rows.filter((r) => r.outcome === o).length;
|
|
223
|
+
return {
|
|
224
|
+
answered: count('answered'),
|
|
225
|
+
dontKnow: count('dont_know'),
|
|
226
|
+
declined: count('declined'),
|
|
227
|
+
unrecorded: rows.filter((r) => r.outcome == null).length,
|
|
228
|
+
};
|
|
229
|
+
}
|
|
230
|
+
//# sourceMappingURL=history-stats.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"history-stats.js","sourceRoot":"","sources":["../../src/eval/history-stats.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;GAsBG;AACH,OAAO,EAAE,mBAAmB,EAAE,MAAM,aAAa,CAAC;AAalD;;;;;;;;GAQG;AACH,MAAM,CAAC,MAAM,aAAa,GAAG,EAAE,CAAC;AAEhC;;;;;;;;;;;;;GAaG;AACH,MAAM,CAAC,MAAM,eAAe,GAAG,CAAC,CAAC;AAsCjC,MAAM,UAAU,WAAW,CAAC,IAAkB;IAC5C,MAAM,SAAS,GAAG,IAAI,GAAG,EAAwB,CAAC;IAClD,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,MAAM,IAAI,GAAG,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;QAC3C,IAAI,IAAI;YAAE,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;;YACpB,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC,UAAU,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;IAC5C,CAAC;IAED,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,IAAI,UAAU,GAAG,CAAC,CAAC;IACnB,IAAI,UAAU,GAAG,CAAC,CAAC;IACnB,IAAI,YAAY,GAAG,CAAC,CAAC;IACrB,IAAI,QAAQ,GAAG,CAAC,CAAC;IAEjB,KAAK,MAAM,IAAI,IAAI,SAAS,CAAC,MAAM,EAAE,EAAE,CAAC;QACtC,MAAM,OAAO,GAAG,CAAC,GAAG,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC;QACtD,MAAM,IAAI,GAAa,EAAE,CAAC;QAC1B,MAAM,MAAM,GAAG,IAAI,GAAG,EAAkB,CAAC;QAEzC,KAAK,MAAM,CAAC,KAAK,EAAE,GAAG,CAAC,IAAI,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC;YAC7C,MAAM,KAAK,GAAG,mBAAmB,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;YAChD,wEAAwE;YACxE,0EAA0E;YAC1E,oEAAoE;YACpE,4DAA4D;YAC5D,IAAI,KAAK,EAAE,CAAC;gBACV,IAAI,KAAK,GAAG,CAAC;oBAAE,UAAU,EAAE,CAAC;gBAE5B,qEAAqE;gBACrE,wEAAwE;gBACxE,sEAAsE;gBACtE,MAAM,QAAQ,GAAG,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC;gBACzC,IAAI,QAAQ,IAAI,CAAC,EAAE,CAAC;oBAClB,OAAO,EAAE,CAAC;oBACV,MAAM,QAAQ,GAAG,IAAI,CAAC,MAAM,GAAG,QAAQ,CAAC;oBACxC,IAAI,QAAQ,IAAI,eAAe;wBAAE,UAAU,EAAE,CAAC;yBACzC,IAAI,QAAQ,IAAI,aAAa;wBAAE,YAAY,EAAE,CAAC;gBACrD,CAAC;gBACD,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;gBACvC,MAAM,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;gBACrB,IAAI,CAAC,GAAG,QAAQ;oBAAE,QAAQ,GAAG,CAAC,CAAC;YACjC,CAAC;YACD,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACnB,CAAC;IACH,CAAC;IAED,OAAO;QACL,QAAQ,EAAE,IAAI,CAAC,MAAM;QACrB,QAAQ,EAAE,SAAS,CAAC,IAAI;QACxB,UAAU;QACV,OAAO;QACP,UAAU;QACV,YAAY;QACZ,WAAW,EAAE,OAAO,GAAG,UAAU,GAAG,YAAY;QAChD,QAAQ;KACT,CAAC;AACJ,CAAC;AA+BD;;;;;;;;;;;;;GAaG;AACH,MAAM,UAAU,YAAY,CAAC,IAAkB;IAC7C,OAAO;QACL,SAAS,EAAE,SAAS,CAAC,IAAI,EAAE,IAAI,CAAC;QAChC,YAAY,EAAE,SAAS,CAAC,IAAI,EAAE,KAAK,CAAC;KACrC,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,SAAS,CAAC,IAAkB,EAAE,qBAAqB,GAAG,IAAI;IACxE,4EAA4E;IAC5E,0EAA0E;IAC1E,qBAAqB;IACrB,MAAM,WAAW,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,KAAK,UAAU,CAAC,CAAC;IACjE,MAAM,MAAM,GAAG,qBAAqB,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,IAAI,IAAI,CAAC,CAAC;IAClG,MAAM,MAAM,GAAG,IAAI,GAAG,EAAwB,CAAC;IAC/C,KAAK,MAAM,GAAG,IAAI,MAAM,EAAE,CAAC;QACzB,MAAM,IAAI,GAAG,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;QACxC,IAAI,IAAI;YAAE,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;;YACpB,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,UAAU,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;IACzC,CAAC;IAED,MAAM,GAAG,GAAc,CAAC,GAAG,MAAM,CAAC,OAAO,EAAE,CAAC;SACzC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC;SACzB,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,IAAI,CAAC,EAAE,EAAE,CAAC,CAAC;QACtB,IAAI;QACJ,QAAQ,EAAE,IAAI,CAAC,MAAM;QACrB,SAAS,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,GAAG,IAAI,CAAC,MAAM;QAC9D,QAAQ,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM;KAChE,CAAC,CAAC,CAAC;IAEN,IAAI,SAAS,GAAG,IAAI,CAAC;IACrB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACpC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,SAAS,IAAI,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,SAAS,IAAI,CAAC,CAAC;YAAE,SAAS,GAAG,KAAK,CAAC;IACjF,CAAC;IAED,OAAO;QACL,KAAK,EAAE,qBAAqB,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,eAAe;QAC7D,IAAI,EAAE,GAAG;QACT,SAAS;QACT,gBAAgB,EAAE,IAAI,CAAC,MAAM,GAAG,WAAW,CAAC,MAAM;QAClD,eAAe,EAAE,WAAW,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM;KACpD,CAAC;AACJ,CAAC;AAoBD,MAAM,MAAM,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC;AAEnC;;;;;GAKG;AACH,SAAS,OAAO,CAAC,EAAU;IACzB,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,GAAG,EAAE,GAAG,CAAC,GAAG,CAAC,CAAC;AAChD,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,QAAQ,CAAC,IAAkB,EAAE,OAAO,GAAG,CAAC;IACtD,MAAM,SAAS,GAAG,IAAI,GAAG,EAAwB,CAAC;IAClD,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,MAAM,IAAI,GAAG,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;QAC3C,IAAI,IAAI;YAAE,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;;YACpB,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC,UAAU,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;IAC5C,CAAC;IAED,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,IAAI,IAAI,GAAG,CAAC,CAAC;IACb,IAAI,kBAAkB,GAAG,CAAC,CAAC;IAE3B,KAAK,MAAM,IAAI,IAAI,SAAS,CAAC,MAAM,EAAE,EAAE,CAAC;QACtC,0EAA0E;QAC1E,yEAAyE;QACzE,0EAA0E;QAC1E,iEAAiE;QACjE,MAAM,OAAO,GAAG,CAAC,GAAG,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC;QAEtD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACxC,MAAM,MAAM,GAAG,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;YAC9B,MAAM,KAAK,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;YACzB,IAAI,CAAC,MAAM,IAAI,CAAC,KAAK;gBAAE,SAAS;YAChC,IAAI,KAAK,CAAC,OAAO,KAAK,UAAU;gBAAE,SAAS;YAE3C,2EAA2E;YAC3E,wEAAwE;YACxE,MAAM,GAAG,GAAG,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;YACnD,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,GAAG,GAAG,OAAO,GAAG,MAAM;gBAAE,SAAS;YAE9D,IAAI,MAAM,CAAC,KAAK,GAAG,CAAC,EAAE,CAAC;gBACrB,kBAAkB,EAAE,CAAC;gBACrB,SAAS;YACX,CAAC;YACD,KAAK,EAAE,CAAC;YACR,IAAI,KAAK,CAAC,KAAK,IAAI,CAAC;gBAAE,IAAI,EAAE,CAAC;QAC/B,CAAC;IACH,CAAC;IAED,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,kBAAkB,EAAE,CAAC;AACtD,CAAC;AAUD,MAAM,UAAU,YAAY,CAAC,IAAkB;IAC7C,MAAM,KAAK,GAAG,CAAC,CAAS,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC;IACxE,OAAO;QACL,QAAQ,EAAE,KAAK,CAAC,UAAU,CAAC;QAC3B,QAAQ,EAAE,KAAK,CAAC,WAAW,CAAC;QAC5B,QAAQ,EAAE,KAAK,CAAC,UAAU,CAAC;QAC3B,UAAU,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,IAAI,IAAI,CAAC,CAAC,MAAM;KACzD,CAAC;AACJ,CAAC"}
|