cc-proficiency 0.2.9 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +79 -3
- package/dist/cli/commands/ai-grade.d.ts +2 -0
- package/dist/cli/commands/ai-grade.d.ts.map +1 -0
- package/dist/cli/commands/ai-grade.js +139 -0
- package/dist/cli/commands/ai-grade.js.map +1 -0
- package/dist/cli/commands/config.d.ts.map +1 -1
- package/dist/cli/commands/config.js +2 -0
- package/dist/cli/commands/config.js.map +1 -1
- package/dist/cli/commands/explain.d.ts.map +1 -1
- package/dist/cli/commands/explain.js +18 -0
- package/dist/cli/commands/explain.js.map +1 -1
- package/dist/cli/commands/process.d.ts.map +1 -1
- package/dist/cli/commands/process.js +30 -0
- package/dist/cli/commands/process.js.map +1 -1
- package/dist/cli/index.js +6 -0
- package/dist/cli/index.js.map +1 -1
- package/dist/cli/services/publishing.d.ts.map +1 -1
- package/dist/cli/services/publishing.js +15 -0
- package/dist/cli/services/publishing.js.map +1 -1
- package/dist/i18n/locales/en.d.ts.map +1 -1
- package/dist/i18n/locales/en.js +28 -1
- package/dist/i18n/locales/en.js.map +1 -1
- package/dist/i18n/locales/es.d.ts.map +1 -1
- package/dist/i18n/locales/es.js +28 -1
- package/dist/i18n/locales/es.js.map +1 -1
- package/dist/i18n/locales/fr.d.ts.map +1 -1
- package/dist/i18n/locales/fr.js +27 -0
- package/dist/i18n/locales/fr.js.map +1 -1
- package/dist/i18n/locales/ja.d.ts.map +1 -1
- package/dist/i18n/locales/ja.js +27 -0
- package/dist/i18n/locales/ja.js.map +1 -1
- package/dist/i18n/locales/ko.d.ts.map +1 -1
- package/dist/i18n/locales/ko.js +27 -0
- package/dist/i18n/locales/ko.js.map +1 -1
- package/dist/i18n/locales/zh-CN.d.ts.map +1 -1
- package/dist/i18n/locales/zh-CN.js +27 -0
- package/dist/i18n/locales/zh-CN.js.map +1 -1
- package/dist/i18n/types.d.ts +21 -0
- package/dist/i18n/types.d.ts.map +1 -1
- package/dist/renderer/ai-animated-svg.d.ts +11 -0
- package/dist/renderer/ai-animated-svg.d.ts.map +1 -0
- package/dist/renderer/ai-animated-svg.js +206 -0
- package/dist/renderer/ai-animated-svg.js.map +1 -0
- package/dist/scoring/ai-evidence.d.ts +108 -0
- package/dist/scoring/ai-evidence.d.ts.map +1 -0
- package/dist/scoring/ai-evidence.js +305 -0
- package/dist/scoring/ai-evidence.js.map +1 -0
- package/dist/scoring/ai-grader.d.ts +64 -0
- package/dist/scoring/ai-grader.d.ts.map +1 -0
- package/dist/scoring/ai-grader.js +229 -0
- package/dist/scoring/ai-grader.js.map +1 -0
- package/dist/scoring/rubric-loader.d.ts +4 -0
- package/dist/scoring/rubric-loader.d.ts.map +1 -0
- package/dist/scoring/rubric-loader.js +102 -0
- package/dist/scoring/rubric-loader.js.map +1 -0
- package/dist/store/achievements.d.ts.map +1 -1
- package/dist/store/achievements.js +25 -0
- package/dist/store/achievements.js.map +1 -1
- package/dist/store/local-store.d.ts +2 -0
- package/dist/store/local-store.d.ts.map +1 -1
- package/dist/store/local-store.js +11 -0
- package/dist/store/local-store.js.map +1 -1
- package/dist/store/queue.d.ts +8 -0
- package/dist/store/queue.d.ts.map +1 -1
- package/dist/store/queue.js +44 -0
- package/dist/store/queue.js.map +1 -1
- package/dist/types.d.ts +51 -0
- package/dist/types.d.ts.map +1 -1
- package/dist/utils/format.d.ts +1 -1
- package/dist/utils/format.d.ts.map +1 -1
- package/dist/utils/format.js +11 -5
- package/dist/utils/format.js.map +1 -1
- package/docs/ai-grading/README.md +39 -0
- package/docs/ai-grading/anti-gaming.md +11 -0
- package/docs/ai-grading/collaboration-quality.md +25 -0
- package/docs/ai-grading/goal-achievement.md +25 -0
- package/docs/ai-grading/growth-learning.md +25 -0
- package/docs/ai-grading/system-prompt-header.md +32 -0
- package/docs/ai-grading/verification-quality.md +25 -0
- package/docs/ai-grading/workflow-mastery.md +25 -0
- package/package.json +2 -1
|
@@ -0,0 +1,305 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.readJsonSafe = readJsonSafe;
|
|
4
|
+
exports.sanitizeProjectPath = sanitizeProjectPath;
|
|
5
|
+
exports.normalizeFrictionKeys = normalizeFrictionKeys;
|
|
6
|
+
exports.getAIPhase = getAIPhase;
|
|
7
|
+
exports.assembleEvidence = assembleEvidence;
|
|
8
|
+
exports.precomputeStats = precomputeStats;
|
|
9
|
+
exports.buildEvidencePayload = buildEvidencePayload;
|
|
10
|
+
exports.computeAIEvidence = computeAIEvidence;
|
|
11
|
+
const node_fs_1 = require("node:fs");
|
|
12
|
+
const node_path_1 = require("node:path");
|
|
13
|
+
const FRICTION_KEY_ALIASES = {
|
|
14
|
+
tool_environment_issue: "tool_environment_issues",
|
|
15
|
+
};
|
|
16
|
+
// ── Public Functions ──
|
|
17
|
+
function readJsonSafe(path) {
|
|
18
|
+
try {
|
|
19
|
+
const raw = stripNullBytes((0, node_fs_1.readFileSync)(path, "utf-8"));
|
|
20
|
+
return JSON.parse(raw);
|
|
21
|
+
}
|
|
22
|
+
catch {
|
|
23
|
+
return null;
|
|
24
|
+
}
|
|
25
|
+
}
|
|
26
|
+
function sanitizeProjectPath(rawPath) {
|
|
27
|
+
if (!rawPath)
|
|
28
|
+
return "unknown";
|
|
29
|
+
const segments = rawPath.replace(/\/+$/, "").split("/").filter(Boolean);
|
|
30
|
+
return (segments[segments.length - 1] ?? "unknown").replace(/[^a-zA-Z0-9._-]/g, "_") || "unknown";
|
|
31
|
+
}
|
|
32
|
+
function normalizeFrictionKeys(counts) {
|
|
33
|
+
const out = {};
|
|
34
|
+
for (const [key, value] of Object.entries(counts)) {
|
|
35
|
+
const k = FRICTION_KEY_ALIASES[key] ?? key;
|
|
36
|
+
out[k] = (out[k] ?? 0) + value;
|
|
37
|
+
}
|
|
38
|
+
return out;
|
|
39
|
+
}
|
|
40
|
+
function getAIPhase(metaCount) {
|
|
41
|
+
if (metaCount < 10)
|
|
42
|
+
return "insufficient";
|
|
43
|
+
if (metaCount <= 30)
|
|
44
|
+
return "early";
|
|
45
|
+
return "full";
|
|
46
|
+
}
|
|
47
|
+
function assembleEvidence(metaDir, facetsDir) {
|
|
48
|
+
const metas = readAllJson(metaDir);
|
|
49
|
+
if (metas.valid.length === 0)
|
|
50
|
+
return null;
|
|
51
|
+
// Read facets (optional) and index by session_id
|
|
52
|
+
const facetsResult = facetsDir ? readAllJson(facetsDir) : { valid: [], corrupt: 0 };
|
|
53
|
+
const facetMap = new Map();
|
|
54
|
+
for (const raw of facetsResult.valid) {
|
|
55
|
+
const facet = raw;
|
|
56
|
+
if (!facet.session_id)
|
|
57
|
+
continue;
|
|
58
|
+
if (facet.friction_counts)
|
|
59
|
+
facet.friction_counts = normalizeFrictionKeys(facet.friction_counts);
|
|
60
|
+
facetMap.set(facet.session_id, facet);
|
|
61
|
+
}
|
|
62
|
+
// Build sessions: meta is primary, LEFT JOIN facets
|
|
63
|
+
const sessions = [];
|
|
64
|
+
let warmupCount = 0;
|
|
65
|
+
for (const raw of metas.valid) {
|
|
66
|
+
const meta = raw;
|
|
67
|
+
if (!meta.session_id)
|
|
68
|
+
continue;
|
|
69
|
+
const facet = facetMap.get(meta.session_id) ?? null;
|
|
70
|
+
const prompt = (meta.first_prompt ?? "").trim();
|
|
71
|
+
if (/^respond with ok$/i.test(prompt))
|
|
72
|
+
warmupCount++;
|
|
73
|
+
sessions.push({ meta, facet, project: sanitizeProjectPath(meta.project_path ?? "") });
|
|
74
|
+
}
|
|
75
|
+
// Sort by start_time for temporal ordering
|
|
76
|
+
sessions.sort((a, b) => {
|
|
77
|
+
const ta = a.meta.start_time ?? "";
|
|
78
|
+
const tb = b.meta.start_time ?? "";
|
|
79
|
+
return ta < tb ? -1 : ta > tb ? 1 : 0;
|
|
80
|
+
});
|
|
81
|
+
const productiveCount = sessions.length - warmupCount;
|
|
82
|
+
const metaCount = metas.valid.length;
|
|
83
|
+
const facetCount = facetMap.size;
|
|
84
|
+
const coverage = metaCount > 0 ? Math.round((facetCount / metaCount) * 100) : 0;
|
|
85
|
+
return {
|
|
86
|
+
sessions,
|
|
87
|
+
metaCount,
|
|
88
|
+
facetCount,
|
|
89
|
+
corruptMeta: metas.corrupt,
|
|
90
|
+
corruptFacets: facetsResult.corrupt,
|
|
91
|
+
productiveCount,
|
|
92
|
+
warmupCount,
|
|
93
|
+
coverage,
|
|
94
|
+
};
|
|
95
|
+
}
|
|
96
|
+
function precomputeStats(evidence) {
|
|
97
|
+
const outcomes = {};
|
|
98
|
+
const frictionTotals = {};
|
|
99
|
+
const toolTotals = {};
|
|
100
|
+
const sessionTypes = {};
|
|
101
|
+
const satisfactionCounts = {};
|
|
102
|
+
const helpfulness = {};
|
|
103
|
+
let gitCommits = 0, gitPushes = 0, totalDuration = 0, durationCount = 0;
|
|
104
|
+
let totalInput = 0, totalOutput = 0, totalLines = 0, totalFiles = 0, warmupCount = 0;
|
|
105
|
+
for (const { facet, meta } of evidence.sessions) {
|
|
106
|
+
if (facet?.outcome)
|
|
107
|
+
incr(outcomes, facet.outcome);
|
|
108
|
+
if (facet?.session_type)
|
|
109
|
+
incr(sessionTypes, facet.session_type);
|
|
110
|
+
if (facet?.friction_counts)
|
|
111
|
+
mergeInto(frictionTotals, facet.friction_counts);
|
|
112
|
+
if (facet?.user_satisfaction_counts)
|
|
113
|
+
mergeInto(satisfactionCounts, facet.user_satisfaction_counts);
|
|
114
|
+
if (facet?.claude_helpfulness)
|
|
115
|
+
incr(helpfulness, facet.claude_helpfulness);
|
|
116
|
+
const prompt = facet?.first_prompt ?? meta.first_prompt ?? "";
|
|
117
|
+
if (prompt.trim().toLowerCase() === "respond with ok")
|
|
118
|
+
warmupCount++;
|
|
119
|
+
if (meta.tool_counts)
|
|
120
|
+
mergeInto(toolTotals, meta.tool_counts);
|
|
121
|
+
gitCommits += meta.git_commits ?? 0;
|
|
122
|
+
gitPushes += meta.git_pushes ?? 0;
|
|
123
|
+
if (meta.duration_minutes && meta.duration_minutes > 0) {
|
|
124
|
+
totalDuration += meta.duration_minutes;
|
|
125
|
+
durationCount++;
|
|
126
|
+
}
|
|
127
|
+
totalInput += meta.input_tokens ?? 0;
|
|
128
|
+
totalOutput += meta.output_tokens ?? 0;
|
|
129
|
+
totalLines += meta.lines_added ?? 0;
|
|
130
|
+
totalFiles += meta.files_modified ?? 0;
|
|
131
|
+
}
|
|
132
|
+
const half = Math.floor(evidence.sessions.length / 2);
|
|
133
|
+
const achieved = (outcomes["fully_achieved"] ?? 0) + (outcomes["mostly_achieved"] ?? 0);
|
|
134
|
+
const totalOutcomes = Object.values(outcomes).reduce((a, b) => a + b, 0);
|
|
135
|
+
const satisfied = satisfactionCounts["likely_satisfied"] ?? 0;
|
|
136
|
+
const totalSat = Object.values(satisfactionCounts).reduce((a, b) => a + b, 0);
|
|
137
|
+
const totalMeta = evidence.metaCount + evidence.corruptMeta;
|
|
138
|
+
// Project distribution: count sessions per project
|
|
139
|
+
const projectDistribution = {};
|
|
140
|
+
let toolActivitySessions = 0;
|
|
141
|
+
let toolDiversitySum = 0;
|
|
142
|
+
for (const s of evidence.sessions) {
|
|
143
|
+
incr(projectDistribution, s.project);
|
|
144
|
+
const toolKeys = Object.keys(s.meta.tool_counts ?? {});
|
|
145
|
+
if (toolKeys.length > 0) {
|
|
146
|
+
toolActivitySessions++;
|
|
147
|
+
toolDiversitySum += toolKeys.length;
|
|
148
|
+
}
|
|
149
|
+
}
|
|
150
|
+
const avgToolDiversity = toolActivitySessions > 0 ? Math.round((toolDiversitySum / toolActivitySessions) * 10) / 10 : 0;
|
|
151
|
+
const facetMatchCount = evidence.sessions.filter((s) => s.facet !== null).length;
|
|
152
|
+
const coveragePercent = totalMeta > 0 ? Math.round((facetMatchCount / totalMeta) * 100) : 0;
|
|
153
|
+
const coverageFlags = {
|
|
154
|
+
hasFacets: evidence.facetCount > 0,
|
|
155
|
+
facetCoverage: coveragePercent,
|
|
156
|
+
sufficientForTrends: evidence.facetCount >= 20,
|
|
157
|
+
};
|
|
158
|
+
return {
|
|
159
|
+
outcomes, achievedRate: totalOutcomes > 0 ? Math.round((achieved / totalOutcomes) * 100) : 0,
|
|
160
|
+
frictionTotals,
|
|
161
|
+
frictionFirstHalf: aggregateFriction(evidence.sessions.slice(0, half)),
|
|
162
|
+
frictionSecondHalf: aggregateFriction(evidence.sessions.slice(half)),
|
|
163
|
+
toolTotals, sessionTypes,
|
|
164
|
+
gitCommitsTotal: gitCommits, gitPushesTotal: gitPushes,
|
|
165
|
+
satisfactionCounts, satisfiedRate: totalSat > 0 ? Math.round((satisfied / totalSat) * 100) : 0,
|
|
166
|
+
helpfulness,
|
|
167
|
+
avgDurationMinutes: durationCount > 0 ? Math.round(totalDuration / durationCount) : 0,
|
|
168
|
+
totalInputTokens: totalInput, totalOutputTokens: totalOutput,
|
|
169
|
+
totalLinesAdded: totalLines, totalFilesModified: totalFiles,
|
|
170
|
+
sessionCount: evidence.sessions.length,
|
|
171
|
+
coveragePercent,
|
|
172
|
+
warmupCount,
|
|
173
|
+
projectDistribution,
|
|
174
|
+
toolActivitySessions,
|
|
175
|
+
avgToolDiversity,
|
|
176
|
+
coverageFlags,
|
|
177
|
+
};
|
|
178
|
+
}
|
|
179
|
+
function buildEvidencePayload(stats, config, ruleScores, features) {
|
|
180
|
+
const flags = stats.coverageFlags ?? { hasFacets: false, facetCoverage: 0, sufficientForTrends: false };
|
|
181
|
+
const productiveCount = stats.sessionCount - stats.warmupCount;
|
|
182
|
+
const warmupPct = stats.sessionCount > 0 ? Math.round((stats.warmupCount / stats.sessionCount) * 100) : 0;
|
|
183
|
+
const toolPct = stats.sessionCount > 0 ? Math.round((stats.toolActivitySessions / stats.sessionCount) * 100) : 0;
|
|
184
|
+
const lines = [
|
|
185
|
+
`SESSION COVERAGE: ${stats.sessionCount} total, ${productiveCount} non-warmup, ${stats.warmupCount} warmup (${warmupPct}%)`,
|
|
186
|
+
`FACET COVERAGE: ${Math.round((flags.facetCoverage / 100) * stats.sessionCount)}/${stats.sessionCount} (${flags.facetCoverage}%) — qualitative signals from this subset only`,
|
|
187
|
+
"",
|
|
188
|
+
"── TIER 1: QUANTITATIVE (from session-meta, all sessions) ──",
|
|
189
|
+
`TOOL ACTIVITY: ${stats.toolActivitySessions} sessions with tool use (${toolPct}%)`,
|
|
190
|
+
`TOOL DISTRIBUTION: ${formatSortedRecord(stats.toolTotals)}`,
|
|
191
|
+
`TOKEN TOTALS: input=${stats.totalInputTokens}, output=${stats.totalOutputTokens}`,
|
|
192
|
+
`PROJECT DISTRIBUTION: ${formatSortedRecord(stats.projectDistribution)}`,
|
|
193
|
+
`GIT: ${stats.gitCommitsTotal} commits, ${stats.gitPushesTotal} pushes`,
|
|
194
|
+
];
|
|
195
|
+
if (flags.hasFacets) {
|
|
196
|
+
lines.push("", `── TIER 2: QUALITATIVE (from facets, ${flags.facetCoverage}% sample) ──`, `OUTCOME DISTRIBUTION: ${formatSortedRecord(stats.outcomes)}`, `ACHIEVED RATE: ${stats.achievedRate}%`, `SATISFACTION: ${formatSortedRecord(stats.satisfactionCounts)}`, `FRICTION TOTALS: ${formatSortedRecord(stats.frictionTotals)}`, `FRICTION TREND: first-half=${formatSortedRecord(stats.frictionFirstHalf)} second-half=${formatSortedRecord(stats.frictionSecondHalf)}`, `SESSION TYPES: ${formatSortedRecord(stats.sessionTypes)}`);
|
|
197
|
+
}
|
|
198
|
+
if (ruleScores) {
|
|
199
|
+
lines.push("", "── TIER 3: RULE ENGINE FEATURES ──", `RULE-BASED DOMAIN SCORES: ${ruleScores.map((d) => `${d.id}=${d.percentage}%`).join(", ")}`);
|
|
200
|
+
if (features) {
|
|
201
|
+
lines.push(`FEATURE INVENTORY: ${formatFeatureInventory(features)}`);
|
|
202
|
+
}
|
|
203
|
+
lines.push(`CONFIG: hooks=${config.hasCustomHooks}(${config.hookWithMatcherCount}), rules=${config.hasRulesFiles}(${config.rulesFileCount}), mcp=${config.hasMcpServers}, memory=${config.hasMemoryFiles}(${config.memoryFileCount}), agents=${config.hasCustomAgents}, skills=${config.hasCustomSkills}`);
|
|
204
|
+
}
|
|
205
|
+
lines.push("", "── COVERAGE FLAGS ──", `FACET_AVAILABLE: ${flags.hasFacets}`, `FACET_COVERAGE: ${flags.facetCoverage}%`, `SUFFICIENT_FOR_TRENDS: ${flags.sufficientForTrends}`);
|
|
206
|
+
return lines.join("\n");
|
|
207
|
+
}
|
|
208
|
+
// ── Helpers ──
|
|
209
|
+
function formatSortedRecord(rec) {
|
|
210
|
+
const entries = Object.entries(rec).sort((a, b) => b[1] - a[1]);
|
|
211
|
+
if (entries.length === 0)
|
|
212
|
+
return "{}";
|
|
213
|
+
return entries.map(([k, v]) => `${k}=${v}`).join(", ");
|
|
214
|
+
}
|
|
215
|
+
function formatFeatureInventory(f) {
|
|
216
|
+
const parts = [];
|
|
217
|
+
if (f.hooks.length > 0)
|
|
218
|
+
parts.push(`hooks=[${f.hooks.map((h) => `${h.name}(${h.count})`).join(",")}]`);
|
|
219
|
+
if (f.skills.length > 0)
|
|
220
|
+
parts.push(`skills=[${f.skills.map((s) => `${s.name}(${s.count})`).join(",")}]`);
|
|
221
|
+
if (f.mcpServers.length > 0)
|
|
222
|
+
parts.push(`mcpServers=[${f.mcpServers.join(",")}]`);
|
|
223
|
+
if (f.topTools.length > 0)
|
|
224
|
+
parts.push(`topTools=[${f.topTools.map((t) => `${t.name}(${t.count})`).join(",")}]`);
|
|
225
|
+
parts.push(`totalToolCalls=${f.totalToolCalls}`);
|
|
226
|
+
parts.push(`uniqueTools=${f.uniqueToolCount}`);
|
|
227
|
+
if (f.usedPlanMode)
|
|
228
|
+
parts.push("planMode=true");
|
|
229
|
+
if (f.hasMemory)
|
|
230
|
+
parts.push("memory=true");
|
|
231
|
+
if (f.hasRules)
|
|
232
|
+
parts.push("rules=true");
|
|
233
|
+
if (f.hasAgents)
|
|
234
|
+
parts.push("agents=true");
|
|
235
|
+
if (f.hasSkills)
|
|
236
|
+
parts.push("skills=true");
|
|
237
|
+
parts.push(`hours=${f.totalHours}`);
|
|
238
|
+
return parts.join(", ");
|
|
239
|
+
}
|
|
240
|
+
function stripNullBytes(raw) {
|
|
241
|
+
return raw.replace(/\0/g, "");
|
|
242
|
+
}
|
|
243
|
+
function incr(map, key) {
|
|
244
|
+
map[key] = (map[key] ?? 0) + 1;
|
|
245
|
+
}
|
|
246
|
+
function mergeInto(target, source) {
|
|
247
|
+
for (const [k, v] of Object.entries(source))
|
|
248
|
+
target[k] = (target[k] ?? 0) + v;
|
|
249
|
+
}
|
|
250
|
+
function readAllJson(dir) {
|
|
251
|
+
let files;
|
|
252
|
+
try {
|
|
253
|
+
files = (0, node_fs_1.readdirSync)(dir).filter((f) => f.endsWith(".json"));
|
|
254
|
+
}
|
|
255
|
+
catch {
|
|
256
|
+
return { valid: [], corrupt: 0 };
|
|
257
|
+
}
|
|
258
|
+
const valid = [];
|
|
259
|
+
let corrupt = 0;
|
|
260
|
+
for (const file of files) {
|
|
261
|
+
const parsed = readJsonSafe((0, node_path_1.join)(dir, file));
|
|
262
|
+
if (parsed !== null)
|
|
263
|
+
valid.push(parsed);
|
|
264
|
+
else
|
|
265
|
+
corrupt++;
|
|
266
|
+
}
|
|
267
|
+
return { valid, corrupt };
|
|
268
|
+
}
|
|
269
|
+
function aggregateFriction(sessions) {
|
|
270
|
+
const totals = {};
|
|
271
|
+
for (const s of sessions) {
|
|
272
|
+
if (s.facet?.friction_counts)
|
|
273
|
+
mergeInto(totals, s.facet.friction_counts);
|
|
274
|
+
}
|
|
275
|
+
return totals;
|
|
276
|
+
}
|
|
277
|
+
/** Compute AI achievement evidence from assembled evidence. */
|
|
278
|
+
function computeAIEvidence(evidence, stats) {
|
|
279
|
+
const hasFacets = evidence.sessions.some((s) => s.facet !== null);
|
|
280
|
+
// When no facets, achievedRate and frictionSatisfiedCount default to 0
|
|
281
|
+
if (!hasFacets) {
|
|
282
|
+
return { achievedRate: 0, frictionSatisfiedCount: 0, promptLengthTrend: computePromptTrend(evidence.sessions) };
|
|
283
|
+
}
|
|
284
|
+
// frictionSatisfiedCount: sessions with friction > 0 AND achieved outcome AND user satisfied
|
|
285
|
+
let frictionSatisfied = 0;
|
|
286
|
+
for (const s of evidence.sessions) {
|
|
287
|
+
if (!s.facet)
|
|
288
|
+
continue;
|
|
289
|
+
const hasFriction = s.facet.friction_counts && Object.values(s.facet.friction_counts).reduce((a, b) => a + b, 0) > 0;
|
|
290
|
+
const achieved = s.facet.outcome === "fully_achieved" || s.facet.outcome === "mostly_achieved";
|
|
291
|
+
const satisfied = s.facet.user_satisfaction_counts?.["likely_satisfied"];
|
|
292
|
+
if (hasFriction && achieved && satisfied && satisfied > 0)
|
|
293
|
+
frictionSatisfied++;
|
|
294
|
+
}
|
|
295
|
+
return { achievedRate: stats.achievedRate, frictionSatisfiedCount: frictionSatisfied, promptLengthTrend: computePromptTrend(evidence.sessions) };
|
|
296
|
+
}
|
|
297
|
+
function computePromptTrend(sessions) {
|
|
298
|
+
const prompts = sessions.map((s) => (s.facet?.first_prompt ?? s.meta.first_prompt ?? "").trim()).filter((p) => p.length > 0);
|
|
299
|
+
if (prompts.length < 20)
|
|
300
|
+
return 0;
|
|
301
|
+
const firstAvg = prompts.slice(0, 10).reduce((sum, p) => sum + p.split(/\s+/).length, 0) / 10;
|
|
302
|
+
const recentAvg = prompts.slice(-10).reduce((sum, p) => sum + p.split(/\s+/).length, 0) / 10;
|
|
303
|
+
return recentAvg - firstAvg;
|
|
304
|
+
}
|
|
305
|
+
//# sourceMappingURL=ai-evidence.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"ai-evidence.js","sourceRoot":"","sources":["../../src/scoring/ai-evidence.ts"],"names":[],"mappings":";;AAkHA,oCAOC;AAED,kDAIC;AAED,sDAOC;AAED,gCAIC;AAED,4CAgDC;AAED,0CAyEC;AAED,oDA2DC;AAiED,8CAuBC;AAhaD,qCAAoD;AACpD,yCAAiC;AA2GjC,MAAM,oBAAoB,GAA2B;IACnD,sBAAsB,EAAE,yBAAyB;CAClD,CAAC;AAEF,yBAAyB;AAEzB,SAAgB,YAAY,CAAC,IAAY;IACvC,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,cAAc,CAAC,IAAA,sBAAY,EAAC,IAAI,EAAE,OAAO,CAAC,CAAC,CAAC;QACxD,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAA4B,CAAC;IACpD,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED,SAAgB,mBAAmB,CAAC,OAAe;IACjD,IAAI,CAAC,OAAO;QAAE,OAAO,SAAS,CAAC;IAC/B,MAAM,QAAQ,GAAG,OAAO,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IACxE,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,IAAI,SAAS,CAAC,CAAC,OAAO,CAAC,kBAAkB,EAAE,GAAG,CAAC,IAAI,SAAS,CAAC;AACpG,CAAC;AAED,SAAgB,qBAAqB,CAAC,MAA8B;IAClE,MAAM,GAAG,GAA2B,EAAE,CAAC;IACvC,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;QAClD,MAAM,CAAC,GAAG,oBAAoB,CAAC,GAAG,CAAC,IAAI,GAAG,CAAC;QAC3C,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,KAAK,CAAC;IACjC,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,SAAgB,UAAU,CAAC,SAAiB;IAC1C,IAAI,SAAS,GAAG,EAAE;QAAE,OAAO,cAAc,CAAC;IAC1C,IAAI,SAAS,IAAI,EAAE;QAAE,OAAO,OAAO,CAAC;IACpC,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAgB,gBAAgB,CAAC,OAAe,EAAE,SAAkB;IAClE,MAAM,KAAK,GAAG,WAAW,CAAC,OAAO,CAAC,CAAC;IACnC,IAAI,KAAK,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IAE1C,iDAAiD;IACjD,MAAM,YAAY,GAAG,SAAS,CAAC,CAAC,CAAC,WAAW,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,EAAE,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC;IACpF,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAqB,CAAC;IAC9C,KAAK,MAAM,GAAG,IAAI,YAAY,CAAC,KAAK,EAAE,CAAC;QACrC,MAAM,KAAK,GAAG,GAA2B,CAAC;QAC1C,IAAI,CAAC,KAAK,CAAC,UAAU;YAAE,SAAS;QAChC,IAAI,KAAK,CAAC,eAAe;YAAE,KAAK,CAAC,eAAe,GAAG,qBAAqB,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC;QAChG,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,UAAU,EAAE,KAAK,CAAC,CAAC;IACxC,CAAC;IAED,oDAAoD;IACpD,MAAM,QAAQ,GAAqB,EAAE,CAAC;IACtC,IAAI,WAAW,GAAG,CAAC,CAAC;IACpB,KAAK,MAAM,GAAG,IAAI,KAAK,CAAC,KAAK,EAAE,CAAC;QAC9B,MAAM,IAAI,GAAG,GAAqC,CAAC;QACnD,IAAI,CAAC,IAAI,CAAC,UAAU;YAAE,SAAS;QAC/B,MAAM,KAAK,GAAG,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,IAAI,CAAC;QACpD,MAAM,MAAM,GAAG,CAAC,IAAI,CAAC,YAAY,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;QAChD,IAAI,oBAAoB,CAAC,IAAI,CAAC,MAAM,CAAC;YAAE,WAAW,EAAE,CAAC;QACrD,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,OAAO,EAAE,mBAAmB,CAAC,IAAI,CAAC,YAAY,IAAI,EAAE,CAAC,EAAE,CAAC,CAAC;IACxF,CAAC;IAED,2CAA2C;IAC3C,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;QACrB,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,CAAC,UAAU,IAAI,EAAE,CAAC;QACnC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,CAAC,UAAU,IAAI,EAAE,CAAC;QACnC,OAAO,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACxC,CAAC,CAAC,CAAC;IAEH,MAAM,eAAe,GAAG,QAAQ,CAAC,MAAM,GAAG,WAAW,CAAC;IACtD,MAAM,SAAS,GAAG,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC;IACrC,MAAM,UAAU,GAAG,QAAQ,CAAC,IAAI,CAAC;IACjC,MAAM,QAAQ,GAAG,SAAS,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,UAAU,GAAG,SAAS,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAEhF,OAAO;QACL,QAAQ;QACR,SAAS;QACT,UAAU;QACV,WAAW,EAAE,KAAK,CAAC,OAAO;QAC1B,aAAa,EAAE,YAAY,CAAC,OAAO;QACnC,eAAe;QACf,WAAW;QACX,QAAQ;KACT,CAAC;AACJ,CAAC;AAED,SAAgB,eAAe,CAAC,QAAyB;IACvD,MAAM,QAAQ,GAA2B,EAAE,CAAC;IAC5C,MAAM,cAAc,GAA2B,EAAE,CAAC;IAClD,MAAM,UAAU,GAA2B,EAAE,CAAC;IAC9C,MAAM,YAAY,GAA2B,EAAE,CAAC;IAChD,MAAM,kBAAkB,GAA2B,EAAE,CAAC;IACtD,MAAM,WAAW,GAA2B,EAAE,CAAC;IAC/C,IAAI,UAAU,GAAG,CAAC,EAAE,SAAS,GAAG,CAAC,EAAE,aAAa,GAAG,CAAC,EAAE,aAAa,GAAG,CAAC,CAAC;IACxE,IAAI,UAAU,GAAG,CAAC,EAAE,WAAW,GAAG,CAAC,EAAE,UAAU,GAAG,CAAC,EAAE,UAAU,GAAG,CAAC,EAAE,WAAW,GAAG,CAAC,CAAC;IAErF,KAAK,MAAM,EAAE,KAAK,EAAE,IAAI,EAAE,IAAI,QAAQ,CAAC,QAAQ,EAAE,CAAC;QAChD,IAAI,KAAK,EAAE,OAAO;YAAE,IAAI,CAAC,QAAQ,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC;QAClD,IAAI,KAAK,EAAE,YAAY;YAAE,IAAI,CAAC,YAAY,EAAE,KAAK,CAAC,YAAY,CAAC,CAAC;QAChE,IAAI,KAAK,EAAE,eAAe;YAAE,SAAS,CAAC,cAAc,EAAE,KAAK,CAAC,eAAe,CAAC,CAAC;QAC7E,IAAI,KAAK,EAAE,wBAAwB;YAAE,SAAS,CAAC,kBAAkB,EAAE,KAAK,CAAC,wBAAwB,CAAC,CAAC;QACnG,IAAI,KAAK,EAAE,kBAAkB;YAAE,IAAI,CAAC,WAAW,EAAE,KAAK,CAAC,kBAAkB,CAAC,CAAC;QAC3E,MAAM,MAAM,GAAG,KAAK,EAAE,YAAY,IAAI,IAAI,CAAC,YAAY,IAAI,EAAE,CAAC;QAC9D,IAAI,MAAM,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,KAAK,iBAAiB;YAAE,WAAW,EAAE,CAAC;QACrE,IAAI,IAAI,CAAC,WAAW;YAAE,SAAS,CAAC,UAAU,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;QAC9D,UAAU,IAAI,IAAI,CAAC,WAAW,IAAI,CAAC,CAAC;QACpC,SAAS,IAAI,IAAI,CAAC,UAAU,IAAI,CAAC,CAAC;QAClC,IAAI,IAAI,CAAC,gBAAgB,IAAI,IAAI,CAAC,gBAAgB,GAAG,CAAC,EAAE,CAAC;YAAC,aAAa,IAAI,IAAI,CAAC,gBAAgB,CAAC;YAAC,aAAa,EAAE,CAAC;QAAC,CAAC;QACpH,UAAU,IAAI,IAAI,CAAC,YAAY,IAAI,CAAC,CAAC;QACrC,WAAW,IAAI,IAAI,CAAC,aAAa,IAAI,CAAC,CAAC;QACvC,UAAU,IAAI,IAAI,CAAC,WAAW,IAAI,CAAC,CAAC;QACpC,UAAU,IAAI,IAAI,CAAC,cAAc,IAAI,CAAC,CAAC;IACzC,CAAC;IAED,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IACtD,MAAM,QAAQ,GAAG,CAAC,QAAQ,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,QAAQ,CAAC,iBAAiB,CAAC,IAAI,CAAC,CAAC,CAAC;IACxF,MAAM,aAAa,GAAG,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;IACzE,MAAM,SAAS,GAAG,kBAAkB,CAAC,kBAAkB,CAAC,IAAI,CAAC,CAAC;IAC9D,MAAM,QAAQ,GAAG,MAAM,CAAC,MAAM,CAAC,kBAAkB,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;IAC9E,MAAM,SAAS,GAAG,QAAQ,CAAC,SAAS,GAAG,QAAQ,CAAC,WAAW,CAAC;IAE5D,mDAAmD;IACnD,MAAM,mBAAmB,GAA2B,EAAE,CAAC;IACvD,IAAI,oBAAoB,GAAG,CAAC,CAAC;IAC7B,IAAI,gBAAgB,GAAG,CAAC,CAAC;IACzB,KAAK,MAAM,CAAC,IAAI,QAAQ,CAAC,QAAQ,EAAE,CAAC;QAClC,IAAI,CAAC,mBAAmB,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC;QACrC,MAAM,QAAQ,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,WAAW,IAAI,EAAE,CAAC,CAAC;QACvD,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAAC,oBAAoB,EAAE,CAAC;YAAC,gBAAgB,IAAI,QAAQ,CAAC,MAAM,CAAC;QAAC,CAAC;IAC3F,CAAC;IACD,MAAM,gBAAgB,GAAG,oBAAoB,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,gBAAgB,GAAG,oBAAoB,CAAC,GAAG,EAAE,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;IACxH,MAAM,eAAe,GAAG,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,KAAK,IAAI,CAAC,CAAC,MAAM,CAAC;IACjF,MAAM,eAAe,GAAG,SAAS,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,eAAe,GAAG,SAAS,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC5F,MAAM,aAAa,GAAkB;QACnC,SAAS,EAAE,QAAQ,CAAC,UAAU,GAAG,CAAC;QAClC,aAAa,EAAE,eAAe;QAC9B,mBAAmB,EAAE,QAAQ,CAAC,UAAU,IAAI,EAAE;KAC/C,CAAC;IAEF,OAAO;QACL,QAAQ,EAAE,YAAY,EAAE,aAAa,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,QAAQ,GAAG,aAAa,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QAC5F,cAAc;QACd,iBAAiB,EAAE,iBAAiB,CAAC,QAAQ,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;QACtE,kBAAkB,EAAE,iBAAiB,CAAC,QAAQ,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QACpE,UAAU,EAAE,YAAY;QACxB,eAAe,EAAE,UAAU,EAAE,cAAc,EAAE,SAAS;QACtD,kBAAkB,EAAE,aAAa,EAAE,QAAQ,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,SAAS,GAAG,QAAQ,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QAC9F,WAAW;QACX,kBAAkB,EAAE,aAAa,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,aAAa,GAAG,aAAa,CAAC,CAAC,CAAC,CAAC,CAAC;QACrF,gBAAgB,EAAE,UAAU,EAAE,iBAAiB,EAAE,WAAW;QAC5D,eAAe,EAAE,UAAU,EAAE,kBAAkB,EAAE,UAAU;QAC3D,YAAY,EAAE,QAAQ,CAAC,QAAQ,CAAC,MAAM;QACtC,eAAe;QACf,WAAW;QACX,mBAAmB;QACnB,oBAAoB;QACpB,gBAAgB;QAChB,aAAa;KACd,CAAC;AACJ,CAAC;AAED,SAAgB,oBAAoB,CAClC,KAAuB,EACvB,MAAqB,EACrB,UAA0B,EAC1B,QAA2B;IAE3B,MAAM,KAAK,GAAG,KAAK,CAAC,aAAa,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,aAAa,EAAE,CAAC,EAAE,mBAAmB,EAAE,KAAK,EAAE,CAAC;IACxG,MAAM,eAAe,GAAG,KAAK,CAAC,YAAY,GAAG,KAAK,CAAC,WAAW,CAAC;IAC/D,MAAM,SAAS,GAAG,KAAK,CAAC,YAAY,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,WAAW,GAAG,KAAK,CAAC,YAAY,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC1G,MAAM,OAAO,GAAG,KAAK,CAAC,YAAY,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,oBAAoB,GAAG,KAAK,CAAC,YAAY,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAEjH,MAAM,KAAK,GAAa;QACtB,qBAAqB,KAAK,CAAC,YAAY,WAAW,eAAe,gBAAgB,KAAK,CAAC,WAAW,YAAY,SAAS,IAAI;QAC3H,mBAAmB,IAAI,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,aAAa,GAAG,GAAG,CAAC,GAAG,KAAK,CAAC,YAAY,CAAC,IAAI,KAAK,CAAC,YAAY,KAAK,KAAK,CAAC,aAAa,gDAAgD;QAC7K,EAAE;QACF,8DAA8D;QAC9D,kBAAkB,KAAK,CAAC,oBAAoB,4BAA4B,OAAO,IAAI;QACnF,sBAAsB,kBAAkB,CAAC,KAAK,CAAC,UAAU,CAAC,EAAE;QAC5D,uBAAuB,KAAK,CAAC,gBAAgB,YAAY,KAAK,CAAC,iBAAiB,EAAE;QAClF,yBAAyB,kBAAkB,CAAC,KAAK,CAAC,mBAAmB,CAAC,EAAE;QACxE,QAAQ,KAAK,CAAC,eAAe,aAAa,KAAK,CAAC,cAAc,SAAS;KACxE,CAAC;IAEF,IAAI,KAAK,CAAC,SAAS,EAAE,CAAC;QACpB,KAAK,CAAC,IAAI,CACR,EAAE,EACF,wCAAwC,KAAK,CAAC,aAAa,cAAc,EACzE,yBAAyB,kBAAkB,CAAC,KAAK,CAAC,QAAQ,CAAC,EAAE,EAC7D,kBAAkB,KAAK,CAAC,YAAY,GAAG,EACvC,iBAAiB,kBAAkB,CAAC,KAAK,CAAC,kBAAkB,CAAC,EAAE,EAC/D,oBAAoB,kBAAkB,CAAC,KAAK,CAAC,cAAc,CAAC,EAAE,EAC9D,8BAA8B,kBAAkB,CAAC,KAAK,CAAC,iBAAiB,CAAC,gBAAgB,kBAAkB,CAAC,KAAK,CAAC,kBAAkB,CAAC,EAAE,EACvI,kBAAkB,kBAAkB,CAAC,KAAK,CAAC,YAAY,CAAC,EAAE,CAC3D,CAAC;IACJ,CAAC;IAED,IAAI,UAAU,EAAE,CAAC;QACf,KAAK,CAAC,IAAI,CACR,EAAE,EACF,oCAAoC,EACpC,6BAA6B,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,UAAU,GAAG,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAC5F,CAAC;QACF,IAAI,QAAQ,EAAE,CAAC;YACb,KAAK,CAAC,IAAI,CAAC,sBAAsB,sBAAsB,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;QACvE,CAAC;QACD,KAAK,CAAC,IAAI,CACR,iBAAiB,MAAM,CAAC,cAAc,IAAI,MAAM,CAAC,oBAAoB,YAAY,MAAM,CAAC,aAAa,IAAI,MAAM,CAAC,cAAc,UAAU,MAAM,CAAC,aAAa,YAAY,MAAM,CAAC,cAAc,IAAI,MAAM,CAAC,eAAe,aAAa,MAAM,CAAC,eAAe,YAAY,MAAM,CAAC,eAAe,EAAE,CAC/R,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,IAAI,CACR,EAAE,EACF,sBAAsB,EACtB,oBAAoB,KAAK,CAAC,SAAS,EAAE,EACrC,mBAAmB,KAAK,CAAC,aAAa,GAAG,EACzC,0BAA0B,KAAK,CAAC,mBAAmB,EAAE,CACtD,CAAC;IAEF,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC;AAED,gBAAgB;AAEhB,SAAS,kBAAkB,CAAC,GAA2B;IACrD,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAChE,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IACtC,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AACzD,CAAC;AAED,SAAS,sBAAsB,CAAC,CAAmB;IACjD,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,IAAI,CAAC,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC;QAAE,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IACvG,IAAI,CAAC,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC;QAAE,KAAK,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IAC1G,IAAI,CAAC,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC;QAAE,KAAK,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IAClF,IAAI,CAAC,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC;QAAE,KAAK,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IAChH,KAAK,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC,cAAc,EAAE,CAAC,CAAC;IACjD,KAAK,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC,eAAe,EAAE,CAAC,CAAC;IAC/C,IAAI,CAAC,CAAC,YAAY;QAAE,KAAK,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;IAChD,IAAI,CAAC,CAAC,SAAS;QAAE,KAAK,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;IAC3C,IAAI,CAAC,CAAC,QAAQ;QAAE,KAAK,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;IACzC,IAAI,CAAC,CAAC,SAAS;QAAE,KAAK,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;IAC3C,IAAI,CAAC,CAAC,SAAS;QAAE,KAAK,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;IAC3C,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,UAAU,EAAE,CAAC,CAAC;IACpC,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC;AAED,SAAS,cAAc,CAAC,GAAW;IACjC,OAAO,GAAG,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;AAChC,CAAC;AAED,SAAS,IAAI,CAAC,GAA2B,EAAE,GAAW;IACpD,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;AACjC,CAAC;AAED,SAAS,SAAS,CAAC,MAA8B,EAAE,MAA8B;IAC/E,KAAK,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC;QAAE,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;AAChF,CAAC;AAED,SAAS,WAAW,CAAC,GAAW;IAC9B,IAAI,KAAe,CAAC;IACpB,IAAI,CAAC;QACH,KAAK,GAAG,IAAA,qBAAW,EAAC,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC;IAC9D,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,KAAK,EAAE,EAAE,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC;IACnC,CAAC;IACD,MAAM,KAAK,GAA8B,EAAE,CAAC;IAC5C,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,MAAM,GAAG,YAAY,CAAC,IAAA,gBAAI,EAAC,GAAG,EAAE,IAAI,CAAC,CAAC,CAAC;QAC7C,IAAI,MAAM,KAAK,IAAI;YAAE,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;;YACnC,OAAO,EAAE,CAAC;IACjB,CAAC;IACD,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC;AAC5B,CAAC;AAED,SAAS,iBAAiB,CAAC,QAA0B;IACnD,MAAM,MAAM,GAA2B,EAAE,CAAC;IAC1C,KAAK,MAAM,CAAC,IAAI,QAAQ,EAAE,CAAC;QACzB,IAAI,CAAC,CAAC,KAAK,EAAE,eAAe;YAAE,SAAS,CAAC,MAAM,EAAE,CAAC,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC;IAC3E,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,+DAA+D;AAC/D,SAAgB,iBAAiB,CAAC,QAAyB,EAAE,KAAuB;IAKlF,MAAM,SAAS,GAAG,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,KAAK,IAAI,CAAC,CAAC;IAElE,uEAAuE;IACvE,IAAI,CAAC,SAAS,EAAE,CAAC;QACf,OAAO,EAAE,YAAY,EAAE,CAAC,EAAE,sBAAsB,EAAE,CAAC,EAAE,iBAAiB,EAAE,kBAAkB,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC;IAClH,CAAC;IAED,6FAA6F;IAC7F,IAAI,iBAAiB,GAAG,CAAC,CAAC;IAC1B,KAAK,MAAM,CAAC,IAAI,QAAQ,CAAC,QAAQ,EAAE,CAAC;QAClC,IAAI,CAAC,CAAC,CAAC,KAAK;YAAE,SAAS;QACvB,MAAM,WAAW,GAAG,CAAC,CAAC,KAAK,CAAC,eAAe,IAAI,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC;QACrH,MAAM,QAAQ,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,KAAK,gBAAgB,IAAI,CAAC,CAAC,KAAK,CAAC,OAAO,KAAK,iBAAiB,CAAC;QAC/F,MAAM,SAAS,GAAG,CAAC,CAAC,KAAK,CAAC,wBAAwB,EAAE,CAAC,kBAAkB,CAAC,CAAC;QACzE,IAAI,WAAW,IAAI,QAAQ,IAAI,SAAS,IAAI,SAAS,GAAG,CAAC;YAAE,iBAAiB,EAAE,CAAC;IACjF,CAAC;IAED,OAAO,EAAE,YAAY,EAAE,KAAK,CAAC,YAAY,EAAE,sBAAsB,EAAE,iBAAiB,EAAE,iBAAiB,EAAE,kBAAkB,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC;AACnJ,CAAC;AAED,SAAS,kBAAkB,CAAC,QAA0B;IACpD,MAAM,OAAO,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,KAAK,EAAE,YAAY,IAAI,CAAC,CAAC,IAAI,CAAC,YAAY,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IAC7H,IAAI,OAAO,CAAC,MAAM,GAAG,EAAE;QAAE,OAAO,CAAC,CAAC;IAClC,MAAM,QAAQ,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC;IAC9F,MAAM,SAAS,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC;IAC7F,OAAO,SAAS,GAAG,QAAQ,CAAC;AAC9B,CAAC"}
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
import type { AIGradingResult, ParsedRubric } from "../types.js";
|
|
2
|
+
interface AICriterionResponse {
|
|
3
|
+
q: string;
|
|
4
|
+
score: number;
|
|
5
|
+
evidence: string;
|
|
6
|
+
}
|
|
7
|
+
interface AIDomainResponse {
|
|
8
|
+
id: string;
|
|
9
|
+
criteria: AICriterionResponse[];
|
|
10
|
+
}
|
|
11
|
+
export interface AICriteriaResponse {
|
|
12
|
+
domains: AIDomainResponse[];
|
|
13
|
+
}
|
|
14
|
+
export declare const AI_GRADING_SCHEMA: {
|
|
15
|
+
type: string;
|
|
16
|
+
properties: {
|
|
17
|
+
domains: {
|
|
18
|
+
type: string;
|
|
19
|
+
items: {
|
|
20
|
+
type: string;
|
|
21
|
+
properties: {
|
|
22
|
+
id: {
|
|
23
|
+
type: string;
|
|
24
|
+
enum: string[];
|
|
25
|
+
};
|
|
26
|
+
criteria: {
|
|
27
|
+
type: string;
|
|
28
|
+
items: {
|
|
29
|
+
type: string;
|
|
30
|
+
properties: {
|
|
31
|
+
q: {
|
|
32
|
+
type: string;
|
|
33
|
+
};
|
|
34
|
+
score: {
|
|
35
|
+
type: string;
|
|
36
|
+
minimum: number;
|
|
37
|
+
maximum: number;
|
|
38
|
+
};
|
|
39
|
+
evidence: {
|
|
40
|
+
type: string;
|
|
41
|
+
};
|
|
42
|
+
};
|
|
43
|
+
required: string[];
|
|
44
|
+
};
|
|
45
|
+
};
|
|
46
|
+
};
|
|
47
|
+
required: string[];
|
|
48
|
+
};
|
|
49
|
+
};
|
|
50
|
+
};
|
|
51
|
+
required: string[];
|
|
52
|
+
};
|
|
53
|
+
export declare function checkClaudeVersion(): boolean;
|
|
54
|
+
export declare function parseVersionSatisfies(versionOutput: string): boolean;
|
|
55
|
+
export interface GradeResult {
|
|
56
|
+
response: AICriteriaResponse;
|
|
57
|
+
actualModel: string;
|
|
58
|
+
}
|
|
59
|
+
export declare function gradeWithClaude(rubricText: string, evidencePayload: string, model?: string): GradeResult | null;
|
|
60
|
+
export declare function parseCriteriaResponse(raw: string): AICriteriaResponse | null;
|
|
61
|
+
export declare function postProcess(response: AICriteriaResponse, rubric: ParsedRubric, model: string, rubricVersion: string): AIGradingResult;
|
|
62
|
+
export declare function computeCacheKey(metaMtime: string, rubricText: string, metaCount: number, configSignals: string, ruleScoresHash: string): string;
|
|
63
|
+
export {};
|
|
64
|
+
//# sourceMappingURL=ai-grader.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"ai-grader.d.ts","sourceRoot":"","sources":["../../src/scoring/ai-grader.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAA8B,eAAe,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAI7F,UAAU,mBAAmB;IAC3B,CAAC,EAAE,MAAM,CAAC;IACV,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED,UAAU,gBAAgB;IACxB,EAAE,EAAE,MAAM,CAAC;IACX,QAAQ,EAAE,mBAAmB,EAAE,CAAC;CACjC;AAED,MAAM,WAAW,kBAAkB;IACjC,OAAO,EAAE,gBAAgB,EAAE,CAAC;CAC7B;AAID,eAAO,MAAM,iBAAiB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA2B7B,CAAC;AAeF,wBAAgB,kBAAkB,IAAI,OAAO,CAW5C;AAED,wBAAgB,qBAAqB,CAAC,aAAa,EAAE,MAAM,GAAG,OAAO,CAUpE;AAID,MAAM,WAAW,WAAW;IAC1B,QAAQ,EAAE,kBAAkB,CAAC;IAC7B,WAAW,EAAE,MAAM,CAAC;CACrB;AAED,wBAAgB,eAAe,CAC7B,UAAU,EAAE,MAAM,EAClB,eAAe,EAAE,MAAM,EACvB,KAAK,CAAC,EAAE,MAAM,GACb,WAAW,GAAG,IAAI,CA6BpB;AAgBD,wBAAgB,qBAAqB,CAAC,GAAG,EAAE,MAAM,GAAG,kBAAkB,GAAG,IAAI,CAsC5E;AAID,wBAAgB,WAAW,CACzB,QAAQ,EAAE,kBAAkB,EAC5B,MAAM,EAAE,YAAY,EACpB,KAAK,EAAE,MAAM,EACb,aAAa,EAAE,MAAM,GACpB,eAAe,CA8DjB;AAYD,wBAAgB,eAAe,CAC7B,SAAS,EAAE,MAAM,EACjB,UAAU,EAAE,MAAM,EAClB,SAAS,EAAE,MAAM,EACjB,aAAa,EAAE,MAAM,EACrB,cAAc,EAAE,MAAM,GACrB,MAAM,CAQR"}
|
|
@@ -0,0 +1,229 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.AI_GRADING_SCHEMA = void 0;
|
|
4
|
+
exports.checkClaudeVersion = checkClaudeVersion;
|
|
5
|
+
exports.parseVersionSatisfies = parseVersionSatisfies;
|
|
6
|
+
exports.gradeWithClaude = gradeWithClaude;
|
|
7
|
+
exports.parseCriteriaResponse = parseCriteriaResponse;
|
|
8
|
+
exports.postProcess = postProcess;
|
|
9
|
+
exports.computeCacheKey = computeCacheKey;
|
|
10
|
+
const node_child_process_1 = require("node:child_process");
|
|
11
|
+
const node_crypto_1 = require("node:crypto");
|
|
12
|
+
// ── JSON Schema for claude --json-schema ──
|
|
13
|
+
exports.AI_GRADING_SCHEMA = {
|
|
14
|
+
type: "object",
|
|
15
|
+
properties: {
|
|
16
|
+
domains: {
|
|
17
|
+
type: "array",
|
|
18
|
+
items: {
|
|
19
|
+
type: "object",
|
|
20
|
+
properties: {
|
|
21
|
+
id: { type: "string", enum: ["goal-achievement", "collaboration-quality", "workflow-mastery", "growth-learning", "verification-quality"] },
|
|
22
|
+
criteria: {
|
|
23
|
+
type: "array",
|
|
24
|
+
items: {
|
|
25
|
+
type: "object",
|
|
26
|
+
properties: {
|
|
27
|
+
q: { type: "string" },
|
|
28
|
+
score: { type: "integer", minimum: 1, maximum: 5 },
|
|
29
|
+
evidence: { type: "string" },
|
|
30
|
+
},
|
|
31
|
+
required: ["q", "score", "evidence"],
|
|
32
|
+
},
|
|
33
|
+
},
|
|
34
|
+
},
|
|
35
|
+
required: ["id", "criteria"],
|
|
36
|
+
},
|
|
37
|
+
},
|
|
38
|
+
},
|
|
39
|
+
required: ["domains"],
|
|
40
|
+
};
|
|
41
|
+
const SYSTEM_PROMPT = "You are a Claude Code proficiency evaluator. Follow the rubric in the user message exactly. Return JSON only. Domain IDs must be exactly: goal-achievement, collaboration-quality, workflow-mastery, growth-learning, verification-quality.";
|
|
42
|
+
const VALID_DOMAIN_IDS = new Set([
|
|
43
|
+
"goal-achievement",
|
|
44
|
+
"collaboration-quality",
|
|
45
|
+
"workflow-mastery",
|
|
46
|
+
"growth-learning",
|
|
47
|
+
"verification-quality",
|
|
48
|
+
]);
|
|
49
|
+
// ── Version Check ──
|
|
50
|
+
function checkClaudeVersion() {
|
|
51
|
+
try {
|
|
52
|
+
const raw = (0, node_child_process_1.execFileSync)("claude", ["--version"], {
|
|
53
|
+
encoding: "utf-8",
|
|
54
|
+
stdio: ["pipe", "pipe", "pipe"],
|
|
55
|
+
timeout: 10_000,
|
|
56
|
+
});
|
|
57
|
+
return parseVersionSatisfies(raw.trim());
|
|
58
|
+
}
|
|
59
|
+
catch {
|
|
60
|
+
return false;
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
function parseVersionSatisfies(versionOutput) {
|
|
64
|
+
// claude --version may output something like "claude 2.3.1" or just "2.3.1"
|
|
65
|
+
const match = versionOutput.match(/(\d+)\.(\d+)\.(\d+)/);
|
|
66
|
+
if (!match)
|
|
67
|
+
return false;
|
|
68
|
+
const major = parseInt(match[1], 10);
|
|
69
|
+
const minor = parseInt(match[2], 10);
|
|
70
|
+
// Require >= 2.1.0
|
|
71
|
+
if (major > 2)
|
|
72
|
+
return true;
|
|
73
|
+
if (major === 2 && minor >= 1)
|
|
74
|
+
return true;
|
|
75
|
+
return false;
|
|
76
|
+
}
|
|
77
|
+
function gradeWithClaude(rubricText, evidencePayload, model) {
|
|
78
|
+
const userPrompt = `${rubricText}\n\n---\n\nEVIDENCE:\n${evidencePayload}`;
|
|
79
|
+
try {
|
|
80
|
+
const result = (0, node_child_process_1.execFileSync)("claude", [
|
|
81
|
+
"-p",
|
|
82
|
+
"--output-format", "json",
|
|
83
|
+
"--json-schema", JSON.stringify(exports.AI_GRADING_SCHEMA),
|
|
84
|
+
"--model", model ?? "sonnet",
|
|
85
|
+
"--system-prompt", SYSTEM_PROMPT,
|
|
86
|
+
"--no-session-persistence",
|
|
87
|
+
"--tools", "",
|
|
88
|
+
], {
|
|
89
|
+
input: userPrompt,
|
|
90
|
+
encoding: "utf-8",
|
|
91
|
+
stdio: ["pipe", "pipe", "pipe"],
|
|
92
|
+
timeout: 300_000,
|
|
93
|
+
maxBuffer: 10 * 1024 * 1024,
|
|
94
|
+
});
|
|
95
|
+
const parsed = parseCriteriaResponse(result);
|
|
96
|
+
if (!parsed)
|
|
97
|
+
return null;
|
|
98
|
+
// Extract actual model from claude's modelUsage response
|
|
99
|
+
const actualModel = extractActualModel(result, model ?? "sonnet");
|
|
100
|
+
return { response: parsed, actualModel };
|
|
101
|
+
}
|
|
102
|
+
catch {
|
|
103
|
+
return null;
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
function extractActualModel(raw, fallback) {
|
|
107
|
+
try {
|
|
108
|
+
const obj = JSON.parse(raw);
|
|
109
|
+
const usage = obj["modelUsage"];
|
|
110
|
+
if (usage) {
|
|
111
|
+
const keys = Object.keys(usage);
|
|
112
|
+
if (keys.length > 0)
|
|
113
|
+
return keys[0];
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
catch { /* ignore */ }
|
|
117
|
+
return fallback;
|
|
118
|
+
}
|
|
119
|
+
// ── Parse & Validate Response ──
|
|
120
|
+
function parseCriteriaResponse(raw) {
|
|
121
|
+
let parsed;
|
|
122
|
+
try {
|
|
123
|
+
parsed = JSON.parse(raw);
|
|
124
|
+
}
|
|
125
|
+
catch {
|
|
126
|
+
return null;
|
|
127
|
+
}
|
|
128
|
+
if (!parsed || typeof parsed !== "object")
|
|
129
|
+
return null;
|
|
130
|
+
const obj = parsed;
|
|
131
|
+
// Handle wrapper: claude --output-format json wraps in { result, structured_output }
|
|
132
|
+
// --json-schema puts data in structured_output; plain mode puts it in result
|
|
133
|
+
const inner = (obj["structured_output"] !== undefined
|
|
134
|
+
? obj["structured_output"]
|
|
135
|
+
: obj["result"] !== undefined
|
|
136
|
+
? obj["result"]
|
|
137
|
+
: obj);
|
|
138
|
+
if (!inner || typeof inner !== "object")
|
|
139
|
+
return null;
|
|
140
|
+
if (!Array.isArray(inner["domains"]))
|
|
141
|
+
return null;
|
|
142
|
+
const domains = inner["domains"];
|
|
143
|
+
for (const d of domains) {
|
|
144
|
+
if (!d || typeof d !== "object")
|
|
145
|
+
return null;
|
|
146
|
+
const domain = d;
|
|
147
|
+
if (typeof domain["id"] !== "string")
|
|
148
|
+
return null;
|
|
149
|
+
if (!Array.isArray(domain["criteria"]))
|
|
150
|
+
return null;
|
|
151
|
+
for (const c of domain["criteria"]) {
|
|
152
|
+
if (!c || typeof c !== "object")
|
|
153
|
+
return null;
|
|
154
|
+
const criterion = c;
|
|
155
|
+
if (typeof criterion["q"] !== "string")
|
|
156
|
+
return null;
|
|
157
|
+
if (typeof criterion["score"] !== "number")
|
|
158
|
+
return null;
|
|
159
|
+
if (typeof criterion["evidence"] !== "string")
|
|
160
|
+
return null;
|
|
161
|
+
}
|
|
162
|
+
}
|
|
163
|
+
return inner;
|
|
164
|
+
}
|
|
165
|
+
// ── Post-Processing (Deterministic) ──
|
|
166
|
+
function postProcess(response, rubric, model, rubricVersion) {
|
|
167
|
+
const gradedDomains = [];
|
|
168
|
+
for (const domain of response.domains) {
|
|
169
|
+
if (!VALID_DOMAIN_IDS.has(domain.id)) {
|
|
170
|
+
throw new Error(`Unknown domain: ${domain.id}`);
|
|
171
|
+
}
|
|
172
|
+
const expected = rubric.domains.find(d => d.id === domain.id);
|
|
173
|
+
if (!expected) {
|
|
174
|
+
throw new Error(`Unknown domain: ${domain.id}`);
|
|
175
|
+
}
|
|
176
|
+
if (domain.criteria.length !== expected.criteria.length) {
|
|
177
|
+
throw new Error(`Domain ${domain.id}: expected ${expected.criteria.length} criteria, got ${domain.criteria.length}`);
|
|
178
|
+
}
|
|
179
|
+
for (let i = 0; i < domain.criteria.length; i++) {
|
|
180
|
+
const score = domain.criteria[i].score;
|
|
181
|
+
if (score < 1 || score > 5 || !Number.isInteger(score)) {
|
|
182
|
+
throw new Error(`Domain ${domain.id} Q${i + 1}: score ${score} out of range 1-5`);
|
|
183
|
+
}
|
|
184
|
+
}
|
|
185
|
+
const n = expected.criteria.length;
|
|
186
|
+
const rawPoints = domain.criteria.reduce((sum, c) => sum + c.score, 0);
|
|
187
|
+
// Normalize: all-1s = 0%, all-5s = 100%
|
|
188
|
+
const total = Math.round(((rawPoints - n) / (n * 4)) * 100);
|
|
189
|
+
const level = total < 34 ? "novice" : total < 67 ? "proficient" : "expert";
|
|
190
|
+
gradedDomains.push({
|
|
191
|
+
id: domain.id,
|
|
192
|
+
criteria: domain.criteria.map(c => ({
|
|
193
|
+
question: c.q,
|
|
194
|
+
score: c.score,
|
|
195
|
+
evidence: c.evidence,
|
|
196
|
+
})),
|
|
197
|
+
total,
|
|
198
|
+
level,
|
|
199
|
+
summary: `${expected.label}: ${level} (${total}%)`,
|
|
200
|
+
});
|
|
201
|
+
}
|
|
202
|
+
// Validate all 5 domains are present
|
|
203
|
+
if (gradedDomains.length !== rubric.domains.length) {
|
|
204
|
+
throw new Error(`Expected ${rubric.domains.length} domains, got ${gradedDomains.length}`);
|
|
205
|
+
}
|
|
206
|
+
return {
|
|
207
|
+
domains: gradedDomains,
|
|
208
|
+
overall: computeOverallSummary(gradedDomains),
|
|
209
|
+
model,
|
|
210
|
+
gradedAt: new Date().toISOString(),
|
|
211
|
+
rubricVersion,
|
|
212
|
+
};
|
|
213
|
+
}
|
|
214
|
+
function computeOverallSummary(domains) {
|
|
215
|
+
const avg = Math.round(domains.reduce((sum, d) => sum + d.total, 0) / domains.length);
|
|
216
|
+
const level = avg < 34 ? "novice" : avg < 67 ? "proficient" : "expert";
|
|
217
|
+
return `Overall: ${level} (${avg}% average)`;
|
|
218
|
+
}
|
|
219
|
+
// ── Cache Key ──
|
|
220
|
+
function computeCacheKey(metaMtime, rubricText, metaCount, configSignals, ruleScoresHash) {
|
|
221
|
+
const hash = (0, node_crypto_1.createHash)("sha256");
|
|
222
|
+
hash.update(metaMtime);
|
|
223
|
+
hash.update(rubricText);
|
|
224
|
+
hash.update(String(metaCount));
|
|
225
|
+
hash.update(configSignals);
|
|
226
|
+
hash.update(ruleScoresHash);
|
|
227
|
+
return hash.digest("hex");
|
|
228
|
+
}
|
|
229
|
+
//# sourceMappingURL=ai-grader.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"ai-grader.js","sourceRoot":"","sources":["../../src/scoring/ai-grader.ts"],"names":[],"mappings":";;;AAkEA,gDAWC;AAED,sDAUC;AASD,0CAiCC;AAgBD,sDAsCC;AAID,kCAmEC;AAYD,0CAcC;AA1RD,2DAAkD;AAClD,6CAAyC;AAqBzC,6CAA6C;AAEhC,QAAA,iBAAiB,GAAG;IAC/B,IAAI,EAAE,QAAQ;IACd,UAAU,EAAE;QACV,OAAO,EAAE;YACP,IAAI,EAAE,OAAO;YACb,KAAK,EAAE;gBACL,IAAI,EAAE,QAAQ;gBACd,UAAU,EAAE;oBACV,EAAE,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,kBAAkB,EAAE,uBAAuB,EAAE,kBAAkB,EAAE,iBAAiB,EAAE,sBAAsB,CAAC,EAAE;oBAC1I,QAAQ,EAAE;wBACR,IAAI,EAAE,OAAO;wBACb,KAAK,EAAE;4BACL,IAAI,EAAE,QAAQ;4BACd,UAAU,EAAE;gCACV,CAAC,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;gCACrB,KAAK,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE;gCAClD,QAAQ,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;6BAC7B;4BACD,QAAQ,EAAE,CAAC,GAAG,EAAE,OAAO,EAAE,UAAU,CAAC;yBACrC;qBACF;iBACF;gBACD,QAAQ,EAAE,CAAC,IAAI,EAAE,UAAU,CAAC;aAC7B;SACF;KACF;IACD,QAAQ,EAAE,CAAC,SAAS,CAAC;CACtB,CAAC;AAEF,MAAM,aAAa,GACjB,6OAA6O,CAAC;AAEhP,MAAM,gBAAgB,GAAwB,IAAI,GAAG,CAAS;IAC5D,kBAAkB;IAClB,uBAAuB;IACvB,kBAAkB;IAClB,iBAAiB;IACjB,sBAAsB;CACvB,CAAC,CAAC;AAEH,sBAAsB;AAEtB,SAAgB,kBAAkB;IAChC,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,IAAA,iCAAY,EAAC,QAAQ,EAAE,CAAC,WAAW,CAAC,EAAE;YAChD,QAAQ,EAAE,OAAO;YACjB,KAAK,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC;YAC/B,OAAO,EAAE,MAAM;SAChB,CAAC,CAAC;QACH,OAAO,qBAAqB,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC;IAC3C,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC;AAED,SAAgB,qBAAqB,CAAC,aAAqB;IACzD,4EAA4E;IAC5E,MAAM,KAAK,GAAG,aAAa,CAAC,KAAK,CAAC,qBAAqB,CAAC,CAAC;IACzD,IAAI,CAAC,KAAK;QAAE,OAAO,KAAK,CAAC;IACzB,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IACrC,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IACrC,mBAAmB;IACnB,IAAI,KAAK,GAAG,CAAC;QAAE,OAAO,IAAI,CAAC;IAC3B,IAAI,KAAK,KAAK,CAAC,IAAI,KAAK,IAAI,CAAC;QAAE,OAAO,IAAI,CAAC;IAC3C,OAAO,KAAK,CAAC;AACf,CAAC;AASD,SAAgB,eAAe,CAC7B,UAAkB,EAClB,eAAuB,EACvB,KAAc;IAEd,MAAM,UAAU,GAAG,GAAG,UAAU,yBAAyB,eAAe,EAAE,CAAC;IAE3E,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,IAAA,iCAAY,EAAC,QAAQ,EAAE;YACpC,IAAI;YACJ,iBAAiB,EAAE,MAAM;YACzB,eAAe,EAAE,IAAI,CAAC,SAAS,CAAC,yBAAiB,CAAC;YAClD,SAAS,EAAE,KAAK,IAAI,QAAQ;YAC5B,iBAAiB,EAAE,aAAa;YAChC,0BAA0B;YAC1B,SAAS,EAAE,EAAE;SACd,EAAE;YACD,KAAK,EAAE,UAAU;YACjB,QAAQ,EAAE,OAAO;YACjB,KAAK,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC;YAC/B,OAAO,EAAE,OAAO;YAChB,SAAS,EAAE,EAAE,GAAG,IAAI,GAAG,IAAI;SAC5B,CAAC,CAAC;QAEH,MAAM,MAAM,GAAG,qBAAqB,CAAC,MAAM,CAAC,CAAC;QAC7C,IAAI,CAAC,MAAM;YAAE,OAAO,IAAI,CAAC;QAEzB,yDAAyD;QACzD,MAAM,WAAW,GAAG,kBAAkB,CAAC,MAAM,EAAE,KAAK,IAAI,QAAQ,CAAC,CAAC;QAClE,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,WAAW,EAAE,CAAC;IAC3C,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED,SAAS,kBAAkB,CAAC,GAAW,EAAE,QAAgB;IACvD,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAA4B,CAAC;QACvD,MAAM,KAAK,GAAG,GAAG,CAAC,YAAY,CAAwC,CAAC;QACvE,IAAI,KAAK,EAAE,CAAC;YACV,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAChC,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC;gBAAE,OAAO,IAAI,CAAC,CAAC,CAAC,CAAC;QACtC,CAAC;IACH,CAAC;IAAC,MAAM,CAAC,CAAC,YAAY,CAAC,CAAC;IACxB,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,kCAAkC;AAElC,SAAgB,qBAAqB,CAAC,GAAW;IAC/C,IAAI,MAAe,CAAC;IACpB,IAAI,CAAC;QACH,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IAC3B,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;IAED,IAAI,CAAC,MAAM,IAAI,OAAO,MAAM,KAAK,QAAQ;QAAE,OAAO,IAAI,CAAC;IACvD,MAAM,GAAG,GAAG,MAAiC,CAAC;IAE9C,qFAAqF;IACrF,6EAA6E;IAC7E,MAAM,KAAK,GAAG,CAAC,GAAG,CAAC,mBAAmB,CAAC,KAAK,SAAS;QACnD,CAAC,CAAC,GAAG,CAAC,mBAAmB,CAAC;QAC1B,CAAC,CAAC,GAAG,CAAC,QAAQ,CAAC,KAAK,SAAS;YAC3B,CAAC,CAAC,GAAG,CAAC,QAAQ,CAAC;YACf,CAAC,CAAC,GAAG,CAA4B,CAAC;IACtC,IAAI,CAAC,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ;QAAE,OAAO,IAAI,CAAC;IAErD,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;QAAE,OAAO,IAAI,CAAC;IAClD,MAAM,OAAO,GAAG,KAAK,CAAC,SAAS,CAAc,CAAC;IAE9C,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;QACxB,IAAI,CAAC,CAAC,IAAI,OAAO,CAAC,KAAK,QAAQ;YAAE,OAAO,IAAI,CAAC;QAC7C,MAAM,MAAM,GAAG,CAA4B,CAAC;QAC5C,IAAI,OAAO,MAAM,CAAC,IAAI,CAAC,KAAK,QAAQ;YAAE,OAAO,IAAI,CAAC;QAClD,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;YAAE,OAAO,IAAI,CAAC;QACpD,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,UAAU,CAAc,EAAE,CAAC;YAChD,IAAI,CAAC,CAAC,IAAI,OAAO,CAAC,KAAK,QAAQ;gBAAE,OAAO,IAAI,CAAC;YAC7C,MAAM,SAAS,GAAG,CAA4B,CAAC;YAC/C,IAAI,OAAO,SAAS,CAAC,GAAG,CAAC,KAAK,QAAQ;gBAAE,OAAO,IAAI,CAAC;YACpD,IAAI,OAAO,SAAS,CAAC,OAAO,CAAC,KAAK,QAAQ;gBAAE,OAAO,IAAI,CAAC;YACxD,IAAI,OAAO,SAAS,CAAC,UAAU,CAAC,KAAK,QAAQ;gBAAE,OAAO,IAAI,CAAC;QAC7D,CAAC;IACH,CAAC;IAED,OAAO,KAAsC,CAAC;AAChD,CAAC;AAED,wCAAwC;AAExC,SAAgB,WAAW,CACzB,QAA4B,EAC5B,MAAoB,EACpB,KAAa,EACb,aAAqB;IAErB,MAAM,aAAa,GAAqB,EAAE,CAAC;IAE3C,KAAK,MAAM,MAAM,IAAI,QAAQ,CAAC,OAAO,EAAE,CAAC;QACtC,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,EAAE,CAAC;YACrC,MAAM,IAAI,KAAK,CAAC,mBAAmB,MAAM,CAAC,EAAE,EAAE,CAAC,CAAC;QAClD,CAAC;QAED,MAAM,QAAQ,GAAG,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,MAAM,CAAC,EAAE,CAAC,CAAC;QAC9D,IAAI,CAAC,QAAQ,EAAE,CAAC;YACd,MAAM,IAAI,KAAK,CAAC,mBAAmB,MAAM,CAAC,EAAE,EAAE,CAAC,CAAC;QAClD,CAAC;QAED,IAAI,MAAM,CAAC,QAAQ,CAAC,MAAM,KAAK,QAAQ,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC;YACxD,MAAM,IAAI,KAAK,CACb,UAAU,MAAM,CAAC,EAAE,cAAc,QAAQ,CAAC,QAAQ,CAAC,MAAM,kBAAkB,MAAM,CAAC,QAAQ,CAAC,MAAM,EAAE,CACpG,CAAC;QACJ,CAAC;QAED,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAChD,MAAM,KAAK,GAAG,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC;YACvC,IAAI,KAAK,GAAG,CAAC,IAAI,KAAK,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,EAAE,CAAC;gBACvD,MAAM,IAAI,KAAK,CACb,UAAU,MAAM,CAAC,EAAE,KAAK,CAAC,GAAG,CAAC,WAAW,KAAK,mBAAmB,CACjE,CAAC;YACJ,CAAC;QACH,CAAC;QAED,MAAM,CAAC,GAAG,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC;QACnC,MAAM,SAAS,GAAG,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;QACvE,wCAAwC;QACxC,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC;QAC5D,MAAM,KAAK,GACT,KAAK,GAAG,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,KAAK,GAAG,EAAE,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,QAAQ,CAAC;QAE/D,aAAa,CAAC,IAAI,CAAC;YACjB,EAAE,EAAE,MAAM,CAAC,EAAgB;YAC3B,QAAQ,EAAE,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;gBAClC,QAAQ,EAAE,CAAC,CAAC,CAAC;gBACb,KAAK,EAAE,CAAC,CAAC,KAAK;gBACd,QAAQ,EAAE,CAAC,CAAC,QAAQ;aACrB,CAAC,CAAC;YACH,KAAK;YACL,KAAK;YACL,OAAO,EAAE,GAAG,QAAQ,CAAC,KAAK,KAAK,KAAK,KAAK,KAAK,IAAI;SACnD,CAAC,CAAC;IACL,CAAC;IAED,qCAAqC;IACrC,IAAI,aAAa,CAAC,MAAM,KAAK,MAAM,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC;QACnD,MAAM,IAAI,KAAK,CACb,YAAY,MAAM,CAAC,OAAO,CAAC,MAAM,iBAAiB,aAAa,CAAC,MAAM,EAAE,CACzE,CAAC;IACJ,CAAC;IAED,OAAO;QACL,OAAO,EAAE,aAAa;QACtB,OAAO,EAAE,qBAAqB,CAAC,aAAa,CAAC;QAC7C,KAAK;QACL,QAAQ,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QAClC,aAAa;KACd,CAAC;AACJ,CAAC;AAED,SAAS,qBAAqB,CAAC,OAAyB;IACtD,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CACpB,OAAO,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,GAAG,OAAO,CAAC,MAAM,CAC9D,CAAC;IACF,MAAM,KAAK,GAAG,GAAG,GAAG,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,GAAG,GAAG,EAAE,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,QAAQ,CAAC;IACvE,OAAO,YAAY,KAAK,KAAK,GAAG,YAAY,CAAC;AAC/C,CAAC;AAED,kBAAkB;AAElB,SAAgB,eAAe,CAC7B,SAAiB,EACjB,UAAkB,EAClB,SAAiB,EACjB,aAAqB,EACrB,cAAsB;IAEtB,MAAM,IAAI,GAAG,IAAA,wBAAU,EAAC,QAAQ,CAAC,CAAC;IAClC,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;IACvB,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;IACxB,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC;IAC/B,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC;IAC3B,IAAI,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC;IAC5B,OAAO,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;AAC5B,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"rubric-loader.d.ts","sourceRoot":"","sources":["../../src/scoring/rubric-loader.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAc,YAAY,EAA6C,MAAM,aAAa,CAAC;AAsFvG,wBAAgB,UAAU,IAAI,YAAY,CAMzC;AAED,wBAAgB,qBAAqB,CAAC,MAAM,EAAE,YAAY,GAAG,MAAM,CAalE"}
|