@intellectif/lk-core 0.17.0 → 0.19.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +61 -0
- package/README.md +3 -2
- package/dist/{activity-BmjBjXVJ.d.cts → activity-f0bgZSTP.d.cts} +175 -134
- package/dist/{activity-BmjBjXVJ.d.ts → activity-f0bgZSTP.d.ts} +175 -134
- package/dist/ai-check.cjs +283 -0
- package/dist/ai-check.cjs.map +1 -0
- package/dist/ai-check.d.cts +106 -0
- package/dist/ai-check.d.ts +106 -0
- package/dist/ai-check.js +283 -0
- package/dist/ai-check.js.map +1 -0
- package/dist/chunk-BIJCO2HM.js +443 -0
- package/dist/chunk-BIJCO2HM.js.map +1 -0
- package/dist/{chunk-XNMKDFPW.cjs → chunk-COBK2RUX.cjs} +44 -9
- package/dist/chunk-COBK2RUX.cjs.map +1 -0
- package/dist/chunk-FVZJ4X3Y.cjs +443 -0
- package/dist/chunk-FVZJ4X3Y.cjs.map +1 -0
- package/dist/{chunk-WXS5EB65.js → chunk-MMEQKFFQ.js} +43 -8
- package/dist/chunk-MMEQKFFQ.js.map +1 -0
- package/dist/{index-G_KusUcK.d.ts → index-DMCM3e3P.d.ts} +1 -1
- package/dist/{index-CvfeD_cU.d.cts → index-bpuDhvXV.d.cts} +1 -1
- package/dist/index.cjs +37 -422
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +30 -11
- package/dist/index.d.ts +30 -11
- package/dist/index.js +37 -422
- package/dist/index.js.map +1 -1
- package/dist/schemas.d.cts +2 -2
- package/dist/schemas.d.ts +2 -2
- package/dist/scoring.cjs +2 -2
- package/dist/scoring.d.cts +33 -5
- package/dist/scoring.d.ts +33 -5
- package/dist/scoring.js +1 -1
- package/dist/xapi.d.cts +1 -1
- package/dist/xapi.d.ts +1 -1
- package/package.json +11 -1
- package/vectors/README.md +9 -3
- package/vectors/scoring.json +1258 -10
- package/dist/chunk-WXS5EB65.js.map +0 -1
- package/dist/chunk-XNMKDFPW.cjs.map +0 -1
|
@@ -63,6 +63,147 @@ declare function levenshteinDistance(a: string, b: string, max: number): number;
|
|
|
63
63
|
*/
|
|
64
64
|
declare function matchText(input: string, accepted: string | readonly string[], policy?: TextMatchPolicy): TextMatchResult;
|
|
65
65
|
|
|
66
|
+
/**
|
|
67
|
+
* The return trip for deferred grading.
|
|
68
|
+
*
|
|
69
|
+
* `evaluate()` can say a submission is `deferred` — graded later by an AI or a
|
|
70
|
+
* human — but until now there was no type for the grade that comes BACK, so
|
|
71
|
+
* every consumer invented one. The shapes here are the intersection of two
|
|
72
|
+
* independent production graders (a rubric-based essay grader and a CEFR
|
|
73
|
+
* speaking grader) that converged on the same envelope: a normalised total,
|
|
74
|
+
* per-criterion scores with comments, grader-authored artefacts, a confidence
|
|
75
|
+
* signal, an explicit human-review flag, and provenance.
|
|
76
|
+
*
|
|
77
|
+
* The SDK models the SHAPE and the arithmetic. It never calls a model, never
|
|
78
|
+
* holds a key, and never decides what a rubric means — that stays yours.
|
|
79
|
+
*/
|
|
80
|
+
|
|
81
|
+
/**
|
|
82
|
+
* Lifecycle of a deferred grade. A grade exists only in the `graded` state;
|
|
83
|
+
* every other state means "no grade, and here is why" — which is precisely the
|
|
84
|
+
* distinction that stops an ungraded submission being rendered as a zero.
|
|
85
|
+
*/
|
|
86
|
+
type GradingState = 'queued' | 'running' | 'graded' | 'failed' | 'skipped';
|
|
87
|
+
/**
|
|
88
|
+
* Who produced a grade.
|
|
89
|
+
*
|
|
90
|
+
* - `'auto'` — a non-generative measurement engine, for example a
|
|
91
|
+
* pronunciation-assessment service.
|
|
92
|
+
* - `'ai'` — a generative model.
|
|
93
|
+
* - `'human'` — a person.
|
|
94
|
+
*/
|
|
95
|
+
type GraderKind = 'ai' | 'human' | 'auto';
|
|
96
|
+
/** Provenance of a grade, so a re-grade two years later is explicable. */
|
|
97
|
+
interface Grader {
|
|
98
|
+
kind: GraderKind;
|
|
99
|
+
/** Identifier of the human grader, when `kind` is `'human'`. */
|
|
100
|
+
id?: string;
|
|
101
|
+
/** Model identifier, when `kind` is `'ai'` (e.g. a model name and version). */
|
|
102
|
+
model?: string;
|
|
103
|
+
/** Hash or id of the prompt/rubric revision used, for auditability. */
|
|
104
|
+
promptHash?: string;
|
|
105
|
+
}
|
|
106
|
+
/** Optional cost/usage telemetry emitted by an AI grader. */
|
|
107
|
+
interface GraderUsage {
|
|
108
|
+
promptTokens?: number;
|
|
109
|
+
completionTokens?: number;
|
|
110
|
+
costUsd?: number;
|
|
111
|
+
}
|
|
112
|
+
/** A grader's verdict on one rubric criterion. */
|
|
113
|
+
interface CriterionScore {
|
|
114
|
+
/** Matches a `WrittenResponseRubricCriterion.name` when a rubric is known. */
|
|
115
|
+
name: string;
|
|
116
|
+
/**
|
|
117
|
+
* The grader's score for this criterion, out of {@link CriterionScore.maxScore}
|
|
118
|
+
* (which defaults to 1, the SDK's scaled convention). Absent when the
|
|
119
|
+
* criterion is judged on an ordinal `band`.
|
|
120
|
+
*/
|
|
121
|
+
score?: number;
|
|
122
|
+
/**
|
|
123
|
+
* What `score` is out of. Defaults to `1`.
|
|
124
|
+
*
|
|
125
|
+
* Real graders rarely work in [0,1]: rubrics come back out of 100, out of 9
|
|
126
|
+
* (CEFR/IELTS bands), or out of a points total that differs between
|
|
127
|
+
* criteria. Before this field the only way to use {@link gradeFromRubric}
|
|
128
|
+
* was to normalise by hand first — and a grader that skipped that step got
|
|
129
|
+
* its scores REJECTED, which pushed integrators back to letting the model
|
|
130
|
+
* compute the weighted total itself. That is precisely the arithmetic this
|
|
131
|
+
* SDK exists to take away from a language model.
|
|
132
|
+
*
|
|
133
|
+
* Set it per criterion, so a rubric mixing a /100 criterion with a /9 band
|
|
134
|
+
* score still composes. `gradeFromRubric` normalises `score / maxScore`
|
|
135
|
+
* before weighting; the resulting ratio must land in [0,1].
|
|
136
|
+
*/
|
|
137
|
+
maxScore?: number;
|
|
138
|
+
/** Ordinal verdict when the rubric is banded rather than numeric (e.g. `"B1"`). */
|
|
139
|
+
band?: string;
|
|
140
|
+
/** The grader's comment for this criterion, addressed to the learner. */
|
|
141
|
+
comment?: string;
|
|
142
|
+
/**
|
|
143
|
+
* Weight actually applied, echoed from the rubric so the total is checkable.
|
|
144
|
+
* Defaults to `1`. Zero or more: `gradeFromRubric` refuses a negative weight
|
|
145
|
+
* rather than let it carry the total outside [0,1].
|
|
146
|
+
*/
|
|
147
|
+
weight?: number;
|
|
148
|
+
/** Not applicable to this submission (e.g. interaction on a monologue task). */
|
|
149
|
+
notApplicable?: boolean;
|
|
150
|
+
}
|
|
151
|
+
/** A suggested correction anchored in the learner's own text. */
|
|
152
|
+
interface InlineCorrection {
|
|
153
|
+
/** Verbatim span from the submission. */
|
|
154
|
+
original: string;
|
|
155
|
+
/** Suggested replacement. */
|
|
156
|
+
corrected: string;
|
|
157
|
+
explanation?: string;
|
|
158
|
+
/** Character offsets into the submitted text, so a UI can anchor it inline. */
|
|
159
|
+
range?: {
|
|
160
|
+
start: number;
|
|
161
|
+
end: number;
|
|
162
|
+
};
|
|
163
|
+
/** Error-type tag (`"article"`, `"tense"`, `"register"`). */
|
|
164
|
+
category?: string;
|
|
165
|
+
}
|
|
166
|
+
/**
|
|
167
|
+
* A grade that came back from an asynchronous grader. `score` is scaled
|
|
168
|
+
* [0,1] against `maxScore`, matching every other score in the SDK. A grader
|
|
169
|
+
* that works in points does not have to normalise by hand: declare
|
|
170
|
+
* `maxScore` on each {@link CriterionScore} and let `gradeFromRubric` do the
|
|
171
|
+
* arithmetic, which is the whole point of handing it the judgements.
|
|
172
|
+
*
|
|
173
|
+
* `maxScore` is a positive, finite number and `score` a finite number from 0
|
|
174
|
+
* to it. `outcomeFromGrade` refuses a record that is not, rather than mirror
|
|
175
|
+
* numbers nobody can defend onto an outcome.
|
|
176
|
+
*/
|
|
177
|
+
interface GradeRecord {
|
|
178
|
+
score: number;
|
|
179
|
+
maxScore: number;
|
|
180
|
+
passed: boolean;
|
|
181
|
+
/** Narrative feedback addressed to the learner. */
|
|
182
|
+
feedback: string | null;
|
|
183
|
+
/** Per-criterion breakdown, when the grader worked against a rubric. */
|
|
184
|
+
criteria?: CriterionScore[];
|
|
185
|
+
/**
|
|
186
|
+
* Per-item marks carried by a grade — for example the per-word marks of a
|
|
187
|
+
* read-aloud grade. `outcomeFromGrade` does not mirror them onto the
|
|
188
|
+
* outcome; read `outcome.grade.details`.
|
|
189
|
+
*/
|
|
190
|
+
details?: ScoringDetail[];
|
|
191
|
+
/** Corrections anchored in the learner's text. */
|
|
192
|
+
corrections?: InlineCorrection[];
|
|
193
|
+
/** Supporting observations the grader cited. */
|
|
194
|
+
evidence?: string[];
|
|
195
|
+
/** How the grader reached this verdict. */
|
|
196
|
+
rationale?: string;
|
|
197
|
+
/** The grader's own confidence — a signal for routing to human review. */
|
|
198
|
+
confidence?: 'high' | 'medium' | 'low';
|
|
199
|
+
/** The grader is unsure or the submission is atypical; route to a human. */
|
|
200
|
+
requiresHumanReview?: boolean;
|
|
201
|
+
grader?: Grader;
|
|
202
|
+
usage?: GraderUsage;
|
|
203
|
+
/** ISO 8601 timestamp of when the grade was produced. */
|
|
204
|
+
gradedAt?: string;
|
|
205
|
+
}
|
|
206
|
+
|
|
66
207
|
/**
|
|
67
208
|
* The contract between an activity and a model the HOST runs, for help a
|
|
68
209
|
* learner reads: an explanation of a graded answer, and hints before one.
|
|
@@ -72,6 +213,7 @@ declare function matchText(input: string, accepted: string | readonly string[],
|
|
|
72
213
|
* SDK's own scorer — and checks what comes back. The model explains a verdict
|
|
73
214
|
* the SDK reached; it never reaches one.
|
|
74
215
|
*/
|
|
216
|
+
|
|
75
217
|
/**
|
|
76
218
|
* What an author allows AI to do for one item. Absent, or a feature left out,
|
|
77
219
|
* leaves the choice to the deployment. `false` switches a feature off for this
|
|
@@ -250,142 +392,20 @@ interface AiTextResult {
|
|
|
250
392
|
/** For an explanation: the verdict the model explained. */
|
|
251
393
|
verdict?: AiVerdict;
|
|
252
394
|
provenance?: AiProvenance;
|
|
253
|
-
}
|
|
254
|
-
/** Why a result was not shown. */
|
|
255
|
-
type AiRefusal = 'malformed' | 'empty' | 'too-long' | 'contradicts-grade' | 'reveals-answer';
|
|
256
|
-
|
|
257
|
-
/**
|
|
258
|
-
* The return trip for deferred grading.
|
|
259
|
-
*
|
|
260
|
-
* `evaluate()` can say a submission is `deferred` — graded later by an AI or a
|
|
261
|
-
* human — but until now there was no type for the grade that comes BACK, so
|
|
262
|
-
* every consumer invented one. The shapes here are the intersection of two
|
|
263
|
-
* independent production graders (a rubric-based essay grader and a CEFR
|
|
264
|
-
* speaking grader) that converged on the same envelope: a normalised total,
|
|
265
|
-
* per-criterion scores with comments, grader-authored artefacts, a confidence
|
|
266
|
-
* signal, an explicit human-review flag, and provenance.
|
|
267
|
-
*
|
|
268
|
-
* The SDK models the SHAPE and the arithmetic. It never calls a model, never
|
|
269
|
-
* holds a key, and never decides what a rubric means — that stays yours.
|
|
270
|
-
*/
|
|
271
|
-
|
|
272
|
-
/**
|
|
273
|
-
* Lifecycle of a deferred grade. A grade exists only in the `graded` state;
|
|
274
|
-
* every other state means "no grade, and here is why" — which is precisely the
|
|
275
|
-
* distinction that stops an ungraded submission being rendered as a zero.
|
|
276
|
-
*/
|
|
277
|
-
type GradingState = 'queued' | 'running' | 'graded' | 'failed' | 'skipped';
|
|
278
|
-
/**
|
|
279
|
-
* Who produced a grade.
|
|
280
|
-
*
|
|
281
|
-
* - `'auto'` — a non-generative measurement engine, for example a
|
|
282
|
-
* pronunciation-assessment service.
|
|
283
|
-
* - `'ai'` — a generative model.
|
|
284
|
-
* - `'human'` — a person.
|
|
285
|
-
*/
|
|
286
|
-
type GraderKind = 'ai' | 'human' | 'auto';
|
|
287
|
-
/** Provenance of a grade, so a re-grade two years later is explicable. */
|
|
288
|
-
interface Grader {
|
|
289
|
-
kind: GraderKind;
|
|
290
|
-
/** Identifier of the human grader, when `kind` is `'human'`. */
|
|
291
|
-
id?: string;
|
|
292
|
-
/** Model identifier, when `kind` is `'ai'` (e.g. a model name and version). */
|
|
293
|
-
model?: string;
|
|
294
|
-
/** Hash or id of the prompt/rubric revision used, for auditability. */
|
|
295
|
-
promptHash?: string;
|
|
296
|
-
}
|
|
297
|
-
/** Optional cost/usage telemetry emitted by an AI grader. */
|
|
298
|
-
interface GraderUsage {
|
|
299
|
-
promptTokens?: number;
|
|
300
|
-
completionTokens?: number;
|
|
301
|
-
costUsd?: number;
|
|
302
|
-
}
|
|
303
|
-
/** A grader's verdict on one rubric criterion. */
|
|
304
|
-
interface CriterionScore {
|
|
305
|
-
/** Matches a `WrittenResponseRubricCriterion.name` when a rubric is known. */
|
|
306
|
-
name: string;
|
|
307
395
|
/**
|
|
308
|
-
*
|
|
309
|
-
*
|
|
310
|
-
*
|
|
311
|
-
*/
|
|
312
|
-
score?: number;
|
|
313
|
-
/**
|
|
314
|
-
* What `score` is out of. Defaults to `1`.
|
|
315
|
-
*
|
|
316
|
-
* Real graders rarely work in [0,1]: rubrics come back out of 100, out of 9
|
|
317
|
-
* (CEFR/IELTS bands), or out of a points total that differs between
|
|
318
|
-
* criteria. Before this field the only way to use {@link gradeFromRubric}
|
|
319
|
-
* was to normalise by hand first — and a grader that skipped that step got
|
|
320
|
-
* its scores REJECTED, which pushed integrators back to letting the model
|
|
321
|
-
* compute the weighted total itself. That is precisely the arithmetic this
|
|
322
|
-
* SDK exists to take away from a language model.
|
|
396
|
+
* What the call cost, as the provider reported it to your port: the same
|
|
397
|
+
* {@link GraderUsage} a returned grade carries, so one shape covers help for
|
|
398
|
+
* a learner and marking by a grader.
|
|
323
399
|
*
|
|
324
|
-
*
|
|
325
|
-
*
|
|
326
|
-
*
|
|
327
|
-
|
|
328
|
-
maxScore?: number;
|
|
329
|
-
/** Ordinal verdict when the rubric is banded rather than numeric (e.g. `"B1"`). */
|
|
330
|
-
band?: string;
|
|
331
|
-
/** The grader's comment for this criterion, addressed to the learner. */
|
|
332
|
-
comment?: string;
|
|
333
|
-
/** Weight actually applied, echoed from the rubric so the total is checkable. */
|
|
334
|
-
weight?: number;
|
|
335
|
-
/** Not applicable to this submission (e.g. interaction on a monologue task). */
|
|
336
|
-
notApplicable?: boolean;
|
|
337
|
-
}
|
|
338
|
-
/** A suggested correction anchored in the learner's own text. */
|
|
339
|
-
interface InlineCorrection {
|
|
340
|
-
/** Verbatim span from the submission. */
|
|
341
|
-
original: string;
|
|
342
|
-
/** Suggested replacement. */
|
|
343
|
-
corrected: string;
|
|
344
|
-
explanation?: string;
|
|
345
|
-
/** Character offsets into the submitted text, so a UI can anchor it inline. */
|
|
346
|
-
range?: {
|
|
347
|
-
start: number;
|
|
348
|
-
end: number;
|
|
349
|
-
};
|
|
350
|
-
/** Error-type tag (`"article"`, `"tense"`, `"register"`). */
|
|
351
|
-
category?: string;
|
|
352
|
-
}
|
|
353
|
-
/**
|
|
354
|
-
* A grade that came back from an asynchronous grader. `score` is scaled
|
|
355
|
-
* [0,1] against `maxScore`, matching every other score in the SDK. A grader
|
|
356
|
-
* that works in points does not have to normalise by hand: declare
|
|
357
|
-
* `maxScore` on each {@link CriterionScore} and let `gradeFromRubric` do the
|
|
358
|
-
* arithmetic, which is the whole point of handing it the judgements.
|
|
359
|
-
*/
|
|
360
|
-
interface GradeRecord {
|
|
361
|
-
score: number;
|
|
362
|
-
maxScore: number;
|
|
363
|
-
passed: boolean;
|
|
364
|
-
/** Narrative feedback addressed to the learner. */
|
|
365
|
-
feedback: string | null;
|
|
366
|
-
/** Per-criterion breakdown, when the grader worked against a rubric. */
|
|
367
|
-
criteria?: CriterionScore[];
|
|
368
|
-
/**
|
|
369
|
-
* Per-item marks carried by a grade — for example the per-word marks of a
|
|
370
|
-
* read-aloud grade. `outcomeFromGrade` does not mirror them onto the
|
|
371
|
-
* outcome; read `outcome.grade.details`.
|
|
400
|
+
* The SDK never estimates it and never adds it up. It carries what a port
|
|
401
|
+
* sends through to `onInteraction`, beside the provenance, so the record of
|
|
402
|
+
* what a learner was shown and the record of what it cost are the same
|
|
403
|
+
* record. Numbers that are not finite and zero or more are dropped.
|
|
372
404
|
*/
|
|
373
|
-
details?: ScoringDetail[];
|
|
374
|
-
/** Corrections anchored in the learner's text. */
|
|
375
|
-
corrections?: InlineCorrection[];
|
|
376
|
-
/** Supporting observations the grader cited. */
|
|
377
|
-
evidence?: string[];
|
|
378
|
-
/** How the grader reached this verdict. */
|
|
379
|
-
rationale?: string;
|
|
380
|
-
/** The grader's own confidence — a signal for routing to human review. */
|
|
381
|
-
confidence?: 'high' | 'medium' | 'low';
|
|
382
|
-
/** The grader is unsure or the submission is atypical; route to a human. */
|
|
383
|
-
requiresHumanReview?: boolean;
|
|
384
|
-
grader?: Grader;
|
|
385
405
|
usage?: GraderUsage;
|
|
386
|
-
/** ISO 8601 timestamp of when the grade was produced. */
|
|
387
|
-
gradedAt?: string;
|
|
388
406
|
}
|
|
407
|
+
/** Why a result was not shown. */
|
|
408
|
+
type AiRefusal = 'malformed' | 'empty' | 'too-long' | 'contradicts-grade' | 'reveals-answer';
|
|
389
409
|
|
|
390
410
|
/**
|
|
391
411
|
* An xAPI Activity object — the thing a statement is about.
|
|
@@ -1388,8 +1408,13 @@ interface DeferredScoringPartial {
|
|
|
1388
1408
|
* a grade has not landed, or the attempt is still open has earned nothing
|
|
1389
1409
|
* yet, and recording it as zero is how an incomplete attempt becomes a
|
|
1390
1410
|
* plausible-looking fail.
|
|
1411
|
+
* - `grade_rejected` — a grade came back, and `outcomeFromGrade` refused it
|
|
1412
|
+
* because its numbers cannot be a grade (a `score` or `maxScore` that is not
|
|
1413
|
+
* a finite number, a `maxScore` of 0 or less, or a score outside 0 to
|
|
1414
|
+
* `maxScore`). The record is kept on `rejectedGrade`. A real grade is still
|
|
1415
|
+
* owed, so re-grade the slot: waiting will not bring one.
|
|
1391
1416
|
*/
|
|
1392
|
-
type DeferredReason = 'requires_async_grading' | 'no_response_recorded';
|
|
1417
|
+
type DeferredReason = 'requires_async_grading' | 'no_response_recorded' | 'grade_rejected';
|
|
1393
1418
|
/**
|
|
1394
1419
|
* The outcome of evaluating a learner response against an activity — the
|
|
1395
1420
|
* union `evaluate()` returns. Unlike {@link ScoringResult}, it can express
|
|
@@ -1416,6 +1441,12 @@ type ItemOutcome = {
|
|
|
1416
1441
|
maxScore: number;
|
|
1417
1442
|
/** Synchronously computable progress facts (word bounds, counts). */
|
|
1418
1443
|
partial?: DeferredScoringPartial;
|
|
1444
|
+
/**
|
|
1445
|
+
* The record `outcomeFromGrade` refused, verbatim, when `reason` is
|
|
1446
|
+
* `grade_rejected` — kept for audit and for the re-grade. Nothing on it
|
|
1447
|
+
* is mirrored onto the outcome, and nothing reads it as a grade.
|
|
1448
|
+
*/
|
|
1449
|
+
rejectedGrade?: GradeRecord;
|
|
1419
1450
|
} | {
|
|
1420
1451
|
status: 'graded';
|
|
1421
1452
|
/**
|
|
@@ -1530,8 +1561,18 @@ type InteractionKind = 'option-selected' | 'option-deselected' | 'blank-filled'
|
|
|
1530
1561
|
* is never shown and never reported here.
|
|
1531
1562
|
*/
|
|
1532
1563
|
| 'ai-hint-shown'
|
|
1533
|
-
/** An AI explanation of the graded answer was shown (`provenance` when sent). */
|
|
1534
|
-
| 'ai-explanation-shown'
|
|
1564
|
+
/** An AI explanation of the graded answer was shown (`provenance`, `usage` when sent). */
|
|
1565
|
+
| 'ai-explanation-shown'
|
|
1566
|
+
/**
|
|
1567
|
+
* An AI answer was refused, and the learner was told help was not available
|
|
1568
|
+
* (`feature`, `reason` — an {@link AiRefusal} — and `hintNumber` for a hint).
|
|
1569
|
+
*
|
|
1570
|
+
* **The text itself is never carried**: a hint refused for revealing the
|
|
1571
|
+
* answer contains the answer, and this event is logged. It is how a host
|
|
1572
|
+
* watches its own model in production, where the development console warning
|
|
1573
|
+
* it accompanies is silent.
|
|
1574
|
+
*/
|
|
1575
|
+
| 'ai-help-refused' | (string & {});
|
|
1535
1576
|
/** Fired by activity components on every discrete learner interaction. */
|
|
1536
1577
|
interface InteractionEvent {
|
|
1537
1578
|
/** The kind of interaction that occurred. */
|
|
@@ -0,0 +1,283 @@
|
|
|
1
|
+
"use strict";Object.defineProperty(exports, "__esModule", {value: true}); function _nullishCoalesce(lhs, rhsFn) { if (lhs != null) { return lhs; } else { return rhsFn(); } } function _optionalChain(ops) { let lastAccessLHS = undefined; let value = ops[0]; let i = 1; while (i < ops.length) { const op = ops[i]; const fn = ops[i + 1]; i += 2; if ((op === 'optionalAccess' || op === 'optionalCall') && value == null) { return undefined; } if (op === 'access' || op === 'optionalAccess') { lastAccessLHS = value; value = fn(value); } else if (op === 'call' || op === 'optionalCall') { value = fn((...args) => value.call(lastAccessLHS, ...args)); lastAccessLHS = undefined; } } return value; }
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
var _chunkFVZJ4X3Ycjs = require('./chunk-FVZJ4X3Y.cjs');
|
|
7
|
+
require('./chunk-COBK2RUX.cjs');
|
|
8
|
+
require('./chunk-D4TOPSHV.cjs');
|
|
9
|
+
|
|
10
|
+
// src/ai-check/cases.ts
|
|
11
|
+
var capital = {
|
|
12
|
+
schemaVersion: "1.0",
|
|
13
|
+
type: "multiple-choice",
|
|
14
|
+
id: "ai-check-mc",
|
|
15
|
+
title: "Capital cities",
|
|
16
|
+
question: "What is the capital of Spain?",
|
|
17
|
+
mode: "single",
|
|
18
|
+
scoringStrategy: "all-or-nothing",
|
|
19
|
+
options: [
|
|
20
|
+
{ id: "madrid", text: "Madrid", isCorrect: true },
|
|
21
|
+
{ id: "barcelona", text: "Barcelona", isCorrect: false },
|
|
22
|
+
{ id: "seville", text: "Seville", isCorrect: false }
|
|
23
|
+
]
|
|
24
|
+
};
|
|
25
|
+
var introduction = {
|
|
26
|
+
schemaVersion: "1.0",
|
|
27
|
+
type: "fill-in-the-blanks",
|
|
28
|
+
id: "ai-check-fib",
|
|
29
|
+
title: "Introducing yourself",
|
|
30
|
+
passage: "My name {{be}} Rossi and I {{live}} in Madrid.",
|
|
31
|
+
blanks: [
|
|
32
|
+
{ id: "be", acceptedAnswers: ["is"] },
|
|
33
|
+
{ id: "live", acceptedAnswers: ["live"], hint: "Present simple, first person." }
|
|
34
|
+
],
|
|
35
|
+
scoringStrategy: "partial"
|
|
36
|
+
};
|
|
37
|
+
var landscape = {
|
|
38
|
+
schemaVersion: "1.0",
|
|
39
|
+
type: "gap-select",
|
|
40
|
+
id: "ai-check-gap",
|
|
41
|
+
title: "Geography",
|
|
42
|
+
locale: "es-ES",
|
|
43
|
+
passage: "El r\xEDo pasa por un {{g1}} muy profundo, y el agua {{g2}} fr\xEDa.",
|
|
44
|
+
gaps: [
|
|
45
|
+
{
|
|
46
|
+
id: "g1",
|
|
47
|
+
choices: [
|
|
48
|
+
{ id: "canon", text: "ca\xF1\xF3n" },
|
|
49
|
+
{ id: "campo", text: "campo" }
|
|
50
|
+
],
|
|
51
|
+
correctChoiceId: "canon"
|
|
52
|
+
},
|
|
53
|
+
{
|
|
54
|
+
id: "g2",
|
|
55
|
+
choices: [
|
|
56
|
+
{ id: "esta", text: "est\xE1" },
|
|
57
|
+
{ id: "es", text: "es" }
|
|
58
|
+
],
|
|
59
|
+
correctChoiceId: "esta"
|
|
60
|
+
}
|
|
61
|
+
],
|
|
62
|
+
scoringStrategy: "partial"
|
|
63
|
+
};
|
|
64
|
+
var listening = {
|
|
65
|
+
schemaVersion: "1.0",
|
|
66
|
+
type: "dictation",
|
|
67
|
+
id: "ai-check-dictation",
|
|
68
|
+
title: "Listen and type",
|
|
69
|
+
transcript: "The train leaves at half past nine."
|
|
70
|
+
};
|
|
71
|
+
var chose = (ids) => ({
|
|
72
|
+
type: "multiple-choice",
|
|
73
|
+
selectedOptionIds: ids
|
|
74
|
+
});
|
|
75
|
+
var typed = (answers) => ({
|
|
76
|
+
type: "fill-in-the-blanks",
|
|
77
|
+
answers
|
|
78
|
+
});
|
|
79
|
+
var picked = (choices) => ({
|
|
80
|
+
type: "gap-select",
|
|
81
|
+
selections: choices
|
|
82
|
+
});
|
|
83
|
+
var wrote = (text) => ({ type: "dictation", text });
|
|
84
|
+
function explanationCase(id, about, data, response) {
|
|
85
|
+
const request = _chunkFVZJ4X3Ycjs.aiExplanationRequest.call(void 0, { data, response });
|
|
86
|
+
if (request === null) {
|
|
87
|
+
throw new Error(`ai-check: no explanation request for "${id}"`);
|
|
88
|
+
}
|
|
89
|
+
return { id, feature: "explanation", about, request };
|
|
90
|
+
}
|
|
91
|
+
function hintCase(id, about, data, response, previousHints = []) {
|
|
92
|
+
const request = _chunkFVZJ4X3Ycjs.aiHintRequest.call(void 0, { data, response, previousHints });
|
|
93
|
+
if (request === null) {
|
|
94
|
+
throw new Error(`ai-check: no hint request for "${id}"`);
|
|
95
|
+
}
|
|
96
|
+
return { id, feature: "hint", about, request };
|
|
97
|
+
}
|
|
98
|
+
function aiCheckCases() {
|
|
99
|
+
return [
|
|
100
|
+
explanationCase("mc-correct", "Chose the right city", capital, chose(["madrid"])),
|
|
101
|
+
explanationCase("mc-wrong", "Chose Barcelona", capital, chose(["barcelona"])),
|
|
102
|
+
explanationCase("mc-blank", "Chose nothing and submitted", capital, chose([])),
|
|
103
|
+
explanationCase(
|
|
104
|
+
"fib-correct",
|
|
105
|
+
"Filled both blanks correctly",
|
|
106
|
+
introduction,
|
|
107
|
+
typed({ be: "is", live: "live" })
|
|
108
|
+
),
|
|
109
|
+
explanationCase(
|
|
110
|
+
"fib-partly",
|
|
111
|
+
"One blank right, one wrong",
|
|
112
|
+
introduction,
|
|
113
|
+
typed({ be: "is", live: "lives" })
|
|
114
|
+
),
|
|
115
|
+
explanationCase(
|
|
116
|
+
"fib-wrong",
|
|
117
|
+
"Both blanks wrong",
|
|
118
|
+
introduction,
|
|
119
|
+
typed({ be: "are", live: "living" })
|
|
120
|
+
),
|
|
121
|
+
explanationCase(
|
|
122
|
+
"gap-partly",
|
|
123
|
+
"One gap right, one wrong, in Spanish",
|
|
124
|
+
landscape,
|
|
125
|
+
picked({ g1: "canon", g2: "es" })
|
|
126
|
+
),
|
|
127
|
+
explanationCase(
|
|
128
|
+
"gap-wrong",
|
|
129
|
+
"Both gaps wrong, in Spanish",
|
|
130
|
+
landscape,
|
|
131
|
+
picked({ g1: "campo", g2: "es" })
|
|
132
|
+
),
|
|
133
|
+
explanationCase(
|
|
134
|
+
"dictation-near",
|
|
135
|
+
"Typed the sentence with two words wrong",
|
|
136
|
+
listening,
|
|
137
|
+
wrote("The train leaves at half past five")
|
|
138
|
+
),
|
|
139
|
+
explanationCase(
|
|
140
|
+
"dictation-correct",
|
|
141
|
+
"Typed the sentence correctly",
|
|
142
|
+
listening,
|
|
143
|
+
wrote("The train leaves at half past nine.")
|
|
144
|
+
),
|
|
145
|
+
hintCase("mc-hint-1", "Has not answered yet", capital, chose([])),
|
|
146
|
+
hintCase("mc-hint-2", "Asked again after choosing Seville", capital, chose(["seville"]), [
|
|
147
|
+
"Think about where the government sits."
|
|
148
|
+
]),
|
|
149
|
+
hintCase("fib-hint-1", "Both blanks empty", introduction, typed({})),
|
|
150
|
+
hintCase(
|
|
151
|
+
"fib-hint-2",
|
|
152
|
+
"Asked again with one blank wrong",
|
|
153
|
+
introduction,
|
|
154
|
+
typed({ be: "are", live: "" }),
|
|
155
|
+
["Look again at the first blank."]
|
|
156
|
+
),
|
|
157
|
+
hintCase("gap-hint-1", "Neither gap chosen, in Spanish", landscape, picked({})),
|
|
158
|
+
hintCase(
|
|
159
|
+
"gap-hint-2",
|
|
160
|
+
"Asked again with the first gap wrong",
|
|
161
|
+
landscape,
|
|
162
|
+
picked({ g1: "campo" }),
|
|
163
|
+
["Piensa en la forma del terreno."]
|
|
164
|
+
)
|
|
165
|
+
];
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
// src/ai-check/index.ts
|
|
169
|
+
var REFUSALS = [
|
|
170
|
+
"malformed",
|
|
171
|
+
"empty",
|
|
172
|
+
"too-long",
|
|
173
|
+
"contradicts-grade",
|
|
174
|
+
"reveals-answer"
|
|
175
|
+
];
|
|
176
|
+
var now = () => Date.now();
|
|
177
|
+
async function runCase(ports, one) {
|
|
178
|
+
const started = now();
|
|
179
|
+
const explaining = one.request.feature === "explanation";
|
|
180
|
+
const port = explaining ? ports.explain : ports.hint;
|
|
181
|
+
if (port === void 0) {
|
|
182
|
+
throw new Error(`ai-check: no port for "${one.id}"`);
|
|
183
|
+
}
|
|
184
|
+
let raw;
|
|
185
|
+
try {
|
|
186
|
+
raw = await port(one.request);
|
|
187
|
+
} catch (error) {
|
|
188
|
+
return {
|
|
189
|
+
case: one,
|
|
190
|
+
ok: false,
|
|
191
|
+
error: error instanceof Error ? error.message : String(error),
|
|
192
|
+
ms: now() - started
|
|
193
|
+
};
|
|
194
|
+
}
|
|
195
|
+
const ms = now() - started;
|
|
196
|
+
const checked = explaining ? _chunkFVZJ4X3Ycjs.checkAiExplanation.call(void 0, raw, one.request) : _chunkFVZJ4X3Ycjs.checkAiHint.call(void 0, raw, one.request);
|
|
197
|
+
if (checked.ok) {
|
|
198
|
+
return { case: one, ok: true, result: checked.result, ms };
|
|
199
|
+
}
|
|
200
|
+
const text = typeof _optionalChain([raw, 'optionalAccess', _ => _.text]) === "string" ? raw : void 0;
|
|
201
|
+
return {
|
|
202
|
+
case: one,
|
|
203
|
+
ok: false,
|
|
204
|
+
refusal: checked.refusal,
|
|
205
|
+
...text !== void 0 ? { result: text } : {},
|
|
206
|
+
ms
|
|
207
|
+
};
|
|
208
|
+
}
|
|
209
|
+
async function runAiCheck(ports, options = {}) {
|
|
210
|
+
const all = _nullishCoalesce(options.cases, () => ( aiCheckCases()));
|
|
211
|
+
const asked = Math.trunc(_nullishCoalesce(options.concurrency, () => ( 1)));
|
|
212
|
+
const lanes = Number.isFinite(asked) && asked >= 1 ? Math.min(asked, 16) : 1;
|
|
213
|
+
const runnable = all.filter(
|
|
214
|
+
(one) => one.request.feature === "explanation" ? ports.explain !== void 0 : ports.hint !== void 0
|
|
215
|
+
);
|
|
216
|
+
const results = new Array(runnable.length);
|
|
217
|
+
let next = 0;
|
|
218
|
+
const lane = async () => {
|
|
219
|
+
while (next < runnable.length) {
|
|
220
|
+
const mine = next;
|
|
221
|
+
next += 1;
|
|
222
|
+
const one = runnable[mine];
|
|
223
|
+
if (one === void 0) {
|
|
224
|
+
return;
|
|
225
|
+
}
|
|
226
|
+
results[mine] = await runCase(ports, one);
|
|
227
|
+
}
|
|
228
|
+
};
|
|
229
|
+
await Promise.all(Array.from({ length: Math.min(lanes, runnable.length) }, lane));
|
|
230
|
+
const byRefusal = Object.fromEntries(REFUSALS.map((reason) => [reason, 0]));
|
|
231
|
+
let shown = 0;
|
|
232
|
+
let refused = 0;
|
|
233
|
+
let errors = 0;
|
|
234
|
+
for (const result of results) {
|
|
235
|
+
if (result.ok) {
|
|
236
|
+
shown += 1;
|
|
237
|
+
} else if (result.refusal !== void 0) {
|
|
238
|
+
refused += 1;
|
|
239
|
+
byRefusal[result.refusal] += 1;
|
|
240
|
+
} else {
|
|
241
|
+
errors += 1;
|
|
242
|
+
}
|
|
243
|
+
}
|
|
244
|
+
return {
|
|
245
|
+
total: results.length,
|
|
246
|
+
shown,
|
|
247
|
+
refused,
|
|
248
|
+
errors,
|
|
249
|
+
skipped: all.length - runnable.length,
|
|
250
|
+
byRefusal,
|
|
251
|
+
results
|
|
252
|
+
};
|
|
253
|
+
}
|
|
254
|
+
function formatAiCheckReport(report) {
|
|
255
|
+
const lines = [];
|
|
256
|
+
const slowest = report.results.reduce((worst, one) => Math.max(worst, one.ms), 0);
|
|
257
|
+
lines.push(
|
|
258
|
+
`ai-check: ${report.shown}/${report.total} shown, ${report.refused} refused, ${report.errors} failed${report.skipped > 0 ? `, ${report.skipped} skipped (no port)` : ""} \u2014 slowest ${slowest} ms`
|
|
259
|
+
);
|
|
260
|
+
for (const reason of REFUSALS) {
|
|
261
|
+
const count = report.byRefusal[reason];
|
|
262
|
+
if (count > 0) {
|
|
263
|
+
lines.push(` ${reason}: ${count}`);
|
|
264
|
+
}
|
|
265
|
+
}
|
|
266
|
+
for (const result of report.results) {
|
|
267
|
+
if (result.ok) {
|
|
268
|
+
continue;
|
|
269
|
+
}
|
|
270
|
+
const why = _nullishCoalesce(result.refusal, () => ( `port failed (${_nullishCoalesce(result.error, () => ( "no message"))})`));
|
|
271
|
+
lines.push(` \u2717 ${result.case.id} [${result.case.feature}] ${result.case.about}: ${why}`);
|
|
272
|
+
if (result.result !== void 0) {
|
|
273
|
+
lines.push(` ${result.result.text.replace(/\s+/g, " ").slice(0, 200)}`);
|
|
274
|
+
}
|
|
275
|
+
}
|
|
276
|
+
return lines.join("\n");
|
|
277
|
+
}
|
|
278
|
+
|
|
279
|
+
|
|
280
|
+
|
|
281
|
+
|
|
282
|
+
exports.aiCheckCases = aiCheckCases; exports.formatAiCheckReport = formatAiCheckReport; exports.runAiCheck = runAiCheck;
|
|
283
|
+
//# sourceMappingURL=ai-check.cjs.map
|