@intellectif/lk-core 1.2.0 → 1.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +29 -0
- package/README.md +2 -2
- package/dist/ai-CnmLPMzd.d.cts +2417 -0
- package/dist/ai-CnmLPMzd.d.cts.map +1 -0
- package/dist/ai-CnmLPMzd.d.ts +2417 -0
- package/dist/ai-CnmLPMzd.d.ts.map +1 -0
- package/dist/ai-check.cjs +845 -620
- package/dist/ai-check.cjs.map +1 -1
- package/dist/ai-check.d.cts +82 -80
- package/dist/ai-check.d.cts.map +1 -0
- package/dist/ai-check.d.ts +82 -80
- package/dist/ai-check.d.ts.map +1 -0
- package/dist/ai-check.js +843 -621
- package/dist/ai-check.js.map +1 -1
- package/dist/ai-writing-CPpH9Qxq.js +2601 -0
- package/dist/ai-writing-CPpH9Qxq.js.map +1 -0
- package/dist/ai-writing-G6tnCGzw.cjs +2846 -0
- package/dist/ai-writing-G6tnCGzw.cjs.map +1 -0
- package/dist/index-2nigSf4N.d.cts +339 -0
- package/dist/index-2nigSf4N.d.cts.map +1 -0
- package/dist/index-B54baFUA.d.ts +339 -0
- package/dist/index-B54baFUA.d.ts.map +1 -0
- package/dist/index-BAfEHCnD.d.cts +1071 -0
- package/dist/index-BAfEHCnD.d.cts.map +1 -0
- package/dist/index-BqMXWuNC.d.cts +153 -0
- package/dist/index-BqMXWuNC.d.cts.map +1 -0
- package/dist/index-CVWvymVb.d.ts +1071 -0
- package/dist/index-CVWvymVb.d.ts.map +1 -0
- package/dist/index-CcHDTxIZ.d.ts +153 -0
- package/dist/index-CcHDTxIZ.d.ts.map +1 -0
- package/dist/index.cjs +1211 -1146
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +941 -861
- package/dist/index.d.cts.map +1 -0
- package/dist/index.d.ts +941 -861
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +1063 -1147
- package/dist/index.js.map +1 -1
- package/dist/item-group-DXiR8nOr.js +445 -0
- package/dist/item-group-DXiR8nOr.js.map +1 -0
- package/dist/item-group-DroFXgLb.cjs +505 -0
- package/dist/item-group-DroFXgLb.cjs.map +1 -0
- package/dist/registry-DVi0LR_i.cjs +5915 -0
- package/dist/registry-DVi0LR_i.cjs.map +1 -0
- package/dist/registry-Dsx9mNsS.js +5449 -0
- package/dist/registry-Dsx9mNsS.js.map +1 -0
- package/dist/schemas-Bdw0nGPo.js +152 -0
- package/dist/schemas-Bdw0nGPo.js.map +1 -0
- package/dist/schemas-DVbAOrtd.cjs +224 -0
- package/dist/schemas-DVbAOrtd.cjs.map +1 -0
- package/dist/schemas.cjs +16 -34
- package/dist/schemas.d.cts +2 -2
- package/dist/schemas.d.ts +2 -2
- package/dist/schemas.js +3 -34
- package/dist/scoring-B2_HEhMT.cjs +1748 -0
- package/dist/scoring-B2_HEhMT.cjs.map +1 -0
- package/dist/scoring-BZqUkuDF.js +1604 -0
- package/dist/scoring-BZqUkuDF.js.map +1 -0
- package/dist/scoring.cjs +37 -75
- package/dist/scoring.d.cts +3 -1103
- package/dist/scoring.d.ts +3 -1103
- package/dist/scoring.js +3 -75
- package/dist/xapi-CMTph7wG.js +332 -0
- package/dist/xapi-CMTph7wG.js.map +1 -0
- package/dist/xapi-DNzkSJ0S.cjs +362 -0
- package/dist/xapi-DNzkSJ0S.cjs.map +1 -0
- package/dist/xapi.cjs +7 -16
- package/dist/xapi.d.cts +2 -150
- package/dist/xapi.d.ts +2 -150
- package/dist/xapi.js +2 -16
- package/package.json +4 -4
- package/dist/activity-D705CXng.d.cts +0 -2162
- package/dist/activity-D705CXng.d.ts +0 -2162
- package/dist/chunk-46BXIGLG.cjs +0 -272
- package/dist/chunk-46BXIGLG.cjs.map +0 -1
- package/dist/chunk-4YANYTFN.cjs +0 -1256
- package/dist/chunk-4YANYTFN.cjs.map +0 -1
- package/dist/chunk-7KOWU4RO.js +0 -2207
- package/dist/chunk-7KOWU4RO.js.map +0 -1
- package/dist/chunk-7THXD7AN.cjs +0 -5057
- package/dist/chunk-7THXD7AN.cjs.map +0 -1
- package/dist/chunk-CYCTIXUN.cjs +0 -118
- package/dist/chunk-CYCTIXUN.cjs.map +0 -1
- package/dist/chunk-CYPA5BW5.js +0 -118
- package/dist/chunk-CYPA5BW5.js.map +0 -1
- package/dist/chunk-JTNDIBZO.js +0 -272
- package/dist/chunk-JTNDIBZO.js.map +0 -1
- package/dist/chunk-MV2N2WNS.js +0 -352
- package/dist/chunk-MV2N2WNS.js.map +0 -1
- package/dist/chunk-MXI26H62.cjs +0 -2207
- package/dist/chunk-MXI26H62.cjs.map +0 -1
- package/dist/chunk-QS7B42ZR.js +0 -5057
- package/dist/chunk-QS7B42ZR.js.map +0 -1
- package/dist/chunk-QX2ERL72.js +0 -1256
- package/dist/chunk-QX2ERL72.js.map +0 -1
- package/dist/chunk-ZWXQSKCX.cjs +0 -352
- package/dist/chunk-ZWXQSKCX.cjs.map +0 -1
- package/dist/index-BChYtrEe.d.ts +0 -548
- package/dist/index-VXt_v3YI.d.cts +0 -548
- package/dist/schemas.cjs.map +0 -1
- package/dist/schemas.js.map +0 -1
- package/dist/scoring.cjs.map +0 -1
- package/dist/scoring.js.map +0 -1
- package/dist/xapi.cjs.map +0 -1
- package/dist/xapi.js.map +0 -1
package/dist/ai-check.cjs
CHANGED
|
@@ -1,662 +1,887 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
1
|
+
Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
|
|
2
|
+
const require_ai_writing = require("./ai-writing-G6tnCGzw.cjs");
|
|
3
|
+
const require_scoring = require("./scoring-B2_HEhMT.cjs");
|
|
4
|
+
//#region src/ai-check/cases.ts
|
|
5
|
+
/**
|
|
6
|
+
* The items the cases are built from. Small, ordinary language-course content
|
|
7
|
+
* — and deliberately awkward in the places a model slips:
|
|
8
|
+
*
|
|
9
|
+
* - an answer a hint can hardly avoid naming (`Madrid`);
|
|
10
|
+
* - an answer of one short word (`is`), which a hint may write as an ordinary
|
|
11
|
+
* word and must not write beside its neighbour in the passage;
|
|
12
|
+
* - an accented answer (`cañón`), so a hint that spells it without the accent
|
|
13
|
+
* is still caught;
|
|
14
|
+
* - a passage with more than one blank, so a hint can be about the wrong one;
|
|
15
|
+
* - a short essay whose mistakes a model must quote exactly as the learner
|
|
16
|
+
* wrote them — straight apostrophes a model likes to curl, a mistake made
|
|
17
|
+
* twice — so a correction of words nobody wrote is caught;
|
|
18
|
+
* - questions in item groups: a reading passage that prints a question's
|
|
19
|
+
* answer, which a hint must point to without quoting, and a run of blanks
|
|
20
|
+
* whose instructions only the group states.
|
|
21
|
+
*/
|
|
22
|
+
const capital = {
|
|
23
|
+
schemaVersion: "1.0",
|
|
24
|
+
type: "multiple-choice",
|
|
25
|
+
id: "ai-check-mc",
|
|
26
|
+
title: "Capital cities",
|
|
27
|
+
question: "What is the capital of Spain?",
|
|
28
|
+
mode: "single",
|
|
29
|
+
scoringStrategy: "all-or-nothing",
|
|
30
|
+
options: [
|
|
31
|
+
{
|
|
32
|
+
id: "madrid",
|
|
33
|
+
text: "Madrid",
|
|
34
|
+
isCorrect: true
|
|
35
|
+
},
|
|
36
|
+
{
|
|
37
|
+
id: "barcelona",
|
|
38
|
+
text: "Barcelona",
|
|
39
|
+
isCorrect: false
|
|
40
|
+
},
|
|
41
|
+
{
|
|
42
|
+
id: "seville",
|
|
43
|
+
text: "Seville",
|
|
44
|
+
isCorrect: false
|
|
45
|
+
}
|
|
46
|
+
]
|
|
47
|
+
};
|
|
48
|
+
const introduction = {
|
|
49
|
+
schemaVersion: "1.0",
|
|
50
|
+
type: "fill-in-the-blanks",
|
|
51
|
+
id: "ai-check-fib",
|
|
52
|
+
title: "Introducing yourself",
|
|
53
|
+
passage: "My name {{be}} Rossi and I {{live}} in Madrid.",
|
|
54
|
+
blanks: [{
|
|
55
|
+
id: "be",
|
|
56
|
+
acceptedAnswers: ["is"]
|
|
57
|
+
}, {
|
|
58
|
+
id: "live",
|
|
59
|
+
acceptedAnswers: ["live"],
|
|
60
|
+
hint: "Present simple, first person."
|
|
61
|
+
}],
|
|
62
|
+
scoringStrategy: "partial"
|
|
63
|
+
};
|
|
64
|
+
const landscape = {
|
|
65
|
+
schemaVersion: "1.0",
|
|
66
|
+
type: "gap-select",
|
|
67
|
+
id: "ai-check-gap",
|
|
68
|
+
title: "Geography",
|
|
69
|
+
locale: "es-ES",
|
|
70
|
+
passage: "El río pasa por un {{g1}} muy profundo, y el agua {{g2}} fría.",
|
|
71
|
+
gaps: [{
|
|
72
|
+
id: "g1",
|
|
73
|
+
choices: [{
|
|
74
|
+
id: "canon",
|
|
75
|
+
text: "cañón"
|
|
76
|
+
}, {
|
|
77
|
+
id: "campo",
|
|
78
|
+
text: "campo"
|
|
79
|
+
}],
|
|
80
|
+
correctChoiceId: "canon"
|
|
81
|
+
}, {
|
|
82
|
+
id: "g2",
|
|
83
|
+
choices: [{
|
|
84
|
+
id: "esta",
|
|
85
|
+
text: "está"
|
|
86
|
+
}, {
|
|
87
|
+
id: "es",
|
|
88
|
+
text: "es"
|
|
89
|
+
}],
|
|
90
|
+
correctChoiceId: "esta"
|
|
91
|
+
}],
|
|
92
|
+
scoringStrategy: "partial"
|
|
93
|
+
};
|
|
94
|
+
const listening = {
|
|
95
|
+
schemaVersion: "1.0",
|
|
96
|
+
type: "dictation",
|
|
97
|
+
id: "ai-check-dictation",
|
|
98
|
+
title: "Listen and type",
|
|
99
|
+
transcript: "The train leaves at half past nine."
|
|
100
|
+
};
|
|
101
|
+
/**
|
|
102
|
+
* A reading group: a passage, a question it answers word for word, and a
|
|
103
|
+
* summary of it. The question's answer is printed in the passage, so a hint
|
|
104
|
+
* that quotes the passage where it answers is refused.
|
|
105
|
+
*/
|
|
106
|
+
const TIDES = "Along most coasts the tide comes in twice a day. It is pulled by the moon: as the Earth turns, the ocean nearest the moon bulges towards it, and so does the ocean on the far side. The highest tides, called spring tides, come when the sun and the moon line up.";
|
|
107
|
+
const springTides = {
|
|
108
|
+
schemaVersion: "1.0",
|
|
109
|
+
type: "multiple-choice",
|
|
110
|
+
id: "ai-check-tides",
|
|
111
|
+
title: "Spring tides",
|
|
112
|
+
question: "When do the highest tides come?",
|
|
113
|
+
mode: "single",
|
|
114
|
+
scoringStrategy: "all-or-nothing",
|
|
115
|
+
options: [
|
|
116
|
+
{
|
|
117
|
+
id: "line-up",
|
|
118
|
+
text: "When the sun and the moon line up",
|
|
119
|
+
isCorrect: true
|
|
120
|
+
},
|
|
121
|
+
{
|
|
122
|
+
id: "full-moon",
|
|
123
|
+
text: "Only at full moon",
|
|
124
|
+
isCorrect: false
|
|
125
|
+
},
|
|
126
|
+
{
|
|
127
|
+
id: "winter",
|
|
128
|
+
text: "In winter",
|
|
129
|
+
isCorrect: false
|
|
130
|
+
}
|
|
131
|
+
]
|
|
35
132
|
};
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
],
|
|
46
|
-
scoringStrategy: "partial"
|
|
133
|
+
const tidesSummary = {
|
|
134
|
+
schemaVersion: "1.0",
|
|
135
|
+
type: "written-response",
|
|
136
|
+
id: "ai-check-tides-summary",
|
|
137
|
+
title: "Summary",
|
|
138
|
+
prompt: "Summarise the text in one or two sentences.",
|
|
139
|
+
minWords: 10,
|
|
140
|
+
maxWords: 60,
|
|
141
|
+
languageTarget: "en-B1"
|
|
47
142
|
};
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
{ id: "campo", text: "campo" }
|
|
61
|
-
],
|
|
62
|
-
correctChoiceId: "canon"
|
|
63
|
-
},
|
|
64
|
-
{
|
|
65
|
-
id: "g2",
|
|
66
|
-
choices: [
|
|
67
|
-
{ id: "esta", text: "est\xE1" },
|
|
68
|
-
{ id: "es", text: "es" }
|
|
69
|
-
],
|
|
70
|
-
correctChoiceId: "esta"
|
|
71
|
-
}
|
|
72
|
-
],
|
|
73
|
-
scoringStrategy: "partial"
|
|
143
|
+
const tides = {
|
|
144
|
+
schemaVersion: "1.0",
|
|
145
|
+
type: "item-group",
|
|
146
|
+
id: "ai-check-tides-group",
|
|
147
|
+
title: "Part 2 — Reading",
|
|
148
|
+
stimulus: {
|
|
149
|
+
id: "ai-check-tides-text",
|
|
150
|
+
kind: "text",
|
|
151
|
+
title: "Tides",
|
|
152
|
+
body: TIDES
|
|
153
|
+
},
|
|
154
|
+
items: [springTides, tidesSummary]
|
|
74
155
|
};
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
156
|
+
/** A part whose questions share instructions the questions themselves never state. */
|
|
157
|
+
const formation = {
|
|
158
|
+
schemaVersion: "1.0",
|
|
159
|
+
type: "fill-in-the-blanks",
|
|
160
|
+
id: "ai-check-formation",
|
|
161
|
+
title: "Word formation",
|
|
162
|
+
passage: "She smiled {{a}} when she heard the news. (HAPPY)",
|
|
163
|
+
blanks: [{
|
|
164
|
+
id: "a",
|
|
165
|
+
acceptedAnswers: ["happily"]
|
|
166
|
+
}],
|
|
167
|
+
scoringStrategy: "all-or-nothing"
|
|
81
168
|
};
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
{ name: "Grammar", description: "Past tenses, articles, agreement", weight: 2 },
|
|
94
|
-
{ name: "Vocabulary", description: "Everyday words, used correctly", weight: 1 },
|
|
95
|
-
{ name: "Task", description: "Says what the learner did, in order", weight: 1 }
|
|
96
|
-
]
|
|
97
|
-
}
|
|
169
|
+
const formationPart = {
|
|
170
|
+
schemaVersion: "1.0",
|
|
171
|
+
type: "item-group",
|
|
172
|
+
id: "ai-check-formation-part",
|
|
173
|
+
title: "Part 1 — Word formation",
|
|
174
|
+
stimulus: {
|
|
175
|
+
id: "ai-check-formation-instructions",
|
|
176
|
+
kind: "text",
|
|
177
|
+
body: "Use the word in capitals at the end of each sentence to form a word that fits the gap."
|
|
178
|
+
},
|
|
179
|
+
items: [formation]
|
|
98
180
|
};
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
181
|
+
const weekend = {
|
|
182
|
+
schemaVersion: "1.0",
|
|
183
|
+
type: "written-response",
|
|
184
|
+
id: "ai-check-writing",
|
|
185
|
+
title: "Your weekend",
|
|
186
|
+
prompt: "Write a few sentences about what you did last weekend.",
|
|
187
|
+
minWords: 15,
|
|
188
|
+
maxWords: 120,
|
|
189
|
+
languageTarget: "en-A2",
|
|
190
|
+
rubric: { criteria: [
|
|
191
|
+
{
|
|
192
|
+
name: "Grammar",
|
|
193
|
+
description: "Past tenses, articles, agreement",
|
|
194
|
+
weight: 2
|
|
195
|
+
},
|
|
196
|
+
{
|
|
197
|
+
name: "Vocabulary",
|
|
198
|
+
description: "Everyday words, used correctly",
|
|
199
|
+
weight: 1
|
|
200
|
+
},
|
|
201
|
+
{
|
|
202
|
+
name: "Task",
|
|
203
|
+
description: "Says what the learner did, in order",
|
|
204
|
+
weight: 1
|
|
205
|
+
}
|
|
206
|
+
] }
|
|
113
207
|
};
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
208
|
+
const weather = {
|
|
209
|
+
schemaVersion: "1.0",
|
|
210
|
+
type: "read-aloud",
|
|
211
|
+
id: "ai-check-reading",
|
|
212
|
+
title: "Read the forecast aloud",
|
|
213
|
+
referenceText: "The weather is warm in the north.",
|
|
214
|
+
locale: "en-US",
|
|
215
|
+
recording: { maxSeconds: 20 },
|
|
216
|
+
scoring: { dimensions: [{
|
|
217
|
+
name: "accuracy",
|
|
218
|
+
weight: 2
|
|
219
|
+
}, {
|
|
220
|
+
name: "fluency",
|
|
221
|
+
weight: 1
|
|
222
|
+
}] }
|
|
223
|
+
};
|
|
224
|
+
/** A word of the forecast as an engine reports it. */
|
|
225
|
+
const said = (text, over = {}) => ({
|
|
226
|
+
text,
|
|
227
|
+
error: "none",
|
|
228
|
+
accuracy: 92,
|
|
229
|
+
...over
|
|
119
230
|
});
|
|
231
|
+
/**
|
|
232
|
+
* An engine's assessment of the forecast. With `slips`, "weather" is
|
|
233
|
+
* mispronounced — its "th" heard as /d/ or /z/ — and "north" is left out: the
|
|
234
|
+
* marks a model must coach, and no others.
|
|
235
|
+
*/
|
|
120
236
|
function forecast(slips) {
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
237
|
+
return {
|
|
238
|
+
assessmentVersion: "1.0",
|
|
239
|
+
status: "assessed",
|
|
240
|
+
task: "scripted",
|
|
241
|
+
locale: "en-US",
|
|
242
|
+
referenceText: weather.referenceText,
|
|
243
|
+
recordingKey: "ai-check-take",
|
|
244
|
+
assessor: {
|
|
245
|
+
kind: "auto",
|
|
246
|
+
id: "pronunciation-engine"
|
|
247
|
+
},
|
|
248
|
+
scale: 100,
|
|
249
|
+
scores: slips ? {
|
|
250
|
+
accuracy: 68,
|
|
251
|
+
fluency: 81,
|
|
252
|
+
completeness: 86
|
|
253
|
+
} : {
|
|
254
|
+
accuracy: 94,
|
|
255
|
+
fluency: 90
|
|
256
|
+
},
|
|
257
|
+
miscue: "assessor",
|
|
258
|
+
phonemeAlphabet: "ipa",
|
|
259
|
+
words: [
|
|
260
|
+
said("The"),
|
|
261
|
+
slips ? said("weather", {
|
|
262
|
+
accuracy: 38,
|
|
263
|
+
error: "mispronunciation",
|
|
264
|
+
phonemes: [
|
|
265
|
+
{
|
|
266
|
+
symbol: "w",
|
|
267
|
+
accuracy: 90
|
|
268
|
+
},
|
|
269
|
+
{
|
|
270
|
+
symbol: "ɛ",
|
|
271
|
+
accuracy: 85
|
|
272
|
+
},
|
|
273
|
+
{
|
|
274
|
+
symbol: "ð",
|
|
275
|
+
accuracy: 12,
|
|
276
|
+
heardAs: [
|
|
277
|
+
{
|
|
278
|
+
symbol: "d",
|
|
279
|
+
score: 64
|
|
280
|
+
},
|
|
281
|
+
{
|
|
282
|
+
symbol: "z",
|
|
283
|
+
score: 21
|
|
284
|
+
},
|
|
285
|
+
{
|
|
286
|
+
symbol: "ð",
|
|
287
|
+
score: 12
|
|
288
|
+
}
|
|
289
|
+
]
|
|
290
|
+
},
|
|
291
|
+
{
|
|
292
|
+
symbol: "ɚ",
|
|
293
|
+
accuracy: 80
|
|
294
|
+
}
|
|
295
|
+
]
|
|
296
|
+
}) : said("weather"),
|
|
297
|
+
said("is"),
|
|
298
|
+
said("warm"),
|
|
299
|
+
said("in"),
|
|
300
|
+
said("the"),
|
|
301
|
+
slips ? {
|
|
302
|
+
text: "north",
|
|
303
|
+
error: "omission"
|
|
304
|
+
} : said("north")
|
|
305
|
+
]
|
|
306
|
+
};
|
|
160
307
|
}
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
308
|
+
const chose = (ids) => ({
|
|
309
|
+
type: "multiple-choice",
|
|
310
|
+
selectedOptionIds: ids
|
|
311
|
+
});
|
|
312
|
+
const typed = (answers) => ({
|
|
313
|
+
type: "fill-in-the-blanks",
|
|
314
|
+
answers
|
|
164
315
|
});
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
316
|
+
const picked = (choices) => ({
|
|
317
|
+
type: "gap-select",
|
|
318
|
+
selections: choices
|
|
168
319
|
});
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
320
|
+
const wrote = (text) => ({
|
|
321
|
+
type: "dictation",
|
|
322
|
+
text
|
|
172
323
|
});
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
wordCount: 0
|
|
324
|
+
const drafted = (text) => ({
|
|
325
|
+
type: "written-response",
|
|
326
|
+
text,
|
|
327
|
+
wordCount: 0
|
|
178
328
|
});
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
329
|
+
/**
|
|
330
|
+
* Items an author asks a model to review. Each is valid, and each but the last
|
|
331
|
+
* has a flaw a rule cannot see: a distractor that is botanically a right
|
|
332
|
+
* answer, and a task far above the level it is set for.
|
|
333
|
+
*/
|
|
334
|
+
const fruit = {
|
|
335
|
+
schemaVersion: "1.0",
|
|
336
|
+
type: "multiple-choice",
|
|
337
|
+
id: "ai-check-fruit",
|
|
338
|
+
title: "Food words",
|
|
339
|
+
question: "Which of these is a fruit?",
|
|
340
|
+
mode: "single",
|
|
341
|
+
scoringStrategy: "all-or-nothing",
|
|
342
|
+
options: [
|
|
343
|
+
{
|
|
344
|
+
id: "f1",
|
|
345
|
+
text: "Apple",
|
|
346
|
+
isCorrect: true
|
|
347
|
+
},
|
|
348
|
+
{
|
|
349
|
+
id: "f2",
|
|
350
|
+
text: "Carrot",
|
|
351
|
+
isCorrect: false
|
|
352
|
+
},
|
|
353
|
+
{
|
|
354
|
+
id: "f3",
|
|
355
|
+
text: "Tomato",
|
|
356
|
+
isCorrect: false
|
|
357
|
+
}
|
|
358
|
+
]
|
|
192
359
|
};
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
360
|
+
const tooHard = {
|
|
361
|
+
schemaVersion: "1.0",
|
|
362
|
+
type: "written-response",
|
|
363
|
+
id: "ai-check-too-hard",
|
|
364
|
+
title: "Cities",
|
|
365
|
+
prompt: "Discuss the socioeconomic ramifications of rapid urbanisation in developing economies.",
|
|
366
|
+
minWords: 40,
|
|
367
|
+
maxWords: 80
|
|
201
368
|
};
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
369
|
+
const cleanCloze = {
|
|
370
|
+
schemaVersion: "1.0",
|
|
371
|
+
type: "fill-in-the-blanks",
|
|
372
|
+
id: "ai-check-clean-cloze",
|
|
373
|
+
title: "Daily routine",
|
|
374
|
+
passage: "Every morning I {{1}} up at seven and {{2}} a shower.",
|
|
375
|
+
blanks: [{
|
|
376
|
+
id: "1",
|
|
377
|
+
acceptedAnswers: ["get", "wake"]
|
|
378
|
+
}, {
|
|
379
|
+
id: "2",
|
|
380
|
+
acceptedAnswers: ["take", "have"]
|
|
381
|
+
}],
|
|
382
|
+
scoringStrategy: "partial"
|
|
213
383
|
};
|
|
214
384
|
function critiqueCase(id, about, type, draft, extra = {}) {
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
385
|
+
const request = require_ai_writing.aiCritiqueRequest({
|
|
386
|
+
type,
|
|
387
|
+
draft,
|
|
388
|
+
...extra
|
|
389
|
+
});
|
|
390
|
+
if (request === null) throw new Error(`ai-check: no critique request for "${id}"`);
|
|
391
|
+
return {
|
|
392
|
+
id,
|
|
393
|
+
feature: "item-critique",
|
|
394
|
+
about,
|
|
395
|
+
request
|
|
396
|
+
};
|
|
220
397
|
}
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
398
|
+
/** A short reading to draft from: plain, and with facts a question can be asked about. */
|
|
399
|
+
const MARKET = "Maria lives in Seville, in the south of Spain. Every Saturday she walks to the market with her grandmother. They buy oranges, fresh bread and a little cheese. On the way home they stop at a café, where Maria drinks hot chocolate and her grandmother reads the newspaper.";
|
|
400
|
+
/** The same reading as a video's captions, with times in seconds. */
|
|
401
|
+
const MARKET_CAPTIONS = [
|
|
402
|
+
{
|
|
403
|
+
start: 0,
|
|
404
|
+
end: 4.2,
|
|
405
|
+
text: "Maria lives in Seville, in the south of Spain."
|
|
406
|
+
},
|
|
407
|
+
{
|
|
408
|
+
start: 4.2,
|
|
409
|
+
end: 8.9,
|
|
410
|
+
text: "Every Saturday she walks to the market with her grandmother."
|
|
411
|
+
},
|
|
412
|
+
{
|
|
413
|
+
start: 8.9,
|
|
414
|
+
end: 12.6,
|
|
415
|
+
text: "They buy oranges, fresh bread and a little cheese."
|
|
416
|
+
},
|
|
417
|
+
{
|
|
418
|
+
start: 12.6,
|
|
419
|
+
end: 18.1,
|
|
420
|
+
text: "On the way home they stop at a café."
|
|
421
|
+
}
|
|
227
422
|
];
|
|
228
423
|
function draftsCase(id, about, types, source, extra = {}) {
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
424
|
+
const request = require_ai_writing.aiDraftsRequest({
|
|
425
|
+
types,
|
|
426
|
+
source,
|
|
427
|
+
...extra
|
|
428
|
+
});
|
|
429
|
+
if (request === null) throw new Error(`ai-check: no drafts request for "${id}"`);
|
|
430
|
+
return {
|
|
431
|
+
id,
|
|
432
|
+
feature: "draft-generation",
|
|
433
|
+
about,
|
|
434
|
+
request
|
|
435
|
+
};
|
|
234
436
|
}
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
437
|
+
/**
|
|
438
|
+
* What a host sets on every read-aloud it drafts: the SDK has no default for a
|
|
439
|
+
* recording's length or the dimensions a reading is graded on.
|
|
440
|
+
*/
|
|
441
|
+
const READ_ALOUD_SETTINGS = {
|
|
442
|
+
locale: "en-US",
|
|
443
|
+
recording: { maxSeconds: 20 },
|
|
444
|
+
scoring: { dimensions: [{
|
|
445
|
+
name: "accuracy",
|
|
446
|
+
weight: 1
|
|
447
|
+
}] }
|
|
239
448
|
};
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
449
|
+
/** Builds a case, failing loudly rather than shipping a case with no request. */
|
|
450
|
+
function explanationCase(id, about, data, response, group) {
|
|
451
|
+
const request = require_ai_writing.aiExplanationRequest({
|
|
452
|
+
data,
|
|
453
|
+
response,
|
|
454
|
+
...group ? { group } : {}
|
|
455
|
+
});
|
|
456
|
+
if (request === null) throw new Error(`ai-check: no explanation request for "${id}"`);
|
|
457
|
+
return {
|
|
458
|
+
id,
|
|
459
|
+
feature: "explanation",
|
|
460
|
+
about,
|
|
461
|
+
request
|
|
462
|
+
};
|
|
246
463
|
}
|
|
247
|
-
function hintCase(id, about, data, response, previousHints = []) {
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
464
|
+
function hintCase(id, about, data, response, previousHints = [], group) {
|
|
465
|
+
const request = require_ai_writing.aiHintRequest({
|
|
466
|
+
data,
|
|
467
|
+
response,
|
|
468
|
+
previousHints,
|
|
469
|
+
...group ? { group } : {}
|
|
470
|
+
});
|
|
471
|
+
if (request === null) throw new Error(`ai-check: no hint request for "${id}"`);
|
|
472
|
+
return {
|
|
473
|
+
id,
|
|
474
|
+
feature: "hint",
|
|
475
|
+
about,
|
|
476
|
+
request
|
|
477
|
+
};
|
|
253
478
|
}
|
|
254
|
-
function writingCase(id, about, text, previousFeedback = [], learnerLocale) {
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
479
|
+
function writingCase(id, about, text, previousFeedback = [], learnerLocale, task = { data: weekend }) {
|
|
480
|
+
const request = require_ai_writing.aiWritingFeedbackRequest({
|
|
481
|
+
data: task.data,
|
|
482
|
+
response: drafted(text),
|
|
483
|
+
previousFeedback,
|
|
484
|
+
...task.group ? { group: task.group } : {},
|
|
485
|
+
...learnerLocale !== void 0 ? { learnerLocale } : {}
|
|
486
|
+
});
|
|
487
|
+
if (request === null) throw new Error(`ai-check: no writing feedback request for "${id}"`);
|
|
488
|
+
return {
|
|
489
|
+
id,
|
|
490
|
+
feature: "writing-feedback",
|
|
491
|
+
about,
|
|
492
|
+
request
|
|
493
|
+
};
|
|
265
494
|
}
|
|
266
495
|
function coachingCase(id, about, marks, learnerLocale) {
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
496
|
+
const assessment = marks.assessment ?? forecast(true);
|
|
497
|
+
const grade = require_scoring.gradeReadAloud(weather, {
|
|
498
|
+
type: "read-aloud",
|
|
499
|
+
recording: {
|
|
500
|
+
key: "ai-check-take",
|
|
501
|
+
mimeType: "audio/wav"
|
|
502
|
+
}
|
|
503
|
+
}, assessment, {
|
|
504
|
+
measured: {
|
|
505
|
+
durationMs: 4e3,
|
|
506
|
+
voicedMs: 3e3
|
|
507
|
+
},
|
|
508
|
+
plausibility: {
|
|
509
|
+
maxWordsPerSecond: 6,
|
|
510
|
+
minVoicedMs: 500
|
|
511
|
+
}
|
|
512
|
+
});
|
|
513
|
+
if ("unscorable" in grade) throw new Error(`ai-check: the reading for "${id}" did not grade: ${grade.reason}`);
|
|
514
|
+
const request = require_ai_writing.aiCoachingRequest({
|
|
515
|
+
data: weather,
|
|
516
|
+
...marks.fromGradeOnly === true ? {} : { assessment },
|
|
517
|
+
grade,
|
|
518
|
+
...learnerLocale !== void 0 ? { learnerLocale } : {}
|
|
519
|
+
});
|
|
520
|
+
if (request === null) throw new Error(`ai-check: no coaching request for "${id}"`);
|
|
521
|
+
return {
|
|
522
|
+
id,
|
|
523
|
+
feature: "pronunciation-coaching",
|
|
524
|
+
about,
|
|
525
|
+
request
|
|
526
|
+
};
|
|
290
527
|
}
|
|
528
|
+
/**
|
|
529
|
+
* The cases {@link runAiCheck} runs: every type the SDK explains, answered
|
|
530
|
+
* right, wrong and — where a type can be — partly right; every type it hints
|
|
531
|
+
* for, before an answer and after a wrong one; and drafts of an essay, with
|
|
532
|
+
* mistakes, without, and revised.
|
|
533
|
+
*
|
|
534
|
+
* They are ordinary calls. What makes them a test is that the SDK knows the
|
|
535
|
+
* answer to each, so it can check what your model writes about it.
|
|
536
|
+
*
|
|
537
|
+
* A fresh array each call, so a caller may filter, slice or add to it without
|
|
538
|
+
* changing what the next caller gets.
|
|
539
|
+
*/
|
|
291
540
|
function aiCheckCases() {
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
"written-response",
|
|
409
|
-
tooHard,
|
|
410
|
-
{ level: "A1" }
|
|
411
|
-
),
|
|
412
|
-
critiqueCase(
|
|
413
|
-
"critique-clean",
|
|
414
|
-
"A cloze with nothing wrong in it",
|
|
415
|
-
"fill-in-the-blanks",
|
|
416
|
-
cleanCloze,
|
|
417
|
-
{
|
|
418
|
-
level: "A2"
|
|
419
|
-
}
|
|
420
|
-
),
|
|
421
|
-
critiqueCase(
|
|
422
|
-
"critique-in-spanish",
|
|
423
|
-
"The fruit question, reviewed in Spanish for its author",
|
|
424
|
-
"multiple-choice",
|
|
425
|
-
fruit,
|
|
426
|
-
{ level: "A1", authorLocale: "es" }
|
|
427
|
-
),
|
|
428
|
-
draftsCase(
|
|
429
|
-
"drafts-questions",
|
|
430
|
-
"Four multiple-choice questions on a short reading",
|
|
431
|
-
["multiple-choice"],
|
|
432
|
-
{ kind: "passage", text: MARKET },
|
|
433
|
-
{ count: 4, level: "A2", locale: "en" }
|
|
434
|
-
),
|
|
435
|
-
draftsCase(
|
|
436
|
-
"drafts-cloze",
|
|
437
|
-
"A cloze on the same reading, from its transcript",
|
|
438
|
-
["fill-in-the-blanks"],
|
|
439
|
-
{ kind: "transcript", text: MARKET },
|
|
440
|
-
{ count: 1, level: "A2", locale: "en", instructions: "Blank the verbs." }
|
|
441
|
-
),
|
|
442
|
-
draftsCase(
|
|
443
|
-
"drafts-gaps",
|
|
444
|
-
"A choose-the-word item on the same reading",
|
|
445
|
-
["gap-select"],
|
|
446
|
-
{ kind: "passage", text: MARKET },
|
|
447
|
-
{ count: 1, level: "A2", locale: "en" }
|
|
448
|
-
),
|
|
449
|
-
draftsCase(
|
|
450
|
-
"drafts-writing",
|
|
451
|
-
"A writing task that follows on from the reading",
|
|
452
|
-
["written-response"],
|
|
453
|
-
{ kind: "passage", text: MARKET },
|
|
454
|
-
{ count: 1, level: "A2", locale: "en" }
|
|
455
|
-
),
|
|
456
|
-
draftsCase(
|
|
457
|
-
"drafts-video",
|
|
458
|
-
"A video quiz from timed captions: multiple choice, then dictation, then read-aloud, as many as the video is worth",
|
|
459
|
-
["multiple-choice", "dictation", "read-aloud"],
|
|
460
|
-
{ kind: "captions", cues: MARKET_CAPTIONS },
|
|
461
|
-
{ level: "A2", locale: "en", settings: { "read-aloud": READ_ALOUD_SETTINGS } }
|
|
462
|
-
),
|
|
463
|
-
draftsCase(
|
|
464
|
-
"drafts-script",
|
|
465
|
-
"The same video from its script alone, with no times, and what the author asked for",
|
|
466
|
-
["multiple-choice", "dictation", "read-aloud"],
|
|
467
|
-
{ kind: "transcript", text: MARKET },
|
|
468
|
-
{
|
|
469
|
-
level: "A2",
|
|
470
|
-
locale: "en",
|
|
471
|
-
instructions: "Focus on what Maria and her grandmother buy.",
|
|
472
|
-
settings: { "read-aloud": READ_ALOUD_SETTINGS }
|
|
473
|
-
}
|
|
474
|
-
)
|
|
475
|
-
];
|
|
541
|
+
return [
|
|
542
|
+
explanationCase("mc-correct", "Chose the right city", capital, chose(["madrid"])),
|
|
543
|
+
explanationCase("mc-wrong", "Chose Barcelona", capital, chose(["barcelona"])),
|
|
544
|
+
explanationCase("mc-blank", "Chose nothing and submitted", capital, chose([])),
|
|
545
|
+
explanationCase("fib-correct", "Filled both blanks correctly", introduction, typed({
|
|
546
|
+
be: "is",
|
|
547
|
+
live: "live"
|
|
548
|
+
})),
|
|
549
|
+
explanationCase("fib-partly", "One blank right, one wrong", introduction, typed({
|
|
550
|
+
be: "is",
|
|
551
|
+
live: "lives"
|
|
552
|
+
})),
|
|
553
|
+
explanationCase("fib-wrong", "Both blanks wrong", introduction, typed({
|
|
554
|
+
be: "are",
|
|
555
|
+
live: "living"
|
|
556
|
+
})),
|
|
557
|
+
explanationCase("gap-partly", "One gap right, one wrong, in Spanish", landscape, picked({
|
|
558
|
+
g1: "canon",
|
|
559
|
+
g2: "es"
|
|
560
|
+
})),
|
|
561
|
+
explanationCase("gap-wrong", "Both gaps wrong, in Spanish", landscape, picked({
|
|
562
|
+
g1: "campo",
|
|
563
|
+
g2: "es"
|
|
564
|
+
})),
|
|
565
|
+
explanationCase("dictation-near", "Typed the sentence with two words wrong", listening, wrote("The train leaves at half past five")),
|
|
566
|
+
explanationCase("dictation-correct", "Typed the sentence correctly", listening, wrote("The train leaves at half past nine.")),
|
|
567
|
+
explanationCase("mc-reading-wrong", "Chose 'Only at full moon', on a question about its group's passage", springTides, chose(["full-moon"]), tides),
|
|
568
|
+
hintCase("mc-hint-1", "Has not answered yet", capital, chose([])),
|
|
569
|
+
hintCase("mc-hint-2", "Asked again after choosing Seville", capital, chose(["seville"]), ["Think about where the government sits."]),
|
|
570
|
+
hintCase("fib-hint-1", "Both blanks empty", introduction, typed({})),
|
|
571
|
+
hintCase("fib-hint-2", "Asked again with one blank wrong", introduction, typed({
|
|
572
|
+
be: "are",
|
|
573
|
+
live: ""
|
|
574
|
+
}), ["Look again at the first blank."]),
|
|
575
|
+
hintCase("gap-hint-1", "Neither gap chosen, in Spanish", landscape, picked({})),
|
|
576
|
+
hintCase("gap-hint-2", "Asked again with the first gap wrong", landscape, picked({ g1: "campo" }), ["Piensa en la forma del terreno."]),
|
|
577
|
+
hintCase("mc-reading-hint", "Has not answered a question whose answer its group's passage prints", springTides, chose([]), [], tides),
|
|
578
|
+
hintCase("fib-formation-hint", "A word-formation blank, under the instructions its group gives", formation, typed({}), [], formationPart),
|
|
579
|
+
writingCase("writing-mistakes", "A first draft with tense, article and agreement mistakes", "Last weekend I go to the market and buyed a apple. My friends was there too, so we don't go home early. Then I go to the cinema."),
|
|
580
|
+
writingCase("writing-clean", "A first draft with no mistakes to correct", "Last weekend I went to the market and bought an apple. My friends were there too, so we stayed until the evening. Then we went to the cinema."),
|
|
581
|
+
writingCase("writing-revised", "A second draft that fixed some mistakes", "Last weekend I went to the market and bought a apple. My friends was there too, so we didn't go home early.", ["Check your past tenses: \"go\" and \"buyed\" should be past forms."]),
|
|
582
|
+
writingCase("writing-feedback-in-spanish", "A first draft, with feedback asked for in Spanish", "Last weekend I visit my grandmother. She cook a big lunch and we eat together in the garden.", [], "es"),
|
|
583
|
+
writingCase("writing-summary", "A summary of its group's passage that gets the spring tides wrong", "The tide comes in twice a day because the moon pulls the ocean. The highest tides come only at full moon.", [], void 0, {
|
|
584
|
+
data: tidesSummary,
|
|
585
|
+
group: tides
|
|
586
|
+
}),
|
|
587
|
+
coachingCase("coaching-slips", "A reading with a mispronounced \"th\" and a word left out", { assessment: forecast(true) }),
|
|
588
|
+
coachingCase("coaching-clean", "A reading the engine marked correct throughout", { assessment: forecast(false) }),
|
|
589
|
+
coachingCase("coaching-from-grade", "The same slips, from a stored grade: marks without sounds", { fromGradeOnly: true }),
|
|
590
|
+
coachingCase("coaching-in-spanish", "The same slips, with coaching asked for in Spanish", { assessment: forecast(true) }, "es"),
|
|
591
|
+
critiqueCase("critique-second-answer", "A question whose distractor \"Tomato\" is botanically a fruit", "multiple-choice", fruit, { level: "A1" }),
|
|
592
|
+
critiqueCase("critique-level", "A writing task far above the A1 level it is set for", "written-response", tooHard, { level: "A1" }),
|
|
593
|
+
critiqueCase("critique-clean", "A cloze with nothing wrong in it", "fill-in-the-blanks", cleanCloze, { level: "A2" }),
|
|
594
|
+
critiqueCase("critique-in-spanish", "The fruit question, reviewed in Spanish for its author", "multiple-choice", fruit, {
|
|
595
|
+
level: "A1",
|
|
596
|
+
authorLocale: "es"
|
|
597
|
+
}),
|
|
598
|
+
draftsCase("drafts-questions", "Four multiple-choice questions on a short reading", ["multiple-choice"], {
|
|
599
|
+
kind: "passage",
|
|
600
|
+
text: MARKET
|
|
601
|
+
}, {
|
|
602
|
+
count: 4,
|
|
603
|
+
level: "A2",
|
|
604
|
+
locale: "en"
|
|
605
|
+
}),
|
|
606
|
+
draftsCase("drafts-cloze", "A cloze on the same reading, from its transcript", ["fill-in-the-blanks"], {
|
|
607
|
+
kind: "transcript",
|
|
608
|
+
text: MARKET
|
|
609
|
+
}, {
|
|
610
|
+
count: 1,
|
|
611
|
+
level: "A2",
|
|
612
|
+
locale: "en",
|
|
613
|
+
instructions: "Blank the verbs."
|
|
614
|
+
}),
|
|
615
|
+
draftsCase("drafts-gaps", "A choose-the-word item on the same reading", ["gap-select"], {
|
|
616
|
+
kind: "passage",
|
|
617
|
+
text: MARKET
|
|
618
|
+
}, {
|
|
619
|
+
count: 1,
|
|
620
|
+
level: "A2",
|
|
621
|
+
locale: "en"
|
|
622
|
+
}),
|
|
623
|
+
draftsCase("drafts-writing", "A writing task that follows on from the reading", ["written-response"], {
|
|
624
|
+
kind: "passage",
|
|
625
|
+
text: MARKET
|
|
626
|
+
}, {
|
|
627
|
+
count: 1,
|
|
628
|
+
level: "A2",
|
|
629
|
+
locale: "en"
|
|
630
|
+
}),
|
|
631
|
+
draftsCase("drafts-video", "A video quiz from timed captions: multiple choice, then dictation, then read-aloud, as many as the video is worth", [
|
|
632
|
+
"multiple-choice",
|
|
633
|
+
"dictation",
|
|
634
|
+
"read-aloud"
|
|
635
|
+
], {
|
|
636
|
+
kind: "captions",
|
|
637
|
+
cues: MARKET_CAPTIONS
|
|
638
|
+
}, {
|
|
639
|
+
level: "A2",
|
|
640
|
+
locale: "en",
|
|
641
|
+
settings: { "read-aloud": READ_ALOUD_SETTINGS }
|
|
642
|
+
}),
|
|
643
|
+
draftsCase("drafts-script", "The same video from its script alone, with no times, and what the author asked for", [
|
|
644
|
+
"multiple-choice",
|
|
645
|
+
"dictation",
|
|
646
|
+
"read-aloud"
|
|
647
|
+
], {
|
|
648
|
+
kind: "transcript",
|
|
649
|
+
text: MARKET
|
|
650
|
+
}, {
|
|
651
|
+
level: "A2",
|
|
652
|
+
locale: "en",
|
|
653
|
+
instructions: "Focus on what Maria and her grandmother buy.",
|
|
654
|
+
settings: { "read-aloud": READ_ALOUD_SETTINGS }
|
|
655
|
+
})
|
|
656
|
+
];
|
|
476
657
|
}
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
658
|
+
//#endregion
|
|
659
|
+
//#region src/ai-check/index.ts
|
|
660
|
+
/**
|
|
661
|
+
* A test kit for the prompts behind your AI ports.
|
|
662
|
+
*
|
|
663
|
+
* The SDK cannot test your model, and will not try. What it can do is make the
|
|
664
|
+
* calls a learner's questions would make — on items whose answers it knows —
|
|
665
|
+
* and run the same checks it runs before a learner sees anything. What comes
|
|
666
|
+
* back is a count of what would have been shown and what would have been
|
|
667
|
+
* refused, and why.
|
|
668
|
+
*
|
|
669
|
+
* Run it where you change a prompt or a model, in your own CI, with your own
|
|
670
|
+
* key. It is a separate entry point (`@intellectif/lk-core/ai-check`) because
|
|
671
|
+
* it is for your test run, not for a learner's browser: nothing here is
|
|
672
|
+
* imported by anything a page renders.
|
|
673
|
+
*
|
|
674
|
+
* ```ts
|
|
675
|
+
* import { aiCheckCases, formatAiCheckReport, runAiCheck } from '@intellectif/lk-core/ai-check';
|
|
676
|
+
*
|
|
677
|
+
* const report = await runAiCheck({
|
|
678
|
+
* explain: (request) => callMyModel(EXPLAIN_PROMPT, request),
|
|
679
|
+
* hint: (request) => callMyModel(HINT_PROMPT, request),
|
|
680
|
+
* writingFeedback: (request) => callMyModel(FEEDBACK_PROMPT, request),
|
|
681
|
+
* pronunciationCoaching: (request) => callMyModel(COACHING_PROMPT, request),
|
|
682
|
+
* critique: (request) => callMyModel(CRITIQUE_PROMPT, request),
|
|
683
|
+
* drafts: (request) => callMyModel(DRAFTS_PROMPT, request),
|
|
684
|
+
* });
|
|
685
|
+
* console.log(formatAiCheckReport(report));
|
|
686
|
+
* if (report.refused > 0 || report.errors > 0) {
|
|
687
|
+
* process.exitCode = 1;
|
|
688
|
+
* }
|
|
689
|
+
* ```
|
|
690
|
+
*
|
|
691
|
+
* A refusal is a real failure, not a warning: it is an answer a learner would
|
|
692
|
+
* have asked for and not received. `reveals-answer` on a hint case is the one
|
|
693
|
+
* to take most seriously — that prompt gives answers away. Next come
|
|
694
|
+
* `misquotes-answer` on a writing case — that prompt corrects words the learner
|
|
695
|
+
* never wrote — and `contradicts-marks` on a coaching case: that prompt coaches
|
|
696
|
+
* words the engine heard as right, or sounds it never reported.
|
|
697
|
+
*/
|
|
698
|
+
const REFUSALS = [
|
|
699
|
+
"malformed",
|
|
700
|
+
"empty",
|
|
701
|
+
"too-long",
|
|
702
|
+
"contradicts-grade",
|
|
703
|
+
"reveals-answer",
|
|
704
|
+
"misquotes-answer",
|
|
705
|
+
"contradicts-marks",
|
|
706
|
+
"contradicts-item"
|
|
488
707
|
];
|
|
708
|
+
/** The port a case is for. */
|
|
489
709
|
function portFor(ports, one) {
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
return ports.critique;
|
|
499
|
-
case "draft-generation":
|
|
500
|
-
return ports.drafts;
|
|
501
|
-
default:
|
|
502
|
-
return ports.writingFeedback;
|
|
503
|
-
}
|
|
710
|
+
switch (one.request.feature) {
|
|
711
|
+
case "explanation": return ports.explain;
|
|
712
|
+
case "hint": return ports.hint;
|
|
713
|
+
case "pronunciation-coaching": return ports.pronunciationCoaching;
|
|
714
|
+
case "item-critique": return ports.critique;
|
|
715
|
+
case "draft-generation": return ports.drafts;
|
|
716
|
+
default: return ports.writingFeedback;
|
|
717
|
+
}
|
|
504
718
|
}
|
|
505
|
-
|
|
719
|
+
/** Milliseconds, from whichever clock this runtime has. */
|
|
720
|
+
const now = () => Date.now();
|
|
721
|
+
/** Runs one case: calls the port, times it, and checks what came back. */
|
|
506
722
|
async function runCase(ports, one) {
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
|
|
723
|
+
const started = now();
|
|
724
|
+
const port = portFor(ports, one);
|
|
725
|
+
if (port === void 0) throw new Error(`ai-check: no port for "${one.id}"`);
|
|
726
|
+
let raw;
|
|
727
|
+
try {
|
|
728
|
+
raw = await port(one.request);
|
|
729
|
+
} catch (error) {
|
|
730
|
+
return {
|
|
731
|
+
case: one,
|
|
732
|
+
ok: false,
|
|
733
|
+
error: error instanceof Error ? error.message : String(error),
|
|
734
|
+
ms: now() - started
|
|
735
|
+
};
|
|
736
|
+
}
|
|
737
|
+
const ms = now() - started;
|
|
738
|
+
if (one.request.feature === "draft-generation") {
|
|
739
|
+
let next = 0;
|
|
740
|
+
const checked = require_ai_writing.checkAiDrafts(raw, one.request, { newId: () => {
|
|
741
|
+
next += 1;
|
|
742
|
+
return `ai-check-${next}`;
|
|
743
|
+
} });
|
|
744
|
+
return checked.ok ? {
|
|
745
|
+
case: one,
|
|
746
|
+
ok: true,
|
|
747
|
+
drafts: checked.drafts,
|
|
748
|
+
ms
|
|
749
|
+
} : refused(one, raw, checked.refusal, ms);
|
|
750
|
+
}
|
|
751
|
+
if (one.request.feature === "item-critique") {
|
|
752
|
+
const checked = require_ai_writing.checkAiCritique(raw, one.request);
|
|
753
|
+
return checked.ok ? {
|
|
754
|
+
case: one,
|
|
755
|
+
ok: true,
|
|
756
|
+
critique: checked.critique,
|
|
757
|
+
ms
|
|
758
|
+
} : refused(one, raw, checked.refusal, ms);
|
|
759
|
+
}
|
|
760
|
+
if (one.request.feature === "pronunciation-coaching") {
|
|
761
|
+
const checked = require_ai_writing.checkAiCoaching(raw, one.request);
|
|
762
|
+
if (checked.ok) {
|
|
763
|
+
const { text, provenance, usage } = checked.coaching;
|
|
764
|
+
return {
|
|
765
|
+
case: one,
|
|
766
|
+
ok: true,
|
|
767
|
+
result: {
|
|
768
|
+
text,
|
|
769
|
+
...provenance !== void 0 ? { provenance } : {},
|
|
770
|
+
...usage !== void 0 ? { usage } : {}
|
|
771
|
+
},
|
|
772
|
+
coaching: checked.coaching,
|
|
773
|
+
ms
|
|
774
|
+
};
|
|
775
|
+
}
|
|
776
|
+
return refused(one, raw, checked.refusal, ms);
|
|
777
|
+
}
|
|
778
|
+
if (one.request.feature === "writing-feedback") {
|
|
779
|
+
const checked = require_ai_writing.checkAiWritingFeedback(raw, one.request);
|
|
780
|
+
if (checked.ok) {
|
|
781
|
+
const { text, provenance, usage } = checked.feedback;
|
|
782
|
+
return {
|
|
783
|
+
case: one,
|
|
784
|
+
ok: true,
|
|
785
|
+
result: {
|
|
786
|
+
text,
|
|
787
|
+
...provenance !== void 0 ? { provenance } : {},
|
|
788
|
+
...usage !== void 0 ? { usage } : {}
|
|
789
|
+
},
|
|
790
|
+
feedback: checked.feedback,
|
|
791
|
+
ms
|
|
792
|
+
};
|
|
793
|
+
}
|
|
794
|
+
return refused(one, raw, checked.refusal, ms);
|
|
795
|
+
}
|
|
796
|
+
const checked = one.request.feature === "explanation" ? require_ai_writing.checkAiExplanation(raw, one.request) : require_ai_writing.checkAiHint(raw, one.request);
|
|
797
|
+
if (checked.ok) return {
|
|
798
|
+
case: one,
|
|
799
|
+
ok: true,
|
|
800
|
+
result: checked.result,
|
|
801
|
+
ms
|
|
802
|
+
};
|
|
803
|
+
return refused(one, raw, checked.refusal, ms);
|
|
579
804
|
}
|
|
805
|
+
/** A case whose answer was refused, with the text that came back when there was one to read. */
|
|
580
806
|
function refused(one, raw, refusal, ms) {
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
|
|
807
|
+
const text = typeof raw?.text === "string" ? raw : void 0;
|
|
808
|
+
return {
|
|
809
|
+
case: one,
|
|
810
|
+
ok: false,
|
|
811
|
+
refusal,
|
|
812
|
+
...text !== void 0 ? { result: text } : {},
|
|
813
|
+
ms
|
|
814
|
+
};
|
|
589
815
|
}
|
|
816
|
+
/**
|
|
817
|
+
* Calls your ports on every case and reports what a learner would have seen.
|
|
818
|
+
*
|
|
819
|
+
* It never throws for a failing case: a port that rejects is an `error` in the
|
|
820
|
+
* report, so one bad call cannot hide the rest.
|
|
821
|
+
*/
|
|
590
822
|
async function runAiCheck(ports, options = {}) {
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
|
|
609
|
-
|
|
610
|
-
|
|
611
|
-
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
shown,
|
|
626
|
-
refused: refused2,
|
|
627
|
-
errors,
|
|
628
|
-
skipped: all.length - runnable.length,
|
|
629
|
-
byRefusal,
|
|
630
|
-
results
|
|
631
|
-
};
|
|
823
|
+
const all = options.cases ?? aiCheckCases();
|
|
824
|
+
const asked = Math.trunc(options.concurrency ?? 1);
|
|
825
|
+
const lanes = Number.isFinite(asked) && asked >= 1 ? Math.min(asked, 16) : 1;
|
|
826
|
+
const runnable = all.filter((one) => portFor(ports, one) !== void 0);
|
|
827
|
+
const results = new Array(runnable.length);
|
|
828
|
+
let next = 0;
|
|
829
|
+
const lane = async () => {
|
|
830
|
+
while (next < runnable.length) {
|
|
831
|
+
const mine = next;
|
|
832
|
+
next += 1;
|
|
833
|
+
const one = runnable[mine];
|
|
834
|
+
if (one === void 0) return;
|
|
835
|
+
results[mine] = await runCase(ports, one);
|
|
836
|
+
}
|
|
837
|
+
};
|
|
838
|
+
await Promise.all(Array.from({ length: Math.min(lanes, runnable.length) }, lane));
|
|
839
|
+
const byRefusal = Object.fromEntries(REFUSALS.map((reason) => [reason, 0]));
|
|
840
|
+
let shown = 0;
|
|
841
|
+
let refused = 0;
|
|
842
|
+
let errors = 0;
|
|
843
|
+
for (const result of results) if (result.ok) shown += 1;
|
|
844
|
+
else if (result.refusal !== void 0) {
|
|
845
|
+
refused += 1;
|
|
846
|
+
byRefusal[result.refusal] += 1;
|
|
847
|
+
} else errors += 1;
|
|
848
|
+
return {
|
|
849
|
+
total: results.length,
|
|
850
|
+
shown,
|
|
851
|
+
refused,
|
|
852
|
+
errors,
|
|
853
|
+
skipped: all.length - runnable.length,
|
|
854
|
+
byRefusal,
|
|
855
|
+
results
|
|
856
|
+
};
|
|
632
857
|
}
|
|
858
|
+
/**
|
|
859
|
+
* The report as a few lines for a CI log: the counts, then every case that
|
|
860
|
+
* failed, with what came back.
|
|
861
|
+
*
|
|
862
|
+
* The refused text is printed — this is your own test run, not a learner's
|
|
863
|
+
* screen, and a hint that gives the answer away is only fixable if you can
|
|
864
|
+
* read it.
|
|
865
|
+
*/
|
|
633
866
|
function formatAiCheckReport(report) {
|
|
634
|
-
|
|
635
|
-
|
|
636
|
-
|
|
637
|
-
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
|
|
641
|
-
|
|
642
|
-
|
|
643
|
-
|
|
644
|
-
}
|
|
645
|
-
|
|
646
|
-
|
|
647
|
-
|
|
648
|
-
}
|
|
649
|
-
const why = _nullishCoalesce(result.refusal, () => ( `port failed (${_nullishCoalesce(result.error, () => ( "no message"))})`));
|
|
650
|
-
lines.push(` \u2717 ${result.case.id} [${result.case.feature}] ${result.case.about}: ${why}`);
|
|
651
|
-
if (result.result !== void 0) {
|
|
652
|
-
lines.push(` ${result.result.text.replace(/\s+/g, " ").slice(0, 200)}`);
|
|
653
|
-
}
|
|
654
|
-
}
|
|
655
|
-
return lines.join("\n");
|
|
867
|
+
const lines = [];
|
|
868
|
+
const slowest = report.results.reduce((worst, one) => Math.max(worst, one.ms), 0);
|
|
869
|
+
lines.push(`ai-check: ${report.shown}/${report.total} shown, ${report.refused} refused, ${report.errors} failed${report.skipped > 0 ? `, ${report.skipped} skipped (no port)` : ""} — slowest ${slowest} ms`);
|
|
870
|
+
for (const reason of REFUSALS) {
|
|
871
|
+
const count = report.byRefusal[reason];
|
|
872
|
+
if (count > 0) lines.push(` ${reason}: ${count}`);
|
|
873
|
+
}
|
|
874
|
+
for (const result of report.results) {
|
|
875
|
+
if (result.ok) continue;
|
|
876
|
+
const why = result.refusal ?? `port failed (${result.error ?? "no message"})`;
|
|
877
|
+
lines.push(` ✗ ${result.case.id} [${result.case.feature}] ${result.case.about}: ${why}`);
|
|
878
|
+
if (result.result !== void 0) lines.push(` ${result.result.text.replace(/\s+/g, " ").slice(0, 200)}`);
|
|
879
|
+
}
|
|
880
|
+
return lines.join("\n");
|
|
656
881
|
}
|
|
882
|
+
//#endregion
|
|
883
|
+
exports.aiCheckCases = aiCheckCases;
|
|
884
|
+
exports.formatAiCheckReport = formatAiCheckReport;
|
|
885
|
+
exports.runAiCheck = runAiCheck;
|
|
657
886
|
|
|
658
|
-
|
|
659
|
-
|
|
660
|
-
|
|
661
|
-
exports.aiCheckCases = aiCheckCases; exports.formatAiCheckReport = formatAiCheckReport; exports.runAiCheck = runAiCheck;
|
|
662
887
|
//# sourceMappingURL=ai-check.cjs.map
|