@studious-lms/server 1.15.0 → 1.16.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/lib/chatAttachmentPrompt.d.ts +47 -0
- package/dist/lib/chatAttachmentPrompt.d.ts.map +1 -0
- package/dist/lib/chatAttachmentPrompt.js +190 -0
- package/dist/lib/chatAttachmentPrompt.js.map +1 -0
- package/dist/lib/inferDocumentFormat.js +3 -3
- package/dist/lib/inferDocumentFormat.js.map +1 -1
- package/dist/pipelines/aiLabChat.d.ts.map +1 -1
- package/dist/pipelines/aiLabChat.js +7 -35
- package/dist/pipelines/aiLabChat.js.map +1 -1
- package/dist/pipelines/aiLabChatContract.d.ts +2 -2
- package/dist/pipelines/aiNewtonChat.d.ts +1 -1
- package/dist/pipelines/aiNewtonChat.d.ts.map +1 -1
- package/dist/pipelines/aiNewtonChat.js +20 -19
- package/dist/pipelines/aiNewtonChat.js.map +1 -1
- package/dist/pipelines/gradeWorksheet.d.ts +1 -0
- package/dist/pipelines/gradeWorksheet.d.ts.map +1 -1
- package/dist/pipelines/gradeWorksheet.js +146 -36
- package/dist/pipelines/gradeWorksheet.js.map +1 -1
- package/dist/pipelines/labChatPrompt.d.ts +4 -20
- package/dist/pipelines/labChatPrompt.d.ts.map +1 -1
- package/dist/pipelines/labChatPrompt.js +22 -65
- package/dist/pipelines/labChatPrompt.js.map +1 -1
- package/dist/routers/_app.d.ts +14 -0
- package/dist/routers/_app.d.ts.map +1 -1
- package/dist/routers/worksheet.d.ts +7 -0
- package/dist/routers/worksheet.d.ts.map +1 -1
- package/dist/routers/worksheet.js +8 -3
- package/dist/routers/worksheet.js.map +1 -1
- package/dist/services/aiProposalTools.js +5 -5
- package/dist/services/aiProposalTools.js.map +1 -1
- package/dist/services/newtonChat.d.ts.map +1 -1
- package/dist/services/newtonChat.js +2 -8
- package/dist/services/newtonChat.js.map +1 -1
- package/dist/services/openaiDocumentGen.d.ts.map +1 -1
- package/dist/services/openaiDocumentGen.js +58 -17
- package/dist/services/openaiDocumentGen.js.map +1 -1
- package/dist/services/worksheet.d.ts +1 -0
- package/dist/services/worksheet.d.ts.map +1 -1
- package/dist/services/worksheet.js +13 -3
- package/dist/services/worksheet.js.map +1 -1
- package/dist/workers/processors/aiNewtonChat.d.ts.map +1 -1
- package/dist/workers/processors/aiNewtonChat.js +3 -3
- package/dist/workers/processors/aiNewtonChat.js.map +1 -1
- package/package.json +2 -1
- package/src/lib/chatAttachmentPrompt.ts +270 -0
- package/src/lib/inferDocumentFormat.ts +1 -1
- package/src/pipelines/aiLabChat.ts +5 -43
- package/src/pipelines/aiNewtonChat.ts +20 -23
- package/src/pipelines/gradeWorksheet.ts +171 -38
- package/src/pipelines/labChatPrompt.ts +27 -99
- package/src/routers/worksheet.ts +11 -0
- package/src/services/aiProposalTools.ts +3 -3
- package/src/services/newtonChat.ts +0 -7
- package/src/services/openaiDocumentGen.ts +57 -15
- package/src/services/worksheet.ts +20 -1
- package/src/workers/processors/aiNewtonChat.ts +0 -1
- package/tests/integration/routers/newtonChat.test.ts +9 -8
- package/tests/unit/lib/chatAttachmentPrompt.test.ts +135 -0
- package/tests/unit/lib/inferDocumentFormat.test.ts +10 -0
- package/tests/unit/pipelines/labChatPrompt.test.ts +6 -6
|
@@ -0,0 +1,270 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Shared chat attachment handling for Newton + Lab pipelines.
|
|
3
|
+
*
|
|
4
|
+
* - Images + PDFs: inlined as multimodal parts (bytes from GCS).
|
|
5
|
+
* - Office docs, spreadsheets, slides, text, csv, json, html: text extracted
|
|
6
|
+
* via officeparser / UTF-8 decode and appended to the user message.
|
|
7
|
+
* - Audio/video/zip: filename mention only (no extraction).
|
|
8
|
+
*/
|
|
9
|
+
import type { FilePart, ImagePart, ModelMessage, TextPart, UserContent } from 'ai';
|
|
10
|
+
import { OfficeParser } from 'officeparser';
|
|
11
|
+
import { bucket } from './googleCloudStorage.js';
|
|
12
|
+
import { logger } from '../utils/logger.js';
|
|
13
|
+
|
|
14
|
+
/** Model used for AI chats that may include document attachments. */
|
|
15
|
+
export const AI_CHAT_MODEL = 'gpt-5.5';
|
|
16
|
+
|
|
17
|
+
const MAX_EXTRACTED_CHARS = 80_000;
|
|
18
|
+
const IMAGE_TYPE_PREFIX = 'image/';
|
|
19
|
+
|
|
20
|
+
export type ChatAttachmentRef = {
|
|
21
|
+
id: string;
|
|
22
|
+
name: string;
|
|
23
|
+
type: string;
|
|
24
|
+
path: string;
|
|
25
|
+
};
|
|
26
|
+
|
|
27
|
+
export type LoadedChatAttachment = ChatAttachmentRef & {
|
|
28
|
+
data?: Uint8Array;
|
|
29
|
+
extractedText?: string;
|
|
30
|
+
};
|
|
31
|
+
|
|
32
|
+
export type ChatHistoryMessage = {
|
|
33
|
+
id?: string;
|
|
34
|
+
content: string;
|
|
35
|
+
senderId: string;
|
|
36
|
+
createdAt?: Date | string | number;
|
|
37
|
+
sender?: {
|
|
38
|
+
username?: string | null;
|
|
39
|
+
profile?: { displayName?: string | null } | null;
|
|
40
|
+
} | null;
|
|
41
|
+
attachments?: ChatAttachmentRef[];
|
|
42
|
+
};
|
|
43
|
+
|
|
44
|
+
type UserContentPart = TextPart | ImagePart | FilePart;
|
|
45
|
+
|
|
46
|
+
const TEXT_DECODE_TYPES = new Set([
|
|
47
|
+
'text/plain',
|
|
48
|
+
'text/csv',
|
|
49
|
+
'text/markdown',
|
|
50
|
+
'text/html',
|
|
51
|
+
'application/json',
|
|
52
|
+
]);
|
|
53
|
+
|
|
54
|
+
const MENTION_ONLY_PREFIXES = ['audio/', 'video/'];
|
|
55
|
+
const MENTION_ONLY_TYPES = new Set(['application/zip']);
|
|
56
|
+
|
|
57
|
+
function extension(name: string): string {
|
|
58
|
+
return name.split('.').pop()?.toLowerCase() ?? '';
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
function isPdf(attachment: { type: string; name: string }): boolean {
|
|
62
|
+
return attachment.type === 'application/pdf' || extension(attachment.name) === 'pdf';
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
function isImage(attachment: { type: string }): boolean {
|
|
66
|
+
return attachment.type.startsWith(IMAGE_TYPE_PREFIX);
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
function isMentionOnly(attachment: { type: string; name: string }): boolean {
|
|
70
|
+
if (MENTION_ONLY_TYPES.has(attachment.type)) return true;
|
|
71
|
+
return MENTION_ONLY_PREFIXES.some((prefix) => attachment.type.startsWith(prefix));
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
function isUtf8Text(attachment: { type: string; name: string }): boolean {
|
|
75
|
+
if (TEXT_DECODE_TYPES.has(attachment.type)) return true;
|
|
76
|
+
const ext = extension(attachment.name);
|
|
77
|
+
return ['txt', 'csv', 'md', 'html', 'json'].includes(ext);
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
function shouldDownload(attachment: ChatAttachmentRef): boolean {
|
|
81
|
+
return Boolean(attachment.path) && !isMentionOnly(attachment);
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
async function extractTextFromBytes(
|
|
85
|
+
bytes: Uint8Array,
|
|
86
|
+
name: string,
|
|
87
|
+
type: string,
|
|
88
|
+
): Promise<string | null> {
|
|
89
|
+
if (isUtf8Text({ type, name })) {
|
|
90
|
+
return new TextDecoder('utf-8', { fatal: false })
|
|
91
|
+
.decode(bytes)
|
|
92
|
+
.slice(0, MAX_EXTRACTED_CHARS);
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
if (isImage({ type }) || isPdf({ type, name })) {
|
|
96
|
+
return null;
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
if (isMentionOnly({ type, name })) {
|
|
100
|
+
return null;
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
try {
|
|
104
|
+
const ast = await OfficeParser.parseOffice(Buffer.from(bytes), {
|
|
105
|
+
includeRawContent: false,
|
|
106
|
+
});
|
|
107
|
+
const text = ast.toText().trim();
|
|
108
|
+
return text ? text.slice(0, MAX_EXTRACTED_CHARS) : null;
|
|
109
|
+
} catch (error) {
|
|
110
|
+
logger.warn('Failed to extract chat attachment text', { name, type, error });
|
|
111
|
+
return null;
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
/** Download bytes from GCS and extract text for every supported attachment type. */
|
|
116
|
+
export async function loadChatAttachments(
|
|
117
|
+
attachments: ChatAttachmentRef[],
|
|
118
|
+
): Promise<LoadedChatAttachment[]> {
|
|
119
|
+
return Promise.all(
|
|
120
|
+
attachments.map(async (attachment) => {
|
|
121
|
+
if (!shouldDownload(attachment)) {
|
|
122
|
+
return { ...attachment };
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
try {
|
|
126
|
+
const [bytes] = await bucket.file(attachment.path).download();
|
|
127
|
+
const data = new Uint8Array(bytes);
|
|
128
|
+
const extractedText =
|
|
129
|
+
(await extractTextFromBytes(data, attachment.name, attachment.type)) ??
|
|
130
|
+
undefined;
|
|
131
|
+
return { ...attachment, data, extractedText };
|
|
132
|
+
} catch (error) {
|
|
133
|
+
logger.warn('Failed to load chat attachment from storage', {
|
|
134
|
+
attachmentId: attachment.id,
|
|
135
|
+
path: attachment.path,
|
|
136
|
+
type: attachment.type,
|
|
137
|
+
error,
|
|
138
|
+
});
|
|
139
|
+
return { ...attachment };
|
|
140
|
+
}
|
|
141
|
+
}),
|
|
142
|
+
);
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
/** @deprecated Use {@link loadChatAttachments} */
|
|
146
|
+
export const loadPromptAttachments = loadChatAttachments;
|
|
147
|
+
|
|
148
|
+
export function buildUserMessageContentWithAttachments(
|
|
149
|
+
textContent: string,
|
|
150
|
+
attachments: LoadedChatAttachment[],
|
|
151
|
+
): UserContent {
|
|
152
|
+
if (attachments.length === 0) {
|
|
153
|
+
return textContent;
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
const parts: UserContentPart[] = [];
|
|
157
|
+
let fullText = textContent;
|
|
158
|
+
|
|
159
|
+
for (const attachment of attachments) {
|
|
160
|
+
if (attachment.extractedText) {
|
|
161
|
+
fullText += `\n\n[Content of ${attachment.name}]:\n${attachment.extractedText}`;
|
|
162
|
+
}
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
const unhandled = attachments.filter((a) => {
|
|
166
|
+
if (a.extractedText) return false;
|
|
167
|
+
if (isImage(a) && a.data) return false;
|
|
168
|
+
if (isPdf(a) && a.data) return false;
|
|
169
|
+
return true;
|
|
170
|
+
});
|
|
171
|
+
|
|
172
|
+
if (unhandled.length > 0) {
|
|
173
|
+
const fileList = unhandled
|
|
174
|
+
.map((a) => `${a.name} (${a.type})`)
|
|
175
|
+
.join(', ');
|
|
176
|
+
fullText = fullText
|
|
177
|
+
? `${fullText}\n[Attached files (content not extracted): ${fileList}]`
|
|
178
|
+
: `[Attached files (content not extracted): ${fileList}]`;
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
if (fullText.trim()) {
|
|
182
|
+
parts.push({ type: 'text', text: fullText });
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
for (const image of attachments) {
|
|
186
|
+
if (isImage(image) && image.data) {
|
|
187
|
+
parts.push({ type: 'image', image: image.data });
|
|
188
|
+
}
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
for (const file of attachments) {
|
|
192
|
+
if (isPdf(file) && file.data) {
|
|
193
|
+
parts.push({
|
|
194
|
+
type: 'file',
|
|
195
|
+
data: file.data,
|
|
196
|
+
mediaType: 'application/pdf',
|
|
197
|
+
filename: file.name,
|
|
198
|
+
});
|
|
199
|
+
}
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
if (parts.length === 0) {
|
|
203
|
+
return textContent || '[Attached files]';
|
|
204
|
+
}
|
|
205
|
+
|
|
206
|
+
if (parts.length === 1 && parts[0].type === 'text') {
|
|
207
|
+
return parts[0].text;
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
return parts;
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
export function sortChatMessagesChronologically<T extends ChatHistoryMessage>(
|
|
214
|
+
messages: T[],
|
|
215
|
+
): T[] {
|
|
216
|
+
return messages
|
|
217
|
+
.map((message, index) => ({ message, index }))
|
|
218
|
+
.sort((left, right) => {
|
|
219
|
+
const leftTime = left.message.createdAt
|
|
220
|
+
? new Date(left.message.createdAt).getTime()
|
|
221
|
+
: null;
|
|
222
|
+
const rightTime = right.message.createdAt
|
|
223
|
+
? new Date(right.message.createdAt).getTime()
|
|
224
|
+
: null;
|
|
225
|
+
|
|
226
|
+
if (leftTime != null && rightTime != null) {
|
|
227
|
+
return leftTime - rightTime;
|
|
228
|
+
}
|
|
229
|
+
if (leftTime != null) return -1;
|
|
230
|
+
if (rightTime != null) return 1;
|
|
231
|
+
return left.index - right.index;
|
|
232
|
+
})
|
|
233
|
+
.map(({ message }) => message);
|
|
234
|
+
}
|
|
235
|
+
|
|
236
|
+
/** Build ModelMessage[] with shared attachment loading for Newton + Lab. */
|
|
237
|
+
export async function buildChatModelMessages({
|
|
238
|
+
messages,
|
|
239
|
+
isAIUser,
|
|
240
|
+
defaultSenderLabel,
|
|
241
|
+
}: {
|
|
242
|
+
messages: ChatHistoryMessage[];
|
|
243
|
+
isAIUser: (senderId: string) => boolean;
|
|
244
|
+
defaultSenderLabel: string;
|
|
245
|
+
}): Promise<ModelMessage[]> {
|
|
246
|
+
const modelMessages: ModelMessage[] = [];
|
|
247
|
+
|
|
248
|
+
for (const message of sortChatMessagesChronologically(messages)) {
|
|
249
|
+
if (isAIUser(message.senderId)) {
|
|
250
|
+
modelMessages.push({ role: 'assistant', content: message.content });
|
|
251
|
+
continue;
|
|
252
|
+
}
|
|
253
|
+
|
|
254
|
+
const senderName =
|
|
255
|
+
message.sender?.profile?.displayName ||
|
|
256
|
+
message.sender?.username ||
|
|
257
|
+
defaultSenderLabel;
|
|
258
|
+
const label = message.content.trim()
|
|
259
|
+
? `${senderName}: ${message.content}`
|
|
260
|
+
: senderName;
|
|
261
|
+
const attachments = await loadChatAttachments(message.attachments ?? []);
|
|
262
|
+
|
|
263
|
+
modelMessages.push({
|
|
264
|
+
role: 'user',
|
|
265
|
+
content: buildUserMessageContentWithAttachments(label, attachments),
|
|
266
|
+
});
|
|
267
|
+
}
|
|
268
|
+
|
|
269
|
+
return modelMessages;
|
|
270
|
+
}
|
|
@@ -9,7 +9,7 @@ const FORMAT_RULES: FormatRule[] = [
|
|
|
9
9
|
{ format: 'xlsx', pattern: /\b(spreadsheet|excel|workbook|xlsx|gradebook|grade book|\.xlsx)\b/i },
|
|
10
10
|
{ format: 'csv', pattern: /\b(csv|comma[- ]separated|\.csv)\b/i },
|
|
11
11
|
{ format: 'docx', pattern: /\b(word doc(ument)?|docx|editable handout|\.docx)\b/i },
|
|
12
|
-
{ format: 'png', pattern: /\b(diagram|chart|graph|plot|figure|infographic|\.png)\b/i },
|
|
12
|
+
{ format: 'png', pattern: /\b(diagram|flowchart|chart|graph|plot|figure|infographic|visual|illustration|mind ?map|timeline|\.png)\b/i },
|
|
13
13
|
{ format: 'md', pattern: /\b(markdown|readme|\.md)\b/i },
|
|
14
14
|
{ format: 'html', pattern: /\b(html|web ?page|\.html)\b/i },
|
|
15
15
|
{ format: 'txt', pattern: /\b(plain text|text file|\.txt)\b/i },
|
|
@@ -12,15 +12,15 @@ import {
|
|
|
12
12
|
} from "../utils/inference.js";
|
|
13
13
|
import { logger } from "../utils/logger.js";
|
|
14
14
|
import { v4 } from "uuid";
|
|
15
|
-
import { bucket
|
|
15
|
+
import { bucket } from "../lib/googleCloudStorage.js";
|
|
16
16
|
import {
|
|
17
17
|
generateDocumentWithOpenAI,
|
|
18
18
|
normalizeDocumentFormat,
|
|
19
19
|
} from "../services/openaiDocumentGen.js";
|
|
20
20
|
import { inferDocumentFormat } from "../lib/inferDocumentFormat.js";
|
|
21
21
|
import { broadcastTextChunks, broadcastStreamComplete } from "../utils/streamBroadcast.js";
|
|
22
|
+
import { AI_CHAT_MODEL } from "../lib/chatAttachmentPrompt.js";
|
|
22
23
|
import { buildLabChatAgentMessages } from "./labChatPrompt.js";
|
|
23
|
-
import type { ConversationMessage } from "./labChatPrompt.js";
|
|
24
24
|
import {
|
|
25
25
|
applyResolvedFormatsToStreamParts,
|
|
26
26
|
enrichStreamPartsWithGeneratedFiles,
|
|
@@ -213,42 +213,6 @@ const loadRecentLabChatMessages = async (
|
|
|
213
213
|
.slice(0, limit);
|
|
214
214
|
};
|
|
215
215
|
|
|
216
|
-
/**
|
|
217
|
-
* Resolves signed URLs for image attachments in messages so the LLM can see them.
|
|
218
|
-
* Non-image files are passed as text descriptions only.
|
|
219
|
-
*/
|
|
220
|
-
async function resolveAttachmentUrls(
|
|
221
|
-
messages: RecentLabChatMessage[],
|
|
222
|
-
): Promise<ConversationMessage[]> {
|
|
223
|
-
const resolved: ConversationMessage[] = [];
|
|
224
|
-
|
|
225
|
-
for (const msg of messages) {
|
|
226
|
-
const attachments = msg.attachments ?? [];
|
|
227
|
-
if (attachments.length === 0) {
|
|
228
|
-
resolved.push({ ...msg, attachments: [] });
|
|
229
|
-
continue;
|
|
230
|
-
}
|
|
231
|
-
|
|
232
|
-
const withUrls = await Promise.all(
|
|
233
|
-
attachments.map(async (a) => {
|
|
234
|
-
if (a.type.startsWith('image/')) {
|
|
235
|
-
try {
|
|
236
|
-
const url = await getSignedUrl(a.path);
|
|
237
|
-
return { id: a.id, name: a.name, type: a.type, url };
|
|
238
|
-
} catch {
|
|
239
|
-
return { id: a.id, name: a.name, type: a.type };
|
|
240
|
-
}
|
|
241
|
-
}
|
|
242
|
-
return { id: a.id, name: a.name, type: a.type };
|
|
243
|
-
}),
|
|
244
|
-
);
|
|
245
|
-
|
|
246
|
-
resolved.push({ ...msg, attachments: withUrls });
|
|
247
|
-
}
|
|
248
|
-
|
|
249
|
-
return resolved;
|
|
250
|
-
}
|
|
251
|
-
|
|
252
216
|
/**
|
|
253
217
|
* Builds schema-aware context for the AI from class data.
|
|
254
218
|
* Formats entities with IDs so the model can reference them when creating assignments.
|
|
@@ -511,13 +475,10 @@ export const generateAndSendLabIntroduction = async (
|
|
|
511
475
|
assignments: classData.assignments,
|
|
512
476
|
});
|
|
513
477
|
|
|
514
|
-
|
|
515
|
-
const messagesWithAttachments = await resolveAttachmentUrls(recentMessages.reverse());
|
|
516
|
-
|
|
517
|
-
const { system, messages } = buildLabChatAgentMessages({
|
|
478
|
+
const { system, messages } = await buildLabChatAgentMessages({
|
|
518
479
|
context: fullLabChat.context,
|
|
519
480
|
classContext,
|
|
520
|
-
recentMessages:
|
|
481
|
+
recentMessages: recentMessages.reverse(),
|
|
521
482
|
isAIUser,
|
|
522
483
|
});
|
|
523
484
|
|
|
@@ -535,6 +496,7 @@ export const generateAndSendLabIntroduction = async (
|
|
|
535
496
|
streamChatResponse({
|
|
536
497
|
conversationId,
|
|
537
498
|
aiMessageId: emitOptions.aiMessageId,
|
|
499
|
+
model: AI_CHAT_MODEL,
|
|
538
500
|
system,
|
|
539
501
|
messages,
|
|
540
502
|
tools,
|
|
@@ -13,9 +13,10 @@ import { isAIUser } from "../utils/aiUser.js";
|
|
|
13
13
|
import { broadcastStreamComplete } from "../utils/streamBroadcast.js";
|
|
14
14
|
import { streamChatResponse } from "../services/aiChatStream.js";
|
|
15
15
|
import { aiTools } from "../services/aiTools.js";
|
|
16
|
-
import
|
|
17
|
-
|
|
18
|
-
|
|
16
|
+
import {
|
|
17
|
+
AI_CHAT_MODEL,
|
|
18
|
+
buildChatModelMessages,
|
|
19
|
+
} from "../lib/chatAttachmentPrompt.js";
|
|
19
20
|
|
|
20
21
|
// AI Policy Levels Configuration
|
|
21
22
|
// Used across assignment creation, editing, and display
|
|
@@ -148,6 +149,7 @@ export const generateAndSendNewtonIntroduction = async (
|
|
|
148
149
|
const { text: response } = await streamChatResponse({
|
|
149
150
|
conversationId,
|
|
150
151
|
aiMessageId,
|
|
152
|
+
model: AI_CHAT_MODEL,
|
|
151
153
|
system: systemPrompt,
|
|
152
154
|
messages: [
|
|
153
155
|
{
|
|
@@ -222,7 +224,6 @@ export const generateAndSendNewtonIntroduction = async (
|
|
|
222
224
|
*/
|
|
223
225
|
export const generateAndSendNewtonResponse = async (
|
|
224
226
|
newtonChatId: string,
|
|
225
|
-
studentMessage: string,
|
|
226
227
|
conversationId: string,
|
|
227
228
|
submission: {
|
|
228
229
|
id: string;
|
|
@@ -258,6 +259,14 @@ export const generateAndSendNewtonIntroduction = async (
|
|
|
258
259
|
},
|
|
259
260
|
},
|
|
260
261
|
},
|
|
262
|
+
attachments: {
|
|
263
|
+
select: {
|
|
264
|
+
id: true,
|
|
265
|
+
name: true,
|
|
266
|
+
type: true,
|
|
267
|
+
path: true,
|
|
268
|
+
},
|
|
269
|
+
},
|
|
261
270
|
},
|
|
262
271
|
orderBy: {
|
|
263
272
|
createdAt: 'desc',
|
|
@@ -302,26 +311,13 @@ export const generateAndSendNewtonIntroduction = async (
|
|
|
302
311
|
|
|
303
312
|
IMPORTANT:
|
|
304
313
|
- Keep responses conversational and educational
|
|
305
|
-
- Focus on helping the student learn, not just completing the assignment
|
|
306
|
-
|
|
307
|
-
const modelMessages: ModelMessage[] = [];
|
|
308
|
-
|
|
309
|
-
// Add recent conversation history
|
|
310
|
-
recentMessages.reverse().forEach(msg => {
|
|
311
|
-
const role = isAIUser(msg.senderId) ? 'assistant' : 'user';
|
|
312
|
-
const senderName = msg.sender?.profile?.displayName || msg.sender?.username || 'Student';
|
|
313
|
-
const content = isAIUser(msg.senderId) ? msg.content : `${senderName}: ${msg.content}`;
|
|
314
|
-
|
|
315
|
-
modelMessages.push({
|
|
316
|
-
role: role as 'user' | 'assistant',
|
|
317
|
-
content,
|
|
318
|
-
});
|
|
319
|
-
});
|
|
314
|
+
- Focus on helping the student learn, not just completing the assignment
|
|
315
|
+
- When the student attaches files (PDFs, images), read and reference them in your reply`;
|
|
320
316
|
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
317
|
+
const modelMessages = await buildChatModelMessages({
|
|
318
|
+
messages: [...recentMessages].reverse(),
|
|
319
|
+
isAIUser,
|
|
320
|
+
defaultSenderLabel: 'Student',
|
|
325
321
|
});
|
|
326
322
|
|
|
327
323
|
const combinedSystem = [
|
|
@@ -337,6 +333,7 @@ export const generateAndSendNewtonIntroduction = async (
|
|
|
337
333
|
const { text: response, parts: streamParts } = await streamChatResponse({
|
|
338
334
|
conversationId,
|
|
339
335
|
aiMessageId,
|
|
336
|
+
model: AI_CHAT_MODEL,
|
|
340
337
|
system: combinedSystem,
|
|
341
338
|
messages: modelMessages,
|
|
342
339
|
...(newtonClassId ? { tools: aiTools(newtonClassId), maxSteps: 5 } : {}),
|
|
@@ -20,6 +20,65 @@ const removeAllPreviousAIComments = async (worksheetQuestionProgressId: string)
|
|
|
20
20
|
});
|
|
21
21
|
};
|
|
22
22
|
|
|
23
|
+
type MarkSchemeItem = { id: string; points: number; description?: string; text?: string };
|
|
24
|
+
|
|
25
|
+
const gradingResponseSchema = z.object({
|
|
26
|
+
isCorrect: z.boolean(),
|
|
27
|
+
points: z.number(),
|
|
28
|
+
markschemeState: z.array(z.object({
|
|
29
|
+
id: z.string(),
|
|
30
|
+
correct: z.boolean(),
|
|
31
|
+
})),
|
|
32
|
+
comments: z.array(z.string()),
|
|
33
|
+
annotations: z.array(z.object({
|
|
34
|
+
quote: z.string().describe('An exact substring from the student response that is relevant to this criterion.'),
|
|
35
|
+
markSchemeId: z.string().describe('The id of the mark scheme criterion this quote relates to.'),
|
|
36
|
+
criterion: z.string().describe('Short label describing what this mark scheme criterion evaluates.'),
|
|
37
|
+
met: z.boolean().describe('Whether the quoted text satisfies this criterion.'),
|
|
38
|
+
})).describe('Highlight specific parts of the student response and map each to a mark scheme criterion.'),
|
|
39
|
+
});
|
|
40
|
+
|
|
41
|
+
function buildGradingPrompt(
|
|
42
|
+
question: { question: string; answer: string; type: WorksheetQuestionType; points: number; markScheme: unknown },
|
|
43
|
+
responseText: string,
|
|
44
|
+
): string {
|
|
45
|
+
const markSchemeItems = Array.isArray(question.markScheme) ? question.markScheme as MarkSchemeItem[] : [];
|
|
46
|
+
const markSchemeBlock = markSchemeItems.length > 0
|
|
47
|
+
? markSchemeItems.map((item, i) =>
|
|
48
|
+
` ${i + 1}. [id="${item.id}", ${item.points} pts] ${item.description || item.text || `Criterion ${i + 1}`}`
|
|
49
|
+
).join('\n')
|
|
50
|
+
: ' No mark scheme provided — grade based on correctness and completeness.';
|
|
51
|
+
|
|
52
|
+
const maxPoints = question.points || markSchemeItems.reduce((sum, item) => sum + (item.points || 0), 0);
|
|
53
|
+
|
|
54
|
+
return `Grade the following student response.
|
|
55
|
+
|
|
56
|
+
Question type: ${question.type}
|
|
57
|
+
Maximum points: ${maxPoints}
|
|
58
|
+
|
|
59
|
+
QUESTION:
|
|
60
|
+
${question.question}
|
|
61
|
+
|
|
62
|
+
CORRECT ANSWER:
|
|
63
|
+
${question.answer || 'No answer key provided.'}
|
|
64
|
+
|
|
65
|
+
MARK SCHEME:
|
|
66
|
+
${markSchemeBlock}
|
|
67
|
+
|
|
68
|
+
STUDENT RESPONSE:
|
|
69
|
+
${responseText}
|
|
70
|
+
|
|
71
|
+
INSTRUCTIONS:
|
|
72
|
+
- Evaluate the student response against each mark scheme criterion.
|
|
73
|
+
- Total points must not exceed ${maxPoints}.
|
|
74
|
+
- In markschemeState, use the EXACT "id" string from the MARK SCHEME items above (e.g. "${markSchemeItems[0]?.id || 'ms-xxx'}"). Do NOT invent new IDs.
|
|
75
|
+
- In annotations, use the same EXACT mark scheme id for "markSchemeId".
|
|
76
|
+
- For each criterion, find the exact substring in the student's response that is relevant (the "quote" field). Copy the text verbatim — do not paraphrase.
|
|
77
|
+
- If no part of the response addresses a criterion, still include an annotation with an empty quote and met=false.
|
|
78
|
+
- Provide constructive feedback comments.
|
|
79
|
+
- Mark isCorrect=true only if the student earned full or near-full marks.`;
|
|
80
|
+
}
|
|
81
|
+
|
|
23
82
|
const gradeWorksheetQuestion = async (worksheetResponseId: string, worksheetQuestionProgressId: string) => {
|
|
24
83
|
|
|
25
84
|
await removeAllPreviousAIComments(worksheetQuestionProgressId);
|
|
@@ -83,38 +142,12 @@ const gradeWorksheetQuestion = async (worksheetResponseId: string, worksheetQues
|
|
|
83
142
|
return updatedProgress;
|
|
84
143
|
}
|
|
85
144
|
|
|
86
|
-
const comments = studentQuestionProgress.comments;
|
|
87
|
-
|
|
88
145
|
try {
|
|
146
|
+
const prompt = buildGradingPrompt(question, responseText);
|
|
147
|
+
|
|
89
148
|
const apiResponse = await inference(
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
Question: ${question.question}
|
|
93
|
-
Response: ${responseText}
|
|
94
|
-
|
|
95
|
-
Comments: ${comments.map((comment) => comment.content).join('\n')}
|
|
96
|
-
Mark Scheme: ${JSON.stringify(question.markScheme)}
|
|
97
|
-
|
|
98
|
-
Justify your reasoning by including comment(s) and mark the question please.
|
|
99
|
-
Return ONLY JSON in the following format (fill in the values as per the question):
|
|
100
|
-
{
|
|
101
|
-
"isCorrect": <boolean>,
|
|
102
|
-
"points": <number>,
|
|
103
|
-
"markschemeState": [
|
|
104
|
-
{ "id": <string>, "correct": <boolean> }
|
|
105
|
-
],
|
|
106
|
-
"comments": [<string>, ...]
|
|
107
|
-
}
|
|
108
|
-
`,
|
|
109
|
-
z.object({
|
|
110
|
-
isCorrect: z.boolean(),
|
|
111
|
-
points: z.number(),
|
|
112
|
-
markschemeState: z.array(z.object({
|
|
113
|
-
id: z.string(),
|
|
114
|
-
correct: z.boolean(),
|
|
115
|
-
})), // @note: this has to be converted to [id: string]: correct boolean
|
|
116
|
-
comments: z.array(z.string()),
|
|
117
|
-
}),
|
|
149
|
+
prompt,
|
|
150
|
+
gradingResponseSchema,
|
|
118
151
|
{
|
|
119
152
|
reason: 'worksheet_autograde',
|
|
120
153
|
metadata: {
|
|
@@ -128,6 +161,47 @@ const gradeWorksheetQuestion = async (worksheetResponseId: string, worksheetQues
|
|
|
128
161
|
throw error;
|
|
129
162
|
});
|
|
130
163
|
|
|
164
|
+
const maxPoints = question.points || 0;
|
|
165
|
+
const clampedPoints = Math.max(0, Math.min(Math.round((apiResponse as { points: number }).points), maxPoints));
|
|
166
|
+
|
|
167
|
+
const markSchemeItems = Array.isArray(question.markScheme) ? question.markScheme as MarkSchemeItem[] : [];
|
|
168
|
+
const validMarkSchemeIds = new Set(markSchemeItems.map(item => item.id));
|
|
169
|
+
|
|
170
|
+
const aiMarkschemeState = (apiResponse as {
|
|
171
|
+
markschemeState: { id: string; correct: boolean }[];
|
|
172
|
+
}).markschemeState;
|
|
173
|
+
|
|
174
|
+
const allIdsValid = aiMarkschemeState.length > 0 &&
|
|
175
|
+
aiMarkschemeState.every(entry => validMarkSchemeIds.has(entry.id));
|
|
176
|
+
|
|
177
|
+
let resolvedState: Record<string, boolean | unknown[]> = {};
|
|
178
|
+
|
|
179
|
+
if (allIdsValid) {
|
|
180
|
+
resolvedState = aiMarkschemeState.reduce((acc, curr) => {
|
|
181
|
+
acc["item-" + curr.id] = curr.correct;
|
|
182
|
+
return acc;
|
|
183
|
+
}, {} as Record<string, boolean>);
|
|
184
|
+
} else if (markSchemeItems.length > 0) {
|
|
185
|
+
logger.warn('AI returned mismatched markscheme IDs, falling back to index mapping', {
|
|
186
|
+
expected: markSchemeItems.map(i => i.id),
|
|
187
|
+
received: aiMarkschemeState.map(i => i.id),
|
|
188
|
+
});
|
|
189
|
+
for (let i = 0; i < markSchemeItems.length; i++) {
|
|
190
|
+
const aiEntry = aiMarkschemeState[i];
|
|
191
|
+
resolvedState["item-" + markSchemeItems[i].id] = aiEntry?.correct ?? false;
|
|
192
|
+
}
|
|
193
|
+
}
|
|
194
|
+
|
|
195
|
+
const annotations = (apiResponse as {
|
|
196
|
+
annotations: { quote: string; markSchemeId: string; criterion: string; met: boolean }[];
|
|
197
|
+
}).annotations
|
|
198
|
+
.filter(a => a.quote.length === 0 || responseText.includes(a.quote));
|
|
199
|
+
|
|
200
|
+
// Store annotations separately so the boolean-only markschemeState
|
|
201
|
+
// isn't broken by non-boolean values (the frontend parser rejects
|
|
202
|
+
// the entire object if any value isn't a boolean).
|
|
203
|
+
resolvedState['_annotations'] = annotations;
|
|
204
|
+
|
|
131
205
|
const updatedStudentQuestionProgress = await prisma.studentQuestionProgress.update({
|
|
132
206
|
where: { id: studentQuestionProgress.id, status: {
|
|
133
207
|
not: {
|
|
@@ -137,13 +211,8 @@ const gradeWorksheetQuestion = async (worksheetResponseId: string, worksheetQues
|
|
|
137
211
|
data: {
|
|
138
212
|
status: JobStatus.COMPLETED,
|
|
139
213
|
isCorrect: (apiResponse as { isCorrect: boolean }).isCorrect,
|
|
140
|
-
points:
|
|
141
|
-
markschemeState:
|
|
142
|
-
markschemeState: { id: string; correct: boolean }[];
|
|
143
|
-
}).markschemeState.reduce((acc, curr) => {
|
|
144
|
-
acc["item-" + curr.id] = curr.correct;
|
|
145
|
-
return acc;
|
|
146
|
-
}, {} as Record<string, boolean>),
|
|
214
|
+
points: clampedPoints,
|
|
215
|
+
markschemeState: resolvedState,
|
|
147
216
|
comments: {
|
|
148
217
|
create: (apiResponse as {
|
|
149
218
|
comments: string[];
|
|
@@ -279,7 +348,14 @@ export const gradeWorksheetPipeline = async (worksheetResponseId: string) => {
|
|
|
279
348
|
continue;
|
|
280
349
|
}
|
|
281
350
|
|
|
282
|
-
|
|
351
|
+
try {
|
|
352
|
+
await gradeWorksheetQuestion(worksheetResponseId, response.id);
|
|
353
|
+
} catch (questionError) {
|
|
354
|
+
logger.error('Failed to grade individual question, continuing with remaining questions', {
|
|
355
|
+
questionId: response.questionId,
|
|
356
|
+
error: questionError,
|
|
357
|
+
});
|
|
358
|
+
}
|
|
283
359
|
|
|
284
360
|
};
|
|
285
361
|
};
|
|
@@ -311,6 +387,63 @@ export const cancelGradePipeline = async (worksheetResponseId: string, worksheet
|
|
|
311
387
|
return updatedStudentQuestionProgress;
|
|
312
388
|
};
|
|
313
389
|
|
|
390
|
+
export const regradeAllPipeline = async (worksheetResponseId: string) => {
|
|
391
|
+
logger.info('Regrading all questions for worksheet response', { worksheetResponseId });
|
|
392
|
+
const worksheetResponse = await prisma.studentWorksheetResponse.findUnique({
|
|
393
|
+
where: { id: worksheetResponseId },
|
|
394
|
+
include: {
|
|
395
|
+
worksheet: true,
|
|
396
|
+
responses: {
|
|
397
|
+
where: {
|
|
398
|
+
question: {
|
|
399
|
+
type: {
|
|
400
|
+
not: {
|
|
401
|
+
in: [WorksheetQuestionType.MULTIPLE_CHOICE, WorksheetQuestionType.TRUE_FALSE],
|
|
402
|
+
},
|
|
403
|
+
},
|
|
404
|
+
},
|
|
405
|
+
},
|
|
406
|
+
include: {
|
|
407
|
+
question: true,
|
|
408
|
+
comments: true,
|
|
409
|
+
},
|
|
410
|
+
},
|
|
411
|
+
},
|
|
412
|
+
});
|
|
413
|
+
|
|
414
|
+
if (!worksheetResponse) {
|
|
415
|
+
logger.error('Worksheet response not found');
|
|
416
|
+
throw new Error('Worksheet response not found');
|
|
417
|
+
}
|
|
418
|
+
|
|
419
|
+
// Reset all non-cancelled responses to PENDING
|
|
420
|
+
for (const response of worksheetResponse.responses) {
|
|
421
|
+
await prisma.studentQuestionProgress.updateMany({
|
|
422
|
+
where: {
|
|
423
|
+
id: response.id,
|
|
424
|
+
status: { not: JobStatus.CANCELLED },
|
|
425
|
+
},
|
|
426
|
+
data: { status: JobStatus.PENDING },
|
|
427
|
+
});
|
|
428
|
+
|
|
429
|
+
pusher.trigger(worksheetChannel(worksheetResponse.id), `set-pending`, {
|
|
430
|
+
id: response.id,
|
|
431
|
+
});
|
|
432
|
+
}
|
|
433
|
+
|
|
434
|
+
// Re-grade each question sequentially
|
|
435
|
+
for (const response of worksheetResponse.responses) {
|
|
436
|
+
try {
|
|
437
|
+
await gradeWorksheetQuestion(worksheetResponseId, response.id);
|
|
438
|
+
} catch (questionError) {
|
|
439
|
+
logger.error('Failed to regrade question, continuing with remaining', {
|
|
440
|
+
questionId: response.questionId,
|
|
441
|
+
error: questionError,
|
|
442
|
+
});
|
|
443
|
+
}
|
|
444
|
+
}
|
|
445
|
+
};
|
|
446
|
+
|
|
314
447
|
export const regradeWorksheetPipeline = async (worksheetResponseId: string, worksheetQuestionProgressId: string) => {
|
|
315
448
|
logger.info('Regrading worksheet response', { worksheetResponseId, worksheetQuestionProgressId });
|
|
316
449
|
try {
|