@studious-lms/server 1.16.0 → 1.16.1

package/src/pipelines/gradeWorksheet.ts

@@ -71,6 +71,8 @@ ${responseText}
 INSTRUCTIONS:
 - Evaluate the student response against each mark scheme criterion.
 - Total points must not exceed ${maxPoints}.
+- In markschemeState, use the EXACT "id" string from the MARK SCHEME items above (e.g. "${markSchemeItems[0]?.id || 'ms-xxx'}"). Do NOT invent new IDs.
+- In annotations, use the same EXACT mark scheme id for "markSchemeId".
 - For each criterion, find the exact substring in the student's response that is relevant (the "quote" field). Copy the text verbatim — do not paraphrase.
 - If no part of the response addresses a criterion, still include an annotation with an empty quote and met=false.
 - Provide constructive feedback comments.
@@ -162,24 +164,44 @@ const gradeWorksheetQuestion = async (worksheetResponseId: string, worksheetQues
         const maxPoints = question.points || 0;
         const clampedPoints = Math.max(0, Math.min(Math.round((apiResponse as { points: number }).points), maxPoints));
 
-        const validMarkSchemeIds = new Set(
-            (Array.isArray(question.markScheme) ? question.markScheme as MarkSchemeItem[] : []).map(item => item.id),
-        );
+        const markSchemeItems = Array.isArray(question.markScheme) ? question.markScheme as MarkSchemeItem[] : [];
+        const validMarkSchemeIds = new Set(markSchemeItems.map(item => item.id));
 
-        const validatedMarkschemeState = (apiResponse as {
+        const aiMarkschemeState = (apiResponse as {
             markschemeState: { id: string; correct: boolean }[];
-        }).markschemeState
-            .filter(item => validMarkSchemeIds.has(item.id))
-            .reduce((acc, curr) => {
+        }).markschemeState;
+
+        const allIdsValid = aiMarkschemeState.length > 0 &&
+            aiMarkschemeState.every(entry => validMarkSchemeIds.has(entry.id));
+
+        let resolvedState: Record<string, boolean | unknown[]> = {};
+
+        if (allIdsValid) {
+            resolvedState = aiMarkschemeState.reduce((acc, curr) => {
                 acc["item-" + curr.id] = curr.correct;
                 return acc;
             }, {} as Record<string, boolean>);
+        } else if (markSchemeItems.length > 0) {
+            logger.warn('AI returned mismatched markscheme IDs, falling back to index mapping', {
+                expected: markSchemeItems.map(i => i.id),
+                received: aiMarkschemeState.map(i => i.id),
+            });
+            for (let i = 0; i < markSchemeItems.length; i++) {
+                const aiEntry = aiMarkschemeState[i];
+                resolvedState["item-" + markSchemeItems[i].id] = aiEntry?.correct ?? false;
+            }
+        }
 
         const annotations = (apiResponse as {
             annotations: { quote: string; markSchemeId: string; criterion: string; met: boolean }[];
         }).annotations
             .filter(a => a.quote.length === 0 || responseText.includes(a.quote));
 
+        // Store annotations separately so the boolean-only markschemeState
+        // isn't broken by non-boolean values (the frontend parser rejects
+        // the entire object if any value isn't a boolean).
+        resolvedState['_annotations'] = annotations;
+
         const updatedStudentQuestionProgress = await prisma.studentQuestionProgress.update({
             where: { id: studentQuestionProgress.id, status: {
                 not: {
@@ -190,10 +212,7 @@ const gradeWorksheetQuestion = async (worksheetResponseId: string, worksheetQues
                 status: JobStatus.COMPLETED,
                 isCorrect: (apiResponse as { isCorrect: boolean }).isCorrect,
                 points: clampedPoints,
-                markschemeState: {
-                    ...validatedMarkschemeState,
-                    _annotations: annotations,
-                },
+                markschemeState: resolvedState,
                 comments: {
                     create: (apiResponse as {
                         comments: string[];
@@ -368,6 +387,63 @@ export const cancelGradePipeline = async (worksheetResponseId: string, worksheet
     return updatedStudentQuestionProgress;
 };
 
+export const regradeAllPipeline = async (worksheetResponseId: string) => {
+    logger.info('Regrading all questions for worksheet response', { worksheetResponseId });
+    const worksheetResponse = await prisma.studentWorksheetResponse.findUnique({
+        where: { id: worksheetResponseId },
+        include: {
+            worksheet: true,
+            responses: {
+                where: {
+                    question: {
+                        type: {
+                            not: {
+                                in: [WorksheetQuestionType.MULTIPLE_CHOICE, WorksheetQuestionType.TRUE_FALSE],
+                            },
+                        },
+                    },
+                },
+                include: {
+                    question: true,
+                    comments: true,
+                },
+            },
+        },
+    });
+
+    if (!worksheetResponse) {
+        logger.error('Worksheet response not found');
+        throw new Error('Worksheet response not found');
+    }
+
+    // Reset all non-cancelled responses to PENDING
+    for (const response of worksheetResponse.responses) {
+        await prisma.studentQuestionProgress.updateMany({
+            where: {
+                id: response.id,
+                status: { not: JobStatus.CANCELLED },
+            },
+            data: { status: JobStatus.PENDING },
+        });
+
+        pusher.trigger(worksheetChannel(worksheetResponse.id), `set-pending`, {
+            id: response.id,
+        });
+    }
+
+    // Re-grade each question sequentially
+    for (const response of worksheetResponse.responses) {
+        try {
+            await gradeWorksheetQuestion(worksheetResponseId, response.id);
+        } catch (questionError) {
+            logger.error('Failed to regrade question, continuing with remaining', {
+                questionId: response.questionId,
+                error: questionError,
+            });
+        }
+    }
+};
+
 export const regradeWorksheetPipeline = async (worksheetResponseId: string, worksheetQuestionProgressId: string) => {
     logger.info('Regrading worksheet response', { worksheetResponseId, worksheetQuestionProgressId });
     try {

package/src/pipelines/labChatPrompt.ts

@@ -1,23 +1,10 @@
 import type { ModelMessage } from "ai";
+import {
+  buildChatModelMessages,
+  type ChatHistoryMessage,
+} from "../lib/chatAttachmentPrompt.js";
 
-export type ConversationMessage = {
-  id?: string;
-  content: string;
-  senderId: string;
-  createdAt?: Date | string | number;
-  sender?: {
-    username?: string | null;
-    profile?: {
-      displayName?: string | null;
-    } | null;
-  } | null;
-  attachments?: {
-    id: string;
-    name: string;
-    type: string;
-    url?: string;
-  }[];
-};
+export type ConversationMessage = ChatHistoryMessage;
 
 const LAB_CHAT_BASE_INSTRUCTIONS = [
   "IMPORTANT INSTRUCTIONS:",
@@ -47,17 +34,27 @@ const LAB_CHAT_REFERENCE_RULES = [
 
 const LAB_CHAT_DOC_RULES = [
   "WHEN CREATING FILES (generate_document tool):",
+  "- Call generate_document with title, content, and format.",
   "- You can generate MANY file types — not just PDF. Always pass format explicitly.",
   "- Match format to what the teacher asked for:",
-  "  * slides / presentation / deck → pptx",
-  "  * spreadsheet / grade table / data → xlsx or csv",
+  "  * slides / presentation / deck → pptx (separate slides with --- or Slide N: headers in content)",
+  "  * spreadsheet / grade table / data → xlsx or csv (include markdown tables or row data in content)",
   "  * editable handout / letter → docx",
   "  * printable worksheet / reading → pdf",
-  "  * diagram / chart image → png",
+  "  * diagram / chart / flowchart / figure / graph / visual → png (describe the visual in detail: labels, arrows, shapes, data — NOT pdf)",
   "  * notes / readme → md or txt",
-  "- Supported: pdf, docx, xlsx, pptx, csv, md, html, txt, json, xml, png, jpeg, gif, zip, py, js, ts, …",
-  "- Write complete classroom-ready content — no placeholders.",
-  "- In your spoken reply, name the actual format (e.g. \"PowerPoint\", \"spreadsheet\") — never say you only make PDFs.",
+  "  * source code → py/js/ts only when the teacher explicitly wants code files",
+  "- Supported: pdf, docx, xlsx, pptx, csv, md, html, txt, json, xml, tex, png, jpeg, gif, zip, py, js, ts, css, sh.",
+  "- Write complete classroom-ready content — no placeholders like [diagram here].",
+  "- One file per tool call; split long packs into logical separate documents if needed.",
+  "- In your spoken reply, name the actual format (e.g. \"PowerPoint\", \"spreadsheet\", \"diagram image\") — never say you only make PDFs.",
+  "- Never include download links or URLs; the file appears as an attachment on this message automatically after you call generate_document.",
+];
+
+const LAB_CHAT_ATTACHMENT_RULES = [
+  "WHEN THE TEACHER ATTACHES FILES:",
+  "- Read attached PDFs, Word docs, PowerPoints, spreadsheets, images, and text files — content is provided inline.",
+  "- Reference specifics from the attachment when answering or creating materials.",
 ];
 
 const LAB_CHAT_WORKSHEET_RULES = [
@@ -97,33 +94,6 @@ FILE GENERATION: You can attach pdf, docx, xlsx, pptx, csv, images, and more via
 
 const buildPrompt = (sections: string[]): string => sections.filter(Boolean).join("\n\n");
 
-const sortMessagesChronologically = (
-  recentMessages: ConversationMessage[],
-): ConversationMessage[] =>
-  recentMessages
-    .map((message, index) => ({ message, index }))
-    .sort((left, right) => {
-      const leftTime = left.message.createdAt
-        ? new Date(left.message.createdAt).getTime()
-        : null;
-      const rightTime = right.message.createdAt
-        ? new Date(right.message.createdAt).getTime()
-        : null;
-
-      if (leftTime != null && rightTime != null) {
-        return leftTime - rightTime;
-      }
-      if (leftTime != null) {
-        return -1;
-      }
-      if (rightTime != null) {
-        return 1;
-      }
-
-      return left.index - right.index;
-    })
-    .map(({ message }) => message);
-
 const LAB_CHAT_TOOL_RULES = [
   "HOW TO CREATE CONTENT (TOOLS):",
   "- You have tools to search class materials and to PROPOSE content for the teacher to review and accept.",
@@ -144,6 +114,7 @@ export const buildLabChatAgentSystemPrompt = (context: string): string =>
     LAB_CHAT_REFERENCE_RULES.join("\n"),
     LAB_CHAT_TOOL_RULES.join("\n"),
     LAB_CHAT_DOC_RULES.join("\n"),
+    LAB_CHAT_ATTACHMENT_RULES.join("\n"),
     LAB_CHAT_WORKSHEET_RULES.join("\n"),
     LAB_CHAT_SECTION_RULES.join("\n"),
     LAB_CHAT_ASSIGNMENT_RULES.join("\n"),
@@ -155,7 +126,7 @@ export const buildLabChatAgentSystemPrompt = (context: string): string =>
  * Returns a `system` string and AI-SDK `ModelMessage[]` history. Per-request
  * class context (IDs) is folded into the system string.
  */
-export const buildLabChatAgentMessages = ({
+export const buildLabChatAgentMessages = async ({
   context,
   classContext,
   recentMessages,
@@ -165,54 +136,14 @@ export const buildLabChatAgentMessages = ({
   classContext: string;
   recentMessages: ConversationMessage[];
   isAIUser: (senderId: string) => boolean;
-}): { system: string; messages: ModelMessage[] } => {
+}): Promise<{ system: string; messages: ModelMessage[] }> => {
   const system = `${buildLabChatAgentSystemPrompt(context)}\n\nCLASS CONTEXT (use these IDs when proposing assignments, worksheets, or attaching files):\n${classContext}`;
 
-  const messages: ModelMessage[] = sortMessagesChronologically(recentMessages).map(
-    (message) => {
-      const role = isAIUser(message.senderId) ? "assistant" : "user";
-      const senderName =
-        message.sender?.profile?.displayName || message.sender?.username || "Teacher";
-
-      if (role === "assistant") {
-        return { role, content: message.content };
-      }
-
-      // Build user message with attachments
-      const textContent = `${senderName}: ${message.content}`;
-      const attachments = message.attachments ?? [];
-
-      if (attachments.length === 0) {
-        return { role, content: textContent };
-      }
-
-      // Separate images (multimodal) from other files (text description)
-      const imageAttachments = attachments.filter((a) => a.type.startsWith('image/'));
-      const otherAttachments = attachments.filter((a) => !a.type.startsWith('image/'));
-
-      const parts: Array<{ type: string; text?: string; image?: string }> = [];
-
-      // Add text content with non-image file descriptions
-      let fullText = textContent;
-      if (otherAttachments.length > 0) {
-        const fileList = otherAttachments.map((a) => `${a.name} (${a.type})`).join(', ');
-        fullText += `\n[Attached files: ${fileList}]`;
-      }
-      parts.push({ type: 'text', text: fullText });
-
-      // Add image attachments as image parts for multimodal
-      for (const img of imageAttachments) {
-        if (img.url) {
-          parts.push({ type: 'image', image: img.url });
-        } else {
-          // Fallback: mention the image name in text if no URL available
-          parts.push({ type: 'text', text: `[Image attached: ${img.name}]` });
-        }
-      }
-
-      return { role, content: parts } as ModelMessage;
-    },
-  );
+  const messages = await buildChatModelMessages({
+    messages: recentMessages,
+    isAIUser,
+    defaultSenderLabel: 'Teacher',
+  });
 
   return { system, messages };
 };

package/src/routers/auth.ts

@@ -5,6 +5,7 @@ import {
   protectedProcedure,
   publicProcedure,
 } from "../trpc.js";
+import { CURRENT_TERMS_VERSION } from "../lib/legal.js";
 import {
   register,
   login,
@@ -27,6 +28,16 @@ const registerSchema = z
     email: z.string().email("Invalid email address"),
     password: z.string().min(6, "Password must be at least 6 characters"),
     confirmPassword: z.string(),
+    agreeToTerms: z.literal(true, {
+      errorMap: () => ({
+        message: "You must accept the Terms of Service and Privacy Policy",
+      }),
+    }),
+    termsVersion: z.literal(CURRENT_TERMS_VERSION, {
+      errorMap: () => ({
+        message: "Please accept the current Terms of Service and Privacy Policy",
+      }),
+    }),
   })
   .refine((data) => data.password === data.confirmPassword, {
     message: "Passwords don't match",
@@ -41,6 +52,8 @@ export const authRouter = createTRPCRouter({
         username: input.username,
         email: input.email,
         password: input.password,
+        agreeToTerms: input.agreeToTerms,
+        termsVersion: input.termsVersion,
       })
     ),
 

package/src/routers/user.ts

@@ -1,3 +1,4 @@
+import { TRPCError } from "@trpc/server";
 import { z } from "zod";
 import { createTRPCRouter, protectedProcedure } from "../trpc.js";
 import {
@@ -5,6 +6,10 @@ import {
   updateProfile,
   getUploadUrl,
 } from "../services/user.js";
+import {
+  deleteUserAccount,
+  exportUserData,
+} from "../services/userPrivacy.js";
 
 const fileUploadSchema = z.object({
   filePath: z.string().min(1, "File path is required"),
@@ -79,4 +84,25 @@ export const userRouter = createTRPCRouter({
     .mutation(({ ctx, input }) =>
       getUploadUrl(ctx.user!.id, input.fileName, input.fileType)
     ),
+
+  exportData: protectedProcedure.query(({ ctx }) =>
+    exportUserData(ctx.user!.id),
+  ),
+
+  deleteAccount: protectedProcedure
+    .input(
+      z.object({
+        confirmUsername: z.string().min(1),
+      }),
+    )
+    .mutation(async ({ ctx, input }) => {
+      const profile = await getProfile(ctx.user!.id);
+      if (profile.username !== input.confirmUsername) {
+        throw new TRPCError({
+          code: "BAD_REQUEST",
+          message: "Username confirmation does not match",
+        });
+      }
+      return deleteUserAccount(ctx.user!.id);
+    }),
 });

package/src/routers/worksheet.ts

@@ -30,6 +30,7 @@ import {
   answerWorksheetQuestion,
   cancelGrading,
   regradeQuestion,
+  regradeAll,
   gradeAnswer,
   addCommentToResponse,
 } from "../services/worksheet.js";
@@ -370,6 +371,16 @@ export const worksheetRouter = createTRPCRouter({
       regradeQuestion(input.worksheetResponseId, input.progressId, ctx.user!.id)
     ),
 
+  regradeAll: protectedProcedure
+    .input(
+      z.object({
+        worksheetResponseId: z.string(),
+      })
+    )
+    .mutation(({ ctx, input }) =>
+      regradeAll(input.worksheetResponseId, ctx.user!.id)
+    ),
+
   gradeAnswer: protectedProcedure
     .input(
       z.object({

package/src/services/aiChatStream.ts

@@ -87,6 +87,13 @@ export function applyResolvedFormatsToStreamParts(
   });
 }
 
+/** Tools that queue async work after the model stream; UI stays loading until the pipeline finishes. */
+const DEFERRED_COMPLETION_TOOLS = new Set(['generate_document']);
+
+function isDeferredCompletionTool(toolName: string): boolean {
+  return DEFERRED_COMPLETION_TOOLS.has(toolName);
+}
+
 /** Attach file IDs to `generate_document` tool parts for inline download chips. */
 export function enrichStreamPartsWithGeneratedFiles(
   parts: StreamPart[],
@@ -126,11 +133,24 @@ export function enrichStreamPartsWithGeneratedFiles(
       ...part,
       fileId: file.fileId,
       fileName: file.fileName,
+      status: 'completed',
       ...(file.format ? { format: file.format } : {}),
     };
   });
 }
 
+/** Mark deferred tools completed (file linked) or failed (generation did not produce a file). */
+export function finalizeDeferredToolParts(parts: StreamPart[]): StreamPart[] {
+  return parts.map((part) => {
+    if (part.type !== 'tool' || !isDeferredCompletionTool(part.toolName)) {
+      return part;
+    }
+    if (part.fileId) return { ...part, status: 'completed' };
+    if (part.status === 'executing') return { ...part, status: 'failed' };
+    return part;
+  });
+}
+
 function toolFieldFromInput(input: unknown, key: string): string | undefined {
   if (input && typeof input === 'object') {
     const val = (input as Record<string, unknown>)[key];
@@ -271,8 +291,15 @@ export async function streamChatResponse(
         }
         case 'tool-result':
         case 'tool-error': {
-          const status: 'completed' | 'failed' =
-            part.type === 'tool-error' ? 'failed' : 'completed';
+          const failed = part.type === 'tool-error';
+          const deferred = isDeferredCompletionTool(part.toolName);
+          // Deferred tools (e.g. generate_document) only queue work during the
+          // stream — keep the spinner until the pipeline attaches a file.
+          const status: 'executing' | 'completed' | 'failed' = failed
+            ? 'failed'
+            : deferred
+              ? 'executing'
+              : 'completed';
           const title = toolTitleFromInput(part.input);
           const format = toolFieldFromInput(part.input, 'format');
           const idx = toolPartIndexById.get(part.toolCallId);
@@ -287,7 +314,7 @@ export async function streamChatResponse(
             toolName: part.toolName,
             input: part.input,
             output: part.type === 'tool-result' ? part.output : undefined,
-            status,
+            status: failed ? 'failed' : 'completed',
           });
           await broadcast('message-tool-call', {
             messageId: aiMessageId,

package/src/services/aiProposalTools.ts

@@ -237,25 +237,26 @@ export function labProposalTools(proposals: LabProposals): ToolSet {
 
     generate_document: {
       description:
-        'Generate a downloadable file and attach it to this message. You are NOT limited to PDF — pick the best format for the request (pptx for slides/presentations, xlsx/csv for data, docx for editable handouts, pdf for print-ready docs, png for diagrams, etc.). Always set format explicitly.',
+        'Generate a downloadable file and attach it to this message. Pick the best format for the request — pptx for slides, xlsx/csv for data, docx for editable handouts, pdf for print-ready docs, png/jpeg for charts/diagrams/figures (NOT pdf for visuals). Use py/js/ts only when the teacher explicitly wants source code. Always set format explicitly.',
       inputSchema: jsonSchema({
         type: 'object',
         properties: {
+          title: { type: 'string', description: 'Document title' },
+          content: {
+            type: 'string',
+            description:
+              'Full body. Structure by format: pptx — separate slides with "---" or "Slide N:" headers; xlsx/csv — markdown tables or row data; png/jpeg — describe the visual in detail (shapes, labels, arrows, data points, colors); pdf/docx — headings, lists, tables; code formats — complete runnable source.',
+          },
           format: {
             type: 'string',
             description:
-              'Output file type. REQUIRED. pptx=slides/presentation, xlsx/csv=spreadsheet/data, docx=Word handout, pdf=printable doc, md/txt/html=text, png/jpeg/gif=image, py/js/ts=code.',
+              'Output file type. REQUIRED. pptx=slides, xlsx/csv=spreadsheet, docx=Word, pdf=printable doc, png/jpeg/gif=chart/diagram/figure image, md/txt/html=text. py/js/ts/css/sh=source code only when requested.',
             enum: [
               'pdf', 'docx', 'doc', 'xlsx', 'pptx', 'csv', 'md', 'html', 'txt',
               'json', 'xml', 'tex', 'png', 'jpeg', 'jpg', 'gif', 'zip', 'tar',
               'py', 'js', 'ts', 'css', 'sh',
             ],
           },
-          title: { type: 'string', description: 'Document title' },
-          content: {
-            type: 'string',
-            description: 'Full body — markdown or plain text with headings, lists, tables, slide notes, etc.',
-          },
         },
         required: ['title', 'content', 'format'],
       }),

package/src/services/aiTools.ts

@@ -251,7 +251,7 @@ export function aiTools(classId: string): ToolSet {
           const client = new OpenAI();
           const response = await client.responses.create({
             model: 'gpt-4o-mini',
-            tools: [{ type: 'web_search_preview' }],
+            tools: [{ type: 'web_search' }],
             input: query,
           });
 

package/src/services/auth.ts

@@ -10,6 +10,7 @@ import { sendMail } from "../utils/email.js";
 import { prismaWrapper } from "../utils/prismaWrapper.js";
 import { env } from "../lib/config/env.js";
 import { logger } from "../utils/logger.js";
+import { CURRENT_TERMS_VERSION } from "../lib/legal.js";
 import {
   findUserByUsername,
   findUserByUsernameOrEmail,
@@ -32,8 +33,17 @@ export async function register(input: {
   username: string;
   email: string;
   password: string;
+  agreeToTerms: true;
+  termsVersion: string;
 }) {
-  const { username, email, password } = input;
+  const { username, email, password, termsVersion } = input;
+
+  if (termsVersion !== CURRENT_TERMS_VERSION) {
+    throw new TRPCError({
+      code: "BAD_REQUEST",
+      message: "Please accept the current Terms of Service and Privacy Policy.",
+    });
+  }
 
   const existingUser = await prismaWrapper.findFirst(
     () => findUserByUsernameOrEmail(username, email),
@@ -70,7 +80,9 @@ export async function register(input: {
         username,
         email,
         password: await hash(password, 10),
-        verified: true,
+        verified: false,
+        termsAcceptedAt: new Date(),
+        termsVersion,
       }),
     "creating new user during registration"
   );

package/src/services/marketing.ts

@@ -4,7 +4,7 @@
 import { v4 } from "uuid";
 import {
   createSchoolDevelopmentProgram,
-  findSchoolDevelopmentProgramById,
+  findSchoolDevelopmentProgramStatusById,
   createEarlyAccessRequest,
 } from "../models/marketing.js";
 
@@ -42,9 +42,9 @@ export async function createSchoolDevelopmentProgramRecord(input: {
   return { id: program.id };
 }
 
-/** Look up a school development program by ID. */
+/** Public application status lookup (no contact PII). */
 export async function searchSchoolDevelopmentPrograms(id: string) {
-  return findSchoolDevelopmentProgramById(id);
+  return findSchoolDevelopmentProgramStatusById(id);
 }
 
 /** Create an early access request. */

package/src/services/newtonChat.ts

@@ -159,13 +159,6 @@ export async function postToNewtonChat(
   ensureMessageHasContentOrAttachments(content, fileIds);
   const normalizedContent = content.trim();
 
-  if (!normalizedContent) {
-    throw new TRPCError({
-      code: "BAD_REQUEST",
-      message: "Newton chat messages must include text content",
-    });
-  }
-
   const result = await prisma.$transaction(async (tx) => {
     const message = await tx.message.create({
       data: {