@studious-lms/server 1.14.0 → 1.16.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@studious-lms/server",
-  "version": "1.14.0",
+  "version": "1.16.0",
   "description": "Backend server for Studious application",
   "main": "dist/exportType.js",
   "types": "dist/exportType.d.ts",

package/prisma/migrations/20260610140000_expand_embedding_source_types/migration.sql

@@ -0,0 +1,4 @@
+-- AlterEnum
+ALTER TYPE "EmbeddingSourceType" ADD VALUE 'DISCUSSION';
+ALTER TYPE "EmbeddingSourceType" ADD VALUE 'SUBMISSION';
+ALTER TYPE "EmbeddingSourceType" ADD VALUE 'SYLLABUS';

package/prisma/schema.prisma

@@ -977,6 +977,9 @@ enum EmbeddingSourceType {
   WORKSHEET
   ANNOUNCEMENT
   SECTION
+  DISCUSSION
+  SUBMISSION
+  SYLLABUS
 }
 
 model Embedding {

package/src/lib/inferDocumentFormat.ts

@@ -0,0 +1,58 @@
+import type { DocumentFormat } from '../services/openaiDocumentGen.js';
+import { normalizeDocumentFormat } from '../services/openaiDocumentGen.js';
+
+type FormatRule = { format: DocumentFormat; pattern: RegExp };
+
+/** Ordered most-specific first. */
+const FORMAT_RULES: FormatRule[] = [
+  { format: 'pptx', pattern: /\b(presentation|powerpoint|power point|slides?|slide deck|deck|pptx|\.pptx)\b/i },
+  { format: 'xlsx', pattern: /\b(spreadsheet|excel|workbook|xlsx|gradebook|grade book|\.xlsx)\b/i },
+  { format: 'csv', pattern: /\b(csv|comma[- ]separated|\.csv)\b/i },
+  { format: 'docx', pattern: /\b(word doc(ument)?|docx|editable handout|\.docx)\b/i },
+  { format: 'png', pattern: /\b(diagram|chart|graph|plot|figure|infographic|\.png)\b/i },
+  { format: 'md', pattern: /\b(markdown|readme|\.md)\b/i },
+  { format: 'html', pattern: /\b(html|web ?page|\.html)\b/i },
+  { format: 'txt', pattern: /\b(plain text|text file|\.txt)\b/i },
+];
+
+function inferFromText(...parts: Array<string | null | undefined>): DocumentFormat | null {
+  const hay = parts.filter(Boolean).join('\n');
+  if (!hay.trim()) return null;
+
+  for (const { format, pattern } of FORMAT_RULES) {
+    if (pattern.test(hay)) return format;
+  }
+  return null;
+}
+
+/**
+ * Resolve the output format for a generated document.
+ *
+ * The lab model often passes `format: "pdf"` even for presentations/spreadsheets.
+ * Keyword signals from the teacher request, title, and body take precedence over
+ * a generic pdf default; a non-pdf explicit format is always respected.
+ */
+export function inferDocumentFormat(
+  title: string,
+  content: string,
+  explicit?: string | null,
+  userRequest?: string | null,
+): DocumentFormat {
+  const explicitNorm = explicit?.trim()
+    ? normalizeDocumentFormat(explicit)
+    : null;
+
+  const fromKeywords = inferFromText(userRequest, title, content);
+
+  // Non-pdf explicit choice wins (model or tool args).
+  if (explicitNorm && explicitNorm !== 'pdf') {
+    return explicitNorm;
+  }
+
+  // Override pdf when the request clearly asks for slides, spreadsheets, etc.
+  if (fromKeywords && fromKeywords !== 'pdf') {
+    return fromKeywords;
+  }
+
+  return explicitNorm ?? 'pdf';
+}

package/src/pipelines/aiLabChat.ts

@@ -1,23 +1,31 @@
 /**
  * AI lab chat pipeline – generates lab introductions and responses.
- * Can create worksheets, sections, assignments, and PDF docs from AI output.
+ * Can create worksheets, sections, assignments, and generated files from AI output.
  */
 import { isAIUser } from "../utils/aiUser.js";
 import { prisma } from "../lib/prisma.js";
-import { pusher, teacherChannel } from "../lib/pusher.js";
+import { pusher, chatChannel, teacherChannel } from "../lib/pusher.js";
 import type { Assignment, Class, File, Section, User } from "@prisma/client";
 import {
   completePendingAIMessage,
   failPendingAIMessage,
 } from "../utils/inference.js";
 import { logger } from "../utils/logger.js";
-import { createPdf } from "../lib/jsonConversion.js";
 import { v4 } from "uuid";
-import { bucket } from "../lib/googleCloudStorage.js";
-import { DocumentBlock } from "../lib/jsonStyles.js";
+import { bucket, getSignedUrl } from "../lib/googleCloudStorage.js";
+import { generateDocumentWithOpenAI } from "../services/openaiDocumentGen.js";
+import { inferDocumentFormat } from "../lib/inferDocumentFormat.js";
+import { normalizeDocumentFormat } from "../services/openaiDocumentGen.js";
 import { broadcastTextChunks, broadcastStreamComplete } from "../utils/streamBroadcast.js";
 import { buildLabChatAgentMessages } from "./labChatPrompt.js";
-import { streamChatResponse } from "../services/aiChatStream.js";
+import type { ConversationMessage } from "./labChatPrompt.js";
+import {
+  applyResolvedFormatsToStreamParts,
+  enrichStreamPartsWithGeneratedFiles,
+  streamChatResponse,
+  type GeneratedDocFileLink,
+  type StreamPart,
+} from "../services/aiChatStream.js";
 import { aiTools } from "../services/aiTools.js";
 import { labProposalTools, createLabProposals } from "../services/aiProposalTools.js";
 
@@ -74,6 +82,12 @@ type RecentLabChatMessage = {
       displayName: string | null;
     } | null;
   } | null;
+  attachments: {
+    id: string;
+    name: string;
+    type: string;
+    path: string;
+  }[];
 };
 
 /**
@@ -118,6 +132,14 @@ const loadRecentLabChatMessages = async (
         },
       },
     },
+    attachments: {
+      select: {
+        id: true,
+        name: true,
+        type: true,
+        path: true,
+      },
+    },
   };
 
   const newestMessages = await prisma.message.findMany({
@@ -188,6 +210,42 @@ const loadRecentLabChatMessages = async (
     .slice(0, limit);
 };
 
+/**
+ * Resolves signed URLs for image attachments in messages so the LLM can see them.
+ * Non-image files are passed as text descriptions only.
+ */
+async function resolveAttachmentUrls(
+  messages: RecentLabChatMessage[],
+): Promise<ConversationMessage[]> {
+  const resolved: ConversationMessage[] = [];
+
+  for (const msg of messages) {
+    const attachments = msg.attachments ?? [];
+    if (attachments.length === 0) {
+      resolved.push({ ...msg, attachments: [] });
+      continue;
+    }
+
+    const withUrls = await Promise.all(
+      attachments.map(async (a) => {
+        if (a.type.startsWith('image/')) {
+          try {
+            const url = await getSignedUrl(a.path);
+            return { id: a.id, name: a.name, type: a.type, url };
+          } catch {
+            return { id: a.id, name: a.name, type: a.type };
+          }
+        }
+        return { id: a.id, name: a.name, type: a.type };
+      }),
+    );
+
+    resolved.push({ ...msg, attachments: withUrls });
+  }
+
+  return resolved;
+}
+
 /**
  * Builds schema-aware context for the AI from class data.
  * Formats entities with IDs so the model can reference them when creating assignments.
@@ -320,7 +378,7 @@ export const generateAndSendLabIntroduction = async (
     aiMessageId: string,
   ): Promise<void> => {
     const introMessage =
-      "Hello teaching team! I'm your AI assistant for course material development. I'll help you create educational content - when you have a clear request, I'll produce it directly. I only ask questions when I need to clarify the topic or scope. What would you like to work on?";
+      "Hello teaching team! I'm your AI assistant for course material development. I can create worksheets, assignments, and downloadable files — PDFs, Word docs, PowerPoints, spreadsheets, and more. When you have a clear request, I'll produce it directly. I only ask questions when I need to clarify the topic or scope. What would you like to work on?";
 
     try {
       await broadcastTextChunks(conversationId, aiMessageId, introMessage);
@@ -352,7 +410,7 @@ export const generateAndSendLabIntroduction = async (
    */
   export const generateAndSendLabResponse = async (
     labChatId: string,
-    _teacherMessage: string,
+    teacherMessage: string,
     emitOptions: { classId: string; aiMessageId: string }
   ): Promise<void> => {
     try {
@@ -450,10 +508,13 @@ export const generateAndSendLabIntroduction = async (
         assignments: classData.assignments,
       });
 
+      // Resolve signed URLs for image attachments so the LLM can see them
+      const messagesWithAttachments = await resolveAttachmentUrls(recentMessages.reverse());
+
       const { system, messages } = buildLabChatAgentMessages({
         context: fullLabChat.context,
         classContext,
-        recentMessages: recentMessages.reverse(),
+        recentMessages: messagesWithAttachments,
         isAIUser,
       });
 
@@ -467,7 +528,7 @@ export const generateAndSendLabIntroduction = async (
         ...labProposalTools(proposals),
       };
 
-      const { text: streamedText, parts: streamParts } = await withTimeout(
+      const { text: streamedText, parts: rawStreamParts } = await withTimeout(
         streamChatResponse({
           conversationId,
           aiMessageId: emitOptions.aiMessageId,
@@ -486,78 +547,128 @@ export const generateAndSendLabIntroduction = async (
       // during the stream by `streamChatResponse`. Just close the stream.
       await broadcastStreamComplete(conversationId, emitOptions.aiMessageId);
 
-      // Generate PDFs for any proposed documents and persist the message.
+      // `generate_document` is an inline tool that only QUEUES into `proposals.docs`
+      // during the stream. OpenAI Code Interpreter renders files here, after the
+      // stream ends, then we link them to this message.
       try {
         const attachmentIds: string[] = [];
-        // Generate PDFs if docs were proposed
-        if (proposals.docs.length > 0) {
-          
-  
-          for (let i = 0; i < proposals.docs.length; i++) {
-            const doc = proposals.docs[i];
-            if (!doc.title || !doc.blocks || !Array.isArray(doc.blocks)) {
-              logger.error(`Document ${i + 1} is missing title or blocks`);
-              continue;
-            } 
-  
-  
-            try {
-              let pdfBytes = await createPdf(doc.blocks as DocumentBlock[]);            
-              if (pdfBytes) {
-                // Sanitize filename - remove special characters and limit length
-                const sanitizedTitle = doc.title
-                  .replace(/[^a-zA-Z0-9\s\-_]/g, '')
-                  .replace(/\s+/g, '_')
-                  .substring(0, 50);
-                
-                const filename = `${sanitizedTitle}_${v4().substring(0, 8)}.pdf`;
-                const filePath = `class/generated/${fullLabChat.classId}/${filename}`;
-  
-                logger.info(`PDF ${i + 1} generated successfully`, { labChatId, title: doc.title });
-                
-                // Upload directly to Google Cloud Storage
-                const gcsFile = bucket.file(filePath);
-                await gcsFile.save(Buffer.from(pdfBytes), {
-                  metadata: {
-                    contentType: 'application/pdf',
-                  }
-                });
-      
-                logger.info(`PDF ${i + 1} uploaded successfully`, { labChatId, filename });
-  
-                const file = await prisma.file.create({
-                  data: {
-                    name: filename,
-                    path: filePath,
-                    type: 'application/pdf',
-                    size: pdfBytes.length,
-                    userId: fullLabChat.createdById,
-                    uploadStatus: 'COMPLETED',
-                    uploadedAt: new Date(),
-                  },
-                });
-                attachmentIds.push(file.id);
-              } else {
-                logger.error(`PDF ${i + 1} creation returned undefined/null`, { labChatId, title: doc.title });
-              }
-            } catch (pdfError) {
-              logger.error(`PDF creation threw an error for document ${i + 1}:`, { 
-                error: pdfError instanceof Error ? {
-                  message: pdfError.message,
-                  stack: pdfError.stack,
-                  name: pdfError.name
-                } : pdfError, 
+        const generatedDocFiles: GeneratedDocFileLink[] = [];
+        const resolvedDocs: Array<{ title: string; format: string }> = [];
+
+        for (const doc of proposals.docs) {
+          if (!doc.title || !doc.content?.trim()) {
+            logger.error('Document missing title or content', { labChatId, doc });
+            continue;
+          }
+
+          const format = inferDocumentFormat(
+            doc.title,
+            doc.content,
+            doc.format,
+            teacherMessage,
+          );
+          resolvedDocs.push({ title: doc.title, format });
+
+          if (format !== normalizeDocumentFormat(doc.format)) {
+            logger.info('Document format overridden from model default', {
+              labChatId,
+              title: doc.title,
+              modelFormat: doc.format,
+              resolvedFormat: format,
+            });
+          }
+
+          try {
+            const generated = await generateDocumentWithOpenAI({
+              title: doc.title,
+              content: doc.content,
+              format,
+            });
+            if (!generated) {
+              logger.error('OpenAI document generation returned null', {
                 labChatId,
-                title: doc.title
+                title: doc.title,
+                format: doc.format,
               });
+              continue;
             }
+
+            const sanitizedTitle = doc.title
+              .replace(/[^a-zA-Z0-9\s\-_]/g, '')
+              .replace(/\s+/g, '_')
+              .substring(0, 50);
+            const filename = `${sanitizedTitle}_${v4().substring(0, 8)}.${generated.format}`;
+            const filePath = `class/generated/${fullLabChat.classId}/${filename}`;
+
+            await bucket.file(filePath).save(generated.bytes, {
+              metadata: { contentType: generated.mimeType },
+            });
+
+            const file = await prisma.file.create({
+              data: {
+                name: filename,
+                path: filePath,
+                type: generated.mimeType,
+                size: generated.bytes.length,
+                userId: fullLabChat.createdById,
+                conversationId,
+                messageId: emitOptions.aiMessageId,
+                uploadStatus: 'COMPLETED',
+                uploadedAt: new Date(),
+              },
+            });
+
+            attachmentIds.push(file.id);
+            generatedDocFiles.push({
+              title: doc.title,
+              fileId: file.id,
+              fileName: filename,
+              format: generated.format,
+            });
+          } catch (docError) {
+            logger.error('Document generation failed', {
+              error: docError instanceof Error ? {
+                message: docError.message,
+                stack: docError.stack,
+                name: docError.name,
+              } : docError,
+              labChatId,
+              title: doc.title,
+              format: doc.format,
+            });
           }
         }
-  
+        
+        const streamParts: StreamPart[] = enrichStreamPartsWithGeneratedFiles(
+          applyResolvedFormatsToStreamParts(rawStreamParts, resolvedDocs),
+          generatedDocFiles,
+        );
+
+        for (const part of streamParts) {
+          if (
+            part.type === 'tool' &&
+            part.toolName === 'generate_document' &&
+            part.fileId
+          ) {
+            await pusher
+              .trigger(chatChannel(conversationId), 'message-tool-call', {
+                messageId: emitOptions.aiMessageId,
+                toolCallId: part.toolCallId,
+                toolName: part.toolName,
+                ...(part.title ? { title: part.title } : {}),
+                ...(part.format ? { format: part.format } : {}),
+                fileId: part.fileId,
+                fileName: part.fileName,
+                status: 'completed',
+              })
+              .catch(() => {});
+          }
+        }
+
         await completePendingAIMessage(emitOptions.aiMessageId, streamedText, {
-          attachments: {
-            connect: attachmentIds.map(id => ({ id })),
-          },
+          ...(attachmentIds.length > 0 && {
+            attachments: { connect: attachmentIds.map((id) => ({ id })) },
+          }),
           meta: {
             assignmentsToCreate: proposals.assignments.length > 0
               ? proposals.assignments.map(assignment => ({ ...assignment, id: v4() }))

package/src/pipelines/aiLabChatContract.ts

@@ -47,26 +47,16 @@ export const assignmentToCreateSchema = z.object({
   })),
 });
 
-export const docBlockSchema = z.object({
-  format: z.number().int().min(0).max(12),
-  content: z.union([z.string(), z.array(z.string())]),
-  metadata: z.object({
-    fontSize: z.number().min(6).nullable().optional(),
-    lineHeight: z.number().min(0.6).nullable().optional(),
-    paragraphSpacing: z.number().min(0).nullable().optional(),
-    indentWidth: z.number().min(0).nullable().optional(),
-    paddingX: z.number().min(0).nullable().optional(),
-    paddingY: z.number().min(0).nullable().optional(),
-    font: z.number().int().min(0).max(5).nullable().optional(),
-    color: z.string().regex(/^#([0-9a-fA-F]{3}|[0-9a-fA-F]{6})$/).nullable().optional(),
-    background: z.string().regex(/^#([0-9a-fA-F]{3}|[0-9a-fA-F]{6})$/).nullable().optional(),
-    align: z.enum(["left", "center", "right"]).nullable().optional(),
-  }).nullable().optional(),
-});
+export const documentFormatSchema = z.enum([
+  'pdf', 'docx', 'doc', 'xlsx', 'pptx', 'csv', 'md', 'html', 'txt', 'json',
+  'xml', 'tex', 'png', 'jpeg', 'jpg', 'gif', 'zip', 'tar', 'py', 'js', 'ts',
+  'css', 'sh',
+]);
 
 export const docToCreateSchema = z.object({
   title: z.string(),
-  blocks: z.array(docBlockSchema),
+  content: z.string().min(1),
+  format: documentFormatSchema.optional().default('pdf'),
 });
 
 export type WorksheetToCreate = z.infer<typeof worksheetToCreateSchema>;

package/src/pipelines/gradeWorksheet.ts

@@ -20,6 +20,63 @@ const removeAllPreviousAIComments = async (worksheetQuestionProgressId: string)
     });
 };
 
+type MarkSchemeItem = { id: string; points: number; description?: string; text?: string };
+
+const gradingResponseSchema = z.object({
+    isCorrect: z.boolean(),
+    points: z.number(),
+    markschemeState: z.array(z.object({
+        id: z.string(),
+        correct: z.boolean(),
+    })),
+    comments: z.array(z.string()),
+    annotations: z.array(z.object({
+        quote: z.string().describe('An exact substring from the student response that is relevant to this criterion.'),
+        markSchemeId: z.string().describe('The id of the mark scheme criterion this quote relates to.'),
+        criterion: z.string().describe('Short label describing what this mark scheme criterion evaluates.'),
+        met: z.boolean().describe('Whether the quoted text satisfies this criterion.'),
+    })).describe('Highlight specific parts of the student response and map each to a mark scheme criterion.'),
+});
+
+function buildGradingPrompt(
+    question: { question: string; answer: string; type: WorksheetQuestionType; points: number; markScheme: unknown },
+    responseText: string,
+): string {
+    const markSchemeItems = Array.isArray(question.markScheme) ? question.markScheme as MarkSchemeItem[] : [];
+    const markSchemeBlock = markSchemeItems.length > 0
+        ? markSchemeItems.map((item, i) =>
+            `  ${i + 1}. [id="${item.id}", ${item.points} pts] ${item.description || item.text || `Criterion ${i + 1}`}`
+        ).join('\n')
+        : '  No mark scheme provided — grade based on correctness and completeness.';
+
+    const maxPoints = question.points || markSchemeItems.reduce((sum, item) => sum + (item.points || 0), 0);
+
+    return `Grade the following student response.
+
+Question type: ${question.type}
+Maximum points: ${maxPoints}
+
+QUESTION:
+${question.question}
+
+CORRECT ANSWER:
+${question.answer || 'No answer key provided.'}
+
+MARK SCHEME:
+${markSchemeBlock}
+
+STUDENT RESPONSE:
+${responseText}
+
+INSTRUCTIONS:
+- Evaluate the student response against each mark scheme criterion.
+- Total points must not exceed ${maxPoints}.
+- For each criterion, find the exact substring in the student's response that is relevant (the "quote" field). Copy the text verbatim — do not paraphrase.
+- If no part of the response addresses a criterion, still include an annotation with an empty quote and met=false.
+- Provide constructive feedback comments.
+- Mark isCorrect=true only if the student earned full or near-full marks.`;
+}
+
 const gradeWorksheetQuestion = async (worksheetResponseId: string, worksheetQuestionProgressId: string) => {
 
     await removeAllPreviousAIComments(worksheetQuestionProgressId);
@@ -83,38 +140,12 @@ const gradeWorksheetQuestion = async (worksheetResponseId: string, worksheetQues
         return updatedProgress;
     }
 
-    const comments = studentQuestionProgress.comments;
-
     try {
+        const prompt = buildGradingPrompt(question, responseText);
+
         const apiResponse = await inference(
-            `Grade the following worksheet response:
-            
-            Question: ${question.question}
-            Response: ${responseText}
-
-            Comments: ${comments.map((comment) => comment.content).join('\n')}
-            Mark Scheme: ${JSON.stringify(question.markScheme)}
-            
-            Justify your reasoning by including comment(s) and mark the question please.         
-            Return ONLY JSON in the following format (fill in the values as per the question):
-           {
-            "isCorrect": <boolean>,
-            "points": <number>,
-            "markschemeState": [
-                { "id": <string>, "correct": <boolean> }
-            ],
-            "comments": [<string>, ...]
-            }
-            `,
-            z.object({
-                isCorrect: z.boolean(),
-                points: z.number(),
-                markschemeState: z.array(z.object({
-                    id: z.string(),
-                    correct: z.boolean(),
-                  })), // @note: this has to be converted to [id: string]: correct boolean
-                comments: z.array(z.string()),
-            }),
+            prompt,
+            gradingResponseSchema,
             {
                 reason: 'worksheet_autograde',
                 metadata: {
@@ -128,6 +159,27 @@ const gradeWorksheetQuestion = async (worksheetResponseId: string, worksheetQues
             throw error;
         });
 
+        const maxPoints = question.points || 0;
+        const clampedPoints = Math.max(0, Math.min(Math.round((apiResponse as { points: number }).points), maxPoints));
+
+        const validMarkSchemeIds = new Set(
+            (Array.isArray(question.markScheme) ? question.markScheme as MarkSchemeItem[] : []).map(item => item.id),
+        );
+
+        const validatedMarkschemeState = (apiResponse as {
+            markschemeState: { id: string; correct: boolean }[];
+        }).markschemeState
+            .filter(item => validMarkSchemeIds.has(item.id))
+            .reduce((acc, curr) => {
+                acc["item-" + curr.id] = curr.correct;
+                return acc;
+            }, {} as Record<string, boolean>);
+
+        const annotations = (apiResponse as {
+            annotations: { quote: string; markSchemeId: string; criterion: string; met: boolean }[];
+        }).annotations
+            .filter(a => a.quote.length === 0 || responseText.includes(a.quote));
+
         const updatedStudentQuestionProgress = await prisma.studentQuestionProgress.update({
             where: { id: studentQuestionProgress.id, status: {
                 not: {
@@ -137,13 +189,11 @@ const gradeWorksheetQuestion = async (worksheetResponseId: string, worksheetQues
             data: {
                 status: JobStatus.COMPLETED,
                 isCorrect: (apiResponse as { isCorrect: boolean }).isCorrect,
-                points: (apiResponse as { points: number }).points,
-                markschemeState: (apiResponse as {
-                    markschemeState: { id: string; correct: boolean }[];
-                }).markschemeState.reduce((acc, curr) => {
-                    acc["item-" + curr.id] = curr.correct;
-                    return acc;
-                }, {} as Record<string, boolean>),
+                points: clampedPoints,
+                markschemeState: {
+                    ...validatedMarkschemeState,
+                    _annotations: annotations,
+                },
                 comments: {
                     create: (apiResponse as {
                         comments: string[];
@@ -279,7 +329,14 @@ export const gradeWorksheetPipeline = async (worksheetResponseId: string) => {
             continue;
         }
 
-        await gradeWorksheetQuestion(worksheetResponseId, response.id);
+        try {
+            await gradeWorksheetQuestion(worksheetResponseId, response.id);
+        } catch (questionError) {
+            logger.error('Failed to grade individual question, continuing with remaining questions', {
+                questionId: response.questionId,
+                error: questionError,
+            });
+        }
 
     };
 };