@studious-lms/server 1.13.0 → 1.15.0

package/src/pipelines/aiLabChat.ts

@@ -1,23 +1,36 @@
 /**
  * AI lab chat pipeline – generates lab introductions and responses.
- * Can create worksheets, sections, assignments, and PDF docs from AI output.
+ * Can create worksheets, sections, assignments, and generated files from AI output.
  */
 import { isAIUser } from "../utils/aiUser.js";
 import { prisma } from "../lib/prisma.js";
-import { pusher, teacherChannel } from "../lib/pusher.js";
+import { pusher, chatChannel, teacherChannel } from "../lib/pusher.js";
 import type { Assignment, Class, File, Section, User } from "@prisma/client";
 import {
-  inference,
   completePendingAIMessage,
   failPendingAIMessage,
 } from "../utils/inference.js";
 import { logger } from "../utils/logger.js";
-import { createPdf } from "../lib/jsonConversion.js";
 import { v4 } from "uuid";
-import { bucket } from "../lib/googleCloudStorage.js";
-import { DocumentBlock } from "../lib/jsonStyles.js";
-import { type LabChatResponse, labChatResponseSchema } from "./aiLabChatContract.js";
-import { buildLabChatResponseMessages } from "./labChatPrompt.js";
+import { bucket, getSignedUrl } from "../lib/googleCloudStorage.js";
+import {
+  generateDocumentWithOpenAI,
+  normalizeDocumentFormat,
+} from "../services/openaiDocumentGen.js";
+import { inferDocumentFormat } from "../lib/inferDocumentFormat.js";
+import { broadcastTextChunks, broadcastStreamComplete } from "../utils/streamBroadcast.js";
+import { buildLabChatAgentMessages } from "./labChatPrompt.js";
+import type { ConversationMessage } from "./labChatPrompt.js";
+import {
+  applyResolvedFormatsToStreamParts,
+  enrichStreamPartsWithGeneratedFiles,
+  finalizeDeferredToolParts,
+  streamChatResponse,
+  type GeneratedDocFileLink,
+  type StreamPart,
+} from "../services/aiChatStream.js";
+import { aiTools } from "../services/aiTools.js";
+import { labProposalTools, createLabProposals } from "../services/aiProposalTools.js";
 
 /** Cap model wait time; leave headroom after inference for PDF/GCS/DB within Vercel maxDuration (500s). */
 const LAB_CHAT_RESPONSE_TIMEOUT_MS = 400_000;
@@ -72,6 +85,12 @@ type RecentLabChatMessage = {
       displayName: string | null;
     } | null;
   } | null;
+  attachments: {
+    id: string;
+    name: string;
+    type: string;
+    path: string;
+  }[];
 };
 
 /**
@@ -116,6 +135,14 @@ const loadRecentLabChatMessages = async (
         },
       },
     },
+    attachments: {
+      select: {
+        id: true,
+        name: true,
+        type: true,
+        path: true,
+      },
+    },
   };
 
   const newestMessages = await prisma.message.findMany({
@@ -186,6 +213,42 @@ const loadRecentLabChatMessages = async (
     .slice(0, limit);
 };
 
+/**
+ * Resolves signed URLs for image attachments in messages so the LLM can see them.
+ * Non-image files are passed as text descriptions only.
+ */
+async function resolveAttachmentUrls(
+  messages: RecentLabChatMessage[],
+): Promise<ConversationMessage[]> {
+  const resolved: ConversationMessage[] = [];
+
+  for (const msg of messages) {
+    const attachments = msg.attachments ?? [];
+    if (attachments.length === 0) {
+      resolved.push({ ...msg, attachments: [] });
+      continue;
+    }
+
+    const withUrls = await Promise.all(
+      attachments.map(async (a) => {
+        if (a.type.startsWith('image/')) {
+          try {
+            const url = await getSignedUrl(a.path);
+            return { id: a.id, name: a.name, type: a.type, url };
+          } catch {
+            return { id: a.id, name: a.name, type: a.type };
+          }
+        }
+        return { id: a.id, name: a.name, type: a.type };
+      }),
+    );
+
+    resolved.push({ ...msg, attachments: withUrls });
+  }
+
+  return resolved;
+}
+
 /**
  * Builds schema-aware context for the AI from class data.
  * Formats entities with IDs so the model can reference them when creating assignments.
@@ -303,33 +366,6 @@ export const getBaseSystemPrompt = (
 
 
 
-/**
- * Generate labchat responses
- * Allow for the generation of the following:
- * - Assignment(s) either individual or bulk as an lesson / course plan.
- * - Worksheet(s) either individual or bulk as an lesson / course plan.
- * - Files (PDFs)
- * @param labChatId 
- */
-// export const sendAiLabChatResponsePipeline = async (labChatId: string) => {
-//     const message = await prisma?.message.create({
-//         data: {
-//             content: "GENERATING_CONTENT",
-//             senderId: getAIUserId(),
-//             conversationId: labChatId,
-//             status: JobStatus.PENDING,
-//         },
-//     });
-
-//     try {
-
-//         inference(`
-//         `)
-//     }
-    
-// };
-
-
 /**
  * Fill the AI introduction placeholder for a brand-new lab chat.
  *
@@ -345,9 +381,10 @@ export const generateAndSendLabIntroduction = async (
     aiMessageId: string,
   ): Promise<void> => {
     const introMessage =
-      "Hello teaching team! I'm your AI assistant for course material development. I'll help you create educational content - when you have a clear request, I'll produce it directly. I only ask questions when I need to clarify the topic or scope. What would you like to work on?";
+      "Hello teaching team! I'm your AI assistant for course material development. I can create worksheets, assignments, and downloadable files — PDFs, Word docs, PowerPoints, spreadsheets, and more. When you have a clear request, I'll produce it directly. I only ask questions when I need to clarify the topic or scope. What would you like to work on?";
 
     try {
+      await broadcastTextChunks(conversationId, aiMessageId, introMessage);
       await completePendingAIMessage(aiMessageId, introMessage);
       logger.info('AI Introduction sent', { labChatId, conversationId, aiMessageId });
     } catch (error) {
@@ -376,7 +413,7 @@ export const generateAndSendLabIntroduction = async (
    */
   export const generateAndSendLabResponse = async (
     labChatId: string,
-    _teacherMessage: string,
+    teacherMessage: string,
     emitOptions: { classId: string; aiMessageId: string }
   ): Promise<void> => {
     try {
@@ -474,133 +511,195 @@ export const generateAndSendLabIntroduction = async (
         assignments: classData.assignments,
       });
 
-      const messages = buildLabChatResponseMessages({
+      // Resolve signed URLs for image attachments so the LLM can see them
+      const messagesWithAttachments = await resolveAttachmentUrls(recentMessages.reverse());
+
+      const { system, messages } = buildLabChatAgentMessages({
         context: fullLabChat.context,
         classContext,
-        recentMessages: recentMessages.reverse(),
+        recentMessages: messagesWithAttachments,
         isAIUser,
       });
-  
-  
-    //   const completion = await inferenceClient.chat.completions.create({
-    //     model: 'command-a-03-2025',
-    //     messages,
-    //     temperature: 0.7,
-    //     response_format: zodTextFormat(labChatResponseSchema, "lab_chat_response_format"),
-    //   });
-
-    const response = await withTimeout(
-      inference<LabChatResponse>(messages, labChatResponseSchema, {
-        reason: 'lab_chat_response',
-        metadata: {
-          labChatId,
+
+      // Genuine agentic generation: the model streams prose token-by-token and
+      // calls real tools inline — read-only RAG lookups plus "propose_*" tools
+      // that record create-actions into `proposals`. `streamChatResponse`
+      // broadcasts the real text chunks and tool-call events as they happen.
+      const proposals = createLabProposals();
+      const tools = {
+        ...aiTools(fullLabChat.classId),
+        ...labProposalTools(proposals),
+      };
+
+      const { text: streamedText, parts: rawStreamParts } = await withTimeout(
+        streamChatResponse({
           conversationId,
-          classId: fullLabChat.classId,
           aiMessageId: emitOptions.aiMessageId,
-        },
-      }),
-      LAB_CHAT_RESPONSE_TIMEOUT_MS,
-      "Lab chat response generation",
-    );
-        
-      if (!response) {
-        throw new Error('No response generated from inference API');
-      }
-      // Parse the JSON response and generate PDF if docs are provided
+          system,
+          messages,
+          tools,
+          temperature: 0.7,
+          maxOutputTokens: 4096,
+          maxSteps: 8,
+        }),
+        LAB_CHAT_RESPONSE_TIMEOUT_MS,
+        "Lab chat response generation",
+      );
+
+      // Real text + tool-call events were already broadcast inline (in order)
+      // during the stream by `streamChatResponse`. Just close the stream.
+      await broadcastStreamComplete(conversationId, emitOptions.aiMessageId);
+
+      // `generate_document` is an inline tool that only QUEUES into `proposals.docs`
+      // during the stream. OpenAI Code Interpreter renders files here, after the
+      // stream ends, then we link them to this message.
       try {
-        const jsonData = response;
-  
-  
         const attachmentIds: string[] = [];
-        // Generate PDFs if docs are provided
-        if (jsonData.docs && Array.isArray(jsonData.docs)) {
-          
-  
-          for (let i = 0; i < jsonData.docs.length; i++) {
-            const doc = jsonData.docs[i];
-            if (!doc.title || !doc.blocks || !Array.isArray(doc.blocks)) {
-              logger.error(`Document ${i + 1} is missing title or blocks`);
-              continue;
-            } 
-  
-  
-            try {
-              let pdfBytes = await createPdf(doc.blocks as DocumentBlock[]);            
-              if (pdfBytes) {
-                // Sanitize filename - remove special characters and limit length
-                const sanitizedTitle = doc.title
-                  .replace(/[^a-zA-Z0-9\s\-_]/g, '')
-                  .replace(/\s+/g, '_')
-                  .substring(0, 50);
-                
-                const filename = `${sanitizedTitle}_${v4().substring(0, 8)}.pdf`;
-                const filePath = `class/generated/${fullLabChat.classId}/${filename}`;
-  
-                logger.info(`PDF ${i + 1} generated successfully`, { labChatId, title: doc.title });
-                
-                // Upload directly to Google Cloud Storage
-                const gcsFile = bucket.file(filePath);
-                await gcsFile.save(Buffer.from(pdfBytes), {
-                  metadata: {
-                    contentType: 'application/pdf',
-                  }
-                });
-      
-                logger.info(`PDF ${i + 1} uploaded successfully`, { labChatId, filename });
-  
-                const file = await prisma.file.create({
-                  data: {
-                    name: filename,
-                    path: filePath,
-                    type: 'application/pdf',
-                    size: pdfBytes.length,
-                    userId: fullLabChat.createdById,
-                    uploadStatus: 'COMPLETED',
-                    uploadedAt: new Date(),
-                  },
-                });
-                attachmentIds.push(file.id);
-              } else {
-                logger.error(`PDF ${i + 1} creation returned undefined/null`, { labChatId, title: doc.title });
-              }
-            } catch (pdfError) {
-              logger.error(`PDF creation threw an error for document ${i + 1}:`, { 
-                error: pdfError instanceof Error ? {
-                  message: pdfError.message,
-                  stack: pdfError.stack,
-                  name: pdfError.name
-                } : pdfError, 
+        const generatedDocFiles: GeneratedDocFileLink[] = [];
+        const resolvedDocs: Array<{ title: string; format: string }> = [];
+
+        for (let i = 0; i < proposals.docs.length; i++) {
+          const doc = proposals.docs[i];
+          if (!doc.title || !doc.content?.trim()) {
+            logger.error(`Document ${i + 1} is missing title or content`, { labChatId, doc });
+            continue;
+          }
+
+          const format = inferDocumentFormat(
+            doc.title,
+            doc.content,
+            doc.format,
+            teacherMessage,
+          );
+          resolvedDocs.push({ title: doc.title, format });
+
+          if (format !== normalizeDocumentFormat(doc.format)) {
+            logger.info('Document format overridden from model default', {
+              labChatId,
+              title: doc.title,
+              modelFormat: doc.format,
+              resolvedFormat: format,
+            });
+          }
+
+          try {
+            const generated = await generateDocumentWithOpenAI({
+              title: doc.title,
+              content: doc.content,
+              format,
+            });
+            if (!generated) {
+              logger.error(`Document ${i + 1} generation returned null`, {
                 labChatId,
-                title: doc.title
+                title: doc.title,
+                format,
               });
+              continue;
             }
+
+            const sanitizedTitle = doc.title
+              .replace(/[^a-zA-Z0-9\s\-_]/g, '')
+              .replace(/\s+/g, '_')
+              .substring(0, 50);
+            const filename = `${sanitizedTitle}_${v4().substring(0, 8)}.${generated.format}`;
+            const filePath = `class/generated/${fullLabChat.classId}/${filename}`;
+
+            logger.info(`Document ${i + 1} generated successfully`, {
+              labChatId,
+              title: doc.title,
+              format: generated.format,
+            });
+
+            await bucket.file(filePath).save(generated.bytes, {
+              metadata: { contentType: generated.mimeType },
+            });
+
+            logger.info(`Document ${i + 1} uploaded successfully`, { labChatId, filename });
+
+            const file = await prisma.file.create({
+              data: {
+                name: filename,
+                path: filePath,
+                type: generated.mimeType,
+                size: generated.bytes.length,
+                userId: fullLabChat.createdById,
+                conversationId,
+                messageId: emitOptions.aiMessageId,
+                uploadStatus: 'COMPLETED',
+                uploadedAt: new Date(),
+              },
+            });
+
+            attachmentIds.push(file.id);
+            generatedDocFiles.push({
+              title: doc.title,
+              fileId: file.id,
+              fileName: filename,
+              format: generated.format,
+            });
+          } catch (docError) {
+            logger.error(`Document generation failed for item ${i + 1}:`, {
+              error: docError instanceof Error ? {
+                message: docError.message,
+                stack: docError.stack,
+                name: docError.name,
+              } : docError,
+              labChatId,
+              title: doc.title,
+              format: doc.format,
+            });
           }
         }
-  
-        await completePendingAIMessage(emitOptions.aiMessageId, jsonData.text, {
-          attachments: {
-            connect: attachmentIds.map(id => ({ id })),
-          },
+
+        const streamParts: StreamPart[] = finalizeDeferredToolParts(
+          enrichStreamPartsWithGeneratedFiles(
+            applyResolvedFormatsToStreamParts(rawStreamParts, resolvedDocs),
+            generatedDocFiles,
+          ),
+        );
+
+        // While the message is still PENDING, resolve deferred tool indicators
+        // (spinner → attachment or error) before the final persist event.
+        for (const part of streamParts) {
+          if (part.type !== 'tool' || part.toolName !== 'generate_document') {
+            continue;
+          }
+          if (part.status === 'executing') continue;
+          await pusher
+            .trigger(chatChannel(conversationId), 'message-tool-call', {
+              messageId: emitOptions.aiMessageId,
+              toolCallId: part.toolCallId,
+              toolName: part.toolName,
+              ...(part.title ? { title: part.title } : {}),
+              ...(part.format ? { format: part.format } : {}),
+              ...(part.fileId ? { fileId: part.fileId, fileName: part.fileName } : {}),
+              status: part.status,
+            })
+            .catch(() => {});
+        }
+
+        await completePendingAIMessage(emitOptions.aiMessageId, streamedText, {
+          ...(attachmentIds.length > 0 && {
+            attachments: { connect: attachmentIds.map((id) => ({ id })) },
+          }),
           meta: {
-            assignmentsToCreate: jsonData.assignmentsToCreate?.map(assignment => ({
-              ...assignment,
-              id: v4(),
-            })) || null,
-            worksheetsToCreate: jsonData.worksheetsToCreate?.map(worksheet => ({
-              ...worksheet,
-              id: v4(),
-            })) || null,
-            sectionsToCreate: jsonData.sectionsToCreate?.map(section => ({
-              ...section,
-              id: v4(),
-            })) || null,
+            assignmentsToCreate: proposals.assignments.length > 0
+              ? proposals.assignments.map(assignment => ({ ...assignment, id: v4() }))
+              : null,
+            worksheetsToCreate: proposals.worksheets.length > 0
+              ? proposals.worksheets.map(worksheet => ({ ...worksheet, id: v4() }))
+              : null,
+            sectionsToCreate: proposals.sections.length > 0
+              ? proposals.sections.map(section => ({ ...section, id: v4() }))
+              : null,
+            streamParts: streamParts.length > 0 ? streamParts : null,
           },
         });
-      } catch (parseError) {
-        logger.error('Failed to parse AI response or generate PDF:', { error: parseError, labChatId });
+      } catch (persistError) {
+        logger.error('Failed to persist AI response or generate PDF:', { error: persistError, labChatId });
         // Fallback: complete the placeholder with the raw text so the
         // teacher still sees something useful instead of FAILED.
-        await completePendingAIMessage(emitOptions.aiMessageId, response.text);
+        await completePendingAIMessage(emitOptions.aiMessageId, streamedText);
       }
 
       try {

package/src/pipelines/aiLabChatContract.ts

@@ -1,75 +1,65 @@
 import z from "zod";
 
-export const labChatResponseSchema = z.object({
-  text: z.string(),
-  worksheetsToCreate: z.array(z.object({
-    title: z.string(),
-    questions: z.array(z.object({
-      type: z.enum(["MULTIPLE_CHOICE", "TRUE_FALSE", "SHORT_ANSWER", "LONG_ANSWER", "MATH_EXPRESSION", "ESSAY"]),
-      question: z.string(),
-      answer: z.string(),
-      options: z.array(z.object({
-        id: z.string(),
-        text: z.string(),
-        isCorrect: z.boolean(),
-      })).optional().default([]),
-      markScheme: z.array(z.object({
-        id: z.string(),
-        points: z.number(),
-        description: z.string(),
-      })).optional().default([]),
-      points: z.number().optional().default(0),
-      order: z.number(),
-    })),
-  })).default([]),
-  sectionsToCreate: z.array(z.object({
-    name: z.string(),
-    color: z.string().regex(/^#([0-9a-fA-F]{3}|[0-9a-fA-F]{6})$/).nullable().optional(),
-  })).default([]),
-  assignmentsToCreate: z.array(z.object({
-    title: z.string(),
-    instructions: z.string(),
-    dueDate: z.string().datetime(),
-    acceptFiles: z.boolean(),
-    acceptExtendedResponse: z.boolean(),
-    acceptWorksheet: z.boolean(),
-    maxGrade: z.number(),
-    gradingBoundaryId: z.string().nullable().optional(),
-    markSchemeId: z.string().nullable().optional(),
-    worksheetIds: z.array(z.string()),
-    studentIds: z.array(z.string()),
-    sectionId: z.string().nullable().optional(),
-    type: z.enum(["HOMEWORK", "QUIZ", "TEST", "PROJECT", "ESSAY", "DISCUSSION", "PRESENTATION", "LAB", "OTHER"]),
-    attachments: z.array(z.object({
-      id: z.string(),
-    })),
-  })).nullable().optional(),
-  docs: z.array(z.object({
-    title: z.string(),
-    blocks: z.array(z.object({
-      format: z.number().int().min(0).max(12),
-      content: z.union([z.string(), z.array(z.string())]),
-      metadata: z.object({
-        fontSize: z.number().min(6).nullable().optional(),
-        lineHeight: z.number().min(0.6).nullable().optional(),
-        paragraphSpacing: z.number().min(0).nullable().optional(),
-        indentWidth: z.number().min(0).nullable().optional(),
-        paddingX: z.number().min(0).nullable().optional(),
-        paddingY: z.number().min(0).nullable().optional(),
-        font: z.number().int().min(0).max(5).nullable().optional(),
-        color: z.string().regex(/^#([0-9a-fA-F]{3}|[0-9a-fA-F]{6})$/).nullable().optional(),
-        background: z.string().regex(/^#([0-9a-fA-F]{3}|[0-9a-fA-F]{6})$/).nullable().optional(),
-        align: z.enum(["left", "center", "right"]).nullable().optional(),
-      }).nullable().optional(),
-    })),
-  })).nullable().optional(),
+export const worksheetQuestionSchema = z.object({
+  type: z.enum(["MULTIPLE_CHOICE", "TRUE_FALSE", "SHORT_ANSWER", "LONG_ANSWER", "MATH_EXPRESSION", "ESSAY"]),
+  question: z.string(),
+  answer: z.string(),
+  options: z.array(z.object({
+    id: z.string(),
+    text: z.string(),
+    isCorrect: z.boolean(),
+  })).optional().default([]),
+  markScheme: z.array(z.object({
+    id: z.string(),
+    points: z.number(),
+    description: z.string(),
+  })).optional().default([]),
+  points: z.number().optional().default(0),
+  order: z.number(),
 });
 
-export type LabChatResponse = z.infer<typeof labChatResponseSchema>;
+export const worksheetToCreateSchema = z.object({
+  title: z.string(),
+  questions: z.array(worksheetQuestionSchema),
+});
+
+export const sectionToCreateSchema = z.object({
+  name: z.string(),
+  color: z.string().regex(/^#([0-9a-fA-F]{3}|[0-9a-fA-F]{6})$/).nullable().optional(),
+});
 
-export const labChatResponseFormat = `{ "text": string, "docs": null | array, "worksheetsToCreate": array, "sectionsToCreate": array, "assignmentsToCreate": null | array }`;
+export const assignmentToCreateSchema = z.object({
+  title: z.string(),
+  instructions: z.string(),
+  dueDate: z.string().datetime(),
+  acceptFiles: z.boolean(),
+  acceptExtendedResponse: z.boolean(),
+  acceptWorksheet: z.boolean(),
+  maxGrade: z.number(),
+  gradingBoundaryId: z.string().nullable().optional(),
+  markSchemeId: z.string().nullable().optional(),
+  worksheetIds: z.array(z.string()),
+  studentIds: z.array(z.string()),
+  sectionId: z.string().nullable().optional(),
+  type: z.enum(["HOMEWORK", "QUIZ", "TEST", "PROJECT", "ESSAY", "DISCUSSION", "PRESENTATION", "LAB", "OTHER"]),
+  attachments: z.array(z.object({
+    id: z.string(),
+  })),
+});
+
+export const documentFormatSchema = z.enum([
+  'pdf', 'docx', 'doc', 'xlsx', 'pptx', 'csv', 'md', 'html', 'txt', 'json',
+  'xml', 'tex', 'png', 'jpeg', 'jpg', 'gif', 'zip', 'tar', 'py', 'js', 'ts',
+  'css', 'sh',
+]);
+
+export const docToCreateSchema = z.object({
+  title: z.string(),
+  content: z.string().min(1),
+  format: documentFormatSchema.optional().default('pdf'),
+});
 
-export const labChatArrayFieldInstructions = [
-  `- "worksheetsToCreate": always output an array. Use [] when there are no worksheets to create.`,
-  `- "sectionsToCreate": always output an array. Use [] when there are no sections to create.`,
-].join("\n");
+export type WorksheetToCreate = z.infer<typeof worksheetToCreateSchema>;
+export type SectionToCreate = z.infer<typeof sectionToCreateSchema>;
+export type AssignmentToCreate = z.infer<typeof assignmentToCreateSchema>;
+export type DocToCreate = z.infer<typeof docToCreateSchema>;