@studious-lms/server 1.16.0 → 1.17.0

package/src/lib/chatAttachmentPrompt.ts

@@ -0,0 +1,270 @@
+/**
+ * Shared chat attachment handling for Newton + Lab pipelines.
+ *
+ * - Images + PDFs: inlined as multimodal parts (bytes from GCS).
+ * - Office docs, spreadsheets, slides, text, csv, json, html: text extracted
+ *   via officeparser / UTF-8 decode and appended to the user message.
+ * - Audio/video/zip: filename mention only (no extraction).
+ */
+import type { FilePart, ImagePart, ModelMessage, TextPart, UserContent } from 'ai';
+import { OfficeParser } from 'officeparser';
+import { bucket } from './googleCloudStorage.js';
+import { logger } from '../utils/logger.js';
+
+/** Model used for AI chats that may include document attachments. */
+export const AI_CHAT_MODEL = 'gpt-5.5';
+
+const MAX_EXTRACTED_CHARS = 80_000;
+const IMAGE_TYPE_PREFIX = 'image/';
+
+export type ChatAttachmentRef = {
+  id: string;
+  name: string;
+  type: string;
+  path: string;
+};
+
+export type LoadedChatAttachment = ChatAttachmentRef & {
+  data?: Uint8Array;
+  extractedText?: string;
+};
+
+export type ChatHistoryMessage = {
+  id?: string;
+  content: string;
+  senderId: string;
+  createdAt?: Date | string | number;
+  sender?: {
+    username?: string | null;
+    profile?: { displayName?: string | null } | null;
+  } | null;
+  attachments?: ChatAttachmentRef[];
+};
+
+type UserContentPart = TextPart | ImagePart | FilePart;
+
+const TEXT_DECODE_TYPES = new Set([
+  'text/plain',
+  'text/csv',
+  'text/markdown',
+  'text/html',
+  'application/json',
+]);
+
+const MENTION_ONLY_PREFIXES = ['audio/', 'video/'];
+const MENTION_ONLY_TYPES = new Set(['application/zip']);
+
+function extension(name: string): string {
+  return name.split('.').pop()?.toLowerCase() ?? '';
+}
+
+function isPdf(attachment: { type: string; name: string }): boolean {
+  return attachment.type === 'application/pdf' || extension(attachment.name) === 'pdf';
+}
+
+function isImage(attachment: { type: string }): boolean {
+  return attachment.type.startsWith(IMAGE_TYPE_PREFIX);
+}
+
+function isMentionOnly(attachment: { type: string; name: string }): boolean {
+  if (MENTION_ONLY_TYPES.has(attachment.type)) return true;
+  return MENTION_ONLY_PREFIXES.some((prefix) => attachment.type.startsWith(prefix));
+}
+
+function isUtf8Text(attachment: { type: string; name: string }): boolean {
+  if (TEXT_DECODE_TYPES.has(attachment.type)) return true;
+  const ext = extension(attachment.name);
+  return ['txt', 'csv', 'md', 'html', 'json'].includes(ext);
+}
+
+function shouldDownload(attachment: ChatAttachmentRef): boolean {
+  return Boolean(attachment.path) && !isMentionOnly(attachment);
+}
+
+async function extractTextFromBytes(
+  bytes: Uint8Array,
+  name: string,
+  type: string,
+): Promise<string | null> {
+  if (isUtf8Text({ type, name })) {
+    return new TextDecoder('utf-8', { fatal: false })
+      .decode(bytes)
+      .slice(0, MAX_EXTRACTED_CHARS);
+  }
+
+  if (isImage({ type }) || isPdf({ type, name })) {
+    return null;
+  }
+
+  if (isMentionOnly({ type, name })) {
+    return null;
+  }
+
+  try {
+    const ast = await OfficeParser.parseOffice(Buffer.from(bytes), {
+      includeRawContent: false,
+    });
+    const text = ast.toText().trim();
+    return text ? text.slice(0, MAX_EXTRACTED_CHARS) : null;
+  } catch (error) {
+    logger.warn('Failed to extract chat attachment text', { name, type, error });
+    return null;
+  }
+}
+
+/** Download bytes from GCS and extract text for every supported attachment type. */
+export async function loadChatAttachments(
+  attachments: ChatAttachmentRef[],
+): Promise<LoadedChatAttachment[]> {
+  return Promise.all(
+    attachments.map(async (attachment) => {
+      if (!shouldDownload(attachment)) {
+        return { ...attachment };
+      }
+
+      try {
+        const [bytes] = await bucket.file(attachment.path).download();
+        const data = new Uint8Array(bytes);
+        const extractedText =
+          (await extractTextFromBytes(data, attachment.name, attachment.type)) ??
+          undefined;
+        return { ...attachment, data, extractedText };
+      } catch (error) {
+        logger.warn('Failed to load chat attachment from storage', {
+          attachmentId: attachment.id,
+          path: attachment.path,
+          type: attachment.type,
+          error,
+        });
+        return { ...attachment };
+      }
+    }),
+  );
+}
+
+/** @deprecated Use {@link loadChatAttachments} */
+export const loadPromptAttachments = loadChatAttachments;
+
+export function buildUserMessageContentWithAttachments(
+  textContent: string,
+  attachments: LoadedChatAttachment[],
+): UserContent {
+  if (attachments.length === 0) {
+    return textContent;
+  }
+
+  const parts: UserContentPart[] = [];
+  let fullText = textContent;
+
+  for (const attachment of attachments) {
+    if (attachment.extractedText) {
+      fullText += `\n\n[Content of ${attachment.name}]:\n${attachment.extractedText}`;
+    }
+  }
+
+  const unhandled = attachments.filter((a) => {
+    if (a.extractedText) return false;
+    if (isImage(a) && a.data) return false;
+    if (isPdf(a) && a.data) return false;
+    return true;
+  });
+
+  if (unhandled.length > 0) {
+    const fileList = unhandled
+      .map((a) => `${a.name} (${a.type})`)
+      .join(', ');
+    fullText = fullText
+      ? `${fullText}\n[Attached files (content not extracted): ${fileList}]`
+      : `[Attached files (content not extracted): ${fileList}]`;
+  }
+
+  if (fullText.trim()) {
+    parts.push({ type: 'text', text: fullText });
+  }
+
+  for (const image of attachments) {
+    if (isImage(image) && image.data) {
+      parts.push({ type: 'image', image: image.data });
+    }
+  }
+
+  for (const file of attachments) {
+    if (isPdf(file) && file.data) {
+      parts.push({
+        type: 'file',
+        data: file.data,
+        mediaType: 'application/pdf',
+        filename: file.name,
+      });
+    }
+  }
+
+  if (parts.length === 0) {
+    return textContent || '[Attached files]';
+  }
+
+  if (parts.length === 1 && parts[0].type === 'text') {
+    return parts[0].text;
+  }
+
+  return parts;
+}
+
+export function sortChatMessagesChronologically<T extends ChatHistoryMessage>(
+  messages: T[],
+): T[] {
+  return messages
+    .map((message, index) => ({ message, index }))
+    .sort((left, right) => {
+      const leftTime = left.message.createdAt
+        ? new Date(left.message.createdAt).getTime()
+        : null;
+      const rightTime = right.message.createdAt
+        ? new Date(right.message.createdAt).getTime()
+        : null;
+
+      if (leftTime != null && rightTime != null) {
+        return leftTime - rightTime;
+      }
+      if (leftTime != null) return -1;
+      if (rightTime != null) return 1;
+      return left.index - right.index;
+    })
+    .map(({ message }) => message);
+}
+
+/** Build ModelMessage[] with shared attachment loading for Newton + Lab. */
+export async function buildChatModelMessages({
+  messages,
+  isAIUser,
+  defaultSenderLabel,
+}: {
+  messages: ChatHistoryMessage[];
+  isAIUser: (senderId: string) => boolean;
+  defaultSenderLabel: string;
+}): Promise<ModelMessage[]> {
+  const modelMessages: ModelMessage[] = [];
+
+  for (const message of sortChatMessagesChronologically(messages)) {
+    if (isAIUser(message.senderId)) {
+      modelMessages.push({ role: 'assistant', content: message.content });
+      continue;
+    }
+
+    const senderName =
+      message.sender?.profile?.displayName ||
+      message.sender?.username ||
+      defaultSenderLabel;
+    const label = message.content.trim()
+      ? `${senderName}: ${message.content}`
+      : senderName;
+    const attachments = await loadChatAttachments(message.attachments ?? []);
+
+    modelMessages.push({
+      role: 'user',
+      content: buildUserMessageContentWithAttachments(label, attachments),
+    });
+  }
+
+  return modelMessages;
+}

package/src/lib/inferDocumentFormat.ts

@@ -9,7 +9,7 @@ const FORMAT_RULES: FormatRule[] = [
   { format: 'xlsx', pattern: /\b(spreadsheet|excel|workbook|xlsx|gradebook|grade book|\.xlsx)\b/i },
   { format: 'csv', pattern: /\b(csv|comma[- ]separated|\.csv)\b/i },
   { format: 'docx', pattern: /\b(word doc(ument)?|docx|editable handout|\.docx)\b/i },
-  { format: 'png', pattern: /\b(diagram|chart|graph|plot|figure|infographic|\.png)\b/i },
+  { format: 'png', pattern: /\b(diagram|flowchart|chart|graph|plot|figure|infographic|visual|illustration|mind ?map|timeline|\.png)\b/i },
   { format: 'md', pattern: /\b(markdown|readme|\.md)\b/i },
   { format: 'html', pattern: /\b(html|web ?page|\.html)\b/i },
   { format: 'txt', pattern: /\b(plain text|text file|\.txt)\b/i },

package/src/lib/legal.ts

@@ -0,0 +1,2 @@
+/** Must match the "Last updated" month in the client privacy/terms copy. */
+export const CURRENT_TERMS_VERSION = "2025-11";

package/src/models/auth.ts

@@ -102,6 +102,8 @@ export function createUser(data: {
   email: string;
   password: string;
   verified?: boolean;
+  termsAcceptedAt?: Date;
+  termsVersion?: string;
 }) {
   return prisma.user.create({
     data: {
@@ -109,6 +111,8 @@ export function createUser(data: {
       email: data.email,
       password: data.password,
       verified: data.verified ?? false,
+      termsAcceptedAt: data.termsAcceptedAt,
+      termsVersion: data.termsVersion,
       profile: { create: {} },
     },
     select: {

package/src/models/marketing.ts

@@ -27,10 +27,18 @@ export function createSchoolDevelopmentProgram(data: {
   });
 }
 
-/** @returns School development program by ID. */
-export function findSchoolDevelopmentProgramById(id: string) {
+/** Public status lookup — omits contact PII (email, phone, address). */
+export function findSchoolDevelopmentProgramStatusById(id: string) {
   return prisma.schoolDevelopementProgram.findUnique({
     where: { id },
+    select: {
+      id: true,
+      name: true,
+      city: true,
+      country: true,
+      status: true,
+      submittedAt: true,
+    },
   });
 }
 

package/src/pipelines/aiLabChat.ts

@@ -12,16 +12,19 @@ import {
 } from "../utils/inference.js";
 import { logger } from "../utils/logger.js";
 import { v4 } from "uuid";
-import { bucket, getSignedUrl } from "../lib/googleCloudStorage.js";
-import { generateDocumentWithOpenAI } from "../services/openaiDocumentGen.js";
+import { bucket } from "../lib/googleCloudStorage.js";
+import {
+  generateDocumentWithOpenAI,
+  normalizeDocumentFormat,
+} from "../services/openaiDocumentGen.js";
 import { inferDocumentFormat } from "../lib/inferDocumentFormat.js";
-import { normalizeDocumentFormat } from "../services/openaiDocumentGen.js";
 import { broadcastTextChunks, broadcastStreamComplete } from "../utils/streamBroadcast.js";
+import { AI_CHAT_MODEL } from "../lib/chatAttachmentPrompt.js";
 import { buildLabChatAgentMessages } from "./labChatPrompt.js";
-import type { ConversationMessage } from "./labChatPrompt.js";
 import {
   applyResolvedFormatsToStreamParts,
   enrichStreamPartsWithGeneratedFiles,
+  finalizeDeferredToolParts,
   streamChatResponse,
   type GeneratedDocFileLink,
   type StreamPart,
@@ -210,42 +213,6 @@ const loadRecentLabChatMessages = async (
     .slice(0, limit);
 };
 
-/**
- * Resolves signed URLs for image attachments in messages so the LLM can see them.
- * Non-image files are passed as text descriptions only.
- */
-async function resolveAttachmentUrls(
-  messages: RecentLabChatMessage[],
-): Promise<ConversationMessage[]> {
-  const resolved: ConversationMessage[] = [];
-
-  for (const msg of messages) {
-    const attachments = msg.attachments ?? [];
-    if (attachments.length === 0) {
-      resolved.push({ ...msg, attachments: [] });
-      continue;
-    }
-
-    const withUrls = await Promise.all(
-      attachments.map(async (a) => {
-        if (a.type.startsWith('image/')) {
-          try {
-            const url = await getSignedUrl(a.path);
-            return { id: a.id, name: a.name, type: a.type, url };
-          } catch {
-            return { id: a.id, name: a.name, type: a.type };
-          }
-        }
-        return { id: a.id, name: a.name, type: a.type };
-      }),
-    );
-
-    resolved.push({ ...msg, attachments: withUrls });
-  }
-
-  return resolved;
-}
-
 /**
  * Builds schema-aware context for the AI from class data.
  * Formats entities with IDs so the model can reference them when creating assignments.
@@ -508,13 +475,10 @@ export const generateAndSendLabIntroduction = async (
         assignments: classData.assignments,
       });
 
-      // Resolve signed URLs for image attachments so the LLM can see them
-      const messagesWithAttachments = await resolveAttachmentUrls(recentMessages.reverse());
-
-      const { system, messages } = buildLabChatAgentMessages({
+      const { system, messages } = await buildLabChatAgentMessages({
         context: fullLabChat.context,
         classContext,
-        recentMessages: messagesWithAttachments,
+        recentMessages: recentMessages.reverse(),
         isAIUser,
       });
 
@@ -532,6 +496,7 @@ export const generateAndSendLabIntroduction = async (
         streamChatResponse({
           conversationId,
           aiMessageId: emitOptions.aiMessageId,
+          model: AI_CHAT_MODEL,
           system,
           messages,
           tools,
@@ -555,9 +520,10 @@ export const generateAndSendLabIntroduction = async (
         const generatedDocFiles: GeneratedDocFileLink[] = [];
         const resolvedDocs: Array<{ title: string; format: string }> = [];
 
-        for (const doc of proposals.docs) {
+        for (let i = 0; i < proposals.docs.length; i++) {
+          const doc = proposals.docs[i];
           if (!doc.title || !doc.content?.trim()) {
-            logger.error('Document missing title or content', { labChatId, doc });
+            logger.error(`Document ${i + 1} is missing title or content`, { labChatId, doc });
             continue;
           }
 
@@ -585,10 +551,10 @@ export const generateAndSendLabIntroduction = async (
               format,
             });
             if (!generated) {
-              logger.error('OpenAI document generation returned null', {
+              logger.error(`Document ${i + 1} generation returned null`, {
                 labChatId,
                 title: doc.title,
-                format: doc.format,
+                format,
               });
               continue;
             }
@@ -600,10 +566,18 @@ export const generateAndSendLabIntroduction = async (
             const filename = `${sanitizedTitle}_${v4().substring(0, 8)}.${generated.format}`;
             const filePath = `class/generated/${fullLabChat.classId}/${filename}`;
 
+            logger.info(`Document ${i + 1} generated successfully`, {
+              labChatId,
+              title: doc.title,
+              format: generated.format,
+            });
+
             await bucket.file(filePath).save(generated.bytes, {
               metadata: { contentType: generated.mimeType },
             });
 
+            logger.info(`Document ${i + 1} uploaded successfully`, { labChatId, filename });
+
             const file = await prisma.file.create({
               data: {
                 name: filename,
@@ -626,7 +600,7 @@ export const generateAndSendLabIntroduction = async (
               format: generated.format,
             });
           } catch (docError) {
-            logger.error('Document generation failed', {
+            logger.error(`Document generation failed for item ${i + 1}:`, {
               error: docError instanceof Error ? {
                 message: docError.message,
                 stack: docError.stack,
@@ -638,31 +612,32 @@ export const generateAndSendLabIntroduction = async (
             });
           }
         }
-        
-        const streamParts: StreamPart[] = enrichStreamPartsWithGeneratedFiles(
-          applyResolvedFormatsToStreamParts(rawStreamParts, resolvedDocs),
-          generatedDocFiles,
+
+        const streamParts: StreamPart[] = finalizeDeferredToolParts(
+          enrichStreamPartsWithGeneratedFiles(
+            applyResolvedFormatsToStreamParts(rawStreamParts, resolvedDocs),
+            generatedDocFiles,
+          ),
         );
 
+        // While the message is still PENDING, resolve deferred tool indicators
+        // (spinner → attachment or error) before the final persist event.
         for (const part of streamParts) {
-          if (
-            part.type === 'tool' &&
-            part.toolName === 'generate_document' &&
-            part.fileId
-          ) {
-            await pusher
-              .trigger(chatChannel(conversationId), 'message-tool-call', {
-                messageId: emitOptions.aiMessageId,
-                toolCallId: part.toolCallId,
-                toolName: part.toolName,
-                ...(part.title ? { title: part.title } : {}),
-                ...(part.format ? { format: part.format } : {}),
-                fileId: part.fileId,
-                fileName: part.fileName,
-                status: 'completed',
-              })
-              .catch(() => {});
+          if (part.type !== 'tool' || part.toolName !== 'generate_document') {
+            continue;
           }
+          if (part.status === 'executing') continue;
+          await pusher
+            .trigger(chatChannel(conversationId), 'message-tool-call', {
+              messageId: emitOptions.aiMessageId,
+              toolCallId: part.toolCallId,
+              toolName: part.toolName,
+              ...(part.title ? { title: part.title } : {}),
+              ...(part.format ? { format: part.format } : {}),
+              ...(part.fileId ? { fileId: part.fileId, fileName: part.fileName } : {}),
+              status: part.status,
+            })
+            .catch(() => {});
         }
 
         await completePendingAIMessage(emitOptions.aiMessageId, streamedText, {

package/src/pipelines/aiNewtonChat.ts

@@ -13,9 +13,10 @@ import { isAIUser } from "../utils/aiUser.js";
 import { broadcastStreamComplete } from "../utils/streamBroadcast.js";
 import { streamChatResponse } from "../services/aiChatStream.js";
 import { aiTools } from "../services/aiTools.js";
-import type { ModelMessage } from "ai";
-
-
+import {
+  AI_CHAT_MODEL,
+  buildChatModelMessages,
+} from "../lib/chatAttachmentPrompt.js";
 
 // AI Policy Levels Configuration
 // Used across assignment creation, editing, and display
@@ -148,6 +149,7 @@ export const generateAndSendNewtonIntroduction = async (
       const { text: response } = await streamChatResponse({
         conversationId,
         aiMessageId,
+        model: AI_CHAT_MODEL,
         system: systemPrompt,
         messages: [
           {
@@ -222,7 +224,6 @@ export const generateAndSendNewtonIntroduction = async (
    */
   export const generateAndSendNewtonResponse = async (
     newtonChatId: string,
-    studentMessage: string,
     conversationId: string,
     submission: {
       id: string;
@@ -258,6 +259,14 @@ export const generateAndSendNewtonIntroduction = async (
               },
             },
           },
+          attachments: {
+            select: {
+              id: true,
+              name: true,
+              type: true,
+              path: true,
+            },
+          },
         },
         orderBy: {
           createdAt: 'desc',
@@ -302,26 +311,13 @@ export const generateAndSendNewtonIntroduction = async (
   
   IMPORTANT:
   - Keep responses conversational and educational
-  - Focus on helping the student learn, not just completing the assignment`;
-  
-      const modelMessages: ModelMessage[] = [];
-  
-      // Add recent conversation history
-      recentMessages.reverse().forEach(msg => {
-        const role = isAIUser(msg.senderId) ? 'assistant' : 'user';
-        const senderName = msg.sender?.profile?.displayName || msg.sender?.username || 'Student';
-        const content = isAIUser(msg.senderId) ? msg.content : `${senderName}: ${msg.content}`;
-        
-        modelMessages.push({
-          role: role as 'user' | 'assistant',
-          content,
-        });
-      });
+  - Focus on helping the student learn, not just completing the assignment
+  - When the student attaches files (PDFs, images), read and reference them in your reply`;
   
-      // Add the new student message
-      modelMessages.push({
-        role: 'user',
-        content: `Student: ${studentMessage}`,
+      const modelMessages = await buildChatModelMessages({
+        messages: [...recentMessages].reverse(),
+        isAIUser,
+        defaultSenderLabel: 'Student',
       });
 
       const combinedSystem = [
@@ -337,6 +333,7 @@ export const generateAndSendNewtonIntroduction = async (
       const { text: response, parts: streamParts } = await streamChatResponse({
         conversationId,
         aiMessageId,
+        model: AI_CHAT_MODEL,
         system: combinedSystem,
         messages: modelMessages,
         ...(newtonClassId ? { tools: aiTools(newtonClassId), maxSteps: 5 } : {}),