@studious-lms/server 1.13.0 → 1.15.0

package/src/services/embedding.ts

@@ -0,0 +1,446 @@
+/**
+ * Embedding service – pgvector-backed RAG for class materials.
+ * Handles text chunking, embedding generation, storage, and semantic search.
+ */
+import { prisma } from '../lib/prisma.js';
+import { openAIClient } from '../utils/inference.js';
+import { logger } from '../utils/logger.js';
+import { enqueueRunJob } from '../workers/queues.js';
+import type { EmbeddingSourceType } from '@prisma/client';
+
+const EMBEDDING_MODEL = 'text-embedding-3-small';
+const EMBEDDING_DIMENSIONS = 1536;
+const CHUNK_SIZE = 512; // tokens (~2048 chars)
+const CHUNK_OVERLAP = 100; // tokens (~400 chars)
+const CHARS_PER_TOKEN = 4;
+
+export interface EmbeddingSearchResult {
+  id: string;
+  sourceType: EmbeddingSourceType;
+  sourceId: string;
+  content: string;
+  metadata: Record<string, unknown>;
+  similarity: number;
+}
+
+/**
+ * Split text into overlapping chunks for embedding.
+ */
+export function chunkText(text: string, chunkSize = CHUNK_SIZE, overlap = CHUNK_OVERLAP): string[] {
+  const maxChars = chunkSize * CHARS_PER_TOKEN;
+  const overlapChars = overlap * CHARS_PER_TOKEN;
+
+  if (text.length <= maxChars) {
+    return [text];
+  }
+
+  const chunks: string[] = [];
+  let start = 0;
+
+  while (start < text.length) {
+    const end = Math.min(start + maxChars, text.length);
+    chunks.push(text.slice(start, end));
+
+    if (end >= text.length) break;
+    start = end - overlapChars;
+  }
+
+  return chunks;
+}
+
+/**
+ * Generate embedding vector for a text string.
+ */
+export async function generateEmbedding(text: string): Promise<number[]> {
+  const response = await openAIClient.embeddings.create({
+    model: EMBEDDING_MODEL,
+    input: text,
+    dimensions: EMBEDDING_DIMENSIONS,
+  });
+
+  return response.data[0].embedding;
+}
+
+/**
+ * Generate embeddings for multiple texts in a single batch.
+ */
+export async function generateEmbeddings(texts: string[]): Promise<number[][]> {
+  if (texts.length === 0) return [];
+
+  const response = await openAIClient.embeddings.create({
+    model: EMBEDDING_MODEL,
+    input: texts,
+    dimensions: EMBEDDING_DIMENSIONS,
+  });
+
+  return response.data
+    .sort((a, b) => a.index - b.index)
+    .map((item) => item.embedding);
+}
+
+/**
+ * Store embeddings for a source document. Replaces any existing embeddings for
+ * the same sourceType + sourceId combination.
+ */
+export async function indexContent(params: {
+  classId: string;
+  sourceType: EmbeddingSourceType;
+  sourceId: string;
+  text: string;
+  metadata?: Record<string, unknown>;
+}): Promise<number> {
+  const { classId, sourceType, sourceId, text, metadata = {} } = params;
+
+  // Remove existing embeddings for this source
+  await prisma.embedding.deleteMany({
+    where: { classId, sourceType, sourceId },
+  });
+
+  const chunks = chunkText(text);
+  if (chunks.length === 0) return 0;
+
+  const embeddings = await generateEmbeddings(chunks);
+
+  // Insert using raw SQL for pgvector support
+  for (let i = 0; i < chunks.length; i++) {
+    const vectorStr = `[${embeddings[i].join(',')}]`;
+    const chunkMetadata = { ...metadata, chunkIndex: i, totalChunks: chunks.length };
+
+    await prisma.$executeRaw`
+      INSERT INTO "Embedding" (id, "classId", "sourceType", "sourceId", content, vector, metadata, "createdAt")
+      VALUES (
+        gen_random_uuid(),
+        ${classId},
+        ${sourceType}::"EmbeddingSourceType",
+        ${sourceId},
+        ${chunks[i]},
+        ${vectorStr}::vector,
+        ${JSON.stringify(chunkMetadata)}::jsonb,
+        NOW()
+      )
+    `;
+  }
+
+  logger.info('Content indexed', { classId, sourceType, sourceId, chunks: chunks.length });
+  return chunks.length;
+}
+
+/**
+ * Semantic search over class embeddings using cosine similarity.
+ */
+export async function searchEmbeddings(params: {
+  classId: string;
+  query: string;
+  limit?: number;
+  sourceTypes?: EmbeddingSourceType[];
+}): Promise<EmbeddingSearchResult[]> {
+  const { classId, query, limit = 5, sourceTypes } = params;
+
+  const queryEmbedding = await generateEmbedding(query);
+  const vectorStr = `[${queryEmbedding.join(',')}]`;
+
+  let results: EmbeddingSearchResult[];
+
+  if (sourceTypes && sourceTypes.length > 0) {
+    results = await prisma.$queryRaw<EmbeddingSearchResult[]>`
+      SELECT
+        id,
+        "sourceType",
+        "sourceId",
+        content,
+        metadata,
+        1 - (vector <=> ${vectorStr}::vector) as similarity
+      FROM "Embedding"
+      WHERE "classId" = ${classId}
+        AND "sourceType" = ANY(${sourceTypes}::"EmbeddingSourceType"[])
+      ORDER BY vector <=> ${vectorStr}::vector
+      LIMIT ${limit}
+    `;
+  } else {
+    results = await prisma.$queryRaw<EmbeddingSearchResult[]>`
+      SELECT
+        id,
+        "sourceType",
+        "sourceId",
+        content,
+        metadata,
+        1 - (vector <=> ${vectorStr}::vector) as similarity
+      FROM "Embedding"
+      WHERE "classId" = ${classId}
+      ORDER BY vector <=> ${vectorStr}::vector
+      LIMIT ${limit}
+    `;
+  }
+
+  return results;
+}
+
+/**
+ * Remove all embeddings for a given source.
+ */
+export async function removeEmbeddings(params: {
+  classId: string;
+  sourceType: EmbeddingSourceType;
+  sourceId: string;
+}): Promise<void> {
+  await prisma.embedding.deleteMany({
+    where: {
+      classId: params.classId,
+      sourceType: params.sourceType,
+      sourceId: params.sourceId,
+    },
+  });
+}
+
+/**
+ * Index an assignment (title + instructions + attached file names) into the vector store.
+ */
+export async function indexAssignment(classId: string, assignmentId: string): Promise<void> {
+  const assignment = await prisma.assignment.findUnique({
+    where: { id: assignmentId },
+    select: {
+      title: true,
+      instructions: true,
+      type: true,
+      attachments: { select: { name: true, type: true } },
+    },
+  });
+
+  if (!assignment) return;
+
+  const parts = [
+    `Title: ${assignment.title}`,
+    `Type: ${assignment.type}`,
+    assignment.instructions ? `Instructions: ${assignment.instructions}` : '',
+  ];
+
+  if (assignment.attachments.length > 0) {
+    const fileList = assignment.attachments.map((f) => `${f.name} (${f.type})`).join(', ');
+    parts.push(`Attached files: ${fileList}`);
+  }
+
+  await indexContent({
+    classId,
+    sourceType: 'ASSIGNMENT',
+    sourceId: assignmentId,
+    text: parts.filter(Boolean).join('\n'),
+    metadata: { title: assignment.title, type: assignment.type, fileCount: assignment.attachments.length },
+  });
+}
+
+/**
+ * Index a worksheet (questions + answers) into the vector store.
+ */
+export async function indexWorksheet(classId: string, worksheetId: string): Promise<void> {
+  const worksheet = await prisma.worksheet.findUnique({
+    where: { id: worksheetId },
+    include: { questions: true },
+  });
+
+  if (!worksheet) return;
+
+  const questionTexts = worksheet.questions.map((q, i) =>
+    `Question ${i + 1} (${q.type}): ${q.question}\nAnswer: ${q.answer}`
+  );
+
+  const text = [
+    `Worksheet: ${worksheet.name}`,
+    ...questionTexts,
+  ].join('\n\n');
+
+  await indexContent({
+    classId,
+    sourceType: 'WORKSHEET',
+    sourceId: worksheetId,
+    text,
+    metadata: { name: worksheet.name, questionCount: worksheet.questions.length },
+  });
+}
+
+/**
+ * Index an announcement into the vector store.
+ */
+export async function indexAnnouncement(classId: string, announcementId: string): Promise<void> {
+  const announcement = await prisma.announcement.findUnique({
+    where: { id: announcementId },
+    select: { remarks: true, teacherId: true },
+  });
+
+  if (!announcement || !announcement.remarks) return;
+
+  const text = `Announcement: ${announcement.remarks}`;
+
+  await indexContent({
+    classId,
+    sourceType: 'ANNOUNCEMENT',
+    sourceId: announcementId,
+    text,
+    metadata: { teacherId: announcement.teacherId },
+  });
+}
+
+/**
+ * Index a discussion (title + body + replies) into the vector store.
+ */
+export async function indexDiscussion(classId: string, discussionId: string): Promise<void> {
+  const discussion = await prisma.discussion.findUnique({
+    where: { id: discussionId },
+    select: {
+      title: true,
+      body: true,
+      type: true,
+      author: { select: { username: true, profile: { select: { displayName: true } } } },
+      replies: {
+        select: { body: true, author: { select: { username: true, profile: { select: { displayName: true } } } } },
+        orderBy: { createdAt: 'asc' },
+        take: 20,
+      },
+    },
+  });
+
+  if (!discussion) return;
+
+  const authorName = discussion.author.profile?.displayName || discussion.author.username;
+  const parts = [
+    `Discussion: ${discussion.title}`,
+    `Type: ${discussion.type}`,
+    `Author: ${authorName}`,
+    `Body: ${discussion.body}`,
+  ];
+
+  if (discussion.replies.length > 0) {
+    parts.push('Replies:');
+    for (const reply of discussion.replies) {
+      const replyAuthor = reply.author.profile?.displayName || reply.author.username;
+      parts.push(`  ${replyAuthor}: ${reply.body.slice(0, 300)}`);
+    }
+  }
+
+  await indexContent({
+    classId,
+    sourceType: 'DISCUSSION',
+    sourceId: discussionId,
+    text: parts.join('\n'),
+    metadata: { title: discussion.title, type: discussion.type, replyCount: discussion.replies.length },
+  });
+}
+
+/**
+ * Index a submission (grade + feedback + file attachments + extended response) into the vector store.
+ */
+export async function indexSubmission(classId: string, submissionId: string): Promise<void> {
+  const submission = await prisma.submission.findUnique({
+    where: { id: submissionId },
+    select: {
+      gradeReceived: true,
+      teacherComments: true,
+      extendedResponse: true,
+      returned: true,
+      student: { select: { username: true, profile: { select: { displayName: true } } } },
+      assignment: { select: { title: true, maxGrade: true, type: true } },
+      attachments: { select: { name: true, type: true } },
+      annotations: { select: { name: true, type: true } },
+    },
+  });
+
+  if (!submission) return;
+
+  const studentName = submission.student.profile?.displayName || submission.student.username;
+  const parts = [
+    `Submission for: ${submission.assignment.title}`,
+    `Student: ${studentName}`,
+    `Assignment type: ${submission.assignment.type}`,
+  ];
+
+  if (submission.gradeReceived != null) {
+    parts.push(`Grade: ${submission.gradeReceived}/${submission.assignment.maxGrade ?? '?'}`);
+  }
+  if (submission.teacherComments) {
+    parts.push(`Teacher feedback: ${submission.teacherComments}`);
+  }
+  if (submission.extendedResponse) {
+    parts.push(`Student response: ${submission.extendedResponse.slice(0, 500)}`);
+  }
+  if (submission.attachments.length > 0) {
+    parts.push(`Submitted files: ${submission.attachments.map((f) => f.name).join(', ')}`);
+  }
+  if (submission.annotations.length > 0) {
+    parts.push(`Teacher annotations: ${submission.annotations.map((f) => f.name).join(', ')}`);
+  }
+
+  await indexContent({
+    classId,
+    sourceType: 'SUBMISSION',
+    sourceId: submissionId,
+    text: parts.join('\n'),
+    metadata: { studentName, assignmentTitle: submission.assignment.title, grade: submission.gradeReceived },
+  });
+}
+
+/**
+ * Index a class syllabus into the vector store.
+ * Uses classId as both classId and sourceId since syllabus is a field on the class.
+ */
+export async function indexSyllabus(classId: string): Promise<void> {
+  const cls = await prisma.class.findUnique({
+    where: { id: classId },
+    select: { name: true, subject: true, syllabus: true },
+  });
+
+  if (!cls?.syllabus) return;
+
+  const text = [
+    `Syllabus for ${cls.name} (${cls.subject})`,
+    cls.syllabus,
+  ].join('\n\n');
+
+  await indexContent({
+    classId,
+    sourceType: 'SYLLABUS',
+    sourceId: classId,
+    text,
+    metadata: { className: cls.name, subject: cls.subject },
+  });
+}
+
+/**
+ * Index a file (name + type + path) into the vector store.
+ * For text-based content, PDF extraction would be added here in the future.
+ */
+export async function indexFile(classId: string, fileId: string): Promise<void> {
+  const file = await prisma.file.findUnique({
+    where: { id: fileId },
+    select: { name: true, path: true, type: true, size: true },
+  });
+
+  if (!file) return;
+
+  const text = [
+    `File: ${file.name}`,
+    `Type: ${file.type}`,
+    `Path: ${file.path}`,
+  ].join('\n');
+
+  await indexContent({
+    classId,
+    sourceType: 'FILE',
+    sourceId: fileId,
+    text,
+    metadata: { name: file.name, type: file.type, size: file.size },
+  });
+}
+
+/**
+ * Fire-and-forget: enqueue a background job to embed content.
+ * Safe to call from service functions after a create/update.
+ */
+export function enqueueEmbedding(
+  classId: string,
+  sourceType: EmbeddingSourceType,
+  sourceId: string,
+): void {
+  enqueueRunJob('EMBED_CONTENT', { classId, sourceType: sourceType as string, sourceId }, { classId })
+    .catch((err) => {
+      logger.error('Failed to enqueue embedding job', { err, classId, sourceType, sourceId });
+    });
+}

package/src/services/folder.ts

@@ -28,6 +28,7 @@ import {
   type DirectFileData,
   confirmDirectUpload,
 } from "../lib/fileUpload.js";
+import { enqueueEmbedding } from "./embedding.js";
 import {
   type AccessLevel,
   getScopeDefault,
@@ -421,9 +422,39 @@ export async function confirmFolderUpload(
     input.uploadSuccess,
     input.errorMessage
   );
+
+  // Index file for RAG search if it belongs to a class folder
+  if (input.uploadSuccess) {
+    const file = await prisma.file.findUnique({
+      where: { id: input.fileId },
+      select: { folderId: true },
+    });
+    if (file?.folderId) {
+      const classId = await resolveFileClassId(file.folderId);
+      if (classId) {
+        enqueueEmbedding(classId, 'FILE', input.fileId);
+      }
+    }
+  }
+
   return { success: true };
 }
 
+/** Walk up folder tree to find classId, if any. */
+async function resolveFileClassId(folderId: string): Promise<string | null> {
+  let currentId: string | null = folderId;
+  while (currentId) {
+    const folder = await prisma.folder.findUnique({
+      where: { id: currentId },
+      select: { classId: true, parentFolderId: true },
+    });
+    if (!folder) return null;
+    if (folder.classId) return folder.classId;
+    currentId = folder.parentFolderId;
+  }
+  return null;
+}
+
 /** What scopes can this user pick from in cloud-storage UIs? */
 export async function listUserScopes(userId: string) {
   const memberships = await prisma.schoolMembership.findMany({

package/src/services/marketing.ts

@@ -4,7 +4,7 @@
 import { v4 } from "uuid";
 import {
   createSchoolDevelopmentProgram,
-  findSchoolDevelopmentProgramById,
+  findSchoolDevelopmentProgramStatusById,
   createEarlyAccessRequest,
 } from "../models/marketing.js";
 
@@ -42,9 +42,9 @@ export async function createSchoolDevelopmentProgramRecord(input: {
   return { id: program.id };
 }
 
-/** Look up a school development program by ID. */
+/** Public application status lookup (no contact PII). */
 export async function searchSchoolDevelopmentPrograms(id: string) {
-  return findSchoolDevelopmentProgramById(id);
+  return findSchoolDevelopmentProgramStatusById(id);
 }
 
 /** Create an early access request. */

package/src/services/message.ts

@@ -2,7 +2,7 @@
  * Message service – send, update, delete messages; mark as read; track mentions.
  * Broadcasts real-time updates via Pusher.
  */
-import type { Prisma } from "@prisma/client";
+import type { AppTransactionClient } from "../lib/prisma.js";
 import { TRPCError } from "@trpc/server";
 import { prisma } from "../lib/prisma.js";
 import { chatChannel, pusher } from "../lib/pusher.js";
@@ -35,7 +35,7 @@ export function ensureMessageHasContentOrAttachments(
 }
 
 export async function claimMessageAttachments(
-  tx: Prisma.TransactionClient,
+  tx: AppTransactionClient,
   userId: string,
   conversationId: string,
   messageId: string,