@studious-lms/server 1.4.9 → 1.6.0

package/src/workers/index.ts

@@ -0,0 +1,71 @@
+/**
+ * Worker process entrypoint.
+ *
+ * Run separately from the HTTP server (e.g. `node dist/workers/index.js` on
+ * Render/Railway/Fly). Importing this file boots every BullMQ Worker
+ * declared under `src/workers/processors/*`. Do NOT import this from the
+ * HTTP server — that would spin up workers in your serverless functions.
+ */
+import '../instrument.js';
+
+import { Worker } from 'bullmq';
+import { prisma } from '../lib/prisma.js';
+import { logger } from '../utils/logger.js';
+import { bullConnection } from './connection.js';
+import { aiLabChatWorker } from './processors/aiLabChat.js';
+import { aiNewtonChatWorker } from './processors/aiNewtonChat.js';
+import { gradeWorksheetWorker } from './processors/gradeWorksheet.js';
+import { pdfToWorksheetWorker } from './processors/pdfToWorksheet.js';
+import { studentProgressChatWorker } from './processors/studentProgressChat.js';
+
+const workers: Worker<any>[] = [
+  pdfToWorksheetWorker,
+  gradeWorksheetWorker,
+  aiLabChatWorker,
+  aiNewtonChatWorker,
+  studentProgressChatWorker,
+];
+
+logger.info('Workers ready', {
+  pid: process.pid,
+  queues: workers.map((w) => w.name),
+});
+
+let shuttingDown = false;
+
+async function shutdown(signal: string): Promise<void> {
+  if (shuttingDown) return;
+  shuttingDown = true;
+
+  logger.info(`Received ${signal}, draining workers...`);
+
+  // `worker.close()` waits for in-flight jobs to finish (up to the worker's
+  // settings) before resolving. Run them in parallel.
+  await Promise.allSettled(workers.map((w) => w.close()));
+
+  await Promise.allSettled([
+    bullConnection.quit(),
+    prisma.$disconnect(),
+  ]);
+
+  logger.info('Workers shut down cleanly');
+  process.exit(0);
+}
+
+process.on('SIGTERM', () => void shutdown('SIGTERM'));
+process.on('SIGINT', () => void shutdown('SIGINT'));
+
+process.on('uncaughtException', (error) => {
+  logger.error('Uncaught exception in worker process', {
+    error: error.message,
+    stack: error.stack,
+  });
+  process.exit(1);
+});
+
+process.on('unhandledRejection', (reason) => {
+  logger.error('Unhandled rejection in worker process', {
+    reason: reason instanceof Error ? reason.message : String(reason),
+  });
+  process.exit(1);
+});

package/src/workers/processors/aiLabChat.ts

@@ -0,0 +1,65 @@
+import {
+  generateAndSendLabIntroduction,
+  generateAndSendLabResponse,
+} from '../../pipelines/aiLabChat.js';
+import { prisma } from '../../lib/prisma.js';
+import { isAIUser } from '../../utils/aiUser.js';
+import { createRunJobWorker } from './createRunJobWorker.js';
+
+export const aiLabChatWorker = createRunJobWorker({
+  type: 'AI_LAB_CHAT',
+  workerOptions: {
+    // Lab responses can produce PDFs and large schemas; cap concurrency to
+    // keep memory under control and to honour upstream model rate limits.
+    concurrency: 1,
+    limiter: { max: 5, duration: 60_000 },
+  },
+  run: async ({ conversationId, aiMessageId }) => {
+    const labChat = await prisma.labChat.findFirst({
+      where: { conversationId },
+      select: {
+        id: true,
+        classId: true,
+        context: true,
+        class: { select: { subject: true } },
+      },
+    });
+
+    if (!labChat) {
+      throw new Error(
+        `LabChat for conversationId=${conversationId} not found`,
+      );
+    }
+
+    // The AI message is a placeholder we created before enqueue; the
+    // pipeline fills it in. Look up the most recent non-AI message in the
+    // conversation to feed the response generator. If none exists, this is
+    // the introductory message at the top of a brand-new lab chat.
+    const triggeringMessage = await prisma.message.findFirst({
+      where: {
+        conversationId,
+        senderId: { not: 'AI_ASSISTANT' },
+      },
+      orderBy: { createdAt: 'desc' },
+      select: { id: true, content: true, senderId: true },
+    });
+
+    if (!triggeringMessage || isAIUser(triggeringMessage.senderId)) {
+      await generateAndSendLabIntroduction(
+        labChat.id,
+        conversationId,
+        labChat.context,
+        labChat.class?.subject || 'Lab',
+        aiMessageId,
+      );
+      return { labChatId: labChat.id, kind: 'introduction', aiMessageId };
+    }
+
+    await generateAndSendLabResponse(labChat.id, triggeringMessage.content, {
+      classId: labChat.classId,
+      aiMessageId,
+    });
+
+    return { labChatId: labChat.id, aiMessageId };
+  },
+});

package/src/workers/processors/aiNewtonChat.ts

@@ -0,0 +1,75 @@
+import {
+  generateAndSendNewtonIntroduction,
+  generateAndSendNewtonResponse,
+} from '../../pipelines/aiNewtonChat.js';
+import { prisma } from '../../lib/prisma.js';
+import { isAIUser } from '../../utils/aiUser.js';
+import { createRunJobWorker } from './createRunJobWorker.js';
+
+export const aiNewtonChatWorker = createRunJobWorker({
+  type: 'AI_NEWTON_CHAT',
+  workerOptions: {
+    concurrency: 2,
+    limiter: { max: 10, duration: 60_000 },
+  },
+  run: async ({ conversationId, aiMessageId }) => {
+    const newtonChat = await prisma.newtonChat.findFirst({
+      where: { conversationId },
+      select: {
+        id: true,
+        submission: {
+          select: {
+            id: true,
+            assignment: {
+              select: {
+                id: true,
+                title: true,
+                instructions: true,
+                class: { select: { subject: true } },
+              },
+            },
+          },
+        },
+      },
+    });
+
+    if (!newtonChat) {
+      throw new Error(
+        `NewtonChat for conversationId=${conversationId} not found`,
+      );
+    }
+
+    // Resolve the triggering student message (if any). On a brand-new
+    // tutor session there are no student messages yet, so we run the
+    // introduction path; otherwise we run the response path against the
+    // most recent non-AI message.
+    const triggeringMessage = await prisma.message.findFirst({
+      where: {
+        conversationId,
+        senderId: { not: 'AI_ASSISTANT' },
+      },
+      orderBy: { createdAt: 'desc' },
+      select: { id: true, content: true, senderId: true },
+    });
+
+    if (!triggeringMessage || isAIUser(triggeringMessage.senderId)) {
+      await generateAndSendNewtonIntroduction(
+        newtonChat.id,
+        conversationId,
+        newtonChat.submission.id,
+        aiMessageId,
+      );
+      return { newtonChatId: newtonChat.id, kind: 'introduction', aiMessageId };
+    }
+
+    await generateAndSendNewtonResponse(
+      newtonChat.id,
+      triggeringMessage.content,
+      conversationId,
+      newtonChat.submission,
+      aiMessageId,
+    );
+
+    return { newtonChatId: newtonChat.id, aiMessageId };
+  },
+});

package/src/workers/processors/createRunJobWorker.ts

@@ -0,0 +1,164 @@
+/**
+ * `createRunJobWorker` — shared factory for every BullMQ processor in this app.
+ *
+ * Wraps the boilerplate that every RunJob-backed worker needs:
+ *   - Idempotent skip if the RunJob already SUCCEEDED (recovery after a crash
+ *     that lost the BullMQ ack).
+ *   - Status transitions: PENDING/RUNNING → RUNNING (with attempts++) → SUCCEEDED.
+ *   - Failure handling: record latest error every attempt, flip to FAILED only
+ *     on the final retry.
+ *   - Standardised structured logging on `error` / `failed` / `completed`.
+ *
+ * A processor implementation only has to provide its `RunJobType`, optional
+ * BullMQ options (concurrency, limiter, etc.), and a `run` callback that does
+ * the actual work. The callback receives the typed `JobInputs[type]` payload
+ * and a context object with `runJobId`, `bullJobId`, `attempt`, plus the
+ * `userId` / `classId` from the original `enqueueRunJob` call.
+ */
+import { Worker, type Job, type WorkerOptions } from 'bullmq';
+import { Prisma, type RunJobType } from '@prisma/client';
+import { bullConnection } from '../connection.js';
+import { prisma } from '../../lib/prisma.js';
+import { logger } from '../../utils/logger.js';
+import { QUEUE_NAMES, type JobInputs } from '../queues.js';
+
+export interface RunJobContext {
+  runJobId: string;
+  bullJobId: string | undefined;
+  /** 1-indexed attempt number of the current run. */
+  attempt: number;
+  /** From `EnqueueOptions.userId` on the producer side. */
+  userId: string | null;
+  /** From `EnqueueOptions.classId` on the producer side. */
+  classId: string | null;
+}
+
+export interface CreateRunJobWorkerOptions<T extends RunJobType> {
+  type: T;
+  /** BullMQ `WorkerOptions` minus `connection`, which is supplied by the helper. */
+  workerOptions?: Omit<WorkerOptions, 'connection'>;
+  /**
+   * The actual work. Return any JSON-serialisable value (or `null`) — it gets
+   * persisted to `RunJob.result` for the client to read back.
+   */
+  run: (
+    input: JobInputs[T],
+    ctx: RunJobContext,
+  ) => Promise<Prisma.InputJsonValue | null>;
+}
+
+type WorkerData<T extends RunJobType> = JobInputs[T] & { runJobId: string };
+
+export function createRunJobWorker<T extends RunJobType>({
+  type,
+  workerOptions,
+  run,
+}: CreateRunJobWorkerOptions<T>): Worker<WorkerData<T>> {
+  const name = QUEUE_NAMES[type];
+
+  const worker = new Worker<WorkerData<T>>(
+    name,
+    async (job: Job<WorkerData<T>>) => {
+      const { runJobId, ...rest } = job.data;
+      // The destructured rest is `Omit<JobInputs[T] & { runJobId }, 'runJobId'>`,
+      // which TS can't simplify back to `JobInputs[T]` while `T` is generic.
+      const input = rest as unknown as JobInputs[T];
+
+      const existing = await prisma.runJob.findUnique({
+        where: { id: runJobId },
+        select: { status: true, result: true, userId: true, classId: true },
+      });
+
+      if (existing?.status === 'SUCCEEDED') {
+        logger.info(`${name} RunJob already succeeded; skipping re-run`, {
+          runJobId,
+          bullJobId: job.id,
+        });
+        return existing.result;
+      }
+
+      await prisma.runJob.update({
+        where: { id: runJobId },
+        data: {
+          status: 'RUNNING',
+          startedAt: existing?.status === 'RUNNING' ? undefined : new Date(),
+          attempts: { increment: 1 },
+        },
+      });
+
+      const result = await run(input, {
+        runJobId,
+        bullJobId: job.id,
+        attempt: job.attemptsMade + 1,
+        userId: existing?.userId ?? null,
+        classId: existing?.classId ?? null,
+      });
+
+      await prisma.runJob.update({
+        where: { id: runJobId },
+        data: {
+          status: 'SUCCEEDED',
+          finishedAt: new Date(),
+          error: null,
+          result: result === null ? Prisma.JsonNull : result,
+        },
+      });
+
+      return result;
+    },
+    { connection: bullConnection, ...workerOptions },
+  );
+
+  worker.on('error', (err) => {
+    logger.error(`${name} worker error`, {
+      error: err.message,
+      stack: err.stack,
+    });
+  });
+
+  worker.on('failed', async (job, err) => {
+    if (!job) return;
+
+    const runJobId = job.data?.runJobId;
+    const isFinalAttempt = job.attemptsMade >= (job.opts.attempts ?? 1);
+
+    logger.error(`${name} job failed`, {
+      bullJobId: job.id,
+      runJobId,
+      attempt: job.attemptsMade,
+      finalAttempt: isFinalAttempt,
+      error: err.message,
+    });
+
+    if (!runJobId) return;
+
+    await prisma.runJob
+      .update({
+        where: { id: runJobId },
+        data: {
+          error: err.message,
+          ...(isFinalAttempt
+            ? { status: 'FAILED', finishedAt: new Date() }
+            : {}),
+        },
+      })
+      .catch((updateError) => {
+        logger.error(`Failed to record ${name} failure on RunJob`, {
+          runJobId,
+          error:
+            updateError instanceof Error
+              ? updateError.message
+              : String(updateError),
+        });
+      });
+  });
+
+  worker.on('completed', (job) => {
+    logger.info(`${name} job completed`, {
+      bullJobId: job.id,
+      runJobId: job.data?.runJobId,
+    });
+  });
+
+  return worker;
+}

package/src/workers/processors/gradeWorksheet.ts

@@ -0,0 +1,16 @@
+import { gradeWorksheetPipeline } from '../../pipelines/gradeWorksheet.js';
+import { createRunJobWorker } from './createRunJobWorker.js';
+
+export const gradeWorksheetWorker = createRunJobWorker({
+  type: 'GRADE_WORKSHEET',
+  workerOptions: {
+    concurrency: 2,
+    // Each pipeline run can fan out to many per-question inferences; keep
+    // throughput modest until we measure real-world usage.
+    limiter: { max: 5, duration: 60_000 },
+  },
+  run: async ({ worksheetResponseId }) => {
+    await gradeWorksheetPipeline(worksheetResponseId);
+    return { worksheetResponseId };
+  },
+});

package/src/workers/processors/pdfToWorksheet.ts

@@ -0,0 +1,17 @@
+import { pdfToWorksheet } from '../../pipelines/pdfToWorksheet.js';
+import { createRunJobWorker } from './createRunJobWorker.js';
+
+export const pdfToWorksheetWorker = createRunJobWorker({
+  type: 'PDF_TO_WORKSHEET',
+  workerOptions: {
+    concurrency: 2,
+    // 5 jobs / minute, e.g. to stay under OpenAI rate limits.
+    limiter: { max: 5, duration: 60_000 },
+  },
+  run: async ({ gcsUrl, classId, worksheetId }) => {
+    const worksheet = await pdfToWorksheet(gcsUrl, classId, { worksheetId });
+    return worksheet
+      ? { worksheetId: worksheet.id, name: worksheet.name }
+      : null;
+  },
+});

package/src/workers/processors/studentProgressChat.ts

@@ -0,0 +1,28 @@
+import { chatAboutStudentProgress } from '../../services/studentProgress.js';
+import { createRunJobWorker } from './createRunJobWorker.js';
+
+export const studentProgressChatWorker = createRunJobWorker({
+  type: 'STUDENT_PROGRESS_CHAT',
+  workerOptions: {
+    concurrency: 4,
+    limiter: { max: 20, duration: 60_000 },
+  },
+  run: async ({ classId, studentId, message }, ctx) => {
+    // `chatAboutStudentProgress` runs an authorization check using the viewer
+    // id, so callers MUST pass `userId` via `enqueueRunJob` options for the
+    // service to receive it here.
+    if (!ctx.userId) {
+      throw new Error(
+        'STUDENT_PROGRESS_CHAT requires `userId` on the RunJob (set via enqueueRunJob options)',
+      );
+    }
+
+    const response = await chatAboutStudentProgress(ctx.userId, {
+      classId,
+      studentId,
+      message,
+    });
+
+    return response;
+  },
+});

package/src/workers/queues.ts

@@ -0,0 +1,174 @@
+/**
+ * BullMQ queue producers and the `enqueueRunJob` helper.
+ *
+ * Producer side only — safe to import from HTTP routes / services running on
+ * Vercel. Importing this file does NOT start any workers; workers live in
+ * `src/workers/processors/*` and are wired up by `src/workers/index.ts`,
+ * which is the entrypoint of the separate long-running worker process.
+ */
+import { Queue, type JobsOptions } from 'bullmq';
+import { Prisma, type RunJobType } from '@prisma/client';
+import { bullConnection } from './connection.js';
+import { prisma } from '../lib/prisma.js';
+import { logger } from '../utils/logger.js';
+
+const DEFAULT_JOB_OPTIONS: JobsOptions = {
+  attempts: 3,
+  backoff: { type: 'exponential', delay: 5_000 },
+  removeOnComplete: { age: 60 * 60 }, // keep completed jobs in Redis for 1h, then drop
+  removeOnFail: { age: 24 * 60 * 60 }, // keep failed jobs for 24h for inspection
+};
+
+/**
+ * Map RunJobType → (queue name, input payload shape).
+ *
+ * Add a new entry here whenever you introduce a new background job type.
+ * The queue name string must match what the corresponding processor passes
+ * to `new Worker(<name>, ...)`.
+ */
+export type JobInputs = {
+  /**
+   * `worksheetId` is the placeholder Worksheet row created synchronously by
+   * the producer (so the client has an id to navigate to). The pipeline fills
+   * that row in asynchronously and broadcasts progress on the per-worksheet
+   * job channel.
+   */
+  PDF_TO_WORKSHEET: { gcsUrl: string; classId: string; worksheetId: string };
+  GRADE_WORKSHEET: { worksheetResponseId: string };
+  /**
+   * `aiMessageId` is the placeholder AI Message row created synchronously by
+   * the producer before enqueue (so the client renders a PENDING bubble in
+   * the timeline). The pipeline fills in the content and flips the row to
+   * COMPLETED / FAILED. Used for both intros and replies — single shape.
+   */
+  AI_LAB_CHAT: { conversationId: string; aiMessageId: string };
+  /**
+   * `aiMessageId` placeholder pattern, same as AI_LAB_CHAT — see above.
+   */
+  AI_NEWTON_CHAT: { conversationId: string; aiMessageId: string };
+  STUDENT_PROGRESS_CHAT: { classId: string; studentId: string; message: string };
+};
+
+export const QUEUE_NAMES: Record<RunJobType, string> = {
+  PDF_TO_WORKSHEET: 'pdfToWorksheet',
+  GRADE_WORKSHEET: 'gradeWorksheet',
+  AI_LAB_CHAT: 'aiLabChat',
+  AI_NEWTON_CHAT: 'aiNewtonChat',
+  STUDENT_PROGRESS_CHAT: 'studentProgressChat',
+};
+
+/**
+ * One Queue instance per type. Created lazily so that importing this module
+ * does not eagerly open queues you may never use in a given runtime.
+ */
+const queueCache = new Map<RunJobType, Queue>();
+
+function getQueue(type: RunJobType): Queue {
+  let queue = queueCache.get(type);
+  if (!queue) {
+    queue = new Queue(QUEUE_NAMES[type], {
+      connection: bullConnection,
+      defaultJobOptions: DEFAULT_JOB_OPTIONS,
+    });
+    queueCache.set(type, queue);
+  }
+  return queue;
+}
+
+export interface EnqueueOptions {
+  classId?: string | null;
+  userId?: string | null;
+  /** Override BullMQ job options (attempts, delay, priority, etc.). */
+  jobOptions?: JobsOptions;
+}
+
+export interface EnqueueResult {
+  runJobId: string;
+  bullJobId: string | null;
+}
+
+/**
+ * Create a `RunJob` audit row, enqueue the BullMQ job, and link the two.
+ *
+ * Returns immediately — the actual work happens on the worker process. The
+ * caller can return `runJobId` to the client and have them poll `getRunJob`
+ * (or subscribe over Pusher) to track progress.
+ *
+ * IMPORTANT: do not call this inside a Prisma `$transaction`. If the outer
+ * transaction rolls back after the BullMQ job is added, the worker will pick
+ * up an orphaned job pointing at a non-existent RunJob row. Enqueue *after*
+ * the transaction commits.
+ */
+export async function enqueueRunJob<T extends RunJobType>(
+  type: T,
+  input: JobInputs[T],
+  options: EnqueueOptions = {},
+): Promise<EnqueueResult> {
+  const runJob = await prisma.runJob.create({
+    data: {
+      type,
+      status: 'PENDING',
+      classId: options.classId ?? null,
+      userId: options.userId ?? null,
+      input: input as unknown as Prisma.InputJsonValue,
+    },
+  });
+
+  try {
+    const queue = getQueue(type);
+    const job = await queue.add(
+      QUEUE_NAMES[type],
+      { runJobId: runJob.id, ...input },
+      options.jobOptions,
+    );
+
+    try {
+      await prisma.runJob.update({
+        where: { id: runJob.id },
+        data: { bullJobId: job.id ?? null },
+      });
+    } catch (linkError) {
+      logger.error('RunJob enqueued but bullJobId linkage failed', {
+        runJobId: runJob.id,
+        bullJobId: job.id ?? null,
+        type,
+        error: linkError instanceof Error ? linkError.message : String(linkError),
+      });
+    }
+
+    return { runJobId: runJob.id, bullJobId: job.id ?? null };
+  } catch (error) {
+    logger.error('Failed to enqueue RunJob; marking FAILED', {
+      runJobId: runJob.id,
+      type,
+      error: error instanceof Error ? error.message : String(error),
+    });
+
+    try {
+      await prisma.runJob.update({
+        where: { id: runJob.id },
+        data: {
+          status: 'FAILED',
+          error: error instanceof Error ? error.message : String(error),
+          finishedAt: new Date(),
+        },
+      });
+    } catch (updateError) {
+      logger.error('Failed to mark RunJob as FAILED after enqueue error', {
+        runJobId: runJob.id,
+        error: updateError instanceof Error ? updateError.message : String(updateError),
+      });
+    }
+
+    throw error;
+  }
+}
+
+/**
+ * Close every cached queue. Call from the HTTP server's shutdown handler so
+ * the producer side disconnects from Redis cleanly.
+ */
+export async function closeAllQueues(): Promise<void> {
+  await Promise.all(Array.from(queueCache.values()).map((q) => q.close()));
+  queueCache.clear();
+}

package/tests/{globalSetup.ts → integration/globalSetup.ts}

@@ -4,10 +4,10 @@ import type { TestProject } from 'vitest/node';
 
 config({ path: resolve(process.cwd(), '.env.test'), override: true });
 
-import { prisma } from '../src/lib/prisma';
-import { appRouter } from '../src/routers/_app';
-import { createTRPCContext } from '../src/trpc';
-import { clearDatabase } from '../src/seedDatabase';
+import { prisma } from '../../src/lib/prisma';
+import { appRouter } from '../../src/routers/_app';
+import { createTRPCContext } from '../../src/trpc';
+import { clearDatabase } from '../../src/seedDatabase';
 import { execSync } from 'child_process';
 import { SessionType } from '@prisma/client';
 

package/tests/{helpers.ts → integration/helpers.ts}

@@ -1,5 +1,5 @@
-import { createTRPCContext } from '../src/trpc';
-import { appRouter } from '../src/routers/_app';
+import { createTRPCContext } from '../../src/trpc';
+import { appRouter } from '../../src/routers/_app';
 
 export async function createUnauthCaller() {
   const ctx = await createTRPCContext({