@studious-lms/server 1.4.7 → 1.5.0

package/src/workers/processors/pdfToWorksheet.ts

@@ -0,0 +1,119 @@
+import { Worker, type Job } from 'bullmq';
+import { Prisma } from '@prisma/client';
+import { bullConnection } from '../connection.js';
+import { prisma } from '../../lib/prisma.js';
+import { logger } from '../../utils/logger.js';
+import { pdfToWorksheet } from '../../pipelines/pdfToWorksheet.js';
+
+interface PdfToWorksheetJobData {
+  runJobId: string;
+  gcsUrl: string;
+  classId: string;
+}
+
+async function process(job: Job<PdfToWorksheetJobData>) {
+  const { runJobId, gcsUrl, classId } = job.data;
+
+  // If a previous attempt already succeeded for this RunJob (rare race after a
+  // crash that lost the BullMQ ack), return the prior result instead of
+  // re-running the AI call and creating a duplicate worksheet.
+  const existing = await prisma.runJob.findUnique({
+    where: { id: runJobId },
+    select: { status: true, result: true },
+  });
+  if (existing?.status === 'SUCCEEDED') {
+    logger.info('pdfToWorksheet RunJob already succeeded; skipping re-run', {
+      runJobId,
+      bullJobId: job.id,
+    });
+    return existing.result;
+  }
+
+  await prisma.runJob.update({
+    where: { id: runJobId },
+    data: {
+      status: 'RUNNING',
+      startedAt: existing?.status === 'RUNNING' ? undefined : new Date(),
+      attempts: { increment: 1 },
+    },
+  });
+
+  const worksheet = await pdfToWorksheet(gcsUrl, classId);
+
+  const result = worksheet ? { worksheetId: worksheet.id, name: worksheet.name } : null;
+
+  await prisma.runJob.update({
+    where: { id: runJobId },
+    data: {
+      status: 'SUCCEEDED',
+      finishedAt: new Date(),
+      error: null,
+      result: result === null ? Prisma.JsonNull : (result as Prisma.InputJsonValue),
+    },
+  });
+
+  return result;
+}
+
+export const pdfToWorksheetWorker = new Worker<PdfToWorksheetJobData>(
+  'pdfToWorksheet',
+  process,
+  {
+    connection: bullConnection,
+    // Conservative defaults — tune once we have real-world numbers.
+    concurrency: 2,
+    limiter: {
+      max: 5,
+      duration: 60_000, // 5 jobs / minute, e.g. to stay under OpenAI rate limits
+    },
+  },
+);
+
+pdfToWorksheetWorker.on('error', (err) => {
+  logger.error('pdfToWorksheet worker error', {
+    error: err.message,
+    stack: err.stack,
+  });
+});
+
+pdfToWorksheetWorker.on('failed', async (job, err) => {
+  if (!job) return;
+
+  const runJobId = job.data?.runJobId;
+  const isFinalAttempt = job.attemptsMade >= (job.opts.attempts ?? 1);
+
+  logger.error('pdfToWorksheet job failed', {
+    bullJobId: job.id,
+    runJobId,
+    attempt: job.attemptsMade,
+    finalAttempt: isFinalAttempt,
+    error: err.message,
+  });
+
+  if (!runJobId) return;
+
+  // Always record the latest error message on the RunJob row.
+  // Only flip to FAILED on the final attempt — earlier failures are still
+  // "RUNNING" because BullMQ is going to retry.
+  await prisma.runJob.update({
+    where: { id: runJobId },
+    data: {
+      error: err.message,
+      ...(isFinalAttempt
+        ? { status: 'FAILED', finishedAt: new Date() }
+        : {}),
+    },
+  }).catch((updateError) => {
+    logger.error('Failed to record pdfToWorksheet failure on RunJob', {
+      runJobId,
+      error: updateError instanceof Error ? updateError.message : String(updateError),
+    });
+  });
+});
+
+pdfToWorksheetWorker.on('completed', (job) => {
+  logger.info('pdfToWorksheet job completed', {
+    bullJobId: job.id,
+    runJobId: job.data?.runJobId,
+  });
+});

package/src/workers/queues.ts

@@ -0,0 +1,159 @@
+/**
+ * BullMQ queue producers and the `enqueueRunJob` helper.
+ *
+ * Producer side only — safe to import from HTTP routes / services running on
+ * Vercel. Importing this file does NOT start any workers; workers live in
+ * `src/workers/processors/*` and are wired up by `src/workers/index.ts`,
+ * which is the entrypoint of the separate long-running worker process.
+ */
+import { Queue, type JobsOptions } from 'bullmq';
+import { Prisma, type RunJobType } from '@prisma/client';
+import { bullConnection } from './connection.js';
+import { prisma } from '../lib/prisma.js';
+import { logger } from '../utils/logger.js';
+
+const DEFAULT_JOB_OPTIONS: JobsOptions = {
+  attempts: 3,
+  backoff: { type: 'exponential', delay: 5_000 },
+  removeOnComplete: { age: 60 * 60 }, // keep completed jobs in Redis for 1h, then drop
+  removeOnFail: { age: 24 * 60 * 60 }, // keep failed jobs for 24h for inspection
+};
+
+/**
+ * Map RunJobType → (queue name, input payload shape).
+ *
+ * Add a new entry here whenever you introduce a new background job type.
+ * The queue name string must match what the corresponding processor passes
+ * to `new Worker(<name>, ...)`.
+ */
+type JobInputs = {
+  PDF_TO_WORKSHEET: { gcsUrl: string; classId: string };
+  GRADE_WORKSHEET: { worksheetResponseId: string };
+  AI_LAB_CHAT: { conversationId: string; messageId: string };
+  AI_NEWTON_CHAT: { conversationId: string; messageId: string };
+  STUDENT_PROGRESS_CHAT: { classId: string; studentId: string; message: string };
+};
+
+const QUEUE_NAMES: Record<RunJobType, string> = {
+  PDF_TO_WORKSHEET: 'pdfToWorksheet',
+  GRADE_WORKSHEET: 'gradeWorksheet',
+  AI_LAB_CHAT: 'aiLabChat',
+  AI_NEWTON_CHAT: 'aiNewtonChat',
+  STUDENT_PROGRESS_CHAT: 'studentProgressChat',
+};
+
+/**
+ * One Queue instance per type. Created lazily so that importing this module
+ * does not eagerly open queues you may never use in a given runtime.
+ */
+const queueCache = new Map<RunJobType, Queue>();
+
+function getQueue(type: RunJobType): Queue {
+  let queue = queueCache.get(type);
+  if (!queue) {
+    queue = new Queue(QUEUE_NAMES[type], {
+      connection: bullConnection,
+      defaultJobOptions: DEFAULT_JOB_OPTIONS,
+    });
+    queueCache.set(type, queue);
+  }
+  return queue;
+}
+
+export interface EnqueueOptions {
+  classId?: string | null;
+  userId?: string | null;
+  /** Override BullMQ job options (attempts, delay, priority, etc.). */
+  jobOptions?: JobsOptions;
+}
+
+export interface EnqueueResult {
+  runJobId: string;
+  bullJobId: string | null;
+}
+
+/**
+ * Create a `RunJob` audit row, enqueue the BullMQ job, and link the two.
+ *
+ * Returns immediately — the actual work happens on the worker process. The
+ * caller can return `runJobId` to the client and have them poll `getRunJob`
+ * (or subscribe over Pusher) to track progress.
+ *
+ * IMPORTANT: do not call this inside a Prisma `$transaction`. If the outer
+ * transaction rolls back after the BullMQ job is added, the worker will pick
+ * up an orphaned job pointing at a non-existent RunJob row. Enqueue *after*
+ * the transaction commits.
+ */
+export async function enqueueRunJob<T extends RunJobType>(
+  type: T,
+  input: JobInputs[T],
+  options: EnqueueOptions = {},
+): Promise<EnqueueResult> {
+  const runJob = await prisma.runJob.create({
+    data: {
+      type,
+      status: 'PENDING',
+      classId: options.classId ?? null,
+      userId: options.userId ?? null,
+      input: input as unknown as Prisma.InputJsonValue,
+    },
+  });
+
+  try {
+    const queue = getQueue(type);
+    const job = await queue.add(
+      QUEUE_NAMES[type],
+      { runJobId: runJob.id, ...input },
+      options.jobOptions,
+    );
+
+    try {
+      await prisma.runJob.update({
+        where: { id: runJob.id },
+        data: { bullJobId: job.id ?? null },
+      });
+    } catch (linkError) {
+      logger.error('RunJob enqueued but bullJobId linkage failed', {
+        runJobId: runJob.id,
+        bullJobId: job.id ?? null,
+        type,
+        error: linkError instanceof Error ? linkError.message : String(linkError),
+      });
+    }
+
+    return { runJobId: runJob.id, bullJobId: job.id ?? null };
+  } catch (error) {
+    logger.error('Failed to enqueue RunJob; marking FAILED', {
+      runJobId: runJob.id,
+      type,
+      error: error instanceof Error ? error.message : String(error),
+    });
+
+    try {
+      await prisma.runJob.update({
+        where: { id: runJob.id },
+        data: {
+          status: 'FAILED',
+          error: error instanceof Error ? error.message : String(error),
+          finishedAt: new Date(),
+        },
+      });
+    } catch (updateError) {
+      logger.error('Failed to mark RunJob as FAILED after enqueue error', {
+        runJobId: runJob.id,
+        error: updateError instanceof Error ? updateError.message : String(updateError),
+      });
+    }
+
+    throw error;
+  }
+}
+
+/**
+ * Close every cached queue. Call from the HTTP server's shutdown handler so
+ * the producer side disconnects from Redis cleanly.
+ */
+export async function closeAllQueues(): Promise<void> {
+  await Promise.all(Array.from(queueCache.values()).map((q) => q.close()));
+  queueCache.clear();
+}

package/tests/{globalSetup.ts → integration/globalSetup.ts}

@@ -4,10 +4,10 @@ import type { TestProject } from 'vitest/node';
 
 config({ path: resolve(process.cwd(), '.env.test'), override: true });
 
-import { prisma } from '../src/lib/prisma';
-import { appRouter } from '../src/routers/_app';
-import { createTRPCContext } from '../src/trpc';
-import { clearDatabase } from '../src/seedDatabase';
+import { prisma } from '../../src/lib/prisma';
+import { appRouter } from '../../src/routers/_app';
+import { createTRPCContext } from '../../src/trpc';
+import { clearDatabase } from '../../src/seedDatabase';
 import { execSync } from 'child_process';
 import { SessionType } from '@prisma/client';
 

package/tests/{helpers.ts → integration/helpers.ts}

@@ -1,5 +1,5 @@
-import { createTRPCContext } from '../src/trpc';
-import { appRouter } from '../src/routers/_app';
+import { createTRPCContext } from '../../src/trpc';
+import { appRouter } from '../../src/routers/_app';
 
 export async function createUnauthCaller() {
   const ctx = await createTRPCContext({

package/tests/{pipelines → integration/pipelines}/pdfToWorksheet.test.ts

@@ -1,18 +1,19 @@
 /**
- * `pdfToWorksheet` integration-style tests: real DB, mocked inference.
- * CI does not call the model, and the PDF URL is passed through directly.
+ * `pdfToWorksheet` integration-style tests: real DB, mocked OpenAI.
+ *
+ * We intercept the `responses.parse` method on the shared openAIClient
+ * instance instead of `vi.mock`ing the inference module: setupFiles loads
+ * the inference module (via appRouter) before any test file's vi.mock can
+ * register, so module mocks don't intercept those cached imports.
  */
 import { describe, test, expect, vi, beforeAll, beforeEach, afterEach } from "vitest";
 import { user1Caller } from "../setup";
-import { prisma } from "../../src/lib/prisma";
-import { logger } from "../../src/utils/logger";
+import { prisma } from "../../../src/lib/prisma";
+import { logger } from "../../../src/utils/logger";
+import { openAIClient } from "../../../src/utils/inference.js";
+import { pdfToWorksheet } from "../../../src/pipelines/pdfToWorksheet.js";
 
-vi.mock("../../src/utils/inference.js", () => ({
-  inference: vi.fn(),
-}));
-
-import { inference } from "../../src/utils/inference.js";
-import { pdfToWorksheet } from "../../src/pipelines/pdfToWorksheet.js";
+const mockedResponsesParse = vi.spyOn(openAIClient.responses, "parse");
 
 /** Same PDF path represented as a signed Google Cloud Storage URL. */
 export const CONV_TEST_PDF_GCS_URL =
@@ -68,7 +69,7 @@ describe("pdfToWorksheet", () => {
   });
 
   beforeEach(() => {
-    vi.mocked(inference).mockReset();
+    mockedResponsesParse.mockReset();
   });
 
   afterEach(async () => {
@@ -79,7 +80,7 @@ describe("pdfToWorksheet", () => {
 
   test("passes the PDF URL directly to inference and persists a worksheet with 8 questions", async () => {
     const draft = draftWithEightQuestions();
-    vi.mocked(inference).mockResolvedValue(draft);
+    mockedResponsesParse.mockResolvedValue({ output_parsed: draft } as any);
 
     const worksheet = await pdfToWorksheet(CONV_TEST_PDF_GCS_URL, classId);
     expect(worksheet).not.toBeNull();
@@ -87,6 +88,9 @@ describe("pdfToWorksheet", () => {
 
     expect(worksheet!.name).toBe(draft.title);
     expect(worksheet!.classId).toBe(classId);
+    expect(worksheet!.sourceUrl).toBe(
+      "https://storage.googleapis.com/studious-test-bucket/worksheets/conv-test.pdf"
+    );
 
     const questions = await prisma.worksheetQuestion.findMany({
       where: { worksheetId: worksheet!.id },
@@ -99,27 +103,17 @@ describe("pdfToWorksheet", () => {
     expect(questions.map((q) => q.order)).toEqual([1, 2, 3, 4, 5, 6, 7, 8]);
     expect(questions.map((q) => q.points)).toEqual([1, 2, 3, 4, 5, 6, 7, 8]);
 
-    expect(inference).toHaveBeenCalledTimes(1);
-    const input = vi.mocked(inference).mock.calls[0][0];
-    expect(Array.isArray(input)).toBe(true);
+    expect(mockedResponsesParse).toHaveBeenCalledTimes(1);
+    const callArgs = mockedResponsesParse.mock.calls[0][0] as { input: unknown };
     type InputPart = { type?: string; file_url?: string; text?: string };
-    const content = (input as { role: string; content: InputPart[] }[])[0].content;
+    const input = callArgs.input as { role: string; content: InputPart[] }[];
+    expect(Array.isArray(input)).toBe(true);
+    const content = input[0].content;
     const filePart = content.find((p) => p.type === "input_file");
     expect(filePart).toEqual({
       type: "input_file",
       file_url: CONV_TEST_PDF_GCS_URL,
     });
-    expect(vi.mocked(inference)).toHaveBeenCalledWith(
-      expect.any(Array),
-      expect.anything(),
-      expect.objectContaining({
-        reason: "pdf_to_worksheet_draft",
-        metadata: expect.objectContaining({
-          classId,
-          filename: "conv-test.pdf",
-        }),
-      })
-    );
   });
 
   test("rejects non-Google Cloud URLs and only logs a sanitized path", async () => {
@@ -133,23 +127,25 @@ describe("pdfToWorksheet", () => {
       "pdfToWorksheet: rejected unsupported PDF URL",
       { sanitizedUrl: "https://drive.google.com/file/d/1BfotteM60Ax0krHzE6rFN1Df8S8kT3kb/view" }
     );
-    expect(inference).not.toHaveBeenCalled();
+    expect(mockedResponsesParse).not.toHaveBeenCalled();
     warnSpy.mockRestore();
   });
 
   test("rounds persisted integer fields from model output", async () => {
-    vi.mocked(inference).mockResolvedValue({
-      title: "Rounded worksheet",
-      questions: [{
-        type: "SHORT_ANSWER",
-        question: "Rounded question",
-        answer: "Rounded answer",
-        options: [],
-        markScheme: [{ id: "step-1", points: 0.6, description: "Half step" }],
-        points: 1.7,
-        order: 1.2,
-      }],
-    } as Awaited<ReturnType<typeof inference>>);
+    mockedResponsesParse.mockResolvedValue({
+      output_parsed: {
+        title: "Rounded worksheet",
+        questions: [{
+          type: "SHORT_ANSWER",
+          question: "Rounded question",
+          answer: "Rounded answer",
+          options: [],
+          markScheme: [{ id: "step-1", points: 0.6, description: "Half step" }],
+          points: 1.7,
+          order: 1.2,
+        }],
+      },
+    } as any);
 
     const worksheet = await pdfToWorksheet(CONV_TEST_PDF_GCS_URL, classId);
     expect(worksheet).not.toBeNull();
@@ -165,12 +161,28 @@ describe("pdfToWorksheet", () => {
   });
 
   test("returns null when inference yields no parsed output", async () => {
-    vi.mocked(inference).mockResolvedValue(
-      null as unknown as Awaited<ReturnType<typeof inference>>
-    );
+    mockedResponsesParse.mockResolvedValue({ output_parsed: null } as any);
 
     const out = await pdfToWorksheet(CONV_TEST_PDF_GCS_URL, classId);
     expect(out).toBeNull();
   });
 
+  test("reuses an existing worksheet for the same PDF source", async () => {
+    mockedResponsesParse.mockResolvedValue({
+      output_parsed: draftWithEightQuestions("Idempotent worksheet"),
+    } as any);
+
+    const first = await pdfToWorksheet(CONV_TEST_PDF_GCS_URL, classId);
+    expect(first).not.toBeNull();
+    createdWorksheetIds.push(first!.id);
+
+    const second = await pdfToWorksheet(
+      "https://storage.googleapis.com/studious-test-bucket/worksheets/conv-test.pdf?X-Goog-Signature=retry-signature",
+      classId
+    );
+
+    expect(second?.id).toBe(first!.id);
+    expect(mockedResponsesParse).toHaveBeenCalledTimes(1);
+  });
+
 });

package/tests/{routers → integration/routers}/attendance.test.ts

@@ -1,7 +1,7 @@
 import { test, expect, describe, beforeEach } from 'vitest';
 import { user1Caller, user2Caller } from '../setup';
 import { expectUnauth } from '../helpers';
-import { prisma } from '../../src/lib/prisma';
+import { prisma } from '../../../src/lib/prisma';
 
 describe('Attendance Router', () => {
   let testClass: any;

package/tests/{routers → integration/routers}/auth.test.ts

@@ -3,9 +3,9 @@
 import { randomUUID } from 'crypto';
 import { SessionType } from '@prisma/client';
 import { test, expect, describe } from 'vitest';
-import { appRouter } from '../../src/routers/_app';
-import { createTRPCContext } from '../../src/trpc';
-import { prisma } from '../../src/lib/prisma';
+import { appRouter } from '../../../src/routers/_app';
+import { createTRPCContext } from '../../../src/trpc';
+import { prisma } from '../../../src/lib/prisma';
 import {
   caller,
   login1,

package/tests/{routers → integration/routers}/class.test.ts

@@ -1,10 +1,10 @@
 import { test, expect, describe, beforeEach, afterEach, vi } from 'vitest';
 import { expectForbidden, expectRejects, expectUnauth } from '../helpers';
-import { prisma } from '../../src/lib/prisma';
+import { prisma } from '../../../src/lib/prisma';
 import { caller, user1Caller, user2Caller, user3Caller } from '../setup';
 
-vi.mock('../../src/lib/googleCloudStorage.js', async (importOriginal) => {
-  const actual = await importOriginal<typeof import('../../src/lib/googleCloudStorage.js')>();
+vi.mock('../../../src/lib/googleCloudStorage.js', async (importOriginal) => {
+  const actual = await importOriginal<typeof import('../../../src/lib/googleCloudStorage.js')>();
   return {
     ...actual,
     copyFile: vi.fn().mockResolvedValue(undefined),
@@ -340,19 +340,5 @@ describe('Class Router', () => {
       expect(rootFolder.name).toBe('Class Files');
     });
 
-
-    test('non-teacher cannot import class', async () => {
-      await user1Caller.folder.getRootFolder({ classId: testClass.id });
-      const exported = await user1Caller.class.exportClass({ classId: testClass.id });
-      const newClassId = `import-forbidden-${Date.now()}`;
-
-      await expect(
-        user3Caller.class.importClass({
-          classId: newClassId,
-          year: new Date().getFullYear(),
-          classData: exported,
-        })
-      ).rejects.toThrow();
-    });
   });
 }); 

package/tests/{routers → integration/routers}/conversation.test.ts

@@ -1,5 +1,5 @@
 import { test, expect, describe, beforeAll } from 'vitest';
-import { prisma } from '../../src/lib/prisma';
+import { prisma } from '../../../src/lib/prisma';
 import { user1Caller, user2Caller, user3Caller } from '../setup';
 import { expectUnauth } from '../helpers';
 

package/tests/{routers → integration/routers}/event.test.ts

@@ -1,7 +1,7 @@
 import { test, expect, describe, beforeEach } from 'vitest';
 import { user1Caller, user2Caller } from '../setup';
 import { expectRejects, expectUnauth } from '../helpers';
-import { prisma } from '../../src/lib/prisma';
+import { prisma } from '../../../src/lib/prisma';
 import { startOfWeek } from 'date-fns';
 
 describe('Event Router', () => {

package/tests/{routers → integration/routers}/message.test.ts

@@ -1,6 +1,6 @@
 import { test, expect, describe, beforeAll } from 'vitest';
 import { user1Caller, user2Caller, user3Caller } from '../setup';
-import { prisma } from '../../src/lib/prisma';
+import { prisma } from '../../../src/lib/prisma';
 
 const expectTRPCError = async (promise: Promise<unknown>, code: string) => {
   await expect(promise).rejects.toMatchObject({ code });

package/tests/{routers → integration/routers}/newtonChat.test.ts

@@ -1,5 +1,5 @@
 import { beforeAll, describe, expect, test } from 'vitest';
-import { prisma } from '../../src/lib/prisma';
+import { prisma } from '../../../src/lib/prisma';
 import { user1Caller, user2Caller } from '../setup';
 
 describe('Newton Chat Router', () => {

package/tests/{routers → integration/routers}/studentProgress.test.ts

@@ -1,5 +1,5 @@
 import { test, expect, describe, beforeEach, vi } from "vitest";
-import { openAIClient } from "../../src/utils/inference.js";
+import { openAIClient } from "../../../src/utils/inference.js";
 import { user1Caller, user2Caller, user3Caller } from "../setup";
 
 const mockedCompletionCreate = vi.spyOn(

package/tests/{user.test.ts → integration/routers/user.test.ts}

@@ -1,6 +1,6 @@
 import { test, expect, describe, beforeEach } from 'vitest';
-import { user1Caller, user2Caller } from './setup';
-import { expectUnauth } from './helpers';
+import { user1Caller, user2Caller } from '../setup';
+import { expectUnauth } from '../helpers';
 
 describe('User Router', () => {
   describe('getProfile', () => {