@studious-lms/server 1.4.9 → 1.5.0

package/src/utils/generateUsername.ts

@@ -0,0 +1,108 @@
+/**
+ * Username generator – produces playful adjective+noun+number combos like "greenegg6767".
+ * Pass an `isTaken` checker to retry until a unique username is found.
+ */
+
+const ADJECTIVES = [
+  "green",
+  "blue",
+  "red",
+  "purple",
+  "yellow",
+  "orange",
+  "pink",
+  "silver",
+  "golden",
+  "fuzzy",
+  "happy",
+  "sleepy",
+  "sneaky",
+  "silly",
+  "brave",
+  "clever",
+  "lucky",
+  "mighty",
+  "quiet",
+  "quick",
+  "loud",
+  "tiny",
+  "giant",
+  "spicy",
+  "jolly",
+  "wild",
+  "shiny",
+  "cosmic",
+  "mystic",
+  "swift",
+  "lazy",
+  "grumpy",
+  "fluffy",
+  "zesty",
+];
+
+const NOUNS = [
+  "egg",
+  "cat",
+  "dog",
+  "fox",
+  "owl",
+  "bear",
+  "duck",
+  "frog",
+  "wolf",
+  "panda",
+  "tiger",
+  "otter",
+  "moose",
+  "shark",
+  "whale",
+  "raven",
+  "robin",
+  "koala",
+  "llama",
+  "sloth",
+  "narwhal",
+  "penguin",
+  "dragon",
+  "phoenix",
+  "muffin",
+  "pickle",
+  "noodle",
+  "potato",
+  "taco",
+  "pancake",
+  "waffle",
+  "cookie",
+  "biscuit",
+  "donut",
+];
+
+function pick<T>(items: readonly T[]) {
+  return items[Math.floor(Math.random() * items.length)]!;
+}
+
+function randomDigits(n: number) {
+  let out = "";
+  for (let i = 0; i < n; i += 1) {
+    out += Math.floor(Math.random() * 10).toString();
+  }
+  return out;
+}
+
+export function generateRandomUsername() {
+  return `${pick(ADJECTIVES)}${pick(NOUNS)}${randomDigits(4)}`;
+}
+
+/** Generate a unique username, calling `isTaken` until one is free. */
+export async function generateUniqueUsername(
+  isTaken: (username: string) => Promise<boolean>,
+  maxAttempts = 10
+) {
+  for (let attempt = 0; attempt < maxAttempts; attempt += 1) {
+    const candidate = generateRandomUsername();
+    if (!(await isTaken(candidate))) {
+      return candidate;
+    }
+  }
+  return `${generateRandomUsername()}${randomDigits(4)}`;
+}

package/src/workers/connection.ts

@@ -0,0 +1,37 @@
+import IORedis from 'ioredis';
+import { env } from '../lib/config/env.js';
+import { logger } from '../utils/logger.js';
+
+/**
+ * BullMQ-specific Redis connection.
+ *
+ * Kept separate from `src/lib/redis.ts` (which is used for caching and
+ * rate-limiting) because BullMQ requires `maxRetriesPerRequest: null` and
+ * `enableReadyCheck: false` on its connection. Mixing those settings with
+ * general Redis usage causes unwanted behaviour on transient errors.
+ */
+function buildBullConnection(): IORedis {
+  if (!env.REDIS_URL) {
+    throw new Error(
+      'REDIS_URL is required for BullMQ. Set it in .env / .env.production / your hosting env.',
+    );
+  }
+
+  const connection = new IORedis(env.REDIS_URL, {
+    maxRetriesPerRequest: null,
+    enableReadyCheck: false,
+    lazyConnect: false,
+  });
+
+  connection.on('error', (err) => {
+    logger.error('BullMQ Redis connection error', { error: err.message });
+  });
+
+  connection.on('connect', () => {
+    logger.info('BullMQ Redis connected');
+  });
+
+  return connection;
+}
+
+export const bullConnection = buildBullConnection();

package/src/workers/index.ts

@@ -0,0 +1,61 @@
+/**
+ * Worker process entrypoint.
+ *
+ * Run separately from the HTTP server (e.g. `node dist/workers/index.js` on
+ * Render/Railway/Fly). Importing this file boots every BullMQ Worker
+ * declared under `src/workers/processors/*`. Do NOT import this from the
+ * HTTP server — that would spin up workers in your serverless functions.
+ */
+import '../instrument.js';
+
+import { Worker } from 'bullmq';
+import { prisma } from '../lib/prisma.js';
+import { logger } from '../utils/logger.js';
+import { bullConnection } from './connection.js';
+import { pdfToWorksheetWorker } from './processors/pdfToWorksheet.js';
+
+const workers: Worker<any>[] = [pdfToWorksheetWorker];
+
+logger.info('Workers ready', {
+  pid: process.pid,
+  queues: workers.map((w) => w.name),
+});
+
+let shuttingDown = false;
+
+async function shutdown(signal: string): Promise<void> {
+  if (shuttingDown) return;
+  shuttingDown = true;
+
+  logger.info(`Received ${signal}, draining workers...`);
+
+  // `worker.close()` waits for in-flight jobs to finish (up to the worker's
+  // settings) before resolving. Run them in parallel.
+  await Promise.allSettled(workers.map((w) => w.close()));
+
+  await Promise.allSettled([
+    bullConnection.quit(),
+    prisma.$disconnect(),
+  ]);
+
+  logger.info('Workers shut down cleanly');
+  process.exit(0);
+}
+
+process.on('SIGTERM', () => void shutdown('SIGTERM'));
+process.on('SIGINT', () => void shutdown('SIGINT'));
+
+process.on('uncaughtException', (error) => {
+  logger.error('Uncaught exception in worker process', {
+    error: error.message,
+    stack: error.stack,
+  });
+  process.exit(1);
+});
+
+process.on('unhandledRejection', (reason) => {
+  logger.error('Unhandled rejection in worker process', {
+    reason: reason instanceof Error ? reason.message : String(reason),
+  });
+  process.exit(1);
+});

package/src/workers/processors/pdfToWorksheet.ts

@@ -0,0 +1,119 @@
+import { Worker, type Job } from 'bullmq';
+import { Prisma } from '@prisma/client';
+import { bullConnection } from '../connection.js';
+import { prisma } from '../../lib/prisma.js';
+import { logger } from '../../utils/logger.js';
+import { pdfToWorksheet } from '../../pipelines/pdfToWorksheet.js';
+
+interface PdfToWorksheetJobData {
+  runJobId: string;
+  gcsUrl: string;
+  classId: string;
+}
+
+async function process(job: Job<PdfToWorksheetJobData>) {
+  const { runJobId, gcsUrl, classId } = job.data;
+
+  // If a previous attempt already succeeded for this RunJob (rare race after a
+  // crash that lost the BullMQ ack), return the prior result instead of
+  // re-running the AI call and creating a duplicate worksheet.
+  const existing = await prisma.runJob.findUnique({
+    where: { id: runJobId },
+    select: { status: true, result: true },
+  });
+  if (existing?.status === 'SUCCEEDED') {
+    logger.info('pdfToWorksheet RunJob already succeeded; skipping re-run', {
+      runJobId,
+      bullJobId: job.id,
+    });
+    return existing.result;
+  }
+
+  await prisma.runJob.update({
+    where: { id: runJobId },
+    data: {
+      status: 'RUNNING',
+      startedAt: existing?.status === 'RUNNING' ? undefined : new Date(),
+      attempts: { increment: 1 },
+    },
+  });
+
+  const worksheet = await pdfToWorksheet(gcsUrl, classId);
+
+  const result = worksheet ? { worksheetId: worksheet.id, name: worksheet.name } : null;
+
+  await prisma.runJob.update({
+    where: { id: runJobId },
+    data: {
+      status: 'SUCCEEDED',
+      finishedAt: new Date(),
+      error: null,
+      result: result === null ? Prisma.JsonNull : (result as Prisma.InputJsonValue),
+    },
+  });
+
+  return result;
+}
+
+export const pdfToWorksheetWorker = new Worker<PdfToWorksheetJobData>(
+  'pdfToWorksheet',
+  process,
+  {
+    connection: bullConnection,
+    // Conservative defaults — tune once we have real-world numbers.
+    concurrency: 2,
+    limiter: {
+      max: 5,
+      duration: 60_000, // 5 jobs / minute, e.g. to stay under OpenAI rate limits
+    },
+  },
+);
+
+pdfToWorksheetWorker.on('error', (err) => {
+  logger.error('pdfToWorksheet worker error', {
+    error: err.message,
+    stack: err.stack,
+  });
+});
+
+pdfToWorksheetWorker.on('failed', async (job, err) => {
+  if (!job) return;
+
+  const runJobId = job.data?.runJobId;
+  const isFinalAttempt = job.attemptsMade >= (job.opts.attempts ?? 1);
+
+  logger.error('pdfToWorksheet job failed', {
+    bullJobId: job.id,
+    runJobId,
+    attempt: job.attemptsMade,
+    finalAttempt: isFinalAttempt,
+    error: err.message,
+  });
+
+  if (!runJobId) return;
+
+  // Always record the latest error message on the RunJob row.
+  // Only flip to FAILED on the final attempt — earlier failures are still
+  // "RUNNING" because BullMQ is going to retry.
+  await prisma.runJob.update({
+    where: { id: runJobId },
+    data: {
+      error: err.message,
+      ...(isFinalAttempt
+        ? { status: 'FAILED', finishedAt: new Date() }
+        : {}),
+    },
+  }).catch((updateError) => {
+    logger.error('Failed to record pdfToWorksheet failure on RunJob', {
+      runJobId,
+      error: updateError instanceof Error ? updateError.message : String(updateError),
+    });
+  });
+});
+
+pdfToWorksheetWorker.on('completed', (job) => {
+  logger.info('pdfToWorksheet job completed', {
+    bullJobId: job.id,
+    runJobId: job.data?.runJobId,
+  });
+});

package/src/workers/queues.ts

@@ -0,0 +1,159 @@
+/**
+ * BullMQ queue producers and the `enqueueRunJob` helper.
+ *
+ * Producer side only — safe to import from HTTP routes / services running on
+ * Vercel. Importing this file does NOT start any workers; workers live in
+ * `src/workers/processors/*` and are wired up by `src/workers/index.ts`,
+ * which is the entrypoint of the separate long-running worker process.
+ */
+import { Queue, type JobsOptions } from 'bullmq';
+import { Prisma, type RunJobType } from '@prisma/client';
+import { bullConnection } from './connection.js';
+import { prisma } from '../lib/prisma.js';
+import { logger } from '../utils/logger.js';
+
+const DEFAULT_JOB_OPTIONS: JobsOptions = {
+  attempts: 3,
+  backoff: { type: 'exponential', delay: 5_000 },
+  removeOnComplete: { age: 60 * 60 }, // keep completed jobs in Redis for 1h, then drop
+  removeOnFail: { age: 24 * 60 * 60 }, // keep failed jobs for 24h for inspection
+};
+
+/**
+ * Map RunJobType → (queue name, input payload shape).
+ *
+ * Add a new entry here whenever you introduce a new background job type.
+ * The queue name string must match what the corresponding processor passes
+ * to `new Worker(<name>, ...)`.
+ */
+type JobInputs = {
+  PDF_TO_WORKSHEET: { gcsUrl: string; classId: string };
+  GRADE_WORKSHEET: { worksheetResponseId: string };
+  AI_LAB_CHAT: { conversationId: string; messageId: string };
+  AI_NEWTON_CHAT: { conversationId: string; messageId: string };
+  STUDENT_PROGRESS_CHAT: { classId: string; studentId: string; message: string };
+};
+
+const QUEUE_NAMES: Record<RunJobType, string> = {
+  PDF_TO_WORKSHEET: 'pdfToWorksheet',
+  GRADE_WORKSHEET: 'gradeWorksheet',
+  AI_LAB_CHAT: 'aiLabChat',
+  AI_NEWTON_CHAT: 'aiNewtonChat',
+  STUDENT_PROGRESS_CHAT: 'studentProgressChat',
+};
+
+/**
+ * One Queue instance per type. Created lazily so that importing this module
+ * does not eagerly open queues you may never use in a given runtime.
+ */
+const queueCache = new Map<RunJobType, Queue>();
+
+function getQueue(type: RunJobType): Queue {
+  let queue = queueCache.get(type);
+  if (!queue) {
+    queue = new Queue(QUEUE_NAMES[type], {
+      connection: bullConnection,
+      defaultJobOptions: DEFAULT_JOB_OPTIONS,
+    });
+    queueCache.set(type, queue);
+  }
+  return queue;
+}
+
+export interface EnqueueOptions {
+  classId?: string | null;
+  userId?: string | null;
+  /** Override BullMQ job options (attempts, delay, priority, etc.). */
+  jobOptions?: JobsOptions;
+}
+
+export interface EnqueueResult {
+  runJobId: string;
+  bullJobId: string | null;
+}
+
+/**
+ * Create a `RunJob` audit row, enqueue the BullMQ job, and link the two.
+ *
+ * Returns immediately — the actual work happens on the worker process. The
+ * caller can return `runJobId` to the client and have them poll `getRunJob`
+ * (or subscribe over Pusher) to track progress.
+ *
+ * IMPORTANT: do not call this inside a Prisma `$transaction`. If the outer
+ * transaction rolls back after the BullMQ job is added, the worker will pick
+ * up an orphaned job pointing at a non-existent RunJob row. Enqueue *after*
+ * the transaction commits.
+ */
+export async function enqueueRunJob<T extends RunJobType>(
+  type: T,
+  input: JobInputs[T],
+  options: EnqueueOptions = {},
+): Promise<EnqueueResult> {
+  const runJob = await prisma.runJob.create({
+    data: {
+      type,
+      status: 'PENDING',
+      classId: options.classId ?? null,
+      userId: options.userId ?? null,
+      input: input as unknown as Prisma.InputJsonValue,
+    },
+  });
+
+  try {
+    const queue = getQueue(type);
+    const job = await queue.add(
+      QUEUE_NAMES[type],
+      { runJobId: runJob.id, ...input },
+      options.jobOptions,
+    );
+
+    try {
+      await prisma.runJob.update({
+        where: { id: runJob.id },
+        data: { bullJobId: job.id ?? null },
+      });
+    } catch (linkError) {
+      logger.error('RunJob enqueued but bullJobId linkage failed', {
+        runJobId: runJob.id,
+        bullJobId: job.id ?? null,
+        type,
+        error: linkError instanceof Error ? linkError.message : String(linkError),
+      });
+    }
+
+    return { runJobId: runJob.id, bullJobId: job.id ?? null };
+  } catch (error) {
+    logger.error('Failed to enqueue RunJob; marking FAILED', {
+      runJobId: runJob.id,
+      type,
+      error: error instanceof Error ? error.message : String(error),
+    });
+
+    try {
+      await prisma.runJob.update({
+        where: { id: runJob.id },
+        data: {
+          status: 'FAILED',
+          error: error instanceof Error ? error.message : String(error),
+          finishedAt: new Date(),
+        },
+      });
+    } catch (updateError) {
+      logger.error('Failed to mark RunJob as FAILED after enqueue error', {
+        runJobId: runJob.id,
+        error: updateError instanceof Error ? updateError.message : String(updateError),
+      });
+    }
+
+    throw error;
+  }
+}
+
+/**
+ * Close every cached queue. Call from the HTTP server's shutdown handler so
+ * the producer side disconnects from Redis cleanly.
+ */
+export async function closeAllQueues(): Promise<void> {
+  await Promise.all(Array.from(queueCache.values()).map((q) => q.close()));
+  queueCache.clear();
+}

package/tests/{globalSetup.ts → integration/globalSetup.ts}

@@ -4,10 +4,10 @@ import type { TestProject } from 'vitest/node';
 
 config({ path: resolve(process.cwd(), '.env.test'), override: true });
 
-import { prisma } from '../src/lib/prisma';
-import { appRouter } from '../src/routers/_app';
-import { createTRPCContext } from '../src/trpc';
-import { clearDatabase } from '../src/seedDatabase';
+import { prisma } from '../../src/lib/prisma';
+import { appRouter } from '../../src/routers/_app';
+import { createTRPCContext } from '../../src/trpc';
+import { clearDatabase } from '../../src/seedDatabase';
 import { execSync } from 'child_process';
 import { SessionType } from '@prisma/client';
 

package/tests/{helpers.ts → integration/helpers.ts}

@@ -1,5 +1,5 @@
-import { createTRPCContext } from '../src/trpc';
-import { appRouter } from '../src/routers/_app';
+import { createTRPCContext } from '../../src/trpc';
+import { appRouter } from '../../src/routers/_app';
 
 export async function createUnauthCaller() {
   const ctx = await createTRPCContext({

package/tests/{pipelines → integration/pipelines}/pdfToWorksheet.test.ts

@@ -1,18 +1,19 @@
 /**
- * `pdfToWorksheet` integration-style tests: real DB, mocked inference.
- * CI does not call the model, and the PDF URL is passed through directly.
+ * `pdfToWorksheet` integration-style tests: real DB, mocked OpenAI.
+ *
+ * We intercept the `responses.parse` method on the shared openAIClient
+ * instance instead of `vi.mock`ing the inference module: setupFiles loads
+ * the inference module (via appRouter) before any test file's vi.mock can
+ * register, so module mocks don't intercept those cached imports.
  */
 import { describe, test, expect, vi, beforeAll, beforeEach, afterEach } from "vitest";
 import { user1Caller } from "../setup";
-import { prisma } from "../../src/lib/prisma";
-import { logger } from "../../src/utils/logger";
+import { prisma } from "../../../src/lib/prisma";
+import { logger } from "../../../src/utils/logger";
+import { openAIClient } from "../../../src/utils/inference.js";
+import { pdfToWorksheet } from "../../../src/pipelines/pdfToWorksheet.js";
 
-vi.mock("../../src/utils/inference.js", () => ({
-  inference: vi.fn(),
-}));
-
-import { inference } from "../../src/utils/inference.js";
-import { pdfToWorksheet } from "../../src/pipelines/pdfToWorksheet.js";
+const mockedResponsesParse = vi.spyOn(openAIClient.responses, "parse");
 
 /** Same PDF path represented as a signed Google Cloud Storage URL. */
 export const CONV_TEST_PDF_GCS_URL =
@@ -68,7 +69,7 @@ describe("pdfToWorksheet", () => {
   });
 
   beforeEach(() => {
-    vi.mocked(inference).mockReset();
+    mockedResponsesParse.mockReset();
   });
 
   afterEach(async () => {
@@ -79,7 +80,7 @@ describe("pdfToWorksheet", () => {
 
   test("passes the PDF URL directly to inference and persists a worksheet with 8 questions", async () => {
     const draft = draftWithEightQuestions();
-    vi.mocked(inference).mockResolvedValue(draft);
+    mockedResponsesParse.mockResolvedValue({ output_parsed: draft } as any);
 
     const worksheet = await pdfToWorksheet(CONV_TEST_PDF_GCS_URL, classId);
     expect(worksheet).not.toBeNull();
@@ -87,6 +88,9 @@ describe("pdfToWorksheet", () => {
 
     expect(worksheet!.name).toBe(draft.title);
     expect(worksheet!.classId).toBe(classId);
+    expect(worksheet!.sourceUrl).toBe(
+      "https://storage.googleapis.com/studious-test-bucket/worksheets/conv-test.pdf"
+    );
 
     const questions = await prisma.worksheetQuestion.findMany({
       where: { worksheetId: worksheet!.id },
@@ -99,27 +103,17 @@ describe("pdfToWorksheet", () => {
     expect(questions.map((q) => q.order)).toEqual([1, 2, 3, 4, 5, 6, 7, 8]);
     expect(questions.map((q) => q.points)).toEqual([1, 2, 3, 4, 5, 6, 7, 8]);
 
-    expect(inference).toHaveBeenCalledTimes(1);
-    const input = vi.mocked(inference).mock.calls[0][0];
-    expect(Array.isArray(input)).toBe(true);
+    expect(mockedResponsesParse).toHaveBeenCalledTimes(1);
+    const callArgs = mockedResponsesParse.mock.calls[0][0] as { input: unknown };
     type InputPart = { type?: string; file_url?: string; text?: string };
-    const content = (input as { role: string; content: InputPart[] }[])[0].content;
+    const input = callArgs.input as { role: string; content: InputPart[] }[];
+    expect(Array.isArray(input)).toBe(true);
+    const content = input[0].content;
     const filePart = content.find((p) => p.type === "input_file");
     expect(filePart).toEqual({
       type: "input_file",
       file_url: CONV_TEST_PDF_GCS_URL,
     });
-    expect(vi.mocked(inference)).toHaveBeenCalledWith(
-      expect.any(Array),
-      expect.anything(),
-      expect.objectContaining({
-        reason: "pdf_to_worksheet_draft",
-        metadata: expect.objectContaining({
-          classId,
-          filename: "conv-test.pdf",
-        }),
-      })
-    );
   });
 
   test("rejects non-Google Cloud URLs and only logs a sanitized path", async () => {
@@ -133,23 +127,25 @@ describe("pdfToWorksheet", () => {
       "pdfToWorksheet: rejected unsupported PDF URL",
       { sanitizedUrl: "https://drive.google.com/file/d/1BfotteM60Ax0krHzE6rFN1Df8S8kT3kb/view" }
     );
-    expect(inference).not.toHaveBeenCalled();
+    expect(mockedResponsesParse).not.toHaveBeenCalled();
     warnSpy.mockRestore();
   });
 
   test("rounds persisted integer fields from model output", async () => {
-    vi.mocked(inference).mockResolvedValue({
-      title: "Rounded worksheet",
-      questions: [{
-        type: "SHORT_ANSWER",
-        question: "Rounded question",
-        answer: "Rounded answer",
-        options: [],
-        markScheme: [{ id: "step-1", points: 0.6, description: "Half step" }],
-        points: 1.7,
-        order: 1.2,
-      }],
-    } as Awaited<ReturnType<typeof inference>>);
+    mockedResponsesParse.mockResolvedValue({
+      output_parsed: {
+        title: "Rounded worksheet",
+        questions: [{
+          type: "SHORT_ANSWER",
+          question: "Rounded question",
+          answer: "Rounded answer",
+          options: [],
+          markScheme: [{ id: "step-1", points: 0.6, description: "Half step" }],
+          points: 1.7,
+          order: 1.2,
+        }],
+      },
+    } as any);
 
     const worksheet = await pdfToWorksheet(CONV_TEST_PDF_GCS_URL, classId);
     expect(worksheet).not.toBeNull();
@@ -165,12 +161,28 @@ describe("pdfToWorksheet", () => {
   });
 
   test("returns null when inference yields no parsed output", async () => {
-    vi.mocked(inference).mockResolvedValue(
-      null as unknown as Awaited<ReturnType<typeof inference>>
-    );
+    mockedResponsesParse.mockResolvedValue({ output_parsed: null } as any);
 
     const out = await pdfToWorksheet(CONV_TEST_PDF_GCS_URL, classId);
     expect(out).toBeNull();
   });
 
+  test("reuses an existing worksheet for the same PDF source", async () => {
+    mockedResponsesParse.mockResolvedValue({
+      output_parsed: draftWithEightQuestions("Idempotent worksheet"),
+    } as any);
+
+    const first = await pdfToWorksheet(CONV_TEST_PDF_GCS_URL, classId);
+    expect(first).not.toBeNull();
+    createdWorksheetIds.push(first!.id);
+
+    const second = await pdfToWorksheet(
+      "https://storage.googleapis.com/studious-test-bucket/worksheets/conv-test.pdf?X-Goog-Signature=retry-signature",
+      classId
+    );
+
+    expect(second?.id).toBe(first!.id);
+    expect(mockedResponsesParse).toHaveBeenCalledTimes(1);
+  });
+
 });

package/tests/{routers → integration/routers}/attendance.test.ts

@@ -1,7 +1,7 @@
 import { test, expect, describe, beforeEach } from 'vitest';
 import { user1Caller, user2Caller } from '../setup';
 import { expectUnauth } from '../helpers';
-import { prisma } from '../../src/lib/prisma';
+import { prisma } from '../../../src/lib/prisma';
 
 describe('Attendance Router', () => {
   let testClass: any;