@studious-lms/server 1.4.2 → 1.4.4

package/src/utils/inference.ts

@@ -2,15 +2,15 @@
  * Inference util – OpenAI client, inference() for structured output, sendAIMessage for chat.
  */
 import OpenAI from 'openai';
+import type { ResponseInput } from "openai/resources/responses/responses.js";
 import { logger } from './logger.js';
 import { prisma } from '../lib/prisma.js';
 import { chatChannel, pusher } from '../lib/pusher.js';
 import { ensureAIUserExists, getAIUserId } from './aiUser.js';
 import { env } from '../lib/config/env.js';
+import { Prisma } from '@prisma/client';
 import { ZodSchema } from 'zod';
-import { zodTextFormat } from "openai/helpers/zod";
-
-
+import { zodTextFormat } from 'openai/helpers/zod';
 
 export const inferenceClient = new OpenAI({
   apiKey: env.INFERENCE_API_KEY,
@@ -19,6 +19,38 @@ export const inferenceClient = new OpenAI({
 
 export const openAIClient = new OpenAI();
 
+const DEFAULT_INFERENCE_MODEL = 'gpt-5-nano';
+
+type InferenceMessage = {
+  role: 'user' | 'assistant' | 'system';
+  content: string;
+};
+
+export interface InferenceOptions {
+  reason?: string;
+  metadata?: Prisma.InputJsonValue;
+  model?: string;
+  temperature?: number;
+  maxTokens?: number;
+}
+
+interface InferenceUsage {
+  promptTokens?: number;
+  completionTokens?: number;
+  totalTokens?: number;
+}
+
+interface InferenceAuditLog {
+  model: string;
+  operation: 'chat.completions' | 'responses.parse';
+  success: boolean;
+  estimatedInputTokens: number;
+  reason: string;
+  metadata?: Prisma.InputJsonValue;
+  usage?: InferenceUsage;
+  error?: unknown;
+}
+
 // Types for lab chat context
 export interface LabChatContext {
   subject: string;
@@ -39,6 +71,116 @@ export interface InferenceResponse {
   finishReason: string;
 }
 
+function normalizeInferenceInput(
+  content: string | OpenAI.Chat.Completions.ChatCompletionMessageParam[],
+): InferenceMessage[] {
+  if (typeof content === 'string') {
+    return [{ role: 'user', content }];
+  }
+
+  return content.map((message) => ({
+    role: message.role as InferenceMessage['role'],
+    content: typeof message.content === 'string' ? message.content : JSON.stringify(message.content),
+  }));
+}
+
+function getInferenceReason(options?: InferenceOptions) {
+  return options?.reason?.trim() || 'unspecified';
+}
+
+function serializeInferenceError(error: unknown) {
+  if (error instanceof Error) {
+    return error.message;
+  }
+
+  if (typeof error === 'string') {
+    return error;
+  }
+
+  try {
+    return JSON.stringify(error);
+  } catch {
+    return 'Unknown inference error';
+  }
+}
+
+async function logInferenceUsage({
+  model,
+  operation,
+  success,
+  estimatedInputTokens,
+  reason,
+  metadata,
+  usage,
+  error,
+}: InferenceAuditLog) {
+  const context = {
+    reason,
+    model,
+    operation,
+    success,
+    estimatedInputTokens,
+    promptTokens: usage?.promptTokens,
+    completionTokens: usage?.completionTokens,
+    totalTokens: usage?.totalTokens,
+    metadata,
+    ...(error ? { error: serializeInferenceError(error) } : {}),
+  };
+
+  try {
+    await prisma.aiUsageLog.create({
+      data: {
+        reason,
+        model,
+        operation,
+        success,
+        estimatedInputTokens,
+        promptTokens: usage?.promptTokens,
+        completionTokens: usage?.completionTokens,
+        totalTokens: usage?.totalTokens,
+        ...(metadata !== undefined ? { metadata } : {}),
+        ...(error ? { error: serializeInferenceError(error) } : {}),
+      },
+    });
+  } catch (loggingError) {
+    logger.error('Failed to persist AI usage log', {
+      ...context,
+      loggingError: serializeInferenceError(loggingError),
+    });
+    return;
+  }
+
+  logger.info('AI usage logged', context);
+}
+
+function getChatCompletionUsage(completion: {
+  usage?: {
+    prompt_tokens?: number;
+    completion_tokens?: number;
+    total_tokens?: number;
+  };
+}): InferenceUsage {
+  return {
+    promptTokens: completion.usage?.prompt_tokens,
+    completionTokens: completion.usage?.completion_tokens,
+    totalTokens: completion.usage?.total_tokens,
+  };
+}
+
+function getResponseUsage(completion: {
+  usage?: {
+    input_tokens?: number;
+    output_tokens?: number;
+    total_tokens?: number;
+  };
+}): InferenceUsage {
+  return {
+    promptTokens: completion.usage?.input_tokens,
+    completionTokens: completion.usage?.output_tokens,
+    totalTokens: completion.usage?.total_tokens,
+  };
+}
+
 /**
  * Centralized function to send AI messages to conversations
  * Handles database storage and Pusher broadcasting
@@ -65,10 +207,8 @@ export async function sendAIMessage(
   createdAt: Date;
   meta?: Record<string, any>;
 }> {
-  // Ensure AI user exists
   await ensureAIUserExists();
 
-  // Create message in database
   const aiMessage = await prisma.message.create({
     data: {
       content,
@@ -94,17 +234,15 @@ export async function sendAIMessage(
     contentLength: content.length,
   });
 
-  // Prepare sender info
   const senderInfo = {
     id: getAIUserId(),
     username: 'Newton_AI',
     profile: {
-      displayName: "Newton AI",
+      displayName: 'Newton AI',
       profilePicture: options.customSender?.profilePicture || null,
     },
   };
 
-  // Broadcast via Pusher
   try {
     await pusher.trigger(chatChannel(conversationId), 'new-message', {
       id: aiMessage.id,
@@ -115,7 +253,7 @@ export async function sendAIMessage(
       sender: senderInfo,
       mentionedUserIds: [],
       meta: aiMessage.meta,
-      attachments: aiMessage.attachments.map(attachment => ({
+      attachments: aiMessage.attachments.map((attachment) => ({
         id: attachment.id,
         attachmentId: attachment.id,
         name: attachment.name,
@@ -138,69 +276,156 @@ export async function sendAIMessage(
   };
 }
 
+/** Responses API user parts use `type: "input_*"`; chat completions use `text` / `image_url` / etc. */
+function isResponseInputWithInputParts(
+  content: OpenAI.Chat.Completions.ChatCompletionMessageParam[] | ResponseInput
+): content is ResponseInput {
+  if (!Array.isArray(content)) return false;
+  for (const msg of content) {
+    if (!msg || typeof msg !== "object" || !("content" in msg)) continue;
+    const parts = (msg as { content?: unknown }).content;
+    if (!Array.isArray(parts)) continue;
+    for (const part of parts) {
+      if (
+        part &&
+        typeof part === "object" &&
+        "type" in part &&
+        typeof (part as { type: string }).type === "string" &&
+        (part as { type: string }).type.startsWith("input_")
+      ) {
+        return true;
+      }
+    }
+  }
+  return false;
+}
+
+function buildResponsesParseInput(
+  content: string | OpenAI.Chat.Completions.ChatCompletionMessageParam[] | ResponseInput
+): ResponseInput {
+  if (typeof content === "string") {
+    return [{ role: "user", content: [{ type: "input_text", text: content }] }];
+  }
+  if (isResponseInputWithInputParts(content)) {
+    return content;
+  }
+  return content as unknown as ResponseInput;
+}
+
 export async function inference<T>(
-  content: string | OpenAI.Chat.Completions.ChatCompletionMessageParam[],
-  format?: ZodSchema
+  content:
+    | string
+    | OpenAI.Chat.Completions.ChatCompletionMessageParam[]
+    | ResponseInput,
+  format?: ZodSchema,
+  options?: InferenceOptions,
 ): Promise<T> {
-  try {
+  const reason = getInferenceReason(options);
+  const model = options?.model ?? DEFAULT_INFERENCE_MODEL;
+  const estimatedInputTokens =
+    typeof content === 'string'
+      ? estimateTokenCount(content)
+      : isResponseInputWithInputParts(content)
+        ? estimateTokenCount(
+            content
+              .flatMap((item) =>
+                typeof (item as any)?.content === 'string'
+                  ? [(item as any).content]
+                  : Array.isArray((item as any)?.content)
+                    ? (item as any).content
+                        .filter((p: any) => p?.type === 'input_text')
+                        .map((p: any) => p?.text ?? '')
+                    : [],
+              )
+              .join('\n'),
+          )
+        : (content as OpenAI.Chat.Completions.ChatCompletionMessageParam[]).reduce(
+            (total, message) =>
+              total +
+              estimateTokenCount(
+                typeof message.content === 'string' ? message.content : '',
+              ),
+            0,
+          );
+  const operation = format ? 'responses.parse' : 'chat.completions';
 
+  try {
     if (!format) {
+      if (typeof content !== 'string' && isResponseInputWithInputParts(content)) {
+        throw new Error(
+          'normalizeInferenceInput cannot handle ResponseInput content with input_* parts when format is undefined; pass a format to use responses.parse.',
+        );
+      }
+
+      const messages = normalizeInferenceInput(
+        content as string | OpenAI.Chat.Completions.ChatCompletionMessageParam[],
+      );
       const completion = await openAIClient.chat.completions.create({
-        model: 'gpt-5-nano',
-        messages: typeof content === 'string' ? [
-          {
-            role: 'user',
-            content: content,
-          },
-        ] : content as Array<{ role: 'user' | 'assistant' | 'system'; content: string }>,
+        model,
+        messages,
+        ...(options?.temperature !== undefined
+          ? { temperature: options.temperature }
+          : {}),
+        ...(options?.maxTokens !== undefined
+          ? { max_tokens: options.maxTokens }
+          : {}),
+      });
+
+      await logInferenceUsage({
+        model,
+        operation,
+        success: true,
+        estimatedInputTokens,
+        reason,
+        metadata: options?.metadata,
+        usage: getChatCompletionUsage(completion),
       });
 
       return completion.choices[0]?.message?.content as T;
     }
 
+    const input = buildResponsesParseInput(content);
     const completion = await openAIClient.responses.parse({
-      model: 'gpt-5-nano',
-      input: typeof content === 'string' ? [
-        {
-          role: 'user',
-          content: content,
-        },
-      ] : content as Array<{ role: 'user' | 'assistant' | 'system'; content: string }>,
-      ...(format ? { text: {
-          format: zodTextFormat(format, "newton_response_format"),
-        },
-      } : {}),
-    });
-
-    console.log({
-      model: 'gpt-5-nano',
-      input: typeof content === 'string' ? [
-        {
-          role: 'user',
-          content: content,
-        },
-      ] : content as Array<{ role: 'user' | 'assistant' | 'system'; content: string }>,
-      ...(format ? { text: {
-          format: zodTextFormat(format, "newton_response_format"),
-        },
-      } : {}),
+      model,
+      input,
+      text: {
+        format: zodTextFormat(format, 'newton_response_format'),
+      },
     });
 
-
     if (!completion) {
       throw new Error('No response generated from inference API');
     }
 
-    // if (format) {
-    //   if (typeof completion.output === 'string') {
-    //     return JSON.parse(completion.output);
-    //   }
-    //   return JSON.parse(completion.output);
-    // }
+    await logInferenceUsage({
+      model,
+      operation,
+      success: true,
+      estimatedInputTokens,
+      reason,
+      metadata: options?.metadata,
+      usage: getResponseUsage(completion),
+    });
 
-    return completion.output_parsed;
+    return completion.output_parsed as T;
   } catch (error) {
-    logger.error('Failed to generate inference response', { error });
+    await logInferenceUsage({
+      model,
+      operation,
+      success: false,
+      estimatedInputTokens,
+      reason,
+      metadata: options?.metadata,
+      error,
+    });
+
+    logger.error('Failed to generate inference response', {
+      error,
+      reason,
+      model,
+      operation,
+      metadata: options?.metadata,
+    });
     throw error;
   }
 }
@@ -217,7 +442,7 @@ export async function getAvailableModels(): Promise<string[]> {
       .sort();
   } catch (error) {
     logger.error('Failed to fetch inference models', { error });
-    return ['command-r-plus', 'command-r', 'command-light']; // Fallback Cohere models
+    return ['command-r-plus', 'command-r', 'command-light'];
   }
 }
 
@@ -225,6 +450,5 @@ export async function getAvailableModels(): Promise<string[]> {
  * Estimate token count for a message (rough approximation)
  */
 export function estimateTokenCount(text: string): number {
-  // Rough approximation: 1 token ≈ 4 characters for English text
   return Math.ceil(text.length / 4);
-}
+}

package/tests/globalSetup.ts

@@ -9,6 +9,7 @@ import { appRouter } from '../src/routers/_app';
 import { createTRPCContext } from '../src/trpc';
 import { clearDatabase } from '../src/seedDatabase';
 import { execSync } from 'child_process';
+import { SessionType } from '@prisma/client';
 
 async function runSetup() {
   const ctx = await createTRPCContext({
@@ -40,9 +41,18 @@ async function runSetup() {
     confirmPassword: 'password_is_1234',
   });
 
-  const session1 = await prisma.session.findFirst({ where: { userId: user1.user.id } });
-  const session2 = await prisma.session.findFirst({ where: { userId: user2.user.id } });
-  const session3 = await prisma.session.findFirst({ where: { userId: user3.user.id } });
+  const session1 = await prisma.session.findFirst({
+    where: { userId: user1.user.id, type: SessionType.EMAIL_VERIFICATION },
+    orderBy: { createdAt: 'desc' },
+  });
+  const session2 = await prisma.session.findFirst({
+    where: { userId: user2.user.id, type: SessionType.EMAIL_VERIFICATION },
+    orderBy: { createdAt: 'desc' },
+  });
+  const session3 = await prisma.session.findFirst({
+    where: { userId: user3.user.id, type: SessionType.EMAIL_VERIFICATION },
+    orderBy: { createdAt: 'desc' },
+  });
 
   if (!session1 || !session2 || !session3) {
     throw new Error('Failed to create sessions for test users');
@@ -59,4 +69,3 @@ export default async function setup(project: TestProject) {
   project.onTestsRerun(runSetup);
   await runSetup();
 }
-

package/tests/pipelines/pdfToWorksheet.test.ts

@@ -0,0 +1,176 @@
+/**
+ * `pdfToWorksheet` integration-style tests: real DB, mocked inference.
+ * CI does not call the model, and the PDF URL is passed through directly.
+ */
+import { describe, test, expect, vi, beforeAll, beforeEach, afterEach } from "vitest";
+import { user1Caller } from "../setup";
+import { prisma } from "../../src/lib/prisma";
+import { logger } from "../../src/utils/logger";
+
+vi.mock("../../src/utils/inference.js", () => ({
+  inference: vi.fn(),
+}));
+
+import { inference } from "../../src/utils/inference.js";
+import { pdfToWorksheet } from "../../src/pipelines/pdfToWorksheet.js";
+
+/** Same PDF path represented as a signed Google Cloud Storage URL. */
+export const CONV_TEST_PDF_GCS_URL =
+  "https://storage.googleapis.com/studious-test-bucket/worksheets/conv-test.pdf?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Signature=test-signature";
+
+export const NON_GCS_PDF_URL =
+  "https://drive.google.com/file/d/1BfotteM60Ax0krHzE6rFN1Df8S8kT3kb/view?usp=drive_link";
+
+/** Mirrors the 8-question conv-test worksheet shape the model is expected to return. */
+function draftWithEightQuestions(title = "Conv test worksheet") {
+  const types = [
+    "SHORT_ANSWER",
+    "MULTIPLE_CHOICE",
+    "TRUE_FALSE",
+    "SHORT_ANSWER",
+    "LONG_ANSWER",
+    "MATH_EXPRESSION",
+    "ESSAY",
+    "SHORT_ANSWER",
+  ] as const;
+
+  return {
+    title,
+    questions: types.map((type, i) => ({
+      type,
+      question: `Question ${i + 1}`,
+      answer: `Answer ${i + 1}`,
+      options:
+        type === "MULTIPLE_CHOICE"
+          ? [
+              { id: "a", text: "Wrong", isCorrect: false },
+              { id: "b", text: "Right", isCorrect: true },
+            ]
+          : [],
+      markScheme: [] as { id: string; points: number; description: string }[],
+      points: i + 1,
+      order: i + 1,
+    })),
+  };
+}
+
+describe("pdfToWorksheet", () => {
+  let classId: string;
+  const createdWorksheetIds: string[] = [];
+
+  beforeAll(async () => {
+    const cls = await user1Caller.class.create({
+      name: "pdfToWorksheet pipeline class",
+      subject: "Science",
+      section: "Test",
+    });
+    classId = cls.id;
+  });
+
+  beforeEach(() => {
+    vi.mocked(inference).mockReset();
+  });
+
+  afterEach(async () => {
+    for (const id of createdWorksheetIds.splice(0)) {
+      await prisma.worksheet.delete({ where: { id } }).catch(() => undefined);
+    }
+  });
+
+  test("passes the PDF URL directly to inference and persists a worksheet with 8 questions", async () => {
+    const draft = draftWithEightQuestions();
+    vi.mocked(inference).mockResolvedValue(draft);
+
+    const worksheet = await pdfToWorksheet(CONV_TEST_PDF_GCS_URL, classId);
+    expect(worksheet).not.toBeNull();
+    createdWorksheetIds.push(worksheet!.id);
+
+    expect(worksheet!.name).toBe(draft.title);
+    expect(worksheet!.classId).toBe(classId);
+
+    const questions = await prisma.worksheetQuestion.findMany({
+      where: { worksheetId: worksheet!.id },
+      orderBy: { order: "asc" },
+    });
+    expect(questions).toHaveLength(8);
+    expect(questions.map((q) => q.question)).toEqual(
+      draft.questions.map((q) => q.question)
+    );
+    expect(questions.map((q) => q.order)).toEqual([1, 2, 3, 4, 5, 6, 7, 8]);
+    expect(questions.map((q) => q.points)).toEqual([1, 2, 3, 4, 5, 6, 7, 8]);
+
+    expect(inference).toHaveBeenCalledTimes(1);
+    const input = vi.mocked(inference).mock.calls[0][0];
+    expect(Array.isArray(input)).toBe(true);
+    type InputPart = { type?: string; file_url?: string; text?: string };
+    const content = (input as { role: string; content: InputPart[] }[])[0].content;
+    const filePart = content.find((p) => p.type === "input_file");
+    expect(filePart).toEqual({
+      type: "input_file",
+      file_url: CONV_TEST_PDF_GCS_URL,
+    });
+    expect(vi.mocked(inference)).toHaveBeenCalledWith(
+      expect.any(Array),
+      expect.anything(),
+      expect.objectContaining({
+        reason: "pdf_to_worksheet_draft",
+        metadata: expect.objectContaining({
+          classId,
+          filename: "conv-test.pdf",
+        }),
+      })
+    );
+  });
+
+  test("rejects non-Google Cloud URLs and only logs a sanitized path", async () => {
+    const warnSpy = vi.spyOn(logger, "warn").mockImplementation(() => undefined);
+
+    await expect(pdfToWorksheet(NON_GCS_PDF_URL, classId)).rejects.toThrow(
+      "Only Google Cloud Storage URLs are supported"
+    );
+
+    expect(warnSpy).toHaveBeenCalledWith(
+      "pdfToWorksheet: rejected unsupported PDF URL",
+      { sanitizedUrl: "https://drive.google.com/file/d/1BfotteM60Ax0krHzE6rFN1Df8S8kT3kb/view" }
+    );
+    expect(inference).not.toHaveBeenCalled();
+    warnSpy.mockRestore();
+  });
+
+  test("rounds persisted integer fields from model output", async () => {
+    vi.mocked(inference).mockResolvedValue({
+      title: "Rounded worksheet",
+      questions: [{
+        type: "SHORT_ANSWER",
+        question: "Rounded question",
+        answer: "Rounded answer",
+        options: [],
+        markScheme: [{ id: "step-1", points: 0.6, description: "Half step" }],
+        points: 1.7,
+        order: 1.2,
+      }],
+    } as Awaited<ReturnType<typeof inference>>);
+
+    const worksheet = await pdfToWorksheet(CONV_TEST_PDF_GCS_URL, classId);
+    expect(worksheet).not.toBeNull();
+    createdWorksheetIds.push(worksheet!.id);
+
+    const [question] = await prisma.worksheetQuestion.findMany({
+      where: { worksheetId: worksheet!.id },
+    });
+
+    expect(question.points).toBe(2);
+    expect(question.order).toBe(1);
+    expect(question.markScheme).toEqual([{ id: "step-1", points: 1, description: "Half step" }]);
+  });
+
+  test("returns null when inference yields no parsed output", async () => {
+    vi.mocked(inference).mockResolvedValue(
+      null as unknown as Awaited<ReturnType<typeof inference>>
+    );
+
+    const out = await pdfToWorksheet(CONV_TEST_PDF_GCS_URL, classId);
+    expect(out).toBeNull();
+  });
+
+});