palaryn 0.4.8 → 0.4.10

package/src/saas/routes.ts

@@ -17,6 +17,7 @@ import { PlanTier } from '../types/subscription';
 import { MODEL_PRICING, resolveModelPricing } from '../budget/model-pricing';
 import { PolicyEngine } from '../policy/engine';
 import { ToolCall } from '../types/tool-call';
+import { detectLlmProvider, llmFetchWithFallback, LlmProvider } from '../llm-utils';
 
 /** Strip HTML tags from user-provided display strings to prevent stored XSS. */
 function stripHtmlTags(input: string): string {
@@ -1239,18 +1240,9 @@ Input: "Allow GET to example.com, block DELETE everywhere"
 Output: [{"name":"allow-get-example","description":"Allow GET requests to example.com","effect":"ALLOW","priority":10,"conditions":{"methods":["GET"],"domains":["example.com"]}},{"name":"block-delete-all","description":"Block all DELETE operations","effect":"DENY","priority":5,"conditions":{"capabilities":["delete"],"methods":["DELETE"]}}]`;
 
     try {
-      const controller = new AbortController();
-      const timeout = setTimeout(() => controller.abort(), 15000);
-
-      // Detect provider from API key prefix
-      const isOpenAI = apiKey.startsWith('sk-proj-') || apiKey.startsWith('sk-');
-      const llmUrl = isOpenAI
-        ? 'https://api.openai.com/v1/chat/completions'
-        : 'https://api.anthropic.com/v1/messages';
-      const llmHeaders: Record<string, string> = isOpenAI
-        ? { 'Content-Type': 'application/json', 'Authorization': `Bearer ${apiKey}` }
-        : { 'Content-Type': 'application/json', 'x-api-key': apiKey, 'anthropic-version': '2023-06-01' };
-      const llmBody = isOpenAI
+      const fallbackKey = process.env.PALARYN_LLM_FALLBACK_KEY;
+
+      const buildBody = (provider: LlmProvider) => provider.isOpenAI
         ? JSON.stringify({
             model: 'gpt-4.1-mini',
             max_tokens: 1024,
@@ -1266,24 +1258,20 @@ Output: [{"name":"allow-get-example","description":"Allow GET requests to exampl
             messages: [{ role: 'user', content: trimmed }],
           });
 
-      const llmRes = await fetch(llmUrl, {
-        method: 'POST',
-        headers: llmHeaders,
-        body: llmBody,
-        signal: controller.signal,
+      const { response: llmRes, provider, usedFallback } = await llmFetchWithFallback({
+        primaryKey: apiKey,
+        fallbackKey,
+        buildBody,
+        timeoutMs: 15000,
+        tag: '[generate-rule]',
       });
 
-      clearTimeout(timeout);
-
-      if (!llmRes.ok) {
-        const errBody = await llmRes.text();
-        console.error('[generate-rule] LLM API error:', llmRes.status, errBody);
-        res.status(502).json({ error: 'Failed to generate rule. LLM API returned an error.' });
-        return;
+      if (usedFallback) {
+        console.log(`[generate-rule] Used fallback provider (${provider.isOpenAI ? 'OpenAI' : 'Anthropic'})`);
       }
 
       const llmData = await llmRes.json() as any;
-      const text = isOpenAI
+      const text = provider.isOpenAI
         ? (llmData.choices?.[0]?.message?.content || '')
         : (llmData.content?.[0]?.text || '');
 
@@ -1326,8 +1314,9 @@ Output: [{"name":"allow-get-example","description":"Allow GET requests to exampl
         res.status(504).json({ error: 'Rule generation timed out. Please try again.' });
         return;
       }
-      console.error('[generate-rule] Unexpected error:', err);
-      res.status(500).json({ error: 'Failed to generate rule. Please try again.' });
+      const errMsg = err instanceof Error ? err.message : String(err);
+      console.error('[generate-rule] Unexpected error:', errMsg);
+      res.status(502).json({ error: 'Failed to generate rule. LLM API returned an error.' });
     }
   });
 
@@ -1457,13 +1446,7 @@ ${current_policy ? `\nThe user has an existing policy. Refine it based on their
     res.flushHeaders();
 
     try {
-      const isOpenAI = apiKey.startsWith('sk-proj-') || apiKey.startsWith('sk-');
-      const llmUrl = isOpenAI
-        ? 'https://api.openai.com/v1/chat/completions'
-        : 'https://api.anthropic.com/v1/messages';
-      const llmHeaders: Record<string, string> = isOpenAI
-        ? { 'Content-Type': 'application/json', 'Authorization': `Bearer ${apiKey}` }
-        : { 'Content-Type': 'application/json', 'x-api-key': apiKey, 'anthropic-version': '2023-06-01' };
+      const fallbackKey = process.env.PALARYN_LLM_FALLBACK_KEY;
 
       // Build message array for multi-turn context
       const chatMessages = messages.map((m: { role: string; content: string }) => ({
@@ -1471,7 +1454,7 @@ ${current_policy ? `\nThe user has an existing policy. Refine it based on their
         content: m.content.slice(0, 2000),
       }));
 
-      const llmBody = isOpenAI
+      const buildBody = (provider: LlmProvider) => provider.isOpenAI
         ? JSON.stringify({
             model: 'gpt-4.1-mini',
             max_tokens: 2048,
@@ -1489,32 +1472,24 @@ ${current_policy ? `\nThe user has an existing policy. Refine it based on their
             messages: chatMessages,
           });
 
-      const controller = new AbortController();
-      const timeout = setTimeout(() => controller.abort(), 60000);
-
-      // Handle client disconnect
-      req.on('close', () => {
-        controller.abort();
-        clearTimeout(timeout);
-      });
+      const clientAbort = new AbortController();
+      req.on('close', () => clientAbort.abort());
 
-      const llmRes = await fetch(llmUrl, {
-        method: 'POST',
-        headers: llmHeaders,
-        body: llmBody,
-        signal: controller.signal,
+      const { response: llmRes, provider, usedFallback } = await llmFetchWithFallback({
+        primaryKey: apiKey,
+        fallbackKey,
+        buildBody,
+        timeoutMs: 15000,
+        signal: clientAbort.signal,
+        tag: '[policy-chat]',
       });
 
-      clearTimeout(timeout);
-
-      if (!llmRes.ok) {
-        const errBody = await llmRes.text();
-        console.error('[policy-chat] LLM API error:', llmRes.status, errBody);
-        res.write(`event: error\ndata: ${JSON.stringify({ error: 'LLM API returned an error.' })}\n\n`);
-        res.end();
-        return;
+      if (usedFallback) {
+        console.log(`[policy-chat] Used fallback provider (${provider.isOpenAI ? 'OpenAI' : 'Anthropic'})`);
       }
 
+      const isOpenAI = provider.isOpenAI;
+
       if (!llmRes.body) {
         res.write(`event: error\ndata: ${JSON.stringify({ error: 'No response stream from LLM.' })}\n\n`);
         res.end();