@almadar/llm 1.0.0

package/src/continuation.ts

@@ -0,0 +1,290 @@
+/**
+ * LLM Continuation Utility
+ *
+ * Handles truncated LLM responses with automatic continuation.
+ * - Detects truncation via finish_reason and JSON structure
+ * - Automatically continues with full context
+ * - Merges partial and continuation responses
+ * - Salvages partial data if max continuations reached
+ *
+ * @packageDocumentation
+ */
+
+import { z } from 'zod';
+import { LLMClient, type LLMFinishReason } from './client.js';
+import { detectTruncation } from './truncation-detector.js';
+import { extractJsonFromText, autoCloseJson, isValidJson } from './json-parser.js';
+
+// ============================================================================
+// Types
+// ============================================================================
+
+export interface ContinuationOptions<T> {
+  client: LLMClient;
+  systemPrompt: string;
+  userPrompt: string;
+  schema?: z.ZodSchema<T>;
+  maxTokens?: number;
+  maxContinuations?: number;
+  maxRetries?: number;
+  buildContinuationPrompt: (
+    partialResponse: string,
+    attempt: number,
+  ) => string;
+  continuationSystemPrompt?: string;
+}
+
+export interface ContinuationResult<T> {
+  data: T;
+  raw: string;
+  continuationCount: number;
+  warnings: string[];
+  wasSalvaged: boolean;
+}
+
+// ============================================================================
+// Constants
+// ============================================================================
+
+const DEFAULT_MAX_TOKENS = 8192;
+const DEFAULT_MAX_CONTINUATIONS = 3;
+
+/**
+ * Default continuation system prompt.
+ * Used when no custom continuationSystemPrompt is provided.
+ */
+const DEFAULT_CONTINUATION_SYSTEM_PROMPT = `You are a JSON continuation assistant. Your ONLY job is to continue generating JSON from where the previous response was truncated.
+
+Rules:
+1. Continue from EXACTLY where the previous output stopped
+2. Do NOT repeat any content already generated
+3. Complete the JSON structure properly with all closing brackets
+4. Do NOT wrap in markdown code blocks
+5. Output ONLY the continuation JSON, nothing else`;
+
+// ============================================================================
+// Helper Functions
+// ============================================================================
+
+export function mergeResponses(
+  previous: string,
+  continuation: string,
+): string {
+  const trimmedPrev = previous.trimEnd();
+  const trimmedCont = continuation.trimStart();
+
+  let cleanedCont = trimmedCont
+    .replace(/^```json?\s*/i, '')
+    .replace(/```\s*$/i, '')
+    .trim();
+
+  if (cleanedCont.startsWith('{')) {
+    try {
+      const contParsed = JSON.parse(autoCloseJson(cleanedCont));
+      const keys = Object.keys(contParsed);
+      if (keys.length === 1 && Array.isArray(contParsed[keys[0]])) {
+        cleanedCont = contParsed[keys[0]]
+          .map((item: unknown) => JSON.stringify(item))
+          .join(',\n');
+      }
+    } catch {
+      // Continue with original cleaning
+    }
+  }
+
+  if (cleanedCont.startsWith('}') || cleanedCont.startsWith(']')) {
+    return trimmedPrev + cleanedCont;
+  }
+
+  const prevEndsWithValue = /[\}\]\"\d]$/.test(trimmedPrev);
+  const contStartsWithValue = /^[\{\[\"]/.test(cleanedCont);
+
+  if (prevEndsWithValue && contStartsWithValue) {
+    return trimmedPrev + ',\n' + cleanedCont;
+  }
+
+  return trimmedPrev + cleanedCont;
+}
+
+export function salvagePartialResponse<T>(rawResponse: string): T | null {
+  console.warn('[Continuation] Attempting to salvage partial response');
+
+  try {
+    const cleanedResponse = extractJsonFromText(rawResponse) || rawResponse;
+    const closed = autoCloseJson(cleanedResponse);
+    const parsed = JSON.parse(closed) as T;
+    console.log('[Continuation] Successfully salvaged partial response');
+    return parsed;
+  } catch (error) {
+    console.error('[Continuation] Could not salvage response:', error);
+  }
+
+  return null;
+}
+
+// ============================================================================
+// Main Function
+// ============================================================================
+
+export async function callWithContinuation<T>(
+  options: ContinuationOptions<T>,
+): Promise<ContinuationResult<T>> {
+  const {
+    client,
+    systemPrompt,
+    userPrompt,
+    schema,
+    maxTokens = DEFAULT_MAX_TOKENS,
+    maxContinuations = DEFAULT_MAX_CONTINUATIONS,
+    buildContinuationPrompt,
+    continuationSystemPrompt = DEFAULT_CONTINUATION_SYSTEM_PROMPT,
+  } = options;
+
+  let rawResponse = '';
+  let continuationCount = 0;
+  const warnings: string[] = [];
+  let wasSalvaged = false;
+
+  console.log('[Continuation] Starting LLM call with continuation support');
+  console.log(
+    `[Continuation] Max tokens: ${maxTokens}, Max continuations: ${maxContinuations}`,
+  );
+
+  try {
+    const response = await client.callRawWithMetadata({
+      systemPrompt,
+      userPrompt,
+      maxTokens,
+    });
+
+    rawResponse = extractJsonFromText(response.raw) || response.raw;
+
+    console.log(
+      `[Continuation] Initial response: ${rawResponse.length} chars, finish_reason: ${response.finishReason}`,
+    );
+
+    let truncation = detectTruncation(rawResponse, response.finishReason);
+
+    while (truncation.isTruncated && continuationCount < maxContinuations) {
+      continuationCount++;
+      const warningMsg = `Response truncated (${truncation.reason}), continuing (attempt ${continuationCount}/${maxContinuations})`;
+      console.log(`[Continuation] ${warningMsg}`);
+      warnings.push(warningMsg);
+
+      const contPrompt = buildContinuationPrompt(
+        rawResponse,
+        continuationCount,
+      );
+
+      const contResponse = await client.callRawWithMetadata({
+        systemPrompt: continuationSystemPrompt,
+        userPrompt: contPrompt,
+        maxTokens,
+      });
+
+      console.log(
+        `[Continuation] Continuation response: ${contResponse.raw.length} chars, finish_reason: ${contResponse.finishReason}`,
+      );
+
+      const cleanedContResponse =
+        extractJsonFromText(contResponse.raw) || contResponse.raw;
+      rawResponse = mergeResponses(rawResponse, cleanedContResponse);
+
+      truncation = detectTruncation(rawResponse, contResponse.finishReason);
+    }
+
+    if (
+      continuationCount >= maxContinuations &&
+      truncation.isTruncated
+    ) {
+      console.warn(
+        `[Continuation] Reached max continuations (${maxContinuations}), attempting to salvage...`,
+      );
+      warnings.push(
+        `Reached max continuations - some content may be incomplete`,
+      );
+      wasSalvaged = true;
+    }
+
+    const cleanedResponse =
+      extractJsonFromText(rawResponse) || rawResponse;
+    let data: T;
+
+    try {
+      if (isValidJson(cleanedResponse)) {
+        data = JSON.parse(cleanedResponse) as T;
+      } else {
+        const closed = autoCloseJson(cleanedResponse);
+        data = JSON.parse(closed) as T;
+        if (!wasSalvaged) {
+          warnings.push('Response required auto-closing of JSON brackets');
+        }
+      }
+    } catch (parseError) {
+      const salvaged = salvagePartialResponse<T>(cleanedResponse);
+      if (salvaged) {
+        data = salvaged;
+        wasSalvaged = true;
+        warnings.push('Response was salvaged from partial data');
+      } else {
+        throw new Error(
+          `Failed to parse response after ${continuationCount} continuations: ${parseError}`,
+        );
+      }
+    }
+
+    if (schema) {
+      try {
+        data = schema.parse(data);
+      } catch (validationError) {
+        console.warn(
+          '[Continuation] Schema validation failed:',
+          validationError,
+        );
+        warnings.push(`Schema validation issue: ${validationError}`);
+      }
+    }
+
+    console.log(
+      `[Continuation] Complete. Continuations: ${continuationCount}, Warnings: ${warnings.length}`,
+    );
+
+    return {
+      data,
+      raw: rawResponse,
+      continuationCount,
+      warnings,
+      wasSalvaged,
+    };
+  } catch (error) {
+    console.error('[Continuation] Error during LLM call:', error);
+    throw error;
+  }
+}
+
+export function buildGenericContinuationPrompt(
+  context: string,
+  partialResponse: string,
+  attempt: number,
+  maxAttempts: number = DEFAULT_MAX_CONTINUATIONS,
+): string {
+  return `## CONTINUATION REQUEST (Attempt ${attempt}/${maxAttempts})
+
+Your previous response was truncated. Continue generating from where you left off.
+
+### ORIGINAL CONTEXT
+${context}
+
+### WHAT YOU GENERATED SO FAR
+\`\`\`json
+${partialResponse}
+\`\`\`
+
+### INSTRUCTIONS
+1. Continue from EXACTLY where the response was cut off
+2. Do NOT repeat any content already generated
+3. Complete the JSON structure properly
+4. Do NOT wrap your response in markdown code blocks
+
+Continue generating now:`;
+}

package/src/index.ts

@@ -0,0 +1,87 @@
+/**
+ * @almadar/llm
+ *
+ * Multi-provider LLM client with rate limiting, token tracking,
+ * structured outputs, and continuation handling.
+ *
+ * @packageDocumentation
+ */
+
+export {
+  LLMClient,
+  getSharedLLMClient,
+  resetSharedLLMClient,
+  createRequirementsClient,
+  createCreativeClient,
+  createFixClient,
+  createDeepSeekClient,
+  createOpenAIClient,
+  createAnthropicClient,
+  createKimiClient,
+  getAvailableProvider,
+  isProviderAvailable,
+  DEEPSEEK_MODELS,
+  OPENAI_MODELS,
+  ANTHROPIC_MODELS,
+  KIMI_MODELS,
+  type LLMProvider,
+  type ProviderConfig,
+  type LLMClientOptions,
+  type LLMCallOptions,
+  type LLMResponse,
+  type LLMUsage,
+  type LLMFinishReason,
+  type CacheableBlock,
+  type CacheAwareLLMCallOptions,
+} from './client.js';
+
+export {
+  RateLimiter,
+  getGlobalRateLimiter,
+  resetGlobalRateLimiter,
+  type RateLimiterOptions,
+} from './rate-limiter.js';
+
+export {
+  TokenTracker,
+  getGlobalTokenTracker,
+  resetGlobalTokenTracker,
+  type TokenUsage,
+} from './token-tracker.js';
+
+export {
+  parseJsonResponse,
+  extractJsonFromText,
+  safeParseJson,
+  isValidJson,
+  autoCloseJson,
+} from './json-parser.js';
+
+export {
+  detectTruncation,
+  findLastCompleteElement,
+  isLikelyTruncated,
+  type TruncationResult,
+  type TruncationReason,
+} from './truncation-detector.js';
+
+export {
+  callWithContinuation,
+  mergeResponses,
+  salvagePartialResponse,
+  buildGenericContinuationPrompt,
+  type ContinuationOptions,
+  type ContinuationResult,
+} from './continuation.js';
+
+export {
+  StructuredOutputClient,
+  getStructuredOutputClient,
+  resetStructuredOutputClient,
+  isStructuredOutputAvailable,
+  STRUCTURED_OUTPUT_MODELS,
+  type StructuredOutputOptions,
+  type StructuredGenerationOptions,
+  type StructuredGenerationResult,
+  type JsonSchema,
+} from './structured-output.js';

package/src/json-parser.ts

@@ -0,0 +1,273 @@
+/**
+ * JSON Parser Utilities
+ *
+ * Robust JSON parsing for LLM responses that may contain:
+ * - Markdown code blocks
+ * - Extra text before/after JSON
+ * - Minor formatting issues
+ *
+ * @packageDocumentation
+ */
+
+import { z } from 'zod';
+
+function extractBalancedBrackets(
+  text: string,
+  startIdx: number,
+  openBracket: string,
+  closeBracket: string,
+): string | null {
+  if (text[startIdx] !== openBracket) return null;
+
+  let depth = 0;
+  let inString = false;
+  let escapeNext = false;
+
+  for (let i = startIdx; i < text.length; i++) {
+    const char = text[i];
+
+    if (escapeNext) {
+      escapeNext = false;
+      continue;
+    }
+
+    if (char === '\\' && inString) {
+      escapeNext = true;
+      continue;
+    }
+
+    if (char === '"') {
+      inString = !inString;
+      continue;
+    }
+
+    if (inString) continue;
+
+    if (char === openBracket) {
+      depth++;
+    } else if (char === closeBracket) {
+      depth--;
+      if (depth === 0) {
+        return text.substring(startIdx, i + 1);
+      }
+    }
+  }
+
+  return null;
+}
+
+/**
+ * Extract JSON from LLM response text.
+ *
+ * Handles markdown code blocks, raw JSON objects/arrays, and primitive values.
+ */
+export function extractJsonFromText(text: string): string | null {
+  const trimmed = text.trim();
+
+  // Try markdown code blocks first
+  const codeBlockMatch = trimmed.match(/```(?:json)?\s*([\s\S]*?)```/);
+  if (codeBlockMatch) {
+    return codeBlockMatch[1].trim();
+  }
+
+  const objectStartIdx = trimmed.indexOf('{');
+  const arrayStartIdx = trimmed.indexOf('[');
+
+  const objectFirst =
+    objectStartIdx !== -1 &&
+    (arrayStartIdx === -1 || objectStartIdx < arrayStartIdx);
+  const arrayFirst =
+    arrayStartIdx !== -1 &&
+    (objectStartIdx === -1 || arrayStartIdx < objectStartIdx);
+
+  if (arrayFirst) {
+    const arrayJson = extractBalancedBrackets(
+      trimmed,
+      arrayStartIdx,
+      '[',
+      ']',
+    );
+    if (arrayJson) return arrayJson;
+    const arrayMatch = trimmed.match(/\[[\s\S]*\]/);
+    if (arrayMatch) return arrayMatch[0];
+  }
+
+  if (objectFirst) {
+    const objectJson = extractBalancedBrackets(
+      trimmed,
+      objectStartIdx,
+      '{',
+      '}',
+    );
+    if (objectJson) return objectJson;
+    const objectMatch = trimmed.match(/\{[\s\S]*\}/);
+    if (objectMatch) return objectMatch[0];
+  }
+
+  // Primitive JSON values
+  if (trimmed.startsWith('"') && trimmed.endsWith('"')) return trimmed;
+  if (/^-?\d+(\.\d+)?([eE][+-]?\d+)?$/.test(trimmed)) return trimmed;
+  if (trimmed === 'true' || trimmed === 'false') return trimmed;
+  if (trimmed === 'null') return trimmed;
+
+  return null;
+}
+
+/**
+ * Parse JSON from LLM response with optional Zod schema validation.
+ */
+export function parseJsonResponse<T>(
+  response: string,
+  schema?: z.ZodSchema<T>,
+): T {
+  const jsonStr = extractJsonFromText(response);
+
+  if (!jsonStr) {
+    throw new Error(
+      'No valid JSON found in response. ' +
+        'Expected a JSON value (object, array, string, number, boolean, or null), ' +
+        'possibly wrapped in markdown code blocks.',
+    );
+  }
+
+  let parsed: unknown;
+  try {
+    parsed = JSON.parse(jsonStr);
+  } catch (parseError) {
+    const fixed = fixCommonJsonIssues(jsonStr);
+    try {
+      parsed = JSON.parse(fixed);
+    } catch {
+      throw new Error(
+        `Failed to parse JSON: ${parseError instanceof Error ? parseError.message : 'Unknown error'}. ` +
+          `Raw text: ${jsonStr.substring(0, 200)}...`,
+      );
+    }
+  }
+
+  if (schema) {
+    const result = schema.safeParse(parsed);
+    if (!result.success) {
+      const errors = result.error.errors
+        .map((e) => `${e.path.join('.')}: ${e.message}`)
+        .join('; ');
+      throw new Error(`Schema validation failed: ${errors}`);
+    }
+    return result.data;
+  }
+
+  return parsed as T;
+}
+
+function fixCommonJsonIssues(json: string): string {
+  let fixed = json;
+  fixed = fixed.replace(/,(\s*[}\]])/g, '$1');
+  fixed = fixed.replace(/([{,]\s*)(\w+)(\s*:)/g, '$1"$2"$3');
+  fixed = fixed.replace(/'/g, '"');
+  fixed = fixed.replace(/[\x00-\x1F\x7F]/g, ' ');
+  return fixed;
+}
+
+/**
+ * Safely parse JSON without throwing.
+ */
+export function safeParseJson<T>(
+  response: string,
+  schema?: z.ZodSchema<T>,
+): { success: true; data: T } | { success: false; error: Error } {
+  try {
+    const data = parseJsonResponse(response, schema);
+    return { success: true, data };
+  } catch (error) {
+    return {
+      success: false,
+      error: error instanceof Error ? error : new Error(String(error)),
+    };
+  }
+}
+
+/**
+ * Check if a string is valid JSON.
+ */
+export function isValidJson(str: string): boolean {
+  try {
+    JSON.parse(str);
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+/**
+ * Attempt to auto-close unclosed JSON brackets.
+ */
+export function autoCloseJson(json: string): string {
+  let result = json.trim();
+
+  // Handle unclosed strings
+  let inString = false;
+  let escaped = false;
+  for (const char of result) {
+    if (escaped) {
+      escaped = false;
+      continue;
+    }
+    if (char === '\\') {
+      escaped = true;
+      continue;
+    }
+    if (char === '"') {
+      inString = !inString;
+    }
+  }
+  if (inString) {
+    result += '"';
+  }
+
+  // Remove trailing incomplete content
+  result = result.replace(/,\s*$/, '');
+  result = result.replace(/:\s*$/, ': null');
+
+  // Build correct closing sequence
+  const closers = buildClosingSequence(result);
+  result += closers;
+
+  return result;
+}
+
+function buildClosingSequence(json: string): string {
+  const stack: string[] = [];
+  let inString = false;
+  let escaped = false;
+
+  for (const char of json) {
+    if (escaped) {
+      escaped = false;
+      continue;
+    }
+
+    if (char === '\\' && inString) {
+      escaped = true;
+      continue;
+    }
+
+    if (char === '"') {
+      inString = !inString;
+      continue;
+    }
+
+    if (inString) continue;
+
+    if (char === '[') {
+      stack.push(']');
+    } else if (char === '{') {
+      stack.push('}');
+    } else if (char === ']' || char === '}') {
+      if (stack.length > 0 && stack[stack.length - 1] === char) {
+        stack.pop();
+      }
+    }
+  }
+
+  return stack.reverse().join('');
+}