@msbayindir/context-rag 1.0.0-beta.3 → 1.0.0-beta.4

package/dist/index.cjs

@@ -4,6 +4,7 @@ var zod = require('zod');
 var crypto = require('crypto');
 var generativeAi = require('@google/generative-ai');
 var server = require('@google/generative-ai/server');
+var zodToJsonSchema = require('zod-to-json-schema');
 var fs = require('fs/promises');
 var path = require('path');
 var pdf = require('pdf-parse');
@@ -388,10 +389,10 @@ Analyze the document and return ONLY a JSON response with the following structur
     "Specific instruction 3 for this document type"
   ],
   
-  "exampleFormats": {
-    "example1": "How a specific format should look",
-    "example2": "Another format example"
-  },
+  "exampleFormats": [
+    { "element": "table", "format": "Markdown table with headers" },
+    { "element": "code", "format": "Code block with language tag" }
+  ],
   
   "chunkStrategy": {
     "maxTokens": 800,
@@ -413,92 +414,104 @@ IMPORTANT RULES:
 
 {{DOCUMENT_TYPE_HINT}}
 `;
-var BASE_EXTRACTION_TEMPLATE = `You are a document processing AI. Extract content following the EXACT format below.
+var BASE_EXTRACTION_TEMPLATE = `You are a document processing AI. Extract content from the document with HIGH FIDELITY and DETAIL.
 
 ## OUTPUT FORMAT (MANDATORY - DO NOT MODIFY)
 
-Use this structure for EVERY content section:
+\u26A0\uFE0F CRITICAL: You MUST use EXACTLY this marker format. Any deviation will cause parsing errors:
 
+\`\`\`
 <!-- SECTION type="[TYPE]" page="[PAGE]" confidence="[0.0-1.0]" -->
 [Content here in Markdown format]
 <!-- /SECTION -->
+\`\`\`
 
-### Valid Types:
-- TEXT: Regular paragraphs and prose
-- TABLE: Data tables in Markdown format
-- LIST: Bullet (-) or numbered (1. 2. 3.) lists
-- HEADING: Section headers with # ## ### levels
-- CODE: Code blocks with language specification
-- QUOTE: Quoted text or citations
-- IMAGE_REF: Description of images, charts, figures
-- QUESTION: Multiple choice questions with options (A, B, C, D, E)
-
-### Format Rules:
-1. **Tables**: Use Markdown table format
-   | Column1 | Column2 | Column3 |
-   |---------|---------|---------|
-   | data    | data    | data    |
+### EXAMPLE OUTPUT (FOLLOW THIS EXACTLY):
+\`\`\`
+<!-- SECTION type="HEADING" page="1" confidence="0.95" -->
+# Introduction to Metabolism
+<!-- /SECTION -->
 
-2. **Lists**: Use consistent format
-   - Bullet item
-   - Another bullet
-   
-   OR
-   
-   1. Numbered item
-   2. Another numbered
+<!-- SECTION type="TEXT" page="1" confidence="0.92" -->
+Metabolism refers to all chemical reactions in an organism. It creates energy...
+(Extract full paragraphs, do not break them up unnecessarily)
+<!-- /SECTION -->
 
-3. **Headings**: Maximum 3 levels, use hierarchy
-   # Main Section
-   ## Subsection
-   ### Sub-subsection
+<!-- SECTION type="LIST" page="2" confidence="0.90" -->
+- First item in the list
+- Second item in the list
+- Third item in the list
+<!-- /SECTION -->
 
-4. **Code**: Specify language
-   \`\`\`python
-   code here
-   \`\`\`
+<!-- SECTION type="TABLE" page="2" confidence="0.88" -->
+| Column1 | Column2 |
+|---------|---------|
+| Data1   | Data2   |
+<!-- /SECTION -->
+\`\`\`
 
-5. **Images**: Describe visual content
-   [IMAGE: Description of what the image shows]
+### Valid Types:
+- TEXT: Regular paragraphs and prose. **PREFER THIS** for standard text.
+- TABLE: **ONLY** for explicit data tables in the source.
+- LIST: **ONLY** for explicit bulleted/numbered lists in source.
+- HEADING: Section headers with # ## ### levels.
+- CODE: Code blocks with language specification.
+- QUOTE: Quoted text or citations.
+- IMAGE_REF: Description of images, charts, figures.
+- QUESTION: Multiple choice questions.
 
-6. **Questions**: Multiple choice questions with options
-   **Question 1:** Question text here?
-   A) Option A text
-   B) Option B text
-   C) Option C text
-   D) Option D text
-   E) Option E text (if exists)
-   **Answer:** [Letter] (if answer is provided in document)
+### Format Rules:
+1. **Tables**: Use Markdown table format.
+2. **Lists**: Use consistent format (bullets or numbers).
+3. **Headings**: Use Markdown headers (#, ##, ###).
+4. **Code**: Use fenced code blocks with language.
+5. **Images**: Describe visual content clearly.
 
 ## DOCUMENT-SPECIFIC INSTRUCTIONS
 {{DOCUMENT_INSTRUCTIONS}}
 
 ## CRITICAL EXTRACTION RULES (DO NOT VIOLATE)
-\u26A0\uFE0F These rules are MANDATORY for legal, medical, and financial document accuracy:
 
 1. **NO SUMMARIZATION**: Extract content EXACTLY as written. Do not summarize, paraphrase, or condense.
-2. **NO INTERPRETATION**: Do not interpret, explain, or add commentary to the content.
-3. **PRESERVE ORIGINAL WORDING**: Keep exact terminology, especially for:
-   - Legal terms, clauses, and article references
-   - Medical terminology, diagnoses, and prescriptions
-   - Financial figures, percentages, and calculations
-   - Technical specifications and measurements
-4. **VERBATIM EXTRACTION**: Copy text word-for-word from the document.
-5. **NO OMISSIONS**: Include all content, even if it seems redundant or repetitive.
-6. **UNCLEAR CONTENT**: If text is unclear or illegible, extract as-is and mark: [UNCLEAR: partial text visible]
-7. **FOREIGN TERMS**: Keep foreign language terms, Latin phrases, and abbreviations exactly as written.
+2. **PRESERVE FLOW**: **DO NOT** break continuous text into lists unless it is explicitly a list in the source. Keep paragraphs together.
+3. **AVOID OVER-SEGMENTATION**: Combine related sentences into single TEXT blocks. Do not create a new section for every sentence.
+4. **PRESERVE ORIGINAL WORDING**: Keep exact terminology, especially for technical, medical, or legal terms.
+5. **NO INTERPRETATION**: Do not interpret or explain the content. Just extract it.
+6. **UNCLEAR CONTENT**: If text is unclear, mark: [UNCLEAR: partial text].
+7. **FOREIGN TERMS**: Keep foreign language terms exactly as written.
 
 ## PROCESSING RULES
-- Extract ALL content completely, do not summarize or skip
-- Preserve original document structure and hierarchy
-- Include page references for each section
-- Maintain technical accuracy and terminology
-- Use appropriate confidence scores based on extraction quality
-- If content spans multiple pages, use the starting page number
+- Extract ALL content completely.
+- Preserve original document structure and hierarchy.
+- Include page references for each section.
+- If content spans multiple pages, use the starting page number.
 
 ## PAGE RANGE
 {{PAGE_RANGE}}
 `;
+var STRUCTURED_EXTRACTION_TEMPLATE = `You are a document processing AI. Extract content from the provided document pages.
+
+Your goal is to extract content accurately, preserving the logical structure and semantics.
+
+## INSTRUCTIONS
+{{DOCUMENT_INSTRUCTIONS}}
+
+## CRITICAL RULES (DO NOT VIOLATE)
+1. **NO SUMMARIZATION**: Extract content EXACTLY as written. Do not summarize, paraphrase, or condense.
+2. **PRESERVE FLOW**: **DO NOT** break continuous text into lists unless it is explicitly a list in the source. Keep paragraphs together.
+3. **AVOID OVER-SEGMENTATION**: Combine related sentences into single TEXT blocks. Do not create a new section for every sentence.
+4. **PRESERVE ORIGINAL WORDING**: Keep exact terminology, especially for technical, medical, or legal terms.
+5. **NO INTERPRETATION**: Do not interpret or explain the content. Just extract it.
+
+## PAGE RANGE
+{{PAGE_RANGE}}
+
+IMPORTANT:
+1. Extract content strictly from the specified page range.
+2. Maintain the order of elements as they appear in the document.
+3. Don't summarize code blocks or tables; extract them fully.
+4. Follow the specific document instructions provided above.
+`;
 var DEFAULT_DOCUMENT_INSTRUCTIONS = `
 - Extract all text content preserving structure
 - Convert tables to Markdown table format
@@ -507,12 +520,21 @@ var DEFAULT_DOCUMENT_INSTRUCTIONS = `
 - Note any images with descriptive text
 - Maintain the logical flow of content
 `;
-function buildExtractionPrompt(documentInstructions, exampleFormats, pageStart, pageEnd) {
+function buildExtractionPrompt(documentInstructions, exampleFormats, pageStart, pageEnd, useStructuredOutput = false) {
   let instructionsBlock = documentInstructions.map((instruction) => `- ${instruction}`).join("\n");
-  if (exampleFormats && Object.keys(exampleFormats).length > 0) {
+  let formats = [];
+  if (Array.isArray(exampleFormats)) {
+    formats = exampleFormats;
+  } else if (exampleFormats) {
+    formats = Object.entries(exampleFormats).map(([key, value]) => ({
+      element: key,
+      format: value
+    }));
+  }
+  if (formats.length > 0) {
     instructionsBlock += "\n\n### Example Formats:\n";
-    for (const [key, value] of Object.entries(exampleFormats)) {
-      instructionsBlock += `- **${key}**: \`${value}\`
+    for (const example of formats) {
+      instructionsBlock += `- **${example.element}**: \`${example.format}\`
 `;
     }
   }
@@ -524,7 +546,8 @@ function buildExtractionPrompt(documentInstructions, exampleFormats, pageStart,
       pageRange = `Process pages ${pageStart}-${pageEnd} of this document.`;
     }
   }
-  return BASE_EXTRACTION_TEMPLATE.replace("{{DOCUMENT_INSTRUCTIONS}}", instructionsBlock || DEFAULT_DOCUMENT_INSTRUCTIONS).replace("{{PAGE_RANGE}}", pageRange);
+  const template = useStructuredOutput ? STRUCTURED_EXTRACTION_TEMPLATE : BASE_EXTRACTION_TEMPLATE;
+  return template.replace("{{DOCUMENT_INSTRUCTIONS}}", instructionsBlock || DEFAULT_DOCUMENT_INSTRUCTIONS).replace("{{PAGE_RANGE}}", pageRange);
 }
 function buildDiscoveryPrompt(documentTypeHint) {
   let hint = "";
@@ -534,7 +557,7 @@ Hint: The user expects this to be a "${documentTypeHint}" document. Consider thi
   }
   return DISCOVERY_TEMPLATE.replace("{{DOCUMENT_TYPE_HINT}}", hint);
 }
-var SECTION_PATTERN = /<!-- SECTION type="(\w+)" page="(\d+)" confidence="([\d.]+)" -->\n?([\s\S]*?)\n?<!-- \/SECTION -->/g;
+var SECTION_PATTERN = /<!-- SECTION (?:type=")?(\w+)"? page="(\d+)" confidence="([\d.]+)" -->\n?([\s\S]*?)\n?<!-- \/SECTION -->/g;
 
 // src/types/enums.ts
 var ChunkTypeEnum = {
@@ -1196,7 +1219,7 @@ var ChunkRepository = class {
       const result = await this.prisma.$queryRaw`
         INSERT INTO context_rag_chunks (
           id, prompt_config_id, document_id, chunk_index, chunk_type,
-          search_content, search_vector, display_content,
+          search_content, enriched_content, context_text, search_vector, display_content,
           source_page_start, source_page_end, confidence_score, metadata, created_at
         ) VALUES (
           gen_random_uuid(),
@@ -1205,6 +1228,8 @@ var ChunkRepository = class {
           ${input.chunkIndex},
           ${input.chunkType},
           ${input.searchContent},
+          ${input.enrichedContent ?? null},
+          ${input.contextText ?? null},
           ${embedding}::vector,
           ${input.displayContent},
           ${input.sourcePageStart},
@@ -1235,7 +1260,7 @@ var ChunkRepository = class {
         const result = await tx.$queryRaw`
           INSERT INTO context_rag_chunks (
             id, prompt_config_id, document_id, chunk_index, chunk_type,
-            search_content, search_vector, display_content,
+            search_content, enriched_content, context_text, search_vector, display_content,
             source_page_start, source_page_end, confidence_score, metadata, created_at
           ) VALUES (
             gen_random_uuid(),
@@ -1244,6 +1269,8 @@ var ChunkRepository = class {
             ${input.chunkIndex},
             ${input.chunkType},
             ${input.searchContent},
+            ${input.enrichedContent ?? null},
+            ${input.contextText ?? null},
             ${embedding}::vector,
             ${input.displayContent},
             ${input.sourcePageStart},
@@ -1453,6 +1480,121 @@ async function getDatabaseStats(prisma) {
     });
   }
 }
+var ChunkTypeSchema = zod.z.enum([
+  "TEXT",
+  "TABLE",
+  "LIST",
+  "HEADING",
+  "CODE",
+  "QUOTE",
+  "IMAGE_REF",
+  "QUESTION",
+  "MIXED"
+]).describe("The type of content in this section. PREFER 'TEXT' for regular paragraphs. Only use 'LIST' for actual bulleted/numbered lists. Only use 'TABLE' for actual tables.");
+var SectionSchema = zod.z.object({
+  /** Content type */
+  type: ChunkTypeSchema,
+  /** Source page number (1-indexed) */
+  page: zod.z.number().int().min(1).describe("The page number where this content starts."),
+  /** Extraction confidence score (0.0-1.0) */
+  confidence: zod.z.number().min(0).max(1).describe("Confidence score (0.0-1.0)"),
+  /** Extracted content in Markdown format */
+  content: zod.z.string().min(1).describe("The full extracted content in Markdown. DO NOT summarize. DO NOT break paragraphs into lists. Maintain the original flow.")
+});
+var SectionArraySchema = zod.z.array(SectionSchema);
+var DetectedElementTypeSchema = zod.z.enum([
+  "table",
+  "list",
+  "code",
+  "image",
+  "chart",
+  "form",
+  "heading"
+]);
+var DetectedElementSchema = zod.z.object({
+  /** Element type */
+  type: DetectedElementTypeSchema,
+  /** Approximate count */
+  count: zod.z.number().int().min(0),
+  /** Example locations (page numbers) */
+  examples: zod.z.array(zod.z.number()).optional()
+});
+var ChunkStrategySchema = zod.z.object({
+  /** Maximum tokens per chunk */
+  maxTokens: zod.z.number().int().min(100).max(2e3).default(500),
+  /** Split method */
+  splitBy: zod.z.enum(["semantic", "page", "paragraph", "section"]).default("semantic"),
+  /** Preserve tables as single chunks */
+  preserveTables: zod.z.boolean().default(true),
+  /** Preserve lists as single chunks */
+  preserveLists: zod.z.boolean().default(true)
+});
+var DiscoveryResponseSchema = zod.z.object({
+  /** Detected document type (e.g., 'Medical', 'Legal') */
+  documentType: zod.z.string().min(1),
+  /** Human-readable document type name */
+  documentTypeName: zod.z.string().min(1),
+  /** Document language (e.g., 'tr', 'en') */
+  language: zod.z.string().optional(),
+  /** Document complexity assessment */
+  complexity: zod.z.enum(["low", "medium", "high"]).optional(),
+  /** Detected elements in document */
+  detectedElements: zod.z.array(DetectedElementSchema).default([]),
+  /** Document-specific extraction instructions */
+  specialInstructions: zod.z.array(zod.z.string()),
+  /** Example formats for each element type */
+  exampleFormats: zod.z.array(zod.z.object({
+    element: zod.z.string(),
+    format: zod.z.string()
+  })).optional(),
+  /** Recommended chunk strategy */
+  chunkStrategy: ChunkStrategySchema.optional(),
+  /** Detection confidence (0.0-1.0) */
+  confidence: zod.z.number().min(0).max(1),
+  /** AI reasoning for the analysis */
+  reasoning: zod.z.string()
+});
+var ContextGenerationSchema = zod.z.object({
+  /** Generated context text */
+  context: zod.z.string(),
+  /** Confidence in the generated context */
+  confidence: zod.z.number().min(0).max(1).optional()
+});
+function zodToGeminiSchema(zodSchema) {
+  const jsonSchema = zodToJsonSchema.zodToJsonSchema(zodSchema, {
+    target: "jsonSchema7",
+    $refStrategy: "none",
+    // Critical: Gemini doesn't support $ref in responseSchema
+    definitionPath: "$defs"
+  });
+  return cleanSchemaForGemini(jsonSchema);
+}
+function cleanSchemaForGemini(schema) {
+  if (typeof schema !== "object" || schema === null) {
+    return schema;
+  }
+  if (Array.isArray(schema)) {
+    return schema.map(cleanSchemaForGemini);
+  }
+  const newObj = {};
+  for (const [key, value] of Object.entries(schema)) {
+    if (key === "additionalProperties" || key === "$schema" || key === "title" || key === "default") {
+      continue;
+    }
+    newObj[key] = cleanSchemaForGemini(value);
+  }
+  return newObj;
+}
+({
+  /** Schema for section extraction */
+  sectionArray: zodToGeminiSchema(SectionArraySchema),
+  /** Schema for discovery response */
+  discovery: zodToGeminiSchema(DiscoveryResponseSchema),
+  /** Schema for context generation */
+  contextGeneration: zodToGeminiSchema(ContextGenerationSchema)
+});
+
+// src/services/gemini.service.ts
 var GeminiService = class {
   genAI;
   fileManager;
@@ -1731,6 +1873,111 @@ ${userContent}` }]
       throw error;
     }
   }
+  /**
+   * Generate structured data from text prompt
+   */
+  async generateStructured(prompt, schema, options) {
+    return this.executeStructuredRetry(
+      [{ role: "user", parts: [{ text: prompt }] }],
+      schema,
+      options
+    );
+  }
+  /**
+   * Generate structured data from PDF
+   */
+  async generateStructuredWithPdf(pdfUri, prompt, schema, options) {
+    return this.executeStructuredRetry(
+      [
+        {
+          role: "user",
+          parts: [
+            { fileData: { mimeType: "application/pdf", fileUri: pdfUri } },
+            { text: prompt }
+          ]
+        }
+      ],
+      schema,
+      options
+    );
+  }
+  /**
+   * Execute structured generation with retry logic
+   */
+  async executeStructuredRetry(contents, schema, options) {
+    const maxRetries = options?.maxRetries ?? 2;
+    let attempt = 0;
+    let lastError;
+    const currentContents = [...contents];
+    while (attempt <= maxRetries) {
+      attempt++;
+      await this.rateLimiter.acquire();
+      try {
+        const result = await this.model.generateContent({
+          contents: currentContents,
+          generationConfig: {
+            responseMimeType: "application/json",
+            // Cast to any because the new schema format might have slight type mismatch 
+            // but is valid for the API
+            responseSchema: zodToGeminiSchema(schema),
+            temperature: options?.temperature ?? 0.2,
+            maxOutputTokens: options?.maxOutputTokens
+          }
+        });
+        const response = result.response;
+        const text = response.text();
+        const usage = response.usageMetadata;
+        this.rateLimiter.reportSuccess();
+        try {
+          const parsed = JSON.parse(text);
+          const data = schema.parse(parsed);
+          return {
+            data,
+            tokenUsage: {
+              input: usage?.promptTokenCount ?? 0,
+              output: usage?.candidatesTokenCount ?? 0,
+              total: usage?.totalTokenCount ?? 0
+            }
+          };
+        } catch (e) {
+          const errorMessage = e instanceof Error ? e.message : String(e);
+          const snippet = text.length > 500 ? text.substring(0, 200) + "...[truncated]..." + text.substring(text.length - 200) : text;
+          this.logger.warn(`Structured validation failed (attempt ${attempt}/${maxRetries + 1})`, {
+            error: errorMessage,
+            snippet: text.substring(0, 100)
+          });
+          lastError = new Error(`Structured output validation failed: ${errorMessage}. Raw response snippet: ${snippet}`);
+          if (attempt <= maxRetries) {
+            currentContents.push({
+              role: "model",
+              parts: [{ text }]
+            });
+            currentContents.push({
+              role: "user",
+              parts: [{ text: `JSON Validation Error: ${errorMessage}
+
+Please fix the JSON output to match the schema exactly.` }]
+            });
+            continue;
+          }
+          throw lastError;
+        }
+      } catch (error) {
+        try {
+          this.handleError(error);
+        } catch (handledError) {
+          throw handledError;
+        }
+        lastError = error;
+        if (attempt <= maxRetries) {
+          this.logger.warn(`Gemini API error (attempt ${attempt}/${maxRetries + 1}), retrying...`, { error: error.message });
+          continue;
+        }
+        throw error;
+      }
+    }
+    throw lastError;
+  }
   /**
    * Handle API errors
    */
@@ -1864,11 +2111,22 @@ var AnthropicHandler = class {
   gemini;
   limit;
   skipTypes;
-  constructor(config, gemini) {
+  constructor(config, mainGemini, resolvedConfig) {
     this.config = config;
-    this.gemini = gemini;
     this.limit = pLimit__default.default(config.concurrencyLimit ?? DEFAULT_ANTHROPIC_CONFIG.concurrencyLimit);
     this.skipTypes = new Set(config.skipChunkTypes ?? DEFAULT_ANTHROPIC_CONFIG.skipChunkTypes);
+    if (config.model && config.model !== resolvedConfig.model) {
+      console.log(`[AnthropicHandler] Using separate model for enhancement: ${config.model}`);
+      const enhancementConfig = {
+        ...resolvedConfig,
+        model: config.model
+      };
+      const rateLimiter = new RateLimiter(resolvedConfig.rateLimitConfig);
+      const logger = createLogger(resolvedConfig.logging);
+      this.gemini = new GeminiService(enhancementConfig, rateLimiter, logger);
+    } else {
+      this.gemini = mainGemini;
+    }
   }
   shouldSkip(chunkType) {
     return this.skipTypes.has(chunkType);
@@ -1916,15 +2174,18 @@ ${doc.fullDocumentText.slice(0, 15e3)}
 ${chunk.content}
 </chunk_to_contextualize>
 
-Bu chunk'\u0131n belgede nerede oldu\u011Funu ve ne hakk\u0131nda oldu\u011Funu 1-2 c\xFCmle ile T\xFCrk\xE7e a\xE7\u0131kla:`;
+Bu i\xE7eri\u011Fin belgenin genel ak\u0131\u015F\u0131 i\xE7indeki yerini, ba\u011Fl\u0131 oldu\u011Fu ana ba\u015Fl\u0131klar\u0131 ve ele ald\u0131\u011F\u0131 konuyu detayl\u0131 bir \u015Fekilde \xF6zetle. \u0130\xE7eri\u011Fin ne oldu\u011Funu de\u011Fil, ba\u011Flam\u0131n\u0131 anlat:`;
     try {
       if (doc.fileUri) {
-        const chunkPrompt = `Bu chunk'\u0131n belgede nerede oldu\u011Funu ve ne hakk\u0131nda oldu\u011Funu 1-2 c\xFCmle ile T\xFCrk\xE7e a\xE7\u0131kla:
+        const chunkPrompt = `Bu i\xE7eri\u011Fin belgenin genel ak\u0131\u015F\u0131 i\xE7indeki yerini, ba\u011Fl\u0131 oldu\u011Fu ana ba\u015Fl\u0131klar\u0131 ve ele ald\u0131\u011F\u0131 konuyu detayl\u0131 bir \u015Fekilde \xF6zetle. \u0130\xE7eri\u011Fin ne oldu\u011Funu de\u011Fil, ba\u011Flam\u0131n\u0131 anlat:
 
 <chunk>
 ${chunk.content}
 </chunk>`;
-        const result2 = await this.gemini.generateWithPdfUri(doc.fileUri, chunkPrompt);
+        const result2 = await this.gemini.generateWithPdfUri(doc.fileUri, chunkPrompt, {
+          maxOutputTokens: 2048,
+          temperature: 0.3
+        });
         return result2.text;
       }
       const result = await this.gemini.generateSimple(fullPrompt);
@@ -1943,7 +2204,7 @@ function createEnhancementHandler(config, _resolvedConfig, gemini) {
   }
   switch (config.approach) {
     case "anthropic_contextual":
-      return new AnthropicHandler(config, gemini);
+      return new AnthropicHandler(config, gemini, _resolvedConfig);
     case "google_grounding":
       throw new Error("Google Grounding is not yet implemented");
     case "custom":
@@ -2168,24 +2429,50 @@ var IngestionEngine = class {
     try {
       const result = await withRetry(
         async () => {
-          const prompt = buildExtractionPrompt(
-            documentInstructions,
-            exampleFormats,
-            batch.pageStart,
-            batch.pageEnd
-          );
-          const fullPrompt = `${prompt}
+          const useStructured = this.config.useStructuredOutput;
+          const getPrompt = (structured) => {
+            const basePrompt = buildExtractionPrompt(
+              documentInstructions,
+              exampleFormats,
+              batch.pageStart,
+              batch.pageEnd,
+              structured
+            );
+            return `${basePrompt}
                     
                     IMPORTANT: You have the FULL document. Restrict your extraction STRICTLY to pages ${batch.pageStart} to ${batch.pageEnd}. Do not extract content from other pages.`;
-          const response = await this.gemini.generateWithPdfUri(
+          };
+          if (useStructured) {
+            try {
+              const structuredResponse = await this.gemini.generateStructuredWithPdf(
+                fileUri,
+                getPrompt(true),
+                SectionArraySchema,
+                {
+                  temperature: this.config.generationConfig?.temperature,
+                  maxOutputTokens: this.config.generationConfig?.maxOutputTokens
+                }
+              );
+              this.logger.debug("Structured extraction success", {
+                batchId: batch.id,
+                chunkCount: structuredResponse.data.length
+              });
+              return structuredResponse;
+            } catch (structuredError) {
+              this.logger.warn("Structured extraction failed, falling back to legacy parsing", {
+                batchId: batch.id,
+                error: structuredError.message
+              });
+            }
+          }
+          return await this.gemini.generateWithPdfUri(
             fileUri,
-            fullPrompt,
+            getPrompt(false),
             {
               temperature: this.config.generationConfig?.temperature,
               maxOutputTokens: this.config.generationConfig?.maxOutputTokens
             }
           );
-          return response;
         },
         {
           ...retryOptions,
@@ -2206,13 +2493,40 @@ var IngestionEngine = class {
           }
         }
       );
-      const chunks = this.parseContentToChunks(
-        result.text,
-        promptConfigId,
-        documentId,
-        batch.pageStart,
-        batch.pageEnd
-      );
+      let chunks;
+      if ("data" in result && Array.isArray(result.data)) {
+        const sections = result.data;
+        chunks = sections.map((section, index) => ({
+          promptConfigId,
+          documentId,
+          chunkIndex: index,
+          chunkType: section.type,
+          searchContent: cleanForSearch(section.content),
+          displayContent: section.content,
+          sourcePageStart: section.page,
+          sourcePageEnd: section.page,
+          confidenceScore: section.confidence,
+          metadata: {
+            type: section.type,
+            pageRange: { start: section.page, end: section.page },
+            confidence: {
+              score: section.confidence,
+              category: section.confidence >= 0.8 ? "HIGH" : section.confidence >= 0.5 ? "MEDIUM" : "LOW"
+            },
+            parsedWithStructuredMarkers: true,
+            parsingMethod: "gemini_response_schema"
+          }
+        }));
+      } else {
+        const textResponse = result;
+        chunks = this.parseContentToChunks(
+          textResponse.text,
+          promptConfigId,
+          documentId,
+          batch.pageStart,
+          batch.pageEnd
+        );
+      }
       const docContext = {
         documentType: void 0,
         // Inferred from processing
@@ -2525,34 +2839,53 @@ var DiscoveryEngine = class {
     const { buffer, metadata } = await this.pdfProcessor.load(options.file);
     const fileUri = await this.gemini.uploadPdfBuffer(buffer, metadata.filename);
     const prompt = buildDiscoveryPrompt(options.documentTypeHint);
-    const response = await this.gemini.generateWithPdfUri(fileUri, prompt);
     let analysisResult;
     try {
-      let jsonStr = response.text;
-      const jsonMatch = jsonStr.match(/```json\s*([\s\S]*?)\s*```/) || jsonStr.match(/```\s*([\s\S]*?)\s*```/);
-      if (jsonMatch?.[1]) {
-        jsonStr = jsonMatch[1];
-      }
-      analysisResult = JSON.parse(jsonStr);
-      if (!analysisResult.documentType) {
-        throw new Error("Missing documentType in response");
-      }
-      if (!Array.isArray(analysisResult.specialInstructions)) {
-        analysisResult.specialInstructions = this.getDefaultInstructions();
-      }
-    } catch (parseError) {
-      this.logger.warn("Failed to parse discovery response as JSON, using defaults", {
-        error: parseError.message
-      });
+      const response = await this.gemini.generateStructuredWithPdf(
+        fileUri,
+        prompt,
+        DiscoveryResponseSchema
+      );
       analysisResult = {
-        documentType: options.documentTypeHint ?? "General",
-        documentTypeName: options.documentTypeHint ?? "General Document",
-        detectedElements: [],
-        specialInstructions: this.getDefaultInstructions(),
-        chunkStrategy: DEFAULT_CHUNK_STRATEGY,
-        confidence: 0.5,
-        reasoning: "Failed to parse AI response, using default configuration"
+        ...response.data,
+        detectedElements: response.data.detectedElements ?? []
       };
+      this.logger.debug("Structured discovery response received", {
+        documentType: analysisResult.documentType,
+        confidence: analysisResult.confidence
+      });
+    } catch (structuredError) {
+      this.logger.warn("Structured output failed, trying legacy parsing", {
+        error: structuredError.message
+      });
+      try {
+        const response = await this.gemini.generateWithPdfUri(fileUri, prompt);
+        let jsonStr = response.text;
+        const jsonMatch = jsonStr.match(/```json\s*([\s\S]*?)\s*```/) || jsonStr.match(/```\s*([\s\S]*?)\s*```/);
+        if (jsonMatch?.[1]) {
+          jsonStr = jsonMatch[1];
+        }
+        const parsed = JSON.parse(jsonStr);
+        analysisResult = DiscoveryResponseSchema.parse(parsed);
+      } catch (legacyError) {
+        this.logger.warn("All parsing methods failed, using defaults", {
+          error: legacyError.message
+        });
+        analysisResult = {
+          documentType: options.documentTypeHint ?? "General",
+          documentTypeName: options.documentTypeHint ?? "General Document",
+          detectedElements: [],
+          specialInstructions: this.getDefaultInstructions(),
+          chunkStrategy: {
+            maxTokens: DEFAULT_CHUNK_STRATEGY.maxTokens,
+            splitBy: DEFAULT_CHUNK_STRATEGY.splitBy,
+            preserveTables: DEFAULT_CHUNK_STRATEGY.preserveTables,
+            preserveLists: DEFAULT_CHUNK_STRATEGY.preserveLists
+          },
+          confidence: 0.5,
+          reasoning: "Failed to parse AI response, using default configuration"
+        };
+      }
     }
     const discoveryResult = {
       id: correlationId,
@@ -2693,7 +3026,9 @@ var ContextRAG = class {
       logging: {
         ...DEFAULT_LOG_CONFIG,
         ...userConfig.logging
-      }
+      },
+      ragEnhancement: userConfig.ragEnhancement,
+      useStructuredOutput: userConfig.useStructuredOutput ?? true
     };
   }
   /**