@minded-ai/mindedjs 3.0.8-beta.12 → 3.1.9-beta.1

package/src/internalTools/documentExtraction/extractStructuredData.ts

@@ -0,0 +1,140 @@
+import { ZodType } from 'zod';
+import { logger } from '../../utils/logger';
+import { BaseLanguageModel } from '@langchain/core/language_models/base';
+
+/**
+ * Extract structured or unstructured data from content using AI.
+ *
+ * This function processes string content with an LLM to extract information:
+ * - With schema: Returns structured data matching the Zod schema
+ * - Without schema: Returns LLM's text analysis of the content based on the provided prompt
+ *
+ * @param options - Extraction options
+ * @param options.content - The content to extract information from
+ * @param options.llm - Language model instance for AI-powered extraction
+ * @param options.schema - Optional Zod schema defining the structure of data to extract
+ * @param options.prompt - Optional prompt to guide the extraction (defaults to expert data-extraction assistant)
+ * @param options.sessionId - Unique session identifier for logging and tracking
+ *
+ * @returns Promise resolving to:
+ *   - Structured data of type T if schema is provided
+ *   - String or object containing LLM analysis if no schema provided
+ *
+ * @throws {Error} If LLM is not provided or doesn't support structured output
+ * @throws {Error} If LLM extraction fails
+ *
+ * @example
+ * ```typescript
+ * import { extractStructuredDataFromString } from '@minded-ai/mindedjs';
+ * import { z } from 'zod';
+ *
+ * // Extract structured data with schema
+ * const result1 = await extractStructuredDataFromString({
+ *   content: 'Invoice #12345\nTotal: $500.00\nDate: 2024-01-15',
+ *   llm: agent.llm,
+ *   schema: z.object({
+ *     invoiceNumber: z.string(),
+ *     totalAmount: z.number(),
+ *     date: z.string(),
+ *   }),
+ * });
+ * // result1: {"invoiceNumber": "12345", "totalAmount": 500, "date": "2024-01-15"}
+ *
+ * // Extract unstructured data with custom prompt (returns object format as string)
+ * const result2 = await extractStructuredDataFromString({
+ *   content: 'Invoice #12345\nTotal: $500.00\nDate: 2024-01-15',
+ *   llm: agent.llm,
+ *   prompt: 'Extract the invoice number, total amount, and date. Return as an object with keys: invoiceNumber, totalAmount, date',
+ * });
+ * // result2: {"invoiceNumber": "12345", "totalAmount": 500, "date": "2024-01-15"}
+ * ```
+ */
+export async function extractStructuredDataFromString<T extends Record<string, any>>({
+  content,
+  llm,
+  schema,
+  prompt,
+  sessionId,
+}: {
+  content: string;
+  llm: BaseLanguageModel;
+  schema?: ZodType<T>;
+  prompt?: string;
+  sessionId: string;
+}): Promise<T | string> {
+  if (!llm) {
+    throw new Error(
+      'LLM instance is required for structured data extraction. Please provide an LLM when calling extractStructuredDataFromString.',
+    );
+  }
+
+  const messages = [
+    {
+      role: 'system',
+      content:
+        'You are an expert data-extraction assistant. ' +
+        'Extract the requested information from the provided document content. ' +
+        'If you cannot find a value for a required field, use "N/A" or a descriptive placeholder. ' +
+        'Be accurate and thorough in your extraction. ' +
+        'The user prompt may include information needed to extract structured data from the document.',
+    },
+    {
+      role: 'user',
+      content: (prompt ? prompt + ' ' : '') + `Please analyze and extract information from this document:\n\n${content}`,
+    },
+  ];
+
+  try {
+    if (schema) {
+      if (!llm.withStructuredOutput) {
+        throw new Error('The provided LLM does not support structured output. Please use a compatible LLM instance.');
+      }
+      const structuredLlm = llm.withStructuredOutput<T>(schema);
+
+      const result = await structuredLlm.invoke(messages);
+
+      logger.debug({
+        msg: '[DocumentProcessor] Structured data extraction completed',
+        sessionId,
+        extractedData: JSON.stringify(result),
+      });
+
+      return result;
+    } else {
+      // Without schema, return the LLM's text response
+      const response = await llm.invoke(messages);
+
+      // Extract the text content from the response
+      let textContent: string;
+      if (typeof response.content === 'string') {
+        textContent = response.content;
+      } else if (Array.isArray(response.content) && response.content.length > 0) {
+        // Handle array of content blocks
+        textContent = response.content.map((block: any) => (typeof block === 'string' ? block : block.text || '')).join('\n');
+      } else {
+        textContent = String(response.content);
+      }
+
+      // Try to parse the response in case it is a valid JSON object.
+      const result = parseJSONSafe(textContent);
+
+      logger.debug({
+        msg: '[DocumentProcessor] Unstructured data extraction completed',
+        sessionId,
+        extractedData: textContent,
+      });
+
+      return result;
+    }
+  } catch (err) {
+    throw new Error(`LLM extraction failed: ${err instanceof Error ? err.message : String(err)}`);
+  }
+}
+
+function parseJSONSafe(str: string) {
+  try {
+    return JSON.parse(str);
+  } catch {
+    return str;
+  }
+}