@claritylabs/cl-sdk 0.11.0 → 0.13.0

package/dist/index.mjs

@@ -1220,6 +1220,12 @@ var PremiumLineSchema = z16.object({
   line: z16.string(),
   amount: z16.string()
 });
+var AuxiliaryFactSchema = z16.object({
+  key: z16.string(),
+  value: z16.string(),
+  subject: z16.string().optional(),
+  context: z16.string().optional()
+});
 var BaseDocumentFields = {
   id: z16.string(),
   carrier: z16.string(),
@@ -1285,7 +1291,8 @@ var BaseDocumentFields = {
   individualClaims: z16.array(ClaimRecordSchema).optional(),
   experienceMod: ExperienceModSchema.optional(),
   cancellationNoticeDays: z16.number().optional(),
-  nonrenewalNoticeDays: z16.number().optional()
+  nonrenewalNoticeDays: z16.number().optional(),
+  supplementaryFacts: z16.array(AuxiliaryFactSchema).optional()
 };
 var PolicyDocumentSchema = z16.object({
   ...BaseDocumentFields,
@@ -1956,6 +1963,7 @@ function assembleDocument(documentId, documentType, memory) {
     ...sanitizeNulls(coverages ?? {}),
     ...sanitizeNulls(premium ?? {}),
     ...sanitizeNulls(supplementary ?? {}),
+    supplementaryFacts: supplementary?.auxiliaryFacts,
     endorsements: endorsements?.endorsements,
     exclusions: exclusions?.exclusions,
     conditions: conditions?.conditions,
@@ -2308,6 +2316,43 @@ Total Cost: ${doc.totalCost}` : ""}`,
       metadata: stringMetadata({ premium: doc.premium, documentType: doc.type })
     });
   }
+  const supplementaryLines = [
+    ...doc.claimsContacts?.map((contact) => `Claims Contact: ${[
+      contact.name,
+      contact.phone,
+      contact.email,
+      contact.hours
+    ].filter(Boolean).join(" | ")}`) ?? [],
+    ...doc.regulatoryContacts?.map((contact) => `Regulatory Contact: ${[
+      contact.name,
+      contact.phone,
+      contact.email
+    ].filter(Boolean).join(" | ")}`) ?? [],
+    ...doc.thirdPartyAdministrators?.map((contact) => `TPA: ${[
+      contact.name,
+      contact.phone,
+      contact.email
+    ].filter(Boolean).join(" | ")}`) ?? [],
+    ...doc.supplementaryFacts?.map((fact) => [
+      fact.subject ? `Subject: ${fact.subject}` : null,
+      `${fact.key}: ${fact.value}`,
+      fact.context ? `Context: ${fact.context}` : null
+    ].filter(Boolean).join(" | ")) ?? [],
+    doc.cancellationNoticeDays != null ? `Cancellation Notice Days: ${doc.cancellationNoticeDays}` : null,
+    doc.nonrenewalNoticeDays != null ? `Nonrenewal Notice Days: ${doc.nonrenewalNoticeDays}` : null
+  ].filter((line) => Boolean(line));
+  if (supplementaryLines.length > 0) {
+    chunks.push({
+      id: `${docId}:supplementary:0`,
+      documentId: docId,
+      type: "supplementary",
+      text: supplementaryLines.join("\n"),
+      metadata: stringMetadata({
+        documentType: doc.type,
+        supplementaryFactCount: doc.supplementaryFacts?.length
+      })
+    });
+  }
   return chunks;
 }
 
@@ -2391,6 +2436,32 @@ function mergeArrayPayload(existing, incoming, arrayKey, keyFn) {
   merged[arrayKey] = mergeUniqueObjects(existingItems, incomingItems, keyFn);
   return merged;
 }
+function mergeSupplementary(existing, incoming) {
+  const merged = mergeShallowPreferPresent(existing, incoming);
+  const mergeContactArray = (arrayKey) => {
+    const existingItems = Array.isArray(existing[arrayKey]) ? existing[arrayKey] : [];
+    const incomingItems = Array.isArray(incoming[arrayKey]) ? incoming[arrayKey] : [];
+    merged[arrayKey] = mergeUniqueObjects(existingItems, incomingItems, (item) => [
+      String(item.name ?? "").toLowerCase(),
+      String(item.phone ?? "").toLowerCase(),
+      String(item.email ?? "").toLowerCase(),
+      String(item.address ?? "").toLowerCase(),
+      String(item.type ?? "").toLowerCase()
+    ].join("|"));
+  };
+  mergeContactArray("regulatoryContacts");
+  mergeContactArray("claimsContacts");
+  mergeContactArray("thirdPartyAdministrators");
+  const existingFacts = Array.isArray(existing.auxiliaryFacts) ? existing.auxiliaryFacts : [];
+  const incomingFacts = Array.isArray(incoming.auxiliaryFacts) ? incoming.auxiliaryFacts : [];
+  merged.auxiliaryFacts = mergeUniqueObjects(existingFacts, incomingFacts, (item) => [
+    String(item.key ?? "").toLowerCase(),
+    String(item.value ?? "").toLowerCase(),
+    String(item.subject ?? "").toLowerCase(),
+    String(item.context ?? "").toLowerCase()
+  ].join("|"));
+  return merged;
+}
 function mergeExtractorResult(extractorName, existing, incoming) {
   if (!existing) return incoming;
   if (!incoming) return existing;
@@ -2401,9 +2472,10 @@ function mergeExtractorResult(extractorName, existing, incoming) {
     case "carrier_info":
     case "named_insured":
     case "loss_history":
-    case "supplementary":
     case "premium_breakdown":
       return mergeShallowPreferPresent(current, next);
+    case "supplementary":
+      return mergeSupplementary(current, next);
     case "coverage_limits":
       return mergeCoverageLimits(current, next);
     case "declarations":
@@ -4000,15 +4072,22 @@ var ContactSchema2 = z33.object({
   address: z33.string().optional().describe("Mailing address"),
   type: z33.string().optional().describe("Contact type, e.g. 'State Department of Insurance'")
 });
+var AuxiliaryFactSchema2 = z33.object({
+  key: z33.string().describe("Normalized machine-readable fact key, e.g. 'policyholder_age' or 'insured_name'"),
+  value: z33.string().describe("Concrete extracted fact value"),
+  subject: z33.string().optional().describe("Person, entity, vehicle, property, or schedule item this fact belongs to"),
+  context: z33.string().optional().describe("Short disambiguating context, such as 'Driver Schedule' or 'Named Insured'")
+});
 var SupplementarySchema = z33.object({
   regulatoryContacts: z33.array(ContactSchema2).optional().describe("Regulatory body contacts (state department of insurance, ombudsman)"),
   claimsContacts: z33.array(ContactSchema2).optional().describe("Claims reporting contacts and instructions"),
   thirdPartyAdministrators: z33.array(ContactSchema2).optional().describe("Third-party administrators for claims handling"),
   cancellationNoticeDays: z33.number().optional().describe("Required notice period for cancellation in days"),
-  nonrenewalNoticeDays: z33.number().optional().describe("Required notice period for nonrenewal in days")
+  nonrenewalNoticeDays: z33.number().optional().describe("Required notice period for nonrenewal in days"),
+  auxiliaryFacts: z33.array(AuxiliaryFactSchema2).optional().describe("Additional retrieval-only facts that do not fit the strict primary schema")
 });
 function buildSupplementaryPrompt() {
-  return `You are an expert insurance document analyst. Extract supplementary and regulatory information from this document.
+  return `You are an expert insurance document analyst. Extract supplementary, retrieval-only information from this document.
 
 Focus on:
 - Regulatory contacts: state department of insurance, regulatory bodies, ombudsman offices \u2014 with phone, email, address
@@ -4018,9 +4097,19 @@ Focus on:
 - Nonrenewal notice period in days
 - Complaint filing procedures and contacts
 - Governing law or jurisdiction provisions
+- Additional policy-specific facts that are useful for memory and retrieval even if they do not belong in the strict primary schema
 
 Look for regulatory notices, complaint contact sections, claims reporting instructions, and cancellation/nonrenewal provisions throughout the document.
 
+For auxiliaryFacts:
+- Capture concrete, policy-specific facts as structured key/value pairs.
+- Prioritize facts that agents may need later but that are often omitted from strict schemas: policyholder names, insured person names, driver names, ages, dates of birth, marital status, garaging information, lienholders, household members, vehicle assignments, schedule row details, and other discrete identifiers.
+- Use short normalized keys like "policyholder_name", "policyholder_age", "insured_name", "driver_age", "driver_date_of_birth", "garaging_zip", "vehicle_principal_driver".
+- Use subject when the fact belongs to a specific person, vehicle, property, or scheduled item.
+- Do not invent facts.
+- Do not include vague boilerplate or generic form language.
+- Do not repeat large narrative excerpts; keep facts atomic.
+
 Return JSON only.`;
 }
 
@@ -4747,6 +4836,28 @@ function createExtractor(config) {
           mergeMemoryResult(result.name, result.data, memory);
         }
       }
+      const supplementaryExtractor = getExtractor("supplementary");
+      if (supplementaryExtractor) {
+        onProgress?.("Extracting supplementary retrieval facts...");
+        try {
+          const supplementaryResult = await runExtractor({
+            name: "supplementary",
+            prompt: supplementaryExtractor.buildPrompt(),
+            schema: supplementaryExtractor.schema,
+            pdfBase64,
+            startPage: 1,
+            endPage: pageCount,
+            generateObject,
+            convertPdfToImages,
+            maxTokens: supplementaryExtractor.maxTokens ?? 4096,
+            providerOptions
+          });
+          trackUsage(supplementaryResult.usage);
+          mergeMemoryResult(supplementaryResult.name, supplementaryResult.data, memory);
+        } catch (error) {
+          await log?.(`Supplementary extractor failed: ${error}`);
+        }
+      }
       await pipelineCtx.save("extract", {
         id,
         pageCount,
@@ -6336,7 +6447,7 @@ Respond with the explanation text only \u2014 no JSON, no field ID, no extra for
 }
 
 // src/prompts/query/classify.ts
-function buildQueryClassifyPrompt(question, conversationContext) {
+function buildQueryClassifyPrompt(question, conversationContext, attachmentContext) {
   return `You are a query classifier for an insurance document intelligence system.
 
 Analyze the user's question and produce a structured classification.
@@ -6346,6 +6457,9 @@ ${question}
 ${conversationContext ? `
 CONVERSATION CONTEXT:
 ${conversationContext}` : ""}
+${attachmentContext ? `
+ATTACHMENT CONTEXT:
+${attachmentContext}` : ""}
 
 INSTRUCTIONS:
 
@@ -6366,6 +6480,7 @@ INSTRUCTIONS:
    - requiresDocumentLookup: true if a specific document needs to be fetched by ID/number/carrier
    - requiresChunkSearch: true if semantic search over document chunks is needed
    - requiresConversationHistory: true if the question references prior conversation
+   - If the user's attachment already contains critical facts, still request chunk/document lookup when policy or quote details should be cross-checked against stored records
 
 CHUNK TYPES (for chunkTypes filter):
 carrier_info, named_insured, coverage, endorsement, exclusion, condition, section, declaration, loss_history, premium, supplementary
@@ -6407,6 +6522,16 @@ var QueryIntentSchema = z35.enum([
   "claims_inquiry",
   "general_knowledge"
 ]);
+var QueryAttachmentKindSchema = z35.enum(["image", "pdf", "text"]);
+var QueryAttachmentSchema = z35.object({
+  id: z35.string().optional().describe("Optional stable attachment ID from the caller"),
+  kind: QueryAttachmentKindSchema,
+  name: z35.string().optional().describe("Original filename or user-facing label"),
+  mimeType: z35.string().optional().describe("MIME type such as image/jpeg or application/pdf"),
+  base64: z35.string().optional().describe("Base64-encoded file content for image/pdf attachments"),
+  text: z35.string().optional().describe("Plain-text attachment content when available"),
+  description: z35.string().optional().describe("Caller-provided description of the attachment")
+});
 var SubQuestionSchema = z35.object({
   question: z35.string().describe("Atomic sub-question to retrieve and answer independently"),
   intent: QueryIntentSchema,
@@ -6427,14 +6552,21 @@ var QueryClassifyResultSchema = z35.object({
   requiresConversationHistory: z35.boolean().describe("Whether conversation history is relevant")
 });
 var EvidenceItemSchema = z35.object({
-  source: z35.enum(["chunk", "document", "conversation"]),
+  source: z35.enum(["chunk", "document", "conversation", "attachment"]),
   chunkId: z35.string().optional(),
   documentId: z35.string().optional(),
   turnId: z35.string().optional(),
+  attachmentId: z35.string().optional(),
   text: z35.string().describe("Text excerpt from the source"),
   relevance: z35.number().min(0).max(1),
   metadata: z35.array(z35.object({ key: z35.string(), value: z35.string() })).optional()
 });
+var AttachmentInterpretationSchema = z35.object({
+  summary: z35.string().describe("Concise summary of what the attachment shows or contains"),
+  extractedFacts: z35.array(z35.string()).describe("Specific observable or document facts grounded in the attachment"),
+  recommendedFocus: z35.array(z35.string()).describe("Important details to incorporate when answering follow-up questions"),
+  confidence: z35.number().min(0).max(1)
+});
 var RetrievalResultSchema = z35.object({
   subQuestion: z35.string(),
   evidence: z35.array(EvidenceItemSchema)
@@ -6755,7 +6887,7 @@ async function verify(originalQuestion, subAnswers, allEvidence, config) {
 
 // src/query/quality.ts
 function sourceIdForEvidence(evidence) {
-  return evidence.chunkId ?? evidence.documentId ?? evidence.turnId;
+  return evidence.chunkId ?? evidence.documentId ?? evidence.turnId ?? evidence.attachmentId;
 }
 function citationSourceId(citation) {
   return citation.chunkId || citation.documentId;
@@ -6859,6 +6991,152 @@ function buildQueryReviewReport(params) {
   };
 }
 
+// src/prompts/query/interpret-attachment.ts
+function buildInterpretAttachmentPrompt(question, attachment) {
+  const attachmentLabel = attachment.name ?? attachment.id ?? "attachment";
+  const descriptor = [
+    `Attachment: ${attachmentLabel}`,
+    `Kind: ${attachment.kind}`,
+    attachment.mimeType ? `MIME type: ${attachment.mimeType}` : null,
+    attachment.description ? `Caller description: ${attachment.description}` : null
+  ].filter(Boolean).join("\n");
+  return `You are interpreting a user-supplied attachment for an insurance-support question.
+
+USER QUESTION:
+${question}
+
+ATTACHMENT METADATA:
+${descriptor}
+
+${attachment.kind === "text" && attachment.text ? `ATTACHMENT TEXT:
+${attachment.text}
+` : "The attachment content is provided separately as a file or image input.\n"}
+INSTRUCTIONS:
+1. Describe what the attachment appears to show or contain in a concise summary.
+2. Extract concrete facts that may matter when answering the user's question.
+3. Note the most important details to carry forward into follow-up questions.
+4. If the attachment is a document, identify the key business or insurance details visible.
+5. If the attachment is a photo of damage or a real-world issue, describe the observable issue without guessing beyond what is visible.
+6. Do not invent unreadable text. If something is unclear, say so in the summary or extracted facts.
+
+Respond with the structured interpretation.`;
+}
+
+// src/query/multimodal.ts
+function attachmentSourceId(attachment, index) {
+  return attachment.id ?? `attachment-${index + 1}`;
+}
+function buildAttachmentProviderOptions(attachment, providerOptions) {
+  const merged = {
+    ...providerOptions,
+    attachments: [
+      {
+        kind: attachment.kind,
+        name: attachment.name,
+        mimeType: attachment.mimeType,
+        base64: attachment.base64,
+        text: attachment.text,
+        description: attachment.description
+      }
+    ]
+  };
+  if (attachment.kind === "pdf" && attachment.base64) {
+    merged.pdfBase64 = attachment.base64;
+  }
+  if (attachment.kind === "image" && attachment.base64) {
+    merged.images = [
+      {
+        imageBase64: attachment.base64,
+        mimeType: attachment.mimeType ?? "image/jpeg"
+      }
+    ];
+  }
+  return merged;
+}
+function buildAttachmentEvidenceText(attachment, interpretation) {
+  const lines = [
+    `Attachment kind: ${attachment.kind}`,
+    attachment.name ? `Attachment name: ${attachment.name}` : null,
+    attachment.mimeType ? `MIME type: ${attachment.mimeType}` : null,
+    attachment.description ? `Caller description: ${attachment.description}` : null,
+    `Summary: ${interpretation.summary}`,
+    interpretation.extractedFacts.length > 0 ? `Extracted facts:
+${interpretation.extractedFacts.map((fact) => `- ${fact}`).join("\n")}` : null,
+    interpretation.recommendedFocus.length > 0 ? `Important follow-up details:
+${interpretation.recommendedFocus.map((item) => `- ${item}`).join("\n")}` : null,
+    attachment.kind === "text" && attachment.text ? `Original text:
+${attachment.text}` : null
+  ];
+  return lines.filter(Boolean).join("\n");
+}
+async function interpretAttachments(params) {
+  const { attachments = [], question, generateObject, providerOptions, log, onUsage } = params;
+  if (attachments.length === 0) {
+    return { evidence: [] };
+  }
+  const evidence = [];
+  for (const [index, attachment] of attachments.entries()) {
+    const id = attachmentSourceId(attachment, index);
+    if (attachment.kind === "text" && attachment.text) {
+      const textEvidence = buildAttachmentEvidenceText(attachment, {
+        summary: attachment.description ?? "User supplied text context.",
+        extractedFacts: [attachment.text],
+        recommendedFocus: [],
+        confidence: 1
+      });
+      evidence.push({
+        source: "attachment",
+        attachmentId: id,
+        chunkId: id,
+        documentId: id,
+        text: textEvidence,
+        relevance: 0.95,
+        metadata: [
+          { key: "kind", value: attachment.kind },
+          ...attachment.name ? [{ key: "name", value: attachment.name }] : []
+        ]
+      });
+      continue;
+    }
+    const prompt = buildInterpretAttachmentPrompt(question, attachment);
+    const { object, usage } = await safeGenerateObject(
+      generateObject,
+      {
+        prompt,
+        schema: AttachmentInterpretationSchema,
+        maxTokens: 2048,
+        providerOptions: buildAttachmentProviderOptions(attachment, providerOptions)
+      },
+      {
+        fallback: {
+          summary: attachment.description ?? `User supplied ${attachment.kind} attachment.`,
+          extractedFacts: [],
+          recommendedFocus: [],
+          confidence: 0.2
+        },
+        log,
+        onError: (error, attempt) => log?.(`Attachment interpretation attempt ${attempt + 1} failed for "${attachment.name ?? id}": ${error}`)
+      }
+    );
+    onUsage?.(usage);
+    evidence.push({
+      source: "attachment",
+      attachmentId: id,
+      chunkId: id,
+      documentId: id,
+      text: buildAttachmentEvidenceText(attachment, object),
+      relevance: Math.max(0.7, object.confidence),
+      metadata: [
+        { key: "kind", value: attachment.kind },
+        ...attachment.name ? [{ key: "name", value: attachment.name }] : []
+      ]
+    });
+  }
+  const contextSummary = evidence.map((item, index) => `Attachment ${index + 1}:
+${item.text}`).join("\n\n");
+  return { evidence, contextSummary };
+}
+
 // src/query/coordinator.ts
 function createQueryAgent(config) {
   const {
@@ -6886,13 +7164,23 @@ function createQueryAgent(config) {
   }
   async function query(input) {
     totalUsage = { inputTokens: 0, outputTokens: 0 };
-    const { question, conversationId, context } = input;
+    const { question, conversationId, context, attachments } = input;
     const pipelineCtx = createPipelineContext({
       id: `query-${Date.now()}`
     });
+    onProgress?.("Interpreting attachments...");
+    const { evidence: attachmentEvidence, contextSummary: attachmentContext } = await interpretAttachments({
+      attachments,
+      question,
+      generateObject,
+      providerOptions,
+      log,
+      onUsage: trackUsage
+    });
+    await pipelineCtx.save("attachments", { attachmentEvidence });
     onProgress?.("Classifying query...");
-    const classification = await classify(question, conversationId);
-    await pipelineCtx.save("classify", { classification });
+    const classification = await classify(question, conversationId, attachmentContext);
+    await pipelineCtx.save("classify", { classification, attachmentEvidence });
     onProgress?.(`Retrieving evidence for ${classification.subQuestions.length} sub-question(s)...`);
     const retrieverConfig = {
       documentStore,
@@ -6905,8 +7193,8 @@ function createQueryAgent(config) {
         (sq) => limit(() => retrieve(sq, conversationId, retrieverConfig))
       )
     );
-    const allEvidence = retrievalResults.flatMap((r) => r.evidence);
-    await pipelineCtx.save("retrieve", { classification, evidence: allEvidence });
+    const allEvidence = [...attachmentEvidence, ...retrievalResults.flatMap((r) => r.evidence)];
+    await pipelineCtx.save("retrieve", { classification, attachmentEvidence, evidence: allEvidence });
     onProgress?.("Reasoning over evidence...");
     const reasonerConfig = { generateObject, providerOptions };
     const reasonResults = await Promise.allSettled(
@@ -6915,7 +7203,7 @@ function createQueryAgent(config) {
           const { subAnswer, usage } = await reason(
             sq.question,
             sq.intent,
-            retrievalResults[i].evidence,
+            [...attachmentEvidence, ...retrievalResults[i].evidence],
             reasonerConfig
           );
           trackUsage(usage);
@@ -6939,7 +7227,7 @@ function createQueryAgent(config) {
         });
       }
     }
-    await pipelineCtx.save("reason", { classification, evidence: allEvidence, subAnswers });
+    await pipelineCtx.save("reason", { classification, attachmentEvidence, evidence: allEvidence, subAnswers });
     onProgress?.("Verifying answer grounding...");
     const verifierConfig = { generateObject, providerOptions };
     const verifyRounds = [];
@@ -6987,7 +7275,7 @@ function createQueryAgent(config) {
                 const { subAnswer, usage: u } = await reason(
                   sq.question,
                   sq.intent,
-                  retryRetrievals[i].evidence,
+                  [...attachmentEvidence, ...retryRetrievals[i].evidence],
                   reasonerConfig
                 );
                 trackUsage(u);
@@ -7022,6 +7310,7 @@ function createQueryAgent(config) {
     });
     await pipelineCtx.save("review", {
       classification,
+      attachmentEvidence,
       evidence: allEvidence,
       subAnswers,
       reviewReport
@@ -7054,7 +7343,7 @@ function createQueryAgent(config) {
     }
     return { ...queryResult, tokenUsage: totalUsage, reviewReport };
   }
-  async function classify(question, conversationId) {
+  async function classify(question, conversationId, attachmentContext) {
     let conversationContext;
     if (conversationId) {
       try {
@@ -7065,7 +7354,7 @@ function createQueryAgent(config) {
       } catch {
       }
     }
-    const prompt = buildQueryClassifyPrompt(question, conversationContext);
+    const prompt = buildQueryClassifyPrompt(question, conversationContext, attachmentContext);
     const { object, usage } = await safeGenerateObject(
       generateObject,
       {
@@ -7280,9 +7569,11 @@ export {
   ApplicationQualityReportSchema,
   ApplicationQualityRoundSchema,
   ApplicationStateSchema,
+  AttachmentInterpretationSchema,
   AuditTypeSchema,
   AutoFillMatchSchema,
   AutoFillResultSchema,
+  AuxiliaryFactSchema,
   BOAT_TYPES,
   BindingAuthoritySchema,
   BoatTypeSchema,
@@ -7401,6 +7692,8 @@ export {
   ProducerInfoSchema,
   ProfessionalLiabilityDeclarationsSchema,
   QUOTE_SECTION_TYPES,
+  QueryAttachmentKindSchema,
+  QueryAttachmentSchema,
   QueryClassifyResultSchema,
   QueryIntentSchema,
   QueryResultSchema,
@@ -7459,6 +7752,7 @@ export {
   buildFormattingPrompt,
   buildIdentityPrompt,
   buildIntentPrompt,
+  buildInterpretAttachmentPrompt,
   buildLookupFillPrompt,
   buildQueryClassifyPrompt,
   buildQuestionBatchPrompt,