@claritylabs/cl-sdk 0.11.0 → 0.12.0

package/dist/index.mjs

@@ -6336,7 +6336,7 @@ Respond with the explanation text only \u2014 no JSON, no field ID, no extra for
 }
 
 // src/prompts/query/classify.ts
-function buildQueryClassifyPrompt(question, conversationContext) {
+function buildQueryClassifyPrompt(question, conversationContext, attachmentContext) {
   return `You are a query classifier for an insurance document intelligence system.
 
 Analyze the user's question and produce a structured classification.
@@ -6346,6 +6346,9 @@ ${question}
 ${conversationContext ? `
 CONVERSATION CONTEXT:
 ${conversationContext}` : ""}
+${attachmentContext ? `
+ATTACHMENT CONTEXT:
+${attachmentContext}` : ""}
 
 INSTRUCTIONS:
 
@@ -6366,6 +6369,7 @@ INSTRUCTIONS:
    - requiresDocumentLookup: true if a specific document needs to be fetched by ID/number/carrier
    - requiresChunkSearch: true if semantic search over document chunks is needed
    - requiresConversationHistory: true if the question references prior conversation
+   - If the user's attachment already contains critical facts, still request chunk/document lookup when policy or quote details should be cross-checked against stored records
 
 CHUNK TYPES (for chunkTypes filter):
 carrier_info, named_insured, coverage, endorsement, exclusion, condition, section, declaration, loss_history, premium, supplementary
@@ -6407,6 +6411,16 @@ var QueryIntentSchema = z35.enum([
   "claims_inquiry",
   "general_knowledge"
 ]);
+var QueryAttachmentKindSchema = z35.enum(["image", "pdf", "text"]);
+var QueryAttachmentSchema = z35.object({
+  id: z35.string().optional().describe("Optional stable attachment ID from the caller"),
+  kind: QueryAttachmentKindSchema,
+  name: z35.string().optional().describe("Original filename or user-facing label"),
+  mimeType: z35.string().optional().describe("MIME type such as image/jpeg or application/pdf"),
+  base64: z35.string().optional().describe("Base64-encoded file content for image/pdf attachments"),
+  text: z35.string().optional().describe("Plain-text attachment content when available"),
+  description: z35.string().optional().describe("Caller-provided description of the attachment")
+});
 var SubQuestionSchema = z35.object({
   question: z35.string().describe("Atomic sub-question to retrieve and answer independently"),
   intent: QueryIntentSchema,
@@ -6427,14 +6441,21 @@ var QueryClassifyResultSchema = z35.object({
   requiresConversationHistory: z35.boolean().describe("Whether conversation history is relevant")
 });
 var EvidenceItemSchema = z35.object({
-  source: z35.enum(["chunk", "document", "conversation"]),
+  source: z35.enum(["chunk", "document", "conversation", "attachment"]),
   chunkId: z35.string().optional(),
   documentId: z35.string().optional(),
   turnId: z35.string().optional(),
+  attachmentId: z35.string().optional(),
   text: z35.string().describe("Text excerpt from the source"),
   relevance: z35.number().min(0).max(1),
   metadata: z35.array(z35.object({ key: z35.string(), value: z35.string() })).optional()
 });
+var AttachmentInterpretationSchema = z35.object({
+  summary: z35.string().describe("Concise summary of what the attachment shows or contains"),
+  extractedFacts: z35.array(z35.string()).describe("Specific observable or document facts grounded in the attachment"),
+  recommendedFocus: z35.array(z35.string()).describe("Important details to incorporate when answering follow-up questions"),
+  confidence: z35.number().min(0).max(1)
+});
 var RetrievalResultSchema = z35.object({
   subQuestion: z35.string(),
   evidence: z35.array(EvidenceItemSchema)
@@ -6755,7 +6776,7 @@ async function verify(originalQuestion, subAnswers, allEvidence, config) {
 
 // src/query/quality.ts
 function sourceIdForEvidence(evidence) {
-  return evidence.chunkId ?? evidence.documentId ?? evidence.turnId;
+  return evidence.chunkId ?? evidence.documentId ?? evidence.turnId ?? evidence.attachmentId;
 }
 function citationSourceId(citation) {
   return citation.chunkId || citation.documentId;
@@ -6859,6 +6880,152 @@ function buildQueryReviewReport(params) {
   };
 }
 
+// src/prompts/query/interpret-attachment.ts
+function buildInterpretAttachmentPrompt(question, attachment) {
+  const attachmentLabel = attachment.name ?? attachment.id ?? "attachment";
+  const descriptor = [
+    `Attachment: ${attachmentLabel}`,
+    `Kind: ${attachment.kind}`,
+    attachment.mimeType ? `MIME type: ${attachment.mimeType}` : null,
+    attachment.description ? `Caller description: ${attachment.description}` : null
+  ].filter(Boolean).join("\n");
+  return `You are interpreting a user-supplied attachment for an insurance-support question.
+
+USER QUESTION:
+${question}
+
+ATTACHMENT METADATA:
+${descriptor}
+
+${attachment.kind === "text" && attachment.text ? `ATTACHMENT TEXT:
+${attachment.text}
+` : "The attachment content is provided separately as a file or image input.\n"}
+INSTRUCTIONS:
+1. Describe what the attachment appears to show or contain in a concise summary.
+2. Extract concrete facts that may matter when answering the user's question.
+3. Note the most important details to carry forward into follow-up questions.
+4. If the attachment is a document, identify the key business or insurance details visible.
+5. If the attachment is a photo of damage or a real-world issue, describe the observable issue without guessing beyond what is visible.
+6. Do not invent unreadable text. If something is unclear, say so in the summary or extracted facts.
+
+Respond with the structured interpretation.`;
+}
+
+// src/query/multimodal.ts
+function attachmentSourceId(attachment, index) {
+  return attachment.id ?? `attachment-${index + 1}`;
+}
+function buildAttachmentProviderOptions(attachment, providerOptions) {
+  const merged = {
+    ...providerOptions,
+    attachments: [
+      {
+        kind: attachment.kind,
+        name: attachment.name,
+        mimeType: attachment.mimeType,
+        base64: attachment.base64,
+        text: attachment.text,
+        description: attachment.description
+      }
+    ]
+  };
+  if (attachment.kind === "pdf" && attachment.base64) {
+    merged.pdfBase64 = attachment.base64;
+  }
+  if (attachment.kind === "image" && attachment.base64) {
+    merged.images = [
+      {
+        imageBase64: attachment.base64,
+        mimeType: attachment.mimeType ?? "image/jpeg"
+      }
+    ];
+  }
+  return merged;
+}
+function buildAttachmentEvidenceText(attachment, interpretation) {
+  const lines = [
+    `Attachment kind: ${attachment.kind}`,
+    attachment.name ? `Attachment name: ${attachment.name}` : null,
+    attachment.mimeType ? `MIME type: ${attachment.mimeType}` : null,
+    attachment.description ? `Caller description: ${attachment.description}` : null,
+    `Summary: ${interpretation.summary}`,
+    interpretation.extractedFacts.length > 0 ? `Extracted facts:
+${interpretation.extractedFacts.map((fact) => `- ${fact}`).join("\n")}` : null,
+    interpretation.recommendedFocus.length > 0 ? `Important follow-up details:
+${interpretation.recommendedFocus.map((item) => `- ${item}`).join("\n")}` : null,
+    attachment.kind === "text" && attachment.text ? `Original text:
+${attachment.text}` : null
+  ];
+  return lines.filter(Boolean).join("\n");
+}
+async function interpretAttachments(params) {
+  const { attachments = [], question, generateObject, providerOptions, log, onUsage } = params;
+  if (attachments.length === 0) {
+    return { evidence: [] };
+  }
+  const evidence = [];
+  for (const [index, attachment] of attachments.entries()) {
+    const id = attachmentSourceId(attachment, index);
+    if (attachment.kind === "text" && attachment.text) {
+      const textEvidence = buildAttachmentEvidenceText(attachment, {
+        summary: attachment.description ?? "User supplied text context.",
+        extractedFacts: [attachment.text],
+        recommendedFocus: [],
+        confidence: 1
+      });
+      evidence.push({
+        source: "attachment",
+        attachmentId: id,
+        chunkId: id,
+        documentId: id,
+        text: textEvidence,
+        relevance: 0.95,
+        metadata: [
+          { key: "kind", value: attachment.kind },
+          ...attachment.name ? [{ key: "name", value: attachment.name }] : []
+        ]
+      });
+      continue;
+    }
+    const prompt = buildInterpretAttachmentPrompt(question, attachment);
+    const { object, usage } = await safeGenerateObject(
+      generateObject,
+      {
+        prompt,
+        schema: AttachmentInterpretationSchema,
+        maxTokens: 2048,
+        providerOptions: buildAttachmentProviderOptions(attachment, providerOptions)
+      },
+      {
+        fallback: {
+          summary: attachment.description ?? `User supplied ${attachment.kind} attachment.`,
+          extractedFacts: [],
+          recommendedFocus: [],
+          confidence: 0.2
+        },
+        log,
+        onError: (error, attempt) => log?.(`Attachment interpretation attempt ${attempt + 1} failed for "${attachment.name ?? id}": ${error}`)
+      }
+    );
+    onUsage?.(usage);
+    evidence.push({
+      source: "attachment",
+      attachmentId: id,
+      chunkId: id,
+      documentId: id,
+      text: buildAttachmentEvidenceText(attachment, object),
+      relevance: Math.max(0.7, object.confidence),
+      metadata: [
+        { key: "kind", value: attachment.kind },
+        ...attachment.name ? [{ key: "name", value: attachment.name }] : []
+      ]
+    });
+  }
+  const contextSummary = evidence.map((item, index) => `Attachment ${index + 1}:
+${item.text}`).join("\n\n");
+  return { evidence, contextSummary };
+}
+
 // src/query/coordinator.ts
 function createQueryAgent(config) {
   const {
@@ -6886,13 +7053,23 @@ function createQueryAgent(config) {
   }
   async function query(input) {
     totalUsage = { inputTokens: 0, outputTokens: 0 };
-    const { question, conversationId, context } = input;
+    const { question, conversationId, context, attachments } = input;
     const pipelineCtx = createPipelineContext({
       id: `query-${Date.now()}`
     });
+    onProgress?.("Interpreting attachments...");
+    const { evidence: attachmentEvidence, contextSummary: attachmentContext } = await interpretAttachments({
+      attachments,
+      question,
+      generateObject,
+      providerOptions,
+      log,
+      onUsage: trackUsage
+    });
+    await pipelineCtx.save("attachments", { attachmentEvidence });
     onProgress?.("Classifying query...");
-    const classification = await classify(question, conversationId);
-    await pipelineCtx.save("classify", { classification });
+    const classification = await classify(question, conversationId, attachmentContext);
+    await pipelineCtx.save("classify", { classification, attachmentEvidence });
     onProgress?.(`Retrieving evidence for ${classification.subQuestions.length} sub-question(s)...`);
     const retrieverConfig = {
       documentStore,
@@ -6905,8 +7082,8 @@ function createQueryAgent(config) {
         (sq) => limit(() => retrieve(sq, conversationId, retrieverConfig))
       )
     );
-    const allEvidence = retrievalResults.flatMap((r) => r.evidence);
-    await pipelineCtx.save("retrieve", { classification, evidence: allEvidence });
+    const allEvidence = [...attachmentEvidence, ...retrievalResults.flatMap((r) => r.evidence)];
+    await pipelineCtx.save("retrieve", { classification, attachmentEvidence, evidence: allEvidence });
     onProgress?.("Reasoning over evidence...");
     const reasonerConfig = { generateObject, providerOptions };
     const reasonResults = await Promise.allSettled(
@@ -6915,7 +7092,7 @@ function createQueryAgent(config) {
           const { subAnswer, usage } = await reason(
             sq.question,
             sq.intent,
-            retrievalResults[i].evidence,
+            [...attachmentEvidence, ...retrievalResults[i].evidence],
             reasonerConfig
           );
           trackUsage(usage);
@@ -6939,7 +7116,7 @@ function createQueryAgent(config) {
         });
       }
     }
-    await pipelineCtx.save("reason", { classification, evidence: allEvidence, subAnswers });
+    await pipelineCtx.save("reason", { classification, attachmentEvidence, evidence: allEvidence, subAnswers });
     onProgress?.("Verifying answer grounding...");
     const verifierConfig = { generateObject, providerOptions };
     const verifyRounds = [];
@@ -6987,7 +7164,7 @@ function createQueryAgent(config) {
                 const { subAnswer, usage: u } = await reason(
                   sq.question,
                   sq.intent,
-                  retryRetrievals[i].evidence,
+                  [...attachmentEvidence, ...retryRetrievals[i].evidence],
                   reasonerConfig
                 );
                 trackUsage(u);
@@ -7022,6 +7199,7 @@ function createQueryAgent(config) {
     });
     await pipelineCtx.save("review", {
       classification,
+      attachmentEvidence,
       evidence: allEvidence,
       subAnswers,
       reviewReport
@@ -7054,7 +7232,7 @@ function createQueryAgent(config) {
     }
     return { ...queryResult, tokenUsage: totalUsage, reviewReport };
   }
-  async function classify(question, conversationId) {
+  async function classify(question, conversationId, attachmentContext) {
     let conversationContext;
     if (conversationId) {
       try {
@@ -7065,7 +7243,7 @@ function createQueryAgent(config) {
       } catch {
       }
     }
-    const prompt = buildQueryClassifyPrompt(question, conversationContext);
+    const prompt = buildQueryClassifyPrompt(question, conversationContext, attachmentContext);
     const { object, usage } = await safeGenerateObject(
       generateObject,
       {
@@ -7280,6 +7458,7 @@ export {
   ApplicationQualityReportSchema,
   ApplicationQualityRoundSchema,
   ApplicationStateSchema,
+  AttachmentInterpretationSchema,
   AuditTypeSchema,
   AutoFillMatchSchema,
   AutoFillResultSchema,
@@ -7401,6 +7580,8 @@ export {
   ProducerInfoSchema,
   ProfessionalLiabilityDeclarationsSchema,
   QUOTE_SECTION_TYPES,
+  QueryAttachmentKindSchema,
+  QueryAttachmentSchema,
   QueryClassifyResultSchema,
   QueryIntentSchema,
   QueryResultSchema,
@@ -7459,6 +7640,7 @@ export {
   buildFormattingPrompt,
   buildIdentityPrompt,
   buildIntentPrompt,
+  buildInterpretAttachmentPrompt,
   buildLookupFillPrompt,
   buildQueryClassifyPrompt,
   buildQuestionBatchPrompt,