@claritylabs/cl-sdk 0.11.0 → 0.12.0

package/dist/index.js

@@ -36,6 +36,7 @@ __export(index_exports, {
   ApplicationQualityReportSchema: () => ApplicationQualityReportSchema,
   ApplicationQualityRoundSchema: () => ApplicationQualityRoundSchema,
   ApplicationStateSchema: () => ApplicationStateSchema,
+  AttachmentInterpretationSchema: () => AttachmentInterpretationSchema,
   AuditTypeSchema: () => AuditTypeSchema,
   AutoFillMatchSchema: () => AutoFillMatchSchema,
   AutoFillResultSchema: () => AutoFillResultSchema,
@@ -157,6 +158,8 @@ __export(index_exports, {
   ProducerInfoSchema: () => ProducerInfoSchema,
   ProfessionalLiabilityDeclarationsSchema: () => ProfessionalLiabilityDeclarationsSchema,
   QUOTE_SECTION_TYPES: () => QUOTE_SECTION_TYPES,
+  QueryAttachmentKindSchema: () => QueryAttachmentKindSchema,
+  QueryAttachmentSchema: () => QueryAttachmentSchema,
   QueryClassifyResultSchema: () => QueryClassifyResultSchema,
   QueryIntentSchema: () => QueryIntentSchema,
   QueryResultSchema: () => QueryResultSchema,
@@ -215,6 +218,7 @@ __export(index_exports, {
   buildFormattingPrompt: () => buildFormattingPrompt,
   buildIdentityPrompt: () => buildIdentityPrompt,
   buildIntentPrompt: () => buildIntentPrompt,
+  buildInterpretAttachmentPrompt: () => buildInterpretAttachmentPrompt,
   buildLookupFillPrompt: () => buildLookupFillPrompt,
   buildQueryClassifyPrompt: () => buildQueryClassifyPrompt,
   buildQuestionBatchPrompt: () => buildQuestionBatchPrompt,
@@ -6575,7 +6579,7 @@ Respond with the explanation text only \u2014 no JSON, no field ID, no extra for
 }
 
 // src/prompts/query/classify.ts
-function buildQueryClassifyPrompt(question, conversationContext) {
+function buildQueryClassifyPrompt(question, conversationContext, attachmentContext) {
   return `You are a query classifier for an insurance document intelligence system.
 
 Analyze the user's question and produce a structured classification.
@@ -6585,6 +6589,9 @@ ${question}
 ${conversationContext ? `
 CONVERSATION CONTEXT:
 ${conversationContext}` : ""}
+${attachmentContext ? `
+ATTACHMENT CONTEXT:
+${attachmentContext}` : ""}
 
 INSTRUCTIONS:
 
@@ -6605,6 +6612,7 @@ INSTRUCTIONS:
    - requiresDocumentLookup: true if a specific document needs to be fetched by ID/number/carrier
    - requiresChunkSearch: true if semantic search over document chunks is needed
    - requiresConversationHistory: true if the question references prior conversation
+   - If the user's attachment already contains critical facts, still request chunk/document lookup when policy or quote details should be cross-checked against stored records
 
 CHUNK TYPES (for chunkTypes filter):
 carrier_info, named_insured, coverage, endorsement, exclusion, condition, section, declaration, loss_history, premium, supplementary
@@ -6646,6 +6654,16 @@ var QueryIntentSchema = import_zod35.z.enum([
   "claims_inquiry",
   "general_knowledge"
 ]);
+var QueryAttachmentKindSchema = import_zod35.z.enum(["image", "pdf", "text"]);
+var QueryAttachmentSchema = import_zod35.z.object({
+  id: import_zod35.z.string().optional().describe("Optional stable attachment ID from the caller"),
+  kind: QueryAttachmentKindSchema,
+  name: import_zod35.z.string().optional().describe("Original filename or user-facing label"),
+  mimeType: import_zod35.z.string().optional().describe("MIME type such as image/jpeg or application/pdf"),
+  base64: import_zod35.z.string().optional().describe("Base64-encoded file content for image/pdf attachments"),
+  text: import_zod35.z.string().optional().describe("Plain-text attachment content when available"),
+  description: import_zod35.z.string().optional().describe("Caller-provided description of the attachment")
+});
 var SubQuestionSchema = import_zod35.z.object({
   question: import_zod35.z.string().describe("Atomic sub-question to retrieve and answer independently"),
   intent: QueryIntentSchema,
@@ -6666,14 +6684,21 @@ var QueryClassifyResultSchema = import_zod35.z.object({
   requiresConversationHistory: import_zod35.z.boolean().describe("Whether conversation history is relevant")
 });
 var EvidenceItemSchema = import_zod35.z.object({
-  source: import_zod35.z.enum(["chunk", "document", "conversation"]),
+  source: import_zod35.z.enum(["chunk", "document", "conversation", "attachment"]),
   chunkId: import_zod35.z.string().optional(),
   documentId: import_zod35.z.string().optional(),
   turnId: import_zod35.z.string().optional(),
+  attachmentId: import_zod35.z.string().optional(),
   text: import_zod35.z.string().describe("Text excerpt from the source"),
   relevance: import_zod35.z.number().min(0).max(1),
   metadata: import_zod35.z.array(import_zod35.z.object({ key: import_zod35.z.string(), value: import_zod35.z.string() })).optional()
 });
+var AttachmentInterpretationSchema = import_zod35.z.object({
+  summary: import_zod35.z.string().describe("Concise summary of what the attachment shows or contains"),
+  extractedFacts: import_zod35.z.array(import_zod35.z.string()).describe("Specific observable or document facts grounded in the attachment"),
+  recommendedFocus: import_zod35.z.array(import_zod35.z.string()).describe("Important details to incorporate when answering follow-up questions"),
+  confidence: import_zod35.z.number().min(0).max(1)
+});
 var RetrievalResultSchema = import_zod35.z.object({
   subQuestion: import_zod35.z.string(),
   evidence: import_zod35.z.array(EvidenceItemSchema)
@@ -6994,7 +7019,7 @@ async function verify(originalQuestion, subAnswers, allEvidence, config) {
 
 // src/query/quality.ts
 function sourceIdForEvidence(evidence) {
-  return evidence.chunkId ?? evidence.documentId ?? evidence.turnId;
+  return evidence.chunkId ?? evidence.documentId ?? evidence.turnId ?? evidence.attachmentId;
 }
 function citationSourceId(citation) {
   return citation.chunkId || citation.documentId;
@@ -7098,6 +7123,152 @@ function buildQueryReviewReport(params) {
   };
 }
 
+// src/prompts/query/interpret-attachment.ts
+function buildInterpretAttachmentPrompt(question, attachment) {
+  const attachmentLabel = attachment.name ?? attachment.id ?? "attachment";
+  const descriptor = [
+    `Attachment: ${attachmentLabel}`,
+    `Kind: ${attachment.kind}`,
+    attachment.mimeType ? `MIME type: ${attachment.mimeType}` : null,
+    attachment.description ? `Caller description: ${attachment.description}` : null
+  ].filter(Boolean).join("\n");
+  return `You are interpreting a user-supplied attachment for an insurance-support question.
+
+USER QUESTION:
+${question}
+
+ATTACHMENT METADATA:
+${descriptor}
+
+${attachment.kind === "text" && attachment.text ? `ATTACHMENT TEXT:
+${attachment.text}
+` : "The attachment content is provided separately as a file or image input.\n"}
+INSTRUCTIONS:
+1. Describe what the attachment appears to show or contain in a concise summary.
+2. Extract concrete facts that may matter when answering the user's question.
+3. Note the most important details to carry forward into follow-up questions.
+4. If the attachment is a document, identify the key business or insurance details visible.
+5. If the attachment is a photo of damage or a real-world issue, describe the observable issue without guessing beyond what is visible.
+6. Do not invent unreadable text. If something is unclear, say so in the summary or extracted facts.
+
+Respond with the structured interpretation.`;
+}
+
+// src/query/multimodal.ts
+function attachmentSourceId(attachment, index) {
+  return attachment.id ?? `attachment-${index + 1}`;
+}
+function buildAttachmentProviderOptions(attachment, providerOptions) {
+  const merged = {
+    ...providerOptions,
+    attachments: [
+      {
+        kind: attachment.kind,
+        name: attachment.name,
+        mimeType: attachment.mimeType,
+        base64: attachment.base64,
+        text: attachment.text,
+        description: attachment.description
+      }
+    ]
+  };
+  if (attachment.kind === "pdf" && attachment.base64) {
+    merged.pdfBase64 = attachment.base64;
+  }
+  if (attachment.kind === "image" && attachment.base64) {
+    merged.images = [
+      {
+        imageBase64: attachment.base64,
+        mimeType: attachment.mimeType ?? "image/jpeg"
+      }
+    ];
+  }
+  return merged;
+}
+function buildAttachmentEvidenceText(attachment, interpretation) {
+  const lines = [
+    `Attachment kind: ${attachment.kind}`,
+    attachment.name ? `Attachment name: ${attachment.name}` : null,
+    attachment.mimeType ? `MIME type: ${attachment.mimeType}` : null,
+    attachment.description ? `Caller description: ${attachment.description}` : null,
+    `Summary: ${interpretation.summary}`,
+    interpretation.extractedFacts.length > 0 ? `Extracted facts:
+${interpretation.extractedFacts.map((fact) => `- ${fact}`).join("\n")}` : null,
+    interpretation.recommendedFocus.length > 0 ? `Important follow-up details:
+${interpretation.recommendedFocus.map((item) => `- ${item}`).join("\n")}` : null,
+    attachment.kind === "text" && attachment.text ? `Original text:
+${attachment.text}` : null
+  ];
+  return lines.filter(Boolean).join("\n");
+}
+async function interpretAttachments(params) {
+  const { attachments = [], question, generateObject, providerOptions, log, onUsage } = params;
+  if (attachments.length === 0) {
+    return { evidence: [] };
+  }
+  const evidence = [];
+  for (const [index, attachment] of attachments.entries()) {
+    const id = attachmentSourceId(attachment, index);
+    if (attachment.kind === "text" && attachment.text) {
+      const textEvidence = buildAttachmentEvidenceText(attachment, {
+        summary: attachment.description ?? "User supplied text context.",
+        extractedFacts: [attachment.text],
+        recommendedFocus: [],
+        confidence: 1
+      });
+      evidence.push({
+        source: "attachment",
+        attachmentId: id,
+        chunkId: id,
+        documentId: id,
+        text: textEvidence,
+        relevance: 0.95,
+        metadata: [
+          { key: "kind", value: attachment.kind },
+          ...attachment.name ? [{ key: "name", value: attachment.name }] : []
+        ]
+      });
+      continue;
+    }
+    const prompt = buildInterpretAttachmentPrompt(question, attachment);
+    const { object, usage } = await safeGenerateObject(
+      generateObject,
+      {
+        prompt,
+        schema: AttachmentInterpretationSchema,
+        maxTokens: 2048,
+        providerOptions: buildAttachmentProviderOptions(attachment, providerOptions)
+      },
+      {
+        fallback: {
+          summary: attachment.description ?? `User supplied ${attachment.kind} attachment.`,
+          extractedFacts: [],
+          recommendedFocus: [],
+          confidence: 0.2
+        },
+        log,
+        onError: (error, attempt) => log?.(`Attachment interpretation attempt ${attempt + 1} failed for "${attachment.name ?? id}": ${error}`)
+      }
+    );
+    onUsage?.(usage);
+    evidence.push({
+      source: "attachment",
+      attachmentId: id,
+      chunkId: id,
+      documentId: id,
+      text: buildAttachmentEvidenceText(attachment, object),
+      relevance: Math.max(0.7, object.confidence),
+      metadata: [
+        { key: "kind", value: attachment.kind },
+        ...attachment.name ? [{ key: "name", value: attachment.name }] : []
+      ]
+    });
+  }
+  const contextSummary = evidence.map((item, index) => `Attachment ${index + 1}:
+${item.text}`).join("\n\n");
+  return { evidence, contextSummary };
+}
+
 // src/query/coordinator.ts
 function createQueryAgent(config) {
   const {
@@ -7125,13 +7296,23 @@ function createQueryAgent(config) {
   }
   async function query(input) {
     totalUsage = { inputTokens: 0, outputTokens: 0 };
-    const { question, conversationId, context } = input;
+    const { question, conversationId, context, attachments } = input;
     const pipelineCtx = createPipelineContext({
       id: `query-${Date.now()}`
     });
+    onProgress?.("Interpreting attachments...");
+    const { evidence: attachmentEvidence, contextSummary: attachmentContext } = await interpretAttachments({
+      attachments,
+      question,
+      generateObject,
+      providerOptions,
+      log,
+      onUsage: trackUsage
+    });
+    await pipelineCtx.save("attachments", { attachmentEvidence });
     onProgress?.("Classifying query...");
-    const classification = await classify(question, conversationId);
-    await pipelineCtx.save("classify", { classification });
+    const classification = await classify(question, conversationId, attachmentContext);
+    await pipelineCtx.save("classify", { classification, attachmentEvidence });
     onProgress?.(`Retrieving evidence for ${classification.subQuestions.length} sub-question(s)...`);
     const retrieverConfig = {
       documentStore,
@@ -7144,8 +7325,8 @@ function createQueryAgent(config) {
         (sq) => limit(() => retrieve(sq, conversationId, retrieverConfig))
       )
     );
-    const allEvidence = retrievalResults.flatMap((r) => r.evidence);
-    await pipelineCtx.save("retrieve", { classification, evidence: allEvidence });
+    const allEvidence = [...attachmentEvidence, ...retrievalResults.flatMap((r) => r.evidence)];
+    await pipelineCtx.save("retrieve", { classification, attachmentEvidence, evidence: allEvidence });
     onProgress?.("Reasoning over evidence...");
     const reasonerConfig = { generateObject, providerOptions };
     const reasonResults = await Promise.allSettled(
@@ -7154,7 +7335,7 @@ function createQueryAgent(config) {
           const { subAnswer, usage } = await reason(
             sq.question,
             sq.intent,
-            retrievalResults[i].evidence,
+            [...attachmentEvidence, ...retrievalResults[i].evidence],
             reasonerConfig
           );
           trackUsage(usage);
@@ -7178,7 +7359,7 @@ function createQueryAgent(config) {
         });
       }
     }
-    await pipelineCtx.save("reason", { classification, evidence: allEvidence, subAnswers });
+    await pipelineCtx.save("reason", { classification, attachmentEvidence, evidence: allEvidence, subAnswers });
     onProgress?.("Verifying answer grounding...");
     const verifierConfig = { generateObject, providerOptions };
     const verifyRounds = [];
@@ -7226,7 +7407,7 @@ function createQueryAgent(config) {
                 const { subAnswer, usage: u } = await reason(
                   sq.question,
                   sq.intent,
-                  retryRetrievals[i].evidence,
+                  [...attachmentEvidence, ...retryRetrievals[i].evidence],
                   reasonerConfig
                 );
                 trackUsage(u);
@@ -7261,6 +7442,7 @@ function createQueryAgent(config) {
     });
     await pipelineCtx.save("review", {
       classification,
+      attachmentEvidence,
       evidence: allEvidence,
       subAnswers,
       reviewReport
@@ -7293,7 +7475,7 @@ function createQueryAgent(config) {
     }
     return { ...queryResult, tokenUsage: totalUsage, reviewReport };
   }
-  async function classify(question, conversationId) {
+  async function classify(question, conversationId, attachmentContext) {
     let conversationContext;
     if (conversationId) {
       try {
@@ -7304,7 +7486,7 @@ function createQueryAgent(config) {
       } catch {
       }
     }
-    const prompt = buildQueryClassifyPrompt(question, conversationContext);
+    const prompt = buildQueryClassifyPrompt(question, conversationContext, attachmentContext);
     const { object, usage } = await safeGenerateObject(
       generateObject,
       {
@@ -7520,6 +7702,7 @@ var AGENT_TOOLS = [
   ApplicationQualityReportSchema,
   ApplicationQualityRoundSchema,
   ApplicationStateSchema,
+  AttachmentInterpretationSchema,
   AuditTypeSchema,
   AutoFillMatchSchema,
   AutoFillResultSchema,
@@ -7641,6 +7824,8 @@ var AGENT_TOOLS = [
   ProducerInfoSchema,
   ProfessionalLiabilityDeclarationsSchema,
   QUOTE_SECTION_TYPES,
+  QueryAttachmentKindSchema,
+  QueryAttachmentSchema,
   QueryClassifyResultSchema,
   QueryIntentSchema,
   QueryResultSchema,
@@ -7699,6 +7884,7 @@ var AGENT_TOOLS = [
   buildFormattingPrompt,
   buildIdentityPrompt,
   buildIntentPrompt,
+  buildInterpretAttachmentPrompt,
   buildLookupFillPrompt,
   buildQueryClassifyPrompt,
   buildQuestionBatchPrompt,