@hed-hog/finance 0.0.233 → 0.0.235

package/src/finance.service.ts

@@ -3,9 +3,10 @@ import { PaginationDTO, PaginationService } from '@hed-hog/api-pagination';
 import { PrismaService } from '@hed-hog/api-prisma';
 import { AiService } from '@hed-hog/core';
 import {
-  BadRequestException,
-  Injectable,
-  NotFoundException
+    BadRequestException,
+    Injectable,
+    Logger,
+    NotFoundException
 } from '@nestjs/common';
 import { CreateBankAccountDto } from './dto/create-bank-account.dto';
 import { CreateFinancialTitleDto } from './dto/create-financial-title.dto';
@@ -15,6 +16,8 @@ type TitleType = 'payable' | 'receivable';
 
 @Injectable()
 export class FinanceService {
+  private readonly logger = new Logger(FinanceService.name);
+
   constructor(
     private readonly prisma: PrismaService,
     private readonly paginationService: PaginationService,
@@ -34,20 +37,25 @@ export class FinanceService {
     const slug = isPayable
       ? 'finance-payable-extractor'
       : 'finance-receivable-extractor';
+    const llmAttachmentDebug = await this.ai.debugAttachmentForLlm(file, fileId);
+    this.logger.warn(
+      `[FINANCE-EXTRACT] start type=${titleType} fileId=${fileId || 'upload'} mode=${llmAttachmentDebug?.mode || 'none'} mime=${llmAttachmentDebug?.mimeType || 'n/a'} textLength=${llmAttachmentDebug?.textLength || 0}`,
+    );
 
     const schema = isPayable
       ? '{"documento":"string|null","fornecedor_nome":"string|null","categoria_nome":"string|null","centro_custo_nome":"string|null","competencia":"YYYY-MM|null","vencimento":"YYYY-MM-DD|null","valor":number|null,"metodo":"boleto|pix|transferencia|cartao|dinheiro|cheque|null","descricao":"string|null","confidence":number|null}'
       : '{"documento":"string|null","cliente_nome":"string|null","categoria_nome":"string|null","centro_custo_nome":"string|null","competencia":"YYYY-MM|null","vencimento":"YYYY-MM-DD|null","valor":number|null,"canal":"boleto|pix|transferencia|cartao|null","descricao":"string|null","confidence":number|null}';
 
     const extractionMessage = isPayable
-      ? 'Extract the payable form fields from the attached file. If uncertain, return null for that field. Response must be JSON only.'
-      : 'Extract the receivable form fields from the attached file. If uncertain, return null for that field. Response must be JSON only.';
+      ? 'Extract the payable form fields from the attached file. IMPORTANT: prioritize Brazilian invoice formats (pt-BR). Monetary values may come as 1.900,00 or 1900.00. Always return numeric value in JSON (example: 1900). Prefer the field explicitly labeled as valor total / total da nota / total a pagar. Never guess; if uncertain, return null. Response must be JSON only.'
+      : 'Extract the receivable form fields from the attached file. IMPORTANT: prioritize Brazilian invoice formats (pt-BR). Monetary values may come as 1.900,00 or 1900.00. Always return numeric value in JSON (example: 1900). Prefer the field explicitly labeled as valor total / total da nota / total a receber. Never guess; if uncertain, return null. Response must be JSON only.';
 
     await this.ai.createAgent({
       slug,
       provider: 'openai',
+      model: 'gpt-4o',
       instructions:
-        `You are a financial OCR extraction assistant. Extract only ${titleType} title fields from invoice/bill files. Always return valid JSON only, no markdown. Use this schema: ${schema}.`,
+        `You are a financial OCR extraction assistant. Extract only ${titleType} title fields from invoice/bill files. Be strict with monetary values in pt-BR format: 1.900,00 means 1900.00. Never confuse punctuation separators. Prefer explicit labels in this order: "valor dos serviços" > "valor total da nota" > "valor líquido". For vencimento, use fields like "vencimento" or "data de vencimento" only; do not use data de emissão as vencimento. If confidence is low, return null. Always return valid JSON only, no markdown. Use this schema: ${schema}.`,
     });
 
     const result = await this.ai.chatWithAgent(
@@ -60,35 +68,415 @@ export class FinanceService {
     );
 
     const parsed = this.parseAiJson(result?.content || '{}');
+    this.logger.warn(
+      `[FINANCE-EXTRACT] pass1 type=${titleType} hasDocumento=${!!this.toNullableString(parsed?.documento)} hasValor=${this.toNullableNumber(parsed?.valor) !== null} hasVencimento=${!!this.normalizeDate(parsed?.vencimento)}`,
+    );
+    const mergedParsed = await this.enhanceParsedExtraction(
+      parsed,
+      slug,
+      isPayable,
+      llmAttachmentDebug,
+      fileId,
+      file,
+    );
 
-    const personName = isPayable ? parsed.fornecedor_nome : parsed.cliente_nome;
+    const personName = isPayable
+      ? mergedParsed.fornecedor_nome
+      : mergedParsed.cliente_nome;
     const personId = await this.matchPersonId(personName);
-    const categoriaId = await this.matchCategoryId(parsed.categoria_nome);
-    const centroCustoId = await this.matchCostCenterId(parsed.centro_custo_nome);
+    const categoriaId = await this.matchCategoryId(mergedParsed.categoria_nome);
+    const centroCustoId = await this.matchCostCenterId(
+      mergedParsed.centro_custo_nome,
+    );
+
+    const metodo = this.normalizePaymentMethod(
+      mergedParsed.metodo || mergedParsed.canal,
+    );
+    const documento = this.toNullableString(mergedParsed.documento);
+    const categoriaNome = this.toNullableString(mergedParsed.categoria_nome);
+    const centroCustoNome = this.toNullableString(mergedParsed.centro_custo_nome);
+    const competencia = this.normalizeMonth(mergedParsed.competencia);
+    const vencimento = this.normalizeDate(mergedParsed.vencimento);
+    const valor = this.toNullableNumber(mergedParsed.valor);
+    const descricao = this.toNullableString(mergedParsed.descricao);
+    const confidence = this.resolveExtractionConfidence(
+      this.toNullableNumber(mergedParsed.confidence),
+      {
+        documento,
+        personId,
+        categoriaId,
+        centroCustoId,
+        vencimento,
+        valor,
+      },
+    );
+    const warnings = this.buildExtractionWarnings(confidence, {
+      valor,
+      vencimento,
+    });
 
-    const metodo = this.normalizePaymentMethod(parsed.metodo || parsed.canal);
+    this.logger.warn(
+      `[FINANCE-EXTRACT] done type=${titleType} confidence=${confidence ?? 'null'} valor=${valor ?? 'null'} vencimento=${vencimento || 'null'} fallback=${mergedParsed?.__fallbackSource || 'none'}`,
+    );
 
     return {
-      documento: this.toNullableString(parsed.documento),
+      documento,
       fornecedorId: isPayable ? personId : '',
       fornecedorNome: isPayable ? this.toNullableString(personName) : null,
       clienteId: !isPayable ? personId : '',
       clienteNome: !isPayable ? this.toNullableString(personName) : null,
       categoriaId,
-      categoriaNome: this.toNullableString(parsed.categoria_nome),
+      categoriaNome,
       centroCustoId,
-      centroCustoNome: this.toNullableString(parsed.centro_custo_nome),
-      competencia: this.normalizeMonth(parsed.competencia),
-      vencimento: this.normalizeDate(parsed.vencimento),
-      valor: this.toNullableNumber(parsed.valor),
+      centroCustoNome,
+      competencia,
+      vencimento,
+      valor,
       metodo,
       canal: metodo,
-      descricao: this.toNullableString(parsed.descricao),
-      confidence: this.toNullableNumber(parsed.confidence),
-      raw: parsed,
+      descricao,
+      confidence,
+      confidenceLevel:
+        confidence === null ? null : confidence < 70 ? 'low' : 'high',
+      warnings,
+      raw: {
+        ...mergedParsed,
+        __llmAttachmentDebug: llmAttachmentDebug,
+      },
+    };
+  }
+
+  private async enhanceParsedExtraction(
+    parsed: any,
+    slug: string,
+    isPayable: boolean,
+    llmAttachmentDebug?: any,
+    fileId?: number,
+    file?: MulterFile,
+  ) {
+    const base = parsed || {};
+
+    const hasDocumento = !!this.toNullableString(base.documento);
+    const hasValor = this.toNullableNumber(base.valor) !== null;
+    const hasVencimento = !!this.normalizeDate(base.vencimento);
+    const hasPessoa = !!this.toNullableString(
+      isPayable ? base.fornecedor_nome : base.cliente_nome,
+    );
+
+    if (hasDocumento || hasValor || hasVencimento || hasPessoa) {
+      this.logger.warn('[FINANCE-EXTRACT] pass2 skipped: pass1 already has key fields');
+      return base;
+    }
+
+    const fallbackSchema = isPayable
+      ? '{"documento":"string|null","fornecedor_nome":"string|null","vencimento":"YYYY-MM-DD|null","valor":number|null,"confidence":number|null}'
+      : '{"documento":"string|null","cliente_nome":"string|null","vencimento":"YYYY-MM-DD|null","valor":number|null,"confidence":number|null}';
+
+    const fallbackMessage = isPayable
+      ? `Second pass. Extract ONLY documento, fornecedor_nome, vencimento, valor. Prefer "valor dos serviços" then "valor total" then "valor líquido". Return JSON only with this schema: ${fallbackSchema}`
+      : `Second pass. Extract ONLY documento, cliente_nome, vencimento, valor. Prefer "valor dos serviços" then "valor total" then "valor líquido". Return JSON only with this schema: ${fallbackSchema}`;
+
+    const retry = await this.ai.chatWithAgent(
+      slug,
+      {
+        message: fallbackMessage,
+        file_id: fileId,
+      },
+      file,
+    );
+
+    const retryParsed = this.parseAiJson(retry?.content || '{}');
+    this.logger.warn(
+      `[FINANCE-EXTRACT] pass2 hasDocumento=${!!this.toNullableString(retryParsed?.documento)} hasValor=${this.toNullableNumber(retryParsed?.valor) !== null} hasVencimento=${!!this.normalizeDate(retryParsed?.vencimento)}`,
+    );
+
+    const mergedAfterRetry = {
+      ...retryParsed,
+      ...base,
+      documento: this.toNullableString(base.documento)
+        ? base.documento
+        : retryParsed.documento,
+      valor:
+        this.toNullableNumber(base.valor) !== null ? base.valor : retryParsed.valor,
+      vencimento: this.normalizeDate(base.vencimento)
+        ? base.vencimento
+        : retryParsed.vencimento,
+      fornecedor_nome: isPayable
+        ? this.toNullableString(base.fornecedor_nome)
+          ? base.fornecedor_nome
+          : retryParsed.fornecedor_nome
+        : base.fornecedor_nome,
+      cliente_nome: !isPayable
+        ? this.toNullableString(base.cliente_nome)
+          ? base.cliente_nome
+          : retryParsed.cliente_nome
+        : base.cliente_nome,
+      confidence:
+        this.toNullableNumber(base.confidence) !== null
+          ? base.confidence
+          : retryParsed.confidence,
+    };
+
+    const stillEmpty =
+      !this.toNullableString(mergedAfterRetry.documento) &&
+      this.toNullableNumber(mergedAfterRetry.valor) === null &&
+      !this.normalizeDate(mergedAfterRetry.vencimento) &&
+      !this.toNullableString(
+        isPayable
+          ? mergedAfterRetry.fornecedor_nome
+          : mergedAfterRetry.cliente_nome,
+      );
+
+    if (!stillEmpty) {
+      this.logger.warn('[FINANCE-EXTRACT] regex fallback skipped: pass2 produced data');
+      return mergedAfterRetry;
+    }
+
+    const rawText = await this.ai.extractAttachmentText(file, fileId);
+    const regexFallback = this.extractFinancialFieldsFromRawText(rawText, isPayable);
+    this.logger.warn(
+      `[FINANCE-EXTRACT] regex fallback activated hasText=${!!rawText} hasDocumento=${!!this.toNullableString(regexFallback?.documento)} hasValor=${this.toNullableNumber(regexFallback?.valor) !== null} hasVencimento=${!!this.normalizeDate(regexFallback?.vencimento)}`,
+    );
+
+    const stillEmptyAfterRegex =
+      !this.toNullableString(regexFallback.documento) &&
+      this.toNullableNumber(regexFallback.valor) === null &&
+      !this.normalizeDate(regexFallback.vencimento);
+
+    if (
+      stillEmptyAfterRegex &&
+      llmAttachmentDebug?.mode === 'pdf-text' &&
+      Number(llmAttachmentDebug?.textLength || 0) === 0
+    ) {
+      this.logger.warn(
+        '[FINANCE-EXTRACT] gemini fallback activated for scanned PDF (no text layer)',
+      );
+
+      const geminiSchema = isPayable
+        ? '{"documento":"string|null","fornecedor_nome":"string|null","vencimento":"YYYY-MM-DD|null","valor":number|null,"confidence":number|null}'
+        : '{"documento":"string|null","cliente_nome":"string|null","vencimento":"YYYY-MM-DD|null","valor":number|null,"confidence":number|null}';
+
+      const geminiPrompt = isPayable
+        ? `Extract documento, fornecedor_nome, vencimento and valor from this invoice PDF. For valor, prefer 'valor dos serviços', then 'valor total', then 'valor líquido'. Return JSON only using schema: ${geminiSchema}`
+        : `Extract documento, cliente_nome, vencimento and valor from this invoice PDF. For valor, prefer 'valor dos serviços', then 'valor total', then 'valor líquido'. Return JSON only using schema: ${geminiSchema}`;
+
+      const geminiModels = [
+        'gemini-2.5-flash',
+        'gemini-2.5-pro',
+        'gemini-2.0-flash',
+        'gemini-2.0-flash-lite',
+        'gemini-1.5-flash',
+        'gemini-1.5-pro',
+        'gemini-1.5-flash-latest',
+        'gemini-1.5-pro-latest',
+      ];
+
+      for (const model of geminiModels) {
+        try {
+          this.logger.warn(
+            `[FINANCE-EXTRACT] gemini fallback trying model=${model}`,
+          );
+
+          const geminiResult = await this.ai.chat(
+            {
+              provider: 'gemini',
+              model,
+              message: geminiPrompt,
+              file_id: fileId,
+            },
+            file,
+          );
+
+          const geminiParsed = this.parseAiJson(geminiResult?.content || '{}');
+          this.logger.warn(
+            `[FINANCE-EXTRACT] gemini fallback result model=${model} hasDocumento=${!!this.toNullableString(geminiParsed?.documento)} hasValor=${this.toNullableNumber(geminiParsed?.valor) !== null} hasVencimento=${!!this.normalizeDate(geminiParsed?.vencimento)}`,
+          );
+
+          return {
+            ...geminiParsed,
+            ...regexFallback,
+            documento: this.toNullableString(geminiParsed?.documento)
+              ? geminiParsed.documento
+              : regexFallback.documento,
+            valor:
+              this.toNullableNumber(geminiParsed?.valor) !== null
+                ? geminiParsed.valor
+                : regexFallback.valor,
+            vencimento: this.normalizeDate(geminiParsed?.vencimento)
+              ? geminiParsed.vencimento
+              : regexFallback.vencimento,
+            competencia: this.normalizeMonth(geminiParsed?.competencia)
+              ? geminiParsed.competencia
+              : regexFallback.competencia,
+            fornecedor_nome: isPayable
+              ? this.toNullableString(geminiParsed?.fornecedor_nome)
+                ? geminiParsed.fornecedor_nome
+                : regexFallback.fornecedor_nome
+              : null,
+            cliente_nome: !isPayable
+              ? this.toNullableString(geminiParsed?.cliente_nome)
+                ? geminiParsed.cliente_nome
+                : regexFallback.cliente_nome
+              : null,
+            confidence:
+              this.toNullableNumber(geminiParsed?.confidence) ?? regexFallback.confidence,
+            __fallbackSource: 'gemini-pdf-binary',
+            __fallbackModel: model,
+            __fallbackHasText: !!rawText,
+          };
+        } catch (error) {
+          const statusCode = (error as any)?.response?.status;
+          const message = (error as any)?.message || 'unknown error';
+          const apiError =
+            (error as any)?.response?.data?.error?.message ||
+            JSON.stringify((error as any)?.response?.data || {});
+          this.logger.warn(
+            `[FINANCE-EXTRACT] gemini fallback failed model=${model} status=${statusCode ?? 'n/a'} message=${message} apiError=${apiError}`,
+          );
+        }
+      }
+    }
+
+    return {
+      ...regexFallback,
+      ...mergedAfterRetry,
+      documento: this.toNullableString(mergedAfterRetry.documento)
+        ? mergedAfterRetry.documento
+        : regexFallback.documento,
+      valor:
+        this.toNullableNumber(mergedAfterRetry.valor) !== null
+          ? mergedAfterRetry.valor
+          : regexFallback.valor,
+      vencimento: this.normalizeDate(mergedAfterRetry.vencimento)
+        ? mergedAfterRetry.vencimento
+        : regexFallback.vencimento,
+      competencia: this.normalizeMonth(mergedAfterRetry.competencia)
+        ? mergedAfterRetry.competencia
+        : regexFallback.competencia,
+      fornecedor_nome: isPayable
+        ? this.toNullableString(mergedAfterRetry.fornecedor_nome)
+          ? mergedAfterRetry.fornecedor_nome
+          : regexFallback.fornecedor_nome
+        : mergedAfterRetry.fornecedor_nome,
+      cliente_nome: !isPayable
+        ? this.toNullableString(mergedAfterRetry.cliente_nome)
+          ? mergedAfterRetry.cliente_nome
+          : regexFallback.cliente_nome
+        : mergedAfterRetry.cliente_nome,
+      __fallbackSource: 'regex-pdf-text',
+      __fallbackHasText: !!rawText,
+    };
+  }
+
+  private extractFinancialFieldsFromRawText(text: string, isPayable: boolean) {
+    const source = text || '';
+
+    const documento =
+      this.firstMatch(
+        source,
+        [
+          /\b(?:n[úu]mero\s+da\s+nota|nota\s+fiscal|nfs-?e|nf-?e|nf)\s*[:#-]?\s*([a-z0-9._\/-]{4,})/i,
+        ],
+      ) || null;
+
+    const emissao = this.firstMatch(source, [
+      /\b(?:data\s+de\s+emiss[aã]o|emiss[aã]o)\s*[:#-]?\s*(\d{2}\/\d{2}\/\d{4})/i,
+    ]);
+
+    const vencimento = this.firstMatch(source, [
+      /\b(?:data\s+de\s+vencimento|vencimento)\s*[:#-]?\s*(\d{2}\/\d{2}\/\d{4})/i,
+    ]);
+
+    const valorLabeled = this.firstMatch(source, [
+      /\bvalor\s+dos\s+servi[cç]os\b[^\d]{0,20}(?:r\$\s*)?([\d.,]+)/i,
+      /\bvalor\s+total\b[^\d]{0,20}(?:r\$\s*)?([\d.,]+)/i,
+      /\btotal\s+da\s+nota\b[^\d]{0,20}(?:r\$\s*)?([\d.,]+)/i,
+      /\bvalor\s+l[ií]quido\b[^\d]{0,20}(?:r\$\s*)?([\d.,]+)/i,
+    ]);
+
+    const fallbackAnyMoney = this.firstMatch(source, [
+      /(?:r\$\s*)([\d]{1,3}(?:\.[\d]{3})*,\d{2})/i,
+    ]);
+
+    const valor = this.toNullableNumber(valorLabeled || fallbackAnyMoney);
+
+    const personName = this.firstMatch(source, [
+      /\b(?:tomador|cliente|fornecedor|prestador)\b\s*[:#-]?\s*([^\n\r]{3,80})/i,
+    ]);
+
+    return {
+      documento,
+      fornecedor_nome: isPayable ? this.toNullableString(personName) : null,
+      cliente_nome: !isPayable ? this.toNullableString(personName) : null,
+      competencia: emissao ? this.normalizeMonth(emissao) : null,
+      vencimento: vencimento ? this.normalizeDate(vencimento) : null,
+      valor,
+      confidence: valor !== null ? 68 : 52,
     };
   }
 
+  private firstMatch(text: string, patterns: RegExp[]): string {
+    for (const pattern of patterns) {
+      const match = text.match(pattern);
+      if (match?.[1]) {
+        return match[1].trim();
+      }
+    }
+
+    return '';
+  }
+
+  private resolveExtractionConfidence(
+    aiConfidence: number | null,
+    data: {
+      documento: string | null;
+      personId: string;
+      categoriaId: string;
+      centroCustoId: string;
+      vencimento: string;
+      valor: number | null;
+    },
+  ): number | null {
+    if (aiConfidence !== null) {
+      const normalized = aiConfidence <= 1 ? aiConfidence * 100 : aiConfidence;
+      return Math.max(0, Math.min(100, Math.round(normalized)));
+    }
+
+    let score = 35;
+    if (data.documento) score += 15;
+    if (data.vencimento) score += 10;
+    if (data.valor !== null && data.valor > 0) score += 20;
+    if (data.personId) score += 10;
+    if (data.categoriaId) score += 5;
+    if (data.centroCustoId) score += 5;
+
+    return Math.max(0, Math.min(100, score));
+  }
+
+  private buildExtractionWarnings(
+    confidence: number | null,
+    data: {
+      valor: number | null;
+      vencimento: string;
+    },
+  ): string[] {
+    const warnings: string[] = [];
+
+    if (confidence !== null && confidence < 70) {
+      warnings.push('Baixa confiança na extração. Revise os campos antes de salvar.');
+    }
+
+    if (data.valor === null || data.valor <= 0) {
+      warnings.push('Valor não identificado com segurança no documento.');
+    }
+
+    if (!data.vencimento) {
+      warnings.push('Data de vencimento não identificada com segurança.');
+    }
+
+    return warnings;
+  }
+
   private parseAiJson(content: string): any {
     const trimmed = (content || '').trim();
     if (!trimmed) {
@@ -263,7 +651,41 @@ export class FinanceService {
 
   private toNullableNumber(value: any): number | null {
     if (value === null || value === undefined || value === '') return null;
-    const num = Number(value);
+    if (typeof value === 'number') {
+      return Number.isFinite(value) ? value : null;
+    }
+
+    const raw = String(value).trim();
+    if (!raw) return null;
+
+    const cleaned = raw.replace(/[^\d,.-]/g, '');
+    if (!cleaned) return null;
+
+    const lastComma = cleaned.lastIndexOf(',');
+    const lastDot = cleaned.lastIndexOf('.');
+
+    let normalized = cleaned;
+
+    if (lastComma >= 0 && lastDot >= 0) {
+      if (lastComma > lastDot) {
+        normalized = cleaned.replace(/\./g, '').replace(',', '.');
+      } else {
+        normalized = cleaned.replace(/,/g, '');
+      }
+    } else if (lastComma >= 0) {
+      normalized = cleaned.replace(/\./g, '').replace(',', '.');
+    } else {
+      const dotParts = cleaned.split('.');
+      if (
+        dotParts.length > 1 &&
+        dotParts[dotParts.length - 1].length === 3 &&
+        dotParts.length <= 4
+      ) {
+        normalized = cleaned.replace(/\./g, '');
+      }
+    }
+
+    const num = Number(normalized);
     return Number.isFinite(num) ? num : null;
   }