@arela/uploader 1.1.4 → 1.2.0

package/.claude/worktrees/agent-multi-profile/src/file-detection.js

@@ -0,0 +1,338 @@
+import fs from 'fs';
+import path from 'path';
+import { PDFParse } from 'pdf-parse';
+
+import { extractDocumentFields } from './document-type-shared.js';
+import { classifyDocument } from './scoring/scoring-engine.js';
+
+// pdf.js can reject internal worker promises on corrupt PDFs (e.g.
+// XRefEntryException) that no try/catch around getText() can reach. Track the
+// last PDF handed to the parser so the global unhandledRejection handler can
+// name it; the event fires after finally blocks run, so it is never cleared.
+let currentPdfFile = null;
+function getCurrentPdfFile() {
+  return currentPdfFile;
+}
+
+// Document types that participate in arela_path composition.
+const ARELA_PATH_TYPES = new Set([
+  'pedimento_simplificado',
+  'pedimento_completo',
+  'pedimento_completo_xml',
+]);
+
+/**
+ * For `pedimento_completo_xml` the patente is not present in the XML body —
+ * it must be parsed from the filename. Three known patterns are tried.
+ */
+function patenteFromXmlFilename(filePath) {
+  if (!filePath) return null;
+  const fileName = path.basename(filePath);
+
+  let m = fileName.match(/^VU_(\d{4})_\d{3}_\d{7}\.xml$/i);
+  if (m) return m[1];
+
+  m = fileName.match(/^\d{3}-(\d{4})-\d{7}\.xml$/i);
+  if (m) return m[1];
+
+  m = fileName.match(/^\d{4}(\d{4})\d{7}(?:_\d{15})?\.xml$/i);
+  if (m) return m[1];
+
+  return null;
+}
+
+/**
+ * Compose arela_path from extracted pedimento fields
+ * Format: RFC/Year/Patente/Aduana/Pedimento/
+ * Example: PED781129JT6/2023/3429/07/3019796/
+ */
+function composeArelaPath(
+  detectedType,
+  fields,
+  detectedPedimentoYear,
+  filePath,
+) {
+  if (!ARELA_PATH_TYPES.has(detectedType)) {
+    return null;
+  }
+
+  // The CoveFact annex resolves to pedimento_completo without payment
+  // evidence (see pedimento-completo.js resolveType), so it must never anchor
+  // an operation on its own: with no self-composed path it only becomes
+  // pushable when a PAID sibling pedimento propagates the folder's
+  // arela_path. Unpaid operations therefore never reach Arela through it.
+  if (/covefact/i.test(filePath ?? '')) {
+    return null;
+  }
+
+  const rfc = fields?.find((f) => f.name === 'rfc')?.value;
+  let patente = fields?.find((f) => f.name === 'patente')?.value;
+  const aduana = fields?.find((f) => f.name === 'aduanaEntradaSalida')?.value;
+  const pedimento = fields?.find((f) => f.name === 'numPedimento')?.value;
+  const year = detectedPedimentoYear;
+
+  // XML matcher does not extract patente from the body — derive from filename.
+  if (!patente && detectedType === 'pedimento_completo_xml') {
+    patente = patenteFromXmlFilename(filePath);
+  }
+
+  // All components are required for a valid arela_path
+  if (!rfc || !year || !patente || !aduana || !pedimento) {
+    console.log('⚠️ Missing required fields for arela_path composition:', {
+      rfc: !!rfc,
+      year: !!year,
+      patente: !!patente,
+      aduana: !!aduana,
+      pedimento: !!pedimento,
+    });
+    return null;
+  }
+
+  // Ensure aduana is padded to 2 digits if needed (07 instead of 7)
+  const aduanaFormatted = aduana.toString().padStart(2, '0');
+
+  // arela_path should be the folder structure only, without filename
+  const arelaPath = `${rfc}/${year}/${patente}/${aduanaFormatted}/${pedimento}/`;
+
+  console.log(`✅ Composed arela_path: ${arelaPath}`);
+  return arelaPath;
+}
+
+/**
+ * File Detection Service
+ * Detects document types and extracts metadata from files
+ */
+export class FileDetectionService {
+  constructor() {
+    // Best-match matchers (adapted from the API). When set, classification uses
+    // the scoring engine; otherwise it falls back to legacy first-match-wins.
+    this.matchers = null;
+    // Per-run classification tally for observability (distribution + unmatched).
+    this.classificationStats = {
+      total: 0,
+      matched: 0,
+      unmatched: 0,
+      byType: {},
+    };
+  }
+
+  /** Provide the resolved+adapted matcher set for scoring-based classification. */
+  setMatchers(matchers) {
+    this.matchers = matchers && matchers.length ? matchers : null;
+  }
+
+  /**
+   * Detect document type, tallying the outcome for a per-run summary. Wraps the
+   * implementation so every return path is counted from one place.
+   */
+  async detectFile(filePath) {
+    const result = await this.#detectFileImpl(filePath);
+    const type = result?.detectedType ?? null;
+    this.classificationStats.total += 1;
+    if (type) {
+      this.classificationStats.matched += 1;
+      this.classificationStats.byType[type] =
+        (this.classificationStats.byType[type] ?? 0) + 1;
+    } else {
+      this.classificationStats.unmatched += 1;
+    }
+    return result;
+  }
+
+  /** Log the classification distribution for the run (call after processing). */
+  logClassificationSummary(logger) {
+    const s = this.classificationStats;
+    if (!s.total) return;
+    const mode = this.matchers ? 'scoring' : 'legacy';
+    const dist = Object.entries(s.byType)
+      .sort((a, b) => b[1] - a[1])
+      .map(([t, n]) => `${t}=${n}`)
+      .join(', ');
+    const log = logger?.info ? logger : console;
+    log.info(
+      `🧩 Classification summary (${mode}): ${s.total} docs, ${s.matched} matched, ` +
+        `${s.unmatched} unmatched${dist ? ` — ${dist}` : ''}`,
+    );
+  }
+
+  /**
+   * Detect document type from a file
+   * @param {string} filePath - Path to the file to analyze
+   * @returns {Promise<{detectedType: string|null, fields: Array, detectedPedimento: string|null, detectedPedimentoYear: number|null, text: string}>}
+   */
+  async #detectFileImpl(filePath) {
+    try {
+      const fileExtension = path
+        .extname(filePath)
+        .toLowerCase()
+        .replace('.', '');
+      const fileName = path.basename(filePath);
+
+      console.log(`🔍 Analyzing file: ${fileName} (${fileExtension})`);
+
+      let text = '';
+
+      // Extract text based on file type
+      switch (fileExtension) {
+        case 'pdf':
+          text = await this.extractTextFromPDF(filePath);
+          break;
+        case 'txt':
+          text = fs.readFileSync(filePath, 'utf8');
+          break;
+        case 'xml':
+          text = fs.readFileSync(filePath, 'utf8');
+          break;
+        default:
+          console.log(`⚠️ Unsupported file type: ${fileExtension}`);
+          return {
+            detectedType: null,
+            fields: [],
+            detectedPedimento: null,
+            detectedPedimentoYear: null,
+            rfc: null,
+            arelaPath: null,
+            text: '',
+            error: `Unsupported file type: ${fileExtension}`,
+          };
+      }
+
+      if (!text || text.trim().length === 0) {
+        console.log('⚠️ No text extracted from file');
+        return {
+          detectedType: null,
+          fields: [],
+          detectedPedimento: null,
+          detectedPedimentoYear: null,
+          rfc: null,
+          arelaPath: null,
+          text: '',
+          error: 'No text could be extracted from file',
+        };
+      }
+
+      // Extract document fields and detect type. Use the best-match scoring
+      // engine when matchers are configured; otherwise legacy first-match-wins.
+      let detectedType, fields, detectedPedimento, detectedPedimentoYear;
+      if (this.matchers) {
+        const r = classifyDocument(this.matchers, {
+          source: text,
+          extension: fileExtension,
+          filePath,
+        });
+        detectedType = r.detectedType;
+        fields = r.fields;
+        detectedPedimento = r.detectedPedimento;
+        detectedPedimentoYear = r.detectedPedimentoYear;
+      } else {
+        [detectedType, fields, detectedPedimento, detectedPedimentoYear] =
+          extractDocumentFields(text, fileExtension, filePath);
+      }
+
+      // Extract RFC from fields
+      const rfc = fields?.find((f) => f.name === 'rfc')?.value ?? null;
+
+      // Compose arela_path for pedimento_simplificado documents
+      const arelaPath = composeArelaPath(
+        detectedType,
+        fields,
+        detectedPedimentoYear,
+        filePath,
+      );
+
+      return {
+        detectedType,
+        fields,
+        detectedPedimento,
+        detectedPedimentoYear,
+        rfc,
+        arelaPath,
+        text,
+        error: null,
+      };
+    } catch (error) {
+      console.error(`❌ Error detecting file ${filePath}:`, error.message);
+      return {
+        detectedType: null,
+        fields: [],
+        detectedPedimento: null,
+        rfc: null,
+        detectedPedimentoYear: null,
+        arelaPath: null,
+        text: '',
+        error: error.message,
+      };
+    }
+  }
+
+  /**
+   * Extract text from PDF file using pdf-parse
+   * More reliable for concurrent operations than office-text-extractor
+   * @param {string} filePath - Path to PDF file
+   * @returns {Promise<string>} - Extracted text
+   */
+  async extractTextFromPDF(filePath) {
+    let parser;
+    currentPdfFile = filePath;
+    try {
+      const dataBuffer = fs.readFileSync(filePath);
+      const uint8Array = new Uint8Array(dataBuffer);
+      parser = new PDFParse({ data: uint8Array });
+      const result = await parser.getText();
+      return result.text;
+    } catch (error) {
+      console.error(
+        `Error extracting text from PDF ${filePath}:`,
+        error.message,
+      );
+      throw new Error(`Failed to extract text from PDF: ${error.message}`);
+    } finally {
+      if (parser) {
+        await parser.destroy();
+      }
+    }
+  }
+
+  /**
+   * Detect multiple files in a directory
+   * @param {Array<string>} filePaths - Array of file paths to analyze
+   * @returns {Promise<Array>} - Array of detection results
+   */
+  async detectFiles(filePaths) {
+    const results = [];
+
+    for (const filePath of filePaths) {
+      const result = await this.detectFile(filePath);
+      results.push({
+        filePath,
+        ...result,
+      });
+    }
+
+    return results;
+  }
+
+  /**
+   * Check if file type is supported for detection
+   * @param {string} filePath - Path to file
+   * @returns {boolean} - True if file type is supported
+   */
+  isSupportedFileType(filePath) {
+    const fileExtension = path.extname(filePath).toLowerCase().replace('.', '');
+    // Support PDF (main documents), XML (metadata/supporting docs), and TXT
+    const supportedExtensions = ['pdf', 'xml', 'txt'];
+    return supportedExtensions.includes(fileExtension);
+  }
+
+  /**
+   * Filter files to only include supported types
+   * @param {Array<string>} filePaths - Array of file paths
+   * @returns {Array<string>} - Filtered array of supported file paths
+   */
+  filterSupportedFiles(filePaths) {
+    return filePaths.filter((filePath) => this.isSupportedFileType(filePath));
+  }
+}
+
+export default FileDetectionService;
+export { composeArelaPath, getCurrentPdfFile };