@arela/uploader 1.1.4 → 1.2.0

package/.claude/worktrees/agent-multi-profile/package.json

@@ -0,0 +1,78 @@
+{
+  "name": "@arela/uploader",
+  "version": "1.2.0",
+  "description": "CLI to upload files/directories to Arela",
+  "bin": {
+    "arela": "./src/index.js"
+  },
+  "type": "module",
+  "scripts": {
+    "start": "node ./src/index.js",
+    "test": "NODE_OPTIONS=--experimental-vm-modules jest",
+    "test:watch": "NODE_OPTIONS=--experimental-vm-modules jest --watch",
+    "test:coverage": "NODE_OPTIONS=--experimental-vm-modules jest --coverage",
+    "format": "prettier --write \"src/**/*.js\""
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/inspiraCode/arela-uploader.git"
+  },
+  "keywords": [
+    "arela",
+    "cli",
+    "uploader",
+    "record-keeping"
+  ],
+  "author": "Alfredo Pacheco",
+  "license": "ISC",
+  "bugs": {
+    "url": "https://github.com/inspiraCode/arela-uploader/issues"
+  },
+  "homepage": "https://github.com/inspiraCode/arela-uploader#readme",
+  "dependencies": {
+    "@supabase/supabase-js": "2.49.4",
+    "bullmq": "^5.71.0",
+    "chokidar": "^4.0.3",
+    "cli-progress": "3.12.0",
+    "commander": "13.1.0",
+    "dotenv": "16.5.0",
+    "form-data": "4.0.4",
+    "formdata-node": "^6.0.3",
+    "globby": "14.1.0",
+    "googleapis": "^171.4.0",
+    "ioredis": "^5.10.0",
+    "mime-types": "3.0.1",
+    "node-fetch": "3.3.2",
+    "office-text-extractor": "3.0.3",
+    "p-limit": "^7.2.0",
+    "pdf-parse": "^2.4.5"
+  },
+  "devDependencies": {
+    "@jest/globals": "^30.2.0",
+    "@trivago/prettier-plugin-sort-imports": "5.2.2",
+    "jest": "^30.2.0",
+    "prettier": "3.5.3"
+  },
+  "jest": {
+    "testEnvironment": "node",
+    "transform": {},
+    "testMatch": [
+      "**/tests/**/*.test.js"
+    ],
+    "testPathIgnorePatterns": [
+      "/node_modules/",
+      "/scripts/"
+    ],
+    "collectCoverageFrom": [
+      "src/commands/**/*.js",
+      "!src/commands/**/index.js"
+    ],
+    "coverageDirectory": "coverage",
+    "coverageReporters": [
+      "text",
+      "text-summary",
+      "html",
+      "lcov"
+    ]
+  }
+}

package/.claude/worktrees/agent-multi-profile/scripts/cleanup-ds-store.js

@@ -0,0 +1,109 @@
+#!/usr/bin/env node
+
+/**
+ * cleanup-ds-store.js
+ * Remove .DS_Store and other system file records from the uploader table
+ */
+
+import { fileURLToPath } from 'url';
+import path from 'path';
+import logger from './src/services/LoggingService.js';
+import { databaseService } from './src/services/DatabaseService.js';
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+
+console.log('\n╔════════════════════════════════════════════════════════════════╗');
+console.log('║         Cleaning System Files from Uploader Table             ║');
+console.log('╚════════════════════════════════════════════════════════════════╝\n');
+
+async function cleanupSystemFiles() {
+  try {
+    const supabase = await databaseService.getSupabaseClient();
+
+    // System file patterns to remove
+    const systemFiles = [
+      '.DS_Store',
+      'Thumbs.db',
+      'desktop.ini',
+      '.directory',
+    ];
+
+    let totalDeleted = 0;
+
+    for (const fileName of systemFiles) {
+      console.log(`\n🔍 Searching for ${fileName}...`);
+      
+      // Find records with this filename
+      const { data: records, error: selectError } = await supabase
+        .from('uploader')
+        .select('*')
+        .eq('filename', fileName);
+
+      if (selectError) {
+        console.error(`❌ Error searching for ${fileName}: ${selectError.message}`);
+        continue;
+      }
+
+      if (!records || records.length === 0) {
+        console.log(`   ✓ No records found for ${fileName}`);
+        continue;
+      }
+
+      console.log(`   Found ${records.length} record(s) for ${fileName}`);
+      
+      // Get the IDs to delete
+      const idsToDelete = records.map(r => r.id);
+      
+      // Delete records
+      const { error: deleteError } = await supabase
+        .from('uploader')
+        .delete()
+        .in('id', idsToDelete);
+
+      if (deleteError) {
+        console.error(`   ❌ Error deleting: ${deleteError.message}`);
+        continue;
+      }
+
+      console.log(`   ✅ Deleted ${records.length} record(s)`);
+      totalDeleted += records.length;
+    }
+
+    // Also delete any records with original_path containing .DS_Store
+    console.log(`\n🔍 Searching for records with .DS_Store in path...`);
+    const { data: pathRecords, error: pathError } = await supabase
+      .from('uploader')
+      .select('*')
+      .ilike('original_path', '%.DS_Store%');
+
+    if (!pathError && pathRecords && pathRecords.length > 0) {
+      console.log(`   Found ${pathRecords.length} record(s)`);
+      const idsToDelete = pathRecords.map(r => r.id);
+      
+      const { error: deleteError } = await supabase
+        .from('uploader')
+        .delete()
+        .in('id', idsToDelete);
+
+      if (!deleteError) {
+        console.log(`   ✅ Deleted ${pathRecords.length} record(s)`);
+        totalDeleted += pathRecords.length;
+      } else {
+        console.error(`   ❌ Error deleting: ${deleteError.message}`);
+      }
+    }
+
+    console.log('\n╔════════════════════════════════════════════════════════════════╗');
+    console.log(`║  ✅ Cleanup Complete: Deleted ${totalDeleted} system file record(s)                  ║`);
+    console.log('╚════════════════════════════════════════════════════════════════╝\n');
+
+    process.exit(0);
+  } catch (error) {
+    console.error('\n❌ ERROR');
+    console.error(error.message);
+    process.exit(1);
+  }
+}
+
+cleanupSystemFiles();

package/.claude/worktrees/agent-multi-profile/scripts/cleanup-system-files.js

@@ -0,0 +1,69 @@
+#!/usr/bin/env node
+
+/**
+ * cleanup-system-files.js
+ * Remove system files (.DS_Store, etc) from the uploader table
+ */
+
+import { fileURLToPath } from 'url';
+import path from 'path';
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+
+console.log('\n╔════════════════════════════════════════════════════════════════╗');
+console.log('║           Cleaning System Files from Database                  ║');
+console.log('╚════════════════════════════════════════════════════════════════╝\n');
+
+try {
+  const databaseService = (await import('./src/services/DatabaseService.js')).default;
+  const supabase = await databaseService.getSupabaseClient();
+
+  // System file patterns to remove
+  const systemFilePatterns = ['.DS_Store', '__pycache__', '.pyc', '.swp', '.swo', 'Thumbs.db', 'desktop.ini'];
+
+  console.log('🔍 Searching for system files...\n');
+
+  let totalRemoved = 0;
+
+  for (const pattern of systemFilePatterns) {
+    const { data: records, error } = await supabase
+      .from('uploader')
+      .select('id, filename')
+      .ilike('filename', `%${pattern}%`);
+
+    if (!error && records && records.length > 0) {
+      console.log(`📝 Found ${records.length} record(s) with "${pattern}"`);
+      records.forEach(r => {
+        console.log(`   - ID: ${r.id}, Filename: ${r.filename}`);
+      });
+
+      // Delete them
+      const ids = records.map(r => r.id);
+      const { error: deleteError } = await supabase
+        .from('uploader')
+        .delete()
+        .in('id', ids);
+
+      if (!deleteError) {
+        console.log(`   ✅ Deleted ${records.length} record(s)\n`);
+        totalRemoved += records.length;
+      } else {
+        console.log(`   ❌ Error deleting records: ${deleteError.message}\n`);
+      }
+    }
+  }
+
+  console.log('╔════════════════════════════════════════════════════════════════╗');
+  console.log(`║                    ✅ CLEANUP COMPLETED                        ║`);
+  console.log(`║                                                                ║`);
+  console.log(`║  Total system files removed: ${totalRemoved}`);
+  console.log('╚════════════════════════════════════════════════════════════════╝\n');
+
+  process.exit(0);
+} catch (error) {
+  console.error('\n❌ CLEANUP FAILED');
+  console.error(error.message);
+  console.error(error.stack);
+  process.exit(1);
+}

package/.claude/worktrees/agent-multi-profile/scripts/scoring-compare.js

@@ -0,0 +1,243 @@
+/**
+ * Scoring engine validation harness (PROTOTYPE).
+ *
+ * Runs the CURRENT first-match-wins engine (`extractDocumentFields`) and the new
+ * best-match scoring engine (`classifyDocument`) over the same corpus and prints
+ * a side-by-side comparison so we can confirm best-match reproduces (or
+ * improves on) the current behaviour before wiring it into the pipeline.
+ *
+ * Usage:
+ *   node scripts/scoring-compare.js                # built-in synthetic samples
+ *   node scripts/scoring-compare.js <folder>       # + real .pdf/.xml/.txt files
+ *
+ * The built-in samples include the `factura_inter_agencia` vs
+ * `factura_comercial` case, which the current engine only resolves via
+ * registration order — the harness shows best-match resolving it by score,
+ * independent of matcher order.
+ */
+import fs from 'fs';
+import path from 'path';
+
+import { extractDocumentFields } from '../src/document-type-shared.js';
+import FileDetectionService from '../src/file-detection.js';
+import { classifyDocument, scoreAll } from '../src/scoring/scoring-engine.js';
+import { scoringMatchers } from '../src/scoring/matchers-seed.js';
+
+// --------------------------- synthetic corpus -------------------------------
+// Compact, representative texts that trigger the relevant clues. Real pdf-parse
+// output is messier — pass a folder to validate against production documents.
+const SAMPLES = [
+  {
+    name: 'simplificado-paid',
+    extension: 'pdf',
+    expected: 'pedimento_simplificado',
+    text: `FORMA SIMPLIFICADA DEL PEDIMENTO
+NUM. PEDIMENTO: 26 07 3429 6000079
+CVE. PEDIMENTO: A1
+T. OPER: IMP
+RFC: CSM9204097Q1
+FECHA DE PAGO: 04/03/2026
+*** PAGO ELECTRONICO ***`,
+  },
+  {
+    name: 'simplificado-unpaid (proforma)',
+    extension: 'pdf',
+    expected: 'proforma',
+    text: `FORMA SIMPLIFICADA DE PEDIMENTO
+NUM. PEDIMENTO: 26 07 3429 6000080
+CVE. PEDIMENTO: A1
+T. OPER: IMP
+RFC: CSM9204097Q1
+*** NO PAGADO ***`,
+  },
+  {
+    name: 'completo',
+    extension: 'pdf',
+    expected: 'pedimento_completo',
+    text: `NUM. PEDIMENTO: 26 07 3429 2002089
+CVE. PEDIMENTO: A1
+T. OPER: IMP
+SEGUNDA COPIA TRANSPORTISTA
+CERTIFICACIONES
+CUADRO DE LIQUIDACION
+*** PAGO ELECTRONICO ***
+FECHA DE PAGO: 02/03/2026`,
+  },
+  {
+    name: 'completo-xml',
+    extension: 'xml',
+    filePath: '/tmp/260734296016642.xml',
+    expected: 'pedimento_completo_xml',
+    text: `<?xml version="1.0"?>
+<ns2:consultarPedimentoCompletoRespuesta>
+  <ns2:pedimento>6016642</ns2:pedimento>
+  <ns2:aduanaEntradaSalida><ns2:clave>70</ns2:clave></ns2:aduanaEntradaSalida>
+  <ns2:fechas><ns2:clave>2</ns2:clave><ns2:fecha>2026-03-02-06:00</ns2:fecha></ns2:fechas>
+  <ns2:fechas><ns2:clave>5</ns2:clave><ns2:fecha>2026-02-20-06:00</ns2:fecha></ns2:fechas>
+  <ns2:rfc>CSM9204097Q1</ns2:rfc>
+</ns2:consultarPedimentoCompletoRespuesta>`,
+  },
+  {
+    name: 'doda-pdf',
+    extension: 'pdf',
+    expected: 'doda_pdf',
+    text: `DOCUMENTO DE OPERACION PARA DESPACHO ADUANERO
+DODA
+VUCEM
+||070|3429|2|4009029|109335668|A231|
+2026-03-02`,
+  },
+  {
+    name: 'doda-xml',
+    extension: 'xml',
+    expected: 'doda_xml',
+    text: `<?xml version="1.0"?>
+<documentoOperacion>
+  <numPedimento>260734292002089</numPedimento>
+  <patenteAduanal>3429</patenteAduanal>
+  <aduanaDespacho>07</aduanaDespacho>
+</documentoOperacion>`,
+  },
+  {
+    name: 'inter-agencia (vs comercial)',
+    extension: 'xml',
+    expected: 'factura_inter_agencia',
+    text: `<cfdi:Comprobante xmlns:cfdi="..." TipoDeComprobante="I">
+  <cfdi:Emisor Rfc="NAA120215F20"/>
+  <cfdi:Receptor Rfc="PCC1008161WA"/>
+  <cfdi:Concepto ClaveProdServ="78141502" Descripcion="Servicios de agente aduanal"/>
+</cfdi:Comprobante>`,
+  },
+  {
+    name: 'factura-comercial',
+    extension: 'xml',
+    expected: 'factura_comercial',
+    text: `<cfdi:Comprobante xmlns:cfdi="..." TipoDeComprobante="I">
+  <cfdi:Emisor Rfc="ABC010101AB1"/>
+  <cfdi:Receptor Rfc="XYZ020202CD2"/>
+  <tfd:TimbreFiscalDigital/>
+  pedimento 26 07 3429 6016477
+</cfdi:Comprobante>`,
+  },
+  {
+    name: 'support-document',
+    extension: 'xml',
+    expected: 'support_document',
+    text: `<?xml version="1.0"?>
+<soapenv:Envelope xmlns:soapenv="http://schemas.xmlsoap.org/soap/envelope/">
+  <oxml:tipoOperacion>IMP</oxml:tipoOperacion>
+  <oxml:patenteAduanal>3429</oxml:patenteAduanal>
+</soapenv:Envelope>`,
+  },
+];
+
+// --------------------------- comparison -------------------------------------
+function firstMatchType(source, extension, filePath) {
+  const [type] = extractDocumentFields(source, extension, filePath);
+  return type;
+}
+
+function bestMatchResult(source, extension, filePath) {
+  return classifyDocument(scoringMatchers, { source, extension, filePath });
+}
+
+function topCandidates(source, extension, filePath, n = 3) {
+  return scoreAll(scoringMatchers, {
+    source,
+    extension,
+    fileName: filePath ? path.basename(filePath) : '',
+  })
+    .slice(0, n)
+    .map((c) => `${c.documentType}:${c.score}`)
+    .join(', ');
+}
+
+function row(name, first, best, expected) {
+  const agree = first === best ? 'sí ' : 'NO ';
+  const vsExp = expected ? (best === expected ? 'ok ' : '⚠️ ') : '   ';
+  return (
+    `${name.padEnd(34)} first=${String(first).padEnd(24)} ` +
+    `best=${String(best).padEnd(24)} coinciden=${agree} esperado=${vsExp}`
+  );
+}
+
+async function run() {
+  const folder = process.argv[2];
+  let total = 0;
+  let disagreements = 0;
+
+  console.log('\n=== Muestras sintéticas ===');
+  for (const s of SAMPLES) {
+    const first = firstMatchType(s.text, s.extension, s.filePath);
+    const best = bestMatchResult(s.text, s.extension, s.filePath).detectedType;
+    total++;
+    if (first !== best) disagreements++;
+    console.log(row(s.name, first, best, s.expected));
+  }
+
+  // Order-independence demonstration for the inter-agencia/comercial case.
+  const ia = SAMPLES.find((s) => s.name.startsWith('inter-agencia'));
+  const reversed = [...scoringMatchers].reverse();
+  const normalWinner = classifyDocument(scoringMatchers, {
+    source: ia.text,
+    extension: ia.extension,
+  }).detectedType;
+  const reversedWinner = classifyDocument(reversed, {
+    source: ia.text,
+    extension: ia.extension,
+  }).detectedType;
+  console.log('\n=== Independencia de orden (inter-agencia) ===');
+  console.log(`candidatos (por score): ${topCandidates(ia.text, ia.extension)}`);
+  console.log(`seed normal   -> ${normalWinner}`);
+  console.log(`seed invertido-> ${reversedWinner}`);
+  console.log(
+    `order-independent: ${normalWinner === reversedWinner ? 'sí ✅' : 'NO ❌'}`,
+  );
+
+  // Optional: real files from a folder.
+  if (folder) {
+    if (!fs.existsSync(folder)) {
+      console.error(`\nCarpeta no existe: ${folder}`);
+    } else {
+      console.log(`\n=== Archivos reales (${folder}) ===`);
+      const detection = new FileDetectionService();
+      const files = walk(folder).filter((f) =>
+        ['.pdf', '.xml', '.txt'].includes(path.extname(f).toLowerCase()),
+      );
+      for (const file of files) {
+        const ext = path.extname(file).toLowerCase().replace('.', '');
+        let text = '';
+        try {
+          text =
+            ext === 'pdf'
+              ? await detection.extractTextFromPDF(file)
+              : fs.readFileSync(file, 'utf8');
+        } catch (err) {
+          console.log(`${path.basename(file).padEnd(34)} ERROR: ${err.message}`);
+          continue;
+        }
+        const first = firstMatchType(text, ext, file);
+        const best = bestMatchResult(text, ext, file).detectedType;
+        total++;
+        if (first !== best) disagreements++;
+        console.log(row(path.basename(file), first, best, null));
+      }
+    }
+  }
+
+  console.log(
+    `\n=== Resumen: ${total} documentos, ${disagreements} divergencias first-vs-best ===\n`,
+  );
+}
+
+function walk(dir) {
+  const out = [];
+  for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {
+    const full = path.join(dir, entry.name);
+    if (entry.isDirectory()) out.push(...walk(full));
+    else out.push(full);
+  }
+  return out;
+}
+
+run();

package/.claude/worktrees/agent-multi-profile/scripts/scoring-phase4-check.js

@@ -0,0 +1,96 @@
+/**
+ * Phase 4 validation: runs the REAL runtime path the uploader now uses
+ * (DB-shape matchers -> adaptDbMatchers -> classifyDocument with rich extraction)
+ * against a corpus and compares it to the legacy engine (extractDocumentFields).
+ *
+ * Usage: node scripts/scoring-phase4-check.js <folder>
+ */
+import fs from 'fs';
+import path from 'path';
+
+import { extractDocumentFields } from '../src/document-type-shared.js';
+import FileDetectionService from '../src/file-detection.js';
+import { adaptDbMatchers } from '../src/scoring/db-matcher-adapter.js';
+import { scoringMatchers } from '../src/scoring/matchers-seed.js';
+import { classifyDocument } from '../src/scoring/scoring-engine.js';
+
+// Serialize the local seed to the shape the API `/resolved` endpoint returns,
+// so we exercise the adapter exactly as in production.
+function toDbShape(matchers) {
+  return matchers.map((m) => ({
+    documentType: m.documentType,
+    extensions: m.extensions,
+    minScore: m.minScore ?? null,
+    priority: m.priority ?? 0,
+    qualify: m.qualify ?? null,
+    clues: (m.clues || []).map((c) => ({
+      kind: c.kind,
+      pattern: c.pattern instanceof RegExp ? c.pattern.source : c.pattern,
+      flags: c.pattern instanceof RegExp ? c.pattern.flags : c.flags || '',
+      weight: c.weight ?? 1,
+      group: c.group ?? null,
+      required: !!c.required,
+      negative: !!c.negative,
+    })),
+    fieldExtractors: [], // rich extraction comes from the registry by documentType
+  }));
+}
+
+const adapted = adaptDbMatchers(toDbShape(scoringMatchers));
+
+function walk(dir) {
+  const out = [];
+  for (const e of fs.readdirSync(dir, { withFileTypes: true })) {
+    const full = path.join(dir, e.name);
+    if (e.isDirectory()) out.push(...walk(full));
+    else out.push(full);
+  }
+  return out;
+}
+
+async function run() {
+  const folder = process.argv[2];
+  if (!folder) {
+    console.error('Pass a folder: node scripts/scoring-phase4-check.js <folder>');
+    process.exit(1);
+  }
+  const detection = new FileDetectionService();
+  const files = walk(folder).filter((f) =>
+    ['.pdf', '.xml', '.txt'].includes(path.extname(f).toLowerCase()),
+  );
+
+  let total = 0;
+  let diverge = 0;
+  const patterns = {};
+
+  for (const file of files) {
+    const ext = path.extname(file).toLowerCase().replace('.', '');
+    let text = '';
+    try {
+      text =
+        ext === 'pdf'
+          ? await detection.extractTextFromPDF(file)
+          : fs.readFileSync(file, 'utf8');
+    } catch {
+      continue;
+    }
+    const legacy = extractDocumentFields(text, ext, file)[0];
+    const phase4 = classifyDocument(adapted, {
+      source: text,
+      extension: ext,
+      filePath: file,
+    }).detectedType;
+    total++;
+    if (legacy !== phase4) {
+      diverge++;
+      const key = `${legacy} -> ${phase4}`;
+      patterns[key] = (patterns[key] || 0) + 1;
+      console.log(`NO  ${path.basename(file).padEnd(40)} ${key}`);
+    }
+  }
+
+  console.log(`\n=== Fase 4 vs legacy: ${total} docs, ${diverge} divergencias ===`);
+  for (const [k, n] of Object.entries(patterns)) console.log(`  ${n}×  ${k}`);
+}
+
+run();