@arela/uploader 1.0.20 → 1.0.21

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@arela/uploader",
-  "version": "1.0.20",
+  "version": "1.0.21",
   "description": "CLI to upload files/directories to Arela",
   "bin": {
     "arela": "./src/index.js"
@@ -39,6 +39,7 @@
     "form-data": "4.0.4",
     "formdata-node": "^6.0.3",
     "globby": "14.1.0",
+    "googleapis": "^171.4.0",
     "ioredis": "^5.10.0",
     "mime-types": "3.0.1",
     "node-fetch": "3.3.2",

package/src/commands/GDriveSyncCommand.js

@@ -0,0 +1,475 @@
+import cliProgress from 'cli-progress';
+import fs from 'fs';
+import fsp from 'fs/promises';
+import pLimit from 'p-limit';
+import path from 'path';
+
+import GoogleDriveService from '../services/GoogleDriveService.js';
+import logger from '../services/LoggingService.js';
+
+import appConfig from '../config/config.js';
+import ErrorHandler from '../errors/ErrorHandler.js';
+import { FileSanitizer } from '../utils/FileSanitizer.js';
+
+const STATE_FILENAME = '.gdrive-sync-state.json';
+const STATE_VERSION = 1;
+
+/**
+ * GDrive Sync Command
+ *
+ * Mirrors a Google Drive folder tree to a local directory so the existing
+ * scan → identify → propagate → push pipeline can run unchanged.
+ *
+ * Idempotent & incremental: maintains a `.gdrive-sync-state.json` at the
+ * mirror root with per-file md5/modifiedTime so re-runs only download
+ * changed/new files.
+ */
+export class GDriveSyncCommand {
+  constructor() {
+    this.errorHandler = new ErrorHandler(logger);
+    this.driveService = null;
+    this.sanitizer = new FileSanitizer();
+    this.onProgress = null;
+  }
+
+  /**
+   * Execute the gdrive-sync command
+   * @param {Object} [options]
+   * @param {string} [options.rootFolder] - override GDRIVE_ROOT_FOLDER_ID
+   * @param {string} [options.dest] - override local mirror path
+   * @param {boolean} [options.full] - ignore state file and re-verify everything
+   * @param {boolean} [options.dryRun] - list/plan only, no downloads or writes
+   * @param {Function} [options.onProgress]
+   */
+  async execute(options = {}) {
+    const startTime = Date.now();
+    this.onProgress = options.onProgress || null;
+
+    try {
+      // Allow CLI overrides before validation
+      if (options.rootFolder) {
+        process.env.GDRIVE_ROOT_FOLDER_ID = options.rootFolder;
+      }
+      if (options.dest) {
+        process.env.GDRIVE_LOCAL_MIRROR_PATH = options.dest;
+      }
+      // Reload config after env mutation
+      const cfg = appConfig.getGDriveConfig();
+      if (options.rootFolder) cfg.rootFolderId = options.rootFolder;
+      if (options.dest) cfg.localMirrorPath = options.dest;
+
+      appConfig.validateGDriveConfig();
+
+      const dryRun = !!options.dryRun;
+      const full = !!options.full;
+
+      this.driveService = new GoogleDriveService();
+
+      this.#say('☁️  Starting arela gdrive-sync command');
+      this.#say(`📁 Root folder ID: ${cfg.rootFolderId}`);
+      this.#say(`💾 Local mirror:   ${cfg.localMirrorPath}`);
+      this.#say(`⚙️  Concurrency:    ${cfg.concurrency}`);
+      this.#say(`📄 Skip native docs: ${cfg.skipNativeDocs}`);
+      if (dryRun) this.#say('🧪 DRY-RUN: no files will be written');
+      if (full) this.#say('♻️  FULL: ignoring state, re-verifying all files');
+
+      // Ensure mirror dir exists
+      if (!dryRun) {
+        await fsp.mkdir(cfg.localMirrorPath, { recursive: true });
+      }
+
+      // Load existing state
+      const statePath = path.join(cfg.localMirrorPath, STATE_FILENAME);
+      const state = full
+        ? this.#emptyState()
+        : await this.#loadState(statePath);
+
+      // Verify root folder exists & is accessible
+      this.#say('\n🔍 Verifying root folder access...');
+      const rootMeta = await this.driveService.getFile(cfg.rootFolderId);
+      if (!GoogleDriveService.isFolder(rootMeta)) {
+        throw new Error(
+          `GDRIVE_ROOT_FOLDER_ID does not point to a folder (mimeType=${rootMeta.mimeType})`,
+        );
+      }
+      logger.success(`  ✓ Root: "${rootMeta.name}"`);
+
+      // Walk Drive tree → produce file plan
+      this.#say('\n🌲 Walking Drive tree...');
+      this.#reportProgress(0, 'Walking Drive tree');
+      const plan = await this.#walkTree(cfg.rootFolderId, '', cfg);
+      this.#say(
+        `📋 Found ${plan.folders.length} folder(s) and ${plan.files.length} file(s)`,
+      );
+
+      // Stats
+      const stats = {
+        foldersCreated: 0,
+        filesAdded: 0,
+        filesUpdated: 0,
+        filesSkipped: 0,
+        filesFailed: 0,
+        nativeDocsSkipped: 0,
+        oversizedSkipped: 0,
+        bytesDownloaded: 0,
+      };
+
+      // Create folders first (cheap, sequential)
+      if (!dryRun) {
+        for (const folderRel of plan.folders) {
+          const abs = path.join(cfg.localMirrorPath, folderRel);
+          try {
+            await fsp.mkdir(abs, { recursive: true });
+            stats.foldersCreated += 1;
+          } catch (err) {
+            logger.warn(`⚠️  mkdir failed for ${abs}: ${err.message}`);
+          }
+        }
+      }
+
+      // Download files concurrently
+      this.#say('\n⬇️  Downloading files...');
+      const limit = pLimit(cfg.concurrency);
+      const progressBar = this.#createProgressBar(plan.files.length, dryRun);
+      let processed = 0;
+
+      const tasks = plan.files.map((file) =>
+        limit(async () => {
+          try {
+            const result = await this.#processFile(file, cfg, state, dryRun);
+            if (result === 'added') stats.filesAdded += 1;
+            else if (result === 'updated') stats.filesUpdated += 1;
+            else if (result === 'skipped-unchanged') stats.filesSkipped += 1;
+            else if (result === 'skipped-native') stats.nativeDocsSkipped += 1;
+            else if (result === 'skipped-oversize') stats.oversizedSkipped += 1;
+            if (
+              (result === 'added' || result === 'updated') &&
+              file.size != null
+            ) {
+              stats.bytesDownloaded += Number(file.size) || 0;
+            }
+          } catch (err) {
+            stats.filesFailed += 1;
+            logger.error(
+              `❌ Failed: ${file.relPath} (${file.id}) — ${err.message}`,
+            );
+          } finally {
+            processed += 1;
+            progressBar.update(processed);
+            if (plan.files.length > 0) {
+              this.#reportProgress(
+                Math.round((processed / plan.files.length) * 100),
+                `Synced ${processed}/${plan.files.length}`,
+              );
+            }
+          }
+        }),
+      );
+
+      await Promise.all(tasks);
+      progressBar.stop();
+
+      // Persist state
+      if (!dryRun) {
+        await this.#saveState(statePath, state);
+      }
+
+      const duration = ((Date.now() - startTime) / 1000).toFixed(2);
+      this.#reportProgress(100, `Sync completed in ${duration}s`);
+
+      logger.success('\n✅ gdrive-sync completed successfully!');
+      this.#say('\n📊 Sync Statistics:');
+      this.#say(`   Folders created:      ${stats.foldersCreated}`);
+      this.#say(`   Files added:          ${stats.filesAdded}`);
+      this.#say(`   Files updated:        ${stats.filesUpdated}`);
+      this.#say(`   Files unchanged:      ${stats.filesSkipped}`);
+      this.#say(`   Native docs skipped:  ${stats.nativeDocsSkipped}`);
+      this.#say(`   Oversized skipped:    ${stats.oversizedSkipped}`);
+      this.#say(`   Files failed:         ${stats.filesFailed}`);
+      this.#say(
+        `   Bytes downloaded:     ${this.#formatBytes(stats.bytesDownloaded)}`,
+      );
+      this.#say(`   Duration:             ${duration}s`);
+      this.#say(
+        `\n💡 Next: run "arela scan" with UPLOAD_BASE_PATH=${cfg.localMirrorPath}`,
+      );
+
+      return { success: true, stats };
+    } catch (error) {
+      this.errorHandler.handleError(error, 'gdrive-sync');
+      return { success: false, error: error.message };
+    }
+  }
+
+  /**
+   * Walk Drive tree starting at folderId.
+   * Returns flat lists of folders (relative paths) and files (with metadata + relPath).
+   * @private
+   */
+  async #walkTree(folderId, relPath, cfg) {
+    const folders = [];
+    const files = [];
+
+    // BFS using a queue
+    const queue = [{ id: folderId, relPath }];
+    // Track sanitized names per folder to dedupe collisions
+    const folderNameMaps = new Map();
+
+    while (queue.length > 0) {
+      const { id, relPath: parentRel } = queue.shift();
+      const usedNames = folderNameMaps.get(parentRel) || new Set();
+
+      for await (const child of this.driveService.listChildren(id)) {
+        const safeName = this.#uniqueSanitizedName(
+          child.name,
+          child.id,
+          usedNames,
+        );
+        const childRel = parentRel ? `${parentRel}/${safeName}` : safeName;
+
+        if (GoogleDriveService.isFolder(child)) {
+          folders.push(childRel);
+          folderNameMaps.set(childRel, new Set());
+          queue.push({ id: child.id, relPath: childRel });
+          continue;
+        }
+
+        // Resolve shortcuts to their targets if enabled
+        let effective = child;
+        if (GoogleDriveService.isShortcut(child)) {
+          if (!cfg.followShortcuts || !child.shortcutDetails?.targetId) {
+            logger.debug(`⏭️  Skipping shortcut: ${childRel}`);
+            continue;
+          }
+          try {
+            effective = await this.driveService.getFile(
+              child.shortcutDetails.targetId,
+            );
+          } catch (err) {
+            logger.warn(
+              `⚠️  Could not resolve shortcut "${childRel}": ${err.message}`,
+            );
+            continue;
+          }
+          if (GoogleDriveService.isFolder(effective)) {
+            folders.push(childRel);
+            folderNameMaps.set(childRel, new Set());
+            queue.push({ id: effective.id, relPath: childRel });
+            continue;
+          }
+        }
+
+        files.push({
+          id: effective.id,
+          name: effective.name,
+          mimeType: effective.mimeType,
+          modifiedTime: effective.modifiedTime,
+          size: effective.size != null ? Number(effective.size) : null,
+          md5Checksum: effective.md5Checksum || null,
+          relPath: childRel,
+          isNativeDoc: GoogleDriveService.isNativeGoogleDoc(effective),
+        });
+      }
+
+      folderNameMaps.set(parentRel, usedNames);
+    }
+
+    return { folders, files };
+  }
+
+  /**
+   * Process a single file: skip / download / update.
+   * @private
+   */
+  async #processFile(file, cfg, state, dryRun) {
+    // Native Google Docs (Docs/Sheets/Slides/...): skip by default
+    if (file.isNativeDoc && cfg.skipNativeDocs) {
+      logger.debug(`⏭️  Native Google Doc skipped: ${file.relPath}`);
+      return 'skipped-native';
+    }
+
+    // Oversize guard
+    if (
+      cfg.maxFileSizeBytes &&
+      file.size != null &&
+      file.size > cfg.maxFileSizeBytes
+    ) {
+      logger.warn(
+        `⚠️  File too large (${this.#formatBytes(file.size)}): ${file.relPath}`,
+      );
+      return 'skipped-oversize';
+    }
+
+    const absPath = path.join(cfg.localMirrorPath, file.relPath);
+    const prev = state.files[file.id];
+
+    // Decide if we can skip (unchanged)
+    const localExists = fs.existsSync(absPath);
+    const unchanged =
+      localExists &&
+      prev &&
+      prev.relPath === file.relPath &&
+      ((file.md5Checksum && prev.md5Checksum === file.md5Checksum) ||
+        (!file.md5Checksum && prev.modifiedTime === file.modifiedTime));
+
+    if (unchanged) {
+      return 'skipped-unchanged';
+    }
+
+    if (dryRun) {
+      logger.info(
+        `   [dry-run] would ${prev ? 'update' : 'add'}: ${file.relPath}`,
+      );
+      return prev ? 'updated' : 'added';
+    }
+
+    // Ensure parent dir
+    await fsp.mkdir(path.dirname(absPath), { recursive: true });
+
+    // Download to .part then atomic rename
+    const partPath = `${absPath}.part`;
+    try {
+      await fsp.rm(partPath, { force: true });
+    } catch {
+      /* ignore */
+    }
+
+    const out = fs.createWriteStream(partPath);
+    await this.driveService.downloadFile(file.id, out);
+    await fsp.rename(partPath, absPath);
+
+    // Set mtime to Drive's modifiedTime so `arela scan` captures changes correctly
+    if (file.modifiedTime) {
+      const mtime = new Date(file.modifiedTime);
+      try {
+        await fsp.utimes(absPath, mtime, mtime);
+      } catch (err) {
+        logger.debug(`⚠️  utimes failed for ${absPath}: ${err.message}`);
+      }
+    }
+
+    // Update state
+    state.files[file.id] = {
+      relPath: file.relPath,
+      md5Checksum: file.md5Checksum,
+      modifiedTime: file.modifiedTime,
+      size: file.size,
+      syncedAt: new Date().toISOString(),
+    };
+
+    return prev ? 'updated' : 'added';
+  }
+
+  /**
+   * Sanitize file/folder name; on duplicate, append " (gdrive-<id>)".
+   * @private
+   */
+  #uniqueSanitizedName(originalName, fileId, usedNames) {
+    let safe = this.sanitizer.sanitizeFileName(originalName);
+    if (!safe) safe = `file-${fileId}`;
+
+    if (usedNames.has(safe)) {
+      const ext = path.extname(safe);
+      const base = path.basename(safe, ext);
+      safe = `${base} (gdrive-${fileId})${ext}`;
+    }
+    usedNames.add(safe);
+    return safe;
+  }
+
+  /**
+   * @private
+   */
+  async #loadState(statePath) {
+    try {
+      const raw = await fsp.readFile(statePath, 'utf-8');
+      const parsed = JSON.parse(raw);
+      if (parsed.version !== STATE_VERSION) {
+        logger.warn(
+          `⚠️  State file version mismatch (${parsed.version} != ${STATE_VERSION}); starting fresh`,
+        );
+        return this.#emptyState();
+      }
+      return parsed;
+    } catch (err) {
+      if (err.code !== 'ENOENT') {
+        logger.warn(`⚠️  Could not read state file: ${err.message}`);
+      }
+      return this.#emptyState();
+    }
+  }
+
+  /**
+   * @private
+   */
+  async #saveState(statePath, state) {
+    state.savedAt = new Date().toISOString();
+    const tmp = `${statePath}.tmp`;
+    await fsp.writeFile(tmp, JSON.stringify(state, null, 2), 'utf-8');
+    await fsp.rename(tmp, statePath);
+  }
+
+  /**
+   * @private
+   */
+  #emptyState() {
+    return { version: STATE_VERSION, savedAt: null, files: {} };
+  }
+
+  /**
+   * @private
+   */
+  #createProgressBar(total, dryRun) {
+    const label = dryRun ? '🧪 Planning' : '⬇️  Downloading';
+    const bar = new cliProgress.SingleBar(
+      {
+        format: `${label} |{bar}| {percentage}% | {value}/{total} files`,
+        barCompleteChar: '\u2588',
+        barIncompleteChar: '\u2591',
+        hideCursor: true,
+      },
+      cliProgress.Presets.shades_classic,
+    );
+    bar.start(Math.max(total, 1), 0);
+    return bar;
+  }
+
+  /**
+   * @private
+   */
+  #reportProgress(percent, message) {
+    if (typeof this.onProgress === 'function') {
+      try {
+        this.onProgress(percent, message);
+      } catch {
+        /* ignore listener errors */
+      }
+    }
+  }
+
+  /**
+   * Print to stdout AND log to file (mirrors ScanCommand's user-facing output style).
+   * @private
+   */
+  #say(message) {
+    console.log(message);
+    logger.info(message);
+  }
+
+  /**
+   * @private
+   */
+  #formatBytes(bytes) {
+    if (!bytes) return '0 B';
+    const units = ['B', 'KB', 'MB', 'GB', 'TB'];
+    let i = 0;
+    let n = bytes;
+    while (n >= 1024 && i < units.length - 1) {
+      n /= 1024;
+      i += 1;
+    }
+    return `${n.toFixed(2)} ${units[i]}`;
+  }
+}
+
+export default new GDriveSyncCommand();

package/src/commands/IdentifyCommand.js

@@ -9,6 +9,24 @@ import ErrorHandler from '../errors/ErrorHandler.js';
 import { ConfigurationError } from '../errors/ErrorTypes.js';
 import FileDetectionService from '../file-detection.js';
 
+/**
+ * Paid pedimento detected_type values. `pedimento_completo_xml` is included
+ * even though the XML matcher is currently disabled in the registry so that
+ * re-enabling it requires no changes here.
+ */
+const DETECTED_PEDIMENTO_TYPES = new Set([
+  'pedimento_simplificado',
+  'pedimento_completo',
+  'pedimento_completo_xml',
+]);
+
+/** Unpaid pedimento detected_type values (proforma variants). */
+const PROFORMA_TYPES = new Set([
+  'proforma',
+  'proforma_completo',
+  'proforma_completo_xml',
+]);
+
 /**
  * Identify Command Handler
  * Optimized replacement for "detect --detect-pdfs"
@@ -247,11 +265,13 @@ export class IdentifyCommand {
 
       // Update statistics
       processedCount += files.length;
-      detectedCount += detectionResults.filter(
-        (r) => r.detectedType === 'pedimento_simplificado',
+      // "Detected" counts paid pedimentos of any flavour (simplificado,
+      // completo, completo_xml when enabled).
+      detectedCount += detectionResults.filter((r) =>
+        DETECTED_PEDIMENTO_TYPES.has(r.detectedType),
       ).length;
-      proformaCount += detectionResults.filter(
-        (r) => r.detectedType === 'proforma',
+      proformaCount += detectionResults.filter((r) =>
+        PROFORMA_TYPES.has(r.detectedType),
       ).length;
       errorCount += detectionResults.filter((r) => r.detectionError).length;
 
@@ -324,8 +344,8 @@ export class IdentifyCommand {
           // Detect using existing FileDetectionService
           const result = await this.detectionService.detectFile(absolutePath);
 
-          // If detection succeeded and found a pedimento_simplificado (paid)
-          if (result.detectedType === 'pedimento_simplificado') {
+          // If detection succeeded and found a paid pedimento (any variant)
+          if (DETECTED_PEDIMENTO_TYPES.has(result.detectedType)) {
             return {
               id: file.id,
               detectedType: result.detectedType,
@@ -338,8 +358,8 @@ export class IdentifyCommand {
             };
           }
 
-          // If detection succeeded and found a proforma (unpaid pedimento)
-          if (result.detectedType === 'proforma') {
+          // If detection succeeded and found a proforma (any variant)
+          if (PROFORMA_TYPES.has(result.detectedType)) {
             return {
               id: file.id,
               detectedType: result.detectedType,
@@ -365,7 +385,7 @@ export class IdentifyCommand {
             detectionError = `DETECTION_ERROR: ${result.error}`;
           } else if (isDefinitelyNotPedimento) {
             detectionError =
-              'NOT_PEDIMENTO: File does not match pedimento-simplificado pattern. Missing key markers: "FORMA SIMPLIFICADA DE PEDIMENTO".';
+              'NOT_PEDIMENTO: File does not match any pedimento pattern. Missing key markers (e.g. "FORMA SIMPLIFICADA DE PEDIMENTO" or "NUM. PEDIMENTO:" + copy markers).';
           } else {
             // Partial match - might be a pedimento with missing fields
             const missingFields = this.#getMissingFields(result);
@@ -434,18 +454,23 @@ export class IdentifyCommand {
       return false;
     }
 
-    // If it was detected as a proforma, it's related to a pedimento structure
-    if (result.detectedType === 'proforma') {
+    // If it was detected as a proforma (any variant), it's related to a
+    // pedimento structure — not "definitely not".
+    if (PROFORMA_TYPES.has(result.detectedType)) {
       return false;
     }
 
-    // Check if the text contains the required pedimento marker
-    // This must match the criteria in pedimento-simplificado.js match function
+    // Check if the text contains any required pedimento marker. This must
+    // stay aligned with the `match()` predicates in pedimento-simplificado.js
+    // and pedimento-completo.js.
     const text = result.text || '';
-    const hasRequiredMarker = /FORMA SIMPLIFICADA DE PEDIMENTO/i.test(text);
+    const hasSimplificadoMarker = /FORMA SIMPLIFICADA DE PEDIMENTO/i.test(text);
+    const hasCompletoMarkers =
+      /NUM\.?\s*PEDIMENTO:/i.test(text) &&
+      /CVE\.?\s*PEDIMENTO:/i.test(text) &&
+      /T\.?\s*OPER:/i.test(text);
 
-    // If the required marker is not found, it's definitely not a pedimento
-    return !hasRequiredMarker;
+    return !hasSimplificadoMarker && !hasCompletoMarkers;
   }
 
   /**

package/src/commands/ScanCommand.js

@@ -592,10 +592,13 @@ export class ScanCommand {
     const relativePath = PathNormalizer.getRelativePath(filePath, basePath);
     const absolutePath = PathNormalizer.normalizeSeparators(filePath);
 
-    // Determine if this is potentially a simplificado document
-    // Must be a PDF and filename must contain 'simp' (case-insensitive)
+    // Determine if this file is potentially a pedimento (simplificado, completo, or CoveFact).
+    // PDFs whose filename contains 'simp', 'pedim' or 'covefact' (case-insensitive)
+    // are flagged so the identify stage prioritizes them. The column name
+    // `likely_simplificado` is preserved for backwards compatibility; semantics
+    // are broader (any likely pedimento PDF).
     const likelySimplificado =
-      fileExtension === 'pdf' && fileName.toLowerCase().includes('simp');
+      fileExtension === 'pdf' && /(simp|pedim|covefact)/i.test(fileName);
 
     return {
       fileName,