@arela/uploader 1.0.20 → 1.0.22

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@arela/uploader",
-  "version": "1.0.20",
+  "version": "1.0.22",
   "description": "CLI to upload files/directories to Arela",
   "bin": {
     "arela": "./src/index.js"
@@ -39,6 +39,7 @@
     "form-data": "4.0.4",
     "formdata-node": "^6.0.3",
     "globby": "14.1.0",
+    "googleapis": "^171.4.0",
     "ioredis": "^5.10.0",
     "mime-types": "3.0.1",
     "node-fetch": "3.3.2",

package/src/commands/DatastageCommand.js

@@ -0,0 +1,164 @@
+import fs from 'fs';
+import path from 'path';
+
+import { DatastageApiService } from '../services/DatastageApiService.js';
+import logger from '../services/LoggingService.js';
+
+import appConfig from '../config/config.js';
+import ErrorHandler from '../errors/ErrorHandler.js';
+
+/**
+ * Datastage Command Handler
+ * Uploads monthly Datastage *.zip files from a directory to the API.
+ * Sequential, idempotent via the cli `datastage_uploads` tracking table.
+ */
+export class DatastageCommand {
+  constructor() {
+    this.errorHandler = new ErrorHandler(logger);
+  }
+
+  /**
+   * @param {Object} options
+   * @param {string} options.dir - directory containing *.zip files (required)
+   * @param {string} [options.api] - 'default'|'agencia'|'cliente'
+   * @param {boolean} [options.retryFailed] - re-attempt files in 'failed' status
+   * @param {boolean} [options.showStats] - print final stats from API
+   */
+  async execute(options = {}) {
+    const startTime = Date.now();
+
+    if (!options.dir) {
+      throw new Error('--dir <path> is required');
+    }
+    const sourceDirectory = path.resolve(options.dir);
+    if (!fs.existsSync(sourceDirectory)) {
+      throw new Error(`Directory not found: ${sourceDirectory}`);
+    }
+    const dirStat = fs.statSync(sourceDirectory);
+    if (!dirStat.isDirectory()) {
+      throw new Error(`Not a directory: ${sourceDirectory}`);
+    }
+
+    const apiTarget = options.api || 'default';
+    const api = new DatastageApiService(apiTarget);
+
+    logger.info('📦 Starting arela datastage command');
+    logger.info(`🎯 API Target: ${apiTarget}`);
+    logger.info(`📂 Source: ${sourceDirectory}`);
+
+    // 1. Enumerate *.zip in root directory (non-recursive)
+    const entries = fs.readdirSync(sourceDirectory, { withFileTypes: true });
+    const zipFiles = entries
+      .filter((e) => e.isFile() && /\.zip$/i.test(e.name))
+      .map((e) => path.join(sourceDirectory, e.name));
+
+    if (zipFiles.length === 0) {
+      logger.warn('No *.zip files found in directory. Nothing to do.');
+      return { uploaded: 0, failed: 0, skipped: 0 };
+    }
+    logger.info(`🗂  Found ${zipFiles.length} zip file(s)`);
+
+    // 2. Register each file (idempotent upsert)
+    logger.info('📝 Registering files...');
+    for (const zipPath of zipFiles) {
+      const stats = fs.statSync(zipPath);
+      try {
+        await api.registerUpload({
+          absolutePath: zipPath,
+          fileName: path.basename(zipPath),
+          sizeBytes: stats.size,
+          fileModifiedAt: stats.mtime.toISOString(),
+          sourceDirectory,
+        });
+      } catch (err) {
+        logger.error(
+          `  ✗ register failed for ${path.basename(zipPath)}: ${err.message}`,
+        );
+        throw err;
+      }
+    }
+
+    // 3. Fetch pending list scoped to this directory
+    const pending = await api.getPending(sourceDirectory);
+    const pendingPaths = new Set(pending.map((p) => p.absolutePath));
+
+    const alreadyUploaded = zipFiles.length - pendingPaths.size;
+    if (alreadyUploaded > 0) {
+      logger.info(`⏭  Skipping ${alreadyUploaded} already uploaded file(s)`);
+    }
+
+    if (pending.length === 0) {
+      logger.success('✅ All files already uploaded. Nothing to do.');
+      if (options.showStats) {
+        const s = await api.getStats(sourceDirectory);
+        logger.info(`📊 Stats: ${JSON.stringify(s)}`);
+      }
+      return { uploaded: 0, failed: 0, skipped: alreadyUploaded };
+    }
+
+    // 4. Sequential upload loop
+    logger.info(`🚀 Uploading ${pending.length} file(s) sequentially...`);
+    let uploaded = 0;
+    let failed = 0;
+
+    for (let i = 0; i < pending.length; i++) {
+      const row = pending[i];
+      const localPath = row.absolutePath;
+      const label = `[${i + 1}/${pending.length}] ${row.fileName}`;
+
+      if (!fs.existsSync(localPath)) {
+        const err = `File missing on disk: ${localPath}`;
+        logger.error(`✗ ${label}: ${err}`);
+        try {
+          await api.markFailed(row.id, err);
+        } catch (e) {
+          logger.error(`  mark-failed error: ${e.message}`);
+        }
+        failed++;
+        continue;
+      }
+
+      try {
+        logger.info(`⬆  ${label}: uploading...`);
+        const result = await api.uploadZip(localPath);
+        const datastageId = result?.id || result?.data?.id;
+        const folio = result?.folio || result?.data?.folio;
+        if (!datastageId) {
+          throw new Error(
+            'API returned no datastage id in response: ' +
+              JSON.stringify(result).slice(0, 300),
+          );
+        }
+        await api.markUploaded(row.id, { datastageId, folio });
+        logger.success(
+          `✓ ${label}: folio=${folio || 'n/a'} datastageId=${datastageId}`,
+        );
+        uploaded++;
+      } catch (err) {
+        logger.error(`✗ ${label}: ${err.message}`);
+        try {
+          await api.markFailed(row.id, err.message);
+        } catch (e) {
+          logger.error(`  mark-failed error: ${e.message}`);
+        }
+        failed++;
+      }
+    }
+
+    const elapsed = ((Date.now() - startTime) / 1000).toFixed(1);
+    logger.info('—'.repeat(60));
+    logger.info(
+      `Done in ${elapsed}s — uploaded=${uploaded} failed=${failed} skipped=${alreadyUploaded}`,
+    );
+
+    if (options.showStats) {
+      const s = await api.getStats(sourceDirectory);
+      logger.info(`📊 Final stats: ${JSON.stringify(s)}`);
+    }
+
+    return { uploaded, failed, skipped: alreadyUploaded };
+  }
+}
+
+const datastageCommand = new DatastageCommand();
+export default datastageCommand;

package/src/commands/GDriveSyncCommand.js

@@ -0,0 +1,475 @@
+import cliProgress from 'cli-progress';
+import fs from 'fs';
+import fsp from 'fs/promises';
+import pLimit from 'p-limit';
+import path from 'path';
+
+import GoogleDriveService from '../services/GoogleDriveService.js';
+import logger from '../services/LoggingService.js';
+
+import appConfig from '../config/config.js';
+import ErrorHandler from '../errors/ErrorHandler.js';
+import { FileSanitizer } from '../utils/FileSanitizer.js';
+
+const STATE_FILENAME = '.gdrive-sync-state.json';
+const STATE_VERSION = 1;
+
+/**
+ * GDrive Sync Command
+ *
+ * Mirrors a Google Drive folder tree to a local directory so the existing
+ * scan → identify → propagate → push pipeline can run unchanged.
+ *
+ * Idempotent & incremental: maintains a `.gdrive-sync-state.json` at the
+ * mirror root with per-file md5/modifiedTime so re-runs only download
+ * changed/new files.
+ */
+export class GDriveSyncCommand {
+  constructor() {
+    this.errorHandler = new ErrorHandler(logger);
+    this.driveService = null;
+    this.sanitizer = new FileSanitizer();
+    this.onProgress = null;
+  }
+
+  /**
+   * Execute the gdrive-sync command
+   * @param {Object} [options]
+   * @param {string} [options.rootFolder] - override GDRIVE_ROOT_FOLDER_ID
+   * @param {string} [options.dest] - override local mirror path
+   * @param {boolean} [options.full] - ignore state file and re-verify everything
+   * @param {boolean} [options.dryRun] - list/plan only, no downloads or writes
+   * @param {Function} [options.onProgress]
+   */
+  async execute(options = {}) {
+    const startTime = Date.now();
+    this.onProgress = options.onProgress || null;
+
+    try {
+      // Allow CLI overrides before validation
+      if (options.rootFolder) {
+        process.env.GDRIVE_ROOT_FOLDER_ID = options.rootFolder;
+      }
+      if (options.dest) {
+        process.env.GDRIVE_LOCAL_MIRROR_PATH = options.dest;
+      }
+      // Reload config after env mutation
+      const cfg = appConfig.getGDriveConfig();
+      if (options.rootFolder) cfg.rootFolderId = options.rootFolder;
+      if (options.dest) cfg.localMirrorPath = options.dest;
+
+      appConfig.validateGDriveConfig();
+
+      const dryRun = !!options.dryRun;
+      const full = !!options.full;
+
+      this.driveService = new GoogleDriveService();
+
+      this.#say('☁️  Starting arela gdrive-sync command');
+      this.#say(`📁 Root folder ID: ${cfg.rootFolderId}`);
+      this.#say(`💾 Local mirror:   ${cfg.localMirrorPath}`);
+      this.#say(`⚙️  Concurrency:    ${cfg.concurrency}`);
+      this.#say(`📄 Skip native docs: ${cfg.skipNativeDocs}`);
+      if (dryRun) this.#say('🧪 DRY-RUN: no files will be written');
+      if (full) this.#say('♻️  FULL: ignoring state, re-verifying all files');
+
+      // Ensure mirror dir exists
+      if (!dryRun) {
+        await fsp.mkdir(cfg.localMirrorPath, { recursive: true });
+      }
+
+      // Load existing state
+      const statePath = path.join(cfg.localMirrorPath, STATE_FILENAME);
+      const state = full
+        ? this.#emptyState()
+        : await this.#loadState(statePath);
+
+      // Verify root folder exists & is accessible
+      this.#say('\n🔍 Verifying root folder access...');
+      const rootMeta = await this.driveService.getFile(cfg.rootFolderId);
+      if (!GoogleDriveService.isFolder(rootMeta)) {
+        throw new Error(
+          `GDRIVE_ROOT_FOLDER_ID does not point to a folder (mimeType=${rootMeta.mimeType})`,
+        );
+      }
+      logger.success(`  ✓ Root: "${rootMeta.name}"`);
+
+      // Walk Drive tree → produce file plan
+      this.#say('\n🌲 Walking Drive tree...');
+      this.#reportProgress(0, 'Walking Drive tree');
+      const plan = await this.#walkTree(cfg.rootFolderId, '', cfg);
+      this.#say(
+        `📋 Found ${plan.folders.length} folder(s) and ${plan.files.length} file(s)`,
+      );
+
+      // Stats
+      const stats = {
+        foldersCreated: 0,
+        filesAdded: 0,
+        filesUpdated: 0,
+        filesSkipped: 0,
+        filesFailed: 0,
+        nativeDocsSkipped: 0,
+        oversizedSkipped: 0,
+        bytesDownloaded: 0,
+      };
+
+      // Create folders first (cheap, sequential)
+      if (!dryRun) {
+        for (const folderRel of plan.folders) {
+          const abs = path.join(cfg.localMirrorPath, folderRel);
+          try {
+            await fsp.mkdir(abs, { recursive: true });
+            stats.foldersCreated += 1;
+          } catch (err) {
+            logger.warn(`⚠️  mkdir failed for ${abs}: ${err.message}`);
+          }
+        }
+      }
+
+      // Download files concurrently
+      this.#say('\n⬇️  Downloading files...');
+      const limit = pLimit(cfg.concurrency);
+      const progressBar = this.#createProgressBar(plan.files.length, dryRun);
+      let processed = 0;
+
+      const tasks = plan.files.map((file) =>
+        limit(async () => {
+          try {
+            const result = await this.#processFile(file, cfg, state, dryRun);
+            if (result === 'added') stats.filesAdded += 1;
+            else if (result === 'updated') stats.filesUpdated += 1;
+            else if (result === 'skipped-unchanged') stats.filesSkipped += 1;
+            else if (result === 'skipped-native') stats.nativeDocsSkipped += 1;
+            else if (result === 'skipped-oversize') stats.oversizedSkipped += 1;
+            if (
+              (result === 'added' || result === 'updated') &&
+              file.size != null
+            ) {
+              stats.bytesDownloaded += Number(file.size) || 0;
+            }
+          } catch (err) {
+            stats.filesFailed += 1;
+            logger.error(
+              `❌ Failed: ${file.relPath} (${file.id}) — ${err.message}`,
+            );
+          } finally {
+            processed += 1;
+            progressBar.update(processed);
+            if (plan.files.length > 0) {
+              this.#reportProgress(
+                Math.round((processed / plan.files.length) * 100),
+                `Synced ${processed}/${plan.files.length}`,
+              );
+            }
+          }
+        }),
+      );
+
+      await Promise.all(tasks);
+      progressBar.stop();
+
+      // Persist state
+      if (!dryRun) {
+        await this.#saveState(statePath, state);
+      }
+
+      const duration = ((Date.now() - startTime) / 1000).toFixed(2);
+      this.#reportProgress(100, `Sync completed in ${duration}s`);
+
+      logger.success('\n✅ gdrive-sync completed successfully!');
+      this.#say('\n📊 Sync Statistics:');
+      this.#say(`   Folders created:      ${stats.foldersCreated}`);
+      this.#say(`   Files added:          ${stats.filesAdded}`);
+      this.#say(`   Files updated:        ${stats.filesUpdated}`);
+      this.#say(`   Files unchanged:      ${stats.filesSkipped}`);
+      this.#say(`   Native docs skipped:  ${stats.nativeDocsSkipped}`);
+      this.#say(`   Oversized skipped:    ${stats.oversizedSkipped}`);
+      this.#say(`   Files failed:         ${stats.filesFailed}`);
+      this.#say(
+        `   Bytes downloaded:     ${this.#formatBytes(stats.bytesDownloaded)}`,
+      );
+      this.#say(`   Duration:             ${duration}s`);
+      this.#say(
+        `\n💡 Next: run "arela scan" with UPLOAD_BASE_PATH=${cfg.localMirrorPath}`,
+      );
+
+      return { success: true, stats };
+    } catch (error) {
+      this.errorHandler.handleError(error, 'gdrive-sync');
+      return { success: false, error: error.message };
+    }
+  }
+
+  /**
+   * Walk Drive tree starting at folderId.
+   * Returns flat lists of folders (relative paths) and files (with metadata + relPath).
+   * @private
+   */
+  async #walkTree(folderId, relPath, cfg) {
+    const folders = [];
+    const files = [];
+
+    // BFS using a queue
+    const queue = [{ id: folderId, relPath }];
+    // Track sanitized names per folder to dedupe collisions
+    const folderNameMaps = new Map();
+
+    while (queue.length > 0) {
+      const { id, relPath: parentRel } = queue.shift();
+      const usedNames = folderNameMaps.get(parentRel) || new Set();
+
+      for await (const child of this.driveService.listChildren(id)) {
+        const safeName = this.#uniqueSanitizedName(
+          child.name,
+          child.id,
+          usedNames,
+        );
+        const childRel = parentRel ? `${parentRel}/${safeName}` : safeName;
+
+        if (GoogleDriveService.isFolder(child)) {
+          folders.push(childRel);
+          folderNameMaps.set(childRel, new Set());
+          queue.push({ id: child.id, relPath: childRel });
+          continue;
+        }
+
+        // Resolve shortcuts to their targets if enabled
+        let effective = child;
+        if (GoogleDriveService.isShortcut(child)) {
+          if (!cfg.followShortcuts || !child.shortcutDetails?.targetId) {
+            logger.debug(`⏭️  Skipping shortcut: ${childRel}`);
+            continue;
+          }
+          try {
+            effective = await this.driveService.getFile(
+              child.shortcutDetails.targetId,
+            );
+          } catch (err) {
+            logger.warn(
+              `⚠️  Could not resolve shortcut "${childRel}": ${err.message}`,
+            );
+            continue;
+          }
+          if (GoogleDriveService.isFolder(effective)) {
+            folders.push(childRel);
+            folderNameMaps.set(childRel, new Set());
+            queue.push({ id: effective.id, relPath: childRel });
+            continue;
+          }
+        }
+
+        files.push({
+          id: effective.id,
+          name: effective.name,
+          mimeType: effective.mimeType,
+          modifiedTime: effective.modifiedTime,
+          size: effective.size != null ? Number(effective.size) : null,
+          md5Checksum: effective.md5Checksum || null,
+          relPath: childRel,
+          isNativeDoc: GoogleDriveService.isNativeGoogleDoc(effective),
+        });
+      }
+
+      folderNameMaps.set(parentRel, usedNames);
+    }
+
+    return { folders, files };
+  }
+
+  /**
+   * Process a single file: skip / download / update.
+   * @private
+   */
+  async #processFile(file, cfg, state, dryRun) {
+    // Native Google Docs (Docs/Sheets/Slides/...): skip by default
+    if (file.isNativeDoc && cfg.skipNativeDocs) {
+      logger.debug(`⏭️  Native Google Doc skipped: ${file.relPath}`);
+      return 'skipped-native';
+    }
+
+    // Oversize guard
+    if (
+      cfg.maxFileSizeBytes &&
+      file.size != null &&
+      file.size > cfg.maxFileSizeBytes
+    ) {
+      logger.warn(
+        `⚠️  File too large (${this.#formatBytes(file.size)}): ${file.relPath}`,
+      );
+      return 'skipped-oversize';
+    }
+
+    const absPath = path.join(cfg.localMirrorPath, file.relPath);
+    const prev = state.files[file.id];
+
+    // Decide if we can skip (unchanged)
+    const localExists = fs.existsSync(absPath);
+    const unchanged =
+      localExists &&
+      prev &&
+      prev.relPath === file.relPath &&
+      ((file.md5Checksum && prev.md5Checksum === file.md5Checksum) ||
+        (!file.md5Checksum && prev.modifiedTime === file.modifiedTime));
+
+    if (unchanged) {
+      return 'skipped-unchanged';
+    }
+
+    if (dryRun) {
+      logger.info(
+        `   [dry-run] would ${prev ? 'update' : 'add'}: ${file.relPath}`,
+      );
+      return prev ? 'updated' : 'added';
+    }
+
+    // Ensure parent dir
+    await fsp.mkdir(path.dirname(absPath), { recursive: true });
+
+    // Download to .part then atomic rename
+    const partPath = `${absPath}.part`;
+    try {
+      await fsp.rm(partPath, { force: true });
+    } catch {
+      /* ignore */
+    }
+
+    const out = fs.createWriteStream(partPath);
+    await this.driveService.downloadFile(file.id, out);
+    await fsp.rename(partPath, absPath);
+
+    // Set mtime to Drive's modifiedTime so `arela scan` captures changes correctly
+    if (file.modifiedTime) {
+      const mtime = new Date(file.modifiedTime);
+      try {
+        await fsp.utimes(absPath, mtime, mtime);
+      } catch (err) {
+        logger.debug(`⚠️  utimes failed for ${absPath}: ${err.message}`);
+      }
+    }
+
+    // Update state
+    state.files[file.id] = {
+      relPath: file.relPath,
+      md5Checksum: file.md5Checksum,
+      modifiedTime: file.modifiedTime,
+      size: file.size,
+      syncedAt: new Date().toISOString(),
+    };
+
+    return prev ? 'updated' : 'added';
+  }
+
+  /**
+   * Sanitize file/folder name; on duplicate, append " (gdrive-<id>)".
+   * @private
+   */
+  #uniqueSanitizedName(originalName, fileId, usedNames) {
+    let safe = this.sanitizer.sanitizeFileName(originalName);
+    if (!safe) safe = `file-${fileId}`;
+
+    if (usedNames.has(safe)) {
+      const ext = path.extname(safe);
+      const base = path.basename(safe, ext);
+      safe = `${base} (gdrive-${fileId})${ext}`;
+    }
+    usedNames.add(safe);
+    return safe;
+  }
+
+  /**
+   * @private
+   */
+  async #loadState(statePath) {
+    try {
+      const raw = await fsp.readFile(statePath, 'utf-8');
+      const parsed = JSON.parse(raw);
+      if (parsed.version !== STATE_VERSION) {
+        logger.warn(
+          `⚠️  State file version mismatch (${parsed.version} != ${STATE_VERSION}); starting fresh`,
+        );
+        return this.#emptyState();
+      }
+      return parsed;
+    } catch (err) {
+      if (err.code !== 'ENOENT') {
+        logger.warn(`⚠️  Could not read state file: ${err.message}`);
+      }
+      return this.#emptyState();
+    }
+  }
+
+  /**
+   * @private
+   */
+  async #saveState(statePath, state) {
+    state.savedAt = new Date().toISOString();
+    const tmp = `${statePath}.tmp`;
+    await fsp.writeFile(tmp, JSON.stringify(state, null, 2), 'utf-8');
+    await fsp.rename(tmp, statePath);
+  }
+
+  /**
+   * @private
+   */
+  #emptyState() {
+    return { version: STATE_VERSION, savedAt: null, files: {} };
+  }
+
+  /**
+   * @private
+   */
+  #createProgressBar(total, dryRun) {
+    const label = dryRun ? '🧪 Planning' : '⬇️  Downloading';
+    const bar = new cliProgress.SingleBar(
+      {
+        format: `${label} |{bar}| {percentage}% | {value}/{total} files`,
+        barCompleteChar: '\u2588',
+        barIncompleteChar: '\u2591',
+        hideCursor: true,
+      },
+      cliProgress.Presets.shades_classic,
+    );
+    bar.start(Math.max(total, 1), 0);
+    return bar;
+  }
+
+  /**
+   * @private
+   */
+  #reportProgress(percent, message) {
+    if (typeof this.onProgress === 'function') {
+      try {
+        this.onProgress(percent, message);
+      } catch {
+        /* ignore listener errors */
+      }
+    }
+  }
+
+  /**
+   * Print to stdout AND log to file (mirrors ScanCommand's user-facing output style).
+   * @private
+   */
+  #say(message) {
+    console.log(message);
+    logger.info(message);
+  }
+
+  /**
+   * @private
+   */
+  #formatBytes(bytes) {
+    if (!bytes) return '0 B';
+    const units = ['B', 'KB', 'MB', 'GB', 'TB'];
+    let i = 0;
+    let n = bytes;
+    while (n >= 1024 && i < units.length - 1) {
+      n /= 1024;
+      i += 1;
+    }
+    return `${n.toFixed(2)} ${units[i]}`;
+  }
+}
+
+export default new GDriveSyncCommand();