@nightowne/tas-cli 2.0.0 → 2.3.0

package/src/sync/sync.js

@@ -13,6 +13,10 @@ import { processFile } from '../index.js';
 // Debounce time in ms to batch rapid file changes
 const DEBOUNCE_MS = 1000;
 
+// Exponential backoff limits for upload errors
+const INITIAL_BACKOFF_MS = 2000;
+const MAX_BACKOFF_MS = 60000;
+
 // Ignore patterns
 const IGNORE_PATTERNS = [
     /^\./, // Hidden files
@@ -29,7 +33,9 @@ export class SyncEngine extends EventEmitter {
         this.dataDir = options.dataDir;
         this.password = options.password;
         this.config = options.config;
+        this.limitRate = options.limitRate || null;
         this.watchers = new Map(); // path -> FSWatcher
+        this.rootWatchers = new Map(); // rootPath -> Set of watched subpaths (for Linux recursive watch fallback)
         this.pendingChanges = new Map(); // path -> timeout
         this.db = null;
         this.running = false;
@@ -98,53 +104,76 @@ export class SyncEngine extends EventEmitter {
         let uploaded = 0;
         let skipped = 0;
 
-        for (const file of files) {
-            const existing = stateMap.get(file.relativePath);
+        // Process files with concurrency limit
+        const CONCURRENCY = 4;
+        const queue = [...files];
+        const promises = [];
 
-            // Check if file has changed (by mtime)
-            if (existing && existing.mtime >= file.mtime) {
-                skipped++;
-                continue;
-            }
+        const worker = async () => {
+            let consecutiveErrors = 0;
 
-            // Calculate hash to detect actual changes
-            const hash = await hashFile(file.path);
+            while (queue.length > 0) {
+                const file = queue.shift();
+                const existing = stateMap.get(file.relativePath);
 
-            if (existing && existing.file_hash === hash) {
-                // File unchanged, just update mtime
-                this.db.updateSyncState(folder.id, file.relativePath, hash, file.mtime);
-                skipped++;
-                continue;
-            }
+                // Check if file has changed (by mtime)
+                if (existing && existing.mtime >= file.mtime) {
+                    skipped++;
+                    continue;
+                }
 
-            // File is new or changed - upload it
-            try {
-                this.emit('file-upload-start', { file: file.relativePath });
+                // Calculate hash to detect actual changes
+                const hash = await hashFile(file.path);
 
-                await processFile(file.path, {
-                    password: this.password,
-                    dataDir: this.dataDir,
-                    customName: file.relativePath, // Use relative path as name
-                    config: this.config,
-                    onProgress: (msg) => this.emit('progress', { file: file.relativePath, message: msg })
-                });
+                if (existing && existing.file_hash === hash) {
+                    // File unchanged, just update mtime
+                    this.db.updateSyncState(folder.id, file.relativePath, hash, file.mtime);
+                    skipped++;
+                    continue;
+                }
 
-                // Update sync state
-                this.db.updateSyncState(folder.id, file.relativePath, hash, file.mtime);
-                uploaded++;
+                // File is new or changed - upload it
+                try {
+                    this.emit('file-upload-start', { file: file.relativePath });
 
-                this.emit('file-upload-complete', { file: file.relativePath });
-            } catch (err) {
-                // File might already exist, skip
-                if (err.message.includes('duplicate')) {
+                    await processFile(file.path, {
+                        password: this.password,
+                        dataDir: this.dataDir,
+                        customName: file.relativePath, // Use relative path as name
+                        config: this.config,
+                        limitRate: this.limitRate ? Math.floor(this.limitRate / CONCURRENCY) : null,
+                        onProgress: (msg) => this.emit('progress', { file: file.relativePath, message: msg })
+                    });
+
+                    // Update sync state
                     this.db.updateSyncState(folder.id, file.relativePath, hash, file.mtime);
-                    skipped++;
-                } else {
-                    this.emit('file-upload-error', { file: file.relativePath, error: err.message });
+                    uploaded++;
+                    consecutiveErrors = 0; // Reset on success
+
+                    this.emit('file-upload-complete', { file: file.relativePath });
+                } catch (err) {
+                    // File might already exist, skip
+                    if (err.message.includes('duplicate')) {
+                        this.db.updateSyncState(folder.id, file.relativePath, hash, file.mtime);
+                        skipped++;
+                        consecutiveErrors = 0;
+                    } else {
+                        consecutiveErrors++;
+                        // Exponential backoff: 2s → 4s → 8s → 16s → ... → 60s max
+                        const delay = Math.min(INITIAL_BACKOFF_MS * Math.pow(2, consecutiveErrors - 1), MAX_BACKOFF_MS);
+                        await new Promise(r => setTimeout(r, delay));
+                        this.emit('file-upload-error', { file: file.relativePath, error: err.message });
+                    }
                 }
             }
+        };
+
+        for (let i = 0; i < CONCURRENCY; i++) {
+            promises.push(worker());
         }
 
+        await Promise.all(promises);
+
         this.emit('sync-complete', { folder: folderPath, uploaded, skipped });
 
         return { uploaded, skipped };
@@ -206,6 +235,7 @@ export class SyncEngine extends EventEmitter {
                 dataDir: this.dataDir,
                 customName: filename,
                 config: this.config,
+                limitRate: this.limitRate,
                 onProgress: (msg) => this.emit('progress', { file: filename, message: msg })
             });
 
@@ -219,37 +249,148 @@ export class SyncEngine extends EventEmitter {
         }
     }
 
+    /**
+     * Recursively list all subdirectories of a directory
+     */
+    getSubdirectories(dirPath) {
+        const subdirs = [];
+        try {
+            const entries = fs.readdirSync(dirPath, { withFileTypes: true });
+            for (const entry of entries) {
+                if (entry.isDirectory()) {
+                    if (this.shouldIgnore(entry.name)) continue;
+                    const fullPath = path.join(dirPath, entry.name);
+                    subdirs.push(fullPath);
+                    subdirs.push(...this.getSubdirectories(fullPath));
+                }
+            }
+        } catch (err) {
+            // Ignore folder reading errors
+        }
+        return subdirs;
+    }
+
+    /**
+     * Watch a single directory (non-recursively) and dynamically watch new folders
+     */
+    watchSingleDir(rootPath, dir) {
+        if (this.watchers.has(dir)) return;
+
+        try {
+            const watcher = fs.watch(dir, { recursive: false }, (event, filename) => {
+                if (!filename) return;
+
+                const fullPath = path.join(dir, filename);
+                const relativePath = path.relative(rootPath, fullPath);
+
+                // If a new directory is created, watch it recursively
+                try {
+                    if (fs.existsSync(fullPath) && fs.statSync(fullPath).isDirectory()) {
+                        if (!this.shouldIgnore(filename)) {
+                            this.watchDirRecursively(rootPath, fullPath);
+                        }
+                    }
+                } catch (e) {
+                    // Ignore stats errors
+                }
+
+                this.handleFileChange(rootPath, relativePath);
+            });
+
+            watcher.on('error', (err) => {
+                this.emit('watch-error', { folder: rootPath, error: err.message });
+            });
+
+            this.watchers.set(dir, watcher);
+
+            if (!this.rootWatchers.has(rootPath)) {
+                this.rootWatchers.set(rootPath, new Set());
+            }
+            this.rootWatchers.get(rootPath).add(dir);
+        } catch (err) {
+            this.emit('watch-error', { folder: rootPath, error: err.message });
+        }
+    }
+
+    /**
+     * Recursively watch a newly created directory and its subdirectories
+     */
+    watchDirRecursively(rootPath, dirPath) {
+        this.watchSingleDir(rootPath, dirPath);
+        const subdirs = this.getSubdirectories(dirPath);
+        for (const subdir of subdirs) {
+            this.watchSingleDir(rootPath, subdir);
+        }
+    }
+
     /**
      * Start watching a folder
      */
     watchFolder(folderPath) {
-        if (this.watchers.has(folderPath)) {
-            return; // Already watching
-        }
+        const isRecursiveSupported = process.platform === 'darwin' || process.platform === 'win32';
+
+        if (isRecursiveSupported) {
+            if (this.watchers.has(folderPath)) {
+                return; // Already watching
+            }
+
+            const watcher = fs.watch(folderPath, { recursive: true }, (event, filename) => {
+                if (filename) {
+                    this.handleFileChange(folderPath, filename);
+                }
+            });
 
-        const watcher = fs.watch(folderPath, { recursive: true }, (event, filename) => {
-            if (filename) {
-                this.handleFileChange(folderPath, filename);
+            watcher.on('error', (err) => {
+                this.emit('watch-error', { folder: folderPath, error: err.message });
+            });
+
+            this.watchers.set(folderPath, watcher);
+            this.emit('watch-start', { folder: folderPath });
+        } else {
+            // Manual recursive watch for Linux/other platforms
+            if (this.rootWatchers.has(folderPath)) {
+                return; // Already watching
             }
-        });
 
-        watcher.on('error', (err) => {
-            this.emit('watch-error', { folder: folderPath, error: err.message });
-        });
+            // Watch root
+            this.watchSingleDir(folderPath, folderPath);
+
+            // Watch all existing subdirectories
+            const subdirs = this.getSubdirectories(folderPath);
+            for (const subdir of subdirs) {
+                this.watchSingleDir(folderPath, subdir);
+            }
 
-        this.watchers.set(folderPath, watcher);
-        this.emit('watch-start', { folder: folderPath });
+            this.emit('watch-start', { folder: folderPath });
+        }
     }
 
     /**
      * Stop watching a folder
      */
     unwatchFolder(folderPath) {
-        const watcher = this.watchers.get(folderPath);
-        if (watcher) {
-            watcher.close();
-            this.watchers.delete(folderPath);
-            this.emit('watch-stop', { folder: folderPath });
+        const isRecursiveSupported = process.platform === 'darwin' || process.platform === 'win32';
+
+        if (isRecursiveSupported) {
+            const watcher = this.watchers.get(folderPath);
+            if (watcher) {
+                watcher.close();
+                this.watchers.delete(folderPath);
+                this.emit('watch-stop', { folder: folderPath });
+            }
+        } else {
+            if (this.rootWatchers.has(folderPath)) {
+                const dirs = this.rootWatchers.get(folderPath);
+                for (const dir of dirs) {
+                    const watcher = this.watchers.get(dir);
+                    if (watcher) {
+                        watcher.close();
+                        this.watchers.delete(dir);
+                    }
+                }
+                this.rootWatchers.delete(folderPath);
+                this.emit('watch-stop', { folder: folderPath });
+            }
         }
     }
 
@@ -283,11 +424,25 @@ export class SyncEngine extends EventEmitter {
         this.pendingChanges.clear();
 
         // Close all watchers
-        for (const [folderPath, watcher] of this.watchers) {
-            watcher.close();
-            this.emit('watch-stop', { folder: folderPath });
+        const isRecursiveSupported = process.platform === 'darwin' || process.platform === 'win32';
+        if (isRecursiveSupported) {
+            for (const [folderPath, watcher] of this.watchers) {
+                watcher.close();
+                this.emit('watch-stop', { folder: folderPath });
+            }
+            this.watchers.clear();
+        } else {
+            for (const rootPath of this.rootWatchers.keys()) {
+                this.unwatchFolder(rootPath);
+            }
+            this.rootWatchers.clear();
+            
+            // Just in case, close any stray watchers
+            for (const watcher of this.watchers.values()) {
+                try { watcher.close(); } catch (e) {}
+            }
+            this.watchers.clear();
         }
-        this.watchers.clear();
 
         if (this.db) {
             this.db.close();

package/src/telegram/client.js

@@ -1,25 +1,66 @@
 /**
  * Telegram Bot client wrapper
  * Uses official Telegram Bot API - 2GB file limit, FREE, no ban risk!
+ * Includes exponential backoff retry and rate limiting for production reliability.
  */
 
 import TelegramBot from 'node-telegram-bot-api';
 import fs from 'fs';
 import path from 'path';
+import { pipeline } from 'stream/promises';
+
+const MAX_RETRIES = 5;
+const BASE_DELAY_MS = 1000;
+const MAX_DELAY_MS = 60000;
+
+/**
+ * Retry an async function with exponential backoff + jitter.
+ * Handles Telegram 429 (rate limit) errors by respecting retry_after.
+ */
+async function withRetry(fn, label = 'operation', retries = MAX_RETRIES) {
+    for (let attempt = 0; attempt <= retries; attempt++) {
+        try {
+            return await fn();
+        } catch (err) {
+            const isRateLimit = err?.response?.statusCode === 429 || err?.message?.includes('429');
+            const isTransient = err?.code === 'ETIMEOUT' || err?.code === 'ECONNRESET' ||
+                err?.code === 'ENOTFOUND' || err?.message?.includes('ETIMEDOUT');
+
+            if (attempt >= retries) throw err;
+            if (!isRateLimit && !isTransient) throw err;
+
+            let delay;
+            if (isRateLimit && err?.response?.body?.parameters?.retry_after) {
+                delay = err.response.body.parameters.retry_after * 1000 + 500;
+            } else {
+                delay = Math.min(BASE_DELAY_MS * Math.pow(2, attempt) + Math.random() * 1000, MAX_DELAY_MS);
+            }
+
+            const sec = (delay / 1000).toFixed(1);
+            console.log(`⏳ ${label} failed (attempt ${attempt + 1}/${retries + 1}), retrying in ${sec}s...`);
+            await new Promise(r => setTimeout(r, delay));
+        }
+    }
+}
 
 export class TelegramClient {
     constructor(dataDir) {
         this.dataDir = dataDir;
         this.bot = null;
         this.chatId = null;
+        this._lastSendTime = 0;
     }
 
     /**
      * Initialize with bot token
      * Get token from @BotFather on Telegram
      */
-    async initialize(token) {
-        this.bot = new TelegramBot(token, { polling: false });
+    async initialize(token, customApiUrl = null) {
+        const options = { polling: false };
+        if (customApiUrl) {
+            options.baseApiUrl = customApiUrl;
+        }
+        this.bot = new TelegramBot(token, options);
 
         // Verify the token works
         try {
@@ -71,49 +112,84 @@ export class TelegramClient {
         });
     }
 
+    /**
+     * Rate-limit: ensure at least 1s between sends to same chat (Telegram limit)
+     */
+    async _rateLimit() {
+        const now = Date.now();
+        const elapsed = now - this._lastSendTime;
+        if (elapsed < 1000) {
+            await new Promise(r => setTimeout(r, 1000 - elapsed));
+        }
+        this._lastSendTime = Date.now();
+    }
+
     /**
      * Send a file to the storage chat
      * Telegram supports up to 2GB for documents!
+     * Includes automatic retry with exponential backoff.
      */
-    async sendFile(filePath, caption = '') {
+    async sendFile(filePath, caption = '', options = {}) {
         if (!this.chatId) {
             throw new Error('Chat ID not set. Run init first.');
         }
 
-        const fileStream = fs.createReadStream(filePath);
+        if (!fs.existsSync(filePath)) {
+            throw new Error(`File not found: ${filePath}`);
+        }
+
         const filename = path.basename(filePath);
 
-        const message = await this.bot.sendDocument(this.chatId, fileStream, {
-            caption: caption
-        }, {
-            filename: filename,
-            contentType: 'application/octet-stream'
-        });
+        return withRetry(async () => {
+            await this._rateLimit();
+
+            let fileStream = fs.createReadStream(filePath);
+
+            if (options.limitRate) {
+                const { Throttle } = await import('../utils/throttle.js');
+                fileStream = fileStream.pipe(new Throttle(options.limitRate));
+            }
+
+            const message = await this.bot.sendDocument(this.chatId, fileStream, {
+                caption: caption
+            }, {
+                filename: filename,
+                contentType: 'application/octet-stream'
+            });
 
-        return {
-            messageId: message.message_id,
-            fileId: message.document.file_id,
-            timestamp: message.date
-        };
+            return {
+                messageId: message.message_id,
+                fileId: message.document.file_id,
+                timestamp: message.date
+            };
+        }, `Upload ${filename}`);
     }
 
     /**
-     * Download a file from Telegram
+     * Download a file from Telegram (In-Memory buffer)
+     * Includes automatic retry with exponential backoff.
      */
     async downloadFile(fileId) {
-        // Get file path from Telegram servers
-        const file = await this.bot.getFile(fileId);
+        return withRetry(async () => {
+            const fileStream = await this.bot.getFileStream(fileId);
 
-        // Download the file
-        const fileStream = await this.bot.getFileStream(fileId);
+            const chunks = [];
+            for await (const chunk of fileStream) {
+                chunks.push(chunk);
+            }
 
-        // Collect chunks into buffer
-        const chunks = [];
-        for await (const chunk of fileStream) {
-            chunks.push(chunk);
-        }
+            return Buffer.concat(chunks);
+        }, `Download ${fileId.substring(0, 12)}...`);
+    }
 
-        return Buffer.concat(chunks);
+    /**
+     * Efficiently download a file from Telegram straight to disk
+     */
+    async downloadFileToPath(fileId, destPath) {
+        const fileStream = await this.bot.getFileStream(fileId);
+        const writeStream = fs.createWriteStream(destPath);
+        await pipeline(fileStream, writeStream);
+        return destPath;
     }
 
     /**

package/src/utils/branding.js

@@ -3,6 +3,13 @@
  */
 
 import chalk from 'chalk';
+import { readFileSync } from 'fs';
+import { fileURLToPath } from 'url';
+import { dirname, join } from 'path';
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = dirname(__filename);
+const pkg = JSON.parse(readFileSync(join(__dirname, '..', '..', 'package.json'), 'utf-8'));
 
 export const LOGO = `
 ████████╗ █████╗ ███████╗
@@ -14,7 +21,7 @@ export const LOGO = `
 `;
 
 export const TAGLINE = 'Telegram as Storage';
-export const VERSION = '1.0.0';
+export const VERSION = pkg.version;
 
 /**
  * Print the TAS banner
@@ -57,10 +64,10 @@ export function warn(msg) {
  * Format file size
  */
 export function formatSize(bytes) {
-    if (bytes === 0) return '0 B';
+    if (!Number.isFinite(bytes) || bytes <= 0) return '0 B';
     const k = 1024;
     const sizes = ['B', 'KB', 'MB', 'GB', 'TB'];
-    const i = Math.floor(Math.log(bytes) / Math.log(k));
+    const i = Math.min(Math.floor(Math.log(bytes) / Math.log(k)), sizes.length - 1);
     return parseFloat((bytes / Math.pow(k, i)).toFixed(2)) + ' ' + sizes[i];
 }
 

package/src/utils/chunker.js

@@ -1,9 +1,11 @@
 /**
  * File chunking utilities for large files
- * WhatsApp document limit is ~2GB, we use 1.9GB chunks to be safe
+ * Telegram Bot API limit is 50 MB for bot uploads (sendDocument).
+ * We use 49 MB to leave room for the 64-byte WAS1 header.
+ * See: https://core.telegram.org/bots/api#senddocument
  */
 
-const MAX_CHUNK_SIZE = 1.9 * 1024 * 1024 * 1024; // 1.9 GB
+const MAX_CHUNK_SIZE = 49 * 1024 * 1024; // 49 MB — Telegram Bot API safe limit
 
 export class Chunker {
     /**
@@ -91,7 +93,17 @@ export function createHeader(filename, originalSize, chunkIndex, totalChunks, fl
     offset += 2;
 
     // Filename length
-    const filenameBytes = Buffer.from(filename, 'utf-8').subarray(0, 42);
+    let filenameBytes = Buffer.from(filename, 'utf-8');
+    if (filenameBytes.length > 42) {
+        // We carefully truncate by chars instead of bytes to avoid splitting a UTF-8 character in half!
+        let truncated = filename;
+        while (Buffer.from(truncated, 'utf-8').length > 42) {
+            // Remove one character at a time from the end
+            truncated = truncated.slice(0, -1);
+        }
+        filenameBytes = Buffer.from(truncated, 'utf-8');
+    }
+
     header.writeUInt16LE(filenameBytes.length, offset);
     offset += 2;
 

package/src/utils/cli-helpers.js

@@ -39,14 +39,13 @@ export async function getPassword(passwordOption, allowCache = true) {
 }
 
 /**
- * Verify password against config
+ * Verify password against config (supports both legacy and new hash formats)
  * @param {string} password - Password to verify
  * @param {Object} config - Config object with passwordHash
  * @returns {boolean}
  */
 export function verifyPassword(password, config) {
-    const encryptor = new Encryptor(password);
-    return encryptor.getPasswordHash() === config.passwordHash;
+    return Encryptor.verifyPasswordHash(password, config.passwordHash);
 }
 
 /**
@@ -62,11 +61,13 @@ export function validateConfig(config) {
         return { valid: false, errors };
     }
 
-    if (!config.botToken || typeof config.botToken !== 'string') {
-        errors.push('Missing or invalid botToken');
+    // v2: encrypted token, v1: plaintext token
+    const hasToken = config.encryptedBotToken || config.botToken;
+    if (!hasToken) {
+        errors.push('Missing bot token (botToken or encryptedBotToken)');
     }
 
-    if (!config.botToken?.includes(':')) {
+    if (config.botToken && !config.botToken.includes(':')) {
         errors.push('Invalid bot token format (should contain :)');
     }
 
@@ -84,6 +85,29 @@ export function validateConfig(config) {
     };
 }
 
+/**
+ * Decrypt bot token from config using password
+ * Supports both v1 (plaintext) and v2 (encrypted) configs
+ * @param {Object} config - Config object
+ * @param {string} password - User's password
+ * @returns {string} - Decrypted bot token
+ */
+export function decryptBotToken(config, password) {
+    // v1: plaintext token (backward compatibility)
+    if (config.botToken) {
+        return config.botToken;
+    }
+
+    // v2: encrypted token
+    if (config.encryptedBotToken) {
+        const encryptor = new Encryptor(password);
+        const encryptedBuffer = Buffer.from(config.encryptedBotToken, 'base64');
+        return encryptor.decrypt(encryptedBuffer).toString('utf-8');
+    }
+
+    throw new Error('No bot token found in config');
+}
+
 /**
  * Load and validate config
  * @param {string} dataDir - Data directory path
@@ -143,3 +167,17 @@ export async function getAndVerifyPassword(passwordOption, dataDir) {
 
     return password;
 }
+
+/**
+ * Resolve config by decrypting bot token if needed.
+ * Returns a config object with a guaranteed plaintext `botToken` field.
+ * @param {Object} config - Raw config from disk
+ * @param {string} password - Verified password
+ * @returns {Object} - Config with decrypted botToken
+ */
+export function resolveConfig(config, password) {
+    return {
+        ...config,
+        botToken: decryptBotToken(config, password)
+    };
+}