@nightowne/tas-cli 1.0.0

package/src/index.js

@@ -0,0 +1,219 @@
+/**
+ * Main processing module
+ * Orchestrates the upload/download pipeline
+ */
+
+import fs from 'fs';
+import path from 'path';
+import { Encryptor, hashFile } from './crypto/encryption.js';
+import { Compressor } from './utils/compression.js';
+import { Chunker, createHeader, parseHeader, HEADER_SIZE } from './utils/chunker.js';
+import { TelegramClient } from './telegram/client.js';
+import { FileIndex } from './db/index.js';
+
+// Telegram has 50MB limit for bots, 2GB for user uploads
+// We'll use 49MB chunks to be safe
+const TELEGRAM_CHUNK_SIZE = 49 * 1024 * 1024;
+
+/**
+ * Process and upload a file to Telegram
+ */
+export async function processFile(filePath, options) {
+    const { password, dataDir, customName, config, onProgress } = options;
+
+    onProgress?.('Reading file...');
+
+    // Read file
+    const filename = customName || path.basename(filePath);
+    const fileData = fs.readFileSync(filePath);
+    const originalSize = fileData.length;
+
+    // Calculate hash
+    onProgress?.('Calculating hash...');
+    const hash = await hashFile(filePath);
+
+    // Check if already uploaded
+    const db = new FileIndex(path.join(dataDir, 'index.db'));
+    db.init();
+
+    if (db.exists(hash)) {
+        db.close();
+        throw new Error('File already uploaded (duplicate hash)');
+    }
+
+    // Compress
+    onProgress?.('Compressing...');
+    const compressor = new Compressor();
+    const { data: compressedData, compressed } = await compressor.compress(fileData, filename);
+
+    // Encrypt
+    onProgress?.('Encrypting...');
+    const encryptor = new Encryptor(password);
+    const encryptedData = encryptor.encrypt(compressedData);
+
+    // Chunk if needed (Telegram bot limit ~50MB per file)
+    onProgress?.('Preparing chunks...');
+    const chunks = [];
+    const numChunks = Math.ceil(encryptedData.length / TELEGRAM_CHUNK_SIZE);
+
+    for (let i = 0; i < numChunks; i++) {
+        const start = i * TELEGRAM_CHUNK_SIZE;
+        const end = Math.min(start + TELEGRAM_CHUNK_SIZE, encryptedData.length);
+        chunks.push({
+            index: i,
+            total: numChunks,
+            data: encryptedData.subarray(start, end)
+        });
+    }
+
+    // Prepare files with headers
+    const tempDir = path.join(dataDir, 'tmp');
+    if (!fs.existsSync(tempDir)) {
+        fs.mkdirSync(tempDir, { recursive: true });
+    }
+
+    const chunkFiles = [];
+    const flags = compressed ? 1 : 0;
+
+    for (const chunk of chunks) {
+        const header = createHeader(filename, originalSize, chunk.index, chunk.total, flags);
+        const chunkData = Buffer.concat([header, chunk.data]);
+
+        const chunkFilename = chunks.length > 1
+            ? `${hash.substring(0, 12)}.part${chunk.index}.tas`
+            : `${hash.substring(0, 12)}.tas`;
+
+        const chunkPath = path.join(tempDir, chunkFilename);
+        fs.writeFileSync(chunkPath, chunkData);
+
+        chunkFiles.push({
+            index: chunk.index,
+            path: chunkPath,
+            size: chunkData.length
+        });
+    }
+
+    // Connect to Telegram
+    onProgress?.('Connecting to Telegram...');
+    const client = new TelegramClient(dataDir);
+    await client.initialize(config.botToken);
+    client.setChatId(config.chatId);
+
+    // Upload chunks
+    const fileId = db.addFile({
+        filename,
+        hash,
+        originalSize,
+        storedSize: encryptedData.length,
+        chunks: chunks.length,
+        compressed
+    });
+
+    for (const chunk of chunkFiles) {
+        onProgress?.(`Uploading chunk ${chunk.index + 1}/${chunkFiles.length}...`);
+
+        const caption = chunks.length > 1
+            ? `📦 ${filename} (${chunk.index + 1}/${chunks.length})`
+            : `📦 ${filename}`;
+
+        const result = await client.sendFile(chunk.path, caption);
+
+        // Store file_id instead of message_id for downloads
+        db.addChunk(fileId, chunk.index, result.messageId.toString(), chunk.size);
+
+        // Also store file_id for easier retrieval
+        db.db.prepare('UPDATE chunks SET file_telegram_id = ? WHERE file_id = ? AND chunk_index = ?')
+            .run(result.fileId, fileId, chunk.index);
+
+        // Clean up temp file
+        fs.unlinkSync(chunk.path);
+    }
+
+    db.close();
+
+    // Clean up temp dir
+    try {
+        fs.rmdirSync(tempDir);
+    } catch (e) {
+        // Ignore if not empty
+    }
+
+    return {
+        filename,
+        hash,
+        originalSize,
+        storedSize: encryptedData.length,
+        chunks: chunks.length,
+        compressed
+    };
+}
+
+/**
+ * Retrieve a file from Telegram
+ */
+export async function retrieveFile(fileRecord, options) {
+    const { password, dataDir, outputPath, config, onProgress } = options;
+
+    onProgress?.('Connecting to Telegram...');
+
+    // Get chunk info
+    const db = new FileIndex(path.join(dataDir, 'index.db'));
+    db.init();
+
+    const chunks = db.getChunks(fileRecord.id);
+    db.close();
+
+    if (chunks.length === 0) {
+        throw new Error('No chunk metadata found for this file');
+    }
+
+    // Connect to Telegram
+    const client = new TelegramClient(dataDir);
+    await client.initialize(config.botToken);
+    client.setChatId(config.chatId);
+
+    // Download all chunks
+    const downloadedChunks = [];
+
+    for (const chunk of chunks) {
+        onProgress?.(`Downloading chunk ${chunk.chunk_index + 1}/${chunks.length}...`);
+
+        const data = await client.downloadFile(chunk.file_telegram_id);
+
+        // Parse header
+        const header = parseHeader(data);
+        const payload = data.subarray(HEADER_SIZE);
+
+        downloadedChunks.push({
+            index: header.chunkIndex,
+            total: header.totalChunks,
+            data: payload,
+            compressed: header.compressed
+        });
+    }
+
+    // Reassemble
+    onProgress?.('Reassembling...');
+    downloadedChunks.sort((a, b) => a.index - b.index);
+    const encryptedData = Buffer.concat(downloadedChunks.map(c => c.data));
+
+    // Decrypt
+    onProgress?.('Decrypting...');
+    const encryptor = new Encryptor(password);
+    const compressedData = encryptor.decrypt(encryptedData);
+
+    // Decompress
+    onProgress?.('Decompressing...');
+    const compressor = new Compressor();
+    const wasCompressed = downloadedChunks[0].compressed;
+    const originalData = await compressor.decompress(compressedData, wasCompressed);
+
+    // Write to output
+    onProgress?.('Writing file...');
+    fs.writeFileSync(outputPath, originalData);
+
+    return {
+        path: outputPath,
+        size: originalData.length
+    };
+}

package/src/sync/sync.js

@@ -0,0 +1,297 @@
+/**
+ * SyncEngine - Watches folders and syncs to Telegram
+ * Dropbox-like auto-sync functionality
+ */
+
+import fs from 'fs';
+import path from 'path';
+import { EventEmitter } from 'events';
+import { FileIndex } from '../db/index.js';
+import { hashFile } from '../crypto/encryption.js';
+import { processFile } from '../index.js';
+
+// Debounce time in ms to batch rapid file changes
+const DEBOUNCE_MS = 1000;
+
+// Ignore patterns
+const IGNORE_PATTERNS = [
+    /^\./, // Hidden files
+    /~$/, // Backup files
+    /\.swp$/, // Vim swap files
+    /\.tmp$/, // Temp files
+    /node_modules/,
+    /\.git/
+];
+
+export class SyncEngine extends EventEmitter {
+    constructor(options) {
+        super();
+        this.dataDir = options.dataDir;
+        this.password = options.password;
+        this.config = options.config;
+        this.watchers = new Map(); // path -> FSWatcher
+        this.pendingChanges = new Map(); // path -> timeout
+        this.db = null;
+        this.running = false;
+    }
+
+    /**
+     * Initialize the sync engine
+     */
+    async initialize() {
+        this.db = new FileIndex(path.join(this.dataDir, 'index.db'));
+        this.db.init();
+    }
+
+    /**
+     * Check if a file should be ignored
+     */
+    shouldIgnore(filename) {
+        return IGNORE_PATTERNS.some(pattern => pattern.test(filename));
+    }
+
+    /**
+     * Get all files in a directory recursively
+     */
+    async scanDirectory(dirPath, relativeTo = dirPath) {
+        const files = [];
+        const entries = fs.readdirSync(dirPath, { withFileTypes: true });
+
+        for (const entry of entries) {
+            if (this.shouldIgnore(entry.name)) continue;
+
+            const fullPath = path.join(dirPath, entry.name);
+            const relativePath = path.relative(relativeTo, fullPath);
+
+            if (entry.isDirectory()) {
+                const subFiles = await this.scanDirectory(fullPath, relativeTo);
+                files.push(...subFiles);
+            } else if (entry.isFile()) {
+                const stats = fs.statSync(fullPath);
+                files.push({
+                    path: fullPath,
+                    relativePath,
+                    mtime: stats.mtimeMs,
+                    size: stats.size
+                });
+            }
+        }
+
+        return files;
+    }
+
+    /**
+     * Sync a single folder - initial scan
+     */
+    async syncFolder(folderPath) {
+        const folder = this.db.getSyncFolderByPath(folderPath);
+        if (!folder) {
+            throw new Error(`Folder not registered: ${folderPath}`);
+        }
+
+        this.emit('sync-start', { folder: folderPath });
+
+        const files = await this.scanDirectory(folderPath);
+        const existingStates = this.db.getFolderSyncStates(folder.id);
+        const stateMap = new Map(existingStates.map(s => [s.relative_path, s]));
+
+        let uploaded = 0;
+        let skipped = 0;
+
+        for (const file of files) {
+            const existing = stateMap.get(file.relativePath);
+
+            // Check if file has changed (by mtime)
+            if (existing && existing.mtime >= file.mtime) {
+                skipped++;
+                continue;
+            }
+
+            // Calculate hash to detect actual changes
+            const hash = await hashFile(file.path);
+
+            if (existing && existing.file_hash === hash) {
+                // File unchanged, just update mtime
+                this.db.updateSyncState(folder.id, file.relativePath, hash, file.mtime);
+                skipped++;
+                continue;
+            }
+
+            // File is new or changed - upload it
+            try {
+                this.emit('file-upload-start', { file: file.relativePath });
+
+                await processFile(file.path, {
+                    password: this.password,
+                    dataDir: this.dataDir,
+                    customName: file.relativePath, // Use relative path as name
+                    config: this.config,
+                    onProgress: (msg) => this.emit('progress', { file: file.relativePath, message: msg })
+                });
+
+                // Update sync state
+                this.db.updateSyncState(folder.id, file.relativePath, hash, file.mtime);
+                uploaded++;
+
+                this.emit('file-upload-complete', { file: file.relativePath });
+            } catch (err) {
+                // File might already exist, skip
+                if (err.message.includes('duplicate')) {
+                    this.db.updateSyncState(folder.id, file.relativePath, hash, file.mtime);
+                    skipped++;
+                } else {
+                    this.emit('file-upload-error', { file: file.relativePath, error: err.message });
+                }
+            }
+        }
+
+        this.emit('sync-complete', { folder: folderPath, uploaded, skipped });
+
+        return { uploaded, skipped };
+    }
+
+    /**
+     * Handle a file change event (debounced)
+     */
+    handleFileChange(folderPath, filename) {
+        if (this.shouldIgnore(filename)) return;
+
+        const fullPath = path.join(folderPath, filename);
+        const key = fullPath;
+
+        // Clear existing timeout
+        if (this.pendingChanges.has(key)) {
+            clearTimeout(this.pendingChanges.get(key));
+        }
+
+        // Set new debounced handler
+        const timeout = setTimeout(async () => {
+            this.pendingChanges.delete(key);
+            await this.processFileChange(folderPath, filename);
+        }, DEBOUNCE_MS);
+
+        this.pendingChanges.set(key, timeout);
+    }
+
+    /**
+     * Process a file change after debounce
+     */
+    async processFileChange(folderPath, filename) {
+        const fullPath = path.join(folderPath, filename);
+
+        // Check if file still exists
+        if (!fs.existsSync(fullPath)) {
+            this.emit('file-deleted', { file: filename });
+            return;
+        }
+
+        const stats = fs.statSync(fullPath);
+        if (!stats.isFile()) return;
+
+        const folder = this.db.getSyncFolderByPath(folderPath);
+        if (!folder) return;
+
+        try {
+            const hash = await hashFile(fullPath);
+            const existing = this.db.getSyncState(folder.id, filename);
+
+            if (existing && existing.file_hash === hash) {
+                return; // No actual change
+            }
+
+            this.emit('file-upload-start', { file: filename });
+
+            await processFile(fullPath, {
+                password: this.password,
+                dataDir: this.dataDir,
+                customName: filename,
+                config: this.config,
+                onProgress: (msg) => this.emit('progress', { file: filename, message: msg })
+            });
+
+            this.db.updateSyncState(folder.id, filename, hash, stats.mtimeMs);
+
+            this.emit('file-upload-complete', { file: filename });
+        } catch (err) {
+            if (!err.message.includes('duplicate')) {
+                this.emit('file-upload-error', { file: filename, error: err.message });
+            }
+        }
+    }
+
+    /**
+     * Start watching a folder
+     */
+    watchFolder(folderPath) {
+        if (this.watchers.has(folderPath)) {
+            return; // Already watching
+        }
+
+        const watcher = fs.watch(folderPath, { recursive: true }, (event, filename) => {
+            if (filename) {
+                this.handleFileChange(folderPath, filename);
+            }
+        });
+
+        watcher.on('error', (err) => {
+            this.emit('watch-error', { folder: folderPath, error: err.message });
+        });
+
+        this.watchers.set(folderPath, watcher);
+        this.emit('watch-start', { folder: folderPath });
+    }
+
+    /**
+     * Stop watching a folder
+     */
+    unwatchFolder(folderPath) {
+        const watcher = this.watchers.get(folderPath);
+        if (watcher) {
+            watcher.close();
+            this.watchers.delete(folderPath);
+            this.emit('watch-stop', { folder: folderPath });
+        }
+    }
+
+    /**
+     * Start syncing all registered folders
+     */
+    async start() {
+        this.running = true;
+        const folders = this.db.getSyncFolders();
+
+        for (const folder of folders) {
+            if (folder.enabled) {
+                // Initial sync
+                await this.syncFolder(folder.local_path);
+                // Start watching
+                this.watchFolder(folder.local_path);
+            }
+        }
+    }
+
+    /**
+     * Stop all watchers
+     */
+    stop() {
+        this.running = false;
+
+        // Clear pending changes
+        for (const timeout of this.pendingChanges.values()) {
+            clearTimeout(timeout);
+        }
+        this.pendingChanges.clear();
+
+        // Close all watchers
+        for (const [folderPath, watcher] of this.watchers) {
+            watcher.close();
+            this.emit('watch-stop', { folder: folderPath });
+        }
+        this.watchers.clear();
+
+        if (this.db) {
+            this.db.close();
+            this.db = null;
+        }
+    }
+}

package/src/telegram/client.js

@@ -0,0 +1,131 @@
+/**
+ * Telegram Bot client wrapper
+ * Uses official Telegram Bot API - 2GB file limit, FREE, no ban risk!
+ */
+
+import TelegramBot from 'node-telegram-bot-api';
+import fs from 'fs';
+import path from 'path';
+
+export class TelegramClient {
+    constructor(dataDir) {
+        this.dataDir = dataDir;
+        this.bot = null;
+        this.chatId = null;
+    }
+
+    /**
+     * Initialize with bot token
+     * Get token from @BotFather on Telegram
+     */
+    async initialize(token) {
+        this.bot = new TelegramBot(token, { polling: false });
+
+        // Verify the token works
+        try {
+            const me = await this.bot.getMe();
+            console.log(`✓ Connected as @${me.username}`);
+            return me;
+        } catch (err) {
+            throw new Error(`Invalid bot token: ${err.message}`);
+        }
+    }
+
+    /**
+     * Set the chat ID for storage (your personal chat with the bot)
+     */
+    setChatId(chatId) {
+        this.chatId = chatId;
+    }
+
+    /**
+     * Wait for user to send a message to get their chat ID
+     * This is needed for first-time setup
+     */
+    async waitForChatId(timeout = 120000) {
+        return new Promise((resolve, reject) => {
+            const pollingBot = new TelegramBot(this.bot.token, { polling: true });
+
+            const timer = setTimeout(() => {
+                pollingBot.stopPolling();
+                reject(new Error('Timeout waiting for message. Please message your bot on Telegram.'));
+            }, timeout);
+
+            pollingBot.on('message', (msg) => {
+                clearTimeout(timer);
+                pollingBot.stopPolling();
+                this.chatId = msg.chat.id;
+                resolve({
+                    chatId: msg.chat.id,
+                    username: msg.from.username,
+                    firstName: msg.from.first_name
+                });
+            });
+
+            pollingBot.on('polling_error', (err) => {
+                // Ignore polling errors during shutdown
+                if (!err.message.includes('ETELEGRAM')) {
+                    console.error('Polling error:', err.message);
+                }
+            });
+        });
+    }
+
+    /**
+     * Send a file to the storage chat
+     * Telegram supports up to 2GB for documents!
+     */
+    async sendFile(filePath, caption = '') {
+        if (!this.chatId) {
+            throw new Error('Chat ID not set. Run init first.');
+        }
+
+        const fileStream = fs.createReadStream(filePath);
+        const filename = path.basename(filePath);
+
+        const message = await this.bot.sendDocument(this.chatId, fileStream, {
+            caption: caption
+        }, {
+            filename: filename,
+            contentType: 'application/octet-stream'
+        });
+
+        return {
+            messageId: message.message_id,
+            fileId: message.document.file_id,
+            timestamp: message.date
+        };
+    }
+
+    /**
+     * Download a file from Telegram
+     */
+    async downloadFile(fileId) {
+        // Get file path from Telegram servers
+        const file = await this.bot.getFile(fileId);
+
+        // Download the file
+        const fileStream = await this.bot.getFileStream(fileId);
+
+        // Collect chunks into buffer
+        const chunks = [];
+        for await (const chunk of fileStream) {
+            chunks.push(chunk);
+        }
+
+        return Buffer.concat(chunks);
+    }
+
+    /**
+     * Delete a message (optional cleanup)
+     */
+    async deleteMessage(messageId) {
+        try {
+            await this.bot.deleteMessage(this.chatId, messageId);
+            return true;
+        } catch (err) {
+            // Message might already be deleted
+            return false;
+        }
+    }
+}