@nightowne/tas-cli 1.1.1 → 2.1.0

package/src/db/index.js

@@ -92,6 +92,22 @@ export class FileIndex {
       );
     `);
 
+    // Create shares table for temporary file sharing
+    this.db.exec(`
+      CREATE TABLE IF NOT EXISTS shares (
+        id INTEGER PRIMARY KEY AUTOINCREMENT,
+        file_id INTEGER NOT NULL,
+        token TEXT UNIQUE NOT NULL,
+        expires_at TEXT NOT NULL,
+        max_downloads INTEGER NOT NULL DEFAULT 1,
+        download_count INTEGER NOT NULL DEFAULT 0,
+        created_at TEXT NOT NULL DEFAULT (datetime('now')),
+        FOREIGN KEY (file_id) REFERENCES files(id) ON DELETE CASCADE
+      );
+
+      CREATE INDEX IF NOT EXISTS idx_shares_token ON shares(token);
+    `);
+
     // Create pending_uploads table for resume functionality
     this.db.exec(`
       CREATE TABLE IF NOT EXISTS pending_uploads (
@@ -492,6 +508,76 @@ export class FileIndex {
     return stmt.get(hash);
   }
 
+  // ============== SHARE METHODS ==============
+
+  /**
+   * Create a share link for a file
+   */
+  addShare(fileId, token, expiresAt, maxDownloads = 1) {
+    const stmt = this.db.prepare(`
+      INSERT INTO shares (file_id, token, expires_at, max_downloads)
+      VALUES (?, ?, ?, ?)
+    `);
+    const result = stmt.run(fileId, token, expiresAt, maxDownloads);
+    return result.lastInsertRowid;
+  }
+
+  /**
+   * Get a share by token (returns null if not found)
+   */
+  getShare(token) {
+    const stmt = this.db.prepare(`
+      SELECT s.*, f.filename, f.original_size
+      FROM shares s
+      JOIN files f ON s.file_id = f.id
+      WHERE s.token = ?
+    `);
+    return stmt.get(token) || null;
+  }
+
+  /**
+   * List all active shares
+   */
+  listShares() {
+    const stmt = this.db.prepare(`
+      SELECT s.*, f.filename, f.original_size
+      FROM shares s
+      JOIN files f ON s.file_id = f.id
+      ORDER BY s.created_at DESC
+    `);
+    return stmt.all();
+  }
+
+  /**
+   * Revoke (delete) a share by token
+   */
+  revokeShare(token) {
+    const stmt = this.db.prepare('DELETE FROM shares WHERE token = ?');
+    const result = stmt.run(token);
+    return result.changes > 0;
+  }
+
+  /**
+   * Increment download count for a share
+   */
+  incrementShareDownload(token) {
+    const stmt = this.db.prepare(`
+      UPDATE shares SET download_count = download_count + 1 WHERE token = ?
+    `);
+    stmt.run(token);
+  }
+
+  /**
+   * Remove expired shares
+   */
+  cleanExpiredShares() {
+    const stmt = this.db.prepare(`
+      DELETE FROM shares WHERE
+        REPLACE(REPLACE(expires_at, 'T', ' '), 'Z', '') < strftime('%Y-%m-%d %H:%M:%f', 'now')
+    `);
+    return stmt.run().changes;
+  }
+
   /**
    * Close database connection
    */

package/src/fuse/mount.js

@@ -130,26 +130,35 @@ export class TelegramFS {
     }
 
     /**
-     * Read file contents
+     * Read file contents from disk cache
      */
     async read(filepath, fd, buffer, length, position, cb) {
         const filename = path.basename(filepath);
 
         try {
+            // Check write buffers first
+            const wb = this.writeBuffers.get(filename);
+            if (wb) {
+                const slice = wb.data.subarray(position, position + length);
+                slice.copy(buffer);
+                return cb(slice.length);
+            }
+
             // Check cache first
-            let data = this.getCached(filename);
+            let cachedPath = this.getCached(filename);
 
-            if (!data) {
-                // Download and decrypt
-                data = await this.downloadFile(filename);
-                this.setCache(filename, data);
+            if (!cachedPath) {
+                // Download, decrypt, and save to disk cache
+                cachedPath = await this.downloadFileToCache(filename);
+                this.setCache(filename, cachedPath);
             }
 
-            // Copy requested portion to buffer
-            const slice = data.subarray(position, position + length);
-            slice.copy(buffer);
+            // Copy requested portion to buffer from disk
+            const fdDisk = fs.openSync(cachedPath, 'r');
+            const bytesRead = fs.readSync(fdDisk, buffer, 0, length, position);
+            fs.closeSync(fdDisk);
 
-            return cb(slice.length);
+            return cb(bytesRead);
         } catch (err) {
             console.error('Read error:', err.message);
             return cb(Fuse.EIO);
@@ -305,10 +314,10 @@ export class TelegramFS {
 
         // Get or load into write buffer
         if (!this.writeBuffers.has(filename)) {
-            const cached = this.getCached(filename);
-            if (cached) {
+            const cachedPath = this.getCached(filename);
+            if (cachedPath) {
                 this.writeBuffers.set(filename, {
-                    data: Buffer.from(cached),
+                    data: fs.readFileSync(cachedPath), // Note: RAM buffer here could be big, but it's okay for truncate/writes right now
                     modified: true
                 });
             } else {
@@ -335,38 +344,81 @@ export class TelegramFS {
 
     // ============== Helper Methods ==============
 
-    async downloadFile(filename) {
+    async downloadFileToCache(filename) {
         const file = this.db.findByName(filename);
         if (!file) throw new Error('File not found');
 
+        const cacheDir = path.join(this.dataDir, 'cache');
+        if (!fs.existsSync(cacheDir)) {
+            fs.mkdirSync(cacheDir, { recursive: true });
+        }
+
+        const outputPath = path.join(cacheDir, file.hash);
+
+        // If it's already fully downloaded and cached on disk, return path
+        if (fs.existsSync(outputPath)) {
+            const stats = fs.statSync(outputPath);
+            if (stats.size === file.original_size) {
+                return outputPath;
+            }
+        }
+
         const chunks = this.db.getChunks(file.id);
         if (chunks.length === 0) throw new Error('No chunks found');
 
-        // Download all chunks
-        const downloadedChunks = [];
-
-        for (const chunk of chunks) {
-            const data = await this.client.downloadFile(chunk.file_telegram_id);
-            const header = parseHeader(data);
-            const payload = data.subarray(HEADER_SIZE);
+        // Pre-sort chunks
+        chunks.sort((a, b) => a.chunk_index - b.chunk_index);
+
+        const firstChunkData = await this.client.downloadFile(chunks[0].file_telegram_id);
+        const header = parseHeader(firstChunkData);
+        let wasCompressed = header.compressed;
+
+        const decryptStream = this.encryptor.getDecryptStream();
+        const decompressStream = this.compressor.getDecompressStream(wasCompressed);
+
+        const { Readable } = await import('stream');
+        const { pipeline } = await import('stream/promises');
+
+        const self = this;
+        let currentChunkIndex = 0;
+        let preloadedFirstChunk = firstChunkData;
+
+        const downloadStream = new Readable({
+            async read() {
+                try {
+                    if (currentChunkIndex >= chunks.length) {
+                        this.push(null);
+                        return;
+                    }
+
+                    const chunk = chunks[currentChunkIndex];
+                    let data;
+                    if (currentChunkIndex === 0 && preloadedFirstChunk) {
+                        data = preloadedFirstChunk;
+                        preloadedFirstChunk = null;
+                    } else {
+                        data = await self.client.downloadFile(chunk.file_telegram_id);
+                    }
+
+                    const payload = data.subarray(HEADER_SIZE);
+                    this.push(payload);
+                    currentChunkIndex++;
+                } catch (err) {
+                    this.destroy(err);
+                }
+            }
+        });
 
-            downloadedChunks.push({
-                index: header.chunkIndex,
-                data: payload,
-                compressed: header.compressed
-            });
-        }
+        const tmpOutputPath = outputPath + '.tmp';
+        const writeStream = fs.createWriteStream(tmpOutputPath);
 
-        // Reassemble
-        downloadedChunks.sort((a, b) => a.index - b.index);
-        const encryptedData = Buffer.concat(downloadedChunks.map(c => c.data));
+        // Pipeline: Telegram -> Decrypt -> Decompress -> Disk Cache
+        await pipeline(downloadStream, decryptStream, decompressStream, writeStream);
 
-        // Decrypt
-        const compressedData = this.encryptor.decrypt(encryptedData);
+        // Rename to final atomic path
+        fs.renameSync(tmpOutputPath, outputPath);
 
-        // Decompress
-        const wasCompressed = downloadedChunks[0].compressed;
-        return await this.compressor.decompress(compressedData, wasCompressed);
+        return outputPath;
     }
 
     async uploadFile(filename, data) {
@@ -399,7 +451,7 @@ export class TelegramFS {
         const encryptedData = this.encryptor.encrypt(compressedData);
 
         // Create temp file with header
-        const tempDir = path.join(this.dataDir, 'tmp');
+        const tempDir = process.env.TAS_TMP_DIR || path.join(this.dataDir, 'tmp');
         if (!fs.existsSync(tempDir)) {
             fs.mkdirSync(tempDir, { recursive: true });
         }
@@ -438,22 +490,34 @@ export class TelegramFS {
         if (!entry) return null;
 
         if (Date.now() - entry.timestamp > CACHE_TTL) {
+            // Expired, delete the file if possible
+            try {
+                if (fs.existsSync(entry.path)) fs.unlinkSync(entry.path);
+            } catch (e) { }
             fileCache.delete(filename);
             return null;
         }
 
-        return entry.data;
+        // Extend cache TTL on read
+        entry.timestamp = Date.now();
+        return entry.path;
     }
 
-    setCache(filename, data) {
+    setCache(filename, cachePath) {
         fileCache.set(filename, {
-            data,
+            path: cachePath,
             timestamp: Date.now()
         });
     }
 
     invalidateCache(filename) {
-        fileCache.delete(filename);
+        const entry = fileCache.get(filename);
+        if (entry) {
+            try {
+                if (fs.existsSync(entry.path)) fs.unlinkSync(entry.path);
+            } catch (e) { }
+            fileCache.delete(filename);
+        }
     }
 
     /**

package/src/index.js

@@ -5,6 +5,7 @@
 
 import fs from 'fs';
 import path from 'path';
+import { pipeline } from 'stream/promises';
 import { Encryptor, hashFile } from './crypto/encryption.js';
 import { Compressor } from './utils/compression.js';
 import { Chunker, createHeader, parseHeader, HEADER_SIZE } from './utils/chunker.js';
@@ -23,10 +24,10 @@ export async function processFile(filePath, options) {
 
     onProgress?.('Reading file...');
 
-    // Read file
+    // Read file initially just to get size
     const filename = customName || path.basename(filePath);
-    const fileData = fs.readFileSync(filePath);
-    const originalSize = fileData.length;
+    const stats = fs.statSync(filePath);
+    const originalSize = stats.size;
 
     // Calculate hash
     onProgress?.('Calculating hash...');
@@ -41,99 +42,134 @@ export async function processFile(filePath, options) {
         throw new Error('File already uploaded (duplicate hash)');
     }
 
-    // Compress
-    onProgress?.('Compressing...');
+    // Prepare processing components
     const compressor = new Compressor();
-    const { data: compressedData, compressed } = await compressor.compress(fileData, filename);
+    const { stream: compressStream, compressed } = compressor.getCompressStream(filename);
+    const flags = compressed ? 1 : 0;
 
-    // Encrypt
-    onProgress?.('Encrypting...');
     const encryptor = new Encryptor(password);
-    const encryptedData = encryptor.encrypt(compressedData);
-
-    // Chunk if needed (Telegram bot limit ~50MB per file)
-    onProgress?.('Preparing chunks...');
-    const chunks = [];
-    const numChunks = Math.ceil(encryptedData.length / TELEGRAM_CHUNK_SIZE);
-
-    for (let i = 0; i < numChunks; i++) {
-        const start = i * TELEGRAM_CHUNK_SIZE;
-        const end = Math.min(start + TELEGRAM_CHUNK_SIZE, encryptedData.length);
-        chunks.push({
-            index: i,
-            total: numChunks,
-            data: encryptedData.subarray(start, end)
-        });
-    }
+    const encryptStream = encryptor.getEncryptStream();
 
-    // Prepare files with headers
-    const tempDir = path.join(dataDir, 'tmp');
+    const tempDir = process.env.TAS_TMP_DIR || path.join(dataDir, 'tmp');
     if (!fs.existsSync(tempDir)) {
         fs.mkdirSync(tempDir, { recursive: true });
     }
 
-    const chunkFiles = [];
-    const flags = compressed ? 1 : 0;
-
-    for (const chunk of chunks) {
-        const header = createHeader(filename, originalSize, chunk.index, chunk.total, flags);
-        const chunkData = Buffer.concat([header, chunk.data]);
-
-        const chunkFilename = chunks.length > 1
-            ? `${hash.substring(0, 12)}.part${chunk.index}.tas`
-            : `${hash.substring(0, 12)}.tas`;
-
-        const chunkPath = path.join(tempDir, chunkFilename);
-        fs.writeFileSync(chunkPath, chunkData);
-
-        chunkFiles.push({
-            index: chunk.index,
-            path: chunkPath,
-            size: chunkData.length
-        });
-    }
-
     // Connect to Telegram
     onProgress?.('Connecting to Telegram...');
     const client = new TelegramClient(dataDir);
     await client.initialize(config.botToken);
     client.setChatId(config.chatId);
 
-    // Upload chunks
+    // We will stream through a custom Writable chunker
+    const { Writable } = await import('stream');
+
+    // First pass estimation (for calculating total chunks and progress)
+    // We don't know the exact final size due to compression and encryption overhead,
+    // so we'll estimate total chunks and update it if needed.
+    // For small files < 49MB we assume 1 chunk.
+    let estimatedSize = compressed ? originalSize : originalSize + 128; // Add encryption overhead
+    if (compressed && originalSize > 1024 * 1024) estimatedSize = originalSize * 0.8; // Rough guess
+    let estimatedChunks = Math.ceil(estimatedSize / TELEGRAM_CHUNK_SIZE) || 1;
+
+    // Register file in DB
     const fileId = db.addFile({
         filename,
         hash,
         originalSize,
-        storedSize: encryptedData.length,
-        chunks: chunks.length,
+        storedSize: 0, // Will update later
+        chunks: estimatedChunks,
         compressed
     });
 
+    onProgress?.('Processing and uploading streams...');
     let uploadedBytes = 0;
-    const totalBytes = chunkFiles.reduce((acc, c) => acc + c.size, 0);
+    let chunkIndex = 0;
+
+    let currentChunkBuffer = Buffer.alloc(0);
+    let totalStoredSize = 0;
+
+    // Helper to upload a single chunk
+    const uploadCurrentChunk = async (isFinal = false) => {
+        if (currentChunkBuffer.length === 0 && !isFinal) return; // Nothing to upload
+        if (currentChunkBuffer.length === 0 && isFinal && chunkIndex > 0) return; // Empty final chunk after perfect split
+
+        // At this point we know if it's the final chunk, so we know the total chunks
+        const totalChunks = isFinal ? chunkIndex + 1 : Math.max(estimatedChunks, chunkIndex + 1);
+
+        const header = createHeader(filename, originalSize, chunkIndex, totalChunks, flags);
+        const chunkData = Buffer.concat([header, currentChunkBuffer]);
 
-    for (const chunk of chunkFiles) {
-        onProgress?.(`Uploading chunk ${chunk.index + 1}/${chunkFiles.length}...`);
+        const chunkFilename = totalChunks > 1
+            ? `${hash.substring(0, 12)}.part${chunkIndex}.tas`
+            : `${hash.substring(0, 12)}.tas`;
+
+        const chunkPath = path.join(tempDir, chunkFilename);
+        fs.writeFileSync(chunkPath, chunkData);
 
-        const caption = chunks.length > 1
-            ? `📦 ${filename} (${chunk.index + 1}/${chunks.length})`
+        const caption = totalChunks > 1
+            ? `📦 ${filename} (${chunkIndex + 1}/${totalChunks})`
             : `📦 ${filename}`;
 
-        const result = await client.sendFile(chunk.path, caption);
+        onProgress?.(`Uploading chunk ${chunkIndex + 1}...`);
+
+        const result = await client.sendFile(chunkPath, caption);
 
-        uploadedBytes += chunk.size;
-        onByteProgress?.({ uploaded: uploadedBytes, total: totalBytes, chunk: chunk.index + 1, totalChunks: chunkFiles.length });
+        uploadedBytes += chunkData.length;
+        totalStoredSize += currentChunkBuffer.length;
 
-        // Store file_id instead of message_id for downloads
-        db.addChunk(fileId, chunk.index, result.messageId.toString(), chunk.size);
+        onByteProgress?.({ uploaded: uploadedBytes, total: estimatedSize, chunk: chunkIndex + 1, totalChunks });
 
-        // Also store file_id for easier retrieval
+        // Store file_id
+        db.addChunk(fileId, chunkIndex, result.messageId.toString(), chunkData.length);
         db.db.prepare('UPDATE chunks SET file_telegram_id = ? WHERE file_id = ? AND chunk_index = ?')
-            .run(result.fileId, fileId, chunk.index);
+            .run(result.fileId, fileId, chunkIndex);
 
-        // Clean up temp file
-        fs.unlinkSync(chunk.path);
-    }
+        // Clean up temp file immediately to save disk space
+        fs.unlinkSync(chunkPath);
+
+        chunkIndex++;
+        currentChunkBuffer = Buffer.alloc(0);
+    };
+
+    const chunkingStream = new Writable({
+        async write(chunk, encoding, callback) {
+            currentChunkBuffer = Buffer.concat([currentChunkBuffer, chunk]);
+
+            // If we exceeded the chunk limit, flush it
+            if (currentChunkBuffer.length >= TELEGRAM_CHUNK_SIZE) {
+                const overflow = currentChunkBuffer.subarray(TELEGRAM_CHUNK_SIZE);
+                currentChunkBuffer = currentChunkBuffer.subarray(0, TELEGRAM_CHUNK_SIZE);
+
+                try {
+                    await uploadCurrentChunk(false);
+                    currentChunkBuffer = overflow; // carry over
+                    callback();
+                } catch (err) {
+                    callback(err);
+                }
+            } else {
+                callback();
+            }
+        },
+        async final(callback) {
+            try {
+                await uploadCurrentChunk(true);
+                callback();
+            } catch (err) {
+                callback(err);
+            }
+        }
+    });
+
+    const readStream = fs.createReadStream(filePath);
+
+    // Run the pipeline: Read -> Compress -> Encrypt -> Chunk & Upload
+    await pipeline(readStream, compressStream, encryptStream, chunkingStream);
+
+    // Update the DB with the final accurate values 
+    db.db.prepare('UPDATE files SET stored_size = ?, chunks = ? WHERE id = ?')
+        .run(totalStoredSize, chunkIndex, fileId);
 
     db.close();
 
@@ -148,8 +184,8 @@ export async function processFile(filePath, options) {
         filename,
         hash,
         originalSize,
-        storedSize: encryptedData.length,
-        chunks: chunks.length,
+        storedSize: totalStoredSize,
+        chunks: chunkIndex,
         compressed
     };
 }
@@ -173,57 +209,91 @@ export async function retrieveFile(fileRecord, options) {
         throw new Error('No chunk metadata found for this file');
     }
 
+    // Prepare components
+    const encryptor = new Encryptor(password);
+    const decryptStream = encryptor.getDecryptStream();
+
+    const tempDir = process.env.TAS_TMP_DIR || path.join(dataDir, 'tmp');
+    if (!fs.existsSync(tempDir)) {
+        fs.mkdirSync(tempDir, { recursive: true });
+    }
+
     // Connect to Telegram
     const client = new TelegramClient(dataDir);
     await client.initialize(config.botToken);
     client.setChatId(config.chatId);
 
-    // Download all chunks
-    const downloadedChunks = [];
-    let downloadedBytes = 0;
-    const totalBytes = fileRecord.stored_size || chunks.reduce((acc, c) => acc + (c.size || 0), 0);
+    // Get total size from first chunk's header, or from DB
+    const firstChunkData = await client.downloadFile(chunks[0].file_telegram_id);
+    const header = parseHeader(firstChunkData);
 
-    for (const chunk of chunks) {
-        onProgress?.(`Downloading chunk ${chunk.chunk_index + 1}/${chunks.length}...`);
+    // Total original uncompressed size
+    let expectedOriginalSize = header.originalSize;
+    let wasCompressed = header.compressed;
 
-        const data = await client.downloadFile(chunk.file_telegram_id);
-        downloadedBytes += data.length;
-        onByteProgress?.({ downloaded: downloadedBytes, total: totalBytes, chunk: chunk.chunk_index + 1, totalChunks: chunks.length });
+    const compressor = new Compressor();
+    const decompressStream = compressor.getDecompressStream(wasCompressed);
 
-        // Parse header
-        const header = parseHeader(data);
-        const payload = data.subarray(HEADER_SIZE);
+    // We need a Readable stream that will lazily fetch chunks from Telegram
+    // and push them into the decryption pipeline.
+    const { Readable } = await import('stream');
 
-        downloadedChunks.push({
-            index: header.chunkIndex,
-            total: header.totalChunks,
-            data: payload,
-            compressed: header.compressed
-        });
-    }
+    const totalBytes = fileRecord.stored_size || chunks.reduce((acc, c) => acc + (c.size || 0), 0);
+    let downloadedBytes = 0;
 
-    // Reassemble
-    onProgress?.('Reassembling...');
-    downloadedChunks.sort((a, b) => a.index - b.index);
-    const encryptedData = Buffer.concat(downloadedChunks.map(c => c.data));
+    // Pre-sort chunks by index so we download them in correct order
+    chunks.sort((a, b) => a.chunk_index - b.chunk_index);
+
+    let currentChunkIndex = 0;
+
+    // We already downloaded the first chunk to inspect its header, we shouldn't discard it.
+    let preloadedFirstChunk = firstChunkData;
+
+    const downloadStream = new Readable({
+        async read() {
+            try {
+                if (currentChunkIndex >= chunks.length) {
+                    this.push(null); // End of stream
+                    return;
+                }
+
+                const chunk = chunks[currentChunkIndex];
+                onProgress?.(`Downloading chunk ${chunk.chunk_index + 1}/${chunks.length}...`);
+
+                let data;
+                if (currentChunkIndex === 0 && preloadedFirstChunk) {
+                    data = preloadedFirstChunk;
+                    preloadedFirstChunk = null;
+                } else {
+                    data = await client.downloadFile(chunk.file_telegram_id);
+                }
+
+                downloadedBytes += data.length;
+                onByteProgress?.({ downloaded: downloadedBytes, total: totalBytes, chunk: chunk.chunk_index + 1, totalChunks: chunks.length });
+
+                // Strip header before pushing
+                const payload = data.subarray(HEADER_SIZE);
+                this.push(payload);
+
+                currentChunkIndex++;
+            } catch (err) {
+                this.destroy(err);
+            }
+        }
+    });
 
-    // Decrypt
-    onProgress?.('Decrypting...');
-    const encryptor = new Encryptor(password);
-    const compressedData = encryptor.decrypt(encryptedData);
+    const writeStream = fs.createWriteStream(outputPath);
+    const { pipeline } = await import('stream/promises');
 
-    // Decompress
-    onProgress?.('Decompressing...');
-    const compressor = new Compressor();
-    const wasCompressed = downloadedChunks[0].compressed;
-    const originalData = await compressor.decompress(compressedData, wasCompressed);
+    onProgress?.('Decrypting, decompressing, and writing file...');
+
+    // Pipeline: Download from Telegram -> Decrypt -> Decompress -> Disk
+    await pipeline(downloadStream, decryptStream, decompressStream, writeStream);
 
-    // Write to output
-    onProgress?.('Writing file...');
-    fs.writeFileSync(outputPath, originalData);
+    const finalStats = fs.statSync(outputPath);
 
     return {
         path: outputPath,
-        size: originalData.length
+        size: finalStats.size
     };
 }