@nightowne/tas-cli 2.3.0 → 2.4.1

package/src/index.js

@@ -8,7 +8,7 @@ import path from 'path';
 import { pipeline } from 'stream/promises';
 import { Encryptor, hashFile } from './crypto/encryption.js';
 import { Compressor } from './utils/compression.js';
-import { createHeader, parseHeader, HEADER_SIZE } from './utils/chunker.js';
+import { createHeader, HEADER_SIZE } from './utils/chunker.js';
 import { TelegramClient } from './telegram/client.js';
 import { FileIndex } from './db/index.js';
 
@@ -37,46 +37,47 @@ export async function processFile(filePath, options) {
     const db = new FileIndex(path.join(dataDir, 'index.db'));
     db.init();
 
-    if (db.exists(hash)) {
-        db.close();
-        throw new Error('File already uploaded (duplicate hash)');
-    }
-
-    // Prepare processing components
-    const compressor = new Compressor();
-    const { stream: compressStream, compressed } = compressor.getCompressStream(filename);
-    const flags = compressed ? 1 : 0;
-
-    const encryptor = new Encryptor(password);
-    const encryptStream = encryptor.getEncryptStream();
-
-    const tempDir = process.env.TAS_TMP_DIR || path.join(dataDir, 'tmp');
-    if (!fs.existsSync(tempDir)) {
-        fs.mkdirSync(tempDir, { recursive: true });
-    }
+    try {
+        if (db.exists(hash)) {
+            throw new Error('File already uploaded (duplicate hash)');
+        }
 
-    // Connect to Telegram
-    onProgress?.('Connecting to Telegram...');
-    const client = new TelegramClient(dataDir);
-    await client.initialize(config.botToken);
-    client.setChatId(config.chatId);
+        // Prepare processing components
+        const compressor = new Compressor();
+        const { stream: compressStream, compressed } = compressor.getCompressStream(filename);
+        const flags = compressed ? 1 : 0;
 
-    // We will stream through a custom Writable chunker
-    const { Writable } = await import('stream');
+        const encryptor = new Encryptor(password);
+        const encryptStream = encryptor.getEncryptStream();
 
-    // First pass estimation (for calculating total chunks and progress)
-    // We don't know the exact final size due to compression and encryption overhead,
-    // so we'll estimate total chunks and update it if needed.
-    // For small files < 49MB we assume 1 chunk.
-    let estimatedSize = compressed ? originalSize : originalSize + 128; // Add encryption overhead
-    if (compressed && originalSize > 1024 * 1024) estimatedSize = originalSize * 0.8; // Rough guess
-    let estimatedChunks = Math.ceil(estimatedSize / TELEGRAM_CHUNK_SIZE) || 1;
+        const tempDir = process.env.TAS_TMP_DIR || path.join(dataDir, 'tmp');
+        if (!fs.existsSync(tempDir)) {
+            try {
+                fs.mkdirSync(tempDir, { recursive: true });
+            } catch (err) {
+                throw new Error(`Failed to create temporary directory at ${tempDir}: ${err.message}. Check permissions or set TAS_TMP_DIR to a writable location.`);
+            }
+        }
 
-    // Register file in DB inside a transaction for atomicity
-    db.db.exec('BEGIN');
-    let fileId;
-    try {
-        fileId = db.addFile({
+        // Connect to Telegram
+        onProgress?.('Connecting to Telegram...');
+        const client = new TelegramClient(dataDir);
+        await client.initialize(config.botToken);
+        client.setChatId(config.chatId);
+
+        // We will stream through a custom Writable chunker
+        const { Writable } = await import('stream');
+
+        // First pass estimation (for calculating total chunks and progress)
+        // We don't know the exact final size due to compression and encryption overhead,
+        // so we'll estimate total chunks and update it if needed.
+        // For small files < 49MB we assume 1 chunk.
+        let estimatedSize = compressed ? originalSize : originalSize + 128; // Add encryption overhead
+        if (compressed && originalSize > 1024 * 1024) estimatedSize = originalSize * 0.8; // Rough guess
+        let estimatedChunks = Math.ceil(estimatedSize / TELEGRAM_CHUNK_SIZE) || 1;
+
+        // Register file in DB
+        const fileId = db.addFile({
             filename,
             hash,
             originalSize,
@@ -84,129 +85,116 @@ export async function processFile(filePath, options) {
             chunks: estimatedChunks,
             compressed
         });
-    } catch (err) {
-        db.db.exec('ROLLBACK');
-        db.close();
-        throw err;
-    }
 
-    onProgress?.('Processing and uploading streams...');
-    let uploadedBytes = 0;
-    let chunkIndex = 0;
+        onProgress?.('Processing and uploading streams...');
+        let uploadedBytes = 0;
+        let chunkIndex = 0;
 
-    let currentChunkBuffer = Buffer.alloc(0);
-    let totalStoredSize = 0;
+        let currentChunkBuffer = Buffer.alloc(0);
+        let totalStoredSize = 0;
 
-    // Helper to upload a single chunk
-    const uploadCurrentChunk = async (isFinal = false) => {
-        if (currentChunkBuffer.length === 0 && !isFinal) return; // Nothing to upload
-        if (currentChunkBuffer.length === 0 && isFinal && chunkIndex > 0) return; // Empty final chunk after perfect split
+        // Helper to upload a single chunk
+        const uploadCurrentChunk = async (isFinal = false) => {
+            if (currentChunkBuffer.length === 0 && !isFinal) return; // Nothing to upload
+            if (currentChunkBuffer.length === 0 && isFinal && chunkIndex > 0) return; // Empty final chunk after perfect split
 
-        // At this point we know if it's the final chunk, so we know the total chunks
-        const totalChunks = isFinal ? chunkIndex + 1 : Math.max(estimatedChunks, chunkIndex + 1);
+            // At this point we know if it's the final chunk, so we know the total chunks
+            const totalChunks = isFinal ? chunkIndex + 1 : Math.max(estimatedChunks, chunkIndex + 1);
 
-        const header = createHeader(filename, originalSize, chunkIndex, totalChunks, flags);
-        const chunkData = Buffer.concat([header, currentChunkBuffer]);
+            const header = createHeader(filename, originalSize, chunkIndex, totalChunks, flags);
+            const chunkData = Buffer.concat([header, currentChunkBuffer]);
 
-        const chunkFilename = totalChunks > 1
-            ? `${hash.substring(0, 12)}.part${chunkIndex}.tas`
-            : `${hash.substring(0, 12)}.tas`;
+            const chunkFilename = totalChunks > 1
+                ? `${hash.substring(0, 12)}.part${chunkIndex}.tas`
+                : `${hash.substring(0, 12)}.tas`;
 
-        const chunkPath = path.join(tempDir, chunkFilename);
-        fs.writeFileSync(chunkPath, chunkData);
+            const chunkPath = path.join(tempDir, chunkFilename);
+            fs.writeFileSync(chunkPath, chunkData);
 
-        const caption = totalChunks > 1
-            ? `📦 ${filename} (${chunkIndex + 1}/${totalChunks})`
-            : `📦 ${filename}`;
+            const caption = totalChunks > 1
+                ? `📦 ${filename} (${chunkIndex + 1}/${totalChunks})`
+                : `📦 ${filename}`;
 
-        onProgress?.(`Uploading chunk ${chunkIndex + 1}...`);
+            onProgress?.(`Uploading chunk ${chunkIndex + 1}...`);
 
-        const result = await client.sendFile(chunkPath, caption);
+            const result = await client.sendFile(chunkPath, caption);
 
-        uploadedBytes += chunkData.length;
-        totalStoredSize += currentChunkBuffer.length;
+            uploadedBytes += chunkData.length;
+            totalStoredSize += currentChunkBuffer.length;
 
-        onByteProgress?.({ uploaded: uploadedBytes, total: estimatedSize, chunk: chunkIndex + 1, totalChunks });
+            onByteProgress?.({ uploaded: uploadedBytes, total: estimatedSize, chunk: chunkIndex + 1, totalChunks });
 
-        // Store file_id
-        db.addChunk(fileId, chunkIndex, result.messageId.toString(), chunkData.length);
-        db.db.prepare('UPDATE chunks SET file_telegram_id = ? WHERE file_id = ? AND chunk_index = ?')
-            .run(result.fileId, fileId, chunkIndex);
+            // Store file_id
+            db.addChunk(fileId, chunkIndex, result.messageId.toString(), chunkData.length);
+            db.db.prepare('UPDATE chunks SET file_telegram_id = ? WHERE file_id = ? AND chunk_index = ?')
+                .run(result.fileId, fileId, chunkIndex);
 
-        // Clean up temp file immediately to save disk space
-        fs.unlinkSync(chunkPath);
+            // Clean up temp file immediately to save disk space
+            fs.unlinkSync(chunkPath);
 
-        chunkIndex++;
-        currentChunkBuffer = Buffer.alloc(0);
-    };
+            chunkIndex++;
+            currentChunkBuffer = Buffer.alloc(0);
+        };
 
-    const chunkingStream = new Writable({
-        async write(chunk, encoding, callback) {
-            currentChunkBuffer = Buffer.concat([currentChunkBuffer, chunk]);
+        const chunkingStream = new Writable({
+            async write(chunk, encoding, callback) {
+                currentChunkBuffer = Buffer.concat([currentChunkBuffer, chunk]);
 
-            // If we exceeded the chunk limit, flush it
-            if (currentChunkBuffer.length >= TELEGRAM_CHUNK_SIZE) {
-                const overflow = currentChunkBuffer.subarray(TELEGRAM_CHUNK_SIZE);
-                currentChunkBuffer = currentChunkBuffer.subarray(0, TELEGRAM_CHUNK_SIZE);
+                // If we exceeded the chunk limit, flush it
+                if (currentChunkBuffer.length >= TELEGRAM_CHUNK_SIZE) {
+                    const overflow = currentChunkBuffer.subarray(TELEGRAM_CHUNK_SIZE);
+                    currentChunkBuffer = currentChunkBuffer.subarray(0, TELEGRAM_CHUNK_SIZE);
 
+                    try {
+                        await uploadCurrentChunk(false);
+                        currentChunkBuffer = overflow; // carry over
+                        callback();
+                    } catch (err) {
+                        callback(err);
+                    }
+                } else {
+                    callback();
+                }
+            },
+            async final(callback) {
                 try {
-                    await uploadCurrentChunk(false);
-                    currentChunkBuffer = overflow; // carry over
+                    await uploadCurrentChunk(true);
                     callback();
                 } catch (err) {
                     callback(err);
                 }
-            } else {
-                callback();
             }
-        },
-        async final(callback) {
-            try {
-                await uploadCurrentChunk(true);
-                callback();
-            } catch (err) {
-                callback(err);
-            }
-        }
-    });
+        });
 
-    try {
         const readStream = fs.createReadStream(filePath);
 
         // Run the pipeline: Read -> Compress -> Encrypt -> Chunk & Upload
         await pipeline(readStream, compressStream, encryptStream, chunkingStream);
-    } catch (pipelineErr) {
-        // Pipeline failed — roll back the DB transaction so no orphaned rows remain
-        try { db.db.exec('ROLLBACK'); } catch (e) { /* already rolled back */ }
-        db.close();
-        throw pipelineErr;
-    }
 
-    // Update the DB with the final accurate values 
-    db.db.prepare('UPDATE files SET stored_size = ?, chunks = ? WHERE id = ?')
-        .run(totalStoredSize, chunkIndex, fileId);
+        // Update the DB with the final accurate values
+        db.db.prepare('UPDATE files SET stored_size = ?, chunks = ? WHERE id = ?')
+            .run(totalStoredSize, chunkIndex, fileId);
 
-    // Commit the transaction — all DB rows are now permanent
-    db.db.exec('COMMIT');
-
-    db.close();
+        // Clean up temp dir (only if empty)
+        try {
+            const remaining = fs.readdirSync(tempDir);
+            if (remaining.length === 0) fs.rmdirSync(tempDir);
+        } catch (e) {
+            // Ignore cleanup errors
+        }
 
-    // Clean up temp dir (only if empty)
-    try {
-        const remaining = fs.readdirSync(tempDir);
-        if (remaining.length === 0) fs.rmdirSync(tempDir);
-    } catch (e) {
-        // Ignore cleanup errors
+        return {
+            filename,
+            hash,
+            originalSize,
+            storedSize: totalStoredSize,
+            chunks: chunkIndex,
+            compressed
+        };
+    } finally {
+        // Always close DB connection, even on error
+        db.close();
     }
-
-    return {
-        filename,
-        hash,
-        originalSize,
-        storedSize: totalStoredSize,
-        chunks: chunkIndex,
-        compressed
-    };
 }
 
 /**
@@ -228,86 +216,32 @@ export async function retrieveFile(fileRecord, options) {
         throw new Error('No chunk metadata found for this file');
     }
 
-    // Prepare components
-    const encryptor = new Encryptor(password);
-    const decryptStream = encryptor.getDecryptStream();
-
-    const tempDir = process.env.TAS_TMP_DIR || path.join(dataDir, 'tmp');
-    if (!fs.existsSync(tempDir)) {
-        fs.mkdirSync(tempDir, { recursive: true });
-    }
-
     // Connect to Telegram
     const client = new TelegramClient(dataDir);
     await client.initialize(config.botToken);
     client.setChatId(config.chatId);
 
-    // Get total size from first chunk's header, or from DB
-    const firstChunkData = await client.downloadFile(chunks[0].file_telegram_id);
-    const header = parseHeader(firstChunkData);
-
-    // Total original uncompressed size
-    let expectedOriginalSize = header.originalSize;
-    let wasCompressed = header.compressed;
-
+    const encryptor = new Encryptor(password);
     const compressor = new Compressor();
-    const decompressStream = compressor.getDecompressStream(wasCompressed);
-
-    // We need a Readable stream that will lazily fetch chunks from Telegram
-    // and push them into the decryption pipeline.
-    const { Readable } = await import('stream');
-
-    const totalBytes = fileRecord.stored_size || chunks.reduce((acc, c) => acc + (c.size || 0), 0);
-    let downloadedBytes = 0;
-
-    // Pre-sort chunks by index so we download them in correct order
-    chunks.sort((a, b) => a.chunk_index - b.chunk_index);
 
-    let currentChunkIndex = 0;
-
-    // We already downloaded the first chunk to inspect its header, we shouldn't discard it.
-    let preloadedFirstChunk = firstChunkData;
-
-    const downloadStream = new Readable({
-        async read() {
-            try {
-                if (currentChunkIndex >= chunks.length) {
-                    this.push(null); // End of stream
-                    return;
-                }
+    const { createDownloadPipeline } = await import('./utils/download-stream.js');
 
-                const chunk = chunks[currentChunkIndex];
-                onProgress?.(`Downloading chunk ${chunk.chunk_index + 1}/${chunks.length}...`);
-
-                let data;
-                if (currentChunkIndex === 0 && preloadedFirstChunk) {
-                    data = preloadedFirstChunk;
-                    preloadedFirstChunk = null;
-                } else {
-                    data = await client.downloadFile(chunk.file_telegram_id);
-                }
-
-                downloadedBytes += data.length;
-                onByteProgress?.({ downloaded: downloadedBytes, total: totalBytes, chunk: chunk.chunk_index + 1, totalChunks: chunks.length });
-
-                // Strip header before pushing
-                const payload = data.subarray(HEADER_SIZE);
-                this.push(payload);
-
-                currentChunkIndex++;
-            } catch (err) {
-                this.destroy(err);
-            }
+    const { readable } = await createDownloadPipeline({
+        client,
+        chunks,
+        encryptor,
+        compressor,
+        onChunkDownloaded({ chunkIndex, totalChunks, bytesDownloaded, totalBytes }) {
+            onProgress?.(`Downloading chunk ${chunkIndex + 1}/${totalChunks}...`);
+            onByteProgress?.({ downloaded: bytesDownloaded, total: totalBytes, chunk: chunkIndex + 1, totalChunks });
         }
     });
 
     const writeStream = fs.createWriteStream(outputPath);
-    const { pipeline } = await import('stream/promises');
 
     onProgress?.('Decrypting, decompressing, and writing file...');
 
-    // Pipeline: Download from Telegram -> Decrypt -> Decompress -> Disk
-    await pipeline(downloadStream, decryptStream, decompressStream, writeStream);
+    await pipeline(readable, writeStream);
 
     const finalStats = fs.statSync(outputPath);
 

package/src/share/server.js

@@ -7,11 +7,12 @@
 import http from 'http';
 import crypto from 'crypto';
 import path from 'path';
+import { pipeline } from 'stream/promises';
 import { FileIndex } from '../db/index.js';
 import { TelegramClient } from '../telegram/client.js';
 import { Encryptor } from '../crypto/encryption.js';
 import { Compressor } from '../utils/compression.js';
-import { parseHeader, HEADER_SIZE } from '../utils/chunker.js';
+import { createDownloadPipeline } from '../utils/download-stream.js';
 
 /**
  * Generate a secure random share token
@@ -239,7 +240,7 @@ export class ShareServer {
         this.password = options.password;
         this.config = options.config;
         this.port = options.port || 3000;
-        this.host = options.host || '0.0.0.0';
+        this.host = options.host || '127.0.0.1';
 
         this.db = null;
         this.client = null;
@@ -261,64 +262,19 @@ export class ShareServer {
     }
 
     /**
-     * Prepare download streams from Telegram — fetches and verifies
-     * the first chunk before returning, so the caller can decide
-     * whether to commit to a 200 response.
-     *
-     * Returns: { downloadStream, decryptStream, decompressStream }
+     * Stream a decrypted file from Telegram directly to the HTTP response
      */
-    async prepareDownloadStreams(fileRecord) {
+    async streamToResponse(fileRecord, res) {
         const chunks = this.db.getChunks(fileRecord.id);
-        if (chunks.length === 0) throw new Error('No chunks found');
-
-        // Pre-sort chunks by index so we download them in correct order
-        chunks.sort((a, b) => a.chunk_index - b.chunk_index);
-
-        // Fetch and verify the first chunk eagerly
-        const firstChunkData = await this.client.downloadFile(chunks[0].file_telegram_id);
-        const header = parseHeader(firstChunkData);
-        let wasCompressed = header.compressed;
-
-        // Prepare streams
-        const decryptStream = this.encryptor.getDecryptStream();
-        const decompressStream = this.compressor.getDecompressStream(wasCompressed);
-
-        const { Readable } = await import('stream');
-
-        const self = this;
-        let currentChunkIndex = 0;
-        let preloadedFirstChunk = firstChunkData;
-
-        const downloadStream = new Readable({
-            async read() {
-                try {
-                    if (currentChunkIndex >= chunks.length) {
-                        this.push(null); // End of stream
-                        return;
-                    }
-
-                    const chunk = chunks[currentChunkIndex];
-                    let data;
-
-                    if (currentChunkIndex === 0 && preloadedFirstChunk) {
-                        data = preloadedFirstChunk;
-                        preloadedFirstChunk = null;
-                    } else {
-                        data = await self.client.downloadFile(chunk.file_telegram_id);
-                    }
-
-                    // Strip header before pushing
-                    const payload = data.subarray(HEADER_SIZE);
-                    this.push(payload);
-
-                    currentChunkIndex++;
-                } catch (err) {
-                    this.destroy(err);
-                }
-            }
+
+        const { readable } = await createDownloadPipeline({
+            client: this.client,
+            chunks,
+            encryptor: this.encryptor,
+            compressor: this.compressor
         });
 
-        return { downloadStream, decryptStream, decompressStream };
+        await pipeline(readable, res);
     }
 
     /**
@@ -401,31 +357,14 @@ export class ShareServer {
             const contentType = contentTypes[ext] || 'application/octet-stream';
             const safeName = sanitizeFilenameForHeader(fileRecord.filename);
 
-            try {
-                // Verify first chunk can be fetched and decrypted BEFORE sending 200 OK.
-                // This prevents sending a truncated/corrupt file on Telegram or decrypt failure.
-                const { downloadStream, decryptStream, decompressStream } = await this.prepareDownloadStreams(fileRecord);
-
-                // First chunk verified — commit to the 200 response
-                res.writeHead(200, {
-                    'Content-Type': contentType,
-                    'Content-Disposition': `attachment; filename="${safeName}"; filename*=UTF-8''${encodeURIComponent(fileRecord.filename)}`,
-                    'Transfer-Encoding': 'chunked'
-                });
+            res.writeHead(200, {
+                'Content-Type': contentType,
+                'Content-Disposition': `attachment; filename="${safeName}"; filename*=UTF-8''${encodeURIComponent(fileRecord.filename)}`,
+                'Transfer-Encoding': 'chunked'
+            });
 
-                const { pipeline } = await import('stream/promises');
-                await pipeline(downloadStream, decryptStream, decompressStream, res);
-            } catch (streamErr) {
-                console.error('Share stream error:', streamErr.message);
-                // Headers not yet sent — we can still return a proper error
-                if (!res.headersSent) {
-                    res.writeHead(500, { 'Content-Type': 'text/plain' });
-                    res.end('Download failed — file could not be decrypted or fetched.');
-                } else {
-                    // Headers already sent (shouldn't happen now, but safety net)
-                    res.end();
-                }
-            }
+            // Download the file from Telegram, decrypt, decompress and stream directly to 'res'
+            await this.streamToResponse(fileRecord, res);
 
         } catch (err) {
             console.error('Share server error:', err.message);