@nightowne/tas-cli 2.4.0 → 3.0.0

package/src/index.js

@@ -8,24 +8,45 @@ import path from 'path';
 import { pipeline } from 'stream/promises';
 import { Encryptor, hashFile } from './crypto/encryption.js';
 import { Compressor } from './utils/compression.js';
-import { createHeader, HEADER_SIZE } from './utils/chunker.js';
-import { TelegramClient } from './telegram/client.js';
+import { createHeader, HEADER_SIZE, MAX_CHUNK_SIZE } from './utils/chunker.js';
+import { TelegramPool } from './telegram/pool.js';
 import { FileIndex } from './db/index.js';
+import { normalizeLogicalPath } from './utils/logical-path.js';
+import { backupRemoteManifest } from './manifest.js';
 
-// Telegram has 50MB limit for bots, 2GB for user uploads
-// We'll use 49MB chunks to be safe
-const TELEGRAM_CHUNK_SIZE = 49 * 1024 * 1024;
+// The hosted Bot API can upload 50 MB but getFile downloads only up to 20 MB.
+// A 19 MiB encrypted payload plus the 64-byte TAS header is round-trip safe.
+export const TELEGRAM_CHUNK_SIZE = MAX_CHUNK_SIZE;
+
+export function createPublicChunkHeader(chunkIndex, totalChunks, flags) {
+    return createHeader('', 0, chunkIndex, totalChunks, flags);
+}
+
+export function createChunkCaption(uploadId, chunkIndex, totalChunks) {
+    return `tas:c1:${uploadId}:${chunkIndex + 1}/${totalChunks}`;
+}
 
 /**
  * Process and upload a file to Telegram
  */
 export async function processFile(filePath, options) {
-    const { password, dataDir, customName, config, onProgress, onByteProgress } = options;
+    const {
+        password,
+        dataDir,
+        customName,
+        config,
+        onProgress,
+        onByteProgress,
+        limitRate,
+        telegramPool,
+        replaceExisting = false,
+        updateManifest = true
+    } = options;
 
     onProgress?.('Reading file...');
 
     // Read file initially just to get size
-    const filename = customName || path.basename(filePath);
+    const filename = normalizeLogicalPath(customName || path.basename(filePath));
     const stats = fs.statSync(filePath);
     const originalSize = stats.size;
 
@@ -33,14 +54,21 @@ export async function processFile(filePath, options) {
     onProgress?.('Calculating hash...');
     const hash = await hashFile(filePath);
 
-    // Check if already uploaded
+    // Logical paths are exact. Identical bytes may legitimately exist under
+    // different paths, so content hash is indexed but no longer globally unique.
     const db = new FileIndex(path.join(dataDir, 'index.db'));
     db.init();
 
-    if (db.exists(hash)) {
+    const existingFile = db.findByExactName(filename);
+    if (existingFile && existingFile.hash === hash) {
         db.close();
-        throw new Error('File already uploaded (duplicate hash)');
+        throw new Error('This logical path already contains the same file');
     }
+    if (existingFile && !replaceExisting) {
+        db.close();
+        throw new Error(`A different file already exists at "${filename}"`);
+    }
+    const existingChunks = existingFile ? db.getChunks(existingFile.id) : [];
 
     // Prepare processing components
     const compressor = new Compressor();
@@ -50,144 +78,182 @@ export async function processFile(filePath, options) {
     const encryptor = new Encryptor(password);
     const encryptStream = encryptor.getEncryptStream();
 
-    const tempDir = process.env.TAS_TMP_DIR || path.join(dataDir, 'tmp');
-    if (!fs.existsSync(tempDir)) {
-        fs.mkdirSync(tempDir, { recursive: true });
-    }
+    const tempRoot = process.env.TAS_TMP_DIR || path.join(dataDir, 'tmp');
+    fs.mkdirSync(tempRoot, { recursive: true });
+    const uploadDir = fs.mkdtempSync(path.join(tempRoot, `${hash.substring(0, 12)}-`));
 
-    // Connect to Telegram
-    onProgress?.('Connecting to Telegram...');
-    const client = new TelegramClient(dataDir);
-    await client.initialize(config.botToken);
-    client.setChatId(config.chatId);
-
-    // We will stream through a custom Writable chunker
+    // First stream the exact encrypted chunks to disk. This makes upload
+    // resumption real: once network transfer starts, every remaining chunk is
+    // already durable locally and every completed Telegram ID is in SQLite.
     const { Writable } = await import('stream');
-
-    // First pass estimation (for calculating total chunks and progress)
-    // We don't know the exact final size due to compression and encryption overhead,
-    // so we'll estimate total chunks and update it if needed.
-    // For small files < 49MB we assume 1 chunk.
-    let estimatedSize = compressed ? originalSize : originalSize + 128; // Add encryption overhead
-    if (compressed && originalSize > 1024 * 1024) estimatedSize = originalSize * 0.8; // Rough guess
-    let estimatedChunks = Math.ceil(estimatedSize / TELEGRAM_CHUNK_SIZE) || 1;
-
-    // Register file in DB
-    const fileId = db.addFile({
-        filename,
-        hash,
-        originalSize,
-        storedSize: 0, // Will update later
-        chunks: estimatedChunks,
-        compressed
-    });
-
-    onProgress?.('Processing and uploading streams...');
-    let uploadedBytes = 0;
-    let chunkIndex = 0;
-
     let currentChunkBuffer = Buffer.alloc(0);
     let totalStoredSize = 0;
-
-    // Helper to upload a single chunk
-    const uploadCurrentChunk = async (isFinal = false) => {
-        if (currentChunkBuffer.length === 0 && !isFinal) return; // Nothing to upload
-        if (currentChunkBuffer.length === 0 && isFinal && chunkIndex > 0) return; // Empty final chunk after perfect split
-
-        // At this point we know if it's the final chunk, so we know the total chunks
-        const totalChunks = isFinal ? chunkIndex + 1 : Math.max(estimatedChunks, chunkIndex + 1);
-
-        const header = createHeader(filename, originalSize, chunkIndex, totalChunks, flags);
-        const chunkData = Buffer.concat([header, currentChunkBuffer]);
-
-        const chunkFilename = totalChunks > 1
-            ? `${hash.substring(0, 12)}.part${chunkIndex}.tas`
-            : `${hash.substring(0, 12)}.tas`;
-
-        const chunkPath = path.join(tempDir, chunkFilename);
-        fs.writeFileSync(chunkPath, chunkData);
-
-        const caption = totalChunks > 1
-            ? `📦 ${filename} (${chunkIndex + 1}/${totalChunks})`
-            : `📦 ${filename}`;
-
-        onProgress?.(`Uploading chunk ${chunkIndex + 1}...`);
-
-        const result = await client.sendFile(chunkPath, caption);
-
-        uploadedBytes += chunkData.length;
-        totalStoredSize += currentChunkBuffer.length;
-
-        onByteProgress?.({ uploaded: uploadedBytes, total: estimatedSize, chunk: chunkIndex + 1, totalChunks });
-
-        // Store file_id
-        db.addChunk(fileId, chunkIndex, result.messageId.toString(), chunkData.length);
-        db.db.prepare('UPDATE chunks SET file_telegram_id = ? WHERE file_id = ? AND chunk_index = ?')
-            .run(result.fileId, fileId, chunkIndex);
-
-        // Clean up temp file immediately to save disk space
-        fs.unlinkSync(chunkPath);
-
-        chunkIndex++;
-        currentChunkBuffer = Buffer.alloc(0);
+    const stagedChunks = [];
+
+    const stageChunk = (payload) => {
+        const index = stagedChunks.length;
+        const chunkPath = path.join(uploadDir, `chunk-${String(index).padStart(6, '0')}.tas`);
+        const header = createPublicChunkHeader(index, 0, flags);
+        fs.writeFileSync(chunkPath, Buffer.concat([header, payload]), { mode: 0o600 });
+        totalStoredSize += payload.length;
+        stagedChunks.push({ index, path: chunkPath, size: header.length + payload.length });
     };
 
     const chunkingStream = new Writable({
-        async write(chunk, encoding, callback) {
-            currentChunkBuffer = Buffer.concat([currentChunkBuffer, chunk]);
-
-            // If we exceeded the chunk limit, flush it
-            if (currentChunkBuffer.length >= TELEGRAM_CHUNK_SIZE) {
-                const overflow = currentChunkBuffer.subarray(TELEGRAM_CHUNK_SIZE);
-                currentChunkBuffer = currentChunkBuffer.subarray(0, TELEGRAM_CHUNK_SIZE);
-
-                try {
-                    await uploadCurrentChunk(false);
-                    currentChunkBuffer = overflow; // carry over
-                    callback();
-                } catch (err) {
-                    callback(err);
+        write(chunk, encoding, callback) {
+            try {
+                currentChunkBuffer = Buffer.concat([currentChunkBuffer, chunk]);
+                while (currentChunkBuffer.length >= TELEGRAM_CHUNK_SIZE) {
+                    stageChunk(currentChunkBuffer.subarray(0, TELEGRAM_CHUNK_SIZE));
+                    currentChunkBuffer = currentChunkBuffer.subarray(TELEGRAM_CHUNK_SIZE);
                 }
-            } else {
                 callback();
+            } catch (error) {
+                callback(error);
             }
         },
-        async final(callback) {
+        final(callback) {
             try {
-                await uploadCurrentChunk(true);
+                if (currentChunkBuffer.length > 0 || stagedChunks.length === 0) stageChunk(currentChunkBuffer);
                 callback();
-            } catch (err) {
-                callback(err);
+            } catch (error) {
+                callback(error);
             }
         }
     });
 
-    const readStream = fs.createReadStream(filePath);
+    onProgress?.('Compressing and encrypting to resumable chunks...');
+    try {
+        await pipeline(fs.createReadStream(filePath), compressStream, encryptStream, chunkingStream);
+    } catch (error) {
+        try { fs.rmSync(uploadDir, { recursive: true, force: true }); } catch { }
+        db.close();
+        throw new Error(`Local processing failed before upload: ${error.message}`);
+    }
 
-    // Run the pipeline: Read -> Compress -> Encrypt -> Chunk & Upload
-    await pipeline(readStream, compressStream, encryptStream, chunkingStream);
+    const totalChunks = stagedChunks.length;
+    if (totalChunks > 0xffff) {
+        try { fs.rmSync(uploadDir, { recursive: true, force: true }); } catch { }
+        db.close();
+        throw new Error(`File requires ${totalChunks} chunks, above the WAS1 limit of 65,535`);
+    }
+    for (const staged of stagedChunks) {
+        const fd = fs.openSync(staged.path, 'r+');
+        try {
+            fs.writeSync(fd, createPublicChunkHeader(staged.index, totalChunks, flags), 0, HEADER_SIZE, 0);
+        } finally {
+            fs.closeSync(fd);
+        }
+    }
 
-    // Update the DB with the final accurate values 
-    db.db.prepare('UPDATE files SET stored_size = ?, chunks = ? WHERE id = ?')
-        .run(totalStoredSize, chunkIndex, fileId);
+    let pendingId;
+    try {
+        pendingId = db.addPendingUpload({
+            filename,
+            filePath,
+            hash,
+            originalSize,
+            storedSize: totalStoredSize,
+            compressed,
+            totalChunks,
+            uploadedChunks: 0,
+            tempDir: uploadDir
+        });
+        for (const staged of stagedChunks) db.addPendingChunk(pendingId, staged.index, staged.path, staged.size);
+    } catch (error) {
+        try { fs.rmSync(uploadDir, { recursive: true, force: true }); } catch { }
+        db.close();
+        throw new Error(`Could not persist resumable upload state: ${error.message}`);
+    }
 
-    db.close();
+    onProgress?.('Connecting to Telegram...');
+    const client = telegramPool || new TelegramPool(dataDir, config.bots);
+    if (!telegramPool) await client.initialize({ includeDisabled: false });
 
-    // Clean up temp dir (only if empty)
+    let uploadedBytes = 0;
     try {
-        const remaining = fs.readdirSync(tempDir);
-        if (remaining.length === 0) fs.rmdirSync(tempDir);
-    } catch (e) {
-        // Ignore cleanup errors
+        for (const staged of stagedChunks) {
+            onProgress?.(`Uploading chunk ${staged.index + 1}/${totalChunks}...`);
+            const botId = client.selectBotId(hash, staged.index);
+            const result = await client.sendFile(
+                staged.path,
+                createChunkCaption(pendingId, staged.index, totalChunks),
+                {
+                    ...(limitRate ? { limitRate } : {}),
+                    botId,
+                    routingKey: hash,
+                    chunkIndex: staged.index
+                }
+            );
+            db.markChunkUploaded(
+                pendingId,
+                staged.index,
+                String(result.messageId),
+                result.fileId,
+                result.botId
+            );
+            uploadedBytes += staged.size;
+            onByteProgress?.({
+                uploaded: uploadedBytes,
+                total: stagedChunks.reduce((sum, chunk) => sum + chunk.size, 0),
+                chunk: staged.index + 1,
+                totalChunks
+            });
+            fs.unlinkSync(staged.path);
+        }
+    } catch (error) {
+        db.close();
+        throw new Error(`${error.message} (upload paused — run \`tas resume\` to continue)`);
+    }
+
+    const uploadedChunks = db.getPendingChunks(pendingId);
+    let fileId;
+    db.db.transaction(() => {
+        fileId = db.addFile({ filename, hash, originalSize, storedSize: totalStoredSize, chunks: totalChunks, compressed });
+        for (const chunk of uploadedChunks) {
+            db.addChunk(
+                fileId,
+                chunk.chunk_index,
+                chunk.message_id,
+                chunk.size,
+                chunk.file_telegram_id,
+                chunk.bot_id || null
+            );
+        }
+        if (existingFile) db.deleteFileCascade(existingFile.id);
+        db.deletePendingUpload(pendingId);
+    })();
+
+    db.close();
+
+    let manifestWarning = null;
+    if (updateManifest) {
+        onProgress?.('Publishing encrypted recovery manifest...');
+        try {
+            await backupRemoteManifest({ dataDir, password, config, telegramPool: client });
+            for (const chunk of existingChunks) {
+                try { await client.deleteMessage(chunk.message_id, chunk.bot_id || null); } catch { }
+            }
+        } catch (error) {
+            manifestWarning = error.message;
+        }
     }
 
+    // Clean up this upload's private staging directory.
+    try {
+        fs.rmdirSync(uploadDir);
+        if (fs.readdirSync(tempRoot).length === 0) fs.rmdirSync(tempRoot);
+    } catch { }
+
     return {
         filename,
         hash,
         originalSize,
         storedSize: totalStoredSize,
-        chunks: chunkIndex,
-        compressed
+        chunks: totalChunks,
+        compressed,
+        manifestWarning,
+        supersededChunks: existingChunks
     };
 }
 
@@ -195,7 +261,7 @@ export async function processFile(filePath, options) {
  * Retrieve a file from Telegram
  */
 export async function retrieveFile(fileRecord, options) {
-    const { password, dataDir, outputPath, config, onProgress, onByteProgress } = options;
+    const { password, dataDir, outputPath, config, onProgress, onByteProgress, telegramPool } = options;
 
     onProgress?.('Connecting to Telegram...');
 
@@ -211,9 +277,7 @@ export async function retrieveFile(fileRecord, options) {
     }
 
     // Connect to Telegram
-    const client = new TelegramClient(dataDir);
-    await client.initialize(config.botToken);
-    client.setChatId(config.chatId);
+    const client = telegramPool || new TelegramPool(dataDir, config.bots);
 
     const encryptor = new Encryptor(password);
     const compressor = new Compressor();

package/src/manifest.js

@@ -0,0 +1,104 @@
+/**
+ * Encrypted remote index manifest.
+ *
+ * Telegram file IDs are not discoverable from ciphertext alone. TAS therefore
+ * keeps a compact encrypted snapshot of files/chunks/tags and stores
+ * the latest manifest pointer in config.json. Losing index.db is recoverable as
+ * long as config.json, the password, and the manifest message still exist.
+ */
+
+import fs from 'fs';
+import path from 'path';
+import os from 'os';
+import zlib from 'zlib';
+import { FileIndex } from './db/index.js';
+import { Encryptor } from './crypto/encryption.js';
+import { TelegramPool } from './telegram/pool.js';
+import { loadConfig, saveConfig } from './utils/cli-helpers.js';
+import { MAX_CHUNK_SIZE } from './utils/chunker.js';
+
+const queues = new Map();
+
+async function writeRemoteManifest({ dataDir, password, config, telegramPool }) {
+    const db = new FileIndex(path.join(dataDir, 'index.db'));
+    db.init();
+    const snapshot = db.exportManifest();
+    db.close();
+
+    const compressed = zlib.gzipSync(Buffer.from(JSON.stringify(snapshot)), { level: 9 });
+    const encrypted = new Encryptor(password).encrypt(compressed);
+    if (encrypted.length > MAX_CHUNK_SIZE) {
+        throw new Error(
+            `Encrypted index manifest is ${encrypted.length} bytes, above the single-message recovery limit. ` +
+            'Refusing to publish an incomplete recovery point.'
+        );
+    }
+
+    const pool = telegramPool || new TelegramPool(dataDir, config.bots);
+    const routingKey = `manifest:${snapshot.createdAt}`;
+    const botId = pool.selectBotId(routingKey, 0);
+    const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'tas-manifest-'));
+    const manifestPath = path.join(tempDir, 'tas-index.m1');
+    fs.writeFileSync(manifestPath, encrypted, { mode: 0o600 });
+
+    let result;
+    try {
+        result = await pool.sendFile(manifestPath, 'tas:m1', { botId, routingKey, chunkIndex: 0 });
+    } finally {
+        try { fs.rmSync(tempDir, { recursive: true, force: true }); } catch { }
+    }
+
+    const rawConfig = loadConfig(dataDir);
+    if (!rawConfig) throw new Error('Config disappeared while publishing remote manifest');
+    const previous = rawConfig.remoteManifest;
+    rawConfig.remoteManifest = {
+        version: 1,
+        botId: result.botId,
+        messageId: String(result.messageId),
+        fileId: result.fileId,
+        createdAt: snapshot.createdAt,
+        files: snapshot.files.length,
+        chunks: snapshot.chunks.length
+    };
+    saveConfig(dataDir, rawConfig);
+
+    // The new pointer is durable locally before the prior recovery point is
+    // removed. Failure to remove merely leaves an encrypted orphan manifest.
+    if (previous?.messageId) {
+        try { await pool.deleteMessage(previous.messageId, previous.botId || null); } catch { }
+    }
+    return rawConfig.remoteManifest;
+}
+
+/** Serialize manifest writes inside one TAS process (notably sync workers). */
+export function backupRemoteManifest(options) {
+    const key = path.resolve(options.dataDir);
+    const previous = queues.get(key) || Promise.resolve();
+    const next = previous.catch(() => { }).then(() => writeRemoteManifest(options));
+    queues.set(key, next);
+    return next.finally(() => {
+        if (queues.get(key) === next) queues.delete(key);
+    });
+}
+
+export async function downloadRemoteManifest({ dataDir, password, config, telegramPool }) {
+    const rawConfig = loadConfig(dataDir);
+    const pointer = rawConfig?.remoteManifest;
+    if (!pointer?.fileId) {
+        throw new Error('No remote manifest pointer exists in config.json');
+    }
+
+    const pool = telegramPool || new TelegramPool(dataDir, config.bots);
+    const encrypted = await pool.downloadFile(pointer.fileId, pointer.botId || null);
+    let manifest;
+    try {
+        const compressed = new Encryptor(password).decrypt(encrypted);
+        manifest = JSON.parse(zlib.gunzipSync(compressed).toString('utf8'));
+    } catch (error) {
+        throw new Error(`Remote manifest authentication/decode failed: ${error.message}`);
+    }
+    if (manifest.schemaVersion !== 1 || !Array.isArray(manifest.files) || !Array.isArray(manifest.chunks)) {
+        throw new Error('Remote manifest is malformed or unsupported');
+    }
+    return manifest;
+}

package/src/share/server.js

@@ -9,7 +9,7 @@ import crypto from 'crypto';
 import path from 'path';
 import { pipeline } from 'stream/promises';
 import { FileIndex } from '../db/index.js';
-import { TelegramClient } from '../telegram/client.js';
+import { TelegramPool } from '../telegram/pool.js';
 import { Encryptor } from '../crypto/encryption.js';
 import { Compressor } from '../utils/compression.js';
 import { createDownloadPipeline } from '../utils/download-stream.js';
@@ -253,9 +253,7 @@ export class ShareServer {
         this.db = new FileIndex(path.join(this.dataDir, 'index.db'));
         this.db.init();
 
-        this.client = new TelegramClient(this.dataDir);
-        await this.client.initialize(this.config.botToken);
-        this.client.setChatId(this.config.chatId);
+        this.client = new TelegramPool(this.dataDir, this.config.bots);
 
         this.encryptor = new Encryptor(this.password);
         this.compressor = new Compressor();
@@ -337,9 +335,6 @@ export class ShareServer {
                 return;
             }
 
-            // Increment download count
-            this.db.incrementShareDownload(token);
-
             // Determine content type
             const ext = path.extname(fileRecord.filename).toLowerCase();
             const contentTypes = {
@@ -360,11 +355,14 @@ export class ShareServer {
             res.writeHead(200, {
                 'Content-Type': contentType,
                 'Content-Disposition': `attachment; filename="${safeName}"; filename*=UTF-8''${encodeURIComponent(fileRecord.filename)}`,
-                'Transfer-Encoding': 'chunked'
+                'Content-Length': fileRecord.original_size
             });
 
-            // Download the file from Telegram, decrypt, decompress and stream directly to 'res'
+            // Download the file from Telegram, decrypt, decompress and stream directly to 'res'.
+            // Count the download only AFTER a successful stream so an aborted
+            // connection doesn't burn a single-use link.
             await this.streamToResponse(fileRecord, res);
+            this.db.incrementShareDownload(token);
 
         } catch (err) {
             console.error('Share server error:', err.message);