@nightowne/tas-cli 2.0.0 → 2.3.0

package/src/fuse/mount.js

@@ -5,21 +5,38 @@
  * This is the killer feature - use Telegram like a regular folder!
  */
 
-import Fuse from 'fuse-native';
 import path from 'path';
 import fs from 'fs';
+import crypto from 'crypto';
+import { processFile } from '../index.js';
+
+let Fuse;
+try {
+    Fuse = (await import('fuse-native')).default;
+} catch {
+    // fuse-native is optional — unavailable on ARM64 or systems without libfuse
+}
 import { TelegramClient } from '../telegram/client.js';
 import { Encryptor } from '../crypto/encryption.js';
 import { Compressor } from '../utils/compression.js';
 import { FileIndex } from '../db/index.js';
 import { createHeader, parseHeader, HEADER_SIZE } from '../utils/chunker.js';
 
-// File cache for performance (avoid re-downloading)
-const fileCache = new Map();
 const CACHE_TTL = 5 * 60 * 1000; // 5 minutes
+const CACHE_MAX_ENTRIES = 100; // Prevent unbounded memory growth
 
 export class TelegramFS {
     constructor(options) {
+        if (!Fuse) {
+            throw new Error(
+                'fuse-native is not available on this system.\n' +
+                '  On Linux x86_64: npm install fuse-native && sudo apt install fuse libfuse-dev\n' +
+                '  On macOS: brew install macfuse && npm install fuse-native\n' +
+                '  On ARM64: see https://github.com/ixchio/tas/issues/1 for a workaround\n' +
+                '  All other TAS commands (push, pull, sync, share) work without FUSE.'
+            );
+        }
+
         this.dataDir = options.dataDir;
         this.password = options.password;
         this.config = options.config;
@@ -35,6 +52,9 @@ export class TelegramFS {
 
         // Pending writes buffer
         this.writeBuffers = new Map();
+        
+        // Instance-level file cache to prevent cross-talk
+        this.fileCache = new Map();
     }
 
     async initialize() {
@@ -70,7 +90,7 @@ export class TelegramFS {
                 mtime: new Date(),
                 atime: new Date(),
                 ctime: new Date(),
-                size: wb.data.length,
+                size: wb.size,
                 mode: 0o100644, // regular file
                 uid: process.getuid?.() || 0,
                 gid: process.getgid?.() || 0
@@ -104,9 +124,14 @@ export class TelegramFS {
         }
 
         const files = this.db.listAll();
-        const names = files.map(f => f.filename);
+        const nameSet = new Set(files.map(f => f.filename));
+
+        // Include files currently being written (not yet released/uploaded)
+        for (const name of this.writeBuffers.keys()) {
+            nameSet.add(name);
+        }
 
-        return cb(0, names);
+        return cb(0, [...nameSet]);
     }
 
     /**
@@ -130,26 +155,34 @@ export class TelegramFS {
     }
 
     /**
-     * Read file contents
+     * Read file contents from disk cache
      */
     async read(filepath, fd, buffer, length, position, cb) {
         const filename = path.basename(filepath);
 
         try {
+            // Check write buffers first
+            const wb = this.writeBuffers.get(filename);
+            if (wb) {
+                const bytesRead = fs.readSync(wb.fd, buffer, 0, length, position);
+                return cb(bytesRead);
+            }
+
             // Check cache first
-            let data = this.getCached(filename);
+            let cachedPath = this.getCached(filename);
 
-            if (!data) {
-                // Download and decrypt
-                data = await this.downloadFile(filename);
-                this.setCache(filename, data);
+            if (!cachedPath) {
+                // Download, decrypt, and save to disk cache
+                cachedPath = await this.downloadFileToCache(filename);
+                this.setCache(filename, cachedPath);
             }
 
-            // Copy requested portion to buffer
-            const slice = data.subarray(position, position + length);
-            slice.copy(buffer);
+            // Copy requested portion to buffer from disk
+            const fdDisk = fs.openSync(cachedPath, 'r');
+            const bytesRead = fs.readSync(fdDisk, buffer, 0, length, position);
+            fs.closeSync(fdDisk);
 
-            return cb(slice.length);
+            return cb(bytesRead);
         } catch (err) {
             console.error('Read error:', err.message);
             return cb(Fuse.EIO);
@@ -157,34 +190,55 @@ export class TelegramFS {
     }
 
     /**
-     * Write to a file (buffers until release)
+     * Helper to get or create a disk-buffered write structure
      */
-    write(filepath, fd, buffer, length, position, cb) {
-        const filename = path.basename(filepath);
+    getOrCreateWriteBuffer(filename) {
+        if (this.writeBuffers.has(filename)) {
+            return this.writeBuffers.get(filename);
+        }
 
-        // Get or create write buffer
-        if (!this.writeBuffers.has(filename)) {
-            this.writeBuffers.set(filename, {
-                data: Buffer.alloc(0),
-                modified: true
-            });
+        const fuseTmpDir = path.join(this.dataDir, 'fuse-tmp');
+        if (!fs.existsSync(fuseTmpDir)) {
+            fs.mkdirSync(fuseTmpDir, { recursive: true });
         }
 
-        const wb = this.writeBuffers.get(filename);
+        const tmpPath = path.join(fuseTmpDir, crypto.randomBytes(16).toString('hex'));
+        const fdDisk = fs.openSync(tmpPath, 'w+');
 
-        // Expand buffer if needed
-        const newSize = Math.max(wb.data.length, position + length);
-        if (newSize > wb.data.length) {
-            const newBuf = Buffer.alloc(newSize);
-            wb.data.copy(newBuf);
-            wb.data = newBuf;
+        // Check if there is an existing file cached
+        const cachedPath = this.getCached(filename);
+        let size = 0;
+        if (cachedPath && fs.existsSync(cachedPath)) {
+            const content = fs.readFileSync(cachedPath);
+            fs.writeSync(fdDisk, content, 0, content.length, 0);
+            size = content.length;
         }
 
-        // Copy incoming data
-        buffer.copy(wb.data, position, 0, length);
-        wb.modified = true;
+        const wb = {
+            tmpPath,
+            fd: fdDisk,
+            size,
+            modified: true
+        };
+        this.writeBuffers.set(filename, wb);
+        return wb;
+    }
 
-        return cb(length);
+    /**
+     * Write to a file (buffers until release)
+     */
+    write(filepath, fd, buffer, length, position, cb) {
+        const filename = path.basename(filepath);
+        try {
+            const wb = this.getOrCreateWriteBuffer(filename);
+            fs.writeSync(wb.fd, buffer, 0, length, position);
+            wb.size = Math.max(wb.size, position + length);
+            wb.modified = true;
+            return cb(length);
+        } catch (err) {
+            console.error('[FUSE] Write error:', err.message);
+            return cb(Fuse.EIO);
+        }
     }
 
     /**
@@ -195,9 +249,19 @@ export class TelegramFS {
 
         console.log(`[FUSE] Creating file: ${filename}`);
 
-        // Initialize empty write buffer
+        // Initialize empty disk-buffered write
+        const fuseTmpDir = path.join(this.dataDir, 'fuse-tmp');
+        if (!fs.existsSync(fuseTmpDir)) {
+            fs.mkdirSync(fuseTmpDir, { recursive: true });
+        }
+
+        const tmpPath = path.join(fuseTmpDir, crypto.randomBytes(16).toString('hex'));
+        const fdDisk = fs.openSync(tmpPath, 'w+');
+
         this.writeBuffers.set(filename, {
-            data: Buffer.alloc(0),
+            tmpPath,
+            fd: fdDisk,
+            size: 0,
             modified: true,
             isNew: true
         });
@@ -219,15 +283,26 @@ export class TelegramFS {
         const filename = path.basename(filepath);
 
         const wb = this.writeBuffers.get(filename);
-        if (!wb || !wb.modified) {
+        if (!wb) {
+            return cb(0);
+        }
+
+        if (!wb.modified) {
+            try { fs.closeSync(wb.fd); } catch (e) {}
+            try { fs.unlinkSync(wb.tmpPath); } catch (e) {}
+            this.writeBuffers.delete(filename);
             return cb(0);
         }
 
         try {
-            // Upload to Telegram
-            await this.uploadFile(filename, wb.data);
+            // Close fd so we can read and upload cleanly
+            try { fs.closeSync(wb.fd); } catch (e) {}
+
+            // Upload to Telegram from the temp path
+            await this.uploadFile(filename, wb.tmpPath);
 
-            // Clear write buffer
+            // Clean up temp file
+            try { fs.unlinkSync(wb.tmpPath); } catch (e) {}
             this.writeBuffers.delete(filename);
 
             // Invalidate cache
@@ -235,7 +310,9 @@ export class TelegramFS {
 
             return cb(0);
         } catch (err) {
-            console.error('Release error:', err.message);
+            console.error('[FUSE] Release error:', err.message);
+            try { fs.unlinkSync(wb.tmpPath); } catch (e) {}
+            this.writeBuffers.delete(filename);
             return cb(Fuse.EIO);
         }
     }
@@ -252,7 +329,7 @@ export class TelegramFS {
         }
 
         try {
-            // Delete from Telegram (optional - could just remove from index)
+            // Delete from Telegram
             const chunks = this.db.getChunks(file.id);
             for (const chunk of chunks) {
                 await this.client.deleteMessage(chunk.message_id);
@@ -266,7 +343,7 @@ export class TelegramFS {
 
             return cb(0);
         } catch (err) {
-            console.error('Unlink error:', err.message);
+            console.error('[FUSE] Unlink error:', err.message);
             return cb(Fuse.EIO);
         }
     }
@@ -288,10 +365,10 @@ export class TelegramFS {
             .run(newName, file.id);
 
         // Update cache key
-        const cached = fileCache.get(oldName);
+        const cached = this.fileCache.get(oldName);
         if (cached) {
-            fileCache.delete(oldName);
-            fileCache.set(newName, cached);
+            this.fileCache.delete(oldName);
+            this.fileCache.set(newName, cached);
         }
 
         return cb(0);
@@ -302,77 +379,98 @@ export class TelegramFS {
      */
     truncate(filepath, size, cb) {
         const filename = path.basename(filepath);
-
-        // Get or load into write buffer
-        if (!this.writeBuffers.has(filename)) {
-            const cached = this.getCached(filename);
-            if (cached) {
-                this.writeBuffers.set(filename, {
-                    data: Buffer.from(cached),
-                    modified: true
-                });
-            } else {
-                this.writeBuffers.set(filename, {
-                    data: Buffer.alloc(0),
-                    modified: true
-                });
-            }
-        }
-
-        const wb = this.writeBuffers.get(filename);
-
-        if (size < wb.data.length) {
-            wb.data = wb.data.subarray(0, size);
-        } else if (size > wb.data.length) {
-            const newBuf = Buffer.alloc(size);
-            wb.data.copy(newBuf);
-            wb.data = newBuf;
+        try {
+            const wb = this.getOrCreateWriteBuffer(filename);
+            fs.ftruncateSync(wb.fd, size);
+            wb.size = size;
+            wb.modified = true;
+            return cb(0);
+        } catch (err) {
+            console.error('[FUSE] Truncate error:', err.message);
+            return cb(Fuse.EIO);
         }
-
-        wb.modified = true;
-        return cb(0);
     }
 
     // ============== Helper Methods ==============
 
-    async downloadFile(filename) {
+    async downloadFileToCache(filename) {
         const file = this.db.findByName(filename);
         if (!file) throw new Error('File not found');
 
+        const cacheDir = path.join(this.dataDir, 'cache');
+        if (!fs.existsSync(cacheDir)) {
+            fs.mkdirSync(cacheDir, { recursive: true });
+        }
+
+        const outputPath = path.join(cacheDir, file.hash);
+
+        // If it's already fully downloaded and cached on disk, return path
+        if (fs.existsSync(outputPath)) {
+            const stats = fs.statSync(outputPath);
+            if (stats.size === file.original_size) {
+                return outputPath;
+            }
+        }
+
         const chunks = this.db.getChunks(file.id);
         if (chunks.length === 0) throw new Error('No chunks found');
 
-        // Download all chunks
-        const downloadedChunks = [];
-
-        for (const chunk of chunks) {
-            const data = await this.client.downloadFile(chunk.file_telegram_id);
-            const header = parseHeader(data);
-            const payload = data.subarray(HEADER_SIZE);
+        // Pre-sort chunks
+        chunks.sort((a, b) => a.chunk_index - b.chunk_index);
+
+        const firstChunkData = await this.client.downloadFile(chunks[0].file_telegram_id);
+        const header = parseHeader(firstChunkData);
+        let wasCompressed = header.compressed;
+
+        const decryptStream = this.encryptor.getDecryptStream();
+        const decompressStream = this.compressor.getDecompressStream(wasCompressed);
+
+        const { Readable } = await import('stream');
+        const { pipeline } = await import('stream/promises');
+
+        const self = this;
+        let currentChunkIndex = 0;
+        let preloadedFirstChunk = firstChunkData;
+
+        const downloadStream = new Readable({
+            async read() {
+                try {
+                    if (currentChunkIndex >= chunks.length) {
+                        this.push(null);
+                        return;
+                    }
+
+                    const chunk = chunks[currentChunkIndex];
+                    let data;
+                    if (currentChunkIndex === 0 && preloadedFirstChunk) {
+                        data = preloadedFirstChunk;
+                        preloadedFirstChunk = null;
+                    } else {
+                        data = await self.client.downloadFile(chunk.file_telegram_id);
+                    }
+
+                    const payload = data.subarray(HEADER_SIZE);
+                    this.push(payload);
+                    currentChunkIndex++;
+                } catch (err) {
+                    this.destroy(err);
+                }
+            }
+        });
 
-            downloadedChunks.push({
-                index: header.chunkIndex,
-                data: payload,
-                compressed: header.compressed
-            });
-        }
+        const tmpOutputPath = outputPath + '.tmp';
+        const writeStream = fs.createWriteStream(tmpOutputPath);
 
-        // Reassemble
-        downloadedChunks.sort((a, b) => a.index - b.index);
-        const encryptedData = Buffer.concat(downloadedChunks.map(c => c.data));
+        // Pipeline: Telegram -> Decrypt -> Decompress -> Disk Cache
+        await pipeline(downloadStream, decryptStream, decompressStream, writeStream);
 
-        // Decrypt
-        const compressedData = this.encryptor.decrypt(encryptedData);
+        // Rename to final atomic path
+        fs.renameSync(tmpOutputPath, outputPath);
 
-        // Decompress
-        const wasCompressed = downloadedChunks[0].compressed;
-        return await this.compressor.decompress(compressedData, wasCompressed);
+        return outputPath;
     }
 
-    async uploadFile(filename, data) {
-        const { hashData } = await import('../crypto/encryption.js');
-        const hash = hashData(data);
-
+    async uploadFile(filename, tempPath) {
         // Check if already exists by name
         const existingByName = this.db.findByName(filename);
         if (existingByName) {
@@ -384,76 +482,65 @@ export class TelegramFS {
             this.db.delete(existingByName.id);
         }
 
-        // Check if already exists by hash (same content, different name)
-        const existingByHash = this.db.findByHash(hash);
-        if (existingByHash) {
-            // Same content already exists, just skip
-            console.log(`[FUSE] File with same content already exists as ${existingByHash.filename}`);
-            return;
-        }
-
-        // Compress
-        const { data: compressedData, compressed } = await this.compressor.compress(data, filename);
-
-        // Encrypt
-        const encryptedData = this.encryptor.encrypt(compressedData);
-
-        // Create temp file with header
-        const tempDir = path.join(this.dataDir, 'tmp');
-        if (!fs.existsSync(tempDir)) {
-            fs.mkdirSync(tempDir, { recursive: true });
-        }
-
-        const flags = compressed ? 1 : 0;
-        const header = createHeader(filename, data.length, 0, 1, flags);
-        const fileData = Buffer.concat([header, encryptedData]);
-
-        const tempPath = path.join(tempDir, `${hash.substring(0, 12)}.tas`);
-        fs.writeFileSync(tempPath, fileData);
-
-        // Upload to Telegram
-        const result = await this.client.sendFile(tempPath, `📦 ${filename}`);
-
-        // Add to index
-        const fileId = this.db.addFile({
-            filename,
-            hash,
-            originalSize: data.length,
-            storedSize: encryptedData.length,
-            chunks: 1,
-            compressed
+        // Run the standard chunked upload pipeline (this handles chunking, encryption, compression, and DB entry)
+        await processFile(tempPath, {
+            password: this.password,
+            dataDir: this.dataDir,
+            customName: filename,
+            config: this.config
         });
-
-        this.db.addChunk(fileId, 0, result.messageId.toString(), fileData.length);
-        this.db.db.prepare('UPDATE chunks SET file_telegram_id = ? WHERE file_id = ? AND chunk_index = ?')
-            .run(result.fileId, fileId, 0);
-
-        // Cleanup
-        fs.unlinkSync(tempPath);
-        try { fs.rmdirSync(tempDir); } catch (e) { }
     }
 
     getCached(filename) {
-        const entry = fileCache.get(filename);
+        const entry = this.fileCache.get(filename);
         if (!entry) return null;
 
         if (Date.now() - entry.timestamp > CACHE_TTL) {
-            fileCache.delete(filename);
+            // Expired, delete the file if possible
+            try {
+                if (fs.existsSync(entry.path)) fs.unlinkSync(entry.path);
+            } catch (e) { }
+            this.fileCache.delete(filename);
             return null;
         }
 
-        return entry.data;
+        // Extend cache TTL on read
+        entry.timestamp = Date.now();
+        return entry.path;
     }
 
-    setCache(filename, data) {
-        fileCache.set(filename, {
-            data,
+    setCache(filename, cachePath) {
+        // Evict oldest entry if cache is full
+        if (this.fileCache.size >= CACHE_MAX_ENTRIES) {
+            let oldestKey = null;
+            let oldestTime = Infinity;
+            for (const [key, entry] of this.fileCache) {
+                if (entry.timestamp < oldestTime) {
+                    oldestTime = entry.timestamp;
+                    oldestKey = key;
+                }
+            }
+            if (oldestKey) {
+                const evicted = this.fileCache.get(oldestKey);
+                try { if (fs.existsSync(evicted.path)) fs.unlinkSync(evicted.path); } catch (e) { }
+                this.fileCache.delete(oldestKey);
+            }
+        }
+
+        this.fileCache.set(filename, {
+            path: cachePath,
             timestamp: Date.now()
         });
     }
 
     invalidateCache(filename) {
-        fileCache.delete(filename);
+        const entry = this.fileCache.get(filename);
+        if (entry) {
+            try {
+                if (fs.existsSync(entry.path)) fs.unlinkSync(entry.path);
+            } catch (e) { }
+            this.fileCache.delete(filename);
+        }
     }
 
     /**
@@ -465,6 +552,17 @@ export class TelegramFS {
             fs.mkdirSync(this.mountPoint, { recursive: true });
         }
 
+        // Clean up any stray temp files from previous sessions
+        const fuseTmpDir = path.join(this.dataDir, 'fuse-tmp');
+        if (fs.existsSync(fuseTmpDir)) {
+            try {
+                const files = fs.readdirSync(fuseTmpDir);
+                for (const file of files) {
+                    fs.unlinkSync(path.join(fuseTmpDir, file));
+                }
+            } catch (e) {}
+        }
+
         const ops = {
             getattr: this.getattr.bind(this),
             readdir: this.readdir.bind(this),