@nightowne/tas-cli 2.4.1 → 3.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +154 -0
- package/FAQ.md +33 -0
- package/QUICKSTART.md +47 -0
- package/README.md +94 -93
- package/package.json +10 -4
- package/src/cli.js +757 -212
- package/src/db/index.js +307 -24
- package/src/fuse/mount.js +336 -175
- package/src/index.js +208 -150
- package/src/manifest.js +104 -0
- package/src/share/server.js +7 -9
- package/src/sync/sync.js +105 -34
- package/src/telegram/client.js +13 -12
- package/src/telegram/pool.js +105 -0
- package/src/utils/branding.js +2 -2
- package/src/utils/chunker.js +4 -3
- package/src/utils/cli-helpers.js +90 -10
- package/src/utils/download-stream.js +10 -4
- package/src/utils/logical-path.js +44 -0
package/src/sync/sync.js
CHANGED
|
@@ -9,18 +9,24 @@ import { EventEmitter } from 'events';
|
|
|
9
9
|
import { FileIndex } from '../db/index.js';
|
|
10
10
|
import { hashFile } from '../crypto/encryption.js';
|
|
11
11
|
import { processFile } from '../index.js';
|
|
12
|
+
import { TelegramPool } from '../telegram/pool.js';
|
|
13
|
+
import { backupRemoteManifest } from '../manifest.js';
|
|
12
14
|
|
|
13
15
|
// Debounce time in ms to batch rapid file changes
|
|
14
16
|
const DEBOUNCE_MS = 1000;
|
|
15
17
|
|
|
16
|
-
// Ignore patterns
|
|
18
|
+
// Ignore patterns.
|
|
19
|
+
// NOTE: dotfiles are intentionally NOT ignored — TAS is advertised as a
|
|
20
|
+
// vault for `.env` files, SSH keys, etc. Only well-known junk is skipped.
|
|
17
21
|
const IGNORE_PATTERNS = [
|
|
18
|
-
|
|
22
|
+
/^\.DS_Store$/, // macOS metadata
|
|
23
|
+
/^\.git$/, // git dir name
|
|
24
|
+
/\.git[\/\\]/, // anything inside .git
|
|
19
25
|
/~$/, // Backup files
|
|
20
26
|
/\.swp$/, // Vim swap files
|
|
21
27
|
/\.tmp$/, // Temp files
|
|
22
|
-
/node_modules/,
|
|
23
|
-
|
|
28
|
+
/(^|[\/\\])node_modules([\/\\]|$)/,
|
|
29
|
+
/(^|[\/\\])\.tas([\/\\]|$)/ // our own data dir if nested
|
|
24
30
|
];
|
|
25
31
|
|
|
26
32
|
export class SyncEngine extends EventEmitter {
|
|
@@ -33,6 +39,7 @@ export class SyncEngine extends EventEmitter {
|
|
|
33
39
|
this.watchers = new Map(); // path -> FSWatcher
|
|
34
40
|
this.pendingChanges = new Map(); // path -> timeout
|
|
35
41
|
this.db = null;
|
|
42
|
+
this.telegramPool = null;
|
|
36
43
|
this.running = false;
|
|
37
44
|
}
|
|
38
45
|
|
|
@@ -42,6 +49,8 @@ export class SyncEngine extends EventEmitter {
|
|
|
42
49
|
async initialize() {
|
|
43
50
|
this.db = new FileIndex(path.join(this.dataDir, 'index.db'));
|
|
44
51
|
this.db.init();
|
|
52
|
+
this.telegramPool = new TelegramPool(this.dataDir, this.config.bots);
|
|
53
|
+
await this.telegramPool.initialize({ includeDisabled: false });
|
|
45
54
|
}
|
|
46
55
|
|
|
47
56
|
/**
|
|
@@ -98,6 +107,7 @@ export class SyncEngine extends EventEmitter {
|
|
|
98
107
|
|
|
99
108
|
let uploaded = 0;
|
|
100
109
|
let skipped = 0;
|
|
110
|
+
const supersededChunks = [];
|
|
101
111
|
|
|
102
112
|
// Process files with concurrency limit
|
|
103
113
|
const CONCURRENCY = 4;
|
|
@@ -129,14 +139,18 @@ export class SyncEngine extends EventEmitter {
|
|
|
129
139
|
try {
|
|
130
140
|
this.emit('file-upload-start', { file: file.relativePath });
|
|
131
141
|
|
|
132
|
-
await processFile(file.path, {
|
|
142
|
+
const result = await processFile(file.path, {
|
|
133
143
|
password: this.password,
|
|
134
144
|
dataDir: this.dataDir,
|
|
135
145
|
customName: file.relativePath, // Use relative path as name
|
|
136
146
|
config: this.config,
|
|
147
|
+
telegramPool: this.telegramPool,
|
|
148
|
+
updateManifest: false,
|
|
149
|
+
replaceExisting: true,
|
|
137
150
|
limitRate: this.limitRate ? Math.floor(this.limitRate / CONCURRENCY) : null,
|
|
138
151
|
onProgress: (msg) => this.emit('progress', { file: file.relativePath, message: msg })
|
|
139
152
|
});
|
|
153
|
+
supersededChunks.push(...(result.supersededChunks || []));
|
|
140
154
|
|
|
141
155
|
// Update sync state
|
|
142
156
|
this.db.updateSyncState(folder.id, file.relativePath, hash, file.mtime);
|
|
@@ -144,15 +158,10 @@ export class SyncEngine extends EventEmitter {
|
|
|
144
158
|
|
|
145
159
|
this.emit('file-upload-complete', { file: file.relativePath });
|
|
146
160
|
} catch (err) {
|
|
147
|
-
//
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
} else {
|
|
152
|
-
// Sleep briefly on non-duplicate error (potential rate limits)
|
|
153
|
-
await new Promise(r => setTimeout(r, 2000));
|
|
154
|
-
this.emit('file-upload-error', { file: file.relativePath, error: err.message });
|
|
155
|
-
}
|
|
161
|
+
// Sleep briefly on a network/provider error before this
|
|
162
|
+
// worker advances; the staged upload remains resumable.
|
|
163
|
+
await new Promise(r => setTimeout(r, 2000));
|
|
164
|
+
this.emit('file-upload-error', { file: file.relativePath, error: err.message });
|
|
156
165
|
}
|
|
157
166
|
}
|
|
158
167
|
};
|
|
@@ -163,6 +172,22 @@ export class SyncEngine extends EventEmitter {
|
|
|
163
172
|
|
|
164
173
|
await Promise.all(promises);
|
|
165
174
|
|
|
175
|
+
if (uploaded > 0) {
|
|
176
|
+
try {
|
|
177
|
+
await backupRemoteManifest({
|
|
178
|
+
dataDir: this.dataDir,
|
|
179
|
+
password: this.password,
|
|
180
|
+
config: this.config,
|
|
181
|
+
telegramPool: this.telegramPool
|
|
182
|
+
});
|
|
183
|
+
for (const chunk of supersededChunks) {
|
|
184
|
+
try { await this.telegramPool.deleteMessage(chunk.message_id, chunk.bot_id || null); } catch { }
|
|
185
|
+
}
|
|
186
|
+
} catch (error) {
|
|
187
|
+
this.emit('manifest-error', { error: error.message });
|
|
188
|
+
}
|
|
189
|
+
}
|
|
190
|
+
|
|
166
191
|
this.emit('sync-complete', { folder: folderPath, uploaded, skipped });
|
|
167
192
|
|
|
168
193
|
return { uploaded, skipped };
|
|
@@ -224,6 +249,8 @@ export class SyncEngine extends EventEmitter {
|
|
|
224
249
|
dataDir: this.dataDir,
|
|
225
250
|
customName: filename,
|
|
226
251
|
config: this.config,
|
|
252
|
+
telegramPool: this.telegramPool,
|
|
253
|
+
replaceExisting: true,
|
|
227
254
|
limitRate: this.limitRate,
|
|
228
255
|
onProgress: (msg) => this.emit('progress', { file: filename, message: msg })
|
|
229
256
|
});
|
|
@@ -232,44 +259,88 @@ export class SyncEngine extends EventEmitter {
|
|
|
232
259
|
|
|
233
260
|
this.emit('file-upload-complete', { file: filename });
|
|
234
261
|
} catch (err) {
|
|
235
|
-
|
|
236
|
-
this.emit('file-upload-error', { file: filename, error: err.message });
|
|
237
|
-
}
|
|
262
|
+
this.emit('file-upload-error', { file: filename, error: err.message });
|
|
238
263
|
}
|
|
239
264
|
}
|
|
240
265
|
|
|
241
266
|
/**
|
|
242
|
-
*
|
|
267
|
+
* Collect all subdirectories under dirPath (including itself).
|
|
268
|
+
* Used because fs.watch({ recursive: true }) only works on macOS/Windows.
|
|
243
269
|
*/
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
270
|
+
_collectDirs(dirPath) {
|
|
271
|
+
const dirs = [dirPath];
|
|
272
|
+
let entries;
|
|
273
|
+
try {
|
|
274
|
+
entries = fs.readdirSync(dirPath, { withFileTypes: true });
|
|
275
|
+
} catch {
|
|
276
|
+
return dirs;
|
|
277
|
+
}
|
|
278
|
+
for (const entry of entries) {
|
|
279
|
+
if (!entry.isDirectory()) continue;
|
|
280
|
+
if (this.shouldIgnore(entry.name)) continue;
|
|
281
|
+
dirs.push(...this._collectDirs(path.join(dirPath, entry.name)));
|
|
247
282
|
}
|
|
283
|
+
return dirs;
|
|
284
|
+
}
|
|
248
285
|
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
286
|
+
_watchSingleDir(watchedDir, rootPath) {
|
|
287
|
+
if (this.watchers.has(watchedDir)) return;
|
|
288
|
+
let watcher;
|
|
289
|
+
try {
|
|
290
|
+
watcher = fs.watch(watchedDir, (event, filename) => {
|
|
291
|
+
if (!filename) return;
|
|
292
|
+
const fullPath = path.join(watchedDir, filename);
|
|
293
|
+
const rel = path.relative(rootPath, fullPath);
|
|
294
|
+
if (!rel || rel.startsWith('..')) return;
|
|
295
|
+
// A new subdirectory appeared — start watching it too
|
|
296
|
+
try {
|
|
297
|
+
if (fs.existsSync(fullPath) && fs.statSync(fullPath).isDirectory()) {
|
|
298
|
+
if (!this.shouldIgnore(filename)) {
|
|
299
|
+
for (const d of this._collectDirs(fullPath)) {
|
|
300
|
+
this._watchSingleDir(d, rootPath);
|
|
301
|
+
}
|
|
302
|
+
}
|
|
303
|
+
return;
|
|
304
|
+
}
|
|
305
|
+
} catch { /* fall through to file handling */ }
|
|
306
|
+
this.handleFileChange(rootPath, rel);
|
|
307
|
+
});
|
|
308
|
+
} catch (err) {
|
|
309
|
+
this.emit('watch-error', { folder: watchedDir, error: err.message });
|
|
310
|
+
return;
|
|
311
|
+
}
|
|
254
312
|
|
|
255
313
|
watcher.on('error', (err) => {
|
|
256
|
-
this.emit('watch-error', { folder:
|
|
314
|
+
this.emit('watch-error', { folder: watchedDir, error: err.message });
|
|
257
315
|
});
|
|
258
316
|
|
|
259
|
-
this.watchers.set(
|
|
317
|
+
this.watchers.set(watchedDir, watcher);
|
|
318
|
+
}
|
|
319
|
+
|
|
320
|
+
/**
|
|
321
|
+
* Start watching a folder (recursive on all platforms)
|
|
322
|
+
*/
|
|
323
|
+
watchFolder(folderPath) {
|
|
324
|
+
if (!fs.existsSync(folderPath)) return;
|
|
325
|
+
for (const dir of this._collectDirs(folderPath)) {
|
|
326
|
+
this._watchSingleDir(dir, folderPath);
|
|
327
|
+
}
|
|
260
328
|
this.emit('watch-start', { folder: folderPath });
|
|
261
329
|
}
|
|
262
330
|
|
|
263
331
|
/**
|
|
264
|
-
* Stop watching a folder
|
|
332
|
+
* Stop watching a folder (closes the root watcher and any subdir watchers)
|
|
265
333
|
*/
|
|
266
334
|
unwatchFolder(folderPath) {
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
335
|
+
let stopped = false;
|
|
336
|
+
for (const [watchedDir, watcher] of [...this.watchers]) {
|
|
337
|
+
if (watchedDir === folderPath || watchedDir.startsWith(folderPath + path.sep)) {
|
|
338
|
+
try { watcher.close(); } catch { }
|
|
339
|
+
this.watchers.delete(watchedDir);
|
|
340
|
+
stopped = true;
|
|
341
|
+
}
|
|
272
342
|
}
|
|
343
|
+
if (stopped) this.emit('watch-stop', { folder: folderPath });
|
|
273
344
|
}
|
|
274
345
|
|
|
275
346
|
/**
|
package/src/telegram/client.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Telegram Bot client wrapper
|
|
3
|
-
* Uses official Telegram Bot API
|
|
3
|
+
* Uses the official Telegram Bot API.
|
|
4
4
|
* Includes exponential backoff retry and rate limiting for production reliability.
|
|
5
5
|
*/
|
|
6
6
|
|
|
@@ -49,6 +49,7 @@ export class TelegramClient {
|
|
|
49
49
|
this.bot = null;
|
|
50
50
|
this.chatId = null;
|
|
51
51
|
this._lastSendTime = 0;
|
|
52
|
+
this._sendQueue = Promise.resolve();
|
|
52
53
|
}
|
|
53
54
|
|
|
54
55
|
/**
|
|
@@ -86,16 +87,13 @@ export class TelegramClient {
|
|
|
86
87
|
async waitForChatId(timeout = 120000) {
|
|
87
88
|
return new Promise((resolve, reject) => {
|
|
88
89
|
const pollingBot = new TelegramBot(this.bot.token, { polling: true });
|
|
89
|
-
let isCompleted = false;
|
|
90
90
|
|
|
91
91
|
const timer = setTimeout(() => {
|
|
92
|
-
isCompleted = true;
|
|
93
92
|
pollingBot.stopPolling();
|
|
94
93
|
reject(new Error('Timeout waiting for message. Please message your bot on Telegram.'));
|
|
95
94
|
}, timeout);
|
|
96
95
|
|
|
97
96
|
pollingBot.on('message', (msg) => {
|
|
98
|
-
isCompleted = true;
|
|
99
97
|
clearTimeout(timer);
|
|
100
98
|
pollingBot.stopPolling();
|
|
101
99
|
this.chatId = msg.chat.id;
|
|
@@ -107,13 +105,10 @@ export class TelegramClient {
|
|
|
107
105
|
});
|
|
108
106
|
|
|
109
107
|
pollingBot.on('polling_error', (err) => {
|
|
110
|
-
//
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
// Suppress expected cleanup errors
|
|
114
|
-
return;
|
|
108
|
+
// Ignore polling errors during shutdown
|
|
109
|
+
if (!err.message.includes('ETELEGRAM')) {
|
|
110
|
+
console.error('Polling error:', err.message);
|
|
115
111
|
}
|
|
116
|
-
console.error('Polling error:', err.message);
|
|
117
112
|
});
|
|
118
113
|
});
|
|
119
114
|
}
|
|
@@ -132,7 +127,7 @@ export class TelegramClient {
|
|
|
132
127
|
|
|
133
128
|
/**
|
|
134
129
|
* Send a file to the storage chat
|
|
135
|
-
*
|
|
130
|
+
* Send a document through the configured Bot API endpoint.
|
|
136
131
|
* Includes automatic retry with exponential backoff.
|
|
137
132
|
*/
|
|
138
133
|
async sendFile(filePath, caption = '', options = {}) {
|
|
@@ -146,7 +141,7 @@ export class TelegramClient {
|
|
|
146
141
|
|
|
147
142
|
const filename = path.basename(filePath);
|
|
148
143
|
|
|
149
|
-
|
|
144
|
+
const operation = async () => withRetry(async () => {
|
|
150
145
|
await this._rateLimit();
|
|
151
146
|
|
|
152
147
|
let fileStream = fs.createReadStream(filePath);
|
|
@@ -169,6 +164,12 @@ export class TelegramClient {
|
|
|
169
164
|
timestamp: message.date
|
|
170
165
|
};
|
|
171
166
|
}, `Upload ${filename}`);
|
|
167
|
+
|
|
168
|
+
// Serialize sends per bot/chat. Without a queue, concurrent sync
|
|
169
|
+
// workers all pass the timestamp check together and burst the API.
|
|
170
|
+
const queued = this._sendQueue.then(operation, operation);
|
|
171
|
+
this._sendQueue = queued.catch(() => { });
|
|
172
|
+
return queued;
|
|
172
173
|
}
|
|
173
174
|
|
|
174
175
|
/**
|
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Multi-bot routing for TAS.
|
|
3
|
+
*
|
|
4
|
+
* A bot is a storage endpoint, not a durability boundary: Telegram controls
|
|
5
|
+
* every endpoint. Bot IDs are persisted with each chunk so reads and deletes
|
|
6
|
+
* always go back through the bot that created the Telegram file ID.
|
|
7
|
+
*/
|
|
8
|
+
|
|
9
|
+
import crypto from 'crypto';
|
|
10
|
+
import { TelegramClient } from './client.js';
|
|
11
|
+
|
|
12
|
+
export const MULTI_BOT_WARNING =
|
|
13
|
+
'Experimental: multiple bots do not guarantee extra quota, durability, ban avoidance, or Terms compliance. ' +
|
|
14
|
+
'Do not use this feature to evade Telegram limits. Use it at your own risk and keep an independent backup.';
|
|
15
|
+
|
|
16
|
+
export function getEnabledBots(bots) {
|
|
17
|
+
return (bots || []).filter(bot => bot.enabled !== false);
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
/**
|
|
21
|
+
* Pick a stable enabled bot for a chunk. The file key chooses the starting
|
|
22
|
+
* offset and the chunk index walks the pool, which distributes a multi-chunk
|
|
23
|
+
* file without making a retry choose a different endpoint.
|
|
24
|
+
*/
|
|
25
|
+
export function selectBotId(bots, routingKey, chunkIndex = 0) {
|
|
26
|
+
const enabled = getEnabledBots(bots);
|
|
27
|
+
if (enabled.length === 0) throw new Error('No enabled Telegram bots configured');
|
|
28
|
+
|
|
29
|
+
const digest = crypto.createHash('sha256').update(String(routingKey || '')).digest();
|
|
30
|
+
const offset = digest.readUInt32BE(0) % enabled.length;
|
|
31
|
+
return enabled[(offset + chunkIndex) % enabled.length].id;
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
export class TelegramPool {
|
|
35
|
+
constructor(dataDir, bots) {
|
|
36
|
+
if (!Array.isArray(bots) || bots.length === 0) {
|
|
37
|
+
throw new Error('No Telegram bots configured');
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
this.dataDir = dataDir;
|
|
41
|
+
this.bots = bots;
|
|
42
|
+
this.botById = new Map(bots.map(bot => [bot.id, bot]));
|
|
43
|
+
this.clientPromises = new Map();
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
async initialize({ includeDisabled = true } = {}) {
|
|
47
|
+
const bots = includeDisabled ? this.bots : getEnabledBots(this.bots);
|
|
48
|
+
await Promise.all(bots.map(bot => this._getClient(bot.id)));
|
|
49
|
+
return this;
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
selectBotId(routingKey, chunkIndex = 0) {
|
|
53
|
+
return selectBotId(this.bots, routingKey, chunkIndex);
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
_normalizeBotId(botId) {
|
|
57
|
+
if (botId && this.botById.has(botId)) return botId;
|
|
58
|
+
if (!botId && this.botById.has('primary')) return 'primary';
|
|
59
|
+
if (!botId && this.bots.length === 1) return this.bots[0].id;
|
|
60
|
+
throw new Error(`Telegram bot ${botId || '(legacy primary)'} is not configured`);
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
async _getClient(botId) {
|
|
64
|
+
const normalizedId = this._normalizeBotId(botId);
|
|
65
|
+
if (!this.clientPromises.has(normalizedId)) {
|
|
66
|
+
const bot = this.botById.get(normalizedId);
|
|
67
|
+
const promise = (async () => {
|
|
68
|
+
const client = new TelegramClient(this.dataDir);
|
|
69
|
+
await client.initialize(bot.botToken, bot.customApiUrl || null);
|
|
70
|
+
client.setChatId(bot.chatId);
|
|
71
|
+
return client;
|
|
72
|
+
})();
|
|
73
|
+
this.clientPromises.set(normalizedId, promise);
|
|
74
|
+
}
|
|
75
|
+
return this.clientPromises.get(normalizedId);
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
async sendFile(filePath, caption = '', options = {}) {
|
|
79
|
+
const botId = options.botId || this.selectBotId(options.routingKey, options.chunkIndex || 0);
|
|
80
|
+
const client = await this._getClient(botId);
|
|
81
|
+
const { botId: _botId, routingKey: _routingKey, chunkIndex: _chunkIndex, ...clientOptions } = options;
|
|
82
|
+
const result = await client.sendFile(filePath, caption, clientOptions);
|
|
83
|
+
return { ...result, botId };
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
async downloadFile(fileId, botId = null) {
|
|
87
|
+
const client = await this._getClient(botId);
|
|
88
|
+
return client.downloadFile(fileId);
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
async downloadFileToPath(fileId, destPath, botId = null) {
|
|
92
|
+
const client = await this._getClient(botId);
|
|
93
|
+
return client.downloadFileToPath(fileId, destPath);
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
async getFile(fileId, botId = null) {
|
|
97
|
+
const client = await this._getClient(botId);
|
|
98
|
+
return client.bot.getFile(fileId);
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
async deleteMessage(messageId, botId = null) {
|
|
102
|
+
const client = await this._getClient(botId);
|
|
103
|
+
return client.deleteMessage(messageId);
|
|
104
|
+
}
|
|
105
|
+
}
|
package/src/utils/branding.js
CHANGED
|
@@ -14,7 +14,7 @@ export const LOGO = `
|
|
|
14
14
|
`;
|
|
15
15
|
|
|
16
16
|
export const TAGLINE = 'Telegram as Storage';
|
|
17
|
-
export const VERSION = '
|
|
17
|
+
export const VERSION = '3.0.0';
|
|
18
18
|
|
|
19
19
|
/**
|
|
20
20
|
* Print the TAS banner
|
|
@@ -22,7 +22,7 @@ export const VERSION = '2.4.1';
|
|
|
22
22
|
export function printBanner() {
|
|
23
23
|
console.log(chalk.cyan(LOGO));
|
|
24
24
|
console.log(chalk.dim(` ${TAGLINE} v${VERSION}`));
|
|
25
|
-
console.log(chalk.dim('
|
|
25
|
+
console.log(chalk.dim(' Experimental • Encrypted • Local-first\n'));
|
|
26
26
|
}
|
|
27
27
|
|
|
28
28
|
/**
|
package/src/utils/chunker.js
CHANGED
|
@@ -1,11 +1,12 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* File chunking utilities for large files
|
|
3
|
-
* Telegram Bot API
|
|
4
|
-
*
|
|
3
|
+
* Telegram's hosted Bot API currently returns files through getFile only up
|
|
4
|
+
* to 20 MB. Keep the full stored document below that read limit so TAS never
|
|
5
|
+
* uploads a chunk that it cannot later pull, mount, verify, or share.
|
|
5
6
|
* See: https://core.telegram.org/bots/api#senddocument
|
|
6
7
|
*/
|
|
7
8
|
|
|
8
|
-
const MAX_CHUNK_SIZE =
|
|
9
|
+
export const MAX_CHUNK_SIZE = 19 * 1024 * 1024; // 19 MiB payload + 64-byte TAS header
|
|
9
10
|
|
|
10
11
|
export class Chunker {
|
|
11
12
|
/**
|
package/src/utils/cli-helpers.js
CHANGED
|
@@ -61,18 +61,32 @@ export function validateConfig(config) {
|
|
|
61
61
|
return { valid: false, errors };
|
|
62
62
|
}
|
|
63
63
|
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
64
|
+
const bots = getBotEntries(config);
|
|
65
|
+
if (bots.length === 0) errors.push('Missing bot configuration');
|
|
66
|
+
|
|
67
|
+
const seenIds = new Set();
|
|
68
|
+
for (const bot of bots) {
|
|
69
|
+
if (!bot.id || !/^[a-z0-9][a-z0-9_-]{0,31}$/i.test(bot.id)) {
|
|
70
|
+
errors.push(`Invalid bot id: ${bot.id || '(missing)'}`);
|
|
71
|
+
} else if (seenIds.has(bot.id)) {
|
|
72
|
+
errors.push(`Duplicate bot id: ${bot.id}`);
|
|
73
|
+
}
|
|
74
|
+
seenIds.add(bot.id);
|
|
69
75
|
|
|
70
|
-
|
|
71
|
-
|
|
76
|
+
if (!bot.encryptedBotToken && !bot.botToken) {
|
|
77
|
+
errors.push(`Missing token for bot ${bot.id || '(unknown)'}`);
|
|
78
|
+
}
|
|
79
|
+
if (bot.botToken && !bot.botToken.includes(':')) {
|
|
80
|
+
errors.push(`Invalid token format for bot ${bot.id || '(unknown)'}`);
|
|
81
|
+
}
|
|
82
|
+
if (bot.chatId === undefined || bot.chatId === null ||
|
|
83
|
+
(typeof bot.chatId !== 'number' && typeof bot.chatId !== 'string')) {
|
|
84
|
+
errors.push(`Missing or invalid chatId for bot ${bot.id || '(unknown)'}`);
|
|
85
|
+
}
|
|
72
86
|
}
|
|
73
87
|
|
|
74
|
-
if (
|
|
75
|
-
errors.push('
|
|
88
|
+
if (bots.length > 0 && !bots.some(bot => bot.enabled !== false)) {
|
|
89
|
+
errors.push('At least one bot must be enabled');
|
|
76
90
|
}
|
|
77
91
|
|
|
78
92
|
if (!config.passwordHash || typeof config.passwordHash !== 'string') {
|
|
@@ -85,6 +99,35 @@ export function validateConfig(config) {
|
|
|
85
99
|
};
|
|
86
100
|
}
|
|
87
101
|
|
|
102
|
+
/**
|
|
103
|
+
* Return normalized raw bot entries. v1/v2 configs are represented as one
|
|
104
|
+
* stable `primary` bot so existing vaults require no migration to keep working.
|
|
105
|
+
*/
|
|
106
|
+
export function getBotEntries(config) {
|
|
107
|
+
if (!config) return [];
|
|
108
|
+
if (Array.isArray(config.bots)) {
|
|
109
|
+
return config.bots.map((bot, index) => ({
|
|
110
|
+
...bot,
|
|
111
|
+
id: bot.id || (index === 0 ? 'primary' : `bot-${index + 1}`),
|
|
112
|
+
enabled: bot.enabled !== false
|
|
113
|
+
}));
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
if (config.encryptedBotToken || config.botToken) {
|
|
117
|
+
return [{
|
|
118
|
+
id: 'primary',
|
|
119
|
+
encryptedBotToken: config.encryptedBotToken,
|
|
120
|
+
botToken: config.botToken,
|
|
121
|
+
chatId: config.chatId,
|
|
122
|
+
username: config.username,
|
|
123
|
+
enabled: true,
|
|
124
|
+
createdAt: config.createdAt
|
|
125
|
+
}];
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
return [];
|
|
129
|
+
}
|
|
130
|
+
|
|
88
131
|
/**
|
|
89
132
|
* Decrypt bot token from config using password
|
|
90
133
|
* Supports both v1 (plaintext) and v2 (encrypted) configs
|
|
@@ -108,6 +151,20 @@ export function decryptBotToken(config, password) {
|
|
|
108
151
|
throw new Error('No bot token found in config');
|
|
109
152
|
}
|
|
110
153
|
|
|
154
|
+
/** Decrypt one normalized bot entry. */
|
|
155
|
+
export function decryptBotEntry(bot, password) {
|
|
156
|
+
return {
|
|
157
|
+
...bot,
|
|
158
|
+
botToken: decryptBotToken(bot, password)
|
|
159
|
+
};
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
/** Encrypt a bot token with the vault password. */
|
|
163
|
+
export function encryptBotToken(token, password) {
|
|
164
|
+
const encryptor = new Encryptor(password);
|
|
165
|
+
return encryptor.encrypt(Buffer.from(token, 'utf-8')).toString('base64');
|
|
166
|
+
}
|
|
167
|
+
|
|
111
168
|
/**
|
|
112
169
|
* Load and validate config
|
|
113
170
|
* @param {string} dataDir - Data directory path
|
|
@@ -176,8 +233,31 @@ export async function getAndVerifyPassword(passwordOption, dataDir) {
|
|
|
176
233
|
* @returns {Object} - Config with decrypted botToken
|
|
177
234
|
*/
|
|
178
235
|
export function resolveConfig(config, password) {
|
|
236
|
+
// Reuse one derived key across the pool; deriving PBKDF2 separately for
|
|
237
|
+
// every bot made startup scale linearly with the number of configured bots.
|
|
238
|
+
const encryptor = new Encryptor(password);
|
|
239
|
+
const bots = getBotEntries(config).map(bot => ({
|
|
240
|
+
...bot,
|
|
241
|
+
botToken: bot.botToken || encryptor.decrypt(Buffer.from(bot.encryptedBotToken, 'base64')).toString('utf-8')
|
|
242
|
+
}));
|
|
243
|
+
const primary = bots.find(bot => bot.id === 'primary') || bots[0];
|
|
244
|
+
|
|
179
245
|
return {
|
|
180
246
|
...config,
|
|
181
|
-
|
|
247
|
+
bots,
|
|
248
|
+
// Keep these aliases for third-party callers that still consume the
|
|
249
|
+
// v1/v2 shape. New code routes through `bots` and persists bot IDs.
|
|
250
|
+
botToken: primary.botToken,
|
|
251
|
+
chatId: primary.chatId,
|
|
252
|
+
username: primary.username
|
|
182
253
|
};
|
|
183
254
|
}
|
|
255
|
+
|
|
256
|
+
/** Write config atomically enough for this local CLI and restore mode 0600. */
|
|
257
|
+
export function saveConfig(dataDir, config) {
|
|
258
|
+
const configPath = path.join(dataDir, 'config.json');
|
|
259
|
+
const tempPath = `${configPath}.tmp`;
|
|
260
|
+
fs.writeFileSync(tempPath, JSON.stringify(config, null, 2), { mode: 0o600 });
|
|
261
|
+
fs.renameSync(tempPath, configPath);
|
|
262
|
+
try { fs.chmodSync(configPath, 0o600); } catch { /* ignore on Windows */ }
|
|
263
|
+
}
|
|
@@ -13,8 +13,8 @@ import { parseHeader, HEADER_SIZE } from './chunker.js';
|
|
|
13
13
|
* Create the download pipeline streams for a file stored in Telegram.
|
|
14
14
|
*
|
|
15
15
|
* @param {object} options
|
|
16
|
-
* @param {object} options.client –
|
|
17
|
-
* @param {Array} options.chunks – Chunk records
|
|
16
|
+
* @param {object} options.client – TelegramPool (or compatible single client)
|
|
17
|
+
* @param {Array} options.chunks – Chunk records with { chunk_index, file_telegram_id, bot_id, size }
|
|
18
18
|
* @param {object} options.encryptor – Encryptor instance
|
|
19
19
|
* @param {object} options.compressor – Compressor instance
|
|
20
20
|
* @param {function} [options.onChunkDownloaded] – Optional callback({ chunkIndex, totalChunks, bytesDownloaded, totalBytes })
|
|
@@ -29,7 +29,10 @@ export async function createDownloadPipeline({ client, chunks, encryptor, compre
|
|
|
29
29
|
const sortedChunks = [...chunks].sort((a, b) => a.chunk_index - b.chunk_index);
|
|
30
30
|
|
|
31
31
|
// Download the first chunk to inspect the header
|
|
32
|
-
const firstChunkData = await client.downloadFile(
|
|
32
|
+
const firstChunkData = await client.downloadFile(
|
|
33
|
+
sortedChunks[0].file_telegram_id,
|
|
34
|
+
sortedChunks[0].bot_id || null
|
|
35
|
+
);
|
|
33
36
|
const header = parseHeader(firstChunkData);
|
|
34
37
|
|
|
35
38
|
const decryptStream = encryptor.getDecryptStream();
|
|
@@ -53,7 +56,10 @@ export async function createDownloadPipeline({ client, chunks, encryptor, compre
|
|
|
53
56
|
data = preloadedFirst;
|
|
54
57
|
preloadedFirst = null;
|
|
55
58
|
} else {
|
|
56
|
-
data = await client.downloadFile(
|
|
59
|
+
data = await client.downloadFile(
|
|
60
|
+
sortedChunks[currentIndex].file_telegram_id,
|
|
61
|
+
sortedChunks[currentIndex].bot_id || null
|
|
62
|
+
);
|
|
57
63
|
}
|
|
58
64
|
|
|
59
65
|
bytesDownloaded += data.length;
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
/** Normalize a TAS logical path to portable POSIX form. */
|
|
2
|
+
export function normalizeLogicalPath(value, { allowRoot = false } = {}) {
|
|
3
|
+
if (typeof value !== 'string' || value.includes('\0')) {
|
|
4
|
+
throw new Error('Invalid logical path');
|
|
5
|
+
}
|
|
6
|
+
|
|
7
|
+
const portable = value.replace(/\\/g, '/').replace(/^\/+/, '');
|
|
8
|
+
const normalized = portable === '' ? '' : portable.split('/').filter(Boolean).join('/');
|
|
9
|
+
const parts = normalized.split('/').filter(Boolean);
|
|
10
|
+
if (parts.some(part => part === '.' || part === '..')) {
|
|
11
|
+
throw new Error('Logical paths cannot contain . or .. segments');
|
|
12
|
+
}
|
|
13
|
+
if (!allowRoot && normalized === '') throw new Error('Logical path cannot be empty');
|
|
14
|
+
return normalized;
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
export function parentLogicalPath(value) {
|
|
18
|
+
const normalized = normalizeLogicalPath(value);
|
|
19
|
+
const index = normalized.lastIndexOf('/');
|
|
20
|
+
return index < 0 ? '' : normalized.slice(0, index);
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
/** Return only immediate children for a virtual directory. */
|
|
24
|
+
export function listLogicalChildren(paths, directory = '') {
|
|
25
|
+
const dir = normalizeLogicalPath(directory, { allowRoot: true });
|
|
26
|
+
const prefix = dir ? `${dir}/` : '';
|
|
27
|
+
const children = new Set();
|
|
28
|
+
|
|
29
|
+
for (const value of paths) {
|
|
30
|
+
const logical = normalizeLogicalPath(value);
|
|
31
|
+
if (!logical.startsWith(prefix)) continue;
|
|
32
|
+
const remainder = logical.slice(prefix.length);
|
|
33
|
+
if (!remainder) continue;
|
|
34
|
+
children.add(remainder.split('/')[0]);
|
|
35
|
+
}
|
|
36
|
+
return [...children].sort((a, b) => a.localeCompare(b));
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
export function isImplicitDirectory(paths, directory) {
|
|
40
|
+
const dir = normalizeLogicalPath(directory, { allowRoot: true });
|
|
41
|
+
if (dir === '') return true;
|
|
42
|
+
const prefix = `${dir}/`;
|
|
43
|
+
return paths.some(value => normalizeLogicalPath(value).startsWith(prefix));
|
|
44
|
+
}
|