nt 0.8.0 → 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +17 -4
- package/dist/hasher.d.ts +117 -5
- package/dist/hasher.js +478 -49
- package/dist/hasher.js.map +1 -1
- package/dist/index.d.ts +3 -3
- package/dist/index.js.map +1 -1
- package/dist/make.d.ts +10 -3
- package/dist/make.js +109 -31
- package/dist/make.js.map +1 -1
- package/dist/schema.js +7 -0
- package/dist/schema.js.map +1 -1
- package/dist/util.d.ts +1 -1
- package/dist/util.js +10 -2
- package/dist/util.js.map +1 -1
- package/package.json +2 -2
package/dist/hasher.js
CHANGED
|
@@ -36,13 +36,16 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
36
36
|
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
37
37
|
};
|
|
38
38
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
39
|
+
exports.isStreamEntry = exports.isReadableStream = void 0;
|
|
39
40
|
const node_crypto_1 = __importDefault(require("node:crypto"));
|
|
40
41
|
const node_fs_1 = __importDefault(require("node:fs"));
|
|
42
|
+
const node_http_1 = require("node:http");
|
|
41
43
|
const node_path_1 = __importDefault(require("node:path"));
|
|
42
44
|
const node_stream_1 = require("node:stream");
|
|
43
45
|
const async_1 = require("async");
|
|
44
46
|
const streamspeed_1 = __importDefault(require("streamspeed"));
|
|
45
47
|
const queue_1 = __importDefault(require("./queue"));
|
|
48
|
+
const schema_1 = require("./schema");
|
|
46
49
|
const util = __importStar(require("./util"));
|
|
47
50
|
/**
|
|
48
51
|
* defaults
|
|
@@ -62,6 +65,65 @@ const round = (num, dec) => {
|
|
|
62
65
|
const pow = 10 ** dec;
|
|
63
66
|
return Math.round(num * pow) / pow;
|
|
64
67
|
};
|
|
68
|
+
const isReadableStream = (val) => typeof val === 'object' &&
|
|
69
|
+
val !== null &&
|
|
70
|
+
typeof val.pipe === 'function';
|
|
71
|
+
exports.isReadableStream = isReadableStream;
|
|
72
|
+
const isStreamEntry = (val) => typeof val === 'object' && val !== null && 'stream' in val;
|
|
73
|
+
exports.isStreamEntry = isStreamEntry;
|
|
74
|
+
const isStreamInput = (item) => (0, exports.isReadableStream)(item) || (0, exports.isStreamEntry)(item);
|
|
75
|
+
/**
|
|
76
|
+
* Infers a path from a stream that knows where its data comes from,
|
|
77
|
+
* such as an `fs.ReadStream`.
|
|
78
|
+
*/
|
|
79
|
+
const inferStreamPath = (stream) => {
|
|
80
|
+
const streamPath = stream.path;
|
|
81
|
+
if (typeof streamPath === 'string')
|
|
82
|
+
return [node_path_1.default.basename(streamPath)];
|
|
83
|
+
if (Buffer.isBuffer(streamPath)) {
|
|
84
|
+
return [node_path_1.default.basename(streamPath.toString())];
|
|
85
|
+
}
|
|
86
|
+
return null;
|
|
87
|
+
};
|
|
88
|
+
const getReadStreamLength = (stream) => {
|
|
89
|
+
// A partially read stream no longer represents its full file.
|
|
90
|
+
if (stream.bytesRead > 0 || !stream.path)
|
|
91
|
+
return Promise.resolve(undefined);
|
|
92
|
+
return new Promise((resolve) => {
|
|
93
|
+
node_fs_1.default.stat(stream.path, (err, stat) => {
|
|
94
|
+
if (err || !stat.isFile()) {
|
|
95
|
+
resolve(undefined);
|
|
96
|
+
return;
|
|
97
|
+
}
|
|
98
|
+
const range = stream;
|
|
99
|
+
const start = typeof range.start === 'number' ? range.start : 0;
|
|
100
|
+
const end = Number.isFinite(range.end)
|
|
101
|
+
? range.end
|
|
102
|
+
: stat.size - 1;
|
|
103
|
+
const lastByte = Math.min(end, stat.size - 1);
|
|
104
|
+
resolve(Math.max(0, lastByte - start + 1));
|
|
105
|
+
});
|
|
106
|
+
});
|
|
107
|
+
};
|
|
108
|
+
/**
|
|
109
|
+
* Infers a stream's length when that information is cheap and reliable.
|
|
110
|
+
* `fs.ReadStream` lengths come from the file size, adjusted for any
|
|
111
|
+
* `start`/`end` range. HTTP responses can use `Content-Length`.
|
|
112
|
+
*/
|
|
113
|
+
const inferStreamLength = (stream) => {
|
|
114
|
+
if (stream instanceof node_fs_1.default.ReadStream)
|
|
115
|
+
return getReadStreamLength(stream);
|
|
116
|
+
if (stream instanceof node_http_1.IncomingMessage) {
|
|
117
|
+
const header = stream.headers['content-length'];
|
|
118
|
+
if (typeof header === 'string') {
|
|
119
|
+
const length = Number.parseInt(header, 10);
|
|
120
|
+
if (Number.isInteger(length) && length >= 0) {
|
|
121
|
+
return Promise.resolve(length);
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
return Promise.resolve(undefined);
|
|
126
|
+
};
|
|
65
127
|
/**
|
|
66
128
|
* Allows arrays of objects with a `path` property to be
|
|
67
129
|
* sorted alphabetically
|
|
@@ -76,6 +138,14 @@ const sortByPath = (a, b) => {
|
|
|
76
138
|
}
|
|
77
139
|
return a.path.length - b.path.length;
|
|
78
140
|
};
|
|
141
|
+
/**
|
|
142
|
+
* Builds a torrent's file list and hashes it into pieces.
|
|
143
|
+
*
|
|
144
|
+
* Inputs are first resolved and validated, then piece offsets are planned. When
|
|
145
|
+
* every input length is known, files are read in parallel. If any stream length
|
|
146
|
+
* is unknown, hashing switches to a single sequential pass because each input's
|
|
147
|
+
* starting offset depends on everything before it.
|
|
148
|
+
*/
|
|
79
149
|
class Hasher extends node_stream_1.Stream {
|
|
80
150
|
pieceLength;
|
|
81
151
|
maxFiles;
|
|
@@ -93,8 +163,9 @@ class Hasher extends node_stream_1.Stream {
|
|
|
93
163
|
totalSize;
|
|
94
164
|
lastPieceLength;
|
|
95
165
|
streamGroup;
|
|
166
|
+
unknownLengths;
|
|
96
167
|
/**
|
|
97
|
-
*
|
|
168
|
+
* Resolves the inputs and prepares the hasher.
|
|
98
169
|
*/
|
|
99
170
|
constructor(dir, list, pieceLength, options) {
|
|
100
171
|
super();
|
|
@@ -111,6 +182,7 @@ class Hasher extends node_stream_1.Stream {
|
|
|
111
182
|
this.hashing = false;
|
|
112
183
|
this.paused = false;
|
|
113
184
|
this.stopped = false;
|
|
185
|
+
this.unknownLengths = false;
|
|
114
186
|
this.readable = false;
|
|
115
187
|
this.filetasks = [];
|
|
116
188
|
this.totalSize = 0;
|
|
@@ -121,48 +193,269 @@ class Hasher extends node_stream_1.Stream {
|
|
|
121
193
|
this.err = err;
|
|
122
194
|
});
|
|
123
195
|
}
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
196
|
+
if (list.some(isStreamInput)) {
|
|
197
|
+
this._resolveInputs(list).then((items) => this._beginHashing(items), (err) => this._emitError(err));
|
|
198
|
+
return;
|
|
199
|
+
}
|
|
200
|
+
this._beginPathInputs(list);
|
|
201
|
+
}
|
|
202
|
+
// ---------------------------------------------------------------------------
|
|
203
|
+
// Input resolution
|
|
204
|
+
// ---------------------------------------------------------------------------
|
|
205
|
+
/**
|
|
206
|
+
* Reports an error on the next tick so callers can attach listeners.
|
|
207
|
+
*/
|
|
208
|
+
_emitError(err) {
|
|
209
|
+
process.nextTick(() => this.emit('error', err));
|
|
210
|
+
}
|
|
211
|
+
/**
|
|
212
|
+
* Resolves streams and paths into a single ordered list of torrent entries.
|
|
213
|
+
*
|
|
214
|
+
* Unlike pure path lists (which are sorted globally), the order of the input
|
|
215
|
+
* list is authoritative here. Directories are expanded in place and their
|
|
216
|
+
* entries are sorted within that directory.
|
|
217
|
+
*/
|
|
218
|
+
async _resolveInputs(list) {
|
|
219
|
+
const baseName = node_path_1.default.basename(this.dir) || 'stream';
|
|
220
|
+
const items = [];
|
|
221
|
+
for (const input of list) {
|
|
222
|
+
if ((0, exports.isReadableStream)(input)) {
|
|
223
|
+
items.push(this._streamItem(input, baseName));
|
|
224
|
+
}
|
|
225
|
+
else if ((0, exports.isStreamEntry)(input)) {
|
|
226
|
+
items.push(this._streamEntryItem(input, baseName));
|
|
127
227
|
}
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
228
|
+
else if (typeof input === 'string') {
|
|
229
|
+
await this._collectPath(input, items);
|
|
230
|
+
}
|
|
231
|
+
else {
|
|
232
|
+
throw new Error('Mixed lists can only contain file paths and streams');
|
|
233
|
+
}
|
|
234
|
+
}
|
|
235
|
+
await Promise.all(items.map(async (item) => {
|
|
236
|
+
if ('stream' in item && item.length === undefined) {
|
|
237
|
+
item.length = await inferStreamLength(item.stream);
|
|
238
|
+
}
|
|
239
|
+
}));
|
|
240
|
+
this._validateItems(items);
|
|
241
|
+
this._assignNames(items);
|
|
242
|
+
return items;
|
|
243
|
+
}
|
|
244
|
+
/**
|
|
245
|
+
* Builds a torrent entry for a bare readable stream.
|
|
246
|
+
*/
|
|
247
|
+
_streamItem(stream, baseName) {
|
|
248
|
+
return {
|
|
249
|
+
stream,
|
|
250
|
+
path: inferStreamPath(stream) ?? [baseName],
|
|
251
|
+
autoNamed: true,
|
|
252
|
+
};
|
|
253
|
+
}
|
|
254
|
+
/**
|
|
255
|
+
* Builds a torrent entry for a `{ stream, path, length }` input.
|
|
256
|
+
*/
|
|
257
|
+
_streamEntryItem(entry, baseName) {
|
|
258
|
+
const rawPath = entry.path;
|
|
259
|
+
let itemPath;
|
|
260
|
+
if (rawPath == null) {
|
|
261
|
+
itemPath = inferStreamPath(entry.stream) ?? [baseName];
|
|
262
|
+
}
|
|
263
|
+
else if (Array.isArray(rawPath)) {
|
|
264
|
+
itemPath = rawPath;
|
|
265
|
+
}
|
|
266
|
+
else {
|
|
267
|
+
itemPath = util.splitPath(rawPath);
|
|
268
|
+
}
|
|
269
|
+
return {
|
|
270
|
+
stream: entry.stream,
|
|
271
|
+
path: itemPath,
|
|
272
|
+
length: entry.length,
|
|
273
|
+
autoNamed: rawPath == null,
|
|
274
|
+
};
|
|
275
|
+
}
|
|
276
|
+
/**
|
|
277
|
+
* Validates an input path, then expands it into files in list order.
|
|
278
|
+
*/
|
|
279
|
+
async _collectPath(input, items) {
|
|
280
|
+
const relPath = node_path_1.default.normalize(input);
|
|
281
|
+
if (node_path_1.default.isAbsolute(relPath) || relPath.split(node_path_1.default.sep).includes('..')) {
|
|
282
|
+
throw new Error(`Path traversal detected: ${relPath}`);
|
|
283
|
+
}
|
|
284
|
+
await this._walkPath(relPath, items);
|
|
285
|
+
}
|
|
286
|
+
/**
|
|
287
|
+
* Recursively walks a path. Directories are expanded in sorted order.
|
|
288
|
+
*/
|
|
289
|
+
async _walkPath(relPath, items) {
|
|
290
|
+
const fullPath = node_path_1.default.join(this.dir, relPath);
|
|
291
|
+
const stat = await node_fs_1.default.promises.stat(fullPath);
|
|
292
|
+
if (stat.isFile()) {
|
|
293
|
+
items.push({
|
|
294
|
+
path: util.splitPath(relPath),
|
|
295
|
+
length: stat.size,
|
|
296
|
+
filepath: fullPath,
|
|
297
|
+
});
|
|
298
|
+
return;
|
|
299
|
+
}
|
|
300
|
+
if (stat.isDirectory()) {
|
|
301
|
+
const names = (await node_fs_1.default.promises.readdir(fullPath)).sort();
|
|
302
|
+
for (const name of names) {
|
|
303
|
+
await this._walkPath(node_path_1.default.join(relPath, name), items);
|
|
304
|
+
}
|
|
305
|
+
}
|
|
306
|
+
}
|
|
307
|
+
/**
|
|
308
|
+
* Rejects invalid paths and stream lengths.
|
|
309
|
+
*/
|
|
310
|
+
_validateItems(items) {
|
|
311
|
+
for (const item of items) {
|
|
312
|
+
const label = item.path.join('/');
|
|
313
|
+
if (item.path.length === 0 ||
|
|
314
|
+
!item.path.every((seg) => typeof seg === 'string' && (0, schema_1.isValidPathSegment)(Buffer.from(seg)))) {
|
|
315
|
+
throw new Error(`Path traversal detected: ${label}`);
|
|
316
|
+
}
|
|
317
|
+
if ('stream' in item &&
|
|
318
|
+
item.length !== undefined &&
|
|
319
|
+
(!Number.isInteger(item.length) || item.length < 0)) {
|
|
320
|
+
throw new Error(`Invalid stream length for "${label}": must be a non-negative integer`);
|
|
321
|
+
}
|
|
322
|
+
}
|
|
323
|
+
}
|
|
324
|
+
/**
|
|
325
|
+
* Reserves explicit paths and gives unnamed streams unique suffixes.
|
|
326
|
+
*/
|
|
327
|
+
_assignNames(items) {
|
|
328
|
+
const usedPaths = new Set();
|
|
329
|
+
// Explicit paths, including files found on disk, must be unique.
|
|
330
|
+
for (const item of items) {
|
|
331
|
+
if (item.autoNamed)
|
|
332
|
+
continue;
|
|
333
|
+
const key = item.path.join('/');
|
|
334
|
+
if (usedPaths.has(key)) {
|
|
335
|
+
throw new Error(`Duplicate path: ${key}`);
|
|
336
|
+
}
|
|
337
|
+
usedPaths.add(key);
|
|
338
|
+
}
|
|
339
|
+
// Unnamed streams avoid both disk paths and other stream paths.
|
|
340
|
+
for (const item of items) {
|
|
341
|
+
if (!item.autoNamed)
|
|
342
|
+
continue;
|
|
343
|
+
const basePath = item.path;
|
|
344
|
+
let candidate = basePath;
|
|
345
|
+
let suffix = 0;
|
|
346
|
+
while (usedPaths.has(candidate.join('/'))) {
|
|
347
|
+
suffix++;
|
|
348
|
+
candidate = [
|
|
349
|
+
...basePath.slice(0, -1),
|
|
350
|
+
`${basePath[basePath.length - 1]}-${suffix}`,
|
|
351
|
+
];
|
|
352
|
+
}
|
|
353
|
+
item.path = candidate;
|
|
354
|
+
usedPaths.add(candidate.join('/'));
|
|
355
|
+
}
|
|
356
|
+
// A file path cannot also be a directory prefix of another path.
|
|
357
|
+
for (const item of items) {
|
|
358
|
+
for (let i = 1; i < item.path.length; i++) {
|
|
359
|
+
const prefix = item.path.slice(0, i).join('/');
|
|
360
|
+
if (usedPaths.has(prefix)) {
|
|
361
|
+
throw new Error(`Path conflict: ${prefix}`);
|
|
137
362
|
}
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
node_fs_1.default.stat(filepath, cb);
|
|
154
|
-
}, this.maxFiles);
|
|
155
|
-
this._getFilesStats(statqueue, list, readTasks);
|
|
156
|
-
// In the case of objects, a file list from a torrent file was given.
|
|
363
|
+
}
|
|
364
|
+
}
|
|
365
|
+
}
|
|
366
|
+
/**
|
|
367
|
+
* Starts reading pure path inputs. Paths are sorted so the resulting torrent
|
|
368
|
+
* does not depend on filesystem iteration order.
|
|
369
|
+
*/
|
|
370
|
+
_beginPathInputs(list) {
|
|
371
|
+
node_fs_1.default.access(this.dir, (err) => {
|
|
372
|
+
if (err) {
|
|
373
|
+
this.emit('error', err);
|
|
374
|
+
return;
|
|
375
|
+
}
|
|
376
|
+
if (typeof list[0] === 'string') {
|
|
377
|
+
this._loadPaths(list);
|
|
157
378
|
}
|
|
158
379
|
else {
|
|
159
|
-
this.
|
|
160
|
-
|
|
380
|
+
this._useFileEntries(list);
|
|
381
|
+
}
|
|
382
|
+
});
|
|
383
|
+
}
|
|
384
|
+
/**
|
|
385
|
+
* Stats and expands paths, then plans the piece layout.
|
|
386
|
+
*/
|
|
387
|
+
_loadPaths(paths) {
|
|
388
|
+
// Limit how many files are opened while walking the tree.
|
|
389
|
+
const statqueue = (0, queue_1.default)((filepath, cb) => {
|
|
390
|
+
node_fs_1.default.stat(filepath, cb);
|
|
391
|
+
}, this.maxFiles);
|
|
392
|
+
this._getFilesStats(statqueue, paths, (err) => {
|
|
393
|
+
// Sort even on failure so partially collected files are deterministic.
|
|
394
|
+
this.files = Array.from(this.files).sort(sortByPath);
|
|
395
|
+
if (err) {
|
|
396
|
+
this.emit('error', err);
|
|
397
|
+
return;
|
|
161
398
|
}
|
|
399
|
+
this._CreateFileTasks();
|
|
400
|
+
this._readyAndStart(() => this._start());
|
|
401
|
+
});
|
|
402
|
+
}
|
|
403
|
+
/**
|
|
404
|
+
* Uses an already-known file list, such as one from an existing torrent.
|
|
405
|
+
* Order is preserved because it determines the piece hashes.
|
|
406
|
+
*/
|
|
407
|
+
_useFileEntries(entries) {
|
|
408
|
+
this.files = Array.from(entries);
|
|
409
|
+
process.nextTick(() => {
|
|
410
|
+
this._CreateFileTasks();
|
|
411
|
+
this._readyAndStart(() => this._start());
|
|
162
412
|
});
|
|
163
413
|
}
|
|
164
414
|
/**
|
|
165
|
-
*
|
|
415
|
+
* Starts hashing a resolved stream/path list.
|
|
416
|
+
*
|
|
417
|
+
* When every length is known, piece offsets can be planned up front and files
|
|
418
|
+
* are hashed in parallel. If any stream length is unknown, hashing falls back
|
|
419
|
+
* to a single sequential pass because later offsets cannot be known yet.
|
|
420
|
+
*/
|
|
421
|
+
_beginHashing(items) {
|
|
422
|
+
if (items.some((item) => item.length === undefined)) {
|
|
423
|
+
this.unknownLengths = true;
|
|
424
|
+
this.files = [];
|
|
425
|
+
this._readyAndStart(() => this._startSequential(items));
|
|
426
|
+
return;
|
|
427
|
+
}
|
|
428
|
+
this.files = items.map((item) => ({
|
|
429
|
+
length: item.length,
|
|
430
|
+
path: item.path,
|
|
431
|
+
}));
|
|
432
|
+
this._CreateFileTasks();
|
|
433
|
+
// Attach stream-backed tasks; path-backed tasks are opened when hashed.
|
|
434
|
+
items.forEach((item, i) => {
|
|
435
|
+
if ('stream' in item)
|
|
436
|
+
this.filetasks[i].stream = item.stream;
|
|
437
|
+
});
|
|
438
|
+
this._readyAndStart(() => this._start());
|
|
439
|
+
}
|
|
440
|
+
/**
|
|
441
|
+
* Marks the hasher ready, then starts hashing on the next tick.
|
|
442
|
+
*/
|
|
443
|
+
_readyAndStart(start) {
|
|
444
|
+
process.nextTick(() => {
|
|
445
|
+
this.ready = true;
|
|
446
|
+
this.emit('ready');
|
|
447
|
+
start();
|
|
448
|
+
});
|
|
449
|
+
}
|
|
450
|
+
// ---------------------------------------------------------------------------
|
|
451
|
+
// Piece planning
|
|
452
|
+
// ---------------------------------------------------------------------------
|
|
453
|
+
/**
|
|
454
|
+
* Plans which pieces each file covers.
|
|
455
|
+
*
|
|
456
|
+
* Files are laid out consecutively, so a file may start partway into a piece
|
|
457
|
+
* and may span several pieces. Each task records its first piece and the
|
|
458
|
+
* offset within that piece.
|
|
166
459
|
*/
|
|
167
460
|
_CreateFileTasks() {
|
|
168
461
|
// Variables to help calculate piece position.
|
|
@@ -188,19 +481,29 @@ class Hasher extends node_stream_1.Stream {
|
|
|
188
481
|
offset = totalSize - this.pieceLength * piece;
|
|
189
482
|
});
|
|
190
483
|
this.totalSize = totalSize;
|
|
191
|
-
this.pieces =
|
|
192
|
-
//
|
|
193
|
-
//
|
|
194
|
-
this.lastPieceLength =
|
|
484
|
+
this.pieces = Math.ceil(totalSize / this.pieceLength);
|
|
485
|
+
// The last piece holds the leftover bytes, or a full piece when the total
|
|
486
|
+
// size divides evenly. A zero-byte torrent has no pieces.
|
|
487
|
+
this.lastPieceLength =
|
|
488
|
+
this.pieces === 0 ? 0 : totalSize - this.pieceLength * (this.pieces - 1);
|
|
195
489
|
}
|
|
490
|
+
// ---------------------------------------------------------------------------
|
|
491
|
+
// Hashing
|
|
492
|
+
// ---------------------------------------------------------------------------
|
|
196
493
|
/**
|
|
197
|
-
*
|
|
494
|
+
* Hashes a fully planned file list in parallel.
|
|
495
|
+
*
|
|
496
|
+
* Only used when every input length is known, since piece offsets must be
|
|
497
|
+
* calculated before reading starts.
|
|
198
498
|
*/
|
|
199
499
|
_start() {
|
|
200
500
|
if (!this.ready) {
|
|
201
501
|
this.emit('error', new Error('Cannot start before being ready'));
|
|
202
502
|
return;
|
|
203
503
|
}
|
|
504
|
+
// A ready listener may have reported an error, such as no files found.
|
|
505
|
+
if (this.err)
|
|
506
|
+
return;
|
|
204
507
|
this.hashing = true;
|
|
205
508
|
this.stopped = false;
|
|
206
509
|
// Create piece objects.
|
|
@@ -233,25 +536,49 @@ class Hasher extends node_stream_1.Stream {
|
|
|
233
536
|
// Keep track how much has been read.
|
|
234
537
|
let readSoFar = 0;
|
|
235
538
|
let pieceIndex = task.piece;
|
|
539
|
+
let finished = false;
|
|
540
|
+
const finish = (err) => {
|
|
541
|
+
if (finished)
|
|
542
|
+
return;
|
|
543
|
+
finished = true;
|
|
544
|
+
if (err && !this.err)
|
|
545
|
+
this.emit('error', err);
|
|
546
|
+
callback();
|
|
547
|
+
};
|
|
236
548
|
// Open file for reading.
|
|
237
549
|
const baseDir = node_path_1.default.resolve(this.dir);
|
|
238
550
|
const file = node_path_1.default.resolve(baseDir, ...task.path);
|
|
239
551
|
const relative = node_path_1.default.relative(baseDir, file);
|
|
240
|
-
if (relative.startsWith('..') ||
|
|
241
|
-
|
|
242
|
-
|
|
552
|
+
if (relative.startsWith('..') ||
|
|
553
|
+
node_path_1.default.isAbsolute(relative) ||
|
|
554
|
+
!task.path.every((seg) => (0, schema_1.isValidPathSegment)(Buffer.from(seg)))) {
|
|
555
|
+
finish(new Error(`Path traversal detected: ${file}`));
|
|
556
|
+
return;
|
|
243
557
|
}
|
|
244
|
-
const rs = node_fs_1.default.createReadStream(file);
|
|
558
|
+
const rs = task.stream || node_fs_1.default.createReadStream(file);
|
|
245
559
|
this.streamGroup.add(rs);
|
|
246
560
|
rs.on('open', () => {
|
|
247
561
|
this.emit('open', file);
|
|
248
562
|
});
|
|
249
563
|
rs.on('data', (chunk) => {
|
|
564
|
+
if (finished)
|
|
565
|
+
return;
|
|
566
|
+
if (this.err) {
|
|
567
|
+
rs.destroy();
|
|
568
|
+
finish();
|
|
569
|
+
return;
|
|
570
|
+
}
|
|
250
571
|
const data = chunk;
|
|
251
|
-
|
|
572
|
+
if (readSoFar + data.length > task.length) {
|
|
573
|
+
rs.destroy();
|
|
574
|
+
finish(new Error(`Stream length mismatch for "${file}": expected ${task.length} bytes but received more`));
|
|
575
|
+
return;
|
|
576
|
+
}
|
|
577
|
+
// A chunk can begin partway through a piece and may span several.
|
|
578
|
+
// Recalculate the current piece from the task's starting offset.
|
|
252
579
|
pieceIndex =
|
|
253
580
|
task.piece + Math.floor((task.offset + readSoFar) / this.pieceLength);
|
|
254
|
-
// Write
|
|
581
|
+
// Write the chunk into its piece buffer(s).
|
|
255
582
|
const targetStart = (task.offset + readSoFar) % this.pieceLength;
|
|
256
583
|
let sourceWritten = this._writePiece(pieces, pieceIndex, targetStart, data, 0, data.length, file, readSoFar);
|
|
257
584
|
while (sourceWritten < data.length) {
|
|
@@ -270,16 +597,23 @@ class Hasher extends node_stream_1.Stream {
|
|
|
270
597
|
});
|
|
271
598
|
// Close this file when there is an error or finished reading.
|
|
272
599
|
rs.on('error', (err) => {
|
|
273
|
-
|
|
274
|
-
callback();
|
|
600
|
+
finish(err);
|
|
275
601
|
});
|
|
276
602
|
rs.on('end', () => {
|
|
277
|
-
|
|
603
|
+
if (finished)
|
|
604
|
+
return;
|
|
605
|
+
if (readSoFar !== task.length) {
|
|
606
|
+
finish(new Error(`Stream length mismatch for "${file}": expected ${task.length} bytes but received ${readSoFar}`));
|
|
607
|
+
return;
|
|
608
|
+
}
|
|
609
|
+
finish();
|
|
278
610
|
});
|
|
279
611
|
}, this.maxFiles);
|
|
280
612
|
// When all file queues are finished, join all the pieces.
|
|
281
613
|
filesqueue.drain = () => {
|
|
282
614
|
this.hashing = false;
|
|
615
|
+
if (this.err)
|
|
616
|
+
return;
|
|
283
617
|
this.emit('end');
|
|
284
618
|
};
|
|
285
619
|
// Start queueing jobs.
|
|
@@ -287,6 +621,101 @@ class Hasher extends node_stream_1.Stream {
|
|
|
287
621
|
filesqueue.push(filetask);
|
|
288
622
|
});
|
|
289
623
|
}
|
|
624
|
+
/**
|
|
625
|
+
* Hashes inputs one at a time in list order.
|
|
626
|
+
*
|
|
627
|
+
* Used whenever any stream length is unknown: later piece offsets cannot be
|
|
628
|
+
* planned until earlier inputs have been read, so parallelism is not possible.
|
|
629
|
+
* File entries are recorded as each input finishes, which lets `make` build
|
|
630
|
+
* metadata once the whole list has been consumed.
|
|
631
|
+
*/
|
|
632
|
+
_startSequential(items) {
|
|
633
|
+
this.hashing = true;
|
|
634
|
+
this.stopped = false;
|
|
635
|
+
this.streamGroup = new streamspeed_1.default();
|
|
636
|
+
this.totalSize = 0;
|
|
637
|
+
const pieceBuffer = Buffer.alloc(this.pieceLength);
|
|
638
|
+
let bytesWritten = 0;
|
|
639
|
+
let pieceIndex = 0;
|
|
640
|
+
const filesqueue = (0, queue_1.default)((item, callback) => {
|
|
641
|
+
if (this.err || this.stopped)
|
|
642
|
+
return callback();
|
|
643
|
+
let fileLength = 0;
|
|
644
|
+
let finished = false;
|
|
645
|
+
const file = node_path_1.default.join(this.dir, ...item.path);
|
|
646
|
+
const rs = 'stream' in item ? item.stream : node_fs_1.default.createReadStream(item.filepath);
|
|
647
|
+
this.streamGroup.add(rs);
|
|
648
|
+
const finish = (err) => {
|
|
649
|
+
if (finished)
|
|
650
|
+
return;
|
|
651
|
+
finished = true;
|
|
652
|
+
if (err && !this.err)
|
|
653
|
+
this.emit('error', err);
|
|
654
|
+
callback();
|
|
655
|
+
};
|
|
656
|
+
rs.on('data', (chunk) => {
|
|
657
|
+
if (finished)
|
|
658
|
+
return;
|
|
659
|
+
const data = chunk;
|
|
660
|
+
fileLength += data.length;
|
|
661
|
+
this.totalSize += data.length;
|
|
662
|
+
// Fill the current piece buffer. Whenever it fills, hash it and
|
|
663
|
+
// continue with the rest of the chunk in the next piece.
|
|
664
|
+
let offset = 0;
|
|
665
|
+
while (offset < data.length) {
|
|
666
|
+
const size = Math.min(data.length - offset, this.pieceLength - bytesWritten);
|
|
667
|
+
data.copy(pieceBuffer, bytesWritten, offset, offset + size);
|
|
668
|
+
bytesWritten += size;
|
|
669
|
+
offset += size;
|
|
670
|
+
if (bytesWritten === this.pieceLength) {
|
|
671
|
+
const buf = Buffer.from(sha1(pieceBuffer));
|
|
672
|
+
this.emit('hash', pieceIndex, buf, file, fileLength - data.length + offset - size, size);
|
|
673
|
+
pieceIndex++;
|
|
674
|
+
bytesWritten = 0;
|
|
675
|
+
}
|
|
676
|
+
}
|
|
677
|
+
});
|
|
678
|
+
rs.on('error', (err) => {
|
|
679
|
+
finish(err);
|
|
680
|
+
});
|
|
681
|
+
rs.on('end', () => {
|
|
682
|
+
if (finished)
|
|
683
|
+
return;
|
|
684
|
+
if (item.length !== undefined && fileLength !== item.length) {
|
|
685
|
+
finish(new Error(`Stream length mismatch for "${file}": expected ${item.length} bytes but received ${fileLength}`));
|
|
686
|
+
return;
|
|
687
|
+
}
|
|
688
|
+
this.files.push({
|
|
689
|
+
length: fileLength,
|
|
690
|
+
path: item.path,
|
|
691
|
+
});
|
|
692
|
+
finish();
|
|
693
|
+
});
|
|
694
|
+
}, 1);
|
|
695
|
+
filesqueue.drain = () => {
|
|
696
|
+
this.hashing = false;
|
|
697
|
+
if (this.err)
|
|
698
|
+
return;
|
|
699
|
+
// Hash the final partial piece, if there is one. When the total size
|
|
700
|
+
// divides evenly, the last full piece was already emitted above.
|
|
701
|
+
if (bytesWritten > 0) {
|
|
702
|
+
const lastSlice = pieceBuffer.subarray(0, bytesWritten);
|
|
703
|
+
const buf = Buffer.from(sha1(lastSlice));
|
|
704
|
+
const lastFile = node_path_1.default.join(this.dir, ...this.files[this.files.length - 1].path);
|
|
705
|
+
this.emit('hash', pieceIndex, buf, lastFile, 0, bytesWritten);
|
|
706
|
+
this.lastPieceLength = bytesWritten;
|
|
707
|
+
pieceIndex++;
|
|
708
|
+
}
|
|
709
|
+
else {
|
|
710
|
+
this.lastPieceLength = this.totalSize === 0 ? 0 : this.pieceLength;
|
|
711
|
+
}
|
|
712
|
+
this.pieces = pieceIndex;
|
|
713
|
+
this.emit('end');
|
|
714
|
+
};
|
|
715
|
+
items.forEach((item) => {
|
|
716
|
+
filesqueue.push(item);
|
|
717
|
+
});
|
|
718
|
+
}
|
|
290
719
|
_writePiece(pieces, index, offset, data, start, end, file, readSoFar) {
|
|
291
720
|
const piece = pieces[index];
|
|
292
721
|
if (!piece)
|