@push.rocks/smartarchive 5.0.1 → 5.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist_ts/00_commitinfo_data.js +2 -2
- package/dist_ts/classes.archiveanalyzer.d.ts +1 -1
- package/dist_ts/classes.archiveanalyzer.js +34 -4
- package/dist_ts/classes.smartarchive.d.ts +4 -4
- package/dist_ts/classes.smartarchive.js +45 -26
- package/dist_ts/classes.tartools.d.ts +74 -23
- package/dist_ts/classes.tartools.js +190 -139
- package/dist_ts/index.d.ts +2 -6
- package/dist_ts/index.js +7 -11
- package/dist_ts/paths.js +1 -1
- package/dist_ts/plugins.d.ts +4 -9
- package/dist_ts/plugins.js +9 -13
- package/{dist_ts → dist_ts_shared}/bzip2/bititerator.d.ts +1 -1
- package/dist_ts_shared/bzip2/bititerator.js +50 -0
- package/{dist_ts → dist_ts_shared}/bzip2/bzip2.d.ts +2 -2
- package/{dist_ts → dist_ts_shared}/bzip2/bzip2.js +1 -1
- package/{dist_ts → dist_ts_shared}/bzip2/index.d.ts +1 -1
- package/dist_ts_shared/bzip2/index.js +92 -0
- package/dist_ts_shared/classes.bzip2tools.d.ts +10 -0
- package/dist_ts_shared/classes.bzip2tools.js +14 -0
- package/dist_ts_shared/classes.gziptools.d.ts +26 -0
- package/dist_ts_shared/classes.gziptools.js +38 -0
- package/dist_ts_shared/classes.tartools.d.ts +30 -0
- package/dist_ts_shared/classes.tartools.js +78 -0
- package/dist_ts_shared/classes.ziptools.d.ts +31 -0
- package/dist_ts_shared/classes.ziptools.js +103 -0
- package/{dist_ts → dist_ts_shared}/errors.js +1 -1
- package/dist_ts_shared/index.d.ts +8 -0
- package/dist_ts_shared/index.js +14 -0
- package/{dist_ts → dist_ts_shared}/interfaces.d.ts +17 -7
- package/{dist_ts → dist_ts_shared}/interfaces.js +1 -1
- package/dist_ts_shared/plugins.d.ts +9 -0
- package/dist_ts_shared/plugins.js +14 -0
- package/dist_ts_web/00_commitinfo_data.d.ts +8 -0
- package/dist_ts_web/00_commitinfo_data.js +9 -0
- package/dist_ts_web/index.d.ts +1 -0
- package/dist_ts_web/index.js +4 -0
- package/dist_ts_web/plugins.d.ts +1 -0
- package/dist_ts_web/plugins.js +4 -0
- package/npmextra.json +13 -7
- package/package.json +24 -13
- package/readme.md +154 -46
- package/ts/00_commitinfo_data.ts +1 -1
- package/ts/classes.archiveanalyzer.ts +33 -5
- package/ts/classes.smartarchive.ts +48 -27
- package/ts/classes.tartools.ts +217 -153
- package/ts/index.ts +7 -11
- package/ts/plugins.ts +8 -16
- package/{ts → ts_shared}/bzip2/bititerator.ts +1 -1
- package/{ts → ts_shared}/bzip2/bzip2.ts +2 -2
- package/{ts → ts_shared}/bzip2/index.ts +6 -6
- package/ts_shared/classes.bzip2tools.ts +14 -0
- package/ts_shared/classes.gziptools.ts +42 -0
- package/ts_shared/classes.tartools.ts +89 -0
- package/ts_shared/classes.ziptools.ts +107 -0
- package/ts_shared/index.ts +17 -0
- package/{ts → ts_shared}/interfaces.ts +18 -7
- package/ts_shared/plugins.ts +22 -0
- package/ts_web/00_commitinfo_data.ts +8 -0
- package/ts_web/index.ts +4 -0
- package/ts_web/plugins.ts +3 -0
- package/dist_ts/bzip2/bititerator.js +0 -50
- package/dist_ts/bzip2/index.js +0 -92
- package/dist_ts/classes.bzip2tools.d.ts +0 -7
- package/dist_ts/classes.bzip2tools.js +0 -11
- package/dist_ts/classes.gziptools.d.ts +0 -53
- package/dist_ts/classes.gziptools.js +0 -116
- package/dist_ts/classes.ziptools.d.ts +0 -56
- package/dist_ts/classes.ziptools.js +0 -171
- package/dist_ts/smartarchive.classes.smartarchive.d.ts +0 -34
- package/dist_ts/smartarchive.classes.smartarchive.js +0 -116
- package/dist_ts/smartarchive.paths.d.ts +0 -2
- package/dist_ts/smartarchive.paths.js +0 -4
- package/dist_ts/smartarchive.plugins.d.ts +0 -14
- package/dist_ts/smartarchive.plugins.js +0 -19
- package/ts/classes.bzip2tools.ts +0 -16
- package/ts/classes.gziptools.ts +0 -138
- package/ts/classes.ziptools.ts +0 -196
- /package/{dist_ts → dist_ts_shared}/errors.d.ts +0 -0
- /package/{ts → ts_shared}/errors.ts +0 -0
|
@@ -6,12 +6,15 @@ import type {
|
|
|
6
6
|
TArchiveFormat,
|
|
7
7
|
TCompressionLevel,
|
|
8
8
|
TEntryFilter,
|
|
9
|
-
} from '
|
|
9
|
+
} from '../ts_shared/interfaces.js';
|
|
10
10
|
|
|
11
|
-
|
|
12
|
-
import {
|
|
11
|
+
// Import browser-compatible tools from ts_shared
|
|
12
|
+
import { Bzip2Tools } from '../ts_shared/classes.bzip2tools.js';
|
|
13
|
+
import { GzipTools } from '../ts_shared/classes.gziptools.js';
|
|
14
|
+
import { ZipTools } from '../ts_shared/classes.ziptools.js';
|
|
15
|
+
|
|
16
|
+
// Import Node.js-extended TarTools
|
|
13
17
|
import { TarTools } from './classes.tartools.js';
|
|
14
|
-
import { ZipTools } from './classes.ziptools.js';
|
|
15
18
|
import { ArchiveAnalyzer, type IAnalyzedResult } from './classes.archiveanalyzer.js';
|
|
16
19
|
|
|
17
20
|
/**
|
|
@@ -62,7 +65,7 @@ export class SmartArchive {
|
|
|
62
65
|
public tarTools = new TarTools();
|
|
63
66
|
public zipTools = new ZipTools();
|
|
64
67
|
public gzipTools = new GzipTools();
|
|
65
|
-
public bzip2Tools = new Bzip2Tools(
|
|
68
|
+
public bzip2Tools = new Bzip2Tools();
|
|
66
69
|
public archiveAnalyzer = new ArchiveAnalyzer(this);
|
|
67
70
|
|
|
68
71
|
// ============================================
|
|
@@ -173,7 +176,7 @@ export class SmartArchive {
|
|
|
173
176
|
public entry(archivePath: string, content: string | Buffer): this {
|
|
174
177
|
this.ensureNotInExtractMode('entry');
|
|
175
178
|
if (!this._mode) this._mode = 'create';
|
|
176
|
-
this.pendingEntries.push({ archivePath, content });
|
|
179
|
+
this.pendingEntries.push({ archivePath, content: content instanceof Buffer ? new Uint8Array(content) : content });
|
|
177
180
|
return this;
|
|
178
181
|
}
|
|
179
182
|
|
|
@@ -184,7 +187,10 @@ export class SmartArchive {
|
|
|
184
187
|
this.ensureNotInExtractMode('entries');
|
|
185
188
|
if (!this._mode) this._mode = 'create';
|
|
186
189
|
for (const e of entriesArg) {
|
|
187
|
-
this.pendingEntries.push({
|
|
190
|
+
this.pendingEntries.push({
|
|
191
|
+
archivePath: e.archivePath,
|
|
192
|
+
content: e.content instanceof Buffer ? new Uint8Array(e.content) : e.content
|
|
193
|
+
});
|
|
188
194
|
}
|
|
189
195
|
return this;
|
|
190
196
|
}
|
|
@@ -374,30 +380,41 @@ export class SmartArchive {
|
|
|
374
380
|
plugins.smartstream.createTransformFunction<IAnalyzedResult, void>(
|
|
375
381
|
async (analyzedResultChunk) => {
|
|
376
382
|
if (analyzedResultChunk.fileType?.mime === 'application/x-tar') {
|
|
377
|
-
|
|
383
|
+
// Use tar-stream for streaming TAR extraction
|
|
384
|
+
// Buffer each entry to ensure tar-stream can proceed to next entry
|
|
385
|
+
const extract = this.tarTools.getExtractStream();
|
|
378
386
|
|
|
379
|
-
|
|
387
|
+
extract.on('entry', (header, stream, next) => {
|
|
380
388
|
if (header.type === 'directory') {
|
|
381
|
-
stream.resume();
|
|
382
|
-
|
|
389
|
+
stream.resume(); // Drain the stream
|
|
390
|
+
next();
|
|
383
391
|
return;
|
|
384
392
|
}
|
|
385
393
|
|
|
386
|
-
|
|
387
|
-
const
|
|
388
|
-
|
|
389
|
-
stream.pipe(passThrough);
|
|
394
|
+
// Buffer the entry content to avoid blocking tar-stream
|
|
395
|
+
const chunks: Buffer[] = [];
|
|
396
|
+
stream.on('data', (chunk: Buffer) => chunks.push(chunk));
|
|
390
397
|
stream.on('end', () => {
|
|
391
|
-
|
|
398
|
+
const content = Buffer.concat(chunks);
|
|
399
|
+
const streamFile = plugins.smartfile.StreamFile.fromBuffer(content);
|
|
400
|
+
streamFile.relativeFilePath = header.name;
|
|
401
|
+
streamFileIntake.push(streamFile);
|
|
392
402
|
next();
|
|
393
403
|
});
|
|
404
|
+
stream.on('error', (err: Error) => {
|
|
405
|
+
streamFileIntake.emit('error', err);
|
|
406
|
+
});
|
|
394
407
|
});
|
|
395
408
|
|
|
396
|
-
|
|
409
|
+
extract.on('finish', () => {
|
|
397
410
|
safeSignalEnd();
|
|
398
411
|
});
|
|
399
412
|
|
|
400
|
-
|
|
413
|
+
extract.on('error', (err: Error) => {
|
|
414
|
+
streamFileIntake.emit('error', err);
|
|
415
|
+
});
|
|
416
|
+
|
|
417
|
+
analyzedResultChunk.resultStream.pipe(extract);
|
|
401
418
|
} else if (analyzedResultChunk.fileType?.mime === 'application/zip') {
|
|
402
419
|
analyzedResultChunk.resultStream
|
|
403
420
|
.pipe(analyzedResultChunk.decompressionStream)
|
|
@@ -544,25 +561,29 @@ export class SmartArchive {
|
|
|
544
561
|
|
|
545
562
|
if (this.creationFormat === 'tar' || this.creationFormat === 'tar.gz' || this.creationFormat === 'tgz') {
|
|
546
563
|
if (this.creationFormat === 'tar') {
|
|
547
|
-
|
|
564
|
+
const result = await this.tarTools.packFiles(entries);
|
|
565
|
+
this.archiveBuffer = Buffer.from(result);
|
|
548
566
|
} else {
|
|
549
|
-
|
|
567
|
+
const result = await this.tarTools.packFilesToTarGz(entries, this._compressionLevel);
|
|
568
|
+
this.archiveBuffer = Buffer.from(result);
|
|
550
569
|
}
|
|
551
570
|
} else if (this.creationFormat === 'zip') {
|
|
552
|
-
|
|
571
|
+
const result = await this.zipTools.createZip(entries, this._compressionLevel);
|
|
572
|
+
this.archiveBuffer = Buffer.from(result);
|
|
553
573
|
} else if (this.creationFormat === 'gz') {
|
|
554
574
|
if (entries.length !== 1) {
|
|
555
575
|
throw new Error('GZIP format only supports a single file');
|
|
556
576
|
}
|
|
557
|
-
let content:
|
|
577
|
+
let content: Uint8Array;
|
|
558
578
|
if (typeof entries[0].content === 'string') {
|
|
559
|
-
content =
|
|
560
|
-
} else if (
|
|
579
|
+
content = new TextEncoder().encode(entries[0].content);
|
|
580
|
+
} else if (entries[0].content instanceof Uint8Array) {
|
|
561
581
|
content = entries[0].content;
|
|
562
582
|
} else {
|
|
563
|
-
throw new Error('GZIP format requires string or
|
|
583
|
+
throw new Error('GZIP format requires string or Uint8Array content');
|
|
564
584
|
}
|
|
565
|
-
|
|
585
|
+
const result = await this.gzipTools.compress(content, this._compressionLevel);
|
|
586
|
+
this.archiveBuffer = Buffer.from(result);
|
|
566
587
|
} else {
|
|
567
588
|
throw new Error(`Unsupported format: ${this.creationFormat}`);
|
|
568
589
|
}
|
|
@@ -808,7 +829,7 @@ export class SmartArchive {
|
|
|
808
829
|
const content = await plugins.fsPromises.readFile(absolutePath);
|
|
809
830
|
this.pendingEntries.push({
|
|
810
831
|
archivePath,
|
|
811
|
-
content,
|
|
832
|
+
content: new Uint8Array(content),
|
|
812
833
|
});
|
|
813
834
|
}
|
|
814
835
|
}
|
package/ts/classes.tartools.ts
CHANGED
|
@@ -1,208 +1,272 @@
|
|
|
1
1
|
import * as plugins from './plugins.js';
|
|
2
|
-
import type { IArchiveEntry, TCompressionLevel } from '
|
|
3
|
-
import {
|
|
2
|
+
import type { IArchiveEntry, TCompressionLevel } from '../ts_shared/interfaces.js';
|
|
3
|
+
import { TarTools as SharedTarTools } from '../ts_shared/classes.tartools.js';
|
|
4
4
|
|
|
5
5
|
/**
|
|
6
|
-
*
|
|
6
|
+
* Options for adding a file to a TAR pack stream
|
|
7
7
|
*/
|
|
8
|
-
export
|
|
8
|
+
export interface ITarPackFileOptions {
|
|
9
|
+
fileName: string;
|
|
10
|
+
content: string | Buffer | Uint8Array | plugins.stream.Readable;
|
|
11
|
+
size?: number;
|
|
12
|
+
mode?: number;
|
|
13
|
+
mtime?: Date;
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
/**
|
|
17
|
+
* Extended TAR archive utilities with Node.js streaming support
|
|
18
|
+
*
|
|
19
|
+
* For small archives: Use inherited buffer-based methods (packFiles, extractTar, etc.)
|
|
20
|
+
* For large archives: Use streaming methods (getPackStream, getExtractStream, etc.)
|
|
21
|
+
*/
|
|
22
|
+
export class TarTools extends SharedTarTools {
|
|
23
|
+
// ============================================
|
|
24
|
+
// STREAMING PACK METHODS (for large files)
|
|
25
|
+
// ============================================
|
|
26
|
+
|
|
27
|
+
/**
|
|
28
|
+
* Get a streaming TAR pack instance
|
|
29
|
+
* Use this for packing large files without buffering everything in memory
|
|
30
|
+
*
|
|
31
|
+
* @example
|
|
32
|
+
* ```typescript
|
|
33
|
+
* const pack = tarTools.getPackStream();
|
|
34
|
+
*
|
|
35
|
+
* await tarTools.addFileToPack(pack, { fileName: 'large.bin', content: readStream, size: fileSize });
|
|
36
|
+
* await tarTools.addFileToPack(pack, { fileName: 'small.txt', content: 'Hello World' });
|
|
37
|
+
*
|
|
38
|
+
* pack.finalize();
|
|
39
|
+
* pack.pipe(fs.createWriteStream('output.tar'));
|
|
40
|
+
* ```
|
|
41
|
+
*/
|
|
42
|
+
public getPackStream(): plugins.tarStream.Pack {
|
|
43
|
+
return plugins.tarStream.pack();
|
|
44
|
+
}
|
|
45
|
+
|
|
9
46
|
/**
|
|
10
47
|
* Add a file to a TAR pack stream
|
|
48
|
+
* Supports strings, buffers, and readable streams
|
|
49
|
+
*
|
|
50
|
+
* @param pack - The pack stream from getPackStream()
|
|
51
|
+
* @param options - File options including name, content, and optional metadata
|
|
11
52
|
*/
|
|
12
53
|
public async addFileToPack(
|
|
13
54
|
pack: plugins.tarStream.Pack,
|
|
14
|
-
|
|
15
|
-
fileName?: string;
|
|
16
|
-
content?:
|
|
17
|
-
| string
|
|
18
|
-
| Buffer
|
|
19
|
-
| plugins.stream.Readable
|
|
20
|
-
| plugins.smartfile.SmartFile
|
|
21
|
-
| plugins.smartfile.StreamFile;
|
|
22
|
-
byteLength?: number;
|
|
23
|
-
filePath?: string;
|
|
24
|
-
}
|
|
55
|
+
options: ITarPackFileOptions
|
|
25
56
|
): Promise<void> {
|
|
26
|
-
|
|
27
|
-
let fileName: string | null = null;
|
|
28
|
-
|
|
29
|
-
if (optionsArg.fileName) {
|
|
30
|
-
fileName = optionsArg.fileName;
|
|
31
|
-
} else if (optionsArg.content instanceof plugins.smartfile.SmartFile) {
|
|
32
|
-
fileName = optionsArg.content.relative;
|
|
33
|
-
} else if (optionsArg.content instanceof plugins.smartfile.StreamFile) {
|
|
34
|
-
fileName = optionsArg.content.relativeFilePath;
|
|
35
|
-
} else if (optionsArg.filePath) {
|
|
36
|
-
fileName = optionsArg.filePath;
|
|
37
|
-
}
|
|
57
|
+
const { fileName, content, mode = 0o644, mtime = new Date() } = options;
|
|
38
58
|
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
59
|
+
if (typeof content === 'string') {
|
|
60
|
+
// String content - convert to buffer
|
|
61
|
+
const buffer = Buffer.from(content, 'utf8');
|
|
62
|
+
const entry = pack.entry({
|
|
63
|
+
name: fileName,
|
|
64
|
+
size: buffer.length,
|
|
65
|
+
mode,
|
|
66
|
+
mtime,
|
|
67
|
+
});
|
|
68
|
+
entry.write(buffer);
|
|
69
|
+
entry.end();
|
|
70
|
+
} else if (Buffer.isBuffer(content) || content instanceof Uint8Array) {
|
|
71
|
+
// Buffer content
|
|
72
|
+
const buffer = Buffer.isBuffer(content) ? content : Buffer.from(content);
|
|
73
|
+
const entry = pack.entry({
|
|
74
|
+
name: fileName,
|
|
75
|
+
size: buffer.length,
|
|
76
|
+
mode,
|
|
77
|
+
mtime,
|
|
78
|
+
});
|
|
79
|
+
entry.write(buffer);
|
|
80
|
+
entry.end();
|
|
81
|
+
} else if (content && typeof (content as any).pipe === 'function') {
|
|
82
|
+
// Readable stream - requires size to be provided
|
|
83
|
+
const size = options.size;
|
|
84
|
+
if (size === undefined) {
|
|
85
|
+
throw new Error('Size must be provided when adding a stream to TAR pack');
|
|
42
86
|
}
|
|
43
87
|
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
} else if (optionsArg.content instanceof plugins.smartfile.StreamFile) {
|
|
55
|
-
contentByteLength = await optionsArg.content.getSize();
|
|
56
|
-
} else if (optionsArg.filePath) {
|
|
57
|
-
const fileStat = await plugins.fsPromises.stat(optionsArg.filePath);
|
|
58
|
-
contentByteLength = fileStat.size;
|
|
59
|
-
}
|
|
88
|
+
return new Promise<void>((resolve, reject) => {
|
|
89
|
+
const entry = pack.entry({
|
|
90
|
+
name: fileName,
|
|
91
|
+
size,
|
|
92
|
+
mode,
|
|
93
|
+
mtime,
|
|
94
|
+
}, (err) => {
|
|
95
|
+
if (err) reject(err);
|
|
96
|
+
else resolve();
|
|
97
|
+
});
|
|
60
98
|
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
} else if (optionsArg.content instanceof plugins.smartfile.SmartFile) {
|
|
68
|
-
content = plugins.stream.Readable.from(optionsArg.content.contents);
|
|
69
|
-
} else if (optionsArg.content instanceof plugins.smartfile.StreamFile) {
|
|
70
|
-
content = await optionsArg.content.createReadStream();
|
|
71
|
-
} else if (optionsArg.content instanceof plugins.stream.Readable) {
|
|
72
|
-
content = optionsArg.content;
|
|
73
|
-
} else if (optionsArg.filePath) {
|
|
74
|
-
content = plugins.fs.createReadStream(optionsArg.filePath);
|
|
75
|
-
} else {
|
|
76
|
-
reject(new Error('No content or filePath specified for TAR entry'));
|
|
77
|
-
return;
|
|
78
|
-
}
|
|
99
|
+
(content as plugins.stream.Readable).pipe(entry);
|
|
100
|
+
});
|
|
101
|
+
} else {
|
|
102
|
+
throw new Error('Unsupported content type for TAR entry');
|
|
103
|
+
}
|
|
104
|
+
}
|
|
79
105
|
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
...(contentByteLength !== undefined ? { size: contentByteLength } : {}),
|
|
84
|
-
},
|
|
85
|
-
(err: Error | null) => {
|
|
86
|
-
if (err) {
|
|
87
|
-
reject(err);
|
|
88
|
-
} else {
|
|
89
|
-
resolve();
|
|
90
|
-
}
|
|
91
|
-
}
|
|
92
|
-
);
|
|
106
|
+
// ============================================
|
|
107
|
+
// STREAMING EXTRACT METHODS (for large files)
|
|
108
|
+
// ============================================
|
|
93
109
|
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
110
|
+
/**
|
|
111
|
+
* Get a streaming TAR extract instance
|
|
112
|
+
* Use this for extracting large archives without buffering everything in memory
|
|
113
|
+
*
|
|
114
|
+
* @example
|
|
115
|
+
* ```typescript
|
|
116
|
+
* const extract = tarTools.getExtractStream();
|
|
117
|
+
*
|
|
118
|
+
* extract.on('entry', (header, stream, next) => {
|
|
119
|
+
* console.log(`Extracting: ${header.name}`);
|
|
120
|
+
* stream.pipe(fs.createWriteStream(`./out/${header.name}`));
|
|
121
|
+
* stream.on('end', next);
|
|
122
|
+
* });
|
|
123
|
+
*
|
|
124
|
+
* fs.createReadStream('archive.tar').pipe(extract);
|
|
125
|
+
* ```
|
|
126
|
+
*/
|
|
127
|
+
public getExtractStream(): plugins.tarStream.Extract {
|
|
128
|
+
return plugins.tarStream.extract();
|
|
97
129
|
}
|
|
98
130
|
|
|
99
131
|
/**
|
|
100
|
-
*
|
|
132
|
+
* Extract a TAR stream to a directory with true streaming (no buffering)
|
|
133
|
+
*
|
|
134
|
+
* @param sourceStream - The TAR archive stream
|
|
135
|
+
* @param targetDir - Directory to extract files to
|
|
101
136
|
*/
|
|
102
|
-
public async
|
|
103
|
-
|
|
104
|
-
|
|
137
|
+
public async extractToDirectory(
|
|
138
|
+
sourceStream: plugins.stream.Readable,
|
|
139
|
+
targetDir: string
|
|
140
|
+
): Promise<void> {
|
|
141
|
+
await plugins.fsPromises.mkdir(targetDir, { recursive: true });
|
|
105
142
|
|
|
106
|
-
|
|
107
|
-
const
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
143
|
+
return new Promise<void>((resolve, reject) => {
|
|
144
|
+
const extract = this.getExtractStream();
|
|
145
|
+
|
|
146
|
+
extract.on('entry', async (header, stream, next) => {
|
|
147
|
+
const filePath = plugins.path.join(targetDir, header.name);
|
|
148
|
+
|
|
149
|
+
if (header.type === 'directory') {
|
|
150
|
+
await plugins.fsPromises.mkdir(filePath, { recursive: true });
|
|
151
|
+
stream.resume(); // Drain the stream
|
|
152
|
+
next();
|
|
153
|
+
} else if (header.type === 'file') {
|
|
154
|
+
await plugins.fsPromises.mkdir(plugins.path.dirname(filePath), { recursive: true });
|
|
155
|
+
const writeStream = plugins.fs.createWriteStream(filePath);
|
|
156
|
+
stream.pipe(writeStream);
|
|
157
|
+
writeStream.on('finish', next);
|
|
158
|
+
writeStream.on('error', reject);
|
|
159
|
+
} else {
|
|
160
|
+
stream.resume(); // Skip other types
|
|
161
|
+
next();
|
|
162
|
+
}
|
|
114
163
|
});
|
|
115
|
-
}
|
|
116
164
|
|
|
117
|
-
|
|
165
|
+
extract.on('finish', resolve);
|
|
166
|
+
extract.on('error', reject);
|
|
167
|
+
|
|
168
|
+
sourceStream.pipe(extract);
|
|
169
|
+
});
|
|
118
170
|
}
|
|
119
171
|
|
|
172
|
+
// ============================================
|
|
173
|
+
// STREAMING DIRECTORY PACK (for large directories)
|
|
174
|
+
// ============================================
|
|
175
|
+
|
|
120
176
|
/**
|
|
121
|
-
*
|
|
177
|
+
* Pack a directory into a TAR stream with true streaming (no buffering)
|
|
178
|
+
* Files are read and written one at a time, never loading everything into memory
|
|
122
179
|
*/
|
|
123
|
-
public async
|
|
124
|
-
|
|
180
|
+
public async getDirectoryPackStream(directoryPath: string): Promise<plugins.tarStream.Pack> {
|
|
181
|
+
const pack = this.getPackStream();
|
|
182
|
+
const fileTree = await plugins.listFileTree(directoryPath, '**/*');
|
|
183
|
+
|
|
184
|
+
// Process files sequentially to avoid memory issues
|
|
185
|
+
(async () => {
|
|
186
|
+
for (const filePath of fileTree) {
|
|
187
|
+
const absolutePath = plugins.path.join(directoryPath, filePath);
|
|
188
|
+
const stat = await plugins.fsPromises.stat(absolutePath);
|
|
189
|
+
|
|
190
|
+
if (stat.isFile()) {
|
|
191
|
+
const readStream = plugins.fs.createReadStream(absolutePath);
|
|
192
|
+
await this.addFileToPack(pack, {
|
|
193
|
+
fileName: filePath,
|
|
194
|
+
content: readStream,
|
|
195
|
+
size: stat.size,
|
|
196
|
+
mode: stat.mode,
|
|
197
|
+
mtime: stat.mtime,
|
|
198
|
+
});
|
|
199
|
+
}
|
|
200
|
+
}
|
|
201
|
+
pack.finalize();
|
|
202
|
+
})().catch((err) => pack.destroy(err));
|
|
203
|
+
|
|
204
|
+
return pack;
|
|
125
205
|
}
|
|
126
206
|
|
|
127
207
|
/**
|
|
128
|
-
*
|
|
208
|
+
* Pack a directory into a TAR.GZ stream with true streaming
|
|
209
|
+
* Uses Node.js zlib for streaming compression
|
|
129
210
|
*/
|
|
130
|
-
public
|
|
131
|
-
|
|
211
|
+
public async getDirectoryPackStreamGz(
|
|
212
|
+
directoryPath: string,
|
|
213
|
+
compressionLevel?: TCompressionLevel
|
|
214
|
+
): Promise<plugins.stream.Readable> {
|
|
215
|
+
const tarStream = await this.getDirectoryPackStream(directoryPath);
|
|
216
|
+
const { createGzip } = await import('node:zlib');
|
|
217
|
+
const gzip = createGzip({ level: compressionLevel ?? 6 });
|
|
218
|
+
return tarStream.pipe(gzip);
|
|
132
219
|
}
|
|
133
220
|
|
|
221
|
+
// ============================================
|
|
222
|
+
// BUFFER-BASED METHODS (inherited + filesystem)
|
|
223
|
+
// ============================================
|
|
224
|
+
|
|
134
225
|
/**
|
|
135
|
-
* Pack
|
|
226
|
+
* Pack a directory into a TAR buffer (loads all files into memory)
|
|
227
|
+
* For large directories, use getDirectoryPackStream() instead
|
|
136
228
|
*/
|
|
137
|
-
public async
|
|
138
|
-
const
|
|
139
|
-
|
|
140
|
-
for (const file of files) {
|
|
141
|
-
await this.addFileToPack(pack, {
|
|
142
|
-
fileName: file.archivePath,
|
|
143
|
-
content: file.content as string | Buffer | plugins.stream.Readable | plugins.smartfile.SmartFile | plugins.smartfile.StreamFile,
|
|
144
|
-
byteLength: file.size,
|
|
145
|
-
});
|
|
146
|
-
}
|
|
229
|
+
public async packDirectory(directoryPath: string): Promise<Uint8Array> {
|
|
230
|
+
const fileTree = await plugins.listFileTree(directoryPath, '**/*');
|
|
231
|
+
const entries: IArchiveEntry[] = [];
|
|
147
232
|
|
|
148
|
-
|
|
233
|
+
for (const filePath of fileTree) {
|
|
234
|
+
const absolutePath = plugins.path.join(directoryPath, filePath);
|
|
235
|
+
const stat = await plugins.fsPromises.stat(absolutePath);
|
|
149
236
|
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
237
|
+
if (stat.isFile()) {
|
|
238
|
+
const content = await plugins.fsPromises.readFile(absolutePath);
|
|
239
|
+
entries.push({
|
|
240
|
+
archivePath: filePath,
|
|
241
|
+
content: new Uint8Array(content),
|
|
242
|
+
});
|
|
243
|
+
}
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
return this.packFiles(entries);
|
|
156
247
|
}
|
|
157
248
|
|
|
158
249
|
/**
|
|
159
|
-
* Pack a directory into a TAR.GZ buffer
|
|
250
|
+
* Pack a directory into a TAR.GZ buffer (loads all files into memory)
|
|
251
|
+
* For large directories, use getDirectoryPackStreamGz() instead
|
|
160
252
|
*/
|
|
161
253
|
public async packDirectoryToTarGz(
|
|
162
254
|
directoryPath: string,
|
|
163
255
|
compressionLevel?: TCompressionLevel
|
|
164
|
-
): Promise<
|
|
165
|
-
const
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
const gzipTools = new GzipTools();
|
|
169
|
-
const gzipStream = gzipTools.getCompressionStream(compressionLevel);
|
|
170
|
-
|
|
171
|
-
const chunks: Buffer[] = [];
|
|
172
|
-
return new Promise((resolve, reject) => {
|
|
173
|
-
pack
|
|
174
|
-
.pipe(gzipStream)
|
|
175
|
-
.on('data', (chunk: Buffer) => chunks.push(chunk))
|
|
176
|
-
.on('end', () => resolve(Buffer.concat(chunks)))
|
|
177
|
-
.on('error', reject);
|
|
178
|
-
});
|
|
256
|
+
): Promise<Uint8Array> {
|
|
257
|
+
const tarBuffer = await this.packDirectory(directoryPath);
|
|
258
|
+
const { gzipSync } = await import('fflate');
|
|
259
|
+
return gzipSync(new Uint8Array(tarBuffer), { level: compressionLevel ?? 6 });
|
|
179
260
|
}
|
|
180
261
|
|
|
181
262
|
/**
|
|
182
263
|
* Pack a directory into a TAR.GZ stream
|
|
264
|
+
* This is now a true streaming implementation
|
|
183
265
|
*/
|
|
184
266
|
public async packDirectoryToTarGzStream(
|
|
185
267
|
directoryPath: string,
|
|
186
268
|
compressionLevel?: TCompressionLevel
|
|
187
269
|
): Promise<plugins.stream.Readable> {
|
|
188
|
-
|
|
189
|
-
pack.finalize();
|
|
190
|
-
|
|
191
|
-
const gzipTools = new GzipTools();
|
|
192
|
-
const gzipStream = gzipTools.getCompressionStream(compressionLevel);
|
|
193
|
-
|
|
194
|
-
return pack.pipe(gzipStream);
|
|
195
|
-
}
|
|
196
|
-
|
|
197
|
-
/**
|
|
198
|
-
* Pack files into a TAR.GZ buffer
|
|
199
|
-
*/
|
|
200
|
-
public async packFilesToTarGz(
|
|
201
|
-
files: IArchiveEntry[],
|
|
202
|
-
compressionLevel?: TCompressionLevel
|
|
203
|
-
): Promise<Buffer> {
|
|
204
|
-
const tarBuffer = await this.packFiles(files);
|
|
205
|
-
const gzipTools = new GzipTools();
|
|
206
|
-
return gzipTools.compress(tarBuffer, compressionLevel);
|
|
270
|
+
return this.getDirectoryPackStreamGz(directoryPath, compressionLevel);
|
|
207
271
|
}
|
|
208
272
|
}
|
package/ts/index.ts
CHANGED
|
@@ -1,15 +1,11 @@
|
|
|
1
|
-
//
|
|
2
|
-
export * from '
|
|
3
|
-
export * from './errors.js';
|
|
1
|
+
// Re-export everything from ts_shared (browser-compatible)
|
|
2
|
+
export * from '../ts_shared/index.js';
|
|
4
3
|
|
|
5
|
-
// Main archive class
|
|
4
|
+
// Node.js-specific: Main archive class with filesystem support
|
|
6
5
|
export * from './classes.smartarchive.js';
|
|
7
6
|
|
|
8
|
-
//
|
|
9
|
-
export * from './classes.tartools.js';
|
|
10
|
-
export * from './classes.ziptools.js';
|
|
11
|
-
export * from './classes.gziptools.js';
|
|
12
|
-
export * from './classes.bzip2tools.js';
|
|
13
|
-
|
|
14
|
-
// Archive analysis
|
|
7
|
+
// Node.js-specific: Archive analysis with SmartArchive integration
|
|
15
8
|
export * from './classes.archiveanalyzer.js';
|
|
9
|
+
|
|
10
|
+
// Node.js-specific: Extended TarTools with streaming support (overrides shared TarTools)
|
|
11
|
+
export { TarTools, type ITarPackFileOptions } from './classes.tartools.js';
|
package/ts/plugins.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
//
|
|
1
|
+
// Node.js native scope
|
|
2
2
|
import * as path from 'node:path';
|
|
3
3
|
import * as stream from 'node:stream';
|
|
4
4
|
import * as fs from 'node:fs';
|
|
@@ -30,32 +30,24 @@ export async function listFileTree(dirPath: string, _pattern: string = '**/*'):
|
|
|
30
30
|
return results;
|
|
31
31
|
}
|
|
32
32
|
|
|
33
|
-
//
|
|
34
|
-
|
|
35
|
-
|
|
33
|
+
// Re-export browser-compatible plugins from ts_shared
|
|
34
|
+
export * from '../ts_shared/plugins.js';
|
|
35
|
+
|
|
36
|
+
// Additional Node.js-specific @pushrocks packages
|
|
36
37
|
import * as smartpath from '@push.rocks/smartpath';
|
|
37
|
-
import * as smartpromise from '@push.rocks/smartpromise';
|
|
38
38
|
import * as smartrequest from '@push.rocks/smartrequest';
|
|
39
39
|
import * as smartunique from '@push.rocks/smartunique';
|
|
40
|
-
import * as smartstream from '@push.rocks/smartstream';
|
|
41
40
|
import * as smartrx from '@push.rocks/smartrx';
|
|
42
41
|
import * as smarturl from '@push.rocks/smarturl';
|
|
43
42
|
|
|
44
43
|
export {
|
|
45
|
-
smartfile,
|
|
46
|
-
smartdelay,
|
|
47
44
|
smartpath,
|
|
48
|
-
smartpromise,
|
|
49
45
|
smartrequest,
|
|
50
46
|
smartunique,
|
|
51
|
-
smartstream,
|
|
52
47
|
smartrx,
|
|
53
48
|
smarturl,
|
|
54
49
|
};
|
|
55
50
|
|
|
56
|
-
//
|
|
57
|
-
import * as
|
|
58
|
-
|
|
59
|
-
import tarStream from 'tar-stream';
|
|
60
|
-
|
|
61
|
-
export { fileType, fflate, tarStream };
|
|
51
|
+
// Node.js-specific: tar-stream for true streaming TAR support
|
|
52
|
+
import * as tarStream from 'tar-stream';
|
|
53
|
+
export { tarStream };
|
|
@@ -6,7 +6,7 @@ const BITMASK = [0, 0x01, 0x03, 0x07, 0x0f, 0x1f, 0x3f, 0x7f, 0xff] as const;
|
|
|
6
6
|
* Creates a bit reader function for BZIP2 decompression.
|
|
7
7
|
* Takes a buffer iterator as input and returns a function that reads bits.
|
|
8
8
|
*/
|
|
9
|
-
export function bitIterator(nextBuffer: () =>
|
|
9
|
+
export function bitIterator(nextBuffer: () => Uint8Array): IBitReader {
|
|
10
10
|
let bit = 0;
|
|
11
11
|
let byte = 0;
|
|
12
12
|
let bytes = nextBuffer();
|
|
@@ -71,7 +71,7 @@ export class Bzip2 {
|
|
|
71
71
|
/**
|
|
72
72
|
* Create a bit reader from a byte array
|
|
73
73
|
*/
|
|
74
|
-
array(bytes: Uint8Array
|
|
74
|
+
array(bytes: Uint8Array): (n: number) => number {
|
|
75
75
|
let bit = 0;
|
|
76
76
|
let byte = 0;
|
|
77
77
|
const BITMASK = [0, 0x01, 0x03, 0x07, 0x0f, 0x1f, 0x3f, 0x7f, 0xff];
|
|
@@ -99,7 +99,7 @@ export class Bzip2 {
|
|
|
99
99
|
/**
|
|
100
100
|
* Simple decompression from a buffer
|
|
101
101
|
*/
|
|
102
|
-
simple(srcbuffer: Uint8Array
|
|
102
|
+
simple(srcbuffer: Uint8Array, stream: (byte: number) => void): void {
|
|
103
103
|
const bits = this.array(srcbuffer);
|
|
104
104
|
const size = this.header(bits as IBitReader);
|
|
105
105
|
let ret: number | null = 0;
|