marcattacks 2.1.0 → 2.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +36 -12
- package/dist/attacker.d.ts +6 -0
- package/dist/attacker.d.ts.map +1 -0
- package/dist/attacker.js +198 -0
- package/dist/attacker.js.map +1 -0
- package/dist/filestream.d.ts +4 -0
- package/dist/filestream.d.ts.map +1 -0
- package/dist/filestream.js +32 -0
- package/dist/filestream.js.map +1 -0
- package/dist/index.js +19 -164
- package/dist/index.js.map +1 -1
- package/dist/input/tsv.d.ts +3 -0
- package/dist/input/tsv.d.ts.map +1 -0
- package/dist/input/tsv.js +45 -0
- package/dist/input/tsv.js.map +1 -0
- package/dist/output/alephseq.d.ts +1 -1
- package/dist/output/alephseq.d.ts.map +1 -1
- package/dist/output/alephseq.js +5 -2
- package/dist/output/alephseq.js.map +1 -1
- package/dist/output/json.d.ts +1 -1
- package/dist/output/json.d.ts.map +1 -1
- package/dist/output/json.js +6 -1
- package/dist/output/json.js.map +1 -1
- package/dist/output/jsonl.d.ts +1 -1
- package/dist/output/jsonl.d.ts.map +1 -1
- package/dist/output/jsonl.js +6 -1
- package/dist/output/jsonl.js.map +1 -1
- package/dist/output/parquet.d.ts +7 -0
- package/dist/output/parquet.d.ts.map +1 -0
- package/dist/output/parquet.js +15 -0
- package/dist/output/parquet.js.map +1 -0
- package/dist/output/rdf.d.ts +1 -1
- package/dist/output/rdf.d.ts.map +1 -1
- package/dist/output/rdf.js +1 -1
- package/dist/output/rdf.js.map +1 -1
- package/dist/output/tsv.d.ts +3 -0
- package/dist/output/tsv.d.ts.map +1 -0
- package/dist/output/tsv.js +42 -0
- package/dist/output/tsv.js.map +1 -0
- package/dist/output/xml.d.ts +1 -1
- package/dist/output/xml.d.ts.map +1 -1
- package/dist/output/xml.js +2 -1
- package/dist/output/xml.js.map +1 -1
- package/dist/s3stream.d.ts +3 -2
- package/dist/s3stream.d.ts.map +1 -1
- package/dist/s3stream.js +55 -3
- package/dist/s3stream.js.map +1 -1
- package/dist/sftpstream.d.ts +3 -3
- package/dist/sftpstream.d.ts.map +1 -1
- package/dist/sftpstream.js +48 -5
- package/dist/sftpstream.js.map +1 -1
- package/dist/transform/avram.d.ts +6 -0
- package/dist/transform/avram.d.ts.map +1 -0
- package/dist/transform/avram.js +37 -0
- package/dist/transform/avram.js.map +1 -0
- package/dist/transform/jsonata.d.ts +1 -1
- package/dist/transform/jsonata.d.ts.map +1 -1
- package/dist/transform/jsonata.js +23 -15
- package/dist/transform/jsonata.js.map +1 -1
- package/dist/transform/marc2rdf.d.ts +1 -1
- package/dist/transform/marc2rdf.d.ts.map +1 -1
- package/dist/transform/marc2rdf.js +4 -2
- package/dist/transform/marc2rdf.js.map +1 -1
- package/dist/transform/marcids.d.ts +6 -0
- package/dist/transform/marcids.d.ts.map +1 -0
- package/dist/transform/marcids.js +22 -0
- package/dist/transform/marcids.js.map +1 -0
- package/dist/transform/notation3.d.ts +1 -1
- package/dist/transform/notation3.d.ts.map +1 -1
- package/dist/transform/notation3.js +26 -21
- package/dist/transform/notation3.js.map +1 -1
- package/dist/util/stream_helpers.d.ts +5 -1
- package/dist/util/stream_helpers.d.ts.map +1 -1
- package/dist/util/stream_helpers.js +3 -4
- package/dist/util/stream_helpers.js.map +1 -1
- package/package.json +7 -5
package/README.md
CHANGED
|
@@ -24,31 +24,37 @@ npm link
|
|
|
24
24
|
Generate JSON:
|
|
25
25
|
|
|
26
26
|
```
|
|
27
|
-
marcattacks --to json
|
|
27
|
+
marcattacks --to json ./data/sample.xml
|
|
28
|
+
```
|
|
29
|
+
|
|
30
|
+
We can also do this for tar (and) gzipped files
|
|
31
|
+
|
|
32
|
+
```
|
|
33
|
+
marcattacks --to json ./data/sample.tar.gz
|
|
28
34
|
```
|
|
29
35
|
|
|
30
36
|
Generate Aleph sequential:
|
|
31
37
|
|
|
32
38
|
```
|
|
33
|
-
marcattacks --to alephseq
|
|
39
|
+
marcattacks --to alephseq ./data/sample.xml
|
|
34
40
|
```
|
|
35
41
|
|
|
36
42
|
Generate RDF:
|
|
37
43
|
|
|
38
44
|
```
|
|
39
|
-
marcattacks --to rdf --map marc2rdf
|
|
45
|
+
marcattacks --to rdf --map marc2rdf ./data/sample.xml
|
|
40
46
|
```
|
|
41
47
|
|
|
42
48
|
Generate XML:
|
|
43
49
|
|
|
44
50
|
```
|
|
45
|
-
marcattacks --to xml
|
|
51
|
+
marcattacks --from alephseq --to xml ./data/one.alephseq
|
|
46
52
|
```
|
|
47
53
|
|
|
48
54
|
Transform the MARC input using a [JSONata](https://docs.jsonata.org/overview.html) expression or file:
|
|
49
55
|
|
|
50
56
|
```
|
|
51
|
-
marcattacks
|
|
57
|
+
marcattacks --param fix=./demo/demo.jsonata ./data/sample.xml
|
|
52
58
|
```
|
|
53
59
|
|
|
54
60
|
## Remote files
|
|
@@ -88,6 +94,7 @@ use `s3s://...` for using an SSL layer.
|
|
|
88
94
|
- jsonl
|
|
89
95
|
- marc (ISO2709)
|
|
90
96
|
- rdf
|
|
97
|
+
- tsv
|
|
91
98
|
- xml (MARCXML)
|
|
92
99
|
|
|
93
100
|
### Output (--to)
|
|
@@ -95,20 +102,24 @@ use `s3s://...` for using an SSL layer.
|
|
|
95
102
|
- alephseq (Aleph sequential)
|
|
96
103
|
- json
|
|
97
104
|
- jsonl
|
|
105
|
+
- parquet
|
|
98
106
|
- rdf
|
|
107
|
+
- tsv
|
|
99
108
|
- xml (MARCXML)
|
|
100
109
|
|
|
101
110
|
### Transform (--map)
|
|
102
111
|
|
|
103
|
-
-
|
|
104
|
-
-
|
|
112
|
+
- avram : A mapper from MARC to [Avram](https://format.gbv.de/schema/avram/specification)
|
|
113
|
+
- jsonata : A jsonata fixer (_default_)
|
|
114
|
+
- marcids : A mapper from MARC to a list of record ids
|
|
115
|
+
- marc2rdf : A mapper from MARC to RDF (demonstrator)
|
|
105
116
|
- notation3 : A [Notation3](https://w3c.github.io/N3/spec/) reasoner
|
|
106
117
|
|
|
107
118
|
Or, provide your own transformers using JavaScript plugins. See: ./plugin/demo.js for an example.
|
|
108
119
|
|
|
109
|
-
###
|
|
120
|
+
### Param (--param)
|
|
110
121
|
|
|
111
|
-
Provide a
|
|
122
|
+
Provide a params to the mapper, input and output. See examples:
|
|
112
123
|
|
|
113
124
|
- `npm run demo:jsonld`
|
|
114
125
|
- `npm run demo:n3`
|
|
@@ -120,9 +131,8 @@ Provide a fix file to the mapper. See examples:
|
|
|
120
131
|
- _file path_
|
|
121
132
|
- sftp://username@host:port/path
|
|
122
133
|
- s3://accessKey:secretKey@host:port/bucket/key (or s3s://)
|
|
123
|
-
- stdin://local
|
|
124
134
|
|
|
125
|
-
### Logging (--info,--debug,--log)
|
|
135
|
+
### Logging (--info,--debug,--trace,--log)
|
|
126
136
|
|
|
127
137
|
Logging messages can be provided with the `--info`, `--debug` and `--trace` options.
|
|
128
138
|
|
|
@@ -134,5 +144,19 @@ Default the logging format is a text format that is written to stderr. This logg
|
|
|
134
144
|
|
|
135
145
|
### Compression (--z,--tar)
|
|
136
146
|
|
|
147
|
+
Gzip and tar compression of input files can be automatically detected by file name extension. If no such extensions are provided the following flags can be set to force decompression:
|
|
148
|
+
|
|
137
149
|
- `--z` : the input file is gzipped
|
|
138
|
-
- `--tar` : the input file is tarred
|
|
150
|
+
- `--tar` : the input file is tarred
|
|
151
|
+
|
|
152
|
+
### Environment Variables
|
|
153
|
+
|
|
154
|
+
SFTP and S3 credentials can be set using environment variables or a local `.env` file.
|
|
155
|
+
Available variables:
|
|
156
|
+
|
|
157
|
+
- SFTP_USERNAME
|
|
158
|
+
- SFTP_PASSWORD
|
|
159
|
+
- S3_ACCESS_KEY
|
|
160
|
+
- S3_SECRET_KEY
|
|
161
|
+
|
|
162
|
+
A SFTP private key can be provided using the `--key-env` command line option. E.g. `--key-env PRIVATE_KEY`, which results reading a `PRIVATE_KEY` environment variable.
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"attacker.d.ts","sourceRoot":"","sources":["../src/attacker.ts"],"names":[],"mappings":"AAsBA,qBAAa,aAAc,SAAQ,KAAK;IACpC,SAAgB,UAAU,EAAE,MAAM,CAAC;gBAEvB,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM;CAUlD;AAED,wBAAsB,MAAM,CAAC,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE,GAAG,GAAI,OAAO,CAAC,MAAM,CAAC,CA0LrE"}
|
package/dist/attacker.js
ADDED
|
@@ -0,0 +1,198 @@
|
|
|
1
|
+
import log4js from 'log4js';
|
|
2
|
+
import { loadPlugin } from './plugin-loader.js';
|
|
3
|
+
import { sftpLatestFile, sftpReadStream, sftpWriteStream } from './sftpstream.js';
|
|
4
|
+
import { httpReadStream } from './httpstream.js';
|
|
5
|
+
import { Readable } from 'stream';
|
|
6
|
+
import { pathToFileURL } from "node:url";
|
|
7
|
+
import {} from 'node:stream';
|
|
8
|
+
import { SlowWritable } from './slow-writable.js';
|
|
9
|
+
import path from "node:path";
|
|
10
|
+
import fs from 'fs';
|
|
11
|
+
import { s3LatestObject, s3ReadStream, s3WriteStream } from './s3stream.js';
|
|
12
|
+
import { pipeline } from 'node:stream/promises';
|
|
13
|
+
import { createCountableSkippedStream, createUntarredStream, createUncompressedStream, createVerboseStream } from './util/stream_helpers.js';
|
|
14
|
+
import { fileLatestFile, fileReadStream } from './filestream.js';
|
|
15
|
+
const logger = log4js.getLogger();
|
|
16
|
+
export class PipelineError extends Error {
|
|
17
|
+
statusCode;
|
|
18
|
+
constructor(message, statusCode) {
|
|
19
|
+
super(message);
|
|
20
|
+
this.statusCode = statusCode;
|
|
21
|
+
Object.setPrototypeOf(this, PipelineError.prototype);
|
|
22
|
+
if (Error.captureStackTrace) {
|
|
23
|
+
Error.captureStackTrace(this, this.constructor);
|
|
24
|
+
}
|
|
25
|
+
}
|
|
26
|
+
}
|
|
27
|
+
export async function attack(url, opts) {
|
|
28
|
+
let result = 0;
|
|
29
|
+
try {
|
|
30
|
+
let inputFile;
|
|
31
|
+
if (fs.existsSync(url)) {
|
|
32
|
+
const filePath = path.resolve(process.cwd(), url);
|
|
33
|
+
inputFile = pathToFileURL(filePath);
|
|
34
|
+
}
|
|
35
|
+
else {
|
|
36
|
+
inputFile = new URL(url);
|
|
37
|
+
}
|
|
38
|
+
logger.info(`using: ${getCleanURL(inputFile)}`);
|
|
39
|
+
let readableStream;
|
|
40
|
+
if (inputFile.protocol.startsWith("http")) {
|
|
41
|
+
readableStream = await httpReadStream(inputFile.toString());
|
|
42
|
+
}
|
|
43
|
+
else if (inputFile.protocol.startsWith("s3")) {
|
|
44
|
+
const url = new URL(inputFile);
|
|
45
|
+
if (process.env.S3_ACCESS_KEY) {
|
|
46
|
+
url.username = process.env.S3_ACCESS_KEY;
|
|
47
|
+
}
|
|
48
|
+
if (process.env.S3_SECRET_KEY) {
|
|
49
|
+
url.password = process.env.S3_SECRET_KEY;
|
|
50
|
+
}
|
|
51
|
+
// optional resolve @latest
|
|
52
|
+
inputFile = await s3LatestObject(url, opts);
|
|
53
|
+
readableStream = await s3ReadStream(inputFile, opts);
|
|
54
|
+
}
|
|
55
|
+
else if (inputFile.protocol === 'sftp:') {
|
|
56
|
+
const url = new URL(inputFile);
|
|
57
|
+
if (process.env.SFTP_USERNAME) {
|
|
58
|
+
url.username = process.env.SFTP_USERNAME;
|
|
59
|
+
}
|
|
60
|
+
if (process.env.SFTP_PASSWORD) {
|
|
61
|
+
url.password = process.env.SFTP_PASSWORD;
|
|
62
|
+
}
|
|
63
|
+
// optional resolve @latest
|
|
64
|
+
inputFile = await sftpLatestFile(url, opts);
|
|
65
|
+
readableStream = await sftpReadStream(inputFile, opts);
|
|
66
|
+
}
|
|
67
|
+
else if (inputFile.protocol === 'stdin:') {
|
|
68
|
+
readableStream = process.stdin;
|
|
69
|
+
}
|
|
70
|
+
else {
|
|
71
|
+
inputFile = await fileLatestFile(inputFile);
|
|
72
|
+
readableStream = await fileReadStream(inputFile);
|
|
73
|
+
}
|
|
74
|
+
const stages = [readableStream];
|
|
75
|
+
if (opts.z || inputFile.pathname.endsWith(".gz")) {
|
|
76
|
+
stages.push(createUncompressedStream());
|
|
77
|
+
}
|
|
78
|
+
if (opts.tar || inputFile.pathname.match(/.tar(.\w+$)?$/) || inputFile.pathname.endsWith(".tgz")) {
|
|
79
|
+
stages.push(await createUntarredStream());
|
|
80
|
+
}
|
|
81
|
+
if (opts.from) {
|
|
82
|
+
const mod = await loadPlugin(opts.from, 'input');
|
|
83
|
+
const transformer = await mod.transform(Object.assign({ path: inputFile }, opts.param));
|
|
84
|
+
transformer.on('error', (error) => {
|
|
85
|
+
if (error.code === 'ERR_STREAM_PREMATURE_CLOSE') {
|
|
86
|
+
stages[0]?.destroy();
|
|
87
|
+
}
|
|
88
|
+
else {
|
|
89
|
+
logger.error("input stream processing error: ", error.message);
|
|
90
|
+
stages[0]?.destroy();
|
|
91
|
+
process.exitCode = 2;
|
|
92
|
+
}
|
|
93
|
+
});
|
|
94
|
+
transformer.on('finish', () => logger.debug('writeable finished'));
|
|
95
|
+
transformer.on('end', () => logger.debug('readable ended'));
|
|
96
|
+
transformer.on('close', () => logger.debug('stream closed'));
|
|
97
|
+
stages.push(transformer);
|
|
98
|
+
}
|
|
99
|
+
else {
|
|
100
|
+
console.error(`Need --from`);
|
|
101
|
+
process.exit(1);
|
|
102
|
+
}
|
|
103
|
+
if (opts.count || opts.skip) {
|
|
104
|
+
stages.push(createCountableSkippedStream(opts.count, opts.skip));
|
|
105
|
+
}
|
|
106
|
+
const verboseStream = createVerboseStream();
|
|
107
|
+
stages.push(verboseStream);
|
|
108
|
+
if (opts.map) {
|
|
109
|
+
const mod = await loadPlugin(opts.map, 'transform');
|
|
110
|
+
const transformer = await mod.transform(opts.param);
|
|
111
|
+
stages.push(transformer);
|
|
112
|
+
}
|
|
113
|
+
let outStream;
|
|
114
|
+
if (isWritableStream(opts.out)) {
|
|
115
|
+
outStream = opts.out;
|
|
116
|
+
}
|
|
117
|
+
else if (opts.out === '@slow') {
|
|
118
|
+
outStream = new SlowWritable({ delayMs: 100 });
|
|
119
|
+
}
|
|
120
|
+
else if (opts.out === '@errors') {
|
|
121
|
+
outStream = new SlowWritable({ simulateErrorEveryN: 2 });
|
|
122
|
+
}
|
|
123
|
+
else if (opts.out) {
|
|
124
|
+
if (opts.out.startsWith("sftp")) {
|
|
125
|
+
const url = new URL(opts.out);
|
|
126
|
+
if (process.env.SFTP_USERNAME) {
|
|
127
|
+
url.username = process.env.SFTP_USERNAME;
|
|
128
|
+
}
|
|
129
|
+
if (process.env.SFTP_PASSWORD) {
|
|
130
|
+
url.password = process.env.SFTP_PASSWORD;
|
|
131
|
+
}
|
|
132
|
+
logger.info(`put ${getCleanURL(url)}`);
|
|
133
|
+
outStream = await sftpWriteStream(url, opts);
|
|
134
|
+
}
|
|
135
|
+
else if (opts.out.startsWith("s3")) {
|
|
136
|
+
const url = new URL(opts.out);
|
|
137
|
+
if (process.env.S3_ACCESS_KEY) {
|
|
138
|
+
url.username = process.env.S3_ACCESS_KEY;
|
|
139
|
+
}
|
|
140
|
+
if (process.env.S3_SECRET_KEY) {
|
|
141
|
+
url.password = process.env.S3_SECRET_KEY;
|
|
142
|
+
}
|
|
143
|
+
logger.info(`put ${getCleanURL(url)}`);
|
|
144
|
+
outStream = await s3WriteStream(url, {});
|
|
145
|
+
}
|
|
146
|
+
else {
|
|
147
|
+
outStream = fs.createWriteStream(opts.out, { encoding: 'utf-8' });
|
|
148
|
+
}
|
|
149
|
+
}
|
|
150
|
+
else {
|
|
151
|
+
outStream = process.stdout;
|
|
152
|
+
}
|
|
153
|
+
if (opts.to) {
|
|
154
|
+
const mod = await loadPlugin(opts.to, 'output');
|
|
155
|
+
stages.push(await mod.transform(opts.param));
|
|
156
|
+
stages.push(outStream);
|
|
157
|
+
try {
|
|
158
|
+
await pipeline(stages);
|
|
159
|
+
result = verboseStream.getCount();
|
|
160
|
+
logger.info("pipeline finished cleanly");
|
|
161
|
+
}
|
|
162
|
+
catch (err) {
|
|
163
|
+
if (err.code === 'ERR_STREAM_PREMATURE_CLOSE') {
|
|
164
|
+
logger.info("Stream closed by limiter.");
|
|
165
|
+
}
|
|
166
|
+
else {
|
|
167
|
+
logger.error("pipeline error:", err);
|
|
168
|
+
throw new PipelineError(err.message, 3);
|
|
169
|
+
}
|
|
170
|
+
}
|
|
171
|
+
}
|
|
172
|
+
}
|
|
173
|
+
catch (e) {
|
|
174
|
+
if (e instanceof PipelineError) {
|
|
175
|
+
throw (e);
|
|
176
|
+
}
|
|
177
|
+
else {
|
|
178
|
+
logger.error(`process crashed with: ${e}`);
|
|
179
|
+
throw (e);
|
|
180
|
+
}
|
|
181
|
+
}
|
|
182
|
+
return result;
|
|
183
|
+
}
|
|
184
|
+
function getCleanURL(url) {
|
|
185
|
+
const tempUrl = new URL(url.href); // Clone to avoid mutating original
|
|
186
|
+
tempUrl.username = '***';
|
|
187
|
+
tempUrl.password = '***';
|
|
188
|
+
return tempUrl;
|
|
189
|
+
}
|
|
190
|
+
function isWritableStream(obj) {
|
|
191
|
+
return (obj !== null &&
|
|
192
|
+
typeof obj === 'object' &&
|
|
193
|
+
typeof obj.write === 'function' &&
|
|
194
|
+
typeof obj.end === 'function' &&
|
|
195
|
+
// internal state check
|
|
196
|
+
obj.writable !== false);
|
|
197
|
+
}
|
|
198
|
+
//# sourceMappingURL=attacker.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"attacker.js","sourceRoot":"","sources":["../src/attacker.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,MAAM,QAAQ,CAAC;AAC5B,OAAO,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AAChD,OAAO,EAAE,cAAc,EAAE,cAAc,EAAG,eAAe,EAAE,MAAM,iBAAiB,CAAC;AACnF,OAAO,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AACjD,OAAO,EAAE,QAAQ,EAAE,MAAM,QAAQ,CAAC;AAClC,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AACzC,OAAO,EAAiC,MAAM,aAAa,CAAC;AAC5D,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,MAAM,IAAI,CAAC;AACpB,OAAO,EAAE,cAAc,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AAC5E,OAAO,EAAE,QAAQ,EAAE,MAAM,sBAAsB,CAAC;AAChD,OAAO,EACH,4BAA4B,EAC5B,oBAAoB,EACpB,wBAAwB,EACxB,mBAAmB,EACtB,MAAM,0BAA0B,CAAC;AAClC,OAAO,EAAE,cAAc,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AAEjE,MAAM,MAAM,GAAG,MAAM,CAAC,SAAS,EAAE,CAAC;AAElC,MAAM,OAAO,aAAc,SAAQ,KAAK;IACpB,UAAU,CAAS;IAEnC,YAAY,OAAe,EAAE,UAAkB;QAC3C,KAAK,CAAC,OAAO,CAAC,CAAC;QACf,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;QAE7B,MAAM,CAAC,cAAc,CAAC,IAAI,EAAE,aAAa,CAAC,SAAS,CAAC,CAAC;QAErD,IAAI,KAAK,CAAC,iBAAiB,EAAE,CAAC;YAC1B,KAAK,CAAC,iBAAiB,CAAC,IAAI,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;QACpD,CAAC;IACL,CAAC;CACJ;AAED,MAAM,CAAC,KAAK,UAAU,MAAM,CAAC,GAAW,EAAE,IAAS;IAC/C,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,IAAI,CAAC;QACD,IAAI,SAAe,CAAC;QAEpB,IAAI,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;YACrB,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,GAAG,CAAC,CAAC;YAClD,SAAS,GAAG,aAAa,CAAC,QAAQ,CAAC,CAAC;QACxC,CAAC;aACI,CAAC;YACF,SAAS,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;QAC7B,CAAC;QAED,MAAM,CAAC,IAAI,CAAC,UAAU,WAAW,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;QAEhD,IAAI,cAAyB,CAAC;QAE9B,IAAI,SAAS,CAAC,QAAQ,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC;YACxC,cAAc,GAAG,MAAM,cAAc,CAAC,SAAS,CAAC,QAAQ,EAAE,CAAC,CAAC;QAChE,CAAC;aACI,IAAI,SAAS,CAAC,QAAQ,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;YAC3C,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,SAAS,CAAC,CAAC;YAE/B,IAAI,OAAO,CAAC,GAAG,CAAC,aAAa,EAAE,CAAC;gBAC5B,GAAG,CAAC,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,aAAa,CAAC;YAC7C,CAAC;YAED,IAAI,OAAO,CAAC,GAAG,CAAC,aAAa,EAAE,CAAC;gBAC5B,GAAG,CAAC,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,aAAa,CAAC;YAC7C,CAAC;YAED,2BAA2B;YAC3B,SAAS,GAAG,MAAM,cAAc,CAAC,GAAG,EAAC,IAAI,CAAC,CAAC;YAC3C,cAAc,GAAG,MAAM,YAAY,CAAC,SAAS,EAAC,IAAI,CAAC,CAAC;QACxD,CAAC;aACI,IAAI,SAAS,CAAC,QAAQ,KAAK,OAAO,EAAE,CAAC;YACtC,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,SAAS,CAAC,CAAC;YAE/B,IAAI,OAAO,CAAC,GAAG,CAAC,aAAa,EAAE,CAAC;gBAC5B,GAAG,CAAC,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,aAAa,CAAC;YAC7C,CAAC;YAED,IAAI,OAAO,CAAC,GAAG,CAAC,aAAa,EAAE,CAAC;gBAC5B,GAAG,CAAC,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,aAAa,CAAC;YAC7C,CAAC;YAED,2BAA2B;YAC3B,SAAS,GAAG,MAAM,cAAc,CAAC,GAAG,EAAC,IAAI,CAAC,CAAC;YAC3C,cAAc,GAAG,MAAM,cAAc,CAAC,SAAS,EAAC,IAAI,CAAC,CAAC;QAC1D,CAAC;aACI,IAAI,SAAS,CAAC,QAAQ,KAAK,QAAQ,EAAE,CAAC;YACvC,cAAc,GAAG,OAAO,CAAC,KAAK,CAAC;QACnC,CAAC;aACI,CAAC;YACF,SAAS,GAAG,MAAM,cAAc,CAAC,SAAS,CAAC,CAAC;YAC5C,cAAc,GAAG,MAAM,cAAc,CAAC,SAAS,CAAC,CAAC;QACrD,CAAC;QAED,MAAM,MAAM,GAAwC,CAAC,cAAc,CAAC,CAAC;QAErE,IAAI,IAAI,CAAC,CAAC,IAAI,SAAS,CAAC,QAAQ,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;YAC/C,MAAM,CAAC,IAAI,CAAC,wBAAwB,EAAE,CAAC,CAAC;QAC5C,CAAC;QAED,IAAI,IAAI,CAAC,GAAG,IAAI,SAAS,CAAC,QAAQ,CAAC,KAAK,CAAC,eAAe,CAAC,IAAI,SAAS,CAAC,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;YAC/F,MAAM,CAAC,IAAI,CAAC,MAAM,oBAAoB,EAAE,CAAC,CAAC;QAC9C,CAAC;QAED,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;YACZ,MAAM,GAAG,GAAG,MAAM,UAAU,CAAC,IAAI,CAAC,IAAI,EAAC,OAAO,CAAC,CAAC;YAChD,MAAM,WAAW,GAAG,MAAM,GAAG,CAAC,SAAS,CAAC,MAAM,CAAC,MAAM,CAAC,EAAC,IAAI,EAAE,SAAS,EAAC,EAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;YACrF,WAAW,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,KAAU,EAAE,EAAE;gBACnC,IAAI,KAAK,CAAC,IAAI,KAAK,4BAA4B,EAAE,CAAC;oBAC9C,MAAM,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,CAAC;gBACzB,CAAC;qBACI,CAAC;oBACF,MAAM,CAAC,KAAK,CAAC,iCAAiC,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC;oBAC/D,MAAM,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,CAAC;oBACrB,OAAO,CAAC,QAAQ,GAAG,CAAC,CAAC;gBACzB,CAAC;YACL,CAAC,CAAC,CAAC;YACH,WAAW,CAAC,EAAE,CAAC,QAAQ,EAAE,GAAG,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,oBAAoB,CAAC,CAAC,CAAC;YACnE,WAAW,CAAC,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAC,CAAC;YAC5D,WAAW,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC,CAAC;YAC7D,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QAC7B,CAAC;aACI,CAAC;YACF,OAAO,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC;YAC7B,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACpB,CAAC;QAED,IAAI,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;YAC1B,MAAM,CAAC,IAAI,CACP,4BAA4B,CACxB,IAAI,CAAC,KAAK,EACV,IAAI,CAAC,IAAI,CACZ,CACJ,CAAC;QACN,CAAC;QAED,MAAM,aAAa,GAAG,mBAAmB,EAAE,CAAC;QAC5C,MAAM,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;QAE3B,IAAI,IAAI,CAAC,GAAG,EAAE,CAAC;YACX,MAAM,GAAG,GAAG,MAAM,UAAU,CAAC,IAAI,CAAC,GAAG,EAAC,WAAW,CAAC,CAAC;YACnD,MAAM,WAAW,GAAe,MAAM,GAAG,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAChE,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QAC7B,CAAC;QAED,IAAI,SAAoB,CAAC;QAEzB,IAAI,gBAAgB,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;YAC7B,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC;QACzB,CAAC;aACI,IAAI,IAAI,CAAC,GAAG,KAAK,OAAO,EAAE,CAAC;YAC5B,SAAS,GAAG,IAAI,YAAY,CAAC,EAAE,OAAO,EAAE,GAAG,EAAE,CAAC,CAAC;QACnD,CAAC;aACI,IAAI,IAAI,CAAC,GAAG,KAAK,SAAS,EAAE,CAAC;YAC9B,SAAS,GAAG,IAAI,YAAY,CAAC,EAAE,mBAAmB,EAAE,CAAC,EAAE,CAAC,CAAC;QAC7D,CAAC;aACI,IAAI,IAAI,CAAC,GAAG,EAAE,CAAC;YAChB,IAAI,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC;gBAC9B,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBAE9B,IAAI,OAAO,CAAC,GAAG,CAAC,aAAa,EAAE,CAAC;oBAC5B,GAAG,CAAC,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,aAAa,CAAC;gBAC7C,CAAC;gBAED,IAAI,OAAO,CAAC,GAAG,CAAC,aAAa,EAAE,CAAC;oBAC5B,GAAG,CAAC,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,aAAa,CAAC;gBAC7C,CAAC;gBAED,MAAM,CAAC,IAAI,CAAC,OAAO,WAAW,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;gBACvC,SAAS,GAAG,MAAM,eAAe,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;YACjD,CAAC;iBACI,IAAI,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;gBACjC,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBAE9B,IAAI,OAAO,CAAC,GAAG,CAAC,aAAa,EAAE,CAAC;oBAC5B,GAAG,CAAC,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,aAAa,CAAC;gBAC7C,CAAC;gBAED,IAAI,OAAO,CAAC,GAAG,CAAC,aAAa,EAAE,CAAC;oBAC5B,GAAG,CAAC,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,aAAa,CAAC;gBAC7C,CAAC;gBAED,MAAM,CAAC,IAAI,CAAC,OAAO,WAAW,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;gBACvC,SAAS,GAAG,MAAM,aAAa,CAAC,GAAG,EAAC,EAAE,CAAC,CAAC;YAC5C,CAAC;iBACI,CAAC;gBACF,SAAS,GAAG,EAAE,CAAC,iBAAiB,CAAC,IAAI,CAAC,GAAG,EAAE,EAAE,QAAQ,EAAE,OAAO,EAAC,CAAC,CAAC;YACrE,CAAC;QACL,CAAC;aACI,CAAC;YACF,SAAS,GAAG,OAAO,CAAC,MAAM,CAAC;QAC/B,CAAC;QAED,IAAI,IAAI,CAAC,EAAE,EAAE,CAAC;YACV,MAAM,GAAG,GAAG,MAAM,UAAU,CAAC,IAAI,CAAC,EAAE,EAAC,QAAQ,CAAC,CAAC;YAC/C,MAAM,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;YAC7C,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YACvB,IAAI,CAAC;gBACD,MAAM,QAAQ,CAAC,MAAM,CAAC,CAAC;gBACvB,MAAM,GAAG,aAAa,CAAC,QAAQ,EAAE,CAAC;gBAClC,MAAM,CAAC,IAAI,CAAC,2BAA2B,CAAC,CAAC;YAC7C,CAAC;YACD,OAAO,GAAS,EAAE,CAAC;gBACf,IAAI,GAAG,CAAC,IAAI,KAAK,4BAA4B,EAAE,CAAC;oBAC5C,MAAM,CAAC,IAAI,CAAC,2BAA2B,CAAC,CAAC;gBAC7C,CAAC;qBAAM,CAAC;oBACJ,MAAM,CAAC,KAAK,CAAC,iBAAiB,EAAE,GAAG,CAAC,CAAC;oBACrC,MAAM,IAAI,aAAa,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;gBAC5C,CAAC;YACL,CAAC;QACL,CAAC;IACL,CAAC;IACD,OAAO,CAAC,EAAE,CAAC;QACP,IAAI,CAAC,YAAY,aAAa,EAAE,CAAC;YAC7B,MAAM,CAAC,CAAC,CAAC,CAAC;QACd,CAAC;aACI,CAAC;YACF,MAAM,CAAC,KAAK,CAAC,yBAAyB,CAAC,EAAE,CAAC,CAAC;YAC3C,MAAM,CAAC,CAAC,CAAC,CAAC;QACd,CAAC;IACL,CAAC;IACD,OAAO,MAAM,CAAC;AAClB,CAAC;AAED,SAAS,WAAW,CAAC,GAAQ;IACzB,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,mCAAmC;IACtE,OAAO,CAAC,QAAQ,GAAG,KAAK,CAAC;IACzB,OAAO,CAAC,QAAQ,GAAG,KAAK,CAAC;IACzB,OAAO,OAAO,CAAC;AACnB,CAAC;AAED,SAAS,gBAAgB,CAAC,GAAQ;IAChC,OAAO,CACL,GAAG,KAAK,IAAI;QACZ,OAAO,GAAG,KAAK,QAAQ;QACvB,OAAO,GAAG,CAAC,KAAK,KAAK,UAAU;QAC/B,OAAO,GAAG,CAAC,GAAG,KAAK,UAAU;QAC7B,uBAAuB;QACvB,GAAG,CAAC,QAAQ,KAAK,KAAK,CACvB,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"filestream.d.ts","sourceRoot":"","sources":["../src/filestream.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,QAAQ,CAAC;AAKvC,wBAAsB,cAAc,CAAC,GAAG,EAAE,GAAG,GAAI,OAAO,CAAC,QAAQ,CAAC,CAEjE;AAED,wBAAsB,cAAc,CAAC,GAAG,EAAE,GAAG,GAAI,OAAO,CAAC,GAAG,CAAC,CA6B5D"}
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
import fs from "fs";
|
|
2
|
+
import log4js from 'log4js';
|
|
3
|
+
const logger = log4js.getLogger();
|
|
4
|
+
export async function fileReadStream(url) {
|
|
5
|
+
return fs.createReadStream(url);
|
|
6
|
+
}
|
|
7
|
+
export async function fileLatestFile(url) {
|
|
8
|
+
return new Promise((resolve) => {
|
|
9
|
+
logger.debug(url.href);
|
|
10
|
+
if (!url.href.includes("@latest:")) {
|
|
11
|
+
resolve(url);
|
|
12
|
+
return;
|
|
13
|
+
}
|
|
14
|
+
const directory = url.pathname.replaceAll(/@latest:.*/g, "");
|
|
15
|
+
const extension = url.pathname.replaceAll(/.*@latest:/g, "");
|
|
16
|
+
logger.debug(`directory: ${directory} ; extension: ${extension}`);
|
|
17
|
+
fs.readdir(directory, (err, files) => {
|
|
18
|
+
if (err) {
|
|
19
|
+
throw new Error("Error finding latest file:", err);
|
|
20
|
+
}
|
|
21
|
+
for (let i = 0; i < files.length; i++) {
|
|
22
|
+
if (files[i]?.toLowerCase().endsWith(extension)) {
|
|
23
|
+
logger.info(`resolved as file://${directory}${files[i]}`);
|
|
24
|
+
resolve(new URL("file://" + directory + files[i]));
|
|
25
|
+
return;
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
throw new Error("No latest file found!");
|
|
29
|
+
});
|
|
30
|
+
});
|
|
31
|
+
}
|
|
32
|
+
//# sourceMappingURL=filestream.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"filestream.js","sourceRoot":"","sources":["../src/filestream.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,IAAI,CAAC;AAEpB,OAAO,MAAM,MAAM,QAAQ,CAAC;AAE5B,MAAM,MAAM,GAAG,MAAM,CAAC,SAAS,EAAE,CAAC;AAElC,MAAM,CAAC,KAAK,UAAU,cAAc,CAAC,GAAQ;IACzC,OAAO,EAAE,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC;AACpC,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,cAAc,CAAC,GAAQ;IACzC,OAAO,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE;QACjC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACvB,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC,EAAE,CAAC;YACjC,OAAO,CAAC,GAAG,CAAC,CAAC;YACb,OAAO;QACX,CAAC;QAED,MAAM,SAAS,GAAG,GAAG,CAAC,QAAQ,CAAC,UAAU,CAAC,aAAa,EAAC,EAAE,CAAC,CAAC;QAC5D,MAAM,SAAS,GAAG,GAAG,CAAC,QAAQ,CAAC,UAAU,CAAC,aAAa,EAAC,EAAE,CAAC,CAAC;QAE5D,MAAM,CAAC,KAAK,CAAC,cAAc,SAAS,iBAAiB,SAAS,EAAE,CAAC,CAAC;QAElE,EAAE,CAAC,OAAO,CAAC,SAAS,EAAE,CAAC,GAAG,EAAC,KAAK,EAAE,EAAE;YACjC,IAAI,GAAG,EAAE,CAAC;gBACT,MAAM,IAAI,KAAK,CAAC,4BAA4B,EAAE,GAAG,CAAC,CAAC;YACpD,CAAC;YAED,KAAK,IAAI,CAAC,GAAG,CAAC,EAAG,CAAC,GAAG,KAAK,CAAC,MAAM,EAAG,CAAC,EAAE,EAAE,CAAC;gBACzC,IAAI,KAAK,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC;oBAC9C,MAAM,CAAC,IAAI,CAAC,sBAAsB,SAAS,GAAG,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;oBAC1D,OAAO,CAAC,IAAI,GAAG,CAAC,SAAS,GAAG,SAAS,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;oBACnD,OAAO;gBACX,CAAC;YACF,CAAC;YAED,MAAM,IAAI,KAAK,CAAC,uBAAuB,CAAC,CAAC;QAC5C,CAAC,CAAC,CAAC;IACP,CAAC,CAAC,CAAC;AACP,CAAC"}
|
package/dist/index.js
CHANGED
|
@@ -1,18 +1,10 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
import log4js from 'log4js';
|
|
3
3
|
import { program } from 'commander';
|
|
4
|
-
import { loadPlugin } from './plugin-loader.js';
|
|
5
|
-
import { sftpReadStream, sftpWriteStream, sftpLatestFile } from './sftpstream.js';
|
|
6
|
-
import { httpReadStream } from './httpstream.js';
|
|
7
|
-
import { Readable } from 'stream';
|
|
8
|
-
import { pathToFileURL } from "node:url";
|
|
9
|
-
import { SlowWritable } from './slow-writable.js';
|
|
10
4
|
import path from "node:path";
|
|
11
|
-
import fs from 'fs';
|
|
12
|
-
import { s3ReaderStream, s3WriterStream } from './s3stream.js';
|
|
13
5
|
import dotenv from 'dotenv';
|
|
14
|
-
import {
|
|
15
|
-
import {
|
|
6
|
+
import { attack, PipelineError } from './attacker.js';
|
|
7
|
+
import { asyncWrapProviders } from 'node:async_hooks';
|
|
16
8
|
program.version('0.1.0')
|
|
17
9
|
.argument('<file>')
|
|
18
10
|
.option('-c,--config <config>', 'config .env', path.join(process.cwd(), '.env'))
|
|
@@ -20,6 +12,7 @@ program.version('0.1.0')
|
|
|
20
12
|
.option('-t,--to <output>', 'output type', 'json')
|
|
21
13
|
.option('-m,--map <map>', 'data mapper', 'jsonata')
|
|
22
14
|
.option('--fix <what>', 'jsonata')
|
|
15
|
+
.option('-p,--param <key=value>', 'repeated params', collect, {})
|
|
23
16
|
.option('-o,--out <file>', 'output file')
|
|
24
17
|
.option('--z', 'uncompress input')
|
|
25
18
|
.option('--tar', 'untar input')
|
|
@@ -116,166 +109,28 @@ async function main() {
|
|
|
116
109
|
console.error(`need an input file`);
|
|
117
110
|
process.exit(2);
|
|
118
111
|
}
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
}
|
|
127
|
-
logger.info(`using: ${getCleanURL(inputFile)}`);
|
|
128
|
-
let readableStream;
|
|
129
|
-
if (inputFile.protocol.startsWith("http")) {
|
|
130
|
-
readableStream = await httpReadStream(inputFile.toString());
|
|
131
|
-
}
|
|
132
|
-
else if (inputFile.protocol.startsWith("s3")) {
|
|
133
|
-
readableStream = await s3ReaderStream(inputFile, {});
|
|
134
|
-
}
|
|
135
|
-
else if (inputFile.protocol === 'sftp:') {
|
|
136
|
-
const config = makeSftpConfig(inputFile, opts);
|
|
137
|
-
let remotePath;
|
|
138
|
-
if (inputFile.pathname.match(/\/@latest:\w+$/)) {
|
|
139
|
-
const remoteDir = inputFile.pathname.replace(/\/@latest.*/, "");
|
|
140
|
-
const extension = inputFile.pathname.replace(/.*\/@latest:/, "");
|
|
141
|
-
remotePath = await sftpLatestFile(config, remoteDir, extension);
|
|
142
|
-
}
|
|
143
|
-
else {
|
|
144
|
-
remotePath = inputFile.pathname;
|
|
145
|
-
}
|
|
146
|
-
readableStream = await sftpReadStream(remotePath, config);
|
|
147
|
-
}
|
|
148
|
-
else if (inputFile.protocol === 'stdin:') {
|
|
149
|
-
readableStream = process.stdin;
|
|
150
|
-
}
|
|
151
|
-
else {
|
|
152
|
-
readableStream = fs.createReadStream(inputFile);
|
|
153
|
-
}
|
|
154
|
-
const stages = [readableStream];
|
|
155
|
-
if (opts.z) {
|
|
156
|
-
stages.push(createUncompressedStream());
|
|
157
|
-
}
|
|
158
|
-
if (opts.tar) {
|
|
159
|
-
stages.push(await createUntarredStream());
|
|
160
|
-
}
|
|
161
|
-
if (opts.from) {
|
|
162
|
-
const mod = await loadPlugin(opts.from, 'input');
|
|
163
|
-
const transformer = await mod.transform({ path: inputFile });
|
|
164
|
-
transformer.on('error', (error) => {
|
|
165
|
-
if (error.code === 'ERR_STREAM_PREMATURE_CLOSE') {
|
|
166
|
-
stages[0]?.destroy();
|
|
167
|
-
}
|
|
168
|
-
else {
|
|
169
|
-
logger.error("input stream processing error: ", error.message);
|
|
170
|
-
stages[0]?.destroy();
|
|
171
|
-
process.exitCode = 2;
|
|
172
|
-
}
|
|
173
|
-
});
|
|
174
|
-
transformer.on('finish', () => logger.debug('writeable finished'));
|
|
175
|
-
transformer.on('end', () => logger.debug('readable ended'));
|
|
176
|
-
transformer.on('close', () => logger.debug('stream closed'));
|
|
177
|
-
stages.push(transformer);
|
|
178
|
-
}
|
|
179
|
-
else {
|
|
180
|
-
console.error(`Need --from`);
|
|
181
|
-
process.exit(1);
|
|
182
|
-
}
|
|
183
|
-
if (opts.count || opts.skip) {
|
|
184
|
-
stages.push(createCountableSkippedStream(opts.count, opts.skip));
|
|
185
|
-
}
|
|
186
|
-
stages.push(createVerboseStream());
|
|
187
|
-
if (opts.map) {
|
|
188
|
-
const mod = await loadPlugin(opts.map, 'transform');
|
|
189
|
-
const transformer = await mod.transform(opts.fix);
|
|
190
|
-
stages.push(transformer);
|
|
191
|
-
}
|
|
192
|
-
let outStream;
|
|
193
|
-
if (opts.out === '@slow') {
|
|
194
|
-
outStream = new SlowWritable({ delayMs: 100 });
|
|
195
|
-
}
|
|
196
|
-
else if (opts.out === '@errors') {
|
|
197
|
-
outStream = new SlowWritable({ simulateErrorEveryN: 2 });
|
|
198
|
-
}
|
|
199
|
-
else if (opts.out) {
|
|
200
|
-
if (opts.out.startsWith("sftp")) {
|
|
201
|
-
const url = new URL(opts.out);
|
|
202
|
-
if (process.env.SFTP_USERNAME) {
|
|
203
|
-
url.username = process.env.SFTP_USERNAME;
|
|
204
|
-
}
|
|
205
|
-
if (process.env.SFTP_PASSWORD) {
|
|
206
|
-
url.password = process.env.SFTP_PASSWORD;
|
|
207
|
-
}
|
|
208
|
-
const config = makeSftpConfig(url, opts);
|
|
209
|
-
logger.info(`put ${getCleanURL(url)}`);
|
|
210
|
-
outStream = await sftpWriteStream(url.href, config);
|
|
211
|
-
}
|
|
212
|
-
else if (opts.out.startsWith("s3")) {
|
|
213
|
-
const url = new URL(opts.out);
|
|
214
|
-
if (process.env.S3_ACCESS_KEY) {
|
|
215
|
-
url.username = process.env.S3_ACCESS_KEY;
|
|
216
|
-
}
|
|
217
|
-
if (process.env.S3_SECRET_KEY) {
|
|
218
|
-
url.password = process.env.S3_SECRET_KEY;
|
|
219
|
-
}
|
|
220
|
-
logger.info(`put ${getCleanURL(url)}`);
|
|
221
|
-
outStream = await s3WriterStream(url, {});
|
|
222
|
-
}
|
|
223
|
-
else {
|
|
224
|
-
outStream = fs.createWriteStream(opts.out, { encoding: 'utf-8' });
|
|
225
|
-
}
|
|
112
|
+
const result = await attack(url, opts);
|
|
113
|
+
logger.info(`total: ${result}`);
|
|
114
|
+
}
|
|
115
|
+
catch (e) {
|
|
116
|
+
if (e instanceof PipelineError) {
|
|
117
|
+
logger.error("pipeline error:", e);
|
|
118
|
+
process.exitCode = e.statusCode;
|
|
226
119
|
}
|
|
227
120
|
else {
|
|
228
|
-
|
|
121
|
+
logger.error("process crashed:", e);
|
|
122
|
+
process.exitCode = 8;
|
|
229
123
|
}
|
|
230
|
-
if (opts.to) {
|
|
231
|
-
const mod = await loadPlugin(opts.to, 'output');
|
|
232
|
-
stages.push(await mod.transform());
|
|
233
|
-
stages.push(outStream);
|
|
234
|
-
try {
|
|
235
|
-
await pipeline(stages);
|
|
236
|
-
logger.info("pipeline finished cleanly");
|
|
237
|
-
}
|
|
238
|
-
catch (err) {
|
|
239
|
-
if (err.code === 'ERR_STREAM_PREMATURE_CLOSE') {
|
|
240
|
-
logger.info("Stream closed by limiter.");
|
|
241
|
-
}
|
|
242
|
-
else {
|
|
243
|
-
logger.error("pipeline error:", err);
|
|
244
|
-
process.exitCode = 3;
|
|
245
|
-
}
|
|
246
|
-
}
|
|
247
|
-
}
|
|
248
|
-
}
|
|
249
|
-
catch (e) {
|
|
250
|
-
logger.error(`process crashed with: ${e}`);
|
|
251
|
-
process.exitCode = 8;
|
|
252
124
|
}
|
|
253
125
|
}
|
|
254
|
-
function
|
|
255
|
-
const
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
}
|
|
260
|
-
function makeSftpConfig(inputFile, opts) {
|
|
261
|
-
let privateKey = undefined;
|
|
262
|
-
if (opts.key) {
|
|
263
|
-
privateKey = fs.readFileSync(opts.key, { encoding: 'utf-8' });
|
|
264
|
-
}
|
|
265
|
-
else if (opts.keyEnv) {
|
|
266
|
-
privateKey = process.env[opts.keyEnv];
|
|
126
|
+
function collect(value, previous) {
|
|
127
|
+
const keyval = value.split("=", 2);
|
|
128
|
+
if (keyval.length == 2 && keyval[0]) {
|
|
129
|
+
previous[keyval[0]] = keyval[1];
|
|
130
|
+
return previous;
|
|
267
131
|
}
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
port: Number(inputFile.port) ?? 22,
|
|
271
|
-
username: inputFile.username
|
|
272
|
-
};
|
|
273
|
-
if (inputFile.password) {
|
|
274
|
-
config.password = inputFile.password;
|
|
275
|
-
}
|
|
276
|
-
if (privateKey) {
|
|
277
|
-
config.privateKey = privateKey;
|
|
132
|
+
else {
|
|
133
|
+
return previous;
|
|
278
134
|
}
|
|
279
|
-
return config;
|
|
280
135
|
}
|
|
281
136
|
//# sourceMappingURL=index.js.map
|
package/dist/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";AAEA,OAAO,MAAM,MAAM,QAAQ,CAAC;AAC5B,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";AAEA,OAAO,MAAM,MAAM,QAAQ,CAAC;AAC5B,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,MAAM,MAAM,QAAQ,CAAC;AAC5B,OAAO,EAAE,MAAM,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AACtD,OAAO,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAC;AAEtD,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC;KACnB,QAAQ,CAAC,QAAQ,CAAC;KAClB,MAAM,CAAC,sBAAsB,EAAC,aAAa,EAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,MAAM,CAAC,CAAC;KAC7E,MAAM,CAAC,kBAAkB,EAAC,YAAY,EAAC,KAAK,CAAC;KAC7C,MAAM,CAAC,kBAAkB,EAAC,aAAa,EAAC,MAAM,CAAC;KAC/C,MAAM,CAAC,gBAAgB,EAAC,aAAa,EAAC,SAAS,CAAC;KAChD,MAAM,CAAC,cAAc,EAAC,SAAS,CAAC;KAChC,MAAM,CAAC,wBAAwB,EAAC,iBAAiB,EAAC,OAAO,EAAC,EAAE,CAAC;KAC7D,MAAM,CAAC,iBAAiB,EAAC,aAAa,CAAC;KACvC,MAAM,CAAC,KAAK,EAAC,kBAAkB,CAAC;KAChC,MAAM,CAAC,OAAO,EAAC,aAAa,CAAC;KAC7B,MAAM,CAAC,eAAe,EAAE,2BAA2B,CAAC;KACpD,MAAM,CAAC,cAAc,EAAE,0BAA0B,CAAC;KAClD,MAAM,CAAC,iBAAiB,EAAE,kBAAkB,CAAC;KAC7C,MAAM,CAAC,iBAAiB,EAAC,kCAAkC,CAAC;KAC5D,MAAM,CAAC,gBAAgB,EAAC,gBAAgB,CAAC;KACzC,MAAM,CAAC,QAAQ,EAAC,2BAA2B,CAAC;KAC5C,MAAM,CAAC,SAAS,EAAC,gCAAgC,CAAC;KAClD,MAAM,CAAC,SAAS,EAAC,qCAAqC,CAAC,CAAC;AAE7D,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;AAE5B,MAAM,IAAI,GAAK,OAAO,CAAC,IAAI,EAAE,CAAC;AAE9B,IAAI,IAAI,CAAC,GAAG,EAAE,CAAC;IACX,IAAI,MAAM,GAAG,QAAQ,CAAC;IACtB,IAAI,IAAI,GAAG,CAAC,CAAC;IAEb,IAAI,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;QAClC,MAAM,GAAG,QAAQ,CAAC;IACtB,CAAC;IACD,IAAI,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC;QAChC,IAAI,GAAG,CAAC,CAAC;IACb,CAAC;IAED,IAAI,IAAI,EAAE,CAAC;QACP,mBAAmB,CAAC,MAAM,CAAC,CAAC;IAChC,CAAC;SACI,CAAC;QACF,sBAAsB,CAAC,MAAM,CAAC,CAAC;IACnC,CAAC;AACL,CAAC;KACI,CAAC;IACF,sBAAsB,CAAC,QAAQ,CAAC,CAAC;AACrC,CAAC;AAED,MAAM,MAAM,GAAG,MAAM,CAAC,SAAS,EAAE,CAAC;AAElC,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;IACZ,MAAM,CAAC,KAAK,GAAG,MAAM,CAAC;AAC1B,CAAC;AAED,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;IACb,MAAM,CAAC,KAAK,GAAG,OAAO,CAAC;AAC3B,CAAC;AAED,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;IACb,MAAM,CAAC,KAAK,GAAG,OAAO,CAAC;AAC3B,CAAC;AAED,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;IACd,MAAM,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,MAAM,EAAG,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;AACvD,CAAC;AAED,IAAI,EAAE,CAAC;AAEP,SAAS,sBAAsB,CAAC,MAAc;IAC1C,MAAM,CAAC,SAAS,CAAC;QACb,SAAS,EAAE;YACP,GAAG,EAAE;gBACD,IAAI,EAAE,MAAM;gBACZ,MAAM,EAAE;oBACJ,IAAI,EAAE,SAAS;oBACf,OAAO,EAAE,oBAAoB;iBAChC;aACJ;SACJ;QACD,UAAU,EAAE;YACR,OAAO,EAAE,EAAE,SAAS,EAAE,CAAC,KAAK,CAAC,EAAE,KAAK,EAAE,KAAK,EAAG,eAAe,EAAE,IAAI,EAAE;SACxE;KACJ,CAAC,CAAC;AACP,CAAC;AAED,SAAS,mBAAmB,CAAC,MAAc;IACvC,MAAM,CAAC,SAAS,CAAC,cAAc,EAAE,CAAC,MAAM,EAAE,EAAE;QACxC,OAAO,CAAC,QAAQ,EAAE,EAAE;YAChB,OAAO,IAAI,CAAC,SAAS,CAAC;gBACtB,SAAS,EAAE,QAAQ,CAAC,SAAS,EAAG,gBAAgB;gBAChD,KAAK,EAAE,QAAQ,CAAC,KAAK,CAAC,QAAQ,EAAE,gBAAgB;gBAChD,QAAQ,EAAE,QAAQ,CAAC,YAAY,EAAC,gBAAgB;gBAChD,OAAO,EAAE,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,gBAAgB;gBAClD,OAAO,EAAE,QAAQ,CAAC,OAAO,EAAO,yBAAyB;gBACzD,GAAG,EAAE,QAAQ,CAAC,GAAG,CAAe,gBAAgB;aAC/C,CAAC,CAAC;QACP,CAAC,CAAC;IACN,CAAC,CAAC,CAAC;IAEH,MAAM,CAAC,SAAS,CAAC;QACb,SAAS,EAAE;YACP,GAAG,EAAE;gBACD,IAAI,EAAE,MAAM;gBACZ,MAAM,EAAE;oBACJ,IAAI,EAAE,cAAc;iBACvB;aACJ;SACJ;QACD,UAAU,EAAE;YACR,OAAO,EAAE,EAAE,SAAS,EAAE,CAAC,KAAK,CAAC,EAAE,KAAK,EAAE,KAAK,EAAG,eAAe,EAAE,IAAI,EAAE;SACxE;KACJ,CAAC,CAAC;AACP,CAAC;AAED,KAAK,UAAU,IAAI;IACf,IAAI,CAAC;QACD,MAAM,GAAG,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAE5B,IAAI,CAAE,GAAG,EAAE,CAAC;YACR,OAAO,CAAC,KAAK,CAAC,oBAAoB,CAAC,CAAC;YACpC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACpB,CAAC;QAED,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,GAAG,EAAC,IAAI,CAAC,CAAC;QACtC,MAAM,CAAC,IAAI,CAAC,UAAU,MAAM,EAAE,CAAC,CAAC;IACpC,CAAC;IACD,OAAO,CAAC,EAAE,CAAC;QACP,IAAI,CAAC,YAAY,aAAa,EAAE,CAAC;YAC7B,MAAM,CAAC,KAAK,CAAC,iBAAiB,EAAE,CAAC,CAAC,CAAC;YACnC,OAAO,CAAC,QAAQ,GAAG,CAAC,CAAC,UAAU,CAAC;QACpC,CAAC;aACI,CAAC;YACF,MAAM,CAAC,KAAK,CAAC,kBAAkB,EAAE,CAAC,CAAC,CAAC;YACpC,OAAO,CAAC,QAAQ,GAAG,CAAC,CAAC;QACzB,CAAC;IACL,CAAC;AACL,CAAC;AAED,SAAS,OAAO,CAAC,KAAY,EAAE,QAAa;IACxC,MAAM,MAAM,GAAG,KAAK,CAAC,KAAK,CAAC,GAAG,EAAC,CAAC,CAAC,CAAC;IAClC,IAAI,MAAM,CAAC,MAAM,IAAI,CAAC,IAAI,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC;QAClC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;QAChC,OAAO,QAAQ,CAAC;IACpB,CAAC;SACI,CAAC;QACF,OAAO,QAAQ,CAAC;IACpB,CAAC;AACL,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"tsv.d.ts","sourceRoot":"","sources":["../../src/input/tsv.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAA0B,MAAM,QAAQ,CAAC;AAK3D,wBAAsB,SAAS,CAAC,KAAK,EAAE,GAAG,GAAG,OAAO,CAAC,SAAS,CAAC,CAgD9D"}
|