marcattacks 2.2.5 → 2.2.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +17 -2
- package/dist/attacker.d.ts +24 -1
- package/dist/attacker.d.ts.map +1 -1
- package/dist/attacker.js +139 -125
- package/dist/attacker.js.map +1 -1
- package/dist/command.d.ts +3 -0
- package/dist/command.d.ts.map +1 -0
- package/dist/{index.js → command.js} +12 -2
- package/dist/command.js.map +1 -0
- package/dist/demo.d.ts +2 -0
- package/dist/demo.d.ts.map +1 -0
- package/dist/demo.js +8 -0
- package/dist/demo.js.map +1 -0
- package/dist/globber.d.ts +3 -0
- package/dist/globber.d.ts.map +1 -0
- package/dist/globber.js +131 -0
- package/dist/globber.js.map +1 -0
- package/dist/output/json.d.ts.map +1 -1
- package/dist/output/json.js +3 -5
- package/dist/output/json.js.map +1 -1
- package/dist/output/rdf.js +2 -2
- package/dist/output/xml.d.ts.map +1 -1
- package/dist/output/xml.js +3 -5
- package/dist/output/xml.js.map +1 -1
- package/dist/{filestream.d.ts → stream/filestream.d.ts} +1 -0
- package/dist/stream/filestream.d.ts.map +1 -0
- package/dist/stream/filestream.js +65 -0
- package/dist/stream/filestream.js.map +1 -0
- package/dist/stream/httpstream.d.ts.map +1 -0
- package/dist/{httpstream.js → stream/httpstream.js} +7 -3
- package/dist/stream/httpstream.js.map +1 -0
- package/dist/{s3stream.d.ts → stream/s3stream.d.ts} +1 -0
- package/dist/stream/s3stream.d.ts.map +1 -0
- package/dist/{s3stream.js → stream/s3stream.js} +64 -1
- package/dist/stream/s3stream.js.map +1 -0
- package/dist/{sftpstream.d.ts → stream/sftpstream.d.ts} +1 -0
- package/dist/stream/sftpstream.d.ts.map +1 -0
- package/dist/{sftpstream.js → stream/sftpstream.js} +66 -2
- package/dist/stream/sftpstream.js.map +1 -0
- package/dist/stream/slow-writable.d.ts.map +1 -0
- package/dist/stream/slow-writable.js.map +1 -0
- package/dist/util/stream_helpers.d.ts.map +1 -1
- package/dist/util/stream_helpers.js +9 -13
- package/dist/util/stream_helpers.js.map +1 -1
- package/package.json +3 -2
- package/dist/filestream.d.ts.map +0 -1
- package/dist/filestream.js +0 -32
- package/dist/filestream.js.map +0 -1
- package/dist/httpstream.d.ts.map +0 -1
- package/dist/httpstream.js.map +0 -1
- package/dist/index.d.ts +0 -3
- package/dist/index.d.ts.map +0 -1
- package/dist/index.js.map +0 -1
- package/dist/s3stream.d.ts.map +0 -1
- package/dist/s3stream.js.map +0 -1
- package/dist/sftpstream.d.ts.map +0 -1
- package/dist/sftpstream.js.map +0 -1
- package/dist/slow-writable.d.ts.map +0 -1
- package/dist/slow-writable.js.map +0 -1
- /package/dist/{httpstream.d.ts → stream/httpstream.d.ts} +0 -0
- /package/dist/{slow-writable.d.ts → stream/slow-writable.d.ts} +0 -0
- /package/dist/{slow-writable.js → stream/slow-writable.js} +0 -0
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"globber.js","sourceRoot":"","sources":["../src/globber.ts"],"names":[],"mappings":";AAEA,OAAO,MAAM,MAAM,QAAQ,CAAC;AAC5B,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,MAAM,MAAM,QAAQ,CAAC;AAC5B,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAC5C,OAAO,EAAE,aAAa,EAAE,MAAM,wBAAwB,CAAC;AACvD,OAAO,EAAE,WAAW,EAAE,MAAM,sBAAsB,CAAC;AACnD,OAAO,EAAE,aAAa,EAAE,MAAM,wBAAwB,CAAC;AACvD,MAAM,OAAO,GAAG,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AAC/C,MAAM,GAAG,GAAG,OAAO,CAAC,iBAAiB,CAAC,CAAC;AAEvC,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC;KACvB,QAAQ,CAAC,QAAQ,CAAC;KAClB,MAAM,CAAC,sBAAsB,EAAC,aAAa,EAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,MAAM,CAAC,CAAC;KAC7E,MAAM,CAAC,iBAAiB,EAAE,kBAAkB,CAAC;KAC7C,MAAM,CAAC,iBAAiB,EAAC,kCAAkC,CAAC;KAC5D,MAAM,CAAC,gBAAgB,EAAC,gBAAgB,CAAC;KACzC,MAAM,CAAC,QAAQ,EAAC,2BAA2B,CAAC;KAC5C,MAAM,CAAC,SAAS,EAAC,gCAAgC,CAAC;KAClD,MAAM,CAAC,SAAS,EAAC,qCAAqC,CAAC,CAAC;AAE7D,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;AAE5B,MAAM,IAAI,GAAK,OAAO,CAAC,IAAI,EAAE,CAAC;AAE9B,IAAI,IAAI,CAAC,GAAG,EAAE,CAAC;IACX,IAAI,MAAM,GAAG,QAAQ,CAAC;IACtB,IAAI,IAAI,GAAG,CAAC,CAAC;IAEb,IAAI,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;QAClC,MAAM,GAAG,QAAQ,CAAC;IACtB,CAAC;IACD,IAAI,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC;QAChC,IAAI,GAAG,CAAC,CAAC;IACb,CAAC;IAED,IAAI,IAAI,EAAE,CAAC;QACP,mBAAmB,CAAC,MAAM,CAAC,CAAC;IAChC,CAAC;SACI,CAAC;QACF,sBAAsB,CAAC,MAAM,CAAC,CAAC;IACnC,CAAC;AACL,CAAC;KACI,CAAC;IACF,sBAAsB,CAAC,QAAQ,CAAC,CAAC;AACrC,CAAC;AAED,MAAM,MAAM,GAAG,MAAM,CAAC,SAAS,EAAE,CAAC;AAElC,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;IACZ,MAAM,CAAC,KAAK,GAAG,MAAM,CAAC;AAC1B,CAAC;AAED,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;IACb,MAAM,CAAC,KAAK,GAAG,OAAO,CAAC;AAC3B,CAAC;AAED,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;IACb,MAAM,CAAC,KAAK,GAAG,OAAO,CAAC;AAC3B,CAAC;AAED,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;IACd,MAAM,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,MAAM,EAAG,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;AACvD,CAAC;AAED,IAAI,EAAE,CAAC;AAEP,SAAS,sBAAsB,CAAC,MAAc;IAC1C,MAAM,CAAC,SAAS,CAAC;QACb,SAAS,EAAE;YACP,GAAG,EAAE;gBACD,IAAI,EAAE,MAAM;gBACZ,MAAM,EAAE;oBACJ,IAAI,EAAE,SAAS;oBACf,OAAO,EAAE,oBAAoB;iBAChC;aACJ;SACJ;QACD,UAAU,EAAE;YACR,OAAO,EAAE,EAAE,SAAS,EAAE,CAAC,KAAK,CAAC,EAAE,KAAK,EAAE,KAAK,EAAG,eAAe,EAAE,IAAI,EAAE;SACxE;KACJ,CAAC,CAAC;AACP,CAAC;AAED,SAAS,mBAAmB,CAAC,MAAc;IACvC,MAAM,CAAC,SAAS,CAAC,cAAc,EAAE,CAAC,MAAM,EAAE,EAAE;QACxC,OAAO,CAAC,QAAQ,EAAE,EAAE;YAChB,OAAO,IAAI,CAAC,SAAS,CAAC;gBACtB,SAAS,EAAE,QAAQ,CAAC,SAAS,EAAG,gBAAgB;gBAChD,KAAK,EAAE,QAAQ,CAAC,KAAK,CAAC,QAAQ,EAAE,gBAAgB;gBAChD,QAAQ,EAAE,QAAQ,CAAC,YAAY,EAAC,gBAAgB;gBAChD,OAAO,EAAE,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,gBAAgB;gBAClD,OAAO,EAAE,QAAQ,CAAC,OAAO,EAAO,yBAAyB;gBACzD,GAAG,EAAE,QAAQ,CAAC,GAAG,CAAe,gBAAgB;aAC/C,CAAC,CAAC;QACP,CAAC,CAAC;IACN,CAAC,CAAC,CAAC;IAEH,MAAM,CAAC,SAAS,CAAC;QACb,SAAS,EAAE;YACP,GAAG,EAAE;gBACD,IAAI,EAAE,MAAM;gBACZ,MAAM,EAAE;oBACJ,IAAI,EAAE,cAAc;iBACvB;aACJ;SACJ;QACD,UAAU,EAAE;YACR,OAAO,EAAE,EAAE,SAAS,EAAE,CAAC,KAAK,CAAC,EAAE,KAAK,EAAE,KAAK,EAAG,eAAe,EAAE,IAAI,EAAE;SACxE;KACJ,CAAC,CAAC;AACP,CAAC;AAED,KAAK,UAAU,IAAI;IACf,IAAI,CAAC;QACD,MAAM,GAAG,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAE5B,IAAI,KAAK,GAAW,EAAE,CAAC;QAEvB,IAAI,GAAG,EAAE,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC;YAC1B,KAAK,GAAG,MAAM,aAAa,CAAC,IAAI,GAAG,CAAC,GAAG,CAAC,EAAC,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC;QAC7D,CAAC;aACI,IAAI,GAAG,EAAE,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;YAC7B,MAAM,KAAK,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;YAE3B,IAAI,OAAO,CAAC,GAAG,CAAC,aAAa,EAAE,CAAC;gBAC5B,KAAK,CAAC,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,aAAa,CAAC;YAC/C,CAAC;YAED,IAAI,OAAO,CAAC,GAAG,CAAC,aAAa,EAAE,CAAC;gBAC5B,KAAK,CAAC,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,aAAa,CAAC;YAC/C,CAAC;YAED,KAAK,GAAG,MAAM,WAAW,CAAC,KAAK,EAAC,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC;QACpD,CAAC;aACI,IAAI,GAAG,EAAE,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC;YAC/B,KAAK,GAAG,MAAM,aAAa,CAAC,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;QAC9C,CAAC;aACI,CAAC;YACF,OAAO,CAAC,KAAK,CAAC,GAAG,GAAG,gBAAgB,CAAC,CAAC;QAC1C,CAAC;QAED,KAAK,CAAC,OAAO,CAAE,CAAC,CAAC,EAAE;YACf,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QACxB,CAAC,CAAC,CAAC;IACP,CAAC;IACD,OAAO,CAAC,EAAE,CAAC;QACP,MAAM,CAAC,KAAK,CAAC,kBAAkB,EAAE,CAAC,CAAC,CAAC;QACpC,OAAO,CAAC,QAAQ,GAAG,CAAC,CAAC;IACzB,CAAC;AACL,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"json.d.ts","sourceRoot":"","sources":["../../src/output/json.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,QAAQ,CAAC;AAKnC,wBAAsB,SAAS,CAAC,MAAM,EAAC,GAAG,GAAI,OAAO,CAAC,SAAS,CAAC,
|
|
1
|
+
{"version":3,"file":"json.d.ts","sourceRoot":"","sources":["../../src/output/json.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,QAAQ,CAAC;AAKnC,wBAAsB,SAAS,CAAC,MAAM,EAAC,GAAG,GAAI,OAAO,CAAC,SAAS,CAAC,CAoC/D"}
|
package/dist/output/json.js
CHANGED
|
@@ -3,7 +3,6 @@ import log4js from 'log4js';
|
|
|
3
3
|
const logger = log4js.getLogger();
|
|
4
4
|
export async function transform(_param) {
|
|
5
5
|
let isFirst = true;
|
|
6
|
-
let hasClosed = false;
|
|
7
6
|
return new Transform({
|
|
8
7
|
objectMode: true,
|
|
9
8
|
transform(data, _encoding, callback) {
|
|
@@ -24,11 +23,10 @@ export async function transform(_param) {
|
|
|
24
23
|
logger.trace(`adding ${output.length} bytes`);
|
|
25
24
|
callback(null, output);
|
|
26
25
|
},
|
|
27
|
-
|
|
28
|
-
logger.debug('
|
|
29
|
-
if (!isFirst
|
|
26
|
+
flush(callback) {
|
|
27
|
+
logger.debug('flush reached');
|
|
28
|
+
if (!isFirst) {
|
|
30
29
|
this.push("]");
|
|
31
|
-
hasClosed = true;
|
|
32
30
|
}
|
|
33
31
|
callback();
|
|
34
32
|
}
|
package/dist/output/json.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"json.js","sourceRoot":"","sources":["../../src/output/json.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,QAAQ,CAAC;AACnC,OAAO,MAAM,MAAM,QAAQ,CAAC;AAE5B,MAAM,MAAM,GAAG,MAAM,CAAC,SAAS,EAAE,CAAC;AAElC,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,MAAU;IACtC,IAAI,OAAO,GAAG,IAAI,CAAC;
|
|
1
|
+
{"version":3,"file":"json.js","sourceRoot":"","sources":["../../src/output/json.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,QAAQ,CAAC;AACnC,OAAO,MAAM,MAAM,QAAQ,CAAC;AAE5B,MAAM,MAAM,GAAG,MAAM,CAAC,SAAS,EAAE,CAAC;AAElC,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,MAAU;IACtC,IAAI,OAAO,GAAG,IAAI,CAAC;IAEnB,OAAO,IAAI,SAAS,CAAC;QACjB,UAAU,EAAE,IAAI;QAChB,SAAS,CAAC,IAAS,EAAE,SAAS,EAAE,QAAQ;YACpC,IAAI,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;gBAChC,MAAM,CAAC,KAAK,CAAC,sBAAsB,CAAC,CAAC;gBACrC,QAAQ,EAAE,CAAC;gBACX,OAAO;YACX,CAAC;YAED,IAAI,MAAM,GAAG,EAAE,CAAC;YAEhB,IAAI,OAAO,EAAE,CAAC;gBACV,MAAM,IAAI,GAAG,CAAC;YAClB,CAAC;iBACI,CAAC;gBACF,MAAM,IAAI,GAAG,CAAC;YAClB,CAAC;YAED,MAAM,IAAI,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;YAE/B,OAAO,GAAG,KAAK,CAAC;YAEhB,MAAM,CAAC,KAAK,CAAC,UAAU,MAAM,CAAC,MAAM,QAAQ,CAAC,CAAC;YAC9C,QAAQ,CAAC,IAAI,EAAC,MAAM,CAAC,CAAC;QAC1B,CAAC;QACD,KAAK,CAAC,QAAQ;YACV,MAAM,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC;YAC9B,IAAI,CAAC,OAAO,EAAE,CAAC;gBACX,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACnB,CAAC;YACD,QAAQ,EAAE,CAAC;QACf,CAAC;KACJ,CAAC,CAAC;AACP,CAAC"}
|
package/dist/output/rdf.js
CHANGED
|
@@ -38,8 +38,8 @@ export async function transform(_param) {
|
|
|
38
38
|
logger.warn(`[${counter}] is not a Record or a JSON-LD`);
|
|
39
39
|
}
|
|
40
40
|
},
|
|
41
|
-
|
|
42
|
-
logger.debug('
|
|
41
|
+
flush(callback) {
|
|
42
|
+
logger.debug('flush reached');
|
|
43
43
|
writer?.end();
|
|
44
44
|
callback();
|
|
45
45
|
}
|
package/dist/output/xml.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"xml.d.ts","sourceRoot":"","sources":["../../src/output/xml.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,QAAQ,CAAC;AAMnC,wBAAsB,SAAS,CAAC,MAAM,EAAC,GAAG,GAAI,OAAO,CAAC,SAAS,CAAC,
|
|
1
|
+
{"version":3,"file":"xml.d.ts","sourceRoot":"","sources":["../../src/output/xml.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,QAAQ,CAAC;AAMnC,wBAAsB,SAAS,CAAC,MAAM,EAAC,GAAG,GAAI,OAAO,CAAC,SAAS,CAAC,CA8D/D;AAED,wBAAgB,SAAS,CACvB,KAAK,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,GAAG,SAAS,EACzC,OAAO,CAAC,EAAE;IAAE,YAAY,CAAC,EAAE,OAAO,CAAA;CAAE,GACnC,MAAM,CAqBR"}
|
package/dist/output/xml.js
CHANGED
|
@@ -4,7 +4,6 @@ import log4js from 'log4js';
|
|
|
4
4
|
const logger = log4js.getLogger();
|
|
5
5
|
export async function transform(_param) {
|
|
6
6
|
let isFirst = true;
|
|
7
|
-
let hasClosed = false;
|
|
8
7
|
return new Transform({
|
|
9
8
|
objectMode: true,
|
|
10
9
|
transform(data, _encoding, callback) {
|
|
@@ -45,14 +44,13 @@ export async function transform(_param) {
|
|
|
45
44
|
logger.trace(`adding ${output.length} bytes`);
|
|
46
45
|
callback(null, output);
|
|
47
46
|
},
|
|
48
|
-
|
|
49
|
-
logger.debug('
|
|
50
|
-
if (!isFirst
|
|
47
|
+
flush(callback) {
|
|
48
|
+
logger.debug('flush reached');
|
|
49
|
+
if (!isFirst) {
|
|
51
50
|
logger.debug("flushing");
|
|
52
51
|
let output = "</marc:collection>\n";
|
|
53
52
|
logger.trace(`adding ${output.length} bytes`);
|
|
54
53
|
this.push(output);
|
|
55
|
-
hasClosed = true;
|
|
56
54
|
}
|
|
57
55
|
callback();
|
|
58
56
|
}
|
package/dist/output/xml.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"xml.js","sourceRoot":"","sources":["../../src/output/xml.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,QAAQ,CAAC;AACnC,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,aAAa,EAAG,cAAc,EAAC,MAAM,eAAe,CAAC;AAChF,OAAO,MAAM,MAAM,QAAQ,CAAC;AAE5B,MAAM,MAAM,GAAG,MAAM,CAAC,SAAS,EAAE,CAAC;AAElC,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,MAAU;IACtC,IAAI,OAAO,GAAG,IAAI,CAAC;
|
|
1
|
+
{"version":3,"file":"xml.js","sourceRoot":"","sources":["../../src/output/xml.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,QAAQ,CAAC;AACnC,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,aAAa,EAAG,cAAc,EAAC,MAAM,eAAe,CAAC;AAChF,OAAO,MAAM,MAAM,QAAQ,CAAC;AAE5B,MAAM,MAAM,GAAG,MAAM,CAAC,SAAS,EAAE,CAAC;AAElC,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,MAAU;IACtC,IAAI,OAAO,GAAG,IAAI,CAAC;IAEnB,OAAO,IAAI,SAAS,CAAC;QACjB,UAAU,EAAE,IAAI;QAChB,SAAS,CAAC,IAAS,EAAE,SAAS,EAAE,QAAQ;YACpC,IAAI,GAAG,GAAgB,IAAI,CAAC,QAAQ,CAAC,CAAC;YAEtC,IAAI,CAAC,GAAG,EAAE,CAAC;gBACP,MAAM,CAAC,KAAK,CAAC,sBAAsB,CAAC,CAAC;gBACrC,QAAQ,EAAE,CAAA;gBACV,OAAO;YACX,CAAC;YAED,IAAI,MAAM,GAAG,EAAE,CAAC;YAEhB,IAAI,OAAO,EAAE,CAAC;gBACV,MAAM,IAAI,8CAA8C,CAAC;gBACzD,MAAM,IAAI,mEAAmE,CAAC;gBAC9E,OAAO,GAAG,KAAK,CAAC;YACpB,CAAC;YAED,MAAM,IAAI,kBAAkB,CAAC;YAE7B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAG,CAAC,GAAG,GAAG,CAAC,MAAM,EAAG,CAAC,EAAE,EAAE,CAAC;gBACpC,IAAI,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;gBAC1B,IAAI,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;gBAC1B,IAAI,GAAG,KAAK,KAAK,EAAE,CAAC,CAAA,CAAC;qBAChB,IAAI,GAAG,KAAK,KAAK,EAAE,CAAC;oBACrB,IAAI,KAAK,GAAG,aAAa,CAAC,GAAG,CAAC,CAAC,CAAE,EAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;oBAC3C,MAAM,IAAI,kBAAkB,SAAS,CAAC,KAAK,CAAC,kBAAkB,CAAC;gBACnE,CAAC;qBACI,IAAI,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC;oBACxB,IAAI,KAAK,GAAG,aAAa,CAAC,GAAG,CAAC,CAAC,CAAE,EAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;oBAC3C,MAAM,IAAI,6BAA6B,GAAG,KAAK,SAAS,CAAC,KAAK,CAAC,wBAAwB,CAAC;gBAC5F,CAAC;qBACI,CAAC;oBACF,MAAM,IAAI,0BAA0B,GAAG,WAAW,GAAG,CAAC,CAAC,CAAC,WAAW,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC;oBAChF,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,IAAI,EAAC,KAAK,EAAE,EAAE;wBAClC,MAAM,IAAI,4BAA4B,IAAI,KAAK,SAAS,CAAC,KAAK,CAAC,oBAAoB,CAAC;oBACxF,CAAC,CAAC,CAAC;oBACH,MAAM,IAAI,uBAAuB,CAAC;gBACtC,CAAC;YACL,CAAC;YAED,MAAM,IAAI,mBAAmB,CAAC;YAE9B,MAAM,CAAC,KAAK,CAAC,UAAU,MAAM,CAAC,MAAM,QAAQ,CAAC,CAAC;YAE9C,QAAQ,CAAC,IAAI,EAAC,MAAM,CAAC,CAAC;QAC1B,CAAC;QACD,KAAK,CAAC,QAAQ;YACV,MAAM,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC;YAC9B,IAAI,CAAC,OAAO,EAAE,CAAC;gBACX,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;gBACzB,IAAI,MAAM,GAAG,sBAAsB,CAAC;gBACpC,MAAM,CAAC,KAAK,CAAC,UAAU,MAAM,CAAC,MAAM,QAAQ,CAAC,CAAC;gBAC9C,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACtB,CAAC;YACD,QAAQ,EAAE,CAAC;QACf,CAAC;KACJ,CAAC,CAAC;AACP,CAAC;AAED,MAAM,UAAU,SAAS,CACvB,KAAyC,EACzC,OAAoC;IAElC,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,KAAK,SAAS;QAAE,OAAO,EAAE,CAAC;IAErD,IAAI,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;IAEtB,uDAAuD;IACvD,0DAA0D;IAC1D,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,uDAAuD,EAAE,EAAE,CAAC,CAAC;IAE3E,6EAA6E;IAC7E,wDAAwD;IACxD,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,0CAA0C,EAAE,OAAO,CAAC,CAAC;IAEnE,kBAAkB;IAClB,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;IAElD,IAAI,OAAO,EAAE,YAAY,EAAE,CAAC;QACxB,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;IAC1D,CAAC;IAED,OAAO,CAAC,CAAC;AACb,CAAC"}
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import type { Readable } from "stream";
|
|
2
2
|
export declare function fileReadStream(url: URL): Promise<Readable>;
|
|
3
3
|
export declare function fileLatestFile(url: URL): Promise<URL>;
|
|
4
|
+
export declare function fileGlobFiles(url: URL): Promise<URL[]>;
|
|
4
5
|
//# sourceMappingURL=filestream.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"filestream.d.ts","sourceRoot":"","sources":["../../src/stream/filestream.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,QAAQ,CAAC;AAKvC,wBAAsB,cAAc,CAAC,GAAG,EAAE,GAAG,GAAI,OAAO,CAAC,QAAQ,CAAC,CAEjE;AAED,wBAAsB,cAAc,CAAC,GAAG,EAAE,GAAG,GAAI,OAAO,CAAC,GAAG,CAAC,CA+B5D;AAED,wBAAsB,aAAa,CAAC,GAAG,EAAE,GAAG,GAAG,OAAO,CAAC,GAAG,EAAE,CAAC,CAsC5D"}
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
import fs from "fs";
|
|
2
|
+
import log4js from 'log4js';
|
|
3
|
+
const logger = log4js.getLogger();
|
|
4
|
+
export async function fileReadStream(url) {
|
|
5
|
+
return fs.createReadStream(url);
|
|
6
|
+
}
|
|
7
|
+
export async function fileLatestFile(url) {
|
|
8
|
+
return new Promise((resolve) => {
|
|
9
|
+
logger.info(`trying to resolve ${url.href}`);
|
|
10
|
+
if (!url.href.includes("@latest:")) {
|
|
11
|
+
logger.info(`resolved as ${url.href}`);
|
|
12
|
+
resolve(url);
|
|
13
|
+
return;
|
|
14
|
+
}
|
|
15
|
+
const directory = url.pathname.replaceAll(/@latest:.*/g, "");
|
|
16
|
+
const extension = url.pathname.replaceAll(/.*@latest:/g, "");
|
|
17
|
+
logger.debug(`directory: ${directory} ; extension: ${extension}`);
|
|
18
|
+
fs.readdir(directory, (err, files) => {
|
|
19
|
+
if (err) {
|
|
20
|
+
throw new Error("Error finding latest file:", err);
|
|
21
|
+
}
|
|
22
|
+
for (let i = 0; i < files.length; i++) {
|
|
23
|
+
if (files[i]?.toLowerCase().endsWith(extension)) {
|
|
24
|
+
logger.info(`resolved as file://${directory}${files[i]}`);
|
|
25
|
+
resolve(new URL("file://" + directory + files[i]));
|
|
26
|
+
return;
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
throw new Error("No latest file found!");
|
|
30
|
+
});
|
|
31
|
+
});
|
|
32
|
+
}
|
|
33
|
+
export async function fileGlobFiles(url) {
|
|
34
|
+
return new Promise((resolve, reject) => {
|
|
35
|
+
logger.info(`trying to glob files for ${url.href}`);
|
|
36
|
+
// Check if the URL follows the @glob: pattern
|
|
37
|
+
if (!url.href.includes("@glob:")) {
|
|
38
|
+
logger.info(`no glob pattern found, returning original URL in array`);
|
|
39
|
+
resolve([url]);
|
|
40
|
+
return;
|
|
41
|
+
}
|
|
42
|
+
// Extract directory path and extension
|
|
43
|
+
const directory = url.pathname.replaceAll(/@glob:.*/g, "");
|
|
44
|
+
const extension = url.pathname.replaceAll(/.*@glob:/g, "");
|
|
45
|
+
logger.debug(`directory: ${directory} ; extension: ${extension}`);
|
|
46
|
+
fs.readdir(directory, (err, files) => {
|
|
47
|
+
if (err) {
|
|
48
|
+
// Reject the promise if the directory cannot be read
|
|
49
|
+
return reject(new Error(`Error reading directory for glob: ${err.message}`));
|
|
50
|
+
}
|
|
51
|
+
const targetExt = extension.toLowerCase();
|
|
52
|
+
const matchedUrls = [];
|
|
53
|
+
// Filter all files in the directory that match the extension
|
|
54
|
+
for (const file of files) {
|
|
55
|
+
if (file.toLowerCase().endsWith(targetExt)) {
|
|
56
|
+
// Reconstruct the file URL
|
|
57
|
+
matchedUrls.push(new URL("file://" + directory + file));
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
logger.info(`glob resolved ${matchedUrls.length} files`);
|
|
61
|
+
resolve(matchedUrls);
|
|
62
|
+
});
|
|
63
|
+
});
|
|
64
|
+
}
|
|
65
|
+
//# sourceMappingURL=filestream.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"filestream.js","sourceRoot":"","sources":["../../src/stream/filestream.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,IAAI,CAAC;AAEpB,OAAO,MAAM,MAAM,QAAQ,CAAC;AAE5B,MAAM,MAAM,GAAG,MAAM,CAAC,SAAS,EAAE,CAAC;AAElC,MAAM,CAAC,KAAK,UAAU,cAAc,CAAC,GAAQ;IACzC,OAAO,EAAE,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC;AACpC,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,cAAc,CAAC,GAAQ;IACzC,OAAO,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE;QACjC,MAAM,CAAC,IAAI,CAAC,qBAAqB,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC;QAE7C,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC,EAAE,CAAC;YACjC,MAAM,CAAC,IAAI,CAAC,eAAe,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC;YACvC,OAAO,CAAC,GAAG,CAAC,CAAC;YACb,OAAO;QACX,CAAC;QAED,MAAM,SAAS,GAAG,GAAG,CAAC,QAAQ,CAAC,UAAU,CAAC,aAAa,EAAC,EAAE,CAAC,CAAC;QAC5D,MAAM,SAAS,GAAG,GAAG,CAAC,QAAQ,CAAC,UAAU,CAAC,aAAa,EAAC,EAAE,CAAC,CAAC;QAE5D,MAAM,CAAC,KAAK,CAAC,cAAc,SAAS,iBAAiB,SAAS,EAAE,CAAC,CAAC;QAElE,EAAE,CAAC,OAAO,CAAC,SAAS,EAAE,CAAC,GAAG,EAAC,KAAK,EAAE,EAAE;YACjC,IAAI,GAAG,EAAE,CAAC;gBACT,MAAM,IAAI,KAAK,CAAC,4BAA4B,EAAE,GAAG,CAAC,CAAC;YACpD,CAAC;YAED,KAAK,IAAI,CAAC,GAAG,CAAC,EAAG,CAAC,GAAG,KAAK,CAAC,MAAM,EAAG,CAAC,EAAE,EAAE,CAAC;gBACzC,IAAI,KAAK,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC;oBAC9C,MAAM,CAAC,IAAI,CAAC,sBAAsB,SAAS,GAAG,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;oBAC1D,OAAO,CAAC,IAAI,GAAG,CAAC,SAAS,GAAG,SAAS,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;oBACnD,OAAO;gBACX,CAAC;YACF,CAAC;YAED,MAAM,IAAI,KAAK,CAAC,uBAAuB,CAAC,CAAC;QAC5C,CAAC,CAAC,CAAC;IACP,CAAC,CAAC,CAAC;AACP,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,aAAa,CAAC,GAAQ;IACxC,OAAO,IAAI,OAAO,CAAQ,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QAC1C,MAAM,CAAC,IAAI,CAAC,4BAA4B,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC;QAEpD,8CAA8C;QAC9C,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC;YAC/B,MAAM,CAAC,IAAI,CAAC,wDAAwD,CAAC,CAAC;YACtE,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;YACf,OAAO;QACX,CAAC;QAED,uCAAuC;QACvC,MAAM,SAAS,GAAG,GAAG,CAAC,QAAQ,CAAC,UAAU,CAAC,WAAW,EAAE,EAAE,CAAC,CAAC;QAC3D,MAAM,SAAS,GAAG,GAAG,CAAC,QAAQ,CAAC,UAAU,CAAC,WAAW,EAAE,EAAE,CAAC,CAAC;QAE3D,MAAM,CAAC,KAAK,CAAC,cAAc,SAAS,iBAAiB,SAAS,EAAE,CAAC,CAAC;QAElE,EAAE,CAAC,OAAO,CAAC,SAAS,EAAE,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE;YACjC,IAAI,GAAG,EAAE,CAAC;gBACN,qDAAqD;gBACrD,OAAO,MAAM,CAAC,IAAI,KAAK,CAAC,qCAAqC,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;YACjF,CAAC;YAED,MAAM,SAAS,GAAG,SAAS,CAAC,WAAW,EAAE,CAAC;YAC1C,MAAM,WAAW,GAAU,EAAE,CAAC;YAE9B,6DAA6D;YAC7D,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;gBACvB,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC;oBACzC,2BAA2B;oBAC3B,WAAW,CAAC,IAAI,CAAC,IAAI,GAAG,CAAC,SAAS,GAAG,SAAS,GAAG,IAAI,CAAC,CAAC,CAAC;gBAC5D,CAAC;YACL,CAAC;YAED,MAAM,CAAC,IAAI,CAAC,iBAAiB,WAAW,CAAC,MAAM,QAAQ,CAAC,CAAC;YACzD,OAAO,CAAC,WAAW,CAAC,CAAC;QACzB,CAAC,CAAC,CAAC;IACP,CAAC,CAAC,CAAC;AACP,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"httpstream.d.ts","sourceRoot":"","sources":["../../src/stream/httpstream.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,QAAQ,CAAC;AAQlC,wBAAgB,cAAc,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,QAAQ,CAAC,CAsCnE"}
|
|
@@ -9,13 +9,17 @@ export function httpReadStream(urlString) {
|
|
|
9
9
|
try {
|
|
10
10
|
const url = new URL(urlString);
|
|
11
11
|
const client = url.protocol === 'http:' ? http : https;
|
|
12
|
+
logger.debug(`resolve ${url.href}`);
|
|
12
13
|
const req = client.get(url, res => {
|
|
13
|
-
|
|
14
|
+
const statusCode = res.statusCode || 0;
|
|
15
|
+
logger.debug(`statusCode = ${statusCode}`);
|
|
16
|
+
if (statusCode >= 400) {
|
|
14
17
|
reject(new Error('HTTP ' + res.statusCode));
|
|
15
18
|
return;
|
|
16
19
|
}
|
|
17
|
-
// Follow redirects
|
|
18
|
-
if (
|
|
20
|
+
// Follow redirects (without any sanity checks..i know)
|
|
21
|
+
if (statusCode >= 300 && statusCode < 400 && res.headers.location) {
|
|
22
|
+
logger.info(`Redirect to ${res.headers.location}...`);
|
|
19
23
|
httpReadStream(res.headers.location).then(resolve).catch(reject);
|
|
20
24
|
return;
|
|
21
25
|
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"httpstream.js","sourceRoot":"","sources":["../../src/stream/httpstream.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,QAAQ,CAAC;AAClC,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAC7B,OAAO,KAAK,KAAK,MAAM,OAAO,CAAC;AAC/B,OAAO,EAAE,GAAG,EAAE,MAAM,KAAK,CAAC;AAC1B,OAAO,MAAM,MAAM,QAAQ,CAAC;AAE5B,MAAM,MAAM,GAAG,MAAM,CAAC,SAAS,EAAE,CAAC;AAElC,MAAM,UAAU,cAAc,CAAC,SAAiB;IAC5C,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QACnC,IAAI,CAAC;YACD,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,SAAS,CAAC,CAAC;YAC/B,MAAM,MAAM,GAAG,GAAG,CAAC,QAAQ,KAAK,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC;YAEvD,MAAM,CAAC,KAAK,CAAC,WAAW,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC;YAEpC,MAAM,GAAG,GAAG,MAAM,CAAC,GAAG,CAAC,GAAG,EAAE,GAAG,CAAC,EAAE;gBAC9B,MAAM,UAAU,GAAG,GAAG,CAAC,UAAU,IAAI,CAAC,CAAC;gBAEvC,MAAM,CAAC,KAAK,CAAC,gBAAgB,UAAU,EAAE,CAAC,CAAC;gBAE3C,IAAI,UAAU,IAAI,GAAG,EAAE,CAAC;oBACpB,MAAM,CAAC,IAAI,KAAK,CAAC,OAAO,GAAG,GAAG,CAAC,UAAU,CAAC,CAAC,CAAC;oBAC5C,OAAO;gBACX,CAAC;gBAED,uDAAuD;gBACvD,IAAI,UAAU,IAAI,GAAG,IAAI,UAAU,GAAG,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC;oBAChE,MAAM,CAAC,IAAI,CAAC,eAAe,GAAG,CAAC,OAAO,CAAC,QAAQ,KAAK,CAAC,CAAC;oBACtD,cAAc,CAAC,GAAG,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;oBACjE,OAAO;gBACX,CAAC;gBAED,OAAO,CAAC,GAAG,CAAC,CAAC;YACjB,CAAC,CAAC,CAAC;YAEH,GAAG,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,EAAE;gBACtB,MAAM,CAAC,KAAK,CAAC,sBAAsB,EAAE,KAAK,CAAC,CAAC;gBAC5C,MAAM,CAAC,KAAK,CAAC,CAAC;YAClB,CAAC,CAAC,CAAC;QACP,CAAC;QACD,OAAO,KAAK,EAAE,CAAC;YACX,MAAM,CAAC,KAAK,CAAC,qBAAqB,EAAE,KAAK,CAAC,CAAC;YAC3C,MAAM,CAAC,KAAK,CAAC,CAAC;QAClB,CAAC;IACL,CAAC,CAAC,CAAC;AACP,CAAC"}
|
|
@@ -6,4 +6,5 @@ export declare function s3WriteStream(url: URL, options: {
|
|
|
6
6
|
partSize?: number;
|
|
7
7
|
}): Promise<Writable>;
|
|
8
8
|
export declare function s3LatestObject(url: URL, opts: any): Promise<URL>;
|
|
9
|
+
export declare function s3GlobFiles(url: URL, opts: any): Promise<URL[]>;
|
|
9
10
|
//# sourceMappingURL=s3stream.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"s3stream.d.ts","sourceRoot":"","sources":["../../src/stream/s3stream.ts"],"names":[],"mappings":"AAWA,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,QAAQ,CAAC;AAa5C,wBAAsB,YAAY,CAAC,GAAG,EAAE,GAAG,EAAE,OAAO,EAAE;IAAE,KAAK,CAAC,EAAE,MAAM,CAAA;CAAE,GAAG,OAAO,CAAC,QAAQ,CAAC,CA0F3F;AAED,wBAAgB,aAAa,CAAC,GAAG,EAAE,GAAG,EAAE,OAAO,EAAE;IAAE,QAAQ,CAAC,EAAE,MAAM,CAAC;CAAC,GAAI,OAAO,CAAC,QAAQ,CAAC,CA+F1F;AAED,wBAAsB,cAAc,CAAC,GAAG,EAAE,GAAG,EAAE,IAAI,EAAE,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,CA2EtE;AAED,wBAAsB,WAAW,CAAC,GAAG,EAAE,GAAG,EAAE,IAAI,EAAE,GAAG,GAAG,OAAO,CAAC,GAAG,EAAE,CAAC,CA2ErE"}
|
|
@@ -175,12 +175,15 @@ export function s3WriteStream(url, options) {
|
|
|
175
175
|
});
|
|
176
176
|
}
|
|
177
177
|
export async function s3LatestObject(url, opts) {
|
|
178
|
+
logger.info(`trying to resolve ${url.href}`);
|
|
178
179
|
if (!url.href.includes("@latest:")) {
|
|
180
|
+
logger.info(`resolved as: ${url.href}`);
|
|
179
181
|
return url;
|
|
180
182
|
}
|
|
181
183
|
const bucket = url.pathname.replaceAll(/@latest:.*/g, "");
|
|
182
184
|
const extension = url.pathname.replaceAll(/.*@latest:/g, "");
|
|
183
185
|
const config = parseURL(url);
|
|
186
|
+
logger.debug(`s3 config:`, config);
|
|
184
187
|
const s3Client = new S3Client(config);
|
|
185
188
|
const paginatorConfig = {
|
|
186
189
|
client: s3Client,
|
|
@@ -226,7 +229,7 @@ export async function s3LatestObject(url, opts) {
|
|
|
226
229
|
url_parts.push(url.port);
|
|
227
230
|
}
|
|
228
231
|
url_parts.push(bucket + '/' + latestFile.Key);
|
|
229
|
-
logger.
|
|
232
|
+
logger.info(`resolved as: ${url_parts.join("")}`);
|
|
230
233
|
return new URL(url_parts.join(""));
|
|
231
234
|
}
|
|
232
235
|
catch (error) {
|
|
@@ -234,6 +237,66 @@ export async function s3LatestObject(url, opts) {
|
|
|
234
237
|
throw error;
|
|
235
238
|
}
|
|
236
239
|
}
|
|
240
|
+
export async function s3GlobFiles(url, opts) {
|
|
241
|
+
logger.info(`trying to glob files for ${url.href}`);
|
|
242
|
+
// Check if the URL follows the @glob: pattern
|
|
243
|
+
if (!url.href.includes("@glob:")) {
|
|
244
|
+
logger.info(`no glob pattern found, returning original URL in array`);
|
|
245
|
+
return [url];
|
|
246
|
+
}
|
|
247
|
+
// Extract bucket name and extension from the pathname
|
|
248
|
+
const bucket = url.pathname.replaceAll(/@glob:.*/g, "").split("/")[1] ?? "";
|
|
249
|
+
const extension = url.pathname.replaceAll(/.*@glob:/g, "");
|
|
250
|
+
const config = parseURL(url);
|
|
251
|
+
logger.debug(`s3 config:`, config);
|
|
252
|
+
const s3Client = new S3Client(config);
|
|
253
|
+
const paginatorConfig = {
|
|
254
|
+
client: s3Client,
|
|
255
|
+
pageSize: 1000
|
|
256
|
+
};
|
|
257
|
+
const commandInput = {
|
|
258
|
+
Bucket: bucket,
|
|
259
|
+
};
|
|
260
|
+
try {
|
|
261
|
+
const matchedUrls = [];
|
|
262
|
+
const targetExt = extension.toLowerCase();
|
|
263
|
+
// Iterate through all pages of the bucket using the V2 paginator
|
|
264
|
+
for await (const page of paginateListObjectsV2(paginatorConfig, commandInput)) {
|
|
265
|
+
const contents = page.Contents || [];
|
|
266
|
+
for (const obj of contents) {
|
|
267
|
+
// Filter by extension match
|
|
268
|
+
if (obj.Key?.toLowerCase().endsWith(targetExt)) {
|
|
269
|
+
// Reconstruct the URL for each matched object
|
|
270
|
+
const url_parts = [];
|
|
271
|
+
url_parts.push(url.protocol);
|
|
272
|
+
url_parts.push('//');
|
|
273
|
+
if (url.username) {
|
|
274
|
+
url_parts.push(url.username);
|
|
275
|
+
if (url.password) {
|
|
276
|
+
url_parts.push(':');
|
|
277
|
+
url_parts.push(url.password);
|
|
278
|
+
}
|
|
279
|
+
url_parts.push('@');
|
|
280
|
+
}
|
|
281
|
+
url_parts.push(url.hostname);
|
|
282
|
+
if (url.port && !["80", "443"].includes(url.port)) {
|
|
283
|
+
url_parts.push(':');
|
|
284
|
+
url_parts.push(url.port);
|
|
285
|
+
}
|
|
286
|
+
// Note: bucket is already part of the path logic in parseURL
|
|
287
|
+
url_parts.push(`/${bucket}/${obj.Key}`);
|
|
288
|
+
matchedUrls.push(new URL(url_parts.join("")));
|
|
289
|
+
}
|
|
290
|
+
}
|
|
291
|
+
}
|
|
292
|
+
logger.info(`glob resolved ${matchedUrls.length} files`);
|
|
293
|
+
return matchedUrls;
|
|
294
|
+
}
|
|
295
|
+
catch (error) {
|
|
296
|
+
logger.error("Error globbing S3 files:", error);
|
|
297
|
+
throw error;
|
|
298
|
+
}
|
|
299
|
+
}
|
|
237
300
|
function isNodeReadable(x) {
|
|
238
301
|
return x && typeof x.pipe === "function" && typeof x.read === "function";
|
|
239
302
|
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"s3stream.js","sourceRoot":"","sources":["../../src/stream/s3stream.ts"],"names":[],"mappings":"AAAA,OAAO,EACH,QAAQ,EACR,gBAAgB,EAChB,gBAAgB,EAChB,iBAAiB,EACjB,4BAA4B,EAC5B,8BAA8B,EAC9B,qBAAqB,EAGxB,MAAM,oBAAoB,CAAC;AAC5B,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,QAAQ,CAAC;AAC5C,OAAO,MAAM,MAAM,QAAQ,CAAC;AAE5B,MAAM,MAAM,GAAG,MAAM,CAAC,SAAS,EAAE,CAAC;AAUlC,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,GAAQ,EAAE,OAA2B;IACpE,MAAM,MAAM,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC;IAE7B,MAAM,CAAC,KAAK,CAAC,YAAY,EAAC,MAAM,CAAC,CAAC;IAElC,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC;IAC7B,MAAM,GAAG,GAAM,MAAM,CAAC,GAAG,CAAC;IAC1B,MAAM,KAAK,GAAI,OAAO,CAAC,KAAK,CAAC;IAC7B,MAAM,EAAE,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC;IAE9B,MAAM,GAAG,GAAG,MAAM,EAAE,CAAC,IAAI,CAAC,IAAI,gBAAgB,CAAC;QAC3C,MAAM,EAAE,MAAM;QACd,GAAG,EAAE,GAAG;QACR,KAAK,EAAE,KAAK;KACf,CAAC,CAAC,CAAC;IAEJ,MAAM,IAAI,GAAG,GAAG,CAAC,IAAI,CAAC;IAEtB,IAAI,CAAC,IAAI,EAAE,CAAC;QACR,MAAM,IAAI,KAAK,CAAC,qCAAqC,CAAC,CAAC;IAC3D,CAAC;IAED,iEAAiE;IACjE,IAAI,cAAc,CAAC,IAAI,CAAC,EAAE,CAAC;QACvB,OAAO,IAAgB,CAAC;IAC5B,CAAC;IAED,6EAA6E;IAC7E,IAAI,gBAAgB,CAAC,IAAI,CAAC,EAAE,CAAC;QACzB,oCAAoC;QACpC,gCAAgC;QAChC,IAAI,OAAQ,QAAgB,CAAC,OAAO,KAAK,UAAU,EAAE,CAAC;YACtD,OAAQ,QAAgB,CAAC,OAAO,CAAC,IAAkC,CAAC,CAAC;QACrE,CAAC;aAAM,CAAC;YACR,sDAAsD;YACtD,MAAM,MAAM,GAAI,IAAmC,CAAC,SAAS,EAAE,CAAC;YAChE,MAAM,UAAU,GAAG,IAAI,QAAQ,CAAC;gBAC5B,IAAI;oBACA,0CAA0C;gBAC9C,CAAC;aACJ,CAAC,CAAC;YACH,CAAC,KAAK,IAAI,EAAE;gBACR,IAAI,CAAC;oBACL,OAAO,IAAI,EAAE,CAAC;wBACV,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;wBAC5C,IAAI,IAAI;4BAAE,MAAM;wBAChB,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;oBACxC,CAAC;oBACD,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBACtB,CAAC;gBAAC,OAAO,GAAG,EAAE,CAAC;oBACf,UAAU,CAAC,OAAO,CAAC,GAAY,CAAC,CAAC;gBACjC,CAAC;YACL,CAAC,CAAC,EAAE,CAAC;YACL,OAAO,UAAU,CAAC;QAClB,CAAC;IACL,CAAC;IAED,uCAAuC;IACvC,IAAI,OAAO,IAAI,KAAK,WAAW,IAAI,IAAI,YAAY,IAAI,EAAE,CAAC;QACtD,MAAM,MAAM,GAAI,IAAa,CAAC,MAAM,EAAE,CAAC;QACvC,IAAI,OAAQ,QAAgB,CAAC,OAAO,KAAK,UAAU,EAAE,CAAC;YAClD,OAAQ,QAAgB,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QAC7C,CAAC;QACD,yBAAyB;QACzB,MAAM,MAAM,GAAG,MAAM,CAAC,SAAS,EAAE,CAAC;QAClC,MAAM,UAAU,GAAG,IAAI,QAAQ,CAAC;YAChC,IAAI,KAAI,CAAC;SACR,CAAC,CAAC;QACH,CAAC,KAAK,IAAI,EAAE;YACZ,IAAI,CAAC;gBACD,OAAO,IAAI,EAAE,CAAC;oBACd,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;oBAC5C,IAAI,IAAI;wBAAE,MAAM;oBAChB,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;gBACpC,CAAC;gBACD,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC1B,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACX,UAAU,CAAC,OAAO,CAAC,GAAY,CAAC,CAAC;YACrC,CAAC;QACD,CAAC,CAAC,EAAE,CAAC;QACL,OAAO,UAAU,CAAC;IACtB,CAAC;IAED,+CAA+C;IAC/C,IAAI,eAAe,CAAC,IAAI,CAAC,EAAE,CAAC;QACxB,OAAO,QAAQ,CAAC,IAAI,CAAC,IAAmD,CAAC,CAAC;IAC9E,CAAC;IAED,qBAAqB;IACrB,MAAM,IAAI,KAAK,CAAC,oCAAoC,CAAC,CAAC;AAC1D,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,GAAQ,EAAE,OAA8B;IAClE,OAAO,IAAI,OAAO,CAAY,CAAC,OAAO,EAAE,EAAE;QACtC,MAAM,MAAM,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC;QAE7B,MAAM,CAAC,KAAK,CAAC,YAAY,EAAE,MAAM,CAAC,CAAC;QACnC,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC;QAC7B,MAAM,GAAG,GAAG,MAAM,CAAC,GAAG,CAAC;QACvB,MAAM,EAAE,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC;QAC9B,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,CAAC,GAAG,IAAI,GAAG,IAAI,CAAC;QAErD,IAAI,QAAQ,GAAkB,IAAI,CAAC;QACnC,IAAI,KAAK,GAA4D,EAAE,CAAC;QACxE,IAAI,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QAC7B,IAAI,UAAU,GAAG,CAAC,CAAC;QAEnB,MAAM,MAAM,GAAG,IAAI,QAAQ,CAAC;YACxB,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,SAAS,EAAE,QAAQ;gBAClC,MAAM,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAC;gBAC/B,IAAI,CAAC;oBACD,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC,CAAC;oBAExC,IAAI,MAAM,CAAC,MAAM,IAAI,QAAQ,EAAE,CAAC;wBAC5B,MAAM,SAAS,EAAE,CAAC;oBACtB,CAAC;oBACD,QAAQ,EAAE,CAAC;gBACf,CAAC;gBAAC,OAAO,GAAG,EAAE,CAAC;oBACX,QAAQ,CAAC,GAAY,CAAC,CAAC;gBAC3B,CAAC;YACL,CAAC;YAED,KAAK,CAAC,KAAK,CAAC,QAAQ;gBAChB,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;gBACzB,IAAI,CAAC;oBACD,MAAM,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC;oBAC7B,MAAM,SAAS,CAAC,IAAI,CAAC,CAAC;oBACtB,MAAM,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAC;oBAChC,MAAM,YAAY,EAAE,CAAC;oBACrB,QAAQ,EAAE,CAAC;gBACf,CAAC;gBAAC,OAAO,GAAG,EAAE,CAAC;oBACX,QAAQ,CAAC,GAAY,CAAC,CAAC;gBAC3B,CAAC;YACL,CAAC;SACJ,CAAC,CAAC;QAEH,KAAK,UAAU,YAAY;YACvB,IAAI,CAAC,QAAQ,EAAE,CAAC;gBACZ,MAAM,GAAG,GAAG,MAAM,EAAE,CAAC,IAAI,CAAC,IAAI,4BAA4B,CAAC;oBACvD,MAAM,EAAE,MAAM;oBACd,GAAG,EAAE,GAAG;iBACX,CAAC,CAAC,CAAC;gBACJ,QAAQ,GAAG,GAAG,CAAC,QAAS,CAAC;YAC7B,CAAC;QACL,CAAC;QAED,KAAK,UAAU,SAAS,CAAC,MAAM,GAAG,KAAK;YACnC,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,IAAI,CAAC,MAAM;gBAAE,OAAO;YAE3C,MAAM,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAC;YAChC,MAAM,YAAY,EAAE,CAAC;YAErB,MAAM,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC;YAC3B,MAAM,GAAG,GAAG,MAAM,EAAE,CAAC,IAAI,CAAC,IAAI,iBAAiB,CAAC;gBAC5C,MAAM,EAAE,MAAM;gBACd,GAAG,EAAE,GAAG;gBACR,UAAU,EAAE,UAAU;gBACtB,QAAQ,EAAE,QAAS;gBACnB,IAAI,EAAE,MAAM;aACf,CAAC,CAAC,CAAC;YAEJ,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,GAAG,CAAC,IAAI,EAAE,UAAU,EAAE,UAAU,EAAE,CAAC,CAAC;YACvD,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;YACzB,UAAU,EAAE,CAAC;QACjB,CAAC;QAED,KAAK,UAAU,YAAY;YACvB,IAAI,CAAC,QAAQ,EAAE,CAAC;gBACZ,wCAAwC;gBACxC,MAAM,EAAE,CAAC,IAAI,CAAC,IAAI,gBAAgB,CAAC;oBAC/B,MAAM,EAAE,MAAM;oBACd,GAAG,EAAE,GAAG;oBACR,IAAI,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;iBACxB,CAAC,CAAC,CAAC;gBACJ,OAAO;YACX,CAAC;YAED,MAAM,EAAE,CAAC,IAAI,CAAC,IAAI,8BAA8B,CAAC;gBAC7C,MAAM,EAAE,MAAM;gBACd,GAAG,EAAE,GAAG;gBACR,QAAQ,EAAE,QAAS;gBACnB,eAAe,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE;aACpC,CAAC,CAAC,CAAC;QACR,CAAC;QAED,OAAO,CAAC,MAAM,CAAC,CAAC;IACpB,CAAC,CAAC,CAAC;AACP,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,cAAc,CAAC,GAAQ,EAAE,IAAS;IACpD,MAAM,CAAC,IAAI,CAAC,qBAAqB,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC;IAE7C,IAAI,CAAE,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC,EAAE,CAAC;QAClC,MAAM,CAAC,IAAI,CAAC,gBAAgB,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC;QACxC,OAAO,GAAG,CAAC;IACf,CAAC;IAED,MAAM,MAAM,GAAG,GAAG,CAAC,QAAQ,CAAC,UAAU,CAAC,aAAa,EAAC,EAAE,CAAC,CAAC;IACzD,MAAM,SAAS,GAAG,GAAG,CAAC,QAAQ,CAAC,UAAU,CAAC,aAAa,EAAC,EAAE,CAAC,CAAC;IAE5D,MAAM,MAAM,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC;IAE7B,MAAM,CAAC,KAAK,CAAC,YAAY,EAAE,MAAM,CAAC,CAAC;IAEnC,MAAM,QAAQ,GAAG,IAAI,QAAQ,CAAC,MAAM,CAAC,CAAC;IAEtC,MAAM,eAAe,GAAG;QACpB,MAAM,EAAE,QAAQ;QAChB,QAAQ,EAAE,IAAI;KACjB,CAAC;IAEF,MAAM,YAAY,GAAG;QACjB,MAAM,EAAE,MAAM;QACd,gCAAgC;KACnC,CAAC;IAEF,IAAI,CAAC;QACD,IAAI,UAAU,GAAmB,IAAI,CAAC;QACtC,MAAM,SAAS,GAAG,SAAS,CAAC,WAAW,EAAE,CAAC;QAE1C,0CAA0C;QAC1C,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,qBAAqB,CAAC,eAAe,EAAE,YAAY,CAAC,EAAE,CAAC;YAC5E,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,IAAI,EAAE,CAAC;YAErC,KAAK,MAAM,GAAG,IAAI,QAAQ,EAAE,CAAC;gBACzB,sBAAsB;gBACtB,IAAI,GAAG,CAAC,GAAG,EAAE,WAAW,EAAE,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC;oBAC7C,6CAA6C;oBAC7C,IAAI,CAAC,UAAU,IAAI,CAAC,GAAG,CAAC,YAAa,GAAG,UAAU,CAAC,YAAa,CAAC,EAAE,CAAC;wBAChE,UAAU,GAAG,GAAG,CAAC;oBACrB,CAAC;gBACL,CAAC;YACL,CAAC;QACL,CAAC;QAED,IAAI,CAAC,UAAU,IAAI,CAAC,UAAU,CAAC,GAAG,EAAE,CAAC;YACjC,MAAM,IAAI,KAAK,CAAC,2BAA2B,SAAS,sBAAsB,MAAM,IAAI,CAAC,CAAC;QAC1F,CAAC;QAED,MAAM,SAAS,GAAc,EAAE,CAAC;QAEhC,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QAC7B,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACrB,IAAI,GAAG,CAAC,QAAQ,EAAE,CAAC;YACf,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;YAC7B,IAAI,GAAG,CAAC,QAAQ,EAAE,CAAC;gBACf,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;gBACnB,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;YACjC,CAAC;YACD,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACxB,CAAC;QACD,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QAC7B,IAAI,CAAC,CAAC,GAAG,CAAC,IAAI,KAAK,IAAI,IAAI,GAAG,CAAC,IAAI,KAAK,KAAK,CAAC,EAAE,CAAC;YAC7C,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACpB,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QAC7B,CAAC;QACD,SAAS,CAAC,IAAI,CAAC,MAAM,GAAG,GAAG,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC;QAE9C,MAAM,CAAC,IAAI,CAAC,gBAAgB,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;QAClD,OAAO,IAAI,GAAG,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC;IACvC,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACb,OAAO,CAAC,KAAK,CAAC,+BAA+B,EAAE,KAAK,CAAC,CAAC;QACtD,MAAM,KAAK,CAAC;IAChB,CAAC;AACL,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,GAAQ,EAAE,IAAS;IACjD,MAAM,CAAC,IAAI,CAAC,4BAA4B,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC;IAEpD,8CAA8C;IAC9C,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC/B,MAAM,CAAC,IAAI,CAAC,wDAAwD,CAAC,CAAC;QACtE,OAAO,CAAC,GAAG,CAAC,CAAC;IACjB,CAAC;IAED,sDAAsD;IACtD,MAAM,MAAM,GAAG,GAAG,CAAC,QAAQ,CAAC,UAAU,CAAC,WAAW,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IAC5E,MAAM,SAAS,GAAG,GAAG,CAAC,QAAQ,CAAC,UAAU,CAAC,WAAW,EAAE,EAAE,CAAC,CAAC;IAE3D,MAAM,MAAM,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC;IAE7B,MAAM,CAAC,KAAK,CAAC,YAAY,EAAE,MAAM,CAAC,CAAC;IAEnC,MAAM,QAAQ,GAAG,IAAI,QAAQ,CAAC,MAAM,CAAC,CAAC;IAEtC,MAAM,eAAe,GAAG;QACpB,MAAM,EAAE,QAAQ;QAChB,QAAQ,EAAE,IAAI;KACjB,CAAC;IAEF,MAAM,YAAY,GAAG;QACjB,MAAM,EAAE,MAAM;KACjB,CAAC;IAEF,IAAI,CAAC;QACD,MAAM,WAAW,GAAU,EAAE,CAAC;QAC9B,MAAM,SAAS,GAAG,SAAS,CAAC,WAAW,EAAE,CAAC;QAE1C,iEAAiE;QACjE,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,qBAAqB,CAAC,eAAe,EAAE,YAAY,CAAC,EAAE,CAAC;YAC5E,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,IAAI,EAAE,CAAC;YAErC,KAAK,MAAM,GAAG,IAAI,QAAQ,EAAE,CAAC;gBACzB,4BAA4B;gBAC5B,IAAI,GAAG,CAAC,GAAG,EAAE,WAAW,EAAE,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC;oBAC7C,8CAA8C;oBAC9C,MAAM,SAAS,GAAa,EAAE,CAAC;oBAE/B,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;oBAC7B,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;oBAErB,IAAI,GAAG,CAAC,QAAQ,EAAE,CAAC;wBACf,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;wBAC7B,IAAI,GAAG,CAAC,QAAQ,EAAE,CAAC;4BACf,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;4BACpB,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;wBACjC,CAAC;wBACD,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;oBACxB,CAAC;oBAED,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;oBAE7B,IAAI,GAAG,CAAC,IAAI,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;wBAChD,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;wBACpB,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;oBAC7B,CAAC;oBAED,6DAA6D;oBAC7D,SAAS,CAAC,IAAI,CAAC,IAAI,MAAM,IAAI,GAAG,CAAC,GAAG,EAAE,CAAC,CAAC;oBAExC,WAAW,CAAC,IAAI,CAAC,IAAI,GAAG,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;gBAClD,CAAC;YACL,CAAC;QACL,CAAC;QAED,MAAM,CAAC,IAAI,CAAC,iBAAiB,WAAW,CAAC,MAAM,QAAQ,CAAC,CAAC;QACzD,OAAO,WAAW,CAAC;IACvB,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACb,MAAM,CAAC,KAAK,CAAC,0BAA0B,EAAE,KAAK,CAAC,CAAC;QAChD,MAAM,KAAK,CAAC;IAChB,CAAC;AACL,CAAC;AAED,SAAS,cAAc,CAAC,CAAM;IAC1B,OAAO,CAAC,IAAI,OAAO,CAAC,CAAC,IAAI,KAAK,UAAU,IAAI,OAAO,CAAC,CAAC,IAAI,KAAK,UAAU,CAAC;AAC7E,CAAC;AAED,SAAS,gBAAgB,CAAC,CAAM;IAC5B,OAAO,OAAO,CAAC,EAAE,SAAS,KAAK,UAAU,CAAC;AAC9C,CAAC;AAED,SAAS,eAAe,CAAC,CAAM;IAC3B,OAAO,CAAC,IAAI,OAAO,CAAC,CAAC,MAAM,CAAC,aAAa,CAAC,KAAK,UAAU,CAAC;AAC9D,CAAC;AAED,SAAS,UAAU,CAAC,MAAgB;IAChC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;IACrB,MAAM,QAAQ,GAAoB;QAC9B,QAAQ,EAAE,MAAM,CAAC,QAAQ;QACzB,cAAc,EAAE,IAAI;QACpB,MAAM,EAAE,MAAM,CAAC,MAAM;KACxB,CAAC;IAEF,IAAI,MAAM,CAAC,WAAW,IAAI,MAAM,CAAC,eAAe,EAAE,CAAC;QAC/C,QAAQ,CAAC,WAAW,GAAG;YACnB,WAAW,EAAE,MAAM,CAAC,WAAW;YAC/B,eAAe,EAAE,MAAM,CAAC,eAAe;SAC1C,CAAC;IACN,CAAC;IACD,OAAO,IAAI,QAAQ,CAAC,QAAQ,CAAC,CAAC;AAClC,CAAC;AAED,SAAS,QAAQ,CAAC,GAAQ;IACtB,MAAM,MAAM,GAAc;QACtB,MAAM,EAAE,WAAW;QACnB,QAAQ,EAAE,uBAAuB;QACjC,MAAM,EAAE,KAAK;QACb,GAAG,EAAE,UAAU;KAClB,CAAC;IAEF,MAAM,MAAM,GAAG,GAAG,CAAC,QAAQ,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC;IACjE,MAAM,CAAC,QAAQ,GAAG,GAAG,MAAM,MAAM,GAAG,CAAC,QAAQ,EAAE,CAAC;IAChD,IAAI,GAAG,CAAC,IAAI,EAAE,CAAC;QACX,MAAM,CAAC,QAAQ,IAAI,IAAI,GAAG,CAAC,IAAI,EAAE,CAAC;IACtC,CAAC;IACD,MAAM,CAAC,MAAM,GAAG,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IACjD,MAAM,CAAC,GAAG,GAAG,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAEzD,IAAI,GAAG,CAAC,QAAQ,EAAE,CAAC;QACf,MAAM,CAAC,WAAW,GAAG,GAAG,CAAC,QAAQ,CAAC;IACtC,CAAC;IAED,IAAI,GAAG,CAAC,QAAQ,EAAE,CAAC;QACf,MAAM,CAAC,eAAe,GAAG,GAAG,CAAC,QAAQ,CAAC;IAC1C,CAAC;IAED,OAAO,MAAM,CAAC;AAClB,CAAC"}
|
|
@@ -9,4 +9,5 @@ export interface SftpConfig {
|
|
|
9
9
|
export declare function sftpReadStream(url: URL, opts: any): Promise<Readable>;
|
|
10
10
|
export declare function sftpWriteStream(url: URL, opts: any): Promise<Writable>;
|
|
11
11
|
export declare function sftpLatestFile(url: URL, opts: any): Promise<URL>;
|
|
12
|
+
export declare function sftpGlobFiles(url: URL, opts: any): Promise<URL[]>;
|
|
12
13
|
//# sourceMappingURL=sftpstream.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"sftpstream.d.ts","sourceRoot":"","sources":["../../src/stream/sftpstream.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,QAAQ,EAAG,QAAQ,EAAE,MAAM,QAAQ,CAAC;AAM7C,MAAM,WAAW,UAAU;IACzB,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,UAAU,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;CAC9B;AAED,wBAAsB,cAAc,CAAC,GAAG,EAAE,GAAG,EAAE,IAAI,EAAE,GAAG,GAAG,OAAO,CAAC,QAAQ,CAAC,CAmC3E;AAED,wBAAsB,eAAe,CAAC,GAAG,EAAE,GAAG,EAAE,IAAI,EAAE,GAAG,GAAG,OAAO,CAAC,QAAQ,CAAC,CAiC5E;AAED,wBAAsB,cAAc,CAAC,GAAG,EAAE,GAAG,EAAE,IAAI,EAAE,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,CA+EtE;AAED,wBAAsB,aAAa,CAAC,GAAG,EAAE,GAAG,EAAE,IAAI,EAAE,GAAG,GAAG,OAAO,CAAC,GAAG,EAAE,CAAC,CA+EvE"}
|
|
@@ -58,9 +58,11 @@ export async function sftpWriteStream(url, opts) {
|
|
|
58
58
|
}
|
|
59
59
|
export async function sftpLatestFile(url, opts) {
|
|
60
60
|
const config = makeSftpConfig(url, opts);
|
|
61
|
+
logger.info(`trying to resolve ${url.href}`);
|
|
61
62
|
logger.debug(`sftp config:`, config);
|
|
62
63
|
return new Promise((resolve, reject) => {
|
|
63
|
-
if (!url.pathname.match(/\/@latest:\
|
|
64
|
+
if (!url.pathname.match(/\/@latest:\S+$/)) {
|
|
65
|
+
logger.info(`resolved as: ${url.href}`);
|
|
64
66
|
resolve(url);
|
|
65
67
|
return;
|
|
66
68
|
}
|
|
@@ -108,7 +110,7 @@ export async function sftpLatestFile(url, opts) {
|
|
|
108
110
|
url_parts.push(url.port);
|
|
109
111
|
}
|
|
110
112
|
url_parts.push(latestPath);
|
|
111
|
-
logger.
|
|
113
|
+
logger.info(`resolved as: ${url_parts.join("")}`);
|
|
112
114
|
resolve(new URL(url_parts.join("")));
|
|
113
115
|
});
|
|
114
116
|
});
|
|
@@ -117,6 +119,68 @@ export async function sftpLatestFile(url, opts) {
|
|
|
117
119
|
conn.connect(config);
|
|
118
120
|
});
|
|
119
121
|
}
|
|
122
|
+
export async function sftpGlobFiles(url, opts) {
|
|
123
|
+
const config = makeSftpConfig(url, opts);
|
|
124
|
+
logger.info(`trying to glob files for ${url.href}`);
|
|
125
|
+
return new Promise((resolve, reject) => {
|
|
126
|
+
// Check if the URL follows the @glob: pattern
|
|
127
|
+
if (!url.pathname.match(/\/@glob:\S+$/)) {
|
|
128
|
+
logger.info(`no glob pattern found, returning original URL in array`);
|
|
129
|
+
resolve([url]);
|
|
130
|
+
return;
|
|
131
|
+
}
|
|
132
|
+
const remoteDir = url.pathname.replace(/\/@glob.*/, "");
|
|
133
|
+
const extension = url.pathname.replace(/.*\/@glob:/, "");
|
|
134
|
+
const conn = new Client();
|
|
135
|
+
conn.on("ready", () => {
|
|
136
|
+
conn.sftp((err, sftp) => {
|
|
137
|
+
if (err) {
|
|
138
|
+
conn.end();
|
|
139
|
+
return reject(err);
|
|
140
|
+
}
|
|
141
|
+
sftp.readdir(remoteDir, (err, list) => {
|
|
142
|
+
if (err) {
|
|
143
|
+
conn.end();
|
|
144
|
+
return reject(err);
|
|
145
|
+
}
|
|
146
|
+
if (!list || list.length === 0) {
|
|
147
|
+
conn.end();
|
|
148
|
+
return resolve([]); // Return empty array if no files exist
|
|
149
|
+
}
|
|
150
|
+
// Filter files by the extension provided after @glob:
|
|
151
|
+
const matchedFiles = list.filter(f => f.filename.toLowerCase().endsWith(extension.toLowerCase()));
|
|
152
|
+
const results = matchedFiles.map(file => {
|
|
153
|
+
const filePath = `${remoteDir}/${file.filename}`;
|
|
154
|
+
// Reconstruct the URL for each matched file
|
|
155
|
+
const url_parts = [];
|
|
156
|
+
url_parts.push(url.protocol);
|
|
157
|
+
url_parts.push('//');
|
|
158
|
+
if (url.username) {
|
|
159
|
+
url_parts.push(url.username);
|
|
160
|
+
if (url.password) {
|
|
161
|
+
url_parts.push(':');
|
|
162
|
+
url_parts.push(url.password);
|
|
163
|
+
}
|
|
164
|
+
url_parts.push('@');
|
|
165
|
+
}
|
|
166
|
+
url_parts.push(url.hostname);
|
|
167
|
+
if (url.port && !["80", "443"].includes(url.port)) {
|
|
168
|
+
url_parts.push(':');
|
|
169
|
+
url_parts.push(url.port);
|
|
170
|
+
}
|
|
171
|
+
url_parts.push(filePath);
|
|
172
|
+
return new URL(url_parts.join(""));
|
|
173
|
+
});
|
|
174
|
+
conn.end();
|
|
175
|
+
logger.info(`glob resolved ${results.length} files`);
|
|
176
|
+
resolve(results);
|
|
177
|
+
});
|
|
178
|
+
});
|
|
179
|
+
});
|
|
180
|
+
conn.on("error", (err) => reject(err));
|
|
181
|
+
conn.connect(config);
|
|
182
|
+
});
|
|
183
|
+
}
|
|
120
184
|
function makeSftpConfig(inputFile, opts) {
|
|
121
185
|
let privateKey = undefined;
|
|
122
186
|
if (opts.key) {
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"sftpstream.js","sourceRoot":"","sources":["../../src/stream/sftpstream.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,MAAM,CAAC;AAC9B,OAAO,EAAE,QAAQ,EAAG,QAAQ,EAAE,MAAM,QAAQ,CAAC;AAC7C,OAAO,EAAE,MAAM,IAAI,CAAC;AACpB,OAAO,MAAM,MAAM,QAAQ,CAAC;AAE5B,MAAM,MAAM,GAAG,MAAM,CAAC,SAAS,EAAE,CAAC;AAUlC,MAAM,CAAC,KAAK,UAAU,cAAc,CAAC,GAAQ,EAAE,IAAS;IACpD,MAAM,MAAM,GAAG,cAAc,CAAC,GAAG,EAAC,IAAI,CAAC,CAAC;IAExC,MAAM,CAAC,KAAK,CAAC,cAAc,EAAE,MAAM,CAAC,CAAC;IAErC,MAAM,UAAU,GAAG,GAAG,CAAC,QAAQ,CAAC;IAEhC,MAAM,CAAC,IAAI,CAAC,eAAe,UAAU,EAAE,CAAC,CAAC;IAEzC,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QACnC,MAAM,IAAI,GAAG,IAAI,MAAM,EAAE,CAAC;QAE1B,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE;YAClB,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE;gBACpB,IAAI,GAAG,EAAE,CAAC;oBACN,IAAI,CAAC,GAAG,EAAE,CAAC;oBACX,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC;gBACvB,CAAC;gBAED,MAAM,MAAM,GAAG,IAAI,CAAC,gBAAgB,CAAC,UAAU,CAAC,CAAC;gBAEjD,kDAAkD;gBAClD,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;gBACrC,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAQ,EAAE,EAAE;oBAC5B,IAAI,CAAC,GAAG,EAAE,CAAC;oBACX,MAAM,CAAC,GAAG,CAAC,CAAC;gBAChB,CAAC,CAAC,CAAC;gBAEH,OAAO,CAAC,MAAM,CAAC,CAAC;YACpB,CAAC,CAAC,CAAC;QACP,CAAC,CAAC,CAAC;QAEH,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;QACvC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;IACzB,CAAC,CAAC,CAAC;AACP,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,eAAe,CAAC,GAAQ,EAAE,IAAS;IACrD,MAAM,MAAM,GAAG,cAAc,CAAC,GAAG,EAAC,IAAI,CAAC,CAAC;IAExC,MAAM,CAAC,KAAK,CAAC,cAAc,EAAE,MAAM,CAAC,CAAC;IAErC,IAAI,UAAU,GAAG,GAAG,CAAC,QAAQ,CAAC;IAE9B,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QACnC,MAAM,IAAI,GAAG,IAAI,MAAM,EAAE,CAAC;QAE1B,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE;YAClB,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE;gBACpB,IAAI,GAAG,EAAE,CAAC;oBACN,IAAI,CAAC,GAAG,EAAE,CAAC;oBACX,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC;gBACvB,CAAC;gBAED,MAAM,MAAM,GAAG,IAAI,CAAC,iBAAiB,CAAC,UAAU,EAAE,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC,CAAC;gBAEzE,kDAAkD;gBAClD,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;gBACrC,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAQ,EAAE,EAAE;oBAC5B,IAAI,CAAC,GAAG,EAAE,CAAC;oBACX,MAAM,CAAC,GAAG,CAAC,CAAC;gBAChB,CAAC,CAAC,CAAC;gBAEH,OAAO,CAAC,MAAM,CAAC,CAAC;YACpB,CAAC,CAAC,CAAC;QACP,CAAC,CAAC,CAAC;QAEH,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;QACvC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;IACzB,CAAC,CAAC,CAAC;AACP,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,cAAc,CAAC,GAAQ,EAAE,IAAS;IACpD,MAAM,MAAM,GAAG,cAAc,CAAC,GAAG,EAAC,IAAI,CAAC,CAAC;IAExC,MAAM,CAAC,IAAI,CAAC,qBAAqB,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC;IAC7C,MAAM,CAAC,KAAK,CAAC,cAAc,EAAE,MAAM,CAAC,CAAC;IAErC,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QACnC,IAAI,CAAE,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,gBAAgB,CAAC,EAAE,CAAC;YACzC,MAAM,CAAC,IAAI,CAAC,gBAAgB,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC;YACxC,OAAO,CAAC,GAAG,CAAC,CAAC;YACb,OAAO;QACX,CAAC;QAED,MAAM,SAAS,GAAG,GAAG,CAAC,QAAQ,CAAC,OAAO,CAAC,aAAa,EAAC,EAAE,CAAC,CAAC;QACzD,MAAM,SAAS,GAAG,GAAG,CAAC,QAAQ,CAAC,OAAO,CAAC,cAAc,EAAC,EAAE,CAAC,CAAC;QAE1D,MAAM,IAAI,GAAG,IAAI,MAAM,EAAE,CAAC;QAE1B,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE;YAClB,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE;gBACpB,IAAI,GAAG,EAAE,CAAC;oBACN,IAAI,CAAC,GAAG,EAAE,CAAC;oBACX,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC;gBACvB,CAAC;gBAED,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE;oBAClC,IAAI,GAAG,EAAE,CAAC;wBACN,IAAI,CAAC,GAAG,EAAE,CAAC;wBACX,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC;oBACvB,CAAC;oBAED,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;wBAC7B,IAAI,CAAC,GAAG,EAAE,CAAC;wBACX,OAAO,MAAM,CAAC,IAAI,KAAK,CAAC,6BAA6B,CAAC,CAAC,CAAC;oBAC5D,CAAC;oBAED,yBAAyB;oBACzB,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,CAAC;oBAE/E,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;wBACvB,IAAI,CAAC,GAAG,EAAE,CAAC;wBACX,OAAO,MAAM,CAAC,IAAI,KAAK,CAAC,MAAM,SAAS,2BAA2B,CAAC,CAAC,CAAC;oBACzE,CAAC;oBAED,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,IAAI,EAAE,EAAE,CACzC,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CACtD,CAAC;oBAEF,MAAM,UAAU,GAAG,GAAG,SAAS,IAAI,MAAM,CAAC,QAAQ,EAAE,CAAC;oBACrD,IAAI,CAAC,GAAG,EAAE,CAAC;oBAEX,MAAM,SAAS,GAAc,EAAE,CAAC;oBAEhC,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;oBAC7B,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;oBACrB,IAAI,GAAG,CAAC,QAAQ,EAAE,CAAC;wBACf,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;wBAC7B,IAAI,GAAG,CAAC,QAAQ,EAAE,CAAC;4BACf,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;4BACnB,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;wBACjC,CAAC;wBACD,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;oBACxB,CAAC;oBACD,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;oBAC7B,IAAI,CAAC,CAAC,GAAG,CAAC,IAAI,KAAK,IAAI,IAAI,GAAG,CAAC,IAAI,KAAK,KAAK,CAAC,EAAE,CAAC;wBAC7C,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;wBACpB,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;oBAC7B,CAAC;oBACD,SAAS,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;oBAE3B,MAAM,CAAC,IAAI,CAAC,gBAAgB,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;oBAClD,OAAO,CAAC,IAAI,GAAG,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;gBACzC,CAAC,CAAC,CAAC;YACP,CAAC,CAAC,CAAC;QACP,CAAC,CAAC,CAAC;QAEH,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;QACvC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;IACzB,CAAC,CAAC,CAAC;AACP,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,aAAa,CAAC,GAAQ,EAAE,IAAS;IACnD,MAAM,MAAM,GAAG,cAAc,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;IAEzC,MAAM,CAAC,IAAI,CAAC,4BAA4B,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC;IAEpD,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QACnC,8CAA8C;QAC9C,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,cAAc,CAAC,EAAE,CAAC;YACtC,MAAM,CAAC,IAAI,CAAC,wDAAwD,CAAC,CAAC;YACtE,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;YACf,OAAO;QACX,CAAC;QAED,MAAM,SAAS,GAAG,GAAG,CAAC,QAAQ,CAAC,OAAO,CAAC,WAAW,EAAE,EAAE,CAAC,CAAC;QACxD,MAAM,SAAS,GAAG,GAAG,CAAC,QAAQ,CAAC,OAAO,CAAC,YAAY,EAAE,EAAE,CAAC,CAAC;QAEzD,MAAM,IAAI,GAAG,IAAI,MAAM,EAAE,CAAC;QAE1B,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE;YAClB,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE;gBACpB,IAAI,GAAG,EAAE,CAAC;oBACN,IAAI,CAAC,GAAG,EAAE,CAAC;oBACX,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC;gBACvB,CAAC;gBAED,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE;oBAClC,IAAI,GAAG,EAAE,CAAC;wBACN,IAAI,CAAC,GAAG,EAAE,CAAC;wBACX,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC;oBACvB,CAAC;oBAED,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;wBAC7B,IAAI,CAAC,GAAG,EAAE,CAAC;wBACX,OAAO,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC,uCAAuC;oBAC/D,CAAC;oBAED,sDAAsD;oBACtD,MAAM,YAAY,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CACjC,CAAC,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,SAAS,CAAC,WAAW,EAAE,CAAC,CAC7D,CAAC;oBAEF,MAAM,OAAO,GAAG,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE;wBACpC,MAAM,QAAQ,GAAG,GAAG,SAAS,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;wBAEjD,4CAA4C;wBAC5C,MAAM,SAAS,GAAa,EAAE,CAAC;wBAC/B,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;wBAC7B,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;wBAErB,IAAI,GAAG,CAAC,QAAQ,EAAE,CAAC;4BACf,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;4BAC7B,IAAI,GAAG,CAAC,QAAQ,EAAE,CAAC;gCACf,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gCACpB,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;4BACjC,CAAC;4BACD,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;wBACxB,CAAC;wBAED,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;wBAE7B,IAAI,GAAG,CAAC,IAAI,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;4BAChD,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;4BACpB,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;wBAC7B,CAAC;wBAED,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;wBACzB,OAAO,IAAI,GAAG,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC;oBACvC,CAAC,CAAC,CAAC;oBAEH,IAAI,CAAC,GAAG,EAAE,CAAC;oBACX,MAAM,CAAC,IAAI,CAAC,iBAAiB,OAAO,CAAC,MAAM,QAAQ,CAAC,CAAC;oBACrD,OAAO,CAAC,OAAO,CAAC,CAAC;gBACrB,CAAC,CAAC,CAAC;YACP,CAAC,CAAC,CAAC;QACP,CAAC,CAAC,CAAC;QAEH,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;QACvC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;IACzB,CAAC,CAAC,CAAC;AACP,CAAC;AAED,SAAS,cAAc,CAAC,SAAc,EAAE,IAAS;IAC7C,IAAI,UAAU,GAAwB,SAAS,CAAC;IAEhD,IAAI,IAAI,CAAC,GAAG,EAAE,CAAC;QACX,UAAU,GAAG,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,GAAG,EAAC,EAAE,QAAQ,EAAE,OAAO,EAAC,CAAC,CAAC;IAChE,CAAC;SACI,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;QACnB,UAAU,GAAG,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAC1C,CAAC;IAED,IAAI,MAAM,GAAe;QACrB,IAAI,EAAE,SAAS,CAAC,QAAQ;QACxB,IAAI,EAAE,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,EAAE;QAClC,QAAQ,EAAE,SAAS,CAAC,QAAQ;KAC/B,CAAC;IAEF,IAAI,SAAS,CAAC,QAAQ,EAAE,CAAC;QAAC,MAAM,CAAC,QAAQ,GAAG,SAAS,CAAC,QAAQ,CAAA;IAAC,CAAC;IAChE,IAAI,UAAU,EAAE,CAAC;QAAC,MAAM,CAAC,UAAU,GAAG,UAAU,CAAA;IAAA,CAAC;IAEjD,OAAO,MAAM,CAAC;AAClB,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"slow-writable.d.ts","sourceRoot":"","sources":["../../src/stream/slow-writable.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,QAAQ,EAAE,KAAK,eAAe,EAAE,MAAM,QAAQ,CAAC;AAKxD,MAAM,WAAW,mBAAoB,SAAQ,eAAe;IAC1D;;OAEG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;IAEjB;;;;OAIG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;IAExB;;;OAGG;IACH,mBAAmB,CAAC,EAAE,MAAM,CAAC;CAC9B;AAED;;;GAGG;AACH,qBAAa,YAAa,SAAQ,QAAQ;IACxC,OAAO,CAAC,OAAO,CAAS;IACxB,OAAO,CAAC,cAAc,CAAS;IAC/B,OAAO,CAAC,mBAAmB,CAAS;IACpC,OAAO,CAAC,QAAQ,CAAK;IACrB,OAAO,CAAC,KAAK,CAKL;IACR,OAAO,CAAC,UAAU,CAAK;IACvB,OAAO,CAAC,aAAa,CAAS;gBAElB,IAAI,GAAE,mBAAwB;IAU1C,MAAM,CAAC,KAAK,EAAE,GAAG,EAAE,QAAQ,EAAE,cAAc,EAAE,QAAQ,EAAE,CAAC,GAAG,CAAC,EAAE,KAAK,GAAG,IAAI,KAAK,IAAI,GAAG,IAAI;IAY1F,OAAO,CAAC,YAAY;YAwBN,iBAAiB;IA6C/B,MAAM,CAAC,QAAQ,EAAE,CAAC,GAAG,CAAC,EAAE,KAAK,GAAG,IAAI,KAAK,IAAI,GAAG,IAAI;IAgBpD,QAAQ,CAAC,GAAG,EAAE,KAAK,GAAG,IAAI,EAAE,QAAQ,EAAE,CAAC,KAAK,CAAC,EAAE,KAAK,GAAG,IAAI,KAAK,IAAI,GAAG,IAAI;CAa5E"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"slow-writable.js","sourceRoot":"","sources":["../../src/stream/slow-writable.ts"],"names":[],"mappings":"AAAA,mBAAmB;AACnB,OAAO,EAAE,QAAQ,EAAwB,MAAM,QAAQ,CAAC;AACxD,OAAO,MAAM,MAAM,QAAQ,CAAC;AAE5B,MAAM,MAAM,GAAG,MAAM,CAAC,SAAS,EAAE,CAAC;AAsBlC;;;GAGG;AACH,MAAM,OAAO,YAAa,SAAQ,QAAQ;IAChC,OAAO,CAAS;IAChB,cAAc,CAAS;IACvB,mBAAmB,CAAS;IAC5B,QAAQ,GAAG,CAAC,CAAC;IACb,KAAK,GAKR,EAAE,CAAC;IACA,UAAU,GAAG,CAAC,CAAC;IACf,aAAa,GAAG,KAAK,CAAC;IAE9B,YAAY,OAA4B,EAAE;QACxC,+EAA+E;QAC/E,MAAM,EAAE,OAAO,GAAG,GAAG,EAAE,cAAc,GAAG,CAAC,EAAE,mBAAmB,GAAG,CAAC,EAAE,GAAG,YAAY,EAAE,GAAG,IAAI,CAAC;QAC7F,KAAK,CAAC,YAAY,CAAC,CAAC;QACpB,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;QACvB,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC,CAAC;QAC9D,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,mBAAmB,CAAC,CAAC,CAAC;IAC1E,CAAC;IAED,uCAAuC;IACvC,MAAM,CAAC,KAAU,EAAE,QAAwB,EAAE,QAAsC;QACjF,IAAI,IAAI,CAAC,aAAa,EAAE,CAAC;YACvB,QAAQ,CAAC,IAAI,KAAK,CAAC,qBAAqB,CAAC,CAAC,CAAC;YAC3C,OAAO;QACT,CAAC;QAED,MAAM,GAAG,GAAG,EAAE,IAAI,CAAC,UAAU,CAAC;QAC9B,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,QAAQ,EAAE,QAAQ,EAAE,GAAG,EAAE,CAAC,CAAC;QACpD,IAAI,CAAC,YAAY,EAAE,CAAC;IACtB,CAAC;IAED,gDAAgD;IACxC,YAAY;QAClB,2DAA2D;QAC3D,OAAO,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,cAAc,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,CAAC;YAC3F,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,EAAG,CAAC;YACjC,IAAI,CAAC,QAAQ,EAAE,CAAC;YAChB,IAAI,CAAC,iBAAiB,CAAC,IAAI,CAAC;iBACzB,IAAI,CAAC,GAAG,EAAE;gBACT,IAAI,CAAC,QAAQ,EAAE,CAAC;gBAChB,2CAA2C;gBAC3C,uCAAuC;gBACvC,OAAO,CAAC,QAAQ,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,YAAY,EAAE,CAAC,CAAC;YAC9C,CAAC,CAAC;iBACD,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;gBACb,IAAI,CAAC,QAAQ,EAAE,CAAC;gBAChB,iEAAiE;gBACjE,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;gBACnB,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;gBACxB,4BAA4B;gBAC5B,OAAO,CAAC,QAAQ,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,YAAY,EAAE,CAAC,CAAC;YAC9C,CAAC,CAAC,CAAC;QACP,CAAC;IACH,CAAC;IAED,yDAAyD;IACjD,KAAK,CAAC,iBAAiB,CAAC,IAK/B;QACC,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACrC,MAAM,UAAU,GACd,IAAI,CAAC,mBAAmB,GAAG,CAAC,IAAI,IAAI,CAAC,GAAG,GAAG,IAAI,CAAC,mBAAmB,KAAK,CAAC,CAAC;YAE5E,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE;gBAC5B,iEAAiE;gBACjE,8CAA8C;gBAC9C,sCAAsC;gBACtC,MAAM,CAAC,IAAI,CAAC,8BAA8B,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;gBAEtD,IAAI,UAAU,EAAE,CAAC;oBACf,MAAM,GAAG,GAAG,IAAI,KAAK,CAAC,0BAA0B,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;oBAC5D,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;oBACnB,MAAM,CAAC,GAAG,CAAC,CAAC;gBACd,CAAC;qBAAM,CAAC;oBACN,IAAI,CAAC,QAAQ,EAAE,CAAC;oBAChB,OAAO,EAAE,CAAC;gBACZ,CAAC;YACH,CAAC,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;YAEjB,yEAAyE;YACzE,MAAM,SAAS,GAAG,GAAG,EAAE;gBACrB,YAAY,CAAC,KAAK,CAAC,CAAC;gBACpB,MAAM,GAAG,GAAG,IAAI,KAAK,CAAC,gCAAgC,CAAC,CAAC;gBACxD,IAAI,CAAC;oBACH,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;gBACrB,CAAC;gBAAC,OAAO,CAAC,EAAE,CAAC;oBACX,SAAS;gBACX,CAAC;gBACD,MAAM,CAAC,GAAG,CAAC,CAAC;YACd,CAAC,CAAC;YAEF,yFAAyF;YACzF,IAAI,IAAI,CAAC,aAAa,EAAE,CAAC;gBACvB,SAAS,EAAE,CAAC;YACd,CAAC;QACH,CAAC,CAAC,CAAC;IACL,CAAC;IAED,MAAM,CAAC,QAAsC;QAC3C,gDAAgD;QAChD,MAAM,KAAK,GAAG,GAAG,EAAE;YACjB,IAAI,IAAI,CAAC,aAAa,EAAE,CAAC;gBACvB,QAAQ,CAAC,IAAI,KAAK,CAAC,oCAAoC,CAAC,CAAC,CAAC;gBAC1D,OAAO;YACT,CAAC;YACD,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC,IAAI,IAAI,CAAC,QAAQ,KAAK,CAAC,EAAE,CAAC;gBACnD,QAAQ,EAAE,CAAC;YACb,CAAC;iBAAM,CAAC;gBACN,UAAU,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;YACxB,CAAC;QACH,CAAC,CAAC;QACF,KAAK,EAAE,CAAC;IACV,CAAC;IAED,QAAQ,CAAC,GAAiB,EAAE,QAAwC;QAClE,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC;QAC1B,sCAAsC;QACtC,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC7B,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,EAAG,CAAC;YACjC,IAAI,CAAC;gBACH,IAAI,CAAC,QAAQ,CAAC,GAAG,IAAI,IAAI,KAAK,CAAC,kBAAkB,CAAC,CAAC,CAAC;YACtD,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACX,SAAS;YACX,CAAC;QACH,CAAC;QACD,QAAQ,CAAC,GAAG,CAAC,CAAC;IAChB,CAAC;CACF"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"stream_helpers.d.ts","sourceRoot":"","sources":["../../src/util/stream_helpers.ts"],"names":[],"mappings":"AAAA,OAAO,EAAY,SAAS,EAA0B,MAAM,QAAQ,CAAC;AAOrE;;;;GAIG;AACH,wBAAgB,4BAA4B,CACxC,KAAK,CAAC,EAAE,MAAM,EACd,IAAI,GAAE,MAAU,GACjB,SAAS,
|
|
1
|
+
{"version":3,"file":"stream_helpers.d.ts","sourceRoot":"","sources":["../../src/util/stream_helpers.ts"],"names":[],"mappings":"AAAA,OAAO,EAAY,SAAS,EAA0B,MAAM,QAAQ,CAAC;AAOrE;;;;GAIG;AACH,wBAAgB,4BAA4B,CACxC,KAAK,CAAC,EAAE,MAAM,EACd,IAAI,GAAE,MAAU,GACjB,SAAS,CAgDX;AAED;;GAEG;AACH,UAAU,aAAc,SAAQ,SAAS;IACrC,QAAQ,IAAI,MAAM,CAAC;CACtB;AACD,wBAAgB,mBAAmB,IAAK,aAAa,CAmCpD;AAED;;GAEG;AACH,wBAAgB,wBAAwB,IAAK,SAAS,CAErD;AAED;;GAEG;AACH,wBAAsB,oBAAoB,IAAI,OAAO,CAAC,SAAS,CAAC,CA8D/D"}
|
|
@@ -49,7 +49,6 @@ export function createCountableSkippedStream(count, skip = 0) {
|
|
|
49
49
|
}
|
|
50
50
|
export function createVerboseStream() {
|
|
51
51
|
let recordNum = 0;
|
|
52
|
-
let flushed = false;
|
|
53
52
|
const start = performance.now();
|
|
54
53
|
const transform = new Transform({
|
|
55
54
|
objectMode: true,
|
|
@@ -66,19 +65,16 @@ export function createVerboseStream() {
|
|
|
66
65
|
}
|
|
67
66
|
callback(null, chunk);
|
|
68
67
|
},
|
|
69
|
-
|
|
68
|
+
flush(callback) {
|
|
70
69
|
logger.debug('final reached');
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
return recordNum;
|
|
80
|
-
};
|
|
81
|
-
}
|
|
70
|
+
const end = performance.now();
|
|
71
|
+
const duration = (end - start) / 1000;
|
|
72
|
+
const speed = recordNum / duration;
|
|
73
|
+
logger.info(`process ${recordNum} records in ${duration.toFixed(2)} seconds (${speed.toFixed(0)} recs/sec)`);
|
|
74
|
+
transform.getCount = () => {
|
|
75
|
+
logger.trace(`called getCount -> ${recordNum}`);
|
|
76
|
+
return recordNum;
|
|
77
|
+
};
|
|
82
78
|
callback();
|
|
83
79
|
}
|
|
84
80
|
});
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"stream_helpers.js","sourceRoot":"","sources":["../../src/util/stream_helpers.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,SAAS,EAA0B,MAAM,QAAQ,CAAC;AACrE,OAAO,MAAM,MAAM,QAAQ,CAAC;AAC5B,OAAO,EAAE,YAAY,EAAE,MAAM,MAAM,CAAC;AACpC,OAAO,GAAG,MAAM,YAAY,CAAC;AAE7B,MAAM,MAAM,GAAG,MAAM,CAAC,SAAS,EAAE,CAAC;AAElC;;;;GAIG;AACH,MAAM,UAAU,4BAA4B,CACxC,KAAc,EACd,OAAe,CAAC;IAEhB,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,IAAI,MAAM,GAAG,CAAC,CAAC;IAEf,OAAO,IAAI,SAAS,CAAC;QACjB,UAAU,EAAE,IAAI;QAChB,SAAS,CAAC,KAAU,EAAE,SAAyB,EAAE,QAA2B;
|
|
1
|
+
{"version":3,"file":"stream_helpers.js","sourceRoot":"","sources":["../../src/util/stream_helpers.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,SAAS,EAA0B,MAAM,QAAQ,CAAC;AACrE,OAAO,MAAM,MAAM,QAAQ,CAAC;AAC5B,OAAO,EAAE,YAAY,EAAE,MAAM,MAAM,CAAC;AACpC,OAAO,GAAG,MAAM,YAAY,CAAC;AAE7B,MAAM,MAAM,GAAG,MAAM,CAAC,SAAS,EAAE,CAAC;AAElC;;;;GAIG;AACH,MAAM,UAAU,4BAA4B,CACxC,KAAc,EACd,OAAe,CAAC;IAEhB,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,IAAI,MAAM,GAAG,CAAC,CAAC;IAEf,OAAO,IAAI,SAAS,CAAC;QACjB,UAAU,EAAE,IAAI;QAChB,SAAS,CAAC,KAAU,EAAE,SAAyB,EAAE,QAA2B;YACxE,IAAI,OAAO,GAAG,IAAI,EAAE,CAAC;gBACjB,OAAO,EAAE,CAAC;gBACV,MAAM,CAAC,KAAK,CAAC,YAAY,OAAO,EAAE,CAAC,CAAC;gBACpC,OAAO,QAAQ,EAAE,CAAC,CAAC,iBAAiB;YACxC,CAAC;YAED,IAAI,KAAK,KAAK,SAAS,IAAI,MAAM,IAAI,KAAK,EAAE,CAAC;gBACzC,MAAM,CAAC,KAAK,CAAC,sCAAsC,CAAC,CAAC;gBAErD,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBAEhB,+DAA+D;gBAC/D,0CAA0C;gBAC1C,UAAU,CAAC,GAAG,EAAE;oBACZ,MAAM,CAAC,KAAK,CAAC,+CAA+C,CAAC,CAAC;oBAC9D,IAAI,CAAC,OAAO,EAAE,CAAC;gBACnB,CAAC,EAAC,IAAI,CAAC,CAAC;gBAER,OAAO;YACX,CAAC;YAED,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACjB,MAAM,EAAE,CAAC;YAET,MAAM,CAAC,KAAK,CAAC,WAAW,MAAM,EAAE,CAAC,CAAC;YAElC,IAAI,KAAK,KAAK,SAAS,IAAI,MAAM,KAAK,KAAK,EAAE,CAAC;gBAC1C,MAAM,CAAC,KAAK,CAAC,sCAAsC,CAAC,CAAC;gBACrD,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBAEhB,+DAA+D;gBAC/D,0CAA0C;gBAC1C,UAAU,CAAC,GAAG,EAAE;oBACZ,MAAM,CAAC,KAAK,CAAC,+CAA+C,CAAC,CAAC;oBAC9D,IAAI,CAAC,OAAO,EAAE,CAAC;gBACnB,CAAC,EAAC,IAAI,CAAC,CAAC;YACZ,CAAC;YAED,QAAQ,EAAE,CAAC;QACf,CAAC;KACJ,CAAC,CAAC;AACP,CAAC;AAQD,MAAM,UAAU,mBAAmB;IAC/B,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,MAAM,KAAK,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;IAChC,MAAM,SAAS,GAAG,IAAI,SAAS,CAAC;QAC5B,UAAU,EAAE,IAAI;QAChB,SAAS,CAAC,KAAU,EAAE,SAAyB,EAAE,QAA2B;YACxE,SAAS,EAAE,CAAC;YAEZ,MAAM,CAAC,KAAK,CAAC,cAAc,SAAS,EAAE,CAAC,CAAC;YACxC,MAAM,CAAC,KAAK,CAAC,mBAAmB,IAAI,CAAC,qBAAqB,aAAa,IAAI,CAAC,qBAAqB,UAAU,CAAC,CAAC;YAE7G,IAAI,SAAS,GAAG,IAAI,KAAK,CAAC,EAAE,CAAC;gBACzB,MAAM,GAAG,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;gBAC9B,MAAM,QAAQ,GAAG,CAAC,GAAG,GAAG,KAAK,CAAC,GAAC,IAAI,CAAC;gBACpC,MAAM,KAAK,GAAG,SAAS,GAAC,QAAQ,CAAC;gBACjC,MAAM,CAAC,KAAK,CAAC,mBAAmB,IAAI,CAAC,qBAAqB,aAAa,IAAI,CAAC,qBAAqB,UAAU,CAAC,CAAC;gBAC7G,MAAM,CAAC,IAAI,CAAC,WAAW,SAAS,KAAK,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC;YACtE,CAAC;YACD,QAAQ,CAAC,IAAI,EAAC,KAAK,CAAC,CAAC;QACzB,CAAC;QACD,KAAK,CAAE,QAA2B;YAC9B,MAAM,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC;YAC9B,MAAM,GAAG,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;YAC9B,MAAM,QAAQ,GAAG,CAAC,GAAG,GAAG,KAAK,CAAC,GAAC,IAAI,CAAC;YACpC,MAAM,KAAK,GAAG,SAAS,GAAC,QAAQ,CAAC;YACjC,MAAM,CAAC,IAAI,CAAC,WAAW,SAAS,eAAe,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,aAAa,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC;YAC7G,SAAS,CAAC,QAAQ,GAAG,GAAG,EAAE;gBACtB,MAAM,CAAC,KAAK,CAAC,sBAAsB,SAAS,EAAE,CAAC,CAAC;gBAChD,OAAO,SAAS,CAAC;YACrB,CAAC,CAAA;YACD,QAAQ,EAAE,CAAC;QACf,CAAC;KACJ,CAAkB,CAAC;IAEpB,OAAO,SAAS,CAAC;AACrB,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,wBAAwB;IACpC,OAAO,YAAY,EAAE,CAAC;AAC1B,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,oBAAoB;IACtC,MAAM,OAAO,GAAG,GAAG,CAAC,OAAO,EAAE,CAAC;IAE9B,MAAM,eAAe,GAAG,IAAI,SAAS,CAAC;QAClC,UAAU,EAAE,IAAI;QAEhB,SAAS,CAAC,KAAU,EAAE,QAAgB,EAAE,QAA2B;YAC/D,4DAA4D;YAC5D,MAAM,MAAM,GAAG,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,EAAE,QAA0B,CAAC,CAAC;YAE/F,wEAAwE;YACxE,MAAM,eAAe,GAAG,OAAO,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;YAE9C,IAAI,CAAC,eAAe,EAAE,CAAC;gBACnB,wBAAwB;gBACxB,OAAO,CAAC,IAAI,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;YACpC,CAAC;iBAAM,CAAC;gBACJ,QAAQ,EAAE,CAAC;YACf,CAAC;QACL,CAAC;QACD,KAAK,CAAC,QAA2B;YAC7B,OAAO,CAAC,GAAG,EAAE,CAAC;YACd,OAAO,CAAC,IAAI,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;QACrC,CAAC;KACJ,CAAC,CAAC;IAEH,OAAO,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,EAAE;QACzC,MAAM,CAAC,IAAI,CAAC,cAAc,MAAM,CAAC,IAAI,kBAAkB,CAAC,CAAC;QAEzD,MAAM,MAAM,GAAa,EAAE,CAAC;QAE5B,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,KAAK,EAAE,EAAE;YACxB,MAAM,CAAC,KAAK,CAAC,0BAA0B,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC;YACvD,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACvB,CAAC,CAAC,CAAC;QAEH,MAAM,CAAC,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE;YAClB,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;YACrC,MAAM,CAAC,KAAK,CAAC,gBAAgB,MAAM,CAAC,IAAI,iBAAiB,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC;YAC1E,eAAe,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC;YAC/C,IAAI,EAAE,CAAC;QACX,CAAC,CAAC,CAAC;QAEH,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE;YACvB,MAAM,CAAC,KAAK,CAAC,uBAAuB,MAAM,CAAC,IAAI,GAAG,EAAE,GAAG,CAAC,CAAC;YACzD,IAAI,CAAC,GAAG,CAAC,CAAC;QACd,CAAC,CAAC,CAAC;QAEH,MAAM,CAAC,MAAM,EAAE,CAAC;IACpB,CAAC,CAAC,CAAC;IAEH,OAAO,CAAC,EAAE,CAAC,QAAQ,EAAE,GAAG,EAAE;QACtB,MAAM,CAAC,KAAK,CAAC,2BAA2B,CAAC,CAAC;QAC1C,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,uBAAuB;IACvD,CAAC,CAAC,CAAC;IAEH,OAAO,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE;QACxB,MAAM,CAAC,KAAK,CAAC,oBAAoB,EAAE,GAAG,CAAC,CAAC;QACxC,eAAe,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IACjC,CAAC,CAAC,CAAC;IAEH,OAAO,eAAe,CAAC;AAC3B,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "marcattacks",
|
|
3
|
-
"version": "2.2.
|
|
3
|
+
"version": "2.2.6",
|
|
4
4
|
"description": "A powerful streaming MARC21 to RDF converter with JSONata transformation and S3 support.",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"marc21",
|
|
@@ -20,7 +20,8 @@
|
|
|
20
20
|
"author": "",
|
|
21
21
|
"repository": "https://codeberg.org/phochste/marcattacks.git",
|
|
22
22
|
"bin": {
|
|
23
|
-
"marcattacks": "./dist/
|
|
23
|
+
"marcattacks": "./dist/command.js",
|
|
24
|
+
"globtrotr": "./dist/globber.js"
|
|
24
25
|
},
|
|
25
26
|
"files": [
|
|
26
27
|
"dist"
|
package/dist/filestream.d.ts.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"filestream.d.ts","sourceRoot":"","sources":["../src/filestream.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,QAAQ,CAAC;AAKvC,wBAAsB,cAAc,CAAC,GAAG,EAAE,GAAG,GAAI,OAAO,CAAC,QAAQ,CAAC,CAEjE;AAED,wBAAsB,cAAc,CAAC,GAAG,EAAE,GAAG,GAAI,OAAO,CAAC,GAAG,CAAC,CA6B5D"}
|