marcattacks 1.9.0 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/dist/index.js +36 -11
- package/dist/index.js.map +1 -1
- package/dist/input/alephseq.d.ts.map +1 -1
- package/dist/input/alephseq.js +3 -8
- package/dist/input/alephseq.js.map +1 -1
- package/dist/input/json.d.ts.map +1 -1
- package/dist/input/json.js +3 -6
- package/dist/input/json.js.map +1 -1
- package/dist/input/jsonl.d.ts.map +1 -1
- package/dist/input/jsonl.js +3 -4
- package/dist/input/jsonl.js.map +1 -1
- package/dist/input/marc.d.ts.map +1 -1
- package/dist/input/marc.js +3 -0
- package/dist/input/marc.js.map +1 -1
- package/dist/input/rdf.d.ts.map +1 -1
- package/dist/input/rdf.js +6 -1
- package/dist/input/rdf.js.map +1 -1
- package/dist/input/xml.d.ts.map +1 -1
- package/dist/input/xml.js +3 -0
- package/dist/input/xml.js.map +1 -1
- package/dist/output/alephseq.d.ts +2 -2
- package/dist/output/alephseq.d.ts.map +1 -1
- package/dist/output/alephseq.js +30 -34
- package/dist/output/alephseq.js.map +1 -1
- package/dist/output/json.d.ts +2 -2
- package/dist/output/json.d.ts.map +1 -1
- package/dist/output/json.js +28 -22
- package/dist/output/json.js.map +1 -1
- package/dist/output/jsonl.d.ts +2 -2
- package/dist/output/jsonl.d.ts.map +1 -1
- package/dist/output/jsonl.js +8 -14
- package/dist/output/jsonl.js.map +1 -1
- package/dist/output/rdf.d.ts +2 -2
- package/dist/output/rdf.d.ts.map +1 -1
- package/dist/output/rdf.js +24 -22
- package/dist/output/rdf.js.map +1 -1
- package/dist/output/xml.d.ts +2 -2
- package/dist/output/xml.d.ts.map +1 -1
- package/dist/output/xml.js +52 -40
- package/dist/output/xml.js.map +1 -1
- package/dist/sftpstream.d.ts.map +1 -1
- package/dist/sftpstream.js +5 -0
- package/dist/sftpstream.js.map +1 -1
- package/dist/transform/notation3.d.ts.map +1 -1
- package/dist/transform/notation3.js +7 -1
- package/dist/transform/notation3.js.map +1 -1
- package/dist/util/marc_n3_helpers.d.ts +2 -0
- package/dist/util/marc_n3_helpers.d.ts.map +1 -0
- package/dist/util/marc_n3_helpers.js +141 -0
- package/dist/util/marc_n3_helpers.js.map +1 -0
- package/dist/util/rdf_parse.d.ts.map +1 -1
- package/dist/util/rdf_parse.js +13 -0
- package/dist/util/rdf_parse.js.map +1 -1
- package/dist/util/stream_helpers.d.ts +12 -0
- package/dist/util/stream_helpers.d.ts.map +1 -0
- package/dist/util/stream_helpers.js +73 -0
- package/dist/util/stream_helpers.js.map +1 -0
- package/package.json +18 -4
package/README.md
CHANGED
|
@@ -88,15 +88,15 @@ use `s3s://...` for using an SSL layer.
|
|
|
88
88
|
- jsonl
|
|
89
89
|
- marc (ISO2709)
|
|
90
90
|
- rdf
|
|
91
|
-
- xml
|
|
91
|
+
- xml (MARCXML)
|
|
92
92
|
|
|
93
93
|
### Output (--to)
|
|
94
94
|
|
|
95
|
-
- xml
|
|
96
95
|
- alephseq (Aleph sequential)
|
|
97
96
|
- json
|
|
98
97
|
- jsonl
|
|
99
98
|
- rdf
|
|
99
|
+
- xml (MARCXML)
|
|
100
100
|
|
|
101
101
|
### Transform (--map)
|
|
102
102
|
|
package/dist/index.js
CHANGED
|
@@ -11,8 +11,9 @@ import path from "node:path";
|
|
|
11
11
|
import fs from 'fs';
|
|
12
12
|
import { s3ReaderStream, s3WriterStream } from './s3stream.js';
|
|
13
13
|
import dotenv from 'dotenv';
|
|
14
|
-
import {
|
|
14
|
+
import { pipeline } from 'node:stream/promises';
|
|
15
15
|
import { createGunzip } from 'zlib';
|
|
16
|
+
import { createCountableSkippedStream, createVerboseStream } from './util/stream_helpers.js';
|
|
16
17
|
program.version('0.1.0')
|
|
17
18
|
.argument('<file>')
|
|
18
19
|
.option('-c,--config <config>', 'config .env', path.join(process.cwd(), '.env'))
|
|
@@ -22,7 +23,10 @@ program.version('0.1.0')
|
|
|
22
23
|
.option('--fix <what>', 'jsonata')
|
|
23
24
|
.option('-o,--out <file>', 'output file')
|
|
24
25
|
.option('-z', 'compressed')
|
|
26
|
+
.option('--count <num>', 'output only <num> records')
|
|
27
|
+
.option('--skip <num>', 'skip first <num> records')
|
|
25
28
|
.option('--key <keyfile>', 'private key file')
|
|
29
|
+
.option('--key-env <env>', 'private key environment variable')
|
|
26
30
|
.option('--log <format>', 'logging format')
|
|
27
31
|
.option('--info', 'output debugging messages')
|
|
28
32
|
.option('--debug', 'output more debugging messages')
|
|
@@ -61,6 +65,7 @@ if (opts.trace) {
|
|
|
61
65
|
if (opts.config) {
|
|
62
66
|
dotenv.config({ path: opts.config, quiet: true });
|
|
63
67
|
}
|
|
68
|
+
logger.info('here');
|
|
64
69
|
main();
|
|
65
70
|
function configureDefaultLogger(output) {
|
|
66
71
|
log4js.configure({
|
|
@@ -161,19 +166,29 @@ async function main() {
|
|
|
161
166
|
path: inputFile
|
|
162
167
|
});
|
|
163
168
|
objectStream.on('error', (error) => {
|
|
164
|
-
|
|
165
|
-
|
|
169
|
+
if (error.code === 'ERR_STREAM_PREMATURE_CLOSE') {
|
|
170
|
+
inputStream.destroy();
|
|
171
|
+
}
|
|
172
|
+
else {
|
|
173
|
+
logger.error("input stream processing error: ", error.message);
|
|
174
|
+
inputStream.destroy();
|
|
175
|
+
process.exitCode = 2;
|
|
176
|
+
}
|
|
166
177
|
});
|
|
167
178
|
}
|
|
168
179
|
else {
|
|
169
180
|
console.error(`Need --from`);
|
|
170
181
|
process.exit(1);
|
|
171
182
|
}
|
|
172
|
-
|
|
183
|
+
const stages = [objectStream];
|
|
184
|
+
if (opts.count || opts.skip) {
|
|
185
|
+
stages.push(createCountableSkippedStream(opts.count, opts.skip));
|
|
186
|
+
}
|
|
187
|
+
stages.push(createVerboseStream());
|
|
173
188
|
if (opts.map) {
|
|
174
189
|
const mod = await loadPlugin(opts.map, 'transform');
|
|
175
190
|
const transformer = await mod.transform(opts.fix);
|
|
176
|
-
|
|
191
|
+
stages.push(transformer);
|
|
177
192
|
}
|
|
178
193
|
let outStream;
|
|
179
194
|
if (opts.out === '@slow') {
|
|
@@ -214,14 +229,21 @@ async function main() {
|
|
|
214
229
|
outStream = process.stdout;
|
|
215
230
|
}
|
|
216
231
|
if (opts.to) {
|
|
232
|
+
const mod = await loadPlugin(opts.to, 'output');
|
|
233
|
+
stages.push(await mod.transform());
|
|
234
|
+
stages.push(outStream);
|
|
217
235
|
try {
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
await finished(outStream);
|
|
236
|
+
await pipeline(stages);
|
|
237
|
+
logger.info("pipeline finished cleanly");
|
|
221
238
|
}
|
|
222
|
-
catch (
|
|
223
|
-
|
|
224
|
-
|
|
239
|
+
catch (err) {
|
|
240
|
+
if (err.code === 'ERR_STREAM_PREMATURE_CLOSE') {
|
|
241
|
+
logger.info("Stream closed by limiter.");
|
|
242
|
+
}
|
|
243
|
+
else {
|
|
244
|
+
logger.error("pipeline error:", err);
|
|
245
|
+
process.exitCode = 3;
|
|
246
|
+
}
|
|
225
247
|
}
|
|
226
248
|
}
|
|
227
249
|
}
|
|
@@ -241,6 +263,9 @@ function makeSftpConfig(inputFile, opts) {
|
|
|
241
263
|
if (opts.key) {
|
|
242
264
|
privateKey = fs.readFileSync(opts.key, { encoding: 'utf-8' });
|
|
243
265
|
}
|
|
266
|
+
else if (opts.keyEnv) {
|
|
267
|
+
privateKey = process.env[opts.keyEnv];
|
|
268
|
+
}
|
|
244
269
|
let config = {
|
|
245
270
|
host: inputFile.hostname,
|
|
246
271
|
port: Number(inputFile.port) ?? 22,
|
package/dist/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";AAEA,OAAO,MAAM,MAAM,QAAQ,CAAC;AAC5B,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AAChD,OAAO,EAAE,cAAc,EAAG,eAAe,EAAG,cAAc,EAAoB,MAAM,iBAAiB,CAAC;AACtG,OAAO,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AACjD,OAAO,EAAE,QAAQ,EAAE,MAAM,QAAQ,CAAC;AAClC,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AAEzC,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,MAAM,IAAI,CAAC;AACpB,OAAO,EAAE,cAAc,EAAE,cAAc,EAAE,MAAM,eAAe,CAAC;AAC/D,OAAO,MAAM,MAAM,QAAQ,CAAC;AAC5B,OAAO,EAAE,QAAQ,EAAE,MAAM,sBAAsB,CAAC;AAChD,OAAO,EAAE,YAAY,EAAE,MAAM,MAAM,CAAC;
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";AAEA,OAAO,MAAM,MAAM,QAAQ,CAAC;AAC5B,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AAChD,OAAO,EAAE,cAAc,EAAG,eAAe,EAAG,cAAc,EAAoB,MAAM,iBAAiB,CAAC;AACtG,OAAO,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AACjD,OAAO,EAAE,QAAQ,EAAE,MAAM,QAAQ,CAAC;AAClC,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AAEzC,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,MAAM,IAAI,CAAC;AACpB,OAAO,EAAE,cAAc,EAAE,cAAc,EAAE,MAAM,eAAe,CAAC;AAC/D,OAAO,MAAM,MAAM,QAAQ,CAAC;AAC5B,OAAO,EAAE,QAAQ,EAAE,MAAM,sBAAsB,CAAC;AAChD,OAAO,EAAE,YAAY,EAAE,MAAM,MAAM,CAAC;AACpC,OAAO,EAAE,4BAA4B,EAAE,mBAAmB,EAAE,MAAM,0BAA0B,CAAC;AAE7F,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC;KACnB,QAAQ,CAAC,QAAQ,CAAC;KAClB,MAAM,CAAC,sBAAsB,EAAC,aAAa,EAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,MAAM,CAAC,CAAC;KAC7E,MAAM,CAAC,kBAAkB,EAAC,YAAY,EAAC,KAAK,CAAC;KAC7C,MAAM,CAAC,kBAAkB,EAAC,aAAa,EAAC,MAAM,CAAC;KAC/C,MAAM,CAAC,gBAAgB,EAAC,aAAa,EAAC,SAAS,CAAC;KAChD,MAAM,CAAC,cAAc,EAAC,SAAS,CAAC;KAChC,MAAM,CAAC,iBAAiB,EAAC,aAAa,CAAC;KACvC,MAAM,CAAC,IAAI,EAAC,YAAY,CAAC;KACzB,MAAM,CAAC,eAAe,EAAE,2BAA2B,CAAC;KACpD,MAAM,CAAC,cAAc,EAAE,0BAA0B,CAAC;KAClD,MAAM,CAAC,iBAAiB,EAAE,kBAAkB,CAAC;KAC7C,MAAM,CAAC,iBAAiB,EAAC,kCAAkC,CAAC;KAC5D,MAAM,CAAC,gBAAgB,EAAC,gBAAgB,CAAC;KACzC,MAAM,CAAC,QAAQ,EAAC,2BAA2B,CAAC;KAC5C,MAAM,CAAC,SAAS,EAAC,gCAAgC,CAAC;KAClD,MAAM,CAAC,SAAS,EAAC,qCAAqC,CAAC,CAAC;AAE7D,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;AAE5B,MAAM,IAAI,GAAK,OAAO,CAAC,IAAI,EAAE,CAAC;AAE9B,IAAI,IAAI,CAAC,GAAG,EAAE,CAAC;IACX,IAAI,MAAM,GAAG,QAAQ,CAAC;IACtB,IAAI,IAAI,GAAG,CAAC,CAAC;IAEb,IAAI,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;QAClC,MAAM,GAAG,QAAQ,CAAC;IACtB,CAAC;IACD,IAAI,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC;QAChC,IAAI,GAAG,CAAC,CAAC;IACb,CAAC;IAED,IAAI,IAAI,EAAE,CAAC;QACP,mBAAmB,CAAC,MAAM,CAAC,CAAC;IAChC,CAAC;SACI,CAAC;QACF,sBAAsB,CAAC,MAAM,CAAC,CAAC;IACnC,CAAC;AACL,CAAC;KACI,CAAC;IACF,sBAAsB,CAAC,QAAQ,CAAC,CAAC;AACrC,CAAC;AAED,MAAM,MAAM,GAAG,MAAM,CAAC,SAAS,EAAE,CAAC;AAElC,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;IACZ,MAAM,CAAC,KAAK,GAAG,MAAM,CAAC;AAC1B,CAAC;AAED,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;IACb,MAAM,CAAC,KAAK,GAAG,OAAO,CAAC;AAC3B,CAAC;AAED,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;IACb,MAAM,CAAC,KAAK,GAAG,OAAO,CAAC;AAC3B,CAAC;AAED,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;IACd,MAAM,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,MAAM,EAAG,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;AACvD,CAAC;AAED,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;AAEpB,IAAI,EAAE,CAAC;AAEP,SAAS,sBAAsB,CAAC,MAAc;IAC1C,MAAM,CAAC,SAAS,CAAC;QACb,SAAS,EAAE;YACP,GAAG,EAAE;gBACD,IAAI,EAAE,MAAM;gBACZ,MAAM,EAAE;oBACJ,IAAI,EAAE,SAAS;oBACf,OAAO,EAAE,oBAAoB;iBAChC;aACJ;SACJ;QACD,UAAU,EAAE;YACR,OAAO,EAAE,EAAE,SAAS,EAAE,CAAC,KAAK,CAAC,EAAE,KAAK,EAAE,KAAK,EAAG,eAAe,EAAE,IAAI,EAAE;SACxE;KACJ,CAAC,CAAC;AACP,CAAC;AAED,SAAS,mBAAmB,CAAC,MAAc;IACvC,MAAM,CAAC,SAAS,CAAC,cAAc,EAAE,CAAC,MAAM,EAAE,EAAE;QACxC,OAAO,CAAC,QAAQ,EAAE,EAAE;YAChB,OAAO,IAAI,CAAC,SAAS,CAAC;gBACtB,SAAS,EAAE,QAAQ,CAAC,SAAS,EAAG,gBAAgB;gBAChD,KAAK,EAAE,QAAQ,CAAC,KAAK,CAAC,QAAQ,EAAE,gBAAgB;gBAChD,QAAQ,EAAE,QAAQ,CAAC,YAAY,EAAC,gBAAgB;gBAChD,OAAO,EAAE,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,gBAAgB;gBAClD,OAAO,EAAE,QAAQ,CAAC,OAAO,EAAO,yBAAyB;gBACzD,GAAG,EAAE,QAAQ,CAAC,GAAG,CAAe,gBAAgB;aAC/C,CAAC,CAAC;QACP,CAAC,CAAC;IACN,CAAC,CAAC,CAAC;IAEH,MAAM,CAAC,SAAS,CAAC;QACb,SAAS,EAAE;YACP,GAAG,EAAE;gBACD,IAAI,EAAE,MAAM;gBACZ,MAAM,EAAE;oBACJ,IAAI,EAAE,cAAc;iBACvB;aACJ;SACJ;QACD,UAAU,EAAE;YACR,OAAO,EAAE,EAAE,SAAS,EAAE,CAAC,KAAK,CAAC,EAAE,KAAK,EAAE,KAAK,EAAG,eAAe,EAAE,IAAI,EAAE;SACxE;KACJ,CAAC,CAAC;AACP,CAAC;AAED,KAAK,UAAU,IAAI;IACf,IAAI,CAAC;QACD,MAAM,GAAG,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAE5B,IAAI,CAAE,GAAG,EAAE,CAAC;YACR,OAAO,CAAC,KAAK,CAAC,oBAAoB,CAAC,CAAC;YACpC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACpB,CAAC;QAED,IAAI,SAAe,CAAC;QAEpB,IAAI,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;YACrB,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,GAAG,CAAC,CAAC;YAClD,SAAS,GAAG,aAAa,CAAC,QAAQ,CAAC,CAAC;QACxC,CAAC;aACI,CAAC;YACF,SAAS,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;QAC7B,CAAC;QAED,MAAM,CAAC,IAAI,CAAC,UAAU,WAAW,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;QAEhD,IAAI,cAAyB,CAAC;QAE9B,IAAI,SAAS,CAAC,QAAQ,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC;YACxC,cAAc,GAAG,MAAM,cAAc,CAAC,SAAS,CAAC,QAAQ,EAAE,CAAC,CAAC;QAChE,CAAC;aACI,IAAI,SAAS,CAAC,QAAQ,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;YAC3C,cAAc,GAAG,MAAM,cAAc,CAAC,SAAS,EAAC,EAAE,CAAC,CAAC;QACxD,CAAC;aACI,IAAI,SAAS,CAAC,QAAQ,KAAK,OAAO,EAAE,CAAC;YACtC,MAAM,MAAM,GAAG,cAAc,CAAC,SAAS,EAAC,IAAI,CAAC,CAAC;YAC9C,IAAI,UAAU,CAAC;YAEf,IAAI,SAAS,CAAC,QAAQ,CAAC,KAAK,CAAC,gBAAgB,CAAC,EAAE,CAAC;gBAC7C,MAAM,SAAS,GAAG,SAAS,CAAC,QAAQ,CAAC,OAAO,CAAC,aAAa,EAAC,EAAE,CAAC,CAAC;gBAC/D,MAAM,SAAS,GAAG,SAAS,CAAC,QAAQ,CAAC,OAAO,CAAC,cAAc,EAAC,EAAE,CAAC,CAAC;gBAChE,UAAU,GAAG,MAAM,cAAc,CAAC,MAAM,EAAC,SAAS,EAAC,SAAS,CAAC,CAAC;YAClE,CAAC;iBACI,CAAC;gBACF,UAAU,GAAG,SAAS,CAAC,QAAQ,CAAC;YACpC,CAAC;YAED,cAAc,GAAG,MAAM,cAAc,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC;QAC9D,CAAC;aACI,IAAI,SAAS,CAAC,QAAQ,KAAK,QAAQ,EAAE,CAAC;YACvC,cAAc,GAAG,OAAO,CAAC,KAAK,CAAC;QACnC,CAAC;aACI,CAAC;YACF,cAAc,GAAG,EAAE,CAAC,gBAAgB,CAAC,SAAS,CAAC,CAAC;QACpD,CAAC;QAED,IAAI,WAAsB,CAAC;QAE3B,IAAI,IAAI,CAAC,CAAC,EAAE,CAAC;YACT,WAAW,GAAG,qBAAqB,CAAC,cAAc,CAAC,CAAC;QACxD,CAAC;aACI,CAAC;YACF,WAAW,GAAG,cAAc,CAAC;QACjC,CAAC;QAED,IAAI,YAAuB,CAAC;QAE5B,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;YACZ,MAAM,GAAG,GAAG,MAAM,UAAU,CAAC,IAAI,CAAC,IAAI,EAAC,OAAO,CAAC,CAAC;YAChD,YAAY,GAAG,MAAM,GAAG,CAAC,eAAe,CAAC,WAAW,EAAE;gBAClD,IAAI,EAAE,SAAS;aAClB,CAAC,CAAC;YACH,YAAY,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,KAAU,EAAE,EAAE;gBACpC,IAAI,KAAK,CAAC,IAAI,KAAK,4BAA4B,EAAE,CAAC;oBAC9C,WAAW,CAAC,OAAO,EAAE,CAAC;gBAC1B,CAAC;qBACI,CAAC;oBACF,MAAM,CAAC,KAAK,CAAC,iCAAiC,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC;oBAC/D,WAAW,CAAC,OAAO,EAAE,CAAC;oBACtB,OAAO,CAAC,QAAQ,GAAG,CAAC,CAAC;gBACzB,CAAC;YACL,CAAC,CAAC,CAAC;QACP,CAAC;aACI,CAAC;YACF,OAAO,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC;YAC7B,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACpB,CAAC;QAED,MAAM,MAAM,GAAwC,CAAC,YAAY,CAAC,CAAC;QAEnE,IAAI,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;YAC1B,MAAM,CAAC,IAAI,CACP,4BAA4B,CACxB,IAAI,CAAC,KAAK,EACV,IAAI,CAAC,IAAI,CACZ,CACJ,CAAC;QACN,CAAC;QAED,MAAM,CAAC,IAAI,CAAC,mBAAmB,EAAE,CAAC,CAAC;QAEnC,IAAI,IAAI,CAAC,GAAG,EAAE,CAAC;YACX,MAAM,GAAG,GAAG,MAAM,UAAU,CAAC,IAAI,CAAC,GAAG,EAAC,WAAW,CAAC,CAAC;YACnD,MAAM,WAAW,GAAe,MAAM,GAAG,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAC9D,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QAC7B,CAAC;QAED,IAAI,SAAoB,CAAC;QAEzB,IAAI,IAAI,CAAC,GAAG,KAAK,OAAO,EAAE,CAAC;YACvB,SAAS,GAAG,IAAI,YAAY,CAAC,EAAE,OAAO,EAAE,GAAG,EAAE,CAAC,CAAC;QACnD,CAAC;aACI,IAAI,IAAI,CAAC,GAAG,KAAK,SAAS,EAAE,CAAC;YAC9B,SAAS,GAAG,IAAI,YAAY,CAAC,EAAE,mBAAmB,EAAE,CAAC,EAAE,CAAC,CAAC;QAC7D,CAAC;aACI,IAAI,IAAI,CAAC,GAAG,EAAE,CAAC;YAChB,IAAI,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC;gBAC9B,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBAE9B,IAAI,OAAO,CAAC,GAAG,CAAC,aAAa,EAAE,CAAC;oBAC5B,GAAG,CAAC,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,aAAa,CAAC;gBAC7C,CAAC;gBAED,IAAI,OAAO,CAAC,GAAG,CAAC,aAAa,EAAE,CAAC;oBAC5B,GAAG,CAAC,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,aAAa,CAAC;gBAC7C,CAAC;gBAED,MAAM,MAAM,GAAG,cAAc,CAAC,GAAG,EAAC,IAAI,CAAC,CAAC;gBACxC,MAAM,CAAC,IAAI,CAAC,OAAO,WAAW,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;gBACvC,SAAS,GAAG,MAAM,eAAe,CAAC,GAAG,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;YACxD,CAAC;iBACI,IAAI,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;gBACjC,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBAE9B,IAAI,OAAO,CAAC,GAAG,CAAC,aAAa,EAAE,CAAC;oBAC5B,GAAG,CAAC,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,aAAa,CAAC;gBAC7C,CAAC;gBAED,IAAI,OAAO,CAAC,GAAG,CAAC,aAAa,EAAE,CAAC;oBAC5B,GAAG,CAAC,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,aAAa,CAAC;gBAC7C,CAAC;gBAED,MAAM,CAAC,IAAI,CAAC,OAAO,WAAW,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;gBACvC,SAAS,GAAG,MAAM,cAAc,CAAC,GAAG,EAAC,EAAE,CAAC,CAAC;YAC7C,CAAC;iBACI,CAAC;gBACF,SAAS,GAAG,EAAE,CAAC,iBAAiB,CAAC,IAAI,CAAC,GAAG,EAAE,EAAE,QAAQ,EAAE,OAAO,EAAC,CAAC,CAAC;YACrE,CAAC;QACL,CAAC;aACI,CAAC;YACF,SAAS,GAAG,OAAO,CAAC,MAAM,CAAC;QAC/B,CAAC;QAED,IAAI,IAAI,CAAC,EAAE,EAAE,CAAC;YACV,MAAM,GAAG,GAAG,MAAM,UAAU,CAAC,IAAI,CAAC,EAAE,EAAC,QAAQ,CAAC,CAAC;YAC/C,MAAM,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,SAAS,EAAE,CAAC,CAAC;YACnC,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YACvB,IAAI,CAAC;gBACD,MAAM,QAAQ,CAAC,MAAM,CAAC,CAAC;gBACvB,MAAM,CAAC,IAAI,CAAC,2BAA2B,CAAC,CAAC;YAC7C,CAAC;YACD,OAAO,GAAS,EAAE,CAAC;gBACf,IAAI,GAAG,CAAC,IAAI,KAAK,4BAA4B,EAAE,CAAC;oBAC5C,MAAM,CAAC,IAAI,CAAC,2BAA2B,CAAC,CAAC;gBAC7C,CAAC;qBAAM,CAAC;oBACJ,MAAM,CAAC,KAAK,CAAC,iBAAiB,EAAE,GAAG,CAAC,CAAC;oBACrC,OAAO,CAAC,QAAQ,GAAG,CAAC,CAAC;gBACzB,CAAC;YACL,CAAC;QACL,CAAC;IACL,CAAC;IACD,OAAO,CAAC,EAAE,CAAC;QACP,MAAM,CAAC,KAAK,CAAC,yBAAyB,CAAC,EAAE,CAAC,CAAC;QAC3C,OAAO,CAAC,QAAQ,GAAG,CAAC,CAAC;IACzB,CAAC;AACL,CAAC;AAED,SAAS,WAAW,CAAC,GAAQ;IACzB,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,mCAAmC;IACtE,OAAO,CAAC,QAAQ,GAAG,KAAK,CAAC;IACzB,OAAO,CAAC,QAAQ,GAAG,KAAK,CAAC;IACzB,OAAO,OAAO,CAAC;AACnB,CAAC;AAED,SAAS,cAAc,CAAC,SAAc,EAAE,IAAS;IAC7C,IAAI,UAAU,GAAwB,SAAS,CAAC;IAEhD,IAAI,IAAI,CAAC,GAAG,EAAE,CAAC;QACX,UAAU,GAAG,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,GAAG,EAAC,EAAE,QAAQ,EAAE,OAAO,EAAC,CAAC,CAAC;IAChE,CAAC;SACI,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;QACnB,UAAU,GAAG,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAC1C,CAAC;IAED,IAAI,MAAM,GAAe;QACrB,IAAI,EAAE,SAAS,CAAC,QAAQ;QACxB,IAAI,EAAE,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,EAAE;QAClC,QAAQ,EAAE,SAAS,CAAC,QAAQ;KAC/B,CAAC;IAEF,IAAI,SAAS,CAAC,QAAQ,EAAE,CAAC;QAAC,MAAM,CAAC,QAAQ,GAAG,SAAS,CAAC,QAAQ,CAAA;IAAC,CAAC;IAChE,IAAI,UAAU,EAAE,CAAC;QAAC,MAAM,CAAC,UAAU,GAAG,UAAU,CAAA;IAAA,CAAC;IAEjD,OAAO,MAAM,CAAC;AAClB,CAAC;AAED,SAAS,qBAAqB,CAAC,YAAsB;IACnD,MAAM,MAAM,GAAG,YAAY,EAAE,CAAC;IAE9B,sDAAsD;IACtD,YAAY,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE;QAC/B,MAAM,CAAC,KAAK,CAAC,gBAAgB,EAAE,GAAG,CAAC,OAAO,CAAC,CAAC;QAC5C,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;IAC5B,CAAC,CAAC,CAAC;IAEH,OAAO,YAAY,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;AACnC,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"alephseq.d.ts","sourceRoot":"","sources":["../../src/input/alephseq.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,QAAQ,CAAC;AAMlC,wBAAsB,eAAe,CAAC,MAAM,EAAE,QAAQ,EAAE,KAAK,EAAE,GAAG,GAAI,OAAO,CAAC,QAAQ,CAAC,
|
|
1
|
+
{"version":3,"file":"alephseq.d.ts","sourceRoot":"","sources":["../../src/input/alephseq.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,QAAQ,CAAC;AAMlC,wBAAsB,eAAe,CAAC,MAAM,EAAE,QAAQ,EAAE,KAAK,EAAE,GAAG,GAAI,OAAO,CAAC,QAAQ,CAAC,CA8FtF"}
|
package/dist/input/alephseq.js
CHANGED
|
@@ -15,6 +15,9 @@ export async function stream2readable(stream, _opts) {
|
|
|
15
15
|
sourcePaused = false;
|
|
16
16
|
}
|
|
17
17
|
},
|
|
18
|
+
destroy() {
|
|
19
|
+
stream.destroy();
|
|
20
|
+
},
|
|
18
21
|
objectMode: true
|
|
19
22
|
});
|
|
20
23
|
let rec = [];
|
|
@@ -47,9 +50,6 @@ export async function stream2readable(stream, _opts) {
|
|
|
47
50
|
}
|
|
48
51
|
rec = [];
|
|
49
52
|
recordNum++;
|
|
50
|
-
if (recordNum % 1000 === 0) {
|
|
51
|
-
logger.info(`record: ${recordNum}`);
|
|
52
|
-
}
|
|
53
53
|
}
|
|
54
54
|
const tag = data?.substring(0, 3);
|
|
55
55
|
const ind1 = data?.substring(3, 4);
|
|
@@ -79,12 +79,7 @@ export async function stream2readable(stream, _opts) {
|
|
|
79
79
|
readableStream.push({
|
|
80
80
|
record: rec
|
|
81
81
|
});
|
|
82
|
-
recordNum++;
|
|
83
|
-
if (recordNum % 1000 === 0) {
|
|
84
|
-
logger.info(`record: ${recordNum}`);
|
|
85
|
-
}
|
|
86
82
|
readableStream.push(null);
|
|
87
|
-
logger.info(`processed ${recordNum} records`);
|
|
88
83
|
});
|
|
89
84
|
return readableStream;
|
|
90
85
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"alephseq.js","sourceRoot":"","sources":["../../src/input/alephseq.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,QAAQ,CAAC;AAClC,OAAO,KAAK,QAAQ,MAAM,eAAe,CAAA;AACzC,OAAO,MAAM,MAAM,QAAQ,CAAC;AAE5B,MAAM,MAAM,GAAG,MAAM,CAAC,SAAS,EAAE,CAAC;AAElC,MAAM,CAAC,KAAK,UAAU,eAAe,CAAC,MAAgB,EAAE,KAAU;IAC9D,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,IAAI,QAAQ,GAAG,KAAK,CAAC;IAErB,MAAM,EAAE,GAAG,QAAQ,CAAC,eAAe,CAAC,EAAC,KAAK,EAAE,MAAM,EAAE,SAAS,EAAE,QAAQ,EAAC,CAAC,CAAC;IAE1E,IAAI,YAAY,GAAG,KAAK,CAAC;IAEzB,MAAM,cAAc,GAAG,IAAI,QAAQ,CAAC;QAChC,IAAI;YACA,IAAI,YAAY,EAAE,CAAC;gBACf,MAAM,CAAC,KAAK,CAAC,kBAAkB,CAAC,CAAC;gBACjC,EAAE,CAAC,MAAM,EAAE,CAAC;gBACZ,YAAY,GAAG,KAAK,CAAC;YACzB,CAAC;QACL,CAAC;QACD,UAAU,EAAE,IAAI;KACnB,CAAC,CAAC;IAEH,IAAI,GAAG,GAAgB,EAAE,CAAC;IAC1B,IAAI,MAAM,GAAY,EAAE,CAAC;IAEzB,EAAE,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,EAAE;QACnB,IAAI,QAAQ;YAAE,OAAO;QAErB,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAEnB,IAAI,IAAI,CAAC,KAAK,CAAC,8BAA8B,CAAC,EAAE,CAAC;YAC7C,KAAK;QACT,CAAC;aACI,CAAC;YACF,MAAM,CAAC,KAAK,CAAC,0BAA0B,SAAS,GAAG,CAAC,EAAE,CAAC,CAAC;YACxD,QAAQ,GAAG,IAAI,CAAC;YAChB,MAAM,CAAC,OAAO,EAAE,CAAC;YACjB,EAAE,CAAC,KAAK,EAAE,CAAC;YACX,cAAc,CAAC,OAAO,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC;YACzD,OAAO;QACX,CAAC;QAED,MAAM,CAAC,EAAE,EAAC,GAAG,IAAI,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QACrC,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAE5B,IAAI,MAAM,IAAI,MAAM,KAAK,EAAE,EAAE,CAAC;YAC1B,MAAM,EAAE,GAAG,cAAc,CAAC,IAAI,CAAC;gBAC3B,MAAM,EAAG,GAAG;aACf,CAAC,CAAC;YAEH,IAAI,CAAC,EAAE,EAAE,CAAC;gBACN,MAAM,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAC;gBAChC,EAAE,CAAC,KAAK,EAAE,CAAC;gBACX,YAAY,GAAG,IAAI,CAAC;YACxB,CAAC;YAED,GAAG,GAAG,EAAE,CAAC;YACT,SAAS,EAAE,CAAC;
|
|
1
|
+
{"version":3,"file":"alephseq.js","sourceRoot":"","sources":["../../src/input/alephseq.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,QAAQ,CAAC;AAClC,OAAO,KAAK,QAAQ,MAAM,eAAe,CAAA;AACzC,OAAO,MAAM,MAAM,QAAQ,CAAC;AAE5B,MAAM,MAAM,GAAG,MAAM,CAAC,SAAS,EAAE,CAAC;AAElC,MAAM,CAAC,KAAK,UAAU,eAAe,CAAC,MAAgB,EAAE,KAAU;IAC9D,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,IAAI,QAAQ,GAAG,KAAK,CAAC;IAErB,MAAM,EAAE,GAAG,QAAQ,CAAC,eAAe,CAAC,EAAC,KAAK,EAAE,MAAM,EAAE,SAAS,EAAE,QAAQ,EAAC,CAAC,CAAC;IAE1E,IAAI,YAAY,GAAG,KAAK,CAAC;IAEzB,MAAM,cAAc,GAAG,IAAI,QAAQ,CAAC;QAChC,IAAI;YACA,IAAI,YAAY,EAAE,CAAC;gBACf,MAAM,CAAC,KAAK,CAAC,kBAAkB,CAAC,CAAC;gBACjC,EAAE,CAAC,MAAM,EAAE,CAAC;gBACZ,YAAY,GAAG,KAAK,CAAC;YACzB,CAAC;QACL,CAAC;QACD,OAAO;YACH,MAAM,CAAC,OAAO,EAAE,CAAC;QACrB,CAAC;QACD,UAAU,EAAE,IAAI;KACnB,CAAC,CAAC;IAEH,IAAI,GAAG,GAAgB,EAAE,CAAC;IAC1B,IAAI,MAAM,GAAY,EAAE,CAAC;IAEzB,EAAE,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,EAAE;QACnB,IAAI,QAAQ;YAAE,OAAO;QAErB,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAEnB,IAAI,IAAI,CAAC,KAAK,CAAC,8BAA8B,CAAC,EAAE,CAAC;YAC7C,KAAK;QACT,CAAC;aACI,CAAC;YACF,MAAM,CAAC,KAAK,CAAC,0BAA0B,SAAS,GAAG,CAAC,EAAE,CAAC,CAAC;YACxD,QAAQ,GAAG,IAAI,CAAC;YAChB,MAAM,CAAC,OAAO,EAAE,CAAC;YACjB,EAAE,CAAC,KAAK,EAAE,CAAC;YACX,cAAc,CAAC,OAAO,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC;YACzD,OAAO;QACX,CAAC;QAED,MAAM,CAAC,EAAE,EAAC,GAAG,IAAI,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QACrC,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAE5B,IAAI,MAAM,IAAI,MAAM,KAAK,EAAE,EAAE,CAAC;YAC1B,MAAM,EAAE,GAAG,cAAc,CAAC,IAAI,CAAC;gBAC3B,MAAM,EAAG,GAAG;aACf,CAAC,CAAC;YAEH,IAAI,CAAC,EAAE,EAAE,CAAC;gBACN,MAAM,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAC;gBAChC,EAAE,CAAC,KAAK,EAAE,CAAC;gBACX,YAAY,GAAG,IAAI,CAAC;YACxB,CAAC;YAED,GAAG,GAAG,EAAE,CAAC;YACT,SAAS,EAAE,CAAC;QAChB,CAAC;QAED,MAAM,GAAG,GAAI,IAAI,EAAE,SAAS,CAAC,CAAC,EAAC,CAAC,CAAC,CAAC;QAClC,MAAM,IAAI,GAAG,IAAI,EAAE,SAAS,CAAC,CAAC,EAAC,CAAC,CAAC,CAAC;QAClC,MAAM,IAAI,GAAG,IAAI,EAAE,SAAS,CAAC,CAAC,EAAC,CAAC,CAAC,CAAC;QAClC,MAAM,EAAE,GAAK,IAAI,EAAE,SAAS,CAAC,CAAC,CAAC,CAAC;QAChC,MAAM,KAAK,GAAG,EAAE,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;QAElC,IAAI,GAAG,IAAI,KAAK,IAAI,GAAG,KAAK,KAAK,IAAI,GAAG,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;YACxD,GAAG,CAAC,IAAI,CAAC;gBACL,GAAG,EAAC,IAAI,EAAC,IAAI;aAChB,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QACnC,CAAC;aACI,CAAC;YACF,GAAG,CAAC,IAAI,CAAC;gBACL,GAAG,EAAC,IAAI,EAAC,IAAI;aAChB,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAC9B,CAAC;QAED,MAAM,GAAG,EAAG,CAAC;IACjB,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,EAAE;QACrB,IAAI,QAAQ;YAAE,OAAO;QACrB,MAAM,CAAC,KAAK,CAAC,kBAAkB,KAAK,EAAE,CAAC,CAAC;IAC5C,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE;QAChB,IAAI,QAAQ;YAAE,OAAO;QACrB,cAAc,CAAC,IAAI,CAAC;YAChB,MAAM,EAAE,GAAG;SACd,CAAC,CAAC;QACH,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC9B,CAAC,CAAC,CAAC;IAEH,OAAO,cAAc,CAAC;AAC1B,CAAC"}
|
package/dist/input/json.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"json.d.ts","sourceRoot":"","sources":["../../src/input/json.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,QAAQ,CAAC;AAMlC,wBAAsB,eAAe,CAAC,MAAM,EAAE,QAAQ,EAAE,KAAK,EAAE,GAAG,GAAI,OAAO,CAAC,QAAQ,CAAC,
|
|
1
|
+
{"version":3,"file":"json.d.ts","sourceRoot":"","sources":["../../src/input/json.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,QAAQ,CAAC;AAMlC,wBAAsB,eAAe,CAAC,MAAM,EAAE,QAAQ,EAAE,KAAK,EAAE,GAAG,GAAI,OAAO,CAAC,QAAQ,CAAC,CAkCtF"}
|
package/dist/input/json.js
CHANGED
|
@@ -3,7 +3,6 @@ import streamArray from "stream-json/streamers/StreamArray.js";
|
|
|
3
3
|
import log4js from 'log4js';
|
|
4
4
|
const logger = log4js.getLogger();
|
|
5
5
|
export async function stream2readable(stream, _opts) {
|
|
6
|
-
let recordNum = 0;
|
|
7
6
|
const pipeline = stream.pipe(streamArray.withParser());
|
|
8
7
|
let sourcePaused = false;
|
|
9
8
|
const readableStream = new Readable({
|
|
@@ -14,6 +13,9 @@ export async function stream2readable(stream, _opts) {
|
|
|
14
13
|
sourcePaused = false;
|
|
15
14
|
}
|
|
16
15
|
},
|
|
16
|
+
destroy() {
|
|
17
|
+
stream.destroy();
|
|
18
|
+
},
|
|
17
19
|
objectMode: true
|
|
18
20
|
});
|
|
19
21
|
pipeline.on('data', (data) => {
|
|
@@ -23,13 +25,8 @@ export async function stream2readable(stream, _opts) {
|
|
|
23
25
|
pipeline.pause();
|
|
24
26
|
sourcePaused = true;
|
|
25
27
|
}
|
|
26
|
-
recordNum++;
|
|
27
|
-
if (recordNum % 1000 === 0) {
|
|
28
|
-
logger.info(`record: ${recordNum}`);
|
|
29
|
-
}
|
|
30
28
|
});
|
|
31
29
|
pipeline.on('end', () => {
|
|
32
|
-
logger.info(`processed ${recordNum} records`);
|
|
33
30
|
readableStream.push(null);
|
|
34
31
|
});
|
|
35
32
|
return readableStream;
|
package/dist/input/json.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"json.js","sourceRoot":"","sources":["../../src/input/json.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,QAAQ,CAAC;AAClC,OAAO,WAAW,MAAM,sCAAsC,CAAC;AAC/D,OAAO,MAAM,MAAM,QAAQ,CAAC;AAE5B,MAAM,MAAM,GAAG,MAAM,CAAC,SAAS,EAAE,CAAC;AAElC,MAAM,CAAC,KAAK,UAAU,eAAe,CAAC,MAAgB,EAAE,KAAU;IAC9D,
|
|
1
|
+
{"version":3,"file":"json.js","sourceRoot":"","sources":["../../src/input/json.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,QAAQ,CAAC;AAClC,OAAO,WAAW,MAAM,sCAAsC,CAAC;AAC/D,OAAO,MAAM,MAAM,QAAQ,CAAC;AAE5B,MAAM,MAAM,GAAG,MAAM,CAAC,SAAS,EAAE,CAAC;AAElC,MAAM,CAAC,KAAK,UAAU,eAAe,CAAC,MAAgB,EAAE,KAAU;IAC9D,MAAM,QAAQ,GAAG,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,UAAU,EAAE,CAAC,CAAC;IAEvD,IAAI,YAAY,GAAG,KAAK,CAAC;IAEzB,MAAM,cAAc,GAAG,IAAI,QAAQ,CAAC;QAChC,IAAI;YACA,IAAI,YAAY,EAAE,CAAC;gBACf,MAAM,CAAC,KAAK,CAAC,kBAAkB,CAAC,CAAC;gBACjC,QAAQ,CAAC,MAAM,EAAE,CAAC;gBAClB,YAAY,GAAG,KAAK,CAAC;YACzB,CAAC;QACL,CAAC;QACD,OAAO;YACH,MAAM,CAAC,OAAO,EAAE,CAAC;QACrB,CAAC;QACD,UAAU,EAAE,IAAI;KACnB,CAAC,CAAC;IAEH,QAAQ,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,IAAS,EAAE,EAAE;QAC9B,MAAM,EAAE,GAAG,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAE3C,IAAI,CAAC,EAAE,EAAE,CAAC;YACN,MAAM,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAA;YAC/B,QAAQ,CAAC,KAAK,EAAE,CAAC;YACjB,YAAY,GAAG,IAAI,CAAC;QACxB,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE;QACpB,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC9B,CAAC,CAAC,CAAC;IAEH,OAAO,cAAc,CAAC;AAC1B,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"jsonl.d.ts","sourceRoot":"","sources":["../../src/input/jsonl.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,QAAQ,CAAC;AAMlC,wBAAsB,eAAe,CAAC,MAAM,EAAE,QAAQ,EAAE,KAAK,EAAE,GAAG,GAAI,OAAO,CAAC,QAAQ,CAAC,
|
|
1
|
+
{"version":3,"file":"jsonl.d.ts","sourceRoot":"","sources":["../../src/input/jsonl.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,QAAQ,CAAC;AAMlC,wBAAsB,eAAe,CAAC,MAAM,EAAE,QAAQ,EAAE,KAAK,EAAE,GAAG,GAAI,OAAO,CAAC,QAAQ,CAAC,CAyDtF"}
|
package/dist/input/jsonl.js
CHANGED
|
@@ -15,6 +15,9 @@ export async function stream2readable(stream, _opts) {
|
|
|
15
15
|
sourcePaused = false;
|
|
16
16
|
}
|
|
17
17
|
},
|
|
18
|
+
destroy() {
|
|
19
|
+
stream.destroy();
|
|
20
|
+
},
|
|
18
21
|
objectMode: true
|
|
19
22
|
});
|
|
20
23
|
rl.on('line', (line) => {
|
|
@@ -29,9 +32,6 @@ export async function stream2readable(stream, _opts) {
|
|
|
29
32
|
sourcePaused = true;
|
|
30
33
|
}
|
|
31
34
|
recordNum++;
|
|
32
|
-
if (recordNum % 1000 === 0) {
|
|
33
|
-
logger.info(`record: ${recordNum}`);
|
|
34
|
-
}
|
|
35
35
|
}
|
|
36
36
|
catch (error) {
|
|
37
37
|
hasError = true;
|
|
@@ -50,7 +50,6 @@ export async function stream2readable(stream, _opts) {
|
|
|
50
50
|
if (hasError)
|
|
51
51
|
return;
|
|
52
52
|
readableStream.push(null);
|
|
53
|
-
logger.info(`processed ${recordNum} records`);
|
|
54
53
|
});
|
|
55
54
|
return readableStream;
|
|
56
55
|
}
|
package/dist/input/jsonl.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"jsonl.js","sourceRoot":"","sources":["../../src/input/jsonl.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,QAAQ,CAAC;AAClC,OAAO,KAAK,QAAQ,MAAM,eAAe,CAAA;AACzC,OAAO,MAAM,MAAM,QAAQ,CAAC;AAE5B,MAAM,MAAM,GAAG,MAAM,CAAC,SAAS,EAAE,CAAC;AAElC,MAAM,CAAC,KAAK,UAAU,eAAe,CAAC,MAAgB,EAAE,KAAU;IAC9D,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,IAAI,QAAQ,GAAG,KAAK,CAAC;IAErB,MAAM,EAAE,GAAG,QAAQ,CAAC,eAAe,CAAC,EAAC,KAAK,EAAE,MAAM,EAAE,SAAS,EAAE,QAAQ,EAAC,CAAC,CAAC;IAE1E,IAAI,YAAY,GAAG,KAAK,CAAC;IAEzB,MAAM,cAAc,GAAG,IAAI,QAAQ,CAAC;QAChC,IAAI;YACA,IAAI,YAAY,EAAE,CAAC;gBACf,MAAM,CAAC,KAAK,CAAC,kBAAkB,CAAC,CAAC;gBACjC,EAAE,CAAC,MAAM,EAAE,CAAC;gBACZ,YAAY,GAAG,KAAK,CAAC;YACzB,CAAC;QACL,CAAC;QACD,UAAU,EAAE,IAAI;KACnB,CAAC,CAAC;IAEH,EAAE,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,EAAE;QACnB,IAAI,QAAQ;YAAE,OAAO;QAErB,SAAS,EAAE,CAAC;QAEZ,IAAI,CAAC;YACD,MAAM,EAAE,GAAG,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC;YAEjD,IAAI,CAAC,EAAE,EAAE,CAAC;gBACN,MAAM,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAC;gBAChC,EAAE,CAAC,KAAK,EAAE,CAAC;gBACX,YAAY,GAAG,IAAI,CAAC;YACxB,CAAC;YAED,SAAS,EAAE,CAAC;
|
|
1
|
+
{"version":3,"file":"jsonl.js","sourceRoot":"","sources":["../../src/input/jsonl.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,QAAQ,CAAC;AAClC,OAAO,KAAK,QAAQ,MAAM,eAAe,CAAA;AACzC,OAAO,MAAM,MAAM,QAAQ,CAAC;AAE5B,MAAM,MAAM,GAAG,MAAM,CAAC,SAAS,EAAE,CAAC;AAElC,MAAM,CAAC,KAAK,UAAU,eAAe,CAAC,MAAgB,EAAE,KAAU;IAC9D,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,IAAI,QAAQ,GAAG,KAAK,CAAC;IAErB,MAAM,EAAE,GAAG,QAAQ,CAAC,eAAe,CAAC,EAAC,KAAK,EAAE,MAAM,EAAE,SAAS,EAAE,QAAQ,EAAC,CAAC,CAAC;IAE1E,IAAI,YAAY,GAAG,KAAK,CAAC;IAEzB,MAAM,cAAc,GAAG,IAAI,QAAQ,CAAC;QAChC,IAAI;YACA,IAAI,YAAY,EAAE,CAAC;gBACf,MAAM,CAAC,KAAK,CAAC,kBAAkB,CAAC,CAAC;gBACjC,EAAE,CAAC,MAAM,EAAE,CAAC;gBACZ,YAAY,GAAG,KAAK,CAAC;YACzB,CAAC;QACL,CAAC;QACD,OAAO;YACH,MAAM,CAAC,OAAO,EAAE,CAAC;QACrB,CAAC;QACD,UAAU,EAAE,IAAI;KACnB,CAAC,CAAC;IAEH,EAAE,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,EAAE;QACnB,IAAI,QAAQ;YAAE,OAAO;QAErB,SAAS,EAAE,CAAC;QAEZ,IAAI,CAAC;YACD,MAAM,EAAE,GAAG,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC;YAEjD,IAAI,CAAC,EAAE,EAAE,CAAC;gBACN,MAAM,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAC;gBAChC,EAAE,CAAC,KAAK,EAAE,CAAC;gBACX,YAAY,GAAG,IAAI,CAAC;YACxB,CAAC;YAED,SAAS,EAAE,CAAC;QAChB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACb,QAAQ,GAAG,IAAI,CAAC;YAChB,MAAM,CAAC,KAAK,CAAC,4BAA4B,SAAS,GAAG,CAAC,KAAK,KAAK,EAAE,CAAC,CAAC;YACpE,MAAM,CAAC,OAAO,EAAE,CAAC;YACjB,EAAE,CAAC,KAAK,EAAE,CAAC;YACX,cAAc,CAAC,OAAO,CAAC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QACtF,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,EAAE;QACrB,IAAI,QAAQ;YAAE,OAAO;QACrB,MAAM,CAAC,KAAK,CAAC,kBAAkB,KAAK,EAAE,CAAC,CAAC;IAC5C,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE;QAChB,IAAI,QAAQ;YAAE,OAAO;QACrB,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC9B,CAAC,CAAC,CAAC;IAEH,OAAO,cAAc,CAAC;AAC1B,CAAC"}
|
package/dist/input/marc.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"marc.d.ts","sourceRoot":"","sources":["../../src/input/marc.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,QAAQ,CAAC;AAMlC,wBAAsB,eAAe,CAAC,MAAM,EAAE,QAAQ,EAAE,KAAK,EAAE,GAAG,GAAI,OAAO,CAAC,QAAQ,CAAC,
|
|
1
|
+
{"version":3,"file":"marc.d.ts","sourceRoot":"","sources":["../../src/input/marc.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,QAAQ,CAAC;AAMlC,wBAAsB,eAAe,CAAC,MAAM,EAAE,QAAQ,EAAE,KAAK,EAAE,GAAG,GAAI,OAAO,CAAC,QAAQ,CAAC,CA4EtF"}
|
package/dist/input/marc.js
CHANGED
package/dist/input/marc.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"marc.js","sourceRoot":"","sources":["../../src/input/marc.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,QAAQ,CAAC;AAClC,OAAO,EAAE,IAAI,EAAE,MAAM,QAAQ,CAAC;AAC9B,OAAO,MAAM,MAAM,QAAQ,CAAC;AAE5B,MAAM,MAAM,GAAG,MAAM,CAAC,SAAS,EAAE,CAAC;AAElC,MAAM,CAAC,KAAK,UAAU,eAAe,CAAC,MAAgB,EAAE,KAAU;IAC9D,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,IAAI,QAAQ,GAAG,KAAK,CAAC;IACrB,IAAI,YAAY,GAAG,KAAK,CAAC;IAEzB,MAAM,MAAM,GAAG,IAAI,CAAC,YAAY,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC;IAEtD,MAAM,cAAc,GAAG,IAAI,QAAQ,CAAC;QAChC,IAAI;YACA,IAAI,YAAY,EAAE,CAAC;gBACf,MAAM,CAAC,KAAK,CAAC,kBAAkB,CAAC,CAAC;gBACjC,MAAM,CAAC,MAAM,EAAE,CAAC;gBAChB,YAAY,GAAG,KAAK,CAAC;YACzB,CAAC;QACL,CAAC;QACD,UAAU,EAAE,IAAI;KACnB,CAAC,CAAC;IAEH,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,MAAM,EAAE,EAAE;QACzB,SAAS,EAAE,CAAC;QAEZ,IAAI,SAAS,GAAG,IAAI,KAAK,CAAC,EAAE,CAAC;YACzB,MAAM,CAAC,IAAI,CAAC,WAAW,SAAS,EAAE,CAAC,CAAC;QACxC,CAAC;QAED,IAAI,GAAG,GAAgB,EAAE,CAAC;QAE1B,GAAG,CAAC,IAAI,CAAC,CAAE,KAAK,EAAG,GAAG,EAAG,GAAG,EAAG,GAAG,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC;QAEpD,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC;YAChC,MAAM,GAAG,GAAI,KAAK,CAAC,CAAC,CAAC,CAAC;YAEtB,IAAI,KAAK,CAAC,MAAM,IAAI,CAAC,IAAI,GAAG,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;gBAC5C,MAAM,IAAI,GAAG,GAAG,CAAC;gBACjB,MAAM,IAAI,GAAG,GAAG,CAAC;gBACjB,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;gBAE5B,GAAG,CAAC,IAAI,CAAC,CAAE,GAAG,EAAG,IAAI,EAAE,IAAI,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC;YAC/C,CAAC;iBACI,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACxB,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;gBAChC,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;gBAChC,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;gBAE5B,GAAG,CAAC,IAAI,CAAC,CAAE,GAAG,EAAG,IAAI,EAAE,IAAI,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC;YAC/C,CAAC;iBACI,CAAC;gBACF,MAAM,CAAC,IAAI,CAAC,cAAc,EAAE,KAAK,CAAC,CAAC;YACvC,CAAC;QACL,CAAC;QAED,MAAM,EAAE,GAAG,cAAc,CAAC,IAAI,CAAC;YAC3B,MAAM,EAAG,GAAG;SACf,CAAC,CAAC;QAEH,IAAI,CAAC,EAAE,EAAE,CAAC;YACN,MAAM,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAC;YAChC,MAAM,CAAC,KAAK,EAAE,CAAC;YACf,YAAY,GAAG,IAAI,CAAC;QACxB,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE;QACpB,IAAI,SAAS,GAAG,IAAI,KAAK,CAAC,EAAE,CAAC;YACzB,MAAM,CAAC,IAAI,CAAC,WAAW,SAAS,EAAE,CAAC,CAAC;QACxC,CAAC;QACD,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC1B,MAAM,CAAC,IAAI,CAAC,aAAa,SAAS,UAAU,CAAC,CAAC;IAClD,CAAC,CAAC,CAAC;IAEH,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAEpB,OAAO,cAAc,CAAC;AAC1B,CAAC"}
|
|
1
|
+
{"version":3,"file":"marc.js","sourceRoot":"","sources":["../../src/input/marc.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,QAAQ,CAAC;AAClC,OAAO,EAAE,IAAI,EAAE,MAAM,QAAQ,CAAC;AAC9B,OAAO,MAAM,MAAM,QAAQ,CAAC;AAE5B,MAAM,MAAM,GAAG,MAAM,CAAC,SAAS,EAAE,CAAC;AAElC,MAAM,CAAC,KAAK,UAAU,eAAe,CAAC,MAAgB,EAAE,KAAU;IAC9D,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,IAAI,QAAQ,GAAG,KAAK,CAAC;IACrB,IAAI,YAAY,GAAG,KAAK,CAAC;IAEzB,MAAM,MAAM,GAAG,IAAI,CAAC,YAAY,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC;IAEtD,MAAM,cAAc,GAAG,IAAI,QAAQ,CAAC;QAChC,IAAI;YACA,IAAI,YAAY,EAAE,CAAC;gBACf,MAAM,CAAC,KAAK,CAAC,kBAAkB,CAAC,CAAC;gBACjC,MAAM,CAAC,MAAM,EAAE,CAAC;gBAChB,YAAY,GAAG,KAAK,CAAC;YACzB,CAAC;QACL,CAAC;QACD,OAAO;YACH,MAAM,CAAC,OAAO,EAAE,CAAC;QACrB,CAAC;QACD,UAAU,EAAE,IAAI;KACnB,CAAC,CAAC;IAEH,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,MAAM,EAAE,EAAE;QACzB,SAAS,EAAE,CAAC;QAEZ,IAAI,SAAS,GAAG,IAAI,KAAK,CAAC,EAAE,CAAC;YACzB,MAAM,CAAC,IAAI,CAAC,WAAW,SAAS,EAAE,CAAC,CAAC;QACxC,CAAC;QAED,IAAI,GAAG,GAAgB,EAAE,CAAC;QAE1B,GAAG,CAAC,IAAI,CAAC,CAAE,KAAK,EAAG,GAAG,EAAG,GAAG,EAAG,GAAG,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC;QAEpD,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC;YAChC,MAAM,GAAG,GAAI,KAAK,CAAC,CAAC,CAAC,CAAC;YAEtB,IAAI,KAAK,CAAC,MAAM,IAAI,CAAC,IAAI,GAAG,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;gBAC5C,MAAM,IAAI,GAAG,GAAG,CAAC;gBACjB,MAAM,IAAI,GAAG,GAAG,CAAC;gBACjB,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;gBAE5B,GAAG,CAAC,IAAI,CAAC,CAAE,GAAG,EAAG,IAAI,EAAE,IAAI,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC;YAC/C,CAAC;iBACI,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACxB,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;gBAChC,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;gBAChC,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;gBAE5B,GAAG,CAAC,IAAI,CAAC,CAAE,GAAG,EAAG,IAAI,EAAE,IAAI,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC;YAC/C,CAAC;iBACI,CAAC;gBACF,MAAM,CAAC,IAAI,CAAC,cAAc,EAAE,KAAK,CAAC,CAAC;YACvC,CAAC;QACL,CAAC;QAED,MAAM,EAAE,GAAG,cAAc,CAAC,IAAI,CAAC;YAC3B,MAAM,EAAG,GAAG;SACf,CAAC,CAAC;QAEH,IAAI,CAAC,EAAE,EAAE,CAAC;YACN,MAAM,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAC;YAChC,MAAM,CAAC,KAAK,EAAE,CAAC;YACf,YAAY,GAAG,IAAI,CAAC;QACxB,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE;QACpB,IAAI,SAAS,GAAG,IAAI,KAAK,CAAC,EAAE,CAAC;YACzB,MAAM,CAAC,IAAI,CAAC,WAAW,SAAS,EAAE,CAAC,CAAC;QACxC,CAAC;QACD,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC1B,MAAM,CAAC,IAAI,CAAC,aAAa,SAAS,UAAU,CAAC,CAAC;IAClD,CAAC,CAAC,CAAC;IAEH,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAEpB,OAAO,cAAc,CAAC;AAC1B,CAAC"}
|
package/dist/input/rdf.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"rdf.d.ts","sourceRoot":"","sources":["../../src/input/rdf.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,QAAQ,CAAC;AASlC,wBAAsB,eAAe,CAAC,MAAM,EAAE,QAAQ,EAAE,IAAI,EAAE,GAAG,GAAI,OAAO,CAAC,QAAQ,CAAC,
|
|
1
|
+
{"version":3,"file":"rdf.d.ts","sourceRoot":"","sources":["../../src/input/rdf.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,QAAQ,CAAC;AASlC,wBAAsB,eAAe,CAAC,MAAM,EAAE,QAAQ,EAAE,IAAI,EAAE,GAAG,GAAI,OAAO,CAAC,QAAQ,CAAC,CAcrF"}
|
package/dist/input/rdf.js
CHANGED
|
@@ -4,7 +4,12 @@ import { parseStream } from "../util/rdf_parse.js";
|
|
|
4
4
|
import log4js from 'log4js';
|
|
5
5
|
const logger = log4js.getLogger();
|
|
6
6
|
export async function stream2readable(stream, opts) {
|
|
7
|
-
const readableStream = new Readable({
|
|
7
|
+
const readableStream = new Readable({
|
|
8
|
+
objectMode: true,
|
|
9
|
+
destroy() {
|
|
10
|
+
stream.destroy();
|
|
11
|
+
}
|
|
12
|
+
});
|
|
8
13
|
const record = await parseStream(stream, opts.path.href);
|
|
9
14
|
readableStream.push(record);
|
|
10
15
|
readableStream.push(null);
|
package/dist/input/rdf.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"rdf.js","sourceRoot":"","sources":["../../src/input/rdf.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,QAAQ,CAAC;AAClC,OAAO,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AAEtC,OAAO,EAAE,WAAW,EAAE,MAAM,sBAAsB,CAAC;AAEnD,OAAO,MAAM,MAAM,QAAQ,CAAC;AAE5B,MAAM,MAAM,GAAG,MAAM,CAAC,SAAS,EAAE,CAAC;AAElC,MAAM,CAAC,KAAK,UAAU,eAAe,CAAC,MAAgB,EAAE,IAAS;IAC7D,MAAM,cAAc,GAAG,IAAI,QAAQ,CAAC,
|
|
1
|
+
{"version":3,"file":"rdf.js","sourceRoot":"","sources":["../../src/input/rdf.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,QAAQ,CAAC;AAClC,OAAO,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AAEtC,OAAO,EAAE,WAAW,EAAE,MAAM,sBAAsB,CAAC;AAEnD,OAAO,MAAM,MAAM,QAAQ,CAAC;AAE5B,MAAM,MAAM,GAAG,MAAM,CAAC,SAAS,EAAE,CAAC;AAElC,MAAM,CAAC,KAAK,UAAU,eAAe,CAAC,MAAgB,EAAE,IAAS;IAC7D,MAAM,cAAc,GAAG,IAAI,QAAQ,CAAC;QAChC,UAAU,EAAE,IAAI;QAChB,OAAO;YACH,MAAM,CAAC,OAAO,EAAE,CAAC;QACrB,CAAC;KACJ,CAAC,CAAC;IAEH,MAAM,MAAM,GAAY,MAAM,WAAW,CAAC,MAAM,EAAE,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAElE,cAAc,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAC5B,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAE1B,OAAO,cAAc,CAAC;AAC1B,CAAC"}
|
package/dist/input/xml.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"xml.d.ts","sourceRoot":"","sources":["../../src/input/xml.ts"],"names":[],"mappings":"AAAA,OAAO,EAAY,QAAQ,EAAE,MAAM,QAAQ,CAAC;AAU5C,wBAAsB,eAAe,CAAC,MAAM,EAAE,QAAQ,GAAI,OAAO,CAAC,QAAQ,CAAC,
|
|
1
|
+
{"version":3,"file":"xml.d.ts","sourceRoot":"","sources":["../../src/input/xml.ts"],"names":[],"mappings":"AAAA,OAAO,EAAY,QAAQ,EAAE,MAAM,QAAQ,CAAC;AAU5C,wBAAsB,eAAe,CAAC,MAAM,EAAE,QAAQ,GAAI,OAAO,CAAC,QAAQ,CAAC,CAgJ1E"}
|
package/dist/input/xml.js
CHANGED
package/dist/input/xml.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"xml.js","sourceRoot":"","sources":["../../src/input/xml.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,QAAQ,CAAC;AAC5C,OAAO,GAAG,MAAM,KAAK,CAAC;AACtB,OAAO,MAAM,MAAM,QAAQ,CAAC;AAM5B,MAAM,MAAM,GAAG,MAAM,CAAC,SAAS,EAAE,CAAC;AAElC,MAAM,CAAC,KAAK,UAAU,eAAe,CAAC,MAAgB;IAClD,IAAI,SAAS,GAAG,CAAC,CAAC;IAElB,IAAI,YAAY,GAAG,KAAK,CAAC;IAEzB,MAAM,cAAc,GAAG,IAAI,QAAQ,CAAC;QAChC,IAAI;YACA,IAAI,YAAY,EAAE,CAAC;gBACf,MAAM,CAAC,KAAK,CAAC,kBAAkB,CAAC,CAAC;gBACjC,MAAM,CAAC,MAAM,EAAE,CAAC;gBAChB,YAAY,GAAG,KAAK,CAAC;YACzB,CAAC;QACL,CAAC;QACD,UAAU,EAAE,IAAI;KACnB,CAAC,CAAC;IAEH,MAAM,MAAM,GAAG,GAAG,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;IAEtC,IAAI,MAAM,GAAgB,EAAE,CAAC;IAC7B,IAAI,QAAQ,GAAc,EAAE,CAAC;IAC7B,IAAI,MAAM,GAAoB,EAAE,CAAC;IACjC,IAAI,OAAO,GAAuB,EAAE,CAAC;IACrC,IAAI,IAAe,CAAC;IACpB,IAAI,IAAI,GAAY,EAAE,CAAC;IACvB,IAAI,QAAQ,GAAG,KAAK,CAAC;IAErB,MAAM,CAAC,EAAE,CAAC,SAAS,EAAE,CAAC,IAAa,EAAE,EAAE;QACnC,IAAI,QAAQ;YAAE,OAAO;QAErB,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,QAAQ,EAAC,EAAE,CAAC,CAAC;QAEpD,IAAI,SAAS,KAAK,YAAY,EAAE,CAAC;YAC7B,sBAAsB;QAC1B,CAAC;aACI,IAAI,SAAS,KAAK,QAAQ,EAAE,CAAC;YAC9B,kBAAkB;QACtB,CAAC;aACI,IAAI,SAAS,KAAK,QAAQ,EAAE,CAAC;YAC9B,IAAI,GAAG,QAAQ,CAAC;QACpB,CAAC;aACI,IAAI,SAAS,IAAI,cAAc,EAAE,CAAC;YACnC,IAAI,GAAG,SAAS,CAAC;YACjB,MAAM,GAAG,IAAI,CAAC,UAAU,CAAA;QAC5B,CAAC;aACI,IAAI,SAAS,KAAK,WAAW,EAAE,CAAC;YACjC,MAAM,GAAG,IAAI,CAAC,UAAU,CAAC;QAC7B,CAAC;aACI,IAAI,SAAS,KAAK,UAAU,EAAE,CAAC;YAChC,OAAO,GAAG,IAAI,CAAC,UAAU,CAAC;QAC9B,CAAC;aACI,CAAC;YACF,MAAM,CAAC,KAAK,CAAC,gBAAgB,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;QAC9C,CAAC;QAED,IAAI,GAAG,EAAE,CAAC;IACd,CAAC,CAAC,CAAC;IAEH,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,CAAS,EAAE,EAAE;QAC5B,IAAI,QAAQ;YAAE,OAAO;QAErB,IAAI,IAAI,CAAC,CAAC;IACd,CAAC,CAAC,CAAC;IAEH,MAAM,CAAC,EAAE,CAAC,UAAU,EAAE,CAAC,GAAW,EAAE,EAAE;QAClC,IAAI,QAAQ;YAAE,OAAO;QAErB,MAAM,SAAS,GAAG,GAAG,CAAC,UAAU,CAAC,QAAQ,EAAC,EAAE,CAAC,CAAC;QAC9C,IAAI,SAAS,KAAK,QAAQ,EAAE,CAAC;YACzB,MAAM,CAAC,IAAI,CAAC,CAAC,KAAK,EAAC,GAAG,EAAC,GAAG,EAAC,GAAG,EAAC,IAAI,CAAC,CAAC,CAAC;QAC1C,CAAC;aACI,IAAI,SAAS,IAAI,cAAc,EAAE,CAAC;YACnC,IAAI,GAAG,GAAG,MAAM,CAAC,GAAa,CAAC;YAC/B,MAAM,CAAC,IAAI,CAAC,CAAC,GAAG,EAAC,GAAG,EAAC,GAAG,EAAC,GAAG,EAAC,IAAI,CAAC,CAAC,CAAC;QACxC,CAAC;aACI,IAAI,SAAS,KAAK,WAAW,EAAE,CAAC;YACjC,IAAI,GAAG,GAAI,MAAM,CAAC,GAAa,CAAC;YAChC,IAAI,IAAI,GAAG,MAAM,CAAC,IAAc,CAAC;YACjC,IAAI,IAAI,GAAG,MAAM,CAAC,IAAc,CAAC;YACjC,MAAM,CAAC,IAAI,CAAC,CAAC,GAAG,EAAC,IAAI,EAAC,IAAI,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC;YAC9C,QAAQ,GAAG,EAAE,CAAC;QAClB,CAAC;aACI,IAAI,SAAS,KAAK,UAAU,EAAE,CAAC;YAChC,IAAI,IAAI,GAAG,OAAO,CAAC,IAAc,CAAC;YAClC,QAAQ,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,IAAI,EAAC,IAAI,CAAC,CAAC,CAAC;QAC5C,CAAC;QACD,IAAI,SAAS,KAAK,QAAQ,EAAE,CAAC;YACzB,MAAM,EAAE,GAAG,cAAc,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC;YAE3C,IAAI,CAAC,EAAE,EAAE,CAAC;gBACN,MAAM,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAC;gBAChC,MAAM,CAAC,KAAK,EAAE,CAAC;gBACf,YAAY,GAAG,IAAI,CAAC;YACxB,CAAC;YAED,MAAM,GAAG,EAAE,CAAC;YACZ,SAAS,EAAE,CAAC;YAEZ,IAAI,SAAS,GAAG,IAAI,KAAK,CAAC,EAAE,CAAC;gBACzB,MAAM,CAAC,IAAI,CAAC,WAAW,SAAS,EAAE,CAAC,CAAC;YACxC,CAAC;QACL,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE;QACvB,IAAI,QAAQ;YAAE,OAAO;QACrB,QAAQ,GAAG,IAAI,CAAC;QAEhB,MAAM,CAAC,KAAK,CAAE,eAAe,EAAE,GAAG,CAAC,OAAO,CAAC,CAAC;QAC5C,MAAM,CAAC,OAAO,EAAE,CAAC;QACjB,MAAM,CAAC,GAAG,EAAE,CAAC;QACb,cAAc,CAAC,OAAO,CAAC,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;IAChF,CAAC,CAAC,CAAC;IAEH,MAAM,CAAC,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE;QAClB,IAAI,QAAQ;YAAE,OAAO;QAErB,MAAM,CAAC,IAAI,CAAC,aAAa,SAAS,UAAU,CAAC,CAAC;QAC9C,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC9B,CAAC,CAAC,CAAC;IAEH,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,KAAK,EAAE,EAAE;QACxB,IAAI,QAAQ;YAAE,OAAO;QACrB,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IACxB,CAAC,CAAC,CAAC;IAEH,MAAM,CAAC,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE;QAClB,IAAI,QAAQ;YAAE,OAAO;QACrB,MAAM,CAAC,GAAG,EAAE,CAAC;IACjB,CAAC,CAAC,CAAC;IAEH,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE;QACvB,IAAI,QAAQ;YAAE,OAAO;QACrB,QAAQ,GAAG,IAAI,CAAC;QAEhB,MAAM,CAAC,KAAK,CAAC,sBAAsB,EAAE,GAAG,CAAC,CAAC;QAC1C,MAAM,CAAC,OAAO,EAAE,CAAC;QACjB,MAAM,CAAC,GAAG,EAAE,CAAC;QACb,cAAc,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IAChC,CAAC,CAAC,CAAC;IAEH,OAAO,cAAc,CAAC;AAC1B,CAAC"}
|
|
1
|
+
{"version":3,"file":"xml.js","sourceRoot":"","sources":["../../src/input/xml.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,QAAQ,CAAC;AAC5C,OAAO,GAAG,MAAM,KAAK,CAAC;AACtB,OAAO,MAAM,MAAM,QAAQ,CAAC;AAM5B,MAAM,MAAM,GAAG,MAAM,CAAC,SAAS,EAAE,CAAC;AAElC,MAAM,CAAC,KAAK,UAAU,eAAe,CAAC,MAAgB;IAClD,IAAI,SAAS,GAAG,CAAC,CAAC;IAElB,IAAI,YAAY,GAAG,KAAK,CAAC;IAEzB,MAAM,cAAc,GAAG,IAAI,QAAQ,CAAC;QAChC,IAAI;YACA,IAAI,YAAY,EAAE,CAAC;gBACf,MAAM,CAAC,KAAK,CAAC,kBAAkB,CAAC,CAAC;gBACjC,MAAM,CAAC,MAAM,EAAE,CAAC;gBAChB,YAAY,GAAG,KAAK,CAAC;YACzB,CAAC;QACL,CAAC;QACD,OAAO;YACH,MAAM,CAAC,OAAO,EAAE,CAAC;QACrB,CAAC;QACD,UAAU,EAAE,IAAI;KACnB,CAAC,CAAC;IAEH,MAAM,MAAM,GAAG,GAAG,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;IAEtC,IAAI,MAAM,GAAgB,EAAE,CAAC;IAC7B,IAAI,QAAQ,GAAc,EAAE,CAAC;IAC7B,IAAI,MAAM,GAAoB,EAAE,CAAC;IACjC,IAAI,OAAO,GAAuB,EAAE,CAAC;IACrC,IAAI,IAAe,CAAC;IACpB,IAAI,IAAI,GAAY,EAAE,CAAC;IACvB,IAAI,QAAQ,GAAG,KAAK,CAAC;IAErB,MAAM,CAAC,EAAE,CAAC,SAAS,EAAE,CAAC,IAAa,EAAE,EAAE;QACnC,IAAI,QAAQ;YAAE,OAAO;QAErB,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,QAAQ,EAAC,EAAE,CAAC,CAAC;QAEpD,IAAI,SAAS,KAAK,YAAY,EAAE,CAAC;YAC7B,sBAAsB;QAC1B,CAAC;aACI,IAAI,SAAS,KAAK,QAAQ,EAAE,CAAC;YAC9B,kBAAkB;QACtB,CAAC;aACI,IAAI,SAAS,KAAK,QAAQ,EAAE,CAAC;YAC9B,IAAI,GAAG,QAAQ,CAAC;QACpB,CAAC;aACI,IAAI,SAAS,IAAI,cAAc,EAAE,CAAC;YACnC,IAAI,GAAG,SAAS,CAAC;YACjB,MAAM,GAAG,IAAI,CAAC,UAAU,CAAA;QAC5B,CAAC;aACI,IAAI,SAAS,KAAK,WAAW,EAAE,CAAC;YACjC,MAAM,GAAG,IAAI,CAAC,UAAU,CAAC;QAC7B,CAAC;aACI,IAAI,SAAS,KAAK,UAAU,EAAE,CAAC;YAChC,OAAO,GAAG,IAAI,CAAC,UAAU,CAAC;QAC9B,CAAC;aACI,CAAC;YACF,MAAM,CAAC,KAAK,CAAC,gBAAgB,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;QAC9C,CAAC;QAED,IAAI,GAAG,EAAE,CAAC;IACd,CAAC,CAAC,CAAC;IAEH,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,CAAS,EAAE,EAAE;QAC5B,IAAI,QAAQ;YAAE,OAAO;QAErB,IAAI,IAAI,CAAC,CAAC;IACd,CAAC,CAAC,CAAC;IAEH,MAAM,CAAC,EAAE,CAAC,UAAU,EAAE,CAAC,GAAW,EAAE,EAAE;QAClC,IAAI,QAAQ;YAAE,OAAO;QAErB,MAAM,SAAS,GAAG,GAAG,CAAC,UAAU,CAAC,QAAQ,EAAC,EAAE,CAAC,CAAC;QAC9C,IAAI,SAAS,KAAK,QAAQ,EAAE,CAAC;YACzB,MAAM,CAAC,IAAI,CAAC,CAAC,KAAK,EAAC,GAAG,EAAC,GAAG,EAAC,GAAG,EAAC,IAAI,CAAC,CAAC,CAAC;QAC1C,CAAC;aACI,IAAI,SAAS,IAAI,cAAc,EAAE,CAAC;YACnC,IAAI,GAAG,GAAG,MAAM,CAAC,GAAa,CAAC;YAC/B,MAAM,CAAC,IAAI,CAAC,CAAC,GAAG,EAAC,GAAG,EAAC,GAAG,EAAC,GAAG,EAAC,IAAI,CAAC,CAAC,CAAC;QACxC,CAAC;aACI,IAAI,SAAS,KAAK,WAAW,EAAE,CAAC;YACjC,IAAI,GAAG,GAAI,MAAM,CAAC,GAAa,CAAC;YAChC,IAAI,IAAI,GAAG,MAAM,CAAC,IAAc,CAAC;YACjC,IAAI,IAAI,GAAG,MAAM,CAAC,IAAc,CAAC;YACjC,MAAM,CAAC,IAAI,CAAC,CAAC,GAAG,EAAC,IAAI,EAAC,IAAI,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC;YAC9C,QAAQ,GAAG,EAAE,CAAC;QAClB,CAAC;aACI,IAAI,SAAS,KAAK,UAAU,EAAE,CAAC;YAChC,IAAI,IAAI,GAAG,OAAO,CAAC,IAAc,CAAC;YAClC,QAAQ,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,IAAI,EAAC,IAAI,CAAC,CAAC,CAAC;QAC5C,CAAC;QACD,IAAI,SAAS,KAAK,QAAQ,EAAE,CAAC;YACzB,MAAM,EAAE,GAAG,cAAc,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC;YAE3C,IAAI,CAAC,EAAE,EAAE,CAAC;gBACN,MAAM,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAC;gBAChC,MAAM,CAAC,KAAK,EAAE,CAAC;gBACf,YAAY,GAAG,IAAI,CAAC;YACxB,CAAC;YAED,MAAM,GAAG,EAAE,CAAC;YACZ,SAAS,EAAE,CAAC;YAEZ,IAAI,SAAS,GAAG,IAAI,KAAK,CAAC,EAAE,CAAC;gBACzB,MAAM,CAAC,IAAI,CAAC,WAAW,SAAS,EAAE,CAAC,CAAC;YACxC,CAAC;QACL,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE;QACvB,IAAI,QAAQ;YAAE,OAAO;QACrB,QAAQ,GAAG,IAAI,CAAC;QAEhB,MAAM,CAAC,KAAK,CAAE,eAAe,EAAE,GAAG,CAAC,OAAO,CAAC,CAAC;QAC5C,MAAM,CAAC,OAAO,EAAE,CAAC;QACjB,MAAM,CAAC,GAAG,EAAE,CAAC;QACb,cAAc,CAAC,OAAO,CAAC,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;IAChF,CAAC,CAAC,CAAC;IAEH,MAAM,CAAC,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE;QAClB,IAAI,QAAQ;YAAE,OAAO;QAErB,MAAM,CAAC,IAAI,CAAC,aAAa,SAAS,UAAU,CAAC,CAAC;QAC9C,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC9B,CAAC,CAAC,CAAC;IAEH,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,KAAK,EAAE,EAAE;QACxB,IAAI,QAAQ;YAAE,OAAO;QACrB,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IACxB,CAAC,CAAC,CAAC;IAEH,MAAM,CAAC,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE;QAClB,IAAI,QAAQ;YAAE,OAAO;QACrB,MAAM,CAAC,GAAG,EAAE,CAAC;IACjB,CAAC,CAAC,CAAC;IAEH,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE;QACvB,IAAI,QAAQ;YAAE,OAAO;QACrB,QAAQ,GAAG,IAAI,CAAC;QAEhB,MAAM,CAAC,KAAK,CAAC,sBAAsB,EAAE,GAAG,CAAC,CAAC;QAC1C,MAAM,CAAC,OAAO,EAAE,CAAC;QACjB,MAAM,CAAC,GAAG,EAAE,CAAC;QACb,cAAc,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IAChC,CAAC,CAAC,CAAC;IAEH,OAAO,cAAc,CAAC;AAC1B,CAAC"}
|
|
@@ -1,3 +1,3 @@
|
|
|
1
|
-
import {
|
|
2
|
-
export declare function
|
|
1
|
+
import { Transform } from 'stream';
|
|
2
|
+
export declare function transform(): Promise<Transform>;
|
|
3
3
|
//# sourceMappingURL=alephseq.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"alephseq.d.ts","sourceRoot":"","sources":["../../src/output/alephseq.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,
|
|
1
|
+
{"version":3,"file":"alephseq.d.ts","sourceRoot":"","sources":["../../src/output/alephseq.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,QAAQ,CAAC;AAMnC,wBAAsB,SAAS,IAAK,OAAO,CAAC,SAAS,CAAC,CAwCrD"}
|
package/dist/output/alephseq.js
CHANGED
|
@@ -1,43 +1,39 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { Transform } from 'stream';
|
|
2
2
|
import { marcmap } from '../marcmap.js';
|
|
3
3
|
import log4js from 'log4js';
|
|
4
4
|
const logger = log4js.getLogger();
|
|
5
|
-
export function
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
let
|
|
16
|
-
let
|
|
17
|
-
for (let
|
|
18
|
-
let
|
|
19
|
-
let
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
5
|
+
export async function transform() {
|
|
6
|
+
return new Transform({
|
|
7
|
+
objectMode: true,
|
|
8
|
+
transform(data, _encoding, callback) {
|
|
9
|
+
let rec = data['record'];
|
|
10
|
+
if (!rec) {
|
|
11
|
+
logger.debug('skipped empty record');
|
|
12
|
+
callback();
|
|
13
|
+
return;
|
|
14
|
+
}
|
|
15
|
+
let id = marcmap(rec, "001", {});
|
|
16
|
+
let output = `${id} FMT L BK\n`;
|
|
17
|
+
for (let i = 0; i < rec.length; i++) {
|
|
18
|
+
let tag = rec[i][0];
|
|
19
|
+
let ind1 = rec[i][1];
|
|
20
|
+
let ind2 = rec[i][2];
|
|
21
|
+
let sf = "";
|
|
22
|
+
for (let j = 3; j < rec[i].length; j += 2) {
|
|
23
|
+
let code = rec[i][j];
|
|
24
|
+
let val = rec[i][j + 1];
|
|
25
|
+
if (tag.match(/^FMT|LDR|00./g)) {
|
|
26
|
+
sf += `${val}`;
|
|
27
|
+
}
|
|
28
|
+
else {
|
|
29
|
+
sf += `\$\$${code}${val}`;
|
|
30
|
+
}
|
|
25
31
|
}
|
|
32
|
+
output += `${id} ${tag}${ind1}${ind2} L ${sf}\n`;
|
|
26
33
|
}
|
|
27
|
-
|
|
34
|
+
logger.debug(`adding ${output.length} bytes`);
|
|
35
|
+
callback(null, output);
|
|
28
36
|
}
|
|
29
|
-
const ok = writable.write(output);
|
|
30
|
-
if (!ok) {
|
|
31
|
-
logger.debug("backpressure on");
|
|
32
|
-
readable.pause();
|
|
33
|
-
writable.once("drain", () => {
|
|
34
|
-
logger.debug("backpressure off");
|
|
35
|
-
readable.resume();
|
|
36
|
-
});
|
|
37
|
-
}
|
|
38
|
-
});
|
|
39
|
-
readable.on('end', () => {
|
|
40
|
-
writable.end();
|
|
41
37
|
});
|
|
42
38
|
}
|
|
43
39
|
//# sourceMappingURL=alephseq.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"alephseq.js","sourceRoot":"","sources":["../../src/output/alephseq.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,
|
|
1
|
+
{"version":3,"file":"alephseq.js","sourceRoot":"","sources":["../../src/output/alephseq.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,QAAQ,CAAC;AACnC,OAAO,EAAE,OAAO,EAAE,MAAM,eAAe,CAAC;AACxC,OAAO,MAAM,MAAM,QAAQ,CAAC;AAE5B,MAAM,MAAM,GAAG,MAAM,CAAC,SAAS,EAAE,CAAC;AAElC,MAAM,CAAC,KAAK,UAAU,SAAS;IAC3B,OAAO,IAAI,SAAS,CAAC;QACjB,UAAU,EAAE,IAAI;QAChB,SAAS,CAAC,IAAS,EAAE,SAAS,EAAE,QAAQ;YACpC,IAAI,GAAG,GAAgB,IAAI,CAAC,QAAQ,CAAC,CAAC;YAEtC,IAAI,CAAC,GAAG,EAAE,CAAC;gBACP,MAAM,CAAC,KAAK,CAAC,sBAAsB,CAAC,CAAC;gBACrC,QAAQ,EAAE,CAAC;gBACX,OAAO;YACX,CAAC;YAED,IAAI,EAAE,GAAG,OAAO,CAAC,GAAG,EAAC,KAAK,EAAC,EAAE,CAAC,CAAC;YAE/B,IAAI,MAAM,GAAG,GAAG,EAAE,eAAe,CAAC;YAElC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAG,CAAC,GAAG,GAAG,CAAC,MAAM,EAAG,CAAC,EAAE,EAAE,CAAC;gBACpC,IAAI,GAAG,GAAI,GAAG,CAAC,CAAC,CAAE,CAAC,CAAC,CAAC,CAAC;gBACtB,IAAI,IAAI,GAAG,GAAG,CAAC,CAAC,CAAE,CAAC,CAAC,CAAC,CAAC;gBACtB,IAAI,IAAI,GAAG,GAAG,CAAC,CAAC,CAAE,CAAC,CAAC,CAAC,CAAC;gBACtB,IAAI,EAAE,GAAG,EAAE,CAAC;gBAEZ,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,CAAE,CAAC,MAAM,EAAG,CAAC,IAAI,CAAC,EAAE,CAAC;oBAC1C,IAAI,IAAI,GAAG,GAAG,CAAC,CAAC,CAAE,CAAC,CAAC,CAAC,CAAC;oBACtB,IAAI,GAAG,GAAI,GAAG,CAAC,CAAC,CAAE,CAAC,CAAC,GAAC,CAAC,CAAC,CAAC;oBACxB,IAAI,GAAI,CAAC,KAAK,CAAC,eAAe,CAAC,EAAE,CAAC;wBAC9B,EAAE,IAAI,GAAG,GAAG,EAAE,CAAC;oBACnB,CAAC;yBACI,CAAC;wBACF,EAAE,IAAI,OAAO,IAAI,GAAG,GAAG,EAAE,CAAC;oBAC9B,CAAC;gBACL,CAAC;gBAED,MAAM,IAAI,GAAG,EAAE,IAAI,GAAG,GAAG,IAAI,GAAG,IAAI,MAAM,EAAE,IAAI,CAAC;YACrD,CAAC;YAED,MAAM,CAAC,KAAK,CAAC,UAAU,MAAM,CAAC,MAAM,QAAQ,CAAC,CAAC;YAC9C,QAAQ,CAAC,IAAI,EAAC,MAAM,CAAC,CAAC;QAC1B,CAAC;KACJ,CAAC,CAAC;AACP,CAAC"}
|
package/dist/output/json.d.ts
CHANGED
|
@@ -1,3 +1,3 @@
|
|
|
1
|
-
import {
|
|
2
|
-
export declare function
|
|
1
|
+
import { Transform } from 'stream';
|
|
2
|
+
export declare function transform(): Promise<Transform>;
|
|
3
3
|
//# sourceMappingURL=json.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"json.d.ts","sourceRoot":"","sources":["../../src/output/json.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,
|
|
1
|
+
{"version":3,"file":"json.d.ts","sourceRoot":"","sources":["../../src/output/json.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,QAAQ,CAAC;AAKnC,wBAAsB,SAAS,IAAK,OAAO,CAAC,SAAS,CAAC,CAmCrD"}
|
package/dist/output/json.js
CHANGED
|
@@ -1,29 +1,35 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { Transform } from 'stream';
|
|
2
2
|
import log4js from 'log4js';
|
|
3
3
|
const logger = log4js.getLogger();
|
|
4
|
-
export function
|
|
4
|
+
export async function transform() {
|
|
5
5
|
let isFirst = true;
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
output
|
|
6
|
+
let hasClosed = false;
|
|
7
|
+
return new Transform({
|
|
8
|
+
objectMode: true,
|
|
9
|
+
transform(data, _encoding, callback) {
|
|
10
|
+
let output = "";
|
|
11
|
+
if (isFirst) {
|
|
12
|
+
output += "[";
|
|
13
|
+
}
|
|
14
|
+
else {
|
|
15
|
+
output += ',';
|
|
16
|
+
}
|
|
17
|
+
output += JSON.stringify(data);
|
|
18
|
+
isFirst = false;
|
|
19
|
+
logger.debug(`adding ${output.length} bytes`);
|
|
20
|
+
callback(null, output);
|
|
21
|
+
},
|
|
22
|
+
flush(callback) {
|
|
23
|
+
// Push the closing bracket to the buffer
|
|
24
|
+
if (!isFirst && !hasClosed) {
|
|
25
|
+
this.push("]");
|
|
26
|
+
hasClosed = true;
|
|
27
|
+
}
|
|
28
|
+
callback();
|
|
29
|
+
},
|
|
30
|
+
destroy(err, callback) {
|
|
31
|
+
callback(err);
|
|
11
32
|
}
|
|
12
|
-
output += JSON.stringify(data);
|
|
13
|
-
const ok = writable.write(output);
|
|
14
|
-
if (!ok) {
|
|
15
|
-
logger.debug("backpressure on");
|
|
16
|
-
readable.pause();
|
|
17
|
-
writable.once('drain', () => {
|
|
18
|
-
logger.debug("backpressure off");
|
|
19
|
-
readable.resume();
|
|
20
|
-
});
|
|
21
|
-
}
|
|
22
|
-
isFirst = false;
|
|
23
|
-
});
|
|
24
|
-
readable.on('close', () => {
|
|
25
|
-
writable.write("]");
|
|
26
|
-
writable.end();
|
|
27
33
|
});
|
|
28
34
|
}
|
|
29
35
|
//# sourceMappingURL=json.js.map
|
package/dist/output/json.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"json.js","sourceRoot":"","sources":["../../src/output/json.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,
|
|
1
|
+
{"version":3,"file":"json.js","sourceRoot":"","sources":["../../src/output/json.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,QAAQ,CAAC;AACnC,OAAO,MAAM,MAAM,QAAQ,CAAC;AAE5B,MAAM,MAAM,GAAG,MAAM,CAAC,SAAS,EAAE,CAAC;AAElC,MAAM,CAAC,KAAK,UAAU,SAAS;IAC3B,IAAI,OAAO,GAAG,IAAI,CAAC;IACnB,IAAI,SAAS,GAAG,KAAK,CAAC;IAEtB,OAAO,IAAI,SAAS,CAAC;QACjB,UAAU,EAAE,IAAI;QAChB,SAAS,CAAC,IAAS,EAAE,SAAS,EAAE,QAAQ;YACpC,IAAI,MAAM,GAAG,EAAE,CAAC;YAEhB,IAAI,OAAO,EAAE,CAAC;gBACV,MAAM,IAAI,GAAG,CAAC;YAClB,CAAC;iBACI,CAAC;gBACF,MAAM,IAAI,GAAG,CAAC;YAClB,CAAC;YAED,MAAM,IAAI,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;YAE/B,OAAO,GAAG,KAAK,CAAC;YAEhB,MAAM,CAAC,KAAK,CAAC,UAAU,MAAM,CAAC,MAAM,QAAQ,CAAC,CAAC;YAC9C,QAAQ,CAAC,IAAI,EAAC,MAAM,CAAC,CAAC;QAC1B,CAAC;QACD,KAAK,CAAC,QAAQ;YACV,yCAAyC;YACzC,IAAI,CAAC,OAAO,IAAI,CAAC,SAAS,EAAE,CAAC;gBACzB,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBACf,SAAS,GAAG,IAAI,CAAC;YACrB,CAAC;YACD,QAAQ,EAAE,CAAC;QACf,CAAC;QACD,OAAO,CAAC,GAAG,EAAE,QAAQ;YACjB,QAAQ,CAAC,GAAG,CAAC,CAAC;QAClB,CAAC;KACJ,CAAC,CAAC;AACP,CAAC"}
|
package/dist/output/jsonl.d.ts
CHANGED
|
@@ -1,3 +1,3 @@
|
|
|
1
|
-
import {
|
|
2
|
-
export declare function
|
|
1
|
+
import { Transform } from 'stream';
|
|
2
|
+
export declare function transform(): Promise<Transform>;
|
|
3
3
|
//# sourceMappingURL=jsonl.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"jsonl.d.ts","sourceRoot":"","sources":["../../src/output/jsonl.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,
|
|
1
|
+
{"version":3,"file":"jsonl.d.ts","sourceRoot":"","sources":["../../src/output/jsonl.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,QAAQ,CAAC;AAKnC,wBAAsB,SAAS,IAAK,OAAO,CAAC,SAAS,CAAC,CASrD"}
|