marcattacks 2.0.0 → 2.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -3
- package/dist/index.js +15 -25
- package/dist/index.js.map +1 -1
- package/dist/input/alephseq.d.ts +2 -2
- package/dist/input/alephseq.d.ts.map +1 -1
- package/dist/input/alephseq.js +46 -76
- package/dist/input/alephseq.js.map +1 -1
- package/dist/input/json.d.ts +2 -2
- package/dist/input/json.d.ts.map +1 -1
- package/dist/input/json.js +15 -27
- package/dist/input/json.js.map +1 -1
- package/dist/input/jsonl.d.ts +2 -2
- package/dist/input/jsonl.d.ts.map +1 -1
- package/dist/input/jsonl.js +27 -46
- package/dist/input/jsonl.js.map +1 -1
- package/dist/input/marc.d.ts +2 -2
- package/dist/input/marc.d.ts.map +1 -1
- package/dist/input/marc.js +18 -37
- package/dist/input/marc.js.map +1 -1
- package/dist/input/rdf.d.ts +2 -2
- package/dist/input/rdf.d.ts.map +1 -1
- package/dist/input/rdf.js +23 -10
- package/dist/input/rdf.js.map +1 -1
- package/dist/input/xml.d.ts +2 -2
- package/dist/input/xml.d.ts.map +1 -1
- package/dist/input/xml.js +28 -92
- package/dist/input/xml.js.map +1 -1
- package/dist/output/alephseq.js +1 -1
- package/dist/output/json.js +1 -1
- package/dist/output/jsonl.js +2 -2
- package/dist/output/jsonl.js.map +1 -1
- package/dist/output/rdf.d.ts.map +1 -1
- package/dist/output/rdf.js +2 -3
- package/dist/output/rdf.js.map +1 -1
- package/dist/output/xml.js +2 -2
- package/dist/s3stream.js.map +1 -1
- package/dist/util/stream_helpers.d.ts +8 -0
- package/dist/util/stream_helpers.d.ts.map +1 -1
- package/dist/util/stream_helpers.js +77 -3
- package/dist/util/stream_helpers.js.map +1 -1
- package/package.json +5 -3
- package/dist/transform/json.d.ts +0 -3
- package/dist/transform/json.d.ts.map +0 -1
- package/dist/transform/json.js +0 -32
- package/dist/transform/json.js.map +0 -1
- package/dist/transform/rdf.d.ts +0 -3
- package/dist/transform/rdf.d.ts.map +0 -1
- package/dist/transform/rdf.js +0 -370
- package/dist/transform/rdf.js.map +0 -1
package/README.md
CHANGED
|
@@ -113,7 +113,6 @@ Provide a fix file to the mapper. See examples:
|
|
|
113
113
|
- `npm run demo:jsonld`
|
|
114
114
|
- `npm run demo:n3`
|
|
115
115
|
- `npm run biblio:one`
|
|
116
|
-
- `npm run biblio:all`
|
|
117
116
|
|
|
118
117
|
### Writable (--out)
|
|
119
118
|
|
|
@@ -133,6 +132,7 @@ Default the logging format is a text format that is written to stderr. This logg
|
|
|
133
132
|
- `--log stdout` : write logs to the stdout
|
|
134
133
|
- `--log json+stdout` : write logs in a JSON format and to the stdout
|
|
135
134
|
|
|
136
|
-
### Compression (
|
|
135
|
+
### Compression (--z,--tar)
|
|
137
136
|
|
|
138
|
-
-
|
|
137
|
+
- `--z` : the input file is gzipped
|
|
138
|
+
- `--tar` : the input file is tarred
|
package/dist/index.js
CHANGED
|
@@ -12,8 +12,7 @@ import fs from 'fs';
|
|
|
12
12
|
import { s3ReaderStream, s3WriterStream } from './s3stream.js';
|
|
13
13
|
import dotenv from 'dotenv';
|
|
14
14
|
import { pipeline } from 'node:stream/promises';
|
|
15
|
-
import {
|
|
16
|
-
import { createCountableSkippedStream, createVerboseStream } from './util/stream_helpers.js';
|
|
15
|
+
import { createCountableSkippedStream, createUntarredStream, createUncompressedStream, createVerboseStream } from './util/stream_helpers.js';
|
|
17
16
|
program.version('0.1.0')
|
|
18
17
|
.argument('<file>')
|
|
19
18
|
.option('-c,--config <config>', 'config .env', path.join(process.cwd(), '.env'))
|
|
@@ -22,7 +21,8 @@ program.version('0.1.0')
|
|
|
22
21
|
.option('-m,--map <map>', 'data mapper', 'jsonata')
|
|
23
22
|
.option('--fix <what>', 'jsonata')
|
|
24
23
|
.option('-o,--out <file>', 'output file')
|
|
25
|
-
.option('
|
|
24
|
+
.option('--z', 'uncompress input')
|
|
25
|
+
.option('--tar', 'untar input')
|
|
26
26
|
.option('--count <num>', 'output only <num> records')
|
|
27
27
|
.option('--skip <num>', 'skip first <num> records')
|
|
28
28
|
.option('--key <keyfile>', 'private key file')
|
|
@@ -65,7 +65,6 @@ if (opts.trace) {
|
|
|
65
65
|
if (opts.config) {
|
|
66
66
|
dotenv.config({ path: opts.config, quiet: true });
|
|
67
67
|
}
|
|
68
|
-
logger.info('here');
|
|
69
68
|
main();
|
|
70
69
|
function configureDefaultLogger(output) {
|
|
71
70
|
log4js.configure({
|
|
@@ -152,35 +151,35 @@ async function main() {
|
|
|
152
151
|
else {
|
|
153
152
|
readableStream = fs.createReadStream(inputFile);
|
|
154
153
|
}
|
|
155
|
-
|
|
154
|
+
const stages = [readableStream];
|
|
156
155
|
if (opts.z) {
|
|
157
|
-
|
|
156
|
+
stages.push(createUncompressedStream());
|
|
158
157
|
}
|
|
159
|
-
|
|
160
|
-
|
|
158
|
+
if (opts.tar) {
|
|
159
|
+
stages.push(await createUntarredStream());
|
|
161
160
|
}
|
|
162
|
-
let objectStream;
|
|
163
161
|
if (opts.from) {
|
|
164
162
|
const mod = await loadPlugin(opts.from, 'input');
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
});
|
|
168
|
-
objectStream.on('error', (error) => {
|
|
163
|
+
const transformer = await mod.transform({ path: inputFile });
|
|
164
|
+
transformer.on('error', (error) => {
|
|
169
165
|
if (error.code === 'ERR_STREAM_PREMATURE_CLOSE') {
|
|
170
|
-
|
|
166
|
+
stages[0]?.destroy();
|
|
171
167
|
}
|
|
172
168
|
else {
|
|
173
169
|
logger.error("input stream processing error: ", error.message);
|
|
174
|
-
|
|
170
|
+
stages[0]?.destroy();
|
|
175
171
|
process.exitCode = 2;
|
|
176
172
|
}
|
|
177
173
|
});
|
|
174
|
+
transformer.on('finish', () => logger.debug('writeable finished'));
|
|
175
|
+
transformer.on('end', () => logger.debug('readable ended'));
|
|
176
|
+
transformer.on('close', () => logger.debug('stream closed'));
|
|
177
|
+
stages.push(transformer);
|
|
178
178
|
}
|
|
179
179
|
else {
|
|
180
180
|
console.error(`Need --from`);
|
|
181
181
|
process.exit(1);
|
|
182
182
|
}
|
|
183
|
-
const stages = [objectStream];
|
|
184
183
|
if (opts.count || opts.skip) {
|
|
185
184
|
stages.push(createCountableSkippedStream(opts.count, opts.skip));
|
|
186
185
|
}
|
|
@@ -279,13 +278,4 @@ function makeSftpConfig(inputFile, opts) {
|
|
|
279
278
|
}
|
|
280
279
|
return config;
|
|
281
280
|
}
|
|
282
|
-
function getDecompressedStream(sourceStream) {
|
|
283
|
-
const gunzip = createGunzip();
|
|
284
|
-
// Forward errors from the source to the gunzip stream
|
|
285
|
-
sourceStream.on('error', (err) => {
|
|
286
|
-
logger.error("gunzip error: ", err.message);
|
|
287
|
-
gunzip.emit('error', err);
|
|
288
|
-
});
|
|
289
|
-
return sourceStream.pipe(gunzip);
|
|
290
|
-
}
|
|
291
281
|
//# sourceMappingURL=index.js.map
|
package/dist/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";AAEA,OAAO,MAAM,MAAM,QAAQ,CAAC;AAC5B,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AAChD,OAAO,EAAE,cAAc,EAAG,eAAe,EAAG,cAAc,EAAoB,MAAM,iBAAiB,CAAC;AACtG,OAAO,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AACjD,OAAO,EAAE,QAAQ,EAAE,MAAM,QAAQ,CAAC;AAClC,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AAEzC,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,MAAM,IAAI,CAAC;AACpB,OAAO,EAAE,cAAc,EAAE,cAAc,EAAE,MAAM,eAAe,CAAC;AAC/D,OAAO,MAAM,MAAM,QAAQ,CAAC;AAC5B,OAAO,EAAE,QAAQ,EAAE,MAAM,sBAAsB,CAAC;AAChD,OAAO,
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";AAEA,OAAO,MAAM,MAAM,QAAQ,CAAC;AAC5B,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AAChD,OAAO,EAAE,cAAc,EAAG,eAAe,EAAG,cAAc,EAAoB,MAAM,iBAAiB,CAAC;AACtG,OAAO,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AACjD,OAAO,EAAE,QAAQ,EAAE,MAAM,QAAQ,CAAC;AAClC,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AAEzC,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,MAAM,IAAI,CAAC;AACpB,OAAO,EAAE,cAAc,EAAE,cAAc,EAAE,MAAM,eAAe,CAAC;AAC/D,OAAO,MAAM,MAAM,QAAQ,CAAC;AAC5B,OAAO,EAAE,QAAQ,EAAE,MAAM,sBAAsB,CAAC;AAChD,OAAO,EACH,4BAA4B,EAC5B,oBAAoB,EACpB,wBAAwB,EACxB,mBAAmB,EACtB,MAAM,0BAA0B,CAAC;AAElC,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC;KACnB,QAAQ,CAAC,QAAQ,CAAC;KAClB,MAAM,CAAC,sBAAsB,EAAC,aAAa,EAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,MAAM,CAAC,CAAC;KAC7E,MAAM,CAAC,kBAAkB,EAAC,YAAY,EAAC,KAAK,CAAC;KAC7C,MAAM,CAAC,kBAAkB,EAAC,aAAa,EAAC,MAAM,CAAC;KAC/C,MAAM,CAAC,gBAAgB,EAAC,aAAa,EAAC,SAAS,CAAC;KAChD,MAAM,CAAC,cAAc,EAAC,SAAS,CAAC;KAChC,MAAM,CAAC,iBAAiB,EAAC,aAAa,CAAC;KACvC,MAAM,CAAC,KAAK,EAAC,kBAAkB,CAAC;KAChC,MAAM,CAAC,OAAO,EAAC,aAAa,CAAC;KAC7B,MAAM,CAAC,eAAe,EAAE,2BAA2B,CAAC;KACpD,MAAM,CAAC,cAAc,EAAE,0BAA0B,CAAC;KAClD,MAAM,CAAC,iBAAiB,EAAE,kBAAkB,CAAC;KAC7C,MAAM,CAAC,iBAAiB,EAAC,kCAAkC,CAAC;KAC5D,MAAM,CAAC,gBAAgB,EAAC,gBAAgB,CAAC;KACzC,MAAM,CAAC,QAAQ,EAAC,2BAA2B,CAAC;KAC5C,MAAM,CAAC,SAAS,EAAC,gCAAgC,CAAC;KAClD,MAAM,CAAC,SAAS,EAAC,qCAAqC,CAAC,CAAC;AAE7D,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;AAE5B,MAAM,IAAI,GAAK,OAAO,CAAC,IAAI,EAAE,CAAC;AAE9B,IAAI,IAAI,CAAC,GAAG,EAAE,CAAC;IACX,IAAI,MAAM,GAAG,QAAQ,CAAC;IACtB,IAAI,IAAI,GAAG,CAAC,CAAC;IAEb,IAAI,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;QAClC,MAAM,GAAG,QAAQ,CAAC;IACtB,CAAC;IACD,IAAI,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC;QAChC,IAAI,GAAG,CAAC,CAAC;IACb,CAAC;IAED,IAAI,IAAI,EAAE,CAAC;QACP,mBAAmB,CAAC,MAAM,CAAC,CAAC;IAChC,CAAC;SACI,CAAC;QACF,sBAAsB,CAAC,MAAM,CAAC,CAAC;IACnC,CAAC;AACL,CAAC;KACI,CAAC;IACF,sBAAsB,CAAC,QAAQ,CAAC,CAAC;AACrC,CAAC;AAED,MAAM,MAAM,GAAG,MAAM,CAAC,SAAS,EAAE,CAAC;AAElC,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;IACZ,MAAM,CAAC,KAAK,GAAG,MAAM,CAAC;AAC1B,CAAC;AAED,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;IACb,MAAM,CAAC,KAAK,GAAG,OAAO,CAAC;AAC3B,CAAC;AAED,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;IACb,MAAM,CAAC,KAAK,GAAG,OAAO,CAAC;AAC3B,CAAC;AAED,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;IACd,MAAM,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,MAAM,EAAG,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;AACvD,CAAC;AAED,IAAI,EAAE,CAAC;AAEP,SAAS,sBAAsB,CAAC,MAAc;IAC1C,MAAM,CAAC,SAAS,CAAC;QACb,SAAS,EAAE;YACP,GAAG,EAAE;gBACD,IAAI,EAAE,MAAM;gBACZ,MAAM,EAAE;oBACJ,IAAI,EAAE,SAAS;oBACf,OAAO,EAAE,oBAAoB;iBAChC;aACJ;SACJ;QACD,UAAU,EAAE;YACR,OAAO,EAAE,EAAE,SAAS,EAAE,CAAC,KAAK,CAAC,EAAE,KAAK,EAAE,KAAK,EAAG,eAAe,EAAE,IAAI,EAAE;SACxE;KACJ,CAAC,CAAC;AACP,CAAC;AAED,SAAS,mBAAmB,CAAC,MAAc;IACvC,MAAM,CAAC,SAAS,CAAC,cAAc,EAAE,CAAC,MAAM,EAAE,EAAE;QACxC,OAAO,CAAC,QAAQ,EAAE,EAAE;YAChB,OAAO,IAAI,CAAC,SAAS,CAAC;gBACtB,SAAS,EAAE,QAAQ,CAAC,SAAS,EAAG,gBAAgB;gBAChD,KAAK,EAAE,QAAQ,CAAC,KAAK,CAAC,QAAQ,EAAE,gBAAgB;gBAChD,QAAQ,EAAE,QAAQ,CAAC,YAAY,EAAC,gBAAgB;gBAChD,OAAO,EAAE,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,gBAAgB;gBAClD,OAAO,EAAE,QAAQ,CAAC,OAAO,EAAO,yBAAyB;gBACzD,GAAG,EAAE,QAAQ,CAAC,GAAG,CAAe,gBAAgB;aAC/C,CAAC,CAAC;QACP,CAAC,CAAC;IACN,CAAC,CAAC,CAAC;IAEH,MAAM,CAAC,SAAS,CAAC;QACb,SAAS,EAAE;YACP,GAAG,EAAE;gBACD,IAAI,EAAE,MAAM;gBACZ,MAAM,EAAE;oBACJ,IAAI,EAAE,cAAc;iBACvB;aACJ;SACJ;QACD,UAAU,EAAE;YACR,OAAO,EAAE,EAAE,SAAS,EAAE,CAAC,KAAK,CAAC,EAAE,KAAK,EAAE,KAAK,EAAG,eAAe,EAAE,IAAI,EAAE;SACxE;KACJ,CAAC,CAAC;AACP,CAAC;AAED,KAAK,UAAU,IAAI;IACf,IAAI,CAAC;QACD,MAAM,GAAG,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAE5B,IAAI,CAAE,GAAG,EAAE,CAAC;YACR,OAAO,CAAC,KAAK,CAAC,oBAAoB,CAAC,CAAC;YACpC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACpB,CAAC;QAED,IAAI,SAAe,CAAC;QAEpB,IAAI,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;YACrB,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,GAAG,CAAC,CAAC;YAClD,SAAS,GAAG,aAAa,CAAC,QAAQ,CAAC,CAAC;QACxC,CAAC;aACI,CAAC;YACF,SAAS,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;QAC7B,CAAC;QAED,MAAM,CAAC,IAAI,CAAC,UAAU,WAAW,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;QAEhD,IAAI,cAAyB,CAAC;QAE9B,IAAI,SAAS,CAAC,QAAQ,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC;YACxC,cAAc,GAAG,MAAM,cAAc,CAAC,SAAS,CAAC,QAAQ,EAAE,CAAC,CAAC;QAChE,CAAC;aACI,IAAI,SAAS,CAAC,QAAQ,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;YAC3C,cAAc,GAAG,MAAM,cAAc,CAAC,SAAS,EAAC,EAAE,CAAC,CAAC;QACxD,CAAC;aACI,IAAI,SAAS,CAAC,QAAQ,KAAK,OAAO,EAAE,CAAC;YACtC,MAAM,MAAM,GAAG,cAAc,CAAC,SAAS,EAAC,IAAI,CAAC,CAAC;YAC9C,IAAI,UAAU,CAAC;YAEf,IAAI,SAAS,CAAC,QAAQ,CAAC,KAAK,CAAC,gBAAgB,CAAC,EAAE,CAAC;gBAC7C,MAAM,SAAS,GAAG,SAAS,CAAC,QAAQ,CAAC,OAAO,CAAC,aAAa,EAAC,EAAE,CAAC,CAAC;gBAC/D,MAAM,SAAS,GAAG,SAAS,CAAC,QAAQ,CAAC,OAAO,CAAC,cAAc,EAAC,EAAE,CAAC,CAAC;gBAChE,UAAU,GAAG,MAAM,cAAc,CAAC,MAAM,EAAC,SAAS,EAAC,SAAS,CAAC,CAAC;YAClE,CAAC;iBACI,CAAC;gBACF,UAAU,GAAG,SAAS,CAAC,QAAQ,CAAC;YACpC,CAAC;YAED,cAAc,GAAG,MAAM,cAAc,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC;QAC9D,CAAC;aACI,IAAI,SAAS,CAAC,QAAQ,KAAK,QAAQ,EAAE,CAAC;YACvC,cAAc,GAAG,OAAO,CAAC,KAAK,CAAC;QACnC,CAAC;aACI,CAAC;YACF,cAAc,GAAG,EAAE,CAAC,gBAAgB,CAAC,SAAS,CAAC,CAAC;QACpD,CAAC;QAED,MAAM,MAAM,GAAwC,CAAC,cAAc,CAAC,CAAC;QAErE,IAAI,IAAI,CAAC,CAAC,EAAE,CAAC;YACT,MAAM,CAAC,IAAI,CAAC,wBAAwB,EAAE,CAAC,CAAC;QAC5C,CAAC;QAED,IAAI,IAAI,CAAC,GAAG,EAAE,CAAC;YACX,MAAM,CAAC,IAAI,CAAC,MAAM,oBAAoB,EAAE,CAAC,CAAC;QAC9C,CAAC;QAED,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;YACZ,MAAM,GAAG,GAAG,MAAM,UAAU,CAAC,IAAI,CAAC,IAAI,EAAC,OAAO,CAAC,CAAC;YAChD,MAAM,WAAW,GAAG,MAAM,GAAG,CAAC,SAAS,CAAC,EAAC,IAAI,EAAE,SAAS,EAAC,CAAC,CAAC;YAC3D,WAAW,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,KAAU,EAAE,EAAE;gBACnC,IAAI,KAAK,CAAC,IAAI,KAAK,4BAA4B,EAAE,CAAC;oBAC9C,MAAM,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,CAAC;gBACzB,CAAC;qBACI,CAAC;oBACF,MAAM,CAAC,KAAK,CAAC,iCAAiC,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC;oBAC/D,MAAM,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,CAAC;oBACrB,OAAO,CAAC,QAAQ,GAAG,CAAC,CAAC;gBACzB,CAAC;YACL,CAAC,CAAC,CAAC;YACH,WAAW,CAAC,EAAE,CAAC,QAAQ,EAAE,GAAG,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,oBAAoB,CAAC,CAAC,CAAC;YACnE,WAAW,CAAC,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAC,CAAC;YAC5D,WAAW,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC,CAAC;YAC7D,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QAC7B,CAAC;aACI,CAAC;YACF,OAAO,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC;YAC7B,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACpB,CAAC;QAED,IAAI,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;YAC1B,MAAM,CAAC,IAAI,CACP,4BAA4B,CACxB,IAAI,CAAC,KAAK,EACV,IAAI,CAAC,IAAI,CACZ,CACJ,CAAC;QACN,CAAC;QAED,MAAM,CAAC,IAAI,CAAC,mBAAmB,EAAE,CAAC,CAAC;QAEnC,IAAI,IAAI,CAAC,GAAG,EAAE,CAAC;YACX,MAAM,GAAG,GAAG,MAAM,UAAU,CAAC,IAAI,CAAC,GAAG,EAAC,WAAW,CAAC,CAAC;YACnD,MAAM,WAAW,GAAe,MAAM,GAAG,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAC9D,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QAC7B,CAAC;QAED,IAAI,SAAoB,CAAC;QAEzB,IAAI,IAAI,CAAC,GAAG,KAAK,OAAO,EAAE,CAAC;YACvB,SAAS,GAAG,IAAI,YAAY,CAAC,EAAE,OAAO,EAAE,GAAG,EAAE,CAAC,CAAC;QACnD,CAAC;aACI,IAAI,IAAI,CAAC,GAAG,KAAK,SAAS,EAAE,CAAC;YAC9B,SAAS,GAAG,IAAI,YAAY,CAAC,EAAE,mBAAmB,EAAE,CAAC,EAAE,CAAC,CAAC;QAC7D,CAAC;aACI,IAAI,IAAI,CAAC,GAAG,EAAE,CAAC;YAChB,IAAI,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC;gBAC9B,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBAE9B,IAAI,OAAO,CAAC,GAAG,CAAC,aAAa,EAAE,CAAC;oBAC5B,GAAG,CAAC,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,aAAa,CAAC;gBAC7C,CAAC;gBAED,IAAI,OAAO,CAAC,GAAG,CAAC,aAAa,EAAE,CAAC;oBAC5B,GAAG,CAAC,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,aAAa,CAAC;gBAC7C,CAAC;gBAED,MAAM,MAAM,GAAG,cAAc,CAAC,GAAG,EAAC,IAAI,CAAC,CAAC;gBACxC,MAAM,CAAC,IAAI,CAAC,OAAO,WAAW,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;gBACvC,SAAS,GAAG,MAAM,eAAe,CAAC,GAAG,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;YACxD,CAAC;iBACI,IAAI,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;gBACjC,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBAE9B,IAAI,OAAO,CAAC,GAAG,CAAC,aAAa,EAAE,CAAC;oBAC5B,GAAG,CAAC,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,aAAa,CAAC;gBAC7C,CAAC;gBAED,IAAI,OAAO,CAAC,GAAG,CAAC,aAAa,EAAE,CAAC;oBAC5B,GAAG,CAAC,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,aAAa,CAAC;gBAC7C,CAAC;gBAED,MAAM,CAAC,IAAI,CAAC,OAAO,WAAW,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;gBACvC,SAAS,GAAG,MAAM,cAAc,CAAC,GAAG,EAAC,EAAE,CAAC,CAAC;YAC7C,CAAC;iBACI,CAAC;gBACF,SAAS,GAAG,EAAE,CAAC,iBAAiB,CAAC,IAAI,CAAC,GAAG,EAAE,EAAE,QAAQ,EAAE,OAAO,EAAC,CAAC,CAAC;YACrE,CAAC;QACL,CAAC;aACI,CAAC;YACF,SAAS,GAAG,OAAO,CAAC,MAAM,CAAC;QAC/B,CAAC;QAED,IAAI,IAAI,CAAC,EAAE,EAAE,CAAC;YACV,MAAM,GAAG,GAAG,MAAM,UAAU,CAAC,IAAI,CAAC,EAAE,EAAC,QAAQ,CAAC,CAAC;YAC/C,MAAM,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,SAAS,EAAE,CAAC,CAAC;YACnC,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YACvB,IAAI,CAAC;gBACD,MAAM,QAAQ,CAAC,MAAM,CAAC,CAAC;gBACvB,MAAM,CAAC,IAAI,CAAC,2BAA2B,CAAC,CAAC;YAC7C,CAAC;YACD,OAAO,GAAS,EAAE,CAAC;gBACf,IAAI,GAAG,CAAC,IAAI,KAAK,4BAA4B,EAAE,CAAC;oBAC5C,MAAM,CAAC,IAAI,CAAC,2BAA2B,CAAC,CAAC;gBAC7C,CAAC;qBAAM,CAAC;oBACJ,MAAM,CAAC,KAAK,CAAC,iBAAiB,EAAE,GAAG,CAAC,CAAC;oBACrC,OAAO,CAAC,QAAQ,GAAG,CAAC,CAAC;gBACzB,CAAC;YACL,CAAC;QACL,CAAC;IACL,CAAC;IACD,OAAO,CAAC,EAAE,CAAC;QACP,MAAM,CAAC,KAAK,CAAC,yBAAyB,CAAC,EAAE,CAAC,CAAC;QAC3C,OAAO,CAAC,QAAQ,GAAG,CAAC,CAAC;IACzB,CAAC;AACL,CAAC;AAED,SAAS,WAAW,CAAC,GAAQ;IACzB,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,mCAAmC;IACtE,OAAO,CAAC,QAAQ,GAAG,KAAK,CAAC;IACzB,OAAO,CAAC,QAAQ,GAAG,KAAK,CAAC;IACzB,OAAO,OAAO,CAAC;AACnB,CAAC;AAED,SAAS,cAAc,CAAC,SAAc,EAAE,IAAS;IAC7C,IAAI,UAAU,GAAwB,SAAS,CAAC;IAEhD,IAAI,IAAI,CAAC,GAAG,EAAE,CAAC;QACX,UAAU,GAAG,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,GAAG,EAAC,EAAE,QAAQ,EAAE,OAAO,EAAC,CAAC,CAAC;IAChE,CAAC;SACI,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;QACnB,UAAU,GAAG,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAC1C,CAAC;IAED,IAAI,MAAM,GAAe;QACrB,IAAI,EAAE,SAAS,CAAC,QAAQ;QACxB,IAAI,EAAE,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,EAAE;QAClC,QAAQ,EAAE,SAAS,CAAC,QAAQ;KAC/B,CAAC;IAEF,IAAI,SAAS,CAAC,QAAQ,EAAE,CAAC;QAAC,MAAM,CAAC,QAAQ,GAAG,SAAS,CAAC,QAAQ,CAAA;IAAC,CAAC;IAChE,IAAI,UAAU,EAAE,CAAC;QAAC,MAAM,CAAC,UAAU,GAAG,UAAU,CAAA;IAAA,CAAC;IAEjD,OAAO,MAAM,CAAC;AAClB,CAAC"}
|
package/dist/input/alephseq.d.ts
CHANGED
|
@@ -1,3 +1,3 @@
|
|
|
1
|
-
import {
|
|
2
|
-
export declare function
|
|
1
|
+
import { Transform } from "stream";
|
|
2
|
+
export declare function transform(_opts: any): Promise<Transform>;
|
|
3
3
|
//# sourceMappingURL=alephseq.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"alephseq.d.ts","sourceRoot":"","sources":["../../src/input/alephseq.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,
|
|
1
|
+
{"version":3,"file":"alephseq.d.ts","sourceRoot":"","sources":["../../src/input/alephseq.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAA0B,MAAM,QAAQ,CAAC;AAK3D,wBAAsB,SAAS,CAAC,KAAK,EAAE,GAAG,GAAG,OAAO,CAAC,SAAS,CAAC,CA8D9D"}
|
package/dist/input/alephseq.js
CHANGED
|
@@ -1,86 +1,56 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import * as readline from 'node:readline';
|
|
1
|
+
import { Transform } from "stream";
|
|
3
2
|
import log4js from 'log4js';
|
|
4
3
|
const logger = log4js.getLogger();
|
|
5
|
-
export async function
|
|
4
|
+
export async function transform(_opts) {
|
|
6
5
|
let recordNum = 0;
|
|
7
|
-
let hasError = false;
|
|
8
|
-
const rl = readline.createInterface({ input: stream, crlfDelay: Infinity });
|
|
9
|
-
let sourcePaused = false;
|
|
10
|
-
const readableStream = new Readable({
|
|
11
|
-
read() {
|
|
12
|
-
if (sourcePaused) {
|
|
13
|
-
logger.debug("backpressure off");
|
|
14
|
-
rl.resume();
|
|
15
|
-
sourcePaused = false;
|
|
16
|
-
}
|
|
17
|
-
},
|
|
18
|
-
destroy() {
|
|
19
|
-
stream.destroy();
|
|
20
|
-
},
|
|
21
|
-
objectMode: true
|
|
22
|
-
});
|
|
23
6
|
let rec = [];
|
|
24
7
|
let previd = "";
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
8
|
+
let tail = "";
|
|
9
|
+
return new Transform({
|
|
10
|
+
objectMode: true,
|
|
11
|
+
transform(chunk, _encoding, callback) {
|
|
12
|
+
const data = tail + chunk.toString();
|
|
13
|
+
const lines = data.split(/\r?\n/);
|
|
14
|
+
tail = lines.pop() || "";
|
|
15
|
+
for (const line of lines) {
|
|
16
|
+
if (line.length === 0)
|
|
17
|
+
continue;
|
|
18
|
+
if (!line.match(/^\w+\s[\x20-\x7E]{5}\sL\s.*/u)) {
|
|
19
|
+
const err = new Error(`syntax error in record ${recordNum + 1}`);
|
|
20
|
+
logger.error(err.message);
|
|
21
|
+
return callback(err);
|
|
22
|
+
}
|
|
23
|
+
const [id, ...rest] = line.split(" ");
|
|
24
|
+
const lineData = rest.join(" ");
|
|
25
|
+
if (previd && previd !== id) {
|
|
26
|
+
this.push({ record: rec });
|
|
27
|
+
rec = [];
|
|
28
|
+
recordNum++;
|
|
29
|
+
}
|
|
30
|
+
const tag = lineData?.substring(0, 3);
|
|
31
|
+
const ind1 = lineData?.substring(3, 4);
|
|
32
|
+
const ind2 = lineData?.substring(4, 5);
|
|
33
|
+
const sf = lineData?.substring(8);
|
|
34
|
+
const parts = sf.split(/\$\$(.)/);
|
|
35
|
+
if (tag === 'FMT' || tag === 'LDR' || tag.startsWith("00")) {
|
|
36
|
+
rec.push([tag, ind1, ind2, "_", ...parts]);
|
|
37
|
+
}
|
|
38
|
+
else {
|
|
39
|
+
rec.push([tag, ind1, ind2, ...parts.slice(1)]);
|
|
40
|
+
}
|
|
41
|
+
previd = id;
|
|
50
42
|
}
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
tag, ind1, ind2
|
|
62
|
-
].concat(["_"].concat(parts)));
|
|
63
|
-
}
|
|
64
|
-
else {
|
|
65
|
-
rec.push([
|
|
66
|
-
tag, ind1, ind2
|
|
67
|
-
].concat(parts.slice(1)));
|
|
43
|
+
callback();
|
|
44
|
+
},
|
|
45
|
+
flush(callback) {
|
|
46
|
+
if (tail) {
|
|
47
|
+
logger.warn("ignoring partial chunk of data: ", tail);
|
|
48
|
+
}
|
|
49
|
+
if (rec.length > 0) {
|
|
50
|
+
this.push({ record: rec });
|
|
51
|
+
}
|
|
52
|
+
callback();
|
|
68
53
|
}
|
|
69
|
-
previd = id;
|
|
70
|
-
});
|
|
71
|
-
rl.on('error', (error) => {
|
|
72
|
-
if (hasError)
|
|
73
|
-
return;
|
|
74
|
-
logger.error(`readline error ${error}`);
|
|
75
|
-
});
|
|
76
|
-
rl.on('close', () => {
|
|
77
|
-
if (hasError)
|
|
78
|
-
return;
|
|
79
|
-
readableStream.push({
|
|
80
|
-
record: rec
|
|
81
|
-
});
|
|
82
|
-
readableStream.push(null);
|
|
83
54
|
});
|
|
84
|
-
return readableStream;
|
|
85
55
|
}
|
|
86
56
|
//# sourceMappingURL=alephseq.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"alephseq.js","sourceRoot":"","sources":["../../src/input/alephseq.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,
|
|
1
|
+
{"version":3,"file":"alephseq.js","sourceRoot":"","sources":["../../src/input/alephseq.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAA0B,MAAM,QAAQ,CAAC;AAC3D,OAAO,MAAM,MAAM,QAAQ,CAAC;AAE5B,MAAM,MAAM,GAAG,MAAM,CAAC,SAAS,EAAE,CAAC;AAElC,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,KAAU;IACtC,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,IAAI,GAAG,GAAe,EAAE,CAAC;IACzB,IAAI,MAAM,GAAW,EAAE,CAAC;IACxB,IAAI,IAAI,GAAG,EAAE,CAAC;IAEd,OAAO,IAAI,SAAS,CAAC;QACjB,UAAU,EAAE,IAAI;QAEhB,SAAS,CAAC,KAAU,EAAE,SAAiB,EAAE,QAA2B;YAChE,MAAM,IAAI,GAAG,IAAI,GAAG,KAAK,CAAC,QAAQ,EAAE,CAAC;YACrC,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;YAElC,IAAI,GAAG,KAAK,CAAC,GAAG,EAAE,IAAI,EAAE,CAAC;YAEzB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;gBACvB,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;oBAAE,SAAS;gBAEhC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,8BAA8B,CAAC,EAAE,CAAC;oBAC9C,MAAM,GAAG,GAAG,IAAI,KAAK,CAAC,0BAA0B,SAAS,GAAG,CAAC,EAAE,CAAC,CAAC;oBACjE,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;oBAC1B,OAAO,QAAQ,CAAC,GAAG,CAAC,CAAC;gBACzB,CAAC;gBAED,MAAM,CAAC,EAAE,EAAE,GAAG,IAAI,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;gBACtC,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBAEhC,IAAI,MAAM,IAAI,MAAM,KAAK,EAAE,EAAE,CAAC;oBAC1B,IAAI,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC,CAAC;oBAC3B,GAAG,GAAG,EAAE,CAAC;oBACT,SAAS,EAAE,CAAC;gBAChB,CAAC;gBAED,MAAM,GAAG,GAAI,QAAQ,EAAE,SAAS,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;gBACvC,MAAM,IAAI,GAAG,QAAQ,EAAE,SAAS,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;gBACvC,MAAM,IAAI,GAAG,QAAQ,EAAE,SAAS,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;gBACvC,MAAM,EAAE,GAAK,QAAQ,EAAE,SAAS,CAAC,CAAC,CAAC,CAAC;gBACpC,MAAM,KAAK,GAAG,EAAE,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;gBAElC,IAAI,GAAG,KAAK,KAAK,IAAI,GAAG,KAAK,KAAK,IAAI,GAAG,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;oBACzD,GAAG,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,IAAI,EAAE,GAAG,EAAE,GAAG,KAAK,CAAC,CAAC,CAAC;gBAC/C,CAAC;qBAAM,CAAC;oBACJ,GAAG,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,IAAI,EAAE,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;gBACnD,CAAC;gBAED,MAAM,GAAG,EAAG,CAAC;YACjB,CAAC;YAED,QAAQ,EAAE,CAAC;QACf,CAAC;QAED,KAAK,CAAC,QAA2B;YAC7B,IAAI,IAAI,EAAE,CAAC;gBACP,MAAM,CAAC,IAAI,CAAC,kCAAkC,EAAE,IAAI,CAAC,CAAC;YAC1D,CAAC;YAED,IAAI,GAAG,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACjB,IAAI,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC,CAAC;YAC/B,CAAC;YACD,QAAQ,EAAE,CAAC;QACf,CAAC;KACJ,CAAC,CAAC;AACP,CAAC"}
|
package/dist/input/json.d.ts
CHANGED
|
@@ -1,3 +1,3 @@
|
|
|
1
|
-
import {
|
|
2
|
-
export declare function
|
|
1
|
+
import { Transform } from "stream";
|
|
2
|
+
export declare function transform(_opts: any): Promise<Transform>;
|
|
3
3
|
//# sourceMappingURL=json.d.ts.map
|
package/dist/input/json.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"json.d.ts","sourceRoot":"","sources":["../../src/input/json.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,
|
|
1
|
+
{"version":3,"file":"json.d.ts","sourceRoot":"","sources":["../../src/input/json.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAA0B,MAAM,QAAQ,CAAC;AAG3D,wBAAsB,SAAS,CAAC,KAAK,EAAE,GAAG,GAAG,OAAO,CAAC,SAAS,CAAC,CAsB9D"}
|
package/dist/input/json.js
CHANGED
|
@@ -1,34 +1,22 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { Transform } from "stream";
|
|
2
2
|
import streamArray from "stream-json/streamers/StreamArray.js";
|
|
3
|
-
|
|
4
|
-
const
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
if (sourcePaused) {
|
|
11
|
-
logger.debug("backpressure off");
|
|
12
|
-
pipeline.resume();
|
|
13
|
-
sourcePaused = false;
|
|
14
|
-
}
|
|
3
|
+
export async function transform(_opts) {
|
|
4
|
+
const jsonParser = streamArray.withParser();
|
|
5
|
+
const transformer = new Transform({
|
|
6
|
+
objectMode: true,
|
|
7
|
+
transform(chunk, encoding, callback) {
|
|
8
|
+
jsonParser.write(chunk, encoding);
|
|
9
|
+
callback();
|
|
15
10
|
},
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
objectMode: true
|
|
20
|
-
});
|
|
21
|
-
pipeline.on('data', (data) => {
|
|
22
|
-
const ok = readableStream.push(data.value);
|
|
23
|
-
if (!ok) {
|
|
24
|
-
logger.debug("backpressure on");
|
|
25
|
-
pipeline.pause();
|
|
26
|
-
sourcePaused = true;
|
|
11
|
+
flush(callback) {
|
|
12
|
+
jsonParser.end();
|
|
13
|
+
callback();
|
|
27
14
|
}
|
|
28
15
|
});
|
|
29
|
-
|
|
30
|
-
|
|
16
|
+
jsonParser.on('data', (data) => {
|
|
17
|
+
transformer.push(data.value);
|
|
31
18
|
});
|
|
32
|
-
|
|
19
|
+
jsonParser.on('error', (err) => transformer.destroy(err));
|
|
20
|
+
return transformer;
|
|
33
21
|
}
|
|
34
22
|
//# sourceMappingURL=json.js.map
|
package/dist/input/json.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"json.js","sourceRoot":"","sources":["../../src/input/json.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,
|
|
1
|
+
{"version":3,"file":"json.js","sourceRoot":"","sources":["../../src/input/json.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAA0B,MAAM,QAAQ,CAAC;AAC3D,OAAO,WAAW,MAAM,sCAAsC,CAAC;AAE/D,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,KAAU;IACtC,MAAM,UAAU,GAAG,WAAW,CAAC,UAAU,EAAE,CAAC;IAE5C,MAAM,WAAW,GAAG,IAAI,SAAS,CAAC;QAC9B,UAAU,EAAE,IAAI;QAChB,SAAS,CAAC,KAAU,EAAE,QAAgB,EAAE,QAA2B;YAC/D,UAAU,CAAC,KAAK,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;YAClC,QAAQ,EAAE,CAAC;QACf,CAAC;QACD,KAAK,CAAC,QAAQ;YACV,UAAU,CAAC,GAAG,EAAE,CAAC;YACjB,QAAQ,EAAE,CAAC;QACf,CAAC;KACJ,CAAC,CAAC;IAEH,UAAU,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,IAAS,EAAE,EAAE;QAChC,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACjC,CAAC,CAAC,CAAC;IAEH,UAAU,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAQ,EAAE,EAAE,CAAC,WAAW,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC;IAE/D,OAAO,WAAW,CAAC;AACvB,CAAC"}
|
package/dist/input/jsonl.d.ts
CHANGED
|
@@ -1,3 +1,3 @@
|
|
|
1
|
-
import {
|
|
2
|
-
export declare function
|
|
1
|
+
import { Transform } from "stream";
|
|
2
|
+
export declare function transform(_opts: any): Promise<Transform>;
|
|
3
3
|
//# sourceMappingURL=jsonl.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"jsonl.d.ts","sourceRoot":"","sources":["../../src/input/jsonl.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,
|
|
1
|
+
{"version":3,"file":"jsonl.d.ts","sourceRoot":"","sources":["../../src/input/jsonl.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAA0B,MAAM,QAAQ,CAAC;AAK3D,wBAAsB,SAAS,CAAC,KAAK,EAAE,GAAG,GAAG,OAAO,CAAC,SAAS,CAAC,CA+B9D"}
|
package/dist/input/jsonl.js
CHANGED
|
@@ -1,56 +1,37 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import * as readline from 'node:readline';
|
|
1
|
+
import { Transform } from "stream";
|
|
3
2
|
import log4js from 'log4js';
|
|
4
3
|
const logger = log4js.getLogger();
|
|
5
|
-
export async function
|
|
4
|
+
export async function transform(_opts) {
|
|
6
5
|
let recordNum = 0;
|
|
7
|
-
let
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
6
|
+
let tail = "";
|
|
7
|
+
return new Transform({
|
|
8
|
+
objectMode: true,
|
|
9
|
+
transform(chunk, _encoding, callback) {
|
|
10
|
+
const lines = (tail + chunk.toString()).split(/\r?\n/);
|
|
11
|
+
tail = lines.pop() || "";
|
|
12
|
+
for (const line of lines) {
|
|
13
|
+
if (!line.trim())
|
|
14
|
+
continue;
|
|
15
|
+
try {
|
|
16
|
+
this.push(JSON.parse(line));
|
|
17
|
+
recordNum++;
|
|
18
|
+
}
|
|
19
|
+
catch (error) {
|
|
20
|
+
logger.error(`JSON parse error at line ${recordNum + 1}: ${error}`);
|
|
21
|
+
return callback(error instanceof Error ? error : new Error(String(error)));
|
|
22
|
+
}
|
|
16
23
|
}
|
|
24
|
+
callback();
|
|
17
25
|
},
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
if (hasError)
|
|
25
|
-
return;
|
|
26
|
-
recordNum++;
|
|
27
|
-
try {
|
|
28
|
-
const ok = readableStream.push(JSON.parse(line));
|
|
29
|
-
if (!ok) {
|
|
30
|
-
logger.debug("backpressure on");
|
|
31
|
-
rl.pause();
|
|
32
|
-
sourcePaused = true;
|
|
26
|
+
flush(callback) {
|
|
27
|
+
if (tail.trim()) {
|
|
28
|
+
try {
|
|
29
|
+
this.push(JSON.parse(tail));
|
|
30
|
+
}
|
|
31
|
+
catch (e) { /* ignore trailing whitespace */ }
|
|
33
32
|
}
|
|
34
|
-
|
|
35
|
-
}
|
|
36
|
-
catch (error) {
|
|
37
|
-
hasError = true;
|
|
38
|
-
logger.error(`JSON parse error at line ${recordNum + 1}: ${error}`);
|
|
39
|
-
stream.destroy();
|
|
40
|
-
rl.close();
|
|
41
|
-
readableStream.destroy(error instanceof Error ? error : new Error(String(error)));
|
|
33
|
+
callback();
|
|
42
34
|
}
|
|
43
35
|
});
|
|
44
|
-
rl.on('error', (error) => {
|
|
45
|
-
if (hasError)
|
|
46
|
-
return;
|
|
47
|
-
logger.error(`readline error ${error}`);
|
|
48
|
-
});
|
|
49
|
-
rl.on('close', () => {
|
|
50
|
-
if (hasError)
|
|
51
|
-
return;
|
|
52
|
-
readableStream.push(null);
|
|
53
|
-
});
|
|
54
|
-
return readableStream;
|
|
55
36
|
}
|
|
56
37
|
//# sourceMappingURL=jsonl.js.map
|
package/dist/input/jsonl.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"jsonl.js","sourceRoot":"","sources":["../../src/input/jsonl.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,
|
|
1
|
+
{"version":3,"file":"jsonl.js","sourceRoot":"","sources":["../../src/input/jsonl.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAA0B,MAAM,QAAQ,CAAC;AAC3D,OAAO,MAAM,MAAM,QAAQ,CAAC;AAE5B,MAAM,MAAM,GAAG,MAAM,CAAC,SAAS,EAAE,CAAC;AAElC,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,KAAU;IACtC,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,IAAI,IAAI,GAAG,EAAE,CAAC;IAEd,OAAO,IAAI,SAAS,CAAC;QACjB,UAAU,EAAE,IAAI;QAChB,SAAS,CAAC,KAAU,EAAE,SAAiB,EAAG,QAA2B;YACjE,MAAM,KAAK,GAAG,CAAC,IAAI,GAAG,KAAK,CAAC,QAAQ,EAAE,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;YACvD,IAAI,GAAG,KAAK,CAAC,GAAG,EAAE,IAAI,EAAE,CAAC;YAEzB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;gBACvB,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE;oBAAE,SAAS;gBAC3B,IAAI,CAAC;oBACD,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC;oBAC5B,SAAS,EAAE,CAAC;gBAChB,CAAC;gBAAC,OAAO,KAAK,EAAE,CAAC;oBACb,MAAM,CAAC,KAAK,CAAC,4BAA4B,SAAS,GAAG,CAAC,KAAK,KAAK,EAAE,CAAC,CAAC;oBACpE,OAAO,QAAQ,CAAC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;gBAC/E,CAAC;YACL,CAAC;YACD,QAAQ,EAAE,CAAC;QACf,CAAC;QACD,KAAK,CAAC,QAAQ;YACV,IAAI,IAAI,CAAC,IAAI,EAAE,EAAE,CAAC;gBACd,IAAI,CAAC;oBACD,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC;gBAChC,CAAC;gBAAC,OAAO,CAAC,EAAE,CAAC,CAAC,gCAAgC,CAAC,CAAC;YACpD,CAAC;YACD,QAAQ,EAAE,CAAC;QACf,CAAC;KACJ,CAAC,CAAC;AACP,CAAC"}
|
package/dist/input/marc.d.ts
CHANGED
|
@@ -1,3 +1,3 @@
|
|
|
1
|
-
import {
|
|
2
|
-
export declare function
|
|
1
|
+
import { Transform } from "stream";
|
|
2
|
+
export declare function transform(_opts: any): Promise<Transform>;
|
|
3
3
|
//# sourceMappingURL=marc.d.ts.map
|
package/dist/input/marc.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"marc.d.ts","sourceRoot":"","sources":["../../src/input/marc.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,
|
|
1
|
+
{"version":3,"file":"marc.d.ts","sourceRoot":"","sources":["../../src/input/marc.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAA0B,MAAM,QAAQ,CAAC;AAM3D,wBAAsB,SAAS,CAAC,KAAK,EAAE,GAAG,GAAG,OAAO,CAAC,SAAS,CAAC,CAgD9D"}
|
package/dist/input/marc.js
CHANGED
|
@@ -1,30 +1,21 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { Transform } from "stream";
|
|
2
2
|
import { Marc } from "marcjs";
|
|
3
3
|
import log4js from 'log4js';
|
|
4
4
|
const logger = log4js.getLogger();
|
|
5
|
-
export async function
|
|
6
|
-
let recordNum = 0;
|
|
7
|
-
let hasError = false;
|
|
8
|
-
let sourcePaused = false;
|
|
5
|
+
export async function transform(_opts) {
|
|
9
6
|
const parser = Marc.createStream('Iso2709', 'Parser');
|
|
10
|
-
const
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
sourcePaused = false;
|
|
16
|
-
}
|
|
17
|
-
},
|
|
18
|
-
destroy() {
|
|
19
|
-
stream.destroy();
|
|
7
|
+
const transformer = new Transform({
|
|
8
|
+
objectMode: true,
|
|
9
|
+
transform(chunk, _encoding, callback) {
|
|
10
|
+
parser.write(chunk); //
|
|
11
|
+
callback();
|
|
20
12
|
},
|
|
21
|
-
|
|
13
|
+
flush(callback) {
|
|
14
|
+
parser.end(); //
|
|
15
|
+
callback();
|
|
16
|
+
}
|
|
22
17
|
});
|
|
23
18
|
parser.on('data', (record) => {
|
|
24
|
-
recordNum++;
|
|
25
|
-
if (recordNum % 1000 === 0) {
|
|
26
|
-
logger.info(`record: ${recordNum}`);
|
|
27
|
-
}
|
|
28
19
|
let rec = [];
|
|
29
20
|
rec.push(['LDR', ' ', ' ', '_', record.leader]);
|
|
30
21
|
for (const field of record.fields) {
|
|
@@ -33,35 +24,25 @@ export async function stream2readable(stream, _opts) {
|
|
|
33
24
|
const ind1 = ' ';
|
|
34
25
|
const ind2 = ' ';
|
|
35
26
|
const data = field.slice(1);
|
|
36
|
-
rec.push([tag, ind1, ind2].concat(data));
|
|
27
|
+
rec.push([tag, ind1, ind2].concat(data)); //
|
|
37
28
|
}
|
|
38
29
|
else if (field.length > 3) {
|
|
39
30
|
const ind1 = field[1].charAt(0);
|
|
40
31
|
const ind2 = field[1].charAt(1);
|
|
41
32
|
const data = field.slice(2);
|
|
42
|
-
rec.push([tag, ind1, ind2].concat(data));
|
|
33
|
+
rec.push([tag, ind1, ind2].concat(data)); //
|
|
43
34
|
}
|
|
44
35
|
else {
|
|
45
|
-
logger.warn("marc error: ", field);
|
|
36
|
+
logger.warn("marc error: ", field); //
|
|
46
37
|
}
|
|
47
38
|
}
|
|
48
|
-
|
|
39
|
+
transformer.push({
|
|
49
40
|
record: rec
|
|
50
41
|
});
|
|
51
|
-
if (!ok) {
|
|
52
|
-
logger.debug("backpressure on");
|
|
53
|
-
parser.pause();
|
|
54
|
-
sourcePaused = true;
|
|
55
|
-
}
|
|
56
42
|
});
|
|
57
|
-
parser.on('
|
|
58
|
-
|
|
59
|
-
logger.info(`record: ${recordNum}`);
|
|
60
|
-
}
|
|
61
|
-
readableStream.push(null);
|
|
62
|
-
logger.info(`processed ${recordNum} records`);
|
|
43
|
+
parser.on('error', (err) => {
|
|
44
|
+
transformer.destroy(err);
|
|
63
45
|
});
|
|
64
|
-
|
|
65
|
-
return readableStream;
|
|
46
|
+
return transformer;
|
|
66
47
|
}
|
|
67
48
|
//# sourceMappingURL=marc.js.map
|
package/dist/input/marc.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"marc.js","sourceRoot":"","sources":["../../src/input/marc.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,
|
|
1
|
+
{"version":3,"file":"marc.js","sourceRoot":"","sources":["../../src/input/marc.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAA0B,MAAM,QAAQ,CAAC;AAC3D,OAAO,EAAE,IAAI,EAAE,MAAM,QAAQ,CAAC;AAC9B,OAAO,MAAM,MAAM,QAAQ,CAAC;AAE5B,MAAM,MAAM,GAAG,MAAM,CAAC,SAAS,EAAE,CAAC;AAElC,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,KAAU;IACtC,MAAM,MAAM,GAAG,IAAI,CAAC,YAAY,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC;IAEtD,MAAM,WAAW,GAAG,IAAI,SAAS,CAAC;QAC9B,UAAU,EAAE,IAAI;QAChB,SAAS,CAAC,KAAU,EAAE,SAAiB,EAAE,QAA2B;YAChE,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE;YACvB,QAAQ,EAAE,CAAC;QACf,CAAC;QACD,KAAK,CAAC,QAA2B;YAC7B,MAAM,CAAC,GAAG,EAAE,CAAC,CAAC,EAAE;YAChB,QAAQ,EAAE,CAAC;QACf,CAAC;KACJ,CAAC,CAAC;IAEH,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,MAAM,EAAE,EAAE;QACzB,IAAI,GAAG,GAAe,EAAE,CAAC;QAEzB,GAAG,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC;QAEhD,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC;YAChC,MAAM,GAAG,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;YAErB,IAAI,KAAK,CAAC,MAAM,IAAI,CAAC,IAAI,GAAG,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;gBAC5C,MAAM,IAAI,GAAG,GAAG,CAAC;gBACjB,MAAM,IAAI,GAAG,GAAG,CAAC;gBACjB,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;gBAC5B,GAAG,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE;YAChD,CAAC;iBAAM,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC1B,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;gBAChC,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;gBAChC,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;gBAC5B,GAAG,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE;YAChD,CAAC;iBAAM,CAAC;gBACJ,MAAM,CAAC,IAAI,CAAC,cAAc,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE;YAC1C,CAAC;QACL,CAAC;QAED,WAAW,CAAC,IAAI,CAAC;YACb,MAAM,EAAE,GAAG;SACd,CAAC,CAAC;IACP,CAAC,CAAC,CAAC;IAEH,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE;QACvB,WAAW,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IAC7B,CAAC,CAAC,CAAC;IAEH,OAAO,WAAW,CAAC;AACvB,CAAC"}
|