marcattacks 1.0.0 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/httpstream.d.ts +3 -0
- package/dist/httpstream.d.ts.map +1 -0
- package/{src/httpstream.ts → dist/httpstream.js} +3 -7
- package/dist/httpstream.js.map +1 -0
- package/dist/index.d.ts +3 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +5 -0
- package/dist/index.js.map +1 -0
- package/dist/input/alephseq.d.ts +3 -0
- package/dist/input/alephseq.d.ts.map +1 -0
- package/{src/input/alephseq.ts → dist/input/alephseq.js} +20 -34
- package/dist/input/alephseq.js.map +1 -0
- package/dist/input/json.d.ts +3 -0
- package/dist/input/json.d.ts.map +1 -0
- package/{src/input/json.ts → dist/input/json.js} +8 -18
- package/dist/input/json.js.map +1 -0
- package/dist/input/jsonl.d.ts +3 -0
- package/dist/input/jsonl.d.ts.map +1 -0
- package/{src/input/jsonl.ts → dist/input/jsonl.js} +11 -21
- package/dist/input/jsonl.js.map +1 -0
- package/dist/input/xml.d.ts +3 -0
- package/dist/input/xml.d.ts.map +1 -0
- package/{src/input/xml.ts → dist/input/xml.js} +29 -50
- package/dist/input/xml.js.map +1 -0
- package/dist/marcmap.d.ts +31 -0
- package/dist/marcmap.d.ts.map +1 -0
- package/{src/marcmap.ts → dist/marcmap.js} +25 -38
- package/dist/marcmap.js.map +1 -0
- package/dist/output/alephseq.d.ts +3 -0
- package/dist/output/alephseq.d.ts.map +1 -0
- package/{src/output/alephseq.ts → dist/output/alephseq.js} +17 -25
- package/dist/output/alephseq.js.map +1 -0
- package/dist/output/json.d.ts +3 -0
- package/dist/output/json.d.ts.map +1 -0
- package/{src/output/json.ts → dist/output/json.js} +4 -13
- package/dist/output/json.js.map +1 -0
- package/dist/output/jsonl.d.ts +3 -0
- package/dist/output/jsonl.d.ts.map +1 -0
- package/{src/output/jsonl.ts → dist/output/jsonl.js} +5 -8
- package/dist/output/jsonl.js.map +1 -0
- package/dist/output/rdf.d.ts +3 -0
- package/dist/output/rdf.d.ts.map +1 -0
- package/dist/output/rdf.js +44 -0
- package/dist/output/rdf.js.map +1 -0
- package/dist/output/xml.d.ts +6 -0
- package/dist/output/xml.d.ts.map +1 -0
- package/{src/output/xml.ts → dist/output/xml.js} +18 -35
- package/dist/output/xml.js.map +1 -0
- package/dist/plugin-loader.d.ts +2 -0
- package/dist/plugin-loader.d.ts.map +1 -0
- package/dist/plugin-loader.js +24 -0
- package/dist/plugin-loader.js.map +1 -0
- package/dist/s3stream.d.ts +8 -0
- package/dist/s3stream.d.ts.map +1 -0
- package/{src/s3stream.ts → dist/s3stream.js} +72 -115
- package/dist/s3stream.js.map +1 -0
- package/dist/sftpstream.d.ts +12 -0
- package/dist/sftpstream.d.ts.map +1 -0
- package/{src/sftpstream.ts → dist/sftpstream.js} +9 -39
- package/dist/sftpstream.js.map +1 -0
- package/dist/slow-writable.d.ts +38 -0
- package/dist/slow-writable.d.ts.map +1 -0
- package/dist/slow-writable.js +126 -0
- package/dist/slow-writable.js.map +1 -0
- package/dist/transform/json.d.ts +3 -0
- package/dist/transform/json.d.ts.map +1 -0
- package/{src/transform/json.ts → dist/transform/json.js} +8 -12
- package/dist/transform/json.js.map +1 -0
- package/dist/transform/rdf.d.ts +3 -0
- package/dist/transform/rdf.d.ts.map +1 -0
- package/{src/transform/rdf.ts → dist/transform/rdf.js} +82 -110
- package/dist/transform/rdf.js.map +1 -0
- package/package.json +7 -2
- package/Dockerfile +0 -23
- package/README-docker.md +0 -39
- package/TYPESCRIPT.txt +0 -6
- package/data/output.rdf +0 -12425
- package/data/sample.xml +0 -2
- package/demo/demo.jsonata +0 -44
- package/docker-compose.yaml +0 -37
- package/logo.jpg +0 -0
- package/plugin/demo.js +0 -12
- package/src/index.ts +0 -177
- package/src/output/rdf.ts +0 -63
- package/src/plugin-loader.ts +0 -27
- package/src/slow-writable.ts +0 -165
- package/tsconfig.json +0 -46
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"httpstream.d.ts","sourceRoot":"","sources":["../src/httpstream.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,QAAQ,CAAC;AAKlC,wBAAgB,cAAc,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,QAAQ,CAAC,CAsBnE"}
|
|
@@ -2,27 +2,23 @@ import { Readable } from 'stream';
|
|
|
2
2
|
import * as http from 'http';
|
|
3
3
|
import * as https from 'https';
|
|
4
4
|
import { URL } from 'url';
|
|
5
|
-
|
|
6
|
-
export function httpReadStream(urlString: string): Promise<Readable> {
|
|
5
|
+
export function httpReadStream(urlString) {
|
|
7
6
|
return new Promise((resolve, reject) => {
|
|
8
7
|
const url = new URL(urlString);
|
|
9
8
|
const client = url.protocol === 'http:' ? http : https;
|
|
10
|
-
|
|
11
9
|
const req = client.get(url, res => {
|
|
12
10
|
if (res.statusCode && res.statusCode >= 400) {
|
|
13
11
|
reject(new Error('HTTP ' + res.statusCode));
|
|
14
12
|
return;
|
|
15
13
|
}
|
|
16
|
-
|
|
17
14
|
// Follow redirects
|
|
18
15
|
if (res.statusCode && res.statusCode >= 300 && res.statusCode < 400 && res.headers.location) {
|
|
19
16
|
httpReadStream(res.headers.location).then(resolve).catch(reject);
|
|
20
17
|
return;
|
|
21
18
|
}
|
|
22
|
-
|
|
23
19
|
resolve(res);
|
|
24
20
|
});
|
|
25
|
-
|
|
26
21
|
req.on('error', reject);
|
|
27
22
|
});
|
|
28
|
-
}
|
|
23
|
+
}
|
|
24
|
+
//# sourceMappingURL=httpstream.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"httpstream.js","sourceRoot":"","sources":["../src/httpstream.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,QAAQ,CAAC;AAClC,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAC7B,OAAO,KAAK,KAAK,MAAM,OAAO,CAAC;AAC/B,OAAO,EAAE,GAAG,EAAE,MAAM,KAAK,CAAC;AAE1B,MAAM,UAAU,cAAc,CAAC,SAAiB;IAC5C,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QACnC,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,SAAS,CAAC,CAAC;QAC/B,MAAM,MAAM,GAAG,GAAG,CAAC,QAAQ,KAAK,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC;QAEvD,MAAM,GAAG,GAAG,MAAM,CAAC,GAAG,CAAC,GAAG,EAAE,GAAG,CAAC,EAAE;YAC9B,IAAI,GAAG,CAAC,UAAU,IAAI,GAAG,CAAC,UAAU,IAAI,GAAG,EAAE,CAAC;gBAC1C,MAAM,CAAC,IAAI,KAAK,CAAC,OAAO,GAAG,GAAG,CAAC,UAAU,CAAC,CAAC,CAAC;gBAC5C,OAAO;YACX,CAAC;YAED,mBAAmB;YACnB,IAAI,GAAG,CAAC,UAAU,IAAI,GAAG,CAAC,UAAU,IAAI,GAAG,IAAI,GAAG,CAAC,UAAU,GAAG,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC;gBAC1F,cAAc,CAAC,GAAG,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;gBACjE,OAAO;YACX,CAAC;YAED,OAAO,CAAC,GAAG,CAAC,CAAC;QACjB,CAAC,CAAC,CAAC;QAEH,GAAG,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;IAC5B,CAAC,CAAC,CAAC;AACP,CAAC"}
|
package/dist/index.d.ts
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":""}
|
package/dist/index.js
CHANGED
|
@@ -10,6 +10,7 @@ import { SlowWritable } from './slow-writable.js';
|
|
|
10
10
|
import path from "node:path";
|
|
11
11
|
import fs from 'fs';
|
|
12
12
|
import { s3ReaderStream, s3WriterStream } from './s3stream.js';
|
|
13
|
+
import dotenv from 'dotenv';
|
|
13
14
|
log4js.configure({
|
|
14
15
|
appenders: {
|
|
15
16
|
err: {
|
|
@@ -26,6 +27,7 @@ log4js.configure({
|
|
|
26
27
|
});
|
|
27
28
|
program.version('0.1.0')
|
|
28
29
|
.argument('<file>')
|
|
30
|
+
.option('-c,--config <config>', 'config .env', path.join(process.cwd(), '.env'))
|
|
29
31
|
.option('-f,--from <from>', 'input type', 'xml')
|
|
30
32
|
.option('-t,--to <output>', 'output type', 'json')
|
|
31
33
|
.option('-m,--map <map>', 'data mapper', 'json')
|
|
@@ -47,6 +49,9 @@ if (opts.debug) {
|
|
|
47
49
|
if (opts.trace) {
|
|
48
50
|
logger.level = "trace";
|
|
49
51
|
}
|
|
52
|
+
if (opts.config) {
|
|
53
|
+
dotenv.config({ path: opts.config });
|
|
54
|
+
}
|
|
50
55
|
main();
|
|
51
56
|
async function main() {
|
|
52
57
|
const url = program.args[0];
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";AAEA,OAAO,MAAM,MAAM,QAAQ,CAAC;AAC5B,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AAChD,OAAO,EAAE,cAAc,EAAG,eAAe,EAAG,cAAc,EAAoB,MAAM,iBAAiB,CAAC;AACtG,OAAO,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AACjD,OAAO,EAAE,QAAQ,EAAE,MAAM,QAAQ,CAAC;AAClC,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AAEzC,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,MAAM,IAAI,CAAC;AACpB,OAAO,EAAE,cAAc,EAAE,cAAc,EAAE,MAAM,eAAe,CAAC;AAC/D,OAAO,MAAM,MAAM,QAAQ,CAAC;AAE5B,MAAM,CAAC,SAAS,CAAC;IACf,SAAS,EAAE;QACT,GAAG,EAAE;YACD,IAAI,EAAE,QAAQ;YACd,MAAM,EAAE;gBACJ,IAAI,EAAE,SAAS;gBACf,OAAO,EAAE,oBAAoB;aAChC;SACJ;KACF;IACD,UAAU,EAAE;QACV,OAAO,EAAE,EAAE,SAAS,EAAE,CAAC,KAAK,CAAC,EAAE,KAAK,EAAE,KAAK,EAAG,eAAe,EAAE,IAAI,EAAE;KACtE;CACF,CAAC,CAAC;AAEH,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC;KACnB,QAAQ,CAAC,QAAQ,CAAC;KAClB,MAAM,CAAC,sBAAsB,EAAC,aAAa,EAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,MAAM,CAAC,CAAC;KAC7E,MAAM,CAAC,kBAAkB,EAAC,YAAY,EAAC,KAAK,CAAC;KAC7C,MAAM,CAAC,kBAAkB,EAAC,aAAa,EAAC,MAAM,CAAC;KAC/C,MAAM,CAAC,gBAAgB,EAAC,aAAa,EAAC,MAAM,CAAC;KAC7C,MAAM,CAAC,cAAc,EAAC,SAAS,CAAC;KAChC,MAAM,CAAC,iBAAiB,EAAC,aAAa,CAAC;KACvC,MAAM,CAAC,iBAAiB,EAAE,kBAAkB,CAAC;KAC7C,MAAM,CAAC,QAAQ,EAAC,2BAA2B,CAAC;KAC5C,MAAM,CAAC,SAAS,EAAC,gCAAgC,CAAC;KAClD,MAAM,CAAC,SAAS,EAAC,qCAAqC,CAAC,CAAC;AAE7D,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;AAE5B,MAAM,IAAI,GAAK,OAAO,CAAC,IAAI,EAAE,CAAC;AAC9B,MAAM,MAAM,GAAG,MAAM,CAAC,SAAS,EAAE,CAAC;AAElC,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;IACZ,MAAM,CAAC,KAAK,GAAG,MAAM,CAAC;AAC1B,CAAC;AAED,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;IACb,MAAM,CAAC,KAAK,GAAG,OAAO,CAAC;AAC3B,CAAC;AAED,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;IACb,MAAM,CAAC,KAAK,GAAG,OAAO,CAAC;AAC3B,CAAC;AAED,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;IACd,MAAM,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;AACzC,CAAC;AAED,IAAI,EAAE,CAAC;AAEP,KAAK,UAAU,IAAI;IACf,MAAM,GAAG,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAE5B,IAAI,CAAE,GAAG,EAAE,CAAC;QACR,OAAO,CAAC,KAAK,CAAC,oBAAoB,CAAC,CAAC;QACpC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACpB,CAAC;IAED,IAAI,SAAe,CAAC;IAEpB,IAAI,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;QACrB,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,GAAG,CAAC,CAAC;QAClD,SAAS,GAAG,aAAa,CAAC,QAAQ,CAAC,CAAC;IACxC,CAAC;SACI,CAAC;QACF,SAAS,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;IAC7B,CAAC;IAED,MAAM,CAAC,IAAI,CAAC,UAAU,SAAS,EAAE,CAAC,CAAC;IAEnC,IAAI,cAAc,CAAC;IAEnB,IAAI,SAAS,CAAC,QAAQ,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC;QACxC,cAAc,GAAG,MAAM,cAAc,CAAC,SAAS,CAAC,QAAQ,EAAE,CAAC,CAAC;IAChE,CAAC;SACI,IAAI,SAAS,CAAC,QAAQ,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;QAC3C,cAAc,GAAG,MAAM,cAAc,CAAC,SAAS,EAAC,EAAE,CAAC,CAAC;IACxD,CAAC;SACI,IAAI,SAAS,CAAC,QAAQ,KAAK,OAAO,EAAE,CAAC;QACtC,MAAM,MAAM,GAAG,cAAc,CAAC,SAAS,EAAC,IAAI,CAAC,CAAC;QAE9C,IAAI,UAAU,CAAC;QAEf,IAAI,SAAS,CAAC,QAAQ,CAAC,KAAK,CAAC,gBAAgB,CAAC,EAAE,CAAC;YAC7C,MAAM,SAAS,GAAG,SAAS,CAAC,QAAQ,CAAC,OAAO,CAAC,aAAa,EAAC,EAAE,CAAC,CAAC;YAC/D,MAAM,SAAS,GAAG,SAAS,CAAC,QAAQ,CAAC,OAAO,CAAC,cAAc,EAAC,EAAE,CAAC,CAAC;YAChE,UAAU,GAAG,MAAM,cAAc,CAAC,MAAM,EAAC,SAAS,EAAC,SAAS,CAAC,CAAC;QAClE,CAAC;aACI,CAAC;YACF,UAAU,GAAG,SAAS,CAAC,QAAQ,CAAC;QACpC,CAAC;QAED,cAAc,GAAG,MAAM,cAAc,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC;IAC9D,CAAC;SACI,CAAC;QACF,cAAc,GAAG,EAAE,CAAC,gBAAgB,CAAC,SAAS,CAAC,CAAC;IACpD,CAAC;IAED,IAAI,YAAuB,CAAC;IAE5B,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;QACZ,MAAM,GAAG,GAAG,MAAM,UAAU,CAAC,IAAI,CAAC,IAAI,EAAC,OAAO,CAAC,CAAC;QAChD,YAAY,GAAG,GAAG,CAAC,eAAe,CAAC,cAAc,CAAC,CAAC;IACvD,CAAC;SACI,CAAC;QACF,OAAO,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC;QAC7B,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACpB,CAAC;IAED,IAAI,YAAY,GAAG,YAAY,CAAC;IAEhC,IAAI,IAAI,CAAC,GAAG,EAAE,CAAC;QACX,MAAM,GAAG,GAAG,MAAM,UAAU,CAAC,IAAI,CAAC,GAAG,EAAC,WAAW,CAAC,CAAC;QACnD,MAAM,WAAW,GAAe,MAAM,GAAG,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAC9D,YAAY,GAAG,YAAY,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;IAClD,CAAC;IAED,IAAI,SAAoB,CAAC;IAEzB,IAAI,IAAI,CAAC,GAAG,KAAK,OAAO,EAAE,CAAC;QACvB,SAAS,GAAG,IAAI,YAAY,CAAC,EAAE,OAAO,EAAE,GAAG,EAAE,CAAC,CAAC;IACnD,CAAC;SACI,IAAI,IAAI,CAAC,GAAG,EAAE,CAAC;QAChB,IAAI,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC;YAC9B,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAC9B,MAAM,MAAM,GAAG,cAAc,CAAC,GAAG,EAAC,IAAI,CAAC,CAAC;YACxC,MAAM,CAAC,IAAI,CAAC,OAAO,GAAG,EAAE,CAAC,CAAC;YAC1B,SAAS,GAAG,MAAM,eAAe,CAAC,GAAG,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;QACxD,CAAC;aACI,IAAI,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;YACjC,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAC9B,MAAM,CAAC,IAAI,CAAC,OAAO,GAAG,EAAE,CAAC,CAAC;YAC1B,SAAS,GAAG,MAAM,cAAc,CAAC,GAAG,EAAC,EAAE,CAAC,CAAC;QAC7C,CAAC;aACI,CAAC;YACF,SAAS,GAAG,EAAE,CAAC,iBAAiB,CAAC,IAAI,CAAC,GAAG,EAAE,EAAE,QAAQ,EAAE,OAAO,EAAC,CAAC,CAAC;QACrE,CAAC;IACL,CAAC;SACI,CAAC;QACF,SAAS,GAAG,OAAO,CAAC,MAAM,CAAC;IAC/B,CAAC;IAED,IAAI,IAAI,CAAC,EAAE,EAAE,CAAC;QACV,MAAM,GAAG,GAAG,MAAM,UAAU,CAAC,IAAI,CAAC,EAAE,EAAC,QAAQ,CAAC,CAAC;QAC/C,GAAG,CAAC,iBAAiB,CAAC,YAAY,EAAE,SAAS,CAAC,CAAC;IACnD,CAAC;AACL,CAAC;AAED,SAAS,cAAc,CAAC,SAAc,EAAE,IAAS;IAC7C,IAAI,UAAU,GAAwB,SAAS,CAAC;IAEhD,IAAI,IAAI,CAAC,GAAG,EAAE,CAAC;QACX,UAAU,GAAG,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,GAAG,EAAC,EAAE,QAAQ,EAAE,OAAO,EAAC,CAAC,CAAC;IAChE,CAAC;IAED,IAAI,MAAM,GAAe;QACrB,IAAI,EAAE,SAAS,CAAC,QAAQ;QACxB,IAAI,EAAE,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,EAAE;QAClC,QAAQ,EAAE,SAAS,CAAC,QAAQ;KAC/B,CAAC;IAEF,IAAI,SAAS,CAAC,QAAQ,EAAE,CAAC;QAAC,MAAM,CAAC,QAAQ,GAAG,SAAS,CAAC,QAAQ,CAAA;IAAC,CAAC;IAChE,IAAI,UAAU,EAAE,CAAC;QAAC,MAAM,CAAC,UAAU,GAAG,UAAU,CAAA;IAAA,CAAC;IAEjD,OAAO,MAAM,CAAC;AAClB,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"alephseq.d.ts","sourceRoot":"","sources":["../../src/input/alephseq.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,QAAQ,CAAC;AAMlC,wBAAgB,eAAe,CAAC,MAAM,EAAE,QAAQ,GAAI,QAAQ,CA4E3D"}
|
|
@@ -1,72 +1,58 @@
|
|
|
1
1
|
import { Readable } from "stream";
|
|
2
|
-
import * as readline from 'node:readline'
|
|
2
|
+
import * as readline from 'node:readline';
|
|
3
3
|
import log4js from 'log4js';
|
|
4
|
-
|
|
5
4
|
const logger = log4js.getLogger();
|
|
6
|
-
|
|
7
|
-
export function stream2readable(stream: Readable) : Readable {
|
|
5
|
+
export function stream2readable(stream) {
|
|
8
6
|
let recordNum = 0;
|
|
9
|
-
|
|
10
|
-
const rl = readline.createInterface({input: stream, crlfDelay: Infinity});
|
|
11
|
-
|
|
7
|
+
const rl = readline.createInterface({ input: stream, crlfDelay: Infinity });
|
|
12
8
|
let sourcePaused = false;
|
|
13
|
-
|
|
14
9
|
const readableStream = new Readable({
|
|
15
10
|
read() {
|
|
16
11
|
if (sourcePaused) {
|
|
17
12
|
logger.debug("backpressure off");
|
|
18
|
-
rl.resume();
|
|
13
|
+
rl.resume();
|
|
19
14
|
sourcePaused = false;
|
|
20
15
|
}
|
|
21
|
-
}
|
|
22
|
-
objectMode: true
|
|
16
|
+
},
|
|
17
|
+
objectMode: true
|
|
23
18
|
});
|
|
24
|
-
|
|
25
|
-
let
|
|
26
|
-
let previd : string = "";
|
|
27
|
-
|
|
19
|
+
let rec = [];
|
|
20
|
+
let previd = "";
|
|
28
21
|
rl.on('line', (line) => {
|
|
29
|
-
const [id
|
|
22
|
+
const [id, ...rest] = line.split(" ");
|
|
30
23
|
const data = rest.join(" ");
|
|
31
|
-
|
|
32
24
|
if (previd && previd !== id) {
|
|
33
25
|
const ok = readableStream.push({
|
|
34
|
-
record:
|
|
26
|
+
record: rec
|
|
35
27
|
});
|
|
36
|
-
|
|
37
28
|
if (!ok) {
|
|
38
29
|
logger.debug("backpressure on");
|
|
39
30
|
rl.pause();
|
|
40
|
-
sourcePaused = true;
|
|
31
|
+
sourcePaused = true;
|
|
41
32
|
}
|
|
42
33
|
rec = [];
|
|
43
34
|
recordNum++;
|
|
44
|
-
|
|
45
35
|
if (recordNum % 1000 === 0) {
|
|
46
36
|
logger.info(`record: ${recordNum}`);
|
|
47
37
|
}
|
|
48
38
|
}
|
|
49
|
-
|
|
50
|
-
const
|
|
51
|
-
const
|
|
52
|
-
const
|
|
53
|
-
const sf = data?.substring(8);
|
|
39
|
+
const tag = data?.substring(0, 3);
|
|
40
|
+
const ind1 = data?.substring(3, 4);
|
|
41
|
+
const ind2 = data?.substring(4, 5);
|
|
42
|
+
const sf = data?.substring(8);
|
|
54
43
|
const parts = sf.split(/\$\$(.)/);
|
|
55
|
-
|
|
56
44
|
if (tag == 'FMT' || tag === 'LDR' || tag.startsWith("00")) {
|
|
57
45
|
rec.push([
|
|
58
|
-
tag,ind1,ind2
|
|
46
|
+
tag, ind1, ind2
|
|
59
47
|
].concat(["_"].concat(parts)));
|
|
60
48
|
}
|
|
61
49
|
else {
|
|
62
50
|
rec.push([
|
|
63
|
-
tag,ind1,ind2
|
|
51
|
+
tag, ind1, ind2
|
|
64
52
|
].concat(parts.slice(1)));
|
|
65
53
|
}
|
|
66
|
-
|
|
67
|
-
previd = id!;
|
|
54
|
+
previd = id;
|
|
68
55
|
});
|
|
69
|
-
|
|
70
56
|
rl.on('close', () => {
|
|
71
57
|
readableStream.push({
|
|
72
58
|
record: rec
|
|
@@ -78,6 +64,6 @@ export function stream2readable(stream: Readable) : Readable {
|
|
|
78
64
|
readableStream.push(null);
|
|
79
65
|
logger.info(`processed ${recordNum} records`);
|
|
80
66
|
});
|
|
81
|
-
|
|
82
67
|
return readableStream;
|
|
83
|
-
}
|
|
68
|
+
}
|
|
69
|
+
//# sourceMappingURL=alephseq.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"alephseq.js","sourceRoot":"","sources":["../../src/input/alephseq.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,QAAQ,CAAC;AAClC,OAAO,KAAK,QAAQ,MAAM,eAAe,CAAA;AACzC,OAAO,MAAM,MAAM,QAAQ,CAAC;AAE5B,MAAM,MAAM,GAAG,MAAM,CAAC,SAAS,EAAE,CAAC;AAElC,MAAM,UAAU,eAAe,CAAC,MAAgB;IAC5C,IAAI,SAAS,GAAG,CAAC,CAAC;IAElB,MAAM,EAAE,GAAG,QAAQ,CAAC,eAAe,CAAC,EAAC,KAAK,EAAE,MAAM,EAAE,SAAS,EAAE,QAAQ,EAAC,CAAC,CAAC;IAE1E,IAAI,YAAY,GAAG,KAAK,CAAC;IAEzB,MAAM,cAAc,GAAG,IAAI,QAAQ,CAAC;QAChC,IAAI;YACA,IAAI,YAAY,EAAE,CAAC;gBACf,MAAM,CAAC,KAAK,CAAC,kBAAkB,CAAC,CAAC;gBACjC,EAAE,CAAC,MAAM,EAAE,CAAC;gBACZ,YAAY,GAAG,KAAK,CAAC;YACzB,CAAC;QACL,CAAC;QACD,UAAU,EAAE,IAAI;KACnB,CAAC,CAAC;IAEH,IAAI,GAAG,GAAgB,EAAE,CAAC;IAC1B,IAAI,MAAM,GAAY,EAAE,CAAC;IAEzB,EAAE,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,EAAE;QACnB,MAAM,CAAC,EAAE,EAAC,GAAG,IAAI,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QACrC,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAE5B,IAAI,MAAM,IAAI,MAAM,KAAK,EAAE,EAAE,CAAC;YAC1B,MAAM,EAAE,GAAG,cAAc,CAAC,IAAI,CAAC;gBAC3B,MAAM,EAAG,GAAG;aACf,CAAC,CAAC;YAEH,IAAI,CAAC,EAAE,EAAE,CAAC;gBACN,MAAM,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAC;gBAChC,EAAE,CAAC,KAAK,EAAE,CAAC;gBACX,YAAY,GAAG,IAAI,CAAC;YACxB,CAAC;YACD,GAAG,GAAG,EAAE,CAAC;YACT,SAAS,EAAE,CAAC;YAEZ,IAAI,SAAS,GAAG,IAAI,KAAK,CAAC,EAAE,CAAC;gBACzB,MAAM,CAAC,IAAI,CAAC,WAAW,SAAS,EAAE,CAAC,CAAC;YACxC,CAAC;QACL,CAAC;QAED,MAAM,GAAG,GAAI,IAAI,EAAE,SAAS,CAAC,CAAC,EAAC,CAAC,CAAC,CAAC;QAClC,MAAM,IAAI,GAAG,IAAI,EAAE,SAAS,CAAC,CAAC,EAAC,CAAC,CAAC,CAAC;QAClC,MAAM,IAAI,GAAG,IAAI,EAAE,SAAS,CAAC,CAAC,EAAC,CAAC,CAAC,CAAC;QAClC,MAAM,EAAE,GAAK,IAAI,EAAE,SAAS,CAAC,CAAC,CAAC,CAAC;QAChC,MAAM,KAAK,GAAG,EAAE,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;QAElC,IAAI,GAAG,IAAI,KAAK,IAAI,GAAG,KAAK,KAAK,IAAI,GAAG,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;YACxD,GAAG,CAAC,IAAI,CAAC;gBACL,GAAG,EAAC,IAAI,EAAC,IAAI;aAChB,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QACnC,CAAC;aACI,CAAC;YACF,GAAG,CAAC,IAAI,CAAC;gBACL,GAAG,EAAC,IAAI,EAAC,IAAI;aAChB,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAC9B,CAAC;QAED,MAAM,GAAG,EAAG,CAAC;IACjB,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE;QAChB,cAAc,CAAC,IAAI,CAAC;YAChB,MAAM,EAAE,GAAG;SACd,CAAC,CAAC;QACH,SAAS,EAAE,CAAC;QACZ,IAAI,SAAS,GAAG,IAAI,KAAK,CAAC,EAAE,CAAC;YACzB,MAAM,CAAC,IAAI,CAAC,WAAW,SAAS,EAAE,CAAC,CAAC;QACxC,CAAC;QACD,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC1B,MAAM,CAAC,IAAI,CAAC,aAAa,SAAS,UAAU,CAAC,CAAC;IAClD,CAAC,CAAC,CAAC;IAEH,OAAO,cAAc,CAAC;AAC1B,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"json.d.ts","sourceRoot":"","sources":["../../src/input/json.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,QAAQ,CAAC;AAMlC,wBAAgB,eAAe,CAAC,MAAM,EAAE,QAAQ,GAAI,QAAQ,CAwC3D"}
|
|
@@ -1,47 +1,37 @@
|
|
|
1
1
|
import { Readable } from "stream";
|
|
2
2
|
import streamArray from "stream-json/streamers/StreamArray.js";
|
|
3
3
|
import log4js from 'log4js';
|
|
4
|
-
|
|
5
4
|
const logger = log4js.getLogger();
|
|
6
|
-
|
|
7
|
-
export function stream2readable(stream: Readable) : Readable {
|
|
5
|
+
export function stream2readable(stream) {
|
|
8
6
|
let recordNum = 0;
|
|
9
|
-
|
|
10
7
|
const pipeline = stream.pipe(streamArray.withParser());
|
|
11
|
-
|
|
12
8
|
let sourcePaused = false;
|
|
13
|
-
|
|
14
9
|
const readableStream = new Readable({
|
|
15
10
|
read() {
|
|
16
11
|
if (sourcePaused) {
|
|
17
12
|
logger.debug("backpressure off");
|
|
18
|
-
pipeline.resume();
|
|
13
|
+
pipeline.resume();
|
|
19
14
|
sourcePaused = false;
|
|
20
15
|
}
|
|
21
|
-
}
|
|
22
|
-
objectMode: true
|
|
16
|
+
},
|
|
17
|
+
objectMode: true
|
|
23
18
|
});
|
|
24
|
-
|
|
25
|
-
pipeline.on('data', (data: any) => {
|
|
19
|
+
pipeline.on('data', (data) => {
|
|
26
20
|
const ok = readableStream.push(data.value);
|
|
27
|
-
|
|
28
21
|
if (!ok) {
|
|
29
|
-
logger.debug("backpressure on")
|
|
22
|
+
logger.debug("backpressure on");
|
|
30
23
|
pipeline.pause();
|
|
31
24
|
sourcePaused = true;
|
|
32
25
|
}
|
|
33
|
-
|
|
34
26
|
recordNum++;
|
|
35
|
-
|
|
36
27
|
if (recordNum % 1000 === 0) {
|
|
37
28
|
logger.info(`record: ${recordNum}`);
|
|
38
29
|
}
|
|
39
30
|
});
|
|
40
|
-
|
|
41
31
|
pipeline.on('end', () => {
|
|
42
32
|
logger.info(`processed ${recordNum} records`);
|
|
43
33
|
readableStream.push(null);
|
|
44
34
|
});
|
|
45
|
-
|
|
46
35
|
return readableStream;
|
|
47
|
-
}
|
|
36
|
+
}
|
|
37
|
+
//# sourceMappingURL=json.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"json.js","sourceRoot":"","sources":["../../src/input/json.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,QAAQ,CAAC;AAClC,OAAO,WAAW,MAAM,sCAAsC,CAAC;AAC/D,OAAO,MAAM,MAAM,QAAQ,CAAC;AAE5B,MAAM,MAAM,GAAG,MAAM,CAAC,SAAS,EAAE,CAAC;AAElC,MAAM,UAAU,eAAe,CAAC,MAAgB;IAC5C,IAAI,SAAS,GAAG,CAAC,CAAC;IAElB,MAAM,QAAQ,GAAG,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,UAAU,EAAE,CAAC,CAAC;IAEvD,IAAI,YAAY,GAAG,KAAK,CAAC;IAEzB,MAAM,cAAc,GAAG,IAAI,QAAQ,CAAC;QAChC,IAAI;YACA,IAAI,YAAY,EAAE,CAAC;gBACf,MAAM,CAAC,KAAK,CAAC,kBAAkB,CAAC,CAAC;gBACjC,QAAQ,CAAC,MAAM,EAAE,CAAC;gBAClB,YAAY,GAAG,KAAK,CAAC;YACzB,CAAC;QACL,CAAC;QACD,UAAU,EAAE,IAAI;KACnB,CAAC,CAAC;IAEH,QAAQ,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,IAAS,EAAE,EAAE;QAC9B,MAAM,EAAE,GAAG,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAE3C,IAAI,CAAC,EAAE,EAAE,CAAC;YACN,MAAM,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAA;YAC/B,QAAQ,CAAC,KAAK,EAAE,CAAC;YACjB,YAAY,GAAG,IAAI,CAAC;QACxB,CAAC;QAED,SAAS,EAAE,CAAC;QAEZ,IAAI,SAAS,GAAG,IAAI,KAAK,CAAC,EAAE,CAAC;YACzB,MAAM,CAAC,IAAI,CAAC,WAAW,SAAS,EAAE,CAAC,CAAC;QACxC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE;QACpB,MAAM,CAAC,IAAI,CAAC,aAAa,SAAS,UAAU,CAAC,CAAC;QAC9C,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC9B,CAAC,CAAC,CAAC;IAEH,OAAO,cAAc,CAAC;AAC1B,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"jsonl.d.ts","sourceRoot":"","sources":["../../src/input/jsonl.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,QAAQ,CAAC;AAMlC,wBAAgB,eAAe,CAAC,MAAM,EAAE,QAAQ,GAAI,QAAQ,CAwC3D"}
|
|
@@ -1,47 +1,37 @@
|
|
|
1
1
|
import { Readable } from "stream";
|
|
2
|
-
import * as readline from 'node:readline'
|
|
2
|
+
import * as readline from 'node:readline';
|
|
3
3
|
import log4js from 'log4js';
|
|
4
|
-
|
|
5
4
|
const logger = log4js.getLogger();
|
|
6
|
-
|
|
7
|
-
export function stream2readable(stream: Readable) : Readable {
|
|
5
|
+
export function stream2readable(stream) {
|
|
8
6
|
let recordNum = 0;
|
|
9
|
-
|
|
10
|
-
const rl = readline.createInterface({input: stream, crlfDelay: Infinity});
|
|
11
|
-
|
|
7
|
+
const rl = readline.createInterface({ input: stream, crlfDelay: Infinity });
|
|
12
8
|
let sourcePaused = false;
|
|
13
|
-
|
|
14
9
|
const readableStream = new Readable({
|
|
15
10
|
read() {
|
|
16
11
|
if (sourcePaused) {
|
|
17
12
|
logger.debug("backpressure off");
|
|
18
|
-
rl.resume();
|
|
13
|
+
rl.resume();
|
|
19
14
|
sourcePaused = false;
|
|
20
15
|
}
|
|
21
|
-
}
|
|
22
|
-
objectMode: true
|
|
16
|
+
},
|
|
17
|
+
objectMode: true
|
|
23
18
|
});
|
|
24
|
-
|
|
25
19
|
rl.on('line', (line) => {
|
|
26
20
|
const ok = readableStream.push(JSON.parse(line));
|
|
27
|
-
|
|
28
21
|
if (!ok) {
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
22
|
+
logger.debug("backpressure on");
|
|
23
|
+
rl.pause();
|
|
24
|
+
sourcePaused = true;
|
|
32
25
|
}
|
|
33
|
-
|
|
34
26
|
recordNum++;
|
|
35
|
-
|
|
36
27
|
if (recordNum % 1000 === 0) {
|
|
37
28
|
logger.info(`record: ${recordNum}`);
|
|
38
29
|
}
|
|
39
30
|
});
|
|
40
|
-
|
|
41
31
|
rl.on('close', () => {
|
|
42
32
|
readableStream.push(null);
|
|
43
33
|
logger.info(`processed ${recordNum} records`);
|
|
44
34
|
});
|
|
45
|
-
|
|
46
35
|
return readableStream;
|
|
47
|
-
}
|
|
36
|
+
}
|
|
37
|
+
//# sourceMappingURL=jsonl.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"jsonl.js","sourceRoot":"","sources":["../../src/input/jsonl.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,QAAQ,CAAC;AAClC,OAAO,KAAK,QAAQ,MAAM,eAAe,CAAA;AACzC,OAAO,MAAM,MAAM,QAAQ,CAAC;AAE5B,MAAM,MAAM,GAAG,MAAM,CAAC,SAAS,EAAE,CAAC;AAElC,MAAM,UAAU,eAAe,CAAC,MAAgB;IAC5C,IAAI,SAAS,GAAG,CAAC,CAAC;IAElB,MAAM,EAAE,GAAG,QAAQ,CAAC,eAAe,CAAC,EAAC,KAAK,EAAE,MAAM,EAAE,SAAS,EAAE,QAAQ,EAAC,CAAC,CAAC;IAE1E,IAAI,YAAY,GAAG,KAAK,CAAC;IAEzB,MAAM,cAAc,GAAG,IAAI,QAAQ,CAAC;QAChC,IAAI;YACA,IAAI,YAAY,EAAE,CAAC;gBACf,MAAM,CAAC,KAAK,CAAC,kBAAkB,CAAC,CAAC;gBACjC,EAAE,CAAC,MAAM,EAAE,CAAC;gBACZ,YAAY,GAAG,KAAK,CAAC;YACzB,CAAC;QACL,CAAC;QACD,UAAU,EAAE,IAAI;KACnB,CAAC,CAAC;IAEH,EAAE,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,EAAE;QACnB,MAAM,EAAE,GAAG,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC;QAEjD,IAAI,CAAC,EAAE,EAAE,CAAC;YACL,MAAM,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAC;YAChC,EAAE,CAAC,KAAK,EAAE,CAAC;YACX,YAAY,GAAG,IAAI,CAAC;QACzB,CAAC;QAED,SAAS,EAAE,CAAC;QAEZ,IAAI,SAAS,GAAG,IAAI,KAAK,CAAC,EAAE,CAAC;YACzB,MAAM,CAAC,IAAI,CAAC,WAAW,SAAS,EAAE,CAAC,CAAC;QACxC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE;QAChB,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC1B,MAAM,CAAC,IAAI,CAAC,aAAa,SAAS,UAAU,CAAC,CAAC;IAClD,CAAC,CAAC,CAAC;IAEH,OAAO,cAAc,CAAC;AAC1B,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"xml.d.ts","sourceRoot":"","sources":["../../src/input/xml.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,QAAQ,CAAC;AAUlC,wBAAgB,eAAe,CAAC,MAAM,EAAE,QAAQ,GAAI,QAAQ,CAkH3D"}
|
|
@@ -1,41 +1,29 @@
|
|
|
1
1
|
import { Readable } from 'stream';
|
|
2
2
|
import sax from 'sax';
|
|
3
3
|
import log4js from 'log4js';
|
|
4
|
-
|
|
5
|
-
type FieldAttribute = { [key: string]: string; } | { [key: string]: sax.QualifiedAttribute; };
|
|
6
|
-
type SubfieldAttribute = { [key: string]: string; };
|
|
7
|
-
type MARCType = 'leader' | 'control' | 'field' | 'subfield' | undefined;
|
|
8
|
-
|
|
9
4
|
const logger = log4js.getLogger();
|
|
10
|
-
|
|
11
|
-
export function stream2readable(stream: Readable) : Readable {
|
|
5
|
+
export function stream2readable(stream) {
|
|
12
6
|
let recordNum = 0;
|
|
13
|
-
|
|
14
7
|
let sourcePaused = false;
|
|
15
|
-
|
|
16
8
|
const readableStream = new Readable({
|
|
17
9
|
read() {
|
|
18
10
|
if (sourcePaused) {
|
|
19
11
|
logger.debug("backpressure off");
|
|
20
|
-
stream.resume();
|
|
12
|
+
stream.resume();
|
|
21
13
|
sourcePaused = false;
|
|
22
14
|
}
|
|
23
|
-
}
|
|
24
|
-
objectMode: true
|
|
15
|
+
},
|
|
16
|
+
objectMode: true
|
|
25
17
|
});
|
|
26
|
-
|
|
27
18
|
const parser = sax.createStream(true);
|
|
28
|
-
|
|
29
|
-
let
|
|
30
|
-
let
|
|
31
|
-
let
|
|
32
|
-
let
|
|
33
|
-
let
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
parser.on('opentag', (node: sax.Tag) => {
|
|
37
|
-
const localName = node.name.replaceAll(/^\w+:/g,'');
|
|
38
|
-
|
|
19
|
+
let record = [];
|
|
20
|
+
let subfield = [];
|
|
21
|
+
let attrib = {};
|
|
22
|
+
let sattrib = {};
|
|
23
|
+
let type;
|
|
24
|
+
let text = '';
|
|
25
|
+
parser.on('opentag', (node) => {
|
|
26
|
+
const localName = node.name.replaceAll(/^\w+:/g, '');
|
|
39
27
|
if (localName === 'collection') {
|
|
40
28
|
// Start collection...
|
|
41
29
|
}
|
|
@@ -47,7 +35,7 @@ export function stream2readable(stream: Readable) : Readable {
|
|
|
47
35
|
}
|
|
48
36
|
else if (localName == 'controlfield') {
|
|
49
37
|
type = 'control';
|
|
50
|
-
attrib = node.attributes
|
|
38
|
+
attrib = node.attributes;
|
|
51
39
|
}
|
|
52
40
|
else if (localName === 'datafield') {
|
|
53
41
|
attrib = node.attributes;
|
|
@@ -58,56 +46,48 @@ export function stream2readable(stream: Readable) : Readable {
|
|
|
58
46
|
else {
|
|
59
47
|
logger.error(`unknown tag: ${node.name}`);
|
|
60
48
|
}
|
|
61
|
-
|
|
62
49
|
text = '';
|
|
63
50
|
});
|
|
64
|
-
|
|
65
|
-
parser.on('text', (t: string) => {
|
|
51
|
+
parser.on('text', (t) => {
|
|
66
52
|
text += t;
|
|
67
53
|
});
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
const localName = tag.replaceAll(/^\w+:/g,'');
|
|
54
|
+
parser.on('closetag', (tag) => {
|
|
55
|
+
const localName = tag.replaceAll(/^\w+:/g, '');
|
|
71
56
|
if (localName === 'leader') {
|
|
72
|
-
record.push(['LDR',' ',' ','_',text]);
|
|
57
|
+
record.push(['LDR', ' ', ' ', '_', text]);
|
|
73
58
|
}
|
|
74
59
|
else if (localName == 'controlfield') {
|
|
75
|
-
let tag = attrib.tag
|
|
76
|
-
record.push([tag,' ',' ','_',text]);
|
|
60
|
+
let tag = attrib.tag;
|
|
61
|
+
record.push([tag, ' ', ' ', '_', text]);
|
|
77
62
|
}
|
|
78
63
|
else if (localName === 'datafield') {
|
|
79
|
-
let tag
|
|
80
|
-
let ind1 = attrib.ind1
|
|
81
|
-
let ind2 = attrib.ind2
|
|
82
|
-
record.push([tag,ind1,ind2].concat(subfield));
|
|
64
|
+
let tag = attrib.tag;
|
|
65
|
+
let ind1 = attrib.ind1;
|
|
66
|
+
let ind2 = attrib.ind2;
|
|
67
|
+
record.push([tag, ind1, ind2].concat(subfield));
|
|
83
68
|
subfield = [];
|
|
84
69
|
}
|
|
85
70
|
else if (localName === 'subfield') {
|
|
86
|
-
let code = sattrib.code
|
|
87
|
-
subfield = subfield.concat([code,text]);
|
|
71
|
+
let code = sattrib.code;
|
|
72
|
+
subfield = subfield.concat([code, text]);
|
|
88
73
|
}
|
|
89
74
|
if (localName === 'record') {
|
|
90
75
|
const ok = readableStream.push({ record });
|
|
91
|
-
|
|
92
76
|
if (!ok) {
|
|
93
77
|
logger.debug("backpressure on");
|
|
94
78
|
stream.pause();
|
|
95
79
|
sourcePaused = true;
|
|
96
80
|
}
|
|
97
|
-
|
|
98
81
|
record = [];
|
|
99
82
|
recordNum++;
|
|
100
|
-
|
|
101
83
|
if (recordNum % 1000 === 0) {
|
|
102
84
|
logger.info(`record: ${recordNum}`);
|
|
103
85
|
}
|
|
104
86
|
}
|
|
105
|
-
});
|
|
106
|
-
|
|
87
|
+
});
|
|
107
88
|
parser.on("error", (err) => {
|
|
108
|
-
logger.error
|
|
89
|
+
logger.error("Parser error:", err);
|
|
109
90
|
});
|
|
110
|
-
|
|
111
91
|
parser.on('end', () => {
|
|
112
92
|
const ok = readableStream.push(null);
|
|
113
93
|
if (!ok) {
|
|
@@ -118,8 +98,7 @@ export function stream2readable(stream: Readable) : Readable {
|
|
|
118
98
|
logger.info(`processed ${recordNum} records`);
|
|
119
99
|
readableStream.destroy();
|
|
120
100
|
});
|
|
121
|
-
|
|
122
101
|
stream.pipe(parser);
|
|
123
|
-
|
|
124
102
|
return readableStream;
|
|
125
|
-
}
|
|
103
|
+
}
|
|
104
|
+
//# sourceMappingURL=xml.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"xml.js","sourceRoot":"","sources":["../../src/input/xml.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,QAAQ,CAAC;AAClC,OAAO,GAAG,MAAM,KAAK,CAAC;AACtB,OAAO,MAAM,MAAM,QAAQ,CAAC;AAM5B,MAAM,MAAM,GAAG,MAAM,CAAC,SAAS,EAAE,CAAC;AAElC,MAAM,UAAU,eAAe,CAAC,MAAgB;IAC5C,IAAI,SAAS,GAAG,CAAC,CAAC;IAElB,IAAI,YAAY,GAAG,KAAK,CAAC;IAEzB,MAAM,cAAc,GAAG,IAAI,QAAQ,CAAC;QAChC,IAAI;YACA,IAAI,YAAY,EAAE,CAAC;gBACf,MAAM,CAAC,KAAK,CAAC,kBAAkB,CAAC,CAAC;gBACjC,MAAM,CAAC,MAAM,EAAE,CAAC;gBAChB,YAAY,GAAG,KAAK,CAAC;YACzB,CAAC;QACL,CAAC;QACD,UAAU,EAAE,IAAI;KACnB,CAAC,CAAC;IAEH,MAAM,MAAM,GAAG,GAAG,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;IAEtC,IAAI,MAAM,GAAgB,EAAE,CAAC;IAC7B,IAAI,QAAQ,GAAc,EAAE,CAAC;IAC7B,IAAI,MAAM,GAAoB,EAAE,CAAC;IACjC,IAAI,OAAO,GAAuB,EAAE,CAAC;IACrC,IAAI,IAAe,CAAC;IACpB,IAAI,IAAI,GAAY,EAAE,CAAC;IAEvB,MAAM,CAAC,EAAE,CAAC,SAAS,EAAE,CAAC,IAAa,EAAE,EAAE;QACnC,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,QAAQ,EAAC,EAAE,CAAC,CAAC;QAEpD,IAAI,SAAS,KAAK,YAAY,EAAE,CAAC;YAC7B,sBAAsB;QAC1B,CAAC;aACI,IAAI,SAAS,KAAK,QAAQ,EAAE,CAAC;YAC9B,kBAAkB;QACtB,CAAC;aACI,IAAI,SAAS,KAAK,QAAQ,EAAE,CAAC;YAC9B,IAAI,GAAG,QAAQ,CAAC;QACpB,CAAC;aACI,IAAI,SAAS,IAAI,cAAc,EAAE,CAAC;YACnC,IAAI,GAAG,SAAS,CAAC;YACjB,MAAM,GAAG,IAAI,CAAC,UAAU,CAAA;QAC5B,CAAC;aACI,IAAI,SAAS,KAAK,WAAW,EAAE,CAAC;YACjC,MAAM,GAAG,IAAI,CAAC,UAAU,CAAC;QAC7B,CAAC;aACI,IAAI,SAAS,KAAK,UAAU,EAAE,CAAC;YAChC,OAAO,GAAG,IAAI,CAAC,UAAU,CAAC;QAC9B,CAAC;aACI,CAAC;YACF,MAAM,CAAC,KAAK,CAAC,gBAAgB,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;QAC9C,CAAC;QAED,IAAI,GAAG,EAAE,CAAC;IACd,CAAC,CAAC,CAAC;IAEH,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,CAAS,EAAE,EAAE;QAC5B,IAAI,IAAI,CAAC,CAAC;IACd,CAAC,CAAC,CAAC;IAEH,MAAM,CAAC,EAAE,CAAC,UAAU,EAAE,CAAC,GAAW,EAAE,EAAE;QAClC,MAAM,SAAS,GAAG,GAAG,CAAC,UAAU,CAAC,QAAQ,EAAC,EAAE,CAAC,CAAC;QAC9C,IAAI,SAAS,KAAK,QAAQ,EAAE,CAAC;YACzB,MAAM,CAAC,IAAI,CAAC,CAAC,KAAK,EAAC,GAAG,EAAC,GAAG,EAAC,GAAG,EAAC,IAAI,CAAC,CAAC,CAAC;QAC1C,CAAC;aACI,IAAI,SAAS,IAAI,cAAc,EAAE,CAAC;YACnC,IAAI,GAAG,GAAG,MAAM,CAAC,GAAa,CAAC;YAC/B,MAAM,CAAC,IAAI,CAAC,CAAC,GAAG,EAAC,GAAG,EAAC,GAAG,EAAC,GAAG,EAAC,IAAI,CAAC,CAAC,CAAC;QACxC,CAAC;aACI,IAAI,SAAS,KAAK,WAAW,EAAE,CAAC;YACjC,IAAI,GAAG,GAAI,MAAM,CAAC,GAAa,CAAC;YAChC,IAAI,IAAI,GAAG,MAAM,CAAC,IAAc,CAAC;YACjC,IAAI,IAAI,GAAG,MAAM,CAAC,IAAc,CAAC;YACjC,MAAM,CAAC,IAAI,CAAC,CAAC,GAAG,EAAC,IAAI,EAAC,IAAI,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC;YAC9C,QAAQ,GAAG,EAAE,CAAC;QAClB,CAAC;aACI,IAAI,SAAS,KAAK,UAAU,EAAE,CAAC;YAChC,IAAI,IAAI,GAAG,OAAO,CAAC,IAAc,CAAC;YAClC,QAAQ,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,IAAI,EAAC,IAAI,CAAC,CAAC,CAAC;QAC5C,CAAC;QACD,IAAI,SAAS,KAAK,QAAQ,EAAE,CAAC;YACzB,MAAM,EAAE,GAAG,cAAc,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC;YAE3C,IAAI,CAAC,EAAE,EAAE,CAAC;gBACN,MAAM,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAC;gBAChC,MAAM,CAAC,KAAK,EAAE,CAAC;gBACf,YAAY,GAAG,IAAI,CAAC;YACxB,CAAC;YAED,MAAM,GAAG,EAAE,CAAC;YACZ,SAAS,EAAE,CAAC;YAEZ,IAAI,SAAS,GAAG,IAAI,KAAK,CAAC,EAAE,CAAC;gBACzB,MAAM,CAAC,IAAI,CAAC,WAAW,SAAS,EAAE,CAAC,CAAC;YACxC,CAAC;QACL,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE;QACvB,MAAM,CAAC,KAAK,CAAE,eAAe,EAAE,GAAG,CAAC,CAAC;IACxC,CAAC,CAAC,CAAC;IAEH,MAAM,CAAC,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE;QAClB,MAAM,EAAE,GAAG,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACrC,IAAI,CAAC,EAAE,EAAE,CAAC;YACN,MAAM,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAC;YAChC,MAAM,CAAC,KAAK,EAAE,CAAC;YACf,YAAY,GAAG,IAAI,CAAC;QACxB,CAAC;QACD,MAAM,CAAC,IAAI,CAAC,aAAa,SAAS,UAAU,CAAC,CAAC;QAC9C,cAAc,CAAC,OAAO,EAAE,CAAC;IAC7B,CAAC,CAAC,CAAC;IAEH,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAEpB,OAAO,cAAc,CAAC;AAC1B,CAAC"}
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
interface MARCMapOpts {
|
|
2
|
+
join_char?: string;
|
|
3
|
+
}
|
|
4
|
+
/**
|
|
5
|
+
* Given a marc row return the marc tag
|
|
6
|
+
*/
|
|
7
|
+
export declare function marctag(row: string[] | undefined): string;
|
|
8
|
+
/**
|
|
9
|
+
* Given a marc row return an array with the indicators
|
|
10
|
+
*/
|
|
11
|
+
export declare function marcind(row: string[] | undefined): string[];
|
|
12
|
+
/**
|
|
13
|
+
* Given a marc and a callback function, call function(tag,row)
|
|
14
|
+
* for each tag
|
|
15
|
+
*/
|
|
16
|
+
export declare function marcForTag(rec: string[][] | undefined, fun: (tag: string, row: string[]) => void): void;
|
|
17
|
+
/**
|
|
18
|
+
* Given a marc row and a callback function, call function(code,value)
|
|
19
|
+
* for each subfield
|
|
20
|
+
*/
|
|
21
|
+
export declare function marcForEachSub(row: string[] | undefined, fun: (code: string, value: string) => void): void;
|
|
22
|
+
/**
|
|
23
|
+
* Given an marc record and a field-path return a string[] with all matching values
|
|
24
|
+
*/
|
|
25
|
+
export declare function marcmap(record: string[][], find: string, opts: MARCMapOpts): string[];
|
|
26
|
+
/**
|
|
27
|
+
* Given a marc row and a code regex, return all matching values
|
|
28
|
+
*/
|
|
29
|
+
export declare function marcsubfields(row: string[], re: RegExp): string[];
|
|
30
|
+
export {};
|
|
31
|
+
//# sourceMappingURL=marcmap.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"marcmap.d.ts","sourceRoot":"","sources":["../src/marcmap.ts"],"names":[],"mappings":"AAAA,UAAU,WAAW;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;CACtB;AAED;;GAEG;AACH,wBAAgB,OAAO,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,SAAS,GAAI,MAAM,CAK1D;AAED;;GAEG;AACH,wBAAgB,OAAO,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,SAAS,GAAI,MAAM,EAAE,CAO5D;AAED;;;GAGG;AACH,wBAAgB,UAAU,CAAC,GAAG,EAAE,MAAM,EAAE,EAAE,GAAG,SAAS,EAAG,GAAG,EAAE,CAAC,GAAG,EAAC,MAAM,EAAE,GAAG,EAAC,MAAM,EAAE,KAAK,IAAI,GAAI,IAAI,CAUvG;AAED;;;GAGG;AACH,wBAAgB,cAAc,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,SAAS,EAAG,GAAG,EAAE,CAAC,IAAI,EAAC,MAAM,EAAE,KAAK,EAAC,MAAM,KAAK,IAAI,GAAI,IAAI,CAU1G;AAED;;GAEG;AACH,wBAAgB,OAAO,CAAC,MAAM,EAAE,MAAM,EAAE,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,WAAW,GAAI,MAAM,EAAE,CAkBtF;AAED;;GAEG;AACH,wBAAgB,aAAa,CAAC,GAAG,EAAE,MAAM,EAAE,EAAE,EAAE,EAAE,MAAM,GAAI,MAAM,EAAE,CASlE"}
|