marcattacks 2.5.0 → 2.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +82 -2
- package/dist/attacker.d.ts.map +1 -1
- package/dist/attacker.js +10 -0
- package/dist/attacker.js.map +1 -1
- package/dist/command.js +1 -0
- package/dist/command.js.map +1 -1
- package/dist/fix/binds.d.ts +5 -0
- package/dist/fix/binds.d.ts.map +1 -0
- package/dist/fix/binds.js +82 -0
- package/dist/fix/binds.js.map +1 -0
- package/dist/fix/conditions.d.ts +5 -0
- package/dist/fix/conditions.d.ts.map +1 -0
- package/dist/fix/conditions.js +111 -0
- package/dist/fix/conditions.js.map +1 -0
- package/dist/fix/fixes.d.ts +7 -0
- package/dist/fix/fixes.d.ts.map +1 -0
- package/dist/fix/fixes.js +478 -0
- package/dist/fix/fixes.js.map +1 -0
- package/dist/fix/index.d.ts +12 -0
- package/dist/fix/index.d.ts.map +1 -0
- package/dist/fix/index.js +40 -0
- package/dist/fix/index.js.map +1 -0
- package/dist/fix/parser.d.ts +25 -0
- package/dist/fix/parser.d.ts.map +1 -0
- package/dist/fix/parser.js +156 -0
- package/dist/fix/parser.js.map +1 -0
- package/dist/fix/path.d.ts +23 -0
- package/dist/fix/path.d.ts.map +1 -0
- package/dist/fix/path.js +351 -0
- package/dist/fix/path.js.map +1 -0
- package/dist/fix/signal.d.ts +2 -0
- package/dist/fix/signal.d.ts.map +1 -0
- package/dist/fix/signal.js +5 -0
- package/dist/fix/signal.js.map +1 -0
- package/dist/fix/util.d.ts +6 -0
- package/dist/fix/util.d.ts.map +1 -0
- package/dist/fix/util.js +133 -0
- package/dist/fix/util.js.map +1 -0
- package/dist/input/alephseq.d.ts.map +1 -1
- package/dist/input/alephseq.js +18 -5
- package/dist/input/alephseq.js.map +1 -1
- package/dist/input/fastxml.d.ts +3 -0
- package/dist/input/fastxml.d.ts.map +1 -0
- package/dist/input/fastxml.js +134 -0
- package/dist/input/fastxml.js.map +1 -0
- package/dist/input/xml.d.ts.map +1 -1
- package/dist/input/xml.js +4 -2
- package/dist/input/xml.js.map +1 -1
- package/dist/marcmap.d.ts.map +1 -1
- package/dist/marcmap.js +15 -3
- package/dist/marcmap.js.map +1 -1
- package/dist/output/alephseq.d.ts.map +1 -1
- package/dist/output/alephseq.js +29 -5
- package/dist/output/alephseq.js.map +1 -1
- package/dist/output/null.d.ts +3 -0
- package/dist/output/null.d.ts.map +1 -0
- package/dist/output/null.js +13 -0
- package/dist/output/null.js.map +1 -0
- package/dist/output/xml.d.ts.map +1 -1
- package/dist/output/xml.js +27 -0
- package/dist/output/xml.js.map +1 -1
- package/dist/stream/framer.d.ts.map +1 -0
- package/dist/stream/framer.js.map +1 -0
- package/dist/stream/worker-pool-str.d.ts.map +1 -0
- package/dist/stream/worker-pool-str.js.map +1 -0
- package/dist/stream/worker-pool.d.ts +10 -0
- package/dist/stream/worker-pool.d.ts.map +1 -0
- package/dist/stream/worker-pool.js +206 -0
- package/dist/stream/worker-pool.js.map +1 -0
- package/dist/transform/fix.d.ts +16 -0
- package/dist/transform/fix.d.ts.map +1 -0
- package/dist/transform/fix.js +46 -0
- package/dist/transform/fix.js.map +1 -0
- package/dist/transform/jsonata.d.ts +4 -0
- package/dist/transform/jsonata.d.ts.map +1 -1
- package/dist/transform/jsonata.js +40 -44
- package/dist/transform/jsonata.js.map +1 -1
- package/dist/transform/marcinrdf.d.ts +1 -1
- package/dist/transform/marcinrdf.d.ts.map +1 -1
- package/dist/transform/marcinrdf.js +16 -3
- package/dist/transform/marcinrdf.js.map +1 -1
- package/dist/util/marc_record.d.ts +19 -0
- package/dist/util/marc_record.d.ts.map +1 -0
- package/dist/util/marc_record.js +16 -0
- package/dist/util/marc_record.js.map +1 -0
- package/dist/worker-str.d.ts.map +1 -0
- package/dist/worker-str.js.map +1 -0
- package/dist/worker.d.ts +2 -0
- package/dist/worker.d.ts.map +1 -0
- package/dist/worker.js +33 -0
- package/dist/worker.js.map +1 -0
- package/package.json +4 -1
- package/dist/output/mulitipart.d.ts +0 -7
- package/dist/output/mulitipart.d.ts.map +0 -1
- package/dist/output/mulitipart.js +0 -53
- package/dist/output/mulitipart.js.map +0 -1
- package/dist/output/text.d.ts +0 -6
- package/dist/output/text.d.ts.map +0 -1
- package/dist/output/text.js +0 -34
- package/dist/output/text.js.map +0 -1
package/dist/output/xml.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"xml.js","sourceRoot":"","sources":["../../src/output/xml.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,QAAQ,CAAC;AACnC,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,aAAa,EAAG,cAAc,EAAC,MAAM,eAAe,CAAC;AAChF,OAAO,MAAM,MAAM,QAAQ,CAAC;AAE5B,MAAM,MAAM,GAAG,MAAM,CAAC,SAAS,EAAE,CAAC;AAElC,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,KAAS;IACrC,IAAI,OAAO,GAAG,IAAI,CAAC;IAEnB,OAAO,IAAI,SAAS,CAAC;QACjB,UAAU,EAAE,IAAI;QAChB,SAAS,CAAC,IAAS,EAAE,SAAS,EAAE,QAAQ;YACpC,IAAI,GAAG,GAAgB,IAAI,CAAC,QAAQ,CAAC,CAAC;YAEtC,IAAI,CAAC,GAAG,EAAE,CAAC;gBACP,MAAM,CAAC,KAAK,CAAC,sBAAsB,CAAC,CAAC;gBACrC,QAAQ,EAAE,CAAA;gBACV,OAAO;YACX,CAAC;YAED,IAAI,MAAM,GAAG,EAAE,CAAC;YAEhB,IAAI,OAAO,EAAE,CAAC;gBACV,MAAM,IAAI,8CAA8C,CAAC;gBACzD,MAAM,IAAI,mEAAmE,CAAC;gBAC9E,OAAO,GAAG,KAAK,CAAC;YACpB,CAAC;YAED,MAAM,IAAI,kBAAkB,CAAC;YAE7B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAG,CAAC,GAAG,GAAG,CAAC,MAAM,EAAG,CAAC,EAAE,EAAE,CAAC;gBACpC,IAAI,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;gBAC1B,IAAI,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;gBAC1B,IAAI,GAAG,KAAK,KAAK,EAAE,CAAC,CAAA,CAAC;qBAChB,IAAI,GAAG,KAAK,KAAK,EAAE,CAAC;oBACrB,IAAI,KAAK,GAAG,aAAa,CAAC,GAAG,CAAC,CAAC,CAAE,EAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;oBAC3C,MAAM,IAAI,kBAAkB,SAAS,CAAC,KAAK,CAAC,kBAAkB,CAAC;gBACnE,CAAC;qBACI,IAAI,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC;oBACxB,IAAI,KAAK,GAAG,aAAa,CAAC,GAAG,CAAC,CAAC,CAAE,EAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;oBAC3C,MAAM,IAAI,6BAA6B,SAAS,CAAC,GAAG,EAAC,EAAC,YAAY,EAAC,IAAI,EAAC,CAAC,KAAK,SAAS,CAAC,KAAK,CAAC,wBAAwB,CAAC;gBAC3H,CAAC;qBACI,CAAC;oBACF,MAAM,IAAI,0BAA0B,SAAS,CAAC,GAAG,CAAC,WAAW,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAC,EAAC,YAAY,EAAC,IAAI,EAAC,CAAC,WAAW,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAC,EAAC,YAAY,EAAC,IAAI,EAAC,CAAC,MAAM,CAAC;oBACzJ,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,IAAI,EAAC,KAAK,EAAE,EAAE;wBAClC,MAAM,IAAI,4BAA4B,SAAS,CAAC,IAAI,CAAC,KAAK,SAAS,CAAC,KAAK,CAAC,oBAAoB,CAAC;oBACnG,CAAC,CAAC,CAAC;oBACH,MAAM,IAAI,uBAAuB,CAAC;gBACtC,CAAC;YACL,CAAC;YAED,MAAM,IAAI,mBAAmB,CAAC;YAE9B,MAAM,CAAC,KAAK,CAAC,UAAU,MAAM,CAAC,MAAM,QAAQ,CAAC,CAAC;YAE9C,QAAQ,CAAC,IAAI,EAAC,MAAM,CAAC,CAAC;QAC1B,CAAC;QACD,KAAK,CAAC,QAAQ;YACV,MAAM,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC;YAC9B,IAAI,CAAC,OAAO,EAAE,CAAC;gBACX,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;gBACzB,IAAI,MAAM,GAAG,sBAAsB,CAAC;gBACpC,MAAM,CAAC,KAAK,CAAC,UAAU,MAAM,CAAC,MAAM,QAAQ,CAAC,CAAC;gBAC9C,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACtB,CAAC;YACD,QAAQ,EAAE,CAAC;QACf,CAAC;KACJ,CAAC,CAAC;AACP,CAAC;AAED,MAAM,UAAU,SAAS,CACvB,KAAyC,EACzC,OAAoC;IAElC,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,KAAK,SAAS;QAAE,OAAO,EAAE,CAAC;IAErD,IAAI,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;IAEtB,qEAAqE;IAErE,6EAA6E;IAC7E,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,qCAAqC,EAAE,QAAQ,CAAC,CAAC,CAAE,2BAA2B;IAC5F,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,sCAAsC,EAAE,QAAQ,CAAC,CAAC,CAAC,0BAA0B;IAE3F,8CAA8C;IAC9C,6DAA6D;IAC7D,yDAAyD;IACzD,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,oEAAoE,EAAE,EAAE,CAAC,CAAC;IAExF,+DAA+D;IAC/D,4DAA4D;IAC5D,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,kBAAkB,EAAE,EAAE,CAAC,CAAC;IAEtC,wCAAwC;IACxC,4DAA4D;IAC5D,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAE7B,kCAAkC;IAClC,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;IAC5B,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;IAE5B,IAAI,OAAO,EAAE,YAAY,EAAE,CAAC;QACxB,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;QAC9B,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;IAClC,CAAC;IAED,OAAO,CAAC,CAAC;AACb,CAAC"}
|
|
1
|
+
{"version":3,"file":"xml.js","sourceRoot":"","sources":["../../src/output/xml.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,QAAQ,CAAC;AACnC,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,aAAa,EAAG,cAAc,EAAC,MAAM,eAAe,CAAC;AAChF,OAAO,MAAM,MAAM,QAAQ,CAAC;AAE5B,MAAM,MAAM,GAAG,MAAM,CAAC,SAAS,EAAE,CAAC;AAElC,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,KAAS;IACrC,IAAI,OAAO,GAAG,IAAI,CAAC;IAEnB,OAAO,IAAI,SAAS,CAAC;QACjB,UAAU,EAAE,IAAI;QAChB,SAAS,CAAC,IAAS,EAAE,SAAS,EAAE,QAAQ;YACpC,IAAI,GAAG,GAAgB,IAAI,CAAC,QAAQ,CAAC,CAAC;YAEtC,IAAI,CAAC,GAAG,EAAE,CAAC;gBACP,MAAM,CAAC,KAAK,CAAC,sBAAsB,CAAC,CAAC;gBACrC,QAAQ,EAAE,CAAA;gBACV,OAAO;YACX,CAAC;YAED,IAAI,MAAM,GAAG,EAAE,CAAC;YAEhB,IAAI,OAAO,EAAE,CAAC;gBACV,MAAM,IAAI,8CAA8C,CAAC;gBACzD,MAAM,IAAI,mEAAmE,CAAC;gBAC9E,OAAO,GAAG,KAAK,CAAC;YACpB,CAAC;YAED,MAAM,IAAI,kBAAkB,CAAC;YAE7B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAG,CAAC,GAAG,GAAG,CAAC,MAAM,EAAG,CAAC,EAAE,EAAE,CAAC;gBACpC,IAAI,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;gBAC1B,IAAI,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;gBAC1B,IAAI,GAAG,KAAK,KAAK,EAAE,CAAC,CAAA,CAAC;qBAChB,IAAI,GAAG,KAAK,KAAK,EAAE,CAAC;oBACrB,IAAI,KAAK,GAAG,aAAa,CAAC,GAAG,CAAC,CAAC,CAAE,EAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;oBAC3C,MAAM,IAAI,kBAAkB,SAAS,CAAC,KAAK,CAAC,kBAAkB,CAAC;gBACnE,CAAC;qBACI,IAAI,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC;oBACxB,IAAI,KAAK,GAAG,aAAa,CAAC,GAAG,CAAC,CAAC,CAAE,EAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;oBAC3C,MAAM,IAAI,6BAA6B,SAAS,CAAC,GAAG,EAAC,EAAC,YAAY,EAAC,IAAI,EAAC,CAAC,KAAK,SAAS,CAAC,KAAK,CAAC,wBAAwB,CAAC;gBAC3H,CAAC;qBACI,CAAC;oBACF,MAAM,IAAI,0BAA0B,SAAS,CAAC,GAAG,CAAC,WAAW,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAC,EAAC,YAAY,EAAC,IAAI,EAAC,CAAC,WAAW,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAC,EAAC,YAAY,EAAC,IAAI,EAAC,CAAC,MAAM,CAAC;oBACzJ,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,IAAI,EAAC,KAAK,EAAE,EAAE;wBAClC,MAAM,IAAI,4BAA4B,SAAS,CAAC,IAAI,CAAC,KAAK,SAAS,CAAC,KAAK,CAAC,oBAAoB,CAAC;oBACnG,CAAC,CAAC,CAAC;oBACH,MAAM,IAAI,uBAAuB,CAAC;gBACtC,CAAC;YACL,CAAC;YAED,MAAM,IAAI,mBAAmB,CAAC;YAE9B,MAAM,CAAC,KAAK,CAAC,UAAU,MAAM,CAAC,MAAM,QAAQ,CAAC,CAAC;YAE9C,QAAQ,CAAC,IAAI,EAAC,MAAM,CAAC,CAAC;QAC1B,CAAC;QACD,KAAK,CAAC,QAAQ;YACV,MAAM,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC;YAC9B,IAAI,CAAC,OAAO,EAAE,CAAC;gBACX,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;gBACzB,IAAI,MAAM,GAAG,sBAAsB,CAAC;gBACpC,MAAM,CAAC,KAAK,CAAC,UAAU,MAAM,CAAC,MAAM,QAAQ,CAAC,CAAC;gBAC9C,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACtB,CAAC;YACD,QAAQ,EAAE,CAAC;QACf,CAAC;KACJ,CAAC,CAAC;AACP,CAAC;AAED,0EAA0E;AAC1E,6EAA6E;AAC7E,6EAA6E;AAC7E,oEAAoE;AACpE,SAAS,WAAW,CAAC,CAAS,EAAE,YAAqB;IACjD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAChC,MAAM,CAAC,GAAG,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;QAC1B,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE;YAAE,OAAO,IAAI,CAAC,CAAiB,QAAQ;QAC3E,IAAI,YAAY,IAAI,CAAC,CAAC,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,CAAC;YAAE,OAAO,IAAI,CAAC,CAAW,MAAM;QACzE,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,IAAI,CAAC;YAAE,OAAO,IAAI,CAAC,CAAC,OAAO;QAC3F,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,IAAI;YAAE,OAAO,IAAI,CAAC;QACxC,IAAI,CAAC,IAAI,MAAM,IAAI,CAAC,IAAI,MAAM;YAAE,OAAO,IAAI,CAAC,CAAuB,aAAa;QAChF,IAAI,CAAC,IAAI,MAAM,IAAI,CAAC,IAAI,MAAM;YAAE,OAAO,IAAI,CAAC,CAAuB,YAAY;QAC/E,IAAI,CAAC,KAAK,MAAM,IAAI,CAAC,KAAK,MAAM;YAAE,OAAO,IAAI,CAAC;IAClD,CAAC;IACD,OAAO,KAAK,CAAC;AACjB,CAAC;AAED,MAAM,UAAU,SAAS,CACvB,KAAyC,EACzC,OAAoC;IAElC,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,KAAK,SAAS;QAAE,OAAO,EAAE,CAAC;IAErD,IAAI,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;IAEtB,sEAAsE;IACtE,IAAI,CAAC,WAAW,CAAC,CAAC,EAAE,OAAO,EAAE,YAAY,KAAK,IAAI,CAAC;QAAE,OAAO,CAAC,CAAC;IAE9D,qEAAqE;IAErE,6EAA6E;IAC7E,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,qCAAqC,EAAE,QAAQ,CAAC,CAAC,CAAE,2BAA2B;IAC5F,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,sCAAsC,EAAE,QAAQ,CAAC,CAAC,CAAC,0BAA0B;IAE3F,8CAA8C;IAC9C,6DAA6D;IAC7D,yDAAyD;IACzD,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,oEAAoE,EAAE,EAAE,CAAC,CAAC;IAExF,+DAA+D;IAC/D,4DAA4D;IAC5D,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,kBAAkB,EAAE,EAAE,CAAC,CAAC;IAEtC,wCAAwC;IACxC,4DAA4D;IAC5D,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAE7B,kCAAkC;IAClC,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;IAC5B,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;IAE5B,IAAI,OAAO,EAAE,YAAY,EAAE,CAAC;QACxB,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;QAC9B,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;IAClC,CAAC;IAED,OAAO,CAAC,CAAC;AACb,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"framer.d.ts","sourceRoot":"","sources":["../../src/stream/framer.ts"],"names":[],"mappings":"AAUA,OAAO,EAAE,SAAS,EAA0B,MAAM,QAAQ,CAAC;AAG3D,eAAO,MAAM,QAAQ,aAAmD,CAAC;AAkCzE,wBAAgB,YAAY,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,SAAS,CA+BnE"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"framer.js","sourceRoot":"","sources":["../../src/stream/framer.ts"],"names":[],"mappings":"AAAA,8EAA8E;AAC9E,6EAA6E;AAC7E,+EAA+E;AAC/E,+EAA+E;AAC/E,4BAA4B;AAC5B,EAAE;AACF,+EAA+E;AAC/E,uDAAuD;AACvD,yEAAyE;AACzE,mEAAmE;AACnE,OAAO,EAAE,SAAS,EAA0B,MAAM,QAAQ,CAAC;AAC3D,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AAEpD,MAAM,CAAC,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,CAAC,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,UAAU,CAAC,CAAC,CAAC;AAEzE,MAAM,YAAY,GAAG,6BAA6B,CAAC;AAEnD,+EAA+E;AAC/E,+EAA+E;AAC/E,SAAS,kBAAkB,CAAC,IAAY,EAAE,GAAW;IACjD,IAAI,IAAI,KAAK,OAAO,EAAE,CAAC;QACnB,MAAM,IAAI,GAAa,EAAE,CAAC;QAC1B,KAAK,IAAI,CAAC,GAAG,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,GAAG,GAAG,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,GAAG,CAAC,CAAC;YAAE,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QACvF,OAAO,IAAI,CAAC;IAChB,CAAC;IACD,IAAI,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;QACvC,MAAM,IAAI,GAAa,EAAE,CAAC;QAC1B,YAAY,CAAC,SAAS,GAAG,CAAC,CAAC;QAC3B,IAAI,CAAyB,CAAC;QAC9B,OAAO,CAAC,CAAC,GAAG,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAAE,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;QACtE,OAAO,IAAI,CAAC;IAChB,CAAC;IACD,2EAA2E;IAC3E,oEAAoE;IACpE,MAAM,IAAI,GAAa,EAAE,CAAC;IAC1B,IAAI,MAAM,GAAkB,IAAI,CAAC;IACjC,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,KAAK,IAAI,CAAC,GAAG,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,GAAG,GAAG,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC;QACnE,MAAM,EAAE,GAAG,GAAG,CAAC,OAAO,CAAC,GAAG,EAAE,SAAS,CAAC,CAAC;QACvC,MAAM,EAAE,GAAG,EAAE,IAAI,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC;QAClF,IAAI,MAAM,KAAK,IAAI,IAAI,EAAE,KAAK,MAAM;YAAE,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,yBAAyB;QACrF,MAAM,GAAG,EAAE,CAAC;QACZ,SAAS,GAAG,CAAC,GAAG,CAAC,CAAC;IACtB,CAAC;IACD,OAAO,IAAI,CAAC;AAChB,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,IAAY,EAAE,KAAa;IACpD,MAAM,OAAO,GAAG,IAAI,aAAa,CAAC,MAAM,CAAC,CAAC;IAC1C,IAAI,GAAG,GAAG,EAAE,CAAC;IACb,IAAI,GAAG,GAAG,CAAC,CAAC;IAEZ,SAAS,IAAI,CAAC,MAAiB,EAAE,KAAc;QAC3C,SAAS,CAAC;YACN,MAAM,IAAI,GAAG,kBAAkB,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;YAC3C,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;gBAAE,MAAM;YAC7B,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,MAAM,GAAG,KAAK;gBAAE,MAAM,CAAO,wBAAwB;YACxE,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAE,CAAC;YACpD,MAAM,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,GAAG,EAAE,EAAE,IAAI,EAAE,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;YACrD,GAAG,GAAG,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;YACrB,IAAI,IAAI,CAAC,MAAM,GAAG,KAAK;gBAAE,MAAM,CAAiB,mBAAmB;QACvE,CAAC;IACL,CAAC;IAED,OAAO,IAAI,SAAS,CAAC;QACjB,kBAAkB,EAAE,IAAI;QACxB,SAAS,CAAC,KAAU,EAAE,IAAY,EAAE,EAAqB;YACrD,GAAG,IAAI,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;YAC5B,IAAI,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;YAClB,EAAE,EAAE,CAAC;QACT,CAAC;QACD,KAAK,CAAC,EAAqB;YACvB,GAAG,IAAI,OAAO,CAAC,GAAG,EAAE,CAAC;YACrB,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;YACjB,IAAI,GAAG,CAAC,MAAM,GAAG,CAAC;gBAAE,IAAI,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,GAAG,EAAE,EAAE,IAAI,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC,0BAA0B;YACpF,EAAE,EAAE,CAAC;QACT,CAAC;KACJ,CAAC,CAAC;AACP,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"worker-pool-str.d.ts","sourceRoot":"","sources":["../../src/stream/worker-pool-str.ts"],"names":[],"mappings":"AAMA,OAAO,EAAE,SAAS,EAA0B,MAAM,QAAQ,CAAC;AAM3D,MAAM,WAAW,WAAW;IAAG,IAAI,EAAE,MAAM,CAAC;IAAC,GAAG,CAAC,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,GAAG,CAAC;IAAC,OAAO,EAAE,MAAM,CAAC;CAAE;AAEzF,wBAAgB,sBAAsB,CAAC,IAAI,EAAE,WAAW,GAAG,SAAS,CA4FnE"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"worker-pool-str.js","sourceRoot":"","sources":["../../src/stream/worker-pool-str.ts"],"names":[],"mappings":"AAAA,+EAA+E;AAC/E,gFAAgF;AAChF,mDAAmD;AACnD,EAAE;AACF,wDAAwD;AACxD,sDAAsD;AACtD,OAAO,EAAE,SAAS,EAA0B,MAAM,QAAQ,CAAC;AAC3D,OAAO,EAAE,MAAM,EAAE,MAAM,qBAAqB,CAAC;AAC7C,OAAO,MAAM,MAAM,QAAQ,CAAC;AAE5B,MAAM,MAAM,GAAG,MAAM,CAAC,SAAS,EAAE,CAAC;AAIlC,MAAM,UAAU,sBAAsB,CAAC,IAAiB;IACpD,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;IACpC,MAAM,GAAG,GAAG,CAAC,GAAG,CAAC,CAAC;IAClB,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,kBAAkB,EAAE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAE/D,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,MAAM,IAAI,GAAa,EAAE,CAAC;IAC1B,MAAM,KAAK,GAAoC,EAAE,CAAC;IAClD,MAAM,OAAO,GAAG,IAAI,GAAG,EAAkB,CAAC;IAE1C,IAAI,QAAQ,GAAG,CAAC,CAAC,CAAK,8BAA8B;IACpD,IAAI,QAAQ,GAAG,CAAC,CAAC,CAAK,8BAA8B;IACpD,IAAI,OAAO,GAAG,CAAC,CAAC,CAAM,6BAA6B;IACnD,IAAI,KAAK,GAAG,CAAC,CAAC,CAAQ,qCAAqC;IAC3D,IAAI,KAAK,GAAG,KAAK,CAAC;IAClB,IAAI,QAAQ,GAAG,CAAC,CAAC;IACjB,IAAI,QAAQ,GAA6B,IAAI,CAAC;IAC9C,IAAI,KAAK,GAA6B,IAAI,CAAC;IAC3C,IAAI,MAAM,GAAiB,IAAI,CAAC;IAEhC,MAAM,MAAM,GAAG,IAAI,SAAS,CAAC;QACzB,kBAAkB,EAAE,IAAI,EAAU,iBAAiB;QACnD,kBAAkB,EAAE,KAAK,EAAS,eAAe;QACjD,SAAS,CAAC,KAAoC,EAAE,IAAI,EAAE,EAAqB;YACvE,IAAI,MAAM,EAAE,CAAC;gBAAC,EAAE,CAAC,MAAM,CAAC,CAAC;gBAAC,OAAO;YAAC,CAAC;YACnC,QAAQ,EAAE,CAAC;YACX,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAClB,QAAQ,EAAE,CAAC;YACX,IAAI,QAAQ,GAAG,GAAG;gBAAE,EAAE,EAAE,CAAC;;gBACpB,QAAQ,GAAG,EAAE,CAAC,CAAW,eAAe;QACjD,CAAC;QACD,KAAK,CAAC,EAAqB;YACvB,IAAI,MAAM,EAAE,CAAC;gBAAC,EAAE,CAAC,MAAM,CAAC,CAAC;gBAAC,OAAO;YAAC,CAAC;YACnC,KAAK,GAAG,IAAI,CAAC;YACb,IAAI,OAAO,KAAK,QAAQ;gBAAE,MAAM,CAAC,EAAE,CAAC,CAAC;;gBAChC,KAAK,GAAG,EAAE,CAAC;QACpB,CAAC;KACJ,CAAC,CAAC;IAEH,SAAS,QAAQ;QACb,OAAO,KAAK,CAAC,MAAM,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YACjC,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,EAAG,CAAC;YACtB,MAAM,GAAG,GAAG,KAAK,CAAC,KAAK,EAAG,CAAC;YAC3B,QAAQ,EAAE,CAAC;YACX,CAAC,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;QACvB,CAAC;IACL,CAAC;IAED,SAAS,QAAQ,CAAC,CAAS,EAAE,GAAQ;QACjC,IAAI,GAAG,CAAC,KAAK,EAAE,CAAC;YAAC,IAAI,CAAC,IAAI,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC;YAAC,OAAO;QAAC,CAAC;QACtD,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,EAAE,GAAG,CAAC,GAAG,CAAC,CAAC;QAC9B,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACb,QAAQ,EAAE,CAAC;QACX,OAAO,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC;YAC3B,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAE,CAAC;YACnC,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;YACzB,QAAQ,EAAE,CAAC;YACX,OAAO,EAAE,CAAC;YACV,IAAI,GAAG,CAAC,MAAM;gBAAE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACrC,CAAC;QACD,KAAK,IAAI,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC;QACpB,QAAQ,EAAE,CAAC;QACX,IAAI,QAAQ,IAAI,QAAQ,GAAG,GAAG,EAAE,CAAC;YAAC,MAAM,CAAC,GAAG,QAAQ,CAAC;YAAC,QAAQ,GAAG,IAAI,CAAC;YAAC,CAAC,EAAE,CAAC;QAAC,CAAC;QAC7E,IAAI,KAAK,IAAI,KAAK,IAAI,OAAO,KAAK,QAAQ,EAAE,CAAC;YAAC,MAAM,CAAC,GAAG,KAAK,CAAC;YAAC,KAAK,GAAG,IAAI,CAAC;YAAC,MAAM,CAAC,CAAC,CAAC,CAAC;QAAC,CAAC;IAC7F,CAAC;IAED,SAAS,IAAI,CAAC,GAAU;QACpB,IAAI,MAAM;YAAE,OAAO;QACnB,MAAM,GAAG,GAAG,CAAC;QACb,KAAK,MAAM,CAAC,IAAI,OAAO;YAAE,CAAC,CAAC,SAAS,EAAE,CAAC;QACvC,IAAI,QAAQ,EAAE,CAAC;YAAC,MAAM,CAAC,GAAG,QAAQ,CAAC;YAAC,QAAQ,GAAG,IAAI,CAAC;YAAC,CAAC,CAAC,GAAG,CAAC,CAAC;QAAC,CAAC;aACzD,IAAI,KAAK,EAAE,CAAC;YAAC,MAAM,CAAC,GAAG,KAAK,CAAC;YAAC,KAAK,GAAG,IAAI,CAAC;YAAC,CAAC,CAAC,GAAG,CAAC,CAAC;QAAC,CAAC;;YACrD,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IAC7B,CAAC;IAED,SAAS,MAAM,CAAC,EAAqB;QACjC,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;IACtF,CAAC;IAED,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QACzB,MAAM,CAAC,GAAG,IAAI,MAAM,CAAC,SAAS,EAAE,EAAE,UAAU,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;QACvG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAChB,CAAC,CAAC,EAAE,CAAC,SAAS,EAAE,CAAC,GAAQ,EAAE,EAAE;YACzB,IAAI,GAAG,IAAI,GAAG,CAAC,KAAK,EAAE,CAAC;gBAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;gBAAC,QAAQ,EAAE,CAAC;YAAC,CAAC;;gBAC9C,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;QAC1B,CAAC,CAAC,CAAC;QACH,CAAC,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;IACtC,CAAC;IAEA,MAAc,CAAC,QAAQ,GAAG,GAAG,EAAE,CAAC,KAAK,CAAC;IACvC,MAAM,CAAC,IAAI,CAAC,mBAAmB,CAAC,yBAAyB,CAAC,CAAC;IAC3D,OAAO,MAAM,CAAC;AAClB,CAAC"}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import { Transform } from 'stream';
|
|
2
|
+
export interface WorkerPoolOpts {
|
|
3
|
+
map: string;
|
|
4
|
+
param: any;
|
|
5
|
+
workers: number;
|
|
6
|
+
batchSize?: number;
|
|
7
|
+
timeoutMs?: number;
|
|
8
|
+
}
|
|
9
|
+
export declare function createWorkerPool(opts: WorkerPoolOpts): Transform;
|
|
10
|
+
//# sourceMappingURL=worker-pool.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"worker-pool.d.ts","sourceRoot":"","sources":["../../src/stream/worker-pool.ts"],"names":[],"mappings":"AAgBA,OAAO,EAAE,SAAS,EAA0B,MAAM,QAAQ,CAAC;AAO3D,MAAM,WAAW,cAAc;IAC3B,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,GAAG,CAAC;IACX,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,SAAS,CAAC,EAAE,MAAM,CAAC;CACtB;AAED,wBAAgB,gBAAgB,CAAC,IAAI,EAAE,cAAc,GAAG,SAAS,CA4IhE"}
|
|
@@ -0,0 +1,206 @@
|
|
|
1
|
+
// A Transform that parallelizes a per-record map across worker threads while
|
|
2
|
+
// preserving record order. Batches incoming records, dispatches batches to a
|
|
3
|
+
// pool of workers, re-orders the results by sequence number, and pushes them
|
|
4
|
+
// downstream in order.
|
|
5
|
+
//
|
|
6
|
+
// In (objectMode): record objects
|
|
7
|
+
// Out (objectMode): mapped record objects, in the original order
|
|
8
|
+
// (records the map rejected are dropped)
|
|
9
|
+
//
|
|
10
|
+
// Hardening over the first prototype:
|
|
11
|
+
// - worker count is clamped to the available parallelism;
|
|
12
|
+
// - real downstream backpressure: results are held in an out-queue and only
|
|
13
|
+
// pushed while push() accepts them; dispatch/accept pause when the output
|
|
14
|
+
// is backed up and resume on _read (so a slow sink can't grow memory);
|
|
15
|
+
// - a per-batch timeout fails the pipeline instead of hanging if a worker
|
|
16
|
+
// dies silently.
|
|
17
|
+
import { Transform } from 'stream';
|
|
18
|
+
import { Worker } from 'node:worker_threads';
|
|
19
|
+
import { availableParallelism } from 'node:os';
|
|
20
|
+
import log4js from 'log4js';
|
|
21
|
+
const logger = log4js.getLogger();
|
|
22
|
+
export function createWorkerPool(opts) {
|
|
23
|
+
const maxN = Math.max(1, availableParallelism());
|
|
24
|
+
const N = Math.min(Math.max(1, Math.floor(opts.workers) || 1), maxN);
|
|
25
|
+
if (N < opts.workers)
|
|
26
|
+
logger.warn(`--workers ${opts.workers} clamped to ${N} (available parallelism)`);
|
|
27
|
+
const BATCH = opts.batchSize ?? 64;
|
|
28
|
+
const CAP = N * 2; // max batches in flight
|
|
29
|
+
const OUT_CAP = CAP * BATCH; // max records buffered awaiting downstream
|
|
30
|
+
const TIMEOUT = opts.timeoutMs ?? 120_000;
|
|
31
|
+
const workerUrl = new URL('../worker.js', import.meta.url);
|
|
32
|
+
const workers = [];
|
|
33
|
+
const idle = [];
|
|
34
|
+
const timers = new Map();
|
|
35
|
+
const queue = [];
|
|
36
|
+
const reorder = new Map();
|
|
37
|
+
const outQueue = []; // mapped records ready to push, in order
|
|
38
|
+
let nextDispatch = 0;
|
|
39
|
+
let nextEmit = 0;
|
|
40
|
+
let inflight = 0;
|
|
41
|
+
let curBatch = [];
|
|
42
|
+
let acceptCb = null; // held _transform callback
|
|
43
|
+
let endCb = null;
|
|
44
|
+
let failed = null;
|
|
45
|
+
let closing = false; // we are terminating workers on purpose
|
|
46
|
+
const stream = new Transform({
|
|
47
|
+
objectMode: true,
|
|
48
|
+
transform(rec, _enc, cb) {
|
|
49
|
+
if (failed) {
|
|
50
|
+
cb(failed);
|
|
51
|
+
return;
|
|
52
|
+
}
|
|
53
|
+
curBatch.push(rec);
|
|
54
|
+
if (curBatch.length >= BATCH) {
|
|
55
|
+
enqueue(curBatch);
|
|
56
|
+
curBatch = [];
|
|
57
|
+
}
|
|
58
|
+
if (canAccept())
|
|
59
|
+
cb();
|
|
60
|
+
else
|
|
61
|
+
acceptCb = cb; // pause upstream until there's room
|
|
62
|
+
},
|
|
63
|
+
flush(cb) {
|
|
64
|
+
if (failed) {
|
|
65
|
+
cb(failed);
|
|
66
|
+
return;
|
|
67
|
+
}
|
|
68
|
+
if (curBatch.length) {
|
|
69
|
+
enqueue(curBatch);
|
|
70
|
+
curBatch = [];
|
|
71
|
+
}
|
|
72
|
+
if (done())
|
|
73
|
+
finish(cb);
|
|
74
|
+
else
|
|
75
|
+
endCb = cb;
|
|
76
|
+
}
|
|
77
|
+
});
|
|
78
|
+
// Resume producing when the downstream consumer asks for more.
|
|
79
|
+
const origRead = stream._read.bind(stream);
|
|
80
|
+
stream._read = (size) => { pump(); origRead(size); };
|
|
81
|
+
function canAccept() {
|
|
82
|
+
return inflight < CAP && outQueue.length < OUT_CAP;
|
|
83
|
+
}
|
|
84
|
+
function done() {
|
|
85
|
+
return nextEmit === nextDispatch && queue.length === 0 && outQueue.length === 0;
|
|
86
|
+
}
|
|
87
|
+
function enqueue(batch) {
|
|
88
|
+
queue.push({ seq: nextDispatch++, batch });
|
|
89
|
+
dispatch();
|
|
90
|
+
}
|
|
91
|
+
const dbg = () => logger.isDebugEnabled();
|
|
92
|
+
function dispatch() {
|
|
93
|
+
// don't start new work while the output is backed up
|
|
94
|
+
if (dbg() && queue.length && (!idle.length || outQueue.length >= OUT_CAP)) {
|
|
95
|
+
logger.debug(`backpressure: ${queue.length} batches queued, idle=${idle.length}, outQueue=${outQueue.length}/${OUT_CAP}`);
|
|
96
|
+
}
|
|
97
|
+
while (queue.length && idle.length && outQueue.length < OUT_CAP) {
|
|
98
|
+
const w = idle.pop();
|
|
99
|
+
const job = queue.shift();
|
|
100
|
+
inflight++;
|
|
101
|
+
timers.set(w, setTimeout(() => fail(new Error(`worker timed out after ${TIMEOUT}ms (batch ${job.seq})`)), TIMEOUT));
|
|
102
|
+
w.postMessage(job);
|
|
103
|
+
if (dbg())
|
|
104
|
+
logger.debug(`dispatch batch ${job.seq} (${job.batch.length} recs) -> worker; inflight=${inflight}`);
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
// push as many ready records as the downstream will take
|
|
108
|
+
function pump() {
|
|
109
|
+
const before = outQueue.length;
|
|
110
|
+
while (outQueue.length && stream.push(outQueue[0]))
|
|
111
|
+
outQueue.shift();
|
|
112
|
+
if (dbg() && outQueue.length > 0 && outQueue.length === before) {
|
|
113
|
+
logger.debug(`downstream full, holding ${outQueue.length} records`);
|
|
114
|
+
}
|
|
115
|
+
dispatch();
|
|
116
|
+
if (acceptCb && canAccept()) {
|
|
117
|
+
const c = acceptCb;
|
|
118
|
+
acceptCb = null;
|
|
119
|
+
c();
|
|
120
|
+
}
|
|
121
|
+
if (endCb && done()) {
|
|
122
|
+
const c = endCb;
|
|
123
|
+
endCb = null;
|
|
124
|
+
finish(c);
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
function onResult(w, msg) {
|
|
128
|
+
const t = timers.get(w);
|
|
129
|
+
if (t) {
|
|
130
|
+
clearTimeout(t);
|
|
131
|
+
timers.delete(w);
|
|
132
|
+
}
|
|
133
|
+
if (msg.error) {
|
|
134
|
+
logger.error(`worker error on batch ${msg.seq}: ${msg.error}`);
|
|
135
|
+
fail(new Error(msg.error));
|
|
136
|
+
return;
|
|
137
|
+
}
|
|
138
|
+
reorder.set(msg.seq, msg.mapped);
|
|
139
|
+
idle.push(w);
|
|
140
|
+
inflight--;
|
|
141
|
+
if (dbg())
|
|
142
|
+
logger.debug(`result batch ${msg.seq} (${msg.mapped.length} mapped); inflight=${inflight}, reorderBuf=${reorder.size}, wantNext=${nextEmit}`);
|
|
143
|
+
// move now-contiguous results into the out-queue (don't push yet)
|
|
144
|
+
while (reorder.has(nextEmit)) {
|
|
145
|
+
const mapped = reorder.get(nextEmit);
|
|
146
|
+
reorder.delete(nextEmit);
|
|
147
|
+
nextEmit++;
|
|
148
|
+
for (const m of mapped)
|
|
149
|
+
if (m !== null)
|
|
150
|
+
outQueue.push(m);
|
|
151
|
+
}
|
|
152
|
+
pump();
|
|
153
|
+
}
|
|
154
|
+
function fail(err) {
|
|
155
|
+
if (failed)
|
|
156
|
+
return;
|
|
157
|
+
failed = err;
|
|
158
|
+
closing = true;
|
|
159
|
+
logger.error(`worker pool failing: ${err.message} (terminating ${workers.length} workers)`);
|
|
160
|
+
for (const t of timers.values())
|
|
161
|
+
clearTimeout(t);
|
|
162
|
+
timers.clear();
|
|
163
|
+
for (const w of workers)
|
|
164
|
+
w.terminate();
|
|
165
|
+
if (acceptCb) {
|
|
166
|
+
const c = acceptCb;
|
|
167
|
+
acceptCb = null;
|
|
168
|
+
c(err);
|
|
169
|
+
}
|
|
170
|
+
else if (endCb) {
|
|
171
|
+
const c = endCb;
|
|
172
|
+
endCb = null;
|
|
173
|
+
c(err);
|
|
174
|
+
}
|
|
175
|
+
else
|
|
176
|
+
stream.destroy(err);
|
|
177
|
+
}
|
|
178
|
+
function finish(cb) {
|
|
179
|
+
closing = true;
|
|
180
|
+
for (const t of timers.values())
|
|
181
|
+
clearTimeout(t);
|
|
182
|
+
timers.clear();
|
|
183
|
+
logger.debug(`map complete: ${nextEmit} batches; terminating ${workers.length} workers`);
|
|
184
|
+
Promise.all(workers.map((w) => w.terminate())).then(() => cb()).catch(() => cb());
|
|
185
|
+
}
|
|
186
|
+
for (let i = 0; i < N; i++) {
|
|
187
|
+
const w = new Worker(workerUrl, { workerData: { map: opts.map, param: opts.param } });
|
|
188
|
+
workers.push(w);
|
|
189
|
+
w.on('message', (msg) => {
|
|
190
|
+
if (msg && msg.ready) {
|
|
191
|
+
idle.push(w);
|
|
192
|
+
dispatch();
|
|
193
|
+
if (dbg())
|
|
194
|
+
logger.debug(`worker ${idle.length}/${N} ready`);
|
|
195
|
+
}
|
|
196
|
+
else
|
|
197
|
+
onResult(w, msg);
|
|
198
|
+
});
|
|
199
|
+
w.on('error', (err) => { logger.error(`worker thread crashed: ${err.message}`); fail(err); });
|
|
200
|
+
w.on('exit', (code) => { if (code !== 0 && !failed && !closing)
|
|
201
|
+
fail(new Error(`worker exited unexpectedly (code ${code})`)); });
|
|
202
|
+
}
|
|
203
|
+
logger.info(`map running on ${N} worker threads (batch ${BATCH}, in-flight cap ${CAP}, timeout ${TIMEOUT}ms)`);
|
|
204
|
+
return stream;
|
|
205
|
+
}
|
|
206
|
+
//# sourceMappingURL=worker-pool.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"worker-pool.js","sourceRoot":"","sources":["../../src/stream/worker-pool.ts"],"names":[],"mappings":"AAAA,6EAA6E;AAC7E,6EAA6E;AAC7E,6EAA6E;AAC7E,uBAAuB;AACvB,EAAE;AACF,mCAAmC;AACnC,iEAAiE;AACjE,2DAA2D;AAC3D,EAAE;AACF,sCAAsC;AACtC,2DAA2D;AAC3D,6EAA6E;AAC7E,6EAA6E;AAC7E,0EAA0E;AAC1E,2EAA2E;AAC3E,oBAAoB;AACpB,OAAO,EAAE,SAAS,EAA0B,MAAM,QAAQ,CAAC;AAC3D,OAAO,EAAE,MAAM,EAAE,MAAM,qBAAqB,CAAC;AAC7C,OAAO,EAAE,oBAAoB,EAAE,MAAM,SAAS,CAAC;AAC/C,OAAO,MAAM,MAAM,QAAQ,CAAC;AAE5B,MAAM,MAAM,GAAG,MAAM,CAAC,SAAS,EAAE,CAAC;AAUlC,MAAM,UAAU,gBAAgB,CAAC,IAAoB;IACjD,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,oBAAoB,EAAE,CAAC,CAAC;IACjD,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;IACrE,IAAI,CAAC,GAAG,IAAI,CAAC,OAAO;QAAE,MAAM,CAAC,IAAI,CAAC,aAAa,IAAI,CAAC,OAAO,eAAe,CAAC,0BAA0B,CAAC,CAAC;IACvG,MAAM,KAAK,GAAG,IAAI,CAAC,SAAS,IAAI,EAAE,CAAC;IACnC,MAAM,GAAG,GAAG,CAAC,GAAG,CAAC,CAAC,CAAiB,wBAAwB;IAC3D,MAAM,OAAO,GAAG,GAAG,GAAG,KAAK,CAAC,CAAO,2CAA2C;IAC9E,MAAM,OAAO,GAAG,IAAI,CAAC,SAAS,IAAI,OAAO,CAAC;IAC1C,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,cAAc,EAAE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAE3D,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,MAAM,IAAI,GAAa,EAAE,CAAC;IAC1B,MAAM,MAAM,GAAG,IAAI,GAAG,EAA0B,CAAC;IACjD,MAAM,KAAK,GAAoC,EAAE,CAAC;IAClD,MAAM,OAAO,GAAG,IAAI,GAAG,EAAiB,CAAC;IACzC,MAAM,QAAQ,GAAU,EAAE,CAAC,CAAQ,yCAAyC;IAE5E,IAAI,YAAY,GAAG,CAAC,CAAC;IACrB,IAAI,QAAQ,GAAG,CAAC,CAAC;IACjB,IAAI,QAAQ,GAAG,CAAC,CAAC;IACjB,IAAI,QAAQ,GAAU,EAAE,CAAC;IACzB,IAAI,QAAQ,GAA6B,IAAI,CAAC,CAAG,2BAA2B;IAC5E,IAAI,KAAK,GAA6B,IAAI,CAAC;IAC3C,IAAI,MAAM,GAAiB,IAAI,CAAC;IAChC,IAAI,OAAO,GAAG,KAAK,CAAC,CAA8B,wCAAwC;IAE1F,MAAM,MAAM,GAAG,IAAI,SAAS,CAAC;QACzB,UAAU,EAAE,IAAI;QAChB,SAAS,CAAC,GAAQ,EAAE,IAAI,EAAE,EAAqB;YAC3C,IAAI,MAAM,EAAE,CAAC;gBAAC,EAAE,CAAC,MAAM,CAAC,CAAC;gBAAC,OAAO;YAAC,CAAC;YACnC,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACnB,IAAI,QAAQ,CAAC,MAAM,IAAI,KAAK,EAAE,CAAC;gBAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;gBAAC,QAAQ,GAAG,EAAE,CAAC;YAAC,CAAC;YACnE,IAAI,SAAS,EAAE;gBAAE,EAAE,EAAE,CAAC;;gBACjB,QAAQ,GAAG,EAAE,CAAC,CAAU,oCAAoC;QACrE,CAAC;QACD,KAAK,CAAC,EAAqB;YACvB,IAAI,MAAM,EAAE,CAAC;gBAAC,EAAE,CAAC,MAAM,CAAC,CAAC;gBAAC,OAAO;YAAC,CAAC;YACnC,IAAI,QAAQ,CAAC,MAAM,EAAE,CAAC;gBAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;gBAAC,QAAQ,GAAG,EAAE,CAAC;YAAC,CAAC;YAC1D,IAAI,IAAI,EAAE;gBAAE,MAAM,CAAC,EAAE,CAAC,CAAC;;gBAClB,KAAK,GAAG,EAAE,CAAC;QACpB,CAAC;KACJ,CAAC,CAAC;IAEH,+DAA+D;IAC/D,MAAM,QAAQ,GAAI,MAAc,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IACnD,MAAc,CAAC,KAAK,GAAG,CAAC,IAAY,EAAE,EAAE,GAAG,IAAI,EAAE,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;IAEtE,SAAS,SAAS;QACd,OAAO,QAAQ,GAAG,GAAG,IAAI,QAAQ,CAAC,MAAM,GAAG,OAAO,CAAC;IACvD,CAAC;IACD,SAAS,IAAI;QACT,OAAO,QAAQ,KAAK,YAAY,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,CAAC;IACpF,CAAC;IAED,SAAS,OAAO,CAAC,KAAY;QACzB,KAAK,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,YAAY,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC;QAC3C,QAAQ,EAAE,CAAC;IACf,CAAC;IAED,MAAM,GAAG,GAAG,GAAG,EAAE,CAAC,MAAM,CAAC,cAAc,EAAE,CAAC;IAE1C,SAAS,QAAQ;QACb,qDAAqD;QACrD,IAAI,GAAG,EAAE,IAAI,KAAK,CAAC,MAAM,IAAI,CAAC,CAAC,IAAI,CAAC,MAAM,IAAI,QAAQ,CAAC,MAAM,IAAI,OAAO,CAAC,EAAE,CAAC;YACxE,MAAM,CAAC,KAAK,CAAC,iBAAiB,KAAK,CAAC,MAAM,yBAAyB,IAAI,CAAC,MAAM,cAAc,QAAQ,CAAC,MAAM,IAAI,OAAO,EAAE,CAAC,CAAC;QAC9H,CAAC;QACD,OAAO,KAAK,CAAC,MAAM,IAAI,IAAI,CAAC,MAAM,IAAI,QAAQ,CAAC,MAAM,GAAG,OAAO,EAAE,CAAC;YAC9D,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,EAAG,CAAC;YACtB,MAAM,GAAG,GAAG,KAAK,CAAC,KAAK,EAAG,CAAC;YAC3B,QAAQ,EAAE,CAAC;YACX,MAAM,CAAC,GAAG,CAAC,CAAC,EAAE,UAAU,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,IAAI,KAAK,CAAC,0BAA0B,OAAO,aAAa,GAAG,CAAC,GAAG,GAAG,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC,CAAC;YACpH,CAAC,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;YACnB,IAAI,GAAG,EAAE;gBAAE,MAAM,CAAC,KAAK,CAAC,kBAAkB,GAAG,CAAC,GAAG,KAAK,GAAG,CAAC,KAAK,CAAC,MAAM,8BAA8B,QAAQ,EAAE,CAAC,CAAC;QACpH,CAAC;IACL,CAAC;IAED,yDAAyD;IACzD,SAAS,IAAI;QACT,MAAM,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAC;QAC/B,OAAO,QAAQ,CAAC,MAAM,IAAI,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;YAAE,QAAQ,CAAC,KAAK,EAAE,CAAC;QACrE,IAAI,GAAG,EAAE,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,IAAI,QAAQ,CAAC,MAAM,KAAK,MAAM,EAAE,CAAC;YAC7D,MAAM,CAAC,KAAK,CAAC,4BAA4B,QAAQ,CAAC,MAAM,UAAU,CAAC,CAAC;QACxE,CAAC;QACD,QAAQ,EAAE,CAAC;QACX,IAAI,QAAQ,IAAI,SAAS,EAAE,EAAE,CAAC;YAAC,MAAM,CAAC,GAAG,QAAQ,CAAC;YAAC,QAAQ,GAAG,IAAI,CAAC;YAAC,CAAC,EAAE,CAAC;QAAC,CAAC;QAC1E,IAAI,KAAK,IAAI,IAAI,EAAE,EAAE,CAAC;YAAC,MAAM,CAAC,GAAG,KAAK,CAAC;YAAC,KAAK,GAAG,IAAI,CAAC;YAAC,MAAM,CAAC,CAAC,CAAC,CAAC;QAAC,CAAC;IACtE,CAAC;IAED,SAAS,QAAQ,CAAC,CAAS,EAAE,GAAQ;QACjC,MAAM,CAAC,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;QACxB,IAAI,CAAC,EAAE,CAAC;YAAC,YAAY,CAAC,CAAC,CAAC,CAAC;YAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;QAAC,CAAC;QAC7C,IAAI,GAAG,CAAC,KAAK,EAAE,CAAC;YAAC,MAAM,CAAC,KAAK,CAAC,yBAAyB,GAAG,CAAC,GAAG,KAAK,GAAG,CAAC,KAAK,EAAE,CAAC,CAAC;YAAC,IAAI,CAAC,IAAI,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC;YAAC,OAAO;QAAC,CAAC;QACtH,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;QACjC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACb,QAAQ,EAAE,CAAC;QACX,IAAI,GAAG,EAAE;YAAE,MAAM,CAAC,KAAK,CAAC,gBAAgB,GAAG,CAAC,GAAG,KAAK,GAAG,CAAC,MAAM,CAAC,MAAM,sBAAsB,QAAQ,gBAAgB,OAAO,CAAC,IAAI,cAAc,QAAQ,EAAE,CAAC,CAAC;QACzJ,kEAAkE;QAClE,OAAO,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC;YAC3B,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAE,CAAC;YACtC,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;YACzB,QAAQ,EAAE,CAAC;YACX,KAAK,MAAM,CAAC,IAAI,MAAM;gBAAE,IAAI,CAAC,KAAK,IAAI;oBAAE,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAC7D,CAAC;QACD,IAAI,EAAE,CAAC;IACX,CAAC;IAED,SAAS,IAAI,CAAC,GAAU;QACpB,IAAI,MAAM;YAAE,OAAO;QACnB,MAAM,GAAG,GAAG,CAAC;QACb,OAAO,GAAG,IAAI,CAAC;QACf,MAAM,CAAC,KAAK,CAAC,wBAAwB,GAAG,CAAC,OAAO,iBAAiB,OAAO,CAAC,MAAM,WAAW,CAAC,CAAC;QAC5F,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,MAAM,EAAE;YAAE,YAAY,CAAC,CAAC,CAAC,CAAC;QACjD,MAAM,CAAC,KAAK,EAAE,CAAC;QACf,KAAK,MAAM,CAAC,IAAI,OAAO;YAAE,CAAC,CAAC,SAAS,EAAE,CAAC;QACvC,IAAI,QAAQ,EAAE,CAAC;YAAC,MAAM,CAAC,GAAG,QAAQ,CAAC;YAAC,QAAQ,GAAG,IAAI,CAAC;YAAC,CAAC,CAAC,GAAG,CAAC,CAAC;QAAC,CAAC;aACzD,IAAI,KAAK,EAAE,CAAC;YAAC,MAAM,CAAC,GAAG,KAAK,CAAC;YAAC,KAAK,GAAG,IAAI,CAAC;YAAC,CAAC,CAAC,GAAG,CAAC,CAAC;QAAC,CAAC;;YACrD,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IAC7B,CAAC;IAED,SAAS,MAAM,CAAC,EAAqB;QACjC,OAAO,GAAG,IAAI,CAAC;QACf,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,MAAM,EAAE;YAAE,YAAY,CAAC,CAAC,CAAC,CAAC;QACjD,MAAM,CAAC,KAAK,EAAE,CAAC;QACf,MAAM,CAAC,KAAK,CAAC,iBAAiB,QAAQ,yBAAyB,OAAO,CAAC,MAAM,UAAU,CAAC,CAAC;QACzF,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;IACtF,CAAC;IAED,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QACzB,MAAM,CAAC,GAAG,IAAI,MAAM,CAAC,SAAS,EAAE,EAAE,UAAU,EAAE,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;QACtF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAChB,CAAC,CAAC,EAAE,CAAC,SAAS,EAAE,CAAC,GAAQ,EAAE,EAAE;YACzB,IAAI,GAAG,IAAI,GAAG,CAAC,KAAK,EAAE,CAAC;gBAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;gBAAC,QAAQ,EAAE,CAAC;gBAAC,IAAI,GAAG,EAAE;oBAAE,MAAM,CAAC,KAAK,CAAC,UAAU,IAAI,CAAC,MAAM,IAAI,CAAC,QAAQ,CAAC,CAAC;YAAC,CAAC;;gBAC3G,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;QAC1B,CAAC,CAAC,CAAC;QACH,CAAC,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE,GAAG,MAAM,CAAC,KAAK,CAAC,0BAA0B,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAC9F,CAAC,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,EAAE,GAAG,IAAI,IAAI,KAAK,CAAC,IAAI,CAAC,MAAM,IAAI,CAAC,OAAO;YAAE,IAAI,CAAC,IAAI,KAAK,CAAC,oCAAoC,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACrI,CAAC;IAED,MAAM,CAAC,IAAI,CAAC,kBAAkB,CAAC,0BAA0B,KAAK,mBAAmB,GAAG,aAAa,OAAO,KAAK,CAAC,CAAC;IAC/G,OAAO,MAAM,CAAC;AAClB,CAAC"}
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
import { Transform } from 'stream';
|
|
2
|
+
/**
|
|
3
|
+
* Apply a Catmandu Fix script to each record.
|
|
4
|
+
*
|
|
5
|
+
* marcattacks --map fix --param fix=./my.fix ...
|
|
6
|
+
*
|
|
7
|
+
* `fix` may be a path to a Fix file or an inline Fix script. The script is
|
|
8
|
+
* parsed and compiled once; the compiled chain runs per record.
|
|
9
|
+
*/
|
|
10
|
+
export declare function createMapper(opts: {
|
|
11
|
+
fix?: string;
|
|
12
|
+
}): Promise<(data: any) => any>;
|
|
13
|
+
export declare function transform(opts: {
|
|
14
|
+
fix?: string;
|
|
15
|
+
}): Promise<Transform>;
|
|
16
|
+
//# sourceMappingURL=fix.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"fix.d.ts","sourceRoot":"","sources":["../../src/transform/fix.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,QAAQ,CAAC;AAOnC;;;;;;;GAOG;AAGH,wBAAsB,YAAY,CAAC,IAAI,EAAE;IAAE,GAAG,CAAC,EAAE,MAAM,CAAA;CAAE,GAAG,OAAO,CAAC,CAAC,IAAI,EAAE,GAAG,KAAK,GAAG,CAAC,CAMtF;AAED,wBAAsB,SAAS,CAAC,IAAI,EAAE;IAAE,GAAG,CAAC,EAAE,MAAM,CAAA;CAAE,GAAG,OAAO,CAAC,SAAS,CAAC,CAoB1E"}
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
import { Transform } from 'stream';
|
|
2
|
+
import fs from 'fs';
|
|
3
|
+
import log4js from 'log4js';
|
|
4
|
+
import { compileFix, REJECT } from '../fix/index.js';
|
|
5
|
+
const logger = log4js.getLogger();
|
|
6
|
+
/**
|
|
7
|
+
* Apply a Catmandu Fix script to each record.
|
|
8
|
+
*
|
|
9
|
+
* marcattacks --map fix --param fix=./my.fix ...
|
|
10
|
+
*
|
|
11
|
+
* `fix` may be a path to a Fix file or an inline Fix script. The script is
|
|
12
|
+
* parsed and compiled once; the compiled chain runs per record.
|
|
13
|
+
*/
|
|
14
|
+
// Pure record -> record mapper (returns REJECT for dropped records). Shared by
|
|
15
|
+
// transform() and the worker pool. Compiled once.
|
|
16
|
+
export async function createMapper(opts) {
|
|
17
|
+
let src = opts.fix ?? '';
|
|
18
|
+
if (src && fs.existsSync(src)) {
|
|
19
|
+
src = fs.readFileSync(src, { encoding: 'utf-8' });
|
|
20
|
+
}
|
|
21
|
+
return compileFix(src);
|
|
22
|
+
}
|
|
23
|
+
export async function transform(opts) {
|
|
24
|
+
let src = opts.fix ?? '';
|
|
25
|
+
if (src && fs.existsSync(src)) {
|
|
26
|
+
src = fs.readFileSync(src, { encoding: 'utf-8' });
|
|
27
|
+
}
|
|
28
|
+
const fix = compileFix(src);
|
|
29
|
+
return new Transform({
|
|
30
|
+
objectMode: true,
|
|
31
|
+
transform(data, _encoding, callback) {
|
|
32
|
+
try {
|
|
33
|
+
const out = fix(data);
|
|
34
|
+
if (out === REJECT)
|
|
35
|
+
callback(); // rejected record -> drop it
|
|
36
|
+
else
|
|
37
|
+
callback(null, out);
|
|
38
|
+
}
|
|
39
|
+
catch (err) {
|
|
40
|
+
logger.error('fix error', err.message);
|
|
41
|
+
callback(err);
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
});
|
|
45
|
+
}
|
|
46
|
+
//# sourceMappingURL=fix.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"fix.js","sourceRoot":"","sources":["../../src/transform/fix.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,QAAQ,CAAC;AACnC,OAAO,EAAE,MAAM,IAAI,CAAC;AACpB,OAAO,MAAM,MAAM,QAAQ,CAAC;AAC5B,OAAO,EAAE,UAAU,EAAE,MAAM,EAAE,MAAM,iBAAiB,CAAC;AAErD,MAAM,MAAM,GAAG,MAAM,CAAC,SAAS,EAAE,CAAC;AAElC;;;;;;;GAOG;AACH,+EAA+E;AAC/E,kDAAkD;AAClD,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,IAAsB;IACrD,IAAI,GAAG,GAAG,IAAI,CAAC,GAAG,IAAI,EAAE,CAAC;IACzB,IAAI,GAAG,IAAI,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;QAC5B,GAAG,GAAG,EAAE,CAAC,YAAY,CAAC,GAAG,EAAE,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC,CAAC;IACtD,CAAC;IACD,OAAO,UAAU,CAAC,GAAG,CAAC,CAAC;AAC3B,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,IAAsB;IAClD,IAAI,GAAG,GAAG,IAAI,CAAC,GAAG,IAAI,EAAE,CAAC;IACzB,IAAI,GAAG,IAAI,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;QAC5B,GAAG,GAAG,EAAE,CAAC,YAAY,CAAC,GAAG,EAAE,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC,CAAC;IACtD,CAAC;IACD,MAAM,GAAG,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC;IAE5B,OAAO,IAAI,SAAS,CAAC;QACjB,UAAU,EAAE,IAAI;QAChB,SAAS,CAAC,IAAS,EAAE,SAAS,EAAE,QAAQ;YACpC,IAAI,CAAC;gBACD,MAAM,GAAG,GAAG,GAAG,CAAC,IAAI,CAAC,CAAC;gBACtB,IAAI,GAAG,KAAK,MAAM;oBAAE,QAAQ,EAAE,CAAC,CAAG,6BAA6B;;oBAC1D,QAAQ,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;YAC7B,CAAC;YAAC,OAAO,GAAQ,EAAE,CAAC;gBAChB,MAAM,CAAC,KAAK,CAAC,WAAW,EAAE,GAAG,CAAC,OAAO,CAAC,CAAC;gBACvC,QAAQ,CAAC,GAAG,CAAC,CAAC;YAClB,CAAC;QACL,CAAC;KACJ,CAAC,CAAC;AACP,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"jsonata.d.ts","sourceRoot":"","sources":["../../src/transform/jsonata.ts"],"names":[],"mappings":"AAAA,OAAe,EAAE,SAAS,EAAE,MAAM,QAAQ,CAAC;
|
|
1
|
+
{"version":3,"file":"jsonata.d.ts","sourceRoot":"","sources":["../../src/transform/jsonata.ts"],"names":[],"mappings":"AAAA,OAAe,EAAE,SAAS,EAAE,MAAM,QAAQ,CAAC;AAa3C,wBAAsB,YAAY,CAAC,IAAI,EAAE;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,GAAI,OAAO,CAAC,CAAC,IAAI,EAAE,GAAG,KAAK,GAAG,CAAC,CA0CtG;AAED,wBAAsB,SAAS,CAAC,IAAI,EAAE;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,GAAI,OAAO,CAAC,SAAS,CAAC,CAe1F"}
|
|
@@ -6,58 +6,54 @@ import { parseStream } from '../util/tsv_parse.js';
|
|
|
6
6
|
import { v4 as uuidv4 } from 'uuid';
|
|
7
7
|
import log4js from 'log4js';
|
|
8
8
|
const logger = log4js.getLogger();
|
|
9
|
-
|
|
10
|
-
|
|
9
|
+
// Build a pure record -> record(Promise) mapper. Shared by the in-process
|
|
10
|
+
// transform() and by the worker pool (so the heavy evaluate() can run on
|
|
11
|
+
// worker threads). Expression + helper functions are compiled once.
|
|
12
|
+
export async function createMapper(opts) {
|
|
11
13
|
let lookup = {};
|
|
12
14
|
if (opts.lookup) {
|
|
13
15
|
lookup = await loadLookup(opts.lookup);
|
|
14
16
|
}
|
|
15
|
-
|
|
17
|
+
// Resolve the query once, up front, instead of on every record.
|
|
18
|
+
let query;
|
|
19
|
+
if (opts.fix) {
|
|
20
|
+
if (fs.existsSync(opts.fix)) {
|
|
21
|
+
query = fs.readFileSync(opts.fix, { encoding: 'utf-8' });
|
|
22
|
+
}
|
|
23
|
+
else {
|
|
24
|
+
throw Error(`no such file ${opts.fix}`);
|
|
25
|
+
}
|
|
26
|
+
}
|
|
27
|
+
else {
|
|
28
|
+
query = '$';
|
|
29
|
+
}
|
|
30
|
+
logger.debug(query);
|
|
31
|
+
// The identity expression is a pure pass-through: skip jsonata entirely.
|
|
32
|
+
if (query.trim() === '$') {
|
|
33
|
+
return (data) => data;
|
|
34
|
+
}
|
|
35
|
+
// Compile the expression and register helper functions ONCE. The helpers
|
|
36
|
+
// read the record currently being processed via `current`; each evaluate
|
|
37
|
+
// is awaited before the next, so this is safe.
|
|
38
|
+
let current;
|
|
39
|
+
const expression = jsonata(query);
|
|
40
|
+
expression.registerFunction('marcmap', (code) => marcmap(current['record'], code, {}));
|
|
41
|
+
expression.registerFunction('marctag', (row) => marctag(row));
|
|
42
|
+
expression.registerFunction('marcind', (row) => marcind(row));
|
|
43
|
+
expression.registerFunction('marcsubfields', (row, regex) => marcsubfields(row, new RegExp(regex)));
|
|
44
|
+
expression.registerFunction('marcrecord', () => current['record']);
|
|
45
|
+
expression.registerFunction('asmarc', (data) => ({ "record": data }));
|
|
46
|
+
expression.registerFunction('genid', () => genid());
|
|
47
|
+
expression.registerFunction('lookup', (key) => lookup[key]);
|
|
48
|
+
return async (data) => { current = data; return expression.evaluate(data); };
|
|
49
|
+
}
|
|
50
|
+
export async function transform(opts) {
|
|
51
|
+
const mapper = await createMapper(opts);
|
|
16
52
|
return new Transform({
|
|
17
53
|
objectMode: true,
|
|
18
54
|
async transform(data, _encoding, callback) {
|
|
19
55
|
try {
|
|
20
|
-
|
|
21
|
-
if (opts.fix) {
|
|
22
|
-
if (fs.existsSync(opts.fix)) {
|
|
23
|
-
query = fs.readFileSync(opts.fix, { encoding: 'utf-8' });
|
|
24
|
-
}
|
|
25
|
-
else {
|
|
26
|
-
throw Error(`no such file ${opts.fix}`);
|
|
27
|
-
}
|
|
28
|
-
}
|
|
29
|
-
else {
|
|
30
|
-
query = '$';
|
|
31
|
-
}
|
|
32
|
-
logger.debug(query);
|
|
33
|
-
}
|
|
34
|
-
const expression = jsonata(query);
|
|
35
|
-
expression.registerFunction('marcmap', (code) => {
|
|
36
|
-
return marcmap(data['record'], code, {});
|
|
37
|
-
});
|
|
38
|
-
expression.registerFunction('marctag', (row) => {
|
|
39
|
-
return marctag(row);
|
|
40
|
-
});
|
|
41
|
-
expression.registerFunction('marcind', (row) => {
|
|
42
|
-
return marcind(row);
|
|
43
|
-
});
|
|
44
|
-
expression.registerFunction('marcsubfields', (row, regex) => {
|
|
45
|
-
return marcsubfields(row, new RegExp(regex));
|
|
46
|
-
});
|
|
47
|
-
expression.registerFunction('marcrecord', () => {
|
|
48
|
-
return data['record'];
|
|
49
|
-
});
|
|
50
|
-
expression.registerFunction('asmarc', (data) => {
|
|
51
|
-
return { "record": data };
|
|
52
|
-
});
|
|
53
|
-
expression.registerFunction('genid', () => {
|
|
54
|
-
return genid();
|
|
55
|
-
});
|
|
56
|
-
expression.registerFunction('lookup', (key) => {
|
|
57
|
-
return lookup[key];
|
|
58
|
-
});
|
|
59
|
-
data = await expression.evaluate(data);
|
|
60
|
-
callback(null, data);
|
|
56
|
+
callback(null, await mapper(data));
|
|
61
57
|
}
|
|
62
58
|
catch (err) {
|
|
63
59
|
logger.info(err);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"jsonata.js","sourceRoot":"","sources":["../../src/transform/jsonata.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,EAAE,EAAE,SAAS,EAAE,MAAM,QAAQ,CAAC;AAC3C,OAAO,OAAO,MAAM,SAAS,CAAC;AAC9B,OAAO,EAAE,MAAM,IAAI,CAAC;AACpB,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AACzE,OAAO,EAAE,WAAW,EAAE,MAAM,sBAAsB,CAAC;AACnD,OAAO,EAAE,EAAE,IAAI,MAAM,EAAE,MAAM,MAAM,CAAC;AACpC,OAAO,MAAM,MAAM,QAAQ,CAAC;AAE5B,MAAM,MAAM,GAAG,MAAM,CAAC,SAAS,EAAE,CAAC;AAElC,MAAM,CAAC,KAAK,UAAU,
|
|
1
|
+
{"version":3,"file":"jsonata.js","sourceRoot":"","sources":["../../src/transform/jsonata.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,EAAE,EAAE,SAAS,EAAE,MAAM,QAAQ,CAAC;AAC3C,OAAO,OAAO,MAAM,SAAS,CAAC;AAC9B,OAAO,EAAE,MAAM,IAAI,CAAC;AACpB,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AACzE,OAAO,EAAE,WAAW,EAAE,MAAM,sBAAsB,CAAC;AACnD,OAAO,EAAE,EAAE,IAAI,MAAM,EAAE,MAAM,MAAM,CAAC;AACpC,OAAO,MAAM,MAAM,QAAQ,CAAC;AAE5B,MAAM,MAAM,GAAG,MAAM,CAAC,SAAS,EAAE,CAAC;AAElC,0EAA0E;AAC1E,yEAAyE;AACzE,oEAAoE;AACpE,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,IAAqC;IACpE,IAAI,MAAM,GAA2B,EAAE,CAAC;IAExC,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;QACd,MAAM,GAAG,MAAM,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAC3C,CAAC;IAED,gEAAgE;IAChE,IAAI,KAAc,CAAC;IACnB,IAAI,IAAI,CAAC,GAAG,EAAE,CAAC;QACX,IAAI,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;YAC1B,KAAK,GAAG,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,GAAG,EAAC,EAAE,QAAQ,EAAE,OAAO,EAAC,CAAC,CAAC;QAC3D,CAAC;aACI,CAAC;YACF,MAAM,KAAK,CAAC,gBAAgB,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;QAC5C,CAAC;IACL,CAAC;SACI,CAAC;QACF,KAAK,GAAG,GAAG,CAAC;IAChB,CAAC;IACD,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IAEpB,yEAAyE;IACzE,IAAI,KAAK,CAAC,IAAI,EAAE,KAAK,GAAG,EAAE,CAAC;QACvB,OAAO,CAAC,IAAS,EAAE,EAAE,CAAC,IAAI,CAAC;IAC/B,CAAC;IAED,yEAAyE;IACzE,yEAAyE;IACzE,+CAA+C;IAC/C,IAAI,OAAY,CAAC;IACjB,MAAM,UAAU,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC;IAClC,UAAU,CAAC,gBAAgB,CAAC,SAAS,EAAE,CAAC,IAAY,EAAE,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,IAAI,EAAE,EAAE,CAAC,CAAC,CAAC;IAC/F,UAAU,CAAC,gBAAgB,CAAC,SAAS,EAAE,CAAC,GAAa,EAAE,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC;IACxE,UAAU,CAAC,gBAAgB,CAAC,SAAS,EAAE,CAAC,GAAa,EAAE,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC;IACxE,UAAU,CAAC,gBAAgB,CAAC,eAAe,EAAE,CAAC,GAAa,EAAE,KAAa,EAAE,EAAE,CAAC,aAAa,CAAC,GAAG,EAAE,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IACtH,UAAU,CAAC,gBAAgB,CAAC,YAAY,EAAE,GAAG,EAAE,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC;IACnE,UAAU,CAAC,gBAAgB,CAAC,QAAQ,EAAE,CAAC,IAAgB,EAAE,EAAE,CAAC,CAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;IAClF,UAAU,CAAC,gBAAgB,CAAC,OAAO,EAAE,GAAG,EAAE,CAAC,KAAK,EAAE,CAAC,CAAC;IACpD,UAAU,CAAC,gBAAgB,CAAC,QAAQ,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;IAE5D,OAAO,KAAK,EAAE,IAAS,EAAE,EAAE,GAAG,OAAO,GAAG,IAAI,CAAC,CAAC,OAAO,UAAU,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;AACtF,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,IAAqC;IACjE,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC,IAAI,CAAC,CAAC;IAExC,OAAO,IAAI,SAAS,CAAC;QACjB,UAAU,EAAE,IAAI;QAChB,KAAK,CAAC,SAAS,CAAC,IAAS,EAAE,SAAyB,EAAE,QAAkC;YACpF,IAAI,CAAC;gBACD,QAAQ,CAAC,IAAI,EAAE,MAAM,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC;YACvC,CAAC;YACD,OAAO,GAAQ,EAAE,CAAC;gBACd,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBACjB,QAAQ,CAAC,GAAY,CAAC,CAAC;YAC3B,CAAC;QACL,CAAC;KACJ,CAAC,CAAC;AACP,CAAC;AAED,SAAS,KAAK;IACV,OAAO,SAAS,MAAM,EAAE,EAAE,CAAC;AAC/B,CAAC;AAED,KAAK,UAAU,UAAU,CAAC,IAAY;IAClC,IAAI,MAAM,GAA4B,EAAE,CAAC;IAEzC,MAAM,OAAO,GAAG,MAAM,WAAW,CAAC,EAAE,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC,CAAC;IAE7D,KAAK,MAAM,GAAG,IAAI,OAAO,EAAE,CAAC;QACxB,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;QACrC,IAAI,IAAI,IAAI,IAAI,CAAC,MAAM,IAAI,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC;YACjD,MAAM,CAAC,GAAG,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;YACvB,MAAM,CAAC,GAAG,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;YACvB,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;gBACT,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;YAClB,CAAC;QACL,CAAC;IACL,CAAC;IAED,OAAO,MAAM,CAAC;AAClB,CAAC"}
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { Transform } from "stream";
|
|
2
2
|
export interface MarcInRDFOptions {
|
|
3
|
-
parse?: "jsonld" | "turtle" | "
|
|
3
|
+
parse?: "jsonld" | "turtle" | "field" | "full";
|
|
4
4
|
}
|
|
5
5
|
export declare function transform(opts?: MarcInRDFOptions): Promise<Transform>;
|
|
6
6
|
//# sourceMappingURL=marcinrdf.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"marcinrdf.d.ts","sourceRoot":"","sources":["../../src/transform/marcinrdf.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,QAAQ,CAAC;AAKnC,MAAM,WAAW,gBAAgB;IAC7B,KAAK,CAAC,EAAE,QAAQ,GAAG,QAAQ,GAAG,
|
|
1
|
+
{"version":3,"file":"marcinrdf.d.ts","sourceRoot":"","sources":["../../src/transform/marcinrdf.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,QAAQ,CAAC;AAKnC,MAAM,WAAW,gBAAgB;IAC7B,KAAK,CAAC,EAAE,QAAQ,GAAG,QAAQ,GAAG,OAAO,GAAG,MAAM,CAAC;CAClD;AAID,wBAAsB,SAAS,CAAC,IAAI,GAAE,gBAAqB,GAAI,OAAO,CAAC,SAAS,CAAC,CAQhF"}
|
|
@@ -36,9 +36,15 @@ async function makeRdfData(data, opts = {}) {
|
|
|
36
36
|
if (parse === "turtle") {
|
|
37
37
|
return await parseString(JSON.stringify(clone), "data.jsonld");
|
|
38
38
|
}
|
|
39
|
-
else if (parse === "
|
|
39
|
+
else if (parse === "field") {
|
|
40
40
|
const record = {
|
|
41
|
-
"text":
|
|
41
|
+
"text": serializeFieldRecord(clone)
|
|
42
|
+
};
|
|
43
|
+
return record;
|
|
44
|
+
}
|
|
45
|
+
else if (parse === "full") {
|
|
46
|
+
const record = {
|
|
47
|
+
"text": serializeFullRecord(clone)
|
|
42
48
|
};
|
|
43
49
|
return record;
|
|
44
50
|
}
|
|
@@ -46,7 +52,7 @@ async function makeRdfData(data, opts = {}) {
|
|
|
46
52
|
return clone;
|
|
47
53
|
}
|
|
48
54
|
}
|
|
49
|
-
function
|
|
55
|
+
function serializeFieldRecord(record) {
|
|
50
56
|
const id = record['@id'];
|
|
51
57
|
const ex = "http://example.org/ns#";
|
|
52
58
|
let result = `<${id}> a <${ex}Record>.\n`;
|
|
@@ -58,6 +64,13 @@ function serializeRecord(record) {
|
|
|
58
64
|
}
|
|
59
65
|
return result.trim();
|
|
60
66
|
}
|
|
67
|
+
function serializeFullRecord(record) {
|
|
68
|
+
const id = record['@id'];
|
|
69
|
+
const ex = "http://example.org/ns#";
|
|
70
|
+
let result = `<${id}> a <${ex}Record>.\n`;
|
|
71
|
+
result += `<${id}> <${ex}record> ${serializeArray(record['record'])}.\n`;
|
|
72
|
+
return result.trim();
|
|
73
|
+
}
|
|
61
74
|
function serializeArray(array) {
|
|
62
75
|
const result = "(" + array.map(x => {
|
|
63
76
|
if (Array.isArray(x)) {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"marcinrdf.js","sourceRoot":"","sources":["../../src/transform/marcinrdf.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,QAAQ,CAAC;AACnC,OAAO,EAAe,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AACzD,OAAO,EAAE,WAAW,EAAE,MAAM,sBAAsB,CAAC;AACnD,OAAO,EAAE,OAAO,EAAE,MAAM,eAAe,CAAC;AAMxC,+DAA+D;AAC/D,sCAAsC;AACtC,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,OAAyB,EAAE;IACvD,OAAO,IAAI,SAAS,CAAC;QACjB,UAAU,EAAE,IAAI;QAChB,KAAK,CAAC,SAAS,CAAC,IAAS,EAAE,SAAS,EAAE,QAAQ;YAC1C,MAAM,OAAO,GAAG,MAAM,WAAW,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;YAC9C,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;QAC5B,CAAC;KACJ,CAAC,CAAC;AACP,CAAC;AAED,KAAK,UAAU,WAAW,CAAC,IAAS,EAAE,OAAyB,EAAE;IAC7D,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,QAAQ,CAAC;IAEjD,kDAAkD;IAClD,sCAAsC;IACtC,IAAI,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;QACjB,OAAO,IAAI,CAAC;IAChB,CAAC;IAED,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAC,KAAK,EAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;IAE/C,MAAM,KAAK,GAAG,eAAe,CAAC,IAAI,CAAC,CAAC;IAEpC,8BAA8B;IAC9B,KAAK,CAAC,UAAU,CAAC,GAAG;QAChB,QAAQ,EAAE,wBAAwB;QAClC,IAAI,EAAE,wBAAwB;QAC9B,QAAQ,EAAE;YACN,KAAK,EAAE,WAAW;YAClB,YAAY,EAAE,OAAO;SACxB;KACJ,CAAC;IAEF,KAAK,CAAC,KAAK,CAAC,GAAG,6BAA6B,EAAE,EAAE,CAAE;IAClD,KAAK,CAAC,OAAO,CAAC,GAAG,WAAW,CAAC;IAE7B,IAAI,KAAK,KAAK,QAAQ,EAAE,CAAC;QACrB,OAAO,MAAM,WAAW,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,EAAE,aAAa,CAAC,CAAC;IACnE,CAAC;SACI,IAAI,KAAK,KAAK,
|
|
1
|
+
{"version":3,"file":"marcinrdf.js","sourceRoot":"","sources":["../../src/transform/marcinrdf.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,QAAQ,CAAC;AACnC,OAAO,EAAe,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AACzD,OAAO,EAAE,WAAW,EAAE,MAAM,sBAAsB,CAAC;AACnD,OAAO,EAAE,OAAO,EAAE,MAAM,eAAe,CAAC;AAMxC,+DAA+D;AAC/D,sCAAsC;AACtC,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,OAAyB,EAAE;IACvD,OAAO,IAAI,SAAS,CAAC;QACjB,UAAU,EAAE,IAAI;QAChB,KAAK,CAAC,SAAS,CAAC,IAAS,EAAE,SAAS,EAAE,QAAQ;YAC1C,MAAM,OAAO,GAAG,MAAM,WAAW,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;YAC9C,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;QAC5B,CAAC;KACJ,CAAC,CAAC;AACP,CAAC;AAED,KAAK,UAAU,WAAW,CAAC,IAAS,EAAE,OAAyB,EAAE;IAC7D,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,QAAQ,CAAC;IAEjD,kDAAkD;IAClD,sCAAsC;IACtC,IAAI,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;QACjB,OAAO,IAAI,CAAC;IAChB,CAAC;IAED,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAC,KAAK,EAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;IAE/C,MAAM,KAAK,GAAG,eAAe,CAAC,IAAI,CAAC,CAAC;IAEpC,8BAA8B;IAC9B,KAAK,CAAC,UAAU,CAAC,GAAG;QAChB,QAAQ,EAAE,wBAAwB;QAClC,IAAI,EAAE,wBAAwB;QAC9B,QAAQ,EAAE;YACN,KAAK,EAAE,WAAW;YAClB,YAAY,EAAE,OAAO;SACxB;KACJ,CAAC;IAEF,KAAK,CAAC,KAAK,CAAC,GAAG,6BAA6B,EAAE,EAAE,CAAE;IAClD,KAAK,CAAC,OAAO,CAAC,GAAG,WAAW,CAAC;IAE7B,IAAI,KAAK,KAAK,QAAQ,EAAE,CAAC;QACrB,OAAO,MAAM,WAAW,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,EAAE,aAAa,CAAC,CAAC;IACnE,CAAC;SACI,IAAI,KAAK,KAAK,OAAO,EAAE,CAAC;QACzB,MAAM,MAAM,GAAS;YACjB,MAAM,EAAG,oBAAoB,CAAC,KAAK,CAAC;SACvC,CAAC;QACF,OAAO,MAAM,CAAC;IAClB,CAAC;SACI,IAAI,KAAK,KAAK,MAAM,EAAE,CAAC;QACxB,MAAM,MAAM,GAAS;YACjB,MAAM,EAAG,mBAAmB,CAAC,KAAK,CAAC;SACtC,CAAC;QACF,OAAO,MAAM,CAAC;IAClB,CAAC;SACI,CAAC;QACF,OAAO,KAAK,CAAC;IACjB,CAAC;AACL,CAAC;AAED,SAAS,oBAAoB,CAAC,MAAW;IACrC,MAAM,EAAE,GAAG,MAAM,CAAE,KAAK,CAAC,CAAC;IAC1B,MAAM,EAAE,GAAG,wBAAwB,CAAC;IACpC,IAAI,MAAM,GAAG,IAAI,EAAE,QAAQ,EAAE,YAAY,CAAC;IAC1C,MAAM,MAAM,GAAG,MAAM,CAAC,QAAQ,CAAC,CAAC;IAEhC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAG,CAAC,GAAG,MAAM,CAAC,MAAM,EAAG,CAAC,EAAE,EAAE,CAAC;QACvC,MAAM,GAAG,GAAI,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAC1B,MAAM,IAAI,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;QACjC,MAAM,IAAI,IAAI,EAAE,MAAM,EAAE,IAAI,GAAG,KAAK,cAAc,CAAC,IAAI,CAAC,KAAK,CAAC;IAClE,CAAC;IAED,OAAO,MAAM,CAAC,IAAI,EAAE,CAAC;AACzB,CAAC;AAED,SAAS,mBAAmB,CAAC,MAAW;IACpC,MAAM,EAAE,GAAG,MAAM,CAAE,KAAK,CAAC,CAAC;IAC1B,MAAM,EAAE,GAAG,wBAAwB,CAAC;IACpC,IAAI,MAAM,GAAG,IAAI,EAAE,QAAQ,EAAE,YAAY,CAAC;IAE1C,MAAM,IAAI,IAAI,EAAE,MAAM,EAAE,WAAW,cAAc,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,KAAK,CAAC;IAEzE,OAAO,MAAM,CAAC,IAAI,EAAE,CAAC;AACzB,CAAC;AAGD,SAAS,cAAc,CAAC,KAAY;IAChC,MAAM,MAAM,GAAY,GAAG,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE;QACxC,IAAI,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC;YACnB,OAAO,cAAc,CAAC,CAAC,CAAC,CAAC;QAC7B,CAAC;aACI,CAAC;YACF,OAAO,cAAc,CAAC,CAAC,CAAC,CAAC;QAC7B,CAAC;IACL,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,GAAG,CAAC;IACnB,OAAO,MAAM,CAAC;AAClB,CAAC;AAED,SAAS,cAAc,CAAC,KAAU;IAC9B,MAAM,MAAM,GAAG,IAAI;QACf,KAAK,CAAC,UAAU,CAAC,KAAK,EAAC,MAAM,CAAC,CAAC,UAAU,CAAC,IAAI,EAAC,MAAM,CAAC;QACtD,IAAI,CAAC;IACT,OAAO,MAAM,CAAC;AAClB,CAAC"}
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Marker for record objects whose string values are already free of control
|
|
3
|
+
* characters (\x00-\x1F and \x7F).
|
|
4
|
+
*
|
|
5
|
+
* Input readers that strip control characters while parsing (e.g. the XML
|
|
6
|
+
* readers) set this on the records they emit. Output writers that would
|
|
7
|
+
* otherwise re-strip every value (e.g. alephseq) can then skip that work.
|
|
8
|
+
*
|
|
9
|
+
* It is a Symbol so it never shows up in `JSON.stringify`, `Object.keys` or
|
|
10
|
+
* `for...in` — outputs that serialise the whole record object (json, jsonl,
|
|
11
|
+
* rdf) are unaffected. A transform that builds a new record object (e.g. a
|
|
12
|
+
* JSONata `fix`) naturally drops the marker, so escaping is re-applied unless
|
|
13
|
+
* cleanliness is re-asserted — a safe default.
|
|
14
|
+
*/
|
|
15
|
+
export declare const CLEAN: unique symbol;
|
|
16
|
+
export type CleanFlag = {
|
|
17
|
+
[CLEAN]?: boolean;
|
|
18
|
+
};
|
|
19
|
+
//# sourceMappingURL=marc_record.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"marc_record.d.ts","sourceRoot":"","sources":["../../src/util/marc_record.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AACH,eAAO,MAAM,KAAK,EAAE,OAAO,MAAoC,CAAC;AAEhE,MAAM,MAAM,SAAS,GAAG;IAAE,CAAC,KAAK,CAAC,CAAC,EAAE,OAAO,CAAA;CAAE,CAAC"}
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Marker for record objects whose string values are already free of control
|
|
3
|
+
* characters (\x00-\x1F and \x7F).
|
|
4
|
+
*
|
|
5
|
+
* Input readers that strip control characters while parsing (e.g. the XML
|
|
6
|
+
* readers) set this on the records they emit. Output writers that would
|
|
7
|
+
* otherwise re-strip every value (e.g. alephseq) can then skip that work.
|
|
8
|
+
*
|
|
9
|
+
* It is a Symbol so it never shows up in `JSON.stringify`, `Object.keys` or
|
|
10
|
+
* `for...in` — outputs that serialise the whole record object (json, jsonl,
|
|
11
|
+
* rdf) are unaffected. A transform that builds a new record object (e.g. a
|
|
12
|
+
* JSONata `fix`) naturally drops the marker, so escaping is re-applied unless
|
|
13
|
+
* cleanliness is re-asserted — a safe default.
|
|
14
|
+
*/
|
|
15
|
+
export const CLEAN = Symbol('marc.record.clean');
|
|
16
|
+
//# sourceMappingURL=marc_record.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"marc_record.js","sourceRoot":"","sources":["../../src/util/marc_record.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AACH,MAAM,CAAC,MAAM,KAAK,GAAkB,MAAM,CAAC,mBAAmB,CAAC,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"worker-str.d.ts","sourceRoot":"","sources":["../src/worker-str.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"worker-str.js","sourceRoot":"","sources":["../src/worker-str.ts"],"names":[],"mappings":"AAAA,gFAAgF;AAChF,8EAA8E;AAC9E,4EAA4E;AAC5E,EAAE;AACF,+EAA+E;AAC/E,YAAY;AACZ,0EAA0E;AAC1E,qCAAqC;AACrC,mEAAmE;AACnE,OAAO,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AAC7D,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AACvC,OAAO,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AAChD,OAAO,EAAE,MAAM,EAAE,MAAM,iBAAiB,CAAC;AAEzC,MAAM,EAAE,IAAI,EAAE,GAAG,EAAE,KAAK,EAAE,GAAG,UAAwD,CAAC;AAEtF,MAAM,QAAQ,GAAG,MAAM,UAAU,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;AACjD,MAAM,MAAM,GAAuB,GAAG;IAClC,CAAC,CAAC,MAAM,CAAC,MAAM,UAAU,CAAC,GAAG,EAAE,WAAW,CAAC,CAAC,CAAC,YAAY,CAAC,KAAK,IAAI,EAAE,CAAC;IACtE,CAAC,CAAC,CAAC,CAAM,EAAE,EAAE,CAAC,CAAC,CAAC;AAEpB,6EAA6E;AAC7E,KAAK,UAAU,KAAK,CAAC,IAAY;IAC7B,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,SAAS,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC;IAClD,MAAM,OAAO,GAAU,EAAE,CAAC;IAC1B,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QACxC,GAAG,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,CAAM,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;QAC5C,GAAG,CAAC,EAAE,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;QACvB,GAAG,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;QACxB,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAClC,CAAC,CAAC,CAAC;IACH,OAAO,OAAO,CAAC;AACnB,CAAC;AAED,UAAW,CAAC,WAAW,CAAC,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;AAEzC,UAAW,CAAC,EAAE,CAAC,SAAS,EAAE,KAAK,EAAE,GAAkC,EAAE,EAAE;IACnE,IAAI,CAAC;QACD,MAAM,OAAO,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACtC,IAAI,GAAG,GAAG,EAAE,CAAC;QACb,IAAI,CAAC,GAAG,CAAC,CAAC;QACV,KAAK,MAAM,GAAG,IAAI,OAAO,EAAE,CAAC;YACxB,MAAM,CAAC,GAAG,MAAM,MAAM,CAAC,GAAG,CAAC,CAAC;YAC5B,IAAI,CAAC,KAAK,MAAM,IAAI,CAAC,KAAK,SAAS,IAAI,CAAC,KAAK,IAAI;gBAAE,SAAS;YAC5D,IAAI,OAAO,CAAC,KAAK,QAAQ,IAAI,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,MAAM,KAAK,CAAC;gBAAE,SAAS,CAAC,uBAAuB;YAC3F,GAAG,IAAI,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC;YAChC,CAAC,EAAE,CAAC;QACR,CAAC;QACD,UAAW,CAAC,WAAW,CAAC,EAAE,GAAG,EAAE,GAAG,CAAC,GAAG,EAAE,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC;IACtD,CAAC;IAAC,OAAO,GAAQ,EAAE,CAAC;QAChB,UAAW,CAAC,WAAW,CAAC,EAAE,GAAG,EAAE,GAAG,CAAC,GAAG,EAAE,KAAK,EAAE,GAAG,EAAE,OAAO,IAAI,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IAClF,CAAC;AACL,CAAC,CAAC,CAAC"}
|
package/dist/worker.d.ts
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"worker.d.ts","sourceRoot":"","sources":["../src/worker.ts"],"names":[],"mappings":""}
|