marcattacks 2.5.1 → 2.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +91 -2
- package/dist/attacker.d.ts +6 -0
- package/dist/attacker.d.ts.map +1 -1
- package/dist/attacker.js +38 -0
- package/dist/attacker.js.map +1 -1
- package/dist/command.js +1 -0
- package/dist/command.js.map +1 -1
- package/dist/fix/binds.d.ts +5 -0
- package/dist/fix/binds.d.ts.map +1 -0
- package/dist/fix/binds.js +82 -0
- package/dist/fix/binds.js.map +1 -0
- package/dist/fix/conditions.d.ts +5 -0
- package/dist/fix/conditions.d.ts.map +1 -0
- package/dist/fix/conditions.js +111 -0
- package/dist/fix/conditions.js.map +1 -0
- package/dist/fix/fixes.d.ts +7 -0
- package/dist/fix/fixes.d.ts.map +1 -0
- package/dist/fix/fixes.js +478 -0
- package/dist/fix/fixes.js.map +1 -0
- package/dist/fix/index.d.ts +12 -0
- package/dist/fix/index.d.ts.map +1 -0
- package/dist/fix/index.js +40 -0
- package/dist/fix/index.js.map +1 -0
- package/dist/fix/parser.d.ts +25 -0
- package/dist/fix/parser.d.ts.map +1 -0
- package/dist/fix/parser.js +156 -0
- package/dist/fix/parser.js.map +1 -0
- package/dist/fix/path.d.ts +23 -0
- package/dist/fix/path.d.ts.map +1 -0
- package/dist/fix/path.js +351 -0
- package/dist/fix/path.js.map +1 -0
- package/dist/fix/signal.d.ts +2 -0
- package/dist/fix/signal.d.ts.map +1 -0
- package/dist/fix/signal.js +5 -0
- package/dist/fix/signal.js.map +1 -0
- package/dist/fix/util.d.ts +6 -0
- package/dist/fix/util.d.ts.map +1 -0
- package/dist/fix/util.js +133 -0
- package/dist/fix/util.js.map +1 -0
- package/dist/input/alephseq.d.ts.map +1 -1
- package/dist/input/alephseq.js +18 -5
- package/dist/input/alephseq.js.map +1 -1
- package/dist/input/fastxml.d.ts +3 -0
- package/dist/input/fastxml.d.ts.map +1 -0
- package/dist/input/fastxml.js +134 -0
- package/dist/input/fastxml.js.map +1 -0
- package/dist/input/xml.d.ts.map +1 -1
- package/dist/input/xml.js +4 -2
- package/dist/input/xml.js.map +1 -1
- package/dist/marcmap.d.ts.map +1 -1
- package/dist/marcmap.js +15 -3
- package/dist/marcmap.js.map +1 -1
- package/dist/output/alephseq.d.ts.map +1 -1
- package/dist/output/alephseq.js +29 -5
- package/dist/output/alephseq.js.map +1 -1
- package/dist/output/null.d.ts +3 -0
- package/dist/output/null.d.ts.map +1 -0
- package/dist/output/null.js +13 -0
- package/dist/output/null.js.map +1 -0
- package/dist/output/xml.d.ts.map +1 -1
- package/dist/output/xml.js +27 -0
- package/dist/output/xml.js.map +1 -1
- package/dist/stream/framer.d.ts.map +1 -0
- package/dist/stream/framer.js.map +1 -0
- package/dist/stream/worker-pool-str.d.ts.map +1 -0
- package/dist/stream/worker-pool-str.js.map +1 -0
- package/dist/stream/worker-pool.d.ts +10 -0
- package/dist/stream/worker-pool.d.ts.map +1 -0
- package/dist/stream/worker-pool.js +217 -0
- package/dist/stream/worker-pool.js.map +1 -0
- package/dist/transform/fix.d.ts +16 -0
- package/dist/transform/fix.d.ts.map +1 -0
- package/dist/transform/fix.js +46 -0
- package/dist/transform/fix.js.map +1 -0
- package/dist/transform/jsonata.d.ts +5 -0
- package/dist/transform/jsonata.d.ts.map +1 -1
- package/dist/transform/jsonata.js +45 -44
- package/dist/transform/jsonata.js.map +1 -1
- package/dist/transform/marcids.d.ts.map +1 -1
- package/dist/transform/marcids.js +3 -0
- package/dist/transform/marcids.js.map +1 -1
- package/dist/util/marc_record.d.ts +19 -0
- package/dist/util/marc_record.d.ts.map +1 -0
- package/dist/util/marc_record.js +16 -0
- package/dist/util/marc_record.js.map +1 -0
- package/dist/worker-str.d.ts.map +1 -0
- package/dist/worker-str.js.map +1 -0
- package/dist/worker.d.ts +2 -0
- package/dist/worker.d.ts.map +1 -0
- package/dist/worker.js +33 -0
- package/dist/worker.js.map +1 -0
- package/package.json +5 -2
- package/dist/output/mulitipart.d.ts +0 -7
- package/dist/output/mulitipart.d.ts.map +0 -1
- package/dist/output/mulitipart.js +0 -53
- package/dist/output/mulitipart.js.map +0 -1
- package/dist/output/text.d.ts +0 -6
- package/dist/output/text.d.ts.map +0 -1
- package/dist/output/text.js +0 -34
- package/dist/output/text.js.map +0 -1
package/dist/output/xml.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"xml.js","sourceRoot":"","sources":["../../src/output/xml.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,QAAQ,CAAC;AACnC,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,aAAa,EAAG,cAAc,EAAC,MAAM,eAAe,CAAC;AAChF,OAAO,MAAM,MAAM,QAAQ,CAAC;AAE5B,MAAM,MAAM,GAAG,MAAM,CAAC,SAAS,EAAE,CAAC;AAElC,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,KAAS;IACrC,IAAI,OAAO,GAAG,IAAI,CAAC;IAEnB,OAAO,IAAI,SAAS,CAAC;QACjB,UAAU,EAAE,IAAI;QAChB,SAAS,CAAC,IAAS,EAAE,SAAS,EAAE,QAAQ;YACpC,IAAI,GAAG,GAAgB,IAAI,CAAC,QAAQ,CAAC,CAAC;YAEtC,IAAI,CAAC,GAAG,EAAE,CAAC;gBACP,MAAM,CAAC,KAAK,CAAC,sBAAsB,CAAC,CAAC;gBACrC,QAAQ,EAAE,CAAA;gBACV,OAAO;YACX,CAAC;YAED,IAAI,MAAM,GAAG,EAAE,CAAC;YAEhB,IAAI,OAAO,EAAE,CAAC;gBACV,MAAM,IAAI,8CAA8C,CAAC;gBACzD,MAAM,IAAI,mEAAmE,CAAC;gBAC9E,OAAO,GAAG,KAAK,CAAC;YACpB,CAAC;YAED,MAAM,IAAI,kBAAkB,CAAC;YAE7B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAG,CAAC,GAAG,GAAG,CAAC,MAAM,EAAG,CAAC,EAAE,EAAE,CAAC;gBACpC,IAAI,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;gBAC1B,IAAI,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;gBAC1B,IAAI,GAAG,KAAK,KAAK,EAAE,CAAC,CAAA,CAAC;qBAChB,IAAI,GAAG,KAAK,KAAK,EAAE,CAAC;oBACrB,IAAI,KAAK,GAAG,aAAa,CAAC,GAAG,CAAC,CAAC,CAAE,EAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;oBAC3C,MAAM,IAAI,kBAAkB,SAAS,CAAC,KAAK,CAAC,kBAAkB,CAAC;gBACnE,CAAC;qBACI,IAAI,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC;oBACxB,IAAI,KAAK,GAAG,aAAa,CAAC,GAAG,CAAC,CAAC,CAAE,EAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;oBAC3C,MAAM,IAAI,6BAA6B,SAAS,CAAC,GAAG,EAAC,EAAC,YAAY,EAAC,IAAI,EAAC,CAAC,KAAK,SAAS,CAAC,KAAK,CAAC,wBAAwB,CAAC;gBAC3H,CAAC;qBACI,CAAC;oBACF,MAAM,IAAI,0BAA0B,SAAS,CAAC,GAAG,CAAC,WAAW,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAC,EAAC,YAAY,EAAC,IAAI,EAAC,CAAC,WAAW,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAC,EAAC,YAAY,EAAC,IAAI,EAAC,CAAC,MAAM,CAAC;oBACzJ,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,IAAI,EAAC,KAAK,EAAE,EAAE;wBAClC,MAAM,IAAI,4BAA4B,SAAS,CAAC,IAAI,CAAC,KAAK,SAAS,CAAC,KAAK,CAAC,oBAAoB,CAAC;oBACnG,CAAC,CAAC,CAAC;oBACH,MAAM,IAAI,uBAAuB,CAAC;gBACtC,CAAC;YACL,CAAC;YAED,MAAM,IAAI,mBAAmB,CAAC;YAE9B,MAAM,CAAC,KAAK,CAAC,UAAU,MAAM,CAAC,MAAM,QAAQ,CAAC,CAAC;YAE9C,QAAQ,CAAC,IAAI,EAAC,MAAM,CAAC,CAAC;QAC1B,CAAC;QACD,KAAK,CAAC,QAAQ;YACV,MAAM,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC;YAC9B,IAAI,CAAC,OAAO,EAAE,CAAC;gBACX,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;gBACzB,IAAI,MAAM,GAAG,sBAAsB,CAAC;gBACpC,MAAM,CAAC,KAAK,CAAC,UAAU,MAAM,CAAC,MAAM,QAAQ,CAAC,CAAC;gBAC9C,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACtB,CAAC;YACD,QAAQ,EAAE,CAAC;QACf,CAAC;KACJ,CAAC,CAAC;AACP,CAAC;AAED,MAAM,UAAU,SAAS,CACvB,KAAyC,EACzC,OAAoC;IAElC,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,KAAK,SAAS;QAAE,OAAO,EAAE,CAAC;IAErD,IAAI,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;IAEtB,qEAAqE;IAErE,6EAA6E;IAC7E,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,qCAAqC,EAAE,QAAQ,CAAC,CAAC,CAAE,2BAA2B;IAC5F,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,sCAAsC,EAAE,QAAQ,CAAC,CAAC,CAAC,0BAA0B;IAE3F,8CAA8C;IAC9C,6DAA6D;IAC7D,yDAAyD;IACzD,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,oEAAoE,EAAE,EAAE,CAAC,CAAC;IAExF,+DAA+D;IAC/D,4DAA4D;IAC5D,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,kBAAkB,EAAE,EAAE,CAAC,CAAC;IAEtC,wCAAwC;IACxC,4DAA4D;IAC5D,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAE7B,kCAAkC;IAClC,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;IAC5B,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;IAE5B,IAAI,OAAO,EAAE,YAAY,EAAE,CAAC;QACxB,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;QAC9B,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;IAClC,CAAC;IAED,OAAO,CAAC,CAAC;AACb,CAAC"}
|
|
1
|
+
{"version":3,"file":"xml.js","sourceRoot":"","sources":["../../src/output/xml.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,QAAQ,CAAC;AACnC,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,aAAa,EAAG,cAAc,EAAC,MAAM,eAAe,CAAC;AAChF,OAAO,MAAM,MAAM,QAAQ,CAAC;AAE5B,MAAM,MAAM,GAAG,MAAM,CAAC,SAAS,EAAE,CAAC;AAElC,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,KAAS;IACrC,IAAI,OAAO,GAAG,IAAI,CAAC;IAEnB,OAAO,IAAI,SAAS,CAAC;QACjB,UAAU,EAAE,IAAI;QAChB,SAAS,CAAC,IAAS,EAAE,SAAS,EAAE,QAAQ;YACpC,IAAI,GAAG,GAAgB,IAAI,CAAC,QAAQ,CAAC,CAAC;YAEtC,IAAI,CAAC,GAAG,EAAE,CAAC;gBACP,MAAM,CAAC,KAAK,CAAC,sBAAsB,CAAC,CAAC;gBACrC,QAAQ,EAAE,CAAA;gBACV,OAAO;YACX,CAAC;YAED,IAAI,MAAM,GAAG,EAAE,CAAC;YAEhB,IAAI,OAAO,EAAE,CAAC;gBACV,MAAM,IAAI,8CAA8C,CAAC;gBACzD,MAAM,IAAI,mEAAmE,CAAC;gBAC9E,OAAO,GAAG,KAAK,CAAC;YACpB,CAAC;YAED,MAAM,IAAI,kBAAkB,CAAC;YAE7B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAG,CAAC,GAAG,GAAG,CAAC,MAAM,EAAG,CAAC,EAAE,EAAE,CAAC;gBACpC,IAAI,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;gBAC1B,IAAI,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;gBAC1B,IAAI,GAAG,KAAK,KAAK,EAAE,CAAC,CAAA,CAAC;qBAChB,IAAI,GAAG,KAAK,KAAK,EAAE,CAAC;oBACrB,IAAI,KAAK,GAAG,aAAa,CAAC,GAAG,CAAC,CAAC,CAAE,EAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;oBAC3C,MAAM,IAAI,kBAAkB,SAAS,CAAC,KAAK,CAAC,kBAAkB,CAAC;gBACnE,CAAC;qBACI,IAAI,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC;oBACxB,IAAI,KAAK,GAAG,aAAa,CAAC,GAAG,CAAC,CAAC,CAAE,EAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;oBAC3C,MAAM,IAAI,6BAA6B,SAAS,CAAC,GAAG,EAAC,EAAC,YAAY,EAAC,IAAI,EAAC,CAAC,KAAK,SAAS,CAAC,KAAK,CAAC,wBAAwB,CAAC;gBAC3H,CAAC;qBACI,CAAC;oBACF,MAAM,IAAI,0BAA0B,SAAS,CAAC,GAAG,CAAC,WAAW,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAC,EAAC,YAAY,EAAC,IAAI,EAAC,CAAC,WAAW,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAC,EAAC,YAAY,EAAC,IAAI,EAAC,CAAC,MAAM,CAAC;oBACzJ,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,IAAI,EAAC,KAAK,EAAE,EAAE;wBAClC,MAAM,IAAI,4BAA4B,SAAS,CAAC,IAAI,CAAC,KAAK,SAAS,CAAC,KAAK,CAAC,oBAAoB,CAAC;oBACnG,CAAC,CAAC,CAAC;oBACH,MAAM,IAAI,uBAAuB,CAAC;gBACtC,CAAC;YACL,CAAC;YAED,MAAM,IAAI,mBAAmB,CAAC;YAE9B,MAAM,CAAC,KAAK,CAAC,UAAU,MAAM,CAAC,MAAM,QAAQ,CAAC,CAAC;YAE9C,QAAQ,CAAC,IAAI,EAAC,MAAM,CAAC,CAAC;QAC1B,CAAC;QACD,KAAK,CAAC,QAAQ;YACV,MAAM,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC;YAC9B,IAAI,CAAC,OAAO,EAAE,CAAC;gBACX,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;gBACzB,IAAI,MAAM,GAAG,sBAAsB,CAAC;gBACpC,MAAM,CAAC,KAAK,CAAC,UAAU,MAAM,CAAC,MAAM,QAAQ,CAAC,CAAC;gBAC9C,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACtB,CAAC;YACD,QAAQ,EAAE,CAAC;QACf,CAAC;KACJ,CAAC,CAAC;AACP,CAAC;AAED,0EAA0E;AAC1E,6EAA6E;AAC7E,6EAA6E;AAC7E,oEAAoE;AACpE,SAAS,WAAW,CAAC,CAAS,EAAE,YAAqB;IACjD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAChC,MAAM,CAAC,GAAG,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;QAC1B,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE;YAAE,OAAO,IAAI,CAAC,CAAiB,QAAQ;QAC3E,IAAI,YAAY,IAAI,CAAC,CAAC,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,CAAC;YAAE,OAAO,IAAI,CAAC,CAAW,MAAM;QACzE,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,IAAI,CAAC;YAAE,OAAO,IAAI,CAAC,CAAC,OAAO;QAC3F,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,IAAI;YAAE,OAAO,IAAI,CAAC;QACxC,IAAI,CAAC,IAAI,MAAM,IAAI,CAAC,IAAI,MAAM;YAAE,OAAO,IAAI,CAAC,CAAuB,aAAa;QAChF,IAAI,CAAC,IAAI,MAAM,IAAI,CAAC,IAAI,MAAM;YAAE,OAAO,IAAI,CAAC,CAAuB,YAAY;QAC/E,IAAI,CAAC,KAAK,MAAM,IAAI,CAAC,KAAK,MAAM;YAAE,OAAO,IAAI,CAAC;IAClD,CAAC;IACD,OAAO,KAAK,CAAC;AACjB,CAAC;AAED,MAAM,UAAU,SAAS,CACvB,KAAyC,EACzC,OAAoC;IAElC,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,KAAK,SAAS;QAAE,OAAO,EAAE,CAAC;IAErD,IAAI,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;IAEtB,sEAAsE;IACtE,IAAI,CAAC,WAAW,CAAC,CAAC,EAAE,OAAO,EAAE,YAAY,KAAK,IAAI,CAAC;QAAE,OAAO,CAAC,CAAC;IAE9D,qEAAqE;IAErE,6EAA6E;IAC7E,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,qCAAqC,EAAE,QAAQ,CAAC,CAAC,CAAE,2BAA2B;IAC5F,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,sCAAsC,EAAE,QAAQ,CAAC,CAAC,CAAC,0BAA0B;IAE3F,8CAA8C;IAC9C,6DAA6D;IAC7D,yDAAyD;IACzD,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,oEAAoE,EAAE,EAAE,CAAC,CAAC;IAExF,+DAA+D;IAC/D,4DAA4D;IAC5D,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,kBAAkB,EAAE,EAAE,CAAC,CAAC;IAEtC,wCAAwC;IACxC,4DAA4D;IAC5D,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAE7B,kCAAkC;IAClC,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;IAC5B,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;IAE5B,IAAI,OAAO,EAAE,YAAY,EAAE,CAAC;QACxB,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;QAC9B,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;IAClC,CAAC;IAED,OAAO,CAAC,CAAC;AACb,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"framer.d.ts","sourceRoot":"","sources":["../../src/stream/framer.ts"],"names":[],"mappings":"AAUA,OAAO,EAAE,SAAS,EAA0B,MAAM,QAAQ,CAAC;AAG3D,eAAO,MAAM,QAAQ,aAAmD,CAAC;AAkCzE,wBAAgB,YAAY,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,SAAS,CA+BnE"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"framer.js","sourceRoot":"","sources":["../../src/stream/framer.ts"],"names":[],"mappings":"AAAA,8EAA8E;AAC9E,6EAA6E;AAC7E,+EAA+E;AAC/E,+EAA+E;AAC/E,4BAA4B;AAC5B,EAAE;AACF,+EAA+E;AAC/E,uDAAuD;AACvD,yEAAyE;AACzE,mEAAmE;AACnE,OAAO,EAAE,SAAS,EAA0B,MAAM,QAAQ,CAAC;AAC3D,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AAEpD,MAAM,CAAC,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,CAAC,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,UAAU,CAAC,CAAC,CAAC;AAEzE,MAAM,YAAY,GAAG,6BAA6B,CAAC;AAEnD,+EAA+E;AAC/E,+EAA+E;AAC/E,SAAS,kBAAkB,CAAC,IAAY,EAAE,GAAW;IACjD,IAAI,IAAI,KAAK,OAAO,EAAE,CAAC;QACnB,MAAM,IAAI,GAAa,EAAE,CAAC;QAC1B,KAAK,IAAI,CAAC,GAAG,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,GAAG,GAAG,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,GAAG,CAAC,CAAC;YAAE,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QACvF,OAAO,IAAI,CAAC;IAChB,CAAC;IACD,IAAI,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;QACvC,MAAM,IAAI,GAAa,EAAE,CAAC;QAC1B,YAAY,CAAC,SAAS,GAAG,CAAC,CAAC;QAC3B,IAAI,CAAyB,CAAC;QAC9B,OAAO,CAAC,CAAC,GAAG,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAAE,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;QACtE,OAAO,IAAI,CAAC;IAChB,CAAC;IACD,2EAA2E;IAC3E,oEAAoE;IACpE,MAAM,IAAI,GAAa,EAAE,CAAC;IAC1B,IAAI,MAAM,GAAkB,IAAI,CAAC;IACjC,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,KAAK,IAAI,CAAC,GAAG,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,GAAG,GAAG,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC;QACnE,MAAM,EAAE,GAAG,GAAG,CAAC,OAAO,CAAC,GAAG,EAAE,SAAS,CAAC,CAAC;QACvC,MAAM,EAAE,GAAG,EAAE,IAAI,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC;QAClF,IAAI,MAAM,KAAK,IAAI,IAAI,EAAE,KAAK,MAAM;YAAE,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,yBAAyB;QACrF,MAAM,GAAG,EAAE,CAAC;QACZ,SAAS,GAAG,CAAC,GAAG,CAAC,CAAC;IACtB,CAAC;IACD,OAAO,IAAI,CAAC;AAChB,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,IAAY,EAAE,KAAa;IACpD,MAAM,OAAO,GAAG,IAAI,aAAa,CAAC,MAAM,CAAC,CAAC;IAC1C,IAAI,GAAG,GAAG,EAAE,CAAC;IACb,IAAI,GAAG,GAAG,CAAC,CAAC;IAEZ,SAAS,IAAI,CAAC,MAAiB,EAAE,KAAc;QAC3C,SAAS,CAAC;YACN,MAAM,IAAI,GAAG,kBAAkB,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;YAC3C,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;gBAAE,MAAM;YAC7B,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,MAAM,GAAG,KAAK;gBAAE,MAAM,CAAO,wBAAwB;YACxE,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAE,CAAC;YACpD,MAAM,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,GAAG,EAAE,EAAE,IAAI,EAAE,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;YACrD,GAAG,GAAG,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;YACrB,IAAI,IAAI,CAAC,MAAM,GAAG,KAAK;gBAAE,MAAM,CAAiB,mBAAmB;QACvE,CAAC;IACL,CAAC;IAED,OAAO,IAAI,SAAS,CAAC;QACjB,kBAAkB,EAAE,IAAI;QACxB,SAAS,CAAC,KAAU,EAAE,IAAY,EAAE,EAAqB;YACrD,GAAG,IAAI,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;YAC5B,IAAI,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;YAClB,EAAE,EAAE,CAAC;QACT,CAAC;QACD,KAAK,CAAC,EAAqB;YACvB,GAAG,IAAI,OAAO,CAAC,GAAG,EAAE,CAAC;YACrB,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;YACjB,IAAI,GAAG,CAAC,MAAM,GAAG,CAAC;gBAAE,IAAI,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,GAAG,EAAE,EAAE,IAAI,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC,0BAA0B;YACpF,EAAE,EAAE,CAAC;QACT,CAAC;KACJ,CAAC,CAAC;AACP,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"worker-pool-str.d.ts","sourceRoot":"","sources":["../../src/stream/worker-pool-str.ts"],"names":[],"mappings":"AAMA,OAAO,EAAE,SAAS,EAA0B,MAAM,QAAQ,CAAC;AAM3D,MAAM,WAAW,WAAW;IAAG,IAAI,EAAE,MAAM,CAAC;IAAC,GAAG,CAAC,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,GAAG,CAAC;IAAC,OAAO,EAAE,MAAM,CAAC;CAAE;AAEzF,wBAAgB,sBAAsB,CAAC,IAAI,EAAE,WAAW,GAAG,SAAS,CA4FnE"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"worker-pool-str.js","sourceRoot":"","sources":["../../src/stream/worker-pool-str.ts"],"names":[],"mappings":"AAAA,+EAA+E;AAC/E,gFAAgF;AAChF,mDAAmD;AACnD,EAAE;AACF,wDAAwD;AACxD,sDAAsD;AACtD,OAAO,EAAE,SAAS,EAA0B,MAAM,QAAQ,CAAC;AAC3D,OAAO,EAAE,MAAM,EAAE,MAAM,qBAAqB,CAAC;AAC7C,OAAO,MAAM,MAAM,QAAQ,CAAC;AAE5B,MAAM,MAAM,GAAG,MAAM,CAAC,SAAS,EAAE,CAAC;AAIlC,MAAM,UAAU,sBAAsB,CAAC,IAAiB;IACpD,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;IACpC,MAAM,GAAG,GAAG,CAAC,GAAG,CAAC,CAAC;IAClB,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,kBAAkB,EAAE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAE/D,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,MAAM,IAAI,GAAa,EAAE,CAAC;IAC1B,MAAM,KAAK,GAAoC,EAAE,CAAC;IAClD,MAAM,OAAO,GAAG,IAAI,GAAG,EAAkB,CAAC;IAE1C,IAAI,QAAQ,GAAG,CAAC,CAAC,CAAK,8BAA8B;IACpD,IAAI,QAAQ,GAAG,CAAC,CAAC,CAAK,8BAA8B;IACpD,IAAI,OAAO,GAAG,CAAC,CAAC,CAAM,6BAA6B;IACnD,IAAI,KAAK,GAAG,CAAC,CAAC,CAAQ,qCAAqC;IAC3D,IAAI,KAAK,GAAG,KAAK,CAAC;IAClB,IAAI,QAAQ,GAAG,CAAC,CAAC;IACjB,IAAI,QAAQ,GAA6B,IAAI,CAAC;IAC9C,IAAI,KAAK,GAA6B,IAAI,CAAC;IAC3C,IAAI,MAAM,GAAiB,IAAI,CAAC;IAEhC,MAAM,MAAM,GAAG,IAAI,SAAS,CAAC;QACzB,kBAAkB,EAAE,IAAI,EAAU,iBAAiB;QACnD,kBAAkB,EAAE,KAAK,EAAS,eAAe;QACjD,SAAS,CAAC,KAAoC,EAAE,IAAI,EAAE,EAAqB;YACvE,IAAI,MAAM,EAAE,CAAC;gBAAC,EAAE,CAAC,MAAM,CAAC,CAAC;gBAAC,OAAO;YAAC,CAAC;YACnC,QAAQ,EAAE,CAAC;YACX,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAClB,QAAQ,EAAE,CAAC;YACX,IAAI,QAAQ,GAAG,GAAG;gBAAE,EAAE,EAAE,CAAC;;gBACpB,QAAQ,GAAG,EAAE,CAAC,CAAW,eAAe;QACjD,CAAC;QACD,KAAK,CAAC,EAAqB;YACvB,IAAI,MAAM,EAAE,CAAC;gBAAC,EAAE,CAAC,MAAM,CAAC,CAAC;gBAAC,OAAO;YAAC,CAAC;YACnC,KAAK,GAAG,IAAI,CAAC;YACb,IAAI,OAAO,KAAK,QAAQ;gBAAE,MAAM,CAAC,EAAE,CAAC,CAAC;;gBAChC,KAAK,GAAG,EAAE,CAAC;QACpB,CAAC;KACJ,CAAC,CAAC;IAEH,SAAS,QAAQ;QACb,OAAO,KAAK,CAAC,MAAM,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YACjC,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,EAAG,CAAC;YACtB,MAAM,GAAG,GAAG,KAAK,CAAC,KAAK,EAAG,CAAC;YAC3B,QAAQ,EAAE,CAAC;YACX,CAAC,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;QACvB,CAAC;IACL,CAAC;IAED,SAAS,QAAQ,CAAC,CAAS,EAAE,GAAQ;QACjC,IAAI,GAAG,CAAC,KAAK,EAAE,CAAC;YAAC,IAAI,CAAC,IAAI,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC;YAAC,OAAO;QAAC,CAAC;QACtD,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,EAAE,GAAG,CAAC,GAAG,CAAC,CAAC;QAC9B,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACb,QAAQ,EAAE,CAAC;QACX,OAAO,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC;YAC3B,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAE,CAAC;YACnC,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;YACzB,QAAQ,EAAE,CAAC;YACX,OAAO,EAAE,CAAC;YACV,IAAI,GAAG,CAAC,MAAM;gBAAE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACrC,CAAC;QACD,KAAK,IAAI,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC;QACpB,QAAQ,EAAE,CAAC;QACX,IAAI,QAAQ,IAAI,QAAQ,GAAG,GAAG,EAAE,CAAC;YAAC,MAAM,CAAC,GAAG,QAAQ,CAAC;YAAC,QAAQ,GAAG,IAAI,CAAC;YAAC,CAAC,EAAE,CAAC;QAAC,CAAC;QAC7E,IAAI,KAAK,IAAI,KAAK,IAAI,OAAO,KAAK,QAAQ,EAAE,CAAC;YAAC,MAAM,CAAC,GAAG,KAAK,CAAC;YAAC,KAAK,GAAG,IAAI,CAAC;YAAC,MAAM,CAAC,CAAC,CAAC,CAAC;QAAC,CAAC;IAC7F,CAAC;IAED,SAAS,IAAI,CAAC,GAAU;QACpB,IAAI,MAAM;YAAE,OAAO;QACnB,MAAM,GAAG,GAAG,CAAC;QACb,KAAK,MAAM,CAAC,IAAI,OAAO;YAAE,CAAC,CAAC,SAAS,EAAE,CAAC;QACvC,IAAI,QAAQ,EAAE,CAAC;YAAC,MAAM,CAAC,GAAG,QAAQ,CAAC;YAAC,QAAQ,GAAG,IAAI,CAAC;YAAC,CAAC,CAAC,GAAG,CAAC,CAAC;QAAC,CAAC;aACzD,IAAI,KAAK,EAAE,CAAC;YAAC,MAAM,CAAC,GAAG,KAAK,CAAC;YAAC,KAAK,GAAG,IAAI,CAAC;YAAC,CAAC,CAAC,GAAG,CAAC,CAAC;QAAC,CAAC;;YACrD,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IAC7B,CAAC;IAED,SAAS,MAAM,CAAC,EAAqB;QACjC,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;IACtF,CAAC;IAED,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QACzB,MAAM,CAAC,GAAG,IAAI,MAAM,CAAC,SAAS,EAAE,EAAE,UAAU,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;QACvG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAChB,CAAC,CAAC,EAAE,CAAC,SAAS,EAAE,CAAC,GAAQ,EAAE,EAAE;YACzB,IAAI,GAAG,IAAI,GAAG,CAAC,KAAK,EAAE,CAAC;gBAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;gBAAC,QAAQ,EAAE,CAAC;YAAC,CAAC;;gBAC9C,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;QAC1B,CAAC,CAAC,CAAC;QACH,CAAC,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;IACtC,CAAC;IAEA,MAAc,CAAC,QAAQ,GAAG,GAAG,EAAE,CAAC,KAAK,CAAC;IACvC,MAAM,CAAC,IAAI,CAAC,mBAAmB,CAAC,yBAAyB,CAAC,CAAC;IAC3D,OAAO,MAAM,CAAC;AAClB,CAAC"}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import { Transform } from 'stream';
|
|
2
|
+
export interface WorkerPoolOpts {
|
|
3
|
+
map: string;
|
|
4
|
+
param: any;
|
|
5
|
+
workers: number;
|
|
6
|
+
batchSize?: number;
|
|
7
|
+
timeoutMs?: number;
|
|
8
|
+
}
|
|
9
|
+
export declare function createWorkerPool(opts: WorkerPoolOpts): Transform;
|
|
10
|
+
//# sourceMappingURL=worker-pool.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"worker-pool.d.ts","sourceRoot":"","sources":["../../src/stream/worker-pool.ts"],"names":[],"mappings":"AAgBA,OAAO,EAAE,SAAS,EAA0B,MAAM,QAAQ,CAAC;AAO3D,MAAM,WAAW,cAAc;IAC3B,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,GAAG,CAAC;IACX,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,SAAS,CAAC,EAAE,MAAM,CAAC;CACtB;AAED,wBAAgB,gBAAgB,CAAC,IAAI,EAAE,cAAc,GAAG,SAAS,CAuJhE"}
|
|
@@ -0,0 +1,217 @@
|
|
|
1
|
+
// A Transform that parallelizes a per-record map across worker threads while
|
|
2
|
+
// preserving record order. Batches incoming records, dispatches batches to a
|
|
3
|
+
// pool of workers, re-orders the results by sequence number, and pushes them
|
|
4
|
+
// downstream in order.
|
|
5
|
+
//
|
|
6
|
+
// In (objectMode): record objects
|
|
7
|
+
// Out (objectMode): mapped record objects, in the original order
|
|
8
|
+
// (records the map rejected are dropped)
|
|
9
|
+
//
|
|
10
|
+
// Hardening over the first prototype:
|
|
11
|
+
// - worker count is clamped to the available parallelism;
|
|
12
|
+
// - real downstream backpressure: results are held in an out-queue and only
|
|
13
|
+
// pushed while push() accepts them; dispatch/accept pause when the output
|
|
14
|
+
// is backed up and resume on _read (so a slow sink can't grow memory);
|
|
15
|
+
// - a per-batch timeout fails the pipeline instead of hanging if a worker
|
|
16
|
+
// dies silently.
|
|
17
|
+
import { Transform } from 'stream';
|
|
18
|
+
import { Worker } from 'node:worker_threads';
|
|
19
|
+
import { availableParallelism } from 'node:os';
|
|
20
|
+
import log4js from 'log4js';
|
|
21
|
+
const logger = log4js.getLogger();
|
|
22
|
+
export function createWorkerPool(opts) {
|
|
23
|
+
const maxN = Math.max(1, availableParallelism());
|
|
24
|
+
const N = Math.min(Math.max(1, Math.floor(opts.workers) || 1), maxN);
|
|
25
|
+
if (N < opts.workers)
|
|
26
|
+
logger.warn(`--workers ${opts.workers} clamped to ${N} (available parallelism)`);
|
|
27
|
+
const BATCH = opts.batchSize ?? 64;
|
|
28
|
+
const CAP = N * 2; // max batches in flight
|
|
29
|
+
const OUT_CAP = CAP * BATCH; // max records buffered awaiting downstream
|
|
30
|
+
const TIMEOUT = opts.timeoutMs ?? 120_000;
|
|
31
|
+
const workerUrl = new URL('../worker.js', import.meta.url);
|
|
32
|
+
const workers = [];
|
|
33
|
+
const idle = [];
|
|
34
|
+
const timers = new Map();
|
|
35
|
+
const queue = [];
|
|
36
|
+
const reorder = new Map();
|
|
37
|
+
const outQueue = []; // mapped records ready to push, in order
|
|
38
|
+
let nextDispatch = 0;
|
|
39
|
+
let nextEmit = 0;
|
|
40
|
+
let inflight = 0;
|
|
41
|
+
let curBatch = [];
|
|
42
|
+
let acceptCb = null; // held _transform callback
|
|
43
|
+
let endCb = null;
|
|
44
|
+
let failed = null;
|
|
45
|
+
let closing = false; // we are terminating workers on purpose
|
|
46
|
+
const stream = new Transform({
|
|
47
|
+
objectMode: true,
|
|
48
|
+
transform(rec, _enc, cb) {
|
|
49
|
+
if (failed) {
|
|
50
|
+
cb(failed);
|
|
51
|
+
return;
|
|
52
|
+
}
|
|
53
|
+
curBatch.push(rec);
|
|
54
|
+
if (curBatch.length >= BATCH) {
|
|
55
|
+
enqueue(curBatch);
|
|
56
|
+
curBatch = [];
|
|
57
|
+
}
|
|
58
|
+
if (canAccept())
|
|
59
|
+
cb();
|
|
60
|
+
else
|
|
61
|
+
acceptCb = cb; // pause upstream until there's room
|
|
62
|
+
},
|
|
63
|
+
flush(cb) {
|
|
64
|
+
if (failed) {
|
|
65
|
+
cb(failed);
|
|
66
|
+
return;
|
|
67
|
+
}
|
|
68
|
+
if (curBatch.length) {
|
|
69
|
+
enqueue(curBatch);
|
|
70
|
+
curBatch = [];
|
|
71
|
+
}
|
|
72
|
+
if (done())
|
|
73
|
+
finish(cb);
|
|
74
|
+
else
|
|
75
|
+
endCb = cb;
|
|
76
|
+
}
|
|
77
|
+
});
|
|
78
|
+
// Resume producing when the downstream consumer asks for more.
|
|
79
|
+
const origRead = stream._read.bind(stream);
|
|
80
|
+
stream._read = (size) => { pump(); origRead(size); };
|
|
81
|
+
function canAccept() {
|
|
82
|
+
return inflight < CAP && outQueue.length < OUT_CAP;
|
|
83
|
+
}
|
|
84
|
+
function done() {
|
|
85
|
+
return nextEmit === nextDispatch && queue.length === 0 && outQueue.length === 0;
|
|
86
|
+
}
|
|
87
|
+
function enqueue(batch) {
|
|
88
|
+
queue.push({ seq: nextDispatch++, batch });
|
|
89
|
+
dispatch();
|
|
90
|
+
}
|
|
91
|
+
const dbg = () => logger.isDebugEnabled();
|
|
92
|
+
function dispatch() {
|
|
93
|
+
// don't start new work while the output is backed up
|
|
94
|
+
if (dbg() && queue.length && (!idle.length || outQueue.length >= OUT_CAP)) {
|
|
95
|
+
logger.debug(`backpressure: ${queue.length} batches queued, idle=${idle.length}, outQueue=${outQueue.length}/${OUT_CAP}`);
|
|
96
|
+
}
|
|
97
|
+
while (queue.length && idle.length && outQueue.length < OUT_CAP) {
|
|
98
|
+
const w = idle.pop();
|
|
99
|
+
const job = queue.shift();
|
|
100
|
+
inflight++;
|
|
101
|
+
timers.set(w, setTimeout(() => fail(new Error(`worker timed out after ${TIMEOUT}ms (batch ${job.seq})`)), TIMEOUT));
|
|
102
|
+
w.postMessage(job);
|
|
103
|
+
if (dbg())
|
|
104
|
+
logger.debug(`dispatch batch ${job.seq} (${job.batch.length} recs) -> worker; inflight=${inflight}`);
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
// push as many ready records as the downstream will take
|
|
108
|
+
function pump() {
|
|
109
|
+
const before = outQueue.length;
|
|
110
|
+
while (outQueue.length && stream.push(outQueue[0]))
|
|
111
|
+
outQueue.shift();
|
|
112
|
+
if (dbg() && outQueue.length > 0 && outQueue.length === before) {
|
|
113
|
+
logger.debug(`downstream full, holding ${outQueue.length} records`);
|
|
114
|
+
}
|
|
115
|
+
dispatch();
|
|
116
|
+
if (acceptCb && canAccept()) {
|
|
117
|
+
const c = acceptCb;
|
|
118
|
+
acceptCb = null;
|
|
119
|
+
c();
|
|
120
|
+
}
|
|
121
|
+
if (endCb && done()) {
|
|
122
|
+
const c = endCb;
|
|
123
|
+
endCb = null;
|
|
124
|
+
finish(c);
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
function onResult(w, msg) {
|
|
128
|
+
const t = timers.get(w);
|
|
129
|
+
if (t) {
|
|
130
|
+
clearTimeout(t);
|
|
131
|
+
timers.delete(w);
|
|
132
|
+
}
|
|
133
|
+
if (msg.error) {
|
|
134
|
+
logger.error(`worker error on batch ${msg.seq}: ${msg.error}`);
|
|
135
|
+
fail(new Error(msg.error));
|
|
136
|
+
return;
|
|
137
|
+
}
|
|
138
|
+
reorder.set(msg.seq, msg.mapped);
|
|
139
|
+
idle.push(w);
|
|
140
|
+
inflight--;
|
|
141
|
+
if (dbg())
|
|
142
|
+
logger.debug(`result batch ${msg.seq} (${msg.mapped.length} mapped); inflight=${inflight}, reorderBuf=${reorder.size}, wantNext=${nextEmit}`);
|
|
143
|
+
// move now-contiguous results into the out-queue (don't push yet)
|
|
144
|
+
while (reorder.has(nextEmit)) {
|
|
145
|
+
const mapped = reorder.get(nextEmit);
|
|
146
|
+
reorder.delete(nextEmit);
|
|
147
|
+
nextEmit++;
|
|
148
|
+
for (const m of mapped)
|
|
149
|
+
if (m !== null)
|
|
150
|
+
outQueue.push(m);
|
|
151
|
+
}
|
|
152
|
+
pump();
|
|
153
|
+
}
|
|
154
|
+
function fail(err) {
|
|
155
|
+
if (failed)
|
|
156
|
+
return;
|
|
157
|
+
failed = err;
|
|
158
|
+
closing = true;
|
|
159
|
+
logger.error(`worker pool failing: ${err.message} (terminating ${workers.length} workers)`);
|
|
160
|
+
for (const t of timers.values())
|
|
161
|
+
clearTimeout(t);
|
|
162
|
+
timers.clear();
|
|
163
|
+
for (const w of workers)
|
|
164
|
+
w.terminate();
|
|
165
|
+
if (acceptCb) {
|
|
166
|
+
const c = acceptCb;
|
|
167
|
+
acceptCb = null;
|
|
168
|
+
c(err);
|
|
169
|
+
}
|
|
170
|
+
else if (endCb) {
|
|
171
|
+
const c = endCb;
|
|
172
|
+
endCb = null;
|
|
173
|
+
c(err);
|
|
174
|
+
}
|
|
175
|
+
else
|
|
176
|
+
stream.destroy(err);
|
|
177
|
+
}
|
|
178
|
+
function finish(cb) {
|
|
179
|
+
closing = true;
|
|
180
|
+
for (const t of timers.values())
|
|
181
|
+
clearTimeout(t);
|
|
182
|
+
timers.clear();
|
|
183
|
+
logger.debug(`map complete: ${nextEmit} batches; terminating ${workers.length} workers`);
|
|
184
|
+
Promise.all(workers.map((w) => w.terminate())).then(() => cb()).catch(() => cb());
|
|
185
|
+
}
|
|
186
|
+
for (let i = 0; i < N; i++) {
|
|
187
|
+
// stdout/stderr: true keeps each worker's stdio OUT of the parent's
|
|
188
|
+
// process.stdout/stderr. By default Node pipes every worker's stdio
|
|
189
|
+
// through to the parent, attaching close/error/finish listeners to the
|
|
190
|
+
// parent WriteStream per worker -> a MaxListenersExceededWarning once
|
|
191
|
+
// --workers crosses ~8. Workers here are pure compute and report results
|
|
192
|
+
// and errors over postMessage, so they never need the parent's console.
|
|
193
|
+
const w = new Worker(workerUrl, {
|
|
194
|
+
workerData: { map: opts.map, param: opts.param },
|
|
195
|
+
stdout: true,
|
|
196
|
+
stderr: true,
|
|
197
|
+
});
|
|
198
|
+
workers.push(w);
|
|
199
|
+
w.on('message', (msg) => {
|
|
200
|
+
if (msg && msg.ready) {
|
|
201
|
+
idle.push(w);
|
|
202
|
+
dispatch();
|
|
203
|
+
if (dbg())
|
|
204
|
+
logger.debug(`worker ${idle.length}/${N} ready`);
|
|
205
|
+
}
|
|
206
|
+
else
|
|
207
|
+
onResult(w, msg);
|
|
208
|
+
});
|
|
209
|
+
w.on('error', (err) => { logger.error(`worker thread crashed: ${err.message}`); fail(err); });
|
|
210
|
+
w.on('exit', (code) => { if (code !== 0 && !failed && !closing)
|
|
211
|
+
fail(new Error(`worker exited unexpectedly (code ${code})`)); });
|
|
212
|
+
}
|
|
213
|
+
logger.info(`map running on ${N} worker threads (batch ${BATCH}, in-flight cap ${CAP}, timeout ${TIMEOUT}ms)`);
|
|
214
|
+
stream.isWorkerPool = true; // lets callers/tests tell a threaded map stage from a serial one
|
|
215
|
+
return stream;
|
|
216
|
+
}
|
|
217
|
+
//# sourceMappingURL=worker-pool.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"worker-pool.js","sourceRoot":"","sources":["../../src/stream/worker-pool.ts"],"names":[],"mappings":"AAAA,6EAA6E;AAC7E,6EAA6E;AAC7E,6EAA6E;AAC7E,uBAAuB;AACvB,EAAE;AACF,mCAAmC;AACnC,iEAAiE;AACjE,2DAA2D;AAC3D,EAAE;AACF,sCAAsC;AACtC,2DAA2D;AAC3D,6EAA6E;AAC7E,6EAA6E;AAC7E,0EAA0E;AAC1E,2EAA2E;AAC3E,oBAAoB;AACpB,OAAO,EAAE,SAAS,EAA0B,MAAM,QAAQ,CAAC;AAC3D,OAAO,EAAE,MAAM,EAAE,MAAM,qBAAqB,CAAC;AAC7C,OAAO,EAAE,oBAAoB,EAAE,MAAM,SAAS,CAAC;AAC/C,OAAO,MAAM,MAAM,QAAQ,CAAC;AAE5B,MAAM,MAAM,GAAG,MAAM,CAAC,SAAS,EAAE,CAAC;AAUlC,MAAM,UAAU,gBAAgB,CAAC,IAAoB;IACjD,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,oBAAoB,EAAE,CAAC,CAAC;IACjD,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;IACrE,IAAI,CAAC,GAAG,IAAI,CAAC,OAAO;QAAE,MAAM,CAAC,IAAI,CAAC,aAAa,IAAI,CAAC,OAAO,eAAe,CAAC,0BAA0B,CAAC,CAAC;IACvG,MAAM,KAAK,GAAG,IAAI,CAAC,SAAS,IAAI,EAAE,CAAC;IACnC,MAAM,GAAG,GAAG,CAAC,GAAG,CAAC,CAAC,CAAiB,wBAAwB;IAC3D,MAAM,OAAO,GAAG,GAAG,GAAG,KAAK,CAAC,CAAO,2CAA2C;IAC9E,MAAM,OAAO,GAAG,IAAI,CAAC,SAAS,IAAI,OAAO,CAAC;IAC1C,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,cAAc,EAAE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAE3D,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,MAAM,IAAI,GAAa,EAAE,CAAC;IAC1B,MAAM,MAAM,GAAG,IAAI,GAAG,EAA0B,CAAC;IACjD,MAAM,KAAK,GAAoC,EAAE,CAAC;IAClD,MAAM,OAAO,GAAG,IAAI,GAAG,EAAiB,CAAC;IACzC,MAAM,QAAQ,GAAU,EAAE,CAAC,CAAQ,yCAAyC;IAE5E,IAAI,YAAY,GAAG,CAAC,CAAC;IACrB,IAAI,QAAQ,GAAG,CAAC,CAAC;IACjB,IAAI,QAAQ,GAAG,CAAC,CAAC;IACjB,IAAI,QAAQ,GAAU,EAAE,CAAC;IACzB,IAAI,QAAQ,GAA6B,IAAI,CAAC,CAAG,2BAA2B;IAC5E,IAAI,KAAK,GAA6B,IAAI,CAAC;IAC3C,IAAI,MAAM,GAAiB,IAAI,CAAC;IAChC,IAAI,OAAO,GAAG,KAAK,CAAC,CAA8B,wCAAwC;IAE1F,MAAM,MAAM,GAAG,IAAI,SAAS,CAAC;QACzB,UAAU,EAAE,IAAI;QAChB,SAAS,CAAC,GAAQ,EAAE,IAAI,EAAE,EAAqB;YAC3C,IAAI,MAAM,EAAE,CAAC;gBAAC,EAAE,CAAC,MAAM,CAAC,CAAC;gBAAC,OAAO;YAAC,CAAC;YACnC,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACnB,IAAI,QAAQ,CAAC,MAAM,IAAI,KAAK,EAAE,CAAC;gBAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;gBAAC,QAAQ,GAAG,EAAE,CAAC;YAAC,CAAC;YACnE,IAAI,SAAS,EAAE;gBAAE,EAAE,EAAE,CAAC;;gBACjB,QAAQ,GAAG,EAAE,CAAC,CAAU,oCAAoC;QACrE,CAAC;QACD,KAAK,CAAC,EAAqB;YACvB,IAAI,MAAM,EAAE,CAAC;gBAAC,EAAE,CAAC,MAAM,CAAC,CAAC;gBAAC,OAAO;YAAC,CAAC;YACnC,IAAI,QAAQ,CAAC,MAAM,EAAE,CAAC;gBAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;gBAAC,QAAQ,GAAG,EAAE,CAAC;YAAC,CAAC;YAC1D,IAAI,IAAI,EAAE;gBAAE,MAAM,CAAC,EAAE,CAAC,CAAC;;gBAClB,KAAK,GAAG,EAAE,CAAC;QACpB,CAAC;KACJ,CAAC,CAAC;IAEH,+DAA+D;IAC/D,MAAM,QAAQ,GAAI,MAAc,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IACnD,MAAc,CAAC,KAAK,GAAG,CAAC,IAAY,EAAE,EAAE,GAAG,IAAI,EAAE,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;IAEtE,SAAS,SAAS;QACd,OAAO,QAAQ,GAAG,GAAG,IAAI,QAAQ,CAAC,MAAM,GAAG,OAAO,CAAC;IACvD,CAAC;IACD,SAAS,IAAI;QACT,OAAO,QAAQ,KAAK,YAAY,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,CAAC;IACpF,CAAC;IAED,SAAS,OAAO,CAAC,KAAY;QACzB,KAAK,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,YAAY,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC;QAC3C,QAAQ,EAAE,CAAC;IACf,CAAC;IAED,MAAM,GAAG,GAAG,GAAG,EAAE,CAAC,MAAM,CAAC,cAAc,EAAE,CAAC;IAE1C,SAAS,QAAQ;QACb,qDAAqD;QACrD,IAAI,GAAG,EAAE,IAAI,KAAK,CAAC,MAAM,IAAI,CAAC,CAAC,IAAI,CAAC,MAAM,IAAI,QAAQ,CAAC,MAAM,IAAI,OAAO,CAAC,EAAE,CAAC;YACxE,MAAM,CAAC,KAAK,CAAC,iBAAiB,KAAK,CAAC,MAAM,yBAAyB,IAAI,CAAC,MAAM,cAAc,QAAQ,CAAC,MAAM,IAAI,OAAO,EAAE,CAAC,CAAC;QAC9H,CAAC;QACD,OAAO,KAAK,CAAC,MAAM,IAAI,IAAI,CAAC,MAAM,IAAI,QAAQ,CAAC,MAAM,GAAG,OAAO,EAAE,CAAC;YAC9D,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,EAAG,CAAC;YACtB,MAAM,GAAG,GAAG,KAAK,CAAC,KAAK,EAAG,CAAC;YAC3B,QAAQ,EAAE,CAAC;YACX,MAAM,CAAC,GAAG,CAAC,CAAC,EAAE,UAAU,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,IAAI,KAAK,CAAC,0BAA0B,OAAO,aAAa,GAAG,CAAC,GAAG,GAAG,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC,CAAC;YACpH,CAAC,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;YACnB,IAAI,GAAG,EAAE;gBAAE,MAAM,CAAC,KAAK,CAAC,kBAAkB,GAAG,CAAC,GAAG,KAAK,GAAG,CAAC,KAAK,CAAC,MAAM,8BAA8B,QAAQ,EAAE,CAAC,CAAC;QACpH,CAAC;IACL,CAAC;IAED,yDAAyD;IACzD,SAAS,IAAI;QACT,MAAM,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAC;QAC/B,OAAO,QAAQ,CAAC,MAAM,IAAI,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;YAAE,QAAQ,CAAC,KAAK,EAAE,CAAC;QACrE,IAAI,GAAG,EAAE,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,IAAI,QAAQ,CAAC,MAAM,KAAK,MAAM,EAAE,CAAC;YAC7D,MAAM,CAAC,KAAK,CAAC,4BAA4B,QAAQ,CAAC,MAAM,UAAU,CAAC,CAAC;QACxE,CAAC;QACD,QAAQ,EAAE,CAAC;QACX,IAAI,QAAQ,IAAI,SAAS,EAAE,EAAE,CAAC;YAAC,MAAM,CAAC,GAAG,QAAQ,CAAC;YAAC,QAAQ,GAAG,IAAI,CAAC;YAAC,CAAC,EAAE,CAAC;QAAC,CAAC;QAC1E,IAAI,KAAK,IAAI,IAAI,EAAE,EAAE,CAAC;YAAC,MAAM,CAAC,GAAG,KAAK,CAAC;YAAC,KAAK,GAAG,IAAI,CAAC;YAAC,MAAM,CAAC,CAAC,CAAC,CAAC;QAAC,CAAC;IACtE,CAAC;IAED,SAAS,QAAQ,CAAC,CAAS,EAAE,GAAQ;QACjC,MAAM,CAAC,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;QACxB,IAAI,CAAC,EAAE,CAAC;YAAC,YAAY,CAAC,CAAC,CAAC,CAAC;YAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;QAAC,CAAC;QAC7C,IAAI,GAAG,CAAC,KAAK,EAAE,CAAC;YAAC,MAAM,CAAC,KAAK,CAAC,yBAAyB,GAAG,CAAC,GAAG,KAAK,GAAG,CAAC,KAAK,EAAE,CAAC,CAAC;YAAC,IAAI,CAAC,IAAI,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC;YAAC,OAAO;QAAC,CAAC;QACtH,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;QACjC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACb,QAAQ,EAAE,CAAC;QACX,IAAI,GAAG,EAAE;YAAE,MAAM,CAAC,KAAK,CAAC,gBAAgB,GAAG,CAAC,GAAG,KAAK,GAAG,CAAC,MAAM,CAAC,MAAM,sBAAsB,QAAQ,gBAAgB,OAAO,CAAC,IAAI,cAAc,QAAQ,EAAE,CAAC,CAAC;QACzJ,kEAAkE;QAClE,OAAO,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC;YAC3B,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAE,CAAC;YACtC,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;YACzB,QAAQ,EAAE,CAAC;YACX,KAAK,MAAM,CAAC,IAAI,MAAM;gBAAE,IAAI,CAAC,KAAK,IAAI;oBAAE,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAC7D,CAAC;QACD,IAAI,EAAE,CAAC;IACX,CAAC;IAED,SAAS,IAAI,CAAC,GAAU;QACpB,IAAI,MAAM;YAAE,OAAO;QACnB,MAAM,GAAG,GAAG,CAAC;QACb,OAAO,GAAG,IAAI,CAAC;QACf,MAAM,CAAC,KAAK,CAAC,wBAAwB,GAAG,CAAC,OAAO,iBAAiB,OAAO,CAAC,MAAM,WAAW,CAAC,CAAC;QAC5F,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,MAAM,EAAE;YAAE,YAAY,CAAC,CAAC,CAAC,CAAC;QACjD,MAAM,CAAC,KAAK,EAAE,CAAC;QACf,KAAK,MAAM,CAAC,IAAI,OAAO;YAAE,CAAC,CAAC,SAAS,EAAE,CAAC;QACvC,IAAI,QAAQ,EAAE,CAAC;YAAC,MAAM,CAAC,GAAG,QAAQ,CAAC;YAAC,QAAQ,GAAG,IAAI,CAAC;YAAC,CAAC,CAAC,GAAG,CAAC,CAAC;QAAC,CAAC;aACzD,IAAI,KAAK,EAAE,CAAC;YAAC,MAAM,CAAC,GAAG,KAAK,CAAC;YAAC,KAAK,GAAG,IAAI,CAAC;YAAC,CAAC,CAAC,GAAG,CAAC,CAAC;QAAC,CAAC;;YACrD,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IAC7B,CAAC;IAED,SAAS,MAAM,CAAC,EAAqB;QACjC,OAAO,GAAG,IAAI,CAAC;QACf,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,MAAM,EAAE;YAAE,YAAY,CAAC,CAAC,CAAC,CAAC;QACjD,MAAM,CAAC,KAAK,EAAE,CAAC;QACf,MAAM,CAAC,KAAK,CAAC,iBAAiB,QAAQ,yBAAyB,OAAO,CAAC,MAAM,UAAU,CAAC,CAAC;QACzF,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;IACtF,CAAC;IAED,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QACzB,oEAAoE;QACpE,oEAAoE;QACpE,uEAAuE;QACvE,sEAAsE;QACtE,yEAAyE;QACzE,wEAAwE;QACxE,MAAM,CAAC,GAAG,IAAI,MAAM,CAAC,SAAS,EAAE;YAC5B,UAAU,EAAE,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE;YAChD,MAAM,EAAE,IAAI;YACZ,MAAM,EAAE,IAAI;SACf,CAAC,CAAC;QACH,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAChB,CAAC,CAAC,EAAE,CAAC,SAAS,EAAE,CAAC,GAAQ,EAAE,EAAE;YACzB,IAAI,GAAG,IAAI,GAAG,CAAC,KAAK,EAAE,CAAC;gBAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;gBAAC,QAAQ,EAAE,CAAC;gBAAC,IAAI,GAAG,EAAE;oBAAE,MAAM,CAAC,KAAK,CAAC,UAAU,IAAI,CAAC,MAAM,IAAI,CAAC,QAAQ,CAAC,CAAC;YAAC,CAAC;;gBAC3G,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;QAC1B,CAAC,CAAC,CAAC;QACH,CAAC,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE,GAAG,MAAM,CAAC,KAAK,CAAC,0BAA0B,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAC9F,CAAC,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,EAAE,GAAG,IAAI,IAAI,KAAK,CAAC,IAAI,CAAC,MAAM,IAAI,CAAC,OAAO;YAAE,IAAI,CAAC,IAAI,KAAK,CAAC,oCAAoC,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACrI,CAAC;IAED,MAAM,CAAC,IAAI,CAAC,kBAAkB,CAAC,0BAA0B,KAAK,mBAAmB,GAAG,aAAa,OAAO,KAAK,CAAC,CAAC;IAC9G,MAAc,CAAC,YAAY,GAAG,IAAI,CAAC,CAAG,iEAAiE;IACxG,OAAO,MAAM,CAAC;AAClB,CAAC"}
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
import { Transform } from 'stream';
|
|
2
|
+
/**
|
|
3
|
+
* Apply a Catmandu Fix script to each record.
|
|
4
|
+
*
|
|
5
|
+
* marcattacks --map fix --param fix=./my.fix ...
|
|
6
|
+
*
|
|
7
|
+
* `fix` may be a path to a Fix file or an inline Fix script. The script is
|
|
8
|
+
* parsed and compiled once; the compiled chain runs per record.
|
|
9
|
+
*/
|
|
10
|
+
export declare function createMapper(opts: {
|
|
11
|
+
fix?: string;
|
|
12
|
+
}): Promise<(data: any) => any>;
|
|
13
|
+
export declare function transform(opts: {
|
|
14
|
+
fix?: string;
|
|
15
|
+
}): Promise<Transform>;
|
|
16
|
+
//# sourceMappingURL=fix.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"fix.d.ts","sourceRoot":"","sources":["../../src/transform/fix.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,QAAQ,CAAC;AAOnC;;;;;;;GAOG;AAGH,wBAAsB,YAAY,CAAC,IAAI,EAAE;IAAE,GAAG,CAAC,EAAE,MAAM,CAAA;CAAE,GAAG,OAAO,CAAC,CAAC,IAAI,EAAE,GAAG,KAAK,GAAG,CAAC,CAMtF;AAED,wBAAsB,SAAS,CAAC,IAAI,EAAE;IAAE,GAAG,CAAC,EAAE,MAAM,CAAA;CAAE,GAAG,OAAO,CAAC,SAAS,CAAC,CAoB1E"}
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
import { Transform } from 'stream';
|
|
2
|
+
import fs from 'fs';
|
|
3
|
+
import log4js from 'log4js';
|
|
4
|
+
import { compileFix, REJECT } from '../fix/index.js';
|
|
5
|
+
const logger = log4js.getLogger();
|
|
6
|
+
/**
|
|
7
|
+
* Apply a Catmandu Fix script to each record.
|
|
8
|
+
*
|
|
9
|
+
* marcattacks --map fix --param fix=./my.fix ...
|
|
10
|
+
*
|
|
11
|
+
* `fix` may be a path to a Fix file or an inline Fix script. The script is
|
|
12
|
+
* parsed and compiled once; the compiled chain runs per record.
|
|
13
|
+
*/
|
|
14
|
+
// Pure record -> record mapper (returns REJECT for dropped records). Shared by
|
|
15
|
+
// transform() and the worker pool. Compiled once.
|
|
16
|
+
export async function createMapper(opts) {
|
|
17
|
+
let src = opts.fix ?? '';
|
|
18
|
+
if (src && fs.existsSync(src)) {
|
|
19
|
+
src = fs.readFileSync(src, { encoding: 'utf-8' });
|
|
20
|
+
}
|
|
21
|
+
return compileFix(src);
|
|
22
|
+
}
|
|
23
|
+
export async function transform(opts) {
|
|
24
|
+
let src = opts.fix ?? '';
|
|
25
|
+
if (src && fs.existsSync(src)) {
|
|
26
|
+
src = fs.readFileSync(src, { encoding: 'utf-8' });
|
|
27
|
+
}
|
|
28
|
+
const fix = compileFix(src);
|
|
29
|
+
return new Transform({
|
|
30
|
+
objectMode: true,
|
|
31
|
+
transform(data, _encoding, callback) {
|
|
32
|
+
try {
|
|
33
|
+
const out = fix(data);
|
|
34
|
+
if (out === REJECT)
|
|
35
|
+
callback(); // rejected record -> drop it
|
|
36
|
+
else
|
|
37
|
+
callback(null, out);
|
|
38
|
+
}
|
|
39
|
+
catch (err) {
|
|
40
|
+
logger.error('fix error', err.message);
|
|
41
|
+
callback(err);
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
});
|
|
45
|
+
}
|
|
46
|
+
//# sourceMappingURL=fix.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"fix.js","sourceRoot":"","sources":["../../src/transform/fix.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,QAAQ,CAAC;AACnC,OAAO,EAAE,MAAM,IAAI,CAAC;AACpB,OAAO,MAAM,MAAM,QAAQ,CAAC;AAC5B,OAAO,EAAE,UAAU,EAAE,MAAM,EAAE,MAAM,iBAAiB,CAAC;AAErD,MAAM,MAAM,GAAG,MAAM,CAAC,SAAS,EAAE,CAAC;AAElC;;;;;;;GAOG;AACH,+EAA+E;AAC/E,kDAAkD;AAClD,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,IAAsB;IACrD,IAAI,GAAG,GAAG,IAAI,CAAC,GAAG,IAAI,EAAE,CAAC;IACzB,IAAI,GAAG,IAAI,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;QAC5B,GAAG,GAAG,EAAE,CAAC,YAAY,CAAC,GAAG,EAAE,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC,CAAC;IACtD,CAAC;IACD,OAAO,UAAU,CAAC,GAAG,CAAC,CAAC;AAC3B,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,IAAsB;IAClD,IAAI,GAAG,GAAG,IAAI,CAAC,GAAG,IAAI,EAAE,CAAC;IACzB,IAAI,GAAG,IAAI,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;QAC5B,GAAG,GAAG,EAAE,CAAC,YAAY,CAAC,GAAG,EAAE,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC,CAAC;IACtD,CAAC;IACD,MAAM,GAAG,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC;IAE5B,OAAO,IAAI,SAAS,CAAC;QACjB,UAAU,EAAE,IAAI;QAChB,SAAS,CAAC,IAAS,EAAE,SAAS,EAAE,QAAQ;YACpC,IAAI,CAAC;gBACD,MAAM,GAAG,GAAG,GAAG,CAAC,IAAI,CAAC,CAAC;gBACtB,IAAI,GAAG,KAAK,MAAM;oBAAE,QAAQ,EAAE,CAAC,CAAG,6BAA6B;;oBAC1D,QAAQ,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;YAC7B,CAAC;YAAC,OAAO,GAAQ,EAAE,CAAC;gBAChB,MAAM,CAAC,KAAK,CAAC,WAAW,EAAE,GAAG,CAAC,OAAO,CAAC,CAAC;gBACvC,QAAQ,CAAC,GAAG,CAAC,CAAC;YAClB,CAAC;QACL,CAAC;KACJ,CAAC,CAAC;AACP,CAAC"}
|
|
@@ -1,4 +1,9 @@
|
|
|
1
1
|
import { Transform } from "stream";
|
|
2
|
+
export declare const autoParallel = true;
|
|
3
|
+
export declare function createMapper(opts: {
|
|
4
|
+
fix: string;
|
|
5
|
+
lookup: string;
|
|
6
|
+
}): Promise<(data: any) => any>;
|
|
2
7
|
export declare function transform(opts: {
|
|
3
8
|
fix: string;
|
|
4
9
|
lookup: string;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"jsonata.d.ts","sourceRoot":"","sources":["../../src/transform/jsonata.ts"],"names":[],"mappings":"AAAA,OAAe,EAAE,SAAS,EAAE,MAAM,QAAQ,CAAC;
|
|
1
|
+
{"version":3,"file":"jsonata.d.ts","sourceRoot":"","sources":["../../src/transform/jsonata.ts"],"names":[],"mappings":"AAAA,OAAe,EAAE,SAAS,EAAE,MAAM,QAAQ,CAAC;AAc3C,eAAO,MAAM,YAAY,OAAO,CAAC;AAKjC,wBAAsB,YAAY,CAAC,IAAI,EAAE;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,GAAI,OAAO,CAAC,CAAC,IAAI,EAAE,GAAG,KAAK,GAAG,CAAC,CA0CtG;AAED,wBAAsB,SAAS,CAAC,IAAI,EAAE;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,GAAI,OAAO,CAAC,SAAS,CAAC,CAe1F"}
|
|
@@ -6,58 +6,59 @@ import { parseStream } from '../util/tsv_parse.js';
|
|
|
6
6
|
import { v4 as uuidv4 } from 'uuid';
|
|
7
7
|
import log4js from 'log4js';
|
|
8
8
|
const logger = log4js.getLogger();
|
|
9
|
-
|
|
10
|
-
|
|
9
|
+
// JSONata is heavy/interpreted, so the `--workers auto` default should spread it
|
|
10
|
+
// across threads. (Cheap maps like `fix` are parallelizable too but do NOT set
|
|
11
|
+
// this, so auto leaves them single-threaded; an explicit --workers N still
|
|
12
|
+
// threads any parallelizable map.)
|
|
13
|
+
export const autoParallel = true;
|
|
14
|
+
// Build a pure record -> record(Promise) mapper. Shared by the in-process
|
|
15
|
+
// transform() and by the worker pool (so the heavy evaluate() can run on
|
|
16
|
+
// worker threads). Expression + helper functions are compiled once.
|
|
17
|
+
export async function createMapper(opts) {
|
|
11
18
|
let lookup = {};
|
|
12
19
|
if (opts.lookup) {
|
|
13
20
|
lookup = await loadLookup(opts.lookup);
|
|
14
21
|
}
|
|
15
|
-
|
|
22
|
+
// Resolve the query once, up front, instead of on every record.
|
|
23
|
+
let query;
|
|
24
|
+
if (opts.fix) {
|
|
25
|
+
if (fs.existsSync(opts.fix)) {
|
|
26
|
+
query = fs.readFileSync(opts.fix, { encoding: 'utf-8' });
|
|
27
|
+
}
|
|
28
|
+
else {
|
|
29
|
+
throw Error(`no such file ${opts.fix}`);
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
else {
|
|
33
|
+
query = '$';
|
|
34
|
+
}
|
|
35
|
+
logger.debug(query);
|
|
36
|
+
// The identity expression is a pure pass-through: skip jsonata entirely.
|
|
37
|
+
if (query.trim() === '$') {
|
|
38
|
+
return (data) => data;
|
|
39
|
+
}
|
|
40
|
+
// Compile the expression and register helper functions ONCE. The helpers
|
|
41
|
+
// read the record currently being processed via `current`; each evaluate
|
|
42
|
+
// is awaited before the next, so this is safe.
|
|
43
|
+
let current;
|
|
44
|
+
const expression = jsonata(query);
|
|
45
|
+
expression.registerFunction('marcmap', (code) => marcmap(current['record'], code, {}));
|
|
46
|
+
expression.registerFunction('marctag', (row) => marctag(row));
|
|
47
|
+
expression.registerFunction('marcind', (row) => marcind(row));
|
|
48
|
+
expression.registerFunction('marcsubfields', (row, regex) => marcsubfields(row, new RegExp(regex)));
|
|
49
|
+
expression.registerFunction('marcrecord', () => current['record']);
|
|
50
|
+
expression.registerFunction('asmarc', (data) => ({ "record": data }));
|
|
51
|
+
expression.registerFunction('genid', () => genid());
|
|
52
|
+
expression.registerFunction('lookup', (key) => lookup[key]);
|
|
53
|
+
return async (data) => { current = data; return expression.evaluate(data); };
|
|
54
|
+
}
|
|
55
|
+
export async function transform(opts) {
|
|
56
|
+
const mapper = await createMapper(opts);
|
|
16
57
|
return new Transform({
|
|
17
58
|
objectMode: true,
|
|
18
59
|
async transform(data, _encoding, callback) {
|
|
19
60
|
try {
|
|
20
|
-
|
|
21
|
-
if (opts.fix) {
|
|
22
|
-
if (fs.existsSync(opts.fix)) {
|
|
23
|
-
query = fs.readFileSync(opts.fix, { encoding: 'utf-8' });
|
|
24
|
-
}
|
|
25
|
-
else {
|
|
26
|
-
throw Error(`no such file ${opts.fix}`);
|
|
27
|
-
}
|
|
28
|
-
}
|
|
29
|
-
else {
|
|
30
|
-
query = '$';
|
|
31
|
-
}
|
|
32
|
-
logger.debug(query);
|
|
33
|
-
}
|
|
34
|
-
const expression = jsonata(query);
|
|
35
|
-
expression.registerFunction('marcmap', (code) => {
|
|
36
|
-
return marcmap(data['record'], code, {});
|
|
37
|
-
});
|
|
38
|
-
expression.registerFunction('marctag', (row) => {
|
|
39
|
-
return marctag(row);
|
|
40
|
-
});
|
|
41
|
-
expression.registerFunction('marcind', (row) => {
|
|
42
|
-
return marcind(row);
|
|
43
|
-
});
|
|
44
|
-
expression.registerFunction('marcsubfields', (row, regex) => {
|
|
45
|
-
return marcsubfields(row, new RegExp(regex));
|
|
46
|
-
});
|
|
47
|
-
expression.registerFunction('marcrecord', () => {
|
|
48
|
-
return data['record'];
|
|
49
|
-
});
|
|
50
|
-
expression.registerFunction('asmarc', (data) => {
|
|
51
|
-
return { "record": data };
|
|
52
|
-
});
|
|
53
|
-
expression.registerFunction('genid', () => {
|
|
54
|
-
return genid();
|
|
55
|
-
});
|
|
56
|
-
expression.registerFunction('lookup', (key) => {
|
|
57
|
-
return lookup[key];
|
|
58
|
-
});
|
|
59
|
-
data = await expression.evaluate(data);
|
|
60
|
-
callback(null, data);
|
|
61
|
+
callback(null, await mapper(data));
|
|
61
62
|
}
|
|
62
63
|
catch (err) {
|
|
63
64
|
logger.info(err);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"jsonata.js","sourceRoot":"","sources":["../../src/transform/jsonata.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,EAAE,EAAE,SAAS,EAAE,MAAM,QAAQ,CAAC;AAC3C,OAAO,OAAO,MAAM,SAAS,CAAC;AAC9B,OAAO,EAAE,MAAM,IAAI,CAAC;AACpB,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AACzE,OAAO,EAAE,WAAW,EAAE,MAAM,sBAAsB,CAAC;AACnD,OAAO,EAAE,EAAE,IAAI,MAAM,EAAE,MAAM,MAAM,CAAC;AACpC,OAAO,MAAM,MAAM,QAAQ,CAAC;AAE5B,MAAM,MAAM,GAAG,MAAM,CAAC,SAAS,EAAE,CAAC;AAElC,MAAM,CAAC,
|
|
1
|
+
{"version":3,"file":"jsonata.js","sourceRoot":"","sources":["../../src/transform/jsonata.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,EAAE,EAAE,SAAS,EAAE,MAAM,QAAQ,CAAC;AAC3C,OAAO,OAAO,MAAM,SAAS,CAAC;AAC9B,OAAO,EAAE,MAAM,IAAI,CAAC;AACpB,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AACzE,OAAO,EAAE,WAAW,EAAE,MAAM,sBAAsB,CAAC;AACnD,OAAO,EAAE,EAAE,IAAI,MAAM,EAAE,MAAM,MAAM,CAAC;AACpC,OAAO,MAAM,MAAM,QAAQ,CAAC;AAE5B,MAAM,MAAM,GAAG,MAAM,CAAC,SAAS,EAAE,CAAC;AAElC,iFAAiF;AACjF,+EAA+E;AAC/E,2EAA2E;AAC3E,mCAAmC;AACnC,MAAM,CAAC,MAAM,YAAY,GAAG,IAAI,CAAC;AAEjC,0EAA0E;AAC1E,yEAAyE;AACzE,oEAAoE;AACpE,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,IAAqC;IACpE,IAAI,MAAM,GAA2B,EAAE,CAAC;IAExC,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;QACd,MAAM,GAAG,MAAM,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAC3C,CAAC;IAED,gEAAgE;IAChE,IAAI,KAAc,CAAC;IACnB,IAAI,IAAI,CAAC,GAAG,EAAE,CAAC;QACX,IAAI,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;YAC1B,KAAK,GAAG,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,GAAG,EAAC,EAAE,QAAQ,EAAE,OAAO,EAAC,CAAC,CAAC;QAC3D,CAAC;aACI,CAAC;YACF,MAAM,KAAK,CAAC,gBAAgB,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;QAC5C,CAAC;IACL,CAAC;SACI,CAAC;QACF,KAAK,GAAG,GAAG,CAAC;IAChB,CAAC;IACD,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IAEpB,yEAAyE;IACzE,IAAI,KAAK,CAAC,IAAI,EAAE,KAAK,GAAG,EAAE,CAAC;QACvB,OAAO,CAAC,IAAS,EAAE,EAAE,CAAC,IAAI,CAAC;IAC/B,CAAC;IAED,yEAAyE;IACzE,yEAAyE;IACzE,+CAA+C;IAC/C,IAAI,OAAY,CAAC;IACjB,MAAM,UAAU,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC;IAClC,UAAU,CAAC,gBAAgB,CAAC,SAAS,EAAE,CAAC,IAAY,EAAE,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,IAAI,EAAE,EAAE,CAAC,CAAC,CAAC;IAC/F,UAAU,CAAC,gBAAgB,CAAC,SAAS,EAAE,CAAC,GAAa,EAAE,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC;IACxE,UAAU,CAAC,gBAAgB,CAAC,SAAS,EAAE,CAAC,GAAa,EAAE,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC;IACxE,UAAU,CAAC,gBAAgB,CAAC,eAAe,EAAE,CAAC,GAAa,EAAE,KAAa,EAAE,EAAE,CAAC,aAAa,CAAC,GAAG,EAAE,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IACtH,UAAU,CAAC,gBAAgB,CAAC,YAAY,EAAE,GAAG,EAAE,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC;IACnE,UAAU,CAAC,gBAAgB,CAAC,QAAQ,EAAE,CAAC,IAAgB,EAAE,EAAE,CAAC,CAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;IAClF,UAAU,CAAC,gBAAgB,CAAC,OAAO,EAAE,GAAG,EAAE,CAAC,KAAK,EAAE,CAAC,CAAC;IACpD,UAAU,CAAC,gBAAgB,CAAC,QAAQ,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;IAE5D,OAAO,KAAK,EAAE,IAAS,EAAE,EAAE,GAAG,OAAO,GAAG,IAAI,CAAC,CAAC,OAAO,UAAU,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;AACtF,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,IAAqC;IACjE,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC,IAAI,CAAC,CAAC;IAExC,OAAO,IAAI,SAAS,CAAC;QACjB,UAAU,EAAE,IAAI;QAChB,KAAK,CAAC,SAAS,CAAC,IAAS,EAAE,SAAyB,EAAE,QAAkC;YACpF,IAAI,CAAC;gBACD,QAAQ,CAAC,IAAI,EAAE,MAAM,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC;YACvC,CAAC;YACD,OAAO,GAAQ,EAAE,CAAC;gBACd,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBACjB,QAAQ,CAAC,GAAY,CAAC,CAAC;YAC3B,CAAC;QACL,CAAC;KACJ,CAAC,CAAC;AACP,CAAC;AAED,SAAS,KAAK;IACV,OAAO,SAAS,MAAM,EAAE,EAAE,CAAC;AAC/B,CAAC;AAED,KAAK,UAAU,UAAU,CAAC,IAAY;IAClC,IAAI,MAAM,GAA4B,EAAE,CAAC;IAEzC,MAAM,OAAO,GAAG,MAAM,WAAW,CAAC,EAAE,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC,CAAC;IAE7D,KAAK,MAAM,GAAG,IAAI,OAAO,EAAE,CAAC;QACxB,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;QACrC,IAAI,IAAI,IAAI,IAAI,CAAC,MAAM,IAAI,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC;YACjD,MAAM,CAAC,GAAG,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;YACvB,MAAM,CAAC,GAAG,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;YACvB,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;gBACT,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;YAClB,CAAC;QACL,CAAC;IACL,CAAC;IAED,OAAO,MAAM,CAAC;AAClB,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"marcids.d.ts","sourceRoot":"","sources":["../../src/transform/marcids.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,QAAQ,CAAC;AAGnC;;GAEG;AACH,wBAAsB,SAAS,CAAC,KAAK,EAAE,GAAG,GAAI,OAAO,CAAC,SAAS,CAAC,
|
|
1
|
+
{"version":3,"file":"marcids.d.ts","sourceRoot":"","sources":["../../src/transform/marcids.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,QAAQ,CAAC;AAGnC;;GAEG;AACH,wBAAsB,SAAS,CAAC,KAAK,EAAE,GAAG,GAAI,OAAO,CAAC,SAAS,CAAC,CAqB/D"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"marcids.js","sourceRoot":"","sources":["../../src/transform/marcids.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,QAAQ,CAAC;AACnC,OAAO,EAAE,OAAO,EAAE,MAAM,eAAe,CAAC;AAExC;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,KAAU;IACtC,OAAO,IAAI,SAAS,CAAC;QACjB,UAAU,EAAE,IAAI;QAChB,SAAS,CAAC,IAAS,EAAE,SAAS,EAAE,QAAQ;YACpC,IAAI,GAAG,GAAgB,IAAI,CAAC,QAAQ,CAAC,CAAC;YAEtC,IAAI,CAAC,GAAG,EAAE,CAAC;gBACP,QAAQ,CAAC,IAAI,CAAC,CAAC;gBACf,OAAO;YACX,CAAC;YAED,IAAI,EAAE,GAAG,OAAO,CAAC,GAAG,EAAC,KAAK,EAAC,EAAE,CAAC,CAAC;YAE/B,IAAI,EAAE,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;gBACjB,QAAQ,CAAC,IAAI,EAAC,EAAC,EAAE,EAAE,EAAE,CAAC,CAAC,CAAC,EAAC,CAAC,CAAC;YAC/B,CAAC;QACL,CAAC;KACJ,CAAC,CAAC;AACP,CAAC"}
|
|
1
|
+
{"version":3,"file":"marcids.js","sourceRoot":"","sources":["../../src/transform/marcids.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,QAAQ,CAAC;AACnC,OAAO,EAAE,OAAO,EAAE,MAAM,eAAe,CAAC;AAExC;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,KAAU;IACtC,OAAO,IAAI,SAAS,CAAC;QACjB,UAAU,EAAE,IAAI;QAChB,SAAS,CAAC,IAAS,EAAE,SAAS,EAAE,QAAQ;YACpC,IAAI,GAAG,GAAgB,IAAI,CAAC,QAAQ,CAAC,CAAC;YAEtC,IAAI,CAAC,GAAG,EAAE,CAAC;gBACP,QAAQ,CAAC,IAAI,CAAC,CAAC;gBACf,OAAO;YACX,CAAC;YAED,IAAI,EAAE,GAAG,OAAO,CAAC,GAAG,EAAC,KAAK,EAAC,EAAE,CAAC,CAAC;YAE/B,IAAI,EAAE,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;gBACjB,QAAQ,CAAC,IAAI,EAAC,EAAC,EAAE,EAAE,EAAE,CAAC,CAAC,CAAC,EAAC,CAAC,CAAC;YAC/B,CAAC;iBACI,CAAC;gBACF,QAAQ,CAAC,IAAI,CAAC,CAAC;YACnB,CAAC;QACL,CAAC;KACJ,CAAC,CAAC;AACP,CAAC"}
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Marker for record objects whose string values are already free of control
|
|
3
|
+
* characters (\x00-\x1F and \x7F).
|
|
4
|
+
*
|
|
5
|
+
* Input readers that strip control characters while parsing (e.g. the XML
|
|
6
|
+
* readers) set this on the records they emit. Output writers that would
|
|
7
|
+
* otherwise re-strip every value (e.g. alephseq) can then skip that work.
|
|
8
|
+
*
|
|
9
|
+
* It is a Symbol so it never shows up in `JSON.stringify`, `Object.keys` or
|
|
10
|
+
* `for...in` — outputs that serialise the whole record object (json, jsonl,
|
|
11
|
+
* rdf) are unaffected. A transform that builds a new record object (e.g. a
|
|
12
|
+
* JSONata `fix`) naturally drops the marker, so escaping is re-applied unless
|
|
13
|
+
* cleanliness is re-asserted — a safe default.
|
|
14
|
+
*/
|
|
15
|
+
export declare const CLEAN: unique symbol;
|
|
16
|
+
export type CleanFlag = {
|
|
17
|
+
[CLEAN]?: boolean;
|
|
18
|
+
};
|
|
19
|
+
//# sourceMappingURL=marc_record.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"marc_record.d.ts","sourceRoot":"","sources":["../../src/util/marc_record.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AACH,eAAO,MAAM,KAAK,EAAE,OAAO,MAAoC,CAAC;AAEhE,MAAM,MAAM,SAAS,GAAG;IAAE,CAAC,KAAK,CAAC,CAAC,EAAE,OAAO,CAAA;CAAE,CAAC"}
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Marker for record objects whose string values are already free of control
|
|
3
|
+
* characters (\x00-\x1F and \x7F).
|
|
4
|
+
*
|
|
5
|
+
* Input readers that strip control characters while parsing (e.g. the XML
|
|
6
|
+
* readers) set this on the records they emit. Output writers that would
|
|
7
|
+
* otherwise re-strip every value (e.g. alephseq) can then skip that work.
|
|
8
|
+
*
|
|
9
|
+
* It is a Symbol so it never shows up in `JSON.stringify`, `Object.keys` or
|
|
10
|
+
* `for...in` — outputs that serialise the whole record object (json, jsonl,
|
|
11
|
+
* rdf) are unaffected. A transform that builds a new record object (e.g. a
|
|
12
|
+
* JSONata `fix`) naturally drops the marker, so escaping is re-applied unless
|
|
13
|
+
* cleanliness is re-asserted — a safe default.
|
|
14
|
+
*/
|
|
15
|
+
export const CLEAN = Symbol('marc.record.clean');
|
|
16
|
+
//# sourceMappingURL=marc_record.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"marc_record.js","sourceRoot":"","sources":["../../src/util/marc_record.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AACH,MAAM,CAAC,MAAM,KAAK,GAAkB,MAAM,CAAC,mBAAmB,CAAC,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"worker-str.d.ts","sourceRoot":"","sources":["../src/worker-str.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"worker-str.js","sourceRoot":"","sources":["../src/worker-str.ts"],"names":[],"mappings":"AAAA,gFAAgF;AAChF,8EAA8E;AAC9E,4EAA4E;AAC5E,EAAE;AACF,+EAA+E;AAC/E,YAAY;AACZ,0EAA0E;AAC1E,qCAAqC;AACrC,mEAAmE;AACnE,OAAO,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AAC7D,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AACvC,OAAO,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AAChD,OAAO,EAAE,MAAM,EAAE,MAAM,iBAAiB,CAAC;AAEzC,MAAM,EAAE,IAAI,EAAE,GAAG,EAAE,KAAK,EAAE,GAAG,UAAwD,CAAC;AAEtF,MAAM,QAAQ,GAAG,MAAM,UAAU,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;AACjD,MAAM,MAAM,GAAuB,GAAG;IAClC,CAAC,CAAC,MAAM,CAAC,MAAM,UAAU,CAAC,GAAG,EAAE,WAAW,CAAC,CAAC,CAAC,YAAY,CAAC,KAAK,IAAI,EAAE,CAAC;IACtE,CAAC,CAAC,CAAC,CAAM,EAAE,EAAE,CAAC,CAAC,CAAC;AAEpB,6EAA6E;AAC7E,KAAK,UAAU,KAAK,CAAC,IAAY;IAC7B,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,SAAS,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC;IAClD,MAAM,OAAO,GAAU,EAAE,CAAC;IAC1B,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QACxC,GAAG,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,CAAM,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;QAC5C,GAAG,CAAC,EAAE,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;QACvB,GAAG,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;QACxB,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAClC,CAAC,CAAC,CAAC;IACH,OAAO,OAAO,CAAC;AACnB,CAAC;AAED,UAAW,CAAC,WAAW,CAAC,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;AAEzC,UAAW,CAAC,EAAE,CAAC,SAAS,EAAE,KAAK,EAAE,GAAkC,EAAE,EAAE;IACnE,IAAI,CAAC;QACD,MAAM,OAAO,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACtC,IAAI,GAAG,GAAG,EAAE,CAAC;QACb,IAAI,CAAC,GAAG,CAAC,CAAC;QACV,KAAK,MAAM,GAAG,IAAI,OAAO,EAAE,CAAC;YACxB,MAAM,CAAC,GAAG,MAAM,MAAM,CAAC,GAAG,CAAC,CAAC;YAC5B,IAAI,CAAC,KAAK,MAAM,IAAI,CAAC,KAAK,SAAS,IAAI,CAAC,KAAK,IAAI;gBAAE,SAAS;YAC5D,IAAI,OAAO,CAAC,KAAK,QAAQ,IAAI,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,MAAM,KAAK,CAAC;gBAAE,SAAS,CAAC,uBAAuB;YAC3F,GAAG,IAAI,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC;YAChC,CAAC,EAAE,CAAC;QACR,CAAC;QACD,UAAW,CAAC,WAAW,CAAC,EAAE,GAAG,EAAE,GAAG,CAAC,GAAG,EAAE,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC;IACtD,CAAC;IAAC,OAAO,GAAQ,EAAE,CAAC;QAChB,UAAW,CAAC,WAAW,CAAC,EAAE,GAAG,EAAE,GAAG,CAAC,GAAG,EAAE,KAAK,EAAE,GAAG,EAAE,OAAO,IAAI,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IAClF,CAAC;AACL,CAAC,CAAC,CAAC"}
|
package/dist/worker.d.ts
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"worker.d.ts","sourceRoot":"","sources":["../src/worker.ts"],"names":[],"mappings":""}
|
package/dist/worker.js
ADDED
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
// Worker thread: runs a parallelizable map (e.g. jsonata, fix) on batches of
|
|
2
|
+
// records. The heavy per-record work (jsonata evaluate / fix chain) runs here;
|
|
3
|
+
// the main thread does I/O, parsing and serialization.
|
|
4
|
+
//
|
|
5
|
+
// Protocol:
|
|
6
|
+
// main -> worker : { seq, batch: record[] }
|
|
7
|
+
// worker -> main : { ready: true } (once, on startup)
|
|
8
|
+
// { seq, mapped: (record|null)[] } (null = rejected)
|
|
9
|
+
// { seq, error: string }
|
|
10
|
+
import { parentPort, workerData } from 'node:worker_threads';
|
|
11
|
+
import { loadPlugin } from './plugin-loader.js';
|
|
12
|
+
import { REJECT } from './fix/signal.js';
|
|
13
|
+
const { map, param } = workerData;
|
|
14
|
+
const mod = await loadPlugin(map, 'transform');
|
|
15
|
+
if (typeof mod.createMapper !== 'function') {
|
|
16
|
+
throw new Error(`map '${map}' is not parallelizable (no createMapper)`);
|
|
17
|
+
}
|
|
18
|
+
const mapper = await mod.createMapper(param ?? {});
|
|
19
|
+
parentPort.postMessage({ ready: true });
|
|
20
|
+
parentPort.on('message', async (msg) => {
|
|
21
|
+
try {
|
|
22
|
+
const mapped = [];
|
|
23
|
+
for (const rec of msg.batch) {
|
|
24
|
+
const r = await mapper(rec);
|
|
25
|
+
mapped.push(r === REJECT ? null : r); // null marks a dropped record
|
|
26
|
+
}
|
|
27
|
+
parentPort.postMessage({ seq: msg.seq, mapped });
|
|
28
|
+
}
|
|
29
|
+
catch (err) {
|
|
30
|
+
parentPort.postMessage({ seq: msg.seq, error: err?.message ?? String(err) });
|
|
31
|
+
}
|
|
32
|
+
});
|
|
33
|
+
//# sourceMappingURL=worker.js.map
|