marcattacks 2.5.1 → 2.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (96) hide show
  1. package/README.md +82 -2
  2. package/dist/attacker.d.ts.map +1 -1
  3. package/dist/attacker.js +10 -0
  4. package/dist/attacker.js.map +1 -1
  5. package/dist/command.js +1 -0
  6. package/dist/command.js.map +1 -1
  7. package/dist/fix/binds.d.ts +5 -0
  8. package/dist/fix/binds.d.ts.map +1 -0
  9. package/dist/fix/binds.js +82 -0
  10. package/dist/fix/binds.js.map +1 -0
  11. package/dist/fix/conditions.d.ts +5 -0
  12. package/dist/fix/conditions.d.ts.map +1 -0
  13. package/dist/fix/conditions.js +111 -0
  14. package/dist/fix/conditions.js.map +1 -0
  15. package/dist/fix/fixes.d.ts +7 -0
  16. package/dist/fix/fixes.d.ts.map +1 -0
  17. package/dist/fix/fixes.js +478 -0
  18. package/dist/fix/fixes.js.map +1 -0
  19. package/dist/fix/index.d.ts +12 -0
  20. package/dist/fix/index.d.ts.map +1 -0
  21. package/dist/fix/index.js +40 -0
  22. package/dist/fix/index.js.map +1 -0
  23. package/dist/fix/parser.d.ts +25 -0
  24. package/dist/fix/parser.d.ts.map +1 -0
  25. package/dist/fix/parser.js +156 -0
  26. package/dist/fix/parser.js.map +1 -0
  27. package/dist/fix/path.d.ts +23 -0
  28. package/dist/fix/path.d.ts.map +1 -0
  29. package/dist/fix/path.js +351 -0
  30. package/dist/fix/path.js.map +1 -0
  31. package/dist/fix/signal.d.ts +2 -0
  32. package/dist/fix/signal.d.ts.map +1 -0
  33. package/dist/fix/signal.js +5 -0
  34. package/dist/fix/signal.js.map +1 -0
  35. package/dist/fix/util.d.ts +6 -0
  36. package/dist/fix/util.d.ts.map +1 -0
  37. package/dist/fix/util.js +133 -0
  38. package/dist/fix/util.js.map +1 -0
  39. package/dist/input/alephseq.d.ts.map +1 -1
  40. package/dist/input/alephseq.js +18 -5
  41. package/dist/input/alephseq.js.map +1 -1
  42. package/dist/input/fastxml.d.ts +3 -0
  43. package/dist/input/fastxml.d.ts.map +1 -0
  44. package/dist/input/fastxml.js +134 -0
  45. package/dist/input/fastxml.js.map +1 -0
  46. package/dist/input/xml.d.ts.map +1 -1
  47. package/dist/input/xml.js +4 -2
  48. package/dist/input/xml.js.map +1 -1
  49. package/dist/marcmap.d.ts.map +1 -1
  50. package/dist/marcmap.js +15 -3
  51. package/dist/marcmap.js.map +1 -1
  52. package/dist/output/alephseq.d.ts.map +1 -1
  53. package/dist/output/alephseq.js +29 -5
  54. package/dist/output/alephseq.js.map +1 -1
  55. package/dist/output/null.d.ts +3 -0
  56. package/dist/output/null.d.ts.map +1 -0
  57. package/dist/output/null.js +13 -0
  58. package/dist/output/null.js.map +1 -0
  59. package/dist/output/xml.d.ts.map +1 -1
  60. package/dist/output/xml.js +27 -0
  61. package/dist/output/xml.js.map +1 -1
  62. package/dist/stream/framer.d.ts.map +1 -0
  63. package/dist/stream/framer.js.map +1 -0
  64. package/dist/stream/worker-pool-str.d.ts.map +1 -0
  65. package/dist/stream/worker-pool-str.js.map +1 -0
  66. package/dist/stream/worker-pool.d.ts +10 -0
  67. package/dist/stream/worker-pool.d.ts.map +1 -0
  68. package/dist/stream/worker-pool.js +206 -0
  69. package/dist/stream/worker-pool.js.map +1 -0
  70. package/dist/transform/fix.d.ts +16 -0
  71. package/dist/transform/fix.d.ts.map +1 -0
  72. package/dist/transform/fix.js +46 -0
  73. package/dist/transform/fix.js.map +1 -0
  74. package/dist/transform/jsonata.d.ts +4 -0
  75. package/dist/transform/jsonata.d.ts.map +1 -1
  76. package/dist/transform/jsonata.js +40 -44
  77. package/dist/transform/jsonata.js.map +1 -1
  78. package/dist/util/marc_record.d.ts +19 -0
  79. package/dist/util/marc_record.d.ts.map +1 -0
  80. package/dist/util/marc_record.js +16 -0
  81. package/dist/util/marc_record.js.map +1 -0
  82. package/dist/worker-str.d.ts.map +1 -0
  83. package/dist/worker-str.js.map +1 -0
  84. package/dist/worker.d.ts +2 -0
  85. package/dist/worker.d.ts.map +1 -0
  86. package/dist/worker.js +33 -0
  87. package/dist/worker.js.map +1 -0
  88. package/package.json +4 -1
  89. package/dist/output/mulitipart.d.ts +0 -7
  90. package/dist/output/mulitipart.d.ts.map +0 -1
  91. package/dist/output/mulitipart.js +0 -53
  92. package/dist/output/mulitipart.js.map +0 -1
  93. package/dist/output/text.d.ts +0 -6
  94. package/dist/output/text.d.ts.map +0 -1
  95. package/dist/output/text.js +0 -34
  96. package/dist/output/text.js.map +0 -1
@@ -1 +1 @@
1
- {"version":3,"file":"xml.js","sourceRoot":"","sources":["../../src/output/xml.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,QAAQ,CAAC;AACnC,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,aAAa,EAAG,cAAc,EAAC,MAAM,eAAe,CAAC;AAChF,OAAO,MAAM,MAAM,QAAQ,CAAC;AAE5B,MAAM,MAAM,GAAG,MAAM,CAAC,SAAS,EAAE,CAAC;AAElC,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,KAAS;IACrC,IAAI,OAAO,GAAG,IAAI,CAAC;IAEnB,OAAO,IAAI,SAAS,CAAC;QACjB,UAAU,EAAE,IAAI;QAChB,SAAS,CAAC,IAAS,EAAE,SAAS,EAAE,QAAQ;YACpC,IAAI,GAAG,GAAgB,IAAI,CAAC,QAAQ,CAAC,CAAC;YAEtC,IAAI,CAAC,GAAG,EAAE,CAAC;gBACP,MAAM,CAAC,KAAK,CAAC,sBAAsB,CAAC,CAAC;gBACrC,QAAQ,EAAE,CAAA;gBACV,OAAO;YACX,CAAC;YAED,IAAI,MAAM,GAAG,EAAE,CAAC;YAEhB,IAAI,OAAO,EAAE,CAAC;gBACV,MAAM,IAAI,8CAA8C,CAAC;gBACzD,MAAM,IAAI,mEAAmE,CAAC;gBAC9E,OAAO,GAAG,KAAK,CAAC;YACpB,CAAC;YAED,MAAM,IAAI,kBAAkB,CAAC;YAE7B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAG,CAAC,GAAG,GAAG,CAAC,MAAM,EAAG,CAAC,EAAE,EAAE,CAAC;gBACpC,IAAI,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;gBAC1B,IAAI,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;gBAC1B,IAAI,GAAG,KAAK,KAAK,EAAE,CAAC,CAAA,CAAC;qBAChB,IAAI,GAAG,KAAK,KAAK,EAAE,CAAC;oBACrB,IAAI,KAAK,GAAG,aAAa,CAAC,GAAG,CAAC,CAAC,CAAE,EAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;oBAC3C,MAAM,IAAI,kBAAkB,SAAS,CAAC,KAAK,CAAC,kBAAkB,CAAC;gBACnE,CAAC;qBACI,IAAI,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC;oBACxB,IAAI,KAAK,GAAG,aAAa,CAAC,GAAG,CAAC,CAAC,CAAE,EAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;oBAC3C,MAAM,IAAI,6BAA6B,SAAS,CAAC,GAAG,EAAC,EAAC,YAAY,EAAC,IAAI,EAAC,CAAC,KAAK,SAAS,CAAC,KAAK,CAAC,wBAAwB,CAAC;gBAC3H,CAAC;qBACI,CAAC;oBACF,MAAM,IAAI,0BAA0B,SAAS,CAAC,GAAG,CAAC,WAAW,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAC,EAAC,YAAY,EAAC,IAAI,EAAC,CAAC,WAAW,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAC,EAAC,YAAY,EAAC,IAAI,EAAC,CAAC,MAAM,CAAC;oBACzJ,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,IAAI,EAAC,KAAK,EAAE,EAAE;wBAClC,MAAM,IAAI,4BAA4B,SAAS,CAAC,IAAI,CAAC,KAAK,SAAS,CAAC,KAAK,CAAC,oBAAoB,CAAC;oBACnG,CAAC,CAAC,CAAC;oBACH,MAAM,IAAI,uBAAuB,CAAC;gBACtC,CAAC;YACL,CAAC;YAED,MAAM,IAAI,mBAAmB,CAAC;YAE9B,MAAM,CAAC,KAAK,CAAC,UAAU,MAAM,CAAC,MAAM,QAAQ,CAAC,CAAC;YAE9C,QAAQ,CAAC,IAAI,EAAC,MAAM,CAAC,CAAC;QAC1B,CAAC;QACD,KAAK,CAAC,QAAQ;YACV,MAAM,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC;YAC9B,IAAI,CAAC,OAAO,EAAE,CAAC;gBACX,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;gBACzB,IAAI,MAAM,GAAG,sBAAsB,CAAC;gBACpC,MAAM,CAAC,KAAK,CAAC,UAAU,MAAM,CAAC,MAAM,QAAQ,CAAC,CAAC;gBAC9C,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACtB,CAAC;YACD,QAAQ,EAAE,CAAC;QACf,CAAC;KACJ,CAAC,CAAC;AACP,CAAC;AAED,MAAM,UAAU,SAAS,CACvB,KAAyC,EACzC,OAAoC;IAElC,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,KAAK,SAAS;QAAE,OAAO,EAAE,CAAC;IAErD,IAAI,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;IAEtB,qEAAqE;IAErE,6EAA6E;IAC7E,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,qCAAqC,EAAE,QAAQ,CAAC,CAAC,CAAE,2BAA2B;IAC5F,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,sCAAsC,EAAE,QAAQ,CAAC,CAAC,CAAC,0BAA0B;IAE3F,8CAA8C;IAC9C,6DAA6D;IAC7D,yDAAyD;IACzD,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,oEAAoE,EAAE,EAAE,CAAC,CAAC;IAExF,+DAA+D;IAC/D,4DAA4D;IAC5D,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,kBAAkB,EAAE,EAAE,CAAC,CAAC;IAEtC,wCAAwC;IACxC,4DAA4D;IAC5D,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAE7B,kCAAkC;IAClC,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;IAC5B,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;IAE5B,IAAI,OAAO,EAAE,YAAY,EAAE,CAAC;QACxB,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;QAC9B,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;IAClC,CAAC;IAED,OAAO,CAAC,CAAC;AACb,CAAC"}
1
+ {"version":3,"file":"xml.js","sourceRoot":"","sources":["../../src/output/xml.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,QAAQ,CAAC;AACnC,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,aAAa,EAAG,cAAc,EAAC,MAAM,eAAe,CAAC;AAChF,OAAO,MAAM,MAAM,QAAQ,CAAC;AAE5B,MAAM,MAAM,GAAG,MAAM,CAAC,SAAS,EAAE,CAAC;AAElC,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,KAAS;IACrC,IAAI,OAAO,GAAG,IAAI,CAAC;IAEnB,OAAO,IAAI,SAAS,CAAC;QACjB,UAAU,EAAE,IAAI;QAChB,SAAS,CAAC,IAAS,EAAE,SAAS,EAAE,QAAQ;YACpC,IAAI,GAAG,GAAgB,IAAI,CAAC,QAAQ,CAAC,CAAC;YAEtC,IAAI,CAAC,GAAG,EAAE,CAAC;gBACP,MAAM,CAAC,KAAK,CAAC,sBAAsB,CAAC,CAAC;gBACrC,QAAQ,EAAE,CAAA;gBACV,OAAO;YACX,CAAC;YAED,IAAI,MAAM,GAAG,EAAE,CAAC;YAEhB,IAAI,OAAO,EAAE,CAAC;gBACV,MAAM,IAAI,8CAA8C,CAAC;gBACzD,MAAM,IAAI,mEAAmE,CAAC;gBAC9E,OAAO,GAAG,KAAK,CAAC;YACpB,CAAC;YAED,MAAM,IAAI,kBAAkB,CAAC;YAE7B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAG,CAAC,GAAG,GAAG,CAAC,MAAM,EAAG,CAAC,EAAE,EAAE,CAAC;gBACpC,IAAI,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;gBAC1B,IAAI,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;gBAC1B,IAAI,GAAG,KAAK,KAAK,EAAE,CAAC,CAAA,CAAC;qBAChB,IAAI,GAAG,KAAK,KAAK,EAAE,CAAC;oBACrB,IAAI,KAAK,GAAG,aAAa,CAAC,GAAG,CAAC,CAAC,CAAE,EAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;oBAC3C,MAAM,IAAI,kBAAkB,SAAS,CAAC,KAAK,CAAC,kBAAkB,CAAC;gBACnE,CAAC;qBACI,IAAI,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC;oBACxB,IAAI,KAAK,GAAG,aAAa,CAAC,GAAG,CAAC,CAAC,CAAE,EAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;oBAC3C,MAAM,IAAI,6BAA6B,SAAS,CAAC,GAAG,EAAC,EAAC,YAAY,EAAC,IAAI,EAAC,CAAC,KAAK,SAAS,CAAC,KAAK,CAAC,wBAAwB,CAAC;gBAC3H,CAAC;qBACI,CAAC;oBACF,MAAM,IAAI,0BAA0B,SAAS,CAAC,GAAG,CAAC,WAAW,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAC,EAAC,YAAY,EAAC,IAAI,EAAC,CAAC,WAAW,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAC,EAAC,YAAY,EAAC,IAAI,EAAC,CAAC,MAAM,CAAC;oBACzJ,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,IAAI,EAAC,KAAK,EAAE,EAAE;wBAClC,MAAM,IAAI,4BAA4B,SAAS,CAAC,IAAI,CAAC,KAAK,SAAS,CAAC,KAAK,CAAC,oBAAoB,CAAC;oBACnG,CAAC,CAAC,CAAC;oBACH,MAAM,IAAI,uBAAuB,CAAC;gBACtC,CAAC;YACL,CAAC;YAED,MAAM,IAAI,mBAAmB,CAAC;YAE9B,MAAM,CAAC,KAAK,CAAC,UAAU,MAAM,CAAC,MAAM,QAAQ,CAAC,CAAC;YAE9C,QAAQ,CAAC,IAAI,EAAC,MAAM,CAAC,CAAC;QAC1B,CAAC;QACD,KAAK,CAAC,QAAQ;YACV,MAAM,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC;YAC9B,IAAI,CAAC,OAAO,EAAE,CAAC;gBACX,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;gBACzB,IAAI,MAAM,GAAG,sBAAsB,CAAC;gBACpC,MAAM,CAAC,KAAK,CAAC,UAAU,MAAM,CAAC,MAAM,QAAQ,CAAC,CAAC;gBAC9C,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACtB,CAAC;YACD,QAAQ,EAAE,CAAC;QACf,CAAC;KACJ,CAAC,CAAC;AACP,CAAC;AAED,0EAA0E;AAC1E,6EAA6E;AAC7E,6EAA6E;AAC7E,oEAAoE;AACpE,SAAS,WAAW,CAAC,CAAS,EAAE,YAAqB;IACjD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAChC,MAAM,CAAC,GAAG,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;QAC1B,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE;YAAE,OAAO,IAAI,CAAC,CAAiB,QAAQ;QAC3E,IAAI,YAAY,IAAI,CAAC,CAAC,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,CAAC;YAAE,OAAO,IAAI,CAAC,CAAW,MAAM;QACzE,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,IAAI,CAAC;YAAE,OAAO,IAAI,CAAC,CAAC,OAAO;QAC3F,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,IAAI;YAAE,OAAO,IAAI,CAAC;QACxC,IAAI,CAAC,IAAI,MAAM,IAAI,CAAC,IAAI,MAAM;YAAE,OAAO,IAAI,CAAC,CAAuB,aAAa;QAChF,IAAI,CAAC,IAAI,MAAM,IAAI,CAAC,IAAI,MAAM;YAAE,OAAO,IAAI,CAAC,CAAuB,YAAY;QAC/E,IAAI,CAAC,KAAK,MAAM,IAAI,CAAC,KAAK,MAAM;YAAE,OAAO,IAAI,CAAC;IAClD,CAAC;IACD,OAAO,KAAK,CAAC;AACjB,CAAC;AAED,MAAM,UAAU,SAAS,CACvB,KAAyC,EACzC,OAAoC;IAElC,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,KAAK,SAAS;QAAE,OAAO,EAAE,CAAC;IAErD,IAAI,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;IAEtB,sEAAsE;IACtE,IAAI,CAAC,WAAW,CAAC,CAAC,EAAE,OAAO,EAAE,YAAY,KAAK,IAAI,CAAC;QAAE,OAAO,CAAC,CAAC;IAE9D,qEAAqE;IAErE,6EAA6E;IAC7E,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,qCAAqC,EAAE,QAAQ,CAAC,CAAC,CAAE,2BAA2B;IAC5F,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,sCAAsC,EAAE,QAAQ,CAAC,CAAC,CAAC,0BAA0B;IAE3F,8CAA8C;IAC9C,6DAA6D;IAC7D,yDAAyD;IACzD,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,oEAAoE,EAAE,EAAE,CAAC,CAAC;IAExF,+DAA+D;IAC/D,4DAA4D;IAC5D,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,kBAAkB,EAAE,EAAE,CAAC,CAAC;IAEtC,wCAAwC;IACxC,4DAA4D;IAC5D,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAE7B,kCAAkC;IAClC,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;IAC5B,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;IAE5B,IAAI,OAAO,EAAE,YAAY,EAAE,CAAC;QACxB,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;QAC9B,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;IAClC,CAAC;IAED,OAAO,CAAC,CAAC;AACb,CAAC"}
@@ -0,0 +1 @@
1
+ {"version":3,"file":"framer.d.ts","sourceRoot":"","sources":["../../src/stream/framer.ts"],"names":[],"mappings":"AAUA,OAAO,EAAE,SAAS,EAA0B,MAAM,QAAQ,CAAC;AAG3D,eAAO,MAAM,QAAQ,aAAmD,CAAC;AAkCzE,wBAAgB,YAAY,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,SAAS,CA+BnE"}
@@ -0,0 +1 @@
1
+ {"version":3,"file":"framer.js","sourceRoot":"","sources":["../../src/stream/framer.ts"],"names":[],"mappings":"AAAA,8EAA8E;AAC9E,6EAA6E;AAC7E,+EAA+E;AAC/E,+EAA+E;AAC/E,4BAA4B;AAC5B,EAAE;AACF,+EAA+E;AAC/E,uDAAuD;AACvD,yEAAyE;AACzE,mEAAmE;AACnE,OAAO,EAAE,SAAS,EAA0B,MAAM,QAAQ,CAAC;AAC3D,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AAEpD,MAAM,CAAC,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,CAAC,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,UAAU,CAAC,CAAC,CAAC;AAEzE,MAAM,YAAY,GAAG,6BAA6B,CAAC;AAEnD,+EAA+E;AAC/E,+EAA+E;AAC/E,SAAS,kBAAkB,CAAC,IAAY,EAAE,GAAW;IACjD,IAAI,IAAI,KAAK,OAAO,EAAE,CAAC;QACnB,MAAM,IAAI,GAAa,EAAE,CAAC;QAC1B,KAAK,IAAI,CAAC,GAAG,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,GAAG,GAAG,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,GAAG,CAAC,CAAC;YAAE,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QACvF,OAAO,IAAI,CAAC;IAChB,CAAC;IACD,IAAI,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;QACvC,MAAM,IAAI,GAAa,EAAE,CAAC;QAC1B,YAAY,CAAC,SAAS,GAAG,CAAC,CAAC;QAC3B,IAAI,CAAyB,CAAC;QAC9B,OAAO,CAAC,CAAC,GAAG,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAAE,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;QACtE,OAAO,IAAI,CAAC;IAChB,CAAC;IACD,2EAA2E;IAC3E,oEAAoE;IACpE,MAAM,IAAI,GAAa,EAAE,CAAC;IAC1B,IAAI,MAAM,GAAkB,IAAI,CAAC;IACjC,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,KAAK,IAAI,CAAC,GAAG,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,GAAG,GAAG,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC;QACnE,MAAM,EAAE,GAAG,GAAG,CAAC,OAAO,CAAC,GAAG,EAAE,SAAS,CAAC,CAAC;QACvC,MAAM,EAAE,GAAG,EAAE,IAAI,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC;QAClF,IAAI,MAAM,KAAK,IAAI,IAAI,EAAE,KAAK,MAAM;YAAE,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,yBAAyB;QACrF,MAAM,GAAG,EAAE,CAAC;QACZ,SAAS,GAAG,CAAC,GAAG,CAAC,CAAC;IACtB,CAAC;IACD,OAAO,IAAI,CAAC;AAChB,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,IAAY,EAAE,KAAa;IACpD,MAAM,OAAO,GAAG,IAAI,aAAa,CAAC,MAAM,CAAC,CAAC;IAC1C,IAAI,GAAG,GAAG,EAAE,CAAC;IACb,IAAI,GAAG,GAAG,CAAC,CAAC;IAEZ,SAAS,IAAI,CAAC,MAAiB,EAAE,KAAc;QAC3C,SAAS,CAAC;YACN,MAAM,IAAI,GAAG,kBAAkB,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;YAC3C,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;gBAAE,MAAM;YAC7B,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,MAAM,GAAG,KAAK;gBAAE,MAAM,CAAO,wBAAwB;YACxE,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAE,CAAC;YACpD,MAAM,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,GAAG,EAAE,EAAE,IAAI,EAAE,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;YACrD,GAAG,GAAG,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;YACrB,IAAI,IAAI,CAAC,MAAM,GAAG,KAAK;gBAAE,MAAM,CAAiB,mBAAmB;QACvE,CAAC;IACL,CAAC;IAED,OAAO,IAAI,SAAS,CAAC;QACjB,kBAAkB,EAAE,IAAI;QACxB,SAAS,CAAC,KAAU,EAAE,IAAY,EAAE,EAAqB;YACrD,GAAG,IAAI,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;YAC5B,IAAI,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;YAClB,EAAE,EAAE,CAAC;QACT,CAAC;QACD,KAAK,CAAC,EAAqB;YACvB,GAAG,IAAI,OAAO,CAAC,GAAG,EAAE,CAAC;YACrB,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;YACjB,IAAI,GAAG,CAAC,MAAM,GAAG,CAAC;gBAAE,IAAI,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,GAAG,EAAE,EAAE,IAAI,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC,0BAA0B;YACpF,EAAE,EAAE,CAAC;QACT,CAAC;KACJ,CAAC,CAAC;AACP,CAAC"}
@@ -0,0 +1 @@
1
+ {"version":3,"file":"worker-pool-str.d.ts","sourceRoot":"","sources":["../../src/stream/worker-pool-str.ts"],"names":[],"mappings":"AAMA,OAAO,EAAE,SAAS,EAA0B,MAAM,QAAQ,CAAC;AAM3D,MAAM,WAAW,WAAW;IAAG,IAAI,EAAE,MAAM,CAAC;IAAC,GAAG,CAAC,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,GAAG,CAAC;IAAC,OAAO,EAAE,MAAM,CAAC;CAAE;AAEzF,wBAAgB,sBAAsB,CAAC,IAAI,EAAE,WAAW,GAAG,SAAS,CA4FnE"}
@@ -0,0 +1 @@
1
+ {"version":3,"file":"worker-pool-str.js","sourceRoot":"","sources":["../../src/stream/worker-pool-str.ts"],"names":[],"mappings":"AAAA,+EAA+E;AAC/E,gFAAgF;AAChF,mDAAmD;AACnD,EAAE;AACF,wDAAwD;AACxD,sDAAsD;AACtD,OAAO,EAAE,SAAS,EAA0B,MAAM,QAAQ,CAAC;AAC3D,OAAO,EAAE,MAAM,EAAE,MAAM,qBAAqB,CAAC;AAC7C,OAAO,MAAM,MAAM,QAAQ,CAAC;AAE5B,MAAM,MAAM,GAAG,MAAM,CAAC,SAAS,EAAE,CAAC;AAIlC,MAAM,UAAU,sBAAsB,CAAC,IAAiB;IACpD,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;IACpC,MAAM,GAAG,GAAG,CAAC,GAAG,CAAC,CAAC;IAClB,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,kBAAkB,EAAE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAE/D,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,MAAM,IAAI,GAAa,EAAE,CAAC;IAC1B,MAAM,KAAK,GAAoC,EAAE,CAAC;IAClD,MAAM,OAAO,GAAG,IAAI,GAAG,EAAkB,CAAC;IAE1C,IAAI,QAAQ,GAAG,CAAC,CAAC,CAAK,8BAA8B;IACpD,IAAI,QAAQ,GAAG,CAAC,CAAC,CAAK,8BAA8B;IACpD,IAAI,OAAO,GAAG,CAAC,CAAC,CAAM,6BAA6B;IACnD,IAAI,KAAK,GAAG,CAAC,CAAC,CAAQ,qCAAqC;IAC3D,IAAI,KAAK,GAAG,KAAK,CAAC;IAClB,IAAI,QAAQ,GAAG,CAAC,CAAC;IACjB,IAAI,QAAQ,GAA6B,IAAI,CAAC;IAC9C,IAAI,KAAK,GAA6B,IAAI,CAAC;IAC3C,IAAI,MAAM,GAAiB,IAAI,CAAC;IAEhC,MAAM,MAAM,GAAG,IAAI,SAAS,CAAC;QACzB,kBAAkB,EAAE,IAAI,EAAU,iBAAiB;QACnD,kBAAkB,EAAE,KAAK,EAAS,eAAe;QACjD,SAAS,CAAC,KAAoC,EAAE,IAAI,EAAE,EAAqB;YACvE,IAAI,MAAM,EAAE,CAAC;gBAAC,EAAE,CAAC,MAAM,CAAC,CAAC;gBAAC,OAAO;YAAC,CAAC;YACnC,QAAQ,EAAE,CAAC;YACX,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAClB,QAAQ,EAAE,CAAC;YACX,IAAI,QAAQ,GAAG,GAAG;gBAAE,EAAE,EAAE,CAAC;;gBACpB,QAAQ,GAAG,EAAE,CAAC,CAAW,eAAe;QACjD,CAAC;QACD,KAAK,CAAC,EAAqB;YACvB,IAAI,MAAM,EAAE,CAAC;gBAAC,EAAE,CAAC,MAAM,CAAC,CAAC;gBAAC,OAAO;YAAC,CAAC;YACnC,KAAK,GAAG,IAAI,CAAC;YACb,IAAI,OAAO,KAAK,QAAQ;gBAAE,MAAM,CAAC,EAAE,CAAC,CAAC;;gBAChC,KAAK,GAAG,EAAE,CAAC;QACpB,CAAC;KACJ,CAAC,CAAC;IAEH,SAAS,QAAQ;QACb,OAAO,KAAK,CAAC,MAAM,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YACjC,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,EAAG,CAAC;YACtB,MAAM,GAAG,GAAG,KAAK,CAAC,KAAK,EAAG,CAAC;YAC3B,QAAQ,EAAE,CAAC;YACX,CAAC,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;QACvB,CAAC;IACL,CAAC;IAED,SAAS,QAAQ,CAAC,CAAS,EAAE,GAAQ;QACjC,IAAI,GAAG,CAAC,KAAK,EAAE,CAAC;YAAC,IAAI,CAAC,IAAI,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC;YAAC,OAAO;QAAC,CAAC;QACtD,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,EAAE,GAAG,CAAC,GAAG,CAAC,CAAC;QAC9B,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACb,QAAQ,EAAE,CAAC;QACX,OAAO,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC;YAC3B,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAE,CAAC;YACnC,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;YACzB,QAAQ,EAAE,CAAC;YACX,OAAO,EAAE,CAAC;YACV,IAAI,GAAG,CAAC,MAAM;gBAAE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACrC,CAAC;QACD,KAAK,IAAI,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC;QACpB,QAAQ,EAAE,CAAC;QACX,IAAI,QAAQ,IAAI,QAAQ,GAAG,GAAG,EAAE,CAAC;YAAC,MAAM,CAAC,GAAG,QAAQ,CAAC;YAAC,QAAQ,GAAG,IAAI,CAAC;YAAC,CAAC,EAAE,CAAC;QAAC,CAAC;QAC7E,IAAI,KAAK,IAAI,KAAK,IAAI,OAAO,KAAK,QAAQ,EAAE,CAAC;YAAC,MAAM,CAAC,GAAG,KAAK,CAAC;YAAC,KAAK,GAAG,IAAI,CAAC;YAAC,MAAM,CAAC,CAAC,CAAC,CAAC;QAAC,CAAC;IAC7F,CAAC;IAED,SAAS,IAAI,CAAC,GAAU;QACpB,IAAI,MAAM;YAAE,OAAO;QACnB,MAAM,GAAG,GAAG,CAAC;QACb,KAAK,MAAM,CAAC,IAAI,OAAO;YAAE,CAAC,CAAC,SAAS,EAAE,CAAC;QACvC,IAAI,QAAQ,EAAE,CAAC;YAAC,MAAM,CAAC,GAAG,QAAQ,CAAC;YAAC,QAAQ,GAAG,IAAI,CAAC;YAAC,CAAC,CAAC,GAAG,CAAC,CAAC;QAAC,CAAC;aACzD,IAAI,KAAK,EAAE,CAAC;YAAC,MAAM,CAAC,GAAG,KAAK,CAAC;YAAC,KAAK,GAAG,IAAI,CAAC;YAAC,CAAC,CAAC,GAAG,CAAC,CAAC;QAAC,CAAC;;YACrD,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IAC7B,CAAC;IAED,SAAS,MAAM,CAAC,EAAqB;QACjC,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;IACtF,CAAC;IAED,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QACzB,MAAM,CAAC,GAAG,IAAI,MAAM,CAAC,SAAS,EAAE,EAAE,UAAU,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;QACvG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAChB,CAAC,CAAC,EAAE,CAAC,SAAS,EAAE,CAAC,GAAQ,EAAE,EAAE;YACzB,IAAI,GAAG,IAAI,GAAG,CAAC,KAAK,EAAE,CAAC;gBAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;gBAAC,QAAQ,EAAE,CAAC;YAAC,CAAC;;gBAC9C,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;QAC1B,CAAC,CAAC,CAAC;QACH,CAAC,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;IACtC,CAAC;IAEA,MAAc,CAAC,QAAQ,GAAG,GAAG,EAAE,CAAC,KAAK,CAAC;IACvC,MAAM,CAAC,IAAI,CAAC,mBAAmB,CAAC,yBAAyB,CAAC,CAAC;IAC3D,OAAO,MAAM,CAAC;AAClB,CAAC"}
@@ -0,0 +1,10 @@
1
+ import { Transform } from 'stream';
2
+ export interface WorkerPoolOpts {
3
+ map: string;
4
+ param: any;
5
+ workers: number;
6
+ batchSize?: number;
7
+ timeoutMs?: number;
8
+ }
9
+ export declare function createWorkerPool(opts: WorkerPoolOpts): Transform;
10
+ //# sourceMappingURL=worker-pool.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"worker-pool.d.ts","sourceRoot":"","sources":["../../src/stream/worker-pool.ts"],"names":[],"mappings":"AAgBA,OAAO,EAAE,SAAS,EAA0B,MAAM,QAAQ,CAAC;AAO3D,MAAM,WAAW,cAAc;IAC3B,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,GAAG,CAAC;IACX,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,SAAS,CAAC,EAAE,MAAM,CAAC;CACtB;AAED,wBAAgB,gBAAgB,CAAC,IAAI,EAAE,cAAc,GAAG,SAAS,CA4IhE"}
@@ -0,0 +1,206 @@
1
+ // A Transform that parallelizes a per-record map across worker threads while
2
+ // preserving record order. Batches incoming records, dispatches batches to a
3
+ // pool of workers, re-orders the results by sequence number, and pushes them
4
+ // downstream in order.
5
+ //
6
+ // In (objectMode): record objects
7
+ // Out (objectMode): mapped record objects, in the original order
8
+ // (records the map rejected are dropped)
9
+ //
10
+ // Hardening over the first prototype:
11
+ // - worker count is clamped to the available parallelism;
12
+ // - real downstream backpressure: results are held in an out-queue and only
13
+ // pushed while push() accepts them; dispatch/accept pause when the output
14
+ // is backed up and resume on _read (so a slow sink can't grow memory);
15
+ // - a per-batch timeout fails the pipeline instead of hanging if a worker
16
+ // dies silently.
17
+ import { Transform } from 'stream';
18
+ import { Worker } from 'node:worker_threads';
19
+ import { availableParallelism } from 'node:os';
20
+ import log4js from 'log4js';
21
+ const logger = log4js.getLogger();
22
+ export function createWorkerPool(opts) {
23
+ const maxN = Math.max(1, availableParallelism());
24
+ const N = Math.min(Math.max(1, Math.floor(opts.workers) || 1), maxN);
25
+ if (N < opts.workers)
26
+ logger.warn(`--workers ${opts.workers} clamped to ${N} (available parallelism)`);
27
+ const BATCH = opts.batchSize ?? 64;
28
+ const CAP = N * 2; // max batches in flight
29
+ const OUT_CAP = CAP * BATCH; // max records buffered awaiting downstream
30
+ const TIMEOUT = opts.timeoutMs ?? 120_000;
31
+ const workerUrl = new URL('../worker.js', import.meta.url);
32
+ const workers = [];
33
+ const idle = [];
34
+ const timers = new Map();
35
+ const queue = [];
36
+ const reorder = new Map();
37
+ const outQueue = []; // mapped records ready to push, in order
38
+ let nextDispatch = 0;
39
+ let nextEmit = 0;
40
+ let inflight = 0;
41
+ let curBatch = [];
42
+ let acceptCb = null; // held _transform callback
43
+ let endCb = null;
44
+ let failed = null;
45
+ let closing = false; // we are terminating workers on purpose
46
+ const stream = new Transform({
47
+ objectMode: true,
48
+ transform(rec, _enc, cb) {
49
+ if (failed) {
50
+ cb(failed);
51
+ return;
52
+ }
53
+ curBatch.push(rec);
54
+ if (curBatch.length >= BATCH) {
55
+ enqueue(curBatch);
56
+ curBatch = [];
57
+ }
58
+ if (canAccept())
59
+ cb();
60
+ else
61
+ acceptCb = cb; // pause upstream until there's room
62
+ },
63
+ flush(cb) {
64
+ if (failed) {
65
+ cb(failed);
66
+ return;
67
+ }
68
+ if (curBatch.length) {
69
+ enqueue(curBatch);
70
+ curBatch = [];
71
+ }
72
+ if (done())
73
+ finish(cb);
74
+ else
75
+ endCb = cb;
76
+ }
77
+ });
78
+ // Resume producing when the downstream consumer asks for more.
79
+ const origRead = stream._read.bind(stream);
80
+ stream._read = (size) => { pump(); origRead(size); };
81
+ function canAccept() {
82
+ return inflight < CAP && outQueue.length < OUT_CAP;
83
+ }
84
+ function done() {
85
+ return nextEmit === nextDispatch && queue.length === 0 && outQueue.length === 0;
86
+ }
87
+ function enqueue(batch) {
88
+ queue.push({ seq: nextDispatch++, batch });
89
+ dispatch();
90
+ }
91
+ const dbg = () => logger.isDebugEnabled();
92
+ function dispatch() {
93
+ // don't start new work while the output is backed up
94
+ if (dbg() && queue.length && (!idle.length || outQueue.length >= OUT_CAP)) {
95
+ logger.debug(`backpressure: ${queue.length} batches queued, idle=${idle.length}, outQueue=${outQueue.length}/${OUT_CAP}`);
96
+ }
97
+ while (queue.length && idle.length && outQueue.length < OUT_CAP) {
98
+ const w = idle.pop();
99
+ const job = queue.shift();
100
+ inflight++;
101
+ timers.set(w, setTimeout(() => fail(new Error(`worker timed out after ${TIMEOUT}ms (batch ${job.seq})`)), TIMEOUT));
102
+ w.postMessage(job);
103
+ if (dbg())
104
+ logger.debug(`dispatch batch ${job.seq} (${job.batch.length} recs) -> worker; inflight=${inflight}`);
105
+ }
106
+ }
107
+ // push as many ready records as the downstream will take
108
+ function pump() {
109
+ const before = outQueue.length;
110
+ while (outQueue.length && stream.push(outQueue[0]))
111
+ outQueue.shift();
112
+ if (dbg() && outQueue.length > 0 && outQueue.length === before) {
113
+ logger.debug(`downstream full, holding ${outQueue.length} records`);
114
+ }
115
+ dispatch();
116
+ if (acceptCb && canAccept()) {
117
+ const c = acceptCb;
118
+ acceptCb = null;
119
+ c();
120
+ }
121
+ if (endCb && done()) {
122
+ const c = endCb;
123
+ endCb = null;
124
+ finish(c);
125
+ }
126
+ }
127
+ function onResult(w, msg) {
128
+ const t = timers.get(w);
129
+ if (t) {
130
+ clearTimeout(t);
131
+ timers.delete(w);
132
+ }
133
+ if (msg.error) {
134
+ logger.error(`worker error on batch ${msg.seq}: ${msg.error}`);
135
+ fail(new Error(msg.error));
136
+ return;
137
+ }
138
+ reorder.set(msg.seq, msg.mapped);
139
+ idle.push(w);
140
+ inflight--;
141
+ if (dbg())
142
+ logger.debug(`result batch ${msg.seq} (${msg.mapped.length} mapped); inflight=${inflight}, reorderBuf=${reorder.size}, wantNext=${nextEmit}`);
143
+ // move now-contiguous results into the out-queue (don't push yet)
144
+ while (reorder.has(nextEmit)) {
145
+ const mapped = reorder.get(nextEmit);
146
+ reorder.delete(nextEmit);
147
+ nextEmit++;
148
+ for (const m of mapped)
149
+ if (m !== null)
150
+ outQueue.push(m);
151
+ }
152
+ pump();
153
+ }
154
+ function fail(err) {
155
+ if (failed)
156
+ return;
157
+ failed = err;
158
+ closing = true;
159
+ logger.error(`worker pool failing: ${err.message} (terminating ${workers.length} workers)`);
160
+ for (const t of timers.values())
161
+ clearTimeout(t);
162
+ timers.clear();
163
+ for (const w of workers)
164
+ w.terminate();
165
+ if (acceptCb) {
166
+ const c = acceptCb;
167
+ acceptCb = null;
168
+ c(err);
169
+ }
170
+ else if (endCb) {
171
+ const c = endCb;
172
+ endCb = null;
173
+ c(err);
174
+ }
175
+ else
176
+ stream.destroy(err);
177
+ }
178
+ function finish(cb) {
179
+ closing = true;
180
+ for (const t of timers.values())
181
+ clearTimeout(t);
182
+ timers.clear();
183
+ logger.debug(`map complete: ${nextEmit} batches; terminating ${workers.length} workers`);
184
+ Promise.all(workers.map((w) => w.terminate())).then(() => cb()).catch(() => cb());
185
+ }
186
+ for (let i = 0; i < N; i++) {
187
+ const w = new Worker(workerUrl, { workerData: { map: opts.map, param: opts.param } });
188
+ workers.push(w);
189
+ w.on('message', (msg) => {
190
+ if (msg && msg.ready) {
191
+ idle.push(w);
192
+ dispatch();
193
+ if (dbg())
194
+ logger.debug(`worker ${idle.length}/${N} ready`);
195
+ }
196
+ else
197
+ onResult(w, msg);
198
+ });
199
+ w.on('error', (err) => { logger.error(`worker thread crashed: ${err.message}`); fail(err); });
200
+ w.on('exit', (code) => { if (code !== 0 && !failed && !closing)
201
+ fail(new Error(`worker exited unexpectedly (code ${code})`)); });
202
+ }
203
+ logger.info(`map running on ${N} worker threads (batch ${BATCH}, in-flight cap ${CAP}, timeout ${TIMEOUT}ms)`);
204
+ return stream;
205
+ }
206
+ //# sourceMappingURL=worker-pool.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"worker-pool.js","sourceRoot":"","sources":["../../src/stream/worker-pool.ts"],"names":[],"mappings":"AAAA,6EAA6E;AAC7E,6EAA6E;AAC7E,6EAA6E;AAC7E,uBAAuB;AACvB,EAAE;AACF,mCAAmC;AACnC,iEAAiE;AACjE,2DAA2D;AAC3D,EAAE;AACF,sCAAsC;AACtC,2DAA2D;AAC3D,6EAA6E;AAC7E,6EAA6E;AAC7E,0EAA0E;AAC1E,2EAA2E;AAC3E,oBAAoB;AACpB,OAAO,EAAE,SAAS,EAA0B,MAAM,QAAQ,CAAC;AAC3D,OAAO,EAAE,MAAM,EAAE,MAAM,qBAAqB,CAAC;AAC7C,OAAO,EAAE,oBAAoB,EAAE,MAAM,SAAS,CAAC;AAC/C,OAAO,MAAM,MAAM,QAAQ,CAAC;AAE5B,MAAM,MAAM,GAAG,MAAM,CAAC,SAAS,EAAE,CAAC;AAUlC,MAAM,UAAU,gBAAgB,CAAC,IAAoB;IACjD,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,oBAAoB,EAAE,CAAC,CAAC;IACjD,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;IACrE,IAAI,CAAC,GAAG,IAAI,CAAC,OAAO;QAAE,MAAM,CAAC,IAAI,CAAC,aAAa,IAAI,CAAC,OAAO,eAAe,CAAC,0BAA0B,CAAC,CAAC;IACvG,MAAM,KAAK,GAAG,IAAI,CAAC,SAAS,IAAI,EAAE,CAAC;IACnC,MAAM,GAAG,GAAG,CAAC,GAAG,CAAC,CAAC,CAAiB,wBAAwB;IAC3D,MAAM,OAAO,GAAG,GAAG,GAAG,KAAK,CAAC,CAAO,2CAA2C;IAC9E,MAAM,OAAO,GAAG,IAAI,CAAC,SAAS,IAAI,OAAO,CAAC;IAC1C,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,cAAc,EAAE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAE3D,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,MAAM,IAAI,GAAa,EAAE,CAAC;IAC1B,MAAM,MAAM,GAAG,IAAI,GAAG,EAA0B,CAAC;IACjD,MAAM,KAAK,GAAoC,EAAE,CAAC;IAClD,MAAM,OAAO,GAAG,IAAI,GAAG,EAAiB,CAAC;IACzC,MAAM,QAAQ,GAAU,EAAE,CAAC,CAAQ,yCAAyC;IAE5E,IAAI,YAAY,GAAG,CAAC,CAAC;IACrB,IAAI,QAAQ,GAAG,CAAC,CAAC;IACjB,IAAI,QAAQ,GAAG,CAAC,CAAC;IACjB,IAAI,QAAQ,GAAU,EAAE,CAAC;IACzB,IAAI,QAAQ,GAA6B,IAAI,CAAC,CAAG,2BAA2B;IAC5E,IAAI,KAAK,GAA6B,IAAI,CAAC;IAC3C,IAAI,MAAM,GAAiB,IAAI,CAAC;IAChC,IAAI,OAAO,GAAG,KAAK,CAAC,CAA8B,wCAAwC;IAE1F,MAAM,MAAM,GAAG,IAAI,SAAS,CAAC;QACzB,UAAU,EAAE,IAAI;QAChB,SAAS,CAAC,GAAQ,EAAE,IAAI,EAAE,EAAqB;YAC3C,IAAI,MAAM,EAAE,CAAC;gBAAC,EAAE,CAAC,MAAM,CAAC,CAAC;gBAAC,OAAO;YAAC,CAAC;YACnC,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACnB,IAAI,QAAQ,CAAC,MAAM,IAAI,KAAK,EAAE,CAAC;gBAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;gBAAC,QAAQ,GAAG,EAAE,CAAC;YAAC,CAAC;YACnE,IAAI,SAAS,EAAE;gBAAE,EAAE,EAAE,CAAC;;gBACjB,QAAQ,GAAG,EAAE,CAAC,CAAU,oCAAoC;QACrE,CAAC;QACD,KAAK,CAAC,EAAqB;YACvB,IAAI,MAAM,EAAE,CAAC;gBAAC,EAAE,CAAC,MAAM,CAAC,CAAC;gBAAC,OAAO;YAAC,CAAC;YACnC,IAAI,QAAQ,CAAC,MAAM,EAAE,CAAC;gBAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;gBAAC,QAAQ,GAAG,EAAE,CAAC;YAAC,CAAC;YAC1D,IAAI,IAAI,EAAE;gBAAE,MAAM,CAAC,EAAE,CAAC,CAAC;;gBAClB,KAAK,GAAG,EAAE,CAAC;QACpB,CAAC;KACJ,CAAC,CAAC;IAEH,+DAA+D;IAC/D,MAAM,QAAQ,GAAI,MAAc,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IACnD,MAAc,CAAC,KAAK,GAAG,CAAC,IAAY,EAAE,EAAE,GAAG,IAAI,EAAE,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;IAEtE,SAAS,SAAS;QACd,OAAO,QAAQ,GAAG,GAAG,IAAI,QAAQ,CAAC,MAAM,GAAG,OAAO,CAAC;IACvD,CAAC;IACD,SAAS,IAAI;QACT,OAAO,QAAQ,KAAK,YAAY,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,CAAC;IACpF,CAAC;IAED,SAAS,OAAO,CAAC,KAAY;QACzB,KAAK,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,YAAY,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC;QAC3C,QAAQ,EAAE,CAAC;IACf,CAAC;IAED,MAAM,GAAG,GAAG,GAAG,EAAE,CAAC,MAAM,CAAC,cAAc,EAAE,CAAC;IAE1C,SAAS,QAAQ;QACb,qDAAqD;QACrD,IAAI,GAAG,EAAE,IAAI,KAAK,CAAC,MAAM,IAAI,CAAC,CAAC,IAAI,CAAC,MAAM,IAAI,QAAQ,CAAC,MAAM,IAAI,OAAO,CAAC,EAAE,CAAC;YACxE,MAAM,CAAC,KAAK,CAAC,iBAAiB,KAAK,CAAC,MAAM,yBAAyB,IAAI,CAAC,MAAM,cAAc,QAAQ,CAAC,MAAM,IAAI,OAAO,EAAE,CAAC,CAAC;QAC9H,CAAC;QACD,OAAO,KAAK,CAAC,MAAM,IAAI,IAAI,CAAC,MAAM,IAAI,QAAQ,CAAC,MAAM,GAAG,OAAO,EAAE,CAAC;YAC9D,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,EAAG,CAAC;YACtB,MAAM,GAAG,GAAG,KAAK,CAAC,KAAK,EAAG,CAAC;YAC3B,QAAQ,EAAE,CAAC;YACX,MAAM,CAAC,GAAG,CAAC,CAAC,EAAE,UAAU,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,IAAI,KAAK,CAAC,0BAA0B,OAAO,aAAa,GAAG,CAAC,GAAG,GAAG,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC,CAAC;YACpH,CAAC,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;YACnB,IAAI,GAAG,EAAE;gBAAE,MAAM,CAAC,KAAK,CAAC,kBAAkB,GAAG,CAAC,GAAG,KAAK,GAAG,CAAC,KAAK,CAAC,MAAM,8BAA8B,QAAQ,EAAE,CAAC,CAAC;QACpH,CAAC;IACL,CAAC;IAED,yDAAyD;IACzD,SAAS,IAAI;QACT,MAAM,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAC;QAC/B,OAAO,QAAQ,CAAC,MAAM,IAAI,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;YAAE,QAAQ,CAAC,KAAK,EAAE,CAAC;QACrE,IAAI,GAAG,EAAE,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,IAAI,QAAQ,CAAC,MAAM,KAAK,MAAM,EAAE,CAAC;YAC7D,MAAM,CAAC,KAAK,CAAC,4BAA4B,QAAQ,CAAC,MAAM,UAAU,CAAC,CAAC;QACxE,CAAC;QACD,QAAQ,EAAE,CAAC;QACX,IAAI,QAAQ,IAAI,SAAS,EAAE,EAAE,CAAC;YAAC,MAAM,CAAC,GAAG,QAAQ,CAAC;YAAC,QAAQ,GAAG,IAAI,CAAC;YAAC,CAAC,EAAE,CAAC;QAAC,CAAC;QAC1E,IAAI,KAAK,IAAI,IAAI,EAAE,EAAE,CAAC;YAAC,MAAM,CAAC,GAAG,KAAK,CAAC;YAAC,KAAK,GAAG,IAAI,CAAC;YAAC,MAAM,CAAC,CAAC,CAAC,CAAC;QAAC,CAAC;IACtE,CAAC;IAED,SAAS,QAAQ,CAAC,CAAS,EAAE,GAAQ;QACjC,MAAM,CAAC,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;QACxB,IAAI,CAAC,EAAE,CAAC;YAAC,YAAY,CAAC,CAAC,CAAC,CAAC;YAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;QAAC,CAAC;QAC7C,IAAI,GAAG,CAAC,KAAK,EAAE,CAAC;YAAC,MAAM,CAAC,KAAK,CAAC,yBAAyB,GAAG,CAAC,GAAG,KAAK,GAAG,CAAC,KAAK,EAAE,CAAC,CAAC;YAAC,IAAI,CAAC,IAAI,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC;YAAC,OAAO;QAAC,CAAC;QACtH,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;QACjC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACb,QAAQ,EAAE,CAAC;QACX,IAAI,GAAG,EAAE;YAAE,MAAM,CAAC,KAAK,CAAC,gBAAgB,GAAG,CAAC,GAAG,KAAK,GAAG,CAAC,MAAM,CAAC,MAAM,sBAAsB,QAAQ,gBAAgB,OAAO,CAAC,IAAI,cAAc,QAAQ,EAAE,CAAC,CAAC;QACzJ,kEAAkE;QAClE,OAAO,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC;YAC3B,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAE,CAAC;YACtC,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;YACzB,QAAQ,EAAE,CAAC;YACX,KAAK,MAAM,CAAC,IAAI,MAAM;gBAAE,IAAI,CAAC,KAAK,IAAI;oBAAE,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAC7D,CAAC;QACD,IAAI,EAAE,CAAC;IACX,CAAC;IAED,SAAS,IAAI,CAAC,GAAU;QACpB,IAAI,MAAM;YAAE,OAAO;QACnB,MAAM,GAAG,GAAG,CAAC;QACb,OAAO,GAAG,IAAI,CAAC;QACf,MAAM,CAAC,KAAK,CAAC,wBAAwB,GAAG,CAAC,OAAO,iBAAiB,OAAO,CAAC,MAAM,WAAW,CAAC,CAAC;QAC5F,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,MAAM,EAAE;YAAE,YAAY,CAAC,CAAC,CAAC,CAAC;QACjD,MAAM,CAAC,KAAK,EAAE,CAAC;QACf,KAAK,MAAM,CAAC,IAAI,OAAO;YAAE,CAAC,CAAC,SAAS,EAAE,CAAC;QACvC,IAAI,QAAQ,EAAE,CAAC;YAAC,MAAM,CAAC,GAAG,QAAQ,CAAC;YAAC,QAAQ,GAAG,IAAI,CAAC;YAAC,CAAC,CAAC,GAAG,CAAC,CAAC;QAAC,CAAC;aACzD,IAAI,KAAK,EAAE,CAAC;YAAC,MAAM,CAAC,GAAG,KAAK,CAAC;YAAC,KAAK,GAAG,IAAI,CAAC;YAAC,CAAC,CAAC,GAAG,CAAC,CAAC;QAAC,CAAC;;YACrD,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IAC7B,CAAC;IAED,SAAS,MAAM,CAAC,EAAqB;QACjC,OAAO,GAAG,IAAI,CAAC;QACf,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,MAAM,EAAE;YAAE,YAAY,CAAC,CAAC,CAAC,CAAC;QACjD,MAAM,CAAC,KAAK,EAAE,CAAC;QACf,MAAM,CAAC,KAAK,CAAC,iBAAiB,QAAQ,yBAAyB,OAAO,CAAC,MAAM,UAAU,CAAC,CAAC;QACzF,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;IACtF,CAAC;IAED,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QACzB,MAAM,CAAC,GAAG,IAAI,MAAM,CAAC,SAAS,EAAE,EAAE,UAAU,EAAE,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;QACtF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAChB,CAAC,CAAC,EAAE,CAAC,SAAS,EAAE,CAAC,GAAQ,EAAE,EAAE;YACzB,IAAI,GAAG,IAAI,GAAG,CAAC,KAAK,EAAE,CAAC;gBAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;gBAAC,QAAQ,EAAE,CAAC;gBAAC,IAAI,GAAG,EAAE;oBAAE,MAAM,CAAC,KAAK,CAAC,UAAU,IAAI,CAAC,MAAM,IAAI,CAAC,QAAQ,CAAC,CAAC;YAAC,CAAC;;gBAC3G,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;QAC1B,CAAC,CAAC,CAAC;QACH,CAAC,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE,GAAG,MAAM,CAAC,KAAK,CAAC,0BAA0B,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAC9F,CAAC,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,EAAE,GAAG,IAAI,IAAI,KAAK,CAAC,IAAI,CAAC,MAAM,IAAI,CAAC,OAAO;YAAE,IAAI,CAAC,IAAI,KAAK,CAAC,oCAAoC,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACrI,CAAC;IAED,MAAM,CAAC,IAAI,CAAC,kBAAkB,CAAC,0BAA0B,KAAK,mBAAmB,GAAG,aAAa,OAAO,KAAK,CAAC,CAAC;IAC/G,OAAO,MAAM,CAAC;AAClB,CAAC"}
@@ -0,0 +1,16 @@
1
+ import { Transform } from 'stream';
2
+ /**
3
+ * Apply a Catmandu Fix script to each record.
4
+ *
5
+ * marcattacks --map fix --param fix=./my.fix ...
6
+ *
7
+ * `fix` may be a path to a Fix file or an inline Fix script. The script is
8
+ * parsed and compiled once; the compiled chain runs per record.
9
+ */
10
+ export declare function createMapper(opts: {
11
+ fix?: string;
12
+ }): Promise<(data: any) => any>;
13
+ export declare function transform(opts: {
14
+ fix?: string;
15
+ }): Promise<Transform>;
16
+ //# sourceMappingURL=fix.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"fix.d.ts","sourceRoot":"","sources":["../../src/transform/fix.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,QAAQ,CAAC;AAOnC;;;;;;;GAOG;AAGH,wBAAsB,YAAY,CAAC,IAAI,EAAE;IAAE,GAAG,CAAC,EAAE,MAAM,CAAA;CAAE,GAAG,OAAO,CAAC,CAAC,IAAI,EAAE,GAAG,KAAK,GAAG,CAAC,CAMtF;AAED,wBAAsB,SAAS,CAAC,IAAI,EAAE;IAAE,GAAG,CAAC,EAAE,MAAM,CAAA;CAAE,GAAG,OAAO,CAAC,SAAS,CAAC,CAoB1E"}
@@ -0,0 +1,46 @@
1
+ import { Transform } from 'stream';
2
+ import fs from 'fs';
3
+ import log4js from 'log4js';
4
+ import { compileFix, REJECT } from '../fix/index.js';
5
+ const logger = log4js.getLogger();
6
+ /**
7
+ * Apply a Catmandu Fix script to each record.
8
+ *
9
+ * marcattacks --map fix --param fix=./my.fix ...
10
+ *
11
+ * `fix` may be a path to a Fix file or an inline Fix script. The script is
12
+ * parsed and compiled once; the compiled chain runs per record.
13
+ */
14
+ // Pure record -> record mapper (returns REJECT for dropped records). Shared by
15
+ // transform() and the worker pool. Compiled once.
16
+ export async function createMapper(opts) {
17
+ let src = opts.fix ?? '';
18
+ if (src && fs.existsSync(src)) {
19
+ src = fs.readFileSync(src, { encoding: 'utf-8' });
20
+ }
21
+ return compileFix(src);
22
+ }
23
+ export async function transform(opts) {
24
+ let src = opts.fix ?? '';
25
+ if (src && fs.existsSync(src)) {
26
+ src = fs.readFileSync(src, { encoding: 'utf-8' });
27
+ }
28
+ const fix = compileFix(src);
29
+ return new Transform({
30
+ objectMode: true,
31
+ transform(data, _encoding, callback) {
32
+ try {
33
+ const out = fix(data);
34
+ if (out === REJECT)
35
+ callback(); // rejected record -> drop it
36
+ else
37
+ callback(null, out);
38
+ }
39
+ catch (err) {
40
+ logger.error('fix error', err.message);
41
+ callback(err);
42
+ }
43
+ }
44
+ });
45
+ }
46
+ //# sourceMappingURL=fix.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"fix.js","sourceRoot":"","sources":["../../src/transform/fix.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,QAAQ,CAAC;AACnC,OAAO,EAAE,MAAM,IAAI,CAAC;AACpB,OAAO,MAAM,MAAM,QAAQ,CAAC;AAC5B,OAAO,EAAE,UAAU,EAAE,MAAM,EAAE,MAAM,iBAAiB,CAAC;AAErD,MAAM,MAAM,GAAG,MAAM,CAAC,SAAS,EAAE,CAAC;AAElC;;;;;;;GAOG;AACH,+EAA+E;AAC/E,kDAAkD;AAClD,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,IAAsB;IACrD,IAAI,GAAG,GAAG,IAAI,CAAC,GAAG,IAAI,EAAE,CAAC;IACzB,IAAI,GAAG,IAAI,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;QAC5B,GAAG,GAAG,EAAE,CAAC,YAAY,CAAC,GAAG,EAAE,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC,CAAC;IACtD,CAAC;IACD,OAAO,UAAU,CAAC,GAAG,CAAC,CAAC;AAC3B,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,IAAsB;IAClD,IAAI,GAAG,GAAG,IAAI,CAAC,GAAG,IAAI,EAAE,CAAC;IACzB,IAAI,GAAG,IAAI,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;QAC5B,GAAG,GAAG,EAAE,CAAC,YAAY,CAAC,GAAG,EAAE,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC,CAAC;IACtD,CAAC;IACD,MAAM,GAAG,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC;IAE5B,OAAO,IAAI,SAAS,CAAC;QACjB,UAAU,EAAE,IAAI;QAChB,SAAS,CAAC,IAAS,EAAE,SAAS,EAAE,QAAQ;YACpC,IAAI,CAAC;gBACD,MAAM,GAAG,GAAG,GAAG,CAAC,IAAI,CAAC,CAAC;gBACtB,IAAI,GAAG,KAAK,MAAM;oBAAE,QAAQ,EAAE,CAAC,CAAG,6BAA6B;;oBAC1D,QAAQ,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;YAC7B,CAAC;YAAC,OAAO,GAAQ,EAAE,CAAC;gBAChB,MAAM,CAAC,KAAK,CAAC,WAAW,EAAE,GAAG,CAAC,OAAO,CAAC,CAAC;gBACvC,QAAQ,CAAC,GAAG,CAAC,CAAC;YAClB,CAAC;QACL,CAAC;KACJ,CAAC,CAAC;AACP,CAAC"}
@@ -1,4 +1,8 @@
1
1
  import { Transform } from "stream";
2
+ export declare function createMapper(opts: {
3
+ fix: string;
4
+ lookup: string;
5
+ }): Promise<(data: any) => any>;
2
6
  export declare function transform(opts: {
3
7
  fix: string;
4
8
  lookup: string;
@@ -1 +1 @@
1
- {"version":3,"file":"jsonata.d.ts","sourceRoot":"","sources":["../../src/transform/jsonata.ts"],"names":[],"mappings":"AAAA,OAAe,EAAE,SAAS,EAAE,MAAM,QAAQ,CAAC;AAU3C,wBAAsB,SAAS,CAAC,IAAI,EAAE;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,GAAI,OAAO,CAAC,SAAS,CAAC,CAgE1F"}
1
+ {"version":3,"file":"jsonata.d.ts","sourceRoot":"","sources":["../../src/transform/jsonata.ts"],"names":[],"mappings":"AAAA,OAAe,EAAE,SAAS,EAAE,MAAM,QAAQ,CAAC;AAa3C,wBAAsB,YAAY,CAAC,IAAI,EAAE;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,GAAI,OAAO,CAAC,CAAC,IAAI,EAAE,GAAG,KAAK,GAAG,CAAC,CA0CtG;AAED,wBAAsB,SAAS,CAAC,IAAI,EAAE;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,GAAI,OAAO,CAAC,SAAS,CAAC,CAe1F"}
@@ -6,58 +6,54 @@ import { parseStream } from '../util/tsv_parse.js';
6
6
  import { v4 as uuidv4 } from 'uuid';
7
7
  import log4js from 'log4js';
8
8
  const logger = log4js.getLogger();
9
- export async function transform(opts) {
10
- let query;
9
+ // Build a pure record -> record(Promise) mapper. Shared by the in-process
10
+ // transform() and by the worker pool (so the heavy evaluate() can run on
11
+ // worker threads). Expression + helper functions are compiled once.
12
+ export async function createMapper(opts) {
11
13
  let lookup = {};
12
14
  if (opts.lookup) {
13
15
  lookup = await loadLookup(opts.lookup);
14
16
  }
15
- logger.info(lookup);
17
+ // Resolve the query once, up front, instead of on every record.
18
+ let query;
19
+ if (opts.fix) {
20
+ if (fs.existsSync(opts.fix)) {
21
+ query = fs.readFileSync(opts.fix, { encoding: 'utf-8' });
22
+ }
23
+ else {
24
+ throw Error(`no such file ${opts.fix}`);
25
+ }
26
+ }
27
+ else {
28
+ query = '$';
29
+ }
30
+ logger.debug(query);
31
+ // The identity expression is a pure pass-through: skip jsonata entirely.
32
+ if (query.trim() === '$') {
33
+ return (data) => data;
34
+ }
35
+ // Compile the expression and register helper functions ONCE. The helpers
36
+ // read the record currently being processed via `current`; each evaluate
37
+ // is awaited before the next, so this is safe.
38
+ let current;
39
+ const expression = jsonata(query);
40
+ expression.registerFunction('marcmap', (code) => marcmap(current['record'], code, {}));
41
+ expression.registerFunction('marctag', (row) => marctag(row));
42
+ expression.registerFunction('marcind', (row) => marcind(row));
43
+ expression.registerFunction('marcsubfields', (row, regex) => marcsubfields(row, new RegExp(regex)));
44
+ expression.registerFunction('marcrecord', () => current['record']);
45
+ expression.registerFunction('asmarc', (data) => ({ "record": data }));
46
+ expression.registerFunction('genid', () => genid());
47
+ expression.registerFunction('lookup', (key) => lookup[key]);
48
+ return async (data) => { current = data; return expression.evaluate(data); };
49
+ }
50
+ export async function transform(opts) {
51
+ const mapper = await createMapper(opts);
16
52
  return new Transform({
17
53
  objectMode: true,
18
54
  async transform(data, _encoding, callback) {
19
55
  try {
20
- if (!query) {
21
- if (opts.fix) {
22
- if (fs.existsSync(opts.fix)) {
23
- query = fs.readFileSync(opts.fix, { encoding: 'utf-8' });
24
- }
25
- else {
26
- throw Error(`no such file ${opts.fix}`);
27
- }
28
- }
29
- else {
30
- query = '$';
31
- }
32
- logger.debug(query);
33
- }
34
- const expression = jsonata(query);
35
- expression.registerFunction('marcmap', (code) => {
36
- return marcmap(data['record'], code, {});
37
- });
38
- expression.registerFunction('marctag', (row) => {
39
- return marctag(row);
40
- });
41
- expression.registerFunction('marcind', (row) => {
42
- return marcind(row);
43
- });
44
- expression.registerFunction('marcsubfields', (row, regex) => {
45
- return marcsubfields(row, new RegExp(regex));
46
- });
47
- expression.registerFunction('marcrecord', () => {
48
- return data['record'];
49
- });
50
- expression.registerFunction('asmarc', (data) => {
51
- return { "record": data };
52
- });
53
- expression.registerFunction('genid', () => {
54
- return genid();
55
- });
56
- expression.registerFunction('lookup', (key) => {
57
- return lookup[key];
58
- });
59
- data = await expression.evaluate(data);
60
- callback(null, data);
56
+ callback(null, await mapper(data));
61
57
  }
62
58
  catch (err) {
63
59
  logger.info(err);
@@ -1 +1 @@
1
- {"version":3,"file":"jsonata.js","sourceRoot":"","sources":["../../src/transform/jsonata.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,EAAE,EAAE,SAAS,EAAE,MAAM,QAAQ,CAAC;AAC3C,OAAO,OAAO,MAAM,SAAS,CAAC;AAC9B,OAAO,EAAE,MAAM,IAAI,CAAC;AACpB,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AACzE,OAAO,EAAE,WAAW,EAAE,MAAM,sBAAsB,CAAC;AACnD,OAAO,EAAE,EAAE,IAAI,MAAM,EAAE,MAAM,MAAM,CAAC;AACpC,OAAO,MAAM,MAAM,QAAQ,CAAC;AAE5B,MAAM,MAAM,GAAG,MAAM,CAAC,SAAS,EAAE,CAAC;AAElC,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,IAAqC;IACjE,IAAI,KAAc,CAAC;IAEnB,IAAI,MAAM,GAA2B,EAAE,CAAC;IAExC,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;QACd,MAAM,GAAG,MAAM,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAC3C,CAAC;IAED,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAEpB,OAAO,IAAI,SAAS,CAAC;QACjB,UAAU,EAAE,IAAI;QAChB,KAAK,CAAC,SAAS,CAAC,IAAS,EAAE,SAAyB,EAAE,QAAkC;YACpF,IAAI,CAAC;gBACD,IAAI,CAAC,KAAK,EAAE,CAAC;oBACT,IAAI,IAAI,CAAC,GAAG,EAAE,CAAC;wBACX,IAAI,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;4BAC1B,KAAK,GAAG,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,GAAG,EAAC,EAAE,QAAQ,EAAE,OAAO,EAAC,CAAC,CAAC;wBAC3D,CAAC;6BACI,CAAC;4BACF,MAAM,KAAK,CAAC,gBAAgB,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;wBAC5C,CAAC;oBACL,CAAC;yBACI,CAAC;wBACF,KAAK,GAAG,GAAG,CAAC;oBAChB,CAAC;oBAED,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;gBACxB,CAAC;gBACD,MAAM,UAAU,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC;gBAClC,UAAU,CAAC,gBAAgB,CAAC,SAAS,EAAE,CAAC,IAAY,EAAE,EAAE;oBACpD,OAAO,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAC,IAAI,EAAC,EAAE,CAAC,CAAC;gBAC3C,CAAC,CAAC,CAAC;gBACH,UAAU,CAAC,gBAAgB,CAAC,SAAS,EAAE,CAAC,GAAa,EAAE,EAAE;oBACrD,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC;gBACxB,CAAC,CAAC,CAAC;gBACH,UAAU,CAAC,gBAAgB,CAAC,SAAS,EAAE,CAAC,GAAa,EAAE,EAAE;oBACrD,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC;gBACxB,CAAC,CAAC,CAAC;gBACH,UAAU,CAAC,gBAAgB,CAAC,eAAe,EAAE,CAAC,GAAa,EAAC,KAAa,EAAE,EAAE;oBACzE,OAAO,aAAa,CAAC,GAAG,EAAE,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;gBACjD,CAAC,CAAC,CAAC;gBACH,UAAU,CAAC,gBAAgB,CAAC,YAAY,EAAE,GAAG,EAAE;oBAC3C,OAAO,IAAI,CAAC,QAAQ,CAAC,CAAC;gBAC1B,CAAC,CAAC,CAAC;gBACH,UAAU,CAAC,gBAAgB,CAAC,QAAQ,EAAE,CAAC,IAAgB,EAAE,EAAE;oBACvD,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAC,CAAC;gBAC7B,CAAC,CAAC,CAAC;gBACH,UAAU,CAAC,gBAAgB,CAAC,OAAO,EAAE,GAAG,EAAE;oBACtC,OAAO,KAAK,EAAE,CAAC;gBACnB,CAAC,CAAC,CAAC;gBACH,UAAU,CAAC,gBAAgB,CAAC,QAAQ,EAAE,CAAC,GAAG,EAAE,EAAE;oBAC1C,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC;gBACvB,CAAC,CAAC,CAAC;gBACH,IAAI,GAAG,MAAM,UAAU,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;gBACvC,QAAQ,CAAC,IAAI,EAAC,IAAI,CAAC,CAAC;YACxB,CAAC;YACD,OAAO,GAAQ,EAAE,CAAC;gBACd,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBACjB,QAAQ,CAAC,GAAY,CAAC,CAAC;YAC3B,CAAC;QACL,CAAC;KACJ,CAAC,CAAC;AACP,CAAC;AAED,SAAS,KAAK;IACV,OAAO,SAAS,MAAM,EAAE,EAAE,CAAC;AAC/B,CAAC;AAED,KAAK,UAAU,UAAU,CAAC,IAAY;IAClC,IAAI,MAAM,GAA4B,EAAE,CAAC;IAEzC,MAAM,OAAO,GAAG,MAAM,WAAW,CAAC,EAAE,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC,CAAC;IAE7D,KAAK,MAAM,GAAG,IAAI,OAAO,EAAE,CAAC;QACxB,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;QACrC,IAAI,IAAI,IAAI,IAAI,CAAC,MAAM,IAAI,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC;YACjD,MAAM,CAAC,GAAG,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;YACvB,MAAM,CAAC,GAAG,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;YACvB,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;gBACT,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;YAClB,CAAC;QACL,CAAC;IACL,CAAC;IAED,OAAO,MAAM,CAAC;AAClB,CAAC"}
1
+ {"version":3,"file":"jsonata.js","sourceRoot":"","sources":["../../src/transform/jsonata.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,EAAE,EAAE,SAAS,EAAE,MAAM,QAAQ,CAAC;AAC3C,OAAO,OAAO,MAAM,SAAS,CAAC;AAC9B,OAAO,EAAE,MAAM,IAAI,CAAC;AACpB,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AACzE,OAAO,EAAE,WAAW,EAAE,MAAM,sBAAsB,CAAC;AACnD,OAAO,EAAE,EAAE,IAAI,MAAM,EAAE,MAAM,MAAM,CAAC;AACpC,OAAO,MAAM,MAAM,QAAQ,CAAC;AAE5B,MAAM,MAAM,GAAG,MAAM,CAAC,SAAS,EAAE,CAAC;AAElC,0EAA0E;AAC1E,yEAAyE;AACzE,oEAAoE;AACpE,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,IAAqC;IACpE,IAAI,MAAM,GAA2B,EAAE,CAAC;IAExC,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;QACd,MAAM,GAAG,MAAM,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAC3C,CAAC;IAED,gEAAgE;IAChE,IAAI,KAAc,CAAC;IACnB,IAAI,IAAI,CAAC,GAAG,EAAE,CAAC;QACX,IAAI,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;YAC1B,KAAK,GAAG,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,GAAG,EAAC,EAAE,QAAQ,EAAE,OAAO,EAAC,CAAC,CAAC;QAC3D,CAAC;aACI,CAAC;YACF,MAAM,KAAK,CAAC,gBAAgB,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;QAC5C,CAAC;IACL,CAAC;SACI,CAAC;QACF,KAAK,GAAG,GAAG,CAAC;IAChB,CAAC;IACD,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IAEpB,yEAAyE;IACzE,IAAI,KAAK,CAAC,IAAI,EAAE,KAAK,GAAG,EAAE,CAAC;QACvB,OAAO,CAAC,IAAS,EAAE,EAAE,CAAC,IAAI,CAAC;IAC/B,CAAC;IAED,yEAAyE;IACzE,yEAAyE;IACzE,+CAA+C;IAC/C,IAAI,OAAY,CAAC;IACjB,MAAM,UAAU,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC;IAClC,UAAU,CAAC,gBAAgB,CAAC,SAAS,EAAE,CAAC,IAAY,EAAE,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,IAAI,EAAE,EAAE,CAAC,CAAC,CAAC;IAC/F,UAAU,CAAC,gBAAgB,CAAC,SAAS,EAAE,CAAC,GAAa,EAAE,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC;IACxE,UAAU,CAAC,gBAAgB,CAAC,SAAS,EAAE,CAAC,GAAa,EAAE,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC;IACxE,UAAU,CAAC,gBAAgB,CAAC,eAAe,EAAE,CAAC,GAAa,EAAE,KAAa,EAAE,EAAE,CAAC,aAAa,CAAC,GAAG,EAAE,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IACtH,UAAU,CAAC,gBAAgB,CAAC,YAAY,EAAE,GAAG,EAAE,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC;IACnE,UAAU,CAAC,gBAAgB,CAAC,QAAQ,EAAE,CAAC,IAAgB,EAAE,EAAE,CAAC,CAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;IAClF,UAAU,CAAC,gBAAgB,CAAC,OAAO,EAAE,GAAG,EAAE,CAAC,KAAK,EAAE,CAAC,CAAC;IACpD,UAAU,CAAC,gBAAgB,CAAC,QAAQ,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;IAE5D,OAAO,KAAK,EAAE,IAAS,EAAE,EAAE,GAAG,OAAO,GAAG,IAAI,CAAC,CAAC,OAAO,UAAU,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;AACtF,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,IAAqC;IACjE,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC,IAAI,CAAC,CAAC;IAExC,OAAO,IAAI,SAAS,CAAC;QACjB,UAAU,EAAE,IAAI;QAChB,KAAK,CAAC,SAAS,CAAC,IAAS,EAAE,SAAyB,EAAE,QAAkC;YACpF,IAAI,CAAC;gBACD,QAAQ,CAAC,IAAI,EAAE,MAAM,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC;YACvC,CAAC;YACD,OAAO,GAAQ,EAAE,CAAC;gBACd,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBACjB,QAAQ,CAAC,GAAY,CAAC,CAAC;YAC3B,CAAC;QACL,CAAC;KACJ,CAAC,CAAC;AACP,CAAC;AAED,SAAS,KAAK;IACV,OAAO,SAAS,MAAM,EAAE,EAAE,CAAC;AAC/B,CAAC;AAED,KAAK,UAAU,UAAU,CAAC,IAAY;IAClC,IAAI,MAAM,GAA4B,EAAE,CAAC;IAEzC,MAAM,OAAO,GAAG,MAAM,WAAW,CAAC,EAAE,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC,CAAC;IAE7D,KAAK,MAAM,GAAG,IAAI,OAAO,EAAE,CAAC;QACxB,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;QACrC,IAAI,IAAI,IAAI,IAAI,CAAC,MAAM,IAAI,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC;YACjD,MAAM,CAAC,GAAG,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;YACvB,MAAM,CAAC,GAAG,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;YACvB,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;gBACT,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;YAClB,CAAC;QACL,CAAC;IACL,CAAC;IAED,OAAO,MAAM,CAAC;AAClB,CAAC"}
@@ -0,0 +1,19 @@
1
+ /**
2
+ * Marker for record objects whose string values are already free of control
3
+ * characters (\x00-\x1F and \x7F).
4
+ *
5
+ * Input readers that strip control characters while parsing (e.g. the XML
6
+ * readers) set this on the records they emit. Output writers that would
7
+ * otherwise re-strip every value (e.g. alephseq) can then skip that work.
8
+ *
9
+ * It is a Symbol so it never shows up in `JSON.stringify`, `Object.keys` or
10
+ * `for...in` — outputs that serialise the whole record object (json, jsonl,
11
+ * rdf) are unaffected. A transform that builds a new record object (e.g. a
12
+ * JSONata `fix`) naturally drops the marker, so escaping is re-applied unless
13
+ * cleanliness is re-asserted — a safe default.
14
+ */
15
+ export declare const CLEAN: unique symbol;
16
+ export type CleanFlag = {
17
+ [CLEAN]?: boolean;
18
+ };
19
+ //# sourceMappingURL=marc_record.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"marc_record.d.ts","sourceRoot":"","sources":["../../src/util/marc_record.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AACH,eAAO,MAAM,KAAK,EAAE,OAAO,MAAoC,CAAC;AAEhE,MAAM,MAAM,SAAS,GAAG;IAAE,CAAC,KAAK,CAAC,CAAC,EAAE,OAAO,CAAA;CAAE,CAAC"}
@@ -0,0 +1,16 @@
1
+ /**
2
+ * Marker for record objects whose string values are already free of control
3
+ * characters (\x00-\x1F and \x7F).
4
+ *
5
+ * Input readers that strip control characters while parsing (e.g. the XML
6
+ * readers) set this on the records they emit. Output writers that would
7
+ * otherwise re-strip every value (e.g. alephseq) can then skip that work.
8
+ *
9
+ * It is a Symbol so it never shows up in `JSON.stringify`, `Object.keys` or
10
+ * `for...in` — outputs that serialise the whole record object (json, jsonl,
11
+ * rdf) are unaffected. A transform that builds a new record object (e.g. a
12
+ * JSONata `fix`) naturally drops the marker, so escaping is re-applied unless
13
+ * cleanliness is re-asserted — a safe default.
14
+ */
15
+ export const CLEAN = Symbol('marc.record.clean');
16
+ //# sourceMappingURL=marc_record.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"marc_record.js","sourceRoot":"","sources":["../../src/util/marc_record.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AACH,MAAM,CAAC,MAAM,KAAK,GAAkB,MAAM,CAAC,mBAAmB,CAAC,CAAC"}
@@ -0,0 +1 @@
1
+ {"version":3,"file":"worker-str.d.ts","sourceRoot":"","sources":["../src/worker-str.ts"],"names":[],"mappings":""}
@@ -0,0 +1 @@
1
+ {"version":3,"file":"worker-str.js","sourceRoot":"","sources":["../src/worker-str.ts"],"names":[],"mappings":"AAAA,gFAAgF;AAChF,8EAA8E;AAC9E,4EAA4E;AAC5E,EAAE;AACF,+EAA+E;AAC/E,YAAY;AACZ,0EAA0E;AAC1E,qCAAqC;AACrC,mEAAmE;AACnE,OAAO,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AAC7D,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AACvC,OAAO,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AAChD,OAAO,EAAE,MAAM,EAAE,MAAM,iBAAiB,CAAC;AAEzC,MAAM,EAAE,IAAI,EAAE,GAAG,EAAE,KAAK,EAAE,GAAG,UAAwD,CAAC;AAEtF,MAAM,QAAQ,GAAG,MAAM,UAAU,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;AACjD,MAAM,MAAM,GAAuB,GAAG;IAClC,CAAC,CAAC,MAAM,CAAC,MAAM,UAAU,CAAC,GAAG,EAAE,WAAW,CAAC,CAAC,CAAC,YAAY,CAAC,KAAK,IAAI,EAAE,CAAC;IACtE,CAAC,CAAC,CAAC,CAAM,EAAE,EAAE,CAAC,CAAC,CAAC;AAEpB,6EAA6E;AAC7E,KAAK,UAAU,KAAK,CAAC,IAAY;IAC7B,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,SAAS,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC;IAClD,MAAM,OAAO,GAAU,EAAE,CAAC;IAC1B,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QACxC,GAAG,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,CAAM,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;QAC5C,GAAG,CAAC,EAAE,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;QACvB,GAAG,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;QACxB,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAClC,CAAC,CAAC,CAAC;IACH,OAAO,OAAO,CAAC;AACnB,CAAC;AAED,UAAW,CAAC,WAAW,CAAC,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;AAEzC,UAAW,CAAC,EAAE,CAAC,SAAS,EAAE,KAAK,EAAE,GAAkC,EAAE,EAAE;IACnE,IAAI,CAAC;QACD,MAAM,OAAO,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACtC,IAAI,GAAG,GAAG,EAAE,CAAC;QACb,IAAI,CAAC,GAAG,CAAC,CAAC;QACV,KAAK,MAAM,GAAG,IAAI,OAAO,EAAE,CAAC;YACxB,MAAM,CAAC,GAAG,MAAM,MAAM,CAAC,GAAG,CAAC,CAAC;YAC5B,IAAI,CAAC,KAAK,MAAM,IAAI,CAAC,KAAK,SAAS,IAAI,CAAC,KAAK,IAAI;gBAAE,SAAS;YAC5D,IAAI,OAAO,CAAC,KAAK,QAAQ,IAAI,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,MAAM,KAAK,CAAC;gBAAE,SAAS,CAAC,uBAAuB;YAC3F,GAAG,IAAI,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC;YAChC,CAAC,EAAE,CAAC;QACR,CAAC;QACD,UAAW,CAAC,WAAW,CAAC,EAAE,GAAG,EAAE,GAAG,CAAC,GAAG,EAAE,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC;IACtD,CAAC;IAAC,OAAO,GAAQ,EAAE,CAAC;QAChB,UAAW,CAAC,WAAW,CAAC,EAAE,GAAG,EAAE,GAAG,CAAC,GAAG,EAAE,KAAK,EAAE,GAAG,EAAE,OAAO,IAAI,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IAClF,CAAC;AACL,CAAC,CAAC,CAAC"}
@@ -0,0 +1,2 @@
1
+ export {};
2
+ //# sourceMappingURL=worker.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"worker.d.ts","sourceRoot":"","sources":["../src/worker.ts"],"names":[],"mappings":""}
package/dist/worker.js ADDED
@@ -0,0 +1,33 @@
1
+ // Worker thread: runs a parallelizable map (e.g. jsonata, fix) on batches of
2
+ // records. The heavy per-record work (jsonata evaluate / fix chain) runs here;
3
+ // the main thread does I/O, parsing and serialization.
4
+ //
5
+ // Protocol:
6
+ // main -> worker : { seq, batch: record[] }
7
+ // worker -> main : { ready: true } (once, on startup)
8
+ // { seq, mapped: (record|null)[] } (null = rejected)
9
+ // { seq, error: string }
10
+ import { parentPort, workerData } from 'node:worker_threads';
11
+ import { loadPlugin } from './plugin-loader.js';
12
+ import { REJECT } from './fix/signal.js';
13
+ const { map, param } = workerData;
14
+ const mod = await loadPlugin(map, 'transform');
15
+ if (typeof mod.createMapper !== 'function') {
16
+ throw new Error(`map '${map}' is not parallelizable (no createMapper)`);
17
+ }
18
+ const mapper = await mod.createMapper(param ?? {});
19
+ parentPort.postMessage({ ready: true });
20
+ parentPort.on('message', async (msg) => {
21
+ try {
22
+ const mapped = [];
23
+ for (const rec of msg.batch) {
24
+ const r = await mapper(rec);
25
+ mapped.push(r === REJECT ? null : r); // null marks a dropped record
26
+ }
27
+ parentPort.postMessage({ seq: msg.seq, mapped });
28
+ }
29
+ catch (err) {
30
+ parentPort.postMessage({ seq: msg.seq, error: err?.message ?? String(err) });
31
+ }
32
+ });
33
+ //# sourceMappingURL=worker.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"worker.js","sourceRoot":"","sources":["../src/worker.ts"],"names":[],"mappings":"AAAA,6EAA6E;AAC7E,+EAA+E;AAC/E,uDAAuD;AACvD,EAAE;AACF,YAAY;AACZ,8CAA8C;AAC9C,gFAAgF;AAChF,+EAA+E;AAC/E,4CAA4C;AAC5C,OAAO,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AAC7D,OAAO,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AAChD,OAAO,EAAE,MAAM,EAAE,MAAM,iBAAiB,CAAC;AAEzC,MAAM,EAAE,GAAG,EAAE,KAAK,EAAE,GAAG,UAAyC,CAAC;AAEjE,MAAM,GAAG,GAAG,MAAM,UAAU,CAAC,GAAG,EAAE,WAAW,CAAC,CAAC;AAC/C,IAAI,OAAO,GAAG,CAAC,YAAY,KAAK,UAAU,EAAE,CAAC;IACzC,MAAM,IAAI,KAAK,CAAC,QAAQ,GAAG,2CAA2C,CAAC,CAAC;AAC5E,CAAC;AACD,MAAM,MAAM,GAAuB,MAAM,GAAG,CAAC,YAAY,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC;AAEvE,UAAW,CAAC,WAAW,CAAC,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;AAEzC,UAAW,CAAC,EAAE,CAAC,SAAS,EAAE,KAAK,EAAE,GAAkC,EAAE,EAAE;IACnE,IAAI,CAAC;QACD,MAAM,MAAM,GAAU,EAAE,CAAC;QACzB,KAAK,MAAM,GAAG,IAAI,GAAG,CAAC,KAAK,EAAE,CAAC;YAC1B,MAAM,CAAC,GAAG,MAAM,MAAM,CAAC,GAAG,CAAC,CAAC;YAC5B,MAAM,CAAC,IAAI,CAAC,CAAC,KAAK,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAG,8BAA8B;QAC1E,CAAC;QACD,UAAW,CAAC,WAAW,CAAC,EAAE,GAAG,EAAE,GAAG,CAAC,GAAG,EAAE,MAAM,EAAE,CAAC,CAAC;IACtD,CAAC;IAAC,OAAO,GAAQ,EAAE,CAAC;QAChB,UAAW,CAAC,WAAW,CAAC,EAAE,GAAG,EAAE,GAAG,CAAC,GAAG,EAAE,KAAK,EAAE,GAAG,EAAE,OAAO,IAAI,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IAClF,CAAC;AACL,CAAC,CAAC,CAAC"}
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "marcattacks",
3
- "version": "2.5.1",
3
+ "version": "2.7.0",
4
4
  "description": "A powerful streaming MARC21 to RDF converter with JSONata transformation and S3 support.",
5
5
  "keywords": [
6
6
  "marc21",
@@ -36,6 +36,9 @@
36
36
  "demo:jsonl": "npx marcattacks --to jsonl data/sample.xml",
37
37
  "demo:rdf": "npx marcattacks --to rdf --map marc2rdf data/sample.xml",
38
38
  "demo:jsonld": "npx marcattacks --to jsonl --param fix=./demo/marc2rdf.jsonata data/sample.xml",
39
+ "demo:fix": "npx marcattacks --to jsonl --map fix --param fix=./demo/example.fix data/sample.xml",
40
+ "demo:fix:rdf": "npx marcattacks --to jsonl --map fix --param fix=./demo/marc2rdf.fix data/sample.xml",
41
+ "demo:workers": "npx marcattacks --to jsonl --param fix=./demo/marc2rdf.jsonata --workers 4 --info data/sample.xml",
39
42
  "demo:biblio:one": "npx marcattacks --from json --info ./data/biblio.json",
40
43
  "demo:biblio:all": "npx marcattacks --from jsonl --info https://biblio.ugent.be/exports/publications.json",
41
44
  "demo:parquet": " npx marcattacks ./data/sample.xml --to parquet --map avram --param schema=./demo/avram.schema --info --out ./data/avram.parquet",
@@ -1,7 +0,0 @@
1
- import { Transform } from 'stream';
2
- export interface OutputMultipartOptions {
3
- header?: string;
4
- delimiter?: string;
5
- }
6
- export declare function transform(opts?: OutputMultipartOptions): Promise<Transform>;
7
- //# sourceMappingURL=mulitipart.d.ts.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"mulitipart.d.ts","sourceRoot":"","sources":["../../src/output/mulitipart.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,QAAQ,CAAC;AAKnC,MAAM,WAAW,sBAAsB;IACnC,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;CACtB;AAED,wBAAsB,SAAS,CAAC,IAAI,GAAE,sBAA2B,GAAI,OAAO,CAAC,SAAS,CAAC,CAyDtF"}