marcattacks 2.5.1 → 2.8.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (100) hide show
  1. package/README.md +91 -2
  2. package/dist/attacker.d.ts +6 -0
  3. package/dist/attacker.d.ts.map +1 -1
  4. package/dist/attacker.js +38 -0
  5. package/dist/attacker.js.map +1 -1
  6. package/dist/command.js +1 -0
  7. package/dist/command.js.map +1 -1
  8. package/dist/fix/binds.d.ts +5 -0
  9. package/dist/fix/binds.d.ts.map +1 -0
  10. package/dist/fix/binds.js +82 -0
  11. package/dist/fix/binds.js.map +1 -0
  12. package/dist/fix/conditions.d.ts +5 -0
  13. package/dist/fix/conditions.d.ts.map +1 -0
  14. package/dist/fix/conditions.js +111 -0
  15. package/dist/fix/conditions.js.map +1 -0
  16. package/dist/fix/fixes.d.ts +7 -0
  17. package/dist/fix/fixes.d.ts.map +1 -0
  18. package/dist/fix/fixes.js +478 -0
  19. package/dist/fix/fixes.js.map +1 -0
  20. package/dist/fix/index.d.ts +12 -0
  21. package/dist/fix/index.d.ts.map +1 -0
  22. package/dist/fix/index.js +40 -0
  23. package/dist/fix/index.js.map +1 -0
  24. package/dist/fix/parser.d.ts +25 -0
  25. package/dist/fix/parser.d.ts.map +1 -0
  26. package/dist/fix/parser.js +156 -0
  27. package/dist/fix/parser.js.map +1 -0
  28. package/dist/fix/path.d.ts +23 -0
  29. package/dist/fix/path.d.ts.map +1 -0
  30. package/dist/fix/path.js +351 -0
  31. package/dist/fix/path.js.map +1 -0
  32. package/dist/fix/signal.d.ts +2 -0
  33. package/dist/fix/signal.d.ts.map +1 -0
  34. package/dist/fix/signal.js +5 -0
  35. package/dist/fix/signal.js.map +1 -0
  36. package/dist/fix/util.d.ts +6 -0
  37. package/dist/fix/util.d.ts.map +1 -0
  38. package/dist/fix/util.js +133 -0
  39. package/dist/fix/util.js.map +1 -0
  40. package/dist/input/alephseq.d.ts.map +1 -1
  41. package/dist/input/alephseq.js +18 -5
  42. package/dist/input/alephseq.js.map +1 -1
  43. package/dist/input/fastxml.d.ts +3 -0
  44. package/dist/input/fastxml.d.ts.map +1 -0
  45. package/dist/input/fastxml.js +134 -0
  46. package/dist/input/fastxml.js.map +1 -0
  47. package/dist/input/xml.d.ts.map +1 -1
  48. package/dist/input/xml.js +4 -2
  49. package/dist/input/xml.js.map +1 -1
  50. package/dist/marcmap.d.ts.map +1 -1
  51. package/dist/marcmap.js +15 -3
  52. package/dist/marcmap.js.map +1 -1
  53. package/dist/output/alephseq.d.ts.map +1 -1
  54. package/dist/output/alephseq.js +29 -5
  55. package/dist/output/alephseq.js.map +1 -1
  56. package/dist/output/null.d.ts +3 -0
  57. package/dist/output/null.d.ts.map +1 -0
  58. package/dist/output/null.js +13 -0
  59. package/dist/output/null.js.map +1 -0
  60. package/dist/output/xml.d.ts.map +1 -1
  61. package/dist/output/xml.js +27 -0
  62. package/dist/output/xml.js.map +1 -1
  63. package/dist/stream/framer.d.ts.map +1 -0
  64. package/dist/stream/framer.js.map +1 -0
  65. package/dist/stream/worker-pool-str.d.ts.map +1 -0
  66. package/dist/stream/worker-pool-str.js.map +1 -0
  67. package/dist/stream/worker-pool.d.ts +10 -0
  68. package/dist/stream/worker-pool.d.ts.map +1 -0
  69. package/dist/stream/worker-pool.js +217 -0
  70. package/dist/stream/worker-pool.js.map +1 -0
  71. package/dist/transform/fix.d.ts +16 -0
  72. package/dist/transform/fix.d.ts.map +1 -0
  73. package/dist/transform/fix.js +46 -0
  74. package/dist/transform/fix.js.map +1 -0
  75. package/dist/transform/jsonata.d.ts +5 -0
  76. package/dist/transform/jsonata.d.ts.map +1 -1
  77. package/dist/transform/jsonata.js +45 -44
  78. package/dist/transform/jsonata.js.map +1 -1
  79. package/dist/transform/marcids.d.ts.map +1 -1
  80. package/dist/transform/marcids.js +3 -0
  81. package/dist/transform/marcids.js.map +1 -1
  82. package/dist/util/marc_record.d.ts +19 -0
  83. package/dist/util/marc_record.d.ts.map +1 -0
  84. package/dist/util/marc_record.js +16 -0
  85. package/dist/util/marc_record.js.map +1 -0
  86. package/dist/worker-str.d.ts.map +1 -0
  87. package/dist/worker-str.js.map +1 -0
  88. package/dist/worker.d.ts +2 -0
  89. package/dist/worker.d.ts.map +1 -0
  90. package/dist/worker.js +33 -0
  91. package/dist/worker.js.map +1 -0
  92. package/package.json +5 -2
  93. package/dist/output/mulitipart.d.ts +0 -7
  94. package/dist/output/mulitipart.d.ts.map +0 -1
  95. package/dist/output/mulitipart.js +0 -53
  96. package/dist/output/mulitipart.js.map +0 -1
  97. package/dist/output/text.d.ts +0 -6
  98. package/dist/output/text.d.ts.map +0 -1
  99. package/dist/output/text.js +0 -34
  100. package/dist/output/text.js.map +0 -1
@@ -1 +1 @@
1
- {"version":3,"file":"xml.js","sourceRoot":"","sources":["../../src/output/xml.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,QAAQ,CAAC;AACnC,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,aAAa,EAAG,cAAc,EAAC,MAAM,eAAe,CAAC;AAChF,OAAO,MAAM,MAAM,QAAQ,CAAC;AAE5B,MAAM,MAAM,GAAG,MAAM,CAAC,SAAS,EAAE,CAAC;AAElC,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,KAAS;IACrC,IAAI,OAAO,GAAG,IAAI,CAAC;IAEnB,OAAO,IAAI,SAAS,CAAC;QACjB,UAAU,EAAE,IAAI;QAChB,SAAS,CAAC,IAAS,EAAE,SAAS,EAAE,QAAQ;YACpC,IAAI,GAAG,GAAgB,IAAI,CAAC,QAAQ,CAAC,CAAC;YAEtC,IAAI,CAAC,GAAG,EAAE,CAAC;gBACP,MAAM,CAAC,KAAK,CAAC,sBAAsB,CAAC,CAAC;gBACrC,QAAQ,EAAE,CAAA;gBACV,OAAO;YACX,CAAC;YAED,IAAI,MAAM,GAAG,EAAE,CAAC;YAEhB,IAAI,OAAO,EAAE,CAAC;gBACV,MAAM,IAAI,8CAA8C,CAAC;gBACzD,MAAM,IAAI,mEAAmE,CAAC;gBAC9E,OAAO,GAAG,KAAK,CAAC;YACpB,CAAC;YAED,MAAM,IAAI,kBAAkB,CAAC;YAE7B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAG,CAAC,GAAG,GAAG,CAAC,MAAM,EAAG,CAAC,EAAE,EAAE,CAAC;gBACpC,IAAI,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;gBAC1B,IAAI,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;gBAC1B,IAAI,GAAG,KAAK,KAAK,EAAE,CAAC,CAAA,CAAC;qBAChB,IAAI,GAAG,KAAK,KAAK,EAAE,CAAC;oBACrB,IAAI,KAAK,GAAG,aAAa,CAAC,GAAG,CAAC,CAAC,CAAE,EAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;oBAC3C,MAAM,IAAI,kBAAkB,SAAS,CAAC,KAAK,CAAC,kBAAkB,CAAC;gBACnE,CAAC;qBACI,IAAI,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC;oBACxB,IAAI,KAAK,GAAG,aAAa,CAAC,GAAG,CAAC,CAAC,CAAE,EAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;oBAC3C,MAAM,IAAI,6BAA6B,SAAS,CAAC,GAAG,EAAC,EAAC,YAAY,EAAC,IAAI,EAAC,CAAC,KAAK,SAAS,CAAC,KAAK,CAAC,wBAAwB,CAAC;gBAC3H,CAAC;qBACI,CAAC;oBACF,MAAM,IAAI,0BAA0B,SAAS,CAAC,GAAG,CAAC,WAAW,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAC,EAAC,YAAY,EAAC,IAAI,EAAC,CAAC,WAAW,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAC,EAAC,YAAY,EAAC,IAAI,EAAC,CAAC,MAAM,CAAC;oBACzJ,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,IAAI,EAAC,KAAK,EAAE,EAAE;wBAClC,MAAM,IAAI,4BAA4B,SAAS,CAAC,IAAI,CAAC,KAAK,SAAS,CAAC,KAAK,CAAC,oBAAoB,CAAC;oBACnG,CAAC,CAAC,CAAC;oBACH,MAAM,IAAI,uBAAuB,CAAC;gBACtC,CAAC;YACL,CAAC;YAED,MAAM,IAAI,mBAAmB,CAAC;YAE9B,MAAM,CAAC,KAAK,CAAC,UAAU,MAAM,CAAC,MAAM,QAAQ,CAAC,CAAC;YAE9C,QAAQ,CAAC,IAAI,EAAC,MAAM,CAAC,CAAC;QAC1B,CAAC;QACD,KAAK,CAAC,QAAQ;YACV,MAAM,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC;YAC9B,IAAI,CAAC,OAAO,EAAE,CAAC;gBACX,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;gBACzB,IAAI,MAAM,GAAG,sBAAsB,CAAC;gBACpC,MAAM,CAAC,KAAK,CAAC,UAAU,MAAM,CAAC,MAAM,QAAQ,CAAC,CAAC;gBAC9C,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACtB,CAAC;YACD,QAAQ,EAAE,CAAC;QACf,CAAC;KACJ,CAAC,CAAC;AACP,CAAC;AAED,MAAM,UAAU,SAAS,CACvB,KAAyC,EACzC,OAAoC;IAElC,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,KAAK,SAAS;QAAE,OAAO,EAAE,CAAC;IAErD,IAAI,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;IAEtB,qEAAqE;IAErE,6EAA6E;IAC7E,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,qCAAqC,EAAE,QAAQ,CAAC,CAAC,CAAE,2BAA2B;IAC5F,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,sCAAsC,EAAE,QAAQ,CAAC,CAAC,CAAC,0BAA0B;IAE3F,8CAA8C;IAC9C,6DAA6D;IAC7D,yDAAyD;IACzD,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,oEAAoE,EAAE,EAAE,CAAC,CAAC;IAExF,+DAA+D;IAC/D,4DAA4D;IAC5D,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,kBAAkB,EAAE,EAAE,CAAC,CAAC;IAEtC,wCAAwC;IACxC,4DAA4D;IAC5D,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAE7B,kCAAkC;IAClC,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;IAC5B,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;IAE5B,IAAI,OAAO,EAAE,YAAY,EAAE,CAAC;QACxB,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;QAC9B,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;IAClC,CAAC;IAED,OAAO,CAAC,CAAC;AACb,CAAC"}
1
+ {"version":3,"file":"xml.js","sourceRoot":"","sources":["../../src/output/xml.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,QAAQ,CAAC;AACnC,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,aAAa,EAAG,cAAc,EAAC,MAAM,eAAe,CAAC;AAChF,OAAO,MAAM,MAAM,QAAQ,CAAC;AAE5B,MAAM,MAAM,GAAG,MAAM,CAAC,SAAS,EAAE,CAAC;AAElC,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,KAAS;IACrC,IAAI,OAAO,GAAG,IAAI,CAAC;IAEnB,OAAO,IAAI,SAAS,CAAC;QACjB,UAAU,EAAE,IAAI;QAChB,SAAS,CAAC,IAAS,EAAE,SAAS,EAAE,QAAQ;YACpC,IAAI,GAAG,GAAgB,IAAI,CAAC,QAAQ,CAAC,CAAC;YAEtC,IAAI,CAAC,GAAG,EAAE,CAAC;gBACP,MAAM,CAAC,KAAK,CAAC,sBAAsB,CAAC,CAAC;gBACrC,QAAQ,EAAE,CAAA;gBACV,OAAO;YACX,CAAC;YAED,IAAI,MAAM,GAAG,EAAE,CAAC;YAEhB,IAAI,OAAO,EAAE,CAAC;gBACV,MAAM,IAAI,8CAA8C,CAAC;gBACzD,MAAM,IAAI,mEAAmE,CAAC;gBAC9E,OAAO,GAAG,KAAK,CAAC;YACpB,CAAC;YAED,MAAM,IAAI,kBAAkB,CAAC;YAE7B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAG,CAAC,GAAG,GAAG,CAAC,MAAM,EAAG,CAAC,EAAE,EAAE,CAAC;gBACpC,IAAI,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;gBAC1B,IAAI,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;gBAC1B,IAAI,GAAG,KAAK,KAAK,EAAE,CAAC,CAAA,CAAC;qBAChB,IAAI,GAAG,KAAK,KAAK,EAAE,CAAC;oBACrB,IAAI,KAAK,GAAG,aAAa,CAAC,GAAG,CAAC,CAAC,CAAE,EAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;oBAC3C,MAAM,IAAI,kBAAkB,SAAS,CAAC,KAAK,CAAC,kBAAkB,CAAC;gBACnE,CAAC;qBACI,IAAI,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC;oBACxB,IAAI,KAAK,GAAG,aAAa,CAAC,GAAG,CAAC,CAAC,CAAE,EAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;oBAC3C,MAAM,IAAI,6BAA6B,SAAS,CAAC,GAAG,EAAC,EAAC,YAAY,EAAC,IAAI,EAAC,CAAC,KAAK,SAAS,CAAC,KAAK,CAAC,wBAAwB,CAAC;gBAC3H,CAAC;qBACI,CAAC;oBACF,MAAM,IAAI,0BAA0B,SAAS,CAAC,GAAG,CAAC,WAAW,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAC,EAAC,YAAY,EAAC,IAAI,EAAC,CAAC,WAAW,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAC,EAAC,YAAY,EAAC,IAAI,EAAC,CAAC,MAAM,CAAC;oBACzJ,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,IAAI,EAAC,KAAK,EAAE,EAAE;wBAClC,MAAM,IAAI,4BAA4B,SAAS,CAAC,IAAI,CAAC,KAAK,SAAS,CAAC,KAAK,CAAC,oBAAoB,CAAC;oBACnG,CAAC,CAAC,CAAC;oBACH,MAAM,IAAI,uBAAuB,CAAC;gBACtC,CAAC;YACL,CAAC;YAED,MAAM,IAAI,mBAAmB,CAAC;YAE9B,MAAM,CAAC,KAAK,CAAC,UAAU,MAAM,CAAC,MAAM,QAAQ,CAAC,CAAC;YAE9C,QAAQ,CAAC,IAAI,EAAC,MAAM,CAAC,CAAC;QAC1B,CAAC;QACD,KAAK,CAAC,QAAQ;YACV,MAAM,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC;YAC9B,IAAI,CAAC,OAAO,EAAE,CAAC;gBACX,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;gBACzB,IAAI,MAAM,GAAG,sBAAsB,CAAC;gBACpC,MAAM,CAAC,KAAK,CAAC,UAAU,MAAM,CAAC,MAAM,QAAQ,CAAC,CAAC;gBAC9C,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACtB,CAAC;YACD,QAAQ,EAAE,CAAC;QACf,CAAC;KACJ,CAAC,CAAC;AACP,CAAC;AAED,0EAA0E;AAC1E,6EAA6E;AAC7E,6EAA6E;AAC7E,oEAAoE;AACpE,SAAS,WAAW,CAAC,CAAS,EAAE,YAAqB;IACjD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAChC,MAAM,CAAC,GAAG,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;QAC1B,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE;YAAE,OAAO,IAAI,CAAC,CAAiB,QAAQ;QAC3E,IAAI,YAAY,IAAI,CAAC,CAAC,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,CAAC;YAAE,OAAO,IAAI,CAAC,CAAW,MAAM;QACzE,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,IAAI,CAAC;YAAE,OAAO,IAAI,CAAC,CAAC,OAAO;QAC3F,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,IAAI;YAAE,OAAO,IAAI,CAAC;QACxC,IAAI,CAAC,IAAI,MAAM,IAAI,CAAC,IAAI,MAAM;YAAE,OAAO,IAAI,CAAC,CAAuB,aAAa;QAChF,IAAI,CAAC,IAAI,MAAM,IAAI,CAAC,IAAI,MAAM;YAAE,OAAO,IAAI,CAAC,CAAuB,YAAY;QAC/E,IAAI,CAAC,KAAK,MAAM,IAAI,CAAC,KAAK,MAAM;YAAE,OAAO,IAAI,CAAC;IAClD,CAAC;IACD,OAAO,KAAK,CAAC;AACjB,CAAC;AAED,MAAM,UAAU,SAAS,CACvB,KAAyC,EACzC,OAAoC;IAElC,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,KAAK,SAAS;QAAE,OAAO,EAAE,CAAC;IAErD,IAAI,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;IAEtB,sEAAsE;IACtE,IAAI,CAAC,WAAW,CAAC,CAAC,EAAE,OAAO,EAAE,YAAY,KAAK,IAAI,CAAC;QAAE,OAAO,CAAC,CAAC;IAE9D,qEAAqE;IAErE,6EAA6E;IAC7E,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,qCAAqC,EAAE,QAAQ,CAAC,CAAC,CAAE,2BAA2B;IAC5F,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,sCAAsC,EAAE,QAAQ,CAAC,CAAC,CAAC,0BAA0B;IAE3F,8CAA8C;IAC9C,6DAA6D;IAC7D,yDAAyD;IACzD,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,oEAAoE,EAAE,EAAE,CAAC,CAAC;IAExF,+DAA+D;IAC/D,4DAA4D;IAC5D,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,kBAAkB,EAAE,EAAE,CAAC,CAAC;IAEtC,wCAAwC;IACxC,4DAA4D;IAC5D,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAE7B,kCAAkC;IAClC,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;IAC5B,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;IAE5B,IAAI,OAAO,EAAE,YAAY,EAAE,CAAC;QACxB,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;QAC9B,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;IAClC,CAAC;IAED,OAAO,CAAC,CAAC;AACb,CAAC"}
@@ -0,0 +1 @@
1
+ {"version":3,"file":"framer.d.ts","sourceRoot":"","sources":["../../src/stream/framer.ts"],"names":[],"mappings":"AAUA,OAAO,EAAE,SAAS,EAA0B,MAAM,QAAQ,CAAC;AAG3D,eAAO,MAAM,QAAQ,aAAmD,CAAC;AAkCzE,wBAAgB,YAAY,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,SAAS,CA+BnE"}
@@ -0,0 +1 @@
1
+ {"version":3,"file":"framer.js","sourceRoot":"","sources":["../../src/stream/framer.ts"],"names":[],"mappings":"AAAA,8EAA8E;AAC9E,6EAA6E;AAC7E,+EAA+E;AAC/E,+EAA+E;AAC/E,4BAA4B;AAC5B,EAAE;AACF,+EAA+E;AAC/E,uDAAuD;AACvD,yEAAyE;AACzE,mEAAmE;AACnE,OAAO,EAAE,SAAS,EAA0B,MAAM,QAAQ,CAAC;AAC3D,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AAEpD,MAAM,CAAC,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,CAAC,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,UAAU,CAAC,CAAC,CAAC;AAEzE,MAAM,YAAY,GAAG,6BAA6B,CAAC;AAEnD,+EAA+E;AAC/E,+EAA+E;AAC/E,SAAS,kBAAkB,CAAC,IAAY,EAAE,GAAW;IACjD,IAAI,IAAI,KAAK,OAAO,EAAE,CAAC;QACnB,MAAM,IAAI,GAAa,EAAE,CAAC;QAC1B,KAAK,IAAI,CAAC,GAAG,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,GAAG,GAAG,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,GAAG,CAAC,CAAC;YAAE,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QACvF,OAAO,IAAI,CAAC;IAChB,CAAC;IACD,IAAI,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;QACvC,MAAM,IAAI,GAAa,EAAE,CAAC;QAC1B,YAAY,CAAC,SAAS,GAAG,CAAC,CAAC;QAC3B,IAAI,CAAyB,CAAC;QAC9B,OAAO,CAAC,CAAC,GAAG,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAAE,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;QACtE,OAAO,IAAI,CAAC;IAChB,CAAC;IACD,2EAA2E;IAC3E,oEAAoE;IACpE,MAAM,IAAI,GAAa,EAAE,CAAC;IAC1B,IAAI,MAAM,GAAkB,IAAI,CAAC;IACjC,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,KAAK,IAAI,CAAC,GAAG,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,GAAG,GAAG,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC;QACnE,MAAM,EAAE,GAAG,GAAG,CAAC,OAAO,CAAC,GAAG,EAAE,SAAS,CAAC,CAAC;QACvC,MAAM,EAAE,GAAG,EAAE,IAAI,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC;QAClF,IAAI,MAAM,KAAK,IAAI,IAAI,EAAE,KAAK,MAAM;YAAE,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,yBAAyB;QACrF,MAAM,GAAG,EAAE,CAAC;QACZ,SAAS,GAAG,CAAC,GAAG,CAAC,CAAC;IACtB,CAAC;IACD,OAAO,IAAI,CAAC;AAChB,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,IAAY,EAAE,KAAa;IACpD,MAAM,OAAO,GAAG,IAAI,aAAa,CAAC,MAAM,CAAC,CAAC;IAC1C,IAAI,GAAG,GAAG,EAAE,CAAC;IACb,IAAI,GAAG,GAAG,CAAC,CAAC;IAEZ,SAAS,IAAI,CAAC,MAAiB,EAAE,KAAc;QAC3C,SAAS,CAAC;YACN,MAAM,IAAI,GAAG,kBAAkB,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;YAC3C,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;gBAAE,MAAM;YAC7B,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,MAAM,GAAG,KAAK;gBAAE,MAAM,CAAO,wBAAwB;YACxE,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAE,CAAC;YACpD,MAAM,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,GAAG,EAAE,EAAE,IAAI,EAAE,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;YACrD,GAAG,GAAG,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;YACrB,IAAI,IAAI,CAAC,MAAM,GAAG,KAAK;gBAAE,MAAM,CAAiB,mBAAmB;QACvE,CAAC;IACL,CAAC;IAED,OAAO,IAAI,SAAS,CAAC;QACjB,kBAAkB,EAAE,IAAI;QACxB,SAAS,CAAC,KAAU,EAAE,IAAY,EAAE,EAAqB;YACrD,GAAG,IAAI,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;YAC5B,IAAI,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;YAClB,EAAE,EAAE,CAAC;QACT,CAAC;QACD,KAAK,CAAC,EAAqB;YACvB,GAAG,IAAI,OAAO,CAAC,GAAG,EAAE,CAAC;YACrB,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;YACjB,IAAI,GAAG,CAAC,MAAM,GAAG,CAAC;gBAAE,IAAI,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,GAAG,EAAE,EAAE,IAAI,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC,0BAA0B;YACpF,EAAE,EAAE,CAAC;QACT,CAAC;KACJ,CAAC,CAAC;AACP,CAAC"}
@@ -0,0 +1 @@
1
+ {"version":3,"file":"worker-pool-str.d.ts","sourceRoot":"","sources":["../../src/stream/worker-pool-str.ts"],"names":[],"mappings":"AAMA,OAAO,EAAE,SAAS,EAA0B,MAAM,QAAQ,CAAC;AAM3D,MAAM,WAAW,WAAW;IAAG,IAAI,EAAE,MAAM,CAAC;IAAC,GAAG,CAAC,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,GAAG,CAAC;IAAC,OAAO,EAAE,MAAM,CAAC;CAAE;AAEzF,wBAAgB,sBAAsB,CAAC,IAAI,EAAE,WAAW,GAAG,SAAS,CA4FnE"}
@@ -0,0 +1 @@
1
+ {"version":3,"file":"worker-pool-str.js","sourceRoot":"","sources":["../../src/stream/worker-pool-str.ts"],"names":[],"mappings":"AAAA,+EAA+E;AAC/E,gFAAgF;AAChF,mDAAmD;AACnD,EAAE;AACF,wDAAwD;AACxD,sDAAsD;AACtD,OAAO,EAAE,SAAS,EAA0B,MAAM,QAAQ,CAAC;AAC3D,OAAO,EAAE,MAAM,EAAE,MAAM,qBAAqB,CAAC;AAC7C,OAAO,MAAM,MAAM,QAAQ,CAAC;AAE5B,MAAM,MAAM,GAAG,MAAM,CAAC,SAAS,EAAE,CAAC;AAIlC,MAAM,UAAU,sBAAsB,CAAC,IAAiB;IACpD,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;IACpC,MAAM,GAAG,GAAG,CAAC,GAAG,CAAC,CAAC;IAClB,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,kBAAkB,EAAE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAE/D,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,MAAM,IAAI,GAAa,EAAE,CAAC;IAC1B,MAAM,KAAK,GAAoC,EAAE,CAAC;IAClD,MAAM,OAAO,GAAG,IAAI,GAAG,EAAkB,CAAC;IAE1C,IAAI,QAAQ,GAAG,CAAC,CAAC,CAAK,8BAA8B;IACpD,IAAI,QAAQ,GAAG,CAAC,CAAC,CAAK,8BAA8B;IACpD,IAAI,OAAO,GAAG,CAAC,CAAC,CAAM,6BAA6B;IACnD,IAAI,KAAK,GAAG,CAAC,CAAC,CAAQ,qCAAqC;IAC3D,IAAI,KAAK,GAAG,KAAK,CAAC;IAClB,IAAI,QAAQ,GAAG,CAAC,CAAC;IACjB,IAAI,QAAQ,GAA6B,IAAI,CAAC;IAC9C,IAAI,KAAK,GAA6B,IAAI,CAAC;IAC3C,IAAI,MAAM,GAAiB,IAAI,CAAC;IAEhC,MAAM,MAAM,GAAG,IAAI,SAAS,CAAC;QACzB,kBAAkB,EAAE,IAAI,EAAU,iBAAiB;QACnD,kBAAkB,EAAE,KAAK,EAAS,eAAe;QACjD,SAAS,CAAC,KAAoC,EAAE,IAAI,EAAE,EAAqB;YACvE,IAAI,MAAM,EAAE,CAAC;gBAAC,EAAE,CAAC,MAAM,CAAC,CAAC;gBAAC,OAAO;YAAC,CAAC;YACnC,QAAQ,EAAE,CAAC;YACX,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAClB,QAAQ,EAAE,CAAC;YACX,IAAI,QAAQ,GAAG,GAAG;gBAAE,EAAE,EAAE,CAAC;;gBACpB,QAAQ,GAAG,EAAE,CAAC,CAAW,eAAe;QACjD,CAAC;QACD,KAAK,CAAC,EAAqB;YACvB,IAAI,MAAM,EAAE,CAAC;gBAAC,EAAE,CAAC,MAAM,CAAC,CAAC;gBAAC,OAAO;YAAC,CAAC;YACnC,KAAK,GAAG,IAAI,CAAC;YACb,IAAI,OAAO,KAAK,QAAQ;gBAAE,MAAM,CAAC,EAAE,CAAC,CAAC;;gBAChC,KAAK,GAAG,EAAE,CAAC;QACpB,CAAC;KACJ,CAAC,CAAC;IAEH,SAAS,QAAQ;QACb,OAAO,KAAK,CAAC,MAAM,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YACjC,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,EAAG,CAAC;YACtB,MAAM,GAAG,GAAG,KAAK,CAAC,KAAK,EAAG,CAAC;YAC3B,QAAQ,EAAE,CAAC;YACX,CAAC,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;QACvB,CAAC;IACL,CAAC;IAED,SAAS,QAAQ,CAAC,CAAS,EAAE,GAAQ;QACjC,IAAI,GAAG,CAAC,KAAK,EAAE,CAAC;YAAC,IAAI,CAAC,IAAI,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC;YAAC,OAAO;QAAC,CAAC;QACtD,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,EAAE,GAAG,CAAC,GAAG,CAAC,CAAC;QAC9B,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACb,QAAQ,EAAE,CAAC;QACX,OAAO,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC;YAC3B,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAE,CAAC;YACnC,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;YACzB,QAAQ,EAAE,CAAC;YACX,OAAO,EAAE,CAAC;YACV,IAAI,GAAG,CAAC,MAAM;gBAAE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACrC,CAAC;QACD,KAAK,IAAI,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC;QACpB,QAAQ,EAAE,CAAC;QACX,IAAI,QAAQ,IAAI,QAAQ,GAAG,GAAG,EAAE,CAAC;YAAC,MAAM,CAAC,GAAG,QAAQ,CAAC;YAAC,QAAQ,GAAG,IAAI,CAAC;YAAC,CAAC,EAAE,CAAC;QAAC,CAAC;QAC7E,IAAI,KAAK,IAAI,KAAK,IAAI,OAAO,KAAK,QAAQ,EAAE,CAAC;YAAC,MAAM,CAAC,GAAG,KAAK,CAAC;YAAC,KAAK,GAAG,IAAI,CAAC;YAAC,MAAM,CAAC,CAAC,CAAC,CAAC;QAAC,CAAC;IAC7F,CAAC;IAED,SAAS,IAAI,CAAC,GAAU;QACpB,IAAI,MAAM;YAAE,OAAO;QACnB,MAAM,GAAG,GAAG,CAAC;QACb,KAAK,MAAM,CAAC,IAAI,OAAO;YAAE,CAAC,CAAC,SAAS,EAAE,CAAC;QACvC,IAAI,QAAQ,EAAE,CAAC;YAAC,MAAM,CAAC,GAAG,QAAQ,CAAC;YAAC,QAAQ,GAAG,IAAI,CAAC;YAAC,CAAC,CAAC,GAAG,CAAC,CAAC;QAAC,CAAC;aACzD,IAAI,KAAK,EAAE,CAAC;YAAC,MAAM,CAAC,GAAG,KAAK,CAAC;YAAC,KAAK,GAAG,IAAI,CAAC;YAAC,CAAC,CAAC,GAAG,CAAC,CAAC;QAAC,CAAC;;YACrD,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IAC7B,CAAC;IAED,SAAS,MAAM,CAAC,EAAqB;QACjC,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;IACtF,CAAC;IAED,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QACzB,MAAM,CAAC,GAAG,IAAI,MAAM,CAAC,SAAS,EAAE,EAAE,UAAU,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;QACvG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAChB,CAAC,CAAC,EAAE,CAAC,SAAS,EAAE,CAAC,GAAQ,EAAE,EAAE;YACzB,IAAI,GAAG,IAAI,GAAG,CAAC,KAAK,EAAE,CAAC;gBAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;gBAAC,QAAQ,EAAE,CAAC;YAAC,CAAC;;gBAC9C,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;QAC1B,CAAC,CAAC,CAAC;QACH,CAAC,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;IACtC,CAAC;IAEA,MAAc,CAAC,QAAQ,GAAG,GAAG,EAAE,CAAC,KAAK,CAAC;IACvC,MAAM,CAAC,IAAI,CAAC,mBAAmB,CAAC,yBAAyB,CAAC,CAAC;IAC3D,OAAO,MAAM,CAAC;AAClB,CAAC"}
@@ -0,0 +1,10 @@
1
+ import { Transform } from 'stream';
2
+ export interface WorkerPoolOpts {
3
+ map: string;
4
+ param: any;
5
+ workers: number;
6
+ batchSize?: number;
7
+ timeoutMs?: number;
8
+ }
9
+ export declare function createWorkerPool(opts: WorkerPoolOpts): Transform;
10
+ //# sourceMappingURL=worker-pool.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"worker-pool.d.ts","sourceRoot":"","sources":["../../src/stream/worker-pool.ts"],"names":[],"mappings":"AAgBA,OAAO,EAAE,SAAS,EAA0B,MAAM,QAAQ,CAAC;AAO3D,MAAM,WAAW,cAAc;IAC3B,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,GAAG,CAAC;IACX,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,SAAS,CAAC,EAAE,MAAM,CAAC;CACtB;AAED,wBAAgB,gBAAgB,CAAC,IAAI,EAAE,cAAc,GAAG,SAAS,CAuJhE"}
@@ -0,0 +1,217 @@
1
+ // A Transform that parallelizes a per-record map across worker threads while
2
+ // preserving record order. Batches incoming records, dispatches batches to a
3
+ // pool of workers, re-orders the results by sequence number, and pushes them
4
+ // downstream in order.
5
+ //
6
+ // In (objectMode): record objects
7
+ // Out (objectMode): mapped record objects, in the original order
8
+ // (records the map rejected are dropped)
9
+ //
10
+ // Hardening over the first prototype:
11
+ // - worker count is clamped to the available parallelism;
12
+ // - real downstream backpressure: results are held in an out-queue and only
13
+ // pushed while push() accepts them; dispatch/accept pause when the output
14
+ // is backed up and resume on _read (so a slow sink can't grow memory);
15
+ // - a per-batch timeout fails the pipeline instead of hanging if a worker
16
+ // dies silently.
17
+ import { Transform } from 'stream';
18
+ import { Worker } from 'node:worker_threads';
19
+ import { availableParallelism } from 'node:os';
20
+ import log4js from 'log4js';
21
+ const logger = log4js.getLogger();
22
+ export function createWorkerPool(opts) {
23
+ const maxN = Math.max(1, availableParallelism());
24
+ const N = Math.min(Math.max(1, Math.floor(opts.workers) || 1), maxN);
25
+ if (N < opts.workers)
26
+ logger.warn(`--workers ${opts.workers} clamped to ${N} (available parallelism)`);
27
+ const BATCH = opts.batchSize ?? 64;
28
+ const CAP = N * 2; // max batches in flight
29
+ const OUT_CAP = CAP * BATCH; // max records buffered awaiting downstream
30
+ const TIMEOUT = opts.timeoutMs ?? 120_000;
31
+ const workerUrl = new URL('../worker.js', import.meta.url);
32
+ const workers = [];
33
+ const idle = [];
34
+ const timers = new Map();
35
+ const queue = [];
36
+ const reorder = new Map();
37
+ const outQueue = []; // mapped records ready to push, in order
38
+ let nextDispatch = 0;
39
+ let nextEmit = 0;
40
+ let inflight = 0;
41
+ let curBatch = [];
42
+ let acceptCb = null; // held _transform callback
43
+ let endCb = null;
44
+ let failed = null;
45
+ let closing = false; // we are terminating workers on purpose
46
+ const stream = new Transform({
47
+ objectMode: true,
48
+ transform(rec, _enc, cb) {
49
+ if (failed) {
50
+ cb(failed);
51
+ return;
52
+ }
53
+ curBatch.push(rec);
54
+ if (curBatch.length >= BATCH) {
55
+ enqueue(curBatch);
56
+ curBatch = [];
57
+ }
58
+ if (canAccept())
59
+ cb();
60
+ else
61
+ acceptCb = cb; // pause upstream until there's room
62
+ },
63
+ flush(cb) {
64
+ if (failed) {
65
+ cb(failed);
66
+ return;
67
+ }
68
+ if (curBatch.length) {
69
+ enqueue(curBatch);
70
+ curBatch = [];
71
+ }
72
+ if (done())
73
+ finish(cb);
74
+ else
75
+ endCb = cb;
76
+ }
77
+ });
78
+ // Resume producing when the downstream consumer asks for more.
79
+ const origRead = stream._read.bind(stream);
80
+ stream._read = (size) => { pump(); origRead(size); };
81
+ function canAccept() {
82
+ return inflight < CAP && outQueue.length < OUT_CAP;
83
+ }
84
+ function done() {
85
+ return nextEmit === nextDispatch && queue.length === 0 && outQueue.length === 0;
86
+ }
87
+ function enqueue(batch) {
88
+ queue.push({ seq: nextDispatch++, batch });
89
+ dispatch();
90
+ }
91
+ const dbg = () => logger.isDebugEnabled();
92
+ function dispatch() {
93
+ // don't start new work while the output is backed up
94
+ if (dbg() && queue.length && (!idle.length || outQueue.length >= OUT_CAP)) {
95
+ logger.debug(`backpressure: ${queue.length} batches queued, idle=${idle.length}, outQueue=${outQueue.length}/${OUT_CAP}`);
96
+ }
97
+ while (queue.length && idle.length && outQueue.length < OUT_CAP) {
98
+ const w = idle.pop();
99
+ const job = queue.shift();
100
+ inflight++;
101
+ timers.set(w, setTimeout(() => fail(new Error(`worker timed out after ${TIMEOUT}ms (batch ${job.seq})`)), TIMEOUT));
102
+ w.postMessage(job);
103
+ if (dbg())
104
+ logger.debug(`dispatch batch ${job.seq} (${job.batch.length} recs) -> worker; inflight=${inflight}`);
105
+ }
106
+ }
107
+ // push as many ready records as the downstream will take
108
+ function pump() {
109
+ const before = outQueue.length;
110
+ while (outQueue.length && stream.push(outQueue[0]))
111
+ outQueue.shift();
112
+ if (dbg() && outQueue.length > 0 && outQueue.length === before) {
113
+ logger.debug(`downstream full, holding ${outQueue.length} records`);
114
+ }
115
+ dispatch();
116
+ if (acceptCb && canAccept()) {
117
+ const c = acceptCb;
118
+ acceptCb = null;
119
+ c();
120
+ }
121
+ if (endCb && done()) {
122
+ const c = endCb;
123
+ endCb = null;
124
+ finish(c);
125
+ }
126
+ }
127
+ function onResult(w, msg) {
128
+ const t = timers.get(w);
129
+ if (t) {
130
+ clearTimeout(t);
131
+ timers.delete(w);
132
+ }
133
+ if (msg.error) {
134
+ logger.error(`worker error on batch ${msg.seq}: ${msg.error}`);
135
+ fail(new Error(msg.error));
136
+ return;
137
+ }
138
+ reorder.set(msg.seq, msg.mapped);
139
+ idle.push(w);
140
+ inflight--;
141
+ if (dbg())
142
+ logger.debug(`result batch ${msg.seq} (${msg.mapped.length} mapped); inflight=${inflight}, reorderBuf=${reorder.size}, wantNext=${nextEmit}`);
143
+ // move now-contiguous results into the out-queue (don't push yet)
144
+ while (reorder.has(nextEmit)) {
145
+ const mapped = reorder.get(nextEmit);
146
+ reorder.delete(nextEmit);
147
+ nextEmit++;
148
+ for (const m of mapped)
149
+ if (m !== null)
150
+ outQueue.push(m);
151
+ }
152
+ pump();
153
+ }
154
+ function fail(err) {
155
+ if (failed)
156
+ return;
157
+ failed = err;
158
+ closing = true;
159
+ logger.error(`worker pool failing: ${err.message} (terminating ${workers.length} workers)`);
160
+ for (const t of timers.values())
161
+ clearTimeout(t);
162
+ timers.clear();
163
+ for (const w of workers)
164
+ w.terminate();
165
+ if (acceptCb) {
166
+ const c = acceptCb;
167
+ acceptCb = null;
168
+ c(err);
169
+ }
170
+ else if (endCb) {
171
+ const c = endCb;
172
+ endCb = null;
173
+ c(err);
174
+ }
175
+ else
176
+ stream.destroy(err);
177
+ }
178
+ function finish(cb) {
179
+ closing = true;
180
+ for (const t of timers.values())
181
+ clearTimeout(t);
182
+ timers.clear();
183
+ logger.debug(`map complete: ${nextEmit} batches; terminating ${workers.length} workers`);
184
+ Promise.all(workers.map((w) => w.terminate())).then(() => cb()).catch(() => cb());
185
+ }
186
+ for (let i = 0; i < N; i++) {
187
+ // stdout/stderr: true keeps each worker's stdio OUT of the parent's
188
+ // process.stdout/stderr. By default Node pipes every worker's stdio
189
+ // through to the parent, attaching close/error/finish listeners to the
190
+ // parent WriteStream per worker -> a MaxListenersExceededWarning once
191
+ // --workers crosses ~8. Workers here are pure compute and report results
192
+ // and errors over postMessage, so they never need the parent's console.
193
+ const w = new Worker(workerUrl, {
194
+ workerData: { map: opts.map, param: opts.param },
195
+ stdout: true,
196
+ stderr: true,
197
+ });
198
+ workers.push(w);
199
+ w.on('message', (msg) => {
200
+ if (msg && msg.ready) {
201
+ idle.push(w);
202
+ dispatch();
203
+ if (dbg())
204
+ logger.debug(`worker ${idle.length}/${N} ready`);
205
+ }
206
+ else
207
+ onResult(w, msg);
208
+ });
209
+ w.on('error', (err) => { logger.error(`worker thread crashed: ${err.message}`); fail(err); });
210
+ w.on('exit', (code) => { if (code !== 0 && !failed && !closing)
211
+ fail(new Error(`worker exited unexpectedly (code ${code})`)); });
212
+ }
213
+ logger.info(`map running on ${N} worker threads (batch ${BATCH}, in-flight cap ${CAP}, timeout ${TIMEOUT}ms)`);
214
+ stream.isWorkerPool = true; // lets callers/tests tell a threaded map stage from a serial one
215
+ return stream;
216
+ }
217
+ //# sourceMappingURL=worker-pool.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"worker-pool.js","sourceRoot":"","sources":["../../src/stream/worker-pool.ts"],"names":[],"mappings":"AAAA,6EAA6E;AAC7E,6EAA6E;AAC7E,6EAA6E;AAC7E,uBAAuB;AACvB,EAAE;AACF,mCAAmC;AACnC,iEAAiE;AACjE,2DAA2D;AAC3D,EAAE;AACF,sCAAsC;AACtC,2DAA2D;AAC3D,6EAA6E;AAC7E,6EAA6E;AAC7E,0EAA0E;AAC1E,2EAA2E;AAC3E,oBAAoB;AACpB,OAAO,EAAE,SAAS,EAA0B,MAAM,QAAQ,CAAC;AAC3D,OAAO,EAAE,MAAM,EAAE,MAAM,qBAAqB,CAAC;AAC7C,OAAO,EAAE,oBAAoB,EAAE,MAAM,SAAS,CAAC;AAC/C,OAAO,MAAM,MAAM,QAAQ,CAAC;AAE5B,MAAM,MAAM,GAAG,MAAM,CAAC,SAAS,EAAE,CAAC;AAUlC,MAAM,UAAU,gBAAgB,CAAC,IAAoB;IACjD,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,oBAAoB,EAAE,CAAC,CAAC;IACjD,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;IACrE,IAAI,CAAC,GAAG,IAAI,CAAC,OAAO;QAAE,MAAM,CAAC,IAAI,CAAC,aAAa,IAAI,CAAC,OAAO,eAAe,CAAC,0BAA0B,CAAC,CAAC;IACvG,MAAM,KAAK,GAAG,IAAI,CAAC,SAAS,IAAI,EAAE,CAAC;IACnC,MAAM,GAAG,GAAG,CAAC,GAAG,CAAC,CAAC,CAAiB,wBAAwB;IAC3D,MAAM,OAAO,GAAG,GAAG,GAAG,KAAK,CAAC,CAAO,2CAA2C;IAC9E,MAAM,OAAO,GAAG,IAAI,CAAC,SAAS,IAAI,OAAO,CAAC;IAC1C,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,cAAc,EAAE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAE3D,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,MAAM,IAAI,GAAa,EAAE,CAAC;IAC1B,MAAM,MAAM,GAAG,IAAI,GAAG,EAA0B,CAAC;IACjD,MAAM,KAAK,GAAoC,EAAE,CAAC;IAClD,MAAM,OAAO,GAAG,IAAI,GAAG,EAAiB,CAAC;IACzC,MAAM,QAAQ,GAAU,EAAE,CAAC,CAAQ,yCAAyC;IAE5E,IAAI,YAAY,GAAG,CAAC,CAAC;IACrB,IAAI,QAAQ,GAAG,CAAC,CAAC;IACjB,IAAI,QAAQ,GAAG,CAAC,CAAC;IACjB,IAAI,QAAQ,GAAU,EAAE,CAAC;IACzB,IAAI,QAAQ,GAA6B,IAAI,CAAC,CAAG,2BAA2B;IAC5E,IAAI,KAAK,GAA6B,IAAI,CAAC;IAC3C,IAAI,MAAM,GAAiB,IAAI,CAAC;IAChC,IAAI,OAAO,GAAG,KAAK,CAAC,CAA8B,wCAAwC;IAE1F,MAAM,MAAM,GAAG,IAAI,SAAS,CAAC;QACzB,UAAU,EAAE,IAAI;QAChB,SAAS,CAAC,GAAQ,EAAE,IAAI,EAAE,EAAqB;YAC3C,IAAI,MAAM,EAAE,CAAC;gBAAC,EAAE,CAAC,MAAM,CAAC,CAAC;gBAAC,OAAO;YAAC,CAAC;YACnC,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACnB,IAAI,QAAQ,CAAC,MAAM,IAAI,KAAK,EAAE,CAAC;gBAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;gBAAC,QAAQ,GAAG,EAAE,CAAC;YAAC,CAAC;YACnE,IAAI,SAAS,EAAE;gBAAE,EAAE,EAAE,CAAC;;gBACjB,QAAQ,GAAG,EAAE,CAAC,CAAU,oCAAoC;QACrE,CAAC;QACD,KAAK,CAAC,EAAqB;YACvB,IAAI,MAAM,EAAE,CAAC;gBAAC,EAAE,CAAC,MAAM,CAAC,CAAC;gBAAC,OAAO;YAAC,CAAC;YACnC,IAAI,QAAQ,CAAC,MAAM,EAAE,CAAC;gBAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;gBAAC,QAAQ,GAAG,EAAE,CAAC;YAAC,CAAC;YAC1D,IAAI,IAAI,EAAE;gBAAE,MAAM,CAAC,EAAE,CAAC,CAAC;;gBAClB,KAAK,GAAG,EAAE,CAAC;QACpB,CAAC;KACJ,CAAC,CAAC;IAEH,+DAA+D;IAC/D,MAAM,QAAQ,GAAI,MAAc,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IACnD,MAAc,CAAC,KAAK,GAAG,CAAC,IAAY,EAAE,EAAE,GAAG,IAAI,EAAE,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;IAEtE,SAAS,SAAS;QACd,OAAO,QAAQ,GAAG,GAAG,IAAI,QAAQ,CAAC,MAAM,GAAG,OAAO,CAAC;IACvD,CAAC;IACD,SAAS,IAAI;QACT,OAAO,QAAQ,KAAK,YAAY,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,CAAC;IACpF,CAAC;IAED,SAAS,OAAO,CAAC,KAAY;QACzB,KAAK,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,YAAY,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC;QAC3C,QAAQ,EAAE,CAAC;IACf,CAAC;IAED,MAAM,GAAG,GAAG,GAAG,EAAE,CAAC,MAAM,CAAC,cAAc,EAAE,CAAC;IAE1C,SAAS,QAAQ;QACb,qDAAqD;QACrD,IAAI,GAAG,EAAE,IAAI,KAAK,CAAC,MAAM,IAAI,CAAC,CAAC,IAAI,CAAC,MAAM,IAAI,QAAQ,CAAC,MAAM,IAAI,OAAO,CAAC,EAAE,CAAC;YACxE,MAAM,CAAC,KAAK,CAAC,iBAAiB,KAAK,CAAC,MAAM,yBAAyB,IAAI,CAAC,MAAM,cAAc,QAAQ,CAAC,MAAM,IAAI,OAAO,EAAE,CAAC,CAAC;QAC9H,CAAC;QACD,OAAO,KAAK,CAAC,MAAM,IAAI,IAAI,CAAC,MAAM,IAAI,QAAQ,CAAC,MAAM,GAAG,OAAO,EAAE,CAAC;YAC9D,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,EAAG,CAAC;YACtB,MAAM,GAAG,GAAG,KAAK,CAAC,KAAK,EAAG,CAAC;YAC3B,QAAQ,EAAE,CAAC;YACX,MAAM,CAAC,GAAG,CAAC,CAAC,EAAE,UAAU,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,IAAI,KAAK,CAAC,0BAA0B,OAAO,aAAa,GAAG,CAAC,GAAG,GAAG,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC,CAAC;YACpH,CAAC,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;YACnB,IAAI,GAAG,EAAE;gBAAE,MAAM,CAAC,KAAK,CAAC,kBAAkB,GAAG,CAAC,GAAG,KAAK,GAAG,CAAC,KAAK,CAAC,MAAM,8BAA8B,QAAQ,EAAE,CAAC,CAAC;QACpH,CAAC;IACL,CAAC;IAED,yDAAyD;IACzD,SAAS,IAAI;QACT,MAAM,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAC;QAC/B,OAAO,QAAQ,CAAC,MAAM,IAAI,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;YAAE,QAAQ,CAAC,KAAK,EAAE,CAAC;QACrE,IAAI,GAAG,EAAE,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,IAAI,QAAQ,CAAC,MAAM,KAAK,MAAM,EAAE,CAAC;YAC7D,MAAM,CAAC,KAAK,CAAC,4BAA4B,QAAQ,CAAC,MAAM,UAAU,CAAC,CAAC;QACxE,CAAC;QACD,QAAQ,EAAE,CAAC;QACX,IAAI,QAAQ,IAAI,SAAS,EAAE,EAAE,CAAC;YAAC,MAAM,CAAC,GAAG,QAAQ,CAAC;YAAC,QAAQ,GAAG,IAAI,CAAC;YAAC,CAAC,EAAE,CAAC;QAAC,CAAC;QAC1E,IAAI,KAAK,IAAI,IAAI,EAAE,EAAE,CAAC;YAAC,MAAM,CAAC,GAAG,KAAK,CAAC;YAAC,KAAK,GAAG,IAAI,CAAC;YAAC,MAAM,CAAC,CAAC,CAAC,CAAC;QAAC,CAAC;IACtE,CAAC;IAED,SAAS,QAAQ,CAAC,CAAS,EAAE,GAAQ;QACjC,MAAM,CAAC,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;QACxB,IAAI,CAAC,EAAE,CAAC;YAAC,YAAY,CAAC,CAAC,CAAC,CAAC;YAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;QAAC,CAAC;QAC7C,IAAI,GAAG,CAAC,KAAK,EAAE,CAAC;YAAC,MAAM,CAAC,KAAK,CAAC,yBAAyB,GAAG,CAAC,GAAG,KAAK,GAAG,CAAC,KAAK,EAAE,CAAC,CAAC;YAAC,IAAI,CAAC,IAAI,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC;YAAC,OAAO;QAAC,CAAC;QACtH,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;QACjC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACb,QAAQ,EAAE,CAAC;QACX,IAAI,GAAG,EAAE;YAAE,MAAM,CAAC,KAAK,CAAC,gBAAgB,GAAG,CAAC,GAAG,KAAK,GAAG,CAAC,MAAM,CAAC,MAAM,sBAAsB,QAAQ,gBAAgB,OAAO,CAAC,IAAI,cAAc,QAAQ,EAAE,CAAC,CAAC;QACzJ,kEAAkE;QAClE,OAAO,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC;YAC3B,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAE,CAAC;YACtC,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;YACzB,QAAQ,EAAE,CAAC;YACX,KAAK,MAAM,CAAC,IAAI,MAAM;gBAAE,IAAI,CAAC,KAAK,IAAI;oBAAE,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAC7D,CAAC;QACD,IAAI,EAAE,CAAC;IACX,CAAC;IAED,SAAS,IAAI,CAAC,GAAU;QACpB,IAAI,MAAM;YAAE,OAAO;QACnB,MAAM,GAAG,GAAG,CAAC;QACb,OAAO,GAAG,IAAI,CAAC;QACf,MAAM,CAAC,KAAK,CAAC,wBAAwB,GAAG,CAAC,OAAO,iBAAiB,OAAO,CAAC,MAAM,WAAW,CAAC,CAAC;QAC5F,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,MAAM,EAAE;YAAE,YAAY,CAAC,CAAC,CAAC,CAAC;QACjD,MAAM,CAAC,KAAK,EAAE,CAAC;QACf,KAAK,MAAM,CAAC,IAAI,OAAO;YAAE,CAAC,CAAC,SAAS,EAAE,CAAC;QACvC,IAAI,QAAQ,EAAE,CAAC;YAAC,MAAM,CAAC,GAAG,QAAQ,CAAC;YAAC,QAAQ,GAAG,IAAI,CAAC;YAAC,CAAC,CAAC,GAAG,CAAC,CAAC;QAAC,CAAC;aACzD,IAAI,KAAK,EAAE,CAAC;YAAC,MAAM,CAAC,GAAG,KAAK,CAAC;YAAC,KAAK,GAAG,IAAI,CAAC;YAAC,CAAC,CAAC,GAAG,CAAC,CAAC;QAAC,CAAC;;YACrD,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IAC7B,CAAC;IAED,SAAS,MAAM,CAAC,EAAqB;QACjC,OAAO,GAAG,IAAI,CAAC;QACf,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,MAAM,EAAE;YAAE,YAAY,CAAC,CAAC,CAAC,CAAC;QACjD,MAAM,CAAC,KAAK,EAAE,CAAC;QACf,MAAM,CAAC,KAAK,CAAC,iBAAiB,QAAQ,yBAAyB,OAAO,CAAC,MAAM,UAAU,CAAC,CAAC;QACzF,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;IACtF,CAAC;IAED,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QACzB,oEAAoE;QACpE,oEAAoE;QACpE,uEAAuE;QACvE,sEAAsE;QACtE,yEAAyE;QACzE,wEAAwE;QACxE,MAAM,CAAC,GAAG,IAAI,MAAM,CAAC,SAAS,EAAE;YAC5B,UAAU,EAAE,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE;YAChD,MAAM,EAAE,IAAI;YACZ,MAAM,EAAE,IAAI;SACf,CAAC,CAAC;QACH,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAChB,CAAC,CAAC,EAAE,CAAC,SAAS,EAAE,CAAC,GAAQ,EAAE,EAAE;YACzB,IAAI,GAAG,IAAI,GAAG,CAAC,KAAK,EAAE,CAAC;gBAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;gBAAC,QAAQ,EAAE,CAAC;gBAAC,IAAI,GAAG,EAAE;oBAAE,MAAM,CAAC,KAAK,CAAC,UAAU,IAAI,CAAC,MAAM,IAAI,CAAC,QAAQ,CAAC,CAAC;YAAC,CAAC;;gBAC3G,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;QAC1B,CAAC,CAAC,CAAC;QACH,CAAC,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE,GAAG,MAAM,CAAC,KAAK,CAAC,0BAA0B,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAC9F,CAAC,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,EAAE,GAAG,IAAI,IAAI,KAAK,CAAC,IAAI,CAAC,MAAM,IAAI,CAAC,OAAO;YAAE,IAAI,CAAC,IAAI,KAAK,CAAC,oCAAoC,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACrI,CAAC;IAED,MAAM,CAAC,IAAI,CAAC,kBAAkB,CAAC,0BAA0B,KAAK,mBAAmB,GAAG,aAAa,OAAO,KAAK,CAAC,CAAC;IAC9G,MAAc,CAAC,YAAY,GAAG,IAAI,CAAC,CAAG,iEAAiE;IACxG,OAAO,MAAM,CAAC;AAClB,CAAC"}
@@ -0,0 +1,16 @@
1
+ import { Transform } from 'stream';
2
+ /**
3
+ * Apply a Catmandu Fix script to each record.
4
+ *
5
+ * marcattacks --map fix --param fix=./my.fix ...
6
+ *
7
+ * `fix` may be a path to a Fix file or an inline Fix script. The script is
8
+ * parsed and compiled once; the compiled chain runs per record.
9
+ */
10
+ export declare function createMapper(opts: {
11
+ fix?: string;
12
+ }): Promise<(data: any) => any>;
13
+ export declare function transform(opts: {
14
+ fix?: string;
15
+ }): Promise<Transform>;
16
+ //# sourceMappingURL=fix.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"fix.d.ts","sourceRoot":"","sources":["../../src/transform/fix.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,QAAQ,CAAC;AAOnC;;;;;;;GAOG;AAGH,wBAAsB,YAAY,CAAC,IAAI,EAAE;IAAE,GAAG,CAAC,EAAE,MAAM,CAAA;CAAE,GAAG,OAAO,CAAC,CAAC,IAAI,EAAE,GAAG,KAAK,GAAG,CAAC,CAMtF;AAED,wBAAsB,SAAS,CAAC,IAAI,EAAE;IAAE,GAAG,CAAC,EAAE,MAAM,CAAA;CAAE,GAAG,OAAO,CAAC,SAAS,CAAC,CAoB1E"}
@@ -0,0 +1,46 @@
1
+ import { Transform } from 'stream';
2
+ import fs from 'fs';
3
+ import log4js from 'log4js';
4
+ import { compileFix, REJECT } from '../fix/index.js';
5
+ const logger = log4js.getLogger();
6
+ /**
7
+ * Apply a Catmandu Fix script to each record.
8
+ *
9
+ * marcattacks --map fix --param fix=./my.fix ...
10
+ *
11
+ * `fix` may be a path to a Fix file or an inline Fix script. The script is
12
+ * parsed and compiled once; the compiled chain runs per record.
13
+ */
14
+ // Pure record -> record mapper (returns REJECT for dropped records). Shared by
15
+ // transform() and the worker pool. Compiled once.
16
+ export async function createMapper(opts) {
17
+ let src = opts.fix ?? '';
18
+ if (src && fs.existsSync(src)) {
19
+ src = fs.readFileSync(src, { encoding: 'utf-8' });
20
+ }
21
+ return compileFix(src);
22
+ }
23
+ export async function transform(opts) {
24
+ let src = opts.fix ?? '';
25
+ if (src && fs.existsSync(src)) {
26
+ src = fs.readFileSync(src, { encoding: 'utf-8' });
27
+ }
28
+ const fix = compileFix(src);
29
+ return new Transform({
30
+ objectMode: true,
31
+ transform(data, _encoding, callback) {
32
+ try {
33
+ const out = fix(data);
34
+ if (out === REJECT)
35
+ callback(); // rejected record -> drop it
36
+ else
37
+ callback(null, out);
38
+ }
39
+ catch (err) {
40
+ logger.error('fix error', err.message);
41
+ callback(err);
42
+ }
43
+ }
44
+ });
45
+ }
46
+ //# sourceMappingURL=fix.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"fix.js","sourceRoot":"","sources":["../../src/transform/fix.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,QAAQ,CAAC;AACnC,OAAO,EAAE,MAAM,IAAI,CAAC;AACpB,OAAO,MAAM,MAAM,QAAQ,CAAC;AAC5B,OAAO,EAAE,UAAU,EAAE,MAAM,EAAE,MAAM,iBAAiB,CAAC;AAErD,MAAM,MAAM,GAAG,MAAM,CAAC,SAAS,EAAE,CAAC;AAElC;;;;;;;GAOG;AACH,+EAA+E;AAC/E,kDAAkD;AAClD,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,IAAsB;IACrD,IAAI,GAAG,GAAG,IAAI,CAAC,GAAG,IAAI,EAAE,CAAC;IACzB,IAAI,GAAG,IAAI,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;QAC5B,GAAG,GAAG,EAAE,CAAC,YAAY,CAAC,GAAG,EAAE,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC,CAAC;IACtD,CAAC;IACD,OAAO,UAAU,CAAC,GAAG,CAAC,CAAC;AAC3B,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,IAAsB;IAClD,IAAI,GAAG,GAAG,IAAI,CAAC,GAAG,IAAI,EAAE,CAAC;IACzB,IAAI,GAAG,IAAI,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;QAC5B,GAAG,GAAG,EAAE,CAAC,YAAY,CAAC,GAAG,EAAE,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC,CAAC;IACtD,CAAC;IACD,MAAM,GAAG,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC;IAE5B,OAAO,IAAI,SAAS,CAAC;QACjB,UAAU,EAAE,IAAI;QAChB,SAAS,CAAC,IAAS,EAAE,SAAS,EAAE,QAAQ;YACpC,IAAI,CAAC;gBACD,MAAM,GAAG,GAAG,GAAG,CAAC,IAAI,CAAC,CAAC;gBACtB,IAAI,GAAG,KAAK,MAAM;oBAAE,QAAQ,EAAE,CAAC,CAAG,6BAA6B;;oBAC1D,QAAQ,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;YAC7B,CAAC;YAAC,OAAO,GAAQ,EAAE,CAAC;gBAChB,MAAM,CAAC,KAAK,CAAC,WAAW,EAAE,GAAG,CAAC,OAAO,CAAC,CAAC;gBACvC,QAAQ,CAAC,GAAG,CAAC,CAAC;YAClB,CAAC;QACL,CAAC;KACJ,CAAC,CAAC;AACP,CAAC"}
@@ -1,4 +1,9 @@
1
1
  import { Transform } from "stream";
2
+ export declare const autoParallel = true;
3
+ export declare function createMapper(opts: {
4
+ fix: string;
5
+ lookup: string;
6
+ }): Promise<(data: any) => any>;
2
7
  export declare function transform(opts: {
3
8
  fix: string;
4
9
  lookup: string;
@@ -1 +1 @@
1
- {"version":3,"file":"jsonata.d.ts","sourceRoot":"","sources":["../../src/transform/jsonata.ts"],"names":[],"mappings":"AAAA,OAAe,EAAE,SAAS,EAAE,MAAM,QAAQ,CAAC;AAU3C,wBAAsB,SAAS,CAAC,IAAI,EAAE;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,GAAI,OAAO,CAAC,SAAS,CAAC,CAgE1F"}
1
+ {"version":3,"file":"jsonata.d.ts","sourceRoot":"","sources":["../../src/transform/jsonata.ts"],"names":[],"mappings":"AAAA,OAAe,EAAE,SAAS,EAAE,MAAM,QAAQ,CAAC;AAc3C,eAAO,MAAM,YAAY,OAAO,CAAC;AAKjC,wBAAsB,YAAY,CAAC,IAAI,EAAE;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,GAAI,OAAO,CAAC,CAAC,IAAI,EAAE,GAAG,KAAK,GAAG,CAAC,CA0CtG;AAED,wBAAsB,SAAS,CAAC,IAAI,EAAE;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,GAAI,OAAO,CAAC,SAAS,CAAC,CAe1F"}
@@ -6,58 +6,59 @@ import { parseStream } from '../util/tsv_parse.js';
6
6
  import { v4 as uuidv4 } from 'uuid';
7
7
  import log4js from 'log4js';
8
8
  const logger = log4js.getLogger();
9
- export async function transform(opts) {
10
- let query;
9
+ // JSONata is heavy/interpreted, so the `--workers auto` default should spread it
10
+ // across threads. (Cheap maps like `fix` are parallelizable too but do NOT set
11
+ // this, so auto leaves them single-threaded; an explicit --workers N still
12
+ // threads any parallelizable map.)
13
+ export const autoParallel = true;
14
+ // Build a pure record -> record(Promise) mapper. Shared by the in-process
15
+ // transform() and by the worker pool (so the heavy evaluate() can run on
16
+ // worker threads). Expression + helper functions are compiled once.
17
+ export async function createMapper(opts) {
11
18
  let lookup = {};
12
19
  if (opts.lookup) {
13
20
  lookup = await loadLookup(opts.lookup);
14
21
  }
15
- logger.info(lookup);
22
+ // Resolve the query once, up front, instead of on every record.
23
+ let query;
24
+ if (opts.fix) {
25
+ if (fs.existsSync(opts.fix)) {
26
+ query = fs.readFileSync(opts.fix, { encoding: 'utf-8' });
27
+ }
28
+ else {
29
+ throw Error(`no such file ${opts.fix}`);
30
+ }
31
+ }
32
+ else {
33
+ query = '$';
34
+ }
35
+ logger.debug(query);
36
+ // The identity expression is a pure pass-through: skip jsonata entirely.
37
+ if (query.trim() === '$') {
38
+ return (data) => data;
39
+ }
40
+ // Compile the expression and register helper functions ONCE. The helpers
41
+ // read the record currently being processed via `current`; each evaluate
42
+ // is awaited before the next, so this is safe.
43
+ let current;
44
+ const expression = jsonata(query);
45
+ expression.registerFunction('marcmap', (code) => marcmap(current['record'], code, {}));
46
+ expression.registerFunction('marctag', (row) => marctag(row));
47
+ expression.registerFunction('marcind', (row) => marcind(row));
48
+ expression.registerFunction('marcsubfields', (row, regex) => marcsubfields(row, new RegExp(regex)));
49
+ expression.registerFunction('marcrecord', () => current['record']);
50
+ expression.registerFunction('asmarc', (data) => ({ "record": data }));
51
+ expression.registerFunction('genid', () => genid());
52
+ expression.registerFunction('lookup', (key) => lookup[key]);
53
+ return async (data) => { current = data; return expression.evaluate(data); };
54
+ }
55
+ export async function transform(opts) {
56
+ const mapper = await createMapper(opts);
16
57
  return new Transform({
17
58
  objectMode: true,
18
59
  async transform(data, _encoding, callback) {
19
60
  try {
20
- if (!query) {
21
- if (opts.fix) {
22
- if (fs.existsSync(opts.fix)) {
23
- query = fs.readFileSync(opts.fix, { encoding: 'utf-8' });
24
- }
25
- else {
26
- throw Error(`no such file ${opts.fix}`);
27
- }
28
- }
29
- else {
30
- query = '$';
31
- }
32
- logger.debug(query);
33
- }
34
- const expression = jsonata(query);
35
- expression.registerFunction('marcmap', (code) => {
36
- return marcmap(data['record'], code, {});
37
- });
38
- expression.registerFunction('marctag', (row) => {
39
- return marctag(row);
40
- });
41
- expression.registerFunction('marcind', (row) => {
42
- return marcind(row);
43
- });
44
- expression.registerFunction('marcsubfields', (row, regex) => {
45
- return marcsubfields(row, new RegExp(regex));
46
- });
47
- expression.registerFunction('marcrecord', () => {
48
- return data['record'];
49
- });
50
- expression.registerFunction('asmarc', (data) => {
51
- return { "record": data };
52
- });
53
- expression.registerFunction('genid', () => {
54
- return genid();
55
- });
56
- expression.registerFunction('lookup', (key) => {
57
- return lookup[key];
58
- });
59
- data = await expression.evaluate(data);
60
- callback(null, data);
61
+ callback(null, await mapper(data));
61
62
  }
62
63
  catch (err) {
63
64
  logger.info(err);
@@ -1 +1 @@
1
- {"version":3,"file":"jsonata.js","sourceRoot":"","sources":["../../src/transform/jsonata.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,EAAE,EAAE,SAAS,EAAE,MAAM,QAAQ,CAAC;AAC3C,OAAO,OAAO,MAAM,SAAS,CAAC;AAC9B,OAAO,EAAE,MAAM,IAAI,CAAC;AACpB,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AACzE,OAAO,EAAE,WAAW,EAAE,MAAM,sBAAsB,CAAC;AACnD,OAAO,EAAE,EAAE,IAAI,MAAM,EAAE,MAAM,MAAM,CAAC;AACpC,OAAO,MAAM,MAAM,QAAQ,CAAC;AAE5B,MAAM,MAAM,GAAG,MAAM,CAAC,SAAS,EAAE,CAAC;AAElC,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,IAAqC;IACjE,IAAI,KAAc,CAAC;IAEnB,IAAI,MAAM,GAA2B,EAAE,CAAC;IAExC,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;QACd,MAAM,GAAG,MAAM,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAC3C,CAAC;IAED,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAEpB,OAAO,IAAI,SAAS,CAAC;QACjB,UAAU,EAAE,IAAI;QAChB,KAAK,CAAC,SAAS,CAAC,IAAS,EAAE,SAAyB,EAAE,QAAkC;YACpF,IAAI,CAAC;gBACD,IAAI,CAAC,KAAK,EAAE,CAAC;oBACT,IAAI,IAAI,CAAC,GAAG,EAAE,CAAC;wBACX,IAAI,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;4BAC1B,KAAK,GAAG,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,GAAG,EAAC,EAAE,QAAQ,EAAE,OAAO,EAAC,CAAC,CAAC;wBAC3D,CAAC;6BACI,CAAC;4BACF,MAAM,KAAK,CAAC,gBAAgB,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;wBAC5C,CAAC;oBACL,CAAC;yBACI,CAAC;wBACF,KAAK,GAAG,GAAG,CAAC;oBAChB,CAAC;oBAED,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;gBACxB,CAAC;gBACD,MAAM,UAAU,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC;gBAClC,UAAU,CAAC,gBAAgB,CAAC,SAAS,EAAE,CAAC,IAAY,EAAE,EAAE;oBACpD,OAAO,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAC,IAAI,EAAC,EAAE,CAAC,CAAC;gBAC3C,CAAC,CAAC,CAAC;gBACH,UAAU,CAAC,gBAAgB,CAAC,SAAS,EAAE,CAAC,GAAa,EAAE,EAAE;oBACrD,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC;gBACxB,CAAC,CAAC,CAAC;gBACH,UAAU,CAAC,gBAAgB,CAAC,SAAS,EAAE,CAAC,GAAa,EAAE,EAAE;oBACrD,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC;gBACxB,CAAC,CAAC,CAAC;gBACH,UAAU,CAAC,gBAAgB,CAAC,eAAe,EAAE,CAAC,GAAa,EAAC,KAAa,EAAE,EAAE;oBACzE,OAAO,aAAa,CAAC,GAAG,EAAE,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;gBACjD,CAAC,CAAC,CAAC;gBACH,UAAU,CAAC,gBAAgB,CAAC,YAAY,EAAE,GAAG,EAAE;oBAC3C,OAAO,IAAI,CAAC,QAAQ,CAAC,CAAC;gBAC1B,CAAC,CAAC,CAAC;gBACH,UAAU,CAAC,gBAAgB,CAAC,QAAQ,EAAE,CAAC,IAAgB,EAAE,EAAE;oBACvD,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAC,CAAC;gBAC7B,CAAC,CAAC,CAAC;gBACH,UAAU,CAAC,gBAAgB,CAAC,OAAO,EAAE,GAAG,EAAE;oBACtC,OAAO,KAAK,EAAE,CAAC;gBACnB,CAAC,CAAC,CAAC;gBACH,UAAU,CAAC,gBAAgB,CAAC,QAAQ,EAAE,CAAC,GAAG,EAAE,EAAE;oBAC1C,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC;gBACvB,CAAC,CAAC,CAAC;gBACH,IAAI,GAAG,MAAM,UAAU,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;gBACvC,QAAQ,CAAC,IAAI,EAAC,IAAI,CAAC,CAAC;YACxB,CAAC;YACD,OAAO,GAAQ,EAAE,CAAC;gBACd,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBACjB,QAAQ,CAAC,GAAY,CAAC,CAAC;YAC3B,CAAC;QACL,CAAC;KACJ,CAAC,CAAC;AACP,CAAC;AAED,SAAS,KAAK;IACV,OAAO,SAAS,MAAM,EAAE,EAAE,CAAC;AAC/B,CAAC;AAED,KAAK,UAAU,UAAU,CAAC,IAAY;IAClC,IAAI,MAAM,GAA4B,EAAE,CAAC;IAEzC,MAAM,OAAO,GAAG,MAAM,WAAW,CAAC,EAAE,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC,CAAC;IAE7D,KAAK,MAAM,GAAG,IAAI,OAAO,EAAE,CAAC;QACxB,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;QACrC,IAAI,IAAI,IAAI,IAAI,CAAC,MAAM,IAAI,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC;YACjD,MAAM,CAAC,GAAG,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;YACvB,MAAM,CAAC,GAAG,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;YACvB,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;gBACT,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;YAClB,CAAC;QACL,CAAC;IACL,CAAC;IAED,OAAO,MAAM,CAAC;AAClB,CAAC"}
1
+ {"version":3,"file":"jsonata.js","sourceRoot":"","sources":["../../src/transform/jsonata.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,EAAE,EAAE,SAAS,EAAE,MAAM,QAAQ,CAAC;AAC3C,OAAO,OAAO,MAAM,SAAS,CAAC;AAC9B,OAAO,EAAE,MAAM,IAAI,CAAC;AACpB,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AACzE,OAAO,EAAE,WAAW,EAAE,MAAM,sBAAsB,CAAC;AACnD,OAAO,EAAE,EAAE,IAAI,MAAM,EAAE,MAAM,MAAM,CAAC;AACpC,OAAO,MAAM,MAAM,QAAQ,CAAC;AAE5B,MAAM,MAAM,GAAG,MAAM,CAAC,SAAS,EAAE,CAAC;AAElC,iFAAiF;AACjF,+EAA+E;AAC/E,2EAA2E;AAC3E,mCAAmC;AACnC,MAAM,CAAC,MAAM,YAAY,GAAG,IAAI,CAAC;AAEjC,0EAA0E;AAC1E,yEAAyE;AACzE,oEAAoE;AACpE,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,IAAqC;IACpE,IAAI,MAAM,GAA2B,EAAE,CAAC;IAExC,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;QACd,MAAM,GAAG,MAAM,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAC3C,CAAC;IAED,gEAAgE;IAChE,IAAI,KAAc,CAAC;IACnB,IAAI,IAAI,CAAC,GAAG,EAAE,CAAC;QACX,IAAI,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;YAC1B,KAAK,GAAG,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,GAAG,EAAC,EAAE,QAAQ,EAAE,OAAO,EAAC,CAAC,CAAC;QAC3D,CAAC;aACI,CAAC;YACF,MAAM,KAAK,CAAC,gBAAgB,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;QAC5C,CAAC;IACL,CAAC;SACI,CAAC;QACF,KAAK,GAAG,GAAG,CAAC;IAChB,CAAC;IACD,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IAEpB,yEAAyE;IACzE,IAAI,KAAK,CAAC,IAAI,EAAE,KAAK,GAAG,EAAE,CAAC;QACvB,OAAO,CAAC,IAAS,EAAE,EAAE,CAAC,IAAI,CAAC;IAC/B,CAAC;IAED,yEAAyE;IACzE,yEAAyE;IACzE,+CAA+C;IAC/C,IAAI,OAAY,CAAC;IACjB,MAAM,UAAU,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC;IAClC,UAAU,CAAC,gBAAgB,CAAC,SAAS,EAAE,CAAC,IAAY,EAAE,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,IAAI,EAAE,EAAE,CAAC,CAAC,CAAC;IAC/F,UAAU,CAAC,gBAAgB,CAAC,SAAS,EAAE,CAAC,GAAa,EAAE,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC;IACxE,UAAU,CAAC,gBAAgB,CAAC,SAAS,EAAE,CAAC,GAAa,EAAE,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC;IACxE,UAAU,CAAC,gBAAgB,CAAC,eAAe,EAAE,CAAC,GAAa,EAAE,KAAa,EAAE,EAAE,CAAC,aAAa,CAAC,GAAG,EAAE,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IACtH,UAAU,CAAC,gBAAgB,CAAC,YAAY,EAAE,GAAG,EAAE,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC;IACnE,UAAU,CAAC,gBAAgB,CAAC,QAAQ,EAAE,CAAC,IAAgB,EAAE,EAAE,CAAC,CAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;IAClF,UAAU,CAAC,gBAAgB,CAAC,OAAO,EAAE,GAAG,EAAE,CAAC,KAAK,EAAE,CAAC,CAAC;IACpD,UAAU,CAAC,gBAAgB,CAAC,QAAQ,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;IAE5D,OAAO,KAAK,EAAE,IAAS,EAAE,EAAE,GAAG,OAAO,GAAG,IAAI,CAAC,CAAC,OAAO,UAAU,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;AACtF,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,IAAqC;IACjE,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC,IAAI,CAAC,CAAC;IAExC,OAAO,IAAI,SAAS,CAAC;QACjB,UAAU,EAAE,IAAI;QAChB,KAAK,CAAC,SAAS,CAAC,IAAS,EAAE,SAAyB,EAAE,QAAkC;YACpF,IAAI,CAAC;gBACD,QAAQ,CAAC,IAAI,EAAE,MAAM,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC;YACvC,CAAC;YACD,OAAO,GAAQ,EAAE,CAAC;gBACd,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBACjB,QAAQ,CAAC,GAAY,CAAC,CAAC;YAC3B,CAAC;QACL,CAAC;KACJ,CAAC,CAAC;AACP,CAAC;AAED,SAAS,KAAK;IACV,OAAO,SAAS,MAAM,EAAE,EAAE,CAAC;AAC/B,CAAC;AAED,KAAK,UAAU,UAAU,CAAC,IAAY;IAClC,IAAI,MAAM,GAA4B,EAAE,CAAC;IAEzC,MAAM,OAAO,GAAG,MAAM,WAAW,CAAC,EAAE,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC,CAAC;IAE7D,KAAK,MAAM,GAAG,IAAI,OAAO,EAAE,CAAC;QACxB,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;QACrC,IAAI,IAAI,IAAI,IAAI,CAAC,MAAM,IAAI,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC;YACjD,MAAM,CAAC,GAAG,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;YACvB,MAAM,CAAC,GAAG,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;YACvB,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;gBACT,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;YAClB,CAAC;QACL,CAAC;IACL,CAAC;IAED,OAAO,MAAM,CAAC;AAClB,CAAC"}
@@ -1 +1 @@
1
- {"version":3,"file":"marcids.d.ts","sourceRoot":"","sources":["../../src/transform/marcids.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,QAAQ,CAAC;AAGnC;;GAEG;AACH,wBAAsB,SAAS,CAAC,KAAK,EAAE,GAAG,GAAI,OAAO,CAAC,SAAS,CAAC,CAkB/D"}
1
+ {"version":3,"file":"marcids.d.ts","sourceRoot":"","sources":["../../src/transform/marcids.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,QAAQ,CAAC;AAGnC;;GAEG;AACH,wBAAsB,SAAS,CAAC,KAAK,EAAE,GAAG,GAAI,OAAO,CAAC,SAAS,CAAC,CAqB/D"}
@@ -16,6 +16,9 @@ export async function transform(_opts) {
16
16
  if (id.length == 1) {
17
17
  callback(null, { id: id[0] });
18
18
  }
19
+ else {
20
+ callback(null);
21
+ }
19
22
  }
20
23
  });
21
24
  }
@@ -1 +1 @@
1
- {"version":3,"file":"marcids.js","sourceRoot":"","sources":["../../src/transform/marcids.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,QAAQ,CAAC;AACnC,OAAO,EAAE,OAAO,EAAE,MAAM,eAAe,CAAC;AAExC;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,KAAU;IACtC,OAAO,IAAI,SAAS,CAAC;QACjB,UAAU,EAAE,IAAI;QAChB,SAAS,CAAC,IAAS,EAAE,SAAS,EAAE,QAAQ;YACpC,IAAI,GAAG,GAAgB,IAAI,CAAC,QAAQ,CAAC,CAAC;YAEtC,IAAI,CAAC,GAAG,EAAE,CAAC;gBACP,QAAQ,CAAC,IAAI,CAAC,CAAC;gBACf,OAAO;YACX,CAAC;YAED,IAAI,EAAE,GAAG,OAAO,CAAC,GAAG,EAAC,KAAK,EAAC,EAAE,CAAC,CAAC;YAE/B,IAAI,EAAE,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;gBACjB,QAAQ,CAAC,IAAI,EAAC,EAAC,EAAE,EAAE,EAAE,CAAC,CAAC,CAAC,EAAC,CAAC,CAAC;YAC/B,CAAC;QACL,CAAC;KACJ,CAAC,CAAC;AACP,CAAC"}
1
+ {"version":3,"file":"marcids.js","sourceRoot":"","sources":["../../src/transform/marcids.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,QAAQ,CAAC;AACnC,OAAO,EAAE,OAAO,EAAE,MAAM,eAAe,CAAC;AAExC;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,KAAU;IACtC,OAAO,IAAI,SAAS,CAAC;QACjB,UAAU,EAAE,IAAI;QAChB,SAAS,CAAC,IAAS,EAAE,SAAS,EAAE,QAAQ;YACpC,IAAI,GAAG,GAAgB,IAAI,CAAC,QAAQ,CAAC,CAAC;YAEtC,IAAI,CAAC,GAAG,EAAE,CAAC;gBACP,QAAQ,CAAC,IAAI,CAAC,CAAC;gBACf,OAAO;YACX,CAAC;YAED,IAAI,EAAE,GAAG,OAAO,CAAC,GAAG,EAAC,KAAK,EAAC,EAAE,CAAC,CAAC;YAE/B,IAAI,EAAE,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;gBACjB,QAAQ,CAAC,IAAI,EAAC,EAAC,EAAE,EAAE,EAAE,CAAC,CAAC,CAAC,EAAC,CAAC,CAAC;YAC/B,CAAC;iBACI,CAAC;gBACF,QAAQ,CAAC,IAAI,CAAC,CAAC;YACnB,CAAC;QACL,CAAC;KACJ,CAAC,CAAC;AACP,CAAC"}
@@ -0,0 +1,19 @@
1
+ /**
2
+ * Marker for record objects whose string values are already free of control
3
+ * characters (\x00-\x1F and \x7F).
4
+ *
5
+ * Input readers that strip control characters while parsing (e.g. the XML
6
+ * readers) set this on the records they emit. Output writers that would
7
+ * otherwise re-strip every value (e.g. alephseq) can then skip that work.
8
+ *
9
+ * It is a Symbol so it never shows up in `JSON.stringify`, `Object.keys` or
10
+ * `for...in` — outputs that serialise the whole record object (json, jsonl,
11
+ * rdf) are unaffected. A transform that builds a new record object (e.g. a
12
+ * JSONata `fix`) naturally drops the marker, so escaping is re-applied unless
13
+ * cleanliness is re-asserted — a safe default.
14
+ */
15
+ export declare const CLEAN: unique symbol;
16
+ export type CleanFlag = {
17
+ [CLEAN]?: boolean;
18
+ };
19
+ //# sourceMappingURL=marc_record.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"marc_record.d.ts","sourceRoot":"","sources":["../../src/util/marc_record.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AACH,eAAO,MAAM,KAAK,EAAE,OAAO,MAAoC,CAAC;AAEhE,MAAM,MAAM,SAAS,GAAG;IAAE,CAAC,KAAK,CAAC,CAAC,EAAE,OAAO,CAAA;CAAE,CAAC"}
@@ -0,0 +1,16 @@
1
+ /**
2
+ * Marker for record objects whose string values are already free of control
3
+ * characters (\x00-\x1F and \x7F).
4
+ *
5
+ * Input readers that strip control characters while parsing (e.g. the XML
6
+ * readers) set this on the records they emit. Output writers that would
7
+ * otherwise re-strip every value (e.g. alephseq) can then skip that work.
8
+ *
9
+ * It is a Symbol so it never shows up in `JSON.stringify`, `Object.keys` or
10
+ * `for...in` — outputs that serialise the whole record object (json, jsonl,
11
+ * rdf) are unaffected. A transform that builds a new record object (e.g. a
12
+ * JSONata `fix`) naturally drops the marker, so escaping is re-applied unless
13
+ * cleanliness is re-asserted — a safe default.
14
+ */
15
+ export const CLEAN = Symbol('marc.record.clean');
16
+ //# sourceMappingURL=marc_record.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"marc_record.js","sourceRoot":"","sources":["../../src/util/marc_record.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AACH,MAAM,CAAC,MAAM,KAAK,GAAkB,MAAM,CAAC,mBAAmB,CAAC,CAAC"}
@@ -0,0 +1 @@
1
+ {"version":3,"file":"worker-str.d.ts","sourceRoot":"","sources":["../src/worker-str.ts"],"names":[],"mappings":""}
@@ -0,0 +1 @@
1
+ {"version":3,"file":"worker-str.js","sourceRoot":"","sources":["../src/worker-str.ts"],"names":[],"mappings":"AAAA,gFAAgF;AAChF,8EAA8E;AAC9E,4EAA4E;AAC5E,EAAE;AACF,+EAA+E;AAC/E,YAAY;AACZ,0EAA0E;AAC1E,qCAAqC;AACrC,mEAAmE;AACnE,OAAO,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AAC7D,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AACvC,OAAO,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AAChD,OAAO,EAAE,MAAM,EAAE,MAAM,iBAAiB,CAAC;AAEzC,MAAM,EAAE,IAAI,EAAE,GAAG,EAAE,KAAK,EAAE,GAAG,UAAwD,CAAC;AAEtF,MAAM,QAAQ,GAAG,MAAM,UAAU,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;AACjD,MAAM,MAAM,GAAuB,GAAG;IAClC,CAAC,CAAC,MAAM,CAAC,MAAM,UAAU,CAAC,GAAG,EAAE,WAAW,CAAC,CAAC,CAAC,YAAY,CAAC,KAAK,IAAI,EAAE,CAAC;IACtE,CAAC,CAAC,CAAC,CAAM,EAAE,EAAE,CAAC,CAAC,CAAC;AAEpB,6EAA6E;AAC7E,KAAK,UAAU,KAAK,CAAC,IAAY;IAC7B,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,SAAS,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC;IAClD,MAAM,OAAO,GAAU,EAAE,CAAC;IAC1B,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QACxC,GAAG,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,CAAM,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;QAC5C,GAAG,CAAC,EAAE,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;QACvB,GAAG,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;QACxB,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAClC,CAAC,CAAC,CAAC;IACH,OAAO,OAAO,CAAC;AACnB,CAAC;AAED,UAAW,CAAC,WAAW,CAAC,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;AAEzC,UAAW,CAAC,EAAE,CAAC,SAAS,EAAE,KAAK,EAAE,GAAkC,EAAE,EAAE;IACnE,IAAI,CAAC;QACD,MAAM,OAAO,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACtC,IAAI,GAAG,GAAG,EAAE,CAAC;QACb,IAAI,CAAC,GAAG,CAAC,CAAC;QACV,KAAK,MAAM,GAAG,IAAI,OAAO,EAAE,CAAC;YACxB,MAAM,CAAC,GAAG,MAAM,MAAM,CAAC,GAAG,CAAC,CAAC;YAC5B,IAAI,CAAC,KAAK,MAAM,IAAI,CAAC,KAAK,SAAS,IAAI,CAAC,KAAK,IAAI;gBAAE,SAAS;YAC5D,IAAI,OAAO,CAAC,KAAK,QAAQ,IAAI,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,MAAM,KAAK,CAAC;gBAAE,SAAS,CAAC,uBAAuB;YAC3F,GAAG,IAAI,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC;YAChC,CAAC,EAAE,CAAC;QACR,CAAC;QACD,UAAW,CAAC,WAAW,CAAC,EAAE,GAAG,EAAE,GAAG,CAAC,GAAG,EAAE,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC;IACtD,CAAC;IAAC,OAAO,GAAQ,EAAE,CAAC;QAChB,UAAW,CAAC,WAAW,CAAC,EAAE,GAAG,EAAE,GAAG,CAAC,GAAG,EAAE,KAAK,EAAE,GAAG,EAAE,OAAO,IAAI,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IAClF,CAAC;AACL,CAAC,CAAC,CAAC"}
@@ -0,0 +1,2 @@
1
+ export {};
2
+ //# sourceMappingURL=worker.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"worker.d.ts","sourceRoot":"","sources":["../src/worker.ts"],"names":[],"mappings":""}
package/dist/worker.js ADDED
@@ -0,0 +1,33 @@
1
+ // Worker thread: runs a parallelizable map (e.g. jsonata, fix) on batches of
2
+ // records. The heavy per-record work (jsonata evaluate / fix chain) runs here;
3
+ // the main thread does I/O, parsing and serialization.
4
+ //
5
+ // Protocol:
6
+ // main -> worker : { seq, batch: record[] }
7
+ // worker -> main : { ready: true } (once, on startup)
8
+ // { seq, mapped: (record|null)[] } (null = rejected)
9
+ // { seq, error: string }
10
+ import { parentPort, workerData } from 'node:worker_threads';
11
+ import { loadPlugin } from './plugin-loader.js';
12
+ import { REJECT } from './fix/signal.js';
13
+ const { map, param } = workerData;
14
+ const mod = await loadPlugin(map, 'transform');
15
+ if (typeof mod.createMapper !== 'function') {
16
+ throw new Error(`map '${map}' is not parallelizable (no createMapper)`);
17
+ }
18
+ const mapper = await mod.createMapper(param ?? {});
19
+ parentPort.postMessage({ ready: true });
20
+ parentPort.on('message', async (msg) => {
21
+ try {
22
+ const mapped = [];
23
+ for (const rec of msg.batch) {
24
+ const r = await mapper(rec);
25
+ mapped.push(r === REJECT ? null : r); // null marks a dropped record
26
+ }
27
+ parentPort.postMessage({ seq: msg.seq, mapped });
28
+ }
29
+ catch (err) {
30
+ parentPort.postMessage({ seq: msg.seq, error: err?.message ?? String(err) });
31
+ }
32
+ });
33
+ //# sourceMappingURL=worker.js.map