marcattacks 2.5.0 → 2.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (100) hide show
  1. package/README.md +82 -2
  2. package/dist/attacker.d.ts.map +1 -1
  3. package/dist/attacker.js +10 -0
  4. package/dist/attacker.js.map +1 -1
  5. package/dist/command.js +1 -0
  6. package/dist/command.js.map +1 -1
  7. package/dist/fix/binds.d.ts +5 -0
  8. package/dist/fix/binds.d.ts.map +1 -0
  9. package/dist/fix/binds.js +82 -0
  10. package/dist/fix/binds.js.map +1 -0
  11. package/dist/fix/conditions.d.ts +5 -0
  12. package/dist/fix/conditions.d.ts.map +1 -0
  13. package/dist/fix/conditions.js +111 -0
  14. package/dist/fix/conditions.js.map +1 -0
  15. package/dist/fix/fixes.d.ts +7 -0
  16. package/dist/fix/fixes.d.ts.map +1 -0
  17. package/dist/fix/fixes.js +478 -0
  18. package/dist/fix/fixes.js.map +1 -0
  19. package/dist/fix/index.d.ts +12 -0
  20. package/dist/fix/index.d.ts.map +1 -0
  21. package/dist/fix/index.js +40 -0
  22. package/dist/fix/index.js.map +1 -0
  23. package/dist/fix/parser.d.ts +25 -0
  24. package/dist/fix/parser.d.ts.map +1 -0
  25. package/dist/fix/parser.js +156 -0
  26. package/dist/fix/parser.js.map +1 -0
  27. package/dist/fix/path.d.ts +23 -0
  28. package/dist/fix/path.d.ts.map +1 -0
  29. package/dist/fix/path.js +351 -0
  30. package/dist/fix/path.js.map +1 -0
  31. package/dist/fix/signal.d.ts +2 -0
  32. package/dist/fix/signal.d.ts.map +1 -0
  33. package/dist/fix/signal.js +5 -0
  34. package/dist/fix/signal.js.map +1 -0
  35. package/dist/fix/util.d.ts +6 -0
  36. package/dist/fix/util.d.ts.map +1 -0
  37. package/dist/fix/util.js +133 -0
  38. package/dist/fix/util.js.map +1 -0
  39. package/dist/input/alephseq.d.ts.map +1 -1
  40. package/dist/input/alephseq.js +18 -5
  41. package/dist/input/alephseq.js.map +1 -1
  42. package/dist/input/fastxml.d.ts +3 -0
  43. package/dist/input/fastxml.d.ts.map +1 -0
  44. package/dist/input/fastxml.js +134 -0
  45. package/dist/input/fastxml.js.map +1 -0
  46. package/dist/input/xml.d.ts.map +1 -1
  47. package/dist/input/xml.js +4 -2
  48. package/dist/input/xml.js.map +1 -1
  49. package/dist/marcmap.d.ts.map +1 -1
  50. package/dist/marcmap.js +15 -3
  51. package/dist/marcmap.js.map +1 -1
  52. package/dist/output/alephseq.d.ts.map +1 -1
  53. package/dist/output/alephseq.js +29 -5
  54. package/dist/output/alephseq.js.map +1 -1
  55. package/dist/output/null.d.ts +3 -0
  56. package/dist/output/null.d.ts.map +1 -0
  57. package/dist/output/null.js +13 -0
  58. package/dist/output/null.js.map +1 -0
  59. package/dist/output/xml.d.ts.map +1 -1
  60. package/dist/output/xml.js +27 -0
  61. package/dist/output/xml.js.map +1 -1
  62. package/dist/stream/framer.d.ts.map +1 -0
  63. package/dist/stream/framer.js.map +1 -0
  64. package/dist/stream/worker-pool-str.d.ts.map +1 -0
  65. package/dist/stream/worker-pool-str.js.map +1 -0
  66. package/dist/stream/worker-pool.d.ts +10 -0
  67. package/dist/stream/worker-pool.d.ts.map +1 -0
  68. package/dist/stream/worker-pool.js +206 -0
  69. package/dist/stream/worker-pool.js.map +1 -0
  70. package/dist/transform/fix.d.ts +16 -0
  71. package/dist/transform/fix.d.ts.map +1 -0
  72. package/dist/transform/fix.js +46 -0
  73. package/dist/transform/fix.js.map +1 -0
  74. package/dist/transform/jsonata.d.ts +4 -0
  75. package/dist/transform/jsonata.d.ts.map +1 -1
  76. package/dist/transform/jsonata.js +40 -44
  77. package/dist/transform/jsonata.js.map +1 -1
  78. package/dist/transform/marcinrdf.d.ts +1 -1
  79. package/dist/transform/marcinrdf.d.ts.map +1 -1
  80. package/dist/transform/marcinrdf.js +16 -3
  81. package/dist/transform/marcinrdf.js.map +1 -1
  82. package/dist/util/marc_record.d.ts +19 -0
  83. package/dist/util/marc_record.d.ts.map +1 -0
  84. package/dist/util/marc_record.js +16 -0
  85. package/dist/util/marc_record.js.map +1 -0
  86. package/dist/worker-str.d.ts.map +1 -0
  87. package/dist/worker-str.js.map +1 -0
  88. package/dist/worker.d.ts +2 -0
  89. package/dist/worker.d.ts.map +1 -0
  90. package/dist/worker.js +33 -0
  91. package/dist/worker.js.map +1 -0
  92. package/package.json +4 -1
  93. package/dist/output/mulitipart.d.ts +0 -7
  94. package/dist/output/mulitipart.d.ts.map +0 -1
  95. package/dist/output/mulitipart.js +0 -53
  96. package/dist/output/mulitipart.js.map +0 -1
  97. package/dist/output/text.d.ts +0 -6
  98. package/dist/output/text.d.ts.map +0 -1
  99. package/dist/output/text.js +0 -34
  100. package/dist/output/text.js.map +0 -1
@@ -1 +1 @@
1
- {"version":3,"file":"xml.js","sourceRoot":"","sources":["../../src/output/xml.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,QAAQ,CAAC;AACnC,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,aAAa,EAAG,cAAc,EAAC,MAAM,eAAe,CAAC;AAChF,OAAO,MAAM,MAAM,QAAQ,CAAC;AAE5B,MAAM,MAAM,GAAG,MAAM,CAAC,SAAS,EAAE,CAAC;AAElC,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,KAAS;IACrC,IAAI,OAAO,GAAG,IAAI,CAAC;IAEnB,OAAO,IAAI,SAAS,CAAC;QACjB,UAAU,EAAE,IAAI;QAChB,SAAS,CAAC,IAAS,EAAE,SAAS,EAAE,QAAQ;YACpC,IAAI,GAAG,GAAgB,IAAI,CAAC,QAAQ,CAAC,CAAC;YAEtC,IAAI,CAAC,GAAG,EAAE,CAAC;gBACP,MAAM,CAAC,KAAK,CAAC,sBAAsB,CAAC,CAAC;gBACrC,QAAQ,EAAE,CAAA;gBACV,OAAO;YACX,CAAC;YAED,IAAI,MAAM,GAAG,EAAE,CAAC;YAEhB,IAAI,OAAO,EAAE,CAAC;gBACV,MAAM,IAAI,8CAA8C,CAAC;gBACzD,MAAM,IAAI,mEAAmE,CAAC;gBAC9E,OAAO,GAAG,KAAK,CAAC;YACpB,CAAC;YAED,MAAM,IAAI,kBAAkB,CAAC;YAE7B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAG,CAAC,GAAG,GAAG,CAAC,MAAM,EAAG,CAAC,EAAE,EAAE,CAAC;gBACpC,IAAI,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;gBAC1B,IAAI,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;gBAC1B,IAAI,GAAG,KAAK,KAAK,EAAE,CAAC,CAAA,CAAC;qBAChB,IAAI,GAAG,KAAK,KAAK,EAAE,CAAC;oBACrB,IAAI,KAAK,GAAG,aAAa,CAAC,GAAG,CAAC,CAAC,CAAE,EAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;oBAC3C,MAAM,IAAI,kBAAkB,SAAS,CAAC,KAAK,CAAC,kBAAkB,CAAC;gBACnE,CAAC;qBACI,IAAI,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC;oBACxB,IAAI,KAAK,GAAG,aAAa,CAAC,GAAG,CAAC,CAAC,CAAE,EAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;oBAC3C,MAAM,IAAI,6BAA6B,SAAS,CAAC,GAAG,EAAC,EAAC,YAAY,EAAC,IAAI,EAAC,CAAC,KAAK,SAAS,CAAC,KAAK,CAAC,wBAAwB,CAAC;gBAC3H,CAAC;qBACI,CAAC;oBACF,MAAM,IAAI,0BAA0B,SAAS,CAAC,GAAG,CAAC,WAAW,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAC,EAAC,YAAY,EAAC,IAAI,EAAC,CAAC,WAAW,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAC,EAAC,YAAY,EAAC,IAAI,EAAC,CAAC,MAAM,CAAC;oBACzJ,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,IAAI,EAAC,KAAK,EAAE,EAAE;wBAClC,MAAM,IAAI,4BAA4B,SAAS,CAAC,IAAI,CAAC,KAAK,SAAS,CAAC,KAAK,CAAC,oBAAoB,CAAC;oBACnG,CAAC,CAAC,CAAC;oBACH,MAAM,IAAI,uBAAuB,CAAC;gBACtC,CAAC;YACL,CAAC;YAED,MAAM,IAAI,mBAAmB,CAAC;YAE9B,MAAM,CAAC,KAAK,CAAC,UAAU,MAAM,CAAC,MAAM,QAAQ,CAAC,CAAC;YAE9C,QAAQ,CAAC,IAAI,EAAC,MAAM,CAAC,CAAC;QAC1B,CAAC;QACD,KAAK,CAAC,QAAQ;YACV,MAAM,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC;YAC9B,IAAI,CAAC,OAAO,EAAE,CAAC;gBACX,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;gBACzB,IAAI,MAAM,GAAG,sBAAsB,CAAC;gBACpC,MAAM,CAAC,KAAK,CAAC,UAAU,MAAM,CAAC,MAAM,QAAQ,CAAC,CAAC;gBAC9C,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACtB,CAAC;YACD,QAAQ,EAAE,CAAC;QACf,CAAC;KACJ,CAAC,CAAC;AACP,CAAC;AAED,MAAM,UAAU,SAAS,CACvB,KAAyC,EACzC,OAAoC;IAElC,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,KAAK,SAAS;QAAE,OAAO,EAAE,CAAC;IAErD,IAAI,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;IAEtB,qEAAqE;IAErE,6EAA6E;IAC7E,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,qCAAqC,EAAE,QAAQ,CAAC,CAAC,CAAE,2BAA2B;IAC5F,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,sCAAsC,EAAE,QAAQ,CAAC,CAAC,CAAC,0BAA0B;IAE3F,8CAA8C;IAC9C,6DAA6D;IAC7D,yDAAyD;IACzD,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,oEAAoE,EAAE,EAAE,CAAC,CAAC;IAExF,+DAA+D;IAC/D,4DAA4D;IAC5D,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,kBAAkB,EAAE,EAAE,CAAC,CAAC;IAEtC,wCAAwC;IACxC,4DAA4D;IAC5D,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAE7B,kCAAkC;IAClC,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;IAC5B,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;IAE5B,IAAI,OAAO,EAAE,YAAY,EAAE,CAAC;QACxB,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;QAC9B,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;IAClC,CAAC;IAED,OAAO,CAAC,CAAC;AACb,CAAC"}
1
+ {"version":3,"file":"xml.js","sourceRoot":"","sources":["../../src/output/xml.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,QAAQ,CAAC;AACnC,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,aAAa,EAAG,cAAc,EAAC,MAAM,eAAe,CAAC;AAChF,OAAO,MAAM,MAAM,QAAQ,CAAC;AAE5B,MAAM,MAAM,GAAG,MAAM,CAAC,SAAS,EAAE,CAAC;AAElC,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,KAAS;IACrC,IAAI,OAAO,GAAG,IAAI,CAAC;IAEnB,OAAO,IAAI,SAAS,CAAC;QACjB,UAAU,EAAE,IAAI;QAChB,SAAS,CAAC,IAAS,EAAE,SAAS,EAAE,QAAQ;YACpC,IAAI,GAAG,GAAgB,IAAI,CAAC,QAAQ,CAAC,CAAC;YAEtC,IAAI,CAAC,GAAG,EAAE,CAAC;gBACP,MAAM,CAAC,KAAK,CAAC,sBAAsB,CAAC,CAAC;gBACrC,QAAQ,EAAE,CAAA;gBACV,OAAO;YACX,CAAC;YAED,IAAI,MAAM,GAAG,EAAE,CAAC;YAEhB,IAAI,OAAO,EAAE,CAAC;gBACV,MAAM,IAAI,8CAA8C,CAAC;gBACzD,MAAM,IAAI,mEAAmE,CAAC;gBAC9E,OAAO,GAAG,KAAK,CAAC;YACpB,CAAC;YAED,MAAM,IAAI,kBAAkB,CAAC;YAE7B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAG,CAAC,GAAG,GAAG,CAAC,MAAM,EAAG,CAAC,EAAE,EAAE,CAAC;gBACpC,IAAI,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;gBAC1B,IAAI,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;gBAC1B,IAAI,GAAG,KAAK,KAAK,EAAE,CAAC,CAAA,CAAC;qBAChB,IAAI,GAAG,KAAK,KAAK,EAAE,CAAC;oBACrB,IAAI,KAAK,GAAG,aAAa,CAAC,GAAG,CAAC,CAAC,CAAE,EAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;oBAC3C,MAAM,IAAI,kBAAkB,SAAS,CAAC,KAAK,CAAC,kBAAkB,CAAC;gBACnE,CAAC;qBACI,IAAI,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC;oBACxB,IAAI,KAAK,GAAG,aAAa,CAAC,GAAG,CAAC,CAAC,CAAE,EAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;oBAC3C,MAAM,IAAI,6BAA6B,SAAS,CAAC,GAAG,EAAC,EAAC,YAAY,EAAC,IAAI,EAAC,CAAC,KAAK,SAAS,CAAC,KAAK,CAAC,wBAAwB,CAAC;gBAC3H,CAAC;qBACI,CAAC;oBACF,MAAM,IAAI,0BAA0B,SAAS,CAAC,GAAG,CAAC,WAAW,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAC,EAAC,YAAY,EAAC,IAAI,EAAC,CAAC,WAAW,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAC,EAAC,YAAY,EAAC,IAAI,EAAC,CAAC,MAAM,CAAC;oBACzJ,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,IAAI,EAAC,KAAK,EAAE,EAAE;wBAClC,MAAM,IAAI,4BAA4B,SAAS,CAAC,IAAI,CAAC,KAAK,SAAS,CAAC,KAAK,CAAC,oBAAoB,CAAC;oBACnG,CAAC,CAAC,CAAC;oBACH,MAAM,IAAI,uBAAuB,CAAC;gBACtC,CAAC;YACL,CAAC;YAED,MAAM,IAAI,mBAAmB,CAAC;YAE9B,MAAM,CAAC,KAAK,CAAC,UAAU,MAAM,CAAC,MAAM,QAAQ,CAAC,CAAC;YAE9C,QAAQ,CAAC,IAAI,EAAC,MAAM,CAAC,CAAC;QAC1B,CAAC;QACD,KAAK,CAAC,QAAQ;YACV,MAAM,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC;YAC9B,IAAI,CAAC,OAAO,EAAE,CAAC;gBACX,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;gBACzB,IAAI,MAAM,GAAG,sBAAsB,CAAC;gBACpC,MAAM,CAAC,KAAK,CAAC,UAAU,MAAM,CAAC,MAAM,QAAQ,CAAC,CAAC;gBAC9C,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACtB,CAAC;YACD,QAAQ,EAAE,CAAC;QACf,CAAC;KACJ,CAAC,CAAC;AACP,CAAC;AAED,0EAA0E;AAC1E,6EAA6E;AAC7E,6EAA6E;AAC7E,oEAAoE;AACpE,SAAS,WAAW,CAAC,CAAS,EAAE,YAAqB;IACjD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAChC,MAAM,CAAC,GAAG,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;QAC1B,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE;YAAE,OAAO,IAAI,CAAC,CAAiB,QAAQ;QAC3E,IAAI,YAAY,IAAI,CAAC,CAAC,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,CAAC;YAAE,OAAO,IAAI,CAAC,CAAW,MAAM;QACzE,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,IAAI,CAAC;YAAE,OAAO,IAAI,CAAC,CAAC,OAAO;QAC3F,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,IAAI;YAAE,OAAO,IAAI,CAAC;QACxC,IAAI,CAAC,IAAI,MAAM,IAAI,CAAC,IAAI,MAAM;YAAE,OAAO,IAAI,CAAC,CAAuB,aAAa;QAChF,IAAI,CAAC,IAAI,MAAM,IAAI,CAAC,IAAI,MAAM;YAAE,OAAO,IAAI,CAAC,CAAuB,YAAY;QAC/E,IAAI,CAAC,KAAK,MAAM,IAAI,CAAC,KAAK,MAAM;YAAE,OAAO,IAAI,CAAC;IAClD,CAAC;IACD,OAAO,KAAK,CAAC;AACjB,CAAC;AAED,MAAM,UAAU,SAAS,CACvB,KAAyC,EACzC,OAAoC;IAElC,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,KAAK,SAAS;QAAE,OAAO,EAAE,CAAC;IAErD,IAAI,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;IAEtB,sEAAsE;IACtE,IAAI,CAAC,WAAW,CAAC,CAAC,EAAE,OAAO,EAAE,YAAY,KAAK,IAAI,CAAC;QAAE,OAAO,CAAC,CAAC;IAE9D,qEAAqE;IAErE,6EAA6E;IAC7E,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,qCAAqC,EAAE,QAAQ,CAAC,CAAC,CAAE,2BAA2B;IAC5F,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,sCAAsC,EAAE,QAAQ,CAAC,CAAC,CAAC,0BAA0B;IAE3F,8CAA8C;IAC9C,6DAA6D;IAC7D,yDAAyD;IACzD,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,oEAAoE,EAAE,EAAE,CAAC,CAAC;IAExF,+DAA+D;IAC/D,4DAA4D;IAC5D,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,kBAAkB,EAAE,EAAE,CAAC,CAAC;IAEtC,wCAAwC;IACxC,4DAA4D;IAC5D,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAE7B,kCAAkC;IAClC,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;IAC5B,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;IAE5B,IAAI,OAAO,EAAE,YAAY,EAAE,CAAC;QACxB,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;QAC9B,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;IAClC,CAAC;IAED,OAAO,CAAC,CAAC;AACb,CAAC"}
@@ -0,0 +1 @@
1
+ {"version":3,"file":"framer.d.ts","sourceRoot":"","sources":["../../src/stream/framer.ts"],"names":[],"mappings":"AAUA,OAAO,EAAE,SAAS,EAA0B,MAAM,QAAQ,CAAC;AAG3D,eAAO,MAAM,QAAQ,aAAmD,CAAC;AAkCzE,wBAAgB,YAAY,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,SAAS,CA+BnE"}
@@ -0,0 +1 @@
1
+ {"version":3,"file":"framer.js","sourceRoot":"","sources":["../../src/stream/framer.ts"],"names":[],"mappings":"AAAA,8EAA8E;AAC9E,6EAA6E;AAC7E,+EAA+E;AAC/E,+EAA+E;AAC/E,4BAA4B;AAC5B,EAAE;AACF,+EAA+E;AAC/E,uDAAuD;AACvD,yEAAyE;AACzE,mEAAmE;AACnE,OAAO,EAAE,SAAS,EAA0B,MAAM,QAAQ,CAAC;AAC3D,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AAEpD,MAAM,CAAC,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,CAAC,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,UAAU,CAAC,CAAC,CAAC;AAEzE,MAAM,YAAY,GAAG,6BAA6B,CAAC;AAEnD,+EAA+E;AAC/E,+EAA+E;AAC/E,SAAS,kBAAkB,CAAC,IAAY,EAAE,GAAW;IACjD,IAAI,IAAI,KAAK,OAAO,EAAE,CAAC;QACnB,MAAM,IAAI,GAAa,EAAE,CAAC;QAC1B,KAAK,IAAI,CAAC,GAAG,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,GAAG,GAAG,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,GAAG,CAAC,CAAC;YAAE,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QACvF,OAAO,IAAI,CAAC;IAChB,CAAC;IACD,IAAI,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;QACvC,MAAM,IAAI,GAAa,EAAE,CAAC;QAC1B,YAAY,CAAC,SAAS,GAAG,CAAC,CAAC;QAC3B,IAAI,CAAyB,CAAC;QAC9B,OAAO,CAAC,CAAC,GAAG,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAAE,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;QACtE,OAAO,IAAI,CAAC;IAChB,CAAC;IACD,2EAA2E;IAC3E,oEAAoE;IACpE,MAAM,IAAI,GAAa,EAAE,CAAC;IAC1B,IAAI,MAAM,GAAkB,IAAI,CAAC;IACjC,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,KAAK,IAAI,CAAC,GAAG,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,GAAG,GAAG,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC;QACnE,MAAM,EAAE,GAAG,GAAG,CAAC,OAAO,CAAC,GAAG,EAAE,SAAS,CAAC,CAAC;QACvC,MAAM,EAAE,GAAG,EAAE,IAAI,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC;QAClF,IAAI,MAAM,KAAK,IAAI,IAAI,EAAE,KAAK,MAAM;YAAE,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,yBAAyB;QACrF,MAAM,GAAG,EAAE,CAAC;QACZ,SAAS,GAAG,CAAC,GAAG,CAAC,CAAC;IACtB,CAAC;IACD,OAAO,IAAI,CAAC;AAChB,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,IAAY,EAAE,KAAa;IACpD,MAAM,OAAO,GAAG,IAAI,aAAa,CAAC,MAAM,CAAC,CAAC;IAC1C,IAAI,GAAG,GAAG,EAAE,CAAC;IACb,IAAI,GAAG,GAAG,CAAC,CAAC;IAEZ,SAAS,IAAI,CAAC,MAAiB,EAAE,KAAc;QAC3C,SAAS,CAAC;YACN,MAAM,IAAI,GAAG,kBAAkB,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;YAC3C,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;gBAAE,MAAM;YAC7B,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,MAAM,GAAG,KAAK;gBAAE,MAAM,CAAO,wBAAwB;YACxE,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAE,CAAC;YACpD,MAAM,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,GAAG,EAAE,EAAE,IAAI,EAAE,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;YACrD,GAAG,GAAG,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;YACrB,IAAI,IAAI,CAAC,MAAM,GAAG,KAAK;gBAAE,MAAM,CAAiB,mBAAmB;QACvE,CAAC;IACL,CAAC;IAED,OAAO,IAAI,SAAS,CAAC;QACjB,kBAAkB,EAAE,IAAI;QACxB,SAAS,CAAC,KAAU,EAAE,IAAY,EAAE,EAAqB;YACrD,GAAG,IAAI,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;YAC5B,IAAI,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;YAClB,EAAE,EAAE,CAAC;QACT,CAAC;QACD,KAAK,CAAC,EAAqB;YACvB,GAAG,IAAI,OAAO,CAAC,GAAG,EAAE,CAAC;YACrB,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;YACjB,IAAI,GAAG,CAAC,MAAM,GAAG,CAAC;gBAAE,IAAI,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,GAAG,EAAE,EAAE,IAAI,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC,0BAA0B;YACpF,EAAE,EAAE,CAAC;QACT,CAAC;KACJ,CAAC,CAAC;AACP,CAAC"}
@@ -0,0 +1 @@
1
+ {"version":3,"file":"worker-pool-str.d.ts","sourceRoot":"","sources":["../../src/stream/worker-pool-str.ts"],"names":[],"mappings":"AAMA,OAAO,EAAE,SAAS,EAA0B,MAAM,QAAQ,CAAC;AAM3D,MAAM,WAAW,WAAW;IAAG,IAAI,EAAE,MAAM,CAAC;IAAC,GAAG,CAAC,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,GAAG,CAAC;IAAC,OAAO,EAAE,MAAM,CAAC;CAAE;AAEzF,wBAAgB,sBAAsB,CAAC,IAAI,EAAE,WAAW,GAAG,SAAS,CA4FnE"}
@@ -0,0 +1 @@
1
+ {"version":3,"file":"worker-pool-str.js","sourceRoot":"","sources":["../../src/stream/worker-pool-str.ts"],"names":[],"mappings":"AAAA,+EAA+E;AAC/E,gFAAgF;AAChF,mDAAmD;AACnD,EAAE;AACF,wDAAwD;AACxD,sDAAsD;AACtD,OAAO,EAAE,SAAS,EAA0B,MAAM,QAAQ,CAAC;AAC3D,OAAO,EAAE,MAAM,EAAE,MAAM,qBAAqB,CAAC;AAC7C,OAAO,MAAM,MAAM,QAAQ,CAAC;AAE5B,MAAM,MAAM,GAAG,MAAM,CAAC,SAAS,EAAE,CAAC;AAIlC,MAAM,UAAU,sBAAsB,CAAC,IAAiB;IACpD,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;IACpC,MAAM,GAAG,GAAG,CAAC,GAAG,CAAC,CAAC;IAClB,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,kBAAkB,EAAE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAE/D,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,MAAM,IAAI,GAAa,EAAE,CAAC;IAC1B,MAAM,KAAK,GAAoC,EAAE,CAAC;IAClD,MAAM,OAAO,GAAG,IAAI,GAAG,EAAkB,CAAC;IAE1C,IAAI,QAAQ,GAAG,CAAC,CAAC,CAAK,8BAA8B;IACpD,IAAI,QAAQ,GAAG,CAAC,CAAC,CAAK,8BAA8B;IACpD,IAAI,OAAO,GAAG,CAAC,CAAC,CAAM,6BAA6B;IACnD,IAAI,KAAK,GAAG,CAAC,CAAC,CAAQ,qCAAqC;IAC3D,IAAI,KAAK,GAAG,KAAK,CAAC;IAClB,IAAI,QAAQ,GAAG,CAAC,CAAC;IACjB,IAAI,QAAQ,GAA6B,IAAI,CAAC;IAC9C,IAAI,KAAK,GAA6B,IAAI,CAAC;IAC3C,IAAI,MAAM,GAAiB,IAAI,CAAC;IAEhC,MAAM,MAAM,GAAG,IAAI,SAAS,CAAC;QACzB,kBAAkB,EAAE,IAAI,EAAU,iBAAiB;QACnD,kBAAkB,EAAE,KAAK,EAAS,eAAe;QACjD,SAAS,CAAC,KAAoC,EAAE,IAAI,EAAE,EAAqB;YACvE,IAAI,MAAM,EAAE,CAAC;gBAAC,EAAE,CAAC,MAAM,CAAC,CAAC;gBAAC,OAAO;YAAC,CAAC;YACnC,QAAQ,EAAE,CAAC;YACX,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAClB,QAAQ,EAAE,CAAC;YACX,IAAI,QAAQ,GAAG,GAAG;gBAAE,EAAE,EAAE,CAAC;;gBACpB,QAAQ,GAAG,EAAE,CAAC,CAAW,eAAe;QACjD,CAAC;QACD,KAAK,CAAC,EAAqB;YACvB,IAAI,MAAM,EAAE,CAAC;gBAAC,EAAE,CAAC,MAAM,CAAC,CAAC;gBAAC,OAAO;YAAC,CAAC;YACnC,KAAK,GAAG,IAAI,CAAC;YACb,IAAI,OAAO,KAAK,QAAQ;gBAAE,MAAM,CAAC,EAAE,CAAC,CAAC;;gBAChC,KAAK,GAAG,EAAE,CAAC;QACpB,CAAC;KACJ,CAAC,CAAC;IAEH,SAAS,QAAQ;QACb,OAAO,KAAK,CAAC,MAAM,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YACjC,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,EAAG,CAAC;YACtB,MAAM,GAAG,GAAG,KAAK,CAAC,KAAK,EAAG,CAAC;YAC3B,QAAQ,EAAE,CAAC;YACX,CAAC,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;QACvB,CAAC;IACL,CAAC;IAED,SAAS,QAAQ,CAAC,CAAS,EAAE,GAAQ;QACjC,IAAI,GAAG,CAAC,KAAK,EAAE,CAAC;YAAC,IAAI,CAAC,IAAI,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC;YAAC,OAAO;QAAC,CAAC;QACtD,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,EAAE,GAAG,CAAC,GAAG,CAAC,CAAC;QAC9B,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACb,QAAQ,EAAE,CAAC;QACX,OAAO,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC;YAC3B,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAE,CAAC;YACnC,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;YACzB,QAAQ,EAAE,CAAC;YACX,OAAO,EAAE,CAAC;YACV,IAAI,GAAG,CAAC,MAAM;gBAAE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACrC,CAAC;QACD,KAAK,IAAI,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC;QACpB,QAAQ,EAAE,CAAC;QACX,IAAI,QAAQ,IAAI,QAAQ,GAAG,GAAG,EAAE,CAAC;YAAC,MAAM,CAAC,GAAG,QAAQ,CAAC;YAAC,QAAQ,GAAG,IAAI,CAAC;YAAC,CAAC,EAAE,CAAC;QAAC,CAAC;QAC7E,IAAI,KAAK,IAAI,KAAK,IAAI,OAAO,KAAK,QAAQ,EAAE,CAAC;YAAC,MAAM,CAAC,GAAG,KAAK,CAAC;YAAC,KAAK,GAAG,IAAI,CAAC;YAAC,MAAM,CAAC,CAAC,CAAC,CAAC;QAAC,CAAC;IAC7F,CAAC;IAED,SAAS,IAAI,CAAC,GAAU;QACpB,IAAI,MAAM;YAAE,OAAO;QACnB,MAAM,GAAG,GAAG,CAAC;QACb,KAAK,MAAM,CAAC,IAAI,OAAO;YAAE,CAAC,CAAC,SAAS,EAAE,CAAC;QACvC,IAAI,QAAQ,EAAE,CAAC;YAAC,MAAM,CAAC,GAAG,QAAQ,CAAC;YAAC,QAAQ,GAAG,IAAI,CAAC;YAAC,CAAC,CAAC,GAAG,CAAC,CAAC;QAAC,CAAC;aACzD,IAAI,KAAK,EAAE,CAAC;YAAC,MAAM,CAAC,GAAG,KAAK,CAAC;YAAC,KAAK,GAAG,IAAI,CAAC;YAAC,CAAC,CAAC,GAAG,CAAC,CAAC;QAAC,CAAC;;YACrD,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IAC7B,CAAC;IAED,SAAS,MAAM,CAAC,EAAqB;QACjC,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;IACtF,CAAC;IAED,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QACzB,MAAM,CAAC,GAAG,IAAI,MAAM,CAAC,SAAS,EAAE,EAAE,UAAU,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;QACvG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAChB,CAAC,CAAC,EAAE,CAAC,SAAS,EAAE,CAAC,GAAQ,EAAE,EAAE;YACzB,IAAI,GAAG,IAAI,GAAG,CAAC,KAAK,EAAE,CAAC;gBAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;gBAAC,QAAQ,EAAE,CAAC;YAAC,CAAC;;gBAC9C,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;QAC1B,CAAC,CAAC,CAAC;QACH,CAAC,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;IACtC,CAAC;IAEA,MAAc,CAAC,QAAQ,GAAG,GAAG,EAAE,CAAC,KAAK,CAAC;IACvC,MAAM,CAAC,IAAI,CAAC,mBAAmB,CAAC,yBAAyB,CAAC,CAAC;IAC3D,OAAO,MAAM,CAAC;AAClB,CAAC"}
@@ -0,0 +1,10 @@
1
+ import { Transform } from 'stream';
2
+ export interface WorkerPoolOpts {
3
+ map: string;
4
+ param: any;
5
+ workers: number;
6
+ batchSize?: number;
7
+ timeoutMs?: number;
8
+ }
9
+ export declare function createWorkerPool(opts: WorkerPoolOpts): Transform;
10
+ //# sourceMappingURL=worker-pool.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"worker-pool.d.ts","sourceRoot":"","sources":["../../src/stream/worker-pool.ts"],"names":[],"mappings":"AAgBA,OAAO,EAAE,SAAS,EAA0B,MAAM,QAAQ,CAAC;AAO3D,MAAM,WAAW,cAAc;IAC3B,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,GAAG,CAAC;IACX,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,SAAS,CAAC,EAAE,MAAM,CAAC;CACtB;AAED,wBAAgB,gBAAgB,CAAC,IAAI,EAAE,cAAc,GAAG,SAAS,CA4IhE"}
@@ -0,0 +1,206 @@
1
+ // A Transform that parallelizes a per-record map across worker threads while
2
+ // preserving record order. Batches incoming records, dispatches batches to a
3
+ // pool of workers, re-orders the results by sequence number, and pushes them
4
+ // downstream in order.
5
+ //
6
+ // In (objectMode): record objects
7
+ // Out (objectMode): mapped record objects, in the original order
8
+ // (records the map rejected are dropped)
9
+ //
10
+ // Hardening over the first prototype:
11
+ // - worker count is clamped to the available parallelism;
12
+ // - real downstream backpressure: results are held in an out-queue and only
13
+ // pushed while push() accepts them; dispatch/accept pause when the output
14
+ // is backed up and resume on _read (so a slow sink can't grow memory);
15
+ // - a per-batch timeout fails the pipeline instead of hanging if a worker
16
+ // dies silently.
17
+ import { Transform } from 'stream';
18
+ import { Worker } from 'node:worker_threads';
19
+ import { availableParallelism } from 'node:os';
20
+ import log4js from 'log4js';
21
+ const logger = log4js.getLogger();
22
+ export function createWorkerPool(opts) {
23
+ const maxN = Math.max(1, availableParallelism());
24
+ const N = Math.min(Math.max(1, Math.floor(opts.workers) || 1), maxN);
25
+ if (N < opts.workers)
26
+ logger.warn(`--workers ${opts.workers} clamped to ${N} (available parallelism)`);
27
+ const BATCH = opts.batchSize ?? 64;
28
+ const CAP = N * 2; // max batches in flight
29
+ const OUT_CAP = CAP * BATCH; // max records buffered awaiting downstream
30
+ const TIMEOUT = opts.timeoutMs ?? 120_000;
31
+ const workerUrl = new URL('../worker.js', import.meta.url);
32
+ const workers = [];
33
+ const idle = [];
34
+ const timers = new Map();
35
+ const queue = [];
36
+ const reorder = new Map();
37
+ const outQueue = []; // mapped records ready to push, in order
38
+ let nextDispatch = 0;
39
+ let nextEmit = 0;
40
+ let inflight = 0;
41
+ let curBatch = [];
42
+ let acceptCb = null; // held _transform callback
43
+ let endCb = null;
44
+ let failed = null;
45
+ let closing = false; // we are terminating workers on purpose
46
+ const stream = new Transform({
47
+ objectMode: true,
48
+ transform(rec, _enc, cb) {
49
+ if (failed) {
50
+ cb(failed);
51
+ return;
52
+ }
53
+ curBatch.push(rec);
54
+ if (curBatch.length >= BATCH) {
55
+ enqueue(curBatch);
56
+ curBatch = [];
57
+ }
58
+ if (canAccept())
59
+ cb();
60
+ else
61
+ acceptCb = cb; // pause upstream until there's room
62
+ },
63
+ flush(cb) {
64
+ if (failed) {
65
+ cb(failed);
66
+ return;
67
+ }
68
+ if (curBatch.length) {
69
+ enqueue(curBatch);
70
+ curBatch = [];
71
+ }
72
+ if (done())
73
+ finish(cb);
74
+ else
75
+ endCb = cb;
76
+ }
77
+ });
78
+ // Resume producing when the downstream consumer asks for more.
79
+ const origRead = stream._read.bind(stream);
80
+ stream._read = (size) => { pump(); origRead(size); };
81
+ function canAccept() {
82
+ return inflight < CAP && outQueue.length < OUT_CAP;
83
+ }
84
+ function done() {
85
+ return nextEmit === nextDispatch && queue.length === 0 && outQueue.length === 0;
86
+ }
87
+ function enqueue(batch) {
88
+ queue.push({ seq: nextDispatch++, batch });
89
+ dispatch();
90
+ }
91
+ const dbg = () => logger.isDebugEnabled();
92
+ function dispatch() {
93
+ // don't start new work while the output is backed up
94
+ if (dbg() && queue.length && (!idle.length || outQueue.length >= OUT_CAP)) {
95
+ logger.debug(`backpressure: ${queue.length} batches queued, idle=${idle.length}, outQueue=${outQueue.length}/${OUT_CAP}`);
96
+ }
97
+ while (queue.length && idle.length && outQueue.length < OUT_CAP) {
98
+ const w = idle.pop();
99
+ const job = queue.shift();
100
+ inflight++;
101
+ timers.set(w, setTimeout(() => fail(new Error(`worker timed out after ${TIMEOUT}ms (batch ${job.seq})`)), TIMEOUT));
102
+ w.postMessage(job);
103
+ if (dbg())
104
+ logger.debug(`dispatch batch ${job.seq} (${job.batch.length} recs) -> worker; inflight=${inflight}`);
105
+ }
106
+ }
107
+ // push as many ready records as the downstream will take
108
+ function pump() {
109
+ const before = outQueue.length;
110
+ while (outQueue.length && stream.push(outQueue[0]))
111
+ outQueue.shift();
112
+ if (dbg() && outQueue.length > 0 && outQueue.length === before) {
113
+ logger.debug(`downstream full, holding ${outQueue.length} records`);
114
+ }
115
+ dispatch();
116
+ if (acceptCb && canAccept()) {
117
+ const c = acceptCb;
118
+ acceptCb = null;
119
+ c();
120
+ }
121
+ if (endCb && done()) {
122
+ const c = endCb;
123
+ endCb = null;
124
+ finish(c);
125
+ }
126
+ }
127
+ function onResult(w, msg) {
128
+ const t = timers.get(w);
129
+ if (t) {
130
+ clearTimeout(t);
131
+ timers.delete(w);
132
+ }
133
+ if (msg.error) {
134
+ logger.error(`worker error on batch ${msg.seq}: ${msg.error}`);
135
+ fail(new Error(msg.error));
136
+ return;
137
+ }
138
+ reorder.set(msg.seq, msg.mapped);
139
+ idle.push(w);
140
+ inflight--;
141
+ if (dbg())
142
+ logger.debug(`result batch ${msg.seq} (${msg.mapped.length} mapped); inflight=${inflight}, reorderBuf=${reorder.size}, wantNext=${nextEmit}`);
143
+ // move now-contiguous results into the out-queue (don't push yet)
144
+ while (reorder.has(nextEmit)) {
145
+ const mapped = reorder.get(nextEmit);
146
+ reorder.delete(nextEmit);
147
+ nextEmit++;
148
+ for (const m of mapped)
149
+ if (m !== null)
150
+ outQueue.push(m);
151
+ }
152
+ pump();
153
+ }
154
+ function fail(err) {
155
+ if (failed)
156
+ return;
157
+ failed = err;
158
+ closing = true;
159
+ logger.error(`worker pool failing: ${err.message} (terminating ${workers.length} workers)`);
160
+ for (const t of timers.values())
161
+ clearTimeout(t);
162
+ timers.clear();
163
+ for (const w of workers)
164
+ w.terminate();
165
+ if (acceptCb) {
166
+ const c = acceptCb;
167
+ acceptCb = null;
168
+ c(err);
169
+ }
170
+ else if (endCb) {
171
+ const c = endCb;
172
+ endCb = null;
173
+ c(err);
174
+ }
175
+ else
176
+ stream.destroy(err);
177
+ }
178
+ function finish(cb) {
179
+ closing = true;
180
+ for (const t of timers.values())
181
+ clearTimeout(t);
182
+ timers.clear();
183
+ logger.debug(`map complete: ${nextEmit} batches; terminating ${workers.length} workers`);
184
+ Promise.all(workers.map((w) => w.terminate())).then(() => cb()).catch(() => cb());
185
+ }
186
+ for (let i = 0; i < N; i++) {
187
+ const w = new Worker(workerUrl, { workerData: { map: opts.map, param: opts.param } });
188
+ workers.push(w);
189
+ w.on('message', (msg) => {
190
+ if (msg && msg.ready) {
191
+ idle.push(w);
192
+ dispatch();
193
+ if (dbg())
194
+ logger.debug(`worker ${idle.length}/${N} ready`);
195
+ }
196
+ else
197
+ onResult(w, msg);
198
+ });
199
+ w.on('error', (err) => { logger.error(`worker thread crashed: ${err.message}`); fail(err); });
200
+ w.on('exit', (code) => { if (code !== 0 && !failed && !closing)
201
+ fail(new Error(`worker exited unexpectedly (code ${code})`)); });
202
+ }
203
+ logger.info(`map running on ${N} worker threads (batch ${BATCH}, in-flight cap ${CAP}, timeout ${TIMEOUT}ms)`);
204
+ return stream;
205
+ }
206
+ //# sourceMappingURL=worker-pool.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"worker-pool.js","sourceRoot":"","sources":["../../src/stream/worker-pool.ts"],"names":[],"mappings":"AAAA,6EAA6E;AAC7E,6EAA6E;AAC7E,6EAA6E;AAC7E,uBAAuB;AACvB,EAAE;AACF,mCAAmC;AACnC,iEAAiE;AACjE,2DAA2D;AAC3D,EAAE;AACF,sCAAsC;AACtC,2DAA2D;AAC3D,6EAA6E;AAC7E,6EAA6E;AAC7E,0EAA0E;AAC1E,2EAA2E;AAC3E,oBAAoB;AACpB,OAAO,EAAE,SAAS,EAA0B,MAAM,QAAQ,CAAC;AAC3D,OAAO,EAAE,MAAM,EAAE,MAAM,qBAAqB,CAAC;AAC7C,OAAO,EAAE,oBAAoB,EAAE,MAAM,SAAS,CAAC;AAC/C,OAAO,MAAM,MAAM,QAAQ,CAAC;AAE5B,MAAM,MAAM,GAAG,MAAM,CAAC,SAAS,EAAE,CAAC;AAUlC,MAAM,UAAU,gBAAgB,CAAC,IAAoB;IACjD,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,oBAAoB,EAAE,CAAC,CAAC;IACjD,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;IACrE,IAAI,CAAC,GAAG,IAAI,CAAC,OAAO;QAAE,MAAM,CAAC,IAAI,CAAC,aAAa,IAAI,CAAC,OAAO,eAAe,CAAC,0BAA0B,CAAC,CAAC;IACvG,MAAM,KAAK,GAAG,IAAI,CAAC,SAAS,IAAI,EAAE,CAAC;IACnC,MAAM,GAAG,GAAG,CAAC,GAAG,CAAC,CAAC,CAAiB,wBAAwB;IAC3D,MAAM,OAAO,GAAG,GAAG,GAAG,KAAK,CAAC,CAAO,2CAA2C;IAC9E,MAAM,OAAO,GAAG,IAAI,CAAC,SAAS,IAAI,OAAO,CAAC;IAC1C,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,cAAc,EAAE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAE3D,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,MAAM,IAAI,GAAa,EAAE,CAAC;IAC1B,MAAM,MAAM,GAAG,IAAI,GAAG,EAA0B,CAAC;IACjD,MAAM,KAAK,GAAoC,EAAE,CAAC;IAClD,MAAM,OAAO,GAAG,IAAI,GAAG,EAAiB,CAAC;IACzC,MAAM,QAAQ,GAAU,EAAE,CAAC,CAAQ,yCAAyC;IAE5E,IAAI,YAAY,GAAG,CAAC,CAAC;IACrB,IAAI,QAAQ,GAAG,CAAC,CAAC;IACjB,IAAI,QAAQ,GAAG,CAAC,CAAC;IACjB,IAAI,QAAQ,GAAU,EAAE,CAAC;IACzB,IAAI,QAAQ,GAA6B,IAAI,CAAC,CAAG,2BAA2B;IAC5E,IAAI,KAAK,GAA6B,IAAI,CAAC;IAC3C,IAAI,MAAM,GAAiB,IAAI,CAAC;IAChC,IAAI,OAAO,GAAG,KAAK,CAAC,CAA8B,wCAAwC;IAE1F,MAAM,MAAM,GAAG,IAAI,SAAS,CAAC;QACzB,UAAU,EAAE,IAAI;QAChB,SAAS,CAAC,GAAQ,EAAE,IAAI,EAAE,EAAqB;YAC3C,IAAI,MAAM,EAAE,CAAC;gBAAC,EAAE,CAAC,MAAM,CAAC,CAAC;gBAAC,OAAO;YAAC,CAAC;YACnC,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACnB,IAAI,QAAQ,CAAC,MAAM,IAAI,KAAK,EAAE,CAAC;gBAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;gBAAC,QAAQ,GAAG,EAAE,CAAC;YAAC,CAAC;YACnE,IAAI,SAAS,EAAE;gBAAE,EAAE,EAAE,CAAC;;gBACjB,QAAQ,GAAG,EAAE,CAAC,CAAU,oCAAoC;QACrE,CAAC;QACD,KAAK,CAAC,EAAqB;YACvB,IAAI,MAAM,EAAE,CAAC;gBAAC,EAAE,CAAC,MAAM,CAAC,CAAC;gBAAC,OAAO;YAAC,CAAC;YACnC,IAAI,QAAQ,CAAC,MAAM,EAAE,CAAC;gBAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;gBAAC,QAAQ,GAAG,EAAE,CAAC;YAAC,CAAC;YAC1D,IAAI,IAAI,EAAE;gBAAE,MAAM,CAAC,EAAE,CAAC,CAAC;;gBAClB,KAAK,GAAG,EAAE,CAAC;QACpB,CAAC;KACJ,CAAC,CAAC;IAEH,+DAA+D;IAC/D,MAAM,QAAQ,GAAI,MAAc,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IACnD,MAAc,CAAC,KAAK,GAAG,CAAC,IAAY,EAAE,EAAE,GAAG,IAAI,EAAE,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;IAEtE,SAAS,SAAS;QACd,OAAO,QAAQ,GAAG,GAAG,IAAI,QAAQ,CAAC,MAAM,GAAG,OAAO,CAAC;IACvD,CAAC;IACD,SAAS,IAAI;QACT,OAAO,QAAQ,KAAK,YAAY,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,CAAC;IACpF,CAAC;IAED,SAAS,OAAO,CAAC,KAAY;QACzB,KAAK,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,YAAY,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC;QAC3C,QAAQ,EAAE,CAAC;IACf,CAAC;IAED,MAAM,GAAG,GAAG,GAAG,EAAE,CAAC,MAAM,CAAC,cAAc,EAAE,CAAC;IAE1C,SAAS,QAAQ;QACb,qDAAqD;QACrD,IAAI,GAAG,EAAE,IAAI,KAAK,CAAC,MAAM,IAAI,CAAC,CAAC,IAAI,CAAC,MAAM,IAAI,QAAQ,CAAC,MAAM,IAAI,OAAO,CAAC,EAAE,CAAC;YACxE,MAAM,CAAC,KAAK,CAAC,iBAAiB,KAAK,CAAC,MAAM,yBAAyB,IAAI,CAAC,MAAM,cAAc,QAAQ,CAAC,MAAM,IAAI,OAAO,EAAE,CAAC,CAAC;QAC9H,CAAC;QACD,OAAO,KAAK,CAAC,MAAM,IAAI,IAAI,CAAC,MAAM,IAAI,QAAQ,CAAC,MAAM,GAAG,OAAO,EAAE,CAAC;YAC9D,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,EAAG,CAAC;YACtB,MAAM,GAAG,GAAG,KAAK,CAAC,KAAK,EAAG,CAAC;YAC3B,QAAQ,EAAE,CAAC;YACX,MAAM,CAAC,GAAG,CAAC,CAAC,EAAE,UAAU,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,IAAI,KAAK,CAAC,0BAA0B,OAAO,aAAa,GAAG,CAAC,GAAG,GAAG,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC,CAAC;YACpH,CAAC,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;YACnB,IAAI,GAAG,EAAE;gBAAE,MAAM,CAAC,KAAK,CAAC,kBAAkB,GAAG,CAAC,GAAG,KAAK,GAAG,CAAC,KAAK,CAAC,MAAM,8BAA8B,QAAQ,EAAE,CAAC,CAAC;QACpH,CAAC;IACL,CAAC;IAED,yDAAyD;IACzD,SAAS,IAAI;QACT,MAAM,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAC;QAC/B,OAAO,QAAQ,CAAC,MAAM,IAAI,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;YAAE,QAAQ,CAAC,KAAK,EAAE,CAAC;QACrE,IAAI,GAAG,EAAE,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,IAAI,QAAQ,CAAC,MAAM,KAAK,MAAM,EAAE,CAAC;YAC7D,MAAM,CAAC,KAAK,CAAC,4BAA4B,QAAQ,CAAC,MAAM,UAAU,CAAC,CAAC;QACxE,CAAC;QACD,QAAQ,EAAE,CAAC;QACX,IAAI,QAAQ,IAAI,SAAS,EAAE,EAAE,CAAC;YAAC,MAAM,CAAC,GAAG,QAAQ,CAAC;YAAC,QAAQ,GAAG,IAAI,CAAC;YAAC,CAAC,EAAE,CAAC;QAAC,CAAC;QAC1E,IAAI,KAAK,IAAI,IAAI,EAAE,EAAE,CAAC;YAAC,MAAM,CAAC,GAAG,KAAK,CAAC;YAAC,KAAK,GAAG,IAAI,CAAC;YAAC,MAAM,CAAC,CAAC,CAAC,CAAC;QAAC,CAAC;IACtE,CAAC;IAED,SAAS,QAAQ,CAAC,CAAS,EAAE,GAAQ;QACjC,MAAM,CAAC,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;QACxB,IAAI,CAAC,EAAE,CAAC;YAAC,YAAY,CAAC,CAAC,CAAC,CAAC;YAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;QAAC,CAAC;QAC7C,IAAI,GAAG,CAAC,KAAK,EAAE,CAAC;YAAC,MAAM,CAAC,KAAK,CAAC,yBAAyB,GAAG,CAAC,GAAG,KAAK,GAAG,CAAC,KAAK,EAAE,CAAC,CAAC;YAAC,IAAI,CAAC,IAAI,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC;YAAC,OAAO;QAAC,CAAC;QACtH,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;QACjC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACb,QAAQ,EAAE,CAAC;QACX,IAAI,GAAG,EAAE;YAAE,MAAM,CAAC,KAAK,CAAC,gBAAgB,GAAG,CAAC,GAAG,KAAK,GAAG,CAAC,MAAM,CAAC,MAAM,sBAAsB,QAAQ,gBAAgB,OAAO,CAAC,IAAI,cAAc,QAAQ,EAAE,CAAC,CAAC;QACzJ,kEAAkE;QAClE,OAAO,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC;YAC3B,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAE,CAAC;YACtC,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;YACzB,QAAQ,EAAE,CAAC;YACX,KAAK,MAAM,CAAC,IAAI,MAAM;gBAAE,IAAI,CAAC,KAAK,IAAI;oBAAE,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAC7D,CAAC;QACD,IAAI,EAAE,CAAC;IACX,CAAC;IAED,SAAS,IAAI,CAAC,GAAU;QACpB,IAAI,MAAM;YAAE,OAAO;QACnB,MAAM,GAAG,GAAG,CAAC;QACb,OAAO,GAAG,IAAI,CAAC;QACf,MAAM,CAAC,KAAK,CAAC,wBAAwB,GAAG,CAAC,OAAO,iBAAiB,OAAO,CAAC,MAAM,WAAW,CAAC,CAAC;QAC5F,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,MAAM,EAAE;YAAE,YAAY,CAAC,CAAC,CAAC,CAAC;QACjD,MAAM,CAAC,KAAK,EAAE,CAAC;QACf,KAAK,MAAM,CAAC,IAAI,OAAO;YAAE,CAAC,CAAC,SAAS,EAAE,CAAC;QACvC,IAAI,QAAQ,EAAE,CAAC;YAAC,MAAM,CAAC,GAAG,QAAQ,CAAC;YAAC,QAAQ,GAAG,IAAI,CAAC;YAAC,CAAC,CAAC,GAAG,CAAC,CAAC;QAAC,CAAC;aACzD,IAAI,KAAK,EAAE,CAAC;YAAC,MAAM,CAAC,GAAG,KAAK,CAAC;YAAC,KAAK,GAAG,IAAI,CAAC;YAAC,CAAC,CAAC,GAAG,CAAC,CAAC;QAAC,CAAC;;YACrD,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IAC7B,CAAC;IAED,SAAS,MAAM,CAAC,EAAqB;QACjC,OAAO,GAAG,IAAI,CAAC;QACf,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,MAAM,EAAE;YAAE,YAAY,CAAC,CAAC,CAAC,CAAC;QACjD,MAAM,CAAC,KAAK,EAAE,CAAC;QACf,MAAM,CAAC,KAAK,CAAC,iBAAiB,QAAQ,yBAAyB,OAAO,CAAC,MAAM,UAAU,CAAC,CAAC;QACzF,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;IACtF,CAAC;IAED,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QACzB,MAAM,CAAC,GAAG,IAAI,MAAM,CAAC,SAAS,EAAE,EAAE,UAAU,EAAE,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;QACtF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAChB,CAAC,CAAC,EAAE,CAAC,SAAS,EAAE,CAAC,GAAQ,EAAE,EAAE;YACzB,IAAI,GAAG,IAAI,GAAG,CAAC,KAAK,EAAE,CAAC;gBAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;gBAAC,QAAQ,EAAE,CAAC;gBAAC,IAAI,GAAG,EAAE;oBAAE,MAAM,CAAC,KAAK,CAAC,UAAU,IAAI,CAAC,MAAM,IAAI,CAAC,QAAQ,CAAC,CAAC;YAAC,CAAC;;gBAC3G,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;QAC1B,CAAC,CAAC,CAAC;QACH,CAAC,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE,GAAG,MAAM,CAAC,KAAK,CAAC,0BAA0B,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAC9F,CAAC,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,EAAE,GAAG,IAAI,IAAI,KAAK,CAAC,IAAI,CAAC,MAAM,IAAI,CAAC,OAAO;YAAE,IAAI,CAAC,IAAI,KAAK,CAAC,oCAAoC,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACrI,CAAC;IAED,MAAM,CAAC,IAAI,CAAC,kBAAkB,CAAC,0BAA0B,KAAK,mBAAmB,GAAG,aAAa,OAAO,KAAK,CAAC,CAAC;IAC/G,OAAO,MAAM,CAAC;AAClB,CAAC"}
@@ -0,0 +1,16 @@
1
+ import { Transform } from 'stream';
2
+ /**
3
+ * Apply a Catmandu Fix script to each record.
4
+ *
5
+ * marcattacks --map fix --param fix=./my.fix ...
6
+ *
7
+ * `fix` may be a path to a Fix file or an inline Fix script. The script is
8
+ * parsed and compiled once; the compiled chain runs per record.
9
+ */
10
+ export declare function createMapper(opts: {
11
+ fix?: string;
12
+ }): Promise<(data: any) => any>;
13
+ export declare function transform(opts: {
14
+ fix?: string;
15
+ }): Promise<Transform>;
16
+ //# sourceMappingURL=fix.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"fix.d.ts","sourceRoot":"","sources":["../../src/transform/fix.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,QAAQ,CAAC;AAOnC;;;;;;;GAOG;AAGH,wBAAsB,YAAY,CAAC,IAAI,EAAE;IAAE,GAAG,CAAC,EAAE,MAAM,CAAA;CAAE,GAAG,OAAO,CAAC,CAAC,IAAI,EAAE,GAAG,KAAK,GAAG,CAAC,CAMtF;AAED,wBAAsB,SAAS,CAAC,IAAI,EAAE;IAAE,GAAG,CAAC,EAAE,MAAM,CAAA;CAAE,GAAG,OAAO,CAAC,SAAS,CAAC,CAoB1E"}
@@ -0,0 +1,46 @@
1
+ import { Transform } from 'stream';
2
+ import fs from 'fs';
3
+ import log4js from 'log4js';
4
+ import { compileFix, REJECT } from '../fix/index.js';
5
+ const logger = log4js.getLogger();
6
+ /**
7
+ * Apply a Catmandu Fix script to each record.
8
+ *
9
+ * marcattacks --map fix --param fix=./my.fix ...
10
+ *
11
+ * `fix` may be a path to a Fix file or an inline Fix script. The script is
12
+ * parsed and compiled once; the compiled chain runs per record.
13
+ */
14
+ // Pure record -> record mapper (returns REJECT for dropped records). Shared by
15
+ // transform() and the worker pool. Compiled once.
16
+ export async function createMapper(opts) {
17
+ let src = opts.fix ?? '';
18
+ if (src && fs.existsSync(src)) {
19
+ src = fs.readFileSync(src, { encoding: 'utf-8' });
20
+ }
21
+ return compileFix(src);
22
+ }
23
+ export async function transform(opts) {
24
+ let src = opts.fix ?? '';
25
+ if (src && fs.existsSync(src)) {
26
+ src = fs.readFileSync(src, { encoding: 'utf-8' });
27
+ }
28
+ const fix = compileFix(src);
29
+ return new Transform({
30
+ objectMode: true,
31
+ transform(data, _encoding, callback) {
32
+ try {
33
+ const out = fix(data);
34
+ if (out === REJECT)
35
+ callback(); // rejected record -> drop it
36
+ else
37
+ callback(null, out);
38
+ }
39
+ catch (err) {
40
+ logger.error('fix error', err.message);
41
+ callback(err);
42
+ }
43
+ }
44
+ });
45
+ }
46
+ //# sourceMappingURL=fix.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"fix.js","sourceRoot":"","sources":["../../src/transform/fix.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,QAAQ,CAAC;AACnC,OAAO,EAAE,MAAM,IAAI,CAAC;AACpB,OAAO,MAAM,MAAM,QAAQ,CAAC;AAC5B,OAAO,EAAE,UAAU,EAAE,MAAM,EAAE,MAAM,iBAAiB,CAAC;AAErD,MAAM,MAAM,GAAG,MAAM,CAAC,SAAS,EAAE,CAAC;AAElC;;;;;;;GAOG;AACH,+EAA+E;AAC/E,kDAAkD;AAClD,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,IAAsB;IACrD,IAAI,GAAG,GAAG,IAAI,CAAC,GAAG,IAAI,EAAE,CAAC;IACzB,IAAI,GAAG,IAAI,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;QAC5B,GAAG,GAAG,EAAE,CAAC,YAAY,CAAC,GAAG,EAAE,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC,CAAC;IACtD,CAAC;IACD,OAAO,UAAU,CAAC,GAAG,CAAC,CAAC;AAC3B,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,IAAsB;IAClD,IAAI,GAAG,GAAG,IAAI,CAAC,GAAG,IAAI,EAAE,CAAC;IACzB,IAAI,GAAG,IAAI,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;QAC5B,GAAG,GAAG,EAAE,CAAC,YAAY,CAAC,GAAG,EAAE,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC,CAAC;IACtD,CAAC;IACD,MAAM,GAAG,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC;IAE5B,OAAO,IAAI,SAAS,CAAC;QACjB,UAAU,EAAE,IAAI;QAChB,SAAS,CAAC,IAAS,EAAE,SAAS,EAAE,QAAQ;YACpC,IAAI,CAAC;gBACD,MAAM,GAAG,GAAG,GAAG,CAAC,IAAI,CAAC,CAAC;gBACtB,IAAI,GAAG,KAAK,MAAM;oBAAE,QAAQ,EAAE,CAAC,CAAG,6BAA6B;;oBAC1D,QAAQ,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;YAC7B,CAAC;YAAC,OAAO,GAAQ,EAAE,CAAC;gBAChB,MAAM,CAAC,KAAK,CAAC,WAAW,EAAE,GAAG,CAAC,OAAO,CAAC,CAAC;gBACvC,QAAQ,CAAC,GAAG,CAAC,CAAC;YAClB,CAAC;QACL,CAAC;KACJ,CAAC,CAAC;AACP,CAAC"}
@@ -1,4 +1,8 @@
1
1
  import { Transform } from "stream";
2
+ export declare function createMapper(opts: {
3
+ fix: string;
4
+ lookup: string;
5
+ }): Promise<(data: any) => any>;
2
6
  export declare function transform(opts: {
3
7
  fix: string;
4
8
  lookup: string;
@@ -1 +1 @@
1
- {"version":3,"file":"jsonata.d.ts","sourceRoot":"","sources":["../../src/transform/jsonata.ts"],"names":[],"mappings":"AAAA,OAAe,EAAE,SAAS,EAAE,MAAM,QAAQ,CAAC;AAU3C,wBAAsB,SAAS,CAAC,IAAI,EAAE;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,GAAI,OAAO,CAAC,SAAS,CAAC,CAgE1F"}
1
+ {"version":3,"file":"jsonata.d.ts","sourceRoot":"","sources":["../../src/transform/jsonata.ts"],"names":[],"mappings":"AAAA,OAAe,EAAE,SAAS,EAAE,MAAM,QAAQ,CAAC;AAa3C,wBAAsB,YAAY,CAAC,IAAI,EAAE;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,GAAI,OAAO,CAAC,CAAC,IAAI,EAAE,GAAG,KAAK,GAAG,CAAC,CA0CtG;AAED,wBAAsB,SAAS,CAAC,IAAI,EAAE;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,GAAI,OAAO,CAAC,SAAS,CAAC,CAe1F"}
@@ -6,58 +6,54 @@ import { parseStream } from '../util/tsv_parse.js';
6
6
  import { v4 as uuidv4 } from 'uuid';
7
7
  import log4js from 'log4js';
8
8
  const logger = log4js.getLogger();
9
- export async function transform(opts) {
10
- let query;
9
+ // Build a pure record -> record(Promise) mapper. Shared by the in-process
10
+ // transform() and by the worker pool (so the heavy evaluate() can run on
11
+ // worker threads). Expression + helper functions are compiled once.
12
+ export async function createMapper(opts) {
11
13
  let lookup = {};
12
14
  if (opts.lookup) {
13
15
  lookup = await loadLookup(opts.lookup);
14
16
  }
15
- logger.info(lookup);
17
+ // Resolve the query once, up front, instead of on every record.
18
+ let query;
19
+ if (opts.fix) {
20
+ if (fs.existsSync(opts.fix)) {
21
+ query = fs.readFileSync(opts.fix, { encoding: 'utf-8' });
22
+ }
23
+ else {
24
+ throw Error(`no such file ${opts.fix}`);
25
+ }
26
+ }
27
+ else {
28
+ query = '$';
29
+ }
30
+ logger.debug(query);
31
+ // The identity expression is a pure pass-through: skip jsonata entirely.
32
+ if (query.trim() === '$') {
33
+ return (data) => data;
34
+ }
35
+ // Compile the expression and register helper functions ONCE. The helpers
36
+ // read the record currently being processed via `current`; each evaluate
37
+ // is awaited before the next, so this is safe.
38
+ let current;
39
+ const expression = jsonata(query);
40
+ expression.registerFunction('marcmap', (code) => marcmap(current['record'], code, {}));
41
+ expression.registerFunction('marctag', (row) => marctag(row));
42
+ expression.registerFunction('marcind', (row) => marcind(row));
43
+ expression.registerFunction('marcsubfields', (row, regex) => marcsubfields(row, new RegExp(regex)));
44
+ expression.registerFunction('marcrecord', () => current['record']);
45
+ expression.registerFunction('asmarc', (data) => ({ "record": data }));
46
+ expression.registerFunction('genid', () => genid());
47
+ expression.registerFunction('lookup', (key) => lookup[key]);
48
+ return async (data) => { current = data; return expression.evaluate(data); };
49
+ }
50
+ export async function transform(opts) {
51
+ const mapper = await createMapper(opts);
16
52
  return new Transform({
17
53
  objectMode: true,
18
54
  async transform(data, _encoding, callback) {
19
55
  try {
20
- if (!query) {
21
- if (opts.fix) {
22
- if (fs.existsSync(opts.fix)) {
23
- query = fs.readFileSync(opts.fix, { encoding: 'utf-8' });
24
- }
25
- else {
26
- throw Error(`no such file ${opts.fix}`);
27
- }
28
- }
29
- else {
30
- query = '$';
31
- }
32
- logger.debug(query);
33
- }
34
- const expression = jsonata(query);
35
- expression.registerFunction('marcmap', (code) => {
36
- return marcmap(data['record'], code, {});
37
- });
38
- expression.registerFunction('marctag', (row) => {
39
- return marctag(row);
40
- });
41
- expression.registerFunction('marcind', (row) => {
42
- return marcind(row);
43
- });
44
- expression.registerFunction('marcsubfields', (row, regex) => {
45
- return marcsubfields(row, new RegExp(regex));
46
- });
47
- expression.registerFunction('marcrecord', () => {
48
- return data['record'];
49
- });
50
- expression.registerFunction('asmarc', (data) => {
51
- return { "record": data };
52
- });
53
- expression.registerFunction('genid', () => {
54
- return genid();
55
- });
56
- expression.registerFunction('lookup', (key) => {
57
- return lookup[key];
58
- });
59
- data = await expression.evaluate(data);
60
- callback(null, data);
56
+ callback(null, await mapper(data));
61
57
  }
62
58
  catch (err) {
63
59
  logger.info(err);
@@ -1 +1 @@
1
- {"version":3,"file":"jsonata.js","sourceRoot":"","sources":["../../src/transform/jsonata.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,EAAE,EAAE,SAAS,EAAE,MAAM,QAAQ,CAAC;AAC3C,OAAO,OAAO,MAAM,SAAS,CAAC;AAC9B,OAAO,EAAE,MAAM,IAAI,CAAC;AACpB,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AACzE,OAAO,EAAE,WAAW,EAAE,MAAM,sBAAsB,CAAC;AACnD,OAAO,EAAE,EAAE,IAAI,MAAM,EAAE,MAAM,MAAM,CAAC;AACpC,OAAO,MAAM,MAAM,QAAQ,CAAC;AAE5B,MAAM,MAAM,GAAG,MAAM,CAAC,SAAS,EAAE,CAAC;AAElC,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,IAAqC;IACjE,IAAI,KAAc,CAAC;IAEnB,IAAI,MAAM,GAA2B,EAAE,CAAC;IAExC,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;QACd,MAAM,GAAG,MAAM,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAC3C,CAAC;IAED,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAEpB,OAAO,IAAI,SAAS,CAAC;QACjB,UAAU,EAAE,IAAI;QAChB,KAAK,CAAC,SAAS,CAAC,IAAS,EAAE,SAAyB,EAAE,QAAkC;YACpF,IAAI,CAAC;gBACD,IAAI,CAAC,KAAK,EAAE,CAAC;oBACT,IAAI,IAAI,CAAC,GAAG,EAAE,CAAC;wBACX,IAAI,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;4BAC1B,KAAK,GAAG,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,GAAG,EAAC,EAAE,QAAQ,EAAE,OAAO,EAAC,CAAC,CAAC;wBAC3D,CAAC;6BACI,CAAC;4BACF,MAAM,KAAK,CAAC,gBAAgB,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;wBAC5C,CAAC;oBACL,CAAC;yBACI,CAAC;wBACF,KAAK,GAAG,GAAG,CAAC;oBAChB,CAAC;oBAED,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;gBACxB,CAAC;gBACD,MAAM,UAAU,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC;gBAClC,UAAU,CAAC,gBAAgB,CAAC,SAAS,EAAE,CAAC,IAAY,EAAE,EAAE;oBACpD,OAAO,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAC,IAAI,EAAC,EAAE,CAAC,CAAC;gBAC3C,CAAC,CAAC,CAAC;gBACH,UAAU,CAAC,gBAAgB,CAAC,SAAS,EAAE,CAAC,GAAa,EAAE,EAAE;oBACrD,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC;gBACxB,CAAC,CAAC,CAAC;gBACH,UAAU,CAAC,gBAAgB,CAAC,SAAS,EAAE,CAAC,GAAa,EAAE,EAAE;oBACrD,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC;gBACxB,CAAC,CAAC,CAAC;gBACH,UAAU,CAAC,gBAAgB,CAAC,eAAe,EAAE,CAAC,GAAa,EAAC,KAAa,EAAE,EAAE;oBACzE,OAAO,aAAa,CAAC,GAAG,EAAE,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;gBACjD,CAAC,CAAC,CAAC;gBACH,UAAU,CAAC,gBAAgB,CAAC,YAAY,EAAE,GAAG,EAAE;oBAC3C,OAAO,IAAI,CAAC,QAAQ,CAAC,CAAC;gBAC1B,CAAC,CAAC,CAAC;gBACH,UAAU,CAAC,gBAAgB,CAAC,QAAQ,EAAE,CAAC,IAAgB,EAAE,EAAE;oBACvD,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAC,CAAC;gBAC7B,CAAC,CAAC,CAAC;gBACH,UAAU,CAAC,gBAAgB,CAAC,OAAO,EAAE,GAAG,EAAE;oBACtC,OAAO,KAAK,EAAE,CAAC;gBACnB,CAAC,CAAC,CAAC;gBACH,UAAU,CAAC,gBAAgB,CAAC,QAAQ,EAAE,CAAC,GAAG,EAAE,EAAE;oBAC1C,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC;gBACvB,CAAC,CAAC,CAAC;gBACH,IAAI,GAAG,MAAM,UAAU,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;gBACvC,QAAQ,CAAC,IAAI,EAAC,IAAI,CAAC,CAAC;YACxB,CAAC;YACD,OAAO,GAAQ,EAAE,CAAC;gBACd,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBACjB,QAAQ,CAAC,GAAY,CAAC,CAAC;YAC3B,CAAC;QACL,CAAC;KACJ,CAAC,CAAC;AACP,CAAC;AAED,SAAS,KAAK;IACV,OAAO,SAAS,MAAM,EAAE,EAAE,CAAC;AAC/B,CAAC;AAED,KAAK,UAAU,UAAU,CAAC,IAAY;IAClC,IAAI,MAAM,GAA4B,EAAE,CAAC;IAEzC,MAAM,OAAO,GAAG,MAAM,WAAW,CAAC,EAAE,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC,CAAC;IAE7D,KAAK,MAAM,GAAG,IAAI,OAAO,EAAE,CAAC;QACxB,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;QACrC,IAAI,IAAI,IAAI,IAAI,CAAC,MAAM,IAAI,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC;YACjD,MAAM,CAAC,GAAG,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;YACvB,MAAM,CAAC,GAAG,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;YACvB,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;gBACT,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;YAClB,CAAC;QACL,CAAC;IACL,CAAC;IAED,OAAO,MAAM,CAAC;AAClB,CAAC"}
1
+ {"version":3,"file":"jsonata.js","sourceRoot":"","sources":["../../src/transform/jsonata.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,EAAE,EAAE,SAAS,EAAE,MAAM,QAAQ,CAAC;AAC3C,OAAO,OAAO,MAAM,SAAS,CAAC;AAC9B,OAAO,EAAE,MAAM,IAAI,CAAC;AACpB,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AACzE,OAAO,EAAE,WAAW,EAAE,MAAM,sBAAsB,CAAC;AACnD,OAAO,EAAE,EAAE,IAAI,MAAM,EAAE,MAAM,MAAM,CAAC;AACpC,OAAO,MAAM,MAAM,QAAQ,CAAC;AAE5B,MAAM,MAAM,GAAG,MAAM,CAAC,SAAS,EAAE,CAAC;AAElC,0EAA0E;AAC1E,yEAAyE;AACzE,oEAAoE;AACpE,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,IAAqC;IACpE,IAAI,MAAM,GAA2B,EAAE,CAAC;IAExC,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;QACd,MAAM,GAAG,MAAM,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAC3C,CAAC;IAED,gEAAgE;IAChE,IAAI,KAAc,CAAC;IACnB,IAAI,IAAI,CAAC,GAAG,EAAE,CAAC;QACX,IAAI,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;YAC1B,KAAK,GAAG,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,GAAG,EAAC,EAAE,QAAQ,EAAE,OAAO,EAAC,CAAC,CAAC;QAC3D,CAAC;aACI,CAAC;YACF,MAAM,KAAK,CAAC,gBAAgB,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;QAC5C,CAAC;IACL,CAAC;SACI,CAAC;QACF,KAAK,GAAG,GAAG,CAAC;IAChB,CAAC;IACD,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IAEpB,yEAAyE;IACzE,IAAI,KAAK,CAAC,IAAI,EAAE,KAAK,GAAG,EAAE,CAAC;QACvB,OAAO,CAAC,IAAS,EAAE,EAAE,CAAC,IAAI,CAAC;IAC/B,CAAC;IAED,yEAAyE;IACzE,yEAAyE;IACzE,+CAA+C;IAC/C,IAAI,OAAY,CAAC;IACjB,MAAM,UAAU,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC;IAClC,UAAU,CAAC,gBAAgB,CAAC,SAAS,EAAE,CAAC,IAAY,EAAE,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,IAAI,EAAE,EAAE,CAAC,CAAC,CAAC;IAC/F,UAAU,CAAC,gBAAgB,CAAC,SAAS,EAAE,CAAC,GAAa,EAAE,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC;IACxE,UAAU,CAAC,gBAAgB,CAAC,SAAS,EAAE,CAAC,GAAa,EAAE,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC;IACxE,UAAU,CAAC,gBAAgB,CAAC,eAAe,EAAE,CAAC,GAAa,EAAE,KAAa,EAAE,EAAE,CAAC,aAAa,CAAC,GAAG,EAAE,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IACtH,UAAU,CAAC,gBAAgB,CAAC,YAAY,EAAE,GAAG,EAAE,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC;IACnE,UAAU,CAAC,gBAAgB,CAAC,QAAQ,EAAE,CAAC,IAAgB,EAAE,EAAE,CAAC,CAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;IAClF,UAAU,CAAC,gBAAgB,CAAC,OAAO,EAAE,GAAG,EAAE,CAAC,KAAK,EAAE,CAAC,CAAC;IACpD,UAAU,CAAC,gBAAgB,CAAC,QAAQ,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;IAE5D,OAAO,KAAK,EAAE,IAAS,EAAE,EAAE,GAAG,OAAO,GAAG,IAAI,CAAC,CAAC,OAAO,UAAU,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;AACtF,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,IAAqC;IACjE,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC,IAAI,CAAC,CAAC;IAExC,OAAO,IAAI,SAAS,CAAC;QACjB,UAAU,EAAE,IAAI;QAChB,KAAK,CAAC,SAAS,CAAC,IAAS,EAAE,SAAyB,EAAE,QAAkC;YACpF,IAAI,CAAC;gBACD,QAAQ,CAAC,IAAI,EAAE,MAAM,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC;YACvC,CAAC;YACD,OAAO,GAAQ,EAAE,CAAC;gBACd,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBACjB,QAAQ,CAAC,GAAY,CAAC,CAAC;YAC3B,CAAC;QACL,CAAC;KACJ,CAAC,CAAC;AACP,CAAC;AAED,SAAS,KAAK;IACV,OAAO,SAAS,MAAM,EAAE,EAAE,CAAC;AAC/B,CAAC;AAED,KAAK,UAAU,UAAU,CAAC,IAAY;IAClC,IAAI,MAAM,GAA4B,EAAE,CAAC;IAEzC,MAAM,OAAO,GAAG,MAAM,WAAW,CAAC,EAAE,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC,CAAC;IAE7D,KAAK,MAAM,GAAG,IAAI,OAAO,EAAE,CAAC;QACxB,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;QACrC,IAAI,IAAI,IAAI,IAAI,CAAC,MAAM,IAAI,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC;YACjD,MAAM,CAAC,GAAG,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;YACvB,MAAM,CAAC,GAAG,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;YACvB,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;gBACT,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;YAClB,CAAC;QACL,CAAC;IACL,CAAC;IAED,OAAO,MAAM,CAAC;AAClB,CAAC"}
@@ -1,6 +1,6 @@
1
1
  import { Transform } from "stream";
2
2
  export interface MarcInRDFOptions {
3
- parse?: "jsonld" | "turtle" | "notation3";
3
+ parse?: "jsonld" | "turtle" | "field" | "full";
4
4
  }
5
5
  export declare function transform(opts?: MarcInRDFOptions): Promise<Transform>;
6
6
  //# sourceMappingURL=marcinrdf.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"marcinrdf.d.ts","sourceRoot":"","sources":["../../src/transform/marcinrdf.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,QAAQ,CAAC;AAKnC,MAAM,WAAW,gBAAgB;IAC7B,KAAK,CAAC,EAAE,QAAQ,GAAG,QAAQ,GAAG,WAAW,CAAC;CAC7C;AAID,wBAAsB,SAAS,CAAC,IAAI,GAAE,gBAAqB,GAAI,OAAO,CAAC,SAAS,CAAC,CAQhF"}
1
+ {"version":3,"file":"marcinrdf.d.ts","sourceRoot":"","sources":["../../src/transform/marcinrdf.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,QAAQ,CAAC;AAKnC,MAAM,WAAW,gBAAgB;IAC7B,KAAK,CAAC,EAAE,QAAQ,GAAG,QAAQ,GAAG,OAAO,GAAG,MAAM,CAAC;CAClD;AAID,wBAAsB,SAAS,CAAC,IAAI,GAAE,gBAAqB,GAAI,OAAO,CAAC,SAAS,CAAC,CAQhF"}
@@ -36,9 +36,15 @@ async function makeRdfData(data, opts = {}) {
36
36
  if (parse === "turtle") {
37
37
  return await parseString(JSON.stringify(clone), "data.jsonld");
38
38
  }
39
- else if (parse === "notation3") {
39
+ else if (parse === "field") {
40
40
  const record = {
41
- "text": serializeRecord(clone)
41
+ "text": serializeFieldRecord(clone)
42
+ };
43
+ return record;
44
+ }
45
+ else if (parse === "full") {
46
+ const record = {
47
+ "text": serializeFullRecord(clone)
42
48
  };
43
49
  return record;
44
50
  }
@@ -46,7 +52,7 @@ async function makeRdfData(data, opts = {}) {
46
52
  return clone;
47
53
  }
48
54
  }
49
- function serializeRecord(record) {
55
+ function serializeFieldRecord(record) {
50
56
  const id = record['@id'];
51
57
  const ex = "http://example.org/ns#";
52
58
  let result = `<${id}> a <${ex}Record>.\n`;
@@ -58,6 +64,13 @@ function serializeRecord(record) {
58
64
  }
59
65
  return result.trim();
60
66
  }
67
+ function serializeFullRecord(record) {
68
+ const id = record['@id'];
69
+ const ex = "http://example.org/ns#";
70
+ let result = `<${id}> a <${ex}Record>.\n`;
71
+ result += `<${id}> <${ex}record> ${serializeArray(record['record'])}.\n`;
72
+ return result.trim();
73
+ }
61
74
  function serializeArray(array) {
62
75
  const result = "(" + array.map(x => {
63
76
  if (Array.isArray(x)) {
@@ -1 +1 @@
1
- {"version":3,"file":"marcinrdf.js","sourceRoot":"","sources":["../../src/transform/marcinrdf.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,QAAQ,CAAC;AACnC,OAAO,EAAe,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AACzD,OAAO,EAAE,WAAW,EAAE,MAAM,sBAAsB,CAAC;AACnD,OAAO,EAAE,OAAO,EAAE,MAAM,eAAe,CAAC;AAMxC,+DAA+D;AAC/D,sCAAsC;AACtC,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,OAAyB,EAAE;IACvD,OAAO,IAAI,SAAS,CAAC;QACjB,UAAU,EAAE,IAAI;QAChB,KAAK,CAAC,SAAS,CAAC,IAAS,EAAE,SAAS,EAAE,QAAQ;YAC1C,MAAM,OAAO,GAAG,MAAM,WAAW,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;YAC9C,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;QAC5B,CAAC;KACJ,CAAC,CAAC;AACP,CAAC;AAED,KAAK,UAAU,WAAW,CAAC,IAAS,EAAE,OAAyB,EAAE;IAC7D,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,QAAQ,CAAC;IAEjD,kDAAkD;IAClD,sCAAsC;IACtC,IAAI,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;QACjB,OAAO,IAAI,CAAC;IAChB,CAAC;IAED,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAC,KAAK,EAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;IAE/C,MAAM,KAAK,GAAG,eAAe,CAAC,IAAI,CAAC,CAAC;IAEpC,8BAA8B;IAC9B,KAAK,CAAC,UAAU,CAAC,GAAG;QAChB,QAAQ,EAAE,wBAAwB;QAClC,IAAI,EAAE,wBAAwB;QAC9B,QAAQ,EAAE;YACN,KAAK,EAAE,WAAW;YAClB,YAAY,EAAE,OAAO;SACxB;KACJ,CAAC;IAEF,KAAK,CAAC,KAAK,CAAC,GAAG,6BAA6B,EAAE,EAAE,CAAE;IAClD,KAAK,CAAC,OAAO,CAAC,GAAG,WAAW,CAAC;IAE7B,IAAI,KAAK,KAAK,QAAQ,EAAE,CAAC;QACrB,OAAO,MAAM,WAAW,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,EAAE,aAAa,CAAC,CAAC;IACnE,CAAC;SACI,IAAI,KAAK,KAAK,WAAW,EAAE,CAAC;QAC7B,MAAM,MAAM,GAAS;YACjB,MAAM,EAAG,eAAe,CAAC,KAAK,CAAC;SAClC,CAAC;QACF,OAAO,MAAM,CAAC;IAClB,CAAC;SACI,CAAC;QACF,OAAO,KAAK,CAAC;IACjB,CAAC;AACL,CAAC;AAED,SAAS,eAAe,CAAC,MAAW;IAChC,MAAM,EAAE,GAAG,MAAM,CAAE,KAAK,CAAC,CAAC;IAC1B,MAAM,EAAE,GAAG,wBAAwB,CAAC;IACpC,IAAI,MAAM,GAAG,IAAI,EAAE,QAAQ,EAAE,YAAY,CAAC;IAC1C,MAAM,MAAM,GAAG,MAAM,CAAC,QAAQ,CAAC,CAAC;IAEhC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAG,CAAC,GAAG,MAAM,CAAC,MAAM,EAAG,CAAC,EAAE,EAAE,CAAC;QACvC,MAAM,GAAG,GAAI,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAC1B,MAAM,IAAI,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;QACjC,MAAM,IAAI,IAAI,EAAE,MAAM,EAAE,IAAI,GAAG,KAAK,cAAc,CAAC,IAAI,CAAC,KAAK,CAAC;IAClE,CAAC;IAED,OAAO,MAAM,CAAC,IAAI,EAAE,CAAC;AACzB,CAAC;AAED,SAAS,cAAc,CAAC,KAAY;IAChC,MAAM,MAAM,GAAY,GAAG,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE;QACxC,IAAI,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC;YACnB,OAAO,cAAc,CAAC,CAAC,CAAC,CAAC;QAC7B,CAAC;aACI,CAAC;YACF,OAAO,cAAc,CAAC,CAAC,CAAC,CAAC;QAC7B,CAAC;IACL,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,GAAG,CAAC;IACnB,OAAO,MAAM,CAAC;AAClB,CAAC;AAED,SAAS,cAAc,CAAC,KAAU;IAC9B,MAAM,MAAM,GAAG,IAAI;QACf,KAAK,CAAC,UAAU,CAAC,KAAK,EAAC,MAAM,CAAC,CAAC,UAAU,CAAC,IAAI,EAAC,MAAM,CAAC;QACtD,IAAI,CAAC;IACT,OAAO,MAAM,CAAC;AAClB,CAAC"}
1
+ {"version":3,"file":"marcinrdf.js","sourceRoot":"","sources":["../../src/transform/marcinrdf.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,QAAQ,CAAC;AACnC,OAAO,EAAe,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AACzD,OAAO,EAAE,WAAW,EAAE,MAAM,sBAAsB,CAAC;AACnD,OAAO,EAAE,OAAO,EAAE,MAAM,eAAe,CAAC;AAMxC,+DAA+D;AAC/D,sCAAsC;AACtC,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,OAAyB,EAAE;IACvD,OAAO,IAAI,SAAS,CAAC;QACjB,UAAU,EAAE,IAAI;QAChB,KAAK,CAAC,SAAS,CAAC,IAAS,EAAE,SAAS,EAAE,QAAQ;YAC1C,MAAM,OAAO,GAAG,MAAM,WAAW,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;YAC9C,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;QAC5B,CAAC;KACJ,CAAC,CAAC;AACP,CAAC;AAED,KAAK,UAAU,WAAW,CAAC,IAAS,EAAE,OAAyB,EAAE;IAC7D,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,QAAQ,CAAC;IAEjD,kDAAkD;IAClD,sCAAsC;IACtC,IAAI,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;QACjB,OAAO,IAAI,CAAC;IAChB,CAAC;IAED,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAC,KAAK,EAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;IAE/C,MAAM,KAAK,GAAG,eAAe,CAAC,IAAI,CAAC,CAAC;IAEpC,8BAA8B;IAC9B,KAAK,CAAC,UAAU,CAAC,GAAG;QAChB,QAAQ,EAAE,wBAAwB;QAClC,IAAI,EAAE,wBAAwB;QAC9B,QAAQ,EAAE;YACN,KAAK,EAAE,WAAW;YAClB,YAAY,EAAE,OAAO;SACxB;KACJ,CAAC;IAEF,KAAK,CAAC,KAAK,CAAC,GAAG,6BAA6B,EAAE,EAAE,CAAE;IAClD,KAAK,CAAC,OAAO,CAAC,GAAG,WAAW,CAAC;IAE7B,IAAI,KAAK,KAAK,QAAQ,EAAE,CAAC;QACrB,OAAO,MAAM,WAAW,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,EAAE,aAAa,CAAC,CAAC;IACnE,CAAC;SACI,IAAI,KAAK,KAAK,OAAO,EAAE,CAAC;QACzB,MAAM,MAAM,GAAS;YACjB,MAAM,EAAG,oBAAoB,CAAC,KAAK,CAAC;SACvC,CAAC;QACF,OAAO,MAAM,CAAC;IAClB,CAAC;SACI,IAAI,KAAK,KAAK,MAAM,EAAE,CAAC;QACxB,MAAM,MAAM,GAAS;YACjB,MAAM,EAAG,mBAAmB,CAAC,KAAK,CAAC;SACtC,CAAC;QACF,OAAO,MAAM,CAAC;IAClB,CAAC;SACI,CAAC;QACF,OAAO,KAAK,CAAC;IACjB,CAAC;AACL,CAAC;AAED,SAAS,oBAAoB,CAAC,MAAW;IACrC,MAAM,EAAE,GAAG,MAAM,CAAE,KAAK,CAAC,CAAC;IAC1B,MAAM,EAAE,GAAG,wBAAwB,CAAC;IACpC,IAAI,MAAM,GAAG,IAAI,EAAE,QAAQ,EAAE,YAAY,CAAC;IAC1C,MAAM,MAAM,GAAG,MAAM,CAAC,QAAQ,CAAC,CAAC;IAEhC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAG,CAAC,GAAG,MAAM,CAAC,MAAM,EAAG,CAAC,EAAE,EAAE,CAAC;QACvC,MAAM,GAAG,GAAI,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAC1B,MAAM,IAAI,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;QACjC,MAAM,IAAI,IAAI,EAAE,MAAM,EAAE,IAAI,GAAG,KAAK,cAAc,CAAC,IAAI,CAAC,KAAK,CAAC;IAClE,CAAC;IAED,OAAO,MAAM,CAAC,IAAI,EAAE,CAAC;AACzB,CAAC;AAED,SAAS,mBAAmB,CAAC,MAAW;IACpC,MAAM,EAAE,GAAG,MAAM,CAAE,KAAK,CAAC,CAAC;IAC1B,MAAM,EAAE,GAAG,wBAAwB,CAAC;IACpC,IAAI,MAAM,GAAG,IAAI,EAAE,QAAQ,EAAE,YAAY,CAAC;IAE1C,MAAM,IAAI,IAAI,EAAE,MAAM,EAAE,WAAW,cAAc,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,KAAK,CAAC;IAEzE,OAAO,MAAM,CAAC,IAAI,EAAE,CAAC;AACzB,CAAC;AAGD,SAAS,cAAc,CAAC,KAAY;IAChC,MAAM,MAAM,GAAY,GAAG,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE;QACxC,IAAI,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC;YACnB,OAAO,cAAc,CAAC,CAAC,CAAC,CAAC;QAC7B,CAAC;aACI,CAAC;YACF,OAAO,cAAc,CAAC,CAAC,CAAC,CAAC;QAC7B,CAAC;IACL,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,GAAG,CAAC;IACnB,OAAO,MAAM,CAAC;AAClB,CAAC;AAED,SAAS,cAAc,CAAC,KAAU;IAC9B,MAAM,MAAM,GAAG,IAAI;QACf,KAAK,CAAC,UAAU,CAAC,KAAK,EAAC,MAAM,CAAC,CAAC,UAAU,CAAC,IAAI,EAAC,MAAM,CAAC;QACtD,IAAI,CAAC;IACT,OAAO,MAAM,CAAC;AAClB,CAAC"}
@@ -0,0 +1,19 @@
1
+ /**
2
+ * Marker for record objects whose string values are already free of control
3
+ * characters (\x00-\x1F and \x7F).
4
+ *
5
+ * Input readers that strip control characters while parsing (e.g. the XML
6
+ * readers) set this on the records they emit. Output writers that would
7
+ * otherwise re-strip every value (e.g. alephseq) can then skip that work.
8
+ *
9
+ * It is a Symbol so it never shows up in `JSON.stringify`, `Object.keys` or
10
+ * `for...in` — outputs that serialise the whole record object (json, jsonl,
11
+ * rdf) are unaffected. A transform that builds a new record object (e.g. a
12
+ * JSONata `fix`) naturally drops the marker, so escaping is re-applied unless
13
+ * cleanliness is re-asserted — a safe default.
14
+ */
15
+ export declare const CLEAN: unique symbol;
16
+ export type CleanFlag = {
17
+ [CLEAN]?: boolean;
18
+ };
19
+ //# sourceMappingURL=marc_record.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"marc_record.d.ts","sourceRoot":"","sources":["../../src/util/marc_record.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AACH,eAAO,MAAM,KAAK,EAAE,OAAO,MAAoC,CAAC;AAEhE,MAAM,MAAM,SAAS,GAAG;IAAE,CAAC,KAAK,CAAC,CAAC,EAAE,OAAO,CAAA;CAAE,CAAC"}
@@ -0,0 +1,16 @@
1
+ /**
2
+ * Marker for record objects whose string values are already free of control
3
+ * characters (\x00-\x1F and \x7F).
4
+ *
5
+ * Input readers that strip control characters while parsing (e.g. the XML
6
+ * readers) set this on the records they emit. Output writers that would
7
+ * otherwise re-strip every value (e.g. alephseq) can then skip that work.
8
+ *
9
+ * It is a Symbol so it never shows up in `JSON.stringify`, `Object.keys` or
10
+ * `for...in` — outputs that serialise the whole record object (json, jsonl,
11
+ * rdf) are unaffected. A transform that builds a new record object (e.g. a
12
+ * JSONata `fix`) naturally drops the marker, so escaping is re-applied unless
13
+ * cleanliness is re-asserted — a safe default.
14
+ */
15
+ export const CLEAN = Symbol('marc.record.clean');
16
+ //# sourceMappingURL=marc_record.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"marc_record.js","sourceRoot":"","sources":["../../src/util/marc_record.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AACH,MAAM,CAAC,MAAM,KAAK,GAAkB,MAAM,CAAC,mBAAmB,CAAC,CAAC"}
@@ -0,0 +1 @@
1
+ {"version":3,"file":"worker-str.d.ts","sourceRoot":"","sources":["../src/worker-str.ts"],"names":[],"mappings":""}
@@ -0,0 +1 @@
1
+ {"version":3,"file":"worker-str.js","sourceRoot":"","sources":["../src/worker-str.ts"],"names":[],"mappings":"AAAA,gFAAgF;AAChF,8EAA8E;AAC9E,4EAA4E;AAC5E,EAAE;AACF,+EAA+E;AAC/E,YAAY;AACZ,0EAA0E;AAC1E,qCAAqC;AACrC,mEAAmE;AACnE,OAAO,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AAC7D,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AACvC,OAAO,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AAChD,OAAO,EAAE,MAAM,EAAE,MAAM,iBAAiB,CAAC;AAEzC,MAAM,EAAE,IAAI,EAAE,GAAG,EAAE,KAAK,EAAE,GAAG,UAAwD,CAAC;AAEtF,MAAM,QAAQ,GAAG,MAAM,UAAU,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;AACjD,MAAM,MAAM,GAAuB,GAAG;IAClC,CAAC,CAAC,MAAM,CAAC,MAAM,UAAU,CAAC,GAAG,EAAE,WAAW,CAAC,CAAC,CAAC,YAAY,CAAC,KAAK,IAAI,EAAE,CAAC;IACtE,CAAC,CAAC,CAAC,CAAM,EAAE,EAAE,CAAC,CAAC,CAAC;AAEpB,6EAA6E;AAC7E,KAAK,UAAU,KAAK,CAAC,IAAY;IAC7B,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,SAAS,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC;IAClD,MAAM,OAAO,GAAU,EAAE,CAAC;IAC1B,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QACxC,GAAG,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,CAAM,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;QAC5C,GAAG,CAAC,EAAE,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;QACvB,GAAG,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;QACxB,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAClC,CAAC,CAAC,CAAC;IACH,OAAO,OAAO,CAAC;AACnB,CAAC;AAED,UAAW,CAAC,WAAW,CAAC,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;AAEzC,UAAW,CAAC,EAAE,CAAC,SAAS,EAAE,KAAK,EAAE,GAAkC,EAAE,EAAE;IACnE,IAAI,CAAC;QACD,MAAM,OAAO,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACtC,IAAI,GAAG,GAAG,EAAE,CAAC;QACb,IAAI,CAAC,GAAG,CAAC,CAAC;QACV,KAAK,MAAM,GAAG,IAAI,OAAO,EAAE,CAAC;YACxB,MAAM,CAAC,GAAG,MAAM,MAAM,CAAC,GAAG,CAAC,CAAC;YAC5B,IAAI,CAAC,KAAK,MAAM,IAAI,CAAC,KAAK,SAAS,IAAI,CAAC,KAAK,IAAI;gBAAE,SAAS;YAC5D,IAAI,OAAO,CAAC,KAAK,QAAQ,IAAI,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,MAAM,KAAK,CAAC;gBAAE,SAAS,CAAC,uBAAuB;YAC3F,GAAG,IAAI,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC;YAChC,CAAC,EAAE,CAAC;QACR,CAAC;QACD,UAAW,CAAC,WAAW,CAAC,EAAE,GAAG,EAAE,GAAG,CAAC,GAAG,EAAE,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC;IACtD,CAAC;IAAC,OAAO,GAAQ,EAAE,CAAC;QAChB,UAAW,CAAC,WAAW,CAAC,EAAE,GAAG,EAAE,GAAG,CAAC,GAAG,EAAE,KAAK,EAAE,GAAG,EAAE,OAAO,IAAI,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IAClF,CAAC;AACL,CAAC,CAAC,CAAC"}
@@ -0,0 +1,2 @@
1
+ export {};
2
+ //# sourceMappingURL=worker.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"worker.d.ts","sourceRoot":"","sources":["../src/worker.ts"],"names":[],"mappings":""}