@terascope/job-components 0.75.1 → 1.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (274) hide show
  1. package/dist/src/builtin/collect/interfaces.d.ts +2 -2
  2. package/dist/src/builtin/collect/interfaces.d.ts.map +1 -1
  3. package/dist/src/builtin/collect/interfaces.js +1 -2
  4. package/dist/src/builtin/collect/processor.d.ts +4 -4
  5. package/dist/src/builtin/collect/processor.d.ts.map +1 -1
  6. package/dist/src/builtin/collect/processor.js +5 -7
  7. package/dist/src/builtin/collect/processor.js.map +1 -1
  8. package/dist/src/builtin/collect/schema.d.ts +2 -2
  9. package/dist/src/builtin/collect/schema.d.ts.map +1 -1
  10. package/dist/src/builtin/collect/schema.js +2 -5
  11. package/dist/src/builtin/collect/schema.js.map +1 -1
  12. package/dist/src/builtin/delay/interfaces.d.ts +1 -1
  13. package/dist/src/builtin/delay/interfaces.d.ts.map +1 -1
  14. package/dist/src/builtin/delay/interfaces.js +1 -2
  15. package/dist/src/builtin/delay/processor.d.ts +2 -2
  16. package/dist/src/builtin/delay/processor.d.ts.map +1 -1
  17. package/dist/src/builtin/delay/processor.js +4 -7
  18. package/dist/src/builtin/delay/processor.js.map +1 -1
  19. package/dist/src/builtin/delay/schema.d.ts +2 -2
  20. package/dist/src/builtin/delay/schema.d.ts.map +1 -1
  21. package/dist/src/builtin/delay/schema.js +2 -5
  22. package/dist/src/builtin/delay/schema.js.map +1 -1
  23. package/dist/src/builtin/index.d.ts +5 -5
  24. package/dist/src/builtin/index.d.ts.map +1 -1
  25. package/dist/src/builtin/index.js +6 -16
  26. package/dist/src/builtin/index.js.map +1 -1
  27. package/dist/src/builtin/noop/processor.d.ts +1 -1
  28. package/dist/src/builtin/noop/processor.d.ts.map +1 -1
  29. package/dist/src/builtin/noop/processor.js +2 -5
  30. package/dist/src/builtin/noop/processor.js.map +1 -1
  31. package/dist/src/builtin/noop/schema.d.ts +1 -1
  32. package/dist/src/builtin/noop/schema.d.ts.map +1 -1
  33. package/dist/src/builtin/noop/schema.js +2 -5
  34. package/dist/src/builtin/noop/schema.js.map +1 -1
  35. package/dist/src/builtin/test-reader/data/fetcher-data.d.ts +1 -1
  36. package/dist/src/builtin/test-reader/data/fetcher-data.d.ts.map +1 -1
  37. package/dist/src/builtin/test-reader/data/fetcher-data.js +1 -2
  38. package/dist/src/builtin/test-reader/data/fetcher-data.js.map +1 -1
  39. package/dist/src/builtin/test-reader/data/slicer-data.d.ts +1 -1
  40. package/dist/src/builtin/test-reader/data/slicer-data.d.ts.map +1 -1
  41. package/dist/src/builtin/test-reader/data/slicer-data.js +1 -2
  42. package/dist/src/builtin/test-reader/data/slicer-data.js.map +1 -1
  43. package/dist/src/builtin/test-reader/fetcher.d.ts +3 -3
  44. package/dist/src/builtin/test-reader/fetcher.d.ts.map +1 -1
  45. package/dist/src/builtin/test-reader/fetcher.js +12 -21
  46. package/dist/src/builtin/test-reader/fetcher.js.map +1 -1
  47. package/dist/src/builtin/test-reader/interfaces.d.ts +1 -1
  48. package/dist/src/builtin/test-reader/interfaces.d.ts.map +1 -1
  49. package/dist/src/builtin/test-reader/interfaces.js +1 -2
  50. package/dist/src/builtin/test-reader/schema.d.ts +2 -2
  51. package/dist/src/builtin/test-reader/schema.d.ts.map +1 -1
  52. package/dist/src/builtin/test-reader/schema.js +2 -5
  53. package/dist/src/builtin/test-reader/schema.js.map +1 -1
  54. package/dist/src/builtin/test-reader/slicer.d.ts +3 -3
  55. package/dist/src/builtin/test-reader/slicer.d.ts.map +1 -1
  56. package/dist/src/builtin/test-reader/slicer.js +10 -19
  57. package/dist/src/builtin/test-reader/slicer.js.map +1 -1
  58. package/dist/src/config-validators.d.ts +1 -1
  59. package/dist/src/config-validators.d.ts.map +1 -1
  60. package/dist/src/config-validators.js +10 -19
  61. package/dist/src/config-validators.js.map +1 -1
  62. package/dist/src/execution-context/api.d.ts +4 -5
  63. package/dist/src/execution-context/api.d.ts.map +1 -1
  64. package/dist/src/execution-context/api.js +16 -16
  65. package/dist/src/execution-context/api.js.map +1 -1
  66. package/dist/src/execution-context/base.d.ts +6 -6
  67. package/dist/src/execution-context/base.d.ts.map +1 -1
  68. package/dist/src/execution-context/base.js +30 -27
  69. package/dist/src/execution-context/base.js.map +1 -1
  70. package/dist/src/execution-context/index.d.ts +9 -12
  71. package/dist/src/execution-context/index.d.ts.map +1 -1
  72. package/dist/src/execution-context/index.js +16 -44
  73. package/dist/src/execution-context/index.js.map +1 -1
  74. package/dist/src/execution-context/interfaces.d.ts +2 -2
  75. package/dist/src/execution-context/interfaces.d.ts.map +1 -1
  76. package/dist/src/execution-context/interfaces.js +1 -2
  77. package/dist/src/execution-context/slicer.d.ts +5 -4
  78. package/dist/src/execution-context/slicer.d.ts.map +1 -1
  79. package/dist/src/execution-context/slicer.js +20 -20
  80. package/dist/src/execution-context/slicer.js.map +1 -1
  81. package/dist/src/execution-context/utils.d.ts +1 -1
  82. package/dist/src/execution-context/utils.d.ts.map +1 -1
  83. package/dist/src/execution-context/utils.js +5 -11
  84. package/dist/src/execution-context/utils.js.map +1 -1
  85. package/dist/src/execution-context/worker.d.ts +6 -5
  86. package/dist/src/execution-context/worker.d.ts.map +1 -1
  87. package/dist/src/execution-context/worker.js +50 -68
  88. package/dist/src/execution-context/worker.js.map +1 -1
  89. package/dist/src/formats.d.ts +2 -2
  90. package/dist/src/formats.d.ts.map +1 -1
  91. package/dist/src/formats.js +21 -27
  92. package/dist/src/formats.js.map +1 -1
  93. package/dist/src/index.d.ts +12 -12
  94. package/dist/src/index.d.ts.map +1 -1
  95. package/dist/src/index.js +13 -29
  96. package/dist/src/index.js.map +1 -1
  97. package/dist/src/interfaces/context.d.ts +23 -140
  98. package/dist/src/interfaces/context.d.ts.map +1 -1
  99. package/dist/src/interfaces/context.js +1 -2
  100. package/dist/src/interfaces/index.d.ts +4 -5
  101. package/dist/src/interfaces/index.d.ts.map +1 -1
  102. package/dist/src/interfaces/index.js +4 -21
  103. package/dist/src/interfaces/index.js.map +1 -1
  104. package/dist/src/interfaces/misc.d.ts +3 -0
  105. package/dist/src/interfaces/misc.d.ts.map +1 -1
  106. package/dist/src/interfaces/misc.js +2 -2
  107. package/dist/src/interfaces/misc.js.map +1 -1
  108. package/dist/src/interfaces/operation-lifecycle.d.ts +4 -3
  109. package/dist/src/interfaces/operation-lifecycle.d.ts.map +1 -1
  110. package/dist/src/interfaces/operation-lifecycle.js +1 -2
  111. package/dist/src/interfaces/operations.d.ts +11 -64
  112. package/dist/src/interfaces/operations.d.ts.map +1 -1
  113. package/dist/src/interfaces/operations.js +1 -4
  114. package/dist/src/interfaces/operations.js.map +1 -1
  115. package/dist/src/job-schemas.d.ts +1 -1
  116. package/dist/src/job-schemas.d.ts.map +1 -1
  117. package/dist/src/job-schemas.js +32 -39
  118. package/dist/src/job-schemas.js.map +1 -1
  119. package/dist/src/job-validator.d.ts +3 -2
  120. package/dist/src/job-validator.d.ts.map +1 -1
  121. package/dist/src/job-validator.js +19 -20
  122. package/dist/src/job-validator.js.map +1 -1
  123. package/dist/src/operation-loader/index.d.ts +2 -2
  124. package/dist/src/operation-loader/index.d.ts.map +1 -1
  125. package/dist/src/operation-loader/index.js +2 -18
  126. package/dist/src/operation-loader/index.js.map +1 -1
  127. package/dist/src/operation-loader/interfaces.d.ts +1 -1
  128. package/dist/src/operation-loader/interfaces.d.ts.map +1 -1
  129. package/dist/src/operation-loader/interfaces.js +15 -18
  130. package/dist/src/operation-loader/interfaces.js.map +1 -1
  131. package/dist/src/operation-loader/loader.d.ts +7 -9
  132. package/dist/src/operation-loader/loader.d.ts.map +1 -1
  133. package/dist/src/operation-loader/loader.js +96 -137
  134. package/dist/src/operation-loader/loader.js.map +1 -1
  135. package/dist/src/operations/api-factory.d.ts +2 -2
  136. package/dist/src/operations/api-factory.d.ts.map +1 -1
  137. package/dist/src/operations/api-factory.js +4 -13
  138. package/dist/src/operations/api-factory.js.map +1 -1
  139. package/dist/src/operations/batch-processor.d.ts +2 -2
  140. package/dist/src/operations/batch-processor.d.ts.map +1 -1
  141. package/dist/src/operations/batch-processor.js +5 -11
  142. package/dist/src/operations/batch-processor.js.map +1 -1
  143. package/dist/src/operations/convict-schema.d.ts +2 -2
  144. package/dist/src/operations/convict-schema.d.ts.map +1 -1
  145. package/dist/src/operations/convict-schema.js +11 -15
  146. package/dist/src/operations/convict-schema.js.map +1 -1
  147. package/dist/src/operations/core/api-core.d.ts +4 -4
  148. package/dist/src/operations/core/api-core.d.ts.map +1 -1
  149. package/dist/src/operations/core/api-core.js +7 -10
  150. package/dist/src/operations/core/api-core.js.map +1 -1
  151. package/dist/src/operations/core/core.d.ts +2 -2
  152. package/dist/src/operations/core/core.d.ts.map +1 -1
  153. package/dist/src/operations/core/core.js +5 -4
  154. package/dist/src/operations/core/core.js.map +1 -1
  155. package/dist/src/operations/core/fetcher-core.d.ts +2 -2
  156. package/dist/src/operations/core/fetcher-core.d.ts.map +1 -1
  157. package/dist/src/operations/core/fetcher-core.js +2 -8
  158. package/dist/src/operations/core/fetcher-core.js.map +1 -1
  159. package/dist/src/operations/core/index.d.ts +6 -6
  160. package/dist/src/operations/core/index.d.ts.map +1 -1
  161. package/dist/src/operations/core/index.js +7 -18
  162. package/dist/src/operations/core/index.js.map +1 -1
  163. package/dist/src/operations/core/operation-core.d.ts +5 -5
  164. package/dist/src/operations/core/operation-core.d.ts.map +1 -1
  165. package/dist/src/operations/core/operation-core.js +8 -11
  166. package/dist/src/operations/core/operation-core.js.map +1 -1
  167. package/dist/src/operations/core/processor-core.d.ts +2 -2
  168. package/dist/src/operations/core/processor-core.d.ts.map +1 -1
  169. package/dist/src/operations/core/processor-core.js +2 -8
  170. package/dist/src/operations/core/processor-core.js.map +1 -1
  171. package/dist/src/operations/core/schema-core.d.ts +1 -1
  172. package/dist/src/operations/core/schema-core.d.ts.map +1 -1
  173. package/dist/src/operations/core/schema-core.js +3 -4
  174. package/dist/src/operations/core/schema-core.js.map +1 -1
  175. package/dist/src/operations/core/slicer-core.d.ts +4 -4
  176. package/dist/src/operations/core/slicer-core.d.ts.map +1 -1
  177. package/dist/src/operations/core/slicer-core.js +13 -14
  178. package/dist/src/operations/core/slicer-core.js.map +1 -1
  179. package/dist/src/operations/each-processor.d.ts +2 -2
  180. package/dist/src/operations/each-processor.d.ts.map +1 -1
  181. package/dist/src/operations/each-processor.js +2 -8
  182. package/dist/src/operations/each-processor.js.map +1 -1
  183. package/dist/src/operations/fetcher.d.ts +2 -2
  184. package/dist/src/operations/fetcher.d.ts.map +1 -1
  185. package/dist/src/operations/fetcher.js +4 -10
  186. package/dist/src/operations/fetcher.js.map +1 -1
  187. package/dist/src/operations/filter-processor.d.ts +2 -2
  188. package/dist/src/operations/filter-processor.d.ts.map +1 -1
  189. package/dist/src/operations/filter-processor.js +2 -8
  190. package/dist/src/operations/filter-processor.js.map +1 -1
  191. package/dist/src/operations/index.d.ts +14 -15
  192. package/dist/src/operations/index.d.ts.map +1 -1
  193. package/dist/src/operations/index.js +15 -47
  194. package/dist/src/operations/index.js.map +1 -1
  195. package/dist/src/operations/interfaces.d.ts +12 -12
  196. package/dist/src/operations/interfaces.d.ts.map +1 -1
  197. package/dist/src/operations/interfaces.js +1 -2
  198. package/dist/src/operations/job-observer.d.ts +3 -3
  199. package/dist/src/operations/job-observer.d.ts.map +1 -1
  200. package/dist/src/operations/job-observer.js +14 -10
  201. package/dist/src/operations/job-observer.js.map +1 -1
  202. package/dist/src/operations/map-processor.d.ts +2 -2
  203. package/dist/src/operations/map-processor.d.ts.map +1 -1
  204. package/dist/src/operations/map-processor.js +4 -10
  205. package/dist/src/operations/map-processor.js.map +1 -1
  206. package/dist/src/operations/observer.d.ts +2 -2
  207. package/dist/src/operations/observer.d.ts.map +1 -1
  208. package/dist/src/operations/observer.js +2 -8
  209. package/dist/src/operations/observer.js.map +1 -1
  210. package/dist/src/operations/operation-api.d.ts +2 -2
  211. package/dist/src/operations/operation-api.d.ts.map +1 -1
  212. package/dist/src/operations/operation-api.js +2 -8
  213. package/dist/src/operations/operation-api.js.map +1 -1
  214. package/dist/src/operations/parallel-slicer.d.ts +2 -2
  215. package/dist/src/operations/parallel-slicer.d.ts.map +1 -1
  216. package/dist/src/operations/parallel-slicer.js +7 -16
  217. package/dist/src/operations/parallel-slicer.js.map +1 -1
  218. package/dist/src/operations/slicer.d.ts +2 -2
  219. package/dist/src/operations/slicer.d.ts.map +1 -1
  220. package/dist/src/operations/slicer.js +7 -16
  221. package/dist/src/operations/slicer.js.map +1 -1
  222. package/dist/src/register-apis.d.ts +2 -3
  223. package/dist/src/register-apis.d.ts.map +1 -1
  224. package/dist/src/register-apis.js +17 -49
  225. package/dist/src/register-apis.js.map +1 -1
  226. package/dist/src/test-helpers.d.ts +11 -31
  227. package/dist/src/test-helpers.d.ts.map +1 -1
  228. package/dist/src/test-helpers.js +67 -150
  229. package/dist/src/test-helpers.js.map +1 -1
  230. package/dist/src/utils.d.ts +1 -1
  231. package/dist/src/utils.d.ts.map +1 -1
  232. package/dist/src/utils.js +9 -14
  233. package/dist/src/utils.js.map +1 -1
  234. package/package.json +10 -7
  235. package/dist/src/interfaces/jobs.d.ts +0 -171
  236. package/dist/src/interfaces/jobs.d.ts.map +0 -1
  237. package/dist/src/interfaces/jobs.js +0 -10
  238. package/dist/src/interfaces/jobs.js.map +0 -1
  239. package/dist/src/operations/shims/index.d.ts +0 -9
  240. package/dist/src/operations/shims/index.d.ts.map +0 -1
  241. package/dist/src/operations/shims/index.js +0 -21
  242. package/dist/src/operations/shims/index.js.map +0 -1
  243. package/dist/src/operations/shims/legacy-processor-shim.d.ts +0 -5
  244. package/dist/src/operations/shims/legacy-processor-shim.d.ts.map +0 -1
  245. package/dist/src/operations/shims/legacy-processor-shim.js +0 -50
  246. package/dist/src/operations/shims/legacy-processor-shim.js.map +0 -1
  247. package/dist/src/operations/shims/legacy-reader-shim.d.ts +0 -7
  248. package/dist/src/operations/shims/legacy-reader-shim.d.ts.map +0 -1
  249. package/dist/src/operations/shims/legacy-reader-shim.js +0 -66
  250. package/dist/src/operations/shims/legacy-reader-shim.js.map +0 -1
  251. package/dist/src/operations/shims/legacy-slice-events-shim.d.ts +0 -10
  252. package/dist/src/operations/shims/legacy-slice-events-shim.d.ts.map +0 -1
  253. package/dist/src/operations/shims/legacy-slice-events-shim.js +0 -34
  254. package/dist/src/operations/shims/legacy-slice-events-shim.js.map +0 -1
  255. package/dist/src/operations/shims/operation-api-shim.d.ts +0 -7
  256. package/dist/src/operations/shims/operation-api-shim.d.ts.map +0 -1
  257. package/dist/src/operations/shims/operation-api-shim.js +0 -10
  258. package/dist/src/operations/shims/operation-api-shim.js.map +0 -1
  259. package/dist/src/operations/shims/processor-shim.d.ts +0 -4
  260. package/dist/src/operations/shims/processor-shim.d.ts.map +0 -1
  261. package/dist/src/operations/shims/processor-shim.js +0 -51
  262. package/dist/src/operations/shims/processor-shim.js.map +0 -1
  263. package/dist/src/operations/shims/reader-shim.d.ts +0 -4
  264. package/dist/src/operations/shims/reader-shim.d.ts.map +0 -1
  265. package/dist/src/operations/shims/reader-shim.js +0 -92
  266. package/dist/src/operations/shims/reader-shim.js.map +0 -1
  267. package/dist/src/operations/shims/schema-shim.d.ts +0 -4
  268. package/dist/src/operations/shims/schema-shim.d.ts.map +0 -1
  269. package/dist/src/operations/shims/schema-shim.js +0 -31
  270. package/dist/src/operations/shims/schema-shim.js.map +0 -1
  271. package/dist/src/operations/shims/shim-utils.d.ts +0 -9
  272. package/dist/src/operations/shims/shim-utils.d.ts.map +0 -1
  273. package/dist/src/operations/shims/shim-utils.js +0 -34
  274. package/dist/src/operations/shims/shim-utils.js.map +0 -1
@@ -1,7 +0,0 @@
1
- import { Context } from '../../interfaces';
2
- import { OperationAPIConstructor } from '../interfaces';
3
- export default function operationAPIShim(context: Context, apis?: APIs): void;
4
- export interface APIs {
5
- [name: string]: OperationAPIConstructor;
6
- }
7
- //# sourceMappingURL=operation-api-shim.d.ts.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"operation-api-shim.d.ts","sourceRoot":"","sources":["../../../../src/operations/shims/operation-api-shim.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAC3C,OAAO,EAAE,uBAAuB,EAAE,MAAM,eAAe,CAAC;AAExD,MAAM,CAAC,OAAO,UAAU,gBAAgB,CAAC,OAAO,EAAE,OAAO,EAAE,IAAI,GAAE,IAAS,GAAG,IAAI,CAKhF;AAED,MAAM,WAAW,IAAI;IACjB,CAAC,IAAI,EAAE,MAAM,GAAG,uBAAuB,CAAC;CAC3C"}
@@ -1,10 +0,0 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- function operationAPIShim(context, apis = {}) {
4
- Object.keys(apis).forEach((name) => {
5
- const api = apis[name];
6
- context.apis.executionContext.addToRegistry(name, api);
7
- });
8
- }
9
- exports.default = operationAPIShim;
10
- //# sourceMappingURL=operation-api-shim.js.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"operation-api-shim.js","sourceRoot":"","sources":["../../../../src/operations/shims/operation-api-shim.ts"],"names":[],"mappings":";;AAGA,SAAwB,gBAAgB,CAAC,OAAgB,EAAE,OAAa,EAAE;IACtE,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE;QAC/B,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC;QACvB,OAAO,CAAC,IAAI,CAAC,gBAAgB,CAAC,aAAa,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;IAC3D,CAAC,CAAC,CAAC;AACP,CAAC;AALD,mCAKC"}
@@ -1,4 +0,0 @@
1
- import { LegacyProcessor } from '../../interfaces';
2
- import { ProcessorModule } from '../interfaces';
3
- export default function processorShim<S extends Record<string, any>>(legacy: LegacyProcessor): ProcessorModule;
4
- //# sourceMappingURL=processor-shim.d.ts.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"processor-shim.d.ts","sourceRoot":"","sources":["../../../../src/operations/shims/processor-shim.ts"],"names":[],"mappings":"AAGA,OAAO,EAEH,eAAe,EAIlB,MAAM,kBAAkB,CAAC;AAG1B,OAAO,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AAGhD,MAAM,CAAC,OAAO,UAAU,aAAa,CAAC,CAAC,SAAS,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,EAC/D,MAAM,EAAE,eAAe,GACxB,eAAe,CA+CjB"}
@@ -1,51 +0,0 @@
1
- "use strict";
2
- /* eslint-disable max-classes-per-file */
3
- var __importDefault = (this && this.__importDefault) || function (mod) {
4
- return (mod && mod.__esModule) ? mod : { "default": mod };
5
- };
6
- Object.defineProperty(exports, "__esModule", { value: true });
7
- const utils_1 = require("@terascope/utils");
8
- const processor_core_1 = __importDefault(require("../core/processor-core"));
9
- const convict_schema_1 = __importDefault(require("../convict-schema"));
10
- const shim_utils_1 = require("./shim-utils");
11
- function processorShim(legacy) {
12
- return {
13
- Processor: class LegacyProcessorShim extends processor_core_1.default {
14
- async initialize() {
15
- this.processorFn = await legacy.newProcessor(this.context, this.opConfig, this.executionConfig);
16
- }
17
- async handle(input, sliceRequest) {
18
- if (this.processorFn != null) {
19
- const result = await this.processorFn(input, this.logger, sliceRequest);
20
- try {
21
- return (0, shim_utils_1.convertResult)(result);
22
- }
23
- catch (err) {
24
- throw new Error(`${this.opConfig._op} failed to convert result: ${(0, utils_1.toString)(err)}`);
25
- }
26
- }
27
- throw new Error('Processor has not been initialized');
28
- }
29
- },
30
- Schema: class LegacySchemaShim extends convict_schema_1.default {
31
- validate(inputConfig) {
32
- const opConfig = super.validate(inputConfig);
33
- if (legacy.selfValidation) {
34
- // @ts-expect-error
35
- legacy.selfValidation(opConfig);
36
- }
37
- return opConfig;
38
- }
39
- validateJob(job) {
40
- if (legacy.crossValidation) {
41
- legacy.crossValidation(job, this.context.sysconfig);
42
- }
43
- }
44
- build(context) {
45
- return legacy.schema(context);
46
- }
47
- }
48
- };
49
- }
50
- exports.default = processorShim;
51
- //# sourceMappingURL=processor-shim.js.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"processor-shim.js","sourceRoot":"","sources":["../../../../src/operations/shims/processor-shim.ts"],"names":[],"mappings":";AAAA,yCAAyC;;;;;AAEzC,4CAAwD;AAQxD,4EAAmD;AACnD,uEAA8C;AAE9C,6CAA6C;AAE7C,SAAwB,aAAa,CACjC,MAAuB;IAEvB,OAAO;QACH,SAAS,EAAE,MAAM,mBAA6C,SAAQ,wBAAgB;YAGlF,KAAK,CAAC,UAAU;gBACZ,IAAI,CAAC,WAAW,GAAG,MAAM,MAAM,CAAC,YAAY,CACxC,IAAI,CAAC,OAAO,EACZ,IAAI,CAAC,QAAQ,EACb,IAAI,CAAC,eAAe,CACvB,CAAC;YACN,CAAC;YAED,KAAK,CAAC,MAAM,CAAC,KAAmB,EAAE,YAA0B;gBACxD,IAAI,IAAI,CAAC,WAAW,IAAI,IAAI,EAAE;oBAC1B,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,KAAK,EAAE,IAAI,CAAC,MAAM,EAAE,YAAY,CAAC,CAAC;oBACxE,IAAI;wBACA,OAAO,IAAA,0BAAa,EAAC,MAAM,CAAC,CAAC;qBAChC;oBAAC,OAAO,GAAG,EAAE;wBACV,MAAM,IAAI,KAAK,CAAC,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,8BAA8B,IAAA,gBAAQ,EAAC,GAAG,CAAC,EAAE,CAAC,CAAC;qBACtF;iBACJ;gBAED,MAAM,IAAI,KAAK,CAAC,oCAAoC,CAAC,CAAC;YAC1D,CAAC;SACJ;QACD,MAAM,EAAE,MAAM,gBAAiB,SAAQ,wBAAgB;YACnD,QAAQ,CAAC,WAAgC;gBACrC,MAAM,QAAQ,GAAG,KAAK,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC;gBAC7C,IAAI,MAAM,CAAC,cAAc,EAAE;oBACvB,mBAAmB;oBACnB,MAAM,CAAC,cAAc,CAAC,QAAQ,CAAC,CAAC;iBACnC;gBACD,OAAO,QAAQ,CAAC;YACpB,CAAC;YAED,WAAW,CAAC,GAAuB;gBAC/B,IAAI,MAAM,CAAC,eAAe,EAAE;oBACxB,MAAM,CAAC,eAAe,CAAC,GAAG,EAAE,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;iBACvD;YACL,CAAC;YAED,KAAK,CAAC,OAAiB;gBACnB,OAAO,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;YAClC,CAAC;SACJ;KACJ,CAAC;AACN,CAAC;AAjDD,gCAiDC"}
@@ -1,4 +0,0 @@
1
- import { LegacyReader } from '../../interfaces';
2
- import { ReaderModule } from '../interfaces';
3
- export default function readerShim<S extends Record<string, any>>(legacy: LegacyReader): ReaderModule;
4
- //# sourceMappingURL=reader-shim.d.ts.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"reader-shim.d.ts","sourceRoot":"","sources":["../../../../src/operations/shims/reader-shim.ts"],"names":[],"mappings":"AAKA,OAAO,EAGH,YAAY,EAMf,MAAM,kBAAkB,CAAC;AAI1B,OAAO,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;AAG7C,MAAM,CAAC,OAAO,UAAU,UAAU,CAAC,CAAC,SAAS,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,EAC5D,MAAM,EAAE,YAAY,GACrB,YAAY,CAgGd"}
@@ -1,92 +0,0 @@
1
- "use strict";
2
- /* eslint-disable max-classes-per-file */
3
- var __importDefault = (this && this.__importDefault) || function (mod) {
4
- return (mod && mod.__esModule) ? mod : { "default": mod };
5
- };
6
- Object.defineProperty(exports, "__esModule", { value: true });
7
- const utils_1 = require("@terascope/utils");
8
- const fetcher_core_1 = __importDefault(require("../core/fetcher-core"));
9
- const parallel_slicer_1 = __importDefault(require("../parallel-slicer"));
10
- const convict_schema_1 = __importDefault(require("../convict-schema"));
11
- const shim_utils_1 = require("./shim-utils");
12
- function readerShim(legacy) {
13
- return {
14
- Slicer: class LegacySlicerShim extends parallel_slicer_1.default {
15
- constructor() {
16
- super(...arguments);
17
- this._maxQueueLength = 10000;
18
- this._dynamicQueueLength = false;
19
- }
20
- /** legacy slicers should recoverable by default */
21
- isRecoverable() {
22
- return true;
23
- }
24
- async initialize(recoveryData) {
25
- const executionContext = {
26
- config: this.executionConfig,
27
- };
28
- if ((0, utils_1.isFunction)(legacy.slicerQueueLength)) {
29
- const result = await legacy.slicerQueueLength(executionContext);
30
- if (result === 'QUEUE_MINIMUM_SIZE') {
31
- this._maxQueueLength = this.executionConfig.workers;
32
- this._dynamicQueueLength = true;
33
- }
34
- else if ((0, utils_1.isInteger)(result) && result > 0) {
35
- this._maxQueueLength = result;
36
- }
37
- }
38
- this.slicerFns = await legacy.newSlicer(this.context, executionContext, recoveryData, this.logger);
39
- await super.initialize(recoveryData);
40
- }
41
- async newSlicer() {
42
- if (this.slicerFns == null) {
43
- throw new Error('Slicer has not been initialized');
44
- }
45
- return this.slicerFns.shift();
46
- }
47
- maxQueueLength() {
48
- if (this._dynamicQueueLength) {
49
- return this.workersConnected;
50
- }
51
- return this._maxQueueLength;
52
- }
53
- },
54
- Fetcher: class LegacyFetcherShim extends fetcher_core_1.default {
55
- async initialize() {
56
- this.fetcherFn = await legacy.newReader(this.context, this.opConfig, this.executionConfig);
57
- }
58
- async handle(sliceRequest) {
59
- if (this.fetcherFn) {
60
- const result = await this.fetcherFn(sliceRequest, this.logger);
61
- try {
62
- return (0, shim_utils_1.convertResult)(result);
63
- }
64
- catch (err) {
65
- throw new Error(`${this.opConfig._op} failed to convert result: ${(0, utils_1.toString)(err)}`);
66
- }
67
- }
68
- throw new Error('Fetcher has not been initialized');
69
- }
70
- },
71
- Schema: class LegacySchemaShim extends convict_schema_1.default {
72
- validate(inputConfig) {
73
- const opConfig = super.validate(inputConfig);
74
- if (legacy.selfValidation) {
75
- // @ts-expect-error
76
- legacy.selfValidation(opConfig);
77
- }
78
- return opConfig;
79
- }
80
- validateJob(job) {
81
- if (legacy.crossValidation) {
82
- legacy.crossValidation(job, this.context.sysconfig);
83
- }
84
- }
85
- build(context) {
86
- return legacy.schema(context);
87
- }
88
- },
89
- };
90
- }
91
- exports.default = readerShim;
92
- //# sourceMappingURL=reader-shim.js.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"reader-shim.js","sourceRoot":"","sources":["../../../../src/operations/shims/reader-shim.ts"],"names":[],"mappings":";AAAA,yCAAyC;;;;;AAEzC,4CAE0B;AAW1B,wEAA+C;AAC/C,yEAAgD;AAChD,uEAA8C;AAE9C,6CAA6C;AAE7C,SAAwB,UAAU,CAC9B,MAAoB;IAEpB,OAAO;QACH,MAAM,EAAE,MAAM,gBAA0C,SAAQ,yBAAiB;YAAzE;;gBACI,oBAAe,GAAG,KAAK,CAAC;gBACxB,wBAAmB,GAAG,KAAK,CAAC;YA8CxC,CAAC;YA3CG,mDAAmD;YACnD,aAAa;gBACT,OAAO,IAAI,CAAC;YAChB,CAAC;YAED,KAAK,CAAC,UAAU,CAAC,YAAkC;gBAC/C,MAAM,gBAAgB,GAAG;oBACrB,MAAM,EAAE,IAAI,CAAC,eAAe;iBACL,CAAC;gBAE5B,IAAI,IAAA,kBAAU,EAAC,MAAM,CAAC,iBAAiB,CAAC,EAAE;oBACtC,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,iBAAiB,CAAC,gBAAgB,CAAC,CAAC;oBAChE,IAAI,MAAM,KAAK,oBAAoB,EAAE;wBACjC,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,eAAe,CAAC,OAAO,CAAC;wBACpD,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC;qBACnC;yBAAM,IAAI,IAAA,iBAAS,EAAC,MAAM,CAAC,IAAI,MAAM,GAAG,CAAC,EAAE;wBACxC,IAAI,CAAC,eAAe,GAAG,MAAM,CAAC;qBACjC;iBACJ;gBAED,IAAI,CAAC,SAAS,GAAG,MAAM,MAAM,CAAC,SAAS,CACnC,IAAI,CAAC,OAAO,EACZ,gBAAgB,EAChB,YAAY,EACZ,IAAI,CAAC,MAAM,CACd,CAAC;gBAEF,MAAM,KAAK,CAAC,UAAU,CAAC,YAAY,CAAC,CAAC;YACzC,CAAC;YAED,KAAK,CAAC,SAAS;gBACX,IAAI,IAAI,CAAC,SAAS,IAAI,IAAI,EAAE;oBACxB,MAAM,IAAI,KAAK,CAAC,iCAAiC,CAAC,CAAC;iBACtD;gBACD,OAAO,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,CAAC;YAClC,CAAC;YAED,cAAc;gBACV,IAAI,IAAI,CAAC,mBAAmB,EAAE;oBAC1B,OAAO,IAAI,CAAC,gBAAgB,CAAC;iBAChC;gBACD,OAAO,IAAI,CAAC,eAAe,CAAC;YAChC,CAAC;SACJ;QACD,OAAO,EAAE,MAAM,iBAA2C,SAAQ,sBAAc;YAG5E,KAAK,CAAC,UAAU;gBACZ,IAAI,CAAC,SAAS,GAAG,MAAM,MAAM,CAAC,SAAS,CACnC,IAAI,CAAC,OAAO,EACZ,IAAI,CAAC,QAAQ,EACb,IAAI,CAAC,eAAe,CACvB,CAAC;YACN,CAAC;YAED,KAAK,CAAC,MAAM,CAAC,YAA0B;gBACnC,IAAI,IAAI,CAAC,SAAS,EAAE;oBAChB,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,YAAY,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;oBAC/D,IAAI;wBACA,OAAO,IAAA,0BAAa,EAAC,MAAM,CAAC,CAAC;qBAChC;oBAAC,OAAO,GAAG,EAAE;wBACV,MAAM,IAAI,KAAK,CAAC,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,8BAA8B,IAAA,gBAAQ,EAAC,GAAG,CAAC,EAAE,CAAC,CAAC;qBACtF;iBACJ;gBAED,MAAM,IAAI,KAAK,CAAC,kCAAkC,CAAC,CAAC;YACxD,CAAC;SACJ;QACD,MAAM,EAAE,MAAM,gBAAiB,SAAQ,wBAAgB;YACnD,QAAQ,CAAC,WAAgC;gBACrC,MAAM,QAAQ,GAAG,KAAK,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC;gBAC7C,IAAI,MAAM,CAAC,cAAc,EAAE;oBACvB,mBAAmB;oBACnB,MAAM,CAAC,cAAc,CAAC,QAAQ,CAAC,CAAC;iBACnC;gBACD,OAAO,QAAQ,CAAC;YACpB,CAAC;YAED,WAAW,CAAC,GAAuB;gBAC/B,IAAI,MAAM,CAAC,eAAe,EAAE;oBACxB,MAAM,CAAC,eAAe,CAAC,GAAG,EAAE,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;iBACvD;YACL,CAAC;YAED,KAAK,CAAC,OAAiB;gBACnB,OAAO,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;YAClC,CAAC;SACJ;KACJ,CAAC;AACN,CAAC;AAlGD,6BAkGC"}
@@ -1,4 +0,0 @@
1
- import { LegacyProcessor } from '../../interfaces';
2
- import { SchemaModule } from '../interfaces';
3
- export default function schemaShim<S extends Record<string, any>>(legacy: LegacyProcessor): SchemaModule;
4
- //# sourceMappingURL=schema-shim.d.ts.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"schema-shim.d.ts","sourceRoot":"","sources":["../../../../src/operations/shims/schema-shim.ts"],"names":[],"mappings":"AAAA,OAAO,EAAW,eAAe,EAAsB,MAAM,kBAAkB,CAAC;AAEhF,OAAO,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;AAE7C,MAAM,CAAC,OAAO,UAAU,UAAU,CAAC,CAAC,SAAS,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,EAC5D,MAAM,EAAE,eAAe,GACxB,YAAY,CAwBd"}
@@ -1,31 +0,0 @@
1
- "use strict";
2
- var __importDefault = (this && this.__importDefault) || function (mod) {
3
- return (mod && mod.__esModule) ? mod : { "default": mod };
4
- };
5
- Object.defineProperty(exports, "__esModule", { value: true });
6
- const convict_schema_1 = __importDefault(require("../convict-schema"));
7
- function schemaShim(legacy) {
8
- return {
9
- Schema: class LegacySchemaShim extends convict_schema_1.default {
10
- // @ts-expect-error
11
- validate(inputConfig) {
12
- const opConfig = super.validate(inputConfig);
13
- if (legacy.selfValidation) {
14
- // @ts-expect-error
15
- legacy.selfValidation(opConfig);
16
- }
17
- return opConfig;
18
- }
19
- validateJob(job) {
20
- if (legacy.crossValidation) {
21
- legacy.crossValidation(job, this.context.sysconfig);
22
- }
23
- }
24
- build(context) {
25
- return legacy.schema(context);
26
- }
27
- }
28
- };
29
- }
30
- exports.default = schemaShim;
31
- //# sourceMappingURL=schema-shim.js.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"schema-shim.js","sourceRoot":"","sources":["../../../../src/operations/shims/schema-shim.ts"],"names":[],"mappings":";;;;;AACA,uEAA8C;AAG9C,SAAwB,UAAU,CAC9B,MAAuB;IAEvB,OAAO;QACH,MAAM,EAAE,MAAM,gBAAiB,SAAQ,wBAAgB;YACnD,mBAAmB;YACnB,QAAQ,CAAC,WAAgB;gBACrB,MAAM,QAAQ,GAAG,KAAK,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC;gBAC7C,IAAI,MAAM,CAAC,cAAc,EAAE;oBACvB,mBAAmB;oBACnB,MAAM,CAAC,cAAc,CAAC,QAAQ,CAAC,CAAC;iBACnC;gBACD,OAAO,QAAQ,CAAC;YACpB,CAAC;YAED,WAAW,CAAC,GAAuB;gBAC/B,IAAI,MAAM,CAAC,eAAe,EAAE;oBACxB,MAAM,CAAC,eAAe,CAAC,GAAG,EAAE,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;iBACvD;YACL,CAAC;YAED,KAAK,CAAC,OAAiB;gBACnB,OAAO,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;YAClC,CAAC;SACJ;KACJ,CAAC;AACN,CAAC;AA1BD,6BA0BC"}
@@ -1,9 +0,0 @@
1
- /// <reference types="node" />
2
- import { DataEntity, DataInput } from '@terascope/utils';
3
- /**
4
- * Convert legacy processor results into DataEntities if possible.
5
- * But in order to be more backwards compatible legacy modules
6
- * can return an array of buffers or strings.
7
- */
8
- export declare function convertResult(input: DataInput[] | Buffer[] | string[]): DataEntity[];
9
- //# sourceMappingURL=shim-utils.d.ts.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"shim-utils.d.ts","sourceRoot":"","sources":["../../../../src/operations/shims/shim-utils.ts"],"names":[],"mappings":";AACA,OAAO,EACH,UAAU,EACV,SAAS,EAKZ,MAAM,kBAAkB,CAAC;AAO1B;;;;EAIE;AACF,wBAAgB,aAAa,CAAC,KAAK,EAAE,SAAS,EAAE,GAAC,MAAM,EAAE,GAAC,MAAM,EAAE,GAAG,UAAU,EAAE,CAgBhF"}
@@ -1,34 +0,0 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.convertResult = void 0;
4
- const util_1 = require("util");
5
- const utils_1 = require("@terascope/utils");
6
- const deprecateType = (0, util_1.deprecate)((result) => (0, utils_1.castArray)(result), 'Legacy processors should return an array of Objects or DataEntities');
7
- /**
8
- * Convert legacy processor results into DataEntities if possible.
9
- * But in order to be more backwards compatible legacy modules
10
- * can return an array of buffers or strings.
11
- */
12
- function convertResult(input) {
13
- if (input == null)
14
- return [];
15
- if (Array.isArray(input) && input.length === 0)
16
- return [];
17
- if (utils_1.DataEntity.isDataEntityArray(input))
18
- return input;
19
- if (utils_1.DataEntity.isDataEntity(input))
20
- return [input];
21
- const first = (0, utils_1.getFirst)(input);
22
- if (first == null)
23
- return [];
24
- // @ts-expect-error
25
- if (Array.isArray(first))
26
- return input;
27
- if (Buffer.isBuffer(first) || (0, utils_1.isString)(first))
28
- return deprecateType(input);
29
- if ((0, utils_1.isPlainObject)(first))
30
- return utils_1.DataEntity.makeArray(input);
31
- throw new Error('Invalid return type for processor');
32
- }
33
- exports.convertResult = convertResult;
34
- //# sourceMappingURL=shim-utils.js.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"shim-utils.js","sourceRoot":"","sources":["../../../../src/operations/shims/shim-utils.ts"],"names":[],"mappings":";;;AAAA,+BAAiC;AACjC,4CAO0B;AAE1B,MAAM,aAAa,GAAG,IAAA,gBAAS,EAC3B,CAAC,MAAW,EAAgB,EAAE,CAAC,IAAA,iBAAS,EAAa,MAAM,CAAC,EAC5D,qEAAqE,CACxE,CAAC;AAEF;;;;EAIE;AACF,SAAgB,aAAa,CAAC,KAAoC;IAC9D,IAAI,KAAK,IAAI,IAAI;QAAE,OAAO,EAAE,CAAC;IAC7B,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IAE1D,IAAI,kBAAU,CAAC,iBAAiB,CAAC,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC;IACtD,IAAI,kBAAU,CAAC,YAAY,CAAC,KAAK,CAAC;QAAE,OAAO,CAAC,KAAK,CAAC,CAAC;IACnD,MAAM,KAAK,GAAG,IAAA,gBAAQ,EAAoC,KAAK,CAAC,CAAC;IACjE,IAAI,KAAK,IAAI,IAAI;QAAE,OAAO,EAAE,CAAC;IAE7B,mBAAmB;IACnB,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC;IAEvC,IAAI,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,IAAA,gBAAQ,EAAC,KAAK,CAAC;QAAE,OAAO,aAAa,CAAC,KAAK,CAAC,CAAC;IAC3E,IAAI,IAAA,qBAAa,EAAC,KAAK,CAAC;QAAE,OAAO,kBAAU,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;IAE7D,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;AACzD,CAAC;AAhBD,sCAgBC"}