@terascope/job-components 0.75.0 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/builtin/collect/interfaces.d.ts +2 -2
- package/dist/src/builtin/collect/interfaces.d.ts.map +1 -1
- package/dist/src/builtin/collect/interfaces.js +1 -2
- package/dist/src/builtin/collect/processor.d.ts +4 -4
- package/dist/src/builtin/collect/processor.d.ts.map +1 -1
- package/dist/src/builtin/collect/processor.js +5 -7
- package/dist/src/builtin/collect/processor.js.map +1 -1
- package/dist/src/builtin/collect/schema.d.ts +2 -2
- package/dist/src/builtin/collect/schema.d.ts.map +1 -1
- package/dist/src/builtin/collect/schema.js +2 -5
- package/dist/src/builtin/collect/schema.js.map +1 -1
- package/dist/src/builtin/delay/interfaces.d.ts +1 -1
- package/dist/src/builtin/delay/interfaces.d.ts.map +1 -1
- package/dist/src/builtin/delay/interfaces.js +1 -2
- package/dist/src/builtin/delay/processor.d.ts +2 -2
- package/dist/src/builtin/delay/processor.d.ts.map +1 -1
- package/dist/src/builtin/delay/processor.js +4 -7
- package/dist/src/builtin/delay/processor.js.map +1 -1
- package/dist/src/builtin/delay/schema.d.ts +2 -2
- package/dist/src/builtin/delay/schema.d.ts.map +1 -1
- package/dist/src/builtin/delay/schema.js +2 -5
- package/dist/src/builtin/delay/schema.js.map +1 -1
- package/dist/src/builtin/index.d.ts +5 -5
- package/dist/src/builtin/index.d.ts.map +1 -1
- package/dist/src/builtin/index.js +6 -16
- package/dist/src/builtin/index.js.map +1 -1
- package/dist/src/builtin/noop/processor.d.ts +1 -1
- package/dist/src/builtin/noop/processor.d.ts.map +1 -1
- package/dist/src/builtin/noop/processor.js +2 -5
- package/dist/src/builtin/noop/processor.js.map +1 -1
- package/dist/src/builtin/noop/schema.d.ts +1 -1
- package/dist/src/builtin/noop/schema.d.ts.map +1 -1
- package/dist/src/builtin/noop/schema.js +2 -5
- package/dist/src/builtin/noop/schema.js.map +1 -1
- package/dist/src/builtin/test-reader/data/fetcher-data.d.ts +1 -1
- package/dist/src/builtin/test-reader/data/fetcher-data.d.ts.map +1 -1
- package/dist/src/builtin/test-reader/data/fetcher-data.js +1 -2
- package/dist/src/builtin/test-reader/data/fetcher-data.js.map +1 -1
- package/dist/src/builtin/test-reader/data/slicer-data.d.ts +1 -1
- package/dist/src/builtin/test-reader/data/slicer-data.d.ts.map +1 -1
- package/dist/src/builtin/test-reader/data/slicer-data.js +1 -2
- package/dist/src/builtin/test-reader/data/slicer-data.js.map +1 -1
- package/dist/src/builtin/test-reader/fetcher.d.ts +3 -3
- package/dist/src/builtin/test-reader/fetcher.d.ts.map +1 -1
- package/dist/src/builtin/test-reader/fetcher.js +12 -21
- package/dist/src/builtin/test-reader/fetcher.js.map +1 -1
- package/dist/src/builtin/test-reader/interfaces.d.ts +1 -1
- package/dist/src/builtin/test-reader/interfaces.d.ts.map +1 -1
- package/dist/src/builtin/test-reader/interfaces.js +1 -2
- package/dist/src/builtin/test-reader/schema.d.ts +2 -2
- package/dist/src/builtin/test-reader/schema.d.ts.map +1 -1
- package/dist/src/builtin/test-reader/schema.js +2 -5
- package/dist/src/builtin/test-reader/schema.js.map +1 -1
- package/dist/src/builtin/test-reader/slicer.d.ts +3 -3
- package/dist/src/builtin/test-reader/slicer.d.ts.map +1 -1
- package/dist/src/builtin/test-reader/slicer.js +10 -19
- package/dist/src/builtin/test-reader/slicer.js.map +1 -1
- package/dist/src/config-validators.d.ts +1 -1
- package/dist/src/config-validators.d.ts.map +1 -1
- package/dist/src/config-validators.js +10 -19
- package/dist/src/config-validators.js.map +1 -1
- package/dist/src/execution-context/api.d.ts +4 -5
- package/dist/src/execution-context/api.d.ts.map +1 -1
- package/dist/src/execution-context/api.js +16 -16
- package/dist/src/execution-context/api.js.map +1 -1
- package/dist/src/execution-context/base.d.ts +6 -6
- package/dist/src/execution-context/base.d.ts.map +1 -1
- package/dist/src/execution-context/base.js +32 -26
- package/dist/src/execution-context/base.js.map +1 -1
- package/dist/src/execution-context/index.d.ts +9 -12
- package/dist/src/execution-context/index.d.ts.map +1 -1
- package/dist/src/execution-context/index.js +16 -44
- package/dist/src/execution-context/index.js.map +1 -1
- package/dist/src/execution-context/interfaces.d.ts +2 -2
- package/dist/src/execution-context/interfaces.d.ts.map +1 -1
- package/dist/src/execution-context/interfaces.js +1 -2
- package/dist/src/execution-context/slicer.d.ts +5 -4
- package/dist/src/execution-context/slicer.d.ts.map +1 -1
- package/dist/src/execution-context/slicer.js +20 -20
- package/dist/src/execution-context/slicer.js.map +1 -1
- package/dist/src/execution-context/utils.d.ts +1 -1
- package/dist/src/execution-context/utils.d.ts.map +1 -1
- package/dist/src/execution-context/utils.js +5 -11
- package/dist/src/execution-context/utils.js.map +1 -1
- package/dist/src/execution-context/worker.d.ts +6 -5
- package/dist/src/execution-context/worker.d.ts.map +1 -1
- package/dist/src/execution-context/worker.js +50 -68
- package/dist/src/execution-context/worker.js.map +1 -1
- package/dist/src/formats.d.ts +2 -2
- package/dist/src/formats.d.ts.map +1 -1
- package/dist/src/formats.js +21 -27
- package/dist/src/formats.js.map +1 -1
- package/dist/src/index.d.ts +12 -12
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/index.js +13 -29
- package/dist/src/index.js.map +1 -1
- package/dist/src/interfaces/context.d.ts +23 -140
- package/dist/src/interfaces/context.d.ts.map +1 -1
- package/dist/src/interfaces/context.js +1 -2
- package/dist/src/interfaces/index.d.ts +4 -5
- package/dist/src/interfaces/index.d.ts.map +1 -1
- package/dist/src/interfaces/index.js +4 -21
- package/dist/src/interfaces/index.js.map +1 -1
- package/dist/src/interfaces/misc.d.ts +3 -0
- package/dist/src/interfaces/misc.d.ts.map +1 -1
- package/dist/src/interfaces/misc.js +2 -2
- package/dist/src/interfaces/misc.js.map +1 -1
- package/dist/src/interfaces/operation-lifecycle.d.ts +4 -3
- package/dist/src/interfaces/operation-lifecycle.d.ts.map +1 -1
- package/dist/src/interfaces/operation-lifecycle.js +1 -2
- package/dist/src/interfaces/operations.d.ts +11 -64
- package/dist/src/interfaces/operations.d.ts.map +1 -1
- package/dist/src/interfaces/operations.js +1 -4
- package/dist/src/interfaces/operations.js.map +1 -1
- package/dist/src/job-schemas.d.ts +1 -1
- package/dist/src/job-schemas.d.ts.map +1 -1
- package/dist/src/job-schemas.js +32 -39
- package/dist/src/job-schemas.js.map +1 -1
- package/dist/src/job-validator.d.ts +3 -2
- package/dist/src/job-validator.d.ts.map +1 -1
- package/dist/src/job-validator.js +19 -20
- package/dist/src/job-validator.js.map +1 -1
- package/dist/src/operation-loader/index.d.ts +2 -2
- package/dist/src/operation-loader/index.d.ts.map +1 -1
- package/dist/src/operation-loader/index.js +2 -18
- package/dist/src/operation-loader/index.js.map +1 -1
- package/dist/src/operation-loader/interfaces.d.ts +1 -1
- package/dist/src/operation-loader/interfaces.d.ts.map +1 -1
- package/dist/src/operation-loader/interfaces.js +15 -18
- package/dist/src/operation-loader/interfaces.js.map +1 -1
- package/dist/src/operation-loader/loader.d.ts +7 -9
- package/dist/src/operation-loader/loader.d.ts.map +1 -1
- package/dist/src/operation-loader/loader.js +96 -137
- package/dist/src/operation-loader/loader.js.map +1 -1
- package/dist/src/operations/api-factory.d.ts +2 -2
- package/dist/src/operations/api-factory.d.ts.map +1 -1
- package/dist/src/operations/api-factory.js +4 -13
- package/dist/src/operations/api-factory.js.map +1 -1
- package/dist/src/operations/batch-processor.d.ts +2 -2
- package/dist/src/operations/batch-processor.d.ts.map +1 -1
- package/dist/src/operations/batch-processor.js +5 -11
- package/dist/src/operations/batch-processor.js.map +1 -1
- package/dist/src/operations/convict-schema.d.ts +2 -2
- package/dist/src/operations/convict-schema.d.ts.map +1 -1
- package/dist/src/operations/convict-schema.js +11 -15
- package/dist/src/operations/convict-schema.js.map +1 -1
- package/dist/src/operations/core/api-core.d.ts +4 -4
- package/dist/src/operations/core/api-core.d.ts.map +1 -1
- package/dist/src/operations/core/api-core.js +7 -10
- package/dist/src/operations/core/api-core.js.map +1 -1
- package/dist/src/operations/core/core.d.ts +2 -2
- package/dist/src/operations/core/core.d.ts.map +1 -1
- package/dist/src/operations/core/core.js +5 -4
- package/dist/src/operations/core/core.js.map +1 -1
- package/dist/src/operations/core/fetcher-core.d.ts +2 -2
- package/dist/src/operations/core/fetcher-core.d.ts.map +1 -1
- package/dist/src/operations/core/fetcher-core.js +2 -8
- package/dist/src/operations/core/fetcher-core.js.map +1 -1
- package/dist/src/operations/core/index.d.ts +6 -6
- package/dist/src/operations/core/index.d.ts.map +1 -1
- package/dist/src/operations/core/index.js +7 -18
- package/dist/src/operations/core/index.js.map +1 -1
- package/dist/src/operations/core/operation-core.d.ts +5 -5
- package/dist/src/operations/core/operation-core.d.ts.map +1 -1
- package/dist/src/operations/core/operation-core.js +8 -11
- package/dist/src/operations/core/operation-core.js.map +1 -1
- package/dist/src/operations/core/processor-core.d.ts +2 -2
- package/dist/src/operations/core/processor-core.d.ts.map +1 -1
- package/dist/src/operations/core/processor-core.js +2 -8
- package/dist/src/operations/core/processor-core.js.map +1 -1
- package/dist/src/operations/core/schema-core.d.ts +1 -1
- package/dist/src/operations/core/schema-core.d.ts.map +1 -1
- package/dist/src/operations/core/schema-core.js +3 -4
- package/dist/src/operations/core/schema-core.js.map +1 -1
- package/dist/src/operations/core/slicer-core.d.ts +4 -4
- package/dist/src/operations/core/slicer-core.d.ts.map +1 -1
- package/dist/src/operations/core/slicer-core.js +13 -14
- package/dist/src/operations/core/slicer-core.js.map +1 -1
- package/dist/src/operations/each-processor.d.ts +2 -2
- package/dist/src/operations/each-processor.d.ts.map +1 -1
- package/dist/src/operations/each-processor.js +2 -8
- package/dist/src/operations/each-processor.js.map +1 -1
- package/dist/src/operations/fetcher.d.ts +2 -2
- package/dist/src/operations/fetcher.d.ts.map +1 -1
- package/dist/src/operations/fetcher.js +4 -10
- package/dist/src/operations/fetcher.js.map +1 -1
- package/dist/src/operations/filter-processor.d.ts +2 -2
- package/dist/src/operations/filter-processor.d.ts.map +1 -1
- package/dist/src/operations/filter-processor.js +2 -8
- package/dist/src/operations/filter-processor.js.map +1 -1
- package/dist/src/operations/index.d.ts +14 -15
- package/dist/src/operations/index.d.ts.map +1 -1
- package/dist/src/operations/index.js +15 -47
- package/dist/src/operations/index.js.map +1 -1
- package/dist/src/operations/interfaces.d.ts +12 -12
- package/dist/src/operations/interfaces.d.ts.map +1 -1
- package/dist/src/operations/interfaces.js +1 -2
- package/dist/src/operations/job-observer.d.ts +3 -3
- package/dist/src/operations/job-observer.d.ts.map +1 -1
- package/dist/src/operations/job-observer.js +14 -10
- package/dist/src/operations/job-observer.js.map +1 -1
- package/dist/src/operations/map-processor.d.ts +2 -2
- package/dist/src/operations/map-processor.d.ts.map +1 -1
- package/dist/src/operations/map-processor.js +4 -10
- package/dist/src/operations/map-processor.js.map +1 -1
- package/dist/src/operations/observer.d.ts +2 -2
- package/dist/src/operations/observer.d.ts.map +1 -1
- package/dist/src/operations/observer.js +2 -8
- package/dist/src/operations/observer.js.map +1 -1
- package/dist/src/operations/operation-api.d.ts +2 -2
- package/dist/src/operations/operation-api.d.ts.map +1 -1
- package/dist/src/operations/operation-api.js +2 -8
- package/dist/src/operations/operation-api.js.map +1 -1
- package/dist/src/operations/parallel-slicer.d.ts +2 -2
- package/dist/src/operations/parallel-slicer.d.ts.map +1 -1
- package/dist/src/operations/parallel-slicer.js +7 -16
- package/dist/src/operations/parallel-slicer.js.map +1 -1
- package/dist/src/operations/slicer.d.ts +2 -2
- package/dist/src/operations/slicer.d.ts.map +1 -1
- package/dist/src/operations/slicer.js +7 -16
- package/dist/src/operations/slicer.js.map +1 -1
- package/dist/src/register-apis.d.ts +2 -3
- package/dist/src/register-apis.d.ts.map +1 -1
- package/dist/src/register-apis.js +17 -49
- package/dist/src/register-apis.js.map +1 -1
- package/dist/src/test-helpers.d.ts +11 -31
- package/dist/src/test-helpers.d.ts.map +1 -1
- package/dist/src/test-helpers.js +65 -151
- package/dist/src/test-helpers.js.map +1 -1
- package/dist/src/utils.d.ts +1 -1
- package/dist/src/utils.d.ts.map +1 -1
- package/dist/src/utils.js +9 -14
- package/dist/src/utils.js.map +1 -1
- package/package.json +9 -6
- package/dist/src/interfaces/jobs.d.ts +0 -171
- package/dist/src/interfaces/jobs.d.ts.map +0 -1
- package/dist/src/interfaces/jobs.js +0 -10
- package/dist/src/interfaces/jobs.js.map +0 -1
- package/dist/src/operations/shims/index.d.ts +0 -9
- package/dist/src/operations/shims/index.d.ts.map +0 -1
- package/dist/src/operations/shims/index.js +0 -21
- package/dist/src/operations/shims/index.js.map +0 -1
- package/dist/src/operations/shims/legacy-processor-shim.d.ts +0 -5
- package/dist/src/operations/shims/legacy-processor-shim.d.ts.map +0 -1
- package/dist/src/operations/shims/legacy-processor-shim.js +0 -50
- package/dist/src/operations/shims/legacy-processor-shim.js.map +0 -1
- package/dist/src/operations/shims/legacy-reader-shim.d.ts +0 -7
- package/dist/src/operations/shims/legacy-reader-shim.d.ts.map +0 -1
- package/dist/src/operations/shims/legacy-reader-shim.js +0 -66
- package/dist/src/operations/shims/legacy-reader-shim.js.map +0 -1
- package/dist/src/operations/shims/legacy-slice-events-shim.d.ts +0 -10
- package/dist/src/operations/shims/legacy-slice-events-shim.d.ts.map +0 -1
- package/dist/src/operations/shims/legacy-slice-events-shim.js +0 -34
- package/dist/src/operations/shims/legacy-slice-events-shim.js.map +0 -1
- package/dist/src/operations/shims/operation-api-shim.d.ts +0 -7
- package/dist/src/operations/shims/operation-api-shim.d.ts.map +0 -1
- package/dist/src/operations/shims/operation-api-shim.js +0 -10
- package/dist/src/operations/shims/operation-api-shim.js.map +0 -1
- package/dist/src/operations/shims/processor-shim.d.ts +0 -4
- package/dist/src/operations/shims/processor-shim.d.ts.map +0 -1
- package/dist/src/operations/shims/processor-shim.js +0 -51
- package/dist/src/operations/shims/processor-shim.js.map +0 -1
- package/dist/src/operations/shims/reader-shim.d.ts +0 -4
- package/dist/src/operations/shims/reader-shim.d.ts.map +0 -1
- package/dist/src/operations/shims/reader-shim.js +0 -92
- package/dist/src/operations/shims/reader-shim.js.map +0 -1
- package/dist/src/operations/shims/schema-shim.d.ts +0 -4
- package/dist/src/operations/shims/schema-shim.d.ts.map +0 -1
- package/dist/src/operations/shims/schema-shim.js +0 -31
- package/dist/src/operations/shims/schema-shim.js.map +0 -1
- package/dist/src/operations/shims/shim-utils.d.ts +0 -9
- package/dist/src/operations/shims/shim-utils.d.ts.map +0 -1
- package/dist/src/operations/shims/shim-utils.js +0 -34
- package/dist/src/operations/shims/shim-utils.js.map +0 -1
|
@@ -1,7 +0,0 @@
|
|
|
1
|
-
import { Context } from '../../interfaces';
|
|
2
|
-
import { OperationAPIConstructor } from '../interfaces';
|
|
3
|
-
export default function operationAPIShim(context: Context, apis?: APIs): void;
|
|
4
|
-
export interface APIs {
|
|
5
|
-
[name: string]: OperationAPIConstructor;
|
|
6
|
-
}
|
|
7
|
-
//# sourceMappingURL=operation-api-shim.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"operation-api-shim.d.ts","sourceRoot":"","sources":["../../../../src/operations/shims/operation-api-shim.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAC3C,OAAO,EAAE,uBAAuB,EAAE,MAAM,eAAe,CAAC;AAExD,MAAM,CAAC,OAAO,UAAU,gBAAgB,CAAC,OAAO,EAAE,OAAO,EAAE,IAAI,GAAE,IAAS,GAAG,IAAI,CAKhF;AAED,MAAM,WAAW,IAAI;IACjB,CAAC,IAAI,EAAE,MAAM,GAAG,uBAAuB,CAAC;CAC3C"}
|
|
@@ -1,10 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
function operationAPIShim(context, apis = {}) {
|
|
4
|
-
Object.keys(apis).forEach((name) => {
|
|
5
|
-
const api = apis[name];
|
|
6
|
-
context.apis.executionContext.addToRegistry(name, api);
|
|
7
|
-
});
|
|
8
|
-
}
|
|
9
|
-
exports.default = operationAPIShim;
|
|
10
|
-
//# sourceMappingURL=operation-api-shim.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"operation-api-shim.js","sourceRoot":"","sources":["../../../../src/operations/shims/operation-api-shim.ts"],"names":[],"mappings":";;AAGA,SAAwB,gBAAgB,CAAC,OAAgB,EAAE,OAAa,EAAE;IACtE,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE;QAC/B,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC;QACvB,OAAO,CAAC,IAAI,CAAC,gBAAgB,CAAC,aAAa,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;IAC3D,CAAC,CAAC,CAAC;AACP,CAAC;AALD,mCAKC"}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"processor-shim.d.ts","sourceRoot":"","sources":["../../../../src/operations/shims/processor-shim.ts"],"names":[],"mappings":"AAGA,OAAO,EAEH,eAAe,EAIlB,MAAM,kBAAkB,CAAC;AAG1B,OAAO,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AAGhD,MAAM,CAAC,OAAO,UAAU,aAAa,CAAC,CAAC,SAAS,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,EAC/D,MAAM,EAAE,eAAe,GACxB,eAAe,CA+CjB"}
|
|
@@ -1,51 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
/* eslint-disable max-classes-per-file */
|
|
3
|
-
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
4
|
-
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
5
|
-
};
|
|
6
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
7
|
-
const utils_1 = require("@terascope/utils");
|
|
8
|
-
const processor_core_1 = __importDefault(require("../core/processor-core"));
|
|
9
|
-
const convict_schema_1 = __importDefault(require("../convict-schema"));
|
|
10
|
-
const shim_utils_1 = require("./shim-utils");
|
|
11
|
-
function processorShim(legacy) {
|
|
12
|
-
return {
|
|
13
|
-
Processor: class LegacyProcessorShim extends processor_core_1.default {
|
|
14
|
-
async initialize() {
|
|
15
|
-
this.processorFn = await legacy.newProcessor(this.context, this.opConfig, this.executionConfig);
|
|
16
|
-
}
|
|
17
|
-
async handle(input, sliceRequest) {
|
|
18
|
-
if (this.processorFn != null) {
|
|
19
|
-
const result = await this.processorFn(input, this.logger, sliceRequest);
|
|
20
|
-
try {
|
|
21
|
-
return (0, shim_utils_1.convertResult)(result);
|
|
22
|
-
}
|
|
23
|
-
catch (err) {
|
|
24
|
-
throw new Error(`${this.opConfig._op} failed to convert result: ${(0, utils_1.toString)(err)}`);
|
|
25
|
-
}
|
|
26
|
-
}
|
|
27
|
-
throw new Error('Processor has not been initialized');
|
|
28
|
-
}
|
|
29
|
-
},
|
|
30
|
-
Schema: class LegacySchemaShim extends convict_schema_1.default {
|
|
31
|
-
validate(inputConfig) {
|
|
32
|
-
const opConfig = super.validate(inputConfig);
|
|
33
|
-
if (legacy.selfValidation) {
|
|
34
|
-
// @ts-expect-error
|
|
35
|
-
legacy.selfValidation(opConfig);
|
|
36
|
-
}
|
|
37
|
-
return opConfig;
|
|
38
|
-
}
|
|
39
|
-
validateJob(job) {
|
|
40
|
-
if (legacy.crossValidation) {
|
|
41
|
-
legacy.crossValidation(job, this.context.sysconfig);
|
|
42
|
-
}
|
|
43
|
-
}
|
|
44
|
-
build(context) {
|
|
45
|
-
return legacy.schema(context);
|
|
46
|
-
}
|
|
47
|
-
}
|
|
48
|
-
};
|
|
49
|
-
}
|
|
50
|
-
exports.default = processorShim;
|
|
51
|
-
//# sourceMappingURL=processor-shim.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"processor-shim.js","sourceRoot":"","sources":["../../../../src/operations/shims/processor-shim.ts"],"names":[],"mappings":";AAAA,yCAAyC;;;;;AAEzC,4CAAwD;AAQxD,4EAAmD;AACnD,uEAA8C;AAE9C,6CAA6C;AAE7C,SAAwB,aAAa,CACjC,MAAuB;IAEvB,OAAO;QACH,SAAS,EAAE,MAAM,mBAA6C,SAAQ,wBAAgB;YAGlF,KAAK,CAAC,UAAU;gBACZ,IAAI,CAAC,WAAW,GAAG,MAAM,MAAM,CAAC,YAAY,CACxC,IAAI,CAAC,OAAO,EACZ,IAAI,CAAC,QAAQ,EACb,IAAI,CAAC,eAAe,CACvB,CAAC;YACN,CAAC;YAED,KAAK,CAAC,MAAM,CAAC,KAAmB,EAAE,YAA0B;gBACxD,IAAI,IAAI,CAAC,WAAW,IAAI,IAAI,EAAE;oBAC1B,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,KAAK,EAAE,IAAI,CAAC,MAAM,EAAE,YAAY,CAAC,CAAC;oBACxE,IAAI;wBACA,OAAO,IAAA,0BAAa,EAAC,MAAM,CAAC,CAAC;qBAChC;oBAAC,OAAO,GAAG,EAAE;wBACV,MAAM,IAAI,KAAK,CAAC,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,8BAA8B,IAAA,gBAAQ,EAAC,GAAG,CAAC,EAAE,CAAC,CAAC;qBACtF;iBACJ;gBAED,MAAM,IAAI,KAAK,CAAC,oCAAoC,CAAC,CAAC;YAC1D,CAAC;SACJ;QACD,MAAM,EAAE,MAAM,gBAAiB,SAAQ,wBAAgB;YACnD,QAAQ,CAAC,WAAgC;gBACrC,MAAM,QAAQ,GAAG,KAAK,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC;gBAC7C,IAAI,MAAM,CAAC,cAAc,EAAE;oBACvB,mBAAmB;oBACnB,MAAM,CAAC,cAAc,CAAC,QAAQ,CAAC,CAAC;iBACnC;gBACD,OAAO,QAAQ,CAAC;YACpB,CAAC;YAED,WAAW,CAAC,GAAuB;gBAC/B,IAAI,MAAM,CAAC,eAAe,EAAE;oBACxB,MAAM,CAAC,eAAe,CAAC,GAAG,EAAE,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;iBACvD;YACL,CAAC;YAED,KAAK,CAAC,OAAiB;gBACnB,OAAO,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;YAClC,CAAC;SACJ;KACJ,CAAC;AACN,CAAC;AAjDD,gCAiDC"}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"reader-shim.d.ts","sourceRoot":"","sources":["../../../../src/operations/shims/reader-shim.ts"],"names":[],"mappings":"AAKA,OAAO,EAGH,YAAY,EAMf,MAAM,kBAAkB,CAAC;AAI1B,OAAO,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;AAG7C,MAAM,CAAC,OAAO,UAAU,UAAU,CAAC,CAAC,SAAS,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,EAC5D,MAAM,EAAE,YAAY,GACrB,YAAY,CAgGd"}
|
|
@@ -1,92 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
/* eslint-disable max-classes-per-file */
|
|
3
|
-
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
4
|
-
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
5
|
-
};
|
|
6
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
7
|
-
const utils_1 = require("@terascope/utils");
|
|
8
|
-
const fetcher_core_1 = __importDefault(require("../core/fetcher-core"));
|
|
9
|
-
const parallel_slicer_1 = __importDefault(require("../parallel-slicer"));
|
|
10
|
-
const convict_schema_1 = __importDefault(require("../convict-schema"));
|
|
11
|
-
const shim_utils_1 = require("./shim-utils");
|
|
12
|
-
function readerShim(legacy) {
|
|
13
|
-
return {
|
|
14
|
-
Slicer: class LegacySlicerShim extends parallel_slicer_1.default {
|
|
15
|
-
constructor() {
|
|
16
|
-
super(...arguments);
|
|
17
|
-
this._maxQueueLength = 10000;
|
|
18
|
-
this._dynamicQueueLength = false;
|
|
19
|
-
}
|
|
20
|
-
/** legacy slicers should recoverable by default */
|
|
21
|
-
isRecoverable() {
|
|
22
|
-
return true;
|
|
23
|
-
}
|
|
24
|
-
async initialize(recoveryData) {
|
|
25
|
-
const executionContext = {
|
|
26
|
-
config: this.executionConfig,
|
|
27
|
-
};
|
|
28
|
-
if ((0, utils_1.isFunction)(legacy.slicerQueueLength)) {
|
|
29
|
-
const result = await legacy.slicerQueueLength(executionContext);
|
|
30
|
-
if (result === 'QUEUE_MINIMUM_SIZE') {
|
|
31
|
-
this._maxQueueLength = this.executionConfig.workers;
|
|
32
|
-
this._dynamicQueueLength = true;
|
|
33
|
-
}
|
|
34
|
-
else if ((0, utils_1.isInteger)(result) && result > 0) {
|
|
35
|
-
this._maxQueueLength = result;
|
|
36
|
-
}
|
|
37
|
-
}
|
|
38
|
-
this.slicerFns = await legacy.newSlicer(this.context, executionContext, recoveryData, this.logger);
|
|
39
|
-
await super.initialize(recoveryData);
|
|
40
|
-
}
|
|
41
|
-
async newSlicer() {
|
|
42
|
-
if (this.slicerFns == null) {
|
|
43
|
-
throw new Error('Slicer has not been initialized');
|
|
44
|
-
}
|
|
45
|
-
return this.slicerFns.shift();
|
|
46
|
-
}
|
|
47
|
-
maxQueueLength() {
|
|
48
|
-
if (this._dynamicQueueLength) {
|
|
49
|
-
return this.workersConnected;
|
|
50
|
-
}
|
|
51
|
-
return this._maxQueueLength;
|
|
52
|
-
}
|
|
53
|
-
},
|
|
54
|
-
Fetcher: class LegacyFetcherShim extends fetcher_core_1.default {
|
|
55
|
-
async initialize() {
|
|
56
|
-
this.fetcherFn = await legacy.newReader(this.context, this.opConfig, this.executionConfig);
|
|
57
|
-
}
|
|
58
|
-
async handle(sliceRequest) {
|
|
59
|
-
if (this.fetcherFn) {
|
|
60
|
-
const result = await this.fetcherFn(sliceRequest, this.logger);
|
|
61
|
-
try {
|
|
62
|
-
return (0, shim_utils_1.convertResult)(result);
|
|
63
|
-
}
|
|
64
|
-
catch (err) {
|
|
65
|
-
throw new Error(`${this.opConfig._op} failed to convert result: ${(0, utils_1.toString)(err)}`);
|
|
66
|
-
}
|
|
67
|
-
}
|
|
68
|
-
throw new Error('Fetcher has not been initialized');
|
|
69
|
-
}
|
|
70
|
-
},
|
|
71
|
-
Schema: class LegacySchemaShim extends convict_schema_1.default {
|
|
72
|
-
validate(inputConfig) {
|
|
73
|
-
const opConfig = super.validate(inputConfig);
|
|
74
|
-
if (legacy.selfValidation) {
|
|
75
|
-
// @ts-expect-error
|
|
76
|
-
legacy.selfValidation(opConfig);
|
|
77
|
-
}
|
|
78
|
-
return opConfig;
|
|
79
|
-
}
|
|
80
|
-
validateJob(job) {
|
|
81
|
-
if (legacy.crossValidation) {
|
|
82
|
-
legacy.crossValidation(job, this.context.sysconfig);
|
|
83
|
-
}
|
|
84
|
-
}
|
|
85
|
-
build(context) {
|
|
86
|
-
return legacy.schema(context);
|
|
87
|
-
}
|
|
88
|
-
},
|
|
89
|
-
};
|
|
90
|
-
}
|
|
91
|
-
exports.default = readerShim;
|
|
92
|
-
//# sourceMappingURL=reader-shim.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"reader-shim.js","sourceRoot":"","sources":["../../../../src/operations/shims/reader-shim.ts"],"names":[],"mappings":";AAAA,yCAAyC;;;;;AAEzC,4CAE0B;AAW1B,wEAA+C;AAC/C,yEAAgD;AAChD,uEAA8C;AAE9C,6CAA6C;AAE7C,SAAwB,UAAU,CAC9B,MAAoB;IAEpB,OAAO;QACH,MAAM,EAAE,MAAM,gBAA0C,SAAQ,yBAAiB;YAAzE;;gBACI,oBAAe,GAAG,KAAK,CAAC;gBACxB,wBAAmB,GAAG,KAAK,CAAC;YA8CxC,CAAC;YA3CG,mDAAmD;YACnD,aAAa;gBACT,OAAO,IAAI,CAAC;YAChB,CAAC;YAED,KAAK,CAAC,UAAU,CAAC,YAAkC;gBAC/C,MAAM,gBAAgB,GAAG;oBACrB,MAAM,EAAE,IAAI,CAAC,eAAe;iBACL,CAAC;gBAE5B,IAAI,IAAA,kBAAU,EAAC,MAAM,CAAC,iBAAiB,CAAC,EAAE;oBACtC,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,iBAAiB,CAAC,gBAAgB,CAAC,CAAC;oBAChE,IAAI,MAAM,KAAK,oBAAoB,EAAE;wBACjC,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,eAAe,CAAC,OAAO,CAAC;wBACpD,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC;qBACnC;yBAAM,IAAI,IAAA,iBAAS,EAAC,MAAM,CAAC,IAAI,MAAM,GAAG,CAAC,EAAE;wBACxC,IAAI,CAAC,eAAe,GAAG,MAAM,CAAC;qBACjC;iBACJ;gBAED,IAAI,CAAC,SAAS,GAAG,MAAM,MAAM,CAAC,SAAS,CACnC,IAAI,CAAC,OAAO,EACZ,gBAAgB,EAChB,YAAY,EACZ,IAAI,CAAC,MAAM,CACd,CAAC;gBAEF,MAAM,KAAK,CAAC,UAAU,CAAC,YAAY,CAAC,CAAC;YACzC,CAAC;YAED,KAAK,CAAC,SAAS;gBACX,IAAI,IAAI,CAAC,SAAS,IAAI,IAAI,EAAE;oBACxB,MAAM,IAAI,KAAK,CAAC,iCAAiC,CAAC,CAAC;iBACtD;gBACD,OAAO,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,CAAC;YAClC,CAAC;YAED,cAAc;gBACV,IAAI,IAAI,CAAC,mBAAmB,EAAE;oBAC1B,OAAO,IAAI,CAAC,gBAAgB,CAAC;iBAChC;gBACD,OAAO,IAAI,CAAC,eAAe,CAAC;YAChC,CAAC;SACJ;QACD,OAAO,EAAE,MAAM,iBAA2C,SAAQ,sBAAc;YAG5E,KAAK,CAAC,UAAU;gBACZ,IAAI,CAAC,SAAS,GAAG,MAAM,MAAM,CAAC,SAAS,CACnC,IAAI,CAAC,OAAO,EACZ,IAAI,CAAC,QAAQ,EACb,IAAI,CAAC,eAAe,CACvB,CAAC;YACN,CAAC;YAED,KAAK,CAAC,MAAM,CAAC,YAA0B;gBACnC,IAAI,IAAI,CAAC,SAAS,EAAE;oBAChB,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,YAAY,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;oBAC/D,IAAI;wBACA,OAAO,IAAA,0BAAa,EAAC,MAAM,CAAC,CAAC;qBAChC;oBAAC,OAAO,GAAG,EAAE;wBACV,MAAM,IAAI,KAAK,CAAC,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,8BAA8B,IAAA,gBAAQ,EAAC,GAAG,CAAC,EAAE,CAAC,CAAC;qBACtF;iBACJ;gBAED,MAAM,IAAI,KAAK,CAAC,kCAAkC,CAAC,CAAC;YACxD,CAAC;SACJ;QACD,MAAM,EAAE,MAAM,gBAAiB,SAAQ,wBAAgB;YACnD,QAAQ,CAAC,WAAgC;gBACrC,MAAM,QAAQ,GAAG,KAAK,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC;gBAC7C,IAAI,MAAM,CAAC,cAAc,EAAE;oBACvB,mBAAmB;oBACnB,MAAM,CAAC,cAAc,CAAC,QAAQ,CAAC,CAAC;iBACnC;gBACD,OAAO,QAAQ,CAAC;YACpB,CAAC;YAED,WAAW,CAAC,GAAuB;gBAC/B,IAAI,MAAM,CAAC,eAAe,EAAE;oBACxB,MAAM,CAAC,eAAe,CAAC,GAAG,EAAE,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;iBACvD;YACL,CAAC;YAED,KAAK,CAAC,OAAiB;gBACnB,OAAO,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;YAClC,CAAC;SACJ;KACJ,CAAC;AACN,CAAC;AAlGD,6BAkGC"}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"schema-shim.d.ts","sourceRoot":"","sources":["../../../../src/operations/shims/schema-shim.ts"],"names":[],"mappings":"AAAA,OAAO,EAAW,eAAe,EAAsB,MAAM,kBAAkB,CAAC;AAEhF,OAAO,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;AAE7C,MAAM,CAAC,OAAO,UAAU,UAAU,CAAC,CAAC,SAAS,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,EAC5D,MAAM,EAAE,eAAe,GACxB,YAAY,CAwBd"}
|
|
@@ -1,31 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
-
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
-
};
|
|
5
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
-
const convict_schema_1 = __importDefault(require("../convict-schema"));
|
|
7
|
-
function schemaShim(legacy) {
|
|
8
|
-
return {
|
|
9
|
-
Schema: class LegacySchemaShim extends convict_schema_1.default {
|
|
10
|
-
// @ts-expect-error
|
|
11
|
-
validate(inputConfig) {
|
|
12
|
-
const opConfig = super.validate(inputConfig);
|
|
13
|
-
if (legacy.selfValidation) {
|
|
14
|
-
// @ts-expect-error
|
|
15
|
-
legacy.selfValidation(opConfig);
|
|
16
|
-
}
|
|
17
|
-
return opConfig;
|
|
18
|
-
}
|
|
19
|
-
validateJob(job) {
|
|
20
|
-
if (legacy.crossValidation) {
|
|
21
|
-
legacy.crossValidation(job, this.context.sysconfig);
|
|
22
|
-
}
|
|
23
|
-
}
|
|
24
|
-
build(context) {
|
|
25
|
-
return legacy.schema(context);
|
|
26
|
-
}
|
|
27
|
-
}
|
|
28
|
-
};
|
|
29
|
-
}
|
|
30
|
-
exports.default = schemaShim;
|
|
31
|
-
//# sourceMappingURL=schema-shim.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"schema-shim.js","sourceRoot":"","sources":["../../../../src/operations/shims/schema-shim.ts"],"names":[],"mappings":";;;;;AACA,uEAA8C;AAG9C,SAAwB,UAAU,CAC9B,MAAuB;IAEvB,OAAO;QACH,MAAM,EAAE,MAAM,gBAAiB,SAAQ,wBAAgB;YACnD,mBAAmB;YACnB,QAAQ,CAAC,WAAgB;gBACrB,MAAM,QAAQ,GAAG,KAAK,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC;gBAC7C,IAAI,MAAM,CAAC,cAAc,EAAE;oBACvB,mBAAmB;oBACnB,MAAM,CAAC,cAAc,CAAC,QAAQ,CAAC,CAAC;iBACnC;gBACD,OAAO,QAAQ,CAAC;YACpB,CAAC;YAED,WAAW,CAAC,GAAuB;gBAC/B,IAAI,MAAM,CAAC,eAAe,EAAE;oBACxB,MAAM,CAAC,eAAe,CAAC,GAAG,EAAE,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;iBACvD;YACL,CAAC;YAED,KAAK,CAAC,OAAiB;gBACnB,OAAO,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;YAClC,CAAC;SACJ;KACJ,CAAC;AACN,CAAC;AA1BD,6BA0BC"}
|
|
@@ -1,9 +0,0 @@
|
|
|
1
|
-
/// <reference types="node" />
|
|
2
|
-
import { DataEntity, DataInput } from '@terascope/utils';
|
|
3
|
-
/**
|
|
4
|
-
* Convert legacy processor results into DataEntities if possible.
|
|
5
|
-
* But in order to be more backwards compatible legacy modules
|
|
6
|
-
* can return an array of buffers or strings.
|
|
7
|
-
*/
|
|
8
|
-
export declare function convertResult(input: DataInput[] | Buffer[] | string[]): DataEntity[];
|
|
9
|
-
//# sourceMappingURL=shim-utils.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"shim-utils.d.ts","sourceRoot":"","sources":["../../../../src/operations/shims/shim-utils.ts"],"names":[],"mappings":";AACA,OAAO,EACH,UAAU,EACV,SAAS,EAKZ,MAAM,kBAAkB,CAAC;AAO1B;;;;EAIE;AACF,wBAAgB,aAAa,CAAC,KAAK,EAAE,SAAS,EAAE,GAAC,MAAM,EAAE,GAAC,MAAM,EAAE,GAAG,UAAU,EAAE,CAgBhF"}
|
|
@@ -1,34 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.convertResult = void 0;
|
|
4
|
-
const util_1 = require("util");
|
|
5
|
-
const utils_1 = require("@terascope/utils");
|
|
6
|
-
const deprecateType = (0, util_1.deprecate)((result) => (0, utils_1.castArray)(result), 'Legacy processors should return an array of Objects or DataEntities');
|
|
7
|
-
/**
|
|
8
|
-
* Convert legacy processor results into DataEntities if possible.
|
|
9
|
-
* But in order to be more backwards compatible legacy modules
|
|
10
|
-
* can return an array of buffers or strings.
|
|
11
|
-
*/
|
|
12
|
-
function convertResult(input) {
|
|
13
|
-
if (input == null)
|
|
14
|
-
return [];
|
|
15
|
-
if (Array.isArray(input) && input.length === 0)
|
|
16
|
-
return [];
|
|
17
|
-
if (utils_1.DataEntity.isDataEntityArray(input))
|
|
18
|
-
return input;
|
|
19
|
-
if (utils_1.DataEntity.isDataEntity(input))
|
|
20
|
-
return [input];
|
|
21
|
-
const first = (0, utils_1.getFirst)(input);
|
|
22
|
-
if (first == null)
|
|
23
|
-
return [];
|
|
24
|
-
// @ts-expect-error
|
|
25
|
-
if (Array.isArray(first))
|
|
26
|
-
return input;
|
|
27
|
-
if (Buffer.isBuffer(first) || (0, utils_1.isString)(first))
|
|
28
|
-
return deprecateType(input);
|
|
29
|
-
if ((0, utils_1.isPlainObject)(first))
|
|
30
|
-
return utils_1.DataEntity.makeArray(input);
|
|
31
|
-
throw new Error('Invalid return type for processor');
|
|
32
|
-
}
|
|
33
|
-
exports.convertResult = convertResult;
|
|
34
|
-
//# sourceMappingURL=shim-utils.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"shim-utils.js","sourceRoot":"","sources":["../../../../src/operations/shims/shim-utils.ts"],"names":[],"mappings":";;;AAAA,+BAAiC;AACjC,4CAO0B;AAE1B,MAAM,aAAa,GAAG,IAAA,gBAAS,EAC3B,CAAC,MAAW,EAAgB,EAAE,CAAC,IAAA,iBAAS,EAAa,MAAM,CAAC,EAC5D,qEAAqE,CACxE,CAAC;AAEF;;;;EAIE;AACF,SAAgB,aAAa,CAAC,KAAoC;IAC9D,IAAI,KAAK,IAAI,IAAI;QAAE,OAAO,EAAE,CAAC;IAC7B,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IAE1D,IAAI,kBAAU,CAAC,iBAAiB,CAAC,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC;IACtD,IAAI,kBAAU,CAAC,YAAY,CAAC,KAAK,CAAC;QAAE,OAAO,CAAC,KAAK,CAAC,CAAC;IACnD,MAAM,KAAK,GAAG,IAAA,gBAAQ,EAAoC,KAAK,CAAC,CAAC;IACjE,IAAI,KAAK,IAAI,IAAI;QAAE,OAAO,EAAE,CAAC;IAE7B,mBAAmB;IACnB,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC;IAEvC,IAAI,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,IAAA,gBAAQ,EAAC,KAAK,CAAC;QAAE,OAAO,aAAa,CAAC,KAAK,CAAC,CAAC;IAC3E,IAAI,IAAA,qBAAa,EAAC,KAAK,CAAC;QAAE,OAAO,kBAAU,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;IAE7D,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;AACzD,CAAC;AAhBD,sCAgBC"}
|