@terascope/job-components 2.5.1 → 2.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"job-validator.d.ts","sourceRoot":"","sources":["../../src/job-validator.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,SAAS,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAC;AAC7D,OAAO,EAAE,OAAO,EAAY,kBAAkB,EAAE,MAAM,cAAc,CAAC;
|
|
1
|
+
{"version":3,"file":"job-validator.d.ts","sourceRoot":"","sources":["../../src/job-validator.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,SAAS,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAC;AAC7D,OAAO,EAAE,OAAO,EAAY,kBAAkB,EAAE,MAAM,cAAc,CAAC;AAUrE,qBAAa,YAAY;IACd,MAAM,EAAE,cAAc,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;IAC1C,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAU;IAClC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAkB;gBAE/B,OAAO,EAAE,OAAO;IAS5B,sFAAsF;IAChF,cAAc,CAChB,OAAO,EAAE,OAAO,CAAC,SAAS,CAAC,SAAS,GAAG,SAAS,CAAC,eAAe,CAAC,GAClE,OAAO,CAAC,kBAAkB,CAAC;IA4H9B,SAAS,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,EAAE,IAAI,EAAE,MAAM,GAAG,IAAI;CAO1D"}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { cloneDeep, pMap, get, isNil, has } from '@terascope/core-utils';
|
|
1
|
+
import { cloneDeep, pMap, get, isNil, has, isTest } from '@terascope/core-utils';
|
|
2
2
|
import { validateJobConfig } from './config-validators.js';
|
|
3
3
|
import { jobSchema } from './job-schemas.js';
|
|
4
4
|
import { OperationLoader, parseName } from './operation-loader/index.js';
|
|
@@ -11,7 +11,7 @@ export class JobValidator {
|
|
|
11
11
|
this.context = context;
|
|
12
12
|
this.opLoader = new OperationLoader({
|
|
13
13
|
assetPath: context.sysconfig.teraslice.assets_directory,
|
|
14
|
-
validate_name_collisions: true
|
|
14
|
+
validate_name_collisions: true,
|
|
15
15
|
});
|
|
16
16
|
this.schema = jobSchema(context);
|
|
17
17
|
}
|
|
@@ -52,12 +52,17 @@ export class JobValidator {
|
|
|
52
52
|
}
|
|
53
53
|
return validatedConfig;
|
|
54
54
|
};
|
|
55
|
+
// jest has an issues with dynamic imports using node experimental vm
|
|
56
|
+
// so we need to load the operations sequentially in test mode to avoid that issue
|
|
57
|
+
// https://github.com/terascope/standard-assets/issues/861
|
|
58
|
+
const concurrency = isTest ? 1 : jobConfig.operations.length;
|
|
59
|
+
const apiConcurrency = isTest ? 1 : jobConfig.apis.length;
|
|
55
60
|
jobConfig.operations = await pMap(jobConfig.operations, async (opConfig, index) => {
|
|
56
61
|
if (index === 0) {
|
|
57
62
|
return handleModule(opConfig, await this.opLoader.loadReader(opConfig._op, assetIds), index);
|
|
58
63
|
}
|
|
59
64
|
return handleModule(opConfig, await this.opLoader.loadProcessor(opConfig._op, assetIds), index);
|
|
60
|
-
});
|
|
65
|
+
}, { concurrency });
|
|
61
66
|
jobConfig.apis = await pMap(jobConfig.apis, async (apiConfig, index) => {
|
|
62
67
|
const { Schema: ApiSchema } = await this.opLoader.loadAPI(apiConfig._name, assetIds);
|
|
63
68
|
const schema = new ApiSchema(this.context, 'api');
|
|
@@ -77,7 +82,7 @@ export class JobValidator {
|
|
|
77
82
|
// this can mutate the job
|
|
78
83
|
validateApisFns.forEach((fn) => {
|
|
79
84
|
fn(jobConfig);
|
|
80
|
-
});
|
|
85
|
+
}, { concurrency: apiConcurrency });
|
|
81
86
|
const apiNames = jobConfig.apis.map((api) => api._name);
|
|
82
87
|
for (const [key] of opAPIMapping.entries()) {
|
|
83
88
|
if (!apiNames.includes(key)) {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"job-validator.js","sourceRoot":"","sources":["../../src/job-validator.ts"],"names":[],"mappings":"AAAA,OAAO,EACH,SAAS,EAAE,IAAI,EAAE,GAAG,EAAE,KAAK,
|
|
1
|
+
{"version":3,"file":"job-validator.js","sourceRoot":"","sources":["../../src/job-validator.ts"],"names":[],"mappings":"AAAA,OAAO,EACH,SAAS,EAAE,IAAI,EAAE,GAAG,EAAE,KAAK,EAAE,GAAG,EAChC,MAAM,EACT,MAAM,uBAAuB,CAAC;AAG/B,OAAO,EAAE,iBAAiB,EAAE,MAAM,wBAAwB,CAAC;AAC3D,OAAO,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAC7C,OAAO,EAAE,eAAe,EAAE,SAAS,EAAE,MAAM,6BAA6B,CAAC;AACzE,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAMlD,MAAM,OAAO,YAAY;IACd,MAAM,CAA6B;IACzB,OAAO,CAAU;IACjB,QAAQ,CAAkB;IAE3C,YAAY,OAAgB;QACxB,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;QACvB,IAAI,CAAC,QAAQ,GAAG,IAAI,eAAe,CAAC;YAChC,SAAS,EAAE,OAAO,CAAC,SAAS,CAAC,SAAS,CAAC,gBAAgB;YACvD,wBAAwB,EAAE,IAAI;SACjC,CAAC,CAAC;QACH,IAAI,CAAC,MAAM,GAAG,SAAS,CAAC,OAAO,CAAC,CAAC;IACrC,CAAC;IAED,sFAAsF;IACtF,KAAK,CAAC,cAAc,CAChB,OAAiE;QAEjE,sDAAsD;QACtD,MAAM,SAAS,GAAG,iBAAiB,CAAC,IAAI,CAAC,MAAM,EAAE,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC;QACrE,MAAM,QAAQ,GAAG,SAAS,CAAC,MAAM,IAAI,EAAE,CAAC;QACxC,MAAM,IAAI,GAA4C,EAAE,CAAC;QAGzD,MAAM,cAAc,GAAoB,EAAE,CAAC;QAC3C,MAAM,eAAe,GAAoB,EAAE,CAAC;QAC5C,MAAM,YAAY,GAAG,IAAI,GAAG,EAAkB,CAAC;QAE/C,MAAM,YAAY,GAAG,CACjB,QAAkB,EAClB,EAAmB,EACnB,KAAa,EACf,EAAE;YACA,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,GAAG,EAAE,GAAG,EAAE,CAAC;YAErC,IAAI,GAAG,IAAI,IAAI,EAAE,CAAC;gBACd,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,GAAG,CAAC;YAC7B,CAAC;YAED,MAAM,MAAM,GAAG,IAAI,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YAE1C,cAAc,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE;gBACxB,IAAI,CAAC,MAAM,CAAC,WAAW;oBAAE,OAAO;gBAEhC,MAAM,YAAY,GAAG,QAAQ,CAAC,GAAG,CAAC;gBAClC,MAAM,EAAE,IAAI,EAAE,GAAG,SAAS,CAAC,YAAY,CAAC,CAAC;gBAEzC,iEAAiE;gBACjE,GAAG,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,GAAG,GAAG,IAAI,CAAC;gBACjC,MAAM,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;gBACxB,+BAA+B;gBAC/B,GAAG,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,GAAG,GAAG,YAAY,CAAC;YAC7C,CAAC,CAAC,CAAC;YAEH,MAAM,eAAe,GAAG,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;YAElD,MAAM,QAAQ,GAAG,GAAG,CAAC,eAAe,EAAE,WAAW,CAAC,IAAI,GAAG,CAAC,eAAe,EAAE,UAAU,CAAC,CAAC;YAEvF,IAAI,QAAQ,EAAE,CAAC;gBACX,MAAM,OAAO,GAAG,GAAG,CAAC,eAAe,EAAE,WAAW,EAAE,IAAI,CAAC,IAAI,GAAG,CAAC,eAAe,EAAE,UAAU,EAAE,IAAI,CAAC,CAAC;gBAElG,IAAI,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC;oBACjB,MAAM,IAAI,KAAK,CAAC,4FAA4F,CAAC,CAAC;gBAClH,CAAC;gBAED,YAAY,CAAC,GAAG,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;YACvC,CAAC;YAED,OAAO,eAAe,CAAC;QAC3B,CAAC,CAAC;QAEF,qEAAqE;QACrE,kFAAkF;QAClF,0DAA0D;QAC1D,MAAM,WAAW,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,UAAU,CAAC,MAAM,CAAC;QAC7D,MAAM,cAAc,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,IAAI,CAAC,MAAM,CAAC;QAE1D,SAAS,CAAC,UAAU,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,UAAU,EAAE,KAAK,EAAE,QAAQ,EAAE,KAAK,EAAE,EAAE;YAC9E,IAAI,KAAK,KAAK,CAAC,EAAE,CAAC;gBACd,OAAO,YAAY,CACf,QAAQ,EACR,MAAM,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC,QAAQ,CAAC,GAAG,EAAE,QAAQ,CAAC,EACtD,KAAK,CACR,CAAC;YACN,CAAC;YAED,OAAO,YAAY,CACf,QAAQ,EACR,MAAM,IAAI,CAAC,QAAQ,CAAC,aAAa,CAAC,QAAQ,CAAC,GAAG,EAAE,QAAQ,CAAC,EACzD,KAAK,CACR,CAAC;QACN,CAAC,EAAE,EAAE,WAAW,EAAE,CACjB,CAAC;QAEF,SAAS,CAAC,IAAI,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,KAAK,EAAE,SAAS,EAAE,KAAK,EAAE,EAAE;YACnE,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,SAAS,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;YACrF,MAAM,MAAM,GAAG,IAAI,SAAS,CAAC,IAAI,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;YAElD,eAAe,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE;gBACzB,IAAI,CAAC,MAAM,CAAC,WAAW;oBAAE,OAAO;gBAEhC,MAAM,YAAY,GAAG,SAAS,CAAC,KAAK,CAAC;gBACrC,MAAM,EAAE,IAAI,EAAE,GAAG,SAAS,CAAC,YAAY,CAAC,CAAC;gBAEzC,iEAAiE;gBACjE,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,KAAK,GAAG,IAAI,CAAC;gBAC7B,MAAM,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;gBACxB,+BAA+B;gBAC/B,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,KAAK,GAAG,YAAY,CAAC;YACzC,CAAC,CAAC,CAAC;YAEH,OAAO,MAAM,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC;QACtC,CAAC,CAAC,CAAC;QAEH,0BAA0B;QAC1B,eAAe,CAAC,OAAO,CAAC,CAAC,EAAE,EAAE,EAAE;YAC3B,EAAE,CAAC,SAAS,CAAC,CAAC;QAClB,CAAC,EAAE,EAAE,WAAW,EAAE,cAAc,EAAE,CAAC,CAAC;QAEpC,MAAM,QAAQ,GAAG,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QAExD,KAAK,MAAM,CAAC,GAAG,CAAC,IAAI,YAAY,CAAC,OAAO,EAAE,EAAE,CAAC;YACzC,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;gBAC1B,MAAM,IAAI,KAAK,CAAC,yCAAyC,GAAG,EAAE,CAAC,CAAC;YACpE,CAAC;QACL,CAAC;QAED,cAAc,CAAC,OAAO,CAAC,CAAC,EAAE,EAAE,EAAE;YAC1B,EAAE,CAAC,SAAS,CAAC,CAAC;QAClB,CAAC,CAAC,CAAC;QAEH,YAAY,CAAC,IAAI,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC;QAEtC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE;YAC/B,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC;YACvB,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,gBAAgB,CAAC,aAAa,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;QAChE,CAAC,CAAC,CAAC;QAEH,OAAO,SAAS,CAAC;IACrB,CAAC;IAED,SAAS,CAAC,GAAwB,EAAE,IAAY;QAC5C,IAAI,CAAC,GAAG,CAAC,MAAM,IAAI,OAAO,GAAG,CAAC,MAAM,KAAK,UAAU,EAAE,CAAC;YAClD,MAAM,IAAI,KAAK,CAAC,GAAG,IAAI,wCAAwC,CAAC,CAAC;QACrE,CAAC;aAAM,IAAI,OAAO,GAAG,CAAC,MAAM,EAAE,KAAK,QAAQ,EAAE,CAAC;YAC1C,MAAM,IAAI,KAAK,CAAC,GAAG,IAAI,mCAAmC,CAAC,CAAC;QAChE,CAAC;IACL,CAAC;CACJ"}
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@terascope/job-components",
|
|
3
3
|
"displayName": "Job Components",
|
|
4
|
-
"version": "2.
|
|
4
|
+
"version": "2.6.0",
|
|
5
5
|
"description": "A teraslice library for validating jobs schemas, registering apis, and defining and running new Job APIs",
|
|
6
6
|
"homepage": "https://github.com/terascope/teraslice/tree/master/packages/job-components#readme",
|
|
7
7
|
"bugs": {
|