@terascope/job-components 2.5.0 → 2.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"job-validator.d.ts","sourceRoot":"","sources":["../../src/job-validator.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,SAAS,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAC;AAC7D,OAAO,EAAE,OAAO,EAAY,kBAAkB,EAAE,MAAM,cAAc,CAAC;
|
|
1
|
+
{"version":3,"file":"job-validator.d.ts","sourceRoot":"","sources":["../../src/job-validator.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,SAAS,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAC;AAC7D,OAAO,EAAE,OAAO,EAAY,kBAAkB,EAAE,MAAM,cAAc,CAAC;AAUrE,qBAAa,YAAY;IACd,MAAM,EAAE,cAAc,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;IAC1C,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAU;IAClC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAkB;gBAE/B,OAAO,EAAE,OAAO;IAS5B,sFAAsF;IAChF,cAAc,CAChB,OAAO,EAAE,OAAO,CAAC,SAAS,CAAC,SAAS,GAAG,SAAS,CAAC,eAAe,CAAC,GAClE,OAAO,CAAC,kBAAkB,CAAC;IA4H9B,SAAS,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,EAAE,IAAI,EAAE,MAAM,GAAG,IAAI;CAO1D"}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { cloneDeep, pMap, get, isNil, has } from '@terascope/core-utils';
|
|
1
|
+
import { cloneDeep, pMap, get, isNil, has, isTest } from '@terascope/core-utils';
|
|
2
2
|
import { validateJobConfig } from './config-validators.js';
|
|
3
3
|
import { jobSchema } from './job-schemas.js';
|
|
4
4
|
import { OperationLoader, parseName } from './operation-loader/index.js';
|
|
@@ -11,7 +11,7 @@ export class JobValidator {
|
|
|
11
11
|
this.context = context;
|
|
12
12
|
this.opLoader = new OperationLoader({
|
|
13
13
|
assetPath: context.sysconfig.teraslice.assets_directory,
|
|
14
|
-
validate_name_collisions: true
|
|
14
|
+
validate_name_collisions: true,
|
|
15
15
|
});
|
|
16
16
|
this.schema = jobSchema(context);
|
|
17
17
|
}
|
|
@@ -52,12 +52,17 @@ export class JobValidator {
|
|
|
52
52
|
}
|
|
53
53
|
return validatedConfig;
|
|
54
54
|
};
|
|
55
|
+
// jest has an issues with dynamic imports using node experimental vm
|
|
56
|
+
// so we need to load the operations sequentially in test mode to avoid that issue
|
|
57
|
+
// https://github.com/terascope/standard-assets/issues/861
|
|
58
|
+
const concurrency = isTest ? 1 : jobConfig.operations.length;
|
|
59
|
+
const apiConcurrency = isTest ? 1 : jobConfig.apis.length;
|
|
55
60
|
jobConfig.operations = await pMap(jobConfig.operations, async (opConfig, index) => {
|
|
56
61
|
if (index === 0) {
|
|
57
62
|
return handleModule(opConfig, await this.opLoader.loadReader(opConfig._op, assetIds), index);
|
|
58
63
|
}
|
|
59
64
|
return handleModule(opConfig, await this.opLoader.loadProcessor(opConfig._op, assetIds), index);
|
|
60
|
-
});
|
|
65
|
+
}, { concurrency });
|
|
61
66
|
jobConfig.apis = await pMap(jobConfig.apis, async (apiConfig, index) => {
|
|
62
67
|
const { Schema: ApiSchema } = await this.opLoader.loadAPI(apiConfig._name, assetIds);
|
|
63
68
|
const schema = new ApiSchema(this.context, 'api');
|
|
@@ -77,7 +82,7 @@ export class JobValidator {
|
|
|
77
82
|
// this can mutate the job
|
|
78
83
|
validateApisFns.forEach((fn) => {
|
|
79
84
|
fn(jobConfig);
|
|
80
|
-
});
|
|
85
|
+
}, { concurrency: apiConcurrency });
|
|
81
86
|
const apiNames = jobConfig.apis.map((api) => api._name);
|
|
82
87
|
for (const [key] of opAPIMapping.entries()) {
|
|
83
88
|
if (!apiNames.includes(key)) {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"job-validator.js","sourceRoot":"","sources":["../../src/job-validator.ts"],"names":[],"mappings":"AAAA,OAAO,EACH,SAAS,EAAE,IAAI,EAAE,GAAG,EAAE,KAAK,
|
|
1
|
+
{"version":3,"file":"job-validator.js","sourceRoot":"","sources":["../../src/job-validator.ts"],"names":[],"mappings":"AAAA,OAAO,EACH,SAAS,EAAE,IAAI,EAAE,GAAG,EAAE,KAAK,EAAE,GAAG,EAChC,MAAM,EACT,MAAM,uBAAuB,CAAC;AAG/B,OAAO,EAAE,iBAAiB,EAAE,MAAM,wBAAwB,CAAC;AAC3D,OAAO,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAC7C,OAAO,EAAE,eAAe,EAAE,SAAS,EAAE,MAAM,6BAA6B,CAAC;AACzE,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAMlD,MAAM,OAAO,YAAY;IACd,MAAM,CAA6B;IACzB,OAAO,CAAU;IACjB,QAAQ,CAAkB;IAE3C,YAAY,OAAgB;QACxB,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;QACvB,IAAI,CAAC,QAAQ,GAAG,IAAI,eAAe,CAAC;YAChC,SAAS,EAAE,OAAO,CAAC,SAAS,CAAC,SAAS,CAAC,gBAAgB;YACvD,wBAAwB,EAAE,IAAI;SACjC,CAAC,CAAC;QACH,IAAI,CAAC,MAAM,GAAG,SAAS,CAAC,OAAO,CAAC,CAAC;IACrC,CAAC;IAED,sFAAsF;IACtF,KAAK,CAAC,cAAc,CAChB,OAAiE;QAEjE,sDAAsD;QACtD,MAAM,SAAS,GAAG,iBAAiB,CAAC,IAAI,CAAC,MAAM,EAAE,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC;QACrE,MAAM,QAAQ,GAAG,SAAS,CAAC,MAAM,IAAI,EAAE,CAAC;QACxC,MAAM,IAAI,GAA4C,EAAE,CAAC;QAGzD,MAAM,cAAc,GAAoB,EAAE,CAAC;QAC3C,MAAM,eAAe,GAAoB,EAAE,CAAC;QAC5C,MAAM,YAAY,GAAG,IAAI,GAAG,EAAkB,CAAC;QAE/C,MAAM,YAAY,GAAG,CACjB,QAAkB,EAClB,EAAmB,EACnB,KAAa,EACf,EAAE;YACA,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,GAAG,EAAE,GAAG,EAAE,CAAC;YAErC,IAAI,GAAG,IAAI,IAAI,EAAE,CAAC;gBACd,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,GAAG,CAAC;YAC7B,CAAC;YAED,MAAM,MAAM,GAAG,IAAI,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YAE1C,cAAc,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE;gBACxB,IAAI,CAAC,MAAM,CAAC,WAAW;oBAAE,OAAO;gBAEhC,MAAM,YAAY,GAAG,QAAQ,CAAC,GAAG,CAAC;gBAClC,MAAM,EAAE,IAAI,EAAE,GAAG,SAAS,CAAC,YAAY,CAAC,CAAC;gBAEzC,iEAAiE;gBACjE,GAAG,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,GAAG,GAAG,IAAI,CAAC;gBACjC,MAAM,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;gBACxB,+BAA+B;gBAC/B,GAAG,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,GAAG,GAAG,YAAY,CAAC;YAC7C,CAAC,CAAC,CAAC;YAEH,MAAM,eAAe,GAAG,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;YAElD,MAAM,QAAQ,GAAG,GAAG,CAAC,eAAe,EAAE,WAAW,CAAC,IAAI,GAAG,CAAC,eAAe,EAAE,UAAU,CAAC,CAAC;YAEvF,IAAI,QAAQ,EAAE,CAAC;gBACX,MAAM,OAAO,GAAG,GAAG,CAAC,eAAe,EAAE,WAAW,EAAE,IAAI,CAAC,IAAI,GAAG,CAAC,eAAe,EAAE,UAAU,EAAE,IAAI,CAAC,CAAC;gBAElG,IAAI,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC;oBACjB,MAAM,IAAI,KAAK,CAAC,4FAA4F,CAAC,CAAC;gBAClH,CAAC;gBAED,YAAY,CAAC,GAAG,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;YACvC,CAAC;YAED,OAAO,eAAe,CAAC;QAC3B,CAAC,CAAC;QAEF,qEAAqE;QACrE,kFAAkF;QAClF,0DAA0D;QAC1D,MAAM,WAAW,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,UAAU,CAAC,MAAM,CAAC;QAC7D,MAAM,cAAc,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,IAAI,CAAC,MAAM,CAAC;QAE1D,SAAS,CAAC,UAAU,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,UAAU,EAAE,KAAK,EAAE,QAAQ,EAAE,KAAK,EAAE,EAAE;YAC9E,IAAI,KAAK,KAAK,CAAC,EAAE,CAAC;gBACd,OAAO,YAAY,CACf,QAAQ,EACR,MAAM,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC,QAAQ,CAAC,GAAG,EAAE,QAAQ,CAAC,EACtD,KAAK,CACR,CAAC;YACN,CAAC;YAED,OAAO,YAAY,CACf,QAAQ,EACR,MAAM,IAAI,CAAC,QAAQ,CAAC,aAAa,CAAC,QAAQ,CAAC,GAAG,EAAE,QAAQ,CAAC,EACzD,KAAK,CACR,CAAC;QACN,CAAC,EAAE,EAAE,WAAW,EAAE,CACjB,CAAC;QAEF,SAAS,CAAC,IAAI,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,KAAK,EAAE,SAAS,EAAE,KAAK,EAAE,EAAE;YACnE,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,SAAS,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;YACrF,MAAM,MAAM,GAAG,IAAI,SAAS,CAAC,IAAI,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;YAElD,eAAe,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE;gBACzB,IAAI,CAAC,MAAM,CAAC,WAAW;oBAAE,OAAO;gBAEhC,MAAM,YAAY,GAAG,SAAS,CAAC,KAAK,CAAC;gBACrC,MAAM,EAAE,IAAI,EAAE,GAAG,SAAS,CAAC,YAAY,CAAC,CAAC;gBAEzC,iEAAiE;gBACjE,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,KAAK,GAAG,IAAI,CAAC;gBAC7B,MAAM,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;gBACxB,+BAA+B;gBAC/B,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,KAAK,GAAG,YAAY,CAAC;YACzC,CAAC,CAAC,CAAC;YAEH,OAAO,MAAM,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC;QACtC,CAAC,CAAC,CAAC;QAEH,0BAA0B;QAC1B,eAAe,CAAC,OAAO,CAAC,CAAC,EAAE,EAAE,EAAE;YAC3B,EAAE,CAAC,SAAS,CAAC,CAAC;QAClB,CAAC,EAAE,EAAE,WAAW,EAAE,cAAc,EAAE,CAAC,CAAC;QAEpC,MAAM,QAAQ,GAAG,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QAExD,KAAK,MAAM,CAAC,GAAG,CAAC,IAAI,YAAY,CAAC,OAAO,EAAE,EAAE,CAAC;YACzC,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;gBAC1B,MAAM,IAAI,KAAK,CAAC,yCAAyC,GAAG,EAAE,CAAC,CAAC;YACpE,CAAC;QACL,CAAC;QAED,cAAc,CAAC,OAAO,CAAC,CAAC,EAAE,EAAE,EAAE;YAC1B,EAAE,CAAC,SAAS,CAAC,CAAC;QAClB,CAAC,CAAC,CAAC;QAEH,YAAY,CAAC,IAAI,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC;QAEtC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE;YAC/B,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC;YACvB,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,gBAAgB,CAAC,aAAa,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;QAChE,CAAC,CAAC,CAAC;QAEH,OAAO,SAAS,CAAC;IACrB,CAAC;IAED,SAAS,CAAC,GAAwB,EAAE,IAAY;QAC5C,IAAI,CAAC,GAAG,CAAC,MAAM,IAAI,OAAO,GAAG,CAAC,MAAM,KAAK,UAAU,EAAE,CAAC;YAClD,MAAM,IAAI,KAAK,CAAC,GAAG,IAAI,wCAAwC,CAAC,CAAC;QACrE,CAAC;aAAM,IAAI,OAAO,GAAG,CAAC,MAAM,EAAE,KAAK,QAAQ,EAAE,CAAC;YAC1C,MAAM,IAAI,KAAK,CAAC,GAAG,IAAI,mCAAmC,CAAC,CAAC;QAChE,CAAC;IACL,CAAC;CACJ"}
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@terascope/job-components",
|
|
3
3
|
"displayName": "Job Components",
|
|
4
|
-
"version": "2.
|
|
4
|
+
"version": "2.6.0",
|
|
5
5
|
"description": "A teraslice library for validating jobs schemas, registering apis, and defining and running new Job APIs",
|
|
6
6
|
"homepage": "https://github.com/terascope/teraslice/tree/master/packages/job-components#readme",
|
|
7
7
|
"bugs": {
|
|
@@ -28,7 +28,7 @@
|
|
|
28
28
|
"prom-client": "~15.1.3",
|
|
29
29
|
"semver": "~7.7.4",
|
|
30
30
|
"uuid": "~13.0.0",
|
|
31
|
-
"@terascope/core-utils": "~2.5.
|
|
31
|
+
"@terascope/core-utils": "~2.5.1",
|
|
32
32
|
"@terascope/types": "~2.5.0"
|
|
33
33
|
},
|
|
34
34
|
"devDependencies": {
|