@skedulo/pulse-solution-services 0.0.13 → 0.0.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +4 -0
- package/README.md +2 -0
- package/dist/core/request-header-constants.js +1 -1
- package/dist/index.d.ts +1 -0
- package/dist/services/graphql/graphql-service.js +1 -1
- package/package.json +1 -1
- package/yarn.lock +12 -4
- package/dist/interfaces/locking-service-interfaces.js +0 -1
- package/dist/monitoring/monitor-call.decorator.js +0 -1
- package/dist/services/batch-processor/batch-orchestrator.d.ts +0 -18
- package/dist/services/batch-processor/batch-orchestrator.js +0 -1
- package/dist/services/batch-processor/constants.d.ts +0 -22
- package/dist/services/batch-processor/constants.js +0 -1
- package/dist/services/batch-processor/fetching-strategies/batch-number-fetching-strategy.d.ts +0 -6
- package/dist/services/batch-processor/fetching-strategies/batch-number-fetching-strategy.js +0 -1
- package/dist/services/batch-processor/fetching-strategies/cursor-based-fetching-strategy.d.ts +0 -8
- package/dist/services/batch-processor/fetching-strategies/cursor-based-fetching-strategy.js +0 -1
- package/dist/services/batch-processor/fetching-strategies/date-based-fetching-strategy.d.ts +0 -4
- package/dist/services/batch-processor/fetching-strategies/date-based-fetching-strategy.js +0 -1
- package/dist/services/batch-processor/fetching-strategies/fetching-strategy-factory.d.ts +0 -9
- package/dist/services/batch-processor/fetching-strategies/fetching-strategy-factory.js +0 -1
- package/dist/services/batch-processor/fetching-strategies/index.d.ts +0 -1
- package/dist/services/batch-processor/fetching-strategies/index.js +0 -1
- package/dist/services/batch-processor/index.d.ts +0 -2
- package/dist/services/batch-processor/index.js +0 -1
- package/dist/services/batch-processor/interfaces.d.ts +0 -33
- package/dist/services/batch-processor/interfaces.js +0 -1
- package/dist/services/batch-processor/workers/custom-worker.d.ts +0 -5
- package/dist/services/batch-processor/workers/custom-worker.js +0 -1
- package/dist/services/batch-processor/workers/delete-worker.d.ts +0 -4
- package/dist/services/batch-processor/workers/delete-worker.js +0 -1
- package/dist/services/batch-processor/workers/index.d.ts +0 -2
- package/dist/services/batch-processor/workers/index.js +0 -1
- package/dist/services/batch-processor/workers/update-worker.d.ts +0 -4
- package/dist/services/batch-processor/workers/update-worker.js +0 -1
- package/dist/services/batch-processor/workers/worker-factory.d.ts +0 -6
- package/dist/services/batch-processor/workers/worker-factory.js +0 -1
- package/dist/services/batch-processor/workers/worker-interfaces.d.ts +0 -4
- package/dist/services/batch-processor/workers/worker-interfaces.js +0 -1
- package/dist/services/locking-service.js +0 -1
package/CHANGELOG.md
CHANGED
package/README.md
CHANGED
|
@@ -537,10 +537,12 @@ const updateResult = await context.graphqlService.mutate({
|
|
|
537
537
|
operationName: 'updateResources',
|
|
538
538
|
records: recordsToUpdate,
|
|
539
539
|
bulkOperation: true // optional
|
|
540
|
+
suppressChangeEvents: true // optional
|
|
540
541
|
});
|
|
541
542
|
```
|
|
542
543
|
|
|
543
544
|
The `bulkOperation` and `readOnly` options help optimize large-scale mutations and read-heavy queries. When enabled, they automatically add the `X-Skedulo-Bulk-Operation` (for mutations) and `X-Skedulo-Read-Only` (for queries) headers to improve performance.
|
|
545
|
+
To reduce the impact on change event processing, especially during high-volume mutations, the `suppressChangeEvents` option can be used. This disables change history tracking, helping to minimize delays.
|
|
544
546
|
|
|
545
547
|
#### Pagination with offset
|
|
546
548
|
```javascript
|
|
@@ -1 +1 @@
|
|
|
1
|
-
"use strict";Object.defineProperty(exports,"__esModule",{value:!0}),exports.REQUEST_HEADERS=void 0,exports.REQUEST_HEADERS={X_SKEDULO_BULK_OPERATION:"X-Skedulo-Bulk-Operation",X_SKEDULO_READ_ONLY:"X-Skedulo-Read-Only",X_SKEDULO_SOURCE:"X-Skedulo-Source",X_GRAPHQL_OPERATION:"X-Graphql-Operation",SKED_FUNCTION_EXECUTION_TYPE:"Sked-Function-Execution-Type",SKED_API_SERVER:"sked-api-server",USER_AGENT:"User-Agent"};
|
|
1
|
+
"use strict";Object.defineProperty(exports,"__esModule",{value:!0}),exports.REQUEST_HEADERS=void 0,exports.REQUEST_HEADERS={X_SKEDULO_BULK_OPERATION:"X-Skedulo-Bulk-Operation",X_SKEDULO_READ_ONLY:"X-Skedulo-Read-Only",X_SKEDULO_SOURCE:"X-Skedulo-Source",X_GRAPHQL_OPERATION:"X-Graphql-Operation",SKED_FUNCTION_EXECUTION_TYPE:"Sked-Function-Execution-Type",SKED_API_SERVER:"sked-api-server",USER_AGENT:"User-Agent",X_SKEDULO_SUPPRESS_CHANGE_EVENTS:"X-Skedulo-Suppress-Change-Events"};
|
package/dist/index.d.ts
CHANGED
|
@@ -571,6 +571,7 @@ export interface GraphqlMurationParams extends GraphqlParams {
|
|
|
571
571
|
records: HasId[];
|
|
572
572
|
operation: GraphqlOperations;
|
|
573
573
|
bulkOperation?: boolean;
|
|
574
|
+
suppressChangeEvents?: boolean;
|
|
574
575
|
}
|
|
575
576
|
export interface GraphqlQueryParams extends GraphqlParams {
|
|
576
577
|
first?: number;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
"use strict";var __decorate=this&&this.__decorate||function(e,t,a,r){var o,n=arguments.length,i=n<3?t:null===r?r=Object.getOwnPropertyDescriptor(t,a):r;if("object"==typeof Reflect&&"function"==typeof Reflect.decorate)i=Reflect.decorate(e,t,a,r);else for(var s=e.length-1;s>=0;s--)(o=e[s])&&(i=(n<3?o(i):n>3?o(t,a,i):o(t,a))||i);return n>3&&i&&Object.defineProperty(t,a,i),i},__metadata=this&&this.__metadata||function(e,t){if("object"==typeof Reflect&&"function"==typeof Reflect.metadata)return Reflect.metadata(e,t)},__awaiter=this&&this.__awaiter||function(e,t,a,r){return new(a||(a=Promise))((function(o,n){function i(e){try{d(r.next(e))}catch(e){n(e)}}function s(e){try{d(r.throw(e))}catch(e){n(e)}}function d(e){var t;e.done?o(e.value):(t=e.value,t instanceof a?t:new a((function(e){e(t)}))).then(i,s)}d((r=r.apply(e,t||[])).next())}))};Object.defineProperty(exports,"__esModule",{value:!0}),exports.GraphQLService=void 0;const constants_1=require("../../constants"),request_header_constants_1=require("../../core/request-header-constants"),logging_1=require("../../logging"),queries_1=require("./queries");class GraphQLService{constructor(e){this.client=e}query(e){return __awaiter(this,void 0,void 0,(function*(){var t,a,r,o,n,i,s,d,u,c,
|
|
1
|
+
"use strict";var __decorate=this&&this.__decorate||function(e,t,a,r){var o,n=arguments.length,i=n<3?t:null===r?r=Object.getOwnPropertyDescriptor(t,a):r;if("object"==typeof Reflect&&"function"==typeof Reflect.decorate)i=Reflect.decorate(e,t,a,r);else for(var s=e.length-1;s>=0;s--)(o=e[s])&&(i=(n<3?o(i):n>3?o(t,a,i):o(t,a))||i);return n>3&&i&&Object.defineProperty(t,a,i),i},__metadata=this&&this.__metadata||function(e,t){if("object"==typeof Reflect&&"function"==typeof Reflect.metadata)return Reflect.metadata(e,t)},__awaiter=this&&this.__awaiter||function(e,t,a,r){return new(a||(a=Promise))((function(o,n){function i(e){try{d(r.next(e))}catch(e){n(e)}}function s(e){try{d(r.throw(e))}catch(e){n(e)}}function d(e){var t;e.done?o(e.value):(t=e.value,t instanceof a?t:new a((function(e){e(t)}))).then(i,s)}d((r=r.apply(e,t||[])).next())}))};Object.defineProperty(exports,"__esModule",{value:!0}),exports.GraphQLService=void 0;const constants_1=require("../../constants"),request_header_constants_1=require("../../core/request-header-constants"),logging_1=require("../../logging"),queries_1=require("./queries");class GraphQLService{constructor(e){this.client=e}query(e){return __awaiter(this,void 0,void 0,(function*(){var t,a,r,o,n,i,s,d,u,_,c,l;e.objectName=this.getQueryName(e.objectName);const p=(0,queries_1.FETCH_RECORDS_QUERY)(e),v=this.getQueryHeaders(e),g=yield this.client.execute(p,v),h={records:(null===(a=null===(t=null==g?void 0:g.data)||void 0===t?void 0:t[e.objectName])||void 0===a?void 0:a.edges.map((e=>e.node)))||[],totalCount:(null===(o=null===(r=null==g?void 0:g.data)||void 0===r?void 0:r[e.objectName])||void 0===o?void 0:o.totalCount)||0,pageInfo:(null===(i=null===(n=null==g?void 0:g.data)||void 0===n?void 0:n[e.objectName])||void 0===i?void 0:i.pageInfo)||{}};if(h.records.length>0){const t=(null===(u=null===(d=null===(s=null==g?void 0:g.data)||void 0===s?void 0:s[e.objectName])||void 0===d?void 0:d.edges)||void 0===u?void 0:u.map((e=>e.offset)))||[],a=(null===(l=null===(c=null===(_=null==g?void 0:g.data)||void 0===_?void 0:_[e.objectName])||void 0===c?void 0:c.edges)||void 0===l?void 0:l.map((e=>e.cursor)))||[];h.endOffset=t[t.length-1],h.endCursor=a[a.length-1]}return h}))}mutate(e){return __awaiter(this,void 0,void 0,(function*(){if(!e.records||0===e.records.length)return Promise.resolve({message:"No records to process"});let t="";switch(e.operation){case constants_1.GraphqlOperations.DELETE:t=(0,queries_1.DELETE_OBJECTS_MUTATION)(e);break;case constants_1.GraphqlOperations.UPDATE:case constants_1.GraphqlOperations.INSERT:t=(0,queries_1.UPSERT_OBJECTS_MUTATION)(e);break;default:throw new Error(`Invalid operation: ${e.operation}`)}const a=this.getMutationHeaders(e);return yield this.client.execute(t,a)}))}delete(e){return __awaiter(this,void 0,void 0,(function*(){return e.operation=constants_1.GraphqlOperations.DELETE,this.mutate(e)}))}update(e){return __awaiter(this,void 0,void 0,(function*(){return e.operation=constants_1.GraphqlOperations.UPDATE,this.mutate(e)}))}insert(e){return __awaiter(this,void 0,void 0,(function*(){return e.operation=constants_1.GraphqlOperations.INSERT,this.mutate(e)}))}extractJobUIDs(e){var t;return(null===(t=null==e?void 0:e.data)||void 0===t?void 0:t.schema)?Object.values(e.data.schema):[]}getQueryName(e){return e.charAt(0).toLowerCase()+e.slice(1)}getQueryHeaders(e){const t=this.getOperationHeader(e.operationName);return e.readOnly&&(t[request_header_constants_1.REQUEST_HEADERS.X_SKEDULO_READ_ONLY]="true"),t}getMutationHeaders(e){const t=this.getOperationHeader(e.operationName);return e.bulkOperation&&(t[request_header_constants_1.REQUEST_HEADERS.X_SKEDULO_BULK_OPERATION]="true"),e.suppressChangeEvents&&(t[request_header_constants_1.REQUEST_HEADERS.X_SKEDULO_SUPPRESS_CHANGE_EVENTS]="true"),t}getOperationHeader(e){return{[request_header_constants_1.REQUEST_HEADERS.X_GRAPHQL_OPERATION]:e}}}exports.GraphQLService=GraphQLService,__decorate([(0,logging_1.LogMethod)(),__metadata("design:type",Function),__metadata("design:paramtypes",[Object]),__metadata("design:returntype",Promise)],GraphQLService.prototype,"query",null),__decorate([(0,logging_1.LogMethod)(),__metadata("design:type",Function),__metadata("design:paramtypes",[Object]),__metadata("design:returntype",Promise)],GraphQLService.prototype,"mutate",null),__decorate([(0,logging_1.LogMethod)(),__metadata("design:type",Function),__metadata("design:paramtypes",[Object]),__metadata("design:returntype",Promise)],GraphQLService.prototype,"delete",null),__decorate([(0,logging_1.LogMethod)(),__metadata("design:type",Function),__metadata("design:paramtypes",[Object]),__metadata("design:returntype",Promise)],GraphQLService.prototype,"update",null),__decorate([(0,logging_1.LogMethod)(),__metadata("design:type",Function),__metadata("design:paramtypes",[Object]),__metadata("design:returntype",Promise)],GraphQLService.prototype,"insert",null);
|
package/package.json
CHANGED
package/yarn.lock
CHANGED
|
@@ -105,12 +105,12 @@
|
|
|
105
105
|
integrity sha512-e/zv1co8pp55dNdEcCynfj9X7nyUKUXoUEwfXqaZt0omVOmDe9oOTdKStH4GmAw6zxMFs50ZayuMfHDKlO7Tfw==
|
|
106
106
|
|
|
107
107
|
"@babel/helpers@^7.26.9":
|
|
108
|
-
version "7.26.
|
|
109
|
-
resolved "https://registry.yarnpkg.com/@babel/helpers/-/helpers-7.26.
|
|
110
|
-
integrity sha512-
|
|
108
|
+
version "7.26.10"
|
|
109
|
+
resolved "https://registry.yarnpkg.com/@babel/helpers/-/helpers-7.26.10.tgz#6baea3cd62ec2d0c1068778d63cb1314f6637384"
|
|
110
|
+
integrity sha512-UPYc3SauzZ3JGgj87GgZ89JVdC5dj0AoetR5Bw6wj4niittNyFh6+eOGonYvJ1ao6B8lEa3Q3klS7ADZ53bc5g==
|
|
111
111
|
dependencies:
|
|
112
112
|
"@babel/template" "^7.26.9"
|
|
113
|
-
"@babel/types" "^7.26.
|
|
113
|
+
"@babel/types" "^7.26.10"
|
|
114
114
|
|
|
115
115
|
"@babel/parser@^7.1.0", "@babel/parser@^7.14.7", "@babel/parser@^7.20.7", "@babel/parser@^7.23.9", "@babel/parser@^7.26.9":
|
|
116
116
|
version "7.26.9"
|
|
@@ -268,6 +268,14 @@
|
|
|
268
268
|
"@babel/helper-string-parser" "^7.25.9"
|
|
269
269
|
"@babel/helper-validator-identifier" "^7.25.9"
|
|
270
270
|
|
|
271
|
+
"@babel/types@^7.26.10":
|
|
272
|
+
version "7.26.10"
|
|
273
|
+
resolved "https://registry.yarnpkg.com/@babel/types/-/types-7.26.10.tgz#396382f6335bd4feb65741eacfc808218f859259"
|
|
274
|
+
integrity sha512-emqcG3vHrpxUKTrxcblR36dcrcoRDvKmnL/dCL6ZsHaShW80qxCAcNhzQZrpeM765VzEos+xOi4s+r4IXzTwdQ==
|
|
275
|
+
dependencies:
|
|
276
|
+
"@babel/helper-string-parser" "^7.25.9"
|
|
277
|
+
"@babel/helper-validator-identifier" "^7.25.9"
|
|
278
|
+
|
|
271
279
|
"@bcoe/v8-coverage@^0.2.3":
|
|
272
280
|
version "0.2.3"
|
|
273
281
|
resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39"
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
"use strict";Object.defineProperty(exports,"__esModule",{value:!0});
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
"use strict";var __awaiter=this&&this.__awaiter||function(t,n,r,o){return new(r||(r=Promise))((function(e,i){function a(t){try{c(o.next(t))}catch(t){i(t)}}function u(t){try{c(o.throw(t))}catch(t){i(t)}}function c(t){var n;t.done?e(t.value):(n=t.value,n instanceof r?n:new r((function(t){t(n)}))).then(a,u)}c((o=o.apply(t,n||[])).next())}))};Object.defineProperty(exports,"__esModule",{value:!0}),exports.MonitorCall=MonitorCall;const monitoring_registry_1=require("./monitoring-registry");function MonitorCall(t){return function(n,r,o){const e=o.value;return o.value=function(...o){return __awaiter(this,void 0,void 0,(function*(){const i=t||`${n.constructor.name}.${r}`;return(0,monitoring_registry_1.getGlobalMonitorManager)().measure(i,(()=>__awaiter(this,void 0,void 0,(function*(){return e.apply(this,o)}))))}))},o}}
|
|
@@ -1,18 +0,0 @@
|
|
|
1
|
-
import { ExecutionContext } from "../../core";
|
|
2
|
-
import { BatchOrchestratorConfig } from "./interfaces";
|
|
3
|
-
export declare class BatchOrchestrator {
|
|
4
|
-
protected context: ExecutionContext;
|
|
5
|
-
protected config: BatchOrchestratorConfig;
|
|
6
|
-
constructor(context: ExecutionContext, config?: BatchOrchestratorConfig);
|
|
7
|
-
processBatchJob(): Promise<void>;
|
|
8
|
-
protected validateJob(): void;
|
|
9
|
-
protected prepareData(): Promise<void>;
|
|
10
|
-
protected sendWorkerStateToCallbackUrl(): Promise<void>;
|
|
11
|
-
protected getWorkerState(): object;
|
|
12
|
-
protected delay(ms: number): Promise<void>;
|
|
13
|
-
protected processWorker(): Promise<void>;
|
|
14
|
-
protected processNextBatch(): Promise<void>;
|
|
15
|
-
protected postProcess(): Promise<void>;
|
|
16
|
-
protected updateJobStatus(): Promise<void>;
|
|
17
|
-
private isCompleted;
|
|
18
|
-
}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
"use strict";var __decorate=this&&this.__decorate||function(t,e,o,a){var n,s=arguments.length,c=s<3?e:null===a?a=Object.getOwnPropertyDescriptor(e,o):a;if("object"==typeof Reflect&&"function"==typeof Reflect.decorate)c=Reflect.decorate(t,e,o,a);else for(var r=t.length-1;r>=0;r--)(n=t[r])&&(c=(s<3?n(c):s>3?n(e,o,c):n(e,o))||c);return s>3&&c&&Object.defineProperty(e,o,c),c},__metadata=this&&this.__metadata||function(t,e){if("object"==typeof Reflect&&"function"==typeof Reflect.metadata)return Reflect.metadata(t,e)},__awaiter=this&&this.__awaiter||function(t,e,o,a){return new(o||(o=Promise))((function(n,s){function c(t){try{i(a.next(t))}catch(t){s(t)}}function r(t){try{i(a.throw(t))}catch(t){s(t)}}function i(t){var e;t.done?n(t.value):(e=t.value,e instanceof o?e:new o((function(t){t(e)}))).then(c,r)}i((a=a.apply(t,e||[])).next())}))};Object.defineProperty(exports,"__esModule",{value:!0}),exports.BatchOrchestrator=void 0;const logging_1=require("../../logging"),constants_1=require("./constants"),fetching_strategies_1=require("./fetching-strategies"),workers_1=require("./workers");class BatchOrchestrator{constructor(t,e={isAsyncProcessing:!0}){this.context=t,this.config=e}processBatchJob(){return __awaiter(this,void 0,void 0,(function*(){this.validateJob(),yield this.prepareData(),yield this.processWorker(),yield this.updateJobStatus(),yield this.postProcess()}))}validateJob(){if(!this.context.contextData.job.JobType)throw new Error("JobType is missing in the job configuration.");if(this.context.contextData.job.Status===constants_1.JobStatus.FAILED)throw new Error("Cannot process a failed job.")}prepareData(){return __awaiter(this,void 0,void 0,(function*(){if(this.context.logger.info("Preparing data for the worker"),this.context.contextData.job.CreatedDate||(this.context.contextData.job.CreatedDate=(new Date).toISOString()),this.context.contextData.job.ObjectName){const t=this.context.contextData.job.FetchingStrategy||constants_1.DataFetchingStrategy.DATE_BASED;this.context.contextData.records=yield fetching_strategies_1.FetchingStrategyFactory.getStrategy(t).execute(this.context)}else this.context.logger.info("No input data required for the worker")}))}sendWorkerStateToCallbackUrl(){return __awaiter(this,void 0,void 0,(function*(){this.context.contextData.job.CallbackURL&&(yield fetch(this.context.contextData.job.CallbackURL,{method:"POST",headers:{"Content-Type":"application/json",Authorization:`Bearer ${this.context.skedContext.auth.token}`},body:JSON.stringify(this.getWorkerState())}))}))}getWorkerState(){var t;return{context:{jobId:this.context.contextData.job.UID,status:this.context.contextData.job.Status},data:this.context.contextData.records,state:null===(t=this.context.contextData.batchState)||void 0===t?void 0:t.worker}}delay(t){return __awaiter(this,void 0,void 0,(function*(){return new Promise((e=>setTimeout(e,t)))}))}processWorker(){return __awaiter(this,void 0,void 0,(function*(){this.context.logger.info("Processing with worker");const t=workers_1.WorkerFactory.getWorker(this.context.contextData.job.JobType),e=this.getWorkerState(),o=yield t.process(this.context,e);this.context.contextData.batchState&&(this.context.contextData.batchState.worker=o),this.context.contextData.job.ProcessedBatches=(this.context.contextData.job.ProcessedBatches||0)+1}))}processNextBatch(){return __awaiter(this,void 0,void 0,(function*(){this.config.isAsyncProcessing?this.processBatchJob():yield this.processBatchJob()}))}postProcess(){return __awaiter(this,void 0,void 0,(function*(){this.context.contextData.job.Status!==constants_1.JobStatus.COMPLETED?(this.context.logger.info("Process next batch"),this.context.contextData.job.DelaySeconds>0&&(yield this.delay(1e3*this.context.contextData.job.DelaySeconds)),yield this.processNextBatch()):this.context.contextData.job.CallbackURL&&(this.context.logger.info("Sending worker state to callback URL"),yield this.sendWorkerStateToCallbackUrl())}))}updateJobStatus(){return __awaiter(this,void 0,void 0,(function*(){this.isCompleted()?(this.context.contextData.job.Status=constants_1.JobStatus.COMPLETED,this.context.contextData.job.CompletedAt=(new Date).toISOString()):this.context.contextData.job.Status===constants_1.JobStatus.QUEUED&&(this.context.contextData.job.Status=constants_1.JobStatus.PROCESSING,this.context.contextData.job.ProcessingAt=(new Date).toISOString())}))}isCompleted(){const t=this.context.contextData.job.ProcessedBatches>=this.context.contextData.job.MaximumBatches,e=!this.context.contextData.records||0===this.context.contextData.records.length,o=this.context.contextData.job.JobType===constants_1.JobType.CUSTOM;return t||!o&&e}}exports.BatchOrchestrator=BatchOrchestrator,__decorate([(0,logging_1.LogMethod)(),__metadata("design:type",Function),__metadata("design:paramtypes",[]),__metadata("design:returntype",Promise)],BatchOrchestrator.prototype,"processBatchJob",null);
|
|
@@ -1,22 +0,0 @@
|
|
|
1
|
-
export declare const JobStatus: {
|
|
2
|
-
PROCESSING: string;
|
|
3
|
-
QUEUED: string;
|
|
4
|
-
ABORTED: string;
|
|
5
|
-
COMPLETED: string;
|
|
6
|
-
FAILED: string;
|
|
7
|
-
};
|
|
8
|
-
export declare const JobType: {
|
|
9
|
-
DELETE: string;
|
|
10
|
-
UPDATE: string;
|
|
11
|
-
CUSTOM: string;
|
|
12
|
-
};
|
|
13
|
-
export declare const BATCH_JOBS = "BatchJobs";
|
|
14
|
-
export declare const ConfigVars: {
|
|
15
|
-
FAILURE_THRESHOLD: string;
|
|
16
|
-
MAX_CONCURRENT_JOBS: string;
|
|
17
|
-
BATCH_PROCESSOR_URL: string;
|
|
18
|
-
};
|
|
19
|
-
export declare const DataFetchingStrategy: {
|
|
20
|
-
CURSOR_BASED: string;
|
|
21
|
-
DATE_BASED: string;
|
|
22
|
-
};
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
"use strict";Object.defineProperty(exports,"__esModule",{value:!0}),exports.DataFetchingStrategy=exports.ConfigVars=exports.BATCH_JOBS=exports.JobType=exports.JobStatus=void 0,exports.JobStatus={PROCESSING:"Processing",QUEUED:"Queued",ABORTED:"Aborted",COMPLETED:"Completed",FAILED:"Failed"},exports.JobType={DELETE:"Delete",UPDATE:"Update",CUSTOM:"Custom"},exports.BATCH_JOBS="BatchJobs",exports.ConfigVars={FAILURE_THRESHOLD:"FAILURE_THRESHOLD",MAX_CONCURRENT_JOBS:"MAX_CONCURRENT_JOBS",BATCH_PROCESSOR_URL:"BATCH_PROCESSOR_URL"},exports.DataFetchingStrategy={CURSOR_BASED:"CursorBased",DATE_BASED:"DateBased"};
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
"use strict";var __awaiter=this&&this.__awaiter||function(t,e,n,r){return new(n||(n=Promise))((function(a,c){function i(t){try{u(r.next(t))}catch(t){c(t)}}function o(t){try{u(r.throw(t))}catch(t){c(t)}}function u(t){var e;t.done?a(t.value):(e=t.value,e instanceof n?e:new n((function(t){t(e)}))).then(i,o)}u((r=r.apply(t,e||[])).next())}))};Object.defineProperty(exports,"__esModule",{value:!0}),exports.BatchNumberFetchingStrategy=void 0,exports.BatchNumberFetchingStrategy={execute(t){return __awaiter(this,void 0,void 0,(function*(){return[{UID:t.contextData.job.ProcessedBatches}]}))}};
|
package/dist/services/batch-processor/fetching-strategies/cursor-based-fetching-strategy.d.ts
DELETED
|
@@ -1,8 +0,0 @@
|
|
|
1
|
-
import { ExecutionContext } from "../../../core/execution-context";
|
|
2
|
-
export declare const CursorBasedFetchingStrategy: {
|
|
3
|
-
/**
|
|
4
|
-
* Fetches the next dataset to process based on the job's configuration.
|
|
5
|
-
* Uses cursor if it is not empty; otherwise, fetches the first batch of records.
|
|
6
|
-
*/
|
|
7
|
-
execute(context: ExecutionContext): Promise<import("../../../interfaces/graphql").HasId[]>;
|
|
8
|
-
};
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
"use strict";var __awaiter=this&&this.__awaiter||function(t,e,o,r){return new(o||(o=Promise))((function(a,i){function n(t){try{s(r.next(t))}catch(t){i(t)}}function c(t){try{s(r.throw(t))}catch(t){i(t)}}function s(t){var e;t.done?a(t.value):(e=t.value,e instanceof o?e:new o((function(t){t(e)}))).then(n,c)}s((r=r.apply(t,e||[])).next())}))};Object.defineProperty(exports,"__esModule",{value:!0}),exports.CursorBasedFetchingStrategy=void 0,exports.CursorBasedFetchingStrategy={execute(t){return __awaiter(this,void 0,void 0,(function*(){var e,o,r,a,i,n;const c=t.contextData.job.BatchSize||200,s=(t.contextData.job.Fields||"UID").split(",").map((t=>t.trim()));s.includes("UID")||s.push("UID");let u={objectName:t.contextData.job.ObjectName,operationName:t.contextData.job.ObjectName,first:c,orderBy:t.contextData.job.OrderBy,after:null===(o=null===(e=t.contextData.batchState)||void 0===e?void 0:e.processor)||void 0===o?void 0:o.cursor,filter:t.contextData.job.FilterCondition,fields:s.join("\n")};const d=t.newQueryBuilder(u);d.withFields(s).withLimit(c).withOrderBy(t.contextData.job.OrderBy).withFilter(t.contextData.job.FilterCondition);const l=yield t.graphqlService.query(d.build()),h=l.records,v=l.endCursor;return null===(i=null===(a=null===(r=t.contextData.batchState)||void 0===r?void 0:r.processor)||void 0===a?void 0:a.allCursors)||void 0===i||i.push(v),(null===(n=t.contextData.batchState)||void 0===n?void 0:n.processor)&&(t.contextData.batchState.processor.cursor=v),h}))}};
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
"use strict";var __awaiter=this&&this.__awaiter||function(e,t,i,r){return new(i||(i=Promise))((function(n,a){function o(e){try{s(r.next(e))}catch(e){a(e)}}function c(e){try{s(r.throw(e))}catch(e){a(e)}}function s(e){var t;e.done?n(e.value):(t=e.value,t instanceof i?t:new i((function(e){e(t)}))).then(o,c)}s((r=r.apply(e,t||[])).next())}))};Object.defineProperty(exports,"__esModule",{value:!0}),exports.DateBasedFetchingStrategy=void 0,exports.DateBasedFetchingStrategy={execute(e){return __awaiter(this,void 0,void 0,(function*(){const t=e.contextData.job;let i=`LastModifiedDate <= ${t.CreatedDate} `;t.FilterCondition&&(i+=` AND ${t.FilterCondition}`);const r=t.OrderBy||"CreatedDate ASC",n=t.BatchSize||10,a=(t.Fields||"UID").split(",").map((e=>e.trim()));return(yield e.graphqlService.query({objectName:t.ObjectName,operationName:t.ObjectName,first:n,filter:i,orderBy:r,fields:a.join("\n")})).records}))}};
|
|
@@ -1,9 +0,0 @@
|
|
|
1
|
-
export declare class FetchingStrategyFactory {
|
|
2
|
-
static getStrategy(strategy: string): {
|
|
3
|
-
execute(context: import("../../..").ExecutionContext): Promise<{
|
|
4
|
-
UID: any;
|
|
5
|
-
}[]>;
|
|
6
|
-
} | {
|
|
7
|
-
execute(context: import("../../..").ExecutionContext): Promise<import("../../..").HasId[]>;
|
|
8
|
-
};
|
|
9
|
-
}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
"use strict";Object.defineProperty(exports,"__esModule",{value:!0}),exports.FetchingStrategyFactory=void 0;const constants_1=require("../constants"),batch_number_fetching_strategy_1=require("./batch-number-fetching-strategy"),cursor_based_fetching_strategy_1=require("./cursor-based-fetching-strategy"),date_based_fetching_strategy_1=require("./date-based-fetching-strategy");class FetchingStrategyFactory{static getStrategy(t){switch(t){case constants_1.DataFetchingStrategy.CURSOR_BASED:return cursor_based_fetching_strategy_1.CursorBasedFetchingStrategy;case constants_1.DataFetchingStrategy.DATE_BASED:return date_based_fetching_strategy_1.DateBasedFetchingStrategy;default:return batch_number_fetching_strategy_1.BatchNumberFetchingStrategy}}}exports.FetchingStrategyFactory=FetchingStrategyFactory;
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
export * from "./fetching-strategy-factory";
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
"use strict";var __createBinding=this&&this.__createBinding||(Object.create?function(e,t,r,i){void 0===i&&(i=r);var n=Object.getOwnPropertyDescriptor(t,r);n&&!("get"in n?!t.__esModule:n.writable||n.configurable)||(n={enumerable:!0,get:function(){return t[r]}}),Object.defineProperty(e,i,n)}:function(e,t,r,i){void 0===i&&(i=r),e[i]=t[r]}),__exportStar=this&&this.__exportStar||function(e,t){for(var r in e)"default"===r||Object.prototype.hasOwnProperty.call(t,r)||__createBinding(t,e,r)};Object.defineProperty(exports,"__esModule",{value:!0}),__exportStar(require("./fetching-strategy-factory"),exports);
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
"use strict";var __createBinding=this&&this.__createBinding||(Object.create?function(e,t,r,i){void 0===i&&(i=r);var o=Object.getOwnPropertyDescriptor(t,r);o&&!("get"in o?!t.__esModule:o.writable||o.configurable)||(o={enumerable:!0,get:function(){return t[r]}}),Object.defineProperty(e,i,o)}:function(e,t,r,i){void 0===i&&(i=r),e[i]=t[r]}),__exportStar=this&&this.__exportStar||function(e,t){for(var r in e)"default"===r||Object.prototype.hasOwnProperty.call(t,r)||__createBinding(t,e,r)};Object.defineProperty(exports,"__esModule",{value:!0}),__exportStar(require("./batch-orchestrator"),exports),__exportStar(require("./interfaces"),exports);
|
|
@@ -1,33 +0,0 @@
|
|
|
1
|
-
import { HasId } from "../../interfaces/graphql";
|
|
2
|
-
export interface BatchJobSchemaData extends HasId {
|
|
3
|
-
StatusDetails?: string;
|
|
4
|
-
MaximumBatches?: number;
|
|
5
|
-
Status?: string;
|
|
6
|
-
ProcessedBatches?: number;
|
|
7
|
-
ObjectName?: string;
|
|
8
|
-
JobType?: string;
|
|
9
|
-
BatchProcessor?: string;
|
|
10
|
-
CreatedDate?: string;
|
|
11
|
-
FilterCondition?: string;
|
|
12
|
-
OrderBy?: string;
|
|
13
|
-
Fields?: string;
|
|
14
|
-
JobHandler?: string;
|
|
15
|
-
State?: string;
|
|
16
|
-
DelaySeconds?: number;
|
|
17
|
-
CallbackURL?: string;
|
|
18
|
-
ProcessingAt?: string;
|
|
19
|
-
CompletedAt?: string;
|
|
20
|
-
FailedBatches?: number;
|
|
21
|
-
FetchingStrategy?: string;
|
|
22
|
-
}
|
|
23
|
-
export interface BatchOrchestratorConfig {
|
|
24
|
-
isAsyncProcessing?: boolean;
|
|
25
|
-
}
|
|
26
|
-
export interface BatchState {
|
|
27
|
-
processor?: {
|
|
28
|
-
cursor?: string;
|
|
29
|
-
allCursors?: string[];
|
|
30
|
-
batchNumber?: number;
|
|
31
|
-
};
|
|
32
|
-
worker?: any;
|
|
33
|
-
}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
"use strict";Object.defineProperty(exports,"__esModule",{value:!0});
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
"use strict";var __awaiter=this&&this.__awaiter||function(t,e,o,r){return new(o||(o=Promise))((function(n,a){function i(t){try{u(r.next(t))}catch(t){a(t)}}function s(t){try{u(r.throw(t))}catch(t){a(t)}}function u(t){var e;t.done?n(t.value):(e=t.value,e instanceof o?e:new o((function(t){t(e)}))).then(i,s)}u((r=r.apply(t,e||[])).next())}))};Object.defineProperty(exports,"__esModule",{value:!0}),exports.CustomWorker=void 0;class CustomWorker{process(t,e){return __awaiter(this,void 0,void 0,(function*(){const{job:o}=t.contextData;if(o.JobHandler&&o.JobHandler.startsWith("http")){let r=yield fetch(o.JobHandler,{method:"POST",headers:{"Content-Type":"application/json",Authorization:`Bearer ${t.skedContext.auth.token}`},body:JSON.stringify(e)});if(!r.ok)throw new Error(`Failed to process job: ${r.status} ${r.statusText}`);return yield r.json()}throw new Error("JobHandler is not a valid URL")}))}}exports.CustomWorker=CustomWorker;
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
"use strict";var __awaiter=this&&this.__awaiter||function(e,t,r,n){return new(r||(r=Promise))((function(o,c){function i(e){try{s(n.next(e))}catch(e){c(e)}}function a(e){try{s(n.throw(e))}catch(e){c(e)}}function s(e){var t;e.done?o(e.value):(t=e.value,t instanceof r?t:new r((function(e){e(t)}))).then(i,a)}s((n=n.apply(e,t||[])).next())}))};Object.defineProperty(exports,"__esModule",{value:!0}),exports.DeleteWorker=void 0;class DeleteWorker{process(e){return __awaiter(this,void 0,void 0,(function*(){const{job:t,records:r}=e.contextData;if(r&&0!==r.length)return console.log(`Deleting ${r.length} ${t.ObjectName}...`),e.graphqlService.delete(t.ObjectName,r)}))}}exports.DeleteWorker=DeleteWorker;
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
"use strict";var __createBinding=this&&this.__createBinding||(Object.create?function(e,r,t,i){void 0===i&&(i=t);var o=Object.getOwnPropertyDescriptor(r,t);o&&!("get"in o?!r.__esModule:o.writable||o.configurable)||(o={enumerable:!0,get:function(){return r[t]}}),Object.defineProperty(e,i,o)}:function(e,r,t,i){void 0===i&&(i=t),e[i]=r[t]}),__exportStar=this&&this.__exportStar||function(e,r){for(var t in e)"default"===t||Object.prototype.hasOwnProperty.call(r,t)||__createBinding(r,e,t)};Object.defineProperty(exports,"__esModule",{value:!0}),__exportStar(require("./worker-factory"),exports),__exportStar(require("./worker-interfaces"),exports);
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
"use strict";var __awaiter=this&&this.__awaiter||function(t,n,o,r){return new(o||(o=Promise))((function(e,a){function i(t){try{c(r.next(t))}catch(t){a(t)}}function s(t){try{c(r.throw(t))}catch(t){a(t)}}function c(t){var n;t.done?e(t.value):(n=t.value,n instanceof o?n:new o((function(t){t(n)}))).then(i,s)}c((r=r.apply(t,n||[])).next())}))};Object.defineProperty(exports,"__esModule",{value:!0}),exports.UpdateWorker=void 0;class UpdateWorker{process(t){return __awaiter(this,void 0,void 0,(function*(){const{job:n,records:o}=t.contextData,{JobHandler:r}=n,e=r;let a={};for(const t of o){const o=applyTransformation(t,e);if(isFlatObject(o)){const t=n.ObjectName;a[t]||(a[t]=[]),a[t].push(o)}else for(const t in o){a[t]||(a[t]=[]);const n=o[t];Array.isArray(n)?a[t].push(...n):a[t].push(n)}}for(const n in a)a[n]&&0!==a[n].length&&(console.log(`Updating ${a[n].length} ${n}...`),yield t.graphqlService.update(n,a[n]))}))}}function applyTransformation(input,transformFunction){const transformData=eval(transformFunction);return transformData(input)}function isFlatObject(t){for(const n in t)if("object"==typeof t[n]&&null!==t[n])return!1;return!0}exports.UpdateWorker=UpdateWorker;
|
|
@@ -1,6 +0,0 @@
|
|
|
1
|
-
import { CustomWorker } from "./custom-worker";
|
|
2
|
-
import { DeleteWorker } from "./delete-worker";
|
|
3
|
-
import { UpdateWorker } from "./update-worker";
|
|
4
|
-
export declare class WorkerFactory {
|
|
5
|
-
static getWorker(jobType: string): CustomWorker | DeleteWorker | UpdateWorker;
|
|
6
|
-
}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
"use strict";Object.defineProperty(exports,"__esModule",{value:!0}),exports.WorkerFactory=void 0;const constants_1=require("../constants"),custom_worker_1=require("./custom-worker"),delete_worker_1=require("./delete-worker"),update_worker_1=require("./update-worker");class WorkerFactory{static getWorker(e){switch(e){case constants_1.JobType.DELETE:return new delete_worker_1.DeleteWorker;case constants_1.JobType.UPDATE:return new update_worker_1.UpdateWorker;case constants_1.JobType.CUSTOM:return new custom_worker_1.CustomWorker;default:throw new Error(`Unsupported JobType: ${e}`)}}}exports.WorkerFactory=WorkerFactory;
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
"use strict";Object.defineProperty(exports,"__esModule",{value:!0});
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
"use strict";var __awaiter=this&&this.__awaiter||function(e,t,i,n){return new(i||(i=Promise))((function(r,c){function o(e){try{l(n.next(e))}catch(e){c(e)}}function a(e){try{l(n.throw(e))}catch(e){c(e)}}function l(e){var t;e.done?r(e.value):(t=e.value,t instanceof i?t:new i((function(e){e(t)}))).then(o,a)}l((n=n.apply(e,t||[])).next())}))};Object.defineProperty(exports,"__esModule",{value:!0}),exports.LockingService=void 0;const KEY_PREFIX="Z_LOCK_";class LockingService{constructor(e){this.configVarClient=e}acquireLock(e){return __awaiter(this,void 0,void 0,(function*(){const t="Z_LOCK_"+e.name,i=new Date(Date.now()+e.ttl).toISOString();try{let n=null;try{n=yield this.configVarClient.get(t)}catch(e){}if(n){if(n.expiryDate&&new Date<new Date(n.expiryDate))return!1;yield this.configVarClient.delete(t)}return yield this.configVarClient.create({key:t,value:e.name,configType:"plain-text",description:e.description,expiryDate:i}),!0}catch(t){return console.error(`Failed to acquire lock: ${e.name}`),!1}}))}releaseLock(e){return __awaiter(this,void 0,void 0,(function*(){const t="Z_LOCK_"+e;try{let e=null;try{e=yield this.configVarClient.get(t)}catch(e){}e&&(yield this.configVarClient.delete(t))}catch(t){console.error(`Failed to release lock: ${e}`)}}))}}exports.LockingService=LockingService;
|