@cloud-copilot/iam-collect 0.1.86 → 0.1.87
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cjs/download/download.d.ts.map +1 -1
- package/dist/cjs/download/download.js +10 -9
- package/dist/cjs/download/download.js.map +1 -1
- package/dist/cjs/jobs/jobRunner.d.ts +62 -0
- package/dist/cjs/jobs/jobRunner.d.ts.map +1 -0
- package/dist/cjs/jobs/jobRunner.js +157 -0
- package/dist/cjs/jobs/jobRunner.js.map +1 -0
- package/dist/esm/download/download.d.ts.map +1 -1
- package/dist/esm/download/download.js +10 -9
- package/dist/esm/download/download.js.map +1 -1
- package/dist/esm/jobs/jobRunner.d.ts +62 -0
- package/dist/esm/jobs/jobRunner.d.ts.map +1 -0
- package/dist/esm/jobs/jobRunner.js +152 -0
- package/dist/esm/jobs/jobRunner.js.map +1 -0
- package/package.json +1 -1
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"download.d.ts","sourceRoot":"","sources":["../../../src/download/download.ts"],"names":[],"mappings":"AAEA,OAAO,EASL,cAAc,EACf,MAAM,qBAAqB,CAAA;AAY5B,wBAAsB,YAAY,CAChC,OAAO,EAAE,cAAc,EAAE,EACzB,UAAU,EAAE,MAAM,EAAE,EACpB,OAAO,EAAE,MAAM,EAAE,EACjB,QAAQ,EAAE,MAAM,EAAE,EAClB,WAAW,EAAE,MAAM,GAAG,SAAS,EAC/B,SAAS,EAAE,OAAO,GACjB,OAAO,CAAC,IAAI,CAAC,
|
|
1
|
+
{"version":3,"file":"download.d.ts","sourceRoot":"","sources":["../../../src/download/download.ts"],"names":[],"mappings":"AAEA,OAAO,EASL,cAAc,EACf,MAAM,qBAAqB,CAAA;AAY5B,wBAAsB,YAAY,CAChC,OAAO,EAAE,cAAc,EAAE,EACzB,UAAU,EAAE,MAAM,EAAE,EACpB,OAAO,EAAE,MAAM,EAAE,EACjB,QAAQ,EAAE,MAAM,EAAE,EAClB,WAAW,EAAE,MAAM,GAAG,SAAS,EAC/B,SAAS,EAAE,OAAO,GACjB,OAAO,CAAC,IAAI,CAAC,CA+Jf"}
|
|
@@ -7,7 +7,7 @@ const config_js_1 = require("../config/config.js");
|
|
|
7
7
|
const partitionDefaults_js_1 = require("../config/partitionDefaults.js");
|
|
8
8
|
const indexMap_js_1 = require("../indexing/indexMap.js");
|
|
9
9
|
const runIndexers_js_1 = require("../indexing/runIndexers.js");
|
|
10
|
-
const
|
|
10
|
+
const jobRunner_js_1 = require("../jobs/jobRunner.js");
|
|
11
11
|
const util_js_1 = require("../jobs/util.js");
|
|
12
12
|
const util_js_2 = require("../persistence/util.js");
|
|
13
13
|
const regions_js_1 = require("../regions.js");
|
|
@@ -35,8 +35,9 @@ async function downloadData(configs, accountIds, regions, services, concurrency,
|
|
|
35
35
|
if (!storageConfig) {
|
|
36
36
|
throw new Error('No storage configuration found. Cannot download data.');
|
|
37
37
|
}
|
|
38
|
-
const jobs = [];
|
|
39
38
|
const indexJobs = [];
|
|
39
|
+
log_js_1.log.debug('Starting download runner', { concurrency });
|
|
40
|
+
const downloadRunner = new jobRunner_js_1.JobRunner(concurrency);
|
|
40
41
|
for (const accountId of accountIds) {
|
|
41
42
|
log_js_1.log.info('Queuing downloads for account', { accountId });
|
|
42
43
|
const authForAccount = (0, config_js_1.getAccountAuthConfig)(accountId, configs);
|
|
@@ -61,7 +62,7 @@ async function downloadData(configs, accountIds, regions, services, concurrency,
|
|
|
61
62
|
const globalRegion = serviceRegions.at(0);
|
|
62
63
|
const globalConfig = (0, config_js_1.accountServiceRegionConfig)(service, accountId, globalRegion, accountConfigs);
|
|
63
64
|
for (const globalSync of globalSyncs) {
|
|
64
|
-
|
|
65
|
+
downloadRunner.enqueue({
|
|
65
66
|
properties: { service, accountId, sync: globalSync.name },
|
|
66
67
|
execute: async (context) => {
|
|
67
68
|
const logDetails = {
|
|
@@ -89,7 +90,7 @@ async function downloadData(configs, accountIds, regions, services, concurrency,
|
|
|
89
90
|
log_js_1.log.debug({ service, accountId, region, syncName: sync.name }, 'Skipping regional sync');
|
|
90
91
|
continue;
|
|
91
92
|
}
|
|
92
|
-
|
|
93
|
+
downloadRunner.enqueue({
|
|
93
94
|
properties: { service, accountId, region, sync: sync.name },
|
|
94
95
|
execute: async (context) => {
|
|
95
96
|
const logDetails = {
|
|
@@ -99,7 +100,7 @@ async function downloadData(configs, accountIds, regions, services, concurrency,
|
|
|
99
100
|
log_js_1.log.debug(logDetails, 'Executing regional sync');
|
|
100
101
|
const regionalCredentials = await (0, auth_js_1.getCredentials)(accountId, asrConfig.auth);
|
|
101
102
|
await sync.execute(accountId, region, regionalCredentials, storage, asrConfig.endpoint, syncOptions);
|
|
102
|
-
log_js_1.log.
|
|
103
|
+
log_js_1.log.trace(logDetails, 'Finished regional sync');
|
|
103
104
|
}
|
|
104
105
|
});
|
|
105
106
|
}
|
|
@@ -115,9 +116,10 @@ async function downloadData(configs, accountIds, regions, services, concurrency,
|
|
|
115
116
|
}
|
|
116
117
|
}
|
|
117
118
|
}
|
|
118
|
-
log_js_1.log.
|
|
119
|
-
|
|
120
|
-
|
|
119
|
+
log_js_1.log.info('Waiting for downloads to complete');
|
|
120
|
+
await downloadRunner.finishAllWork();
|
|
121
|
+
log_js_1.log.info('Finished downloads', { jobs: downloadRunner.getResults().length });
|
|
122
|
+
const failedJobs = downloadRunner.getResults().filter((r) => r.status === 'rejected');
|
|
121
123
|
if (failedJobs.length > 0) {
|
|
122
124
|
log_js_1.log.error('Some downloads failed', { failedJobs: failedJobs.length });
|
|
123
125
|
for (const failedJob of failedJobs) {
|
|
@@ -125,7 +127,6 @@ async function downloadData(configs, accountIds, regions, services, concurrency,
|
|
|
125
127
|
}
|
|
126
128
|
throw new Error(`Failed to download some data. See logs for details.`);
|
|
127
129
|
}
|
|
128
|
-
log_js_1.log.info('Finished downloads', { jobs: jobs.length });
|
|
129
130
|
if (skipIndex) {
|
|
130
131
|
log_js_1.log.info('Skipping indexing');
|
|
131
132
|
return;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"download.js","sourceRoot":"","sources":["../../../src/download/download.ts"],"names":[],"mappings":";;AAwBA,
|
|
1
|
+
{"version":3,"file":"download.js","sourceRoot":"","sources":["../../../src/download/download.ts"],"names":[],"mappings":";;AAwBA,oCAsKC;AA9LD,4CAA+C;AAC/C,oDAA6D;AAC7D,mDAU4B;AAC5B,yEAAqE;AACrE,yDAA+D;AAC/D,+DAAmE;AACnE,uDAAgD;AAChD,6CAAoD;AACpD,oDAA4D;AAC5D,8CAAiD;AACjD,gDAA4C;AAC5C,oDAA0F;AAC1F,4CAAqC;AAE9B,KAAK,UAAU,YAAY,CAChC,OAAyB,EACzB,UAAoB,EACpB,OAAiB,EACjB,QAAkB,EAClB,WAA+B,EAC/B,SAAkB;IAElB,IAAI,WAAW,KAAK,SAAS,IAAI,WAAW,IAAI,CAAC,EAAE,CAAC;QAClD,WAAW,GAAG,IAAA,4BAAkB,GAAE,CAAA;IACpC,CAAC;IAED,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC5B,MAAM,kBAAkB,GAAG,IAAA,iCAAqB,EAAC,OAAO,CAAC,CAAA;QACzD,IAAI,kBAAkB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAClC,UAAU,GAAG,kBAAkB,CAAA;QACjC,CAAC;aAAM,CAAC;YACN,MAAM,iBAAiB,GAAG,IAAA,gCAAoB,EAAC,OAAO,CAAC,CAAA;YACvD,MAAM,kBAAkB,GAAG,MAAM,IAAA,sCAAwB,EAAC,iBAAiB,EAAE;gBAC3E,KAAK,EAAE,kBAAkB;aAC1B,CAAC,CAAA;YACF,UAAU,GAAG,CAAC,kBAAkB,CAAC,SAAS,CAAC,CAAA;QAC7C,CAAC;IACH,CAAC;IAED,MAAM,aAAa,GAAG,IAAA,4BAAgB,EAAC,OAAO,CAAC,CAAA;IAC/C,IAAI,CAAC,aAAa,EAAE,CAAC;QACnB,MAAM,IAAI,KAAK,CAAC,uDAAuD,CAAC,CAAA;IAC1E,CAAC;IAED,MAAM,SAAS,GAAe,EAAE,CAAA;IAEhC,YAAG,CAAC,KAAK,CAAC,0BAA0B,EAAE,EAAE,WAAW,EAAE,CAAC,CAAA;IACtD,MAAM,cAAc,GAAG,IAAI,wBAAS,CAAC,WAAW,CAAC,CAAA;IAEjD,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE,CAAC;QACnC,YAAG,CAAC,IAAI,CAAC,+BAA+B,EAAE,EAAE,SAAS,EAAE,CAAC,CAAA;QACxD,MAAM,cAAc,GAAG,IAAA,gCAAoB,EAAC,SAAS,EAAE,OAAO,CAAC,CAAA;QAC/D,MAAM,WAAW,GAAG,MAAM,IAAA,wBAAc,EAAC,SAAS,EAAE,cAAc,CAAC,CAAA;QACnE,MAAM,SAAS,GAAG,WAAW,CAAC,SAAS,CAAA;QACvC,MAAM,eAAe,GAAG,IAAA,2CAAoB,EAAC,SAAS,CAAC,CAAA;QACvD,MAAM,cAAc,GAAG,CAAC,eAAe,EAAE,GAAG,OAAO,CAAC,CAAA;QAEpD,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACzB,OAAO,GAAG,MAAM,IAAA,8BAAiB,EAAC,WAAW,CAAC,CAAA;QAChD,CAAC;QAED,MAAM,OAAO,GAAG,IAAA,6BAAmB,EAAC,aAAa,EAAE,SAAS,CAAC,CAAA;QAC7D,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC1B,QAAQ,GAAG,yBAAkC,CAAA;QAC/C,CAAC;QACD,MAAM,WAAW,GAAG,EAAE,CAAA;QAEtB,MAAM,eAAe,GAAG,IAAA,8BAAkB,EAAC,SAAS,EAAE,cAAc,EAAE,QAAQ,CAAC,CAAA;QAC/E,KAAK,MAAM,OAAO,IAAI,eAAe,EAAE,CAAC;YACtC,YAAG,CAAC,IAAI,CAAC,mBAAmB,EAAE,EAAE,OAAO,EAAE,SAAS,EAAE,CAAC,CAAA;YACrD,MAAM,cAAc,GAAG,IAAA,6BAAiB,EAAC,OAAO,EAAE,SAAS,EAAE,cAAc,EAAE,OAAO,CAAC,CAAA;YAErF,8BAA8B;YAC9B,MAAM,WAAW,GAAG,IAAA,qCAAwB,EAAC,OAAO,CAAC,CAAA;YACrD,MAAM,YAAY,GAAG,cAAc,CAAC,EAAE,CAAC,CAAC,CAAE,CAAA;YAC1C,MAAM,YAAY,GAAG,IAAA,sCAA0B,EAC7C,OAAO,EACP,SAAS,EACT,YAAY,EACZ,cAAc,CACf,CAAA;YAED,KAAK,MAAM,UAAU,IAAI,WAAW,EAAE,CAAC;gBACrC,cAAc,CAAC,OAAO,CAAC;oBACrB,UAAU,EAAE,EAAE,OAAO,EAAE,SAAS,EAAE,IAAI,EAAE,UAAU,CAAC,IAAI,EAAE;oBACzD,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE;wBACzB,MAAM,UAAU,GAAG;4BACjB,QAAQ,EAAE,OAAO,CAAC,QAAQ;4BAC1B,GAAG,OAAO,CAAC,UAAU;yBACtB,CAAA;wBACD,MAAM,iBAAiB,GAAG,MAAM,IAAA,wBAAc,EAAC,SAAS,EAAE,YAAY,CAAC,IAAI,CAAC,CAAA;wBAC5E,YAAG,CAAC,KAAK,CAAC,UAAU,EAAE,uBAAuB,CAAC,CAAA;wBAC9C,MAAM,UAAU,CAAC,OAAO,CACtB,SAAS,EACT,YAAY,EACZ,iBAAiB,EACjB,OAAO,EACP,YAAY,CAAC,QAAQ,EACrB,WAAW,CACZ,CAAA;wBACD,YAAG,CAAC,KAAK,CAAC,UAAU,EAAE,sBAAsB,CAAC,CAAA;oBAC/C,CAAC;iBACF,CAAC,CAAA;YACJ,CAAC;YAED,MAAM,aAAa,GAAG,IAAA,uCAA0B,EAAC,OAAO,CAAC,CAAA;YACzD,gCAAgC;YAChC,KAAK,MAAM,MAAM,IAAI,cAAc,EAAE,CAAC;gBACpC,YAAG,CAAC,KAAK,CAAC,EAAE,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,EAAE,wBAAwB,CAAC,CAAA;gBACnE,IAAI,aAAa,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;oBAC/B,SAAQ;gBACV,CAAC;gBACD,MAAM,SAAS,GAAG,IAAA,sCAA0B,EAAC,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,cAAc,CAAC,CAAA;gBAExF,KAAK,MAAM,IAAI,IAAI,aAAa,EAAE,CAAC;oBACjC,MAAM,WAAW,GAAG,IAAA,gCAAoB,EACtC,SAAS,EACT,OAAO,EACP,IAAI,CAAC,IAAI,EACT,cAAc,EACd,MAAM,CACP,CAAA;oBACD,IAAI,CAAC,WAAW,EAAE,CAAC;wBACjB,YAAG,CAAC,KAAK,CAAC,EAAE,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,QAAQ,EAAE,IAAI,CAAC,IAAI,EAAE,EAAE,wBAAwB,CAAC,CAAA;wBACxF,SAAQ;oBACV,CAAC;oBACD,cAAc,CAAC,OAAO,CAAC;wBACrB,UAAU,EAAE,EAAE,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE;wBAC3D,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE;4BACzB,MAAM,UAAU,GAAG;gCACjB,QAAQ,EAAE,OAAO,CAAC,QAAQ;gCAC1B,GAAG,OAAO,CAAC,UAAU;6BACtB,CAAA;4BACD,YAAG,CAAC,KAAK,CAAC,UAAU,EAAE,yBAAyB,CAAC,CAAA;4BAChD,MAAM,mBAAmB,GAAG,MAAM,IAAA,wBAAc,EAAC,SAAS,EAAE,SAAS,CAAC,IAAI,CAAC,CAAA;4BAE3E,MAAM,IAAI,CAAC,OAAO,CAChB,SAAS,EACT,MAAM,EACN,mBAAmB,EACnB,OAAO,EACP,SAAS,CAAC,QAAQ,EAClB,WAAW,CACZ,CAAA;4BACD,YAAG,CAAC,KAAK,CAAC,UAAU,EAAE,wBAAwB,CAAC,CAAA;wBACjD,CAAC;qBACF,CAAC,CAAA;gBACJ,CAAC;YACH,CAAC;YAED,MAAM,QAAQ,GAAG,IAAA,mCAAqB,EAAC,OAAO,CAAC,CAAA;YAC/C,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;gBAC/B,SAAS,CAAC,IAAI,CAAC;oBACb,OAAO;oBACP,SAAS;oBACT,SAAS;oBACT,OAAO,EAAE,cAAc;iBACxB,CAAC,CAAA;YACJ,CAAC;QACH,CAAC;IACH,CAAC;IAED,YAAG,CAAC,IAAI,CAAC,mCAAmC,CAAC,CAAA;IAC7C,MAAM,cAAc,CAAC,aAAa,EAAE,CAAA;IACpC,YAAG,CAAC,IAAI,CAAC,oBAAoB,EAAE,EAAE,IAAI,EAAE,cAAc,CAAC,UAAU,EAAE,CAAC,MAAM,EAAE,CAAC,CAAA;IAC5E,MAAM,UAAU,GAAG,cAAc,CAAC,UAAU,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,UAAU,CAAC,CAAA;IACrF,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC1B,YAAG,CAAC,KAAK,CAAC,uBAAuB,EAAE,EAAE,UAAU,EAAE,UAAU,CAAC,MAAM,EAAE,CAAC,CAAA;QACrE,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE,CAAC;YACnC,YAAG,CAAC,KAAK,CAAC,iBAAiB,EAAE,SAAS,CAAC,MAAM,EAAE,SAAS,CAAC,UAAU,CAAC,CAAA;QACtE,CAAC;QACD,MAAM,IAAI,KAAK,CAAC,qDAAqD,CAAC,CAAA;IACxE,CAAC;IAED,IAAI,SAAS,EAAE,CAAC;QACd,YAAG,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAA;QAC7B,OAAM;IACR,CAAC;IAED,MAAM,IAAA,6BAAY,EAAC,SAAS,EAAE,aAAa,EAAE,WAAW,CAAC,CAAA;AAC3D,CAAC"}
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
import { Job, JobResult } from './jobQueue.js';
|
|
2
|
+
/**
|
|
3
|
+
* Job runner that allows running jobs as they are created.
|
|
4
|
+
* This will wait for jobs to be added to it and run them up the
|
|
5
|
+
* maximum concurrency.
|
|
6
|
+
*
|
|
7
|
+
* Results are available via `getResults()`.
|
|
8
|
+
*/
|
|
9
|
+
export declare class JobRunner<T = void, P = Record<string, unknown>> {
|
|
10
|
+
private concurrency;
|
|
11
|
+
private queue;
|
|
12
|
+
private results;
|
|
13
|
+
private activeJobs;
|
|
14
|
+
private waitingResolvers;
|
|
15
|
+
private workers;
|
|
16
|
+
private isAcceptingWork;
|
|
17
|
+
private workAvailablePromise;
|
|
18
|
+
private resolveWorkAvailable;
|
|
19
|
+
/**
|
|
20
|
+
* Create a new runner with the specified concurrency.
|
|
21
|
+
*
|
|
22
|
+
* @param concurrency - The maximum number of jobs to run concurrently.
|
|
23
|
+
*/
|
|
24
|
+
constructor(concurrency: number);
|
|
25
|
+
private worker;
|
|
26
|
+
private waitForWorkAvailable;
|
|
27
|
+
private ensureWorkers;
|
|
28
|
+
private notifyWorkersOfNewWork;
|
|
29
|
+
private checkIfIdle;
|
|
30
|
+
/**
|
|
31
|
+
* Add a job to the queue
|
|
32
|
+
*/
|
|
33
|
+
enqueue(job: Job<T, P>): void;
|
|
34
|
+
/**
|
|
35
|
+
* Add multiple jobs to the queue
|
|
36
|
+
*/
|
|
37
|
+
enqueueAll(jobs: Job<T, P>[]): void;
|
|
38
|
+
/**
|
|
39
|
+
* Returns a promise that resolves when all queued work is complete
|
|
40
|
+
*/
|
|
41
|
+
waitForIdle(): Promise<void>;
|
|
42
|
+
/**
|
|
43
|
+
* Get all results accumulated so far
|
|
44
|
+
*/
|
|
45
|
+
getResults(): JobResult<T, P>[];
|
|
46
|
+
/**
|
|
47
|
+
* Shutdown the queue - no new jobs will be accepted, but existing jobs will complete.
|
|
48
|
+
*
|
|
49
|
+
* Returns when a promise that resolves when all jobs have been processed and
|
|
50
|
+
* are available in `getResults()`.
|
|
51
|
+
*/
|
|
52
|
+
finishAllWork(): Promise<void>;
|
|
53
|
+
/**
|
|
54
|
+
* Get the current queue length
|
|
55
|
+
*/
|
|
56
|
+
get queueLength(): number;
|
|
57
|
+
/**
|
|
58
|
+
* Get the number of currently active jobs
|
|
59
|
+
*/
|
|
60
|
+
get activeJobCount(): number;
|
|
61
|
+
}
|
|
62
|
+
//# sourceMappingURL=jobRunner.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"jobRunner.d.ts","sourceRoot":"","sources":["../../../src/jobs/jobRunner.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,GAAG,EAAc,SAAS,EAAE,MAAM,eAAe,CAAA;AAE1D;;;;;;GAMG;AAEH,qBAAa,SAAS,CAAC,CAAC,GAAG,IAAI,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC;IAe9C,OAAO,CAAC,WAAW;IAd/B,OAAO,CAAC,KAAK,CAAkB;IAC/B,OAAO,CAAC,OAAO,CAAwB;IACvC,OAAO,CAAC,UAAU,CAAI;IACtB,OAAO,CAAC,gBAAgB,CAAqB;IAC7C,OAAO,CAAC,OAAO,CAAsB;IACrC,OAAO,CAAC,eAAe,CAAO;IAC9B,OAAO,CAAC,oBAAoB,CAA6B;IACzD,OAAO,CAAC,oBAAoB,CAA4B;IAExD;;;;OAIG;gBACiB,WAAW,EAAE,MAAM;YAEzB,MAAM;IAsCpB,OAAO,CAAC,oBAAoB;IAS5B,OAAO,CAAC,aAAa;IAQrB,OAAO,CAAC,sBAAsB;IAS9B,OAAO,CAAC,WAAW;IAQnB;;OAEG;IACH,OAAO,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,IAAI;IAS7B;;OAEG;IACH,UAAU,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,GAAG,IAAI;IAInC;;OAEG;IACH,WAAW,IAAI,OAAO,CAAC,IAAI,CAAC;IAW5B;;OAEG;IACH,UAAU,IAAI,SAAS,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE;IAI/B;;;;;OAKG;IACG,aAAa,IAAI,OAAO,CAAC,IAAI,CAAC;IASpC;;OAEG;IACH,IAAI,WAAW,IAAI,MAAM,CAExB;IAED;;OAEG;IACH,IAAI,cAAc,IAAI,MAAM,CAE3B;CACF"}
|
|
@@ -0,0 +1,157 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.JobRunner = void 0;
|
|
4
|
+
const log_js_1 = require("../utils/log.js");
|
|
5
|
+
/**
|
|
6
|
+
* Job runner that allows running jobs as they are created.
|
|
7
|
+
* This will wait for jobs to be added to it and run them up the
|
|
8
|
+
* maximum concurrency.
|
|
9
|
+
*
|
|
10
|
+
* Results are available via `getResults()`.
|
|
11
|
+
*/
|
|
12
|
+
class JobRunner {
|
|
13
|
+
concurrency;
|
|
14
|
+
queue = [];
|
|
15
|
+
results = [];
|
|
16
|
+
activeJobs = 0;
|
|
17
|
+
waitingResolvers = [];
|
|
18
|
+
workers = [];
|
|
19
|
+
isAcceptingWork = true;
|
|
20
|
+
workAvailablePromise = null;
|
|
21
|
+
resolveWorkAvailable = null;
|
|
22
|
+
/**
|
|
23
|
+
* Create a new runner with the specified concurrency.
|
|
24
|
+
*
|
|
25
|
+
* @param concurrency - The maximum number of jobs to run concurrently.
|
|
26
|
+
*/
|
|
27
|
+
constructor(concurrency) {
|
|
28
|
+
this.concurrency = concurrency;
|
|
29
|
+
}
|
|
30
|
+
async worker(workerId) {
|
|
31
|
+
while (this.isAcceptingWork || this.queue.length > 0) {
|
|
32
|
+
const job = this.queue.shift();
|
|
33
|
+
if (!job) {
|
|
34
|
+
if (!this.isAcceptingWork) {
|
|
35
|
+
// No longer accepting work and no jobs left, exit immediately
|
|
36
|
+
return;
|
|
37
|
+
}
|
|
38
|
+
// No work available, wait for new work to be added
|
|
39
|
+
await this.waitForWorkAvailable();
|
|
40
|
+
continue;
|
|
41
|
+
}
|
|
42
|
+
this.activeJobs++;
|
|
43
|
+
const context = { workerId };
|
|
44
|
+
const startTime = Date.now();
|
|
45
|
+
const interval = setInterval(() => {
|
|
46
|
+
log_js_1.log.warn(`Long-running job detected.`, { minutes: Math.floor((Date.now() - startTime) / 60000) }, { ...context, ...job.properties });
|
|
47
|
+
}, 60_000);
|
|
48
|
+
try {
|
|
49
|
+
const value = await job.execute({ ...context, properties: job.properties });
|
|
50
|
+
this.results.push({ status: 'fulfilled', value, properties: job.properties });
|
|
51
|
+
}
|
|
52
|
+
catch (reason) {
|
|
53
|
+
this.results.push({ status: 'rejected', reason, properties: job.properties });
|
|
54
|
+
}
|
|
55
|
+
finally {
|
|
56
|
+
clearInterval(interval);
|
|
57
|
+
this.activeJobs--;
|
|
58
|
+
this.checkIfIdle();
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
waitForWorkAvailable() {
|
|
63
|
+
if (!this.workAvailablePromise) {
|
|
64
|
+
this.workAvailablePromise = new Promise((resolve) => {
|
|
65
|
+
this.resolveWorkAvailable = resolve;
|
|
66
|
+
});
|
|
67
|
+
}
|
|
68
|
+
return this.workAvailablePromise;
|
|
69
|
+
}
|
|
70
|
+
ensureWorkers() {
|
|
71
|
+
if (this.workers.length === 0 && this.isAcceptingWork) {
|
|
72
|
+
for (let i = 0; i < this.concurrency; i++) {
|
|
73
|
+
this.workers.push(this.worker(i + 1));
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
notifyWorkersOfNewWork() {
|
|
78
|
+
// Wake up waiting workers
|
|
79
|
+
if (this.resolveWorkAvailable) {
|
|
80
|
+
this.resolveWorkAvailable();
|
|
81
|
+
this.workAvailablePromise = null;
|
|
82
|
+
this.resolveWorkAvailable = null;
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
checkIfIdle() {
|
|
86
|
+
if (this.activeJobs === 0 && this.queue.length === 0) {
|
|
87
|
+
// Notify all waiting resolvers
|
|
88
|
+
this.waitingResolvers.forEach((resolve) => resolve());
|
|
89
|
+
this.waitingResolvers = [];
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
/**
|
|
93
|
+
* Add a job to the queue
|
|
94
|
+
*/
|
|
95
|
+
enqueue(job) {
|
|
96
|
+
if (!this.isAcceptingWork) {
|
|
97
|
+
throw new Error('Cannot enqueue jobs after shutdown');
|
|
98
|
+
}
|
|
99
|
+
this.queue.push(job);
|
|
100
|
+
this.ensureWorkers();
|
|
101
|
+
this.notifyWorkersOfNewWork();
|
|
102
|
+
}
|
|
103
|
+
/**
|
|
104
|
+
* Add multiple jobs to the queue
|
|
105
|
+
*/
|
|
106
|
+
enqueueAll(jobs) {
|
|
107
|
+
jobs.forEach((job) => this.enqueue(job));
|
|
108
|
+
}
|
|
109
|
+
/**
|
|
110
|
+
* Returns a promise that resolves when all queued work is complete
|
|
111
|
+
*/
|
|
112
|
+
waitForIdle() {
|
|
113
|
+
// log.debug('waitForIdle called', this.activeJobs, this.queue.length)
|
|
114
|
+
return new Promise((resolve) => {
|
|
115
|
+
if (this.activeJobs === 0 && this.queue.length === 0) {
|
|
116
|
+
resolve();
|
|
117
|
+
}
|
|
118
|
+
else {
|
|
119
|
+
this.waitingResolvers.push(resolve);
|
|
120
|
+
}
|
|
121
|
+
});
|
|
122
|
+
}
|
|
123
|
+
/**
|
|
124
|
+
* Get all results accumulated so far
|
|
125
|
+
*/
|
|
126
|
+
getResults() {
|
|
127
|
+
return this.results;
|
|
128
|
+
}
|
|
129
|
+
/**
|
|
130
|
+
* Shutdown the queue - no new jobs will be accepted, but existing jobs will complete.
|
|
131
|
+
*
|
|
132
|
+
* Returns when a promise that resolves when all jobs have been processed and
|
|
133
|
+
* are available in `getResults()`.
|
|
134
|
+
*/
|
|
135
|
+
async finishAllWork() {
|
|
136
|
+
this.isAcceptingWork = false;
|
|
137
|
+
// Wake up any sleeping workers so they can process remaining jobs or exit
|
|
138
|
+
this.notifyWorkersOfNewWork();
|
|
139
|
+
// Check if we're already idle and notify any waiting resolvers
|
|
140
|
+
await Promise.all(this.workers);
|
|
141
|
+
this.workers = [];
|
|
142
|
+
}
|
|
143
|
+
/**
|
|
144
|
+
* Get the current queue length
|
|
145
|
+
*/
|
|
146
|
+
get queueLength() {
|
|
147
|
+
return this.queue.length;
|
|
148
|
+
}
|
|
149
|
+
/**
|
|
150
|
+
* Get the number of currently active jobs
|
|
151
|
+
*/
|
|
152
|
+
get activeJobCount() {
|
|
153
|
+
return this.activeJobs;
|
|
154
|
+
}
|
|
155
|
+
}
|
|
156
|
+
exports.JobRunner = JobRunner;
|
|
157
|
+
//# sourceMappingURL=jobRunner.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"jobRunner.js","sourceRoot":"","sources":["../../../src/jobs/jobRunner.ts"],"names":[],"mappings":";;;AAAA,4CAAqC;AAGrC;;;;;;GAMG;AAEH,MAAa,SAAS;IAeA;IAdZ,KAAK,GAAgB,EAAE,CAAA;IACvB,OAAO,GAAsB,EAAE,CAAA;IAC/B,UAAU,GAAG,CAAC,CAAA;IACd,gBAAgB,GAAmB,EAAE,CAAA;IACrC,OAAO,GAAoB,EAAE,CAAA;IAC7B,eAAe,GAAG,IAAI,CAAA;IACtB,oBAAoB,GAAyB,IAAI,CAAA;IACjD,oBAAoB,GAAwB,IAAI,CAAA;IAExD;;;;OAIG;IACH,YAAoB,WAAmB;QAAnB,gBAAW,GAAX,WAAW,CAAQ;IAAG,CAAC;IAEnC,KAAK,CAAC,MAAM,CAAC,QAAgB;QACnC,OAAO,IAAI,CAAC,eAAe,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACrD,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAA;YAC9B,IAAI,CAAC,GAAG,EAAE,CAAC;gBACT,IAAI,CAAC,IAAI,CAAC,eAAe,EAAE,CAAC;oBAC1B,8DAA8D;oBAC9D,OAAM;gBACR,CAAC;gBACD,mDAAmD;gBACnD,MAAM,IAAI,CAAC,oBAAoB,EAAE,CAAA;gBACjC,SAAQ;YACV,CAAC;YAED,IAAI,CAAC,UAAU,EAAE,CAAA;YACjB,MAAM,OAAO,GAAe,EAAE,QAAQ,EAAE,CAAA;YAExC,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAA;YAC5B,MAAM,QAAQ,GAAG,WAAW,CAAC,GAAG,EAAE;gBAChC,YAAG,CAAC,IAAI,CACN,4BAA4B,EAC5B,EAAE,OAAO,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC,GAAG,KAAK,CAAC,EAAE,EACzD,EAAE,GAAG,OAAO,EAAE,GAAG,GAAG,CAAC,UAAU,EAAE,CAClC,CAAA;YACH,CAAC,EAAE,MAAM,CAAC,CAAA;YAEV,IAAI,CAAC;gBACH,MAAM,KAAK,GAAG,MAAM,GAAG,CAAC,OAAO,CAAC,EAAE,GAAG,OAAO,EAAE,UAAU,EAAE,GAAG,CAAC,UAAU,EAAE,CAAC,CAAA;gBAC3E,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,WAAW,EAAE,KAAK,EAAE,UAAU,EAAE,GAAG,CAAC,UAAU,EAAE,CAAC,CAAA;YAC/E,CAAC;YAAC,OAAO,MAAM,EAAE,CAAC;gBAChB,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,EAAE,UAAU,EAAE,GAAG,CAAC,UAAU,EAAE,CAAC,CAAA;YAC/E,CAAC;oBAAS,CAAC;gBACT,aAAa,CAAC,QAAQ,CAAC,CAAA;gBACvB,IAAI,CAAC,UAAU,EAAE,CAAA;gBACjB,IAAI,CAAC,WAAW,EAAE,CAAA;YACpB,CAAC;QACH,CAAC;IACH,CAAC;IAEO,oBAAoB;QAC1B,IAAI,CAAC,IAAI,CAAC,oBAAoB,EAAE,CAAC;YAC/B,IAAI,CAAC,oBAAoB,GAAG,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE;gBACxD,IAAI,CAAC,oBAAoB,GAAG,OAAO,CAAA;YACrC,CAAC,CAAC,CAAA;QACJ,CAAC;QACD,OAAO,IAAI,CAAC,oBAAoB,CAAA;IAClC,CAAC;IAEO,aAAa;QACnB,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,KAAK,CAAC,IAAI,IAAI,CAAC,eAAe,EAAE,CAAC;YACtD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC,EAAE,EAAE,CAAC;gBAC1C,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAA;YACvC,CAAC;QACH,CAAC;IACH,CAAC;IAEO,sBAAsB;QAC5B,0BAA0B;QAC1B,IAAI,IAAI,CAAC,oBAAoB,EAAE,CAAC;YAC9B,IAAI,CAAC,oBAAoB,EAAE,CAAA;YAC3B,IAAI,CAAC,oBAAoB,GAAG,IAAI,CAAA;YAChC,IAAI,CAAC,oBAAoB,GAAG,IAAI,CAAA;QAClC,CAAC;IACH,CAAC;IAEO,WAAW;QACjB,IAAI,IAAI,CAAC,UAAU,KAAK,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACrD,+BAA+B;YAC/B,IAAI,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,EAAE,CAAC,CAAA;YACrD,IAAI,CAAC,gBAAgB,GAAG,EAAE,CAAA;QAC5B,CAAC;IACH,CAAC;IAED;;OAEG;IACH,OAAO,CAAC,GAAc;QACpB,IAAI,CAAC,IAAI,CAAC,eAAe,EAAE,CAAC;YAC1B,MAAM,IAAI,KAAK,CAAC,oCAAoC,CAAC,CAAA;QACvD,CAAC;QACD,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;QACpB,IAAI,CAAC,aAAa,EAAE,CAAA;QACpB,IAAI,CAAC,sBAAsB,EAAE,CAAA;IAC/B,CAAC;IAED;;OAEG;IACH,UAAU,CAAC,IAAiB;QAC1B,IAAI,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAA;IAC1C,CAAC;IAED;;OAEG;IACH,WAAW;QACT,sEAAsE;QACtE,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE;YAC7B,IAAI,IAAI,CAAC,UAAU,KAAK,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACrD,OAAO,EAAE,CAAA;YACX,CAAC;iBAAM,CAAC;gBACN,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;YACrC,CAAC;QACH,CAAC,CAAC,CAAA;IACJ,CAAC;IAED;;OAEG;IACH,UAAU;QACR,OAAO,IAAI,CAAC,OAAO,CAAA;IACrB,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,aAAa;QACjB,IAAI,CAAC,eAAe,GAAG,KAAK,CAAA;QAC5B,0EAA0E;QAC1E,IAAI,CAAC,sBAAsB,EAAE,CAAA;QAC7B,+DAA+D;QAC/D,MAAM,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;QAC/B,IAAI,CAAC,OAAO,GAAG,EAAE,CAAA;IACnB,CAAC;IAED;;OAEG;IACH,IAAI,WAAW;QACb,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,CAAA;IAC1B,CAAC;IAED;;OAEG;IACH,IAAI,cAAc;QAChB,OAAO,IAAI,CAAC,UAAU,CAAA;IACxB,CAAC;CACF;AA7JD,8BA6JC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"download.d.ts","sourceRoot":"","sources":["../../../src/download/download.ts"],"names":[],"mappings":"AAEA,OAAO,EASL,cAAc,EACf,MAAM,qBAAqB,CAAA;AAY5B,wBAAsB,YAAY,CAChC,OAAO,EAAE,cAAc,EAAE,EACzB,UAAU,EAAE,MAAM,EAAE,EACpB,OAAO,EAAE,MAAM,EAAE,EACjB,QAAQ,EAAE,MAAM,EAAE,EAClB,WAAW,EAAE,MAAM,GAAG,SAAS,EAC/B,SAAS,EAAE,OAAO,GACjB,OAAO,CAAC,IAAI,CAAC,
|
|
1
|
+
{"version":3,"file":"download.d.ts","sourceRoot":"","sources":["../../../src/download/download.ts"],"names":[],"mappings":"AAEA,OAAO,EASL,cAAc,EACf,MAAM,qBAAqB,CAAA;AAY5B,wBAAsB,YAAY,CAChC,OAAO,EAAE,cAAc,EAAE,EACzB,UAAU,EAAE,MAAM,EAAE,EACpB,OAAO,EAAE,MAAM,EAAE,EACjB,QAAQ,EAAE,MAAM,EAAE,EAClB,WAAW,EAAE,MAAM,GAAG,SAAS,EAC/B,SAAS,EAAE,OAAO,GACjB,OAAO,CAAC,IAAI,CAAC,CA+Jf"}
|
|
@@ -4,7 +4,7 @@ import { accountServiceRegionConfig, getAccountAuthConfig, getConfiguredAccounts
|
|
|
4
4
|
import { getPartitionDefaults } from '../config/partitionDefaults.js';
|
|
5
5
|
import { getIndexersForService } from '../indexing/indexMap.js';
|
|
6
6
|
import { runIndexJobs } from '../indexing/runIndexers.js';
|
|
7
|
-
import {
|
|
7
|
+
import { JobRunner } from '../jobs/jobRunner.js';
|
|
8
8
|
import { defaultConcurrency } from '../jobs/util.js';
|
|
9
9
|
import { createStorageClient } from '../persistence/util.js';
|
|
10
10
|
import { getEnabledRegions } from '../regions.js';
|
|
@@ -32,8 +32,9 @@ export async function downloadData(configs, accountIds, regions, services, concu
|
|
|
32
32
|
if (!storageConfig) {
|
|
33
33
|
throw new Error('No storage configuration found. Cannot download data.');
|
|
34
34
|
}
|
|
35
|
-
const jobs = [];
|
|
36
35
|
const indexJobs = [];
|
|
36
|
+
log.debug('Starting download runner', { concurrency });
|
|
37
|
+
const downloadRunner = new JobRunner(concurrency);
|
|
37
38
|
for (const accountId of accountIds) {
|
|
38
39
|
log.info('Queuing downloads for account', { accountId });
|
|
39
40
|
const authForAccount = getAccountAuthConfig(accountId, configs);
|
|
@@ -58,7 +59,7 @@ export async function downloadData(configs, accountIds, regions, services, concu
|
|
|
58
59
|
const globalRegion = serviceRegions.at(0);
|
|
59
60
|
const globalConfig = accountServiceRegionConfig(service, accountId, globalRegion, accountConfigs);
|
|
60
61
|
for (const globalSync of globalSyncs) {
|
|
61
|
-
|
|
62
|
+
downloadRunner.enqueue({
|
|
62
63
|
properties: { service, accountId, sync: globalSync.name },
|
|
63
64
|
execute: async (context) => {
|
|
64
65
|
const logDetails = {
|
|
@@ -86,7 +87,7 @@ export async function downloadData(configs, accountIds, regions, services, concu
|
|
|
86
87
|
log.debug({ service, accountId, region, syncName: sync.name }, 'Skipping regional sync');
|
|
87
88
|
continue;
|
|
88
89
|
}
|
|
89
|
-
|
|
90
|
+
downloadRunner.enqueue({
|
|
90
91
|
properties: { service, accountId, region, sync: sync.name },
|
|
91
92
|
execute: async (context) => {
|
|
92
93
|
const logDetails = {
|
|
@@ -96,7 +97,7 @@ export async function downloadData(configs, accountIds, regions, services, concu
|
|
|
96
97
|
log.debug(logDetails, 'Executing regional sync');
|
|
97
98
|
const regionalCredentials = await getCredentials(accountId, asrConfig.auth);
|
|
98
99
|
await sync.execute(accountId, region, regionalCredentials, storage, asrConfig.endpoint, syncOptions);
|
|
99
|
-
log.
|
|
100
|
+
log.trace(logDetails, 'Finished regional sync');
|
|
100
101
|
}
|
|
101
102
|
});
|
|
102
103
|
}
|
|
@@ -112,9 +113,10 @@ export async function downloadData(configs, accountIds, regions, services, concu
|
|
|
112
113
|
}
|
|
113
114
|
}
|
|
114
115
|
}
|
|
115
|
-
log.
|
|
116
|
-
|
|
117
|
-
|
|
116
|
+
log.info('Waiting for downloads to complete');
|
|
117
|
+
await downloadRunner.finishAllWork();
|
|
118
|
+
log.info('Finished downloads', { jobs: downloadRunner.getResults().length });
|
|
119
|
+
const failedJobs = downloadRunner.getResults().filter((r) => r.status === 'rejected');
|
|
118
120
|
if (failedJobs.length > 0) {
|
|
119
121
|
log.error('Some downloads failed', { failedJobs: failedJobs.length });
|
|
120
122
|
for (const failedJob of failedJobs) {
|
|
@@ -122,7 +124,6 @@ export async function downloadData(configs, accountIds, regions, services, concu
|
|
|
122
124
|
}
|
|
123
125
|
throw new Error(`Failed to download some data. See logs for details.`);
|
|
124
126
|
}
|
|
125
|
-
log.info('Finished downloads', { jobs: jobs.length });
|
|
126
127
|
if (skipIndex) {
|
|
127
128
|
log.info('Skipping indexing');
|
|
128
129
|
return;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"download.js","sourceRoot":"","sources":["../../../src/download/download.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAA;AAC/C,OAAO,EAAE,wBAAwB,EAAE,MAAM,oBAAoB,CAAA;AAC7D,OAAO,EACL,0BAA0B,EAC1B,oBAAoB,EACpB,qBAAqB,EACrB,oBAAoB,EACpB,gBAAgB,EAChB,iBAAiB,EACjB,kBAAkB,EAClB,oBAAoB,EAErB,MAAM,qBAAqB,CAAA;AAC5B,OAAO,EAAE,oBAAoB,EAAE,MAAM,gCAAgC,CAAA;AACrE,OAAO,EAAE,qBAAqB,EAAE,MAAM,yBAAyB,CAAA;AAC/D,OAAO,EAAY,YAAY,EAAE,MAAM,4BAA4B,CAAA;AACnE,OAAO,
|
|
1
|
+
{"version":3,"file":"download.js","sourceRoot":"","sources":["../../../src/download/download.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAA;AAC/C,OAAO,EAAE,wBAAwB,EAAE,MAAM,oBAAoB,CAAA;AAC7D,OAAO,EACL,0BAA0B,EAC1B,oBAAoB,EACpB,qBAAqB,EACrB,oBAAoB,EACpB,gBAAgB,EAChB,iBAAiB,EACjB,kBAAkB,EAClB,oBAAoB,EAErB,MAAM,qBAAqB,CAAA;AAC5B,OAAO,EAAE,oBAAoB,EAAE,MAAM,gCAAgC,CAAA;AACrE,OAAO,EAAE,qBAAqB,EAAE,MAAM,yBAAyB,CAAA;AAC/D,OAAO,EAAY,YAAY,EAAE,MAAM,4BAA4B,CAAA;AACnE,OAAO,EAAE,SAAS,EAAE,MAAM,sBAAsB,CAAA;AAChD,OAAO,EAAE,kBAAkB,EAAE,MAAM,iBAAiB,CAAA;AACpD,OAAO,EAAE,mBAAmB,EAAE,MAAM,wBAAwB,CAAA;AAC5D,OAAO,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAA;AACjD,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAA;AAC5C,OAAO,EAAE,wBAAwB,EAAE,0BAA0B,EAAE,MAAM,qBAAqB,CAAA;AAC1F,OAAO,EAAE,GAAG,EAAE,MAAM,iBAAiB,CAAA;AAErC,MAAM,CAAC,KAAK,UAAU,YAAY,CAChC,OAAyB,EACzB,UAAoB,EACpB,OAAiB,EACjB,QAAkB,EAClB,WAA+B,EAC/B,SAAkB;IAElB,IAAI,WAAW,KAAK,SAAS,IAAI,WAAW,IAAI,CAAC,EAAE,CAAC;QAClD,WAAW,GAAG,kBAAkB,EAAE,CAAA;IACpC,CAAC;IAED,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC5B,MAAM,kBAAkB,GAAG,qBAAqB,CAAC,OAAO,CAAC,CAAA;QACzD,IAAI,kBAAkB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAClC,UAAU,GAAG,kBAAkB,CAAA;QACjC,CAAC;aAAM,CAAC;YACN,MAAM,iBAAiB,GAAG,oBAAoB,CAAC,OAAO,CAAC,CAAA;YACvD,MAAM,kBAAkB,GAAG,MAAM,wBAAwB,CAAC,iBAAiB,EAAE;gBAC3E,KAAK,EAAE,kBAAkB;aAC1B,CAAC,CAAA;YACF,UAAU,GAAG,CAAC,kBAAkB,CAAC,SAAS,CAAC,CAAA;QAC7C,CAAC;IACH,CAAC;IAED,MAAM,aAAa,GAAG,gBAAgB,CAAC,OAAO,CAAC,CAAA;IAC/C,IAAI,CAAC,aAAa,EAAE,CAAC;QACnB,MAAM,IAAI,KAAK,CAAC,uDAAuD,CAAC,CAAA;IAC1E,CAAC;IAED,MAAM,SAAS,GAAe,EAAE,CAAA;IAEhC,GAAG,CAAC,KAAK,CAAC,0BAA0B,EAAE,EAAE,WAAW,EAAE,CAAC,CAAA;IACtD,MAAM,cAAc,GAAG,IAAI,SAAS,CAAC,WAAW,CAAC,CAAA;IAEjD,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE,CAAC;QACnC,GAAG,CAAC,IAAI,CAAC,+BAA+B,EAAE,EAAE,SAAS,EAAE,CAAC,CAAA;QACxD,MAAM,cAAc,GAAG,oBAAoB,CAAC,SAAS,EAAE,OAAO,CAAC,CAAA;QAC/D,MAAM,WAAW,GAAG,MAAM,cAAc,CAAC,SAAS,EAAE,cAAc,CAAC,CAAA;QACnE,MAAM,SAAS,GAAG,WAAW,CAAC,SAAS,CAAA;QACvC,MAAM,eAAe,GAAG,oBAAoB,CAAC,SAAS,CAAC,CAAA;QACvD,MAAM,cAAc,GAAG,CAAC,eAAe,EAAE,GAAG,OAAO,CAAC,CAAA;QAEpD,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACzB,OAAO,GAAG,MAAM,iBAAiB,CAAC,WAAW,CAAC,CAAA;QAChD,CAAC;QAED,MAAM,OAAO,GAAG,mBAAmB,CAAC,aAAa,EAAE,SAAS,CAAC,CAAA;QAC7D,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC1B,QAAQ,GAAG,WAAkC,CAAA;QAC/C,CAAC;QACD,MAAM,WAAW,GAAG,EAAE,CAAA;QAEtB,MAAM,eAAe,GAAG,kBAAkB,CAAC,SAAS,EAAE,cAAc,EAAE,QAAQ,CAAC,CAAA;QAC/E,KAAK,MAAM,OAAO,IAAI,eAAe,EAAE,CAAC;YACtC,GAAG,CAAC,IAAI,CAAC,mBAAmB,EAAE,EAAE,OAAO,EAAE,SAAS,EAAE,CAAC,CAAA;YACrD,MAAM,cAAc,GAAG,iBAAiB,CAAC,OAAO,EAAE,SAAS,EAAE,cAAc,EAAE,OAAO,CAAC,CAAA;YAErF,8BAA8B;YAC9B,MAAM,WAAW,GAAG,wBAAwB,CAAC,OAAO,CAAC,CAAA;YACrD,MAAM,YAAY,GAAG,cAAc,CAAC,EAAE,CAAC,CAAC,CAAE,CAAA;YAC1C,MAAM,YAAY,GAAG,0BAA0B,CAC7C,OAAO,EACP,SAAS,EACT,YAAY,EACZ,cAAc,CACf,CAAA;YAED,KAAK,MAAM,UAAU,IAAI,WAAW,EAAE,CAAC;gBACrC,cAAc,CAAC,OAAO,CAAC;oBACrB,UAAU,EAAE,EAAE,OAAO,EAAE,SAAS,EAAE,IAAI,EAAE,UAAU,CAAC,IAAI,EAAE;oBACzD,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE;wBACzB,MAAM,UAAU,GAAG;4BACjB,QAAQ,EAAE,OAAO,CAAC,QAAQ;4BAC1B,GAAG,OAAO,CAAC,UAAU;yBACtB,CAAA;wBACD,MAAM,iBAAiB,GAAG,MAAM,cAAc,CAAC,SAAS,EAAE,YAAY,CAAC,IAAI,CAAC,CAAA;wBAC5E,GAAG,CAAC,KAAK,CAAC,UAAU,EAAE,uBAAuB,CAAC,CAAA;wBAC9C,MAAM,UAAU,CAAC,OAAO,CACtB,SAAS,EACT,YAAY,EACZ,iBAAiB,EACjB,OAAO,EACP,YAAY,CAAC,QAAQ,EACrB,WAAW,CACZ,CAAA;wBACD,GAAG,CAAC,KAAK,CAAC,UAAU,EAAE,sBAAsB,CAAC,CAAA;oBAC/C,CAAC;iBACF,CAAC,CAAA;YACJ,CAAC;YAED,MAAM,aAAa,GAAG,0BAA0B,CAAC,OAAO,CAAC,CAAA;YACzD,gCAAgC;YAChC,KAAK,MAAM,MAAM,IAAI,cAAc,EAAE,CAAC;gBACpC,GAAG,CAAC,KAAK,CAAC,EAAE,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,EAAE,wBAAwB,CAAC,CAAA;gBACnE,IAAI,aAAa,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;oBAC/B,SAAQ;gBACV,CAAC;gBACD,MAAM,SAAS,GAAG,0BAA0B,CAAC,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,cAAc,CAAC,CAAA;gBAExF,KAAK,MAAM,IAAI,IAAI,aAAa,EAAE,CAAC;oBACjC,MAAM,WAAW,GAAG,oBAAoB,CACtC,SAAS,EACT,OAAO,EACP,IAAI,CAAC,IAAI,EACT,cAAc,EACd,MAAM,CACP,CAAA;oBACD,IAAI,CAAC,WAAW,EAAE,CAAC;wBACjB,GAAG,CAAC,KAAK,CAAC,EAAE,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,QAAQ,EAAE,IAAI,CAAC,IAAI,EAAE,EAAE,wBAAwB,CAAC,CAAA;wBACxF,SAAQ;oBACV,CAAC;oBACD,cAAc,CAAC,OAAO,CAAC;wBACrB,UAAU,EAAE,EAAE,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE;wBAC3D,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE;4BACzB,MAAM,UAAU,GAAG;gCACjB,QAAQ,EAAE,OAAO,CAAC,QAAQ;gCAC1B,GAAG,OAAO,CAAC,UAAU;6BACtB,CAAA;4BACD,GAAG,CAAC,KAAK,CAAC,UAAU,EAAE,yBAAyB,CAAC,CAAA;4BAChD,MAAM,mBAAmB,GAAG,MAAM,cAAc,CAAC,SAAS,EAAE,SAAS,CAAC,IAAI,CAAC,CAAA;4BAE3E,MAAM,IAAI,CAAC,OAAO,CAChB,SAAS,EACT,MAAM,EACN,mBAAmB,EACnB,OAAO,EACP,SAAS,CAAC,QAAQ,EAClB,WAAW,CACZ,CAAA;4BACD,GAAG,CAAC,KAAK,CAAC,UAAU,EAAE,wBAAwB,CAAC,CAAA;wBACjD,CAAC;qBACF,CAAC,CAAA;gBACJ,CAAC;YACH,CAAC;YAED,MAAM,QAAQ,GAAG,qBAAqB,CAAC,OAAO,CAAC,CAAA;YAC/C,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;gBAC/B,SAAS,CAAC,IAAI,CAAC;oBACb,OAAO;oBACP,SAAS;oBACT,SAAS;oBACT,OAAO,EAAE,cAAc;iBACxB,CAAC,CAAA;YACJ,CAAC;QACH,CAAC;IACH,CAAC;IAED,GAAG,CAAC,IAAI,CAAC,mCAAmC,CAAC,CAAA;IAC7C,MAAM,cAAc,CAAC,aAAa,EAAE,CAAA;IACpC,GAAG,CAAC,IAAI,CAAC,oBAAoB,EAAE,EAAE,IAAI,EAAE,cAAc,CAAC,UAAU,EAAE,CAAC,MAAM,EAAE,CAAC,CAAA;IAC5E,MAAM,UAAU,GAAG,cAAc,CAAC,UAAU,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,UAAU,CAAC,CAAA;IACrF,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC1B,GAAG,CAAC,KAAK,CAAC,uBAAuB,EAAE,EAAE,UAAU,EAAE,UAAU,CAAC,MAAM,EAAE,CAAC,CAAA;QACrE,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE,CAAC;YACnC,GAAG,CAAC,KAAK,CAAC,iBAAiB,EAAE,SAAS,CAAC,MAAM,EAAE,SAAS,CAAC,UAAU,CAAC,CAAA;QACtE,CAAC;QACD,MAAM,IAAI,KAAK,CAAC,qDAAqD,CAAC,CAAA;IACxE,CAAC;IAED,IAAI,SAAS,EAAE,CAAC;QACd,GAAG,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAA;QAC7B,OAAM;IACR,CAAC;IAED,MAAM,YAAY,CAAC,SAAS,EAAE,aAAa,EAAE,WAAW,CAAC,CAAA;AAC3D,CAAC"}
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
import { Job, JobResult } from './jobQueue.js';
|
|
2
|
+
/**
|
|
3
|
+
* Job runner that allows running jobs as they are created.
|
|
4
|
+
* This will wait for jobs to be added to it and run them up the
|
|
5
|
+
* maximum concurrency.
|
|
6
|
+
*
|
|
7
|
+
* Results are available via `getResults()`.
|
|
8
|
+
*/
|
|
9
|
+
export declare class JobRunner<T = void, P = Record<string, unknown>> {
|
|
10
|
+
private concurrency;
|
|
11
|
+
private queue;
|
|
12
|
+
private results;
|
|
13
|
+
private activeJobs;
|
|
14
|
+
private waitingResolvers;
|
|
15
|
+
private workers;
|
|
16
|
+
private isAcceptingWork;
|
|
17
|
+
private workAvailablePromise;
|
|
18
|
+
private resolveWorkAvailable;
|
|
19
|
+
/**
|
|
20
|
+
* Create a new runner with the specified concurrency.
|
|
21
|
+
*
|
|
22
|
+
* @param concurrency - The maximum number of jobs to run concurrently.
|
|
23
|
+
*/
|
|
24
|
+
constructor(concurrency: number);
|
|
25
|
+
private worker;
|
|
26
|
+
private waitForWorkAvailable;
|
|
27
|
+
private ensureWorkers;
|
|
28
|
+
private notifyWorkersOfNewWork;
|
|
29
|
+
private checkIfIdle;
|
|
30
|
+
/**
|
|
31
|
+
* Add a job to the queue
|
|
32
|
+
*/
|
|
33
|
+
enqueue(job: Job<T, P>): void;
|
|
34
|
+
/**
|
|
35
|
+
* Add multiple jobs to the queue
|
|
36
|
+
*/
|
|
37
|
+
enqueueAll(jobs: Job<T, P>[]): void;
|
|
38
|
+
/**
|
|
39
|
+
* Returns a promise that resolves when all queued work is complete
|
|
40
|
+
*/
|
|
41
|
+
waitForIdle(): Promise<void>;
|
|
42
|
+
/**
|
|
43
|
+
* Get all results accumulated so far
|
|
44
|
+
*/
|
|
45
|
+
getResults(): JobResult<T, P>[];
|
|
46
|
+
/**
|
|
47
|
+
* Shutdown the queue - no new jobs will be accepted, but existing jobs will complete.
|
|
48
|
+
*
|
|
49
|
+
* Returns when a promise that resolves when all jobs have been processed and
|
|
50
|
+
* are available in `getResults()`.
|
|
51
|
+
*/
|
|
52
|
+
finishAllWork(): Promise<void>;
|
|
53
|
+
/**
|
|
54
|
+
* Get the current queue length
|
|
55
|
+
*/
|
|
56
|
+
get queueLength(): number;
|
|
57
|
+
/**
|
|
58
|
+
* Get the number of currently active jobs
|
|
59
|
+
*/
|
|
60
|
+
get activeJobCount(): number;
|
|
61
|
+
}
|
|
62
|
+
//# sourceMappingURL=jobRunner.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"jobRunner.d.ts","sourceRoot":"","sources":["../../../src/jobs/jobRunner.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,GAAG,EAAc,SAAS,EAAE,MAAM,eAAe,CAAA;AAE1D;;;;;;GAMG;AAEH,qBAAa,SAAS,CAAC,CAAC,GAAG,IAAI,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC;IAe9C,OAAO,CAAC,WAAW;IAd/B,OAAO,CAAC,KAAK,CAAkB;IAC/B,OAAO,CAAC,OAAO,CAAwB;IACvC,OAAO,CAAC,UAAU,CAAI;IACtB,OAAO,CAAC,gBAAgB,CAAqB;IAC7C,OAAO,CAAC,OAAO,CAAsB;IACrC,OAAO,CAAC,eAAe,CAAO;IAC9B,OAAO,CAAC,oBAAoB,CAA6B;IACzD,OAAO,CAAC,oBAAoB,CAA4B;IAExD;;;;OAIG;gBACiB,WAAW,EAAE,MAAM;YAEzB,MAAM;IAsCpB,OAAO,CAAC,oBAAoB;IAS5B,OAAO,CAAC,aAAa;IAQrB,OAAO,CAAC,sBAAsB;IAS9B,OAAO,CAAC,WAAW;IAQnB;;OAEG;IACH,OAAO,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,IAAI;IAS7B;;OAEG;IACH,UAAU,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,GAAG,IAAI;IAInC;;OAEG;IACH,WAAW,IAAI,OAAO,CAAC,IAAI,CAAC;IAW5B;;OAEG;IACH,UAAU,IAAI,SAAS,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE;IAI/B;;;;;OAKG;IACG,aAAa,IAAI,OAAO,CAAC,IAAI,CAAC;IASpC;;OAEG;IACH,IAAI,WAAW,IAAI,MAAM,CAExB;IAED;;OAEG;IACH,IAAI,cAAc,IAAI,MAAM,CAE3B;CACF"}
|
|
@@ -0,0 +1,152 @@
|
|
|
1
|
+
import { log } from '../utils/log.js';
|
|
2
|
+
/**
|
|
3
|
+
* Job runner that allows running jobs as they are created.
|
|
4
|
+
* This will wait for jobs to be added to it and run them up the
|
|
5
|
+
* maximum concurrency.
|
|
6
|
+
*
|
|
7
|
+
* Results are available via `getResults()`.
|
|
8
|
+
*/
|
|
9
|
+
export class JobRunner {
|
|
10
|
+
/**
|
|
11
|
+
* Create a new runner with the specified concurrency.
|
|
12
|
+
*
|
|
13
|
+
* @param concurrency - The maximum number of jobs to run concurrently.
|
|
14
|
+
*/
|
|
15
|
+
constructor(concurrency) {
|
|
16
|
+
this.concurrency = concurrency;
|
|
17
|
+
this.queue = [];
|
|
18
|
+
this.results = [];
|
|
19
|
+
this.activeJobs = 0;
|
|
20
|
+
this.waitingResolvers = [];
|
|
21
|
+
this.workers = [];
|
|
22
|
+
this.isAcceptingWork = true;
|
|
23
|
+
this.workAvailablePromise = null;
|
|
24
|
+
this.resolveWorkAvailable = null;
|
|
25
|
+
}
|
|
26
|
+
async worker(workerId) {
|
|
27
|
+
while (this.isAcceptingWork || this.queue.length > 0) {
|
|
28
|
+
const job = this.queue.shift();
|
|
29
|
+
if (!job) {
|
|
30
|
+
if (!this.isAcceptingWork) {
|
|
31
|
+
// No longer accepting work and no jobs left, exit immediately
|
|
32
|
+
return;
|
|
33
|
+
}
|
|
34
|
+
// No work available, wait for new work to be added
|
|
35
|
+
await this.waitForWorkAvailable();
|
|
36
|
+
continue;
|
|
37
|
+
}
|
|
38
|
+
this.activeJobs++;
|
|
39
|
+
const context = { workerId };
|
|
40
|
+
const startTime = Date.now();
|
|
41
|
+
const interval = setInterval(() => {
|
|
42
|
+
log.warn(`Long-running job detected.`, { minutes: Math.floor((Date.now() - startTime) / 60000) }, { ...context, ...job.properties });
|
|
43
|
+
}, 60000);
|
|
44
|
+
try {
|
|
45
|
+
const value = await job.execute({ ...context, properties: job.properties });
|
|
46
|
+
this.results.push({ status: 'fulfilled', value, properties: job.properties });
|
|
47
|
+
}
|
|
48
|
+
catch (reason) {
|
|
49
|
+
this.results.push({ status: 'rejected', reason, properties: job.properties });
|
|
50
|
+
}
|
|
51
|
+
finally {
|
|
52
|
+
clearInterval(interval);
|
|
53
|
+
this.activeJobs--;
|
|
54
|
+
this.checkIfIdle();
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
waitForWorkAvailable() {
|
|
59
|
+
if (!this.workAvailablePromise) {
|
|
60
|
+
this.workAvailablePromise = new Promise((resolve) => {
|
|
61
|
+
this.resolveWorkAvailable = resolve;
|
|
62
|
+
});
|
|
63
|
+
}
|
|
64
|
+
return this.workAvailablePromise;
|
|
65
|
+
}
|
|
66
|
+
ensureWorkers() {
|
|
67
|
+
if (this.workers.length === 0 && this.isAcceptingWork) {
|
|
68
|
+
for (let i = 0; i < this.concurrency; i++) {
|
|
69
|
+
this.workers.push(this.worker(i + 1));
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
notifyWorkersOfNewWork() {
|
|
74
|
+
// Wake up waiting workers
|
|
75
|
+
if (this.resolveWorkAvailable) {
|
|
76
|
+
this.resolveWorkAvailable();
|
|
77
|
+
this.workAvailablePromise = null;
|
|
78
|
+
this.resolveWorkAvailable = null;
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
checkIfIdle() {
|
|
82
|
+
if (this.activeJobs === 0 && this.queue.length === 0) {
|
|
83
|
+
// Notify all waiting resolvers
|
|
84
|
+
this.waitingResolvers.forEach((resolve) => resolve());
|
|
85
|
+
this.waitingResolvers = [];
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
/**
|
|
89
|
+
* Add a job to the queue
|
|
90
|
+
*/
|
|
91
|
+
enqueue(job) {
|
|
92
|
+
if (!this.isAcceptingWork) {
|
|
93
|
+
throw new Error('Cannot enqueue jobs after shutdown');
|
|
94
|
+
}
|
|
95
|
+
this.queue.push(job);
|
|
96
|
+
this.ensureWorkers();
|
|
97
|
+
this.notifyWorkersOfNewWork();
|
|
98
|
+
}
|
|
99
|
+
/**
|
|
100
|
+
* Add multiple jobs to the queue
|
|
101
|
+
*/
|
|
102
|
+
enqueueAll(jobs) {
|
|
103
|
+
jobs.forEach((job) => this.enqueue(job));
|
|
104
|
+
}
|
|
105
|
+
/**
|
|
106
|
+
* Returns a promise that resolves when all queued work is complete
|
|
107
|
+
*/
|
|
108
|
+
waitForIdle() {
|
|
109
|
+
// log.debug('waitForIdle called', this.activeJobs, this.queue.length)
|
|
110
|
+
return new Promise((resolve) => {
|
|
111
|
+
if (this.activeJobs === 0 && this.queue.length === 0) {
|
|
112
|
+
resolve();
|
|
113
|
+
}
|
|
114
|
+
else {
|
|
115
|
+
this.waitingResolvers.push(resolve);
|
|
116
|
+
}
|
|
117
|
+
});
|
|
118
|
+
}
|
|
119
|
+
/**
|
|
120
|
+
* Get all results accumulated so far
|
|
121
|
+
*/
|
|
122
|
+
getResults() {
|
|
123
|
+
return this.results;
|
|
124
|
+
}
|
|
125
|
+
/**
|
|
126
|
+
* Shutdown the queue - no new jobs will be accepted, but existing jobs will complete.
|
|
127
|
+
*
|
|
128
|
+
* Returns when a promise that resolves when all jobs have been processed and
|
|
129
|
+
* are available in `getResults()`.
|
|
130
|
+
*/
|
|
131
|
+
async finishAllWork() {
|
|
132
|
+
this.isAcceptingWork = false;
|
|
133
|
+
// Wake up any sleeping workers so they can process remaining jobs or exit
|
|
134
|
+
this.notifyWorkersOfNewWork();
|
|
135
|
+
// Check if we're already idle and notify any waiting resolvers
|
|
136
|
+
await Promise.all(this.workers);
|
|
137
|
+
this.workers = [];
|
|
138
|
+
}
|
|
139
|
+
/**
|
|
140
|
+
* Get the current queue length
|
|
141
|
+
*/
|
|
142
|
+
get queueLength() {
|
|
143
|
+
return this.queue.length;
|
|
144
|
+
}
|
|
145
|
+
/**
|
|
146
|
+
* Get the number of currently active jobs
|
|
147
|
+
*/
|
|
148
|
+
get activeJobCount() {
|
|
149
|
+
return this.activeJobs;
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
//# sourceMappingURL=jobRunner.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"jobRunner.js","sourceRoot":"","sources":["../../../src/jobs/jobRunner.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,GAAG,EAAE,MAAM,iBAAiB,CAAA;AAGrC;;;;;;GAMG;AAEH,MAAM,OAAO,SAAS;IAUpB;;;;OAIG;IACH,YAAoB,WAAmB;QAAnB,gBAAW,GAAX,WAAW,CAAQ;QAd/B,UAAK,GAAgB,EAAE,CAAA;QACvB,YAAO,GAAsB,EAAE,CAAA;QAC/B,eAAU,GAAG,CAAC,CAAA;QACd,qBAAgB,GAAmB,EAAE,CAAA;QACrC,YAAO,GAAoB,EAAE,CAAA;QAC7B,oBAAe,GAAG,IAAI,CAAA;QACtB,yBAAoB,GAAyB,IAAI,CAAA;QACjD,yBAAoB,GAAwB,IAAI,CAAA;IAOd,CAAC;IAEnC,KAAK,CAAC,MAAM,CAAC,QAAgB;QACnC,OAAO,IAAI,CAAC,eAAe,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACrD,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAA;YAC9B,IAAI,CAAC,GAAG,EAAE,CAAC;gBACT,IAAI,CAAC,IAAI,CAAC,eAAe,EAAE,CAAC;oBAC1B,8DAA8D;oBAC9D,OAAM;gBACR,CAAC;gBACD,mDAAmD;gBACnD,MAAM,IAAI,CAAC,oBAAoB,EAAE,CAAA;gBACjC,SAAQ;YACV,CAAC;YAED,IAAI,CAAC,UAAU,EAAE,CAAA;YACjB,MAAM,OAAO,GAAe,EAAE,QAAQ,EAAE,CAAA;YAExC,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAA;YAC5B,MAAM,QAAQ,GAAG,WAAW,CAAC,GAAG,EAAE;gBAChC,GAAG,CAAC,IAAI,CACN,4BAA4B,EAC5B,EAAE,OAAO,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC,GAAG,KAAK,CAAC,EAAE,EACzD,EAAE,GAAG,OAAO,EAAE,GAAG,GAAG,CAAC,UAAU,EAAE,CAClC,CAAA;YACH,CAAC,EAAE,KAAM,CAAC,CAAA;YAEV,IAAI,CAAC;gBACH,MAAM,KAAK,GAAG,MAAM,GAAG,CAAC,OAAO,CAAC,EAAE,GAAG,OAAO,EAAE,UAAU,EAAE,GAAG,CAAC,UAAU,EAAE,CAAC,CAAA;gBAC3E,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,WAAW,EAAE,KAAK,EAAE,UAAU,EAAE,GAAG,CAAC,UAAU,EAAE,CAAC,CAAA;YAC/E,CAAC;YAAC,OAAO,MAAM,EAAE,CAAC;gBAChB,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,EAAE,UAAU,EAAE,GAAG,CAAC,UAAU,EAAE,CAAC,CAAA;YAC/E,CAAC;oBAAS,CAAC;gBACT,aAAa,CAAC,QAAQ,CAAC,CAAA;gBACvB,IAAI,CAAC,UAAU,EAAE,CAAA;gBACjB,IAAI,CAAC,WAAW,EAAE,CAAA;YACpB,CAAC;QACH,CAAC;IACH,CAAC;IAEO,oBAAoB;QAC1B,IAAI,CAAC,IAAI,CAAC,oBAAoB,EAAE,CAAC;YAC/B,IAAI,CAAC,oBAAoB,GAAG,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE;gBACxD,IAAI,CAAC,oBAAoB,GAAG,OAAO,CAAA;YACrC,CAAC,CAAC,CAAA;QACJ,CAAC;QACD,OAAO,IAAI,CAAC,oBAAoB,CAAA;IAClC,CAAC;IAEO,aAAa;QACnB,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,KAAK,CAAC,IAAI,IAAI,CAAC,eAAe,EAAE,CAAC;YACtD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC,EAAE,EAAE,CAAC;gBAC1C,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAA;YACvC,CAAC;QACH,CAAC;IACH,CAAC;IAEO,sBAAsB;QAC5B,0BAA0B;QAC1B,IAAI,IAAI,CAAC,oBAAoB,EAAE,CAAC;YAC9B,IAAI,CAAC,oBAAoB,EAAE,CAAA;YAC3B,IAAI,CAAC,oBAAoB,GAAG,IAAI,CAAA;YAChC,IAAI,CAAC,oBAAoB,GAAG,IAAI,CAAA;QAClC,CAAC;IACH,CAAC;IAEO,WAAW;QACjB,IAAI,IAAI,CAAC,UAAU,KAAK,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACrD,+BAA+B;YAC/B,IAAI,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,EAAE,CAAC,CAAA;YACrD,IAAI,CAAC,gBAAgB,GAAG,EAAE,CAAA;QAC5B,CAAC;IACH,CAAC;IAED;;OAEG;IACH,OAAO,CAAC,GAAc;QACpB,IAAI,CAAC,IAAI,CAAC,eAAe,EAAE,CAAC;YAC1B,MAAM,IAAI,KAAK,CAAC,oCAAoC,CAAC,CAAA;QACvD,CAAC;QACD,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;QACpB,IAAI,CAAC,aAAa,EAAE,CAAA;QACpB,IAAI,CAAC,sBAAsB,EAAE,CAAA;IAC/B,CAAC;IAED;;OAEG;IACH,UAAU,CAAC,IAAiB;QAC1B,IAAI,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAA;IAC1C,CAAC;IAED;;OAEG;IACH,WAAW;QACT,sEAAsE;QACtE,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE;YAC7B,IAAI,IAAI,CAAC,UAAU,KAAK,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACrD,OAAO,EAAE,CAAA;YACX,CAAC;iBAAM,CAAC;gBACN,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;YACrC,CAAC;QACH,CAAC,CAAC,CAAA;IACJ,CAAC;IAED;;OAEG;IACH,UAAU;QACR,OAAO,IAAI,CAAC,OAAO,CAAA;IACrB,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,aAAa;QACjB,IAAI,CAAC,eAAe,GAAG,KAAK,CAAA;QAC5B,0EAA0E;QAC1E,IAAI,CAAC,sBAAsB,EAAE,CAAA;QAC7B,+DAA+D;QAC/D,MAAM,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;QAC/B,IAAI,CAAC,OAAO,GAAG,EAAE,CAAA;IACnB,CAAC;IAED;;OAEG;IACH,IAAI,WAAW;QACb,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,CAAA;IAC1B,CAAC;IAED;;OAEG;IACH,IAAI,cAAc;QAChB,OAAO,IAAI,CAAC,UAAU,CAAA;IACxB,CAAC;CACF"}
|