@cloud-copilot/iam-collect 0.1.85 → 0.1.87

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (35) hide show
  1. package/dist/cjs/download/download.d.ts.map +1 -1
  2. package/dist/cjs/download/download.js +10 -9
  3. package/dist/cjs/download/download.js.map +1 -1
  4. package/dist/cjs/indexing/indexMap.js +2 -2
  5. package/dist/cjs/indexing/indexMap.js.map +1 -1
  6. package/dist/cjs/indexing/indexers/vpcs.d.ts +16 -0
  7. package/dist/cjs/indexing/indexers/vpcs.d.ts.map +1 -0
  8. package/dist/cjs/indexing/indexers/vpcs.js +94 -0
  9. package/dist/cjs/indexing/indexers/vpcs.js.map +1 -0
  10. package/dist/cjs/jobs/jobRunner.d.ts +62 -0
  11. package/dist/cjs/jobs/jobRunner.d.ts.map +1 -0
  12. package/dist/cjs/jobs/jobRunner.js +157 -0
  13. package/dist/cjs/jobs/jobRunner.js.map +1 -0
  14. package/dist/esm/download/download.d.ts.map +1 -1
  15. package/dist/esm/download/download.js +10 -9
  16. package/dist/esm/download/download.js.map +1 -1
  17. package/dist/esm/indexing/indexMap.js +1 -1
  18. package/dist/esm/indexing/indexMap.js.map +1 -1
  19. package/dist/esm/indexing/indexers/vpcs.d.ts +16 -0
  20. package/dist/esm/indexing/indexers/vpcs.d.ts.map +1 -0
  21. package/dist/esm/indexing/indexers/vpcs.js +91 -0
  22. package/dist/esm/indexing/indexers/vpcs.js.map +1 -0
  23. package/dist/esm/jobs/jobRunner.d.ts +62 -0
  24. package/dist/esm/jobs/jobRunner.d.ts.map +1 -0
  25. package/dist/esm/jobs/jobRunner.js +152 -0
  26. package/dist/esm/jobs/jobRunner.js.map +1 -0
  27. package/package.json +1 -1
  28. package/dist/cjs/indexing/indexers/vpcEndpoints.d.ts +0 -3
  29. package/dist/cjs/indexing/indexers/vpcEndpoints.d.ts.map +0 -1
  30. package/dist/cjs/indexing/indexers/vpcEndpoints.js +0 -57
  31. package/dist/cjs/indexing/indexers/vpcEndpoints.js.map +0 -1
  32. package/dist/esm/indexing/indexers/vpcEndpoints.d.ts +0 -3
  33. package/dist/esm/indexing/indexers/vpcEndpoints.d.ts.map +0 -1
  34. package/dist/esm/indexing/indexers/vpcEndpoints.js +0 -54
  35. package/dist/esm/indexing/indexers/vpcEndpoints.js.map +0 -1
@@ -1 +1 @@
1
- {"version":3,"file":"download.d.ts","sourceRoot":"","sources":["../../../src/download/download.ts"],"names":[],"mappings":"AAEA,OAAO,EASL,cAAc,EACf,MAAM,qBAAqB,CAAA;AAY5B,wBAAsB,YAAY,CAChC,OAAO,EAAE,cAAc,EAAE,EACzB,UAAU,EAAE,MAAM,EAAE,EACpB,OAAO,EAAE,MAAM,EAAE,EACjB,QAAQ,EAAE,MAAM,EAAE,EAClB,WAAW,EAAE,MAAM,GAAG,SAAS,EAC/B,SAAS,EAAE,OAAO,GACjB,OAAO,CAAC,IAAI,CAAC,CA6Jf"}
1
+ {"version":3,"file":"download.d.ts","sourceRoot":"","sources":["../../../src/download/download.ts"],"names":[],"mappings":"AAEA,OAAO,EASL,cAAc,EACf,MAAM,qBAAqB,CAAA;AAY5B,wBAAsB,YAAY,CAChC,OAAO,EAAE,cAAc,EAAE,EACzB,UAAU,EAAE,MAAM,EAAE,EACpB,OAAO,EAAE,MAAM,EAAE,EACjB,QAAQ,EAAE,MAAM,EAAE,EAClB,WAAW,EAAE,MAAM,GAAG,SAAS,EAC/B,SAAS,EAAE,OAAO,GACjB,OAAO,CAAC,IAAI,CAAC,CA+Jf"}
@@ -7,7 +7,7 @@ const config_js_1 = require("../config/config.js");
7
7
  const partitionDefaults_js_1 = require("../config/partitionDefaults.js");
8
8
  const indexMap_js_1 = require("../indexing/indexMap.js");
9
9
  const runIndexers_js_1 = require("../indexing/runIndexers.js");
10
- const jobQueue_js_1 = require("../jobs/jobQueue.js");
10
+ const jobRunner_js_1 = require("../jobs/jobRunner.js");
11
11
  const util_js_1 = require("../jobs/util.js");
12
12
  const util_js_2 = require("../persistence/util.js");
13
13
  const regions_js_1 = require("../regions.js");
@@ -35,8 +35,9 @@ async function downloadData(configs, accountIds, regions, services, concurrency,
35
35
  if (!storageConfig) {
36
36
  throw new Error('No storage configuration found. Cannot download data.');
37
37
  }
38
- const jobs = [];
39
38
  const indexJobs = [];
39
+ log_js_1.log.debug('Starting download runner', { concurrency });
40
+ const downloadRunner = new jobRunner_js_1.JobRunner(concurrency);
40
41
  for (const accountId of accountIds) {
41
42
  log_js_1.log.info('Queuing downloads for account', { accountId });
42
43
  const authForAccount = (0, config_js_1.getAccountAuthConfig)(accountId, configs);
@@ -61,7 +62,7 @@ async function downloadData(configs, accountIds, regions, services, concurrency,
61
62
  const globalRegion = serviceRegions.at(0);
62
63
  const globalConfig = (0, config_js_1.accountServiceRegionConfig)(service, accountId, globalRegion, accountConfigs);
63
64
  for (const globalSync of globalSyncs) {
64
- jobs.push({
65
+ downloadRunner.enqueue({
65
66
  properties: { service, accountId, sync: globalSync.name },
66
67
  execute: async (context) => {
67
68
  const logDetails = {
@@ -89,7 +90,7 @@ async function downloadData(configs, accountIds, regions, services, concurrency,
89
90
  log_js_1.log.debug({ service, accountId, region, syncName: sync.name }, 'Skipping regional sync');
90
91
  continue;
91
92
  }
92
- jobs.push({
93
+ downloadRunner.enqueue({
93
94
  properties: { service, accountId, region, sync: sync.name },
94
95
  execute: async (context) => {
95
96
  const logDetails = {
@@ -99,7 +100,7 @@ async function downloadData(configs, accountIds, regions, services, concurrency,
99
100
  log_js_1.log.debug(logDetails, 'Executing regional sync');
100
101
  const regionalCredentials = await (0, auth_js_1.getCredentials)(accountId, asrConfig.auth);
101
102
  await sync.execute(accountId, region, regionalCredentials, storage, asrConfig.endpoint, syncOptions);
102
- log_js_1.log.debug(logDetails, 'Finished regional sync');
103
+ log_js_1.log.trace(logDetails, 'Finished regional sync');
103
104
  }
104
105
  });
105
106
  }
@@ -115,9 +116,10 @@ async function downloadData(configs, accountIds, regions, services, concurrency,
115
116
  }
116
117
  }
117
118
  }
118
- log_js_1.log.debug('Starting downloads', { jobs: jobs.length, concurrency });
119
- const results = await (0, jobQueue_js_1.runJobs)(jobs, concurrency);
120
- const failedJobs = results.filter((r) => r.status === 'rejected');
119
+ log_js_1.log.info('Waiting for downloads to complete');
120
+ await downloadRunner.finishAllWork();
121
+ log_js_1.log.info('Finished downloads', { jobs: downloadRunner.getResults().length });
122
+ const failedJobs = downloadRunner.getResults().filter((r) => r.status === 'rejected');
121
123
  if (failedJobs.length > 0) {
122
124
  log_js_1.log.error('Some downloads failed', { failedJobs: failedJobs.length });
123
125
  for (const failedJob of failedJobs) {
@@ -125,7 +127,6 @@ async function downloadData(configs, accountIds, regions, services, concurrency,
125
127
  }
126
128
  throw new Error(`Failed to download some data. See logs for details.`);
127
129
  }
128
- log_js_1.log.info('Finished downloads', { jobs: jobs.length });
129
130
  if (skipIndex) {
130
131
  log_js_1.log.info('Skipping indexing');
131
132
  return;
@@ -1 +1 @@
1
- {"version":3,"file":"download.js","sourceRoot":"","sources":["../../../src/download/download.ts"],"names":[],"mappings":";;AAwBA,oCAoKC;AA5LD,4CAA+C;AAC/C,oDAA6D;AAC7D,mDAU4B;AAC5B,yEAAqE;AACrE,yDAA+D;AAC/D,+DAAmE;AACnE,qDAAkD;AAClD,6CAAoD;AACpD,oDAA4D;AAC5D,8CAAiD;AACjD,gDAA4C;AAC5C,oDAA0F;AAC1F,4CAAqC;AAE9B,KAAK,UAAU,YAAY,CAChC,OAAyB,EACzB,UAAoB,EACpB,OAAiB,EACjB,QAAkB,EAClB,WAA+B,EAC/B,SAAkB;IAElB,IAAI,WAAW,KAAK,SAAS,IAAI,WAAW,IAAI,CAAC,EAAE,CAAC;QAClD,WAAW,GAAG,IAAA,4BAAkB,GAAE,CAAA;IACpC,CAAC;IAED,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC5B,MAAM,kBAAkB,GAAG,IAAA,iCAAqB,EAAC,OAAO,CAAC,CAAA;QACzD,IAAI,kBAAkB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAClC,UAAU,GAAG,kBAAkB,CAAA;QACjC,CAAC;aAAM,CAAC;YACN,MAAM,iBAAiB,GAAG,IAAA,gCAAoB,EAAC,OAAO,CAAC,CAAA;YACvD,MAAM,kBAAkB,GAAG,MAAM,IAAA,sCAAwB,EAAC,iBAAiB,EAAE;gBAC3E,KAAK,EAAE,kBAAkB;aAC1B,CAAC,CAAA;YACF,UAAU,GAAG,CAAC,kBAAkB,CAAC,SAAS,CAAC,CAAA;QAC7C,CAAC;IACH,CAAC;IAED,MAAM,aAAa,GAAG,IAAA,4BAAgB,EAAC,OAAO,CAAC,CAAA;IAC/C,IAAI,CAAC,aAAa,EAAE,CAAC;QACnB,MAAM,IAAI,KAAK,CAAC,uDAAuD,CAAC,CAAA;IAC1E,CAAC;IAED,MAAM,IAAI,GAAU,EAAE,CAAA;IACtB,MAAM,SAAS,GAAe,EAAE,CAAA;IAEhC,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE,CAAC;QACnC,YAAG,CAAC,IAAI,CAAC,+BAA+B,EAAE,EAAE,SAAS,EAAE,CAAC,CAAA;QACxD,MAAM,cAAc,GAAG,IAAA,gCAAoB,EAAC,SAAS,EAAE,OAAO,CAAC,CAAA;QAC/D,MAAM,WAAW,GAAG,MAAM,IAAA,wBAAc,EAAC,SAAS,EAAE,cAAc,CAAC,CAAA;QACnE,MAAM,SAAS,GAAG,WAAW,CAAC,SAAS,CAAA;QACvC,MAAM,eAAe,GAAG,IAAA,2CAAoB,EAAC,SAAS,CAAC,CAAA;QACvD,MAAM,cAAc,GAAG,CAAC,eAAe,EAAE,GAAG,OAAO,CAAC,CAAA;QAEpD,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACzB,OAAO,GAAG,MAAM,IAAA,8BAAiB,EAAC,WAAW,CAAC,CAAA;QAChD,CAAC;QAED,MAAM,OAAO,GAAG,IAAA,6BAAmB,EAAC,aAAa,EAAE,SAAS,CAAC,CAAA;QAC7D,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC1B,QAAQ,GAAG,yBAAkC,CAAA;QAC/C,CAAC;QACD,MAAM,WAAW,GAAG,EAAE,CAAA;QAEtB,MAAM,eAAe,GAAG,IAAA,8BAAkB,EAAC,SAAS,EAAE,cAAc,EAAE,QAAQ,CAAC,CAAA;QAC/E,KAAK,MAAM,OAAO,IAAI,eAAe,EAAE,CAAC;YACtC,YAAG,CAAC,IAAI,CAAC,mBAAmB,EAAE,EAAE,OAAO,EAAE,SAAS,EAAE,CAAC,CAAA;YACrD,MAAM,cAAc,GAAG,IAAA,6BAAiB,EAAC,OAAO,EAAE,SAAS,EAAE,cAAc,EAAE,OAAO,CAAC,CAAA;YAErF,8BAA8B;YAC9B,MAAM,WAAW,GAAG,IAAA,qCAAwB,EAAC,OAAO,CAAC,CAAA;YACrD,MAAM,YAAY,GAAG,cAAc,CAAC,EAAE,CAAC,CAAC,CAAE,CAAA;YAC1C,MAAM,YAAY,GAAG,IAAA,sCAA0B,EAC7C,OAAO,EACP,SAAS,EACT,YAAY,EACZ,cAAc,CACf,CAAA;YAED,KAAK,MAAM,UAAU,IAAI,WAAW,EAAE,CAAC;gBACrC,IAAI,CAAC,IAAI,CAAC;oBACR,UAAU,EAAE,EAAE,OAAO,EAAE,SAAS,EAAE,IAAI,EAAE,UAAU,CAAC,IAAI,EAAE;oBACzD,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE;wBACzB,MAAM,UAAU,GAAG;4BACjB,QAAQ,EAAE,OAAO,CAAC,QAAQ;4BAC1B,GAAG,OAAO,CAAC,UAAU;yBACtB,CAAA;wBACD,MAAM,iBAAiB,GAAG,MAAM,IAAA,wBAAc,EAAC,SAAS,EAAE,YAAY,CAAC,IAAI,CAAC,CAAA;wBAC5E,YAAG,CAAC,KAAK,CAAC,UAAU,EAAE,uBAAuB,CAAC,CAAA;wBAC9C,MAAM,UAAU,CAAC,OAAO,CACtB,SAAS,EACT,YAAY,EACZ,iBAAiB,EACjB,OAAO,EACP,YAAY,CAAC,QAAQ,EACrB,WAAW,CACZ,CAAA;wBACD,YAAG,CAAC,KAAK,CAAC,UAAU,EAAE,sBAAsB,CAAC,CAAA;oBAC/C,CAAC;iBACF,CAAC,CAAA;YACJ,CAAC;YAED,MAAM,aAAa,GAAG,IAAA,uCAA0B,EAAC,OAAO,CAAC,CAAA;YACzD,gCAAgC;YAChC,KAAK,MAAM,MAAM,IAAI,cAAc,EAAE,CAAC;gBACpC,YAAG,CAAC,KAAK,CAAC,EAAE,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,EAAE,wBAAwB,CAAC,CAAA;gBACnE,IAAI,aAAa,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;oBAC/B,SAAQ;gBACV,CAAC;gBACD,MAAM,SAAS,GAAG,IAAA,sCAA0B,EAAC,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,cAAc,CAAC,CAAA;gBAExF,KAAK,MAAM,IAAI,IAAI,aAAa,EAAE,CAAC;oBACjC,MAAM,WAAW,GAAG,IAAA,gCAAoB,EACtC,SAAS,EACT,OAAO,EACP,IAAI,CAAC,IAAI,EACT,cAAc,EACd,MAAM,CACP,CAAA;oBACD,IAAI,CAAC,WAAW,EAAE,CAAC;wBACjB,YAAG,CAAC,KAAK,CAAC,EAAE,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,QAAQ,EAAE,IAAI,CAAC,IAAI,EAAE,EAAE,wBAAwB,CAAC,CAAA;wBACxF,SAAQ;oBACV,CAAC;oBACD,IAAI,CAAC,IAAI,CAAC;wBACR,UAAU,EAAE,EAAE,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE;wBAC3D,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE;4BACzB,MAAM,UAAU,GAAG;gCACjB,QAAQ,EAAE,OAAO,CAAC,QAAQ;gCAC1B,GAAG,OAAO,CAAC,UAAU;6BACtB,CAAA;4BACD,YAAG,CAAC,KAAK,CAAC,UAAU,EAAE,yBAAyB,CAAC,CAAA;4BAChD,MAAM,mBAAmB,GAAG,MAAM,IAAA,wBAAc,EAAC,SAAS,EAAE,SAAS,CAAC,IAAI,CAAC,CAAA;4BAE3E,MAAM,IAAI,CAAC,OAAO,CAChB,SAAS,EACT,MAAM,EACN,mBAAmB,EACnB,OAAO,EACP,SAAS,CAAC,QAAQ,EAClB,WAAW,CACZ,CAAA;4BACD,YAAG,CAAC,KAAK,CAAC,UAAU,EAAE,wBAAwB,CAAC,CAAA;wBACjD,CAAC;qBACF,CAAC,CAAA;gBACJ,CAAC;YACH,CAAC;YAED,MAAM,QAAQ,GAAG,IAAA,mCAAqB,EAAC,OAAO,CAAC,CAAA;YAC/C,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;gBAC/B,SAAS,CAAC,IAAI,CAAC;oBACb,OAAO;oBACP,SAAS;oBACT,SAAS;oBACT,OAAO,EAAE,cAAc;iBACxB,CAAC,CAAA;YACJ,CAAC;QACH,CAAC;IACH,CAAC;IAED,YAAG,CAAC,KAAK,CAAC,oBAAoB,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,MAAM,EAAE,WAAW,EAAE,CAAC,CAAA;IACnE,MAAM,OAAO,GAAG,MAAM,IAAA,qBAAO,EAAC,IAAI,EAAE,WAAW,CAAC,CAAA;IAChD,MAAM,UAAU,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,UAAU,CAAC,CAAA;IACjE,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC1B,YAAG,CAAC,KAAK,CAAC,uBAAuB,EAAE,EAAE,UAAU,EAAE,UAAU,CAAC,MAAM,EAAE,CAAC,CAAA;QACrE,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE,CAAC;YACnC,YAAG,CAAC,KAAK,CAAC,iBAAiB,EAAE,SAAS,CAAC,MAAM,EAAE,SAAS,CAAC,UAAU,CAAC,CAAA;QACtE,CAAC;QACD,MAAM,IAAI,KAAK,CAAC,qDAAqD,CAAC,CAAA;IACxE,CAAC;IACD,YAAG,CAAC,IAAI,CAAC,oBAAoB,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC,CAAA;IAErD,IAAI,SAAS,EAAE,CAAC;QACd,YAAG,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAA;QAC7B,OAAM;IACR,CAAC;IAED,MAAM,IAAA,6BAAY,EAAC,SAAS,EAAE,aAAa,EAAE,WAAW,CAAC,CAAA;AAC3D,CAAC"}
1
+ {"version":3,"file":"download.js","sourceRoot":"","sources":["../../../src/download/download.ts"],"names":[],"mappings":";;AAwBA,oCAsKC;AA9LD,4CAA+C;AAC/C,oDAA6D;AAC7D,mDAU4B;AAC5B,yEAAqE;AACrE,yDAA+D;AAC/D,+DAAmE;AACnE,uDAAgD;AAChD,6CAAoD;AACpD,oDAA4D;AAC5D,8CAAiD;AACjD,gDAA4C;AAC5C,oDAA0F;AAC1F,4CAAqC;AAE9B,KAAK,UAAU,YAAY,CAChC,OAAyB,EACzB,UAAoB,EACpB,OAAiB,EACjB,QAAkB,EAClB,WAA+B,EAC/B,SAAkB;IAElB,IAAI,WAAW,KAAK,SAAS,IAAI,WAAW,IAAI,CAAC,EAAE,CAAC;QAClD,WAAW,GAAG,IAAA,4BAAkB,GAAE,CAAA;IACpC,CAAC;IAED,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC5B,MAAM,kBAAkB,GAAG,IAAA,iCAAqB,EAAC,OAAO,CAAC,CAAA;QACzD,IAAI,kBAAkB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAClC,UAAU,GAAG,kBAAkB,CAAA;QACjC,CAAC;aAAM,CAAC;YACN,MAAM,iBAAiB,GAAG,IAAA,gCAAoB,EAAC,OAAO,CAAC,CAAA;YACvD,MAAM,kBAAkB,GAAG,MAAM,IAAA,sCAAwB,EAAC,iBAAiB,EAAE;gBAC3E,KAAK,EAAE,kBAAkB;aAC1B,CAAC,CAAA;YACF,UAAU,GAAG,CAAC,kBAAkB,CAAC,SAAS,CAAC,CAAA;QAC7C,CAAC;IACH,CAAC;IAED,MAAM,aAAa,GAAG,IAAA,4BAAgB,EAAC,OAAO,CAAC,CAAA;IAC/C,IAAI,CAAC,aAAa,EAAE,CAAC;QACnB,MAAM,IAAI,KAAK,CAAC,uDAAuD,CAAC,CAAA;IAC1E,CAAC;IAED,MAAM,SAAS,GAAe,EAAE,CAAA;IAEhC,YAAG,CAAC,KAAK,CAAC,0BAA0B,EAAE,EAAE,WAAW,EAAE,CAAC,CAAA;IACtD,MAAM,cAAc,GAAG,IAAI,wBAAS,CAAC,WAAW,CAAC,CAAA;IAEjD,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE,CAAC;QACnC,YAAG,CAAC,IAAI,CAAC,+BAA+B,EAAE,EAAE,SAAS,EAAE,CAAC,CAAA;QACxD,MAAM,cAAc,GAAG,IAAA,gCAAoB,EAAC,SAAS,EAAE,OAAO,CAAC,CAAA;QAC/D,MAAM,WAAW,GAAG,MAAM,IAAA,wBAAc,EAAC,SAAS,EAAE,cAAc,CAAC,CAAA;QACnE,MAAM,SAAS,GAAG,WAAW,CAAC,SAAS,CAAA;QACvC,MAAM,eAAe,GAAG,IAAA,2CAAoB,EAAC,SAAS,CAAC,CAAA;QACvD,MAAM,cAAc,GAAG,CAAC,eAAe,EAAE,GAAG,OAAO,CAAC,CAAA;QAEpD,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACzB,OAAO,GAAG,MAAM,IAAA,8BAAiB,EAAC,WAAW,CAAC,CAAA;QAChD,CAAC;QAED,MAAM,OAAO,GAAG,IAAA,6BAAmB,EAAC,aAAa,EAAE,SAAS,CAAC,CAAA;QAC7D,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC1B,QAAQ,GAAG,yBAAkC,CAAA;QAC/C,CAAC;QACD,MAAM,WAAW,GAAG,EAAE,CAAA;QAEtB,MAAM,eAAe,GAAG,IAAA,8BAAkB,EAAC,SAAS,EAAE,cAAc,EAAE,QAAQ,CAAC,CAAA;QAC/E,KAAK,MAAM,OAAO,IAAI,eAAe,EAAE,CAAC;YACtC,YAAG,CAAC,IAAI,CAAC,mBAAmB,EAAE,EAAE,OAAO,EAAE,SAAS,EAAE,CAAC,CAAA;YACrD,MAAM,cAAc,GAAG,IAAA,6BAAiB,EAAC,OAAO,EAAE,SAAS,EAAE,cAAc,EAAE,OAAO,CAAC,CAAA;YAErF,8BAA8B;YAC9B,MAAM,WAAW,GAAG,IAAA,qCAAwB,EAAC,OAAO,CAAC,CAAA;YACrD,MAAM,YAAY,GAAG,cAAc,CAAC,EAAE,CAAC,CAAC,CAAE,CAAA;YAC1C,MAAM,YAAY,GAAG,IAAA,sCAA0B,EAC7C,OAAO,EACP,SAAS,EACT,YAAY,EACZ,cAAc,CACf,CAAA;YAED,KAAK,MAAM,UAAU,IAAI,WAAW,EAAE,CAAC;gBACrC,cAAc,CAAC,OAAO,CAAC;oBACrB,UAAU,EAAE,EAAE,OAAO,EAAE,SAAS,EAAE,IAAI,EAAE,UAAU,CAAC,IAAI,EAAE;oBACzD,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE;wBACzB,MAAM,UAAU,GAAG;4BACjB,QAAQ,EAAE,OAAO,CAAC,QAAQ;4BAC1B,GAAG,OAAO,CAAC,UAAU;yBACtB,CAAA;wBACD,MAAM,iBAAiB,GAAG,MAAM,IAAA,wBAAc,EAAC,SAAS,EAAE,YAAY,CAAC,IAAI,CAAC,CAAA;wBAC5E,YAAG,CAAC,KAAK,CAAC,UAAU,EAAE,uBAAuB,CAAC,CAAA;wBAC9C,MAAM,UAAU,CAAC,OAAO,CACtB,SAAS,EACT,YAAY,EACZ,iBAAiB,EACjB,OAAO,EACP,YAAY,CAAC,QAAQ,EACrB,WAAW,CACZ,CAAA;wBACD,YAAG,CAAC,KAAK,CAAC,UAAU,EAAE,sBAAsB,CAAC,CAAA;oBAC/C,CAAC;iBACF,CAAC,CAAA;YACJ,CAAC;YAED,MAAM,aAAa,GAAG,IAAA,uCAA0B,EAAC,OAAO,CAAC,CAAA;YACzD,gCAAgC;YAChC,KAAK,MAAM,MAAM,IAAI,cAAc,EAAE,CAAC;gBACpC,YAAG,CAAC,KAAK,CAAC,EAAE,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,EAAE,wBAAwB,CAAC,CAAA;gBACnE,IAAI,aAAa,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;oBAC/B,SAAQ;gBACV,CAAC;gBACD,MAAM,SAAS,GAAG,IAAA,sCAA0B,EAAC,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,cAAc,CAAC,CAAA;gBAExF,KAAK,MAAM,IAAI,IAAI,aAAa,EAAE,CAAC;oBACjC,MAAM,WAAW,GAAG,IAAA,gCAAoB,EACtC,SAAS,EACT,OAAO,EACP,IAAI,CAAC,IAAI,EACT,cAAc,EACd,MAAM,CACP,CAAA;oBACD,IAAI,CAAC,WAAW,EAAE,CAAC;wBACjB,YAAG,CAAC,KAAK,CAAC,EAAE,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,QAAQ,EAAE,IAAI,CAAC,IAAI,EAAE,EAAE,wBAAwB,CAAC,CAAA;wBACxF,SAAQ;oBACV,CAAC;oBACD,cAAc,CAAC,OAAO,CAAC;wBACrB,UAAU,EAAE,EAAE,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE;wBAC3D,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE;4BACzB,MAAM,UAAU,GAAG;gCACjB,QAAQ,EAAE,OAAO,CAAC,QAAQ;gCAC1B,GAAG,OAAO,CAAC,UAAU;6BACtB,CAAA;4BACD,YAAG,CAAC,KAAK,CAAC,UAAU,EAAE,yBAAyB,CAAC,CAAA;4BAChD,MAAM,mBAAmB,GAAG,MAAM,IAAA,wBAAc,EAAC,SAAS,EAAE,SAAS,CAAC,IAAI,CAAC,CAAA;4BAE3E,MAAM,IAAI,CAAC,OAAO,CAChB,SAAS,EACT,MAAM,EACN,mBAAmB,EACnB,OAAO,EACP,SAAS,CAAC,QAAQ,EAClB,WAAW,CACZ,CAAA;4BACD,YAAG,CAAC,KAAK,CAAC,UAAU,EAAE,wBAAwB,CAAC,CAAA;wBACjD,CAAC;qBACF,CAAC,CAAA;gBACJ,CAAC;YACH,CAAC;YAED,MAAM,QAAQ,GAAG,IAAA,mCAAqB,EAAC,OAAO,CAAC,CAAA;YAC/C,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;gBAC/B,SAAS,CAAC,IAAI,CAAC;oBACb,OAAO;oBACP,SAAS;oBACT,SAAS;oBACT,OAAO,EAAE,cAAc;iBACxB,CAAC,CAAA;YACJ,CAAC;QACH,CAAC;IACH,CAAC;IAED,YAAG,CAAC,IAAI,CAAC,mCAAmC,CAAC,CAAA;IAC7C,MAAM,cAAc,CAAC,aAAa,EAAE,CAAA;IACpC,YAAG,CAAC,IAAI,CAAC,oBAAoB,EAAE,EAAE,IAAI,EAAE,cAAc,CAAC,UAAU,EAAE,CAAC,MAAM,EAAE,CAAC,CAAA;IAC5E,MAAM,UAAU,GAAG,cAAc,CAAC,UAAU,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,UAAU,CAAC,CAAA;IACrF,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC1B,YAAG,CAAC,KAAK,CAAC,uBAAuB,EAAE,EAAE,UAAU,EAAE,UAAU,CAAC,MAAM,EAAE,CAAC,CAAA;QACrE,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE,CAAC;YACnC,YAAG,CAAC,KAAK,CAAC,iBAAiB,EAAE,SAAS,CAAC,MAAM,EAAE,SAAS,CAAC,UAAU,CAAC,CAAA;QACtE,CAAC;QACD,MAAM,IAAI,KAAK,CAAC,qDAAqD,CAAC,CAAA;IACxE,CAAC;IAED,IAAI,SAAS,EAAE,CAAC;QACd,YAAG,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAA;QAC7B,OAAM;IACR,CAAC;IAED,MAAM,IAAA,6BAAY,EAAC,SAAS,EAAE,aAAa,EAAE,WAAW,CAAC,CAAA;AAC3D,CAAC"}
@@ -6,13 +6,13 @@ const accountOrgs_js_1 = require("./indexers/accountOrgs.js");
6
6
  const apigateways_js_1 = require("./indexers/apigateways.js");
7
7
  const buckets_js_1 = require("./indexers/buckets.js");
8
8
  const iamPrincipalsToTrustPolicies_js_1 = require("./indexers/iamPrincipalsToTrustPolicies.js");
9
- const vpcEndpoints_js_1 = require("./indexers/vpcEndpoints.js");
9
+ const vpcs_js_1 = require("./indexers/vpcs.js");
10
10
  const allIndexers = [
11
11
  accountOrgs_js_1.AccountOrganizationIndexer,
12
12
  apigateways_js_1.ApiGatewayIndexer,
13
13
  iamPrincipalsToTrustPolicies_js_1.IamPrincipalsToTrustPoliciesIndexer,
14
14
  buckets_js_1.S3BucketIndexer,
15
- vpcEndpoints_js_1.VpcEndpointIndexer
15
+ vpcs_js_1.VpcEndpointIndexer
16
16
  ];
17
17
  const indexMap = new Map();
18
18
  for (const indexer of allIndexers) {
@@ -1 +1 @@
1
- {"version":3,"file":"indexMap.js","sourceRoot":"","sources":["../../../src/indexing/indexMap.ts"],"names":[],"mappings":";;AAgCA,sDAEC;AAlCD,gDAA6D;AAE7D,8DAAsE;AACtE,8DAA6D;AAC7D,sDAAuD;AACvD,gGAAgG;AAChG,gEAA+D;AAE/D,MAAM,WAAW,GAAmB;IAClC,2CAA0B;IAC1B,kCAAiB;IACjB,qEAAmC;IACnC,4BAAe;IACf,oCAAkB;CACnB,CAAA;AAED,MAAM,QAAQ,GAAG,IAAI,GAAG,EAA0B,CAAA;AAElD,KAAK,MAAM,OAAO,IAAI,WAAW,EAAE,CAAC;IAClC,MAAM,OAAO,GAAG,IAAA,8BAAgB,EAAC,OAAO,CAAC,UAAU,CAAC,CAAA;IACpD,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC;QAC3B,QAAQ,CAAC,GAAG,CAAC,OAAO,EAAE,EAAE,CAAC,CAAA;IAC3B,CAAC;IACD,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAE,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;AACtC,CAAC;AAED;;;;;GAKG;AACH,SAAgB,qBAAqB,CAAC,UAAsB;IAC1D,OAAO,QAAQ,CAAC,GAAG,CAAC,IAAA,8BAAgB,EAAC,UAAU,CAAC,CAAC,IAAI,EAAE,CAAA;AACzD,CAAC"}
1
+ {"version":3,"file":"indexMap.js","sourceRoot":"","sources":["../../../src/indexing/indexMap.ts"],"names":[],"mappings":";;AAgCA,sDAEC;AAlCD,gDAA6D;AAE7D,8DAAsE;AACtE,8DAA6D;AAC7D,sDAAuD;AACvD,gGAAgG;AAChG,gDAAuD;AAEvD,MAAM,WAAW,GAAmB;IAClC,2CAA0B;IAC1B,kCAAiB;IACjB,qEAAmC;IACnC,4BAAe;IACf,4BAAkB;CACnB,CAAA;AAED,MAAM,QAAQ,GAAG,IAAI,GAAG,EAA0B,CAAA;AAElD,KAAK,MAAM,OAAO,IAAI,WAAW,EAAE,CAAC;IAClC,MAAM,OAAO,GAAG,IAAA,8BAAgB,EAAC,OAAO,CAAC,UAAU,CAAC,CAAA;IACpD,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC;QAC3B,QAAQ,CAAC,GAAG,CAAC,OAAO,EAAE,EAAE,CAAC,CAAA;IAC3B,CAAC;IACD,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAE,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;AACtC,CAAC;AAED;;;;;GAKG;AACH,SAAgB,qBAAqB,CAAC,UAAsB;IAC1D,OAAO,QAAQ,CAAC,GAAG,CAAC,IAAA,8BAAgB,EAAC,UAAU,CAAC,CAAC,IAAI,EAAE,CAAA;AACzD,CAAC"}
@@ -0,0 +1,16 @@
1
+ import { Indexer } from '../indexer.js';
2
+ export interface VpcIndex {
3
+ vpcs: Record<string, {
4
+ arn: string;
5
+ endpoints: {
6
+ id: string;
7
+ service: string;
8
+ }[];
9
+ }>;
10
+ endpoints: Record<string, {
11
+ arn: string;
12
+ vpc: string;
13
+ }>;
14
+ }
15
+ export declare const VpcEndpointIndexer: Indexer<VpcIndex>;
16
+ //# sourceMappingURL=vpcs.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"vpcs.d.ts","sourceRoot":"","sources":["../../../../src/indexing/indexers/vpcs.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,OAAO,EAAE,MAAM,eAAe,CAAA;AAYvC,MAAM,WAAW,QAAQ;IACvB,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE;QAAE,GAAG,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE;YAAE,EAAE,EAAE,MAAM,CAAC;YAAC,OAAO,EAAE,MAAM,CAAA;SAAE,EAAE,CAAA;KAAE,CAAC,CAAA;IAEnF,SAAS,EAAE,MAAM,CAAC,MAAM,EAAE;QAAE,GAAG,EAAE,MAAM,CAAC;QAAC,GAAG,EAAE,MAAM,CAAA;KAAE,CAAC,CAAA;CACxD;AAID,eAAO,MAAM,kBAAkB,EAAE,OAAO,CAAC,QAAQ,CAgGhD,CAAA"}
@@ -0,0 +1,94 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.VpcEndpointIndexer = void 0;
4
+ const iam_utils_1 = require("@cloud-copilot/iam-utils");
5
+ const indexName = 'vpcs';
6
+ exports.VpcEndpointIndexer = {
7
+ awsService: 'ec2',
8
+ name: 'vpcs',
9
+ getCache: async (storage) => {
10
+ const data = await storage.getIndex(indexName, {
11
+ vpcs: {},
12
+ endpoints: {}
13
+ });
14
+ return data;
15
+ },
16
+ saveCache: async (storage, cache, lockId) => {
17
+ return storage.saveIndex(indexName, cache, lockId);
18
+ },
19
+ updateCache: async (existingCache, accountId, regions, storage) => {
20
+ const regionsSet = new Set(regions);
21
+ const matchesRegion = (region) => {
22
+ return region && (regionsSet.size == 0 || regionsSet.has(region));
23
+ };
24
+ const { vpcs, endpoints } = existingCache;
25
+ const currentVpcKeys = Object.keys(vpcs);
26
+ const currentEndpointKeys = Object.keys(endpoints);
27
+ // Remove all existing vpcs for the account in the specified regions
28
+ for (const key of currentVpcKeys) {
29
+ const arnParts = (0, iam_utils_1.splitArnParts)(vpcs[key].arn);
30
+ if (arnParts.accountId == accountId && matchesRegion(arnParts.region)) {
31
+ delete vpcs[key];
32
+ }
33
+ }
34
+ for (const key of currentEndpointKeys) {
35
+ const arnParts = (0, iam_utils_1.splitArnParts)(endpoints[key].arn);
36
+ if (arnParts.accountId == accountId && matchesRegion(arnParts.region)) {
37
+ delete endpoints[key];
38
+ }
39
+ }
40
+ const currentEndpoints = [];
41
+ const currentVpcs = [];
42
+ if (regions.length == 0) {
43
+ const gateways = await storage.findResourceMetadata(accountId, {
44
+ service: 'ec2',
45
+ region: '*',
46
+ resourceType: 'vpc-endpoint'
47
+ });
48
+ currentEndpoints.push(...gateways);
49
+ const vpcs = await storage.findResourceMetadata(accountId, {
50
+ service: 'ec2',
51
+ region: '*',
52
+ resourceType: 'vpc'
53
+ });
54
+ currentVpcs.push(...vpcs);
55
+ }
56
+ else {
57
+ for (const region of regions) {
58
+ const gateways = await storage.findResourceMetadata(accountId, {
59
+ service: 'ec2',
60
+ region: region,
61
+ resourceType: 'vpc-endpoint'
62
+ });
63
+ currentEndpoints.push(...gateways);
64
+ const vpcs = await storage.findResourceMetadata(accountId, {
65
+ service: 'ec2',
66
+ region: '*',
67
+ resourceType: 'vpc'
68
+ });
69
+ currentVpcs.push(...vpcs);
70
+ }
71
+ }
72
+ for (const endpoint of currentEndpoints) {
73
+ const vpcId = (0, iam_utils_1.splitArnParts)(endpoint.vpc).resourcePath;
74
+ const endpointId = (0, iam_utils_1.splitArnParts)(endpoint.arn).resourcePath;
75
+ endpoints[endpointId] = { arn: endpoint.arn, vpc: vpcId };
76
+ if (!vpcs[vpcId]) {
77
+ vpcs[vpcId] = { arn: endpoint.vpc, endpoints: [] };
78
+ }
79
+ const service = endpoint.serviceName.split('.').slice(3).join('.');
80
+ vpcs[vpcId].endpoints.push({ id: endpointId, service });
81
+ }
82
+ for (const vpc of currentVpcs) {
83
+ const vpcId = (0, iam_utils_1.splitArnParts)(vpc.arn).resourcePath;
84
+ if (!vpcs[vpcId]) {
85
+ vpcs[vpcId] = { arn: vpc.arn, endpoints: [] };
86
+ }
87
+ }
88
+ // return {
89
+ // vpcs,
90
+ // endpoints
91
+ // }
92
+ }
93
+ };
94
+ //# sourceMappingURL=vpcs.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"vpcs.js","sourceRoot":"","sources":["../../../../src/indexing/indexers/vpcs.ts"],"names":[],"mappings":";;;AAAA,wDAAwD;AAmBxD,MAAM,SAAS,GAAG,MAAM,CAAA;AAEX,QAAA,kBAAkB,GAAsB;IACnD,UAAU,EAAE,KAAK;IACjB,IAAI,EAAE,MAAM;IACZ,QAAQ,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE;QAC1B,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,QAAQ,CAAC,SAAS,EAAE;YAC7C,IAAI,EAAE,EAAE;YACR,SAAS,EAAE,EAAE;SACd,CAAC,CAAA;QACF,OAAO,IAAW,CAAA;IACpB,CAAC;IACD,SAAS,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE;QAC1C,OAAO,OAAO,CAAC,SAAS,CAAC,SAAS,EAAE,KAAK,EAAE,MAAM,CAAC,CAAA;IACpD,CAAC;IACD,WAAW,EAAE,KAAK,EAAE,aAAa,EAAE,SAAS,EAAE,OAAO,EAAE,OAAO,EAAE,EAAE;QAChE,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,CAAA;QACnC,MAAM,aAAa,GAAG,CAAC,MAA0B,EAAE,EAAE;YACnD,OAAO,MAAM,IAAI,CAAC,UAAU,CAAC,IAAI,IAAI,CAAC,IAAI,UAAU,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAA;QACnE,CAAC,CAAA;QAED,MAAM,EAAE,IAAI,EAAE,SAAS,EAAE,GAAG,aAAa,CAAA;QAEzC,MAAM,cAAc,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACxC,MAAM,mBAAmB,GAAG,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAA;QAElD,oEAAoE;QACpE,KAAK,MAAM,GAAG,IAAI,cAAc,EAAE,CAAC;YACjC,MAAM,QAAQ,GAAG,IAAA,yBAAa,EAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAA;YAC7C,IAAI,QAAQ,CAAC,SAAS,IAAI,SAAS,IAAI,aAAa,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;gBACtE,OAAO,IAAI,CAAC,GAAG,CAAC,CAAA;YAClB,CAAC;QACH,CAAC;QAED,KAAK,MAAM,GAAG,IAAI,mBAAmB,EAAE,CAAC;YACtC,MAAM,QAAQ,GAAG,IAAA,yBAAa,EAAC,SAAS,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAA;YAClD,IAAI,QAAQ,CAAC,SAAS,IAAI,SAAS,IAAI,aAAa,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;gBACtE,OAAO,SAAS,CAAC,GAAG,CAAC,CAAA;YACvB,CAAC;QACH,CAAC;QAED,MAAM,gBAAgB,GAA0B,EAAE,CAAA;QAClD,MAAM,WAAW,GAAkB,EAAE,CAAA;QAErC,IAAI,OAAO,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;YACxB,MAAM,QAAQ,GAAG,MAAM,OAAO,CAAC,oBAAoB,CAAsB,SAAS,EAAE;gBAClF,OAAO,EAAE,KAAK;gBACd,MAAM,EAAE,GAAG;gBACX,YAAY,EAAE,cAAc;aAC7B,CAAC,CAAA;YACF,gBAAgB,CAAC,IAAI,CAAC,GAAG,QAAQ,CAAC,CAAA;YAElC,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,oBAAoB,CAAc,SAAS,EAAE;gBACtE,OAAO,EAAE,KAAK;gBACd,MAAM,EAAE,GAAG;gBACX,YAAY,EAAE,KAAK;aACpB,CAAC,CAAA;YACF,WAAW,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,CAAA;QAC3B,CAAC;aAAM,CAAC;YACN,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;gBAC7B,MAAM,QAAQ,GAAG,MAAM,OAAO,CAAC,oBAAoB,CAAsB,SAAS,EAAE;oBAClF,OAAO,EAAE,KAAK;oBACd,MAAM,EAAE,MAAM;oBACd,YAAY,EAAE,cAAc;iBAC7B,CAAC,CAAA;gBACF,gBAAgB,CAAC,IAAI,CAAC,GAAG,QAAQ,CAAC,CAAA;gBAClC,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,oBAAoB,CAAc,SAAS,EAAE;oBACtE,OAAO,EAAE,KAAK;oBACd,MAAM,EAAE,GAAG;oBACX,YAAY,EAAE,KAAK;iBACpB,CAAC,CAAA;gBACF,WAAW,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,CAAA;YAC3B,CAAC;QACH,CAAC;QAED,KAAK,MAAM,QAAQ,IAAI,gBAAgB,EAAE,CAAC;YACxC,MAAM,KAAK,GAAG,IAAA,yBAAa,EAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,YAAa,CAAA;YACvD,MAAM,UAAU,GAAG,IAAA,yBAAa,EAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,YAAa,CAAA;YAC5D,SAAS,CAAC,UAAU,CAAC,GAAG,EAAE,GAAG,EAAE,QAAQ,CAAC,GAAG,EAAE,GAAG,EAAE,KAAK,EAAE,CAAA;YAEzD,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;gBACjB,IAAI,CAAC,KAAK,CAAC,GAAG,EAAE,GAAG,EAAE,QAAQ,CAAC,GAAG,EAAE,SAAS,EAAE,EAAE,EAAE,CAAA;YACpD,CAAC;YACD,MAAM,OAAO,GAAG,QAAQ,CAAC,WAAW,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;YAClE,IAAI,CAAC,KAAK,CAAC,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,UAAU,EAAE,OAAO,EAAE,CAAC,CAAA;QACzD,CAAC;QACD,KAAK,MAAM,GAAG,IAAI,WAAW,EAAE,CAAC;YAC9B,MAAM,KAAK,GAAG,IAAA,yBAAa,EAAC,GAAG,CAAC,GAAG,CAAC,CAAC,YAAa,CAAA;YAClD,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;gBACjB,IAAI,CAAC,KAAK,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,GAAG,EAAE,SAAS,EAAE,EAAE,EAAE,CAAA;YAC/C,CAAC;QACH,CAAC;QAED,WAAW;QACX,UAAU;QACV,cAAc;QACd,IAAI;IACN,CAAC;CACF,CAAA"}
@@ -0,0 +1,62 @@
1
+ import { Job, JobResult } from './jobQueue.js';
2
+ /**
3
+ * Job runner that allows running jobs as they are created.
4
+ * This will wait for jobs to be added to it and run them up the
5
+ * maximum concurrency.
6
+ *
7
+ * Results are available via `getResults()`.
8
+ */
9
+ export declare class JobRunner<T = void, P = Record<string, unknown>> {
10
+ private concurrency;
11
+ private queue;
12
+ private results;
13
+ private activeJobs;
14
+ private waitingResolvers;
15
+ private workers;
16
+ private isAcceptingWork;
17
+ private workAvailablePromise;
18
+ private resolveWorkAvailable;
19
+ /**
20
+ * Create a new runner with the specified concurrency.
21
+ *
22
+ * @param concurrency - The maximum number of jobs to run concurrently.
23
+ */
24
+ constructor(concurrency: number);
25
+ private worker;
26
+ private waitForWorkAvailable;
27
+ private ensureWorkers;
28
+ private notifyWorkersOfNewWork;
29
+ private checkIfIdle;
30
+ /**
31
+ * Add a job to the queue
32
+ */
33
+ enqueue(job: Job<T, P>): void;
34
+ /**
35
+ * Add multiple jobs to the queue
36
+ */
37
+ enqueueAll(jobs: Job<T, P>[]): void;
38
+ /**
39
+ * Returns a promise that resolves when all queued work is complete
40
+ */
41
+ waitForIdle(): Promise<void>;
42
+ /**
43
+ * Get all results accumulated so far
44
+ */
45
+ getResults(): JobResult<T, P>[];
46
+ /**
47
+ * Shutdown the queue - no new jobs will be accepted, but existing jobs will complete.
48
+ *
49
+ * Returns when a promise that resolves when all jobs have been processed and
50
+ * are available in `getResults()`.
51
+ */
52
+ finishAllWork(): Promise<void>;
53
+ /**
54
+ * Get the current queue length
55
+ */
56
+ get queueLength(): number;
57
+ /**
58
+ * Get the number of currently active jobs
59
+ */
60
+ get activeJobCount(): number;
61
+ }
62
+ //# sourceMappingURL=jobRunner.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"jobRunner.d.ts","sourceRoot":"","sources":["../../../src/jobs/jobRunner.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,GAAG,EAAc,SAAS,EAAE,MAAM,eAAe,CAAA;AAE1D;;;;;;GAMG;AAEH,qBAAa,SAAS,CAAC,CAAC,GAAG,IAAI,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC;IAe9C,OAAO,CAAC,WAAW;IAd/B,OAAO,CAAC,KAAK,CAAkB;IAC/B,OAAO,CAAC,OAAO,CAAwB;IACvC,OAAO,CAAC,UAAU,CAAI;IACtB,OAAO,CAAC,gBAAgB,CAAqB;IAC7C,OAAO,CAAC,OAAO,CAAsB;IACrC,OAAO,CAAC,eAAe,CAAO;IAC9B,OAAO,CAAC,oBAAoB,CAA6B;IACzD,OAAO,CAAC,oBAAoB,CAA4B;IAExD;;;;OAIG;gBACiB,WAAW,EAAE,MAAM;YAEzB,MAAM;IAsCpB,OAAO,CAAC,oBAAoB;IAS5B,OAAO,CAAC,aAAa;IAQrB,OAAO,CAAC,sBAAsB;IAS9B,OAAO,CAAC,WAAW;IAQnB;;OAEG;IACH,OAAO,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,IAAI;IAS7B;;OAEG;IACH,UAAU,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,GAAG,IAAI;IAInC;;OAEG;IACH,WAAW,IAAI,OAAO,CAAC,IAAI,CAAC;IAW5B;;OAEG;IACH,UAAU,IAAI,SAAS,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE;IAI/B;;;;;OAKG;IACG,aAAa,IAAI,OAAO,CAAC,IAAI,CAAC;IASpC;;OAEG;IACH,IAAI,WAAW,IAAI,MAAM,CAExB;IAED;;OAEG;IACH,IAAI,cAAc,IAAI,MAAM,CAE3B;CACF"}
@@ -0,0 +1,157 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.JobRunner = void 0;
4
+ const log_js_1 = require("../utils/log.js");
5
+ /**
6
+ * Job runner that allows running jobs as they are created.
7
+ * This will wait for jobs to be added to it and run them up the
8
+ * maximum concurrency.
9
+ *
10
+ * Results are available via `getResults()`.
11
+ */
12
+ class JobRunner {
13
+ concurrency;
14
+ queue = [];
15
+ results = [];
16
+ activeJobs = 0;
17
+ waitingResolvers = [];
18
+ workers = [];
19
+ isAcceptingWork = true;
20
+ workAvailablePromise = null;
21
+ resolveWorkAvailable = null;
22
+ /**
23
+ * Create a new runner with the specified concurrency.
24
+ *
25
+ * @param concurrency - The maximum number of jobs to run concurrently.
26
+ */
27
+ constructor(concurrency) {
28
+ this.concurrency = concurrency;
29
+ }
30
+ async worker(workerId) {
31
+ while (this.isAcceptingWork || this.queue.length > 0) {
32
+ const job = this.queue.shift();
33
+ if (!job) {
34
+ if (!this.isAcceptingWork) {
35
+ // No longer accepting work and no jobs left, exit immediately
36
+ return;
37
+ }
38
+ // No work available, wait for new work to be added
39
+ await this.waitForWorkAvailable();
40
+ continue;
41
+ }
42
+ this.activeJobs++;
43
+ const context = { workerId };
44
+ const startTime = Date.now();
45
+ const interval = setInterval(() => {
46
+ log_js_1.log.warn(`Long-running job detected.`, { minutes: Math.floor((Date.now() - startTime) / 60000) }, { ...context, ...job.properties });
47
+ }, 60_000);
48
+ try {
49
+ const value = await job.execute({ ...context, properties: job.properties });
50
+ this.results.push({ status: 'fulfilled', value, properties: job.properties });
51
+ }
52
+ catch (reason) {
53
+ this.results.push({ status: 'rejected', reason, properties: job.properties });
54
+ }
55
+ finally {
56
+ clearInterval(interval);
57
+ this.activeJobs--;
58
+ this.checkIfIdle();
59
+ }
60
+ }
61
+ }
62
+ waitForWorkAvailable() {
63
+ if (!this.workAvailablePromise) {
64
+ this.workAvailablePromise = new Promise((resolve) => {
65
+ this.resolveWorkAvailable = resolve;
66
+ });
67
+ }
68
+ return this.workAvailablePromise;
69
+ }
70
+ ensureWorkers() {
71
+ if (this.workers.length === 0 && this.isAcceptingWork) {
72
+ for (let i = 0; i < this.concurrency; i++) {
73
+ this.workers.push(this.worker(i + 1));
74
+ }
75
+ }
76
+ }
77
+ notifyWorkersOfNewWork() {
78
+ // Wake up waiting workers
79
+ if (this.resolveWorkAvailable) {
80
+ this.resolveWorkAvailable();
81
+ this.workAvailablePromise = null;
82
+ this.resolveWorkAvailable = null;
83
+ }
84
+ }
85
+ checkIfIdle() {
86
+ if (this.activeJobs === 0 && this.queue.length === 0) {
87
+ // Notify all waiting resolvers
88
+ this.waitingResolvers.forEach((resolve) => resolve());
89
+ this.waitingResolvers = [];
90
+ }
91
+ }
92
+ /**
93
+ * Add a job to the queue
94
+ */
95
+ enqueue(job) {
96
+ if (!this.isAcceptingWork) {
97
+ throw new Error('Cannot enqueue jobs after shutdown');
98
+ }
99
+ this.queue.push(job);
100
+ this.ensureWorkers();
101
+ this.notifyWorkersOfNewWork();
102
+ }
103
+ /**
104
+ * Add multiple jobs to the queue
105
+ */
106
+ enqueueAll(jobs) {
107
+ jobs.forEach((job) => this.enqueue(job));
108
+ }
109
+ /**
110
+ * Returns a promise that resolves when all queued work is complete
111
+ */
112
+ waitForIdle() {
113
+ // log.debug('waitForIdle called', this.activeJobs, this.queue.length)
114
+ return new Promise((resolve) => {
115
+ if (this.activeJobs === 0 && this.queue.length === 0) {
116
+ resolve();
117
+ }
118
+ else {
119
+ this.waitingResolvers.push(resolve);
120
+ }
121
+ });
122
+ }
123
+ /**
124
+ * Get all results accumulated so far
125
+ */
126
+ getResults() {
127
+ return this.results;
128
+ }
129
+ /**
130
+ * Shutdown the queue - no new jobs will be accepted, but existing jobs will complete.
131
+ *
132
+ * Returns when a promise that resolves when all jobs have been processed and
133
+ * are available in `getResults()`.
134
+ */
135
+ async finishAllWork() {
136
+ this.isAcceptingWork = false;
137
+ // Wake up any sleeping workers so they can process remaining jobs or exit
138
+ this.notifyWorkersOfNewWork();
139
+ // Check if we're already idle and notify any waiting resolvers
140
+ await Promise.all(this.workers);
141
+ this.workers = [];
142
+ }
143
+ /**
144
+ * Get the current queue length
145
+ */
146
+ get queueLength() {
147
+ return this.queue.length;
148
+ }
149
+ /**
150
+ * Get the number of currently active jobs
151
+ */
152
+ get activeJobCount() {
153
+ return this.activeJobs;
154
+ }
155
+ }
156
+ exports.JobRunner = JobRunner;
157
+ //# sourceMappingURL=jobRunner.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"jobRunner.js","sourceRoot":"","sources":["../../../src/jobs/jobRunner.ts"],"names":[],"mappings":";;;AAAA,4CAAqC;AAGrC;;;;;;GAMG;AAEH,MAAa,SAAS;IAeA;IAdZ,KAAK,GAAgB,EAAE,CAAA;IACvB,OAAO,GAAsB,EAAE,CAAA;IAC/B,UAAU,GAAG,CAAC,CAAA;IACd,gBAAgB,GAAmB,EAAE,CAAA;IACrC,OAAO,GAAoB,EAAE,CAAA;IAC7B,eAAe,GAAG,IAAI,CAAA;IACtB,oBAAoB,GAAyB,IAAI,CAAA;IACjD,oBAAoB,GAAwB,IAAI,CAAA;IAExD;;;;OAIG;IACH,YAAoB,WAAmB;QAAnB,gBAAW,GAAX,WAAW,CAAQ;IAAG,CAAC;IAEnC,KAAK,CAAC,MAAM,CAAC,QAAgB;QACnC,OAAO,IAAI,CAAC,eAAe,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACrD,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAA;YAC9B,IAAI,CAAC,GAAG,EAAE,CAAC;gBACT,IAAI,CAAC,IAAI,CAAC,eAAe,EAAE,CAAC;oBAC1B,8DAA8D;oBAC9D,OAAM;gBACR,CAAC;gBACD,mDAAmD;gBACnD,MAAM,IAAI,CAAC,oBAAoB,EAAE,CAAA;gBACjC,SAAQ;YACV,CAAC;YAED,IAAI,CAAC,UAAU,EAAE,CAAA;YACjB,MAAM,OAAO,GAAe,EAAE,QAAQ,EAAE,CAAA;YAExC,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAA;YAC5B,MAAM,QAAQ,GAAG,WAAW,CAAC,GAAG,EAAE;gBAChC,YAAG,CAAC,IAAI,CACN,4BAA4B,EAC5B,EAAE,OAAO,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC,GAAG,KAAK,CAAC,EAAE,EACzD,EAAE,GAAG,OAAO,EAAE,GAAG,GAAG,CAAC,UAAU,EAAE,CAClC,CAAA;YACH,CAAC,EAAE,MAAM,CAAC,CAAA;YAEV,IAAI,CAAC;gBACH,MAAM,KAAK,GAAG,MAAM,GAAG,CAAC,OAAO,CAAC,EAAE,GAAG,OAAO,EAAE,UAAU,EAAE,GAAG,CAAC,UAAU,EAAE,CAAC,CAAA;gBAC3E,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,WAAW,EAAE,KAAK,EAAE,UAAU,EAAE,GAAG,CAAC,UAAU,EAAE,CAAC,CAAA;YAC/E,CAAC;YAAC,OAAO,MAAM,EAAE,CAAC;gBAChB,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,EAAE,UAAU,EAAE,GAAG,CAAC,UAAU,EAAE,CAAC,CAAA;YAC/E,CAAC;oBAAS,CAAC;gBACT,aAAa,CAAC,QAAQ,CAAC,CAAA;gBACvB,IAAI,CAAC,UAAU,EAAE,CAAA;gBACjB,IAAI,CAAC,WAAW,EAAE,CAAA;YACpB,CAAC;QACH,CAAC;IACH,CAAC;IAEO,oBAAoB;QAC1B,IAAI,CAAC,IAAI,CAAC,oBAAoB,EAAE,CAAC;YAC/B,IAAI,CAAC,oBAAoB,GAAG,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE;gBACxD,IAAI,CAAC,oBAAoB,GAAG,OAAO,CAAA;YACrC,CAAC,CAAC,CAAA;QACJ,CAAC;QACD,OAAO,IAAI,CAAC,oBAAoB,CAAA;IAClC,CAAC;IAEO,aAAa;QACnB,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,KAAK,CAAC,IAAI,IAAI,CAAC,eAAe,EAAE,CAAC;YACtD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC,EAAE,EAAE,CAAC;gBAC1C,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAA;YACvC,CAAC;QACH,CAAC;IACH,CAAC;IAEO,sBAAsB;QAC5B,0BAA0B;QAC1B,IAAI,IAAI,CAAC,oBAAoB,EAAE,CAAC;YAC9B,IAAI,CAAC,oBAAoB,EAAE,CAAA;YAC3B,IAAI,CAAC,oBAAoB,GAAG,IAAI,CAAA;YAChC,IAAI,CAAC,oBAAoB,GAAG,IAAI,CAAA;QAClC,CAAC;IACH,CAAC;IAEO,WAAW;QACjB,IAAI,IAAI,CAAC,UAAU,KAAK,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACrD,+BAA+B;YAC/B,IAAI,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,EAAE,CAAC,CAAA;YACrD,IAAI,CAAC,gBAAgB,GAAG,EAAE,CAAA;QAC5B,CAAC;IACH,CAAC;IAED;;OAEG;IACH,OAAO,CAAC,GAAc;QACpB,IAAI,CAAC,IAAI,CAAC,eAAe,EAAE,CAAC;YAC1B,MAAM,IAAI,KAAK,CAAC,oCAAoC,CAAC,CAAA;QACvD,CAAC;QACD,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;QACpB,IAAI,CAAC,aAAa,EAAE,CAAA;QACpB,IAAI,CAAC,sBAAsB,EAAE,CAAA;IAC/B,CAAC;IAED;;OAEG;IACH,UAAU,CAAC,IAAiB;QAC1B,IAAI,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAA;IAC1C,CAAC;IAED;;OAEG;IACH,WAAW;QACT,sEAAsE;QACtE,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE;YAC7B,IAAI,IAAI,CAAC,UAAU,KAAK,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACrD,OAAO,EAAE,CAAA;YACX,CAAC;iBAAM,CAAC;gBACN,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;YACrC,CAAC;QACH,CAAC,CAAC,CAAA;IACJ,CAAC;IAED;;OAEG;IACH,UAAU;QACR,OAAO,IAAI,CAAC,OAAO,CAAA;IACrB,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,aAAa;QACjB,IAAI,CAAC,eAAe,GAAG,KAAK,CAAA;QAC5B,0EAA0E;QAC1E,IAAI,CAAC,sBAAsB,EAAE,CAAA;QAC7B,+DAA+D;QAC/D,MAAM,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;QAC/B,IAAI,CAAC,OAAO,GAAG,EAAE,CAAA;IACnB,CAAC;IAED;;OAEG;IACH,IAAI,WAAW;QACb,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,CAAA;IAC1B,CAAC;IAED;;OAEG;IACH,IAAI,cAAc;QAChB,OAAO,IAAI,CAAC,UAAU,CAAA;IACxB,CAAC;CACF;AA7JD,8BA6JC"}
@@ -1 +1 @@
1
- {"version":3,"file":"download.d.ts","sourceRoot":"","sources":["../../../src/download/download.ts"],"names":[],"mappings":"AAEA,OAAO,EASL,cAAc,EACf,MAAM,qBAAqB,CAAA;AAY5B,wBAAsB,YAAY,CAChC,OAAO,EAAE,cAAc,EAAE,EACzB,UAAU,EAAE,MAAM,EAAE,EACpB,OAAO,EAAE,MAAM,EAAE,EACjB,QAAQ,EAAE,MAAM,EAAE,EAClB,WAAW,EAAE,MAAM,GAAG,SAAS,EAC/B,SAAS,EAAE,OAAO,GACjB,OAAO,CAAC,IAAI,CAAC,CA6Jf"}
1
+ {"version":3,"file":"download.d.ts","sourceRoot":"","sources":["../../../src/download/download.ts"],"names":[],"mappings":"AAEA,OAAO,EASL,cAAc,EACf,MAAM,qBAAqB,CAAA;AAY5B,wBAAsB,YAAY,CAChC,OAAO,EAAE,cAAc,EAAE,EACzB,UAAU,EAAE,MAAM,EAAE,EACpB,OAAO,EAAE,MAAM,EAAE,EACjB,QAAQ,EAAE,MAAM,EAAE,EAClB,WAAW,EAAE,MAAM,GAAG,SAAS,EAC/B,SAAS,EAAE,OAAO,GACjB,OAAO,CAAC,IAAI,CAAC,CA+Jf"}
@@ -4,7 +4,7 @@ import { accountServiceRegionConfig, getAccountAuthConfig, getConfiguredAccounts
4
4
  import { getPartitionDefaults } from '../config/partitionDefaults.js';
5
5
  import { getIndexersForService } from '../indexing/indexMap.js';
6
6
  import { runIndexJobs } from '../indexing/runIndexers.js';
7
- import { runJobs } from '../jobs/jobQueue.js';
7
+ import { JobRunner } from '../jobs/jobRunner.js';
8
8
  import { defaultConcurrency } from '../jobs/util.js';
9
9
  import { createStorageClient } from '../persistence/util.js';
10
10
  import { getEnabledRegions } from '../regions.js';
@@ -32,8 +32,9 @@ export async function downloadData(configs, accountIds, regions, services, concu
32
32
  if (!storageConfig) {
33
33
  throw new Error('No storage configuration found. Cannot download data.');
34
34
  }
35
- const jobs = [];
36
35
  const indexJobs = [];
36
+ log.debug('Starting download runner', { concurrency });
37
+ const downloadRunner = new JobRunner(concurrency);
37
38
  for (const accountId of accountIds) {
38
39
  log.info('Queuing downloads for account', { accountId });
39
40
  const authForAccount = getAccountAuthConfig(accountId, configs);
@@ -58,7 +59,7 @@ export async function downloadData(configs, accountIds, regions, services, concu
58
59
  const globalRegion = serviceRegions.at(0);
59
60
  const globalConfig = accountServiceRegionConfig(service, accountId, globalRegion, accountConfigs);
60
61
  for (const globalSync of globalSyncs) {
61
- jobs.push({
62
+ downloadRunner.enqueue({
62
63
  properties: { service, accountId, sync: globalSync.name },
63
64
  execute: async (context) => {
64
65
  const logDetails = {
@@ -86,7 +87,7 @@ export async function downloadData(configs, accountIds, regions, services, concu
86
87
  log.debug({ service, accountId, region, syncName: sync.name }, 'Skipping regional sync');
87
88
  continue;
88
89
  }
89
- jobs.push({
90
+ downloadRunner.enqueue({
90
91
  properties: { service, accountId, region, sync: sync.name },
91
92
  execute: async (context) => {
92
93
  const logDetails = {
@@ -96,7 +97,7 @@ export async function downloadData(configs, accountIds, regions, services, concu
96
97
  log.debug(logDetails, 'Executing regional sync');
97
98
  const regionalCredentials = await getCredentials(accountId, asrConfig.auth);
98
99
  await sync.execute(accountId, region, regionalCredentials, storage, asrConfig.endpoint, syncOptions);
99
- log.debug(logDetails, 'Finished regional sync');
100
+ log.trace(logDetails, 'Finished regional sync');
100
101
  }
101
102
  });
102
103
  }
@@ -112,9 +113,10 @@ export async function downloadData(configs, accountIds, regions, services, concu
112
113
  }
113
114
  }
114
115
  }
115
- log.debug('Starting downloads', { jobs: jobs.length, concurrency });
116
- const results = await runJobs(jobs, concurrency);
117
- const failedJobs = results.filter((r) => r.status === 'rejected');
116
+ log.info('Waiting for downloads to complete');
117
+ await downloadRunner.finishAllWork();
118
+ log.info('Finished downloads', { jobs: downloadRunner.getResults().length });
119
+ const failedJobs = downloadRunner.getResults().filter((r) => r.status === 'rejected');
118
120
  if (failedJobs.length > 0) {
119
121
  log.error('Some downloads failed', { failedJobs: failedJobs.length });
120
122
  for (const failedJob of failedJobs) {
@@ -122,7 +124,6 @@ export async function downloadData(configs, accountIds, regions, services, concu
122
124
  }
123
125
  throw new Error(`Failed to download some data. See logs for details.`);
124
126
  }
125
- log.info('Finished downloads', { jobs: jobs.length });
126
127
  if (skipIndex) {
127
128
  log.info('Skipping indexing');
128
129
  return;
@@ -1 +1 @@
1
- {"version":3,"file":"download.js","sourceRoot":"","sources":["../../../src/download/download.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAA;AAC/C,OAAO,EAAE,wBAAwB,EAAE,MAAM,oBAAoB,CAAA;AAC7D,OAAO,EACL,0BAA0B,EAC1B,oBAAoB,EACpB,qBAAqB,EACrB,oBAAoB,EACpB,gBAAgB,EAChB,iBAAiB,EACjB,kBAAkB,EAClB,oBAAoB,EAErB,MAAM,qBAAqB,CAAA;AAC5B,OAAO,EAAE,oBAAoB,EAAE,MAAM,gCAAgC,CAAA;AACrE,OAAO,EAAE,qBAAqB,EAAE,MAAM,yBAAyB,CAAA;AAC/D,OAAO,EAAY,YAAY,EAAE,MAAM,4BAA4B,CAAA;AACnE,OAAO,EAAO,OAAO,EAAE,MAAM,qBAAqB,CAAA;AAClD,OAAO,EAAE,kBAAkB,EAAE,MAAM,iBAAiB,CAAA;AACpD,OAAO,EAAE,mBAAmB,EAAE,MAAM,wBAAwB,CAAA;AAC5D,OAAO,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAA;AACjD,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAA;AAC5C,OAAO,EAAE,wBAAwB,EAAE,0BAA0B,EAAE,MAAM,qBAAqB,CAAA;AAC1F,OAAO,EAAE,GAAG,EAAE,MAAM,iBAAiB,CAAA;AAErC,MAAM,CAAC,KAAK,UAAU,YAAY,CAChC,OAAyB,EACzB,UAAoB,EACpB,OAAiB,EACjB,QAAkB,EAClB,WAA+B,EAC/B,SAAkB;IAElB,IAAI,WAAW,KAAK,SAAS,IAAI,WAAW,IAAI,CAAC,EAAE,CAAC;QAClD,WAAW,GAAG,kBAAkB,EAAE,CAAA;IACpC,CAAC;IAED,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC5B,MAAM,kBAAkB,GAAG,qBAAqB,CAAC,OAAO,CAAC,CAAA;QACzD,IAAI,kBAAkB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAClC,UAAU,GAAG,kBAAkB,CAAA;QACjC,CAAC;aAAM,CAAC;YACN,MAAM,iBAAiB,GAAG,oBAAoB,CAAC,OAAO,CAAC,CAAA;YACvD,MAAM,kBAAkB,GAAG,MAAM,wBAAwB,CAAC,iBAAiB,EAAE;gBAC3E,KAAK,EAAE,kBAAkB;aAC1B,CAAC,CAAA;YACF,UAAU,GAAG,CAAC,kBAAkB,CAAC,SAAS,CAAC,CAAA;QAC7C,CAAC;IACH,CAAC;IAED,MAAM,aAAa,GAAG,gBAAgB,CAAC,OAAO,CAAC,CAAA;IAC/C,IAAI,CAAC,aAAa,EAAE,CAAC;QACnB,MAAM,IAAI,KAAK,CAAC,uDAAuD,CAAC,CAAA;IAC1E,CAAC;IAED,MAAM,IAAI,GAAU,EAAE,CAAA;IACtB,MAAM,SAAS,GAAe,EAAE,CAAA;IAEhC,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE,CAAC;QACnC,GAAG,CAAC,IAAI,CAAC,+BAA+B,EAAE,EAAE,SAAS,EAAE,CAAC,CAAA;QACxD,MAAM,cAAc,GAAG,oBAAoB,CAAC,SAAS,EAAE,OAAO,CAAC,CAAA;QAC/D,MAAM,WAAW,GAAG,MAAM,cAAc,CAAC,SAAS,EAAE,cAAc,CAAC,CAAA;QACnE,MAAM,SAAS,GAAG,WAAW,CAAC,SAAS,CAAA;QACvC,MAAM,eAAe,GAAG,oBAAoB,CAAC,SAAS,CAAC,CAAA;QACvD,MAAM,cAAc,GAAG,CAAC,eAAe,EAAE,GAAG,OAAO,CAAC,CAAA;QAEpD,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACzB,OAAO,GAAG,MAAM,iBAAiB,CAAC,WAAW,CAAC,CAAA;QAChD,CAAC;QAED,MAAM,OAAO,GAAG,mBAAmB,CAAC,aAAa,EAAE,SAAS,CAAC,CAAA;QAC7D,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC1B,QAAQ,GAAG,WAAkC,CAAA;QAC/C,CAAC;QACD,MAAM,WAAW,GAAG,EAAE,CAAA;QAEtB,MAAM,eAAe,GAAG,kBAAkB,CAAC,SAAS,EAAE,cAAc,EAAE,QAAQ,CAAC,CAAA;QAC/E,KAAK,MAAM,OAAO,IAAI,eAAe,EAAE,CAAC;YACtC,GAAG,CAAC,IAAI,CAAC,mBAAmB,EAAE,EAAE,OAAO,EAAE,SAAS,EAAE,CAAC,CAAA;YACrD,MAAM,cAAc,GAAG,iBAAiB,CAAC,OAAO,EAAE,SAAS,EAAE,cAAc,EAAE,OAAO,CAAC,CAAA;YAErF,8BAA8B;YAC9B,MAAM,WAAW,GAAG,wBAAwB,CAAC,OAAO,CAAC,CAAA;YACrD,MAAM,YAAY,GAAG,cAAc,CAAC,EAAE,CAAC,CAAC,CAAE,CAAA;YAC1C,MAAM,YAAY,GAAG,0BAA0B,CAC7C,OAAO,EACP,SAAS,EACT,YAAY,EACZ,cAAc,CACf,CAAA;YAED,KAAK,MAAM,UAAU,IAAI,WAAW,EAAE,CAAC;gBACrC,IAAI,CAAC,IAAI,CAAC;oBACR,UAAU,EAAE,EAAE,OAAO,EAAE,SAAS,EAAE,IAAI,EAAE,UAAU,CAAC,IAAI,EAAE;oBACzD,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE;wBACzB,MAAM,UAAU,GAAG;4BACjB,QAAQ,EAAE,OAAO,CAAC,QAAQ;4BAC1B,GAAG,OAAO,CAAC,UAAU;yBACtB,CAAA;wBACD,MAAM,iBAAiB,GAAG,MAAM,cAAc,CAAC,SAAS,EAAE,YAAY,CAAC,IAAI,CAAC,CAAA;wBAC5E,GAAG,CAAC,KAAK,CAAC,UAAU,EAAE,uBAAuB,CAAC,CAAA;wBAC9C,MAAM,UAAU,CAAC,OAAO,CACtB,SAAS,EACT,YAAY,EACZ,iBAAiB,EACjB,OAAO,EACP,YAAY,CAAC,QAAQ,EACrB,WAAW,CACZ,CAAA;wBACD,GAAG,CAAC,KAAK,CAAC,UAAU,EAAE,sBAAsB,CAAC,CAAA;oBAC/C,CAAC;iBACF,CAAC,CAAA;YACJ,CAAC;YAED,MAAM,aAAa,GAAG,0BAA0B,CAAC,OAAO,CAAC,CAAA;YACzD,gCAAgC;YAChC,KAAK,MAAM,MAAM,IAAI,cAAc,EAAE,CAAC;gBACpC,GAAG,CAAC,KAAK,CAAC,EAAE,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,EAAE,wBAAwB,CAAC,CAAA;gBACnE,IAAI,aAAa,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;oBAC/B,SAAQ;gBACV,CAAC;gBACD,MAAM,SAAS,GAAG,0BAA0B,CAAC,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,cAAc,CAAC,CAAA;gBAExF,KAAK,MAAM,IAAI,IAAI,aAAa,EAAE,CAAC;oBACjC,MAAM,WAAW,GAAG,oBAAoB,CACtC,SAAS,EACT,OAAO,EACP,IAAI,CAAC,IAAI,EACT,cAAc,EACd,MAAM,CACP,CAAA;oBACD,IAAI,CAAC,WAAW,EAAE,CAAC;wBACjB,GAAG,CAAC,KAAK,CAAC,EAAE,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,QAAQ,EAAE,IAAI,CAAC,IAAI,EAAE,EAAE,wBAAwB,CAAC,CAAA;wBACxF,SAAQ;oBACV,CAAC;oBACD,IAAI,CAAC,IAAI,CAAC;wBACR,UAAU,EAAE,EAAE,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE;wBAC3D,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE;4BACzB,MAAM,UAAU,GAAG;gCACjB,QAAQ,EAAE,OAAO,CAAC,QAAQ;gCAC1B,GAAG,OAAO,CAAC,UAAU;6BACtB,CAAA;4BACD,GAAG,CAAC,KAAK,CAAC,UAAU,EAAE,yBAAyB,CAAC,CAAA;4BAChD,MAAM,mBAAmB,GAAG,MAAM,cAAc,CAAC,SAAS,EAAE,SAAS,CAAC,IAAI,CAAC,CAAA;4BAE3E,MAAM,IAAI,CAAC,OAAO,CAChB,SAAS,EACT,MAAM,EACN,mBAAmB,EACnB,OAAO,EACP,SAAS,CAAC,QAAQ,EAClB,WAAW,CACZ,CAAA;4BACD,GAAG,CAAC,KAAK,CAAC,UAAU,EAAE,wBAAwB,CAAC,CAAA;wBACjD,CAAC;qBACF,CAAC,CAAA;gBACJ,CAAC;YACH,CAAC;YAED,MAAM,QAAQ,GAAG,qBAAqB,CAAC,OAAO,CAAC,CAAA;YAC/C,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;gBAC/B,SAAS,CAAC,IAAI,CAAC;oBACb,OAAO;oBACP,SAAS;oBACT,SAAS;oBACT,OAAO,EAAE,cAAc;iBACxB,CAAC,CAAA;YACJ,CAAC;QACH,CAAC;IACH,CAAC;IAED,GAAG,CAAC,KAAK,CAAC,oBAAoB,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,MAAM,EAAE,WAAW,EAAE,CAAC,CAAA;IACnE,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,IAAI,EAAE,WAAW,CAAC,CAAA;IAChD,MAAM,UAAU,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,UAAU,CAAC,CAAA;IACjE,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC1B,GAAG,CAAC,KAAK,CAAC,uBAAuB,EAAE,EAAE,UAAU,EAAE,UAAU,CAAC,MAAM,EAAE,CAAC,CAAA;QACrE,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE,CAAC;YACnC,GAAG,CAAC,KAAK,CAAC,iBAAiB,EAAE,SAAS,CAAC,MAAM,EAAE,SAAS,CAAC,UAAU,CAAC,CAAA;QACtE,CAAC;QACD,MAAM,IAAI,KAAK,CAAC,qDAAqD,CAAC,CAAA;IACxE,CAAC;IACD,GAAG,CAAC,IAAI,CAAC,oBAAoB,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC,CAAA;IAErD,IAAI,SAAS,EAAE,CAAC;QACd,GAAG,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAA;QAC7B,OAAM;IACR,CAAC;IAED,MAAM,YAAY,CAAC,SAAS,EAAE,aAAa,EAAE,WAAW,CAAC,CAAA;AAC3D,CAAC"}
1
+ {"version":3,"file":"download.js","sourceRoot":"","sources":["../../../src/download/download.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAA;AAC/C,OAAO,EAAE,wBAAwB,EAAE,MAAM,oBAAoB,CAAA;AAC7D,OAAO,EACL,0BAA0B,EAC1B,oBAAoB,EACpB,qBAAqB,EACrB,oBAAoB,EACpB,gBAAgB,EAChB,iBAAiB,EACjB,kBAAkB,EAClB,oBAAoB,EAErB,MAAM,qBAAqB,CAAA;AAC5B,OAAO,EAAE,oBAAoB,EAAE,MAAM,gCAAgC,CAAA;AACrE,OAAO,EAAE,qBAAqB,EAAE,MAAM,yBAAyB,CAAA;AAC/D,OAAO,EAAY,YAAY,EAAE,MAAM,4BAA4B,CAAA;AACnE,OAAO,EAAE,SAAS,EAAE,MAAM,sBAAsB,CAAA;AAChD,OAAO,EAAE,kBAAkB,EAAE,MAAM,iBAAiB,CAAA;AACpD,OAAO,EAAE,mBAAmB,EAAE,MAAM,wBAAwB,CAAA;AAC5D,OAAO,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAA;AACjD,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAA;AAC5C,OAAO,EAAE,wBAAwB,EAAE,0BAA0B,EAAE,MAAM,qBAAqB,CAAA;AAC1F,OAAO,EAAE,GAAG,EAAE,MAAM,iBAAiB,CAAA;AAErC,MAAM,CAAC,KAAK,UAAU,YAAY,CAChC,OAAyB,EACzB,UAAoB,EACpB,OAAiB,EACjB,QAAkB,EAClB,WAA+B,EAC/B,SAAkB;IAElB,IAAI,WAAW,KAAK,SAAS,IAAI,WAAW,IAAI,CAAC,EAAE,CAAC;QAClD,WAAW,GAAG,kBAAkB,EAAE,CAAA;IACpC,CAAC;IAED,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC5B,MAAM,kBAAkB,GAAG,qBAAqB,CAAC,OAAO,CAAC,CAAA;QACzD,IAAI,kBAAkB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAClC,UAAU,GAAG,kBAAkB,CAAA;QACjC,CAAC;aAAM,CAAC;YACN,MAAM,iBAAiB,GAAG,oBAAoB,CAAC,OAAO,CAAC,CAAA;YACvD,MAAM,kBAAkB,GAAG,MAAM,wBAAwB,CAAC,iBAAiB,EAAE;gBAC3E,KAAK,EAAE,kBAAkB;aAC1B,CAAC,CAAA;YACF,UAAU,GAAG,CAAC,kBAAkB,CAAC,SAAS,CAAC,CAAA;QAC7C,CAAC;IACH,CAAC;IAED,MAAM,aAAa,GAAG,gBAAgB,CAAC,OAAO,CAAC,CAAA;IAC/C,IAAI,CAAC,aAAa,EAAE,CAAC;QACnB,MAAM,IAAI,KAAK,CAAC,uDAAuD,CAAC,CAAA;IAC1E,CAAC;IAED,MAAM,SAAS,GAAe,EAAE,CAAA;IAEhC,GAAG,CAAC,KAAK,CAAC,0BAA0B,EAAE,EAAE,WAAW,EAAE,CAAC,CAAA;IACtD,MAAM,cAAc,GAAG,IAAI,SAAS,CAAC,WAAW,CAAC,CAAA;IAEjD,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE,CAAC;QACnC,GAAG,CAAC,IAAI,CAAC,+BAA+B,EAAE,EAAE,SAAS,EAAE,CAAC,CAAA;QACxD,MAAM,cAAc,GAAG,oBAAoB,CAAC,SAAS,EAAE,OAAO,CAAC,CAAA;QAC/D,MAAM,WAAW,GAAG,MAAM,cAAc,CAAC,SAAS,EAAE,cAAc,CAAC,CAAA;QACnE,MAAM,SAAS,GAAG,WAAW,CAAC,SAAS,CAAA;QACvC,MAAM,eAAe,GAAG,oBAAoB,CAAC,SAAS,CAAC,CAAA;QACvD,MAAM,cAAc,GAAG,CAAC,eAAe,EAAE,GAAG,OAAO,CAAC,CAAA;QAEpD,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACzB,OAAO,GAAG,MAAM,iBAAiB,CAAC,WAAW,CAAC,CAAA;QAChD,CAAC;QAED,MAAM,OAAO,GAAG,mBAAmB,CAAC,aAAa,EAAE,SAAS,CAAC,CAAA;QAC7D,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC1B,QAAQ,GAAG,WAAkC,CAAA;QAC/C,CAAC;QACD,MAAM,WAAW,GAAG,EAAE,CAAA;QAEtB,MAAM,eAAe,GAAG,kBAAkB,CAAC,SAAS,EAAE,cAAc,EAAE,QAAQ,CAAC,CAAA;QAC/E,KAAK,MAAM,OAAO,IAAI,eAAe,EAAE,CAAC;YACtC,GAAG,CAAC,IAAI,CAAC,mBAAmB,EAAE,EAAE,OAAO,EAAE,SAAS,EAAE,CAAC,CAAA;YACrD,MAAM,cAAc,GAAG,iBAAiB,CAAC,OAAO,EAAE,SAAS,EAAE,cAAc,EAAE,OAAO,CAAC,CAAA;YAErF,8BAA8B;YAC9B,MAAM,WAAW,GAAG,wBAAwB,CAAC,OAAO,CAAC,CAAA;YACrD,MAAM,YAAY,GAAG,cAAc,CAAC,EAAE,CAAC,CAAC,CAAE,CAAA;YAC1C,MAAM,YAAY,GAAG,0BAA0B,CAC7C,OAAO,EACP,SAAS,EACT,YAAY,EACZ,cAAc,CACf,CAAA;YAED,KAAK,MAAM,UAAU,IAAI,WAAW,EAAE,CAAC;gBACrC,cAAc,CAAC,OAAO,CAAC;oBACrB,UAAU,EAAE,EAAE,OAAO,EAAE,SAAS,EAAE,IAAI,EAAE,UAAU,CAAC,IAAI,EAAE;oBACzD,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE;wBACzB,MAAM,UAAU,GAAG;4BACjB,QAAQ,EAAE,OAAO,CAAC,QAAQ;4BAC1B,GAAG,OAAO,CAAC,UAAU;yBACtB,CAAA;wBACD,MAAM,iBAAiB,GAAG,MAAM,cAAc,CAAC,SAAS,EAAE,YAAY,CAAC,IAAI,CAAC,CAAA;wBAC5E,GAAG,CAAC,KAAK,CAAC,UAAU,EAAE,uBAAuB,CAAC,CAAA;wBAC9C,MAAM,UAAU,CAAC,OAAO,CACtB,SAAS,EACT,YAAY,EACZ,iBAAiB,EACjB,OAAO,EACP,YAAY,CAAC,QAAQ,EACrB,WAAW,CACZ,CAAA;wBACD,GAAG,CAAC,KAAK,CAAC,UAAU,EAAE,sBAAsB,CAAC,CAAA;oBAC/C,CAAC;iBACF,CAAC,CAAA;YACJ,CAAC;YAED,MAAM,aAAa,GAAG,0BAA0B,CAAC,OAAO,CAAC,CAAA;YACzD,gCAAgC;YAChC,KAAK,MAAM,MAAM,IAAI,cAAc,EAAE,CAAC;gBACpC,GAAG,CAAC,KAAK,CAAC,EAAE,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,EAAE,wBAAwB,CAAC,CAAA;gBACnE,IAAI,aAAa,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;oBAC/B,SAAQ;gBACV,CAAC;gBACD,MAAM,SAAS,GAAG,0BAA0B,CAAC,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,cAAc,CAAC,CAAA;gBAExF,KAAK,MAAM,IAAI,IAAI,aAAa,EAAE,CAAC;oBACjC,MAAM,WAAW,GAAG,oBAAoB,CACtC,SAAS,EACT,OAAO,EACP,IAAI,CAAC,IAAI,EACT,cAAc,EACd,MAAM,CACP,CAAA;oBACD,IAAI,CAAC,WAAW,EAAE,CAAC;wBACjB,GAAG,CAAC,KAAK,CAAC,EAAE,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,QAAQ,EAAE,IAAI,CAAC,IAAI,EAAE,EAAE,wBAAwB,CAAC,CAAA;wBACxF,SAAQ;oBACV,CAAC;oBACD,cAAc,CAAC,OAAO,CAAC;wBACrB,UAAU,EAAE,EAAE,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE;wBAC3D,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE;4BACzB,MAAM,UAAU,GAAG;gCACjB,QAAQ,EAAE,OAAO,CAAC,QAAQ;gCAC1B,GAAG,OAAO,CAAC,UAAU;6BACtB,CAAA;4BACD,GAAG,CAAC,KAAK,CAAC,UAAU,EAAE,yBAAyB,CAAC,CAAA;4BAChD,MAAM,mBAAmB,GAAG,MAAM,cAAc,CAAC,SAAS,EAAE,SAAS,CAAC,IAAI,CAAC,CAAA;4BAE3E,MAAM,IAAI,CAAC,OAAO,CAChB,SAAS,EACT,MAAM,EACN,mBAAmB,EACnB,OAAO,EACP,SAAS,CAAC,QAAQ,EAClB,WAAW,CACZ,CAAA;4BACD,GAAG,CAAC,KAAK,CAAC,UAAU,EAAE,wBAAwB,CAAC,CAAA;wBACjD,CAAC;qBACF,CAAC,CAAA;gBACJ,CAAC;YACH,CAAC;YAED,MAAM,QAAQ,GAAG,qBAAqB,CAAC,OAAO,CAAC,CAAA;YAC/C,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;gBAC/B,SAAS,CAAC,IAAI,CAAC;oBACb,OAAO;oBACP,SAAS;oBACT,SAAS;oBACT,OAAO,EAAE,cAAc;iBACxB,CAAC,CAAA;YACJ,CAAC;QACH,CAAC;IACH,CAAC;IAED,GAAG,CAAC,IAAI,CAAC,mCAAmC,CAAC,CAAA;IAC7C,MAAM,cAAc,CAAC,aAAa,EAAE,CAAA;IACpC,GAAG,CAAC,IAAI,CAAC,oBAAoB,EAAE,EAAE,IAAI,EAAE,cAAc,CAAC,UAAU,EAAE,CAAC,MAAM,EAAE,CAAC,CAAA;IAC5E,MAAM,UAAU,GAAG,cAAc,CAAC,UAAU,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,UAAU,CAAC,CAAA;IACrF,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC1B,GAAG,CAAC,KAAK,CAAC,uBAAuB,EAAE,EAAE,UAAU,EAAE,UAAU,CAAC,MAAM,EAAE,CAAC,CAAA;QACrE,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE,CAAC;YACnC,GAAG,CAAC,KAAK,CAAC,iBAAiB,EAAE,SAAS,CAAC,MAAM,EAAE,SAAS,CAAC,UAAU,CAAC,CAAA;QACtE,CAAC;QACD,MAAM,IAAI,KAAK,CAAC,qDAAqD,CAAC,CAAA;IACxE,CAAC;IAED,IAAI,SAAS,EAAE,CAAC;QACd,GAAG,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAA;QAC7B,OAAM;IACR,CAAC;IAED,MAAM,YAAY,CAAC,SAAS,EAAE,aAAa,EAAE,WAAW,CAAC,CAAA;AAC3D,CAAC"}
@@ -3,7 +3,7 @@ import { AccountOrganizationIndexer } from './indexers/accountOrgs.js';
3
3
  import { ApiGatewayIndexer } from './indexers/apigateways.js';
4
4
  import { S3BucketIndexer } from './indexers/buckets.js';
5
5
  import { IamPrincipalsToTrustPoliciesIndexer } from './indexers/iamPrincipalsToTrustPolicies.js';
6
- import { VpcEndpointIndexer } from './indexers/vpcEndpoints.js';
6
+ import { VpcEndpointIndexer } from './indexers/vpcs.js';
7
7
  const allIndexers = [
8
8
  AccountOrganizationIndexer,
9
9
  ApiGatewayIndexer,
@@ -1 +1 @@
1
- {"version":3,"file":"indexMap.js","sourceRoot":"","sources":["../../../src/indexing/indexMap.ts"],"names":[],"mappings":"AAAA,OAAO,EAAc,gBAAgB,EAAE,MAAM,gBAAgB,CAAA;AAE7D,OAAO,EAAE,0BAA0B,EAAE,MAAM,2BAA2B,CAAA;AACtE,OAAO,EAAE,iBAAiB,EAAE,MAAM,2BAA2B,CAAA;AAC7D,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAA;AACvD,OAAO,EAAE,mCAAmC,EAAE,MAAM,4CAA4C,CAAA;AAChG,OAAO,EAAE,kBAAkB,EAAE,MAAM,4BAA4B,CAAA;AAE/D,MAAM,WAAW,GAAmB;IAClC,0BAA0B;IAC1B,iBAAiB;IACjB,mCAAmC;IACnC,eAAe;IACf,kBAAkB;CACnB,CAAA;AAED,MAAM,QAAQ,GAAG,IAAI,GAAG,EAA0B,CAAA;AAElD,KAAK,MAAM,OAAO,IAAI,WAAW,EAAE,CAAC;IAClC,MAAM,OAAO,GAAG,gBAAgB,CAAC,OAAO,CAAC,UAAU,CAAC,CAAA;IACpD,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC;QAC3B,QAAQ,CAAC,GAAG,CAAC,OAAO,EAAE,EAAE,CAAC,CAAA;IAC3B,CAAC;IACD,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAE,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;AACtC,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,qBAAqB,CAAC,UAAsB;IAC1D,OAAO,QAAQ,CAAC,GAAG,CAAC,gBAAgB,CAAC,UAAU,CAAC,CAAC,IAAI,EAAE,CAAA;AACzD,CAAC"}
1
+ {"version":3,"file":"indexMap.js","sourceRoot":"","sources":["../../../src/indexing/indexMap.ts"],"names":[],"mappings":"AAAA,OAAO,EAAc,gBAAgB,EAAE,MAAM,gBAAgB,CAAA;AAE7D,OAAO,EAAE,0BAA0B,EAAE,MAAM,2BAA2B,CAAA;AACtE,OAAO,EAAE,iBAAiB,EAAE,MAAM,2BAA2B,CAAA;AAC7D,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAA;AACvD,OAAO,EAAE,mCAAmC,EAAE,MAAM,4CAA4C,CAAA;AAChG,OAAO,EAAE,kBAAkB,EAAE,MAAM,oBAAoB,CAAA;AAEvD,MAAM,WAAW,GAAmB;IAClC,0BAA0B;IAC1B,iBAAiB;IACjB,mCAAmC;IACnC,eAAe;IACf,kBAAkB;CACnB,CAAA;AAED,MAAM,QAAQ,GAAG,IAAI,GAAG,EAA0B,CAAA;AAElD,KAAK,MAAM,OAAO,IAAI,WAAW,EAAE,CAAC;IAClC,MAAM,OAAO,GAAG,gBAAgB,CAAC,OAAO,CAAC,UAAU,CAAC,CAAA;IACpD,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC;QAC3B,QAAQ,CAAC,GAAG,CAAC,OAAO,EAAE,EAAE,CAAC,CAAA;IAC3B,CAAC;IACD,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAE,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;AACtC,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,qBAAqB,CAAC,UAAsB;IAC1D,OAAO,QAAQ,CAAC,GAAG,CAAC,gBAAgB,CAAC,UAAU,CAAC,CAAC,IAAI,EAAE,CAAA;AACzD,CAAC"}
@@ -0,0 +1,16 @@
1
+ import { Indexer } from '../indexer.js';
2
+ export interface VpcIndex {
3
+ vpcs: Record<string, {
4
+ arn: string;
5
+ endpoints: {
6
+ id: string;
7
+ service: string;
8
+ }[];
9
+ }>;
10
+ endpoints: Record<string, {
11
+ arn: string;
12
+ vpc: string;
13
+ }>;
14
+ }
15
+ export declare const VpcEndpointIndexer: Indexer<VpcIndex>;
16
+ //# sourceMappingURL=vpcs.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"vpcs.d.ts","sourceRoot":"","sources":["../../../../src/indexing/indexers/vpcs.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,OAAO,EAAE,MAAM,eAAe,CAAA;AAYvC,MAAM,WAAW,QAAQ;IACvB,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE;QAAE,GAAG,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE;YAAE,EAAE,EAAE,MAAM,CAAC;YAAC,OAAO,EAAE,MAAM,CAAA;SAAE,EAAE,CAAA;KAAE,CAAC,CAAA;IAEnF,SAAS,EAAE,MAAM,CAAC,MAAM,EAAE;QAAE,GAAG,EAAE,MAAM,CAAC;QAAC,GAAG,EAAE,MAAM,CAAA;KAAE,CAAC,CAAA;CACxD;AAID,eAAO,MAAM,kBAAkB,EAAE,OAAO,CAAC,QAAQ,CAgGhD,CAAA"}
@@ -0,0 +1,91 @@
1
+ import { splitArnParts } from '@cloud-copilot/iam-utils';
2
+ const indexName = 'vpcs';
3
+ export const VpcEndpointIndexer = {
4
+ awsService: 'ec2',
5
+ name: 'vpcs',
6
+ getCache: async (storage) => {
7
+ const data = await storage.getIndex(indexName, {
8
+ vpcs: {},
9
+ endpoints: {}
10
+ });
11
+ return data;
12
+ },
13
+ saveCache: async (storage, cache, lockId) => {
14
+ return storage.saveIndex(indexName, cache, lockId);
15
+ },
16
+ updateCache: async (existingCache, accountId, regions, storage) => {
17
+ const regionsSet = new Set(regions);
18
+ const matchesRegion = (region) => {
19
+ return region && (regionsSet.size == 0 || regionsSet.has(region));
20
+ };
21
+ const { vpcs, endpoints } = existingCache;
22
+ const currentVpcKeys = Object.keys(vpcs);
23
+ const currentEndpointKeys = Object.keys(endpoints);
24
+ // Remove all existing vpcs for the account in the specified regions
25
+ for (const key of currentVpcKeys) {
26
+ const arnParts = splitArnParts(vpcs[key].arn);
27
+ if (arnParts.accountId == accountId && matchesRegion(arnParts.region)) {
28
+ delete vpcs[key];
29
+ }
30
+ }
31
+ for (const key of currentEndpointKeys) {
32
+ const arnParts = splitArnParts(endpoints[key].arn);
33
+ if (arnParts.accountId == accountId && matchesRegion(arnParts.region)) {
34
+ delete endpoints[key];
35
+ }
36
+ }
37
+ const currentEndpoints = [];
38
+ const currentVpcs = [];
39
+ if (regions.length == 0) {
40
+ const gateways = await storage.findResourceMetadata(accountId, {
41
+ service: 'ec2',
42
+ region: '*',
43
+ resourceType: 'vpc-endpoint'
44
+ });
45
+ currentEndpoints.push(...gateways);
46
+ const vpcs = await storage.findResourceMetadata(accountId, {
47
+ service: 'ec2',
48
+ region: '*',
49
+ resourceType: 'vpc'
50
+ });
51
+ currentVpcs.push(...vpcs);
52
+ }
53
+ else {
54
+ for (const region of regions) {
55
+ const gateways = await storage.findResourceMetadata(accountId, {
56
+ service: 'ec2',
57
+ region: region,
58
+ resourceType: 'vpc-endpoint'
59
+ });
60
+ currentEndpoints.push(...gateways);
61
+ const vpcs = await storage.findResourceMetadata(accountId, {
62
+ service: 'ec2',
63
+ region: '*',
64
+ resourceType: 'vpc'
65
+ });
66
+ currentVpcs.push(...vpcs);
67
+ }
68
+ }
69
+ for (const endpoint of currentEndpoints) {
70
+ const vpcId = splitArnParts(endpoint.vpc).resourcePath;
71
+ const endpointId = splitArnParts(endpoint.arn).resourcePath;
72
+ endpoints[endpointId] = { arn: endpoint.arn, vpc: vpcId };
73
+ if (!vpcs[vpcId]) {
74
+ vpcs[vpcId] = { arn: endpoint.vpc, endpoints: [] };
75
+ }
76
+ const service = endpoint.serviceName.split('.').slice(3).join('.');
77
+ vpcs[vpcId].endpoints.push({ id: endpointId, service });
78
+ }
79
+ for (const vpc of currentVpcs) {
80
+ const vpcId = splitArnParts(vpc.arn).resourcePath;
81
+ if (!vpcs[vpcId]) {
82
+ vpcs[vpcId] = { arn: vpc.arn, endpoints: [] };
83
+ }
84
+ }
85
+ // return {
86
+ // vpcs,
87
+ // endpoints
88
+ // }
89
+ }
90
+ };
91
+ //# sourceMappingURL=vpcs.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"vpcs.js","sourceRoot":"","sources":["../../../../src/indexing/indexers/vpcs.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,0BAA0B,CAAA;AAmBxD,MAAM,SAAS,GAAG,MAAM,CAAA;AAExB,MAAM,CAAC,MAAM,kBAAkB,GAAsB;IACnD,UAAU,EAAE,KAAK;IACjB,IAAI,EAAE,MAAM;IACZ,QAAQ,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE;QAC1B,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,QAAQ,CAAC,SAAS,EAAE;YAC7C,IAAI,EAAE,EAAE;YACR,SAAS,EAAE,EAAE;SACd,CAAC,CAAA;QACF,OAAO,IAAW,CAAA;IACpB,CAAC;IACD,SAAS,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE;QAC1C,OAAO,OAAO,CAAC,SAAS,CAAC,SAAS,EAAE,KAAK,EAAE,MAAM,CAAC,CAAA;IACpD,CAAC;IACD,WAAW,EAAE,KAAK,EAAE,aAAa,EAAE,SAAS,EAAE,OAAO,EAAE,OAAO,EAAE,EAAE;QAChE,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,CAAA;QACnC,MAAM,aAAa,GAAG,CAAC,MAA0B,EAAE,EAAE;YACnD,OAAO,MAAM,IAAI,CAAC,UAAU,CAAC,IAAI,IAAI,CAAC,IAAI,UAAU,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAA;QACnE,CAAC,CAAA;QAED,MAAM,EAAE,IAAI,EAAE,SAAS,EAAE,GAAG,aAAa,CAAA;QAEzC,MAAM,cAAc,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACxC,MAAM,mBAAmB,GAAG,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAA;QAElD,oEAAoE;QACpE,KAAK,MAAM,GAAG,IAAI,cAAc,EAAE,CAAC;YACjC,MAAM,QAAQ,GAAG,aAAa,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAA;YAC7C,IAAI,QAAQ,CAAC,SAAS,IAAI,SAAS,IAAI,aAAa,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;gBACtE,OAAO,IAAI,CAAC,GAAG,CAAC,CAAA;YAClB,CAAC;QACH,CAAC;QAED,KAAK,MAAM,GAAG,IAAI,mBAAmB,EAAE,CAAC;YACtC,MAAM,QAAQ,GAAG,aAAa,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAA;YAClD,IAAI,QAAQ,CAAC,SAAS,IAAI,SAAS,IAAI,aAAa,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;gBACtE,OAAO,SAAS,CAAC,GAAG,CAAC,CAAA;YACvB,CAAC;QACH,CAAC;QAED,MAAM,gBAAgB,GAA0B,EAAE,CAAA;QAClD,MAAM,WAAW,GAAkB,EAAE,CAAA;QAErC,IAAI,OAAO,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;YACxB,MAAM,QAAQ,GAAG,MAAM,OAAO,CAAC,oBAAoB,CAAsB,SAAS,EAAE;gBAClF,OAAO,EAAE,KAAK;gBACd,MAAM,EAAE,GAAG;gBACX,YAAY,EAAE,cAAc;aAC7B,CAAC,CAAA;YACF,gBAAgB,CAAC,IAAI,CAAC,GAAG,QAAQ,CAAC,CAAA;YAElC,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,oBAAoB,CAAc,SAAS,EAAE;gBACtE,OAAO,EAAE,KAAK;gBACd,MAAM,EAAE,GAAG;gBACX,YAAY,EAAE,KAAK;aACpB,CAAC,CAAA;YACF,WAAW,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,CAAA;QAC3B,CAAC;aAAM,CAAC;YACN,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;gBAC7B,MAAM,QAAQ,GAAG,MAAM,OAAO,CAAC,oBAAoB,CAAsB,SAAS,EAAE;oBAClF,OAAO,EAAE,KAAK;oBACd,MAAM,EAAE,MAAM;oBACd,YAAY,EAAE,cAAc;iBAC7B,CAAC,CAAA;gBACF,gBAAgB,CAAC,IAAI,CAAC,GAAG,QAAQ,CAAC,CAAA;gBAClC,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,oBAAoB,CAAc,SAAS,EAAE;oBACtE,OAAO,EAAE,KAAK;oBACd,MAAM,EAAE,GAAG;oBACX,YAAY,EAAE,KAAK;iBACpB,CAAC,CAAA;gBACF,WAAW,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,CAAA;YAC3B,CAAC;QACH,CAAC;QAED,KAAK,MAAM,QAAQ,IAAI,gBAAgB,EAAE,CAAC;YACxC,MAAM,KAAK,GAAG,aAAa,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,YAAa,CAAA;YACvD,MAAM,UAAU,GAAG,aAAa,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,YAAa,CAAA;YAC5D,SAAS,CAAC,UAAU,CAAC,GAAG,EAAE,GAAG,EAAE,QAAQ,CAAC,GAAG,EAAE,GAAG,EAAE,KAAK,EAAE,CAAA;YAEzD,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;gBACjB,IAAI,CAAC,KAAK,CAAC,GAAG,EAAE,GAAG,EAAE,QAAQ,CAAC,GAAG,EAAE,SAAS,EAAE,EAAE,EAAE,CAAA;YACpD,CAAC;YACD,MAAM,OAAO,GAAG,QAAQ,CAAC,WAAW,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;YAClE,IAAI,CAAC,KAAK,CAAC,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,UAAU,EAAE,OAAO,EAAE,CAAC,CAAA;QACzD,CAAC;QACD,KAAK,MAAM,GAAG,IAAI,WAAW,EAAE,CAAC;YAC9B,MAAM,KAAK,GAAG,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,YAAa,CAAA;YAClD,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;gBACjB,IAAI,CAAC,KAAK,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,GAAG,EAAE,SAAS,EAAE,EAAE,EAAE,CAAA;YAC/C,CAAC;QACH,CAAC;QAED,WAAW;QACX,UAAU;QACV,cAAc;QACd,IAAI;IACN,CAAC;CACF,CAAA"}
@@ -0,0 +1,62 @@
1
+ import { Job, JobResult } from './jobQueue.js';
2
+ /**
3
+ * Job runner that allows running jobs as they are created.
4
+ * This will wait for jobs to be added to it and run them up the
5
+ * maximum concurrency.
6
+ *
7
+ * Results are available via `getResults()`.
8
+ */
9
+ export declare class JobRunner<T = void, P = Record<string, unknown>> {
10
+ private concurrency;
11
+ private queue;
12
+ private results;
13
+ private activeJobs;
14
+ private waitingResolvers;
15
+ private workers;
16
+ private isAcceptingWork;
17
+ private workAvailablePromise;
18
+ private resolveWorkAvailable;
19
+ /**
20
+ * Create a new runner with the specified concurrency.
21
+ *
22
+ * @param concurrency - The maximum number of jobs to run concurrently.
23
+ */
24
+ constructor(concurrency: number);
25
+ private worker;
26
+ private waitForWorkAvailable;
27
+ private ensureWorkers;
28
+ private notifyWorkersOfNewWork;
29
+ private checkIfIdle;
30
+ /**
31
+ * Add a job to the queue
32
+ */
33
+ enqueue(job: Job<T, P>): void;
34
+ /**
35
+ * Add multiple jobs to the queue
36
+ */
37
+ enqueueAll(jobs: Job<T, P>[]): void;
38
+ /**
39
+ * Returns a promise that resolves when all queued work is complete
40
+ */
41
+ waitForIdle(): Promise<void>;
42
+ /**
43
+ * Get all results accumulated so far
44
+ */
45
+ getResults(): JobResult<T, P>[];
46
+ /**
47
+ * Shutdown the queue - no new jobs will be accepted, but existing jobs will complete.
48
+ *
49
+ * Returns when a promise that resolves when all jobs have been processed and
50
+ * are available in `getResults()`.
51
+ */
52
+ finishAllWork(): Promise<void>;
53
+ /**
54
+ * Get the current queue length
55
+ */
56
+ get queueLength(): number;
57
+ /**
58
+ * Get the number of currently active jobs
59
+ */
60
+ get activeJobCount(): number;
61
+ }
62
+ //# sourceMappingURL=jobRunner.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"jobRunner.d.ts","sourceRoot":"","sources":["../../../src/jobs/jobRunner.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,GAAG,EAAc,SAAS,EAAE,MAAM,eAAe,CAAA;AAE1D;;;;;;GAMG;AAEH,qBAAa,SAAS,CAAC,CAAC,GAAG,IAAI,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC;IAe9C,OAAO,CAAC,WAAW;IAd/B,OAAO,CAAC,KAAK,CAAkB;IAC/B,OAAO,CAAC,OAAO,CAAwB;IACvC,OAAO,CAAC,UAAU,CAAI;IACtB,OAAO,CAAC,gBAAgB,CAAqB;IAC7C,OAAO,CAAC,OAAO,CAAsB;IACrC,OAAO,CAAC,eAAe,CAAO;IAC9B,OAAO,CAAC,oBAAoB,CAA6B;IACzD,OAAO,CAAC,oBAAoB,CAA4B;IAExD;;;;OAIG;gBACiB,WAAW,EAAE,MAAM;YAEzB,MAAM;IAsCpB,OAAO,CAAC,oBAAoB;IAS5B,OAAO,CAAC,aAAa;IAQrB,OAAO,CAAC,sBAAsB;IAS9B,OAAO,CAAC,WAAW;IAQnB;;OAEG;IACH,OAAO,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,IAAI;IAS7B;;OAEG;IACH,UAAU,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,GAAG,IAAI;IAInC;;OAEG;IACH,WAAW,IAAI,OAAO,CAAC,IAAI,CAAC;IAW5B;;OAEG;IACH,UAAU,IAAI,SAAS,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE;IAI/B;;;;;OAKG;IACG,aAAa,IAAI,OAAO,CAAC,IAAI,CAAC;IASpC;;OAEG;IACH,IAAI,WAAW,IAAI,MAAM,CAExB;IAED;;OAEG;IACH,IAAI,cAAc,IAAI,MAAM,CAE3B;CACF"}
@@ -0,0 +1,152 @@
1
+ import { log } from '../utils/log.js';
2
+ /**
3
+ * Job runner that allows running jobs as they are created.
4
+ * This will wait for jobs to be added to it and run them up the
5
+ * maximum concurrency.
6
+ *
7
+ * Results are available via `getResults()`.
8
+ */
9
+ export class JobRunner {
10
+ /**
11
+ * Create a new runner with the specified concurrency.
12
+ *
13
+ * @param concurrency - The maximum number of jobs to run concurrently.
14
+ */
15
+ constructor(concurrency) {
16
+ this.concurrency = concurrency;
17
+ this.queue = [];
18
+ this.results = [];
19
+ this.activeJobs = 0;
20
+ this.waitingResolvers = [];
21
+ this.workers = [];
22
+ this.isAcceptingWork = true;
23
+ this.workAvailablePromise = null;
24
+ this.resolveWorkAvailable = null;
25
+ }
26
+ async worker(workerId) {
27
+ while (this.isAcceptingWork || this.queue.length > 0) {
28
+ const job = this.queue.shift();
29
+ if (!job) {
30
+ if (!this.isAcceptingWork) {
31
+ // No longer accepting work and no jobs left, exit immediately
32
+ return;
33
+ }
34
+ // No work available, wait for new work to be added
35
+ await this.waitForWorkAvailable();
36
+ continue;
37
+ }
38
+ this.activeJobs++;
39
+ const context = { workerId };
40
+ const startTime = Date.now();
41
+ const interval = setInterval(() => {
42
+ log.warn(`Long-running job detected.`, { minutes: Math.floor((Date.now() - startTime) / 60000) }, { ...context, ...job.properties });
43
+ }, 60000);
44
+ try {
45
+ const value = await job.execute({ ...context, properties: job.properties });
46
+ this.results.push({ status: 'fulfilled', value, properties: job.properties });
47
+ }
48
+ catch (reason) {
49
+ this.results.push({ status: 'rejected', reason, properties: job.properties });
50
+ }
51
+ finally {
52
+ clearInterval(interval);
53
+ this.activeJobs--;
54
+ this.checkIfIdle();
55
+ }
56
+ }
57
+ }
58
+ waitForWorkAvailable() {
59
+ if (!this.workAvailablePromise) {
60
+ this.workAvailablePromise = new Promise((resolve) => {
61
+ this.resolveWorkAvailable = resolve;
62
+ });
63
+ }
64
+ return this.workAvailablePromise;
65
+ }
66
+ ensureWorkers() {
67
+ if (this.workers.length === 0 && this.isAcceptingWork) {
68
+ for (let i = 0; i < this.concurrency; i++) {
69
+ this.workers.push(this.worker(i + 1));
70
+ }
71
+ }
72
+ }
73
+ notifyWorkersOfNewWork() {
74
+ // Wake up waiting workers
75
+ if (this.resolveWorkAvailable) {
76
+ this.resolveWorkAvailable();
77
+ this.workAvailablePromise = null;
78
+ this.resolveWorkAvailable = null;
79
+ }
80
+ }
81
+ checkIfIdle() {
82
+ if (this.activeJobs === 0 && this.queue.length === 0) {
83
+ // Notify all waiting resolvers
84
+ this.waitingResolvers.forEach((resolve) => resolve());
85
+ this.waitingResolvers = [];
86
+ }
87
+ }
88
+ /**
89
+ * Add a job to the queue
90
+ */
91
+ enqueue(job) {
92
+ if (!this.isAcceptingWork) {
93
+ throw new Error('Cannot enqueue jobs after shutdown');
94
+ }
95
+ this.queue.push(job);
96
+ this.ensureWorkers();
97
+ this.notifyWorkersOfNewWork();
98
+ }
99
+ /**
100
+ * Add multiple jobs to the queue
101
+ */
102
+ enqueueAll(jobs) {
103
+ jobs.forEach((job) => this.enqueue(job));
104
+ }
105
+ /**
106
+ * Returns a promise that resolves when all queued work is complete
107
+ */
108
+ waitForIdle() {
109
+ // log.debug('waitForIdle called', this.activeJobs, this.queue.length)
110
+ return new Promise((resolve) => {
111
+ if (this.activeJobs === 0 && this.queue.length === 0) {
112
+ resolve();
113
+ }
114
+ else {
115
+ this.waitingResolvers.push(resolve);
116
+ }
117
+ });
118
+ }
119
+ /**
120
+ * Get all results accumulated so far
121
+ */
122
+ getResults() {
123
+ return this.results;
124
+ }
125
+ /**
126
+ * Shutdown the queue - no new jobs will be accepted, but existing jobs will complete.
127
+ *
128
+ * Returns when a promise that resolves when all jobs have been processed and
129
+ * are available in `getResults()`.
130
+ */
131
+ async finishAllWork() {
132
+ this.isAcceptingWork = false;
133
+ // Wake up any sleeping workers so they can process remaining jobs or exit
134
+ this.notifyWorkersOfNewWork();
135
+ // Check if we're already idle and notify any waiting resolvers
136
+ await Promise.all(this.workers);
137
+ this.workers = [];
138
+ }
139
+ /**
140
+ * Get the current queue length
141
+ */
142
+ get queueLength() {
143
+ return this.queue.length;
144
+ }
145
+ /**
146
+ * Get the number of currently active jobs
147
+ */
148
+ get activeJobCount() {
149
+ return this.activeJobs;
150
+ }
151
+ }
152
+ //# sourceMappingURL=jobRunner.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"jobRunner.js","sourceRoot":"","sources":["../../../src/jobs/jobRunner.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,GAAG,EAAE,MAAM,iBAAiB,CAAA;AAGrC;;;;;;GAMG;AAEH,MAAM,OAAO,SAAS;IAUpB;;;;OAIG;IACH,YAAoB,WAAmB;QAAnB,gBAAW,GAAX,WAAW,CAAQ;QAd/B,UAAK,GAAgB,EAAE,CAAA;QACvB,YAAO,GAAsB,EAAE,CAAA;QAC/B,eAAU,GAAG,CAAC,CAAA;QACd,qBAAgB,GAAmB,EAAE,CAAA;QACrC,YAAO,GAAoB,EAAE,CAAA;QAC7B,oBAAe,GAAG,IAAI,CAAA;QACtB,yBAAoB,GAAyB,IAAI,CAAA;QACjD,yBAAoB,GAAwB,IAAI,CAAA;IAOd,CAAC;IAEnC,KAAK,CAAC,MAAM,CAAC,QAAgB;QACnC,OAAO,IAAI,CAAC,eAAe,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACrD,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAA;YAC9B,IAAI,CAAC,GAAG,EAAE,CAAC;gBACT,IAAI,CAAC,IAAI,CAAC,eAAe,EAAE,CAAC;oBAC1B,8DAA8D;oBAC9D,OAAM;gBACR,CAAC;gBACD,mDAAmD;gBACnD,MAAM,IAAI,CAAC,oBAAoB,EAAE,CAAA;gBACjC,SAAQ;YACV,CAAC;YAED,IAAI,CAAC,UAAU,EAAE,CAAA;YACjB,MAAM,OAAO,GAAe,EAAE,QAAQ,EAAE,CAAA;YAExC,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAA;YAC5B,MAAM,QAAQ,GAAG,WAAW,CAAC,GAAG,EAAE;gBAChC,GAAG,CAAC,IAAI,CACN,4BAA4B,EAC5B,EAAE,OAAO,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC,GAAG,KAAK,CAAC,EAAE,EACzD,EAAE,GAAG,OAAO,EAAE,GAAG,GAAG,CAAC,UAAU,EAAE,CAClC,CAAA;YACH,CAAC,EAAE,KAAM,CAAC,CAAA;YAEV,IAAI,CAAC;gBACH,MAAM,KAAK,GAAG,MAAM,GAAG,CAAC,OAAO,CAAC,EAAE,GAAG,OAAO,EAAE,UAAU,EAAE,GAAG,CAAC,UAAU,EAAE,CAAC,CAAA;gBAC3E,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,WAAW,EAAE,KAAK,EAAE,UAAU,EAAE,GAAG,CAAC,UAAU,EAAE,CAAC,CAAA;YAC/E,CAAC;YAAC,OAAO,MAAM,EAAE,CAAC;gBAChB,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,EAAE,UAAU,EAAE,GAAG,CAAC,UAAU,EAAE,CAAC,CAAA;YAC/E,CAAC;oBAAS,CAAC;gBACT,aAAa,CAAC,QAAQ,CAAC,CAAA;gBACvB,IAAI,CAAC,UAAU,EAAE,CAAA;gBACjB,IAAI,CAAC,WAAW,EAAE,CAAA;YACpB,CAAC;QACH,CAAC;IACH,CAAC;IAEO,oBAAoB;QAC1B,IAAI,CAAC,IAAI,CAAC,oBAAoB,EAAE,CAAC;YAC/B,IAAI,CAAC,oBAAoB,GAAG,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE;gBACxD,IAAI,CAAC,oBAAoB,GAAG,OAAO,CAAA;YACrC,CAAC,CAAC,CAAA;QACJ,CAAC;QACD,OAAO,IAAI,CAAC,oBAAoB,CAAA;IAClC,CAAC;IAEO,aAAa;QACnB,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,KAAK,CAAC,IAAI,IAAI,CAAC,eAAe,EAAE,CAAC;YACtD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC,EAAE,EAAE,CAAC;gBAC1C,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAA;YACvC,CAAC;QACH,CAAC;IACH,CAAC;IAEO,sBAAsB;QAC5B,0BAA0B;QAC1B,IAAI,IAAI,CAAC,oBAAoB,EAAE,CAAC;YAC9B,IAAI,CAAC,oBAAoB,EAAE,CAAA;YAC3B,IAAI,CAAC,oBAAoB,GAAG,IAAI,CAAA;YAChC,IAAI,CAAC,oBAAoB,GAAG,IAAI,CAAA;QAClC,CAAC;IACH,CAAC;IAEO,WAAW;QACjB,IAAI,IAAI,CAAC,UAAU,KAAK,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACrD,+BAA+B;YAC/B,IAAI,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,EAAE,CAAC,CAAA;YACrD,IAAI,CAAC,gBAAgB,GAAG,EAAE,CAAA;QAC5B,CAAC;IACH,CAAC;IAED;;OAEG;IACH,OAAO,CAAC,GAAc;QACpB,IAAI,CAAC,IAAI,CAAC,eAAe,EAAE,CAAC;YAC1B,MAAM,IAAI,KAAK,CAAC,oCAAoC,CAAC,CAAA;QACvD,CAAC;QACD,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;QACpB,IAAI,CAAC,aAAa,EAAE,CAAA;QACpB,IAAI,CAAC,sBAAsB,EAAE,CAAA;IAC/B,CAAC;IAED;;OAEG;IACH,UAAU,CAAC,IAAiB;QAC1B,IAAI,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAA;IAC1C,CAAC;IAED;;OAEG;IACH,WAAW;QACT,sEAAsE;QACtE,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE;YAC7B,IAAI,IAAI,CAAC,UAAU,KAAK,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACrD,OAAO,EAAE,CAAA;YACX,CAAC;iBAAM,CAAC;gBACN,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;YACrC,CAAC;QACH,CAAC,CAAC,CAAA;IACJ,CAAC;IAED;;OAEG;IACH,UAAU;QACR,OAAO,IAAI,CAAC,OAAO,CAAA;IACrB,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,aAAa;QACjB,IAAI,CAAC,eAAe,GAAG,KAAK,CAAA;QAC5B,0EAA0E;QAC1E,IAAI,CAAC,sBAAsB,EAAE,CAAA;QAC7B,+DAA+D;QAC/D,MAAM,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;QAC/B,IAAI,CAAC,OAAO,GAAG,EAAE,CAAA;IACnB,CAAC;IAED;;OAEG;IACH,IAAI,WAAW;QACb,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,CAAA;IAC1B,CAAC;IAED;;OAEG;IACH,IAAI,cAAc;QAChB,OAAO,IAAI,CAAC,UAAU,CAAA;IACxB,CAAC;CACF"}
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@cloud-copilot/iam-collect",
3
- "version": "0.1.85",
3
+ "version": "0.1.87",
4
4
  "description": "Collect IAM information from AWS Accounts",
5
5
  "repository": {
6
6
  "type": "git",
@@ -1,3 +0,0 @@
1
- import { Indexer } from '../indexer.js';
2
- export declare const VpcEndpointIndexer: Indexer<Record<string, string[]>>;
3
- //# sourceMappingURL=vpcEndpoints.d.ts.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"vpcEndpoints.d.ts","sourceRoot":"","sources":["../../../../src/indexing/indexers/vpcEndpoints.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,OAAO,EAAE,MAAM,eAAe,CAAA;AASvC,eAAO,MAAM,kBAAkB,EAAE,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,CAqDhE,CAAA"}
@@ -1,57 +0,0 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.VpcEndpointIndexer = void 0;
4
- const iam_utils_1 = require("@cloud-copilot/iam-utils");
5
- const indexName = 'vpcs-to-endpoints';
6
- exports.VpcEndpointIndexer = {
7
- awsService: 'ec2',
8
- name: 'vpcsToEndpoints',
9
- getCache: async (storage) => {
10
- const data = await storage.getIndex(indexName, {});
11
- return data;
12
- },
13
- saveCache: async (storage, cache, lockId) => {
14
- return storage.saveIndex(indexName, cache, lockId);
15
- },
16
- updateCache: async (existingCache, accountId, regions, storage) => {
17
- const regionsSet = new Set(regions);
18
- const matchesRegion = (region) => {
19
- return region && (regionsSet.size == 0 || regionsSet.has(region));
20
- };
21
- const currentCacheKeys = Object.keys(existingCache);
22
- // Remove all existing vpcs for the account in the specified regions
23
- for (const key of currentCacheKeys) {
24
- const arnParts = (0, iam_utils_1.splitArnParts)(key);
25
- if (arnParts.accountId == accountId && matchesRegion(arnParts.region)) {
26
- delete existingCache[key];
27
- }
28
- }
29
- const currentEndpoints = [];
30
- if (regions.length == 0) {
31
- const gateways = await storage.findResourceMetadata(accountId, {
32
- service: 'ec2',
33
- region: '*',
34
- resourceType: 'vpc-endpoint'
35
- });
36
- currentEndpoints.push(...gateways);
37
- }
38
- else {
39
- for (const region of regions) {
40
- const gateways = await storage.findResourceMetadata(accountId, {
41
- service: 'ec2',
42
- region: region,
43
- resourceType: 'vpc-endpoint'
44
- });
45
- currentEndpoints.push(...gateways);
46
- }
47
- }
48
- for (const endpoint of currentEndpoints) {
49
- const vpcId = endpoint.vpc;
50
- if (!existingCache[vpcId]) {
51
- existingCache[vpcId] = [];
52
- }
53
- existingCache[vpcId].push(endpoint.arn);
54
- }
55
- }
56
- };
57
- //# sourceMappingURL=vpcEndpoints.js.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"vpcEndpoints.js","sourceRoot":"","sources":["../../../../src/indexing/indexers/vpcEndpoints.ts"],"names":[],"mappings":";;;AAAA,wDAAwD;AAQxD,MAAM,SAAS,GAAG,mBAAmB,CAAA;AAExB,QAAA,kBAAkB,GAAsC;IACnE,UAAU,EAAE,KAAK;IACjB,IAAI,EAAE,iBAAiB;IACvB,QAAQ,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE;QAC1B,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,QAAQ,CAAC,SAAS,EAAE,EAAE,CAAC,CAAA;QAClD,OAAO,IAAI,CAAA;IACb,CAAC;IACD,SAAS,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE;QAC1C,OAAO,OAAO,CAAC,SAAS,CAAC,SAAS,EAAE,KAAK,EAAE,MAAM,CAAC,CAAA;IACpD,CAAC;IACD,WAAW,EAAE,KAAK,EAAE,aAAa,EAAE,SAAS,EAAE,OAAO,EAAE,OAAO,EAAE,EAAE;QAChE,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,CAAA;QACnC,MAAM,aAAa,GAAG,CAAC,MAA0B,EAAE,EAAE;YACnD,OAAO,MAAM,IAAI,CAAC,UAAU,CAAC,IAAI,IAAI,CAAC,IAAI,UAAU,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAA;QACnE,CAAC,CAAA;QACD,MAAM,gBAAgB,GAAG,MAAM,CAAC,IAAI,CAAC,aAAa,CAAC,CAAA;QAEnD,oEAAoE;QACpE,KAAK,MAAM,GAAG,IAAI,gBAAgB,EAAE,CAAC;YACnC,MAAM,QAAQ,GAAG,IAAA,yBAAa,EAAC,GAAG,CAAC,CAAA;YACnC,IAAI,QAAQ,CAAC,SAAS,IAAI,SAAS,IAAI,aAAa,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;gBACtE,OAAO,aAAa,CAAC,GAAG,CAAC,CAAA;YAC3B,CAAC;QACH,CAAC;QAED,MAAM,gBAAgB,GAA0B,EAAE,CAAA;QAElD,IAAI,OAAO,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;YACxB,MAAM,QAAQ,GAAG,MAAM,OAAO,CAAC,oBAAoB,CAAsB,SAAS,EAAE;gBAClF,OAAO,EAAE,KAAK;gBACd,MAAM,EAAE,GAAG;gBACX,YAAY,EAAE,cAAc;aAC7B,CAAC,CAAA;YACF,gBAAgB,CAAC,IAAI,CAAC,GAAG,QAAQ,CAAC,CAAA;QACpC,CAAC;aAAM,CAAC;YACN,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;gBAC7B,MAAM,QAAQ,GAAG,MAAM,OAAO,CAAC,oBAAoB,CAAsB,SAAS,EAAE;oBAClF,OAAO,EAAE,KAAK;oBACd,MAAM,EAAE,MAAM;oBACd,YAAY,EAAE,cAAc;iBAC7B,CAAC,CAAA;gBACF,gBAAgB,CAAC,IAAI,CAAC,GAAG,QAAQ,CAAC,CAAA;YACpC,CAAC;QACH,CAAC;QAED,KAAK,MAAM,QAAQ,IAAI,gBAAgB,EAAE,CAAC;YACxC,MAAM,KAAK,GAAG,QAAQ,CAAC,GAAG,CAAA;YAC1B,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,EAAE,CAAC;gBAC1B,aAAa,CAAC,KAAK,CAAC,GAAG,EAAE,CAAA;YAC3B,CAAC;YACD,aAAa,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAA;QACzC,CAAC;IACH,CAAC;CACF,CAAA"}
@@ -1,3 +0,0 @@
1
- import { Indexer } from '../indexer.js';
2
- export declare const VpcEndpointIndexer: Indexer<Record<string, string[]>>;
3
- //# sourceMappingURL=vpcEndpoints.d.ts.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"vpcEndpoints.d.ts","sourceRoot":"","sources":["../../../../src/indexing/indexers/vpcEndpoints.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,OAAO,EAAE,MAAM,eAAe,CAAA;AASvC,eAAO,MAAM,kBAAkB,EAAE,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,CAqDhE,CAAA"}
@@ -1,54 +0,0 @@
1
- import { splitArnParts } from '@cloud-copilot/iam-utils';
2
- const indexName = 'vpcs-to-endpoints';
3
- export const VpcEndpointIndexer = {
4
- awsService: 'ec2',
5
- name: 'vpcsToEndpoints',
6
- getCache: async (storage) => {
7
- const data = await storage.getIndex(indexName, {});
8
- return data;
9
- },
10
- saveCache: async (storage, cache, lockId) => {
11
- return storage.saveIndex(indexName, cache, lockId);
12
- },
13
- updateCache: async (existingCache, accountId, regions, storage) => {
14
- const regionsSet = new Set(regions);
15
- const matchesRegion = (region) => {
16
- return region && (regionsSet.size == 0 || regionsSet.has(region));
17
- };
18
- const currentCacheKeys = Object.keys(existingCache);
19
- // Remove all existing vpcs for the account in the specified regions
20
- for (const key of currentCacheKeys) {
21
- const arnParts = splitArnParts(key);
22
- if (arnParts.accountId == accountId && matchesRegion(arnParts.region)) {
23
- delete existingCache[key];
24
- }
25
- }
26
- const currentEndpoints = [];
27
- if (regions.length == 0) {
28
- const gateways = await storage.findResourceMetadata(accountId, {
29
- service: 'ec2',
30
- region: '*',
31
- resourceType: 'vpc-endpoint'
32
- });
33
- currentEndpoints.push(...gateways);
34
- }
35
- else {
36
- for (const region of regions) {
37
- const gateways = await storage.findResourceMetadata(accountId, {
38
- service: 'ec2',
39
- region: region,
40
- resourceType: 'vpc-endpoint'
41
- });
42
- currentEndpoints.push(...gateways);
43
- }
44
- }
45
- for (const endpoint of currentEndpoints) {
46
- const vpcId = endpoint.vpc;
47
- if (!existingCache[vpcId]) {
48
- existingCache[vpcId] = [];
49
- }
50
- existingCache[vpcId].push(endpoint.arn);
51
- }
52
- }
53
- };
54
- //# sourceMappingURL=vpcEndpoints.js.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"vpcEndpoints.js","sourceRoot":"","sources":["../../../../src/indexing/indexers/vpcEndpoints.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,0BAA0B,CAAA;AAQxD,MAAM,SAAS,GAAG,mBAAmB,CAAA;AAErC,MAAM,CAAC,MAAM,kBAAkB,GAAsC;IACnE,UAAU,EAAE,KAAK;IACjB,IAAI,EAAE,iBAAiB;IACvB,QAAQ,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE;QAC1B,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,QAAQ,CAAC,SAAS,EAAE,EAAE,CAAC,CAAA;QAClD,OAAO,IAAI,CAAA;IACb,CAAC;IACD,SAAS,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE;QAC1C,OAAO,OAAO,CAAC,SAAS,CAAC,SAAS,EAAE,KAAK,EAAE,MAAM,CAAC,CAAA;IACpD,CAAC;IACD,WAAW,EAAE,KAAK,EAAE,aAAa,EAAE,SAAS,EAAE,OAAO,EAAE,OAAO,EAAE,EAAE;QAChE,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,CAAA;QACnC,MAAM,aAAa,GAAG,CAAC,MAA0B,EAAE,EAAE;YACnD,OAAO,MAAM,IAAI,CAAC,UAAU,CAAC,IAAI,IAAI,CAAC,IAAI,UAAU,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAA;QACnE,CAAC,CAAA;QACD,MAAM,gBAAgB,GAAG,MAAM,CAAC,IAAI,CAAC,aAAa,CAAC,CAAA;QAEnD,oEAAoE;QACpE,KAAK,MAAM,GAAG,IAAI,gBAAgB,EAAE,CAAC;YACnC,MAAM,QAAQ,GAAG,aAAa,CAAC,GAAG,CAAC,CAAA;YACnC,IAAI,QAAQ,CAAC,SAAS,IAAI,SAAS,IAAI,aAAa,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;gBACtE,OAAO,aAAa,CAAC,GAAG,CAAC,CAAA;YAC3B,CAAC;QACH,CAAC;QAED,MAAM,gBAAgB,GAA0B,EAAE,CAAA;QAElD,IAAI,OAAO,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;YACxB,MAAM,QAAQ,GAAG,MAAM,OAAO,CAAC,oBAAoB,CAAsB,SAAS,EAAE;gBAClF,OAAO,EAAE,KAAK;gBACd,MAAM,EAAE,GAAG;gBACX,YAAY,EAAE,cAAc;aAC7B,CAAC,CAAA;YACF,gBAAgB,CAAC,IAAI,CAAC,GAAG,QAAQ,CAAC,CAAA;QACpC,CAAC;aAAM,CAAC;YACN,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;gBAC7B,MAAM,QAAQ,GAAG,MAAM,OAAO,CAAC,oBAAoB,CAAsB,SAAS,EAAE;oBAClF,OAAO,EAAE,KAAK;oBACd,MAAM,EAAE,MAAM;oBACd,YAAY,EAAE,cAAc;iBAC7B,CAAC,CAAA;gBACF,gBAAgB,CAAC,IAAI,CAAC,GAAG,QAAQ,CAAC,CAAA;YACpC,CAAC;QACH,CAAC;QAED,KAAK,MAAM,QAAQ,IAAI,gBAAgB,EAAE,CAAC;YACxC,MAAM,KAAK,GAAG,QAAQ,CAAC,GAAG,CAAA;YAC1B,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,EAAE,CAAC;gBAC1B,aAAa,CAAC,KAAK,CAAC,GAAG,EAAE,CAAA;YAC3B,CAAC;YACD,aAAa,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAA;QACzC,CAAC;IACH,CAAC;CACF,CAAA"}