@cloud-copilot/iam-collect 0.1.180 → 0.1.181
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cjs/aws/auth.js +3 -3
- package/dist/cjs/aws/auth.js.map +1 -1
- package/dist/cjs/aws/coreAuth.js +7 -7
- package/dist/cjs/aws/coreAuth.js.map +1 -1
- package/dist/cjs/cli.js +5 -5
- package/dist/cjs/cli.js.map +1 -1
- package/dist/cjs/config/version.d.ts +1 -1
- package/dist/cjs/config/version.js +1 -1
- package/dist/cjs/download/download.js +21 -21
- package/dist/cjs/download/download.js.map +1 -1
- package/dist/cjs/indexing/runIndexers.js +7 -7
- package/dist/cjs/indexing/runIndexers.js.map +1 -1
- package/dist/cjs/jobs/jobQueue.js +2 -2
- package/dist/cjs/jobs/jobQueue.js.map +1 -1
- package/dist/cjs/jobs/jobRunner.js +2 -2
- package/dist/cjs/jobs/jobRunner.js.map +1 -1
- package/dist/cjs/persistence/s3/S3PathBasedPersistenceAdapter.js +2 -2
- package/dist/cjs/persistence/s3/S3PathBasedPersistenceAdapter.js.map +1 -1
- package/dist/cjs/syncs/apigateway/gateways.js +2 -2
- package/dist/cjs/syncs/apigateway/gateways.js.map +1 -1
- package/dist/cjs/syncs/iam/authorizationDetails.js +3 -3
- package/dist/cjs/syncs/iam/authorizationDetails.js.map +1 -1
- package/dist/cjs/syncs/s3outposts/s3OutpostsSyncs.js +2 -2
- package/dist/cjs/syncs/s3outposts/s3OutpostsSyncs.js.map +1 -1
- package/dist/cjs/syncs/sns/topics.js +2 -2
- package/dist/cjs/syncs/sns/topics.js.map +1 -1
- package/dist/cjs/syncs/sso/ssoInstances.js +2 -2
- package/dist/cjs/syncs/sso/ssoInstances.js.map +1 -1
- package/dist/cjs/syncs/typedSync.js +4 -4
- package/dist/cjs/syncs/typedSync.js.map +1 -1
- package/dist/cjs/utils/client-tools.js +2 -2
- package/dist/cjs/utils/client-tools.js.map +1 -1
- package/dist/esm/aws/auth.js +1 -1
- package/dist/esm/aws/auth.js.map +1 -1
- package/dist/esm/aws/coreAuth.js +1 -1
- package/dist/esm/aws/coreAuth.js.map +1 -1
- package/dist/esm/cli.js +4 -4
- package/dist/esm/cli.js.map +1 -1
- package/dist/esm/config/version.d.ts +1 -1
- package/dist/esm/config/version.js +1 -1
- package/dist/esm/download/download.js +1 -1
- package/dist/esm/download/download.js.map +1 -1
- package/dist/esm/indexing/runIndexers.js +1 -1
- package/dist/esm/indexing/runIndexers.js.map +1 -1
- package/dist/esm/jobs/jobQueue.js +1 -1
- package/dist/esm/jobs/jobQueue.js.map +1 -1
- package/dist/esm/jobs/jobRunner.js +1 -1
- package/dist/esm/jobs/jobRunner.js.map +1 -1
- package/dist/esm/persistence/s3/S3PathBasedPersistenceAdapter.js +1 -1
- package/dist/esm/syncs/apigateway/gateways.js +1 -1
- package/dist/esm/syncs/iam/authorizationDetails.js +1 -1
- package/dist/esm/syncs/s3outposts/s3OutpostsSyncs.js +1 -1
- package/dist/esm/syncs/sns/topics.js +1 -1
- package/dist/esm/syncs/sso/ssoInstances.js +1 -1
- package/dist/esm/syncs/typedSync.js +1 -1
- package/dist/esm/syncs/typedSync.js.map +1 -1
- package/dist/esm/utils/client-tools.js +1 -1
- package/dist/esm/utils/client-tools.js.map +1 -1
- package/package.json +4 -3
- package/dist/cjs/utils/log.d.ts +0 -11
- package/dist/cjs/utils/log.d.ts.map +0 -1
- package/dist/cjs/utils/log.js +0 -100
- package/dist/cjs/utils/log.js.map +0 -1
- package/dist/esm/utils/log.d.ts +0 -11
- package/dist/esm/utils/log.d.ts.map +0 -1
- package/dist/esm/utils/log.js +0 -96
- package/dist/esm/utils/log.js.map +0 -1
package/dist/cjs/aws/auth.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.getCredentials = getCredentials;
|
|
4
|
-
const
|
|
4
|
+
const log_1 = require("@cloud-copilot/log");
|
|
5
5
|
const coreAuth_js_1 = require("./coreAuth.js");
|
|
6
6
|
/**
|
|
7
7
|
* We cache credentials with a timeout
|
|
@@ -58,7 +58,7 @@ async function getCredentials(accountId, authConfig) {
|
|
|
58
58
|
const cacheKey = credentialsCacheKey(accountId, authConfig);
|
|
59
59
|
const cachedCredentials = getCachedCredentials(cacheKey);
|
|
60
60
|
if (cachedCredentials) {
|
|
61
|
-
|
|
61
|
+
log_1.log.trace({ accountId }, 'Using cached credentials');
|
|
62
62
|
return cachedCredentials;
|
|
63
63
|
}
|
|
64
64
|
if (credentialRequestCache[cacheKey] !== undefined) {
|
|
@@ -67,7 +67,7 @@ async function getCredentials(accountId, authConfig) {
|
|
|
67
67
|
//Create a new promise and store it in case another request comes in while this one is being processed.
|
|
68
68
|
return (credentialRequestCache[cacheKey] = (async () => {
|
|
69
69
|
try {
|
|
70
|
-
|
|
70
|
+
log_1.log.trace({ accountId }, 'Creating new credentials');
|
|
71
71
|
const newCredentials = await (0, coreAuth_js_1.getNewCredentials)(accountId, authConfig);
|
|
72
72
|
setCachedCredentials(cacheKey, newCredentials);
|
|
73
73
|
return newCredentials;
|
package/dist/cjs/aws/auth.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"auth.js","sourceRoot":"","sources":["../../../src/aws/auth.ts"],"names":[],"mappings":";;AAsEA,wCA0BC;AA/FD,
|
|
1
|
+
{"version":3,"file":"auth.js","sourceRoot":"","sources":["../../../src/aws/auth.ts"],"names":[],"mappings":";;AAsEA,wCA0BC;AA/FD,4CAAwC;AACxC,+CAA8F;AAE9F;;GAEG;AACH,MAAM,gBAAgB,GAMlB,IAAI,GAAG,EAAE,CAAA;AAEb,mCAAmC;AACnC,MAAM,wBAAwB,GAAG,GAAG,GAAG,IAAI,CAAA;AAE3C;;GAEG;AACH,MAAM,sBAAsB,GAA+D,EAAE,CAAA;AAE7F;;;;;;GAMG;AACH,SAAS,mBAAmB,CAAC,SAAiB,EAAE,UAAkC;IAChF,OAAO,UAAU,CAAC,CAAC,CAAC,GAAG,SAAS,IAAI,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC,CAAC,SAAS,CAAA;AAC9E,CAAC;AAED;;;;;GAKG;AACH,SAAS,oBAAoB,CAAC,QAAgB;IAC5C,MAAM,MAAM,GAAG,gBAAgB,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAA;IAC7C,IAAI,MAAM,IAAI,MAAM,CAAC,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC;QAC7C,OAAO,MAAM,CAAC,WAAW,CAAA;IAC3B,CAAC;IACD,gBAAgB,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAA;IACjC,OAAO,SAAS,CAAA;AAClB,CAAC;AAED;;;;;GAKG;AACH,SAAS,oBAAoB,CAC3B,QAAgB,EAChB,WAA8C;IAE9C,MAAM,UAAU,GAAG,IAAA,iBAAG,GAAE,GAAG,wBAAwB,CAAA;IACnD,gBAAgB,CAAC,GAAG,CAAC,QAAQ,EAAE,EAAE,UAAU,EAAE,WAAW,EAAE,CAAC,CAAA;AAC7D,CAAC;AAED;;;;;;GAMG;AACI,KAAK,UAAU,cAAc,CAClC,SAAiB,EACjB,UAAkC;IAElC,MAAM,QAAQ,GAAG,mBAAmB,CAAC,SAAS,EAAE,UAAU,CAAC,CAAA;IAC3D,MAAM,iBAAiB,GAAG,oBAAoB,CAAC,QAAQ,CAAC,CAAA;IACxD,IAAI,iBAAiB,EAAE,CAAC;QACtB,SAAG,CAAC,KAAK,CAAC,EAAE,SAAS,EAAE,EAAE,0BAA0B,CAAC,CAAA;QACpD,OAAO,iBAAiB,CAAA;IAC1B,CAAC;IAED,IAAI,sBAAsB,CAAC,QAAQ,CAAC,KAAK,SAAS,EAAE,CAAC;QACnD,OAAO,sBAAsB,CAAC,QAAQ,CAAC,CAAA;IACzC,CAAC;IAED,uGAAuG;IACvG,OAAO,CAAC,sBAAsB,CAAC,QAAQ,CAAC,GAAG,CAAC,KAAK,IAAI,EAAE;QACrD,IAAI,CAAC;YACH,SAAG,CAAC,KAAK,CAAC,EAAE,SAAS,EAAE,EAAE,0BAA0B,CAAC,CAAA;YACpD,MAAM,cAAc,GAAG,MAAM,IAAA,+BAAiB,EAAC,SAAS,EAAE,UAAU,CAAC,CAAA;YACrE,oBAAoB,CAAC,QAAQ,EAAE,cAAc,CAAC,CAAA;YAC9C,OAAO,cAAc,CAAA;QACvB,CAAC;gBAAS,CAAC;YACT,OAAO,sBAAsB,CAAC,QAAQ,CAAC,CAAA,CAAC,uDAAuD;QACjG,CAAC;IACH,CAAC,CAAC,EAAE,CAAC,CAAA;AACP,CAAC"}
|
package/dist/cjs/aws/coreAuth.js
CHANGED
|
@@ -5,7 +5,7 @@ exports.getNewCredentials = getNewCredentials;
|
|
|
5
5
|
exports.getNewInitialCredentials = getNewInitialCredentials;
|
|
6
6
|
exports.buildRoleArn = buildRoleArn;
|
|
7
7
|
const credential_providers_1 = require("@aws-sdk/credential-providers");
|
|
8
|
-
const
|
|
8
|
+
const log_1 = require("@cloud-copilot/log");
|
|
9
9
|
const strings_js_1 = require("../utils/strings.js");
|
|
10
10
|
const tokens_js_1 = require("./tokens.js");
|
|
11
11
|
/**
|
|
@@ -35,7 +35,7 @@ async function getNewCredentials(accountId, authConfig) {
|
|
|
35
35
|
let credentials = baseCredentials;
|
|
36
36
|
if (authConfig?.role) {
|
|
37
37
|
const roleArn = buildRoleArn(baseCredentials.partition, accountId, authConfig.role.pathAndName);
|
|
38
|
-
|
|
38
|
+
log_1.log.trace({ accountId, roleArn, sourceAccount: baseCredentials.accountId }, 'Assuming role for account with credentials');
|
|
39
39
|
const roleProvider = (0, credential_providers_1.fromTemporaryCredentials)({
|
|
40
40
|
masterCredentials: baseCredentials.provider,
|
|
41
41
|
params: {
|
|
@@ -54,7 +54,7 @@ async function getNewCredentials(accountId, authConfig) {
|
|
|
54
54
|
else if (baseCredentials.accountId != accountId) {
|
|
55
55
|
// If the account ID from the credentials doesn't match the expected account ID and no role is specified
|
|
56
56
|
// throw an error to indicate that the credentials do not match the expected account
|
|
57
|
-
|
|
57
|
+
log_1.log.error('Auth config, account mismatch', {
|
|
58
58
|
desiredAccountId: accountId,
|
|
59
59
|
currentAccountId: baseCredentials.accountId
|
|
60
60
|
});
|
|
@@ -77,17 +77,17 @@ async function getNewInitialCredentials(authConfig, logInfo = {}) {
|
|
|
77
77
|
let credentials;
|
|
78
78
|
const cacheKey = await (0, strings_js_1.shortHash)(JSON.stringify({ authConfig }));
|
|
79
79
|
if (authConfig?.profile) {
|
|
80
|
-
|
|
80
|
+
log_1.log.trace({ ...logInfo, profile: authConfig.profile }, 'Using profile for credentials');
|
|
81
81
|
provider = (0, credential_providers_1.fromIni)({ profile: authConfig.profile });
|
|
82
82
|
credentials = await provider();
|
|
83
83
|
}
|
|
84
84
|
else {
|
|
85
|
-
|
|
85
|
+
log_1.log.trace(logInfo, 'Using default SDK credential chain');
|
|
86
86
|
provider = (0, credential_providers_1.fromNodeProviderChain)();
|
|
87
87
|
credentials = await provider();
|
|
88
88
|
}
|
|
89
89
|
let tokenInfo = await (0, tokens_js_1.getTokenInfo)(credentials);
|
|
90
|
-
|
|
90
|
+
log_1.log.trace('initial credentials', tokenInfo);
|
|
91
91
|
if (authConfig?.initialRole) {
|
|
92
92
|
let roleArn;
|
|
93
93
|
if ('arn' in authConfig?.initialRole) {
|
|
@@ -96,7 +96,7 @@ async function getNewInitialCredentials(authConfig, logInfo = {}) {
|
|
|
96
96
|
else {
|
|
97
97
|
roleArn = buildRoleArn(tokenInfo.partition, tokenInfo.accountId, authConfig.initialRole.pathAndName);
|
|
98
98
|
}
|
|
99
|
-
|
|
99
|
+
log_1.log.trace({ roleArn, sourceAccount: tokenInfo.accountId, ...logInfo }, 'Assuming initial role for account with credentials');
|
|
100
100
|
const roleProvider = (0, credential_providers_1.fromTemporaryCredentials)({
|
|
101
101
|
masterCredentials: credentials,
|
|
102
102
|
params: {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"coreAuth.js","sourceRoot":"","sources":["../../../src/aws/coreAuth.ts"],"names":[],"mappings":";;AA4DA,kBAEC;AAWD,8CA4CC;AAYD,4DAwDC;AAUD,oCASC;AA5MD,wEAIsC;AAOtC,
|
|
1
|
+
{"version":3,"file":"coreAuth.js","sourceRoot":"","sources":["../../../src/aws/coreAuth.ts"],"names":[],"mappings":";;AA4DA,kBAEC;AAWD,8CA4CC;AAYD,4DAwDC;AAUD,oCASC;AA5MD,wEAIsC;AAOtC,4CAAwC;AACxC,oDAAiE;AACjE,2CAA0C;AAwC1C;;;;;;GAMG;AACH,SAAgB,GAAG;IACjB,OAAO,IAAI,CAAC,GAAG,EAAE,CAAA;AACnB,CAAC;AAED;;;;;;;;GAQG;AACI,KAAK,UAAU,iBAAiB,CACrC,SAAiB,EACjB,UAAkC;IAElC,MAAM,QAAQ,GAAG,MAAM,IAAA,sBAAS,EAAC,IAAI,CAAC,SAAS,CAAC,EAAE,SAAS,EAAE,UAAU,EAAE,CAAC,CAAC,CAAA;IAC3E,MAAM,eAAe,GAAG,MAAM,wBAAwB,CAAC,UAAU,EAAE;QACjE,SAAS;KACV,CAAC,CAAA;IAEF,IAAI,WAAW,GAAG,eAAe,CAAA;IACjC,IAAI,UAAU,EAAE,IAAI,EAAE,CAAC;QACrB,MAAM,OAAO,GAAG,YAAY,CAAC,eAAe,CAAC,SAAS,EAAE,SAAS,EAAE,UAAU,CAAC,IAAI,CAAC,WAAW,CAAC,CAAA;QAC/F,SAAG,CAAC,KAAK,CACP,EAAE,SAAS,EAAE,OAAO,EAAE,aAAa,EAAE,eAAe,CAAC,SAAS,EAAE,EAChE,4CAA4C,CAC7C,CAAA;QACD,MAAM,YAAY,GAAG,IAAA,+CAAwB,EAAC;YAC5C,iBAAiB,EAAE,eAAe,CAAC,QAAQ;YAC3C,MAAM,EAAE;gBACN,OAAO,EAAE,OAAO;gBAChB,UAAU,EAAE,UAAU,CAAC,IAAI,CAAC,UAAU;gBACtC,eAAe,EAAE,UAAU,CAAC,IAAI,CAAC,WAAW,IAAI,eAAe,IAAA,6BAAgB,GAAE,EAAE;aACpF;SACF,CAAC,CAAA;QAEF,WAAW,GAAG;YACZ,QAAQ;YACR,QAAQ,EAAE,YAAY;YACtB,SAAS,EAAE,SAAS;YACpB,SAAS,EAAE,eAAe,CAAC,SAAS;SACrC,CAAA;IACH,CAAC;SAAM,IAAI,eAAe,CAAC,SAAS,IAAI,SAAS,EAAE,CAAC;QAClD,wGAAwG;QACxG,oFAAoF;QACpF,SAAG,CAAC,KAAK,CAAC,+BAA+B,EAAE;YACzC,gBAAgB,EAAE,SAAS;YAC3B,gBAAgB,EAAE,eAAe,CAAC,SAAS;SAC5C,CAAC,CAAA;QACF,MAAM,IAAI,KAAK,CACb,iEAAiE,SAAS,WAAW,eAAe,CAAC,SAAS,yCAAyC,CACxJ,CAAA;IACH,CAAC;IAED,OAAO,WAAW,CAAA;AACpB,CAAC;AAED;;;;;;;;;GASG;AACI,KAAK,UAAU,wBAAwB,CAC5C,UAAkC,EAClC,UAAmC,EAAE;IAErC,IAAI,QAAgC,CAAA;IACpC,IAAI,WAAkC,CAAA;IACtC,MAAM,QAAQ,GAAG,MAAM,IAAA,sBAAS,EAAC,IAAI,CAAC,SAAS,CAAC,EAAE,UAAU,EAAE,CAAC,CAAC,CAAA;IAChE,IAAI,UAAU,EAAE,OAAO,EAAE,CAAC;QACxB,SAAG,CAAC,KAAK,CAAC,EAAE,GAAG,OAAO,EAAE,OAAO,EAAE,UAAU,CAAC,OAAO,EAAE,EAAE,+BAA+B,CAAC,CAAA;QACvF,QAAQ,GAAG,IAAA,8BAAO,EAAC,EAAE,OAAO,EAAE,UAAU,CAAC,OAAO,EAAE,CAAC,CAAA;QACnD,WAAW,GAAG,MAAM,QAAQ,EAAE,CAAA;IAChC,CAAC;SAAM,CAAC;QACN,SAAG,CAAC,KAAK,CAAC,OAAO,EAAE,oCAAoC,CAAC,CAAA;QACxD,QAAQ,GAAG,IAAA,4CAAqB,GAAE,CAAA;QAClC,WAAW,GAAG,MAAM,QAAQ,EAAE,CAAA;IAChC,CAAC;IAED,IAAI,SAAS,GAAG,MAAM,IAAA,wBAAY,EAAC,WAAW,CAAC,CAAA;IAC/C,SAAG,CAAC,KAAK,CAAC,qBAAqB,EAAE,SAAS,CAAC,CAAA;IAE3C,IAAI,UAAU,EAAE,WAAW,EAAE,CAAC;QAC5B,IAAI,OAAe,CAAA;QACnB,IAAI,KAAK,IAAI,UAAU,EAAE,WAAW,EAAE,CAAC;YACrC,OAAO,GAAG,UAAU,CAAC,WAAW,CAAC,GAAG,CAAA;QACtC,CAAC;aAAM,CAAC;YACN,OAAO,GAAG,YAAY,CACpB,SAAS,CAAC,SAAS,EACnB,SAAS,CAAC,SAAS,EACnB,UAAU,CAAC,WAAW,CAAC,WAAW,CACnC,CAAA;QACH,CAAC;QAED,SAAG,CAAC,KAAK,CACP,EAAE,OAAO,EAAE,aAAa,EAAE,SAAS,CAAC,SAAS,EAAE,GAAG,OAAO,EAAE,EAC3D,oDAAoD,CACrD,CAAA;QACD,MAAM,YAAY,GAAG,IAAA,+CAAwB,EAAC;YAC5C,iBAAiB,EAAE,WAAW;YAC9B,MAAM,EAAE;gBACN,OAAO,EAAE,OAAO;gBAChB,UAAU,EAAE,UAAU,CAAC,WAAW,CAAC,UAAU;gBAC7C,eAAe,EAAE,UAAU,CAAC,WAAW,CAAC,WAAW,IAAI,eAAe,IAAA,6BAAgB,GAAE,EAAE;aAC3F;SACF,CAAC,CAAA;QAEF,QAAQ,GAAG,YAAY,CAAA;QACvB,WAAW,GAAG,MAAM,YAAY,EAAE,CAAA;QAClC,SAAS,GAAG,MAAM,IAAA,wBAAY,EAAC,WAAW,CAAC,CAAA;IAC7C,CAAC;IAED,OAAO;QACL,QAAQ;QACR,QAAQ;QACR,SAAS,EAAE,SAAS,CAAC,SAAS;QAC9B,SAAS,EAAE,SAAS,CAAC,SAAS;KAC/B,CAAA;AACH,CAAC;AAED;;;;;;;GAOG;AACH,SAAgB,YAAY,CAC1B,SAAiB,EACjB,SAAiB,EACjB,eAAuB;IAEvB,IAAI,CAAC,eAAe,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;QACrC,eAAe,GAAG,IAAI,eAAe,EAAE,CAAA;IACzC,CAAC;IACD,OAAO,OAAO,SAAS,SAAS,SAAS,QAAQ,eAAe,EAAE,CAAA;AACpE,CAAC"}
|
package/dist/cjs/cli.js
CHANGED
|
@@ -10,7 +10,7 @@ const packageVersion_js_1 = require("./config/packageVersion.js");
|
|
|
10
10
|
const download_js_1 = require("./download/download.js");
|
|
11
11
|
const index_js_1 = require("./index/index.js");
|
|
12
12
|
const mergeSqlite_js_1 = require("./mergeSqlite/mergeSqlite.js");
|
|
13
|
-
const
|
|
13
|
+
const log_1 = require("@cloud-copilot/log");
|
|
14
14
|
/**
|
|
15
15
|
* For some reason the AWS SDK v3 looks for AWS_REGION and not AWS_DEFAULT_REGION
|
|
16
16
|
* even though other SDKs and the CLI use AWS_DEFAULT_REGION. To make things easier
|
|
@@ -107,7 +107,7 @@ const main = async () => {
|
|
|
107
107
|
}, {
|
|
108
108
|
log: (0, cli_1.enumArgument)({
|
|
109
109
|
description: 'The log level to use',
|
|
110
|
-
validValues: [...
|
|
110
|
+
validValues: [...log_1.LogLevels] // Convert readonly to mutable array
|
|
111
111
|
})
|
|
112
112
|
}, {
|
|
113
113
|
envPrefix: 'IAM_COLLECT',
|
|
@@ -119,9 +119,9 @@ const main = async () => {
|
|
|
119
119
|
checkForUpdates: '@cloud-copilot/iam-collect'
|
|
120
120
|
}
|
|
121
121
|
});
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
}
|
|
122
|
+
(0, log_1.setLogger)(new log_1.StandardLogger({
|
|
123
|
+
logLevel: cli.args.log ?? undefined
|
|
124
|
+
}));
|
|
125
125
|
if (cli.subcommand === 'init') {
|
|
126
126
|
if ((0, defaultConfig_js_1.defaultConfigExists)()) {
|
|
127
127
|
console.error('Configuration file already exists');
|
package/dist/cjs/cli.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"cli.js","sourceRoot":"","sources":["../../src/cli.ts"],"names":[],"mappings":";;;AAEA,4CAO2B;AAC3B,sDAA0D;AAC1D,0DAAwD;AACxD,sEAAyE;AACzE,gEAA+D;AAC/D,kEAA8D;AAC9D,wDAAqD;AACrD,+CAAwC;AACxC,iEAAmE;AAEnE,
|
|
1
|
+
{"version":3,"file":"cli.js","sourceRoot":"","sources":["../../src/cli.ts"],"names":[],"mappings":";;;AAEA,4CAO2B;AAC3B,sDAA0D;AAC1D,0DAAwD;AACxD,sEAAyE;AACzE,gEAA+D;AAC/D,kEAA8D;AAC9D,wDAAqD;AACrD,+CAAwC;AACxC,iEAAmE;AAEnE,4CAAwF;AAExF;;;;GAIG;AACH,IAAI,OAAO,CAAC,GAAG,CAAC,kBAAkB,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,UAAU,EAAE,CAAC;IAC9D,OAAO,CAAC,GAAG,CAAC,UAAU,GAAG,OAAO,CAAC,GAAG,CAAC,kBAAkB,CAAA;AACzD,CAAC;AAED,MAAM,IAAI,GAAG,KAAK,IAAI,EAAE;IACtB,MAAM,GAAG,GAAG,MAAM,IAAA,uBAAiB,EACjC,aAAa,EACb;QACE,IAAI,EAAE;YACJ,WAAW,EAAE,+CAA+C;YAC5D,SAAS,EAAE,EAAE;SACd;QACD,QAAQ,EAAE;YACR,WAAW,EAAE,sCAAsC;YACnD,SAAS,EAAE;gBACT,WAAW,EAAE,IAAA,yBAAmB,EAAC;oBAC/B,WAAW,EAAE,gCAAgC;oBAC7C,YAAY,EAAE,EAAE;iBACjB,CAAC;gBACF,QAAQ,EAAE,IAAA,yBAAmB,EAAC;oBAC5B,WAAW,EAAE,kCAAkC;oBAC/C,YAAY,EAAE,EAAE;iBACjB,CAAC;gBACF,OAAO,EAAE,IAAA,yBAAmB,EAAC;oBAC3B,WAAW,EAAE,8BAA8B;oBAC3C,YAAY,EAAE,EAAE;iBACjB,CAAC;gBACF,QAAQ,EAAE,IAAA,yBAAmB,EAAC;oBAC5B,WAAW,EAAE,0BAA0B;oBACvC,YAAY,EAAE,EAAE;iBACjB,CAAC;gBACF,WAAW,EAAE,IAAA,oBAAc,EAAC;oBAC1B,WAAW,EACT,yFAAyF;iBAC5F,CAAC;gBACF,OAAO,EAAE,IAAA,qBAAe,EAAC;oBACvB,WAAW,EAAE,+CAA+C;oBAC5D,SAAS,EAAE,GAAG;iBACf,CAAC;gBACF,SAAS,EAAE,IAAA,qBAAe,EAAC;oBACzB,WAAW,EACT,sIAAsI;oBACxI,SAAS,EAAE,GAAG;iBACf,CAAC;aACH;SACF;QACD,KAAK,EAAE;YACL,WAAW,EAAE,8BAA8B;YAC3C,SAAS,EAAE;gBACT,WAAW,EAAE,IAAA,yBAAmB,EAAC;oBAC/B,WAAW,EAAE,gCAAgC;oBAC7C,YAAY,EAAE,EAAE;iBACjB,CAAC;gBACF,SAAS,EAAE,IAAA,oBAAc,EAAC;oBACxB,WAAW,EAAE,0DAA0D;oBACvE,YAAY,EAAE,KAAK;iBACpB,CAAC;gBACF,QAAQ,EAAE,IAAA,yBAAmB,EAAC;oBAC5B,WAAW,EAAE,2CAA2C;oBACxD,YAAY,EAAE,EAAE;iBACjB,CAAC;gBACF,OAAO,EAAE,IAAA,yBAAmB,EAAC;oBAC3B,WAAW,EAAE,uCAAuC;oBACpD,YAAY,EAAE,EAAE;iBACjB,CAAC;gBACF,QAAQ,EAAE,IAAA,yBAAmB,EAAC;oBAC5B,WAAW,EAAE,wCAAwC;oBACrD,YAAY,EAAE,EAAE;iBACjB,CAAC;gBACF,WAAW,EAAE,IAAA,oBAAc,EAAC;oBAC1B,WAAW,EACT,sFAAsF;iBACzF,CAAC;aACH;SACF;QACD,iBAAiB,EAAE;YACjB,WAAW,EAAE,sDAAsD;YACnE,SAAS,EAAE;gBACT,cAAc,EAAE,IAAA,oBAAc,EAAC;oBAC7B,WAAW,EACT,wIAAwI;iBAC3I,CAAC;gBACF,eAAe,EAAE,IAAA,yBAAmB,EAAC;oBACnC,WAAW,EAAE,oCAAoC;iBAClD,CAAC;aACH;SACF;QACD,cAAc,EAAE;YACd,WAAW,EAAE,kEAAkE;YAC/E,SAAS,EAAE;gBACT,OAAO,EAAE,IAAA,oBAAc,EAAC;oBACtB,WAAW,EAAE,qCAAqC;iBACnD,CAAC;aACH;SACF;KACF,EACD;QACE,GAAG,EAAE,IAAA,kBAAY,EAAC;YAChB,WAAW,EAAE,sBAAsB;YACnC,WAAW,EAAE,CAAC,GAAG,eAAS,CAAC,CAAC,oCAAoC;SACjE,CAAC;KACH,EACD;QACE,SAAS,EAAE,aAAa;QACxB,gBAAgB,EAAE,IAAI;QACtB,iBAAiB,EAAE,IAAI;QACvB,cAAc,EAAE,KAAK;QACrB,OAAO,EAAE;YACP,cAAc,EAAE,qCAAiB;YACjC,eAAe,EAAE,4BAA4B;SAC9C;KACF,CACF,CAAA;IAED,IAAA,eAAS,EACP,IAAI,oBAAc,CAAC;QACjB,QAAQ,EAAG,GAAG,CAAC,IAAI,CAAC,GAAgB,IAAI,SAAS;KAClD,CAAC,CACH,CAAA;IAED,IAAI,GAAG,CAAC,UAAU,KAAK,MAAM,EAAE,CAAC;QAC9B,IAAI,IAAA,sCAAmB,GAAE,EAAE,CAAC;YAC1B,OAAO,CAAC,KAAK,CAAC,mCAAmC,CAAC,CAAA;YAClD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QACjB,CAAC;QACD,OAAO,CAAC,GAAG,CAAC,iBAAiB,CAAC,CAAA;QAC9B,MAAM,IAAA,gDAA0B,GAAE,CAAA;IACpC,CAAC;SAAM,IAAI,GAAG,CAAC,UAAU,KAAK,UAAU,EAAE,CAAC;QACzC,MAAM,aAAa,GAAG,qBAAqB,CAAA;QAC3C,MAAM,WAAW,GAAG,GAAG,CAAC,IAAI,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC,CAAA;QAC5F,MAAM,OAAO,GAAG,IAAA,+BAAe,EAAC,WAAW,CAAC,CAAA;QAC5C,MAAM,IAAA,0BAAY,EAChB,OAAO,EACP,GAAG,CAAC,IAAI,CAAC,QAAQ,EACjB,GAAG,CAAC,IAAI,CAAC,OAAO,EAChB,GAAG,CAAC,IAAI,CAAC,QAAQ,EACjB,GAAG,CAAC,IAAI,CAAC,WAAW,EACpB,GAAG,CAAC,IAAI,CAAC,OAAO,EAChB,GAAG,CAAC,IAAI,CAAC,SAAS,CACnB,CAAA;IACH,CAAC;SAAM,IAAI,GAAG,CAAC,UAAU,KAAK,OAAO,EAAE,CAAC;QACtC,MAAM,aAAa,GAAG,qBAAqB,CAAA;QAC3C,MAAM,WAAW,GAAG,GAAG,CAAC,IAAI,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC,CAAA;QAC5F,MAAM,OAAO,GAAG,IAAA,+BAAe,EAAC,WAAW,CAAC,CAAA;QAC5C,MAAM,IAAA,gBAAK,EACT,OAAO,EACP,GAAG,CAAC,IAAI,CAAC,SAAS,IAAI,KAAK,EAC3B,GAAG,CAAC,IAAI,CAAC,QAAQ,EACjB,GAAG,CAAC,IAAI,CAAC,OAAO,EAChB,GAAG,CAAC,IAAI,CAAC,QAAwB,EACjC,GAAG,CAAC,IAAI,CAAC,WAAW,CACrB,CAAA;IACH,CAAC;SAAM,IAAI,GAAG,CAAC,UAAU,KAAK,iBAAiB,EAAE,CAAC;QAChD,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,cAAc,EAAE,CAAC;YAC7B,OAAO,CAAC,KAAK,CAAC,4DAA4D,CAAC,CAAA;YAC3E,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QACjB,CAAC;QACD,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,eAAe,EAAE,CAAC;YAC9B,OAAO,CAAC,KAAK,CAAC,wEAAwE,CAAC,CAAA;YACvF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QACjB,CAAC;QACD,MAAM,IAAA,qCAAoB,EAAC,GAAG,CAAC,IAAI,CAAC,cAAc,EAAE,GAAG,CAAC,IAAI,CAAC,eAAe,CAAC,CAAA;IAC/E,CAAC;SAAM,IAAI,GAAG,CAAC,UAAU,KAAK,cAAc,EAAE,CAAC;QAC7C,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC;YACtB,OAAO,CAAC,KAAK,CAAC,yDAAyD,CAAC,CAAA;YACxE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QACjB,CAAC;QACD,MAAM,WAAW,GAAG,MAAM,IAAA,+BAAkB,EAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;QAC9D,IAAI,CAAC,WAAW,EAAE,CAAC;YACjB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QACjB,CAAC;IACH,CAAC;AACH,CAAC,CAAA;AAED,IAAI,EAAE;KACH,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE;IACX,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAA;IAChB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;AACjB,CAAC,CAAC;KACD,IAAI,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC;KACd,OAAO,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAA"}
|
|
@@ -15,7 +15,7 @@ const util_js_2 = require("../persistence/util.js");
|
|
|
15
15
|
const regions_js_1 = require("../regions.js");
|
|
16
16
|
const services_js_1 = require("../services.js");
|
|
17
17
|
const syncMap_js_1 = require("../syncs/syncMap.js");
|
|
18
|
-
const
|
|
18
|
+
const log_1 = require("@cloud-copilot/log");
|
|
19
19
|
/**
|
|
20
20
|
* Download data from AWS services.
|
|
21
21
|
*
|
|
@@ -61,17 +61,17 @@ async function downloadData(configs, accountIds, regions, services, concurrency,
|
|
|
61
61
|
}
|
|
62
62
|
const storage = (0, util_js_2.createStorageClient)(storageConfig, defaultPartition, deleteData);
|
|
63
63
|
const indexJobs = [];
|
|
64
|
-
|
|
64
|
+
log_1.log.debug('Starting download runner', { concurrency });
|
|
65
65
|
const downloadRunner = new jobRunner_js_1.JobRunner(concurrency);
|
|
66
|
-
const workerPool = new job_1.ConcurrentWorkerPool(concurrency,
|
|
66
|
+
const workerPool = new job_1.ConcurrentWorkerPool(concurrency, log_1.log);
|
|
67
67
|
for (const accountId of accountIds) {
|
|
68
|
-
|
|
68
|
+
log_1.log.info('Queuing downloads for account', { accountId });
|
|
69
69
|
const authForAccount = (0, config_js_1.getAccountAuthConfig)(accountId, configs);
|
|
70
70
|
const credentials = await getCredentialsForSync(clientPool, defaultPartition, accountId, authForAccount);
|
|
71
71
|
const accountPartition = credentials.partition;
|
|
72
72
|
if (accountPartition !== defaultPartition) {
|
|
73
73
|
//TODO: Consider updating the storage client to handle multiple partitions
|
|
74
|
-
|
|
74
|
+
log_1.log.error(`Account ${accountId} is in partition ${accountPartition}, but the default account is in partition ${defaultPartition}. This is not supported.`);
|
|
75
75
|
throw new Error('Cannot download data for multiple partitions in one run.');
|
|
76
76
|
}
|
|
77
77
|
const partitionConfig = (0, partitionDefaults_js_1.getPartitionDefaults)(accountPartition);
|
|
@@ -87,7 +87,7 @@ async function downloadData(configs, accountIds, regions, services, concurrency,
|
|
|
87
87
|
};
|
|
88
88
|
const enabledServices = (0, config_js_1.servicesForAccount)(accountId, accountConfigs, services);
|
|
89
89
|
for (const service of enabledServices) {
|
|
90
|
-
|
|
90
|
+
log_1.log.info('Queuing downloads', { service, accountId });
|
|
91
91
|
const serviceRegions = (0, config_js_1.regionsForService)(service, accountId, accountConfigs, accountRegions);
|
|
92
92
|
//Global syncs for the service
|
|
93
93
|
const globalSyncs = (0, syncMap_js_1.getGlobalSyncsForService)(service);
|
|
@@ -96,7 +96,7 @@ async function downloadData(configs, accountIds, regions, services, concurrency,
|
|
|
96
96
|
for (const globalSync of globalSyncs) {
|
|
97
97
|
const customConfig = (0, config_js_1.customConfigForSync)(service, globalSync.name, accountId, globalRegion, accountConfigs);
|
|
98
98
|
if (!clientPool.isSyncSupported(service, globalSync.name, globalRegion)) {
|
|
99
|
-
|
|
99
|
+
log_1.log.info({
|
|
100
100
|
skippedSync: true,
|
|
101
101
|
service,
|
|
102
102
|
accountId,
|
|
@@ -113,9 +113,9 @@ async function downloadData(configs, accountIds, regions, services, concurrency,
|
|
|
113
113
|
...context.properties
|
|
114
114
|
};
|
|
115
115
|
const globalCredentials = await getCredentialsForSync(clientPool, defaultPartition, accountId, globalConfig.auth);
|
|
116
|
-
|
|
116
|
+
log_1.log.debug(logDetails, 'Executing global sync');
|
|
117
117
|
await globalSync.execute(accountId, globalRegion, globalCredentials, storage, globalConfig.endpoint, { ...syncOptions, customConfig });
|
|
118
|
-
|
|
118
|
+
log_1.log.trace(logDetails, 'Finished global sync');
|
|
119
119
|
}
|
|
120
120
|
});
|
|
121
121
|
}
|
|
@@ -125,16 +125,16 @@ async function downloadData(configs, accountIds, regions, services, concurrency,
|
|
|
125
125
|
if (regionalSyncs.length === 0) {
|
|
126
126
|
continue;
|
|
127
127
|
}
|
|
128
|
-
|
|
128
|
+
log_1.log.debug({ service, accountId, region }, 'Queuing regional syncs');
|
|
129
129
|
const asrConfig = (0, config_js_1.accountServiceRegionConfig)(service, accountId, region, accountConfigs);
|
|
130
130
|
for (const sync of regionalSyncs) {
|
|
131
131
|
const includeSync = (0, config_js_1.syncEnabledForRegion)(accountId, service, sync.name, accountConfigs, region);
|
|
132
132
|
if (!includeSync) {
|
|
133
|
-
|
|
133
|
+
log_1.log.debug({ service, accountId, region, syncName: sync.name }, 'Skipping regional sync');
|
|
134
134
|
continue;
|
|
135
135
|
}
|
|
136
136
|
if (!clientPool.isSyncSupported(service, sync.name, region)) {
|
|
137
|
-
|
|
137
|
+
log_1.log.info({
|
|
138
138
|
skippedSync: true,
|
|
139
139
|
service,
|
|
140
140
|
accountId,
|
|
@@ -152,10 +152,10 @@ async function downloadData(configs, accountIds, regions, services, concurrency,
|
|
|
152
152
|
workerId: context.workerId,
|
|
153
153
|
...context.properties
|
|
154
154
|
};
|
|
155
|
-
|
|
155
|
+
log_1.log.debug(logDetails, 'Executing regional sync');
|
|
156
156
|
const regionalCredentials = await getCredentialsForSync(clientPool, accountPartition, accountId, asrConfig.auth);
|
|
157
157
|
await sync.execute(accountId, region, regionalCredentials, storage, asrConfig.endpoint, { ...syncOptions, customConfig });
|
|
158
|
-
|
|
158
|
+
log_1.log.trace(logDetails, 'Finished regional sync');
|
|
159
159
|
}
|
|
160
160
|
});
|
|
161
161
|
}
|
|
@@ -171,19 +171,19 @@ async function downloadData(configs, accountIds, regions, services, concurrency,
|
|
|
171
171
|
}
|
|
172
172
|
}
|
|
173
173
|
}
|
|
174
|
-
|
|
174
|
+
log_1.log.info('Waiting for downloads to complete');
|
|
175
175
|
await downloadRunner.finishAllWork();
|
|
176
|
-
|
|
176
|
+
log_1.log.info('Finished downloads', { jobs: downloadRunner.getResults().length });
|
|
177
177
|
const failedJobs = downloadRunner.getResults().filter((r) => r.status === 'rejected');
|
|
178
178
|
if (failedJobs.length > 0) {
|
|
179
|
-
|
|
179
|
+
log_1.log.error('Some downloads failed', { failedJobs: failedJobs.length });
|
|
180
180
|
for (const failedJob of failedJobs) {
|
|
181
|
-
|
|
181
|
+
log_1.log.error('Download failed', failedJob.reason, failedJob.properties);
|
|
182
182
|
}
|
|
183
183
|
throw new Error(`Failed to download some data. See logs for details.`);
|
|
184
184
|
}
|
|
185
185
|
if (skipIndex) {
|
|
186
|
-
|
|
186
|
+
log_1.log.info('Skipping indexing');
|
|
187
187
|
return;
|
|
188
188
|
}
|
|
189
189
|
await (0, runIndexers_js_1.runIndexJobs)(indexJobs, storageConfig, concurrency);
|
|
@@ -203,11 +203,11 @@ async function getAccountRegions(allRegions, accountId, configs, accountCredenti
|
|
|
203
203
|
}
|
|
204
204
|
const configuredRegions = (0, config_js_1.configuredRegionListForAccount)(configs, accountId);
|
|
205
205
|
if (configuredRegions) {
|
|
206
|
-
|
|
206
|
+
log_1.log.debug('Using configured regions', { regions: configuredRegions, accountId });
|
|
207
207
|
return configuredRegions;
|
|
208
208
|
}
|
|
209
209
|
else {
|
|
210
|
-
|
|
210
|
+
log_1.log.debug('No configured regions found, discovering regions for account', {
|
|
211
211
|
accountId
|
|
212
212
|
});
|
|
213
213
|
return (0, regions_js_1.getEnabledRegions)(accountCredentials, clientPool);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"download.js","sourceRoot":"","sources":["../../../src/download/download.ts"],"names":[],"mappings":";;AA6CA,oCA8PC;AA3SD,4CAAyD;AACzD,4CAA+C;AAE/C,oDAG2B;AAC3B,mDAc4B;AAC5B,2DAA0D;AAC1D,yEAAqE;AACrE,yDAA+D;AAC/D,+DAAwE;AACxE,uDAAgD;AAChD,6CAAoD;AACpD,oDAA4D;AAC5D,8CAAiD;AACjD,gDAA4C;AAC5C,oDAA0F;AAC1F,
|
|
1
|
+
{"version":3,"file":"download.js","sourceRoot":"","sources":["../../../src/download/download.ts"],"names":[],"mappings":";;AA6CA,oCA8PC;AA3SD,4CAAyD;AACzD,4CAA+C;AAE/C,oDAG2B;AAC3B,mDAc4B;AAC5B,2DAA0D;AAC1D,yEAAqE;AACrE,yDAA+D;AAC/D,+DAAwE;AACxE,uDAAgD;AAChD,6CAAoD;AACpD,oDAA4D;AAC5D,8CAAiD;AACjD,gDAA4C;AAC5C,oDAA0F;AAC1F,4CAAwC;AAExC;;;;;;;;;;GAUG;AACI,KAAK,UAAU,YAAY,CAChC,OAAyB,EACzB,UAAoB,EACpB,OAAiB,EACjB,QAAkB,EAClB,WAA+B,EAC/B,SAAkB,EAClB,SAAkB;IAElB,IAAI,WAAW,KAAK,SAAS,IAAI,WAAW,IAAI,CAAC,EAAE,CAAC;QAClD,WAAW,GAAG,IAAA,4BAAkB,GAAE,CAAA;IACpC,CAAC;IACD,MAAM,UAAU,GAAG,CAAC,SAAS,CAAA;IAE7B,MAAM,iBAAiB,GAAG,IAAA,gCAAoB,EAAC,OAAO,CAAC,CAAA;IACvD,MAAM,kBAAkB,GAAG,MAAM,IAAA,sCAAwB,EAAC,iBAAiB,EAAE;QAC3E,KAAK,EAAE,6BAA6B;KACrC,CAAC,CAAA;IAEF,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC5B;;;;WAIG;QACH,MAAM,kBAAkB,GAAG,IAAA,iCAAqB,EAAC,OAAO,CAAC,CAAA;QACzD,IAAI,kBAAkB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAClC,UAAU,GAAG,kBAAkB,CAAA;QACjC,CAAC;aAAM,CAAC;YACN,UAAU,GAAG,CAAC,kBAAkB,CAAC,SAAS,CAAC,CAAA;QAC7C,CAAC;IACH,CAAC;IAED,2CAA2C;IAC3C,MAAM,gBAAgB,GAAG,IAAA,mCAAuB,EAAC,OAAO,CAAC,CAAA;IACzD,MAAM,UAAU,GAAG,MAAM,IAAA,gCAAgB,EAAC,gBAAgB,CAAC,CAAA;IAC3D,MAAM,UAAU,CAAC,IAAI,EAAE,CAAA;IAEvB,MAAM,gBAAgB,GAAG,kBAAkB,CAAC,SAAS,CAAA;IAErD,MAAM,aAAa,GAAG,IAAA,4BAAgB,EAAC,OAAO,CAAC,CAAA;IAC/C,IAAI,CAAC,aAAa,EAAE,CAAC;QACnB,MAAM,IAAI,KAAK,CAAC,uDAAuD,CAAC,CAAA;IAC1E,CAAC;IAED,MAAM,OAAO,GAAG,IAAA,6BAAmB,EAAC,aAAa,EAAE,gBAAgB,EAAE,UAAU,CAAC,CAAA;IAEhF,MAAM,SAAS,GAAe,EAAE,CAAA;IAEhC,SAAG,CAAC,KAAK,CAAC,0BAA0B,EAAE,EAAE,WAAW,EAAE,CAAC,CAAA;IACtD,MAAM,cAAc,GAAG,IAAI,wBAAS,CAAC,WAAW,CAAC,CAAA;IACjD,MAAM,UAAU,GAAG,IAAI,0BAAoB,CAAC,WAAW,EAAE,SAAG,CAAC,CAAA;IAE7D,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE,CAAC;QACnC,SAAG,CAAC,IAAI,CAAC,+BAA+B,EAAE,EAAE,SAAS,EAAE,CAAC,CAAA;QACxD,MAAM,cAAc,GAAG,IAAA,gCAAoB,EAAC,SAAS,EAAE,OAAO,CAAC,CAAA;QAC/D,MAAM,WAAW,GAAG,MAAM,qBAAqB,CAC7C,UAAU,EACV,gBAAgB,EAChB,SAAS,EACT,cAAc,CACf,CAAA;QACD,MAAM,gBAAgB,GAAG,WAAW,CAAC,SAAS,CAAA;QAC9C,IAAI,gBAAgB,KAAK,gBAAgB,EAAE,CAAC;YAC1C,0EAA0E;YAC1E,SAAG,CAAC,KAAK,CACP,WAAW,SAAS,oBAAoB,gBAAgB,6CAA6C,gBAAgB,0BAA0B,CAChJ,CAAA;YACD,MAAM,IAAI,KAAK,CAAC,0DAA0D,CAAC,CAAA;QAC7E,CAAC;QACD,MAAM,eAAe,GAAG,IAAA,2CAAoB,EAAC,gBAAgB,CAAC,CAAA;QAC9D,MAAM,cAAc,GAAG,CAAC,eAAe,EAAE,GAAG,OAAO,CAAC,CAAA;QACpD,MAAM,cAAc,GAAG,MAAM,iBAAiB,CAC5C,OAAO,EACP,SAAS,EACT,OAAO,EACP,WAAW,EACX,UAAU,CACX,CAAA;QAED,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC1B,QAAQ,GAAG,yBAAkC,CAAA;QAC/C,CAAC;QACD,MAAM,WAAW,GAAG;YAClB,UAAU;YACV,SAAS;YACT,UAAU;SACX,CAAA;QAED,MAAM,eAAe,GAAG,IAAA,8BAAkB,EAAC,SAAS,EAAE,cAAc,EAAE,QAAQ,CAAC,CAAA;QAC/E,KAAK,MAAM,OAAO,IAAI,eAAe,EAAE,CAAC;YACtC,SAAG,CAAC,IAAI,CAAC,mBAAmB,EAAE,EAAE,OAAO,EAAE,SAAS,EAAE,CAAC,CAAA;YACrD,MAAM,cAAc,GAAG,IAAA,6BAAiB,EAAC,OAAO,EAAE,SAAS,EAAE,cAAc,EAAE,cAAc,CAAC,CAAA;YAE5F,8BAA8B;YAC9B,MAAM,WAAW,GAAG,IAAA,qCAAwB,EAAC,OAAO,CAAC,CAAA;YACrD,MAAM,YAAY,GAAG,cAAc,CAAC,EAAE,CAAC,CAAC,CAAE,CAAA;YAC1C,MAAM,YAAY,GAAG,IAAA,sCAA0B,EAC7C,OAAO,EACP,SAAS,EACT,YAAY,EACZ,cAAc,CACf,CAAA;YAED,KAAK,MAAM,UAAU,IAAI,WAAW,EAAE,CAAC;gBACrC,MAAM,YAAY,GAAG,IAAA,+BAAmB,EACtC,OAAO,EACP,UAAU,CAAC,IAAI,EACf,SAAS,EACT,YAAY,EACZ,cAAc,CACf,CAAA;gBAED,IAAI,CAAC,UAAU,CAAC,eAAe,CAAC,OAAO,EAAE,UAAU,CAAC,IAAI,EAAE,YAAY,CAAC,EAAE,CAAC;oBACxE,SAAG,CAAC,IAAI,CACN;wBACE,WAAW,EAAE,IAAI;wBACjB,OAAO;wBACP,SAAS;wBACT,IAAI,EAAE,UAAU,CAAC,IAAI;wBACrB,UAAU,EAAE,UAAU,CAAC,WAAW,CAAC,IAAI;qBACxC,EACD,oDAAoD,CACrD,CAAA;oBACD,SAAQ;gBACV,CAAC;gBACD,cAAc,CAAC,OAAO,CAAC;oBACrB,UAAU,EAAE,EAAE,OAAO,EAAE,SAAS,EAAE,IAAI,EAAE,UAAU,CAAC,IAAI,EAAE;oBACzD,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE;wBACzB,MAAM,UAAU,GAAG;4BACjB,QAAQ,EAAE,OAAO,CAAC,QAAQ;4BAC1B,GAAG,OAAO,CAAC,UAAU;yBACtB,CAAA;wBACD,MAAM,iBAAiB,GAAG,MAAM,qBAAqB,CACnD,UAAU,EACV,gBAAgB,EAChB,SAAS,EACT,YAAY,CAAC,IAAI,CAClB,CAAA;wBACD,SAAG,CAAC,KAAK,CAAC,UAAU,EAAE,uBAAuB,CAAC,CAAA;wBAC9C,MAAM,UAAU,CAAC,OAAO,CACtB,SAAS,EACT,YAAY,EACZ,iBAAiB,EACjB,OAAO,EACP,YAAY,CAAC,QAAQ,EACrB,EAAE,GAAG,WAAW,EAAE,YAAY,EAAE,CACjC,CAAA;wBACD,SAAG,CAAC,KAAK,CAAC,UAAU,EAAE,sBAAsB,CAAC,CAAA;oBAC/C,CAAC;iBACF,CAAC,CAAA;YACJ,CAAC;YAED,MAAM,aAAa,GAAG,IAAA,uCAA0B,EAAC,OAAO,CAAC,CAAA;YACzD,gCAAgC;YAChC,KAAK,MAAM,MAAM,IAAI,cAAc,EAAE,CAAC;gBACpC,IAAI,aAAa,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;oBAC/B,SAAQ;gBACV,CAAC;gBACD,SAAG,CAAC,KAAK,CAAC,EAAE,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,EAAE,wBAAwB,CAAC,CAAA;gBACnE,MAAM,SAAS,GAAG,IAAA,sCAA0B,EAAC,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,cAAc,CAAC,CAAA;gBAExF,KAAK,MAAM,IAAI,IAAI,aAAa,EAAE,CAAC;oBACjC,MAAM,WAAW,GAAG,IAAA,gCAAoB,EACtC,SAAS,EACT,OAAO,EACP,IAAI,CAAC,IAAI,EACT,cAAc,EACd,MAAM,CACP,CAAA;oBACD,IAAI,CAAC,WAAW,EAAE,CAAC;wBACjB,SAAG,CAAC,KAAK,CAAC,EAAE,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,QAAQ,EAAE,IAAI,CAAC,IAAI,EAAE,EAAE,wBAAwB,CAAC,CAAA;wBACxF,SAAQ;oBACV,CAAC;oBACD,IAAI,CAAC,UAAU,CAAC,eAAe,CAAC,OAAO,EAAE,IAAI,CAAC,IAAI,EAAE,MAAM,CAAC,EAAE,CAAC;wBAC5D,SAAG,CAAC,IAAI,CACN;4BACE,WAAW,EAAE,IAAI;4BACjB,OAAO;4BACP,SAAS;4BACT,IAAI,EAAE,IAAI,CAAC,IAAI;4BACf,MAAM;4BACN,UAAU,EAAE,UAAU,CAAC,WAAW,CAAC,IAAI;yBACxC,EACD,sDAAsD,CACvD,CAAA;wBACD,SAAQ;oBACV,CAAC;oBACD,MAAM,YAAY,GAAG,IAAA,+BAAmB,EACtC,OAAO,EACP,IAAI,CAAC,IAAI,EACT,SAAS,EACT,MAAM,EACN,cAAc,CACf,CAAA;oBACD,cAAc,CAAC,OAAO,CAAC;wBACrB,UAAU,EAAE,EAAE,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE;wBAC3D,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE;4BACzB,MAAM,UAAU,GAAG;gCACjB,QAAQ,EAAE,OAAO,CAAC,QAAQ;gCAC1B,GAAG,OAAO,CAAC,UAAU;6BACtB,CAAA;4BACD,SAAG,CAAC,KAAK,CAAC,UAAU,EAAE,yBAAyB,CAAC,CAAA;4BAChD,MAAM,mBAAmB,GAAG,MAAM,qBAAqB,CACrD,UAAU,EACV,gBAAgB,EAChB,SAAS,EACT,SAAS,CAAC,IAAI,CACf,CAAA;4BAED,MAAM,IAAI,CAAC,OAAO,CAChB,SAAS,EACT,MAAM,EACN,mBAAmB,EACnB,OAAO,EACP,SAAS,CAAC,QAAQ,EAClB,EAAE,GAAG,WAAW,EAAE,YAAY,EAAE,CACjC,CAAA;4BACD,SAAG,CAAC,KAAK,CAAC,UAAU,EAAE,wBAAwB,CAAC,CAAA;wBACjD,CAAC;qBACF,CAAC,CAAA;gBACJ,CAAC;YACH,CAAC;YAED,MAAM,QAAQ,GAAG,IAAA,mCAAqB,EAAC,OAAO,CAAC,CAAA;YAC/C,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;gBAC/B,SAAS,CAAC,IAAI,CAAC;oBACb,OAAO;oBACP,SAAS,EAAE,gBAAgB;oBAC3B,SAAS;oBACT,OAAO,EAAE,cAAc;iBACxB,CAAC,CAAA;YACJ,CAAC;QACH,CAAC;IACH,CAAC;IAED,SAAG,CAAC,IAAI,CAAC,mCAAmC,CAAC,CAAA;IAC7C,MAAM,cAAc,CAAC,aAAa,EAAE,CAAA;IACpC,SAAG,CAAC,IAAI,CAAC,oBAAoB,EAAE,EAAE,IAAI,EAAE,cAAc,CAAC,UAAU,EAAE,CAAC,MAAM,EAAE,CAAC,CAAA;IAC5E,MAAM,UAAU,GAAG,cAAc,CAAC,UAAU,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,UAAU,CAAC,CAAA;IACrF,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC1B,SAAG,CAAC,KAAK,CAAC,uBAAuB,EAAE,EAAE,UAAU,EAAE,UAAU,CAAC,MAAM,EAAE,CAAC,CAAA;QACrE,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE,CAAC;YACnC,SAAG,CAAC,KAAK,CAAC,iBAAiB,EAAE,SAAS,CAAC,MAAM,EAAE,SAAS,CAAC,UAAU,CAAC,CAAA;QACtE,CAAC;QACD,MAAM,IAAI,KAAK,CAAC,qDAAqD,CAAC,CAAA;IACxE,CAAC;IAED,IAAI,SAAS,EAAE,CAAC;QACd,SAAG,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAA;QAC7B,OAAM;IACR,CAAC;IAED,MAAM,IAAA,6BAAY,EAAC,SAAS,EAAE,aAAa,EAAE,WAAW,CAAC,CAAA;AAC3D,CAAC;AAED;;;;;;;;GAQG;AACH,KAAK,UAAU,iBAAiB,CAC9B,UAAoB,EACpB,SAAiB,EACjB,OAAyB,EACzB,kBAAqD,EACrD,UAAyB;IAEzB,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC1B,OAAO,UAAU,CAAA;IACnB,CAAC;IAED,MAAM,iBAAiB,GAAG,IAAA,0CAA8B,EAAC,OAAO,EAAE,SAAS,CAAC,CAAA;IAC5E,IAAI,iBAAiB,EAAE,CAAC;QACtB,SAAG,CAAC,KAAK,CAAC,0BAA0B,EAAE,EAAE,OAAO,EAAE,iBAAiB,EAAE,SAAS,EAAE,CAAC,CAAA;QAChF,OAAO,iBAAiB,CAAA;IAC1B,CAAC;SAAM,CAAC;QACN,SAAG,CAAC,KAAK,CAAC,8DAA8D,EAAE;YACxE,SAAS;SACV,CAAC,CAAA;QACF,OAAO,IAAA,8BAAiB,EAAC,kBAAkB,EAAE,UAAU,CAAC,CAAA;IAC1D,CAAC;AACH,CAAC;AAED,KAAK,UAAU,qBAAqB,CAClC,UAAyB,EACzB,gBAAwB,EACxB,SAAiB,EACjB,UAAkC;IAElC,IAAI,UAAU,CAAC,sBAAsB,EAAE,EAAE,CAAC;QACxC,OAAO,IAAA,wBAAc,EAAC,SAAS,EAAE,UAAU,CAAC,CAAA;IAC9C,CAAC;IAED,OAAO;QACL,SAAS,EAAE,SAAS;QACpB,SAAS,EAAE,gBAAgB;QAC3B,QAAQ,EAAE,gBAAgB;QAC1B,QAAQ,EAAE,KAAK,IAAI,EAAE;YACnB,MAAM,IAAI,KAAK,CAAC,8CAA8C,CAAC,CAAA;QACjE,CAAC;KACF,CAAA;AACH,CAAC"}
|
|
@@ -3,7 +3,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
|
3
3
|
exports.runIndexJobs = runIndexJobs;
|
|
4
4
|
const jobQueue_js_1 = require("../jobs/jobQueue.js");
|
|
5
5
|
const util_js_1 = require("../persistence/util.js");
|
|
6
|
-
const
|
|
6
|
+
const log_1 = require("@cloud-copilot/log");
|
|
7
7
|
/**
|
|
8
8
|
* Run index jobs in parallel with a specified concurrency.
|
|
9
9
|
*
|
|
@@ -38,24 +38,24 @@ async function runIndexJobs(indexJobs, storageConfig, concurrency) {
|
|
|
38
38
|
numIndexers: indexerJobs.length
|
|
39
39
|
},
|
|
40
40
|
execute: async (context) => {
|
|
41
|
-
|
|
41
|
+
log_1.log.debug('Running indexers', { workerId: context.workerId, ...context.properties });
|
|
42
42
|
await runIndexers(partition, storageConfig, indexerJobs);
|
|
43
|
-
|
|
43
|
+
log_1.log.trace('Finished indexers', { workerId: context.workerId, ...context.properties });
|
|
44
44
|
}
|
|
45
45
|
});
|
|
46
46
|
}
|
|
47
47
|
}
|
|
48
|
-
|
|
48
|
+
log_1.log.debug('Starting indexing', { jobs: jobs.length, concurrency });
|
|
49
49
|
const indexResults = await (0, jobQueue_js_1.runJobs)(jobs, concurrency);
|
|
50
50
|
const failedIndexes = indexResults.filter((r) => r.status === 'rejected');
|
|
51
51
|
if (failedIndexes.length > 0) {
|
|
52
|
-
|
|
52
|
+
log_1.log.error('Some indexers failed', { failedJobs: failedIndexes.length });
|
|
53
53
|
for (const failedJob of failedIndexes) {
|
|
54
|
-
|
|
54
|
+
log_1.log.error('Indexer failed', failedJob.reason, failedJob.properties);
|
|
55
55
|
}
|
|
56
56
|
throw new Error(`Failed to index some data. See logs for details.`);
|
|
57
57
|
}
|
|
58
|
-
|
|
58
|
+
log_1.log.info('Finished indexing', { jobs: jobs.length });
|
|
59
59
|
return indexResults;
|
|
60
60
|
}
|
|
61
61
|
/**
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"runIndexers.js","sourceRoot":"","sources":["../../../src/indexing/runIndexers.ts"],"names":[],"mappings":";;AAsCA,oCAqDC;AA1FD,qDAAuE;AACvE,oDAA4D;AAC5D,
|
|
1
|
+
{"version":3,"file":"runIndexers.js","sourceRoot":"","sources":["../../../src/indexing/runIndexers.ts"],"names":[],"mappings":";;AAsCA,oCAqDC;AA1FD,qDAAuE;AACvE,oDAA4D;AAC5D,4CAAwC;AA4BxC;;;;;;GAMG;AACI,KAAK,UAAU,YAAY,CAChC,SAAqB,EACrB,aAA4B,EAC5B,WAAmB;IAEnB,MAAM,UAAU,GAAyC,IAAI,GAAG,EAAE,CAAA;IAClE,KAAK,MAAM,GAAG,IAAI,SAAS,EAAE,CAAC;QAC5B,MAAM,SAAS,GAAG,GAAG,CAAC,SAAS,CAAA;QAC/B,MAAM,WAAW,GAAG,GAAG,CAAC,OAAO,CAAC,IAAI,CAAA;QACpC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC;YAC/B,UAAU,CAAC,GAAG,CAAC,SAAS,EAAE,IAAI,GAAG,EAAE,CAAC,CAAA;QACtC,CAAC;QACD,MAAM,aAAa,GAAG,UAAU,CAAC,GAAG,CAAC,SAAS,CAAE,CAAA;QAChD,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,WAAW,CAAC,EAAE,CAAC;YACpC,aAAa,CAAC,GAAG,CAAC,WAAW,EAAE,EAAE,CAAC,CAAA;QACpC,CAAC;QACD,aAAa,CAAC,GAAG,CAAC,WAAW,CAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;IAC3C,CAAC;IAED,MAAM,IAAI,GAAU,EAAE,CAAA;IACtB,KAAK,MAAM,CAAC,SAAS,EAAE,aAAa,CAAC,IAAI,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC;QAC9D,KAAK,MAAM,CAAC,WAAW,EAAE,WAAW,CAAC,IAAI,aAAa,CAAC,OAAO,EAAE,EAAE,CAAC;YACjE,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBAC7B,SAAQ;YACV,CAAC;YACD,IAAI,CAAC,IAAI,CAAC;gBACR,UAAU,EAAE;oBACV,SAAS;oBACT,WAAW;oBACX,WAAW,EAAE,WAAW,CAAC,MAAM;iBAChC;gBACD,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE;oBACzB,SAAG,CAAC,KAAK,CAAC,kBAAkB,EAAE,EAAE,QAAQ,EAAE,OAAO,CAAC,QAAQ,EAAE,GAAG,OAAO,CAAC,UAAU,EAAE,CAAC,CAAA;oBACpF,MAAM,WAAW,CAAC,SAAS,EAAE,aAAa,EAAE,WAAW,CAAC,CAAA;oBACxD,SAAG,CAAC,KAAK,CAAC,mBAAmB,EAAE,EAAE,QAAQ,EAAE,OAAO,CAAC,QAAQ,EAAE,GAAG,OAAO,CAAC,UAAU,EAAE,CAAC,CAAA;gBACvF,CAAC;aACF,CAAC,CAAA;QACJ,CAAC;IACH,CAAC;IAED,SAAG,CAAC,KAAK,CAAC,mBAAmB,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,MAAM,EAAE,WAAW,EAAE,CAAC,CAAA;IAClE,MAAM,YAAY,GAAG,MAAM,IAAA,qBAAO,EAAC,IAAI,EAAE,WAAW,CAAC,CAAA;IACrD,MAAM,aAAa,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,UAAU,CAAC,CAAA;IACzE,IAAI,aAAa,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC7B,SAAG,CAAC,KAAK,CAAC,sBAAsB,EAAE,EAAE,UAAU,EAAE,aAAa,CAAC,MAAM,EAAE,CAAC,CAAA;QACvE,KAAK,MAAM,SAAS,IAAI,aAAa,EAAE,CAAC;YACtC,SAAG,CAAC,KAAK,CAAC,gBAAgB,EAAE,SAAS,CAAC,MAAM,EAAE,SAAS,CAAC,UAAU,CAAC,CAAA;QACrE,CAAC;QACD,MAAM,IAAI,KAAK,CAAC,kDAAkD,CAAC,CAAA;IACrE,CAAC;IACD,SAAG,CAAC,IAAI,CAAC,mBAAmB,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC,CAAA;IAEpD,OAAO,YAAY,CAAA;AACrB,CAAC;AAED;;;;;;;GAOG;AACH,KAAK,UAAU,WAAW,CACxB,SAAiB,EACjB,aAA4B,EAC5B,WAAuB;IAEvB,IAAI,KAAK,GAAY,KAAK,CAAA;IAC1B,IAAI,YAAY,GAAW,CAAC,CAAA;IAC5B,MAAM,OAAO,GAAG,WAAW,CAAC,CAAC,CAAC,CAAC,OAAO,CAAA;IACtC,MAAM,OAAO,GAAG,IAAA,6BAAmB,EAAC,aAAa,EAAE,SAAS,EAAE,KAAK,CAAC,CAAA;IAEpE,OAAO,CAAC,KAAK,IAAI,YAAY,GAAG,CAAC,EAAE,CAAC;QAClC,MAAM,KAAK,GAAG,MAAM,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAA;QAC7C,KAAK,MAAM,GAAG,IAAI,WAAW,EAAE,CAAC;YAC9B,MAAM,OAAO,CAAC,WAAW,CAAC,KAAK,CAAC,IAAI,EAAE,GAAG,CAAC,SAAS,EAAE,GAAG,CAAC,OAAO,EAAE,OAAO,CAAC,CAAA;QAC5E,CAAC;QACD,KAAK,GAAG,MAAM,OAAO,CAAC,SAAS,CAAC,OAAO,EAAE,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,MAAM,CAAC,CAAA;QAClE,YAAY,EAAE,CAAA;IAChB,CAAC;IACD,IAAI,CAAC,KAAK,EAAE,CAAC;QACX,MAAM,IAAI,KAAK,CAAC,0BAA0B,OAAO,CAAC,IAAI,UAAU,YAAY,WAAW,CAAC,CAAA;IAC1F,CAAC;AACH,CAAC"}
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.runJobs = runJobs;
|
|
4
|
-
const
|
|
4
|
+
const log_1 = require("@cloud-copilot/log");
|
|
5
5
|
/**
|
|
6
6
|
* Runs the given jobs with up to `concurrency` tasks in flight at once.
|
|
7
7
|
* Resolves with an array of results in the same order as the jobs.
|
|
@@ -20,7 +20,7 @@ async function runJobs(jobs, concurrency) {
|
|
|
20
20
|
};
|
|
21
21
|
const startTime = Date.now();
|
|
22
22
|
const interval = setInterval(() => {
|
|
23
|
-
|
|
23
|
+
log_1.log.warn(`Long-running job detected.`, { minutes: Math.floor((Date.now() - startTime) / 60000) }, { ...context, ...jobs[i].properties });
|
|
24
24
|
}, 60_000);
|
|
25
25
|
try {
|
|
26
26
|
const value = await jobs[i].execute({ ...context, properties: jobs[i].properties });
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"jobQueue.js","sourceRoot":"","sources":["../../../src/jobs/jobQueue.ts"],"names":[],"mappings":";;AAqCA,0BA6CC;AAlFD,
|
|
1
|
+
{"version":3,"file":"jobQueue.js","sourceRoot":"","sources":["../../../src/jobs/jobQueue.ts"],"names":[],"mappings":";;AAqCA,0BA6CC;AAlFD,4CAAwC;AAiCxC;;;GAGG;AACI,KAAK,UAAU,OAAO,CAC3B,IAAiB,EACjB,WAAmB;IAEnB,MAAM,OAAO,GAAsB,EAAE,CAAA;IACrC,IAAI,SAAS,GAAG,CAAC,CAAA;IAEjB,oFAAoF;IACpF,KAAK,UAAU,MAAM,CAAC,QAAgB;QACpC,OAAO,IAAI,EAAE,CAAC;YACZ,MAAM,CAAC,GAAG,SAAS,EAAE,CAAA;YACrB,IAAI,CAAC,IAAI,IAAI,CAAC,MAAM;gBAAE,OAAM;YAE5B,MAAM,OAAO,GAAe;gBAC1B,QAAQ;aACT,CAAA;YAED,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAA;YAC5B,MAAM,QAAQ,GAAG,WAAW,CAAC,GAAG,EAAE;gBAChC,SAAG,CAAC,IAAI,CACN,4BAA4B,EAC5B,EAAE,OAAO,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC,GAAG,KAAK,CAAC,EAAE,EACzD,EAAE,GAAG,OAAO,EAAE,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,UAAU,EAAE,CACtC,CAAA;YACH,CAAC,EAAE,MAAM,CAAC,CAAA;YACV,IAAI,CAAC;gBACH,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,EAAE,GAAG,OAAO,EAAE,UAAU,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,UAAU,EAAE,CAAC,CAAA;gBACnF,OAAO,CAAC,CAAC,CAAC,GAAG,EAAE,MAAM,EAAE,WAAW,EAAE,KAAK,EAAE,UAAU,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,UAAU,EAAE,CAAA;YAC7E,CAAC;YAAC,OAAO,MAAM,EAAE,CAAC;gBAChB,OAAO,CAAC,CAAC,CAAC,GAAG,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,EAAE,UAAU,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,UAAU,EAAE,CAAA;YAC7E,CAAC;oBAAS,CAAC;gBACT,aAAa,CAAC,QAAQ,CAAC,CAAA;YACzB,CAAC;QACH,CAAC;IACH,CAAC;IAED,4EAA4E;IAC5E,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,WAAW,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;SACtD,IAAI,CAAC,IAAI,CAAC;SACV,GAAG,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,EAAE,CAAC,MAAM,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAA;IAEnC,iCAAiC;IACjC,MAAM,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAA;IAE1B,OAAO,OAAO,CAAA;AAChB,CAAC"}
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.JobRunner = void 0;
|
|
4
|
-
const
|
|
4
|
+
const log_1 = require("@cloud-copilot/log");
|
|
5
5
|
/**
|
|
6
6
|
* Job runner that allows running jobs as they are created.
|
|
7
7
|
* This will wait for jobs to be added to it and run them up the
|
|
@@ -43,7 +43,7 @@ class JobRunner {
|
|
|
43
43
|
const context = { workerId };
|
|
44
44
|
const startTime = Date.now();
|
|
45
45
|
const interval = setInterval(() => {
|
|
46
|
-
|
|
46
|
+
log_1.log.warn(`Long-running job detected.`, { minutes: Math.floor((Date.now() - startTime) / 60000) }, { ...context, ...job.properties });
|
|
47
47
|
}, 60_000);
|
|
48
48
|
try {
|
|
49
49
|
const value = await job.execute({ ...context, properties: job.properties });
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"jobRunner.js","sourceRoot":"","sources":["../../../src/jobs/jobRunner.ts"],"names":[],"mappings":";;;AAAA,
|
|
1
|
+
{"version":3,"file":"jobRunner.js","sourceRoot":"","sources":["../../../src/jobs/jobRunner.ts"],"names":[],"mappings":";;;AAAA,4CAAwC;AAGxC;;;;;;GAMG;AAEH,MAAa,SAAS;IAeA;IAdZ,KAAK,GAAgB,EAAE,CAAA;IACvB,OAAO,GAAsB,EAAE,CAAA;IAC/B,UAAU,GAAG,CAAC,CAAA;IACd,gBAAgB,GAAmB,EAAE,CAAA;IACrC,OAAO,GAAoB,EAAE,CAAA;IAC7B,eAAe,GAAG,IAAI,CAAA;IACtB,oBAAoB,GAAyB,IAAI,CAAA;IACjD,oBAAoB,GAAwB,IAAI,CAAA;IAExD;;;;OAIG;IACH,YAAoB,WAAmB;QAAnB,gBAAW,GAAX,WAAW,CAAQ;IAAG,CAAC;IAEnC,KAAK,CAAC,MAAM,CAAC,QAAgB;QACnC,OAAO,IAAI,CAAC,eAAe,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACrD,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAA;YAC9B,IAAI,CAAC,GAAG,EAAE,CAAC;gBACT,IAAI,CAAC,IAAI,CAAC,eAAe,EAAE,CAAC;oBAC1B,8DAA8D;oBAC9D,OAAM;gBACR,CAAC;gBACD,mDAAmD;gBACnD,MAAM,IAAI,CAAC,oBAAoB,EAAE,CAAA;gBACjC,SAAQ;YACV,CAAC;YAED,IAAI,CAAC,UAAU,EAAE,CAAA;YACjB,MAAM,OAAO,GAAe,EAAE,QAAQ,EAAE,CAAA;YAExC,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAA;YAC5B,MAAM,QAAQ,GAAG,WAAW,CAAC,GAAG,EAAE;gBAChC,SAAG,CAAC,IAAI,CACN,4BAA4B,EAC5B,EAAE,OAAO,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC,GAAG,KAAK,CAAC,EAAE,EACzD,EAAE,GAAG,OAAO,EAAE,GAAG,GAAG,CAAC,UAAU,EAAE,CAClC,CAAA;YACH,CAAC,EAAE,MAAM,CAAC,CAAA;YAEV,IAAI,CAAC;gBACH,MAAM,KAAK,GAAG,MAAM,GAAG,CAAC,OAAO,CAAC,EAAE,GAAG,OAAO,EAAE,UAAU,EAAE,GAAG,CAAC,UAAU,EAAE,CAAC,CAAA;gBAC3E,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,WAAW,EAAE,KAAK,EAAE,UAAU,EAAE,GAAG,CAAC,UAAU,EAAE,CAAC,CAAA;YAC/E,CAAC;YAAC,OAAO,MAAM,EAAE,CAAC;gBAChB,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,EAAE,UAAU,EAAE,GAAG,CAAC,UAAU,EAAE,CAAC,CAAA;YAC/E,CAAC;oBAAS,CAAC;gBACT,aAAa,CAAC,QAAQ,CAAC,CAAA;gBACvB,IAAI,CAAC,UAAU,EAAE,CAAA;gBACjB,IAAI,CAAC,WAAW,EAAE,CAAA;YACpB,CAAC;QACH,CAAC;IACH,CAAC;IAEO,oBAAoB;QAC1B,IAAI,CAAC,IAAI,CAAC,oBAAoB,EAAE,CAAC;YAC/B,IAAI,CAAC,oBAAoB,GAAG,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE;gBACxD,IAAI,CAAC,oBAAoB,GAAG,OAAO,CAAA;YACrC,CAAC,CAAC,CAAA;QACJ,CAAC;QACD,OAAO,IAAI,CAAC,oBAAoB,CAAA;IAClC,CAAC;IAEO,aAAa;QACnB,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,KAAK,CAAC,IAAI,IAAI,CAAC,eAAe,EAAE,CAAC;YACtD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC,EAAE,EAAE,CAAC;gBAC1C,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAA;YACvC,CAAC;QACH,CAAC;IACH,CAAC;IAEO,sBAAsB;QAC5B,0BAA0B;QAC1B,IAAI,IAAI,CAAC,oBAAoB,EAAE,CAAC;YAC9B,IAAI,CAAC,oBAAoB,EAAE,CAAA;YAC3B,IAAI,CAAC,oBAAoB,GAAG,IAAI,CAAA;YAChC,IAAI,CAAC,oBAAoB,GAAG,IAAI,CAAA;QAClC,CAAC;IACH,CAAC;IAEO,WAAW;QACjB,IAAI,IAAI,CAAC,UAAU,KAAK,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACrD,+BAA+B;YAC/B,IAAI,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,EAAE,CAAC,CAAA;YACrD,IAAI,CAAC,gBAAgB,GAAG,EAAE,CAAA;QAC5B,CAAC;IACH,CAAC;IAED;;OAEG;IACH,OAAO,CAAC,GAAc;QACpB,IAAI,CAAC,IAAI,CAAC,eAAe,EAAE,CAAC;YAC1B,MAAM,IAAI,KAAK,CAAC,oCAAoC,CAAC,CAAA;QACvD,CAAC;QACD,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;QACpB,IAAI,CAAC,aAAa,EAAE,CAAA;QACpB,IAAI,CAAC,sBAAsB,EAAE,CAAA;IAC/B,CAAC;IAED;;OAEG;IACH,UAAU,CAAC,IAAiB;QAC1B,IAAI,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAA;IAC1C,CAAC;IAED;;OAEG;IACH,WAAW;QACT,sEAAsE;QACtE,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE;YAC7B,IAAI,IAAI,CAAC,UAAU,KAAK,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACrD,OAAO,EAAE,CAAA;YACX,CAAC;iBAAM,CAAC;gBACN,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;YACrC,CAAC;QACH,CAAC,CAAC,CAAA;IACJ,CAAC;IAED;;OAEG;IACH,UAAU;QACR,OAAO,IAAI,CAAC,OAAO,CAAA;IACrB,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,aAAa;QACjB,IAAI,CAAC,eAAe,GAAG,KAAK,CAAA;QAC5B,0EAA0E;QAC1E,IAAI,CAAC,sBAAsB,EAAE,CAAA;QAC7B,+DAA+D;QAC/D,MAAM,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;QAC/B,IAAI,CAAC,OAAO,GAAG,EAAE,CAAA;IACnB,CAAC;IAED;;OAEG;IACH,IAAI,WAAW;QACb,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,CAAA;IAC1B,CAAC;IAED;;OAEG;IACH,IAAI,cAAc;QAChB,OAAO,IAAI,CAAC,UAAU,CAAA;IACxB,CAAC;CACF;AA7JD,8BA6JC"}
|
|
@@ -7,7 +7,7 @@ const auth_js_1 = require("../../aws/auth.js");
|
|
|
7
7
|
const ClientPool_js_1 = require("../../aws/ClientPool.js");
|
|
8
8
|
const coreAuth_js_1 = require("../../aws/coreAuth.js");
|
|
9
9
|
const client_tools_js_1 = require("../../utils/client-tools.js");
|
|
10
|
-
const
|
|
10
|
+
const log_1 = require("@cloud-copilot/log");
|
|
11
11
|
class S3PathBasedPersistenceAdapter {
|
|
12
12
|
storageConfig;
|
|
13
13
|
deleteData;
|
|
@@ -66,7 +66,7 @@ class S3PathBasedPersistenceAdapter {
|
|
|
66
66
|
}
|
|
67
67
|
catch (error) {
|
|
68
68
|
if (error.name === 'PreconditionFailed' || error.name === 'ConditionalRequestConflict') {
|
|
69
|
-
|
|
69
|
+
log_1.log.debug({ filePath, lockId }, 'Optimistic locking failed. The object may have been modified by another process.');
|
|
70
70
|
// PreconditionFailed indicates that the ETag does not match the current version of the object
|
|
71
71
|
// ConditionalRequestConflict indicates a conflicting operation occurred during the request
|
|
72
72
|
// In either case, we can return false and let the caller handle the conflict
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"S3PathBasedPersistenceAdapter.js","sourceRoot":"","sources":["../../../../src/persistence/s3/S3PathBasedPersistenceAdapter.ts"],"names":[],"mappings":";;;AAAA,kDAS2B;AAC3B,wDAAwD;AACxD,+CAAkD;AAClD,2DAAuD;AACvD,uDAAgE;AAEhE,iEAA4D;AAC5D
|
|
1
|
+
{"version":3,"file":"S3PathBasedPersistenceAdapter.js","sourceRoot":"","sources":["../../../../src/persistence/s3/S3PathBasedPersistenceAdapter.ts"],"names":[],"mappings":";;;AAAA,kDAS2B;AAC3B,wDAAwD;AACxD,+CAAkD;AAClD,2DAAuD;AACvD,uDAAgE;AAEhE,iEAA4D;AAC5D,4CAAwC;AAGxC,MAAa,6BAA6B;IAGrB;IACA;IAHX,oBAAoB,CAAoB;IAChD,YACmB,aAA8B,EAC9B,UAAmB;QADnB,kBAAa,GAAb,aAAa,CAAiB;QAC9B,eAAU,GAAV,UAAU,CAAS;IACnC,CAAC;IAEI,KAAK,CAAC,SAAS;QACrB;;;;;WAKG;QACH,IAAI,CAAC,IAAI,CAAC,oBAAoB,EAAE,CAAC;YAC/B,MAAM,UAAU,GAAG,IAAI,CAAC,aAAa,CAAC,IAAI,CAAA;YAE1C,sCAAsC;YACtC,IAAI,UAAU,IAAI,UAAU,CAAC,WAAW,IAAI,KAAK,IAAI,UAAU,CAAC,WAAW,EAAE,CAAC;gBAC5E,IAAI,CAAC,oBAAoB,GAAG,IAAA,yBAAa,EAAC,UAAU,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,SAAS,CAAA;YACjF,CAAC;YAED,2DAA2D;YAC3D,IAAI,CAAC,IAAI,CAAC,oBAAoB,EAAE,CAAC;gBAC/B,MAAM,kBAAkB,GAAG,MAAM,IAAA,sCAAwB,EAAC,UAAU,EAAE;oBACpE,KAAK,EAAE,+BAA+B;iBACvC,CAAC,CAAA;gBACF,IAAI,CAAC,oBAAoB,GAAG,kBAAkB,CAAC,SAAS,CAAA;YAC1D,CAAC;QACH,CAAC;QAED,MAAM,WAAW,GAAG,MAAM,IAAA,wBAAc,EAAC,IAAI,CAAC,oBAAoB,EAAE,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,CAAA;QAC5F,MAAM,MAAM,GAAG,6BAAa,CAAC,eAAe,CAAC,MAAM,CACjD,oBAAQ,EACR,WAAW,EACX,IAAI,CAAC,aAAa,CAAC,MAAM,EACzB,IAAI,CAAC,aAAa,CAAC,QAAQ,CAC5B,CAAA;QACD,OAAO,MAAM,CAAA;IACf,CAAC;IAED,KAAK,CAAC,SAAS,CAAC,QAAgB,EAAE,IAAqB;QACrD,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,SAAS,EAAE,CAAA;QACrC,MAAM,MAAM,CAAC,IAAI,CACf,IAAI,4BAAgB,CAAC;YACnB,MAAM,EAAE,IAAI,CAAC,aAAa,CAAC,MAAM;YACjC,GAAG,EAAE,QAAQ;YACb,IAAI,EAAE,IAAI;SACX,CAAC,CACH,CAAA;IACH,CAAC;IAED,KAAK,CAAC,uBAAuB,CAC3B,QAAgB,EAChB,IAAqB,EACrB,MAAc;QAEd,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,SAAS,EAAE,CAAA;QACrC,IAAI,CAAC;YACH,MAAM,MAAM,GAA0B;gBACpC,MAAM,EAAE,IAAI,CAAC,aAAa,CAAC,MAAM;gBACjC,GAAG,EAAE,QAAQ;gBACb,IAAI,EAAE,IAAI;aACX,CAAA;YAED,yDAAyD;YACzD,IAAI,MAAM,IAAI,MAAM,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC;gBACnC,MAAM,CAAC,SAAS,CAAC,GAAG,MAAM,CAAA;YAC5B,CAAC;YAED,MAAM,MAAM,CAAC,IAAI,CAAC,IAAI,4BAAgB,CAAC,MAAM,CAAC,CAAC,CAAA;YAC/C,OAAO,IAAI,CAAA;QACb,CAAC;QAAC,OAAO,KAAU,EAAE,CAAC;YACpB,IAAI,KAAK,CAAC,IAAI,KAAK,oBAAoB,IAAI,KAAK,CAAC,IAAI,KAAK,4BAA4B,EAAE,CAAC;gBACvF,SAAG,CAAC,KAAK,CACP,EAAE,QAAQ,EAAE,MAAM,EAAE,EACpB,kFAAkF,CACnF,CAAA;gBACD,8FAA8F;gBAC9F,2FAA2F;gBAC3F,6EAA6E;gBAC7E,OAAO,KAAK,CAAA;YACd,CAAC;YACD,iFAAiF;YACjF,MAAM,KAAK,CAAA;QACb,CAAC;IACH,CAAC;IAED,KAAK,CAAC,QAAQ,CAAC,QAAgB;QAC7B,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,QAAQ,CAAC,CAAA;QACtD,OAAO,QAAQ,EAAE,IAAI,CAAA;IACvB,CAAC;IAED,KAAK,CAAC,gBAAgB,CAAC,QAAgB;QACrC,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,SAAS,EAAE,CAAA;QAErC,MAAM,QAAQ,GAAG,MAAM,IAAA,gCAAc,EAAC,KAAK,IAAI,EAAE;YAC/C,OAAO,MAAM,CAAC,IAAI,CAChB,IAAI,4BAAgB,CAAC;gBACnB,MAAM,EAAE,IAAI,CAAC,aAAa,CAAC,MAAM;gBACjC,GAAG,EAAE,QAAQ;aACd,CAAC,CACH,CAAA;QACH,CAAC,CAAC,CAAA;QAEF,IAAI,CAAC,QAAQ,EAAE,CAAC;YACd,OAAO,SAAS,CAAA;QAClB,CAAC;QAED,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,iBAAiB,EAAG,CAAA;QACtD,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAK,CAAA;QAC3B,OAAO;YACL,IAAI;YACJ,IAAI;SACL,CAAA;IACH,CAAC;IAED,KAAK,CAAC,UAAU,CAAC,QAAgB;QAC/B,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,CAAC;YACrB,OAAM;QACR,CAAC;QACD,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,SAAS,EAAE,CAAA;QACrC,MAAM,MAAM,CAAC,IAAI,CACf,IAAI,+BAAmB,CAAC;YACtB,MAAM,EAAE,IAAI,CAAC,aAAa,CAAC,MAAM;YACjC,GAAG,EAAE,QAAQ;SACd,CAAC,CACH,CAAA;IACH,CAAC;IAED,KAAK,CAAC,eAAe,CAAC,OAAe;QACnC,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,CAAC;YACrB,OAAM;QACR,CAAC;QACD,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,SAAS,EAAE,CAAA;QACrC,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;YAC3B,OAAO,IAAI,GAAG,CAAA;QAChB,CAAC;QAED,6EAA6E;QAC7E,6EAA6E;QAC7E,IAAI,iBAAiB,GAAuB,SAAS,CAAA;QAErD,GAAG,CAAC;YACF,mCAAmC;YACnC,MAAM,IAAI,GAA+B,MAAM,MAAM,CAAC,IAAI,CACxD,IAAI,gCAAoB,CAAC;gBACvB,MAAM,EAAE,IAAI,CAAC,aAAa,CAAC,MAAM;gBACjC,MAAM,EAAE,OAAO;gBACf,iBAAiB;aAClB,CAAC,CACH,CAAA;YAED,MAAM,OAAO,GAAG,IAAI,CAAC,QAAQ,IAAI,EAAE,CAAA;YACnC,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACvB,8BAA8B;gBAC9B,MAAM,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,GAAI,EAAE,CAAC,CAAC,CAAA;gBACtD,MAAM,MAAM,CAAC,IAAI,CACf,IAAI,gCAAoB,CAAC;oBACvB,MAAM,EAAE,IAAI,CAAC,aAAa,CAAC,MAAM;oBACjC,MAAM,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,IAAI,EAAE;iBAC3C,CAAC,CACH,CAAA;YACH,CAAC;YAED,0BAA0B;YAC1B,iBAAiB,GAAG,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,qBAAqB,CAAC,CAAC,CAAC,SAAS,CAAA;QAC/E,CAAC,QAAQ,iBAAiB,EAAC;IAC7B,CAAC;IAED,KAAK,CAAC,aAAa,CAAC,OAAe;QACjC,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,SAAS,EAAE,CAAA;QACrC,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;YAC3B,OAAO,IAAI,GAAG,CAAA;QAChB,CAAC;QAED,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,IAAI,CAChC,IAAI,gCAAoB,CAAC;YACvB,MAAM,EAAE,IAAI,CAAC,aAAa,CAAC,MAAM;YACjC,MAAM,EAAE,OAAO;YACf,SAAS,EAAE,GAAG;SACf,CAAC,CACH,CAAA;QAED,IAAI,CAAC,QAAQ,CAAC,cAAc,EAAE,CAAC;YAC7B,OAAO,EAAE,CAAA;QACX,CAAC;QAED,OAAO,CACL,QAAQ,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,MAAO,CAAC;aAC5C,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,KAAK,OAAO,CAAC;YACjC,0EAA0E;aACzE,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CACxD,CAAA;IACH,CAAC;IAED,KAAK,CAAC,eAAe,CAAC,OAAe,EAAE,SAAmB,EAAE,QAAgB;QAC1E,IAAI,QAAQ,GAAG,CAAC,OAAO,CAAC,CAAA;QACxB,KAAK,MAAM,IAAI,IAAI,SAAS,EAAE,CAAC;YAC7B,IAAI,IAAI,IAAI,GAAG,EAAE,CAAC;gBAChB,MAAM,OAAO,GAAG,EAAE,CAAA;gBAClB,KAAK,MAAM,GAAG,IAAI,QAAQ,EAAE,CAAC;oBAC3B,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,CAAA;oBAC7C,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;wBAC5B,IAAI,KAAK,KAAK,QAAQ,EAAE,CAAC;4BACvB,OAAO,CAAC,IAAI,CAAC,GAAG,GAAG,GAAG,GAAG,KAAK,CAAC,CAAA;wBACjC,CAAC;oBACH,CAAC;gBACH,CAAC;gBACD,QAAQ,GAAG,OAAO,CAAA;YACpB,CAAC;iBAAM,CAAC;gBACN,QAAQ,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,GAAG,GAAG,GAAG,IAAI,CAAC,CAAA;YACpD,CAAC;QACH,CAAC;QAED,MAAM,OAAO,GAAa,EAAE,CAAA;QAC5B,KAAK,MAAM,GAAG,IAAI,QAAQ,EAAE,CAAC;YAC3B,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,GAAG,GAAG,GAAG,GAAG,QAAQ,CAAC,CAAA;YACtD,IAAI,IAAI,EAAE,CAAC;gBACT,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;YACpB,CAAC;QACH,CAAC;QAED,OAAO,OAAO,CAAA;IAChB,CAAC;CACF;AAjOD,sEAiOC"}
|
|
@@ -3,7 +3,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
|
3
3
|
exports.RestApisSync = void 0;
|
|
4
4
|
exports.parseApiGatewayPolicy = parseApiGatewayPolicy;
|
|
5
5
|
const client_api_gateway_1 = require("@aws-sdk/client-api-gateway");
|
|
6
|
-
const
|
|
6
|
+
const log_1 = require("@cloud-copilot/log");
|
|
7
7
|
const typedSync_js_1 = require("../typedSync.js");
|
|
8
8
|
exports.RestApisSync = (0, typedSync_js_1.createTypedSyncOperation)('apigateway', 'gateways', (0, typedSync_js_1.createResourceSyncType)({
|
|
9
9
|
client: client_api_gateway_1.APIGatewayClient,
|
|
@@ -58,7 +58,7 @@ function parseApiGatewayPolicy(apiId, policy) {
|
|
|
58
58
|
return JSON.parse(policy);
|
|
59
59
|
}
|
|
60
60
|
catch (error) {
|
|
61
|
-
|
|
61
|
+
log_1.log.error('Failed to parse API Gateway policy', { apiId }, error);
|
|
62
62
|
throw new Error(`Unable to parse policy for API Gateway ${apiId}`);
|
|
63
63
|
}
|
|
64
64
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"gateways.js","sourceRoot":"","sources":["../../../../src/syncs/apigateway/gateways.ts"],"names":[],"mappings":";;;AAuDA,sDAeC;AAtED,oEAAkF;AAClF
|
|
1
|
+
{"version":3,"file":"gateways.js","sourceRoot":"","sources":["../../../../src/syncs/apigateway/gateways.ts"],"names":[],"mappings":";;;AAuDA,sDAeC;AAtED,oEAAkF;AAClF,4CAAwC;AAExC,kDAAkF;AAErE,QAAA,YAAY,GAAS,IAAA,uCAAwB,EACxD,YAAY,EACZ,UAAU,EACV,IAAA,qCAAsB,EAAC;IACrB,MAAM,EAAE,qCAAgB;IACxB,OAAO,EAAE,uCAAkB;IAC3B,GAAG,EAAE,OAAO;IACZ,gBAAgB,EAAE;QAChB,QAAQ,EAAE,UAAU;QACpB,SAAS,EAAE,UAAU;KACtB;IACD,GAAG,EAAE,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,SAAS,EAAE,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC,EAAG,EAAE,MAAM,EAAE,SAAS,CAAC;IAChF,IAAI,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI;IACvB,iBAAiB,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE,CAAC,CAAC;QACvC,OAAO;QACP,OAAO,EAAE,YAAY;QACrB,MAAM;QACN,YAAY,EAAE,UAAU;KACzB,CAAC;IACF,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;QACjB,QAAQ,EAAE;YACR,EAAE,EAAE,GAAG,CAAC,EAAE;YACV,IAAI,EAAE,GAAG,CAAC,IAAI;SACf;QACD,MAAM,EAAE,qBAAqB,CAAC,GAAG,CAAC,EAAG,EAAE,GAAG,CAAC,MAAM,CAAC;KACnD,CAAC;CACH,CAAC,CACH,CAAA;AAED;;;;;;;GAOG;AACH,SAAS,UAAU,CAAC,KAAa,EAAE,MAAc,EAAE,SAAiB;IAClE,OAAO,OAAO,SAAS,eAAe,MAAM,eAAe,KAAK,EAAE,CAAA;AACpE,CAAC;AAED;;;;;;;;GAQG;AACH,SAAgB,qBAAqB,CACnC,KAAa,EACb,MAA0B;IAE1B,IAAI,CAAC,MAAM,EAAE,CAAC;QACZ,OAAO,SAAS,CAAA;IAClB,CAAC;IACD,MAAM,GAAG,MAAM,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC,CAAA;IAE1D,IAAI,CAAC;QACH,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAA;IAC3B,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,SAAG,CAAC,KAAK,CAAC,oCAAoC,EAAE,EAAE,KAAK,EAAE,EAAE,KAAK,CAAC,CAAA;QACjE,MAAM,IAAI,KAAK,CAAC,0CAA0C,KAAK,EAAE,CAAC,CAAA;IACpE,CAAC;AACH,CAAC"}
|
|
@@ -4,7 +4,7 @@ exports.AuthorizationDetailsSync = void 0;
|
|
|
4
4
|
exports.getAuthorizationDetails = getAuthorizationDetails;
|
|
5
5
|
const client_iam_1 = require("@aws-sdk/client-iam");
|
|
6
6
|
const client_tools_js_1 = require("../../utils/client-tools.js");
|
|
7
|
-
const
|
|
7
|
+
const log_1 = require("@cloud-copilot/log");
|
|
8
8
|
const tags_js_1 = require("../../utils/tags.js");
|
|
9
9
|
const sync_js_1 = require("../sync.js");
|
|
10
10
|
exports.AuthorizationDetailsSync = {
|
|
@@ -172,7 +172,7 @@ async function getAuthorizationDetails(client, concurrentWorkerPool, customConfi
|
|
|
172
172
|
} while (isTruncated);
|
|
173
173
|
let policiesWithTags = policyDetails;
|
|
174
174
|
if (customConfig?.includePolicyTags) {
|
|
175
|
-
|
|
175
|
+
log_1.log.info('Including policy tags in authorization details sync');
|
|
176
176
|
policiesWithTags = await getTagsForManagedPolicies(client, policyDetails, concurrentWorkerPool);
|
|
177
177
|
}
|
|
178
178
|
return {
|
|
@@ -218,7 +218,7 @@ async function getTagsForManagedPolicies(client, policies, concurrentWorkerPool)
|
|
|
218
218
|
else {
|
|
219
219
|
// Log the error but continue processing other policies
|
|
220
220
|
const policyArn = jobResult.properties.policy.Arn;
|
|
221
|
-
|
|
221
|
+
log_1.log.error('Failed to get tags for policy', jobResult.reason, {
|
|
222
222
|
policyArn
|
|
223
223
|
});
|
|
224
224
|
throw new Error(`Failed to get tags for policy ${policyArn}. See logs for details.`);
|