@backstage/plugin-catalog-backend-module-aws 0.2.2 → 0.2.3-next.1

package/CHANGELOG.md

@@ -1,5 +1,44 @@
 # @backstage/plugin-catalog-backend-module-aws
 
+## 0.2.3-next.1
+
+### Patch Changes
+
+- 629cbd194a87: Use `coreServices.rootConfig` instead of `coreService.config`
+- 4b82382ed8c2: Fixed invalid configuration schema. The configuration schema may be more strict as a result.
+- b4222908b0c3: Added option to configure AWS `accountId` in `AwsS3EntityProvider`
+- Updated dependencies
+  - @backstage/backend-common@0.19.2-next.1
+  - @backstage/plugin-catalog-node@1.4.1-next.1
+  - @backstage/backend-plugin-api@0.6.0-next.1
+  - @backstage/backend-tasks@0.5.5-next.1
+  - @backstage/integration@1.5.1
+  - @backstage/integration-aws-node@0.1.5
+  - @backstage/catalog-model@1.4.1
+  - @backstage/config@1.0.8
+  - @backstage/errors@1.2.1
+  - @backstage/types@1.1.0
+  - @backstage/plugin-catalog-common@1.0.15
+  - @backstage/plugin-kubernetes-common@0.6.5
+
+## 0.2.3-next.0
+
+### Patch Changes
+
+- Updated dependencies
+  - @backstage/backend-common@0.19.2-next.0
+  - @backstage/backend-plugin-api@0.5.5-next.0
+  - @backstage/backend-tasks@0.5.5-next.0
+  - @backstage/catalog-model@1.4.1
+  - @backstage/config@1.0.8
+  - @backstage/errors@1.2.1
+  - @backstage/integration@1.5.1
+  - @backstage/integration-aws-node@0.1.5
+  - @backstage/types@1.1.0
+  - @backstage/plugin-catalog-common@1.0.15
+  - @backstage/plugin-catalog-node@1.4.1-next.0
+  - @backstage/plugin-kubernetes-common@0.6.5
+
 ## 0.2.2
 
 ### Patch Changes

package/alpha/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@backstage/plugin-catalog-backend-module-aws",
-  "version": "0.2.2",
+  "version": "0.2.3-next.1",
   "main": "../dist/alpha.cjs.js",
   "types": "../dist/alpha.d.ts"
 }

package/config.d.ts

@@ -54,14 +54,19 @@ export interface Config {
              * @see https://docs.aws.amazon.com/sdk-for-javascript/v3/developer-guide/setting-region.html
              */
             region?: string;
+            /**
+             * (Optional) AWS Account id.
+             * If not set, main account is used.
+             * @see https://github.com/backstage/backstage/blob/master/packages/integration-aws-node/README.md
+             */
+            accountId?: string;
             /**
              * (Optional) TaskScheduleDefinition for the refresh.
              */
             schedule?: TaskScheduleDefinitionConfig;
           }
-        | Record<
-            string,
-            {
+        | {
+            [name: string]: {
               /**
                * (Required) AWS S3 Bucket Name
                */
@@ -77,12 +82,18 @@ export interface Config {
                * @see https://docs.aws.amazon.com/sdk-for-javascript/v3/developer-guide/setting-region.html
                */
               region?: string;
+              /**
+               * (Optional) AWS Account id.
+               * If not set, main account is used.
+               * @see https://github.com/backstage/backstage/blob/master/packages/integration-aws-node/README.md
+               */
+              accountId?: string;
               /**
                * (Optional) TaskScheduleDefinition for the refresh.
                */
               schedule?: TaskScheduleDefinitionConfig;
-            }
-          >;
+            };
+          };
     };
   };
 }

package/dist/alpha.cjs.js

@@ -5,7 +5,7 @@ Object.defineProperty(exports, '__esModule', { value: true });
 var backendCommon = require('@backstage/backend-common');
 var backendPluginApi = require('@backstage/backend-plugin-api');
 var alpha = require('@backstage/plugin-catalog-node/alpha');
-var AwsS3EntityProvider = require('./cjs/AwsS3EntityProvider-45cf9dc1.cjs.js');
+var AwsS3EntityProvider = require('./cjs/AwsS3EntityProvider-8bab60be.cjs.js');
 require('@backstage/integration');
 require('@backstage/plugin-catalog-node');
 require('@backstage/backend-tasks');
@@ -20,7 +20,7 @@ const catalogModuleAwsS3EntityProvider = backendPluginApi.createBackendModule({
   register(env) {
     env.registerInit({
       deps: {
-        config: backendPluginApi.coreServices.config,
+        config: backendPluginApi.coreServices.rootConfig,
         catalog: alpha.catalogProcessingExtensionPoint,
         logger: backendPluginApi.coreServices.logger,
         scheduler: backendPluginApi.coreServices.scheduler

package/dist/alpha.cjs.js.map

@@ -1 +1 @@
-{"version":3,"file":"alpha.cjs.js","sources":["../src/module/catalogModuleAwsS3EntityProvider.ts"],"sourcesContent":["/*\n * Copyright 2022 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport { loggerToWinstonLogger } from '@backstage/backend-common';\nimport {\n  coreServices,\n  createBackendModule,\n} from '@backstage/backend-plugin-api';\nimport { catalogProcessingExtensionPoint } from '@backstage/plugin-catalog-node/alpha';\nimport { AwsS3EntityProvider } from '../providers';\n\n/**\n * Registers the AwsS3EntityProvider with the catalog processing extension point.\n *\n * @alpha\n */\nexport const catalogModuleAwsS3EntityProvider = createBackendModule({\n  pluginId: 'catalog',\n  moduleId: 'awsS3EntityProvider',\n  register(env) {\n    env.registerInit({\n      deps: {\n        config: coreServices.config,\n        catalog: catalogProcessingExtensionPoint,\n        logger: coreServices.logger,\n        scheduler: coreServices.scheduler,\n      },\n      async init({ config, catalog, logger, scheduler }) {\n        catalog.addEntityProvider(\n          AwsS3EntityProvider.fromConfig(config, {\n            logger: loggerToWinstonLogger(logger),\n            scheduler,\n          }),\n        );\n      },\n    });\n  },\n});\n"],"names":["createBackendModule","coreServices","catalogProcessingExtensionPoint","AwsS3EntityProvider","loggerToWinstonLogger"],"mappings":";;;;;;;;;;;;;;;;AA6BO,MAAM,mCAAmCA,oCAAoB,CAAA;AAAA,EAClE,QAAU,EAAA,SAAA;AAAA,EACV,QAAU,EAAA,qBAAA;AAAA,EACV,SAAS,GAAK,EAAA;AACZ,IAAA,GAAA,CAAI,YAAa,CAAA;AAAA,MACf,IAAM,EAAA;AAAA,QACJ,QAAQC,6BAAa,CAAA,MAAA;AAAA,QACrB,OAAS,EAAAC,qCAAA;AAAA,QACT,QAAQD,6BAAa,CAAA,MAAA;AAAA,QACrB,WAAWA,6BAAa,CAAA,SAAA;AAAA,OAC1B;AAAA,MACA,MAAM,IAAK,CAAA,EAAE,QAAQ,OAAS,EAAA,MAAA,EAAQ,WAAa,EAAA;AACjD,QAAQ,OAAA,CAAA,iBAAA;AAAA,UACNE,uCAAA,CAAoB,WAAW,MAAQ,EAAA;AAAA,YACrC,MAAA,EAAQC,oCAAsB,MAAM,CAAA;AAAA,YACpC,SAAA;AAAA,WACD,CAAA;AAAA,SACH,CAAA;AAAA,OACF;AAAA,KACD,CAAA,CAAA;AAAA,GACH;AACF,CAAC;;;;"}
+{"version":3,"file":"alpha.cjs.js","sources":["../src/module/catalogModuleAwsS3EntityProvider.ts"],"sourcesContent":["/*\n * Copyright 2022 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport { loggerToWinstonLogger } from '@backstage/backend-common';\nimport {\n  coreServices,\n  createBackendModule,\n} from '@backstage/backend-plugin-api';\nimport { catalogProcessingExtensionPoint } from '@backstage/plugin-catalog-node/alpha';\nimport { AwsS3EntityProvider } from '../providers';\n\n/**\n * Registers the AwsS3EntityProvider with the catalog processing extension point.\n *\n * @alpha\n */\nexport const catalogModuleAwsS3EntityProvider = createBackendModule({\n  pluginId: 'catalog',\n  moduleId: 'awsS3EntityProvider',\n  register(env) {\n    env.registerInit({\n      deps: {\n        config: coreServices.rootConfig,\n        catalog: catalogProcessingExtensionPoint,\n        logger: coreServices.logger,\n        scheduler: coreServices.scheduler,\n      },\n      async init({ config, catalog, logger, scheduler }) {\n        catalog.addEntityProvider(\n          AwsS3EntityProvider.fromConfig(config, {\n            logger: loggerToWinstonLogger(logger),\n            scheduler,\n          }),\n        );\n      },\n    });\n  },\n});\n"],"names":["createBackendModule","coreServices","catalogProcessingExtensionPoint","AwsS3EntityProvider","loggerToWinstonLogger"],"mappings":";;;;;;;;;;;;;;;;AA6BO,MAAM,mCAAmCA,oCAAoB,CAAA;AAAA,EAClE,QAAU,EAAA,SAAA;AAAA,EACV,QAAU,EAAA,qBAAA;AAAA,EACV,SAAS,GAAK,EAAA;AACZ,IAAA,GAAA,CAAI,YAAa,CAAA;AAAA,MACf,IAAM,EAAA;AAAA,QACJ,QAAQC,6BAAa,CAAA,UAAA;AAAA,QACrB,OAAS,EAAAC,qCAAA;AAAA,QACT,QAAQD,6BAAa,CAAA,MAAA;AAAA,QACrB,WAAWA,6BAAa,CAAA,SAAA;AAAA,OAC1B;AAAA,MACA,MAAM,IAAK,CAAA,EAAE,QAAQ,OAAS,EAAA,MAAA,EAAQ,WAAa,EAAA;AACjD,QAAQ,OAAA,CAAA,iBAAA;AAAA,UACNE,uCAAA,CAAoB,WAAW,MAAQ,EAAA;AAAA,YACrC,MAAA,EAAQC,oCAAsB,MAAM,CAAA;AAAA,YACpC,SAAA;AAAA,WACD,CAAA;AAAA,SACH,CAAA;AAAA,OACF;AAAA,KACD,CAAA,CAAA;AAAA,GACH;AACF,CAAC;;;;"}

package/dist/cjs/{AwsS3EntityProvider-45cf9dc1.cjs.js → AwsS3EntityProvider-8bab60be.cjs.js}

@@ -48,13 +48,15 @@ function readAwsS3Config(id, config) {
   const bucketName = config.getString("bucketName");
   const region = config.getOptionalString("region");
   const prefix = config.getOptionalString("prefix");
+  const accountId = config.getOptionalString("accountId");
   const schedule = config.has("schedule") ? backendTasks.readTaskScheduleDefinitionFromConfig(config.getConfig("schedule")) : void 0;
   return {
     id,
     bucketName,
     region,
     prefix,
-    schedule
+    schedule,
+    accountId
   };
 }
 
@@ -133,17 +135,20 @@ class AwsS3EntityProvider {
   /** {@inheritdoc @backstage/plugin-catalog-backend#EntityProvider.connect} */
   async connect(connection) {
     this.connection = connection;
-    const credProvider = await this.awsCredentialsManager.getCredentialProvider();
+    const { accountId, region, bucketName } = this.config;
+    const credProvider = await this.awsCredentialsManager.getCredentialProvider(
+      accountId ? { accountId } : void 0
+    );
     this.s3 = new clientS3.S3({
       apiVersion: "2006-03-01",
       credentialDefaultProvider: () => credProvider.sdkCredentialProvider,
       endpoint: this.integration.config.endpoint,
-      region: this.config.region,
+      region,
       forcePathStyle: this.integration.config.s3ForcePathStyle
     });
     const endpoint = await middlewareEndpoint.getEndpointFromInstructions(
       {
-        Bucket: this.config.bucketName
+        Bucket: bucketName
       },
       clientS3.ListObjectsV2Command,
       this.s3.config
@@ -208,4 +213,4 @@ class AwsS3EntityProvider {
 }
 
 exports.AwsS3EntityProvider = AwsS3EntityProvider;
-//# sourceMappingURL=AwsS3EntityProvider-45cf9dc1.cjs.js.map
+//# sourceMappingURL=AwsS3EntityProvider-8bab60be.cjs.js.map

package/dist/cjs/AwsS3EntityProvider-8bab60be.cjs.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"AwsS3EntityProvider-8bab60be.cjs.js","sources":["../../src/providers/config.ts","../../src/providers/AwsS3EntityProvider.ts"],"sourcesContent":["/*\n * Copyright 2022 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport { readTaskScheduleDefinitionFromConfig } from '@backstage/backend-tasks';\nimport { Config } from '@backstage/config';\nimport { AwsS3Config } from './types';\n\nconst DEFAULT_PROVIDER_ID = 'default';\n\nexport function readAwsS3Configs(config: Config): AwsS3Config[] {\n  const configs: AwsS3Config[] = [];\n\n  const providerConfigs = config.getOptionalConfig('catalog.providers.awsS3');\n  if (!providerConfigs) {\n    return configs;\n  }\n\n  if (providerConfigs.has('bucketName')) {\n    // simple/single config variant\n    configs.push(readAwsS3Config(DEFAULT_PROVIDER_ID, providerConfigs));\n\n    return configs;\n  }\n\n  for (const id of providerConfigs.keys()) {\n    configs.push(readAwsS3Config(id, providerConfigs.getConfig(id)));\n  }\n\n  return configs;\n}\n\nfunction readAwsS3Config(id: string, config: Config): AwsS3Config {\n  const bucketName = config.getString('bucketName');\n  const region = config.getOptionalString('region');\n  const prefix = config.getOptionalString('prefix');\n  const accountId = config.getOptionalString('accountId');\n\n  const schedule = config.has('schedule')\n    ? readTaskScheduleDefinitionFromConfig(config.getConfig('schedule'))\n    : undefined;\n\n  return {\n    id,\n    bucketName,\n    region,\n    prefix,\n    schedule,\n    accountId,\n  };\n}\n","/*\n * Copyright 2022 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport { PluginTaskScheduler, TaskRunner } from '@backstage/backend-tasks';\nimport { Config } from '@backstage/config';\nimport { AwsS3Integration, ScmIntegrations } from '@backstage/integration';\nimport {\n  EntityProvider,\n  EntityProviderConnection,\n  locationSpecToLocationEntity,\n} from '@backstage/plugin-catalog-node';\nimport { LocationSpec } from '@backstage/plugin-catalog-common';\nimport { readAwsS3Configs } from './config';\nimport { AwsS3Config } from './types';\nimport {\n  ListObjectsV2Command,\n  ListObjectsV2Output,\n  S3,\n} from '@aws-sdk/client-s3';\nimport * as uuid from 'uuid';\nimport { Logger } from 'winston';\nimport { getEndpointFromInstructions } from '@aws-sdk/middleware-endpoint';\nimport {\n  AwsCredentialsManager,\n  DefaultAwsCredentialsManager,\n} from '@backstage/integration-aws-node';\n\n// TODO: event-based updates using S3 events (+ queue like SQS)?\n/**\n * Provider which discovers catalog files (any name) within an S3 bucket.\n *\n * Use `AwsS3EntityProvider.fromConfig(...)` to create instances.\n *\n * @public\n */\nexport class AwsS3EntityProvider implements EntityProvider {\n  private readonly logger: Logger;\n  private s3?: S3;\n  private readonly scheduleFn: () => Promise<void>;\n  private connection?: EntityProviderConnection;\n  private endpoint?: string;\n\n  static fromConfig(\n    configRoot: Config,\n    options: {\n      logger: Logger;\n      schedule?: TaskRunner;\n      scheduler?: PluginTaskScheduler;\n    },\n  ): AwsS3EntityProvider[] {\n    const providerConfigs = readAwsS3Configs(configRoot);\n\n    // Even though the awsS3 integration allows a config array\n    // there is no *real* support for multiple configs.\n    // Usually, there will be just the integration for the default host.\n    // In case, a custom endpoint is used, the host from this endpoint\n    // will be extracted and used as host (e.g., localhost when used with LocalStack)\n    // and the default integration will be added as second integration.\n    // In this case, we still want the first one though, but have no means to select it\n    // just from the bucket name (and region).\n    const integration = ScmIntegrations.fromConfig(configRoot).awsS3.list()[0];\n    if (!integration) {\n      throw new Error('No integration found for awsS3');\n    }\n\n    if (!options.schedule && !options.scheduler) {\n      throw new Error('Either schedule or scheduler must be provided.');\n    }\n\n    return providerConfigs.map(providerConfig => {\n      if (!options.schedule && !providerConfig.schedule) {\n        throw new Error(\n          `No schedule provided neither via code nor config for awsS3-provider:${providerConfig.id}.`,\n        );\n      }\n      const awsCredentialsManager =\n        DefaultAwsCredentialsManager.fromConfig(configRoot);\n      const taskRunner =\n        options.schedule ??\n        options.scheduler!.createScheduledTaskRunner(providerConfig.schedule!);\n\n      return new AwsS3EntityProvider(\n        providerConfig,\n        integration,\n        awsCredentialsManager,\n        options.logger,\n        taskRunner,\n      );\n    });\n  }\n\n  private constructor(\n    private readonly config: AwsS3Config,\n    private readonly integration: AwsS3Integration,\n    private readonly awsCredentialsManager: AwsCredentialsManager,\n    logger: Logger,\n    taskRunner: TaskRunner,\n  ) {\n    this.logger = logger.child({\n      target: this.getProviderName(),\n    });\n\n    this.scheduleFn = this.createScheduleFn(taskRunner);\n  }\n\n  private createScheduleFn(taskRunner: TaskRunner): () => Promise<void> {\n    return async () => {\n      const taskId = `${this.getProviderName()}:refresh`;\n      return taskRunner.run({\n        id: taskId,\n        fn: async () => {\n          const logger = this.logger.child({\n            class: AwsS3EntityProvider.prototype.constructor.name,\n            taskId,\n            taskInstanceId: uuid.v4(),\n          });\n\n          try {\n            await this.refresh(logger);\n          } catch (error) {\n            logger.error(`${this.getProviderName()} refresh failed`, error);\n          }\n        },\n      });\n    };\n  }\n\n  /** {@inheritdoc @backstage/plugin-catalog-backend#EntityProvider.getProviderName} */\n  getProviderName(): string {\n    return `awsS3-provider:${this.config.id}`;\n  }\n\n  /** {@inheritdoc @backstage/plugin-catalog-backend#EntityProvider.connect} */\n  async connect(connection: EntityProviderConnection): Promise<void> {\n    this.connection = connection;\n    const { accountId, region, bucketName } = this.config;\n    const credProvider = await this.awsCredentialsManager.getCredentialProvider(\n      accountId ? { accountId } : undefined,\n    );\n    this.s3 = new S3({\n      apiVersion: '2006-03-01',\n      credentialDefaultProvider: () => credProvider.sdkCredentialProvider,\n      endpoint: this.integration.config.endpoint,\n      region,\n      forcePathStyle: this.integration.config.s3ForcePathStyle,\n    });\n\n    // https://github.com/aws/aws-sdk-js-v3/issues/4122#issuecomment-1298968804\n    const endpoint = await getEndpointFromInstructions(\n      {\n        Bucket: bucketName,\n      },\n      ListObjectsV2Command,\n      this.s3.config as unknown as Record<string, unknown>,\n    );\n    if (endpoint?.url)\n      this.endpoint = endpoint.url.href.endsWith('/')\n        ? endpoint.url.href\n        : `${endpoint.url.href}/`;\n    await this.scheduleFn();\n  }\n\n  async refresh(logger: Logger) {\n    if (!this.connection) {\n      throw new Error('Not initialized');\n    }\n\n    logger.info('Discovering AWS S3 objects');\n\n    const keys = await this.listAllObjectKeys();\n    logger.info(`Discovered ${keys.length} AWS S3 objects`);\n\n    const locations = keys.map(key => this.createLocationSpec(key));\n\n    await this.connection.applyMutation({\n      type: 'full',\n      entities: locations.map(location => {\n        return {\n          locationKey: this.getProviderName(),\n          entity: locationSpecToLocationEntity({ location }),\n        };\n      }),\n    });\n\n    logger.info(`Committed ${locations.length} Locations for AWS S3 objects`);\n  }\n\n  private async listAllObjectKeys(): Promise<string[]> {\n    if (!this.s3) {\n      throw new Error('Not initialized');\n    }\n\n    const keys: string[] = [];\n\n    let continuationToken: string | undefined = undefined;\n    let output: ListObjectsV2Output;\n    do {\n      output = await this.s3.listObjectsV2({\n        Bucket: this.config.bucketName,\n        ContinuationToken: continuationToken,\n        Prefix: this.config.prefix,\n      });\n\n      if (output.Contents) {\n        output.Contents.forEach(item => {\n          if (item.Key && !item.Key.endsWith('/')) {\n            keys.push(item.Key);\n          }\n        });\n      }\n      continuationToken = output.NextContinuationToken;\n    } while (continuationToken);\n\n    return keys;\n  }\n\n  private createLocationSpec(key: string): LocationSpec {\n    return {\n      type: 'url',\n      target: this.createObjectUrl(key),\n      presence: 'required',\n    };\n  }\n\n  private createObjectUrl(key: string): string {\n    return new URL(key, this.endpoint).href;\n  }\n}\n"],"names":["readTaskScheduleDefinitionFromConfig","integration","ScmIntegrations","DefaultAwsCredentialsManager","uuid","S3","getEndpointFromInstructions","ListObjectsV2Command","locationSpecToLocationEntity"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAoBA,MAAM,mBAAsB,GAAA,SAAA,CAAA;AAErB,SAAS,iBAAiB,MAA+B,EAAA;AAC9D,EAAA,MAAM,UAAyB,EAAC,CAAA;AAEhC,EAAM,MAAA,eAAA,GAAkB,MAAO,CAAA,iBAAA,CAAkB,yBAAyB,CAAA,CAAA;AAC1E,EAAA,IAAI,CAAC,eAAiB,EAAA;AACpB,IAAO,OAAA,OAAA,CAAA;AAAA,GACT;AAEA,EAAI,IAAA,eAAA,CAAgB,GAAI,CAAA,YAAY,CAAG,EAAA;AAErC,IAAA,OAAA,CAAQ,IAAK,CAAA,eAAA,CAAgB,mBAAqB,EAAA,eAAe,CAAC,CAAA,CAAA;AAElE,IAAO,OAAA,OAAA,CAAA;AAAA,GACT;AAEA,EAAW,KAAA,MAAA,EAAA,IAAM,eAAgB,CAAA,IAAA,EAAQ,EAAA;AACvC,IAAA,OAAA,CAAQ,KAAK,eAAgB,CAAA,EAAA,EAAI,gBAAgB,SAAU,CAAA,EAAE,CAAC,CAAC,CAAA,CAAA;AAAA,GACjE;AAEA,EAAO,OAAA,OAAA,CAAA;AACT,CAAA;AAEA,SAAS,eAAA,CAAgB,IAAY,MAA6B,EAAA;AAChE,EAAM,MAAA,UAAA,GAAa,MAAO,CAAA,SAAA,CAAU,YAAY,CAAA,CAAA;AAChD,EAAM,MAAA,MAAA,GAAS,MAAO,CAAA,iBAAA,CAAkB,QAAQ,CAAA,CAAA;AAChD,EAAM,MAAA,MAAA,GAAS,MAAO,CAAA,iBAAA,CAAkB,QAAQ,CAAA,CAAA;AAChD,EAAM,MAAA,SAAA,GAAY,MAAO,CAAA,iBAAA,CAAkB,WAAW,CAAA,CAAA;AAEtD,EAAM,MAAA,QAAA,GAAW,MAAO,CAAA,GAAA,CAAI,UAAU,CAAA,GAClCA,kDAAqC,MAAO,CAAA,SAAA,CAAU,UAAU,CAAC,CACjE,GAAA,KAAA,CAAA,CAAA;AAEJ,EAAO,OAAA;AAAA,IACL,EAAA;AAAA,IACA,UAAA;AAAA,IACA,MAAA;AAAA,IACA,MAAA;AAAA,IACA,QAAA;AAAA,IACA,SAAA;AAAA,GACF,CAAA;AACF;;;;;;;;ACdO,MAAM,mBAA8C,CAAA;AAAA,EAwDjD,WACW,CAAA,MAAA,EACA,WACA,EAAA,qBAAA,EACjB,QACA,UACA,EAAA;AALiB,IAAA,IAAA,CAAA,MAAA,GAAA,MAAA,CAAA;AACA,IAAA,IAAA,CAAA,WAAA,GAAA,WAAA,CAAA;AACA,IAAA,IAAA,CAAA,qBAAA,GAAA,qBAAA,CAAA;AA1DnB,IAAiB,aAAA,CAAA,IAAA,EAAA,QAAA,CAAA,CAAA;AACjB,IAAQ,aAAA,CAAA,IAAA,EAAA,IAAA,CAAA,CAAA;AACR,IAAiB,aAAA,CAAA,IAAA,EAAA,YAAA,CAAA,CAAA;AACjB,IAAQ,aAAA,CAAA,IAAA,EAAA,YAAA,CAAA,CAAA;AACR,IAAQ,aAAA,CAAA,IAAA,EAAA,UAAA,CAAA,CAAA;AA0DN,IAAK,IAAA,CAAA,MAAA,GAAS,OAAO,KAAM,CAAA;AAAA,MACzB,MAAA,EAAQ,KAAK,eAAgB,EAAA;AAAA,KAC9B,CAAA,CAAA;AAED,IAAK,IAAA,CAAA,UAAA,GAAa,IAAK,CAAA,gBAAA,CAAiB,UAAU,CAAA,CAAA;AAAA,GACpD;AAAA,EA7DA,OAAO,UACL,CAAA,UAAA,EACA,OAKuB,EAAA;AACvB,IAAM,MAAA,eAAA,GAAkB,iBAAiB,UAAU,CAAA,CAAA;AAUnD,IAAM,MAAAC,aAAA,GAAcC,4BAAgB,UAAW,CAAA,UAAU,EAAE,KAAM,CAAA,IAAA,GAAO,CAAC,CAAA,CAAA;AACzE,IAAA,IAAI,CAACD,aAAa,EAAA;AAChB,MAAM,MAAA,IAAI,MAAM,gCAAgC,CAAA,CAAA;AAAA,KAClD;AAEA,IAAA,IAAI,CAAC,OAAA,CAAQ,QAAY,IAAA,CAAC,QAAQ,SAAW,EAAA;AAC3C,MAAM,MAAA,IAAI,MAAM,gDAAgD,CAAA,CAAA;AAAA,KAClE;AAEA,IAAO,OAAA,eAAA,CAAgB,IAAI,CAAkB,cAAA,KAAA;AAlFjD,MAAA,IAAA,EAAA,CAAA;AAmFM,MAAA,IAAI,CAAC,OAAA,CAAQ,QAAY,IAAA,CAAC,eAAe,QAAU,EAAA;AACjD,QAAA,MAAM,IAAI,KAAA;AAAA,UACR,CAAA,oEAAA,EAAuE,eAAe,EAAE,CAAA,CAAA,CAAA;AAAA,SAC1F,CAAA;AAAA,OACF;AACA,MAAM,MAAA,qBAAA,GACJE,+CAA6B,CAAA,UAAA,CAAW,UAAU,CAAA,CAAA;AACpD,MAAM,MAAA,UAAA,GAAA,CACJ,aAAQ,QAAR,KAAA,IAAA,GAAA,EAAA,GACA,QAAQ,SAAW,CAAA,yBAAA,CAA0B,eAAe,QAAS,CAAA,CAAA;AAEvE,MAAA,OAAO,IAAI,mBAAA;AAAA,QACT,cAAA;AAAA,QACAF,aAAA;AAAA,QACA,qBAAA;AAAA,QACA,OAAQ,CAAA,MAAA;AAAA,QACR,UAAA;AAAA,OACF,CAAA;AAAA,KACD,CAAA,CAAA;AAAA,GACH;AAAA,EAgBQ,iBAAiB,UAA6C,EAAA;AACpE,IAAA,OAAO,YAAY;AACjB,MAAA,MAAM,MAAS,GAAA,CAAA,EAAG,IAAK,CAAA,eAAA,EAAiB,CAAA,QAAA,CAAA,CAAA;AACxC,MAAA,OAAO,WAAW,GAAI,CAAA;AAAA,QACpB,EAAI,EAAA,MAAA;AAAA,QACJ,IAAI,YAAY;AACd,UAAM,MAAA,MAAA,GAAS,IAAK,CAAA,MAAA,CAAO,KAAM,CAAA;AAAA,YAC/B,KAAA,EAAO,mBAAoB,CAAA,SAAA,CAAU,WAAY,CAAA,IAAA;AAAA,YACjD,MAAA;AAAA,YACA,cAAA,EAAgBG,gBAAK,EAAG,EAAA;AAAA,WACzB,CAAA,CAAA;AAED,UAAI,IAAA;AACF,YAAM,MAAA,IAAA,CAAK,QAAQ,MAAM,CAAA,CAAA;AAAA,mBAClB,KAAO,EAAA;AACd,YAAA,MAAA,CAAO,MAAM,CAAG,EAAA,IAAA,CAAK,eAAgB,EAAC,mBAAmB,KAAK,CAAA,CAAA;AAAA,WAChE;AAAA,SACF;AAAA,OACD,CAAA,CAAA;AAAA,KACH,CAAA;AAAA,GACF;AAAA;AAAA,EAGA,eAA0B,GAAA;AACxB,IAAO,OAAA,CAAA,eAAA,EAAkB,IAAK,CAAA,MAAA,CAAO,EAAE,CAAA,CAAA,CAAA;AAAA,GACzC;AAAA;AAAA,EAGA,MAAM,QAAQ,UAAqD,EAAA;AACjE,IAAA,IAAA,CAAK,UAAa,GAAA,UAAA,CAAA;AAClB,IAAA,MAAM,EAAE,SAAA,EAAW,MAAQ,EAAA,UAAA,KAAe,IAAK,CAAA,MAAA,CAAA;AAC/C,IAAM,MAAA,YAAA,GAAe,MAAM,IAAA,CAAK,qBAAsB,CAAA,qBAAA;AAAA,MACpD,SAAA,GAAY,EAAE,SAAA,EAAc,GAAA,KAAA,CAAA;AAAA,KAC9B,CAAA;AACA,IAAK,IAAA,CAAA,EAAA,GAAK,IAAIC,WAAG,CAAA;AAAA,MACf,UAAY,EAAA,YAAA;AAAA,MACZ,yBAAA,EAA2B,MAAM,YAAa,CAAA,qBAAA;AAAA,MAC9C,QAAA,EAAU,IAAK,CAAA,WAAA,CAAY,MAAO,CAAA,QAAA;AAAA,MAClC,MAAA;AAAA,MACA,cAAA,EAAgB,IAAK,CAAA,WAAA,CAAY,MAAO,CAAA,gBAAA;AAAA,KACzC,CAAA,CAAA;AAGD,IAAA,MAAM,WAAW,MAAMC,8CAAA;AAAA,MACrB;AAAA,QACE,MAAQ,EAAA,UAAA;AAAA,OACV;AAAA,MACAC,6BAAA;AAAA,MACA,KAAK,EAAG,CAAA,MAAA;AAAA,KACV,CAAA;AACA,IAAA,IAAI,QAAU,IAAA,IAAA,GAAA,KAAA,CAAA,GAAA,QAAA,CAAA,GAAA;AACZ,MAAA,IAAA,CAAK,QAAW,GAAA,QAAA,CAAS,GAAI,CAAA,IAAA,CAAK,QAAS,CAAA,GAAG,CAC1C,GAAA,QAAA,CAAS,GAAI,CAAA,IAAA,GACb,CAAG,EAAA,QAAA,CAAS,IAAI,IAAI,CAAA,CAAA,CAAA,CAAA;AAC1B,IAAA,MAAM,KAAK,UAAW,EAAA,CAAA;AAAA,GACxB;AAAA,EAEA,MAAM,QAAQ,MAAgB,EAAA;AAC5B,IAAI,IAAA,CAAC,KAAK,UAAY,EAAA;AACpB,MAAM,MAAA,IAAI,MAAM,iBAAiB,CAAA,CAAA;AAAA,KACnC;AAEA,IAAA,MAAA,CAAO,KAAK,4BAA4B,CAAA,CAAA;AAExC,IAAM,MAAA,IAAA,GAAO,MAAM,IAAA,CAAK,iBAAkB,EAAA,CAAA;AAC1C,IAAA,MAAA,CAAO,IAAK,CAAA,CAAA,WAAA,EAAc,IAAK,CAAA,MAAM,CAAiB,eAAA,CAAA,CAAA,CAAA;AAEtD,IAAA,MAAM,YAAY,IAAK,CAAA,GAAA,CAAI,SAAO,IAAK,CAAA,kBAAA,CAAmB,GAAG,CAAC,CAAA,CAAA;AAE9D,IAAM,MAAA,IAAA,CAAK,WAAW,aAAc,CAAA;AAAA,MAClC,IAAM,EAAA,MAAA;AAAA,MACN,QAAA,EAAU,SAAU,CAAA,GAAA,CAAI,CAAY,QAAA,KAAA;AAClC,QAAO,OAAA;AAAA,UACL,WAAA,EAAa,KAAK,eAAgB,EAAA;AAAA,UAClC,MAAQ,EAAAC,8CAAA,CAA6B,EAAE,QAAA,EAAU,CAAA;AAAA,SACnD,CAAA;AAAA,OACD,CAAA;AAAA,KACF,CAAA,CAAA;AAED,IAAA,MAAA,CAAO,IAAK,CAAA,CAAA,UAAA,EAAa,SAAU,CAAA,MAAM,CAA+B,6BAAA,CAAA,CAAA,CAAA;AAAA,GAC1E;AAAA,EAEA,MAAc,iBAAuC,GAAA;AACnD,IAAI,IAAA,CAAC,KAAK,EAAI,EAAA;AACZ,MAAM,MAAA,IAAI,MAAM,iBAAiB,CAAA,CAAA;AAAA,KACnC;AAEA,IAAA,MAAM,OAAiB,EAAC,CAAA;AAExB,IAAA,IAAI,iBAAwC,GAAA,KAAA,CAAA,CAAA;AAC5C,IAAI,IAAA,MAAA,CAAA;AACJ,IAAG,GAAA;AACD,MAAS,MAAA,GAAA,MAAM,IAAK,CAAA,EAAA,CAAG,aAAc,CAAA;AAAA,QACnC,MAAA,EAAQ,KAAK,MAAO,CAAA,UAAA;AAAA,QACpB,iBAAmB,EAAA,iBAAA;AAAA,QACnB,MAAA,EAAQ,KAAK,MAAO,CAAA,MAAA;AAAA,OACrB,CAAA,CAAA;AAED,MAAA,IAAI,OAAO,QAAU,EAAA;AACnB,QAAO,MAAA,CAAA,QAAA,CAAS,QAAQ,CAAQ,IAAA,KAAA;AAC9B,UAAA,IAAI,KAAK,GAAO,IAAA,CAAC,KAAK,GAAI,CAAA,QAAA,CAAS,GAAG,CAAG,EAAA;AACvC,YAAK,IAAA,CAAA,IAAA,CAAK,KAAK,GAAG,CAAA,CAAA;AAAA,WACpB;AAAA,SACD,CAAA,CAAA;AAAA,OACH;AACA,MAAA,iBAAA,GAAoB,MAAO,CAAA,qBAAA,CAAA;AAAA,KACpB,QAAA,iBAAA,EAAA;AAET,IAAO,OAAA,IAAA,CAAA;AAAA,GACT;AAAA,EAEQ,mBAAmB,GAA2B,EAAA;AACpD,IAAO,OAAA;AAAA,MACL,IAAM,EAAA,KAAA;AAAA,MACN,MAAA,EAAQ,IAAK,CAAA,eAAA,CAAgB,GAAG,CAAA;AAAA,MAChC,QAAU,EAAA,UAAA;AAAA,KACZ,CAAA;AAAA,GACF;AAAA,EAEQ,gBAAgB,GAAqB,EAAA;AAC3C,IAAA,OAAO,IAAI,GAAA,CAAI,GAAK,EAAA,IAAA,CAAK,QAAQ,CAAE,CAAA,IAAA,CAAA;AAAA,GACrC;AACF;;;;"}

package/dist/index.cjs.js

@@ -9,7 +9,7 @@ var pluginCatalogNode = require('@backstage/plugin-catalog-node');
 var clientOrganizations = require('@aws-sdk/client-organizations');
 var errors = require('@backstage/errors');
 var limiterFactory = require('p-limit');
-var AwsS3EntityProvider = require('./cjs/AwsS3EntityProvider-45cf9dc1.cjs.js');
+var AwsS3EntityProvider = require('./cjs/AwsS3EntityProvider-8bab60be.cjs.js');
 require('@backstage/integration');
 require('@backstage/backend-tasks');
 require('@aws-sdk/client-s3');

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@backstage/plugin-catalog-backend-module-aws",
   "description": "A Backstage catalog backend module that helps integrate towards AWS",
-  "version": "0.2.2",
+  "version": "0.2.3-next.1",
   "main": "./dist/index.cjs.js",
   "types": "./dist/index.d.ts",
   "license": "Apache-2.0",
@@ -49,16 +49,16 @@
     "@aws-sdk/credential-providers": "^3.350.0",
     "@aws-sdk/middleware-endpoint": "^3.347.0",
     "@aws-sdk/types": "^3.347.0",
-    "@backstage/backend-common": "^0.19.1",
-    "@backstage/backend-plugin-api": "^0.5.4",
-    "@backstage/backend-tasks": "^0.5.4",
+    "@backstage/backend-common": "^0.19.2-next.1",
+    "@backstage/backend-plugin-api": "^0.6.0-next.1",
+    "@backstage/backend-tasks": "^0.5.5-next.1",
     "@backstage/catalog-model": "^1.4.1",
     "@backstage/config": "^1.0.8",
     "@backstage/errors": "^1.2.1",
     "@backstage/integration": "^1.5.1",
     "@backstage/integration-aws-node": "^0.1.5",
     "@backstage/plugin-catalog-common": "^1.0.15",
-    "@backstage/plugin-catalog-node": "^1.4.0",
+    "@backstage/plugin-catalog-node": "^1.4.1-next.1",
     "@backstage/plugin-kubernetes-common": "^0.6.5",
     "@backstage/types": "^1.1.0",
     "lodash": "^4.17.21",
@@ -68,8 +68,8 @@
   },
   "devDependencies": {
     "@aws-sdk/util-stream-node": "^3.350.0",
-    "@backstage/backend-test-utils": "^0.1.39",
-    "@backstage/cli": "^0.22.9",
+    "@backstage/backend-test-utils": "^0.2.0-next.1",
+    "@backstage/cli": "^0.22.10-next.1",
     "@types/lodash": "^4.14.151",
     "aws-sdk-client-mock": "^2.0.0",
     "aws-sdk-client-mock-jest": "^2.0.0",

package/dist/cjs/AwsS3EntityProvider-45cf9dc1.cjs.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"AwsS3EntityProvider-45cf9dc1.cjs.js","sources":["../../src/providers/config.ts","../../src/providers/AwsS3EntityProvider.ts"],"sourcesContent":["/*\n * Copyright 2022 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport { readTaskScheduleDefinitionFromConfig } from '@backstage/backend-tasks';\nimport { Config } from '@backstage/config';\nimport { AwsS3Config } from './types';\n\nconst DEFAULT_PROVIDER_ID = 'default';\n\nexport function readAwsS3Configs(config: Config): AwsS3Config[] {\n  const configs: AwsS3Config[] = [];\n\n  const providerConfigs = config.getOptionalConfig('catalog.providers.awsS3');\n  if (!providerConfigs) {\n    return configs;\n  }\n\n  if (providerConfigs.has('bucketName')) {\n    // simple/single config variant\n    configs.push(readAwsS3Config(DEFAULT_PROVIDER_ID, providerConfigs));\n\n    return configs;\n  }\n\n  for (const id of providerConfigs.keys()) {\n    configs.push(readAwsS3Config(id, providerConfigs.getConfig(id)));\n  }\n\n  return configs;\n}\n\nfunction readAwsS3Config(id: string, config: Config): AwsS3Config {\n  const bucketName = config.getString('bucketName');\n  const region = config.getOptionalString('region');\n  const prefix = config.getOptionalString('prefix');\n\n  const schedule = config.has('schedule')\n    ? readTaskScheduleDefinitionFromConfig(config.getConfig('schedule'))\n    : undefined;\n\n  return {\n    id,\n    bucketName,\n    region,\n    prefix,\n    schedule,\n  };\n}\n","/*\n * Copyright 2022 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport { PluginTaskScheduler, TaskRunner } from '@backstage/backend-tasks';\nimport { Config } from '@backstage/config';\nimport { AwsS3Integration, ScmIntegrations } from '@backstage/integration';\nimport {\n  EntityProvider,\n  EntityProviderConnection,\n  locationSpecToLocationEntity,\n} from '@backstage/plugin-catalog-node';\nimport { LocationSpec } from '@backstage/plugin-catalog-common';\nimport { readAwsS3Configs } from './config';\nimport { AwsS3Config } from './types';\nimport {\n  ListObjectsV2Command,\n  ListObjectsV2Output,\n  S3,\n} from '@aws-sdk/client-s3';\nimport * as uuid from 'uuid';\nimport { Logger } from 'winston';\nimport { getEndpointFromInstructions } from '@aws-sdk/middleware-endpoint';\nimport {\n  AwsCredentialsManager,\n  DefaultAwsCredentialsManager,\n} from '@backstage/integration-aws-node';\n\n// TODO: event-based updates using S3 events (+ queue like SQS)?\n/**\n * Provider which discovers catalog files (any name) within an S3 bucket.\n *\n * Use `AwsS3EntityProvider.fromConfig(...)` to create instances.\n *\n * @public\n */\nexport class AwsS3EntityProvider implements EntityProvider {\n  private readonly logger: Logger;\n  private s3?: S3;\n  private readonly scheduleFn: () => Promise<void>;\n  private connection?: EntityProviderConnection;\n  private endpoint?: string;\n\n  static fromConfig(\n    configRoot: Config,\n    options: {\n      logger: Logger;\n      schedule?: TaskRunner;\n      scheduler?: PluginTaskScheduler;\n    },\n  ): AwsS3EntityProvider[] {\n    const providerConfigs = readAwsS3Configs(configRoot);\n\n    // Even though the awsS3 integration allows a config array\n    // there is no *real* support for multiple configs.\n    // Usually, there will be just the integration for the default host.\n    // In case, a custom endpoint is used, the host from this endpoint\n    // will be extracted and used as host (e.g., localhost when used with LocalStack)\n    // and the default integration will be added as second integration.\n    // In this case, we still want the first one though, but have no means to select it\n    // just from the bucket name (and region).\n    const integration = ScmIntegrations.fromConfig(configRoot).awsS3.list()[0];\n    if (!integration) {\n      throw new Error('No integration found for awsS3');\n    }\n\n    if (!options.schedule && !options.scheduler) {\n      throw new Error('Either schedule or scheduler must be provided.');\n    }\n\n    return providerConfigs.map(providerConfig => {\n      if (!options.schedule && !providerConfig.schedule) {\n        throw new Error(\n          `No schedule provided neither via code nor config for awsS3-provider:${providerConfig.id}.`,\n        );\n      }\n      const awsCredentialsManager =\n        DefaultAwsCredentialsManager.fromConfig(configRoot);\n      const taskRunner =\n        options.schedule ??\n        options.scheduler!.createScheduledTaskRunner(providerConfig.schedule!);\n\n      return new AwsS3EntityProvider(\n        providerConfig,\n        integration,\n        awsCredentialsManager,\n        options.logger,\n        taskRunner,\n      );\n    });\n  }\n\n  private constructor(\n    private readonly config: AwsS3Config,\n    private readonly integration: AwsS3Integration,\n    private readonly awsCredentialsManager: AwsCredentialsManager,\n    logger: Logger,\n    taskRunner: TaskRunner,\n  ) {\n    this.logger = logger.child({\n      target: this.getProviderName(),\n    });\n\n    this.scheduleFn = this.createScheduleFn(taskRunner);\n  }\n\n  private createScheduleFn(taskRunner: TaskRunner): () => Promise<void> {\n    return async () => {\n      const taskId = `${this.getProviderName()}:refresh`;\n      return taskRunner.run({\n        id: taskId,\n        fn: async () => {\n          const logger = this.logger.child({\n            class: AwsS3EntityProvider.prototype.constructor.name,\n            taskId,\n            taskInstanceId: uuid.v4(),\n          });\n\n          try {\n            await this.refresh(logger);\n          } catch (error) {\n            logger.error(`${this.getProviderName()} refresh failed`, error);\n          }\n        },\n      });\n    };\n  }\n\n  /** {@inheritdoc @backstage/plugin-catalog-backend#EntityProvider.getProviderName} */\n  getProviderName(): string {\n    return `awsS3-provider:${this.config.id}`;\n  }\n\n  /** {@inheritdoc @backstage/plugin-catalog-backend#EntityProvider.connect} */\n  async connect(connection: EntityProviderConnection): Promise<void> {\n    this.connection = connection;\n    const credProvider =\n      await this.awsCredentialsManager.getCredentialProvider();\n    this.s3 = new S3({\n      apiVersion: '2006-03-01',\n      credentialDefaultProvider: () => credProvider.sdkCredentialProvider,\n      endpoint: this.integration.config.endpoint,\n      region: this.config.region,\n      forcePathStyle: this.integration.config.s3ForcePathStyle,\n    });\n\n    // https://github.com/aws/aws-sdk-js-v3/issues/4122#issuecomment-1298968804\n    const endpoint = await getEndpointFromInstructions(\n      {\n        Bucket: this.config.bucketName,\n      },\n      ListObjectsV2Command,\n      this.s3.config as unknown as Record<string, unknown>,\n    );\n    if (endpoint?.url)\n      this.endpoint = endpoint.url.href.endsWith('/')\n        ? endpoint.url.href\n        : `${endpoint.url.href}/`;\n    await this.scheduleFn();\n  }\n\n  async refresh(logger: Logger) {\n    if (!this.connection) {\n      throw new Error('Not initialized');\n    }\n\n    logger.info('Discovering AWS S3 objects');\n\n    const keys = await this.listAllObjectKeys();\n    logger.info(`Discovered ${keys.length} AWS S3 objects`);\n\n    const locations = keys.map(key => this.createLocationSpec(key));\n\n    await this.connection.applyMutation({\n      type: 'full',\n      entities: locations.map(location => {\n        return {\n          locationKey: this.getProviderName(),\n          entity: locationSpecToLocationEntity({ location }),\n        };\n      }),\n    });\n\n    logger.info(`Committed ${locations.length} Locations for AWS S3 objects`);\n  }\n\n  private async listAllObjectKeys(): Promise<string[]> {\n    if (!this.s3) {\n      throw new Error('Not initialized');\n    }\n\n    const keys: string[] = [];\n\n    let continuationToken: string | undefined = undefined;\n    let output: ListObjectsV2Output;\n    do {\n      output = await this.s3.listObjectsV2({\n        Bucket: this.config.bucketName,\n        ContinuationToken: continuationToken,\n        Prefix: this.config.prefix,\n      });\n\n      if (output.Contents) {\n        output.Contents.forEach(item => {\n          if (item.Key && !item.Key.endsWith('/')) {\n            keys.push(item.Key);\n          }\n        });\n      }\n      continuationToken = output.NextContinuationToken;\n    } while (continuationToken);\n\n    return keys;\n  }\n\n  private createLocationSpec(key: string): LocationSpec {\n    return {\n      type: 'url',\n      target: this.createObjectUrl(key),\n      presence: 'required',\n    };\n  }\n\n  private createObjectUrl(key: string): string {\n    return new URL(key, this.endpoint).href;\n  }\n}\n"],"names":["readTaskScheduleDefinitionFromConfig","integration","ScmIntegrations","DefaultAwsCredentialsManager","uuid","S3","getEndpointFromInstructions","ListObjectsV2Command","locationSpecToLocationEntity"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAoBA,MAAM,mBAAsB,GAAA,SAAA,CAAA;AAErB,SAAS,iBAAiB,MAA+B,EAAA;AAC9D,EAAA,MAAM,UAAyB,EAAC,CAAA;AAEhC,EAAM,MAAA,eAAA,GAAkB,MAAO,CAAA,iBAAA,CAAkB,yBAAyB,CAAA,CAAA;AAC1E,EAAA,IAAI,CAAC,eAAiB,EAAA;AACpB,IAAO,OAAA,OAAA,CAAA;AAAA,GACT;AAEA,EAAI,IAAA,eAAA,CAAgB,GAAI,CAAA,YAAY,CAAG,EAAA;AAErC,IAAA,OAAA,CAAQ,IAAK,CAAA,eAAA,CAAgB,mBAAqB,EAAA,eAAe,CAAC,CAAA,CAAA;AAElE,IAAO,OAAA,OAAA,CAAA;AAAA,GACT;AAEA,EAAW,KAAA,MAAA,EAAA,IAAM,eAAgB,CAAA,IAAA,EAAQ,EAAA;AACvC,IAAA,OAAA,CAAQ,KAAK,eAAgB,CAAA,EAAA,EAAI,gBAAgB,SAAU,CAAA,EAAE,CAAC,CAAC,CAAA,CAAA;AAAA,GACjE;AAEA,EAAO,OAAA,OAAA,CAAA;AACT,CAAA;AAEA,SAAS,eAAA,CAAgB,IAAY,MAA6B,EAAA;AAChE,EAAM,MAAA,UAAA,GAAa,MAAO,CAAA,SAAA,CAAU,YAAY,CAAA,CAAA;AAChD,EAAM,MAAA,MAAA,GAAS,MAAO,CAAA,iBAAA,CAAkB,QAAQ,CAAA,CAAA;AAChD,EAAM,MAAA,MAAA,GAAS,MAAO,CAAA,iBAAA,CAAkB,QAAQ,CAAA,CAAA;AAEhD,EAAM,MAAA,QAAA,GAAW,MAAO,CAAA,GAAA,CAAI,UAAU,CAAA,GAClCA,kDAAqC,MAAO,CAAA,SAAA,CAAU,UAAU,CAAC,CACjE,GAAA,KAAA,CAAA,CAAA;AAEJ,EAAO,OAAA;AAAA,IACL,EAAA;AAAA,IACA,UAAA;AAAA,IACA,MAAA;AAAA,IACA,MAAA;AAAA,IACA,QAAA;AAAA,GACF,CAAA;AACF;;;;;;;;ACZO,MAAM,mBAA8C,CAAA;AAAA,EAwDjD,WACW,CAAA,MAAA,EACA,WACA,EAAA,qBAAA,EACjB,QACA,UACA,EAAA;AALiB,IAAA,IAAA,CAAA,MAAA,GAAA,MAAA,CAAA;AACA,IAAA,IAAA,CAAA,WAAA,GAAA,WAAA,CAAA;AACA,IAAA,IAAA,CAAA,qBAAA,GAAA,qBAAA,CAAA;AA1DnB,IAAiB,aAAA,CAAA,IAAA,EAAA,QAAA,CAAA,CAAA;AACjB,IAAQ,aAAA,CAAA,IAAA,EAAA,IAAA,CAAA,CAAA;AACR,IAAiB,aAAA,CAAA,IAAA,EAAA,YAAA,CAAA,CAAA;AACjB,IAAQ,aAAA,CAAA,IAAA,EAAA,YAAA,CAAA,CAAA;AACR,IAAQ,aAAA,CAAA,IAAA,EAAA,UAAA,CAAA,CAAA;AA0DN,IAAK,IAAA,CAAA,MAAA,GAAS,OAAO,KAAM,CAAA;AAAA,MACzB,MAAA,EAAQ,KAAK,eAAgB,EAAA;AAAA,KAC9B,CAAA,CAAA;AAED,IAAK,IAAA,CAAA,UAAA,GAAa,IAAK,CAAA,gBAAA,CAAiB,UAAU,CAAA,CAAA;AAAA,GACpD;AAAA,EA7DA,OAAO,UACL,CAAA,UAAA,EACA,OAKuB,EAAA;AACvB,IAAM,MAAA,eAAA,GAAkB,iBAAiB,UAAU,CAAA,CAAA;AAUnD,IAAM,MAAAC,aAAA,GAAcC,4BAAgB,UAAW,CAAA,UAAU,EAAE,KAAM,CAAA,IAAA,GAAO,CAAC,CAAA,CAAA;AACzE,IAAA,IAAI,CAACD,aAAa,EAAA;AAChB,MAAM,MAAA,IAAI,MAAM,gCAAgC,CAAA,CAAA;AAAA,KAClD;AAEA,IAAA,IAAI,CAAC,OAAA,CAAQ,QAAY,IAAA,CAAC,QAAQ,SAAW,EAAA;AAC3C,MAAM,MAAA,IAAI,MAAM,gDAAgD,CAAA,CAAA;AAAA,KAClE;AAEA,IAAO,OAAA,eAAA,CAAgB,IAAI,CAAkB,cAAA,KAAA;AAlFjD,MAAA,IAAA,EAAA,CAAA;AAmFM,MAAA,IAAI,CAAC,OAAA,CAAQ,QAAY,IAAA,CAAC,eAAe,QAAU,EAAA;AACjD,QAAA,MAAM,IAAI,KAAA;AAAA,UACR,CAAA,oEAAA,EAAuE,eAAe,EAAE,CAAA,CAAA,CAAA;AAAA,SAC1F,CAAA;AAAA,OACF;AACA,MAAM,MAAA,qBAAA,GACJE,+CAA6B,CAAA,UAAA,CAAW,UAAU,CAAA,CAAA;AACpD,MAAM,MAAA,UAAA,GAAA,CACJ,aAAQ,QAAR,KAAA,IAAA,GAAA,EAAA,GACA,QAAQ,SAAW,CAAA,yBAAA,CAA0B,eAAe,QAAS,CAAA,CAAA;AAEvE,MAAA,OAAO,IAAI,mBAAA;AAAA,QACT,cAAA;AAAA,QACAF,aAAA;AAAA,QACA,qBAAA;AAAA,QACA,OAAQ,CAAA,MAAA;AAAA,QACR,UAAA;AAAA,OACF,CAAA;AAAA,KACD,CAAA,CAAA;AAAA,GACH;AAAA,EAgBQ,iBAAiB,UAA6C,EAAA;AACpE,IAAA,OAAO,YAAY;AACjB,MAAA,MAAM,MAAS,GAAA,CAAA,EAAG,IAAK,CAAA,eAAA,EAAiB,CAAA,QAAA,CAAA,CAAA;AACxC,MAAA,OAAO,WAAW,GAAI,CAAA;AAAA,QACpB,EAAI,EAAA,MAAA;AAAA,QACJ,IAAI,YAAY;AACd,UAAM,MAAA,MAAA,GAAS,IAAK,CAAA,MAAA,CAAO,KAAM,CAAA;AAAA,YAC/B,KAAA,EAAO,mBAAoB,CAAA,SAAA,CAAU,WAAY,CAAA,IAAA;AAAA,YACjD,MAAA;AAAA,YACA,cAAA,EAAgBG,gBAAK,EAAG,EAAA;AAAA,WACzB,CAAA,CAAA;AAED,UAAI,IAAA;AACF,YAAM,MAAA,IAAA,CAAK,QAAQ,MAAM,CAAA,CAAA;AAAA,mBAClB,KAAO,EAAA;AACd,YAAA,MAAA,CAAO,MAAM,CAAG,EAAA,IAAA,CAAK,eAAgB,EAAC,mBAAmB,KAAK,CAAA,CAAA;AAAA,WAChE;AAAA,SACF;AAAA,OACD,CAAA,CAAA;AAAA,KACH,CAAA;AAAA,GACF;AAAA;AAAA,EAGA,eAA0B,GAAA;AACxB,IAAO,OAAA,CAAA,eAAA,EAAkB,IAAK,CAAA,MAAA,CAAO,EAAE,CAAA,CAAA,CAAA;AAAA,GACzC;AAAA;AAAA,EAGA,MAAM,QAAQ,UAAqD,EAAA;AACjE,IAAA,IAAA,CAAK,UAAa,GAAA,UAAA,CAAA;AAClB,IAAA,MAAM,YACJ,GAAA,MAAM,IAAK,CAAA,qBAAA,CAAsB,qBAAsB,EAAA,CAAA;AACzD,IAAK,IAAA,CAAA,EAAA,GAAK,IAAIC,WAAG,CAAA;AAAA,MACf,UAAY,EAAA,YAAA;AAAA,MACZ,yBAAA,EAA2B,MAAM,YAAa,CAAA,qBAAA;AAAA,MAC9C,QAAA,EAAU,IAAK,CAAA,WAAA,CAAY,MAAO,CAAA,QAAA;AAAA,MAClC,MAAA,EAAQ,KAAK,MAAO,CAAA,MAAA;AAAA,MACpB,cAAA,EAAgB,IAAK,CAAA,WAAA,CAAY,MAAO,CAAA,gBAAA;AAAA,KACzC,CAAA,CAAA;AAGD,IAAA,MAAM,WAAW,MAAMC,8CAAA;AAAA,MACrB;AAAA,QACE,MAAA,EAAQ,KAAK,MAAO,CAAA,UAAA;AAAA,OACtB;AAAA,MACAC,6BAAA;AAAA,MACA,KAAK,EAAG,CAAA,MAAA;AAAA,KACV,CAAA;AACA,IAAA,IAAI,QAAU,IAAA,IAAA,GAAA,KAAA,CAAA,GAAA,QAAA,CAAA,GAAA;AACZ,MAAA,IAAA,CAAK,QAAW,GAAA,QAAA,CAAS,GAAI,CAAA,IAAA,CAAK,QAAS,CAAA,GAAG,CAC1C,GAAA,QAAA,CAAS,GAAI,CAAA,IAAA,GACb,CAAG,EAAA,QAAA,CAAS,IAAI,IAAI,CAAA,CAAA,CAAA,CAAA;AAC1B,IAAA,MAAM,KAAK,UAAW,EAAA,CAAA;AAAA,GACxB;AAAA,EAEA,MAAM,QAAQ,MAAgB,EAAA;AAC5B,IAAI,IAAA,CAAC,KAAK,UAAY,EAAA;AACpB,MAAM,MAAA,IAAI,MAAM,iBAAiB,CAAA,CAAA;AAAA,KACnC;AAEA,IAAA,MAAA,CAAO,KAAK,4BAA4B,CAAA,CAAA;AAExC,IAAM,MAAA,IAAA,GAAO,MAAM,IAAA,CAAK,iBAAkB,EAAA,CAAA;AAC1C,IAAA,MAAA,CAAO,IAAK,CAAA,CAAA,WAAA,EAAc,IAAK,CAAA,MAAM,CAAiB,eAAA,CAAA,CAAA,CAAA;AAEtD,IAAA,MAAM,YAAY,IAAK,CAAA,GAAA,CAAI,SAAO,IAAK,CAAA,kBAAA,CAAmB,GAAG,CAAC,CAAA,CAAA;AAE9D,IAAM,MAAA,IAAA,CAAK,WAAW,aAAc,CAAA;AAAA,MAClC,IAAM,EAAA,MAAA;AAAA,MACN,QAAA,EAAU,SAAU,CAAA,GAAA,CAAI,CAAY,QAAA,KAAA;AAClC,QAAO,OAAA;AAAA,UACL,WAAA,EAAa,KAAK,eAAgB,EAAA;AAAA,UAClC,MAAQ,EAAAC,8CAAA,CAA6B,EAAE,QAAA,EAAU,CAAA;AAAA,SACnD,CAAA;AAAA,OACD,CAAA;AAAA,KACF,CAAA,CAAA;AAED,IAAA,MAAA,CAAO,IAAK,CAAA,CAAA,UAAA,EAAa,SAAU,CAAA,MAAM,CAA+B,6BAAA,CAAA,CAAA,CAAA;AAAA,GAC1E;AAAA,EAEA,MAAc,iBAAuC,GAAA;AACnD,IAAI,IAAA,CAAC,KAAK,EAAI,EAAA;AACZ,MAAM,MAAA,IAAI,MAAM,iBAAiB,CAAA,CAAA;AAAA,KACnC;AAEA,IAAA,MAAM,OAAiB,EAAC,CAAA;AAExB,IAAA,IAAI,iBAAwC,GAAA,KAAA,CAAA,CAAA;AAC5C,IAAI,IAAA,MAAA,CAAA;AACJ,IAAG,GAAA;AACD,MAAS,MAAA,GAAA,MAAM,IAAK,CAAA,EAAA,CAAG,aAAc,CAAA;AAAA,QACnC,MAAA,EAAQ,KAAK,MAAO,CAAA,UAAA;AAAA,QACpB,iBAAmB,EAAA,iBAAA;AAAA,QACnB,MAAA,EAAQ,KAAK,MAAO,CAAA,MAAA;AAAA,OACrB,CAAA,CAAA;AAED,MAAA,IAAI,OAAO,QAAU,EAAA;AACnB,QAAO,MAAA,CAAA,QAAA,CAAS,QAAQ,CAAQ,IAAA,KAAA;AAC9B,UAAA,IAAI,KAAK,GAAO,IAAA,CAAC,KAAK,GAAI,CAAA,QAAA,CAAS,GAAG,CAAG,EAAA;AACvC,YAAK,IAAA,CAAA,IAAA,CAAK,KAAK,GAAG,CAAA,CAAA;AAAA,WACpB;AAAA,SACD,CAAA,CAAA;AAAA,OACH;AACA,MAAA,iBAAA,GAAoB,MAAO,CAAA,qBAAA,CAAA;AAAA,KACpB,QAAA,iBAAA,EAAA;AAET,IAAO,OAAA,IAAA,CAAA;AAAA,GACT;AAAA,EAEQ,mBAAmB,GAA2B,EAAA;AACpD,IAAO,OAAA;AAAA,MACL,IAAM,EAAA,KAAA;AAAA,MACN,MAAA,EAAQ,IAAK,CAAA,eAAA,CAAgB,GAAG,CAAA;AAAA,MAChC,QAAU,EAAA,UAAA;AAAA,KACZ,CAAA;AAAA,GACF;AAAA,EAEQ,gBAAgB,GAAqB,EAAA;AAC3C,IAAA,OAAO,IAAI,GAAA,CAAI,GAAK,EAAA,IAAA,CAAK,QAAQ,CAAE,CAAA,IAAA,CAAA;AAAA,GACrC;AACF;;;;"}