@backstage/plugin-catalog-backend-module-aws 0.1.1 → 0.1.3

package/CHANGELOG.md

@@ -1,5 +1,37 @@
 # @backstage/plugin-catalog-backend-module-aws
 
+## 0.1.3
+
+### Patch Changes
+
+- Updated dependencies
+  - @backstage/plugin-catalog-backend@1.0.0
+  - @backstage/backend-common@0.13.1
+  - @backstage/catalog-model@1.0.0
+  - @backstage/config@1.0.0
+  - @backstage/errors@1.0.0
+  - @backstage/types@1.0.0
+
+## 0.1.2
+
+### Patch Changes
+
+- f115a7f8fd: Added `AwsS3DiscoveryProcessor`, which was moved here from `@backstage/plugin-catalog-backend` where it previously resided.
+- Updated dependencies
+  - @backstage/backend-common@0.13.0
+  - @backstage/plugin-catalog-backend@0.24.0
+  - @backstage/catalog-model@0.13.0
+
+## 0.1.2-next.0
+
+### Patch Changes
+
+- f115a7f8fd: Added `AwsS3DiscoveryProcessor`, which was moved here from `@backstage/plugin-catalog-backend` where it previously resided.
+- Updated dependencies
+  - @backstage/backend-common@0.13.0-next.0
+  - @backstage/plugin-catalog-backend@0.24.0-next.0
+  - @backstage/catalog-model@0.13.0-next.0
+
 ## 0.1.1
 
 ### Patch Changes

package/dist/index.cjs.js

@@ -4,10 +4,13 @@ Object.defineProperty(exports, '__esModule', { value: true });
 
 var pluginCatalogBackend = require('@backstage/plugin-catalog-backend');
 var AWS = require('aws-sdk');
+var errors = require('@backstage/errors');
+var limiterFactory = require('p-limit');
 
 function _interopDefaultLegacy (e) { return e && typeof e === 'object' && 'default' in e ? e : { 'default': e }; }
 
 var AWS__default = /*#__PURE__*/_interopDefaultLegacy(AWS);
+var limiterFactory__default = /*#__PURE__*/_interopDefaultLegacy(limiterFactory);
 
 function readAwsOrganizationConfig(config) {
   const providerConfig = config.getOptionalConfig("provider");
@@ -116,5 +119,51 @@ class AwsOrganizationCloudAccountProcessor {
   }
 }
 
+class AwsS3DiscoveryProcessor {
+  constructor(reader) {
+    this.reader = reader;
+  }
+  getProcessorName() {
+    return "AwsS3DiscoveryProcessor";
+  }
+  async readLocation(location, optional, emit, parser) {
+    if (location.type !== "s3-discovery") {
+      return false;
+    }
+    try {
+      const output = await this.doRead(location.target);
+      for (const item of output) {
+        for await (const parseResult of parser({
+          data: item.data,
+          location: { type: location.type, target: item.url }
+        })) {
+          emit(parseResult);
+        }
+      }
+    } catch (error) {
+      const message = `Unable to read ${location.type}, ${error}`;
+      if (errors.isError(error) && error.name === "NotFoundError") {
+        if (!optional) {
+          emit(pluginCatalogBackend.processingResult.notFoundError(location, message));
+        }
+      } else {
+        emit(pluginCatalogBackend.processingResult.generalError(location, message));
+      }
+    }
+    return true;
+  }
+  async doRead(location) {
+    const limiter = limiterFactory__default["default"](5);
+    const response = await this.reader.readTree(location);
+    const responseFiles = await response.files();
+    const output = responseFiles.map(async (file) => ({
+      url: file.path,
+      data: await limiter(file.content)
+    }));
+    return Promise.all(output);
+  }
+}
+
 exports.AwsOrganizationCloudAccountProcessor = AwsOrganizationCloudAccountProcessor;
+exports.AwsS3DiscoveryProcessor = AwsS3DiscoveryProcessor;
 //# sourceMappingURL=index.cjs.js.map

package/dist/index.cjs.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.cjs.js","sources":["../src/awsOrganization/config.ts","../src/processors/AwsOrganizationCloudAccountProcessor.ts"],"sourcesContent":["/*\n * Copyright 2020 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport { Config } from '@backstage/config';\n\n/**\n * The configuration parameters for a single AWS Organization Processor\n */\nexport type AwsOrganizationProviderConfig = {\n  /**\n   * The role to assume for the processor.\n   */\n  roleArn?: string;\n};\n\nexport function readAwsOrganizationConfig(\n  config: Config,\n): AwsOrganizationProviderConfig {\n  const providerConfig = config.getOptionalConfig('provider');\n\n  const roleArn = providerConfig?.getOptionalString('roleArn');\n  return {\n    roleArn,\n  };\n}\n","/*\n * Copyright 2020 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport { ResourceEntityV1alpha1 } from '@backstage/catalog-model';\nimport { Config } from '@backstage/config';\nimport {\n  CatalogProcessor,\n  CatalogProcessorEmit,\n  LocationSpec,\n  processingResult,\n} from '@backstage/plugin-catalog-backend';\nimport AWS, { Credentials, Organizations } from 'aws-sdk';\nimport { Account, ListAccountsResponse } from 'aws-sdk/clients/organizations';\nimport { Logger } from 'winston';\nimport {\n  AwsOrganizationProviderConfig,\n  readAwsOrganizationConfig,\n} from '../awsOrganization/config';\n\nconst AWS_ORGANIZATION_REGION = 'us-east-1';\nconst LOCATION_TYPE = 'aws-cloud-accounts';\n\nconst ACCOUNTID_ANNOTATION = 'amazonaws.com/account-id';\nconst ARN_ANNOTATION = 'amazonaws.com/arn';\nconst ORGANIZATION_ANNOTATION = 'amazonaws.com/organization-id';\n\n/**\n * A processor for ingesting AWS Accounts from AWS Organizations.\n *\n * If custom authentication is needed, it can be achieved by configuring the\n * global AWS.credentials object.\n *\n * @public\n */\nexport class AwsOrganizationCloudAccountProcessor implements CatalogProcessor {\n  private readonly organizations: Organizations;\n  private readonly provider: AwsOrganizationProviderConfig;\n\n  static fromConfig(config: Config, options: { logger: Logger }) {\n    const c = config.getOptionalConfig('catalog.processors.awsOrganization');\n    return new AwsOrganizationCloudAccountProcessor({\n      ...options,\n      provider: c ? readAwsOrganizationConfig(c) : {},\n    });\n  }\n\n  private static buildCredentials(\n    config: AwsOrganizationProviderConfig,\n  ): Credentials | undefined {\n    const roleArn = config.roleArn;\n    if (!roleArn) {\n      return undefined;\n    }\n\n    return new AWS.ChainableTemporaryCredentials({\n      params: {\n        RoleSessionName: 'backstage-aws-organization-processor',\n        RoleArn: roleArn,\n      },\n    });\n  }\n\n  private constructor(options: { provider: AwsOrganizationProviderConfig }) {\n    this.provider = options.provider;\n    const credentials = AwsOrganizationCloudAccountProcessor.buildCredentials(\n      this.provider,\n    );\n    this.organizations = new AWS.Organizations({\n      credentials,\n      region: AWS_ORGANIZATION_REGION,\n    }); // Only available in us-east-1\n  }\n\n  getProcessorName(): string {\n    return 'AwsOrganizationCloudAccountProcessor';\n  }\n\n  async readLocation(\n    location: LocationSpec,\n    _optional: boolean,\n    emit: CatalogProcessorEmit,\n  ): Promise<boolean> {\n    if (location.type !== LOCATION_TYPE) {\n      return false;\n    }\n\n    (await this.getAwsAccounts())\n      .map(account => this.mapAccountToComponent(account))\n      .filter(entity => {\n        if (location.target !== '') {\n          if (entity.metadata.annotations) {\n            return (\n              entity.metadata.annotations[ORGANIZATION_ANNOTATION] ===\n              location.target\n            );\n          }\n          return false;\n        }\n        return true;\n      })\n      .forEach(entity => {\n        emit(processingResult.entity(location, entity));\n      });\n\n    return true;\n  }\n\n  private normalizeName(name: string): string {\n    return name\n      .trim()\n      .toLocaleLowerCase('en-US')\n      .replace(/[^a-zA-Z0-9\\-]/g, '-');\n  }\n\n  private extractInformationFromArn(arn: string): {\n    accountId: string;\n    organizationId: string;\n  } {\n    const parts = arn.split('/');\n\n    return {\n      accountId: parts[parts.length - 1],\n      organizationId: parts[parts.length - 2],\n    };\n  }\n\n  private async getAwsAccounts(): Promise<Account[]> {\n    let awsAccounts: Account[] = [];\n    let isInitialAttempt = true;\n    let nextToken = undefined;\n    while (isInitialAttempt || nextToken) {\n      isInitialAttempt = false;\n      const orgAccounts: ListAccountsResponse = await this.organizations\n        .listAccounts({ NextToken: nextToken })\n        .promise();\n      if (orgAccounts.Accounts) {\n        awsAccounts = awsAccounts.concat(orgAccounts.Accounts);\n      }\n      nextToken = orgAccounts.NextToken;\n    }\n\n    return awsAccounts;\n  }\n\n  private mapAccountToComponent(account: Account): ResourceEntityV1alpha1 {\n    const { accountId, organizationId } = this.extractInformationFromArn(\n      account.Arn as string,\n    );\n    return {\n      apiVersion: 'backstage.io/v1alpha1',\n      kind: 'Resource',\n      metadata: {\n        annotations: {\n          [ACCOUNTID_ANNOTATION]: accountId,\n          [ARN_ANNOTATION]: account.Arn || '',\n          [ORGANIZATION_ANNOTATION]: organizationId,\n        },\n        name: this.normalizeName(account.Name || ''),\n        namespace: 'default',\n      },\n      spec: {\n        type: 'cloud-account',\n        owner: 'unknown',\n      },\n    };\n  }\n}\n"],"names":["AWS","processingResult"],"mappings":";;;;;;;;;;;mCA6BE,QAC+B;AAC/B,QAAM,iBAAiB,OAAO,kBAAkB;AAEhD,QAAM,UAAU,iDAAgB,kBAAkB;AAClD,SAAO;AAAA,IACL;AAAA;AAAA;;ACHJ,MAAM,0BAA0B;AAChC,MAAM,gBAAgB;AAEtB,MAAM,uBAAuB;AAC7B,MAAM,iBAAiB;AACvB,MAAM,0BAA0B;2CAU8C;AAAA,SAIrE,WAAW,QAAgB,SAA6B;AAC7D,UAAM,IAAI,OAAO,kBAAkB;AACnC,WAAO,IAAI,qCAAqC;AAAA,SAC3C;AAAA,MACH,UAAU,IAAI,0BAA0B,KAAK;AAAA;AAAA;AAAA,SAIlC,iBACb,QACyB;AACzB,UAAM,UAAU,OAAO;AACvB,QAAI,CAAC,SAAS;AACZ,aAAO;AAAA;AAGT,WAAO,IAAIA,wBAAI,8BAA8B;AAAA,MAC3C,QAAQ;AAAA,QACN,iBAAiB;AAAA,QACjB,SAAS;AAAA;AAAA;AAAA;AAAA,EAKP,YAAY,SAAsD;AACxE,SAAK,WAAW,QAAQ;AACxB,UAAM,cAAc,qCAAqC,iBACvD,KAAK;AAEP,SAAK,gBAAgB,IAAIA,wBAAI,cAAc;AAAA,MACzC;AAAA,MACA,QAAQ;AAAA;AAAA;AAAA,EAIZ,mBAA2B;AACzB,WAAO;AAAA;AAAA,QAGH,aACJ,UACA,WACA,MACkB;AAClB,QAAI,SAAS,SAAS,eAAe;AACnC,aAAO;AAAA;AAGT,IAAC,OAAM,KAAK,kBACT,IAAI,aAAW,KAAK,sBAAsB,UAC1C,OAAO,YAAU;AAChB,UAAI,SAAS,WAAW,IAAI;AAC1B,YAAI,OAAO,SAAS,aAAa;AAC/B,iBACE,OAAO,SAAS,YAAY,6BAC5B,SAAS;AAAA;AAGb,eAAO;AAAA;AAET,aAAO;AAAA,OAER,QAAQ,YAAU;AACjB,WAAKC,sCAAiB,OAAO,UAAU;AAAA;AAG3C,WAAO;AAAA;AAAA,EAGD,cAAc,MAAsB;AAC1C,WAAO,KACJ,OACA,kBAAkB,SAClB,QAAQ,mBAAmB;AAAA;AAAA,EAGxB,0BAA0B,KAGhC;AACA,UAAM,QAAQ,IAAI,MAAM;AAExB,WAAO;AAAA,MACL,WAAW,MAAM,MAAM,SAAS;AAAA,MAChC,gBAAgB,MAAM,MAAM,SAAS;AAAA;AAAA;AAAA,QAI3B,iBAAqC;AACjD,QAAI,cAAyB;AAC7B,QAAI,mBAAmB;AACvB,QAAI,YAAY;AAChB,WAAO,oBAAoB,WAAW;AACpC,yBAAmB;AACnB,YAAM,cAAoC,MAAM,KAAK,cAClD,aAAa,EAAE,WAAW,aAC1B;AACH,UAAI,YAAY,UAAU;AACxB,sBAAc,YAAY,OAAO,YAAY;AAAA;AAE/C,kBAAY,YAAY;AAAA;AAG1B,WAAO;AAAA;AAAA,EAGD,sBAAsB,SAA0C;AACtE,UAAM,EAAE,WAAW,mBAAmB,KAAK,0BACzC,QAAQ;AAEV,WAAO;AAAA,MACL,YAAY;AAAA,MACZ,MAAM;AAAA,MACN,UAAU;AAAA,QACR,aAAa;AAAA,WACV,uBAAuB;AAAA,WACvB,iBAAiB,QAAQ,OAAO;AAAA,WAChC,0BAA0B;AAAA;AAAA,QAE7B,MAAM,KAAK,cAAc,QAAQ,QAAQ;AAAA,QACzC,WAAW;AAAA;AAAA,MAEb,MAAM;AAAA,QACJ,MAAM;AAAA,QACN,OAAO;AAAA;AAAA;AAAA;AAAA;;;;"}
+{"version":3,"file":"index.cjs.js","sources":["../src/awsOrganization/config.ts","../src/processors/AwsOrganizationCloudAccountProcessor.ts","../src/processors/AwsS3DiscoveryProcessor.ts"],"sourcesContent":["/*\n * Copyright 2020 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport { Config } from '@backstage/config';\n\n/**\n * The configuration parameters for a single AWS Organization Processor\n */\nexport type AwsOrganizationProviderConfig = {\n  /**\n   * The role to assume for the processor.\n   */\n  roleArn?: string;\n};\n\nexport function readAwsOrganizationConfig(\n  config: Config,\n): AwsOrganizationProviderConfig {\n  const providerConfig = config.getOptionalConfig('provider');\n\n  const roleArn = providerConfig?.getOptionalString('roleArn');\n  return {\n    roleArn,\n  };\n}\n","/*\n * Copyright 2020 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport { ResourceEntityV1alpha1 } from '@backstage/catalog-model';\nimport { Config } from '@backstage/config';\nimport {\n  CatalogProcessor,\n  CatalogProcessorEmit,\n  LocationSpec,\n  processingResult,\n} from '@backstage/plugin-catalog-backend';\nimport AWS, { Credentials, Organizations } from 'aws-sdk';\nimport { Account, ListAccountsResponse } from 'aws-sdk/clients/organizations';\nimport { Logger } from 'winston';\nimport {\n  AwsOrganizationProviderConfig,\n  readAwsOrganizationConfig,\n} from '../awsOrganization/config';\n\nconst AWS_ORGANIZATION_REGION = 'us-east-1';\nconst LOCATION_TYPE = 'aws-cloud-accounts';\n\nconst ACCOUNTID_ANNOTATION = 'amazonaws.com/account-id';\nconst ARN_ANNOTATION = 'amazonaws.com/arn';\nconst ORGANIZATION_ANNOTATION = 'amazonaws.com/organization-id';\n\n/**\n * A processor for ingesting AWS Accounts from AWS Organizations.\n *\n * If custom authentication is needed, it can be achieved by configuring the\n * global AWS.credentials object.\n *\n * @public\n */\nexport class AwsOrganizationCloudAccountProcessor implements CatalogProcessor {\n  private readonly organizations: Organizations;\n  private readonly provider: AwsOrganizationProviderConfig;\n\n  static fromConfig(config: Config, options: { logger: Logger }) {\n    const c = config.getOptionalConfig('catalog.processors.awsOrganization');\n    return new AwsOrganizationCloudAccountProcessor({\n      ...options,\n      provider: c ? readAwsOrganizationConfig(c) : {},\n    });\n  }\n\n  private static buildCredentials(\n    config: AwsOrganizationProviderConfig,\n  ): Credentials | undefined {\n    const roleArn = config.roleArn;\n    if (!roleArn) {\n      return undefined;\n    }\n\n    return new AWS.ChainableTemporaryCredentials({\n      params: {\n        RoleSessionName: 'backstage-aws-organization-processor',\n        RoleArn: roleArn,\n      },\n    });\n  }\n\n  private constructor(options: { provider: AwsOrganizationProviderConfig }) {\n    this.provider = options.provider;\n    const credentials = AwsOrganizationCloudAccountProcessor.buildCredentials(\n      this.provider,\n    );\n    this.organizations = new AWS.Organizations({\n      credentials,\n      region: AWS_ORGANIZATION_REGION,\n    }); // Only available in us-east-1\n  }\n\n  getProcessorName(): string {\n    return 'AwsOrganizationCloudAccountProcessor';\n  }\n\n  async readLocation(\n    location: LocationSpec,\n    _optional: boolean,\n    emit: CatalogProcessorEmit,\n  ): Promise<boolean> {\n    if (location.type !== LOCATION_TYPE) {\n      return false;\n    }\n\n    (await this.getAwsAccounts())\n      .map(account => this.mapAccountToComponent(account))\n      .filter(entity => {\n        if (location.target !== '') {\n          if (entity.metadata.annotations) {\n            return (\n              entity.metadata.annotations[ORGANIZATION_ANNOTATION] ===\n              location.target\n            );\n          }\n          return false;\n        }\n        return true;\n      })\n      .forEach(entity => {\n        emit(processingResult.entity(location, entity));\n      });\n\n    return true;\n  }\n\n  private normalizeName(name: string): string {\n    return name\n      .trim()\n      .toLocaleLowerCase('en-US')\n      .replace(/[^a-zA-Z0-9\\-]/g, '-');\n  }\n\n  private extractInformationFromArn(arn: string): {\n    accountId: string;\n    organizationId: string;\n  } {\n    const parts = arn.split('/');\n\n    return {\n      accountId: parts[parts.length - 1],\n      organizationId: parts[parts.length - 2],\n    };\n  }\n\n  private async getAwsAccounts(): Promise<Account[]> {\n    let awsAccounts: Account[] = [];\n    let isInitialAttempt = true;\n    let nextToken = undefined;\n    while (isInitialAttempt || nextToken) {\n      isInitialAttempt = false;\n      const orgAccounts: ListAccountsResponse = await this.organizations\n        .listAccounts({ NextToken: nextToken })\n        .promise();\n      if (orgAccounts.Accounts) {\n        awsAccounts = awsAccounts.concat(orgAccounts.Accounts);\n      }\n      nextToken = orgAccounts.NextToken;\n    }\n\n    return awsAccounts;\n  }\n\n  private mapAccountToComponent(account: Account): ResourceEntityV1alpha1 {\n    const { accountId, organizationId } = this.extractInformationFromArn(\n      account.Arn as string,\n    );\n    return {\n      apiVersion: 'backstage.io/v1alpha1',\n      kind: 'Resource',\n      metadata: {\n        annotations: {\n          [ACCOUNTID_ANNOTATION]: accountId,\n          [ARN_ANNOTATION]: account.Arn || '',\n          [ORGANIZATION_ANNOTATION]: organizationId,\n        },\n        name: this.normalizeName(account.Name || ''),\n        namespace: 'default',\n      },\n      spec: {\n        type: 'cloud-account',\n        owner: 'unknown',\n      },\n    };\n  }\n}\n","/*\n * Copyright 2020 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport { UrlReader } from '@backstage/backend-common';\nimport { isError } from '@backstage/errors';\nimport {\n  CatalogProcessor,\n  CatalogProcessorEmit,\n  CatalogProcessorParser,\n  LocationSpec,\n  processingResult,\n} from '@backstage/plugin-catalog-backend';\nimport limiterFactory from 'p-limit';\n\n/**\n * A processor for automatic discovery of entities from S3 buckets. Handles the\n * `s3-discovery` location type, and target bucket URLs e.g. on the form\n * `https://testbucket.s3.us-east-2.amazonaws.com`.\n *\n * @public\n */\nexport class AwsS3DiscoveryProcessor implements CatalogProcessor {\n  constructor(private readonly reader: UrlReader) {}\n\n  getProcessorName(): string {\n    return 'AwsS3DiscoveryProcessor';\n  }\n\n  async readLocation(\n    location: LocationSpec,\n    optional: boolean,\n    emit: CatalogProcessorEmit,\n    parser: CatalogProcessorParser,\n  ): Promise<boolean> {\n    if (location.type !== 's3-discovery') {\n      return false;\n    }\n\n    try {\n      const output = await this.doRead(location.target);\n      for (const item of output) {\n        for await (const parseResult of parser({\n          data: item.data,\n          location: { type: location.type, target: item.url },\n        })) {\n          emit(parseResult);\n        }\n      }\n    } catch (error) {\n      const message = `Unable to read ${location.type}, ${error}`;\n\n      if (isError(error) && error.name === 'NotFoundError') {\n        if (!optional) {\n          emit(processingResult.notFoundError(location, message));\n        }\n      } else {\n        emit(processingResult.generalError(location, message));\n      }\n    }\n    return true;\n  }\n\n  private async doRead(\n    location: string,\n  ): Promise<{ data: Buffer; url: string }[]> {\n    const limiter = limiterFactory(5);\n    const response = await this.reader.readTree(location);\n    const responseFiles = await response.files();\n    const output = responseFiles.map(async file => ({\n      url: file.path,\n      data: await limiter(file.content),\n    }));\n    return Promise.all(output);\n  }\n}\n"],"names":["AWS","processingResult","isError","limiterFactory"],"mappings":";;;;;;;;;;;;;;AA4BO,SAAA,yBAAA,CACL,MAC+B,EAAA;AAC/B,EAAM,MAAA,cAAA,GAAiB,OAAO,iBAAkB,CAAA,UAAA,CAAA,CAAA;AAEhD,EAAM,MAAA,OAAA,GAAU,iDAAgB,iBAAkB,CAAA,SAAA,CAAA,CAAA;AAClD,EAAO,OAAA;AAAA,IACL,OAAA;AAAA,GAAA,CAAA;AAAA;;ACHJ,MAAM,uBAA0B,GAAA,WAAA,CAAA;AAChC,MAAM,aAAgB,GAAA,oBAAA,CAAA;AAEtB,MAAM,oBAAuB,GAAA,0BAAA,CAAA;AAC7B,MAAM,cAAiB,GAAA,mBAAA,CAAA;AACvB,MAAM,uBAA0B,GAAA,+BAAA,CAAA;AAU8C,MAAA,oCAAA,CAAA;AAAA,EAIrE,OAAA,UAAA,CAAW,QAAgB,OAA6B,EAAA;AAC7D,IAAM,MAAA,CAAA,GAAI,OAAO,iBAAkB,CAAA,oCAAA,CAAA,CAAA;AACnC,IAAA,OAAO,IAAI,oCAAqC,CAAA;AAAA,MAC3C,GAAA,OAAA;AAAA,MACH,QAAA,EAAU,CAAI,GAAA,yBAAA,CAA0B,CAAK,CAAA,GAAA,EAAA;AAAA,KAAA,CAAA,CAAA;AAAA,GAAA;AAAA,EAAA,OAIlC,iBACb,MACyB,EAAA;AACzB,IAAA,MAAM,UAAU,MAAO,CAAA,OAAA,CAAA;AACvB,IAAA,IAAI,CAAC,OAAS,EAAA;AACZ,MAAO,OAAA,KAAA,CAAA,CAAA;AAAA,KAAA;AAGT,IAAO,OAAA,IAAIA,wBAAI,6BAA8B,CAAA;AAAA,MAC3C,MAAQ,EAAA;AAAA,QACN,eAAiB,EAAA,sCAAA;AAAA,QACjB,OAAS,EAAA,OAAA;AAAA,OAAA;AAAA,KAAA,CAAA,CAAA;AAAA,GAAA;AAAA,EAKP,YAAY,OAAsD,EAAA;AACxE,IAAA,IAAA,CAAK,WAAW,OAAQ,CAAA,QAAA,CAAA;AACxB,IAAM,MAAA,WAAA,GAAc,oCAAqC,CAAA,gBAAA,CACvD,IAAK,CAAA,QAAA,CAAA,CAAA;AAEP,IAAK,IAAA,CAAA,aAAA,GAAgB,IAAIA,uBAAA,CAAI,aAAc,CAAA;AAAA,MACzC,WAAA;AAAA,MACA,MAAQ,EAAA,uBAAA;AAAA,KAAA,CAAA,CAAA;AAAA,GAAA;AAAA,EAIZ,gBAA2B,GAAA;AACzB,IAAO,OAAA,sCAAA,CAAA;AAAA,GAAA;AAAA,EAGH,MAAA,YAAA,CACJ,QACA,EAAA,SAAA,EACA,IACkB,EAAA;AAClB,IAAI,IAAA,QAAA,CAAS,SAAS,aAAe,EAAA;AACnC,MAAO,OAAA,KAAA,CAAA;AAAA,KAAA;AAGT,IAAC,CAAA,MAAM,KAAK,cACT,EAAA,EAAA,GAAA,CAAI,aAAW,IAAK,CAAA,qBAAA,CAAsB,OAC1C,CAAA,CAAA,CAAA,MAAA,CAAO,CAAU,MAAA,KAAA;AAChB,MAAI,IAAA,QAAA,CAAS,WAAW,EAAI,EAAA;AAC1B,QAAI,IAAA,MAAA,CAAO,SAAS,WAAa,EAAA;AAC/B,UAAA,OACE,MAAO,CAAA,QAAA,CAAS,WAAY,CAAA,uBAAA,CAAA,KAC5B,QAAS,CAAA,MAAA,CAAA;AAAA,SAAA;AAGb,QAAO,OAAA,KAAA,CAAA;AAAA,OAAA;AAET,MAAO,OAAA,IAAA,CAAA;AAAA,KAAA,CAAA,CAER,QAAQ,CAAU,MAAA,KAAA;AACjB,MAAK,IAAA,CAAAC,qCAAA,CAAiB,OAAO,QAAU,EAAA,MAAA,CAAA,CAAA,CAAA;AAAA,KAAA,CAAA,CAAA;AAG3C,IAAO,OAAA,IAAA,CAAA;AAAA,GAAA;AAAA,EAGD,cAAc,IAAsB,EAAA;AAC1C,IAAA,OAAO,IACJ,CAAA,IAAA,EAAA,CACA,iBAAkB,CAAA,OAAA,CAAA,CAClB,QAAQ,iBAAmB,EAAA,GAAA,CAAA,CAAA;AAAA,GAAA;AAAA,EAGxB,0BAA0B,GAGhC,EAAA;AACA,IAAM,MAAA,KAAA,GAAQ,IAAI,KAAM,CAAA,GAAA,CAAA,CAAA;AAExB,IAAO,OAAA;AAAA,MACL,SAAA,EAAW,KAAM,CAAA,KAAA,CAAM,MAAS,GAAA,CAAA,CAAA;AAAA,MAChC,cAAA,EAAgB,KAAM,CAAA,KAAA,CAAM,MAAS,GAAA,CAAA,CAAA;AAAA,KAAA,CAAA;AAAA,GAAA;AAAA,EAAA,MAI3B,cAAqC,GAAA;AACjD,IAAA,IAAI,WAAyB,GAAA,EAAA,CAAA;AAC7B,IAAA,IAAI,gBAAmB,GAAA,IAAA,CAAA;AACvB,IAAA,IAAI,SAAY,GAAA,KAAA,CAAA,CAAA;AAChB,IAAA,OAAO,oBAAoB,SAAW,EAAA;AACpC,MAAmB,gBAAA,GAAA,KAAA,CAAA;AACnB,MAAA,MAAM,cAAoC,MAAM,IAAA,CAAK,cAClD,YAAa,CAAA,EAAE,WAAW,SAC1B,EAAA,CAAA,CAAA,OAAA,EAAA,CAAA;AACH,MAAA,IAAI,YAAY,QAAU,EAAA;AACxB,QAAc,WAAA,GAAA,WAAA,CAAY,OAAO,WAAY,CAAA,QAAA,CAAA,CAAA;AAAA,OAAA;AAE/C,MAAA,SAAA,GAAY,WAAY,CAAA,SAAA,CAAA;AAAA,KAAA;AAG1B,IAAO,OAAA,WAAA,CAAA;AAAA,GAAA;AAAA,EAGD,sBAAsB,OAA0C,EAAA;AACtE,IAAA,MAAM,EAAE,SAAA,EAAW,cAAmB,EAAA,GAAA,IAAA,CAAK,0BACzC,OAAQ,CAAA,GAAA,CAAA,CAAA;AAEV,IAAO,OAAA;AAAA,MACL,UAAY,EAAA,uBAAA;AAAA,MACZ,IAAM,EAAA,UAAA;AAAA,MACN,QAAU,EAAA;AAAA,QACR,WAAa,EAAA;AAAA,UAAA,CACV,oBAAuB,GAAA,SAAA;AAAA,UACvB,CAAA,cAAA,GAAiB,QAAQ,GAAO,IAAA,EAAA;AAAA,UAAA,CAChC,uBAA0B,GAAA,cAAA;AAAA,SAAA;AAAA,QAE7B,IAAM,EAAA,IAAA,CAAK,aAAc,CAAA,OAAA,CAAQ,IAAQ,IAAA,EAAA,CAAA;AAAA,QACzC,SAAW,EAAA,SAAA;AAAA,OAAA;AAAA,MAEb,IAAM,EAAA;AAAA,QACJ,IAAM,EAAA,eAAA;AAAA,QACN,KAAO,EAAA,SAAA;AAAA,OAAA;AAAA,KAAA,CAAA;AAAA,GAAA;AAAA;;AC7IkD,MAAA,uBAAA,CAAA;AAAA,EAC/D,YAA6B,MAAmB,EAAA;AAAnB,IAAA,IAAA,CAAA,MAAA,GAAA,MAAA,CAAA;AAAA,GAAA;AAAA,EAE7B,gBAA2B,GAAA;AACzB,IAAO,OAAA,yBAAA,CAAA;AAAA,GAAA;AAAA,EAAA,MAGH,YACJ,CAAA,QAAA,EACA,QACA,EAAA,IAAA,EACA,MACkB,EAAA;AAClB,IAAI,IAAA,QAAA,CAAS,SAAS,cAAgB,EAAA;AACpC,MAAO,OAAA,KAAA,CAAA;AAAA,KAAA;AAGT,IAAI,IAAA;AACF,MAAA,MAAM,MAAS,GAAA,MAAM,IAAK,CAAA,MAAA,CAAO,QAAS,CAAA,MAAA,CAAA,CAAA;AAC1C,MAAA,KAAA,MAAW,QAAQ,MAAQ,EAAA;AACzB,QAAA,WAAA,MAAiB,eAAe,MAAO,CAAA;AAAA,UACrC,MAAM,IAAK,CAAA,IAAA;AAAA,UACX,UAAU,EAAE,IAAA,EAAM,QAAS,CAAA,IAAA,EAAM,QAAQ,IAAK,CAAA,GAAA,EAAA;AAAA,SAC5C,CAAA,EAAA;AACF,UAAK,IAAA,CAAA,WAAA,CAAA,CAAA;AAAA,SAAA;AAAA,OAAA;AAAA,KAAA,CAAA,OAGF,KAAP,EAAA;AACA,MAAM,MAAA,OAAA,GAAU,CAAkB,eAAA,EAAA,QAAA,CAAS,IAAS,CAAA,EAAA,EAAA,KAAA,CAAA,CAAA,CAAA;AAEpD,MAAA,IAAIC,cAAQ,CAAA,KAAA,CAAA,IAAU,KAAM,CAAA,IAAA,KAAS,eAAiB,EAAA;AACpD,QAAA,IAAI,CAAC,QAAU,EAAA;AACb,UAAK,IAAA,CAAAD,qCAAA,CAAiB,cAAc,QAAU,EAAA,OAAA,CAAA,CAAA,CAAA;AAAA,SAAA;AAAA,OAE3C,MAAA;AACL,QAAK,IAAA,CAAAA,qCAAA,CAAiB,aAAa,QAAU,EAAA,OAAA,CAAA,CAAA,CAAA;AAAA,OAAA;AAAA,KAAA;AAGjD,IAAO,OAAA,IAAA,CAAA;AAAA,GAAA;AAAA,EAAA,MAGK,OACZ,QAC0C,EAAA;AAC1C,IAAA,MAAM,UAAUE,kCAAe,CAAA,CAAA,CAAA,CAAA;AAC/B,IAAA,MAAM,QAAW,GAAA,MAAM,IAAK,CAAA,MAAA,CAAO,QAAS,CAAA,QAAA,CAAA,CAAA;AAC5C,IAAM,MAAA,aAAA,GAAgB,MAAM,QAAS,CAAA,KAAA,EAAA,CAAA;AACrC,IAAA,MAAM,MAAS,GAAA,aAAA,CAAc,GAAI,CAAA,OAAM,IAAS,MAAA;AAAA,MAC9C,KAAK,IAAK,CAAA,IAAA;AAAA,MACV,IAAA,EAAM,MAAM,OAAA,CAAQ,IAAK,CAAA,OAAA,CAAA;AAAA,KAAA,CAAA,CAAA,CAAA;AAE3B,IAAA,OAAO,QAAQ,GAAI,CAAA,MAAA,CAAA,CAAA;AAAA,GAAA;AAAA;;;;;"}

package/dist/index.d.ts

@@ -1,6 +1,7 @@
 import { Config } from '@backstage/config';
-import { CatalogProcessor, LocationSpec, CatalogProcessorEmit } from '@backstage/plugin-catalog-backend';
+import { CatalogProcessor, LocationSpec, CatalogProcessorEmit, CatalogProcessorParser } from '@backstage/plugin-catalog-backend';
 import { Logger } from 'winston';
+import { UrlReader } from '@backstage/backend-common';
 
 /**
  * A processor for ingesting AWS Accounts from AWS Organizations.
@@ -26,4 +27,19 @@ declare class AwsOrganizationCloudAccountProcessor implements CatalogProcessor {
     private mapAccountToComponent;
 }
 
-export { AwsOrganizationCloudAccountProcessor };
+/**
+ * A processor for automatic discovery of entities from S3 buckets. Handles the
+ * `s3-discovery` location type, and target bucket URLs e.g. on the form
+ * `https://testbucket.s3.us-east-2.amazonaws.com`.
+ *
+ * @public
+ */
+declare class AwsS3DiscoveryProcessor implements CatalogProcessor {
+    private readonly reader;
+    constructor(reader: UrlReader);
+    getProcessorName(): string;
+    readLocation(location: LocationSpec, optional: boolean, emit: CatalogProcessorEmit, parser: CatalogProcessorParser): Promise<boolean>;
+    private doRead;
+}
+
+export { AwsOrganizationCloudAccountProcessor, AwsS3DiscoveryProcessor };

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@backstage/plugin-catalog-backend-module-aws",
   "description": "A Backstage catalog backend module that helps integrate towards AWS",
-  "version": "0.1.1",
+  "version": "0.1.3",
   "main": "dist/index.cjs.js",
   "types": "dist/index.d.ts",
   "license": "Apache-2.0",
@@ -33,24 +33,27 @@
     "start": "backstage-cli package start"
   },
   "dependencies": {
-    "@backstage/catalog-model": "^0.12.0",
-    "@backstage/config": "^0.1.15",
-    "@backstage/errors": "^0.2.2",
-    "@backstage/plugin-catalog-backend": "^0.23.0",
-    "@backstage/types": "^0.1.3",
+    "@backstage/backend-common": "^0.13.1",
+    "@backstage/catalog-model": "^1.0.0",
+    "@backstage/config": "^1.0.0",
+    "@backstage/errors": "^1.0.0",
+    "@backstage/plugin-catalog-backend": "^1.0.0",
+    "@backstage/types": "^1.0.0",
     "aws-sdk": "^2.840.0",
     "lodash": "^4.17.21",
+    "p-limit": "^3.0.2",
     "winston": "^3.2.1"
   },
   "devDependencies": {
-    "@backstage/cli": "^0.15.0",
+    "@backstage/cli": "^0.16.0",
     "@types/lodash": "^4.14.151",
-    "aws-sdk-mock": "^5.2.1"
+    "aws-sdk-mock": "^5.2.1",
+    "yaml": "^1.9.2"
   },
   "files": [
     "dist",
     "config.d.ts"
   ],
   "configSchema": "config.d.ts",
-  "gitHead": "04bb0dd824b78f6b57dac62c3015e681f094045c"
+  "gitHead": "e9496f746b31600dbfac7fa76987479e66426257"
 }