@infrascan/s3-connector 0.1.0

package/README.md

@@ -0,0 +1,42 @@
+# @infrascan/s3-connector
+
+Infrascan connectors act as the plumbing between the Infrascan SDK and wherever you persist your scan state.
+
+The S3 connector uses an AWS S3 Bucket to store its state. This connector library exposes an S3Connector class as the default export, which wraps an cache internally to reduce the overhead of S3 reads.
+
+All state files are written into the bucket in the format: /service-function/account/region.json.
+This is to support both searching state globally and reading state by function call per region.
+
+## Quickstart
+
+```js
+import { fromIni } from "@aws-sdk/credential-providers";
+import { S3Client } from "@aws-sdk/s3-client";
+import Infrascan from "@infrascan/sdk";
+import S3Connector from "@infrascan/s3-connector";
+import { registerAwsScanners } from "@infrascan/aws";
+
+const credentials = fromIni({ profile: "dev" });
+const s3Client = new S3Client({ credentials });
+const connector = new S3Connector({
+  S3: s3Client,
+  prefix: 'my-scan-tenant',
+  bucket: 'my-scan-output-bucket'
+});
+const infrascan = registerAwsScanners(new Infrascan());
+
+infrascan.performScan(
+  credentials,
+  connector
+).then(function (scanMetadata) {
+  console.log("Scan Complete!", scanMetadata);
+  return infrascan.generateGraph(
+    scanMetadata,
+    connector
+  );
+}).then(function (graphData) {
+  console.log("Graph generated!", graphData);
+}).catch(function (err) {
+  console.error("Failed to scan", err);
+});
+```

package/dist/index.d.mts

@@ -0,0 +1,47 @@
+import { S3Client, PutObjectCommandOutput } from '@aws-sdk/client-s3';
+
+declare class Cache<T> {
+    sizeLimit: number;
+    size: number;
+    state: Record<string, T>;
+    keyOrders: Array<string>;
+    constructor(maxSize: number);
+    set(key: string, val: T): void;
+    get(key: string): T | undefined;
+    has(key: string): boolean;
+}
+
+interface ConstructorArgs {
+    S3: S3Client;
+    prefix: string;
+    bucket: string;
+    cacheLimit?: number;
+}
+declare class S3Connector {
+    S3: S3Client;
+    prefix: string;
+    bucket: string;
+    cache: Cache<any>;
+    constructor({ S3, prefix, bucket, cacheLimit }: ConstructorArgs);
+    recordServiceCall(filePath: string, state: any): Promise<PutObjectCommandOutput>;
+    readStateForCall(filePath: string): Promise<any>;
+    /**
+     * Format for file paths is somewhat unintuitive — the account and region are put at the end of the path
+     * This makes it easier for retrieving global state by service and function call where only list by prefix is supported (e.g. S3)
+     * Example:
+     * - /1234567890/us-east-1/Lambda/listFunctions.json cannot be globally listed by prefix, only by account and region
+     * - /Lambda-listFunctions/1234567890/us-east-1.json can be listed by function which allows for cross account resolution
+     */
+    buildFilePathForServiceCall(account: string, region: string, service: string, functionName: string): string;
+    onServiceScanCompleteCallback(account: string, region: string, service: string, functionName: string, functionState: any): Promise<PutObjectCommandOutput>;
+    resolveStateForServiceFunction(account: string, region: string, service: string, functionCall: string): Promise<any>;
+    listObjectsForService(service: string, functionCall: string): Promise<Array<string>>;
+    getGlobalStateForServiceFunction(service: string, functionCall: string): Promise<Array<any | undefined>>;
+    getMetadataPath(): string;
+    writeMetadata(metadata: any): Promise<PutObjectCommandOutput>;
+    readMetadata(): Promise<any>;
+    getGraphPath(): string;
+    writeGraphData(graphBlob: any): Promise<PutObjectCommandOutput>;
+}
+
+export { ConstructorArgs, S3Connector as default };

package/dist/index.d.ts

@@ -0,0 +1,47 @@
+import { S3Client, PutObjectCommandOutput } from '@aws-sdk/client-s3';
+
+declare class Cache<T> {
+    sizeLimit: number;
+    size: number;
+    state: Record<string, T>;
+    keyOrders: Array<string>;
+    constructor(maxSize: number);
+    set(key: string, val: T): void;
+    get(key: string): T | undefined;
+    has(key: string): boolean;
+}
+
+interface ConstructorArgs {
+    S3: S3Client;
+    prefix: string;
+    bucket: string;
+    cacheLimit?: number;
+}
+declare class S3Connector {
+    S3: S3Client;
+    prefix: string;
+    bucket: string;
+    cache: Cache<any>;
+    constructor({ S3, prefix, bucket, cacheLimit }: ConstructorArgs);
+    recordServiceCall(filePath: string, state: any): Promise<PutObjectCommandOutput>;
+    readStateForCall(filePath: string): Promise<any>;
+    /**
+     * Format for file paths is somewhat unintuitive — the account and region are put at the end of the path
+     * This makes it easier for retrieving global state by service and function call where only list by prefix is supported (e.g. S3)
+     * Example:
+     * - /1234567890/us-east-1/Lambda/listFunctions.json cannot be globally listed by prefix, only by account and region
+     * - /Lambda-listFunctions/1234567890/us-east-1.json can be listed by function which allows for cross account resolution
+     */
+    buildFilePathForServiceCall(account: string, region: string, service: string, functionName: string): string;
+    onServiceScanCompleteCallback(account: string, region: string, service: string, functionName: string, functionState: any): Promise<PutObjectCommandOutput>;
+    resolveStateForServiceFunction(account: string, region: string, service: string, functionCall: string): Promise<any>;
+    listObjectsForService(service: string, functionCall: string): Promise<Array<string>>;
+    getGlobalStateForServiceFunction(service: string, functionCall: string): Promise<Array<any | undefined>>;
+    getMetadataPath(): string;
+    writeMetadata(metadata: any): Promise<PutObjectCommandOutput>;
+    readMetadata(): Promise<any>;
+    getGraphPath(): string;
+    writeGraphData(graphBlob: any): Promise<PutObjectCommandOutput>;
+}
+
+export { ConstructorArgs, S3Connector as default };

package/dist/index.js

@@ -0,0 +1,153 @@
+"use strict";
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+
+// src/index.ts
+var src_exports = {};
+__export(src_exports, {
+  default: () => S3Connector
+});
+module.exports = __toCommonJS(src_exports);
+var import_client_s3 = require("@aws-sdk/client-s3");
+
+// src/cache.ts
+var Cache = class {
+  constructor(maxSize) {
+    if (maxSize < 1) {
+      throw new Error("Cannot create a cache with a size less than 1");
+    }
+    this.sizeLimit = maxSize;
+    this.size = 0;
+    this.state = {};
+    this.keyOrders = [];
+  }
+  set(key, val) {
+    if (this.size === this.sizeLimit) {
+      const keyToRemove = this.keyOrders.shift();
+      delete this.state[keyToRemove];
+      this.size -= 1;
+    }
+    this.keyOrders.push(key);
+    this.state[key] = val;
+    this.size += 1;
+  }
+  get(key) {
+    return this.state[key];
+  }
+  has(key) {
+    return this.state[key] != null;
+  }
+};
+
+// src/index.ts
+var isNotNull = (value) => value != null;
+var S3Connector = class {
+  constructor({ S3, prefix, bucket, cacheLimit }) {
+    this.S3 = S3;
+    this.prefix = prefix;
+    this.bucket = bucket;
+    this.cache = new Cache(cacheLimit ?? 10);
+  }
+  async recordServiceCall(filePath, state) {
+    const putObjectCmd = new import_client_s3.PutObjectCommand({
+      Bucket: this.bucket,
+      Key: filePath,
+      Body: JSON.stringify(state)
+    });
+    const putObjectResponse = await this.S3.send(putObjectCmd);
+    this.cache.set(filePath, state);
+    return putObjectResponse;
+  }
+  async readStateForCall(filePath) {
+    if (this.cache.has(filePath)) {
+      return this.cache.get(filePath);
+    }
+    const getStateCommand = new import_client_s3.GetObjectCommand({
+      Bucket: this.bucket,
+      Key: filePath
+    });
+    const rawState = await this.S3.send(getStateCommand);
+    const state = await rawState?.Body?.transformToString("utf8");
+    return state ? JSON.parse(state) : void 0;
+  }
+  /**
+   * Format for file paths is somewhat unintuitive — the account and region are put at the end of the path
+   * This makes it easier for retrieving global state by service and function call where only list by prefix is supported (e.g. S3)
+   * Example:
+   * - /1234567890/us-east-1/Lambda/listFunctions.json cannot be globally listed by prefix, only by account and region
+   * - /Lambda-listFunctions/1234567890/us-east-1.json can be listed by function which allows for cross account resolution
+   */
+  buildFilePathForServiceCall(account, region, service, functionName) {
+    return `${this.prefix}/${service}-${functionName}/${account}/${region}.json`;
+  }
+  onServiceScanCompleteCallback(account, region, service, functionName, functionState) {
+    const filePath = this.buildFilePathForServiceCall(
+      account,
+      region,
+      service,
+      functionName
+    );
+    return this.recordServiceCall(filePath, functionState);
+  }
+  resolveStateForServiceFunction(account, region, service, functionCall) {
+    const filePath = this.buildFilePathForServiceCall(
+      account,
+      region,
+      service,
+      functionCall
+    );
+    return this.readStateForCall(filePath);
+  }
+  async listObjectsForService(service, functionCall) {
+    const servicePrefix = `${this.prefix}/${service}-${functionCall}`;
+    const listObjectCommand = new import_client_s3.ListObjectsV2Command({
+      Bucket: this.bucket,
+      Prefix: servicePrefix
+    });
+    const response = await this.S3.send(listObjectCommand);
+    return response?.Contents?.map(({ Key }) => Key).filter(isNotNull) ?? [];
+  }
+  async getGlobalStateForServiceFunction(service, functionCall) {
+    const allKeys = await this.listObjectsForService(service, functionCall);
+    let globalState = [];
+    for (let idx = 0; idx < allKeys.length; idx += 1) {
+      const serviceState = await this.readStateForCall(allKeys[idx]);
+      if (serviceState != null) {
+        globalState = globalState.concat(serviceState);
+      } else {
+        console.log("No state returned for", allKeys[idx]);
+      }
+    }
+    return globalState;
+  }
+  getMetadataPath() {
+    return `${this.prefix}/metadata.json`;
+  }
+  writeMetadata(metadata) {
+    return this.recordServiceCall(this.getMetadataPath(), metadata);
+  }
+  readMetadata() {
+    return this.readStateForCall(this.getMetadataPath());
+  }
+  getGraphPath() {
+    return `${this.prefix}/graph.json`;
+  }
+  writeGraphData(graphBlob) {
+    return this.recordServiceCall(this.getGraphPath(), graphBlob);
+  }
+};

package/dist/index.mjs

@@ -0,0 +1,136 @@
+// src/index.ts
+import {
+  PutObjectCommand,
+  GetObjectCommand,
+  ListObjectsV2Command
+} from "@aws-sdk/client-s3";
+
+// src/cache.ts
+var Cache = class {
+  constructor(maxSize) {
+    if (maxSize < 1) {
+      throw new Error("Cannot create a cache with a size less than 1");
+    }
+    this.sizeLimit = maxSize;
+    this.size = 0;
+    this.state = {};
+    this.keyOrders = [];
+  }
+  set(key, val) {
+    if (this.size === this.sizeLimit) {
+      const keyToRemove = this.keyOrders.shift();
+      delete this.state[keyToRemove];
+      this.size -= 1;
+    }
+    this.keyOrders.push(key);
+    this.state[key] = val;
+    this.size += 1;
+  }
+  get(key) {
+    return this.state[key];
+  }
+  has(key) {
+    return this.state[key] != null;
+  }
+};
+
+// src/index.ts
+var isNotNull = (value) => value != null;
+var S3Connector = class {
+  constructor({ S3, prefix, bucket, cacheLimit }) {
+    this.S3 = S3;
+    this.prefix = prefix;
+    this.bucket = bucket;
+    this.cache = new Cache(cacheLimit ?? 10);
+  }
+  async recordServiceCall(filePath, state) {
+    const putObjectCmd = new PutObjectCommand({
+      Bucket: this.bucket,
+      Key: filePath,
+      Body: JSON.stringify(state)
+    });
+    const putObjectResponse = await this.S3.send(putObjectCmd);
+    this.cache.set(filePath, state);
+    return putObjectResponse;
+  }
+  async readStateForCall(filePath) {
+    if (this.cache.has(filePath)) {
+      return this.cache.get(filePath);
+    }
+    const getStateCommand = new GetObjectCommand({
+      Bucket: this.bucket,
+      Key: filePath
+    });
+    const rawState = await this.S3.send(getStateCommand);
+    const state = await rawState?.Body?.transformToString("utf8");
+    return state ? JSON.parse(state) : void 0;
+  }
+  /**
+   * Format for file paths is somewhat unintuitive — the account and region are put at the end of the path
+   * This makes it easier for retrieving global state by service and function call where only list by prefix is supported (e.g. S3)
+   * Example:
+   * - /1234567890/us-east-1/Lambda/listFunctions.json cannot be globally listed by prefix, only by account and region
+   * - /Lambda-listFunctions/1234567890/us-east-1.json can be listed by function which allows for cross account resolution
+   */
+  buildFilePathForServiceCall(account, region, service, functionName) {
+    return `${this.prefix}/${service}-${functionName}/${account}/${region}.json`;
+  }
+  onServiceScanCompleteCallback(account, region, service, functionName, functionState) {
+    const filePath = this.buildFilePathForServiceCall(
+      account,
+      region,
+      service,
+      functionName
+    );
+    return this.recordServiceCall(filePath, functionState);
+  }
+  resolveStateForServiceFunction(account, region, service, functionCall) {
+    const filePath = this.buildFilePathForServiceCall(
+      account,
+      region,
+      service,
+      functionCall
+    );
+    return this.readStateForCall(filePath);
+  }
+  async listObjectsForService(service, functionCall) {
+    const servicePrefix = `${this.prefix}/${service}-${functionCall}`;
+    const listObjectCommand = new ListObjectsV2Command({
+      Bucket: this.bucket,
+      Prefix: servicePrefix
+    });
+    const response = await this.S3.send(listObjectCommand);
+    return response?.Contents?.map(({ Key }) => Key).filter(isNotNull) ?? [];
+  }
+  async getGlobalStateForServiceFunction(service, functionCall) {
+    const allKeys = await this.listObjectsForService(service, functionCall);
+    let globalState = [];
+    for (let idx = 0; idx < allKeys.length; idx += 1) {
+      const serviceState = await this.readStateForCall(allKeys[idx]);
+      if (serviceState != null) {
+        globalState = globalState.concat(serviceState);
+      } else {
+        console.log("No state returned for", allKeys[idx]);
+      }
+    }
+    return globalState;
+  }
+  getMetadataPath() {
+    return `${this.prefix}/metadata.json`;
+  }
+  writeMetadata(metadata) {
+    return this.recordServiceCall(this.getMetadataPath(), metadata);
+  }
+  readMetadata() {
+    return this.readStateForCall(this.getMetadataPath());
+  }
+  getGraphPath() {
+    return `${this.prefix}/graph.json`;
+  }
+  writeGraphData(graphBlob) {
+    return this.recordServiceCall(this.getGraphPath(), graphBlob);
+  }
+};
+export {
+  S3Connector as default
+};

package/package.json

@@ -0,0 +1,61 @@
+{
+  "name": "@infrascan/s3-connector",
+  "version": "0.1.0",
+  "description": "Connector for the Infrascan SDK to read and write state to an S3 bucket",
+  "main": "dist/index.js",
+  "module": "dist/index.mjs",
+  "types": "dist/index.d.ts",
+  "exports": {
+    ".": {
+      "require": "./dist/index.js",
+      "import": "./dist/index.mjs",
+      "types": "./dist/index.d.ts"
+    }
+  },
+  "files": [
+    "dist"
+  ],
+  "scripts": {
+    "build": "rm -rf dist ; tsup",
+    "prepublish": "rm -rf dist ; tsup",
+    "lint": "tsc --noEmit ; eslint .",
+    "clean": "rm -rf dist node_modules",
+    "test": "tap test/*.test.ts --allow-incomplete-coverage -- --typescript"
+  },
+  "tsup": {
+    "clean": true,
+    "cjsInterop": true,
+    "splitting": false,
+    "dts": true,
+    "entry": [
+      "src/index.ts"
+    ],
+    "format": [
+      "esm",
+      "cjs"
+    ],
+    "target": "node18"
+  },
+  "keywords": [
+    "AWS",
+    "Infrastructure Diagram",
+    "Infrastructure Graph",
+    "Cloud Infrastructure",
+    "Connector"
+  ],
+  "author": "Liam Farrelly <liam@l3f7.dev> (https://whoisli.am/)",
+  "license": "MPL-2.0",
+  "devDependencies": {
+    "@aws-sdk/util-stream-node": "^3.374.0",
+    "@infrascan/shared-types": "^0.2.2",
+    "aws-sdk-client-mock": "^3.0.0",
+    "eslint-config-custom": "*",
+    "tap": "^18.5.7",
+    "tsconfig": "*",
+    "typescript": "^5.1.6"
+  },
+  "dependencies": {
+    "@aws-sdk/client-s3": "3.428.0",
+    "minimatch": "^9.0.3"
+  }
+}