@atproto/aws 0.0.1-beta.0

package/dist/src/cloudfront.d.ts

@@ -0,0 +1,16 @@
+import * as aws from '@aws-sdk/client-cloudfront';
+export declare type CloudfrontConfig = {
+    distributionId: string;
+    pathPrefix?: string;
+} & Omit<aws.CloudFrontClientConfig, 'apiVersion'>;
+export declare class CloudfrontInvalidator implements ImageInvalidator {
+    distributionId: string;
+    pathPrefix: string;
+    client: aws.CloudFront;
+    constructor(cfg: CloudfrontConfig);
+    invalidate(subject: string, paths: string[]): Promise<void>;
+}
+export default CloudfrontInvalidator;
+interface ImageInvalidator {
+    invalidate(subject: string, paths: string[]): Promise<void>;
+}

package/dist/src/index.d.ts

@@ -0,0 +1,3 @@
+export * from './kms';
+export * from './s3';
+export * from './cloudfront';

package/dist/src/kms.d.ts

@@ -0,0 +1,16 @@
+import * as aws from '@aws-sdk/client-kms';
+import * as crypto from '@atproto/crypto';
+export declare type KmsConfig = {
+    keyId: string;
+} & Omit<aws.KMSClientConfig, 'apiVersion'>;
+export declare class KmsKeypair implements crypto.Keypair {
+    private client;
+    private keyId;
+    private publicKey;
+    jwtAlg: string;
+    constructor(client: aws.KMS, keyId: string, publicKey: Uint8Array);
+    static load(cfg: KmsConfig): Promise<KmsKeypair>;
+    did(): string;
+    sign(msg: Uint8Array): Promise<Uint8Array>;
+}
+export default KmsKeypair;

package/dist/src/s3.d.ts

@@ -0,0 +1,30 @@
+/// <reference types="node" />
+import * as aws from '@aws-sdk/client-s3';
+import { BlobStore } from '@atproto/repo';
+import { CID } from 'multiformats/cid';
+import stream from 'stream';
+export declare type S3Config = {
+    bucket: string;
+} & Omit<aws.S3ClientConfig, 'apiVersion'>;
+export declare class S3BlobStore implements BlobStore {
+    private client;
+    private bucket;
+    constructor(cfg: S3Config);
+    private genKey;
+    private getTmpPath;
+    private getStoredPath;
+    private getQuarantinedPath;
+    putTemp(bytes: Uint8Array | stream.Readable): Promise<string>;
+    makePermanent(key: string, cid: CID): Promise<void>;
+    putPermanent(cid: CID, bytes: Uint8Array | stream.Readable): Promise<void>;
+    quarantine(cid: CID): Promise<void>;
+    unquarantine(cid: CID): Promise<void>;
+    private getObject;
+    getBytes(cid: CID): Promise<Uint8Array>;
+    getStream(cid: CID): Promise<stream.Readable>;
+    delete(cid: CID): Promise<void>;
+    hasStored(cid: CID): Promise<boolean>;
+    private deleteKey;
+    private move;
+}
+export default S3BlobStore;

package/package.json

@@ -0,0 +1,30 @@
+{
+  "name": "@atproto/aws",
+  "version": "0.0.1-beta.0",
+  "main": "dist/index.js",
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/bluesky-social/atproto.git",
+    "directory": "packages/aws"
+  },
+  "scripts": {
+    "prettier": "prettier --check src/",
+    "prettier:fix": "prettier --write src/",
+    "lint": "eslint . --ext .ts,.tsx",
+    "lint:fix": "yarn lint --fix",
+    "verify": "run-p prettier lint",
+    "verify:fix": "yarn prettier:fix && yarn lint:fix",
+    "build": "node ./build.js",
+    "postbuild": "tsc --build tsconfig.build.json"
+  },
+  "dependencies": {
+    "@atproto/crypto": "*",
+    "@aws-sdk/client-cloudfront": "^3.261.0",
+    "@aws-sdk/client-kms": "^3.196.0",
+    "@aws-sdk/client-s3": "^3.224.0",
+    "@aws-sdk/lib-storage": "^3.226.0",
+    "@noble/secp256k1": "^1.7.0",
+    "key-encoder": "^2.0.3"
+  }
+}

package/src/cloudfront.ts

@@ -0,0 +1,41 @@
+import * as aws from '@aws-sdk/client-cloudfront'
+
+export type CloudfrontConfig = {
+  distributionId: string
+  pathPrefix?: string
+} & Omit<aws.CloudFrontClientConfig, 'apiVersion'>
+
+export class CloudfrontInvalidator implements ImageInvalidator {
+  distributionId: string
+  pathPrefix: string
+  client: aws.CloudFront
+  constructor(cfg: CloudfrontConfig) {
+    const { distributionId, pathPrefix, ...rest } = cfg
+    this.distributionId = distributionId
+    this.pathPrefix = pathPrefix ?? ''
+    this.client = new aws.CloudFront({
+      ...rest,
+      apiVersion: '2020-05-31',
+    })
+  }
+  async invalidate(subject: string, paths: string[]) {
+    await this.client.createInvalidation({
+      DistributionId: this.distributionId,
+      InvalidationBatch: {
+        CallerReference: `cf-invalidator-${subject}-${Date.now()}`,
+        Paths: {
+          Quantity: paths.length,
+          Items: paths.map((path) => this.pathPrefix + path),
+        },
+      },
+    })
+  }
+}
+
+export default CloudfrontInvalidator
+
+// @NOTE keep in sync with same interface in pds/src/image/invalidator.ts
+// this is separate to avoid the dependency on @atproto/pds.
+interface ImageInvalidator {
+  invalidate(subject: string, paths: string[]): Promise<void>
+}

package/src/index.ts

@@ -0,0 +1,3 @@
+export * from './kms'
+export * from './s3'
+export * from './cloudfront'

package/src/kms.ts

@@ -0,0 +1,66 @@
+import * as aws from '@aws-sdk/client-kms'
+import * as secp from '@noble/secp256k1'
+import * as crypto from '@atproto/crypto'
+import KeyEncoder from 'key-encoder'
+
+const keyEncoder = new KeyEncoder('secp256k1')
+
+export type KmsConfig = { keyId: string } & Omit<
+  aws.KMSClientConfig,
+  'apiVersion'
+>
+
+export class KmsKeypair implements crypto.Keypair {
+  jwtAlg = crypto.SECP256K1_JWT_ALG
+
+  constructor(
+    private client: aws.KMS,
+    private keyId: string,
+    private publicKey: Uint8Array,
+  ) {}
+
+  static async load(cfg: KmsConfig) {
+    const { keyId, ...rest } = cfg
+    const client = new aws.KMS({
+      ...rest,
+      apiVersion: '2014-11-01',
+    })
+    const res = await client.getPublicKey({ KeyId: keyId })
+    if (!res.PublicKey) {
+      throw new Error('Could not find public key')
+    }
+    // public key comes back DER-encoded, so we translate it to raw 65 byte encoding
+    const rawPublicKeyHex = keyEncoder.encodePublic(
+      Buffer.from(res.PublicKey),
+      'der',
+      'raw',
+    )
+    const publicKey = secp.utils.hexToBytes(rawPublicKeyHex)
+    return new KmsKeypair(client, keyId, publicKey)
+  }
+
+  did(): string {
+    return crypto.formatDidKey(this.jwtAlg, this.publicKey)
+  }
+
+  async sign(msg: Uint8Array): Promise<Uint8Array> {
+    const res = await this.client.sign({
+      KeyId: this.keyId,
+      Message: msg,
+      SigningAlgorithm: 'ECDSA_SHA_256',
+    })
+    if (!res.Signature) {
+      throw new Error('Could not get signature')
+    }
+    // signature comes back DER encoded & not-normalized
+    // we translate to raw 64 byte encoding
+    // we also normalize s as no more than 1/2 prime order to pass strict verification
+    // (prevents duplicating a signature)
+    // more: https://github.com/bitcoin-core/secp256k1/blob/a1102b12196ea27f44d6201de4d25926a2ae9640/include/secp256k1.h#L530-L534
+    const sig = secp.Signature.fromDER(res.Signature)
+    const normalized = sig.normalizeS()
+    return normalized.toCompactRawBytes()
+  }
+}
+
+export default KmsKeypair

package/src/s3.ts

@@ -0,0 +1,157 @@
+import * as aws from '@aws-sdk/client-s3'
+import { Upload } from '@aws-sdk/lib-storage'
+import { BlobStore, BlobNotFoundError } from '@atproto/repo'
+import { randomStr } from '@atproto/crypto'
+import { CID } from 'multiformats/cid'
+import stream from 'stream'
+
+export type S3Config = { bucket: string } & Omit<
+  aws.S3ClientConfig,
+  'apiVersion'
+>
+
+// @NOTE we use Upload rather than client.putObject because stream
+// length is not known in advance. See also aws/aws-sdk-js-v3#2348.
+
+export class S3BlobStore implements BlobStore {
+  private client: aws.S3
+  private bucket: string
+
+  constructor(cfg: S3Config) {
+    const { bucket, ...rest } = cfg
+    this.bucket = bucket
+    this.client = new aws.S3({
+      ...rest,
+      apiVersion: '2006-03-01',
+    })
+  }
+
+  private genKey() {
+    return randomStr(32, 'base32')
+  }
+
+  private getTmpPath(key: string): string {
+    return `tmp/${key}`
+  }
+
+  private getStoredPath(cid: CID): string {
+    return `blocks/${cid.toString()}`
+  }
+
+  private getQuarantinedPath(cid: CID): string {
+    return `quarantine/${cid.toString()}`
+  }
+
+  async putTemp(bytes: Uint8Array | stream.Readable): Promise<string> {
+    const key = this.genKey()
+    await new Upload({
+      client: this.client,
+      params: {
+        Bucket: this.bucket,
+        Body: bytes,
+        Key: this.getTmpPath(key),
+      },
+    }).done()
+    return key
+  }
+
+  async makePermanent(key: string, cid: CID): Promise<void> {
+    const alreadyHas = await this.hasStored(cid)
+    if (!alreadyHas) {
+      await this.move({
+        from: this.getTmpPath(key),
+        to: this.getStoredPath(cid),
+      })
+    } else {
+      // already saved, so we no-op & just delete the temp
+      await this.deleteKey(this.getTmpPath(key))
+    }
+  }
+
+  async putPermanent(
+    cid: CID,
+    bytes: Uint8Array | stream.Readable,
+  ): Promise<void> {
+    await new Upload({
+      client: this.client,
+      params: {
+        Bucket: this.bucket,
+        Body: bytes,
+        Key: this.getStoredPath(cid),
+      },
+    }).done()
+  }
+
+  async quarantine(cid: CID): Promise<void> {
+    await this.move({
+      from: this.getStoredPath(cid),
+      to: this.getQuarantinedPath(cid),
+    })
+  }
+
+  async unquarantine(cid: CID): Promise<void> {
+    await this.move({
+      from: this.getQuarantinedPath(cid),
+      to: this.getStoredPath(cid),
+    })
+  }
+
+  private async getObject(cid: CID) {
+    const res = await this.client.getObject({
+      Bucket: this.bucket,
+      Key: this.getStoredPath(cid),
+    })
+    if (res.Body) {
+      return res.Body
+    } else {
+      throw new BlobNotFoundError()
+    }
+  }
+
+  async getBytes(cid: CID): Promise<Uint8Array> {
+    const res = await this.getObject(cid)
+    return res.transformToByteArray()
+  }
+
+  async getStream(cid: CID): Promise<stream.Readable> {
+    const res = await this.getObject(cid)
+    return res as stream.Readable
+  }
+
+  async delete(cid: CID): Promise<void> {
+    await this.deleteKey(this.getStoredPath(cid))
+  }
+
+  async hasStored(cid: CID): Promise<boolean> {
+    try {
+      const res = await this.client.headObject({
+        Bucket: this.bucket,
+        Key: this.getStoredPath(cid),
+      })
+      return res.$metadata.httpStatusCode === 200
+    } catch (err) {
+      return false
+    }
+  }
+
+  private async deleteKey(key: string) {
+    await this.client.deleteObject({
+      Bucket: this.bucket,
+      Key: key,
+    })
+  }
+
+  private async move(keys: { from: string; to: string }) {
+    await this.client.copyObject({
+      Bucket: this.bucket,
+      CopySource: `${this.bucket}/${keys.from}`,
+      Key: keys.to,
+    })
+    await this.client.deleteObject({
+      Bucket: this.bucket,
+      Key: keys.from,
+    })
+  }
+}
+
+export default S3BlobStore

package/tsconfig.build.json

@@ -0,0 +1,4 @@
+{
+  "extends": "./tsconfig.json",
+  "exclude": ["**/*.spec.ts", "**/*.test.ts"]
+}

package/tsconfig.json

@@ -0,0 +1,8 @@
+{
+  "extends": "../../tsconfig.json",
+  "compilerOptions": {
+    "outDir": "./dist", // Your outDir,
+    "emitDeclarationOnly": true
+  },
+  "include": ["./src","__tests__/**/**.ts"]
+}