@kutur-kit/assets 0.0.0-stage → 0.1.0-alpha.0

package/src/internal/sigv4.ts

@@ -0,0 +1,84 @@
+import type * as AssetSigning from "../AssetSigning.ts";
+
+import * as Credentials from "@distilled.cloud/aws/Credentials";
+import * as Presign from "@distilled.cloud/aws/Presign";
+import * as Region from "@distilled.cloud/aws/Region";
+import { Clock, DateTime, Duration, Effect } from "effect";
+
+import * as AssetBucket from "../AssetBucket.ts";
+import { encodeKey } from "./url.ts";
+
+/** Where signed URLs point: path-style, `${endpoint}/${bucket}/${key}`. */
+export interface Target {
+  readonly endpoint: string;
+  readonly bucket: string;
+}
+
+// The longest S3 allows for a presigned URL.
+const maxExpiry = Duration.days(7);
+
+/**
+ * Signs with SigV4 for the S3 API. The target and the `Credentials` are
+ * resolved for each request, so stores that deliver them at runtime, such as
+ * alchemy's R2 bindings, can sign too.
+ */
+export const make = <R>(
+  bucketId: string,
+  target: Effect.Effect<Target, never, R>,
+): Effect.Effect<AssetSigning.Signing, never, Credentials.Credentials | R> =>
+  Effect.map(Effect.context<Credentials.Credentials | R>(), (context) => {
+    const sign = (
+      method: "PUT" | "GET",
+      key: string,
+      expiresIn: Duration.Duration,
+      signed: Readonly<Record<string, string>>,
+    ) =>
+      Effect.gen(function* () {
+        const { endpoint, bucket } = yield* target;
+        const credentials = yield* yield* Credentials.Credentials;
+        const now = yield* Clock.currentTimeMillis;
+        const seconds = Math.ceil(Duration.toMillis(Duration.min(expiresIn, maxExpiry)) / 1000);
+
+        const url = yield* Presign.presignUrl({
+          method,
+          url: `${endpoint.replace(/\/+$/u, "")}/${bucket}/${encodeKey(key)}`,
+          service: "s3",
+          region: credentials.region,
+          expiresIn: seconds,
+          headers: signed,
+          // The signing time comes from Effect's clock, as the expiry does.
+          datetime: new Date(now).toISOString().replace(/[-:]|\.\d{3}/gu, ""),
+        }).pipe(Effect.provideService(Region.Region, Effect.succeed(credentials.region)));
+
+        return {
+          method,
+          url,
+          // Browsers set Content-Length from the body; the rest is sent as given.
+          headers: Object.fromEntries(
+            Object.entries(signed).filter(([name]) => name !== "content-length"),
+          ),
+          expiresAt: DateTime.makeUnsafe(now + seconds * 1000),
+        };
+      }).pipe(
+        Effect.provideContext(context),
+        Effect.mapError(
+          (cause) =>
+            new AssetBucket.AssetBucketError({ operation: "sign", bucket: bucketId, key, cause }),
+        ),
+      );
+
+    return {
+      // Content-Length is signed: without it, the storage accepts any size.
+      // `If-None-Match: *` makes the URL write-once: replayed after its
+      // upload, even with other bytes of the same length, it is refused.
+      signPut: ({ key, contentType, contentLength, cacheControl, expiresIn }) =>
+        sign("PUT", key, expiresIn, {
+          "content-type": contentType,
+          "content-length": String(contentLength),
+          "if-none-match": "*",
+          // Signed, or browsers' `no-cache` for conditional requests is stored.
+          "cache-control": cacheControl,
+        }),
+      signGet: ({ key, expiresIn }) => sign("GET", key, expiresIn, {}),
+    };
+  });

package/src/internal/url.ts

@@ -0,0 +1,2 @@
+/** A key in a URL path: segments encoded, slashes kept. */
+export const encodeKey = (key: string) => key.split("/").map(encodeURIComponent).join("/");

package/src/memory.ts

@@ -0,0 +1,65 @@
+import type * as Asset from "./Asset.ts";
+import type * as AssetBucket from "./AssetBucket.ts";
+
+import { ByteSize, Effect, Layer, Option, Stream } from "effect";
+
+import { concat } from "./internal/bytes.ts";
+
+interface Entry {
+  readonly info: AssetBucket.Info;
+  readonly bytes: Uint8Array;
+}
+
+/**
+ * Backs a bucket with memory for the lifetime of the layer. Bodies are copied
+ * in and out, so callers cannot change stored bytes.
+ */
+export const layer = <Self, Id extends string>(
+  bucket: AssetBucket.Tag<Self, Id>,
+): Layer.Layer<Self> =>
+  Layer.sync(bucket, () => {
+    const objects = new Map<string, Entry>();
+
+    return bucket.of({
+      put: ({ key, contentType, cacheControl, body }) =>
+        Stream.runCollect(body).pipe(
+          Effect.map((chunks) => {
+            const bytes = concat(chunks);
+            const info = { size: ByteSize.bytes(bytes.length), contentType, cacheControl };
+
+            objects.set(key, { info, bytes });
+
+            return info;
+          }),
+        ),
+      get: (key) =>
+        Effect.sync(() =>
+          Option.map(Option.fromUndefinedOr(objects.get(key)), ({ info, bytes }) => ({
+            info,
+            body: Stream.succeed(bytes.slice()),
+          })),
+        ),
+      head: (key) =>
+        Effect.sync(() => Option.map(Option.fromUndefinedOr(objects.get(key)), ({ info }) => info)),
+      delete: (key) =>
+        Effect.sync(() => {
+          objects.delete(key);
+        }),
+    });
+  });
+
+/** Backs the buckets of the given assets with memory, one store per bucket. */
+export const toLayer = <const Assets extends ReadonlyArray<Asset.Any>>(
+  assets: Assets,
+): Layer.Layer<Asset.BucketOf<Assets[number]>> => {
+  // SAFETY: asset buckets are tags from AssetBucket.Service.
+  const buckets = new Set(assets.map((asset) => asset.bucket as AssetBucket.Tag<unknown>));
+
+  const merged = [...buckets].reduce<Layer.Layer<never>>(
+    (layers, bucket) => Layer.merge(layers, layer(bucket)),
+    Layer.empty,
+  );
+
+  // SAFETY: the merged layers provide exactly the buckets of the listed assets.
+  return merged as Layer.Layer<Asset.BucketOf<Assets[number]>>;
+};

package/src/s3.ts

@@ -0,0 +1,138 @@
+import type * as AssetSigning from "./AssetSigning.ts";
+import type * as Credentials from "@distilled.cloud/aws/Credentials";
+import type * as HttpClient from "effect/http/HttpClient";
+
+import * as S3 from "@distilled.cloud/aws/s3";
+import { ByteSize, Effect, Layer, Option, Stream } from "effect";
+
+import * as AssetBucket from "./AssetBucket.ts";
+import * as AssetSigningModule from "./AssetSigning.ts";
+import { concat } from "./internal/bytes.ts";
+import * as SigV4 from "./internal/sigv4.ts";
+
+export interface Options {
+  /** The bucket's name in the storage. */
+  readonly bucket: string;
+}
+
+export interface SigningOptions extends Options {
+  /**
+   * Where clients reach the storage, such as
+   * `https://<account>.r2.cloudflarestorage.com` or `http://localhost:9000`.
+   * Signed URLs are path-style under it: `${endpoint}/${bucket}/${key}`.
+   */
+  readonly endpoint: string;
+}
+
+type Requirements = Credentials.Credentials | HttpClient.HttpClient;
+
+const failure =
+  (bucket: string, operation: AssetBucket.AssetBucketError["operation"], key: string) =>
+  (cause: unknown) =>
+    new AssetBucket.AssetBucketError({ operation, bucket, key, cause });
+
+/**
+ * Backs a bucket with S3 or a store with its API, such as R2 or RustFS,
+ * through `@distilled.cloud/aws`. Provide its `Credentials` and an
+ * `HttpClient`, and an `Endpoint` for stores other than AWS.
+ *
+ * Bodies are collected before they are sent, so a failing body sends nothing.
+ * Uploads through the app are small by `maxSize`; larger files go to storage
+ * directly, through `signing`.
+ */
+export const layer = <Self, Id extends string>(
+  bucket: AssetBucket.Tag<Self, Id>,
+  options: Options,
+): Layer.Layer<Self, never, Requirements> =>
+  Layer.effect(
+    bucket,
+    Effect.gen(function* () {
+      // S3 reads the optional `Endpoint` and `Region` when called, so the
+      // whole context is kept, not just the required services.
+      const context = yield* Effect.context<Requirements>();
+      const Bucket = options.bucket;
+
+      const fail = (operation: AssetBucket.AssetBucketError["operation"], key: string) =>
+        failure(bucket.id, operation, key);
+
+      return bucket.of({
+        put: ({ key, contentType, cacheControl, body }) =>
+          Effect.gen(function* () {
+            const bytes = concat(yield* Stream.runCollect(body));
+
+            const request: S3.PutObjectRequest = {
+              Bucket,
+              Key: key,
+              Body: bytes,
+              ContentType: contentType,
+            };
+
+            if (cacheControl !== undefined) request.CacheControl = cacheControl;
+
+            yield* S3.putObject(request).pipe(
+              Effect.provideContext(context),
+              Effect.mapError(fail("put", key)),
+            );
+
+            return { size: ByteSize.bytes(bytes.length), contentType, cacheControl };
+          }),
+        get: (key) =>
+          S3.getObject({ Bucket, Key: key }).pipe(
+            Effect.map((object) =>
+              Option.some({
+                info: {
+                  size: ByteSize.bytes(object.ContentLength ?? 0),
+                  contentType: object.ContentType ?? "application/octet-stream",
+                  cacheControl: object.CacheControl,
+                },
+                body:
+                  object.Body === undefined
+                    ? Stream.empty
+                    : Stream.mapError(object.Body, fail("get", key)),
+              }),
+            ),
+            Effect.catchTag("NoSuchKey", () => Effect.succeedNone),
+            Effect.provideContext(context),
+            Effect.mapError(fail("get", key)),
+          ),
+        head: (key) =>
+          S3.headObject({ Bucket, Key: key }).pipe(
+            Effect.map((object) =>
+              Option.some({
+                size: ByteSize.bytes(object.ContentLength ?? 0),
+                contentType: object.ContentType ?? "application/octet-stream",
+                cacheControl: object.CacheControl,
+              }),
+            ),
+            Effect.catchTag("NotFound", () => Effect.succeedNone),
+            Effect.provideContext(context),
+            Effect.mapError(fail("head", key)),
+          ),
+        delete: (key) =>
+          S3.deleteObject({ Bucket, Key: key }).pipe(
+            Effect.asVoid,
+            Effect.provideContext(context),
+            Effect.mapError(fail("delete", key)),
+          ),
+      });
+    }),
+  );
+
+/**
+ * Signs URLs for a bucket with SigV4, for clients to upload to and download
+ * from the storage directly. Uploads are signed with their content type and
+ * length, the cache control to store, and to create an object only, which
+ * the storage enforces.
+ */
+export const signing = <Self, Id extends string>(
+  bucket: AssetBucket.Tag<Self, Id>,
+  options: SigningOptions,
+): Layer.Layer<AssetSigning.SigningOf<Self>, never, Credentials.Credentials> =>
+  Layer.effect(AssetSigningModule.of(bucket), SigV4.make(bucket.id, Effect.succeed(options)));
+
+/**
+ * R2's S3 endpoint for an account, for `signing` and an S3 `Endpoint`.
+ * Requests to it are signed for the region `auto`.
+ */
+export const r2Endpoint = (accountId: string, jurisdiction?: "eu" | "fedramp") =>
+  `https://${accountId}.${jurisdiction === undefined ? "" : `${jurisdiction}.`}r2.cloudflarestorage.com`;

package/src/signing/local.ts

@@ -0,0 +1,210 @@
+import type * as AssetBucket from "../AssetBucket.ts";
+
+import {
+  ByteSize,
+  Clock,
+  DateTime,
+  Duration,
+  Effect,
+  Layer,
+  Option,
+  Redacted,
+  Stream,
+} from "effect";
+import { HttpRouter, HttpServerResponse } from "effect/http";
+
+import * as AssetSigning from "../AssetSigning.ts";
+import { mount, respond, target } from "../internal/http.ts";
+import { encodeKey } from "../internal/url.ts";
+
+export interface Options {
+  /**
+   * Where `routes` are mounted, such as `/storage/private` or
+   * `http://localhost:3000/storage/private`. Signed URLs start with it.
+   */
+  readonly baseUrl: string;
+  /** Signs URLs. Changing it invalidates URLs already handed out. */
+  readonly secret: Redacted.Redacted<string>;
+}
+
+interface Signed {
+  readonly method: "PUT" | "GET";
+  readonly key: string;
+  readonly expires: number;
+  readonly contentType?: string | undefined;
+  readonly contentLength?: string | undefined;
+  /** `*` on uploads: they create an object and never replace one. */
+  readonly ifNoneMatch?: string | undefined;
+  readonly cacheControl?: string | undefined;
+}
+
+const hex = (bytes: ArrayBuffer) =>
+  Array.from(new Uint8Array(bytes), (byte) => byte.toString(16).padStart(2, "0")).join("");
+
+// The signature covers everything a client must not change.
+const sign = (secret: Redacted.Redacted<string>, signed: Signed) =>
+  Effect.promise(async () => {
+    const key = await crypto.subtle.importKey(
+      "raw",
+      new TextEncoder().encode(Redacted.value(secret)),
+      { name: "HMAC", hash: "SHA-256" },
+      false,
+      ["sign"],
+    );
+
+    const message = [
+      signed.method,
+      signed.key,
+      signed.expires,
+      signed.contentType ?? "",
+      signed.contentLength ?? "",
+      signed.ifNoneMatch ?? "",
+      signed.cacheControl ?? "",
+    ].join("\n");
+
+    return hex(await crypto.subtle.sign("HMAC", key, new TextEncoder().encode(message)));
+  });
+
+// Compares in time independent of where the strings differ.
+const equal = (a: string, b: string) => {
+  let difference = a.length ^ b.length;
+
+  for (let index = 0; index < Math.max(a.length, b.length); index++)
+    difference |= (a.charCodeAt(index) || 0) ^ (b.charCodeAt(index) || 0);
+
+  return difference === 0;
+};
+
+/**
+ * Signs URLs for a bucket with an HMAC, for development, tests or a single
+ * self-hosted server. `routes` verifies them, so storage needs no signing.
+ */
+export const layer = <Self, Id extends string>(
+  bucket: AssetBucket.Tag<Self, Id>,
+  options: Options,
+): Layer.Layer<AssetSigning.SigningOf<Self>> => {
+  const request = (signed: Omit<Signed, "expires">, expiresIn: Duration.Duration) =>
+    Effect.gen(function* () {
+      const now = yield* Clock.currentTimeMillis;
+      const expires = Math.ceil((now + Duration.toMillis(expiresIn)) / 1000);
+      const signature = yield* sign(options.secret, { ...signed, expires });
+      const query = new URLSearchParams({ expires: String(expires), signature });
+      const headers: Record<string, string> = {};
+
+      if (signed.contentType !== undefined) headers["content-type"] = signed.contentType;
+
+      if (signed.ifNoneMatch !== undefined) headers["if-none-match"] = signed.ifNoneMatch;
+
+      if (signed.cacheControl !== undefined) headers["cache-control"] = signed.cacheControl;
+
+      return {
+        method: signed.method,
+        url: `${options.baseUrl}/${encodeKey(signed.key)}?${query.toString()}`,
+        headers,
+        expiresAt: DateTime.makeUnsafe(expires * 1000),
+      };
+    });
+
+  return Layer.succeed(AssetSigning.of(bucket), {
+    signPut: ({ key, contentType, contentLength, cacheControl, expiresIn }) =>
+      request(
+        {
+          method: "PUT",
+          key,
+          contentType,
+          contentLength: String(contentLength),
+          ifNoneMatch: "*",
+          cacheControl,
+        },
+        expiresIn,
+      ),
+    signGet: ({ key, expiresIn }) => request({ method: "GET", key }, expiresIn),
+  });
+};
+
+const refused = HttpServerResponse.text("Forbidden", { status: 403 });
+
+/**
+ * Accepts signed uploads and serves signed downloads of a bucket under
+ * `baseUrl`. Refuses a request that differs from what was signed: another
+ * method, key, content type, length or cache control, or one sent after it
+ * expires. An upload to a key that already holds an object is refused with
+ * 412.
+ */
+export const routes = <Self, Id extends string>(
+  bucket: AssetBucket.Tag<Self, Id>,
+  options: Options,
+) => {
+  const { prefix, path } = mount(options.baseUrl);
+
+  const verify = (query: URLSearchParams, signed: Omit<Signed, "expires">) =>
+    Effect.gen(function* () {
+      const expires = Number(query.get("expires"));
+      const signature = query.get("signature") ?? "";
+      const now = yield* Clock.currentTimeMillis;
+      const expected = yield* sign(options.secret, { ...signed, expires });
+
+      return now <= expires * 1000 && equal(signature, expected);
+    });
+
+  const upload = Effect.gen(function* () {
+    const { request, key, query } = yield* target(prefix);
+    const contentType = request.headers["content-type"];
+    const contentLength = request.headers["content-length"];
+    const ifNoneMatch = request.headers["if-none-match"];
+    const cacheControl = request.headers["cache-control"];
+
+    // The signed length is required, so the body's is checked against it.
+    const length = Option.flatMap(Option.fromUndefinedOr(contentLength), (header) =>
+      ByteSize.fromInput(Number(header)),
+    );
+
+    const allowed =
+      Option.isSome(key) &&
+      Option.isSome(length) &&
+      (yield* verify(query, {
+        method: "PUT",
+        key: key.value,
+        contentType,
+        contentLength,
+        ifNoneMatch,
+        cacheControl,
+      }));
+
+    if (!allowed || Option.isNone(key) || Option.isNone(length)) return refused;
+
+    const store = yield* bucket;
+
+    // A signed upload writes once, as S3 answers `If-None-Match: *`. Checked
+    // before writing, so two uploads racing to one key may both succeed: these
+    // routes serve development and tests.
+    const existing = yield* store.head(key.value);
+
+    if (Option.isSome(existing))
+      return HttpServerResponse.text("Precondition Failed", { status: 412 });
+
+    // Node ends the body at its Content-Length; a client that sends less
+    // fails the stream, so the bucket keeps what it had.
+    yield* store.put({
+      key: key.value,
+      contentType: contentType ?? "",
+      cacheControl,
+      body: Stream.orDie(request.stream),
+      contentLength: length.value,
+    });
+
+    return HttpServerResponse.empty({ status: 200 });
+  }).pipe(Effect.orDie);
+
+  const download = Effect.gen(function* () {
+    const { key, query } = yield* target(prefix);
+
+    const allowed = Option.isSome(key) && (yield* verify(query, { method: "GET", key: key.value }));
+
+    if (!allowed || Option.isNone(key)) return refused;
+
+    return yield* respond(yield* bucket, key.value);
+  }).pipe(Effect.orDie);
+
+  return Layer.mergeAll(HttpRouter.add("PUT", path, upload), HttpRouter.add("GET", path, download));
+};