@kutur-kit/assets 0.0.0-stage → 0.1.0-alpha.0

package/src/AssetSigning.ts

@@ -0,0 +1,206 @@
+import type * as Asset from "./Asset.ts";
+import type * as AssetBucket from "./AssetBucket.ts";
+
+import { ByteSize, Context, Duration, Effect, Option, Schema, SchemaGetter } from "effect";
+
+/**
+ * A request a client sends to storage itself, such as a browser uploading a
+ * file without passing it through the app. A schema, so APIs can return it.
+ */
+export const SignedRequest = Schema.Struct({
+  method: Schema.Literals(["PUT", "GET"]),
+  url: Schema.String,
+  /**
+   * Headers the client must send as given: they are part of the signature.
+   * Uploads use only `uploadHeaders`.
+   */
+  headers: Schema.Record(Schema.String, Schema.String),
+  expiresAt: Schema.DateTimeUtc,
+});
+
+export type SignedRequest = typeof SignedRequest.Type;
+
+/**
+ * The headers a signed upload may carry. Browsers send a CORS preflight
+ * before uploading with them to storage on another origin, so the bucket's
+ * CORS rules must allow them, not only `content-type`:
+ *
+ * ```ts
+ * { allowedOrigins: [appOrigin], allowedMethods: ["PUT"], allowedHeaders: [...AssetSigning.uploadHeaders] }
+ * ```
+ */
+export const uploadHeaders = ["content-type", "if-none-match", "cache-control"] as const;
+
+export interface SignPutInput {
+  readonly key: string;
+  readonly contentType: string;
+  /** The exact length the upload must have. */
+  readonly contentLength: ByteSize.ByteSize;
+  /** Stored with the object; the client must send it as given. */
+  readonly cacheControl: string;
+  readonly expiresIn: Duration.Duration;
+}
+
+export interface SignGetInput {
+  readonly key: string;
+  readonly expiresIn: Duration.Duration;
+}
+
+/**
+ * Signs requests that clients send to a bucket's storage directly. Storage
+ * must refuse a signed request that differs from what was signed: another
+ * method, key, content type, length or cache control, or one sent after it
+ * expires. Signed uploads create an object and never replace one, so a URL
+ * replayed after its upload is refused.
+ */
+export interface Signing {
+  readonly signPut: (
+    input: SignPutInput,
+  ) => Effect.Effect<SignedRequest, AssetBucket.AssetBucketError>;
+  readonly signGet: (
+    input: SignGetInput,
+  ) => Effect.Effect<SignedRequest, AssetBucket.AssetBucketError>;
+}
+
+/** The service a bucket's `Signing` is provided as. */
+export interface SigningOf<Self> {
+  readonly "~@kutur-kit/assets/AssetSigning": Self;
+}
+
+/** The key a bucket's `Signing` is provided and required by. */
+export const of = <Self, Id extends string>(
+  bucket: AssetBucket.Tag<Self, Id>,
+): Context.Service<SigningOf<Self>, Signing> =>
+  Context.Service<SigningOf<Self>, Signing>(`@kutur-kit/assets/AssetSigning/${bucket.id}`);
+
+/**
+ * The stored object does not match the upload it should hold: it is missing,
+ * or its size or content type differs from what the client declared.
+ */
+export class UploadNotConfirmed extends Schema.TaggedError<
+  UploadNotConfirmed,
+  { readonly brand: unique symbol }
+>()(
+  "AssetUploadNotConfirmed",
+  { asset: Schema.String, reason: Schema.Literals(["missing", "size", "contentType"]) },
+  { httpApiStatus: 409 },
+) {}
+
+/** An upload a client declares before sending it: its content type and size. */
+export interface Upload<Type extends Asset.MediaType> {
+  readonly contentType: Type;
+  readonly size: ByteSize.ByteSize;
+}
+
+// A size in a JSON body: a whole number of bytes.
+const Size = Schema.Int.check(Schema.isGreaterThanOrEqualTo(0)).pipe(
+  Schema.decodeTo(Schema.ByteSize, {
+    decode: SchemaGetter.transform((size: number) => ByteSize.bytes(size)),
+    encode: SchemaGetter.transform((size: ByteSize.ByteSize) => Number(size)),
+  }),
+);
+
+/** The schema of an upload declaration, a struct apps can extend with its `fields`. */
+export type UploadSchema<Type extends Asset.MediaType> = Schema.Struct<{
+  readonly contentType: Schema.Codec<Type, string>;
+  readonly size: typeof Size;
+}>;
+
+/**
+ * Decodes an upload a client declares, such as a request body, with a content
+ * type the asset accepts. Sizes over its `maxSize` are refused by
+ * `presignPut`, as `Asset.TooLarge`. Extend it for requests that carry more:
+ *
+ * ```ts
+ * Schema.Struct({ ...AssetSigning.Upload(ProjectFile).fields, name: Schema.String })
+ * ```
+ */
+export const Upload = <Type extends Asset.MediaType, K, B>(
+  asset: Asset.Asset<string, Type, K, B>,
+): UploadSchema<Type> => Schema.Struct({ contentType: asset.ContentType, size: Size });
+
+export interface SignOptions {
+  /** How long the URL stays valid. 15 minutes by default. */
+  readonly expiresIn?: Duration.Input | undefined;
+}
+
+const expiry = (options: SignOptions | undefined) =>
+  Duration.fromInputUnsafe(options?.expiresIn ?? "15 minutes");
+
+/**
+ * Signs a URL the client uploads the object to, bound to its content type,
+ * size and the asset's cache control.
+ */
+export const presignPut = <Type extends Asset.MediaType, K, B>(
+  asset: Asset.Asset<string, Type, K, B>,
+  key: K,
+  upload: Upload<Type>,
+  options?: SignOptions,
+): Effect.Effect<
+  SignedRequest,
+  Asset.InvalidKey | Asset.UnsupportedContentType | Asset.TooLarge | AssetBucket.AssetBucketError,
+  SigningOf<B>
+> =>
+  Effect.gen(function* () {
+    const stored = yield* asset.storedKey(key);
+
+    yield* asset.check({ contentType: upload.contentType, contentLength: upload.size });
+
+    const signing = yield* of(asset.bucket);
+
+    return yield* signing.signPut({
+      key: stored,
+      contentType: upload.contentType,
+      contentLength: upload.size,
+      cacheControl: asset.cacheControl,
+      expiresIn: expiry(options),
+    });
+  });
+
+/** Signs a URL the client downloads the object from. */
+export const presignGet = <Type extends Asset.MediaType, K, B>(
+  asset: Asset.Asset<string, Type, K, B>,
+  key: K,
+  options?: SignOptions,
+): Effect.Effect<SignedRequest, Asset.InvalidKey | AssetBucket.AssetBucketError, SigningOf<B>> =>
+  Effect.gen(function* () {
+    const stored = yield* asset.storedKey(key);
+    const signing = yield* of(asset.bucket);
+
+    return yield* signing.signGet({ key: stored, expiresIn: expiry(options) });
+  });
+
+/**
+ * Checks that a client's upload arrived as declared, before the app records
+ * it: the object exists with the declared size and content type.
+ */
+export const confirm = <Type extends Asset.MediaType, K, B>(
+  asset: Asset.Asset<string, Type, K, B>,
+  key: K,
+  upload: Upload<Type>,
+): Effect.Effect<
+  Asset.Info<Type>,
+  Asset.InvalidKey | UploadNotConfirmed | AssetBucket.AssetBucketError,
+  B
+> =>
+  Effect.gen(function* () {
+    const stored = yield* asset.storedKey(key);
+    const bucket = yield* asset.bucket;
+    const info = yield* bucket.head(stored);
+
+    const reason = Option.match(info, {
+      onNone: () => Option.some("missing" as const),
+      onSome: ({ size, contentType }) =>
+        size !== upload.size
+          ? Option.some("size" as const)
+          : contentType === upload.contentType
+            ? Option.none()
+            : Option.some("contentType" as const),
+    });
+
+    return yield* Option.match(reason, {
+      onNone: () => Effect.succeed({ size: upload.size, contentType: upload.contentType }),
+      onSome: (because) =>
+        Effect.fail(new UploadNotConfirmed({ asset: asset.id, reason: because })),
+    });
+  });

package/src/alchemy/r2.ts

@@ -0,0 +1,151 @@
+import type * as AssetSigning from "../AssetSigning.ts";
+import type { Bucket, BucketCorsRule } from "alchemy/Cloudflare/R2/Bucket";
+import type { PutOptions, R2Object } from "alchemy/Cloudflare/R2/BucketTypes";
+import type { WorkerEnvironment } from "alchemy/Cloudflare/Workers/Worker";
+
+import * as Credentials from "@distilled.cloud/aws/Credentials";
+import { ReadWriteBucket } from "alchemy/Cloudflare/R2/ReadWriteBucket";
+import { S3Credentials } from "alchemy/Cloudflare/R2/S3Credentials";
+import { RuntimeContext } from "alchemy/RuntimeContext";
+import { ByteSize, Effect, Layer, Option, Stream } from "effect";
+
+import * as AssetBucket from "../AssetBucket.ts";
+import * as AssetSigningModule from "../AssetSigning.ts";
+import { concat } from "../internal/bytes.ts";
+import * as SigV4 from "../internal/sigv4.ts";
+
+/** An R2 bucket resource, or the Effect that declares it. */
+export type R2Bucket<R = never> = Bucket | Effect.Effect<Bucket, never, R>;
+
+// Alchemy marks binding methods as callable only while handling a request.
+// `Bucket` methods carry no requirements, so the marker is satisfied here,
+// as alchemy does where it implements other interfaces. The methods read
+// nothing from it, but they fail if called while a Worker is constructed.
+const handling = <A, E>(effect: Effect.Effect<A, E, RuntimeContext>): Effect.Effect<A, E> =>
+  Effect.provide(effect, RuntimeContext.phantom);
+
+/** The HTTP metadata R2 stores with an object and serves with it. */
+type HttpMetadata = Exclude<NonNullable<PutOptions["httpMetadata"]>, Headers>;
+
+const infoOf = (object: R2Object): AssetBucket.Info => ({
+  size: ByteSize.bytes(object.size),
+  contentType: object.httpMetadata?.contentType ?? "application/octet-stream",
+  cacheControl: object.httpMetadata?.cacheControl,
+});
+
+/**
+ * Backs a bucket with an R2 bucket through alchemy's `ReadWriteBucket`.
+ * Provide its implementation for where the app runs: `ReadWriteBucketBinding`
+ * in a Worker, `ReadWriteBucketHttp` elsewhere alchemy deploys. Content type
+ * and cache control are stored as the object's HTTP metadata, which R2
+ * serves on its public domains.
+ *
+ * Bodies are collected before they are sent, so a failing body sends
+ * nothing. Uploads through the app are small by `maxSize`; larger files go
+ * to storage directly, through `signing`.
+ */
+export const layer = <Self, Id extends string, R = never>(
+  bucket: AssetBucket.Tag<Self, Id>,
+  r2: R2Bucket<R>,
+): Layer.Layer<Self, never, ReadWriteBucket | R> =>
+  Layer.effect(
+    bucket,
+    Effect.gen(function* () {
+      const client = yield* ReadWriteBucket(r2);
+
+      const fail = (operation: AssetBucket.AssetBucketError["operation"], key: string) =>
+        Effect.mapError(
+          (cause: unknown) =>
+            new AssetBucket.AssetBucketError({ operation, bucket: bucket.id, key, cause }),
+        );
+
+      return bucket.of({
+        put: ({ key, contentType, cacheControl, body }) =>
+          Effect.gen(function* () {
+            const bytes = concat(yield* Stream.runCollect(body));
+            const httpMetadata: HttpMetadata = { contentType };
+
+            if (cacheControl !== undefined) httpMetadata.cacheControl = cacheControl;
+
+            const object = yield* handling(client.put(key, bytes, { httpMetadata })).pipe(
+              fail("put", key),
+            );
+
+            return { ...infoOf(object), contentType, cacheControl };
+          }),
+        get: (key) =>
+          handling(client.get(key)).pipe(
+            Effect.map((object) =>
+              object === null
+                ? Option.none()
+                : Option.some({
+                    info: infoOf(object),
+                    body: Stream.mapError(
+                      object.body,
+                      (cause) =>
+                        new AssetBucket.AssetBucketError({
+                          operation: "get",
+                          bucket: bucket.id,
+                          key,
+                          cause,
+                        }),
+                    ),
+                  }),
+            ),
+            fail("get", key),
+          ),
+        head: (key) =>
+          handling(client.head(key)).pipe(
+            Effect.map((object) => (object === null ? Option.none() : Option.some(infoOf(object)))),
+            fail("head", key),
+          ),
+        delete: (key) => handling(client.delete(key)).pipe(fail("delete", key)),
+      });
+    }),
+  );
+
+/**
+ * Signs URLs for an R2 bucket from a Worker, with the S3 credentials alchemy
+ * derives for it: a token scoped to the bucket when deployed, the local S3
+ * endpoint under `alchemy dev`. Signed as `S3Assets.signing` signs: uploads
+ * are bound to their content type, length and cache control, and write once.
+ *
+ * Outside a Worker, use `S3Assets.signing` with `S3Assets.r2Endpoint` and
+ * the S3 keys of an R2 API token.
+ */
+export const signing = <Self, Id extends string, R = never>(
+  bucket: AssetBucket.Tag<Self, Id>,
+  r2: R2Bucket<R>,
+): Layer.Layer<AssetSigning.SigningOf<Self>, never, WorkerEnvironment | R> =>
+  Layer.effect(
+    AssetSigningModule.of(bucket),
+    Effect.gen(function* () {
+      const credentials = handling(yield* S3Credentials(r2, { access: "read-write" }));
+
+      return yield* SigV4.make(
+        bucket.id,
+        Effect.map(credentials, ({ endpoint, bucketName }) => ({ endpoint, bucket: bucketName })),
+      ).pipe(
+        Effect.provideService(
+          Credentials.Credentials,
+          Effect.map(credentials, ({ accessKeyId, secretAccessKey, region }) =>
+            Credentials.fromAwsCredentialIdentity({ accessKeyId, secretAccessKey }, region),
+          ),
+        ),
+      );
+    }),
+  );
+
+/**
+ * The CORS rule an R2 bucket needs for browsers on `origins` to upload to it
+ * with signed URLs: `PUT` with `AssetSigning.uploadHeaders`.
+ *
+ * ```ts
+ * Cloudflare.R2.Bucket("Uploads", { cors: [AssetsR2.corsRule(["https://app.example.com"])] })
+ * ```
+ */
+export const corsRule = (origins: ReadonlyArray<string>): BucketCorsRule => ({
+  allowedMethods: ["PUT"],
+  allowedOrigins: [...origins],
+  allowedHeaders: [...AssetSigningModule.uploadHeaders],
+});

package/src/delivery/local.ts

@@ -0,0 +1,41 @@
+import type * as AssetBucket from "../AssetBucket.ts";
+
+import { Effect, Option } from "effect";
+import { HttpRouter, HttpServerResponse } from "effect/http";
+
+import { mount, respond, target } from "../internal/http.ts";
+
+export interface Options {
+  /**
+   * Where `routes` are mounted, such as `/cdn/public` or
+   * `http://localhost:3000/cdn/public`. Give `AssetDelivery.layer` the same
+   * base URL, so the public URLs it makes reach the routes.
+   */
+  readonly baseUrl: string;
+}
+
+/**
+ * Serves every object of a bucket under `baseUrl` without a signature, as a
+ * CDN serves a public bucket, for development and tests. Objects are served
+ * with the cache control stored with them, as R2 and S3 serve it.
+ */
+export const routes = <Self, Id extends string>(
+  bucket: AssetBucket.Tag<Self, Id>,
+  options: Options,
+) => {
+  const { prefix, path } = mount(options.baseUrl);
+
+  return HttpRouter.add(
+    "GET",
+    path,
+    Effect.gen(function* () {
+      const { key } = yield* target(prefix);
+      const store = yield* bucket;
+
+      return yield* Option.match(key, {
+        onNone: () => Effect.succeed(HttpServerResponse.empty({ status: 404 })),
+        onSome: (value) => respond(store, value),
+      });
+    }).pipe(Effect.orDie),
+  );
+};

package/src/filesystem.ts

@@ -0,0 +1,166 @@
+import { Effect, FileSystem, Layer, Option, Path, Schema, Stream } from "effect";
+
+import * as AssetBucket from "./AssetBucket.ts";
+
+export interface Options {
+  /** The directory holding the bucket's files. It is created when needed. */
+  readonly directory: string;
+}
+
+// What a stored object's record holds. `file` names the body's file, a new
+// one for each write, so a replacement is visible only once its record is.
+const Entry = Schema.Struct({
+  key: Schema.String,
+  contentType: Schema.String,
+  cacheControl: Schema.optional(Schema.String),
+  size: Schema.ByteSize,
+  file: Schema.String,
+});
+
+const EntryJson = Schema.fromJsonString(Schema.toCodecJson(Entry));
+
+const infoOf = (entry: typeof Entry.Type): AssetBucket.Info => ({
+  size: entry.size,
+  contentType: entry.contentType,
+  cacheControl: entry.cacheControl,
+});
+
+class BodyFailure<E> {
+  constructor(readonly error: E) {}
+}
+
+const hex = (bytes: ArrayBuffer) =>
+  Array.from(new Uint8Array(bytes), (byte) => byte.toString(16).padStart(2, "0")).join("");
+
+/**
+ * Backs a bucket with files under a directory, for development and tests
+ * without object storage. Keys are stored by their SHA-256, so any key fits
+ * any file system: case-insensitive ones and Windows included. Writes are
+ * atomic for one process; concurrent writers to one key may leave unused
+ * files behind.
+ */
+export const layer = <Self, Id extends string>(
+  bucket: AssetBucket.Tag<Self, Id>,
+  options: Options,
+): Layer.Layer<Self, never, FileSystem.FileSystem | Path.Path> =>
+  Layer.effect(
+    bucket,
+    Effect.gen(function* () {
+      const fs = yield* FileSystem.FileSystem;
+      const path = yield* Path.Path;
+
+      const failure =
+        (operation: AssetBucket.AssetBucketError["operation"], key: string) => (cause: unknown) =>
+          new AssetBucket.AssetBucketError({ operation, bucket: bucket.id, key, cause });
+
+      // Two hex characters of the hash shard the directory, as Git does.
+      const locate = (key: string) =>
+        Effect.promise(() => crypto.subtle.digest("SHA-256", new TextEncoder().encode(key))).pipe(
+          Effect.map((digest) => {
+            const hash = hex(digest);
+            const directory = path.join(options.directory, hash.slice(0, 2));
+
+            return { directory, hash, record: path.join(directory, `${hash}.json`) };
+          }),
+        );
+
+      const read = (key: string, operation: AssetBucket.AssetBucketError["operation"]) =>
+        Effect.gen(function* () {
+          const { directory, record } = yield* locate(key);
+
+          const json = yield* fs.readFileString(record).pipe(
+            Effect.asSome,
+            Effect.catchReason("PlatformError", "NotFound", () => Effect.succeedNone),
+          );
+
+          if (Option.isNone(json)) return Option.none();
+
+          const entry = yield* Schema.decodeEffect(EntryJson)(json.value);
+
+          return Option.some({ entry, file: path.join(directory, entry.file) });
+        }).pipe(Effect.mapError(failure(operation, key)));
+
+      return bucket.of({
+        put: ({ key, contentType, cacheControl, body }) =>
+          Effect.gen(function* () {
+            const { directory, hash, record } = yield* locate(key).pipe(
+              Effect.tap(({ directory }) => fs.makeDirectory(directory, { recursive: true })),
+              Effect.mapError(failure("put", key)),
+            );
+
+            const previous = yield* read(key, "put");
+            const version = crypto.randomUUID();
+            const file = `${hash}.${version}`;
+            const target = path.join(directory, file);
+
+            // Body errors pass through the sink, so they are wrapped to keep
+            // them apart from the file system's.
+            const size = yield* Stream.run(
+              Stream.mapError(body, (error) => new BodyFailure(error)),
+              fs.sink(target),
+            ).pipe(
+              Effect.andThen(fs.stat(target)),
+              Effect.mapError((error) =>
+                error instanceof BodyFailure ? error.error : failure("put", key)(error),
+              ),
+              Effect.map((info) => info.size),
+              Effect.onError(() => Effect.ignore(fs.remove(target, { force: true }))),
+            );
+
+            const entry = { key, contentType, cacheControl, size, file };
+            const staged = `${record}.${version}`;
+
+            yield* Schema.encodeEffect(EntryJson)(entry).pipe(
+              Effect.flatMap((json) => fs.writeFileString(staged, json)),
+              Effect.andThen(fs.rename(staged, record)),
+              Effect.onError(() =>
+                Effect.ignore(
+                  Effect.all([
+                    fs.remove(staged, { force: true }),
+                    fs.remove(target, { force: true }),
+                  ]),
+                ),
+              ),
+              Effect.mapError(failure("put", key)),
+            );
+
+            // The replaced body is no longer referenced. Leaving it on failure
+            // costs only disk space.
+            yield* Option.match(previous, {
+              onNone: () => Effect.void,
+              onSome: ({ file }) => Effect.ignore(fs.remove(file, { force: true })),
+            });
+
+            return { size: entry.size, contentType, cacheControl };
+          }),
+        get: (key) =>
+          Effect.map(
+            read(key, "get"),
+            Option.map(({ entry, file }) => ({
+              info: infoOf(entry),
+              body: Stream.mapError(fs.stream(file), failure("get", key)),
+            })),
+          ),
+        head: (key) =>
+          Effect.map(
+            read(key, "head"),
+            Option.map(({ entry }) => infoOf(entry)),
+          ),
+        delete: (key) =>
+          Effect.gen(function* () {
+            const stored = yield* read(key, "delete");
+
+            if (Option.isNone(stored)) return;
+
+            const { record } = yield* locate(key);
+
+            yield* fs
+              .remove(record, { force: true })
+              .pipe(
+                Effect.andThen(fs.remove(stored.value.file, { force: true })),
+                Effect.mapError(failure("delete", key)),
+              );
+          }),
+      });
+    }),
+  );

package/src/index.ts

@@ -0,0 +1,3 @@
+export * as Asset from "./Asset.ts";
+
+export * as AssetBucket from "./AssetBucket.ts";

package/src/internal/bytes.ts

@@ -0,0 +1,12 @@
+/** Joins chunks into one array. */
+export const concat = (chunks: ReadonlyArray<Uint8Array>): Uint8Array => {
+  const bytes = new Uint8Array(chunks.reduce((size, chunk) => size + chunk.length, 0));
+  let offset = 0;
+
+  for (const chunk of chunks) {
+    bytes.set(chunk, offset);
+    offset += chunk.length;
+  }
+
+  return bytes;
+};

package/src/internal/http.ts

@@ -0,0 +1,54 @@
+import type * as AssetBucket from "../AssetBucket.ts";
+
+import { Effect, Option } from "effect";
+import { HttpServerRequest, HttpServerResponse } from "effect/http";
+
+/** The routes' prefix and wildcard path under a base URL, absolute or not. */
+export const mount = (baseUrl: string) => {
+  const prefix = new URL(baseUrl, "http://localhost").pathname.replace(/\/$/u, "");
+
+  return { prefix, path: `/${prefix.slice(1)}/*` as const };
+};
+
+/** The current request's key under the prefix, and its query; no key for a malformed path. */
+export const target = (prefix: string) =>
+  Effect.map(HttpServerRequest.HttpServerRequest, (request) => {
+    const url = new URL(request.url, "http://localhost");
+
+    const key = Option.liftThrowable(() =>
+      url.pathname
+        .slice(prefix.length + 1)
+        .split("/")
+        .map(decodeURIComponent)
+        .join("/"),
+    )();
+
+    return { request, key, query: url.searchParams };
+  });
+
+// Objects are served from the app's origin in development. Even a file a
+// browser would read as something else must not be sniffed or run.
+const inert = {
+  "x-content-type-options": "nosniff",
+  "content-security-policy": "default-src 'none'; sandbox",
+};
+
+/** Serves a stored object with its type, length and cache control, or 404. */
+export const respond = (bucket: AssetBucket.Bucket, key: string) =>
+  Effect.map(
+    bucket.get(key),
+    Option.match({
+      onNone: () => HttpServerResponse.empty({ status: 404 }),
+      onSome: ({ info, body }) => {
+        const response = HttpServerResponse.stream(body, {
+          contentType: info.contentType,
+          contentLength: Number(info.size),
+          headers: inert,
+        });
+
+        return info.cacheControl === undefined
+          ? response
+          : HttpServerResponse.setHeader(response, "cache-control", info.cacheControl);
+      },
+    }),
+  );