@distilled.cloud/core 0.30.3 → 1.0.0-rc.10

package/src/protocol-rest.ts

@@ -0,0 +1,391 @@
+/**
+ * Generic REST protocol factory for simple bearer/header-auth JSON APIs
+ * (Neon, Fly.io, PlanetScale, Supabase, Turso, …).
+ *
+ * `makeRestProtocol(options)` builds a core {@link API.Protocol} layer:
+ *
+ *   request:  credentials resolved from the CALLING fiber's context on every
+ *             request (the layer itself is memoized per process — see
+ *             `core/api`), auth headers + base URL from the provider's
+ *             options, request built by `buildRequest` from
+ *             `core/protocol-http` (deep-unwrapping any `Redacted` input
+ *             values first)
+ *
+ *   response: 2xx JSON → optional `transformResponse` → recursive wire→TS
+ *             key mapping (`mapKeys`) → `Redacted` wrapping of members
+ *             marked with {@link SensitiveValue}; non-2xx → typed error:
+ *             per-op matcher classes (`matchTypedError`), then the status
+ *             map (default `HTTP_STATUS_MAP`), then an `InternalServerError`
+ *             for unmapped 5xx, then the provider's `unknownError` fallback
+ *             — with `retryAfter` durations stamped from the standard hint
+ *             headers on retryable statuses.
+ *
+ * Anything provider-specific (envelope quirks, bespoke error codes) belongs
+ * in the provider package — via these options or a hand-written protocol.
+ */
+import * as Effect from "effect/Effect";
+import * as Layer from "effect/Layer";
+import * as Redacted from "effect/Redacted";
+import type * as AST from "effect/SchemaAST";
+import type * as HttpClientRequest from "effect/unstable/http/HttpClientRequest";
+import type * as HttpClientResponse from "effect/unstable/http/HttpClientResponse";
+import * as API from "./api.ts";
+import { httpSymbol, makeAnnotation, type HttpTrait } from "./trait.ts";
+import {
+  buildRequest,
+  getAnn,
+  getPropAnn,
+  getProps,
+  isOpaqueValue,
+  mapKeys,
+  matchTypedError,
+  resolveNode,
+} from "./protocol-http.ts";
+import { HTTP_STATUS_MAP, InternalServerError } from "./errors.ts";
+import { parseRetryAfterForStatus } from "./retry-after.ts";
+
+// =============================================================================
+// Traits
+// =============================================================================
+
+export const sensitiveValueSymbol = Symbol.for(
+  "@distilled.cloud/core/sensitive-value",
+);
+/**
+ * Marks a string member as sensitive (mirrors `smithy.api#sensitive`).
+ * The REST protocol wraps decoded values in `Redacted` on the way out and
+ * accepts `string | Redacted<string>` on the way in (unwrapped before
+ * serialization). Takes an ignored argument so generators can inline the
+ * smithy trait value (`T.SensitiveValue({})`).
+ */
+export const SensitiveValue = (_value?: unknown) =>
+  makeAnnotation(sensitiveValueSymbol, true);
+
+export const rawResponseSymbol = Symbol.for(
+  "@distilled.cloud/core/raw-response",
+);
+/**
+ * Marks the sole output member that carries a bare (array/scalar) response
+ * body (mirrors `com.distilled.openapi#rawResponse`).
+ */
+export const RawResponse = (_value?: unknown) =>
+  makeAnnotation(rawResponseSymbol, true);
+
+export const rawResponseRootSymbol = Symbol.for(
+  "@distilled.cloud/core/raw-response-root",
+);
+/**
+ * Marks a response schema whose ENTIRE value is the response body (the
+ * generator's `rootPipe` for synthesized bare-payload wrappers): the emitted
+ * response type IS the payload type and the protocol returns the mapped body
+ * directly.
+ */
+export const RawResponseRoot = () =>
+  makeAnnotation(rawResponseRootSymbol, true);
+
+// =============================================================================
+// Value helpers
+// =============================================================================
+
+const isPlainObject = (v: unknown): v is Record<string, unknown> =>
+  v !== null &&
+  typeof v === "object" &&
+  !Array.isArray(v) &&
+  !isOpaqueValue(v) &&
+  (Object.getPrototypeOf(v) === Object.prototype ||
+    Object.getPrototypeOf(v) === null);
+
+/**
+ * Deep-unwrap `Redacted` values in an input (plain objects/arrays only —
+ * class instances, files, and binary payloads pass through untouched).
+ * Sensitive input members accept `string | Redacted<string>`; the wire wants
+ * the raw string.
+ */
+export const unwrapRedactedDeep = (value: unknown): unknown => {
+  if (Redacted.isRedacted(value)) return Redacted.value(value);
+  if (Array.isArray(value)) return value.map(unwrapRedactedDeep);
+  if (isPlainObject(value)) {
+    const out: Record<string, unknown> = {};
+    for (const [k, v] of Object.entries(value)) {
+      if (v === undefined) continue;
+      out[k] = unwrapRedactedDeep(v);
+    }
+    return out;
+  }
+  return value;
+};
+
+/**
+ * Walk a decoded (TS-named) value alongside its schema AST, wrapping members
+ * marked {@link SensitiveValue} in `Redacted`. Keys the schema doesn't model
+ * pass through verbatim.
+ */
+export const wrapSensitive = (ast: AST.AST, value: unknown): unknown => {
+  if (value === null || typeof value !== "object" || isOpaqueValue(value)) {
+    return value;
+  }
+  const node = resolveNode(ast);
+  if (node._tag === "Arrays") {
+    if (!Array.isArray(value)) return value;
+    const elem = (node as any).rest?.[0] as AST.AST | undefined;
+    return elem ? value.map((v) => wrapSensitive(elem, v)) : value;
+  }
+  if (node._tag === "Objects" && !Array.isArray(value)) {
+    const props = getProps(node);
+    if (props.length === 0) return value;
+    const byName = new Map(props.map((p) => [String(p.name), p]));
+    const out: Record<string, unknown> = {};
+    for (const [k, v] of Object.entries(value as Record<string, unknown>)) {
+      if (v === undefined) continue;
+      const prop = byName.get(k);
+      if (!prop) {
+        out[k] = v;
+      } else if (
+        getPropAnn(prop, sensitiveValueSymbol) !== undefined &&
+        typeof v === "string"
+      ) {
+        out[k] = Redacted.make(v);
+      } else {
+        out[k] = wrapSensitive(prop.type, v);
+      }
+    }
+    return out;
+  }
+  return value;
+};
+
+// =============================================================================
+// Protocol factory
+// =============================================================================
+
+/** What the wire said about a failure, for the `unknownError` fallback. */
+export interface RestErrorInfo {
+  readonly status: number;
+  /** Error code from the envelope (services vary between string and number). */
+  readonly code?: string | number;
+  readonly message: string;
+  /** Parsed JSON body, or the raw text when the body wasn't JSON. */
+  readonly body: unknown;
+  readonly headers: Record<string, string | undefined>;
+}
+
+export interface RestErrorEnvelope {
+  readonly code?: string | number;
+  readonly message?: string;
+}
+
+export interface RestProtocolOptions<C> {
+  /**
+   * Resolve credentials ON THE CALLING FIBER — evaluated per request, never
+   * at layer build time, so context-provided credentials and token refreshes
+   * are picked up. Typically `Effect.gen(function* () { const resolve =
+   * yield* Credentials; return yield* resolve; })` for the distilled
+   * credentials-service convention. Its error/requirement channels are
+   * erased at the protocol boundary (Protocol effects carry none) and
+   * reintroduced for callers by the generated `<Sdk>OpError` /
+   * `<Sdk>OpContext` annotations.
+   */
+  readonly credentials: Effect.Effect<C, any, any>;
+  /**
+   * API base URL from the resolved credentials. Receives the operation's
+   * route so APIs with more than one endpoint (e.g. S2's account vs
+   * per-basin hosts, chosen by path) can route per request; single-endpoint
+   * providers ignore the argument. A THROWN error (e.g. a required scope
+   * missing from the credentials) is caught and surfaced on the calling
+   * effect's error channel rather than dying as a defect.
+   */
+  readonly baseUrl: (
+    credentials: C,
+    target: { readonly uri: string; readonly method: string },
+  ) => string;
+  /** Auth (and any fixed) headers from the resolved credentials. */
+  readonly headers: (credentials: C) => Record<string, string>;
+  /**
+   * Extract `{ code?, message? }` from a parsed non-2xx JSON body. Default:
+   * lenient `{ code?, message? | error? }` (covers the common REST error
+   * envelopes).
+   */
+  readonly errorEnvelope?: (body: unknown) => RestErrorEnvelope | undefined;
+  /**
+   * HTTP status → error class constructed as `new Cls({ message, retryAfter
+   * })`. Default: core `HTTP_STATUS_MAP`. Consulted after per-op typed error
+   * matching; unmapped 5xx fall back to `InternalServerError`, everything
+   * else to {@link unknownError}.
+   */
+  readonly statusMap?: Readonly<Record<number, new (args: any) => any>>;
+  /** Fallback error for failures nothing else matched. */
+  readonly unknownError: (info: RestErrorInfo) => unknown;
+  /** Transform the parsed 2xx JSON before decoding (e.g. stripNulls). */
+  readonly transformResponse?: (body: unknown) => unknown;
+  /** Passed through to `buildRequest` (member-header transforms). */
+  readonly mapMemberHeader?: (name: string, value: string) => string;
+  /** Passed through to `buildRequest` (wire names for unmodeled input keys). */
+  readonly unknownKeyToWire?: (key: string) => string;
+}
+
+const defaultErrorEnvelope = (body: unknown): RestErrorEnvelope | undefined => {
+  if (body === null || typeof body !== "object") return undefined;
+  const b = body as Record<string, unknown>;
+  const code =
+    typeof b.code === "string" || typeof b.code === "number"
+      ? b.code
+      : undefined;
+  const message =
+    typeof b.message === "string"
+      ? b.message
+      : typeof b.error === "string"
+        ? b.error
+        : undefined;
+  return { code, message };
+};
+
+// Bridge: Protocol.decode is typed as Effect<unknown> (no error channel),
+// but REST failures are real typed errors that operations re-surface via
+// their `errors: [...]` lists. Fail with the instance and erase the type
+// here; the generated operation annotations reintroduce it for callers.
+const fail = (e: unknown): Effect.Effect<never> =>
+  Effect.fail(e) as Effect.Effect<never>;
+
+/**
+ * Build a `Layer<Protocol>` for a simple REST JSON API. Assign the result to
+ * a module-level const in the provider's `protocol.ts` — `API.make` memoizes
+ * protocol layers by value identity.
+ */
+export const makeRestProtocol = <C>(
+  options: RestProtocolOptions<C>,
+): Layer.Layer<API.Protocol> => {
+  const errorEnvelope = options.errorEnvelope ?? defaultErrorEnvelope;
+  const statusMap: Readonly<
+    Record<number, (new (args: any) => any) | undefined>
+  > = options.statusMap ?? HTTP_STATUS_MAP;
+
+  const encode = ({
+    input,
+    inputAst,
+  }: {
+    readonly input: unknown;
+    readonly inputAst: AST.AST;
+  }) =>
+    Effect.gen(function* () {
+      const creds = yield* options.credentials as Effect.Effect<C>;
+      const http = getAnn(inputAst, httpSymbol) as HttpTrait | undefined;
+      let baseUrl: string;
+      try {
+        baseUrl = options.baseUrl(creds, {
+          uri: http?.uri ?? "",
+          method: http?.method ?? "",
+        });
+      } catch (e) {
+        // A baseUrl that refuses the route (missing scope in the
+        // credentials) fails the call with that error, typed for callers by
+        // the operation's declared error channel.
+        return yield* Effect.fail(e);
+      }
+      return buildRequest({
+        input: unwrapRedactedDeep(input),
+        inputAst,
+        baseUrl,
+        headers: options.headers(creds),
+        mapMemberHeader: options.mapMemberHeader,
+        unknownKeyToWire: options.unknownKeyToWire,
+      });
+    });
+
+  const decode = ({
+    response,
+    outputAst,
+    errors: errorClasses,
+  }: {
+    readonly response: HttpClientResponse.HttpClientResponse;
+    readonly outputAst: AST.AST;
+    readonly errors: ReadonlyArray<unknown>;
+  }) =>
+    Effect.gen(function* () {
+      // Read as text and parse tolerantly — error pages are often non-JSON.
+      const text = (yield* response.text.pipe(Effect.orDie)) ?? "";
+      if (process.env.DISTILLED_DEBUG_HTTP) {
+        console.error(
+          `[distilled] <- ${response.status} ${text.slice(0, 400)}`,
+        );
+      }
+      let json: unknown;
+      let nonJson = false;
+      if (text.trim().length > 0) {
+        try {
+          json = JSON.parse(text);
+        } catch {
+          nonJson = true;
+        }
+      }
+      const status = response.status;
+      const headers = response.headers as Record<string, string | undefined>;
+
+      if (status >= 400) {
+        const env = (nonJson ? undefined : errorEnvelope(json)) ?? {};
+        const message =
+          env.message ??
+          (nonJson && text.trim() ? text.trim() : `HTTP ${status}`);
+
+        // 1. Per-operation typed error (matcher metadata on the class).
+        const typed = matchTypedError(errorClasses, status, [
+          {
+            code: typeof env.code === "number" ? env.code : undefined,
+            message,
+          },
+        ]);
+        if (typed !== undefined) return yield* fail(typed);
+
+        // 2. Status-mapped class (retryAfter only stamps on retryable
+        //    statuses — parseRetryAfterForStatus gates on that).
+        const StatusErrorClass = statusMap[status];
+        if (StatusErrorClass) {
+          return yield* fail(
+            new StatusErrorClass({
+              message,
+              retryAfter: parseRetryAfterForStatus(status, headers),
+            }),
+          );
+        }
+
+        // 3. Unmapped 5xx (e.g. proxy-specific statuses) → retryable server
+        //    error rather than the unknown fallback.
+        if (status >= 500) {
+          return yield* fail(
+            new InternalServerError({
+              message,
+              retryAfter: parseRetryAfterForStatus(status, headers),
+            }),
+          );
+        }
+
+        // 4. Provider fallback.
+        return yield* fail(
+          options.unknownError({
+            status,
+            code: env.code,
+            message,
+            body: nonJson ? text : json,
+            headers,
+          }),
+        );
+      }
+
+      // 2xx: the response body IS the payload (no envelope). Wire→TS key
+      // mapping is schema-driven; `RawResponseRoot` responses are the body
+      // verbatim (mapKeys handles arrays/scalars structurally either way).
+      let body: unknown = nonJson ? text : (json ?? {});
+      if (options.transformResponse) body = options.transformResponse(body);
+      return wrapSensitive(outputAst, mapKeys(outputAst, body, "decode"));
+    });
+
+  return Layer.succeed(
+    API.Protocol,
+    API.Protocol.of({
+      // Erase encode's credentials requirement (resolved on the calling
+      // fiber; see RestProtocolOptions.credentials).
+      encode: (args) =>
+        encode(args) as Effect.Effect<HttpClientRequest.HttpClientRequest>,
+      decode,
+    }),
+  );
+};

package/src/retry.ts

@@ -8,7 +8,6 @@
  * for that SDK without wrapping every call with `Effect.retry`.
  */
 import * as Config from "effect/Config";
-import * as ConfigProvider from "effect/ConfigProvider";
 import * as Context from "effect/Context";
 import * as Duration from "effect/Duration";
 import * as Effect from "effect/Effect";
@@ -84,9 +83,7 @@ export const jittered = Schedule.addDelay(() =>
  */
 export const capped = (max: Duration.Duration) =>
   Schedule.modifyDelay(({ duration }) =>
-    Effect.succeed(
-      Duration.isGreaterThan(duration, max) ? Duration.millis(5000) : duration,
-    ),
+    Effect.succeed(Duration.isGreaterThan(duration, max) ? max : duration),
   );
 
 // ============================================================================
@@ -124,7 +121,7 @@ export const ServerRetryHintCapMs = Context.Service<number>(
 export const serverRetryHintCapLayer = (capMs: number) =>
   Layer.succeed(ServerRetryHintCapMs, capMs);
 
-const serverRetryHintCapMsConfig: Config.Config<number> = Config.string(
+const serverRetryHintCapMsConfig: Config.Config<number> = Config.String(
   ENV_SERVER_RETRY_HINT_CAP_MS,
 ).pipe(
   Config.map((raw) => {
@@ -148,12 +145,6 @@ export const readServerRetryHintCapMsFromEnv = (): number =>
   Effect.runSync(
     serverRetryHintCapMsConfig.pipe(
       Effect.orElseSucceed(() => DEFAULT_SERVER_RETRY_HINT_CAP_MS),
-      // The default ConfigProvider caches env reads; use a fresh provider so
-      // each call observes the current process.env value.
-      Effect.provideService(
-        ConfigProvider.ConfigProvider,
-        ConfigProvider.fromEnv(),
-      ),
     ),
   );
 
@@ -166,10 +157,6 @@ const resolveServerRetryHintCapMs = (): Effect.Effect<number, never, never> =>
     }
     return yield* serverRetryHintCapMsConfig.pipe(
       Effect.orElseSucceed(() => DEFAULT_SERVER_RETRY_HINT_CAP_MS),
-      Effect.provideService(
-        ConfigProvider.ConfigProvider,
-        ConfigProvider.fromEnv(),
-      ),
     );
   });
 
@@ -223,25 +210,37 @@ const honorServerHint = (
  * - Honors `error.retryAfter` (server-provided hint) with precedence, capped
  *   by {@link DEFAULT_SERVER_RETRY_HINT_CAP_MS} by default; override with
  *   `DISTILLED_SERVER_RETRY_HINT_CAP_MS` or {@link ServerRetryHintCapMs}
- * - Otherwise uses exponential backoff starting at 100ms with a factor of 2
+ * - Otherwise uses exponential backoff starting at 250ms with a factor of 2,
+ *   capped at 5s per delay
  * - Ensures at least 500ms delay for throttling errors
- * - Limits to 5 retry attempts
+ * - Limits to 8 retry attempts
  * - Applies jitter to avoid thundering herd
+ *
+ * The 250ms/5s-cap/8-attempt shape (~20s of total patience) comes from
+ * alchemy's Cloudflare provider suite, where transient auth blips under
+ * high request concurrency routinely outlast a short 5-attempt/3s policy
+ * while a genuinely broken credential still fails within seconds.
  */
 export const makeDefault: Factory = (lastError) => ({
   while: (error) => isTransientError(error),
+  // The 5s cap applies to the exponential backoff BEFORE the server hint is
+  // considered, so a server-provided retryAfter longer than 5s is honored in
+  // full (bounded only by the 60s hint cap inside honorServerHint). Capping
+  // after the hint would silently clamp e.g. a Retry-After: 30 to 5s.
   schedule: Schedule.max([
     pipe(
-      Schedule.exponential(100, 2),
+      Schedule.exponential(250, 2),
+      capped(Duration.seconds(5)),
       honorServerHint(lastError, (duration, error) => {
         if (isThrottling(error) && Duration.toMillis(duration) < 500) {
           return Duration.millis(500);
         }
         return duration;
       }),
+      jittered,
     ),
-    Schedule.recurs(5),
-  ]).pipe(jittered),
+    Schedule.recurs(8),
+  ]),
 });
 
 /**
@@ -252,8 +251,8 @@ export const throttlingFactory: Factory = (lastError) => ({
   while: (error) => isThrottling(error),
   schedule: pipe(
     Schedule.exponential(1000, 2),
-    honorServerHint(lastError),
     capped(Duration.seconds(5)),
+    honorServerHint(lastError),
     jittered,
   ),
 });
@@ -282,8 +281,8 @@ export const transientFactory: Factory = (lastError) => ({
   while: isTransientError,
   schedule: pipe(
     Schedule.exponential(1000, 2),
-    honorServerHint(lastError),
     capped(Duration.seconds(5)),
+    honorServerHint(lastError),
     jittered,
   ),
 });

package/src/schema.ts

@@ -2,16 +2,15 @@
  * `any`-collapsing re-export of `effect/Schema` for generated service files.
  *
  * This is the single, shared definition consumed by every code-generated
- * provider package (AWS, Azure, Cloudflare, GCP, Kubernetes, …) via
- * `@distilled.cloud/core/schema`. Do not copy it into individual packages.
+ * provider package via `@distilled.cloud/core/schema`. Do not copy it into
+ * individual packages.
  *
  * Every real TYPE (`Schema`, `Codec`, `Top`, …) is re-exported untouched, so
- * the `Schema.Schema<Foo>` / `Schema.Codec<Foo>` annotations on generated
- * consts (and the public `.d.ts`) stay precise. The schema *construction
- * surface* (`Struct`, `optional`, `suspend`, …) and the leaf scalar schemas are
- * retyped to `any`, so the compiler instantiates none of the heavy Schema
- * generics while building a service file — the explicit annotations carry the
- * real types.
+ * the `Schema.Schema<Foo>` annotations on generated consts (and the public
+ * `.d.ts`) stay precise. The schema *construction surface* (`Struct`,
+ * `optional`, `suspend`, …) and the leaf scalar schemas are retyped to `any`,
+ * so the compiler instantiates none of the heavy Schema generics while
+ * building a service file — the explicit annotations carry the real types.
  *
  * Overriding a *value* export never affects the same-named *type* export (they
  * live in separate namespaces and continue to flow through `export *`), so
@@ -24,7 +23,7 @@ export * from "effect/Schema";
 
 import * as S from "effect/Schema";
 
-type AnyFn = (...args: any[]) => any;
+export type AnyFn = (...args: any[]) => any;
 
 // Construction surface — collapse to `any` so generics are never instantiated.
 export const optional: AnyFn = S.optional as AnyFn;
@@ -49,6 +48,6 @@ export const Null: any = S.Null;
 export const Void: any = S.Void;
 export const Any: any = S.Any;
 
-// NOTE: `Schema` / `Codec` (types) and `TaggedErrorClass` are intentionally NOT
+// NOTE: `Schema` / `Codec` (types) and `TaggedError` are intentionally NOT
 // overridden — they flow through `export *` with real types so cast targets and
 // typed error classes stay precise.