@distilled.cloud/core 0.0.0 → 0.2.0

package/src/json-patch.ts

@@ -0,0 +1,261 @@
+/**
+ * JSON Patch (RFC 6902) Implementation
+ *
+ * Provides a unified spec patching system for all SDKs.
+ * Patches are applied to OpenAPI/Discovery/Smithy specs before code generation
+ * to add error types, fix nullable fields, mark sensitive data, etc.
+ *
+ * @example
+ * ```ts
+ * import { applyAllPatches } from "@distilled.cloud/sdk-core/json-patch";
+ *
+ * const spec = JSON.parse(fs.readFileSync("openapi.json", "utf-8"));
+ * const { applied, errors } = applyAllPatches(spec, "./patches");
+ * ```
+ */
+import * as fs from "fs";
+import * as path from "path";
+
+// ============================================================================
+// Types
+// ============================================================================
+
+export interface JsonPatchOperation {
+  op: "add" | "remove" | "replace" | "move" | "copy" | "test";
+  path: string;
+  value?: unknown;
+  from?: string;
+}
+
+export type JsonPatch = JsonPatchOperation[];
+
+export interface PatchFile {
+  description: string;
+  patches: JsonPatch;
+}
+
+// ============================================================================
+// JSON Pointer (RFC 6901)
+// ============================================================================
+
+/**
+ * Parse a JSON Pointer (RFC 6901) path into segments.
+ */
+export function parseJsonPointer(pointer: string): string[] {
+  if (pointer === "") return [];
+  if (!pointer.startsWith("/")) {
+    throw new Error(`Invalid JSON Pointer: ${pointer}`);
+  }
+  return pointer
+    .slice(1)
+    .split("/")
+    .map((segment) => segment.replace(/~1/g, "/").replace(/~0/g, "~"));
+}
+
+/**
+ * Get a value at a JSON Pointer path.
+ */
+export function getValueAtPath(obj: unknown, pointer: string): unknown {
+  const segments = parseJsonPointer(pointer);
+  let current: unknown = obj;
+
+  for (const segment of segments) {
+    if (current === null || typeof current !== "object") {
+      throw new Error(`Cannot traverse path ${pointer}: not an object`);
+    }
+    if (Array.isArray(current)) {
+      const index = segment === "-" ? current.length : parseInt(segment, 10);
+      current = current[index];
+    } else {
+      current = (current as Record<string, unknown>)[segment];
+    }
+  }
+
+  return current;
+}
+
+/**
+ * Set a value at a JSON Pointer path.
+ */
+export function setValueAtPath(
+  obj: unknown,
+  pointer: string,
+  value: unknown,
+): void {
+  const segments = parseJsonPointer(pointer);
+  if (segments.length === 0) {
+    throw new Error("Cannot set value at root path");
+  }
+
+  let current: unknown = obj;
+
+  for (let i = 0; i < segments.length - 1; i++) {
+    const segment = segments[i]!;
+    if (current === null || typeof current !== "object") {
+      throw new Error(`Cannot traverse path ${pointer}: not an object`);
+    }
+    if (Array.isArray(current)) {
+      const index = parseInt(segment, 10);
+      current = current[index];
+    } else {
+      current = (current as Record<string, unknown>)[segment];
+    }
+  }
+
+  const lastSegment = segments[segments.length - 1]!;
+  if (current === null || typeof current !== "object") {
+    throw new Error(
+      `Cannot set value at path ${pointer}: parent is not an object`,
+    );
+  }
+
+  if (Array.isArray(current)) {
+    const index =
+      lastSegment === "-" ? current.length : parseInt(lastSegment, 10);
+    if (lastSegment === "-") {
+      current.push(value);
+    } else {
+      current[index] = value;
+    }
+  } else {
+    (current as Record<string, unknown>)[lastSegment] = value;
+  }
+}
+
+/**
+ * Remove a value at a JSON Pointer path.
+ */
+export function removeValueAtPath(obj: unknown, pointer: string): void {
+  const segments = parseJsonPointer(pointer);
+  if (segments.length === 0) {
+    throw new Error("Cannot remove root");
+  }
+
+  let current: unknown = obj;
+
+  for (let i = 0; i < segments.length - 1; i++) {
+    const segment = segments[i]!;
+    if (current === null || typeof current !== "object") {
+      throw new Error(`Cannot traverse path ${pointer}: not an object`);
+    }
+    if (Array.isArray(current)) {
+      current = current[parseInt(segment, 10)];
+    } else {
+      current = (current as Record<string, unknown>)[segment];
+    }
+  }
+
+  const lastSegment = segments[segments.length - 1]!;
+  if (current === null || typeof current !== "object") {
+    throw new Error(
+      `Cannot remove at path ${pointer}: parent is not an object`,
+    );
+  }
+
+  if (Array.isArray(current)) {
+    current.splice(parseInt(lastSegment, 10), 1);
+  } else {
+    delete (current as Record<string, unknown>)[lastSegment];
+  }
+}
+
+// ============================================================================
+// Patch Operations
+// ============================================================================
+
+/**
+ * Apply a single JSON Patch operation.
+ */
+export function applyOperation(
+  obj: unknown,
+  operation: JsonPatchOperation,
+): void {
+  switch (operation.op) {
+    case "add":
+      setValueAtPath(obj, operation.path, operation.value);
+      break;
+    case "remove":
+      removeValueAtPath(obj, operation.path);
+      break;
+    case "replace":
+      // For replace, the path must exist
+      getValueAtPath(obj, operation.path); // throws if doesn't exist
+      setValueAtPath(obj, operation.path, operation.value);
+      break;
+    case "move": {
+      if (!operation.from) throw new Error("move operation requires 'from'");
+      const moveValue = getValueAtPath(obj, operation.from);
+      removeValueAtPath(obj, operation.from);
+      setValueAtPath(obj, operation.path, moveValue);
+      break;
+    }
+    case "copy": {
+      if (!operation.from) throw new Error("copy operation requires 'from'");
+      const copyValue = getValueAtPath(obj, operation.from);
+      setValueAtPath(
+        obj,
+        operation.path,
+        JSON.parse(JSON.stringify(copyValue)),
+      );
+      break;
+    }
+    case "test": {
+      const testValue = getValueAtPath(obj, operation.path);
+      if (JSON.stringify(testValue) !== JSON.stringify(operation.value)) {
+        throw new Error(
+          `Test operation failed at ${operation.path}: expected ${JSON.stringify(operation.value)}, got ${JSON.stringify(testValue)}`,
+        );
+      }
+      break;
+    }
+    default:
+      throw new Error(`Unknown operation: ${(operation as { op: string }).op}`);
+  }
+}
+
+/**
+ * Apply a JSON Patch to an object (mutates in place).
+ */
+export function applyPatch(obj: unknown, patch: JsonPatch): void {
+  for (const operation of patch) {
+    applyOperation(obj, operation);
+  }
+}
+
+/**
+ * Load and apply all patches from a directory.
+ * Finds all *.patch.json files and applies them.
+ */
+export function applyAllPatches(
+  spec: unknown,
+  patchDir: string,
+): { applied: string[]; errors: string[] } {
+  const applied: string[] = [];
+  const errors: string[] = [];
+
+  if (!fs.existsSync(patchDir)) {
+    return { applied, errors };
+  }
+
+  // Find all .patch.json files
+  const files = fs
+    .readdirSync(patchDir)
+    .filter((f) => f.endsWith(".patch.json"))
+    .sort(); // Sort for deterministic application order
+
+  for (const file of files) {
+    const filePath = path.join(patchDir, file);
+    try {
+      const content = fs.readFileSync(filePath, "utf-8");
+      const patchFile: PatchFile = JSON.parse(content);
+      applyPatch(spec, patchFile.patches);
+      applied.push(`${file}: ${patchFile.description}`);
+    } catch (error) {
+      errors.push(
+        `${file}: ${error instanceof Error ? error.message : String(error)}`,
+      );
+    }
+  }
+
+  return { applied, errors };
+}

package/src/pagination.ts

@@ -0,0 +1,222 @@
+/**
+ * Pagination utilities for streaming through paginated API responses.
+ *
+ * Supports multiple pagination styles:
+ * - Page-based (e.g., PlanetScale): page/per_page with next_page number
+ * - Cursor-based (e.g., Neon): cursor/limit with next cursor string
+ * - Token-based (e.g., AWS): NextToken/MaxResults with continuation tokens
+ *
+ * Each SDK defines its own pagination trait configuration, and these
+ * shared utilities handle the streaming logic.
+ *
+ * @example
+ * ```ts
+ * import * as Pagination from "@distilled.cloud/sdk-core/pagination";
+ *
+ * // Page-based pagination
+ * const allPages = Pagination.paginatePages(listDatabases, { organization: "my-org" }, {
+ *   inputToken: "page",
+ *   outputToken: "next_page",
+ *   items: "data",
+ * });
+ * ```
+ */
+import * as Effect from "effect/Effect";
+import * as Stream from "effect/Stream";
+import { getPath } from "./traits.ts";
+
+// ============================================================================
+// Pagination Trait
+// ============================================================================
+
+/**
+ * Pagination trait describing how to navigate between pages.
+ */
+export interface PaginatedTrait {
+  /** The name of the input member containing the page/cursor token */
+  inputToken: string;
+  /** The path to the output member containing the next page/cursor token */
+  outputToken: string;
+  /** The path to the output member containing the paginated items */
+  items?: string;
+  /** The name of the input member that limits page size */
+  pageSize?: string;
+}
+
+// ============================================================================
+// Page-based Pagination (PlanetScale style)
+// ============================================================================
+
+/**
+ * Creates a stream of pages using page-number pagination.
+ *
+ * @param operation - The paginated operation to call
+ * @param input - The initial input (without page parameter)
+ * @param pagination - The pagination trait configuration
+ * @returns A Stream of full page responses
+ */
+export const paginatePageNumber = <
+  Input extends Record<string, unknown>,
+  Output,
+  E,
+  R,
+>(
+  operation: (input: Input) => Effect.Effect<Output, E, R>,
+  input: Omit<Input, string>,
+  pagination: PaginatedTrait,
+): Stream.Stream<Output, E, R> => {
+  type State = { page: number; done: boolean };
+
+  const unfoldFn = (state: State) =>
+    Effect.gen(function* () {
+      if (state.done) {
+        return undefined;
+      }
+
+      const requestPayload = {
+        ...input,
+        [pagination.inputToken]: state.page,
+      } as Input;
+
+      const response = yield* operation(requestPayload);
+
+      const nextPage = getPath(response, pagination.outputToken) as
+        | number
+        | null
+        | undefined;
+
+      const nextState: State = {
+        page: nextPage ?? state.page + 1,
+        done: nextPage === null || nextPage === undefined,
+      };
+
+      return [response, nextState] as const;
+    });
+
+  return Stream.unfold({ page: 1, done: false } as State, unfoldFn);
+};
+
+// ============================================================================
+// Cursor-based Pagination (Neon style)
+// ============================================================================
+
+/**
+ * Creates a stream of pages using cursor-based pagination.
+ *
+ * @param operation - The paginated operation to call
+ * @param input - The initial input (without cursor parameter)
+ * @param pagination - The pagination trait configuration
+ * @returns A Stream of full page responses
+ */
+export const paginateCursor = <
+  Input extends Record<string, unknown>,
+  Output,
+  E,
+  R,
+>(
+  operation: (input: Input) => Effect.Effect<Output, E, R>,
+  input: Omit<Input, string>,
+  pagination: PaginatedTrait,
+): Stream.Stream<Output, E, R> => {
+  type State = { cursor: string | undefined; done: boolean };
+
+  const unfoldFn = (state: State) =>
+    Effect.gen(function* () {
+      if (state.done) {
+        return undefined;
+      }
+
+      const requestPayload = {
+        ...input,
+        ...(state.cursor ? { [pagination.inputToken]: state.cursor } : {}),
+      } as Input;
+
+      const response = yield* operation(requestPayload);
+
+      const nextCursor = getPath(response, pagination.outputToken) as
+        | string
+        | null
+        | undefined;
+
+      const nextState: State = {
+        cursor: nextCursor ?? undefined,
+        done: nextCursor === null || nextCursor === undefined,
+      };
+
+      return [response, nextState] as const;
+    });
+
+  return Stream.unfold({ cursor: undefined, done: false } as State, unfoldFn);
+};
+
+// ============================================================================
+// Token-based Pagination (AWS style)
+// ============================================================================
+
+/**
+ * Creates a stream of pages using token-based pagination.
+ *
+ * @param operation - The paginated operation to call
+ * @param input - The initial input
+ * @param pagination - The pagination trait configuration
+ * @returns A Stream of full page responses
+ */
+export const paginateToken = <
+  Input extends Record<string, unknown>,
+  Output,
+  E,
+  R,
+>(
+  operation: (input: Input) => Effect.Effect<Output, E, R>,
+  input: Input,
+  pagination: PaginatedTrait,
+): Stream.Stream<Output, E, R> => {
+  type State = { token: unknown; done: boolean };
+
+  const unfoldFn = (state: State) =>
+    Effect.gen(function* () {
+      if (state.done) {
+        return undefined;
+      }
+
+      const requestPayload =
+        state.token !== undefined
+          ? { ...input, [pagination.inputToken]: state.token }
+          : input;
+
+      const response = yield* operation(requestPayload as Input);
+
+      const nextToken = getPath(response, pagination.outputToken);
+
+      const nextState: State = {
+        token: nextToken,
+        done: nextToken === undefined || nextToken === null,
+      };
+
+      return [response, nextState] as const;
+    });
+
+  return Stream.unfold({ token: undefined, done: false } as State, unfoldFn);
+};
+
+// ============================================================================
+// Item extraction
+// ============================================================================
+
+/**
+ * Extracts individual items from a page stream.
+ *
+ * @param pages - A stream of page responses
+ * @param itemsPath - Dot-separated path to the items array in the page response
+ * @returns A Stream of individual items
+ */
+export const extractItems = <Output, Item, E, R>(
+  pages: Stream.Stream<Output, E, R>,
+  itemsPath: string,
+): Stream.Stream<Item, E, R> =>
+  pages.pipe(
+    Stream.flatMap((page) => {
+      const items = getPath(page, itemsPath) as readonly Item[] | undefined;
+      return Stream.fromIterable(items ?? []);
+    }),
+  );

package/src/retry.ts

@@ -0,0 +1,177 @@
+/**
+ * Retry Policy System
+ *
+ * Provides configurable retry policies for API operations.
+ * Each SDK creates its own Retry service tag but uses these shared utilities
+ * for building retry schedules and policies.
+ *
+ * @example
+ * ```ts
+ * import * as Retry from "@distilled.cloud/sdk-core/retry";
+ *
+ * // Use the default retry policy
+ * myEffect.pipe(Retry.policy(myRetryService, Retry.makeDefault()))
+ *
+ * // Disable retries
+ * myEffect.pipe(Retry.none(myRetryService))
+ * ```
+ */
+import * as Duration from "effect/Duration";
+import * as Effect from "effect/Effect";
+import { pipe } from "effect/Function";
+import * as Layer from "effect/Layer";
+import * as Ref from "effect/Ref";
+import * as Schedule from "effect/Schedule";
+import * as ServiceMap from "effect/ServiceMap";
+import { isThrottling, isTransientError } from "./category.ts";
+
+// ============================================================================
+// Retry Policy Types
+// ============================================================================
+
+/**
+ * Retry policy options that match the Effect.retry contract.
+ */
+export interface Options {
+  /**
+   * Predicate to determine if an error should trigger a retry.
+   */
+  readonly while?: (error: unknown) => boolean;
+  /**
+   * The schedule to use for retrying.
+   */
+  readonly schedule?: Schedule.Schedule<unknown>;
+}
+
+/**
+ * A factory function that creates retry policy options with access to the last error ref.
+ * This allows dynamic policies that can inspect the last error for retry-after headers, etc.
+ */
+export type Factory = (lastError: Ref.Ref<unknown>) => Options;
+
+/**
+ * A retry policy can be either static options or a factory that receives the last error ref.
+ */
+export type Policy = Options | Factory;
+
+// ============================================================================
+// Retry Service Factory
+// ============================================================================
+
+/**
+ * Create a typed Retry service class for an SDK.
+ * Each SDK should create its own Retry service using this factory.
+ *
+ * @example
+ * ```ts
+ * // In planetscale-sdk/src/retry.ts
+ * export class Retry extends makeRetryService("PlanetScaleRetry") {}
+ * ```
+ */
+export const makeRetryService = (name: string) =>
+  ServiceMap.Service<any, Policy>()(name);
+
+/**
+ * Provides a custom retry policy for API calls.
+ */
+export const policy =
+  (Service: any, optionsOrFactory: Policy) =>
+  <A, E, R>(effect: Effect.Effect<A, E, R>) =>
+    Effect.provide(effect, Layer.succeed(Service, optionsOrFactory) as any);
+
+/**
+ * Disables all automatic retries.
+ */
+export const none =
+  (Service: any) =>
+  <A, E, R>(effect: Effect.Effect<A, E, R>) =>
+    Effect.provide(
+      effect,
+      Layer.succeed(Service, { while: () => false }) as any,
+    );
+
+// ============================================================================
+// Retry Schedule Utilities
+// ============================================================================
+
+/**
+ * Custom jittered schedule helper.
+ * Adds random jitter between 0-50ms to avoid thundering herd.
+ */
+export const jittered = Schedule.addDelay(() =>
+  Effect.succeed(Duration.millis(Math.random() * 50)),
+);
+
+/**
+ * Cap delay at a maximum duration.
+ */
+export const capped = (max: Duration.Duration) =>
+  Schedule.modifyDelay((duration: Duration.Duration) =>
+    Effect.succeed(
+      Duration.isGreaterThan(duration, max) ? Duration.millis(5000) : duration,
+    ),
+  );
+
+// ============================================================================
+// Default Retry Policies
+// ============================================================================
+
+/**
+ * Creates the default retry policy.
+ *
+ * This policy:
+ * - Retries transient errors (throttling, server, network, locked errors)
+ * - Uses exponential backoff starting at 100ms with a factor of 2
+ * - Ensures at least 500ms delay for throttling errors
+ * - Limits to 5 retry attempts
+ * - Applies jitter to avoid thundering herd
+ */
+export const makeDefault: Factory = (lastError) => ({
+  while: (error) => isTransientError(error),
+  schedule: pipe(
+    Schedule.exponential(100, 2),
+    Schedule.modifyDelay(
+      Effect.fnUntraced(function* (duration) {
+        const error = yield* Ref.get(lastError);
+        if (isThrottling(error)) {
+          if (Duration.toMillis(duration) < 500) {
+            return Duration.toMillis(Duration.millis(500));
+          }
+        }
+        return Duration.toMillis(duration);
+      }),
+    ),
+    Schedule.both(Schedule.recurs(5)),
+    jittered,
+  ),
+});
+
+/**
+ * Retry options that retries all throttling errors indefinitely.
+ */
+export const throttlingOptions: Options = {
+  while: (error) => isThrottling(error),
+  schedule: pipe(
+    Schedule.exponential(1000, 2),
+    capped(Duration.seconds(5)),
+    jittered,
+  ),
+};
+
+/**
+ * Retry options that retries all transient errors indefinitely.
+ *
+ * This includes:
+ * 1. Throttling errors
+ * 2. Server errors
+ * 3. Network errors
+ * 4. Locked errors (423)
+ */
+export const transientOptions: Options = {
+  while: isTransientError,
+  schedule: pipe(
+    Schedule.exponential(1000, 2),
+    capped(Duration.seconds(5)),
+    jittered,
+  ),
+};