okengine 0.21.0 → 0.22.0

package/src/kernel/idempotency.ts

@@ -0,0 +1,304 @@
+/**
+ * HTTP idempotency policy.
+ *
+ * A flow honors `Idempotency-Key` when it is reached over HTTP as REST
+ * non-GET or RPC, its inferred effects mutate or it uses `fx.raw`, and it
+ * is not a stream, live feed, or binary response. The IETF draft this header
+ * follows expired without becoming an RFC.
+ *
+ * @module
+ */
+
+import { parseDurationMs } from "../elements/clock/duration.ts";
+import type { Effects } from "../manifest/types.ts";
+import { fail } from "./errors.ts";
+import type { EffectKind } from "./effects.ts";
+import type { IdempotencyRow, StoredResponse } from "./idempotency-store.ts";
+
+/** Request header. */
+export const IDEMPOTENCY_HEADER = "idempotency-key";
+
+/** Set on a replayed response. */
+export const REPLAYED_HEADER = "idempotent-replayed";
+
+/** Default TTL stamped when the flow omits one. */
+export const IDEMPOTENCY_DEFAULT_TTL = "24h";
+
+/** Author option on `flow({ idempotency })`. */
+export type FlowIdempotencyOption =
+  | false
+  | "required"
+  | { readonly required?: boolean; readonly ttl?: string };
+
+/** Manifest / client stamp. */
+export interface ResolvedIdempotency {
+  readonly mode: "off" | "auto" | "required";
+  readonly ttl: string;
+}
+
+const MUTATING: ReadonlySet<EffectKind> = new Set([
+  "write",
+  "emit",
+  "send",
+  "call",
+  "fetch",
+  "ask",
+  "embed",
+]);
+
+const KEY_PATTERN = /^[\x20-\x7E]{16,255}$/;
+
+/**
+ * True when inferred effects include anything besides reads and secrets.
+ *
+ * @param effects - Declared or inferred effect set
+ */
+export function hasMutatingEffects(effects: Effects | undefined): boolean {
+  if (effects === undefined) return false;
+  return (
+    (effects.writes?.length ?? 0) > 0 ||
+    (effects.emits?.length ?? 0) > 0 ||
+    (effects.sends?.length ?? 0) > 0 ||
+    (effects.calls?.length ?? 0) > 0 ||
+    (effects.fetches?.length ?? 0) > 0 ||
+    (effects.asks?.length ?? 0) > 0 ||
+    (effects.embeds?.length ?? 0) > 0
+  );
+}
+
+/**
+ * True when this HTTP entry should look at `Idempotency-Key`.
+ *
+ * @param args - Trigger shape and inferred effects
+ */
+export function idempotencyEligible(args: {
+  readonly hasRequest: boolean;
+  /** HTTP method. RPC is `POST`. */
+  readonly method: string | undefined;
+  readonly stream: boolean;
+  readonly live: boolean;
+  readonly effects: Effects | undefined;
+  readonly usesRaw: boolean;
+}): boolean {
+  if (!args.hasRequest || args.stream || args.live) return false;
+  const method = (args.method ?? "POST").toUpperCase();
+  if (method === "GET") return false;
+  return args.usesRaw || hasMutatingEffects(args.effects);
+}
+
+/** What this request should do with the header. */
+export interface IdempotencyAttempt {
+  readonly claiming: boolean;
+  readonly mode: "off" | "auto" | "required";
+  readonly ttlMs: number;
+  readonly usesRaw: boolean;
+}
+
+/**
+ * Decide whether this request claims a key.
+ *
+ * A stamped `required` flow still ignores the header when this entry is not
+ * an eligible HTTP call (signal delivery, GET, stream, live, read-only).
+ *
+ * @param args - Flow stamp plus this request
+ */
+export function prepareIdempotencyAttempt(args: {
+  readonly hasRequest: boolean;
+  readonly method: string | undefined;
+  readonly stream: boolean;
+  readonly live: boolean;
+  readonly effects: Effects | undefined;
+  readonly usesRaw: boolean;
+  readonly option: FlowIdempotencyOption | undefined;
+  readonly resolved: ResolvedIdempotency | undefined;
+  readonly header: string | null;
+}): IdempotencyAttempt {
+  const eligible = idempotencyEligible(args);
+  const resolved = args.resolved ?? resolveIdempotencyMode(args.option, eligible);
+  const mode = eligible ? resolved.mode : "off";
+  const headerPresent = args.header !== null && args.header.trim().length > 0;
+  const ttlMs = idempotencyTtlMs(resolved.ttl);
+  return {
+    claiming: mode === "required" || (mode === "auto" && headerPresent),
+    mode,
+    ttlMs: ttlMs > 0 ? ttlMs : 24 * 60 * 60 * 1000,
+    usesRaw: args.usesRaw,
+  };
+}
+
+/**
+ * Resolve `off | auto | required` and the TTL string.
+ *
+ * Ineligible flows are `off` even when the option says required. Callers
+ * that parse source should reject that combination before calling this.
+ *
+ * @param option - Author option
+ * @param eligible - {@link idempotencyEligible}
+ */
+export function resolveIdempotencyMode(
+  option: FlowIdempotencyOption | undefined,
+  eligible: boolean,
+): ResolvedIdempotency {
+  const ttl = ttlOf(option);
+  if (!eligible || option === false) return { mode: "off", ttl };
+  if (option === "required") return { mode: "required", ttl };
+  if (typeof option === "object" && option.required === true) return { mode: "required", ttl };
+  return { mode: "auto", ttl };
+}
+
+/**
+ * Parse a TTL string. `0` means the string is not a duration.
+ *
+ * @param ttl - Duration such as `24h`
+ */
+export function idempotencyTtlMs(ttl: string): number {
+  return parseDurationMs(ttl);
+}
+
+/**
+ * `user:<id>`, else `apikey:<id>`, else `anon`.
+ *
+ * @param auth - Principal after the gate
+ */
+export function idempotencyPrincipal(auth: {
+  readonly userId: string | null;
+  readonly apiKeyId?: string | null;
+}): string {
+  if (auth.userId !== null && auth.userId.length > 0) return `user:${auth.userId}`;
+  if (auth.apiKeyId !== null && auth.apiKeyId !== undefined && auth.apiKeyId.length > 0) {
+    return `apikey:${auth.apiKeyId}`;
+  }
+  return "anon";
+}
+
+/**
+ * SHA-256 of canonical JSON `{ flow, input }`. Computed once at claim.
+ *
+ * @param flow - Flow name
+ * @param input - Validated input
+ */
+export function idempotencyFingerprint(flow: string, input: unknown): string {
+  const hasher = new Bun.CryptoHasher("sha256");
+  hasher.update(canonicalJson({ flow, input }));
+  return hasher.digest("hex");
+}
+
+/**
+ * Classify a raw header value.
+ *
+ * @param header - Header value, or null when absent
+ */
+export function classifyIdempotencyKey(header: string | null): "missing" | "invalid" | string {
+  if (header === null) return "missing";
+  const key = header.trim();
+  if (!KEY_PATTERN.test(key)) return "invalid";
+  return key;
+}
+
+/**
+ * True when this attempt recorded a non-read effect. Secrets do not count.
+ * Failed attempts stay on the ledger.
+ *
+ * @param entries - This run's ledger
+ */
+export function attemptedMutation(entries: readonly { readonly kind: EffectKind }[]): boolean {
+  return entries.some((entry) => MUTATING.has(entry.kind));
+}
+
+/**
+ * Replay a stored response and mark it `Idempotent-Replayed`.
+ *
+ * @param row - Completed row
+ */
+export function replayResponse(row: IdempotencyRow): Response {
+  const headers = new Headers();
+  const contentType = row.responseHeaders?.contentType;
+  const location = row.responseHeaders?.location;
+  if (contentType !== undefined && contentType.length > 0) headers.set("content-type", contentType);
+  if (location !== undefined && location.length > 0) headers.set("location", location);
+  headers.set(REPLAYED_HEADER, "true");
+  const status = row.responseStatus ?? 200;
+  if (status === 204) return new Response(null, { status, headers });
+  return new Response(row.responseBody ?? "", { status, headers });
+}
+
+/**
+ * Protocol failure. `IdempotencyInProgress` also sets `Retry-After`.
+ *
+ * @param code - Builtin idempotency code
+ * @param retryAfterSeconds - Seconds until the lease expires, for 409
+ */
+export function idempotencyErrorResponse(
+  code:
+    | "IdempotencyKeyMissing"
+    | "IdempotencyKeyInvalid"
+    | "IdempotencyKeyReused"
+    | "IdempotencyInProgress",
+  retryAfterSeconds?: number,
+): Response {
+  const failure = fail(
+    code,
+    code === "IdempotencyInProgress" ? { retryAfter: retryAfterSeconds } : {},
+  );
+  const headers = new Headers();
+  if (code === "IdempotencyInProgress") {
+    headers.set("retry-after", String(Math.max(1, retryAfterSeconds ?? 1)));
+  }
+  const status =
+    code === "IdempotencyKeyReused" ? 422 : code === "IdempotencyInProgress" ? 409 : 400;
+  return Response.json({ data: null, error: failure.error }, { status, headers });
+}
+
+/**
+ * Buffer a JSON envelope for storage. Streams and non-JSON bodies return
+ * undefined so the caller deletes the in-progress row.
+ *
+ * @param res - Encoded response (not the live stream)
+ */
+export async function storedFromResponse(res: {
+  readonly status: number;
+  readonly headers: { get(name: string): string | null };
+  text(): Promise<string>;
+}): Promise<StoredResponse | undefined> {
+  const contentType = res.headers.get("content-type") ?? undefined;
+  if (contentType?.includes("text/event-stream")) return undefined;
+  if (
+    contentType !== undefined &&
+    !contentType.includes("json") &&
+    !contentType.includes("text/") &&
+    res.status !== 204
+  ) {
+    return undefined;
+  }
+  const location = res.headers.get("location") ?? undefined;
+  const body = res.status === 204 ? "" : await res.text();
+  return {
+    status: res.status,
+    body,
+    ...(contentType !== undefined ? { contentType } : {}),
+    ...(location !== undefined ? { location } : {}),
+  };
+}
+
+/**
+ * Author TTL, or {@link IDEMPOTENCY_DEFAULT_TTL}.
+ *
+ * @param option - Author option
+ */
+export function ttlOf(option: FlowIdempotencyOption | undefined): string {
+  if (typeof option === "object" && option !== null && typeof option.ttl === "string") {
+    return option.ttl;
+  }
+  return IDEMPOTENCY_DEFAULT_TTL;
+}
+
+function canonicalJson(value: unknown): string {
+  if (value === undefined) return "null";
+  if (value === null || typeof value !== "object") return JSON.stringify(value) ?? "null";
+  if (Array.isArray(value)) return `[${value.map((item) => canonicalJson(item)).join(",")}]`;
+  const obj = value as Record<string, unknown>;
+  const keys = Object.keys(obj)
+    .filter((key) => obj[key] !== undefined)
+    .sort();
+  return `{${keys.map((key) => `${JSON.stringify(key)}:${canonicalJson(obj[key])}`).join(",")}}`;
+}

package/src/kernel/index.ts

@@ -212,6 +212,7 @@ export {
   currentAbortSignal,
   isAbortError,
   linkAbort,
+  requestSignal,
   withAbortSignal,
 } from "./abort-scope.ts";
 

package/src/kernel/journal.ts

@@ -10,7 +10,9 @@
 import { mkdir } from "node:fs/promises";
 import { dirname } from "node:path";
 import { okid } from "../okid.ts";
+import type { IdempotencyStore } from "./idempotency-store.ts";
 import { JournalSuspend } from "./journal-suspend.ts";
+import { lazyRequire } from "./lazy-require.ts";
 
 export { JournalSuspend, isJournalSuspend } from "./journal-suspend.ts";
 
@@ -151,8 +153,21 @@ export interface JournalLeaseStore {
   listOrphans(now: number): Promise<readonly JournalRun[]>;
 }
 
+/**
+ * Load the idempotency table implementation without a static import.
+ * Computed stem so the edge profile does not inline it.
+ */
+function loadIdempotencyStore(): typeof import("./idempotency-store.ts") {
+  return lazyRequire(import.meta.dir, ["idempotency", "store"].join("-"));
+}
+
 /** Persistence backend for journal runs. */
 export interface JournalStore extends Partial<JournalLeaseStore> {
+  /**
+   * Idempotency records on this same driver. Absent on a custom store that
+   * only implements run `get` / `put`.
+   */
+  readonly idempotency?: IdempotencyStore;
   /**
    * Load a run by id.
    *
@@ -286,6 +301,11 @@ export function createMemoryJournalStore(seed?: readonly JournalRun[]): JournalS
       return [...runs.values()].map(cloneRun);
     },
     ...leaseMethods(load),
+    get idempotency(): IdempotencyStore {
+      const created = loadIdempotencyStore().createMemoryIdempotencyStore();
+      Object.defineProperty(this, "idempotency", { value: created });
+      return created;
+    },
   };
 }
 
@@ -332,6 +352,13 @@ export function createFileJournalStore(path: string): JournalStore {
     },
     // Single-host file: leases coordinate same-machine processes only.
     ...leaseMethods(load, flush),
+    get idempotency(): IdempotencyStore {
+      const created = loadIdempotencyStore().createFileIdempotencyStore(
+        `${dirname(path)}/idempotency.json`,
+      );
+      Object.defineProperty(this, "idempotency", { value: created });
+      return created;
+    },
   };
 }
 
@@ -428,6 +455,35 @@ export interface JournalSession {
   ): Promise<void>;
 }
 
+/**
+ * A journal session that forwards to a real session once one is assigned.
+ *
+ * Idempotent durable flows start the journal only after the claim, which is
+ * after `fx` has already been created.
+ */
+export function createJournalSlot(): {
+  readonly slot: { session?: JournalSession };
+  readonly facade: JournalSession;
+} {
+  const slot: { session?: JournalSession } = {};
+  const facade = new Proxy({} as JournalSession, {
+    get(_target, prop, receiver) {
+      const session = slot.session;
+      if (!session) {
+        if (prop === "runId") return "";
+        if (prop === "stampTenant") return async () => undefined;
+        if (prop === "rewind") return () => undefined;
+        return undefined;
+      }
+      const value = Reflect.get(session, prop, receiver);
+      return typeof value === "function"
+        ? (value as (...args: unknown[]) => unknown).bind(session)
+        : value;
+    },
+  });
+  return { slot, facade };
+}
+
 /** Journal facade. */
 export interface Journal {
   readonly store: JournalStore;

package/src/kernel/json-result.ts

@@ -0,0 +1,59 @@
+/**
+ * `fx.json` carriers — brand and guards, with no `fx` / Store graph.
+ *
+ * `project-out` and the HTTP encoder need these on the cold-start path.
+ * Importing them from `fx.ts` would evaluate the Store barrel (Zod, hybrid
+ * search) before the server listens.
+ */
+
+/** Brand for {@link JsonResult} (kept internal — flows never construct it). */
+export const jsonResultBrand: unique symbol = Symbol.for("oke.json");
+
+/** Carrier from `fx.json` — status + body read by the response encoder. */
+export interface JsonResult<T = unknown> {
+  readonly [jsonResultBrand]: true;
+  readonly status: number;
+  readonly value?: T;
+  readonly meta?: Record<string, unknown>;
+  readonly kind?: undefined;
+}
+
+/** SSE carrier from `fx.json.stream` / `fx.live`. */
+export interface JsonStreamResult {
+  readonly [jsonResultBrand]: true;
+  readonly kind: "stream";
+  readonly status: 200;
+  readonly chunks: AsyncIterable<unknown>;
+  /** Awaited before the 200 SSE body; throws OKE1210 on a missing resume cursor. */
+  ready?: () => Promise<void>;
+  /** Set by the kernel to commit journal / Runs after the stream settles. */
+  finalize?: () => Promise<void>;
+}
+
+/**
+ * True when `value` is an `fx.json` envelope carrier.
+ *
+ * @param value - Unknown handler output
+ */
+export function isJsonResult(value: unknown): value is JsonResult {
+  return (
+    typeof value === "object" &&
+    value !== null &&
+    (value as JsonResult)[jsonResultBrand] === true &&
+    (value as JsonStreamResult).kind !== "stream"
+  );
+}
+
+/**
+ * True when `value` is an SSE stream carrier from `fx.json.stream`.
+ *
+ * @param value - Unknown handler output
+ */
+export function isJsonStreamResult(value: unknown): value is JsonStreamResult {
+  return (
+    typeof value === "object" &&
+    value !== null &&
+    (value as JsonStreamResult)[jsonResultBrand] === true &&
+    (value as JsonStreamResult).kind === "stream"
+  );
+}

package/src/kernel/project-out.ts

@@ -12,7 +12,12 @@
 
 import type { SchemaInput } from "../validation/standard-schema.ts";
 import { validate } from "../validation/standard-schema.ts";
-import { isJsonResult, isJsonStreamResult, jsonResultBrand, type JsonResult } from "./fx.ts";
+import {
+  isJsonResult,
+  isJsonStreamResult,
+  jsonResultBrand,
+  type JsonResult,
+} from "./json-result.ts";
 
 /**
  * Project `output` through `schema` when the exposure declared `out`.

package/src/manifest/types.ts

@@ -201,6 +201,14 @@ export interface Flow {
   source?: string;
   plane?: FlowPlane;
   durable?: boolean;
+  /**
+   * HTTP idempotency stamp. `off` ignores `Idempotency-Key`.
+   * `auto` honors it when present. `required` rejects a missing header.
+   * `ttl` is how long a stored response (possibly PII) is kept.
+   */
+  idempotency?: { mode: "off" | "auto" | "required"; ttl: string };
+  /** True when the body calls `fx.raw`. Throws then keep the idempotency record. */
+  usesRaw?: boolean;
   /** Live signal name when this flow streams `delivery: "live"` SSE. */
   live?: string;
   cache?: boolean | string;

package/src/okid-extended.ts

@@ -0,0 +1,175 @@
+/**
+ * `okid({ … })` options path — sortable, prefix, and alphabet toggles.
+ *
+ * Kept off the kernel edge profile. `okid()` and `okid(length)` never load it.
+ */
+
+import {
+  assertLength,
+  OKID_ALPHABET,
+  OKID_DEFAULT_LENGTH,
+  OKID_LOOKALIKE_CHARS,
+  OKID_MAX_PREFIX_LENGTH,
+  OKID_MIN_LENGTH,
+  OKID_SORTABLE_ALPHABET,
+  OKID_SORTABLE_MIN_LENGTH,
+  type OkidOptions,
+} from "./okid-shared.ts";
+
+/** Character groups addressable through {@link OkidOptions} toggles. */
+const GROUPS = {
+  numbers: "0123456789",
+  lowercase: "abcdefghijklmnopqrstuvwxyz",
+  uppercase: "ABCDEFGHIJKLMNOPQRSTUVWXYZ",
+  symbols: "-_",
+} as const;
+
+/** Resolved alphabet + encoding metadata for one options combination. */
+interface ResolvedAlphabet {
+  readonly chars: string;
+  readonly size: number;
+  /** Bitmask covering `size` values (`size` is always a power of two here). */
+  readonly mask: number;
+}
+
+/** Memoized resolutions keyed by the toggle bitmask (32 combinations max). */
+const ALPHABET_CACHE = new Map<number, ResolvedAlphabet>();
+
+/**
+ * Resolve a toggle combination to an alphabet and rejection-sampling mask.
+ *
+ * @param numbers - Include `0-9`
+ * @param lowercase - Include `a-z`
+ * @param uppercase - Include `A-Z`
+ * @param symbols - Include `-` and `_`
+ * @param lookAlikes - Include visually confusable characters
+ */
+function resolveAlphabet(
+  numbers: boolean,
+  lowercase: boolean,
+  uppercase: boolean,
+  symbols: boolean,
+  lookAlikes: boolean,
+): ResolvedAlphabet {
+  const key =
+    (numbers ? 1 : 0) |
+    (lowercase ? 2 : 0) |
+    (uppercase ? 4 : 0) |
+    (symbols ? 8 : 0) |
+    (lookAlikes ? 0 : 16);
+  const cached = ALPHABET_CACHE.get(key);
+  if (cached) return cached;
+
+  let chars = "";
+  if (numbers) chars += GROUPS.numbers;
+  if (lowercase) chars += GROUPS.lowercase;
+  if (uppercase) chars += GROUPS.uppercase;
+  if (symbols) chars += GROUPS.symbols;
+  if (!chars) {
+    throw new RangeError("okid: alphabet is empty — enable at least one character group");
+  }
+  if (!lookAlikes) {
+    chars = [...chars].filter((c) => !OKID_LOOKALIKE_CHARS.includes(c)).join("");
+  }
+
+  // Round up to a power of two for mask-based rejection sampling: bytes below
+  // `size` map uniformly, bytes above are discarded and re-drawn — unbiased at
+  // every alphabet size, unlike naive modulo.
+  const rawSize = chars.length;
+  const size = 1 << Math.ceil(Math.log2(rawSize));
+  const resolved: ResolvedAlphabet = { chars, size: rawSize, mask: size - 1 };
+  ALPHABET_CACHE.set(key, resolved);
+  return resolved;
+}
+
+/**
+ * Encode one random byte stream into `length` characters of `alphabet`.
+ *
+ * @param alphabet - Resolved charset
+ * @param length - Output length
+ */
+function encodeAlphabet(alphabet: ResolvedAlphabet, length: number): string {
+  const { chars, size, mask } = alphabet;
+  const bytes = new Uint8Array(length + Math.ceil(length >> 2));
+  crypto.getRandomValues(bytes.subarray(0, length));
+  let out = "";
+  let i = 0;
+  while (out.length < length && i < bytes.length) {
+    const byte = bytes[i++]!;
+    if ((byte & mask) < size) out += chars[byte & mask];
+  }
+  return out;
+}
+
+/**
+ * Pack epoch-ms into exactly 8 codepoint-ordered characters (48 bits).
+ *
+ * @param nowMs - Epoch milliseconds
+ */
+function encodeTimestamp(nowMs: number): string {
+  let t = nowMs % 2 ** 48;
+  let out = "";
+  for (let i = 0; i < 8; i++) {
+    out = OKID_SORTABLE_ALPHABET[t & 63]! + out;
+    t = Math.floor(t / 64);
+  }
+  return out;
+}
+
+/**
+ * Assert a semantic prefix is within bounds and stays on the URL-safe alphabet.
+ *
+ * @param prefix - Caller-supplied label
+ */
+function assertPrefix(prefix: string): void {
+  if (prefix.length > OKID_MAX_PREFIX_LENGTH) {
+    throw new RangeError(
+      `okid: prefix length ${prefix.length} exceeds max ${OKID_MAX_PREFIX_LENGTH}`,
+    );
+  }
+  for (const char of prefix) {
+    if (!OKID_ALPHABET.includes(char)) {
+      throw new RangeError(
+        `okid: prefix contains invalid character ${JSON.stringify(char)} — use characters from OKID_ALPHABET`,
+      );
+    }
+  }
+}
+
+/**
+ * Generate an id from an options object.
+ *
+ * @param options - Length, prefix, sortable, and alphabet toggles
+ */
+export function okidWithOptions(options: OkidOptions): string {
+  const {
+    length = OKID_DEFAULT_LENGTH,
+    prefix = "",
+    sortable = false,
+    lowercase = true,
+    uppercase = true,
+    numbers = true,
+    symbols = true,
+    lookAlikes = true,
+  } = options;
+
+  if (prefix) assertPrefix(prefix);
+
+  let body: string;
+  if (sortable) {
+    assertLength(length, OKID_SORTABLE_MIN_LENGTH, "length");
+    // Time ordering requires lexicographic encoding, which requires the full
+    // codepoint-ordered alphabet — partial subsets cannot preserve both the
+    // caller's charset choice AND cross-ms ordering, so toggles are ignored.
+    const alphabet = resolveAlphabet(true, true, true, true, true);
+    body = encodeTimestamp(Date.now()) + encodeAlphabet(alphabet, length - 8);
+  } else {
+    assertLength(length, OKID_MIN_LENGTH, "length");
+    body = encodeAlphabet(
+      resolveAlphabet(numbers, lowercase, uppercase, symbols, lookAlikes),
+      length,
+    );
+  }
+
+  return prefix ? prefix + body : body;
+}