okengine 0.21.1 → 0.23.0

package/src/kernel/idempotency.ts

@@ -0,0 +1,304 @@
+/**
+ * HTTP idempotency policy.
+ *
+ * A flow honors `Idempotency-Key` when it is reached over HTTP as REST
+ * non-GET or RPC, its inferred effects mutate or it uses `fx.raw`, and it
+ * is not a stream, live feed, or binary response. The IETF draft this header
+ * follows expired without becoming an RFC.
+ *
+ * @module
+ */
+
+import { parseDurationMs } from "../elements/clock/duration.ts";
+import type { Effects } from "../manifest/types.ts";
+import { fail } from "./errors.ts";
+import type { EffectKind } from "./effects.ts";
+import type { IdempotencyRow, StoredResponse } from "./idempotency-store.ts";
+
+/** Request header. */
+export const IDEMPOTENCY_HEADER = "idempotency-key";
+
+/** Set on a replayed response. */
+export const REPLAYED_HEADER = "idempotent-replayed";
+
+/** Default TTL stamped when the flow omits one. */
+export const IDEMPOTENCY_DEFAULT_TTL = "24h";
+
+/** Author option on `flow({ idempotency })`. */
+export type FlowIdempotencyOption =
+  | false
+  | "required"
+  | { readonly required?: boolean; readonly ttl?: string };
+
+/** Manifest / client stamp. */
+export interface ResolvedIdempotency {
+  readonly mode: "off" | "auto" | "required";
+  readonly ttl: string;
+}
+
+const MUTATING: ReadonlySet<EffectKind> = new Set([
+  "write",
+  "emit",
+  "send",
+  "call",
+  "fetch",
+  "ask",
+  "embed",
+]);
+
+const KEY_PATTERN = /^[\x20-\x7E]{16,255}$/;
+
+/**
+ * True when inferred effects include anything besides reads and secrets.
+ *
+ * @param effects - Declared or inferred effect set
+ */
+export function hasMutatingEffects(effects: Effects | undefined): boolean {
+  if (effects === undefined) return false;
+  return (
+    (effects.writes?.length ?? 0) > 0 ||
+    (effects.emits?.length ?? 0) > 0 ||
+    (effects.sends?.length ?? 0) > 0 ||
+    (effects.calls?.length ?? 0) > 0 ||
+    (effects.fetches?.length ?? 0) > 0 ||
+    (effects.asks?.length ?? 0) > 0 ||
+    (effects.embeds?.length ?? 0) > 0
+  );
+}
+
+/**
+ * True when this HTTP entry should look at `Idempotency-Key`.
+ *
+ * @param args - Trigger shape and inferred effects
+ */
+export function idempotencyEligible(args: {
+  readonly hasRequest: boolean;
+  /** HTTP method. RPC is `POST`. */
+  readonly method: string | undefined;
+  readonly stream: boolean;
+  readonly live: boolean;
+  readonly effects: Effects | undefined;
+  readonly usesRaw: boolean;
+}): boolean {
+  if (!args.hasRequest || args.stream || args.live) return false;
+  const method = (args.method ?? "POST").toUpperCase();
+  if (method === "GET") return false;
+  return args.usesRaw || hasMutatingEffects(args.effects);
+}
+
+/** What this request should do with the header. */
+export interface IdempotencyAttempt {
+  readonly claiming: boolean;
+  readonly mode: "off" | "auto" | "required";
+  readonly ttlMs: number;
+  readonly usesRaw: boolean;
+}
+
+/**
+ * Decide whether this request claims a key.
+ *
+ * A stamped `required` flow still ignores the header when this entry is not
+ * an eligible HTTP call (signal delivery, GET, stream, live, read-only).
+ *
+ * @param args - Flow stamp plus this request
+ */
+export function prepareIdempotencyAttempt(args: {
+  readonly hasRequest: boolean;
+  readonly method: string | undefined;
+  readonly stream: boolean;
+  readonly live: boolean;
+  readonly effects: Effects | undefined;
+  readonly usesRaw: boolean;
+  readonly option: FlowIdempotencyOption | undefined;
+  readonly resolved: ResolvedIdempotency | undefined;
+  readonly header: string | null;
+}): IdempotencyAttempt {
+  const eligible = idempotencyEligible(args);
+  const resolved = args.resolved ?? resolveIdempotencyMode(args.option, eligible);
+  const mode = eligible ? resolved.mode : "off";
+  const headerPresent = args.header !== null && args.header.trim().length > 0;
+  const ttlMs = idempotencyTtlMs(resolved.ttl);
+  return {
+    claiming: mode === "required" || (mode === "auto" && headerPresent),
+    mode,
+    ttlMs: ttlMs > 0 ? ttlMs : 24 * 60 * 60 * 1000,
+    usesRaw: args.usesRaw,
+  };
+}
+
+/**
+ * Resolve `off | auto | required` and the TTL string.
+ *
+ * Ineligible flows are `off` even when the option says required. Callers
+ * that parse source should reject that combination before calling this.
+ *
+ * @param option - Author option
+ * @param eligible - {@link idempotencyEligible}
+ */
+export function resolveIdempotencyMode(
+  option: FlowIdempotencyOption | undefined,
+  eligible: boolean,
+): ResolvedIdempotency {
+  const ttl = ttlOf(option);
+  if (!eligible || option === false) return { mode: "off", ttl };
+  if (option === "required") return { mode: "required", ttl };
+  if (typeof option === "object" && option.required === true) return { mode: "required", ttl };
+  return { mode: "auto", ttl };
+}
+
+/**
+ * Parse a TTL string. `0` means the string is not a duration.
+ *
+ * @param ttl - Duration such as `24h`
+ */
+export function idempotencyTtlMs(ttl: string): number {
+  return parseDurationMs(ttl);
+}
+
+/**
+ * `user:<id>`, else `apikey:<id>`, else `anon`.
+ *
+ * @param auth - Principal after the gate
+ */
+export function idempotencyPrincipal(auth: {
+  readonly userId: string | null;
+  readonly apiKeyId?: string | null;
+}): string {
+  if (auth.userId !== null && auth.userId.length > 0) return `user:${auth.userId}`;
+  if (auth.apiKeyId !== null && auth.apiKeyId !== undefined && auth.apiKeyId.length > 0) {
+    return `apikey:${auth.apiKeyId}`;
+  }
+  return "anon";
+}
+
+/**
+ * SHA-256 of canonical JSON `{ flow, input }`. Computed once at claim.
+ *
+ * @param flow - Flow name
+ * @param input - Validated input
+ */
+export function idempotencyFingerprint(flow: string, input: unknown): string {
+  const hasher = new Bun.CryptoHasher("sha256");
+  hasher.update(canonicalJson({ flow, input }));
+  return hasher.digest("hex");
+}
+
+/**
+ * Classify a raw header value.
+ *
+ * @param header - Header value, or null when absent
+ */
+export function classifyIdempotencyKey(header: string | null): "missing" | "invalid" | string {
+  if (header === null) return "missing";
+  const key = header.trim();
+  if (!KEY_PATTERN.test(key)) return "invalid";
+  return key;
+}
+
+/**
+ * True when this attempt recorded a non-read effect. Secrets do not count.
+ * Failed attempts stay on the ledger.
+ *
+ * @param entries - This run's ledger
+ */
+export function attemptedMutation(entries: readonly { readonly kind: EffectKind }[]): boolean {
+  return entries.some((entry) => MUTATING.has(entry.kind));
+}
+
+/**
+ * Replay a stored response and mark it `Idempotent-Replayed`.
+ *
+ * @param row - Completed row
+ */
+export function replayResponse(row: IdempotencyRow): Response {
+  const headers = new Headers();
+  const contentType = row.responseHeaders?.contentType;
+  const location = row.responseHeaders?.location;
+  if (contentType !== undefined && contentType.length > 0) headers.set("content-type", contentType);
+  if (location !== undefined && location.length > 0) headers.set("location", location);
+  headers.set(REPLAYED_HEADER, "true");
+  const status = row.responseStatus ?? 200;
+  if (status === 204) return new Response(null, { status, headers });
+  return new Response(row.responseBody ?? "", { status, headers });
+}
+
+/**
+ * Protocol failure. `IdempotencyInProgress` also sets `Retry-After`.
+ *
+ * @param code - Builtin idempotency code
+ * @param retryAfterSeconds - Seconds until the lease expires, for 409
+ */
+export function idempotencyErrorResponse(
+  code:
+    | "IdempotencyKeyMissing"
+    | "IdempotencyKeyInvalid"
+    | "IdempotencyKeyReused"
+    | "IdempotencyInProgress",
+  retryAfterSeconds?: number,
+): Response {
+  const failure = fail(
+    code,
+    code === "IdempotencyInProgress" ? { retryAfter: retryAfterSeconds } : {},
+  );
+  const headers = new Headers();
+  if (code === "IdempotencyInProgress") {
+    headers.set("retry-after", String(Math.max(1, retryAfterSeconds ?? 1)));
+  }
+  const status =
+    code === "IdempotencyKeyReused" ? 422 : code === "IdempotencyInProgress" ? 409 : 400;
+  return Response.json({ data: null, error: failure.error }, { status, headers });
+}
+
+/**
+ * Buffer a JSON envelope for storage. Streams and non-JSON bodies return
+ * undefined so the caller deletes the in-progress row.
+ *
+ * @param res - Encoded response (not the live stream)
+ */
+export async function storedFromResponse(res: {
+  readonly status: number;
+  readonly headers: { get(name: string): string | null };
+  text(): Promise<string>;
+}): Promise<StoredResponse | undefined> {
+  const contentType = res.headers.get("content-type") ?? undefined;
+  if (contentType?.includes("text/event-stream")) return undefined;
+  if (
+    contentType !== undefined &&
+    !contentType.includes("json") &&
+    !contentType.includes("text/") &&
+    res.status !== 204
+  ) {
+    return undefined;
+  }
+  const location = res.headers.get("location") ?? undefined;
+  const body = res.status === 204 ? "" : await res.text();
+  return {
+    status: res.status,
+    body,
+    ...(contentType !== undefined ? { contentType } : {}),
+    ...(location !== undefined ? { location } : {}),
+  };
+}
+
+/**
+ * Author TTL, or {@link IDEMPOTENCY_DEFAULT_TTL}.
+ *
+ * @param option - Author option
+ */
+export function ttlOf(option: FlowIdempotencyOption | undefined): string {
+  if (typeof option === "object" && option !== null && typeof option.ttl === "string") {
+    return option.ttl;
+  }
+  return IDEMPOTENCY_DEFAULT_TTL;
+}
+
+function canonicalJson(value: unknown): string {
+  if (value === undefined) return "null";
+  if (value === null || typeof value !== "object") return JSON.stringify(value) ?? "null";
+  if (Array.isArray(value)) return `[${value.map((item) => canonicalJson(item)).join(",")}]`;
+  const obj = value as Record<string, unknown>;
+  const keys = Object.keys(obj)
+    .filter((key) => obj[key] !== undefined)
+    .sort();
+  return `{${keys.map((key) => `${JSON.stringify(key)}:${canonicalJson(obj[key])}`).join(",")}}`;
+}

package/src/kernel/index.ts

@@ -212,6 +212,7 @@ export {
   currentAbortSignal,
   isAbortError,
   linkAbort,
+  requestSignal,
   withAbortSignal,
 } from "./abort-scope.ts";
 

package/src/kernel/journal.ts

@@ -10,7 +10,10 @@
 import { mkdir } from "node:fs/promises";
 import { dirname } from "node:path";
 import { okid } from "../okid.ts";
+import type { DecisionLabelStore } from "./decision-label-store.ts";
+import type { IdempotencyStore } from "./idempotency-store.ts";
 import { JournalSuspend } from "./journal-suspend.ts";
+import { lazyRequire } from "./lazy-require.ts";
 
 export { JournalSuspend, isJournalSuspend } from "./journal-suspend.ts";
 
@@ -126,6 +129,24 @@ export interface JournalLeaseStore {
    * @param leaseMs - Lease duration
    */
   acquireLease(runId: string, instanceId: string, now: number, leaseMs: number): Promise<boolean>;
+  /**
+   * Take the lease and write `update(run)` in that same hold.
+   * `undefined` from `update` keeps the previous entries and still holds the lease.
+   * `"lease"` means another holder won. `"missing"` means no run.
+   *
+   * @param runId - Run id
+   * @param instanceId - Claimant instance
+   * @param now - Epoch-ms
+   * @param leaseMs - Lease duration
+   * @param update - Next snapshot, or undefined to leave entries
+   */
+  cas(
+    runId: string,
+    instanceId: string,
+    now: number,
+    leaseMs: number,
+    update: (run: JournalRun) => JournalRun | undefined,
+  ): Promise<"ok" | "missing" | { readonly lease: true; readonly leaseExpiresAt?: number }>;
   /**
    * Release a lease held by `instanceId` (no-op for other holders).
    *
@@ -151,8 +172,45 @@ export interface JournalLeaseStore {
   listOrphans(now: number): Promise<readonly JournalRun[]>;
 }
 
+/**
+ * Load the idempotency table implementation without a static import.
+ * Computed stem so the edge profile does not inline it.
+ */
+function loadIdempotencyStore(): typeof import("./idempotency-store.ts") {
+  return lazyRequire(import.meta.dir, ["idempotency", "store"].join("-"));
+}
+
+/**
+ * Load the decision-label table without a static import.
+ */
+function loadDecisionLabelStore(): typeof import("./decision-label-store.ts") {
+  return lazyRequire(import.meta.dir, ["decision", "label", "store"].join("-"));
+}
+
+/**
+ * Load the agent-event table without a static import.
+ */
+function loadAgentEventStore(): typeof import("./agent-event-store.ts") {
+  return lazyRequire(import.meta.dir, ["agent", "event", "store"].join("-"));
+}
+
 /** Persistence backend for journal runs. */
 export interface JournalStore extends Partial<JournalLeaseStore> {
+  /**
+   * Idempotency records on this same driver. Absent on a custom store that
+   * only implements run `get` / `put`.
+   */
+  readonly idempotency?: IdempotencyStore;
+  /**
+   * Decision labels and the drift flag on this same driver. Opened only when
+   * the app declares decisions.
+   */
+  readonly decisions?: DecisionLabelStore;
+  /**
+   * Agent-run event log on this same driver. Followers resume from these rows
+   * after a restart and from another instance.
+   */
+  readonly agentEvents?: import("./agent-event-store.ts").AgentEventStore;
   /**
    * Load a run by id.
    *
@@ -219,6 +277,7 @@ function leaseMethods(
   load: () => Promise<Map<string, JournalRun>>,
   flush?: (map: Map<string, JournalRun>) => Promise<void>,
 ): JournalLeaseStore {
+  let gate: Promise<void> = Promise.resolve();
   return {
     async acquireLease(runId, instanceId, now, leaseMs) {
       const map = await load();
@@ -229,6 +288,31 @@ function leaseMethods(
       await flush?.(map);
       return true;
     },
+    async cas(runId, instanceId, now, leaseMs, update) {
+      let release!: () => void;
+      const slot = new Promise<void>((resolve) => {
+        release = resolve;
+      });
+      const prev = gate;
+      gate = slot;
+      await prev;
+      try {
+        const map = await load();
+        const run = map.get(runId);
+        if (!run) return "missing";
+        if (!claimable(run, instanceId, now)) {
+          return { lease: true, leaseExpiresAt: run.leaseExpiresAt };
+        }
+        const next = update(cloneRun(run)) ?? run;
+        next.lockedBy = instanceId;
+        next.leaseExpiresAt = now + leaseMs;
+        map.set(runId, next);
+        await flush?.(map);
+        return "ok";
+      } finally {
+        release();
+      }
+    },
     async releaseLease(runId, instanceId) {
       const map = await load();
       const run = map.get(runId);
@@ -286,6 +370,21 @@ export function createMemoryJournalStore(seed?: readonly JournalRun[]): JournalS
       return [...runs.values()].map(cloneRun);
     },
     ...leaseMethods(load),
+    get idempotency(): IdempotencyStore {
+      const created = loadIdempotencyStore().createMemoryIdempotencyStore();
+      Object.defineProperty(this, "idempotency", { value: created });
+      return created;
+    },
+    get decisions(): DecisionLabelStore {
+      const created = loadDecisionLabelStore().createMemoryDecisionLabelStore();
+      Object.defineProperty(this, "decisions", { value: created });
+      return created;
+    },
+    get agentEvents(): import("./agent-event-store.ts").AgentEventStore {
+      const created = loadAgentEventStore().createMemoryAgentEventStore();
+      Object.defineProperty(this, "agentEvents", { value: created });
+      return created;
+    },
   };
 }
 
@@ -332,6 +431,27 @@ export function createFileJournalStore(path: string): JournalStore {
     },
     // Single-host file: leases coordinate same-machine processes only.
     ...leaseMethods(load, flush),
+    get idempotency(): IdempotencyStore {
+      const created = loadIdempotencyStore().createFileIdempotencyStore(
+        `${dirname(path)}/idempotency.json`,
+      );
+      Object.defineProperty(this, "idempotency", { value: created });
+      return created;
+    },
+    get decisions(): DecisionLabelStore {
+      const created = loadDecisionLabelStore().createFileDecisionLabelStore(
+        `${dirname(path)}/decision-labels.json`,
+      );
+      Object.defineProperty(this, "decisions", { value: created });
+      return created;
+    },
+    get agentEvents(): import("./agent-event-store.ts").AgentEventStore {
+      const created = loadAgentEventStore().createFileAgentEventStore(
+        `${dirname(path)}/agent-events`,
+      );
+      Object.defineProperty(this, "agentEvents", { value: created });
+      return created;
+    },
   };
 }
 
@@ -405,6 +525,11 @@ export interface JournalSession {
    * instead of treating the cursor as past them. Clears the in-memory undo stack
    * so frames re-bind on the next walk.
    */
+  /**
+   * Increments on {@link JournalSession.rewind}. Agent run ids key off this
+   * so `flow.retry` replays the same id.
+   */
+  readonly epoch: number;
   rewind(): void;
   /** Registered per-step undos in persist/replay order (LIFO compensate). */
   undoStack(): readonly JournalUndoFrame[];
@@ -428,6 +553,36 @@ export interface JournalSession {
   ): Promise<void>;
 }
 
+/**
+ * A journal session that forwards to a real session once one is assigned.
+ *
+ * Idempotent durable flows start the journal only after the claim, which is
+ * after `fx` has already been created.
+ */
+export function createJournalSlot(): {
+  readonly slot: { session?: JournalSession };
+  readonly facade: JournalSession;
+} {
+  const slot: { session?: JournalSession } = {};
+  const facade = new Proxy({} as JournalSession, {
+    get(_target, prop, receiver) {
+      const session = slot.session;
+      if (!session) {
+        if (prop === "runId") return "";
+        if (prop === "epoch") return 0;
+        if (prop === "stampTenant") return async () => undefined;
+        if (prop === "rewind") return () => undefined;
+        return undefined;
+      }
+      const value = Reflect.get(session, prop, receiver);
+      return typeof value === "function"
+        ? (value as (...args: unknown[]) => unknown).bind(session)
+        : value;
+    },
+  });
+  return { slot, facade };
+}
+
 /** Journal facade. */
 export interface Journal {
   readonly store: JournalStore;
@@ -460,6 +615,7 @@ export function createJournal(options: CreateJournalOptions): Journal {
   function openSession(run: JournalRun, leased: boolean): JournalSession {
     /** Next entry index to consume on replay. */
     let cursor = 0;
+    let epoch = 0;
     let leaseHeld = leased;
     let registrationPass = false;
     let undoExecution = false;
@@ -609,9 +765,13 @@ export function createJournal(options: CreateJournalOptions): Journal {
         await persist();
         return value;
       },
+      get epoch() {
+        return epoch;
+      },
       rewind() {
         cursor = 0;
         undos.length = 0;
+        epoch += 1;
       },
       undoStack() {
         return undos;

package/src/kernel/json-result.ts

@@ -28,6 +28,16 @@ export interface JsonStreamResult {
   ready?: () => Promise<void>;
   /** Set by the kernel to commit journal / Runs after the stream settles. */
   finalize?: () => Promise<void>;
+  /**
+   * Agent result once a streamed `fx.run` settles.
+   * Absent for text and live streams.
+   */
+  result?: Promise<unknown>;
+  /**
+   * Set when iteration parked a durable run, after `[DONE]` is queued.
+   * Finalize commits the journal as sleeping from this.
+   */
+  parked?: { wakeAt: number; label: string };
 }
 
 /**

package/src/kernel/pipeline.test.ts

@@ -14,6 +14,7 @@ import { createClockRuntime } from "../elements/clock.ts";
 import { gate } from "../elements/gate.ts";
 import { createRunsRuntime, memoryRunsDriver } from "../runs/index.ts";
 import { oke } from "./app.ts";
+import { HTTP_FRAME_REDACTED } from "./http-frame.ts";
 import { flow, resetFlowSeq } from "./flow.ts";
 import { on, resetBindings } from "./on.ts";
 import { http } from "./triggers.ts";
@@ -196,6 +197,50 @@ describe("pipeline — Unauthorized for anonymous", () => {
   });
 });
 
+describe("pipeline — HTTP wire frame", () => {
+  test("fetch stores query, headers, and status beside the body", async () => {
+    resetBindings();
+    resetFlowSeq();
+
+    on(
+      http.get("/tasks", { out: z.object({ ok: z.literal(true) }) }).public(),
+      flow("tasks.list", {
+        do: () => ({ ok: true as const }),
+      }),
+    );
+
+    const runs = createRunsRuntime({ driver: memoryRunsDriver });
+    await runs.open();
+    const app = oke({
+      name: "http-frame",
+      gate: { policies: [gate.public] },
+      runs,
+      env: "test",
+    });
+    await app.boot({ env: "test", gates: [gate.public], runs });
+
+    const res = await app.fetch(
+      new Request("http://localhost/tasks?limit=20", {
+        headers: { accept: "application/json", cookie: "sid=secret" },
+      }),
+    );
+    expect(res.status).toBe(200);
+
+    const event = (await runs.all()).find((row) => row.flow === "tasks.list");
+    expect(event?.input).toBeUndefined();
+    expect(event?.http).toMatchObject({
+      request: {
+        method: "GET",
+        path: "/tasks",
+        query: { limit: "20" },
+        headers: { cookie: HTTP_FRAME_REDACTED, accept: "application/json" },
+      },
+      response: { status: 200 },
+    });
+    await app.bootResult?.close();
+  });
+});
+
 describe("pipeline — evaluated gates on the run", () => {
   test("recorded run contains evaluated chain, not just declared names", async () => {
     resetBindings();

package/src/kernel/sse-id.ts

@@ -0,0 +1,32 @@
+/**
+ * SSE id carried on a live agent event without changing its JSON payload.
+ *
+ * The follow log stores a seq. The live `fx.run` frame for that row uses the
+ * same seq as `id:` so a client can resume with `Last-Event-ID`.
+ */
+
+const sseIdBrand: unique symbol = Symbol.for("oke.sse.id");
+
+/**
+ * Copy `value` and attach an SSE id the encoder writes as `id:`.
+ * The id is not enumerable, so it is not part of the JSON payload.
+ *
+ * @param value - Event object
+ * @param id - SSE id (the stored log seq)
+ */
+export function withSseId<T extends object>(value: T, id: string): T {
+  const copy: T = { ...value };
+  Object.defineProperty(copy, sseIdBrand, { value: id, enumerable: false });
+  return copy;
+}
+
+/**
+ * SSE id attached by {@link withSseId}, if any.
+ *
+ * @param value - Stream chunk
+ */
+export function readSseId(value: unknown): string | undefined {
+  if (typeof value !== "object" || value === null) return undefined;
+  const id = (value as { readonly [sseIdBrand]?: unknown })[sseIdBrand];
+  return typeof id === "string" ? id : undefined;
+}

package/src/manifest/diff.ts

@@ -52,6 +52,7 @@ const EFFECT_KEYS = [
   "secrets",
   "calls",
   "fetches",
+  "decides",
 ] as const;
 
 /** Isolation strength: higher = stronger tenant separation. */