okengine 0.21.0 → 0.22.0

package/src/compiler/extract.test.ts

@@ -63,6 +63,7 @@ describe("extractManifest — five trigger types", () => {
       http: { method: "POST", path: "/orders" },
     });
     expect(manifest.flows?.["triggers.http"]?.effects?.writes).toEqual(["sql:orders"]);
+    expect(manifest.flows?.["triggers.http"]?.idempotency).toEqual({ mode: "auto", ttl: "24h" });
 
     expect(manifest.flows?.["triggers.every"]?.trigger).toEqual({
       every: "10m",
@@ -86,6 +87,7 @@ describe("extractManifest — five trigger types", () => {
       true,
     );
     expect(internal?.effects?.reads).toEqual(["sql:links"]);
+    expect(internal?.idempotency).toEqual({ mode: "off", ttl: "24h" });
   });
 });
 
@@ -99,9 +101,44 @@ describe("extractManifest — fx.raw", () => {
     });
 
     expect(manifest.flows?.["raw.unannotated"]?.cache).toBe(false);
+    expect(manifest.flows?.["raw.unannotated"]?.usesRaw).toBe(true);
+    expect(manifest.flows?.["raw.unannotated"]?.idempotency).toEqual({ mode: "off", ttl: "24h" });
     expect(manifest.flows?.["raw.annotated"]?.cache).toBeUndefined();
+    expect(manifest.flows?.["raw.annotated"]?.usesRaw).toBe(true);
     expect(manifest.flows?.["raw.annotated"]?.effects?.reads).toEqual(["sql:orders"]);
   });
+
+  test("required on a GET flow is an extract error; a bad ttl is an extract error", async () => {
+    const requiredGet = `
+      import { on, flow, http } from "okengine";
+      export const read = on(
+        http.get("/read"),
+        flow("pay.read", { idempotency: "required", do: async () => ({ ok: true }) }),
+      );
+    `;
+    await expect(extractFromSources({ "src/flows/pay.ts": requiredGet })).rejects.toThrow(
+      /header would be ignored/,
+    );
+
+    const badTtl = `
+      import { on, flow, http, store, field } from "okengine";
+      export const db = store.sql("db");
+      export const payments = store.schema.table("payments", { id: field.text().primaryKey() });
+      export const charge = on(
+        http.post("/charge"),
+        flow("pay.charge", {
+          idempotency: { ttl: "nope" },
+          do: async (input, fx) => {
+            await fx.store(db).insert(payments).values(input);
+            return { ok: true };
+          },
+        }),
+      );
+    `;
+    await expect(extractFromSources({ "src/flows/pay.ts": badTtl })).rejects.toThrow(
+      /not a duration/,
+    );
+  });
 });
 
 describe("extractManifest — performance", () => {

package/src/compiler/extract.ts

@@ -6,6 +6,7 @@
  * from source to `manifest.oke.json`.
  */
 
+import { readdir } from "node:fs/promises";
 import { parseSync } from "oxc-parser";
 import { join } from "node:path";
 
@@ -15,6 +16,13 @@ import {
   getAiProviderEntry,
 } from "../elements/ai/providers.ts";
 import { buildCronExpression, type CronField } from "../elements/clock/cron-fields.ts";
+import {
+  hasMutatingEffects,
+  idempotencyTtlMs,
+  resolveIdempotencyMode,
+  ttlOf,
+  type FlowIdempotencyOption,
+} from "../kernel/idempotency.ts";
 import { SearchConfigError } from "../elements/store/search-errors.ts";
 
 import type {
@@ -45,6 +53,7 @@ import type {
 } from "../manifest/types.ts";
 import { sqlTableRef } from "../manifest/sql-resource.ts";
 import {
+  effectsMissingFromDeclaration,
   identifierName,
   inferEffects,
   resolveCallTarget,
@@ -56,6 +65,7 @@ import {
   type InferBinding,
   type Literal,
 } from "./effects-infer.ts";
+import { buildFxIndex, readTsconfigPaths, type FxIndex } from "./fx-index.ts";
 import {
   isFlowsTreeFile,
   isSkippedExtractSource,
@@ -131,6 +141,8 @@ interface ProjectScope {
   /** Store-wide live defaulting — `oke({ store: { live: true } })`. */
   store?: Manifest["store"];
   flows: Record<string, Flow>;
+  /** Project functions that can receive `fx`, built once per extract. */
+  fxIndex: FxIndex;
   /** Export name → flow id (for agent tools). */
   flowExports: Map<string, string>;
   /** Local binding name → store.resource declaration (for on(http.resource)). */
@@ -168,6 +180,7 @@ export async function extractManifest(options: ExtractManifestOptions = {}): Pro
 
   const scope: ProjectScope = {
     app: options.app ?? "app",
+    fxIndex: buildFxIndex([]),
     bindings: new Map(),
     signals: {},
     stores: {},
@@ -198,6 +211,12 @@ export async function extractManifest(options: ExtractManifestOptions = {}): Pro
     return { file, program: result.program as unknown as AstNode };
   });
 
+  const tsPaths = options.rootDir ? await readTsconfigPaths(options.rootDir) : [];
+  scope.fxIndex = buildFxIndex(
+    parsed.map(({ file, program }) => ({ path: file.path, program })),
+    tsPaths,
+  );
+
   // Pass 1 — declarations (elements, config, named bindings).
   for (const { file, program } of parsed) {
     collectDeclarations(file.path, program, scope);
@@ -295,21 +314,50 @@ export async function extractFromSources(
 }
 
 async function readSources(rootDir: string, pattern: string): Promise<SourceFile[]> {
-  // Bun.Glob (global) — bare `import … from "bun"` is rejected by JSR.
-  const glob = new Bun.Glob(pattern);
   const files: SourceFile[] = [];
-  for await (const path of glob.scan({
-    cwd: rootDir,
-    onlyFiles: true,
-  })) {
-    if (isSkippedExtractSource(path)) continue;
-    const abs = join(rootDir, path);
-    files.push({ path: toPosixPath(path), source: await Bun.file(abs).text() });
+  if (pattern === "**/*.{ts,tsx}") {
+    await walkSources(rootDir, rootDir, files);
+  } else {
+    // Bun.Glob (global) — bare `import … from "bun"` is rejected by JSR.
+    const glob = new Bun.Glob(pattern);
+    for await (const path of glob.scan({ cwd: rootDir, onlyFiles: true })) {
+      if (isSkippedExtractSource(path)) continue;
+      const abs = join(rootDir, path);
+      files.push({ path: toPosixPath(path), source: await Bun.file(abs).text() });
+    }
   }
   files.sort((a, b) => a.path.localeCompare(b.path));
   return files;
 }
 
+/**
+ * Walk an app tree without entering `node_modules` or directory symlinks.
+ *
+ * A template's `node_modules/okengine` link points back at this repo, which
+ * contains the template again. Globbing through that link never terminates.
+ */
+async function walkSources(rootDir: string, dir: string, files: SourceFile[]): Promise<void> {
+  let entries;
+  try {
+    entries = await readdir(dir, { withFileTypes: true });
+  } catch {
+    return;
+  }
+  for (const entry of entries) {
+    if (entry.name === "node_modules" || entry.name === ".git" || entry.isSymbolicLink()) continue;
+    const abs = join(dir, entry.name);
+    if (entry.isDirectory()) {
+      await walkSources(rootDir, abs, files);
+      continue;
+    }
+    if (!entry.isFile()) continue;
+    if (!entry.name.endsWith(".ts") && !entry.name.endsWith(".tsx")) continue;
+    const rel = toPosixPath(abs.slice(rootDir.length + 1));
+    if (isSkippedExtractSource(rel)) continue;
+    files.push({ path: rel, source: await Bun.file(abs).text() });
+  }
+}
+
 function collectDeclarations(filePath: string, program: AstNode, scope: ProjectScope): void {
   walk(program, (node) => {
     if (node.type === "CallExpression") {
@@ -1882,13 +1930,17 @@ function registerFlow(args: {
   }
   args.scope.bindings.set(name, { kind: "flow", ref: name });
 
-  const hasExplicitEffects = objectProp(opts, "effects") !== undefined;
+  const effectsNode = objectProp(opts, "effects");
+  const hasExplicitEffects = effectsNode !== undefined;
   const doNode = objectProp(opts, "do");
   const inferred = doNode
     ? inferEffects({
         doNode,
         bindings: args.scope.bindings,
         hasExplicitEffects,
+        flowName: name,
+        file: args.file.path,
+        resolveCallee: (callee, file) => args.scope.fxIndex.resolve(file, callee),
       })
     : {
         effects: {} as Effects,
@@ -1900,8 +1952,16 @@ function registerFlow(args: {
       };
 
   let effects: Effects | undefined;
-  if (hasExplicitEffects) {
-    effects = parseEffectsObject(objectProp(opts, "effects"));
+  if (hasExplicitEffects && effectsNode?.type === "ObjectExpression") {
+    effects = parseEffectsObject(effectsNode);
+    const missing = effectsMissingFromDeclaration(effects ?? {}, inferred.effects);
+    if (missing.length > 0) {
+      throw new Error(
+        `OKE1900: flow "${name}" declares effects that omit ${missing.join(", ")}. An effects block must include every effect inference can see.`,
+      );
+    }
+  } else if (hasExplicitEffects) {
+    effects = parseEffectsObject(effectsNode);
   } else {
     effects = inferred.effects;
   }
@@ -2039,6 +2099,24 @@ function registerFlow(args: {
     flow.cacheKeys = `computed:${effects.reads[0]}/userId`;
   }
 
+  const idempotencyOption = parseIdempotencyOption(opts);
+  const httpMethod = manifestTrigger?.http?.method;
+  const stream = boolProp(opts, "stream") === true;
+  const live = typeof liveFromTrigger === "string";
+  const canHonor =
+    httpMethod !== "GET" && !stream && !live && (inferred.usesRaw || hasMutatingEffects(effects));
+  if (idempotencyRequired(idempotencyOption) && !canHonor) {
+    throw new Error(
+      `flow "${name}" sets idempotency required, but the header would be ignored (read-only, GET, stream, or live).`,
+    );
+  }
+  const ttl = ttlOf(idempotencyOption);
+  if (idempotencyTtlMs(ttl) <= 0) {
+    throw new Error(`flow "${name}" has an idempotency ttl that is not a duration ("${ttl}").`);
+  }
+  flow.idempotency = resolveIdempotencyMode(idempotencyOption, canHonor);
+  if (inferred.usesRaw) flow.usesRaw = true;
+
   putFlow(args.scope, name, flow);
 }
 
@@ -3279,6 +3357,29 @@ function numberProp(obj: AstNode | undefined, key: string): number | undefined {
   return typeof v === "number" ? v : undefined;
 }
 
+function parseIdempotencyOption(opts: AstNode | undefined): FlowIdempotencyOption | undefined {
+  const node = objectProp(opts, "idempotency");
+  if (!node) return undefined;
+  if (node.type === "Literal") {
+    const value = (node as Literal).value;
+    if (value === false) return false;
+    if (value === "required") return "required";
+  }
+  if (node.type === "ObjectExpression") {
+    const required = boolProp(node, "required");
+    const ttl = stringProp(node, "ttl");
+    return {
+      ...(required !== undefined ? { required } : {}),
+      ...(ttl !== undefined ? { ttl } : {}),
+    };
+  }
+  return undefined;
+}
+
+function idempotencyRequired(option: FlowIdempotencyOption | undefined): boolean {
+  return option === "required" || (typeof option === "object" && option.required === true);
+}
+
 function boolProp(obj: AstNode | undefined, key: string): boolean | undefined {
   const node = objectProp(obj, key);
   if (!node || node.type !== "Literal") return undefined;

package/src/compiler/fixtures/skyport.expected.json

@@ -3,6 +3,10 @@
   "app": "skyport",
   "flows": {
     "bookings.create": {
+      "idempotency": {
+        "mode": "auto",
+        "ttl": "24h"
+      },
       "trigger": {
         "http": {
           "method": "POST",
@@ -29,12 +33,20 @@
       "plane": "user"
     },
     "bookings.getBooking": {
+      "idempotency": {
+        "mode": "off",
+        "ttl": "24h"
+      },
       "effects": {
         "reads": ["sql:bookings"]
       },
       "source": "src/flows/bookings/index.ts:50"
     },
     "bookings.mine": {
+      "idempotency": {
+        "mode": "off",
+        "ttl": "24h"
+      },
       "trigger": {
         "http": {
           "method": "GET",
@@ -49,6 +61,10 @@
       "cacheKeys": "computed:sql:bookings/userId"
     },
     "bookings.refundBooking": {
+      "idempotency": {
+        "mode": "auto",
+        "ttl": "24h"
+      },
       "effects": {
         "reads": ["sql:bookings"],
         "writes": ["sql:bookings"]
@@ -56,6 +72,10 @@
       "source": "src/flows/bookings/index.ts:54"
     },
     "payments.chargeBooking": {
+      "idempotency": {
+        "mode": "off",
+        "ttl": "24h"
+      },
       "effects": {
         "secrets": ["STRIPE_KEY"]
       },
@@ -64,6 +84,10 @@
       "steps": ["create-intent", "confirm"]
     },
     "support.createTicket": {
+      "idempotency": {
+        "mode": "auto",
+        "ttl": "24h"
+      },
       "trigger": {
         "http": {
           "method": "POST",

package/src/compiler/fx-follow.test.ts

@@ -0,0 +1,164 @@
+/**
+ * Effect inference follows `fx` into helpers and same-function chain aliases.
+ */
+
+import { describe, expect, test } from "bun:test";
+import { extractFromSources } from "./extract.ts";
+
+const header = `
+import { on, flow, http, store } from "okengine";
+export const db = store.sql("app");
+export const notes = store.schema.table("notes", { id: field.text().primaryKey() });
+`;
+
+describe("extractManifest — fx helpers and chain aliases", () => {
+  test("a same-file helper's store read is part of the flow", async () => {
+    const manifest = await extractFromSources({
+      "src/flows/notes/get.ts": `
+${header}
+async function load(fx: Fx) {
+  await fx.store(db).select().from(notes);
+}
+export const get = on(http.get("/notes"), flow("notes.get", {
+  do: async (_input, fx) => {
+    await load(fx);
+    return { ok: true };
+  },
+}));
+`,
+    });
+    expect(manifest.flows?.["notes.get"]?.effects?.reads).toEqual(["sql:notes"]);
+  });
+
+  test("an imported helper's store read is part of the flow", async () => {
+    const manifest = await extractFromSources({
+      "src/flows/notes/_shared.ts": `
+${header}
+export async function load(fx: Fx) {
+  await fx.store(db).select().from(notes);
+}
+`,
+      "src/flows/notes/get.ts": `
+import { on, flow, http } from "okengine";
+import { load } from "./_shared";
+export const get = on(http.get("/notes"), flow("notes.get", {
+  do: async (_input, fx) => load(fx),
+}));
+`,
+    });
+    expect(manifest.flows?.["notes.get"]?.effects?.reads).toEqual(["sql:notes"]);
+  });
+
+  test("a same-function chain alias records the read", async () => {
+    const manifest = await extractFromSources({
+      "src/flows/notes/get.ts": `
+${header}
+export const get = on(http.get("/notes"), flow("notes.get", {
+  do: async (_input, fx) => {
+    const q = fx.store(db);
+    const rows = await q.select().from(notes);
+    return rows;
+  },
+}));
+`,
+    });
+    expect(manifest.flows?.["notes.get"]?.effects?.reads).toEqual(["sql:notes"]);
+  });
+
+  test("an as-cast store alias records the index write", async () => {
+    const manifest = await extractFromSources({
+      "src/flows/search/index.ts": `
+import { on, flow, http, store } from "okengine";
+export const taskIndex = store.index("tasks");
+export const sync = on(http.post("/sync"), flow("search.sync", {
+  do: async (_input, fx) => {
+    const idx = fx.store(taskIndex) as {
+      driverId: string;
+      upsert: (id: string, doc: unknown) => Promise<void>;
+    };
+    if (idx.driverId === "meilisearch") {
+      await idx.upsert("1", { id: "1" });
+    }
+  },
+}));
+`,
+    });
+    expect(manifest.flows?.["search.sync"]?.effects?.writes).toEqual(["index:tasks"]);
+  });
+
+  test("a chain alias that escapes fails extract", async () => {
+    const bodies = [
+      `const q = fx.store(db); return q;`,
+      `const q = fx.store(db); await other(q);`,
+      `let q = fx.store(db); q = fx.store(db);`,
+      `const q = fx.store(db); const bag = { q }; return bag;`,
+      `const q = fx.store(db); const bag = [q]; return bag;`,
+    ];
+    for (const body of bodies) {
+      await expect(
+        extractFromSources({
+          "src/flows/notes/get.ts": `
+${header}
+export const get = on(http.get("/notes"), flow("notes.get", {
+  do: async (_input, fx) => {
+    ${body}
+  },
+}));
+`,
+        }),
+      ).rejects.toThrow(/OKE1900/);
+    }
+  });
+
+  test("renaming or destructuring fx fails extract", async () => {
+    const bodies = [
+      `do: async (input, ctx) => ({ ok: true })`,
+      `do: async (_input, fx) => { const ctx = fx; return ctx.id(); }`,
+      `do: async (_input, fx) => { const { store } = fx; return store; }`,
+    ];
+    for (const body of bodies) {
+      await expect(
+        extractFromSources({
+          "src/flows/notes/get.ts": `
+${header}
+export const get = on(http.get("/notes"), flow("notes.get", { ${body} }));
+`,
+        }),
+      ).rejects.toThrow(/OKE1900/);
+    }
+  });
+
+  test("an effects block must cover inferred reads and may add keys", async () => {
+    await expect(
+      extractFromSources({
+        "src/flows/notes/get.ts": `
+${header}
+async function load(fx: Fx) {
+  await fx.store(db).select().from(notes);
+}
+export const get = on(http.get("/notes"), flow("notes.get", {
+  effects: { reads: ["sql:other"] },
+  do: async (_input, fx) => load(fx),
+}));
+`,
+      }),
+    ).rejects.toThrow(/OKE1900/);
+
+    const manifest = await extractFromSources({
+      "src/flows/notes/get.ts": `
+${header}
+async function load(fx: Fx) {
+  await fx.store(db).select().from(notes);
+}
+export const get = on(http.get("/notes"), flow("notes.get", {
+  effects: { reads: ["sql:notes"], writes: ["kv:extra"] },
+  do: async (_input, fx) => load(fx),
+}));
+`,
+    });
+    expect(manifest.flows?.["notes.get"]?.effects).toMatchObject({
+      reads: ["sql:notes"],
+      writes: ["kv:extra"],
+    });
+  });
+});