khotan-data 0.2.0 → 0.3.1

package/dist/templates/pass.example.ts

@@ -4,43 +4,51 @@
 //
 // Copy this file, rename it for your source/destination pair, and register the
 // exported pass handler in {outputDir}/khotan.ts.
+//
+// IMPORTANT — Workflow step structure:
+// Declare "use step" functions at module top level and pass them only
+// serializable values (the `ctx` object is plain data and is safe to pass).
+// Do NOT nest step functions inside the "use workflow" function — the Workflow
+// compiler cannot hoist closures that capture workflow scope, and they fail at
+// runtime in the sandbox. Keep the workflow body limited to orchestration.
 // ============================================================================
 
 import { pass, type PassContext } from "./pass";
 
-async function stripeToSlackWorkflow(ctx: PassContext) {
-  "use workflow";
-
-  async function forwardEvent() {
-    "use step";
-    console.log("Forwarding webhook event", {
-      eventType: ctx.eventType,
-      khotanRunId: ctx.khotanRunId,
-    });
+// Step: full Node.js access, retried independently. Receives serializable ctx.
+async function forwardEvent(ctx: PassContext) {
+  "use step";
+  console.log("Forwarding webhook event", {
+    eventType: ctx.eventType,
+    khotanRunId: ctx.khotanRunId,
+  });
 
-    await fetch("https://slack.com/api/chat.postMessage", {
-      method: "POST",
-      headers: {
-        Authorization: `Bearer ${ctx.destVars["botToken"] ?? ""}`,
-        "Content-Type": "application/json",
-      },
-      body: JSON.stringify({
-        channel: ctx.destVars["channelId"],
-        text: `Received ${ctx.eventType}`,
-        blocks: [
-          {
-            type: "section",
-            text: {
-              type: "mrkdwn",
-              text: `Received ${ctx.eventType} from Stripe`,
-            },
+  await fetch("https://slack.com/api/chat.postMessage", {
+    method: "POST",
+    headers: {
+      Authorization: `Bearer ${ctx.destVars["botToken"] ?? ""}`,
+      "Content-Type": "application/json",
+    },
+    body: JSON.stringify({
+      channel: ctx.destVars["channelId"],
+      text: `Received ${ctx.eventType}`,
+      blocks: [
+        {
+          type: "section",
+          text: {
+            type: "mrkdwn",
+            text: `Received ${ctx.eventType} from Stripe`,
           },
-        ],
-      }),
-    });
-  }
+        },
+      ],
+    }),
+  });
+}
 
-  await forwardEvent();
+// Workflow: orchestration only. Calls top-level steps with serializable args.
+async function stripeToSlackWorkflow(ctx: PassContext) {
+  "use workflow";
+  await forwardEvent(ctx);
 }
 
 export const stripeToSlack = pass({

package/dist/templates/pass.ts

@@ -85,36 +85,41 @@ export function pass(config: PassConfig): PassRegistration {
 // import { pass, type PassContext } from "./pass";
 // import { plug } from "../plugs/plug";
 //
-// async function pollinateToSlackWorkflow(ctx: PassContext) {
-//   "use workflow";
+// Declare "use step" functions at MODULE TOP LEVEL and pass them serializable
+// values only (`ctx` is plain data). Do NOT nest steps inside the "use workflow"
+// function — closures over workflow scope cannot be hoisted and fail at runtime.
 //
-//   async function forwardEvent() {
-//     "use step";
-//     const cache = khotanCache(ctx, "pollinate-forwarded-events");
-//     const eventId = String(ctx.event["id"] ?? "");
-//     if (eventId && (await cache.get<boolean>(eventId))) return;
+// // Step: top-level, full Node.js access, retried independently.
+// async function forwardEvent(ctx: PassContext) {
+//   "use step";
+//   const cache = khotanCache(ctx, "pollinate-forwarded-events");
+//   const eventId = String(ctx.event["id"] ?? "");
+//   if (eventId && (await cache.get<boolean>(eventId))) return;
 //
-//     // Construct destination plug from destVars
-//     const slackPlug = plug({
-//       name: "slack",
-//       baseUrl: "https://hooks.slack.com",
-//       authType: "bearer",
-//       auth: { bearer: { token: ctx.destVars["token"] ?? "" } },
-//     });
+//   // Construct destination plug from destVars
+//   const slackPlug = plug({
+//     name: "slack",
+//     baseUrl: "https://hooks.slack.com",
+//     authType: "bearer",
+//     auth: { bearer: { token: ctx.destVars["token"] ?? "" } },
+//   });
 //
-//     await slackPlug.post("/services/webhook", {
-//       body: {
-//         text: `Received ${ctx.eventType} event from pollinate`,
-//         event: ctx.event,
-//       },
-//     });
+//   await slackPlug.post("/services/webhook", {
+//     body: {
+//       text: `Received ${ctx.eventType} event from pollinate`,
+//       event: ctx.event,
+//     },
+//   });
 //
-//     if (eventId) {
-//       await cache.set(eventId, true);
-//     }
+//   if (eventId) {
+//     await cache.set(eventId, true);
 //   }
+// }
 //
-//   await forwardEvent();
+// // Workflow: orchestration only.
+// async function pollinateToSlackWorkflow(ctx: PassContext) {
+//   "use workflow";
+//   await forwardEvent(ctx);
 // }
 //
 // export const pollinateToSlack = pass({

package/dist/templates/relay.example.ts

@@ -4,62 +4,70 @@
 //
 // Copy this file, rename it for your source/destination pair, and register the
 // exported flow in {outputDir}/khotan.ts.
+//
+// IMPORTANT — Workflow step structure:
+// Declare "use step" functions at module top level and pass them only
+// serializable values (the `ctx` object is plain data and is safe to pass).
+// Do NOT nest step functions inside the "use workflow" function — the Workflow
+// compiler cannot hoist closures that capture workflow scope, and they fail at
+// runtime in the sandbox. Keep the workflow body limited to orchestration.
 // ============================================================================
 
 import { khotanCache } from "khotan-data/factory";
 import { relay, type RelayContext } from "./relay";
 
-async function shopifyToHubspotWorkflow(ctx: RelayContext) {
-  "use workflow";
+// Step: full Node.js access, retried independently. Receives serializable ctx.
+async function forwardProducts(ctx: RelayContext) {
+  "use step";
+  console.log("Starting relay", {
+    flow: ctx.flow.name,
+    to: ctx.flow.to,
+    khotanRunId: ctx.khotanRunId,
+    runType: ctx.runType,
+  });
 
-  async function forwardProducts() {
-    "use step";
-    console.log("Starting relay", {
-      flow: ctx.flow.name,
-      to: ctx.flow.to,
-      khotanRunId: ctx.khotanRunId,
-      runType: ctx.runType,
-    });
+  const sourceResponse = await fetch("https://source.example.com/products", {
+    headers: {
+      Authorization: `Bearer ${ctx.vars["sourceToken"] ?? ""}`,
+    },
+  });
+  const payload = (await sourceResponse.json()) as {
+    data?: Array<Record<string, unknown>>;
+  };
+  const records = Array.isArray(payload.data) ? payload.data : [];
+  const snapshotCache = khotanCache(ctx, "shopify-products-snapshot");
+  const previousRecords =
+    (await snapshotCache.get<Array<Record<string, unknown>>>("latest")) ?? [];
+
+  await snapshotCache.set("latest", records, { ttl: "6h" });
 
-    const sourceResponse = await fetch("https://source.example.com/products", {
+  for (const record of records) {
+    await fetch("https://destination.example.com/products", {
+      method: "POST",
       headers: {
-        Authorization: `Bearer ${ctx.vars["sourceToken"] ?? ""}`,
+        Authorization: `Bearer ${ctx.vars["destinationToken"] ?? ""}`,
+        "Content-Type": "application/json",
       },
+      body: JSON.stringify(record),
     });
-    const payload = (await sourceResponse.json()) as {
-      data?: Array<Record<string, unknown>>;
-    };
-    const records = Array.isArray(payload.data) ? payload.data : [];
-    const snapshotCache = khotanCache(ctx, "shopify-products-snapshot");
-    const previousRecords =
-      (await snapshotCache.get<Array<Record<string, unknown>>>("latest")) ?? [];
-
-    await snapshotCache.set("latest", records, { ttl: "6h" });
-
-    for (const record of records) {
-      await fetch("https://destination.example.com/products", {
-        method: "POST",
-        headers: {
-          Authorization: `Bearer ${ctx.vars["destinationToken"] ?? ""}`,
-          "Content-Type": "application/json",
-        },
-        body: JSON.stringify(record),
-      });
-    }
-
-    return {
-      extracted: records.length,
-      transformed: records.length,
-      created: records.length,
-      metadata: {
-        relay: ctx.flow.name,
-        to: ctx.flow.to,
-        previousCount: previousRecords.length,
-      },
-    };
   }
 
-  return forwardProducts();
+  return {
+    extracted: records.length,
+    transformed: records.length,
+    created: records.length,
+    metadata: {
+      relay: ctx.flow.name,
+      to: ctx.flow.to,
+      previousCount: previousRecords.length,
+    },
+  };
+}
+
+// Workflow: orchestration only. Calls top-level steps with serializable args.
+async function shopifyToHubspotWorkflow(ctx: RelayContext) {
+  "use workflow";
+  return forwardProducts(ctx);
 }
 
 export const shopifyToHubspotRelay = relay({

package/dist/templates/relay.ts

@@ -51,48 +51,53 @@ export function relay(config: RelayConfig): FlowRegistration {
 // Usage Example (create a file like flows/shopify-to-hubspot.ts)
 // ---------------------------------------------------------------------------
 //
+// Declare "use step" functions at MODULE TOP LEVEL and pass them serializable
+// values only (`ctx` is plain data). Do NOT nest steps inside the "use workflow"
+// function — closures over workflow scope cannot be hoisted and fail at runtime.
+//
 // import { bindWorkflowPlug, khotanCache, relay, type RelayContext } from "khotan-data/factory";
 // import { shopifyPlug } from "../plugs/shopify";
 // import { hubspotPlug } from "../plugs/hubspot";
 //
-// async function shopifyToHubspotWorkflow(ctx: RelayContext) {
-//   "use workflow";
-//
-//   async function forwardProducts() {
-//     "use step";
-//     console.log("Starting relay", {
-//       flow: ctx.flow.name,
-//       to: ctx.flow.to,
-//       khotanRunId: ctx.khotanRunId,
-//       runType: ctx.runType,
-//     });
-//     const shopify = bindWorkflowPlug(shopifyPlug, ctx);
-//     const hubspot = bindWorkflowPlug(hubspotPlug, ctx, "hubspot");
-//
-//     const snapshotCache = khotanCache(ctx, "shopify-products-snapshot");
-//     const previous = await snapshotCache.get<Array<Record<string, unknown>>>("latest");
+// // Step: top-level, full Node.js access, retried independently.
+// async function forwardProducts(ctx: RelayContext) {
+//   "use step";
+//   console.log("Starting relay", {
+//     flow: ctx.flow.name,
+//     to: ctx.flow.to,
+//     khotanRunId: ctx.khotanRunId,
+//     runType: ctx.runType,
+//   });
+//   const shopify = bindWorkflowPlug(shopifyPlug, ctx);
+//   const hubspot = bindWorkflowPlug(hubspotPlug, ctx, "hubspot");
 //
-//     const response = await shopify.get<{ data?: Array<Record<string, unknown>> }>("/products");
-//     const records = Array.isArray(response.data) ? response.data : [];
-//     await snapshotCache.set("latest", records);
+//   const snapshotCache = khotanCache(ctx, "shopify-products-snapshot");
+//   const previous = await snapshotCache.get<Array<Record<string, unknown>>>("latest");
 //
-//     for (const record of records) {
-//       await hubspot.post("/products", { body: record });
-//     }
+//   const response = await shopify.get<{ data?: Array<Record<string, unknown>> }>("/products");
+//   const records = Array.isArray(response.data) ? response.data : [];
+//   await snapshotCache.set("latest", records);
 //
-//     return {
-//       extracted: records.length,
-//       transformed: records.length,
-//       created: records.length,
-//       metadata: {
-//         relay: ctx.flow.name,
-//         to: ctx.flow.to,
-//         previousCount: previous?.length ?? 0,
-//       },
-//     };
+//   for (const record of records) {
+//     await hubspot.post("/products", { body: record });
 //   }
 //
-//   return forwardProducts();
+//   return {
+//     extracted: records.length,
+//     transformed: records.length,
+//     created: records.length,
+//     metadata: {
+//       relay: ctx.flow.name,
+//       to: ctx.flow.to,
+//       previousCount: previous?.length ?? 0,
+//     },
+//   };
+// }
+//
+// // Workflow: orchestration only.
+// async function shopifyToHubspotWorkflow(ctx: RelayContext) {
+//   "use workflow";
+//   return forwardProducts(ctx);
 // }
 //
 // export const shopifyToHubspotRelay = relay({

package/dist/templates/skill-dashboard.md

@@ -20,7 +20,7 @@ The Hub scaffolds three components to `src/components/khotan/`:
 
 | File | Purpose |
 |------|---------|
-| `hub.tsx` | Main `<KhotanHub />` — plug cards, flow table, enable/disable toggles |
+| `hub.tsx` | Main `<KhotanHub />` — plug cards, flow table, enable/disable toggles, per-flow "Run now" trigger |
 | `var-panel.tsx` | Variables panel for configuring plug vars |
 | `wire-panel.tsx` | Webhook subscription management (connect/disconnect) |
 
@@ -44,6 +44,7 @@ Or use `npx khotan add config-page-1` to scaffold a `/config` page automatically
 
 - Lists all registered plugs with status badges (connected/error/idle)
 - Click a plug to see its flows with enable/disable toggles
+- "Run now" button on each flow row triggers a tracked run via `POST /api/khotan/flows/:id/runs` (uses the browser session, so it passes your `authorize` hook)
 - VarPanel: configure plug variables (stored encrypted via `KHOTAN_SECRET`)
 - WirePanel: manage webhook subscriptions (requires wires configured on plug)
 - Debug button on each plug card (visible when `KHOTAN_DEBUG=1`)

package/dist/templates/skill-setup.md

@@ -120,7 +120,11 @@ const khotanData = khotan({
 
 Notes:
 - `authorize` is **not** a replacement for `KHOTAN_SECRET` — that key only
-  encrypts credentials at rest, it does not authenticate requests.
+  encrypts credentials at rest, it does not authenticate requests. Conversely,
+  `KHOTAN_SECRET` is **not** an HTTP credential: do not send it as a Bearer
+  token. Management routes are gated solely by `authorize` (plus the dev-only
+  CLI HMAC token). A rejected request returns `401` with `code:
+  authorize_rejected` and a `hint` explaining how to authenticate.
 - Inbound webhooks (`POST /webhook/:plug`, verified per-plug via `onVerify`),
   the cron dispatcher (`CRON_SECRET`), and debug routes (`KHOTAN_DEBUG`,
   non-production only) are exempt from `authorize` automatically.
@@ -139,6 +143,76 @@ const nextConfig = {
 };
 ```
 
+## Workflow Runtime & Middleware/Proxy
+
+Inflows, outflows, relays, catch, and pass run on **Vercel Workflow**, which
+communicates over `/.well-known/workflow/*`. If your app has a `middleware.ts`
+(or `proxy.ts`) whose `matcher` captures these paths, durable runs **silently
+fail** — steps never get invoked and runs hang.
+
+`npx khotan init` detects a middleware/proxy file and warns when it may
+intercept these paths. Exclude them from the matcher:
+
+```typescript
+// middleware.ts
+export const config = {
+  matcher: ["/((?!_next|.well-known/workflow).*)"],
+};
+```
+
+If you do auth or rewrites manually (not via `matcher`), short-circuit early:
+
+```typescript
+export function middleware(request: NextRequest) {
+  if (request.nextUrl.pathname.startsWith("/.well-known/workflow")) {
+    return NextResponse.next();
+  }
+  // ...your logic
+}
+```
+
+Vercel Workflow also requires AI Gateway OIDC — run `vercel link` and
+`vercel env pull` so `VERCEL_OIDC_TOKEN` is available locally.
+
+## Triggering Flows
+
+Start a flow through khotan (never call the workflow function directly) so run
+tracking and Workflow IDs are recorded. The API is `khotanData.flow(name).start()`:
+
+```typescript
+import khotanData from "@/lib/khotan/khotan";
+
+await khotanData.flow("products-inflow", { plugName: "shopify" }).start({
+  runType: "delta", // or "full"
+});
+```
+
+`plugName` is only needed to disambiguate when the same flow name is registered
+under multiple plugs. There is no `khotanData.api.*` or `flow().run()` surface —
+`flow(name).start(options)` is the single entry point for manual and scheduled
+runs alike. The cron dispatcher (`/api/khotan/cron`) calls this same path.
+
+### Triggering over HTTP (scripts / external services)
+
+There is **no** `POST /flows/:name/run` route. The HTTP trigger is:
+
+```
+POST /api/khotan/flows/{flowId}/runs    body: { "runType": "delta" }
+```
+
+This is a **management route**, so it goes through your `authorize` hook. Common
+gotcha: `KHOTAN_SECRET` is an encryption key, **not** an HTTP credential — sending
+`Authorization: Bearer <KHOTAN_SECRET>` returns `401` with `code: authorize_rejected`.
+To trigger from outside the app, authenticate with a credential your `authorize`
+hook accepts (a session cookie, or your own token you validate inside `authorize`).
+
+Prefer triggering server-side with `khotanData.flow(name).start()` whenever you
+can — it needs no HTTP round-trip or auth.
+
+The `npx khotan flows trigger <name>` CLI works in **dev** without any of this: it
+signs a short-lived HMAC token from `KHOTAN_SECRET` (the `KhotanCLI` auth scheme,
+disabled when `NODE_ENV=production`). The raw secret never leaves your machine.
+
 ## Verify Setup
 
 ```bash
@@ -194,3 +268,5 @@ This keeps sync logic grounded in real API payloads before you write pagination,
 - **"Cannot find module khotan-data"**: Add to `serverExternalPackages` in next.config.ts
 - **Migration fails**: Ensure `DATABASE_URL` is set and Postgres is reachable
 - **Init won't overwrite**: By design — delete the file manually if you need to re-scaffold
+- **Flow/workflow runs hang or never start**: Check your `middleware.ts`/`proxy.ts` matcher excludes `/.well-known/workflow/*` (see "Workflow Runtime & Middleware/Proxy")
+- **Step "is not a function" / fails to resolve at runtime**: Declare `"use step"` functions at module top level and pass `ctx` as an argument — never nest them inside the `"use workflow"` function (closures over workflow scope cannot be hoisted)

package/dist/templates/skill-webhook.md

@@ -110,19 +110,28 @@ npx khotan add catch --yes
 
 Process webhook events durably via Vercel Workflow:
 
-```typescript
-import { catchEvent } from "./webhooks/catch";
-
-const processInvoice = catchEvent(async (ctx) => {
-  "use workflow";
+Declare `"use step"` functions at module top level and pass `ctx` (serializable
+data) as an argument. Nesting steps inside the `"use workflow"` function fails at
+runtime — closures over workflow scope cannot be hoisted.
 
-  async function persist() {
-    "use step";
-    // Write to database — retried on failure
-    await db.insert(invoices).values(ctx.event);
-  }
-
-  await persist();
+```typescript
+import { catchEvent, type CatchContext } from "./webhooks/catch";
+import { db } from "@/db";
+import { invoices } from "@/db/schema";
+
+// Step: top-level, full Node.js access, retried on failure.
+async function persistInvoice(ctx: CatchContext) {
+  "use step";
+  await db.insert(invoices).values(ctx.event);
+}
+
+const processInvoice = catchEvent({
+  name: "stripe-invoices",
+  events: ["invoice.paid"],
+  workflow: async (ctx) => {
+    "use workflow";
+    await persistInvoice(ctx);
+  },
 });
 ```
 
@@ -140,22 +149,35 @@ npx khotan add pass --yes
 
 Forward webhook events to another service:
 
+The context exposes `ctx.event`, `ctx.eventType`, and `ctx.destVars` (the
+decrypted credentials for the destination plug). There is no `ctx.destPlug` —
+construct the destination plug from `destVars` inside a top-level step.
+
 ```typescript
-import { pass } from "./webhooks/pass";
+import { pass, type PassContext } from "./webhooks/pass";
+import { plug } from "@/lib/khotan/plugs/plug";
+
+// Step: top-level. Build the destination plug from ctx.destVars.
+async function forwardToSlackStep(ctx: PassContext) {
+  "use step";
+  const slack = plug({
+    name: "slack",
+    baseUrl: "https://slack.com/api",
+    authType: "bearer",
+    auth: { bearer: { token: ctx.destVars["botToken"] ?? "" } },
+  });
+  await slack.post("/chat.postMessage", {
+    body: { channel: ctx.destVars["channelId"], text: `New event: ${ctx.eventType}` },
+  });
+}
 
 const forwardToSlack = pass({
-  to: "slack",  // Destination plug name (must be registered)
+  name: "stripe-to-slack",
+  to: "slack", // Destination plug name (must be registered)
+  events: ["invoice.paid"],
   workflow: async (ctx) => {
     "use workflow";
-    // ctx.event — the incoming webhook payload
-    // ctx.destVars — destination plug variables
-    async function forward() {
-      "use step";
-      await ctx.destPlug.post("/messages", {
-        body: { text: `New event: ${ctx.event.type}` },
-      });
-    }
-    await forward();
+    await forwardToSlackStep(ctx);
   },
 });
 ```

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "khotan-data",
-  "version": "0.2.0",
-  "description": "Data primitives for TypeScript — ETL pipelines, transforms, and Drizzle Postgres integration.",
+  "version": "0.3.1",
+  "description": "Data sync, ETL, and webhook primitives for Next.js + Drizzle + Postgres. shadcn for data plumbing.",
   "type": "module",
   "main": "./dist/index.cjs",
   "module": "./dist/index.js",
@@ -107,11 +107,11 @@
     "transform",
     "data-management"
   ],
-  "author": "",
+  "author": "Khotan",
   "license": "MIT",
   "repository": {
     "type": "git",
-    "url": ""
+    "url": "https://github.com/khotan-io/khotan-data"
   },
   "engines": {
     "node": ">=18"