khotan-data 0.2.0 → 0.3.1

package/dist/templates/catch.example.ts

@@ -4,29 +4,37 @@
 //
 // Copy this file, rename it for your webhook source/event, and register the
 // exported catch handler in {outputDir}/khotan.ts.
+//
+// IMPORTANT — Workflow step structure:
+// Declare "use step" functions at module top level and pass them only
+// serializable values (the `ctx` object is plain data and is safe to pass).
+// Do NOT nest step functions inside the "use workflow" function — the Workflow
+// compiler cannot hoist closures that capture workflow scope, and they fail at
+// runtime in the sandbox. Keep the workflow body limited to orchestration.
 // ============================================================================
 
 import { catchEvent, type CatchContext } from "./catch";
 
-async function stripeInvoiceCatchWorkflow(ctx: CatchContext) {
-  "use workflow";
-
-  async function persistEvent() {
-    "use step";
-    console.log("Handling webhook event", {
-      eventType: ctx.eventType,
-      khotanRunId: ctx.khotanRunId,
-    });
+// Step: full Node.js access, retried independently. Receives serializable ctx.
+async function persistEvent(ctx: CatchContext) {
+  "use step";
+  console.log("Handling webhook event", {
+    eventType: ctx.eventType,
+    khotanRunId: ctx.khotanRunId,
+  });
 
-    // Khotan already records webhook deliveries. Add app-specific side effects
-    // here, such as updating a local table or enqueueing downstream work.
-    console.log("Webhook payload", {
-      event: ctx.event,
-      headers: ctx.headers,
-    });
-  }
+  // Khotan already records webhook deliveries. Add app-specific side effects
+  // here, such as updating a local table or enqueueing downstream work.
+  console.log("Webhook payload", {
+    event: ctx.event,
+    headers: ctx.headers,
+  });
+}
 
-  await persistEvent();
+// Workflow: orchestration only. Calls top-level steps with serializable args.
+async function stripeInvoiceCatchWorkflow(ctx: CatchContext) {
+  "use workflow";
+  await persistEvent(ctx);
 }
 
 export const stripeInvoiceCatch = catchEvent({

package/dist/templates/catch.ts

@@ -81,26 +81,31 @@ export function catchEvent(config: CatchConfig): CatchRegistration {
 // them to khotan_runs. Use your catch workflow for app-specific side effects
 // and optionally khotanCache(ctx, "name") for dedupe or cursor state.
 //
-// async function pollinateCatchWorkflow(ctx: CatchContext) {
-//   "use workflow";
+// Declare "use step" functions at MODULE TOP LEVEL and pass them serializable
+// values only (`ctx` is plain data). Do NOT nest steps inside the "use workflow"
+// function — closures over workflow scope cannot be hoisted and fail at runtime.
 //
-//   async function notifyOps() {
-//     "use step";
-//     const cache = khotanCache(ctx, "pollinate-webhook-markers");
-//     const eventId = String(ctx.event["id"] ?? "");
-//     if (eventId && (await cache.get<boolean>(eventId))) return;
+// // Step: top-level, full Node.js access, retried independently.
+// async function notifyOps(ctx: CatchContext) {
+//   "use step";
+//   const cache = khotanCache(ctx, "pollinate-webhook-markers");
+//   const eventId = String(ctx.event["id"] ?? "");
+//   if (eventId && (await cache.get<boolean>(eventId))) return;
 //
-//     console.log("Handled webhook", {
-//       eventType: ctx.eventType,
-//       khotanRunId: ctx.khotanRunId,
-//     });
+//   console.log("Handled webhook", {
+//     eventType: ctx.eventType,
+//     khotanRunId: ctx.khotanRunId,
+//   });
 //
-//     if (eventId) {
-//       await cache.set(eventId, true);
-//     }
+//   if (eventId) {
+//     await cache.set(eventId, true);
 //   }
+// }
 //
-//   await notifyOps();
+// // Workflow: orchestration only.
+// async function pollinateCatchWorkflow(ctx: CatchContext) {
+//   "use workflow";
+//   await notifyOps(ctx);
 // }
 //
 // export const pollinateCatch = catchEvent({

package/dist/templates/hub.tsx

@@ -46,7 +46,14 @@ interface Flow {
   enabled: boolean;
   schedule: string | null;
   lastRunAt: string | null;
-  lastRunStatus: "completed" | "partial" | "failed" | "cancelled" | null;
+  lastRunStatus:
+    | "pending"
+    | "running"
+    | "completed"
+    | "partial"
+    | "failed"
+    | "cancelled"
+    | null;
   plugName: string | null;
 }
 
@@ -99,10 +106,11 @@ const runStatusVariant: Record<string, StatusVariant> = {
 export function KhotanHub({
   webhookUrl,
   debugHref,
-  logsHref = "/logs",
+  logsHref,
 }: {
   webhookUrl?: string;
   debugHref?: (plugName: string) => string;
+  /** When set, shows an "Open Logs" button linking here. Hidden if omitted. */
   logsHref?: string;
 } = {}) {
   const [plugs, setPlugs] = useState<Plug[]>([]);
@@ -112,6 +120,11 @@ export function KhotanHub({
   const [error, setError] = useState<unknown>(null);
   const [selectedPlugId, setSelectedPlugId] = useState<string | null>(null);
   const [debugEnabled, setDebugEnabled] = useState(false);
+  const [runningFlowId, setRunningFlowId] = useState<string | null>(null);
+  const [runNotice, setRunNotice] = useState<{
+    type: "ok" | "error";
+    text: string;
+  } | null>(null);
 
   useEffect(() => {
     fetch("/api/khotan/debug")
@@ -147,6 +160,35 @@ export function KhotanHub({
     });
   }
 
+  async function runFlow(flowId: string, flowName: string) {
+    setRunningFlowId(flowId);
+    setRunNotice(null);
+    try {
+      const res = await fetch(`/api/khotan/flows/${flowId}/runs`, {
+        method: "POST",
+        headers: { "Content-Type": "application/json" },
+        body: JSON.stringify({ runType: "full" }),
+      });
+      if (!res.ok) {
+        const data = (await res.json().catch(() => ({}))) as {
+          error?: string;
+          hint?: string;
+        };
+        setRunNotice({
+          type: "error",
+          text: data.hint ?? data.error ?? `Failed to trigger ${flowName}.`,
+        });
+        return;
+      }
+      setRunNotice({ type: "ok", text: `Triggered ${flowName}.` });
+      await fetchData();
+    } catch {
+      setRunNotice({ type: "error", text: `Failed to trigger ${flowName}.` });
+    } finally {
+      setRunningFlowId(null);
+    }
+  }
+
   async function toggleWebhookHandler(handlerId: string, enabled: boolean) {
     setWebhookHandlers((prev) =>
       prev.map((h) => (h.id === handlerId ? { ...h, enabled } : h)),
@@ -227,23 +269,27 @@ export function KhotanHub({
     <div className="space-y-6">
       <div className="flex items-center justify-between gap-4">
         <h2 className="text-2xl font-bold tracking-tight">Khotan Hub</h2>
-        <Button
-          variant="outline"
-          size="sm"
-          onClick={() => {
-            window.location.href = logsHref;
-          }}
-        >
-          Open Logs
-        </Button>
+        {logsHref && (
+          <Button
+            variant="outline"
+            size="sm"
+            onClick={() => {
+              window.location.href = logsHref;
+            }}
+          >
+            Open Logs
+          </Button>
+        )}
       </div>
 
       <div className="grid gap-4 md:grid-cols-2 lg:grid-cols-3">
         {plugs.map((plug) => (
           <Card
             key={plug.id}
-            className={`cursor-pointer transition-colors hover:border-primary ${
-              selectedPlugId === plug.id ? "border-primary" : ""
+            className={`cursor-pointer transition-all hover:border-primary ${
+              selectedPlugId === plug.id
+                ? "border-primary ring-2 ring-primary ring-offset-2 ring-offset-background shadow-md"
+                : "border-border"
             }`}
             onClick={() =>
               setSelectedPlugId(selectedPlugId === plug.id ? null : plug.id)
@@ -287,6 +333,18 @@ export function KhotanHub({
         ))}
       </div>
 
+      {!selectedPlug && (
+        <Card className="border-dashed">
+          <CardContent className="py-10 text-center">
+            <p className="text-muted-foreground mb-1">No plug selected yet.</p>
+            <p className="text-sm text-muted-foreground">
+              Select a plug above to view its flows, webhook handlers, and
+              credentials.
+            </p>
+          </CardContent>
+        </Card>
+      )}
+
       {selectedPlug && (
         <div className="space-y-4">
           <Card>
@@ -294,6 +352,17 @@ export function KhotanHub({
               <CardTitle>{selectedPlug.name} — Flows</CardTitle>
             </CardHeader>
             <CardContent>
+              {runNotice && (
+                <p
+                  className={`mb-3 text-sm ${
+                    runNotice.type === "error"
+                      ? "text-destructive"
+                      : "text-muted-foreground"
+                  }`}
+                >
+                  {runNotice.text}
+                </p>
+              )}
               {plugFlows.length === 0 ? (
                 <p className="text-sm text-muted-foreground">
                   No flows configured for this plug.
@@ -307,6 +376,7 @@ export function KhotanHub({
                       <TableHead>Schedule</TableHead>
                       <TableHead>Last Run</TableHead>
                       <TableHead>Enabled</TableHead>
+                      <TableHead className="text-right">Actions</TableHead>
                     </TableRow>
                   </TableHeader>
                   <TableBody>
@@ -345,6 +415,19 @@ export function KhotanHub({
                             onCheckedChange={(v) => toggleFlow(flow.id, v)}
                           />
                         </TableCell>
+                        <TableCell className="text-right">
+                          <Button
+                            size="sm"
+                            variant="outline"
+                            className="h-7 px-2 text-xs"
+                            disabled={runningFlowId !== null}
+                            onClick={() => runFlow(flow.id, flow.name)}
+                          >
+                            {runningFlowId === flow.id
+                              ? "Running…"
+                              : "Run now"}
+                          </Button>
+                        </TableCell>
                       </TableRow>
                     ))}
                   </TableBody>

package/dist/templates/inflow.example.ts

@@ -4,53 +4,61 @@
 //
 // Copy this file, rename it for your source service/resource, and register the
 // exported flow in {outputDir}/khotan.ts.
+//
+// IMPORTANT — Workflow step structure:
+// Declare "use step" functions at module top level and pass them only
+// serializable values (the `ctx` object is plain data and is safe to pass).
+// Do NOT nest step functions inside the "use workflow" function — the Workflow
+// compiler cannot hoist closures that capture workflow scope, and they fail at
+// runtime in the sandbox. Keep the workflow body limited to orchestration.
 // ============================================================================
 
 import { inflow, type InflowContext } from "./inflow";
 import { sendUpdate } from "khotan-data/factory";
 
-async function shopifyProductsWorkflow(ctx: InflowContext) {
-  "use workflow";
-
-  async function extractAndLoad() {
-    "use step";
-    console.log("Starting inflow", {
-      flow: ctx.flow.name,
-      khotanRunId: ctx.khotanRunId,
-      runType: ctx.runType,
-    });
-    await sendUpdate({
-      message: "Starting product inflow",
-      metadata: { flow: ctx.flow.name, runType: ctx.runType },
-    });
+// Step: full Node.js access, retried independently. Receives serializable ctx.
+async function extractAndLoad(ctx: InflowContext) {
+  "use step";
+  console.log("Starting inflow", {
+    flow: ctx.flow.name,
+    khotanRunId: ctx.khotanRunId,
+    runType: ctx.runType,
+  });
+  await sendUpdate({
+    message: "Starting product inflow",
+    metadata: { flow: ctx.flow.name, runType: ctx.runType },
+  });
 
-    const response = await fetch("https://api.example.com/products", {
-      headers: {
-        Authorization: `Bearer ${ctx.vars["apiToken"] ?? ""}`,
-      },
-    });
-    const payload = (await response.json()) as {
-      data?: Array<Record<string, unknown>>;
-    };
-    const records = Array.isArray(payload.data) ? payload.data : [];
+  const response = await fetch("https://api.example.com/products", {
+    headers: {
+      Authorization: `Bearer ${ctx.vars["apiToken"] ?? ""}`,
+    },
+  });
+  const payload = (await response.json()) as {
+    data?: Array<Record<string, unknown>>;
+  };
+  const records = Array.isArray(payload.data) ? payload.data : [];
 
-    // Replace this with your app-specific transform and DB upsert.
-    console.log("Fetched records", records.length);
-    await sendUpdate({
-      message: `Fetched ${String(records.length)} products`,
-      extracted: records.length,
-      progress: 50,
-    });
+  // Replace this with your app-specific transform and DB upsert.
+  console.log("Fetched records", records.length);
+  await sendUpdate({
+    message: `Fetched ${String(records.length)} products`,
+    extracted: records.length,
+    progress: 50,
+  });
 
-    return {
-      extracted: records.length,
-      transformed: records.length,
-      created: records.length,
-      metadata: { source: ctx.flow.name },
-    };
-  }
+  return {
+    extracted: records.length,
+    transformed: records.length,
+    created: records.length,
+    metadata: { source: ctx.flow.name },
+  };
+}
 
-  return extractAndLoad();
+// Workflow: orchestration only. Calls top-level steps with serializable args.
+async function shopifyProductsWorkflow(ctx: InflowContext) {
+  "use workflow";
+  return extractAndLoad(ctx);
 }
 
 export const shopifyProductsInflow = inflow({

package/dist/templates/inflow.ts

@@ -48,46 +48,52 @@ export function inflow(config: InflowConfig): FlowRegistration {
 // Usage Example (create a file like flows/shopify-products.ts)
 // ---------------------------------------------------------------------------
 //
+// Declare "use step" functions at MODULE TOP LEVEL and pass them serializable
+// values only. The `ctx` object is plain data and is safe to pass. Do NOT nest
+// step functions inside the "use workflow" function — the Workflow compiler
+// cannot hoist closures that capture workflow scope, so they fail at runtime.
+//
 // import { bindWorkflowPlug, inflow, type InflowContext, sendUpdate } from "khotan-data/factory";
 // import { db } from "@/db";
 // import { products } from "@/db/schema";
 // import { shopifyPlug } from "../plugs/shopify";
 //
-// async function shopifyProductsWorkflow(ctx: InflowContext) {
-//   "use workflow";
-//
-//   async function extractAndLoad() {
-//     "use step";
-//     console.log("Starting inflow", {
-//       flow: ctx.flow.name,
-//       khotanRunId: ctx.khotanRunId,
-//       runType: ctx.runType,
-//     });
-//     await sendUpdate({ message: "Starting Shopify products inflow" });
-//     const shopify = bindWorkflowPlug(shopifyPlug, ctx);
-//
-//     const response = await shopify.get<{ data?: Array<{ id: string; sku?: string }> }>("/products");
-//     const records = Array.isArray(response.data) ? response.data : [];
-//     await sendUpdate({ message: `Fetched ${records.length} products`, extracted: records.length });
+// // Step: top-level, full Node.js access, retried independently.
+// async function extractAndLoad(ctx: InflowContext) {
+//   "use step";
+//   console.log("Starting inflow", {
+//     flow: ctx.flow.name,
+//     khotanRunId: ctx.khotanRunId,
+//     runType: ctx.runType,
+//   });
+//   await sendUpdate({ message: "Starting Shopify products inflow" });
+//   const shopify = bindWorkflowPlug(shopifyPlug, ctx);
 //
-//     if (records.length > 0) {
-//       await db.insert(products).values(
-//         records.map((record) => ({
-//           externalId: record.id,
-//           sku: record.sku ?? record.id,
-//         })),
-//       );
-//     }
+//   const response = await shopify.get<{ data?: Array<{ id: string; sku?: string }> }>("/products");
+//   const records = Array.isArray(response.data) ? response.data : [];
+//   await sendUpdate({ message: `Fetched ${records.length} products`, extracted: records.length });
 //
-//     return {
-//       extracted: records.length,
-//       transformed: records.length,
-//       created: records.length,
-//       metadata: { source: ctx.flow.name },
-//     };
+//   if (records.length > 0) {
+//     await db.insert(products).values(
+//       records.map((record) => ({
+//         externalId: record.id,
+//         sku: record.sku ?? record.id,
+//       })),
+//     );
 //   }
 //
-//   return extractAndLoad();
+//   return {
+//     extracted: records.length,
+//     transformed: records.length,
+//     created: records.length,
+//     metadata: { source: ctx.flow.name },
+//   };
+// }
+//
+// // Workflow: orchestration only.
+// async function shopifyProductsWorkflow(ctx: InflowContext) {
+//   "use workflow";
+//   return extractAndLoad(ctx);
 // }
 //
 // export const shopifyProductsInflow = inflow({

package/dist/templates/khotan-config.ts

@@ -42,14 +42,13 @@ const khotanData = khotan({
   plugs: [
     // Example plug registration:
     //
+    // import { stripePlug } from "./plugs/stripe";
+    // import { stripeProductsInflow } from "./flows/stripe-products";
+    //
     // {
     //   name: "stripe",
-    //   baseUrl: "https://api.stripe.com",
-    //   authType: "bearer",
-    //   flows: [
-    //     { name: "products-inflow", type: "inflow", schedule: "0 * * * *", resource: "products" },
-    //     { name: "invoices-inflow", type: "inflow", schedule: "0 0 * * *" },
-    //   ],
+    //   plug: stripePlug,
+    //   flows: [stripeProductsInflow],
     // },
   ],
 
@@ -64,3 +63,14 @@ const khotanData = khotan({
 });
 
 export default khotanData;
+
+// Trigger a flow run from server code (route handler, action, cron):
+//
+//   import khotanData from "@/lib/khotan/khotan";
+//   await khotanData.flow("products-inflow", { plugName: "stripe" }).start({
+//     runType: "delta", // or "full"
+//   });
+//
+// `flow(name).start(options)` is the single entry point — there is no
+// `khotanData.api.*` surface. `plugName` only disambiguates when the same flow
+// name is registered under multiple plugs.

package/dist/templates/outflow.example.ts

@@ -4,44 +4,52 @@
 //
 // Copy this file, rename it for your destination service/resource, and register
 // the exported flow in {outputDir}/khotan.ts.
+//
+// IMPORTANT — Workflow step structure:
+// Declare "use step" functions at module top level and pass them only
+// serializable values (the `ctx` object is plain data and is safe to pass).
+// Do NOT nest step functions inside the "use workflow" function — the Workflow
+// compiler cannot hoist closures that capture workflow scope, and they fail at
+// runtime in the sandbox. Keep the workflow body limited to orchestration.
 // ============================================================================
 
 import { outflow, type OutflowContext } from "./outflow";
 
-async function hubspotProductsWorkflow(ctx: OutflowContext) {
-  "use workflow";
-
-  async function loadAndPush() {
-    "use step";
-    console.log("Starting outflow", {
-      flow: ctx.flow.name,
-      khotanRunId: ctx.khotanRunId,
-      runType: ctx.runType,
+// Step: full Node.js access, retried independently. Receives serializable ctx.
+async function loadAndPush(ctx: OutflowContext) {
+  "use step";
+  console.log("Starting outflow", {
+    flow: ctx.flow.name,
+    khotanRunId: ctx.khotanRunId,
+    runType: ctx.runType,
+  });
+
+  // Replace this with your app-specific DB query.
+  const records: Array<Record<string, unknown>> = [];
+
+  for (const record of records) {
+    await fetch("https://api.example.com/products", {
+      method: "POST",
+      headers: {
+        Authorization: `Bearer ${ctx.vars["apiToken"] ?? ""}`,
+        "Content-Type": "application/json",
+      },
+      body: JSON.stringify(record),
     });
-
-    // Replace this with your app-specific DB query.
-    const records: Array<Record<string, unknown>> = [];
-
-    for (const record of records) {
-      await fetch("https://api.example.com/products", {
-        method: "POST",
-        headers: {
-          Authorization: `Bearer ${ctx.vars["apiToken"] ?? ""}`,
-          "Content-Type": "application/json",
-        },
-        body: JSON.stringify(record),
-      });
-    }
-
-    return {
-      extracted: records.length,
-      transformed: records.length,
-      created: records.length,
-      metadata: { destination: ctx.flow.name },
-    };
   }
 
-  return loadAndPush();
+  return {
+    extracted: records.length,
+    transformed: records.length,
+    created: records.length,
+    metadata: { destination: ctx.flow.name },
+  };
+}
+
+// Workflow: orchestration only. Calls top-level steps with serializable args.
+async function hubspotProductsWorkflow(ctx: OutflowContext) {
+  "use workflow";
+  return loadAndPush(ctx);
 }
 
 export const hubspotProductsOutflow = outflow({

package/dist/templates/outflow.ts

@@ -48,38 +48,43 @@ export function outflow(config: OutflowConfig): FlowRegistration {
 // Usage Example (create a file like flows/hubspot-products.ts)
 // ---------------------------------------------------------------------------
 //
+// Declare "use step" functions at MODULE TOP LEVEL and pass them serializable
+// values only (`ctx` is plain data). Do NOT nest steps inside the "use workflow"
+// function — closures over workflow scope cannot be hoisted and fail at runtime.
+//
 // import { bindWorkflowPlug, outflow, type OutflowContext } from "khotan-data/factory";
 // import { db } from "@/db";
 // import { products } from "@/db/schema";
 // import { hubspotPlug } from "../plugs/hubspot";
 //
-// async function hubspotProductsWorkflow(ctx: OutflowContext) {
-//   "use workflow";
-//
-//   async function loadAndPush() {
-//     "use step";
-//     console.log("Starting outflow", {
-//       flow: ctx.flow.name,
-//       khotanRunId: ctx.khotanRunId,
-//       runType: ctx.runType,
-//     });
-//     const hubspot = bindWorkflowPlug(hubspotPlug, ctx);
-//
-//     const records = await db.select().from(products);
+// // Step: top-level, full Node.js access, retried independently.
+// async function loadAndPush(ctx: OutflowContext) {
+//   "use step";
+//   console.log("Starting outflow", {
+//     flow: ctx.flow.name,
+//     khotanRunId: ctx.khotanRunId,
+//     runType: ctx.runType,
+//   });
+//   const hubspot = bindWorkflowPlug(hubspotPlug, ctx);
 //
-//     for (const record of records) {
-//       await hubspot.post("/products", { body: record });
-//     }
+//   const records = await db.select().from(products);
 //
-//     return {
-//       extracted: records.length,
-//       transformed: records.length,
-//       created: records.length,
-//       metadata: { destination: ctx.flow.name },
-//     };
+//   for (const record of records) {
+//     await hubspot.post("/products", { body: record });
 //   }
 //
-//   return loadAndPush();
+//   return {
+//     extracted: records.length,
+//     transformed: records.length,
+//     created: records.length,
+//     metadata: { destination: ctx.flow.name },
+//   };
+// }
+//
+// // Workflow: orchestration only.
+// async function hubspotProductsWorkflow(ctx: OutflowContext) {
+//   "use workflow";
+//   return loadAndPush(ctx);
 // }
 //
 // export const hubspotProductsOutflow = outflow({