khotan-data 0.1.1 → 0.3.0

package/dist/templates/outflow.example.ts

@@ -4,44 +4,52 @@
 //
 // Copy this file, rename it for your destination service/resource, and register
 // the exported flow in {outputDir}/khotan.ts.
+//
+// IMPORTANT — Workflow step structure:
+// Declare "use step" functions at module top level and pass them only
+// serializable values (the `ctx` object is plain data and is safe to pass).
+// Do NOT nest step functions inside the "use workflow" function — the Workflow
+// compiler cannot hoist closures that capture workflow scope, and they fail at
+// runtime in the sandbox. Keep the workflow body limited to orchestration.
 // ============================================================================
 
 import { outflow, type OutflowContext } from "./outflow";
 
-async function hubspotProductsWorkflow(ctx: OutflowContext) {
-  "use workflow";
-
-  async function loadAndPush() {
-    "use step";
-    console.log("Starting outflow", {
-      flow: ctx.flow.name,
-      khotanRunId: ctx.khotanRunId,
-      runType: ctx.runType,
+// Step: full Node.js access, retried independently. Receives serializable ctx.
+async function loadAndPush(ctx: OutflowContext) {
+  "use step";
+  console.log("Starting outflow", {
+    flow: ctx.flow.name,
+    khotanRunId: ctx.khotanRunId,
+    runType: ctx.runType,
+  });
+
+  // Replace this with your app-specific DB query.
+  const records: Array<Record<string, unknown>> = [];
+
+  for (const record of records) {
+    await fetch("https://api.example.com/products", {
+      method: "POST",
+      headers: {
+        Authorization: `Bearer ${ctx.vars["apiToken"] ?? ""}`,
+        "Content-Type": "application/json",
+      },
+      body: JSON.stringify(record),
     });
-
-    // Replace this with your app-specific DB query.
-    const records: Array<Record<string, unknown>> = [];
-
-    for (const record of records) {
-      await fetch("https://api.example.com/products", {
-        method: "POST",
-        headers: {
-          Authorization: `Bearer ${ctx.vars["apiToken"] ?? ""}`,
-          "Content-Type": "application/json",
-        },
-        body: JSON.stringify(record),
-      });
-    }
-
-    return {
-      extracted: records.length,
-      transformed: records.length,
-      created: records.length,
-      metadata: { destination: ctx.flow.name },
-    };
   }
 
-  return loadAndPush();
+  return {
+    extracted: records.length,
+    transformed: records.length,
+    created: records.length,
+    metadata: { destination: ctx.flow.name },
+  };
+}
+
+// Workflow: orchestration only. Calls top-level steps with serializable args.
+async function hubspotProductsWorkflow(ctx: OutflowContext) {
+  "use workflow";
+  return loadAndPush(ctx);
 }
 
 export const hubspotProductsOutflow = outflow({

package/dist/templates/outflow.ts

@@ -48,38 +48,43 @@ export function outflow(config: OutflowConfig): FlowRegistration {
 // Usage Example (create a file like flows/hubspot-products.ts)
 // ---------------------------------------------------------------------------
 //
+// Declare "use step" functions at MODULE TOP LEVEL and pass them serializable
+// values only (`ctx` is plain data). Do NOT nest steps inside the "use workflow"
+// function — closures over workflow scope cannot be hoisted and fail at runtime.
+//
 // import { bindWorkflowPlug, outflow, type OutflowContext } from "khotan-data/factory";
 // import { db } from "@/db";
 // import { products } from "@/db/schema";
 // import { hubspotPlug } from "../plugs/hubspot";
 //
-// async function hubspotProductsWorkflow(ctx: OutflowContext) {
-//   "use workflow";
-//
-//   async function loadAndPush() {
-//     "use step";
-//     console.log("Starting outflow", {
-//       flow: ctx.flow.name,
-//       khotanRunId: ctx.khotanRunId,
-//       runType: ctx.runType,
-//     });
-//     const hubspot = bindWorkflowPlug(hubspotPlug, ctx);
-//
-//     const records = await db.select().from(products);
+// // Step: top-level, full Node.js access, retried independently.
+// async function loadAndPush(ctx: OutflowContext) {
+//   "use step";
+//   console.log("Starting outflow", {
+//     flow: ctx.flow.name,
+//     khotanRunId: ctx.khotanRunId,
+//     runType: ctx.runType,
+//   });
+//   const hubspot = bindWorkflowPlug(hubspotPlug, ctx);
 //
-//     for (const record of records) {
-//       await hubspot.post("/products", { body: record });
-//     }
+//   const records = await db.select().from(products);
 //
-//     return {
-//       extracted: records.length,
-//       transformed: records.length,
-//       created: records.length,
-//       metadata: { destination: ctx.flow.name },
-//     };
+//   for (const record of records) {
+//     await hubspot.post("/products", { body: record });
 //   }
 //
-//   return loadAndPush();
+//   return {
+//     extracted: records.length,
+//     transformed: records.length,
+//     created: records.length,
+//     metadata: { destination: ctx.flow.name },
+//   };
+// }
+//
+// // Workflow: orchestration only.
+// async function hubspotProductsWorkflow(ctx: OutflowContext) {
+//   "use workflow";
+//   return loadAndPush(ctx);
 // }
 //
 // export const hubspotProductsOutflow = outflow({

package/dist/templates/pass.example.ts

@@ -4,43 +4,51 @@
 //
 // Copy this file, rename it for your source/destination pair, and register the
 // exported pass handler in {outputDir}/khotan.ts.
+//
+// IMPORTANT — Workflow step structure:
+// Declare "use step" functions at module top level and pass them only
+// serializable values (the `ctx` object is plain data and is safe to pass).
+// Do NOT nest step functions inside the "use workflow" function — the Workflow
+// compiler cannot hoist closures that capture workflow scope, and they fail at
+// runtime in the sandbox. Keep the workflow body limited to orchestration.
 // ============================================================================
 
 import { pass, type PassContext } from "./pass";
 
-async function stripeToSlackWorkflow(ctx: PassContext) {
-  "use workflow";
-
-  async function forwardEvent() {
-    "use step";
-    console.log("Forwarding webhook event", {
-      eventType: ctx.eventType,
-      khotanRunId: ctx.khotanRunId,
-    });
+// Step: full Node.js access, retried independently. Receives serializable ctx.
+async function forwardEvent(ctx: PassContext) {
+  "use step";
+  console.log("Forwarding webhook event", {
+    eventType: ctx.eventType,
+    khotanRunId: ctx.khotanRunId,
+  });
 
-    await fetch("https://slack.com/api/chat.postMessage", {
-      method: "POST",
-      headers: {
-        Authorization: `Bearer ${ctx.destVars["botToken"] ?? ""}`,
-        "Content-Type": "application/json",
-      },
-      body: JSON.stringify({
-        channel: ctx.destVars["channelId"],
-        text: `Received ${ctx.eventType}`,
-        blocks: [
-          {
-            type: "section",
-            text: {
-              type: "mrkdwn",
-              text: `Received ${ctx.eventType} from Stripe`,
-            },
+  await fetch("https://slack.com/api/chat.postMessage", {
+    method: "POST",
+    headers: {
+      Authorization: `Bearer ${ctx.destVars["botToken"] ?? ""}`,
+      "Content-Type": "application/json",
+    },
+    body: JSON.stringify({
+      channel: ctx.destVars["channelId"],
+      text: `Received ${ctx.eventType}`,
+      blocks: [
+        {
+          type: "section",
+          text: {
+            type: "mrkdwn",
+            text: `Received ${ctx.eventType} from Stripe`,
           },
-        ],
-      }),
-    });
-  }
+        },
+      ],
+    }),
+  });
+}
 
-  await forwardEvent();
+// Workflow: orchestration only. Calls top-level steps with serializable args.
+async function stripeToSlackWorkflow(ctx: PassContext) {
+  "use workflow";
+  await forwardEvent(ctx);
 }
 
 export const stripeToSlack = pass({

package/dist/templates/pass.ts

@@ -85,36 +85,41 @@ export function pass(config: PassConfig): PassRegistration {
 // import { pass, type PassContext } from "./pass";
 // import { plug } from "../plugs/plug";
 //
-// async function pollinateToSlackWorkflow(ctx: PassContext) {
-//   "use workflow";
+// Declare "use step" functions at MODULE TOP LEVEL and pass them serializable
+// values only (`ctx` is plain data). Do NOT nest steps inside the "use workflow"
+// function — closures over workflow scope cannot be hoisted and fail at runtime.
 //
-//   async function forwardEvent() {
-//     "use step";
-//     const cache = khotanCache(ctx, "pollinate-forwarded-events");
-//     const eventId = String(ctx.event["id"] ?? "");
-//     if (eventId && (await cache.get<boolean>(eventId))) return;
+// // Step: top-level, full Node.js access, retried independently.
+// async function forwardEvent(ctx: PassContext) {
+//   "use step";
+//   const cache = khotanCache(ctx, "pollinate-forwarded-events");
+//   const eventId = String(ctx.event["id"] ?? "");
+//   if (eventId && (await cache.get<boolean>(eventId))) return;
 //
-//     // Construct destination plug from destVars
-//     const slackPlug = plug({
-//       name: "slack",
-//       baseUrl: "https://hooks.slack.com",
-//       authType: "bearer",
-//       auth: { bearer: { token: ctx.destVars["token"] ?? "" } },
-//     });
+//   // Construct destination plug from destVars
+//   const slackPlug = plug({
+//     name: "slack",
+//     baseUrl: "https://hooks.slack.com",
+//     authType: "bearer",
+//     auth: { bearer: { token: ctx.destVars["token"] ?? "" } },
+//   });
 //
-//     await slackPlug.post("/services/webhook", {
-//       body: {
-//         text: `Received ${ctx.eventType} event from pollinate`,
-//         event: ctx.event,
-//       },
-//     });
+//   await slackPlug.post("/services/webhook", {
+//     body: {
+//       text: `Received ${ctx.eventType} event from pollinate`,
+//       event: ctx.event,
+//     },
+//   });
 //
-//     if (eventId) {
-//       await cache.set(eventId, true);
-//     }
+//   if (eventId) {
+//     await cache.set(eventId, true);
 //   }
+// }
 //
-//   await forwardEvent();
+// // Workflow: orchestration only.
+// async function pollinateToSlackWorkflow(ctx: PassContext) {
+//   "use workflow";
+//   await forwardEvent(ctx);
 // }
 //
 // export const pollinateToSlack = pass({

package/dist/templates/plug-debugger.tsx

@@ -5,6 +5,7 @@ import { Badge } from "@/components/ui/badge";
 import { Button } from "@/components/ui/button";
 import { Input } from "@/components/ui/input";
 import { Label } from "@/components/ui/label";
+import { khotanFetch, isKhotanApiError, ApiErrorState } from "./api-state";
 
 // ============================================================================
 // Plug Debugger — Lightweight Postman for your plugs
@@ -331,6 +332,7 @@ export function PlugDebugger({
 }: PlugDebuggerProps) {
   const [meta, setMeta] = useState<PlugMeta | null>(null);
   const [loading, setLoading] = useState(true);
+  const [metaError, setMetaError] = useState<unknown>(null);
 
   const [method, setMethod] = useState<string>("GET");
   const [path, setPath] = useState("");
@@ -353,15 +355,17 @@ export function PlugDebugger({
   const containerRef = useRef<HTMLDivElement>(null);
 
   const fetchMeta = useCallback(async () => {
+    setLoading(true);
+    setMetaError(null);
     try {
-      const res = await fetch(`${basePath}/debug/${plugName}`);
-      if (!res.ok) {
-        setMeta(null);
-        return;
-      }
-      setMeta(await res.json());
-    } catch {
+      setMeta(await khotanFetch<PlugMeta>(`${basePath}/debug/${plugName}`));
+    } catch (err) {
       setMeta(null);
+      // A 404 means debug is off or the plug isn't registered — handled by the
+      // dedicated "not available" message below. Surface everything else.
+      if (!(isKhotanApiError(err) && err.status === 404)) {
+        setMetaError(err);
+      }
     } finally {
       setLoading(false);
     }
@@ -401,6 +405,10 @@ export function PlugDebugger({
     );
   }
 
+  if (metaError) {
+    return <ApiErrorState error={metaError} onRetry={() => void fetchMeta()} />;
+  }
+
   if (!meta) {
     return (
       <div className="rounded-lg border border-border p-6 text-center">

package/dist/templates/relay.example.ts

@@ -4,62 +4,70 @@
 //
 // Copy this file, rename it for your source/destination pair, and register the
 // exported flow in {outputDir}/khotan.ts.
+//
+// IMPORTANT — Workflow step structure:
+// Declare "use step" functions at module top level and pass them only
+// serializable values (the `ctx` object is plain data and is safe to pass).
+// Do NOT nest step functions inside the "use workflow" function — the Workflow
+// compiler cannot hoist closures that capture workflow scope, and they fail at
+// runtime in the sandbox. Keep the workflow body limited to orchestration.
 // ============================================================================
 
 import { khotanCache } from "khotan-data/factory";
 import { relay, type RelayContext } from "./relay";
 
-async function shopifyToHubspotWorkflow(ctx: RelayContext) {
-  "use workflow";
+// Step: full Node.js access, retried independently. Receives serializable ctx.
+async function forwardProducts(ctx: RelayContext) {
+  "use step";
+  console.log("Starting relay", {
+    flow: ctx.flow.name,
+    to: ctx.flow.to,
+    khotanRunId: ctx.khotanRunId,
+    runType: ctx.runType,
+  });
 
-  async function forwardProducts() {
-    "use step";
-    console.log("Starting relay", {
-      flow: ctx.flow.name,
-      to: ctx.flow.to,
-      khotanRunId: ctx.khotanRunId,
-      runType: ctx.runType,
-    });
+  const sourceResponse = await fetch("https://source.example.com/products", {
+    headers: {
+      Authorization: `Bearer ${ctx.vars["sourceToken"] ?? ""}`,
+    },
+  });
+  const payload = (await sourceResponse.json()) as {
+    data?: Array<Record<string, unknown>>;
+  };
+  const records = Array.isArray(payload.data) ? payload.data : [];
+  const snapshotCache = khotanCache(ctx, "shopify-products-snapshot");
+  const previousRecords =
+    (await snapshotCache.get<Array<Record<string, unknown>>>("latest")) ?? [];
+
+  await snapshotCache.set("latest", records, { ttl: "6h" });
 
-    const sourceResponse = await fetch("https://source.example.com/products", {
+  for (const record of records) {
+    await fetch("https://destination.example.com/products", {
+      method: "POST",
       headers: {
-        Authorization: `Bearer ${ctx.vars["sourceToken"] ?? ""}`,
+        Authorization: `Bearer ${ctx.vars["destinationToken"] ?? ""}`,
+        "Content-Type": "application/json",
       },
+      body: JSON.stringify(record),
     });
-    const payload = (await sourceResponse.json()) as {
-      data?: Array<Record<string, unknown>>;
-    };
-    const records = Array.isArray(payload.data) ? payload.data : [];
-    const snapshotCache = khotanCache(ctx, "shopify-products-snapshot");
-    const previousRecords =
-      (await snapshotCache.get<Array<Record<string, unknown>>>("latest")) ?? [];
-
-    await snapshotCache.set("latest", records, { ttl: "6h" });
-
-    for (const record of records) {
-      await fetch("https://destination.example.com/products", {
-        method: "POST",
-        headers: {
-          Authorization: `Bearer ${ctx.vars["destinationToken"] ?? ""}`,
-          "Content-Type": "application/json",
-        },
-        body: JSON.stringify(record),
-      });
-    }
-
-    return {
-      extracted: records.length,
-      transformed: records.length,
-      created: records.length,
-      metadata: {
-        relay: ctx.flow.name,
-        to: ctx.flow.to,
-        previousCount: previousRecords.length,
-      },
-    };
   }
 
-  return forwardProducts();
+  return {
+    extracted: records.length,
+    transformed: records.length,
+    created: records.length,
+    metadata: {
+      relay: ctx.flow.name,
+      to: ctx.flow.to,
+      previousCount: previousRecords.length,
+    },
+  };
+}
+
+// Workflow: orchestration only. Calls top-level steps with serializable args.
+async function shopifyToHubspotWorkflow(ctx: RelayContext) {
+  "use workflow";
+  return forwardProducts(ctx);
 }
 
 export const shopifyToHubspotRelay = relay({

package/dist/templates/relay.ts

@@ -51,48 +51,53 @@ export function relay(config: RelayConfig): FlowRegistration {
 // Usage Example (create a file like flows/shopify-to-hubspot.ts)
 // ---------------------------------------------------------------------------
 //
+// Declare "use step" functions at MODULE TOP LEVEL and pass them serializable
+// values only (`ctx` is plain data). Do NOT nest steps inside the "use workflow"
+// function — closures over workflow scope cannot be hoisted and fail at runtime.
+//
 // import { bindWorkflowPlug, khotanCache, relay, type RelayContext } from "khotan-data/factory";
 // import { shopifyPlug } from "../plugs/shopify";
 // import { hubspotPlug } from "../plugs/hubspot";
 //
-// async function shopifyToHubspotWorkflow(ctx: RelayContext) {
-//   "use workflow";
-//
-//   async function forwardProducts() {
-//     "use step";
-//     console.log("Starting relay", {
-//       flow: ctx.flow.name,
-//       to: ctx.flow.to,
-//       khotanRunId: ctx.khotanRunId,
-//       runType: ctx.runType,
-//     });
-//     const shopify = bindWorkflowPlug(shopifyPlug, ctx);
-//     const hubspot = bindWorkflowPlug(hubspotPlug, ctx, "hubspot");
-//
-//     const snapshotCache = khotanCache(ctx, "shopify-products-snapshot");
-//     const previous = await snapshotCache.get<Array<Record<string, unknown>>>("latest");
+// // Step: top-level, full Node.js access, retried independently.
+// async function forwardProducts(ctx: RelayContext) {
+//   "use step";
+//   console.log("Starting relay", {
+//     flow: ctx.flow.name,
+//     to: ctx.flow.to,
+//     khotanRunId: ctx.khotanRunId,
+//     runType: ctx.runType,
+//   });
+//   const shopify = bindWorkflowPlug(shopifyPlug, ctx);
+//   const hubspot = bindWorkflowPlug(hubspotPlug, ctx, "hubspot");
 //
-//     const response = await shopify.get<{ data?: Array<Record<string, unknown>> }>("/products");
-//     const records = Array.isArray(response.data) ? response.data : [];
-//     await snapshotCache.set("latest", records);
+//   const snapshotCache = khotanCache(ctx, "shopify-products-snapshot");
+//   const previous = await snapshotCache.get<Array<Record<string, unknown>>>("latest");
 //
-//     for (const record of records) {
-//       await hubspot.post("/products", { body: record });
-//     }
+//   const response = await shopify.get<{ data?: Array<Record<string, unknown>> }>("/products");
+//   const records = Array.isArray(response.data) ? response.data : [];
+//   await snapshotCache.set("latest", records);
 //
-//     return {
-//       extracted: records.length,
-//       transformed: records.length,
-//       created: records.length,
-//       metadata: {
-//         relay: ctx.flow.name,
-//         to: ctx.flow.to,
-//         previousCount: previous?.length ?? 0,
-//       },
-//     };
+//   for (const record of records) {
+//     await hubspot.post("/products", { body: record });
 //   }
 //
-//   return forwardProducts();
+//   return {
+//     extracted: records.length,
+//     transformed: records.length,
+//     created: records.length,
+//     metadata: {
+//       relay: ctx.flow.name,
+//       to: ctx.flow.to,
+//       previousCount: previous?.length ?? 0,
+//     },
+//   };
+// }
+//
+// // Workflow: orchestration only.
+// async function shopifyToHubspotWorkflow(ctx: RelayContext) {
+//   "use workflow";
+//   return forwardProducts(ctx);
 // }
 //
 // export const shopifyToHubspotRelay = relay({