khotan-data 0.1.1 → 0.3.0

package/dist/templates/hub.tsx

@@ -15,6 +15,7 @@ import { Switch } from "@/components/ui/switch";
 import { Button } from "@/components/ui/button";
 import { WirePanel } from "./wire";
 import { VarPanel } from "./var-panel";
+import { khotanFetch, ApiErrorState } from "./api-state";
 
 // ============================================================================
 // Khotan Hub — Dashboard for configured plugs and flows
@@ -45,7 +46,14 @@ interface Flow {
   enabled: boolean;
   schedule: string | null;
   lastRunAt: string | null;
-  lastRunStatus: "completed" | "partial" | "failed" | "cancelled" | null;
+  lastRunStatus:
+    | "pending"
+    | "running"
+    | "completed"
+    | "partial"
+    | "failed"
+    | "cancelled"
+    | null;
   plugName: string | null;
 }
 
@@ -98,19 +106,25 @@ const runStatusVariant: Record<string, StatusVariant> = {
 export function KhotanHub({
   webhookUrl,
   debugHref,
-  logsHref = "/logs",
+  logsHref,
 }: {
   webhookUrl?: string;
   debugHref?: (plugName: string) => string;
+  /** When set, shows an "Open Logs" button linking here. Hidden if omitted. */
   logsHref?: string;
 } = {}) {
   const [plugs, setPlugs] = useState<Plug[]>([]);
   const [flows, setFlows] = useState<Flow[]>([]);
   const [webhookHandlers, setWebhookHandlers] = useState<WebhookHandler[]>([]);
   const [loading, setLoading] = useState(true);
-  const [error, setError] = useState<string | null>(null);
+  const [error, setError] = useState<unknown>(null);
   const [selectedPlugId, setSelectedPlugId] = useState<string | null>(null);
   const [debugEnabled, setDebugEnabled] = useState(false);
+  const [runningFlowId, setRunningFlowId] = useState<string | null>(null);
+  const [runNotice, setRunNotice] = useState<{
+    type: "ok" | "error";
+    text: string;
+  } | null>(null);
 
   useEffect(() => {
     fetch("/api/khotan/debug")
@@ -122,17 +136,14 @@ export function KhotanHub({
     setLoading(true);
     setError(null);
     try {
-      const [plugsRes, flowsRes] = await Promise.all([
-        fetch("/api/khotan/plugs"),
-        fetch("/api/khotan/flows"),
+      const [plugsData, flowsData] = await Promise.all([
+        khotanFetch<Plug[]>("/api/khotan/plugs"),
+        khotanFetch<Flow[]>("/api/khotan/flows"),
       ]);
-      if (!plugsRes.ok || !flowsRes.ok) {
-        throw new Error("Failed to fetch khotan data");
-      }
-      setPlugs(await plugsRes.json());
-      setFlows(await flowsRes.json());
+      setPlugs(plugsData);
+      setFlows(flowsData);
     } catch (err) {
-      setError(err instanceof Error ? err.message : "Unknown error");
+      setError(err);
     } finally {
       setLoading(false);
     }
@@ -149,6 +160,35 @@ export function KhotanHub({
     });
   }
 
+  async function runFlow(flowId: string, flowName: string) {
+    setRunningFlowId(flowId);
+    setRunNotice(null);
+    try {
+      const res = await fetch(`/api/khotan/flows/${flowId}/runs`, {
+        method: "POST",
+        headers: { "Content-Type": "application/json" },
+        body: JSON.stringify({ runType: "full" }),
+      });
+      if (!res.ok) {
+        const data = (await res.json().catch(() => ({}))) as {
+          error?: string;
+          hint?: string;
+        };
+        setRunNotice({
+          type: "error",
+          text: data.hint ?? data.error ?? `Failed to trigger ${flowName}.`,
+        });
+        return;
+      }
+      setRunNotice({ type: "ok", text: `Triggered ${flowName}.` });
+      await fetchData();
+    } catch {
+      setRunNotice({ type: "error", text: `Failed to trigger ${flowName}.` });
+    } finally {
+      setRunningFlowId(null);
+    }
+  }
+
   async function toggleWebhookHandler(handlerId: string, enabled: boolean) {
     setWebhookHandlers((prev) =>
       prev.map((h) => (h.id === handlerId ? { ...h, enabled } : h)),
@@ -201,19 +241,7 @@ export function KhotanHub({
   }
 
   if (error) {
-    return (
-      <Card>
-        <CardContent className="py-8 text-center">
-          <p className="text-destructive mb-4">{error}</p>
-          <button
-            onClick={fetchData}
-            className="text-sm underline hover:no-underline"
-          >
-            Retry
-          </button>
-        </CardContent>
-      </Card>
-    );
+    return <ApiErrorState error={error} onRetry={fetchData} />;
   }
 
   if (plugs.length === 0) {
@@ -241,23 +269,27 @@ export function KhotanHub({
     <div className="space-y-6">
       <div className="flex items-center justify-between gap-4">
         <h2 className="text-2xl font-bold tracking-tight">Khotan Hub</h2>
-        <Button
-          variant="outline"
-          size="sm"
-          onClick={() => {
-            window.location.href = logsHref;
-          }}
-        >
-          Open Logs
-        </Button>
+        {logsHref && (
+          <Button
+            variant="outline"
+            size="sm"
+            onClick={() => {
+              window.location.href = logsHref;
+            }}
+          >
+            Open Logs
+          </Button>
+        )}
       </div>
 
       <div className="grid gap-4 md:grid-cols-2 lg:grid-cols-3">
         {plugs.map((plug) => (
           <Card
             key={plug.id}
-            className={`cursor-pointer transition-colors hover:border-primary ${
-              selectedPlugId === plug.id ? "border-primary" : ""
+            className={`cursor-pointer transition-all hover:border-primary ${
+              selectedPlugId === plug.id
+                ? "border-primary ring-2 ring-primary ring-offset-2 ring-offset-background shadow-md"
+                : "border-border"
             }`}
             onClick={() =>
               setSelectedPlugId(selectedPlugId === plug.id ? null : plug.id)
@@ -301,6 +333,18 @@ export function KhotanHub({
         ))}
       </div>
 
+      {!selectedPlug && (
+        <Card className="border-dashed">
+          <CardContent className="py-10 text-center">
+            <p className="text-muted-foreground mb-1">No plug selected yet.</p>
+            <p className="text-sm text-muted-foreground">
+              Select a plug above to view its flows, webhook handlers, and
+              credentials.
+            </p>
+          </CardContent>
+        </Card>
+      )}
+
       {selectedPlug && (
         <div className="space-y-4">
           <Card>
@@ -308,6 +352,17 @@ export function KhotanHub({
               <CardTitle>{selectedPlug.name} — Flows</CardTitle>
             </CardHeader>
             <CardContent>
+              {runNotice && (
+                <p
+                  className={`mb-3 text-sm ${
+                    runNotice.type === "error"
+                      ? "text-destructive"
+                      : "text-muted-foreground"
+                  }`}
+                >
+                  {runNotice.text}
+                </p>
+              )}
               {plugFlows.length === 0 ? (
                 <p className="text-sm text-muted-foreground">
                   No flows configured for this plug.
@@ -321,6 +376,7 @@ export function KhotanHub({
                       <TableHead>Schedule</TableHead>
                       <TableHead>Last Run</TableHead>
                       <TableHead>Enabled</TableHead>
+                      <TableHead className="text-right">Actions</TableHead>
                     </TableRow>
                   </TableHeader>
                   <TableBody>
@@ -359,6 +415,19 @@ export function KhotanHub({
                             onCheckedChange={(v) => toggleFlow(flow.id, v)}
                           />
                         </TableCell>
+                        <TableCell className="text-right">
+                          <Button
+                            size="sm"
+                            variant="outline"
+                            className="h-7 px-2 text-xs"
+                            disabled={runningFlowId !== null}
+                            onClick={() => runFlow(flow.id, flow.name)}
+                          >
+                            {runningFlowId === flow.id
+                              ? "Running…"
+                              : "Run now"}
+                          </Button>
+                        </TableCell>
                       </TableRow>
                     ))}
                   </TableBody>

package/dist/templates/inflow.example.ts

@@ -4,53 +4,61 @@
 //
 // Copy this file, rename it for your source service/resource, and register the
 // exported flow in {outputDir}/khotan.ts.
+//
+// IMPORTANT — Workflow step structure:
+// Declare "use step" functions at module top level and pass them only
+// serializable values (the `ctx` object is plain data and is safe to pass).
+// Do NOT nest step functions inside the "use workflow" function — the Workflow
+// compiler cannot hoist closures that capture workflow scope, and they fail at
+// runtime in the sandbox. Keep the workflow body limited to orchestration.
 // ============================================================================
 
 import { inflow, type InflowContext } from "./inflow";
 import { sendUpdate } from "khotan-data/factory";
 
-async function shopifyProductsWorkflow(ctx: InflowContext) {
-  "use workflow";
-
-  async function extractAndLoad() {
-    "use step";
-    console.log("Starting inflow", {
-      flow: ctx.flow.name,
-      khotanRunId: ctx.khotanRunId,
-      runType: ctx.runType,
-    });
-    await sendUpdate({
-      message: "Starting product inflow",
-      metadata: { flow: ctx.flow.name, runType: ctx.runType },
-    });
+// Step: full Node.js access, retried independently. Receives serializable ctx.
+async function extractAndLoad(ctx: InflowContext) {
+  "use step";
+  console.log("Starting inflow", {
+    flow: ctx.flow.name,
+    khotanRunId: ctx.khotanRunId,
+    runType: ctx.runType,
+  });
+  await sendUpdate({
+    message: "Starting product inflow",
+    metadata: { flow: ctx.flow.name, runType: ctx.runType },
+  });
 
-    const response = await fetch("https://api.example.com/products", {
-      headers: {
-        Authorization: `Bearer ${ctx.vars["apiToken"] ?? ""}`,
-      },
-    });
-    const payload = (await response.json()) as {
-      data?: Array<Record<string, unknown>>;
-    };
-    const records = Array.isArray(payload.data) ? payload.data : [];
+  const response = await fetch("https://api.example.com/products", {
+    headers: {
+      Authorization: `Bearer ${ctx.vars["apiToken"] ?? ""}`,
+    },
+  });
+  const payload = (await response.json()) as {
+    data?: Array<Record<string, unknown>>;
+  };
+  const records = Array.isArray(payload.data) ? payload.data : [];
 
-    // Replace this with your app-specific transform and DB upsert.
-    console.log("Fetched records", records.length);
-    await sendUpdate({
-      message: `Fetched ${String(records.length)} products`,
-      extracted: records.length,
-      progress: 50,
-    });
+  // Replace this with your app-specific transform and DB upsert.
+  console.log("Fetched records", records.length);
+  await sendUpdate({
+    message: `Fetched ${String(records.length)} products`,
+    extracted: records.length,
+    progress: 50,
+  });
 
-    return {
-      extracted: records.length,
-      transformed: records.length,
-      created: records.length,
-      metadata: { source: ctx.flow.name },
-    };
-  }
+  return {
+    extracted: records.length,
+    transformed: records.length,
+    created: records.length,
+    metadata: { source: ctx.flow.name },
+  };
+}
 
-  return extractAndLoad();
+// Workflow: orchestration only. Calls top-level steps with serializable args.
+async function shopifyProductsWorkflow(ctx: InflowContext) {
+  "use workflow";
+  return extractAndLoad(ctx);
 }
 
 export const shopifyProductsInflow = inflow({

package/dist/templates/inflow.ts

@@ -48,46 +48,52 @@ export function inflow(config: InflowConfig): FlowRegistration {
 // Usage Example (create a file like flows/shopify-products.ts)
 // ---------------------------------------------------------------------------
 //
+// Declare "use step" functions at MODULE TOP LEVEL and pass them serializable
+// values only. The `ctx` object is plain data and is safe to pass. Do NOT nest
+// step functions inside the "use workflow" function — the Workflow compiler
+// cannot hoist closures that capture workflow scope, so they fail at runtime.
+//
 // import { bindWorkflowPlug, inflow, type InflowContext, sendUpdate } from "khotan-data/factory";
 // import { db } from "@/db";
 // import { products } from "@/db/schema";
 // import { shopifyPlug } from "../plugs/shopify";
 //
-// async function shopifyProductsWorkflow(ctx: InflowContext) {
-//   "use workflow";
-//
-//   async function extractAndLoad() {
-//     "use step";
-//     console.log("Starting inflow", {
-//       flow: ctx.flow.name,
-//       khotanRunId: ctx.khotanRunId,
-//       runType: ctx.runType,
-//     });
-//     await sendUpdate({ message: "Starting Shopify products inflow" });
-//     const shopify = bindWorkflowPlug(shopifyPlug, ctx);
-//
-//     const response = await shopify.get<{ data?: Array<{ id: string; sku?: string }> }>("/products");
-//     const records = Array.isArray(response.data) ? response.data : [];
-//     await sendUpdate({ message: `Fetched ${records.length} products`, extracted: records.length });
+// // Step: top-level, full Node.js access, retried independently.
+// async function extractAndLoad(ctx: InflowContext) {
+//   "use step";
+//   console.log("Starting inflow", {
+//     flow: ctx.flow.name,
+//     khotanRunId: ctx.khotanRunId,
+//     runType: ctx.runType,
+//   });
+//   await sendUpdate({ message: "Starting Shopify products inflow" });
+//   const shopify = bindWorkflowPlug(shopifyPlug, ctx);
 //
-//     if (records.length > 0) {
-//       await db.insert(products).values(
-//         records.map((record) => ({
-//           externalId: record.id,
-//           sku: record.sku ?? record.id,
-//         })),
-//       );
-//     }
+//   const response = await shopify.get<{ data?: Array<{ id: string; sku?: string }> }>("/products");
+//   const records = Array.isArray(response.data) ? response.data : [];
+//   await sendUpdate({ message: `Fetched ${records.length} products`, extracted: records.length });
 //
-//     return {
-//       extracted: records.length,
-//       transformed: records.length,
-//       created: records.length,
-//       metadata: { source: ctx.flow.name },
-//     };
+//   if (records.length > 0) {
+//     await db.insert(products).values(
+//       records.map((record) => ({
+//         externalId: record.id,
+//         sku: record.sku ?? record.id,
+//       })),
+//     );
 //   }
 //
-//   return extractAndLoad();
+//   return {
+//     extracted: records.length,
+//     transformed: records.length,
+//     created: records.length,
+//     metadata: { source: ctx.flow.name },
+//   };
+// }
+//
+// // Workflow: orchestration only.
+// async function shopifyProductsWorkflow(ctx: InflowContext) {
+//   "use workflow";
+//   return extractAndLoad(ctx);
 // }
 //
 // export const shopifyProductsInflow = inflow({

package/dist/templates/khotan-config.ts

@@ -13,6 +13,23 @@ import { db } from "@/db";
 const khotanData = khotan({
   adapter: drizzleAdapter(db),
 
+  // ── Security ──────────────────────────────────────────────────────────────
+  // The management API (/api/khotan/*) and the Hub dashboard expose plug
+  // credentials and operational controls. Gate every management route behind
+  // your auth layer with `authorize`. It receives the raw Request, so it works
+  // directly with session libraries like better-auth:
+  //
+  //   import { auth } from "@/lib/auth";
+  //
+  //   authorize: async (request) => {
+  //     const session = await auth.api.getSession({ headers: request.headers });
+  //     return Boolean(session?.user); // or: session?.user?.role === "admin"
+  //   },
+  //
+  // Inbound webhooks, the cron dispatcher (CRON_SECRET), and debug routes are
+  // exempt automatically. Without `authorize`, the API is PUBLIC.
+  // authorize: async (request) => { /* return true to allow */ return false; },
+
   // Resources define logical entity types for cross-referencing across plugs.
   // The mapping block declares the shared identity contract for that resource.
   resources: [
@@ -47,3 +64,14 @@ const khotanData = khotan({
 });
 
 export default khotanData;
+
+// Trigger a flow run from server code (route handler, action, cron):
+//
+//   import khotanData from "@/lib/khotan/khotan";
+//   await khotanData.flow("products-inflow", { plugName: "stripe" }).start({
+//     runType: "delta", // or "full"
+//   });
+//
+// `flow(name).start(options)` is the single entry point — there is no
+// `khotanData.api.*` surface. `plugName` only disambiguates when the same flow
+// name is registered under multiple plugs.

package/dist/templates/mapping-browser.tsx

@@ -1,6 +1,7 @@
 "use client";
 
 import { useEffect, useMemo, useState } from "react";
+import { khotanFetch, ApiErrorState } from "./api-state";
 import { Button } from "@/components/ui/button";
 import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card";
 import { Input } from "@/components/ui/input";
@@ -60,7 +61,9 @@ function readErrorMessage(error: unknown): string {
   return "Unknown error";
 }
 
-function toPrettyJson(value: Record<string, unknown> | null | undefined): string {
+function toPrettyJson(
+  value: Record<string, unknown> | null | undefined,
+): string {
   return value ? JSON.stringify(value, null, 2) : "";
 }
 
@@ -92,7 +95,10 @@ function parseConnectValueInput(
   }
 
   const parsed = JSON.parse(trimmed) as unknown;
-  if (!Array.isArray(parsed) || parsed.some((value) => typeof value !== "string")) {
+  if (
+    !Array.isArray(parsed) ||
+    parsed.some((value) => typeof value !== "string")
+  ) {
     throw new Error(
       "Composite connect values must be provided as a JSON string array in declared field order.",
     );
@@ -119,7 +125,7 @@ export function KhotanMappingBrowser({
   const [mappingsLoading, setMappingsLoading] = useState(false);
   const [search, setSearch] = useState("");
   const [offset, setOffset] = useState(0);
-  const [error, setError] = useState<string | null>(null);
+  const [error, setError] = useState<unknown>(null);
   const [actionError, setActionError] = useState<string | null>(null);
   const [submitting, setSubmitting] = useState(false);
   const [formMode, setFormMode] = useState<FormMode | null>(null);
@@ -142,11 +148,7 @@ export function KhotanMappingBrowser({
     setResourcesLoading(true);
     setError(null);
     try {
-      const res = await fetch("/api/khotan/resources");
-      if (!res.ok) {
-        throw new Error("Failed to fetch resources from /api/khotan/resources");
-      }
-      const data = (await res.json()) as ResourceRecord[];
+      const data = await khotanFetch<ResourceRecord[]>("/api/khotan/resources");
       setResources(data);
 
       setSelectedResourceId((current) => {
@@ -156,13 +158,17 @@ export function KhotanMappingBrowser({
         return current || data[0]!.id;
       });
     } catch (error) {
-      setError(readErrorMessage(error));
+      setError(error);
     } finally {
       setResourcesLoading(false);
     }
   }
 
-  async function fetchMappings(resourceId: string, nextOffset: number, term: string) {
+  async function fetchMappings(
+    resourceId: string,
+    nextOffset: number,
+    term: string,
+  ) {
     setMappingsLoading(true);
     setError(null);
     try {
@@ -175,15 +181,11 @@ export function KhotanMappingBrowser({
       if (term.trim()) {
         url.searchParams.set("search", term.trim());
       }
-      const res = await fetch(url.toString());
-      if (!res.ok) {
-        throw new Error("Failed to fetch mappings for the selected resource");
-      }
-      const data = (await res.json()) as MappingPage;
+      const data = await khotanFetch<MappingPage>(url.toString());
       setMappings(data.items);
       setPage(data.page);
     } catch (error) {
-      setError(readErrorMessage(error));
+      setError(error);
       setMappings([]);
       setPage(null);
     } finally {
@@ -238,7 +240,10 @@ export function KhotanMappingBrowser({
 
     if (declaredPlugNames.length > 0) {
       const nextDeclaredRefs = Object.fromEntries(
-        declaredPlugNames.map((plugName) => [plugName, mapping.refs[plugName] ?? ""]),
+        declaredPlugNames.map((plugName) => [
+          plugName,
+          mapping.refs[plugName] ?? "",
+        ]),
       );
       setDeclaredRefs(nextDeclaredRefs);
       setDynamicRefs([]);
@@ -355,7 +360,9 @@ export function KhotanMappingBrowser({
       }
 
       const nextOffset =
-        mappings.length === 1 && offset > 0 ? Math.max(offset - pageSize, 0) : offset;
+        mappings.length === 1 && offset > 0
+          ? Math.max(offset - pageSize, 0)
+          : offset;
       setOffset(nextOffset);
       await fetchMappings(mapping.resourceId, nextOffset, search);
     } catch (error) {
@@ -460,33 +467,30 @@ export function KhotanMappingBrowser({
           </div>
 
           {resourcesLoading ? (
-            <div className="text-muted-foreground text-sm">Loading resources...</div>
+            <div className="text-muted-foreground text-sm">
+              Loading resources...
+            </div>
           ) : null}
 
           {!resourcesLoading && resources.length === 0 ? (
             <div className="text-muted-foreground text-sm">
-              No resources are registered yet. Mappings require registered resources
-              in your `khotan()` config.
+              No resources are registered yet. Mappings require registered
+              resources in your `khotan()` config.
             </div>
           ) : null}
 
           {error ? (
-            <div className="space-y-2 rounded-md border border-destructive/30 bg-destructive/5 p-4">
-              <p className="text-sm text-destructive">{error}</p>
-              <Button
-                variant="outline"
-                size="sm"
-                onClick={() => {
-                  if (selectedResourceId) {
-                    void fetchMappings(selectedResourceId, offset, search);
-                  } else {
-                    void fetchResources();
-                  }
-                }}
-              >
-                Retry
-              </Button>
-            </div>
+            <ApiErrorState
+              error={error}
+              onRetry={() => {
+                if (selectedResourceId) {
+                  void fetchMappings(selectedResourceId, offset, search);
+                } else {
+                  void fetchResources();
+                }
+              }}
+              compact
+            />
           ) : null}
         </CardContent>
       </Card>
@@ -537,8 +541,8 @@ export function KhotanMappingBrowser({
                         id={`ref-${plugName}`}
                         value={declaredRefs[plugName] ?? ""}
                         placeholder={
-                          selectedResource?.mapping.plugs?.[plugName]?.uniqueIdentifier ??
-                          "External ID"
+                          selectedResource?.mapping.plugs?.[plugName]
+                            ?.uniqueIdentifier ?? "External ID"
                         }
                         onChange={(event) =>
                           setDeclaredRefs((current) => ({
@@ -590,7 +594,9 @@ export function KhotanMappingBrowser({
                           setDynamicRefs((current) =>
                             current.length === 1
                               ? [{ plugName: "", ref: "" }]
-                              : current.filter((_, itemIndex) => itemIndex !== index),
+                              : current.filter(
+                                  (_, itemIndex) => itemIndex !== index,
+                                ),
                           )
                         }
                       >
@@ -643,7 +649,11 @@ export function KhotanMappingBrowser({
                     ? "Create Mapping"
                     : "Save Changes"}
               </Button>
-              <Button variant="outline" onClick={resetForm} disabled={submitting}>
+              <Button
+                variant="outline"
+                onClick={resetForm}
+                disabled={submitting}
+              >
                 Cancel
               </Button>
             </div>
@@ -662,7 +672,9 @@ export function KhotanMappingBrowser({
         </CardHeader>
         <CardContent>
           {mappingsLoading ? (
-            <div className="text-muted-foreground text-sm">Loading mappings...</div>
+            <div className="text-muted-foreground text-sm">
+              Loading mappings...
+            </div>
           ) : null}
 
           {!mappingsLoading &&
@@ -729,8 +741,8 @@ export function KhotanMappingBrowser({
               {page ? (
                 <div className="flex flex-wrap items-center justify-between gap-3">
                   <p className="text-muted-foreground text-sm">
-                    Showing {page.offset + 1}-
-                    {page.offset + mappings.length} of {page.total}
+                    Showing {page.offset + 1}-{page.offset + mappings.length} of{" "}
+                    {page.total}
                   </p>
                   <div className="flex gap-2">
                     <Button