khotan-data 0.0.1 → 0.1.1

package/dist/templates/hub.tsx

@@ -0,0 +1,450 @@
+"use client";
+
+import { useEffect, useState } from "react";
+import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card";
+import { Badge } from "@/components/ui/badge";
+import {
+  Table,
+  TableBody,
+  TableCell,
+  TableHead,
+  TableHeader,
+  TableRow,
+} from "@/components/ui/table";
+import { Switch } from "@/components/ui/switch";
+import { Button } from "@/components/ui/button";
+import { WirePanel } from "./wire";
+import { VarPanel } from "./var-panel";
+
+// ============================================================================
+// Khotan Hub — Dashboard for configured plugs and flows
+// Generated by khotan CLI · https://github.com/khotan-data
+//
+// This file is yours. Restyle, extend, or embed it however you like.
+// It uses shadcn/ui primitives and fetches from /api/khotan.
+// ============================================================================
+
+interface Plug {
+  id: string;
+  name: string;
+  baseUrl: string;
+  authType: string;
+  enabled: boolean;
+  status: "connected" | "error" | "idle";
+  statusMessage: string | null;
+  flowCount: number;
+  createdAt: string;
+  updatedAt: string;
+}
+
+interface Flow {
+  id: string;
+  plugId: string;
+  name: string;
+  type: "inflow" | "outflow" | "relay" | "webhook";
+  enabled: boolean;
+  schedule: string | null;
+  lastRunAt: string | null;
+  lastRunStatus: "completed" | "partial" | "failed" | "cancelled" | null;
+  plugName: string | null;
+}
+
+interface WebhookHandler {
+  id: string;
+  wireId: string;
+  name: string;
+  type: "catch" | "pass";
+  destinationPlugId: string | null;
+  events: string[] | null;
+  enabled: boolean;
+  lastRunAt: string | null;
+  lastRunStatus:
+    | "pending"
+    | "running"
+    | "completed"
+    | "partial"
+    | "failed"
+    | "cancelled"
+    | null;
+  createdAt: string;
+  updatedAt: string;
+}
+
+type StatusVariant = "default" | "secondary" | "destructive" | "outline";
+type FlowTypeVariant = "default" | "secondary" | "outline";
+
+const statusVariant: Record<string, StatusVariant> = {
+  connected: "default",
+  idle: "secondary",
+  error: "destructive",
+};
+
+const flowTypeVariant: Record<string, FlowTypeVariant> = {
+  inflow: "default",
+  outflow: "secondary",
+  relay: "outline",
+  webhook: "outline",
+};
+
+const runStatusVariant: Record<string, StatusVariant> = {
+  pending: "outline",
+  running: "secondary",
+  completed: "default",
+  partial: "secondary",
+  failed: "destructive",
+  cancelled: "outline",
+};
+
+export function KhotanHub({
+  webhookUrl,
+  debugHref,
+  logsHref = "/logs",
+}: {
+  webhookUrl?: string;
+  debugHref?: (plugName: string) => string;
+  logsHref?: string;
+} = {}) {
+  const [plugs, setPlugs] = useState<Plug[]>([]);
+  const [flows, setFlows] = useState<Flow[]>([]);
+  const [webhookHandlers, setWebhookHandlers] = useState<WebhookHandler[]>([]);
+  const [loading, setLoading] = useState(true);
+  const [error, setError] = useState<string | null>(null);
+  const [selectedPlugId, setSelectedPlugId] = useState<string | null>(null);
+  const [debugEnabled, setDebugEnabled] = useState(false);
+
+  useEffect(() => {
+    fetch("/api/khotan/debug")
+      .then((res) => setDebugEnabled(res.ok))
+      .catch(() => setDebugEnabled(false));
+  }, []);
+
+  async function fetchData() {
+    setLoading(true);
+    setError(null);
+    try {
+      const [plugsRes, flowsRes] = await Promise.all([
+        fetch("/api/khotan/plugs"),
+        fetch("/api/khotan/flows"),
+      ]);
+      if (!plugsRes.ok || !flowsRes.ok) {
+        throw new Error("Failed to fetch khotan data");
+      }
+      setPlugs(await plugsRes.json());
+      setFlows(await flowsRes.json());
+    } catch (err) {
+      setError(err instanceof Error ? err.message : "Unknown error");
+    } finally {
+      setLoading(false);
+    }
+  }
+
+  async function toggleFlow(flowId: string, enabled: boolean) {
+    setFlows((prev) =>
+      prev.map((flow) => (flow.id === flowId ? { ...flow, enabled } : flow)),
+    );
+    await fetch(`/api/khotan/flows/${flowId}`, {
+      method: "PATCH",
+      headers: { "Content-Type": "application/json" },
+      body: JSON.stringify({ enabled }),
+    });
+  }
+
+  async function toggleWebhookHandler(handlerId: string, enabled: boolean) {
+    setWebhookHandlers((prev) =>
+      prev.map((h) => (h.id === handlerId ? { ...h, enabled } : h)),
+    );
+    await fetch(`/api/khotan/webhook-handlers/${handlerId}`, {
+      method: "PATCH",
+      headers: { "Content-Type": "application/json" },
+      body: JSON.stringify({ enabled }),
+    });
+  }
+
+  useEffect(() => {
+    fetchData();
+  }, []);
+
+  useEffect(() => {
+    if (!selectedPlugId) {
+      setWebhookHandlers([]);
+      return;
+    }
+    const plug = plugs.find((p) => p.id === selectedPlugId);
+    if (!plug) return;
+    fetch(`/api/khotan/webhook-handlers/${plug.name}`)
+      .then((res) => (res.ok ? res.json() : []))
+      .then((data) => setWebhookHandlers(data))
+      .catch(() => setWebhookHandlers([]));
+  }, [selectedPlugId, plugs]);
+
+  if (loading) {
+    return (
+      <div className="space-y-4">
+        <div className="h-8 w-48 animate-pulse rounded bg-muted" />
+        <div className="grid gap-4 md:grid-cols-2 lg:grid-cols-3">
+          {[1, 2, 3].map((i) => (
+            <Card key={i}>
+              <CardHeader>
+                <div className="h-5 w-32 animate-pulse rounded bg-muted" />
+              </CardHeader>
+              <CardContent>
+                <div className="space-y-2">
+                  <div className="h-4 w-full animate-pulse rounded bg-muted" />
+                  <div className="h-4 w-2/3 animate-pulse rounded bg-muted" />
+                </div>
+              </CardContent>
+            </Card>
+          ))}
+        </div>
+      </div>
+    );
+  }
+
+  if (error) {
+    return (
+      <Card>
+        <CardContent className="py-8 text-center">
+          <p className="text-destructive mb-4">{error}</p>
+          <button
+            onClick={fetchData}
+            className="text-sm underline hover:no-underline"
+          >
+            Retry
+          </button>
+        </CardContent>
+      </Card>
+    );
+  }
+
+  if (plugs.length === 0) {
+    return (
+      <Card>
+        <CardContent className="py-8 text-center">
+          <p className="text-muted-foreground mb-2">No plugs configured yet.</p>
+          <p className="text-sm text-muted-foreground">
+            Register plugs in your <code>khotan.ts</code> config file, then
+            restart the dev server.
+          </p>
+        </CardContent>
+      </Card>
+    );
+  }
+
+  const selectedPlug = selectedPlugId
+    ? plugs.find((p) => p.id === selectedPlugId)
+    : null;
+  const plugFlows = selectedPlugId
+    ? flows.filter((flow) => flow.plugId === selectedPlugId)
+    : [];
+
+  return (
+    <div className="space-y-6">
+      <div className="flex items-center justify-between gap-4">
+        <h2 className="text-2xl font-bold tracking-tight">Khotan Hub</h2>
+        <Button
+          variant="outline"
+          size="sm"
+          onClick={() => {
+            window.location.href = logsHref;
+          }}
+        >
+          Open Logs
+        </Button>
+      </div>
+
+      <div className="grid gap-4 md:grid-cols-2 lg:grid-cols-3">
+        {plugs.map((plug) => (
+          <Card
+            key={plug.id}
+            className={`cursor-pointer transition-colors hover:border-primary ${
+              selectedPlugId === plug.id ? "border-primary" : ""
+            }`}
+            onClick={() =>
+              setSelectedPlugId(selectedPlugId === plug.id ? null : plug.id)
+            }
+          >
+            <CardHeader className="flex flex-row items-center justify-between space-y-0 pb-2">
+              <CardTitle className="text-sm font-medium">{plug.name}</CardTitle>
+              <Badge variant={statusVariant[plug.status] ?? "secondary"}>
+                {plug.status}
+              </Badge>
+            </CardHeader>
+            <CardContent>
+              <p className="text-xs text-muted-foreground truncate">
+                {plug.baseUrl}
+              </p>
+              <div className="mt-2 flex items-center justify-between">
+                <p className="text-xs text-muted-foreground">
+                  {plug.authType} · {plug.flowCount} flow
+                  {plug.flowCount !== 1 ? "s" : ""}
+                </p>
+                {debugEnabled && (
+                  <Button
+                    size="sm"
+                    variant="ghost"
+                    className="h-6 px-2 text-[11px]"
+                    onClick={(e) => {
+                      e.stopPropagation();
+                      if (debugHref) {
+                        window.location.href = debugHref(plug.name);
+                      } else {
+                        window.location.href = `/debug/${plug.name}`;
+                      }
+                    }}
+                  >
+                    Debug
+                  </Button>
+                )}
+              </div>
+            </CardContent>
+          </Card>
+        ))}
+      </div>
+
+      {selectedPlug && (
+        <div className="space-y-4">
+          <Card>
+            <CardHeader>
+              <CardTitle>{selectedPlug.name} — Flows</CardTitle>
+            </CardHeader>
+            <CardContent>
+              {plugFlows.length === 0 ? (
+                <p className="text-sm text-muted-foreground">
+                  No flows configured for this plug.
+                </p>
+              ) : (
+                <Table>
+                  <TableHeader>
+                    <TableRow>
+                      <TableHead>Name</TableHead>
+                      <TableHead>Type</TableHead>
+                      <TableHead>Schedule</TableHead>
+                      <TableHead>Last Run</TableHead>
+                      <TableHead>Enabled</TableHead>
+                    </TableRow>
+                  </TableHeader>
+                  <TableBody>
+                    {plugFlows.map((flow) => (
+                      <TableRow key={flow.id}>
+                        <TableCell className="font-medium">
+                          {flow.name}
+                        </TableCell>
+                        <TableCell>
+                          <Badge
+                            variant={flowTypeVariant[flow.type] ?? "outline"}
+                          >
+                            {flow.type}
+                          </Badge>
+                        </TableCell>
+                        <TableCell className="text-muted-foreground">
+                          {flow.schedule ?? "—"}
+                        </TableCell>
+                        <TableCell>
+                          {flow.lastRunStatus ? (
+                            <Badge
+                              variant={
+                                runStatusVariant[flow.lastRunStatus] ??
+                                "secondary"
+                              }
+                            >
+                              {flow.lastRunStatus}
+                            </Badge>
+                          ) : (
+                            <span className="text-muted-foreground">never</span>
+                          )}
+                        </TableCell>
+                        <TableCell>
+                          <Switch
+                            checked={flow.enabled}
+                            onCheckedChange={(v) => toggleFlow(flow.id, v)}
+                          />
+                        </TableCell>
+                      </TableRow>
+                    ))}
+                  </TableBody>
+                </Table>
+              )}
+            </CardContent>
+          </Card>
+
+          {webhookHandlers.length > 0 && (
+            <Card>
+              <CardHeader>
+                <CardTitle>{selectedPlug.name} — Webhook Handlers</CardTitle>
+              </CardHeader>
+              <CardContent>
+                <Table>
+                  <TableHeader>
+                    <TableRow>
+                      <TableHead>Name</TableHead>
+                      <TableHead>Type</TableHead>
+                      <TableHead>Events</TableHead>
+                      <TableHead>Destination</TableHead>
+                      <TableHead>Last Run</TableHead>
+                      <TableHead>Enabled</TableHead>
+                    </TableRow>
+                  </TableHeader>
+                  <TableBody>
+                    {webhookHandlers.map((handler) => (
+                      <TableRow key={handler.id}>
+                        <TableCell className="font-medium">
+                          {handler.name}
+                        </TableCell>
+                        <TableCell>
+                          <Badge
+                            variant={
+                              handler.type === "catch" ? "default" : "secondary"
+                            }
+                          >
+                            {handler.type}
+                          </Badge>
+                        </TableCell>
+                        <TableCell className="text-muted-foreground">
+                          {handler.events?.length
+                            ? handler.events.join(", ")
+                            : "from wire config"}
+                        </TableCell>
+                        <TableCell className="text-muted-foreground">
+                          {handler.destinationPlugId
+                            ? (plugs.find(
+                                (p) => p.id === handler.destinationPlugId,
+                              )?.name ?? handler.destinationPlugId)
+                            : "—"}
+                        </TableCell>
+                        <TableCell>
+                          {handler.lastRunStatus ? (
+                            <Badge
+                              variant={
+                                runStatusVariant[handler.lastRunStatus] ??
+                                "secondary"
+                              }
+                            >
+                              {handler.lastRunStatus}
+                            </Badge>
+                          ) : (
+                            <span className="text-muted-foreground">never</span>
+                          )}
+                        </TableCell>
+                        <TableCell>
+                          <Switch
+                            checked={handler.enabled}
+                            onCheckedChange={(v) =>
+                              toggleWebhookHandler(handler.id, v)
+                            }
+                          />
+                        </TableCell>
+                      </TableRow>
+                    ))}
+                  </TableBody>
+                </Table>
+              </CardContent>
+            </Card>
+          )}
+
+          <VarPanel plugName={selectedPlug.name} />
+          <WirePanel plugName={selectedPlug.name} webhookUrl={webhookUrl} />
+        </div>
+      )}
+    </div>
+  );
+}

package/dist/templates/inflow.example.ts

@@ -0,0 +1,61 @@
+// ============================================================================
+// Example: Inflow
+// Generated by khotan CLI · https://github.com/khotan-data
+//
+// Copy this file, rename it for your source service/resource, and register the
+// exported flow in {outputDir}/khotan.ts.
+// ============================================================================
+
+import { inflow, type InflowContext } from "./inflow";
+import { sendUpdate } from "khotan-data/factory";
+
+async function shopifyProductsWorkflow(ctx: InflowContext) {
+  "use workflow";
+
+  async function extractAndLoad() {
+    "use step";
+    console.log("Starting inflow", {
+      flow: ctx.flow.name,
+      khotanRunId: ctx.khotanRunId,
+      runType: ctx.runType,
+    });
+    await sendUpdate({
+      message: "Starting product inflow",
+      metadata: { flow: ctx.flow.name, runType: ctx.runType },
+    });
+
+    const response = await fetch("https://api.example.com/products", {
+      headers: {
+        Authorization: `Bearer ${ctx.vars["apiToken"] ?? ""}`,
+      },
+    });
+    const payload = (await response.json()) as {
+      data?: Array<Record<string, unknown>>;
+    };
+    const records = Array.isArray(payload.data) ? payload.data : [];
+
+    // Replace this with your app-specific transform and DB upsert.
+    console.log("Fetched records", records.length);
+    await sendUpdate({
+      message: `Fetched ${String(records.length)} products`,
+      extracted: records.length,
+      progress: 50,
+    });
+
+    return {
+      extracted: records.length,
+      transformed: records.length,
+      created: records.length,
+      metadata: { source: ctx.flow.name },
+    };
+  }
+
+  return extractAndLoad();
+}
+
+export const shopifyProductsInflow = inflow({
+  name: "shopify-products-inflow",
+  resource: "products",
+  schedule: "0 * * * *",
+  workflow: shopifyProductsWorkflow,
+});

package/dist/templates/inflow.ts

@@ -0,0 +1,98 @@
+// ============================================================================
+// Inflow — durable external service → app data movement
+// Generated by khotan CLI · https://github.com/khotan-data
+//
+// This file defines the inflow() builder and types. Create per-service flow
+// files (e.g. shopify-products.ts) using this builder to pull records from an
+// external service, transform them, and load them into your app with durable,
+// retryable Vercel Workflow steps. Inflow workflows can also use
+// khotanCache(ctx, "name") for checkpoints and last-seen markers across runs.
+// ============================================================================
+
+import type {
+  FlowRegistration,
+  FlowRunResult,
+  FlowWorkflowContext,
+} from "khotan-data/factory";
+
+export type InflowContext = FlowWorkflowContext & {
+  flow: FlowWorkflowContext["flow"] & { type: "inflow" };
+};
+
+export type InflowWorkflow = (
+  ctx: InflowContext,
+) => Promise<FlowRunResult | void>;
+
+export interface InflowConfig {
+  /** Unique name for this flow (used for DB tracking and Hub display) */
+  name: string;
+  /** Logical resource this flow feeds, e.g. "products" */
+  resource?: string;
+  /** Optional cron schedule used by scheduler integrations */
+  schedule?: string;
+  /** Durable workflow that extracts, transforms, and loads records */
+  workflow: InflowWorkflow;
+}
+
+export function inflow(config: InflowConfig): FlowRegistration {
+  return {
+    name: config.name,
+    type: "inflow",
+    resource: config.resource,
+    schedule: config.schedule,
+    workflow: config.workflow as FlowRegistration["workflow"],
+  };
+}
+
+// ---------------------------------------------------------------------------
+// Usage Example (create a file like flows/shopify-products.ts)
+// ---------------------------------------------------------------------------
+//
+// import { bindWorkflowPlug, inflow, type InflowContext, sendUpdate } from "khotan-data/factory";
+// import { db } from "@/db";
+// import { products } from "@/db/schema";
+// import { shopifyPlug } from "../plugs/shopify";
+//
+// async function shopifyProductsWorkflow(ctx: InflowContext) {
+//   "use workflow";
+//
+//   async function extractAndLoad() {
+//     "use step";
+//     console.log("Starting inflow", {
+//       flow: ctx.flow.name,
+//       khotanRunId: ctx.khotanRunId,
+//       runType: ctx.runType,
+//     });
+//     await sendUpdate({ message: "Starting Shopify products inflow" });
+//     const shopify = bindWorkflowPlug(shopifyPlug, ctx);
+//
+//     const response = await shopify.get<{ data?: Array<{ id: string; sku?: string }> }>("/products");
+//     const records = Array.isArray(response.data) ? response.data : [];
+//     await sendUpdate({ message: `Fetched ${records.length} products`, extracted: records.length });
+//
+//     if (records.length > 0) {
+//       await db.insert(products).values(
+//         records.map((record) => ({
+//           externalId: record.id,
+//           sku: record.sku ?? record.id,
+//         })),
+//       );
+//     }
+//
+//     return {
+//       extracted: records.length,
+//       transformed: records.length,
+//       created: records.length,
+//       metadata: { source: ctx.flow.name },
+//     };
+//   }
+//
+//   return extractAndLoad();
+// }
+//
+// export const shopifyProductsInflow = inflow({
+//   name: "shopify-products-inflow",
+//   resource: "products",
+//   schedule: "0 * * * *",
+//   workflow: shopifyProductsWorkflow,
+// });

package/dist/templates/khotan-config.ts

@@ -0,0 +1,49 @@
+// ============================================================================
+// Khotan Config — register your plugs, caches, and flows here
+// Generated by khotan CLI · https://github.com/khotan-data
+//
+// Import your Drizzle database instance and register plugs below.
+// Restart the dev server after making changes.
+// ============================================================================
+
+import { khotan, drizzleAdapter } from "khotan-data/factory";
+// TODO: Update this import to your Drizzle database instance
+import { db } from "@/db";
+
+const khotanData = khotan({
+  adapter: drizzleAdapter(db),
+
+  // Resources define logical entity types for cross-referencing across plugs.
+  // The mapping block declares the shared identity contract for that resource.
+  resources: [
+    // Example resource registration:
+    //
+    // { name: "products", mapping: { connectField: "sku" }, description: "Product catalog" },
+    // { name: "orders", mapping: { connectField: "order_number" } },
+  ],
+
+  plugs: [
+    // Example plug registration:
+    //
+    // {
+    //   name: "stripe",
+    //   baseUrl: "https://api.stripe.com",
+    //   authType: "bearer",
+    //   flows: [
+    //     { name: "products-inflow", type: "inflow", schedule: "0 * * * *", resource: "products" },
+    //     { name: "invoices-inflow", type: "inflow", schedule: "0 0 * * *" },
+    //   ],
+    // },
+  ],
+
+  // Caches store durable key/value state for relays, inflows, outflows, and
+  // webhook workflows. Register named caches you want to access via
+  // khotanCache(ctx, "cache-name") or khotanData.cache("cache-name").
+  caches: [
+    // Example cache registration:
+    //
+    // cin7ProductsSnapshotCache,
+  ],
+});
+
+export default khotanData;

package/dist/templates/khotan-route.ts

@@ -0,0 +1,13 @@
+// ============================================================================
+// Khotan API Route — catch-all handler for /api/khotan/*
+// Generated by khotan CLI · https://github.com/khotan-data
+//
+// This file wires your khotan config into a Next.js App Router route.
+// ============================================================================
+
+import { toNextJsHandler } from "khotan-data/factory";
+import khotanData from "@/khotan/khotan";
+
+export const { GET, POST, PUT, PATCH, DELETE } = toNextJsHandler(
+  khotanData.handler,
+);

package/dist/templates/logs-page.tsx

@@ -0,0 +1,9 @@
+import { KhotanLogs } from "@/components/khotan/logs";
+
+export default function KhotanLogsPage() {
+  return (
+    <main className="container mx-auto max-w-7xl px-4 py-10">
+      <KhotanLogs />
+    </main>
+  );
+}