khotan-data 0.1.0 → 0.2.0

package/dist/templates/skill-plug.md

@@ -98,28 +98,39 @@ export const myPlug = plug({
 
 Endpoints power the plug debugger UI, `khotan plug --compare`, and typed clients.
 
-## Typed Client (Contract Pattern)
+## Preferred Pattern
 
-For separate contract definition + type-safe calls:
+Keep each integration in a single app-owned plug file when possible:
 
 ```typescript
-import { defineContract, createPlugClient } from "khotan-data/plug";
-
-const contract = defineContract({
-  listProducts: {
-    method: "GET",
-    path: "/products",
-    query: z.object({ page: z.number().optional() }),
-    responses: { 200: z.object({ data: z.array(ProductSchema), total: z.number() }) },
-  },
+import { z } from "zod";
+import { plug, basic } from "./plug";
+
+const ProductSchema = z.object({
+  id: z.string(),
+  sku: z.string(),
+  name: z.string(),
 });
 
-const client = createPlugClient(contract, myPlug);
-const result = await client.listProducts({ query: { page: 1 } });
-// result.status — 200
-// result.body.data — typed as Product[]
+export type Product = z.infer<typeof ProductSchema>;
+
+export const myPlug = plug({
+  name: "my-service",
+  baseUrl: "https://api.example.com",
+  auth: basic(process.env.API_USER!, process.env.API_KEY!),
+  endpoints: {
+    listProducts: {
+      method: "GET",
+      path: "/products",
+      query: z.object({ page: z.number().optional(), limit: z.number().optional() }),
+      responses: { 200: z.array(ProductSchema) },
+    },
+  },
+});
 ```
 
+This keeps the runtime plug, debugger metadata, `khotan plug --compare`, and any exported types in one place.
+
 ## Hooks
 
 ```typescript
@@ -179,6 +190,18 @@ npx khotan plug myPlug --endpoint listProducts --compare  # Check schema
 
 Set `KHOTAN_DEBUG=1` for verbose `[khotan:auth]` and `[khotan:request]` console logs.
 
+### Recommended Plug Workflow
+
+1. Create the plug file and auth/hook setup.
+2. Add a small set of typed endpoints directly on the plug (`listProducts`, `getProduct`, etc).
+3. Run the app with `KHOTAN_DEBUG=1`.
+4. Use `npx khotan plug myPlug --info` to confirm the endpoints are visible to the debugger.
+5. Use `npx khotan plug myPlug --endpoint listProducts --compare` against the live API.
+6. Tighten schemas until the compare output matches the real payload shape you care about.
+7. Only then build inflows, relays, outflows, or webhook handlers on top of those endpoints.
+
+The package does not paginate or delta-sync for you automatically inside user flows. Your app code decides which typed endpoints to call, what page size to use, when to stop, and how to implement full, test, partial, backfill, reconcile, or delta runs.
+
 ## Managing Vars
 
 Use the CLI to inspect and update stored plug variables:

package/dist/templates/skill-setup.md

@@ -42,8 +42,8 @@ import { stripeChargesInflow } from "./flows/stripe-charges";
 const khotanData = khotan({
   adapter: drizzleAdapter(db),
   resources: [
-    { name: "products", connectField: "sku" },
-    { name: "orders", connectField: "order_number" },
+    { name: "products", mapping: { connectField: "sku" } },
+    { name: "orders", mapping: { connectField: "order_number" } },
   ],
   plugs: [
     {
@@ -90,8 +90,44 @@ The schema command auto-detects your Drizzle schema directory, updates `drizzle.
 |----------|----------|---------|
 | `DATABASE_URL` | Yes | Postgres connection (used by Drizzle) |
 | `KHOTAN_SECRET` | For variables | AES-256-GCM key for encrypting plug vars |
-| `KHOTAN_DEBUG` | For debugging | Enables `/debug/*` routes and the `plug` CLI (`probe` alias) |
+| `KHOTAN_DEBUG` | For debugging | Enables `/debug/*` routes and the `plug` CLI (`probe` alias). Automatically disabled when `NODE_ENV=production` |
 | `KHOTAN_WEBHOOK_URL` | For webhooks | Public URL for wire callbacks |
+| `CRON_SECRET` | For production cron | Protects the built-in `/api/khotan/cron` dispatcher route. The route fails closed in production when this is unset |
+
+## Securing the Management API
+
+The management API (`/api/khotan/*`) and the Hub dashboard expose plug
+credentials and operational controls. **They are public unless you gate them.**
+
+Pass an `authorize` hook to `khotan({ ... })`. It receives the raw `Request` and
+returns `true` to allow the request or `false` to reject it with `401`. It
+composes directly with session libraries like better-auth:
+
+```typescript
+import { khotan, drizzleAdapter } from "khotan-data/factory";
+import { auth } from "@/lib/auth";
+import { db } from "@/db";
+
+const khotanData = khotan({
+  adapter: drizzleAdapter(db),
+  authorize: async (request) => {
+    const session = await auth.api.getSession({ headers: request.headers });
+    return Boolean(session?.user); // or: session?.user?.role === "admin"
+  },
+  plugs: [/* ... */],
+});
+```
+
+Notes:
+- `authorize` is **not** a replacement for `KHOTAN_SECRET` — that key only
+  encrypts credentials at rest, it does not authenticate requests.
+- Inbound webhooks (`POST /webhook/:plug`, verified per-plug via `onVerify`),
+  the cron dispatcher (`CRON_SECRET`), and debug routes (`KHOTAN_DEBUG`,
+  non-production only) are exempt from `authorize` automatically.
+- Also protect the Hub dashboard page (e.g. `/config`) with your app's
+  middleware — `authorize` only guards the API, not your React pages.
+- Without `authorize`, khotan logs a startup warning. Always configure it
+  before deploying.
 
 ## Next.js Config
 
@@ -111,6 +147,47 @@ curl http://localhost:3000/api/khotan/flows      # Should list flows
 curl http://localhost:3000/api/khotan/resources   # Should list resources
 ```
 
+## Scheduled Flows On Vercel
+
+Khotan flow `schedule` values are runtime source-of-truth metadata. On Vercel, prefer a single dispatcher CRON instead of defining one platform CRON per flow.
+
+Add one entry to `vercel.json`:
+
+```json
+{
+  "crons": [
+    { "path": "/api/khotan/cron", "schedule": "* * * * *" }
+  ]
+}
+```
+
+Then define schedules only on your flows in `{outputDir}/khotan.ts`:
+
+```typescript
+{
+  name: "products-inflow",
+  type: "inflow",
+  schedule: "0 * * * *",
+  resource: "products",
+}
+```
+
+The dispatcher route evaluates which flows are due on each tick and starts them through the normal run-tracking path. If `CRON_SECRET` is set, Vercel should call the route with `Authorization: Bearer <CRON_SECRET>`.
+
+## Typical Build Order
+
+After init and schema setup, the usual path to a working sync is:
+
+1. Add or author a plug file for the external service.
+2. Define a few typed endpoints directly on the plug with Zod response schemas.
+3. Start the app with `KHOTAN_DEBUG=1`.
+4. Verify the plug is visible with `npx khotan plug --list` and `npx khotan plug myPlug --info`.
+5. Hit live endpoints with `npx khotan plug myPlug --endpoint listProducts --compare` until the schemas match the real API shape you intend to use.
+6. Register the plug in `{outputDir}/khotan.ts` with resources and flows.
+7. Only after endpoint verification, build inflows, relays, outflows, or webhook handlers on top of those live-checked endpoints.
+
+This keeps sync logic grounded in real API payloads before you write pagination, mapping, or transformation code.
+
 ## Troubleshooting
 
 - **Empty plug list**: Factory upserts on first request — hit any endpoint first, then check `/plugs`

package/dist/templates/topology-canvas.tsx

@@ -18,6 +18,7 @@ import {
   type NodeProps,
 } from "@xyflow/react";
 import "@xyflow/react/dist/style.css";
+import { khotanFetch, ApiErrorState } from "./api-state";
 import { Badge } from "@/components/ui/badge";
 import {
   Card,
@@ -1056,8 +1057,9 @@ function TopologyCanvasInner() {
   const [nodes, setNodes, onNodesChange] = useNodesState<TopologyNodeData>([]);
   const [edges, setEdges, onEdgesChange] = useEdgesState([]);
   const [loading, setLoading] = useState(true);
-  const [error, setError] = useState<string | null>(null);
+  const [error, setError] = useState<unknown>(null);
   const [lastUpdatedAt, setLastUpdatedAt] = useState<string | null>(null);
+  const [refreshKey, setRefreshKey] = useState(0);
 
   useEffect(() => {
     let cancelled = false;
@@ -1068,30 +1070,22 @@ function TopologyCanvasInner() {
           setError(null);
         }
 
-        const [plugsRes, flowsRes, runsRes] = await Promise.all([
-          fetch("/api/khotan/plugs"),
-          fetch("/api/khotan/flows"),
-          fetch("/api/khotan/runs?limit=100"),
+        const [plugs, flows, runsRaw] = await Promise.all([
+          khotanFetch<PlugRecord[]>("/api/khotan/plugs"),
+          khotanFetch<FlowRecord[]>("/api/khotan/flows"),
+          khotanFetch<unknown>("/api/khotan/runs?limit=100"),
         ]);
 
-        if (!plugsRes.ok || !flowsRes.ok || !runsRes.ok) {
-          throw new Error("Failed to load topology data from /api/khotan");
-        }
-
-        const plugs = (await plugsRes.json()) as PlugRecord[];
-        const flows = (await flowsRes.json()) as FlowRecord[];
-        const runs = normalizeRuns(await runsRes.json());
+        const runs = normalizeRuns(runsRaw);
 
         const webhookGroups = await Promise.all(
           plugs.map(async (plug) => {
             try {
-              const res = await fetch(
+              const handlers = await khotanFetch<
+                Array<Omit<WebhookHandlerRecord, "plugName">>
+              >(
                 `/api/khotan/webhook-handlers/${encodeURIComponent(plug.name)}`,
               );
-              if (!res.ok) return [] as WebhookHandlerRecord[];
-              const handlers = (await res.json()) as Array<
-                Omit<WebhookHandlerRecord, "plugName">
-              >;
               return handlers.map((handler) => ({
                 ...handler,
                 plugName: plug.name,
@@ -1113,11 +1107,7 @@ function TopologyCanvasInner() {
         setLastUpdatedAt(new Date().toISOString());
       } catch (loadError) {
         if (!cancelled) {
-          setError(
-            loadError instanceof Error
-              ? loadError.message
-              : "Unknown topology load failure",
-          );
+          setError(loadError);
         }
       } finally {
         if (!cancelled) {
@@ -1135,7 +1125,7 @@ function TopologyCanvasInner() {
       cancelled = true;
       window.clearInterval(interval);
     };
-  }, []);
+  }, [refreshKey]);
 
   const model = useMemo(() => {
     return snapshot ? buildTopologyModel(snapshot) : null;
@@ -1180,19 +1170,18 @@ function TopologyCanvasInner() {
 
   if (error) {
     return (
-      <Card className="border-red-200 bg-white/80 shadow-xl backdrop-blur">
+      <Card className="border-white/70 bg-white/80 shadow-xl backdrop-blur">
         <CardHeader>
           <CardTitle>Topology Canvas</CardTitle>
           <CardDescription>
             Could not load the graph from the local Khotan API.
           </CardDescription>
         </CardHeader>
-        <CardContent className="space-y-3 text-sm">
-          <p className="text-red-600">{error}</p>
-          <p className="text-slate-500">
-            Make sure the catch-all Khotan route is mounted and your dev server
-            is running.
-          </p>
+        <CardContent>
+          <ApiErrorState
+            error={error}
+            onRetry={() => setRefreshKey((v) => v + 1)}
+          />
         </CardContent>
       </Card>
     );

package/dist/templates/var-panel.tsx

@@ -6,6 +6,7 @@ import { Badge } from "@/components/ui/badge";
 import { Button } from "@/components/ui/button";
 import { Input } from "@/components/ui/input";
 import { Label } from "@/components/ui/label";
+import { khotanFetch, ApiErrorState } from "./api-state";
 
 // ============================================================================
 // Var Panel — UI for managing plug variables
@@ -44,6 +45,7 @@ export function VarPanel({
   const [configured, setConfigured] = useState(false);
   const [loading, setLoading] = useState(true);
   const [saving, setSaving] = useState(false);
+  const [loadError, setLoadError] = useState<unknown>(null);
   const [error, setError] = useState<string | null>(null);
   const [success, setSuccess] = useState<string | null>(null);
   const [editing, setEditing] = useState(false);
@@ -52,21 +54,23 @@ export function VarPanel({
   );
 
   const fetchVariables = useCallback(async () => {
+    setLoading(true);
+    setLoadError(null);
     try {
-      const res = await fetch(`${basePath}/variables/${plugName}`);
-      if (!res.ok) {
-        setFields([]);
-        setConfigured(false);
-        return;
-      }
-      const data = await res.json();
+      const data = await khotanFetch<{
+        fields?: VarField[];
+        values?: Record<string, string>;
+        configured?: boolean;
+      }>(`${basePath}/variables/${plugName}`);
       setFields((data.fields ?? []).filter((f: VarField) => !f.hidden));
       setValues(data.values ?? {});
       setConfigured(data.configured ?? false);
-      setFormValues(data.configured ? data.values : {});
+      setFormValues(data.configured ? (data.values ?? {}) : {});
       setError(null);
-    } catch {
-      setError("Failed to load variables");
+    } catch (err) {
+      setFields([]);
+      setConfigured(false);
+      setLoadError(err);
     } finally {
       setLoading(false);
     }
@@ -144,6 +148,25 @@ export function VarPanel({
     );
   }
 
+  if (loadError) {
+    return (
+      <Card>
+        <CardHeader className="pb-2">
+          <CardTitle className="text-sm font-medium capitalize">
+            {displayName} Variables
+          </CardTitle>
+        </CardHeader>
+        <CardContent>
+          <ApiErrorState
+            error={loadError}
+            onRetry={() => void fetchVariables()}
+            compact
+          />
+        </CardContent>
+      </Card>
+    );
+  }
+
   if (fields.length === 0) return null;
 
   return (

package/dist/templates/webhook-events-table.tsx

@@ -1,6 +1,7 @@
 "use client";
 
 import { useEffect, useState } from "react";
+import { khotanFetch, ApiErrorState } from "./api-state";
 import { Badge } from "@/components/ui/badge";
 import { Button } from "@/components/ui/button";
 import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card";
@@ -75,7 +76,7 @@ export function KhotanWebhookEventsTable({
   const [data, setData] = useState<PageResponse<WebhookEventItem> | null>(null);
   const [offset, setOffset] = useState(0);
   const [loading, setLoading] = useState(true);
-  const [error, setError] = useState<string | null>(null);
+  const [error, setError] = useState<unknown>(null);
   const [refreshKey, setRefreshKey] = useState(0);
 
   useEffect(() => {
@@ -85,19 +86,15 @@ export function KhotanWebhookEventsTable({
       setLoading(true);
       setError(null);
       try {
-        const res = await fetch(
+        const json = await khotanFetch<PageResponse<WebhookEventItem>>(
           `/api/khotan/webhook-events?limit=${String(pageSize)}&offset=${String(offset)}`,
         );
-        if (!res.ok) {
-          throw new Error("Failed to load webhook events");
-        }
-        const json = (await res.json()) as PageResponse<WebhookEventItem>;
         if (!cancelled) {
           setData(json);
         }
       } catch (err) {
         if (!cancelled) {
-          setError(err instanceof Error ? err.message : "Unknown error");
+          setError(err);
         }
       } finally {
         if (!cancelled) {
@@ -131,110 +128,116 @@ export function KhotanWebhookEventsTable({
       </CardHeader>
       <CardContent className="space-y-4">
         {error ? (
-          <div className="rounded-md border border-destructive/30 bg-destructive/5 p-3 text-sm text-destructive">
-            {error}
-          </div>
+          <ApiErrorState
+            error={error}
+            onRetry={() => setRefreshKey((v) => v + 1)}
+            compact
+          />
         ) : null}
 
-        <Table>
-          <TableHeader>
-            <TableRow>
-              <TableHead>Received</TableHead>
-              <TableHead>Event</TableHead>
-              <TableHead>Handler</TableHead>
-              <TableHead>Plug</TableHead>
-              <TableHead>Run</TableHead>
-              <TableHead>Payload</TableHead>
-            </TableRow>
-          </TableHeader>
-          <TableBody>
-            {loading ? (
+        {error ? null : (
+          <Table>
+            <TableHeader>
               <TableRow>
-                <TableCell
-                  colSpan={6}
-                  className="text-sm text-muted-foreground"
-                >
-                  Loading webhook events...
-                </TableCell>
+                <TableHead>Received</TableHead>
+                <TableHead>Event</TableHead>
+                <TableHead>Handler</TableHead>
+                <TableHead>Plug</TableHead>
+                <TableHead>Run</TableHead>
+                <TableHead>Payload</TableHead>
               </TableRow>
-            ) : data?.items.length ? (
-              data.items.map((item) => (
-                <TableRow key={item.id}>
-                  <TableCell className="text-sm text-muted-foreground">
-                    {formatDateTime(item.receivedAt)}
-                  </TableCell>
-                  <TableCell className="font-medium">
-                    {item.eventType}
-                  </TableCell>
-                  <TableCell className="text-sm text-muted-foreground">
-                    {formatHandler(item)}
-                  </TableCell>
-                  <TableCell className="text-muted-foreground">
-                    {item.plugName ?? "-"}
-                  </TableCell>
-                  <TableCell className="space-y-1 text-xs">
-                    <div className="font-mono text-muted-foreground">
-                      {item.khotanRunId}
-                    </div>
-                    <div className="flex flex-wrap items-center gap-2">
-                      {item.runStatus ? (
-                        <Badge variant={statusVariant[item.runStatus]}>
-                          {item.runStatus}
-                        </Badge>
-                      ) : null}
-                      <span className="font-mono text-muted-foreground">
-                        {item.workflowRunId ?? "-"}
-                      </span>
-                    </div>
+            </TableHeader>
+            <TableBody>
+              {loading ? (
+                <TableRow>
+                  <TableCell
+                    colSpan={6}
+                    className="text-sm text-muted-foreground"
+                  >
+                    Loading webhook events...
                   </TableCell>
-                  <TableCell className="max-w-80">
-                    <details>
-                      <summary className="cursor-pointer text-sm text-primary">
-                        View payload
-                      </summary>
-                      <pre className="mt-2 max-h-64 overflow-auto rounded-md bg-muted p-3 text-xs">
-                        {JSON.stringify(item.payload, null, 2)}
-                      </pre>
-                    </details>
+                </TableRow>
+              ) : data?.items.length ? (
+                data.items.map((item) => (
+                  <TableRow key={item.id}>
+                    <TableCell className="text-sm text-muted-foreground">
+                      {formatDateTime(item.receivedAt)}
+                    </TableCell>
+                    <TableCell className="font-medium">
+                      {item.eventType}
+                    </TableCell>
+                    <TableCell className="text-sm text-muted-foreground">
+                      {formatHandler(item)}
+                    </TableCell>
+                    <TableCell className="text-muted-foreground">
+                      {item.plugName ?? "-"}
+                    </TableCell>
+                    <TableCell className="space-y-1 text-xs">
+                      <div className="font-mono text-muted-foreground">
+                        {item.khotanRunId}
+                      </div>
+                      <div className="flex flex-wrap items-center gap-2">
+                        {item.runStatus ? (
+                          <Badge variant={statusVariant[item.runStatus]}>
+                            {item.runStatus}
+                          </Badge>
+                        ) : null}
+                        <span className="font-mono text-muted-foreground">
+                          {item.workflowRunId ?? "-"}
+                        </span>
+                      </div>
+                    </TableCell>
+                    <TableCell className="max-w-80">
+                      <details>
+                        <summary className="cursor-pointer text-sm text-primary">
+                          View payload
+                        </summary>
+                        <pre className="mt-2 max-h-64 overflow-auto rounded-md bg-muted p-3 text-xs">
+                          {JSON.stringify(item.payload, null, 2)}
+                        </pre>
+                      </details>
+                    </TableCell>
+                  </TableRow>
+                ))
+              ) : (
+                <TableRow>
+                  <TableCell
+                    colSpan={6}
+                    className="text-sm text-muted-foreground"
+                  >
+                    No webhook events recorded yet.
                   </TableCell>
                 </TableRow>
-              ))
-            ) : (
-              <TableRow>
-                <TableCell
-                  colSpan={6}
-                  className="text-sm text-muted-foreground"
-                >
-                  No webhook events recorded yet.
-                </TableCell>
-              </TableRow>
-            )}
-          </TableBody>
-        </Table>
+              )}
+            </TableBody>
+          </Table>
+        )}
 
-        <div className="flex items-center justify-between gap-3">
-          <p className="text-sm text-muted-foreground">
-            Page {Math.floor(offset / pageSize) + 1}
-          </p>
-          <div className="flex items-center gap-2">
-            <Button
-              variant="outline"
-              size="sm"
-              disabled={offset === 0 || loading}
-              onClick={() => setOffset(Math.max(offset - pageSize, 0))}
-            >
-              Previous
-            </Button>
-            <Button
-              variant="outline"
-              size="sm"
-              disabled={!data?.page.hasMore || loading}
-              onClick={() => setOffset(offset + pageSize)}
-            >
-              Next
-            </Button>
+        {error ? null : (
+          <div className="flex items-center justify-between gap-3">
+            <p className="text-sm text-muted-foreground">
+              Page {Math.floor(offset / pageSize) + 1}
+            </p>
+            <div className="flex items-center gap-2">
+              <Button
+                variant="outline"
+                size="sm"
+                disabled={offset === 0 || loading}
+                onClick={() => setOffset(Math.max(offset - pageSize, 0))}
+              >
+                Previous
+              </Button>
+              <Button
+                variant="outline"
+                size="sm"
+                disabled={!data?.page.hasMore || loading}
+                onClick={() => setOffset(offset + pageSize)}
+              >
+                Next
+              </Button>
+            </div>
           </div>
-        </div>
+        )}
       </CardContent>
     </Card>
   );