khotan-data 0.1.1 → 0.3.0

package/dist/templates/runs-table.tsx

@@ -2,6 +2,7 @@
 
 import { Fragment, useCallback, useEffect, useState } from "react";
 import { RefreshCw } from "lucide-react";
+import { khotanFetch, ApiErrorState } from "./api-state";
 import { Badge } from "@/components/ui/badge";
 import { Button } from "@/components/ui/button";
 import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card";
@@ -359,7 +360,7 @@ export function KhotanRunsTable({ pageSize = 10 }: { pageSize?: number } = {}) {
   const [data, setData] = useState<PageResponse<RunLogItem> | null>(null);
   const [offset, setOffset] = useState(0);
   const [loading, setLoading] = useState(true);
-  const [error, setError] = useState<string | null>(null);
+  const [error, setError] = useState<unknown>(null);
   const [refreshKey, setRefreshKey] = useState(0);
   const [expandedRunId, setExpandedRunId] = useState<string | null>(null);
   const [streamingEnabled, setStreamingEnabled] = useState(false);
@@ -378,20 +379,16 @@ export function KhotanRunsTable({ pageSize = 10 }: { pageSize?: number } = {}) {
       setLoading(true);
       setError(null);
       try {
-        const res = await fetch(
+        const json = await khotanFetch<PageResponse<RunLogItem>>(
           `/api/khotan/runs?limit=${String(pageSize)}&offset=${String(offset)}`,
         );
-        if (!res.ok) {
-          throw new Error("Failed to load runs");
-        }
-        const json = (await res.json()) as PageResponse<RunLogItem>;
         if (!cancelled) {
           setData(json);
           setLastUpdatedAt(new Date().toISOString());
         }
       } catch (err) {
         if (!cancelled) {
-          setError(err instanceof Error ? err.message : "Unknown error");
+          setError(err);
         }
       } finally {
         if (!cancelled) {
@@ -454,138 +451,144 @@ export function KhotanRunsTable({ pageSize = 10 }: { pageSize?: number } = {}) {
       </CardHeader>
       <CardContent className="space-y-4">
         {error ? (
-          <div className="rounded-md border border-destructive/30 bg-destructive/5 p-3 text-sm text-destructive">
-            {error}
-          </div>
+          <ApiErrorState
+            error={error}
+            onRetry={() => setRefreshKey((v) => v + 1)}
+            compact
+          />
         ) : null}
 
-        <Table>
-          <TableHeader>
-            <TableRow>
-              <TableHead>Started</TableHead>
-              <TableHead>Status</TableHead>
-              <TableHead>Source</TableHead>
-              <TableHead>Plug</TableHead>
-              <TableHead>Run Type</TableHead>
-              <TableHead>Counts</TableHead>
-              <TableHead>Workflow</TableHead>
-              <TableHead />
-            </TableRow>
-          </TableHeader>
-          <TableBody>
-            {loading ? (
+        {error ? null : (
+          <Table>
+            <TableHeader>
               <TableRow>
-                <TableCell
-                  colSpan={8}
-                  className="text-sm text-muted-foreground"
-                >
-                  Loading runs...
-                </TableCell>
+                <TableHead>Started</TableHead>
+                <TableHead>Status</TableHead>
+                <TableHead>Source</TableHead>
+                <TableHead>Plug</TableHead>
+                <TableHead>Run Type</TableHead>
+                <TableHead>Counts</TableHead>
+                <TableHead>Workflow</TableHead>
+                <TableHead />
               </TableRow>
-            ) : data?.items.length ? (
-              data.items.map((item) => (
-                <Fragment key={item.id}>
-                  <TableRow>
-                    <TableCell className="text-sm text-muted-foreground">
-                      <div>{formatDateTime(item.startedAt)}</div>
-                      <div className="text-xs">
-                        {item.completedAt
-                          ? `completed ${formatDateTime(item.completedAt)}`
-                          : "in progress"}
-                      </div>
-                    </TableCell>
-                    <TableCell>
-                      <Badge variant={statusVariant[item.status]}>
-                        {statusLabel[item.status]}
-                      </Badge>
-                      {item.error ? (
-                        <div
-                          className="mt-1 max-w-56 truncate text-xs text-destructive"
-                          title={item.error}
-                        >
-                          {item.error}
-                        </div>
-                      ) : null}
-                    </TableCell>
-                    <TableCell className="font-medium">
-                      {formatSource(item)}
-                    </TableCell>
-                    <TableCell className="text-muted-foreground">
-                      {item.plugName ?? "-"}
-                    </TableCell>
-                    <TableCell className="font-mono text-xs">
-                      {item.runType}
-                    </TableCell>
-                    <TableCell className="max-w-64 text-xs text-muted-foreground">
-                      {formatCounts(item)}
-                    </TableCell>
-                    <TableCell className="font-mono text-xs text-muted-foreground">
-                      {item.workflowRunId ?? "-"}
-                    </TableCell>
-                    <TableCell className="text-right">
-                      <Button
-                        variant="outline"
-                        size="sm"
-                        onClick={() =>
-                          setExpandedRunId((current) =>
-                            current === item.id ? null : item.id,
-                          )
-                        }
-                      >
-                        {expandedRunId === item.id ? "Hide" : "Details"}
-                      </Button>
-                    </TableCell>
-                  </TableRow>
-                  {expandedRunId === item.id ? (
+            </TableHeader>
+            <TableBody>
+              {loading ? (
+                <TableRow>
+                  <TableCell
+                    colSpan={8}
+                    className="text-sm text-muted-foreground"
+                  >
+                    Loading runs...
+                  </TableCell>
+                </TableRow>
+              ) : data?.items.length ? (
+                data.items.map((item) => (
+                  <Fragment key={item.id}>
                     <TableRow>
-                      <TableCell colSpan={8}>
-                        <RunDetails
-                          run={item}
-                          streamingEnabled={streamingEnabled}
-                          onChanged={() => setRefreshKey((v) => v + 1)}
-                          onStreamInbound={pulseLiveIndicator}
-                        />
+                      <TableCell className="text-sm text-muted-foreground">
+                        <div>{formatDateTime(item.startedAt)}</div>
+                        <div className="text-xs">
+                          {item.completedAt
+                            ? `completed ${formatDateTime(item.completedAt)}`
+                            : "in progress"}
+                        </div>
+                      </TableCell>
+                      <TableCell>
+                        <Badge variant={statusVariant[item.status]}>
+                          {statusLabel[item.status]}
+                        </Badge>
+                        {item.error ? (
+                          <div
+                            className="mt-1 max-w-56 truncate text-xs text-destructive"
+                            title={item.error}
+                          >
+                            {item.error}
+                          </div>
+                        ) : null}
+                      </TableCell>
+                      <TableCell className="font-medium">
+                        {formatSource(item)}
+                      </TableCell>
+                      <TableCell className="text-muted-foreground">
+                        {item.plugName ?? "-"}
+                      </TableCell>
+                      <TableCell className="font-mono text-xs">
+                        {item.runType}
+                      </TableCell>
+                      <TableCell className="max-w-64 text-xs text-muted-foreground">
+                        {formatCounts(item)}
+                      </TableCell>
+                      <TableCell className="font-mono text-xs text-muted-foreground">
+                        {item.workflowRunId ?? "-"}
+                      </TableCell>
+                      <TableCell className="text-right">
+                        <Button
+                          variant="outline"
+                          size="sm"
+                          onClick={() =>
+                            setExpandedRunId((current) =>
+                              current === item.id ? null : item.id,
+                            )
+                          }
+                        >
+                          {expandedRunId === item.id ? "Hide" : "Details"}
+                        </Button>
                       </TableCell>
                     </TableRow>
-                  ) : null}
-                </Fragment>
-              ))
-            ) : (
-              <TableRow>
-                <TableCell
-                  colSpan={8}
-                  className="text-sm text-muted-foreground"
-                >
-                  No runs recorded yet.
-                </TableCell>
-              </TableRow>
-            )}
-          </TableBody>
-        </Table>
+                    {expandedRunId === item.id ? (
+                      <TableRow>
+                        <TableCell colSpan={8}>
+                          <RunDetails
+                            run={item}
+                            streamingEnabled={streamingEnabled}
+                            onChanged={() => setRefreshKey((v) => v + 1)}
+                            onStreamInbound={pulseLiveIndicator}
+                          />
+                        </TableCell>
+                      </TableRow>
+                    ) : null}
+                  </Fragment>
+                ))
+              ) : (
+                <TableRow>
+                  <TableCell
+                    colSpan={8}
+                    className="text-sm text-muted-foreground"
+                  >
+                    No runs recorded yet.
+                  </TableCell>
+                </TableRow>
+              )}
+            </TableBody>
+          </Table>
+        )}
 
-        <div className="flex items-center justify-between gap-3">
-          <p className="text-sm text-muted-foreground">
-            Page {Math.floor(offset / pageSize) + 1}
-          </p>
-          <div className="flex items-center gap-2">
-            <Button
-              variant="outline"
-              size="sm"
-              disabled={offset === 0 || loading}
-              onClick={() => setOffset(Math.max(offset - pageSize, 0))}
-            >
-              Previous
-            </Button>
-            <Button
-              variant="outline"
-              size="sm"
-              disabled={!data?.page.hasMore || loading}
-              onClick={() => setOffset(offset + pageSize)}
-            >
-              Next
-            </Button>
+        {error ? null : (
+          <div className="flex items-center justify-between gap-3">
+            <p className="text-sm text-muted-foreground">
+              Page {Math.floor(offset / pageSize) + 1}
+            </p>
+            <div className="flex items-center gap-2">
+              <Button
+                variant="outline"
+                size="sm"
+                disabled={offset === 0 || loading}
+                onClick={() => setOffset(Math.max(offset - pageSize, 0))}
+              >
+                Previous
+              </Button>
+              <Button
+                variant="outline"
+                size="sm"
+                disabled={!data?.page.hasMore || loading}
+                onClick={() => setOffset(offset + pageSize)}
+              >
+                Next
+              </Button>
+            </div>
           </div>
-        </div>
+        )}
       </CardContent>
     </Card>
   );

package/dist/templates/skill-dashboard.md

@@ -20,7 +20,7 @@ The Hub scaffolds three components to `src/components/khotan/`:
 
 | File | Purpose |
 |------|---------|
-| `hub.tsx` | Main `<KhotanHub />` — plug cards, flow table, enable/disable toggles |
+| `hub.tsx` | Main `<KhotanHub />` — plug cards, flow table, enable/disable toggles, per-flow "Run now" trigger |
 | `var-panel.tsx` | Variables panel for configuring plug vars |
 | `wire-panel.tsx` | Webhook subscription management (connect/disconnect) |
 
@@ -44,6 +44,7 @@ Or use `npx khotan add config-page-1` to scaffold a `/config` page automatically
 
 - Lists all registered plugs with status badges (connected/error/idle)
 - Click a plug to see its flows with enable/disable toggles
+- "Run now" button on each flow row triggers a tracked run via `POST /api/khotan/flows/:id/runs` (uses the browser session, so it passes your `authorize` hook)
 - VarPanel: configure plug variables (stored encrypted via `KHOTAN_SECRET`)
 - WirePanel: manage webhook subscriptions (requires wires configured on plug)
 - Debug button on each plug card (visible when `KHOTAN_DEBUG=1`)

package/dist/templates/skill-setup.md

@@ -90,9 +90,48 @@ The schema command auto-detects your Drizzle schema directory, updates `drizzle.
 |----------|----------|---------|
 | `DATABASE_URL` | Yes | Postgres connection (used by Drizzle) |
 | `KHOTAN_SECRET` | For variables | AES-256-GCM key for encrypting plug vars |
-| `KHOTAN_DEBUG` | For debugging | Enables `/debug/*` routes and the `plug` CLI (`probe` alias) |
+| `KHOTAN_DEBUG` | For debugging | Enables `/debug/*` routes and the `plug` CLI (`probe` alias). Automatically disabled when `NODE_ENV=production` |
 | `KHOTAN_WEBHOOK_URL` | For webhooks | Public URL for wire callbacks |
-| `CRON_SECRET` | For production cron | Protects the built-in `/api/khotan/cron` dispatcher route |
+| `CRON_SECRET` | For production cron | Protects the built-in `/api/khotan/cron` dispatcher route. The route fails closed in production when this is unset |
+
+## Securing the Management API
+
+The management API (`/api/khotan/*`) and the Hub dashboard expose plug
+credentials and operational controls. **They are public unless you gate them.**
+
+Pass an `authorize` hook to `khotan({ ... })`. It receives the raw `Request` and
+returns `true` to allow the request or `false` to reject it with `401`. It
+composes directly with session libraries like better-auth:
+
+```typescript
+import { khotan, drizzleAdapter } from "khotan-data/factory";
+import { auth } from "@/lib/auth";
+import { db } from "@/db";
+
+const khotanData = khotan({
+  adapter: drizzleAdapter(db),
+  authorize: async (request) => {
+    const session = await auth.api.getSession({ headers: request.headers });
+    return Boolean(session?.user); // or: session?.user?.role === "admin"
+  },
+  plugs: [/* ... */],
+});
+```
+
+Notes:
+- `authorize` is **not** a replacement for `KHOTAN_SECRET` — that key only
+  encrypts credentials at rest, it does not authenticate requests. Conversely,
+  `KHOTAN_SECRET` is **not** an HTTP credential: do not send it as a Bearer
+  token. Management routes are gated solely by `authorize` (plus the dev-only
+  CLI HMAC token). A rejected request returns `401` with `code:
+  authorize_rejected` and a `hint` explaining how to authenticate.
+- Inbound webhooks (`POST /webhook/:plug`, verified per-plug via `onVerify`),
+  the cron dispatcher (`CRON_SECRET`), and debug routes (`KHOTAN_DEBUG`,
+  non-production only) are exempt from `authorize` automatically.
+- Also protect the Hub dashboard page (e.g. `/config`) with your app's
+  middleware — `authorize` only guards the API, not your React pages.
+- Without `authorize`, khotan logs a startup warning. Always configure it
+  before deploying.
 
 ## Next.js Config
 
@@ -104,6 +143,76 @@ const nextConfig = {
 };
 ```
 
+## Workflow Runtime & Middleware/Proxy
+
+Inflows, outflows, relays, catch, and pass run on **Vercel Workflow**, which
+communicates over `/.well-known/workflow/*`. If your app has a `middleware.ts`
+(or `proxy.ts`) whose `matcher` captures these paths, durable runs **silently
+fail** — steps never get invoked and runs hang.
+
+`npx khotan init` detects a middleware/proxy file and warns when it may
+intercept these paths. Exclude them from the matcher:
+
+```typescript
+// middleware.ts
+export const config = {
+  matcher: ["/((?!_next|.well-known/workflow).*)"],
+};
+```
+
+If you do auth or rewrites manually (not via `matcher`), short-circuit early:
+
+```typescript
+export function middleware(request: NextRequest) {
+  if (request.nextUrl.pathname.startsWith("/.well-known/workflow")) {
+    return NextResponse.next();
+  }
+  // ...your logic
+}
+```
+
+Vercel Workflow also requires AI Gateway OIDC — run `vercel link` and
+`vercel env pull` so `VERCEL_OIDC_TOKEN` is available locally.
+
+## Triggering Flows
+
+Start a flow through khotan (never call the workflow function directly) so run
+tracking and Workflow IDs are recorded. The API is `khotanData.flow(name).start()`:
+
+```typescript
+import khotanData from "@/lib/khotan/khotan";
+
+await khotanData.flow("products-inflow", { plugName: "shopify" }).start({
+  runType: "delta", // or "full"
+});
+```
+
+`plugName` is only needed to disambiguate when the same flow name is registered
+under multiple plugs. There is no `khotanData.api.*` or `flow().run()` surface —
+`flow(name).start(options)` is the single entry point for manual and scheduled
+runs alike. The cron dispatcher (`/api/khotan/cron`) calls this same path.
+
+### Triggering over HTTP (scripts / external services)
+
+There is **no** `POST /flows/:name/run` route. The HTTP trigger is:
+
+```
+POST /api/khotan/flows/{flowId}/runs    body: { "runType": "delta" }
+```
+
+This is a **management route**, so it goes through your `authorize` hook. Common
+gotcha: `KHOTAN_SECRET` is an encryption key, **not** an HTTP credential — sending
+`Authorization: Bearer <KHOTAN_SECRET>` returns `401` with `code: authorize_rejected`.
+To trigger from outside the app, authenticate with a credential your `authorize`
+hook accepts (a session cookie, or your own token you validate inside `authorize`).
+
+Prefer triggering server-side with `khotanData.flow(name).start()` whenever you
+can — it needs no HTTP round-trip or auth.
+
+The `npx khotan flows trigger <name>` CLI works in **dev** without any of this: it
+signs a short-lived HMAC token from `KHOTAN_SECRET` (the `KhotanCLI` auth scheme,
+disabled when `NODE_ENV=production`). The raw secret never leaves your machine.
+
 ## Verify Setup
 
 ```bash
@@ -159,3 +268,5 @@ This keeps sync logic grounded in real API payloads before you write pagination,
 - **"Cannot find module khotan-data"**: Add to `serverExternalPackages` in next.config.ts
 - **Migration fails**: Ensure `DATABASE_URL` is set and Postgres is reachable
 - **Init won't overwrite**: By design — delete the file manually if you need to re-scaffold
+- **Flow/workflow runs hang or never start**: Check your `middleware.ts`/`proxy.ts` matcher excludes `/.well-known/workflow/*` (see "Workflow Runtime & Middleware/Proxy")
+- **Step "is not a function" / fails to resolve at runtime**: Declare `"use step"` functions at module top level and pass `ctx` as an argument — never nest them inside the `"use workflow"` function (closures over workflow scope cannot be hoisted)

package/dist/templates/skill-webhook.md

@@ -110,19 +110,28 @@ npx khotan add catch --yes
 
 Process webhook events durably via Vercel Workflow:
 
-```typescript
-import { catchEvent } from "./webhooks/catch";
-
-const processInvoice = catchEvent(async (ctx) => {
-  "use workflow";
+Declare `"use step"` functions at module top level and pass `ctx` (serializable
+data) as an argument. Nesting steps inside the `"use workflow"` function fails at
+runtime — closures over workflow scope cannot be hoisted.
 
-  async function persist() {
-    "use step";
-    // Write to database — retried on failure
-    await db.insert(invoices).values(ctx.event);
-  }
-
-  await persist();
+```typescript
+import { catchEvent, type CatchContext } from "./webhooks/catch";
+import { db } from "@/db";
+import { invoices } from "@/db/schema";
+
+// Step: top-level, full Node.js access, retried on failure.
+async function persistInvoice(ctx: CatchContext) {
+  "use step";
+  await db.insert(invoices).values(ctx.event);
+}
+
+const processInvoice = catchEvent({
+  name: "stripe-invoices",
+  events: ["invoice.paid"],
+  workflow: async (ctx) => {
+    "use workflow";
+    await persistInvoice(ctx);
+  },
 });
 ```
 
@@ -140,22 +149,35 @@ npx khotan add pass --yes
 
 Forward webhook events to another service:
 
+The context exposes `ctx.event`, `ctx.eventType`, and `ctx.destVars` (the
+decrypted credentials for the destination plug). There is no `ctx.destPlug` —
+construct the destination plug from `destVars` inside a top-level step.
+
 ```typescript
-import { pass } from "./webhooks/pass";
+import { pass, type PassContext } from "./webhooks/pass";
+import { plug } from "@/lib/khotan/plugs/plug";
+
+// Step: top-level. Build the destination plug from ctx.destVars.
+async function forwardToSlackStep(ctx: PassContext) {
+  "use step";
+  const slack = plug({
+    name: "slack",
+    baseUrl: "https://slack.com/api",
+    authType: "bearer",
+    auth: { bearer: { token: ctx.destVars["botToken"] ?? "" } },
+  });
+  await slack.post("/chat.postMessage", {
+    body: { channel: ctx.destVars["channelId"], text: `New event: ${ctx.eventType}` },
+  });
+}
 
 const forwardToSlack = pass({
-  to: "slack",  // Destination plug name (must be registered)
+  name: "stripe-to-slack",
+  to: "slack", // Destination plug name (must be registered)
+  events: ["invoice.paid"],
   workflow: async (ctx) => {
     "use workflow";
-    // ctx.event — the incoming webhook payload
-    // ctx.destVars — destination plug variables
-    async function forward() {
-      "use step";
-      await ctx.destPlug.post("/messages", {
-        body: { text: `New event: ${ctx.event.type}` },
-      });
-    }
-    await forward();
+    await forwardToSlackStep(ctx);
   },
 });
 ```

package/dist/templates/topology-canvas.tsx

@@ -18,6 +18,7 @@ import {
   type NodeProps,
 } from "@xyflow/react";
 import "@xyflow/react/dist/style.css";
+import { khotanFetch, ApiErrorState } from "./api-state";
 import { Badge } from "@/components/ui/badge";
 import {
   Card,
@@ -1056,8 +1057,9 @@ function TopologyCanvasInner() {
   const [nodes, setNodes, onNodesChange] = useNodesState<TopologyNodeData>([]);
   const [edges, setEdges, onEdgesChange] = useEdgesState([]);
   const [loading, setLoading] = useState(true);
-  const [error, setError] = useState<string | null>(null);
+  const [error, setError] = useState<unknown>(null);
   const [lastUpdatedAt, setLastUpdatedAt] = useState<string | null>(null);
+  const [refreshKey, setRefreshKey] = useState(0);
 
   useEffect(() => {
     let cancelled = false;
@@ -1068,30 +1070,22 @@ function TopologyCanvasInner() {
           setError(null);
         }
 
-        const [plugsRes, flowsRes, runsRes] = await Promise.all([
-          fetch("/api/khotan/plugs"),
-          fetch("/api/khotan/flows"),
-          fetch("/api/khotan/runs?limit=100"),
+        const [plugs, flows, runsRaw] = await Promise.all([
+          khotanFetch<PlugRecord[]>("/api/khotan/plugs"),
+          khotanFetch<FlowRecord[]>("/api/khotan/flows"),
+          khotanFetch<unknown>("/api/khotan/runs?limit=100"),
         ]);
 
-        if (!plugsRes.ok || !flowsRes.ok || !runsRes.ok) {
-          throw new Error("Failed to load topology data from /api/khotan");
-        }
-
-        const plugs = (await plugsRes.json()) as PlugRecord[];
-        const flows = (await flowsRes.json()) as FlowRecord[];
-        const runs = normalizeRuns(await runsRes.json());
+        const runs = normalizeRuns(runsRaw);
 
         const webhookGroups = await Promise.all(
           plugs.map(async (plug) => {
             try {
-              const res = await fetch(
+              const handlers = await khotanFetch<
+                Array<Omit<WebhookHandlerRecord, "plugName">>
+              >(
                 `/api/khotan/webhook-handlers/${encodeURIComponent(plug.name)}`,
               );
-              if (!res.ok) return [] as WebhookHandlerRecord[];
-              const handlers = (await res.json()) as Array<
-                Omit<WebhookHandlerRecord, "plugName">
-              >;
               return handlers.map((handler) => ({
                 ...handler,
                 plugName: plug.name,
@@ -1113,11 +1107,7 @@ function TopologyCanvasInner() {
         setLastUpdatedAt(new Date().toISOString());
       } catch (loadError) {
         if (!cancelled) {
-          setError(
-            loadError instanceof Error
-              ? loadError.message
-              : "Unknown topology load failure",
-          );
+          setError(loadError);
         }
       } finally {
         if (!cancelled) {
@@ -1135,7 +1125,7 @@ function TopologyCanvasInner() {
       cancelled = true;
       window.clearInterval(interval);
     };
-  }, []);
+  }, [refreshKey]);
 
   const model = useMemo(() => {
     return snapshot ? buildTopologyModel(snapshot) : null;
@@ -1180,19 +1170,18 @@ function TopologyCanvasInner() {
 
   if (error) {
     return (
-      <Card className="border-red-200 bg-white/80 shadow-xl backdrop-blur">
+      <Card className="border-white/70 bg-white/80 shadow-xl backdrop-blur">
         <CardHeader>
           <CardTitle>Topology Canvas</CardTitle>
           <CardDescription>
             Could not load the graph from the local Khotan API.
           </CardDescription>
         </CardHeader>
-        <CardContent className="space-y-3 text-sm">
-          <p className="text-red-600">{error}</p>
-          <p className="text-slate-500">
-            Make sure the catch-all Khotan route is mounted and your dev server
-            is running.
-          </p>
+        <CardContent>
+          <ApiErrorState
+            error={error}
+            onRetry={() => setRefreshKey((v) => v + 1)}
+          />
         </CardContent>
       </Card>
     );