khotan-data 0.1.0 → 0.2.0

package/dist/templates/mappings-page.tsx

@@ -0,0 +1,9 @@
+import { KhotanMappingBrowser } from "@/components/khotan/mapping-browser";
+
+export default function KhotanMappingsPage() {
+  return (
+    <main className="container mx-auto max-w-7xl px-4 py-10">
+      <KhotanMappingBrowser />
+    </main>
+  );
+}

package/dist/templates/outflow.ts

@@ -5,6 +5,8 @@
 // This file defines the outflow() builder and types. Create per-service flow
 // files (e.g. crm-audiences.ts) using this builder to read app data and push it
 // to an external service with durable, retryable Vercel Workflow steps.
+// Outflow workflows can also use khotanCache(ctx, "name") for checkpoints, cursor
+// state, or dedupe markers between runs.
 // ============================================================================
 
 import type {
@@ -46,7 +48,7 @@ export function outflow(config: OutflowConfig): FlowRegistration {
 // Usage Example (create a file like flows/hubspot-products.ts)
 // ---------------------------------------------------------------------------
 //
-// import { outflow, type OutflowContext } from "./outflow";
+// import { bindWorkflowPlug, outflow, type OutflowContext } from "khotan-data/factory";
 // import { db } from "@/db";
 // import { products } from "@/db/schema";
 // import { hubspotPlug } from "../plugs/hubspot";
@@ -61,14 +63,12 @@ export function outflow(config: OutflowConfig): FlowRegistration {
 //       khotanRunId: ctx.khotanRunId,
 //       runType: ctx.runType,
 //     });
+//     const hubspot = bindWorkflowPlug(hubspotPlug, ctx);
 //
 //     const records = await db.select().from(products);
 //
 //     for (const record of records) {
-//       await hubspotPlug.post("/products", {
-//         vars: ctx.vars,
-//         body: record,
-//       });
+//       await hubspot.post("/products", { body: record });
 //     }
 //
 //     return {

package/dist/templates/pass.ts

@@ -26,6 +26,8 @@ export interface PassContext {
   destVars: Record<string, string>;
   /** Khotan run ID created for this webhook handler execution */
   khotanRunId: string;
+  /** Internal Khotan instance identifier for helper APIs */
+  khotanInstanceId: string;
 }
 
 // ---------------------------------------------------------------------------
@@ -79,6 +81,7 @@ export function pass(config: PassConfig): PassRegistration {
 // Usage Example (create a file like webhooks/pollinate-to-slack.ts)
 // ---------------------------------------------------------------------------
 //
+// import { khotanCache } from "khotan-data/factory";
 // import { pass, type PassContext } from "./pass";
 // import { plug } from "../plugs/plug";
 //
@@ -87,6 +90,9 @@ export function pass(config: PassConfig): PassRegistration {
 //
 //   async function forwardEvent() {
 //     "use step";
+//     const cache = khotanCache(ctx, "pollinate-forwarded-events");
+//     const eventId = String(ctx.event["id"] ?? "");
+//     if (eventId && (await cache.get<boolean>(eventId))) return;
 //
 //     // Construct destination plug from destVars
 //     const slackPlug = plug({
@@ -102,6 +108,10 @@ export function pass(config: PassConfig): PassRegistration {
 //         event: ctx.event,
 //       },
 //     });
+//
+//     if (eventId) {
+//       await cache.set(eventId, true);
+//     }
 //   }
 //
 //   await forwardEvent();

package/dist/templates/plug-debugger.tsx

@@ -5,6 +5,7 @@ import { Badge } from "@/components/ui/badge";
 import { Button } from "@/components/ui/button";
 import { Input } from "@/components/ui/input";
 import { Label } from "@/components/ui/label";
+import { khotanFetch, isKhotanApiError, ApiErrorState } from "./api-state";
 
 // ============================================================================
 // Plug Debugger — Lightweight Postman for your plugs
@@ -331,6 +332,7 @@ export function PlugDebugger({
 }: PlugDebuggerProps) {
   const [meta, setMeta] = useState<PlugMeta | null>(null);
   const [loading, setLoading] = useState(true);
+  const [metaError, setMetaError] = useState<unknown>(null);
 
   const [method, setMethod] = useState<string>("GET");
   const [path, setPath] = useState("");
@@ -353,15 +355,17 @@ export function PlugDebugger({
   const containerRef = useRef<HTMLDivElement>(null);
 
   const fetchMeta = useCallback(async () => {
+    setLoading(true);
+    setMetaError(null);
     try {
-      const res = await fetch(`${basePath}/debug/${plugName}`);
-      if (!res.ok) {
-        setMeta(null);
-        return;
-      }
-      setMeta(await res.json());
-    } catch {
+      setMeta(await khotanFetch<PlugMeta>(`${basePath}/debug/${plugName}`));
+    } catch (err) {
       setMeta(null);
+      // A 404 means debug is off or the plug isn't registered — handled by the
+      // dedicated "not available" message below. Surface everything else.
+      if (!(isKhotanApiError(err) && err.status === 404)) {
+        setMetaError(err);
+      }
     } finally {
       setLoading(false);
     }
@@ -401,6 +405,10 @@ export function PlugDebugger({
     );
   }
 
+  if (metaError) {
+    return <ApiErrorState error={metaError} onRetry={() => void fetchMeta()} />;
+  }
+
   if (!meta) {
     return (
       <div className="rounded-lg border border-border p-6 text-center">

package/dist/templates/relay.example.ts

@@ -6,6 +6,7 @@
 // exported flow in {outputDir}/khotan.ts.
 // ============================================================================
 
+import { khotanCache } from "khotan-data/factory";
 import { relay, type RelayContext } from "./relay";
 
 async function shopifyToHubspotWorkflow(ctx: RelayContext) {
@@ -29,6 +30,11 @@ async function shopifyToHubspotWorkflow(ctx: RelayContext) {
       data?: Array<Record<string, unknown>>;
     };
     const records = Array.isArray(payload.data) ? payload.data : [];
+    const snapshotCache = khotanCache(ctx, "shopify-products-snapshot");
+    const previousRecords =
+      (await snapshotCache.get<Array<Record<string, unknown>>>("latest")) ?? [];
+
+    await snapshotCache.set("latest", records, { ttl: "6h" });
 
     for (const record of records) {
       await fetch("https://destination.example.com/products", {
@@ -45,7 +51,11 @@ async function shopifyToHubspotWorkflow(ctx: RelayContext) {
       extracted: records.length,
       transformed: records.length,
       created: records.length,
-      metadata: { relay: ctx.flow.name, to: ctx.flow.to },
+      metadata: {
+        relay: ctx.flow.name,
+        to: ctx.flow.to,
+        previousCount: previousRecords.length,
+      },
     };
   }
 

package/dist/templates/relay.ts

@@ -5,7 +5,8 @@
 // This file defines the relay() builder and types. Create per-service flow
 // files (e.g. shopify-to-hubspot.ts) using this builder to read from the source
 // plug and forward to a destination system with durable, retryable Vercel
-// Workflow steps.
+// Workflow steps. Relay workflows can also use khotanCache(ctx, "name") for durable
+// snapshots, checkpoints, and dedupe state between runs.
 // ============================================================================
 
 import type {
@@ -50,7 +51,7 @@ export function relay(config: RelayConfig): FlowRegistration {
 // Usage Example (create a file like flows/shopify-to-hubspot.ts)
 // ---------------------------------------------------------------------------
 //
-// import { relay, type RelayContext } from "./relay";
+// import { bindWorkflowPlug, khotanCache, relay, type RelayContext } from "khotan-data/factory";
 // import { shopifyPlug } from "../plugs/shopify";
 // import { hubspotPlug } from "../plugs/hubspot";
 //
@@ -65,21 +66,29 @@ export function relay(config: RelayConfig): FlowRegistration {
 //       khotanRunId: ctx.khotanRunId,
 //       runType: ctx.runType,
 //     });
+//     const shopify = bindWorkflowPlug(shopifyPlug, ctx);
+//     const hubspot = bindWorkflowPlug(hubspotPlug, ctx, "hubspot");
 //
-//     const response = await shopifyPlug.get<{ data?: Array<Record<string, unknown>> }>("/products", {
-//       vars: ctx.vars,
-//     });
+//     const snapshotCache = khotanCache(ctx, "shopify-products-snapshot");
+//     const previous = await snapshotCache.get<Array<Record<string, unknown>>>("latest");
+//
+//     const response = await shopify.get<{ data?: Array<Record<string, unknown>> }>("/products");
 //     const records = Array.isArray(response.data) ? response.data : [];
+//     await snapshotCache.set("latest", records);
 //
 //     for (const record of records) {
-//       await hubspotPlug.post("/products", { body: record });
+//       await hubspot.post("/products", { body: record });
 //     }
 //
 //     return {
 //       extracted: records.length,
 //       transformed: records.length,
 //       created: records.length,
-//       metadata: { relay: ctx.flow.name, to: ctx.flow.to },
+//       metadata: {
+//         relay: ctx.flow.name,
+//         to: ctx.flow.to,
+//         previousCount: previous?.length ?? 0,
+//       },
 //     };
 //   }
 //

package/dist/templates/runs-table.tsx

@@ -2,6 +2,7 @@
 
 import { Fragment, useCallback, useEffect, useState } from "react";
 import { RefreshCw } from "lucide-react";
+import { khotanFetch, ApiErrorState } from "./api-state";
 import { Badge } from "@/components/ui/badge";
 import { Button } from "@/components/ui/button";
 import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card";
@@ -359,7 +360,7 @@ export function KhotanRunsTable({ pageSize = 10 }: { pageSize?: number } = {}) {
   const [data, setData] = useState<PageResponse<RunLogItem> | null>(null);
   const [offset, setOffset] = useState(0);
   const [loading, setLoading] = useState(true);
-  const [error, setError] = useState<string | null>(null);
+  const [error, setError] = useState<unknown>(null);
   const [refreshKey, setRefreshKey] = useState(0);
   const [expandedRunId, setExpandedRunId] = useState<string | null>(null);
   const [streamingEnabled, setStreamingEnabled] = useState(false);
@@ -378,20 +379,16 @@ export function KhotanRunsTable({ pageSize = 10 }: { pageSize?: number } = {}) {
       setLoading(true);
       setError(null);
       try {
-        const res = await fetch(
+        const json = await khotanFetch<PageResponse<RunLogItem>>(
           `/api/khotan/runs?limit=${String(pageSize)}&offset=${String(offset)}`,
         );
-        if (!res.ok) {
-          throw new Error("Failed to load runs");
-        }
-        const json = (await res.json()) as PageResponse<RunLogItem>;
         if (!cancelled) {
           setData(json);
           setLastUpdatedAt(new Date().toISOString());
         }
       } catch (err) {
         if (!cancelled) {
-          setError(err instanceof Error ? err.message : "Unknown error");
+          setError(err);
         }
       } finally {
         if (!cancelled) {
@@ -454,138 +451,144 @@ export function KhotanRunsTable({ pageSize = 10 }: { pageSize?: number } = {}) {
       </CardHeader>
       <CardContent className="space-y-4">
         {error ? (
-          <div className="rounded-md border border-destructive/30 bg-destructive/5 p-3 text-sm text-destructive">
-            {error}
-          </div>
+          <ApiErrorState
+            error={error}
+            onRetry={() => setRefreshKey((v) => v + 1)}
+            compact
+          />
         ) : null}
 
-        <Table>
-          <TableHeader>
-            <TableRow>
-              <TableHead>Started</TableHead>
-              <TableHead>Status</TableHead>
-              <TableHead>Source</TableHead>
-              <TableHead>Plug</TableHead>
-              <TableHead>Run Type</TableHead>
-              <TableHead>Counts</TableHead>
-              <TableHead>Workflow</TableHead>
-              <TableHead />
-            </TableRow>
-          </TableHeader>
-          <TableBody>
-            {loading ? (
+        {error ? null : (
+          <Table>
+            <TableHeader>
               <TableRow>
-                <TableCell
-                  colSpan={8}
-                  className="text-sm text-muted-foreground"
-                >
-                  Loading runs...
-                </TableCell>
+                <TableHead>Started</TableHead>
+                <TableHead>Status</TableHead>
+                <TableHead>Source</TableHead>
+                <TableHead>Plug</TableHead>
+                <TableHead>Run Type</TableHead>
+                <TableHead>Counts</TableHead>
+                <TableHead>Workflow</TableHead>
+                <TableHead />
               </TableRow>
-            ) : data?.items.length ? (
-              data.items.map((item) => (
-                <Fragment key={item.id}>
-                  <TableRow>
-                    <TableCell className="text-sm text-muted-foreground">
-                      <div>{formatDateTime(item.startedAt)}</div>
-                      <div className="text-xs">
-                        {item.completedAt
-                          ? `completed ${formatDateTime(item.completedAt)}`
-                          : "in progress"}
-                      </div>
-                    </TableCell>
-                    <TableCell>
-                      <Badge variant={statusVariant[item.status]}>
-                        {statusLabel[item.status]}
-                      </Badge>
-                      {item.error ? (
-                        <div
-                          className="mt-1 max-w-56 truncate text-xs text-destructive"
-                          title={item.error}
-                        >
-                          {item.error}
-                        </div>
-                      ) : null}
-                    </TableCell>
-                    <TableCell className="font-medium">
-                      {formatSource(item)}
-                    </TableCell>
-                    <TableCell className="text-muted-foreground">
-                      {item.plugName ?? "-"}
-                    </TableCell>
-                    <TableCell className="font-mono text-xs">
-                      {item.runType}
-                    </TableCell>
-                    <TableCell className="max-w-64 text-xs text-muted-foreground">
-                      {formatCounts(item)}
-                    </TableCell>
-                    <TableCell className="font-mono text-xs text-muted-foreground">
-                      {item.workflowRunId ?? "-"}
-                    </TableCell>
-                    <TableCell className="text-right">
-                      <Button
-                        variant="outline"
-                        size="sm"
-                        onClick={() =>
-                          setExpandedRunId((current) =>
-                            current === item.id ? null : item.id,
-                          )
-                        }
-                      >
-                        {expandedRunId === item.id ? "Hide" : "Details"}
-                      </Button>
-                    </TableCell>
-                  </TableRow>
-                  {expandedRunId === item.id ? (
+            </TableHeader>
+            <TableBody>
+              {loading ? (
+                <TableRow>
+                  <TableCell
+                    colSpan={8}
+                    className="text-sm text-muted-foreground"
+                  >
+                    Loading runs...
+                  </TableCell>
+                </TableRow>
+              ) : data?.items.length ? (
+                data.items.map((item) => (
+                  <Fragment key={item.id}>
                     <TableRow>
-                      <TableCell colSpan={8}>
-                        <RunDetails
-                          run={item}
-                          streamingEnabled={streamingEnabled}
-                          onChanged={() => setRefreshKey((v) => v + 1)}
-                          onStreamInbound={pulseLiveIndicator}
-                        />
+                      <TableCell className="text-sm text-muted-foreground">
+                        <div>{formatDateTime(item.startedAt)}</div>
+                        <div className="text-xs">
+                          {item.completedAt
+                            ? `completed ${formatDateTime(item.completedAt)}`
+                            : "in progress"}
+                        </div>
+                      </TableCell>
+                      <TableCell>
+                        <Badge variant={statusVariant[item.status]}>
+                          {statusLabel[item.status]}
+                        </Badge>
+                        {item.error ? (
+                          <div
+                            className="mt-1 max-w-56 truncate text-xs text-destructive"
+                            title={item.error}
+                          >
+                            {item.error}
+                          </div>
+                        ) : null}
+                      </TableCell>
+                      <TableCell className="font-medium">
+                        {formatSource(item)}
+                      </TableCell>
+                      <TableCell className="text-muted-foreground">
+                        {item.plugName ?? "-"}
+                      </TableCell>
+                      <TableCell className="font-mono text-xs">
+                        {item.runType}
+                      </TableCell>
+                      <TableCell className="max-w-64 text-xs text-muted-foreground">
+                        {formatCounts(item)}
+                      </TableCell>
+                      <TableCell className="font-mono text-xs text-muted-foreground">
+                        {item.workflowRunId ?? "-"}
+                      </TableCell>
+                      <TableCell className="text-right">
+                        <Button
+                          variant="outline"
+                          size="sm"
+                          onClick={() =>
+                            setExpandedRunId((current) =>
+                              current === item.id ? null : item.id,
+                            )
+                          }
+                        >
+                          {expandedRunId === item.id ? "Hide" : "Details"}
+                        </Button>
                       </TableCell>
                     </TableRow>
-                  ) : null}
-                </Fragment>
-              ))
-            ) : (
-              <TableRow>
-                <TableCell
-                  colSpan={8}
-                  className="text-sm text-muted-foreground"
-                >
-                  No runs recorded yet.
-                </TableCell>
-              </TableRow>
-            )}
-          </TableBody>
-        </Table>
+                    {expandedRunId === item.id ? (
+                      <TableRow>
+                        <TableCell colSpan={8}>
+                          <RunDetails
+                            run={item}
+                            streamingEnabled={streamingEnabled}
+                            onChanged={() => setRefreshKey((v) => v + 1)}
+                            onStreamInbound={pulseLiveIndicator}
+                          />
+                        </TableCell>
+                      </TableRow>
+                    ) : null}
+                  </Fragment>
+                ))
+              ) : (
+                <TableRow>
+                  <TableCell
+                    colSpan={8}
+                    className="text-sm text-muted-foreground"
+                  >
+                    No runs recorded yet.
+                  </TableCell>
+                </TableRow>
+              )}
+            </TableBody>
+          </Table>
+        )}
 
-        <div className="flex items-center justify-between gap-3">
-          <p className="text-sm text-muted-foreground">
-            Page {Math.floor(offset / pageSize) + 1}
-          </p>
-          <div className="flex items-center gap-2">
-            <Button
-              variant="outline"
-              size="sm"
-              disabled={offset === 0 || loading}
-              onClick={() => setOffset(Math.max(offset - pageSize, 0))}
-            >
-              Previous
-            </Button>
-            <Button
-              variant="outline"
-              size="sm"
-              disabled={!data?.page.hasMore || loading}
-              onClick={() => setOffset(offset + pageSize)}
-            >
-              Next
-            </Button>
+        {error ? null : (
+          <div className="flex items-center justify-between gap-3">
+            <p className="text-sm text-muted-foreground">
+              Page {Math.floor(offset / pageSize) + 1}
+            </p>
+            <div className="flex items-center gap-2">
+              <Button
+                variant="outline"
+                size="sm"
+                disabled={offset === 0 || loading}
+                onClick={() => setOffset(Math.max(offset - pageSize, 0))}
+              >
+                Previous
+              </Button>
+              <Button
+                variant="outline"
+                size="sm"
+                disabled={!data?.page.hasMore || loading}
+                onClick={() => setOffset(offset + pageSize)}
+              >
+                Next
+              </Button>
+            </div>
           </div>
-        </div>
+        )}
       </CardContent>
     </Card>
   );

package/dist/templates/schema.ts

@@ -303,6 +303,67 @@ export const khotanMappings = pgTable(
   ],
 );
 
+// ---------------------------------------------------------------------------
+// khotan_caches — one row per registered durable cache namespace
+// ---------------------------------------------------------------------------
+
+export const khotanCaches = pgTable(
+  "khotan_caches",
+  {
+    id: text("id")
+      .primaryKey()
+      .$defaultFn(() => crypto.randomUUID()),
+    name: text("name").notNull().unique(),
+    scope: jsonb("scope").$type<{
+      plug?: string;
+      resource?: string;
+      flow?: string;
+    }>(),
+    ttlSeconds: integer("ttl_seconds"),
+    createdAt: timestamp("created_at", { withTimezone: true })
+      .defaultNow()
+      .notNull(),
+    updatedAt: timestamp("updated_at", { withTimezone: true })
+      .defaultNow()
+      .notNull(),
+  },
+  (table) => [index("khotan_caches_name_idx").on(table.name)],
+);
+
+// ---------------------------------------------------------------------------
+// khotan_cache_entries — latest-value durable cache rows keyed by cache + key
+// ---------------------------------------------------------------------------
+
+export const khotanCacheEntries = pgTable(
+  "khotan_cache_entries",
+  {
+    id: text("id")
+      .primaryKey()
+      .$defaultFn(() => crypto.randomUUID()),
+    cacheId: text("cache_id")
+      .notNull()
+      .references(() => khotanCaches.id),
+    key: text("key").notNull(),
+    value: jsonb("value").notNull(),
+    expiresAt: timestamp("expires_at", { withTimezone: true }),
+    createdAt: timestamp("created_at", { withTimezone: true })
+      .defaultNow()
+      .notNull(),
+    updatedAt: timestamp("updated_at", { withTimezone: true })
+      .defaultNow()
+      .notNull(),
+  },
+  (table) => [
+    unique("khotan_cache_entries_cache_id_key_unique").on(
+      table.cacheId,
+      table.key,
+    ),
+    index("khotan_cache_entries_cache_id_idx").on(table.cacheId),
+    index("khotan_cache_entries_cache_id_key_idx").on(table.cacheId, table.key),
+    index("khotan_cache_entries_expires_at_idx").on(table.expiresAt),
+  ],
+);
+
 // ---------------------------------------------------------------------------
 // Relations
 // ---------------------------------------------------------------------------
@@ -395,6 +456,20 @@ export const khotanMappingsRelations = relations(khotanMappings, ({ one }) => ({
   }),
 }));
 
+export const khotanCachesRelations = relations(khotanCaches, ({ many }) => ({
+  entries: many(khotanCacheEntries),
+}));
+
+export const khotanCacheEntriesRelations = relations(
+  khotanCacheEntries,
+  ({ one }) => ({
+    cache: one(khotanCaches, {
+      fields: [khotanCacheEntries.cacheId],
+      references: [khotanCaches.id],
+    }),
+  }),
+);
+
 // ---------------------------------------------------------------------------
 // Type helpers
 // ---------------------------------------------------------------------------
@@ -422,3 +497,9 @@ export type NewKhotanResource = typeof khotanResources.$inferInsert;
 
 export type KhotanMapping = typeof khotanMappings.$inferSelect;
 export type NewKhotanMapping = typeof khotanMappings.$inferInsert;
+
+export type KhotanCache = typeof khotanCaches.$inferSelect;
+export type NewKhotanCache = typeof khotanCaches.$inferInsert;
+
+export type KhotanCacheEntry = typeof khotanCacheEntries.$inferSelect;
+export type NewKhotanCacheEntry = typeof khotanCacheEntries.$inferInsert;