khotan-data 0.0.1 → 0.1.1

package/dist/templates/schema.ts

@@ -0,0 +1,505 @@
+// ============================================================================
+// Schema — Drizzle table definitions for khotan plugs, flows, and runs
+// Generated by khotan CLI · https://github.com/khotan-data
+//
+// This file is yours. Edit anything — column types, defaults, indexes,
+// relations. It has zero runtime dependencies on khotan-data.
+//
+// Re-export from your Drizzle schema barrel file:
+//   export * from "@/lib/khotan/schema";
+// ============================================================================
+
+import { relations } from "drizzle-orm";
+import {
+  boolean,
+  index,
+  integer,
+  jsonb,
+  pgTable,
+  text,
+  timestamp,
+  unique,
+} from "drizzle-orm/pg-core";
+
+// ---------------------------------------------------------------------------
+// khotan_plugs — one row per configured external service connection
+// ---------------------------------------------------------------------------
+
+export const khotanPlugs = pgTable("khotan_plugs", {
+  id: text("id")
+    .primaryKey()
+    .$defaultFn(() => crypto.randomUUID()),
+  name: text("name").notNull().unique(),
+  baseUrl: text("base_url").notNull(),
+  authType: text("auth_type", {
+    enum: ["bearer", "basic", "apiKey", "custom"],
+  }).notNull(),
+  enabled: boolean("enabled").default(true).notNull(),
+  status: text("status", {
+    enum: ["connected", "error", "idle"],
+  })
+    .default("idle")
+    .notNull(),
+  statusMessage: text("status_message"),
+  encryptedVars: text("encrypted_vars"),
+  createdAt: timestamp("created_at", { withTimezone: true })
+    .defaultNow()
+    .notNull(),
+  updatedAt: timestamp("updated_at", { withTimezone: true })
+    .defaultNow()
+    .notNull(),
+});
+
+// ---------------------------------------------------------------------------
+// khotan_resources — one row per logical entity type (e.g. products, orders)
+// ---------------------------------------------------------------------------
+
+export const khotanResources = pgTable("khotan_resources", {
+  id: text("id")
+    .primaryKey()
+    .$defaultFn(() => crypto.randomUUID()),
+  name: text("name").notNull().unique(),
+  connectField: text("connect_field").notNull(),
+  description: text("description"),
+  createdAt: timestamp("created_at", { withTimezone: true })
+    .defaultNow()
+    .notNull(),
+  updatedAt: timestamp("updated_at", { withTimezone: true })
+    .defaultNow()
+    .notNull(),
+});
+
+// ---------------------------------------------------------------------------
+// khotan_flows — one row per data flow tied to a plug
+// ---------------------------------------------------------------------------
+
+export const khotanFlows = pgTable(
+  "khotan_flows",
+  {
+    id: text("id")
+      .primaryKey()
+      .$defaultFn(() => crypto.randomUUID()),
+    plugId: text("plug_id")
+      .notNull()
+      .references(() => khotanPlugs.id),
+    name: text("name").notNull(),
+    type: text("type", {
+      enum: ["inflow", "outflow", "relay", "webhook"],
+    }).notNull(),
+    enabled: boolean("enabled").default(true).notNull(),
+    schedule: text("schedule"),
+    resourceId: text("resource_id").references(() => khotanResources.id),
+    lastRunAt: timestamp("last_run_at", { withTimezone: true }),
+    lastRunStatus: text("last_run_status", {
+      enum: ["completed", "partial", "failed", "cancelled"],
+    }),
+    createdAt: timestamp("created_at", { withTimezone: true })
+      .defaultNow()
+      .notNull(),
+    updatedAt: timestamp("updated_at", { withTimezone: true })
+      .defaultNow()
+      .notNull(),
+  },
+  (table) => [
+    unique("khotan_flows_plug_id_name_unique").on(table.plugId, table.name),
+    index("khotan_flows_plug_id_idx").on(table.plugId),
+    index("khotan_flows_resource_id_idx").on(table.resourceId),
+  ],
+);
+
+// ---------------------------------------------------------------------------
+// khotan_wires — one row per webhook subscription managed by a plug
+// ---------------------------------------------------------------------------
+
+export const khotanWires = pgTable(
+  "khotan_wires",
+  {
+    id: text("id")
+      .primaryKey()
+      .$defaultFn(() => crypto.randomUUID()),
+    plugId: text("plug_id")
+      .notNull()
+      .references(() => khotanPlugs.id),
+    remoteId: text("remote_id").notNull(),
+    callbackUrl: text("callback_url").notNull(),
+    eventTypes: jsonb("event_types").notNull().$type<string[]>(),
+    status: text("status", {
+      enum: ["active", "disabled"],
+    })
+      .default("active")
+      .notNull(),
+    metadata: jsonb("metadata"),
+    createdAt: timestamp("created_at", { withTimezone: true })
+      .defaultNow()
+      .notNull(),
+    updatedAt: timestamp("updated_at", { withTimezone: true })
+      .defaultNow()
+      .notNull(),
+  },
+  (table) => [
+    index("khotan_wires_plug_id_idx").on(table.plugId),
+    index("khotan_wires_status_idx").on(table.status),
+  ],
+);
+
+// ---------------------------------------------------------------------------
+// khotan_webhook_handlers — catches and passes attached to a wire
+// ---------------------------------------------------------------------------
+
+export const khotanWebhookHandlers = pgTable(
+  "khotan_webhook_handlers",
+  {
+    id: text("id")
+      .primaryKey()
+      .$defaultFn(() => crypto.randomUUID()),
+    wireId: text("wire_id")
+      .notNull()
+      .references(() => khotanWires.id),
+    name: text("name").notNull(),
+    type: text("type", {
+      enum: ["catch", "pass"],
+    }).notNull(),
+    destinationPlugId: text("destination_plug_id"),
+    enabled: boolean("enabled").default(true).notNull(),
+    createdAt: timestamp("created_at", { withTimezone: true })
+      .defaultNow()
+      .notNull(),
+    updatedAt: timestamp("updated_at", { withTimezone: true })
+      .defaultNow()
+      .notNull(),
+  },
+  (table) => [
+    unique("khotan_webhook_handlers_wire_id_name_unique").on(
+      table.wireId,
+      table.name,
+    ),
+    index("khotan_webhook_handlers_wire_id_idx").on(table.wireId),
+  ],
+);
+
+// ---------------------------------------------------------------------------
+// khotan_webhook_events — one row per webhook event routed to a handler
+// ---------------------------------------------------------------------------
+
+export const khotanWebhookEvents = pgTable(
+  "khotan_webhook_events",
+  {
+    id: text("id")
+      .primaryKey()
+      .$defaultFn(() => crypto.randomUUID()),
+    wireId: text("wire_id")
+      .notNull()
+      .references(() => khotanWires.id),
+    webhookHandlerId: text("webhook_handler_id")
+      .notNull()
+      .references(() => khotanWebhookHandlers.id),
+    khotanRunId: text("khotan_run_id")
+      .notNull()
+      .references(() => khotanRuns.id),
+    eventType: text("event_type").notNull(),
+    payload: jsonb("payload").notNull().$type<Record<string, unknown>>(),
+    headers: jsonb("headers").notNull().$type<Record<string, string>>(),
+    receivedAt: timestamp("received_at", { withTimezone: true })
+      .defaultNow()
+      .notNull(),
+  },
+  (table) => [
+    index("khotan_webhook_events_wire_id_idx").on(table.wireId),
+    index("khotan_webhook_events_webhook_handler_id_idx").on(
+      table.webhookHandlerId,
+    ),
+    index("khotan_webhook_events_khotan_run_id_idx").on(table.khotanRunId),
+    index("khotan_webhook_events_received_at_idx").on(table.receivedAt.desc()),
+  ],
+);
+
+// ---------------------------------------------------------------------------
+// khotan_runs — one row per execution of a flow, wire, or webhook handler
+// ---------------------------------------------------------------------------
+
+export const khotanRuns = pgTable(
+  "khotan_runs",
+  {
+    id: text("id")
+      .primaryKey()
+      .$defaultFn(() => crypto.randomUUID()),
+    flowId: text("flow_id").references(() => khotanFlows.id),
+    wireId: text("wire_id").references(() => khotanWires.id),
+    webhookHandlerId: text("webhook_handler_id").references(
+      () => khotanWebhookHandlers.id,
+    ),
+    workflowRunId: text("workflow_run_id"),
+    runType: text("run_type", {
+      enum: ["full", "delta", "backfill", "reconcile", "dry-run", "webhook"],
+    }).notNull(),
+    status: text("status", {
+      enum: [
+        "pending",
+        "running",
+        "completed",
+        "partial",
+        "failed",
+        "cancelled",
+      ],
+    })
+      .default("pending")
+      .notNull(),
+    startedAt: timestamp("started_at", { withTimezone: true })
+      .defaultNow()
+      .notNull(),
+    completedAt: timestamp("completed_at", { withTimezone: true }),
+    durationMs: integer("duration_ms"),
+    extracted: integer("extracted").default(0).notNull(),
+    transformed: integer("transformed").default(0).notNull(),
+    created: integer("created").default(0).notNull(),
+    updated: integer("updated").default(0).notNull(),
+    deleted: integer("deleted").default(0).notNull(),
+    failed: integer("failed").default(0).notNull(),
+    error: text("error"),
+    metadata: jsonb("metadata"),
+  },
+  (table) => [
+    index("khotan_runs_flow_id_idx").on(table.flowId),
+    index("khotan_runs_wire_id_idx").on(table.wireId),
+    index("khotan_runs_webhook_handler_id_idx").on(table.webhookHandlerId),
+    index("khotan_runs_status_idx").on(table.status),
+    index("khotan_runs_flow_id_started_at_idx").on(
+      table.flowId,
+      table.startedAt.desc(),
+    ),
+  ],
+);
+
+// ---------------------------------------------------------------------------
+// khotan_mappings — one row per entity instance within a resource
+// ---------------------------------------------------------------------------
+
+export const khotanMappings = pgTable(
+  "khotan_mappings",
+  {
+    id: text("id")
+      .primaryKey()
+      .$defaultFn(() => crypto.randomUUID()),
+    resourceId: text("resource_id")
+      .notNull()
+      .references(() => khotanResources.id),
+    connectValue: text("connect_value").notNull(),
+    refs: jsonb("refs").notNull().$type<Record<string, string>>().default({}),
+    metadata: jsonb("metadata").$type<Record<string, unknown>>(),
+    createdAt: timestamp("created_at", { withTimezone: true })
+      .defaultNow()
+      .notNull(),
+    updatedAt: timestamp("updated_at", { withTimezone: true })
+      .defaultNow()
+      .notNull(),
+  },
+  (table) => [
+    unique("khotan_mappings_resource_id_connect_value_unique").on(
+      table.resourceId,
+      table.connectValue,
+    ),
+    index("khotan_mappings_resource_id_idx").on(table.resourceId),
+    index("khotan_mappings_refs_gin_idx").using("gin", table.refs),
+  ],
+);
+
+// ---------------------------------------------------------------------------
+// khotan_caches — one row per registered durable cache namespace
+// ---------------------------------------------------------------------------
+
+export const khotanCaches = pgTable(
+  "khotan_caches",
+  {
+    id: text("id")
+      .primaryKey()
+      .$defaultFn(() => crypto.randomUUID()),
+    name: text("name").notNull().unique(),
+    scope: jsonb("scope").$type<{
+      plug?: string;
+      resource?: string;
+      flow?: string;
+    }>(),
+    ttlSeconds: integer("ttl_seconds"),
+    createdAt: timestamp("created_at", { withTimezone: true })
+      .defaultNow()
+      .notNull(),
+    updatedAt: timestamp("updated_at", { withTimezone: true })
+      .defaultNow()
+      .notNull(),
+  },
+  (table) => [index("khotan_caches_name_idx").on(table.name)],
+);
+
+// ---------------------------------------------------------------------------
+// khotan_cache_entries — latest-value durable cache rows keyed by cache + key
+// ---------------------------------------------------------------------------
+
+export const khotanCacheEntries = pgTable(
+  "khotan_cache_entries",
+  {
+    id: text("id")
+      .primaryKey()
+      .$defaultFn(() => crypto.randomUUID()),
+    cacheId: text("cache_id")
+      .notNull()
+      .references(() => khotanCaches.id),
+    key: text("key").notNull(),
+    value: jsonb("value").notNull(),
+    expiresAt: timestamp("expires_at", { withTimezone: true }),
+    createdAt: timestamp("created_at", { withTimezone: true })
+      .defaultNow()
+      .notNull(),
+    updatedAt: timestamp("updated_at", { withTimezone: true })
+      .defaultNow()
+      .notNull(),
+  },
+  (table) => [
+    unique("khotan_cache_entries_cache_id_key_unique").on(
+      table.cacheId,
+      table.key,
+    ),
+    index("khotan_cache_entries_cache_id_idx").on(table.cacheId),
+    index("khotan_cache_entries_cache_id_key_idx").on(table.cacheId, table.key),
+    index("khotan_cache_entries_expires_at_idx").on(table.expiresAt),
+  ],
+);
+
+// ---------------------------------------------------------------------------
+// Relations
+// ---------------------------------------------------------------------------
+
+export const khotanPlugsRelations = relations(khotanPlugs, ({ many }) => ({
+  flows: many(khotanFlows),
+  wires: many(khotanWires),
+}));
+
+export const khotanFlowsRelations = relations(khotanFlows, ({ one, many }) => ({
+  plug: one(khotanPlugs, {
+    fields: [khotanFlows.plugId],
+    references: [khotanPlugs.id],
+  }),
+  resource: one(khotanResources, {
+    fields: [khotanFlows.resourceId],
+    references: [khotanResources.id],
+  }),
+  runs: many(khotanRuns),
+}));
+
+export const khotanWiresRelations = relations(khotanWires, ({ one, many }) => ({
+  plug: one(khotanPlugs, {
+    fields: [khotanWires.plugId],
+    references: [khotanPlugs.id],
+  }),
+  webhookHandlers: many(khotanWebhookHandlers),
+  webhookEvents: many(khotanWebhookEvents),
+  runs: many(khotanRuns),
+}));
+
+export const khotanWebhookHandlersRelations = relations(
+  khotanWebhookHandlers,
+  ({ one, many }) => ({
+    wire: one(khotanWires, {
+      fields: [khotanWebhookHandlers.wireId],
+      references: [khotanWires.id],
+    }),
+    webhookEvents: many(khotanWebhookEvents),
+    runs: many(khotanRuns),
+  }),
+);
+
+export const khotanRunsRelations = relations(khotanRuns, ({ one, many }) => ({
+  flow: one(khotanFlows, {
+    fields: [khotanRuns.flowId],
+    references: [khotanFlows.id],
+  }),
+  wire: one(khotanWires, {
+    fields: [khotanRuns.wireId],
+    references: [khotanWires.id],
+  }),
+  webhookHandler: one(khotanWebhookHandlers, {
+    fields: [khotanRuns.webhookHandlerId],
+    references: [khotanWebhookHandlers.id],
+  }),
+  webhookEvents: many(khotanWebhookEvents),
+}));
+
+export const khotanWebhookEventsRelations = relations(
+  khotanWebhookEvents,
+  ({ one }) => ({
+    wire: one(khotanWires, {
+      fields: [khotanWebhookEvents.wireId],
+      references: [khotanWires.id],
+    }),
+    webhookHandler: one(khotanWebhookHandlers, {
+      fields: [khotanWebhookEvents.webhookHandlerId],
+      references: [khotanWebhookHandlers.id],
+    }),
+    khotanRun: one(khotanRuns, {
+      fields: [khotanWebhookEvents.khotanRunId],
+      references: [khotanRuns.id],
+    }),
+  }),
+);
+
+export const khotanResourcesRelations = relations(
+  khotanResources,
+  ({ many }) => ({
+    flows: many(khotanFlows),
+    mappings: many(khotanMappings),
+  }),
+);
+
+export const khotanMappingsRelations = relations(khotanMappings, ({ one }) => ({
+  resource: one(khotanResources, {
+    fields: [khotanMappings.resourceId],
+    references: [khotanResources.id],
+  }),
+}));
+
+export const khotanCachesRelations = relations(khotanCaches, ({ many }) => ({
+  entries: many(khotanCacheEntries),
+}));
+
+export const khotanCacheEntriesRelations = relations(
+  khotanCacheEntries,
+  ({ one }) => ({
+    cache: one(khotanCaches, {
+      fields: [khotanCacheEntries.cacheId],
+      references: [khotanCaches.id],
+    }),
+  }),
+);
+
+// ---------------------------------------------------------------------------
+// Type helpers
+// ---------------------------------------------------------------------------
+
+export type KhotanPlug = typeof khotanPlugs.$inferSelect;
+export type NewKhotanPlug = typeof khotanPlugs.$inferInsert;
+
+export type KhotanFlow = typeof khotanFlows.$inferSelect;
+export type NewKhotanFlow = typeof khotanFlows.$inferInsert;
+
+export type KhotanWire = typeof khotanWires.$inferSelect;
+export type NewKhotanWire = typeof khotanWires.$inferInsert;
+
+export type KhotanWebhookHandler = typeof khotanWebhookHandlers.$inferSelect;
+export type NewKhotanWebhookHandler = typeof khotanWebhookHandlers.$inferInsert;
+
+export type KhotanWebhookEvent = typeof khotanWebhookEvents.$inferSelect;
+export type NewKhotanWebhookEvent = typeof khotanWebhookEvents.$inferInsert;
+
+export type KhotanRun = typeof khotanRuns.$inferSelect;
+export type NewKhotanRun = typeof khotanRuns.$inferInsert;
+
+export type KhotanResource = typeof khotanResources.$inferSelect;
+export type NewKhotanResource = typeof khotanResources.$inferInsert;
+
+export type KhotanMapping = typeof khotanMappings.$inferSelect;
+export type NewKhotanMapping = typeof khotanMappings.$inferInsert;
+
+export type KhotanCache = typeof khotanCaches.$inferSelect;
+export type NewKhotanCache = typeof khotanCaches.$inferInsert;
+
+export type KhotanCacheEntry = typeof khotanCacheEntries.$inferSelect;
+export type NewKhotanCacheEntry = typeof khotanCacheEntries.$inferInsert;

package/dist/templates/skill-dashboard.md

@@ -0,0 +1,144 @@
+---
+name: khotan-dashboard
+description: >
+  Set up khotan dashboard UI — the Hub for managing plugs, flows,
+  variables, and webhooks, plus the Plug Debugger for testing API
+  requests. Use when adding a management interface, configuring plug
+  variables in the browser, or setting up debug pages.
+---
+
+Set up khotan dashboard UI — the Hub for managing plugs, flows, variables, and webhooks, plus the Plug Debugger for testing API requests. Use when adding a management interface, configuring plug variables in the browser, or setting up debug pages.
+
+## Hub (Management Dashboard)
+
+```bash
+npx khotan add hub --yes
+npx khotan add config-page-1 --yes    # Ready-made /config route
+```
+
+The Hub scaffolds three components to `src/components/khotan/`:
+
+| File | Purpose |
+|------|---------|
+| `hub.tsx` | Main `<KhotanHub />` — plug cards, flow table, enable/disable toggles |
+| `var-panel.tsx` | Variables panel for configuring plug vars |
+| `wire-panel.tsx` | Webhook subscription management (connect/disconnect) |
+
+### Rendering the Hub
+
+```tsx
+import { KhotanHub } from "@/components/khotan/hub";
+
+export default function ConfigPage() {
+  return (
+    <main className="container mx-auto max-w-5xl px-4 py-10">
+      <KhotanHub />
+    </main>
+  );
+}
+```
+
+Or use `npx khotan add config-page-1` to scaffold a `/config` page automatically.
+
+### Hub Features
+
+- Lists all registered plugs with status badges (connected/error/idle)
+- Click a plug to see its flows with enable/disable toggles
+- VarPanel: configure plug variables (stored encrypted via `KHOTAN_SECRET`)
+- WirePanel: manage webhook subscriptions (requires wires configured on plug)
+- Debug button on each plug card (visible when `KHOTAN_DEBUG=1`)
+
+### Hub Props
+
+```tsx
+<KhotanHub
+  webhookUrl="https://your-domain.com"  // Base URL for wire callbacks
+/>
+```
+
+### API Endpoints Used by Hub
+
+| Endpoint | Purpose |
+|----------|---------|
+| `GET /api/khotan/plugs` | List plugs with flow counts |
+| `GET /api/khotan/flows` | List all flows |
+| `PATCH /api/khotan/flows/:id` | Toggle flow enabled/disabled |
+| `POST /api/khotan/flows/:id/runs` | Start a tracked flow run |
+| `GET /api/khotan/runs/:id` | Get run detail with live Workflow status |
+| `GET /api/khotan/runs/:id/stream` | Stream Workflow progress updates |
+| `POST /api/khotan/runs/:id/cancel` | Cancel a running Workflow-backed run |
+| `POST /api/khotan/runs/:id/retry` | Retry a flow run with the same run type |
+| `PATCH /api/khotan/plugs/:id` | Toggle plug enabled/disabled |
+| `GET /api/khotan/variables/:plugName` | Get var fields + masked values |
+| `POST /api/khotan/variables/:plugName` | Save encrypted variables |
+| `DELETE /api/khotan/variables/:plugName` | Clear variables |
+| `GET /api/khotan/wires/:plugName` | Get wire status |
+| `POST /api/khotan/wires/:plugName` | Create webhook subscription |
+| `DELETE /api/khotan/wires/:plugName` | Remove webhook subscription |
+
+From server code, prefer the Khotan-native starter instead of calling `workflow/api.start()` directly:
+
+```typescript
+await khotanData.flow("products-inflow", { plugName: "shopify" }).start({
+  runType: "delta",
+});
+```
+
+### Variables In Code And CLI
+
+- Declare optional `defaultValue` on plug var fields to seed initial DB-backed values.
+- Use `npx khotan plug vars <plugName>` to inspect masked values from the terminal.
+- Use `npx khotan plug vars <plugName> set --json '{...}'` to update variables without opening the Hub.
+- Use `npx khotan plug vars <plugName> clear` to remove all stored overrides for a plug.
+
+## Plug Debugger (Dev Testing UI)
+
+```bash
+npx khotan add plug-debugger --yes
+npx khotan add debug-page-1 --yes    # Routes at /debug and /debug/[plugName]
+```
+
+Requires `KHOTAN_DEBUG=1` in your environment.
+
+### Features
+
+- Postman-like interface for testing plug requests
+- Typed endpoint sidebar showing Zod schemas
+- Path parameter interpolation
+- Response validation (green/amber/red diff)
+- Request history
+- Auto-format JSON bodies
+- Keyboard shortcut: Cmd+Enter to send
+
+### Debug Routes
+
+| Route | Purpose |
+|-------|---------|
+| `/debug` | Index page listing all plugs |
+| `/debug/[plugName]` | Per-plug debugger |
+
+### Debug API Endpoints
+
+| Endpoint | Purpose |
+|----------|---------|
+| `GET /api/khotan/debug` | Check if debug mode is active |
+| `GET /api/khotan/debug/:plugName` | Plug metadata + endpoint schemas |
+| `POST /api/khotan/debug/:plugName` | Fire request through the real plug code path |
+
+## shadcn Dependencies
+
+Both components require shadcn/ui. The CLI will offer to install missing components:
+
+- **Hub**: card, badge, table, switch, button, input, label
+- **Plug Debugger**: card, badge, button, input, label
+
+Run `npx shadcn@latest init --defaults --yes` first if shadcn is not set up.
+
+## Skip UI
+
+Use `--without-ui` to scaffold only the backend code without React components:
+
+```bash
+npx khotan add hub --without-ui
+npx khotan add wire --without-ui
+```