khotan-data 0.1.0 → 0.1.1

package/dist/templates/mappings-page.tsx

@@ -0,0 +1,9 @@
+import { KhotanMappingBrowser } from "@/components/khotan/mapping-browser";
+
+export default function KhotanMappingsPage() {
+  return (
+    <main className="container mx-auto max-w-7xl px-4 py-10">
+      <KhotanMappingBrowser />
+    </main>
+  );
+}

package/dist/templates/outflow.ts

@@ -5,6 +5,8 @@
 // This file defines the outflow() builder and types. Create per-service flow
 // files (e.g. crm-audiences.ts) using this builder to read app data and push it
 // to an external service with durable, retryable Vercel Workflow steps.
+// Outflow workflows can also use khotanCache(ctx, "name") for checkpoints, cursor
+// state, or dedupe markers between runs.
 // ============================================================================
 
 import type {
@@ -46,7 +48,7 @@ export function outflow(config: OutflowConfig): FlowRegistration {
 // Usage Example (create a file like flows/hubspot-products.ts)
 // ---------------------------------------------------------------------------
 //
-// import { outflow, type OutflowContext } from "./outflow";
+// import { bindWorkflowPlug, outflow, type OutflowContext } from "khotan-data/factory";
 // import { db } from "@/db";
 // import { products } from "@/db/schema";
 // import { hubspotPlug } from "../plugs/hubspot";
@@ -61,14 +63,12 @@ export function outflow(config: OutflowConfig): FlowRegistration {
 //       khotanRunId: ctx.khotanRunId,
 //       runType: ctx.runType,
 //     });
+//     const hubspot = bindWorkflowPlug(hubspotPlug, ctx);
 //
 //     const records = await db.select().from(products);
 //
 //     for (const record of records) {
-//       await hubspotPlug.post("/products", {
-//         vars: ctx.vars,
-//         body: record,
-//       });
+//       await hubspot.post("/products", { body: record });
 //     }
 //
 //     return {

package/dist/templates/pass.ts

@@ -26,6 +26,8 @@ export interface PassContext {
   destVars: Record<string, string>;
   /** Khotan run ID created for this webhook handler execution */
   khotanRunId: string;
+  /** Internal Khotan instance identifier for helper APIs */
+  khotanInstanceId: string;
 }
 
 // ---------------------------------------------------------------------------
@@ -79,6 +81,7 @@ export function pass(config: PassConfig): PassRegistration {
 // Usage Example (create a file like webhooks/pollinate-to-slack.ts)
 // ---------------------------------------------------------------------------
 //
+// import { khotanCache } from "khotan-data/factory";
 // import { pass, type PassContext } from "./pass";
 // import { plug } from "../plugs/plug";
 //
@@ -87,6 +90,9 @@ export function pass(config: PassConfig): PassRegistration {
 //
 //   async function forwardEvent() {
 //     "use step";
+//     const cache = khotanCache(ctx, "pollinate-forwarded-events");
+//     const eventId = String(ctx.event["id"] ?? "");
+//     if (eventId && (await cache.get<boolean>(eventId))) return;
 //
 //     // Construct destination plug from destVars
 //     const slackPlug = plug({
@@ -102,6 +108,10 @@ export function pass(config: PassConfig): PassRegistration {
 //         event: ctx.event,
 //       },
 //     });
+//
+//     if (eventId) {
+//       await cache.set(eventId, true);
+//     }
 //   }
 //
 //   await forwardEvent();

package/dist/templates/relay.example.ts

@@ -6,6 +6,7 @@
 // exported flow in {outputDir}/khotan.ts.
 // ============================================================================
 
+import { khotanCache } from "khotan-data/factory";
 import { relay, type RelayContext } from "./relay";
 
 async function shopifyToHubspotWorkflow(ctx: RelayContext) {
@@ -29,6 +30,11 @@ async function shopifyToHubspotWorkflow(ctx: RelayContext) {
       data?: Array<Record<string, unknown>>;
     };
     const records = Array.isArray(payload.data) ? payload.data : [];
+    const snapshotCache = khotanCache(ctx, "shopify-products-snapshot");
+    const previousRecords =
+      (await snapshotCache.get<Array<Record<string, unknown>>>("latest")) ?? [];
+
+    await snapshotCache.set("latest", records, { ttl: "6h" });
 
     for (const record of records) {
       await fetch("https://destination.example.com/products", {
@@ -45,7 +51,11 @@ async function shopifyToHubspotWorkflow(ctx: RelayContext) {
       extracted: records.length,
       transformed: records.length,
       created: records.length,
-      metadata: { relay: ctx.flow.name, to: ctx.flow.to },
+      metadata: {
+        relay: ctx.flow.name,
+        to: ctx.flow.to,
+        previousCount: previousRecords.length,
+      },
     };
   }
 

package/dist/templates/relay.ts

@@ -5,7 +5,8 @@
 // This file defines the relay() builder and types. Create per-service flow
 // files (e.g. shopify-to-hubspot.ts) using this builder to read from the source
 // plug and forward to a destination system with durable, retryable Vercel
-// Workflow steps.
+// Workflow steps. Relay workflows can also use khotanCache(ctx, "name") for durable
+// snapshots, checkpoints, and dedupe state between runs.
 // ============================================================================
 
 import type {
@@ -50,7 +51,7 @@ export function relay(config: RelayConfig): FlowRegistration {
 // Usage Example (create a file like flows/shopify-to-hubspot.ts)
 // ---------------------------------------------------------------------------
 //
-// import { relay, type RelayContext } from "./relay";
+// import { bindWorkflowPlug, khotanCache, relay, type RelayContext } from "khotan-data/factory";
 // import { shopifyPlug } from "../plugs/shopify";
 // import { hubspotPlug } from "../plugs/hubspot";
 //
@@ -65,21 +66,29 @@ export function relay(config: RelayConfig): FlowRegistration {
 //       khotanRunId: ctx.khotanRunId,
 //       runType: ctx.runType,
 //     });
+//     const shopify = bindWorkflowPlug(shopifyPlug, ctx);
+//     const hubspot = bindWorkflowPlug(hubspotPlug, ctx, "hubspot");
 //
-//     const response = await shopifyPlug.get<{ data?: Array<Record<string, unknown>> }>("/products", {
-//       vars: ctx.vars,
-//     });
+//     const snapshotCache = khotanCache(ctx, "shopify-products-snapshot");
+//     const previous = await snapshotCache.get<Array<Record<string, unknown>>>("latest");
+//
+//     const response = await shopify.get<{ data?: Array<Record<string, unknown>> }>("/products");
 //     const records = Array.isArray(response.data) ? response.data : [];
+//     await snapshotCache.set("latest", records);
 //
 //     for (const record of records) {
-//       await hubspotPlug.post("/products", { body: record });
+//       await hubspot.post("/products", { body: record });
 //     }
 //
 //     return {
 //       extracted: records.length,
 //       transformed: records.length,
 //       created: records.length,
-//       metadata: { relay: ctx.flow.name, to: ctx.flow.to },
+//       metadata: {
+//         relay: ctx.flow.name,
+//         to: ctx.flow.to,
+//         previousCount: previous?.length ?? 0,
+//       },
 //     };
 //   }
 //

package/dist/templates/schema.ts

@@ -303,6 +303,67 @@ export const khotanMappings = pgTable(
   ],
 );
 
+// ---------------------------------------------------------------------------
+// khotan_caches — one row per registered durable cache namespace
+// ---------------------------------------------------------------------------
+
+export const khotanCaches = pgTable(
+  "khotan_caches",
+  {
+    id: text("id")
+      .primaryKey()
+      .$defaultFn(() => crypto.randomUUID()),
+    name: text("name").notNull().unique(),
+    scope: jsonb("scope").$type<{
+      plug?: string;
+      resource?: string;
+      flow?: string;
+    }>(),
+    ttlSeconds: integer("ttl_seconds"),
+    createdAt: timestamp("created_at", { withTimezone: true })
+      .defaultNow()
+      .notNull(),
+    updatedAt: timestamp("updated_at", { withTimezone: true })
+      .defaultNow()
+      .notNull(),
+  },
+  (table) => [index("khotan_caches_name_idx").on(table.name)],
+);
+
+// ---------------------------------------------------------------------------
+// khotan_cache_entries — latest-value durable cache rows keyed by cache + key
+// ---------------------------------------------------------------------------
+
+export const khotanCacheEntries = pgTable(
+  "khotan_cache_entries",
+  {
+    id: text("id")
+      .primaryKey()
+      .$defaultFn(() => crypto.randomUUID()),
+    cacheId: text("cache_id")
+      .notNull()
+      .references(() => khotanCaches.id),
+    key: text("key").notNull(),
+    value: jsonb("value").notNull(),
+    expiresAt: timestamp("expires_at", { withTimezone: true }),
+    createdAt: timestamp("created_at", { withTimezone: true })
+      .defaultNow()
+      .notNull(),
+    updatedAt: timestamp("updated_at", { withTimezone: true })
+      .defaultNow()
+      .notNull(),
+  },
+  (table) => [
+    unique("khotan_cache_entries_cache_id_key_unique").on(
+      table.cacheId,
+      table.key,
+    ),
+    index("khotan_cache_entries_cache_id_idx").on(table.cacheId),
+    index("khotan_cache_entries_cache_id_key_idx").on(table.cacheId, table.key),
+    index("khotan_cache_entries_expires_at_idx").on(table.expiresAt),
+  ],
+);
+
 // ---------------------------------------------------------------------------
 // Relations
 // ---------------------------------------------------------------------------
@@ -395,6 +456,20 @@ export const khotanMappingsRelations = relations(khotanMappings, ({ one }) => ({
   }),
 }));
 
+export const khotanCachesRelations = relations(khotanCaches, ({ many }) => ({
+  entries: many(khotanCacheEntries),
+}));
+
+export const khotanCacheEntriesRelations = relations(
+  khotanCacheEntries,
+  ({ one }) => ({
+    cache: one(khotanCaches, {
+      fields: [khotanCacheEntries.cacheId],
+      references: [khotanCaches.id],
+    }),
+  }),
+);
+
 // ---------------------------------------------------------------------------
 // Type helpers
 // ---------------------------------------------------------------------------
@@ -422,3 +497,9 @@ export type NewKhotanResource = typeof khotanResources.$inferInsert;
 
 export type KhotanMapping = typeof khotanMappings.$inferSelect;
 export type NewKhotanMapping = typeof khotanMappings.$inferInsert;
+
+export type KhotanCache = typeof khotanCaches.$inferSelect;
+export type NewKhotanCache = typeof khotanCaches.$inferInsert;
+
+export type KhotanCacheEntry = typeof khotanCacheEntries.$inferSelect;
+export type NewKhotanCacheEntry = typeof khotanCacheEntries.$inferInsert;

package/dist/templates/skill-plug.md

@@ -98,28 +98,39 @@ export const myPlug = plug({
 
 Endpoints power the plug debugger UI, `khotan plug --compare`, and typed clients.
 
-## Typed Client (Contract Pattern)
+## Preferred Pattern
 
-For separate contract definition + type-safe calls:
+Keep each integration in a single app-owned plug file when possible:
 
 ```typescript
-import { defineContract, createPlugClient } from "khotan-data/plug";
-
-const contract = defineContract({
-  listProducts: {
-    method: "GET",
-    path: "/products",
-    query: z.object({ page: z.number().optional() }),
-    responses: { 200: z.object({ data: z.array(ProductSchema), total: z.number() }) },
-  },
+import { z } from "zod";
+import { plug, basic } from "./plug";
+
+const ProductSchema = z.object({
+  id: z.string(),
+  sku: z.string(),
+  name: z.string(),
 });
 
-const client = createPlugClient(contract, myPlug);
-const result = await client.listProducts({ query: { page: 1 } });
-// result.status — 200
-// result.body.data — typed as Product[]
+export type Product = z.infer<typeof ProductSchema>;
+
+export const myPlug = plug({
+  name: "my-service",
+  baseUrl: "https://api.example.com",
+  auth: basic(process.env.API_USER!, process.env.API_KEY!),
+  endpoints: {
+    listProducts: {
+      method: "GET",
+      path: "/products",
+      query: z.object({ page: z.number().optional(), limit: z.number().optional() }),
+      responses: { 200: z.array(ProductSchema) },
+    },
+  },
+});
 ```
 
+This keeps the runtime plug, debugger metadata, `khotan plug --compare`, and any exported types in one place.
+
 ## Hooks
 
 ```typescript
@@ -179,6 +190,18 @@ npx khotan plug myPlug --endpoint listProducts --compare  # Check schema
 
 Set `KHOTAN_DEBUG=1` for verbose `[khotan:auth]` and `[khotan:request]` console logs.
 
+### Recommended Plug Workflow
+
+1. Create the plug file and auth/hook setup.
+2. Add a small set of typed endpoints directly on the plug (`listProducts`, `getProduct`, etc).
+3. Run the app with `KHOTAN_DEBUG=1`.
+4. Use `npx khotan plug myPlug --info` to confirm the endpoints are visible to the debugger.
+5. Use `npx khotan plug myPlug --endpoint listProducts --compare` against the live API.
+6. Tighten schemas until the compare output matches the real payload shape you care about.
+7. Only then build inflows, relays, outflows, or webhook handlers on top of those endpoints.
+
+The package does not paginate or delta-sync for you automatically inside user flows. Your app code decides which typed endpoints to call, what page size to use, when to stop, and how to implement full, test, partial, backfill, reconcile, or delta runs.
+
 ## Managing Vars
 
 Use the CLI to inspect and update stored plug variables:

package/dist/templates/skill-setup.md

@@ -42,8 +42,8 @@ import { stripeChargesInflow } from "./flows/stripe-charges";
 const khotanData = khotan({
   adapter: drizzleAdapter(db),
   resources: [
-    { name: "products", connectField: "sku" },
-    { name: "orders", connectField: "order_number" },
+    { name: "products", mapping: { connectField: "sku" } },
+    { name: "orders", mapping: { connectField: "order_number" } },
   ],
   plugs: [
     {
@@ -92,6 +92,7 @@ The schema command auto-detects your Drizzle schema directory, updates `drizzle.
 | `KHOTAN_SECRET` | For variables | AES-256-GCM key for encrypting plug vars |
 | `KHOTAN_DEBUG` | For debugging | Enables `/debug/*` routes and the `plug` CLI (`probe` alias) |
 | `KHOTAN_WEBHOOK_URL` | For webhooks | Public URL for wire callbacks |
+| `CRON_SECRET` | For production cron | Protects the built-in `/api/khotan/cron` dispatcher route |
 
 ## Next.js Config
 
@@ -111,6 +112,47 @@ curl http://localhost:3000/api/khotan/flows      # Should list flows
 curl http://localhost:3000/api/khotan/resources   # Should list resources
 ```
 
+## Scheduled Flows On Vercel
+
+Khotan flow `schedule` values are runtime source-of-truth metadata. On Vercel, prefer a single dispatcher CRON instead of defining one platform CRON per flow.
+
+Add one entry to `vercel.json`:
+
+```json
+{
+  "crons": [
+    { "path": "/api/khotan/cron", "schedule": "* * * * *" }
+  ]
+}
+```
+
+Then define schedules only on your flows in `{outputDir}/khotan.ts`:
+
+```typescript
+{
+  name: "products-inflow",
+  type: "inflow",
+  schedule: "0 * * * *",
+  resource: "products",
+}
+```
+
+The dispatcher route evaluates which flows are due on each tick and starts them through the normal run-tracking path. If `CRON_SECRET` is set, Vercel should call the route with `Authorization: Bearer <CRON_SECRET>`.
+
+## Typical Build Order
+
+After init and schema setup, the usual path to a working sync is:
+
+1. Add or author a plug file for the external service.
+2. Define a few typed endpoints directly on the plug with Zod response schemas.
+3. Start the app with `KHOTAN_DEBUG=1`.
+4. Verify the plug is visible with `npx khotan plug --list` and `npx khotan plug myPlug --info`.
+5. Hit live endpoints with `npx khotan plug myPlug --endpoint listProducts --compare` until the schemas match the real API shape you intend to use.
+6. Register the plug in `{outputDir}/khotan.ts` with resources and flows.
+7. Only after endpoint verification, build inflows, relays, outflows, or webhook handlers on top of those live-checked endpoints.
+
+This keeps sync logic grounded in real API payloads before you write pagination, mapping, or transformation code.
+
 ## Troubleshooting
 
 - **Empty plug list**: Factory upserts on first request — hit any endpoint first, then check `/plugs`

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "khotan-data",
-  "version": "0.1.0",
+  "version": "0.1.1",
   "description": "Data primitives for TypeScript — ETL pipelines, transforms, and Drizzle Postgres integration.",
   "type": "module",
   "main": "./dist/index.cjs",