khotan-data 0.1.0 → 0.2.0

package/README.md

@@ -4,6 +4,14 @@ Data primitives for TypeScript — ETL pipelines, transforms, and Drizzle Postgr
 
 Built for **Next.js + Drizzle + Postgres** projects. Think better-auth for data management.
 
+## Install
+
+```bash
+npm i khotan-data
+```
+
+Requires `drizzle-orm` as a peer dependency (you almost certainly already have it).
+
 ## CLI
 
 Scaffold components into your Next.js + Drizzle project:
@@ -17,6 +25,7 @@ npx khotan init --full
 
 # Add components (reusable building blocks — never create pages)
 npx khotan add schema    # Drizzle table definitions (plugs, flows, runs, resources, mappings)
+npx khotan add cache     # Durable key/value caches for workflows and relays
 npx khotan add plug      # Fetch wrapper with auth, retry, pagination
 npx khotan add inflow    # Workflow-backed flow for pulling data in
 npx khotan add outflow   # Workflow-backed flow for pushing data out
@@ -33,18 +42,27 @@ npx khotan add hub --yes        # Auto-accept dependency install prompts
 
 ## Factory (Runtime Engine)
 
-Register plugs, flows, and resources — the factory upserts them on boot and serves a REST API:
+Register plugs, caches, flows, and resources — the factory upserts them on boot and serves a REST API:
 
 ```typescript
 import { khotan, drizzleAdapter, toNextJsHandler } from "khotan-data/factory";
 import { db } from "@/db";
 import { shopifyPlug } from "@/lib/khotan/plugs/shopify";
 import { shopifyProductsInflow } from "@/lib/khotan/flows/shopify-products";
+import { shopifyProductsSnapshotCache } from "@/lib/khotan/caches/shopify-products-snapshot";
 
 const khotanData = khotan({
   adapter: drizzleAdapter(db),
+  // Gate the management API behind your auth layer (see "Security" below).
+  authorize: async (request) => {
+    const session = await auth.api.getSession({ headers: request.headers });
+    return Boolean(session?.user);
+  },
   resources: [
-    { name: "products", connectField: "sku" },
+    { name: "products", mapping: { connectField: "sku" } },
+  ],
+  caches: [
+    shopifyProductsSnapshotCache,
   ],
   plugs: [
     {
@@ -66,13 +84,78 @@ await khotanData.flow("products-inflow", { plugName: "shopify" }).start({
 });
 ```
 
-## Install
+## Security
 
-```bash
-npm install khotan-data
+The management API (`/api/khotan/*`) exposes plug credentials and operational
+controls. It is **public unless you gate it**. Pass an `authorize` hook — it
+receives the raw `Request` and returns `true`/`false`, so it composes directly
+with session libraries like better-auth:
+
+```typescript
+authorize: async (request) => {
+  const session = await auth.api.getSession({ headers: request.headers });
+  return session?.user?.role === "admin";
+},
 ```
 
-Requires `drizzle-orm` as a peer dependency (you almost certainly already have it).
+- `KHOTAN_SECRET` encrypts plug credentials **at rest** (AES-256-GCM). It is not
+  an auth credential — it never gates requests. Set it to a high-entropy value.
+- Inbound webhooks (verified via per-plug `onVerify`), the cron dispatcher
+  (`CRON_SECRET`), and debug routes (`KHOTAN_DEBUG`, non-production only) are
+  exempt from `authorize` automatically.
+- `KHOTAN_DEBUG` is force-disabled when `NODE_ENV=production`. The cron route
+  fails closed in production when `CRON_SECRET` is unset.
+- Protect the Hub dashboard page (e.g. `/config`) with your app's middleware —
+  `authorize` only guards the API.
+
+## Caches
+
+Use first-class caches when a flow, relay, catch, or pass needs durable state between runs.
+
+```typescript
+import { cache } from "@/lib/khotan/caches/cache";
+
+export const shopifyProductsSnapshotCache = cache({
+  name: "shopify-products-snapshot",
+  scope: {
+    plug: "shopify",
+    resource: "products",
+    flow: "shopify-products-inflow",
+  },
+  ttl: "6h",
+});
+```
+
+Inside workflows, use `khotanCache(ctx, "name")` for snapshots, cursors, and dedupe markers:
+
+```typescript
+import { khotanCache } from "khotan-data/factory";
+
+async function shopifyProductsWorkflow(ctx: InflowContext) {
+  "use workflow";
+
+  async function syncProducts() {
+    "use step";
+    const snapshotCache = khotanCache(ctx, "shopify-products-snapshot");
+    const previous =
+      (await snapshotCache.get<Array<Record<string, unknown>>>("latest")) ?? [];
+
+    const response = await shopifyPlug.get<{ data?: Array<Record<string, unknown>> }>("/products");
+    const records = Array.isArray(response.data) ? response.data : [];
+
+    await snapshotCache.set("latest", records);
+
+    return {
+      extracted: records.length,
+      transformed: records.length,
+      created: records.length,
+      metadata: { previousCount: previous.length },
+    };
+  }
+
+  return syncProducts();
+}
+```
 
 ## Quick Start