khotan-data 0.1.0 → 0.1.1

package/README.md

@@ -4,6 +4,14 @@ Data primitives for TypeScript — ETL pipelines, transforms, and Drizzle Postgr
 
 Built for **Next.js + Drizzle + Postgres** projects. Think better-auth for data management.
 
+## Install
+
+```bash
+npm i khotan-data
+```
+
+Requires `drizzle-orm` as a peer dependency (you almost certainly already have it).
+
 ## CLI
 
 Scaffold components into your Next.js + Drizzle project:
@@ -17,6 +25,7 @@ npx khotan init --full
 
 # Add components (reusable building blocks — never create pages)
 npx khotan add schema    # Drizzle table definitions (plugs, flows, runs, resources, mappings)
+npx khotan add cache     # Durable key/value caches for workflows and relays
 npx khotan add plug      # Fetch wrapper with auth, retry, pagination
 npx khotan add inflow    # Workflow-backed flow for pulling data in
 npx khotan add outflow   # Workflow-backed flow for pushing data out
@@ -33,18 +42,22 @@ npx khotan add hub --yes        # Auto-accept dependency install prompts
 
 ## Factory (Runtime Engine)
 
-Register plugs, flows, and resources — the factory upserts them on boot and serves a REST API:
+Register plugs, caches, flows, and resources — the factory upserts them on boot and serves a REST API:
 
 ```typescript
 import { khotan, drizzleAdapter, toNextJsHandler } from "khotan-data/factory";
 import { db } from "@/db";
 import { shopifyPlug } from "@/lib/khotan/plugs/shopify";
 import { shopifyProductsInflow } from "@/lib/khotan/flows/shopify-products";
+import { shopifyProductsSnapshotCache } from "@/lib/khotan/caches/shopify-products-snapshot";
 
 const khotanData = khotan({
   adapter: drizzleAdapter(db),
   resources: [
-    { name: "products", connectField: "sku" },
+    { name: "products", mapping: { connectField: "sku" } },
+  ],
+  caches: [
+    shopifyProductsSnapshotCache,
   ],
   plugs: [
     {
@@ -66,13 +79,54 @@ await khotanData.flow("products-inflow", { plugName: "shopify" }).start({
 });
 ```
 
-## Install
+## Caches
 
-```bash
-npm install khotan-data
+Use first-class caches when a flow, relay, catch, or pass needs durable state between runs.
+
+```typescript
+import { cache } from "@/lib/khotan/caches/cache";
+
+export const shopifyProductsSnapshotCache = cache({
+  name: "shopify-products-snapshot",
+  scope: {
+    plug: "shopify",
+    resource: "products",
+    flow: "shopify-products-inflow",
+  },
+  ttl: "6h",
+});
 ```
 
-Requires `drizzle-orm` as a peer dependency (you almost certainly already have it).
+Inside workflows, use `khotanCache(ctx, "name")` for snapshots, cursors, and dedupe markers:
+
+```typescript
+import { khotanCache } from "khotan-data/factory";
+
+async function shopifyProductsWorkflow(ctx: InflowContext) {
+  "use workflow";
+
+  async function syncProducts() {
+    "use step";
+    const snapshotCache = khotanCache(ctx, "shopify-products-snapshot");
+    const previous =
+      (await snapshotCache.get<Array<Record<string, unknown>>>("latest")) ?? [];
+
+    const response = await shopifyPlug.get<{ data?: Array<Record<string, unknown>> }>("/products");
+    const records = Array.isArray(response.data) ? response.data : [];
+
+    await snapshotCache.set("latest", records);
+
+    return {
+      extracted: records.length,
+      transformed: records.length,
+      created: records.length,
+      metadata: { previousCount: previous.length },
+    };
+  }
+
+  return syncProducts();
+}
+```
 
 ## Quick Start
 

package/dist/cli.js

@@ -168,6 +168,23 @@ var COMPONENTS = {
       npmPackages: ["drizzle-orm"]
     }
   },
+  cache: {
+    name: "cache",
+    description: "First-class durable cache definitions for khotan sync workloads",
+    requires: ["schema"],
+    files: [
+      {
+        templatePath: path2.resolve(__dirname$1, "templates", "cache.ts"),
+        outputFile: "caches/cache.ts",
+        outputBase: "outputDir"
+      },
+      {
+        templatePath: path2.resolve(__dirname$1, "templates", "cache.example.ts"),
+        outputFile: "caches/cache.example.ts",
+        outputBase: "outputDir"
+      }
+    ]
+  },
   hub: {
     name: "hub",
     description: "Dashboard UI for managing plugs and flows",
@@ -231,6 +248,25 @@ var COMPONENTS = {
       }
     ]
   },
+  "mapping-browser": {
+    name: "mapping-browser",
+    description: "Searchable mappings browser for listing, creating, editing, and deleting resource mappings",
+    requiresShadcn: true,
+    dependencies: {
+      shadcnComponents: ["card", "table", "button", "input", "label"]
+    },
+    files: [
+      {
+        templatePath: path2.resolve(
+          __dirname$1,
+          "templates",
+          "mapping-browser.tsx"
+        ),
+        outputFile: "mapping-browser.tsx",
+        outputBase: "components"
+      }
+    ]
+  },
   "plug-debugger": {
     name: "plug-debugger",
     description: "Dev-only debug panel for testing plug requests interactively",
@@ -462,6 +498,22 @@ var BLOCKS = {
       }
     ]
   },
+  "mappings-page-1": {
+    name: "mappings-page-1",
+    description: "Page route at /mappings that renders the reusable Khotan mappings browser",
+    requires: ["mapping-browser"],
+    files: [
+      {
+        templatePath: path2.resolve(
+          __dirname$1,
+          "templates",
+          "mappings-page.tsx"
+        ),
+        outputFile: "mappings/page.tsx",
+        outputBase: "appRoot"
+      }
+    ]
+  },
   graph: {
     name: "graph",
     description: "Standalone topology graph page at /graph with filtering and run-state overlays",
@@ -1722,13 +1774,13 @@ function diffSchemas(expected, actual, basePath = "$") {
   }
   return diffObjectSchema(expected, actual, basePath);
 }
-function diffTypedNode(expected, actual, path14) {
+function diffTypedNode(expected, actual, path15) {
   const expectedType = expected["_type"];
   if (expectedType === "array") {
     if (actual.type !== "array") {
       return [
         {
-          path: path14,
+          path: path15,
           issue: "type_mismatch",
           note: `expected array, got ${actual.type}`
         }
@@ -1736,13 +1788,13 @@ function diffTypedNode(expected, actual, path14) {
     }
     const itemSchema = expected["items"];
     if (!itemSchema || !actual.items) return [];
-    return diffSchemas(itemSchema, actual.items, `${path14}[]`);
+    return diffSchemas(itemSchema, actual.items, `${path15}[]`);
   }
   const normalizedExpected = normalizeType(expectedType);
   if (normalizedExpected !== actual.type && actual.type !== "null") {
     return [
       {
-        path: path14,
+        path: path15,
         issue: "type_mismatch",
         note: `expected ${expectedType}, got ${actual.type}`
       }
@@ -1750,11 +1802,11 @@ function diffTypedNode(expected, actual, path14) {
   }
   return [];
 }
-function diffObjectSchema(expected, actual, path14) {
+function diffObjectSchema(expected, actual, path15) {
   if (actual.type !== "object") {
     return [
       {
-        path: path14,
+        path: path15,
         issue: "type_mismatch",
         note: `expected object, got ${actual.type}`
       }
@@ -1763,7 +1815,7 @@ function diffObjectSchema(expected, actual, path14) {
   const mismatches = [];
   const actualProps = actual.properties;
   for (const [key, typeDesc] of Object.entries(expected)) {
-    const childPath = path14 === "$" ? `$.${key}` : `${path14}.${key}`;
+    const childPath = path15 === "$" ? `$.${key}` : `${path15}.${key}`;
     const typeStr = typeof typeDesc === "string" ? typeDesc : null;
     const isOptional = typeStr?.endsWith("?") ?? false;
     if (!(key in actualProps)) {
@@ -1801,7 +1853,7 @@ function diffObjectSchema(expected, actual, path14) {
   }
   for (const key of Object.keys(actualProps)) {
     if (!(key in expected)) {
-      const childPath = path14 === "$" ? `$.${key}` : `${path14}.${key}`;
+      const childPath = path15 === "$" ? `$.${key}` : `${path15}.${key}`;
       mismatches.push({ path: childPath, issue: "extra" });
     }
   }
@@ -2571,6 +2623,237 @@ withApiOptions(
   );
   output4({ ok: true, action: "cancel", run });
 });
+function output5(obj) {
+  process.stdout.write(JSON.stringify(obj, null, 2) + "\n");
+}
+function fail5(error, hint) {
+  output5({ ok: false, error, hint });
+  process.exit(1);
+}
+function parseEnvFile4(filePath) {
+  try {
+    const content = fs4.readFileSync(filePath, "utf-8");
+    const values = {};
+    for (const line of content.split("\n")) {
+      const trimmed = line.trim();
+      if (!trimmed || trimmed.startsWith("#")) continue;
+      const match = /^([A-Z0-9_]+)\s*=\s*["']?(.*?)["']?$/.exec(trimmed);
+      if (match) values[match[1]] = match[2];
+    }
+    return values;
+  } catch {
+    return {};
+  }
+}
+function parsePortFromEnvFile5(filePath) {
+  const raw = parseEnvFile4(filePath)["PORT"];
+  if (!raw) return null;
+  const parsed = parseInt(raw, 10);
+  return Number.isFinite(parsed) ? parsed : null;
+}
+function resolvePort5(portFlag) {
+  if (portFlag) return parseInt(portFlag, 10);
+  const cwd = process.cwd();
+  return parsePortFromEnvFile5(path2.join(cwd, ".env.local")) ?? parsePortFromEnvFile5(path2.join(cwd, ".env")) ?? 3e3;
+}
+function resolveBaseUrl2(opts) {
+  return `http://localhost:${resolvePort5(opts.port)}${opts.basePath}`;
+}
+async function fetchJson2(url, init) {
+  const res = await fetch(url, init);
+  const data = await res.json().catch(() => ({}));
+  if (!res.ok) {
+    fail5(
+      "request_failed",
+      data.error ?? `Request to ${url} failed with status ${res.status}`
+    );
+  }
+  return data;
+}
+async function checkConnectivity5(baseUrl) {
+  try {
+    const res = await fetch(`${baseUrl}/flows`);
+    if (!res.ok) {
+      fail5(
+        "api_unavailable",
+        `Could not reach Khotan flows API at ${baseUrl}. Check your base path and dev server.`
+      );
+    }
+  } catch {
+    fail5(
+      "connect_failed",
+      `Could not connect to dev server at ${baseUrl.replace("http://", "")}. Is it running?`
+    );
+  }
+}
+function parseJsonObjectOption(value, label) {
+  if (!value) return void 0;
+  try {
+    const parsed = JSON.parse(value);
+    if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) {
+      fail5("invalid_json", `${label} must be a JSON object.`);
+    }
+    return parsed;
+  } catch {
+    fail5("invalid_json", `${label} must be valid JSON.`);
+  }
+}
+function withApiOptions2(command) {
+  return command.option("--port <port>", "Dev server port").option("--base-path <path>", "API base path", "/api/khotan");
+}
+async function listResources(baseUrl) {
+  return fetchJson2(`${baseUrl}/resources`);
+}
+async function resolveResource(baseUrl, resourceNameOrId) {
+  const resources = await listResources(baseUrl);
+  const match = resources.find(
+    (resource) => resource.id === resourceNameOrId || resource.name === resourceNameOrId
+  );
+  if (!match) {
+    fail5(
+      "resource_not_found",
+      `Resource "${resourceNameOrId}" is not registered in the running Khotan config.`
+    );
+  }
+  return match;
+}
+var mappingsCommand = new Command("mappings").description("List, lookup, and mutate mappings via the running Khotan API");
+withApiOptions2(
+  mappingsCommand.command("list").description("List mappings for one resource").argument("<resource>", "Resource name or ID").option("--limit <limit>", "Page size", "20").option("--offset <offset>", "Page offset", "0").option("--search <term>", "Search connectValue, refs, and metadata")
+).action(
+  async (resourceNameOrId, opts) => {
+    const baseUrl = resolveBaseUrl2(opts);
+    await checkConnectivity5(baseUrl);
+    const resource = await resolveResource(baseUrl, resourceNameOrId);
+    const limit = Math.max(parseInt(opts.limit, 10) || 20, 1);
+    const offset = Math.max(parseInt(opts.offset, 10) || 0, 0);
+    const url = new URL(
+      `${baseUrl}/resources/${encodeURIComponent(resource.id)}/mappings`
+    );
+    url.searchParams.set("limit", String(limit));
+    url.searchParams.set("offset", String(offset));
+    if (opts.search) {
+      url.searchParams.set("search", opts.search);
+    }
+    const data = await fetchJson2(url.toString());
+    output5({
+      ok: true,
+      resource,
+      items: data.items,
+      page: data.page
+    });
+  }
+);
+withApiOptions2(
+  mappingsCommand.command("lookup").description("Lookup a mapping by connect value or plug ref").argument("<resource>", "Resource name or ID").option("--connect-value <value>", "Lookup by canonical connect value").option("--plug <plugName>", "Lookup by plug name").option("--ref <ref>", "Lookup by plug ref")
+).action(
+  async (resourceNameOrId, opts) => {
+    const usingConnectValue = typeof opts.connectValue === "string";
+    const usingPlugRef = typeof opts.plug === "string" || typeof opts.ref === "string";
+    if (!usingConnectValue && !usingPlugRef) {
+      fail5(
+        "validation_error",
+        "Pass either --connect-value <value> or --plug <plugName> with --ref <ref>."
+      );
+    }
+    if (usingConnectValue && usingPlugRef) {
+      fail5(
+        "validation_error",
+        "Choose one lookup mode: either --connect-value or --plug with --ref."
+      );
+    }
+    if (opts.plug && !opts.ref || !opts.plug && opts.ref) {
+      fail5(
+        "validation_error",
+        "Plug-ref lookup requires both --plug <plugName> and --ref <ref>."
+      );
+    }
+    const baseUrl = resolveBaseUrl2(opts);
+    await checkConnectivity5(baseUrl);
+    const resource = await resolveResource(baseUrl, resourceNameOrId);
+    const payload = usingConnectValue ? {
+      resourceId: resource.id,
+      connectValue: opts.connectValue
+    } : {
+      resourceId: resource.id,
+      plugName: opts.plug,
+      ref: opts.ref
+    };
+    const mapping = await fetchJson2(
+      `${baseUrl}/mappings/lookup`,
+      {
+        method: "POST",
+        headers: { "Content-Type": "application/json" },
+        body: JSON.stringify(payload)
+      }
+    );
+    output5({ ok: true, resource, mapping });
+  }
+);
+withApiOptions2(
+  mappingsCommand.command("upsert").description("Create or update one mapping by canonical connect value").argument("<resource>", "Resource name or ID").requiredOption("--connect-value <value>", "Canonical connect value").requiredOption("--refs <json>", "JSON object of refs keyed by plug name").option("--metadata <json>", "Optional JSON object of contextual metadata")
+).action(
+  async (resourceNameOrId, opts) => {
+    const baseUrl = resolveBaseUrl2(opts);
+    await checkConnectivity5(baseUrl);
+    const resource = await resolveResource(baseUrl, resourceNameOrId);
+    const refs = parseJsonObjectOption(opts.refs, "--refs");
+    const metadata = parseJsonObjectOption(opts.metadata, "--metadata");
+    const mapping = await fetchJson2(`${baseUrl}/mappings`, {
+      method: "POST",
+      headers: { "Content-Type": "application/json" },
+      body: JSON.stringify({
+        resourceId: resource.id,
+        connectValue: opts.connectValue,
+        refs,
+        metadata
+      })
+    });
+    output5({ ok: true, action: "upsert", resource, mapping });
+  }
+);
+withApiOptions2(
+  mappingsCommand.command("update").description("Update one mapping by row ID").argument("<mappingId>", "Mapping row ID").requiredOption("--resource <resource>", "Resource name or ID").requiredOption("--connect-value <value>", "Canonical connect value").requiredOption("--refs <json>", "JSON object of refs keyed by plug name").option("--metadata <json>", "Optional JSON object of contextual metadata")
+).action(
+  async (mappingId, opts) => {
+    const baseUrl = resolveBaseUrl2(opts);
+    await checkConnectivity5(baseUrl);
+    const resource = await resolveResource(baseUrl, opts.resource);
+    const refs = parseJsonObjectOption(opts.refs, "--refs");
+    const metadata = parseJsonObjectOption(opts.metadata, "--metadata");
+    const mapping = await fetchJson2(
+      `${baseUrl}/mappings/${encodeURIComponent(mappingId)}`,
+      {
+        method: "PUT",
+        headers: { "Content-Type": "application/json" },
+        body: JSON.stringify({
+          resourceId: resource.id,
+          connectValue: opts.connectValue,
+          refs,
+          metadata
+        })
+      }
+    );
+    output5({ ok: true, action: "update", resource, mapping });
+  }
+);
+withApiOptions2(
+  mappingsCommand.command("delete").description("Delete one mapping by row ID").argument("<mappingId>", "Mapping row ID")
+).action(async (mappingId, opts) => {
+  const baseUrl = resolveBaseUrl2(opts);
+  await checkConnectivity5(baseUrl);
+  const res = await fetch(`${baseUrl}/mappings/${encodeURIComponent(mappingId)}`, {
+    method: "DELETE"
+  });
+  if (!res.ok) {
+    const data = await res.json().catch(() => ({}));
+    fail5(
+      "request_failed",
+      data.error ?? `Delete request failed for mapping "${mappingId}" with status ${res.status}.`
+    );
+  }
+  output5({ ok: true, action: "delete", id: mappingId });
+});
 
 // src/cli/index.ts
 var program = new Command();
@@ -2582,4 +2865,5 @@ program.addCommand(migrateCommand);
 program.addCommand(plugCommand);
 program.addCommand(wireCommand);
 program.addCommand(flowsCommand);
+program.addCommand(mappingsCommand);
 program.parse();