khotan-data 0.2.0 → 0.3.1

package/README.md

@@ -1,8 +1,8 @@
 # khotan-data
 
-Data primitives for TypeScript — ETL pipelines, transforms, and Drizzle Postgres integration.
+Data sync, ETL, and webhook primitives for Next.js + Drizzle + Postgres. shadcn for data plumbing.
 
-Built for **Next.js + Drizzle + Postgres** projects. Think better-auth for data management.
+Built for **Next.js + Drizzle + Postgres** projects. Think shadcn × better-auth, but for data.
 
 ## Install
 
@@ -99,7 +99,13 @@ authorize: async (request) => {
 ```
 
 - `KHOTAN_SECRET` encrypts plug credentials **at rest** (AES-256-GCM). It is not
-  an auth credential — it never gates requests. Set it to a high-entropy value.
+  an auth credential — it never gates requests, and **must not** be sent as a
+  `Bearer` token. Management routes are gated only by `authorize` (plus a
+  dev-only CLI HMAC token derived from the secret). A rejected request returns
+  `401` with `code: "authorize_rejected"` and a `hint`. To trigger a flow over
+  HTTP (`POST /api/khotan/flows/{flowId}/runs`), send a credential your
+  `authorize` hook accepts — or just call `khotanData.flow(name).start()` from
+  server code, which needs no auth. Set the secret to a high-entropy value.
 - Inbound webhooks (verified via per-plug `onVerify`), the cron dispatcher
   (`CRON_SECRET`), and debug routes (`KHOTAN_DEBUG`, non-production only) are
   exempt from `authorize` automatically.
@@ -128,32 +134,38 @@ export const shopifyProductsSnapshotCache = cache({
 
 Inside workflows, use `khotanCache(ctx, "name")` for snapshots, cursors, and dedupe markers:
 
+Declare `"use step"` functions at module top level and pass them serializable
+values only (`ctx` is plain data). Nesting steps inside the `"use workflow"`
+function fails at runtime — the Workflow compiler cannot hoist closures that
+capture workflow scope.
+
 ```typescript
 import { khotanCache } from "khotan-data/factory";
 
-async function shopifyProductsWorkflow(ctx: InflowContext) {
-  "use workflow";
-
-  async function syncProducts() {
-    "use step";
-    const snapshotCache = khotanCache(ctx, "shopify-products-snapshot");
-    const previous =
-      (await snapshotCache.get<Array<Record<string, unknown>>>("latest")) ?? [];
+// Step: top-level, retried independently, full Node.js access.
+async function syncProducts(ctx: InflowContext) {
+  "use step";
+  const snapshotCache = khotanCache(ctx, "shopify-products-snapshot");
+  const previous =
+    (await snapshotCache.get<Array<Record<string, unknown>>>("latest")) ?? [];
 
-    const response = await shopifyPlug.get<{ data?: Array<Record<string, unknown>> }>("/products");
-    const records = Array.isArray(response.data) ? response.data : [];
+  const response = await shopifyPlug.get<{ data?: Array<Record<string, unknown>> }>("/products");
+  const records = Array.isArray(response.data) ? response.data : [];
 
-    await snapshotCache.set("latest", records);
+  await snapshotCache.set("latest", records);
 
-    return {
-      extracted: records.length,
-      transformed: records.length,
-      created: records.length,
-      metadata: { previousCount: previous.length },
-    };
-  }
+  return {
+    extracted: records.length,
+    transformed: records.length,
+    created: records.length,
+    metadata: { previousCount: previous.length },
+  };
+}
 
-  return syncProducts();
+// Workflow: orchestration only.
+async function shopifyProductsWorkflow(ctx: InflowContext) {
+  "use workflow";
+  return syncProducts(ctx);
 }
 ```
 

package/dist/cli.js

@@ -46,7 +46,7 @@ function checkNpmPackages(cwd, packages) {
       ...pkgJson.dependencies,
       ...pkgJson.devDependencies
     };
-    return packages.filter((pkg) => !(pkg in allDeps));
+    return packages.filter((pkg2) => !(pkg2 in allDeps));
   } catch {
     return packages;
   }
@@ -678,6 +678,15 @@ function resolveAgentsMdPaths(content, targets) {
 // src/cli/commands/init.ts
 var __dirname2 = path2.dirname(fileURLToPath(import.meta.url));
 function resolveOutputDir(projectRoot) {
+  const configPath = path2.join(projectRoot, "khotan.config.ts");
+  if (fs4.existsSync(configPath)) {
+    try {
+      const content = fs4.readFileSync(configPath, "utf-8");
+      const match = /outputDir:\s*["']([^"']+)["']/.exec(content);
+      if (match?.[1]) return match[1];
+    } catch {
+    }
+  }
   if (fs4.existsSync(path2.join(projectRoot, "src", "app"))) {
     return "src/khotan";
   }
@@ -699,11 +708,6 @@ function scaffoldCoreFiles(cwd, outputDir) {
     "templates",
     "khotan-config.ts"
   );
-  const routeTemplatePath = path2.resolve(
-    __dirname2,
-    "templates",
-    "khotan-route.ts"
-  );
   const khotanTsPath = path2.join(path2.resolve(cwd, outputDir), "khotan.ts");
   if (!fs4.existsSync(khotanTsPath)) {
     fs4.mkdirSync(path2.dirname(khotanTsPath), { recursive: true });
@@ -721,13 +725,67 @@ function scaffoldCoreFiles(cwd, outputDir) {
   const routePath = path2.join(routeDir, "route.ts");
   if (!fs4.existsSync(routePath)) {
     fs4.mkdirSync(routeDir, { recursive: true });
-    fs4.copyFileSync(routeTemplatePath, routePath);
+    const khotanImportPath = outputDir.startsWith("src/") ? `@/${outputDir.slice(4)}/khotan` : `@/${outputDir}/khotan`;
+    const routeContent = `// ============================================================================
+// Khotan API Route \u2014 catch-all handler for /api/khotan/*
+// Generated by khotan CLI \xB7 https://github.com/khotan-io/khotan-data
+//
+// This file wires your khotan config into a Next.js App Router route.
+// ============================================================================
+
+import { toNextJsHandler } from "khotan-data/factory";
+import khotanData from "${khotanImportPath}";
+
+export const { GET, POST, PUT, PATCH, DELETE } = toNextJsHandler(
+  khotanData.handler,
+);
+`;
+    fs4.writeFileSync(routePath, routeContent, "utf-8");
     created.push(path2.relative(cwd, routePath));
   } else {
     console.log(`\u2713 ${path2.relative(cwd, routePath)} already exists, skipping`);
   }
   return created;
 }
+var MIDDLEWARE_CANDIDATES = [
+  "middleware.ts",
+  "middleware.js",
+  "src/middleware.ts",
+  "src/middleware.js",
+  "proxy.ts",
+  "proxy.js",
+  "src/proxy.ts",
+  "src/proxy.js"
+];
+function warnAboutWorkflowProxy(cwd) {
+  const found = MIDDLEWARE_CANDIDATES.map((rel) => ({
+    rel,
+    abs: path2.join(cwd, rel)
+  })).find((c) => fs4.existsSync(c.abs));
+  if (!found) return false;
+  let contents = "";
+  try {
+    contents = fs4.readFileSync(found.abs, "utf-8");
+  } catch {
+    return false;
+  }
+  if (/\.well-known|workflow/i.test(contents)) {
+    return false;
+  }
+  console.log(
+    `
+\u26A0 Detected ${found.rel}. Vercel Workflow (used by inflows, outflows,
+  relays, catch, and pass) communicates over /.well-known/workflow/*.
+  If your middleware/proxy matcher captures these paths, durable runs
+  will silently fail. Exclude them from your matcher, e.g.:
+
+    export const config = {
+      matcher: ["/((?!_next|.well-known/workflow).*)"],
+    };
+`
+  );
+  return true;
+}
 async function runFullSetup(cwd) {
   const results = [];
   const pm = detectPackageManager(cwd);
@@ -839,6 +897,14 @@ Installing shadcn components: ${missingShadcn.join(", ")}...`
     results.push({ name: "Scaffold core files", status: "skipped" });
   }
   results.push(ensureKhotanDataInstalled(cwd));
+  warnAboutWorkflowProxy(cwd);
+  const skillCount = scaffoldAgentSkills(cwd);
+  if (skillCount > 0) {
+    console.log(`\u2713 Installed ${String(skillCount)} agent skills`);
+    results.push({ name: "Install agent skills", status: "success" });
+  } else {
+    results.push({ name: "Install agent skills", status: "skipped" });
+  }
   return results;
 }
 function ensureKhotanDataInstalled(cwd) {
@@ -867,6 +933,7 @@ async function runInit(cwd) {
   }
   scaffoldCoreFiles(cwd, outputDir);
   ensureKhotanDataInstalled(cwd);
+  warnAboutWorkflowProxy(cwd);
   return fs4.existsSync(configPath);
 }
 var SKILL_COMPONENTS = [
@@ -939,6 +1006,7 @@ ${String(failed.length)} step(s) failed. You may need to run them manually.`
   }
   const coreFiles = scaffoldCoreFiles(cwd, outputDir);
   ensureKhotanDataInstalled(cwd);
+  warnAboutWorkflowProxy(cwd);
   let installSkills2 = opts.yes ?? false;
   if (!installSkills2 && process.stdin.isTTY) {
     const response = await prompts2({
@@ -1805,13 +1873,13 @@ function diffSchemas(expected, actual, basePath = "$") {
   }
   return diffObjectSchema(expected, actual, basePath);
 }
-function diffTypedNode(expected, actual, path16) {
+function diffTypedNode(expected, actual, path17) {
   const expectedType = expected["_type"];
   if (expectedType === "array") {
     if (actual.type !== "array") {
       return [
         {
-          path: path16,
+          path: path17,
           issue: "type_mismatch",
           note: `expected array, got ${actual.type}`
         }
@@ -1819,13 +1887,13 @@ function diffTypedNode(expected, actual, path16) {
     }
     const itemSchema = expected["items"];
     if (!itemSchema || !actual.items) return [];
-    return diffSchemas(itemSchema, actual.items, `${path16}[]`);
+    return diffSchemas(itemSchema, actual.items, `${path17}[]`);
   }
   const normalizedExpected = normalizeType(expectedType);
   if (normalizedExpected !== actual.type && actual.type !== "null") {
     return [
       {
-        path: path16,
+        path: path17,
         issue: "type_mismatch",
         note: `expected ${expectedType}, got ${actual.type}`
       }
@@ -1833,11 +1901,11 @@ function diffTypedNode(expected, actual, path16) {
   }
   return [];
 }
-function diffObjectSchema(expected, actual, path16) {
+function diffObjectSchema(expected, actual, path17) {
   if (actual.type !== "object") {
     return [
       {
-        path: path16,
+        path: path17,
         issue: "type_mismatch",
         note: `expected object, got ${actual.type}`
       }
@@ -1846,7 +1914,7 @@ function diffObjectSchema(expected, actual, path16) {
   const mismatches = [];
   const actualProps = actual.properties;
   for (const [key, typeDesc] of Object.entries(expected)) {
-    const childPath = path16 === "$" ? `$.${key}` : `${path16}.${key}`;
+    const childPath = path17 === "$" ? `$.${key}` : `${path17}.${key}`;
     const typeStr = typeof typeDesc === "string" ? typeDesc : null;
     const isOptional = typeStr?.endsWith("?") ?? false;
     if (!(key in actualProps)) {
@@ -1884,7 +1952,7 @@ function diffObjectSchema(expected, actual, path16) {
   }
   for (const key of Object.keys(actualProps)) {
     if (!(key in expected)) {
-      const childPath = path16 === "$" ? `$.${key}` : `${path16}.${key}`;
+      const childPath = path17 === "$" ? `$.${key}` : `${path17}.${key}`;
       mismatches.push({ path: childPath, issue: "extra" });
     }
   }
@@ -2942,8 +3010,12 @@ withApiOptions2(
 });
 
 // src/cli/index.ts
+var __cliDirname = path2.dirname(fileURLToPath(import.meta.url));
+var pkg = JSON.parse(
+  fs4.readFileSync(path2.resolve(__cliDirname, "..", "package.json"), "utf-8")
+);
 var program = new Command();
-program.name("khotan").description("Scaffold data components into your project").version("0.0.1");
+program.name("khotan").description("Scaffold data components into your project").version(pkg.version);
 program.addCommand(initCommand);
 program.addCommand(addCommand);
 program.addCommand(generateCommand);

package/dist/factory.cjs

@@ -2301,7 +2301,14 @@ function khotan(config) {
         }
       }
       if (!allowed) {
-        return Response.json({ error: "Unauthorized" }, { status: 401 });
+        return Response.json(
+          {
+            error: "Unauthorized",
+            code: "authorize_rejected",
+            hint: "Management routes (/api/khotan/*) require your `authorize` hook to pass. KHOTAN_SECRET is an encryption key, not an HTTP credential \u2014 sending it as a Bearer token will not authenticate the request. To trigger a flow: call khotanData.flow(name).start() from server code (no HTTP/auth needed), or send a credential your authorize hook accepts (e.g. a session cookie or your own token). The khotan CLI authenticates automatically via a dev-only token derived from KHOTAN_SECRET."
+          },
+          { status: 401 }
+        );
       }
     }
     const limit = Math.min(