@clivly/sdk 0.1.0 → 0.3.0-next.1

package/README.md

@@ -1,52 +1,222 @@
 # @clivly/sdk
 
 Connect your app to Clivly Cloud. The SDK sends a heartbeat, reports your schema
-for entity mapping, and mirrors your contacts and companies into Clivly — over
-outbound HTTPS only (no inbound ports, no tunnel).
+for entity mapping, and mirrors your contacts and companies into Clivly over
+outbound HTTPS. If you configure `syncTrigger`, Clivly can also call back into
+your app to start a sync from the dashboard.
+
+For the end-to-end onboarding flow, start with
+[`docs/guides/connecting-to-clivly-cloud.md`](../../docs/guides/connecting-to-clivly-cloud.md).
+This README stays focused on the SDK surface itself.
 
 ## Install
 
 ```bash
-npm install @clivly/sdk
+npm install @clivly/sdk @clivly/core clivly
 ```
 
+Add `drizzle-orm` when you use `@clivly/sdk/drizzle`, and `vite` when you use
+the Vite/TanStack Start plugin.
+
 ## Usage
 
 ```ts
 import { createClivlySDK } from "@clivly/sdk";
-import { and, gt } from "drizzle-orm";
+import { fromDrizzle } from "@clivly/sdk/drizzle";
+import { defineClivlyConfig } from "@clivly/core";
 import { db, participants, accounts } from "./db";
 
+// Declarative entity config — the non-deprecated source of truth. It maps your
+// host tables/columns onto CRM concepts, and `defineClivlyConfig` validates it
+// at load (warning on any non-canonical field key that would mirror as empty).
+const entities = defineClivlyConfig({
+  entities: {
+    participants: {
+      concept: "contact",
+      source: "participants",
+      fields: { name: "full_name", email: "email" },
+    },
+    accounts: {
+      concept: "company",
+      source: "accounts",
+      fields: { name: "legal_name", domain: "website" },
+    },
+  },
+});
+
 const clivly = createClivlySDK({
   apiKey: process.env.CLIVLY_API_KEY!, // sk_live_… from Settings → API keys
-  contactEntity: "participants",
-  contactFields: ["id", "full_name", "email", "account_id", "updated_at"],
+  entities,
   source: {
-    contacts: {
-      entity: "participants",
-      cursorField: "updated_at",
-      fetchPage: ({ since, limit }) =>
-        db.select().from(participants)
-          .where(since ? gt(participants.updated_at, since) : undefined)
-          .orderBy(participants.updated_at)
-          .limit(limit),
-    },
-    companies: {
-      entity: "accounts",
-      cursorField: "updated_at",
-      fetchPage: ({ since, limit }) =>
-        db.select().from(accounts)
-          .where(since ? gt(accounts.updated_at, since) : undefined)
-          .orderBy(accounts.updated_at)
-          .limit(limit),
-    },
+    contacts: fromDrizzle(db, participants, { entity: "participants", cursorField: "updatedAt" }),
+    companies: fromDrizzle(db, accounts, { entity: "accounts", cursorField: "updatedAt" }),
   },
   sync: { onBoot: true, intervalMs: 30_000, batchSize: 500 },
 });
 
+const status = await clivly.connect(); // rich result: { ok, status, reason, retryable, org? }
+if (!status.ok) throw new Error(`Clivly: ${status.reason}`);
 await clivly.start();      // heartbeat + backfill + interval delta sync
-await clivly.sync();       // force a delta sync now
+
+// Optional: let Clivly's dashboard start a sync in this app.
+const remote = createClivlySDK({
+  apiKey: process.env.CLIVLY_API_KEY!,
+  entities,
+  source: { /* … */ },
+  syncTrigger: {
+    url: "https://yourapp.com/api/clivly/tick",
+    secret: process.env.CLIVLY_TRIGGER_SECRET, // whsec_… from Integrations → Remote sync
+  },
+});
+
+export async function POST(request: Request) {
+  return remote.createClivlyHandler()(request);
+}
 ```
 
-`fetchPage` returns rows keyed by your source column names. Clivly applies the
-field map you confirmed during onboarding, so re-mapping never needs a redeploy.
+`fromDrizzle` generates the keyset-paginated `fetchPage` the SDK needs, inferring the
+tie-breaker id from the table's primary key — pass `idField` to override.
+`cursorField` and `idField` are Drizzle property keys (`updatedAt`), not database
+column names (`updated_at`). Rows are keyed by your source column names; Clivly
+applies the field map you confirmed during onboarding, so re-mapping never needs
+a redeploy. Non-Drizzle hosts can still pass a hand-written `fetchPage` closure
+instead.
+
+With `syncTrigger.url` + `syncTrigger.secret` configured, the SDK reports the
+trigger URL on each heartbeat and rejects unsigned or badly-signed trigger
+requests. Without `syncTrigger`, cloud mode is push-only: Clivly can observe
+the app but cannot start a sync from the dashboard.
+
+If you rotate a remote-sync secret from Clivly's dashboard after the encryption
+rollout, Clivly stores it encrypted at rest. The server-side deployment must
+have `CLIVLY_TRIGGER_SECRET_WRAPPING_KEY` configured, while your app continues
+to use `CLIVLY_TRIGGER_SECRET` unchanged. Existing plaintext rows remain valid
+during rollout, and older rows can be backfilled with
+`node ./scripts/backfill-sync-trigger-secrets.mjs --write`.
+
+## Vite / TanStack Start
+
+For long-lived Vite dev servers, wire the SDK lifecycle through the Vite plugin:
+
+```ts
+import { clivlyVite } from "@clivly/sdk/vite";
+import clivly from "./clivly.config";
+import { defineConfig } from "vite";
+
+export default defineConfig({
+  plugins: [clivlyVite(clivly)],
+});
+```
+
+The plugin calls `connect()` for a clear startup status line, then `start()` for
+the heartbeat and sync loop. It is dev-server only; use `runScheduled()` or
+`createClivlyHandler()` for serverless production runtimes.
+
+> **Legacy:** the flat `contactEntity` / `contactFields` form is still accepted but
+> **deprecated** — prefer the `entities` config above, which is the same model the
+> mapping table and sync engine share.
+
+## Verify your setup
+
+Put your instance in `clivly.config.ts` (`export default createClivlySDK({...})`), then:
+
+```bash
+npx clivly ping     # CLIVLY_API_KEY + API reachability
+npx clivly doctor   # env → config → schema → columns → sample fetch → heartbeat
+```
+
+In a monorepo, pass the app config explicitly:
+
+```bash
+npx clivly doctor --config ./apps/web/clivly.config.ts
+```
+
+Or in code: `await clivly.doctor()` returns `{ ok, checks }` — the same six read-only checks.
+
+## Serverless / edge
+
+On request-scoped runtimes (Cloudflare Workers, Vercel, Lambda) there is no long-lived
+process, so `start()`'s `setInterval` keep-alive loop can't persist — it stops ticking after
+the first response. **Don't call `start()` there.** Instead drive one-shot runs from a
+scheduled trigger with `runScheduled()`, or mount `createClivlyHandler()` at a route:
+
+```ts
+// Cloudflare Workers
+export default {
+  scheduled: () => clivly.runScheduled(), // cron: one heartbeat + one sync
+  fetch: clivly.createClivlyHandler(),    // or drive via an uptime ping / external scheduler
+};
+```
+
+`runScheduled()` throws if a sync push fails (so a cron run is marked failed);
+`createClivlyHandler()` returns `200` when the triggered run finishes successfully,
+`500` otherwise. If the run carries a Clivly-issued run id, the SDK also reports
+the final outcome back to Clivly so the dashboard reflects `running` vs
+`succeeded` vs `failed` truthfully. `start()` emits a warning if it detects a
+serverless runtime. Verify a connection from the terminal any time with
+`npx clivly ping`.
+
+### Persisting the delta cursor
+
+Delta sync tracks a per-entity cursor so each run only pages rows changed since the last one.
+By default that cursor lives in memory — fine for a long-lived process, but **lost on every
+serverless cold start**, so a fresh isolate would re-page the whole entity. Pass a
+`cursorStore` backed by your own KV/DB to persist it:
+
+```ts
+const clivly = createClivlySDK({
+  apiKey: process.env.CLIVLY_API_KEY!,
+  entities,
+  source: { /* … */ },
+  cursorStore: {
+    // `SyncCursor` is { id: string; time: Date } — serialize `time` (e.g. ISO string).
+    get: async (entity) => {
+      const raw = await kv.get(`clivly:cursor:${entity}`);
+      return raw ? { id: raw.id, time: new Date(raw.time) } : null;
+    },
+    set: (entity, cursor) =>
+      kv.put(`clivly:cursor:${entity}`, cursor && { id: cursor.id, time: cursor.time.toISOString() }),
+  },
+});
+```
+
+Delta sync is **at-least-once**: after a cold start with no persisted cursor, rows since the
+last push may be re-sent (the server dedupes by id) — it never silently skips rows. Use
+`sync({ mode: "full" })` when you need the server to reconcile deletions (archive-on-leave).
+
+## Many-to-many company membership
+
+If a mapping links contacts to companies through a join table
+(`via: { through: … }`), that entity's source must return **one row per
+membership**, each row carrying the join table's foreign-key column.
+
+```ts
+// Ada belongs to two organizations: two rows, differing only in org_id.
+{ id: "u1", full_name: "Ada", email: "ada@acme.test", org_id: "acme"   }
+{ id: "u1", full_name: "Ada", email: "ada@acme.test", org_id: "globex" }
+```
+
+This mirrors what the embedded model produces with `LEFT JOIN <join table>`, and
+it is what lets Clivly keep one CRM row per membership instead of collapsing the
+person onto whichever company happened to come first.
+
+A person with no membership may omit the column entirely or send it as `null` —
+both are accepted. But if **no** pushed row carries the column, the push is
+rejected with `400 invalid_config`: silently keeping one membership would be
+worse than failing, because the row counts look correct either way.
+
+## Known limitations
+
+- `start()` is for long-lived runtimes only. On serverless/edge, use
+  `runScheduled()` or `createClivlyHandler()`.
+- `fromDrizzle` is the first-class generated source path today. Non-Drizzle hosts
+  should provide a hand-written `fetchPage`.
+- The flat `contactEntity` / `contactFields` config still works but is deprecated
+  in favor of `defineClivlyConfig({ entities: ... })`.
+
+## More docs
+
+- Onboarding guide:
+  [`docs/guides/connecting-to-clivly-cloud.md`](../../docs/guides/connecting-to-clivly-cloud.md)
+- CLI docs: [`../cli/README.md`](../cli/README.md)
+- Release notes: [`../../CHANGELOG.md`](../../CHANGELOG.md)

package/dist/drizzle-meta-DMx_9nlj.mjs

@@ -0,0 +1,4 @@
+//#region src/drizzle-meta.ts
+const DRIZZLE_META = "__clivly_drizzle_source_meta__";
+//#endregion
+export { DRIZZLE_META as t };

package/dist/drizzle-meta-iAu8w1hj.cjs

@@ -0,0 +1,9 @@
+//#region src/drizzle-meta.ts
+const DRIZZLE_META = "__clivly_drizzle_source_meta__";
+//#endregion
+Object.defineProperty(exports, "DRIZZLE_META", {
+	enumerable: true,
+	get: function() {
+		return DRIZZLE_META;
+	}
+});

package/dist/drizzle.cjs

@@ -0,0 +1,107 @@
+Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
+const require_drizzle_meta = require("./drizzle-meta-iAu8w1hj.cjs");
+let drizzle_orm = require("drizzle-orm");
+let drizzle_orm_pg_core = require("drizzle-orm/pg-core");
+//#region src/drizzle.ts
+function buildColumnsMeta(table) {
+	const columns = (0, drizzle_orm.getTableColumns)(table);
+	const fkByColumn = /* @__PURE__ */ new Map();
+	try {
+		for (const fk of (0, drizzle_orm_pg_core.getTableConfig)(table).foreignKeys) {
+			const ref = fk.reference();
+			const foreignTable = (0, drizzle_orm.getTableName)(ref.foreignTable);
+			ref.columns.forEach((col, i) => {
+				const foreignCol = ref.foreignColumns[i];
+				if (foreignCol) fkByColumn.set(col.name, {
+					table: foreignTable,
+					column: foreignCol.name
+				});
+			});
+		}
+	} catch {}
+	return Object.values(columns).map((col) => {
+		const references = fkByColumn.get(col.name);
+		let type;
+		try {
+			type = col.getSQLType();
+		} catch {
+			type = void 0;
+		}
+		return {
+			name: col.name,
+			type,
+			nullable: !col.notNull,
+			isPrimaryKey: Boolean(col.primary),
+			isForeignKey: references !== void 0,
+			...references ? { references } : {}
+		};
+	});
+}
+function inferIdField(table) {
+	const columns = (0, drizzle_orm.getTableColumns)(table);
+	const primaryKeys = Object.entries(columns).filter(([, column]) => column.primary);
+	if (primaryKeys.length === 1) {
+		const primaryKeyField = primaryKeys[0]?.[0];
+		if (!primaryKeyField) throw new Error(`fromDrizzle: unable to extract primary key field name from table "${(0, drizzle_orm.getTableName)(table)}".`);
+		return primaryKeyField;
+	}
+	const kind = primaryKeys.length === 0 ? "no" : "a composite";
+	throw new Error(`fromDrizzle: table "${(0, drizzle_orm.getTableName)(table)}" has ${kind} single-column primary key — pass { idField } explicitly.`);
+}
+/**
+* Build a `SyncSource` from a Drizzle table, generating the keyset-paginated
+* `fetchPage` the SDK needs. `cursorField`/`idField` are drizzle property keys;
+* `idField` defaults to the table's single-column primary key.
+*
+* `drizzle-orm` is an optional peer dependency — installed only when you import
+* `@clivly/sdk/drizzle`.
+*/
+function fromDrizzle(db, table, opts) {
+	const drizzleDb = db;
+	const idField = opts.idField ?? inferIdField(table);
+	const columns = (0, drizzle_orm.getTableColumns)(table);
+	const cursorColumn = columns[opts.cursorField];
+	const idColumn = columns[idField];
+	if (!cursorColumn) throw new Error(`fromDrizzle: cursorField "${opts.cursorField}" is not a column on "${(0, drizzle_orm.getTableName)(table)}".`);
+	if (!idColumn) throw new Error(`fromDrizzle: idField "${idField}" is not a column on "${(0, drizzle_orm.getTableName)(table)}".`);
+	const source = {
+		entity: opts.entity,
+		cursorField: opts.cursorField,
+		idField,
+		fetchPage: ({ since, limit }) => drizzleDb.select().from(table).where(since ? drizzle_orm.sql`(${cursorColumn}, ${idColumn}) > (${since.time}, ${since.id})` : void 0).orderBy((0, drizzle_orm.asc)(cursorColumn), (0, drizzle_orm.asc)(idColumn)).limit(limit)
+	};
+	const meta = {
+		columnKeys: Object.keys(columns),
+		cursorField: opts.cursorField,
+		idField,
+		tableName: (0, drizzle_orm.getTableName)(table),
+		columnsMeta: buildColumnsMeta(table)
+	};
+	Object.defineProperty(source, require_drizzle_meta.DRIZZLE_META, {
+		value: meta,
+		enumerable: false
+	});
+	return source;
+}
+/**
+* Build a `NamespaceIntrospector` from any Drizzle client that exposes
+* `.execute()`. Reads `information_schema` in the connection's default schema
+* (`current_schema()`), so it reports exactly the tables the app can reach.
+*/
+function drizzleIntrospector(db) {
+	const client = db;
+	return {
+		async listTables() {
+			return (await client.execute(drizzle_orm.sql`
+        select table_name from information_schema.tables
+        where table_schema = current_schema()
+      `)).rows.map((r) => String(r.table_name));
+		},
+		async sampleSelect(table) {
+			await client.execute(drizzle_orm.sql`select 1 from ${drizzle_orm.sql.identifier(table)} limit 1`);
+		}
+	};
+}
+//#endregion
+exports.drizzleIntrospector = drizzleIntrospector;
+exports.fromDrizzle = fromDrizzle;

package/dist/drizzle.d.cts

@@ -0,0 +1,28 @@
+import { d as SyncSource, t as NamespaceIntrospector } from "./namespace-probe-DIBgCQca.cjs";
+import { Table } from "drizzle-orm";
+
+//#region src/drizzle.d.ts
+interface DrizzleDb {
+  select: (...args: any[]) => any;
+}
+/**
+ * Build a `SyncSource` from a Drizzle table, generating the keyset-paginated
+ * `fetchPage` the SDK needs. `cursorField`/`idField` are drizzle property keys;
+ * `idField` defaults to the table's single-column primary key.
+ *
+ * `drizzle-orm` is an optional peer dependency — installed only when you import
+ * `@clivly/sdk/drizzle`.
+ */
+declare function fromDrizzle(db: DrizzleDb, table: Table, opts: {
+  entity: string;
+  cursorField: string;
+  idField?: string;
+}): SyncSource;
+/**
+ * Build a `NamespaceIntrospector` from any Drizzle client that exposes
+ * `.execute()`. Reads `information_schema` in the connection's default schema
+ * (`current_schema()`), so it reports exactly the tables the app can reach.
+ */
+declare function drizzleIntrospector(db: unknown): NamespaceIntrospector;
+//#endregion
+export { drizzleIntrospector, fromDrizzle };

package/dist/drizzle.d.mts

@@ -0,0 +1,28 @@
+import { d as SyncSource, t as NamespaceIntrospector } from "./namespace-probe-DFpvfl7T.mjs";
+import { Table } from "drizzle-orm";
+
+//#region src/drizzle.d.ts
+interface DrizzleDb {
+  select: (...args: any[]) => any;
+}
+/**
+ * Build a `SyncSource` from a Drizzle table, generating the keyset-paginated
+ * `fetchPage` the SDK needs. `cursorField`/`idField` are drizzle property keys;
+ * `idField` defaults to the table's single-column primary key.
+ *
+ * `drizzle-orm` is an optional peer dependency — installed only when you import
+ * `@clivly/sdk/drizzle`.
+ */
+declare function fromDrizzle(db: DrizzleDb, table: Table, opts: {
+  entity: string;
+  cursorField: string;
+  idField?: string;
+}): SyncSource;
+/**
+ * Build a `NamespaceIntrospector` from any Drizzle client that exposes
+ * `.execute()`. Reads `information_schema` in the connection's default schema
+ * (`current_schema()`), so it reports exactly the tables the app can reach.
+ */
+declare function drizzleIntrospector(db: unknown): NamespaceIntrospector;
+//#endregion
+export { drizzleIntrospector, fromDrizzle };

package/dist/drizzle.mjs

@@ -0,0 +1,105 @@
+import { t as DRIZZLE_META } from "./drizzle-meta-DMx_9nlj.mjs";
+import { asc, getTableColumns, getTableName, sql } from "drizzle-orm";
+import { getTableConfig } from "drizzle-orm/pg-core";
+//#region src/drizzle.ts
+function buildColumnsMeta(table) {
+	const columns = getTableColumns(table);
+	const fkByColumn = /* @__PURE__ */ new Map();
+	try {
+		for (const fk of getTableConfig(table).foreignKeys) {
+			const ref = fk.reference();
+			const foreignTable = getTableName(ref.foreignTable);
+			ref.columns.forEach((col, i) => {
+				const foreignCol = ref.foreignColumns[i];
+				if (foreignCol) fkByColumn.set(col.name, {
+					table: foreignTable,
+					column: foreignCol.name
+				});
+			});
+		}
+	} catch {}
+	return Object.values(columns).map((col) => {
+		const references = fkByColumn.get(col.name);
+		let type;
+		try {
+			type = col.getSQLType();
+		} catch {
+			type = void 0;
+		}
+		return {
+			name: col.name,
+			type,
+			nullable: !col.notNull,
+			isPrimaryKey: Boolean(col.primary),
+			isForeignKey: references !== void 0,
+			...references ? { references } : {}
+		};
+	});
+}
+function inferIdField(table) {
+	const columns = getTableColumns(table);
+	const primaryKeys = Object.entries(columns).filter(([, column]) => column.primary);
+	if (primaryKeys.length === 1) {
+		const primaryKeyField = primaryKeys[0]?.[0];
+		if (!primaryKeyField) throw new Error(`fromDrizzle: unable to extract primary key field name from table "${getTableName(table)}".`);
+		return primaryKeyField;
+	}
+	const kind = primaryKeys.length === 0 ? "no" : "a composite";
+	throw new Error(`fromDrizzle: table "${getTableName(table)}" has ${kind} single-column primary key — pass { idField } explicitly.`);
+}
+/**
+* Build a `SyncSource` from a Drizzle table, generating the keyset-paginated
+* `fetchPage` the SDK needs. `cursorField`/`idField` are drizzle property keys;
+* `idField` defaults to the table's single-column primary key.
+*
+* `drizzle-orm` is an optional peer dependency — installed only when you import
+* `@clivly/sdk/drizzle`.
+*/
+function fromDrizzle(db, table, opts) {
+	const drizzleDb = db;
+	const idField = opts.idField ?? inferIdField(table);
+	const columns = getTableColumns(table);
+	const cursorColumn = columns[opts.cursorField];
+	const idColumn = columns[idField];
+	if (!cursorColumn) throw new Error(`fromDrizzle: cursorField "${opts.cursorField}" is not a column on "${getTableName(table)}".`);
+	if (!idColumn) throw new Error(`fromDrizzle: idField "${idField}" is not a column on "${getTableName(table)}".`);
+	const source = {
+		entity: opts.entity,
+		cursorField: opts.cursorField,
+		idField,
+		fetchPage: ({ since, limit }) => drizzleDb.select().from(table).where(since ? sql`(${cursorColumn}, ${idColumn}) > (${since.time}, ${since.id})` : void 0).orderBy(asc(cursorColumn), asc(idColumn)).limit(limit)
+	};
+	const meta = {
+		columnKeys: Object.keys(columns),
+		cursorField: opts.cursorField,
+		idField,
+		tableName: getTableName(table),
+		columnsMeta: buildColumnsMeta(table)
+	};
+	Object.defineProperty(source, DRIZZLE_META, {
+		value: meta,
+		enumerable: false
+	});
+	return source;
+}
+/**
+* Build a `NamespaceIntrospector` from any Drizzle client that exposes
+* `.execute()`. Reads `information_schema` in the connection's default schema
+* (`current_schema()`), so it reports exactly the tables the app can reach.
+*/
+function drizzleIntrospector(db) {
+	const client = db;
+	return {
+		async listTables() {
+			return (await client.execute(sql`
+        select table_name from information_schema.tables
+        where table_schema = current_schema()
+      `)).rows.map((r) => String(r.table_name));
+		},
+		async sampleSelect(table) {
+			await client.execute(sql`select 1 from ${sql.identifier(table)} limit 1`);
+		}
+	};
+}
+//#endregion
+export { drizzleIntrospector, fromDrizzle };