clivly 0.9.5 → 0.9.7

package/{sdk-CM1UO-I7.cjs → sdk-BL9BCzyD.cjs}

@@ -1,5 +1,6 @@
-const require_entity_config = require("./entity-config-7sGCk_nw.cjs");
-const require_mappings_config = require("./mappings-config-CLyEg0sf.cjs");
+const require_projection = require("./projection-DEwL-D8R.cjs");
+const require_entity_config = require("./entity-config-CPM-Tp-d.cjs");
+const require_mappings_config = require("./mappings-config-CBwL3FVh.cjs");
 const require_drizzle_meta = require("./drizzle-meta-Bb1JWYqf.cjs");
 //#region ../sdk/src/deployment-identity.ts
 const MAX_DEPLOYMENT_ID_LENGTH = 128;
@@ -572,12 +573,21 @@ async function runDoctor(args) {
 		ok: entityCount > 0,
 		detail: entityCount > 0 ? `${entityCount} entit${entityCount === 1 ? "y" : "ies"} configured.` : "No `entities` config — pass defineClivlyConfig(...) as `entities`."
 	});
+	const sourcesDetail = (count, deferred) => {
+		if (deferred) return "Deferred — the coverage check below names the entity and the source key it expected.";
+		return count === 0 ? "No `source.contacts` — nothing to sync." : `${count} source(s) to mirror.`;
+	};
+	const coverage = checkConceptCoverage(config);
+	const coverageOwnsIt = !(coverage.ok || coverage.skipped);
+	const noSources = sources.length === 0;
+	const deferred = noSources && coverageOwnsIt;
 	checks.push({
 		name: "Sources configured",
-		ok: sources.length > 0,
-		detail: sources.length > 0 ? `${sources.length} source(s) to mirror.` : "No `source.contacts` — nothing to sync."
+		ok: !noSources,
+		skipped: deferred,
+		detail: sourcesDetail(sources.length, deferred)
 	});
-	checks.push(checkConceptCoverage(config));
+	checks.push(coverage);
 	checks.push(...cursorColumnChecks(sources));
 	for (const source of sources) {
 		const check = cursorMonotonicityCheck(source);
@@ -980,6 +990,240 @@ function describeSilentPush(target, body) {
 	return `Clivly sync: pushed ${received} row(s) for ${objectType ? `"${source}" → "${objectType}"` : `"${source}"`} and none were stored — no active mapping in this workspace targets it, so the rows were dropped. Map it in Integrations → Mapping, then sync again.`;
 }
 //#endregion
+//#region ../sdk/src/remote-projections.ts
+const SYNC_CUSTOM_PATH$1 = "/clivly/sdk/sync/custom";
+const MISSING_INTROSPECTOR = "this app cannot run dashboard-authored derived objects yet — add `introspector: drizzleIntrospector(db)` to clivly.config.ts";
+/**
+* Turn definitions fetched from the cloud into runnable sync sources.
+*
+* Revalidates every projection locally against the schema THIS app discovered,
+* never against the cloud's stored copy. Two reasons, and both matter:
+*
+* 1. Security. `checkTablesExist`/`checkRefs` bound every table and column ref
+*    to the schema passed in, so validating against the locally-discovered
+*    schema makes the blast radius exactly the namespace the developer handed
+*    to `discoverFromDrizzle`. Trusting the cloud's verdict instead would let a
+*    compromised cloud read any table in the database.
+* 2. Truth. The stored copy can be stale. A column dropped in the host app
+*    becomes a readable quarantine here instead of a raw Postgres error out of
+*    `sql.raw` at execution time.
+*/
+function resolveRemoteProjections(args) {
+	const { entries, introspector, schema } = args;
+	return entries.map((entry) => resolveOne(entry, schema, introspector));
+}
+/**
+* The base table's PK, resolved from the schema THIS app discovered — never
+* from the cloud's copy.
+*
+* `validateProjection` proved its uniqueness case against this exact source
+* (`primaryKeyColumns` over the locally-discovered schema, see
+* `projection-validate.ts`). Compiling a *different* column — the cloud's
+* `basePkColumn`, which can go stale the moment a host renames a PK and the
+* `updatedAt`-only ETag never invalidates — would defeat that proof and
+* revive the raw-Postgres-error failure mode local revalidation exists to
+* prevent. Only relevant when the identity is `base_pk` with no explicit
+* column; every other identity kind ignores `basePkColumn` entirely.
+*/
+function resolveLocalBasePk(projection, schema) {
+	if (projection.identity.kind !== "base_pk" || projection.identity.column) return;
+	const baseTable = schema.find((t) => t.name === projection.baseTable);
+	if (!baseTable) return;
+	return require_entity_config.primaryKeyColumns(baseTable)[0];
+}
+function resolveOne(entry, schema, introspector) {
+	const parsed = require_projection.projectionSchema.safeParse(entry.projection);
+	if (!parsed.success) return {
+		id: entry.id,
+		status: "invalid",
+		reason: `projection did not parse: ${parsed.error.issues.map((i) => `${i.path.join(".")}: ${i.message}`).join("; ")}`
+	};
+	const projection = parsed.data;
+	const errors = require_entity_config.validateProjection(projection, schema, entry.entityKey);
+	if (errors.length > 0) return {
+		id: entry.id,
+		status: "invalid",
+		reason: errors.map((e) => `${e.path}: ${e.message}`).join("; ")
+	};
+	if (!introspector?.projectionSource) return {
+		id: entry.id,
+		status: "skipped",
+		reason: MISSING_INTROSPECTOR
+	};
+	try {
+		const basePkColumn = resolveLocalBasePk(projection, schema) ?? entry.basePkColumn;
+		const source = introspector.projectionSource(projection, {
+			objectType: entry.objectType,
+			...basePkColumn ? { basePkColumn } : {}
+		});
+		return {
+			id: entry.id,
+			status: "ok",
+			source
+		};
+	} catch (error) {
+		return {
+			id: entry.id,
+			status: "failed",
+			reason: error instanceof Error ? error.message : String(error)
+		};
+	}
+}
+const PROJECTIONS_PATH = "/clivly/sdk/projections";
+const PROJECTIONS_REPORT_PATH = "/clivly/sdk/projections/report";
+const MAX_REASON = 2e3;
+/**
+* Merge what resolution decided with what the push actually did.
+*
+* A `status: "ok"` outcome means the projection *resolved*, not that its rows
+* landed. If the push then failed, the object did not sync and must not be
+* reported as healthy — so the push result wins for those.
+*/
+function buildRemoteProjectionReport(args) {
+	return {
+		...args.runId ? { runId: args.runId } : {},
+		results: args.outcomes.map((outcome) => {
+			if (outcome.status !== "ok") return {
+				id: outcome.id,
+				status: outcome.status,
+				reason: outcome.reason.slice(0, MAX_REASON)
+			};
+			const result = args.pushed.get(outcome.source.entity);
+			if (result?.ok) return {
+				id: outcome.id,
+				status: "ok",
+				rows: result.count
+			};
+			return {
+				id: outcome.id,
+				status: "failed",
+				reason: (result?.reason ?? "the push to Clivly failed; rows were read but not stored").slice(0, MAX_REASON)
+			};
+		})
+	};
+}
+function createRemoteProjectionFetcher(args) {
+	let etag = null;
+	let cached = [];
+	return { async fetch() {
+		try {
+			const response = await args.fetcher(`${args.apiUrl}${PROJECTIONS_PATH}`, { headers: {
+				authorization: `Bearer ${args.apiKey}`,
+				...etag ? { "if-none-match": etag } : {}
+			} });
+			if (response.status === 304) return cached;
+			if (!response.ok) return [];
+			const body = await response.json();
+			etag = body.etag ?? null;
+			cached = body.projections ?? [];
+			return cached;
+		} catch {
+			return [];
+		}
+	} };
+}
+const REMOTE_PAGE_SIZE = 500;
+/**
+* Page one source to exhaustion, keyset-paging on identity alone (the id
+* field, no timestamp). Split out of `pushRemoteProjections` so that
+* function's own cognitive complexity stays under the lint threshold — this
+* is the piece that can throw (a `fetchPage` execution error), kept separate
+* from push-failure handling below.
+*/
+async function fetchAllPages(source, pageSize) {
+	const idField = source.idField ?? "id";
+	const all = [];
+	let since = null;
+	for (;;) {
+		const rows = await source.fetchPage({
+			since,
+			limit: pageSize
+		});
+		if (rows.length === 0) break;
+		all.push(...rows);
+		const last = rows.at(-1);
+		const nextId = String(last?.[idField]);
+		if (since && nextId === since.id) break;
+		since = { id: nextId };
+		if (rows.length < pageSize) break;
+	}
+	return all;
+}
+/**
+* Page a remote projection by identity and push one `full` batch.
+*
+* `full` rather than `delta` because only a full push lets the server archive
+* rows that dropped out when a join or filter stopped matching — a delta cannot
+* express absence. And identity rather than a timestamp because a
+* dashboard-authored projection has no watermark column, while validation has
+* already proven the identity unique.
+*
+* `pageSize` defaults to `REMOTE_PAGE_SIZE` and exists so a test can force a
+* short page without either mutating the exported constant or constructing a
+* fixture hundreds of rows deep.
+*
+* Returns one entry per source, keyed by `source.entity`. `count` and `ok`
+* are independent: `count` is how many rows were pushed (0 for a projection
+* that legitimately matches nothing this run), `ok` is whether the push
+* itself succeeded. A caller that folds failures into a report must branch
+* on `ok`, never on `count === 0` — a correctly-working, legitimately-empty
+* projection has `ok: true, count: 0` and is not a failure.
+*
+* Each source's paging-and-push is wrapped individually: a projection can
+* validate cleanly and still fail at execution — permission denied, a type
+* error, a column dropped between validation and this run — and `fetchPage`
+* throwing is the realistic shape of that. Quarantine that one source as
+* `{ count: 0, ok: false, reason }` and keep going. The design's promise is
+* one bad object never takes down the others or the run's report; letting a
+* throw here escape the loop would break that promise for every source
+* queued after the failing one.
+*
+* Paging and pushing are caught separately, on purpose: `reason` is set only
+* when paging itself threw (no rows were ever read), so the reported cause
+* names execution — not the push, which is what actually failed when
+* `args.push` returns `false`. Conflating the two sent an author whose
+* projection can't even run to look at their push credentials instead of
+* their SQL.
+*/
+async function pushRemoteProjections(args) {
+	const pageSize = args.pageSize ?? REMOTE_PAGE_SIZE;
+	const pushed = /* @__PURE__ */ new Map();
+	for (const source of args.sources) {
+		let all;
+		try {
+			all = await fetchAllPages(source, pageSize);
+		} catch (error) {
+			const message = error instanceof Error ? error.message : String(error);
+			pushed.set(source.entity, {
+				count: 0,
+				ok: false,
+				reason: `the projection failed to execute: ${message}`.slice(0, MAX_REASON)
+			});
+			continue;
+		}
+		try {
+			const ok = await args.push(SYNC_CUSTOM_PATH$1, {
+				source: source.entity,
+				rows: all,
+				mode: "full",
+				fetched: all.length,
+				cursor: null,
+				objectType: source.objectType
+			});
+			pushed.set(source.entity, {
+				count: all.length,
+				ok
+			});
+		} catch {
+			pushed.set(source.entity, {
+				count: 0,
+				ok: false
+			});
+		}
+	}
+	return pushed;
+}
+//#endregion
 //#region ../sdk/src/self-check.ts
 function selfCheck(input) {
 	const issues = [];
@@ -1328,12 +1572,25 @@ function resolveContactEntity(config) {
 }
 function collectSchemaMeta(config) {
 	const byTable = /* @__PURE__ */ new Map();
-	for (const table of config.schema ?? []) if (table.columnsMeta) byTable.set(table.name, table.columnsMeta);
+	const merge = (name, meta) => {
+		const existing = byTable.get(name);
+		byTable.set(name, {
+			columnsMeta: meta.columnsMeta ?? existing?.columnsMeta,
+			uniqueConstraints: meta.uniqueConstraints ?? existing?.uniqueConstraints
+		});
+	};
+	for (const table of config.schema ?? []) if (table.columnsMeta || table.uniqueConstraints) merge(table.name, {
+		columnsMeta: table.columnsMeta,
+		uniqueConstraints: table.uniqueConstraints
+	});
 	const sources = [];
 	if (config.source) sources.push(config.source.contacts, config.source.companies, config.source.deals, ...config.source.custom ?? []);
 	for (const source of sources) {
 		const meta = source?.[require_drizzle_meta.DRIZZLE_META];
-		if (meta?.tableName && meta.columnsMeta) byTable.set(meta.tableName, meta.columnsMeta);
+		if (meta?.tableName && (meta.columnsMeta || meta.uniqueConstraints)) merge(meta.tableName, {
+			columnsMeta: meta.columnsMeta,
+			uniqueConstraints: meta.uniqueConstraints
+		});
 	}
 	return byTable;
 }
@@ -1348,12 +1605,13 @@ function buildSchema(config) {
 	if (config.contactEntity && !byName.has(config.contactEntity)) byName.set(config.contactEntity, new Set(config.contactFields ?? []));
 	const metaByTable = collectSchemaMeta(config);
 	return [...byName].map(([name, columns]) => {
-		const columnsMeta = metaByTable.get(name);
-		if (columnsMeta) for (const col of columnsMeta) columns.add(col.name);
+		const meta = metaByTable.get(name);
+		for (const col of meta?.columnsMeta ?? []) columns.add(col.name);
 		return {
 			name,
 			columns: [...columns],
-			...columnsMeta ? { columnsMeta } : {}
+			...meta?.columnsMeta ? { columnsMeta: meta.columnsMeta } : {},
+			...meta?.uniqueConstraints ? { uniqueConstraints: meta.uniqueConstraints } : {}
 		};
 	});
 }
@@ -1533,6 +1791,15 @@ async function handlePreviewRequest(config, preview) {
 		sampledCount: rows.length
 	}, 200);
 }
+async function handleProjectionPreviewRequest(config, request) {
+	const run = config.introspector?.runProjection;
+	if (!run) return jsonResponse({ error: "This app cannot preview derived objects. Set `introspector: drizzleIntrospector(db)` in clivly.config.ts, on a clivly version that supports runProjection." }, 400);
+	const rows = await run.call(config.introspector, request.projection, { limit: request.limit });
+	return jsonResponse({
+		rows,
+		sampledCount: rows.length
+	}, 200);
+}
 /**
 * Create the Clivly SDK. The implemented surface — heartbeat/discovery reporting
 * and the auth-verify handler — connects a developer's app to a running Clivly
@@ -1560,9 +1827,15 @@ function createClivlySDK(config) {
 	const cursorStore = config.cursorStore ?? createMemoryCursorStore();
 	const fetcher = createFetcher(config.retry);
 	let warnedUnsignedTrigger = false;
+	const remoteProjections = createRemoteProjectionFetcher({
+		apiKey: config.apiKey,
+		apiUrl,
+		fetcher
+	});
 	let observedUrl;
 	let lastPushFailure = null;
 	let lastPush = { failed: false };
+	let lastRemoteOutcomes = [];
 	const warnedSilentTargets = /* @__PURE__ */ new Set();
 	const warnIfSilent = async (target, response) => {
 		if (typeof response.json !== "function") return;
@@ -1634,18 +1907,58 @@ function createClivlySDK(config) {
 		for (const customSource of customSources ?? []) if (!await syncOne(customSource, SYNC_CUSTOM_PATH, mode, customSource.objectType, runId)) failed.push(customSource.entity);
 		return failed;
 	};
+	const syncRemoteSources = async (sources, runId) => {
+		const pushed = await pushRemoteProjections({
+			push: createPush(runId),
+			sources
+		});
+		const failed = [];
+		for (const [entity, result] of pushed) if (!result.ok) failed.push(entity);
+		return {
+			failed,
+			pushed
+		};
+	};
+	const reportRemoteOutcomes = async (outcomes, pushed, runId) => {
+		if (outcomes.length === 0) return;
+		try {
+			const report = buildRemoteProjectionReport({
+				outcomes,
+				pushed,
+				runId
+			});
+			await fetcher(`${apiUrl}${PROJECTIONS_REPORT_PATH}`, {
+				method: "POST",
+				headers: {
+					authorization: `Bearer ${config.apiKey}`,
+					"content-type": "application/json"
+				},
+				body: JSON.stringify(report)
+			});
+		} catch {}
+	};
 	const runSync = async (options) => {
-		if (!config.source) return;
+		const remoteOutcomes = resolveRemoteProjections({
+			entries: await remoteProjections.fetch(),
+			introspector: config.introspector,
+			schema: buildSchema(config)
+		});
+		lastRemoteOutcomes = remoteOutcomes;
+		const remoteSources = remoteOutcomes.filter((o) => o.status === "ok").map((o) => o.source);
+		if (!config.source && remoteSources.length === 0 && remoteOutcomes.length === 0) return;
 		lastPushFailure = null;
 		const mode = options?.mode ?? "delta";
 		const runId = options?.runId;
 		const failed = [];
-		if (config.source.companies) {
+		if (config.source?.companies) {
 			if (!await syncOne(config.source.companies, SYNC_COMPANIES_PATH, mode, void 0, runId)) failed.push(config.source.companies.entity);
 		}
-		if (!await syncOne(config.source.contacts, SYNC_CONTACTS_PATH, mode, void 0, runId)) failed.push(config.source.contacts.entity);
-		if (config.source.deals && !await syncOne(config.source.deals, SYNC_DEALS_PATH, mode, void 0, runId)) failed.push(config.source.deals.entity);
-		failed.push(...await syncCustomSources(config.source.custom, mode, runId));
+		if (config.source?.contacts && !await syncOne(config.source.contacts, SYNC_CONTACTS_PATH, mode, void 0, runId)) failed.push(config.source.contacts.entity);
+		if (config.source?.deals && !await syncOne(config.source.deals, SYNC_DEALS_PATH, mode, void 0, runId)) failed.push(config.source.deals.entity);
+		failed.push(...await syncCustomSources(config.source?.custom, mode, runId));
+		const remote = await syncRemoteSources(remoteSources, runId);
+		failed.push(...remote.failed);
+		await reportRemoteOutcomes(lastRemoteOutcomes, remote.pushed, runId);
 		if (failed.length > 0) {
 			const message = describePushFailure(failed);
 			lastPush = {
@@ -1662,13 +1975,31 @@ function createClivlySDK(config) {
 		lastPushFailure = null;
 		if (!await syncOne(resolved.source, resolved.path, "delta", resolved.objectType, runId, SAMPLE_BATCH_SIZE, SAMPLE_BATCH_SIZE, false)) throw new Error(describePushFailure([resolved.source.entity]));
 	};
-	const sendHeartbeat = () => {
-		const triggerUrl = resolveTriggerUrl({
+	const describeSyncTrigger = () => ({
+		configured: config.syncTrigger?.url,
+		resolved: resolveTriggerUrl({
 			configuredUrl: config.syncTrigger?.url,
 			configuredPath: config.syncTrigger?.path,
 			env: config.env ?? process.env,
 			observedUrl
-		});
+		})
+	});
+	const describeRemoteProjections = () => {
+		const counts = {
+			failed: 0,
+			invalid: 0,
+			ok: 0,
+			skipped: 0
+		};
+		for (const outcome of lastRemoteOutcomes) counts[outcome.status] += 1;
+		return {
+			capable: Boolean(config.introspector?.projectionSource),
+			ran: lastRemoteOutcomes.length > 0,
+			...counts
+		};
+	};
+	const sendHeartbeat = () => {
+		const triggerUrl = describeSyncTrigger().resolved;
 		return fetcher(`${apiUrl}/clivly/sdk/heartbeat`, {
 			method: "POST",
 			headers: {
@@ -1865,7 +2196,7 @@ function createClivlySDK(config) {
 	});
 	const runScheduled = async (options) => {
 		await heartbeat();
-		if (config.source) await runSync(options);
+		if (config.source || config.introspector) await runSync(options);
 	};
 	const triggerRequestAllowed = async (req, rawBody) => {
 		const secret = config.syncTrigger?.secret;
@@ -1896,6 +2227,7 @@ function createClivlySDK(config) {
 		const startedAt = Date.now();
 		try {
 			if (payload.preview) return await handlePreviewRequest(config, payload.preview);
+			if (payload.previewProjection) return await handleProjectionPreviewRequest(config, payload.previewProjection);
 			if (payload.sample) await runSampleSync(payload.sample.entityKey, payload.runId);
 			else await runScheduled({
 				mode: payload.mode,
@@ -1958,7 +2290,7 @@ function createClivlySDK(config) {
 			presenceClient = null;
 			console.warn("Clivly presence socket unavailable; using HTTP heartbeat", error);
 		}
-		if (!timers.sync && config.source) {
+		if (!timers.sync && (config.source || config.introspector)) {
 			timers.sync = setInterval(() => {
 				runSync().catch(() => {});
 			}, syncIntervalMs);
@@ -1991,6 +2323,8 @@ function createClivlySDK(config) {
 		connect,
 		ping,
 		doctor,
+		describeSyncTrigger,
+		describeRemoteProjections,
 		runScheduled,
 		createClivlyHandler,
 		sync: runSync,