clivly 0.13.0 → 0.14.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +10 -3
- package/{adapter-DhPBRNpi.d.mts → adapter-B-iBMhl0.d.mts} +13 -0
- package/{adapter-DTMeGGsl.d.cts → adapter-JpXSAnj-.d.cts} +13 -0
- package/core-adapter.d.cts +1 -1
- package/core-adapter.d.mts +1 -1
- package/core-sync-engine.cjs +3 -1
- package/core-sync-engine.d.cts +2 -2
- package/core-sync-engine.d.mts +2 -2
- package/core-sync-engine.mjs +2 -2
- package/core.cjs +3 -1
- package/core.d.cts +3 -3
- package/core.d.mts +3 -3
- package/core.mjs +2 -2
- package/index.cjs +102 -1
- package/index.mjs +102 -1
- package/internal-adapter.d.cts +1 -1
- package/internal-adapter.d.mts +1 -1
- package/internal-sync-engine.cjs +3 -1
- package/internal-sync-engine.d.cts +2 -2
- package/internal-sync-engine.d.mts +2 -2
- package/internal-sync-engine.mjs +2 -2
- package/internal.cjs +3 -1
- package/internal.d.cts +3 -3
- package/internal.d.mts +3 -3
- package/internal.mjs +2 -2
- package/package.json +1 -1
- package/{sdk-B-dp1tZN.mjs → sdk-DLoVPlGB.mjs} +139 -5
- package/{sdk-DKfKhJD_.cjs → sdk-ZMy4C4VK.cjs} +156 -4
- package/sdk.cjs +4 -1
- package/sdk.d.cts +32 -0
- package/sdk.d.mts +32 -0
- package/sdk.mjs +2 -2
- package/{sync-engine-Dwm_KDUS.cjs → sync-engine-BGVDsrOQ.cjs} +131 -71
- package/{sync-engine-V3_UutGv.d.cts → sync-engine-CYsWOujz.d.cts} +63 -22
- package/{sync-engine--M9EaONC.d.mts → sync-engine-C_5TpvfX.d.mts} +63 -22
- package/{sync-engine-Df2MCs_n.mjs → sync-engine-DH058G3w.mjs} +120 -72
package/internal.mjs
CHANGED
|
@@ -6,6 +6,6 @@ import { a as hostOwnedFieldMessage, c as rejectedContactFields, i as toObjectTy
|
|
|
6
6
|
import { n as buildRelationships, r as relationshipViaSignature, t as buildFilterFromDiscriminator } from "./mapping-form-DigkuvFc.mjs";
|
|
7
7
|
import { a as compileProjectionPreview, i as compileProjectionJoins, n as compileProjectionFrom, o as compileProjectionSelect, r as compileProjectionIdentity, s as quoteIdent, t as PREVIEW_MAX_ROWS } from "./projection-sql-DweUaWRc.mjs";
|
|
8
8
|
import { i as explainEntitiesConfig, n as compileEntityView, r as compileEntityViews, t as DEFAULT_VIEW_PREFIX } from "./view-compiler-C-6RydKK.mjs";
|
|
9
|
-
import {
|
|
9
|
+
import { a as resolveActions, i as reconcile, n as isRemovalAction, o as runSync } from "./sync-engine-DH058G3w.mjs";
|
|
10
10
|
import { n as isClosedStageType, t as OPEN_STAGE_TYPE } from "./types-YJTkXKzU.mjs";
|
|
11
|
-
export { BASE_REF, CANONICAL_FIELDS, CRM_CONCEPTS, ClivlyConfigError, DEFAULT_VIEW_PREFIX, MATERIALIZING_CONCEPTS, OPEN_STAGE_TYPE, PREVIEW_MAX_ROWS, PROJECTION_KEY_PATTERN, REQUIRED_FIELDS, authCustom, buildFilterFromDiscriminator, buildRelationships, columnMetaFor, compileEntityView, compileEntityViews, compileProjectionFrom, compileProjectionIdentity, compileProjectionJoins, compileProjectionPreview, compileProjectionSelect, computeDealValue, coversUniqueKey, defineClivlyConfig, entitiesConfigSchema, evaluateStage, explainEntitiesConfig, fanOutJoins, filterSchema, hasRichMeta, hostOwnedFieldMessage, identityRefs, isClosedStageType, isHostOwned, joinFansOut, mappingsToEntitiesConfig, matchesCondition, missingRequiredFields, primaryKeyColumns, projectionFields, projectionSchema, quoteIdent, reconcile, rejectedCompanyFields, rejectedContactFields, relationshipSchema, relationshipViaSignature, relationshipsSchema, requiredFieldsFor, runSync, slugify, toObjectTypeSlug, validateEntitiesConfig, validateProjection, validateProjectionFilter };
|
|
11
|
+
export { BASE_REF, CANONICAL_FIELDS, CRM_CONCEPTS, ClivlyConfigError, DEFAULT_VIEW_PREFIX, MATERIALIZING_CONCEPTS, OPEN_STAGE_TYPE, PREVIEW_MAX_ROWS, PROJECTION_KEY_PATTERN, REQUIRED_FIELDS, authCustom, buildFilterFromDiscriminator, buildRelationships, columnMetaFor, compileEntityView, compileEntityViews, compileProjectionFrom, compileProjectionIdentity, compileProjectionJoins, compileProjectionPreview, compileProjectionSelect, computeDealValue, coversUniqueKey, defineClivlyConfig, entitiesConfigSchema, evaluateStage, explainEntitiesConfig, fanOutJoins, filterSchema, hasRichMeta, hostOwnedFieldMessage, identityRefs, isClosedStageType, isHostOwned, isRemovalAction, joinFansOut, mappingsToEntitiesConfig, matchesCondition, missingRequiredFields, primaryKeyColumns, projectionFields, projectionSchema, quoteIdent, reconcile, rejectedCompanyFields, rejectedContactFields, relationshipSchema, relationshipViaSignature, relationshipsSchema, requiredFieldsFor, resolveActions, runSync, slugify, toObjectTypeSlug, validateEntitiesConfig, validateProjection, validateProjectionFilter };
|
package/package.json
CHANGED
|
@@ -1391,6 +1391,51 @@ async function fullSyncSource(args) {
|
|
|
1391
1391
|
};
|
|
1392
1392
|
}
|
|
1393
1393
|
//#endregion
|
|
1394
|
+
//#region ../sdk/src/trigger-method.ts
|
|
1395
|
+
/** Host-side override, mirroring `CLIVLY_INSTANT_PUSH_DISABLED`. */
|
|
1396
|
+
const REQUIRE_POST_ENV_VAR = "CLIVLY_TRIGGER_REQUIRE_POST";
|
|
1397
|
+
const NON_POST_DEPRECATION_WARNING = `Clivly: this sync trigger was called with GET. It still ran the sync, but a future release will refuse any method except POST with a 405. Change the caller — your cron job or uptime check — to POST. To adopt the new behaviour now, set ${REQUIRE_POST_ENV_VAR}=true.`;
|
|
1398
|
+
const METHOD_NOT_ALLOWED = 405;
|
|
1399
|
+
const NO_CONTENT = 204;
|
|
1400
|
+
const OK = 200;
|
|
1401
|
+
/**
|
|
1402
|
+
* Resolve the phase. Only the exact strings `"true"` and `"false"` count —
|
|
1403
|
+
* anything else (unset, empty, `"1"`, `"TRUE"`) falls through to the default
|
|
1404
|
+
* rather than guessing, so a typo can never silently disable a guard.
|
|
1405
|
+
*/
|
|
1406
|
+
function enforcementEnabled(input) {
|
|
1407
|
+
const override = input.env[REQUIRE_POST_ENV_VAR];
|
|
1408
|
+
if (override === "true") return true;
|
|
1409
|
+
if (override === "false") return false;
|
|
1410
|
+
return input.enforceDefault;
|
|
1411
|
+
}
|
|
1412
|
+
function resolveTriggerMethod(input) {
|
|
1413
|
+
const method = input.method.toUpperCase();
|
|
1414
|
+
if (method === "POST") return {
|
|
1415
|
+
runSync: true,
|
|
1416
|
+
status: OK
|
|
1417
|
+
};
|
|
1418
|
+
if (method === "HEAD") return {
|
|
1419
|
+
runSync: false,
|
|
1420
|
+
status: NO_CONTENT
|
|
1421
|
+
};
|
|
1422
|
+
const refuse = {
|
|
1423
|
+
body: {
|
|
1424
|
+
error: `Method ${method} is not allowed on the Clivly sync trigger. Use POST.`,
|
|
1425
|
+
ok: false
|
|
1426
|
+
},
|
|
1427
|
+
runSync: false,
|
|
1428
|
+
status: METHOD_NOT_ALLOWED
|
|
1429
|
+
};
|
|
1430
|
+
if (method !== "GET") return refuse;
|
|
1431
|
+
if (enforcementEnabled(input)) return refuse;
|
|
1432
|
+
return {
|
|
1433
|
+
runSync: true,
|
|
1434
|
+
status: OK,
|
|
1435
|
+
warn: NON_POST_DEPRECATION_WARNING
|
|
1436
|
+
};
|
|
1437
|
+
}
|
|
1438
|
+
//#endregion
|
|
1394
1439
|
//#region ../sdk/src/trigger-url.ts
|
|
1395
1440
|
const SCHEME_REGEX = /^https?:\/\//;
|
|
1396
1441
|
const INVALID_HOST_CHARS_REGEX = /[/\\@?#\s]/;
|
|
@@ -1702,7 +1747,7 @@ function buildSchema(config) {
|
|
|
1702
1747
|
});
|
|
1703
1748
|
}
|
|
1704
1749
|
const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
|
|
1705
|
-
function
|
|
1750
|
+
function memoryCursorStore() {
|
|
1706
1751
|
const cursors = /* @__PURE__ */ new Map();
|
|
1707
1752
|
return {
|
|
1708
1753
|
get: (entity) => cursors.get(entity) ?? null,
|
|
@@ -1711,6 +1756,56 @@ function createMemoryCursorStore() {
|
|
|
1711
1756
|
}
|
|
1712
1757
|
};
|
|
1713
1758
|
}
|
|
1759
|
+
/** Default key prefix for {@link kvCursorStore}. */
|
|
1760
|
+
const CURSOR_KEY_PREFIX = "clivly:cursor:";
|
|
1761
|
+
/**
|
|
1762
|
+
* A persistent `CursorStore` backed by any key-value store.
|
|
1763
|
+
*
|
|
1764
|
+
* Serverless runtimes discard memory between invocations, so without a
|
|
1765
|
+
* persisted cursor every `delta` sync starts from `null` and re-reads the
|
|
1766
|
+
* whole source table. That is invisible — the run succeeds — and bills like a
|
|
1767
|
+
* full sync forever.
|
|
1768
|
+
*
|
|
1769
|
+
* This exists because the hand-written version has a trap in it:
|
|
1770
|
+
* `SyncCursor.time` is a `Date`, which a JSON round-trip turns into a string,
|
|
1771
|
+
* and `fromDrizzle` then calls `since.time.toISOString()` on it. The cursor is
|
|
1772
|
+
* stored as an ISO string and parsed back to a `Date` here so no caller has to
|
|
1773
|
+
* remember that.
|
|
1774
|
+
*
|
|
1775
|
+
* ```ts
|
|
1776
|
+
* cursorStore: kvCursorStore(env.CLIVLY_CURSORS) // Workers KV binding
|
|
1777
|
+
* ```
|
|
1778
|
+
*
|
|
1779
|
+
* A malformed or partial stored value resolves to `null` rather than throwing:
|
|
1780
|
+
* the worst case is one full re-read, where throwing would wedge sync entirely.
|
|
1781
|
+
*/
|
|
1782
|
+
function kvCursorStore(kv, options) {
|
|
1783
|
+
const prefix = options?.prefix ?? "clivly:cursor:";
|
|
1784
|
+
const keyFor = (entity) => `${prefix}${entity}`;
|
|
1785
|
+
return {
|
|
1786
|
+
get: async (entity) => {
|
|
1787
|
+
const raw = await kv.get(keyFor(entity));
|
|
1788
|
+
if (!raw) return null;
|
|
1789
|
+
try {
|
|
1790
|
+
const parsed = JSON.parse(raw);
|
|
1791
|
+
if (typeof parsed.id !== "string" || typeof parsed.time !== "string") return null;
|
|
1792
|
+
const time = new Date(parsed.time);
|
|
1793
|
+
return Number.isNaN(time.getTime()) ? null : {
|
|
1794
|
+
id: parsed.id,
|
|
1795
|
+
time
|
|
1796
|
+
};
|
|
1797
|
+
} catch {
|
|
1798
|
+
return null;
|
|
1799
|
+
}
|
|
1800
|
+
},
|
|
1801
|
+
set: async (entity, cursor) => {
|
|
1802
|
+
await kv.put(keyFor(entity), cursor === null ? "" : JSON.stringify({
|
|
1803
|
+
id: cursor.id,
|
|
1804
|
+
time: cursor.time.toISOString()
|
|
1805
|
+
}));
|
|
1806
|
+
}
|
|
1807
|
+
};
|
|
1808
|
+
}
|
|
1714
1809
|
function isLikelyServerless() {
|
|
1715
1810
|
const runtime = globalThis;
|
|
1716
1811
|
if (runtime.navigator?.userAgent === "Cloudflare-Workers") return true;
|
|
@@ -1940,9 +2035,11 @@ function createClivlySDK(config) {
|
|
|
1940
2035
|
let presenceClient = null;
|
|
1941
2036
|
let httpHeartbeatActive = true;
|
|
1942
2037
|
const batchSize = config.sync?.batchSize ?? DEFAULT_BATCH_SIZE;
|
|
1943
|
-
|
|
2038
|
+
if (serverless && !config.cursorStore) console.warn("Clivly: no `cursorStore` is configured and this looks like a serverless/edge runtime, where memory does not survive between invocations. Every `delta` sync will start from scratch and re-read your entire source tables — a full sync, on every run, billed as such. Pass a persistent store: `cursorStore: kvCursorStore(env.MY_KV)` (exported from `@clivly/sdk`). See https://clivly.com/docs/concepts/sync#cursor-stores");
|
|
2039
|
+
const cursorStore = config.cursorStore ?? memoryCursorStore();
|
|
1944
2040
|
const fetcher = createFetcher(config.retry);
|
|
1945
2041
|
let warnedUnsignedTrigger = false;
|
|
2042
|
+
let warnedNonPostTrigger = false;
|
|
1946
2043
|
const remoteProjections = createRemoteProjectionFetcher({
|
|
1947
2044
|
apiKey: config.apiKey,
|
|
1948
2045
|
apiUrl,
|
|
@@ -2368,8 +2465,45 @@ function createClivlySDK(config) {
|
|
|
2368
2465
|
return {};
|
|
2369
2466
|
}
|
|
2370
2467
|
};
|
|
2468
|
+
/**
|
|
2469
|
+
* The response to send instead of running a sync, or null to proceed.
|
|
2470
|
+
*
|
|
2471
|
+
* Split out of the handler so the method decision reads as one thing rather
|
|
2472
|
+
* than three branches ahead of the request plumbing — the same shape as
|
|
2473
|
+
* `triggerRequestAllowed` below.
|
|
2474
|
+
*/
|
|
2475
|
+
const refuseDisallowedMethod = (req) => {
|
|
2476
|
+
const decision = resolveTriggerMethod({
|
|
2477
|
+
enforceDefault: false,
|
|
2478
|
+
env: config.env ?? process.env,
|
|
2479
|
+
method: req.method
|
|
2480
|
+
});
|
|
2481
|
+
if (decision.warn && !warnedNonPostTrigger) {
|
|
2482
|
+
warnedNonPostTrigger = true;
|
|
2483
|
+
console.warn(decision.warn);
|
|
2484
|
+
}
|
|
2485
|
+
if (decision.runSync) return null;
|
|
2486
|
+
return decision.body ? jsonResponse(decision.body, decision.status) : new Response(null, { status: decision.status });
|
|
2487
|
+
};
|
|
2488
|
+
/**
|
|
2489
|
+
* Serve a preview request, or null when this payload is a real sync.
|
|
2490
|
+
*
|
|
2491
|
+
* Preview (T10) is a signed, ZERO-WRITE dry run: it returns ≤5 RAW source
|
|
2492
|
+
* rows, slimmed to only the columns the draft maps or filters on and coerced
|
|
2493
|
+
* to JSON scalars, which the cloud then projects through the same path as a
|
|
2494
|
+
* real sync. It never syncs, never pushes, never writes a cursor and never
|
|
2495
|
+
* ingests. Callers keep it inside their `try` so a malformed body or a
|
|
2496
|
+
* throwing `fetchPage` degrades to the same graceful 500 as the sync paths.
|
|
2497
|
+
*/
|
|
2498
|
+
const handleIfPreview = async (payload) => {
|
|
2499
|
+
if (payload.preview) return await handlePreviewRequest(config, payload.preview);
|
|
2500
|
+
if (payload.previewProjection) return await handleProjectionPreviewRequest(config, payload.previewProjection);
|
|
2501
|
+
return null;
|
|
2502
|
+
};
|
|
2371
2503
|
const createClivlyHandler = () => async (req) => {
|
|
2372
2504
|
observedUrl = observedTriggerUrl(req) ?? observedUrl;
|
|
2505
|
+
const refusal = refuseDisallowedMethod(req);
|
|
2506
|
+
if (refusal) return refusal;
|
|
2373
2507
|
const rawBody = await req.text().catch(() => "");
|
|
2374
2508
|
if (!await triggerRequestAllowed(req, rawBody)) return jsonResponse({
|
|
2375
2509
|
ok: false,
|
|
@@ -2379,8 +2513,8 @@ function createClivlySDK(config) {
|
|
|
2379
2513
|
const runId = payload.runId ?? null;
|
|
2380
2514
|
const startedAt = Date.now();
|
|
2381
2515
|
try {
|
|
2382
|
-
|
|
2383
|
-
if (
|
|
2516
|
+
const previewResponse = await handleIfPreview(payload);
|
|
2517
|
+
if (previewResponse) return previewResponse;
|
|
2384
2518
|
if (payload.sample) await runSampleSync(payload.sample.entityKey, payload.runId);
|
|
2385
2519
|
else await runScheduled({
|
|
2386
2520
|
mode: payload.mode,
|
|
@@ -2486,4 +2620,4 @@ function createClivlySDK(config) {
|
|
|
2486
2620
|
};
|
|
2487
2621
|
}
|
|
2488
2622
|
//#endregion
|
|
2489
|
-
export {
|
|
2623
|
+
export { ClivlyAuthError as a, ClivlyRateLimitError as c, memoryCursorStore as i, createChatSessionHandler as l, createClivlySDK as n, ClivlyError as o, kvCursorStore as r, ClivlyNetworkError as s, CURSOR_KEY_PREFIX as t, isPlaceholderUrl as u };
|
|
@@ -1391,6 +1391,51 @@ async function fullSyncSource(args) {
|
|
|
1391
1391
|
};
|
|
1392
1392
|
}
|
|
1393
1393
|
//#endregion
|
|
1394
|
+
//#region ../sdk/src/trigger-method.ts
|
|
1395
|
+
/** Host-side override, mirroring `CLIVLY_INSTANT_PUSH_DISABLED`. */
|
|
1396
|
+
const REQUIRE_POST_ENV_VAR = "CLIVLY_TRIGGER_REQUIRE_POST";
|
|
1397
|
+
const NON_POST_DEPRECATION_WARNING = `Clivly: this sync trigger was called with GET. It still ran the sync, but a future release will refuse any method except POST with a 405. Change the caller — your cron job or uptime check — to POST. To adopt the new behaviour now, set ${REQUIRE_POST_ENV_VAR}=true.`;
|
|
1398
|
+
const METHOD_NOT_ALLOWED = 405;
|
|
1399
|
+
const NO_CONTENT = 204;
|
|
1400
|
+
const OK = 200;
|
|
1401
|
+
/**
|
|
1402
|
+
* Resolve the phase. Only the exact strings `"true"` and `"false"` count —
|
|
1403
|
+
* anything else (unset, empty, `"1"`, `"TRUE"`) falls through to the default
|
|
1404
|
+
* rather than guessing, so a typo can never silently disable a guard.
|
|
1405
|
+
*/
|
|
1406
|
+
function enforcementEnabled(input) {
|
|
1407
|
+
const override = input.env[REQUIRE_POST_ENV_VAR];
|
|
1408
|
+
if (override === "true") return true;
|
|
1409
|
+
if (override === "false") return false;
|
|
1410
|
+
return input.enforceDefault;
|
|
1411
|
+
}
|
|
1412
|
+
function resolveTriggerMethod(input) {
|
|
1413
|
+
const method = input.method.toUpperCase();
|
|
1414
|
+
if (method === "POST") return {
|
|
1415
|
+
runSync: true,
|
|
1416
|
+
status: OK
|
|
1417
|
+
};
|
|
1418
|
+
if (method === "HEAD") return {
|
|
1419
|
+
runSync: false,
|
|
1420
|
+
status: NO_CONTENT
|
|
1421
|
+
};
|
|
1422
|
+
const refuse = {
|
|
1423
|
+
body: {
|
|
1424
|
+
error: `Method ${method} is not allowed on the Clivly sync trigger. Use POST.`,
|
|
1425
|
+
ok: false
|
|
1426
|
+
},
|
|
1427
|
+
runSync: false,
|
|
1428
|
+
status: METHOD_NOT_ALLOWED
|
|
1429
|
+
};
|
|
1430
|
+
if (method !== "GET") return refuse;
|
|
1431
|
+
if (enforcementEnabled(input)) return refuse;
|
|
1432
|
+
return {
|
|
1433
|
+
runSync: true,
|
|
1434
|
+
status: OK,
|
|
1435
|
+
warn: NON_POST_DEPRECATION_WARNING
|
|
1436
|
+
};
|
|
1437
|
+
}
|
|
1438
|
+
//#endregion
|
|
1394
1439
|
//#region ../sdk/src/trigger-url.ts
|
|
1395
1440
|
const SCHEME_REGEX = /^https?:\/\//;
|
|
1396
1441
|
const INVALID_HOST_CHARS_REGEX = /[/\\@?#\s]/;
|
|
@@ -1702,7 +1747,7 @@ function buildSchema(config) {
|
|
|
1702
1747
|
});
|
|
1703
1748
|
}
|
|
1704
1749
|
const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
|
|
1705
|
-
function
|
|
1750
|
+
function memoryCursorStore() {
|
|
1706
1751
|
const cursors = /* @__PURE__ */ new Map();
|
|
1707
1752
|
return {
|
|
1708
1753
|
get: (entity) => cursors.get(entity) ?? null,
|
|
@@ -1711,6 +1756,56 @@ function createMemoryCursorStore() {
|
|
|
1711
1756
|
}
|
|
1712
1757
|
};
|
|
1713
1758
|
}
|
|
1759
|
+
/** Default key prefix for {@link kvCursorStore}. */
|
|
1760
|
+
const CURSOR_KEY_PREFIX = "clivly:cursor:";
|
|
1761
|
+
/**
|
|
1762
|
+
* A persistent `CursorStore` backed by any key-value store.
|
|
1763
|
+
*
|
|
1764
|
+
* Serverless runtimes discard memory between invocations, so without a
|
|
1765
|
+
* persisted cursor every `delta` sync starts from `null` and re-reads the
|
|
1766
|
+
* whole source table. That is invisible — the run succeeds — and bills like a
|
|
1767
|
+
* full sync forever.
|
|
1768
|
+
*
|
|
1769
|
+
* This exists because the hand-written version has a trap in it:
|
|
1770
|
+
* `SyncCursor.time` is a `Date`, which a JSON round-trip turns into a string,
|
|
1771
|
+
* and `fromDrizzle` then calls `since.time.toISOString()` on it. The cursor is
|
|
1772
|
+
* stored as an ISO string and parsed back to a `Date` here so no caller has to
|
|
1773
|
+
* remember that.
|
|
1774
|
+
*
|
|
1775
|
+
* ```ts
|
|
1776
|
+
* cursorStore: kvCursorStore(env.CLIVLY_CURSORS) // Workers KV binding
|
|
1777
|
+
* ```
|
|
1778
|
+
*
|
|
1779
|
+
* A malformed or partial stored value resolves to `null` rather than throwing:
|
|
1780
|
+
* the worst case is one full re-read, where throwing would wedge sync entirely.
|
|
1781
|
+
*/
|
|
1782
|
+
function kvCursorStore(kv, options) {
|
|
1783
|
+
const prefix = options?.prefix ?? "clivly:cursor:";
|
|
1784
|
+
const keyFor = (entity) => `${prefix}${entity}`;
|
|
1785
|
+
return {
|
|
1786
|
+
get: async (entity) => {
|
|
1787
|
+
const raw = await kv.get(keyFor(entity));
|
|
1788
|
+
if (!raw) return null;
|
|
1789
|
+
try {
|
|
1790
|
+
const parsed = JSON.parse(raw);
|
|
1791
|
+
if (typeof parsed.id !== "string" || typeof parsed.time !== "string") return null;
|
|
1792
|
+
const time = new Date(parsed.time);
|
|
1793
|
+
return Number.isNaN(time.getTime()) ? null : {
|
|
1794
|
+
id: parsed.id,
|
|
1795
|
+
time
|
|
1796
|
+
};
|
|
1797
|
+
} catch {
|
|
1798
|
+
return null;
|
|
1799
|
+
}
|
|
1800
|
+
},
|
|
1801
|
+
set: async (entity, cursor) => {
|
|
1802
|
+
await kv.put(keyFor(entity), cursor === null ? "" : JSON.stringify({
|
|
1803
|
+
id: cursor.id,
|
|
1804
|
+
time: cursor.time.toISOString()
|
|
1805
|
+
}));
|
|
1806
|
+
}
|
|
1807
|
+
};
|
|
1808
|
+
}
|
|
1714
1809
|
function isLikelyServerless() {
|
|
1715
1810
|
const runtime = globalThis;
|
|
1716
1811
|
if (runtime.navigator?.userAgent === "Cloudflare-Workers") return true;
|
|
@@ -1940,9 +2035,11 @@ function createClivlySDK(config) {
|
|
|
1940
2035
|
let presenceClient = null;
|
|
1941
2036
|
let httpHeartbeatActive = true;
|
|
1942
2037
|
const batchSize = config.sync?.batchSize ?? DEFAULT_BATCH_SIZE;
|
|
1943
|
-
|
|
2038
|
+
if (serverless && !config.cursorStore) console.warn("Clivly: no `cursorStore` is configured and this looks like a serverless/edge runtime, where memory does not survive between invocations. Every `delta` sync will start from scratch and re-read your entire source tables — a full sync, on every run, billed as such. Pass a persistent store: `cursorStore: kvCursorStore(env.MY_KV)` (exported from `@clivly/sdk`). See https://clivly.com/docs/concepts/sync#cursor-stores");
|
|
2039
|
+
const cursorStore = config.cursorStore ?? memoryCursorStore();
|
|
1944
2040
|
const fetcher = createFetcher(config.retry);
|
|
1945
2041
|
let warnedUnsignedTrigger = false;
|
|
2042
|
+
let warnedNonPostTrigger = false;
|
|
1946
2043
|
const remoteProjections = createRemoteProjectionFetcher({
|
|
1947
2044
|
apiKey: config.apiKey,
|
|
1948
2045
|
apiUrl,
|
|
@@ -2368,8 +2465,45 @@ function createClivlySDK(config) {
|
|
|
2368
2465
|
return {};
|
|
2369
2466
|
}
|
|
2370
2467
|
};
|
|
2468
|
+
/**
|
|
2469
|
+
* The response to send instead of running a sync, or null to proceed.
|
|
2470
|
+
*
|
|
2471
|
+
* Split out of the handler so the method decision reads as one thing rather
|
|
2472
|
+
* than three branches ahead of the request plumbing — the same shape as
|
|
2473
|
+
* `triggerRequestAllowed` below.
|
|
2474
|
+
*/
|
|
2475
|
+
const refuseDisallowedMethod = (req) => {
|
|
2476
|
+
const decision = resolveTriggerMethod({
|
|
2477
|
+
enforceDefault: false,
|
|
2478
|
+
env: config.env ?? process.env,
|
|
2479
|
+
method: req.method
|
|
2480
|
+
});
|
|
2481
|
+
if (decision.warn && !warnedNonPostTrigger) {
|
|
2482
|
+
warnedNonPostTrigger = true;
|
|
2483
|
+
console.warn(decision.warn);
|
|
2484
|
+
}
|
|
2485
|
+
if (decision.runSync) return null;
|
|
2486
|
+
return decision.body ? jsonResponse(decision.body, decision.status) : new Response(null, { status: decision.status });
|
|
2487
|
+
};
|
|
2488
|
+
/**
|
|
2489
|
+
* Serve a preview request, or null when this payload is a real sync.
|
|
2490
|
+
*
|
|
2491
|
+
* Preview (T10) is a signed, ZERO-WRITE dry run: it returns ≤5 RAW source
|
|
2492
|
+
* rows, slimmed to only the columns the draft maps or filters on and coerced
|
|
2493
|
+
* to JSON scalars, which the cloud then projects through the same path as a
|
|
2494
|
+
* real sync. It never syncs, never pushes, never writes a cursor and never
|
|
2495
|
+
* ingests. Callers keep it inside their `try` so a malformed body or a
|
|
2496
|
+
* throwing `fetchPage` degrades to the same graceful 500 as the sync paths.
|
|
2497
|
+
*/
|
|
2498
|
+
const handleIfPreview = async (payload) => {
|
|
2499
|
+
if (payload.preview) return await handlePreviewRequest(config, payload.preview);
|
|
2500
|
+
if (payload.previewProjection) return await handleProjectionPreviewRequest(config, payload.previewProjection);
|
|
2501
|
+
return null;
|
|
2502
|
+
};
|
|
2371
2503
|
const createClivlyHandler = () => async (req) => {
|
|
2372
2504
|
observedUrl = observedTriggerUrl(req) ?? observedUrl;
|
|
2505
|
+
const refusal = refuseDisallowedMethod(req);
|
|
2506
|
+
if (refusal) return refusal;
|
|
2373
2507
|
const rawBody = await req.text().catch(() => "");
|
|
2374
2508
|
if (!await triggerRequestAllowed(req, rawBody)) return jsonResponse({
|
|
2375
2509
|
ok: false,
|
|
@@ -2379,8 +2513,8 @@ function createClivlySDK(config) {
|
|
|
2379
2513
|
const runId = payload.runId ?? null;
|
|
2380
2514
|
const startedAt = Date.now();
|
|
2381
2515
|
try {
|
|
2382
|
-
|
|
2383
|
-
if (
|
|
2516
|
+
const previewResponse = await handleIfPreview(payload);
|
|
2517
|
+
if (previewResponse) return previewResponse;
|
|
2384
2518
|
if (payload.sample) await runSampleSync(payload.sample.entityKey, payload.runId);
|
|
2385
2519
|
else await runScheduled({
|
|
2386
2520
|
mode: payload.mode,
|
|
@@ -2486,6 +2620,12 @@ function createClivlySDK(config) {
|
|
|
2486
2620
|
};
|
|
2487
2621
|
}
|
|
2488
2622
|
//#endregion
|
|
2623
|
+
Object.defineProperty(exports, "CURSOR_KEY_PREFIX", {
|
|
2624
|
+
enumerable: true,
|
|
2625
|
+
get: function() {
|
|
2626
|
+
return CURSOR_KEY_PREFIX;
|
|
2627
|
+
}
|
|
2628
|
+
});
|
|
2489
2629
|
Object.defineProperty(exports, "ClivlyAuthError", {
|
|
2490
2630
|
enumerable: true,
|
|
2491
2631
|
get: function() {
|
|
@@ -2528,3 +2668,15 @@ Object.defineProperty(exports, "isPlaceholderUrl", {
|
|
|
2528
2668
|
return isPlaceholderUrl;
|
|
2529
2669
|
}
|
|
2530
2670
|
});
|
|
2671
|
+
Object.defineProperty(exports, "kvCursorStore", {
|
|
2672
|
+
enumerable: true,
|
|
2673
|
+
get: function() {
|
|
2674
|
+
return kvCursorStore;
|
|
2675
|
+
}
|
|
2676
|
+
});
|
|
2677
|
+
Object.defineProperty(exports, "memoryCursorStore", {
|
|
2678
|
+
enumerable: true,
|
|
2679
|
+
get: function() {
|
|
2680
|
+
return memoryCursorStore;
|
|
2681
|
+
}
|
|
2682
|
+
});
|
package/sdk.cjs
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
|
|
2
2
|
const require_entity_config = require("./entity-config-BI4RYNKL.cjs");
|
|
3
|
-
const require_sdk = require("./sdk-
|
|
3
|
+
const require_sdk = require("./sdk-ZMy4C4VK.cjs");
|
|
4
|
+
exports.CURSOR_KEY_PREFIX = require_sdk.CURSOR_KEY_PREFIX;
|
|
4
5
|
exports.ClivlyAuthError = require_sdk.ClivlyAuthError;
|
|
5
6
|
exports.ClivlyError = require_sdk.ClivlyError;
|
|
6
7
|
exports.ClivlyNetworkError = require_sdk.ClivlyNetworkError;
|
|
@@ -10,4 +11,6 @@ exports.createClivlySDK = require_sdk.createClivlySDK;
|
|
|
10
11
|
exports.customObjectDeclarations = require_entity_config.customObjectDeclarations;
|
|
11
12
|
exports.defineClivlyConfig = require_entity_config.defineClivlyConfig;
|
|
12
13
|
exports.isPlaceholderUrl = require_sdk.isPlaceholderUrl;
|
|
14
|
+
exports.kvCursorStore = require_sdk.kvCursorStore;
|
|
15
|
+
exports.memoryCursorStore = require_sdk.memoryCursorStore;
|
|
13
16
|
exports.validateEntitiesConfig = require_entity_config.validateEntitiesConfig;
|
package/sdk.d.cts
CHANGED
|
@@ -28,6 +28,38 @@ export declare class ClivlyRateLimitError extends ClivlyError {}
|
|
|
28
28
|
export declare class ClivlyNetworkError extends ClivlyError {}
|
|
29
29
|
//#endregion
|
|
30
30
|
//#region ../sdk/src/index.d.ts
|
|
31
|
+
export declare function memoryCursorStore(): CursorStore;
|
|
32
|
+
/** The shape every key-value client shares: Workers KV, Upstash, node-redis. */
|
|
33
|
+
export interface CursorKV {
|
|
34
|
+
get(key: string): Promise<string | null> | string | null;
|
|
35
|
+
put(key: string, value: string): Promise<unknown> | unknown;
|
|
36
|
+
}
|
|
37
|
+
/** Default key prefix for {@link kvCursorStore}. */
|
|
38
|
+
export declare const CURSOR_KEY_PREFIX = "clivly:cursor:";
|
|
39
|
+
/**
|
|
40
|
+
* A persistent `CursorStore` backed by any key-value store.
|
|
41
|
+
*
|
|
42
|
+
* Serverless runtimes discard memory between invocations, so without a
|
|
43
|
+
* persisted cursor every `delta` sync starts from `null` and re-reads the
|
|
44
|
+
* whole source table. That is invisible — the run succeeds — and bills like a
|
|
45
|
+
* full sync forever.
|
|
46
|
+
*
|
|
47
|
+
* This exists because the hand-written version has a trap in it:
|
|
48
|
+
* `SyncCursor.time` is a `Date`, which a JSON round-trip turns into a string,
|
|
49
|
+
* and `fromDrizzle` then calls `since.time.toISOString()` on it. The cursor is
|
|
50
|
+
* stored as an ISO string and parsed back to a `Date` here so no caller has to
|
|
51
|
+
* remember that.
|
|
52
|
+
*
|
|
53
|
+
* ```ts
|
|
54
|
+
* cursorStore: kvCursorStore(env.CLIVLY_CURSORS) // Workers KV binding
|
|
55
|
+
* ```
|
|
56
|
+
*
|
|
57
|
+
* A malformed or partial stored value resolves to `null` rather than throwing:
|
|
58
|
+
* the worst case is one full re-read, where throwing would wedge sync entirely.
|
|
59
|
+
*/
|
|
60
|
+
export declare function kvCursorStore(kv: CursorKV, options?: {
|
|
61
|
+
prefix?: string;
|
|
62
|
+
}): CursorStore;
|
|
31
63
|
/**
|
|
32
64
|
* Create the Clivly SDK. The implemented surface — heartbeat/discovery reporting
|
|
33
65
|
* and the auth-verify handler — connects a developer's app to a running Clivly
|
package/sdk.d.mts
CHANGED
|
@@ -28,6 +28,38 @@ export declare class ClivlyRateLimitError extends ClivlyError {}
|
|
|
28
28
|
export declare class ClivlyNetworkError extends ClivlyError {}
|
|
29
29
|
//#endregion
|
|
30
30
|
//#region ../sdk/src/index.d.ts
|
|
31
|
+
export declare function memoryCursorStore(): CursorStore;
|
|
32
|
+
/** The shape every key-value client shares: Workers KV, Upstash, node-redis. */
|
|
33
|
+
export interface CursorKV {
|
|
34
|
+
get(key: string): Promise<string | null> | string | null;
|
|
35
|
+
put(key: string, value: string): Promise<unknown> | unknown;
|
|
36
|
+
}
|
|
37
|
+
/** Default key prefix for {@link kvCursorStore}. */
|
|
38
|
+
export declare const CURSOR_KEY_PREFIX = "clivly:cursor:";
|
|
39
|
+
/**
|
|
40
|
+
* A persistent `CursorStore` backed by any key-value store.
|
|
41
|
+
*
|
|
42
|
+
* Serverless runtimes discard memory between invocations, so without a
|
|
43
|
+
* persisted cursor every `delta` sync starts from `null` and re-reads the
|
|
44
|
+
* whole source table. That is invisible — the run succeeds — and bills like a
|
|
45
|
+
* full sync forever.
|
|
46
|
+
*
|
|
47
|
+
* This exists because the hand-written version has a trap in it:
|
|
48
|
+
* `SyncCursor.time` is a `Date`, which a JSON round-trip turns into a string,
|
|
49
|
+
* and `fromDrizzle` then calls `since.time.toISOString()` on it. The cursor is
|
|
50
|
+
* stored as an ISO string and parsed back to a `Date` here so no caller has to
|
|
51
|
+
* remember that.
|
|
52
|
+
*
|
|
53
|
+
* ```ts
|
|
54
|
+
* cursorStore: kvCursorStore(env.CLIVLY_CURSORS) // Workers KV binding
|
|
55
|
+
* ```
|
|
56
|
+
*
|
|
57
|
+
* A malformed or partial stored value resolves to `null` rather than throwing:
|
|
58
|
+
* the worst case is one full re-read, where throwing would wedge sync entirely.
|
|
59
|
+
*/
|
|
60
|
+
export declare function kvCursorStore(kv: CursorKV, options?: {
|
|
61
|
+
prefix?: string;
|
|
62
|
+
}): CursorStore;
|
|
31
63
|
/**
|
|
32
64
|
* Create the Clivly SDK. The implemented surface — heartbeat/discovery reporting
|
|
33
65
|
* and the auth-verify handler — connects a developer's app to a running Clivly
|
package/sdk.mjs
CHANGED
|
@@ -1,3 +1,3 @@
|
|
|
1
1
|
import { c as defineClivlyConfig, g as validateEntitiesConfig, s as customObjectDeclarations } from "./entity-config-Ccfb2IL9.mjs";
|
|
2
|
-
import { a as ClivlyRateLimitError, i as
|
|
3
|
-
export { ClivlyAuthError, ClivlyError, ClivlyNetworkError, ClivlyRateLimitError, createChatSessionHandler, createClivlySDK, customObjectDeclarations, defineClivlyConfig, isPlaceholderUrl, validateEntitiesConfig };
|
|
2
|
+
import { a as ClivlyAuthError, c as ClivlyRateLimitError, i as memoryCursorStore, l as createChatSessionHandler, n as createClivlySDK, o as ClivlyError, r as kvCursorStore, s as ClivlyNetworkError, t as CURSOR_KEY_PREFIX, u as isPlaceholderUrl } from "./sdk-DLoVPlGB.mjs";
|
|
3
|
+
export { CURSOR_KEY_PREFIX, ClivlyAuthError, ClivlyError, ClivlyNetworkError, ClivlyRateLimitError, createChatSessionHandler, createClivlySDK, customObjectDeclarations, defineClivlyConfig, isPlaceholderUrl, kvCursorStore, memoryCursorStore, validateEntitiesConfig };
|