clivly 0.12.0 → 0.14.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +10 -3
- package/{adapter-DhPBRNpi.d.mts → adapter-B-iBMhl0.d.mts} +13 -0
- package/{adapter-DTMeGGsl.d.cts → adapter-JpXSAnj-.d.cts} +13 -0
- package/core-adapter.d.cts +1 -1
- package/core-adapter.d.mts +1 -1
- package/core-sync-engine.cjs +3 -1
- package/core-sync-engine.d.cts +2 -2
- package/core-sync-engine.d.mts +2 -2
- package/core-sync-engine.mjs +2 -2
- package/core.cjs +3 -1
- package/core.d.cts +3 -3
- package/core.d.mts +3 -3
- package/core.mjs +2 -2
- package/drizzle.cjs +65 -0
- package/drizzle.d.cts +51 -1
- package/drizzle.d.mts +51 -1
- package/drizzle.mjs +64 -1
- package/index.cjs +102 -1
- package/index.mjs +102 -1
- package/internal-adapter.d.cts +1 -1
- package/internal-adapter.d.mts +1 -1
- package/internal-sync-engine.cjs +3 -1
- package/internal-sync-engine.d.cts +2 -2
- package/internal-sync-engine.d.mts +2 -2
- package/internal-sync-engine.mjs +2 -2
- package/internal.cjs +3 -1
- package/internal.d.cts +3 -3
- package/internal.d.mts +3 -3
- package/internal.mjs +2 -2
- package/{namespace-probe-CNQKSmxz.d.mts → namespace-probe-DykR0kgh.d.mts} +42 -0
- package/{namespace-probe-C5RLl74M.d.cts → namespace-probe-H7yWc0JV.d.cts} +42 -0
- package/package.json +1 -1
- package/{sdk-C-Pjxj2y.mjs → sdk-DLoVPlGB.mjs} +177 -6
- package/{sdk-BvGSxBQc.cjs → sdk-ZMy4C4VK.cjs} +194 -5
- package/sdk.cjs +4 -1
- package/sdk.d.cts +33 -1
- package/sdk.d.mts +33 -1
- package/sdk.mjs +2 -2
- package/{sync-engine-Dwm_KDUS.cjs → sync-engine-BGVDsrOQ.cjs} +131 -71
- package/{sync-engine-V3_UutGv.d.cts → sync-engine-CYsWOujz.d.cts} +63 -22
- package/{sync-engine--M9EaONC.d.mts → sync-engine-C_5TpvfX.d.mts} +63 -22
- package/{sync-engine-Df2MCs_n.mjs → sync-engine-DH058G3w.mjs} +120 -72
- package/vite.d.cts +1 -1
- package/vite.d.mts +1 -1
|
@@ -699,6 +699,20 @@ function detectHostEnvironment(env, configured) {
|
|
|
699
699
|
return fromNodeEnv(env.NODE_ENV?.trim());
|
|
700
700
|
}
|
|
701
701
|
//#endregion
|
|
702
|
+
//#region ../sdk/src/instant-push-support.ts
|
|
703
|
+
/**
|
|
704
|
+
* The fast rollback switch for instant push (see
|
|
705
|
+
* docs/reviews/2026-09-19-near-real-time-sync-plan.md, question 5).
|
|
706
|
+
*
|
|
707
|
+
* A host-side env var, not a Clivly-Cloud-side toggle: it must keep working
|
|
708
|
+
* even when Clivly Cloud is the thing misbehaving (e.g. its ingest endpoint
|
|
709
|
+
* is what instant push is flooding), and it must take effect without a
|
|
710
|
+
* redeploy — just an env var change and the next cold start.
|
|
711
|
+
*/
|
|
712
|
+
function isInstantPushDisabled(env) {
|
|
713
|
+
return env.CLIVLY_INSTANT_PUSH_DISABLED === "true";
|
|
714
|
+
}
|
|
715
|
+
//#endregion
|
|
702
716
|
//#region ../sdk/src/presence-machine.ts
|
|
703
717
|
const PRESENCE_BACKOFF_BASE_MS = 1e3;
|
|
704
718
|
const PRESENCE_BACKOFF_MAX_MS = 3e4;
|
|
@@ -1377,6 +1391,51 @@ async function fullSyncSource(args) {
|
|
|
1377
1391
|
};
|
|
1378
1392
|
}
|
|
1379
1393
|
//#endregion
|
|
1394
|
+
//#region ../sdk/src/trigger-method.ts
|
|
1395
|
+
/** Host-side override, mirroring `CLIVLY_INSTANT_PUSH_DISABLED`. */
|
|
1396
|
+
const REQUIRE_POST_ENV_VAR = "CLIVLY_TRIGGER_REQUIRE_POST";
|
|
1397
|
+
const NON_POST_DEPRECATION_WARNING = `Clivly: this sync trigger was called with GET. It still ran the sync, but a future release will refuse any method except POST with a 405. Change the caller — your cron job or uptime check — to POST. To adopt the new behaviour now, set ${REQUIRE_POST_ENV_VAR}=true.`;
|
|
1398
|
+
const METHOD_NOT_ALLOWED = 405;
|
|
1399
|
+
const NO_CONTENT = 204;
|
|
1400
|
+
const OK = 200;
|
|
1401
|
+
/**
|
|
1402
|
+
* Resolve the phase. Only the exact strings `"true"` and `"false"` count —
|
|
1403
|
+
* anything else (unset, empty, `"1"`, `"TRUE"`) falls through to the default
|
|
1404
|
+
* rather than guessing, so a typo can never silently disable a guard.
|
|
1405
|
+
*/
|
|
1406
|
+
function enforcementEnabled(input) {
|
|
1407
|
+
const override = input.env[REQUIRE_POST_ENV_VAR];
|
|
1408
|
+
if (override === "true") return true;
|
|
1409
|
+
if (override === "false") return false;
|
|
1410
|
+
return input.enforceDefault;
|
|
1411
|
+
}
|
|
1412
|
+
function resolveTriggerMethod(input) {
|
|
1413
|
+
const method = input.method.toUpperCase();
|
|
1414
|
+
if (method === "POST") return {
|
|
1415
|
+
runSync: true,
|
|
1416
|
+
status: OK
|
|
1417
|
+
};
|
|
1418
|
+
if (method === "HEAD") return {
|
|
1419
|
+
runSync: false,
|
|
1420
|
+
status: NO_CONTENT
|
|
1421
|
+
};
|
|
1422
|
+
const refuse = {
|
|
1423
|
+
body: {
|
|
1424
|
+
error: `Method ${method} is not allowed on the Clivly sync trigger. Use POST.`,
|
|
1425
|
+
ok: false
|
|
1426
|
+
},
|
|
1427
|
+
runSync: false,
|
|
1428
|
+
status: METHOD_NOT_ALLOWED
|
|
1429
|
+
};
|
|
1430
|
+
if (method !== "GET") return refuse;
|
|
1431
|
+
if (enforcementEnabled(input)) return refuse;
|
|
1432
|
+
return {
|
|
1433
|
+
runSync: true,
|
|
1434
|
+
status: OK,
|
|
1435
|
+
warn: NON_POST_DEPRECATION_WARNING
|
|
1436
|
+
};
|
|
1437
|
+
}
|
|
1438
|
+
//#endregion
|
|
1380
1439
|
//#region ../sdk/src/trigger-url.ts
|
|
1381
1440
|
const SCHEME_REGEX = /^https?:\/\//;
|
|
1382
1441
|
const INVALID_HOST_CHARS_REGEX = /[/\\@?#\s]/;
|
|
@@ -1688,7 +1747,7 @@ function buildSchema(config) {
|
|
|
1688
1747
|
});
|
|
1689
1748
|
}
|
|
1690
1749
|
const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
|
|
1691
|
-
function
|
|
1750
|
+
function memoryCursorStore() {
|
|
1692
1751
|
const cursors = /* @__PURE__ */ new Map();
|
|
1693
1752
|
return {
|
|
1694
1753
|
get: (entity) => cursors.get(entity) ?? null,
|
|
@@ -1697,6 +1756,56 @@ function createMemoryCursorStore() {
|
|
|
1697
1756
|
}
|
|
1698
1757
|
};
|
|
1699
1758
|
}
|
|
1759
|
+
/** Default key prefix for {@link kvCursorStore}. */
|
|
1760
|
+
const CURSOR_KEY_PREFIX = "clivly:cursor:";
|
|
1761
|
+
/**
|
|
1762
|
+
* A persistent `CursorStore` backed by any key-value store.
|
|
1763
|
+
*
|
|
1764
|
+
* Serverless runtimes discard memory between invocations, so without a
|
|
1765
|
+
* persisted cursor every `delta` sync starts from `null` and re-reads the
|
|
1766
|
+
* whole source table. That is invisible — the run succeeds — and bills like a
|
|
1767
|
+
* full sync forever.
|
|
1768
|
+
*
|
|
1769
|
+
* This exists because the hand-written version has a trap in it:
|
|
1770
|
+
* `SyncCursor.time` is a `Date`, which a JSON round-trip turns into a string,
|
|
1771
|
+
* and `fromDrizzle` then calls `since.time.toISOString()` on it. The cursor is
|
|
1772
|
+
* stored as an ISO string and parsed back to a `Date` here so no caller has to
|
|
1773
|
+
* remember that.
|
|
1774
|
+
*
|
|
1775
|
+
* ```ts
|
|
1776
|
+
* cursorStore: kvCursorStore(env.CLIVLY_CURSORS) // Workers KV binding
|
|
1777
|
+
* ```
|
|
1778
|
+
*
|
|
1779
|
+
* A malformed or partial stored value resolves to `null` rather than throwing:
|
|
1780
|
+
* the worst case is one full re-read, where throwing would wedge sync entirely.
|
|
1781
|
+
*/
|
|
1782
|
+
function kvCursorStore(kv, options) {
|
|
1783
|
+
const prefix = options?.prefix ?? "clivly:cursor:";
|
|
1784
|
+
const keyFor = (entity) => `${prefix}${entity}`;
|
|
1785
|
+
return {
|
|
1786
|
+
get: async (entity) => {
|
|
1787
|
+
const raw = await kv.get(keyFor(entity));
|
|
1788
|
+
if (!raw) return null;
|
|
1789
|
+
try {
|
|
1790
|
+
const parsed = JSON.parse(raw);
|
|
1791
|
+
if (typeof parsed.id !== "string" || typeof parsed.time !== "string") return null;
|
|
1792
|
+
const time = new Date(parsed.time);
|
|
1793
|
+
return Number.isNaN(time.getTime()) ? null : {
|
|
1794
|
+
id: parsed.id,
|
|
1795
|
+
time
|
|
1796
|
+
};
|
|
1797
|
+
} catch {
|
|
1798
|
+
return null;
|
|
1799
|
+
}
|
|
1800
|
+
},
|
|
1801
|
+
set: async (entity, cursor) => {
|
|
1802
|
+
await kv.put(keyFor(entity), cursor === null ? "" : JSON.stringify({
|
|
1803
|
+
id: cursor.id,
|
|
1804
|
+
time: cursor.time.toISOString()
|
|
1805
|
+
}));
|
|
1806
|
+
}
|
|
1807
|
+
};
|
|
1808
|
+
}
|
|
1700
1809
|
function isLikelyServerless() {
|
|
1701
1810
|
const runtime = globalThis;
|
|
1702
1811
|
if (runtime.navigator?.userAgent === "Cloudflare-Workers") return true;
|
|
@@ -1926,9 +2035,11 @@ function createClivlySDK(config) {
|
|
|
1926
2035
|
let presenceClient = null;
|
|
1927
2036
|
let httpHeartbeatActive = true;
|
|
1928
2037
|
const batchSize = config.sync?.batchSize ?? DEFAULT_BATCH_SIZE;
|
|
1929
|
-
|
|
2038
|
+
if (serverless && !config.cursorStore) console.warn("Clivly: no `cursorStore` is configured and this looks like a serverless/edge runtime, where memory does not survive between invocations. Every `delta` sync will start from scratch and re-read your entire source tables — a full sync, on every run, billed as such. Pass a persistent store: `cursorStore: kvCursorStore(env.MY_KV)` (exported from `@clivly/sdk`). See https://clivly.com/docs/concepts/sync#cursor-stores");
|
|
2039
|
+
const cursorStore = config.cursorStore ?? memoryCursorStore();
|
|
1930
2040
|
const fetcher = createFetcher(config.retry);
|
|
1931
2041
|
let warnedUnsignedTrigger = false;
|
|
2042
|
+
let warnedNonPostTrigger = false;
|
|
1932
2043
|
const remoteProjections = createRemoteProjectionFetcher({
|
|
1933
2044
|
apiKey: config.apiKey,
|
|
1934
2045
|
apiUrl,
|
|
@@ -2090,6 +2201,28 @@ function createClivlySDK(config) {
|
|
|
2090
2201
|
lastPushFailure = null;
|
|
2091
2202
|
if (!await syncOne(resolved.source, resolved.path, "delta", resolved.objectType, runId, SAMPLE_BATCH_SIZE, SAMPLE_BATCH_SIZE, false)) throw pushFailureError(describePushFailure([resolved.source.entity]));
|
|
2092
2203
|
};
|
|
2204
|
+
const pushOne = async (entityKey, row) => {
|
|
2205
|
+
if (config.sync?.instant !== true) return;
|
|
2206
|
+
if (isInstantPushDisabled(config.env ?? process.env)) return;
|
|
2207
|
+
const resolved = resolveEntitySource(config, entityKey);
|
|
2208
|
+
if (!resolved) {
|
|
2209
|
+
console.warn(`Clivly instant push: no source is configured for entity "${entityKey}" — this row was not pushed. It will still sync on the next delta run once \`entities\` declares it.`);
|
|
2210
|
+
return;
|
|
2211
|
+
}
|
|
2212
|
+
const path = CONCEPT_PATHS[resolved.expectedKey];
|
|
2213
|
+
if (!path) return;
|
|
2214
|
+
const push = createPush();
|
|
2215
|
+
try {
|
|
2216
|
+
if (!await push(path, {
|
|
2217
|
+
source: pushSourceOf(resolved.source),
|
|
2218
|
+
rows: [row],
|
|
2219
|
+
mode: "delta",
|
|
2220
|
+
...resolved.expectedKey === "custom" ? { objectType: resolved.source.objectType } : {}
|
|
2221
|
+
})) console.warn(`Clivly instant push failed for "${entityKey}". This row will still sync on the next delta run.`);
|
|
2222
|
+
} catch (error) {
|
|
2223
|
+
console.warn(`Clivly instant push failed for "${entityKey}": ${error instanceof Error ? error.message : String(error)}. This row will still sync on the next delta run.`);
|
|
2224
|
+
}
|
|
2225
|
+
};
|
|
2093
2226
|
const describeSyncTrigger = () => ({
|
|
2094
2227
|
configured: config.syncTrigger?.url,
|
|
2095
2228
|
resolved: resolveTriggerUrl({
|
|
@@ -2332,8 +2465,45 @@ function createClivlySDK(config) {
|
|
|
2332
2465
|
return {};
|
|
2333
2466
|
}
|
|
2334
2467
|
};
|
|
2468
|
+
/**
|
|
2469
|
+
* The response to send instead of running a sync, or null to proceed.
|
|
2470
|
+
*
|
|
2471
|
+
* Split out of the handler so the method decision reads as one thing rather
|
|
2472
|
+
* than three branches ahead of the request plumbing — the same shape as
|
|
2473
|
+
* `triggerRequestAllowed` below.
|
|
2474
|
+
*/
|
|
2475
|
+
const refuseDisallowedMethod = (req) => {
|
|
2476
|
+
const decision = resolveTriggerMethod({
|
|
2477
|
+
enforceDefault: false,
|
|
2478
|
+
env: config.env ?? process.env,
|
|
2479
|
+
method: req.method
|
|
2480
|
+
});
|
|
2481
|
+
if (decision.warn && !warnedNonPostTrigger) {
|
|
2482
|
+
warnedNonPostTrigger = true;
|
|
2483
|
+
console.warn(decision.warn);
|
|
2484
|
+
}
|
|
2485
|
+
if (decision.runSync) return null;
|
|
2486
|
+
return decision.body ? jsonResponse(decision.body, decision.status) : new Response(null, { status: decision.status });
|
|
2487
|
+
};
|
|
2488
|
+
/**
|
|
2489
|
+
* Serve a preview request, or null when this payload is a real sync.
|
|
2490
|
+
*
|
|
2491
|
+
* Preview (T10) is a signed, ZERO-WRITE dry run: it returns ≤5 RAW source
|
|
2492
|
+
* rows, slimmed to only the columns the draft maps or filters on and coerced
|
|
2493
|
+
* to JSON scalars, which the cloud then projects through the same path as a
|
|
2494
|
+
* real sync. It never syncs, never pushes, never writes a cursor and never
|
|
2495
|
+
* ingests. Callers keep it inside their `try` so a malformed body or a
|
|
2496
|
+
* throwing `fetchPage` degrades to the same graceful 500 as the sync paths.
|
|
2497
|
+
*/
|
|
2498
|
+
const handleIfPreview = async (payload) => {
|
|
2499
|
+
if (payload.preview) return await handlePreviewRequest(config, payload.preview);
|
|
2500
|
+
if (payload.previewProjection) return await handleProjectionPreviewRequest(config, payload.previewProjection);
|
|
2501
|
+
return null;
|
|
2502
|
+
};
|
|
2335
2503
|
const createClivlyHandler = () => async (req) => {
|
|
2336
2504
|
observedUrl = observedTriggerUrl(req) ?? observedUrl;
|
|
2505
|
+
const refusal = refuseDisallowedMethod(req);
|
|
2506
|
+
if (refusal) return refusal;
|
|
2337
2507
|
const rawBody = await req.text().catch(() => "");
|
|
2338
2508
|
if (!await triggerRequestAllowed(req, rawBody)) return jsonResponse({
|
|
2339
2509
|
ok: false,
|
|
@@ -2343,8 +2513,8 @@ function createClivlySDK(config) {
|
|
|
2343
2513
|
const runId = payload.runId ?? null;
|
|
2344
2514
|
const startedAt = Date.now();
|
|
2345
2515
|
try {
|
|
2346
|
-
|
|
2347
|
-
if (
|
|
2516
|
+
const previewResponse = await handleIfPreview(payload);
|
|
2517
|
+
if (previewResponse) return previewResponse;
|
|
2348
2518
|
if (payload.sample) await runSampleSync(payload.sample.entityKey, payload.runId);
|
|
2349
2519
|
else await runScheduled({
|
|
2350
2520
|
mode: payload.mode,
|
|
@@ -2445,8 +2615,9 @@ function createClivlySDK(config) {
|
|
|
2445
2615
|
runScheduled,
|
|
2446
2616
|
createClivlyHandler,
|
|
2447
2617
|
sync: runSync,
|
|
2448
|
-
createAuthVerifyHandler
|
|
2618
|
+
createAuthVerifyHandler,
|
|
2619
|
+
pushOne
|
|
2449
2620
|
};
|
|
2450
2621
|
}
|
|
2451
2622
|
//#endregion
|
|
2452
|
-
export {
|
|
2623
|
+
export { ClivlyAuthError as a, ClivlyRateLimitError as c, memoryCursorStore as i, createChatSessionHandler as l, createClivlySDK as n, ClivlyError as o, kvCursorStore as r, ClivlyNetworkError as s, CURSOR_KEY_PREFIX as t, isPlaceholderUrl as u };
|
|
@@ -699,6 +699,20 @@ function detectHostEnvironment(env, configured) {
|
|
|
699
699
|
return fromNodeEnv(env.NODE_ENV?.trim());
|
|
700
700
|
}
|
|
701
701
|
//#endregion
|
|
702
|
+
//#region ../sdk/src/instant-push-support.ts
|
|
703
|
+
/**
|
|
704
|
+
* The fast rollback switch for instant push (see
|
|
705
|
+
* docs/reviews/2026-09-19-near-real-time-sync-plan.md, question 5).
|
|
706
|
+
*
|
|
707
|
+
* A host-side env var, not a Clivly-Cloud-side toggle: it must keep working
|
|
708
|
+
* even when Clivly Cloud is the thing misbehaving (e.g. its ingest endpoint
|
|
709
|
+
* is what instant push is flooding), and it must take effect without a
|
|
710
|
+
* redeploy — just an env var change and the next cold start.
|
|
711
|
+
*/
|
|
712
|
+
function isInstantPushDisabled(env) {
|
|
713
|
+
return env.CLIVLY_INSTANT_PUSH_DISABLED === "true";
|
|
714
|
+
}
|
|
715
|
+
//#endregion
|
|
702
716
|
//#region ../sdk/src/presence-machine.ts
|
|
703
717
|
const PRESENCE_BACKOFF_BASE_MS = 1e3;
|
|
704
718
|
const PRESENCE_BACKOFF_MAX_MS = 3e4;
|
|
@@ -1377,6 +1391,51 @@ async function fullSyncSource(args) {
|
|
|
1377
1391
|
};
|
|
1378
1392
|
}
|
|
1379
1393
|
//#endregion
|
|
1394
|
+
//#region ../sdk/src/trigger-method.ts
|
|
1395
|
+
/** Host-side override, mirroring `CLIVLY_INSTANT_PUSH_DISABLED`. */
|
|
1396
|
+
const REQUIRE_POST_ENV_VAR = "CLIVLY_TRIGGER_REQUIRE_POST";
|
|
1397
|
+
const NON_POST_DEPRECATION_WARNING = `Clivly: this sync trigger was called with GET. It still ran the sync, but a future release will refuse any method except POST with a 405. Change the caller — your cron job or uptime check — to POST. To adopt the new behaviour now, set ${REQUIRE_POST_ENV_VAR}=true.`;
|
|
1398
|
+
const METHOD_NOT_ALLOWED = 405;
|
|
1399
|
+
const NO_CONTENT = 204;
|
|
1400
|
+
const OK = 200;
|
|
1401
|
+
/**
|
|
1402
|
+
* Resolve the phase. Only the exact strings `"true"` and `"false"` count —
|
|
1403
|
+
* anything else (unset, empty, `"1"`, `"TRUE"`) falls through to the default
|
|
1404
|
+
* rather than guessing, so a typo can never silently disable a guard.
|
|
1405
|
+
*/
|
|
1406
|
+
function enforcementEnabled(input) {
|
|
1407
|
+
const override = input.env[REQUIRE_POST_ENV_VAR];
|
|
1408
|
+
if (override === "true") return true;
|
|
1409
|
+
if (override === "false") return false;
|
|
1410
|
+
return input.enforceDefault;
|
|
1411
|
+
}
|
|
1412
|
+
function resolveTriggerMethod(input) {
|
|
1413
|
+
const method = input.method.toUpperCase();
|
|
1414
|
+
if (method === "POST") return {
|
|
1415
|
+
runSync: true,
|
|
1416
|
+
status: OK
|
|
1417
|
+
};
|
|
1418
|
+
if (method === "HEAD") return {
|
|
1419
|
+
runSync: false,
|
|
1420
|
+
status: NO_CONTENT
|
|
1421
|
+
};
|
|
1422
|
+
const refuse = {
|
|
1423
|
+
body: {
|
|
1424
|
+
error: `Method ${method} is not allowed on the Clivly sync trigger. Use POST.`,
|
|
1425
|
+
ok: false
|
|
1426
|
+
},
|
|
1427
|
+
runSync: false,
|
|
1428
|
+
status: METHOD_NOT_ALLOWED
|
|
1429
|
+
};
|
|
1430
|
+
if (method !== "GET") return refuse;
|
|
1431
|
+
if (enforcementEnabled(input)) return refuse;
|
|
1432
|
+
return {
|
|
1433
|
+
runSync: true,
|
|
1434
|
+
status: OK,
|
|
1435
|
+
warn: NON_POST_DEPRECATION_WARNING
|
|
1436
|
+
};
|
|
1437
|
+
}
|
|
1438
|
+
//#endregion
|
|
1380
1439
|
//#region ../sdk/src/trigger-url.ts
|
|
1381
1440
|
const SCHEME_REGEX = /^https?:\/\//;
|
|
1382
1441
|
const INVALID_HOST_CHARS_REGEX = /[/\\@?#\s]/;
|
|
@@ -1688,7 +1747,7 @@ function buildSchema(config) {
|
|
|
1688
1747
|
});
|
|
1689
1748
|
}
|
|
1690
1749
|
const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
|
|
1691
|
-
function
|
|
1750
|
+
function memoryCursorStore() {
|
|
1692
1751
|
const cursors = /* @__PURE__ */ new Map();
|
|
1693
1752
|
return {
|
|
1694
1753
|
get: (entity) => cursors.get(entity) ?? null,
|
|
@@ -1697,6 +1756,56 @@ function createMemoryCursorStore() {
|
|
|
1697
1756
|
}
|
|
1698
1757
|
};
|
|
1699
1758
|
}
|
|
1759
|
+
/** Default key prefix for {@link kvCursorStore}. */
|
|
1760
|
+
const CURSOR_KEY_PREFIX = "clivly:cursor:";
|
|
1761
|
+
/**
|
|
1762
|
+
* A persistent `CursorStore` backed by any key-value store.
|
|
1763
|
+
*
|
|
1764
|
+
* Serverless runtimes discard memory between invocations, so without a
|
|
1765
|
+
* persisted cursor every `delta` sync starts from `null` and re-reads the
|
|
1766
|
+
* whole source table. That is invisible — the run succeeds — and bills like a
|
|
1767
|
+
* full sync forever.
|
|
1768
|
+
*
|
|
1769
|
+
* This exists because the hand-written version has a trap in it:
|
|
1770
|
+
* `SyncCursor.time` is a `Date`, which a JSON round-trip turns into a string,
|
|
1771
|
+
* and `fromDrizzle` then calls `since.time.toISOString()` on it. The cursor is
|
|
1772
|
+
* stored as an ISO string and parsed back to a `Date` here so no caller has to
|
|
1773
|
+
* remember that.
|
|
1774
|
+
*
|
|
1775
|
+
* ```ts
|
|
1776
|
+
* cursorStore: kvCursorStore(env.CLIVLY_CURSORS) // Workers KV binding
|
|
1777
|
+
* ```
|
|
1778
|
+
*
|
|
1779
|
+
* A malformed or partial stored value resolves to `null` rather than throwing:
|
|
1780
|
+
* the worst case is one full re-read, where throwing would wedge sync entirely.
|
|
1781
|
+
*/
|
|
1782
|
+
function kvCursorStore(kv, options) {
|
|
1783
|
+
const prefix = options?.prefix ?? "clivly:cursor:";
|
|
1784
|
+
const keyFor = (entity) => `${prefix}${entity}`;
|
|
1785
|
+
return {
|
|
1786
|
+
get: async (entity) => {
|
|
1787
|
+
const raw = await kv.get(keyFor(entity));
|
|
1788
|
+
if (!raw) return null;
|
|
1789
|
+
try {
|
|
1790
|
+
const parsed = JSON.parse(raw);
|
|
1791
|
+
if (typeof parsed.id !== "string" || typeof parsed.time !== "string") return null;
|
|
1792
|
+
const time = new Date(parsed.time);
|
|
1793
|
+
return Number.isNaN(time.getTime()) ? null : {
|
|
1794
|
+
id: parsed.id,
|
|
1795
|
+
time
|
|
1796
|
+
};
|
|
1797
|
+
} catch {
|
|
1798
|
+
return null;
|
|
1799
|
+
}
|
|
1800
|
+
},
|
|
1801
|
+
set: async (entity, cursor) => {
|
|
1802
|
+
await kv.put(keyFor(entity), cursor === null ? "" : JSON.stringify({
|
|
1803
|
+
id: cursor.id,
|
|
1804
|
+
time: cursor.time.toISOString()
|
|
1805
|
+
}));
|
|
1806
|
+
}
|
|
1807
|
+
};
|
|
1808
|
+
}
|
|
1700
1809
|
function isLikelyServerless() {
|
|
1701
1810
|
const runtime = globalThis;
|
|
1702
1811
|
if (runtime.navigator?.userAgent === "Cloudflare-Workers") return true;
|
|
@@ -1926,9 +2035,11 @@ function createClivlySDK(config) {
|
|
|
1926
2035
|
let presenceClient = null;
|
|
1927
2036
|
let httpHeartbeatActive = true;
|
|
1928
2037
|
const batchSize = config.sync?.batchSize ?? DEFAULT_BATCH_SIZE;
|
|
1929
|
-
|
|
2038
|
+
if (serverless && !config.cursorStore) console.warn("Clivly: no `cursorStore` is configured and this looks like a serverless/edge runtime, where memory does not survive between invocations. Every `delta` sync will start from scratch and re-read your entire source tables — a full sync, on every run, billed as such. Pass a persistent store: `cursorStore: kvCursorStore(env.MY_KV)` (exported from `@clivly/sdk`). See https://clivly.com/docs/concepts/sync#cursor-stores");
|
|
2039
|
+
const cursorStore = config.cursorStore ?? memoryCursorStore();
|
|
1930
2040
|
const fetcher = createFetcher(config.retry);
|
|
1931
2041
|
let warnedUnsignedTrigger = false;
|
|
2042
|
+
let warnedNonPostTrigger = false;
|
|
1932
2043
|
const remoteProjections = createRemoteProjectionFetcher({
|
|
1933
2044
|
apiKey: config.apiKey,
|
|
1934
2045
|
apiUrl,
|
|
@@ -2090,6 +2201,28 @@ function createClivlySDK(config) {
|
|
|
2090
2201
|
lastPushFailure = null;
|
|
2091
2202
|
if (!await syncOne(resolved.source, resolved.path, "delta", resolved.objectType, runId, SAMPLE_BATCH_SIZE, SAMPLE_BATCH_SIZE, false)) throw pushFailureError(describePushFailure([resolved.source.entity]));
|
|
2092
2203
|
};
|
|
2204
|
+
const pushOne = async (entityKey, row) => {
|
|
2205
|
+
if (config.sync?.instant !== true) return;
|
|
2206
|
+
if (isInstantPushDisabled(config.env ?? process.env)) return;
|
|
2207
|
+
const resolved = resolveEntitySource(config, entityKey);
|
|
2208
|
+
if (!resolved) {
|
|
2209
|
+
console.warn(`Clivly instant push: no source is configured for entity "${entityKey}" — this row was not pushed. It will still sync on the next delta run once \`entities\` declares it.`);
|
|
2210
|
+
return;
|
|
2211
|
+
}
|
|
2212
|
+
const path = CONCEPT_PATHS[resolved.expectedKey];
|
|
2213
|
+
if (!path) return;
|
|
2214
|
+
const push = createPush();
|
|
2215
|
+
try {
|
|
2216
|
+
if (!await push(path, {
|
|
2217
|
+
source: pushSourceOf(resolved.source),
|
|
2218
|
+
rows: [row],
|
|
2219
|
+
mode: "delta",
|
|
2220
|
+
...resolved.expectedKey === "custom" ? { objectType: resolved.source.objectType } : {}
|
|
2221
|
+
})) console.warn(`Clivly instant push failed for "${entityKey}". This row will still sync on the next delta run.`);
|
|
2222
|
+
} catch (error) {
|
|
2223
|
+
console.warn(`Clivly instant push failed for "${entityKey}": ${error instanceof Error ? error.message : String(error)}. This row will still sync on the next delta run.`);
|
|
2224
|
+
}
|
|
2225
|
+
};
|
|
2093
2226
|
const describeSyncTrigger = () => ({
|
|
2094
2227
|
configured: config.syncTrigger?.url,
|
|
2095
2228
|
resolved: resolveTriggerUrl({
|
|
@@ -2332,8 +2465,45 @@ function createClivlySDK(config) {
|
|
|
2332
2465
|
return {};
|
|
2333
2466
|
}
|
|
2334
2467
|
};
|
|
2468
|
+
/**
|
|
2469
|
+
* The response to send instead of running a sync, or null to proceed.
|
|
2470
|
+
*
|
|
2471
|
+
* Split out of the handler so the method decision reads as one thing rather
|
|
2472
|
+
* than three branches ahead of the request plumbing — the same shape as
|
|
2473
|
+
* `triggerRequestAllowed` below.
|
|
2474
|
+
*/
|
|
2475
|
+
const refuseDisallowedMethod = (req) => {
|
|
2476
|
+
const decision = resolveTriggerMethod({
|
|
2477
|
+
enforceDefault: false,
|
|
2478
|
+
env: config.env ?? process.env,
|
|
2479
|
+
method: req.method
|
|
2480
|
+
});
|
|
2481
|
+
if (decision.warn && !warnedNonPostTrigger) {
|
|
2482
|
+
warnedNonPostTrigger = true;
|
|
2483
|
+
console.warn(decision.warn);
|
|
2484
|
+
}
|
|
2485
|
+
if (decision.runSync) return null;
|
|
2486
|
+
return decision.body ? jsonResponse(decision.body, decision.status) : new Response(null, { status: decision.status });
|
|
2487
|
+
};
|
|
2488
|
+
/**
|
|
2489
|
+
* Serve a preview request, or null when this payload is a real sync.
|
|
2490
|
+
*
|
|
2491
|
+
* Preview (T10) is a signed, ZERO-WRITE dry run: it returns ≤5 RAW source
|
|
2492
|
+
* rows, slimmed to only the columns the draft maps or filters on and coerced
|
|
2493
|
+
* to JSON scalars, which the cloud then projects through the same path as a
|
|
2494
|
+
* real sync. It never syncs, never pushes, never writes a cursor and never
|
|
2495
|
+
* ingests. Callers keep it inside their `try` so a malformed body or a
|
|
2496
|
+
* throwing `fetchPage` degrades to the same graceful 500 as the sync paths.
|
|
2497
|
+
*/
|
|
2498
|
+
const handleIfPreview = async (payload) => {
|
|
2499
|
+
if (payload.preview) return await handlePreviewRequest(config, payload.preview);
|
|
2500
|
+
if (payload.previewProjection) return await handleProjectionPreviewRequest(config, payload.previewProjection);
|
|
2501
|
+
return null;
|
|
2502
|
+
};
|
|
2335
2503
|
const createClivlyHandler = () => async (req) => {
|
|
2336
2504
|
observedUrl = observedTriggerUrl(req) ?? observedUrl;
|
|
2505
|
+
const refusal = refuseDisallowedMethod(req);
|
|
2506
|
+
if (refusal) return refusal;
|
|
2337
2507
|
const rawBody = await req.text().catch(() => "");
|
|
2338
2508
|
if (!await triggerRequestAllowed(req, rawBody)) return jsonResponse({
|
|
2339
2509
|
ok: false,
|
|
@@ -2343,8 +2513,8 @@ function createClivlySDK(config) {
|
|
|
2343
2513
|
const runId = payload.runId ?? null;
|
|
2344
2514
|
const startedAt = Date.now();
|
|
2345
2515
|
try {
|
|
2346
|
-
|
|
2347
|
-
if (
|
|
2516
|
+
const previewResponse = await handleIfPreview(payload);
|
|
2517
|
+
if (previewResponse) return previewResponse;
|
|
2348
2518
|
if (payload.sample) await runSampleSync(payload.sample.entityKey, payload.runId);
|
|
2349
2519
|
else await runScheduled({
|
|
2350
2520
|
mode: payload.mode,
|
|
@@ -2445,10 +2615,17 @@ function createClivlySDK(config) {
|
|
|
2445
2615
|
runScheduled,
|
|
2446
2616
|
createClivlyHandler,
|
|
2447
2617
|
sync: runSync,
|
|
2448
|
-
createAuthVerifyHandler
|
|
2618
|
+
createAuthVerifyHandler,
|
|
2619
|
+
pushOne
|
|
2449
2620
|
};
|
|
2450
2621
|
}
|
|
2451
2622
|
//#endregion
|
|
2623
|
+
Object.defineProperty(exports, "CURSOR_KEY_PREFIX", {
|
|
2624
|
+
enumerable: true,
|
|
2625
|
+
get: function() {
|
|
2626
|
+
return CURSOR_KEY_PREFIX;
|
|
2627
|
+
}
|
|
2628
|
+
});
|
|
2452
2629
|
Object.defineProperty(exports, "ClivlyAuthError", {
|
|
2453
2630
|
enumerable: true,
|
|
2454
2631
|
get: function() {
|
|
@@ -2491,3 +2668,15 @@ Object.defineProperty(exports, "isPlaceholderUrl", {
|
|
|
2491
2668
|
return isPlaceholderUrl;
|
|
2492
2669
|
}
|
|
2493
2670
|
});
|
|
2671
|
+
Object.defineProperty(exports, "kvCursorStore", {
|
|
2672
|
+
enumerable: true,
|
|
2673
|
+
get: function() {
|
|
2674
|
+
return kvCursorStore;
|
|
2675
|
+
}
|
|
2676
|
+
});
|
|
2677
|
+
Object.defineProperty(exports, "memoryCursorStore", {
|
|
2678
|
+
enumerable: true,
|
|
2679
|
+
get: function() {
|
|
2680
|
+
return memoryCursorStore;
|
|
2681
|
+
}
|
|
2682
|
+
});
|
package/sdk.cjs
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
|
|
2
2
|
const require_entity_config = require("./entity-config-BI4RYNKL.cjs");
|
|
3
|
-
const require_sdk = require("./sdk-
|
|
3
|
+
const require_sdk = require("./sdk-ZMy4C4VK.cjs");
|
|
4
|
+
exports.CURSOR_KEY_PREFIX = require_sdk.CURSOR_KEY_PREFIX;
|
|
4
5
|
exports.ClivlyAuthError = require_sdk.ClivlyAuthError;
|
|
5
6
|
exports.ClivlyError = require_sdk.ClivlyError;
|
|
6
7
|
exports.ClivlyNetworkError = require_sdk.ClivlyNetworkError;
|
|
@@ -10,4 +11,6 @@ exports.createClivlySDK = require_sdk.createClivlySDK;
|
|
|
10
11
|
exports.customObjectDeclarations = require_entity_config.customObjectDeclarations;
|
|
11
12
|
exports.defineClivlyConfig = require_entity_config.defineClivlyConfig;
|
|
12
13
|
exports.isPlaceholderUrl = require_sdk.isPlaceholderUrl;
|
|
14
|
+
exports.kvCursorStore = require_sdk.kvCursorStore;
|
|
15
|
+
exports.memoryCursorStore = require_sdk.memoryCursorStore;
|
|
13
16
|
exports.validateEntitiesConfig = require_entity_config.validateEntitiesConfig;
|
package/sdk.d.cts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { C as customObjectDeclarations, M as validateEntitiesConfig, a as ClivlyEntitiesConfig, f as CustomObjectDeclaration, l as ConfigValidationError, m as DiscoveredSchemaTable, u as ConfigValidationResult, w as defineClivlyConfig } from "./entity-config-ApnT5j3X.cjs";
|
|
2
|
-
import { _ as RemoteProjectionsDescription, a as ClivlyConnectResult, c as ClivlyUser, d as DiscoveredTable, f as DoctorCheck, g as EnsuredObjectTypeResult, h as EnsuredMappingResult, i as ClivlyConnectReason, l as CreateChatSessionHandlerOptions, m as EnsureSchemaResult, n as ChatSessionRequestBody, o as ClivlySDK, p as DoctorReport, r as ChatWidgetVisitor, s as ClivlySDKConfig, u as CursorStore, y as SyncTriggerDescription } from "./namespace-probe-
|
|
2
|
+
import { _ as RemoteProjectionsDescription, a as ClivlyConnectResult, c as ClivlyUser, d as DiscoveredTable, f as DoctorCheck, g as EnsuredObjectTypeResult, h as EnsuredMappingResult, i as ClivlyConnectReason, l as CreateChatSessionHandlerOptions, m as EnsureSchemaResult, n as ChatSessionRequestBody, o as ClivlySDK, p as DoctorReport, r as ChatWidgetVisitor, s as ClivlySDKConfig, u as CursorStore, y as SyncTriggerDescription } from "./namespace-probe-H7yWc0JV.cjs";
|
|
3
3
|
//#region ../sdk/src/trigger-url.d.ts
|
|
4
4
|
export declare function isPlaceholderUrl(value: string): boolean;
|
|
5
5
|
//#endregion
|
|
@@ -28,6 +28,38 @@ export declare class ClivlyRateLimitError extends ClivlyError {}
|
|
|
28
28
|
export declare class ClivlyNetworkError extends ClivlyError {}
|
|
29
29
|
//#endregion
|
|
30
30
|
//#region ../sdk/src/index.d.ts
|
|
31
|
+
export declare function memoryCursorStore(): CursorStore;
|
|
32
|
+
/** The shape every key-value client shares: Workers KV, Upstash, node-redis. */
|
|
33
|
+
export interface CursorKV {
|
|
34
|
+
get(key: string): Promise<string | null> | string | null;
|
|
35
|
+
put(key: string, value: string): Promise<unknown> | unknown;
|
|
36
|
+
}
|
|
37
|
+
/** Default key prefix for {@link kvCursorStore}. */
|
|
38
|
+
export declare const CURSOR_KEY_PREFIX = "clivly:cursor:";
|
|
39
|
+
/**
|
|
40
|
+
* A persistent `CursorStore` backed by any key-value store.
|
|
41
|
+
*
|
|
42
|
+
* Serverless runtimes discard memory between invocations, so without a
|
|
43
|
+
* persisted cursor every `delta` sync starts from `null` and re-reads the
|
|
44
|
+
* whole source table. That is invisible — the run succeeds — and bills like a
|
|
45
|
+
* full sync forever.
|
|
46
|
+
*
|
|
47
|
+
* This exists because the hand-written version has a trap in it:
|
|
48
|
+
* `SyncCursor.time` is a `Date`, which a JSON round-trip turns into a string,
|
|
49
|
+
* and `fromDrizzle` then calls `since.time.toISOString()` on it. The cursor is
|
|
50
|
+
* stored as an ISO string and parsed back to a `Date` here so no caller has to
|
|
51
|
+
* remember that.
|
|
52
|
+
*
|
|
53
|
+
* ```ts
|
|
54
|
+
* cursorStore: kvCursorStore(env.CLIVLY_CURSORS) // Workers KV binding
|
|
55
|
+
* ```
|
|
56
|
+
*
|
|
57
|
+
* A malformed or partial stored value resolves to `null` rather than throwing:
|
|
58
|
+
* the worst case is one full re-read, where throwing would wedge sync entirely.
|
|
59
|
+
*/
|
|
60
|
+
export declare function kvCursorStore(kv: CursorKV, options?: {
|
|
61
|
+
prefix?: string;
|
|
62
|
+
}): CursorStore;
|
|
31
63
|
/**
|
|
32
64
|
* Create the Clivly SDK. The implemented surface — heartbeat/discovery reporting
|
|
33
65
|
* and the auth-verify handler — connects a developer's app to a running Clivly
|
package/sdk.d.mts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { C as customObjectDeclarations, M as validateEntitiesConfig, a as ClivlyEntitiesConfig, f as CustomObjectDeclaration, l as ConfigValidationError, m as DiscoveredSchemaTable, u as ConfigValidationResult, w as defineClivlyConfig } from "./entity-config-CKcj4-M_.mjs";
|
|
2
|
-
import { _ as RemoteProjectionsDescription, a as ClivlyConnectResult, c as ClivlyUser, d as DiscoveredTable, f as DoctorCheck, g as EnsuredObjectTypeResult, h as EnsuredMappingResult, i as ClivlyConnectReason, l as CreateChatSessionHandlerOptions, m as EnsureSchemaResult, n as ChatSessionRequestBody, o as ClivlySDK, p as DoctorReport, r as ChatWidgetVisitor, s as ClivlySDKConfig, u as CursorStore, y as SyncTriggerDescription } from "./namespace-probe-
|
|
2
|
+
import { _ as RemoteProjectionsDescription, a as ClivlyConnectResult, c as ClivlyUser, d as DiscoveredTable, f as DoctorCheck, g as EnsuredObjectTypeResult, h as EnsuredMappingResult, i as ClivlyConnectReason, l as CreateChatSessionHandlerOptions, m as EnsureSchemaResult, n as ChatSessionRequestBody, o as ClivlySDK, p as DoctorReport, r as ChatWidgetVisitor, s as ClivlySDKConfig, u as CursorStore, y as SyncTriggerDescription } from "./namespace-probe-DykR0kgh.mjs";
|
|
3
3
|
//#region ../sdk/src/trigger-url.d.ts
|
|
4
4
|
export declare function isPlaceholderUrl(value: string): boolean;
|
|
5
5
|
//#endregion
|
|
@@ -28,6 +28,38 @@ export declare class ClivlyRateLimitError extends ClivlyError {}
|
|
|
28
28
|
export declare class ClivlyNetworkError extends ClivlyError {}
|
|
29
29
|
//#endregion
|
|
30
30
|
//#region ../sdk/src/index.d.ts
|
|
31
|
+
export declare function memoryCursorStore(): CursorStore;
|
|
32
|
+
/** The shape every key-value client shares: Workers KV, Upstash, node-redis. */
|
|
33
|
+
export interface CursorKV {
|
|
34
|
+
get(key: string): Promise<string | null> | string | null;
|
|
35
|
+
put(key: string, value: string): Promise<unknown> | unknown;
|
|
36
|
+
}
|
|
37
|
+
/** Default key prefix for {@link kvCursorStore}. */
|
|
38
|
+
export declare const CURSOR_KEY_PREFIX = "clivly:cursor:";
|
|
39
|
+
/**
|
|
40
|
+
* A persistent `CursorStore` backed by any key-value store.
|
|
41
|
+
*
|
|
42
|
+
* Serverless runtimes discard memory between invocations, so without a
|
|
43
|
+
* persisted cursor every `delta` sync starts from `null` and re-reads the
|
|
44
|
+
* whole source table. That is invisible — the run succeeds — and bills like a
|
|
45
|
+
* full sync forever.
|
|
46
|
+
*
|
|
47
|
+
* This exists because the hand-written version has a trap in it:
|
|
48
|
+
* `SyncCursor.time` is a `Date`, which a JSON round-trip turns into a string,
|
|
49
|
+
* and `fromDrizzle` then calls `since.time.toISOString()` on it. The cursor is
|
|
50
|
+
* stored as an ISO string and parsed back to a `Date` here so no caller has to
|
|
51
|
+
* remember that.
|
|
52
|
+
*
|
|
53
|
+
* ```ts
|
|
54
|
+
* cursorStore: kvCursorStore(env.CLIVLY_CURSORS) // Workers KV binding
|
|
55
|
+
* ```
|
|
56
|
+
*
|
|
57
|
+
* A malformed or partial stored value resolves to `null` rather than throwing:
|
|
58
|
+
* the worst case is one full re-read, where throwing would wedge sync entirely.
|
|
59
|
+
*/
|
|
60
|
+
export declare function kvCursorStore(kv: CursorKV, options?: {
|
|
61
|
+
prefix?: string;
|
|
62
|
+
}): CursorStore;
|
|
31
63
|
/**
|
|
32
64
|
* Create the Clivly SDK. The implemented surface — heartbeat/discovery reporting
|
|
33
65
|
* and the auth-verify handler — connects a developer's app to a running Clivly
|
package/sdk.mjs
CHANGED
|
@@ -1,3 +1,3 @@
|
|
|
1
1
|
import { c as defineClivlyConfig, g as validateEntitiesConfig, s as customObjectDeclarations } from "./entity-config-Ccfb2IL9.mjs";
|
|
2
|
-
import { a as ClivlyRateLimitError, i as
|
|
3
|
-
export { ClivlyAuthError, ClivlyError, ClivlyNetworkError, ClivlyRateLimitError, createChatSessionHandler, createClivlySDK, customObjectDeclarations, defineClivlyConfig, isPlaceholderUrl, validateEntitiesConfig };
|
|
2
|
+
import { a as ClivlyAuthError, c as ClivlyRateLimitError, i as memoryCursorStore, l as createChatSessionHandler, n as createClivlySDK, o as ClivlyError, r as kvCursorStore, s as ClivlyNetworkError, t as CURSOR_KEY_PREFIX, u as isPlaceholderUrl } from "./sdk-DLoVPlGB.mjs";
|
|
3
|
+
export { CURSOR_KEY_PREFIX, ClivlyAuthError, ClivlyError, ClivlyNetworkError, ClivlyRateLimitError, createChatSessionHandler, createClivlySDK, customObjectDeclarations, defineClivlyConfig, isPlaceholderUrl, kvCursorStore, memoryCursorStore, validateEntitiesConfig };
|