@clivly/sdk 0.13.0 → 0.15.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +94 -21
- package/dist/index.cjs +141 -4
- package/dist/index.d.cts +32 -0
- package/dist/index.d.mts +32 -0
- package/dist/index.mjs +139 -5
- package/package.json +3 -3
package/README.md
CHANGED
|
@@ -272,10 +272,16 @@ production runtimes.
|
|
|
272
272
|
// Cloudflare Workers
|
|
273
273
|
export default {
|
|
274
274
|
scheduled: () => clivly.runScheduled(), // cron: one heartbeat + one sync
|
|
275
|
-
fetch: clivly.createClivlyHandler(), //
|
|
275
|
+
fetch: clivly.createClivlyHandler(), // the route Clivly Cloud POSTs to
|
|
276
276
|
};
|
|
277
277
|
```
|
|
278
278
|
|
|
279
|
+
**Only `POST` triggers a sync.** `HEAD` answers `204` and does no work — that
|
|
280
|
+
is the liveness probe. `GET` still syncs but logs a deprecation warning, and a
|
|
281
|
+
future release will refuse it with `405`, so point any cron or uptime check at
|
|
282
|
+
`POST`. Set `CLIVLY_TRIGGER_REQUIRE_POST=true` to adopt that now and confirm
|
|
283
|
+
nothing in your setup depends on `GET`.
|
|
284
|
+
|
|
279
285
|
`runScheduled()` throws if a sync push fails, so a cron run is marked failed.
|
|
280
286
|
`createClivlyHandler()` returns `200` when the triggered run finishes
|
|
281
287
|
successfully and `500` otherwise, with `{ ok, runId, durationMs }` or
|
|
@@ -350,7 +356,8 @@ it, dashboard-authored objects are fetched and then skipped; nothing errors, and
|
|
|
350
356
|
| Nothing connects, no error | You imported the config but never called an entry point | Call `start()` / `runScheduled()`, or add `clivlyVite(sdk)` |
|
|
351
357
|
| `reason: "invalid_api_key"` | `CLIVLY_SECRET_KEY` missing, wrong, or revoked by a regeneration | Re-run `npx clivly login` |
|
|
352
358
|
| Sync works once on serverless, then stops | `start()`'s `setInterval` cannot persist | Use `runScheduled()` or `createClivlyHandler()` |
|
|
353
|
-
| Every
|
|
359
|
+
| Every run re-reads everything | The default cursor store is in-memory, which serverless discards between invocations | Pass [`kvCursorStore`](#persisting-the-delta-cursor) |
|
|
360
|
+
| A cron stopped triggering syncs | Only `POST` triggers a sync | Change the caller to `POST` |
|
|
354
361
|
| `401` on the tick route | Trigger secret differs between your app and Clivly | Redeploy with the current `CLIVLY_SYNC_TRIGGER_SECRET` |
|
|
355
362
|
| An entity syncs nothing, silently | `source` keyed by entity key instead of CRM concept | Key it `contacts` / `companies` / `deals` / `custom` |
|
|
356
363
|
| Rows sync on insert but never on edit | `cursorField` does not change on edit | Add an `updated_at` column and use it. `doctor()` warns about this |
|
|
@@ -372,35 +379,92 @@ failed `connect()` result into a throwable.
|
|
|
372
379
|
|
|
373
380
|
Delta sync tracks a per-entity cursor so each run only pages rows changed since
|
|
374
381
|
the last one. By default that cursor lives in memory — fine for a long-lived
|
|
375
|
-
process, but **
|
|
376
|
-
re-
|
|
382
|
+
process, but **discarded between invocations on serverless**. Every delta run
|
|
383
|
+
then starts from nothing and re-reads your entire source tables: a full sync,
|
|
384
|
+
on every run, reported as a success. The SDK warns at startup when it detects a
|
|
385
|
+
serverless runtime with no `cursorStore`, and `clivly status` flags it.
|
|
386
|
+
|
|
387
|
+
Use the store the SDK ships. It works with any key-value client that has `get`
|
|
388
|
+
and `put` — Workers KV, Upstash, node-redis:
|
|
377
389
|
|
|
378
390
|
```ts
|
|
391
|
+
import { createClivlySDK, kvCursorStore } from "@clivly/sdk";
|
|
392
|
+
|
|
379
393
|
const clivly = createClivlySDK({
|
|
380
394
|
apiKey: process.env.CLIVLY_SECRET_KEY!,
|
|
381
395
|
entities,
|
|
382
396
|
source: { /* … */ },
|
|
383
|
-
cursorStore:
|
|
384
|
-
// `SyncCursor` is { id: string; time: Date } — serialize `time` yourself.
|
|
385
|
-
get: async (entity) => {
|
|
386
|
-
const raw = await kv.get(`clivly:cursor:${entity}`);
|
|
387
|
-
return raw ? { id: raw.id, time: new Date(raw.time) } : null;
|
|
388
|
-
},
|
|
389
|
-
set: (entity, cursor) =>
|
|
390
|
-
kv.put(
|
|
391
|
-
`clivly:cursor:${entity}`,
|
|
392
|
-
cursor && { id: cursor.id, time: cursor.time.toISOString() }
|
|
393
|
-
),
|
|
394
|
-
},
|
|
397
|
+
cursorStore: kvCursorStore(env.CLIVLY_CURSORS),
|
|
395
398
|
});
|
|
396
399
|
```
|
|
397
400
|
|
|
401
|
+
Writing your own is fine too, but mind the one trap: **`SyncCursor.time` is a
|
|
402
|
+
`Date`**. A JSON round-trip turns it into a string, and the sync path then
|
|
403
|
+
calls `.toISOString()` on it and throws. Serialize to ISO on the way out and
|
|
404
|
+
parse back to a `Date` on the way in — which is exactly what `kvCursorStore`
|
|
405
|
+
does for you:
|
|
406
|
+
|
|
407
|
+
```ts
|
|
408
|
+
cursorStore: {
|
|
409
|
+
get: async (entity) => {
|
|
410
|
+
const raw = await kv.get(`clivly:cursor:${entity}`);
|
|
411
|
+
if (!raw) return null;
|
|
412
|
+
const { id, time } = JSON.parse(raw);
|
|
413
|
+
return { id, time: new Date(time) }; // back to a Date
|
|
414
|
+
},
|
|
415
|
+
set: (entity, cursor) =>
|
|
416
|
+
kv.put(
|
|
417
|
+
`clivly:cursor:${entity}`,
|
|
418
|
+
cursor ? JSON.stringify({ id: cursor.id, time: cursor.time.toISOString() }) : ""
|
|
419
|
+
),
|
|
420
|
+
}
|
|
421
|
+
```
|
|
422
|
+
|
|
423
|
+
Return `null` rather than throwing on a missing or malformed value: the cost is
|
|
424
|
+
one full re-read, where throwing would stop sync entirely.
|
|
425
|
+
|
|
426
|
+
`memoryCursorStore()` is also exported, for tests and for long-lived hosts that
|
|
427
|
+
want the default named explicitly.
|
|
428
|
+
|
|
398
429
|
Delta sync is **at-least-once**: after a cold start with no persisted cursor,
|
|
399
430
|
rows since the last push may be re-sent (the server dedupes by id) — it never
|
|
400
431
|
silently skips rows. Use `sync({ mode: "full" })` when you need the server to
|
|
401
432
|
reconcile deletions (archive-on-leave); a delta cannot tell absence from
|
|
402
433
|
"unchanged".
|
|
403
434
|
|
|
435
|
+
### Pushing a row the moment it is written
|
|
436
|
+
|
|
437
|
+
Scheduled syncs pick new rows up eventually. To push one immediately (a signup,
|
|
438
|
+
say), opt in with `sync: { instant: true }` and wrap the insert with
|
|
439
|
+
`insertAndPush`. You create it once per mapped table, and it inserts,
|
|
440
|
+
translates the row and pushes it. Without the opt-in, both helpers do nothing
|
|
441
|
+
and the row arrives on the next delta sync instead. The `entity` argument is
|
|
442
|
+
the key from your `entities` config. A push never throws: a failed push only
|
|
443
|
+
logs a warning, and the next delta sync still delivers the row.
|
|
444
|
+
|
|
445
|
+
```ts
|
|
446
|
+
import { insertAndPush } from "clivly/drizzle";
|
|
447
|
+
|
|
448
|
+
const insertUser = insertAndPush(clivly, db, schema.users, { entity: "user" });
|
|
449
|
+
|
|
450
|
+
// In the signup handler. Pass `waitUntil` on Workers so the push outlives the response.
|
|
451
|
+
const created = await insertUser(newUserValues, c.executionCtx.waitUntil.bind(c.executionCtx));
|
|
452
|
+
```
|
|
453
|
+
|
|
454
|
+
When a library performs the insert itself (an auth library's lifecycle hook,
|
|
455
|
+
for example) there is no call to wrap. Re-read the row and translate it with
|
|
456
|
+
`toColumnKeyedRow`, then push it:
|
|
457
|
+
|
|
458
|
+
```ts
|
|
459
|
+
import { toColumnKeyedRow } from "clivly/drizzle";
|
|
460
|
+
|
|
461
|
+
await clivly.pushOne("user", toColumnKeyedRow(schema.users, row));
|
|
462
|
+
```
|
|
463
|
+
|
|
464
|
+
The translation matters. A Drizzle row is keyed by property name (`fullName`),
|
|
465
|
+
but mappings and the cloud address columns by their database name
|
|
466
|
+
(`full_name`), so an untranslated row pushes empty fields without any error.
|
|
467
|
+
|
|
404
468
|
### Sync ordering
|
|
405
469
|
|
|
406
470
|
`runSync` pushes **companies first**, then contacts, then deals, then custom
|
|
@@ -418,9 +482,17 @@ membership**, each row carrying the join table's foreign-key column.
|
|
|
418
482
|
{ id: "u1", full_name: "Ada", email: "ada@acme.test", org_id: "globex" }
|
|
419
483
|
```
|
|
420
484
|
|
|
421
|
-
This mirrors what the embedded model produces with `LEFT JOIN <join table
|
|
422
|
-
|
|
423
|
-
|
|
485
|
+
This mirrors what the embedded model produces with `LEFT JOIN <join table>`.
|
|
486
|
+
Clivly keeps Ada as **one contact** (identity is her `id`, not `id` + company)
|
|
487
|
+
with **two memberships**, one per organisation. The contact list shows her once,
|
|
488
|
+
as `Acme +1`, and both organisations' contact counts include her. Her primary
|
|
489
|
+
organisation is the lowest company ref for a new contact, and it stays put
|
|
490
|
+
while she still belongs to it.
|
|
491
|
+
|
|
492
|
+
When she leaves one organisation, a **full** sync ends that membership and
|
|
493
|
+
keeps the contact and her other memberships live. A delta sync never ends a
|
|
494
|
+
membership, because a missing row may just be on the next page of the push.
|
|
495
|
+
She is archived only when she leaves the source entirely.
|
|
424
496
|
|
|
425
497
|
A person with no membership may omit the column entirely or send it as `null` —
|
|
426
498
|
both are accepted. But if **no** pushed row carries the column, the push is
|
|
@@ -450,13 +522,14 @@ is honoured. Other `4xx` responses return immediately.
|
|
|
450
522
|
| `entities` | `ClivlyEntitiesConfig` | — | From `defineClivlyConfig({ entities })` |
|
|
451
523
|
| `schema` | `DiscoveredTable[]` | — | Use `discoverFromDrizzle(schema)`. It reports keys, types, nullability and FK edges alongside column names; [derived objects](#derived-objects) cannot be authored without them, so prefer it over a hand-written array |
|
|
452
524
|
| `source` | `{ contacts, companies?, deals?, custom? }` | — | Keyed by CRM concept. `contacts` is required when `source` is present |
|
|
453
|
-
| `cursorStore` | `CursorStore` | in-memory |
|
|
525
|
+
| `cursorStore` | `CursorStore` | in-memory | **Required on serverless** — `kvCursorStore(kv)` |
|
|
454
526
|
| `heartbeatIntervalMs` | `number` | `60000` | HTTP presence cadence and default sync cadence; paused when opt-in WebSocket presence is ready |
|
|
455
527
|
| `presence.transport` | `"http" \\| "websocket"` | `"http"` | WebSocket is for long-lived Node hosts; HTTP resumes during reconnect/fallback |
|
|
456
528
|
| `presence.deploymentId` | `string` | platform-derived | Optional release identity for a WebSocket deployment |
|
|
457
529
|
| `sync.onBoot` | `boolean` | `true` | Only affects `start()` |
|
|
458
530
|
| `sync.intervalMs` | `number` | `heartbeatIntervalMs` | |
|
|
459
531
|
| `sync.batchSize` | `number` | `500` | Rows per page |
|
|
532
|
+
| `sync.instant` | `boolean` | `false` | Lets `pushOne()` / `insertAndPush()` push a single row immediately; see [Pushing a row the moment it is written](#pushing-a-row-the-moment-it-is-written) |
|
|
460
533
|
| `retry.maxAttempts` / `retry.baseDelayMs` | `number` | `3` / `500` | |
|
|
461
534
|
| `syncTrigger.path` / `.url` / `.secret` | `string` | — | See above |
|
|
462
535
|
| `verifyToken` | `string` | — | For `createAuthVerifyHandler`. **Nothing calls that endpoint today** |
|
|
@@ -518,7 +591,7 @@ rejected key and a dead network all mean the question went unanswered.
|
|
|
518
591
|
| From `clivly` | From `@clivly/sdk` | Contents |
|
|
519
592
|
| --- | --- | --- |
|
|
520
593
|
| `clivly/sdk` | `@clivly/sdk` | `createClivlySDK`, `defineClivlyConfig`, `createChatSessionHandler`, error classes, all types |
|
|
521
|
-
| `clivly/drizzle` | `@clivly/sdk/drizzle` | `fromDrizzle`, `discoverFromDrizzle`, `drizzleIntrospector`, `buildUniqueConstraints` |
|
|
594
|
+
| `clivly/drizzle` | `@clivly/sdk/drizzle` | `fromDrizzle`, `discoverFromDrizzle`, `drizzleIntrospector`, `buildUniqueConstraints`, `insertAndPush`, `toColumnKeyedRow` |
|
|
522
595
|
| `clivly/vite` | `@clivly/sdk/vite` | `clivlyVite` |
|
|
523
596
|
| — | `@clivly/sdk/chat-session` | `createChatSessionHandler` (also on the root) |
|
|
524
597
|
|
package/dist/index.cjs
CHANGED
|
@@ -1393,6 +1393,51 @@ async function fullSyncSource(args) {
|
|
|
1393
1393
|
};
|
|
1394
1394
|
}
|
|
1395
1395
|
//#endregion
|
|
1396
|
+
//#region src/trigger-method.ts
|
|
1397
|
+
/** Host-side override, mirroring `CLIVLY_INSTANT_PUSH_DISABLED`. */
|
|
1398
|
+
const REQUIRE_POST_ENV_VAR = "CLIVLY_TRIGGER_REQUIRE_POST";
|
|
1399
|
+
const NON_POST_DEPRECATION_WARNING = `Clivly: this sync trigger was called with GET. It still ran the sync, but a future release will refuse any method except POST with a 405. Change the caller — your cron job or uptime check — to POST. To adopt the new behaviour now, set ${REQUIRE_POST_ENV_VAR}=true.`;
|
|
1400
|
+
const METHOD_NOT_ALLOWED = 405;
|
|
1401
|
+
const NO_CONTENT = 204;
|
|
1402
|
+
const OK = 200;
|
|
1403
|
+
/**
|
|
1404
|
+
* Resolve the phase. Only the exact strings `"true"` and `"false"` count —
|
|
1405
|
+
* anything else (unset, empty, `"1"`, `"TRUE"`) falls through to the default
|
|
1406
|
+
* rather than guessing, so a typo can never silently disable a guard.
|
|
1407
|
+
*/
|
|
1408
|
+
function enforcementEnabled(input) {
|
|
1409
|
+
const override = input.env[REQUIRE_POST_ENV_VAR];
|
|
1410
|
+
if (override === "true") return true;
|
|
1411
|
+
if (override === "false") return false;
|
|
1412
|
+
return input.enforceDefault;
|
|
1413
|
+
}
|
|
1414
|
+
function resolveTriggerMethod(input) {
|
|
1415
|
+
const method = input.method.toUpperCase();
|
|
1416
|
+
if (method === "POST") return {
|
|
1417
|
+
runSync: true,
|
|
1418
|
+
status: OK
|
|
1419
|
+
};
|
|
1420
|
+
if (method === "HEAD") return {
|
|
1421
|
+
runSync: false,
|
|
1422
|
+
status: NO_CONTENT
|
|
1423
|
+
};
|
|
1424
|
+
const refuse = {
|
|
1425
|
+
body: {
|
|
1426
|
+
error: `Method ${method} is not allowed on the Clivly sync trigger. Use POST.`,
|
|
1427
|
+
ok: false
|
|
1428
|
+
},
|
|
1429
|
+
runSync: false,
|
|
1430
|
+
status: METHOD_NOT_ALLOWED
|
|
1431
|
+
};
|
|
1432
|
+
if (method !== "GET") return refuse;
|
|
1433
|
+
if (enforcementEnabled(input)) return refuse;
|
|
1434
|
+
return {
|
|
1435
|
+
runSync: true,
|
|
1436
|
+
status: OK,
|
|
1437
|
+
warn: NON_POST_DEPRECATION_WARNING
|
|
1438
|
+
};
|
|
1439
|
+
}
|
|
1440
|
+
//#endregion
|
|
1396
1441
|
//#region src/trigger-url.ts
|
|
1397
1442
|
const SCHEME_REGEX = /^https?:\/\//;
|
|
1398
1443
|
const INVALID_HOST_CHARS_REGEX = /[/\\@?#\s]/;
|
|
@@ -1584,7 +1629,7 @@ function buildSchema(config) {
|
|
|
1584
1629
|
});
|
|
1585
1630
|
}
|
|
1586
1631
|
const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
|
|
1587
|
-
function
|
|
1632
|
+
function memoryCursorStore() {
|
|
1588
1633
|
const cursors = /* @__PURE__ */ new Map();
|
|
1589
1634
|
return {
|
|
1590
1635
|
get: (entity) => cursors.get(entity) ?? null,
|
|
@@ -1593,6 +1638,56 @@ function createMemoryCursorStore() {
|
|
|
1593
1638
|
}
|
|
1594
1639
|
};
|
|
1595
1640
|
}
|
|
1641
|
+
/** Default key prefix for {@link kvCursorStore}. */
|
|
1642
|
+
const CURSOR_KEY_PREFIX = "clivly:cursor:";
|
|
1643
|
+
/**
|
|
1644
|
+
* A persistent `CursorStore` backed by any key-value store.
|
|
1645
|
+
*
|
|
1646
|
+
* Serverless runtimes discard memory between invocations, so without a
|
|
1647
|
+
* persisted cursor every `delta` sync starts from `null` and re-reads the
|
|
1648
|
+
* whole source table. That is invisible — the run succeeds — and bills like a
|
|
1649
|
+
* full sync forever.
|
|
1650
|
+
*
|
|
1651
|
+
* This exists because the hand-written version has a trap in it:
|
|
1652
|
+
* `SyncCursor.time` is a `Date`, which a JSON round-trip turns into a string,
|
|
1653
|
+
* and `fromDrizzle` then calls `since.time.toISOString()` on it. The cursor is
|
|
1654
|
+
* stored as an ISO string and parsed back to a `Date` here so no caller has to
|
|
1655
|
+
* remember that.
|
|
1656
|
+
*
|
|
1657
|
+
* ```ts
|
|
1658
|
+
* cursorStore: kvCursorStore(env.CLIVLY_CURSORS) // Workers KV binding
|
|
1659
|
+
* ```
|
|
1660
|
+
*
|
|
1661
|
+
* A malformed or partial stored value resolves to `null` rather than throwing:
|
|
1662
|
+
* the worst case is one full re-read, where throwing would wedge sync entirely.
|
|
1663
|
+
*/
|
|
1664
|
+
function kvCursorStore(kv, options) {
|
|
1665
|
+
const prefix = options?.prefix ?? "clivly:cursor:";
|
|
1666
|
+
const keyFor = (entity) => `${prefix}${entity}`;
|
|
1667
|
+
return {
|
|
1668
|
+
get: async (entity) => {
|
|
1669
|
+
const raw = await kv.get(keyFor(entity));
|
|
1670
|
+
if (!raw) return null;
|
|
1671
|
+
try {
|
|
1672
|
+
const parsed = JSON.parse(raw);
|
|
1673
|
+
if (typeof parsed.id !== "string" || typeof parsed.time !== "string") return null;
|
|
1674
|
+
const time = new Date(parsed.time);
|
|
1675
|
+
return Number.isNaN(time.getTime()) ? null : {
|
|
1676
|
+
id: parsed.id,
|
|
1677
|
+
time
|
|
1678
|
+
};
|
|
1679
|
+
} catch {
|
|
1680
|
+
return null;
|
|
1681
|
+
}
|
|
1682
|
+
},
|
|
1683
|
+
set: async (entity, cursor) => {
|
|
1684
|
+
await kv.put(keyFor(entity), cursor === null ? "" : JSON.stringify({
|
|
1685
|
+
id: cursor.id,
|
|
1686
|
+
time: cursor.time.toISOString()
|
|
1687
|
+
}));
|
|
1688
|
+
}
|
|
1689
|
+
};
|
|
1690
|
+
}
|
|
1596
1691
|
function isLikelyServerless() {
|
|
1597
1692
|
const runtime = globalThis;
|
|
1598
1693
|
if (runtime.navigator?.userAgent === "Cloudflare-Workers") return true;
|
|
@@ -1822,9 +1917,11 @@ function createClivlySDK(config) {
|
|
|
1822
1917
|
let presenceClient = null;
|
|
1823
1918
|
let httpHeartbeatActive = true;
|
|
1824
1919
|
const batchSize = config.sync?.batchSize ?? DEFAULT_BATCH_SIZE;
|
|
1825
|
-
|
|
1920
|
+
if (serverless && !config.cursorStore) console.warn("Clivly: no `cursorStore` is configured and this looks like a serverless/edge runtime, where memory does not survive between invocations. Every `delta` sync will start from scratch and re-read your entire source tables — a full sync, on every run, billed as such. Pass a persistent store: `cursorStore: kvCursorStore(env.MY_KV)` (exported from `@clivly/sdk`). See https://clivly.com/docs/concepts/sync#cursor-stores");
|
|
1921
|
+
const cursorStore = config.cursorStore ?? memoryCursorStore();
|
|
1826
1922
|
const fetcher = createFetcher(config.retry);
|
|
1827
1923
|
let warnedUnsignedTrigger = false;
|
|
1924
|
+
let warnedNonPostTrigger = false;
|
|
1828
1925
|
const remoteProjections = createRemoteProjectionFetcher({
|
|
1829
1926
|
apiKey: config.apiKey,
|
|
1830
1927
|
apiUrl,
|
|
@@ -2250,8 +2347,45 @@ function createClivlySDK(config) {
|
|
|
2250
2347
|
return {};
|
|
2251
2348
|
}
|
|
2252
2349
|
};
|
|
2350
|
+
/**
|
|
2351
|
+
* The response to send instead of running a sync, or null to proceed.
|
|
2352
|
+
*
|
|
2353
|
+
* Split out of the handler so the method decision reads as one thing rather
|
|
2354
|
+
* than three branches ahead of the request plumbing — the same shape as
|
|
2355
|
+
* `triggerRequestAllowed` below.
|
|
2356
|
+
*/
|
|
2357
|
+
const refuseDisallowedMethod = (req) => {
|
|
2358
|
+
const decision = resolveTriggerMethod({
|
|
2359
|
+
enforceDefault: false,
|
|
2360
|
+
env: config.env ?? process.env,
|
|
2361
|
+
method: req.method
|
|
2362
|
+
});
|
|
2363
|
+
if (decision.warn && !warnedNonPostTrigger) {
|
|
2364
|
+
warnedNonPostTrigger = true;
|
|
2365
|
+
console.warn(decision.warn);
|
|
2366
|
+
}
|
|
2367
|
+
if (decision.runSync) return null;
|
|
2368
|
+
return decision.body ? jsonResponse(decision.body, decision.status) : new Response(null, { status: decision.status });
|
|
2369
|
+
};
|
|
2370
|
+
/**
|
|
2371
|
+
* Serve a preview request, or null when this payload is a real sync.
|
|
2372
|
+
*
|
|
2373
|
+
* Preview (T10) is a signed, ZERO-WRITE dry run: it returns ≤5 RAW source
|
|
2374
|
+
* rows, slimmed to only the columns the draft maps or filters on and coerced
|
|
2375
|
+
* to JSON scalars, which the cloud then projects through the same path as a
|
|
2376
|
+
* real sync. It never syncs, never pushes, never writes a cursor and never
|
|
2377
|
+
* ingests. Callers keep it inside their `try` so a malformed body or a
|
|
2378
|
+
* throwing `fetchPage` degrades to the same graceful 500 as the sync paths.
|
|
2379
|
+
*/
|
|
2380
|
+
const handleIfPreview = async (payload) => {
|
|
2381
|
+
if (payload.preview) return await handlePreviewRequest(config, payload.preview);
|
|
2382
|
+
if (payload.previewProjection) return await handleProjectionPreviewRequest(config, payload.previewProjection);
|
|
2383
|
+
return null;
|
|
2384
|
+
};
|
|
2253
2385
|
const createClivlyHandler = () => async (req) => {
|
|
2254
2386
|
observedUrl = observedTriggerUrl(req) ?? observedUrl;
|
|
2387
|
+
const refusal = refuseDisallowedMethod(req);
|
|
2388
|
+
if (refusal) return refusal;
|
|
2255
2389
|
const rawBody = await req.text().catch(() => "");
|
|
2256
2390
|
if (!await triggerRequestAllowed(req, rawBody)) return jsonResponse({
|
|
2257
2391
|
ok: false,
|
|
@@ -2261,8 +2395,8 @@ function createClivlySDK(config) {
|
|
|
2261
2395
|
const runId = payload.runId ?? null;
|
|
2262
2396
|
const startedAt = Date.now();
|
|
2263
2397
|
try {
|
|
2264
|
-
|
|
2265
|
-
if (
|
|
2398
|
+
const previewResponse = await handleIfPreview(payload);
|
|
2399
|
+
if (previewResponse) return previewResponse;
|
|
2266
2400
|
if (payload.sample) await runSampleSync(payload.sample.entityKey, payload.runId);
|
|
2267
2401
|
else await runScheduled({
|
|
2268
2402
|
mode: payload.mode,
|
|
@@ -2368,6 +2502,7 @@ function createClivlySDK(config) {
|
|
|
2368
2502
|
};
|
|
2369
2503
|
}
|
|
2370
2504
|
//#endregion
|
|
2505
|
+
exports.CURSOR_KEY_PREFIX = CURSOR_KEY_PREFIX;
|
|
2371
2506
|
exports.ClivlyAuthError = ClivlyAuthError;
|
|
2372
2507
|
exports.ClivlyError = ClivlyError;
|
|
2373
2508
|
exports.ClivlyNetworkError = ClivlyNetworkError;
|
|
@@ -2387,6 +2522,8 @@ Object.defineProperty(exports, "defineClivlyConfig", {
|
|
|
2387
2522
|
}
|
|
2388
2523
|
});
|
|
2389
2524
|
exports.isPlaceholderUrl = isPlaceholderUrl;
|
|
2525
|
+
exports.kvCursorStore = kvCursorStore;
|
|
2526
|
+
exports.memoryCursorStore = memoryCursorStore;
|
|
2390
2527
|
Object.defineProperty(exports, "validateEntitiesConfig", {
|
|
2391
2528
|
enumerable: true,
|
|
2392
2529
|
get: function() {
|
package/dist/index.d.cts
CHANGED
|
@@ -26,6 +26,38 @@ export declare class ClivlyRateLimitError extends ClivlyError {}
|
|
|
26
26
|
export declare class ClivlyNetworkError extends ClivlyError {}
|
|
27
27
|
//#endregion
|
|
28
28
|
//#region src/index.d.ts
|
|
29
|
+
export declare function memoryCursorStore(): CursorStore;
|
|
30
|
+
/** The shape every key-value client shares: Workers KV, Upstash, node-redis. */
|
|
31
|
+
export interface CursorKV {
|
|
32
|
+
get(key: string): Promise<string | null> | string | null;
|
|
33
|
+
put(key: string, value: string): Promise<unknown> | unknown;
|
|
34
|
+
}
|
|
35
|
+
/** Default key prefix for {@link kvCursorStore}. */
|
|
36
|
+
export declare const CURSOR_KEY_PREFIX = "clivly:cursor:";
|
|
37
|
+
/**
|
|
38
|
+
* A persistent `CursorStore` backed by any key-value store.
|
|
39
|
+
*
|
|
40
|
+
* Serverless runtimes discard memory between invocations, so without a
|
|
41
|
+
* persisted cursor every `delta` sync starts from `null` and re-reads the
|
|
42
|
+
* whole source table. That is invisible — the run succeeds — and bills like a
|
|
43
|
+
* full sync forever.
|
|
44
|
+
*
|
|
45
|
+
* This exists because the hand-written version has a trap in it:
|
|
46
|
+
* `SyncCursor.time` is a `Date`, which a JSON round-trip turns into a string,
|
|
47
|
+
* and `fromDrizzle` then calls `since.time.toISOString()` on it. The cursor is
|
|
48
|
+
* stored as an ISO string and parsed back to a `Date` here so no caller has to
|
|
49
|
+
* remember that.
|
|
50
|
+
*
|
|
51
|
+
* ```ts
|
|
52
|
+
* cursorStore: kvCursorStore(env.CLIVLY_CURSORS) // Workers KV binding
|
|
53
|
+
* ```
|
|
54
|
+
*
|
|
55
|
+
* A malformed or partial stored value resolves to `null` rather than throwing:
|
|
56
|
+
* the worst case is one full re-read, where throwing would wedge sync entirely.
|
|
57
|
+
*/
|
|
58
|
+
export declare function kvCursorStore(kv: CursorKV, options?: {
|
|
59
|
+
prefix?: string;
|
|
60
|
+
}): CursorStore;
|
|
29
61
|
/**
|
|
30
62
|
* Create the Clivly SDK. The implemented surface — heartbeat/discovery reporting
|
|
31
63
|
* and the auth-verify handler — connects a developer's app to a running Clivly
|
package/dist/index.d.mts
CHANGED
|
@@ -26,6 +26,38 @@ export declare class ClivlyRateLimitError extends ClivlyError {}
|
|
|
26
26
|
export declare class ClivlyNetworkError extends ClivlyError {}
|
|
27
27
|
//#endregion
|
|
28
28
|
//#region src/index.d.ts
|
|
29
|
+
export declare function memoryCursorStore(): CursorStore;
|
|
30
|
+
/** The shape every key-value client shares: Workers KV, Upstash, node-redis. */
|
|
31
|
+
export interface CursorKV {
|
|
32
|
+
get(key: string): Promise<string | null> | string | null;
|
|
33
|
+
put(key: string, value: string): Promise<unknown> | unknown;
|
|
34
|
+
}
|
|
35
|
+
/** Default key prefix for {@link kvCursorStore}. */
|
|
36
|
+
export declare const CURSOR_KEY_PREFIX = "clivly:cursor:";
|
|
37
|
+
/**
|
|
38
|
+
* A persistent `CursorStore` backed by any key-value store.
|
|
39
|
+
*
|
|
40
|
+
* Serverless runtimes discard memory between invocations, so without a
|
|
41
|
+
* persisted cursor every `delta` sync starts from `null` and re-reads the
|
|
42
|
+
* whole source table. That is invisible — the run succeeds — and bills like a
|
|
43
|
+
* full sync forever.
|
|
44
|
+
*
|
|
45
|
+
* This exists because the hand-written version has a trap in it:
|
|
46
|
+
* `SyncCursor.time` is a `Date`, which a JSON round-trip turns into a string,
|
|
47
|
+
* and `fromDrizzle` then calls `since.time.toISOString()` on it. The cursor is
|
|
48
|
+
* stored as an ISO string and parsed back to a `Date` here so no caller has to
|
|
49
|
+
* remember that.
|
|
50
|
+
*
|
|
51
|
+
* ```ts
|
|
52
|
+
* cursorStore: kvCursorStore(env.CLIVLY_CURSORS) // Workers KV binding
|
|
53
|
+
* ```
|
|
54
|
+
*
|
|
55
|
+
* A malformed or partial stored value resolves to `null` rather than throwing:
|
|
56
|
+
* the worst case is one full re-read, where throwing would wedge sync entirely.
|
|
57
|
+
*/
|
|
58
|
+
export declare function kvCursorStore(kv: CursorKV, options?: {
|
|
59
|
+
prefix?: string;
|
|
60
|
+
}): CursorStore;
|
|
29
61
|
/**
|
|
30
62
|
* Create the Clivly SDK. The implemented surface — heartbeat/discovery reporting
|
|
31
63
|
* and the auth-verify handler — connects a developer's app to a running Clivly
|
package/dist/index.mjs
CHANGED
|
@@ -1392,6 +1392,51 @@ async function fullSyncSource(args) {
|
|
|
1392
1392
|
};
|
|
1393
1393
|
}
|
|
1394
1394
|
//#endregion
|
|
1395
|
+
//#region src/trigger-method.ts
|
|
1396
|
+
/** Host-side override, mirroring `CLIVLY_INSTANT_PUSH_DISABLED`. */
|
|
1397
|
+
const REQUIRE_POST_ENV_VAR = "CLIVLY_TRIGGER_REQUIRE_POST";
|
|
1398
|
+
const NON_POST_DEPRECATION_WARNING = `Clivly: this sync trigger was called with GET. It still ran the sync, but a future release will refuse any method except POST with a 405. Change the caller — your cron job or uptime check — to POST. To adopt the new behaviour now, set ${REQUIRE_POST_ENV_VAR}=true.`;
|
|
1399
|
+
const METHOD_NOT_ALLOWED = 405;
|
|
1400
|
+
const NO_CONTENT = 204;
|
|
1401
|
+
const OK = 200;
|
|
1402
|
+
/**
|
|
1403
|
+
* Resolve the phase. Only the exact strings `"true"` and `"false"` count —
|
|
1404
|
+
* anything else (unset, empty, `"1"`, `"TRUE"`) falls through to the default
|
|
1405
|
+
* rather than guessing, so a typo can never silently disable a guard.
|
|
1406
|
+
*/
|
|
1407
|
+
function enforcementEnabled(input) {
|
|
1408
|
+
const override = input.env[REQUIRE_POST_ENV_VAR];
|
|
1409
|
+
if (override === "true") return true;
|
|
1410
|
+
if (override === "false") return false;
|
|
1411
|
+
return input.enforceDefault;
|
|
1412
|
+
}
|
|
1413
|
+
function resolveTriggerMethod(input) {
|
|
1414
|
+
const method = input.method.toUpperCase();
|
|
1415
|
+
if (method === "POST") return {
|
|
1416
|
+
runSync: true,
|
|
1417
|
+
status: OK
|
|
1418
|
+
};
|
|
1419
|
+
if (method === "HEAD") return {
|
|
1420
|
+
runSync: false,
|
|
1421
|
+
status: NO_CONTENT
|
|
1422
|
+
};
|
|
1423
|
+
const refuse = {
|
|
1424
|
+
body: {
|
|
1425
|
+
error: `Method ${method} is not allowed on the Clivly sync trigger. Use POST.`,
|
|
1426
|
+
ok: false
|
|
1427
|
+
},
|
|
1428
|
+
runSync: false,
|
|
1429
|
+
status: METHOD_NOT_ALLOWED
|
|
1430
|
+
};
|
|
1431
|
+
if (method !== "GET") return refuse;
|
|
1432
|
+
if (enforcementEnabled(input)) return refuse;
|
|
1433
|
+
return {
|
|
1434
|
+
runSync: true,
|
|
1435
|
+
status: OK,
|
|
1436
|
+
warn: NON_POST_DEPRECATION_WARNING
|
|
1437
|
+
};
|
|
1438
|
+
}
|
|
1439
|
+
//#endregion
|
|
1395
1440
|
//#region src/trigger-url.ts
|
|
1396
1441
|
const SCHEME_REGEX = /^https?:\/\//;
|
|
1397
1442
|
const INVALID_HOST_CHARS_REGEX = /[/\\@?#\s]/;
|
|
@@ -1583,7 +1628,7 @@ function buildSchema(config) {
|
|
|
1583
1628
|
});
|
|
1584
1629
|
}
|
|
1585
1630
|
const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
|
|
1586
|
-
function
|
|
1631
|
+
function memoryCursorStore() {
|
|
1587
1632
|
const cursors = /* @__PURE__ */ new Map();
|
|
1588
1633
|
return {
|
|
1589
1634
|
get: (entity) => cursors.get(entity) ?? null,
|
|
@@ -1592,6 +1637,56 @@ function createMemoryCursorStore() {
|
|
|
1592
1637
|
}
|
|
1593
1638
|
};
|
|
1594
1639
|
}
|
|
1640
|
+
/** Default key prefix for {@link kvCursorStore}. */
|
|
1641
|
+
const CURSOR_KEY_PREFIX = "clivly:cursor:";
|
|
1642
|
+
/**
|
|
1643
|
+
* A persistent `CursorStore` backed by any key-value store.
|
|
1644
|
+
*
|
|
1645
|
+
* Serverless runtimes discard memory between invocations, so without a
|
|
1646
|
+
* persisted cursor every `delta` sync starts from `null` and re-reads the
|
|
1647
|
+
* whole source table. That is invisible — the run succeeds — and bills like a
|
|
1648
|
+
* full sync forever.
|
|
1649
|
+
*
|
|
1650
|
+
* This exists because the hand-written version has a trap in it:
|
|
1651
|
+
* `SyncCursor.time` is a `Date`, which a JSON round-trip turns into a string,
|
|
1652
|
+
* and `fromDrizzle` then calls `since.time.toISOString()` on it. The cursor is
|
|
1653
|
+
* stored as an ISO string and parsed back to a `Date` here so no caller has to
|
|
1654
|
+
* remember that.
|
|
1655
|
+
*
|
|
1656
|
+
* ```ts
|
|
1657
|
+
* cursorStore: kvCursorStore(env.CLIVLY_CURSORS) // Workers KV binding
|
|
1658
|
+
* ```
|
|
1659
|
+
*
|
|
1660
|
+
* A malformed or partial stored value resolves to `null` rather than throwing:
|
|
1661
|
+
* the worst case is one full re-read, where throwing would wedge sync entirely.
|
|
1662
|
+
*/
|
|
1663
|
+
function kvCursorStore(kv, options) {
|
|
1664
|
+
const prefix = options?.prefix ?? "clivly:cursor:";
|
|
1665
|
+
const keyFor = (entity) => `${prefix}${entity}`;
|
|
1666
|
+
return {
|
|
1667
|
+
get: async (entity) => {
|
|
1668
|
+
const raw = await kv.get(keyFor(entity));
|
|
1669
|
+
if (!raw) return null;
|
|
1670
|
+
try {
|
|
1671
|
+
const parsed = JSON.parse(raw);
|
|
1672
|
+
if (typeof parsed.id !== "string" || typeof parsed.time !== "string") return null;
|
|
1673
|
+
const time = new Date(parsed.time);
|
|
1674
|
+
return Number.isNaN(time.getTime()) ? null : {
|
|
1675
|
+
id: parsed.id,
|
|
1676
|
+
time
|
|
1677
|
+
};
|
|
1678
|
+
} catch {
|
|
1679
|
+
return null;
|
|
1680
|
+
}
|
|
1681
|
+
},
|
|
1682
|
+
set: async (entity, cursor) => {
|
|
1683
|
+
await kv.put(keyFor(entity), cursor === null ? "" : JSON.stringify({
|
|
1684
|
+
id: cursor.id,
|
|
1685
|
+
time: cursor.time.toISOString()
|
|
1686
|
+
}));
|
|
1687
|
+
}
|
|
1688
|
+
};
|
|
1689
|
+
}
|
|
1595
1690
|
function isLikelyServerless() {
|
|
1596
1691
|
const runtime = globalThis;
|
|
1597
1692
|
if (runtime.navigator?.userAgent === "Cloudflare-Workers") return true;
|
|
@@ -1821,9 +1916,11 @@ function createClivlySDK(config) {
|
|
|
1821
1916
|
let presenceClient = null;
|
|
1822
1917
|
let httpHeartbeatActive = true;
|
|
1823
1918
|
const batchSize = config.sync?.batchSize ?? DEFAULT_BATCH_SIZE;
|
|
1824
|
-
|
|
1919
|
+
if (serverless && !config.cursorStore) console.warn("Clivly: no `cursorStore` is configured and this looks like a serverless/edge runtime, where memory does not survive between invocations. Every `delta` sync will start from scratch and re-read your entire source tables — a full sync, on every run, billed as such. Pass a persistent store: `cursorStore: kvCursorStore(env.MY_KV)` (exported from `@clivly/sdk`). See https://clivly.com/docs/concepts/sync#cursor-stores");
|
|
1920
|
+
const cursorStore = config.cursorStore ?? memoryCursorStore();
|
|
1825
1921
|
const fetcher = createFetcher(config.retry);
|
|
1826
1922
|
let warnedUnsignedTrigger = false;
|
|
1923
|
+
let warnedNonPostTrigger = false;
|
|
1827
1924
|
const remoteProjections = createRemoteProjectionFetcher({
|
|
1828
1925
|
apiKey: config.apiKey,
|
|
1829
1926
|
apiUrl,
|
|
@@ -2249,8 +2346,45 @@ function createClivlySDK(config) {
|
|
|
2249
2346
|
return {};
|
|
2250
2347
|
}
|
|
2251
2348
|
};
|
|
2349
|
+
/**
|
|
2350
|
+
* The response to send instead of running a sync, or null to proceed.
|
|
2351
|
+
*
|
|
2352
|
+
* Split out of the handler so the method decision reads as one thing rather
|
|
2353
|
+
* than three branches ahead of the request plumbing — the same shape as
|
|
2354
|
+
* `triggerRequestAllowed` below.
|
|
2355
|
+
*/
|
|
2356
|
+
const refuseDisallowedMethod = (req) => {
|
|
2357
|
+
const decision = resolveTriggerMethod({
|
|
2358
|
+
enforceDefault: false,
|
|
2359
|
+
env: config.env ?? process.env,
|
|
2360
|
+
method: req.method
|
|
2361
|
+
});
|
|
2362
|
+
if (decision.warn && !warnedNonPostTrigger) {
|
|
2363
|
+
warnedNonPostTrigger = true;
|
|
2364
|
+
console.warn(decision.warn);
|
|
2365
|
+
}
|
|
2366
|
+
if (decision.runSync) return null;
|
|
2367
|
+
return decision.body ? jsonResponse(decision.body, decision.status) : new Response(null, { status: decision.status });
|
|
2368
|
+
};
|
|
2369
|
+
/**
|
|
2370
|
+
* Serve a preview request, or null when this payload is a real sync.
|
|
2371
|
+
*
|
|
2372
|
+
* Preview (T10) is a signed, ZERO-WRITE dry run: it returns ≤5 RAW source
|
|
2373
|
+
* rows, slimmed to only the columns the draft maps or filters on and coerced
|
|
2374
|
+
* to JSON scalars, which the cloud then projects through the same path as a
|
|
2375
|
+
* real sync. It never syncs, never pushes, never writes a cursor and never
|
|
2376
|
+
* ingests. Callers keep it inside their `try` so a malformed body or a
|
|
2377
|
+
* throwing `fetchPage` degrades to the same graceful 500 as the sync paths.
|
|
2378
|
+
*/
|
|
2379
|
+
const handleIfPreview = async (payload) => {
|
|
2380
|
+
if (payload.preview) return await handlePreviewRequest(config, payload.preview);
|
|
2381
|
+
if (payload.previewProjection) return await handleProjectionPreviewRequest(config, payload.previewProjection);
|
|
2382
|
+
return null;
|
|
2383
|
+
};
|
|
2252
2384
|
const createClivlyHandler = () => async (req) => {
|
|
2253
2385
|
observedUrl = observedTriggerUrl(req) ?? observedUrl;
|
|
2386
|
+
const refusal = refuseDisallowedMethod(req);
|
|
2387
|
+
if (refusal) return refusal;
|
|
2254
2388
|
const rawBody = await req.text().catch(() => "");
|
|
2255
2389
|
if (!await triggerRequestAllowed(req, rawBody)) return jsonResponse({
|
|
2256
2390
|
ok: false,
|
|
@@ -2260,8 +2394,8 @@ function createClivlySDK(config) {
|
|
|
2260
2394
|
const runId = payload.runId ?? null;
|
|
2261
2395
|
const startedAt = Date.now();
|
|
2262
2396
|
try {
|
|
2263
|
-
|
|
2264
|
-
if (
|
|
2397
|
+
const previewResponse = await handleIfPreview(payload);
|
|
2398
|
+
if (previewResponse) return previewResponse;
|
|
2265
2399
|
if (payload.sample) await runSampleSync(payload.sample.entityKey, payload.runId);
|
|
2266
2400
|
else await runScheduled({
|
|
2267
2401
|
mode: payload.mode,
|
|
@@ -2367,4 +2501,4 @@ function createClivlySDK(config) {
|
|
|
2367
2501
|
};
|
|
2368
2502
|
}
|
|
2369
2503
|
//#endregion
|
|
2370
|
-
export { ClivlyAuthError, ClivlyError, ClivlyNetworkError, ClivlyRateLimitError, createChatSessionHandler, createClivlySDK, customObjectDeclarations, defineClivlyConfig, isPlaceholderUrl, validateEntitiesConfig };
|
|
2504
|
+
export { CURSOR_KEY_PREFIX, ClivlyAuthError, ClivlyError, ClivlyNetworkError, ClivlyRateLimitError, createChatSessionHandler, createClivlySDK, customObjectDeclarations, defineClivlyConfig, isPlaceholderUrl, kvCursorStore, memoryCursorStore, validateEntitiesConfig };
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@clivly/sdk",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.15.0",
|
|
4
4
|
"private": false,
|
|
5
5
|
"description": "Clivly SDK — connect your app to Clivly Cloud (heartbeat, discovery, auth bridge).",
|
|
6
6
|
"license": "MIT",
|
|
@@ -53,14 +53,14 @@
|
|
|
53
53
|
"README.md"
|
|
54
54
|
],
|
|
55
55
|
"dependencies": {
|
|
56
|
-
"@clivly/core": "0.
|
|
56
|
+
"@clivly/core": "0.15.0"
|
|
57
57
|
},
|
|
58
58
|
"devDependencies": {
|
|
59
59
|
"@electric-sql/pglite": "^0.5.4",
|
|
60
60
|
"drizzle-orm": "^0.45.2",
|
|
61
61
|
"tsdown": "^0.23.0",
|
|
62
62
|
"typescript": "^6",
|
|
63
|
-
"vite": "^8.3.
|
|
63
|
+
"vite": "^8.3.5",
|
|
64
64
|
"vitest": "^3.2.0"
|
|
65
65
|
},
|
|
66
66
|
"peerDependencies": {
|