create-ncblock 0.0.39 → 0.0.41

package/templates/worker/.agents/skills/sync-guide/examples/incremental-basic.ts

@@ -0,0 +1,103 @@
+/**
+ * Delta sync — opaque cursor for incremental change tracking.
+ *
+ * This is the delta half of a backfill+delta pair. It handles ongoing
+ * change detection for APIs that sort results by updated_at and return
+ * an opaque pagination cursor. On each cycle, the cursor picks up where
+ * the last cycle left off, fetching only newly modified records.
+ *
+ * Example: Shopify GraphQL (sortKey: UPDATED_AT), any API with opaque
+ * cursor pagination that returns results in modification order.
+ *
+ * Key points:
+ * - State is just { cursor: string | null } — no phase discrimination
+ * - First run: cursor is null, starts from the beginning
+ * - Subsequent runs: cursor picks up where the last cycle left off
+ * - The cursor never resets in incremental mode — it persists forever
+ * - Cursor preservation: if the API returns no endCursor (empty page at
+ *   frontier), keep the existing cursor rather than regressing to null
+ *
+ * For a full dataset load (backfill), add a separate replace-mode sync
+ * targeting the same database — see replace-paginated.ts for the pattern.
+ * Trigger it via CLI:
+ *   ntn workers sync state reset ordersBackfill && ntn workers sync trigger ordersBackfill
+ */
+
+import { Worker } from "@notionhq/workers"
+import * as Builder from "@notionhq/workers/builder"
+import * as Schema from "@notionhq/workers/schema"
+
+const worker = new Worker()
+export default worker
+
+// Database shared between backfill (replace) and delta (incremental) syncs
+const orders = worker.database("orders", {
+	type: "managed",
+	initialTitle: "Orders",
+	primaryKeyProperty: "Order ID",
+	schema: {
+		properties: {
+			Title: Schema.title(),
+			"Order ID": Schema.richText(),
+		},
+	},
+})
+
+// Rate-limit API calls — shared across all syncs hitting this API
+const apiPacer = worker.pacer("shopify", {
+	allowedRequests: 4,
+	intervalMs: 1000,
+})
+
+type CursorState = { cursor: string | null }
+
+// Delta sync: incremental mode, runs every 5 minutes to pick up changes
+worker.sync("ordersDelta", {
+	database: orders,
+	mode: "incremental",
+	schedule: "5m",
+	execute: async (state: CursorState | undefined) => {
+		const cursor = state?.cursor ?? null
+
+		// GraphQL query with Relay-style pagination, sorted by UPDATED_AT
+		const query = `
+      query ($after: String) {
+        orders(first: 100, sortKey: UPDATED_AT, after: $after) {
+          edges { node { id name } }
+          pageInfo { hasNextPage endCursor }
+        }
+      }
+    `
+
+		await apiPacer.wait()
+		const response = await fetch(
+			"https://shop.example.com/admin/api/graphql.json",
+			{
+				method: "POST",
+				headers: {
+					"Content-Type": "application/json",
+					"X-Access-Token": process.env.SHOP_TOKEN ?? "",
+				},
+				body: JSON.stringify({ query, variables: { after: cursor } }),
+			},
+		)
+		const { data } = await response.json()
+		const { edges, pageInfo } = data.orders
+
+		return {
+			changes: edges.map((edge: { node: { id: string; name: string } }) => ({
+				type: "upsert" as const,
+				key: edge.node.id,
+				properties: {
+					Title: Builder.title(edge.node.name),
+					"Order ID": Builder.richText(edge.node.id),
+				},
+			})),
+			hasMore: pageInfo.hasNextPage,
+			nextState: {
+				// Preserve existing cursor if API returns null (empty frontier)
+				cursor: pageInfo.endCursor ?? cursor,
+			},
+		}
+	},
+})

package/templates/worker/.agents/skills/sync-guide/examples/incremental-bimodal.ts

@@ -0,0 +1,207 @@
+/**
+ * Two-sync pattern — backfill + delta writing to the same database.
+ *
+ * Instead of a single sync with a bi-modal state machine (phase: "backfill" | "delta"),
+ * this uses two separate syncs targeting the same database:
+ *
+ * - **Backfill sync** (replace mode, manual schedule): Paginates the full dataset
+ *   using keyset pagination on (created_at, id). Triggered on demand via CLI.
+ * - **Delta sync** (incremental mode, 5m schedule): Fetches only recently modified
+ *   records using keyset pagination on (updated_at, id) with a 15s consistency buffer.
+ *
+ * Advantages over the single-sync bi-modal approach:
+ * - No phase discrimination in state — each sync has simple, focused state
+ * - No backfill-to-delta transition logic
+ * - Backfill and delta run independently — re-backfill anytime without disrupting delta
+ * - Easier to reason about and debug
+ *
+ * This is the Salesforce/HubSpot pattern:
+ * - Backfill: keyset pagination on (created_at, id)
+ * - Delta: keyset pagination on (updated_at, id) with consistency buffer
+ *
+ * To trigger a full re-backfill:
+ *   ntn workers sync state reset contactsBackfill && ntn workers sync trigger contactsBackfill
+ */
+
+import { Worker } from "@notionhq/workers"
+import * as Builder from "@notionhq/workers/builder"
+import * as Schema from "@notionhq/workers/schema"
+
+const worker = new Worker()
+export default worker
+
+// One database shared by both syncs
+const contacts = worker.database("contacts", {
+	type: "managed",
+	initialTitle: "Contacts",
+	primaryKeyProperty: "Contact ID",
+	schema: {
+		properties: {
+			Name: Schema.title(),
+			"Contact ID": Schema.richText(),
+		},
+	},
+})
+
+// One pacer shared by both syncs — budget is apportioned evenly
+const apiPacer = worker.pacer("crm", {
+	allowedRequests: 10,
+	intervalMs: 1000,
+})
+
+const BATCH_SIZE = 100
+const CONSISTENCY_BUFFER_MS = 15_000 // 15 seconds
+
+// ---------------------------------------------------------------------------
+// Shared helper — maps a contact record to a sync upsert change
+// ---------------------------------------------------------------------------
+function toUpsert(record: { id: string; name: string }) {
+	return {
+		type: "upsert" as const,
+		key: record.id,
+		properties: {
+			Name: Builder.title(record.name),
+			"Contact ID": Builder.richText(record.id),
+		},
+	}
+}
+
+// ---------------------------------------------------------------------------
+// Backfill sync: replace mode, manual schedule
+// ---------------------------------------------------------------------------
+// Paginates the full dataset using keyset pagination on (created_at, id).
+// Replace mode means the runtime will delete any records not seen during the
+// full cycle, ensuring the database is a faithful mirror of the source.
+
+type BackfillState = {
+	cursorTimestamp: string | null
+	cursorId: string | null
+}
+
+worker.sync("contactsBackfill", {
+	database: contacts,
+	mode: "replace",
+	schedule: "manual",
+	execute: async (state: BackfillState | undefined) => {
+		// Keyset pagination: WHERE created_at > X OR (created_at = X AND id > Y)
+		const params = new URLSearchParams({
+			limit: String(BATCH_SIZE),
+			order_by: "created_at,id",
+		})
+		if (state?.cursorTimestamp) {
+			params.set("created_after", state.cursorTimestamp)
+			params.set("created_after_id", state.cursorId ?? "")
+		}
+
+		await apiPacer.wait()
+		const response = await fetch(`https://api.example.com/contacts?${params}`, {
+			headers: { Authorization: `Bearer ${process.env.API_TOKEN}` },
+		})
+		const data = await response.json()
+		const records: Array<{
+			id: string
+			name: string
+			created_at: string
+		}> = data.contacts
+		const done = records.length < BATCH_SIZE
+
+		if (done) {
+			return {
+				changes: records.map(toUpsert),
+				hasMore: false,
+			}
+		}
+
+		const last = records[records.length - 1]
+		return {
+			changes: records.map(toUpsert),
+			hasMore: true,
+			nextState: {
+				cursorTimestamp: last.created_at,
+				cursorId: last.id,
+			},
+		}
+	},
+})
+
+// ---------------------------------------------------------------------------
+// Delta sync: incremental mode, every 5 minutes
+// ---------------------------------------------------------------------------
+// Fetches only records modified since the cursor. Uses keyset pagination on
+// (updated_at, id) with a 15s consistency buffer to avoid advancing past
+// records that the API hasn't indexed yet.
+
+type DeltaState = {
+	cursorTimestamp: string
+	cursorId: string
+}
+
+worker.sync("contactsDelta", {
+	database: contacts,
+	mode: "incremental",
+	schedule: "5m",
+	execute: async (state: DeltaState | undefined) => {
+		// On first run, start from "now" minus the consistency buffer.
+		// This means the first delta cycle won't fetch any historical data —
+		// that's the backfill sync's job.
+		if (!state) {
+			const startTs = new Date(Date.now() - CONSISTENCY_BUFFER_MS).toISOString()
+			return {
+				changes: [],
+				hasMore: false,
+				nextState: { cursorTimestamp: startTs, cursorId: "" },
+			}
+		}
+
+		const params = new URLSearchParams({
+			limit: String(BATCH_SIZE),
+			order_by: "updated_at,id",
+			updated_after: state.cursorTimestamp,
+			updated_after_id: state.cursorId,
+		})
+
+		await apiPacer.wait()
+		const response = await fetch(`https://api.example.com/contacts?${params}`, {
+			headers: { Authorization: `Bearer ${process.env.API_TOKEN}` },
+		})
+		const data = await response.json()
+		const records: Array<{
+			id: string
+			name: string
+			updated_at: string
+		}> = data.contacts
+		const done = records.length < BATCH_SIZE
+
+		// Consistency buffer: never advance the cursor closer than 15s to "now".
+		// In incremental mode the cursor never resets, so if we advance past a record
+		// that hasn't been indexed yet, it's lost permanently.
+		const bufferTs = new Date(Date.now() - CONSISTENCY_BUFFER_MS).toISOString()
+		const last = records[records.length - 1]
+
+		let nextCursorTs: string
+		let nextCursorId: string
+		if (done) {
+			// Caught up — cap the cursor at the buffer boundary
+			nextCursorTs =
+				last && last.updated_at < bufferTs
+					? last.updated_at
+					: state.cursorTimestamp < bufferTs
+						? bufferTs
+						: state.cursorTimestamp
+			nextCursorId = last?.id ?? state.cursorId
+		} else {
+			// More pages — advance cursor to last record on this page
+			nextCursorTs = last.updated_at
+			nextCursorId = last.id
+		}
+
+		return {
+			changes: records.map(toUpsert),
+			hasMore: !done,
+			nextState: {
+				cursorTimestamp: nextCursorTs,
+				cursorId: nextCursorId,
+			},
+		}
+	},
+})

package/templates/worker/.agents/skills/sync-guide/examples/incremental-events.ts

@@ -0,0 +1,132 @@
+/**
+ * Delta sync — event-feed pattern (Stripe-style).
+ *
+ * This is the delta half of a backfill+delta pair. It reads from an events/
+ * changelog endpoint to detect changes incrementally. A separate replace-mode
+ * backfill sync (not shown here) handles the full dataset load.
+ *
+ * The event feed provides a reliable change stream: each event has a unique ID
+ * that serves as a cursor. Events can produce both upserts and deletes.
+ *
+ * Key points:
+ * - State is just { eventCursor: string } — no phase discrimination
+ * - First run: fetch the latest event ID as the starting cursor
+ * - Consistency buffer: skip events younger than 10 seconds (they may not be
+ *   fully consistent yet, and since the cursor never resets, skipping = permanent loss)
+ * - Events can map to both upserts and deletes
+ *
+ * For the backfill half, use a replace-mode sync targeting the same database —
+ * see replace-paginated.ts for the pattern. Trigger it via CLI:
+ *   ntn workers sync state reset customersBackfill && ntn workers sync trigger customersBackfill
+ */
+
+import { Worker } from "@notionhq/workers"
+import * as Builder from "@notionhq/workers/builder"
+import * as Schema from "@notionhq/workers/schema"
+
+const worker = new Worker()
+export default worker
+
+// Database shared between backfill (replace) and this delta (incremental) sync
+const customers = worker.database("customers", {
+	type: "managed",
+	initialTitle: "Customers",
+	primaryKeyProperty: "Customer ID",
+	schema: {
+		properties: {
+			Name: Schema.title(),
+			"Customer ID": Schema.richText(),
+		},
+	},
+})
+
+// Rate-limit API calls — shared across all syncs hitting this API
+const apiPacer = worker.pacer("stripe", {
+	allowedRequests: 25,
+	intervalMs: 1000,
+})
+
+const CONSISTENCY_BUFFER_SECONDS = 10
+
+type DeltaState = { eventCursor: string }
+
+// Delta sync: reads the event feed for incremental changes
+worker.sync("customersDelta", {
+	database: customers,
+	mode: "incremental",
+	schedule: "5m",
+	execute: async (state: DeltaState | undefined) => {
+		// First run: capture the latest event ID as the starting cursor.
+		// We don't process any events on the first run — the backfill sync
+		// handles the full dataset. This just establishes "start here" for
+		// future delta cycles.
+		if (!state) {
+			await apiPacer.wait()
+			const anchorResponse = await apiCall("/v1/events?limit=1")
+			const eventCursor = anchorResponse.data[0]?.id ?? ""
+			return {
+				changes: [],
+				hasMore: false,
+				nextState: { eventCursor },
+			}
+		}
+
+		// Read events from the changelog endpoint.
+		// Events are returned in reverse-chronological order, so we read backwards
+		// from the latest event to our cursor position.
+		await apiPacer.wait()
+		const response = await apiCall(
+			`/v1/events?limit=100&ending_before=${state.eventCursor}`,
+		)
+		const events: Array<{
+			id: string
+			type: string
+			created: number
+			data: { object: { id: string; name: string } }
+		}> = response.data
+
+		// Consistency buffer: skip events younger than 10 seconds.
+		// The event stream may not be fully consistent for very recent events.
+		// Since the cursor never resets, advancing past an inconsistent event
+		// means we'd miss the final state of that record permanently.
+		const cutoff = Date.now() / 1000 - CONSISTENCY_BUFFER_SECONDS
+		const safeEvents = events.filter(e => e.created < cutoff)
+
+		// Map events to changes (upserts or deletes)
+		const changes = safeEvents.map(event => {
+			if (event.type.endsWith(".deleted")) {
+				return { type: "delete" as const, key: event.data.object.id }
+			}
+			return toUpsert(event.data.object)
+		})
+
+		// Only advance cursor if we have safe events to process.
+		// If all events are too recent, cursor stays put — we'll re-check next cycle.
+		const lastSafe = safeEvents[safeEvents.length - 1]
+		const nextCursor = lastSafe?.id ?? state.eventCursor
+
+		return {
+			changes,
+			hasMore: response.has_more && safeEvents.length > 0,
+			nextState: { eventCursor: nextCursor },
+		}
+	},
+})
+
+function toUpsert(customer: { id: string; name: string }) {
+	return {
+		type: "upsert" as const,
+		key: customer.id,
+		properties: {
+			Name: Builder.title(customer.name),
+			"Customer ID": Builder.richText(customer.id),
+		},
+	}
+}
+
+async function apiCall(path: string) {
+	const response = await fetch(`https://api.stripe.com${path}`, {
+		headers: { Authorization: `Bearer ${process.env.STRIPE_SECRET_KEY}` },
+	})
+	return response.json()
+}

package/templates/worker/.agents/skills/sync-guide/examples/replace-paginated.ts

@@ -0,0 +1,79 @@
+/**
+ * Replace mode — paginated API.
+ *
+ * Fetches all records page by page. Each cycle re-fetches everything.
+ * The runtime deletes any records not seen during the cycle.
+ *
+ * This pattern is also used for backfill syncs (with schedule: "manual"),
+ * where you want to re-import all data on demand rather than on a timer.
+ *
+ * Key points:
+ * - State is just within-cycle pagination: { page: number }
+ * - State effectively resets between cycles (each cycle starts from page 1)
+ * - hasMore: true while there are more pages to fetch
+ * - Batch size ~100 to avoid overloading a single execute call
+ * - A pacer rate-limits API calls to avoid hitting upstream rate limits
+ */
+
+import { Worker } from "@notionhq/workers"
+import * as Builder from "@notionhq/workers/builder"
+import * as Schema from "@notionhq/workers/schema"
+
+const worker = new Worker()
+export default worker
+
+const productsDb = worker.database("productsDb", {
+	type: "managed",
+	initialTitle: "Products",
+	primaryKeyProperty: "Product ID",
+	schema: {
+		properties: {
+			Name: Schema.title(),
+			"Product ID": Schema.richText(),
+		},
+	},
+})
+
+// Rate-limit API calls: 10 requests per second
+const apiPacer = worker.pacer("exampleApi", {
+	allowedRequests: 10,
+	intervalMs: 1000,
+})
+
+// State is simple — just track which page we're on within this cycle
+type PaginationState = { page: number }
+
+worker.sync("productsSync", {
+	database: productsDb,
+	mode: "replace",
+	execute: async (state: PaginationState | undefined) => {
+		const page = state?.page ?? 1
+		const pageSize = 100
+
+		// Wait for the pacer before each API call
+		await apiPacer.wait()
+
+		// Fetch one page from the API
+		const response = await fetch(
+			`https://api.example.com/products?page=${page}&limit=${pageSize}`,
+			{ headers: { Authorization: `Bearer ${process.env.API_TOKEN}` } },
+		)
+		const data = await response.json()
+
+		const hasMore = data.products.length === pageSize
+
+		return {
+			changes: data.products.map((product: { id: string; name: string }) => ({
+				type: "upsert" as const,
+				key: product.id,
+				properties: {
+					Name: Builder.title(product.name),
+					"Product ID": Builder.richText(product.id),
+				},
+			})),
+			hasMore,
+			// Next page, or undefined if done (cycle complete)
+			nextState: hasMore ? { page: page + 1 } : undefined,
+		}
+	},
+})

package/templates/worker/.agents/skills/sync-guide/examples/replace-simple.ts

@@ -0,0 +1,57 @@
+/**
+ * Replace mode — static data, no pagination.
+ *
+ * The simplest possible sync. Returns hardcoded data with no API calls.
+ * Good for: testing the sync setup, syncing small static datasets.
+ *
+ * Key points:
+ * - mode: "replace" means the runtime deletes any records not returned each cycle
+ * - No state needed — there's no pagination and no cursor
+ * - hasMore: false on every call — single-page cycle
+ * - Database is declared separately via worker.database() and referenced by handle
+ */
+
+import { Worker } from "@notionhq/workers"
+import * as Builder from "@notionhq/workers/builder"
+import * as Schema from "@notionhq/workers/schema"
+
+const worker = new Worker()
+export default worker
+
+const teamDb = worker.database("teamDb", {
+	type: "managed",
+	initialTitle: "Team Members",
+	primaryKeyProperty: "Member ID",
+	schema: {
+		properties: {
+			Name: Schema.title(),
+			"Member ID": Schema.richText(),
+		},
+	},
+})
+
+worker.sync("teamSync", {
+	database: teamDb,
+	mode: "replace",
+	execute: async () => {
+		// In a real sync, you'd fetch this from an API
+		const members = [
+			{ id: "m-1", name: "Alice" },
+			{ id: "m-2", name: "Bob" },
+			{ id: "m-3", name: "Charlie" },
+		]
+
+		return {
+			changes: members.map(m => ({
+				type: "upsert" as const,
+				key: m.id,
+				properties: {
+					Name: Builder.title(m.name),
+					"Member ID": Builder.richText(m.id),
+				},
+			})),
+			// No more pages — cycle is complete
+			hasMore: false,
+		}
+	},
+})

package/templates/worker/.agents/skills/sync-validate/SKILL.md

@@ -0,0 +1,60 @@
+---
+name: sync-validate
+description: Review a sync capability for common bugs — cursor advancement, pagination termination, state persistence, bi-modal correctness, consistency buffers, and deletion handling
+user-invocable: true
+disable-model-invocation: true
+allowed-tools: ["Read", "Glob", "Grep", "Bash"]
+---
+
+## Instructions
+
+Read the sync capabilities in `src/index.ts` (and any imported modules). For each sync found, run through the checklist below. Report findings grouped by severity.
+
+Before starting, read `.agents/skills/sync-guide/SKILL.md` for the full sync concepts reference.
+
+### Critical Issues (will break the sync or cause data loss)
+
+1. **No pagination termination**: Does `hasMore` eventually become `false`? Look for: infinite loops where `nextState` doesn't advance, missing base cases, conditions that can never be met.
+
+2. **Cursor doesn't advance**: Does `nextState` change between iterations? If the cursor is the same as the previous state, the sync will loop forever. Check that each execute call makes progress.
+
+3. **Missing first-run handling**: When `state` is `undefined` (first run), does the code handle it gracefully? Look for: `state.cursor` without `state?.cursor`, property access on potentially undefined state.
+
+4. **Batch too large**: Is the sync returning thousands of changes in one execution? Recommend batches of ~100. Large batches will fail.
+
+5. **Replace mode when API supports change tracking**: If mode is `replace` (or unset — it defaults to `replace`), does the source API support `updated_at` filters, event feeds, or similar change tracking? If so, recommend switching to `incremental` to avoid re-fetching everything each cycle.
+
+6. **State persistence misunderstanding**: In incremental mode, the cursor never resets between cycles. The next cycle starts exactly where the last one left off. Check for code that assumes a fresh start each cycle — this will cause records to be re-fetched or skipped permanently.
+
+### Structural Issues (bi-modal correctness)
+
+7. **Single-mode cursor for incremental sync**: Is the sync using the same cursor strategy for both backfill and delta? Unless the API sorts by `updated_at` and uses an opaque cursor (where one cursor naturally serves both), the sync should have a discriminated state union with separate backfill and delta phases.
+
+8. **Missing backfill-to-delta transition**: For bi-modal syncs, how is the delta cursor seeded? It must come from a marker captured *before* the backfill started (event anchor, timestamp with overlap), NOT from the last record in the backfill. Otherwise, changes during the backfill window are lost permanently.
+
+9. **No overlap in transition**: When transitioning from backfill to delta, is there an overlap window (e.g., `backfillStartedAt - 5 minutes`)? Without overlap, records modified during the backfill but before the delta cursor starts are missed permanently.
+
+### Warnings (may cause subtle data quality bugs)
+
+10. **No consistency buffer**: For incremental syncs hitting eventually consistent APIs, the cursor should lag behind "now" by 10-60 seconds. Without this, the cursor can advance past records that haven't been indexed by the source API yet — those records are lost permanently since the cursor never resets.
+
+11. **Timestamp cursor without tie-breaking**: If using `updated_at` as a cursor, can multiple records share the same timestamp? If yes (batch imports, bulk updates, low-resolution timestamps), recommend the keyset pattern: `(timestamp, id)` with a query like `WHERE ts > X OR (ts = X AND id > Y)`.
+
+12. **Missing delete handling**: In incremental mode, are deletions handled? Check:
+    - Does the source API have a delete signal (audit log, archived filter, events)?
+    - If yes, is the sync emitting `{ type: "delete", key }` markers?
+    - If the delete signal is on a separate endpoint, is the flip-flop pattern used (alternate streams at cycle boundaries)?
+    - If no delete signal exists, should this be a replace-mode sync instead?
+
+13. **Hardcoded secrets**: Are API keys, tokens, or credentials in the code instead of `process.env`? Flag any string that looks like a secret.
+
+14. **Missing error handling on fetch**: Network calls without error handling will crash the sync on any transient failure. Consider whether the sync should catch and handle API errors or let them propagate (the runtime will retry the cycle).
+
+### Output Format
+
+For each issue found:
+- **What**: Name the issue and point to the specific code location
+- **Why it matters**: What will happen in production if this isn't fixed
+- **Fix**: Provide a concrete code snippet showing the fix
+
+If no issues are found, say so. Don't invent problems.

package/templates/worker/.claudeignore

@@ -0,0 +1,2 @@
+.env
+.env.*

package/templates/worker/.codexignore

@@ -0,0 +1,2 @@
+.env
+.env.*