@haathie/pgmb 0.2.6 → 0.2.7

package/src/retry-handler.ts

@@ -0,0 +1,125 @@
+import { RETRY_EVENT } from './consts.ts'
+import type { IReadNextEventsResult } from './queries.ts'
+import { findEvents, scheduleEventRetry } from './queries.ts'
+import type { PgClientLike } from './query-types.ts'
+import type { IEvent, IEventData, IEventHandler, IFindEventsFn, IReadEvent, IRetryEventPayload, IRetryHandlerOpts } from './types.ts'
+
+const defaultFindEvents = findEvents.run.bind(findEvents)
+
+export function createRetryHandler<T extends IEventData>(
+	{ retriesS }: IRetryHandlerOpts,
+	handler: IEventHandler<T>,
+): IEventHandler<T> {
+	return async(ev, ctx) => {
+		const { name, client, subscriptionId, logger } = ctx
+
+		try {
+			await handler(ev,	ctx)
+		} catch(err) {
+			const retryNumber = (ev.retry?.retryNumber ?? 0)
+			const nextRetryGapS = retriesS[retryNumber]
+			logger.error({ err, nextRetryGapS }, 'error in event handler')
+
+			if(!nextRetryGapS) {
+				return
+			}
+
+			await scheduleEventRetry.run(
+				{
+					subscriptionId,
+					ids: ev.items.map(i => i.id),
+					retryNumber: retryNumber + 1,
+					delayInterval: `${nextRetryGapS} seconds`,
+					handlerName: name,
+				},
+				client
+			)
+		}
+	}
+}
+
+export async function normaliseRetryEventsInReadEventMap<T extends IEventData>(
+	rows: IReadNextEventsResult[],
+	client: PgClientLike,
+	findEvents: IFindEventsFn = defaultFindEvents,
+) {
+	const map: { [sid: string]: IReadEvent<T>[] } = {}
+	const evsToPopulate: IReadEvent<T>[] = []
+	const idsToLoad: string[] = []
+
+	// reverse the map, do subscriptionId -> events
+	const subToEventMap: { [sid: string]: IReadNextEventsResult[] } = {}
+	for(const row of rows) {
+		for(const subId of row.subscriptionIds) {
+			subToEventMap[subId] ||= []
+			subToEventMap[subId].push(row)
+		}
+	}
+
+	const subEventList = Object.entries(subToEventMap)
+	for(const [subscriptionId, items] of subEventList) {
+		for(let i = 0;i < items.length;i) {
+			const item = items[i]
+			if(item.topic !== RETRY_EVENT) {
+				i++
+				continue
+			}
+
+			const retry = item.payload as IRetryEventPayload
+			if(!retry.ids?.length) {
+				continue
+			}
+
+			idsToLoad.push(...retry.ids)
+
+			map[subscriptionId] ||= []
+
+			const ev: IReadEvent<T> = { items: [], retry }
+			map[subscriptionId].push(ev)
+			evsToPopulate.push(ev)
+			items.splice(i, 1)
+		}
+
+		if(!items.length) {
+			continue
+		}
+
+		map[subscriptionId] ||= []
+		map[subscriptionId].push({ items: items as unknown as IEvent<T>[] })
+	}
+
+	if(!idsToLoad.length) {
+		return { map, retryEvents: 0, retryItemCount: 0 }
+	}
+
+	const fetchedEvents = await findEvents({ ids: idsToLoad }, client)
+	const fetchedEventMap = fetchedEvents.reduce(
+		(map, ev) => {
+			map[ev.id] = ev as IEvent<T>
+			return map
+		},
+		{} as { [id: string]: IEvent<T> }
+	)
+
+	// populate the events
+	for(const { items, retry } of evsToPopulate) {
+		if(!retry) {
+			continue
+		}
+
+		for(const id of retry.ids) {
+			const ev = fetchedEventMap[id]
+			if(!ev) {
+				continue
+			}
+
+			items.push(ev)
+		}
+	}
+
+	return {
+		map,
+		retryEvents: evsToPopulate.length,
+		retryItemCount: idsToLoad.length,
+	}
+}

package/src/sse.ts

@@ -0,0 +1,148 @@
+import assert, { AssertionError } from 'node:assert'
+import type { IncomingMessage, ServerResponse } from 'node:http'
+import type { PgmbClient } from './client.ts'
+import type { IReplayEventsResult } from './queries.ts'
+import { replayEvents } from './queries.ts'
+import type { IEphemeralListener, IEvent, IEventData, SSERequestHandlerOpts } from './types.ts'
+import { getCreateDateFromSubscriptionId, getDateFromMessageId } from './utils.ts'
+
+export function createSSERequestHandler<T extends IEventData>(
+	this: PgmbClient<T>,
+	{
+		getSubscriptionOpts,
+		maxReplayEvents = 1000,
+		maxReplayIntervalMs = 5 * 60 * 1000,
+		jsonifier = JSON
+	}: SSERequestHandlerOpts,
+) {
+	const replayEnabled = maxReplayEvents > 0
+	return handleSSERequest.bind(this)
+
+	async function handleSSERequest(
+		this: PgmbClient<T>,
+		req: IncomingMessage,
+		res: ServerResponse
+	) {
+		let sub: IEphemeralListener<T> | undefined
+		let eventsToReplay: IReplayEventsResult[] = []
+
+		try {
+			assert(
+				req.method?.toLowerCase() === 'get',
+				'SSE only supports GET requests'
+			)
+			// validate last-event-id header
+			const fromEventId = req.headers['last-event-id']
+			if(fromEventId) {
+				assert(replayEnabled, 'replay disabled on server')
+				assert(typeof fromEventId === 'string', 'invalid last-event-id header')
+				const fromDt = getDateFromMessageId(fromEventId)
+				assert(fromDt, 'invalid last-event-id header value')
+				assert(
+					fromDt.getTime() >= (Date.now() - maxReplayIntervalMs),
+					'last-event-id is too old to replay'
+				)
+			}
+
+			sub = await this.registerFireAndForgetHandler({
+				...await getSubscriptionOpts(req),
+				expiryInterval: `${maxReplayIntervalMs * 2} milliseconds`
+			})
+
+			if(fromEventId) {
+				const fromDt = getDateFromMessageId(fromEventId)!
+				const subDt = getCreateDateFromSubscriptionId(sub.id)
+				assert(subDt, 'internal: invalid subscription id format')
+				assert(
+					fromDt >= subDt,
+					'last-event-id is before subscription creation, cannot replay'
+				)
+
+				eventsToReplay = await replayEvents.run(
+					{
+						groupId: this.groupId,
+						subscriptionId: sub.id,
+						fromEventId: fromEventId,
+						maxEvents: maxReplayEvents
+					},
+					this.client
+				)
+
+				this.logger.trace(
+					{ subId: sub.id, count: eventsToReplay.length },
+					'got events to replay'
+				)
+			}
+
+			if(res.writableEnded) {
+				throw new Error('response already ended')
+			}
+		} catch(err) {
+			this.logger
+				.error({ subId: sub?.id, err }, 'error in sse subscription setup')
+
+			await sub?.throw(err).catch(() => { })
+
+			if(res.writableEnded) {
+				return
+			}
+
+			const message = err instanceof Error ? err.message : String(err)
+			// if an assertion failed, we cannot connect with these parameters
+			// so use 204 No Content
+			const code = err instanceof AssertionError ? 204 : 500
+			res
+				.writeHead(code, message)
+				.end()
+			return
+		}
+
+		res.once('close', () => {
+			sub?.return()
+		})
+		res.once('error', err => {
+			sub?.throw(err).catch(() => {})
+		})
+
+		res.writeHead(200, {
+			'content-type': 'text/event-stream',
+			'cache-control': 'no-cache',
+			'connection': 'keep-alive',
+			'transfer-encoding': 'chunked',
+		})
+		res.flushHeaders()
+
+		try {
+			// send replayed events first
+			writeSseEvents(res, eventsToReplay as IEvent<T>[])
+
+			for await (const { items } of sub) {
+				writeSseEvents(res, items)
+			}
+		} catch(err) {
+			this.logger.error({ err }, 'error in sse subscription')
+			if(res.writableEnded) {
+				return
+			}
+
+			// send error event
+			const message = err instanceof Error ? err.message : String(err)
+			const errData	= jsonifier.stringify({ message })
+			res.write(`event: error\ndata: ${errData}\nretry: 250\n\n`)
+			res.end()
+		}
+	}
+
+	function writeSseEvents(res: ServerResponse, items: IEvent<T>[]) {
+		for(const { id, payload, topic } of items) {
+			const data = jsonifier.stringify(payload)
+			if(!replayEnabled) {
+				// if replay is disabled, do not send an id field
+				res.write(`event: ${topic}\ndata: ${data}\n\n`)
+				continue
+			}
+
+			res.write(`id: ${id}\nevent: ${topic}\ndata: ${data}\n\n`)
+		}
+	}
+}

package/src/types.ts

@@ -0,0 +1,267 @@
+import type { IDatabaseConnection } from '@pgtyped/runtime'
+import type { IncomingMessage } from 'node:http'
+import type { Logger } from 'pino'
+import type { HeaderRecord } from 'undici-types/header.js'
+import type { AbortableAsyncIterator } from './abortable-async-iterator.ts'
+import type { IAssertSubscriptionParams, IFindEventsParams, IFindEventsResult, IReadNextEventsParams, IReadNextEventsResult } from './queries.ts'
+import type { PgClientLike } from './query-types.ts'
+
+export type ISplitFn<T extends IEventData>
+	= (event: IReadEvent<T>) => IReadEvent<T>[]
+
+export type SerialisedEvent = {
+	body: Buffer | string
+	contentType: string
+}
+
+export type WebhookInfo = {
+	id: string
+	url: string | URL
+}
+
+export type GetWebhookInfoFn = (
+	subscriptionIds: string[]
+) => Promise<{ [id: string]: WebhookInfo[] }> | { [id: string]: WebhookInfo[] }
+
+export type PgmbWebhookOpts<T extends IEventData> = {
+	/**
+	 * Maximum time to wait for webhook request to complete
+	 * @default 5 seconds
+	 */
+	timeoutMs?: number
+	headers?: HeaderRecord
+	/**
+	 * Configure retry intervals in seconds for failed webhook requests.
+	 * If null, a failed handler will fail the event processor. Use carefully.
+	 */
+	retryOpts?: IRetryHandlerOpts | null
+	splitBy?: ISplitFn<T>
+	jsonifier?: JSONifier
+	serialiseEvent?(ev: IReadEvent, logger: Logger): SerialisedEvent
+}
+
+export interface IEventData {
+	topic: string
+	payload: unknown
+	metadata?: unknown
+}
+
+export type IEvent<T extends IEventData> = (T & { id: string })
+
+export type PGMBEventBatcherOpts<T extends IEventData> = {
+	/**
+	 * Whether a particular published message should be logged.
+	 * By default, all messages are logged -- in case of certain
+	 * failures, the logs can be used to replay the messages.
+	 */
+	shouldLog?(msg: T): boolean
+
+	publish(...msgs: T[]): Promise<{ id: string }[]>
+
+	logger?: Logger
+	/**
+	 * Automatically flush after this interval.
+	 * Set to undefined or 0 to disable. Will need to
+	 * manually call `flush()` to publish messages.
+	 * @default undefined
+	 */
+	flushIntervalMs?: number
+	/**
+	 * Max number of messages to send in a batch
+	 * @default 2500
+	 */
+	maxBatchSize?: number
+}
+
+export type IReadNextEventsFn = (parmas: IReadNextEventsParams, db: IDatabaseConnection)
+	=> Promise<IReadNextEventsResult[]>
+
+export type IFindEventsFn = (parmas: IFindEventsParams, db: IDatabaseConnection)
+	=> Promise<IFindEventsResult[]>
+
+export type Pgmb2ClientOpts<T extends IEventData> = {
+	client: PgClientLike
+	/**
+	 * Globally unique identifier for this Pgmb2Client instance. All subs
+	 * registered with this client will use this groupId.
+	 */
+	groupId: string
+	logger?: Logger
+	/**
+	 * How long to sleep between polling for new events from
+	 * the global events table.
+	 * Only one global call is required across all clients.
+	 * Set to 0 to disable polling.
+	 *
+	 * @default 1 second
+	 * */
+	pollEventsIntervalMs?: number
+	/**
+	 * Group level configuration for how often to read new events
+	 * relevant to the group's subscriptions.
+	 * @default 1 second
+	 */
+	readEventsIntervalMs?: number
+	/**
+	 * How often to mark subscriptions as active,
+	 * and remove expired ones.
+	 * @default 1 minute
+	 */
+	subscriptionMaintenanceMs?: number
+	/**
+	 * How often to maintain the events tables
+	 * (drop old partitions, create new ones, etc)
+	 * Set to 0 to disable automatic maintenance.
+	 *
+	 * @default 5 minutes
+	 */
+	tableMaintainanceMs?: number
+
+	readChunkSize?: number
+	/**
+	 * As we process in batches, a single handler taking time to finish
+	 * can lead to buildup of unprocessed checkpoints. To avoid this,
+	 * we keep moving forward while handlers run in the background, but
+	 * to avoid an unbounded number of items being backlogged, we limit
+	 * how much further we can go ahead from the earliest uncompleted checkpoint.
+	 * @default 10
+	 */
+	maxActiveCheckpoints?: number
+	webhookHandlerOpts?: Partial<PgmbWebhookOpts<T>>
+	getWebhookInfo?: GetWebhookInfoFn
+	/**
+	 * Override the default readNextEvents implementation
+	 */
+	readNextEvents?: IReadNextEventsFn
+	/**
+	 * Override the default findEvents implementation
+	 */
+	findEvents?: IFindEventsFn
+} & Pick<
+	PGMBEventBatcherOpts<IEventData>,
+	'flushIntervalMs' | 'maxBatchSize' | 'shouldLog'
+>
+
+export type IReadEvent<T extends IEventData = IEventData> = {
+	items: IEvent<T>[]
+	retry?: IRetryEventPayload
+}
+
+export type RegisterSubscriptionParams
+	= Omit<IAssertSubscriptionParams, 'groupId'>
+
+export type registerReliableHandlerParams<T extends IEventData = IEventData> = RegisterSubscriptionParams & {
+	/**
+	 * Name for the retry handler, used to ensure retries for a particular
+	 * handler are not mixed with another handler. This name need only be
+	 * unique for a particular subscription.
+	 */
+	name?: string
+	retryOpts?: IRetryHandlerOpts
+	/**
+	 * If provided, will split an incoming event into multiple events
+	 * as determined by the function.
+	 */
+	splitBy?: ISplitFn<T>
+}
+
+export type CreateTopicalSubscriptionOpts<T extends IEventData> = {
+	/**
+	 * The topics to subscribe to.
+	 */
+	topics: T['topic'][]
+	/**
+	 * To scale out processing, you can partition the subscriptions.
+	 * For example, with `current: 0, total: 3`, only messages
+	 * where `hashtext(e.id) % 3 == 0` will be received by this subscription.
+	 * This will result in an approximate even split for all processors, the only
+	 * caveat being it requires knowing the number of event processors on this
+	 * subscription beforehand.
+	 */
+	partition?: {
+		current: number
+		total: number
+	}
+	/**
+	 * Add any additional params to filter by.
+	 * i.e "s.params @> jsonb_build_object(...additionalFilters)"
+	 * The value should be a valid SQL snippet.
+	 */
+	additionalFilters?: Record<string, string>
+	/** JSON to populate params */
+	additionalParams?: Record<string, any>
+
+	expiryInterval?: RegisterSubscriptionParams['expiryInterval']
+}
+
+export interface IEphemeralListener<T extends IEventData>
+	extends AbortableAsyncIterator<IReadEvent<T>> {
+	id: string
+}
+
+export type IEventHandlerContext = {
+	logger: Logger
+	client: PgClientLike
+	subscriptionId: string
+	/** registered name of the handler */
+	name: string
+	extra?: unknown
+}
+
+export type IEventHandler<T extends IEventData = IEventData>
+	= (item: IReadEvent<T>, ctx: IEventHandlerContext) => Promise<void>
+
+export type IRetryEventPayload = {
+	ids: string[]
+	handlerName: string
+	retryNumber: number
+}
+
+type SSESubscriptionOpts
+	= Pick<RegisterSubscriptionParams, 'conditionsSql' | 'params'>
+
+export type SSERequestHandlerOpts = {
+	getSubscriptionOpts(req: IncomingMessage):
+		Promise<SSESubscriptionOpts> | SSESubscriptionOpts
+	/**
+	 * Maximum interval to replay events for an SSE subscription.
+	 * @default 5 minutes
+	 */
+	maxReplayIntervalMs?: number
+	/**
+	 * Max number of events to replay for an SSE subscription.
+	 * Set to 0 to disable replaying events.
+	 * @default 1000
+	 */
+	maxReplayEvents?: number
+
+	jsonifier?: JSONifier
+}
+
+export type IRetryHandlerOpts = {
+	retriesS: number[]
+}
+
+export interface JSONifier {
+	stringify(data: unknown): string
+	parse(data: string): unknown
+}
+
+export type ITableMutationEventData<T, N extends string> = {
+	topic: `${N}.insert`
+	payload: T
+	metadata: {}
+} | {
+	topic: `${N}.delete`
+	payload: T
+	metadata: {}
+} | {
+	topic: `${N}.update`
+	/**
+	 * The fields that were updated in the row
+	 */
+	payload: Partial<T>
+	metadata: {
+		old: T
+	}
+}

package/src/utils.ts

@@ -0,0 +1,71 @@
+import assert from 'node:assert'
+import type { CreateTopicalSubscriptionOpts, IEventData, RegisterSubscriptionParams } from './types'
+
+/**
+ * Extract the date from a message ID, same as the PG function
+ */
+export function getDateFromMessageId(messageId: string) {
+	if(!messageId.startsWith('pm')) {
+		return undefined
+	}
+
+	const micros = parseInt(messageId.slice(2, 15), 16)
+	if(isNaN(micros)) {
+		return undefined
+	}
+
+	const date = new Date(micros / 1000)
+	return date
+}
+
+/**
+ * Extract the date from a subscription ID
+ */
+export function getCreateDateFromSubscriptionId(id: string) {
+	if(!id.startsWith('su')) {
+		return undefined
+	}
+
+	return getDateFromMessageId('pm' + id.slice(2))
+}
+
+/**
+ * Creates subscription params for a subscription that matches
+ * 1 or more topics. Also supports partitioning the subscription
+ * such that only a subset of messages are received.
+ */
+export function createTopicalSubscriptionParams<T extends IEventData>({
+	topics,
+	partition,
+	additionalFilters = {},
+	additionalParams = {},
+	...rest
+}: CreateTopicalSubscriptionOpts<T>): RegisterSubscriptionParams {
+	assert(topics.length > 0, 'At least one topic must be provided')
+
+	const filters = { ...additionalFilters }
+	filters['topics'] ||= 'ARRAY[e.topic]'
+	if(partition) {
+		filters['partition'] = `hashtext(e.id) % ${partition.total}`
+	}
+
+	const strs = Object.entries(filters)
+		.map(([k, v]) => `'${k}',${v}`)
+	return {
+		conditionsSql: `s.params @> jsonb_build_object(${strs.join(',')})`,
+		params: { topics, partition: partition?.current, ...additionalParams },
+		...rest
+	}
+}
+
+/**
+ * Get an environment variable as a number
+ */
+export function getEnvNumber(key: string, defaultValue = 0) {
+	const num = +(process.env[key] || defaultValue)
+	if(isNaN(num) || !isFinite(num)) {
+		return defaultValue
+	}
+
+	return num
+}

package/src/webhook-handler.ts

@@ -0,0 +1,91 @@
+import assert from 'node:assert'
+import { createHash } from 'node:crypto'
+import { createRetryHandler } from './retry-handler.ts'
+import type { IEventData, IEventHandler, IReadEvent, JSONifier, PgmbWebhookOpts, SerialisedEvent } from './types.ts'
+
+/**
+ * Create a handler that sends events to a webhook URL via HTTP POST.
+ * @param url Where to send the webhook requests
+ */
+export function createWebhookHandler<T extends IEventData>(
+	{
+		timeoutMs = 5_000,
+		headers,
+		retryOpts = {
+			// retry after 5 minutes, then after 30 minutes
+			retriesS: [5 * 60, 30 * 60]
+		},
+		jsonifier = JSON,
+		serialiseEvent = createSimpleSerialiser(jsonifier)
+	}: Partial<PgmbWebhookOpts<T>>
+) {
+	const handler: IEventHandler = async(ev, { logger, extra }) => {
+		assert(
+			typeof extra === 'object'
+			&& extra !== null
+			&& 'url' in extra
+			&& (
+				typeof extra.url === 'string'
+				|| extra.url instanceof URL
+			),
+			'webhook handler requires extra.url parameter'
+		)
+		const { url } = extra
+		const idempotencyKey = getIdempotencyKeyHeader(ev)
+		logger = logger.child({ idempotencyKey })
+
+		const { body, contentType } = serialiseEvent(ev, logger)
+
+		const { status, statusText, body: res } = await fetch(url, {
+			method: 'POST',
+			headers: {
+				'content-type': contentType,
+				'idempotency-key': idempotencyKey,
+				...headers
+			},
+			body,
+			redirect: 'manual',
+			signal: AbortSignal.timeout(timeoutMs)
+		})
+		// don't care about response body
+		await res?.cancel().catch(() => { })
+		// see: https://developer.mozilla.org/en-US/docs/Web/HTTP/Reference/Headers/Idempotency-Key
+		if(status === 422) { // unprocessable request, do not retry
+			logger.warn('webhook returned 422, dropping event')
+			return
+		}
+
+		if(status < 200 || status >= 300) {
+			throw new Error(`Non-2xx response: ${status} (${statusText})`)
+		}
+
+		logger.info({ status }, 'webhook sent successfully')
+	}
+
+	if(!retryOpts) {
+		return handler
+	}
+
+	return createRetryHandler(retryOpts, handler)
+}
+
+function getIdempotencyKeyHeader(ev: IReadEvent) {
+	const hasher = createHash('sha256')
+	for(const item of ev.items) {
+		hasher.update(item.id)
+	}
+
+	return hasher.digest('hex').slice(0, 16)
+}
+
+function createSimpleSerialiser(
+	jsonifier: JSONifier
+): ((ev: IReadEvent) => SerialisedEvent) {
+	return ev => ({
+		body: jsonifier.stringify({
+			items: ev.items
+				.map(({ id, payload, topic }) => ({ id, payload, topic }))
+		}),
+		contentType: 'application/json'
+	})
+}

package/lib/abortable-async-iterator.d.ts

@@ -1,14 +0,0 @@
-type AAResult<T> = IteratorResult<T>;
-export declare class AbortableAsyncIterator<T> implements AsyncIterableIterator<T> {
-    #private;
-    readonly signal: AbortSignal;
-    readonly onEnd: () => void;
-    ended: boolean;
-    constructor(signal: AbortSignal, onEnd?: () => void);
-    next(): Promise<AAResult<T>>;
-    enqueue(value: T): void;
-    throw(reason?: unknown): Promise<AAResult<T>>;
-    return(value?: any): Promise<AAResult<T>>;
-    [Symbol.asyncIterator](): this;
-}
-export {};