@zeyolabs/kit 0.0.0-stage → 2026.11.6

package/monitoring/prometheus/client.ts

@@ -0,0 +1,147 @@
+import { create, toBinary } from '@bufbuild/protobuf';
+import * as snappy from 'snappyjs';
+
+// Direct push into Prometheus over a VPC-style binding (Workers VPC, no public host). No ambient
+// context, no app-specific knowledge — the binding + credentials are passed in.
+import { waitUntil } from '../../cloudflare/wait-until';
+import { WriteRequestSchema } from './gen/remote_write_pb';
+
+export type PrometheusSample = {
+	labels?: Record<string, string>;
+	name: string;
+	timestampMs?: number;
+	value: number;
+};
+
+/** Just the shape callers depend on — a test can hand-write a fake object matching this
+ *  (`{ push: vi.fn(), pushAndForget: vi.fn(), pushWithRetry: vi.fn() }`) instead of constructing
+ *  `PrometheusClient` with a fake `vpc.fetch`. */
+export interface IPrometheusClient {
+	push(...samples: PrometheusSample[]): Promise<void>;
+	pushAndForget(...samples: PrometheusSample[]): void;
+	pushWithRetry(attempts?: number, ...samples: PrometheusSample[]): Promise<void>;
+}
+
+export type PrometheusConfig = {
+	password: string;
+	user: string;
+	/** The resolved Workers VPC Service binding — the caller reads it off its own `EnvStore`
+	 *  (`env.vpc('PROMETHEUS_VPC')`), not a binding name for this client to look up itself. */
+	vpc: Fetcher;
+};
+
+/** The real implementation — pushes over Prometheus's remote-write protocol. Construct directly
+ *  via `new PrometheusClient(config)`. */
+export class PrometheusClient implements IPrometheusClient {
+	constructor(private readonly config: PrometheusConfig) {}
+
+	/** Zero samples is a real no-op (nothing to push). */
+	async push(...samples: PrometheusSample[]): Promise<void> {
+		if (samples.length === 0) return;
+		const { password, user, vpc } = this.config;
+
+		const now = BigInt(Date.now());
+		const writeRequest = create(WriteRequestSchema, {
+			timeseries: samples.map((s) => ({
+				labels: [
+					{ name: '__name__', value: s.name },
+					...Object.entries(s.labels ?? {}).map(([name, value]) => ({ name, value })),
+				],
+				samples: [
+					{ timestamp: s.timestampMs != null ? BigInt(s.timestampMs) : now, value: s.value },
+				],
+			})),
+		});
+
+		const res = await vpc.fetch('http://localhost/api/v1/write', {
+			body: snappy.compress(toBinary(WriteRequestSchema, writeRequest)),
+			headers: {
+				Authorization: `Basic ${btoa(`${user}:${password}`)}`,
+				'Content-Encoding': 'snappy',
+				'Content-Type': 'application/x-protobuf',
+				'X-Prometheus-Remote-Write-Version': '0.1.0',
+			},
+			method: 'POST',
+		});
+
+		if (!res.ok) {
+			throw new Error(
+				`PrometheusClient.push: Prometheus returned ${res.status}: ${await res.text()}`
+			);
+		}
+	}
+
+	/** Swallow-and-log — a Prometheus outage must never block the caller's own critical path.
+	 *  Extends its own lifetime past the caller's response where that's possible (the real
+	 *  Workers runtime), since without it the platform can kill this push mid-flight the moment
+	 *  the caller's own response has been sent. */
+	pushAndForget(...samples: PrometheusSample[]): void {
+		if (samples.length === 0) return;
+		const result = this.push(...samples).catch((error) => {
+			console.warn('metrics push failed', {
+				error: error instanceof Error ? error.message : String(error),
+			});
+		});
+		void waitUntil(result);
+	}
+
+	/** Retries with exponential backoff + jitter instead of swallowing — for a caller that needs
+	 *  the push to land. Still throws after exhausting `attempts`. */
+	async pushWithRetry(attempts = 3, ...samples: PrometheusSample[]): Promise<void> {
+		if (samples.length === 0) return;
+		for (let i = 0; ; i++) {
+			try {
+				return await this.push(...samples);
+			} catch (error) {
+				if (i >= attempts - 1) throw error;
+				const backoff = 2 ** i * 100 + Math.random() * 100;
+				await new Promise((resolve) => setTimeout(resolve, backoff));
+			}
+		}
+	}
+}
+
+// Metric-type-shaped `PrometheusSample` builders — thin naming-convention wrappers, not real
+// client-side accumulation (a Worker invocation is stateless; every call is one point-in-time
+// observation the caller already computed). `bucket`/`sum` apply the `_bucket`/`_sum` suffixes
+// Prometheus's histogram functions (`histogram_quantile`, `rate`) require to find the right
+// series — `le` is the caller's own bucket boundary, folded into labels as a string (`"+Inf"` is
+// a valid value for the top bucket). `count` does NOT auto-suffix `_total` — unlike `_bucket`/
+// `_sum`, that's a naming convention `rate()`/`increase()` don't require, so it stays opt-in.
+
+export function bucket(
+	name: string,
+	le: number | string,
+	value: number,
+	labels?: Record<string, string>,
+	timestampMs?: number
+): PrometheusSample {
+	return { labels: { ...labels, le: String(le) }, name: `${name}_bucket`, timestampMs, value };
+}
+
+export function count(
+	name: string,
+	value: number,
+	labels?: Record<string, string>,
+	timestampMs?: number
+): PrometheusSample {
+	return { labels, name, timestampMs, value };
+}
+
+export function gauge(
+	name: string,
+	value: number,
+	labels?: Record<string, string>,
+	timestampMs?: number
+): PrometheusSample {
+	return { labels, name, timestampMs, value };
+}
+
+export function sum(
+	name: string,
+	value: number,
+	labels?: Record<string, string>,
+	timestampMs?: number
+): PrometheusSample {
+	return { labels, name: `${name}_sum`, timestampMs, value };
+}

package/monitoring/prometheus/dev.ts

@@ -0,0 +1,23 @@
+import type { IPrometheusClient, PrometheusSample } from './client';
+
+/** Stands in for `PrometheusClient` where there's no `PROMETHEUS_VPC` binding to push to (local
+ *  dev) — logs instead of sending, so metrics are still visible without a real Prometheus. */
+export class DevPrometheusClient implements IPrometheusClient {
+	async push(...samples: PrometheusSample[]): Promise<void> {
+		this.log(samples);
+	}
+
+	pushAndForget(...samples: PrometheusSample[]): void {
+		this.log(samples);
+	}
+
+	async pushWithRetry(_attempts?: number, ...samples: PrometheusSample[]): Promise<void> {
+		this.log(samples);
+	}
+
+	private log(samples: PrometheusSample[]): void {
+		for (const s of samples) {
+			console.log(`[prometheus:dev] ${s.name} ${s.value}`, s.labels ?? {});
+		}
+	}
+}

package/monitoring/prometheus/gen/remote_write_pb.ts

@@ -0,0 +1,105 @@
+// Minimal Prometheus remote_write v1 wire schema — only the fields the workers actually send
+// (a plain gauge sample per series, no exemplars/histograms/metadata). Field numbers match upstream
+// prometheus/prometheus's prompb/{types,remote}.proto exactly, so the wire bytes are compatible; the
+// gogoproto annotations there are Go-codegen-only and don't affect the wire format, so this file
+// omits that dependency entirely.
+
+// @generated by protoc-gen-es v2.16.0 with parameter "target=ts"
+// @generated from file remote_write.proto (package zeyo.prometheus.remote_write.v1, syntax proto3)
+/* eslint-disable */
+
+import type { GenFile, GenMessage } from "@bufbuild/protobuf/codegenv2";
+import { fileDesc, messageDesc } from "@bufbuild/protobuf/codegenv2";
+import type { Message } from "@bufbuild/protobuf";
+
+/**
+ * Describes the file remote_write.proto.
+ */
+export const file_remote_write: GenFile = /*@__PURE__*/
+  fileDesc("ChJyZW1vdGVfd3JpdGUucHJvdG8SH3pleW8ucHJvbWV0aGV1cy5yZW1vdGVfd3JpdGUudjEiJAoFTGFiZWwSDAoEbmFtZRgBIAEoCRINCgV2YWx1ZRgCIAEoCSIqCgZTYW1wbGUSDQoFdmFsdWUYASABKAESEQoJdGltZXN0YW1wGAIgASgDIn4KClRpbWVTZXJpZXMSNgoGbGFiZWxzGAEgAygLMiYuemV5by5wcm9tZXRoZXVzLnJlbW90ZV93cml0ZS52MS5MYWJlbBI4CgdzYW1wbGVzGAIgAygLMicuemV5by5wcm9tZXRoZXVzLnJlbW90ZV93cml0ZS52MS5TYW1wbGUiTwoMV3JpdGVSZXF1ZXN0Ej8KCnRpbWVzZXJpZXMYASADKAsyKy56ZXlvLnByb21ldGhldXMucmVtb3RlX3dyaXRlLnYxLlRpbWVTZXJpZXNiBnByb3RvMw");
+
+/**
+ * @generated from message zeyo.prometheus.remote_write.v1.Label
+ */
+export type Label = Message<"zeyo.prometheus.remote_write.v1.Label"> & {
+  /**
+   * @generated from field: string name = 1;
+   */
+  name: string;
+
+  /**
+   * @generated from field: string value = 2;
+   */
+  value: string;
+};
+
+/**
+ * Describes the message zeyo.prometheus.remote_write.v1.Label.
+ * Use `create(LabelSchema)` to create a new message.
+ */
+export const LabelSchema: GenMessage<Label> = /*@__PURE__*/
+  messageDesc(file_remote_write, 0);
+
+/**
+ * @generated from message zeyo.prometheus.remote_write.v1.Sample
+ */
+export type Sample = Message<"zeyo.prometheus.remote_write.v1.Sample"> & {
+  /**
+   * @generated from field: double value = 1;
+   */
+  value: number;
+
+  /**
+   * ms since epoch
+   *
+   * @generated from field: int64 timestamp = 2;
+   */
+  timestamp: bigint;
+};
+
+/**
+ * Describes the message zeyo.prometheus.remote_write.v1.Sample.
+ * Use `create(SampleSchema)` to create a new message.
+ */
+export const SampleSchema: GenMessage<Sample> = /*@__PURE__*/
+  messageDesc(file_remote_write, 1);
+
+/**
+ * @generated from message zeyo.prometheus.remote_write.v1.TimeSeries
+ */
+export type TimeSeries = Message<"zeyo.prometheus.remote_write.v1.TimeSeries"> & {
+  /**
+   * @generated from field: repeated zeyo.prometheus.remote_write.v1.Label labels = 1;
+   */
+  labels: Label[];
+
+  /**
+   * @generated from field: repeated zeyo.prometheus.remote_write.v1.Sample samples = 2;
+   */
+  samples: Sample[];
+};
+
+/**
+ * Describes the message zeyo.prometheus.remote_write.v1.TimeSeries.
+ * Use `create(TimeSeriesSchema)` to create a new message.
+ */
+export const TimeSeriesSchema: GenMessage<TimeSeries> = /*@__PURE__*/
+  messageDesc(file_remote_write, 2);
+
+/**
+ * @generated from message zeyo.prometheus.remote_write.v1.WriteRequest
+ */
+export type WriteRequest = Message<"zeyo.prometheus.remote_write.v1.WriteRequest"> & {
+  /**
+   * @generated from field: repeated zeyo.prometheus.remote_write.v1.TimeSeries timeseries = 1;
+   */
+  timeseries: TimeSeries[];
+};
+
+/**
+ * Describes the message zeyo.prometheus.remote_write.v1.WriteRequest.
+ * Use `create(WriteRequestSchema)` to create a new message.
+ */
+export const WriteRequestSchema: GenMessage<WriteRequest> = /*@__PURE__*/
+  messageDesc(file_remote_write, 3);
+

package/monitoring/prometheus/metrics.ts

@@ -0,0 +1,180 @@
+import type { Handle } from '@sveltejs/kit/hooks';
+
+import type { WorkerMiddleware } from '../../cloudflare/workers';
+import type { EnvStore } from '../../utils/env';
+
+import { runWithContext } from '../../cloudflare/context';
+// Workers analog of rust-common's RequestMetrics/track_metrics — pushes rather than scrapes.
+import { waitUntil } from '../../cloudflare/wait-until';
+import { envStore } from '../../utils/env';
+import { count, gauge, type IPrometheusClient, PrometheusClient } from './client';
+
+/** Does nothing — the fallback `client()` returns when `PROMETHEUS_*` isn't configured in this
+ *  env, so a caller expecting "swallow-and-log" fire-and-forget metrics (see `pushAndForget`'s own
+ *  doc comment) doesn't get an unhandled construction throw instead. */
+const NOOP_CLIENT: IPrometheusClient = {
+	push: async () => {},
+	pushAndForget: () => {},
+	pushWithRetry: async () => {},
+};
+
+let instance: IPrometheusClient | null = null;
+
+/** Isolate-level singleton, not per-request — the binding is isolate-stable. Private to this
+ *  file: the generic middleware below has no `env` param to thread through, so it needs its own
+ *  ambient access; a caller with an `env` in hand should construct `PrometheusClient` directly.
+ *
+ *  Construction itself (reading `PROMETHEUS_*` secrets/bindings) can throw when this env has none
+ *  configured — local dev, most notably. Every caller here treats metrics as fire-and-forget, so
+ *  that must degrade to a no-op instead of throwing: an unhandled construction error surfaces as
+ *  an unhandled promise rejection in `pushRequestCount`'s async body, which crashes the whole
+ *  process outside the real Workers runtime (Node kills on unhandled rejection by default). */
+function client(): IPrometheusClient {
+	if (instance) return instance;
+	try {
+		const env = envStore();
+		return (instance = new PrometheusClient({
+			password: env.secret('PROMETHEUS_PASSWORD'),
+			user: env.string('PROMETHEUS_USER'),
+			vpc: env.vpc('PROMETHEUS_VPC'),
+		}));
+	} catch (error) {
+		console.warn('Prometheus metrics disabled — PROMETHEUS_* not configured in this env', {
+			error: error instanceof Error ? error.message : String(error),
+		});
+		return (instance = NOOP_CLIENT);
+	}
+}
+
+/** `colo` splits this metric per Cloudflare colo, so concurrent pushes from different colos don't race Prometheus's per-series ordering and get rejected as "out of order". */
+async function pushRequestCount(
+	service: string,
+	method: string,
+	route: string,
+	status: number,
+	colo?: string
+): Promise<void> {
+	client().pushAndForget(
+		count('platform_http_requests_total', 1, {
+			method,
+			route,
+			service,
+			status: String(status),
+			...(colo ? { colo } : {}),
+		})
+	);
+}
+
+export const Metrics = {
+	/** Add to `sequence(...)` right after `contextHandle`. Falls back to fire-and-forget on apps
+	 *  with no `Platform.ctx` (marketing, ui). */
+	trackHTTP(service: string): Handle {
+		return async ({ event, resolve }) => {
+			const response = await resolve(event);
+			const push = pushRequestCount(
+				service,
+				event.request.method,
+				event.route.id ?? 'unmatched',
+				response.status,
+				(event.request as Request & { cf?: { colo?: string } }).cf?.colo
+			);
+			void waitUntil(push);
+			return response;
+		};
+	},
+
+	/** One call wires whichever surfaces a router has: HTTP request counts, `.queue(...)`
+	 *  success/failure, and `.cron(...)` success/failure + last-run — `WorkerRouter.use()`/
+	 *  `WorkerBindingRouter.use()` each apply only the fields that surface actually has.
+	 */
+	WorkerMiddleware<Env>(service: string): WorkerMiddleware<Env> {
+		return {
+			cron: async (cron, _env, _executionCtx, next) => {
+				try {
+					await next();
+					client().pushAndForget(
+						count('platform_cron_runs_total', 1, { cron, outcome: 'ok', service })
+					);
+					client().pushAndForget(
+						gauge('platform_cron_last_run_timestamp_ms', Date.now(), { cron, service })
+					);
+				} catch (e) {
+					client().pushAndForget(
+						count('platform_cron_runs_total', 1, { cron, outcome: 'error', service })
+					);
+					throw e;
+				}
+			},
+			http: async (c, next) => {
+				const response = await next();
+				const colo = (c.req.cf as undefined | { colo?: string })?.colo;
+				c.executionCtx.waitUntil(
+					pushRequestCount(service, c.req.method, c.route, response.status, colo)
+				);
+				return response;
+			},
+			queue: async (queueName, _env, _executionCtx, next) => {
+				try {
+					await next();
+					client().pushAndForget(
+						count('platform_queue_messages_total', 1, { outcome: 'ok', queue: queueName, service })
+					);
+				} catch (e) {
+					client().pushAndForget(
+						count('platform_queue_messages_total', 1, {
+							outcome: 'error',
+							queue: queueName,
+							service,
+						})
+					);
+					throw e;
+				}
+			},
+		};
+	},
+
+	/** For a `TickingDurableObject.onTick()` implementation (`.../workers/ticking-do`) that wants
+	 *  to push `platform_do_tick_runs_total`/`platform_do_tick_last_run_timestamp_ms`. */
+	TickingDO(service: string, env: EnvStore): { onTick(outcome: 'error' | 'ok'): void } {
+		const client = new PrometheusClient({
+			password: env.secret('PROMETHEUS_PASSWORD'),
+			user: env.string('PROMETHEUS_USER'),
+			vpc: env.vpc('PROMETHEUS_VPC'),
+		});
+		return {
+			onTick(outcome) {
+				client.pushAndForget(count('platform_do_tick_runs_total', 1, { outcome, service }));
+				if (outcome === 'ok') {
+					client.pushAndForget(
+						gauge('platform_do_tick_last_run_timestamp_ms', Date.now(), { service })
+					);
+				}
+			},
+		};
+	},
+
+	/** Plain-Worker `fetch` wrapper. `classifyRoute` collapses the path onto a fixed route set (no
+	 *  router here to supply one, and an unbounded label would blow up cardinality). `secretNames`:
+	 *  the app's secret-store binding names. */
+	withHTTP<Env extends object>(
+		service: string,
+		classifyRoute: (path: string) => string,
+		secretNames: ReadonlySet<string>,
+		handler: (request: Request, env: Env, ctx: ExecutionContext) => Promise<Response>
+	): (request: Request, env: Env, ctx: ExecutionContext) => Promise<Response> {
+		return (request, env, ctx) =>
+			runWithContext(env, secretNames, async () => {
+				const response = await handler(request, env, ctx);
+				ctx.waitUntil(
+					pushRequestCount(
+						service,
+						request.method,
+						classifyRoute(new URL(request.url).pathname),
+						response.status,
+						(request.cf as undefined | { colo?: string })?.colo
+					)
+				);
+				return response;
+			});
+	},
+};

package/monitoring/prometheus/testing.ts

@@ -0,0 +1,19 @@
+import type { IPrometheusClient, PrometheusSample } from './client';
+
+/** Records every sample passed to `push`/`pushAndForget`/`pushWithRetry` instead of sending
+ *  anything — for asserting what a service pushed, without a mock `vpc.fetch`. */
+export class FakePrometheusClient implements IPrometheusClient {
+	readonly pushed: PrometheusSample[] = [];
+
+	async push(...samples: PrometheusSample[]): Promise<void> {
+		this.pushed.push(...samples);
+	}
+
+	pushAndForget(...samples: PrometheusSample[]): void {
+		this.pushed.push(...samples);
+	}
+
+	async pushWithRetry(_attempts?: number, ...samples: PrometheusSample[]): Promise<void> {
+		this.pushed.push(...samples);
+	}
+}

package/monitoring/slack/api.ts

@@ -0,0 +1,68 @@
+// Generic Slack Web API + Events API primitives — no app domain knowledge. `SlackClient`
+// (./client.ts) is the separate Incoming-Webhook poster; this file is the bot-token
+// (`chat.postMessage`) side plus request-signature verification.
+
+/** Post a message (or thread reply) to a Slack channel via the bot token. Returns the posted
+ *  message's `ts`. */
+export async function slackPostMessage(
+	token: string,
+	params: {
+		blocks?: Record<string, unknown>[];
+		channel: string;
+		icon_url?: string;
+		text: string;
+		thread_ts?: string;
+		username?: string;
+	}
+): Promise<{ channel: string; ts: string }> {
+	const res = await fetch('https://slack.com/api/chat.postMessage', {
+		body: JSON.stringify(params),
+		headers: {
+			Authorization: `Bearer ${token}`,
+			'Content-Type': 'application/json',
+		},
+		method: 'POST',
+	});
+	const data = (await res.json()) as { channel: string; error?: string; ok: boolean; ts: string };
+	if (!data.ok) throw new Error(`Slack postMessage: ${data.error ?? 'unknown error'}`);
+	return { channel: data.channel, ts: data.ts };
+}
+
+/**
+ * Verify a Slack request signature (v0 scheme, HMAC-SHA256). Returns false when the timestamp
+ * is stale (> 5 minutes) or the computed signature does not match.
+ */
+export async function verifySlackSignature(
+	signingSecret: string,
+	timestamp: string,
+	rawBody: string,
+	signature: string
+): Promise<boolean> {
+	// Guard against replay attacks.
+	if (Math.abs(Date.now() / 1000 - Number(timestamp)) > 300) return false;
+
+	const key = await crypto.subtle.importKey(
+		'raw',
+		new TextEncoder().encode(signingSecret),
+		{ hash: 'SHA-256', name: 'HMAC' },
+		false,
+		['sign']
+	);
+	const base = `v0:${timestamp}:${rawBody}`;
+	const buf = await crypto.subtle.sign('HMAC', key, new TextEncoder().encode(base));
+	const hex = Array.from(new Uint8Array(buf))
+		.map((b) => b.toString(16).padStart(2, '0'))
+		.join('');
+	return timingSafeEqual(signature, `v0=${hex}`);
+}
+
+// Web Crypto has no `timingSafeEqual` — compare every byte instead of short-circuiting on the
+// first mismatch, so response timing can't leak how many signature bytes an attacker got right.
+function timingSafeEqual(a: string, b: string): boolean {
+	if (a.length !== b.length) return false;
+	let diff = 0;
+	for (let i = 0; i < a.length; i++) {
+		diff |= a.charCodeAt(i) ^ b.charCodeAt(i);
+	}
+	return diff === 0;
+}

package/monitoring/slack/client.ts

@@ -0,0 +1,33 @@
+import type { EnvStore } from '../../utils/env';
+
+import { logger } from '../logger/client';
+
+/**
+ * Posts a message to a Slack channel via an incoming webhook. Never throws on a failed post —
+ * losing an alert must not fail the caller's actual work — but a webhook request failure (network,
+ * non-2xx) is still logged.
+ */
+export class SlackClient {
+	constructor(private readonly webhookUrl: string) {}
+
+	/** `blocks` is optional, for a richer message than plain `text`. */
+	async post(payload: { blocks?: Record<string, unknown>[]; text: string }): Promise<boolean> {
+		try {
+			const res = await fetch(this.webhookUrl, {
+				body: JSON.stringify(payload),
+				headers: { 'content-type': 'application/json' },
+				method: 'POST',
+			});
+			if (!res.ok) throw new Error(`Slack webhook responded ${res.status}`);
+			return true;
+		} catch (err) {
+			await logger.error(err, { alert: payload.text });
+			return false;
+		}
+	}
+}
+
+/** Built from `SLACK_OPS_WEBHOOK_URL` off the ambient env store. */
+export function slackClient(env: EnvStore): SlackClient {
+	return new SlackClient(env.string('SLACK_OPS_WEBHOOK_URL'));
+}

package/monitoring/slack/events.ts

@@ -0,0 +1,15 @@
+// Slack Events API envelope shapes — generic wire format, no app domain knowledge.
+export type SlackMessageEvent = {
+	bot_id?: string;
+	channel: string;
+	subtype?: string;
+	text: string;
+	thread_ts?: string;
+	ts: string;
+	type: 'message';
+	user?: string;
+};
+
+export type SlackPayload =
+	| { challenge: string; type: 'url_verification' }
+	| { event: SlackMessageEvent; event_id: string; type: 'event_callback' };