@xeonr/upload-pool-sdk 1.0.0 → 1.2.0

package/src/pool.ts

@@ -3,6 +3,11 @@
  * to the pool's job stream, dispatches job:dispatch events to the right
  * handler keyed by content type URN, and orchestrates accept/complete/
  * report-error against IntegrationQueueService.
+ *
+ * Every transition emits a structured log line via the injected Logger
+ * so a worker's k8s logs surface SSE connection state, dispatched jobs,
+ * handler outcomes, and RPC errors without callers needing to wire up
+ * their own onError hook.
  */
 import { hostname } from "node:os";
 import { randomBytes } from "node:crypto";
@@ -16,6 +21,7 @@ import { createRpcClients, type RpcClients } from "./rpc-clients.js";
 import { SseClient } from "./sse-client.js";
 import { createJobContext, type JobEnvelope } from "./job-context.js";
 import { NonRetryableError } from "./errors.js";
+import { JsonLogger, type Logger } from "./logger.js";
 import type {
 	JobContext,
 	JobHandler,
@@ -26,6 +32,7 @@ export class Pool {
 	private readonly config: PoolConfig;
 	private readonly rpc: RpcClients;
 	private readonly sse: SseClient;
+	private readonly logger: Logger;
 	private inFlight = 0;
 	private readonly workerId: string;
 	private readonly capabilities: string[];
@@ -33,46 +40,85 @@ export class Pool {
 	constructor(config: PoolConfig) {
 		this.config = {
 			concurrency: 1,
-			onError: defaultErrorHandler,
 			...config,
 		};
 		this.workerId = config.workerId ?? `${hostname()}-${randomBytes(4).toString("hex")}`;
 		this.capabilities = Object.keys(config.handlers);
-		this.rpc = createRpcClients(config.endpoint);
+		this.logger = (config.logger ?? new JsonLogger()).child({
+			workerId: this.workerId,
+		});
+		this.rpc = createRpcClients({
+			apiEndpoint: config.apiEndpoint,
+			pipelineEndpoint: config.pipelineEndpoint,
+			logger: this.logger,
+		});
 		this.sse = new SseClient({
-			endpoint: config.endpoint,
+			endpoint: config.pipelineEndpoint,
 			token: config.token,
 			workerId: this.workerId,
 			capabilities: this.capabilities,
+			logger: this.logger.child({ component: "sse" }),
 			onConnected: () => {
-				/* lifecycle: future event emitter hook */
+				/* logged inside sse-client; callers can hook lifecycle here later */
 			},
 			onDisconnected: () => {
-				/* lifecycle: future event emitter hook */
+				/* logged inside sse-client */
 			},
 			onJobDispatch: (payload) => this.handleDispatch(payload as JobEnvelope),
 		});
+
+		this.logger.info("sdk.boot", {
+			apiEndpoint: config.apiEndpoint,
+			pipelineEndpoint: config.pipelineEndpoint,
+			capabilities: this.capabilities,
+			concurrency: this.config.concurrency,
+			version: SDK_VERSION,
+		});
+
+		if (this.capabilities.length === 0) {
+			this.logger.warn("sdk.boot.no_handlers", {
+				note: "no handlers registered — worker will not receive any jobs",
+			});
+		}
 	}
 
 	async start(): Promise<void> {
+		this.logger.info("sdk.start");
 		this.sse.start();
 	}
 
 	async stop(): Promise<void> {
+		this.logger.info("sdk.stop", { inFlight: this.inFlight });
 		this.sse.stop();
 	}
 
 	private async handleDispatch(envelope: JobEnvelope): Promise<void> {
+		const jobLogger = this.logger.child({
+			jobId: envelope.jobId,
+			uploadId: envelope.uploadId,
+			urn: envelope.contentTypeContext.urn,
+		});
+
 		if (this.inFlight >= (this.config.concurrency ?? 1)) {
 			// Pipeline only dispatches to idle workers (zero in-flight), so
 			// this branch is defensive — if it ever fires we silently
 			// drop the dispatch and let the pipeline timeout requeue.
+			jobLogger.warn("job.dispatched.dropped_at_capacity", {
+				inFlight: this.inFlight,
+				concurrency: this.config.concurrency,
+			});
 			return;
 		}
 		this.inFlight++;
+		jobLogger.info("job.dispatched", {
+			filename: envelope.filename,
+			mimeType: envelope.mimeType,
+			inFlight: this.inFlight,
+		});
 
 		const handler = this.resolveHandler(envelope.contentTypeContext.urn);
-		const ctx = createJobContext(envelope, this.rpc);
+		const ctx = createJobContext(envelope, this.rpc, jobLogger);
+		const startedAt = Date.now();
 
 		try {
 			// AcceptJob — clears the pipeline's accept-timeout.
@@ -83,11 +129,16 @@ export class Pool {
 					queueToken: this.config.token,
 				}),
 			);
+			jobLogger.info("job.accepted");
 
 			if (!handler) {
+				jobLogger.warn("job.unhandled", {
+					availableHandlers: this.capabilities,
+				});
 				await this.reportError(
 					envelope.jobId,
 					ctx,
+					jobLogger,
 					new NonRetryableError(`no handler for URN ${envelope.contentTypeContext.urn}`),
 				);
 				return;
@@ -102,8 +153,15 @@ export class Pool {
 					queueToken: this.config.token,
 				}),
 			);
+			jobLogger.info("job.completed", {
+				durationMs: Date.now() - startedAt,
+			});
 		} catch (err) {
-			await this.reportError(envelope.jobId, ctx, err as Error);
+			jobLogger.error("job.failed", {
+				err,
+				durationMs: Date.now() - startedAt,
+			});
+			await this.reportError(envelope.jobId, ctx, jobLogger, err as Error);
 		} finally {
 			this.inFlight--;
 		}
@@ -116,6 +174,7 @@ export class Pool {
 	private async reportError(
 		jobId: string,
 		ctx: JobContext,
+		jobLogger: Logger,
 		err: Error,
 	): Promise<void> {
 		const retry = !(err instanceof NonRetryableError);
@@ -130,14 +189,12 @@ export class Pool {
 					retry,
 				}),
 			);
+			jobLogger.info("job.error_reported", { retry });
 		} catch (rptErr) {
+			jobLogger.error("job.error_report_failed", { err: rptErr });
 			this.config.onError?.(rptErr as Error, ctx);
 		}
 	}
 }
 
-function defaultErrorHandler(err: Error, ctx?: JobContext): void {
-	const where = ctx ? `job ${ctx.jobId} (${ctx.contentType.urn})` : "pool";
-	// eslint-disable-next-line no-console
-	console.error(`[@xeonr/upload-pool-sdk] ${where}: ${err.message}`);
-}
+const SDK_VERSION = "1.2.0";

package/src/rpc-clients.ts

@@ -1,29 +1,90 @@
 /**
  * ConnectRPC clients for the two services the SDK talks to:
- *   - InternalUploadsService — per-upload callbacks (UpdateUpload,
- *     RequestMetaUpload, ConfirmMetaUpload, GetProcessingContext).
- *     Auth: the per-job update_token from the job envelope, passed
- *     as the in-band `updateToken` field on each request.
- *   - IntegrationQueueService — accept / complete / report-error.
- *     Auth: the pool token, passed as `queueToken` on each request.
+ *   - InternalUploadsService (`apiEndpoint`) — per-upload callbacks
+ *     (UpdateUpload, RequestMetaUpload, ConfirmMetaUpload,
+ *     GetProcessingContext). Auth: the per-job `update_token` from the
+ *     job envelope, passed as the in-band `updateToken` field on each
+ *     request.
+ *   - IntegrationQueueService (`pipelineEndpoint`) — accept / complete /
+ *     report-error. Auth: the pool token, passed as `queueToken` on each
+ *     request.
+ *
+ * The two services live on different processes (Go uploads-api vs Node
+ * pipeline-api) so we keep them on separate transports and don't try to
+ * share a baseUrl.
+ *
+ * Each request is wrapped in a logging interceptor that emits one
+ * `rpc.request` line on dispatch and one of `rpc.response` /
+ * `rpc.error` on completion. Latency and connect-error codes are
+ * captured so failed RPCs are diagnosable from worker logs alone.
  */
 import { createConnectTransport } from "@connectrpc/connect-node";
-import { createClient, type Client } from "@connectrpc/connect";
+import {
+	createClient,
+	type Client,
+	type Interceptor,
+	ConnectError,
+	Code,
+} from "@connectrpc/connect";
 import { InternalUploadsService } from "./protocol/uplim/api/v1/uploads_pb.js";
 import { IntegrationQueueService } from "./protocol/uplim/workflow/v1/integration_queue_pb.js";
+import type { Logger } from "./logger.js";
 
 export interface RpcClients {
 	internalUploads: Client<typeof InternalUploadsService>;
 	integrationQueue: Client<typeof IntegrationQueueService>;
 }
 
-export function createRpcClients(endpoint: string): RpcClients {
-	const transport = createConnectTransport({
-		baseUrl: endpoint,
+function loggingInterceptor(logger: Logger, target: string): Interceptor {
+	return (next) => async (req) => {
+		const startedAt = Date.now();
+		const method = `${req.service.typeName}/${req.method.name}`;
+		logger.debug("rpc.request", { method, target });
+		try {
+			const res = await next(req);
+			logger.debug("rpc.response", {
+				method,
+				target,
+				durationMs: Date.now() - startedAt,
+			});
+			return res;
+		} catch (err) {
+			const code =
+				err instanceof ConnectError ? Code[err.code] : undefined;
+			const message = err instanceof Error ? err.message : String(err);
+			logger.error("rpc.error", {
+				method,
+				target,
+				code,
+				message,
+				durationMs: Date.now() - startedAt,
+			});
+			throw err;
+		}
+	};
+}
+
+export interface RpcClientsConfig {
+	apiEndpoint: string;
+	pipelineEndpoint: string;
+	logger: Logger;
+}
+
+export function createRpcClients(config: RpcClientsConfig): RpcClients {
+	const rpcLogger = config.logger.child({ component: "rpc" });
+
+	const apiTransport = createConnectTransport({
+		baseUrl: config.apiEndpoint,
+		httpVersion: "1.1",
+		interceptors: [loggingInterceptor(rpcLogger, "api")],
+	});
+	const pipelineTransport = createConnectTransport({
+		baseUrl: config.pipelineEndpoint,
 		httpVersion: "1.1",
+		interceptors: [loggingInterceptor(rpcLogger, "pipeline")],
 	});
 	return {
-		internalUploads: createClient(InternalUploadsService, transport),
-		integrationQueue: createClient(IntegrationQueueService, transport),
+		internalUploads: createClient(InternalUploadsService, apiTransport),
+		integrationQueue: createClient(IntegrationQueueService, pipelineTransport),
 	};
 }

package/src/sse-client.ts

@@ -5,14 +5,21 @@
  *
  * Reconnect strategy: exponential backoff (1s, 2s, 4s, 8s, capped at 30s).
  * On `job:dispatch` events the SDK invokes the registered onJob callback.
+ *
+ * All lifecycle transitions emit structured logs via the injected
+ * `Logger` so connection failures are visible in the worker's k8s logs
+ * (otherwise a worker that can't connect at all looks identical to one
+ * that's idle).
  */
 import { EventSource } from "eventsource";
+import type { Logger } from "./logger.js";
 
 export interface SseClientConfig {
 	endpoint: string;
 	token: string;
 	workerId: string;
 	capabilities: string[];
+	logger: Logger;
 	onConnected: () => void;
 	onDisconnected: (reason: string) => void;
 	onJobDispatch: (payload: unknown) => void;
@@ -25,6 +32,8 @@ export class SseClient {
 	private es: EventSource | null = null;
 	private stopped = false;
 	private reconnectDelay = RECONNECT_INITIAL_MS;
+	private connectAttempt = 0;
+	private connectedAt: number | null = null;
 
 	constructor(private readonly config: SseClientConfig) {}
 
@@ -39,11 +48,13 @@ export class SseClient {
 			this.es.close();
 			this.es = null;
 		}
+		this.config.logger.info("sse.stopped");
 	}
 
 	private connect(): void {
 		if (this.stopped) return;
 
+		this.connectAttempt++;
 		const url = new URL(`${this.config.endpoint}/queue/connect`);
 		url.searchParams.set("queueToken", this.config.token);
 		url.searchParams.set("workerId", this.config.workerId);
@@ -51,31 +62,84 @@ export class SseClient {
 			url.searchParams.set("capabilities", this.config.capabilities.join(","));
 		}
 
+		// Strip the token from the logged URL — the queueToken param is the
+		// pool secret, no reason to bake it into telemetry.
+		const loggedUrl = new URL(url.toString());
+		loggedUrl.searchParams.set("queueToken", "[redacted]");
+
+		this.config.logger.info("sse.connect.start", {
+			url: loggedUrl.toString(),
+			attempt: this.connectAttempt,
+			capabilities: this.config.capabilities,
+		});
+
 		this.es = new EventSource(url.toString());
 
+		this.es.addEventListener("open", () => {
+			this.config.logger.debug("sse.open");
+		});
+
 		this.es.addEventListener("connected", () => {
 			this.reconnectDelay = RECONNECT_INITIAL_MS;
+			this.connectedAt = Date.now();
+			this.config.logger.info("sse.connect.opened", {
+				attempt: this.connectAttempt,
+			});
 			this.config.onConnected();
 		});
 
 		this.es.addEventListener("job:dispatch", (event) => {
 			try {
 				const data = JSON.parse((event as MessageEvent).data);
+				this.config.logger.debug("sse.event.job_dispatch", {
+					jobId: (data as { jobId?: string }).jobId,
+				});
 				this.config.onJobDispatch(data);
 			} catch (err) {
-				this.config.onDisconnected(`bad job payload: ${(err as Error).message}`);
+				const reason = `bad job payload: ${(err as Error).message}`;
+				this.config.logger.error("sse.event.parse_error", {
+					err,
+					rawData: (event as MessageEvent).data,
+				});
+				this.config.onDisconnected(reason);
 			}
 		});
 
-		// heartbeat events are no-ops — receipt alone keeps the connection alive.
+		// heartbeat events are no-ops at the dispatch layer — receipt alone
+		// keeps the connection alive. We log at debug so noisy keep-alive
+		// traffic doesn't drown the info stream.
+		this.es.addEventListener("heartbeat", () => {
+			this.config.logger.debug("sse.event.heartbeat");
+		});
 
-		this.es.onerror = () => {
+		this.es.onerror = (err: unknown) => {
 			if (this.stopped) return;
-			this.config.onDisconnected("sse error");
+			const errMessage =
+				(err as { message?: string; type?: string; status?: number })?.message ??
+				(err as { type?: string })?.type ??
+				"unknown";
+			const status = (err as { status?: number; code?: number })?.status ??
+				(err as { code?: number })?.code;
+			const wasConnected = this.connectedAt !== null;
+			const uptimeMs = wasConnected ? Date.now() - this.connectedAt! : null;
+
+			this.config.logger.warn("sse.disconnected", {
+				message: errMessage,
+				status,
+				wasConnected,
+				uptimeMs,
+				attempt: this.connectAttempt,
+				nextReconnectMs: this.reconnectDelay,
+			});
+
+			this.config.onDisconnected(`sse error: ${errMessage}`);
+			this.connectedAt = null;
+
 			if (this.es) {
 				this.es.close();
 				this.es = null;
 			}
+
 			setTimeout(() => this.connect(), this.reconnectDelay);
 			this.reconnectDelay = Math.min(this.reconnectDelay * 2, RECONNECT_MAX_MS);
 		};

package/src/types.ts

@@ -17,16 +17,47 @@
 export interface PoolConfig {
 	/** Pool token, prefixed with "tpq_". From the pool's creation response. */
 	token: string;
-	/** API endpoint base URL, e.g. "https://api.upl.im". */
-	endpoint: string;
+	/**
+	 * Public uploads-api base URL, e.g. "https://uploads-api.xeonr.dev".
+	 * Hosts `InternalUploadsService` (RequestMetaUpload, ConfirmMetaUpload,
+	 * UpdateUpload, GetProcessingContext).
+	 *
+	 * Use the public https URL even when running inside the same k8s cluster
+	 * as the API — keeps third-party workers and first-party workers on a
+	 * single code path (TLS, normal ingress, no service-mesh assumptions).
+	 */
+	apiEndpoint: string;
+	/**
+	 * Public pipeline-api base URL, e.g. "https://uploads-pipeline-api.xeonr.dev".
+	 * Hosts the SSE `/queue/connect` endpoint and `IntegrationQueueService`
+	 * (AcceptJob, CompleteJob, ReportError).
+	 *
+	 * Distinct from `apiEndpoint` because the pipeline runs as a separate
+	 * Node service; conflating the two would silently route worker traffic
+	 * to the wrong process.
+	 */
+	pipelineEndpoint: string;
 	/** Optional worker identifier. Defaults to hostname + random suffix. */
 	workerId?: string;
 	/** Handlers keyed by content type URN (e.g. "default:image"). */
 	handlers: Record<string, JobHandler>;
 	/** Fallback handler if URN has no specific entry in `handlers`. Optional. */
 	onUnhandled?: JobHandler;
-	/** Lifecycle / error hook. Defaults to console-based logging. */
+	/**
+	 * Lifecycle / error hook. Receives every handler exception alongside
+	 * the job context. The SDK also logs every failure via `logger` —
+	 * `onError` is for hooks that need to fan out to external systems
+	 * (sentry, pagerduty). Optional.
+	 */
 	onError?: (err: Error, ctx?: JobContext) => void;
+	/**
+	 * Structured logger. Defaults to a JSON-line logger that writes to
+	 * stdout (`info`/`debug`) and stderr (`warn`/`error`), honoring
+	 * `UPL_LOG_LEVEL`. Pass `noopLogger` from this package for silence,
+	 * or supply your own implementing the `Logger` interface to bridge
+	 * to pino/winston.
+	 */
+	logger?: import("./logger.js").Logger;
 	/** Max concurrent in-flight jobs. Defaults to 1. */
 	concurrency?: number;
 }