@xeonr/upload-pool-sdk 1.0.0

package/src/rpc-clients.ts

@@ -0,0 +1,29 @@
+/**
+ * ConnectRPC clients for the two services the SDK talks to:
+ *   - InternalUploadsService — per-upload callbacks (UpdateUpload,
+ *     RequestMetaUpload, ConfirmMetaUpload, GetProcessingContext).
+ *     Auth: the per-job update_token from the job envelope, passed
+ *     as the in-band `updateToken` field on each request.
+ *   - IntegrationQueueService — accept / complete / report-error.
+ *     Auth: the pool token, passed as `queueToken` on each request.
+ */
+import { createConnectTransport } from "@connectrpc/connect-node";
+import { createClient, type Client } from "@connectrpc/connect";
+import { InternalUploadsService } from "./protocol/uplim/api/v1/uploads_pb.js";
+import { IntegrationQueueService } from "./protocol/uplim/workflow/v1/integration_queue_pb.js";
+
+export interface RpcClients {
+	internalUploads: Client<typeof InternalUploadsService>;
+	integrationQueue: Client<typeof IntegrationQueueService>;
+}
+
+export function createRpcClients(endpoint: string): RpcClients {
+	const transport = createConnectTransport({
+		baseUrl: endpoint,
+		httpVersion: "1.1",
+	});
+	return {
+		internalUploads: createClient(InternalUploadsService, transport),
+		integrationQueue: createClient(IntegrationQueueService, transport),
+	};
+}

package/src/sse-client.ts

@@ -0,0 +1,83 @@
+/**
+ * SSE client for the pool's job dispatch stream. Uses the `eventsource`
+ * npm package because Node's built-in EventSource (Node 22+) varies in
+ * support across our deployment targets.
+ *
+ * Reconnect strategy: exponential backoff (1s, 2s, 4s, 8s, capped at 30s).
+ * On `job:dispatch` events the SDK invokes the registered onJob callback.
+ */
+import { EventSource } from "eventsource";
+
+export interface SseClientConfig {
+	endpoint: string;
+	token: string;
+	workerId: string;
+	capabilities: string[];
+	onConnected: () => void;
+	onDisconnected: (reason: string) => void;
+	onJobDispatch: (payload: unknown) => void;
+}
+
+const RECONNECT_INITIAL_MS = 1_000;
+const RECONNECT_MAX_MS = 30_000;
+
+export class SseClient {
+	private es: EventSource | null = null;
+	private stopped = false;
+	private reconnectDelay = RECONNECT_INITIAL_MS;
+
+	constructor(private readonly config: SseClientConfig) {}
+
+	start(): void {
+		this.stopped = false;
+		this.connect();
+	}
+
+	stop(): void {
+		this.stopped = true;
+		if (this.es) {
+			this.es.close();
+			this.es = null;
+		}
+	}
+
+	private connect(): void {
+		if (this.stopped) return;
+
+		const url = new URL(`${this.config.endpoint}/queue/connect`);
+		url.searchParams.set("queueToken", this.config.token);
+		url.searchParams.set("workerId", this.config.workerId);
+		if (this.config.capabilities.length > 0) {
+			url.searchParams.set("capabilities", this.config.capabilities.join(","));
+		}
+
+		this.es = new EventSource(url.toString());
+
+		this.es.addEventListener("connected", () => {
+			this.reconnectDelay = RECONNECT_INITIAL_MS;
+			this.config.onConnected();
+		});
+
+		this.es.addEventListener("job:dispatch", (event) => {
+			try {
+				const data = JSON.parse((event as MessageEvent).data);
+				this.config.onJobDispatch(data);
+			} catch (err) {
+				this.config.onDisconnected(`bad job payload: ${(err as Error).message}`);
+			}
+		});
+
+		// heartbeat events are no-ops — receipt alone keeps the connection alive.
+
+		this.es.onerror = () => {
+			if (this.stopped) return;
+			this.config.onDisconnected("sse error");
+			if (this.es) {
+				this.es.close();
+				this.es = null;
+			}
+			setTimeout(() => this.connect(), this.reconnectDelay);
+			this.reconnectDelay = Math.min(this.reconnectDelay * 2, RECONNECT_MAX_MS);
+		};
+	}
+}

package/src/types.ts

@@ -0,0 +1,169 @@
+/**
+ * Public types for the upl-im worker SDK.
+ *
+ * Customers implement a handler per content type URN. The SDK handles:
+ *   - SSE connection to the pool's job queue (with reconnect)
+ *   - Job acceptance and completion via the IntegrationQueueService RPCs
+ *   - Source download from a presigned URL
+ *   - Thumbnail upload to presigned URLs (RequestMetaUpload + S3 PUT + ConfirmMetaUpload)
+ *   - Metadata writes via UpdateUpload, scoped to the job's upload by the
+ *     per-upload `update_token` JWT embedded in the job envelope
+ *
+ * Auth model: pool token (`tpq_…`) authenticates the SSE connection.
+ * Per-job `update_token` (the existing 24h JWT) authenticates the four
+ * InternalUploadsService callbacks.
+ */
+
+export interface PoolConfig {
+	/** Pool token, prefixed with "tpq_". From the pool's creation response. */
+	token: string;
+	/** API endpoint base URL, e.g. "https://api.upl.im". */
+	endpoint: string;
+	/** Optional worker identifier. Defaults to hostname + random suffix. */
+	workerId?: string;
+	/** Handlers keyed by content type URN (e.g. "default:image"). */
+	handlers: Record<string, JobHandler>;
+	/** Fallback handler if URN has no specific entry in `handlers`. Optional. */
+	onUnhandled?: JobHandler;
+	/** Lifecycle / error hook. Defaults to console-based logging. */
+	onError?: (err: Error, ctx?: JobContext) => void;
+	/** Max concurrent in-flight jobs. Defaults to 1. */
+	concurrency?: number;
+}
+
+export type JobHandler = (ctx: JobContext) => Promise<void>;
+
+export interface JobContext {
+	jobId: string;
+	uploadId: string;
+	filename: string;
+	mimeType: string;
+	contentType: {
+		urn: string;
+		id: string;
+		/** Typed proto blob — content-type-specific. May be empty. */
+		config: Uint8Array;
+	};
+	folderContext?: FolderContext;
+
+	/** Direct S3 download URL — short-lived (≈15 min). */
+	sourceUrl: string;
+	/** Stream the source file. Prefer for large files. */
+	download(): Promise<ReadableStream<Uint8Array>>;
+	/** Buffer the entire source file in memory. Convenient for thumbnails. */
+	downloadBuffer(): Promise<Uint8Array>;
+	/** Stream to a local file path. */
+	downloadToFile(path: string): Promise<void>;
+
+	/** Mint a presigned upload URL, PUT data, confirm. Atomic from caller's POV. */
+	uploadMeta(opts: UploadMetaOpts): Promise<void>;
+	/**
+	 * Write upload-row fields. Accepts either the loose UploadMetadataPartial
+	 * (most handlers) or a fully-constructed UploadMetadata proto (handlers
+	 * that need typed oneof sub-messages like the archive metadata's
+	 * directory tree). When passing a proto, construct via the schema from
+	 * `@upl-im/default-workers-shared` or your own proto bindings.
+	 */
+	setMetadata(metadata: UploadMetadataPartial | UploadMetadataProto): Promise<void>;
+	/** Convenience wrapper for setMetadata with description + tags. */
+	setDescription(text: string, tags?: string[]): Promise<void>;
+	/** Mark has_thumbnail=true and stamp the thumbnail_generation_version. */
+	markHasThumbnail(version: number): Promise<void>;
+
+	/**
+	 * Lower-level: mint a presigned upload URL for the meta-upload type
+	 * and pass it to the callback. After the callback resolves the SDK
+	 * calls ConfirmMetaUpload; on throw it leaves the row unfinalised
+	 * (matches pipeline semantics — unfinalised rows get cleaned up by
+	 * a follow-up confirm or expire after 24h).
+	 *
+	 * Use this when the renderer writes to the presigned URL itself
+	 * (e.g., the legacy localTasks generateThumbnailLocal that fans
+	 * the same file out to multiple URLs in parallel via fetch PUT).
+	 * Otherwise prefer uploadMeta().
+	 *
+	 * Callback receives an array of MetaUploadHandles (uploadUrl,
+	 * downloadUrl, metadataId) parallel to `opts`. downloadUrl is what
+	 * AI description generators want as input after a thumbnail/sheet
+	 * has been written.
+	 */
+	withPresignedUploads<T>(
+		opts: PresignedUploadRequest[],
+		callback: (handles: MetaUploadHandle[]) => Promise<T>,
+	): Promise<T>;
+}
+
+export interface PresignedUploadRequest {
+	type: UploadMetaOpts["type"];
+	instance?: number;
+	filename?: string;
+}
+
+export interface MetaUploadHandle {
+	/** Presigned S3 URL for the renderer to PUT to. */
+	uploadUrl: string;
+	/** Public download URL — usable as AI description input once the upload + confirm round-trip completes. */
+	downloadUrl: string;
+	/** Internal metadata row id. Mostly opaque to handlers; SDK uses it for the confirm step. */
+	metadataId: string;
+}
+
+export interface FolderContext {
+	folderId: string;
+	folderPath: string;
+	files: Array<{
+		uploadId: string;
+		filename: string;
+		downloadUrl: string;
+	}>;
+}
+
+export interface UploadMetaOpts {
+	type: "THUMBNAIL_LIGHT" | "THUMBNAIL_DARK" | "VIDEO_TIMELINE" | "ARCHIVE_FILE" | "IMAGE_COMPAT" | "AI_CONTENT" | "RENDERER_CONFIG";
+	data: Uint8Array | ReadableStream<Uint8Array>;
+	/** For VIDEO_TIMELINE: which frame index. */
+	instance?: number;
+	/** For ARCHIVE_FILE: the file path inside the archive. */
+	filename?: string;
+	/** S3 content-type header. Defaults to image/png for thumbnails. */
+	contentType?: string;
+}
+
+/**
+ * Marker type for callers passing a fully-constructed UploadMetadata proto.
+ * Detected at runtime by the presence of the @bufbuild/protobuf $typeName
+ * field — workers shouldn't need to import the proto type here.
+ */
+export type UploadMetadataProto = { $typeName: string; [k: string]: unknown };
+
+/**
+ * Partial UploadMetadata — workers populate the fields they computed; the
+ * server writes them as-is. Mirrors the shape of the `UploadMetadata` proto
+ * but is intentionally loose so the SDK doesn't churn on every proto edit.
+ */
+export interface UploadMetadataPartial {
+	image?: {
+		width?: number;
+		height?: number;
+	};
+	video?: {
+		width?: number;
+		height?: number;
+		durationMs?: number;
+	};
+	audio?: {
+		durationMs?: number;
+	};
+	pdf?: {
+		pageCount?: number;
+	};
+	thumbnailGenerationVersion?: number;
+}
+
+export interface PoolLifecycleEvents {
+	connected: { workerId: string; poolId: string };
+	disconnected: { workerId: string; reason: string };
+	jobStarted: { jobId: string; urn: string };
+	jobCompleted: { jobId: string; urn: string; durationMs: number };
+	jobFailed: { jobId: string; urn: string; error: Error };
+}