@pipe0/client 1.0.1 → 1.0.2

package/dist/pipe0.mjs

@@ -1,3 +1,4 @@
+import { CapacityTracker } from "./capacity-tracker.mjs";
 import createFetchClient from "openapi-fetch";
 
 //#region src/pipe0.ts
@@ -17,6 +18,17 @@ var Pipe0AbortError = class extends Error {
 		this.runId = runId;
 	}
 };
+/**
+* The server reports the run as `canceled` (via `pipes.cancel` /
+* `searches.cancel` or another client). Subclasses {@link Pipe0AbortError} so
+* existing `instanceof Pipe0AbortError` handling keeps working.
+*/
+var Pipe0CanceledError = class extends Pipe0AbortError {
+	constructor(message, runId) {
+		super(message, runId);
+		this.name = "Pipe0CanceledError";
+	}
+};
 var Pipe0BatchError = class extends Error {
 	constructor(message, errors, successfulBatches) {
 		super(message);
@@ -36,11 +48,53 @@ var Pipe0TaskError = class extends Error {
 	}
 };
 var Pipe0ServerError = class extends Error {
-	constructor(message) {
+	/** HTTP status of the error response, when known. */
+	status;
+	/** RFC 7807 problem `type` discriminator (e.g. "queue-limit-exceeded", "org-rate-limited"). */
+	problemType;
+	constructor(message, options = {}) {
 		super(message);
 		this.name = "Pipe0ServerError";
+		this.status = options.status;
+		this.problemType = options.problemType;
+	}
+};
+/**
+* Thrown only after the SDK's automatic 429 handling exhausted its wait
+* budget (`rateLimitMaxWaitMs` / `queueFullMaxWaitMs`), or immediately when
+* `autoRetry429` is disabled the error surfaces as a plain
+* {@link Pipe0ServerError} instead. Subclasses Pipe0ServerError so existing
+* handling keeps working.
+*/
+var Pipe0RateLimitError = class extends Pipe0ServerError {
+	/** The delay the SDK would have waited next, in ms. */
+	retryAfterMs;
+	/** Attempts made before giving up. */
+	attempts;
+	constructor(message, options = {}) {
+		super(message, {
+			status: 429,
+			problemType: options.problemType
+		});
+		this.name = "Pipe0RateLimitError";
+		this.retryAfterMs = options.retryAfterMs;
+		this.attempts = options.attempts ?? 1;
 	}
 };
+/** The run-creating endpoints — the only responses that carry capacity headers. */
+const CAPACITY_HEADER_PATHS = new Set([
+	"/v1/pipes/run",
+	"/v1/pipes/run/sync",
+	"/v1/search/run",
+	"/v1/search/run/sync",
+	"/v1/searches/run",
+	"/v1/searches/run/sync",
+	"/v1/pipes/cancel/{run_id}",
+	"/v1/search/cancel/{run_id}"
+]);
+function extractProblemType(error) {
+	if (error !== null && typeof error === "object" && "type" in error && typeof error.type === "string") return error.type;
+}
 var Pipe0 = class {
 	client;
 	apiKey;
@@ -49,6 +103,15 @@ var Pipe0 = class {
 	maxPollingIntervalMs;
 	defaultBatchSize;
 	maxConcurrentBatches;
+	autoRetry429;
+	retryAfterCapMs;
+	rateLimitMaxWaitMs;
+	queueFullMaxWaitMs;
+	defaultCancelOnAbort;
+	defaultPriorityFloor;
+	pacingProbeMs;
+	onCapacity;
+	tracker = new CapacityTracker();
 	constructor(options = {}) {
 		const envApiKey = typeof process !== "undefined" ? process.env?.PIPE0_API_KEY : void 0;
 		this.apiKey = options.apiKey ?? envApiKey;
@@ -57,13 +120,92 @@ var Pipe0 = class {
 		this.maxPollingIntervalMs = options.maxPollingIntervalMs ?? this.minPollingIntervalMs * 3;
 		this.defaultBatchSize = options.defaultBatchSize ?? 100;
 		this.maxConcurrentBatches = options.maxConcurrentBatches ?? 5;
+		this.autoRetry429 = options.autoRetry429 ?? true;
+		this.retryAfterCapMs = options.retryAfterCapMs ?? 3e4;
+		this.rateLimitMaxWaitMs = options.rateLimitMaxWaitMs ?? 12e4;
+		this.queueFullMaxWaitMs = options.queueFullMaxWaitMs ?? 9e5;
+		this.defaultCancelOnAbort = options.cancelOnAbort ?? true;
+		this.defaultPriorityFloor = options.priorityFloor ?? 2;
+		this.pacingProbeMs = options.pacingProbeMs ?? 15e3;
+		this.onCapacity = options.onCapacity;
 		const headers = {};
 		if (this.apiKey) headers.Authorization = `Bearer ${this.apiKey}`;
 		this.client = createFetchClient({
 			baseUrl: options.baseUrl ?? "https://api.pipe0.com",
 			credentials: options.credentials ? options.credentials : options.baseUrl ? "same-origin" : "include",
-			headers
+			headers,
+			fetch: options.fetch
 		});
+		this.client.use({ onResponse: ({ schemaPath, response }) => {
+			if (!CAPACITY_HEADER_PATHS.has(schemaPath)) return void 0;
+			const snapshot = this.tracker.observeHeaders(response.headers);
+			if (snapshot && this.onCapacity) try {
+				this.onCapacity(snapshot);
+			} catch {}
+		} });
+	}
+	/**
+	* The SDK's current estimate of your organization's queue capacity, from
+	* the most recent run-creating response's headers adjusted by runs the SDK
+	* has since observed completing. Null until the first create.
+	*/
+	get capacity() {
+		return this.tracker.snapshot();
+	}
+	/**
+	* The single request funnel: every API call goes through here so 429
+	* handling and capacity bookkeeping live in exactly one place.
+	*
+	* - `org-rate-limited` (request bucket, has Retry-After): sleep and retry,
+	*   budget `rateLimitMaxWaitMs`.
+	* - `queue-limit-exceeded` (queued-record hard ceiling, no Retry-After):
+	*   exponential backoff retry, budget `queueFullMaxWaitMs` — re-issuing the
+	*   create is safe because the ceiling rejects before anything is queued.
+	* - Anything else throws {@link Pipe0ServerError} with `status` +
+	*   `problemType` attached.
+	*/
+	async request(fn, opts = {}) {
+		const retryEnabled = opts.retry ?? this.autoRetry429;
+		const startedAt = Date.now();
+		let attempts = 0;
+		let queueDelayMs = 5e3;
+		while (true) {
+			attempts += 1;
+			if (opts.capacity) this.tracker.onCreateIssued(opts.capacity.records);
+			let result;
+			try {
+				result = await fn();
+			} finally {
+				if (opts.capacity) this.tracker.onCreateSettled(opts.capacity.records);
+			}
+			if (!result.error) return result.data;
+			const status = result.response?.status;
+			const problemType = extractProblemType(result.error);
+			if (status !== 429 || !retryEnabled) throw new Pipe0ServerError(JSON.stringify(result.error), {
+				status,
+				problemType
+			});
+			const isQueueFull = problemType === "queue-limit-exceeded";
+			let delayMs;
+			let budgetMs;
+			if (isQueueFull) {
+				delayMs = queueDelayMs;
+				queueDelayMs = Math.min(queueDelayMs * 2, 6e4);
+				budgetMs = this.queueFullMaxWaitMs;
+			} else {
+				const retryAfterSeconds = Number(result.response.headers.get("Retry-After"));
+				const baseMs = Number.isFinite(retryAfterSeconds) && retryAfterSeconds > 0 ? retryAfterSeconds * 1e3 : 5e3;
+				delayMs = Math.min(Math.max(baseMs, 1e3), this.retryAfterCapMs);
+				budgetMs = this.rateLimitMaxWaitMs;
+			}
+			delayMs = Math.round(delayMs * (1 + Math.random() * .25));
+			if (Date.now() - startedAt + delayMs > budgetMs) throw new Pipe0RateLimitError(`Request still rate limited (${problemType ?? "429"}) after ${attempts} attempt(s) over ${Date.now() - startedAt}ms`, {
+				problemType,
+				retryAfterMs: delayMs,
+				attempts
+			});
+			await this.sleep(delayMs, opts.signal);
+		}
 	}
 	/**
 	* Centralized logic to poll until a task completes, fails, or times out.
@@ -73,107 +215,184 @@ var Pipe0 = class {
 		const timeout = this.pollingTimeoutMs;
 		const initialInterval = this.minPollingIntervalMs;
 		const maxInterval = this.maxPollingIntervalMs;
+		const cancelOnAbort = options.cancelOnAbort ?? this.defaultCancelOnAbort;
 		const startTime = Date.now();
 		let currentInterval = initialInterval;
-		while (true) {
-			if (Date.now() - startTime >= timeout) throw new Pipe0TimeoutError(`${taskType} polling timed out after ${timeout}ms`, resolvedRunId);
-			if (options.signal?.aborted) throw new Pipe0AbortError(`${taskType} polling was aborted`, resolvedRunId);
-			const polledResponse = await checkFn(resolvedRunId);
-			if (options.onPoll) await options.onPoll(polledResponse);
-			if (polledResponse.status === "completed") return polledResponse;
-			if (polledResponse.status === "skipped") return polledResponse;
-			if (polledResponse.status === "canceled") throw new Pipe0AbortError(`${taskType} run was canceled`, resolvedRunId);
-			if (polledResponse.status === "failed") throw new Pipe0TaskError(`${taskType} failed with status: ${polledResponse.status}`, polledResponse);
-			await this.sleep(currentInterval, options.signal);
-			currentInterval = Math.min(currentInterval * 1.1, maxInterval);
+		try {
+			while (true) {
+				if (Date.now() - startTime >= timeout) throw new Pipe0TimeoutError(`${taskType} polling timed out after ${timeout}ms`, resolvedRunId);
+				if (options.signal?.aborted) throw new Pipe0AbortError(`${taskType} polling was aborted`, resolvedRunId);
+				const polledResponse = await checkFn(resolvedRunId);
+				if (options.onPoll) await options.onPoll(polledResponse);
+				if (polledResponse.status === "completed") {
+					this.tracker.creditTerminal(resolvedRunId);
+					return polledResponse;
+				}
+				if (polledResponse.status === "skipped") {
+					this.tracker.creditTerminal(resolvedRunId);
+					return polledResponse;
+				}
+				if (polledResponse.status === "canceled") {
+					this.tracker.creditTerminal(resolvedRunId);
+					throw new Pipe0CanceledError(`${taskType} run was canceled`, resolvedRunId);
+				}
+				if (polledResponse.status === "failed") {
+					this.tracker.creditTerminal(resolvedRunId);
+					throw new Pipe0TaskError(`${taskType} failed with status: ${polledResponse.status}`, polledResponse);
+				}
+				await this.sleep(currentInterval, options.signal);
+				currentInterval = Math.min(currentInterval * 1.1, maxInterval);
+			}
+		} catch (error) {
+			if (cancelOnAbort && options.cancelFn && options.signal?.aborted && !(error instanceof Pipe0CanceledError)) options.cancelFn(resolvedRunId).catch(() => {});
+			throw error;
 		}
 	}
 	pipes = {
 		pipe: async (payload, options = {}) => {
-			return this.pipes.waitUntilComplete(await this.pipes.create(payload), options);
+			return this.pipes.waitUntilComplete(await this.pipes.create(payload, options), options);
+		},
+		create: async (payload, options = {}) => {
+			const records = Array.isArray(payload.input) ? payload.input.length : 1;
+			const data = await this.request(() => this.client.POST("/v1/pipes/run", { body: payload }), {
+				signal: options.signal,
+				capacity: { records }
+			});
+			this.tracker.registerRun(data.id, records);
+			return data.id;
 		},
-		create: async (payload) => {
-			const response = await this.client.POST("/v1/pipes/run", { body: payload });
-			if (response.error) throw new Pipe0ServerError(JSON.stringify(response.error));
-			return response.data.id;
+		check: async (runId, options = {}) => {
+			return this.request(() => this.client.GET("/v1/pipes/check/{run_id}", {
+				params: { path: { run_id: runId } },
+				signal: options.signal
+			}), { signal: options.signal });
 		},
-		check: async (runId) => {
-			const response = await this.client.GET("/v1/pipes/check/{run_id}", { params: { path: { run_id: runId } } });
-			if (response.error) throw new Pipe0ServerError(JSON.stringify(response.error));
-			return response.data;
+		cancel: async (runId, options = {}) => {
+			const data = await this.request(() => this.client.POST("/v1/pipes/cancel/{run_id}", {
+				params: { path: { run_id: runId } },
+				signal: options.signal
+			}), { signal: options.signal });
+			this.tracker.creditTerminal(runId);
+			return data;
 		},
 		waitUntilComplete: async (runId, options = {}) => {
-			return this._pollUntilComplete(runId, this.pipes.check, "Pipe", options);
+			return this._pollUntilComplete(runId, (id) => this.pipes.check(id, { signal: options.signal }), "Pipe", {
+				...options,
+				cancelFn: (id) => this.pipes.cancel(id)
+			});
 		},
-		pipeInBatches: async (payload, { stopOnError = true, ...options } = {}) => {
+		pipeInBatches: async (payload, { stopOnError = true, priorityFloor, cancelOnAbort, ...options } = {}) => {
 			const batchSize = this.defaultBatchSize;
 			const maxConcurrency = this.maxConcurrentBatches;
+			const floor = priorityFloor ?? this.defaultPriorityFloor;
+			const doCancelOnAbort = cancelOnAbort ?? this.defaultCancelOnAbort;
 			if (!Array.isArray(payload.input)) throw new Error("payload.input must be an array for batched operations");
 			const inputArray = payload.input;
-			const totalItems = inputArray.length;
 			const batches = [];
-			for (let i = 0; i < totalItems; i += batchSize) {
-				const batchInput = inputArray.slice(i, i + batchSize);
-				batches.push({
-					...payload,
-					input: batchInput
-				});
-			}
-			const resultsArr = [];
+			for (let i = 0; i < inputArray.length; i += batchSize) batches.push({
+				...payload,
+				input: inputArray.slice(i, i + batchSize)
+			});
+			const results = new Array(batches.length);
 			const errors = [];
-			for (let i = 0; i < batches.length; i += maxConcurrency) {
-				if (options.signal?.aborted) throw new Pipe0AbortError("Batch operation was aborted");
-				const batchPromises = batches.slice(i, i + maxConcurrency).map(async (batch, groupIndex) => {
-					const batchIndex = i + groupIndex;
-					try {
-						const runId = await this.pipes.create(batch);
-						const result = await this.pipes.waitUntilComplete(runId, {
-							signal: options.signal,
-							onPoll: options.onPoll ? (response) => options.onPoll(batchIndex, response) : void 0
-						});
-						if (options.onBatchComplete) options.onBatchComplete(batchIndex, result);
-						return {
-							batchIndex,
-							result
-						};
-					} catch (error) {
-						const err = error instanceof Error ? error : new Error(String(error));
-						errors.push({
-							batchIndex,
-							error: err
-						});
-						if (stopOnError) throw new Pipe0BatchError(`Batch ${batchIndex} failed and stopOnError is enabled`, errors, resultsArr);
-						return {
-							batchIndex,
-							result: null
-						};
+			const inFlight = /* @__PURE__ */ new Set();
+			/** batchIndex → runId while the run is still cancelable. */
+			const pendingRuns = /* @__PURE__ */ new Map();
+			let next = 0;
+			let stopped = false;
+			const sweepCancel = () => {
+				if (!doCancelOnAbort) return;
+				for (const runId of pendingRuns.values()) this.pipes.cancel(runId).catch(() => {});
+				pendingRuns.clear();
+			};
+			const runBatch = async (batchIndex) => {
+				try {
+					const runId = await this.pipes.create(batches[batchIndex], { signal: options.signal });
+					pendingRuns.set(batchIndex, runId);
+					const result = await this.pipes.waitUntilComplete(runId, {
+						signal: options.signal,
+						cancelOnAbort: false,
+						onPoll: options.onPoll ? (response) => options.onPoll(batchIndex, response) : void 0
+					});
+					pendingRuns.delete(batchIndex);
+					results[batchIndex] = result;
+					if (options.onBatchComplete) options.onBatchComplete(batchIndex, result);
+				} catch (error) {
+					if (error instanceof Pipe0AbortError && options.signal?.aborted) return;
+					pendingRuns.delete(batchIndex);
+					errors.push({
+						batchIndex,
+						error: error instanceof Error ? error : new Error(String(error))
+					});
+					if (stopOnError) stopped = true;
+				}
+			};
+			const submit = (batchIndex) => {
+				const p = runBatch(batchIndex).finally(() => inFlight.delete(p));
+				inFlight.add(p);
+			};
+			try {
+				while (next < batches.length && !stopped || inFlight.size > 0) {
+					if (options.signal?.aborted) throw new Pipe0AbortError("Batch operation was aborted");
+					if (!stopped && next < batches.length && inFlight.size < maxConcurrency) {
+						const records = batches[next].input.length;
+						const estimate = this.tracker.estimate(floor);
+						if (estimate === null || estimate >= records) {
+							submit(next++);
+							continue;
+						}
+						if (inFlight.size === 0) {
+							await this.sleep(this.pacingProbeMs, options.signal);
+							if (options.signal?.aborted) throw new Pipe0AbortError("Batch operation was aborted");
+							submit(next++);
+							continue;
+						}
 					}
-				});
-				const batchResults = await Promise.all(batchPromises);
-				for (const { result } of batchResults) if (result) resultsArr.push(result);
+					if (inFlight.size > 0) await Promise.race(inFlight);
+				}
+			} catch (error) {
+				sweepCancel();
+				throw error;
 			}
-			if (errors.length > 0) throw new Pipe0BatchError(`${errors.length} batch(es) failed out of ${batches.length} total`, errors, resultsArr);
-			return resultsArr;
+			const compact = results.filter((r) => r !== void 0);
+			if (stopped) throw new Pipe0BatchError(`Batch ${errors[0]?.batchIndex} failed and stopOnError is enabled`, errors, compact);
+			if (errors.length > 0) throw new Pipe0BatchError(`${errors.length} batch(es) failed out of ${batches.length} total`, errors, compact);
+			return compact;
 		}
 	};
 	searches = {
 		search: async (payload, options = {}) => {
-			return this.searches.waitUntilComplete(await this.searches.create(payload), options);
+			return this.searches.waitUntilComplete(await this.searches.create(payload, options), options);
+		},
+		create: async (payload, options = {}) => {
+			const data = await this.request(() => this.client.POST("/v1/search/run", { body: payload }), {
+				signal: options.signal,
+				capacity: { records: 1 }
+			});
+			this.tracker.registerRun(data.id, 1);
+			return data.id;
 		},
-		create: async (payload) => {
-			const response = await this.client.POST("/v1/search/run", { body: payload });
-			if (response.error) throw new Pipe0ServerError(JSON.stringify(response.error));
-			return response.data.id;
+		check: async (runId, options = {}) => {
+			return this.request(() => this.client.GET("/v1/search/check/{run_id}", {
+				params: { path: { run_id: runId } },
+				signal: options.signal
+			}), { signal: options.signal });
 		},
-		check: async (runId) => {
-			const response = await this.client.GET("/v1/search/check/{run_id}", { params: { path: { run_id: runId } } });
-			if (response.error) throw new Pipe0ServerError(JSON.stringify(response.error));
-			return response.data;
+		cancel: async (runId, options = {}) => {
+			const data = await this.request(() => this.client.POST("/v1/search/cancel/{run_id}", {
+				params: { path: { run_id: runId } },
+				signal: options.signal
+			}), { signal: options.signal });
+			this.tracker.creditTerminal(runId);
+			return data;
 		},
 		waitUntilComplete: async (runId, options = {}) => {
-			return this._pollUntilComplete(runId, this.searches.check, "Search", options);
+			return this._pollUntilComplete(runId, (id) => this.searches.check(id, { signal: options.signal }), "Search", {
+				...options,
+				cancelFn: (id) => this.searches.cancel(id)
+			});
 		},
-		searchAll: async ({ searches, config, dedupeBy, maxPages = 1, normalize = true, maxConcurrency = this.maxConcurrentBatches, signal, stopOnError = true }) => {
+		searchAll: async ({ searches, config, dedupeBy, maxPages = 1, normalize = true, maxConcurrency = this.maxConcurrentBatches, signal, stopOnError = true, cancelOnAbort }) => {
 			const errors = [];
 			const merged = [];
 			const runItem = async (item) => {
@@ -185,7 +404,10 @@ var Pipe0 = class {
 					const res = await this.searches.search({
 						config,
 						search
-					}, { signal });
+					}, {
+						signal,
+						cancelOnAbort
+					});
 					rows.push(...res.results);
 					const next = res.next_page;
 					if (!next) break;
@@ -220,23 +442,22 @@ var Pipe0 = class {
 	};
 	sheets = { effects: {
 		run: async (sheetId, payload) => {
-			const response = await this.client.POST("/v1/sheets/{sheet_id}/effects/run", {
+			return this.request(() => this.client.POST("/v1/sheets/{sheet_id}/effects/run", {
 				params: { path: { sheet_id: sheetId } },
 				body: payload
-			});
-			if (response.error) throw new Pipe0ServerError(JSON.stringify(response.error));
-			return response.data;
+			}));
 		},
-		check: async (sheetId, runId) => {
-			const response = await this.client.GET("/v1/sheets/{sheet_id}/effects/check/{run_id}", { params: { path: {
-				sheet_id: sheetId,
-				run_id: runId
-			} } });
-			if (response.error) throw new Pipe0ServerError(JSON.stringify(response.error));
-			return response.data;
+		check: async (sheetId, runId, options = {}) => {
+			return this.request(() => this.client.GET("/v1/sheets/{sheet_id}/effects/check/{run_id}", {
+				params: { path: {
+					sheet_id: sheetId,
+					run_id: runId
+				} },
+				signal: options.signal
+			}), { signal: options.signal });
 		},
 		waitUntilComplete: async (sheetId, runId, options = {}) => {
-			return this._pollUntilComplete(runId, (id) => this.sheets.effects.check(sheetId, id), "Effects", options);
+			return this._pollUntilComplete(runId, (id) => this.sheets.effects.check(sheetId, id, { signal: options.signal }), "Effects", options);
 		},
 		runUntilComplete: async (sheetId, payload, options = {}) => {
 			const { id } = await this.sheets.effects.run(sheetId, payload);
@@ -284,7 +505,7 @@ function dedupeResults(rows, fields) {
 			out.push(row);
 			continue;
 		}
-		const key = parts.join("\0");
+		const key = parts.join(" ");
 		if (seen.has(key)) continue;
 		seen.add(key);
 		out.push(row);
@@ -308,5 +529,5 @@ function normalizeResults(rows) {
 }
 
 //#endregion
-export { Pipe0, Pipe0AbortError, Pipe0BatchError, Pipe0ServerError, Pipe0TaskError, Pipe0TimeoutError };
+export { Pipe0, Pipe0AbortError, Pipe0BatchError, Pipe0CanceledError, Pipe0RateLimitError, Pipe0ServerError, Pipe0TaskError, Pipe0TimeoutError };
 //# sourceMappingURL=pipe0.mjs.map