@interactive-inc/flume 0.9.4 → 0.10.1

package/dist/github.js

@@ -1,4 +1,4 @@
-import { c as safeNormalizeError, i as safeNow, l as safeErrorMessage, o as FlumeParseError, r as FlumeLogger, s as attempt, t as FlumeSource } from "./flume-source.js";
+import { a as safeNow, c as attempt, i as FlumeLogger, l as safeNormalizeError, s as FlumeParseError, t as FlumeSource, u as safeErrorMessage } from "./flume-source.js";
 import { t as FlumeHttpError } from "./http-error.js";
 import { t as safeJsonParse } from "./safe-json-parse.js";
 import { t as safeReadText } from "./safe-read-text.js";
@@ -38,7 +38,12 @@ var FlumeGitHubSeenCache = class {
 	has(id, updatedAt) {
 		return this.seen.get(id) === updatedAt;
 	}
+	/**
+	* 既存キーは delete → set で挿入順を末尾へ更新する (LRU)。
+	* これをしないと trim() が「長寿命で更新され続ける thread」から先に追い出す
+	*/
 	add(id, updatedAt) {
+		this.seen.delete(id);
 		this.seen.set(id, updatedAt);
 	}
 	trim() {
@@ -59,8 +64,11 @@ var FlumeGitHubSeenCache = class {
 const NOTIFICATIONS_URL = "https://api.github.com/notifications";
 const CONSECUTIVE_ERRORS_TO_DISCONNECT = 3;
 const SEEN_CACHE_MAX = 5e3;
+const MAX_PAGES = 1e3;
+const MAX_TIMER_DELAY_MS = 2147483647;
 /**
 * GitHub /notifications を条件付きポーリングする (ETag / Last-Modified)。
+* Link: rel="next" を MAX_PAGES まで辿って全ページを集約し、
 * 304 / X-Poll-Interval / レート制限を尊重し、stop() / abort() で in-flight 通信を打ち切る
 */
 var FlumeGitHubPoller = class {
@@ -69,6 +77,7 @@ var FlumeGitHubPoller = class {
 	cache = new FlumeGitHubSeenCache({ maxSize: SEEN_CACHE_MAX });
 	timer = null;
 	rateLimitTimer = null;
+	rateLimitPauseActive = false;
 	since = null;
 	etag = null;
 	lastModified = null;
@@ -76,6 +85,7 @@ var FlumeGitHubPoller = class {
 	isStoppedFlag = false;
 	inFlight = false;
 	consecutiveErrors = 0;
+	degraded = false;
 	effectiveIntervalSec;
 	controller = null;
 	constructor(props) {
@@ -108,7 +118,7 @@ var FlumeGitHubPoller = class {
 		const error = await this.poll();
 		if (error) return error;
 		if (this.isStoppedFlag) return null;
-		if (this.rateLimitTimer !== null) return null;
+		if (this.rateLimitPauseActive) return null;
 		this.scheduleInterval();
 		return null;
 	}
@@ -122,6 +132,7 @@ var FlumeGitHubPoller = class {
 		this.controller = null;
 		this.clearTimer();
 		this.clearRateLimitTimer();
+		this.rateLimitPauseActive = false;
 	}
 	scheduleInterval() {
 		this.clearTimer();
@@ -134,7 +145,7 @@ var FlumeGitHubPoller = class {
 					error
 				});
 			});
-		}, this.effectiveIntervalSec * 1e3));
+		}, Math.min(this.effectiveIntervalSec * 1e3, MAX_TIMER_DELAY_MS)));
 		if (intervalResult instanceof Error) {
 			this.log.error({
 				action: "poller.interval.schedule.error",
@@ -176,7 +187,114 @@ var FlumeGitHubPoller = class {
 			per_page: "50"
 		});
 		if (this.since) params.set("since", this.since);
-		const url = `${NOTIFICATIONS_URL}?${params}`;
+		const response = await this.fetchPage(`${NOTIFICATIONS_URL}?${params}`, true);
+		if (this.isStoppedFlag) return null;
+		if (response instanceof Error) return this.recordFailure({
+			kind: "network",
+			message: response.message,
+			cause: response
+		});
+		this.followPollIntervalHeader(response.headers.get("X-Poll-Interval"));
+		if (response.status === 304) {
+			this.consecutiveErrors = 0;
+			this.log.debug({
+				action: "poll.not-modified",
+				message: "304 Not Modified"
+			});
+			this.notifyRecoveredIfDegraded();
+			return null;
+		}
+		if (await this.isRateLimited(response)) {
+			this.handleRateLimit(response);
+			return null;
+		}
+		if (!response.ok) {
+			const error = new FlumeHttpError({
+				message: `HTTP ${response.status}`,
+				status: response.status
+			});
+			return this.recordFailure({
+				kind: "http",
+				message: safeErrorMessage({ error }),
+				error
+			});
+		}
+		const nextEtag = response.headers.get("ETag");
+		const nextLastModified = response.headers.get("Last-Modified");
+		const rawNotifications = await this.collectPages(response);
+		if (!Array.isArray(rawNotifications)) return rawNotifications;
+		this.processNotifications(rawNotifications);
+		this.consecutiveErrors = 0;
+		this.etag = nextEtag;
+		this.lastModified = nextLastModified;
+		this.advanceCursor();
+		this.notifyRecoveredIfDegraded();
+		return null;
+	}
+	/**
+	* 先頭ページの body を読み、Link: rel="next" を MAX_PAGES まで辿って
+	* 全ページの生 notifications を集約する。2 ページ目以降は無条件リクエスト
+	* (If-None-Match / If-Modified-Since なし)。失敗・停止・rate limit 時は
+	* Error | null を返し pollOnce がそのまま返す
+	*/
+	async collectPages(firstResponse) {
+		const rawNotifications = [];
+		let pageResponse = firstResponse;
+		let pageCount = 1;
+		while (true) {
+			const body = await this.readPageBody(pageResponse);
+			if (this.isStoppedFlag) return null;
+			if (body instanceof Error) return this.recordFailure({
+				kind: "http",
+				message: body.message,
+				error: body
+			});
+			if (body === null) return null;
+			for (const item of body) rawNotifications.push(item);
+			const nextUrl = this.findNextPageUrl(pageResponse.headers.get("Link"));
+			if (nextUrl === null) return rawNotifications;
+			if (pageCount >= MAX_PAGES) {
+				const error = new FlumeHttpError({
+					message: `pagination exceeded the safety limit of ${MAX_PAGES} pages`,
+					status: 0
+				});
+				return this.recordFailure({
+					kind: "http",
+					message: error.message,
+					error
+				});
+			}
+			const nextResponse = await this.fetchPage(nextUrl, false);
+			if (this.isStoppedFlag) return null;
+			if (nextResponse instanceof Error) return this.recordFailure({
+				kind: "network",
+				message: nextResponse.message,
+				cause: nextResponse
+			});
+			if (await this.isRateLimited(nextResponse)) {
+				this.handleRateLimit(nextResponse);
+				return null;
+			}
+			if (!nextResponse.ok) {
+				const error = new FlumeHttpError({
+					message: `HTTP ${nextResponse.status}`,
+					status: nextResponse.status
+				});
+				return this.recordFailure({
+					kind: "http",
+					message: safeErrorMessage({ error }),
+					error
+				});
+			}
+			pageResponse = nextResponse;
+			pageCount++;
+		}
+	}
+	/**
+	* 1 ページ分を fetch する。条件付きヘッダ (If-None-Match / If-Modified-Since) は
+	* 先頭ページのみ付与する
+	*/
+	async fetchPage(url, isFirstPage) {
 		this.log.debug({
 			action: "http.request",
 			message: `GET ${url}`
@@ -186,24 +304,19 @@ var FlumeGitHubPoller = class {
 			Accept: "application/vnd.github+json",
 			"X-GitHub-Api-Version": "2022-11-28"
 		};
-		if (this.etag) headers["If-None-Match"] = this.etag;
-		if (this.lastModified) headers["If-Modified-Since"] = this.lastModified;
+		if (isFirstPage && this.etag) headers["If-None-Match"] = this.etag;
+		if (isFirstPage && this.lastModified) headers["If-Modified-Since"] = this.lastModified;
 		const response = await attempt(() => this.props.deps.fetch(url, {
 			headers,
 			signal: this.controller?.signal
 		}));
-		if (this.isStoppedFlag) return null;
 		if (response instanceof Error) {
 			this.log.error({
 				action: "http.error",
 				message: safeErrorMessage({ error: response }),
 				error: response
 			});
-			return this.recordFailure({
-				kind: "network",
-				message: response.message,
-				cause: response
-			});
+			return response;
 		}
 		this.log.debug({
 			action: "http.response",
@@ -213,49 +326,24 @@ var FlumeGitHubPoller = class {
 				url
 			}
 		});
-		this.maybeWidenInterval(response.headers.get("X-Poll-Interval"));
-		if (response.status === 304) {
-			this.consecutiveErrors = 0;
-			this.log.debug({
-				action: "poll.not-modified",
-				message: "304 Not Modified"
-			});
-			return null;
-		}
-		if (this.isRateLimited(response)) {
-			this.handleRateLimit(response);
-			return null;
-		}
-		if (!response.ok) {
-			const error = new FlumeHttpError({
-				message: `HTTP ${response.status}`,
-				status: response.status
-			});
-			return this.recordFailure({
-				kind: "http",
-				message: safeErrorMessage({ error }),
-				error
-			});
-		}
-		this.consecutiveErrors = 0;
-		this.etag = response.headers.get("ETag");
-		this.lastModified = response.headers.get("Last-Modified");
+		return response;
+	}
+	/**
+	* body を読んで JSON 配列に解決する。読み取り失敗は FlumeHttpError (呼び出し側で
+	* recordFailure する)、JSON 破損・非配列は warn 済みの null (poll ごと破棄) を返す
+	*/
+	async readPageBody(response) {
 		const text = await safeReadText({
 			response,
 			context: "notifications"
 		});
-		if (this.isStoppedFlag) return null;
 		if (text instanceof FlumeHttpError) {
 			this.log.warn({
 				action: "http.body.read",
 				message: safeErrorMessage({ error: text }),
 				error: text
 			});
-			return this.recordFailure({
-				kind: "http",
-				message: text.message,
-				error: text
-			});
+			return text;
 		}
 		const json = safeJsonParse(text);
 		if (json instanceof FlumeParseError) {
@@ -274,7 +362,15 @@ var FlumeGitHubPoller = class {
 			});
 			return null;
 		}
-		this.processNotifications(json);
+		return json;
+	}
+	findNextPageUrl(linkHeader) {
+		if (linkHeader === null) return null;
+		for (const part of linkHeader.split(",")) {
+			if (!part.includes("rel=\"next\"")) continue;
+			const url = part.match(/<([^>]+)>/)?.[1];
+			if (url !== void 0) return url;
+		}
 		return null;
 	}
 	recordFailure(input) {
@@ -290,14 +386,40 @@ var FlumeGitHubPoller = class {
 			error,
 			detail: { consecutiveErrors: this.consecutiveErrors }
 		});
-		if (this.consecutiveErrors >= CONSECUTIVE_ERRORS_TO_DISCONNECT && !this.isStoppedFlag) this.props.onDisconnected(input.kind === "network" ? "network error" : input.message);
+		if (this.consecutiveErrors >= CONSECUTIVE_ERRORS_TO_DISCONNECT && !this.isStoppedFlag) {
+			this.degraded = true;
+			this.props.onDisconnected(input.kind === "network" ? "network error" : input.message);
+		}
 		if (!this.bootstrapped) return error;
 		return null;
 	}
-	isRateLimited(response) {
+	/**
+	* onDisconnected 通知後に poll が完全成功したら connected へ戻す。
+	* degraded でなければ何もしない (冪等)
+	*/
+	notifyRecoveredIfDegraded() {
+		if (!this.degraded) return;
+		this.degraded = false;
+		this.props.onConnected();
+	}
+	/**
+	* 429 は常に rate limit。403 は Retry-After 付き、X-RateLimit-Remaining が 0、
+	* または本文が secondary rate limit / abuse detection を示す場合
+	*/
+	async isRateLimited(response) {
 		if (response.status === 429) return true;
-		if (response.status === 403 && response.headers.get("X-RateLimit-Remaining") === "0") return true;
-		return false;
+		if (response.status !== 403) return false;
+		if (response.headers.get("Retry-After") !== null) return true;
+		if (response.headers.get("X-RateLimit-Remaining") === "0") return true;
+		const cloned = attempt(() => response.clone());
+		if (cloned instanceof Error) return false;
+		const body = await safeReadText({
+			response: cloned,
+			context: "rate limit response"
+		});
+		if (body instanceof Error) return false;
+		const normalized = body.toLowerCase();
+		return normalized.includes("secondary rate limit") || normalized.includes("abuse detection");
 	}
 	handleRateLimit(response) {
 		const retryAfter = response.headers.get("Retry-After");
@@ -316,11 +438,21 @@ var FlumeGitHubPoller = class {
 		});
 		this.clearTimer();
 		this.clearRateLimitTimer();
+		this.rateLimitPauseActive = true;
 		const timerResult = attempt(() => this.props.deps.setTimeout(() => {
 			this.rateLimitTimer = null;
+			this.rateLimitPauseActive = false;
 			if (this.isStoppedFlag) return;
 			this.scheduleInterval();
-		}, delaySec * 1e3));
+			this.poll().catch((err) => {
+				const error = safeNormalizeError({ value: err });
+				this.log.error({
+					action: "poll.unhandled",
+					message: safeErrorMessage({ error }),
+					error
+				});
+			});
+		}, Math.min(delaySec * 1e3, MAX_TIMER_DELAY_MS)));
 		if (timerResult instanceof Error) {
 			this.log.error({
 				action: "poller.rate-limit.schedule.error",
@@ -328,6 +460,7 @@ var FlumeGitHubPoller = class {
 				error: timerResult
 			});
 			this.rateLimitTimer = null;
+			if (!this.isStoppedFlag) this.props.onDisconnected("rate-limit timer scheduling rejected by runtime");
 		} else this.rateLimitTimer = timerResult;
 	}
 	clearTimer() {
@@ -352,19 +485,25 @@ var FlumeGitHubPoller = class {
 		});
 		this.rateLimitTimer = null;
 	}
-	maybeWidenInterval(headerValue) {
+	/**
+	* X-Poll-Interval を双方向に追従する。下限はユーザー指定 interval。
+	* ヘッダ欠落・非数値は現在の実効値を維持する
+	*/
+	followPollIntervalHeader(headerValue) {
 		if (headerValue === null) return;
-		const required = Number.parseInt(headerValue, 10);
-		if (!Number.isFinite(required) || required <= this.effectiveIntervalSec) return;
+		const headerSec = Number.parseInt(headerValue, 10);
+		if (!Number.isFinite(headerSec)) return;
+		const nextSec = Math.min(Math.max(this.props.interval, headerSec), Math.floor(MAX_TIMER_DELAY_MS / 1e3));
+		if (nextSec === this.effectiveIntervalSec) return;
 		this.log.info({
-			action: "poll.widen-interval",
-			message: `widening interval ${this.effectiveIntervalSec}s -> ${required}s per X-Poll-Interval`,
+			action: "poll.interval.follow",
+			message: `adjusting interval ${this.effectiveIntervalSec}s -> ${nextSec}s per X-Poll-Interval`,
 			detail: {
 				from: this.effectiveIntervalSec,
-				to: required
+				to: nextSec
 			}
 		});
-		this.effectiveIntervalSec = required;
+		this.effectiveIntervalSec = nextSec;
 		if (this.timer !== null) this.scheduleInterval();
 	}
 	processNotifications(raw) {
@@ -389,11 +528,11 @@ var FlumeGitHubPoller = class {
 		if (!this.bootstrapped) {
 			this.bootstrapped = true;
 			for (const notification of notifications) this.cache.add(notification.id, notification.updated_at);
-			this.advanceCursor();
 			this.log.info({
 				action: "poller.bootstrap",
 				message: `seeded ${notifications.length} existing notifications`
 			});
+			this.degraded = false;
 			this.props.onConnected();
 			return;
 		}
@@ -403,7 +542,6 @@ var FlumeGitHubPoller = class {
 			return true;
 		});
 		this.cache.trim();
-		this.advanceCursor();
 		if (fresh.length > 0) {
 			this.log.info({
 				action: "poll.fresh",
@@ -452,6 +590,8 @@ var FlumeGitHubPoller = class {
 };
 //#endregion
 //#region lib/github/github-source.ts
+const DEFAULT_POLL_INTERVAL_SEC = 60;
+const MAX_POLL_INTERVAL_SEC = Math.floor(2147483647 / 1e3);
 var FlumeGitHubSource = class extends FlumeSource {
 	options;
 	name = "github";
@@ -464,7 +604,7 @@ var FlumeGitHubSource = class extends FlumeSource {
 		this.setStatus("connecting");
 		this.poller = new FlumeGitHubPoller({
 			token: this.options.token,
-			interval: this.options.pollInterval ?? 60,
+			interval: this.getPollIntervalSec(),
 			onLog: ctx.log.handler,
 			deps: ctx.deps,
 			onNotifications: (notifications) => this.handleNotifications(ctx, notifications),
@@ -487,6 +627,16 @@ var FlumeGitHubSource = class extends FlumeSource {
 		this.poller?.stop();
 		this.poller = null;
 	}
+	/**
+	* pollInterval が非数値・非有限・0 以下の場合は既定値へフォールバックする
+	*/
+	getPollIntervalSec() {
+		const requested = this.options.pollInterval;
+		if (typeof requested !== "number") return DEFAULT_POLL_INTERVAL_SEC;
+		if (!Number.isFinite(requested)) return DEFAULT_POLL_INTERVAL_SEC;
+		if (requested <= 0) return DEFAULT_POLL_INTERVAL_SEC;
+		return Math.min(requested, MAX_POLL_INTERVAL_SEC);
+	}
 	handleNotifications(ctx, notifications) {
 		for (const notification of notifications) this.emit({
 			source: "github",

package/dist/http-error.js

@@ -1,10 +1,14 @@
 //#region lib/errors/http-error.ts
 var FlumeHttpError = class extends Error {
 	status;
+	code;
+	retryAfterMs;
 	constructor(props) {
 		super(props.message, props.cause === void 0 ? void 0 : { cause: props.cause });
 		this.name = "FlumeHttpError";
 		this.status = props.status;
+		this.code = props.code ?? null;
+		this.retryAfterMs = props.retryAfterMs ?? null;
 		Object.freeze(this);
 	}
 };