hazo_collect 0.2.7 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGE_LOG.md +93 -0
- package/README.md +70 -3
- package/config/hazo_collect_config.ini.sample +44 -0
- package/db_setup_postgres.sql +189 -0
- package/db_setup_sqlite.sql +87 -0
- package/dist/chunk-TORPW2KF.js +19 -0
- package/dist/ddl/postgres.sql +141 -6
- package/dist/ddl/sqlite.sql +23 -6
- package/dist/index.d.ts +2 -0
- package/dist/index.js +4 -0
- package/dist/lib/index.d.ts +14 -0
- package/dist/lib/index.js +6 -0
- package/dist/locales/en.json +1 -0
- package/dist/server/index.d.ts +108 -1
- package/dist/server/index.js +283 -26
- package/package.json +15 -8
package/dist/ddl/postgres.sql
CHANGED
|
@@ -8,6 +8,7 @@
|
|
|
8
8
|
-- hazo_collect_plugin_registry — discovered manifest catalogue
|
|
9
9
|
-- hazo_collect_watermarks — per-plugin incremental watermark
|
|
10
10
|
-- hazo_collect_landing — raw as-fetched payloads (idempotent on plugin+key)
|
|
11
|
+
-- hazo_collect_rate_limits — shared per-host token-bucket state (0.3.0+)
|
|
11
12
|
|
|
12
13
|
CREATE TABLE IF NOT EXISTS hazo_collect_plugin_runs (
|
|
13
14
|
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
|
@@ -34,14 +35,19 @@ CREATE INDEX IF NOT EXISTS hazo_collect_plugin_runs_run_id_idx
|
|
|
34
35
|
ON hazo_collect_plugin_runs (run_id);
|
|
35
36
|
|
|
36
37
|
CREATE TABLE IF NOT EXISTS hazo_collect_plugin_health (
|
|
37
|
-
plugin
|
|
38
|
-
last_success_at
|
|
39
|
-
last_run_at
|
|
40
|
-
last_status
|
|
41
|
-
state
|
|
42
|
-
consecutive_failures
|
|
38
|
+
plugin TEXT PRIMARY KEY,
|
|
39
|
+
last_success_at TIMESTAMPTZ,
|
|
40
|
+
last_run_at TIMESTAMPTZ,
|
|
41
|
+
last_status TEXT,
|
|
42
|
+
state TEXT NOT NULL DEFAULT 'ok' CHECK (state IN ('ok', 'stale', 'failed')),
|
|
43
|
+
consecutive_failures INT NOT NULL DEFAULT 0,
|
|
44
|
+
last_rate_limit_wait_ms INT NOT NULL DEFAULT 0
|
|
43
45
|
);
|
|
44
46
|
|
|
47
|
+
-- 0.3.0: existing deployments — add the new column if upgrading in place.
|
|
48
|
+
ALTER TABLE hazo_collect_plugin_health
|
|
49
|
+
ADD COLUMN IF NOT EXISTS last_rate_limit_wait_ms INT NOT NULL DEFAULT 0;
|
|
50
|
+
|
|
45
51
|
CREATE TABLE IF NOT EXISTS hazo_collect_plugin_registry (
|
|
46
52
|
name TEXT PRIMARY KEY,
|
|
47
53
|
kind TEXT NOT NULL CHECK (kind IN ('source', 'sink')),
|
|
@@ -75,4 +81,133 @@ CREATE TABLE IF NOT EXISTS hazo_collect_landing (
|
|
|
75
81
|
CREATE INDEX IF NOT EXISTS hazo_collect_landing_plugin_idx
|
|
76
82
|
ON hazo_collect_landing (plugin, fetched_at DESC);
|
|
77
83
|
|
|
84
|
+
-- ===========================================================================
|
|
85
|
+
-- Shared per-host rate limiting (0.3.0+)
|
|
86
|
+
--
|
|
87
|
+
-- hazo_collect_rate_limits holds one token-bucket row per host, shared by
|
|
88
|
+
-- every worker/process that collects through this database — ctx.http
|
|
89
|
+
-- enforces it transparently for hosts configured via ManagerOptions.rateLimits
|
|
90
|
+
-- (see createNodeRuntime / createRateLimitManager in src/server/rate-limit.ts).
|
|
91
|
+
--
|
|
92
|
+
-- hazo_collect_take_token_rpc is the atomic "take a token" operation. It is a
|
|
93
|
+
-- SINGLE UPDATE statement (the UPDATE ... FROM (... FOR UPDATE) subquery below
|
|
94
|
+
-- is one statement, not a separate SELECT then UPDATE over two round trips) so
|
|
95
|
+
-- no lock is ever held across requests — safe to call from PostgREST via
|
|
96
|
+
-- POST /rpc/hazo_collect_take_token_rpc, and safe under concurrent callers.
|
|
97
|
+
--
|
|
98
|
+
-- hazo_collect_apply_backoff_rpc records a 429 + Retry-After: it drains the
|
|
99
|
+
-- bucket and raises backoff_until so every caller sharing this host's bucket
|
|
100
|
+
-- (any process, not just the one that saw the 429) backs off too.
|
|
101
|
+
--
|
|
102
|
+
-- Consumers using the PostgREST path must GRANT EXECUTE on both functions to
|
|
103
|
+
-- the role PostgREST connects as (see README's "PostgREST setup" note).
|
|
104
|
+
-- ===========================================================================
|
|
105
|
+
|
|
106
|
+
CREATE TABLE IF NOT EXISTS hazo_collect_rate_limits (
|
|
107
|
+
host TEXT PRIMARY KEY,
|
|
108
|
+
requests_per_second DOUBLE PRECISION NOT NULL,
|
|
109
|
+
burst INT NOT NULL,
|
|
110
|
+
tokens_available DOUBLE PRECISION NOT NULL,
|
|
111
|
+
last_refill_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
|
112
|
+
backoff_until TIMESTAMPTZ,
|
|
113
|
+
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
|
|
114
|
+
);
|
|
115
|
+
|
|
116
|
+
DROP FUNCTION IF EXISTS hazo_collect_take_token_rpc(text, double precision, int, int);
|
|
117
|
+
|
|
118
|
+
CREATE OR REPLACE FUNCTION hazo_collect_take_token_rpc(
|
|
119
|
+
p_host text,
|
|
120
|
+
p_requests_per_second double precision,
|
|
121
|
+
p_burst int,
|
|
122
|
+
p_cost int DEFAULT 1
|
|
123
|
+
) RETURNS TABLE(allowed boolean, wait_ms int)
|
|
124
|
+
LANGUAGE plpgsql
|
|
125
|
+
SECURITY DEFINER
|
|
126
|
+
AS $$
|
|
127
|
+
DECLARE
|
|
128
|
+
now_ts timestamptz := clock_timestamp();
|
|
129
|
+
BEGIN
|
|
130
|
+
-- First use: seed the bucket at full burst. Race-safe — a concurrent
|
|
131
|
+
-- racer's insert simply loses (ON CONFLICT DO NOTHING) and the UPDATE
|
|
132
|
+
-- below re-reads whichever row won.
|
|
133
|
+
INSERT INTO hazo_collect_rate_limits
|
|
134
|
+
(host, requests_per_second, burst, tokens_available, last_refill_at, updated_at)
|
|
135
|
+
VALUES (p_host, p_requests_per_second, p_burst, p_burst, now_ts, now_ts)
|
|
136
|
+
ON CONFLICT (host) DO NOTHING;
|
|
137
|
+
|
|
138
|
+
RETURN QUERY
|
|
139
|
+
UPDATE hazo_collect_rate_limits r
|
|
140
|
+
SET tokens_available = CASE
|
|
141
|
+
WHEN old.is_allowed THEN old.refilled - p_cost
|
|
142
|
+
ELSE old.refilled
|
|
143
|
+
END,
|
|
144
|
+
last_refill_at = now_ts,
|
|
145
|
+
requests_per_second = p_requests_per_second,
|
|
146
|
+
burst = p_burst,
|
|
147
|
+
updated_at = now_ts
|
|
148
|
+
FROM (
|
|
149
|
+
-- FOR UPDATE locks the row before computing refill/allowed, so a
|
|
150
|
+
-- concurrent caller blocks here (not just at the outer UPDATE) and,
|
|
151
|
+
-- once unblocked, re-reads the fresh committed row — no lost updates.
|
|
152
|
+
SELECT
|
|
153
|
+
host,
|
|
154
|
+
LEAST(
|
|
155
|
+
p_burst::double precision,
|
|
156
|
+
tokens_available + EXTRACT(EPOCH FROM (now_ts - last_refill_at)) * p_requests_per_second
|
|
157
|
+
) AS refilled,
|
|
158
|
+
backoff_until,
|
|
159
|
+
(backoff_until IS NULL OR backoff_until <= now_ts)
|
|
160
|
+
AND LEAST(
|
|
161
|
+
p_burst::double precision,
|
|
162
|
+
tokens_available + EXTRACT(EPOCH FROM (now_ts - last_refill_at)) * p_requests_per_second
|
|
163
|
+
) >= p_cost AS is_allowed
|
|
164
|
+
FROM hazo_collect_rate_limits
|
|
165
|
+
WHERE host = p_host
|
|
166
|
+
FOR UPDATE
|
|
167
|
+
) AS old
|
|
168
|
+
WHERE r.host = old.host
|
|
169
|
+
RETURNING
|
|
170
|
+
old.is_allowed AS allowed,
|
|
171
|
+
CASE
|
|
172
|
+
WHEN old.is_allowed THEN 0
|
|
173
|
+
WHEN old.backoff_until IS NOT NULL AND old.backoff_until > now_ts
|
|
174
|
+
THEN CEIL(EXTRACT(EPOCH FROM (old.backoff_until - now_ts)) * 1000)::int
|
|
175
|
+
WHEN p_requests_per_second > 0
|
|
176
|
+
THEN CEIL((p_cost - old.refilled) / p_requests_per_second * 1000)::int
|
|
177
|
+
ELSE 2147483647
|
|
178
|
+
END AS wait_ms;
|
|
179
|
+
END;
|
|
180
|
+
$$;
|
|
181
|
+
|
|
182
|
+
COMMENT ON FUNCTION hazo_collect_take_token_rpc(text, double precision, int, int) IS
|
|
183
|
+
'hazo_collect: atomically take a rate-limit token for a host (refill-on-read token bucket). '
|
|
184
|
+
'Callable via PostgREST POST /rpc/hazo_collect_take_token_rpc. GRANT EXECUTE to the PostgREST role.';
|
|
185
|
+
|
|
186
|
+
DROP FUNCTION IF EXISTS hazo_collect_apply_backoff_rpc(text, timestamptz, double precision, int);
|
|
187
|
+
|
|
188
|
+
CREATE OR REPLACE FUNCTION hazo_collect_apply_backoff_rpc(
|
|
189
|
+
p_host text,
|
|
190
|
+
p_backoff_until timestamptz,
|
|
191
|
+
p_requests_per_second double precision,
|
|
192
|
+
p_burst int
|
|
193
|
+
) RETURNS void
|
|
194
|
+
LANGUAGE plpgsql
|
|
195
|
+
SECURITY DEFINER
|
|
196
|
+
AS $$
|
|
197
|
+
BEGIN
|
|
198
|
+
INSERT INTO hazo_collect_rate_limits
|
|
199
|
+
(host, requests_per_second, burst, tokens_available, last_refill_at, backoff_until, updated_at)
|
|
200
|
+
VALUES (p_host, p_requests_per_second, p_burst, 0, clock_timestamp(), p_backoff_until, clock_timestamp())
|
|
201
|
+
ON CONFLICT (host) DO UPDATE
|
|
202
|
+
SET tokens_available = 0,
|
|
203
|
+
backoff_until = GREATEST(hazo_collect_rate_limits.backoff_until, EXCLUDED.backoff_until),
|
|
204
|
+
updated_at = clock_timestamp();
|
|
205
|
+
END;
|
|
206
|
+
$$;
|
|
207
|
+
|
|
208
|
+
COMMENT ON FUNCTION hazo_collect_apply_backoff_rpc(text, timestamptz, double precision, int) IS
|
|
209
|
+
'hazo_collect: record a 429 + Retry-After for a host — drains the shared bucket and raises '
|
|
210
|
+
'backoff_until so every caller sharing this host backs off. Callable via PostgREST POST '
|
|
211
|
+
'/rpc/hazo_collect_apply_backoff_rpc. GRANT EXECUTE to the PostgREST role.';
|
|
212
|
+
|
|
78
213
|
NOTIFY pgrst, 'reload schema';
|
package/dist/ddl/sqlite.sql
CHANGED
|
@@ -10,6 +10,12 @@
|
|
|
10
10
|
-- hazo_collect_plugin_registry
|
|
11
11
|
-- hazo_collect_watermarks
|
|
12
12
|
-- hazo_collect_landing
|
|
13
|
+
-- hazo_collect_rate_limits — shared per-host token-bucket state (0.3.0+).
|
|
14
|
+
-- SQLite has no RPC/plpgsql, so the atomic "take a token" operation is
|
|
15
|
+
-- an optimistic compare-and-swap retry loop over this table (see
|
|
16
|
+
-- src/server/rate-limit.ts, createSqliteRateLimitStore) instead of a
|
|
17
|
+
-- stored function — same technique hazo_jobs uses for its SQLite
|
|
18
|
+
-- group-completion CAS (src/server/group-tracker.ts).
|
|
13
19
|
|
|
14
20
|
CREATE TABLE IF NOT EXISTS hazo_collect_plugin_runs (
|
|
15
21
|
id TEXT PRIMARY KEY,
|
|
@@ -33,12 +39,13 @@ CREATE INDEX IF NOT EXISTS hazo_collect_plugin_runs_plugin_idx
|
|
|
33
39
|
ON hazo_collect_plugin_runs (plugin, started_at DESC);
|
|
34
40
|
|
|
35
41
|
CREATE TABLE IF NOT EXISTS hazo_collect_plugin_health (
|
|
36
|
-
plugin
|
|
37
|
-
last_success_at
|
|
38
|
-
last_run_at
|
|
39
|
-
last_status
|
|
40
|
-
state
|
|
41
|
-
consecutive_failures
|
|
42
|
+
plugin TEXT PRIMARY KEY,
|
|
43
|
+
last_success_at TEXT,
|
|
44
|
+
last_run_at TEXT,
|
|
45
|
+
last_status TEXT,
|
|
46
|
+
state TEXT NOT NULL DEFAULT 'ok' CHECK (state IN ('ok', 'stale', 'failed')),
|
|
47
|
+
consecutive_failures INTEGER NOT NULL DEFAULT 0,
|
|
48
|
+
last_rate_limit_wait_ms INTEGER NOT NULL DEFAULT 0
|
|
42
49
|
);
|
|
43
50
|
|
|
44
51
|
CREATE TABLE IF NOT EXISTS hazo_collect_plugin_registry (
|
|
@@ -73,3 +80,13 @@ CREATE TABLE IF NOT EXISTS hazo_collect_landing (
|
|
|
73
80
|
|
|
74
81
|
CREATE INDEX IF NOT EXISTS hazo_collect_landing_plugin_idx
|
|
75
82
|
ON hazo_collect_landing (plugin, fetched_at DESC);
|
|
83
|
+
|
|
84
|
+
CREATE TABLE IF NOT EXISTS hazo_collect_rate_limits (
|
|
85
|
+
host TEXT PRIMARY KEY,
|
|
86
|
+
requests_per_second REAL NOT NULL,
|
|
87
|
+
burst INTEGER NOT NULL,
|
|
88
|
+
tokens_available REAL NOT NULL,
|
|
89
|
+
last_refill_at TEXT NOT NULL DEFAULT (datetime('now')),
|
|
90
|
+
backoff_until TEXT,
|
|
91
|
+
updated_at TEXT NOT NULL DEFAULT (datetime('now'))
|
|
92
|
+
);
|
package/dist/index.d.ts
CHANGED
|
@@ -1,7 +1,9 @@
|
|
|
1
1
|
import { M as ManifestInput } from './run-envelope-BOV1BPah.js';
|
|
2
2
|
export { a as Manifest, b as ManifestSchema, R as RunEnvelope, c as RunEnvelopeSchema, p as parseManifest, d as parseRunEnvelope } from './run-envelope-BOV1BPah.js';
|
|
3
3
|
export { R as RunError, a as RunResult, b as RunResultSchema, p as parseRunResult } from './run-result-yw8Iqipg.js';
|
|
4
|
+
export { t } from './lib/index.js';
|
|
4
5
|
import 'zod';
|
|
6
|
+
import 'hazo_core/i18n';
|
|
5
7
|
|
|
6
8
|
declare const CONTRACT_VERSION = "1.0.0";
|
|
7
9
|
|
package/dist/index.js
CHANGED
|
@@ -6,6 +6,9 @@ import {
|
|
|
6
6
|
parseRunResult,
|
|
7
7
|
validateInputs
|
|
8
8
|
} from "./chunk-6RFFJN6X.js";
|
|
9
|
+
import {
|
|
10
|
+
t
|
|
11
|
+
} from "./chunk-TORPW2KF.js";
|
|
9
12
|
|
|
10
13
|
// src/schema/run-envelope.ts
|
|
11
14
|
import { z } from "zod";
|
|
@@ -41,5 +44,6 @@ export {
|
|
|
41
44
|
parseManifest,
|
|
42
45
|
parseRunEnvelope,
|
|
43
46
|
parseRunResult,
|
|
47
|
+
t,
|
|
44
48
|
validateInputs
|
|
45
49
|
};
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import * as hazo_core_i18n from 'hazo_core/i18n';
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Future-proofing plumbing only: hazo_collect is a "zero-string" package —
|
|
5
|
+
* it's a headless collector-manager engine with no rendered UI and no
|
|
6
|
+
* user-facing copy (all string literals in this package are internal error
|
|
7
|
+
* messages for plugin/collector authors, not end-user text). `en` is an
|
|
8
|
+
* empty flat catalog today. If this package ever grows a user-facing
|
|
9
|
+
* string, add it to `src/locales/en.json` and read it via `t('some.key')`
|
|
10
|
+
* instead of a literal.
|
|
11
|
+
*/
|
|
12
|
+
declare const t: hazo_core_i18n.TFunc;
|
|
13
|
+
|
|
14
|
+
export { t };
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{}
|
package/dist/server/index.d.ts
CHANGED
|
@@ -34,15 +34,108 @@ declare function persistRegistry(adapter: HazoConnectAdapter, entries: Array<{
|
|
|
34
34
|
reseed?: boolean;
|
|
35
35
|
}): Promise<DbResult<void>>;
|
|
36
36
|
|
|
37
|
+
/** Per-host token-bucket config. Hosts not present in the manager's `hosts`
|
|
38
|
+
* map keep today's unthrottled behaviour — this feature is strictly opt-in
|
|
39
|
+
* per host. */
|
|
40
|
+
interface HostRateLimitConfig {
|
|
41
|
+
requestsPerSecond: number;
|
|
42
|
+
burst: number;
|
|
43
|
+
}
|
|
44
|
+
type FetchFn = (url: RequestInfo | URL, init?: RequestInit) => Promise<Response>;
|
|
45
|
+
interface TakeTokenResult {
|
|
46
|
+
allowed: boolean;
|
|
47
|
+
/** Milliseconds the caller should wait before trying again when `allowed` is false. 0 when allowed. */
|
|
48
|
+
waitMs: number;
|
|
49
|
+
}
|
|
50
|
+
/**
|
|
51
|
+
* The DB-backed, atomic per-host token bucket. One implementation per
|
|
52
|
+
* hazo_connect adapter dialect — see `createRateLimitStore`.
|
|
53
|
+
*/
|
|
54
|
+
interface RateLimitStore {
|
|
55
|
+
takeToken(host: string, cfg: HostRateLimitConfig, cost: number): Promise<TakeTokenResult>;
|
|
56
|
+
applyBackoff(host: string, backoffUntilIso: string, cfg: HostRateLimitConfig): Promise<void>;
|
|
57
|
+
}
|
|
58
|
+
interface WaitTracker {
|
|
59
|
+
totalWaitMs: number;
|
|
60
|
+
}
|
|
61
|
+
interface TokenBucketState {
|
|
62
|
+
tokensAvailable: number;
|
|
63
|
+
lastRefillAtMs: number;
|
|
64
|
+
backoffUntilMs: number | null;
|
|
65
|
+
}
|
|
66
|
+
interface ComputedTokenBucket {
|
|
67
|
+
allowed: boolean;
|
|
68
|
+
tokensAvailable: number;
|
|
69
|
+
waitMs: number;
|
|
70
|
+
}
|
|
71
|
+
/**
|
|
72
|
+
* Refill-on-read token bucket. Always advances the notion of "last refill"
|
|
73
|
+
* to `nowMs` and lets tokens accrue (capped at `burst`) even while a backoff
|
|
74
|
+
* is active — a backoff means "don't send requests", not "stop the clock" —
|
|
75
|
+
* but `allowed` is gated by the backoff window regardless of token count.
|
|
76
|
+
*/
|
|
77
|
+
declare function computeTokenBucket(state: TokenBucketState, cfg: HostRateLimitConfig, cost: number, nowMs: number): ComputedTokenBucket;
|
|
78
|
+
/** Parses a `Retry-After` header value (seconds, or an HTTP-date) into a
|
|
79
|
+
* millisecond delay relative to `nowMs`. Returns null when missing/unparsable. */
|
|
80
|
+
declare function parseRetryAfterMs(headerValue: string | null | undefined, nowMs: number): number | null;
|
|
81
|
+
declare function createRateLimitStore(adapter: HazoConnectAdapter): Promise<RateLimitStore>;
|
|
82
|
+
interface CreateRateLimitManagerOptions {
|
|
83
|
+
adapter: HazoConnectAdapter;
|
|
84
|
+
/** Per-host config. A host absent from this map is never throttled. */
|
|
85
|
+
hosts: Record<string, HostRateLimitConfig>;
|
|
86
|
+
/** The real network fetch — injectable so consumers (and this package's
|
|
87
|
+
* own tests) can point `ctx.http` at a fake host while the limit still
|
|
88
|
+
* applies. Defaults to hazo_core's `fetchWithRequestId`. */
|
|
89
|
+
fetchFn?: FetchFn;
|
|
90
|
+
/** Testing seam: bypass dialect detection with a store directly (used by
|
|
91
|
+
* this package's own unit tests; consumers normally never pass this). */
|
|
92
|
+
store?: RateLimitStore;
|
|
93
|
+
now?: () => number;
|
|
94
|
+
sleep?: (ms: number) => Promise<void>;
|
|
95
|
+
/** Safety valve so a pathological config can't hang a run forever waiting
|
|
96
|
+
* on tokens. Default 60s. */
|
|
97
|
+
maxWaitMs?: number;
|
|
98
|
+
}
|
|
99
|
+
interface RateLimitManager {
|
|
100
|
+
/** Returns a fetch-compatible function enforcing the shared per-host
|
|
101
|
+
* bucket. Pass a `waitTracker` to accumulate how long this call spent
|
|
102
|
+
* waiting on the limit — used to populate collector health. */
|
|
103
|
+
createScopedFetch(waitTracker?: WaitTracker): FetchFn;
|
|
104
|
+
}
|
|
105
|
+
declare function createRateLimitManager(opts: CreateRateLimitManagerOptions): RateLimitManager;
|
|
106
|
+
|
|
37
107
|
interface RuntimeAdapter {
|
|
38
108
|
run(def: CollectorDefinition, envelope: RunEnvelope, adapter: HazoConnectAdapter): Promise<{
|
|
39
109
|
ok: true;
|
|
40
110
|
outcome: CollectorOutcome;
|
|
111
|
+
rateLimitWaitMs: number;
|
|
41
112
|
} | {
|
|
42
113
|
ok: false;
|
|
43
114
|
error: unknown;
|
|
115
|
+
rateLimitWaitMs: number;
|
|
44
116
|
}>;
|
|
45
117
|
}
|
|
118
|
+
interface NodeRuntimeOptions {
|
|
119
|
+
/**
|
|
120
|
+
* Injected fetch implementation used by every collector's `ctx.http`.
|
|
121
|
+
* Lets consumers (and this package's own tests) point `ctx.http` at a
|
|
122
|
+
* fake host while per-host rate limiting still applies. Defaults to
|
|
123
|
+
* hazo_core's `fetchWithRequestId`.
|
|
124
|
+
*/
|
|
125
|
+
fetch?: FetchFn;
|
|
126
|
+
/**
|
|
127
|
+
* Per-host token-bucket config (`requestsPerSecond` + `burst`). A host
|
|
128
|
+
* absent from this map keeps today's unthrottled behaviour.
|
|
129
|
+
*/
|
|
130
|
+
rateLimits?: Record<string, HostRateLimitConfig>;
|
|
131
|
+
}
|
|
132
|
+
/**
|
|
133
|
+
* Builds a `RuntimeAdapter`. `nodeRuntime` below is the default instance
|
|
134
|
+
* (no fetch override, no rate limits — identical to pre-0.3.0 behaviour).
|
|
135
|
+
* `createManager({ fetch, rateLimits })` calls this factory so the manager
|
|
136
|
+
* is the seam consumers configure instead of touching the runtime directly.
|
|
137
|
+
*/
|
|
138
|
+
declare function createNodeRuntime(options?: NodeRuntimeOptions): RuntimeAdapter;
|
|
46
139
|
declare const nodeRuntime: RuntimeAdapter;
|
|
47
140
|
|
|
48
141
|
interface SecretsProvider {
|
|
@@ -54,6 +147,20 @@ interface ManagerOptions {
|
|
|
54
147
|
registry: Registry;
|
|
55
148
|
secrets?: SecretsProvider;
|
|
56
149
|
runtime?: RuntimeAdapter;
|
|
150
|
+
/**
|
|
151
|
+
* Injected fetch implementation used by `ctx.http` in every run this
|
|
152
|
+
* manager dispatches — lets consumers (and tests) point collectors at a
|
|
153
|
+
* fake host while per-host rate limiting still applies. Ignored when
|
|
154
|
+
* `runtime` is also supplied (that runtime owns its own fetch wiring).
|
|
155
|
+
* Defaults to hazo_core's `fetchWithRequestId`.
|
|
156
|
+
*/
|
|
157
|
+
fetch?: FetchFn;
|
|
158
|
+
/**
|
|
159
|
+
* Per-host token-bucket rate limits (`requestsPerSecond` + `burst`)
|
|
160
|
+
* enforced inside `ctx.http`. A host absent from this map keeps today's
|
|
161
|
+
* unthrottled behaviour. Ignored when `runtime` is also supplied.
|
|
162
|
+
*/
|
|
163
|
+
rateLimits?: Record<string, HostRateLimitConfig>;
|
|
57
164
|
}
|
|
58
165
|
interface Manager {
|
|
59
166
|
runNow(pluginName: string, opts?: {
|
|
@@ -63,4 +170,4 @@ interface Manager {
|
|
|
63
170
|
}
|
|
64
171
|
declare function createManager(opts: ManagerOptions): Manager;
|
|
65
172
|
|
|
66
|
-
export { type DiscoverOptions, type Manager, type ManagerOptions, type Registry, type RegistryEntry, type RuntimeAdapter, createInMemoryRegistry, createManager, discover, nodeRuntime, persistRegistry };
|
|
173
|
+
export { type DiscoverOptions, type FetchFn, type HostRateLimitConfig, type Manager, type ManagerOptions, type NodeRuntimeOptions, type RateLimitManager, type RateLimitStore, type Registry, type RegistryEntry, type RuntimeAdapter, type TakeTokenResult, type WaitTracker, computeTokenBucket, createInMemoryRegistry, createManager, createNodeRuntime, createRateLimitManager, createRateLimitStore, discover, nodeRuntime, parseRetryAfterMs, persistRegistry };
|