hazo_collect 0.2.7 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -8,6 +8,7 @@
8
8
  -- hazo_collect_plugin_registry — discovered manifest catalogue
9
9
  -- hazo_collect_watermarks — per-plugin incremental watermark
10
10
  -- hazo_collect_landing — raw as-fetched payloads (idempotent on plugin+key)
11
+ -- hazo_collect_rate_limits — shared per-host token-bucket state (0.3.0+)
11
12
 
12
13
  CREATE TABLE IF NOT EXISTS hazo_collect_plugin_runs (
13
14
  id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
@@ -34,14 +35,19 @@ CREATE INDEX IF NOT EXISTS hazo_collect_plugin_runs_run_id_idx
34
35
  ON hazo_collect_plugin_runs (run_id);
35
36
 
36
37
  CREATE TABLE IF NOT EXISTS hazo_collect_plugin_health (
37
- plugin TEXT PRIMARY KEY,
38
- last_success_at TIMESTAMPTZ,
39
- last_run_at TIMESTAMPTZ,
40
- last_status TEXT,
41
- state TEXT NOT NULL DEFAULT 'ok' CHECK (state IN ('ok', 'stale', 'failed')),
42
- consecutive_failures INT NOT NULL DEFAULT 0
38
+ plugin TEXT PRIMARY KEY,
39
+ last_success_at TIMESTAMPTZ,
40
+ last_run_at TIMESTAMPTZ,
41
+ last_status TEXT,
42
+ state TEXT NOT NULL DEFAULT 'ok' CHECK (state IN ('ok', 'stale', 'failed')),
43
+ consecutive_failures INT NOT NULL DEFAULT 0,
44
+ last_rate_limit_wait_ms INT NOT NULL DEFAULT 0
43
45
  );
44
46
 
47
+ -- 0.3.0: existing deployments — add the new column if upgrading in place.
48
+ ALTER TABLE hazo_collect_plugin_health
49
+ ADD COLUMN IF NOT EXISTS last_rate_limit_wait_ms INT NOT NULL DEFAULT 0;
50
+
45
51
  CREATE TABLE IF NOT EXISTS hazo_collect_plugin_registry (
46
52
  name TEXT PRIMARY KEY,
47
53
  kind TEXT NOT NULL CHECK (kind IN ('source', 'sink')),
@@ -75,4 +81,133 @@ CREATE TABLE IF NOT EXISTS hazo_collect_landing (
75
81
  CREATE INDEX IF NOT EXISTS hazo_collect_landing_plugin_idx
76
82
  ON hazo_collect_landing (plugin, fetched_at DESC);
77
83
 
84
+ -- ===========================================================================
85
+ -- Shared per-host rate limiting (0.3.0+)
86
+ --
87
+ -- hazo_collect_rate_limits holds one token-bucket row per host, shared by
88
+ -- every worker/process that collects through this database — ctx.http
89
+ -- enforces it transparently for hosts configured via ManagerOptions.rateLimits
90
+ -- (see createNodeRuntime / createRateLimitManager in src/server/rate-limit.ts).
91
+ --
92
+ -- hazo_collect_take_token_rpc is the atomic "take a token" operation. It is a
93
+ -- SINGLE UPDATE statement (the UPDATE ... FROM (... FOR UPDATE) subquery below
94
+ -- is one statement, not a separate SELECT then UPDATE over two round trips) so
95
+ -- no lock is ever held across requests — safe to call from PostgREST via
96
+ -- POST /rpc/hazo_collect_take_token_rpc, and safe under concurrent callers.
97
+ --
98
+ -- hazo_collect_apply_backoff_rpc records a 429 + Retry-After: it drains the
99
+ -- bucket and raises backoff_until so every caller sharing this host's bucket
100
+ -- (any process, not just the one that saw the 429) backs off too.
101
+ --
102
+ -- Consumers using the PostgREST path must GRANT EXECUTE on both functions to
103
+ -- the role PostgREST connects as (see README's "PostgREST setup" note).
104
+ -- ===========================================================================
105
+
106
+ CREATE TABLE IF NOT EXISTS hazo_collect_rate_limits (
107
+ host TEXT PRIMARY KEY,
108
+ requests_per_second DOUBLE PRECISION NOT NULL,
109
+ burst INT NOT NULL,
110
+ tokens_available DOUBLE PRECISION NOT NULL,
111
+ last_refill_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
112
+ backoff_until TIMESTAMPTZ,
113
+ updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
114
+ );
115
+
116
+ DROP FUNCTION IF EXISTS hazo_collect_take_token_rpc(text, double precision, int, int);
117
+
118
+ CREATE OR REPLACE FUNCTION hazo_collect_take_token_rpc(
119
+ p_host text,
120
+ p_requests_per_second double precision,
121
+ p_burst int,
122
+ p_cost int DEFAULT 1
123
+ ) RETURNS TABLE(allowed boolean, wait_ms int)
124
+ LANGUAGE plpgsql
125
+ SECURITY DEFINER
126
+ AS $$
127
+ DECLARE
128
+ now_ts timestamptz := clock_timestamp();
129
+ BEGIN
130
+ -- First use: seed the bucket at full burst. Race-safe — a concurrent
131
+ -- racer's insert simply loses (ON CONFLICT DO NOTHING) and the UPDATE
132
+ -- below re-reads whichever row won.
133
+ INSERT INTO hazo_collect_rate_limits
134
+ (host, requests_per_second, burst, tokens_available, last_refill_at, updated_at)
135
+ VALUES (p_host, p_requests_per_second, p_burst, p_burst, now_ts, now_ts)
136
+ ON CONFLICT (host) DO NOTHING;
137
+
138
+ RETURN QUERY
139
+ UPDATE hazo_collect_rate_limits r
140
+ SET tokens_available = CASE
141
+ WHEN old.is_allowed THEN old.refilled - p_cost
142
+ ELSE old.refilled
143
+ END,
144
+ last_refill_at = now_ts,
145
+ requests_per_second = p_requests_per_second,
146
+ burst = p_burst,
147
+ updated_at = now_ts
148
+ FROM (
149
+ -- FOR UPDATE locks the row before computing refill/allowed, so a
150
+ -- concurrent caller blocks here (not just at the outer UPDATE) and,
151
+ -- once unblocked, re-reads the fresh committed row — no lost updates.
152
+ SELECT
153
+ host,
154
+ LEAST(
155
+ p_burst::double precision,
156
+ tokens_available + EXTRACT(EPOCH FROM (now_ts - last_refill_at)) * p_requests_per_second
157
+ ) AS refilled,
158
+ backoff_until,
159
+ (backoff_until IS NULL OR backoff_until <= now_ts)
160
+ AND LEAST(
161
+ p_burst::double precision,
162
+ tokens_available + EXTRACT(EPOCH FROM (now_ts - last_refill_at)) * p_requests_per_second
163
+ ) >= p_cost AS is_allowed
164
+ FROM hazo_collect_rate_limits
165
+ WHERE host = p_host
166
+ FOR UPDATE
167
+ ) AS old
168
+ WHERE r.host = old.host
169
+ RETURNING
170
+ old.is_allowed AS allowed,
171
+ CASE
172
+ WHEN old.is_allowed THEN 0
173
+ WHEN old.backoff_until IS NOT NULL AND old.backoff_until > now_ts
174
+ THEN CEIL(EXTRACT(EPOCH FROM (old.backoff_until - now_ts)) * 1000)::int
175
+ WHEN p_requests_per_second > 0
176
+ THEN CEIL((p_cost - old.refilled) / p_requests_per_second * 1000)::int
177
+ ELSE 2147483647
178
+ END AS wait_ms;
179
+ END;
180
+ $$;
181
+
182
+ COMMENT ON FUNCTION hazo_collect_take_token_rpc(text, double precision, int, int) IS
183
+ 'hazo_collect: atomically take a rate-limit token for a host (refill-on-read token bucket). '
184
+ 'Callable via PostgREST POST /rpc/hazo_collect_take_token_rpc. GRANT EXECUTE to the PostgREST role.';
185
+
186
+ DROP FUNCTION IF EXISTS hazo_collect_apply_backoff_rpc(text, timestamptz, double precision, int);
187
+
188
+ CREATE OR REPLACE FUNCTION hazo_collect_apply_backoff_rpc(
189
+ p_host text,
190
+ p_backoff_until timestamptz,
191
+ p_requests_per_second double precision,
192
+ p_burst int
193
+ ) RETURNS void
194
+ LANGUAGE plpgsql
195
+ SECURITY DEFINER
196
+ AS $$
197
+ BEGIN
198
+ INSERT INTO hazo_collect_rate_limits
199
+ (host, requests_per_second, burst, tokens_available, last_refill_at, backoff_until, updated_at)
200
+ VALUES (p_host, p_requests_per_second, p_burst, 0, clock_timestamp(), p_backoff_until, clock_timestamp())
201
+ ON CONFLICT (host) DO UPDATE
202
+ SET tokens_available = 0,
203
+ backoff_until = GREATEST(hazo_collect_rate_limits.backoff_until, EXCLUDED.backoff_until),
204
+ updated_at = clock_timestamp();
205
+ END;
206
+ $$;
207
+
208
+ COMMENT ON FUNCTION hazo_collect_apply_backoff_rpc(text, timestamptz, double precision, int) IS
209
+ 'hazo_collect: record a 429 + Retry-After for a host — drains the shared bucket and raises '
210
+ 'backoff_until so every caller sharing this host backs off. Callable via PostgREST POST '
211
+ '/rpc/hazo_collect_apply_backoff_rpc. GRANT EXECUTE to the PostgREST role.';
212
+
78
213
  NOTIFY pgrst, 'reload schema';
@@ -10,6 +10,12 @@
10
10
  -- hazo_collect_plugin_registry
11
11
  -- hazo_collect_watermarks
12
12
  -- hazo_collect_landing
13
+ -- hazo_collect_rate_limits — shared per-host token-bucket state (0.3.0+).
14
+ -- SQLite has no RPC/plpgsql, so the atomic "take a token" operation is
15
+ -- an optimistic compare-and-swap retry loop over this table (see
16
+ -- src/server/rate-limit.ts, createSqliteRateLimitStore) instead of a
17
+ -- stored function — same technique hazo_jobs uses for its SQLite
18
+ -- group-completion CAS (src/server/group-tracker.ts).
13
19
 
14
20
  CREATE TABLE IF NOT EXISTS hazo_collect_plugin_runs (
15
21
  id TEXT PRIMARY KEY,
@@ -33,12 +39,13 @@ CREATE INDEX IF NOT EXISTS hazo_collect_plugin_runs_plugin_idx
33
39
  ON hazo_collect_plugin_runs (plugin, started_at DESC);
34
40
 
35
41
  CREATE TABLE IF NOT EXISTS hazo_collect_plugin_health (
36
- plugin TEXT PRIMARY KEY,
37
- last_success_at TEXT,
38
- last_run_at TEXT,
39
- last_status TEXT,
40
- state TEXT NOT NULL DEFAULT 'ok' CHECK (state IN ('ok', 'stale', 'failed')),
41
- consecutive_failures INTEGER NOT NULL DEFAULT 0
42
+ plugin TEXT PRIMARY KEY,
43
+ last_success_at TEXT,
44
+ last_run_at TEXT,
45
+ last_status TEXT,
46
+ state TEXT NOT NULL DEFAULT 'ok' CHECK (state IN ('ok', 'stale', 'failed')),
47
+ consecutive_failures INTEGER NOT NULL DEFAULT 0,
48
+ last_rate_limit_wait_ms INTEGER NOT NULL DEFAULT 0
42
49
  );
43
50
 
44
51
  CREATE TABLE IF NOT EXISTS hazo_collect_plugin_registry (
@@ -73,3 +80,13 @@ CREATE TABLE IF NOT EXISTS hazo_collect_landing (
73
80
 
74
81
  CREATE INDEX IF NOT EXISTS hazo_collect_landing_plugin_idx
75
82
  ON hazo_collect_landing (plugin, fetched_at DESC);
83
+
84
+ CREATE TABLE IF NOT EXISTS hazo_collect_rate_limits (
85
+ host TEXT PRIMARY KEY,
86
+ requests_per_second REAL NOT NULL,
87
+ burst INTEGER NOT NULL,
88
+ tokens_available REAL NOT NULL,
89
+ last_refill_at TEXT NOT NULL DEFAULT (datetime('now')),
90
+ backoff_until TEXT,
91
+ updated_at TEXT NOT NULL DEFAULT (datetime('now'))
92
+ );
package/dist/index.d.ts CHANGED
@@ -1,7 +1,9 @@
1
1
  import { M as ManifestInput } from './run-envelope-BOV1BPah.js';
2
2
  export { a as Manifest, b as ManifestSchema, R as RunEnvelope, c as RunEnvelopeSchema, p as parseManifest, d as parseRunEnvelope } from './run-envelope-BOV1BPah.js';
3
3
  export { R as RunError, a as RunResult, b as RunResultSchema, p as parseRunResult } from './run-result-yw8Iqipg.js';
4
+ export { t } from './lib/index.js';
4
5
  import 'zod';
6
+ import 'hazo_core/i18n';
5
7
 
6
8
  declare const CONTRACT_VERSION = "1.0.0";
7
9
 
package/dist/index.js CHANGED
@@ -6,6 +6,9 @@ import {
6
6
  parseRunResult,
7
7
  validateInputs
8
8
  } from "./chunk-6RFFJN6X.js";
9
+ import {
10
+ t
11
+ } from "./chunk-TORPW2KF.js";
9
12
 
10
13
  // src/schema/run-envelope.ts
11
14
  import { z } from "zod";
@@ -41,5 +44,6 @@ export {
41
44
  parseManifest,
42
45
  parseRunEnvelope,
43
46
  parseRunResult,
47
+ t,
44
48
  validateInputs
45
49
  };
@@ -0,0 +1,14 @@
1
+ import * as hazo_core_i18n from 'hazo_core/i18n';
2
+
3
+ /**
4
+ * Future-proofing plumbing only: hazo_collect is a "zero-string" package —
5
+ * it's a headless collector-manager engine with no rendered UI and no
6
+ * user-facing copy (all string literals in this package are internal error
7
+ * messages for plugin/collector authors, not end-user text). `en` is an
8
+ * empty flat catalog today. If this package ever grows a user-facing
9
+ * string, add it to `src/locales/en.json` and read it via `t('some.key')`
10
+ * instead of a literal.
11
+ */
12
+ declare const t: hazo_core_i18n.TFunc;
13
+
14
+ export { t };
@@ -0,0 +1,6 @@
1
+ import {
2
+ t
3
+ } from "../chunk-TORPW2KF.js";
4
+ export {
5
+ t
6
+ };
@@ -0,0 +1 @@
1
+ {}
@@ -34,15 +34,108 @@ declare function persistRegistry(adapter: HazoConnectAdapter, entries: Array<{
34
34
  reseed?: boolean;
35
35
  }): Promise<DbResult<void>>;
36
36
 
37
+ /** Per-host token-bucket config. Hosts not present in the manager's `hosts`
38
+ * map keep today's unthrottled behaviour — this feature is strictly opt-in
39
+ * per host. */
40
+ interface HostRateLimitConfig {
41
+ requestsPerSecond: number;
42
+ burst: number;
43
+ }
44
+ type FetchFn = (url: RequestInfo | URL, init?: RequestInit) => Promise<Response>;
45
+ interface TakeTokenResult {
46
+ allowed: boolean;
47
+ /** Milliseconds the caller should wait before trying again when `allowed` is false. 0 when allowed. */
48
+ waitMs: number;
49
+ }
50
+ /**
51
+ * The DB-backed, atomic per-host token bucket. One implementation per
52
+ * hazo_connect adapter dialect — see `createRateLimitStore`.
53
+ */
54
+ interface RateLimitStore {
55
+ takeToken(host: string, cfg: HostRateLimitConfig, cost: number): Promise<TakeTokenResult>;
56
+ applyBackoff(host: string, backoffUntilIso: string, cfg: HostRateLimitConfig): Promise<void>;
57
+ }
58
+ interface WaitTracker {
59
+ totalWaitMs: number;
60
+ }
61
+ interface TokenBucketState {
62
+ tokensAvailable: number;
63
+ lastRefillAtMs: number;
64
+ backoffUntilMs: number | null;
65
+ }
66
+ interface ComputedTokenBucket {
67
+ allowed: boolean;
68
+ tokensAvailable: number;
69
+ waitMs: number;
70
+ }
71
+ /**
72
+ * Refill-on-read token bucket. Always advances the notion of "last refill"
73
+ * to `nowMs` and lets tokens accrue (capped at `burst`) even while a backoff
74
+ * is active — a backoff means "don't send requests", not "stop the clock" —
75
+ * but `allowed` is gated by the backoff window regardless of token count.
76
+ */
77
+ declare function computeTokenBucket(state: TokenBucketState, cfg: HostRateLimitConfig, cost: number, nowMs: number): ComputedTokenBucket;
78
+ /** Parses a `Retry-After` header value (seconds, or an HTTP-date) into a
79
+ * millisecond delay relative to `nowMs`. Returns null when missing/unparsable. */
80
+ declare function parseRetryAfterMs(headerValue: string | null | undefined, nowMs: number): number | null;
81
+ declare function createRateLimitStore(adapter: HazoConnectAdapter): Promise<RateLimitStore>;
82
+ interface CreateRateLimitManagerOptions {
83
+ adapter: HazoConnectAdapter;
84
+ /** Per-host config. A host absent from this map is never throttled. */
85
+ hosts: Record<string, HostRateLimitConfig>;
86
+ /** The real network fetch — injectable so consumers (and this package's
87
+ * own tests) can point `ctx.http` at a fake host while the limit still
88
+ * applies. Defaults to hazo_core's `fetchWithRequestId`. */
89
+ fetchFn?: FetchFn;
90
+ /** Testing seam: bypass dialect detection with a store directly (used by
91
+ * this package's own unit tests; consumers normally never pass this). */
92
+ store?: RateLimitStore;
93
+ now?: () => number;
94
+ sleep?: (ms: number) => Promise<void>;
95
+ /** Safety valve so a pathological config can't hang a run forever waiting
96
+ * on tokens. Default 60s. */
97
+ maxWaitMs?: number;
98
+ }
99
+ interface RateLimitManager {
100
+ /** Returns a fetch-compatible function enforcing the shared per-host
101
+ * bucket. Pass a `waitTracker` to accumulate how long this call spent
102
+ * waiting on the limit — used to populate collector health. */
103
+ createScopedFetch(waitTracker?: WaitTracker): FetchFn;
104
+ }
105
+ declare function createRateLimitManager(opts: CreateRateLimitManagerOptions): RateLimitManager;
106
+
37
107
  interface RuntimeAdapter {
38
108
  run(def: CollectorDefinition, envelope: RunEnvelope, adapter: HazoConnectAdapter): Promise<{
39
109
  ok: true;
40
110
  outcome: CollectorOutcome;
111
+ rateLimitWaitMs: number;
41
112
  } | {
42
113
  ok: false;
43
114
  error: unknown;
115
+ rateLimitWaitMs: number;
44
116
  }>;
45
117
  }
118
+ interface NodeRuntimeOptions {
119
+ /**
120
+ * Injected fetch implementation used by every collector's `ctx.http`.
121
+ * Lets consumers (and this package's own tests) point `ctx.http` at a
122
+ * fake host while per-host rate limiting still applies. Defaults to
123
+ * hazo_core's `fetchWithRequestId`.
124
+ */
125
+ fetch?: FetchFn;
126
+ /**
127
+ * Per-host token-bucket config (`requestsPerSecond` + `burst`). A host
128
+ * absent from this map keeps today's unthrottled behaviour.
129
+ */
130
+ rateLimits?: Record<string, HostRateLimitConfig>;
131
+ }
132
+ /**
133
+ * Builds a `RuntimeAdapter`. `nodeRuntime` below is the default instance
134
+ * (no fetch override, no rate limits — identical to pre-0.3.0 behaviour).
135
+ * `createManager({ fetch, rateLimits })` calls this factory so the manager
136
+ * is the seam consumers configure instead of touching the runtime directly.
137
+ */
138
+ declare function createNodeRuntime(options?: NodeRuntimeOptions): RuntimeAdapter;
46
139
  declare const nodeRuntime: RuntimeAdapter;
47
140
 
48
141
  interface SecretsProvider {
@@ -54,6 +147,20 @@ interface ManagerOptions {
54
147
  registry: Registry;
55
148
  secrets?: SecretsProvider;
56
149
  runtime?: RuntimeAdapter;
150
+ /**
151
+ * Injected fetch implementation used by `ctx.http` in every run this
152
+ * manager dispatches — lets consumers (and tests) point collectors at a
153
+ * fake host while per-host rate limiting still applies. Ignored when
154
+ * `runtime` is also supplied (that runtime owns its own fetch wiring).
155
+ * Defaults to hazo_core's `fetchWithRequestId`.
156
+ */
157
+ fetch?: FetchFn;
158
+ /**
159
+ * Per-host token-bucket rate limits (`requestsPerSecond` + `burst`)
160
+ * enforced inside `ctx.http`. A host absent from this map keeps today's
161
+ * unthrottled behaviour. Ignored when `runtime` is also supplied.
162
+ */
163
+ rateLimits?: Record<string, HostRateLimitConfig>;
57
164
  }
58
165
  interface Manager {
59
166
  runNow(pluginName: string, opts?: {
@@ -63,4 +170,4 @@ interface Manager {
63
170
  }
64
171
  declare function createManager(opts: ManagerOptions): Manager;
65
172
 
66
- export { type DiscoverOptions, type Manager, type ManagerOptions, type Registry, type RegistryEntry, type RuntimeAdapter, createInMemoryRegistry, createManager, discover, nodeRuntime, persistRegistry };
173
+ export { type DiscoverOptions, type FetchFn, type HostRateLimitConfig, type Manager, type ManagerOptions, type NodeRuntimeOptions, type RateLimitManager, type RateLimitStore, type Registry, type RegistryEntry, type RuntimeAdapter, type TakeTokenResult, type WaitTracker, computeTokenBucket, createInMemoryRegistry, createManager, createNodeRuntime, createRateLimitManager, createRateLimitStore, discover, nodeRuntime, parseRetryAfterMs, persistRegistry };