hazo_collect 0.2.7 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -54,7 +54,7 @@ import { QueryBuilder as QueryBuilder2, wrapResult as wrapResult2 } from "hazo_c
54
54
  import { generateRequestId as generateRequestId2, getCorrelationId, withContext } from "hazo_core";
55
55
 
56
56
  // src/runtime/node.ts
57
- import { createLogger } from "hazo_core";
57
+ import { createLogger, fetchWithRequestId as fetchWithRequestId3 } from "hazo_core";
58
58
 
59
59
  // src/sdk/http.ts
60
60
  import { fetchWithRequestId, sleep } from "hazo_core";
@@ -88,29 +88,278 @@ function createHttpHelper(_requestId, fetchFn = fetchWithRequestId) {
88
88
  };
89
89
  }
90
90
 
91
- // src/runtime/node.ts
92
- var nodeRuntime = {
93
- async run(def, envelope, adapter) {
94
- const writeAdapter = createWriteAdapter(adapter);
95
- const log = createLogger("hazo_collect").child({
96
- collector: def.manifest.name,
97
- runId: envelope.run_id
91
+ // src/server/rate-limit.ts
92
+ import { fetchWithRequestId as fetchWithRequestId2, sleep as coreSleep } from "hazo_core";
93
+ function computeTokenBucket(state, cfg, cost, nowMs) {
94
+ const elapsedSec = Math.max(0, (nowMs - state.lastRefillAtMs) / 1e3);
95
+ const refilled = Math.min(cfg.burst, state.tokensAvailable + elapsedSec * cfg.requestsPerSecond);
96
+ const inBackoff = state.backoffUntilMs != null && state.backoffUntilMs > nowMs;
97
+ const allowed = !inBackoff && refilled >= cost;
98
+ const tokensAvailable = allowed ? refilled - cost : refilled;
99
+ let waitMs;
100
+ if (allowed) {
101
+ waitMs = 0;
102
+ } else if (inBackoff) {
103
+ waitMs = Math.max(0, state.backoffUntilMs - nowMs);
104
+ } else if (cfg.requestsPerSecond > 0) {
105
+ const deficit = cost - refilled;
106
+ waitMs = Math.ceil(deficit / cfg.requestsPerSecond * 1e3);
107
+ } else {
108
+ waitMs = Number.MAX_SAFE_INTEGER;
109
+ }
110
+ return { allowed, tokensAvailable, waitMs };
111
+ }
112
+ var DEFAULT_BACKOFF_MS = 1e3;
113
+ function parseRetryAfterMs(headerValue, nowMs) {
114
+ if (!headerValue) return null;
115
+ const asSeconds = Number(headerValue);
116
+ if (Number.isFinite(asSeconds)) return Math.max(0, asSeconds * 1e3);
117
+ const asDateMs = Date.parse(headerValue);
118
+ if (!Number.isNaN(asDateMs)) return Math.max(0, asDateMs - nowMs);
119
+ return null;
120
+ }
121
+ function hasRaw(adapter) {
122
+ return typeof adapter.raw === "function";
123
+ }
124
+ async function detectDialect(adapter) {
125
+ if (hasRaw(adapter)) return "pg";
126
+ let cfg;
127
+ try {
128
+ cfg = await adapter.getConfig();
129
+ } catch {
130
+ cfg = void 0;
131
+ }
132
+ if (cfg && typeof cfg.base_url === "string") return "postgrest";
133
+ return "sqlite";
134
+ }
135
+ function createPgRateLimitStore(adapter) {
136
+ return {
137
+ async takeToken(host, cfg, cost) {
138
+ const rows = await adapter.raw(
139
+ "SELECT allowed, wait_ms FROM hazo_collect_take_token_rpc($1, $2, $3, $4)",
140
+ [host, cfg.requestsPerSecond, cfg.burst, cost]
141
+ );
142
+ const row = rows[0];
143
+ return { allowed: Boolean(row.allowed), waitMs: Number(row.wait_ms) };
144
+ },
145
+ async applyBackoff(host, backoffUntilIso, cfg) {
146
+ await adapter.raw("SELECT hazo_collect_apply_backoff_rpc($1, $2, $3, $4)", [
147
+ host,
148
+ backoffUntilIso,
149
+ cfg.requestsPerSecond,
150
+ cfg.burst
151
+ ]);
152
+ }
153
+ };
154
+ }
155
+ function createPostgrestRateLimitStore(adapter) {
156
+ return {
157
+ async takeToken(host, cfg, cost) {
158
+ const result = await adapter.rawQuery("/rpc/hazo_collect_take_token_rpc", {
159
+ method: "POST",
160
+ headers: { "Content-Type": "application/json" },
161
+ body: JSON.stringify({
162
+ p_host: host,
163
+ p_requests_per_second: cfg.requestsPerSecond,
164
+ p_burst: cfg.burst,
165
+ p_cost: cost
166
+ })
167
+ });
168
+ const row = Array.isArray(result) ? result[0] : result;
169
+ return { allowed: Boolean(row.allowed), waitMs: Number(row.wait_ms) };
170
+ },
171
+ async applyBackoff(host, backoffUntilIso, cfg) {
172
+ await adapter.rawQuery("/rpc/hazo_collect_apply_backoff_rpc", {
173
+ method: "POST",
174
+ headers: { "Content-Type": "application/json" },
175
+ body: JSON.stringify({
176
+ p_host: host,
177
+ p_backoff_until: backoffUntilIso,
178
+ p_requests_per_second: cfg.requestsPerSecond,
179
+ p_burst: cfg.burst
180
+ })
181
+ });
182
+ }
183
+ };
184
+ }
185
+ var RATE_LIMIT_TABLE = "hazo_collect_rate_limits";
186
+ function createSqliteRateLimitStore(adapter, maxAttempts = 50) {
187
+ async function readRow(host) {
188
+ const rows = await adapter.rawQuery(`SELECT * FROM ${RATE_LIMIT_TABLE} WHERE host = ?`, {
189
+ params: [host]
98
190
  });
99
- const http = createHttpHelper(envelope.run_id);
100
- const ctx = {
101
- envelope,
102
- write: writeAdapter.write.bind(writeAdapter),
103
- http,
104
- log
105
- };
106
- try {
107
- const outcome = await def.run(ctx);
108
- return { ok: true, outcome };
109
- } catch (error) {
110
- return { ok: false, error };
191
+ return rows[0];
192
+ }
193
+ async function seedRow(host, cfg, tokens, nowIso) {
194
+ await adapter.rawQuery(
195
+ `INSERT OR IGNORE INTO ${RATE_LIMIT_TABLE}
196
+ (host, requests_per_second, burst, tokens_available, last_refill_at, updated_at)
197
+ VALUES (?, ?, ?, ?, ?, ?)`,
198
+ { params: [host, cfg.requestsPerSecond, cfg.burst, tokens, nowIso, nowIso] }
199
+ );
200
+ }
201
+ return {
202
+ async takeToken(host, cfg, cost) {
203
+ for (let attempt = 0; attempt < maxAttempts; attempt++) {
204
+ const existing = await readRow(host);
205
+ const nowMs = Date.now();
206
+ const nowIso = new Date(nowMs).toISOString();
207
+ if (!existing) {
208
+ await seedRow(host, cfg, cfg.burst, nowIso);
209
+ continue;
210
+ }
211
+ const computed = computeTokenBucket(
212
+ {
213
+ tokensAvailable: Number(existing.tokens_available),
214
+ lastRefillAtMs: Date.parse(existing.last_refill_at),
215
+ backoffUntilMs: existing.backoff_until ? Date.parse(existing.backoff_until) : null
216
+ },
217
+ cfg,
218
+ cost,
219
+ nowMs
220
+ );
221
+ const updated = await adapter.rawQuery(
222
+ `UPDATE ${RATE_LIMIT_TABLE}
223
+ SET tokens_available = ?, last_refill_at = ?, requests_per_second = ?, burst = ?, updated_at = ?
224
+ WHERE host = ?
225
+ AND last_refill_at = ?
226
+ AND IFNULL(backoff_until, '') = IFNULL(?, '')
227
+ RETURNING tokens_available`,
228
+ {
229
+ params: [
230
+ computed.tokensAvailable,
231
+ nowIso,
232
+ cfg.requestsPerSecond,
233
+ cfg.burst,
234
+ nowIso,
235
+ host,
236
+ existing.last_refill_at,
237
+ existing.backoff_until
238
+ ]
239
+ }
240
+ );
241
+ if (!updated.length) continue;
242
+ return { allowed: computed.allowed, waitMs: computed.waitMs };
243
+ }
244
+ throw new Error(`hazo_collect: exhausted retries taking a rate-limit token for host "${host}"`);
245
+ },
246
+ async applyBackoff(host, backoffUntilIso, cfg) {
247
+ const nowIso = (/* @__PURE__ */ new Date()).toISOString();
248
+ await seedRow(host, cfg, 0, nowIso);
249
+ await adapter.rawQuery(
250
+ `UPDATE ${RATE_LIMIT_TABLE}
251
+ SET tokens_available = 0,
252
+ backoff_until = CASE
253
+ WHEN backoff_until IS NULL OR backoff_until < ? THEN ?
254
+ ELSE backoff_until
255
+ END,
256
+ updated_at = ?
257
+ WHERE host = ?`,
258
+ { params: [backoffUntilIso, backoffUntilIso, nowIso, host] }
259
+ );
111
260
  }
261
+ };
262
+ }
263
+ async function createRateLimitStore(adapter) {
264
+ const dialect = await detectDialect(adapter);
265
+ switch (dialect) {
266
+ case "pg":
267
+ return createPgRateLimitStore(adapter);
268
+ case "postgrest":
269
+ return createPostgrestRateLimitStore(adapter);
270
+ case "sqlite":
271
+ return createSqliteRateLimitStore(adapter);
112
272
  }
113
- };
273
+ }
274
+ function hostnameOf(url) {
275
+ try {
276
+ const raw = url instanceof Request ? url.url : url;
277
+ return (typeof raw === "string" ? new URL(raw) : raw).hostname;
278
+ } catch {
279
+ return String(url);
280
+ }
281
+ }
282
+ function createRateLimitManager(opts) {
283
+ const hosts = opts.hosts ?? {};
284
+ const fetchFn = opts.fetchFn ?? fetchWithRequestId2;
285
+ const now = opts.now ?? (() => Date.now());
286
+ const sleepFn = opts.sleep ?? coreSleep;
287
+ const maxWaitMs = opts.maxWaitMs ?? 6e4;
288
+ let storePromise;
289
+ function getStore() {
290
+ if (opts.store) return Promise.resolve(opts.store);
291
+ if (!storePromise) storePromise = createRateLimitStore(opts.adapter);
292
+ return storePromise;
293
+ }
294
+ return {
295
+ createScopedFetch(waitTracker) {
296
+ return async (url, init) => {
297
+ const host = hostnameOf(url);
298
+ const cfg = hosts[host];
299
+ if (!cfg) return fetchFn(url, init);
300
+ const store = await getStore();
301
+ let waitedMs = 0;
302
+ while (true) {
303
+ const { allowed, waitMs } = await store.takeToken(host, cfg, 1);
304
+ if (allowed) break;
305
+ const sleepMs = Math.max(1, waitMs);
306
+ waitedMs += sleepMs;
307
+ if (waitedMs > maxWaitMs) break;
308
+ await sleepFn(sleepMs);
309
+ }
310
+ if (waitTracker) waitTracker.totalWaitMs += waitedMs;
311
+ const response = await fetchFn(url, init);
312
+ if (response.status === 429) {
313
+ const retryAfter = parseRetryAfterMs(response.headers?.get?.("Retry-After") ?? null, now());
314
+ const backoffUntilIso = new Date(now() + (retryAfter ?? DEFAULT_BACKOFF_MS)).toISOString();
315
+ await store.applyBackoff(host, backoffUntilIso, cfg);
316
+ }
317
+ return response;
318
+ };
319
+ }
320
+ };
321
+ }
322
+
323
+ // src/runtime/node.ts
324
+ function createNodeRuntime(options = {}) {
325
+ const baseFetch = options.fetch ?? fetchWithRequestId3;
326
+ const hosts = options.rateLimits ?? {};
327
+ const rateLimitingEnabled = Object.keys(hosts).length > 0;
328
+ const managers = /* @__PURE__ */ new WeakMap();
329
+ function managerFor(adapter) {
330
+ let manager = managers.get(adapter);
331
+ if (!manager) {
332
+ manager = createRateLimitManager({ adapter, hosts, fetchFn: baseFetch });
333
+ managers.set(adapter, manager);
334
+ }
335
+ return manager;
336
+ }
337
+ return {
338
+ async run(def, envelope, adapter) {
339
+ const writeAdapter = createWriteAdapter(adapter);
340
+ const log = createLogger("hazo_collect").child({
341
+ collector: def.manifest.name,
342
+ runId: envelope.run_id
343
+ });
344
+ const waitTracker = { totalWaitMs: 0 };
345
+ const scopedFetch = rateLimitingEnabled ? managerFor(adapter).createScopedFetch(waitTracker) : baseFetch;
346
+ const http = createHttpHelper(envelope.run_id, scopedFetch);
347
+ const ctx = {
348
+ envelope,
349
+ write: writeAdapter.write.bind(writeAdapter),
350
+ http,
351
+ log
352
+ };
353
+ try {
354
+ const outcome = await def.run(ctx);
355
+ return { ok: true, outcome, rateLimitWaitMs: waitTracker.totalWaitMs };
356
+ } catch (error) {
357
+ return { ok: false, error, rateLimitWaitMs: waitTracker.totalWaitMs };
358
+ }
359
+ }
360
+ };
361
+ }
362
+ var nodeRuntime = createNodeRuntime();
114
363
 
115
364
  // src/manager/index.ts
116
365
  async function buildDefaultSecretsProvider() {
@@ -193,7 +442,7 @@ async function upsertWatermark(adapter, plugin, watermark) {
193
442
  updated_at: nowISO()
194
443
  });
195
444
  }
196
- async function upsertHealth(adapter, plugin, status, now) {
445
+ async function upsertHealth(adapter, plugin, status, now, rateLimitWaitMs) {
197
446
  if (status === "success") {
198
447
  const qb = new QueryBuilder2().from("hazo_collect_plugin_health").onConflict(["plugin"]).doUpdate();
199
448
  await adapter.query(qb, "POST", {
@@ -202,7 +451,8 @@ async function upsertHealth(adapter, plugin, status, now) {
202
451
  last_run_at: now,
203
452
  last_status: "success",
204
453
  consecutive_failures: 0,
205
- state: "ok"
454
+ state: "ok",
455
+ last_rate_limit_wait_ms: rateLimitWaitMs
206
456
  });
207
457
  return;
208
458
  }
@@ -213,7 +463,8 @@ async function upsertHealth(adapter, plugin, status, now) {
213
463
  plugin,
214
464
  last_run_at: now,
215
465
  last_status: status,
216
- consecutive_failures: current + 1
466
+ consecutive_failures: current + 1,
467
+ last_rate_limit_wait_ms: rateLimitWaitMs
217
468
  };
218
469
  if (status === "failed") {
219
470
  upsertRow.state = "failed";
@@ -223,7 +474,7 @@ async function upsertHealth(adapter, plugin, status, now) {
223
474
  }
224
475
  function createManager(opts) {
225
476
  const { adapter, registry } = opts;
226
- const runtime = opts.runtime ?? nodeRuntime;
477
+ const runtime = opts.runtime ?? (opts.fetch || opts.rateLimits ? createNodeRuntime({ fetch: opts.fetch, rateLimits: opts.rateLimits }) : nodeRuntime);
227
478
  let _secretsProviderPromise;
228
479
  function getSecretsProvider() {
229
480
  if (opts.secrets) return Promise.resolve(opts.secrets);
@@ -243,6 +494,7 @@ function createManager(opts) {
243
494
  started_at
244
495
  });
245
496
  const dispatchResult = await runtime.run(entry.worker, envelope, adapter);
497
+ const rateLimitWaitMs = dispatchResult.rateLimitWaitMs ?? 0;
246
498
  let result;
247
499
  if (dispatchResult.ok) {
248
500
  const outcome = dispatchResult.outcome;
@@ -283,7 +535,7 @@ function createManager(opts) {
283
535
  if (result.status === "success" && result.watermark) {
284
536
  await upsertWatermark(adapter, envelope.plugin, result.watermark);
285
537
  }
286
- await upsertHealth(adapter, envelope.plugin, result.status, nowISO());
538
+ await upsertHealth(adapter, envelope.plugin, result.status, nowISO(), rateLimitWaitMs);
287
539
  return result;
288
540
  });
289
541
  }
@@ -470,10 +722,15 @@ async function persistRegistry(adapter, entries, opts) {
470
722
  });
471
723
  }
472
724
  export {
725
+ computeTokenBucket,
473
726
  createInMemoryRegistry,
474
727
  createManager,
728
+ createNodeRuntime,
729
+ createRateLimitManager,
730
+ createRateLimitStore,
475
731
  createWriteAdapter,
476
732
  discover,
477
733
  nodeRuntime,
734
+ parseRetryAfterMs,
478
735
  persistRegistry
479
736
  };
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "hazo_collect",
3
- "version": "0.2.7",
3
+ "version": "0.3.0",
4
4
  "description": "Collector-manager engine for the Ocdata platform",
5
5
  "type": "module",
6
6
  "module": "./dist/index.js",
@@ -27,6 +27,9 @@
27
27
  },
28
28
  "files": [
29
29
  "dist",
30
+ "config/hazo_collect_config.ini.sample",
31
+ "db_setup_postgres.sql",
32
+ "db_setup_sqlite.sql",
30
33
  "README.md",
31
34
  "CHANGE_LOG.md",
32
35
  "SETUP_CHECKLIST.md"
@@ -43,12 +46,13 @@
43
46
  "ajv": "^8.17.1"
44
47
  },
45
48
  "peerDependencies": {
46
- "hazo_core": "^2.0.0",
47
- "hazo_connect": "^4.0.0",
48
- "hazo_secure": "^1.7.1",
49
+ "hazo_core": "^2.5.0",
50
+ "hazo_connect": "^4.3.5",
51
+ "hazo_secure": "^1.9.1",
49
52
  "react": "^18.0.0 || ^19.0.0",
50
53
  "react-dom": "^18.0.0 || ^19.0.0",
51
- "hazo_ui": "^6.8.0"
54
+ "hazo_ui": "^6.21.0",
55
+ "hazo_i18n": "^0.17.0"
52
56
  },
53
57
  "peerDependenciesMeta": {
54
58
  "hazo_secure": {
@@ -62,6 +66,9 @@
62
66
  },
63
67
  "hazo_ui": {
64
68
  "optional": true
69
+ },
70
+ "hazo_i18n": {
71
+ "optional": true
65
72
  }
66
73
  },
67
74
  "devDependencies": {
@@ -74,9 +81,9 @@
74
81
  "@types/node": "^22.10.0",
75
82
  "@types/react": "^19.0.0",
76
83
  "@types/react-dom": "^19.0.0",
77
- "hazo_core": "^2.0.0",
78
- "hazo_connect": "^4.0.0",
79
- "hazo_secure": "^1.7.1"
84
+ "hazo_core": "^2.5.0",
85
+ "hazo_connect": "^4.3.5",
86
+ "hazo_secure": "^1.9.1"
80
87
  },
81
88
  "keywords": [],
82
89
  "author": "Pubs Abayasiri",