bulltrackers-module 1.0.155 → 1.0.156

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (3) hide show
  1. package/functions/core/utils/intelligent_proxy_manager.js +13 -66
  2. package/index.js +42 -98
  3. package/package.json +1 -1
package/functions/core/utils/intelligent_proxy_manager.js CHANGED
@@ -6,8 +6,6 @@
6
6
  * --- MODIFIED: Now includes exponential backoff and retries specifically for rate-limit errors. ---
7
7
  */
8
8
  const { FieldValue } = require('@google-cloud/firestore');
9
-
10
- // --- NEW: Added sleep utility ---
11
9
  const sleep = (ms) => new Promise(resolve => setTimeout(resolve, ms));
12
10
 
13
11
  class IntelligentProxyManager {
@@ -33,11 +31,8 @@ class IntelligentProxyManager {
33
31
  this.proxyLockingEnabled = config.proxyLockingEnabled !== false;
34
32
  this.proxies = {};
35
33
  this.configLastLoaded = 0;
36
-
37
- // --- NEW: Retry configuration ---
38
34
  this.MAX_RETRIES = 3;
39
35
  this.INITIAL_BACKOFF_MS = 1000;
40
-
41
36
  if (this.proxyUrls.length === 0) { this.logger.log('WARN', '[ProxyManager] No proxy URLs provided in config.');
42
37
  } else { const lockingStatus = this.proxyLockingEnabled ? "Locking Mechanism Enabled" : "Locking Mechanism DISABLED"; this.logger.log('INFO', `[ProxyManager] Initialized with ${this.proxyUrls.length} proxies and ${lockingStatus}.`); }
43
38
  }
@@ -68,10 +63,8 @@ class IntelligentProxyManager {
68
63
  */
69
64
  async _selectProxy() {
70
65
  await this._loadConfig();
71
-
72
66
  const availableProxies = this.proxyLockingEnabled ? Object.values(this.proxies).filter(p => p.status === 'unlocked') : Object.values(this.proxies);
73
- if (availableProxies.length === 0) { const errorMsg = this.proxyLockingEnabled ? "All proxies are locked. No proxy available." : "No proxies are loaded. Cannot make request.";
74
- this.logger.log('ERROR', `[ProxyManager] ${errorMsg}`); throw new Error(errorMsg); }
67
+ if (availableProxies.length === 0) { const errorMsg = this.proxyLockingEnabled ? "All proxies are locked. No proxy available." : "No proxies are loaded. Cannot make request."; this.logger.log('ERROR', `[ProxyManager] ${errorMsg}`); throw new Error(errorMsg); }
75
68
  const selected = availableProxies[Math.floor(Math.random() * availableProxies.length)];
76
69
  return { owner: selected.owner, url: selected.url };
77
70
  }
@@ -84,8 +77,7 @@ class IntelligentProxyManager {
84
77
  if (!this.proxyLockingEnabled) { this.logger.log('TRACE', `[ProxyManager] Locking skipped for ${owner} (locking is disabled).`); return; }
85
78
  if (this.proxies[owner]) { this.proxies[owner].status = 'locked'; }
86
79
  this.logger.log('WARN', `[ProxyManager] Locking proxy: ${owner}`);
87
- try { const docRef = this.firestore.doc(this.PERFORMANCE_DOC_PATH);
88
- await docRef.set({ locks: { [owner]: { locked: true, lastLocked: FieldValue.serverTimestamp() } } }, { merge: true });
80
+ try { const docRef = this.firestore.doc(this.PERFORMANCE_DOC_PATH); await docRef.set({ locks: { [owner]: { locked: true, lastLocked: FieldValue.serverTimestamp() } } }, { merge: true });
89
81
  } catch (error) { this.logger.log('ERROR', `[ProxyManager] Failed to write lock for ${owner} to Firestore.`, { errorMessage: error.message }); }
90
82
  }
91
83
 
@@ -97,46 +89,20 @@ class IntelligentProxyManager {
97
89
  */
98
90
  async fetch(targetUrl, options = {}) {
99
91
  let proxy = null;
100
- try {
101
- proxy = await this._selectProxy();
102
- } catch (error) {
103
- // This happens if *all* proxies are locked.
104
- return { ok: false, status: 503, error: { message: error.message }, headers: new Headers() };
105
- }
106
-
92
+ try { proxy = await this._selectProxy(); } catch (error) { return { ok: false, status: 503, error: { message: error.message }, headers: new Headers() }; }
107
93
  let backoff = this.INITIAL_BACKOFF_MS;
108
94
  let lastResponse = null;
109
-
110
95
  for (let attempt = 1; attempt <= this.MAX_RETRIES; attempt++) {
111
96
  const response = await this._fetchViaAppsScript(proxy.url, targetUrl, options);
112
- lastResponse = response; // Always store the last response
113
-
97
+ lastResponse = response;
114
98
  // 1. Success
115
- if (response.ok) {
116
- return response;
117
- }
118
-
99
+ if (response.ok) { return response; }
119
100
  // 2. Rate Limit Error (Retryable)
120
- if (response.isRateLimitError) {
121
- this.logger.log('WARN', `[ProxyManager] Rate limit hit on proxy ${proxy.owner} (Attempt ${attempt}/${this.MAX_RETRIES}). Backing off for ${backoff}ms...`, { url: targetUrl });
122
- await sleep(backoff);
123
- backoff *= 2; // Exponential backoff
124
- // Continue to the next attempt
125
- continue;
126
- }
127
-
101
+ if (response.isRateLimitError) { this.logger.log('WARN', `[ProxyManager] Rate limit hit on proxy ${proxy.owner} (Attempt ${attempt}/${this.MAX_RETRIES}). Backing off for ${backoff}ms...`, { url: targetUrl }); await sleep(backoff); backoff *= 2; continue; }
128
102
  // 3. Other Fetch Error (Non-Retryable, Lock Proxy)
129
- if (response.isUrlFetchError) {
130
- this.logger.log('ERROR', `[ProxyManager] Proxy ${proxy.owner} failed (non-rate-limit). Locking proxy.`, { url: targetUrl, status: response.status });
131
- await this.lockProxy(proxy.owner);
132
- return response; // Fail fast and return
133
- }
134
-
103
+ if (response.isUrlFetchError) { this.logger.log('ERROR', `[ProxyManager] Proxy ${proxy.owner} failed (non-rate-limit). Locking proxy.`, { url: targetUrl, status: response.status }); await this.lockProxy(proxy.owner); return response; }
135
104
  // 4. Standard Error (e.g., 404, 500 from *target* URL, not proxy)
136
- // This was a "successful" proxy fetch of a failing URL. Not retryable.
137
- return response;
138
- }
139
-
105
+ return response; }
140
106
  // If loop finishes, all retries failed (likely all were rate-limit errors)
141
107
  this.logger.log('ERROR', `[ProxyManager] Request failed after ${this.MAX_RETRIES} rate-limit retries.`, { url: targetUrl });
142
108
  return lastResponse;
@@ -152,30 +118,16 @@ class IntelligentProxyManager {
152
118
  const payload = { url: targetUrl, ...options };
153
119
  try {
154
120
  const response = await fetch(proxyUrl, { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(payload) });
155
-
156
- // This is an error with the *proxy function itself* (e.g., 500, 429)
157
121
  if (!response.ok) {
158
122
  const errorText = await response.text();
159
123
  this.logger.log('WARN', `[ProxyManager] Proxy infrastructure itself failed.`, { status: response.status, proxy: proxyUrl, error: errorText });
160
124
  const isRateLimit = response.status === 429;
161
- return { ok: false, status: response.status, isUrlFetchError: true, isRateLimitError: isRateLimit, error: { message: `Proxy infrastructure failed with status ${response.status}` }, headers: response.headers, text: () => Promise.resolve(errorText) };
162
- }
163
-
125
+ return { ok: false, status: response.status, isUrlFetchError: true, isRateLimitError: isRateLimit, error: { message: `Proxy infrastructure failed with status ${response.status}` }, headers: response.headers, text: () => Promise.resolve(errorText) }; }
164
126
  const proxyResponse = await response.json();
165
-
166
- // This is an error *returned by the proxy* (e.g., UrlFetchApp failed)
167
127
  if (proxyResponse.error) {
168
128
  const errorMsg = proxyResponse.error.message || '';
169
- // --- NEW: Check for AppScript's rate limit error text ---
170
- if (errorMsg.toLowerCase().includes('service invoked too many times')) {
171
- this.logger.log('WARN', `[ProxyManager] Proxy quota error: ${proxyUrl}`, { error: proxyResponse.error });
172
- return { ok: false, status: 500, error: proxyResponse.error, isUrlFetchError: true, isRateLimitError: true, headers: new Headers() }; // <-- Set flag
173
- }
174
- // Other UrlFetchApp error
175
- return { ok: false, status: 500, error: proxyResponse.error, isUrlFetchError: true, isRateLimitError: false, headers: new Headers(), text: () => Promise.resolve(errorMsg) };
176
- }
177
-
178
- // Success. The proxy fetched the target URL.
129
+ if (errorMsg.toLowerCase().includes('service invoked too many times')) { this.logger.log('WARN', `[ProxyManager] Proxy quota error: ${proxyUrl}`, { error: proxyResponse.error }); return { ok: false, status: 500, error: proxyResponse.error, isUrlFetchError: true, isRateLimitError: true, headers: new Headers() }; }
130
+ return { ok: false, status: 500, error: proxyResponse.error, isUrlFetchError: true, isRateLimitError: false, headers: new Headers(), text: () => Promise.resolve(errorMsg) }; }
179
131
  return {
180
132
  ok: proxyResponse.statusCode >= 200 && proxyResponse.statusCode < 300,
181
133
  status: proxyResponse.statusCode,
@@ -183,13 +135,8 @@ class IntelligentProxyManager {
183
135
  json: () => Promise.resolve(JSON.parse(proxyResponse.body)),
184
136
  text: () => Promise.resolve(proxyResponse.body),
185
137
  isUrlFetchError: false,
186
- isRateLimitError: false
187
- };
188
- } catch (networkError) {
189
- this.logger.log('ERROR', `[ProxyManager] Network error calling proxy: ${proxyUrl}`, { errorMessage: networkError.message });
190
- return { ok: false, status: 0, isUrlFetchError: true, isRateLimitError: false, error: { message: `Network error: ${networkError.message}` }, headers: new Headers() };
191
- }
192
- }
138
+ isRateLimitError: false };
139
+ } catch (networkError) { this.logger.log('ERROR', `[ProxyManager] Network error calling proxy: ${proxyUrl}`, { errorMessage: networkError.message }); return { ok: false, status: 0, isUrlFetchError: true, isRateLimitError: false, error: { message: `Network error: ${networkError.message}` }, headers: new Headers() }; } }
193
140
  }
194
141
 
195
142
  module.exports = { IntelligentProxyManager };
package/index.js CHANGED
@@ -1,101 +1,45 @@
1
1
  /**
2
2
  * @fileoverview Main entry point for the Bulltrackers shared module.
3
- * This module consolidates all core logic into a single 'pipe' object
4
- * to enforce a clear naming convention and dependency injection pattern.
3
+ * Export the pipes!
5
4
  */
6
-
7
- // --- Core Utilities (Classes and Stateless Helpers) ---
8
-
9
- const core = {
10
- IntelligentHeaderManager : require('./functions/core/utils/intelligent_header_manager') .IntelligentHeaderManager,
11
- IntelligentProxyManager : require('./functions/core/utils/intelligent_proxy_manager') .IntelligentProxyManager,
12
- FirestoreBatchManager : require('./functions/task-engine/utils/firestore_batch_manager').FirestoreBatchManager,
13
- firestoreUtils : require('./functions/core/utils/firestore_utils'),
14
- pubsubUtils : require('./functions/core/utils/pubsub_utils'),
15
- };
16
-
17
- // --- Pipe 1: Orchestrator ---
18
-
19
- const orchestrator = {
20
- // Main Pipes (Entry points for Cloud Functions)
21
- runDiscoveryOrchestrator : require('./functions/orchestrator/index').runDiscoveryOrchestrator,
22
- runUpdateOrchestrator : require('./functions/orchestrator/index').runUpdateOrchestrator,
23
-
24
- // Sub-Pipes (Discovery)
25
- checkDiscoveryNeed : require('./functions/orchestrator/helpers/discovery_helpers').checkDiscoveryNeed,
26
- getDiscoveryCandidates : require('./functions/orchestrator/helpers/discovery_helpers').getDiscoveryCandidates,
27
- dispatchDiscovery : require('./functions/orchestrator/helpers/discovery_helpers').dispatchDiscovery,
28
-
29
- // Sub-Pipes (Updates)
30
- getUpdateTargets : require('./functions/orchestrator/helpers/update_helpers').getUpdateTargets,
31
- dispatchUpdates : require('./functions/orchestrator/helpers/update_helpers').dispatchUpdates,
32
- };
33
-
34
-
35
- // --- Pipe 2: Dispatcher ---
36
-
37
- const dispatcher = {
38
- handleRequest : require('./functions/dispatcher/index').handleRequest,
39
- dispatchTasksInBatches : require('./functions/dispatcher/helpers/dispatch_helpers').dispatchTasksInBatches,
40
- };
41
-
42
-
43
- // --- Pipe 3: Task Engine ---
44
-
45
- const taskEngine = {
46
- handleRequest : require('./functions/task-engine/handler_creator').handleRequest,
47
- handleDiscover : require('./functions/task-engine/helpers/discover_helpers').handleDiscover,
48
- handleVerify : require('./functions/task-engine/helpers/verify_helpers').handleVerify,
49
- handleUpdate : require('./functions/task-engine/helpers/update_helpers').handleUpdate,
50
- };
51
-
52
-
53
- // --- Pipe 4: Computation System ---
54
-
55
- const computationSystem = {
56
- runComputationPass : require('./functions/computation-system/helpers/computation_pass_runner').runComputationPass,
57
- dataLoader : require('./functions/computation-system/utils/data_loader'),
58
- computationUtils : require('./functions/computation-system/utils/utils'),
59
- };
60
-
61
-
62
- // --- Pipe 5: API ---
63
-
64
- const api = {
65
- createApiApp : require('./functions/generic-api/index').createApiApp,
66
- helpers : require('./functions/generic-api/helpers/api_helpers'),
67
- };
68
-
69
-
70
- // --- Pipe 6: Maintenance ---
71
-
72
- const maintenance = {
73
- runSpeculatorCleanup : require('./functions/speculator-cleanup-orchestrator/helpers/cleanup_helpers') .runCleanup,
74
- handleInvalidSpeculator : require('./functions/invalid-speculator-handler/helpers/handler_helpers') .handleInvalidSpeculator,
75
- runFetchInsights : require('./functions/fetch-insights/helpers/handler_helpers').fetchAndStoreInsights,
76
- runFetchPrices : require('./functions/etoro-price-fetcher/helpers/handler_helpers').fetchAndStorePrices,
77
- runSocialOrchestrator : require('./functions/social-orchestrator/helpers/orchestrator_helpers') .runSocialOrchestrator,
78
- handleSocialTask : require('./functions/social-task-handler/helpers/handler_helpers') .handleSocialTask,
79
- runBackfillAssetPrices : require('./functions/price-backfill/helpers/handler_helpers') .runBackfillAssetPrices,
80
- };
81
-
82
-
83
- // --- Pipe 7: Proxy ---
84
-
85
- const proxy = {
86
- handlePost : require('./functions/appscript-api/index').handlePost,
87
- };
88
-
89
-
90
- module.exports = {
91
- pipe: {
92
- core,
93
- orchestrator,
94
- dispatcher,
95
- taskEngine,
96
- computationSystem,
97
- api,
98
- maintenance,
99
- proxy,
100
- }
101
- };
5
+ // Core
6
+ const core = { IntelligentHeaderManager : require('./functions/core/utils/intelligent_header_manager') .IntelligentHeaderManager,
7
+ IntelligentProxyManager : require('./functions/core/utils/intelligent_proxy_manager') .IntelligentProxyManager,
8
+ FirestoreBatchManager : require('./functions/task-engine/utils/firestore_batch_manager') .FirestoreBatchManager,
9
+ firestoreUtils : require('./functions/core/utils/firestore_utils'),
10
+ pubsubUtils : require('./functions/core/utils/pubsub_utils') };
11
+ // Orchestrator
12
+ const orchestrator = { runDiscoveryOrchestrator : require('./functions/orchestrator/index') .runDiscoveryOrchestrator,
13
+ runUpdateOrchestrator : require('./functions/orchestrator/index') .runUpdateOrchestrator,
14
+ checkDiscoveryNeed : require('./functions/orchestrator/helpers/discovery_helpers') .checkDiscoveryNeed,
15
+ getDiscoveryCandidates : require('./functions/orchestrator/helpers/discovery_helpers') .getDiscoveryCandidates,
16
+ dispatchDiscovery : require('./functions/orchestrator/helpers/discovery_helpers') .dispatchDiscovery,
17
+ getUpdateTargets : require('./functions/orchestrator/helpers/update_helpers') .getUpdateTargets,
18
+ dispatchUpdates : require('./functions/orchestrator/helpers/update_helpers') .dispatchUpdates };
19
+ // Dispatcher
20
+ const dispatcher = { handleRequest : require('./functions/dispatcher/index') .handleRequest ,
21
+ dispatchTasksInBatches : require('./functions/dispatcher/helpers/dispatch_helpers') .dispatchTasksInBatches };
22
+ // Task Engine
23
+ const taskEngine = { handleRequest : require('./functions/task-engine/handler_creator') .handleRequest ,
24
+ handleDiscover : require('./functions/task-engine/helpers/discover_helpers') .handleDiscover,
25
+ handleVerify : require('./functions/task-engine/helpers/verify_helpers') .handleVerify ,
26
+ handleUpdate : require('./functions/task-engine/helpers/update_helpers') .handleUpdate };
27
+ // Computation System
28
+ const computationSystem = { runComputationPass : require('./functions/computation-system/helpers/computation_pass_runner') .runComputationPass,
29
+ dataLoader : require('./functions/computation-system/utils/data_loader'),
30
+ computationUtils : require('./functions/computation-system/utils/utils') };
31
+ // API
32
+ const api = { createApiApp : require('./functions/generic-api/index') .createApiApp,
33
+ helpers : require('./functions/generic-api/helpers/api_helpers') };
34
+ // Maintenance
35
+ const maintenance = { runSpeculatorCleanup : require('./functions/speculator-cleanup-orchestrator/helpers/cleanup_helpers') .runCleanup,
36
+ handleInvalidSpeculator : require('./functions/invalid-speculator-handler/helpers/handler_helpers') .handleInvalidSpeculator,
37
+ runFetchInsights : require('./functions/fetch-insights/helpers/handler_helpers') .fetchAndStoreInsights,
38
+ runFetchPrices : require('./functions/etoro-price-fetcher/helpers/handler_helpers') .fetchAndStorePrices,
39
+ runSocialOrchestrator : require('./functions/social-orchestrator/helpers/orchestrator_helpers') .runSocialOrchestrator,
40
+ handleSocialTask : require('./functions/social-task-handler/helpers/handler_helpers') .handleSocialTask,
41
+ runBackfillAssetPrices : require('./functions/price-backfill/helpers/handler_helpers') .runBackfillAssetPrices };
42
+ // Proxy
43
+ const proxy = { handlePost : require('./functions/appscript-api/index') .handlePost };
44
+
45
+ module.exports = { pipe: { core, orchestrator, dispatcher, taskEngine, computationSystem, api, maintenance, proxy } };
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "bulltrackers-module",
3
- "version": "1.0.155",
3
+ "version": "1.0.156",
4
4
  "description": "Helper Functions for Bulltrackers.",
5
5
  "main": "index.js",
6
6
  "files": [