bulltrackers-module 1.0.104 → 1.0.106
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.MD +222 -222
- package/functions/appscript-api/helpers/errors.js +19 -19
- package/functions/appscript-api/index.js +58 -58
- package/functions/computation-system/helpers/orchestration_helpers.js +647 -113
- package/functions/computation-system/utils/data_loader.js +191 -191
- package/functions/computation-system/utils/utils.js +149 -254
- package/functions/core/utils/firestore_utils.js +433 -433
- package/functions/core/utils/pubsub_utils.js +53 -53
- package/functions/dispatcher/helpers/dispatch_helpers.js +47 -47
- package/functions/dispatcher/index.js +52 -52
- package/functions/etoro-price-fetcher/helpers/handler_helpers.js +124 -124
- package/functions/fetch-insights/helpers/handler_helpers.js +91 -91
- package/functions/generic-api/helpers/api_helpers.js +379 -379
- package/functions/generic-api/index.js +150 -150
- package/functions/invalid-speculator-handler/helpers/handler_helpers.js +75 -75
- package/functions/orchestrator/helpers/discovery_helpers.js +226 -226
- package/functions/orchestrator/helpers/update_helpers.js +92 -92
- package/functions/orchestrator/index.js +147 -147
- package/functions/price-backfill/helpers/handler_helpers.js +116 -123
- package/functions/social-orchestrator/helpers/orchestrator_helpers.js +61 -61
- package/functions/social-task-handler/helpers/handler_helpers.js +288 -288
- package/functions/task-engine/handler_creator.js +78 -78
- package/functions/task-engine/helpers/discover_helpers.js +125 -125
- package/functions/task-engine/helpers/update_helpers.js +118 -118
- package/functions/task-engine/helpers/verify_helpers.js +162 -162
- package/functions/task-engine/utils/firestore_batch_manager.js +258 -258
- package/index.js +105 -113
- package/package.json +45 -45
- package/functions/computation-system/computation_dependencies.json +0 -120
- package/functions/computation-system/helpers/worker_helpers.js +0 -340
- package/functions/computation-system/utils/computation_state_manager.js +0 -178
- package/functions/computation-system/utils/dependency_graph.js +0 -191
- package/functions/speculator-cleanup-orchestrator/helpers/cleanup_helpers.js +0 -160
|
@@ -1,91 +1,91 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* @fileoverview Main pipe: pipe.maintenance.runFetchInsights
|
|
3
|
-
* REFACTORED: Now stateless and receives dependencies.
|
|
4
|
-
*/
|
|
5
|
-
const { FieldValue } = require('@google-cloud/firestore');
|
|
6
|
-
|
|
7
|
-
/**
|
|
8
|
-
* Main pipe: pipe.maintenance.runFetchInsights
|
|
9
|
-
* @param {object} config - Configuration object.
|
|
10
|
-
* @param {object} dependencies - Contains db, logger, headerManager, proxyManager.
|
|
11
|
-
* @returns {Promise<{success: boolean, message: string, instrumentCount?: number}>}
|
|
12
|
-
*/
|
|
13
|
-
exports.fetchAndStoreInsights = async (config, dependencies) => {
|
|
14
|
-
const { db, logger, headerManager, proxyManager } = dependencies;
|
|
15
|
-
|
|
16
|
-
logger.log('INFO', '[FetchInsightsHelpers] Starting eToro insights data fetch...');
|
|
17
|
-
let selectedHeader = null;
|
|
18
|
-
let wasSuccessful = false;
|
|
19
|
-
|
|
20
|
-
try {
|
|
21
|
-
if (!config.etoroInsightsUrl || !config.insightsCollectionName) {
|
|
22
|
-
throw new Error("Missing required configuration: etoroInsightsUrl or insightsCollectionName.");
|
|
23
|
-
}
|
|
24
|
-
|
|
25
|
-
selectedHeader = await headerManager.selectHeader();
|
|
26
|
-
if (!selectedHeader || !selectedHeader.header) {
|
|
27
|
-
throw new Error("Could not select a valid header for the request.");
|
|
28
|
-
}
|
|
29
|
-
|
|
30
|
-
logger.log('INFO', `[FetchInsightsHelpers] Using header ID: ${selectedHeader.id}`, {
|
|
31
|
-
userAgent: selectedHeader.header['User-Agent']
|
|
32
|
-
});
|
|
33
|
-
|
|
34
|
-
const fetchOptions = {
|
|
35
|
-
headers: selectedHeader.header,
|
|
36
|
-
timeout: 30000
|
|
37
|
-
};
|
|
38
|
-
|
|
39
|
-
const response = await proxyManager.fetch(config.etoroInsightsUrl, fetchOptions);
|
|
40
|
-
|
|
41
|
-
if (!response || typeof response.text !== 'function') {
|
|
42
|
-
const responseString = JSON.stringify(response, null, 2);
|
|
43
|
-
logger.log('ERROR', `[FetchInsightsHelpers] Invalid or incomplete response received. Response object: ${responseString}`);
|
|
44
|
-
throw new Error(`Invalid response structure received from proxy.`);
|
|
45
|
-
}
|
|
46
|
-
|
|
47
|
-
if (!response.ok) {
|
|
48
|
-
const errorText = await response.text();
|
|
49
|
-
throw new Error(`API request failed via proxy with status ${response.status}: ${errorText}`);
|
|
50
|
-
}
|
|
51
|
-
|
|
52
|
-
wasSuccessful = true;
|
|
53
|
-
const insightsData = await response.json();
|
|
54
|
-
|
|
55
|
-
if (!Array.isArray(insightsData) || insightsData.length === 0) {
|
|
56
|
-
throw new Error('API returned empty or invalid data.');
|
|
57
|
-
}
|
|
58
|
-
|
|
59
|
-
const today = new Date().toISOString().slice(0, 10);
|
|
60
|
-
const docRef = db.collection(config.insightsCollectionName).doc(today); // Use db
|
|
61
|
-
|
|
62
|
-
const firestorePayload = {
|
|
63
|
-
fetchedAt: FieldValue.serverTimestamp(),
|
|
64
|
-
instrumentCount: insightsData.length,
|
|
65
|
-
insights: insightsData
|
|
66
|
-
};
|
|
67
|
-
|
|
68
|
-
await docRef.set(firestorePayload);
|
|
69
|
-
|
|
70
|
-
const successMsg = `Successfully fetched and stored ${insightsData.length} instrument insights for ${today}.`;
|
|
71
|
-
logger.log('SUCCESS', `[FetchInsightsHelpers] ${successMsg}`, {
|
|
72
|
-
documentId: today,
|
|
73
|
-
instrumentCount: insightsData.length
|
|
74
|
-
});
|
|
75
|
-
return { success: true, message: successMsg, instrumentCount: insightsData.length };
|
|
76
|
-
|
|
77
|
-
} catch (error) {
|
|
78
|
-
logger.log('ERROR', '[FetchInsightsHelpers] Error fetching eToro insights', {
|
|
79
|
-
errorMessage: error.message,
|
|
80
|
-
errorStack: error.stack,
|
|
81
|
-
headerId: selectedHeader ? selectedHeader.id : 'not-selected'
|
|
82
|
-
});
|
|
83
|
-
throw error;
|
|
84
|
-
} finally {
|
|
85
|
-
if (selectedHeader) {
|
|
86
|
-
await headerManager.updatePerformance(selectedHeader.id, wasSuccessful);
|
|
87
|
-
// Also flush performance, as this is a standalone function
|
|
88
|
-
await headerManager.flushPerformanceUpdates();
|
|
89
|
-
}
|
|
90
|
-
}
|
|
91
|
-
};
|
|
1
|
+
/**
|
|
2
|
+
* @fileoverview Main pipe: pipe.maintenance.runFetchInsights
|
|
3
|
+
* REFACTORED: Now stateless and receives dependencies.
|
|
4
|
+
*/
|
|
5
|
+
const { FieldValue } = require('@google-cloud/firestore');
|
|
6
|
+
|
|
7
|
+
/**
|
|
8
|
+
* Main pipe: pipe.maintenance.runFetchInsights
|
|
9
|
+
* @param {object} config - Configuration object.
|
|
10
|
+
* @param {object} dependencies - Contains db, logger, headerManager, proxyManager.
|
|
11
|
+
* @returns {Promise<{success: boolean, message: string, instrumentCount?: number}>}
|
|
12
|
+
*/
|
|
13
|
+
exports.fetchAndStoreInsights = async (config, dependencies) => {
|
|
14
|
+
const { db, logger, headerManager, proxyManager } = dependencies;
|
|
15
|
+
|
|
16
|
+
logger.log('INFO', '[FetchInsightsHelpers] Starting eToro insights data fetch...');
|
|
17
|
+
let selectedHeader = null;
|
|
18
|
+
let wasSuccessful = false;
|
|
19
|
+
|
|
20
|
+
try {
|
|
21
|
+
if (!config.etoroInsightsUrl || !config.insightsCollectionName) {
|
|
22
|
+
throw new Error("Missing required configuration: etoroInsightsUrl or insightsCollectionName.");
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
selectedHeader = await headerManager.selectHeader();
|
|
26
|
+
if (!selectedHeader || !selectedHeader.header) {
|
|
27
|
+
throw new Error("Could not select a valid header for the request.");
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
logger.log('INFO', `[FetchInsightsHelpers] Using header ID: ${selectedHeader.id}`, {
|
|
31
|
+
userAgent: selectedHeader.header['User-Agent']
|
|
32
|
+
});
|
|
33
|
+
|
|
34
|
+
const fetchOptions = {
|
|
35
|
+
headers: selectedHeader.header,
|
|
36
|
+
timeout: 30000
|
|
37
|
+
};
|
|
38
|
+
|
|
39
|
+
const response = await proxyManager.fetch(config.etoroInsightsUrl, fetchOptions);
|
|
40
|
+
|
|
41
|
+
if (!response || typeof response.text !== 'function') {
|
|
42
|
+
const responseString = JSON.stringify(response, null, 2);
|
|
43
|
+
logger.log('ERROR', `[FetchInsightsHelpers] Invalid or incomplete response received. Response object: ${responseString}`);
|
|
44
|
+
throw new Error(`Invalid response structure received from proxy.`);
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
if (!response.ok) {
|
|
48
|
+
const errorText = await response.text();
|
|
49
|
+
throw new Error(`API request failed via proxy with status ${response.status}: ${errorText}`);
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
wasSuccessful = true;
|
|
53
|
+
const insightsData = await response.json();
|
|
54
|
+
|
|
55
|
+
if (!Array.isArray(insightsData) || insightsData.length === 0) {
|
|
56
|
+
throw new Error('API returned empty or invalid data.');
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
const today = new Date().toISOString().slice(0, 10);
|
|
60
|
+
const docRef = db.collection(config.insightsCollectionName).doc(today); // Use db
|
|
61
|
+
|
|
62
|
+
const firestorePayload = {
|
|
63
|
+
fetchedAt: FieldValue.serverTimestamp(),
|
|
64
|
+
instrumentCount: insightsData.length,
|
|
65
|
+
insights: insightsData
|
|
66
|
+
};
|
|
67
|
+
|
|
68
|
+
await docRef.set(firestorePayload);
|
|
69
|
+
|
|
70
|
+
const successMsg = `Successfully fetched and stored ${insightsData.length} instrument insights for ${today}.`;
|
|
71
|
+
logger.log('SUCCESS', `[FetchInsightsHelpers] ${successMsg}`, {
|
|
72
|
+
documentId: today,
|
|
73
|
+
instrumentCount: insightsData.length
|
|
74
|
+
});
|
|
75
|
+
return { success: true, message: successMsg, instrumentCount: insightsData.length };
|
|
76
|
+
|
|
77
|
+
} catch (error) {
|
|
78
|
+
logger.log('ERROR', '[FetchInsightsHelpers] Error fetching eToro insights', {
|
|
79
|
+
errorMessage: error.message,
|
|
80
|
+
errorStack: error.stack,
|
|
81
|
+
headerId: selectedHeader ? selectedHeader.id : 'not-selected'
|
|
82
|
+
});
|
|
83
|
+
throw error;
|
|
84
|
+
} finally {
|
|
85
|
+
if (selectedHeader) {
|
|
86
|
+
await headerManager.updatePerformance(selectedHeader.id, wasSuccessful);
|
|
87
|
+
// Also flush performance, as this is a standalone function
|
|
88
|
+
await headerManager.flushPerformanceUpdates();
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
};
|