bulltrackers-module 1.0.152 → 1.0.154
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/functions/appscript-api/index.js +8 -38
- package/functions/computation-system/helpers/computation_pass_runner.js +38 -183
- package/functions/computation-system/helpers/orchestration_helpers.js +105 -326
- package/functions/computation-system/utils/data_loader.js +38 -133
- package/functions/computation-system/utils/schema_capture.js +7 -41
- package/functions/computation-system/utils/utils.js +37 -124
- package/functions/core/utils/firestore_utils.js +8 -46
- package/functions/core/utils/intelligent_header_manager.js +26 -128
- package/functions/core/utils/intelligent_proxy_manager.js +33 -171
- package/functions/core/utils/pubsub_utils.js +7 -24
- package/functions/dispatcher/helpers/dispatch_helpers.js +9 -30
- package/functions/dispatcher/index.js +7 -30
- package/functions/etoro-price-fetcher/helpers/handler_helpers.js +12 -80
- package/functions/fetch-insights/helpers/handler_helpers.js +18 -70
- package/functions/generic-api/helpers/api_helpers.js +28 -167
- package/functions/generic-api/index.js +49 -188
- package/functions/invalid-speculator-handler/helpers/handler_helpers.js +10 -47
- package/functions/orchestrator/helpers/discovery_helpers.js +1 -5
- package/functions/orchestrator/index.js +1 -6
- package/functions/price-backfill/helpers/handler_helpers.js +13 -69
- package/functions/social-orchestrator/helpers/orchestrator_helpers.js +5 -37
- package/functions/social-task-handler/helpers/handler_helpers.js +29 -186
- package/functions/speculator-cleanup-orchestrator/helpers/cleanup_helpers.js +19 -78
- package/functions/task-engine/handler_creator.js +2 -8
- package/functions/task-engine/helpers/update_helpers.js +74 -100
- package/functions/task-engine/helpers/verify_helpers.js +11 -56
- package/functions/task-engine/utils/firestore_batch_manager.js +29 -65
- package/functions/task-engine/utils/task_engine_utils.js +14 -37
- package/index.js +45 -43
- package/package.json +1 -1
|
@@ -17,109 +17,41 @@ const SHARD_SIZE = 40;
|
|
|
17
17
|
*/
|
|
18
18
|
exports.fetchAndStorePrices = async (config, dependencies) => {
|
|
19
19
|
const { db, logger, headerManager, proxyManager } = dependencies;
|
|
20
|
-
|
|
21
20
|
logger.log('INFO', '[PriceFetcherHelpers] Starting Daily Closing Price Update...');
|
|
22
21
|
let selectedHeader = null;
|
|
23
22
|
let wasSuccessful = false;
|
|
24
|
-
|
|
25
|
-
// --- NEW: Use the new config key, or fallback to the old one --- TODO Implement the config
|
|
26
23
|
const priceCollectionName = 'asset_prices';
|
|
27
|
-
|
|
28
|
-
try {
|
|
29
|
-
if (!config.etoroApiUrl) {
|
|
30
|
-
throw new Error("Missing required configuration: etoroApiUrl.");
|
|
31
|
-
}
|
|
32
|
-
|
|
24
|
+
try { if (!config.etoroApiUrl) { throw new Error("Missing required configuration: etoroApiUrl."); }
|
|
33
25
|
selectedHeader = await headerManager.selectHeader();
|
|
34
|
-
if (!selectedHeader || !selectedHeader.header) {
|
|
35
|
-
|
|
36
|
-
}
|
|
37
|
-
|
|
38
|
-
const fetchOptions = {
|
|
39
|
-
headers: selectedHeader.header,
|
|
40
|
-
timeout: 60000
|
|
41
|
-
};
|
|
42
|
-
|
|
26
|
+
if (!selectedHeader || !selectedHeader.header) { throw new Error("Could not select a valid header for the request."); }
|
|
27
|
+
const fetchOptions = { headers: selectedHeader.header, timeout: 60000 };
|
|
43
28
|
logger.log('INFO', `[PriceFetcherHelpers] Using header ID: ${selectedHeader.id}`);
|
|
44
|
-
|
|
45
29
|
const response = await proxyManager.fetch(config.etoroApiUrl, fetchOptions);
|
|
46
|
-
|
|
47
|
-
if (!response
|
|
48
|
-
throw new Error(`Invalid response structure received from proxy.`);
|
|
49
|
-
}
|
|
50
|
-
|
|
51
|
-
if (!response.ok) {
|
|
52
|
-
const errorBody = await response.text();
|
|
53
|
-
throw new Error(`API returned status ${response.status}: ${errorBody}`);
|
|
54
|
-
}
|
|
30
|
+
if (!response || typeof response.text !== 'function') { throw new Error(`Invalid response structure received from proxy.`); }
|
|
31
|
+
if (!response.ok) { const errorBody = await response.text(); throw new Error(`API returned status ${response.status}: ${errorBody}`); }
|
|
55
32
|
wasSuccessful = true;
|
|
56
|
-
|
|
57
33
|
const results = await response.json();
|
|
58
|
-
if (!Array.isArray(results)) {
|
|
59
|
-
throw new Error('Invalid response format from API. Expected an array.');
|
|
60
|
-
}
|
|
61
|
-
|
|
62
|
-
// --- START MODIFICATION ---
|
|
63
|
-
|
|
34
|
+
if (!Array.isArray(results)) { throw new Error('Invalid response format from API. Expected an array.'); }
|
|
64
35
|
logger.log('INFO', `[PriceFetcherHelpers] Received ${results.length} instrument prices. Sharding...`);
|
|
65
|
-
const shardUpdates = {};
|
|
66
|
-
|
|
36
|
+
const shardUpdates = {};
|
|
67
37
|
for (const instrumentData of results) {
|
|
68
38
|
const dailyData = instrumentData?.ClosingPrices?.Daily;
|
|
69
39
|
const instrumentId = instrumentData.InstrumentId;
|
|
70
|
-
|
|
71
40
|
if (instrumentId && dailyData?.Price && dailyData?.Date) {
|
|
72
41
|
const instrumentIdStr = String(instrumentId);
|
|
73
42
|
const dateKey = dailyData.Date.substring(0, 10);
|
|
74
|
-
|
|
75
|
-
// Determine shard ID
|
|
76
43
|
const shardId = `shard_${parseInt(instrumentIdStr, 10) % SHARD_SIZE}`;
|
|
77
|
-
|
|
78
|
-
if (!shardUpdates[shardId]) {
|
|
79
|
-
shardUpdates[shardId] = {};
|
|
80
|
-
}
|
|
81
|
-
|
|
82
|
-
// Use dot notation to define the update path
|
|
44
|
+
if (!shardUpdates[shardId]) { shardUpdates[shardId] = {}; }
|
|
83
45
|
const pricePath = `${instrumentIdStr}.prices.${dateKey}`;
|
|
84
46
|
const updatePath = `${instrumentIdStr}.lastUpdated`;
|
|
85
|
-
|
|
86
47
|
shardUpdates[shardId][pricePath] = dailyData.Price;
|
|
87
|
-
shardUpdates[shardId][updatePath] = FieldValue.serverTimestamp();
|
|
88
|
-
}
|
|
89
|
-
}
|
|
90
|
-
|
|
91
|
-
// Commit all shard updates in parallel
|
|
48
|
+
shardUpdates[shardId][updatePath] = FieldValue.serverTimestamp(); } }
|
|
92
49
|
const batchPromises = [];
|
|
93
|
-
for (const shardId in shardUpdates) {
|
|
94
|
-
const docRef = db.collection(priceCollectionName).doc(shardId);
|
|
95
|
-
const payload = shardUpdates[shardId];
|
|
96
|
-
|
|
97
|
-
// --- THIS IS THE FIX ---
|
|
98
|
-
// Use .update() to correctly merge data into nested maps.
|
|
99
|
-
// Using .set(payload, { merge: true }) creates the flat, broken keys.
|
|
100
|
-
batchPromises.push(docRef.update(payload));
|
|
101
|
-
// --- END FIX ---
|
|
102
|
-
}
|
|
103
|
-
|
|
50
|
+
for (const shardId in shardUpdates) { const docRef = db.collection(priceCollectionName).doc(shardId); const payload = shardUpdates[shardId]; batchPromises.push(docRef.update(payload)); }
|
|
104
51
|
await Promise.all(batchPromises);
|
|
105
|
-
|
|
106
|
-
// --- END MODIFICATION ---
|
|
107
|
-
|
|
108
52
|
const successMessage = `Successfully processed and saved daily prices for ${results.length} instruments to ${batchPromises.length} shards.`;
|
|
109
53
|
logger.log('SUCCESS', `[PriceFetcherHelpers] ${successMessage}`);
|
|
110
54
|
return { success: true, message: successMessage, instrumentsProcessed: results.length };
|
|
111
|
-
|
|
112
|
-
}
|
|
113
|
-
logger.log('ERROR', '[PriceFetcherHelpers] Fatal error during closing price update', {
|
|
114
|
-
errorMessage: error.message,
|
|
115
|
-
errorStack: error.stack,
|
|
116
|
-
headerId: selectedHeader ? selectedHeader.id : 'not-selected'
|
|
117
|
-
});
|
|
118
|
-
throw error;
|
|
119
|
-
} finally {
|
|
120
|
-
if (selectedHeader) {
|
|
121
|
-
await headerManager.updatePerformance(selectedHeader.id, wasSuccessful);
|
|
122
|
-
await headerManager.flushPerformanceUpdates();
|
|
123
|
-
}
|
|
124
|
-
}
|
|
55
|
+
} catch (error) { logger.log('ERROR', '[PriceFetcherHelpers] Fatal error during closing price update', { errorMessage: error.message, errorStack: error.stack, headerId: selectedHeader ? selectedHeader.id : 'not-selected' }); throw error;
|
|
56
|
+
} finally { if (selectedHeader) { await headerManager.updatePerformance(selectedHeader.id, wasSuccessful); await headerManager.flushPerformanceUpdates(); } }
|
|
125
57
|
};
|
|
@@ -12,80 +12,28 @@ const { FieldValue } = require('@google-cloud/firestore');
|
|
|
12
12
|
*/
|
|
13
13
|
exports.fetchAndStoreInsights = async (config, dependencies) => {
|
|
14
14
|
const { db, logger, headerManager, proxyManager } = dependencies;
|
|
15
|
-
|
|
16
15
|
logger.log('INFO', '[FetchInsightsHelpers] Starting eToro insights data fetch...');
|
|
17
|
-
let selectedHeader = null;
|
|
18
|
-
let wasSuccessful = false;
|
|
19
|
-
|
|
16
|
+
let selectedHeader = null; let wasSuccessful = false;
|
|
20
17
|
try {
|
|
21
|
-
if (!config.etoroInsightsUrl || !config.insightsCollectionName) {
|
|
22
|
-
throw new Error("Missing required configuration: etoroInsightsUrl or insightsCollectionName.");
|
|
23
|
-
}
|
|
24
|
-
|
|
18
|
+
if (!config.etoroInsightsUrl || !config.insightsCollectionName) { throw new Error("Missing required configuration: etoroInsightsUrl or insightsCollectionName."); }
|
|
25
19
|
selectedHeader = await headerManager.selectHeader();
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
}
|
|
29
|
-
|
|
30
|
-
logger.log('INFO', `[FetchInsightsHelpers] Using header ID: ${selectedHeader.id}`, {
|
|
31
|
-
userAgent: selectedHeader.header['User-Agent']
|
|
32
|
-
});
|
|
33
|
-
|
|
34
|
-
const fetchOptions = {
|
|
35
|
-
headers: selectedHeader.header,
|
|
36
|
-
timeout: 30000
|
|
37
|
-
};
|
|
38
|
-
|
|
20
|
+
if (!selectedHeader || !selectedHeader.header) { throw new Error("Could not select a valid header for the request."); }
|
|
21
|
+
logger.log('INFO', `[FetchInsightsHelpers] Using header ID: ${selectedHeader.id}`, { userAgent: selectedHeader.header['User-Agent'] });
|
|
22
|
+
const fetchOptions = { headers: selectedHeader.header, timeout: 30000 };
|
|
39
23
|
const response = await proxyManager.fetch(config.etoroInsightsUrl, fetchOptions);
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
throw new Error(`Invalid response structure received from proxy.`);
|
|
45
|
-
}
|
|
46
|
-
|
|
47
|
-
if (!response.ok) {
|
|
48
|
-
const errorText = await response.text();
|
|
49
|
-
throw new Error(`API request failed via proxy with status ${response.status}: ${errorText}`);
|
|
50
|
-
}
|
|
51
|
-
|
|
24
|
+
if (!response || typeof response.text !== 'function') { const responseString = JSON.stringify(response, null, 2);
|
|
25
|
+
logger.log('ERROR', `[FetchInsightsHelpers] Invalid or incomplete response received. Response object: ${responseString}`); throw new Error(`Invalid response structure received from proxy.`); }
|
|
26
|
+
if (!response.ok) { const errorText = await response.text();
|
|
27
|
+
throw new Error(`API request failed via proxy with status ${response.status}: ${errorText}`); }
|
|
52
28
|
wasSuccessful = true;
|
|
53
29
|
const insightsData = await response.json();
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
fetchedAt: FieldValue.serverTimestamp(),
|
|
64
|
-
instrumentCount: insightsData.length,
|
|
65
|
-
insights: insightsData
|
|
66
|
-
};
|
|
67
|
-
|
|
68
|
-
await docRef.set(firestorePayload);
|
|
69
|
-
|
|
70
|
-
const successMsg = `Successfully fetched and stored ${insightsData.length} instrument insights for ${today}.`;
|
|
71
|
-
logger.log('SUCCESS', `[FetchInsightsHelpers] ${successMsg}`, {
|
|
72
|
-
documentId: today,
|
|
73
|
-
instrumentCount: insightsData.length
|
|
74
|
-
});
|
|
75
|
-
return { success: true, message: successMsg, instrumentCount: insightsData.length };
|
|
76
|
-
|
|
77
|
-
} catch (error) {
|
|
78
|
-
logger.log('ERROR', '[FetchInsightsHelpers] Error fetching eToro insights', {
|
|
79
|
-
errorMessage: error.message,
|
|
80
|
-
errorStack: error.stack,
|
|
81
|
-
headerId: selectedHeader ? selectedHeader.id : 'not-selected'
|
|
82
|
-
});
|
|
83
|
-
throw error;
|
|
84
|
-
} finally {
|
|
85
|
-
if (selectedHeader) {
|
|
86
|
-
await headerManager.updatePerformance(selectedHeader.id, wasSuccessful);
|
|
87
|
-
// Also flush performance, as this is a standalone function
|
|
88
|
-
await headerManager.flushPerformanceUpdates();
|
|
89
|
-
}
|
|
90
|
-
}
|
|
30
|
+
if (!Array.isArray(insightsData) || insightsData.length === 0) { throw new Error('API returned empty or invalid data.'); }
|
|
31
|
+
const today = new Date().toISOString().slice(0, 10);
|
|
32
|
+
const docRef = db.collection(config.insightsCollectionName).doc(today);
|
|
33
|
+
const firestorePayload = { fetchedAt: FieldValue.serverTimestamp(), instrumentCount: insightsData.length, insights: insightsData };
|
|
34
|
+
await docRef.set(firestorePayload);
|
|
35
|
+
const successMsg = `Successfully fetched and stored ${insightsData.length} instrument insights for ${today}.`;
|
|
36
|
+
logger.log('SUCCESS', `[FetchInsightsHelpers] ${successMsg}`, { documentId: today, instrumentCount: insightsData.length }); return { success: true, message: successMsg, instrumentCount: insightsData.length };
|
|
37
|
+
} catch (error) { logger.log('ERROR', '[FetchInsightsHelpers] Error fetching eToro insights', { errorMessage: error.message, errorStack: error.stack, headerId: selectedHeader ? selectedHeader.id : 'not-selected' }); throw error;
|
|
38
|
+
} finally { if (selectedHeader) { await headerManager.updatePerformance(selectedHeader.id, wasSuccessful); await headerManager.flushPerformanceUpdates(); } }
|
|
91
39
|
};
|
|
@@ -7,7 +7,6 @@
|
|
|
7
7
|
|
|
8
8
|
const { FieldPath } = require('@google-cloud/firestore');
|
|
9
9
|
|
|
10
|
-
// --- All Mocks are REMOVED ---
|
|
11
10
|
|
|
12
11
|
/**
|
|
13
12
|
* Sub-pipe: pipe.api.helpers.validateRequest
|
|
@@ -16,16 +15,13 @@ const validateRequest = (query, config) => {
|
|
|
16
15
|
if (!query.computations) return "Missing 'computations' parameter.";
|
|
17
16
|
if (!query.startDate || !/^\d{4}-\d{2}-\d{2}$/.test(query.startDate)) return "Missing or invalid 'startDate'.";
|
|
18
17
|
if (!query.endDate || !/^\d{4}-\d{2}-\d{2}$/.test(query.endDate)) return "Missing or invalid 'endDate'.";
|
|
19
|
-
|
|
20
18
|
const start = new Date(query.startDate);
|
|
21
19
|
const end = new Date(query.endDate);
|
|
22
20
|
if (end < start) return "'endDate' must be after 'startDate'.";
|
|
23
|
-
|
|
24
21
|
const maxDateRange = config.maxDateRange || 100;
|
|
25
22
|
const diffTime = Math.abs(end - start);
|
|
26
23
|
const diffDays = Math.ceil(diffTime / (1000 * 60 * 60 * 24)) + 1;
|
|
27
24
|
if (diffDays > maxDateRange) return `Date range cannot exceed ${maxDateRange} days.`;
|
|
28
|
-
|
|
29
25
|
return null;
|
|
30
26
|
};
|
|
31
27
|
|
|
@@ -38,29 +34,8 @@ const validateRequest = (query, config) => {
|
|
|
38
34
|
const buildCalculationMap = (unifiedCalculations) => {
|
|
39
35
|
const calcMap = {};
|
|
40
36
|
for (const category in unifiedCalculations) {
|
|
41
|
-
for (const subKey in unifiedCalculations[category]) {
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
// Handle historical subdirectory
|
|
45
|
-
if (subKey === 'historical' && typeof item === 'object') {
|
|
46
|
-
for (const calcName in item) {
|
|
47
|
-
calcMap[calcName] = {
|
|
48
|
-
category: category,
|
|
49
|
-
class: item[calcName] // <-- Store the class
|
|
50
|
-
};
|
|
51
|
-
}
|
|
52
|
-
}
|
|
53
|
-
// Handle regular daily/meta/social calc
|
|
54
|
-
else if (typeof item === 'function') {
|
|
55
|
-
const calcName = subKey;
|
|
56
|
-
calcMap[calcName] = {
|
|
57
|
-
category: category,
|
|
58
|
-
class: item // <-- Store the class
|
|
59
|
-
};
|
|
60
|
-
}
|
|
61
|
-
}
|
|
62
|
-
}
|
|
63
|
-
return calcMap;
|
|
37
|
+
for (const subKey in unifiedCalculations[category]) { const item = unifiedCalculations[category][subKey]; if (subKey === 'historical' && typeof item === 'object') { for (const calcName in item) { calcMap[calcName] = { category: category, class: item[calcName] }; } }
|
|
38
|
+
else if (typeof item === 'function') { const calcName = subKey; calcMap[calcName] = { category: category, class: item }; }}} return calcMap;
|
|
64
39
|
};
|
|
65
40
|
|
|
66
41
|
/**
|
|
@@ -70,11 +45,7 @@ const getDateStringsInRange = (startDate, endDate) => {
|
|
|
70
45
|
const dates = [];
|
|
71
46
|
const current = new Date(startDate + 'T00:00:00Z');
|
|
72
47
|
const end = new Date(endDate + 'T00:00:00Z');
|
|
73
|
-
|
|
74
|
-
while (current <= end) {
|
|
75
|
-
dates.push(current.toISOString().slice(0, 10));
|
|
76
|
-
current.setUTCDate(current.getUTCDate() + 1);
|
|
77
|
-
}
|
|
48
|
+
while (current <= end) { dates.push(current.toISOString().slice(0, 10)); current.setUTCDate(current.getUTCDate() + 1); }
|
|
78
49
|
return dates;
|
|
79
50
|
};
|
|
80
51
|
|
|
@@ -87,40 +58,19 @@ const fetchUnifiedData = async (config, dependencies, calcKeys, dateStrings, cal
|
|
|
87
58
|
const insightsCollection = config.unifiedInsightsCollection || 'unified_insights';
|
|
88
59
|
const resultsSub = config.resultsSubcollection || 'results';
|
|
89
60
|
const compsSub = config.computationsSubcollection || 'computations';
|
|
90
|
-
|
|
91
61
|
try {
|
|
92
62
|
for (const date of dateStrings) {
|
|
93
63
|
response[date] = {};
|
|
94
64
|
const docRefs = [];
|
|
95
65
|
const keyPaths = [];
|
|
96
|
-
|
|
97
66
|
for (const key of calcKeys) {
|
|
98
67
|
const pathInfo = calcMap[key];
|
|
99
|
-
if (pathInfo) {
|
|
100
|
-
|
|
101
|
-
.collection(resultsSub).doc(pathInfo.category)
|
|
102
|
-
.collection(compsSub).doc(key);
|
|
103
|
-
docRefs.push(docRef);
|
|
104
|
-
keyPaths.push(key);
|
|
105
|
-
} else {
|
|
106
|
-
logger.log('WARN', `[${date}] No path info found for computation key: ${key}`);
|
|
107
|
-
}
|
|
108
|
-
}
|
|
68
|
+
if (pathInfo) { const docRef = db.collection(insightsCollection).doc(date) .collection(resultsSub).doc(pathInfo.category) .collection(compsSub).doc(key); docRefs.push(docRef); keyPaths.push(key);
|
|
69
|
+
} else { logger.log('WARN', `[${date}] No path info found for computation key: ${key}`); } }
|
|
109
70
|
if (docRefs.length === 0) continue;
|
|
110
71
|
const snapshots = await db.getAll(...docRefs);
|
|
111
|
-
snapshots.forEach((doc, i) => {
|
|
112
|
-
|
|
113
|
-
if (doc.exists) {
|
|
114
|
-
response[date][key] = doc.data();
|
|
115
|
-
} else {
|
|
116
|
-
response[date][key] = null;
|
|
117
|
-
}
|
|
118
|
-
});
|
|
119
|
-
}
|
|
120
|
-
} catch (error) {
|
|
121
|
-
logger.log('ERROR', 'API: Error fetching data from Firestore.', { errorMessage: error.message });
|
|
122
|
-
throw new Error('Failed to retrieve computation data.');
|
|
123
|
-
}
|
|
72
|
+
snapshots.forEach((doc, i) => { const key = keyPaths[i]; if (doc.exists) { response[date][key] = doc.data(); } else { response[date][key] = null; } }); }
|
|
73
|
+
} catch (error) { logger.log('ERROR', 'API: Error fetching data from Firestore.', { errorMessage: error.message }); throw new Error('Failed to retrieve computation data.'); }
|
|
124
74
|
return response;
|
|
125
75
|
};
|
|
126
76
|
|
|
@@ -129,31 +79,14 @@ const fetchUnifiedData = async (config, dependencies, calcKeys, dateStrings, cal
|
|
|
129
79
|
*/
|
|
130
80
|
const createApiHandler = (config, dependencies, calcMap) => {
|
|
131
81
|
const { logger } = dependencies;
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
const validationError = validateRequest(req.query, config);
|
|
135
|
-
if (validationError) {
|
|
136
|
-
logger.log('WARN', 'API Bad Request', { error: validationError, query: req.query });
|
|
137
|
-
return res.status(400).send({ status: 'error', message: validationError });
|
|
138
|
-
}
|
|
82
|
+
return async (req, res) => { const validationError = validateRequest(req.query, config);
|
|
83
|
+
if (validationError) { logger.log('WARN', 'API Bad Request', { error: validationError, query: req.query }); return res.status(400).send({ status: 'error', message: validationError }); }
|
|
139
84
|
try {
|
|
140
85
|
const computationKeys = req.query.computations.split(',');
|
|
141
86
|
const dateStrings = getDateStringsInRange(req.query.startDate, req.query.endDate);
|
|
142
87
|
const data = await fetchUnifiedData(config, dependencies, computationKeys, dateStrings, calcMap);
|
|
143
|
-
res.status(200).send({
|
|
144
|
-
|
|
145
|
-
metadata: {
|
|
146
|
-
computations: computationKeys,
|
|
147
|
-
startDate: req.query.startDate,
|
|
148
|
-
endDate: req.query.endDate,
|
|
149
|
-
},
|
|
150
|
-
data,
|
|
151
|
-
});
|
|
152
|
-
} catch (error) {
|
|
153
|
-
logger.log('ERROR', 'API processing failed.', { errorMessage: error.message, stack: error.stack });
|
|
154
|
-
res.status(500).send({ status: 'error', message: 'An internal error occurred.' });
|
|
155
|
-
}
|
|
156
|
-
};
|
|
88
|
+
res.status(200).send({ status: 'success', metadata: { computations: computationKeys, startDate: req.query.startDate, endDate: req.query.endDate, }, data, });
|
|
89
|
+
} catch (error) { logger.log('ERROR', 'API processing failed.', { errorMessage: error.message, stack: error.stack }); res.status(500).send({ status: 'error', message: 'An internal error occurred.' }); } };
|
|
157
90
|
};
|
|
158
91
|
|
|
159
92
|
/**
|
|
@@ -166,13 +99,9 @@ function createStructureSnippet(data, maxKeys = 20) {
|
|
|
166
99
|
if (typeof data === 'boolean') return true;
|
|
167
100
|
return data;
|
|
168
101
|
}
|
|
169
|
-
if (Array.isArray(data)) {
|
|
170
|
-
if (data.length === 0) return "<empty array>";
|
|
171
|
-
return [ createStructureSnippet(data[0], maxKeys) ];
|
|
172
|
-
}
|
|
102
|
+
if (Array.isArray(data)) { if (data.length === 0) return "<empty array>"; return [ createStructureSnippet(data[0], maxKeys) ]; }
|
|
173
103
|
const newObj = {};
|
|
174
|
-
const keys = Object.keys(data)
|
|
175
|
-
|
|
104
|
+
const keys = Object.keys(data)
|
|
176
105
|
if (keys.length > 0 && keys.every(k => k.match(/^[A-Z.]+$/) || k.includes('_') || k.match(/^[0-9]+$/))) {
|
|
177
106
|
const exampleKey = keys[0];
|
|
178
107
|
newObj[exampleKey] = createStructureSnippet(data[exampleKey], maxKeys);
|
|
@@ -184,10 +113,7 @@ function createStructureSnippet(data, maxKeys = 20) {
|
|
|
184
113
|
newObj[firstKey] = createStructureSnippet(data[firstKey], maxKeys);
|
|
185
114
|
newObj[`... (${keys.length - 1} more keys)`] = "<object>";
|
|
186
115
|
} else {
|
|
187
|
-
for (const key of keys) {
|
|
188
|
-
newObj[key] = createStructureSnippet(data[key], maxKeys);
|
|
189
|
-
}
|
|
190
|
-
}
|
|
116
|
+
for (const key of keys) { newObj[key] = createStructureSnippet(data[key], maxKeys); } }
|
|
191
117
|
return newObj;
|
|
192
118
|
}
|
|
193
119
|
|
|
@@ -198,40 +124,22 @@ async function getComputationStructure(computationName, calcMap, config, depende
|
|
|
198
124
|
const { db, logger } = dependencies;
|
|
199
125
|
try {
|
|
200
126
|
const pathInfo = calcMap[computationName];
|
|
201
|
-
if (!pathInfo) {
|
|
202
|
-
return { status: 'error', computation: computationName, message: `Computation not found in calculation map.` };
|
|
203
|
-
}
|
|
127
|
+
if (!pathInfo) { return { status: 'error', computation: computationName, message: `Computation not found in calculation map.` }; }
|
|
204
128
|
const { category } = pathInfo;
|
|
205
129
|
const insightsCollection = config.unifiedInsightsCollection || 'unified_insights';
|
|
206
130
|
const resultsSub = config.resultsSubcollection || 'results';
|
|
207
131
|
const compsSub = config.computationsSubcollection || 'computations';
|
|
208
132
|
const computationQueryPath = `${category}.${computationName}`;
|
|
209
|
-
|
|
210
|
-
const dateQuery = db.collection(insightsCollection)
|
|
211
|
-
.where(computationQueryPath, '==', true)
|
|
212
|
-
.orderBy(FieldPath.documentId(), 'desc')
|
|
213
|
-
.limit(1);
|
|
133
|
+
const dateQuery = db.collection(insightsCollection) .where(computationQueryPath, '==', true) .orderBy(FieldPath.documentId(), 'desc') .limit(1);
|
|
214
134
|
const dateSnapshot = await dateQuery.get();
|
|
215
|
-
if (dateSnapshot.empty) {
|
|
216
|
-
return { status: 'error', computation: computationName, message: `No computed data found. (Query path: ${computationQueryPath})` };
|
|
217
|
-
}
|
|
135
|
+
if (dateSnapshot.empty) { return { status: 'error', computation: computationName, message: `No computed data found. (Query path: ${computationQueryPath})` }; }
|
|
218
136
|
const latestStoredDate = dateSnapshot.docs[0].id;
|
|
219
|
-
const docRef = db.collection(insightsCollection).doc(latestStoredDate)
|
|
220
|
-
.collection(resultsSub).doc(category)
|
|
221
|
-
.collection(compsSub).doc(computationName);
|
|
137
|
+
const docRef = db.collection(insightsCollection).doc(latestStoredDate) .collection(resultsSub).doc(category) .collection(compsSub).doc(computationName);
|
|
222
138
|
const doc = await docRef.get();
|
|
223
|
-
if (!doc.exists) {
|
|
224
|
-
return { status: 'error', computation: computationName, message: `Summary flag was present for ${latestStoredDate} but doc is missing.` };
|
|
225
|
-
}
|
|
139
|
+
if (!doc.exists) { return { status: 'error', computation: computationName, message: `Summary flag was present for ${latestStoredDate} but doc is missing.` }; }
|
|
226
140
|
const fullData = doc.data();
|
|
227
141
|
const structureSnippet = createStructureSnippet(fullData);
|
|
228
|
-
return {
|
|
229
|
-
status: 'success',
|
|
230
|
-
computation: computationName,
|
|
231
|
-
category: category,
|
|
232
|
-
latestStoredDate: latestStoredDate,
|
|
233
|
-
structureSnippet: structureSnippet,
|
|
234
|
-
};
|
|
142
|
+
return { status: 'success', computation: computationName, category: category, latestStoredDate: latestStoredDate, structureSnippet: structureSnippet, };
|
|
235
143
|
} catch (error) {
|
|
236
144
|
logger.log('ERROR', `API /structure/${computationName} helper failed.`, { errorMessage: error.message });
|
|
237
145
|
return { status: 'error', computation: computationName, message: error.message };
|
|
@@ -244,17 +152,10 @@ async function getComputationStructure(computationName, calcMap, config, depende
|
|
|
244
152
|
*/
|
|
245
153
|
async function getDynamicSchema(CalcClass, calcName) {
|
|
246
154
|
if (CalcClass && typeof CalcClass.getSchema === 'function') {
|
|
247
|
-
try {
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
console.error(`Error running static getSchema() for ${calcName}: ${e.message}`);
|
|
251
|
-
return { "ERROR": `Failed to get static schema: ${e.message}` };
|
|
252
|
-
}
|
|
253
|
-
} else {
|
|
254
|
-
return { "ERROR": `Computation '${calcName}' does not have a static getSchema() method defined.` };
|
|
255
|
-
}
|
|
155
|
+
try { return CalcClass.getSchema();
|
|
156
|
+
} catch (e) { console.error(`Error running static getSchema() for ${calcName}: ${e.message}`); return { "ERROR": `Failed to get static schema: ${e.message}` };}
|
|
157
|
+
} else { return { "ERROR": `Computation '${calcName}' does not have a static getSchema() method defined.` }; }
|
|
256
158
|
}
|
|
257
|
-
// --- END UPDATED HARNESS ---
|
|
258
159
|
|
|
259
160
|
|
|
260
161
|
/**
|
|
@@ -263,58 +164,18 @@ async function getDynamicSchema(CalcClass, calcName) {
|
|
|
263
164
|
const createManifestHandler = (config, dependencies, calcMap) => {
|
|
264
165
|
const { db, logger } = dependencies;
|
|
265
166
|
const schemaCollection = config.schemaCollection || 'computation_schemas';
|
|
266
|
-
|
|
267
167
|
return async (req, res) => {
|
|
268
168
|
try {
|
|
269
169
|
logger.log('INFO', '[API /manifest] Fetching all computation schemas...');
|
|
270
170
|
const snapshot = await db.collection(schemaCollection).get();
|
|
271
|
-
if (snapshot.empty) {
|
|
272
|
-
logger.log('WARN', '[API /manifest] No schemas found in collection.');
|
|
273
|
-
return res.status(404).send({ status: 'error', message: 'No computation schemas have been generated yet.' });
|
|
274
|
-
}
|
|
275
|
-
|
|
171
|
+
if (snapshot.empty) { logger.log('WARN', '[API /manifest] No schemas found in collection.'); return res.status(404).send({ status: 'error', message: 'No computation schemas have been generated yet.' }); }
|
|
276
172
|
const manifest = {};
|
|
277
|
-
snapshot.forEach(doc => {
|
|
278
|
-
|
|
279
|
-
manifest[doc.id] = {
|
|
280
|
-
// --- CHANGED: Return the structure consistent with your file ---
|
|
281
|
-
category: data.category,
|
|
282
|
-
structure: data.schema, // Use 'structure' key
|
|
283
|
-
metadata: data.metadata,
|
|
284
|
-
lastUpdated: data.lastUpdated
|
|
285
|
-
};
|
|
286
|
-
});
|
|
287
|
-
|
|
288
|
-
res.status(200).send({
|
|
289
|
-
status: 'success',
|
|
290
|
-
// --- CHANGED: Use the structure from your file ---
|
|
291
|
-
summary: {
|
|
292
|
-
source: 'firestore_computation_schemas',
|
|
293
|
-
totalComputations: snapshot.size,
|
|
294
|
-
schemasAvailable: snapshot.size,
|
|
295
|
-
schemasFailed: 0,
|
|
296
|
-
lastUpdated: Math.max(...Object.values(manifest).map(m =>
|
|
297
|
-
m.lastUpdated ? m.lastUpdated.toMillis() : 0
|
|
298
|
-
))
|
|
299
|
-
},
|
|
300
|
-
manifest: manifest
|
|
301
|
-
});
|
|
302
|
-
|
|
173
|
+
snapshot.forEach(doc => { const data = doc.data(); manifest[doc.id] = { category: data.category, structure: data.schema, metadata: data.metadata, lastUpdated: data.lastUpdated }; });
|
|
174
|
+
res.status(200).send({ status: 'success', summary: { source: 'firestore_computation_schemas', totalComputations: snapshot.size, schemasAvailable: snapshot.size, schemasFailed: 0, lastUpdated: Math.max(...Object.values(manifest).map(m => m.lastUpdated ? m.lastUpdated.toMillis() : 0 )) }, manifest: manifest });
|
|
303
175
|
} catch (error) {
|
|
304
176
|
logger.log('ERROR', 'API /manifest handler failed.', { errorMessage: error.message, stack: error.stack });
|
|
305
|
-
res.status(500).send({ status: 'error', message: 'An internal error occurred.' });
|
|
306
|
-
}
|
|
307
|
-
};
|
|
177
|
+
res.status(500).send({ status: 'error', message: 'An internal error occurred.' }); } };
|
|
308
178
|
};
|
|
309
|
-
// --- END NEW HANDLER ---
|
|
310
179
|
|
|
311
180
|
|
|
312
|
-
module.exports = {
|
|
313
|
-
validateRequest,
|
|
314
|
-
buildCalculationMap,
|
|
315
|
-
fetchUnifiedData,
|
|
316
|
-
createApiHandler,
|
|
317
|
-
getComputationStructure,
|
|
318
|
-
getDynamicSchema,
|
|
319
|
-
createManifestHandler // <-- EXPORT NEW HANDLER
|
|
320
|
-
};
|
|
181
|
+
module.exports = { validateRequest, buildCalculationMap, fetchUnifiedData, createApiHandler, getComputationStructure, getDynamicSchema, createManifestHandler };
|