bulltrackers-module 1.0.731 → 1.0.733
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/functions/orchestrator/index.js +19 -17
- package/index.js +8 -29
- package/package.json +6 -5
- package/functions/computation-system/WorkflowOrchestrator.js +0 -213
- package/functions/computation-system/config/monitoring_config.js +0 -31
- package/functions/computation-system/config/validation_overrides.js +0 -10
- package/functions/computation-system/context/ContextFactory.js +0 -132
- package/functions/computation-system/context/ManifestBuilder.js +0 -379
- package/functions/computation-system/data/AvailabilityChecker.js +0 -236
- package/functions/computation-system/data/CachedDataLoader.js +0 -325
- package/functions/computation-system/data/DependencyFetcher.js +0 -455
- package/functions/computation-system/executors/MetaExecutor.js +0 -279
- package/functions/computation-system/executors/PriceBatchExecutor.js +0 -108
- package/functions/computation-system/executors/StandardExecutor.js +0 -465
- package/functions/computation-system/helpers/computation_dispatcher.js +0 -750
- package/functions/computation-system/helpers/computation_worker.js +0 -375
- package/functions/computation-system/helpers/monitor.js +0 -64
- package/functions/computation-system/helpers/on_demand_helpers.js +0 -154
- package/functions/computation-system/layers/extractors.js +0 -1097
- package/functions/computation-system/layers/index.js +0 -40
- package/functions/computation-system/layers/mathematics.js +0 -522
- package/functions/computation-system/layers/profiling.js +0 -537
- package/functions/computation-system/layers/validators.js +0 -170
- package/functions/computation-system/legacy/AvailabilityCheckerOld.js +0 -388
- package/functions/computation-system/legacy/CachedDataLoaderOld.js +0 -357
- package/functions/computation-system/legacy/DependencyFetcherOld.js +0 -478
- package/functions/computation-system/legacy/MetaExecutorold.js +0 -364
- package/functions/computation-system/legacy/StandardExecutorold.js +0 -476
- package/functions/computation-system/legacy/computation_dispatcherold.js +0 -944
- package/functions/computation-system/logger/logger.js +0 -297
- package/functions/computation-system/persistence/ContractValidator.js +0 -81
- package/functions/computation-system/persistence/FirestoreUtils.js +0 -56
- package/functions/computation-system/persistence/ResultCommitter.js +0 -283
- package/functions/computation-system/persistence/ResultsValidator.js +0 -130
- package/functions/computation-system/persistence/RunRecorder.js +0 -142
- package/functions/computation-system/persistence/StatusRepository.js +0 -52
- package/functions/computation-system/reporter_epoch.js +0 -6
- package/functions/computation-system/scripts/UpdateContracts.js +0 -128
- package/functions/computation-system/services/SnapshotService.js +0 -148
- package/functions/computation-system/simulation/Fabricator.js +0 -285
- package/functions/computation-system/simulation/SeededRandom.js +0 -41
- package/functions/computation-system/simulation/SimRunner.js +0 -51
- package/functions/computation-system/system_epoch.js +0 -2
- package/functions/computation-system/tools/BuildReporter.js +0 -531
- package/functions/computation-system/tools/ContractDiscoverer.js +0 -144
- package/functions/computation-system/tools/DeploymentValidator.js +0 -536
- package/functions/computation-system/tools/FinalSweepReporter.js +0 -322
- package/functions/computation-system/topology/HashManager.js +0 -55
- package/functions/computation-system/topology/ManifestLoader.js +0 -47
- package/functions/computation-system/utils/data_loader.js +0 -597
- package/functions/computation-system/utils/schema_capture.js +0 -121
- package/functions/computation-system/utils/utils.js +0 -188
|
@@ -11,6 +11,15 @@ const { CloudTasksClient } = require('@google-cloud/tasks');
|
|
|
11
11
|
// Initialize Cloud Tasks Client
|
|
12
12
|
const cloudTasksClient = new CloudTasksClient();
|
|
13
13
|
|
|
14
|
+
// --- HARDCODED CONFIGURATION ---
|
|
15
|
+
const CONFIG = {
|
|
16
|
+
projectId: 'stocks-12345',
|
|
17
|
+
location: 'us-central1',
|
|
18
|
+
queueName: 'task-engine-queue',
|
|
19
|
+
// Default Dispatcher URL (can be overridden by environment variable)
|
|
20
|
+
dispatcherUrl: process.env.DISPATCHER_URL || 'https://us-central1-stocks-12345.cloudfunctions.net/dispatcher'
|
|
21
|
+
};
|
|
22
|
+
|
|
14
23
|
/**
|
|
15
24
|
* ENTRY POINT: HTTP Handler for Workflow Interaction
|
|
16
25
|
*/
|
|
@@ -28,11 +37,10 @@ async function handleOrchestratorHttp(req, res, dependencies, config) {
|
|
|
28
37
|
throw new Error("Missing userType or date for PLAN action");
|
|
29
38
|
}
|
|
30
39
|
|
|
31
|
-
// Determine self-URL for callback
|
|
32
|
-
|
|
33
|
-
const
|
|
34
|
-
|
|
35
|
-
const orchestratorUrl = orchestratorUrlOverride || `https://${location}-${project}.cloudfunctions.net/orchestrator-http`;
|
|
40
|
+
// Determine self-URL for callback (Cloud Task needs to call this function back)
|
|
41
|
+
// We use the hardcoded location/project unless overridden
|
|
42
|
+
const orchestratorUrl = orchestratorUrlOverride ||
|
|
43
|
+
`https://${CONFIG.location}-${CONFIG.projectId}.cloudfunctions.net/orchestrator-http`;
|
|
36
44
|
|
|
37
45
|
const result = await planDailyUpdates(userType, date, windows || 10, config, dependencies, orchestratorUrl);
|
|
38
46
|
res.status(200).send(result);
|
|
@@ -68,13 +76,8 @@ async function handleOrchestratorHttp(req, res, dependencies, config) {
|
|
|
68
76
|
async function planDailyUpdates(userType, date, numberOfWindows, config, deps, orchestratorUrl) {
|
|
69
77
|
const { logger, db } = deps;
|
|
70
78
|
|
|
71
|
-
// Cloud Tasks Config
|
|
72
|
-
const
|
|
73
|
-
const location = process.env.GCP_LOCATION || 'us-central1';
|
|
74
|
-
const queue = process.env.GCP_QUEUE_NAME || 'orchestrator-queue'; // MUST EXIST in Cloud Console
|
|
75
|
-
|
|
76
|
-
// Construct the fully qualified queue name
|
|
77
|
-
const parentQueue = cloudTasksClient.queuePath(project, location, queue);
|
|
79
|
+
// Use Hardcoded Cloud Tasks Config
|
|
80
|
+
const parentQueue = cloudTasksClient.queuePath(CONFIG.projectId, CONFIG.location, CONFIG.queueName);
|
|
78
81
|
|
|
79
82
|
// 1. Get ALL targets
|
|
80
83
|
const now = new Date();
|
|
@@ -147,9 +150,10 @@ async function planDailyUpdates(userType, date, numberOfWindows, config, deps, o
|
|
|
147
150
|
url: orchestratorUrl,
|
|
148
151
|
headers: { 'Content-Type': 'application/json' },
|
|
149
152
|
body: Buffer.from(JSON.stringify(taskPayload)).toString('base64'),
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
+
// Use OIDC if service account is available in env, otherwise skip
|
|
154
|
+
oidcToken: process.env.GCP_SERVICE_ACCOUNT_EMAIL
|
|
155
|
+
? { serviceAccountEmail: process.env.GCP_SERVICE_ACCOUNT_EMAIL }
|
|
156
|
+
: undefined
|
|
153
157
|
},
|
|
154
158
|
scheduleTime: {
|
|
155
159
|
seconds: scheduleTimeSeconds
|
|
@@ -168,13 +172,11 @@ async function planDailyUpdates(userType, date, numberOfWindows, config, deps, o
|
|
|
168
172
|
// Dispatch Cloud Tasks
|
|
169
173
|
logger.log('INFO', `[Orchestrator Plan] Scheduling ${tasksToCreate.length} tasks for ${userType}...`);
|
|
170
174
|
|
|
171
|
-
// Create tasks in parallel (with limit if needed, but usually fine for <50 tasks)
|
|
172
175
|
const scheduleResults = await Promise.allSettled(tasksToCreate.map(req => cloudTasksClient.createTask(req)));
|
|
173
176
|
|
|
174
177
|
const failedCount = scheduleResults.filter(r => r.status === 'rejected').length;
|
|
175
178
|
if (failedCount > 0) {
|
|
176
179
|
logger.log('ERROR', `[Orchestrator Plan] Failed to schedule ${failedCount} tasks.`);
|
|
177
|
-
// Note: We don't rollback Firestore here; manual retry or idempotency handles it.
|
|
178
180
|
}
|
|
179
181
|
|
|
180
182
|
return {
|
package/index.js
CHANGED
|
@@ -30,24 +30,12 @@ const { handleDiscover } = require('./functions
|
|
|
30
30
|
const { handleVerify } = require('./functions/task-engine/helpers/verify_helpers');
|
|
31
31
|
const { handleUpdate } = require('./functions/task-engine/helpers/update_helpers');
|
|
32
32
|
|
|
33
|
-
//
|
|
34
|
-
const
|
|
35
|
-
const { dispatchComputationPass } = require('./functions/computation-system/helpers/computation_dispatcher');
|
|
36
|
-
const { handleComputationTask } = require('./functions/computation-system/helpers/computation_worker');
|
|
37
|
-
const {
|
|
38
|
-
requestBuildReport,
|
|
39
|
-
handleBuildReportTrigger,
|
|
40
|
-
generateBuildReport
|
|
41
|
-
} = require('./functions/computation-system/tools/BuildReporter');
|
|
42
|
-
|
|
43
|
-
const { checkPassStatus } = require('./functions/computation-system/helpers/monitor');
|
|
44
|
-
|
|
45
|
-
const dataLoader = require('./functions/computation-system/utils/data_loader');
|
|
46
|
-
const computationUtils = require('./functions/computation-system/utils/utils');
|
|
47
|
-
|
|
33
|
+
// --- COMPUTATION SYSTEM V2 (Replaces v1) ---
|
|
34
|
+
const computationSystemV2 = require('./functions/computation-system-v2/handlers/index');
|
|
48
35
|
|
|
49
36
|
const { createApiV2App } = require('./functions/api-v2/index');
|
|
50
37
|
|
|
38
|
+
// Maintenance & Backfills
|
|
51
39
|
const { fetchAndStoreInsights } = require('./functions/fetch-insights/helpers/handler_helpers');
|
|
52
40
|
const { fetchAndStorePrices } = require('./functions/etoro-price-fetcher/helpers/handler_helpers');
|
|
53
41
|
const { runSocialOrchestrator } = require('./functions/social-orchestrator/helpers/orchestrator_helpers');
|
|
@@ -103,16 +91,11 @@ const taskEngine = {
|
|
|
103
91
|
handleUpdate,
|
|
104
92
|
};
|
|
105
93
|
|
|
94
|
+
// Expose v2 handlers directly
|
|
106
95
|
const computationSystem = {
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
computationUtils,
|
|
111
|
-
buildManifest,
|
|
112
|
-
requestBuildReport,
|
|
113
|
-
handleBuildReportTrigger,
|
|
114
|
-
generateBuildReport,
|
|
115
|
-
checkPassStatus
|
|
96
|
+
computeScheduler: computationSystemV2.computeScheduler,
|
|
97
|
+
computeDispatcher: computationSystemV2.computeDispatcher,
|
|
98
|
+
computeOnDemand: computationSystemV2.computeOnDemand
|
|
116
99
|
};
|
|
117
100
|
|
|
118
101
|
const api = {
|
|
@@ -154,7 +137,7 @@ module.exports = {
|
|
|
154
137
|
if (require.main === module) {
|
|
155
138
|
const args = process.argv.slice(2);
|
|
156
139
|
|
|
157
|
-
// Check if this looks like a backfill command
|
|
140
|
+
// Check if this looks like a backfill command
|
|
158
141
|
const isBackfillCommand = args.some(arg =>
|
|
159
142
|
arg.startsWith('--startDate=') ||
|
|
160
143
|
arg.startsWith('--endDate=') ||
|
|
@@ -162,14 +145,10 @@ if (require.main === module) {
|
|
|
162
145
|
);
|
|
163
146
|
|
|
164
147
|
if (isBackfillCommand) {
|
|
165
|
-
// Route to backfill function
|
|
166
148
|
console.log('🚀 Starting backfill from main entry point...\n');
|
|
167
149
|
backfillTaskEngineData(null, null).catch(error => {
|
|
168
150
|
console.error('Fatal error:', error);
|
|
169
151
|
process.exit(1);
|
|
170
152
|
});
|
|
171
|
-
} else {
|
|
172
|
-
// No recognized command, just export (normal module behavior)
|
|
173
|
-
// This allows the file to still work as a module when imported
|
|
174
153
|
}
|
|
175
154
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "bulltrackers-module",
|
|
3
|
-
"version": "1.0.
|
|
3
|
+
"version": "1.0.733",
|
|
4
4
|
"description": "Helper Functions for Bulltrackers.",
|
|
5
5
|
"main": "index.js",
|
|
6
6
|
"files": [
|
|
@@ -38,24 +38,25 @@
|
|
|
38
38
|
"finance"
|
|
39
39
|
],
|
|
40
40
|
"dependencies": {
|
|
41
|
+
"@google-cloud/bigquery": "^7.9.4",
|
|
41
42
|
"@google-cloud/firestore": "^7.11.3",
|
|
42
43
|
"@google-cloud/monitoring": "latest",
|
|
43
44
|
"@google-cloud/pubsub": "latest",
|
|
45
|
+
"@google-cloud/storage": "^7.18.0",
|
|
46
|
+
"@google-cloud/tasks": "^5.0.0",
|
|
44
47
|
"aiden-shared-calculations-unified": "^1.0.110",
|
|
45
48
|
"cors": "^2.8.5",
|
|
46
49
|
"dotenv": "latest",
|
|
47
50
|
"express": "^4.19.2",
|
|
48
51
|
"express-rate-limit": "^8.2.1",
|
|
52
|
+
"firebase-admin": "^13.6.0",
|
|
49
53
|
"google-auth-library": "^10.5.0",
|
|
50
54
|
"graphviz": "latest",
|
|
51
55
|
"node-graphviz": "^0.1.1",
|
|
52
56
|
"p-limit": "^3.1.0",
|
|
53
57
|
"require-all": "^3.0.0",
|
|
54
58
|
"sharedsetup": "latest",
|
|
55
|
-
"zod": "^4.3.5"
|
|
56
|
-
"@google-cloud/storage": "^7.18.0",
|
|
57
|
-
"@google-cloud/bigquery": "^7.3.0",
|
|
58
|
-
"@google-cloud/tasks": "^5.0.0"
|
|
59
|
+
"zod": "^4.3.5"
|
|
59
60
|
},
|
|
60
61
|
"devDependencies": {
|
|
61
62
|
"bulltracker-deployer": "file:../bulltracker-deployer"
|
|
@@ -1,213 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* FILENAME: computation-system/WorkflowOrchestrator.js
|
|
3
|
-
* UPDATED: Fixed 'Version Mismatch' deadlock for historical chains.
|
|
4
|
-
* UPDATED: Added Force-Run logic for 'isTest' computations on current day.
|
|
5
|
-
*/
|
|
6
|
-
|
|
7
|
-
const { normalizeName, DEFINITIVE_EARLIEST_DATES } = require('./utils/utils');
|
|
8
|
-
const { checkRootDataAvailability, checkRootDependencies } = require('./data/AvailabilityChecker');
|
|
9
|
-
const { fetchExistingResults } = require('./data/DependencyFetcher');
|
|
10
|
-
const { fetchComputationStatus, updateComputationStatus } = require('./persistence/StatusRepository'); // Unused
|
|
11
|
-
const { StandardExecutor } = require('./executors/StandardExecutor');
|
|
12
|
-
const { MetaExecutor } = require('./executors/MetaExecutor');
|
|
13
|
-
|
|
14
|
-
const STATUS_IMPOSSIBLE_PREFIX = 'IMPOSSIBLE';
|
|
15
|
-
|
|
16
|
-
function groupByPass(manifest) {
|
|
17
|
-
const passes = {};
|
|
18
|
-
manifest.forEach(calc => {
|
|
19
|
-
if (!passes[calc.pass]) passes[calc.pass] = [];
|
|
20
|
-
passes[calc.pass].push(calc);
|
|
21
|
-
});
|
|
22
|
-
return passes;
|
|
23
|
-
}
|
|
24
|
-
|
|
25
|
-
function isDependencyReady(depName, isHistoricalSelf, currentStatusMap, prevStatusMap, manifestMap, storedStatus) {
|
|
26
|
-
const norm = normalizeName(depName);
|
|
27
|
-
const targetStatus = isHistoricalSelf ? (prevStatusMap ? prevStatusMap[norm] : null) : currentStatusMap[norm];
|
|
28
|
-
const depManifest = manifestMap.get(norm);
|
|
29
|
-
|
|
30
|
-
if (!targetStatus) return { ready: false, reason: 'Missing' };
|
|
31
|
-
if (String(targetStatus.hash).startsWith(STATUS_IMPOSSIBLE_PREFIX)) return { ready: false, reason: 'Impossible Upstream' };
|
|
32
|
-
|
|
33
|
-
if (depManifest && targetStatus.hash !== depManifest.hash) {
|
|
34
|
-
return { ready: false, reason: 'Dependency Version Mismatch' };
|
|
35
|
-
}
|
|
36
|
-
|
|
37
|
-
if (storedStatus && storedStatus.dependencyResultHashes) {
|
|
38
|
-
// FIX: Normalize the dependency name for lookup
|
|
39
|
-
const lastSeenResultHash = storedStatus.dependencyResultHashes[depName] || storedStatus.dependencyResultHashes[norm];
|
|
40
|
-
if (lastSeenResultHash && targetStatus.resultHash !== lastSeenResultHash) {
|
|
41
|
-
return { ready: true, dataChanged: true, reason: 'Dependency Data Update' };
|
|
42
|
-
}
|
|
43
|
-
}
|
|
44
|
-
|
|
45
|
-
return { ready: true, dataChanged: false };
|
|
46
|
-
}
|
|
47
|
-
|
|
48
|
-
function analyzeDateExecution(dateStr, calcsInPass, rootDataStatus, dailyStatus, manifestMap, prevDailyStatus = null) {
|
|
49
|
-
const report = { runnable: [], blocked: [], impossible: [], failedDependency: [], reRuns: [], skipped: [] };
|
|
50
|
-
const simulationStatus = { ...dailyStatus };
|
|
51
|
-
const isToday = dateStr === new Date().toISOString().slice(0, 10);
|
|
52
|
-
|
|
53
|
-
for (const calc of calcsInPass) {
|
|
54
|
-
const cName = normalizeName(calc.name);
|
|
55
|
-
const stored = simulationStatus[cName];
|
|
56
|
-
const currentHash = calc.hash;
|
|
57
|
-
|
|
58
|
-
// [NEW] Rule: 'isTest' computations always re-run on the current day.
|
|
59
|
-
// This ensures debug/test probes fire immediately to test system changes.
|
|
60
|
-
const shouldForceRun = isToday && (calc.isTest === true);
|
|
61
|
-
|
|
62
|
-
// 1. Root Data Check
|
|
63
|
-
const rootCheck = checkRootDependencies(calc, rootDataStatus);
|
|
64
|
-
if (!rootCheck.canRun) {
|
|
65
|
-
if (dateStr !== new Date().toISOString().slice(0, 10)) {
|
|
66
|
-
report.impossible.push({ name: cName, reason: `Missing Root: ${rootCheck.missing.join(', ')}` });
|
|
67
|
-
simulationStatus[cName] = { hash: `${STATUS_IMPOSSIBLE_PREFIX}:NO_DATA` };
|
|
68
|
-
} else {
|
|
69
|
-
report.blocked.push({ name: cName, reason: `Waiting for Root Data` });
|
|
70
|
-
}
|
|
71
|
-
continue;
|
|
72
|
-
}
|
|
73
|
-
|
|
74
|
-
// --- OPTIMIZATION: Early skip if code matches AND data is stable ---
|
|
75
|
-
// [UPDATED] We bypass this optimization if shouldForceRun is true
|
|
76
|
-
if (stored?.hash === currentHash && !shouldForceRun) {
|
|
77
|
-
let hasDataDrift = false;
|
|
78
|
-
let isBlocked = false;
|
|
79
|
-
let missingDeps = [];
|
|
80
|
-
|
|
81
|
-
if (calc.dependencies) {
|
|
82
|
-
for (const dep of calc.dependencies) {
|
|
83
|
-
const check = isDependencyReady(dep, false, simulationStatus, null, manifestMap, stored);
|
|
84
|
-
if (!check.ready) missingDeps.push(dep);
|
|
85
|
-
else if (check.dataChanged) { hasDataDrift = true; break; }
|
|
86
|
-
}
|
|
87
|
-
}
|
|
88
|
-
|
|
89
|
-
if (!hasDataDrift && missingDeps.length === 0 && calc.isHistorical) {
|
|
90
|
-
const yesterday = new Date(dateStr + 'T00:00:00Z');
|
|
91
|
-
yesterday.setUTCDate(yesterday.getUTCDate() - 1);
|
|
92
|
-
if (yesterday >= DEFINITIVE_EARLIEST_DATES.absoluteEarliest) {
|
|
93
|
-
const check = isDependencyReady(calc.name, true, null, prevDailyStatus, manifestMap, stored);
|
|
94
|
-
if (!check.ready) {
|
|
95
|
-
// [FIX]: Treat Missing OR Version Mismatch as Data Drift (Fresh Start)
|
|
96
|
-
if (check.reason === 'Missing' || check.reason === 'Dependency Version Mismatch') {
|
|
97
|
-
hasDataDrift = true;
|
|
98
|
-
} else {
|
|
99
|
-
isBlocked = true;
|
|
100
|
-
}
|
|
101
|
-
}
|
|
102
|
-
else if (check.dataChanged) hasDataDrift = true;
|
|
103
|
-
}
|
|
104
|
-
}
|
|
105
|
-
|
|
106
|
-
if (!hasDataDrift && !isBlocked && missingDeps.length === 0) {
|
|
107
|
-
report.skipped.push({ name: cName, reason: "Up To Date" });
|
|
108
|
-
continue;
|
|
109
|
-
}
|
|
110
|
-
}
|
|
111
|
-
// --- END OPTIMIZATION ---
|
|
112
|
-
|
|
113
|
-
const missingDeps = [];
|
|
114
|
-
let hasDataDrift = false;
|
|
115
|
-
let isBlocked = false;
|
|
116
|
-
|
|
117
|
-
if (calc.dependencies) {
|
|
118
|
-
for (const dep of calc.dependencies) {
|
|
119
|
-
const check = isDependencyReady(dep, false, simulationStatus, null, manifestMap, stored);
|
|
120
|
-
if (!check.ready) missingDeps.push(dep);
|
|
121
|
-
else if (check.dataChanged) hasDataDrift = true;
|
|
122
|
-
}
|
|
123
|
-
}
|
|
124
|
-
|
|
125
|
-
if (calc.isHistorical) {
|
|
126
|
-
const yesterday = new Date(dateStr + 'T00:00:00Z');
|
|
127
|
-
yesterday.setUTCDate(yesterday.getUTCDate() - 1);
|
|
128
|
-
if (yesterday >= DEFINITIVE_EARLIEST_DATES.absoluteEarliest) {
|
|
129
|
-
const check = isDependencyReady(calc.name, true, null, prevDailyStatus, manifestMap, stored);
|
|
130
|
-
if (!check.ready) {
|
|
131
|
-
// [FIX]: Allow fresh start if yesterday is Missing OR has different Code Version
|
|
132
|
-
if (check.reason === 'Missing' || check.reason === 'Dependency Version Mismatch') {
|
|
133
|
-
hasDataDrift = true;
|
|
134
|
-
} else {
|
|
135
|
-
isBlocked = true;
|
|
136
|
-
}
|
|
137
|
-
}
|
|
138
|
-
else if (check.dataChanged) hasDataDrift = true;
|
|
139
|
-
}
|
|
140
|
-
}
|
|
141
|
-
|
|
142
|
-
if (missingDeps.length > 0) {
|
|
143
|
-
const isImpossible = missingDeps.some(d => String(simulationStatus[normalizeName(d)]?.hash).startsWith(STATUS_IMPOSSIBLE_PREFIX));
|
|
144
|
-
if (isImpossible) {
|
|
145
|
-
report.impossible.push({ name: cName, reason: 'Upstream Impossible' });
|
|
146
|
-
simulationStatus[cName] = { hash: `${STATUS_IMPOSSIBLE_PREFIX}:UPSTREAM` };
|
|
147
|
-
} else {
|
|
148
|
-
report.failedDependency.push({ name: cName, missing: missingDeps });
|
|
149
|
-
}
|
|
150
|
-
continue;
|
|
151
|
-
}
|
|
152
|
-
|
|
153
|
-
if (isBlocked) {
|
|
154
|
-
report.blocked.push({ name: cName, reason: 'Waiting for Yesterday' });
|
|
155
|
-
continue;
|
|
156
|
-
}
|
|
157
|
-
|
|
158
|
-
const currentDependencyResultHashes = {};
|
|
159
|
-
if (calc.dependencies) {
|
|
160
|
-
calc.dependencies.forEach(d => {
|
|
161
|
-
const resHash = simulationStatus[normalizeName(d)]?.resultHash;
|
|
162
|
-
if (resHash) currentDependencyResultHashes[d] = resHash;
|
|
163
|
-
});
|
|
164
|
-
}
|
|
165
|
-
|
|
166
|
-
const taskPayload = { name: cName, dependencyResultHashes: currentDependencyResultHashes };
|
|
167
|
-
|
|
168
|
-
if (!stored?.hash) {
|
|
169
|
-
report.runnable.push({ ...taskPayload, reason: "New Calculation" });
|
|
170
|
-
} else if (stored.hash !== currentHash) {
|
|
171
|
-
report.reRuns.push({ ...taskPayload, oldHash: stored.hash, newHash: currentHash, reason: "Hash Mismatch" });
|
|
172
|
-
} else if (shouldForceRun) {
|
|
173
|
-
// [NEW] Logic to handle the forced run for Test probes
|
|
174
|
-
report.reRuns.push({ ...taskPayload, oldHash: stored.hash, newHash: currentHash, reason: "Test Computation (Always Run Today)" });
|
|
175
|
-
} else if (hasDataDrift) {
|
|
176
|
-
report.runnable.push({ ...taskPayload, reason: "Input Data Changed" });
|
|
177
|
-
}
|
|
178
|
-
}
|
|
179
|
-
return report;
|
|
180
|
-
}
|
|
181
|
-
|
|
182
|
-
async function executeDispatchTask(dateStr, pass, targetComputation, config, dependencies, computationManifest, previousCategory = null, dependencyResultHashes = {}, metadata = {}) {
|
|
183
|
-
const { logger } = dependencies;
|
|
184
|
-
const manifestMap = new Map(computationManifest.map(c => [normalizeName(c.name), c]));
|
|
185
|
-
const calcManifest = manifestMap.get(normalizeName(targetComputation));
|
|
186
|
-
|
|
187
|
-
if (!calcManifest) throw new Error(`Calc '${targetComputation}' not found.`);
|
|
188
|
-
|
|
189
|
-
// Merge runtime metadata (like targetCid) into the manifest
|
|
190
|
-
// This allows the Executor to access it via 'calcInstance.manifest'
|
|
191
|
-
Object.assign(calcManifest, metadata);
|
|
192
|
-
|
|
193
|
-
calcManifest.dependencyResultHashes = dependencyResultHashes;
|
|
194
|
-
|
|
195
|
-
const rootData = await checkRootDataAvailability(dateStr, config, dependencies, DEFINITIVE_EARLIEST_DATES);
|
|
196
|
-
const calcsToRun = [calcManifest];
|
|
197
|
-
|
|
198
|
-
const existingResults = await fetchExistingResults(dateStr, calcsToRun, computationManifest, config, dependencies, false);
|
|
199
|
-
let previousResults = {};
|
|
200
|
-
if (calcManifest.isHistorical) {
|
|
201
|
-
const prev = new Date(dateStr + 'T00:00:00Z'); prev.setUTCDate(prev.getUTCDate() - 1);
|
|
202
|
-
previousResults = await fetchExistingResults(prev.toISOString().slice(0, 10), calcsToRun, computationManifest, config, dependencies, true);
|
|
203
|
-
}
|
|
204
|
-
|
|
205
|
-
const execDate = new Date(dateStr + 'T00:00:00Z');
|
|
206
|
-
const updates = (calcManifest.type === 'standard')
|
|
207
|
-
? await StandardExecutor.run(execDate, calcsToRun, `Pass ${pass}`, config, dependencies, rootData, existingResults, previousResults)
|
|
208
|
-
: await MetaExecutor.run(execDate, calcsToRun, `Pass ${pass}`, config, dependencies, rootData, existingResults, previousResults);
|
|
209
|
-
|
|
210
|
-
return { date: dateStr, updates };
|
|
211
|
-
}
|
|
212
|
-
|
|
213
|
-
module.exports = { executeDispatchTask, analyzeDateExecution, groupByPass };
|
|
@@ -1,31 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* computation-system/config/monitoring_config.js
|
|
3
|
-
* Configuration for Google Cloud Monitoring, Logging, and Tracing.
|
|
4
|
-
*/
|
|
5
|
-
|
|
6
|
-
module.exports = {
|
|
7
|
-
// Toggle to easily disable all GCP telemetry without changing code
|
|
8
|
-
enabled: true,
|
|
9
|
-
|
|
10
|
-
// Your Google Cloud Project ID
|
|
11
|
-
project: process.env.GOOGLE_CLOUD_PROJECT_ID || process.env.GCP_PROJECT || 'your-project-id',
|
|
12
|
-
|
|
13
|
-
// Configuration for the "Eyeball" (Trace)
|
|
14
|
-
trace: {
|
|
15
|
-
enabled: true,
|
|
16
|
-
// Force sampling for these computations (can use '*' for all)
|
|
17
|
-
sampledTasks: ['*']
|
|
18
|
-
},
|
|
19
|
-
|
|
20
|
-
// Definitions for Custom Metrics
|
|
21
|
-
metrics: {
|
|
22
|
-
// The heartbeat metric replaces the Firestore 'active' check over time
|
|
23
|
-
heartbeat: 'custom.googleapis.com/computation/worker_heartbeat',
|
|
24
|
-
|
|
25
|
-
// Tracks memory usage per worker/task
|
|
26
|
-
memory: 'custom.googleapis.com/computation/memory_usage',
|
|
27
|
-
|
|
28
|
-
// Tracks duration of specific processing stages
|
|
29
|
-
duration: 'custom.googleapis.com/computation/stage_duration'
|
|
30
|
-
}
|
|
31
|
-
};
|
|
@@ -1,10 +0,0 @@
|
|
|
1
|
-
module.exports = {
|
|
2
|
-
// Only add calculations here if the HeuristicValidator is too aggressive
|
|
3
|
-
// EXAMPLES :
|
|
4
|
-
// "bankruptcy-detector": { maxZeroPct: 100 }, // It's rare, so 100% 0s is fine
|
|
5
|
-
// "sparse-signal-generator": { maxNullPct: 99 }
|
|
6
|
-
|
|
7
|
-
"instrument-price-change-1d": { maxZeroPct: 100 }, // Because weekeends/holidays return 0 change, technically crypto means this can't hit 100% but it's usually quite close, so we override
|
|
8
|
-
"instrument-price-momentum-20d ": { maxZeroPct: 100 }, // Some assets can be very stagnant over a month, especially bonds or stablecoins
|
|
9
|
-
"mimetic-latency": { maxFlatlinePct: 100 },
|
|
10
|
-
};
|
|
@@ -1,132 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* @fileoverview Factory for creating the Computation Context.
|
|
3
|
-
* UPDATED: Injects verification and rankings data into context globally and locally.
|
|
4
|
-
* UPDATED: Added support for historical ranking data in both Standard and Meta contexts.
|
|
5
|
-
* UPDATED: Added support for 'series' data (historical root data or computation results) in Global Data.
|
|
6
|
-
*/
|
|
7
|
-
const mathLayer = require('../layers/index');
|
|
8
|
-
const { LEGACY_MAPPING } = require('../topology/HashManager');
|
|
9
|
-
|
|
10
|
-
class ContextFactory {
|
|
11
|
-
static buildMathContext() {
|
|
12
|
-
const mathContext = {};
|
|
13
|
-
for (const [key, value] of Object.entries(mathLayer)) {
|
|
14
|
-
mathContext[key] = value;
|
|
15
|
-
const legacyKey = LEGACY_MAPPING[key];
|
|
16
|
-
if (legacyKey) { mathContext[legacyKey] = value; }
|
|
17
|
-
}
|
|
18
|
-
return mathContext;
|
|
19
|
-
}
|
|
20
|
-
|
|
21
|
-
static buildPerUserContext(options) {
|
|
22
|
-
const {
|
|
23
|
-
todayPortfolio,
|
|
24
|
-
yesterdayPortfolio,
|
|
25
|
-
todayHistory,
|
|
26
|
-
yesterdayHistory,
|
|
27
|
-
userId,
|
|
28
|
-
userType,
|
|
29
|
-
dateStr,
|
|
30
|
-
metadata,
|
|
31
|
-
mappings,
|
|
32
|
-
insights,
|
|
33
|
-
socialData,
|
|
34
|
-
computedDependencies,
|
|
35
|
-
previousComputedDependencies,
|
|
36
|
-
config,
|
|
37
|
-
deps,
|
|
38
|
-
verification,
|
|
39
|
-
rankings,
|
|
40
|
-
yesterdayRankings,
|
|
41
|
-
allRankings,
|
|
42
|
-
allRankingsYesterday,
|
|
43
|
-
allVerifications,
|
|
44
|
-
ratings,
|
|
45
|
-
pageViews,
|
|
46
|
-
watchlistMembership,
|
|
47
|
-
alertHistory,
|
|
48
|
-
piMasterList,
|
|
49
|
-
seriesData
|
|
50
|
-
} = options;
|
|
51
|
-
|
|
52
|
-
return {
|
|
53
|
-
user: {
|
|
54
|
-
id: userId,
|
|
55
|
-
type: userType,
|
|
56
|
-
portfolio: { today: todayPortfolio, yesterday: yesterdayPortfolio },
|
|
57
|
-
history: { today: todayHistory, yesterday: yesterdayHistory },
|
|
58
|
-
verification: verification || null,
|
|
59
|
-
rankEntry: rankings || null,
|
|
60
|
-
rankEntryYesterday: yesterdayRankings || null
|
|
61
|
-
},
|
|
62
|
-
date: { today: dateStr },
|
|
63
|
-
insights: { today: insights?.today, yesterday: insights?.yesterday },
|
|
64
|
-
social: { today: socialData?.today, yesterday: socialData?.yesterday },
|
|
65
|
-
mappings: mappings || {},
|
|
66
|
-
math: ContextFactory.buildMathContext(),
|
|
67
|
-
computed: computedDependencies || {},
|
|
68
|
-
previousComputed: previousComputedDependencies || {},
|
|
69
|
-
meta: metadata, config, deps,
|
|
70
|
-
globalData: {
|
|
71
|
-
rankings: allRankings || [],
|
|
72
|
-
rankingsYesterday: allRankingsYesterday || [],
|
|
73
|
-
verifications: allVerifications || {},
|
|
74
|
-
ratings: ratings || {},
|
|
75
|
-
pageViews: pageViews || {},
|
|
76
|
-
watchlistMembership: watchlistMembership || {},
|
|
77
|
-
alertHistory: alertHistory || {},
|
|
78
|
-
piMasterList: piMasterList || {},
|
|
79
|
-
series: seriesData || {}
|
|
80
|
-
}
|
|
81
|
-
};
|
|
82
|
-
}
|
|
83
|
-
|
|
84
|
-
static buildMetaContext(options) {
|
|
85
|
-
const {
|
|
86
|
-
dateStr,
|
|
87
|
-
metadata,
|
|
88
|
-
mappings,
|
|
89
|
-
insights,
|
|
90
|
-
socialData,
|
|
91
|
-
prices,
|
|
92
|
-
computedDependencies,
|
|
93
|
-
previousComputedDependencies,
|
|
94
|
-
config,
|
|
95
|
-
deps,
|
|
96
|
-
allRankings,
|
|
97
|
-
allRankingsYesterday,
|
|
98
|
-
allVerifications,
|
|
99
|
-
ratings,
|
|
100
|
-
pageViews,
|
|
101
|
-
watchlistMembership,
|
|
102
|
-
alertHistory,
|
|
103
|
-
seriesData,
|
|
104
|
-
piMasterList
|
|
105
|
-
} = options;
|
|
106
|
-
|
|
107
|
-
return {
|
|
108
|
-
date: { today: dateStr },
|
|
109
|
-
insights: { today: insights?.today, yesterday: insights?.yesterday },
|
|
110
|
-
social: { today: socialData?.today, yesterday: socialData?.yesterday },
|
|
111
|
-
prices: prices || {},
|
|
112
|
-
mappings: mappings || {},
|
|
113
|
-
math: ContextFactory.buildMathContext(),
|
|
114
|
-
computed: computedDependencies || {},
|
|
115
|
-
previousComputed: previousComputedDependencies || {},
|
|
116
|
-
meta: metadata, config, deps,
|
|
117
|
-
globalData: {
|
|
118
|
-
rankings: allRankings || [],
|
|
119
|
-
rankingsYesterday: allRankingsYesterday || [],
|
|
120
|
-
verifications: allVerifications || {},
|
|
121
|
-
ratings: ratings || {},
|
|
122
|
-
pageViews: pageViews || {},
|
|
123
|
-
watchlistMembership: watchlistMembership || {},
|
|
124
|
-
alertHistory: alertHistory || {},
|
|
125
|
-
series: seriesData || {},
|
|
126
|
-
piMasterList: piMasterList || {} // <--- ADDED TO GLOBAL DATA
|
|
127
|
-
}
|
|
128
|
-
};
|
|
129
|
-
}
|
|
130
|
-
}
|
|
131
|
-
|
|
132
|
-
module.exports = { ContextFactory };
|