bulltrackers-module 1.0.213 → 1.0.215
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/functions/computation-system/controllers/computation_controller.js +25 -82
- package/functions/computation-system/helpers/computation_dispatcher.js +11 -24
- package/functions/computation-system/helpers/computation_manifest_builder.js +52 -89
- package/functions/computation-system/helpers/computation_pass_runner.js +23 -63
- package/functions/computation-system/helpers/computation_worker.js +11 -53
- package/functions/computation-system/helpers/orchestration_helpers.js +89 -208
- package/functions/task-engine/helpers/update_helpers.js +34 -70
- package/package.json +1 -1
- package/functions/computation-system/layers/math_primitives.js +0 -744
|
@@ -4,8 +4,7 @@
|
|
|
4
4
|
*/
|
|
5
5
|
|
|
6
6
|
// Load all layers dynamically from the index
|
|
7
|
-
const mathLayer
|
|
8
|
-
|
|
7
|
+
const mathLayer = require('../layers/index');
|
|
9
8
|
const { loadDailyInsights, loadDailySocialPostInsights, getRelevantShardRefs, getPriceShardRefs } = require('../utils/data_loader');
|
|
10
9
|
|
|
11
10
|
// Legacy Keys Mapping (Ensures backward compatibility with existing Calculations)
|
|
@@ -26,76 +25,20 @@ const LEGACY_MAPPING = {
|
|
|
26
25
|
};
|
|
27
26
|
|
|
28
27
|
class DataLoader {
|
|
29
|
-
constructor(config, dependencies) {
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
}
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
async loadMappings() {
|
|
38
|
-
if (this.cache.mappings) return this.cache.mappings;
|
|
39
|
-
const { calculationUtils } = this.deps;
|
|
40
|
-
this.cache.mappings = await calculationUtils.loadInstrumentMappings();
|
|
41
|
-
return this.cache.mappings;
|
|
42
|
-
}
|
|
43
|
-
async loadInsights(dateStr) {
|
|
44
|
-
if (this.cache.insights.has(dateStr)) return this.cache.insights.get(dateStr);
|
|
45
|
-
const insights = await loadDailyInsights(this.config, this.deps, dateStr);
|
|
46
|
-
this.cache.insights.set(dateStr, insights);
|
|
47
|
-
return insights;
|
|
48
|
-
}
|
|
49
|
-
async loadSocial(dateStr) {
|
|
50
|
-
if (this.cache.social.has(dateStr)) return this.cache.social.get(dateStr);
|
|
51
|
-
const social = await loadDailySocialPostInsights(this.config, this.deps, dateStr);
|
|
52
|
-
this.cache.social.set(dateStr, social);
|
|
53
|
-
return social;
|
|
54
|
-
}
|
|
55
|
-
|
|
56
|
-
async getPriceShardReferences() {
|
|
57
|
-
return getPriceShardRefs(this.config, this.deps);
|
|
58
|
-
}
|
|
59
|
-
|
|
60
|
-
async getSpecificPriceShardReferences(targetInstrumentIds) {
|
|
61
|
-
return getRelevantShardRefs(this.config, this.deps, targetInstrumentIds);
|
|
62
|
-
}
|
|
63
|
-
|
|
64
|
-
async loadPriceShard(docRef) {
|
|
65
|
-
try {
|
|
66
|
-
const snap = await docRef.get();
|
|
67
|
-
if (!snap.exists) return {};
|
|
68
|
-
return snap.data();
|
|
69
|
-
} catch (e) {
|
|
70
|
-
console.error(`Error loading shard ${docRef.path}:`, e);
|
|
71
|
-
return {};
|
|
72
|
-
}
|
|
73
|
-
}
|
|
28
|
+
constructor(config, dependencies) { this.config = config; this.deps = dependencies; this.cache = { mappings: null, insights: new Map(), social: new Map(), prices: null }; }
|
|
29
|
+
get mappings() { return this.cache.mappings; }
|
|
30
|
+
async loadMappings() { if (this.cache.mappings) return this.cache.mappings; const { calculationUtils } = this.deps; this.cache.mappings = await calculationUtils.loadInstrumentMappings(); return this.cache.mappings; }
|
|
31
|
+
async loadInsights(dateStr) { if (this.cache.insights.has(dateStr)) return this.cache.insights.get(dateStr); const insights = await loadDailyInsights(this.config, this.deps, dateStr); this.cache.insights.set(dateStr, insights); return insights; }
|
|
32
|
+
async loadSocial(dateStr) { if (this.cache.social.has(dateStr)) return this.cache.social.get(dateStr); const social = await loadDailySocialPostInsights(this.config, this.deps, dateStr); this.cache.social.set(dateStr, social); return social; }
|
|
33
|
+
async getPriceShardReferences() { return getPriceShardRefs(this.config, this.deps); }
|
|
34
|
+
async getSpecificPriceShardReferences (targetInstrumentIds) { return getRelevantShardRefs(this.config, this.deps, targetInstrumentIds); }
|
|
35
|
+
async loadPriceShard(docRef) { try { const snap = await docRef.get(); if (!snap.exists) return {}; return snap.data(); } catch (e) { console.error(`Error loading shard ${docRef.path}:`, e); return {}; } }
|
|
74
36
|
}
|
|
75
37
|
|
|
76
38
|
class ContextBuilder {
|
|
77
|
-
|
|
78
|
-
/**
|
|
79
|
-
* dynamically constructs the 'math' object.
|
|
80
|
-
* 1. Iterates over all exports from layers/index.js
|
|
81
|
-
* 2. Maps standard classes to legacy keys (extract, compute, etc.)
|
|
82
|
-
* 3. Adds ALL classes by their actual name to support new features automatically.
|
|
83
|
-
*/
|
|
84
39
|
static buildMathContext() {
|
|
85
40
|
const mathContext = {};
|
|
86
|
-
|
|
87
|
-
// 1. Auto-discover and map
|
|
88
|
-
for (const [key, value] of Object.entries(mathLayer)) {
|
|
89
|
-
// Add by actual name (e.g. math.NewFeature)
|
|
90
|
-
mathContext[key] = value;
|
|
91
|
-
|
|
92
|
-
// Map to legacy key if exists (e.g. math.extract)
|
|
93
|
-
const legacyKey = LEGACY_MAPPING[key];
|
|
94
|
-
if (legacyKey) {
|
|
95
|
-
mathContext[legacyKey] = value;
|
|
96
|
-
}
|
|
97
|
-
}
|
|
98
|
-
|
|
41
|
+
for (const [key, value] of Object.entries(mathLayer)) { mathContext[key] = value; const legacyKey = LEGACY_MAPPING[key]; if (legacyKey) { mathContext[legacyKey] = value; } }
|
|
99
42
|
return mathContext;
|
|
100
43
|
}
|
|
101
44
|
|
|
@@ -107,7 +50,7 @@ class ContextBuilder {
|
|
|
107
50
|
insights: { today: insights?.today, yesterday: insights?.yesterday },
|
|
108
51
|
social: { today: socialData?.today, yesterday: socialData?.yesterday },
|
|
109
52
|
mappings: mappings || {},
|
|
110
|
-
math: ContextBuilder.buildMathContext(),
|
|
53
|
+
math: ContextBuilder.buildMathContext(),
|
|
111
54
|
computed: computedDependencies || {},
|
|
112
55
|
previousComputed: previousComputedDependencies || {},
|
|
113
56
|
meta: metadata, config, deps
|
|
@@ -122,7 +65,7 @@ class ContextBuilder {
|
|
|
122
65
|
social: { today: socialData?.today, yesterday: socialData?.yesterday },
|
|
123
66
|
prices: prices || {},
|
|
124
67
|
mappings: mappings || {},
|
|
125
|
-
math: ContextBuilder.buildMathContext(),
|
|
68
|
+
math: ContextBuilder.buildMathContext(),
|
|
126
69
|
computed: computedDependencies || {},
|
|
127
70
|
previousComputed: previousComputedDependencies || {},
|
|
128
71
|
meta: metadata, config, deps
|
|
@@ -133,23 +76,23 @@ class ContextBuilder {
|
|
|
133
76
|
class ComputationExecutor {
|
|
134
77
|
constructor(config, dependencies, dataLoader) {
|
|
135
78
|
this.config = config;
|
|
136
|
-
this.deps
|
|
79
|
+
this.deps = dependencies;
|
|
137
80
|
this.loader = dataLoader;
|
|
138
81
|
}
|
|
139
82
|
|
|
140
83
|
async executePerUser(calcInstance, metadata, dateStr, portfolioData, yesterdayPortfolioData, historyData, computedDeps, prevDeps) {
|
|
141
|
-
const { logger }
|
|
84
|
+
const { logger } = this.deps;
|
|
142
85
|
const targetUserType = metadata.userType;
|
|
143
|
-
const mappings
|
|
144
|
-
const insights
|
|
86
|
+
const mappings = await this.loader.loadMappings();
|
|
87
|
+
const insights = metadata.rootDataDependencies?.includes('insights') ? { today: await this.loader.loadInsights(dateStr) } : null;
|
|
145
88
|
|
|
146
89
|
// Access SCHEMAS dynamically from the loaded layer
|
|
147
90
|
const SCHEMAS = mathLayer.SCHEMAS;
|
|
148
91
|
|
|
149
92
|
for (const [userId, todayPortfolio] of Object.entries(portfolioData)) {
|
|
150
93
|
const yesterdayPortfolio = yesterdayPortfolioData ? yesterdayPortfolioData[userId] : null;
|
|
151
|
-
const todayHistory
|
|
152
|
-
const actualUserType
|
|
94
|
+
const todayHistory = historyData ? historyData[userId] : null;
|
|
95
|
+
const actualUserType = todayPortfolio.PublicPositions ? SCHEMAS.USER_TYPES.SPECULATOR : SCHEMAS.USER_TYPES.NORMAL;
|
|
153
96
|
if (targetUserType !== 'all') {
|
|
154
97
|
const mappedTarget = (targetUserType === 'speculator') ? SCHEMAS.USER_TYPES.SPECULATOR : SCHEMAS.USER_TYPES.NORMAL;
|
|
155
98
|
if (mappedTarget !== actualUserType) continue;
|
|
@@ -160,10 +103,10 @@ class ComputationExecutor {
|
|
|
160
103
|
}
|
|
161
104
|
|
|
162
105
|
async executeOncePerDay(calcInstance, metadata, dateStr, computedDeps, prevDeps) {
|
|
163
|
-
const mappings
|
|
106
|
+
const mappings = await this.loader.loadMappings();
|
|
164
107
|
const { logger } = this.deps;
|
|
165
|
-
const insights
|
|
166
|
-
const social
|
|
108
|
+
const insights = metadata.rootDataDependencies?.includes('insights') ? { today: await this.loader.loadInsights(dateStr) } : null;
|
|
109
|
+
const social = metadata.rootDataDependencies?.includes('social') ? { today: await this.loader.loadSocial(dateStr) } : null;
|
|
167
110
|
|
|
168
111
|
if (metadata.rootDataDependencies?.includes('price')) {
|
|
169
112
|
logger.log('INFO', `[Executor] Running Batched/Sharded Execution for ${metadata.name}`);
|
|
@@ -171,7 +114,7 @@ class ComputationExecutor {
|
|
|
171
114
|
if (shardRefs.length === 0) { logger.log('WARN', '[Executor] No price shards found.'); return {}; }
|
|
172
115
|
let processedCount = 0;
|
|
173
116
|
for (const ref of shardRefs) {
|
|
174
|
-
const shardData
|
|
117
|
+
const shardData = await this.loader.loadPriceShard(ref);
|
|
175
118
|
const partialContext = ContextBuilder.buildMetaContext({ dateStr, metadata, mappings, insights, socialData: social, prices: { history: shardData }, computedDependencies: computedDeps, previousComputedDependencies: prevDeps, config: this.config, deps: this.deps });
|
|
176
119
|
await calcInstance.process(partialContext);
|
|
177
120
|
partialContext.prices = null;
|
|
@@ -189,9 +132,9 @@ class ComputationExecutor {
|
|
|
189
132
|
|
|
190
133
|
class ComputationController {
|
|
191
134
|
constructor(config, dependencies) {
|
|
192
|
-
this.config
|
|
193
|
-
this.deps
|
|
194
|
-
this.loader
|
|
135
|
+
this.config = config;
|
|
136
|
+
this.deps = dependencies;
|
|
137
|
+
this.loader = new DataLoader(config, dependencies);
|
|
195
138
|
this.executor = new ComputationExecutor(config, dependencies, this.loader);
|
|
196
139
|
}
|
|
197
140
|
}
|
|
@@ -17,53 +17,42 @@ const TOPIC_NAME = 'computation-tasks';
|
|
|
17
17
|
*/
|
|
18
18
|
async function dispatchComputationPass(config, dependencies, computationManifest) {
|
|
19
19
|
const { logger } = dependencies;
|
|
20
|
-
|
|
21
|
-
// Create fresh PubSubUtils instance
|
|
22
20
|
const pubsubUtils = new PubSubUtils(dependencies);
|
|
23
|
-
|
|
24
21
|
const passToRun = String(config.COMPUTATION_PASS_TO_RUN);
|
|
25
|
-
|
|
26
|
-
if (!passToRun) {
|
|
27
|
-
return logger.log('ERROR', '[Dispatcher] No pass defined (COMPUTATION_PASS_TO_RUN). Aborting.');
|
|
28
|
-
}
|
|
22
|
+
if (!passToRun) { return logger.log('ERROR', '[Dispatcher] No pass defined (COMPUTATION_PASS_TO_RUN). Aborting.'); }
|
|
29
23
|
|
|
30
24
|
// 1. Validate Pass Existence
|
|
31
25
|
const passes = groupByPass(computationManifest);
|
|
32
26
|
const calcsInThisPass = passes[passToRun] || [];
|
|
33
27
|
|
|
34
|
-
if (!calcsInThisPass.length) {
|
|
35
|
-
return logger.log('WARN', `[Dispatcher] No calcs for Pass ${passToRun}. Exiting.`);
|
|
36
|
-
}
|
|
28
|
+
if (!calcsInThisPass.length) { return logger.log('WARN', `[Dispatcher] No calcs for Pass ${passToRun}. Exiting.`); }
|
|
37
29
|
|
|
38
30
|
const calcNames = calcsInThisPass.map(c => c.name).join(', ');
|
|
39
31
|
logger.log('INFO', `🚀 [Dispatcher] Preparing PASS ${passToRun}.`);
|
|
40
32
|
logger.log('INFO', `[Dispatcher] Included Calculations: [${calcNames}]`);
|
|
41
33
|
|
|
42
34
|
// 2. Determine Date Range
|
|
43
|
-
// Hardcoded earliest dates - keep synced with PassRunner for now
|
|
44
35
|
const earliestDates = {
|
|
45
36
|
portfolio: new Date('2025-09-25T00:00:00Z'),
|
|
46
|
-
history:
|
|
47
|
-
social:
|
|
48
|
-
insights:
|
|
49
|
-
price:
|
|
37
|
+
history: new Date('2025-11-05T00:00:00Z'),
|
|
38
|
+
social: new Date('2025-10-30T00:00:00Z'),
|
|
39
|
+
insights: new Date('2025-08-26T00:00:00Z'),
|
|
40
|
+
price: new Date('2025-08-01T00:00:00Z')
|
|
50
41
|
};
|
|
51
|
-
const passEarliestDate = Object.values(earliestDates).reduce((a, b) => a < b ? a : b);
|
|
52
|
-
const endDateUTC = new Date(Date.UTC(new Date().getUTCFullYear(), new Date().getUTCMonth(), new Date().getUTCDate() - 1));
|
|
53
42
|
|
|
43
|
+
const passEarliestDate = Object.values(earliestDates).reduce((a, b) => a < b ? a : b);
|
|
44
|
+
const endDateUTC = new Date(Date.UTC(new Date().getUTCFullYear(), new Date().getUTCMonth(), new Date().getUTCDate() - 1));
|
|
54
45
|
const allExpectedDates = getExpectedDateStrings(passEarliestDate, endDateUTC);
|
|
55
46
|
|
|
56
47
|
logger.log('INFO', `[Dispatcher] Dispatches checks for ${allExpectedDates.length} dates (${allExpectedDates[0]} to ${allExpectedDates[allExpectedDates.length - 1]}). Workers will validate dependencies.`);
|
|
57
48
|
|
|
58
49
|
// 3. Dispatch Messages
|
|
59
50
|
let dispatchedCount = 0;
|
|
60
|
-
const BATCH_SIZE
|
|
51
|
+
const BATCH_SIZE = 50;
|
|
61
52
|
|
|
62
53
|
// We can publish in parallel batches
|
|
63
54
|
const chunks = [];
|
|
64
|
-
for (let i = 0; i < allExpectedDates.length; i += BATCH_SIZE) {
|
|
65
|
-
chunks.push(allExpectedDates.slice(i, i + BATCH_SIZE));
|
|
66
|
-
}
|
|
55
|
+
for (let i = 0; i < allExpectedDates.length; i += BATCH_SIZE) { chunks.push(allExpectedDates.slice(i, i + BATCH_SIZE)); }
|
|
67
56
|
|
|
68
57
|
for (const chunk of chunks) {
|
|
69
58
|
const messages = chunk.map(dateStr => ({
|
|
@@ -79,9 +68,7 @@ async function dispatchComputationPass(config, dependencies, computationManifest
|
|
|
79
68
|
await pubsubUtils.publishMessageBatch(TOPIC_NAME, messages);
|
|
80
69
|
dispatchedCount += messages.length;
|
|
81
70
|
logger.log('INFO', `[Dispatcher] Dispatched batch of ${messages.length} tasks.`);
|
|
82
|
-
} catch (err) {
|
|
83
|
-
logger.log('ERROR', `[Dispatcher] Failed to dispatch batch: ${err.message}`);
|
|
84
|
-
}
|
|
71
|
+
} catch (err) { logger.log('ERROR', `[Dispatcher] Failed to dispatch batch: ${err.message}`); }
|
|
85
72
|
}
|
|
86
73
|
|
|
87
74
|
logger.log('INFO', `[Dispatcher] Finished. Dispatched ${dispatchedCount} checks for Pass ${passToRun}.`);
|
|
@@ -1,26 +1,19 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* @fileoverview
|
|
3
|
-
* Dynamic Manifest Builder (
|
|
3
|
+
* Dynamic Manifest Builder (v6 - Merkle Tree Dependency Hashing)
|
|
4
4
|
*
|
|
5
|
-
*
|
|
6
|
-
*
|
|
7
|
-
*
|
|
8
|
-
*
|
|
9
|
-
*
|
|
10
|
-
*
|
|
11
|
-
*
|
|
12
|
-
* * SAFE MODE:
|
|
13
|
-
* If a calculation uses no detectable layer keywords, it defaults to
|
|
14
|
-
* depending on ALL layers to prevent staleness.
|
|
15
|
-
* * FIXED:
|
|
16
|
-
* - Broadened LAYER_TRIGGERS to detect context usage (e.g. 'math.signals')
|
|
17
|
-
* - Updated isHistorical check to look for 'previousComputed'
|
|
5
|
+
* KEY FEATURES:
|
|
6
|
+
* 1. Smart Layer Hashing: Detects used layers (Math, Extractors) to avoid stale helper code.
|
|
7
|
+
* 2. Cascading Invalidation (Merkle Hashing):
|
|
8
|
+
* The final hash of a computation is derived from:
|
|
9
|
+
* [Own Code] + [Layer States] + [Hashes of all Dependencies]
|
|
10
|
+
* * This guarantees that if Calculation A is updated, Calculation B (which depends on A)
|
|
11
|
+
* will automatically generate a new hash, forcing the system to re-run it.
|
|
18
12
|
*/
|
|
19
13
|
|
|
20
14
|
const { generateCodeHash } = require('../utils/utils');
|
|
21
15
|
|
|
22
16
|
// 1. Import Layers directly to generate their "State Hashes"
|
|
23
|
-
// We import them individually to hash them as distinct domains.
|
|
24
17
|
const MathematicsLayer = require('../layers/mathematics');
|
|
25
18
|
const ExtractorsLayer = require('../layers/extractors');
|
|
26
19
|
const ProfilingLayer = require('../layers/profiling');
|
|
@@ -30,23 +23,15 @@ const ValidatorsLayer = require('../layers/validators');
|
|
|
30
23
|
* 1. Layer Hash Generation
|
|
31
24
|
* -------------------------------------------------- */
|
|
32
25
|
|
|
33
|
-
/**
|
|
34
|
-
* Generates a single hash representing the entire state of a layer module.
|
|
35
|
-
* Combines all exported classes/functions/objects into one deterministic string.
|
|
36
|
-
*/
|
|
37
26
|
function generateLayerHash(layerExports, layerName) {
|
|
38
|
-
const keys = Object.keys(layerExports).sort();
|
|
27
|
+
const keys = Object.keys(layerExports).sort();
|
|
39
28
|
let combinedSource = `LAYER:${layerName}`;
|
|
40
29
|
|
|
41
30
|
for (const key of keys) {
|
|
42
31
|
const item = layerExports[key];
|
|
43
|
-
if (typeof item === 'function')
|
|
44
|
-
|
|
45
|
-
} else
|
|
46
|
-
combinedSource += JSON.stringify(item);
|
|
47
|
-
} else {
|
|
48
|
-
combinedSource += String(item);
|
|
49
|
-
}
|
|
32
|
+
if (typeof item === 'function') { combinedSource += item.toString();
|
|
33
|
+
} else if (typeof item === 'object' && item !== null) { combinedSource += JSON.stringify(item);
|
|
34
|
+
} else { combinedSource += String(item); }
|
|
50
35
|
}
|
|
51
36
|
return generateCodeHash(combinedSource);
|
|
52
37
|
}
|
|
@@ -54,13 +39,12 @@ function generateLayerHash(layerExports, layerName) {
|
|
|
54
39
|
// Pre-compute layer hashes at startup
|
|
55
40
|
const LAYER_HASHES = {
|
|
56
41
|
'mathematics': generateLayerHash(MathematicsLayer, 'mathematics'),
|
|
57
|
-
'extractors': generateLayerHash(ExtractorsLayer,
|
|
58
|
-
'profiling': generateLayerHash(ProfilingLayer,
|
|
59
|
-
'validators': generateLayerHash(ValidatorsLayer,
|
|
42
|
+
'extractors': generateLayerHash(ExtractorsLayer, 'extractors'),
|
|
43
|
+
'profiling': generateLayerHash(ProfilingLayer, 'profiling'),
|
|
44
|
+
'validators': generateLayerHash(ValidatorsLayer, 'validators')
|
|
60
45
|
};
|
|
61
46
|
|
|
62
47
|
// Map code patterns to Layer dependencies
|
|
63
|
-
// If a calculation's code contains these strings, it depends on that layer.
|
|
64
48
|
const LAYER_TRIGGERS = {
|
|
65
49
|
'mathematics': [
|
|
66
50
|
'math.compute', 'MathPrimitives',
|
|
@@ -93,11 +77,11 @@ const LAYER_TRIGGERS = {
|
|
|
93
77
|
* Pretty Console Helpers
|
|
94
78
|
* -------------------------------------------------- */
|
|
95
79
|
const log = {
|
|
96
|
-
info: (msg) =>
|
|
97
|
-
step: (msg) =>
|
|
98
|
-
warn: (msg) =>
|
|
99
|
-
success: (msg) =>
|
|
100
|
-
error: (msg) =>
|
|
80
|
+
info: (msg) => console.log('ℹ︎ ' + msg),
|
|
81
|
+
step: (msg) => console.log('› ' + msg),
|
|
82
|
+
warn: (msg) => console.warn('⚠︎ ' + msg),
|
|
83
|
+
success: (msg) => console.log('✔︎ ' + msg),
|
|
84
|
+
error: (msg) => console.error('✖ ' + msg),
|
|
101
85
|
fatal: (msg) => { console.error('✖ FATAL ✖ ' + msg); console.error('✖ FATAL ✖ Manifest build FAILED.'); },
|
|
102
86
|
divider: (label) => { const line = ''.padEnd(60, '─'); console.log(`\n${line}\n${label}\n${line}\n`); },
|
|
103
87
|
};
|
|
@@ -124,18 +108,6 @@ function suggestClosest(name, candidates, n = 3) {
|
|
|
124
108
|
return scores.slice(0, n).map(s => s[0]);
|
|
125
109
|
}
|
|
126
110
|
|
|
127
|
-
function findCycles(manifestMap, adjacencyList) {
|
|
128
|
-
const visited = new Set(), stack = new Set(), cycles = [];
|
|
129
|
-
const dfs = (node, path) => {
|
|
130
|
-
if (stack.has(node)) { const idx = path.indexOf(node); cycles.push([...path.slice(idx), node]); return; }
|
|
131
|
-
if (visited.has(node)) return;
|
|
132
|
-
visited.add(node); stack.add(node);
|
|
133
|
-
for (const nb of adjacencyList.get(node) || []) dfs(nb, [...path, nb]);
|
|
134
|
-
stack.delete(node); };
|
|
135
|
-
for (const name of manifestMap.keys()) dfs(name, [name]);
|
|
136
|
-
return cycles;
|
|
137
|
-
}
|
|
138
|
-
|
|
139
111
|
function getDependencySet(endpoints, adjacencyList) {
|
|
140
112
|
const required = new Set(endpoints);
|
|
141
113
|
const queue = [...endpoints];
|
|
@@ -149,72 +121,53 @@ function getDependencySet(endpoints, adjacencyList) {
|
|
|
149
121
|
* -------------------------------------------------- */
|
|
150
122
|
|
|
151
123
|
function buildManifest(productLinesToRun = [], calculations) {
|
|
152
|
-
log.divider('Building Dynamic Manifest (
|
|
124
|
+
log.divider('Building Dynamic Manifest (Merkle Hashing)');
|
|
153
125
|
log.info(`Target Product Lines: [${productLinesToRun.join(', ')}]`);
|
|
154
|
-
|
|
155
|
-
|
|
126
|
+
|
|
156
127
|
const manifestMap = new Map();
|
|
157
128
|
const adjacency = new Map();
|
|
158
129
|
const reverseAdjacency = new Map();
|
|
159
130
|
const inDegree = new Map();
|
|
160
131
|
let hasFatalError = false;
|
|
161
132
|
|
|
162
|
-
/* ---------------- 1. Load All Calculations ---------------- */
|
|
133
|
+
/* ---------------- 1. Load All Calculations (Phase 1: Intrinsic Hash) ---------------- */
|
|
163
134
|
log.step('Loading and validating all calculation classes…');
|
|
164
|
-
const allCalculationClasses = new Map();
|
|
165
135
|
|
|
166
136
|
function processCalc(Class, name, folderName) {
|
|
167
137
|
if (!Class || typeof Class !== 'function') return;
|
|
168
138
|
const normalizedName = normalizeName(name);
|
|
169
|
-
allCalculationClasses.set(normalizedName, Class);
|
|
170
139
|
|
|
171
|
-
if (typeof Class.getMetadata
|
|
140
|
+
if (typeof Class.getMetadata !== 'function') { log.fatal(`Calculation "${normalizedName}" is missing static getMetadata().`); hasFatalError = true; return; }
|
|
172
141
|
if (typeof Class.getDependencies !== 'function') { log.fatal(`Calculation "${normalizedName}" is missing static getDependencies().`); hasFatalError = true;return; }
|
|
173
142
|
|
|
174
143
|
const metadata = Class.getMetadata();
|
|
175
144
|
const dependencies = Class.getDependencies().map(normalizeName);
|
|
176
145
|
|
|
177
|
-
// FIX: Updated check to include 'previousComputed'
|
|
178
146
|
const codeStr = Class.toString();
|
|
179
147
|
if (metadata.isHistorical === true && !codeStr.includes('yesterday') && !codeStr.includes('previousComputed')) {
|
|
180
|
-
log.warn(`Calculation "${normalizedName}" marked 'isHistorical' but no 'previousComputed'
|
|
148
|
+
log.warn(`Calculation "${normalizedName}" marked 'isHistorical' but no 'previousComputed' state reference found.`);
|
|
181
149
|
}
|
|
182
150
|
|
|
183
|
-
let finalCategory = folderName;
|
|
184
|
-
if (folderName === 'core') {
|
|
185
|
-
if (metadata.category) finalCategory = metadata.category;
|
|
186
|
-
} else {
|
|
187
|
-
finalCategory = folderName;
|
|
188
|
-
}
|
|
151
|
+
let finalCategory = folderName === 'core' && metadata.category ? metadata.category : folderName;
|
|
189
152
|
|
|
190
|
-
// ---
|
|
191
|
-
|
|
153
|
+
// --- PHASE 1: INTRINSIC HASH (Code + Layers) ---
|
|
154
|
+
// We do NOT include dependencies yet.
|
|
155
|
+
let compositeHashString = generateCodeHash(codeStr);
|
|
192
156
|
const usedLayers = [];
|
|
193
157
|
|
|
194
|
-
//
|
|
158
|
+
// Check for specific layer usage
|
|
195
159
|
for (const [layerName, triggers] of Object.entries(LAYER_TRIGGERS)) {
|
|
196
|
-
|
|
197
|
-
if (triggers.some(trigger => codeStr.includes(trigger))) {
|
|
198
|
-
compositeHashString += LAYER_HASHES[layerName]; // Append Layer Hash
|
|
199
|
-
usedLayers.push(layerName);
|
|
200
|
-
}
|
|
160
|
+
if (triggers.some(trigger => codeStr.includes(trigger))) { compositeHashString += LAYER_HASHES[layerName]; usedLayers.push(layerName); }
|
|
201
161
|
}
|
|
202
162
|
|
|
203
|
-
//
|
|
204
|
-
// If we found NO specific triggers (e.g. legacy code or unique structure),
|
|
205
|
-
// assume it might use anything. Depend on ALL layers to be safe.
|
|
163
|
+
// Safe Mode Fallback
|
|
206
164
|
let isSafeMode = false;
|
|
207
165
|
if (usedLayers.length === 0) {
|
|
208
166
|
isSafeMode = true;
|
|
209
167
|
Object.values(LAYER_HASHES).forEach(h => compositeHashString += h);
|
|
210
168
|
}
|
|
211
169
|
|
|
212
|
-
|
|
213
|
-
const finalHash = generateCodeHash(compositeHashString);
|
|
214
|
-
|
|
215
|
-
if (isSafeMode) {
|
|
216
|
-
log.warn(`[Hash] ${normalizedName}: No specific dependencies detected. Enforcing SAFE MODE (Linked to ALL Layers).`);
|
|
217
|
-
}
|
|
170
|
+
const baseHash = generateCodeHash(compositeHashString);
|
|
218
171
|
|
|
219
172
|
const manifestEntry = {
|
|
220
173
|
name: normalizedName,
|
|
@@ -227,7 +180,7 @@ function buildManifest(productLinesToRun = [], calculations) {
|
|
|
227
180
|
userType: metadata.userType,
|
|
228
181
|
dependencies: dependencies,
|
|
229
182
|
pass: 0,
|
|
230
|
-
hash:
|
|
183
|
+
hash: baseHash,
|
|
231
184
|
debugUsedLayers: isSafeMode ? ['ALL (Safe Mode)'] : usedLayers
|
|
232
185
|
};
|
|
233
186
|
|
|
@@ -241,7 +194,6 @@ function buildManifest(productLinesToRun = [], calculations) {
|
|
|
241
194
|
throw new Error('Manifest build failed: Invalid calculations object.');
|
|
242
195
|
}
|
|
243
196
|
|
|
244
|
-
// Iterate over folders (folderName becomes the default category)
|
|
245
197
|
for (const folderName in calculations) {
|
|
246
198
|
if (folderName === 'legacy') continue;
|
|
247
199
|
const group = calculations[folderName];
|
|
@@ -274,13 +226,8 @@ function buildManifest(productLinesToRun = [], calculations) {
|
|
|
274
226
|
|
|
275
227
|
/* ---------------- 3. Filter for Product Lines ---------------- */
|
|
276
228
|
const productLineEndpoints = [];
|
|
277
|
-
for (const [name, entry] of manifestMap.entries()) {
|
|
278
|
-
|
|
279
|
-
}
|
|
280
|
-
// Always include core
|
|
281
|
-
for (const [name, entry] of manifestMap.entries()) {
|
|
282
|
-
if (entry.sourcePackage === 'core') { productLineEndpoints.push(name); }
|
|
283
|
-
}
|
|
229
|
+
for (const [name, entry] of manifestMap.entries()) { if (productLinesToRun.includes(entry.category)) { productLineEndpoints.push(name); } }
|
|
230
|
+
for (const [name, entry] of manifestMap.entries()) { if (entry.sourcePackage === 'core') { productLineEndpoints.push(name); } }
|
|
284
231
|
|
|
285
232
|
const requiredCalcs = getDependencySet(productLineEndpoints, adjacency);
|
|
286
233
|
log.info(`Filtered down to ${requiredCalcs.size} active calculations.`);
|
|
@@ -288,6 +235,7 @@ function buildManifest(productLinesToRun = [], calculations) {
|
|
|
288
235
|
const filteredManifestMap = new Map();
|
|
289
236
|
const filteredInDegree = new Map();
|
|
290
237
|
const filteredReverseAdjacency = new Map();
|
|
238
|
+
|
|
291
239
|
for (const name of requiredCalcs) { filteredManifestMap.set(name, manifestMap.get(name)); filteredInDegree.set(name, inDegree.get(name));
|
|
292
240
|
const consumers = (reverseAdjacency.get(name) || []).filter(consumer => requiredCalcs.has(consumer)); filteredReverseAdjacency.set(name, consumers); }
|
|
293
241
|
|
|
@@ -312,6 +260,21 @@ function buildManifest(productLinesToRun = [], calculations) {
|
|
|
312
260
|
if (sortedManifest.length !== filteredManifestMap.size) {
|
|
313
261
|
throw new Error('Circular dependency detected. Manifest build failed.'); }
|
|
314
262
|
|
|
263
|
+
/* ---------------- 5. Phase 2: Cascading Dependency Hashing ---------------- */
|
|
264
|
+
log.step('Computing Cascading Merkle Hashes...');
|
|
265
|
+
|
|
266
|
+
for (const entry of sortedManifest) {
|
|
267
|
+
// Start with the intrinsic hash (Code + Layers)
|
|
268
|
+
let dependencySignature = entry.hash;
|
|
269
|
+
|
|
270
|
+
if (entry.dependencies && entry.dependencies.length > 0) {
|
|
271
|
+
const depHashes = entry.dependencies.map(depName => { const depEntry = filteredManifestMap.get(depName); if (!depEntry) return ''; return depEntry.hash; }).join('|');
|
|
272
|
+
dependencySignature += `|DEPS:${depHashes}`;
|
|
273
|
+
}
|
|
274
|
+
// Generate the Final Smart Hash
|
|
275
|
+
entry.hash = generateCodeHash(dependencySignature);
|
|
276
|
+
}
|
|
277
|
+
|
|
315
278
|
log.success(`Total passes required: ${maxPass}`);
|
|
316
279
|
return sortedManifest;
|
|
317
280
|
}
|