bulltrackers-module 1.0.308 → 1.0.310
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/functions/computation-system/WorkflowOrchestrator.js +54 -23
- package/functions/computation-system/context/ManifestBuilder.js +12 -16
- package/functions/computation-system/tools/BuildReporter.js +32 -62
- package/functions/computation-system/topology/HashManager.js +12 -61
- package/functions/computation-system/workflows/bulltrackers_pipeline.yaml +44 -2
- package/package.json +1 -1
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* FILENAME: computation-system/WorkflowOrchestrator.js
|
|
3
|
-
* UPDATED:
|
|
4
|
-
* Includes Content-Based Short-Circuiting for both Upstream and Historical dependencies.
|
|
3
|
+
* UPDATED: Implements Data-Drift Detection (Content-Addressable Execution).
|
|
5
4
|
*/
|
|
6
5
|
|
|
7
6
|
const { normalizeName, DEFINITIVE_EARLIEST_DATES } = require('./utils/utils');
|
|
@@ -14,7 +13,7 @@ const { MetaExecutor } = require('./executor
|
|
|
14
13
|
const STATUS_IMPOSSIBLE_PREFIX = 'IMPOSSIBLE';
|
|
15
14
|
|
|
16
15
|
/**
|
|
17
|
-
*
|
|
16
|
+
* Groups manifest entries by their pass number.
|
|
18
17
|
* Required by the Dispatcher to identify current work-sets.
|
|
19
18
|
*/
|
|
20
19
|
function groupByPass(manifest) {
|
|
@@ -27,28 +26,39 @@ function groupByPass(manifest) {
|
|
|
27
26
|
}
|
|
28
27
|
|
|
29
28
|
/**
|
|
30
|
-
*
|
|
31
|
-
* Checks if a dependency
|
|
29
|
+
* Core Short-Circuit Logic.
|
|
30
|
+
* Checks if a dependency is satisfied AND checks for Data Drift.
|
|
31
|
+
* Returns { ready: boolean, dataChanged: boolean, reason: string }
|
|
32
32
|
*/
|
|
33
33
|
function isDependencyReady(depName, isHistoricalSelf, currentStatusMap, prevStatusMap, manifestMap, storedStatus) {
|
|
34
34
|
const norm = normalizeName(depName);
|
|
35
35
|
const targetStatus = isHistoricalSelf ? (prevStatusMap ? prevStatusMap[norm] : null) : currentStatusMap[norm];
|
|
36
36
|
const depManifest = manifestMap.get(norm);
|
|
37
37
|
|
|
38
|
+
// 1. Availability Check
|
|
38
39
|
if (!targetStatus) return { ready: false, reason: 'Missing' };
|
|
39
40
|
if (String(targetStatus.hash).startsWith(STATUS_IMPOSSIBLE_PREFIX)) return { ready: false, reason: 'Impossible Upstream' };
|
|
40
41
|
|
|
41
|
-
//
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
// If our code didn't change, check if the output of the dependency is what we expect.
|
|
46
|
-
const lastSeenResultHash = storedStatus?.dependencyResultHashes?.[depName];
|
|
47
|
-
if (lastSeenResultHash && targetStatus.resultHash === lastSeenResultHash) {
|
|
48
|
-
return { ready: true, shortCircuited: true };
|
|
42
|
+
// 2. Code Hash Check (The dependency must be running the correct version)
|
|
43
|
+
// If the dependency's hash doesn't match its manifest, it means the dependency itself needs to run/update first.
|
|
44
|
+
if (depManifest && targetStatus.hash !== depManifest.hash) {
|
|
45
|
+
return { ready: false, reason: 'Dependency Version Mismatch' };
|
|
49
46
|
}
|
|
50
47
|
|
|
51
|
-
|
|
48
|
+
// 3. Data Integrity Check (The Short-Circuit Logic)
|
|
49
|
+
// We check if the result hash of the dependency matches what we remember using last time.
|
|
50
|
+
if (storedStatus && storedStatus.dependencyResultHashes) {
|
|
51
|
+
const lastSeenResultHash = storedStatus.dependencyResultHashes[depName];
|
|
52
|
+
|
|
53
|
+
// If we recorded a dependency hash last time, and it differs from the current live status,
|
|
54
|
+
// then the dependency has produced NEW data. We are NOT ready to skip.
|
|
55
|
+
// We return 'ready: true' (it exists) but we flag 'dataChanged: true' to force execution.
|
|
56
|
+
if (lastSeenResultHash && targetStatus.resultHash !== lastSeenResultHash) {
|
|
57
|
+
return { ready: true, dataChanged: true, reason: 'Dependency Data Update' };
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
return { ready: true, dataChanged: false };
|
|
52
62
|
}
|
|
53
63
|
|
|
54
64
|
function analyzeDateExecution(dateStr, calcsInPass, rootDataStatus, dailyStatus, manifestMap, prevDailyStatus = null) {
|
|
@@ -74,17 +84,22 @@ function analyzeDateExecution(dateStr, calcsInPass, rootDataStatus, dailyStatus,
|
|
|
74
84
|
|
|
75
85
|
// 2. Dependency & Temporal Check
|
|
76
86
|
const missingDeps = [];
|
|
77
|
-
let
|
|
87
|
+
let hasDataDrift = false; // Tracks if any dependency produced new results
|
|
88
|
+
let isBlocked = false;
|
|
78
89
|
|
|
79
90
|
// A. Standard Upstream Dependencies
|
|
80
91
|
if (calc.dependencies) {
|
|
81
92
|
for (const dep of calc.dependencies) {
|
|
82
93
|
const check = isDependencyReady(dep, false, simulationStatus, null, manifestMap, stored);
|
|
83
|
-
if (!check.ready)
|
|
94
|
+
if (!check.ready) {
|
|
95
|
+
missingDeps.push(dep);
|
|
96
|
+
} else if (check.dataChanged) {
|
|
97
|
+
hasDataDrift = true;
|
|
98
|
+
}
|
|
84
99
|
}
|
|
85
100
|
}
|
|
86
101
|
|
|
87
|
-
// B.
|
|
102
|
+
// B. Temporal Dependency (Yesterday's Self)
|
|
88
103
|
if (calc.isHistorical) {
|
|
89
104
|
const yesterday = new Date(dateStr + 'T00:00:00Z');
|
|
90
105
|
yesterday.setUTCDate(yesterday.getUTCDate() - 1);
|
|
@@ -92,7 +107,8 @@ function analyzeDateExecution(dateStr, calcsInPass, rootDataStatus, dailyStatus,
|
|
|
92
107
|
// Only block if yesterday is a valid data date.
|
|
93
108
|
if (yesterday >= DEFINITIVE_EARLIEST_DATES.absoluteEarliest) {
|
|
94
109
|
const check = isDependencyReady(calc.name, true, null, prevDailyStatus, manifestMap, stored);
|
|
95
|
-
if (!check.ready)
|
|
110
|
+
if (!check.ready) isBlocked = true;
|
|
111
|
+
else if (check.dataChanged) hasDataDrift = true; // Historical drift implies we need to re-run
|
|
96
112
|
}
|
|
97
113
|
}
|
|
98
114
|
|
|
@@ -107,12 +123,14 @@ function analyzeDateExecution(dateStr, calcsInPass, rootDataStatus, dailyStatus,
|
|
|
107
123
|
continue;
|
|
108
124
|
}
|
|
109
125
|
|
|
110
|
-
if (
|
|
126
|
+
if (isBlocked) {
|
|
111
127
|
report.blocked.push({ name: cName, reason: 'Waiting for Yesterday' });
|
|
112
128
|
continue;
|
|
113
129
|
}
|
|
114
130
|
|
|
115
|
-
// 3.
|
|
131
|
+
// 3. Execution Decision
|
|
132
|
+
|
|
133
|
+
// Collect current dependency result hashes to be saved if we run
|
|
116
134
|
const currentDependencyResultHashes = {};
|
|
117
135
|
if (calc.dependencies) {
|
|
118
136
|
calc.dependencies.forEach(d => {
|
|
@@ -120,16 +138,26 @@ function analyzeDateExecution(dateStr, calcsInPass, rootDataStatus, dailyStatus,
|
|
|
120
138
|
if (resHash) currentDependencyResultHashes[d] = resHash;
|
|
121
139
|
});
|
|
122
140
|
}
|
|
123
|
-
|
|
141
|
+
|
|
124
142
|
const taskPayload = { name: cName, dependencyResultHashes: currentDependencyResultHashes };
|
|
125
143
|
|
|
126
144
|
if (!stored?.hash) {
|
|
145
|
+
// Case A: New Calculation (Never run)
|
|
127
146
|
report.runnable.push({ ...taskPayload, reason: "New Calculation" });
|
|
128
147
|
simulationStatus[cName] = { hash: currentHash, resultHash: 'SIMULATED' };
|
|
129
|
-
}
|
|
148
|
+
}
|
|
149
|
+
else if (stored.hash !== currentHash) {
|
|
150
|
+
// Case B: Code Hash Mismatch (Logic Changed)
|
|
130
151
|
report.reRuns.push({ ...taskPayload, oldHash: stored.hash, newHash: currentHash, reason: "Hash Mismatch" });
|
|
131
152
|
simulationStatus[cName] = { hash: currentHash, resultHash: 'SIMULATED' };
|
|
132
|
-
}
|
|
153
|
+
}
|
|
154
|
+
else if (hasDataDrift) {
|
|
155
|
+
// Case C: Code Matches, BUT Input Data Changed (The Holy Grail Optimization)
|
|
156
|
+
report.runnable.push({ ...taskPayload, reason: "Input Data Changed" });
|
|
157
|
+
simulationStatus[cName] = { hash: currentHash, resultHash: 'SIMULATED' };
|
|
158
|
+
}
|
|
159
|
+
else {
|
|
160
|
+
// Case D: Code Matches AND Data Matches -> Short Circuit
|
|
133
161
|
report.skipped.push({ name: cName, reason: "Up To Date" });
|
|
134
162
|
}
|
|
135
163
|
}
|
|
@@ -142,6 +170,9 @@ async function executeDispatchTask(dateStr, pass, targetComputation, config, dep
|
|
|
142
170
|
const calcManifest = manifestMap.get(normalizeName(targetComputation));
|
|
143
171
|
|
|
144
172
|
if (!calcManifest) throw new Error(`Calc '${targetComputation}' not found.`);
|
|
173
|
+
|
|
174
|
+
// [CRITICAL] Inject the fresh dependency result hashes so they are saved to DB on commit.
|
|
175
|
+
// This enables the "lastSeenResultHash" check in future runs.
|
|
145
176
|
calcManifest.dependencyResultHashes = dependencyResultHashes;
|
|
146
177
|
|
|
147
178
|
const rootData = await checkRootDataAvailability(dateStr, config, dependencies, DEFINITIVE_EARLIEST_DATES);
|
|
@@ -1,10 +1,14 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* {
|
|
3
|
+
* type: uploaded file
|
|
4
|
+
* fileName: computation-system/context/ManifestBuilder.js
|
|
5
|
+
* }
|
|
6
|
+
*/
|
|
1
7
|
/**
|
|
2
8
|
* @fileoverview Dynamic Manifest Builder - Handles Topological Sort and Auto-Discovery.
|
|
3
|
-
* UPDATED:
|
|
4
|
-
* UPGRADE: Implements Tarjan's Algorithm for Precise Cycle Detection.
|
|
5
|
-
* FIXED: Now incorporates System Infrastructure Hash into Calculation Hashes.
|
|
9
|
+
* UPDATED: Removed Automatic Infra Hashing. Now relies strictly on SYSTEM_EPOCH.
|
|
6
10
|
*/
|
|
7
|
-
const { generateCodeHash,
|
|
11
|
+
const { generateCodeHash, LEGACY_MAPPING } = require('../topology/HashManager.js');
|
|
8
12
|
const { normalizeName } = require('../utils/utils');
|
|
9
13
|
|
|
10
14
|
const SYSTEM_EPOCH = require('../system_epoch');
|
|
@@ -86,7 +90,6 @@ function getDependencySet(endpoints, adjacencyList) {
|
|
|
86
90
|
|
|
87
91
|
/**
|
|
88
92
|
* Helper: Detects cycles using Tarjan's SCC Algorithm.
|
|
89
|
-
* Returns a string description of the first cycle found.
|
|
90
93
|
*/
|
|
91
94
|
function detectCircularDependencies(manifestMap) {
|
|
92
95
|
let index = 0;
|
|
@@ -106,8 +109,7 @@ function detectCircularDependencies(manifestMap) {
|
|
|
106
109
|
const entry = manifestMap.get(v);
|
|
107
110
|
if (entry && entry.dependencies) {
|
|
108
111
|
for (const w of entry.dependencies) {
|
|
109
|
-
if (!manifestMap.has(w)) continue;
|
|
110
|
-
|
|
112
|
+
if (!manifestMap.has(w)) continue;
|
|
111
113
|
if (!indices.has(w)) {
|
|
112
114
|
strongconnect(w);
|
|
113
115
|
lowLinks.set(v, Math.min(lowLinks.get(v), lowLinks.get(w)));
|
|
@@ -147,10 +149,7 @@ function detectCircularDependencies(manifestMap) {
|
|
|
147
149
|
|
|
148
150
|
function buildManifest(productLinesToRun = [], calculations) {
|
|
149
151
|
log.divider('Building Dynamic Manifest');
|
|
150
|
-
|
|
151
|
-
// [CRITICAL FIX] Calculate Infrastructure Hash once per build
|
|
152
|
-
const INFRA_HASH = getInfrastructureHash();
|
|
153
|
-
log.info(`[ManifestBuilder] System Infrastructure Hash: ${INFRA_HASH.substring(0, 8)}`);
|
|
152
|
+
log.info(`[ManifestBuilder] Global System Epoch: ${SYSTEM_EPOCH}`);
|
|
154
153
|
|
|
155
154
|
const requestedLog = (!productLinesToRun || productLinesToRun.length === 0)
|
|
156
155
|
? "ALL (Wildcard/Empty)"
|
|
@@ -175,9 +174,8 @@ function buildManifest(productLinesToRun = [], calculations) {
|
|
|
175
174
|
const codeStr = Class.toString();
|
|
176
175
|
const selfCodeHash = generateCodeHash(codeStr);
|
|
177
176
|
|
|
178
|
-
// [
|
|
179
|
-
|
|
180
|
-
let compositeHashString = selfCodeHash + `|EPOCH:${SYSTEM_EPOCH}|INFRA:${INFRA_HASH}`;
|
|
177
|
+
// [UPDATED] Composite Hash now depends ONLY on Code + Epoch + Layers
|
|
178
|
+
let compositeHashString = selfCodeHash + `|EPOCH:${SYSTEM_EPOCH}`;
|
|
181
179
|
|
|
182
180
|
const usedDeps = [];
|
|
183
181
|
const usedLayerHashes = {};
|
|
@@ -227,7 +225,6 @@ function buildManifest(productLinesToRun = [], calculations) {
|
|
|
227
225
|
composition: {
|
|
228
226
|
epoch: SYSTEM_EPOCH,
|
|
229
227
|
code: selfCodeHash,
|
|
230
|
-
infra: INFRA_HASH, // Stored in composition for audit
|
|
231
228
|
layers: layerComposition,
|
|
232
229
|
deps: {}
|
|
233
230
|
},
|
|
@@ -279,7 +276,6 @@ function buildManifest(productLinesToRun = [], calculations) {
|
|
|
279
276
|
|
|
280
277
|
const activeList = Array.from(activePackages).sort().join(', ');
|
|
281
278
|
log.info(`[ManifestBuilder] ✅ FINAL ACTIVE PRODUCT LINES: [${activeList}]`);
|
|
282
|
-
log.info(`[ManifestBuilder] Total Active Calculations: ${requiredCalcs.size}`);
|
|
283
279
|
|
|
284
280
|
const filteredManifestMap = new Map();
|
|
285
281
|
const filteredInDegree = new Map();
|
|
@@ -1,70 +1,31 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* {
|
|
3
|
+
* type: uploaded file
|
|
4
|
+
* fileName: computation-system/tools/BuildReporter.js
|
|
5
|
+
* }
|
|
6
|
+
*/
|
|
1
7
|
const { analyzeDateExecution } = require('../WorkflowOrchestrator');
|
|
2
8
|
const { fetchComputationStatus, updateComputationStatus } = require('../persistence/StatusRepository');
|
|
3
|
-
const { normalizeName, getExpectedDateStrings, DEFINITIVE_EARLIEST_DATES } = require('../utils/utils');
|
|
9
|
+
const { normalizeName, getExpectedDateStrings, DEFINITIVE_EARLIEST_DATES, generateCodeHash } = require('../utils/utils');
|
|
4
10
|
const { checkRootDataAvailability } = require('../data/AvailabilityChecker');
|
|
5
11
|
const SimRunner = require('../simulation/SimRunner');
|
|
12
|
+
const SYSTEM_EPOCH = require('../system_epoch'); // Relies on manual epoch
|
|
6
13
|
const pLimit = require('p-limit');
|
|
7
14
|
const path = require('path');
|
|
8
15
|
const crypto = require('crypto');
|
|
9
16
|
const fs = require('fs');
|
|
10
17
|
const packageJson = require(path.join(__dirname, '..', '..', '..', 'package.json'));
|
|
11
18
|
const packageVersion = packageJson.version;
|
|
12
|
-
const { generateCodeHash } = require('../utils/utils');
|
|
13
19
|
|
|
14
20
|
const SIMHASH_REGISTRY_COLLECTION = 'system_simhash_registry';
|
|
15
21
|
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
const IGNORED_FILES = new Set(['package-lock.json', '.DS_Store', '.env']);
|
|
20
|
-
|
|
21
|
-
function walkSync(dir, fileList = []) {
|
|
22
|
-
const files = fs.readdirSync(dir);
|
|
23
|
-
files.forEach(file => {
|
|
24
|
-
if (IGNORED_FILES.has(file)) return;
|
|
25
|
-
const filePath = path.join(dir, file);
|
|
26
|
-
const stat = fs.statSync(filePath);
|
|
27
|
-
if (stat.isDirectory()) {
|
|
28
|
-
if (!IGNORED_DIRS.has(file)) {
|
|
29
|
-
walkSync(filePath, fileList);
|
|
30
|
-
}
|
|
31
|
-
} else {
|
|
32
|
-
if (file.endsWith('.js') || file.endsWith('.json') || file.endsWith('.yaml')) {
|
|
33
|
-
fileList.push(filePath);
|
|
34
|
-
}
|
|
35
|
-
}
|
|
36
|
-
});
|
|
37
|
-
return fileList;
|
|
38
|
-
}
|
|
39
|
-
|
|
40
|
-
function getInfrastructureHash() {
|
|
41
|
-
try {
|
|
42
|
-
const allFiles = walkSync(SYSTEM_ROOT);
|
|
43
|
-
allFiles.sort();
|
|
44
|
-
const bigHash = crypto.createHash('sha256');
|
|
45
|
-
for (const filePath of allFiles) {
|
|
46
|
-
const content = fs.readFileSync(filePath, 'utf8');
|
|
47
|
-
const relativePath = path.relative(SYSTEM_ROOT, filePath);
|
|
48
|
-
let cleanContent = content;
|
|
49
|
-
if (filePath.endsWith('.js')) {
|
|
50
|
-
cleanContent = generateCodeHash(content);
|
|
51
|
-
} else {
|
|
52
|
-
cleanContent = content.replace(/\s+/g, '');
|
|
53
|
-
}
|
|
54
|
-
bigHash.update(`${relativePath}:${cleanContent}|`);
|
|
55
|
-
}
|
|
56
|
-
return bigHash.digest('hex');
|
|
57
|
-
} catch (e) {
|
|
58
|
-
console.warn(`[BuildReporter] ⚠️ Failed to generate infra hash: ${e.message}`);
|
|
59
|
-
return 'infra_hash_error';
|
|
60
|
-
}
|
|
61
|
-
}
|
|
62
|
-
|
|
22
|
+
/**
|
|
23
|
+
* Replaces expensive file walking with System Epoch + Manifest Hash.
|
|
24
|
+
*/
|
|
63
25
|
function getSystemFingerprint(manifest) {
|
|
64
26
|
const sortedManifestHashes = manifest.map(c => c.hash).sort().join('|');
|
|
65
|
-
const infraHash = getInfrastructureHash();
|
|
66
27
|
return crypto.createHash('sha256')
|
|
67
|
-
.update(sortedManifestHashes +
|
|
28
|
+
.update(sortedManifestHashes + SYSTEM_EPOCH)
|
|
68
29
|
.digest('hex');
|
|
69
30
|
}
|
|
70
31
|
|
|
@@ -104,11 +65,14 @@ function calculateBlastRadius(targetCalcName, reverseGraph) {
|
|
|
104
65
|
};
|
|
105
66
|
}
|
|
106
67
|
|
|
68
|
+
/**
|
|
69
|
+
* [OPTIMIZED] Logic Stability Check
|
|
70
|
+
* We trust SimHash here. If the code logic (behavior) is stable, we mark it stable.
|
|
71
|
+
* We do NOT check dependency drift here. The WorkflowOrchestrator handles that at runtime.
|
|
72
|
+
*/
|
|
107
73
|
async function verifyBehavioralStability(candidates, manifestMap, dailyStatus, logger, simHashCache, db) {
|
|
108
74
|
const trueReRuns = [];
|
|
109
75
|
const stableUpdates = [];
|
|
110
|
-
|
|
111
|
-
// Concurrency for simulations
|
|
112
76
|
const limit = pLimit(10);
|
|
113
77
|
|
|
114
78
|
const checks = candidates.map(item => limit(async () => {
|
|
@@ -116,11 +80,13 @@ async function verifyBehavioralStability(candidates, manifestMap, dailyStatus, l
|
|
|
116
80
|
const manifest = manifestMap.get(item.name);
|
|
117
81
|
const stored = dailyStatus[item.name];
|
|
118
82
|
|
|
83
|
+
// 1. If no history exists, it must run.
|
|
119
84
|
if (!stored || !stored.simHash || !manifest) {
|
|
120
85
|
trueReRuns.push(item);
|
|
121
86
|
return;
|
|
122
87
|
}
|
|
123
88
|
|
|
89
|
+
// 2. Fetch or Compute SimHash
|
|
124
90
|
let newSimHash = simHashCache.get(manifest.hash);
|
|
125
91
|
if (!newSimHash) {
|
|
126
92
|
newSimHash = await SimRunner.run(manifest, manifestMap);
|
|
@@ -132,7 +98,11 @@ async function verifyBehavioralStability(candidates, manifestMap, dailyStatus, l
|
|
|
132
98
|
}).catch(err => logger.log('WARN', `Failed to write SimHash registry for ${manifest.name}: ${err.message}`));
|
|
133
99
|
}
|
|
134
100
|
|
|
101
|
+
// 3. Behavioral Comparison
|
|
135
102
|
if (newSimHash === stored.simHash) {
|
|
103
|
+
// STABLE: The logic is identical for mocked data.
|
|
104
|
+
// We mark this as stable, allowing the Orchestrator to skip it
|
|
105
|
+
// UNLESS the actual production input data has changed.
|
|
136
106
|
stableUpdates.push({
|
|
137
107
|
...item,
|
|
138
108
|
reason: "Code Updated (Logic Stable)",
|
|
@@ -140,6 +110,7 @@ async function verifyBehavioralStability(candidates, manifestMap, dailyStatus, l
|
|
|
140
110
|
newHash: manifest.hash
|
|
141
111
|
});
|
|
142
112
|
} else {
|
|
113
|
+
// UNSTABLE: The logic actually produces different results.
|
|
143
114
|
trueReRuns.push({
|
|
144
115
|
...item,
|
|
145
116
|
reason: item.reason + ` [SimHash Mismatch]`,
|
|
@@ -240,7 +211,6 @@ async function generateBuildReport(config, dependencies, manifest, daysBack = 90
|
|
|
240
211
|
|
|
241
212
|
let totalRun = 0, totalReRun = 0, totalStable = 0;
|
|
242
213
|
|
|
243
|
-
// [FIX] Reduced concurrency from 20 to 5 to avoid Firestore DEADLINE_EXCEEDED
|
|
244
214
|
const limit = pLimit(5);
|
|
245
215
|
|
|
246
216
|
const processingPromises = datesToCheck.map(dateStr => limit(async () => {
|
|
@@ -305,12 +275,15 @@ async function generateBuildReport(config, dependencies, manifest, daysBack = 90
|
|
|
305
275
|
const updatesPayload = {};
|
|
306
276
|
for (const stable of stableUpdates) {
|
|
307
277
|
const m = manifestMap.get(stable.name);
|
|
308
|
-
|
|
278
|
+
// We must grab the EXISTING data from the DB to preserve the OLD result hash.
|
|
279
|
+
// This allows the Orchestrator to compare Old Result vs New Dependency Result.
|
|
280
|
+
const stored = dailyStatus[stable.name];
|
|
281
|
+
if (m && stored) {
|
|
309
282
|
updatesPayload[stable.name] = {
|
|
310
|
-
hash: m.hash,
|
|
311
|
-
simHash: stable.simHash,
|
|
312
|
-
resultHash:
|
|
313
|
-
dependencyResultHashes:
|
|
283
|
+
hash: m.hash, // The NEW Code Hash
|
|
284
|
+
simHash: stable.simHash, // The NEW SimHash (Same as old)
|
|
285
|
+
resultHash: stored.resultHash, // The OLD Result Hash (Preserved)
|
|
286
|
+
dependencyResultHashes: stored.dependencyResultHashes || {}, // OLD Deps (Preserved)
|
|
314
287
|
category: m.category,
|
|
315
288
|
composition: m.composition,
|
|
316
289
|
lastUpdated: new Date()
|
|
@@ -324,19 +297,16 @@ async function generateBuildReport(config, dependencies, manifest, daysBack = 90
|
|
|
324
297
|
}
|
|
325
298
|
}
|
|
326
299
|
|
|
327
|
-
// 3. BLOCKED / IMPOSSIBLE / UPTODATE
|
|
328
300
|
analysis.blocked.forEach (item => pushIfValid(dateSummary.blocked, item));
|
|
329
301
|
analysis.failedDependency.forEach (item => pushIfValid(dateSummary.blocked, item, "Dependency Missing"));
|
|
330
302
|
analysis.impossible.forEach (item => pushIfValid(dateSummary.impossible, item));
|
|
331
303
|
analysis.skipped.forEach (item => pushIfValid(dateSummary.uptodate, item, "Up To Date"));
|
|
332
304
|
|
|
333
|
-
// Meta stats
|
|
334
305
|
const includedCount = dateSummary.run.length + dateSummary.rerun.length + dateSummary.stable.length +
|
|
335
306
|
dateSummary.blocked.length + dateSummary.impossible.length + dateSummary.uptodate.length;
|
|
336
307
|
dateSummary.meta.totalIncluded = includedCount;
|
|
337
308
|
dateSummary.meta.match = (includedCount === expectedCount);
|
|
338
309
|
|
|
339
|
-
// Write Immediately
|
|
340
310
|
await db.collection('computation_build_records')
|
|
341
311
|
.doc(buildId)
|
|
342
312
|
.collection('details')
|
|
@@ -1,10 +1,14 @@
|
|
|
1
1
|
/**
|
|
2
|
-
*
|
|
3
|
-
*
|
|
2
|
+
* {
|
|
3
|
+
* type: uploaded file
|
|
4
|
+
* fileName: computation-system/topology/HashManager.js
|
|
5
|
+
* }
|
|
6
|
+
*/
|
|
7
|
+
/**
|
|
8
|
+
* @fileoverview Manages code versioning and legacy mappings.
|
|
9
|
+
* UPDATED: Removed global infrastructure scanning. Now relies on Manual Epochs.
|
|
4
10
|
*/
|
|
5
11
|
const crypto = require('crypto');
|
|
6
|
-
const fs = require('fs');
|
|
7
|
-
const path = require('path');
|
|
8
12
|
|
|
9
13
|
// Legacy Keys Mapping (Ensures backward compatibility)
|
|
10
14
|
const LEGACY_MAPPING = {
|
|
@@ -39,66 +43,13 @@ function generateCodeHash(codeString) {
|
|
|
39
43
|
return crypto.createHash('sha256').update(clean).digest('hex');
|
|
40
44
|
}
|
|
41
45
|
|
|
42
|
-
// --- INFRASTRUCTURE HASHING (The "System Fingerprint") ---
|
|
43
|
-
|
|
44
|
-
const SYSTEM_ROOT = path.resolve(__dirname, '..');
|
|
45
|
-
const IGNORED_DIRS = new Set(['node_modules', '.git', '.idea', 'coverage', 'logs', 'tests', 'docs']);
|
|
46
|
-
const IGNORED_FILES = new Set(['package-lock.json', '.DS_Store', '.env', 'README.md']);
|
|
47
|
-
|
|
48
|
-
function walkSync(dir, fileList = []) {
|
|
49
|
-
const files = fs.readdirSync(dir);
|
|
50
|
-
files.forEach(file => {
|
|
51
|
-
if (IGNORED_FILES.has(file)) return;
|
|
52
|
-
const filePath = path.join(dir, file);
|
|
53
|
-
const stat = fs.statSync(filePath);
|
|
54
|
-
if (stat.isDirectory()) {
|
|
55
|
-
if (!IGNORED_DIRS.has(file)) {
|
|
56
|
-
walkSync(filePath, fileList);
|
|
57
|
-
}
|
|
58
|
-
} else {
|
|
59
|
-
// Hash JS, JSON, and YAML (Workflows)
|
|
60
|
-
if (file.endsWith('.js') || file.endsWith('.json') || file.endsWith('.yaml')) {
|
|
61
|
-
fileList.push(filePath);
|
|
62
|
-
}
|
|
63
|
-
}
|
|
64
|
-
});
|
|
65
|
-
return fileList;
|
|
66
|
-
}
|
|
67
|
-
|
|
68
46
|
/**
|
|
69
|
-
*
|
|
70
|
-
*
|
|
71
|
-
*
|
|
47
|
+
* DEPRECATED: Previously walked the file system.
|
|
48
|
+
* Now returns a static string because we rely on SYSTEM_EPOCH for global versioning.
|
|
49
|
+
* Kept for backward compatibility with older consumers.
|
|
72
50
|
*/
|
|
73
51
|
function getInfrastructureHash() {
|
|
74
|
-
|
|
75
|
-
const allFiles = walkSync(SYSTEM_ROOT);
|
|
76
|
-
allFiles.sort(); // Crucial for determinism
|
|
77
|
-
|
|
78
|
-
const bigHash = crypto.createHash('sha256');
|
|
79
|
-
|
|
80
|
-
for (const filePath of allFiles) {
|
|
81
|
-
const content = fs.readFileSync(filePath, 'utf8');
|
|
82
|
-
const relativePath = path.relative(SYSTEM_ROOT, filePath);
|
|
83
|
-
|
|
84
|
-
let cleanContent = content;
|
|
85
|
-
|
|
86
|
-
// Reuse the standard code hash logic for JS files to be consistent
|
|
87
|
-
if (filePath.endsWith('.js')) {
|
|
88
|
-
cleanContent = generateCodeHash(content);
|
|
89
|
-
} else {
|
|
90
|
-
// For JSON/YAML, just strip whitespace
|
|
91
|
-
cleanContent = content.replace(/\s+/g, '');
|
|
92
|
-
}
|
|
93
|
-
|
|
94
|
-
bigHash.update(`${relativePath}:${cleanContent}|`);
|
|
95
|
-
}
|
|
96
|
-
|
|
97
|
-
return bigHash.digest('hex');
|
|
98
|
-
} catch (e) {
|
|
99
|
-
console.warn(`[HashManager] ⚠️ Failed to generate infra hash: ${e.message}`);
|
|
100
|
-
return 'infra_error';
|
|
101
|
-
}
|
|
52
|
+
return 'MANUAL_EPOCH_MODE';
|
|
102
53
|
}
|
|
103
54
|
|
|
104
55
|
module.exports = { LEGACY_MAPPING, generateCodeHash, getInfrastructureHash };
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
# Cloud Workflows: Precision Cursor-Based Orchestrator
|
|
2
2
|
# PURPOSE: Orchestrates 5 passes with dynamic date detection and cursor logic.
|
|
3
|
+
# UPDATED: Added Short-Circuit logic to break infinite loops on empty dispatches.
|
|
3
4
|
|
|
4
5
|
main:
|
|
5
6
|
params: [input]
|
|
@@ -26,6 +27,7 @@ main:
|
|
|
26
27
|
assign:
|
|
27
28
|
- n_cursor: 1
|
|
28
29
|
- pass_complete: false
|
|
30
|
+
- consecutive_empty_dispatches: 0 # Track consecutive "duds" to prevent infinite loops
|
|
29
31
|
|
|
30
32
|
- sequential_date_loop:
|
|
31
33
|
switch:
|
|
@@ -49,9 +51,12 @@ main:
|
|
|
49
51
|
assign:
|
|
50
52
|
- pass_complete: true
|
|
51
53
|
|
|
52
|
-
# State 2: Tasks were dispatched
|
|
54
|
+
# State 2: Tasks were dispatched (Healthy State)
|
|
53
55
|
- condition: '${dispatch_res.body.dispatched > 0}'
|
|
54
56
|
steps:
|
|
57
|
+
- reset_retry_counter:
|
|
58
|
+
assign:
|
|
59
|
+
- consecutive_empty_dispatches: 0 # Reset counter because progress was made
|
|
55
60
|
- log_dispatch:
|
|
56
61
|
call: sys.log
|
|
57
62
|
args:
|
|
@@ -64,8 +69,45 @@ main:
|
|
|
64
69
|
assign:
|
|
65
70
|
# If n_cursor_ignored is true, stay on same N to retry (e.g. for high-mem)
|
|
66
71
|
- n_cursor: '${if(dispatch_res.body.n_cursor_ignored, n_cursor, n_cursor + 1)}'
|
|
67
|
-
-
|
|
72
|
+
- next_loop_work:
|
|
68
73
|
next: sequential_date_loop
|
|
69
74
|
|
|
75
|
+
# State 3: No tasks dispatched (Potential Infinite Loop Scenario)
|
|
76
|
+
# The Dispatcher is "Continuing" but found nothing runnable on the target date.
|
|
77
|
+
- condition: '${dispatch_res.body.dispatched == 0}'
|
|
78
|
+
steps:
|
|
79
|
+
- increment_retry:
|
|
80
|
+
assign:
|
|
81
|
+
- consecutive_empty_dispatches: '${consecutive_empty_dispatches + 1}'
|
|
82
|
+
- check_break_condition:
|
|
83
|
+
switch:
|
|
84
|
+
# If we have tried 3 times in a row with 0 results, assume the date is "stuck"
|
|
85
|
+
- condition: '${consecutive_empty_dispatches >= 3}'
|
|
86
|
+
steps:
|
|
87
|
+
- log_break:
|
|
88
|
+
call: sys.log
|
|
89
|
+
args:
|
|
90
|
+
text: '${"Pass " + pass_id + " - 🛑 FORCE BREAK: 3 consecutive empty dispatches. Moving to next pass to prevent infinite loop."}'
|
|
91
|
+
- force_complete:
|
|
92
|
+
assign:
|
|
93
|
+
- pass_complete: true
|
|
94
|
+
# Otherwise, wait briefly and retry (or move cursor depending on dispatcher logic)
|
|
95
|
+
- condition: '${true}'
|
|
96
|
+
steps:
|
|
97
|
+
- log_retry:
|
|
98
|
+
call: sys.log
|
|
99
|
+
args:
|
|
100
|
+
text: '${"Pass " + pass_id + " - Empty dispatch (" + string(consecutive_empty_dispatches) + "/3). Retrying..."}'
|
|
101
|
+
- wait_short:
|
|
102
|
+
call: sys.sleep
|
|
103
|
+
args:
|
|
104
|
+
seconds: 5
|
|
105
|
+
- update_cursor_retry:
|
|
106
|
+
assign:
|
|
107
|
+
# Still advance cursor if it wasn't a strict reroute, to try next date
|
|
108
|
+
- n_cursor: '${if(dispatch_res.body.n_cursor_ignored, n_cursor, n_cursor + 1)}'
|
|
109
|
+
- next_loop_retry:
|
|
110
|
+
next: sequential_date_loop
|
|
111
|
+
|
|
70
112
|
- finish:
|
|
71
113
|
return: "Pipeline Execution Satiated and Complete"
|