bulltrackers-module 1.0.307 → 1.0.309
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/functions/computation-system/WorkflowOrchestrator.js +68 -23
- package/functions/computation-system/context/ManifestBuilder.js +12 -16
- package/functions/computation-system/helpers/computation_worker.js +4 -3
- package/functions/computation-system/tools/BuildReporter.js +32 -62
- package/functions/computation-system/topology/HashManager.js +12 -61
- package/package.json +1 -1
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* FILENAME: computation-system/WorkflowOrchestrator.js
|
|
3
|
-
* UPDATED:
|
|
4
|
-
* Includes Content-Based Short-Circuiting for both Upstream and Historical dependencies.
|
|
3
|
+
* UPDATED: Implements Data-Drift Detection (Content-Addressable Execution).
|
|
5
4
|
*/
|
|
6
5
|
|
|
7
6
|
const { normalizeName, DEFINITIVE_EARLIEST_DATES } = require('./utils/utils');
|
|
@@ -14,28 +13,52 @@ const { MetaExecutor } = require('./executor
|
|
|
14
13
|
const STATUS_IMPOSSIBLE_PREFIX = 'IMPOSSIBLE';
|
|
15
14
|
|
|
16
15
|
/**
|
|
17
|
-
*
|
|
18
|
-
*
|
|
16
|
+
* Groups manifest entries by their pass number.
|
|
17
|
+
* Required by the Dispatcher to identify current work-sets.
|
|
18
|
+
*/
|
|
19
|
+
function groupByPass(manifest) {
|
|
20
|
+
const passes = {};
|
|
21
|
+
manifest.forEach(calc => {
|
|
22
|
+
if (!passes[calc.pass]) passes[calc.pass] = [];
|
|
23
|
+
passes[calc.pass].push(calc);
|
|
24
|
+
});
|
|
25
|
+
return passes;
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
/**
|
|
29
|
+
* Core Short-Circuit Logic.
|
|
30
|
+
* Checks if a dependency is satisfied AND checks for Data Drift.
|
|
31
|
+
* Returns { ready: boolean, dataChanged: boolean, reason: string }
|
|
19
32
|
*/
|
|
20
33
|
function isDependencyReady(depName, isHistoricalSelf, currentStatusMap, prevStatusMap, manifestMap, storedStatus) {
|
|
21
34
|
const norm = normalizeName(depName);
|
|
22
35
|
const targetStatus = isHistoricalSelf ? (prevStatusMap ? prevStatusMap[norm] : null) : currentStatusMap[norm];
|
|
23
36
|
const depManifest = manifestMap.get(norm);
|
|
24
37
|
|
|
38
|
+
// 1. Availability Check
|
|
25
39
|
if (!targetStatus) return { ready: false, reason: 'Missing' };
|
|
26
40
|
if (String(targetStatus.hash).startsWith(STATUS_IMPOSSIBLE_PREFIX)) return { ready: false, reason: 'Impossible Upstream' };
|
|
27
41
|
|
|
28
|
-
//
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
42
|
+
// 2. Code Hash Check (The dependency must be running the correct version)
|
|
43
|
+
// If the dependency's hash doesn't match its manifest, it means the dependency itself needs to run/update first.
|
|
44
|
+
if (depManifest && targetStatus.hash !== depManifest.hash) {
|
|
45
|
+
return { ready: false, reason: 'Dependency Version Mismatch' };
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
// 3. Data Integrity Check (The Short-Circuit Logic)
|
|
49
|
+
// We check if the result hash of the dependency matches what we remember using last time.
|
|
50
|
+
if (storedStatus && storedStatus.dependencyResultHashes) {
|
|
51
|
+
const lastSeenResultHash = storedStatus.dependencyResultHashes[depName];
|
|
52
|
+
|
|
53
|
+
// If we recorded a dependency hash last time, and it differs from the current live status,
|
|
54
|
+
// then the dependency has produced NEW data. We are NOT ready to skip.
|
|
55
|
+
// We return 'ready: true' (it exists) but we flag 'dataChanged: true' to force execution.
|
|
56
|
+
if (lastSeenResultHash && targetStatus.resultHash !== lastSeenResultHash) {
|
|
57
|
+
return { ready: true, dataChanged: true, reason: 'Dependency Data Update' };
|
|
58
|
+
}
|
|
36
59
|
}
|
|
37
60
|
|
|
38
|
-
return { ready:
|
|
61
|
+
return { ready: true, dataChanged: false };
|
|
39
62
|
}
|
|
40
63
|
|
|
41
64
|
function analyzeDateExecution(dateStr, calcsInPass, rootDataStatus, dailyStatus, manifestMap, prevDailyStatus = null) {
|
|
@@ -61,24 +84,31 @@ function analyzeDateExecution(dateStr, calcsInPass, rootDataStatus, dailyStatus,
|
|
|
61
84
|
|
|
62
85
|
// 2. Dependency & Temporal Check
|
|
63
86
|
const missingDeps = [];
|
|
64
|
-
let
|
|
87
|
+
let hasDataDrift = false; // Tracks if any dependency produced new results
|
|
88
|
+
let isBlocked = false;
|
|
65
89
|
|
|
66
90
|
// A. Standard Upstream Dependencies
|
|
67
91
|
if (calc.dependencies) {
|
|
68
92
|
for (const dep of calc.dependencies) {
|
|
69
93
|
const check = isDependencyReady(dep, false, simulationStatus, null, manifestMap, stored);
|
|
70
|
-
if (!check.ready)
|
|
94
|
+
if (!check.ready) {
|
|
95
|
+
missingDeps.push(dep);
|
|
96
|
+
} else if (check.dataChanged) {
|
|
97
|
+
hasDataDrift = true;
|
|
98
|
+
}
|
|
71
99
|
}
|
|
72
100
|
}
|
|
73
101
|
|
|
74
|
-
// B.
|
|
102
|
+
// B. Temporal Dependency (Yesterday's Self)
|
|
75
103
|
if (calc.isHistorical) {
|
|
76
104
|
const yesterday = new Date(dateStr + 'T00:00:00Z');
|
|
77
105
|
yesterday.setUTCDate(yesterday.getUTCDate() - 1);
|
|
78
106
|
|
|
107
|
+
// Only block if yesterday is a valid data date.
|
|
79
108
|
if (yesterday >= DEFINITIVE_EARLIEST_DATES.absoluteEarliest) {
|
|
80
109
|
const check = isDependencyReady(calc.name, true, null, prevDailyStatus, manifestMap, stored);
|
|
81
|
-
if (!check.ready)
|
|
110
|
+
if (!check.ready) isBlocked = true;
|
|
111
|
+
else if (check.dataChanged) hasDataDrift = true; // Historical drift implies we need to re-run
|
|
82
112
|
}
|
|
83
113
|
}
|
|
84
114
|
|
|
@@ -93,12 +123,14 @@ function analyzeDateExecution(dateStr, calcsInPass, rootDataStatus, dailyStatus,
|
|
|
93
123
|
continue;
|
|
94
124
|
}
|
|
95
125
|
|
|
96
|
-
if (
|
|
126
|
+
if (isBlocked) {
|
|
97
127
|
report.blocked.push({ name: cName, reason: 'Waiting for Yesterday' });
|
|
98
128
|
continue;
|
|
99
129
|
}
|
|
100
130
|
|
|
101
|
-
// 3.
|
|
131
|
+
// 3. Execution Decision
|
|
132
|
+
|
|
133
|
+
// Collect current dependency result hashes to be saved if we run
|
|
102
134
|
const currentDependencyResultHashes = {};
|
|
103
135
|
if (calc.dependencies) {
|
|
104
136
|
calc.dependencies.forEach(d => {
|
|
@@ -106,16 +138,26 @@ function analyzeDateExecution(dateStr, calcsInPass, rootDataStatus, dailyStatus,
|
|
|
106
138
|
if (resHash) currentDependencyResultHashes[d] = resHash;
|
|
107
139
|
});
|
|
108
140
|
}
|
|
109
|
-
|
|
141
|
+
|
|
110
142
|
const taskPayload = { name: cName, dependencyResultHashes: currentDependencyResultHashes };
|
|
111
143
|
|
|
112
144
|
if (!stored?.hash) {
|
|
145
|
+
// Case A: New Calculation (Never run)
|
|
113
146
|
report.runnable.push({ ...taskPayload, reason: "New Calculation" });
|
|
114
147
|
simulationStatus[cName] = { hash: currentHash, resultHash: 'SIMULATED' };
|
|
115
|
-
}
|
|
148
|
+
}
|
|
149
|
+
else if (stored.hash !== currentHash) {
|
|
150
|
+
// Case B: Code Hash Mismatch (Logic Changed)
|
|
116
151
|
report.reRuns.push({ ...taskPayload, oldHash: stored.hash, newHash: currentHash, reason: "Hash Mismatch" });
|
|
117
152
|
simulationStatus[cName] = { hash: currentHash, resultHash: 'SIMULATED' };
|
|
118
|
-
}
|
|
153
|
+
}
|
|
154
|
+
else if (hasDataDrift) {
|
|
155
|
+
// Case C: Code Matches, BUT Input Data Changed (The Holy Grail Optimization)
|
|
156
|
+
report.runnable.push({ ...taskPayload, reason: "Input Data Changed" });
|
|
157
|
+
simulationStatus[cName] = { hash: currentHash, resultHash: 'SIMULATED' };
|
|
158
|
+
}
|
|
159
|
+
else {
|
|
160
|
+
// Case D: Code Matches AND Data Matches -> Short Circuit
|
|
119
161
|
report.skipped.push({ name: cName, reason: "Up To Date" });
|
|
120
162
|
}
|
|
121
163
|
}
|
|
@@ -128,6 +170,9 @@ async function executeDispatchTask(dateStr, pass, targetComputation, config, dep
|
|
|
128
170
|
const calcManifest = manifestMap.get(normalizeName(targetComputation));
|
|
129
171
|
|
|
130
172
|
if (!calcManifest) throw new Error(`Calc '${targetComputation}' not found.`);
|
|
173
|
+
|
|
174
|
+
// [CRITICAL] Inject the fresh dependency result hashes so they are saved to DB on commit.
|
|
175
|
+
// This enables the "lastSeenResultHash" check in future runs.
|
|
131
176
|
calcManifest.dependencyResultHashes = dependencyResultHashes;
|
|
132
177
|
|
|
133
178
|
const rootData = await checkRootDataAvailability(dateStr, config, dependencies, DEFINITIVE_EARLIEST_DATES);
|
|
@@ -148,4 +193,4 @@ async function executeDispatchTask(dateStr, pass, targetComputation, config, dep
|
|
|
148
193
|
return { date: dateStr, updates };
|
|
149
194
|
}
|
|
150
195
|
|
|
151
|
-
module.exports = { executeDispatchTask, analyzeDateExecution };
|
|
196
|
+
module.exports = { executeDispatchTask, analyzeDateExecution, groupByPass };
|
|
@@ -1,10 +1,14 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* {
|
|
3
|
+
* type: uploaded file
|
|
4
|
+
* fileName: computation-system/context/ManifestBuilder.js
|
|
5
|
+
* }
|
|
6
|
+
*/
|
|
1
7
|
/**
|
|
2
8
|
* @fileoverview Dynamic Manifest Builder - Handles Topological Sort and Auto-Discovery.
|
|
3
|
-
* UPDATED:
|
|
4
|
-
* UPGRADE: Implements Tarjan's Algorithm for Precise Cycle Detection.
|
|
5
|
-
* FIXED: Now incorporates System Infrastructure Hash into Calculation Hashes.
|
|
9
|
+
* UPDATED: Removed Automatic Infra Hashing. Now relies strictly on SYSTEM_EPOCH.
|
|
6
10
|
*/
|
|
7
|
-
const { generateCodeHash,
|
|
11
|
+
const { generateCodeHash, LEGACY_MAPPING } = require('../topology/HashManager.js');
|
|
8
12
|
const { normalizeName } = require('../utils/utils');
|
|
9
13
|
|
|
10
14
|
const SYSTEM_EPOCH = require('../system_epoch');
|
|
@@ -86,7 +90,6 @@ function getDependencySet(endpoints, adjacencyList) {
|
|
|
86
90
|
|
|
87
91
|
/**
|
|
88
92
|
* Helper: Detects cycles using Tarjan's SCC Algorithm.
|
|
89
|
-
* Returns a string description of the first cycle found.
|
|
90
93
|
*/
|
|
91
94
|
function detectCircularDependencies(manifestMap) {
|
|
92
95
|
let index = 0;
|
|
@@ -106,8 +109,7 @@ function detectCircularDependencies(manifestMap) {
|
|
|
106
109
|
const entry = manifestMap.get(v);
|
|
107
110
|
if (entry && entry.dependencies) {
|
|
108
111
|
for (const w of entry.dependencies) {
|
|
109
|
-
if (!manifestMap.has(w)) continue;
|
|
110
|
-
|
|
112
|
+
if (!manifestMap.has(w)) continue;
|
|
111
113
|
if (!indices.has(w)) {
|
|
112
114
|
strongconnect(w);
|
|
113
115
|
lowLinks.set(v, Math.min(lowLinks.get(v), lowLinks.get(w)));
|
|
@@ -147,10 +149,7 @@ function detectCircularDependencies(manifestMap) {
|
|
|
147
149
|
|
|
148
150
|
function buildManifest(productLinesToRun = [], calculations) {
|
|
149
151
|
log.divider('Building Dynamic Manifest');
|
|
150
|
-
|
|
151
|
-
// [CRITICAL FIX] Calculate Infrastructure Hash once per build
|
|
152
|
-
const INFRA_HASH = getInfrastructureHash();
|
|
153
|
-
log.info(`[ManifestBuilder] System Infrastructure Hash: ${INFRA_HASH.substring(0, 8)}`);
|
|
152
|
+
log.info(`[ManifestBuilder] Global System Epoch: ${SYSTEM_EPOCH}`);
|
|
154
153
|
|
|
155
154
|
const requestedLog = (!productLinesToRun || productLinesToRun.length === 0)
|
|
156
155
|
? "ALL (Wildcard/Empty)"
|
|
@@ -175,9 +174,8 @@ function buildManifest(productLinesToRun = [], calculations) {
|
|
|
175
174
|
const codeStr = Class.toString();
|
|
176
175
|
const selfCodeHash = generateCodeHash(codeStr);
|
|
177
176
|
|
|
178
|
-
// [
|
|
179
|
-
|
|
180
|
-
let compositeHashString = selfCodeHash + `|EPOCH:${SYSTEM_EPOCH}|INFRA:${INFRA_HASH}`;
|
|
177
|
+
// [UPDATED] Composite Hash now depends ONLY on Code + Epoch + Layers
|
|
178
|
+
let compositeHashString = selfCodeHash + `|EPOCH:${SYSTEM_EPOCH}`;
|
|
181
179
|
|
|
182
180
|
const usedDeps = [];
|
|
183
181
|
const usedLayerHashes = {};
|
|
@@ -227,7 +225,6 @@ function buildManifest(productLinesToRun = [], calculations) {
|
|
|
227
225
|
composition: {
|
|
228
226
|
epoch: SYSTEM_EPOCH,
|
|
229
227
|
code: selfCodeHash,
|
|
230
|
-
infra: INFRA_HASH, // Stored in composition for audit
|
|
231
228
|
layers: layerComposition,
|
|
232
229
|
deps: {}
|
|
233
230
|
},
|
|
@@ -279,7 +276,6 @@ function buildManifest(productLinesToRun = [], calculations) {
|
|
|
279
276
|
|
|
280
277
|
const activeList = Array.from(activePackages).sort().join(', ');
|
|
281
278
|
log.info(`[ManifestBuilder] ✅ FINAL ACTIVE PRODUCT LINES: [${activeList}]`);
|
|
282
|
-
log.info(`[ManifestBuilder] Total Active Calculations: ${requiredCalcs.size}`);
|
|
283
279
|
|
|
284
280
|
const filteredManifestMap = new Map();
|
|
285
281
|
const filteredInDegree = new Map();
|
|
@@ -48,7 +48,7 @@ async function handleComputationTask(message, config, dependencies) {
|
|
|
48
48
|
|
|
49
49
|
logger.log('INFO', `[Worker] 📥 Task: ${computation} (${date}) [Tier: ${resourceTier}]`);
|
|
50
50
|
|
|
51
|
-
// 1. Audit Lease
|
|
51
|
+
// 1. Audit Lease (Lease on life for the monitor to see)
|
|
52
52
|
await db.doc(ledgerPath).set({
|
|
53
53
|
status: 'IN_PROGRESS',
|
|
54
54
|
workerId: process.env.K_REVISION || os.hostname(),
|
|
@@ -84,6 +84,7 @@ async function handleComputationTask(message, config, dependencies) {
|
|
|
84
84
|
composition: calcUpdate.composition
|
|
85
85
|
};
|
|
86
86
|
|
|
87
|
+
// Mark ledger as completed
|
|
87
88
|
await db.doc(ledgerPath).update({ status: 'COMPLETED', completedAt: new Date() });
|
|
88
89
|
await recordRunAttempt(db, { date, computation, pass }, 'SUCCESS', null, metrics, triggerReason, resourceTier);
|
|
89
90
|
|
|
@@ -94,9 +95,9 @@ async function handleComputationTask(message, config, dependencies) {
|
|
|
94
95
|
if (isDeterministic || (message.deliveryAttempt || 1) >= MAX_RETRIES) {
|
|
95
96
|
await db.doc(ledgerPath).set({ status: 'FAILED', error: err.message, failedAt: new Date() }, { merge: true });
|
|
96
97
|
await recordRunAttempt(db, { date, computation, pass }, 'FAILURE', { message: err.message, stage: err.stage || 'FATAL' }, { peakMemoryMB: heartbeat.getPeak() }, triggerReason, resourceTier);
|
|
97
|
-
return; //
|
|
98
|
+
return; // Exit without throwing to prevent endless Pub/Sub retries for dead logic
|
|
98
99
|
}
|
|
99
|
-
throw err; // Trigger Pub/Sub retry
|
|
100
|
+
throw err; // Trigger Pub/Sub retry for non-deterministic transient errors
|
|
100
101
|
}
|
|
101
102
|
}
|
|
102
103
|
|
|
@@ -1,70 +1,31 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* {
|
|
3
|
+
* type: uploaded file
|
|
4
|
+
* fileName: computation-system/tools/BuildReporter.js
|
|
5
|
+
* }
|
|
6
|
+
*/
|
|
1
7
|
const { analyzeDateExecution } = require('../WorkflowOrchestrator');
|
|
2
8
|
const { fetchComputationStatus, updateComputationStatus } = require('../persistence/StatusRepository');
|
|
3
|
-
const { normalizeName, getExpectedDateStrings, DEFINITIVE_EARLIEST_DATES } = require('../utils/utils');
|
|
9
|
+
const { normalizeName, getExpectedDateStrings, DEFINITIVE_EARLIEST_DATES, generateCodeHash } = require('../utils/utils');
|
|
4
10
|
const { checkRootDataAvailability } = require('../data/AvailabilityChecker');
|
|
5
11
|
const SimRunner = require('../simulation/SimRunner');
|
|
12
|
+
const SYSTEM_EPOCH = require('../system_epoch'); // Relies on manual epoch
|
|
6
13
|
const pLimit = require('p-limit');
|
|
7
14
|
const path = require('path');
|
|
8
15
|
const crypto = require('crypto');
|
|
9
16
|
const fs = require('fs');
|
|
10
17
|
const packageJson = require(path.join(__dirname, '..', '..', '..', 'package.json'));
|
|
11
18
|
const packageVersion = packageJson.version;
|
|
12
|
-
const { generateCodeHash } = require('../utils/utils');
|
|
13
19
|
|
|
14
20
|
const SIMHASH_REGISTRY_COLLECTION = 'system_simhash_registry';
|
|
15
21
|
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
const IGNORED_FILES = new Set(['package-lock.json', '.DS_Store', '.env']);
|
|
20
|
-
|
|
21
|
-
function walkSync(dir, fileList = []) {
|
|
22
|
-
const files = fs.readdirSync(dir);
|
|
23
|
-
files.forEach(file => {
|
|
24
|
-
if (IGNORED_FILES.has(file)) return;
|
|
25
|
-
const filePath = path.join(dir, file);
|
|
26
|
-
const stat = fs.statSync(filePath);
|
|
27
|
-
if (stat.isDirectory()) {
|
|
28
|
-
if (!IGNORED_DIRS.has(file)) {
|
|
29
|
-
walkSync(filePath, fileList);
|
|
30
|
-
}
|
|
31
|
-
} else {
|
|
32
|
-
if (file.endsWith('.js') || file.endsWith('.json') || file.endsWith('.yaml')) {
|
|
33
|
-
fileList.push(filePath);
|
|
34
|
-
}
|
|
35
|
-
}
|
|
36
|
-
});
|
|
37
|
-
return fileList;
|
|
38
|
-
}
|
|
39
|
-
|
|
40
|
-
function getInfrastructureHash() {
|
|
41
|
-
try {
|
|
42
|
-
const allFiles = walkSync(SYSTEM_ROOT);
|
|
43
|
-
allFiles.sort();
|
|
44
|
-
const bigHash = crypto.createHash('sha256');
|
|
45
|
-
for (const filePath of allFiles) {
|
|
46
|
-
const content = fs.readFileSync(filePath, 'utf8');
|
|
47
|
-
const relativePath = path.relative(SYSTEM_ROOT, filePath);
|
|
48
|
-
let cleanContent = content;
|
|
49
|
-
if (filePath.endsWith('.js')) {
|
|
50
|
-
cleanContent = generateCodeHash(content);
|
|
51
|
-
} else {
|
|
52
|
-
cleanContent = content.replace(/\s+/g, '');
|
|
53
|
-
}
|
|
54
|
-
bigHash.update(`${relativePath}:${cleanContent}|`);
|
|
55
|
-
}
|
|
56
|
-
return bigHash.digest('hex');
|
|
57
|
-
} catch (e) {
|
|
58
|
-
console.warn(`[BuildReporter] ⚠️ Failed to generate infra hash: ${e.message}`);
|
|
59
|
-
return 'infra_hash_error';
|
|
60
|
-
}
|
|
61
|
-
}
|
|
62
|
-
|
|
22
|
+
/**
|
|
23
|
+
* Replaces expensive file walking with System Epoch + Manifest Hash.
|
|
24
|
+
*/
|
|
63
25
|
function getSystemFingerprint(manifest) {
|
|
64
26
|
const sortedManifestHashes = manifest.map(c => c.hash).sort().join('|');
|
|
65
|
-
const infraHash = getInfrastructureHash();
|
|
66
27
|
return crypto.createHash('sha256')
|
|
67
|
-
.update(sortedManifestHashes +
|
|
28
|
+
.update(sortedManifestHashes + SYSTEM_EPOCH)
|
|
68
29
|
.digest('hex');
|
|
69
30
|
}
|
|
70
31
|
|
|
@@ -104,11 +65,14 @@ function calculateBlastRadius(targetCalcName, reverseGraph) {
|
|
|
104
65
|
};
|
|
105
66
|
}
|
|
106
67
|
|
|
68
|
+
/**
|
|
69
|
+
* [OPTIMIZED] Logic Stability Check
|
|
70
|
+
* We trust SimHash here. If the code logic (behavior) is stable, we mark it stable.
|
|
71
|
+
* We do NOT check dependency drift here. The WorkflowOrchestrator handles that at runtime.
|
|
72
|
+
*/
|
|
107
73
|
async function verifyBehavioralStability(candidates, manifestMap, dailyStatus, logger, simHashCache, db) {
|
|
108
74
|
const trueReRuns = [];
|
|
109
75
|
const stableUpdates = [];
|
|
110
|
-
|
|
111
|
-
// Concurrency for simulations
|
|
112
76
|
const limit = pLimit(10);
|
|
113
77
|
|
|
114
78
|
const checks = candidates.map(item => limit(async () => {
|
|
@@ -116,11 +80,13 @@ async function verifyBehavioralStability(candidates, manifestMap, dailyStatus, l
|
|
|
116
80
|
const manifest = manifestMap.get(item.name);
|
|
117
81
|
const stored = dailyStatus[item.name];
|
|
118
82
|
|
|
83
|
+
// 1. If no history exists, it must run.
|
|
119
84
|
if (!stored || !stored.simHash || !manifest) {
|
|
120
85
|
trueReRuns.push(item);
|
|
121
86
|
return;
|
|
122
87
|
}
|
|
123
88
|
|
|
89
|
+
// 2. Fetch or Compute SimHash
|
|
124
90
|
let newSimHash = simHashCache.get(manifest.hash);
|
|
125
91
|
if (!newSimHash) {
|
|
126
92
|
newSimHash = await SimRunner.run(manifest, manifestMap);
|
|
@@ -132,7 +98,11 @@ async function verifyBehavioralStability(candidates, manifestMap, dailyStatus, l
|
|
|
132
98
|
}).catch(err => logger.log('WARN', `Failed to write SimHash registry for ${manifest.name}: ${err.message}`));
|
|
133
99
|
}
|
|
134
100
|
|
|
101
|
+
// 3. Behavioral Comparison
|
|
135
102
|
if (newSimHash === stored.simHash) {
|
|
103
|
+
// STABLE: The logic is identical for mocked data.
|
|
104
|
+
// We mark this as stable, allowing the Orchestrator to skip it
|
|
105
|
+
// UNLESS the actual production input data has changed.
|
|
136
106
|
stableUpdates.push({
|
|
137
107
|
...item,
|
|
138
108
|
reason: "Code Updated (Logic Stable)",
|
|
@@ -140,6 +110,7 @@ async function verifyBehavioralStability(candidates, manifestMap, dailyStatus, l
|
|
|
140
110
|
newHash: manifest.hash
|
|
141
111
|
});
|
|
142
112
|
} else {
|
|
113
|
+
// UNSTABLE: The logic actually produces different results.
|
|
143
114
|
trueReRuns.push({
|
|
144
115
|
...item,
|
|
145
116
|
reason: item.reason + ` [SimHash Mismatch]`,
|
|
@@ -240,7 +211,6 @@ async function generateBuildReport(config, dependencies, manifest, daysBack = 90
|
|
|
240
211
|
|
|
241
212
|
let totalRun = 0, totalReRun = 0, totalStable = 0;
|
|
242
213
|
|
|
243
|
-
// [FIX] Reduced concurrency from 20 to 5 to avoid Firestore DEADLINE_EXCEEDED
|
|
244
214
|
const limit = pLimit(5);
|
|
245
215
|
|
|
246
216
|
const processingPromises = datesToCheck.map(dateStr => limit(async () => {
|
|
@@ -305,12 +275,15 @@ async function generateBuildReport(config, dependencies, manifest, daysBack = 90
|
|
|
305
275
|
const updatesPayload = {};
|
|
306
276
|
for (const stable of stableUpdates) {
|
|
307
277
|
const m = manifestMap.get(stable.name);
|
|
308
|
-
|
|
278
|
+
// We must grab the EXISTING data from the DB to preserve the OLD result hash.
|
|
279
|
+
// This allows the Orchestrator to compare Old Result vs New Dependency Result.
|
|
280
|
+
const stored = dailyStatus[stable.name];
|
|
281
|
+
if (m && stored) {
|
|
309
282
|
updatesPayload[stable.name] = {
|
|
310
|
-
hash: m.hash,
|
|
311
|
-
simHash: stable.simHash,
|
|
312
|
-
resultHash:
|
|
313
|
-
dependencyResultHashes:
|
|
283
|
+
hash: m.hash, // The NEW Code Hash
|
|
284
|
+
simHash: stable.simHash, // The NEW SimHash (Same as old)
|
|
285
|
+
resultHash: stored.resultHash, // The OLD Result Hash (Preserved)
|
|
286
|
+
dependencyResultHashes: stored.dependencyResultHashes || {}, // OLD Deps (Preserved)
|
|
314
287
|
category: m.category,
|
|
315
288
|
composition: m.composition,
|
|
316
289
|
lastUpdated: new Date()
|
|
@@ -324,19 +297,16 @@ async function generateBuildReport(config, dependencies, manifest, daysBack = 90
|
|
|
324
297
|
}
|
|
325
298
|
}
|
|
326
299
|
|
|
327
|
-
// 3. BLOCKED / IMPOSSIBLE / UPTODATE
|
|
328
300
|
analysis.blocked.forEach (item => pushIfValid(dateSummary.blocked, item));
|
|
329
301
|
analysis.failedDependency.forEach (item => pushIfValid(dateSummary.blocked, item, "Dependency Missing"));
|
|
330
302
|
analysis.impossible.forEach (item => pushIfValid(dateSummary.impossible, item));
|
|
331
303
|
analysis.skipped.forEach (item => pushIfValid(dateSummary.uptodate, item, "Up To Date"));
|
|
332
304
|
|
|
333
|
-
// Meta stats
|
|
334
305
|
const includedCount = dateSummary.run.length + dateSummary.rerun.length + dateSummary.stable.length +
|
|
335
306
|
dateSummary.blocked.length + dateSummary.impossible.length + dateSummary.uptodate.length;
|
|
336
307
|
dateSummary.meta.totalIncluded = includedCount;
|
|
337
308
|
dateSummary.meta.match = (includedCount === expectedCount);
|
|
338
309
|
|
|
339
|
-
// Write Immediately
|
|
340
310
|
await db.collection('computation_build_records')
|
|
341
311
|
.doc(buildId)
|
|
342
312
|
.collection('details')
|
|
@@ -1,10 +1,14 @@
|
|
|
1
1
|
/**
|
|
2
|
-
*
|
|
3
|
-
*
|
|
2
|
+
* {
|
|
3
|
+
* type: uploaded file
|
|
4
|
+
* fileName: computation-system/topology/HashManager.js
|
|
5
|
+
* }
|
|
6
|
+
*/
|
|
7
|
+
/**
|
|
8
|
+
* @fileoverview Manages code versioning and legacy mappings.
|
|
9
|
+
* UPDATED: Removed global infrastructure scanning. Now relies on Manual Epochs.
|
|
4
10
|
*/
|
|
5
11
|
const crypto = require('crypto');
|
|
6
|
-
const fs = require('fs');
|
|
7
|
-
const path = require('path');
|
|
8
12
|
|
|
9
13
|
// Legacy Keys Mapping (Ensures backward compatibility)
|
|
10
14
|
const LEGACY_MAPPING = {
|
|
@@ -39,66 +43,13 @@ function generateCodeHash(codeString) {
|
|
|
39
43
|
return crypto.createHash('sha256').update(clean).digest('hex');
|
|
40
44
|
}
|
|
41
45
|
|
|
42
|
-
// --- INFRASTRUCTURE HASHING (The "System Fingerprint") ---
|
|
43
|
-
|
|
44
|
-
const SYSTEM_ROOT = path.resolve(__dirname, '..');
|
|
45
|
-
const IGNORED_DIRS = new Set(['node_modules', '.git', '.idea', 'coverage', 'logs', 'tests', 'docs']);
|
|
46
|
-
const IGNORED_FILES = new Set(['package-lock.json', '.DS_Store', '.env', 'README.md']);
|
|
47
|
-
|
|
48
|
-
function walkSync(dir, fileList = []) {
|
|
49
|
-
const files = fs.readdirSync(dir);
|
|
50
|
-
files.forEach(file => {
|
|
51
|
-
if (IGNORED_FILES.has(file)) return;
|
|
52
|
-
const filePath = path.join(dir, file);
|
|
53
|
-
const stat = fs.statSync(filePath);
|
|
54
|
-
if (stat.isDirectory()) {
|
|
55
|
-
if (!IGNORED_DIRS.has(file)) {
|
|
56
|
-
walkSync(filePath, fileList);
|
|
57
|
-
}
|
|
58
|
-
} else {
|
|
59
|
-
// Hash JS, JSON, and YAML (Workflows)
|
|
60
|
-
if (file.endsWith('.js') || file.endsWith('.json') || file.endsWith('.yaml')) {
|
|
61
|
-
fileList.push(filePath);
|
|
62
|
-
}
|
|
63
|
-
}
|
|
64
|
-
});
|
|
65
|
-
return fileList;
|
|
66
|
-
}
|
|
67
|
-
|
|
68
46
|
/**
|
|
69
|
-
*
|
|
70
|
-
*
|
|
71
|
-
*
|
|
47
|
+
* DEPRECATED: Previously walked the file system.
|
|
48
|
+
* Now returns a static string because we rely on SYSTEM_EPOCH for global versioning.
|
|
49
|
+
* Kept for backward compatibility with older consumers.
|
|
72
50
|
*/
|
|
73
51
|
function getInfrastructureHash() {
|
|
74
|
-
|
|
75
|
-
const allFiles = walkSync(SYSTEM_ROOT);
|
|
76
|
-
allFiles.sort(); // Crucial for determinism
|
|
77
|
-
|
|
78
|
-
const bigHash = crypto.createHash('sha256');
|
|
79
|
-
|
|
80
|
-
for (const filePath of allFiles) {
|
|
81
|
-
const content = fs.readFileSync(filePath, 'utf8');
|
|
82
|
-
const relativePath = path.relative(SYSTEM_ROOT, filePath);
|
|
83
|
-
|
|
84
|
-
let cleanContent = content;
|
|
85
|
-
|
|
86
|
-
// Reuse the standard code hash logic for JS files to be consistent
|
|
87
|
-
if (filePath.endsWith('.js')) {
|
|
88
|
-
cleanContent = generateCodeHash(content);
|
|
89
|
-
} else {
|
|
90
|
-
// For JSON/YAML, just strip whitespace
|
|
91
|
-
cleanContent = content.replace(/\s+/g, '');
|
|
92
|
-
}
|
|
93
|
-
|
|
94
|
-
bigHash.update(`${relativePath}:${cleanContent}|`);
|
|
95
|
-
}
|
|
96
|
-
|
|
97
|
-
return bigHash.digest('hex');
|
|
98
|
-
} catch (e) {
|
|
99
|
-
console.warn(`[HashManager] ⚠️ Failed to generate infra hash: ${e.message}`);
|
|
100
|
-
return 'infra_error';
|
|
101
|
-
}
|
|
52
|
+
return 'MANUAL_EPOCH_MODE';
|
|
102
53
|
}
|
|
103
54
|
|
|
104
55
|
module.exports = { LEGACY_MAPPING, generateCodeHash, getInfrastructureHash };
|