@cocreate/server-autoscaler 1.0.1 → 1.0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.github/workflows/automated.yml +1 -1
- package/CHANGELOG.md +7 -0
- package/README.md +139 -55
- package/package.json +13 -10
- package/release.config.js +1 -1
- package/src/geoScaling.js +223 -0
- package/src/index.js +305 -364
- package/src/test.js +113 -0
package/src/index.js
CHANGED
|
@@ -17,415 +17,377 @@
|
|
|
17
17
|
********************************************************************************/
|
|
18
18
|
|
|
19
19
|
import cluster from 'node:cluster';
|
|
20
|
-
import https from 'node:https';
|
|
21
|
-
import os from 'node:os';
|
|
22
20
|
import Config from '@cocreate/config';
|
|
23
21
|
import { getValueFromObject } from '@cocreate/utils';
|
|
22
|
+
import { getCloudProvider } from './geoScaling.js';
|
|
24
23
|
|
|
25
24
|
// Guard clause verification to ensure cluster API abstraction integrity
|
|
26
|
-
const
|
|
25
|
+
const isMaster = cluster.isMaster;
|
|
27
26
|
|
|
28
|
-
//
|
|
29
|
-
// Module-Level Sandbox State (ESM Singleton Container)
|
|
30
|
-
// ==============================================================================
|
|
31
|
-
let serverCtx = null; // Central reference to CoCreateServer Primary instance
|
|
27
|
+
let server = null; // Central reference to CoCreateServer Master instance
|
|
32
28
|
let autoscaleConfig = null; // Configured thresholds and duration constants
|
|
33
29
|
let isRunning = false; // Execution loop flag
|
|
34
30
|
|
|
35
|
-
// Consecutive tracking counters to filter transient spikes
|
|
31
|
+
// Consecutive tracking counters to filter transient CPU spikes
|
|
36
32
|
let consecutiveCpuSpikes = 0;
|
|
37
|
-
let
|
|
33
|
+
let consecutiveLowLoadFlushes = 0;
|
|
38
34
|
|
|
39
|
-
//
|
|
40
|
-
let lastScaleOperationTimestamp = 0;
|
|
41
|
-
|
|
42
|
-
// Default operational values for metrics-driven autoscale thresholds
|
|
35
|
+
// Default operational values for telemetry-driven autoscale thresholds
|
|
43
36
|
const defaults = {
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
37
|
+
// HORIZONTAL THRESHOLDS & SCALE OUT
|
|
38
|
+
scaleOut: {
|
|
39
|
+
cpuThreshold: 85, // % CPU trigger
|
|
40
|
+
memoryThreshold: 85, // % Memory trigger
|
|
41
|
+
consecutiveFlushes: 3, // Sustained load required
|
|
42
|
+
maxClusterSize: 0 // Global ceiling (0 = uncapped)
|
|
43
|
+
},
|
|
44
|
+
|
|
45
|
+
// SCALE IN THRESHOLDS & STRATEGY
|
|
46
|
+
scaleIn: {
|
|
47
|
+
cpuThreshold: 10, // % CPU low load trigger
|
|
48
|
+
memoryThreshold: 45, // % Memory low load trigger
|
|
49
|
+
consecutiveFlushes: 10, // Sustained low load (~10 min window)
|
|
50
|
+
preservation: "continent", // "global" | "continent" | "country" | "region"
|
|
51
|
+
minClusterSize: 2, // Minimum cluster floor
|
|
52
|
+
decommissionStrategy: "local" // "local" (autonomous self-decommissioning) | "oldest" (target oldest idle node)
|
|
53
|
+
},
|
|
54
|
+
|
|
55
|
+
// GEO-SCALING & EDGE SHEDDING
|
|
56
|
+
geoScale: {
|
|
57
|
+
enabled: true,
|
|
58
|
+
sheddingThreshold: 20, // % unserved traffic required to spawn edge node
|
|
59
|
+
densityBy: "continent" // "continent" | "country" | "region"
|
|
60
|
+
},
|
|
61
|
+
|
|
62
|
+
// REGIONAL VERTICAL TIERING
|
|
63
|
+
vertical: {
|
|
64
|
+
enabled: true,
|
|
65
|
+
maxNodesPerTier: 2, // Up to 2 horizontal nodes per tier in a region before stepping up to next tier
|
|
66
|
+
tiers: [
|
|
67
|
+
{ name: "nano", vCPU: 1, memoryGb: 2 },
|
|
68
|
+
{ name: "small", vCPU: 2, memoryGb: 8 },
|
|
69
|
+
{ name: "medium", vCPU: 4, memoryGb: 16 },
|
|
70
|
+
{ name: "large", vCPU: 8, memoryGb: 32 },
|
|
71
|
+
{ name: "xlarge", vCPU: 16, memoryGb: 64 },
|
|
72
|
+
{ name: "2xlarge", vCPU: 32, memoryGb: 128 },
|
|
73
|
+
{ name: "4xlarge", vCPU: 64, memoryGb: 256 }
|
|
74
|
+
]
|
|
75
|
+
}
|
|
53
76
|
};
|
|
54
77
|
|
|
55
78
|
/**
|
|
56
|
-
*
|
|
57
|
-
|
|
58
|
-
function normalizePeersList(result) {
|
|
59
|
-
if (!result) return [];
|
|
60
|
-
if (Array.isArray(result)) return result;
|
|
61
|
-
if (Array.isArray(result.object)) return result.object;
|
|
62
|
-
if (Array.isArray(result.data)) return result.data;
|
|
63
|
-
return [];
|
|
64
|
-
}
|
|
65
|
-
|
|
66
|
-
/**
|
|
67
|
-
* Recursively parses and evaluates an AST rule against a provided context.
|
|
79
|
+
* Queries the database via CRUD for active servers matching given filter query where master is running.
|
|
80
|
+
* Accepts optional query parameters to target specific geographic zones or regions.
|
|
68
81
|
*
|
|
69
|
-
* @param {Object}
|
|
70
|
-
* @
|
|
71
|
-
* @returns {boolean} Outcome of the evaluation statement
|
|
82
|
+
* @param {Object} [query={}] - Optional query filters to merge with default master status
|
|
83
|
+
* @returns {Object|null} Cluster snapshot including averages and raw active servers array
|
|
72
84
|
*/
|
|
73
|
-
function
|
|
74
|
-
|
|
85
|
+
async function getClusterDetails(query = {}) {
|
|
86
|
+
try {
|
|
87
|
+
// Query servers array for active servers where master is running
|
|
88
|
+
const readResponse = await server.crud.send({
|
|
89
|
+
method: 'object.read',
|
|
90
|
+
array: 'servers',
|
|
91
|
+
$filter: {
|
|
92
|
+
query: {
|
|
93
|
+
'master.status': 'running',
|
|
94
|
+
...query
|
|
95
|
+
}
|
|
96
|
+
},
|
|
97
|
+
organization_id: server.organization_id,
|
|
98
|
+
host: server.host
|
|
99
|
+
});
|
|
75
100
|
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
const leftVal = evaluateAST(node.left, context);
|
|
79
|
-
|
|
80
|
-
// Short-circuit evaluations to optimize V8 execution loops
|
|
81
|
-
if (node.operator === 'OR' && leftVal) return true;
|
|
82
|
-
if (node.operator === 'AND' && !leftVal) return false;
|
|
83
|
-
|
|
84
|
-
const rightVal = evaluateAST(node.right, context);
|
|
85
|
-
if (node.operator === 'OR') return leftVal || rightVal;
|
|
86
|
-
if (node.operator === 'AND') return leftVal && rightVal;
|
|
87
|
-
break;
|
|
88
|
-
}
|
|
101
|
+
const activeServers = Array.isArray(readResponse?.object) ? readResponse.object : [];
|
|
102
|
+
if (activeServers.length === 0) return null;
|
|
89
103
|
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
const targetValue = node.value;
|
|
104
|
+
let totalCpu = 0;
|
|
105
|
+
let totalMem = 0;
|
|
93
106
|
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
switch (node.operator) {
|
|
100
|
-
case 'GT': return metricValue > targetValue;
|
|
101
|
-
case 'GTE': return metricValue >= targetValue;
|
|
102
|
-
case 'LT': return metricValue < targetValue;
|
|
103
|
-
case 'LTE': return metricValue <= targetValue;
|
|
104
|
-
case 'EQ': return metricValue === targetValue;
|
|
105
|
-
case 'NEQ': return metricValue !== targetValue;
|
|
106
|
-
default: return false;
|
|
107
|
-
}
|
|
107
|
+
for (const serverRecord of activeServers) {
|
|
108
|
+
const peerTelemetry = serverRecord.telemetry;
|
|
109
|
+
totalCpu += peerTelemetry?.cpu?.totalUsage || 0;
|
|
110
|
+
totalMem += peerTelemetry?.memory?.percentage || 0;
|
|
108
111
|
}
|
|
109
112
|
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
+
const count = activeServers.length;
|
|
114
|
+
|
|
115
|
+
return {
|
|
116
|
+
avgCpu: totalCpu / count,
|
|
117
|
+
avgMemory: totalMem / count,
|
|
118
|
+
totalCpu,
|
|
119
|
+
totalMem,
|
|
120
|
+
totalCount: count,
|
|
121
|
+
servers: activeServers // Raw active server array from database
|
|
122
|
+
};
|
|
123
|
+
} catch (err) {
|
|
124
|
+
console.error("[@cocreate/server-autoscaler] Error querying cluster details from DB:", err.message);
|
|
125
|
+
return null;
|
|
113
126
|
}
|
|
114
|
-
return false;
|
|
115
127
|
}
|
|
116
128
|
|
|
117
129
|
/**
|
|
118
|
-
*
|
|
119
|
-
*
|
|
130
|
+
* Validates whether scaling out is necessary by checking Node-level average first,
|
|
131
|
+
* followed by single-flush Cluster capacity from active server records in the database.
|
|
132
|
+
* Also enforces maxClusterSize if explicitly configured (> 0).
|
|
120
133
|
*
|
|
121
|
-
* @param {
|
|
122
|
-
* @
|
|
123
|
-
* @param {Object} actionConfig - Specific configuration parameters extracted from AST action node
|
|
134
|
+
* @param {Object} telemetry - Telemetry payload frame
|
|
135
|
+
* @returns {boolean} True if scaling out is authorized, False if cordoning alone is sufficient or max cap reached
|
|
124
136
|
*/
|
|
125
|
-
async function
|
|
126
|
-
const
|
|
127
|
-
const
|
|
128
|
-
|
|
129
|
-
// Build standard AST-modeled API Payload Object
|
|
130
|
-
const payload = {
|
|
131
|
-
method: actionConfig.method || 'infrastructure.scale',
|
|
132
|
-
timestamp: new Date().toISOString(),
|
|
133
|
-
serverId: serverCtx?.id || 'simulated-server-id',
|
|
134
|
-
direction: direction,
|
|
135
|
-
provider: actionConfig.provider || 'round-robin',
|
|
136
|
-
spec: actionConfig.spec || 'standard',
|
|
137
|
-
region: actionConfig.region || 'local',
|
|
138
|
-
reason: {
|
|
139
|
-
avgCpu: metrics.cpu?.avg || metrics.cpu?.totalUsage || 0,
|
|
140
|
-
avgMemory: metrics.memory?.avg_used_pct || metrics.memory?.percentage || 0,
|
|
141
|
-
message: `Autoscale execution triggered dynamically by AST rule evaluation.`
|
|
142
|
-
},
|
|
143
|
-
token: clusterSecret
|
|
144
|
-
};
|
|
145
|
-
|
|
146
|
-
// 1. Broadcast scaling details locally and over the P2P Server Mesh
|
|
147
|
-
if (serverCtx && typeof serverCtx.send === 'function') {
|
|
148
|
-
serverCtx.send({
|
|
149
|
-
method: 'infrastructure.scale',
|
|
150
|
-
broadcast: true,
|
|
151
|
-
payload: payload
|
|
152
|
-
});
|
|
137
|
+
async function shouldScaleOutCluster(telemetry) {
|
|
138
|
+
const scaleOutCpu = parseInt(getValueFromObject(autoscaleConfig, 'autoscale.scaleOut.cpuThreshold'), 10) || defaults.scaleOut.cpuThreshold;
|
|
139
|
+
const targetMemory = parseInt(getValueFromObject(autoscaleConfig, 'autoscale.scaleOut.memoryThreshold'), 10) || defaults.scaleOut.memoryThreshold;
|
|
140
|
+
const maxClusterSize = parseInt(getValueFromObject(autoscaleConfig, 'autoscale.scaleOut.maxClusterSize'), 10) || defaults.scaleOut.maxClusterSize;
|
|
153
141
|
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
direction,
|
|
158
|
-
metrics: payload.reason
|
|
159
|
-
});
|
|
160
|
-
}
|
|
142
|
+
// STEP 1: Evaluate local Node average across workers
|
|
143
|
+
const nodeAvg = calculateNodeAverage(telemetry);
|
|
144
|
+
console.log(`[@cocreate/server-autoscaler] [EVALUATION TIER 1] Node Average -> CPU: ${nodeAvg.avgCpu.toFixed(1)}%, Memory: ${nodeAvg.avgMemory.toFixed(1)}%`);
|
|
161
145
|
|
|
162
|
-
|
|
163
|
-
console.log(`\n=============================================================================`);
|
|
164
|
-
console.log(`[AUTOSCALER ALERT] DYNAMIC RESOURCE SCALE TRIGGERED`);
|
|
165
|
-
console.log(`=============================================================================`);
|
|
166
|
-
console.log(`* ACTION DIRECTION : ${direction.toUpperCase()}`);
|
|
167
|
-
console.log(`* TARGET REGION : ${payload.region.toUpperCase()}`);
|
|
168
|
-
console.log(`* SYSTEM PROFILE : ${payload.spec.toUpperCase()}`);
|
|
169
|
-
console.log(`* INSTANCE METRIC : CPU [${payload.reason.avgCpu}%], RAM [${payload.reason.avgMemory}%]`);
|
|
170
|
-
console.log(`* TRIGGER MESSAGE : ${payload.reason.message}`);
|
|
171
|
-
console.log(`=============================================================================`);
|
|
146
|
+
const isNodeOverloaded = nodeAvg.avgCpu >= scaleOutCpu || nodeAvg.avgMemory >= targetMemory;
|
|
172
147
|
|
|
173
|
-
|
|
148
|
+
if (!isNodeOverloaded) {
|
|
149
|
+
console.log(`[@cocreate/server-autoscaler] [EVALUATION TIER 1] Node average is below thresholds. Cordoning worker is sufficient. Scaling out CANCELLED.`);
|
|
150
|
+
return false;
|
|
151
|
+
}
|
|
174
152
|
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
hostname: urlObj.hostname,
|
|
179
|
-
port: urlObj.port || 443,
|
|
180
|
-
path: urlObj.pathname + urlObj.search,
|
|
181
|
-
method: 'POST',
|
|
182
|
-
headers: {
|
|
183
|
-
'Content-Type': 'application/json',
|
|
184
|
-
'Content-Length': Buffer.byteLength(JSON.stringify(payload))
|
|
185
|
-
},
|
|
186
|
-
timeout: 5000 // Limit network stall times
|
|
187
|
-
};
|
|
153
|
+
// STEP 2: Evaluate Cluster capacity across all running servers in DB
|
|
154
|
+
console.log(`[@cocreate/server-autoscaler] [EVALUATION TIER 2] Node is overloaded. Querying DB for active cluster details...`);
|
|
155
|
+
const clusterData = await getClusterDetails();
|
|
188
156
|
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
} else {
|
|
198
|
-
reject(new Error(`Scaling HTTP request failed: Status ${res.statusCode}`));
|
|
199
|
-
}
|
|
200
|
-
});
|
|
201
|
-
});
|
|
157
|
+
if (clusterData && clusterData.totalCount > 0) {
|
|
158
|
+
console.log(`[@cocreate/server-autoscaler] [EVALUATION TIER 2] Cluster Average (${clusterData.totalCount} servers) -> CPU: ${clusterData.avgCpu.toFixed(1)}%, Memory: ${clusterData.avgMemory.toFixed(1)}%`);
|
|
159
|
+
|
|
160
|
+
// Check optional hard cluster size ceiling (if configured)
|
|
161
|
+
if (maxClusterSize > 0 && clusterData.totalCount >= maxClusterSize) {
|
|
162
|
+
console.warn(`[@cocreate/server-autoscaler] [EVALUATION TIER 2] Max cluster size ceiling reached (${clusterData.totalCount}/${maxClusterSize}). Scaling out CANCELLED.`);
|
|
163
|
+
return false;
|
|
164
|
+
}
|
|
202
165
|
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
console.log(
|
|
211
|
-
return { simulated: true, status: 'acknowledged', direction };
|
|
166
|
+
const isClusterOverloaded = clusterData.avgCpu >= scaleOutCpu || clusterData.avgMemory >= targetMemory;
|
|
167
|
+
|
|
168
|
+
if (!isClusterOverloaded) {
|
|
169
|
+
console.log(`[@cocreate/server-autoscaler] [EVALUATION TIER 2] Remaining cluster has adequate capacity. Cordoning will rebalance traffic. Scaling out CANCELLED.`);
|
|
170
|
+
return false;
|
|
171
|
+
}
|
|
172
|
+
} else {
|
|
173
|
+
console.log(`[@cocreate/server-autoscaler] [EVALUATION TIER 2] Server telemetries unavailable in DB. Proceeding with node-based scaling decision.`);
|
|
212
174
|
}
|
|
213
|
-
}
|
|
214
175
|
|
|
215
|
-
|
|
216
|
-
* Calculates a dynamic safety memory threshold percentage based on the physical core-to-memory ratio.
|
|
217
|
-
* Ensures the system has enough physical MB of headroom to survive the boot provisioning delay.
|
|
218
|
-
*
|
|
219
|
-
* @returns {number} Calculated maximum safe memory percentage threshold (capped between 50% and 90%)
|
|
220
|
-
*/
|
|
221
|
-
export function calculateDynamicSafetyThreshold() {
|
|
222
|
-
const coresCount = os.cpus().length;
|
|
223
|
-
const totalMemoryMb = os.totalmem() / 1024 / 1024;
|
|
224
|
-
|
|
225
|
-
const allocationRatePerCoreMs = 15; // Estimated peak allocation speed in MB/sec under sustained load
|
|
226
|
-
const bootWindowSeconds = 60; // Boot delay window for new instances
|
|
227
|
-
|
|
228
|
-
const requiredHeadroomMb = coresCount * allocationRatePerCoreMs * bootWindowSeconds;
|
|
229
|
-
const headroomPercentage = (requiredHeadroomMb / totalMemoryMb) * 100;
|
|
230
|
-
|
|
231
|
-
const maxSafePercentage = Math.round(100 - headroomPercentage);
|
|
232
|
-
|
|
233
|
-
return Math.max(50, Math.min(90, maxSafePercentage));
|
|
176
|
+
return true;
|
|
234
177
|
}
|
|
235
178
|
|
|
236
179
|
/**
|
|
237
|
-
*
|
|
238
|
-
*
|
|
180
|
+
* Evaluates whether it is safe for a node to scale in.
|
|
181
|
+
* Verifies global minimum limits, regional preservation, and projected cluster capacity.
|
|
239
182
|
*
|
|
240
|
-
* @
|
|
183
|
+
* @param {Object} telemetry - Telemetry context frame
|
|
184
|
+
* @returns {boolean} True if node scale-in is authorized
|
|
241
185
|
*/
|
|
242
|
-
async function
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
186
|
+
async function shouldScaleInCluster(telemetry) {
|
|
187
|
+
const minSize = parseInt(getValueFromObject(autoscaleConfig, 'autoscale.scaleIn.minClusterSize'), 10) || defaults.scaleIn.minClusterSize;
|
|
188
|
+
const scaleOutCpu = parseInt(getValueFromObject(autoscaleConfig, 'autoscale.scaleOut.cpuThreshold'), 10) || defaults.scaleOut.cpuThreshold;
|
|
189
|
+
const targetMemory = parseInt(getValueFromObject(autoscaleConfig, 'autoscale.scaleOut.memoryThreshold'), 10) || defaults.scaleOut.memoryThreshold;
|
|
190
|
+
const preservation = getValueFromObject(autoscaleConfig, 'autoscale.scaleIn.preservation') ?? defaults.scaleIn.preservation;
|
|
191
|
+
|
|
192
|
+
// Evaluate geographic preservation if enabled ("continent" | "country" | "region")
|
|
193
|
+
if (preservation && preservation !== "global") {
|
|
194
|
+
const localZone = server?.location?.[preservation] || server?.[preservation];
|
|
195
|
+
if (localZone) {
|
|
196
|
+
// Targeted query for active servers in this node's geographic zone
|
|
197
|
+
const zoneClusterData = await getClusterDetails({
|
|
198
|
+
[`location.${preservation}`]: localZone
|
|
199
|
+
});
|
|
200
|
+
|
|
201
|
+
if (!zoneClusterData || zoneClusterData.totalCount <= 1) {
|
|
202
|
+
console.log(`[@cocreate/server-autoscaler] [CONSENSUS] Scale-in rejected. Removing this node would leave '${localZone}' with no active nodes (Preservation: ${preservation}).`);
|
|
203
|
+
return false;
|
|
204
|
+
}
|
|
205
|
+
}
|
|
246
206
|
}
|
|
247
207
|
|
|
248
|
-
const
|
|
208
|
+
const clusterData = await getClusterDetails();
|
|
249
209
|
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
$filter: {
|
|
255
|
-
query: { status: 'active' },
|
|
256
|
-
limit: 0
|
|
257
|
-
},
|
|
258
|
-
organization_id: serverCtx.organization_id,
|
|
259
|
-
host: serverCtx.host
|
|
260
|
-
});
|
|
210
|
+
if (!clusterData || clusterData.totalCount <= 1) {
|
|
211
|
+
console.log(`[@cocreate/server-autoscaler] [CONSENSUS] Scale-in rejected. Insufficient active nodes found.`);
|
|
212
|
+
return false;
|
|
213
|
+
}
|
|
261
214
|
|
|
262
|
-
|
|
215
|
+
if (clusterData.totalCount <= minSize) {
|
|
216
|
+
console.log(`[@cocreate/server-autoscaler] [CONSENSUS] Scale-in rejected. Active nodes (${clusterData.totalCount}) is already at or below the minimum cluster limit (${minSize}).`);
|
|
217
|
+
return false;
|
|
218
|
+
}
|
|
263
219
|
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
220
|
+
// Calculate projected load on remaining nodes if this node is removed
|
|
221
|
+
const remainingCount = clusterData.totalCount - 1;
|
|
222
|
+
const projectedCpu = clusterData.totalCpu / remainingCount;
|
|
223
|
+
const projectedMemory = clusterData.totalMem / remainingCount;
|
|
268
224
|
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
console.
|
|
225
|
+
console.log(`[@cocreate/server-autoscaler] [SCALE IN CHECK] Projected remaining cluster load (${remainingCount} servers) -> CPU: ${projectedCpu.toFixed(1)}%, Memory: ${projectedMemory.toFixed(1)}%`);
|
|
226
|
+
|
|
227
|
+
if (projectedCpu >= scaleOutCpu || projectedMemory >= targetMemory) {
|
|
228
|
+
console.log(`[@cocreate/server-autoscaler] [CONSENSUS] Scale-in rejected. Removing this node would push remaining cluster load above scale-out thresholds.`);
|
|
273
229
|
return false;
|
|
274
230
|
}
|
|
231
|
+
|
|
232
|
+
console.log(`[@cocreate/server-autoscaler] [CONSENSUS] Average cluster threshold met for scale-in. Scale-in authorized. Node marked for scale in.`);
|
|
233
|
+
return true;
|
|
275
234
|
}
|
|
276
235
|
|
|
277
236
|
/**
|
|
278
|
-
*
|
|
237
|
+
* Triggers external cloud infrastructure provisioning APIs to add or remove node instances.
|
|
238
|
+
* Incorporates geographic edge targeting metadata and maxNodesPerTier vertical tier checks.
|
|
279
239
|
*
|
|
280
|
-
* @param {
|
|
240
|
+
* @param {string} action - 'out' or 'in'
|
|
241
|
+
* @param {Object} telemetry - Telemetry context frame
|
|
242
|
+
* @param {Object} meta - Optional metadata specs
|
|
281
243
|
*/
|
|
282
|
-
async function
|
|
283
|
-
|
|
244
|
+
async function executeScaleAction(action, telemetry, meta = {}) {
|
|
245
|
+
// If scaling out, calculate target region and determine spec tier using maxNodesPerTier
|
|
246
|
+
if (action === 'out') {
|
|
247
|
+
const clients = Array.from(
|
|
248
|
+
server?.wsManager?.clients?.values() ?? [],
|
|
249
|
+
client => client?.clientInfo
|
|
250
|
+
).filter(Boolean);
|
|
251
|
+
|
|
252
|
+
const geoTarget = getCloudProvider(clients, autoscaleConfig);
|
|
253
|
+
if (geoTarget) {
|
|
254
|
+
meta.geo = geoTarget;
|
|
255
|
+
console.log(`[@cocreate/server-autoscaler] Geo-Scaling active. Requesting provision in region: ${geoTarget.targetRegion}`);
|
|
256
|
+
}
|
|
284
257
|
|
|
285
|
-
|
|
258
|
+
// Check regional tier counts against maxNodesPerTier to decide whether to upgrade tier
|
|
259
|
+
const maxNodesPerTier = parseInt(getValueFromObject(autoscaleConfig, 'autoscale.vertical.maxNodesPerTier'), 10) || defaults.vertical.maxNodesPerTier;
|
|
260
|
+
const tiers = getValueFromObject(autoscaleConfig, 'autoscale.vertical.tiers') || defaults.vertical.tiers;
|
|
261
|
+
|
|
262
|
+
const targetRegion = geoTarget?.targetRegion || server?.infrastructure?.region || 'default-region';
|
|
263
|
+
|
|
264
|
+
// Targeted query for servers in the specific target infrastructure region
|
|
265
|
+
const regionalClusterData = await getClusterDetails({
|
|
266
|
+
'infrastructure.region': targetRegion
|
|
267
|
+
});
|
|
286
268
|
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
269
|
+
if (regionalClusterData && Array.isArray(regionalClusterData.servers)) {
|
|
270
|
+
const regionalTierCounts = {};
|
|
271
|
+
for (const serverRecord of regionalClusterData.servers) {
|
|
272
|
+
const peerTelemetry = serverRecord.telemetry;
|
|
273
|
+
const vCPU = peerTelemetry?.cpu?.cores?.length || serverRecord.totalWorkers || 1;
|
|
274
|
+
const memoryTotalBytes = peerTelemetry?.memory?.total || 0;
|
|
275
|
+
const memoryGb = memoryTotalBytes > 0 ? Math.round(memoryTotalBytes / (1024 * 1024 * 1024)) : 2;
|
|
290
276
|
|
|
291
|
-
|
|
277
|
+
const tier = matchTierName(vCPU, memoryGb);
|
|
278
|
+
regionalTierCounts[tier] = (regionalTierCounts[tier] || 0) + 1;
|
|
279
|
+
}
|
|
292
280
|
|
|
293
|
-
|
|
294
|
-
consecutiveCpuSpikes = 0;
|
|
295
|
-
consecutiveLowLoadTicks = 0;
|
|
281
|
+
let selectedTier = tiers[0]; // Start at smallest tier (e.g., nano)
|
|
296
282
|
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
283
|
+
for (const tierSpec of tiers) {
|
|
284
|
+
const countAtTier = regionalTierCounts[tierSpec.name] || 0;
|
|
285
|
+
if (countAtTier < maxNodesPerTier) {
|
|
286
|
+
selectedTier = tierSpec;
|
|
287
|
+
break;
|
|
288
|
+
}
|
|
289
|
+
// If maxNodesPerTier reached, step up to next tier spec
|
|
290
|
+
selectedTier = tierSpec;
|
|
291
|
+
}
|
|
292
|
+
|
|
293
|
+
meta.tier = selectedTier;
|
|
294
|
+
console.log(`[@cocreate/server-autoscaler] Vertical Tiering selected spec '${selectedTier.name}' (${selectedTier.vCPU} vCPU, ${selectedTier.memoryGb} GB RAM) for region '${targetRegion}'.`);
|
|
295
|
+
}
|
|
296
|
+
}
|
|
297
|
+
|
|
298
|
+
console.log(`[@cocreate/server-autoscaler] Requesting infrastructure scale-${action.toUpperCase()}...`, meta);
|
|
299
|
+
|
|
300
|
+
// Emit event across server event bus for infrastructure orchestrator
|
|
301
|
+
if (server?.emit) {
|
|
302
|
+
server.emit('telemetry.scaleAction', {
|
|
303
|
+
action,
|
|
304
|
+
telemetry,
|
|
305
|
+
meta,
|
|
306
|
+
timestamp: Date.now()
|
|
307
|
+
});
|
|
308
|
+
}
|
|
307
309
|
}
|
|
308
310
|
|
|
309
311
|
/**
|
|
310
|
-
*
|
|
311
|
-
*
|
|
312
|
+
* Processes telemetry frame payloads.
|
|
313
|
+
* - Memory evaluated INSTANTLY on every TICK to protect against sudden heap OOM crashes.
|
|
314
|
+
* - CPU evaluated across SUSTAINED FLUSHES to avoid over-provisioning on transient spikes.
|
|
315
|
+
* - Scale down evaluated across DEBOUNCED LOW FLUSHES.
|
|
312
316
|
*
|
|
313
|
-
* @param {Object}
|
|
317
|
+
* @param {Object} telemetry - Telemetry frame payload received on tick or flush dispatch
|
|
314
318
|
*/
|
|
315
|
-
async function
|
|
319
|
+
async function processTelemetryFrame(telemetry) {
|
|
316
320
|
if (!isRunning) return;
|
|
317
321
|
|
|
318
|
-
const
|
|
319
|
-
const
|
|
320
|
-
const
|
|
321
|
-
const
|
|
322
|
-
const spikesRequired = parseInt(getValueFromObject(autoscaleConfig, 'autoscale.
|
|
323
|
-
const
|
|
322
|
+
const scaleOutCpu = parseInt(getValueFromObject(autoscaleConfig, 'autoscale.scaleOut.cpuThreshold'), 10) || defaults.scaleOut.cpuThreshold;
|
|
323
|
+
const scaleInCpu = parseInt(getValueFromObject(autoscaleConfig, 'autoscale.scaleIn.cpuThreshold'), 10) || defaults.scaleIn.cpuThreshold;
|
|
324
|
+
const targetMemoryThreshold = parseInt(getValueFromObject(autoscaleConfig, 'autoscale.scaleOut.memoryThreshold'), 10) || defaults.scaleOut.memoryThreshold;
|
|
325
|
+
const scaleInMemory = parseInt(getValueFromObject(autoscaleConfig, 'autoscale.scaleIn.memoryThreshold'), 10) || defaults.scaleIn.memoryThreshold;
|
|
326
|
+
const spikesRequired = parseInt(getValueFromObject(autoscaleConfig, 'autoscale.scaleOut.consecutiveFlushes'), 10) || defaults.scaleOut.consecutiveFlushes;
|
|
327
|
+
const lowFlushesRequired = parseInt(getValueFromObject(autoscaleConfig, 'autoscale.scaleIn.consecutiveFlushes'), 10) || defaults.scaleIn.consecutiveFlushes;
|
|
324
328
|
|
|
325
|
-
|
|
329
|
+
const cpuAvg = telemetry.cpu?.totalUsage || 0;
|
|
330
|
+
const memoryAvg = telemetry.memory?.percentage || 0;
|
|
326
331
|
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
if (isSafetyLimitsEnabled) {
|
|
330
|
-
const dynamicSafetyThreshold = calculateDynamicSafetyThreshold();
|
|
331
|
-
if (targetMemoryThreshold > dynamicSafetyThreshold) {
|
|
332
|
-
console.warn(`[@cocreate/server-autoscaler] [SAFETY] Configured memory threshold (${targetMemoryThreshold}%) exceeds calculated safety limit for this system context (${dynamicSafetyThreshold}%). Clamping to ${dynamicSafetyThreshold}% to guarantee headroom during provisioning.`);
|
|
333
|
-
targetMemoryThreshold = dynamicSafetyThreshold;
|
|
334
|
-
}
|
|
335
|
-
}
|
|
332
|
+
// TRACK 1: INSTANT PER-TICK MEMORY EVALUATION
|
|
333
|
+
const isMemoryPeaked = memoryAvg >= targetMemoryThreshold;
|
|
336
334
|
|
|
337
|
-
if (
|
|
338
|
-
|
|
339
|
-
|
|
335
|
+
if (isMemoryPeaked) {
|
|
336
|
+
console.warn(`[@cocreate/server-autoscaler] [INSTANT TICK] Memory threshold exceeded (${memoryAvg}% >= limit: ${targetMemoryThreshold}%). Triggering immediate worker cordon & scale evaluation...`);
|
|
337
|
+
|
|
338
|
+
cordonWorkerNode(telemetry);
|
|
340
339
|
|
|
341
|
-
|
|
342
|
-
|
|
340
|
+
const authorized = await shouldScaleOutCluster(telemetry);
|
|
341
|
+
if (authorized) {
|
|
342
|
+
consecutiveCpuSpikes = 0;
|
|
343
|
+
await executeScaleAction('out', telemetry, { spec: 'memory-optimized', trigger: 'memory-instant-tick' });
|
|
344
|
+
}
|
|
345
|
+
return;
|
|
346
|
+
}
|
|
343
347
|
|
|
344
|
-
//
|
|
345
|
-
if (cpuAvg >=
|
|
348
|
+
// TRACK 2: SUSTAINED FLUSH-BASED CPU EVALUATION
|
|
349
|
+
if (cpuAvg >= scaleOutCpu) {
|
|
346
350
|
consecutiveCpuSpikes++;
|
|
347
|
-
|
|
351
|
+
consecutiveLowLoadFlushes = 0;
|
|
348
352
|
} else {
|
|
349
353
|
consecutiveCpuSpikes = 0;
|
|
350
354
|
}
|
|
351
355
|
|
|
352
|
-
const isUnderloaded = cpuAvg <=
|
|
356
|
+
const isUnderloaded = cpuAvg <= scaleInCpu && memoryAvg <= scaleInMemory;
|
|
353
357
|
|
|
354
358
|
if (isUnderloaded) {
|
|
355
|
-
|
|
359
|
+
consecutiveLowLoadFlushes++;
|
|
356
360
|
} else {
|
|
357
|
-
|
|
361
|
+
consecutiveLowLoadFlushes = 0;
|
|
358
362
|
}
|
|
359
363
|
|
|
360
|
-
|
|
361
|
-
const evaluationContext = {
|
|
362
|
-
...metricPayload,
|
|
363
|
-
consecutiveCpuSpikes,
|
|
364
|
-
consecutiveLowLoadTicks
|
|
365
|
-
};
|
|
364
|
+
const isCpuSustainedPeak = consecutiveCpuSpikes >= spikesRequired;
|
|
366
365
|
|
|
367
|
-
|
|
368
|
-
|
|
366
|
+
if (isCpuSustainedPeak) {
|
|
367
|
+
console.warn(`[@cocreate/server-autoscaler] [SUSTAINED FLUSH] Sustained CPU threshold exceeded for ${consecutiveCpuSpikes} consecutive flushes (${cpuAvg}% >= limit: ${scaleOutCpu}%). Triggering worker cordon & scale evaluation...`);
|
|
369
368
|
|
|
370
|
-
|
|
371
|
-
// --- DECLARATIVE AST EXECUTION TRACK ---
|
|
372
|
-
console.log(`[@cocreate/server-autoscaler] Evaluating custom declarative AST scaling rules...`);
|
|
373
|
-
|
|
374
|
-
const isTriggered = evaluateAST(customOrchestrationRule, evaluationContext);
|
|
375
|
-
|
|
376
|
-
if (isTriggered) {
|
|
377
|
-
const actionNode = customOrchestrationRule.action || {};
|
|
378
|
-
lastScaleOperationTimestamp = Date.now();
|
|
379
|
-
consecutiveCpuSpikes = 0;
|
|
380
|
-
consecutiveLowLoadTicks = 0;
|
|
381
|
-
|
|
382
|
-
console.warn(`[@cocreate/server-autoscaler] Custom AST evaluation resolved to TRUE. Triggering custom scaled action...`);
|
|
383
|
-
await executeScaleAction('up', metricPayload, actionNode);
|
|
384
|
-
return;
|
|
385
|
-
}
|
|
386
|
-
} else {
|
|
387
|
-
// --- TRADITIONAL DUAL-TRACK HARDCODED FALLBACK ---
|
|
388
|
-
const isMemoryPeaked = memoryAvg >= targetMemoryThreshold;
|
|
389
|
-
const isCpuExtremePeak = cpuAvg >= scaleUpCpuExtreme;
|
|
390
|
-
const isCpuSustainedPeak = consecutiveCpuSpikes >= spikesRequired;
|
|
391
|
-
|
|
392
|
-
if (isCpuExtremePeak || isCpuSustainedPeak || isMemoryPeaked) {
|
|
393
|
-
let triggerReason = '';
|
|
394
|
-
let targetSpec = 'standard';
|
|
395
|
-
|
|
396
|
-
if (isMemoryPeaked) {
|
|
397
|
-
triggerReason = `Incompressible memory threshold exceeded (${memoryAvg}% >= limit: ${targetMemoryThreshold}%)`;
|
|
398
|
-
targetSpec = 'memory-optimized';
|
|
399
|
-
} else if (isCpuExtremePeak) {
|
|
400
|
-
triggerReason = `Extreme CPU ceiling exceeded on a single flush (${cpuAvg}% >= limit: ${scaleUpCpuExtreme}%)`;
|
|
401
|
-
targetSpec = 'compute-optimized';
|
|
402
|
-
} else {
|
|
403
|
-
triggerReason = `Sustained CPU threshold exceeded for ${consecutiveCpuSpikes} consecutive cycles (${cpuAvg}% >= limit: ${scaleUpCpu}%)`;
|
|
404
|
-
targetSpec = 'compute-optimized';
|
|
405
|
-
}
|
|
369
|
+
cordonWorkerNode(telemetry);
|
|
406
370
|
|
|
407
|
-
|
|
371
|
+
const authorized = await shouldScaleOutCluster(telemetry);
|
|
372
|
+
if (authorized) {
|
|
408
373
|
consecutiveCpuSpikes = 0;
|
|
409
|
-
|
|
410
|
-
console.warn(`[@cocreate/server-autoscaler] Peak limit hit! ${triggerReason}. Scaling up...`);
|
|
411
|
-
await executeScaleAction('up', metricPayload, { spec: targetSpec });
|
|
412
|
-
return;
|
|
374
|
+
await executeScaleAction('out', telemetry, { spec: 'compute-optimized', trigger: 'cpu-sustained-flushes' });
|
|
413
375
|
}
|
|
376
|
+
return;
|
|
414
377
|
}
|
|
415
378
|
|
|
416
|
-
//
|
|
417
|
-
if (
|
|
418
|
-
console.log(`[@cocreate/server-autoscaler] Low load sustained for ${
|
|
379
|
+
// TRACK 3: SCALE IN EVALUATION (SUSTAINED LOW LOAD ACROSS FLUSHES)
|
|
380
|
+
if (consecutiveLowLoadFlushes >= lowFlushesRequired) {
|
|
381
|
+
console.log(`[@cocreate/server-autoscaler] Low load sustained for ${consecutiveLowLoadFlushes} consecutive flushes. Checking cluster state to verify consensus...`);
|
|
419
382
|
|
|
420
|
-
const
|
|
421
|
-
if (
|
|
422
|
-
|
|
423
|
-
consecutiveLowLoadTicks = 0;
|
|
383
|
+
const safeToScaleIn = await shouldScaleInCluster(telemetry);
|
|
384
|
+
if (safeToScaleIn) {
|
|
385
|
+
consecutiveLowLoadFlushes = 0;
|
|
424
386
|
|
|
425
|
-
console.log(`[@cocreate/server-autoscaler] Safe-
|
|
426
|
-
await executeScaleAction('
|
|
387
|
+
console.log(`[@cocreate/server-autoscaler] Safe-in consensus verified. Scaling in...`);
|
|
388
|
+
await executeScaleAction('in', telemetry);
|
|
427
389
|
} else {
|
|
428
|
-
|
|
390
|
+
consecutiveLowLoadFlushes = Math.round(lowFlushesRequired / 2);
|
|
429
391
|
}
|
|
430
392
|
}
|
|
431
393
|
}
|
|
@@ -433,71 +395,50 @@ async function processMetricFlush(metricPayload) {
|
|
|
433
395
|
/**
|
|
434
396
|
* Boots the autoscaler system exclusively on the Master/Primary process context.
|
|
435
397
|
*
|
|
436
|
-
* @param {Object}
|
|
398
|
+
* @param {Object} serverInstance - Central CoCreateServer instance references
|
|
437
399
|
* @returns {Object} Active autoscale control API
|
|
438
400
|
*/
|
|
439
|
-
export async function init(
|
|
440
|
-
|
|
441
|
-
|
|
401
|
+
export async function init(serverInstance) {
|
|
402
|
+
server = serverInstance;
|
|
403
|
+
|
|
404
|
+
if (!isMaster) {
|
|
405
|
+
console.log('[@cocreate/server-autoscaler] Autoscaler initialized on worker node (Standby).');
|
|
406
|
+
return;
|
|
442
407
|
}
|
|
443
408
|
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
'
|
|
448
|
-
|
|
409
|
+
try {
|
|
410
|
+
autoscaleConfig = await Config.get('autoscale');
|
|
411
|
+
} catch (e) {
|
|
412
|
+
console.warn('[@cocreate/server-autoscaler] Failed to load config, falling back to defaults.', e.message);
|
|
413
|
+
autoscaleConfig = {};
|
|
414
|
+
}
|
|
449
415
|
|
|
450
416
|
const isEnabled = getValueFromObject(autoscaleConfig, 'autoscale.enabled') !== false && getValueFromObject(autoscaleConfig, 'autoscale.enabled') !== 'false';
|
|
451
417
|
|
|
452
418
|
if (!isEnabled) {
|
|
453
|
-
console.log('[@cocreate/server-autoscaler]
|
|
454
|
-
return
|
|
419
|
+
console.log('[@cocreate/server-autoscaler] Autoscaling is disabled by configuration.');
|
|
420
|
+
return;
|
|
455
421
|
}
|
|
456
422
|
|
|
457
423
|
isRunning = true;
|
|
458
|
-
|
|
459
|
-
consecutiveCpuSpikes = 0;
|
|
460
|
-
consecutiveLowLoadTicks = 0;
|
|
461
424
|
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
console.error("[@cocreate/server-autoscaler] Error processing autoscale execution tick:", err);
|
|
466
|
-
});
|
|
467
|
-
}
|
|
425
|
+
// Subscribe to incoming telemetry frame dispatches published by server-telemetry.js
|
|
426
|
+
server.on('telemetry', (telemetry) => {
|
|
427
|
+
processTelemetryFrame(telemetry);
|
|
468
428
|
});
|
|
469
429
|
|
|
470
|
-
|
|
471
|
-
if (payload && payload.data) {
|
|
472
|
-
processEmergencyAlert(payload.data).catch(err => {
|
|
473
|
-
console.error("[@cocreate/server-autoscaler] Error processing emergency autoscale trigger:", err);
|
|
474
|
-
});
|
|
475
|
-
}
|
|
476
|
-
});
|
|
477
|
-
|
|
478
|
-
console.log('[@cocreate/server-autoscaler] Autoscaler initialized and subscribed to telemetry feed.');
|
|
479
|
-
|
|
480
|
-
return {
|
|
481
|
-
init,
|
|
482
|
-
stop,
|
|
483
|
-
processMetricFlush,
|
|
484
|
-
processEmergencyAlert,
|
|
485
|
-
executeScaleAction,
|
|
486
|
-
calculateDynamicSafetyThreshold
|
|
487
|
-
};
|
|
430
|
+
console.log('[@cocreate/server-autoscaler] Initialized and listening for telemetry on Master node.');
|
|
488
431
|
}
|
|
489
432
|
|
|
490
433
|
/**
|
|
491
|
-
*
|
|
434
|
+
* Halts any active autoscaler processing
|
|
492
435
|
*/
|
|
493
436
|
export function stop() {
|
|
494
437
|
isRunning = false;
|
|
495
|
-
|
|
496
|
-
consecutiveLowLoadTicks = 0;
|
|
497
|
-
console.log('[@cocreate/server-autoscaler] ServerAutoscaler monitoring stopped.');
|
|
438
|
+
console.log('[@cocreate/server-autoscaler] Execution loop halted.');
|
|
498
439
|
}
|
|
499
440
|
|
|
500
|
-
export default {
|
|
501
|
-
init,
|
|
502
|
-
stop
|
|
441
|
+
export default {
|
|
442
|
+
init,
|
|
443
|
+
stop
|
|
503
444
|
};
|