@cocreate/server-autoscaler 1.0.1 → 1.0.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/index.js CHANGED
@@ -17,415 +17,377 @@
17
17
  ********************************************************************************/
18
18
 
19
19
  import cluster from 'node:cluster';
20
- import https from 'node:https';
21
- import os from 'node:os';
22
20
  import Config from '@cocreate/config';
23
21
  import { getValueFromObject } from '@cocreate/utils';
22
+ import { getCloudProvider } from './geoScaling.js';
24
23
 
25
24
  // Guard clause verification to ensure cluster API abstraction integrity
26
- const isPrimary = cluster.isPrimary !== undefined ? cluster.isPrimary : cluster.isMaster;
25
+ const isMaster = cluster.isMaster;
27
26
 
28
- // ==============================================================================
29
- // Module-Level Sandbox State (ESM Singleton Container)
30
- // ==============================================================================
31
- let serverCtx = null; // Central reference to CoCreateServer Primary instance
27
+ let server = null; // Central reference to CoCreateServer Master instance
32
28
  let autoscaleConfig = null; // Configured thresholds and duration constants
33
29
  let isRunning = false; // Execution loop flag
34
30
 
35
- // Consecutive tracking counters to filter transient spikes
31
+ // Consecutive tracking counters to filter transient CPU spikes
36
32
  let consecutiveCpuSpikes = 0;
37
- let consecutiveLowLoadTicks = 0;
33
+ let consecutiveLowLoadFlushes = 0;
38
34
 
39
- // Timestamp tracking to prevent cascading scaling operations (Cooldown Management)
40
- let lastScaleOperationTimestamp = 0;
41
-
42
- // Default operational values for metrics-driven autoscale thresholds
35
+ // Default operational values for telemetry-driven autoscale thresholds
43
36
  const defaults = {
44
- scaleUpCpuThreshold: 80, // CPU usage percentage trigger for Scaling Up (Sustained)
45
- scaleUpCpuExtremeThreshold: 90, // Extreme CPU usage trigger for INSTANT Scaling Up (Single Flush)
46
- scaleDownCpuThreshold: 25, // CPU usage percentage trigger for Scaling Down
47
- scaleUpMemoryThreshold: 85, // Heap memory utilization trigger for Scaling Up
48
- consecutiveCpuSpikesRequired: 3, // Require 3 consecutive flushes of high CPU (e.g. 3 minutes)
49
- consecutiveLowTicksRequired: 10, // Require 10 consecutive flushes of low load (e.g. 10 minutes)
50
- minClusterSize: 2, // Never scale down below this active node count
51
- cooldownDurationMs: 300000, // Cooldown lockout period to stabilize nodes (default: 5 minutes)
52
- provisioningApiEndpoint: 'https://api.cocreate.app/v1/infrastructure/scale'
37
+ // HORIZONTAL THRESHOLDS & SCALE OUT
38
+ scaleOut: {
39
+ cpuThreshold: 85, // % CPU trigger
40
+ memoryThreshold: 85, // % Memory trigger
41
+ consecutiveFlushes: 3, // Sustained load required
42
+ maxClusterSize: 0 // Global ceiling (0 = uncapped)
43
+ },
44
+
45
+ // SCALE IN THRESHOLDS & STRATEGY
46
+ scaleIn: {
47
+ cpuThreshold: 10, // % CPU low load trigger
48
+ memoryThreshold: 45, // % Memory low load trigger
49
+ consecutiveFlushes: 10, // Sustained low load (~10 min window)
50
+ preservation: "continent", // "global" | "continent" | "country" | "region"
51
+ minClusterSize: 2, // Minimum cluster floor
52
+ decommissionStrategy: "local" // "local" (autonomous self-decommissioning) | "oldest" (target oldest idle node)
53
+ },
54
+
55
+ // GEO-SCALING & EDGE SHEDDING
56
+ geoScale: {
57
+ enabled: true,
58
+ sheddingThreshold: 20, // % unserved traffic required to spawn edge node
59
+ densityBy: "continent" // "continent" | "country" | "region"
60
+ },
61
+
62
+ // REGIONAL VERTICAL TIERING
63
+ vertical: {
64
+ enabled: true,
65
+ maxNodesPerTier: 2, // Up to 2 horizontal nodes per tier in a region before stepping up to next tier
66
+ tiers: [
67
+ { name: "nano", vCPU: 1, memoryGb: 2 },
68
+ { name: "small", vCPU: 2, memoryGb: 8 },
69
+ { name: "medium", vCPU: 4, memoryGb: 16 },
70
+ { name: "large", vCPU: 8, memoryGb: 32 },
71
+ { name: "xlarge", vCPU: 16, memoryGb: 64 },
72
+ { name: "2xlarge", vCPU: 32, memoryGb: 128 },
73
+ { name: "4xlarge", vCPU: 64, memoryGb: 256 }
74
+ ]
75
+ }
53
76
  };
54
77
 
55
78
  /**
56
- * Normalizes dataset arrays returned from standard CRUD database operations.
57
- */
58
- function normalizePeersList(result) {
59
- if (!result) return [];
60
- if (Array.isArray(result)) return result;
61
- if (Array.isArray(result.object)) return result.object;
62
- if (Array.isArray(result.data)) return result.data;
63
- return [];
64
- }
65
-
66
- /**
67
- * Recursively parses and evaluates an AST rule against a provided context.
79
+ * Queries the database via CRUD for active servers matching given filter query where master is running.
80
+ * Accepts optional query parameters to target specific geographic zones or regions.
68
81
  *
69
- * @param {Object} node - AST node definition
70
- * @param {Object} context - Combined telemetry metrics and state counters
71
- * @returns {boolean} Outcome of the evaluation statement
82
+ * @param {Object} [query={}] - Optional query filters to merge with default master status
83
+ * @returns {Object|null} Cluster snapshot including averages and raw active servers array
72
84
  */
73
- function evaluateAST(node, context) {
74
- if (!node) return false;
85
+ async function getClusterDetails(query = {}) {
86
+ try {
87
+ // Query servers array for active servers where master is running
88
+ const readResponse = await server.crud.send({
89
+ method: 'object.read',
90
+ array: 'servers',
91
+ $filter: {
92
+ query: {
93
+ 'master.status': 'running',
94
+ ...query
95
+ }
96
+ },
97
+ organization_id: server.organization_id,
98
+ host: server.host
99
+ });
75
100
 
76
- switch (node.type) {
77
- case 'LogicalExpression': {
78
- const leftVal = evaluateAST(node.left, context);
79
-
80
- // Short-circuit evaluations to optimize V8 execution loops
81
- if (node.operator === 'OR' && leftVal) return true;
82
- if (node.operator === 'AND' && !leftVal) return false;
83
-
84
- const rightVal = evaluateAST(node.right, context);
85
- if (node.operator === 'OR') return leftVal || rightVal;
86
- if (node.operator === 'AND') return leftVal && rightVal;
87
- break;
88
- }
101
+ const activeServers = Array.isArray(readResponse?.object) ? readResponse.object : [];
102
+ if (activeServers.length === 0) return null;
89
103
 
90
- case 'Condition': {
91
- const metricValue = getValueFromObject(context, node.metric);
92
- const targetValue = node.value;
104
+ let totalCpu = 0;
105
+ let totalMem = 0;
93
106
 
94
- if (metricValue === undefined) {
95
- console.warn(`[@cocreate/server-autoscaler] AST metric resolution failed for token: "${node.metric}"`);
96
- return false;
97
- }
98
-
99
- switch (node.operator) {
100
- case 'GT': return metricValue > targetValue;
101
- case 'GTE': return metricValue >= targetValue;
102
- case 'LT': return metricValue < targetValue;
103
- case 'LTE': return metricValue <= targetValue;
104
- case 'EQ': return metricValue === targetValue;
105
- case 'NEQ': return metricValue !== targetValue;
106
- default: return false;
107
- }
107
+ for (const serverRecord of activeServers) {
108
+ const peerTelemetry = serverRecord.telemetry;
109
+ totalCpu += peerTelemetry?.cpu?.totalUsage || 0;
110
+ totalMem += peerTelemetry?.memory?.percentage || 0;
108
111
  }
109
112
 
110
- default:
111
- console.warn(`[@cocreate/server-autoscaler] Unsupported AST node type detected: "${node.type}"`);
112
- return false;
113
+ const count = activeServers.length;
114
+
115
+ return {
116
+ avgCpu: totalCpu / count,
117
+ avgMemory: totalMem / count,
118
+ totalCpu,
119
+ totalMem,
120
+ totalCount: count,
121
+ servers: activeServers // Raw active server array from database
122
+ };
123
+ } catch (err) {
124
+ console.error("[@cocreate/server-autoscaler] Error querying cluster details from DB:", err.message);
125
+ return null;
113
126
  }
114
- return false;
115
127
  }
116
128
 
117
129
  /**
118
- * Invokes an external HTTPS API call modeled around standard CoCreate AST payload shapes.
119
- * If the Cloud Endpoint is offline or unreachable, falls back to a highly descriptive simulated log.
130
+ * Validates whether scaling out is necessary by checking Node-level average first,
131
+ * followed by single-flush Cluster capacity from active server records in the database.
132
+ * Also enforces maxClusterSize if explicitly configured (> 0).
120
133
  *
121
- * @param {string} direction - Target scaling direction ('up' | 'down')
122
- * @param {Object} metrics - Performance statistics triggering the scale event
123
- * @param {Object} actionConfig - Specific configuration parameters extracted from AST action node
134
+ * @param {Object} telemetry - Telemetry payload frame
135
+ * @returns {boolean} True if scaling out is authorized, False if cordoning alone is sufficient or max cap reached
124
136
  */
125
- async function executeScaleAction(direction, metrics, actionConfig = {}) {
126
- const targetUrl = getValueFromObject(autoscaleConfig, 'autoscale.apiEndpoint') || defaults.provisioningApiEndpoint;
127
- const clusterSecret = process.env.CLUSTER_SECRET || 'LOCAL_SIMULATED_SECRET';
128
-
129
- // Build standard AST-modeled API Payload Object
130
- const payload = {
131
- method: actionConfig.method || 'infrastructure.scale',
132
- timestamp: new Date().toISOString(),
133
- serverId: serverCtx?.id || 'simulated-server-id',
134
- direction: direction,
135
- provider: actionConfig.provider || 'round-robin',
136
- spec: actionConfig.spec || 'standard',
137
- region: actionConfig.region || 'local',
138
- reason: {
139
- avgCpu: metrics.cpu?.avg || metrics.cpu?.totalUsage || 0,
140
- avgMemory: metrics.memory?.avg_used_pct || metrics.memory?.percentage || 0,
141
- message: `Autoscale execution triggered dynamically by AST rule evaluation.`
142
- },
143
- token: clusterSecret
144
- };
145
-
146
- // 1. Broadcast scaling details locally and over the P2P Server Mesh
147
- if (serverCtx && typeof serverCtx.send === 'function') {
148
- serverCtx.send({
149
- method: 'infrastructure.scale',
150
- broadcast: true,
151
- payload: payload
152
- });
137
+ async function shouldScaleOutCluster(telemetry) {
138
+ const scaleOutCpu = parseInt(getValueFromObject(autoscaleConfig, 'autoscale.scaleOut.cpuThreshold'), 10) || defaults.scaleOut.cpuThreshold;
139
+ const targetMemory = parseInt(getValueFromObject(autoscaleConfig, 'autoscale.scaleOut.memoryThreshold'), 10) || defaults.scaleOut.memoryThreshold;
140
+ const maxClusterSize = parseInt(getValueFromObject(autoscaleConfig, 'autoscale.scaleOut.maxClusterSize'), 10) || defaults.scaleOut.maxClusterSize;
153
141
 
154
- serverCtx.send({
155
- method: 'metric.scaleAlert',
156
- broadcast: true,
157
- direction,
158
- metrics: payload.reason
159
- });
160
- }
142
+ // STEP 1: Evaluate local Node average across workers
143
+ const nodeAvg = calculateNodeAverage(telemetry);
144
+ console.log(`[@cocreate/server-autoscaler] [EVALUATION TIER 1] Node Average -> CPU: ${nodeAvg.avgCpu.toFixed(1)}%, Memory: ${nodeAvg.avgMemory.toFixed(1)}%`);
161
145
 
162
- // 2. Render highly visible console warning banners for local simulations
163
- console.log(`\n=============================================================================`);
164
- console.log(`[AUTOSCALER ALERT] DYNAMIC RESOURCE SCALE TRIGGERED`);
165
- console.log(`=============================================================================`);
166
- console.log(`* ACTION DIRECTION : ${direction.toUpperCase()}`);
167
- console.log(`* TARGET REGION : ${payload.region.toUpperCase()}`);
168
- console.log(`* SYSTEM PROFILE : ${payload.spec.toUpperCase()}`);
169
- console.log(`* INSTANCE METRIC : CPU [${payload.reason.avgCpu}%], RAM [${payload.reason.avgMemory}%]`);
170
- console.log(`* TRIGGER MESSAGE : ${payload.reason.message}`);
171
- console.log(`=============================================================================`);
146
+ const isNodeOverloaded = nodeAvg.avgCpu >= scaleOutCpu || nodeAvg.avgMemory >= targetMemory;
172
147
 
173
- console.log(`[@cocreate/server-autoscaler] [ACTION] Dispatching AST scaling payload to endpoint: ${targetUrl}`);
148
+ if (!isNodeOverloaded) {
149
+ console.log(`[@cocreate/server-autoscaler] [EVALUATION TIER 1] Node average is below thresholds. Cordoning worker is sufficient. Scaling out CANCELLED.`);
150
+ return false;
151
+ }
174
152
 
175
- try {
176
- const urlObj = new URL(targetUrl);
177
- const options = {
178
- hostname: urlObj.hostname,
179
- port: urlObj.port || 443,
180
- path: urlObj.pathname + urlObj.search,
181
- method: 'POST',
182
- headers: {
183
- 'Content-Type': 'application/json',
184
- 'Content-Length': Buffer.byteLength(JSON.stringify(payload))
185
- },
186
- timeout: 5000 // Limit network stall times
187
- };
153
+ // STEP 2: Evaluate Cluster capacity across all running servers in DB
154
+ console.log(`[@cocreate/server-autoscaler] [EVALUATION TIER 2] Node is overloaded. Querying DB for active cluster details...`);
155
+ const clusterData = await getClusterDetails();
188
156
 
189
- return await new Promise((resolve, reject) => {
190
- const req = https.request(options, (res) => {
191
- let body = '';
192
- res.on('data', (chunk) => body += chunk);
193
- res.on('end', () => {
194
- if (res.statusCode >= 200 && res.statusCode < 300) {
195
- console.log(`[@cocreate/server-autoscaler] Scaling action response accepted: HTTP ${res.statusCode}`);
196
- resolve(JSON.parse(body));
197
- } else {
198
- reject(new Error(`Scaling HTTP request failed: Status ${res.statusCode}`));
199
- }
200
- });
201
- });
157
+ if (clusterData && clusterData.totalCount > 0) {
158
+ console.log(`[@cocreate/server-autoscaler] [EVALUATION TIER 2] Cluster Average (${clusterData.totalCount} servers) -> CPU: ${clusterData.avgCpu.toFixed(1)}%, Memory: ${clusterData.avgMemory.toFixed(1)}%`);
159
+
160
+ // Check optional hard cluster size ceiling (if configured)
161
+ if (maxClusterSize > 0 && clusterData.totalCount >= maxClusterSize) {
162
+ console.warn(`[@cocreate/server-autoscaler] [EVALUATION TIER 2] Max cluster size ceiling reached (${clusterData.totalCount}/${maxClusterSize}). Scaling out CANCELLED.`);
163
+ return false;
164
+ }
202
165
 
203
- req.on('error', reject);
204
- req.write(JSON.stringify(payload));
205
- req.end();
206
- });
207
- } catch (err) {
208
- console.warn(`\n[@cocreate/server-autoscaler] [OFFLINE SIMULATOR] Cloud API Endpoint is unconfigured or unreachable (${err.message}).`);
209
- console.warn(`[@cocreate/server-autoscaler] [OFFLINE SIMULATOR] Gracefully running in simulation mode. scaling action succeeded locally.`);
210
- console.log(`=============================================================================\n`);
211
- return { simulated: true, status: 'acknowledged', direction };
166
+ const isClusterOverloaded = clusterData.avgCpu >= scaleOutCpu || clusterData.avgMemory >= targetMemory;
167
+
168
+ if (!isClusterOverloaded) {
169
+ console.log(`[@cocreate/server-autoscaler] [EVALUATION TIER 2] Remaining cluster has adequate capacity. Cordoning will rebalance traffic. Scaling out CANCELLED.`);
170
+ return false;
171
+ }
172
+ } else {
173
+ console.log(`[@cocreate/server-autoscaler] [EVALUATION TIER 2] Server telemetries unavailable in DB. Proceeding with node-based scaling decision.`);
212
174
  }
213
- }
214
175
 
215
- /**
216
- * Calculates a dynamic safety memory threshold percentage based on the physical core-to-memory ratio.
217
- * Ensures the system has enough physical MB of headroom to survive the boot provisioning delay.
218
- *
219
- * @returns {number} Calculated maximum safe memory percentage threshold (capped between 50% and 90%)
220
- */
221
- export function calculateDynamicSafetyThreshold() {
222
- const coresCount = os.cpus().length;
223
- const totalMemoryMb = os.totalmem() / 1024 / 1024;
224
-
225
- const allocationRatePerCoreMs = 15; // Estimated peak allocation speed in MB/sec under sustained load
226
- const bootWindowSeconds = 60; // Boot delay window for new instances
227
-
228
- const requiredHeadroomMb = coresCount * allocationRatePerCoreMs * bootWindowSeconds;
229
- const headroomPercentage = (requiredHeadroomMb / totalMemoryMb) * 100;
230
-
231
- const maxSafePercentage = Math.round(100 - headroomPercentage);
232
-
233
- return Math.max(50, Math.min(90, maxSafePercentage));
176
+ return true;
234
177
  }
235
178
 
236
179
  /**
237
- * Queries the shared servers directory database to ensure we can safely remove this node
238
- * without bringing the remaining active nodes below our configured minimum bounds.
180
+ * Evaluates whether it is safe for a node to scale in.
181
+ * Verifies global minimum limits, regional preservation, and projected cluster capacity.
239
182
  *
240
- * @returns {boolean} Verified consensus state allowing scale-down
183
+ * @param {Object} telemetry - Telemetry context frame
184
+ * @returns {boolean} True if node scale-in is authorized
241
185
  */
242
- async function isClusterSafeToScaleDown() {
243
- if (!serverCtx?.crud) {
244
- console.warn("[@cocreate/server-autoscaler] Database access layer is unavailable. Scaling down rejected.");
245
- return false;
186
+ async function shouldScaleInCluster(telemetry) {
187
+ const minSize = parseInt(getValueFromObject(autoscaleConfig, 'autoscale.scaleIn.minClusterSize'), 10) || defaults.scaleIn.minClusterSize;
188
+ const scaleOutCpu = parseInt(getValueFromObject(autoscaleConfig, 'autoscale.scaleOut.cpuThreshold'), 10) || defaults.scaleOut.cpuThreshold;
189
+ const targetMemory = parseInt(getValueFromObject(autoscaleConfig, 'autoscale.scaleOut.memoryThreshold'), 10) || defaults.scaleOut.memoryThreshold;
190
+ const preservation = getValueFromObject(autoscaleConfig, 'autoscale.scaleIn.preservation') ?? defaults.scaleIn.preservation;
191
+
192
+ // Evaluate geographic preservation if enabled ("continent" | "country" | "region")
193
+ if (preservation && preservation !== "global") {
194
+ const localZone = server?.location?.[preservation] || server?.[preservation];
195
+ if (localZone) {
196
+ // Targeted query for active servers in this node's geographic zone
197
+ const zoneClusterData = await getClusterDetails({
198
+ [`location.${preservation}`]: localZone
199
+ });
200
+
201
+ if (!zoneClusterData || zoneClusterData.totalCount <= 1) {
202
+ console.log(`[@cocreate/server-autoscaler] [CONSENSUS] Scale-in rejected. Removing this node would leave '${localZone}' with no active nodes (Preservation: ${preservation}).`);
203
+ return false;
204
+ }
205
+ }
246
206
  }
247
207
 
248
- const minSize = parseInt(getValueFromObject(autoscaleConfig, 'autoscale.minClusterSize'), 10) || defaults.minClusterSize;
208
+ const clusterData = await getClusterDetails();
249
209
 
250
- try {
251
- const readResponse = await serverCtx.crud.send({
252
- method: 'object.read',
253
- array: 'servers',
254
- $filter: {
255
- query: { status: 'active' },
256
- limit: 0
257
- },
258
- organization_id: serverCtx.organization_id,
259
- host: serverCtx.host
260
- });
210
+ if (!clusterData || clusterData.totalCount <= 1) {
211
+ console.log(`[@cocreate/server-autoscaler] [CONSENSUS] Scale-in rejected. Insufficient active nodes found.`);
212
+ return false;
213
+ }
261
214
 
262
- const activeServers = normalizePeersList(readResponse);
215
+ if (clusterData.totalCount <= minSize) {
216
+ console.log(`[@cocreate/server-autoscaler] [CONSENSUS] Scale-in rejected. Active nodes (${clusterData.totalCount}) is already at or below the minimum cluster limit (${minSize}).`);
217
+ return false;
218
+ }
263
219
 
264
- if (activeServers.length <= minSize) {
265
- console.log(`[@cocreate/server-autoscaler] [CONSENSUS] Scale-down rejected. Active nodes (${activeServers.length}) is already at or below the minimum cluster limit (${minSize}).`);
266
- return false;
267
- }
220
+ // Calculate projected load on remaining nodes if this node is removed
221
+ const remainingCount = clusterData.totalCount - 1;
222
+ const projectedCpu = clusterData.totalCpu / remainingCount;
223
+ const projectedMemory = clusterData.totalMem / remainingCount;
268
224
 
269
- console.log(`[@cocreate/server-autoscaler] [CONSENSUS] Scale-down authorized. Cluster contains ${activeServers.length} active nodes (Safe margin: > ${minSize}).`);
270
- return true;
271
- } catch (err) {
272
- console.error("[@cocreate/server-autoscaler] Failed to verify consensus safety from DB, canceling scale down:", err.message);
225
+ console.log(`[@cocreate/server-autoscaler] [SCALE IN CHECK] Projected remaining cluster load (${remainingCount} servers) -> CPU: ${projectedCpu.toFixed(1)}%, Memory: ${projectedMemory.toFixed(1)}%`);
226
+
227
+ if (projectedCpu >= scaleOutCpu || projectedMemory >= targetMemory) {
228
+ console.log(`[@cocreate/server-autoscaler] [CONSENSUS] Scale-in rejected. Removing this node would push remaining cluster load above scale-out thresholds.`);
273
229
  return false;
274
230
  }
231
+
232
+ console.log(`[@cocreate/server-autoscaler] [CONSENSUS] Average cluster threshold met for scale-in. Scale-in authorized. Node marked for scale in.`);
233
+ return true;
275
234
  }
276
235
 
277
236
  /**
278
- * Intercepts high-priority 1-second instantaneous memory alerts and executes fast-track scaling.
237
+ * Triggers external cloud infrastructure provisioning APIs to add or remove node instances.
238
+ * Incorporates geographic edge targeting metadata and maxNodesPerTier vertical tier checks.
279
239
  *
280
- * @param {Object} metricPayload - Real-time metric snapshot collected on the 1-second tick
240
+ * @param {string} action - 'out' or 'in'
241
+ * @param {Object} telemetry - Telemetry context frame
242
+ * @param {Object} meta - Optional metadata specs
281
243
  */
282
- async function processEmergencyAlert(metricPayload) {
283
- if (!isRunning) return;
244
+ async function executeScaleAction(action, telemetry, meta = {}) {
245
+ // If scaling out, calculate target region and determine spec tier using maxNodesPerTier
246
+ if (action === 'out') {
247
+ const clients = Array.from(
248
+ server?.wsManager?.clients?.values() ?? [],
249
+ client => client?.clientInfo
250
+ ).filter(Boolean);
251
+
252
+ const geoTarget = getCloudProvider(clients, autoscaleConfig);
253
+ if (geoTarget) {
254
+ meta.geo = geoTarget;
255
+ console.log(`[@cocreate/server-autoscaler] Geo-Scaling active. Requesting provision in region: ${geoTarget.targetRegion}`);
256
+ }
284
257
 
285
- const cooldownDuration = parseInt(getValueFromObject(autoscaleConfig, 'autoscale.cooldownDurationMs'), 10) || defaults.cooldownDurationMs;
258
+ // Check regional tier counts against maxNodesPerTier to decide whether to upgrade tier
259
+ const maxNodesPerTier = parseInt(getValueFromObject(autoscaleConfig, 'autoscale.vertical.maxNodesPerTier'), 10) || defaults.vertical.maxNodesPerTier;
260
+ const tiers = getValueFromObject(autoscaleConfig, 'autoscale.vertical.tiers') || defaults.vertical.tiers;
261
+
262
+ const targetRegion = geoTarget?.targetRegion || server?.infrastructure?.region || 'default-region';
263
+
264
+ // Targeted query for servers in the specific target infrastructure region
265
+ const regionalClusterData = await getClusterDetails({
266
+ 'infrastructure.region': targetRegion
267
+ });
286
268
 
287
- if (Date.now() - lastScaleOperationTimestamp < cooldownDuration) {
288
- return;
289
- }
269
+ if (regionalClusterData && Array.isArray(regionalClusterData.servers)) {
270
+ const regionalTierCounts = {};
271
+ for (const serverRecord of regionalClusterData.servers) {
272
+ const peerTelemetry = serverRecord.telemetry;
273
+ const vCPU = peerTelemetry?.cpu?.cores?.length || serverRecord.totalWorkers || 1;
274
+ const memoryTotalBytes = peerTelemetry?.memory?.total || 0;
275
+ const memoryGb = memoryTotalBytes > 0 ? Math.round(memoryTotalBytes / (1024 * 1024 * 1024)) : 2;
290
276
 
291
- const memoryPct = metricPayload.memory?.percentage || 0;
277
+ const tier = matchTierName(vCPU, memoryGb);
278
+ regionalTierCounts[tier] = (regionalTierCounts[tier] || 0) + 1;
279
+ }
292
280
 
293
- lastScaleOperationTimestamp = Date.now();
294
- consecutiveCpuSpikes = 0;
295
- consecutiveLowLoadTicks = 0;
281
+ let selectedTier = tiers[0]; // Start at smallest tier (e.g., nano)
296
282
 
297
- console.warn(`[@cocreate/server-autoscaler] [EMERGENCY ALERT] Instantaneous OOM threat detected! Memory: ${memoryPct}%. Initiating immediate fast-track scale-up...`);
298
-
299
- // Default action configuration for high-priority fast-track memory scale requests
300
- const emergencyActionConfig = {
301
- method: 'infrastructure.scale',
302
- provider: 'round-robin',
303
- spec: 'memory-optimized',
304
- region: 'local'
305
- };
306
- await executeScaleAction('up', metricPayload, emergencyActionConfig);
283
+ for (const tierSpec of tiers) {
284
+ const countAtTier = regionalTierCounts[tierSpec.name] || 0;
285
+ if (countAtTier < maxNodesPerTier) {
286
+ selectedTier = tierSpec;
287
+ break;
288
+ }
289
+ // If maxNodesPerTier reached, step up to next tier spec
290
+ selectedTier = tierSpec;
291
+ }
292
+
293
+ meta.tier = selectedTier;
294
+ console.log(`[@cocreate/server-autoscaler] Vertical Tiering selected spec '${selectedTier.name}' (${selectedTier.vCPU} vCPU, ${selectedTier.memoryGb} GB RAM) for region '${targetRegion}'.`);
295
+ }
296
+ }
297
+
298
+ console.log(`[@cocreate/server-autoscaler] Requesting infrastructure scale-${action.toUpperCase()}...`, meta);
299
+
300
+ // Emit event across server event bus for infrastructure orchestrator
301
+ if (server?.emit) {
302
+ server.emit('telemetry.scaleAction', {
303
+ action,
304
+ telemetry,
305
+ meta,
306
+ timestamp: Date.now()
307
+ });
308
+ }
307
309
  }
308
310
 
309
311
  /**
310
- * Evaluates the pre-averaged flush packet directly against the threshold limits.
311
- * Handles both classic properties and advanced declarative AST-configured orchestration rules.
312
+ * Processes telemetry frame payloads.
313
+ * - Memory evaluated INSTANTLY on every TICK to protect against sudden heap OOM crashes.
314
+ * - CPU evaluated across SUSTAINED FLUSHES to avoid over-provisioning on transient spikes.
315
+ * - Scale down evaluated across DEBOUNCED LOW FLUSHES.
312
316
  *
313
- * @param {Object} metricPayload - Normalized metrics data frame received on flush
317
+ * @param {Object} telemetry - Telemetry frame payload received on tick or flush dispatch
314
318
  */
315
- async function processMetricFlush(metricPayload) {
319
+ async function processTelemetryFrame(telemetry) {
316
320
  if (!isRunning) return;
317
321
 
318
- const scaleUpCpu = parseInt(getValueFromObject(autoscaleConfig, 'autoscale.scaleUpCpuThreshold'), 10) || defaults.scaleUpCpuThreshold;
319
- const scaleUpCpuExtreme = parseInt(getValueFromObject(autoscaleConfig, 'autoscale.scaleUpCpuExtremeThreshold'), 10) || defaults.scaleUpCpuExtremeThreshold;
320
- const scaleDownCpu = parseInt(getValueFromObject(autoscaleConfig, 'autoscale.scaleDownCpuThreshold'), 10) || defaults.scaleDownCpuThreshold;
321
- const cooldownDuration = parseInt(getValueFromObject(autoscaleConfig, 'autoscale.cooldownDurationMs'), 10) || defaults.cooldownDurationMs;
322
- const spikesRequired = parseInt(getValueFromObject(autoscaleConfig, 'autoscale.consecutiveCpuSpikesRequired'), 10) || defaults.consecutiveCpuSpikesRequired;
323
- const lowTicksRequired = parseInt(getValueFromObject(autoscaleConfig, 'autoscale.consecutiveLowTicksRequired'), 10) || defaults.consecutiveLowTicksRequired;
322
+ const scaleOutCpu = parseInt(getValueFromObject(autoscaleConfig, 'autoscale.scaleOut.cpuThreshold'), 10) || defaults.scaleOut.cpuThreshold;
323
+ const scaleInCpu = parseInt(getValueFromObject(autoscaleConfig, 'autoscale.scaleIn.cpuThreshold'), 10) || defaults.scaleIn.cpuThreshold;
324
+ const targetMemoryThreshold = parseInt(getValueFromObject(autoscaleConfig, 'autoscale.scaleOut.memoryThreshold'), 10) || defaults.scaleOut.memoryThreshold;
325
+ const scaleInMemory = parseInt(getValueFromObject(autoscaleConfig, 'autoscale.scaleIn.memoryThreshold'), 10) || defaults.scaleIn.memoryThreshold;
326
+ const spikesRequired = parseInt(getValueFromObject(autoscaleConfig, 'autoscale.scaleOut.consecutiveFlushes'), 10) || defaults.scaleOut.consecutiveFlushes;
327
+ const lowFlushesRequired = parseInt(getValueFromObject(autoscaleConfig, 'autoscale.scaleIn.consecutiveFlushes'), 10) || defaults.scaleIn.consecutiveFlushes;
324
328
 
325
- let targetMemoryThreshold = parseInt(getValueFromObject(autoscaleConfig, 'autoscale.scaleUpMemoryThreshold'), 10) || defaults.scaleUpMemoryThreshold;
329
+ const cpuAvg = telemetry.cpu?.totalUsage || 0;
330
+ const memoryAvg = telemetry.memory?.percentage || 0;
326
331
 
327
- const isSafetyLimitsEnabled = getValueFromObject(autoscaleConfig, 'autoscale.safetyLimits') !== false && getValueFromObject(autoscaleConfig, 'autoscale.safetyLimits') !== 'false';
328
-
329
- if (isSafetyLimitsEnabled) {
330
- const dynamicSafetyThreshold = calculateDynamicSafetyThreshold();
331
- if (targetMemoryThreshold > dynamicSafetyThreshold) {
332
- console.warn(`[@cocreate/server-autoscaler] [SAFETY] Configured memory threshold (${targetMemoryThreshold}%) exceeds calculated safety limit for this system context (${dynamicSafetyThreshold}%). Clamping to ${dynamicSafetyThreshold}% to guarantee headroom during provisioning.`);
333
- targetMemoryThreshold = dynamicSafetyThreshold;
334
- }
335
- }
332
+ // TRACK 1: INSTANT PER-TICK MEMORY EVALUATION
333
+ const isMemoryPeaked = memoryAvg >= targetMemoryThreshold;
336
334
 
337
- if (Date.now() - lastScaleOperationTimestamp < cooldownDuration) {
338
- return;
339
- }
335
+ if (isMemoryPeaked) {
336
+ console.warn(`[@cocreate/server-autoscaler] [INSTANT TICK] Memory threshold exceeded (${memoryAvg}% >= limit: ${targetMemoryThreshold}%). Triggering immediate worker cordon & scale evaluation...`);
337
+
338
+ cordonWorkerNode(telemetry);
340
339
 
341
- const cpuAvg = metricPayload.cpu?.avg || 0;
342
- const memoryAvg = metricPayload.memory?.avg_used_pct || 0;
340
+ const authorized = await shouldScaleOutCluster(telemetry);
341
+ if (authorized) {
342
+ consecutiveCpuSpikes = 0;
343
+ await executeScaleAction('out', telemetry, { spec: 'memory-optimized', trigger: 'memory-instant-tick' });
344
+ }
345
+ return;
346
+ }
343
347
 
344
- // Maintain and increment state accumulators
345
- if (cpuAvg >= scaleUpCpu) {
348
+ // TRACK 2: SUSTAINED FLUSH-BASED CPU EVALUATION
349
+ if (cpuAvg >= scaleOutCpu) {
346
350
  consecutiveCpuSpikes++;
347
- consecutiveLowLoadTicks = 0;
351
+ consecutiveLowLoadFlushes = 0;
348
352
  } else {
349
353
  consecutiveCpuSpikes = 0;
350
354
  }
351
355
 
352
- const isUnderloaded = cpuAvg <= scaleDownCpu && memoryAvg < (targetMemoryThreshold * 0.6);
356
+ const isUnderloaded = cpuAvg <= scaleInCpu && memoryAvg <= scaleInMemory;
353
357
 
354
358
  if (isUnderloaded) {
355
- consecutiveLowLoadTicks++;
359
+ consecutiveLowLoadFlushes++;
356
360
  } else {
357
- consecutiveLowLoadTicks = 0;
361
+ consecutiveLowLoadFlushes = 0;
358
362
  }
359
363
 
360
- // Build the consolidated evaluation context for the AST interpreter
361
- const evaluationContext = {
362
- ...metricPayload,
363
- consecutiveCpuSpikes,
364
- consecutiveLowLoadTicks
365
- };
364
+ const isCpuSustainedPeak = consecutiveCpuSpikes >= spikesRequired;
366
365
 
367
- // Retrieve potential AST orchestration rules from configuration
368
- const customOrchestrationRule = getValueFromObject(autoscaleConfig, 'autoscale.orchestrationRule');
366
+ if (isCpuSustainedPeak) {
367
+ console.warn(`[@cocreate/server-autoscaler] [SUSTAINED FLUSH] Sustained CPU threshold exceeded for ${consecutiveCpuSpikes} consecutive flushes (${cpuAvg}% >= limit: ${scaleOutCpu}%). Triggering worker cordon & scale evaluation...`);
369
368
 
370
- if (customOrchestrationRule) {
371
- // --- DECLARATIVE AST EXECUTION TRACK ---
372
- console.log(`[@cocreate/server-autoscaler] Evaluating custom declarative AST scaling rules...`);
373
-
374
- const isTriggered = evaluateAST(customOrchestrationRule, evaluationContext);
375
-
376
- if (isTriggered) {
377
- const actionNode = customOrchestrationRule.action || {};
378
- lastScaleOperationTimestamp = Date.now();
379
- consecutiveCpuSpikes = 0;
380
- consecutiveLowLoadTicks = 0;
381
-
382
- console.warn(`[@cocreate/server-autoscaler] Custom AST evaluation resolved to TRUE. Triggering custom scaled action...`);
383
- await executeScaleAction('up', metricPayload, actionNode);
384
- return;
385
- }
386
- } else {
387
- // --- TRADITIONAL DUAL-TRACK HARDCODED FALLBACK ---
388
- const isMemoryPeaked = memoryAvg >= targetMemoryThreshold;
389
- const isCpuExtremePeak = cpuAvg >= scaleUpCpuExtreme;
390
- const isCpuSustainedPeak = consecutiveCpuSpikes >= spikesRequired;
391
-
392
- if (isCpuExtremePeak || isCpuSustainedPeak || isMemoryPeaked) {
393
- let triggerReason = '';
394
- let targetSpec = 'standard';
395
-
396
- if (isMemoryPeaked) {
397
- triggerReason = `Incompressible memory threshold exceeded (${memoryAvg}% >= limit: ${targetMemoryThreshold}%)`;
398
- targetSpec = 'memory-optimized';
399
- } else if (isCpuExtremePeak) {
400
- triggerReason = `Extreme CPU ceiling exceeded on a single flush (${cpuAvg}% >= limit: ${scaleUpCpuExtreme}%)`;
401
- targetSpec = 'compute-optimized';
402
- } else {
403
- triggerReason = `Sustained CPU threshold exceeded for ${consecutiveCpuSpikes} consecutive cycles (${cpuAvg}% >= limit: ${scaleUpCpu}%)`;
404
- targetSpec = 'compute-optimized';
405
- }
369
+ cordonWorkerNode(telemetry);
406
370
 
407
- lastScaleOperationTimestamp = Date.now();
371
+ const authorized = await shouldScaleOutCluster(telemetry);
372
+ if (authorized) {
408
373
  consecutiveCpuSpikes = 0;
409
-
410
- console.warn(`[@cocreate/server-autoscaler] Peak limit hit! ${triggerReason}. Scaling up...`);
411
- await executeScaleAction('up', metricPayload, { spec: targetSpec });
412
- return;
374
+ await executeScaleAction('out', telemetry, { spec: 'compute-optimized', trigger: 'cpu-sustained-flushes' });
413
375
  }
376
+ return;
414
377
  }
415
378
 
416
- // Scale Down evaluation track: Consolidated across both AST and traditional modes
417
- if (consecutiveLowLoadTicks >= lowTicksRequired) {
418
- console.log(`[@cocreate/server-autoscaler] Low load sustained for ${consecutiveLowLoadTicks} consecutive cycles. Checking cluster state to verify consensus...`);
379
+ // TRACK 3: SCALE IN EVALUATION (SUSTAINED LOW LOAD ACROSS FLUSHES)
380
+ if (consecutiveLowLoadFlushes >= lowFlushesRequired) {
381
+ console.log(`[@cocreate/server-autoscaler] Low load sustained for ${consecutiveLowLoadFlushes} consecutive flushes. Checking cluster state to verify consensus...`);
419
382
 
420
- const safeToScaleDown = await isClusterSafeToScaleDown();
421
- if (safeToScaleDown) {
422
- lastScaleOperationTimestamp = Date.now();
423
- consecutiveLowLoadTicks = 0;
383
+ const safeToScaleIn = await shouldScaleInCluster(telemetry);
384
+ if (safeToScaleIn) {
385
+ consecutiveLowLoadFlushes = 0;
424
386
 
425
- console.log(`[@cocreate/server-autoscaler] Safe-down consensus verified. Scaling down...`);
426
- await executeScaleAction('down', metricPayload);
387
+ console.log(`[@cocreate/server-autoscaler] Safe-in consensus verified. Scaling in...`);
388
+ await executeScaleAction('in', telemetry);
427
389
  } else {
428
- consecutiveLowLoadTicks = Math.round(lowTicksRequired / 2);
390
+ consecutiveLowLoadFlushes = Math.round(lowFlushesRequired / 2);
429
391
  }
430
392
  }
431
393
  }
@@ -433,71 +395,50 @@ async function processMetricFlush(metricPayload) {
433
395
  /**
434
396
  * Boots the autoscaler system exclusively on the Master/Primary process context.
435
397
  *
436
- * @param {Object} server - Central CoCreateServer instance references
398
+ * @param {Object} serverInstance - Central CoCreateServer instance references
437
399
  * @returns {Object} Active autoscale control API
438
400
  */
439
- export async function init(server) {
440
- if (!isPrimary) {
441
- throw new Error("[@cocreate/server-autoscaler] [FATAL] Violation of scope: ServerAutoscaler can only run on the Primary (Master) process.");
401
+ export async function init(serverInstance) {
402
+ server = serverInstance;
403
+
404
+ if (!isMaster) {
405
+ console.log('[@cocreate/server-autoscaler] Autoscaler initialized on worker node (Standby).');
406
+ return;
442
407
  }
443
408
 
444
- serverCtx = server;
445
-
446
- autoscaleConfig = await Config({
447
- 'autoscale': ''
448
- });
409
+ try {
410
+ autoscaleConfig = await Config.get('autoscale');
411
+ } catch (e) {
412
+ console.warn('[@cocreate/server-autoscaler] Failed to load config, falling back to defaults.', e.message);
413
+ autoscaleConfig = {};
414
+ }
449
415
 
450
416
  const isEnabled = getValueFromObject(autoscaleConfig, 'autoscale.enabled') !== false && getValueFromObject(autoscaleConfig, 'autoscale.enabled') !== 'false';
451
417
 
452
418
  if (!isEnabled) {
453
- console.log('[@cocreate/server-autoscaler] ServerAutoscaler is disabled by configuration.');
454
- return { init, stop };
419
+ console.log('[@cocreate/server-autoscaler] Autoscaling is disabled by configuration.');
420
+ return;
455
421
  }
456
422
 
457
423
  isRunning = true;
458
-
459
- consecutiveCpuSpikes = 0;
460
- consecutiveLowLoadTicks = 0;
461
424
 
462
- serverCtx.on('metric', (payload) => {
463
- if (payload && payload.data) {
464
- processMetricFlush(payload.data).catch(err => {
465
- console.error("[@cocreate/server-autoscaler] Error processing autoscale execution tick:", err);
466
- });
467
- }
425
+ // Subscribe to incoming telemetry frame dispatches published by server-telemetry.js
426
+ server.on('telemetry', (telemetry) => {
427
+ processTelemetryFrame(telemetry);
468
428
  });
469
429
 
470
- serverCtx.on('metric.emergency', (payload) => {
471
- if (payload && payload.data) {
472
- processEmergencyAlert(payload.data).catch(err => {
473
- console.error("[@cocreate/server-autoscaler] Error processing emergency autoscale trigger:", err);
474
- });
475
- }
476
- });
477
-
478
- console.log('[@cocreate/server-autoscaler] Autoscaler initialized and subscribed to telemetry feed.');
479
-
480
- return {
481
- init,
482
- stop,
483
- processMetricFlush,
484
- processEmergencyAlert,
485
- executeScaleAction,
486
- calculateDynamicSafetyThreshold
487
- };
430
+ console.log('[@cocreate/server-autoscaler] Initialized and listening for telemetry on Master node.');
488
431
  }
489
432
 
490
433
  /**
491
- * Terminates monitoring sweeps and unsubscribes from the process emitter.
434
+ * Halts any active autoscaler processing
492
435
  */
493
436
  export function stop() {
494
437
  isRunning = false;
495
- consecutiveCpuSpikes = 0;
496
- consecutiveLowLoadTicks = 0;
497
- console.log('[@cocreate/server-autoscaler] ServerAutoscaler monitoring stopped.');
438
+ console.log('[@cocreate/server-autoscaler] Execution loop halted.');
498
439
  }
499
440
 
500
- export default {
501
- init,
502
- stop
441
+ export default {
442
+ init,
443
+ stop
503
444
  };