@xenon-device-management/xenon 1.2.0 → 1.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +74 -0
- package/lib/package.json +1 -1
- package/lib/public/assets/{Layouts-D0WSzKOh.js → Layouts-D6IPfwoe.js} +1 -1
- package/lib/public/assets/{ai-settings-DQWDdNd7.js → ai-settings-CflyFKan.js} +1 -1
- package/lib/public/assets/{apps-1sLWHOGO.js → apps-Da4dvQ1J.js} +1 -1
- package/lib/public/assets/{badge-BiR1gmMm.js → badge-BNR9umdu.js} +1 -1
- package/lib/public/assets/{button-BVazt4Z1.js → button-hZFV1ypT.js} +1 -1
- package/lib/public/assets/{calendar-yMyP2_Nc.js → calendar-fehdBtun.js} +1 -1
- package/lib/public/assets/{clock-CsVplnJ2.js → clock-DrpxSvCL.js} +1 -1
- package/lib/public/assets/{cpu-DNC8n7kK.js → cpu-tuyMVZ4I.js} +1 -1
- package/lib/public/assets/{device-explorer-DFu8Gxj4.js → device-explorer-DOfRH3zm.js} +1 -1
- package/lib/public/assets/{index-S71J2rWg.js → index-BaTiUCeH.js} +18 -18
- package/lib/public/assets/{lock-BstCxnX6.js → lock-C6CoqSr2.js} +1 -1
- package/lib/public/assets/{maintenance-settings-BwfG9cu2.js → maintenance-settings-CM2oC7-i.js} +1 -1
- package/lib/public/assets/{mouse-pointer-2-CSn_Wnc9.js → mouse-pointer-2-CXdnjXIg.js} +1 -1
- package/lib/public/assets/{plus-DfjM7G6e.js → plus-B4B1Hukt.js} +1 -1
- package/lib/public/assets/{session-dashboard-C6ek4z65.js → session-dashboard-B5OPMTz5.js} +1 -1
- package/lib/public/assets/{settings-BDYP8ULf.js → settings-BTHP7fj3.js} +1 -1
- package/lib/public/assets/{trash-2-CZWUMK5b.js → trash-2-NJMZJ2Ol.js} +1 -1
- package/lib/public/assets/{useSocket-CliVeWS3.js → useSocket-Ct2wo7P2.js} +2 -2
- package/lib/public/assets/{webhook-settings-tPiwWf8y.js → webhook-settings-Cz35-QJ7.js} +1 -1
- package/lib/public/assets/{zap-ZrK5B58i.js → zap-CssSMAN5.js} +1 -1
- package/lib/public/index.html +1 -1
- package/lib/schema.json +85 -38
- package/lib/src/InternalHttpClient.js +69 -14
- package/lib/src/app/index.js +92 -24
- package/lib/src/app/routers/apikeys.js +33 -0
- package/lib/src/app/routers/apps.js +4 -0
- package/lib/src/app/routers/auth.js +36 -0
- package/lib/src/app/routers/config.js +4 -0
- package/lib/src/app/routers/control.js +61 -10
- package/lib/src/app/routers/dashboard.js +5 -6
- package/lib/src/app/routers/grid.js +30 -12
- package/lib/src/app/routers/processes.js +24 -0
- package/lib/src/app/routers/reservation.js +15 -0
- package/lib/src/app/routers/webhook.js +6 -3
- package/lib/src/auth/nodeSecret.js +33 -0
- package/lib/src/config.js +5 -0
- package/lib/src/data-service/prisma-store.js +17 -1
- package/lib/src/device-managers/AndroidDeviceManager.js +2 -2
- package/lib/src/device-managers/NodeDevices.js +8 -1
- package/lib/src/device-managers/ios/IOSDiscoveryService.js +7 -4
- package/lib/src/device-managers/ios/IOSStreamService.js +7 -0
- package/lib/src/device-managers/ios/WDAClient.js +2 -0
- package/lib/src/device-utils.js +29 -4
- package/lib/src/generated/client/edge.js +2 -2
- package/lib/src/generated/client/index.js +2 -2
- package/lib/src/generated/client/package.json +1 -1
- package/lib/src/generated/client/schema.prisma +3 -0
- package/lib/src/helpers/UniversalMjpegProxy.js +23 -0
- package/lib/src/index.js +10 -2
- package/lib/src/interceptors/CommandInterceptor.js +29 -0
- package/lib/src/interfaces/IPluginArgs.js +0 -1
- package/lib/src/logger.js +30 -2
- package/lib/src/logging/sessionContext.js +28 -0
- package/lib/src/middleware/apiKeyMiddleware.js +49 -0
- package/lib/src/middleware/csrfMiddleware.js +73 -0
- package/lib/src/middleware/nodeSecretMiddleware.js +38 -0
- package/lib/src/middleware/rateLimitMiddleware.js +68 -0
- package/lib/src/middleware/scopeGuard.js +41 -0
- package/lib/src/plugin.js +1 -1
- package/lib/src/services/AIService.js +43 -8
- package/lib/src/services/ApiKeyService.js +102 -0
- package/lib/src/services/CircuitBreaker.js +158 -0
- package/lib/src/services/CleanupService.js +137 -39
- package/lib/src/services/DeviceReconciler.js +102 -0
- package/lib/src/services/MetricsService.js +78 -0
- package/lib/src/services/PortAllocator.js +13 -0
- package/lib/src/services/ProcessMetricsService.js +99 -0
- package/lib/src/services/ProcessRegistry.js +123 -0
- package/lib/src/services/ServerManager.js +14 -2
- package/lib/src/services/SessionLifecycleService.js +80 -23
- package/lib/src/services/ShutdownCoordinator.js +89 -0
- package/lib/src/services/SocketClient.js +11 -0
- package/lib/src/services/SocketServer.js +109 -6
- package/lib/src/services/VideoPipelineService.js +2 -0
- package/lib/src/services/healing/HealingMetrics.js +63 -0
- package/lib/src/services/healing/HealingOrchestrator.js +32 -4
- package/lib/src/services/healing/OcrHealingProvider.js +7 -0
- package/lib/test/unit/ApiKeyService.test.js +101 -0
- package/lib/test/unit/PortAllocator.test.js +14 -0
- package/lib/test/unit/ProcessRegistry.test.js +70 -0
- package/lib/test/unit/apiKeyMiddleware.test.js +58 -0
- package/lib/test/unit/nodeSecretMiddleware.test.js +38 -0
- package/lib/test/unit/rateLimitMiddleware.test.js +37 -0
- package/lib/tsconfig.tsbuildinfo +1 -1
- package/package.json +2 -2
- package/prisma/migrations/20260423081701_add_session_indexes/migration.sql +8 -0
- package/prisma/schema.prisma +3 -0
- package/schema.json +85 -38
|
@@ -54,6 +54,7 @@ const logger_1 = __importDefault(require("../logger"));
|
|
|
54
54
|
const fs = __importStar(require("fs"));
|
|
55
55
|
const path = __importStar(require("path"));
|
|
56
56
|
const config_1 = require("../config");
|
|
57
|
+
const CircuitBreaker_1 = require("./CircuitBreaker");
|
|
57
58
|
class GeminiProvider {
|
|
58
59
|
constructor(apiKey, modelName) {
|
|
59
60
|
this.genAI = new generative_ai_1.GoogleGenerativeAI(apiKey.trim());
|
|
@@ -222,6 +223,29 @@ class AIService {
|
|
|
222
223
|
this.isMock = false;
|
|
223
224
|
this.initializeProvider();
|
|
224
225
|
}
|
|
226
|
+
// Key the breaker by provider + resolved model name so a misbehaving model
|
|
227
|
+
// (rate-limited preview tier, deprecated snapshot) doesn't trip sibling
|
|
228
|
+
// models on the same provider. Falls back to 'default' if model unresolved.
|
|
229
|
+
breakerKey() {
|
|
230
|
+
const provider = config_1.config.aiProvider || 'unknown';
|
|
231
|
+
const model = provider === 'gemini'
|
|
232
|
+
? config_1.config.geminiModel
|
|
233
|
+
: provider === 'openai'
|
|
234
|
+
? config_1.config.openaiModel
|
|
235
|
+
: provider === 'anthropic'
|
|
236
|
+
? config_1.config.anthropicModel
|
|
237
|
+
: provider === 'ollama'
|
|
238
|
+
? config_1.config.ollamaModel
|
|
239
|
+
: config_1.config.aiModel;
|
|
240
|
+
return `ai:${provider}:${model || 'default'}`;
|
|
241
|
+
}
|
|
242
|
+
// Single choke point for every LLM call so circuit-breaker state is shared
|
|
243
|
+
// across analyzeFailure / visualFind / healLocator.
|
|
244
|
+
callProvider(prompt, screenshotBase64) {
|
|
245
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
246
|
+
return CircuitBreaker_1.CIRCUIT_BREAKERS.execute(this.breakerKey(), () => this.provider.analyze(prompt, screenshotBase64));
|
|
247
|
+
});
|
|
248
|
+
}
|
|
225
249
|
initializeProvider() {
|
|
226
250
|
const providerType = config_1.config.aiProvider;
|
|
227
251
|
const genericModel = config_1.config.aiModel;
|
|
@@ -281,12 +305,17 @@ Fix: Add a pre-emptive check for the location permission dialog or use the \`aut
|
|
|
281
305
|
try {
|
|
282
306
|
const prompt = this.constructPrompt(context);
|
|
283
307
|
const screenshotBase64 = this.getScreenshotBase64(context.screenshotPath);
|
|
284
|
-
const text = yield this.
|
|
308
|
+
const text = yield this.callProvider(prompt, screenshotBase64 || undefined);
|
|
285
309
|
logger_1.default.info(`[AIService] Analysis complete for ${context.sessionId}`);
|
|
286
310
|
return text;
|
|
287
311
|
}
|
|
288
312
|
catch (err) {
|
|
289
|
-
|
|
313
|
+
if (err instanceof CircuitBreaker_1.CircuitOpenError) {
|
|
314
|
+
logger_1.default.debug(`[AIService] Analysis skipped: ${err.message}`);
|
|
315
|
+
}
|
|
316
|
+
else {
|
|
317
|
+
logger_1.default.error(`[AIService] Analysis failed: ${err.message}`);
|
|
318
|
+
}
|
|
290
319
|
return null;
|
|
291
320
|
}
|
|
292
321
|
});
|
|
@@ -307,13 +336,16 @@ Fix: Add a pre-emptive check for the location permission dialog or use the \`aut
|
|
|
307
336
|
Instructions: Look at the provided screenshot and find the exact center of the specified element.
|
|
308
337
|
`;
|
|
309
338
|
try {
|
|
310
|
-
const response = yield this.
|
|
339
|
+
const response = yield this.callProvider(prompt, screenshotBase64);
|
|
311
340
|
const data = JSON.parse(response.replace(/```json|```/g, '').trim());
|
|
312
341
|
return data;
|
|
313
342
|
}
|
|
314
343
|
catch (err) {
|
|
315
|
-
// Log service unavailability at debug level (expected)
|
|
316
|
-
|
|
344
|
+
// Log service unavailability / tripped breaker at debug level (expected);
|
|
345
|
+
// genuine errors at warn.
|
|
346
|
+
if (err instanceof CircuitBreaker_1.CircuitOpenError ||
|
|
347
|
+
((_a = err.message) === null || _a === void 0 ? void 0 : _a.includes('unavailable')) ||
|
|
348
|
+
((_b = err.response) === null || _b === void 0 ? void 0 : _b.status) === 404) {
|
|
317
349
|
logger_1.default.debug(`[AIService] visualFind skipped: ${err.message}`);
|
|
318
350
|
}
|
|
319
351
|
else {
|
|
@@ -350,13 +382,16 @@ Fix: Add a pre-emptive check for the location permission dialog or use the \`aut
|
|
|
350
382
|
Response Format: JSON only, strictly { "recommendedXpath": "string", "reason": "string" }.
|
|
351
383
|
`;
|
|
352
384
|
try {
|
|
353
|
-
const response = yield this.
|
|
385
|
+
const response = yield this.callProvider(prompt, context.screenshotBase64);
|
|
354
386
|
const data = JSON.parse(response.replace(/```json|```/g, '').trim());
|
|
355
387
|
return data;
|
|
356
388
|
}
|
|
357
389
|
catch (err) {
|
|
358
|
-
// Log service unavailability at debug level (expected)
|
|
359
|
-
|
|
390
|
+
// Log service unavailability / tripped breaker at debug level (expected);
|
|
391
|
+
// genuine errors at warn.
|
|
392
|
+
if (err instanceof CircuitBreaker_1.CircuitOpenError ||
|
|
393
|
+
((_a = err.message) === null || _a === void 0 ? void 0 : _a.includes('unavailable')) ||
|
|
394
|
+
((_b = err.response) === null || _b === void 0 ? void 0 : _b.status) === 404) {
|
|
360
395
|
logger_1.default.debug(`[AIService] healLocator skipped: ${err.message}`);
|
|
361
396
|
}
|
|
362
397
|
else {
|
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
|
|
3
|
+
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
|
|
4
|
+
if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
|
|
5
|
+
else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
|
|
6
|
+
return c > 3 && r && Object.defineProperty(target, key, r), r;
|
|
7
|
+
};
|
|
8
|
+
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
9
|
+
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
10
|
+
return new (P || (P = Promise))(function (resolve, reject) {
|
|
11
|
+
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
12
|
+
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
13
|
+
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
14
|
+
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
15
|
+
});
|
|
16
|
+
};
|
|
17
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
18
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
19
|
+
};
|
|
20
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
21
|
+
exports.ApiKeyService = void 0;
|
|
22
|
+
const typedi_1 = require("typedi");
|
|
23
|
+
const crypto_1 = __importDefault(require("crypto"));
|
|
24
|
+
const fs_1 = __importDefault(require("fs"));
|
|
25
|
+
const path_1 = __importDefault(require("path"));
|
|
26
|
+
const prisma_1 = require("../prisma");
|
|
27
|
+
const logger_1 = __importDefault(require("../logger"));
|
|
28
|
+
let ApiKeyService = class ApiKeyService {
|
|
29
|
+
constructor() {
|
|
30
|
+
this.log = logger_1.default.scope('ApiKey');
|
|
31
|
+
}
|
|
32
|
+
hash(raw) {
|
|
33
|
+
return crypto_1.default.createHash('sha256').update(raw).digest('hex');
|
|
34
|
+
}
|
|
35
|
+
generateRaw() {
|
|
36
|
+
return crypto_1.default.randomBytes(32).toString('hex');
|
|
37
|
+
}
|
|
38
|
+
bootstrapIfEmpty(keyFilePath) {
|
|
39
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
40
|
+
const count = yield prisma_1.prisma.apiKey.count();
|
|
41
|
+
if (count > 0)
|
|
42
|
+
return null;
|
|
43
|
+
const raw = this.generateRaw();
|
|
44
|
+
const keyHash = this.hash(raw);
|
|
45
|
+
fs_1.default.mkdirSync(path_1.default.dirname(keyFilePath), { recursive: true });
|
|
46
|
+
fs_1.default.writeFileSync(keyFilePath, raw + '\n', { mode: 0o600 });
|
|
47
|
+
yield prisma_1.prisma.apiKey.create({
|
|
48
|
+
data: {
|
|
49
|
+
name: 'bootstrap',
|
|
50
|
+
keyHash,
|
|
51
|
+
scopes: 'admin',
|
|
52
|
+
rateLimit: 300,
|
|
53
|
+
},
|
|
54
|
+
});
|
|
55
|
+
this.log.warn(`No API keys found. Bootstrap key written to ${keyFilePath}. Rotate within 24h via POST /xenon/api/apikeys.`);
|
|
56
|
+
return raw;
|
|
57
|
+
});
|
|
58
|
+
}
|
|
59
|
+
verify(raw) {
|
|
60
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
61
|
+
if (!raw)
|
|
62
|
+
return null;
|
|
63
|
+
const row = yield prisma_1.prisma.apiKey.findUnique({ where: { keyHash: this.hash(raw) } });
|
|
64
|
+
if (!row || row.revokedAt)
|
|
65
|
+
return null;
|
|
66
|
+
prisma_1.prisma.apiKey
|
|
67
|
+
.update({ where: { id: row.id }, data: { lastUsedAt: new Date() } })
|
|
68
|
+
.catch(() => undefined);
|
|
69
|
+
return row;
|
|
70
|
+
});
|
|
71
|
+
}
|
|
72
|
+
hasScope(row, required) {
|
|
73
|
+
const owned = new Set(row.scopes.split(',').map((s) => s.trim()));
|
|
74
|
+
if (owned.has('admin'))
|
|
75
|
+
return true;
|
|
76
|
+
return required.some((r) => owned.has(r));
|
|
77
|
+
}
|
|
78
|
+
create(params) {
|
|
79
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
80
|
+
var _a;
|
|
81
|
+
const raw = this.generateRaw();
|
|
82
|
+
const row = yield prisma_1.prisma.apiKey.create({
|
|
83
|
+
data: {
|
|
84
|
+
name: params.name,
|
|
85
|
+
keyHash: this.hash(raw),
|
|
86
|
+
scopes: params.scopes.join(','),
|
|
87
|
+
rateLimit: (_a = params.rateLimit) !== null && _a !== void 0 ? _a : 300,
|
|
88
|
+
},
|
|
89
|
+
});
|
|
90
|
+
return { id: row.id, raw };
|
|
91
|
+
});
|
|
92
|
+
}
|
|
93
|
+
revoke(id) {
|
|
94
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
95
|
+
yield prisma_1.prisma.apiKey.update({ where: { id }, data: { revokedAt: new Date() } });
|
|
96
|
+
});
|
|
97
|
+
}
|
|
98
|
+
};
|
|
99
|
+
exports.ApiKeyService = ApiKeyService;
|
|
100
|
+
exports.ApiKeyService = ApiKeyService = __decorate([
|
|
101
|
+
(0, typedi_1.Service)()
|
|
102
|
+
], ApiKeyService);
|
|
@@ -0,0 +1,158 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
3
|
+
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
4
|
+
return new (P || (P = Promise))(function (resolve, reject) {
|
|
5
|
+
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
6
|
+
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
7
|
+
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
8
|
+
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
9
|
+
});
|
|
10
|
+
};
|
|
11
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
12
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
13
|
+
};
|
|
14
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
15
|
+
exports.CIRCUIT_BREAKERS = exports.CircuitOpenError = void 0;
|
|
16
|
+
exports.isTransientFailure = isTransientFailure;
|
|
17
|
+
const logger_1 = __importDefault(require("../logger"));
|
|
18
|
+
class CircuitOpenError extends Error {
|
|
19
|
+
constructor(key, retryAfterMs) {
|
|
20
|
+
super(`Circuit open for '${key}'; retry in ${retryAfterMs}ms`);
|
|
21
|
+
this.name = 'CircuitOpenError';
|
|
22
|
+
this.key = key;
|
|
23
|
+
this.retryAfterMs = retryAfterMs;
|
|
24
|
+
}
|
|
25
|
+
}
|
|
26
|
+
exports.CircuitOpenError = CircuitOpenError;
|
|
27
|
+
class Breaker {
|
|
28
|
+
constructor(key, opts) {
|
|
29
|
+
this.key = key;
|
|
30
|
+
this.opts = opts;
|
|
31
|
+
this.state = 'closed';
|
|
32
|
+
this.consecutiveFailures = 0;
|
|
33
|
+
this.openedAt = 0;
|
|
34
|
+
}
|
|
35
|
+
// Returns whether the next request should go through. Transitions open ->
|
|
36
|
+
// half_open here if the cooldown has elapsed — caller gets exactly one
|
|
37
|
+
// probe attempt which then drives the closed/open decision.
|
|
38
|
+
admit() {
|
|
39
|
+
if (this.state === 'closed' || this.state === 'half_open') {
|
|
40
|
+
return { allowed: true };
|
|
41
|
+
}
|
|
42
|
+
const elapsed = Date.now() - this.openedAt;
|
|
43
|
+
if (elapsed >= this.opts.openDurationMs) {
|
|
44
|
+
this.state = 'half_open';
|
|
45
|
+
logger_1.default.info(`[CircuitBreaker] ${this.key} -> half_open (cooldown elapsed)`);
|
|
46
|
+
return { allowed: true };
|
|
47
|
+
}
|
|
48
|
+
return { allowed: false, retryAfterMs: this.opts.openDurationMs - elapsed };
|
|
49
|
+
}
|
|
50
|
+
recordSuccess() {
|
|
51
|
+
const wasRecovering = this.state !== 'closed';
|
|
52
|
+
this.state = 'closed';
|
|
53
|
+
this.consecutiveFailures = 0;
|
|
54
|
+
if (wasRecovering) {
|
|
55
|
+
logger_1.default.info(`[CircuitBreaker] ${this.key} -> closed (probe succeeded)`);
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
recordFailure() {
|
|
59
|
+
if (this.state === 'half_open') {
|
|
60
|
+
// Probe failed -> re-open immediately with a fresh cooldown window.
|
|
61
|
+
this.state = 'open';
|
|
62
|
+
this.openedAt = Date.now();
|
|
63
|
+
logger_1.default.warn(`[CircuitBreaker] ${this.key} -> open (probe failed)`);
|
|
64
|
+
return;
|
|
65
|
+
}
|
|
66
|
+
this.consecutiveFailures++;
|
|
67
|
+
if (this.state === 'closed' && this.consecutiveFailures >= this.opts.failureThreshold) {
|
|
68
|
+
this.state = 'open';
|
|
69
|
+
this.openedAt = Date.now();
|
|
70
|
+
logger_1.default.warn(`[CircuitBreaker] ${this.key} -> open after ${this.consecutiveFailures} consecutive failures (cooldown ${this.opts.openDurationMs}ms)`);
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
snapshot() {
|
|
74
|
+
return {
|
|
75
|
+
key: this.key,
|
|
76
|
+
state: this.state,
|
|
77
|
+
consecutiveFailures: this.consecutiveFailures,
|
|
78
|
+
openedAt: this.openedAt,
|
|
79
|
+
};
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
// Classifier: only transient / server-side failures should trip the breaker.
|
|
83
|
+
// Caller-induced failures (400s, parse errors, bad API key) stay uncounted
|
|
84
|
+
// so a misconfigured request doesn't take the breaker out for everyone.
|
|
85
|
+
function isTransientFailure(err) {
|
|
86
|
+
var _a, _b;
|
|
87
|
+
const status = (_b = (_a = err === null || err === void 0 ? void 0 : err.response) === null || _a === void 0 ? void 0 : _a.status) !== null && _b !== void 0 ? _b : err === null || err === void 0 ? void 0 : err.status;
|
|
88
|
+
if (typeof status === 'number') {
|
|
89
|
+
if (status >= 500)
|
|
90
|
+
return true;
|
|
91
|
+
if (status === 429)
|
|
92
|
+
return true; // rate limit — back off
|
|
93
|
+
return false; // 4xx (auth, bad model, bad request) won't heal itself via retry
|
|
94
|
+
}
|
|
95
|
+
const code = String((err === null || err === void 0 ? void 0 : err.code) || '').toUpperCase();
|
|
96
|
+
const transientCodes = [
|
|
97
|
+
'ECONNRESET',
|
|
98
|
+
'ECONNREFUSED',
|
|
99
|
+
'ETIMEDOUT',
|
|
100
|
+
'ENOTFOUND',
|
|
101
|
+
'EAI_AGAIN',
|
|
102
|
+
'EPIPE',
|
|
103
|
+
'ECONNABORTED',
|
|
104
|
+
'ERR_NETWORK',
|
|
105
|
+
];
|
|
106
|
+
if (transientCodes.includes(code))
|
|
107
|
+
return true;
|
|
108
|
+
const msg = String((err === null || err === void 0 ? void 0 : err.message) || '').toLowerCase();
|
|
109
|
+
if (msg.includes('timeout') || msg.includes('timed out'))
|
|
110
|
+
return true;
|
|
111
|
+
if (msg.includes('service unavailable') || msg.includes('unavailable'))
|
|
112
|
+
return true;
|
|
113
|
+
// SDK errors often embed HTTP status as text (Gemini/Anthropic wrap raw responses).
|
|
114
|
+
if (/\b(5\d{2}|429)\b/.test(msg))
|
|
115
|
+
return true;
|
|
116
|
+
return false;
|
|
117
|
+
}
|
|
118
|
+
class CircuitBreakerRegistry {
|
|
119
|
+
constructor() {
|
|
120
|
+
this.breakers = new Map();
|
|
121
|
+
this.defaults = {
|
|
122
|
+
failureThreshold: 5,
|
|
123
|
+
openDurationMs: 60000,
|
|
124
|
+
};
|
|
125
|
+
}
|
|
126
|
+
execute(key, task, opts) {
|
|
127
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
128
|
+
const breaker = this.getOrCreate(key, opts);
|
|
129
|
+
const gate = breaker.admit();
|
|
130
|
+
if (!gate.allowed) {
|
|
131
|
+
throw new CircuitOpenError(key, gate.retryAfterMs);
|
|
132
|
+
}
|
|
133
|
+
try {
|
|
134
|
+
const result = yield task();
|
|
135
|
+
breaker.recordSuccess();
|
|
136
|
+
return result;
|
|
137
|
+
}
|
|
138
|
+
catch (err) {
|
|
139
|
+
if (isTransientFailure(err)) {
|
|
140
|
+
breaker.recordFailure();
|
|
141
|
+
}
|
|
142
|
+
throw err;
|
|
143
|
+
}
|
|
144
|
+
});
|
|
145
|
+
}
|
|
146
|
+
snapshot() {
|
|
147
|
+
return Array.from(this.breakers.values()).map((b) => b.snapshot());
|
|
148
|
+
}
|
|
149
|
+
getOrCreate(key, opts) {
|
|
150
|
+
let breaker = this.breakers.get(key);
|
|
151
|
+
if (!breaker) {
|
|
152
|
+
breaker = new Breaker(key, Object.assign(Object.assign({}, this.defaults), opts));
|
|
153
|
+
this.breakers.set(key, breaker);
|
|
154
|
+
}
|
|
155
|
+
return breaker;
|
|
156
|
+
}
|
|
157
|
+
}
|
|
158
|
+
exports.CIRCUIT_BREAKERS = new CircuitBreakerRegistry();
|
|
@@ -23,22 +23,29 @@ const prisma_1 = require("../prisma");
|
|
|
23
23
|
const typedi_1 = require("typedi");
|
|
24
24
|
const logger_1 = __importDefault(require("../logger"));
|
|
25
25
|
const fs_1 = __importDefault(require("fs"));
|
|
26
|
+
const path_1 = __importDefault(require("path"));
|
|
27
|
+
const config_1 = require("../config");
|
|
26
28
|
let CleanupService = class CleanupService {
|
|
27
29
|
constructor() {
|
|
28
30
|
this.log = logger_1.default.scope('CleanupService');
|
|
29
31
|
}
|
|
30
32
|
/**
|
|
31
|
-
* Orchestrates
|
|
33
|
+
* Orchestrates build + session cleanup based on retention policy.
|
|
34
|
+
*
|
|
35
|
+
* Two phases:
|
|
36
|
+
* 1. Purge old builds (by age AND count cap) together with their sessions.
|
|
37
|
+
* 2. Sweep orphan sessions (no build_id) older than the retention window —
|
|
38
|
+
* otherwise ad-hoc sessions accumulate forever.
|
|
32
39
|
*/
|
|
33
40
|
runCleanup(pluginArgs) {
|
|
34
41
|
return __awaiter(this, void 0, void 0, function* () {
|
|
35
42
|
const { buildCleanupDays = 30, buildCleanupMaxCount = 100, deleteBuildAssets = true, } = pluginArgs;
|
|
36
43
|
this.log.info(`Starting cleanup: Retention = ${buildCleanupDays} days, Max Builds = ${buildCleanupMaxCount}, Purge Assets = ${deleteBuildAssets}`);
|
|
44
|
+
const expirationDate = new Date();
|
|
45
|
+
expirationDate.setDate(expirationDate.getDate() - buildCleanupDays);
|
|
37
46
|
try {
|
|
38
47
|
const buildIdsToPurge = new Set();
|
|
39
48
|
// 1. Identify builds to delete (by age)
|
|
40
|
-
const expirationDate = new Date();
|
|
41
|
-
expirationDate.setDate(expirationDate.getDate() - buildCleanupDays);
|
|
42
49
|
const buildsByAge = yield prisma_1.prisma.build.findMany({
|
|
43
50
|
where: {
|
|
44
51
|
createdAt: {
|
|
@@ -64,15 +71,19 @@ let CleanupService = class CleanupService {
|
|
|
64
71
|
}
|
|
65
72
|
}
|
|
66
73
|
}
|
|
67
|
-
if (buildIdsToPurge.size
|
|
68
|
-
this.log.info(
|
|
69
|
-
|
|
74
|
+
if (buildIdsToPurge.size > 0) {
|
|
75
|
+
this.log.info(`Identified ${buildIdsToPurge.size} builds for purging.`);
|
|
76
|
+
for (const buildId of buildIdsToPurge) {
|
|
77
|
+
yield this.purgeBuild(buildId, deleteBuildAssets);
|
|
78
|
+
}
|
|
70
79
|
}
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
for (const buildId of buildIdsToPurge) {
|
|
74
|
-
yield this.purgeBuild(buildId, deleteBuildAssets);
|
|
80
|
+
else {
|
|
81
|
+
this.log.info('No builds identified for cleanup.');
|
|
75
82
|
}
|
|
83
|
+
// 3. Orphan sessions (no build_id) older than retention window. Without
|
|
84
|
+
// this, ad-hoc sessions (WebDriver runs with no build capability) are
|
|
85
|
+
// never cleaned up — their assets accumulate on disk forever.
|
|
86
|
+
yield this.purgeOrphanSessions(expirationDate, deleteBuildAssets);
|
|
76
87
|
this.log.info('✅ Build cleanup completed successfully.');
|
|
77
88
|
}
|
|
78
89
|
catch (err) {
|
|
@@ -87,33 +98,15 @@ let CleanupService = class CleanupService {
|
|
|
87
98
|
return __awaiter(this, void 0, void 0, function* () {
|
|
88
99
|
const sessions = yield prisma_1.prisma.session.findMany({
|
|
89
100
|
where: { build_id: buildId },
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
},
|
|
101
|
+
select: {
|
|
102
|
+
id: true,
|
|
103
|
+
video_recording: true,
|
|
104
|
+
performance_trace: true,
|
|
95
105
|
},
|
|
96
106
|
});
|
|
97
107
|
for (const session of sessions) {
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
if (session.video_recording) {
|
|
101
|
-
this.unlinkSilently(session.video_recording);
|
|
102
|
-
}
|
|
103
|
-
// Delete screenshots
|
|
104
|
-
for (const logItem of session.SessionLog) {
|
|
105
|
-
if (logItem.screenshot) {
|
|
106
|
-
this.unlinkSilently(logItem.screenshot);
|
|
107
|
-
}
|
|
108
|
-
}
|
|
109
|
-
}
|
|
110
|
-
// Delete relations
|
|
111
|
-
// Note: We use deleteMany which is efficient.
|
|
112
|
-
yield Promise.all([
|
|
113
|
-
prisma_1.prisma.sessionLog.deleteMany({ where: { session_id: session.id } }),
|
|
114
|
-
prisma_1.prisma.log.deleteMany({ where: { session_id: session.id } }),
|
|
115
|
-
prisma_1.prisma.profiling.deleteMany({ where: { session_id: session.id } }),
|
|
116
|
-
]);
|
|
108
|
+
yield this.purgeSessionAssets(session, deleteAssets);
|
|
109
|
+
yield this.deleteSessionChildren(session.id);
|
|
117
110
|
}
|
|
118
111
|
// Delete sessions for this build
|
|
119
112
|
yield prisma_1.prisma.session.deleteMany({ where: { build_id: buildId } });
|
|
@@ -122,16 +115,121 @@ let CleanupService = class CleanupService {
|
|
|
122
115
|
this.log.debug(`Purged build: ${buildId} (${sessions.length} sessions removed)`);
|
|
123
116
|
});
|
|
124
117
|
}
|
|
118
|
+
/**
|
|
119
|
+
* Purge sessions with no parent build that are older than the retention
|
|
120
|
+
* window. Runs after build purging so build-owned sessions are already gone.
|
|
121
|
+
*/
|
|
122
|
+
purgeOrphanSessions(cutoff, deleteAssets) {
|
|
123
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
124
|
+
const orphans = yield prisma_1.prisma.session.findMany({
|
|
125
|
+
where: {
|
|
126
|
+
build_id: null,
|
|
127
|
+
createdAt: { lt: cutoff },
|
|
128
|
+
},
|
|
129
|
+
select: {
|
|
130
|
+
id: true,
|
|
131
|
+
video_recording: true,
|
|
132
|
+
performance_trace: true,
|
|
133
|
+
},
|
|
134
|
+
});
|
|
135
|
+
if (orphans.length === 0) {
|
|
136
|
+
this.log.info('No orphan sessions to purge.');
|
|
137
|
+
return;
|
|
138
|
+
}
|
|
139
|
+
this.log.info(`Purging ${orphans.length} orphan session(s) older than cutoff.`);
|
|
140
|
+
for (const session of orphans) {
|
|
141
|
+
yield this.purgeSessionAssets(session, deleteAssets);
|
|
142
|
+
yield this.deleteSessionChildren(session.id);
|
|
143
|
+
}
|
|
144
|
+
yield prisma_1.prisma.session.deleteMany({
|
|
145
|
+
where: {
|
|
146
|
+
build_id: null,
|
|
147
|
+
createdAt: { lt: cutoff },
|
|
148
|
+
},
|
|
149
|
+
});
|
|
150
|
+
});
|
|
151
|
+
}
|
|
152
|
+
/**
|
|
153
|
+
* Deletes the assets owned by a single session: video, performance trace,
|
|
154
|
+
* all screenshots referenced by its logs, and finally a recursive sweep of
|
|
155
|
+
* the session's on-disk directory. That last sweep catches anything the DB
|
|
156
|
+
* didn't know about — partial pipeline writes, aborted recordings, stray
|
|
157
|
+
* screenshot dumps — so the session folder is guaranteed empty after cleanup.
|
|
158
|
+
*/
|
|
159
|
+
purgeSessionAssets(session, deleteAssets) {
|
|
160
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
161
|
+
if (!deleteAssets)
|
|
162
|
+
return;
|
|
163
|
+
if (session.video_recording) {
|
|
164
|
+
this.unlinkSilently(session.video_recording);
|
|
165
|
+
}
|
|
166
|
+
if (session.performance_trace) {
|
|
167
|
+
this.unlinkSilently(session.performance_trace);
|
|
168
|
+
}
|
|
169
|
+
// Screenshots live on SessionLog rows, one per UI command. Fetch just the
|
|
170
|
+
// paths so we don't pull full payloads into memory for long sessions.
|
|
171
|
+
const shots = yield prisma_1.prisma.sessionLog.findMany({
|
|
172
|
+
where: { session_id: session.id, screenshot: { not: null } },
|
|
173
|
+
select: { screenshot: true },
|
|
174
|
+
});
|
|
175
|
+
for (const s of shots) {
|
|
176
|
+
if (s.screenshot)
|
|
177
|
+
this.unlinkSilently(s.screenshot);
|
|
178
|
+
}
|
|
179
|
+
// Final sweep: rm -rf the session directory. Files we already unlinked
|
|
180
|
+
// are no-ops; this phase catches the untracked strays.
|
|
181
|
+
this.removeSessionDirectory(session.id);
|
|
182
|
+
});
|
|
183
|
+
}
|
|
184
|
+
deleteSessionChildren(sessionId) {
|
|
185
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
186
|
+
yield Promise.all([
|
|
187
|
+
prisma_1.prisma.sessionLog.deleteMany({ where: { session_id: sessionId } }),
|
|
188
|
+
prisma_1.prisma.log.deleteMany({ where: { session_id: sessionId } }),
|
|
189
|
+
prisma_1.prisma.profiling.deleteMany({ where: { session_id: sessionId } }),
|
|
190
|
+
]);
|
|
191
|
+
});
|
|
192
|
+
}
|
|
193
|
+
removeSessionDirectory(sessionId) {
|
|
194
|
+
// Defense against path traversal: reject anything that, when joined and
|
|
195
|
+
// normalized, escapes sessionAssetsPath. sessionId is a UUID in normal
|
|
196
|
+
// use but a corrupted DB row shouldn't let us rm -rf arbitrary paths.
|
|
197
|
+
const base = path_1.default.resolve(config_1.config.sessionAssetsPath);
|
|
198
|
+
const target = path_1.default.resolve(base, sessionId);
|
|
199
|
+
if (!target.startsWith(base + path_1.default.sep) && target !== base) {
|
|
200
|
+
this.log.warn(`Refusing to remove suspicious session dir: ${sessionId}`);
|
|
201
|
+
return;
|
|
202
|
+
}
|
|
203
|
+
if (target === base) {
|
|
204
|
+
this.log.warn('Refusing to remove sessionAssetsPath itself');
|
|
205
|
+
return;
|
|
206
|
+
}
|
|
207
|
+
try {
|
|
208
|
+
fs_1.default.rmSync(target, { recursive: true, force: true });
|
|
209
|
+
this.log.debug(`Removed session directory: ${target}`);
|
|
210
|
+
}
|
|
211
|
+
catch (err) {
|
|
212
|
+
this.log.warn(`Failed to remove session dir ${target}: ${err.message}`);
|
|
213
|
+
}
|
|
214
|
+
}
|
|
125
215
|
unlinkSilently(filePath) {
|
|
216
|
+
// Asset paths in the DB are stored relative to sessionAssetsPath (see
|
|
217
|
+
// asset-manager.ts). Resolving them against cwd — which the prior
|
|
218
|
+
// implementation did implicitly — means fs.existsSync always returned
|
|
219
|
+
// false and nothing ever got unlinked. Always go via path.resolve.
|
|
220
|
+
if (!filePath)
|
|
221
|
+
return;
|
|
222
|
+
const resolved = path_1.default.isAbsolute(filePath)
|
|
223
|
+
? filePath
|
|
224
|
+
: path_1.default.resolve(config_1.config.sessionAssetsPath, filePath);
|
|
126
225
|
try {
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
this.log.debug(`Deleted asset: ${filePath}`);
|
|
226
|
+
if (fs_1.default.existsSync(resolved)) {
|
|
227
|
+
fs_1.default.unlinkSync(resolved);
|
|
228
|
+
this.log.debug(`Deleted asset: ${resolved}`);
|
|
131
229
|
}
|
|
132
230
|
}
|
|
133
231
|
catch (e) {
|
|
134
|
-
this.log.warn(`Failed to delete asset ${
|
|
232
|
+
this.log.warn(`Failed to delete asset ${resolved}: ${e.message}`);
|
|
135
233
|
}
|
|
136
234
|
}
|
|
137
235
|
};
|