buildhive-agent 1.0.0-beta.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +166 -0
- package/dist/__tests__/fakes/FakeDockerManager.d.ts +115 -0
- package/dist/__tests__/fakes/FakeDockerManager.d.ts.map +1 -0
- package/dist/__tests__/fakes/FakeDockerManager.js +203 -0
- package/dist/__tests__/fakes/FakeDockerManager.js.map +1 -0
- package/dist/acceptanceChecker.d.ts +26 -0
- package/dist/acceptanceChecker.d.ts.map +1 -0
- package/dist/acceptanceChecker.js +64 -0
- package/dist/acceptanceChecker.js.map +1 -0
- package/dist/advancedAgent.d.ts +161 -0
- package/dist/advancedAgent.d.ts.map +1 -0
- package/dist/advancedAgent.js +604 -0
- package/dist/advancedAgent.js.map +1 -0
- package/dist/agent.d.ts +101 -0
- package/dist/agent.d.ts.map +1 -0
- package/dist/agent.js +490 -0
- package/dist/agent.js.map +1 -0
- package/dist/api/jobStatusApi.d.ts +88 -0
- package/dist/api/jobStatusApi.d.ts.map +1 -0
- package/dist/api/jobStatusApi.js +240 -0
- package/dist/api/jobStatusApi.js.map +1 -0
- package/dist/autoUpdater.d.ts +135 -0
- package/dist/autoUpdater.d.ts.map +1 -0
- package/dist/autoUpdater.js +494 -0
- package/dist/autoUpdater.js.map +1 -0
- package/dist/cacheManager.d.ts +108 -0
- package/dist/cacheManager.d.ts.map +1 -0
- package/dist/cacheManager.js +300 -0
- package/dist/cacheManager.js.map +1 -0
- package/dist/cli.d.ts +11 -0
- package/dist/cli.d.ts.map +1 -0
- package/dist/cli.js +749 -0
- package/dist/cli.js.map +1 -0
- package/dist/config/index.d.ts +30 -0
- package/dist/config/index.d.ts.map +1 -0
- package/dist/config/index.js +35 -0
- package/dist/config/index.js.map +1 -0
- package/dist/config/loader.d.ts +45 -0
- package/dist/config/loader.d.ts.map +1 -0
- package/dist/config/loader.js +269 -0
- package/dist/config/loader.js.map +1 -0
- package/dist/config/types.d.ts +193 -0
- package/dist/config/types.d.ts.map +1 -0
- package/dist/config/types.js +90 -0
- package/dist/config/types.js.map +1 -0
- package/dist/config/validation.d.ts +28 -0
- package/dist/config/validation.d.ts.map +1 -0
- package/dist/config/validation.js +397 -0
- package/dist/config/validation.js.map +1 -0
- package/dist/docker.d.ts +96 -0
- package/dist/docker.d.ts.map +1 -0
- package/dist/docker.js +411 -0
- package/dist/docker.js.map +1 -0
- package/dist/enhancedJobExecutor.d.ts +81 -0
- package/dist/enhancedJobExecutor.d.ts.map +1 -0
- package/dist/enhancedJobExecutor.js +223 -0
- package/dist/enhancedJobExecutor.js.map +1 -0
- package/dist/executors/executorFactory.d.ts +46 -0
- package/dist/executors/executorFactory.d.ts.map +1 -0
- package/dist/executors/executorFactory.js +80 -0
- package/dist/executors/executorFactory.js.map +1 -0
- package/dist/executors/index.d.ts +7 -0
- package/dist/executors/index.d.ts.map +1 -0
- package/dist/executors/index.js +6 -0
- package/dist/executors/index.js.map +1 -0
- package/dist/executors/nativeExecutor.d.ts +60 -0
- package/dist/executors/nativeExecutor.d.ts.map +1 -0
- package/dist/executors/nativeExecutor.js +311 -0
- package/dist/executors/nativeExecutor.js.map +1 -0
- package/dist/executors/types.d.ts +38 -0
- package/dist/executors/types.d.ts.map +1 -0
- package/dist/executors/types.js +9 -0
- package/dist/executors/types.js.map +1 -0
- package/dist/healthMonitor.d.ts +213 -0
- package/dist/healthMonitor.d.ts.map +1 -0
- package/dist/healthMonitor.js +547 -0
- package/dist/healthMonitor.js.map +1 -0
- package/dist/index.d.ts +16 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +16 -0
- package/dist/index.js.map +1 -0
- package/dist/jobExecutor.d.ts +117 -0
- package/dist/jobExecutor.d.ts.map +1 -0
- package/dist/jobExecutor.js +458 -0
- package/dist/jobExecutor.js.map +1 -0
- package/dist/lifecycleExecutor.d.ts +54 -0
- package/dist/lifecycleExecutor.d.ts.map +1 -0
- package/dist/lifecycleExecutor.js +230 -0
- package/dist/lifecycleExecutor.js.map +1 -0
- package/dist/main.d.ts +15 -0
- package/dist/main.d.ts.map +1 -0
- package/dist/main.js +77 -0
- package/dist/main.js.map +1 -0
- package/dist/metrics.d.ts +103 -0
- package/dist/metrics.d.ts.map +1 -0
- package/dist/metrics.js +360 -0
- package/dist/metrics.js.map +1 -0
- package/dist/recipes/builtinRecipes.d.ts +11 -0
- package/dist/recipes/builtinRecipes.d.ts.map +1 -0
- package/dist/recipes/builtinRecipes.js +688 -0
- package/dist/recipes/builtinRecipes.js.map +1 -0
- package/dist/recipes/index.d.ts +18 -0
- package/dist/recipes/index.d.ts.map +1 -0
- package/dist/recipes/index.js +17 -0
- package/dist/recipes/index.js.map +1 -0
- package/dist/recipes/recipeRegistry.d.ts +49 -0
- package/dist/recipes/recipeRegistry.d.ts.map +1 -0
- package/dist/recipes/recipeRegistry.js +264 -0
- package/dist/recipes/recipeRegistry.js.map +1 -0
- package/dist/recipes/types.d.ts +116 -0
- package/dist/recipes/types.d.ts.map +1 -0
- package/dist/recipes/types.js +10 -0
- package/dist/recipes/types.js.map +1 -0
- package/dist/recovery.d.ts +133 -0
- package/dist/recovery.d.ts.map +1 -0
- package/dist/recovery.js +299 -0
- package/dist/recovery.js.map +1 -0
- package/dist/registration/apiClient.d.ts +44 -0
- package/dist/registration/apiClient.d.ts.map +1 -0
- package/dist/registration/apiClient.js +149 -0
- package/dist/registration/apiClient.js.map +1 -0
- package/dist/registration/index.d.ts +41 -0
- package/dist/registration/index.d.ts.map +1 -0
- package/dist/registration/index.js +141 -0
- package/dist/registration/index.js.map +1 -0
- package/dist/registration/machineId.d.ts +30 -0
- package/dist/registration/machineId.d.ts.map +1 -0
- package/dist/registration/machineId.js +89 -0
- package/dist/registration/machineId.js.map +1 -0
- package/dist/registration/types.d.ts +32 -0
- package/dist/registration/types.d.ts.map +1 -0
- package/dist/registration/types.js +9 -0
- package/dist/registration/types.js.map +1 -0
- package/dist/resourceGovernor.d.ts +57 -0
- package/dist/resourceGovernor.d.ts.map +1 -0
- package/dist/resourceGovernor.js +125 -0
- package/dist/resourceGovernor.js.map +1 -0
- package/dist/security/secretManager.d.ts +107 -0
- package/dist/security/secretManager.d.ts.map +1 -0
- package/dist/security/secretManager.js +361 -0
- package/dist/security/secretManager.js.map +1 -0
- package/dist/security.d.ts +134 -0
- package/dist/security.d.ts.map +1 -0
- package/dist/security.js +470 -0
- package/dist/security.js.map +1 -0
- package/dist/storage/artifactUploader.d.ts +155 -0
- package/dist/storage/artifactUploader.d.ts.map +1 -0
- package/dist/storage/artifactUploader.js +554 -0
- package/dist/storage/artifactUploader.js.map +1 -0
- package/dist/types.d.ts +49 -0
- package/dist/types.d.ts.map +1 -0
- package/dist/types.js +7 -0
- package/dist/types.js.map +1 -0
- package/dist/utils/capabilities.d.ts +23 -0
- package/dist/utils/capabilities.d.ts.map +1 -0
- package/dist/utils/capabilities.js +200 -0
- package/dist/utils/capabilities.js.map +1 -0
- package/dist/utils/logger.d.ts +20 -0
- package/dist/utils/logger.d.ts.map +1 -0
- package/dist/utils/logger.js +188 -0
- package/dist/utils/logger.js.map +1 -0
- package/dist/utils/sdkScanner.d.ts +105 -0
- package/dist/utils/sdkScanner.d.ts.map +1 -0
- package/dist/utils/sdkScanner.js +459 -0
- package/dist/utils/sdkScanner.js.map +1 -0
- package/dist/websocketClient.d.ts +154 -0
- package/dist/websocketClient.d.ts.map +1 -0
- package/dist/websocketClient.js +422 -0
- package/dist/websocketClient.js.map +1 -0
- package/package.json +64 -0
|
@@ -0,0 +1,133 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Recovery Manager - Error recovery and retry logic
|
|
3
|
+
*
|
|
4
|
+
* Provides robust error recovery mechanisms for the BuildHive agent including:
|
|
5
|
+
* - Retry logic for failed job status updates
|
|
6
|
+
* - Docker daemon health checking and recovery
|
|
7
|
+
* - Partial job failure detection and recovery
|
|
8
|
+
* - Error tracking and analysis
|
|
9
|
+
*
|
|
10
|
+
* Requirements: MVP.4.3.2 - Error recovery
|
|
11
|
+
*/
|
|
12
|
+
import { JobStatusApi } from './api/jobStatusApi.js';
|
|
13
|
+
import Docker from 'dockerode';
|
|
14
|
+
export interface RetryConfig {
|
|
15
|
+
maxAttempts: number;
|
|
16
|
+
initialDelayMs: number;
|
|
17
|
+
maxDelayMs: number;
|
|
18
|
+
backoffMultiplier: number;
|
|
19
|
+
retryableErrors?: string[];
|
|
20
|
+
}
|
|
21
|
+
export interface RecoveryStats {
|
|
22
|
+
totalRetries: number;
|
|
23
|
+
successfulRetries: number;
|
|
24
|
+
failedRetries: number;
|
|
25
|
+
dockerHealthChecks: number;
|
|
26
|
+
dockerRecoveries: number;
|
|
27
|
+
errorsSinceLastSuccess: number;
|
|
28
|
+
lastErrorTimestamp?: Date;
|
|
29
|
+
lastSuccessTimestamp?: Date;
|
|
30
|
+
}
|
|
31
|
+
export declare class RecoveryManager {
|
|
32
|
+
private docker;
|
|
33
|
+
private stats;
|
|
34
|
+
private defaultRetryConfig;
|
|
35
|
+
constructor(docker?: Docker);
|
|
36
|
+
/**
|
|
37
|
+
* Retry an async operation with exponential backoff
|
|
38
|
+
*
|
|
39
|
+
* @param operation - Async function to retry
|
|
40
|
+
* @param config - Retry configuration
|
|
41
|
+
* @param operationName - Name for logging
|
|
42
|
+
* @returns Result of the operation
|
|
43
|
+
*/
|
|
44
|
+
retryOperation<T>(operation: () => Promise<T>, config?: Partial<RetryConfig>, operationName?: string): Promise<T>;
|
|
45
|
+
/**
|
|
46
|
+
* Retry job status update with exponential backoff
|
|
47
|
+
*
|
|
48
|
+
* @param statusApi - JobStatusApi instance
|
|
49
|
+
* @param jobId - Job ID
|
|
50
|
+
* @param update - Status update data
|
|
51
|
+
* @param config - Custom retry configuration
|
|
52
|
+
*/
|
|
53
|
+
retryJobStatusUpdate(statusApi: JobStatusApi, jobId: string, update: any, config?: Partial<RetryConfig>): Promise<void>;
|
|
54
|
+
/**
|
|
55
|
+
* Check if Docker daemon is healthy
|
|
56
|
+
*
|
|
57
|
+
* @returns true if Docker is healthy, false otherwise
|
|
58
|
+
*/
|
|
59
|
+
checkDockerHealth(): Promise<boolean>;
|
|
60
|
+
/**
|
|
61
|
+
* Attempt to recover Docker connection
|
|
62
|
+
*
|
|
63
|
+
* @param maxAttempts - Maximum recovery attempts
|
|
64
|
+
* @returns true if recovery successful
|
|
65
|
+
*/
|
|
66
|
+
recoverDockerConnection(maxAttempts?: number): Promise<boolean>;
|
|
67
|
+
/**
|
|
68
|
+
* Monitor Docker daemon health continuously
|
|
69
|
+
*
|
|
70
|
+
* @param intervalMs - Health check interval in milliseconds
|
|
71
|
+
* @param onUnhealthy - Callback when Docker becomes unhealthy
|
|
72
|
+
* @returns Interval timer ID
|
|
73
|
+
*/
|
|
74
|
+
startDockerHealthMonitoring(intervalMs?: number, onUnhealthy?: () => void): NodeJS.Timeout;
|
|
75
|
+
/**
|
|
76
|
+
* Detect and recover from partial job failures
|
|
77
|
+
*
|
|
78
|
+
* This method checks for common failure patterns and attempts recovery.
|
|
79
|
+
*
|
|
80
|
+
* @param error - Error that occurred
|
|
81
|
+
* @param jobId - Job ID
|
|
82
|
+
* @param retryFn - Function to retry the job
|
|
83
|
+
* @returns true if recovery was attempted
|
|
84
|
+
*/
|
|
85
|
+
recoverFromJobFailure(error: any, jobId: string, retryFn?: () => Promise<void>): Promise<boolean>;
|
|
86
|
+
/**
|
|
87
|
+
* Get recovery statistics
|
|
88
|
+
*
|
|
89
|
+
* @returns Recovery statistics object
|
|
90
|
+
*/
|
|
91
|
+
getStats(): RecoveryStats;
|
|
92
|
+
/**
|
|
93
|
+
* Reset recovery statistics
|
|
94
|
+
*/
|
|
95
|
+
resetStats(): void;
|
|
96
|
+
/**
|
|
97
|
+
* Check if an error is retryable
|
|
98
|
+
*
|
|
99
|
+
* @param error - Error to check
|
|
100
|
+
* @param config - Retry configuration
|
|
101
|
+
* @returns true if error is retryable
|
|
102
|
+
*/
|
|
103
|
+
private isRetryableError;
|
|
104
|
+
/**
|
|
105
|
+
* Check if a job failure is recoverable
|
|
106
|
+
*
|
|
107
|
+
* @param error - Error to analyze
|
|
108
|
+
* @returns true if failure is recoverable
|
|
109
|
+
*/
|
|
110
|
+
private isRecoverableJobFailure;
|
|
111
|
+
/**
|
|
112
|
+
* Calculate retry delay with exponential backoff
|
|
113
|
+
*
|
|
114
|
+
* @param attempt - Current attempt number
|
|
115
|
+
* @param config - Retry configuration
|
|
116
|
+
* @returns Delay in milliseconds
|
|
117
|
+
*/
|
|
118
|
+
private calculateDelay;
|
|
119
|
+
/**
|
|
120
|
+
* Sleep for specified milliseconds
|
|
121
|
+
*
|
|
122
|
+
* @param ms - Milliseconds to sleep
|
|
123
|
+
*/
|
|
124
|
+
private sleep;
|
|
125
|
+
}
|
|
126
|
+
/**
|
|
127
|
+
* Create a recovery manager instance
|
|
128
|
+
*
|
|
129
|
+
* @param docker - Docker instance (optional)
|
|
130
|
+
* @returns RecoveryManager instance
|
|
131
|
+
*/
|
|
132
|
+
export declare function createRecoveryManager(docker?: Docker): RecoveryManager;
|
|
133
|
+
//# sourceMappingURL=recovery.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"recovery.d.ts","sourceRoot":"","sources":["../src/recovery.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAGH,OAAO,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAC;AACrD,OAAO,MAAM,MAAM,WAAW,CAAC;AAI/B,MAAM,WAAW,WAAW;IAC1B,WAAW,EAAE,MAAM,CAAC;IACpB,cAAc,EAAE,MAAM,CAAC;IACvB,UAAU,EAAE,MAAM,CAAC;IACnB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,eAAe,CAAC,EAAE,MAAM,EAAE,CAAC;CAC5B;AAED,MAAM,WAAW,aAAa;IAC5B,YAAY,EAAE,MAAM,CAAC;IACrB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,aAAa,EAAE,MAAM,CAAC;IACtB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,gBAAgB,EAAE,MAAM,CAAC;IACzB,sBAAsB,EAAE,MAAM,CAAC;IAC/B,kBAAkB,CAAC,EAAE,IAAI,CAAC;IAC1B,oBAAoB,CAAC,EAAE,IAAI,CAAC;CAC7B;AAED,qBAAa,eAAe;IAC1B,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,KAAK,CAOX;IAEF,OAAO,CAAC,kBAAkB,CAcxB;gBAEU,MAAM,CAAC,EAAE,MAAM;IAK3B;;;;;;;OAOG;IACG,cAAc,CAAC,CAAC,EACpB,SAAS,EAAE,MAAM,OAAO,CAAC,CAAC,CAAC,EAC3B,MAAM,GAAE,OAAO,CAAC,WAAW,CAAM,EACjC,aAAa,GAAE,MAAoB,GAClC,OAAO,CAAC,CAAC,CAAC;IAsDb;;;;;;;OAOG;IACG,oBAAoB,CACxB,SAAS,EAAE,YAAY,EACvB,KAAK,EAAE,MAAM,EACb,MAAM,EAAE,GAAG,EACX,MAAM,CAAC,EAAE,OAAO,CAAC,WAAW,CAAC,GAC5B,OAAO,CAAC,IAAI,CAAC;IAQhB;;;;OAIG;IACG,iBAAiB,IAAI,OAAO,CAAC,OAAO,CAAC;IAuB3C;;;;;OAKG;IACG,uBAAuB,CAAC,WAAW,GAAE,MAAU,GAAG,OAAO,CAAC,OAAO,CAAC;IAyBxE;;;;;;OAMG;IACH,2BAA2B,CACzB,UAAU,GAAE,MAAc,EAC1B,WAAW,CAAC,EAAE,MAAM,IAAI,GACvB,MAAM,CAAC,OAAO;IAmBjB;;;;;;;;;OASG;IACG,qBAAqB,CACzB,KAAK,EAAE,GAAG,EACV,KAAK,EAAE,MAAM,EACb,OAAO,CAAC,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,GAC5B,OAAO,CAAC,OAAO,CAAC;IA8BnB;;;;OAIG;IACH,QAAQ,IAAI,aAAa;IAIzB;;OAEG;IACH,UAAU,IAAI,IAAI;IAYlB;;;;;;OAMG;IACH,OAAO,CAAC,gBAAgB;IAkBxB;;;;;OAKG;IACH,OAAO,CAAC,uBAAuB;IAiB/B;;;;;;OAMG;IACH,OAAO,CAAC,cAAc;IAKtB;;;;OAIG;IACH,OAAO,CAAC,KAAK;CAGd;AAED;;;;;GAKG;AACH,wBAAgB,qBAAqB,CAAC,MAAM,CAAC,EAAE,MAAM,GAAG,eAAe,CAEtE"}
|
package/dist/recovery.js
ADDED
|
@@ -0,0 +1,299 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Recovery Manager - Error recovery and retry logic
|
|
3
|
+
*
|
|
4
|
+
* Provides robust error recovery mechanisms for the BuildHive agent including:
|
|
5
|
+
* - Retry logic for failed job status updates
|
|
6
|
+
* - Docker daemon health checking and recovery
|
|
7
|
+
* - Partial job failure detection and recovery
|
|
8
|
+
* - Error tracking and analysis
|
|
9
|
+
*
|
|
10
|
+
* Requirements: MVP.4.3.2 - Error recovery
|
|
11
|
+
*/
|
|
12
|
+
import { createLogger } from './utils/logger.js';
|
|
13
|
+
import Docker from 'dockerode';
|
|
14
|
+
const logger = createLogger('recovery');
|
|
15
|
+
export class RecoveryManager {
|
|
16
|
+
docker;
|
|
17
|
+
stats = {
|
|
18
|
+
totalRetries: 0,
|
|
19
|
+
successfulRetries: 0,
|
|
20
|
+
failedRetries: 0,
|
|
21
|
+
dockerHealthChecks: 0,
|
|
22
|
+
dockerRecoveries: 0,
|
|
23
|
+
errorsSinceLastSuccess: 0
|
|
24
|
+
};
|
|
25
|
+
defaultRetryConfig = {
|
|
26
|
+
maxAttempts: 3,
|
|
27
|
+
initialDelayMs: 1000,
|
|
28
|
+
maxDelayMs: 30000,
|
|
29
|
+
backoffMultiplier: 2,
|
|
30
|
+
retryableErrors: [
|
|
31
|
+
'ECONNREFUSED',
|
|
32
|
+
'ETIMEDOUT',
|
|
33
|
+
'ENOTFOUND',
|
|
34
|
+
'ENETUNREACH',
|
|
35
|
+
'503',
|
|
36
|
+
'502',
|
|
37
|
+
'504'
|
|
38
|
+
]
|
|
39
|
+
};
|
|
40
|
+
constructor(docker) {
|
|
41
|
+
this.docker = docker || new Docker();
|
|
42
|
+
logger.info('RecoveryManager initialized');
|
|
43
|
+
}
|
|
44
|
+
/**
|
|
45
|
+
* Retry an async operation with exponential backoff
|
|
46
|
+
*
|
|
47
|
+
* @param operation - Async function to retry
|
|
48
|
+
* @param config - Retry configuration
|
|
49
|
+
* @param operationName - Name for logging
|
|
50
|
+
* @returns Result of the operation
|
|
51
|
+
*/
|
|
52
|
+
async retryOperation(operation, config = {}, operationName = 'operation') {
|
|
53
|
+
const retryConfig = { ...this.defaultRetryConfig, ...config };
|
|
54
|
+
let lastError;
|
|
55
|
+
let attempt = 0;
|
|
56
|
+
while (attempt < retryConfig.maxAttempts) {
|
|
57
|
+
attempt++;
|
|
58
|
+
try {
|
|
59
|
+
logger.debug(`Attempting ${operationName} (attempt ${attempt}/${retryConfig.maxAttempts})`);
|
|
60
|
+
const result = await operation();
|
|
61
|
+
if (attempt > 1) {
|
|
62
|
+
this.stats.successfulRetries++;
|
|
63
|
+
logger.info(`${operationName} succeeded after ${attempt} attempts`);
|
|
64
|
+
}
|
|
65
|
+
this.stats.errorsSinceLastSuccess = 0;
|
|
66
|
+
this.stats.lastSuccessTimestamp = new Date();
|
|
67
|
+
return result;
|
|
68
|
+
}
|
|
69
|
+
catch (error) {
|
|
70
|
+
lastError = error;
|
|
71
|
+
this.stats.totalRetries++;
|
|
72
|
+
this.stats.errorsSinceLastSuccess++;
|
|
73
|
+
this.stats.lastErrorTimestamp = new Date();
|
|
74
|
+
const isRetryable = this.isRetryableError(error, retryConfig);
|
|
75
|
+
if (!isRetryable || attempt >= retryConfig.maxAttempts) {
|
|
76
|
+
this.stats.failedRetries++;
|
|
77
|
+
logger.error(`${operationName} failed after ${attempt} attempts`, {
|
|
78
|
+
error: error.message,
|
|
79
|
+
retryable: isRetryable
|
|
80
|
+
});
|
|
81
|
+
throw error;
|
|
82
|
+
}
|
|
83
|
+
const delay = this.calculateDelay(attempt, retryConfig);
|
|
84
|
+
logger.warn(`${operationName} failed, retrying in ${delay}ms`, {
|
|
85
|
+
attempt,
|
|
86
|
+
maxAttempts: retryConfig.maxAttempts,
|
|
87
|
+
error: error.message
|
|
88
|
+
});
|
|
89
|
+
await this.sleep(delay);
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
throw lastError || new Error(`${operationName} failed after ${retryConfig.maxAttempts} attempts`);
|
|
93
|
+
}
|
|
94
|
+
/**
|
|
95
|
+
* Retry job status update with exponential backoff
|
|
96
|
+
*
|
|
97
|
+
* @param statusApi - JobStatusApi instance
|
|
98
|
+
* @param jobId - Job ID
|
|
99
|
+
* @param update - Status update data
|
|
100
|
+
* @param config - Custom retry configuration
|
|
101
|
+
*/
|
|
102
|
+
async retryJobStatusUpdate(statusApi, jobId, update, config) {
|
|
103
|
+
return this.retryOperation(() => statusApi.updateJobStatus(jobId, update), config, `job status update for ${jobId}`);
|
|
104
|
+
}
|
|
105
|
+
/**
|
|
106
|
+
* Check if Docker daemon is healthy
|
|
107
|
+
*
|
|
108
|
+
* @returns true if Docker is healthy, false otherwise
|
|
109
|
+
*/
|
|
110
|
+
async checkDockerHealth() {
|
|
111
|
+
this.stats.dockerHealthChecks++;
|
|
112
|
+
try {
|
|
113
|
+
logger.debug('Checking Docker daemon health');
|
|
114
|
+
// Try to ping Docker
|
|
115
|
+
const ping = await this.docker.ping();
|
|
116
|
+
if (ping.toString() === 'OK') {
|
|
117
|
+
logger.debug('Docker daemon is healthy');
|
|
118
|
+
return true;
|
|
119
|
+
}
|
|
120
|
+
logger.warn('Docker ping returned unexpected response', { ping });
|
|
121
|
+
return false;
|
|
122
|
+
}
|
|
123
|
+
catch (error) {
|
|
124
|
+
logger.error('Docker health check failed', { error: error.message });
|
|
125
|
+
return false;
|
|
126
|
+
}
|
|
127
|
+
}
|
|
128
|
+
/**
|
|
129
|
+
* Attempt to recover Docker connection
|
|
130
|
+
*
|
|
131
|
+
* @param maxAttempts - Maximum recovery attempts
|
|
132
|
+
* @returns true if recovery successful
|
|
133
|
+
*/
|
|
134
|
+
async recoverDockerConnection(maxAttempts = 5) {
|
|
135
|
+
logger.info('Attempting Docker connection recovery');
|
|
136
|
+
for (let attempt = 1; attempt <= maxAttempts; attempt++) {
|
|
137
|
+
logger.info(`Docker recovery attempt ${attempt}/${maxAttempts}`);
|
|
138
|
+
const isHealthy = await this.checkDockerHealth();
|
|
139
|
+
if (isHealthy) {
|
|
140
|
+
this.stats.dockerRecoveries++;
|
|
141
|
+
logger.info('Docker connection recovered successfully');
|
|
142
|
+
return true;
|
|
143
|
+
}
|
|
144
|
+
if (attempt < maxAttempts) {
|
|
145
|
+
const delay = Math.min(1000 * Math.pow(2, attempt), 30000);
|
|
146
|
+
logger.info(`Waiting ${delay}ms before next recovery attempt`);
|
|
147
|
+
await this.sleep(delay);
|
|
148
|
+
}
|
|
149
|
+
}
|
|
150
|
+
logger.error('Failed to recover Docker connection after all attempts');
|
|
151
|
+
return false;
|
|
152
|
+
}
|
|
153
|
+
/**
|
|
154
|
+
* Monitor Docker daemon health continuously
|
|
155
|
+
*
|
|
156
|
+
* @param intervalMs - Health check interval in milliseconds
|
|
157
|
+
* @param onUnhealthy - Callback when Docker becomes unhealthy
|
|
158
|
+
* @returns Interval timer ID
|
|
159
|
+
*/
|
|
160
|
+
startDockerHealthMonitoring(intervalMs = 60000, onUnhealthy) {
|
|
161
|
+
logger.info('Starting Docker health monitoring', { intervalMs });
|
|
162
|
+
return setInterval(async () => {
|
|
163
|
+
const isHealthy = await this.checkDockerHealth();
|
|
164
|
+
if (!isHealthy) {
|
|
165
|
+
logger.warn('Docker health check failed, attempting recovery');
|
|
166
|
+
const recovered = await this.recoverDockerConnection();
|
|
167
|
+
if (!recovered && onUnhealthy) {
|
|
168
|
+
logger.error('Docker recovery failed, calling unhealthy callback');
|
|
169
|
+
onUnhealthy();
|
|
170
|
+
}
|
|
171
|
+
}
|
|
172
|
+
}, intervalMs);
|
|
173
|
+
}
|
|
174
|
+
/**
|
|
175
|
+
* Detect and recover from partial job failures
|
|
176
|
+
*
|
|
177
|
+
* This method checks for common failure patterns and attempts recovery.
|
|
178
|
+
*
|
|
179
|
+
* @param error - Error that occurred
|
|
180
|
+
* @param jobId - Job ID
|
|
181
|
+
* @param retryFn - Function to retry the job
|
|
182
|
+
* @returns true if recovery was attempted
|
|
183
|
+
*/
|
|
184
|
+
async recoverFromJobFailure(error, jobId, retryFn) {
|
|
185
|
+
logger.info('Analyzing job failure for recovery', { jobId, error: error.message });
|
|
186
|
+
// Check for recoverable failure patterns
|
|
187
|
+
const isRecoverable = this.isRecoverableJobFailure(error);
|
|
188
|
+
if (!isRecoverable) {
|
|
189
|
+
logger.info('Job failure is not recoverable', { jobId });
|
|
190
|
+
return false;
|
|
191
|
+
}
|
|
192
|
+
if (!retryFn) {
|
|
193
|
+
logger.warn('No retry function provided for job recovery', { jobId });
|
|
194
|
+
return false;
|
|
195
|
+
}
|
|
196
|
+
try {
|
|
197
|
+
logger.info('Attempting job recovery', { jobId });
|
|
198
|
+
await this.retryOperation(retryFn, {}, `job recovery for ${jobId}`);
|
|
199
|
+
logger.info('Job recovery successful', { jobId });
|
|
200
|
+
return true;
|
|
201
|
+
}
|
|
202
|
+
catch (recoveryError) {
|
|
203
|
+
logger.error('Job recovery failed', {
|
|
204
|
+
jobId,
|
|
205
|
+
error: recoveryError.message
|
|
206
|
+
});
|
|
207
|
+
return false;
|
|
208
|
+
}
|
|
209
|
+
}
|
|
210
|
+
/**
|
|
211
|
+
* Get recovery statistics
|
|
212
|
+
*
|
|
213
|
+
* @returns Recovery statistics object
|
|
214
|
+
*/
|
|
215
|
+
getStats() {
|
|
216
|
+
return { ...this.stats };
|
|
217
|
+
}
|
|
218
|
+
/**
|
|
219
|
+
* Reset recovery statistics
|
|
220
|
+
*/
|
|
221
|
+
resetStats() {
|
|
222
|
+
this.stats = {
|
|
223
|
+
totalRetries: 0,
|
|
224
|
+
successfulRetries: 0,
|
|
225
|
+
failedRetries: 0,
|
|
226
|
+
dockerHealthChecks: 0,
|
|
227
|
+
dockerRecoveries: 0,
|
|
228
|
+
errorsSinceLastSuccess: 0
|
|
229
|
+
};
|
|
230
|
+
logger.info('Recovery statistics reset');
|
|
231
|
+
}
|
|
232
|
+
/**
|
|
233
|
+
* Check if an error is retryable
|
|
234
|
+
*
|
|
235
|
+
* @param error - Error to check
|
|
236
|
+
* @param config - Retry configuration
|
|
237
|
+
* @returns true if error is retryable
|
|
238
|
+
*/
|
|
239
|
+
isRetryableError(error, config) {
|
|
240
|
+
const errorMessage = error.message || '';
|
|
241
|
+
const errorCode = error.code || '';
|
|
242
|
+
const statusCode = error.response?.status?.toString() || '';
|
|
243
|
+
for (const retryableError of config.retryableErrors || []) {
|
|
244
|
+
if (errorMessage.includes(retryableError) ||
|
|
245
|
+
errorCode === retryableError ||
|
|
246
|
+
statusCode === retryableError) {
|
|
247
|
+
return true;
|
|
248
|
+
}
|
|
249
|
+
}
|
|
250
|
+
return false;
|
|
251
|
+
}
|
|
252
|
+
/**
|
|
253
|
+
* Check if a job failure is recoverable
|
|
254
|
+
*
|
|
255
|
+
* @param error - Error to analyze
|
|
256
|
+
* @returns true if failure is recoverable
|
|
257
|
+
*/
|
|
258
|
+
isRecoverableJobFailure(error) {
|
|
259
|
+
const recoverablePatterns = [
|
|
260
|
+
'ECONNREFUSED',
|
|
261
|
+
'ETIMEDOUT',
|
|
262
|
+
'Container exited unexpectedly',
|
|
263
|
+
'Docker daemon not responding',
|
|
264
|
+
'Network timeout',
|
|
265
|
+
'Temporary failure'
|
|
266
|
+
];
|
|
267
|
+
const errorMessage = (error.message || '').toLowerCase();
|
|
268
|
+
return recoverablePatterns.some(pattern => errorMessage.includes(pattern.toLowerCase()));
|
|
269
|
+
}
|
|
270
|
+
/**
|
|
271
|
+
* Calculate retry delay with exponential backoff
|
|
272
|
+
*
|
|
273
|
+
* @param attempt - Current attempt number
|
|
274
|
+
* @param config - Retry configuration
|
|
275
|
+
* @returns Delay in milliseconds
|
|
276
|
+
*/
|
|
277
|
+
calculateDelay(attempt, config) {
|
|
278
|
+
const delay = config.initialDelayMs * Math.pow(config.backoffMultiplier, attempt - 1);
|
|
279
|
+
return Math.min(delay, config.maxDelayMs);
|
|
280
|
+
}
|
|
281
|
+
/**
|
|
282
|
+
* Sleep for specified milliseconds
|
|
283
|
+
*
|
|
284
|
+
* @param ms - Milliseconds to sleep
|
|
285
|
+
*/
|
|
286
|
+
sleep(ms) {
|
|
287
|
+
return new Promise(resolve => setTimeout(resolve, ms));
|
|
288
|
+
}
|
|
289
|
+
}
|
|
290
|
+
/**
|
|
291
|
+
* Create a recovery manager instance
|
|
292
|
+
*
|
|
293
|
+
* @param docker - Docker instance (optional)
|
|
294
|
+
* @returns RecoveryManager instance
|
|
295
|
+
*/
|
|
296
|
+
export function createRecoveryManager(docker) {
|
|
297
|
+
return new RecoveryManager(docker);
|
|
298
|
+
}
|
|
299
|
+
//# sourceMappingURL=recovery.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"recovery.js","sourceRoot":"","sources":["../src/recovery.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,OAAO,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AAEjD,OAAO,MAAM,MAAM,WAAW,CAAC;AAE/B,MAAM,MAAM,GAAG,YAAY,CAAC,UAAU,CAAC,CAAC;AAqBxC,MAAM,OAAO,eAAe;IAClB,MAAM,CAAS;IACf,KAAK,GAAkB;QAC7B,YAAY,EAAE,CAAC;QACf,iBAAiB,EAAE,CAAC;QACpB,aAAa,EAAE,CAAC;QAChB,kBAAkB,EAAE,CAAC;QACrB,gBAAgB,EAAE,CAAC;QACnB,sBAAsB,EAAE,CAAC;KAC1B,CAAC;IAEM,kBAAkB,GAAgB;QACxC,WAAW,EAAE,CAAC;QACd,cAAc,EAAE,IAAI;QACpB,UAAU,EAAE,KAAK;QACjB,iBAAiB,EAAE,CAAC;QACpB,eAAe,EAAE;YACf,cAAc;YACd,WAAW;YACX,WAAW;YACX,aAAa;YACb,KAAK;YACL,KAAK;YACL,KAAK;SACN;KACF,CAAC;IAEF,YAAY,MAAe;QACzB,IAAI,CAAC,MAAM,GAAG,MAAM,IAAI,IAAI,MAAM,EAAE,CAAC;QACrC,MAAM,CAAC,IAAI,CAAC,6BAA6B,CAAC,CAAC;IAC7C,CAAC;IAED;;;;;;;OAOG;IACH,KAAK,CAAC,cAAc,CAClB,SAA2B,EAC3B,SAA+B,EAAE,EACjC,gBAAwB,WAAW;QAEnC,MAAM,WAAW,GAAG,EAAE,GAAG,IAAI,CAAC,kBAAkB,EAAE,GAAG,MAAM,EAAE,CAAC;QAC9D,IAAI,SAA4B,CAAC;QACjC,IAAI,OAAO,GAAG,CAAC,CAAC;QAEhB,OAAO,OAAO,GAAG,WAAW,CAAC,WAAW,EAAE,CAAC;YACzC,OAAO,EAAE,CAAC;YAEV,IAAI,CAAC;gBACH,MAAM,CAAC,KAAK,CAAC,cAAc,aAAa,aAAa,OAAO,IAAI,WAAW,CAAC,WAAW,GAAG,CAAC,CAAC;gBAE5F,MAAM,MAAM,GAAG,MAAM,SAAS,EAAE,CAAC;gBAEjC,IAAI,OAAO,GAAG,CAAC,EAAE,CAAC;oBAChB,IAAI,CAAC,KAAK,CAAC,iBAAiB,EAAE,CAAC;oBAC/B,MAAM,CAAC,IAAI,CAAC,GAAG,aAAa,oBAAoB,OAAO,WAAW,CAAC,CAAC;gBACtE,CAAC;gBAED,IAAI,CAAC,KAAK,CAAC,sBAAsB,GAAG,CAAC,CAAC;gBACtC,IAAI,CAAC,KAAK,CAAC,oBAAoB,GAAG,IAAI,IAAI,EAAE,CAAC;gBAE7C,OAAO,MAAM,CAAC;YAEhB,CAAC;YAAC,OAAO,KAAU,EAAE,CAAC;gBACpB,SAAS,GAAG,KAAK,CAAC;gBAClB,IAAI,CAAC,KAAK,CAAC,YAAY,EAAE,CAAC;gBAC1B,IAAI,CAAC,KAAK,CAAC,sBAAsB,EAAE,CAAC;gBACpC,IAAI,CAAC,KAAK,CAAC,kBAAkB,GAAG,IAAI,IAAI,EAAE,CAAC;gBAE3C,MAAM,WAAW,GAAG,IAAI,CAAC,gBAAgB,CAAC,KAAK,EAAE,WAAW,CAAC,CAAC;gBAE9D,IAAI,CAAC,WAAW,IAAI,OAAO,IAAI,WAAW,CAAC,WAAW,EAAE,CAAC;oBACvD,IAAI,CAAC,KAAK,CAAC,aAAa,EAAE,CAAC;oBAC3B,MAAM,CAAC,KAAK,CAAC,GAAG,aAAa,iBAAiB,OAAO,WAAW,EAAE;wBAChE,KAAK,EAAE,KAAK,CAAC,OAAO;wBACpB,SAAS,EAAE,WAAW;qBACvB,CAAC,CAAC;oBACH,MAAM,KAAK,CAAC;gBACd,CAAC;gBAED,MAAM,KAAK,GAAG,IAAI,CAAC,cAAc,CAAC,OAAO,EAAE,WAAW,CAAC,CAAC;gBACxD,MAAM,CAAC,IAAI,CAAC,GAAG,aAAa,wBAAwB,KAAK,IAAI,EAAE;oBAC7D,OAAO;oBACP,WAAW,EAAE,WAAW,CAAC,WAAW;oBACpC,KAAK,EAAE,KAAK,CAAC,OAAO;iBACrB,CAAC,CAAC;gBAEH,MAAM,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;YAC1B,CAAC;QACH,CAAC;QAED,MAAM,SAAS,IAAI,IAAI,KAAK,CAAC,GAAG,aAAa,iBAAiB,WAAW,CAAC,WAAW,WAAW,CAAC,CAAC;IACpG,CAAC;IAED;;;;;;;OAOG;IACH,KAAK,CAAC,oBAAoB,CACxB,SAAuB,EACvB,KAAa,EACb,MAAW,EACX,MAA6B;QAE7B,OAAO,IAAI,CAAC,cAAc,CACxB,GAAG,EAAE,CAAC,SAAS,CAAC,eAAe,CAAC,KAAK,EAAE,MAAM,CAAC,EAC9C,MAAM,EACN,yBAAyB,KAAK,EAAE,CACjC,CAAC;IACJ,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,iBAAiB;QACrB,IAAI,CAAC,KAAK,CAAC,kBAAkB,EAAE,CAAC;QAEhC,IAAI,CAAC;YACH,MAAM,CAAC,KAAK,CAAC,+BAA+B,CAAC,CAAC;YAE9C,qBAAqB;YACrB,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;YAEtC,IAAI,IAAI,CAAC,QAAQ,EAAE,KAAK,IAAI,EAAE,CAAC;gBAC7B,MAAM,CAAC,KAAK,CAAC,0BAA0B,CAAC,CAAC;gBACzC,OAAO,IAAI,CAAC;YACd,CAAC;YAED,MAAM,CAAC,IAAI,CAAC,0CAA0C,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC;YAClE,OAAO,KAAK,CAAC;QAEf,CAAC;QAAC,OAAO,KAAU,EAAE,CAAC;YACpB,MAAM,CAAC,KAAK,CAAC,4BAA4B,EAAE,EAAE,KAAK,EAAE,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;YACrE,OAAO,KAAK,CAAC;QACf,CAAC;IACH,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,uBAAuB,CAAC,cAAsB,CAAC;QACnD,MAAM,CAAC,IAAI,CAAC,uCAAuC,CAAC,CAAC;QAErD,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,WAAW,EAAE,OAAO,EAAE,EAAE,CAAC;YACxD,MAAM,CAAC,IAAI,CAAC,2BAA2B,OAAO,IAAI,WAAW,EAAE,CAAC,CAAC;YAEjE,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,iBAAiB,EAAE,CAAC;YAEjD,IAAI,SAAS,EAAE,CAAC;gBACd,IAAI,CAAC,KAAK,CAAC,gBAAgB,EAAE,CAAC;gBAC9B,MAAM,CAAC,IAAI,CAAC,0CAA0C,CAAC,CAAC;gBACxD,OAAO,IAAI,CAAC;YACd,CAAC;YAED,IAAI,OAAO,GAAG,WAAW,EAAE,CAAC;gBAC1B,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,CAAC,EAAE,KAAK,CAAC,CAAC;gBAC3D,MAAM,CAAC,IAAI,CAAC,WAAW,KAAK,iCAAiC,CAAC,CAAC;gBAC/D,MAAM,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;YAC1B,CAAC;QACH,CAAC;QAED,MAAM,CAAC,KAAK,CAAC,wDAAwD,CAAC,CAAC;QACvE,OAAO,KAAK,CAAC;IACf,CAAC;IAED;;;;;;OAMG;IACH,2BAA2B,CACzB,aAAqB,KAAK,EAC1B,WAAwB;QAExB,MAAM,CAAC,IAAI,CAAC,mCAAmC,EAAE,EAAE,UAAU,EAAE,CAAC,CAAC;QAEjE,OAAO,WAAW,CAAC,KAAK,IAAI,EAAE;YAC5B,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,iBAAiB,EAAE,CAAC;YAEjD,IAAI,CAAC,SAAS,EAAE,CAAC;gBACf,MAAM,CAAC,IAAI,CAAC,iDAAiD,CAAC,CAAC;gBAE/D,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,uBAAuB,EAAE,CAAC;gBAEvD,IAAI,CAAC,SAAS,IAAI,WAAW,EAAE,CAAC;oBAC9B,MAAM,CAAC,KAAK,CAAC,oDAAoD,CAAC,CAAC;oBACnE,WAAW,EAAE,CAAC;gBAChB,CAAC;YACH,CAAC;QACH,CAAC,EAAE,UAAU,CAAC,CAAC;IACjB,CAAC;IAED;;;;;;;;;OASG;IACH,KAAK,CAAC,qBAAqB,CACzB,KAAU,EACV,KAAa,EACb,OAA6B;QAE7B,MAAM,CAAC,IAAI,CAAC,oCAAoC,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;QAEnF,yCAAyC;QACzC,MAAM,aAAa,GAAG,IAAI,CAAC,uBAAuB,CAAC,KAAK,CAAC,CAAC;QAE1D,IAAI,CAAC,aAAa,EAAE,CAAC;YACnB,MAAM,CAAC,IAAI,CAAC,gCAAgC,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC;YACzD,OAAO,KAAK,CAAC;QACf,CAAC;QAED,IAAI,CAAC,OAAO,EAAE,CAAC;YACb,MAAM,CAAC,IAAI,CAAC,6CAA6C,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC;YACtE,OAAO,KAAK,CAAC;QACf,CAAC;QAED,IAAI,CAAC;YACH,MAAM,CAAC,IAAI,CAAC,yBAAyB,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC;YAClD,MAAM,IAAI,CAAC,cAAc,CAAC,OAAO,EAAE,EAAE,EAAE,oBAAoB,KAAK,EAAE,CAAC,CAAC;YACpE,MAAM,CAAC,IAAI,CAAC,yBAAyB,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC;YAClD,OAAO,IAAI,CAAC;QACd,CAAC;QAAC,OAAO,aAAkB,EAAE,CAAC;YAC5B,MAAM,CAAC,KAAK,CAAC,qBAAqB,EAAE;gBAClC,KAAK;gBACL,KAAK,EAAE,aAAa,CAAC,OAAO;aAC7B,CAAC,CAAC;YACH,OAAO,KAAK,CAAC;QACf,CAAC;IACH,CAAC;IAED;;;;OAIG;IACH,QAAQ;QACN,OAAO,EAAE,GAAG,IAAI,CAAC,KAAK,EAAE,CAAC;IAC3B,CAAC;IAED;;OAEG;IACH,UAAU;QACR,IAAI,CAAC,KAAK,GAAG;YACX,YAAY,EAAE,CAAC;YACf,iBAAiB,EAAE,CAAC;YACpB,aAAa,EAAE,CAAC;YAChB,kBAAkB,EAAE,CAAC;YACrB,gBAAgB,EAAE,CAAC;YACnB,sBAAsB,EAAE,CAAC;SAC1B,CAAC;QACF,MAAM,CAAC,IAAI,CAAC,2BAA2B,CAAC,CAAC;IAC3C,CAAC;IAED;;;;;;OAMG;IACK,gBAAgB,CAAC,KAAU,EAAE,MAAmB;QACtD,MAAM,YAAY,GAAG,KAAK,CAAC,OAAO,IAAI,EAAE,CAAC;QACzC,MAAM,SAAS,GAAG,KAAK,CAAC,IAAI,IAAI,EAAE,CAAC;QACnC,MAAM,UAAU,GAAG,KAAK,CAAC,QAAQ,EAAE,MAAM,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;QAE5D,KAAK,MAAM,cAAc,IAAI,MAAM,CAAC,eAAe,IAAI,EAAE,EAAE,CAAC;YAC1D,IACE,YAAY,CAAC,QAAQ,CAAC,cAAc,CAAC;gBACrC,SAAS,KAAK,cAAc;gBAC5B,UAAU,KAAK,cAAc,EAC7B,CAAC;gBACD,OAAO,IAAI,CAAC;YACd,CAAC;QACH,CAAC;QAED,OAAO,KAAK,CAAC;IACf,CAAC;IAED;;;;;OAKG;IACK,uBAAuB,CAAC,KAAU;QACxC,MAAM,mBAAmB,GAAG;YAC1B,cAAc;YACd,WAAW;YACX,+BAA+B;YAC/B,8BAA8B;YAC9B,iBAAiB;YACjB,mBAAmB;SACpB,CAAC;QAEF,MAAM,YAAY,GAAG,CAAC,KAAK,CAAC,OAAO,IAAI,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC;QAEzD,OAAO,mBAAmB,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CACxC,YAAY,CAAC,QAAQ,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC,CAC7C,CAAC;IACJ,CAAC;IAED;;;;;;OAMG;IACK,cAAc,CAAC,OAAe,EAAE,MAAmB;QACzD,MAAM,KAAK,GAAG,MAAM,CAAC,cAAc,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,iBAAiB,EAAE,OAAO,GAAG,CAAC,CAAC,CAAC;QACtF,OAAO,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,MAAM,CAAC,UAAU,CAAC,CAAC;IAC5C,CAAC;IAED;;;;OAIG;IACK,KAAK,CAAC,EAAU;QACtB,OAAO,IAAI,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;IACzD,CAAC;CACF;AAED;;;;;GAKG;AACH,MAAM,UAAU,qBAAqB,CAAC,MAAe;IACnD,OAAO,IAAI,eAAe,CAAC,MAAM,CAAC,CAAC;AACrC,CAAC"}
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* BuildHive API Client
|
|
3
|
+
*
|
|
4
|
+
* HTTP client for communicating with the BuildHive platform REST API.
|
|
5
|
+
* Handles agent registration, authentication, and heartbeat updates.
|
|
6
|
+
*
|
|
7
|
+
* Requirements: MVP.4.1.2
|
|
8
|
+
*/
|
|
9
|
+
import { RegistrationRequest, RegistrationResponse, AuthenticationRequest, AuthenticationResponse } from './types.js';
|
|
10
|
+
export interface HeartbeatRequest {
|
|
11
|
+
agentId: string;
|
|
12
|
+
status: 'ONLINE' | 'OFFLINE' | 'BUSY' | 'MAINTENANCE';
|
|
13
|
+
currentLoad: number;
|
|
14
|
+
activeJobs: number;
|
|
15
|
+
cpuUsage?: number;
|
|
16
|
+
memoryUsage?: number;
|
|
17
|
+
diskUsage?: number;
|
|
18
|
+
}
|
|
19
|
+
export declare class BuildHiveApiClient {
|
|
20
|
+
private client;
|
|
21
|
+
private platformUrl;
|
|
22
|
+
constructor(platformUrl: string);
|
|
23
|
+
/**
|
|
24
|
+
* Register agent with the BuildHive platform
|
|
25
|
+
*/
|
|
26
|
+
register(request: RegistrationRequest): Promise<RegistrationResponse>;
|
|
27
|
+
/**
|
|
28
|
+
* Authenticate agent and get JWT token
|
|
29
|
+
*/
|
|
30
|
+
authenticate(request: AuthenticationRequest): Promise<AuthenticationResponse>;
|
|
31
|
+
/**
|
|
32
|
+
* Send heartbeat to platform
|
|
33
|
+
*/
|
|
34
|
+
sendHeartbeat(request: HeartbeatRequest, apiKey: string): Promise<void>;
|
|
35
|
+
/**
|
|
36
|
+
* Test connection to platform
|
|
37
|
+
*/
|
|
38
|
+
testConnection(): Promise<boolean>;
|
|
39
|
+
/**
|
|
40
|
+
* Get platform URL
|
|
41
|
+
*/
|
|
42
|
+
getPlatformUrl(): string;
|
|
43
|
+
}
|
|
44
|
+
//# sourceMappingURL=apiClient.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"apiClient.d.ts","sourceRoot":"","sources":["../../src/registration/apiClient.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAIH,OAAO,EACL,mBAAmB,EACnB,oBAAoB,EACpB,qBAAqB,EACrB,sBAAsB,EACvB,MAAM,YAAY,CAAC;AAIpB,MAAM,WAAW,gBAAgB;IAC/B,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,QAAQ,GAAG,SAAS,GAAG,MAAM,GAAG,aAAa,CAAC;IACtD,WAAW,EAAE,MAAM,CAAC;IACpB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,qBAAa,kBAAkB;IAC7B,OAAO,CAAC,MAAM,CAAgB;IAC9B,OAAO,CAAC,WAAW,CAAS;gBAEhB,WAAW,EAAE,MAAM;IA0C/B;;OAEG;IACG,QAAQ,CAAC,OAAO,EAAE,mBAAmB,GAAG,OAAO,CAAC,oBAAoB,CAAC;IAsC3E;;OAEG;IACG,YAAY,CAAC,OAAO,EAAE,qBAAqB,GAAG,OAAO,CAAC,sBAAsB,CAAC;IAiCnF;;OAEG;IACG,aAAa,CAAC,OAAO,EAAE,gBAAgB,EAAE,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAoB7E;;OAEG;IACG,cAAc,IAAI,OAAO,CAAC,OAAO,CAAC;IAWxC;;OAEG;IACH,cAAc,IAAI,MAAM;CAGzB"}
|
|
@@ -0,0 +1,149 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* BuildHive API Client
|
|
3
|
+
*
|
|
4
|
+
* HTTP client for communicating with the BuildHive platform REST API.
|
|
5
|
+
* Handles agent registration, authentication, and heartbeat updates.
|
|
6
|
+
*
|
|
7
|
+
* Requirements: MVP.4.1.2
|
|
8
|
+
*/
|
|
9
|
+
import axios from 'axios';
|
|
10
|
+
import { createLogger } from '../utils/logger.js';
|
|
11
|
+
const logger = createLogger('apiClient');
|
|
12
|
+
export class BuildHiveApiClient {
|
|
13
|
+
client;
|
|
14
|
+
platformUrl;
|
|
15
|
+
constructor(platformUrl) {
|
|
16
|
+
this.platformUrl = platformUrl;
|
|
17
|
+
this.client = axios.create({
|
|
18
|
+
baseURL: platformUrl,
|
|
19
|
+
timeout: 30000,
|
|
20
|
+
headers: {
|
|
21
|
+
'Content-Type': 'application/json',
|
|
22
|
+
'User-Agent': 'BuildHive-Agent/1.0.0',
|
|
23
|
+
},
|
|
24
|
+
});
|
|
25
|
+
// Add request interceptor for logging
|
|
26
|
+
this.client.interceptors.request.use((config) => {
|
|
27
|
+
logger.debug(`API Request: ${config.method?.toUpperCase()} ${config.url}`);
|
|
28
|
+
return config;
|
|
29
|
+
}, (error) => {
|
|
30
|
+
logger.error('API Request Error:', error);
|
|
31
|
+
return Promise.reject(error);
|
|
32
|
+
});
|
|
33
|
+
// Add response interceptor for logging
|
|
34
|
+
this.client.interceptors.response.use((response) => {
|
|
35
|
+
logger.debug(`API Response: ${response.status} ${response.config.url}`);
|
|
36
|
+
return response;
|
|
37
|
+
}, (error) => {
|
|
38
|
+
if (axios.isAxiosError(error)) {
|
|
39
|
+
logger.error(`API Error: ${error.response?.status} ${error.config?.url} - ${error.response?.data?.message || error.message}`);
|
|
40
|
+
}
|
|
41
|
+
return Promise.reject(error);
|
|
42
|
+
});
|
|
43
|
+
}
|
|
44
|
+
/**
|
|
45
|
+
* Register agent with the BuildHive platform
|
|
46
|
+
*/
|
|
47
|
+
async register(request) {
|
|
48
|
+
logger.info('Registering agent with BuildHive platform');
|
|
49
|
+
try {
|
|
50
|
+
const response = await this.client.post('/api/agents/register', request);
|
|
51
|
+
logger.info('Agent registered successfully', {
|
|
52
|
+
agentId: response.data.agentId,
|
|
53
|
+
expiresAt: response.data.expiresAt,
|
|
54
|
+
});
|
|
55
|
+
return response.data;
|
|
56
|
+
}
|
|
57
|
+
catch (error) {
|
|
58
|
+
if (axios.isAxiosError(error)) {
|
|
59
|
+
const message = error.response?.data?.message || error.message;
|
|
60
|
+
const statusCode = error.response?.status;
|
|
61
|
+
logger.error(`Registration failed [${statusCode}]: ${message}`);
|
|
62
|
+
if (statusCode === 409) {
|
|
63
|
+
throw new Error('Agent already registered with this machine ID');
|
|
64
|
+
}
|
|
65
|
+
else if (statusCode === 400) {
|
|
66
|
+
throw new Error(`Invalid registration data: ${message}`);
|
|
67
|
+
}
|
|
68
|
+
else if (statusCode === 401 || statusCode === 403) {
|
|
69
|
+
throw new Error('Authentication failed - check your API credentials');
|
|
70
|
+
}
|
|
71
|
+
else if (statusCode && statusCode >= 500) {
|
|
72
|
+
throw new Error('Server error during registration - please try again later');
|
|
73
|
+
}
|
|
74
|
+
throw new Error(`Registration failed: ${message}`);
|
|
75
|
+
}
|
|
76
|
+
throw error;
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
/**
|
|
80
|
+
* Authenticate agent and get JWT token
|
|
81
|
+
*/
|
|
82
|
+
async authenticate(request) {
|
|
83
|
+
logger.info('Authenticating agent');
|
|
84
|
+
try {
|
|
85
|
+
const response = await this.client.post('/api/agents/auth', request);
|
|
86
|
+
logger.info('Agent authenticated successfully', {
|
|
87
|
+
expiresAt: response.data.expiresAt,
|
|
88
|
+
});
|
|
89
|
+
return response.data;
|
|
90
|
+
}
|
|
91
|
+
catch (error) {
|
|
92
|
+
if (axios.isAxiosError(error)) {
|
|
93
|
+
const message = error.response?.data?.message || error.message;
|
|
94
|
+
const statusCode = error.response?.status;
|
|
95
|
+
logger.error(`Authentication failed [${statusCode}]: ${message}`);
|
|
96
|
+
if (statusCode === 401) {
|
|
97
|
+
throw new Error('Invalid agent ID or API key');
|
|
98
|
+
}
|
|
99
|
+
else if (statusCode === 404) {
|
|
100
|
+
throw new Error('Agent not found - please register first');
|
|
101
|
+
}
|
|
102
|
+
throw new Error(`Authentication failed: ${message}`);
|
|
103
|
+
}
|
|
104
|
+
throw error;
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
/**
|
|
108
|
+
* Send heartbeat to platform
|
|
109
|
+
*/
|
|
110
|
+
async sendHeartbeat(request, apiKey) {
|
|
111
|
+
logger.debug('Sending heartbeat to platform');
|
|
112
|
+
try {
|
|
113
|
+
await this.client.put('/api/agents/heartbeat', request, {
|
|
114
|
+
headers: {
|
|
115
|
+
Authorization: `Bearer ${apiKey}`,
|
|
116
|
+
},
|
|
117
|
+
});
|
|
118
|
+
logger.debug('Heartbeat sent successfully');
|
|
119
|
+
}
|
|
120
|
+
catch (error) {
|
|
121
|
+
if (axios.isAxiosError(error)) {
|
|
122
|
+
const message = error.response?.data?.message || error.message;
|
|
123
|
+
logger.error(`Heartbeat failed: ${message}`);
|
|
124
|
+
// Don't throw - heartbeat failures should be handled gracefully
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
}
|
|
128
|
+
/**
|
|
129
|
+
* Test connection to platform
|
|
130
|
+
*/
|
|
131
|
+
async testConnection() {
|
|
132
|
+
try {
|
|
133
|
+
await this.client.get('/health');
|
|
134
|
+
logger.info('Platform connection test successful');
|
|
135
|
+
return true;
|
|
136
|
+
}
|
|
137
|
+
catch (error) {
|
|
138
|
+
logger.error('Platform connection test failed:', error);
|
|
139
|
+
return false;
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
/**
|
|
143
|
+
* Get platform URL
|
|
144
|
+
*/
|
|
145
|
+
getPlatformUrl() {
|
|
146
|
+
return this.platformUrl;
|
|
147
|
+
}
|
|
148
|
+
}
|
|
149
|
+
//# sourceMappingURL=apiClient.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"apiClient.js","sourceRoot":"","sources":["../../src/registration/apiClient.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,KAAwB,MAAM,OAAO,CAAC;AAC7C,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAQlD,MAAM,MAAM,GAAG,YAAY,CAAC,WAAW,CAAC,CAAC;AAYzC,MAAM,OAAO,kBAAkB;IACrB,MAAM,CAAgB;IACtB,WAAW,CAAS;IAE5B,YAAY,WAAmB;QAC7B,IAAI,CAAC,WAAW,GAAG,WAAW,CAAC;QAC/B,IAAI,CAAC,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC;YACzB,OAAO,EAAE,WAAW;YACpB,OAAO,EAAE,KAAK;YACd,OAAO,EAAE;gBACP,cAAc,EAAE,kBAAkB;gBAClC,YAAY,EAAE,uBAAuB;aACtC;SACF,CAAC,CAAC;QAEH,sCAAsC;QACtC,IAAI,CAAC,MAAM,CAAC,YAAY,CAAC,OAAO,CAAC,GAAG,CAClC,CAAC,MAAM,EAAE,EAAE;YACT,MAAM,CAAC,KAAK,CAAC,gBAAgB,MAAM,CAAC,MAAM,EAAE,WAAW,EAAE,IAAI,MAAM,CAAC,GAAG,EAAE,CAAC,CAAC;YAC3E,OAAO,MAAM,CAAC;QAChB,CAAC,EACD,CAAC,KAAK,EAAE,EAAE;YACR,MAAM,CAAC,KAAK,CAAC,oBAAoB,EAAE,KAAK,CAAC,CAAC;YAC1C,OAAO,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QAC/B,CAAC,CACF,CAAC;QAEF,uCAAuC;QACvC,IAAI,CAAC,MAAM,CAAC,YAAY,CAAC,QAAQ,CAAC,GAAG,CACnC,CAAC,QAAQ,EAAE,EAAE;YACX,MAAM,CAAC,KAAK,CAAC,iBAAiB,QAAQ,CAAC,MAAM,IAAI,QAAQ,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC,CAAC;YACxE,OAAO,QAAQ,CAAC;QAClB,CAAC,EACD,CAAC,KAAK,EAAE,EAAE;YACR,IAAI,KAAK,CAAC,YAAY,CAAC,KAAK,CAAC,EAAE,CAAC;gBAC9B,MAAM,CAAC,KAAK,CACV,cAAc,KAAK,CAAC,QAAQ,EAAE,MAAM,IAAI,KAAK,CAAC,MAAM,EAAE,GAAG,MACvD,KAAK,CAAC,QAAQ,EAAE,IAAI,EAAE,OAAO,IAAI,KAAK,CAAC,OACzC,EAAE,CACH,CAAC;YACJ,CAAC;YACD,OAAO,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QAC/B,CAAC,CACF,CAAC;IACJ,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,QAAQ,CAAC,OAA4B;QACzC,MAAM,CAAC,IAAI,CAAC,2CAA2C,CAAC,CAAC;QAEzD,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,IAAI,CACrC,sBAAsB,EACtB,OAAO,CACR,CAAC;YAEF,MAAM,CAAC,IAAI,CAAC,+BAA+B,EAAE;gBAC3C,OAAO,EAAE,QAAQ,CAAC,IAAI,CAAC,OAAO;gBAC9B,SAAS,EAAE,QAAQ,CAAC,IAAI,CAAC,SAAS;aACnC,CAAC,CAAC;YAEH,OAAO,QAAQ,CAAC,IAAI,CAAC;QACvB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,KAAK,CAAC,YAAY,CAAC,KAAK,CAAC,EAAE,CAAC;gBAC9B,MAAM,OAAO,GAAG,KAAK,CAAC,QAAQ,EAAE,IAAI,EAAE,OAAO,IAAI,KAAK,CAAC,OAAO,CAAC;gBAC/D,MAAM,UAAU,GAAG,KAAK,CAAC,QAAQ,EAAE,MAAM,CAAC;gBAC1C,MAAM,CAAC,KAAK,CAAC,wBAAwB,UAAU,MAAM,OAAO,EAAE,CAAC,CAAC;gBAEhE,IAAI,UAAU,KAAK,GAAG,EAAE,CAAC;oBACvB,MAAM,IAAI,KAAK,CAAC,+CAA+C,CAAC,CAAC;gBACnE,CAAC;qBAAM,IAAI,UAAU,KAAK,GAAG,EAAE,CAAC;oBAC9B,MAAM,IAAI,KAAK,CAAC,8BAA8B,OAAO,EAAE,CAAC,CAAC;gBAC3D,CAAC;qBAAM,IAAI,UAAU,KAAK,GAAG,IAAI,UAAU,KAAK,GAAG,EAAE,CAAC;oBACpD,MAAM,IAAI,KAAK,CAAC,oDAAoD,CAAC,CAAC;gBACxE,CAAC;qBAAM,IAAI,UAAU,IAAI,UAAU,IAAI,GAAG,EAAE,CAAC;oBAC3C,MAAM,IAAI,KAAK,CAAC,2DAA2D,CAAC,CAAC;gBAC/E,CAAC;gBAED,MAAM,IAAI,KAAK,CAAC,wBAAwB,OAAO,EAAE,CAAC,CAAC;YACrD,CAAC;YAED,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,YAAY,CAAC,OAA8B;QAC/C,MAAM,CAAC,IAAI,CAAC,sBAAsB,CAAC,CAAC;QAEpC,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,IAAI,CACrC,kBAAkB,EAClB,OAAO,CACR,CAAC;YAEF,MAAM,CAAC,IAAI,CAAC,kCAAkC,EAAE;gBAC9C,SAAS,EAAE,QAAQ,CAAC,IAAI,CAAC,SAAS;aACnC,CAAC,CAAC;YAEH,OAAO,QAAQ,CAAC,IAAI,CAAC;QACvB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,KAAK,CAAC,YAAY,CAAC,KAAK,CAAC,EAAE,CAAC;gBAC9B,MAAM,OAAO,GAAG,KAAK,CAAC,QAAQ,EAAE,IAAI,EAAE,OAAO,IAAI,KAAK,CAAC,OAAO,CAAC;gBAC/D,MAAM,UAAU,GAAG,KAAK,CAAC,QAAQ,EAAE,MAAM,CAAC;gBAC1C,MAAM,CAAC,KAAK,CAAC,0BAA0B,UAAU,MAAM,OAAO,EAAE,CAAC,CAAC;gBAElE,IAAI,UAAU,KAAK,GAAG,EAAE,CAAC;oBACvB,MAAM,IAAI,KAAK,CAAC,6BAA6B,CAAC,CAAC;gBACjD,CAAC;qBAAM,IAAI,UAAU,KAAK,GAAG,EAAE,CAAC;oBAC9B,MAAM,IAAI,KAAK,CAAC,yCAAyC,CAAC,CAAC;gBAC7D,CAAC;gBAED,MAAM,IAAI,KAAK,CAAC,0BAA0B,OAAO,EAAE,CAAC,CAAC;YACvD,CAAC;YAED,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,aAAa,CAAC,OAAyB,EAAE,MAAc;QAC3D,MAAM,CAAC,KAAK,CAAC,+BAA+B,CAAC,CAAC;QAE9C,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,uBAAuB,EAAE,OAAO,EAAE;gBACtD,OAAO,EAAE;oBACP,aAAa,EAAE,UAAU,MAAM,EAAE;iBAClC;aACF,CAAC,CAAC;YAEH,MAAM,CAAC,KAAK,CAAC,6BAA6B,CAAC,CAAC;QAC9C,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,KAAK,CAAC,YAAY,CAAC,KAAK,CAAC,EAAE,CAAC;gBAC9B,MAAM,OAAO,GAAG,KAAK,CAAC,QAAQ,EAAE,IAAI,EAAE,OAAO,IAAI,KAAK,CAAC,OAAO,CAAC;gBAC/D,MAAM,CAAC,KAAK,CAAC,qBAAqB,OAAO,EAAE,CAAC,CAAC;gBAC7C,gEAAgE;YAClE,CAAC;QACH,CAAC;IACH,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,cAAc;QAClB,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;YACjC,MAAM,CAAC,IAAI,CAAC,qCAAqC,CAAC,CAAC;YACnD,OAAO,IAAI,CAAC;QACd,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,CAAC,KAAK,CAAC,kCAAkC,EAAE,KAAK,CAAC,CAAC;YACxD,OAAO,KAAK,CAAC;QACf,CAAC;IACH,CAAC;IAED;;OAEG;IACH,cAAc;QACZ,OAAO,IAAI,CAAC,WAAW,CAAC;IAC1B,CAAC;CACF"}
|