agent-web-interface 4.1.0 → 4.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/browser/session-manager.d.ts +28 -1
- package/dist/src/browser/session-manager.d.ts.map +1 -1
- package/dist/src/browser/session-manager.js +73 -0
- package/dist/src/browser/session-manager.js.map +1 -1
- package/dist/src/index.d.ts +10 -1
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/index.js +47 -0
- package/dist/src/index.js.map +1 -1
- package/dist/src/server/mcp-server.d.ts +14 -1
- package/dist/src/server/mcp-server.d.ts.map +1 -1
- package/dist/src/server/mcp-server.js +21 -6
- package/dist/src/server/mcp-server.js.map +1 -1
- package/dist/src/server/session-store.d.ts +18 -3
- package/dist/src/server/session-store.d.ts.map +1 -1
- package/dist/src/server/session-store.js +27 -3
- package/dist/src/server/session-store.js.map +1 -1
- package/dist/src/session/session-worker-binding.d.ts +57 -0
- package/dist/src/session/session-worker-binding.d.ts.map +1 -0
- package/dist/src/session/session-worker-binding.js +109 -0
- package/dist/src/session/session-worker-binding.js.map +1 -0
- package/dist/src/shared/services/logging.service.d.ts +8 -0
- package/dist/src/shared/services/logging.service.d.ts.map +1 -1
- package/dist/src/shared/services/logging.service.js +10 -0
- package/dist/src/shared/services/logging.service.js.map +1 -1
- package/dist/src/snapshot/snapshot-compiler.d.ts +14 -0
- package/dist/src/snapshot/snapshot-compiler.d.ts.map +1 -1
- package/dist/src/snapshot/snapshot-compiler.js +119 -3
- package/dist/src/snapshot/snapshot-compiler.js.map +1 -1
- package/dist/src/state/actionables-filter.d.ts.map +1 -1
- package/dist/src/state/actionables-filter.js +8 -30
- package/dist/src/state/actionables-filter.js.map +1 -1
- package/dist/src/state/locator-generator.d.ts.map +1 -1
- package/dist/src/state/locator-generator.js +1 -10
- package/dist/src/state/locator-generator.js.map +1 -1
- package/dist/src/state/node-layer.d.ts +22 -0
- package/dist/src/state/node-layer.d.ts.map +1 -0
- package/dist/src/state/node-layer.js +42 -0
- package/dist/src/state/node-layer.js.map +1 -0
- package/dist/src/state/state-manager.d.ts.map +1 -1
- package/dist/src/state/state-manager.js +9 -8
- package/dist/src/state/state-manager.js.map +1 -1
- package/dist/src/tools/tool-schemas.d.ts +22 -22
- package/dist/src/tools/tool-schemas.d.ts.map +1 -1
- package/dist/src/tools/tool-schemas.js.map +1 -1
- package/dist/src/worker/chrome-worker-process.d.ts +123 -0
- package/dist/src/worker/chrome-worker-process.d.ts.map +1 -0
- package/dist/src/worker/chrome-worker-process.js +294 -0
- package/dist/src/worker/chrome-worker-process.js.map +1 -0
- package/dist/src/worker/errors/index.d.ts +5 -0
- package/dist/src/worker/errors/index.d.ts.map +1 -0
- package/dist/src/worker/errors/index.js +5 -0
- package/dist/src/worker/errors/index.js.map +1 -0
- package/dist/src/worker/errors/worker.error.d.ts +122 -0
- package/dist/src/worker/errors/worker.error.d.ts.map +1 -0
- package/dist/src/worker/errors/worker.error.js +199 -0
- package/dist/src/worker/errors/worker.error.js.map +1 -0
- package/dist/src/worker/health-monitor.d.ts +141 -0
- package/dist/src/worker/health-monitor.d.ts.map +1 -0
- package/dist/src/worker/health-monitor.js +260 -0
- package/dist/src/worker/health-monitor.js.map +1 -0
- package/dist/src/worker/index.d.ts +16 -0
- package/dist/src/worker/index.d.ts.map +1 -0
- package/dist/src/worker/index.js +19 -0
- package/dist/src/worker/index.js.map +1 -0
- package/dist/src/worker/lease-manager.d.ts +137 -0
- package/dist/src/worker/lease-manager.d.ts.map +1 -0
- package/dist/src/worker/lease-manager.js +334 -0
- package/dist/src/worker/lease-manager.js.map +1 -0
- package/dist/src/worker/multi-tenant-config.d.ts +46 -0
- package/dist/src/worker/multi-tenant-config.d.ts.map +1 -0
- package/dist/src/worker/multi-tenant-config.js +94 -0
- package/dist/src/worker/multi-tenant-config.js.map +1 -0
- package/dist/src/worker/port-allocator.d.ts +96 -0
- package/dist/src/worker/port-allocator.d.ts.map +1 -0
- package/dist/src/worker/port-allocator.js +153 -0
- package/dist/src/worker/port-allocator.js.map +1 -0
- package/dist/src/worker/types.d.ts +218 -0
- package/dist/src/worker/types.d.ts.map +1 -0
- package/dist/src/worker/types.js +38 -0
- package/dist/src/worker/types.js.map +1 -0
- package/dist/src/worker/worker-manager.d.ts +157 -0
- package/dist/src/worker/worker-manager.d.ts.map +1 -0
- package/dist/src/worker/worker-manager.js +500 -0
- package/dist/src/worker/worker-manager.js.map +1 -0
- package/package.json +1 -1
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"worker-manager.d.ts","sourceRoot":"","sources":["../../../src/worker/worker-manager.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,EAAE,YAAY,EAAE,MAAM,QAAQ,CAAC;AAGtC,OAAO,KAAK,EACV,mBAAmB,EACnB,gBAAgB,EAChB,sBAAsB,EACtB,mBAAmB,EACpB,MAAM,YAAY,CAAC;AAUpB;;GAEG;AACH,UAAU,oBAAoB;IAC5B,EAAE,CAAC,CAAC,SAAS,MAAM,mBAAmB,EACpC,KAAK,EAAE,CAAC,EACR,QAAQ,EAAE,CAAC,IAAI,EAAE,mBAAmB,CAAC,CAAC,CAAC,KAAK,IAAI,GAC/C,IAAI,CAAC;IACR,IAAI,CAAC,CAAC,SAAS,MAAM,mBAAmB,EACtC,KAAK,EAAE,CAAC,EACR,QAAQ,EAAE,CAAC,IAAI,EAAE,mBAAmB,CAAC,CAAC,CAAC,KAAK,IAAI,GAC/C,IAAI,CAAC;IACR,IAAI,CAAC,CAAC,SAAS,MAAM,mBAAmB,EAAE,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,mBAAmB,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC;IAC3F,GAAG,CAAC,CAAC,SAAS,MAAM,mBAAmB,EACrC,KAAK,EAAE,CAAC,EACR,QAAQ,EAAE,CAAC,IAAI,EAAE,mBAAmB,CAAC,CAAC,CAAC,KAAK,IAAI,GAC/C,IAAI,CAAC;IACR,kBAAkB,CAAC,CAAC,SAAS,MAAM,mBAAmB,EAAE,KAAK,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC;CAC1E;AAaD;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AACH,qBAAa,aAAc,SAAQ,YAAa,YAAW,oBAAoB;IAC7E,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAsB;IAC7C,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAgB;IAC9C,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAgB;IAC9C,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAe;IAE5C,kCAAkC;IAClC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAkC;IAC1D,kCAAkC;IAClC,OAAO,CAAC,QAAQ,CAAC,cAAc,CAA6B;IAE5D,OAAO,CAAC,eAAe,CAAS;gBAEpB,MAAM,EAAE,mBAAmB;IAkCvC;;OAEG;IACH,IAAI,WAAW,IAAI,MAAM,CAExB;IAED;;OAEG;IACH,IAAI,UAAU,IAAI,MAAM,CAEvB;IAED;;OAEG;IACH,IAAI,cAAc,IAAI,OAAO,CAE5B;IAED;;;;;;;;OAQG;IACG,gBAAgB,CACpB,QAAQ,EAAE,MAAM,EAChB,YAAY,EAAE,MAAM,EACpB,KAAK,CAAC,EAAE,MAAM,GACb,OAAO,CAAC,sBAAsB,CAAC;IA8ElC;;;;;;OAMG;IACH,YAAY,CAAC,QAAQ,EAAE,MAAM,EAAE,YAAY,CAAC,EAAE,MAAM,GAAG,OAAO;IAuB9D;;;;;;OAMG;IACH,YAAY,CAAC,QAAQ,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,MAAM,GAAG,OAAO;IAqBvD;;;;;OAKG;IACH,cAAc,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS;IAQpD;;OAEG;IACH,cAAc,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO;IAIzC;;OAEG;IACH,aAAa,CAAC,QAAQ,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,GAAG,OAAO;IAI9D;;OAEG;IACH,kBAAkB,CAAC,QAAQ,EAAE,MAAM,GAAG,gBAAgB,GAAG,SAAS;IAOlE;;;;;OAKG;IACG,UAAU,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAOjE;;OAEG;IACG,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC;IAsB/B;;OAEG;YACW,YAAY;IAwF1B;;OAEG;YACW,aAAa;IA4C3B;;OAEG;IACH,OAAO,CAAC,kBAAkB;IAuC1B;;OAEG;IACH,OAAO,CAAC,0BAA0B;IA6BlC;;OAEG;IACH,OAAO,CAAC,cAAc;IAetB;;OAEG;IACH,OAAO,CAAC,cAAc;IAQtB;;OAEG;IACH,OAAO,CAAC,iBAAiB;CAS1B"}
|
|
@@ -0,0 +1,500 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Worker Manager
|
|
3
|
+
*
|
|
4
|
+
* Main orchestrator for managing tenant-bound Chrome workers with exclusive
|
|
5
|
+
* lease-based access control. Coordinates worker lifecycle, health monitoring,
|
|
6
|
+
* and lease management.
|
|
7
|
+
*/
|
|
8
|
+
import { EventEmitter } from 'events';
|
|
9
|
+
import { randomUUID } from 'crypto';
|
|
10
|
+
import { join } from 'path';
|
|
11
|
+
import { PortAllocator } from './port-allocator.js';
|
|
12
|
+
import { ChromeWorkerProcess } from './chrome-worker-process.js';
|
|
13
|
+
import { HealthMonitor } from './health-monitor.js';
|
|
14
|
+
import { LeaseManager } from './lease-manager.js';
|
|
15
|
+
import { WorkerError } from './errors/index.js';
|
|
16
|
+
import { createLogger } from '../shared/services/logging.service.js';
|
|
17
|
+
const logger = createLogger('WorkerManager');
|
|
18
|
+
/**
|
|
19
|
+
* Manages tenant-bound Chrome workers with exclusive lease-based access.
|
|
20
|
+
*
|
|
21
|
+
* @example
|
|
22
|
+
* ```typescript
|
|
23
|
+
* const manager = new WorkerManager({
|
|
24
|
+
* profileBaseDir: '/var/lib/athena/profiles',
|
|
25
|
+
* idleTimeoutMs: 300000,
|
|
26
|
+
* hardTtlMs: 7200000,
|
|
27
|
+
* leaseTtlMs: 300000,
|
|
28
|
+
* healthCheckIntervalMs: 30000,
|
|
29
|
+
* portRange: { min: 9300, max: 9399 },
|
|
30
|
+
* maxWorkers: 100,
|
|
31
|
+
* });
|
|
32
|
+
*
|
|
33
|
+
* // Acquire a worker for a tenant
|
|
34
|
+
* const result = await manager.acquireForTenant('tenant-a', 'controller-1');
|
|
35
|
+
* if (result.success) {
|
|
36
|
+
* console.log(`CDP endpoint: ${result.cdpEndpoint}`);
|
|
37
|
+
* }
|
|
38
|
+
*
|
|
39
|
+
* // Release when done
|
|
40
|
+
* manager.releaseLease('tenant-a');
|
|
41
|
+
*
|
|
42
|
+
* // Shutdown all workers
|
|
43
|
+
* await manager.shutdown();
|
|
44
|
+
* ```
|
|
45
|
+
*/
|
|
46
|
+
export class WorkerManager extends EventEmitter {
|
|
47
|
+
config;
|
|
48
|
+
portAllocator;
|
|
49
|
+
healthMonitor;
|
|
50
|
+
leaseManager;
|
|
51
|
+
/** Worker entries by worker ID */
|
|
52
|
+
workers = new Map();
|
|
53
|
+
/** Tenant to worker ID mapping */
|
|
54
|
+
tenantToWorker = new Map();
|
|
55
|
+
_isShuttingDown = false;
|
|
56
|
+
constructor(config) {
|
|
57
|
+
super();
|
|
58
|
+
this.config = config;
|
|
59
|
+
// Initialize sub-components
|
|
60
|
+
this.portAllocator = new PortAllocator({
|
|
61
|
+
min: config.portRange.min,
|
|
62
|
+
max: config.portRange.max,
|
|
63
|
+
});
|
|
64
|
+
this.healthMonitor = new HealthMonitor(config.healthCheckIntervalMs, {
|
|
65
|
+
timeoutMs: 5000,
|
|
66
|
+
failureThreshold: 3,
|
|
67
|
+
});
|
|
68
|
+
this.leaseManager = new LeaseManager({
|
|
69
|
+
defaultTtlMs: config.leaseTtlMs,
|
|
70
|
+
cleanupIntervalMs: Math.min(config.leaseTtlMs / 2, 60000),
|
|
71
|
+
});
|
|
72
|
+
// Set up event handlers
|
|
73
|
+
this.setupEventHandlers();
|
|
74
|
+
// Start health monitoring
|
|
75
|
+
this.healthMonitor.start();
|
|
76
|
+
logger.info('WorkerManager initialized', {
|
|
77
|
+
profileBaseDir: config.profileBaseDir,
|
|
78
|
+
portRange: config.portRange,
|
|
79
|
+
maxWorkers: config.maxWorkers,
|
|
80
|
+
});
|
|
81
|
+
}
|
|
82
|
+
/**
|
|
83
|
+
* Get current worker count
|
|
84
|
+
*/
|
|
85
|
+
get workerCount() {
|
|
86
|
+
return this.workers.size;
|
|
87
|
+
}
|
|
88
|
+
/**
|
|
89
|
+
* Get current lease count
|
|
90
|
+
*/
|
|
91
|
+
get leaseCount() {
|
|
92
|
+
return this.leaseManager.leaseCount;
|
|
93
|
+
}
|
|
94
|
+
/**
|
|
95
|
+
* Check if shutdown is in progress
|
|
96
|
+
*/
|
|
97
|
+
get isShuttingDown() {
|
|
98
|
+
return this._isShuttingDown;
|
|
99
|
+
}
|
|
100
|
+
/**
|
|
101
|
+
* Acquire a worker for a tenant.
|
|
102
|
+
* Creates a new worker if one doesn't exist for the tenant.
|
|
103
|
+
*
|
|
104
|
+
* @param tenantId - Tenant identifier
|
|
105
|
+
* @param controllerId - Controller/session identifier
|
|
106
|
+
* @param ttlMs - Optional custom lease TTL
|
|
107
|
+
* @returns Lease acquisition result with CDP endpoint
|
|
108
|
+
*/
|
|
109
|
+
async acquireForTenant(tenantId, controllerId, ttlMs) {
|
|
110
|
+
if (this._isShuttingDown) {
|
|
111
|
+
return {
|
|
112
|
+
success: false,
|
|
113
|
+
error: 'WorkerManager is shutting down',
|
|
114
|
+
errorCode: 'OPERATION_FAILED',
|
|
115
|
+
};
|
|
116
|
+
}
|
|
117
|
+
// Check if tenant already has a worker
|
|
118
|
+
const workerId = this.tenantToWorker.get(tenantId);
|
|
119
|
+
if (workerId) {
|
|
120
|
+
const entry = this.workers.get(workerId);
|
|
121
|
+
if (entry?.descriptor.state === 'running') {
|
|
122
|
+
// Try to acquire lease on existing worker
|
|
123
|
+
const leaseResult = this.leaseManager.acquire(tenantId, controllerId, workerId, ttlMs);
|
|
124
|
+
if (leaseResult.success) {
|
|
125
|
+
this.resetIdleTimer(entry);
|
|
126
|
+
this.emit('leaseAcquired', {
|
|
127
|
+
leaseId: leaseResult.lease.leaseId,
|
|
128
|
+
tenantId,
|
|
129
|
+
controllerId,
|
|
130
|
+
});
|
|
131
|
+
return {
|
|
132
|
+
...leaseResult,
|
|
133
|
+
cdpEndpoint: entry.descriptor.cdpEndpoint,
|
|
134
|
+
};
|
|
135
|
+
}
|
|
136
|
+
return leaseResult;
|
|
137
|
+
}
|
|
138
|
+
// Worker exists but not running - clean it up
|
|
139
|
+
await this.cleanupWorker(workerId, 'not running');
|
|
140
|
+
}
|
|
141
|
+
// Create new worker for tenant
|
|
142
|
+
try {
|
|
143
|
+
const newWorkerId = await this.createWorker(tenantId);
|
|
144
|
+
// Acquire lease on new worker
|
|
145
|
+
const leaseResult = this.leaseManager.acquire(tenantId, controllerId, newWorkerId, ttlMs);
|
|
146
|
+
if (leaseResult.success) {
|
|
147
|
+
const entry = this.workers.get(newWorkerId);
|
|
148
|
+
this.emit('leaseAcquired', {
|
|
149
|
+
leaseId: leaseResult.lease.leaseId,
|
|
150
|
+
tenantId,
|
|
151
|
+
controllerId,
|
|
152
|
+
});
|
|
153
|
+
return {
|
|
154
|
+
...leaseResult,
|
|
155
|
+
cdpEndpoint: entry.descriptor.cdpEndpoint,
|
|
156
|
+
};
|
|
157
|
+
}
|
|
158
|
+
return leaseResult;
|
|
159
|
+
}
|
|
160
|
+
catch (error) {
|
|
161
|
+
const workerError = error instanceof WorkerError
|
|
162
|
+
? error
|
|
163
|
+
: WorkerError.operationFailed('acquire worker', error);
|
|
164
|
+
return {
|
|
165
|
+
success: false,
|
|
166
|
+
error: workerError.message,
|
|
167
|
+
errorCode: workerError.code,
|
|
168
|
+
};
|
|
169
|
+
}
|
|
170
|
+
}
|
|
171
|
+
/**
|
|
172
|
+
* Release a tenant's lease.
|
|
173
|
+
*
|
|
174
|
+
* @param tenantId - Tenant identifier
|
|
175
|
+
* @param controllerId - Optional controller ID to validate ownership
|
|
176
|
+
* @returns true if lease was released
|
|
177
|
+
*/
|
|
178
|
+
releaseLease(tenantId, controllerId) {
|
|
179
|
+
const lease = this.leaseManager.getLease(tenantId);
|
|
180
|
+
const released = this.leaseManager.release(tenantId, controllerId);
|
|
181
|
+
if (released && lease) {
|
|
182
|
+
this.emit('leaseReleased', {
|
|
183
|
+
leaseId: lease.leaseId,
|
|
184
|
+
tenantId,
|
|
185
|
+
});
|
|
186
|
+
// Start idle timer for the worker
|
|
187
|
+
const workerId = this.tenantToWorker.get(tenantId);
|
|
188
|
+
if (workerId) {
|
|
189
|
+
const entry = this.workers.get(workerId);
|
|
190
|
+
if (entry) {
|
|
191
|
+
this.startIdleTimer(entry);
|
|
192
|
+
}
|
|
193
|
+
}
|
|
194
|
+
}
|
|
195
|
+
return released;
|
|
196
|
+
}
|
|
197
|
+
/**
|
|
198
|
+
* Refresh a tenant's lease.
|
|
199
|
+
*
|
|
200
|
+
* @param tenantId - Tenant identifier
|
|
201
|
+
* @param ttlMs - Optional new TTL
|
|
202
|
+
* @returns true if lease was refreshed
|
|
203
|
+
*/
|
|
204
|
+
refreshLease(tenantId, ttlMs) {
|
|
205
|
+
try {
|
|
206
|
+
const refreshed = this.leaseManager.refresh(tenantId, ttlMs);
|
|
207
|
+
if (refreshed) {
|
|
208
|
+
// Reset idle timer
|
|
209
|
+
const workerId = this.tenantToWorker.get(tenantId);
|
|
210
|
+
if (workerId) {
|
|
211
|
+
const entry = this.workers.get(workerId);
|
|
212
|
+
if (entry) {
|
|
213
|
+
this.resetIdleTimer(entry);
|
|
214
|
+
}
|
|
215
|
+
}
|
|
216
|
+
}
|
|
217
|
+
return refreshed;
|
|
218
|
+
}
|
|
219
|
+
catch {
|
|
220
|
+
return false;
|
|
221
|
+
}
|
|
222
|
+
}
|
|
223
|
+
/**
|
|
224
|
+
* Get the CDP endpoint for a tenant's worker.
|
|
225
|
+
*
|
|
226
|
+
* @param tenantId - Tenant identifier
|
|
227
|
+
* @returns CDP endpoint URL or undefined if no active worker
|
|
228
|
+
*/
|
|
229
|
+
getCdpEndpoint(tenantId) {
|
|
230
|
+
const workerId = this.tenantToWorker.get(tenantId);
|
|
231
|
+
if (!workerId)
|
|
232
|
+
return undefined;
|
|
233
|
+
const entry = this.workers.get(workerId);
|
|
234
|
+
return entry?.descriptor.cdpEndpoint;
|
|
235
|
+
}
|
|
236
|
+
/**
|
|
237
|
+
* Check if a tenant has an active lease.
|
|
238
|
+
*/
|
|
239
|
+
hasActiveLease(tenantId) {
|
|
240
|
+
return this.leaseManager.hasLease(tenantId);
|
|
241
|
+
}
|
|
242
|
+
/**
|
|
243
|
+
* Check if a specific controller holds the lease for a tenant.
|
|
244
|
+
*/
|
|
245
|
+
isLeaseHeldBy(tenantId, controllerId) {
|
|
246
|
+
return this.leaseManager.isLeaseHeldBy(tenantId, controllerId);
|
|
247
|
+
}
|
|
248
|
+
/**
|
|
249
|
+
* Get worker descriptor for a tenant.
|
|
250
|
+
*/
|
|
251
|
+
getWorkerForTenant(tenantId) {
|
|
252
|
+
const workerId = this.tenantToWorker.get(tenantId);
|
|
253
|
+
if (!workerId)
|
|
254
|
+
return undefined;
|
|
255
|
+
return this.workers.get(workerId)?.descriptor;
|
|
256
|
+
}
|
|
257
|
+
/**
|
|
258
|
+
* Stop a specific worker.
|
|
259
|
+
*
|
|
260
|
+
* @param tenantId - Tenant identifier
|
|
261
|
+
* @param reason - Reason for stopping
|
|
262
|
+
*/
|
|
263
|
+
async stopWorker(tenantId, reason) {
|
|
264
|
+
const workerId = this.tenantToWorker.get(tenantId);
|
|
265
|
+
if (!workerId)
|
|
266
|
+
return;
|
|
267
|
+
await this.cleanupWorker(workerId, reason);
|
|
268
|
+
}
|
|
269
|
+
/**
|
|
270
|
+
* Shutdown the WorkerManager and all workers.
|
|
271
|
+
*/
|
|
272
|
+
async shutdown() {
|
|
273
|
+
if (this._isShuttingDown)
|
|
274
|
+
return;
|
|
275
|
+
this._isShuttingDown = true;
|
|
276
|
+
logger.info('WorkerManager shutting down');
|
|
277
|
+
// Stop health monitoring
|
|
278
|
+
this.healthMonitor.stop();
|
|
279
|
+
// Stop lease cleanup
|
|
280
|
+
this.leaseManager.stop();
|
|
281
|
+
// Stop all workers
|
|
282
|
+
const stopPromises = Array.from(this.workers.keys()).map((workerId) => this.cleanupWorker(workerId, 'shutdown'));
|
|
283
|
+
await Promise.all(stopPromises);
|
|
284
|
+
logger.info('WorkerManager shutdown complete');
|
|
285
|
+
}
|
|
286
|
+
/**
|
|
287
|
+
* Create a new worker for a tenant.
|
|
288
|
+
*/
|
|
289
|
+
async createWorker(tenantId) {
|
|
290
|
+
// Check worker limit
|
|
291
|
+
if (this.workers.size >= this.config.maxWorkers) {
|
|
292
|
+
throw WorkerError.maxWorkersReached(this.config.maxWorkers, this.workers.size);
|
|
293
|
+
}
|
|
294
|
+
// Allocate port
|
|
295
|
+
const port = await this.portAllocator.allocateVerified();
|
|
296
|
+
const workerId = `w-${randomUUID().slice(0, 8)}`;
|
|
297
|
+
const profileDir = join(this.config.profileBaseDir, tenantId);
|
|
298
|
+
logger.info('Creating worker', { workerId, tenantId, port, profileDir });
|
|
299
|
+
// Create worker descriptor
|
|
300
|
+
const now = Date.now();
|
|
301
|
+
const descriptor = {
|
|
302
|
+
workerId,
|
|
303
|
+
tenantId,
|
|
304
|
+
state: 'starting',
|
|
305
|
+
port,
|
|
306
|
+
profileDir,
|
|
307
|
+
createdAt: now,
|
|
308
|
+
lastUsedAt: now,
|
|
309
|
+
};
|
|
310
|
+
// Create Chrome process
|
|
311
|
+
const process = new ChromeWorkerProcess({
|
|
312
|
+
workerId,
|
|
313
|
+
port,
|
|
314
|
+
profileDir,
|
|
315
|
+
chromePath: this.config.chromePath,
|
|
316
|
+
});
|
|
317
|
+
// Create entry
|
|
318
|
+
const entry = {
|
|
319
|
+
descriptor,
|
|
320
|
+
process,
|
|
321
|
+
tenantId,
|
|
322
|
+
};
|
|
323
|
+
this.workers.set(workerId, entry);
|
|
324
|
+
this.tenantToWorker.set(tenantId, workerId);
|
|
325
|
+
// Set up process event handlers
|
|
326
|
+
this.setupWorkerProcessHandlers(entry);
|
|
327
|
+
try {
|
|
328
|
+
// Start the process
|
|
329
|
+
await process.start();
|
|
330
|
+
// Update descriptor
|
|
331
|
+
descriptor.state = 'running';
|
|
332
|
+
descriptor.pid = process.pid;
|
|
333
|
+
descriptor.cdpEndpoint = process.cdpEndpoint;
|
|
334
|
+
descriptor.startedAt = Date.now();
|
|
335
|
+
// Register with health monitor
|
|
336
|
+
this.healthMonitor.registerWorker(workerId, port);
|
|
337
|
+
// Start hard TTL timer
|
|
338
|
+
this.startHardTtlTimer(entry);
|
|
339
|
+
this.emit('workerCreated', { workerId, tenantId });
|
|
340
|
+
this.emit('workerStarted', {
|
|
341
|
+
workerId,
|
|
342
|
+
tenantId,
|
|
343
|
+
cdpEndpoint: descriptor.cdpEndpoint,
|
|
344
|
+
});
|
|
345
|
+
logger.info('Worker started', {
|
|
346
|
+
workerId,
|
|
347
|
+
tenantId,
|
|
348
|
+
pid: descriptor.pid,
|
|
349
|
+
cdpEndpoint: descriptor.cdpEndpoint,
|
|
350
|
+
});
|
|
351
|
+
return workerId;
|
|
352
|
+
}
|
|
353
|
+
catch (error) {
|
|
354
|
+
// Clean up on failure
|
|
355
|
+
this.workers.delete(workerId);
|
|
356
|
+
this.tenantToWorker.delete(tenantId);
|
|
357
|
+
this.portAllocator.release(port);
|
|
358
|
+
throw error;
|
|
359
|
+
}
|
|
360
|
+
}
|
|
361
|
+
/**
|
|
362
|
+
* Clean up a worker.
|
|
363
|
+
*/
|
|
364
|
+
async cleanupWorker(workerId, reason) {
|
|
365
|
+
const entry = this.workers.get(workerId);
|
|
366
|
+
if (!entry)
|
|
367
|
+
return;
|
|
368
|
+
logger.info('Cleaning up worker', { workerId, tenantId: entry.tenantId, reason });
|
|
369
|
+
// Clear timers
|
|
370
|
+
if (entry.idleTimerId) {
|
|
371
|
+
clearTimeout(entry.idleTimerId);
|
|
372
|
+
}
|
|
373
|
+
if (entry.hardTtlTimerId) {
|
|
374
|
+
clearTimeout(entry.hardTtlTimerId);
|
|
375
|
+
}
|
|
376
|
+
// Revoke any lease
|
|
377
|
+
this.leaseManager.revoke(entry.tenantId, reason);
|
|
378
|
+
// Unregister from health monitor
|
|
379
|
+
this.healthMonitor.unregisterWorker(workerId);
|
|
380
|
+
// Stop the process
|
|
381
|
+
try {
|
|
382
|
+
await entry.process.stop();
|
|
383
|
+
}
|
|
384
|
+
catch (error) {
|
|
385
|
+
logger.warning('Error stopping worker process', { workerId, error });
|
|
386
|
+
entry.process.kill();
|
|
387
|
+
}
|
|
388
|
+
// Release port
|
|
389
|
+
this.portAllocator.release(entry.descriptor.port);
|
|
390
|
+
// Remove from maps
|
|
391
|
+
this.workers.delete(workerId);
|
|
392
|
+
this.tenantToWorker.delete(entry.tenantId);
|
|
393
|
+
entry.descriptor.state = 'stopped';
|
|
394
|
+
this.emit('workerStopped', {
|
|
395
|
+
workerId,
|
|
396
|
+
tenantId: entry.tenantId,
|
|
397
|
+
reason,
|
|
398
|
+
});
|
|
399
|
+
}
|
|
400
|
+
/**
|
|
401
|
+
* Set up event handlers for sub-components.
|
|
402
|
+
*/
|
|
403
|
+
setupEventHandlers() {
|
|
404
|
+
// Health monitor events
|
|
405
|
+
this.healthMonitor.on('healthChange', ({ workerId, healthy }) => {
|
|
406
|
+
if (!healthy) {
|
|
407
|
+
const entry = this.workers.get(workerId);
|
|
408
|
+
if (entry) {
|
|
409
|
+
logger.warning('Worker became unhealthy', { workerId, tenantId: entry.tenantId });
|
|
410
|
+
// Could trigger restart here if needed
|
|
411
|
+
}
|
|
412
|
+
}
|
|
413
|
+
});
|
|
414
|
+
// Lease expiration events
|
|
415
|
+
this.leaseManager.onLeaseExpired((lease) => {
|
|
416
|
+
this.emit('leaseExpired', {
|
|
417
|
+
leaseId: lease.leaseId,
|
|
418
|
+
tenantId: lease.tenantId,
|
|
419
|
+
});
|
|
420
|
+
// Start idle timer for the worker
|
|
421
|
+
const entry = this.workers.get(lease.workerId);
|
|
422
|
+
if (entry) {
|
|
423
|
+
this.startIdleTimer(entry);
|
|
424
|
+
}
|
|
425
|
+
});
|
|
426
|
+
// Lease revocation events
|
|
427
|
+
this.leaseManager.onLeaseRevoked((lease, reason) => {
|
|
428
|
+
this.emit('leaseRevoked', {
|
|
429
|
+
leaseId: lease.leaseId,
|
|
430
|
+
tenantId: lease.tenantId,
|
|
431
|
+
reason,
|
|
432
|
+
});
|
|
433
|
+
});
|
|
434
|
+
}
|
|
435
|
+
/**
|
|
436
|
+
* Set up event handlers for a worker process.
|
|
437
|
+
*/
|
|
438
|
+
setupWorkerProcessHandlers(entry) {
|
|
439
|
+
const { process, descriptor } = entry;
|
|
440
|
+
process.on('exit', ({ code, signal }) => {
|
|
441
|
+
// Only handle if not already cleaned up
|
|
442
|
+
if (!this.workers.has(descriptor.workerId))
|
|
443
|
+
return;
|
|
444
|
+
descriptor.state = 'crashed';
|
|
445
|
+
this.emit('workerCrashed', {
|
|
446
|
+
workerId: descriptor.workerId,
|
|
447
|
+
tenantId: entry.tenantId,
|
|
448
|
+
exitCode: code,
|
|
449
|
+
});
|
|
450
|
+
// Revoke lease on crash
|
|
451
|
+
this.leaseManager.revoke(entry.tenantId, `worker crashed (exit: ${code}, signal: ${signal})`);
|
|
452
|
+
// Clean up
|
|
453
|
+
void this.cleanupWorker(descriptor.workerId, 'crashed');
|
|
454
|
+
});
|
|
455
|
+
process.on('error', ({ error }) => {
|
|
456
|
+
logger.error('Worker process error', error, {
|
|
457
|
+
workerId: descriptor.workerId,
|
|
458
|
+
});
|
|
459
|
+
});
|
|
460
|
+
}
|
|
461
|
+
/**
|
|
462
|
+
* Start idle timer for a worker.
|
|
463
|
+
*/
|
|
464
|
+
startIdleTimer(entry) {
|
|
465
|
+
// Clear existing timer
|
|
466
|
+
if (entry.idleTimerId) {
|
|
467
|
+
clearTimeout(entry.idleTimerId);
|
|
468
|
+
}
|
|
469
|
+
entry.idleTimerId = setTimeout(() => {
|
|
470
|
+
logger.info('Worker idle timeout reached', {
|
|
471
|
+
workerId: entry.descriptor.workerId,
|
|
472
|
+
tenantId: entry.tenantId,
|
|
473
|
+
});
|
|
474
|
+
void this.cleanupWorker(entry.descriptor.workerId, 'idle timeout');
|
|
475
|
+
}, this.config.idleTimeoutMs);
|
|
476
|
+
}
|
|
477
|
+
/**
|
|
478
|
+
* Reset idle timer (on activity).
|
|
479
|
+
*/
|
|
480
|
+
resetIdleTimer(entry) {
|
|
481
|
+
if (entry.idleTimerId) {
|
|
482
|
+
clearTimeout(entry.idleTimerId);
|
|
483
|
+
entry.idleTimerId = undefined;
|
|
484
|
+
}
|
|
485
|
+
entry.descriptor.lastUsedAt = Date.now();
|
|
486
|
+
}
|
|
487
|
+
/**
|
|
488
|
+
* Start hard TTL timer for a worker.
|
|
489
|
+
*/
|
|
490
|
+
startHardTtlTimer(entry) {
|
|
491
|
+
entry.hardTtlTimerId = setTimeout(() => {
|
|
492
|
+
logger.info('Worker hard TTL reached', {
|
|
493
|
+
workerId: entry.descriptor.workerId,
|
|
494
|
+
tenantId: entry.tenantId,
|
|
495
|
+
});
|
|
496
|
+
void this.cleanupWorker(entry.descriptor.workerId, 'hard TTL');
|
|
497
|
+
}, this.config.hardTtlMs);
|
|
498
|
+
}
|
|
499
|
+
}
|
|
500
|
+
//# sourceMappingURL=worker-manager.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"worker-manager.js","sourceRoot":"","sources":["../../../src/worker/worker-manager.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,EAAE,YAAY,EAAE,MAAM,QAAQ,CAAC;AACtC,OAAO,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AACpC,OAAO,EAAE,IAAI,EAAE,MAAM,MAAM,CAAC;AAO5B,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACpD,OAAO,EAAE,mBAAmB,EAAE,MAAM,4BAA4B,CAAC;AACjE,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACpD,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,EAAE,YAAY,EAAE,MAAM,uCAAuC,CAAC;AAErE,MAAM,MAAM,GAAG,YAAY,CAAC,eAAe,CAAC,CAAC;AAiC7C;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AACH,MAAM,OAAO,aAAc,SAAQ,YAAY;IAC5B,MAAM,CAAsB;IAC5B,aAAa,CAAgB;IAC7B,aAAa,CAAgB;IAC7B,YAAY,CAAe;IAE5C,kCAAkC;IACjB,OAAO,GAAG,IAAI,GAAG,EAAuB,CAAC;IAC1D,kCAAkC;IACjB,cAAc,GAAG,IAAI,GAAG,EAAkB,CAAC;IAEpD,eAAe,GAAG,KAAK,CAAC;IAEhC,YAAY,MAA2B;QACrC,KAAK,EAAE,CAAC;QAER,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QAErB,4BAA4B;QAC5B,IAAI,CAAC,aAAa,GAAG,IAAI,aAAa,CAAC;YACrC,GAAG,EAAE,MAAM,CAAC,SAAS,CAAC,GAAG;YACzB,GAAG,EAAE,MAAM,CAAC,SAAS,CAAC,GAAG;SAC1B,CAAC,CAAC;QAEH,IAAI,CAAC,aAAa,GAAG,IAAI,aAAa,CAAC,MAAM,CAAC,qBAAqB,EAAE;YACnE,SAAS,EAAE,IAAI;YACf,gBAAgB,EAAE,CAAC;SACpB,CAAC,CAAC;QAEH,IAAI,CAAC,YAAY,GAAG,IAAI,YAAY,CAAC;YACnC,YAAY,EAAE,MAAM,CAAC,UAAU;YAC/B,iBAAiB,EAAE,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,UAAU,GAAG,CAAC,EAAE,KAAK,CAAC;SAC1D,CAAC,CAAC;QAEH,wBAAwB;QACxB,IAAI,CAAC,kBAAkB,EAAE,CAAC;QAE1B,0BAA0B;QAC1B,IAAI,CAAC,aAAa,CAAC,KAAK,EAAE,CAAC;QAE3B,MAAM,CAAC,IAAI,CAAC,2BAA2B,EAAE;YACvC,cAAc,EAAE,MAAM,CAAC,cAAc;YACrC,SAAS,EAAE,MAAM,CAAC,SAAS;YAC3B,UAAU,EAAE,MAAM,CAAC,UAAU;SAC9B,CAAC,CAAC;IACL,CAAC;IAED;;OAEG;IACH,IAAI,WAAW;QACb,OAAO,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC;IAC3B,CAAC;IAED;;OAEG;IACH,IAAI,UAAU;QACZ,OAAO,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC;IACtC,CAAC;IAED;;OAEG;IACH,IAAI,cAAc;QAChB,OAAO,IAAI,CAAC,eAAe,CAAC;IAC9B,CAAC;IAED;;;;;;;;OAQG;IACH,KAAK,CAAC,gBAAgB,CACpB,QAAgB,EAChB,YAAoB,EACpB,KAAc;QAEd,IAAI,IAAI,CAAC,eAAe,EAAE,CAAC;YACzB,OAAO;gBACL,OAAO,EAAE,KAAK;gBACd,KAAK,EAAE,gCAAgC;gBACvC,SAAS,EAAE,kBAAkB;aAC9B,CAAC;QACJ,CAAC;QAED,uCAAuC;QACvC,MAAM,QAAQ,GAAG,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QAEnD,IAAI,QAAQ,EAAE,CAAC;YACb,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;YAEzC,IAAI,KAAK,EAAE,UAAU,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;gBAC1C,0CAA0C;gBAC1C,MAAM,WAAW,GAAG,IAAI,CAAC,YAAY,CAAC,OAAO,CAAC,QAAQ,EAAE,YAAY,EAAE,QAAQ,EAAE,KAAK,CAAC,CAAC;gBAEvF,IAAI,WAAW,CAAC,OAAO,EAAE,CAAC;oBACxB,IAAI,CAAC,cAAc,CAAC,KAAK,CAAC,CAAC;oBAE3B,IAAI,CAAC,IAAI,CAAC,eAAe,EAAE;wBACzB,OAAO,EAAE,WAAW,CAAC,KAAM,CAAC,OAAO;wBACnC,QAAQ;wBACR,YAAY;qBACb,CAAC,CAAC;oBAEH,OAAO;wBACL,GAAG,WAAW;wBACd,WAAW,EAAE,KAAK,CAAC,UAAU,CAAC,WAAW;qBAC1C,CAAC;gBACJ,CAAC;gBAED,OAAO,WAAW,CAAC;YACrB,CAAC;YAED,8CAA8C;YAC9C,MAAM,IAAI,CAAC,aAAa,CAAC,QAAQ,EAAE,aAAa,CAAC,CAAC;QACpD,CAAC;QAED,+BAA+B;QAC/B,IAAI,CAAC;YACH,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,QAAQ,CAAC,CAAC;YAEtD,8BAA8B;YAC9B,MAAM,WAAW,GAAG,IAAI,CAAC,YAAY,CAAC,OAAO,CAAC,QAAQ,EAAE,YAAY,EAAE,WAAW,EAAE,KAAK,CAAC,CAAC;YAE1F,IAAI,WAAW,CAAC,OAAO,EAAE,CAAC;gBACxB,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,WAAW,CAAE,CAAC;gBAE7C,IAAI,CAAC,IAAI,CAAC,eAAe,EAAE;oBACzB,OAAO,EAAE,WAAW,CAAC,KAAM,CAAC,OAAO;oBACnC,QAAQ;oBACR,YAAY;iBACb,CAAC,CAAC;gBAEH,OAAO;oBACL,GAAG,WAAW;oBACd,WAAW,EAAE,KAAK,CAAC,UAAU,CAAC,WAAW;iBAC1C,CAAC;YACJ,CAAC;YAED,OAAO,WAAW,CAAC;QACrB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,WAAW,GACf,KAAK,YAAY,WAAW;gBAC1B,CAAC,CAAC,KAAK;gBACP,CAAC,CAAC,WAAW,CAAC,eAAe,CAAC,gBAAgB,EAAE,KAAc,CAAC,CAAC;YAEpE,OAAO;gBACL,OAAO,EAAE,KAAK;gBACd,KAAK,EAAE,WAAW,CAAC,OAAO;gBAC1B,SAAS,EAAE,WAAW,CAAC,IAAI;aAC5B,CAAC;QACJ,CAAC;IACH,CAAC;IAED;;;;;;OAMG;IACH,YAAY,CAAC,QAAgB,EAAE,YAAqB;QAClD,MAAM,KAAK,GAAG,IAAI,CAAC,YAAY,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;QACnD,MAAM,QAAQ,GAAG,IAAI,CAAC,YAAY,CAAC,OAAO,CAAC,QAAQ,EAAE,YAAY,CAAC,CAAC;QAEnE,IAAI,QAAQ,IAAI,KAAK,EAAE,CAAC;YACtB,IAAI,CAAC,IAAI,CAAC,eAAe,EAAE;gBACzB,OAAO,EAAE,KAAK,CAAC,OAAO;gBACtB,QAAQ;aACT,CAAC,CAAC;YAEH,kCAAkC;YAClC,MAAM,QAAQ,GAAG,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;YACnD,IAAI,QAAQ,EAAE,CAAC;gBACb,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;gBACzC,IAAI,KAAK,EAAE,CAAC;oBACV,IAAI,CAAC,cAAc,CAAC,KAAK,CAAC,CAAC;gBAC7B,CAAC;YACH,CAAC;QACH,CAAC;QAED,OAAO,QAAQ,CAAC;IAClB,CAAC;IAED;;;;;;OAMG;IACH,YAAY,CAAC,QAAgB,EAAE,KAAc;QAC3C,IAAI,CAAC;YACH,MAAM,SAAS,GAAG,IAAI,CAAC,YAAY,CAAC,OAAO,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;YAE7D,IAAI,SAAS,EAAE,CAAC;gBACd,mBAAmB;gBACnB,MAAM,QAAQ,GAAG,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;gBACnD,IAAI,QAAQ,EAAE,CAAC;oBACb,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;oBACzC,IAAI,KAAK,EAAE,CAAC;wBACV,IAAI,CAAC,cAAc,CAAC,KAAK,CAAC,CAAC;oBAC7B,CAAC;gBACH,CAAC;YACH,CAAC;YAED,OAAO,SAAS,CAAC;QACnB,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,KAAK,CAAC;QACf,CAAC;IACH,CAAC;IAED;;;;;OAKG;IACH,cAAc,CAAC,QAAgB;QAC7B,MAAM,QAAQ,GAAG,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QACnD,IAAI,CAAC,QAAQ;YAAE,OAAO,SAAS,CAAC;QAEhC,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QACzC,OAAO,KAAK,EAAE,UAAU,CAAC,WAAW,CAAC;IACvC,CAAC;IAED;;OAEG;IACH,cAAc,CAAC,QAAgB;QAC7B,OAAO,IAAI,CAAC,YAAY,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;IAC9C,CAAC;IAED;;OAEG;IACH,aAAa,CAAC,QAAgB,EAAE,YAAoB;QAClD,OAAO,IAAI,CAAC,YAAY,CAAC,aAAa,CAAC,QAAQ,EAAE,YAAY,CAAC,CAAC;IACjE,CAAC;IAED;;OAEG;IACH,kBAAkB,CAAC,QAAgB;QACjC,MAAM,QAAQ,GAAG,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QACnD,IAAI,CAAC,QAAQ;YAAE,OAAO,SAAS,CAAC;QAEhC,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,UAAU,CAAC;IAChD,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,UAAU,CAAC,QAAgB,EAAE,MAAc;QAC/C,MAAM,QAAQ,GAAG,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QACnD,IAAI,CAAC,QAAQ;YAAE,OAAO;QAEtB,MAAM,IAAI,CAAC,aAAa,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;IAC7C,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,QAAQ;QACZ,IAAI,IAAI,CAAC,eAAe;YAAE,OAAO;QAEjC,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC;QAC5B,MAAM,CAAC,IAAI,CAAC,6BAA6B,CAAC,CAAC;QAE3C,yBAAyB;QACzB,IAAI,CAAC,aAAa,CAAC,IAAI,EAAE,CAAC;QAE1B,qBAAqB;QACrB,IAAI,CAAC,YAAY,CAAC,IAAI,EAAE,CAAC;QAEzB,mBAAmB;QACnB,MAAM,YAAY,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CACpE,IAAI,CAAC,aAAa,CAAC,QAAQ,EAAE,UAAU,CAAC,CACzC,CAAC;QAEF,MAAM,OAAO,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC;QAEhC,MAAM,CAAC,IAAI,CAAC,iCAAiC,CAAC,CAAC;IACjD,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,YAAY,CAAC,QAAgB;QACzC,qBAAqB;QACrB,IAAI,IAAI,CAAC,OAAO,CAAC,IAAI,IAAI,IAAI,CAAC,MAAM,CAAC,UAAU,EAAE,CAAC;YAChD,MAAM,WAAW,CAAC,iBAAiB,CAAC,IAAI,CAAC,MAAM,CAAC,UAAU,EAAE,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;QACjF,CAAC;QAED,gBAAgB;QAChB,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,gBAAgB,EAAE,CAAC;QAEzD,MAAM,QAAQ,GAAG,KAAK,UAAU,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC;QACjD,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,cAAc,EAAE,QAAQ,CAAC,CAAC;QAE9D,MAAM,CAAC,IAAI,CAAC,iBAAiB,EAAE,EAAE,QAAQ,EAAE,QAAQ,EAAE,IAAI,EAAE,UAAU,EAAE,CAAC,CAAC;QAEzE,2BAA2B;QAC3B,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACvB,MAAM,UAAU,GAAqB;YACnC,QAAQ;YACR,QAAQ;YACR,KAAK,EAAE,UAAU;YACjB,IAAI;YACJ,UAAU;YACV,SAAS,EAAE,GAAG;YACd,UAAU,EAAE,GAAG;SAChB,CAAC;QAEF,wBAAwB;QACxB,MAAM,OAAO,GAAG,IAAI,mBAAmB,CAAC;YACtC,QAAQ;YACR,IAAI;YACJ,UAAU;YACV,UAAU,EAAE,IAAI,CAAC,MAAM,CAAC,UAAU;SACnC,CAAC,CAAC;QAEH,eAAe;QACf,MAAM,KAAK,GAAgB;YACzB,UAAU;YACV,OAAO;YACP,QAAQ;SACT,CAAC;QAEF,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;QAClC,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;QAE5C,gCAAgC;QAChC,IAAI,CAAC,0BAA0B,CAAC,KAAK,CAAC,CAAC;QAEvC,IAAI,CAAC;YACH,oBAAoB;YACpB,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC;YAEtB,oBAAoB;YACpB,UAAU,CAAC,KAAK,GAAG,SAAS,CAAC;YAC7B,UAAU,CAAC,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC;YAC7B,UAAU,CAAC,WAAW,GAAG,OAAO,CAAC,WAAW,CAAC;YAC7C,UAAU,CAAC,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YAElC,+BAA+B;YAC/B,IAAI,CAAC,aAAa,CAAC,cAAc,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;YAElD,uBAAuB;YACvB,IAAI,CAAC,iBAAiB,CAAC,KAAK,CAAC,CAAC;YAE9B,IAAI,CAAC,IAAI,CAAC,eAAe,EAAE,EAAE,QAAQ,EAAE,QAAQ,EAAE,CAAC,CAAC;YACnD,IAAI,CAAC,IAAI,CAAC,eAAe,EAAE;gBACzB,QAAQ;gBACR,QAAQ;gBACR,WAAW,EAAE,UAAU,CAAC,WAAY;aACrC,CAAC,CAAC;YAEH,MAAM,CAAC,IAAI,CAAC,gBAAgB,EAAE;gBAC5B,QAAQ;gBACR,QAAQ;gBACR,GAAG,EAAE,UAAU,CAAC,GAAG;gBACnB,WAAW,EAAE,UAAU,CAAC,WAAW;aACpC,CAAC,CAAC;YAEH,OAAO,QAAQ,CAAC;QAClB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,sBAAsB;YACtB,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;YAC9B,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;YACrC,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;YAEjC,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,aAAa,CAAC,QAAgB,EAAE,MAAc;QAC1D,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QACzC,IAAI,CAAC,KAAK;YAAE,OAAO;QAEnB,MAAM,CAAC,IAAI,CAAC,oBAAoB,EAAE,EAAE,QAAQ,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,EAAE,MAAM,EAAE,CAAC,CAAC;QAElF,eAAe;QACf,IAAI,KAAK,CAAC,WAAW,EAAE,CAAC;YACtB,YAAY,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;QAClC,CAAC;QACD,IAAI,KAAK,CAAC,cAAc,EAAE,CAAC;YACzB,YAAY,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC;QACrC,CAAC;QAED,mBAAmB;QACnB,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,KAAK,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;QAEjD,iCAAiC;QACjC,IAAI,CAAC,aAAa,CAAC,gBAAgB,CAAC,QAAQ,CAAC,CAAC;QAE9C,mBAAmB;QACnB,IAAI,CAAC;YACH,MAAM,KAAK,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;QAC7B,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,CAAC,OAAO,CAAC,+BAA+B,EAAE,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,CAAC;YACrE,KAAK,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;QACvB,CAAC;QAED,eAAe;QACf,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,KAAK,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC;QAElD,mBAAmB;QACnB,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;QAC9B,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;QAE3C,KAAK,CAAC,UAAU,CAAC,KAAK,GAAG,SAAS,CAAC;QAEnC,IAAI,CAAC,IAAI,CAAC,eAAe,EAAE;YACzB,QAAQ;YACR,QAAQ,EAAE,KAAK,CAAC,QAAQ;YACxB,MAAM;SACP,CAAC,CAAC;IACL,CAAC;IAED;;OAEG;IACK,kBAAkB;QACxB,wBAAwB;QACxB,IAAI,CAAC,aAAa,CAAC,EAAE,CACnB,cAAc,EACd,CAAC,EAAE,QAAQ,EAAE,OAAO,EAA0C,EAAE,EAAE;YAChE,IAAI,CAAC,OAAO,EAAE,CAAC;gBACb,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;gBACzC,IAAI,KAAK,EAAE,CAAC;oBACV,MAAM,CAAC,OAAO,CAAC,yBAAyB,EAAE,EAAE,QAAQ,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,EAAE,CAAC,CAAC;oBAClF,uCAAuC;gBACzC,CAAC;YACH,CAAC;QACH,CAAC,CACF,CAAC;QAEF,0BAA0B;QAC1B,IAAI,CAAC,YAAY,CAAC,cAAc,CAAC,CAAC,KAAK,EAAE,EAAE;YACzC,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE;gBACxB,OAAO,EAAE,KAAK,CAAC,OAAO;gBACtB,QAAQ,EAAE,KAAK,CAAC,QAAQ;aACzB,CAAC,CAAC;YAEH,kCAAkC;YAClC,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;YAC/C,IAAI,KAAK,EAAE,CAAC;gBACV,IAAI,CAAC,cAAc,CAAC,KAAK,CAAC,CAAC;YAC7B,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,0BAA0B;QAC1B,IAAI,CAAC,YAAY,CAAC,cAAc,CAAC,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE;YACjD,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE;gBACxB,OAAO,EAAE,KAAK,CAAC,OAAO;gBACtB,QAAQ,EAAE,KAAK,CAAC,QAAQ;gBACxB,MAAM;aACP,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;IACL,CAAC;IAED;;OAEG;IACK,0BAA0B,CAAC,KAAkB;QACnD,MAAM,EAAE,OAAO,EAAE,UAAU,EAAE,GAAG,KAAK,CAAC;QAEtC,OAAO,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAkD,EAAE,EAAE;YACtF,wCAAwC;YACxC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC,QAAQ,CAAC;gBAAE,OAAO;YAEnD,UAAU,CAAC,KAAK,GAAG,SAAS,CAAC;YAE7B,IAAI,CAAC,IAAI,CAAC,eAAe,EAAE;gBACzB,QAAQ,EAAE,UAAU,CAAC,QAAQ;gBAC7B,QAAQ,EAAE,KAAK,CAAC,QAAQ;gBACxB,QAAQ,EAAE,IAAI;aACf,CAAC,CAAC;YAEH,wBAAwB;YACxB,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,KAAK,CAAC,QAAQ,EAAE,yBAAyB,IAAI,aAAa,MAAM,GAAG,CAAC,CAAC;YAE9F,WAAW;YACX,KAAK,IAAI,CAAC,aAAa,CAAC,UAAU,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC;QAC1D,CAAC,CAAC,CAAC;QAEH,OAAO,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,EAAE,KAAK,EAAoB,EAAE,EAAE;YAClD,MAAM,CAAC,KAAK,CAAC,sBAAsB,EAAE,KAAK,EAAE;gBAC1C,QAAQ,EAAE,UAAU,CAAC,QAAQ;aAC9B,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;IACL,CAAC;IAED;;OAEG;IACK,cAAc,CAAC,KAAkB;QACvC,uBAAuB;QACvB,IAAI,KAAK,CAAC,WAAW,EAAE,CAAC;YACtB,YAAY,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;QAClC,CAAC;QAED,KAAK,CAAC,WAAW,GAAG,UAAU,CAAC,GAAG,EAAE;YAClC,MAAM,CAAC,IAAI,CAAC,6BAA6B,EAAE;gBACzC,QAAQ,EAAE,KAAK,CAAC,UAAU,CAAC,QAAQ;gBACnC,QAAQ,EAAE,KAAK,CAAC,QAAQ;aACzB,CAAC,CAAC;YACH,KAAK,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,UAAU,CAAC,QAAQ,EAAE,cAAc,CAAC,CAAC;QACrE,CAAC,EAAE,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC;IAChC,CAAC;IAED;;OAEG;IACK,cAAc,CAAC,KAAkB;QACvC,IAAI,KAAK,CAAC,WAAW,EAAE,CAAC;YACtB,YAAY,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;YAChC,KAAK,CAAC,WAAW,GAAG,SAAS,CAAC;QAChC,CAAC;QACD,KAAK,CAAC,UAAU,CAAC,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAC3C,CAAC;IAED;;OAEG;IACK,iBAAiB,CAAC,KAAkB;QAC1C,KAAK,CAAC,cAAc,GAAG,UAAU,CAAC,GAAG,EAAE;YACrC,MAAM,CAAC,IAAI,CAAC,yBAAyB,EAAE;gBACrC,QAAQ,EAAE,KAAK,CAAC,UAAU,CAAC,QAAQ;gBACnC,QAAQ,EAAE,KAAK,CAAC,QAAQ;aACzB,CAAC,CAAC;YACH,KAAK,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,UAAU,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;QACjE,CAAC,EAAE,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;IAC5B,CAAC;CACF"}
|