@kapeta/local-cluster-service 0.11.1 → 0.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +7 -0
- package/definitions.d.ts +7 -0
- package/dist/cjs/src/config/routes.js +1 -1
- package/dist/cjs/src/containerManager.d.ts +2 -1
- package/dist/cjs/src/containerManager.js +125 -21
- package/dist/cjs/src/definitionsManager.d.ts +1 -0
- package/dist/cjs/src/definitionsManager.js +7 -4
- package/dist/cjs/src/instanceManager.d.ts +8 -1
- package/dist/cjs/src/instanceManager.js +48 -19
- package/dist/cjs/src/operatorManager.d.ts +2 -0
- package/dist/cjs/src/operatorManager.js +69 -67
- package/dist/cjs/src/socketManager.d.ts +1 -0
- package/dist/cjs/src/socketManager.js +3 -0
- package/dist/cjs/src/types.d.ts +1 -0
- package/dist/cjs/src/utils/BlockInstanceRunner.js +2 -2
- package/dist/esm/src/config/routes.js +1 -1
- package/dist/esm/src/containerManager.d.ts +2 -1
- package/dist/esm/src/containerManager.js +126 -22
- package/dist/esm/src/definitionsManager.d.ts +1 -0
- package/dist/esm/src/definitionsManager.js +8 -5
- package/dist/esm/src/instanceManager.d.ts +8 -1
- package/dist/esm/src/instanceManager.js +48 -19
- package/dist/esm/src/operatorManager.d.ts +2 -0
- package/dist/esm/src/operatorManager.js +67 -65
- package/dist/esm/src/socketManager.d.ts +1 -0
- package/dist/esm/src/socketManager.js +3 -0
- package/dist/esm/src/types.d.ts +1 -0
- package/dist/esm/src/utils/BlockInstanceRunner.js +2 -2
- package/dist/esm/src/utils/utils.js +1 -1
- package/package.json +1 -1
- package/src/config/routes.ts +1 -1
- package/src/containerManager.ts +178 -43
- package/src/definitionsManager.ts +9 -5
- package/src/instanceManager.ts +70 -40
- package/src/instances/routes.ts +1 -1
- package/src/operatorManager.ts +72 -70
- package/src/socketManager.ts +4 -0
- package/src/types.ts +1 -1
- package/src/utils/BlockInstanceRunner.ts +12 -22
- package/src/utils/utils.ts +2 -2
package/dist/cjs/src/types.d.ts
CHANGED
@@ -200,7 +200,7 @@ class BlockInstanceRunner {
|
|
200
200
|
`KAPETA_LOCAL_CLUSTER_PORT=${clusterService_1.clusterService.getClusterServicePort()}`,
|
201
201
|
...Object.entries({
|
202
202
|
...env,
|
203
|
-
...addonEnv
|
203
|
+
...addonEnv,
|
204
204
|
}).map(([key, value]) => `${key}=${value}`),
|
205
205
|
],
|
206
206
|
HostConfig: {
|
@@ -330,7 +330,7 @@ class BlockInstanceRunner {
|
|
330
330
|
async _handleContainer(container) {
|
331
331
|
return {
|
332
332
|
type: types_1.InstanceType.DOCKER,
|
333
|
-
pid: container.id
|
333
|
+
pid: container.id,
|
334
334
|
};
|
335
335
|
}
|
336
336
|
}
|
@@ -32,7 +32,7 @@ router.put('/instance', async (req, res) => {
|
|
32
32
|
if (req.kapeta.instanceId) {
|
33
33
|
configManager.setConfigForSection(req.kapeta.systemId, req.kapeta.instanceId, config);
|
34
34
|
//Restart the instance if it is running after config change
|
35
|
-
await instanceManager.
|
35
|
+
await instanceManager.prepareForRestart(req.kapeta.systemId, req.kapeta.instanceId);
|
36
36
|
}
|
37
37
|
else {
|
38
38
|
configManager.setConfigForSystem(req.kapeta.systemId, config);
|
@@ -1,6 +1,6 @@
|
|
1
1
|
import { Docker } from 'node-docker-api';
|
2
2
|
import { Container } from 'node-docker-api/lib/container';
|
3
|
-
import { InstanceInfo, LogEntry } from
|
3
|
+
import { InstanceInfo, LogEntry } from './types';
|
4
4
|
type StringMap = {
|
5
5
|
[key: string]: string;
|
6
6
|
};
|
@@ -49,6 +49,7 @@ declare class ContainerManager {
|
|
49
49
|
private _alive;
|
50
50
|
private _mountDir;
|
51
51
|
private _version;
|
52
|
+
private _lastDockerAccessCheck;
|
52
53
|
constructor();
|
53
54
|
initialize(): Promise<void>;
|
54
55
|
checkAlive(): Promise<boolean>;
|
@@ -8,7 +8,10 @@ import { parseKapetaUri } from '@kapeta/nodejs-utils';
|
|
8
8
|
import ClusterConfiguration from '@kapeta/local-cluster-config';
|
9
9
|
import uuid from 'node-uuid';
|
10
10
|
import md5 from 'md5';
|
11
|
-
import { getBlockInstanceContainerName } from
|
11
|
+
import { getBlockInstanceContainerName } from './utils/utils';
|
12
|
+
import { socketManager } from './socketManager';
|
13
|
+
import { KapetaAPI } from '@kapeta/nodejs-api-client';
|
14
|
+
const EVENT_IMAGE_PULL = 'docker-image-pull';
|
12
15
|
export const CONTAINER_LABEL_PORT_PREFIX = 'kapeta_port-';
|
13
16
|
const NANO_SECOND = 1000000;
|
14
17
|
const HEALTH_CHECK_INTERVAL = 3000;
|
@@ -26,6 +29,7 @@ class ContainerManager {
|
|
26
29
|
_alive;
|
27
30
|
_mountDir;
|
28
31
|
_version;
|
32
|
+
_lastDockerAccessCheck = 0;
|
29
33
|
constructor() {
|
30
34
|
this._docker = null;
|
31
35
|
this._alive = false;
|
@@ -160,17 +164,121 @@ class ContainerManager {
|
|
160
164
|
console.log('Image found: %s', image);
|
161
165
|
return false;
|
162
166
|
}
|
163
|
-
|
164
|
-
|
165
|
-
|
167
|
+
const timeStarted = Date.now();
|
168
|
+
socketManager.emitGlobal(EVENT_IMAGE_PULL, { image, percent: 0 });
|
169
|
+
const api = new KapetaAPI();
|
170
|
+
const accessToken = await api.getAccessToken();
|
171
|
+
const auth = image.startsWith('docker.kapeta.com/')
|
172
|
+
? {
|
173
|
+
username: 'kapeta',
|
174
|
+
password: accessToken,
|
175
|
+
serveraddress: 'docker.kapeta.com',
|
176
|
+
}
|
177
|
+
: {};
|
178
|
+
const stream = (await this.docker().image.create(auth, {
|
166
179
|
fromImage: imageName,
|
167
180
|
tag: tag,
|
168
|
-
});
|
169
|
-
|
170
|
-
|
181
|
+
}));
|
182
|
+
const chunks = {};
|
183
|
+
let lastEmitted = Date.now();
|
184
|
+
await promisifyStream(stream, (rawData) => {
|
185
|
+
const lines = rawData.toString().trim().split('\n');
|
186
|
+
lines.forEach((line) => {
|
187
|
+
const data = JSON.parse(line);
|
188
|
+
if (![
|
189
|
+
'Waiting',
|
190
|
+
'Downloading',
|
191
|
+
'Extracting',
|
192
|
+
'Download complete',
|
193
|
+
'Pull complete',
|
194
|
+
'Already exists',
|
195
|
+
].includes(data.status)) {
|
196
|
+
return;
|
197
|
+
}
|
198
|
+
if (!chunks[data.id]) {
|
199
|
+
chunks[data.id] = {
|
200
|
+
downloading: {
|
201
|
+
total: 0,
|
202
|
+
current: 0,
|
203
|
+
},
|
204
|
+
extracting: {
|
205
|
+
total: 0,
|
206
|
+
current: 0,
|
207
|
+
},
|
208
|
+
done: false,
|
209
|
+
};
|
210
|
+
}
|
211
|
+
const chunk = chunks[data.id];
|
212
|
+
switch (data.status) {
|
213
|
+
case 'Downloading':
|
214
|
+
chunk.downloading = data.progressDetail;
|
215
|
+
break;
|
216
|
+
case 'Extracting':
|
217
|
+
chunk.extracting = data.progressDetail;
|
218
|
+
break;
|
219
|
+
case 'Download complete':
|
220
|
+
chunk.downloading.current = chunks[data.id].downloading.total;
|
221
|
+
break;
|
222
|
+
case 'Pull complete':
|
223
|
+
chunk.extracting.current = chunks[data.id].extracting.total;
|
224
|
+
chunk.done = true;
|
225
|
+
break;
|
226
|
+
case 'Already exists':
|
227
|
+
// Force layer to be done
|
228
|
+
chunk.downloading.current = 1;
|
229
|
+
chunk.downloading.total = 1;
|
230
|
+
chunk.extracting.current = 1;
|
231
|
+
chunk.extracting.total = 1;
|
232
|
+
chunk.done = true;
|
233
|
+
break;
|
234
|
+
}
|
235
|
+
});
|
236
|
+
if (Date.now() - lastEmitted < 1000) {
|
237
|
+
return;
|
238
|
+
}
|
239
|
+
const chunkList = Object.values(chunks);
|
240
|
+
let totals = {
|
241
|
+
downloading: {
|
242
|
+
total: 0,
|
243
|
+
current: 0,
|
244
|
+
},
|
245
|
+
extracting: {
|
246
|
+
total: 0,
|
247
|
+
current: 0,
|
248
|
+
},
|
249
|
+
total: chunkList.length,
|
250
|
+
done: 0,
|
251
|
+
};
|
252
|
+
chunkList.forEach((chunk) => {
|
253
|
+
if (chunk.downloading.current > 0) {
|
254
|
+
totals.downloading.current += chunk.downloading.current;
|
255
|
+
}
|
256
|
+
if (chunk.downloading.total > 0) {
|
257
|
+
totals.downloading.total += chunk.downloading.total;
|
258
|
+
}
|
259
|
+
if (chunk.extracting.current > 0) {
|
260
|
+
totals.extracting.current += chunk.extracting.current;
|
261
|
+
}
|
262
|
+
if (chunk.extracting.total > 0) {
|
263
|
+
totals.extracting.total += chunk.extracting.total;
|
264
|
+
}
|
265
|
+
if (chunk.done) {
|
266
|
+
totals.done++;
|
267
|
+
}
|
268
|
+
});
|
269
|
+
const percent = totals.total > 0 ? (totals.done / totals.total) * 100 : 0;
|
270
|
+
//We emit at most every second to not spam the client
|
271
|
+
socketManager.emitGlobal(EVENT_IMAGE_PULL, {
|
272
|
+
image,
|
273
|
+
percent,
|
274
|
+
status: totals,
|
275
|
+
timeTaken: Date.now() - timeStarted,
|
276
|
+
});
|
277
|
+
lastEmitted = Date.now();
|
278
|
+
//console.log('Pulling image %s: %s % [done: %s, total: %s]', image, Math.round(percent), totals.done, totals.total);
|
171
279
|
});
|
172
280
|
IMAGE_PULL_CACHE[image] = Date.now();
|
173
|
-
|
281
|
+
socketManager.emitGlobal(EVENT_IMAGE_PULL, { image, percent: 100, timeTaken: Date.now() - timeStarted });
|
174
282
|
return true;
|
175
283
|
}
|
176
284
|
toDockerMounts(mounts) {
|
@@ -210,13 +318,7 @@ class ContainerManager {
|
|
210
318
|
return container;
|
211
319
|
}
|
212
320
|
async createOrUpdateContainer(opts) {
|
213
|
-
let imagePulled =
|
214
|
-
try {
|
215
|
-
imagePulled = await this.pull(opts.Image);
|
216
|
-
}
|
217
|
-
catch (e) {
|
218
|
-
console.warn('Failed to pull image. Continuing...', e);
|
219
|
-
}
|
321
|
+
let imagePulled = await this.pull(opts.Image);
|
220
322
|
this.applyHash(opts);
|
221
323
|
if (!opts.name) {
|
222
324
|
console.log('Starting unnamed container: %s', opts.Image);
|
@@ -341,12 +443,14 @@ class ContainerManager {
|
|
341
443
|
const containerName = getBlockInstanceContainerName(instance.systemId, instance.instanceId);
|
342
444
|
const containerInfo = await this.getContainerByName(containerName);
|
343
445
|
if (!containerInfo) {
|
344
|
-
return [
|
345
|
-
|
346
|
-
|
446
|
+
return [
|
447
|
+
{
|
448
|
+
source: 'stdout',
|
449
|
+
level: 'ERROR',
|
347
450
|
time: Date.now(),
|
348
|
-
message:
|
349
|
-
}
|
451
|
+
message: 'Container not found',
|
452
|
+
},
|
453
|
+
];
|
350
454
|
}
|
351
455
|
return containerInfo.getLogs();
|
352
456
|
}
|
@@ -434,13 +538,13 @@ export class ContainerInfo {
|
|
434
538
|
return ports;
|
435
539
|
}
|
436
540
|
async getLogs() {
|
437
|
-
const logStream = await this.native.logs({
|
541
|
+
const logStream = (await this.native.logs({
|
438
542
|
stdout: true,
|
439
543
|
stderr: true,
|
440
544
|
follow: false,
|
441
545
|
tail: 100,
|
442
546
|
timestamps: true,
|
443
|
-
});
|
547
|
+
}));
|
444
548
|
const out = [];
|
445
549
|
await promisifyStream(logStream, (data) => {
|
446
550
|
const buf = data;
|
@@ -7,6 +7,7 @@ declare class DefinitionsManager {
|
|
7
7
|
getDefinitions(kindFilter?: string | string[]): DefinitionInfo[];
|
8
8
|
exists(ref: string): boolean;
|
9
9
|
getProviderDefinitions(): DefinitionInfo[];
|
10
|
+
getDefinition(ref: string): DefinitionInfo | undefined;
|
10
11
|
}
|
11
12
|
export declare const definitionsManager: DefinitionsManager;
|
12
13
|
export {};
|
@@ -1,5 +1,5 @@
|
|
1
1
|
import ClusterConfiguration from '@kapeta/local-cluster-config';
|
2
|
-
import { parseKapetaUri } from
|
2
|
+
import { parseKapetaUri } from '@kapeta/nodejs-utils';
|
3
3
|
const CACHE_TTL = 60 * 1000; // 1 min
|
4
4
|
class DefinitionsManager {
|
5
5
|
cache = {};
|
@@ -33,13 +33,16 @@ class DefinitionsManager {
|
|
33
33
|
return this.doCached(key, () => ClusterConfiguration.getDefinitions(kindFilter));
|
34
34
|
}
|
35
35
|
exists(ref) {
|
36
|
-
|
37
|
-
return !!this.getDefinitions().find((d) => {
|
38
|
-
return parseKapetaUri(`${d.definition.metadata.name}:${d.version}`).id === uri.id;
|
39
|
-
});
|
36
|
+
return !!this.getDefinition(ref);
|
40
37
|
}
|
41
38
|
getProviderDefinitions() {
|
42
39
|
return this.doCached('providers', () => ClusterConfiguration.getProviderDefinitions());
|
43
40
|
}
|
41
|
+
getDefinition(ref) {
|
42
|
+
const uri = parseKapetaUri(ref);
|
43
|
+
return this.getDefinitions().find((d) => {
|
44
|
+
return parseKapetaUri(`${d.definition.metadata.name}:${d.version}`).id === uri.id;
|
45
|
+
});
|
46
|
+
}
|
44
47
|
}
|
45
48
|
export const definitionsManager = new DefinitionsManager();
|
@@ -23,7 +23,14 @@ export declare class InstanceManager {
|
|
23
23
|
private stopInner;
|
24
24
|
stopAllForPlan(systemId: string): Promise<void>;
|
25
25
|
start(systemId: string, instanceId: string): Promise<InstanceInfo>;
|
26
|
-
|
26
|
+
/**
|
27
|
+
* Stops an instance but does not remove it from the list of active instances
|
28
|
+
*
|
29
|
+
* It will be started again next time the system checks the status of the instance
|
30
|
+
*
|
31
|
+
* We do it this way to not cause the user to wait for the instance to start again
|
32
|
+
*/
|
33
|
+
prepareForRestart(systemId: string, instanceId: string): Promise<void>;
|
27
34
|
stopAll(): Promise<void>;
|
28
35
|
private stopInstances;
|
29
36
|
private save;
|
@@ -10,6 +10,9 @@ import { containerManager, HEALTH_CHECK_TIMEOUT } from './containerManager';
|
|
10
10
|
import { configManager } from './configManager';
|
11
11
|
import { DesiredInstanceStatus, InstanceOwner, InstanceStatus, InstanceType } from './types';
|
12
12
|
import { getBlockInstanceContainerName, normalizeKapetaUri } from './utils/utils';
|
13
|
+
import { KIND_OPERATOR, operatorManager } from './operatorManager';
|
14
|
+
import { parseKapetaUri } from '@kapeta/nodejs-utils';
|
15
|
+
import { definitionsManager } from './definitionsManager';
|
13
16
|
const CHECK_INTERVAL = 5000;
|
14
17
|
const DEFAULT_HEALTH_PORT_TYPE = 'rest';
|
15
18
|
const EVENT_STATUS_CHANGED = 'status-changed';
|
@@ -55,7 +58,10 @@ export class InstanceManager {
|
|
55
58
|
async exclusive(systemId, instanceId, fn) {
|
56
59
|
systemId = normalizeKapetaUri(systemId);
|
57
60
|
const key = `${systemId}/${instanceId}`;
|
58
|
-
|
61
|
+
//console.log(`Acquiring lock for ${key}`, this.instanceLocks.isBusy(key));
|
62
|
+
const result = await this.instanceLocks.acquire(key, fn);
|
63
|
+
//console.log(`Releasing lock for ${key}`, this.instanceLocks.isBusy(key));
|
64
|
+
return result;
|
59
65
|
}
|
60
66
|
async getLogs(systemId, instanceId) {
|
61
67
|
const instance = this.getInstance(systemId, instanceId);
|
@@ -66,19 +72,23 @@ export class InstanceManager {
|
|
66
72
|
case InstanceType.DOCKER:
|
67
73
|
return await containerManager.getLogs(instance);
|
68
74
|
case InstanceType.UNKNOWN:
|
69
|
-
return [
|
75
|
+
return [
|
76
|
+
{
|
70
77
|
level: 'INFO',
|
71
78
|
message: 'Instance is starting...',
|
72
79
|
time: Date.now(),
|
73
80
|
source: 'stdout',
|
74
|
-
}
|
81
|
+
},
|
82
|
+
];
|
75
83
|
case InstanceType.LOCAL:
|
76
|
-
return [
|
84
|
+
return [
|
85
|
+
{
|
77
86
|
level: 'INFO',
|
78
87
|
message: 'Instance started outside Kapeta - logs not available...',
|
79
88
|
time: Date.now(),
|
80
89
|
source: 'stdout',
|
81
|
-
}
|
90
|
+
},
|
91
|
+
];
|
82
92
|
}
|
83
93
|
return [];
|
84
94
|
}
|
@@ -119,7 +129,8 @@ export class InstanceManager {
|
|
119
129
|
const address = await serviceManager.getProviderAddress(systemId, instanceId, info.portType ?? DEFAULT_HEALTH_PORT_TYPE);
|
120
130
|
const healthUrl = this.getHealthUrl(info, address);
|
121
131
|
if (instance) {
|
122
|
-
if (instance.status === InstanceStatus.STOPPING &&
|
132
|
+
if (instance.status === InstanceStatus.STOPPING &&
|
133
|
+
instance.desiredStatus === DesiredInstanceStatus.STOP) {
|
123
134
|
//If instance is stopping do not interfere
|
124
135
|
return;
|
125
136
|
}
|
@@ -229,8 +240,7 @@ export class InstanceManager {
|
|
229
240
|
if (instance.status === InstanceStatus.STOPPED) {
|
230
241
|
return;
|
231
242
|
}
|
232
|
-
if (changeDesired &&
|
233
|
-
instance.desiredStatus !== DesiredInstanceStatus.EXTERNAL) {
|
243
|
+
if (changeDesired && instance.desiredStatus !== DesiredInstanceStatus.EXTERNAL) {
|
234
244
|
instance.desiredStatus = DesiredInstanceStatus.STOP;
|
235
245
|
}
|
236
246
|
instance.status = InstanceStatus.STOPPING;
|
@@ -324,6 +334,24 @@ export class InstanceManager {
|
|
324
334
|
console.log('Starting instance: %s::%s [desired: %s]', systemId, instanceId, instance.desiredStatus);
|
325
335
|
// Save the instance before starting it, so that we can track the status
|
326
336
|
await this.saveInternalInstance(instance);
|
337
|
+
const blockSpec = blockAsset.data.spec;
|
338
|
+
if (blockSpec.consumers) {
|
339
|
+
const promises = blockSpec.consumers.map((consumer) => {
|
340
|
+
const consumerUri = parseKapetaUri(consumer.kind);
|
341
|
+
const asset = definitionsManager.getDefinition(consumer.kind);
|
342
|
+
if (!asset) {
|
343
|
+
// Definition not found
|
344
|
+
return Promise.resolve();
|
345
|
+
}
|
346
|
+
if (KIND_OPERATOR.toLowerCase() !== asset.definition.kind.toLowerCase()) {
|
347
|
+
// Not an operator
|
348
|
+
return Promise.resolve();
|
349
|
+
}
|
350
|
+
console.log('Ensuring resource: %s in %s', consumerUri.id, systemId);
|
351
|
+
return operatorManager.ensureResource(systemId, consumerUri.fullName, consumerUri.version);
|
352
|
+
});
|
353
|
+
await Promise.all(promises);
|
354
|
+
}
|
327
355
|
if (existingInstance) {
|
328
356
|
// Check if the instance is already running - but after we've commmuicated the desired status
|
329
357
|
const currentStatus = await this.requestInstanceStatus(existingInstance);
|
@@ -348,7 +376,7 @@ export class InstanceManager {
|
|
348
376
|
});
|
349
377
|
}
|
350
378
|
catch (e) {
|
351
|
-
console.warn('Failed to start instance', e);
|
379
|
+
console.warn('Failed to start instance: ', systemId, instanceId, blockRef, e.message);
|
352
380
|
const logs = [
|
353
381
|
{
|
354
382
|
source: 'stdout',
|
@@ -364,6 +392,7 @@ export class InstanceManager {
|
|
364
392
|
health: null,
|
365
393
|
portType: DEFAULT_HEALTH_PORT_TYPE,
|
366
394
|
status: InstanceStatus.FAILED,
|
395
|
+
errorMessage: e.message ?? 'Failed to start - Check logs for details.',
|
367
396
|
});
|
368
397
|
this.emitInstanceEvent(systemId, instanceId, EVENT_INSTANCE_LOG, logs[0]);
|
369
398
|
this.emitInstanceEvent(systemId, blockInstance.id, EVENT_INSTANCE_EXITED, {
|
@@ -375,15 +404,16 @@ export class InstanceManager {
|
|
375
404
|
}
|
376
405
|
});
|
377
406
|
}
|
378
|
-
|
407
|
+
/**
|
408
|
+
* Stops an instance but does not remove it from the list of active instances
|
409
|
+
*
|
410
|
+
* It will be started again next time the system checks the status of the instance
|
411
|
+
*
|
412
|
+
* We do it this way to not cause the user to wait for the instance to start again
|
413
|
+
*/
|
414
|
+
async prepareForRestart(systemId, instanceId) {
|
379
415
|
systemId = normalizeKapetaUri(systemId);
|
380
416
|
await this.stopInner(systemId, instanceId);
|
381
|
-
const existingInstance = this.getInstance(systemId, instanceId);
|
382
|
-
if (existingInstance?.desiredStatus === DesiredInstanceStatus.STOP) {
|
383
|
-
// Internal instance was marked as stopped - abort restart
|
384
|
-
return existingInstance;
|
385
|
-
}
|
386
|
-
return this.start(systemId, instanceId);
|
387
417
|
}
|
388
418
|
async stopAll() {
|
389
419
|
return this.stopInstances(this._instances);
|
@@ -441,8 +471,7 @@ export class InstanceManager {
|
|
441
471
|
const oldStatus = instance.status;
|
442
472
|
const skipUpdate = (newStatus === InstanceStatus.STOPPED && instance.status === InstanceStatus.FAILED) ||
|
443
473
|
([InstanceStatus.READY, InstanceStatus.UNHEALTHY].includes(newStatus) &&
|
444
|
-
instance.status === InstanceStatus.STOPPING
|
445
|
-
instance.desiredStatus === DesiredInstanceStatus.STOP) ||
|
474
|
+
instance.status === InstanceStatus.STOPPING) ||
|
446
475
|
(newStatus === InstanceStatus.STOPPED &&
|
447
476
|
instance.status === InstanceStatus.STARTING &&
|
448
477
|
instance.desiredStatus === DesiredInstanceStatus.RUN);
|
@@ -482,7 +511,7 @@ export class InstanceManager {
|
|
482
511
|
//If the instance is unhealthy, try to restart it
|
483
512
|
console.log('Restarting unhealthy instance', instance);
|
484
513
|
try {
|
485
|
-
await this.
|
514
|
+
await this.prepareForRestart(instance.systemId, instance.instanceId);
|
486
515
|
}
|
487
516
|
catch (e) {
|
488
517
|
console.warn('Failed to restart instance', instance.systemId, instance.instanceId, e);
|
@@ -1,5 +1,6 @@
|
|
1
1
|
import { ContainerInfo } from './containerManager';
|
2
2
|
import { EnvironmentType, OperatorInfo } from './types';
|
3
|
+
export declare const KIND_OPERATOR = "core/resource-type-operator";
|
3
4
|
declare class Operator {
|
4
5
|
private _data;
|
5
6
|
constructor(data: any);
|
@@ -8,6 +9,7 @@ declare class Operator {
|
|
8
9
|
}
|
9
10
|
declare class OperatorManager {
|
10
11
|
private _mountDir;
|
12
|
+
private operatorLock;
|
11
13
|
constructor();
|
12
14
|
_getMountPoint(operatorType: string, mountName: string): string;
|
13
15
|
/**
|
@@ -8,7 +8,8 @@ import FSExtra from 'fs-extra';
|
|
8
8
|
import { definitionsManager } from './definitionsManager';
|
9
9
|
import { getBindHost, normalizeKapetaUri } from './utils/utils';
|
10
10
|
import _ from 'lodash';
|
11
|
-
|
11
|
+
import AsyncLock from 'async-lock';
|
12
|
+
export const KIND_OPERATOR = 'core/resource-type-operator';
|
12
13
|
class Operator {
|
13
14
|
_data;
|
14
15
|
constructor(data) {
|
@@ -23,6 +24,7 @@ class Operator {
|
|
23
24
|
}
|
24
25
|
class OperatorManager {
|
25
26
|
_mountDir;
|
27
|
+
operatorLock = new AsyncLock();
|
26
28
|
constructor() {
|
27
29
|
this._mountDir = Path.join(storageService.getKapetaBasedir(), 'mounts');
|
28
30
|
FSExtra.mkdirpSync(this._mountDir);
|
@@ -107,74 +109,74 @@ class OperatorManager {
|
|
107
109
|
* @return {Promise<ContainerInfo>}
|
108
110
|
*/
|
109
111
|
async ensureResource(systemId, resourceType, version) {
|
110
|
-
|
111
|
-
const
|
112
|
-
|
113
|
-
|
114
|
-
|
115
|
-
|
116
|
-
|
117
|
-
|
118
|
-
|
119
|
-
|
120
|
-
containerPortInfo =
|
121
|
-
|
122
|
-
|
123
|
-
|
112
|
+
systemId = normalizeKapetaUri(systemId);
|
113
|
+
const key = `${systemId}#${resourceType}:${version}`;
|
114
|
+
return await this.operatorLock.acquire(key, async () => {
|
115
|
+
const operator = this.getOperator(resourceType, version);
|
116
|
+
const operatorData = operator.getData();
|
117
|
+
const portTypes = Object.keys(operatorData.ports);
|
118
|
+
portTypes.sort();
|
119
|
+
const ports = {};
|
120
|
+
for (let i = 0; i < portTypes.length; i++) {
|
121
|
+
const portType = portTypes[i];
|
122
|
+
let containerPortInfo = operatorData.ports[portType];
|
123
|
+
const hostPort = await serviceManager.ensureServicePort(systemId, resourceType, portType);
|
124
|
+
if (typeof containerPortInfo === 'number' || typeof containerPortInfo === 'string') {
|
125
|
+
containerPortInfo = { port: containerPortInfo, type: 'tcp' };
|
126
|
+
}
|
127
|
+
if (!containerPortInfo.type) {
|
128
|
+
containerPortInfo.type = 'tcp';
|
129
|
+
}
|
130
|
+
const portId = containerPortInfo.port + '/' + containerPortInfo.type;
|
131
|
+
ports[portId] = {
|
132
|
+
type: portType,
|
133
|
+
hostPort,
|
134
|
+
};
|
124
135
|
}
|
125
|
-
const
|
126
|
-
|
127
|
-
|
128
|
-
|
136
|
+
const mounts = await containerManager.createMounts(systemId, resourceType, operatorData.mounts);
|
137
|
+
const nameParts = [systemId, resourceType.toLowerCase(), version];
|
138
|
+
const containerName = `kapeta-resource-${md5(nameParts.join('_'))}`;
|
139
|
+
const PortBindings = {};
|
140
|
+
const Env = [];
|
141
|
+
const Labels = {
|
142
|
+
kapeta: 'true',
|
129
143
|
};
|
130
|
-
|
131
|
-
|
132
|
-
|
133
|
-
|
134
|
-
|
135
|
-
|
136
|
-
|
137
|
-
|
138
|
-
|
139
|
-
|
140
|
-
|
141
|
-
|
142
|
-
|
143
|
-
|
144
|
-
|
145
|
-
|
146
|
-
|
147
|
-
|
148
|
-
|
149
|
-
|
150
|
-
|
144
|
+
const bindHost = getBindHost();
|
145
|
+
const ExposedPorts = {};
|
146
|
+
_.forEach(ports, (portInfo, containerPort) => {
|
147
|
+
ExposedPorts['' + containerPort] = {};
|
148
|
+
PortBindings['' + containerPort] = [
|
149
|
+
{
|
150
|
+
HostPort: '' + portInfo.hostPort,
|
151
|
+
HostIp: bindHost,
|
152
|
+
},
|
153
|
+
];
|
154
|
+
Labels[CONTAINER_LABEL_PORT_PREFIX + portInfo.hostPort] = portInfo.type;
|
155
|
+
});
|
156
|
+
const Mounts = containerManager.toDockerMounts(mounts);
|
157
|
+
_.forEach(operatorData.env, (value, name) => {
|
158
|
+
Env.push(name + '=' + value);
|
159
|
+
});
|
160
|
+
let HealthCheck = undefined;
|
161
|
+
if (operatorData.health) {
|
162
|
+
HealthCheck = containerManager.toDockerHealth(operatorData.health);
|
163
|
+
}
|
164
|
+
const container = await containerManager.ensureContainer({
|
165
|
+
name: containerName,
|
166
|
+
Image: operatorData.image,
|
167
|
+
Hostname: containerName + '.kapeta',
|
168
|
+
Labels,
|
169
|
+
Cmd: operatorData.cmd,
|
170
|
+
ExposedPorts,
|
171
|
+
Env,
|
172
|
+
HealthCheck,
|
173
|
+
HostConfig: {
|
174
|
+
PortBindings,
|
175
|
+
Mounts,
|
151
176
|
},
|
152
|
-
|
153
|
-
|
154
|
-
});
|
155
|
-
const Mounts = containerManager.toDockerMounts(mounts);
|
156
|
-
_.forEach(operatorData.env, (value, name) => {
|
157
|
-
Env.push(name + '=' + value);
|
158
|
-
});
|
159
|
-
let HealthCheck = undefined;
|
160
|
-
if (operatorData.health) {
|
161
|
-
HealthCheck = containerManager.toDockerHealth(operatorData.health);
|
162
|
-
}
|
163
|
-
const container = await containerManager.ensureContainer({
|
164
|
-
name: containerName,
|
165
|
-
Image: operatorData.image,
|
166
|
-
Hostname: containerName + '.kapeta',
|
167
|
-
Labels,
|
168
|
-
Cmd: operatorData.cmd,
|
169
|
-
ExposedPorts,
|
170
|
-
Env,
|
171
|
-
HealthCheck,
|
172
|
-
HostConfig: {
|
173
|
-
PortBindings,
|
174
|
-
Mounts,
|
175
|
-
},
|
177
|
+
});
|
178
|
+
return new ContainerInfo(container);
|
176
179
|
});
|
177
|
-
return new ContainerInfo(container);
|
178
180
|
}
|
179
181
|
}
|
180
182
|
export const operatorManager = new OperatorManager();
|
@@ -7,6 +7,7 @@ export declare class SocketManager {
|
|
7
7
|
isAlive(): boolean;
|
8
8
|
private get io();
|
9
9
|
emit(context: string, type: string, payload: any): void;
|
10
|
+
emitGlobal(type: string, payload: any): void;
|
10
11
|
_bindIO(): void;
|
11
12
|
_handleSocketCreated(socket: Socket): void;
|
12
13
|
_bindSocket(socket: Socket): void;
|
@@ -23,6 +23,9 @@ export class SocketManager {
|
|
23
23
|
emit(context, type, payload) {
|
24
24
|
this.io.to(context).emit(type, { context, payload });
|
25
25
|
}
|
26
|
+
emitGlobal(type, payload) {
|
27
|
+
this.io.emit(type, { payload });
|
28
|
+
}
|
26
29
|
_bindIO() {
|
27
30
|
this.io.on('connection', (socket) => this._handleSocketCreated(socket));
|
28
31
|
}
|