@kapeta/local-cluster-service 0.10.1 → 0.11.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +14 -0
- package/dist/cjs/src/containerManager.d.ts +6 -4
- package/dist/cjs/src/containerManager.js +100 -45
- package/dist/cjs/src/definitionsManager.d.ts +1 -0
- package/dist/cjs/src/definitionsManager.js +7 -0
- package/dist/cjs/src/instanceManager.d.ts +6 -2
- package/dist/cjs/src/instanceManager.js +240 -233
- package/dist/cjs/src/instances/routes.js +10 -4
- package/dist/cjs/src/operatorManager.js +8 -6
- package/dist/cjs/src/repositoryManager.js +4 -4
- package/dist/cjs/src/types.d.ts +0 -9
- package/dist/cjs/src/utils/BlockInstanceRunner.js +9 -64
- package/dist/cjs/src/utils/utils.d.ts +1 -1
- package/dist/cjs/src/utils/utils.js +3 -2
- package/dist/esm/src/containerManager.d.ts +6 -4
- package/dist/esm/src/containerManager.js +100 -45
- package/dist/esm/src/definitionsManager.d.ts +1 -0
- package/dist/esm/src/definitionsManager.js +7 -0
- package/dist/esm/src/instanceManager.d.ts +6 -2
- package/dist/esm/src/instanceManager.js +240 -233
- package/dist/esm/src/instances/routes.js +10 -4
- package/dist/esm/src/operatorManager.js +8 -6
- package/dist/esm/src/repositoryManager.js +4 -4
- package/dist/esm/src/types.d.ts +0 -9
- package/dist/esm/src/utils/BlockInstanceRunner.js +9 -64
- package/dist/esm/src/utils/utils.d.ts +1 -1
- package/dist/esm/src/utils/utils.js +3 -2
- package/package.json +3 -1
- package/src/containerManager.ts +126 -49
- package/src/definitionsManager.ts +8 -0
- package/src/instanceManager.ts +270 -255
- package/src/instances/routes.ts +9 -4
- package/src/operatorManager.ts +9 -8
- package/src/repositoryManager.ts +5 -5
- package/src/types.ts +0 -7
- package/src/utils/BlockInstanceRunner.ts +10 -66
- package/src/utils/LogData.ts +1 -0
- package/src/utils/utils.ts +3 -2
@@ -1,5 +1,6 @@
|
|
1
1
|
import _ from 'lodash';
|
2
2
|
import request from 'request';
|
3
|
+
import AsyncLock from 'async-lock';
|
3
4
|
import { BlockInstanceRunner } from './utils/BlockInstanceRunner';
|
4
5
|
import { storageService } from './storageService';
|
5
6
|
import { socketManager } from './socketManager';
|
@@ -19,6 +20,7 @@ const MIN_TIME_RUNNING = 30000; //If something didnt run for more than 30 secs -
|
|
19
20
|
export class InstanceManager {
|
20
21
|
_interval = undefined;
|
21
22
|
_instances = [];
|
23
|
+
instanceLocks = new AsyncLock();
|
22
24
|
constructor() {
|
23
25
|
this._instances = storageService.section('instances', []);
|
24
26
|
// We need to wait a bit before running the first check
|
@@ -50,6 +52,36 @@ export class InstanceManager {
|
|
50
52
|
systemId = normalizeKapetaUri(systemId);
|
51
53
|
return this._instances.find((i) => i.systemId === systemId && i.instanceId === instanceId);
|
52
54
|
}
|
55
|
+
async exclusive(systemId, instanceId, fn) {
|
56
|
+
systemId = normalizeKapetaUri(systemId);
|
57
|
+
const key = `${systemId}/${instanceId}`;
|
58
|
+
return this.instanceLocks.acquire(key, fn);
|
59
|
+
}
|
60
|
+
async getLogs(systemId, instanceId) {
|
61
|
+
const instance = this.getInstance(systemId, instanceId);
|
62
|
+
if (!instance) {
|
63
|
+
throw new Error(`Instance ${systemId}/${instanceId} not found`);
|
64
|
+
}
|
65
|
+
switch (instance.type) {
|
66
|
+
case InstanceType.DOCKER:
|
67
|
+
return await containerManager.getLogs(instance);
|
68
|
+
case InstanceType.UNKNOWN:
|
69
|
+
return [{
|
70
|
+
level: 'INFO',
|
71
|
+
message: 'Instance is starting...',
|
72
|
+
time: Date.now(),
|
73
|
+
source: 'stdout',
|
74
|
+
}];
|
75
|
+
case InstanceType.LOCAL:
|
76
|
+
return [{
|
77
|
+
level: 'INFO',
|
78
|
+
message: 'Instance started outside Kapeta - logs not available...',
|
79
|
+
time: Date.now(),
|
80
|
+
source: 'stdout',
|
81
|
+
}];
|
82
|
+
}
|
83
|
+
return [];
|
84
|
+
}
|
53
85
|
async saveInternalInstance(instance) {
|
54
86
|
instance.systemId = normalizeKapetaUri(instance.systemId);
|
55
87
|
if (instance.ref) {
|
@@ -80,58 +112,59 @@ export class InstanceManager {
|
|
80
112
|
* which self-registers with the cluster service locally on startup.
|
81
113
|
*/
|
82
114
|
async registerInstanceFromSDK(systemId, instanceId, info) {
|
83
|
-
systemId
|
84
|
-
|
85
|
-
|
86
|
-
|
87
|
-
|
88
|
-
|
89
|
-
if (instance
|
90
|
-
|
91
|
-
|
92
|
-
|
93
|
-
if (info.owner === InstanceOwner.EXTERNAL) {
|
94
|
-
//If instance was started externally - then we want to replace the internal instance with that
|
95
|
-
if (instance.owner === InstanceOwner.INTERNAL &&
|
96
|
-
(instance.status === InstanceStatus.READY ||
|
97
|
-
instance.status === InstanceStatus.STARTING ||
|
98
|
-
instance.status === InstanceStatus.UNHEALTHY)) {
|
99
|
-
throw new Error(`Instance ${instanceId} is already running`);
|
115
|
+
return this.exclusive(systemId, instanceId, async () => {
|
116
|
+
systemId = normalizeKapetaUri(systemId);
|
117
|
+
let instance = this.getInstance(systemId, instanceId);
|
118
|
+
//Get target address
|
119
|
+
const address = await serviceManager.getProviderAddress(systemId, instanceId, info.portType ?? DEFAULT_HEALTH_PORT_TYPE);
|
120
|
+
const healthUrl = this.getHealthUrl(info, address);
|
121
|
+
if (instance) {
|
122
|
+
if (instance.status === InstanceStatus.STOPPING && instance.desiredStatus === DesiredInstanceStatus.STOP) {
|
123
|
+
//If instance is stopping do not interfere
|
124
|
+
return;
|
100
125
|
}
|
101
|
-
|
102
|
-
|
103
|
-
|
104
|
-
|
105
|
-
|
106
|
-
|
107
|
-
|
108
|
-
|
109
|
-
|
110
|
-
|
126
|
+
if (info.owner === InstanceOwner.EXTERNAL) {
|
127
|
+
//If instance was started externally - then we want to replace the internal instance with that
|
128
|
+
if (instance.owner === InstanceOwner.INTERNAL &&
|
129
|
+
(instance.status === InstanceStatus.READY ||
|
130
|
+
instance.status === InstanceStatus.STARTING ||
|
131
|
+
instance.status === InstanceStatus.UNHEALTHY)) {
|
132
|
+
throw new Error(`Instance ${instanceId} is already running`);
|
133
|
+
}
|
134
|
+
instance.desiredStatus = info.desiredStatus;
|
135
|
+
instance.owner = info.owner;
|
136
|
+
instance.status = InstanceStatus.STARTING;
|
137
|
+
instance.startedAt = Date.now();
|
138
|
+
}
|
139
|
+
instance.pid = info.pid;
|
140
|
+
instance.address = address;
|
141
|
+
if (info.type) {
|
142
|
+
instance.type = info.type;
|
143
|
+
}
|
144
|
+
if (healthUrl) {
|
145
|
+
instance.health = healthUrl;
|
146
|
+
}
|
147
|
+
this.emitSystemEvent(systemId, EVENT_STATUS_CHANGED, instance);
|
111
148
|
}
|
112
|
-
|
113
|
-
instance
|
149
|
+
else {
|
150
|
+
//If instance was not found - then we're receiving an externally started instance
|
151
|
+
instance = {
|
152
|
+
...info,
|
153
|
+
systemId,
|
154
|
+
instanceId,
|
155
|
+
status: InstanceStatus.STARTING,
|
156
|
+
startedAt: Date.now(),
|
157
|
+
desiredStatus: DesiredInstanceStatus.EXTERNAL,
|
158
|
+
owner: InstanceOwner.EXTERNAL,
|
159
|
+
health: healthUrl,
|
160
|
+
address,
|
161
|
+
};
|
162
|
+
this._instances.push(instance);
|
163
|
+
this.emitSystemEvent(systemId, EVENT_INSTANCE_CREATED, instance);
|
114
164
|
}
|
115
|
-
this.
|
116
|
-
|
117
|
-
|
118
|
-
//If instance was not found - then we're receiving an externally started instance
|
119
|
-
instance = {
|
120
|
-
...info,
|
121
|
-
systemId,
|
122
|
-
instanceId,
|
123
|
-
status: InstanceStatus.STARTING,
|
124
|
-
startedAt: Date.now(),
|
125
|
-
desiredStatus: DesiredInstanceStatus.EXTERNAL,
|
126
|
-
owner: InstanceOwner.EXTERNAL,
|
127
|
-
health: healthUrl,
|
128
|
-
address,
|
129
|
-
};
|
130
|
-
this._instances.push(instance);
|
131
|
-
this.emitSystemEvent(systemId, EVENT_INSTANCE_CREATED, instance);
|
132
|
-
}
|
133
|
-
this.save();
|
134
|
-
return instance;
|
165
|
+
this.save();
|
166
|
+
return instance;
|
167
|
+
});
|
135
168
|
}
|
136
169
|
getHealthUrl(info, address) {
|
137
170
|
let healthUrl = null;
|
@@ -145,15 +178,17 @@ export class InstanceManager {
|
|
145
178
|
return healthUrl;
|
146
179
|
}
|
147
180
|
markAsStopped(systemId, instanceId) {
|
148
|
-
systemId
|
149
|
-
|
150
|
-
|
151
|
-
instance.status
|
152
|
-
|
153
|
-
|
154
|
-
|
155
|
-
|
156
|
-
|
181
|
+
return this.exclusive(systemId, instanceId, async () => {
|
182
|
+
systemId = normalizeKapetaUri(systemId);
|
183
|
+
const instance = _.find(this._instances, { systemId, instanceId });
|
184
|
+
if (instance && instance.owner === InstanceOwner.EXTERNAL && instance.status !== InstanceStatus.STOPPED) {
|
185
|
+
instance.status = InstanceStatus.STOPPED;
|
186
|
+
instance.pid = null;
|
187
|
+
instance.health = null;
|
188
|
+
this.emitSystemEvent(systemId, EVENT_STATUS_CHANGED, instance);
|
189
|
+
this.save();
|
190
|
+
}
|
191
|
+
});
|
157
192
|
}
|
158
193
|
async startAllForPlan(systemId) {
|
159
194
|
systemId = normalizeKapetaUri(systemId);
|
@@ -182,54 +217,60 @@ export class InstanceManager {
|
|
182
217
|
return settled.map((p) => (p.status === 'fulfilled' ? p.value : null)).filter((p) => !!p);
|
183
218
|
}
|
184
219
|
async stop(systemId, instanceId) {
|
185
|
-
systemId
|
186
|
-
|
187
|
-
|
188
|
-
|
189
|
-
|
190
|
-
|
191
|
-
|
192
|
-
|
193
|
-
|
194
|
-
instance.
|
195
|
-
|
196
|
-
|
197
|
-
|
198
|
-
|
199
|
-
|
200
|
-
|
201
|
-
|
202
|
-
|
203
|
-
|
204
|
-
|
205
|
-
|
206
|
-
|
207
|
-
|
208
|
-
|
209
|
-
|
220
|
+
return this.stopInner(systemId, instanceId, true);
|
221
|
+
}
|
222
|
+
async stopInner(systemId, instanceId, changeDesired = false) {
|
223
|
+
return this.exclusive(systemId, instanceId, async () => {
|
224
|
+
systemId = normalizeKapetaUri(systemId);
|
225
|
+
const instance = this.getInstance(systemId, instanceId);
|
226
|
+
if (!instance) {
|
227
|
+
return;
|
228
|
+
}
|
229
|
+
if (instance.status === InstanceStatus.STOPPED) {
|
230
|
+
return;
|
231
|
+
}
|
232
|
+
if (changeDesired &&
|
233
|
+
instance.desiredStatus !== DesiredInstanceStatus.EXTERNAL) {
|
234
|
+
instance.desiredStatus = DesiredInstanceStatus.STOP;
|
235
|
+
}
|
236
|
+
instance.status = InstanceStatus.STOPPING;
|
237
|
+
this.emitSystemEvent(systemId, EVENT_STATUS_CHANGED, instance);
|
238
|
+
console.log('Stopping instance: %s::%s [desired: %s]', systemId, instanceId, instance.desiredStatus);
|
239
|
+
this.save();
|
240
|
+
try {
|
241
|
+
if (instance.type === 'docker') {
|
242
|
+
const containerName = getBlockInstanceContainerName(instance.systemId, instance.instanceId);
|
243
|
+
const container = await containerManager.getContainerByName(containerName);
|
244
|
+
if (container) {
|
245
|
+
try {
|
246
|
+
await container.stop();
|
247
|
+
instance.status = InstanceStatus.STOPPED;
|
248
|
+
this.emitSystemEvent(systemId, EVENT_STATUS_CHANGED, instance);
|
249
|
+
this.save();
|
250
|
+
}
|
251
|
+
catch (e) {
|
252
|
+
console.error('Failed to stop container', e);
|
253
|
+
}
|
210
254
|
}
|
211
|
-
|
212
|
-
console.
|
255
|
+
else {
|
256
|
+
console.warn('Container not found', containerName);
|
213
257
|
}
|
258
|
+
return;
|
214
259
|
}
|
215
|
-
|
216
|
-
|
260
|
+
if (!instance.pid) {
|
261
|
+
instance.status = InstanceStatus.STOPPED;
|
262
|
+
this.save();
|
263
|
+
return;
|
217
264
|
}
|
218
|
-
|
219
|
-
}
|
220
|
-
if (!instance.pid) {
|
265
|
+
process.kill(instance.pid, 'SIGTERM');
|
221
266
|
instance.status = InstanceStatus.STOPPED;
|
267
|
+
this.emitSystemEvent(systemId, EVENT_STATUS_CHANGED, instance);
|
222
268
|
this.save();
|
223
|
-
return;
|
224
269
|
}
|
225
|
-
|
226
|
-
|
227
|
-
|
228
|
-
|
229
|
-
}
|
230
|
-
catch (e) {
|
231
|
-
console.error('Failed to stop process', e);
|
232
|
-
}
|
270
|
+
catch (e) {
|
271
|
+
console.error('Failed to stop process', e);
|
272
|
+
}
|
273
|
+
});
|
233
274
|
}
|
234
275
|
async stopAllForPlan(systemId) {
|
235
276
|
systemId = normalizeKapetaUri(systemId);
|
@@ -237,143 +278,111 @@ export class InstanceManager {
|
|
237
278
|
return this.stopInstances(instancesForPlan);
|
238
279
|
}
|
239
280
|
async start(systemId, instanceId) {
|
240
|
-
systemId
|
241
|
-
|
242
|
-
|
243
|
-
|
244
|
-
|
245
|
-
const blockInstance = plan.spec && plan.spec.blocks ? _.find(plan.spec.blocks, { id: instanceId }) : null;
|
246
|
-
if (!blockInstance) {
|
247
|
-
throw new Error('Block instance not found: ' + instanceId);
|
248
|
-
}
|
249
|
-
const blockRef = normalizeKapetaUri(blockInstance.block.ref);
|
250
|
-
const blockAsset = await assetManager.getAsset(blockRef, true);
|
251
|
-
if (!blockAsset) {
|
252
|
-
throw new Error('Block not found: ' + blockRef);
|
253
|
-
}
|
254
|
-
const existingInstance = this.getInstance(systemId, instanceId);
|
255
|
-
if (existingInstance) {
|
256
|
-
if (existingInstance.status === InstanceStatus.READY) {
|
257
|
-
// Instance is already running
|
258
|
-
return existingInstance;
|
281
|
+
return this.exclusive(systemId, instanceId, async () => {
|
282
|
+
systemId = normalizeKapetaUri(systemId);
|
283
|
+
const plan = await assetManager.getPlan(systemId, true);
|
284
|
+
if (!plan) {
|
285
|
+
throw new Error('Plan not found: ' + systemId);
|
259
286
|
}
|
260
|
-
|
261
|
-
|
262
|
-
|
263
|
-
return existingInstance;
|
287
|
+
const blockInstance = plan.spec && plan.spec.blocks ? _.find(plan.spec.blocks, { id: instanceId }) : null;
|
288
|
+
if (!blockInstance) {
|
289
|
+
throw new Error('Block instance not found: ' + instanceId);
|
264
290
|
}
|
265
|
-
|
266
|
-
|
267
|
-
|
268
|
-
|
291
|
+
const blockRef = normalizeKapetaUri(blockInstance.block.ref);
|
292
|
+
const blockAsset = await assetManager.getAsset(blockRef, true);
|
293
|
+
if (!blockAsset) {
|
294
|
+
throw new Error('Block not found: ' + blockRef);
|
269
295
|
}
|
270
|
-
|
271
|
-
|
272
|
-
|
273
|
-
|
274
|
-
|
275
|
-
name: blockAsset.data.metadata.name,
|
276
|
-
desiredStatus: DesiredInstanceStatus.RUN,
|
277
|
-
owner: InstanceOwner.INTERNAL,
|
278
|
-
type: InstanceType.UNKNOWN,
|
279
|
-
status: InstanceStatus.STARTING,
|
280
|
-
startedAt: Date.now(),
|
281
|
-
};
|
282
|
-
console.log('Starting instance: %s::%s [desired: %s]', systemId, instanceId, instance.desiredStatus);
|
283
|
-
// Save the instance before starting it, so that we can track the status
|
284
|
-
await this.saveInternalInstance(instance);
|
285
|
-
if (existingInstance) {
|
286
|
-
// Check if the instance is already running - but after we've commmuicated the desired status
|
287
|
-
const currentStatus = await this.requestInstanceStatus(existingInstance);
|
288
|
-
if (currentStatus === InstanceStatus.READY) {
|
289
|
-
// Instance is already running
|
290
|
-
return existingInstance;
|
291
|
-
}
|
292
|
-
}
|
293
|
-
const instanceConfig = await configManager.getConfigForSection(systemId, instanceId);
|
294
|
-
const runner = new BlockInstanceRunner(systemId);
|
295
|
-
const startTime = Date.now();
|
296
|
-
try {
|
297
|
-
const processInfo = await runner.start(blockRef, instanceId, instanceConfig);
|
298
|
-
//emit stdout/stderr via sockets
|
299
|
-
processInfo.output.on('data', (data) => {
|
300
|
-
const payload = {
|
301
|
-
source: 'stdout',
|
302
|
-
level: 'INFO',
|
303
|
-
message: data.toString(),
|
304
|
-
time: Date.now(),
|
305
|
-
};
|
306
|
-
this.emitInstanceEvent(systemId, instanceId, EVENT_INSTANCE_LOG, payload);
|
307
|
-
});
|
308
|
-
processInfo.output.on('exit', (exitCode) => {
|
309
|
-
const timeRunning = Date.now() - startTime;
|
310
|
-
const instance = this.getInstance(systemId, instanceId);
|
311
|
-
if (instance?.status === InstanceStatus.READY) {
|
312
|
-
//It's already been running
|
313
|
-
return;
|
296
|
+
const existingInstance = this.getInstance(systemId, instanceId);
|
297
|
+
if (existingInstance) {
|
298
|
+
if (existingInstance.status === InstanceStatus.READY) {
|
299
|
+
// Instance is already running
|
300
|
+
return existingInstance;
|
314
301
|
}
|
315
|
-
if (
|
316
|
-
|
317
|
-
//
|
318
|
-
return;
|
302
|
+
if (existingInstance.desiredStatus === DesiredInstanceStatus.RUN &&
|
303
|
+
existingInstance.status === InstanceStatus.STARTING) {
|
304
|
+
// Internal instance is already starting - don't start it again
|
305
|
+
return existingInstance;
|
319
306
|
}
|
320
|
-
if (
|
321
|
-
|
322
|
-
|
323
|
-
|
324
|
-
this.save();
|
325
|
-
}
|
326
|
-
this.emitSystemEvent(systemId, EVENT_INSTANCE_EXITED, {
|
327
|
-
error: 'Failed to start instance',
|
328
|
-
status: EVENT_INSTANCE_EXITED,
|
329
|
-
instanceId: blockInstance.id,
|
330
|
-
});
|
307
|
+
if (existingInstance.owner === InstanceOwner.EXTERNAL &&
|
308
|
+
existingInstance.status === InstanceStatus.STARTING) {
|
309
|
+
// External instance is already starting - don't start it again
|
310
|
+
return existingInstance;
|
331
311
|
}
|
332
|
-
}
|
333
|
-
instance
|
334
|
-
|
335
|
-
|
336
|
-
|
337
|
-
|
338
|
-
|
339
|
-
|
340
|
-
|
341
|
-
|
342
|
-
|
343
|
-
|
344
|
-
|
345
|
-
|
346
|
-
|
347
|
-
|
348
|
-
|
349
|
-
|
350
|
-
{
|
351
|
-
|
352
|
-
|
353
|
-
|
354
|
-
|
355
|
-
|
356
|
-
|
357
|
-
const
|
358
|
-
|
359
|
-
|
360
|
-
|
361
|
-
|
362
|
-
|
363
|
-
|
364
|
-
|
365
|
-
|
366
|
-
|
367
|
-
|
368
|
-
|
369
|
-
|
370
|
-
|
371
|
-
|
372
|
-
|
312
|
+
}
|
313
|
+
let instance = {
|
314
|
+
systemId,
|
315
|
+
instanceId,
|
316
|
+
ref: blockRef,
|
317
|
+
name: blockAsset.data.metadata.name,
|
318
|
+
desiredStatus: DesiredInstanceStatus.RUN,
|
319
|
+
owner: InstanceOwner.INTERNAL,
|
320
|
+
type: existingInstance?.type ?? InstanceType.UNKNOWN,
|
321
|
+
status: InstanceStatus.STARTING,
|
322
|
+
startedAt: Date.now(),
|
323
|
+
};
|
324
|
+
console.log('Starting instance: %s::%s [desired: %s]', systemId, instanceId, instance.desiredStatus);
|
325
|
+
// Save the instance before starting it, so that we can track the status
|
326
|
+
await this.saveInternalInstance(instance);
|
327
|
+
if (existingInstance) {
|
328
|
+
// Check if the instance is already running - but after we've commmuicated the desired status
|
329
|
+
const currentStatus = await this.requestInstanceStatus(existingInstance);
|
330
|
+
if (currentStatus === InstanceStatus.READY) {
|
331
|
+
// Instance is already running
|
332
|
+
return existingInstance;
|
333
|
+
}
|
334
|
+
}
|
335
|
+
const instanceConfig = await configManager.getConfigForSection(systemId, instanceId);
|
336
|
+
const runner = new BlockInstanceRunner(systemId);
|
337
|
+
const startTime = Date.now();
|
338
|
+
try {
|
339
|
+
const processInfo = await runner.start(blockRef, instanceId, instanceConfig);
|
340
|
+
instance.status = InstanceStatus.READY;
|
341
|
+
return this.saveInternalInstance({
|
342
|
+
...instance,
|
343
|
+
type: processInfo.type,
|
344
|
+
pid: processInfo.pid ?? -1,
|
345
|
+
health: null,
|
346
|
+
portType: processInfo.portType,
|
347
|
+
status: InstanceStatus.READY,
|
348
|
+
});
|
349
|
+
}
|
350
|
+
catch (e) {
|
351
|
+
console.warn('Failed to start instance', e);
|
352
|
+
const logs = [
|
353
|
+
{
|
354
|
+
source: 'stdout',
|
355
|
+
level: 'ERROR',
|
356
|
+
message: e.message,
|
357
|
+
time: Date.now(),
|
358
|
+
},
|
359
|
+
];
|
360
|
+
const out = await this.saveInternalInstance({
|
361
|
+
...instance,
|
362
|
+
type: InstanceType.LOCAL,
|
363
|
+
pid: null,
|
364
|
+
health: null,
|
365
|
+
portType: DEFAULT_HEALTH_PORT_TYPE,
|
366
|
+
status: InstanceStatus.FAILED,
|
367
|
+
});
|
368
|
+
this.emitInstanceEvent(systemId, instanceId, EVENT_INSTANCE_LOG, logs[0]);
|
369
|
+
this.emitInstanceEvent(systemId, blockInstance.id, EVENT_INSTANCE_EXITED, {
|
370
|
+
error: `Failed to start instance: ${e.message}`,
|
371
|
+
status: EVENT_INSTANCE_EXITED,
|
372
|
+
instanceId: blockInstance.id,
|
373
|
+
});
|
374
|
+
return out;
|
375
|
+
}
|
376
|
+
});
|
373
377
|
}
|
374
378
|
async restart(systemId, instanceId) {
|
375
379
|
systemId = normalizeKapetaUri(systemId);
|
376
|
-
await this.
|
380
|
+
await this.stopInner(systemId, instanceId);
|
381
|
+
const existingInstance = this.getInstance(systemId, instanceId);
|
382
|
+
if (existingInstance?.desiredStatus === DesiredInstanceStatus.STOP) {
|
383
|
+
// Internal instance was marked as stopped - abort restart
|
384
|
+
return existingInstance;
|
385
|
+
}
|
377
386
|
return this.start(systemId, instanceId);
|
378
387
|
}
|
379
388
|
async stopAll() {
|
@@ -387,9 +396,7 @@ export class InstanceManager {
|
|
387
396
|
save() {
|
388
397
|
try {
|
389
398
|
storageService.put('instances', this._instances.map((instance) => {
|
390
|
-
|
391
|
-
delete copy.internal;
|
392
|
-
return copy;
|
399
|
+
return { ...instance };
|
393
400
|
}));
|
394
401
|
}
|
395
402
|
catch (e) {
|
@@ -402,7 +409,7 @@ export class InstanceManager {
|
|
402
409
|
const all = [...this._instances];
|
403
410
|
while (all.length > 0) {
|
404
411
|
// Check a few instances at a time - docker doesn't like too many concurrent requests
|
405
|
-
const chunk = all.splice(0,
|
412
|
+
const chunk = all.splice(0, 30);
|
406
413
|
const promises = chunk.map(async (instance) => {
|
407
414
|
if (!instance.systemId) {
|
408
415
|
return;
|
@@ -462,7 +469,7 @@ export class InstanceManager {
|
|
462
469
|
[InstanceStatus.READY, InstanceStatus.STARTING, InstanceStatus.UNHEALTHY].includes(newStatus)) {
|
463
470
|
//If the instance is running but we want it to stop, stop it
|
464
471
|
try {
|
465
|
-
await this.
|
472
|
+
await this.stopInner(instance.systemId, instance.instanceId);
|
466
473
|
}
|
467
474
|
catch (e) {
|
468
475
|
console.warn('Failed to stop instance', instance.systemId, instance.instanceId, e);
|
@@ -491,7 +498,7 @@ export class InstanceManager {
|
|
491
498
|
}
|
492
499
|
async getExternalStatus(instance) {
|
493
500
|
if (instance.type === InstanceType.DOCKER) {
|
494
|
-
const containerName = getBlockInstanceContainerName(instance.instanceId);
|
501
|
+
const containerName = getBlockInstanceContainerName(instance.systemId, instance.instanceId);
|
495
502
|
const container = await containerManager.getContainerByName(containerName);
|
496
503
|
if (!container) {
|
497
504
|
// If the container doesn't exist, we consider the instance stopped
|
@@ -62,15 +62,21 @@ router.post('/:systemId/:instanceId/stop', async (req, res) => {
|
|
62
62
|
/**
|
63
63
|
* Get logs for instance in a plan
|
64
64
|
*/
|
65
|
-
router.get('/:systemId/:instanceId/logs', (req, res) => {
|
65
|
+
router.get('/:systemId/:instanceId/logs', async (req, res) => {
|
66
66
|
const instanceInfo = instanceManager.getInstance(req.params.systemId, req.params.instanceId);
|
67
67
|
if (!instanceInfo) {
|
68
68
|
res.status(404).send({ ok: false });
|
69
69
|
return;
|
70
70
|
}
|
71
|
-
|
72
|
-
logs
|
73
|
-
|
71
|
+
try {
|
72
|
+
const logs = await instanceManager.getLogs(req.params.systemId, req.params.instanceId);
|
73
|
+
res.status(200).send({
|
74
|
+
logs,
|
75
|
+
});
|
76
|
+
}
|
77
|
+
catch (e) {
|
78
|
+
res.status(500).send({ ok: false, error: e.message });
|
79
|
+
}
|
74
80
|
});
|
75
81
|
/**
|
76
82
|
* Get public address for instance in a plan if available
|
@@ -111,13 +111,11 @@ class OperatorManager {
|
|
111
111
|
const operatorData = operator.getData();
|
112
112
|
const portTypes = Object.keys(operatorData.ports);
|
113
113
|
portTypes.sort();
|
114
|
-
const containerBaseName = 'kapeta-resource';
|
115
|
-
const nameParts = [resourceType.toLowerCase()];
|
116
114
|
const ports = {};
|
117
115
|
for (let i = 0; i < portTypes.length; i++) {
|
118
116
|
const portType = portTypes[i];
|
119
117
|
let containerPortInfo = operatorData.ports[portType];
|
120
|
-
const hostPort = await serviceManager.ensureServicePort(resourceType, portType);
|
118
|
+
const hostPort = await serviceManager.ensureServicePort(systemId, resourceType, portType);
|
121
119
|
if (typeof containerPortInfo === 'number' || typeof containerPortInfo === 'string') {
|
122
120
|
containerPortInfo = { port: containerPortInfo, type: 'tcp' };
|
123
121
|
}
|
@@ -125,14 +123,18 @@ class OperatorManager {
|
|
125
123
|
containerPortInfo.type = 'tcp';
|
126
124
|
}
|
127
125
|
const portId = containerPortInfo.port + '/' + containerPortInfo.type;
|
128
|
-
nameParts.push(portType + '-' + portId + '-' + hostPort);
|
129
126
|
ports[portId] = {
|
130
127
|
type: portType,
|
131
128
|
hostPort,
|
132
129
|
};
|
133
130
|
}
|
134
|
-
const mounts = containerManager.createMounts(resourceType, operatorData.mounts);
|
135
|
-
const
|
131
|
+
const mounts = await containerManager.createMounts(systemId, resourceType, operatorData.mounts);
|
132
|
+
const nameParts = [
|
133
|
+
systemId,
|
134
|
+
resourceType.toLowerCase(),
|
135
|
+
version
|
136
|
+
];
|
137
|
+
const containerName = `kapeta-resource-${md5(nameParts.join('_'))}`;
|
136
138
|
const PortBindings = {};
|
137
139
|
const Env = [];
|
138
140
|
const Labels = {
|