@kapeta/local-cluster-service 0.10.1 → 0.11.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (38) hide show
  1. package/CHANGELOG.md +14 -0
  2. package/dist/cjs/src/containerManager.d.ts +6 -4
  3. package/dist/cjs/src/containerManager.js +100 -45
  4. package/dist/cjs/src/definitionsManager.d.ts +1 -0
  5. package/dist/cjs/src/definitionsManager.js +7 -0
  6. package/dist/cjs/src/instanceManager.d.ts +6 -2
  7. package/dist/cjs/src/instanceManager.js +240 -233
  8. package/dist/cjs/src/instances/routes.js +10 -4
  9. package/dist/cjs/src/operatorManager.js +8 -6
  10. package/dist/cjs/src/repositoryManager.js +4 -4
  11. package/dist/cjs/src/types.d.ts +0 -9
  12. package/dist/cjs/src/utils/BlockInstanceRunner.js +9 -64
  13. package/dist/cjs/src/utils/utils.d.ts +1 -1
  14. package/dist/cjs/src/utils/utils.js +3 -2
  15. package/dist/esm/src/containerManager.d.ts +6 -4
  16. package/dist/esm/src/containerManager.js +100 -45
  17. package/dist/esm/src/definitionsManager.d.ts +1 -0
  18. package/dist/esm/src/definitionsManager.js +7 -0
  19. package/dist/esm/src/instanceManager.d.ts +6 -2
  20. package/dist/esm/src/instanceManager.js +240 -233
  21. package/dist/esm/src/instances/routes.js +10 -4
  22. package/dist/esm/src/operatorManager.js +8 -6
  23. package/dist/esm/src/repositoryManager.js +4 -4
  24. package/dist/esm/src/types.d.ts +0 -9
  25. package/dist/esm/src/utils/BlockInstanceRunner.js +9 -64
  26. package/dist/esm/src/utils/utils.d.ts +1 -1
  27. package/dist/esm/src/utils/utils.js +3 -2
  28. package/package.json +3 -1
  29. package/src/containerManager.ts +126 -49
  30. package/src/definitionsManager.ts +8 -0
  31. package/src/instanceManager.ts +270 -255
  32. package/src/instances/routes.ts +9 -4
  33. package/src/operatorManager.ts +9 -8
  34. package/src/repositoryManager.ts +5 -5
  35. package/src/types.ts +0 -7
  36. package/src/utils/BlockInstanceRunner.ts +10 -66
  37. package/src/utils/LogData.ts +1 -0
  38. package/src/utils/utils.ts +3 -2
@@ -1,5 +1,6 @@
1
1
  import _ from 'lodash';
2
2
  import request from 'request';
3
+ import AsyncLock from 'async-lock';
3
4
  import { BlockInstanceRunner } from './utils/BlockInstanceRunner';
4
5
  import { storageService } from './storageService';
5
6
  import { socketManager } from './socketManager';
@@ -19,6 +20,7 @@ const MIN_TIME_RUNNING = 30000; //If something didnt run for more than 30 secs -
19
20
  export class InstanceManager {
20
21
  _interval = undefined;
21
22
  _instances = [];
23
+ instanceLocks = new AsyncLock();
22
24
  constructor() {
23
25
  this._instances = storageService.section('instances', []);
24
26
  // We need to wait a bit before running the first check
@@ -50,6 +52,36 @@ export class InstanceManager {
50
52
  systemId = normalizeKapetaUri(systemId);
51
53
  return this._instances.find((i) => i.systemId === systemId && i.instanceId === instanceId);
52
54
  }
55
+ async exclusive(systemId, instanceId, fn) {
56
+ systemId = normalizeKapetaUri(systemId);
57
+ const key = `${systemId}/${instanceId}`;
58
+ return this.instanceLocks.acquire(key, fn);
59
+ }
60
+ async getLogs(systemId, instanceId) {
61
+ const instance = this.getInstance(systemId, instanceId);
62
+ if (!instance) {
63
+ throw new Error(`Instance ${systemId}/${instanceId} not found`);
64
+ }
65
+ switch (instance.type) {
66
+ case InstanceType.DOCKER:
67
+ return await containerManager.getLogs(instance);
68
+ case InstanceType.UNKNOWN:
69
+ return [{
70
+ level: 'INFO',
71
+ message: 'Instance is starting...',
72
+ time: Date.now(),
73
+ source: 'stdout',
74
+ }];
75
+ case InstanceType.LOCAL:
76
+ return [{
77
+ level: 'INFO',
78
+ message: 'Instance started outside Kapeta - logs not available...',
79
+ time: Date.now(),
80
+ source: 'stdout',
81
+ }];
82
+ }
83
+ return [];
84
+ }
53
85
  async saveInternalInstance(instance) {
54
86
  instance.systemId = normalizeKapetaUri(instance.systemId);
55
87
  if (instance.ref) {
@@ -80,58 +112,59 @@ export class InstanceManager {
80
112
  * which self-registers with the cluster service locally on startup.
81
113
  */
82
114
  async registerInstanceFromSDK(systemId, instanceId, info) {
83
- systemId = normalizeKapetaUri(systemId);
84
- let instance = this.getInstance(systemId, instanceId);
85
- //Get target address
86
- const address = await serviceManager.getProviderAddress(systemId, instanceId, info.portType ?? DEFAULT_HEALTH_PORT_TYPE);
87
- const healthUrl = this.getHealthUrl(info, address);
88
- if (instance) {
89
- if (instance.status === InstanceStatus.STOPPING && instance.desiredStatus === DesiredInstanceStatus.STOP) {
90
- //If instance is stopping do not interfere
91
- return;
92
- }
93
- if (info.owner === InstanceOwner.EXTERNAL) {
94
- //If instance was started externally - then we want to replace the internal instance with that
95
- if (instance.owner === InstanceOwner.INTERNAL &&
96
- (instance.status === InstanceStatus.READY ||
97
- instance.status === InstanceStatus.STARTING ||
98
- instance.status === InstanceStatus.UNHEALTHY)) {
99
- throw new Error(`Instance ${instanceId} is already running`);
115
+ return this.exclusive(systemId, instanceId, async () => {
116
+ systemId = normalizeKapetaUri(systemId);
117
+ let instance = this.getInstance(systemId, instanceId);
118
+ //Get target address
119
+ const address = await serviceManager.getProviderAddress(systemId, instanceId, info.portType ?? DEFAULT_HEALTH_PORT_TYPE);
120
+ const healthUrl = this.getHealthUrl(info, address);
121
+ if (instance) {
122
+ if (instance.status === InstanceStatus.STOPPING && instance.desiredStatus === DesiredInstanceStatus.STOP) {
123
+ //If instance is stopping do not interfere
124
+ return;
100
125
  }
101
- instance.desiredStatus = info.desiredStatus;
102
- instance.owner = info.owner;
103
- instance.internal = undefined;
104
- instance.status = InstanceStatus.STARTING;
105
- instance.startedAt = Date.now();
106
- }
107
- instance.pid = info.pid;
108
- instance.address = address;
109
- if (info.type) {
110
- instance.type = info.type;
126
+ if (info.owner === InstanceOwner.EXTERNAL) {
127
+ //If instance was started externally - then we want to replace the internal instance with that
128
+ if (instance.owner === InstanceOwner.INTERNAL &&
129
+ (instance.status === InstanceStatus.READY ||
130
+ instance.status === InstanceStatus.STARTING ||
131
+ instance.status === InstanceStatus.UNHEALTHY)) {
132
+ throw new Error(`Instance ${instanceId} is already running`);
133
+ }
134
+ instance.desiredStatus = info.desiredStatus;
135
+ instance.owner = info.owner;
136
+ instance.status = InstanceStatus.STARTING;
137
+ instance.startedAt = Date.now();
138
+ }
139
+ instance.pid = info.pid;
140
+ instance.address = address;
141
+ if (info.type) {
142
+ instance.type = info.type;
143
+ }
144
+ if (healthUrl) {
145
+ instance.health = healthUrl;
146
+ }
147
+ this.emitSystemEvent(systemId, EVENT_STATUS_CHANGED, instance);
111
148
  }
112
- if (healthUrl) {
113
- instance.health = healthUrl;
149
+ else {
150
+ //If instance was not found - then we're receiving an externally started instance
151
+ instance = {
152
+ ...info,
153
+ systemId,
154
+ instanceId,
155
+ status: InstanceStatus.STARTING,
156
+ startedAt: Date.now(),
157
+ desiredStatus: DesiredInstanceStatus.EXTERNAL,
158
+ owner: InstanceOwner.EXTERNAL,
159
+ health: healthUrl,
160
+ address,
161
+ };
162
+ this._instances.push(instance);
163
+ this.emitSystemEvent(systemId, EVENT_INSTANCE_CREATED, instance);
114
164
  }
115
- this.emitSystemEvent(systemId, EVENT_STATUS_CHANGED, instance);
116
- }
117
- else {
118
- //If instance was not found - then we're receiving an externally started instance
119
- instance = {
120
- ...info,
121
- systemId,
122
- instanceId,
123
- status: InstanceStatus.STARTING,
124
- startedAt: Date.now(),
125
- desiredStatus: DesiredInstanceStatus.EXTERNAL,
126
- owner: InstanceOwner.EXTERNAL,
127
- health: healthUrl,
128
- address,
129
- };
130
- this._instances.push(instance);
131
- this.emitSystemEvent(systemId, EVENT_INSTANCE_CREATED, instance);
132
- }
133
- this.save();
134
- return instance;
165
+ this.save();
166
+ return instance;
167
+ });
135
168
  }
136
169
  getHealthUrl(info, address) {
137
170
  let healthUrl = null;
@@ -145,15 +178,17 @@ export class InstanceManager {
145
178
  return healthUrl;
146
179
  }
147
180
  markAsStopped(systemId, instanceId) {
148
- systemId = normalizeKapetaUri(systemId);
149
- const instance = _.find(this._instances, { systemId, instanceId });
150
- if (instance && instance.owner === InstanceOwner.EXTERNAL && instance.status !== InstanceStatus.STOPPED) {
151
- instance.status = InstanceStatus.STOPPED;
152
- instance.pid = null;
153
- instance.health = null;
154
- this.emitSystemEvent(systemId, EVENT_STATUS_CHANGED, instance);
155
- this.save();
156
- }
181
+ return this.exclusive(systemId, instanceId, async () => {
182
+ systemId = normalizeKapetaUri(systemId);
183
+ const instance = _.find(this._instances, { systemId, instanceId });
184
+ if (instance && instance.owner === InstanceOwner.EXTERNAL && instance.status !== InstanceStatus.STOPPED) {
185
+ instance.status = InstanceStatus.STOPPED;
186
+ instance.pid = null;
187
+ instance.health = null;
188
+ this.emitSystemEvent(systemId, EVENT_STATUS_CHANGED, instance);
189
+ this.save();
190
+ }
191
+ });
157
192
  }
158
193
  async startAllForPlan(systemId) {
159
194
  systemId = normalizeKapetaUri(systemId);
@@ -182,54 +217,60 @@ export class InstanceManager {
182
217
  return settled.map((p) => (p.status === 'fulfilled' ? p.value : null)).filter((p) => !!p);
183
218
  }
184
219
  async stop(systemId, instanceId) {
185
- systemId = normalizeKapetaUri(systemId);
186
- const instance = this.getInstance(systemId, instanceId);
187
- if (!instance) {
188
- return;
189
- }
190
- if (instance.status === InstanceStatus.STOPPED) {
191
- return;
192
- }
193
- if (instance.desiredStatus !== DesiredInstanceStatus.EXTERNAL) {
194
- instance.desiredStatus = DesiredInstanceStatus.STOP;
195
- }
196
- instance.status = InstanceStatus.STOPPING;
197
- this.emitSystemEvent(systemId, EVENT_STATUS_CHANGED, instance);
198
- console.log('Stopping instance: %s::%s [desired: %s]', systemId, instanceId, instance.desiredStatus);
199
- this.save();
200
- try {
201
- if (instance.type === 'docker') {
202
- const containerName = getBlockInstanceContainerName(instance.instanceId);
203
- const container = await containerManager.getContainerByName(containerName);
204
- if (container) {
205
- try {
206
- await container.stop();
207
- instance.status = InstanceStatus.STOPPED;
208
- this.emitSystemEvent(systemId, EVENT_STATUS_CHANGED, instance);
209
- this.save();
220
+ return this.stopInner(systemId, instanceId, true);
221
+ }
222
+ async stopInner(systemId, instanceId, changeDesired = false) {
223
+ return this.exclusive(systemId, instanceId, async () => {
224
+ systemId = normalizeKapetaUri(systemId);
225
+ const instance = this.getInstance(systemId, instanceId);
226
+ if (!instance) {
227
+ return;
228
+ }
229
+ if (instance.status === InstanceStatus.STOPPED) {
230
+ return;
231
+ }
232
+ if (changeDesired &&
233
+ instance.desiredStatus !== DesiredInstanceStatus.EXTERNAL) {
234
+ instance.desiredStatus = DesiredInstanceStatus.STOP;
235
+ }
236
+ instance.status = InstanceStatus.STOPPING;
237
+ this.emitSystemEvent(systemId, EVENT_STATUS_CHANGED, instance);
238
+ console.log('Stopping instance: %s::%s [desired: %s]', systemId, instanceId, instance.desiredStatus);
239
+ this.save();
240
+ try {
241
+ if (instance.type === 'docker') {
242
+ const containerName = getBlockInstanceContainerName(instance.systemId, instance.instanceId);
243
+ const container = await containerManager.getContainerByName(containerName);
244
+ if (container) {
245
+ try {
246
+ await container.stop();
247
+ instance.status = InstanceStatus.STOPPED;
248
+ this.emitSystemEvent(systemId, EVENT_STATUS_CHANGED, instance);
249
+ this.save();
250
+ }
251
+ catch (e) {
252
+ console.error('Failed to stop container', e);
253
+ }
210
254
  }
211
- catch (e) {
212
- console.error('Failed to stop container', e);
255
+ else {
256
+ console.warn('Container not found', containerName);
213
257
  }
258
+ return;
214
259
  }
215
- else {
216
- console.warn('Container not found', containerName);
260
+ if (!instance.pid) {
261
+ instance.status = InstanceStatus.STOPPED;
262
+ this.save();
263
+ return;
217
264
  }
218
- return;
219
- }
220
- if (!instance.pid) {
265
+ process.kill(instance.pid, 'SIGTERM');
221
266
  instance.status = InstanceStatus.STOPPED;
267
+ this.emitSystemEvent(systemId, EVENT_STATUS_CHANGED, instance);
222
268
  this.save();
223
- return;
224
269
  }
225
- process.kill(instance.pid, 'SIGTERM');
226
- instance.status = InstanceStatus.STOPPED;
227
- this.emitSystemEvent(systemId, EVENT_STATUS_CHANGED, instance);
228
- this.save();
229
- }
230
- catch (e) {
231
- console.error('Failed to stop process', e);
232
- }
270
+ catch (e) {
271
+ console.error('Failed to stop process', e);
272
+ }
273
+ });
233
274
  }
234
275
  async stopAllForPlan(systemId) {
235
276
  systemId = normalizeKapetaUri(systemId);
@@ -237,143 +278,111 @@ export class InstanceManager {
237
278
  return this.stopInstances(instancesForPlan);
238
279
  }
239
280
  async start(systemId, instanceId) {
240
- systemId = normalizeKapetaUri(systemId);
241
- const plan = await assetManager.getPlan(systemId, true);
242
- if (!plan) {
243
- throw new Error('Plan not found: ' + systemId);
244
- }
245
- const blockInstance = plan.spec && plan.spec.blocks ? _.find(plan.spec.blocks, { id: instanceId }) : null;
246
- if (!blockInstance) {
247
- throw new Error('Block instance not found: ' + instanceId);
248
- }
249
- const blockRef = normalizeKapetaUri(blockInstance.block.ref);
250
- const blockAsset = await assetManager.getAsset(blockRef, true);
251
- if (!blockAsset) {
252
- throw new Error('Block not found: ' + blockRef);
253
- }
254
- const existingInstance = this.getInstance(systemId, instanceId);
255
- if (existingInstance) {
256
- if (existingInstance.status === InstanceStatus.READY) {
257
- // Instance is already running
258
- return existingInstance;
281
+ return this.exclusive(systemId, instanceId, async () => {
282
+ systemId = normalizeKapetaUri(systemId);
283
+ const plan = await assetManager.getPlan(systemId, true);
284
+ if (!plan) {
285
+ throw new Error('Plan not found: ' + systemId);
259
286
  }
260
- if (existingInstance.desiredStatus === DesiredInstanceStatus.RUN &&
261
- existingInstance.status === InstanceStatus.STARTING) {
262
- // Internal instance is already starting - don't start it again
263
- return existingInstance;
287
+ const blockInstance = plan.spec && plan.spec.blocks ? _.find(plan.spec.blocks, { id: instanceId }) : null;
288
+ if (!blockInstance) {
289
+ throw new Error('Block instance not found: ' + instanceId);
264
290
  }
265
- if (existingInstance.owner === InstanceOwner.EXTERNAL &&
266
- existingInstance.status === InstanceStatus.STARTING) {
267
- // External instance is already starting - don't start it again
268
- return existingInstance;
291
+ const blockRef = normalizeKapetaUri(blockInstance.block.ref);
292
+ const blockAsset = await assetManager.getAsset(blockRef, true);
293
+ if (!blockAsset) {
294
+ throw new Error('Block not found: ' + blockRef);
269
295
  }
270
- }
271
- let instance = {
272
- systemId,
273
- instanceId,
274
- ref: blockRef,
275
- name: blockAsset.data.metadata.name,
276
- desiredStatus: DesiredInstanceStatus.RUN,
277
- owner: InstanceOwner.INTERNAL,
278
- type: InstanceType.UNKNOWN,
279
- status: InstanceStatus.STARTING,
280
- startedAt: Date.now(),
281
- };
282
- console.log('Starting instance: %s::%s [desired: %s]', systemId, instanceId, instance.desiredStatus);
283
- // Save the instance before starting it, so that we can track the status
284
- await this.saveInternalInstance(instance);
285
- if (existingInstance) {
286
- // Check if the instance is already running - but after we've commmuicated the desired status
287
- const currentStatus = await this.requestInstanceStatus(existingInstance);
288
- if (currentStatus === InstanceStatus.READY) {
289
- // Instance is already running
290
- return existingInstance;
291
- }
292
- }
293
- const instanceConfig = await configManager.getConfigForSection(systemId, instanceId);
294
- const runner = new BlockInstanceRunner(systemId);
295
- const startTime = Date.now();
296
- try {
297
- const processInfo = await runner.start(blockRef, instanceId, instanceConfig);
298
- //emit stdout/stderr via sockets
299
- processInfo.output.on('data', (data) => {
300
- const payload = {
301
- source: 'stdout',
302
- level: 'INFO',
303
- message: data.toString(),
304
- time: Date.now(),
305
- };
306
- this.emitInstanceEvent(systemId, instanceId, EVENT_INSTANCE_LOG, payload);
307
- });
308
- processInfo.output.on('exit', (exitCode) => {
309
- const timeRunning = Date.now() - startTime;
310
- const instance = this.getInstance(systemId, instanceId);
311
- if (instance?.status === InstanceStatus.READY) {
312
- //It's already been running
313
- return;
296
+ const existingInstance = this.getInstance(systemId, instanceId);
297
+ if (existingInstance) {
298
+ if (existingInstance.status === InstanceStatus.READY) {
299
+ // Instance is already running
300
+ return existingInstance;
314
301
  }
315
- if (exitCode === 143 || exitCode === 137) {
316
- //Process got SIGTERM (143) or SIGKILL (137)
317
- //TODO: Windows?
318
- return;
302
+ if (existingInstance.desiredStatus === DesiredInstanceStatus.RUN &&
303
+ existingInstance.status === InstanceStatus.STARTING) {
304
+ // Internal instance is already starting - don't start it again
305
+ return existingInstance;
319
306
  }
320
- if (exitCode !== 0 || timeRunning < MIN_TIME_RUNNING) {
321
- const instance = this.getInstance(systemId, instanceId);
322
- if (instance) {
323
- instance.status = InstanceStatus.FAILED;
324
- this.save();
325
- }
326
- this.emitSystemEvent(systemId, EVENT_INSTANCE_EXITED, {
327
- error: 'Failed to start instance',
328
- status: EVENT_INSTANCE_EXITED,
329
- instanceId: blockInstance.id,
330
- });
307
+ if (existingInstance.owner === InstanceOwner.EXTERNAL &&
308
+ existingInstance.status === InstanceStatus.STARTING) {
309
+ // External instance is already starting - don't start it again
310
+ return existingInstance;
331
311
  }
332
- });
333
- instance.status = InstanceStatus.READY;
334
- return this.saveInternalInstance({
335
- ...instance,
336
- type: processInfo.type,
337
- pid: processInfo.pid ?? -1,
338
- health: null,
339
- portType: processInfo.portType,
340
- status: InstanceStatus.READY,
341
- internal: {
342
- logs: processInfo.logs,
343
- output: processInfo.output,
344
- },
345
- });
346
- }
347
- catch (e) {
348
- console.warn('Failed to start instance', e);
349
- const logs = [
350
- {
351
- source: 'stdout',
352
- level: 'ERROR',
353
- message: e.message,
354
- time: Date.now(),
355
- },
356
- ];
357
- const out = await this.saveInternalInstance({
358
- ...instance,
359
- type: InstanceType.LOCAL,
360
- pid: null,
361
- health: null,
362
- portType: DEFAULT_HEALTH_PORT_TYPE,
363
- status: InstanceStatus.FAILED,
364
- });
365
- this.emitInstanceEvent(systemId, instanceId, EVENT_INSTANCE_LOG, logs[0]);
366
- this.emitInstanceEvent(systemId, blockInstance.id, EVENT_INSTANCE_EXITED, {
367
- error: `Failed to start instance: ${e.message}`,
368
- status: EVENT_INSTANCE_EXITED,
369
- instanceId: blockInstance.id,
370
- });
371
- return out;
372
- }
312
+ }
313
+ let instance = {
314
+ systemId,
315
+ instanceId,
316
+ ref: blockRef,
317
+ name: blockAsset.data.metadata.name,
318
+ desiredStatus: DesiredInstanceStatus.RUN,
319
+ owner: InstanceOwner.INTERNAL,
320
+ type: existingInstance?.type ?? InstanceType.UNKNOWN,
321
+ status: InstanceStatus.STARTING,
322
+ startedAt: Date.now(),
323
+ };
324
+ console.log('Starting instance: %s::%s [desired: %s]', systemId, instanceId, instance.desiredStatus);
325
+ // Save the instance before starting it, so that we can track the status
326
+ await this.saveInternalInstance(instance);
327
+ if (existingInstance) {
328
+ // Check if the instance is already running - but after we've commmuicated the desired status
329
+ const currentStatus = await this.requestInstanceStatus(existingInstance);
330
+ if (currentStatus === InstanceStatus.READY) {
331
+ // Instance is already running
332
+ return existingInstance;
333
+ }
334
+ }
335
+ const instanceConfig = await configManager.getConfigForSection(systemId, instanceId);
336
+ const runner = new BlockInstanceRunner(systemId);
337
+ const startTime = Date.now();
338
+ try {
339
+ const processInfo = await runner.start(blockRef, instanceId, instanceConfig);
340
+ instance.status = InstanceStatus.READY;
341
+ return this.saveInternalInstance({
342
+ ...instance,
343
+ type: processInfo.type,
344
+ pid: processInfo.pid ?? -1,
345
+ health: null,
346
+ portType: processInfo.portType,
347
+ status: InstanceStatus.READY,
348
+ });
349
+ }
350
+ catch (e) {
351
+ console.warn('Failed to start instance', e);
352
+ const logs = [
353
+ {
354
+ source: 'stdout',
355
+ level: 'ERROR',
356
+ message: e.message,
357
+ time: Date.now(),
358
+ },
359
+ ];
360
+ const out = await this.saveInternalInstance({
361
+ ...instance,
362
+ type: InstanceType.LOCAL,
363
+ pid: null,
364
+ health: null,
365
+ portType: DEFAULT_HEALTH_PORT_TYPE,
366
+ status: InstanceStatus.FAILED,
367
+ });
368
+ this.emitInstanceEvent(systemId, instanceId, EVENT_INSTANCE_LOG, logs[0]);
369
+ this.emitInstanceEvent(systemId, blockInstance.id, EVENT_INSTANCE_EXITED, {
370
+ error: `Failed to start instance: ${e.message}`,
371
+ status: EVENT_INSTANCE_EXITED,
372
+ instanceId: blockInstance.id,
373
+ });
374
+ return out;
375
+ }
376
+ });
373
377
  }
374
378
  async restart(systemId, instanceId) {
375
379
  systemId = normalizeKapetaUri(systemId);
376
- await this.stop(systemId, instanceId);
380
+ await this.stopInner(systemId, instanceId);
381
+ const existingInstance = this.getInstance(systemId, instanceId);
382
+ if (existingInstance?.desiredStatus === DesiredInstanceStatus.STOP) {
383
+ // Internal instance was marked as stopped - abort restart
384
+ return existingInstance;
385
+ }
377
386
  return this.start(systemId, instanceId);
378
387
  }
379
388
  async stopAll() {
@@ -387,9 +396,7 @@ export class InstanceManager {
387
396
  save() {
388
397
  try {
389
398
  storageService.put('instances', this._instances.map((instance) => {
390
- const copy = { ...instance };
391
- delete copy.internal;
392
- return copy;
399
+ return { ...instance };
393
400
  }));
394
401
  }
395
402
  catch (e) {
@@ -402,7 +409,7 @@ export class InstanceManager {
402
409
  const all = [...this._instances];
403
410
  while (all.length > 0) {
404
411
  // Check a few instances at a time - docker doesn't like too many concurrent requests
405
- const chunk = all.splice(0, 20);
412
+ const chunk = all.splice(0, 30);
406
413
  const promises = chunk.map(async (instance) => {
407
414
  if (!instance.systemId) {
408
415
  return;
@@ -462,7 +469,7 @@ export class InstanceManager {
462
469
  [InstanceStatus.READY, InstanceStatus.STARTING, InstanceStatus.UNHEALTHY].includes(newStatus)) {
463
470
  //If the instance is running but we want it to stop, stop it
464
471
  try {
465
- await this.stop(instance.systemId, instance.instanceId);
472
+ await this.stopInner(instance.systemId, instance.instanceId);
466
473
  }
467
474
  catch (e) {
468
475
  console.warn('Failed to stop instance', instance.systemId, instance.instanceId, e);
@@ -491,7 +498,7 @@ export class InstanceManager {
491
498
  }
492
499
  async getExternalStatus(instance) {
493
500
  if (instance.type === InstanceType.DOCKER) {
494
- const containerName = getBlockInstanceContainerName(instance.instanceId);
501
+ const containerName = getBlockInstanceContainerName(instance.systemId, instance.instanceId);
495
502
  const container = await containerManager.getContainerByName(containerName);
496
503
  if (!container) {
497
504
  // If the container doesn't exist, we consider the instance stopped
@@ -62,15 +62,21 @@ router.post('/:systemId/:instanceId/stop', async (req, res) => {
62
62
  /**
63
63
  * Get logs for instance in a plan
64
64
  */
65
- router.get('/:systemId/:instanceId/logs', (req, res) => {
65
+ router.get('/:systemId/:instanceId/logs', async (req, res) => {
66
66
  const instanceInfo = instanceManager.getInstance(req.params.systemId, req.params.instanceId);
67
67
  if (!instanceInfo) {
68
68
  res.status(404).send({ ok: false });
69
69
  return;
70
70
  }
71
- res.status(202).send({
72
- logs: instanceInfo.internal?.logs() ?? [],
73
- });
71
+ try {
72
+ const logs = await instanceManager.getLogs(req.params.systemId, req.params.instanceId);
73
+ res.status(200).send({
74
+ logs,
75
+ });
76
+ }
77
+ catch (e) {
78
+ res.status(500).send({ ok: false, error: e.message });
79
+ }
74
80
  });
75
81
  /**
76
82
  * Get public address for instance in a plan if available
@@ -111,13 +111,11 @@ class OperatorManager {
111
111
  const operatorData = operator.getData();
112
112
  const portTypes = Object.keys(operatorData.ports);
113
113
  portTypes.sort();
114
- const containerBaseName = 'kapeta-resource';
115
- const nameParts = [resourceType.toLowerCase()];
116
114
  const ports = {};
117
115
  for (let i = 0; i < portTypes.length; i++) {
118
116
  const portType = portTypes[i];
119
117
  let containerPortInfo = operatorData.ports[portType];
120
- const hostPort = await serviceManager.ensureServicePort(resourceType, portType);
118
+ const hostPort = await serviceManager.ensureServicePort(systemId, resourceType, portType);
121
119
  if (typeof containerPortInfo === 'number' || typeof containerPortInfo === 'string') {
122
120
  containerPortInfo = { port: containerPortInfo, type: 'tcp' };
123
121
  }
@@ -125,14 +123,18 @@ class OperatorManager {
125
123
  containerPortInfo.type = 'tcp';
126
124
  }
127
125
  const portId = containerPortInfo.port + '/' + containerPortInfo.type;
128
- nameParts.push(portType + '-' + portId + '-' + hostPort);
129
126
  ports[portId] = {
130
127
  type: portType,
131
128
  hostPort,
132
129
  };
133
130
  }
134
- const mounts = containerManager.createMounts(resourceType, operatorData.mounts);
135
- const containerName = containerBaseName + '-' + md5(nameParts.join('_'));
131
+ const mounts = await containerManager.createMounts(systemId, resourceType, operatorData.mounts);
132
+ const nameParts = [
133
+ systemId,
134
+ resourceType.toLowerCase(),
135
+ version
136
+ ];
137
+ const containerName = `kapeta-resource-${md5(nameParts.join('_'))}`;
136
138
  const PortBindings = {};
137
139
  const Env = [];
138
140
  const Labels = {