@wopr-network/platform-core 1.39.4 → 1.39.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/credits/credit-expiry-cron.test.js +30 -0
- package/dist/credits/ledger.js +11 -5
- package/dist/credits/ledger.test.js +87 -0
- package/dist/db/schema/ledger.js +6 -0
- package/dist/fleet/fleet-manager.d.ts +14 -35
- package/dist/fleet/fleet-manager.js +52 -236
- package/dist/fleet/fleet-manager.test.js +13 -85
- package/dist/fleet/instance.d.ts +58 -3
- package/dist/fleet/instance.js +297 -33
- package/dist/fleet/instance.test.d.ts +1 -0
- package/dist/fleet/instance.test.js +343 -0
- package/dist/node-agent/types.d.ts +1 -1
- package/package.json +1 -1
- package/src/credits/README.md +106 -0
- package/src/credits/credit-expiry-cron.test.ts +36 -0
- package/src/credits/ledger.test.ts +113 -0
- package/src/credits/ledger.ts +13 -7
- package/src/db/schema/ledger.ts +6 -0
- package/src/fleet/fleet-manager.test.ts +13 -111
- package/src/fleet/fleet-manager.ts +50 -255
- package/src/fleet/instance.test.ts +390 -0
- package/src/fleet/instance.ts +318 -33
|
@@ -126,74 +126,21 @@ describe("FleetManager", () => {
|
|
|
126
126
|
expect(container.stop).toHaveBeenCalled();
|
|
127
127
|
});
|
|
128
128
|
});
|
|
129
|
-
|
|
130
|
-
it("pulls image before restarting a running container", async () => {
|
|
131
|
-
// Store the profile first; default mockContainer has state "running" — valid for restart
|
|
132
|
-
await store.save({ id: "bot-id", ...PROFILE_PARAMS });
|
|
133
|
-
docker.listContainers.mockResolvedValue([{ Id: "container-123" }]);
|
|
134
|
-
await fleet.restart("bot-id");
|
|
135
|
-
// Pull is called first
|
|
136
|
-
expect(docker.pull).toHaveBeenCalledWith(PROFILE_PARAMS.image, {});
|
|
137
|
-
// Then restart
|
|
138
|
-
expect(container.restart).toHaveBeenCalled();
|
|
139
|
-
});
|
|
140
|
-
it("restarts a container in exited state (crash recovery)", async () => {
|
|
141
|
-
await store.save({ id: "bot-id", ...PROFILE_PARAMS });
|
|
142
|
-
container.inspect.mockResolvedValue({
|
|
143
|
-
Id: "container-123",
|
|
144
|
-
Created: "2026-01-01T00:00:00Z",
|
|
145
|
-
State: { Status: "exited", Running: false, StartedAt: "", Health: null },
|
|
146
|
-
});
|
|
147
|
-
docker.listContainers.mockResolvedValue([{ Id: "container-123" }]);
|
|
148
|
-
await fleet.restart("bot-id");
|
|
149
|
-
expect(container.restart).toHaveBeenCalled();
|
|
150
|
-
});
|
|
151
|
-
it("restarts a container in stopped state", async () => {
|
|
152
|
-
await store.save({ id: "bot-id", ...PROFILE_PARAMS });
|
|
153
|
-
container.inspect.mockResolvedValue({
|
|
154
|
-
Id: "container-123",
|
|
155
|
-
Created: "2026-01-01T00:00:00Z",
|
|
156
|
-
State: { Status: "stopped", Running: false, StartedAt: "", Health: null },
|
|
157
|
-
});
|
|
158
|
-
docker.listContainers.mockResolvedValue([{ Id: "container-123" }]);
|
|
159
|
-
await fleet.restart("bot-id");
|
|
160
|
-
expect(container.restart).toHaveBeenCalled();
|
|
161
|
-
});
|
|
162
|
-
it("restarts a container in dead state (crash recovery)", async () => {
|
|
163
|
-
await store.save({ id: "bot-id", ...PROFILE_PARAMS });
|
|
164
|
-
container.inspect.mockResolvedValue({
|
|
165
|
-
Id: "container-123",
|
|
166
|
-
Created: "2026-01-01T00:00:00Z",
|
|
167
|
-
State: { Status: "dead", Running: false, StartedAt: "", Health: null },
|
|
168
|
-
});
|
|
169
|
-
docker.listContainers.mockResolvedValue([{ Id: "container-123" }]);
|
|
170
|
-
await fleet.restart("bot-id");
|
|
171
|
-
expect(container.restart).toHaveBeenCalled();
|
|
172
|
-
});
|
|
173
|
-
it("does not restart if pull fails", async () => {
|
|
174
|
-
await store.save({ id: "bot-id", ...PROFILE_PARAMS });
|
|
175
|
-
docker.modem.followProgress.mockImplementation((_stream, cb) => cb(new Error("Pull failed")));
|
|
176
|
-
await expect(fleet.restart("bot-id")).rejects.toThrow("Pull failed");
|
|
177
|
-
expect(container.restart).not.toHaveBeenCalled();
|
|
178
|
-
});
|
|
179
|
-
it("throws BotNotFoundError for missing profile", async () => {
|
|
180
|
-
await expect(fleet.restart("missing")).rejects.toThrow(BotNotFoundError);
|
|
181
|
-
});
|
|
182
|
-
});
|
|
129
|
+
// restart tests moved to instance.test.ts — Instance.restart() + Instance.pullImage()
|
|
183
130
|
describe("remove", () => {
|
|
184
131
|
it("stops running container, removes it, and deletes profile", async () => {
|
|
185
132
|
await store.save({ id: "bot-id", ...PROFILE_PARAMS });
|
|
186
133
|
docker.listContainers.mockResolvedValue([{ Id: "container-123" }]);
|
|
187
134
|
await fleet.remove("bot-id");
|
|
188
135
|
expect(container.stop).toHaveBeenCalled();
|
|
189
|
-
expect(container.remove).toHaveBeenCalledWith({ v: false });
|
|
136
|
+
expect(container.remove).toHaveBeenCalledWith({ force: true, v: false });
|
|
190
137
|
expect(store.delete).toHaveBeenCalledWith("bot-id");
|
|
191
138
|
});
|
|
192
139
|
it("removes volumes when requested", async () => {
|
|
193
140
|
await store.save({ id: "bot-id", ...PROFILE_PARAMS });
|
|
194
141
|
docker.listContainers.mockResolvedValue([{ Id: "container-123" }]);
|
|
195
142
|
await fleet.remove("bot-id", true);
|
|
196
|
-
expect(container.remove).toHaveBeenCalledWith({ v: true });
|
|
143
|
+
expect(container.remove).toHaveBeenCalledWith({ force: true, v: true });
|
|
197
144
|
});
|
|
198
145
|
it("deletes profile even when no container exists", async () => {
|
|
199
146
|
await store.save({ id: "bot-id", ...PROFILE_PARAMS });
|
|
@@ -240,13 +187,13 @@ describe("FleetManager", () => {
|
|
|
240
187
|
});
|
|
241
188
|
describe("logs", () => {
|
|
242
189
|
it("returns container logs", async () => {
|
|
190
|
+
await store.save({ id: "bot-id", ...PROFILE_PARAMS });
|
|
243
191
|
docker.listContainers.mockResolvedValue([{ Id: "container-123" }]);
|
|
244
192
|
const logs = await fleet.logs("bot-id", 50);
|
|
245
193
|
expect(container.logs).toHaveBeenCalledWith(expect.objectContaining({ tail: 50, stdout: true, stderr: true }));
|
|
246
194
|
expect(logs).toContain("log line 1");
|
|
247
195
|
});
|
|
248
|
-
it("throws BotNotFoundError when
|
|
249
|
-
docker.listContainers.mockResolvedValue([]);
|
|
196
|
+
it("throws BotNotFoundError when profile not found", async () => {
|
|
250
197
|
await expect(fleet.logs("missing")).rejects.toThrow(BotNotFoundError);
|
|
251
198
|
});
|
|
252
199
|
});
|
|
@@ -255,6 +202,7 @@ describe("FleetManager", () => {
|
|
|
255
202
|
const { PassThrough } = await import("node:stream");
|
|
256
203
|
const mockStream = new PassThrough();
|
|
257
204
|
container.logs.mockResolvedValue(mockStream);
|
|
205
|
+
await store.save({ id: "bot-id", ...PROFILE_PARAMS });
|
|
258
206
|
docker.listContainers.mockResolvedValue([{ Id: "container-123" }]);
|
|
259
207
|
const stream = await fleet.logStream("bot-id", { since: "2026-01-01T00:00:00Z", tail: 50 });
|
|
260
208
|
// Result is a PassThrough (demuxed), not the raw multiplexed stream
|
|
@@ -276,6 +224,7 @@ describe("FleetManager", () => {
|
|
|
276
224
|
const { PassThrough } = await import("node:stream");
|
|
277
225
|
const mockStream = new PassThrough();
|
|
278
226
|
container.logs.mockResolvedValue(mockStream);
|
|
227
|
+
await store.save({ id: "bot-id", ...PROFILE_PARAMS });
|
|
279
228
|
docker.listContainers.mockResolvedValue([{ Id: "container-123" }]);
|
|
280
229
|
await fleet.logStream("bot-id", {});
|
|
281
230
|
expect(container.logs).toHaveBeenCalledWith({
|
|
@@ -287,8 +236,7 @@ describe("FleetManager", () => {
|
|
|
287
236
|
});
|
|
288
237
|
mockStream.destroy();
|
|
289
238
|
});
|
|
290
|
-
it("throws BotNotFoundError when
|
|
291
|
-
docker.listContainers.mockResolvedValue([]);
|
|
239
|
+
it("throws BotNotFoundError when profile not found", async () => {
|
|
292
240
|
await expect(fleet.logStream("missing", {})).rejects.toThrow(BotNotFoundError);
|
|
293
241
|
});
|
|
294
242
|
it("proxies via node-agent for remote bots", async () => {
|
|
@@ -557,31 +505,7 @@ describe("FleetManager", () => {
|
|
|
557
505
|
expect(bus.send).not.toHaveBeenCalled();
|
|
558
506
|
expect(docker.pull).toHaveBeenCalled();
|
|
559
507
|
});
|
|
560
|
-
|
|
561
|
-
const container = mockContainer();
|
|
562
|
-
const docker = mockDocker(container);
|
|
563
|
-
const store = mockStore();
|
|
564
|
-
const bus = mockCommandBus();
|
|
565
|
-
const instanceRepo = mockInstanceRepo();
|
|
566
|
-
const botId = "bot-restart-1";
|
|
567
|
-
await instanceRepo.create({ id: botId, tenantId: "t1", name: "restart-bot", nodeId: "node-3" });
|
|
568
|
-
const fleet = new FleetManager(docker, store, undefined, undefined, undefined, bus, instanceRepo);
|
|
569
|
-
await store.save({
|
|
570
|
-
id: botId,
|
|
571
|
-
tenantId: "t1",
|
|
572
|
-
name: "restart-bot",
|
|
573
|
-
description: "",
|
|
574
|
-
image: "ghcr.io/wopr-network/wopr:latest",
|
|
575
|
-
env: {},
|
|
576
|
-
restartPolicy: "unless-stopped",
|
|
577
|
-
});
|
|
578
|
-
await fleet.restart(botId);
|
|
579
|
-
expect(bus.send).toHaveBeenCalledWith("node-3", {
|
|
580
|
-
type: "bot.restart",
|
|
581
|
-
payload: { name: "restart-bot" },
|
|
582
|
-
});
|
|
583
|
-
expect(container.restart).not.toHaveBeenCalled();
|
|
584
|
-
});
|
|
508
|
+
// remote restart test removed — restart moved to Instance
|
|
585
509
|
it("should dispatch remove via commandBus when bot has nodeId", async () => {
|
|
586
510
|
const container = mockContainer();
|
|
587
511
|
const docker = mockDocker(container);
|
|
@@ -645,6 +569,7 @@ describe("FleetManager", () => {
|
|
|
645
569
|
const containerWithExec = mockContainer({
|
|
646
570
|
exec: vi.fn().mockResolvedValue(execMock),
|
|
647
571
|
});
|
|
572
|
+
await store.save({ id: "bot-id", ...PROFILE_PARAMS });
|
|
648
573
|
docker.listContainers.mockResolvedValue([{ Id: "container-123" }]);
|
|
649
574
|
docker.getContainer.mockReturnValue(containerWithExec);
|
|
650
575
|
const result = await fleet.getVolumeUsage("bot-id");
|
|
@@ -655,6 +580,7 @@ describe("FleetManager", () => {
|
|
|
655
580
|
});
|
|
656
581
|
});
|
|
657
582
|
it("returns null when container is not found", async () => {
|
|
583
|
+
await store.save({ id: "bot-id", ...PROFILE_PARAMS });
|
|
658
584
|
docker.listContainers.mockResolvedValue([]);
|
|
659
585
|
const result = await fleet.getVolumeUsage("bot-id");
|
|
660
586
|
expect(result).toBeNull();
|
|
@@ -666,6 +592,7 @@ describe("FleetManager", () => {
|
|
|
666
592
|
State: { Running: false },
|
|
667
593
|
}),
|
|
668
594
|
});
|
|
595
|
+
await store.save({ id: "bot-id", ...PROFILE_PARAMS });
|
|
669
596
|
docker.listContainers.mockResolvedValue([{ Id: "container-123" }]);
|
|
670
597
|
docker.getContainer.mockReturnValue(stoppedContainer);
|
|
671
598
|
const result = await fleet.getVolumeUsage("bot-id");
|
|
@@ -675,6 +602,7 @@ describe("FleetManager", () => {
|
|
|
675
602
|
const containerWithFailingExec = mockContainer({
|
|
676
603
|
exec: vi.fn().mockRejectedValue(new Error("exec failed")),
|
|
677
604
|
});
|
|
605
|
+
await store.save({ id: "bot-id", ...PROFILE_PARAMS });
|
|
678
606
|
docker.listContainers.mockResolvedValue([{ Id: "container-123" }]);
|
|
679
607
|
docker.getContainer.mockReturnValue(containerWithFailingExec);
|
|
680
608
|
const result = await fleet.getVolumeUsage("bot-id");
|
package/dist/fleet/instance.d.ts
CHANGED
|
@@ -1,8 +1,9 @@
|
|
|
1
1
|
import type Docker from "dockerode";
|
|
2
|
+
import type { BotMetricsTracker } from "../gateway/bot-metrics-tracker.js";
|
|
2
3
|
import type { ProxyManagerInterface } from "../proxy/types.js";
|
|
3
4
|
import type { IBotInstanceRepository } from "./bot-instance-repository.js";
|
|
4
5
|
import type { FleetEventEmitter } from "./fleet-event-emitter.js";
|
|
5
|
-
import type { BotProfile } from "./types.js";
|
|
6
|
+
import type { BotProfile, BotStatus } from "./types.js";
|
|
6
7
|
/**
|
|
7
8
|
* Instance — a runtime handle to a container.
|
|
8
9
|
*
|
|
@@ -22,6 +23,7 @@ export interface InstanceDeps {
|
|
|
22
23
|
instanceRepo?: IBotInstanceRepository;
|
|
23
24
|
proxyManager?: ProxyManagerInterface;
|
|
24
25
|
eventEmitter?: FleetEventEmitter;
|
|
26
|
+
botMetricsTracker?: BotMetricsTracker;
|
|
25
27
|
}
|
|
26
28
|
export declare class Instance {
|
|
27
29
|
readonly id: string;
|
|
@@ -33,13 +35,64 @@ export declare class Instance {
|
|
|
33
35
|
private readonly instanceRepo;
|
|
34
36
|
private readonly proxyManager;
|
|
35
37
|
private readonly eventEmitter;
|
|
38
|
+
private readonly botMetricsTracker;
|
|
39
|
+
/** Simple per-instance mutex to serialize start/stop/restart/remove. */
|
|
40
|
+
private lockPromise;
|
|
36
41
|
constructor(deps: InstanceDeps);
|
|
42
|
+
/**
|
|
43
|
+
* Remote instances have containerId like "remote:node-3".
|
|
44
|
+
* Local Docker operations are not supported — callers (e.g. wopr-platform)
|
|
45
|
+
* handle remote delegation at a higher level via NodeCommandBus.
|
|
46
|
+
*/
|
|
47
|
+
private get isRemote();
|
|
48
|
+
private assertLocal;
|
|
49
|
+
private withLock;
|
|
37
50
|
/** Emit bot.created — call only from FleetManager.create(), not getInstance() */
|
|
38
51
|
emitCreated(): void;
|
|
39
52
|
start(): Promise<void>;
|
|
40
53
|
stop(): Promise<void>;
|
|
41
|
-
|
|
42
|
-
|
|
54
|
+
/**
|
|
55
|
+
* Restart the container.
|
|
56
|
+
* Callers that need an image update should call pullImage() first.
|
|
57
|
+
*/
|
|
58
|
+
restart(): Promise<void>;
|
|
59
|
+
/**
|
|
60
|
+
* Pull the latest version of this instance's image.
|
|
61
|
+
* Call before restart() to update the image before restarting.
|
|
62
|
+
*/
|
|
63
|
+
pullImage(): Promise<void>;
|
|
64
|
+
remove(removeVolumes?: boolean): Promise<void>;
|
|
65
|
+
/**
|
|
66
|
+
* Simple container state check (running / stopped / gone).
|
|
67
|
+
*/
|
|
68
|
+
containerState(): Promise<"running" | "stopped" | "gone">;
|
|
69
|
+
/**
|
|
70
|
+
* Full status including profile data, container state, resource stats,
|
|
71
|
+
* and application metrics. Returns BotStatus.
|
|
72
|
+
*/
|
|
73
|
+
status(): Promise<BotStatus>;
|
|
74
|
+
/**
|
|
75
|
+
* Get container logs (demultiplexed to plain text).
|
|
76
|
+
*/
|
|
77
|
+
logs(tail?: number): Promise<string>;
|
|
78
|
+
/**
|
|
79
|
+
* Stream container logs in real-time (follow mode).
|
|
80
|
+
* Returns a Node.js ReadableStream that emits plain-text log chunks (already demultiplexed).
|
|
81
|
+
* Caller is responsible for destroying the stream when done.
|
|
82
|
+
*/
|
|
83
|
+
logStream(opts: {
|
|
84
|
+
since?: string;
|
|
85
|
+
tail?: number;
|
|
86
|
+
}): Promise<NodeJS.ReadableStream>;
|
|
87
|
+
/**
|
|
88
|
+
* Get disk usage for this instance's /data volume.
|
|
89
|
+
* Returns null if the container is not running or exec fails.
|
|
90
|
+
*/
|
|
91
|
+
getVolumeUsage(): Promise<{
|
|
92
|
+
usedBytes: number;
|
|
93
|
+
totalBytes: number;
|
|
94
|
+
availableBytes: number;
|
|
95
|
+
} | null>;
|
|
43
96
|
/**
|
|
44
97
|
* Register this instance in the billing system.
|
|
45
98
|
* Skip for ephemeral instances — they bill per-token, not per-instance.
|
|
@@ -50,5 +103,7 @@ export declare class Instance {
|
|
|
50
103
|
* Skip for ephemeral instances — they're accessed directly via Docker DNS.
|
|
51
104
|
*/
|
|
52
105
|
setupProxy(): Promise<void>;
|
|
106
|
+
private offlineStatus;
|
|
107
|
+
private getStats;
|
|
53
108
|
private emit;
|
|
54
109
|
}
|
package/dist/fleet/instance.js
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { PassThrough } from "node:stream";
|
|
1
2
|
import { logger } from "../config/logger.js";
|
|
2
3
|
export class Instance {
|
|
3
4
|
id;
|
|
@@ -9,6 +10,9 @@ export class Instance {
|
|
|
9
10
|
instanceRepo;
|
|
10
11
|
proxyManager;
|
|
11
12
|
eventEmitter;
|
|
13
|
+
botMetricsTracker;
|
|
14
|
+
/** Simple per-instance mutex to serialize start/stop/restart/remove. */
|
|
15
|
+
lockPromise = Promise.resolve();
|
|
12
16
|
constructor(deps) {
|
|
13
17
|
this.id = deps.profile.id;
|
|
14
18
|
this.containerId = deps.containerId;
|
|
@@ -19,55 +23,139 @@ export class Instance {
|
|
|
19
23
|
this.instanceRepo = deps.instanceRepo;
|
|
20
24
|
this.proxyManager = deps.proxyManager;
|
|
21
25
|
this.eventEmitter = deps.eventEmitter;
|
|
26
|
+
this.botMetricsTracker = deps.botMetricsTracker;
|
|
27
|
+
}
|
|
28
|
+
/**
|
|
29
|
+
* Remote instances have containerId like "remote:node-3".
|
|
30
|
+
* Local Docker operations are not supported — callers (e.g. wopr-platform)
|
|
31
|
+
* handle remote delegation at a higher level via NodeCommandBus.
|
|
32
|
+
*/
|
|
33
|
+
get isRemote() {
|
|
34
|
+
return this.containerId.startsWith("remote:");
|
|
35
|
+
}
|
|
36
|
+
assertLocal(operation) {
|
|
37
|
+
if (this.isRemote) {
|
|
38
|
+
throw new Error(`${operation} is not supported on remote instances — use node agent`);
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
async withLock(fn) {
|
|
42
|
+
const prev = this.lockPromise;
|
|
43
|
+
let resolve;
|
|
44
|
+
this.lockPromise = new Promise((r) => {
|
|
45
|
+
resolve = r;
|
|
46
|
+
});
|
|
47
|
+
try {
|
|
48
|
+
await prev;
|
|
49
|
+
return await fn();
|
|
50
|
+
}
|
|
51
|
+
finally {
|
|
52
|
+
resolve();
|
|
53
|
+
}
|
|
22
54
|
}
|
|
23
55
|
/** Emit bot.created — call only from FleetManager.create(), not getInstance() */
|
|
24
56
|
emitCreated() {
|
|
25
57
|
this.emit("bot.created");
|
|
26
58
|
}
|
|
27
59
|
async start() {
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
60
|
+
this.assertLocal("start()");
|
|
61
|
+
return this.withLock(async () => {
|
|
62
|
+
const container = this.docker.getContainer(this.containerId);
|
|
63
|
+
await container.start();
|
|
64
|
+
logger.info(`Instance started`, { id: this.id, containerName: this.containerName, url: this.url });
|
|
65
|
+
this.emit("bot.started");
|
|
66
|
+
});
|
|
32
67
|
}
|
|
33
68
|
async stop() {
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
const msg = err instanceof Error ? err.message : String(err);
|
|
40
|
-
if (!msg.includes("not running") && !msg.includes("already stopped")) {
|
|
41
|
-
throw err;
|
|
69
|
+
this.assertLocal("stop()");
|
|
70
|
+
return this.withLock(async () => {
|
|
71
|
+
const container = this.docker.getContainer(this.containerId);
|
|
72
|
+
try {
|
|
73
|
+
await container.stop({ t: 10 });
|
|
42
74
|
}
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
75
|
+
catch (err) {
|
|
76
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
77
|
+
if (!msg.includes("not running") && !msg.includes("already stopped")) {
|
|
78
|
+
throw err;
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
logger.info(`Instance stopped`, { id: this.id, containerName: this.containerName });
|
|
82
|
+
this.emit("bot.stopped");
|
|
83
|
+
});
|
|
46
84
|
}
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
85
|
+
/**
|
|
86
|
+
* Restart the container.
|
|
87
|
+
* Callers that need an image update should call pullImage() first.
|
|
88
|
+
*/
|
|
89
|
+
async restart() {
|
|
90
|
+
this.assertLocal("restart()");
|
|
91
|
+
return this.withLock(async () => {
|
|
92
|
+
this.botMetricsTracker?.reset(this.id);
|
|
93
|
+
const container = this.docker.getContainer(this.containerId);
|
|
94
|
+
const info = await container.inspect();
|
|
95
|
+
const validStates = new Set(["running", "stopped", "exited", "dead"]);
|
|
96
|
+
const currentState = typeof info.State.Status === "string" && info.State.Status ? info.State.Status : "unknown";
|
|
97
|
+
if (!validStates.has(currentState)) {
|
|
98
|
+
throw new Error(`Cannot restart instance ${this.id}: container is in state "${currentState}". ` +
|
|
99
|
+
`Valid states: ${[...validStates].join(", ")}.`);
|
|
57
100
|
}
|
|
58
|
-
|
|
59
|
-
|
|
101
|
+
await container.restart();
|
|
102
|
+
logger.info(`Instance restarted`, { id: this.id, containerName: this.containerName });
|
|
103
|
+
this.emit("bot.restarted");
|
|
104
|
+
});
|
|
105
|
+
}
|
|
106
|
+
/**
|
|
107
|
+
* Pull the latest version of this instance's image.
|
|
108
|
+
* Call before restart() to update the image before restarting.
|
|
109
|
+
*/
|
|
110
|
+
async pullImage() {
|
|
111
|
+
this.assertLocal("pullImage()");
|
|
112
|
+
logger.info(`Pulling image ${this.profile.image}`, { id: this.id });
|
|
113
|
+
const username = process.env.REGISTRY_USERNAME;
|
|
114
|
+
const password = process.env.REGISTRY_PASSWORD;
|
|
115
|
+
const server = process.env.REGISTRY_SERVER;
|
|
116
|
+
const authconfig = username && password ? { username, password, serveraddress: server ?? "ghcr.io" } : undefined;
|
|
117
|
+
const stream = await this.docker.pull(this.profile.image, authconfig ? { authconfig } : {});
|
|
118
|
+
await new Promise((resolve, reject) => {
|
|
119
|
+
this.docker.modem.followProgress(stream, (err) => {
|
|
120
|
+
if (err)
|
|
121
|
+
reject(err);
|
|
122
|
+
else
|
|
123
|
+
resolve();
|
|
124
|
+
});
|
|
125
|
+
});
|
|
126
|
+
logger.info(`Image pulled`, { id: this.id, image: this.profile.image });
|
|
127
|
+
}
|
|
128
|
+
async remove(removeVolumes = false) {
|
|
129
|
+
this.assertLocal("remove()");
|
|
130
|
+
return this.withLock(async () => {
|
|
131
|
+
const container = this.docker.getContainer(this.containerId);
|
|
60
132
|
try {
|
|
61
|
-
await
|
|
133
|
+
await container.stop({ t: 5 }).catch(() => { });
|
|
134
|
+
await container.remove({ force: true, v: removeVolumes });
|
|
62
135
|
}
|
|
63
136
|
catch (err) {
|
|
64
|
-
|
|
137
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
138
|
+
if (!msg.includes("No such container")) {
|
|
139
|
+
throw err;
|
|
140
|
+
}
|
|
65
141
|
}
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
142
|
+
if (this.proxyManager) {
|
|
143
|
+
try {
|
|
144
|
+
await this.proxyManager.removeRoute(this.id);
|
|
145
|
+
}
|
|
146
|
+
catch (err) {
|
|
147
|
+
logger.warn("Proxy route cleanup failed (non-fatal)", { id: this.id, err });
|
|
148
|
+
}
|
|
149
|
+
}
|
|
150
|
+
logger.info(`Instance removed`, { id: this.id, containerName: this.containerName });
|
|
151
|
+
this.emit("bot.removed");
|
|
152
|
+
});
|
|
69
153
|
}
|
|
70
|
-
|
|
154
|
+
/**
|
|
155
|
+
* Simple container state check (running / stopped / gone).
|
|
156
|
+
*/
|
|
157
|
+
async containerState() {
|
|
158
|
+
this.assertLocal("containerState()");
|
|
71
159
|
try {
|
|
72
160
|
const container = this.docker.getContainer(this.containerId);
|
|
73
161
|
const info = await container.inspect();
|
|
@@ -77,6 +165,149 @@ export class Instance {
|
|
|
77
165
|
return "gone";
|
|
78
166
|
}
|
|
79
167
|
}
|
|
168
|
+
/**
|
|
169
|
+
* Full status including profile data, container state, resource stats,
|
|
170
|
+
* and application metrics. Returns BotStatus.
|
|
171
|
+
*/
|
|
172
|
+
async status() {
|
|
173
|
+
this.assertLocal("status()");
|
|
174
|
+
try {
|
|
175
|
+
const container = this.docker.getContainer(this.containerId);
|
|
176
|
+
const info = await container.inspect();
|
|
177
|
+
let stats = null;
|
|
178
|
+
if (info.State.Running) {
|
|
179
|
+
try {
|
|
180
|
+
stats = await this.getStats(container);
|
|
181
|
+
}
|
|
182
|
+
catch {
|
|
183
|
+
// stats not available
|
|
184
|
+
}
|
|
185
|
+
}
|
|
186
|
+
const now = new Date().toISOString();
|
|
187
|
+
return {
|
|
188
|
+
id: this.profile.id,
|
|
189
|
+
name: this.profile.name,
|
|
190
|
+
description: this.profile.description,
|
|
191
|
+
image: this.profile.image,
|
|
192
|
+
containerId: info.Id,
|
|
193
|
+
state: info.State.Status,
|
|
194
|
+
health: info.State.Health?.Status ?? null,
|
|
195
|
+
uptime: info.State.Running && info.State.StartedAt ? info.State.StartedAt : null,
|
|
196
|
+
startedAt: info.State.StartedAt || null,
|
|
197
|
+
createdAt: info.Created || now,
|
|
198
|
+
updatedAt: now,
|
|
199
|
+
stats,
|
|
200
|
+
applicationMetrics: this.botMetricsTracker?.getMetrics(this.profile.id) ?? null,
|
|
201
|
+
};
|
|
202
|
+
}
|
|
203
|
+
catch {
|
|
204
|
+
return this.offlineStatus();
|
|
205
|
+
}
|
|
206
|
+
}
|
|
207
|
+
/**
|
|
208
|
+
* Get container logs (demultiplexed to plain text).
|
|
209
|
+
*/
|
|
210
|
+
async logs(tail = 100) {
|
|
211
|
+
this.assertLocal("logs()");
|
|
212
|
+
const container = this.docker.getContainer(this.containerId);
|
|
213
|
+
const logBuffer = await container.logs({
|
|
214
|
+
stdout: true,
|
|
215
|
+
stderr: true,
|
|
216
|
+
tail,
|
|
217
|
+
timestamps: true,
|
|
218
|
+
});
|
|
219
|
+
// Docker returns multiplexed binary frames when Tty is false (the default).
|
|
220
|
+
// Demultiplex by stripping the 8-byte header from each frame so callers
|
|
221
|
+
// receive plain text instead of binary garbage interleaved with log lines.
|
|
222
|
+
const buf = Buffer.isBuffer(logBuffer) ? logBuffer : Buffer.from(logBuffer, "binary");
|
|
223
|
+
const chunks = [];
|
|
224
|
+
let offset = 0;
|
|
225
|
+
while (offset + 8 <= buf.length) {
|
|
226
|
+
const frameSize = buf.readUInt32BE(offset + 4);
|
|
227
|
+
const end = offset + 8 + frameSize;
|
|
228
|
+
if (end > buf.length)
|
|
229
|
+
break;
|
|
230
|
+
chunks.push(buf.subarray(offset + 8, end));
|
|
231
|
+
offset = end;
|
|
232
|
+
}
|
|
233
|
+
// If demux produced nothing (e.g. TTY container), fall back to raw string
|
|
234
|
+
return chunks.length > 0 ? Buffer.concat(chunks).toString("utf-8") : buf.toString("utf-8");
|
|
235
|
+
}
|
|
236
|
+
/**
|
|
237
|
+
* Stream container logs in real-time (follow mode).
|
|
238
|
+
* Returns a Node.js ReadableStream that emits plain-text log chunks (already demultiplexed).
|
|
239
|
+
* Caller is responsible for destroying the stream when done.
|
|
240
|
+
*/
|
|
241
|
+
async logStream(opts) {
|
|
242
|
+
this.assertLocal("logStream()");
|
|
243
|
+
const container = this.docker.getContainer(this.containerId);
|
|
244
|
+
const logOpts = {
|
|
245
|
+
stdout: true,
|
|
246
|
+
stderr: true,
|
|
247
|
+
follow: true,
|
|
248
|
+
tail: opts.tail ?? 100,
|
|
249
|
+
timestamps: true,
|
|
250
|
+
};
|
|
251
|
+
if (opts.since) {
|
|
252
|
+
logOpts.since = opts.since;
|
|
253
|
+
}
|
|
254
|
+
// Docker returns a multiplexed binary stream when Tty is false (the default for
|
|
255
|
+
// containers created by createContainer without Tty:true). Demultiplex it so
|
|
256
|
+
// callers receive plain text without 8-byte binary frame headers.
|
|
257
|
+
const multiplexed = (await container.logs(logOpts));
|
|
258
|
+
const pt = new PassThrough();
|
|
259
|
+
this.docker.modem.demuxStream(multiplexed, pt, pt);
|
|
260
|
+
return pt;
|
|
261
|
+
}
|
|
262
|
+
/**
|
|
263
|
+
* Get disk usage for this instance's /data volume.
|
|
264
|
+
* Returns null if the container is not running or exec fails.
|
|
265
|
+
*/
|
|
266
|
+
async getVolumeUsage() {
|
|
267
|
+
this.assertLocal("getVolumeUsage()");
|
|
268
|
+
try {
|
|
269
|
+
const container = this.docker.getContainer(this.containerId);
|
|
270
|
+
const info = await container.inspect();
|
|
271
|
+
if (!info.State.Running)
|
|
272
|
+
return null;
|
|
273
|
+
const exec = await container.exec({
|
|
274
|
+
Cmd: ["df", "-B1", "/data"],
|
|
275
|
+
AttachStdout: true,
|
|
276
|
+
AttachStderr: false,
|
|
277
|
+
});
|
|
278
|
+
const output = await new Promise((resolve, reject) => {
|
|
279
|
+
exec.start({}, (err, stream) => {
|
|
280
|
+
if (err)
|
|
281
|
+
return reject(err);
|
|
282
|
+
if (!stream)
|
|
283
|
+
return reject(new Error("No stream from exec"));
|
|
284
|
+
let data = "";
|
|
285
|
+
stream.on("data", (chunk) => {
|
|
286
|
+
data += chunk.toString();
|
|
287
|
+
});
|
|
288
|
+
stream.on("end", () => resolve(data));
|
|
289
|
+
stream.on("error", reject);
|
|
290
|
+
});
|
|
291
|
+
});
|
|
292
|
+
// Parse df output — second line has the numbers
|
|
293
|
+
const lines = output.trim().split("\n");
|
|
294
|
+
if (lines.length < 2)
|
|
295
|
+
return null;
|
|
296
|
+
const parts = lines[lines.length - 1].split(/\s+/);
|
|
297
|
+
if (parts.length < 4)
|
|
298
|
+
return null;
|
|
299
|
+
const totalBytes = parseInt(parts[1], 10);
|
|
300
|
+
const usedBytes = parseInt(parts[2], 10);
|
|
301
|
+
const availableBytes = parseInt(parts[3], 10);
|
|
302
|
+
if (Number.isNaN(totalBytes) || Number.isNaN(usedBytes) || Number.isNaN(availableBytes))
|
|
303
|
+
return null;
|
|
304
|
+
return { usedBytes, totalBytes, availableBytes };
|
|
305
|
+
}
|
|
306
|
+
catch {
|
|
307
|
+
logger.warn(`Failed to get volume usage for instance ${this.id}`);
|
|
308
|
+
return null;
|
|
309
|
+
}
|
|
310
|
+
}
|
|
80
311
|
/**
|
|
81
312
|
* Register this instance in the billing system.
|
|
82
313
|
* Skip for ephemeral instances — they bill per-token, not per-instance.
|
|
@@ -123,6 +354,39 @@ export class Instance {
|
|
|
123
354
|
logger.warn("Proxy route registration failed (non-fatal)", { id: this.id, err });
|
|
124
355
|
}
|
|
125
356
|
}
|
|
357
|
+
offlineStatus() {
|
|
358
|
+
const now = new Date().toISOString();
|
|
359
|
+
return {
|
|
360
|
+
id: this.profile.id,
|
|
361
|
+
name: this.profile.name,
|
|
362
|
+
description: this.profile.description,
|
|
363
|
+
image: this.profile.image,
|
|
364
|
+
containerId: null,
|
|
365
|
+
state: "stopped",
|
|
366
|
+
health: null,
|
|
367
|
+
uptime: null,
|
|
368
|
+
startedAt: null,
|
|
369
|
+
createdAt: now,
|
|
370
|
+
updatedAt: now,
|
|
371
|
+
stats: null,
|
|
372
|
+
applicationMetrics: null,
|
|
373
|
+
};
|
|
374
|
+
}
|
|
375
|
+
async getStats(container) {
|
|
376
|
+
const raw = await container.stats({ stream: false });
|
|
377
|
+
const cpuDelta = raw.cpu_stats.cpu_usage.total_usage - raw.precpu_stats.cpu_usage.total_usage;
|
|
378
|
+
const systemDelta = raw.cpu_stats.system_cpu_usage - raw.precpu_stats.system_cpu_usage;
|
|
379
|
+
const numCpus = raw.cpu_stats.online_cpus || 1;
|
|
380
|
+
const cpuPercent = systemDelta > 0 ? (cpuDelta / systemDelta) * numCpus * 100 : 0;
|
|
381
|
+
const memUsage = raw.memory_stats.usage || 0;
|
|
382
|
+
const memLimit = raw.memory_stats.limit || 1;
|
|
383
|
+
return {
|
|
384
|
+
cpuPercent: Math.round(cpuPercent * 100) / 100,
|
|
385
|
+
memoryUsageMb: Math.round(memUsage / 1024 / 1024),
|
|
386
|
+
memoryLimitMb: Math.round(memLimit / 1024 / 1024),
|
|
387
|
+
memoryPercent: Math.round((memUsage / memLimit) * 100 * 100) / 100,
|
|
388
|
+
};
|
|
389
|
+
}
|
|
126
390
|
emit(type) {
|
|
127
391
|
if (this.eventEmitter) {
|
|
128
392
|
this.eventEmitter.emit({
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|