@wopr-network/platform-core 1.39.4 → 1.39.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/credits/credit-expiry-cron.test.js +30 -0
- package/dist/credits/ledger.js +11 -5
- package/dist/credits/ledger.test.js +87 -0
- package/dist/db/schema/ledger.js +6 -0
- package/dist/fleet/fleet-manager.d.ts +14 -35
- package/dist/fleet/fleet-manager.js +52 -236
- package/dist/fleet/fleet-manager.test.js +13 -85
- package/dist/fleet/instance.d.ts +58 -3
- package/dist/fleet/instance.js +297 -33
- package/dist/fleet/instance.test.d.ts +1 -0
- package/dist/fleet/instance.test.js +343 -0
- package/dist/node-agent/types.d.ts +1 -1
- package/package.json +1 -1
- package/src/credits/README.md +106 -0
- package/src/credits/credit-expiry-cron.test.ts +36 -0
- package/src/credits/ledger.test.ts +113 -0
- package/src/credits/ledger.ts +13 -7
- package/src/db/schema/ledger.ts +6 -0
- package/src/fleet/fleet-manager.test.ts +13 -111
- package/src/fleet/fleet-manager.ts +50 -255
- package/src/fleet/instance.test.ts +390 -0
- package/src/fleet/instance.ts +318 -33
package/src/credits/ledger.ts
CHANGED
|
@@ -372,19 +372,25 @@ export class DrizzleLedger implements ILedger {
|
|
|
372
372
|
}
|
|
373
373
|
|
|
374
374
|
// Verify balance before hitting DB
|
|
375
|
-
let totalDebit =
|
|
376
|
-
let totalCredit =
|
|
375
|
+
let totalDebit = 0n;
|
|
376
|
+
let totalCredit = 0n;
|
|
377
377
|
for (const line of input.lines) {
|
|
378
378
|
if (line.amount.isZero() || line.amount.isNegative()) {
|
|
379
379
|
throw new Error("Journal line amounts must be positive");
|
|
380
380
|
}
|
|
381
|
-
if (line.side === "debit") totalDebit += line.amount.toRaw();
|
|
382
|
-
else totalCredit += line.amount.toRaw();
|
|
381
|
+
if (line.side === "debit") totalDebit += BigInt(line.amount.toRaw());
|
|
382
|
+
else totalCredit += BigInt(line.amount.toRaw());
|
|
383
383
|
}
|
|
384
384
|
if (totalDebit !== totalCredit) {
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
385
|
+
const fmtDebit =
|
|
386
|
+
totalDebit <= BigInt(Number.MAX_SAFE_INTEGER)
|
|
387
|
+
? Credit.fromRaw(Number(totalDebit)).toDisplayString()
|
|
388
|
+
: `${totalDebit} raw`;
|
|
389
|
+
const fmtCredit =
|
|
390
|
+
totalCredit <= BigInt(Number.MAX_SAFE_INTEGER)
|
|
391
|
+
? Credit.fromRaw(Number(totalCredit)).toDisplayString()
|
|
392
|
+
: `${totalCredit} raw`;
|
|
393
|
+
throw new Error(`Unbalanced entry: debits=${fmtDebit}, credits=${fmtCredit}`);
|
|
388
394
|
}
|
|
389
395
|
|
|
390
396
|
return this.db.transaction(async (tx) => {
|
package/src/db/schema/ledger.ts
CHANGED
|
@@ -70,6 +70,9 @@ export const journalLines = pgTable(
|
|
|
70
70
|
accountId: text("account_id")
|
|
71
71
|
.notNull()
|
|
72
72
|
.references(() => accounts.id),
|
|
73
|
+
// mode: "number" truncates values > Number.MAX_SAFE_INTEGER (≈$9.007M in nanodollars).
|
|
74
|
+
// Credit.fromRaw() guards against this with a RangeError.
|
|
75
|
+
// If balances approach $9M, migrate to mode: "bigint" (returns string from Drizzle).
|
|
73
76
|
amount: bigint("amount", { mode: "number" }).notNull(), // nanodollars, always positive
|
|
74
77
|
side: entrySideEnum("side").notNull(),
|
|
75
78
|
},
|
|
@@ -89,6 +92,9 @@ export const accountBalances = pgTable("account_balances", {
|
|
|
89
92
|
accountId: text("account_id")
|
|
90
93
|
.primaryKey()
|
|
91
94
|
.references(() => accounts.id),
|
|
95
|
+
// mode: "number" truncates values > Number.MAX_SAFE_INTEGER (≈$9.007M in nanodollars).
|
|
96
|
+
// Credit.fromRaw() guards against this with a RangeError.
|
|
97
|
+
// If balances approach $9M, migrate to mode: "bigint" (returns string from Drizzle).
|
|
92
98
|
balance: bigint("balance", { mode: "number" }).notNull().default(0), // net balance in nanodollars
|
|
93
99
|
lastUpdated: text("last_updated").notNull().default(sql`(now())`),
|
|
94
100
|
});
|
|
@@ -157,73 +157,7 @@ describe("FleetManager", () => {
|
|
|
157
157
|
});
|
|
158
158
|
});
|
|
159
159
|
|
|
160
|
-
|
|
161
|
-
it("pulls image before restarting a running container", async () => {
|
|
162
|
-
// Store the profile first; default mockContainer has state "running" — valid for restart
|
|
163
|
-
await store.save({ id: "bot-id", ...PROFILE_PARAMS });
|
|
164
|
-
docker.listContainers.mockResolvedValue([{ Id: "container-123" }]);
|
|
165
|
-
|
|
166
|
-
await fleet.restart("bot-id");
|
|
167
|
-
|
|
168
|
-
// Pull is called first
|
|
169
|
-
expect(docker.pull).toHaveBeenCalledWith(PROFILE_PARAMS.image, {});
|
|
170
|
-
// Then restart
|
|
171
|
-
expect(container.restart).toHaveBeenCalled();
|
|
172
|
-
});
|
|
173
|
-
|
|
174
|
-
it("restarts a container in exited state (crash recovery)", async () => {
|
|
175
|
-
await store.save({ id: "bot-id", ...PROFILE_PARAMS });
|
|
176
|
-
container.inspect.mockResolvedValue({
|
|
177
|
-
Id: "container-123",
|
|
178
|
-
Created: "2026-01-01T00:00:00Z",
|
|
179
|
-
State: { Status: "exited", Running: false, StartedAt: "", Health: null },
|
|
180
|
-
});
|
|
181
|
-
docker.listContainers.mockResolvedValue([{ Id: "container-123" }]);
|
|
182
|
-
|
|
183
|
-
await fleet.restart("bot-id");
|
|
184
|
-
expect(container.restart).toHaveBeenCalled();
|
|
185
|
-
});
|
|
186
|
-
|
|
187
|
-
it("restarts a container in stopped state", async () => {
|
|
188
|
-
await store.save({ id: "bot-id", ...PROFILE_PARAMS });
|
|
189
|
-
container.inspect.mockResolvedValue({
|
|
190
|
-
Id: "container-123",
|
|
191
|
-
Created: "2026-01-01T00:00:00Z",
|
|
192
|
-
State: { Status: "stopped", Running: false, StartedAt: "", Health: null },
|
|
193
|
-
});
|
|
194
|
-
docker.listContainers.mockResolvedValue([{ Id: "container-123" }]);
|
|
195
|
-
|
|
196
|
-
await fleet.restart("bot-id");
|
|
197
|
-
expect(container.restart).toHaveBeenCalled();
|
|
198
|
-
});
|
|
199
|
-
|
|
200
|
-
it("restarts a container in dead state (crash recovery)", async () => {
|
|
201
|
-
await store.save({ id: "bot-id", ...PROFILE_PARAMS });
|
|
202
|
-
container.inspect.mockResolvedValue({
|
|
203
|
-
Id: "container-123",
|
|
204
|
-
Created: "2026-01-01T00:00:00Z",
|
|
205
|
-
State: { Status: "dead", Running: false, StartedAt: "", Health: null },
|
|
206
|
-
});
|
|
207
|
-
docker.listContainers.mockResolvedValue([{ Id: "container-123" }]);
|
|
208
|
-
|
|
209
|
-
await fleet.restart("bot-id");
|
|
210
|
-
expect(container.restart).toHaveBeenCalled();
|
|
211
|
-
});
|
|
212
|
-
|
|
213
|
-
it("does not restart if pull fails", async () => {
|
|
214
|
-
await store.save({ id: "bot-id", ...PROFILE_PARAMS });
|
|
215
|
-
docker.modem.followProgress.mockImplementation((_stream: unknown, cb: (err: Error | null) => void) =>
|
|
216
|
-
cb(new Error("Pull failed")),
|
|
217
|
-
);
|
|
218
|
-
|
|
219
|
-
await expect(fleet.restart("bot-id")).rejects.toThrow("Pull failed");
|
|
220
|
-
expect(container.restart).not.toHaveBeenCalled();
|
|
221
|
-
});
|
|
222
|
-
|
|
223
|
-
it("throws BotNotFoundError for missing profile", async () => {
|
|
224
|
-
await expect(fleet.restart("missing")).rejects.toThrow(BotNotFoundError);
|
|
225
|
-
});
|
|
226
|
-
});
|
|
160
|
+
// restart tests moved to instance.test.ts — Instance.restart() + Instance.pullImage()
|
|
227
161
|
|
|
228
162
|
describe("remove", () => {
|
|
229
163
|
it("stops running container, removes it, and deletes profile", async () => {
|
|
@@ -233,7 +167,7 @@ describe("FleetManager", () => {
|
|
|
233
167
|
await fleet.remove("bot-id");
|
|
234
168
|
|
|
235
169
|
expect(container.stop).toHaveBeenCalled();
|
|
236
|
-
expect(container.remove).toHaveBeenCalledWith({ v: false });
|
|
170
|
+
expect(container.remove).toHaveBeenCalledWith({ force: true, v: false });
|
|
237
171
|
expect(store.delete).toHaveBeenCalledWith("bot-id");
|
|
238
172
|
});
|
|
239
173
|
|
|
@@ -243,7 +177,7 @@ describe("FleetManager", () => {
|
|
|
243
177
|
|
|
244
178
|
await fleet.remove("bot-id", true);
|
|
245
179
|
|
|
246
|
-
expect(container.remove).toHaveBeenCalledWith({ v: true });
|
|
180
|
+
expect(container.remove).toHaveBeenCalledWith({ force: true, v: true });
|
|
247
181
|
});
|
|
248
182
|
|
|
249
183
|
it("deletes profile even when no container exists", async () => {
|
|
@@ -304,6 +238,7 @@ describe("FleetManager", () => {
|
|
|
304
238
|
|
|
305
239
|
describe("logs", () => {
|
|
306
240
|
it("returns container logs", async () => {
|
|
241
|
+
await store.save({ id: "bot-id", ...PROFILE_PARAMS });
|
|
307
242
|
docker.listContainers.mockResolvedValue([{ Id: "container-123" }]);
|
|
308
243
|
|
|
309
244
|
const logs = await fleet.logs("bot-id", 50);
|
|
@@ -312,8 +247,7 @@ describe("FleetManager", () => {
|
|
|
312
247
|
expect(logs).toContain("log line 1");
|
|
313
248
|
});
|
|
314
249
|
|
|
315
|
-
it("throws BotNotFoundError when
|
|
316
|
-
docker.listContainers.mockResolvedValue([]);
|
|
250
|
+
it("throws BotNotFoundError when profile not found", async () => {
|
|
317
251
|
await expect(fleet.logs("missing")).rejects.toThrow(BotNotFoundError);
|
|
318
252
|
});
|
|
319
253
|
});
|
|
@@ -323,6 +257,7 @@ describe("FleetManager", () => {
|
|
|
323
257
|
const { PassThrough } = await import("node:stream");
|
|
324
258
|
const mockStream = new PassThrough();
|
|
325
259
|
container.logs.mockResolvedValue(mockStream);
|
|
260
|
+
await store.save({ id: "bot-id", ...PROFILE_PARAMS });
|
|
326
261
|
docker.listContainers.mockResolvedValue([{ Id: "container-123" }]);
|
|
327
262
|
|
|
328
263
|
const stream = await fleet.logStream("bot-id", { since: "2026-01-01T00:00:00Z", tail: 50 });
|
|
@@ -346,6 +281,7 @@ describe("FleetManager", () => {
|
|
|
346
281
|
const { PassThrough } = await import("node:stream");
|
|
347
282
|
const mockStream = new PassThrough();
|
|
348
283
|
container.logs.mockResolvedValue(mockStream);
|
|
284
|
+
await store.save({ id: "bot-id", ...PROFILE_PARAMS });
|
|
349
285
|
docker.listContainers.mockResolvedValue([{ Id: "container-123" }]);
|
|
350
286
|
|
|
351
287
|
await fleet.logStream("bot-id", {});
|
|
@@ -359,8 +295,7 @@ describe("FleetManager", () => {
|
|
|
359
295
|
mockStream.destroy();
|
|
360
296
|
});
|
|
361
297
|
|
|
362
|
-
it("throws BotNotFoundError when
|
|
363
|
-
docker.listContainers.mockResolvedValue([]);
|
|
298
|
+
it("throws BotNotFoundError when profile not found", async () => {
|
|
364
299
|
await expect(fleet.logStream("missing", {})).rejects.toThrow(BotNotFoundError);
|
|
365
300
|
});
|
|
366
301
|
|
|
@@ -718,44 +653,7 @@ describe("FleetManager", () => {
|
|
|
718
653
|
expect(docker.pull).toHaveBeenCalled();
|
|
719
654
|
});
|
|
720
655
|
|
|
721
|
-
|
|
722
|
-
const container = mockContainer();
|
|
723
|
-
const docker = mockDocker(container);
|
|
724
|
-
const store = mockStore();
|
|
725
|
-
const bus = mockCommandBus();
|
|
726
|
-
const instanceRepo = mockInstanceRepo();
|
|
727
|
-
|
|
728
|
-
const botId = "bot-restart-1";
|
|
729
|
-
await instanceRepo.create({ id: botId, tenantId: "t1", name: "restart-bot", nodeId: "node-3" });
|
|
730
|
-
|
|
731
|
-
const fleet = new FleetManager(
|
|
732
|
-
docker as unknown as Docker,
|
|
733
|
-
store,
|
|
734
|
-
undefined,
|
|
735
|
-
undefined,
|
|
736
|
-
undefined,
|
|
737
|
-
bus,
|
|
738
|
-
instanceRepo,
|
|
739
|
-
);
|
|
740
|
-
|
|
741
|
-
await store.save({
|
|
742
|
-
id: botId,
|
|
743
|
-
tenantId: "t1",
|
|
744
|
-
name: "restart-bot",
|
|
745
|
-
description: "",
|
|
746
|
-
image: "ghcr.io/wopr-network/wopr:latest",
|
|
747
|
-
env: {},
|
|
748
|
-
restartPolicy: "unless-stopped",
|
|
749
|
-
} as BotProfile);
|
|
750
|
-
|
|
751
|
-
await fleet.restart(botId);
|
|
752
|
-
|
|
753
|
-
expect(bus.send).toHaveBeenCalledWith("node-3", {
|
|
754
|
-
type: "bot.restart",
|
|
755
|
-
payload: { name: "restart-bot" },
|
|
756
|
-
});
|
|
757
|
-
expect(container.restart).not.toHaveBeenCalled();
|
|
758
|
-
});
|
|
656
|
+
// remote restart test removed — restart moved to Instance
|
|
759
657
|
|
|
760
658
|
it("should dispatch remove via commandBus when bot has nodeId", async () => {
|
|
761
659
|
const container = mockContainer();
|
|
@@ -850,6 +748,7 @@ describe("FleetManager", () => {
|
|
|
850
748
|
const containerWithExec = mockContainer({
|
|
851
749
|
exec: vi.fn().mockResolvedValue(execMock),
|
|
852
750
|
});
|
|
751
|
+
await store.save({ id: "bot-id", ...PROFILE_PARAMS });
|
|
853
752
|
docker.listContainers.mockResolvedValue([{ Id: "container-123" }]);
|
|
854
753
|
docker.getContainer.mockReturnValue(containerWithExec);
|
|
855
754
|
|
|
@@ -863,6 +762,7 @@ describe("FleetManager", () => {
|
|
|
863
762
|
});
|
|
864
763
|
|
|
865
764
|
it("returns null when container is not found", async () => {
|
|
765
|
+
await store.save({ id: "bot-id", ...PROFILE_PARAMS });
|
|
866
766
|
docker.listContainers.mockResolvedValue([]);
|
|
867
767
|
const result = await fleet.getVolumeUsage("bot-id");
|
|
868
768
|
expect(result).toBeNull();
|
|
@@ -875,6 +775,7 @@ describe("FleetManager", () => {
|
|
|
875
775
|
State: { Running: false },
|
|
876
776
|
}),
|
|
877
777
|
});
|
|
778
|
+
await store.save({ id: "bot-id", ...PROFILE_PARAMS });
|
|
878
779
|
docker.listContainers.mockResolvedValue([{ Id: "container-123" }]);
|
|
879
780
|
docker.getContainer.mockReturnValue(stoppedContainer);
|
|
880
781
|
|
|
@@ -886,6 +787,7 @@ describe("FleetManager", () => {
|
|
|
886
787
|
const containerWithFailingExec = mockContainer({
|
|
887
788
|
exec: vi.fn().mockRejectedValue(new Error("exec failed")),
|
|
888
789
|
});
|
|
790
|
+
await store.save({ id: "bot-id", ...PROFILE_PARAMS });
|
|
889
791
|
docker.listContainers.mockResolvedValue([{ Id: "container-123" }]);
|
|
890
792
|
docker.getContainer.mockReturnValue(containerWithFailingExec);
|
|
891
793
|
|
|
@@ -14,7 +14,7 @@ import { Instance } from "./instance.js";
|
|
|
14
14
|
import type { INodeCommandBus } from "./node-command-bus.js";
|
|
15
15
|
import type { IProfileStore } from "./profile-store.js";
|
|
16
16
|
import { getSharedVolumeConfig } from "./shared-volume-config.js";
|
|
17
|
-
import type { BotProfile, BotStatus
|
|
17
|
+
import type { BotProfile, BotStatus } from "./types.js";
|
|
18
18
|
|
|
19
19
|
const CONTAINER_LABEL = "wopr.managed";
|
|
20
20
|
const CONTAINER_ID_LABEL = "wopr.bot-id";
|
|
@@ -134,6 +134,7 @@ export class FleetManager {
|
|
|
134
134
|
instanceRepo: this.instanceRepo,
|
|
135
135
|
proxyManager: this.proxyManager,
|
|
136
136
|
eventEmitter: this.eventEmitter,
|
|
137
|
+
botMetricsTracker: this.botMetricsTracker,
|
|
137
138
|
});
|
|
138
139
|
remoteInstance.emitCreated();
|
|
139
140
|
return remoteInstance;
|
|
@@ -203,46 +204,15 @@ export class FleetManager {
|
|
|
203
204
|
instanceRepo: this.instanceRepo,
|
|
204
205
|
proxyManager: this.proxyManager,
|
|
205
206
|
eventEmitter: this.eventEmitter,
|
|
206
|
-
|
|
207
|
-
}
|
|
208
|
-
|
|
209
|
-
/**
|
|
210
|
-
* Restart: pull new image BEFORE restarting container to avoid downtime on pull failure.
|
|
211
|
-
* Valid from: running, stopped, exited, dead states.
|
|
212
|
-
* Throws InvalidStateTransitionError if the container is in an invalid state (e.g. paused).
|
|
213
|
-
* For remote bots, delegates to the node agent via NodeCommandBus.
|
|
214
|
-
*/
|
|
215
|
-
async restart(id: string): Promise<void> {
|
|
216
|
-
return this.withLock(id, async () => {
|
|
217
|
-
this.botMetricsTracker?.reset(id);
|
|
218
|
-
const profile = await this.store.get(id);
|
|
219
|
-
if (!profile) throw new BotNotFoundError(id);
|
|
220
|
-
|
|
221
|
-
const remote = await this.resolveNodeId(id);
|
|
222
|
-
if (remote) {
|
|
223
|
-
await remote.commandBus.send(remote.nodeId, {
|
|
224
|
-
type: "bot.restart",
|
|
225
|
-
payload: { name: profile.name },
|
|
226
|
-
});
|
|
227
|
-
} else {
|
|
228
|
-
// Pull new image first — if this fails, old container is unchanged
|
|
229
|
-
await this.pullImage(profile.image);
|
|
230
|
-
|
|
231
|
-
const container = await this.findContainer(id);
|
|
232
|
-
if (!container) throw new BotNotFoundError(id);
|
|
233
|
-
const info = await container.inspect();
|
|
234
|
-
const validRestartStates = new Set(["running", "stopped", "exited", "dead"]);
|
|
235
|
-
this.assertValidState(id, info.State.Status, "restart", validRestartStates);
|
|
236
|
-
await container.restart();
|
|
237
|
-
}
|
|
238
|
-
logger.info(`Restarted bot ${id}`);
|
|
239
|
-
this.emitEvent("bot.restarted", id, profile.tenantId);
|
|
207
|
+
botMetricsTracker: this.botMetricsTracker,
|
|
240
208
|
});
|
|
241
209
|
}
|
|
242
210
|
|
|
243
211
|
/**
|
|
244
212
|
* Remove a bot: stop container, remove it, optionally remove volumes, delete profile.
|
|
245
213
|
* For remote bots, delegates stop+remove to the node agent via NodeCommandBus.
|
|
214
|
+
* Container removal is delegated to Instance.remove(); fleet-level cleanup
|
|
215
|
+
* (profile store, network policy) stays here.
|
|
246
216
|
*/
|
|
247
217
|
async remove(id: string, removeVolumes = false): Promise<void> {
|
|
248
218
|
return this.withLock(id, async () => {
|
|
@@ -256,13 +226,11 @@ export class FleetManager {
|
|
|
256
226
|
payload: { name: profile.name, removeVolumes },
|
|
257
227
|
});
|
|
258
228
|
} else {
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
}
|
|
265
|
-
await container.remove({ v: removeVolumes });
|
|
229
|
+
try {
|
|
230
|
+
const instance = await this.buildInstance(profile);
|
|
231
|
+
await instance.remove(removeVolumes);
|
|
232
|
+
} catch {
|
|
233
|
+
// Container may already be gone — not fatal for fleet-level cleanup
|
|
266
234
|
}
|
|
267
235
|
}
|
|
268
236
|
|
|
@@ -272,9 +240,6 @@ export class FleetManager {
|
|
|
272
240
|
}
|
|
273
241
|
|
|
274
242
|
await this.store.delete(id);
|
|
275
|
-
if (this.proxyManager) {
|
|
276
|
-
this.proxyManager.removeRoute(id);
|
|
277
|
-
}
|
|
278
243
|
logger.info(`Removed bot ${id}`);
|
|
279
244
|
this.emitEvent("bot.removed", id, profile.tenantId);
|
|
280
245
|
});
|
|
@@ -282,17 +247,13 @@ export class FleetManager {
|
|
|
282
247
|
|
|
283
248
|
/**
|
|
284
249
|
* Get live status of a single bot.
|
|
250
|
+
* Delegates to Instance.status() which returns a full BotStatus.
|
|
251
|
+
* Falls back to offline status when no container exists.
|
|
285
252
|
*/
|
|
286
253
|
async status(id: string): Promise<BotStatus> {
|
|
287
254
|
const profile = await this.store.get(id);
|
|
288
255
|
if (!profile) throw new BotNotFoundError(id);
|
|
289
|
-
|
|
290
|
-
const container = await this.findContainer(id);
|
|
291
|
-
if (!container) {
|
|
292
|
-
return this.offlineStatus(profile);
|
|
293
|
-
}
|
|
294
|
-
|
|
295
|
-
return this.buildStatus(profile, container);
|
|
256
|
+
return this.statusForProfile(profile);
|
|
296
257
|
}
|
|
297
258
|
|
|
298
259
|
/**
|
|
@@ -313,41 +274,17 @@ export class FleetManager {
|
|
|
313
274
|
}
|
|
314
275
|
|
|
315
276
|
/**
|
|
316
|
-
* Get container logs.
|
|
277
|
+
* Get container logs. Delegates to Instance.logs().
|
|
317
278
|
*/
|
|
318
279
|
async logs(id: string, tail = 100): Promise<string> {
|
|
319
|
-
const
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
const logBuffer = await container.logs({
|
|
323
|
-
stdout: true,
|
|
324
|
-
stderr: true,
|
|
325
|
-
tail,
|
|
326
|
-
timestamps: true,
|
|
327
|
-
});
|
|
328
|
-
|
|
329
|
-
// Docker returns multiplexed binary frames when Tty is false (the default).
|
|
330
|
-
// Demultiplex by stripping the 8-byte header from each frame so callers
|
|
331
|
-
// receive plain text instead of binary garbage interleaved with log lines.
|
|
332
|
-
const buf = Buffer.isBuffer(logBuffer) ? logBuffer : Buffer.from(logBuffer as unknown as string, "binary");
|
|
333
|
-
const chunks: Buffer[] = [];
|
|
334
|
-
let offset = 0;
|
|
335
|
-
while (offset + 8 <= buf.length) {
|
|
336
|
-
const frameSize = buf.readUInt32BE(offset + 4);
|
|
337
|
-
const end = offset + 8 + frameSize;
|
|
338
|
-
if (end > buf.length) break;
|
|
339
|
-
chunks.push(buf.subarray(offset + 8, end));
|
|
340
|
-
offset = end;
|
|
341
|
-
}
|
|
342
|
-
// If demux produced nothing (e.g. TTY container), fall back to raw string
|
|
343
|
-
return chunks.length > 0 ? Buffer.concat(chunks).toString("utf-8") : buf.toString("utf-8");
|
|
280
|
+
const instance = await this.getInstance(id);
|
|
281
|
+
return instance.logs(tail);
|
|
344
282
|
}
|
|
345
283
|
|
|
346
284
|
/**
|
|
347
285
|
* Stream container logs in real-time (follow mode).
|
|
348
|
-
* Returns a Node.js ReadableStream that emits plain-text log chunks (already demultiplexed).
|
|
349
286
|
* For remote bots, proxies via node-agent bot.logs command and returns a one-shot stream.
|
|
350
|
-
*
|
|
287
|
+
* For local bots, delegates to Instance.logStream().
|
|
351
288
|
*/
|
|
352
289
|
async logStream(id: string, opts: { since?: string; tail?: number }): Promise<NodeJS.ReadableStream> {
|
|
353
290
|
// Check for remote node assignment first (mirrors start/stop/restart pattern)
|
|
@@ -365,31 +302,20 @@ export class FleetManager {
|
|
|
365
302
|
return pt;
|
|
366
303
|
}
|
|
367
304
|
|
|
368
|
-
const
|
|
369
|
-
|
|
305
|
+
const instance = await this.getInstance(id);
|
|
306
|
+
return instance.logStream(opts);
|
|
307
|
+
}
|
|
370
308
|
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
309
|
+
/**
|
|
310
|
+
* Get disk usage for a bot's /data volume. Delegates to Instance.getVolumeUsage().
|
|
311
|
+
*/
|
|
312
|
+
async getVolumeUsage(id: string): Promise<{ usedBytes: number; totalBytes: number; availableBytes: number } | null> {
|
|
313
|
+
try {
|
|
314
|
+
const instance = await this.getInstance(id);
|
|
315
|
+
return instance.getVolumeUsage();
|
|
316
|
+
} catch {
|
|
317
|
+
return null;
|
|
380
318
|
}
|
|
381
|
-
|
|
382
|
-
// Docker returns a multiplexed binary stream when Tty is false (the default for
|
|
383
|
-
// containers created by createContainer without Tty:true). Demultiplex it so
|
|
384
|
-
// callers receive plain text without 8-byte binary frame headers.
|
|
385
|
-
const multiplexed = (await container.logs(logOpts)) as unknown as NodeJS.ReadableStream;
|
|
386
|
-
const pt = new PassThrough();
|
|
387
|
-
(
|
|
388
|
-
this.docker.modem as unknown as {
|
|
389
|
-
demuxStream(stream: NodeJS.ReadableStream, stdout: PassThrough, stderr: PassThrough): void;
|
|
390
|
-
}
|
|
391
|
-
).demuxStream(multiplexed, pt, pt);
|
|
392
|
-
return pt;
|
|
393
319
|
}
|
|
394
320
|
|
|
395
321
|
/** Fields that require container recreation when changed. */
|
|
@@ -465,57 +391,6 @@ export class FleetManager {
|
|
|
465
391
|
});
|
|
466
392
|
}
|
|
467
393
|
|
|
468
|
-
/**
|
|
469
|
-
* Get disk usage for a bot's /data volume.
|
|
470
|
-
* Returns null if the container is not running or exec fails.
|
|
471
|
-
*/
|
|
472
|
-
async getVolumeUsage(id: string): Promise<{ usedBytes: number; totalBytes: number; availableBytes: number } | null> {
|
|
473
|
-
const container = await this.findContainer(id);
|
|
474
|
-
if (!container) return null;
|
|
475
|
-
|
|
476
|
-
try {
|
|
477
|
-
const info = await container.inspect();
|
|
478
|
-
if (!info.State.Running) return null;
|
|
479
|
-
|
|
480
|
-
const exec = await container.exec({
|
|
481
|
-
Cmd: ["df", "-B1", "/data"],
|
|
482
|
-
AttachStdout: true,
|
|
483
|
-
AttachStderr: false,
|
|
484
|
-
});
|
|
485
|
-
|
|
486
|
-
const output = await new Promise<string>((resolve, reject) => {
|
|
487
|
-
exec.start({}, (err: Error | null, stream: import("node:stream").Duplex | undefined) => {
|
|
488
|
-
if (err) return reject(err);
|
|
489
|
-
if (!stream) return reject(new Error("No stream from exec"));
|
|
490
|
-
let data = "";
|
|
491
|
-
stream.on("data", (chunk: Buffer) => {
|
|
492
|
-
data += chunk.toString();
|
|
493
|
-
});
|
|
494
|
-
stream.on("end", () => resolve(data));
|
|
495
|
-
stream.on("error", reject);
|
|
496
|
-
});
|
|
497
|
-
});
|
|
498
|
-
|
|
499
|
-
// Parse df output — second line has the numbers
|
|
500
|
-
const lines = output.trim().split("\n");
|
|
501
|
-
if (lines.length < 2) return null;
|
|
502
|
-
|
|
503
|
-
const parts = lines[lines.length - 1].split(/\s+/);
|
|
504
|
-
if (parts.length < 4) return null;
|
|
505
|
-
|
|
506
|
-
const totalBytes = parseInt(parts[1], 10);
|
|
507
|
-
const usedBytes = parseInt(parts[2], 10);
|
|
508
|
-
const availableBytes = parseInt(parts[3], 10);
|
|
509
|
-
|
|
510
|
-
if (Number.isNaN(totalBytes) || Number.isNaN(usedBytes) || Number.isNaN(availableBytes)) return null;
|
|
511
|
-
|
|
512
|
-
return { usedBytes, totalBytes, availableBytes };
|
|
513
|
-
} catch {
|
|
514
|
-
logger.warn(`Failed to get volume usage for bot ${id}`);
|
|
515
|
-
return null;
|
|
516
|
-
}
|
|
517
|
-
}
|
|
518
|
-
|
|
519
394
|
/** Get the underlying profile store */
|
|
520
395
|
get profiles(): IProfileStore {
|
|
521
396
|
return this.store;
|
|
@@ -523,18 +398,6 @@ export class FleetManager {
|
|
|
523
398
|
|
|
524
399
|
// --- Private helpers ---
|
|
525
400
|
|
|
526
|
-
/**
|
|
527
|
-
* Assert that a container's current state is valid for the requested operation.
|
|
528
|
-
* Guards against undefined/null Status values from Docker (uses "unknown" as fallback).
|
|
529
|
-
* Throws InvalidStateTransitionError when the state is not in validStates.
|
|
530
|
-
*/
|
|
531
|
-
private assertValidState(id: string, rawStatus: unknown, operation: string, validStates: Set<string>): void {
|
|
532
|
-
const currentState = typeof rawStatus === "string" && rawStatus ? rawStatus : "unknown";
|
|
533
|
-
if (!validStates.has(currentState)) {
|
|
534
|
-
throw new InvalidStateTransitionError(id, operation, currentState, [...validStates]);
|
|
535
|
-
}
|
|
536
|
-
}
|
|
537
|
-
|
|
538
401
|
private async pullImage(image: string): Promise<void> {
|
|
539
402
|
logger.info(`Pulling image ${image}`);
|
|
540
403
|
|
|
@@ -655,77 +518,28 @@ export class FleetManager {
|
|
|
655
518
|
}
|
|
656
519
|
|
|
657
520
|
private async statusForProfile(profile: BotProfile): Promise<BotStatus> {
|
|
658
|
-
|
|
659
|
-
|
|
660
|
-
|
|
661
|
-
|
|
662
|
-
|
|
663
|
-
|
|
664
|
-
|
|
665
|
-
|
|
666
|
-
|
|
667
|
-
|
|
668
|
-
|
|
669
|
-
|
|
670
|
-
|
|
671
|
-
|
|
672
|
-
|
|
521
|
+
try {
|
|
522
|
+
const instance = await this.buildInstance(profile);
|
|
523
|
+
return instance.status();
|
|
524
|
+
} catch {
|
|
525
|
+
// Container not found — return offline status
|
|
526
|
+
const now = new Date().toISOString();
|
|
527
|
+
return {
|
|
528
|
+
id: profile.id,
|
|
529
|
+
name: profile.name,
|
|
530
|
+
description: profile.description,
|
|
531
|
+
image: profile.image,
|
|
532
|
+
containerId: null,
|
|
533
|
+
state: "stopped",
|
|
534
|
+
health: null,
|
|
535
|
+
uptime: null,
|
|
536
|
+
startedAt: null,
|
|
537
|
+
createdAt: now,
|
|
538
|
+
updatedAt: now,
|
|
539
|
+
stats: null,
|
|
540
|
+
applicationMetrics: null,
|
|
541
|
+
};
|
|
673
542
|
}
|
|
674
|
-
|
|
675
|
-
const now = new Date().toISOString();
|
|
676
|
-
return {
|
|
677
|
-
id: profile.id,
|
|
678
|
-
name: profile.name,
|
|
679
|
-
description: profile.description,
|
|
680
|
-
image: profile.image,
|
|
681
|
-
containerId: info.Id,
|
|
682
|
-
state: info.State.Status as BotStatus["state"],
|
|
683
|
-
health: info.State.Health?.Status ?? null,
|
|
684
|
-
uptime: info.State.Running && info.State.StartedAt ? info.State.StartedAt : null,
|
|
685
|
-
startedAt: info.State.StartedAt || null,
|
|
686
|
-
createdAt: info.Created || now,
|
|
687
|
-
updatedAt: now,
|
|
688
|
-
stats,
|
|
689
|
-
applicationMetrics: this.botMetricsTracker?.getMetrics(profile.id) ?? null,
|
|
690
|
-
};
|
|
691
|
-
}
|
|
692
|
-
|
|
693
|
-
private offlineStatus(profile: BotProfile): BotStatus {
|
|
694
|
-
const now = new Date().toISOString();
|
|
695
|
-
return {
|
|
696
|
-
id: profile.id,
|
|
697
|
-
name: profile.name,
|
|
698
|
-
description: profile.description,
|
|
699
|
-
image: profile.image,
|
|
700
|
-
containerId: null,
|
|
701
|
-
state: "stopped",
|
|
702
|
-
health: null,
|
|
703
|
-
uptime: null,
|
|
704
|
-
startedAt: null,
|
|
705
|
-
createdAt: now,
|
|
706
|
-
updatedAt: now,
|
|
707
|
-
stats: null,
|
|
708
|
-
applicationMetrics: null,
|
|
709
|
-
};
|
|
710
|
-
}
|
|
711
|
-
|
|
712
|
-
private async getStats(container: Docker.Container): Promise<ContainerStats> {
|
|
713
|
-
const raw = await container.stats({ stream: false });
|
|
714
|
-
|
|
715
|
-
const cpuDelta = raw.cpu_stats.cpu_usage.total_usage - raw.precpu_stats.cpu_usage.total_usage;
|
|
716
|
-
const systemDelta = raw.cpu_stats.system_cpu_usage - raw.precpu_stats.system_cpu_usage;
|
|
717
|
-
const numCpus = raw.cpu_stats.online_cpus || 1;
|
|
718
|
-
const cpuPercent = systemDelta > 0 ? (cpuDelta / systemDelta) * numCpus * 100 : 0;
|
|
719
|
-
|
|
720
|
-
const memUsage = raw.memory_stats.usage || 0;
|
|
721
|
-
const memLimit = raw.memory_stats.limit || 1;
|
|
722
|
-
|
|
723
|
-
return {
|
|
724
|
-
cpuPercent: Math.round(cpuPercent * 100) / 100,
|
|
725
|
-
memoryUsageMb: Math.round(memUsage / 1024 / 1024),
|
|
726
|
-
memoryLimitMb: Math.round(memLimit / 1024 / 1024),
|
|
727
|
-
memoryPercent: Math.round((memUsage / memLimit) * 100 * 100) / 100,
|
|
728
|
-
};
|
|
729
543
|
}
|
|
730
544
|
}
|
|
731
545
|
|
|
@@ -735,22 +549,3 @@ export class BotNotFoundError extends Error {
|
|
|
735
549
|
this.name = "BotNotFoundError";
|
|
736
550
|
}
|
|
737
551
|
}
|
|
738
|
-
|
|
739
|
-
export class InvalidStateTransitionError extends Error {
|
|
740
|
-
readonly botId: string;
|
|
741
|
-
readonly operation: string;
|
|
742
|
-
readonly currentState: string;
|
|
743
|
-
readonly validStates: string[];
|
|
744
|
-
|
|
745
|
-
constructor(botId: string, operation: string, currentState: string, validStates: string[]) {
|
|
746
|
-
super(
|
|
747
|
-
`Cannot ${operation} bot ${botId}: container is in state "${currentState}". ` +
|
|
748
|
-
`Valid states for ${operation}: ${validStates.join(", ")}.`,
|
|
749
|
-
);
|
|
750
|
-
this.name = "InvalidStateTransitionError";
|
|
751
|
-
this.botId = botId;
|
|
752
|
-
this.operation = operation;
|
|
753
|
-
this.currentState = currentState;
|
|
754
|
-
this.validStates = validStates;
|
|
755
|
-
}
|
|
756
|
-
}
|