@wopr-network/platform-core 1.39.4 → 1.39.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/credits/credit-expiry-cron.test.js +30 -0
- package/dist/credits/ledger.js +11 -5
- package/dist/credits/ledger.test.js +87 -0
- package/dist/db/schema/ledger.js +6 -0
- package/dist/fleet/fleet-manager.d.ts +14 -35
- package/dist/fleet/fleet-manager.js +52 -236
- package/dist/fleet/fleet-manager.test.js +13 -85
- package/dist/fleet/instance.d.ts +58 -3
- package/dist/fleet/instance.js +297 -33
- package/dist/fleet/instance.test.d.ts +1 -0
- package/dist/fleet/instance.test.js +343 -0
- package/dist/node-agent/types.d.ts +1 -1
- package/package.json +1 -1
- package/src/credits/README.md +106 -0
- package/src/credits/credit-expiry-cron.test.ts +36 -0
- package/src/credits/ledger.test.ts +113 -0
- package/src/credits/ledger.ts +13 -7
- package/src/db/schema/ledger.ts +6 -0
- package/src/fleet/fleet-manager.test.ts +13 -111
- package/src/fleet/fleet-manager.ts +50 -255
- package/src/fleet/instance.test.ts +390 -0
- package/src/fleet/instance.ts +318 -33
|
@@ -81,6 +81,36 @@ describe("runCreditExpiryCron", () => {
|
|
|
81
81
|
const balanceAfterSecond = await ledger.balance("tenant-1");
|
|
82
82
|
expect(balanceAfterSecond.toCents()).toBe(balanceAfterFirst.toCents());
|
|
83
83
|
});
|
|
84
|
+
it("skips expiry when balance has been fully consumed before cron runs", async () => {
|
|
85
|
+
// Simulate: grant expires but tenant spent everything before cron ran
|
|
86
|
+
await ledger.credit("tenant-1", Credit.fromCents(500), "promo", {
|
|
87
|
+
description: "Promo",
|
|
88
|
+
referenceId: "promo:fully-consumed",
|
|
89
|
+
expiresAt: "2026-01-10T00:00:00Z",
|
|
90
|
+
});
|
|
91
|
+
// Tenant spends entire balance before expiry cron runs
|
|
92
|
+
await ledger.debit("tenant-1", Credit.fromCents(500), "bot_runtime", { description: "Full spend" });
|
|
93
|
+
const result = await runCreditExpiryCron({ ledger, now: "2026-01-15T00:00:00Z" });
|
|
94
|
+
// Zero balance — nothing to expire
|
|
95
|
+
expect(result.processed).toBe(0);
|
|
96
|
+
const balance = await ledger.balance("tenant-1");
|
|
97
|
+
expect(balance.toCents()).toBe(0);
|
|
98
|
+
});
|
|
99
|
+
it("only expires remaining balance when usage reduced it between grant and expiry", async () => {
|
|
100
|
+
// Grant $5, spend $3 before cron, cron should only expire remaining $2
|
|
101
|
+
await ledger.credit("tenant-1", Credit.fromCents(500), "promo", {
|
|
102
|
+
description: "Promo",
|
|
103
|
+
referenceId: "promo:partial-concurrent",
|
|
104
|
+
expiresAt: "2026-01-10T00:00:00Z",
|
|
105
|
+
});
|
|
106
|
+
await ledger.debit("tenant-1", Credit.fromCents(300), "bot_runtime", { description: "Partial spend" });
|
|
107
|
+
const result = await runCreditExpiryCron({ ledger, now: "2026-01-15T00:00:00Z" });
|
|
108
|
+
expect(result.processed).toBe(1);
|
|
109
|
+
expect(result.expired).toContain("tenant-1");
|
|
110
|
+
// $5 granted - $3 used - $2 expired = $0
|
|
111
|
+
const balance = await ledger.balance("tenant-1");
|
|
112
|
+
expect(balance.toCents()).toBe(0);
|
|
113
|
+
});
|
|
84
114
|
it("does not return unknown entry type even with expiresAt metadata", async () => {
|
|
85
115
|
// Simulate a hypothetical new entry type that has expiresAt in metadata.
|
|
86
116
|
// With the old denylist approach, this would be incorrectly returned.
|
package/dist/credits/ledger.js
CHANGED
|
@@ -169,19 +169,25 @@ export class DrizzleLedger {
|
|
|
169
169
|
throw new Error("Journal entry must have at least 2 lines");
|
|
170
170
|
}
|
|
171
171
|
// Verify balance before hitting DB
|
|
172
|
-
let totalDebit =
|
|
173
|
-
let totalCredit =
|
|
172
|
+
let totalDebit = 0n;
|
|
173
|
+
let totalCredit = 0n;
|
|
174
174
|
for (const line of input.lines) {
|
|
175
175
|
if (line.amount.isZero() || line.amount.isNegative()) {
|
|
176
176
|
throw new Error("Journal line amounts must be positive");
|
|
177
177
|
}
|
|
178
178
|
if (line.side === "debit")
|
|
179
|
-
totalDebit += line.amount.toRaw();
|
|
179
|
+
totalDebit += BigInt(line.amount.toRaw());
|
|
180
180
|
else
|
|
181
|
-
totalCredit += line.amount.toRaw();
|
|
181
|
+
totalCredit += BigInt(line.amount.toRaw());
|
|
182
182
|
}
|
|
183
183
|
if (totalDebit !== totalCredit) {
|
|
184
|
-
|
|
184
|
+
const fmtDebit = totalDebit <= BigInt(Number.MAX_SAFE_INTEGER)
|
|
185
|
+
? Credit.fromRaw(Number(totalDebit)).toDisplayString()
|
|
186
|
+
: `${totalDebit} raw`;
|
|
187
|
+
const fmtCredit = totalCredit <= BigInt(Number.MAX_SAFE_INTEGER)
|
|
188
|
+
? Credit.fromRaw(Number(totalCredit)).toDisplayString()
|
|
189
|
+
: `${totalCredit} raw`;
|
|
190
|
+
throw new Error(`Unbalanced entry: debits=${fmtDebit}, credits=${fmtCredit}`);
|
|
185
191
|
}
|
|
186
192
|
return this.db.transaction(async (tx) => {
|
|
187
193
|
const entryId = crypto.randomUUID();
|
|
@@ -38,6 +38,21 @@ describe("DrizzleLedger", () => {
|
|
|
38
38
|
],
|
|
39
39
|
})).rejects.toThrow("Unbalanced");
|
|
40
40
|
});
|
|
41
|
+
it("throws Unbalanced entry (not RangeError) when BigInt totals exceed MAX_SAFE_INTEGER", async () => {
|
|
42
|
+
// Two debit lines each at MAX_SAFE_INTEGER raw — their BigInt sum exceeds MAX_SAFE_INTEGER.
|
|
43
|
+
// Before the fix, Credit.fromRaw(Number(totalDebit)) would throw RangeError, masking the real error.
|
|
44
|
+
const big = Credit.fromRaw(Number.MAX_SAFE_INTEGER);
|
|
45
|
+
const small = Credit.fromRaw(1);
|
|
46
|
+
await expect(ledger.post({
|
|
47
|
+
entryType: "purchase",
|
|
48
|
+
tenantId: "t1",
|
|
49
|
+
lines: [
|
|
50
|
+
{ accountCode: "1000", amount: big, side: "debit" },
|
|
51
|
+
{ accountCode: "1000", amount: big, side: "debit" },
|
|
52
|
+
{ accountCode: "2000:t1", amount: small, side: "credit" },
|
|
53
|
+
],
|
|
54
|
+
})).rejects.toThrow("Unbalanced entry");
|
|
55
|
+
});
|
|
41
56
|
it("rejects zero-amount lines", async () => {
|
|
42
57
|
await expect(ledger.post({
|
|
43
58
|
entryType: "purchase",
|
|
@@ -209,6 +224,22 @@ describe("DrizzleLedger", () => {
|
|
|
209
224
|
it("rejects zero amount", async () => {
|
|
210
225
|
await expect(ledger.debit("t1", Credit.ZERO, "bot_runtime")).rejects.toThrow("must be positive");
|
|
211
226
|
});
|
|
227
|
+
it("concurrent debits do not overdraft", async () => {
|
|
228
|
+
// Balance is $10.00 from beforeEach credit.
|
|
229
|
+
// Two concurrent $8 debits — only one should succeed.
|
|
230
|
+
const results = await Promise.allSettled([
|
|
231
|
+
ledger.debit("t1", Credit.fromCents(800), "bot_runtime"),
|
|
232
|
+
ledger.debit("t1", Credit.fromCents(800), "bot_runtime"),
|
|
233
|
+
]);
|
|
234
|
+
const successes = results.filter((r) => r.status === "fulfilled");
|
|
235
|
+
const failures = results.filter((r) => r.status === "rejected");
|
|
236
|
+
expect(successes).toHaveLength(1);
|
|
237
|
+
expect(failures).toHaveLength(1);
|
|
238
|
+
expect(failures[0].reason).toBeInstanceOf(InsufficientBalanceError);
|
|
239
|
+
// Balance should be $2.00, not -$6.00
|
|
240
|
+
const bal = await ledger.balance("t1");
|
|
241
|
+
expect(bal.toCentsRounded()).toBe(200);
|
|
242
|
+
});
|
|
212
243
|
});
|
|
213
244
|
// -----------------------------------------------------------------------
|
|
214
245
|
// balance()
|
|
@@ -268,6 +299,20 @@ describe("DrizzleLedger", () => {
|
|
|
268
299
|
expect(tb.balanced).toBe(true);
|
|
269
300
|
expect(tb.totalDebits.equals(tb.totalCredits)).toBe(true);
|
|
270
301
|
});
|
|
302
|
+
it("detects imbalance from direct DB corruption", async () => {
|
|
303
|
+
await ledger.credit("t1", Credit.fromCents(100), "purchase");
|
|
304
|
+
// Corrupt the ledger: insert an unmatched debit line directly into journal_lines.
|
|
305
|
+
const entryRows = await pool.query("SELECT id FROM journal_entries LIMIT 1");
|
|
306
|
+
const accountRows = await pool.query("SELECT id FROM accounts WHERE code = '1000' LIMIT 1");
|
|
307
|
+
const entryId = entryRows.rows[0].id;
|
|
308
|
+
const accountId = accountRows.rows[0].id;
|
|
309
|
+
// Insert an unmatched debit line worth 999 raw units — no corresponding credit
|
|
310
|
+
await pool.query(`INSERT INTO journal_lines (id, journal_entry_id, account_id, amount, side)
|
|
311
|
+
VALUES ('corrupt-line-1', $1, $2, 999, 'debit')`, [entryId, accountId]);
|
|
312
|
+
const tb = await ledger.trialBalance();
|
|
313
|
+
expect(tb.balanced).toBe(false);
|
|
314
|
+
expect(tb.difference.toRaw()).toBe(999);
|
|
315
|
+
});
|
|
271
316
|
});
|
|
272
317
|
// -----------------------------------------------------------------------
|
|
273
318
|
// history()
|
|
@@ -491,4 +536,46 @@ describe("DrizzleLedger", () => {
|
|
|
491
536
|
expect(tb.balanced).toBe(true);
|
|
492
537
|
});
|
|
493
538
|
});
|
|
539
|
+
// -----------------------------------------------------------------------
|
|
540
|
+
// deadlock prevention — concurrent multi-line entries
|
|
541
|
+
// -----------------------------------------------------------------------
|
|
542
|
+
describe("deadlock prevention", () => {
|
|
543
|
+
it("concurrent multi-line entries with overlapping accounts succeed", async () => {
|
|
544
|
+
// Fund two tenants
|
|
545
|
+
await ledger.credit("t1", Credit.fromCents(1000), "purchase");
|
|
546
|
+
await ledger.credit("t2", Credit.fromCents(1000), "purchase");
|
|
547
|
+
// Two entries that touch accounts in potentially reverse order.
|
|
548
|
+
// Both touch account 4000 (revenue), creating a potential lock conflict.
|
|
549
|
+
const results = await Promise.allSettled([
|
|
550
|
+
ledger.debit("t1", Credit.fromCents(50), "bot_runtime"),
|
|
551
|
+
ledger.debit("t2", Credit.fromCents(30), "bot_runtime"),
|
|
552
|
+
]);
|
|
553
|
+
// Both should succeed — lock ordering prevents deadlock
|
|
554
|
+
expect(results.every((r) => r.status === "fulfilled")).toBe(true);
|
|
555
|
+
// Verify balances are correct
|
|
556
|
+
expect((await ledger.balance("t1")).toCentsRounded()).toBe(950);
|
|
557
|
+
expect((await ledger.balance("t2")).toCentsRounded()).toBe(970);
|
|
558
|
+
// Revenue account should reflect both debits
|
|
559
|
+
expect((await ledger.accountBalance("4000")).toCentsRounded()).toBe(80);
|
|
560
|
+
// Trial balance must still be balanced
|
|
561
|
+
const tb = await ledger.trialBalance();
|
|
562
|
+
expect(tb.balanced).toBe(true);
|
|
563
|
+
});
|
|
564
|
+
it("concurrent multi-line entries on same tenant serialize correctly", async () => {
|
|
565
|
+
await ledger.credit("t1", Credit.fromCents(1000), "purchase");
|
|
566
|
+
// Two debits on the same tenant, touching overlapping accounts.
|
|
567
|
+
const results = await Promise.allSettled([
|
|
568
|
+
ledger.debit("t1", Credit.fromCents(100), "bot_runtime"),
|
|
569
|
+
ledger.debit("t1", Credit.fromCents(200), "adapter_usage"),
|
|
570
|
+
]);
|
|
571
|
+
expect(results.every((r) => r.status === "fulfilled")).toBe(true);
|
|
572
|
+
// Balance: $10 - $1 - $2 = $7
|
|
573
|
+
expect((await ledger.balance("t1")).toCentsRounded()).toBe(700);
|
|
574
|
+
// Revenue accounts
|
|
575
|
+
expect((await ledger.accountBalance("4000")).toCentsRounded()).toBe(100); // bot_runtime
|
|
576
|
+
expect((await ledger.accountBalance("4010")).toCentsRounded()).toBe(200); // adapter_usage
|
|
577
|
+
const tb = await ledger.trialBalance();
|
|
578
|
+
expect(tb.balanced).toBe(true);
|
|
579
|
+
});
|
|
580
|
+
});
|
|
494
581
|
});
|
package/dist/db/schema/ledger.js
CHANGED
|
@@ -55,6 +55,9 @@ export const journalLines = pgTable("journal_lines", {
|
|
|
55
55
|
accountId: text("account_id")
|
|
56
56
|
.notNull()
|
|
57
57
|
.references(() => accounts.id),
|
|
58
|
+
// mode: "number" truncates values > Number.MAX_SAFE_INTEGER (≈$9.007M in nanodollars).
|
|
59
|
+
// Credit.fromRaw() guards against this with a RangeError.
|
|
60
|
+
// If balances approach $9M, migrate to mode: "bigint" (returns string from Drizzle).
|
|
58
61
|
amount: bigint("amount", { mode: "number" }).notNull(), // nanodollars, always positive
|
|
59
62
|
side: entrySideEnum("side").notNull(),
|
|
60
63
|
}, (table) => [
|
|
@@ -71,6 +74,9 @@ export const accountBalances = pgTable("account_balances", {
|
|
|
71
74
|
accountId: text("account_id")
|
|
72
75
|
.primaryKey()
|
|
73
76
|
.references(() => accounts.id),
|
|
77
|
+
// mode: "number" truncates values > Number.MAX_SAFE_INTEGER (≈$9.007M in nanodollars).
|
|
78
|
+
// Credit.fromRaw() guards against this with a RangeError.
|
|
79
|
+
// If balances approach $9M, migrate to mode: "bigint" (returns string from Drizzle).
|
|
74
80
|
balance: bigint("balance", { mode: "number" }).notNull().default(0), // net balance in nanodollars
|
|
75
81
|
lastUpdated: text("last_updated").notNull().default(sql `(now())`),
|
|
76
82
|
});
|
|
@@ -53,20 +53,17 @@ export declare class FleetManager {
|
|
|
53
53
|
*/
|
|
54
54
|
getInstance(id: string): Promise<Instance>;
|
|
55
55
|
private buildInstance;
|
|
56
|
-
/**
|
|
57
|
-
* Restart: pull new image BEFORE restarting container to avoid downtime on pull failure.
|
|
58
|
-
* Valid from: running, stopped, exited, dead states.
|
|
59
|
-
* Throws InvalidStateTransitionError if the container is in an invalid state (e.g. paused).
|
|
60
|
-
* For remote bots, delegates to the node agent via NodeCommandBus.
|
|
61
|
-
*/
|
|
62
|
-
restart(id: string): Promise<void>;
|
|
63
56
|
/**
|
|
64
57
|
* Remove a bot: stop container, remove it, optionally remove volumes, delete profile.
|
|
65
58
|
* For remote bots, delegates stop+remove to the node agent via NodeCommandBus.
|
|
59
|
+
* Container removal is delegated to Instance.remove(); fleet-level cleanup
|
|
60
|
+
* (profile store, network policy) stays here.
|
|
66
61
|
*/
|
|
67
62
|
remove(id: string, removeVolumes?: boolean): Promise<void>;
|
|
68
63
|
/**
|
|
69
64
|
* Get live status of a single bot.
|
|
65
|
+
* Delegates to Instance.status() which returns a full BotStatus.
|
|
66
|
+
* Falls back to offline status when no container exists.
|
|
70
67
|
*/
|
|
71
68
|
status(id: string): Promise<BotStatus>;
|
|
72
69
|
/**
|
|
@@ -78,58 +75,40 @@ export declare class FleetManager {
|
|
|
78
75
|
*/
|
|
79
76
|
listByTenant(tenantId: string): Promise<BotStatus[]>;
|
|
80
77
|
/**
|
|
81
|
-
* Get container logs.
|
|
78
|
+
* Get container logs. Delegates to Instance.logs().
|
|
82
79
|
*/
|
|
83
80
|
logs(id: string, tail?: number): Promise<string>;
|
|
84
81
|
/**
|
|
85
82
|
* Stream container logs in real-time (follow mode).
|
|
86
|
-
* Returns a Node.js ReadableStream that emits plain-text log chunks (already demultiplexed).
|
|
87
83
|
* For remote bots, proxies via node-agent bot.logs command and returns a one-shot stream.
|
|
88
|
-
*
|
|
84
|
+
* For local bots, delegates to Instance.logStream().
|
|
89
85
|
*/
|
|
90
86
|
logStream(id: string, opts: {
|
|
91
87
|
since?: string;
|
|
92
88
|
tail?: number;
|
|
93
89
|
}): Promise<NodeJS.ReadableStream>;
|
|
94
|
-
/** Fields that require container recreation when changed. */
|
|
95
|
-
private static readonly CONTAINER_FIELDS;
|
|
96
|
-
/**
|
|
97
|
-
* Update a bot profile. Only recreates the container if container-relevant
|
|
98
|
-
* fields changed. Rolls back the profile if container recreation fails.
|
|
99
|
-
*/
|
|
100
|
-
update(id: string, updates: Partial<Omit<BotProfile, "id">>): Promise<BotProfile>;
|
|
101
90
|
/**
|
|
102
|
-
* Get disk usage for a bot's /data volume.
|
|
103
|
-
* Returns null if the container is not running or exec fails.
|
|
91
|
+
* Get disk usage for a bot's /data volume. Delegates to Instance.getVolumeUsage().
|
|
104
92
|
*/
|
|
105
93
|
getVolumeUsage(id: string): Promise<{
|
|
106
94
|
usedBytes: number;
|
|
107
95
|
totalBytes: number;
|
|
108
96
|
availableBytes: number;
|
|
109
97
|
} | null>;
|
|
110
|
-
/**
|
|
111
|
-
|
|
98
|
+
/** Fields that require container recreation when changed. */
|
|
99
|
+
private static readonly CONTAINER_FIELDS;
|
|
112
100
|
/**
|
|
113
|
-
*
|
|
114
|
-
*
|
|
115
|
-
* Throws InvalidStateTransitionError when the state is not in validStates.
|
|
101
|
+
* Update a bot profile. Only recreates the container if container-relevant
|
|
102
|
+
* fields changed. Rolls back the profile if container recreation fails.
|
|
116
103
|
*/
|
|
117
|
-
|
|
104
|
+
update(id: string, updates: Partial<Omit<BotProfile, "id">>): Promise<BotProfile>;
|
|
105
|
+
/** Get the underlying profile store */
|
|
106
|
+
get profiles(): IProfileStore;
|
|
118
107
|
private pullImage;
|
|
119
108
|
private createContainer;
|
|
120
109
|
private findContainer;
|
|
121
110
|
private statusForProfile;
|
|
122
|
-
private buildStatus;
|
|
123
|
-
private offlineStatus;
|
|
124
|
-
private getStats;
|
|
125
111
|
}
|
|
126
112
|
export declare class BotNotFoundError extends Error {
|
|
127
113
|
constructor(id: string);
|
|
128
114
|
}
|
|
129
|
-
export declare class InvalidStateTransitionError extends Error {
|
|
130
|
-
readonly botId: string;
|
|
131
|
-
readonly operation: string;
|
|
132
|
-
readonly currentState: string;
|
|
133
|
-
readonly validStates: string[];
|
|
134
|
-
constructor(botId: string, operation: string, currentState: string, validStates: string[]);
|
|
135
|
-
}
|
|
@@ -106,6 +106,7 @@ export class FleetManager {
|
|
|
106
106
|
instanceRepo: this.instanceRepo,
|
|
107
107
|
proxyManager: this.proxyManager,
|
|
108
108
|
eventEmitter: this.eventEmitter,
|
|
109
|
+
botMetricsTracker: this.botMetricsTracker,
|
|
109
110
|
});
|
|
110
111
|
remoteInstance.emitCreated();
|
|
111
112
|
return remoteInstance;
|
|
@@ -173,45 +174,14 @@ export class FleetManager {
|
|
|
173
174
|
instanceRepo: this.instanceRepo,
|
|
174
175
|
proxyManager: this.proxyManager,
|
|
175
176
|
eventEmitter: this.eventEmitter,
|
|
176
|
-
|
|
177
|
-
}
|
|
178
|
-
/**
|
|
179
|
-
* Restart: pull new image BEFORE restarting container to avoid downtime on pull failure.
|
|
180
|
-
* Valid from: running, stopped, exited, dead states.
|
|
181
|
-
* Throws InvalidStateTransitionError if the container is in an invalid state (e.g. paused).
|
|
182
|
-
* For remote bots, delegates to the node agent via NodeCommandBus.
|
|
183
|
-
*/
|
|
184
|
-
async restart(id) {
|
|
185
|
-
return this.withLock(id, async () => {
|
|
186
|
-
this.botMetricsTracker?.reset(id);
|
|
187
|
-
const profile = await this.store.get(id);
|
|
188
|
-
if (!profile)
|
|
189
|
-
throw new BotNotFoundError(id);
|
|
190
|
-
const remote = await this.resolveNodeId(id);
|
|
191
|
-
if (remote) {
|
|
192
|
-
await remote.commandBus.send(remote.nodeId, {
|
|
193
|
-
type: "bot.restart",
|
|
194
|
-
payload: { name: profile.name },
|
|
195
|
-
});
|
|
196
|
-
}
|
|
197
|
-
else {
|
|
198
|
-
// Pull new image first — if this fails, old container is unchanged
|
|
199
|
-
await this.pullImage(profile.image);
|
|
200
|
-
const container = await this.findContainer(id);
|
|
201
|
-
if (!container)
|
|
202
|
-
throw new BotNotFoundError(id);
|
|
203
|
-
const info = await container.inspect();
|
|
204
|
-
const validRestartStates = new Set(["running", "stopped", "exited", "dead"]);
|
|
205
|
-
this.assertValidState(id, info.State.Status, "restart", validRestartStates);
|
|
206
|
-
await container.restart();
|
|
207
|
-
}
|
|
208
|
-
logger.info(`Restarted bot ${id}`);
|
|
209
|
-
this.emitEvent("bot.restarted", id, profile.tenantId);
|
|
177
|
+
botMetricsTracker: this.botMetricsTracker,
|
|
210
178
|
});
|
|
211
179
|
}
|
|
212
180
|
/**
|
|
213
181
|
* Remove a bot: stop container, remove it, optionally remove volumes, delete profile.
|
|
214
182
|
* For remote bots, delegates stop+remove to the node agent via NodeCommandBus.
|
|
183
|
+
* Container removal is delegated to Instance.remove(); fleet-level cleanup
|
|
184
|
+
* (profile store, network policy) stays here.
|
|
215
185
|
*/
|
|
216
186
|
async remove(id, removeVolumes = false) {
|
|
217
187
|
return this.withLock(id, async () => {
|
|
@@ -226,13 +196,12 @@ export class FleetManager {
|
|
|
226
196
|
});
|
|
227
197
|
}
|
|
228
198
|
else {
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
await container.remove({ v: removeVolumes });
|
|
199
|
+
try {
|
|
200
|
+
const instance = await this.buildInstance(profile);
|
|
201
|
+
await instance.remove(removeVolumes);
|
|
202
|
+
}
|
|
203
|
+
catch {
|
|
204
|
+
// Container may already be gone — not fatal for fleet-level cleanup
|
|
236
205
|
}
|
|
237
206
|
}
|
|
238
207
|
// Clean up tenant network if no more containers remain
|
|
@@ -240,25 +209,20 @@ export class FleetManager {
|
|
|
240
209
|
await this.networkPolicy.cleanupAfterRemoval(profile.tenantId);
|
|
241
210
|
}
|
|
242
211
|
await this.store.delete(id);
|
|
243
|
-
if (this.proxyManager) {
|
|
244
|
-
this.proxyManager.removeRoute(id);
|
|
245
|
-
}
|
|
246
212
|
logger.info(`Removed bot ${id}`);
|
|
247
213
|
this.emitEvent("bot.removed", id, profile.tenantId);
|
|
248
214
|
});
|
|
249
215
|
}
|
|
250
216
|
/**
|
|
251
217
|
* Get live status of a single bot.
|
|
218
|
+
* Delegates to Instance.status() which returns a full BotStatus.
|
|
219
|
+
* Falls back to offline status when no container exists.
|
|
252
220
|
*/
|
|
253
221
|
async status(id) {
|
|
254
222
|
const profile = await this.store.get(id);
|
|
255
223
|
if (!profile)
|
|
256
224
|
throw new BotNotFoundError(id);
|
|
257
|
-
|
|
258
|
-
if (!container) {
|
|
259
|
-
return this.offlineStatus(profile);
|
|
260
|
-
}
|
|
261
|
-
return this.buildStatus(profile, container);
|
|
225
|
+
return this.statusForProfile(profile);
|
|
262
226
|
}
|
|
263
227
|
/**
|
|
264
228
|
* List all bots with live status.
|
|
@@ -276,40 +240,16 @@ export class FleetManager {
|
|
|
276
240
|
return Promise.all(tenantProfiles.map((p) => this.statusForProfile(p)));
|
|
277
241
|
}
|
|
278
242
|
/**
|
|
279
|
-
* Get container logs.
|
|
243
|
+
* Get container logs. Delegates to Instance.logs().
|
|
280
244
|
*/
|
|
281
245
|
async logs(id, tail = 100) {
|
|
282
|
-
const
|
|
283
|
-
|
|
284
|
-
throw new BotNotFoundError(id);
|
|
285
|
-
const logBuffer = await container.logs({
|
|
286
|
-
stdout: true,
|
|
287
|
-
stderr: true,
|
|
288
|
-
tail,
|
|
289
|
-
timestamps: true,
|
|
290
|
-
});
|
|
291
|
-
// Docker returns multiplexed binary frames when Tty is false (the default).
|
|
292
|
-
// Demultiplex by stripping the 8-byte header from each frame so callers
|
|
293
|
-
// receive plain text instead of binary garbage interleaved with log lines.
|
|
294
|
-
const buf = Buffer.isBuffer(logBuffer) ? logBuffer : Buffer.from(logBuffer, "binary");
|
|
295
|
-
const chunks = [];
|
|
296
|
-
let offset = 0;
|
|
297
|
-
while (offset + 8 <= buf.length) {
|
|
298
|
-
const frameSize = buf.readUInt32BE(offset + 4);
|
|
299
|
-
const end = offset + 8 + frameSize;
|
|
300
|
-
if (end > buf.length)
|
|
301
|
-
break;
|
|
302
|
-
chunks.push(buf.subarray(offset + 8, end));
|
|
303
|
-
offset = end;
|
|
304
|
-
}
|
|
305
|
-
// If demux produced nothing (e.g. TTY container), fall back to raw string
|
|
306
|
-
return chunks.length > 0 ? Buffer.concat(chunks).toString("utf-8") : buf.toString("utf-8");
|
|
246
|
+
const instance = await this.getInstance(id);
|
|
247
|
+
return instance.logs(tail);
|
|
307
248
|
}
|
|
308
249
|
/**
|
|
309
250
|
* Stream container logs in real-time (follow mode).
|
|
310
|
-
* Returns a Node.js ReadableStream that emits plain-text log chunks (already demultiplexed).
|
|
311
251
|
* For remote bots, proxies via node-agent bot.logs command and returns a one-shot stream.
|
|
312
|
-
*
|
|
252
|
+
* For local bots, delegates to Instance.logStream().
|
|
313
253
|
*/
|
|
314
254
|
async logStream(id, opts) {
|
|
315
255
|
// Check for remote node assignment first (mirrors start/stop/restart pattern)
|
|
@@ -327,26 +267,20 @@ export class FleetManager {
|
|
|
327
267
|
pt.end(logData);
|
|
328
268
|
return pt;
|
|
329
269
|
}
|
|
330
|
-
const
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
270
|
+
const instance = await this.getInstance(id);
|
|
271
|
+
return instance.logStream(opts);
|
|
272
|
+
}
|
|
273
|
+
/**
|
|
274
|
+
* Get disk usage for a bot's /data volume. Delegates to Instance.getVolumeUsage().
|
|
275
|
+
*/
|
|
276
|
+
async getVolumeUsage(id) {
|
|
277
|
+
try {
|
|
278
|
+
const instance = await this.getInstance(id);
|
|
279
|
+
return instance.getVolumeUsage();
|
|
280
|
+
}
|
|
281
|
+
catch {
|
|
282
|
+
return null;
|
|
342
283
|
}
|
|
343
|
-
// Docker returns a multiplexed binary stream when Tty is false (the default for
|
|
344
|
-
// containers created by createContainer without Tty:true). Demultiplex it so
|
|
345
|
-
// callers receive plain text without 8-byte binary frame headers.
|
|
346
|
-
const multiplexed = (await container.logs(logOpts));
|
|
347
|
-
const pt = new PassThrough();
|
|
348
|
-
this.docker.modem.demuxStream(multiplexed, pt, pt);
|
|
349
|
-
return pt;
|
|
350
284
|
}
|
|
351
285
|
/** Fields that require container recreation when changed. */
|
|
352
286
|
static CONTAINER_FIELDS = new Set([
|
|
@@ -417,72 +351,11 @@ export class FleetManager {
|
|
|
417
351
|
return updated;
|
|
418
352
|
});
|
|
419
353
|
}
|
|
420
|
-
/**
|
|
421
|
-
* Get disk usage for a bot's /data volume.
|
|
422
|
-
* Returns null if the container is not running or exec fails.
|
|
423
|
-
*/
|
|
424
|
-
async getVolumeUsage(id) {
|
|
425
|
-
const container = await this.findContainer(id);
|
|
426
|
-
if (!container)
|
|
427
|
-
return null;
|
|
428
|
-
try {
|
|
429
|
-
const info = await container.inspect();
|
|
430
|
-
if (!info.State.Running)
|
|
431
|
-
return null;
|
|
432
|
-
const exec = await container.exec({
|
|
433
|
-
Cmd: ["df", "-B1", "/data"],
|
|
434
|
-
AttachStdout: true,
|
|
435
|
-
AttachStderr: false,
|
|
436
|
-
});
|
|
437
|
-
const output = await new Promise((resolve, reject) => {
|
|
438
|
-
exec.start({}, (err, stream) => {
|
|
439
|
-
if (err)
|
|
440
|
-
return reject(err);
|
|
441
|
-
if (!stream)
|
|
442
|
-
return reject(new Error("No stream from exec"));
|
|
443
|
-
let data = "";
|
|
444
|
-
stream.on("data", (chunk) => {
|
|
445
|
-
data += chunk.toString();
|
|
446
|
-
});
|
|
447
|
-
stream.on("end", () => resolve(data));
|
|
448
|
-
stream.on("error", reject);
|
|
449
|
-
});
|
|
450
|
-
});
|
|
451
|
-
// Parse df output — second line has the numbers
|
|
452
|
-
const lines = output.trim().split("\n");
|
|
453
|
-
if (lines.length < 2)
|
|
454
|
-
return null;
|
|
455
|
-
const parts = lines[lines.length - 1].split(/\s+/);
|
|
456
|
-
if (parts.length < 4)
|
|
457
|
-
return null;
|
|
458
|
-
const totalBytes = parseInt(parts[1], 10);
|
|
459
|
-
const usedBytes = parseInt(parts[2], 10);
|
|
460
|
-
const availableBytes = parseInt(parts[3], 10);
|
|
461
|
-
if (Number.isNaN(totalBytes) || Number.isNaN(usedBytes) || Number.isNaN(availableBytes))
|
|
462
|
-
return null;
|
|
463
|
-
return { usedBytes, totalBytes, availableBytes };
|
|
464
|
-
}
|
|
465
|
-
catch {
|
|
466
|
-
logger.warn(`Failed to get volume usage for bot ${id}`);
|
|
467
|
-
return null;
|
|
468
|
-
}
|
|
469
|
-
}
|
|
470
354
|
/** Get the underlying profile store */
|
|
471
355
|
get profiles() {
|
|
472
356
|
return this.store;
|
|
473
357
|
}
|
|
474
358
|
// --- Private helpers ---
|
|
475
|
-
/**
|
|
476
|
-
* Assert that a container's current state is valid for the requested operation.
|
|
477
|
-
* Guards against undefined/null Status values from Docker (uses "unknown" as fallback).
|
|
478
|
-
* Throws InvalidStateTransitionError when the state is not in validStates.
|
|
479
|
-
*/
|
|
480
|
-
assertValidState(id, rawStatus, operation, validStates) {
|
|
481
|
-
const currentState = typeof rawStatus === "string" && rawStatus ? rawStatus : "unknown";
|
|
482
|
-
if (!validStates.has(currentState)) {
|
|
483
|
-
throw new InvalidStateTransitionError(id, operation, currentState, [...validStates]);
|
|
484
|
-
}
|
|
485
|
-
}
|
|
486
359
|
async pullImage(image) {
|
|
487
360
|
logger.info(`Pulling image ${image}`);
|
|
488
361
|
// Build authconfig from environment variables if present.
|
|
@@ -589,71 +462,29 @@ export class FleetManager {
|
|
|
589
462
|
return this.docker.getContainer(containers[0].Id);
|
|
590
463
|
}
|
|
591
464
|
async statusForProfile(profile) {
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
return
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
|
|
465
|
+
try {
|
|
466
|
+
const instance = await this.buildInstance(profile);
|
|
467
|
+
return instance.status();
|
|
468
|
+
}
|
|
469
|
+
catch {
|
|
470
|
+
// Container not found — return offline status
|
|
471
|
+
const now = new Date().toISOString();
|
|
472
|
+
return {
|
|
473
|
+
id: profile.id,
|
|
474
|
+
name: profile.name,
|
|
475
|
+
description: profile.description,
|
|
476
|
+
image: profile.image,
|
|
477
|
+
containerId: null,
|
|
478
|
+
state: "stopped",
|
|
479
|
+
health: null,
|
|
480
|
+
uptime: null,
|
|
481
|
+
startedAt: null,
|
|
482
|
+
createdAt: now,
|
|
483
|
+
updatedAt: now,
|
|
484
|
+
stats: null,
|
|
485
|
+
applicationMetrics: null,
|
|
486
|
+
};
|
|
607
487
|
}
|
|
608
|
-
const now = new Date().toISOString();
|
|
609
|
-
return {
|
|
610
|
-
id: profile.id,
|
|
611
|
-
name: profile.name,
|
|
612
|
-
description: profile.description,
|
|
613
|
-
image: profile.image,
|
|
614
|
-
containerId: info.Id,
|
|
615
|
-
state: info.State.Status,
|
|
616
|
-
health: info.State.Health?.Status ?? null,
|
|
617
|
-
uptime: info.State.Running && info.State.StartedAt ? info.State.StartedAt : null,
|
|
618
|
-
startedAt: info.State.StartedAt || null,
|
|
619
|
-
createdAt: info.Created || now,
|
|
620
|
-
updatedAt: now,
|
|
621
|
-
stats,
|
|
622
|
-
applicationMetrics: this.botMetricsTracker?.getMetrics(profile.id) ?? null,
|
|
623
|
-
};
|
|
624
|
-
}
|
|
625
|
-
offlineStatus(profile) {
|
|
626
|
-
const now = new Date().toISOString();
|
|
627
|
-
return {
|
|
628
|
-
id: profile.id,
|
|
629
|
-
name: profile.name,
|
|
630
|
-
description: profile.description,
|
|
631
|
-
image: profile.image,
|
|
632
|
-
containerId: null,
|
|
633
|
-
state: "stopped",
|
|
634
|
-
health: null,
|
|
635
|
-
uptime: null,
|
|
636
|
-
startedAt: null,
|
|
637
|
-
createdAt: now,
|
|
638
|
-
updatedAt: now,
|
|
639
|
-
stats: null,
|
|
640
|
-
applicationMetrics: null,
|
|
641
|
-
};
|
|
642
|
-
}
|
|
643
|
-
async getStats(container) {
|
|
644
|
-
const raw = await container.stats({ stream: false });
|
|
645
|
-
const cpuDelta = raw.cpu_stats.cpu_usage.total_usage - raw.precpu_stats.cpu_usage.total_usage;
|
|
646
|
-
const systemDelta = raw.cpu_stats.system_cpu_usage - raw.precpu_stats.system_cpu_usage;
|
|
647
|
-
const numCpus = raw.cpu_stats.online_cpus || 1;
|
|
648
|
-
const cpuPercent = systemDelta > 0 ? (cpuDelta / systemDelta) * numCpus * 100 : 0;
|
|
649
|
-
const memUsage = raw.memory_stats.usage || 0;
|
|
650
|
-
const memLimit = raw.memory_stats.limit || 1;
|
|
651
|
-
return {
|
|
652
|
-
cpuPercent: Math.round(cpuPercent * 100) / 100,
|
|
653
|
-
memoryUsageMb: Math.round(memUsage / 1024 / 1024),
|
|
654
|
-
memoryLimitMb: Math.round(memLimit / 1024 / 1024),
|
|
655
|
-
memoryPercent: Math.round((memUsage / memLimit) * 100 * 100) / 100,
|
|
656
|
-
};
|
|
657
488
|
}
|
|
658
489
|
}
|
|
659
490
|
export class BotNotFoundError extends Error {
|
|
@@ -662,18 +493,3 @@ export class BotNotFoundError extends Error {
|
|
|
662
493
|
this.name = "BotNotFoundError";
|
|
663
494
|
}
|
|
664
495
|
}
|
|
665
|
-
export class InvalidStateTransitionError extends Error {
|
|
666
|
-
botId;
|
|
667
|
-
operation;
|
|
668
|
-
currentState;
|
|
669
|
-
validStates;
|
|
670
|
-
constructor(botId, operation, currentState, validStates) {
|
|
671
|
-
super(`Cannot ${operation} bot ${botId}: container is in state "${currentState}". ` +
|
|
672
|
-
`Valid states for ${operation}: ${validStates.join(", ")}.`);
|
|
673
|
-
this.name = "InvalidStateTransitionError";
|
|
674
|
-
this.botId = botId;
|
|
675
|
-
this.operation = operation;
|
|
676
|
-
this.currentState = currentState;
|
|
677
|
-
this.validStates = validStates;
|
|
678
|
-
}
|
|
679
|
-
}
|