@wopr-network/platform-core 1.69.0 → 1.70.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/db/schema/pool-config.d.ts +41 -0
- package/dist/db/schema/pool-config.js +5 -0
- package/dist/db/schema/pool-instances.d.ts +126 -0
- package/dist/db/schema/pool-instances.js +10 -0
- package/dist/server/__tests__/container.test.js +4 -1
- package/dist/server/container.d.ts +18 -2
- package/dist/server/container.js +18 -4
- package/dist/server/lifecycle.js +10 -0
- package/dist/server/routes/admin.d.ts +18 -0
- package/dist/server/routes/admin.js +21 -0
- package/dist/server/services/hot-pool-claim.d.ts +30 -0
- package/dist/server/services/hot-pool-claim.js +92 -0
- package/dist/server/services/hot-pool.d.ts +25 -0
- package/dist/server/services/hot-pool.js +129 -0
- package/dist/server/services/pool-repository.d.ts +44 -0
- package/dist/server/services/pool-repository.js +72 -0
- package/drizzle/migrations/0025_hot_pool_tables.sql +29 -0
- package/package.json +1 -1
- package/src/db/schema/pool-config.ts +6 -0
- package/src/db/schema/pool-instances.ts +11 -0
- package/src/server/__tests__/container.test.ts +4 -1
- package/src/server/container.ts +34 -7
- package/src/server/lifecycle.ts +10 -0
- package/src/server/routes/admin.ts +26 -0
- package/src/server/services/hot-pool-claim.ts +130 -0
- package/src/server/services/hot-pool.ts +174 -0
- package/src/server/services/pool-repository.ts +107 -0
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
export declare const poolConfig: import("drizzle-orm/pg-core").PgTableWithColumns<{
|
|
2
|
+
name: "pool_config";
|
|
3
|
+
schema: undefined;
|
|
4
|
+
columns: {
|
|
5
|
+
id: import("drizzle-orm/pg-core").PgColumn<{
|
|
6
|
+
name: "id";
|
|
7
|
+
tableName: "pool_config";
|
|
8
|
+
dataType: "number";
|
|
9
|
+
columnType: "PgInteger";
|
|
10
|
+
data: number;
|
|
11
|
+
driverParam: string | number;
|
|
12
|
+
notNull: true;
|
|
13
|
+
hasDefault: true;
|
|
14
|
+
isPrimaryKey: true;
|
|
15
|
+
isAutoincrement: false;
|
|
16
|
+
hasRuntimeDefault: false;
|
|
17
|
+
enumValues: undefined;
|
|
18
|
+
baseColumn: never;
|
|
19
|
+
identity: undefined;
|
|
20
|
+
generated: undefined;
|
|
21
|
+
}, {}, {}>;
|
|
22
|
+
poolSize: import("drizzle-orm/pg-core").PgColumn<{
|
|
23
|
+
name: "pool_size";
|
|
24
|
+
tableName: "pool_config";
|
|
25
|
+
dataType: "number";
|
|
26
|
+
columnType: "PgInteger";
|
|
27
|
+
data: number;
|
|
28
|
+
driverParam: string | number;
|
|
29
|
+
notNull: true;
|
|
30
|
+
hasDefault: true;
|
|
31
|
+
isPrimaryKey: false;
|
|
32
|
+
isAutoincrement: false;
|
|
33
|
+
hasRuntimeDefault: false;
|
|
34
|
+
enumValues: undefined;
|
|
35
|
+
baseColumn: never;
|
|
36
|
+
identity: undefined;
|
|
37
|
+
generated: undefined;
|
|
38
|
+
}, {}, {}>;
|
|
39
|
+
};
|
|
40
|
+
dialect: "pg";
|
|
41
|
+
}>;
|
|
@@ -0,0 +1,126 @@
|
|
|
1
|
+
export declare const poolInstances: import("drizzle-orm/pg-core").PgTableWithColumns<{
|
|
2
|
+
name: "pool_instances";
|
|
3
|
+
schema: undefined;
|
|
4
|
+
columns: {
|
|
5
|
+
id: import("drizzle-orm/pg-core").PgColumn<{
|
|
6
|
+
name: "id";
|
|
7
|
+
tableName: "pool_instances";
|
|
8
|
+
dataType: "string";
|
|
9
|
+
columnType: "PgText";
|
|
10
|
+
data: string;
|
|
11
|
+
driverParam: string;
|
|
12
|
+
notNull: true;
|
|
13
|
+
hasDefault: false;
|
|
14
|
+
isPrimaryKey: true;
|
|
15
|
+
isAutoincrement: false;
|
|
16
|
+
hasRuntimeDefault: false;
|
|
17
|
+
enumValues: [string, ...string[]];
|
|
18
|
+
baseColumn: never;
|
|
19
|
+
identity: undefined;
|
|
20
|
+
generated: undefined;
|
|
21
|
+
}, {}, {}>;
|
|
22
|
+
containerId: import("drizzle-orm/pg-core").PgColumn<{
|
|
23
|
+
name: "container_id";
|
|
24
|
+
tableName: "pool_instances";
|
|
25
|
+
dataType: "string";
|
|
26
|
+
columnType: "PgText";
|
|
27
|
+
data: string;
|
|
28
|
+
driverParam: string;
|
|
29
|
+
notNull: true;
|
|
30
|
+
hasDefault: false;
|
|
31
|
+
isPrimaryKey: false;
|
|
32
|
+
isAutoincrement: false;
|
|
33
|
+
hasRuntimeDefault: false;
|
|
34
|
+
enumValues: [string, ...string[]];
|
|
35
|
+
baseColumn: never;
|
|
36
|
+
identity: undefined;
|
|
37
|
+
generated: undefined;
|
|
38
|
+
}, {}, {}>;
|
|
39
|
+
status: import("drizzle-orm/pg-core").PgColumn<{
|
|
40
|
+
name: "status";
|
|
41
|
+
tableName: "pool_instances";
|
|
42
|
+
dataType: "string";
|
|
43
|
+
columnType: "PgText";
|
|
44
|
+
data: string;
|
|
45
|
+
driverParam: string;
|
|
46
|
+
notNull: true;
|
|
47
|
+
hasDefault: true;
|
|
48
|
+
isPrimaryKey: false;
|
|
49
|
+
isAutoincrement: false;
|
|
50
|
+
hasRuntimeDefault: false;
|
|
51
|
+
enumValues: [string, ...string[]];
|
|
52
|
+
baseColumn: never;
|
|
53
|
+
identity: undefined;
|
|
54
|
+
generated: undefined;
|
|
55
|
+
}, {}, {}>;
|
|
56
|
+
tenantId: import("drizzle-orm/pg-core").PgColumn<{
|
|
57
|
+
name: "tenant_id";
|
|
58
|
+
tableName: "pool_instances";
|
|
59
|
+
dataType: "string";
|
|
60
|
+
columnType: "PgText";
|
|
61
|
+
data: string;
|
|
62
|
+
driverParam: string;
|
|
63
|
+
notNull: false;
|
|
64
|
+
hasDefault: false;
|
|
65
|
+
isPrimaryKey: false;
|
|
66
|
+
isAutoincrement: false;
|
|
67
|
+
hasRuntimeDefault: false;
|
|
68
|
+
enumValues: [string, ...string[]];
|
|
69
|
+
baseColumn: never;
|
|
70
|
+
identity: undefined;
|
|
71
|
+
generated: undefined;
|
|
72
|
+
}, {}, {}>;
|
|
73
|
+
name: import("drizzle-orm/pg-core").PgColumn<{
|
|
74
|
+
name: "name";
|
|
75
|
+
tableName: "pool_instances";
|
|
76
|
+
dataType: "string";
|
|
77
|
+
columnType: "PgText";
|
|
78
|
+
data: string;
|
|
79
|
+
driverParam: string;
|
|
80
|
+
notNull: false;
|
|
81
|
+
hasDefault: false;
|
|
82
|
+
isPrimaryKey: false;
|
|
83
|
+
isAutoincrement: false;
|
|
84
|
+
hasRuntimeDefault: false;
|
|
85
|
+
enumValues: [string, ...string[]];
|
|
86
|
+
baseColumn: never;
|
|
87
|
+
identity: undefined;
|
|
88
|
+
generated: undefined;
|
|
89
|
+
}, {}, {}>;
|
|
90
|
+
createdAt: import("drizzle-orm/pg-core").PgColumn<{
|
|
91
|
+
name: "created_at";
|
|
92
|
+
tableName: "pool_instances";
|
|
93
|
+
dataType: "date";
|
|
94
|
+
columnType: "PgTimestamp";
|
|
95
|
+
data: Date;
|
|
96
|
+
driverParam: string;
|
|
97
|
+
notNull: true;
|
|
98
|
+
hasDefault: true;
|
|
99
|
+
isPrimaryKey: false;
|
|
100
|
+
isAutoincrement: false;
|
|
101
|
+
hasRuntimeDefault: false;
|
|
102
|
+
enumValues: undefined;
|
|
103
|
+
baseColumn: never;
|
|
104
|
+
identity: undefined;
|
|
105
|
+
generated: undefined;
|
|
106
|
+
}, {}, {}>;
|
|
107
|
+
claimedAt: import("drizzle-orm/pg-core").PgColumn<{
|
|
108
|
+
name: "claimed_at";
|
|
109
|
+
tableName: "pool_instances";
|
|
110
|
+
dataType: "date";
|
|
111
|
+
columnType: "PgTimestamp";
|
|
112
|
+
data: Date;
|
|
113
|
+
driverParam: string;
|
|
114
|
+
notNull: false;
|
|
115
|
+
hasDefault: false;
|
|
116
|
+
isPrimaryKey: false;
|
|
117
|
+
isAutoincrement: false;
|
|
118
|
+
hasRuntimeDefault: false;
|
|
119
|
+
enumValues: undefined;
|
|
120
|
+
baseColumn: never;
|
|
121
|
+
identity: undefined;
|
|
122
|
+
generated: undefined;
|
|
123
|
+
}, {}, {}>;
|
|
124
|
+
};
|
|
125
|
+
dialect: "pg";
|
|
126
|
+
}>;
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import { pgTable, text, timestamp } from "drizzle-orm/pg-core";
|
|
2
|
+
export const poolInstances = pgTable("pool_instances", {
|
|
3
|
+
id: text("id").primaryKey(),
|
|
4
|
+
containerId: text("container_id").notNull(),
|
|
5
|
+
status: text("status").notNull().default("warm"),
|
|
6
|
+
tenantId: text("tenant_id"),
|
|
7
|
+
name: text("name"),
|
|
8
|
+
createdAt: timestamp("created_at").notNull().defaultNow(),
|
|
9
|
+
claimedAt: timestamp("claimed_at"),
|
|
10
|
+
});
|
|
@@ -144,7 +144,10 @@ describe("createTestContainer", () => {
|
|
|
144
144
|
serviceKeyRepo: {},
|
|
145
145
|
};
|
|
146
146
|
const hotPool = {
|
|
147
|
-
|
|
147
|
+
start: async () => ({ stop: () => { } }),
|
|
148
|
+
claim: async () => null,
|
|
149
|
+
getPoolSize: async () => 2,
|
|
150
|
+
setPoolSize: async () => { },
|
|
148
151
|
};
|
|
149
152
|
const c = createTestContainer({ fleet, crypto, stripe, gateway, hotPool });
|
|
150
153
|
expect(c.fleet).not.toBeNull();
|
|
@@ -45,8 +45,24 @@ export interface GatewayServices {
|
|
|
45
45
|
serviceKeyRepo: IServiceKeyRepository;
|
|
46
46
|
}
|
|
47
47
|
export interface HotPoolServices {
|
|
48
|
-
/**
|
|
49
|
-
|
|
48
|
+
/** Start the pool manager (replenish loop + cleanup). */
|
|
49
|
+
start: () => Promise<{
|
|
50
|
+
stop: () => void;
|
|
51
|
+
}>;
|
|
52
|
+
/** Claim a warm instance from the pool. Returns null if empty. */
|
|
53
|
+
claim: (name: string, tenantId: string, adminUser: {
|
|
54
|
+
id: string;
|
|
55
|
+
email: string;
|
|
56
|
+
name: string;
|
|
57
|
+
}) => Promise<{
|
|
58
|
+
id: string;
|
|
59
|
+
name: string;
|
|
60
|
+
subdomain: string;
|
|
61
|
+
} | null>;
|
|
62
|
+
/** Get current pool size from DB. */
|
|
63
|
+
getPoolSize: () => Promise<number>;
|
|
64
|
+
/** Set pool size in DB. */
|
|
65
|
+
setPoolSize: (size: number) => Promise<void>;
|
|
50
66
|
}
|
|
51
67
|
export interface PlatformContainer {
|
|
52
68
|
db: DrizzleDb;
|
package/dist/server/container.js
CHANGED
|
@@ -115,9 +115,8 @@ export async function buildContainer(bootConfig) {
|
|
|
115
115
|
const serviceKeyRepo = new DrizzleServiceKeyRepository(db);
|
|
116
116
|
gateway = { serviceKeyRepo };
|
|
117
117
|
}
|
|
118
|
-
//
|
|
119
|
-
const
|
|
120
|
-
return {
|
|
118
|
+
// 12. Build the container (hotPool bound after construction)
|
|
119
|
+
const result = {
|
|
121
120
|
db,
|
|
122
121
|
pool,
|
|
123
122
|
productConfig,
|
|
@@ -129,6 +128,21 @@ export async function buildContainer(bootConfig) {
|
|
|
129
128
|
crypto,
|
|
130
129
|
stripe,
|
|
131
130
|
gateway,
|
|
132
|
-
hotPool,
|
|
131
|
+
hotPool: null,
|
|
133
132
|
};
|
|
133
|
+
// Bind hot pool after container construction (closures need the full container)
|
|
134
|
+
if (bootConfig.features.hotPool && fleet) {
|
|
135
|
+
const { startHotPool, setPoolSize: setSize, getPoolSize: getSize } = await import("./services/hot-pool.js");
|
|
136
|
+
const { claimPoolInstance } = await import("./services/hot-pool-claim.js");
|
|
137
|
+
const { DrizzlePoolRepository } = await import("./services/pool-repository.js");
|
|
138
|
+
const poolRepo = new DrizzlePoolRepository(pool);
|
|
139
|
+
const hotPoolConfig = { provisionSecret: bootConfig.provisionSecret };
|
|
140
|
+
result.hotPool = {
|
|
141
|
+
start: () => startHotPool(result, poolRepo, hotPoolConfig),
|
|
142
|
+
claim: (name, tenantId, adminUser) => claimPoolInstance(result, poolRepo, name, tenantId, adminUser, hotPoolConfig),
|
|
143
|
+
getPoolSize: () => getSize(poolRepo),
|
|
144
|
+
setPoolSize: (size) => setSize(poolRepo, size),
|
|
145
|
+
};
|
|
146
|
+
}
|
|
147
|
+
return result;
|
|
134
148
|
}
|
package/dist/server/lifecycle.js
CHANGED
|
@@ -26,6 +26,16 @@ export async function startBackgroundServices(container) {
|
|
|
26
26
|
// Non-fatal — proxy sync will retry on next health tick
|
|
27
27
|
}
|
|
28
28
|
}
|
|
29
|
+
// Hot pool manager (if enabled)
|
|
30
|
+
if (container.hotPool) {
|
|
31
|
+
try {
|
|
32
|
+
const poolHandles = await container.hotPool.start();
|
|
33
|
+
handles.unsubscribes.push(poolHandles.stop);
|
|
34
|
+
}
|
|
35
|
+
catch {
|
|
36
|
+
// Non-fatal — pool will be empty but claiming falls back to cold create
|
|
37
|
+
}
|
|
38
|
+
}
|
|
29
39
|
return handles;
|
|
30
40
|
}
|
|
31
41
|
// ---------------------------------------------------------------------------
|
|
@@ -107,5 +107,23 @@ export declare function createAdminRouter(container: PlatformContainer, config?:
|
|
|
107
107
|
};
|
|
108
108
|
meta: object;
|
|
109
109
|
}>;
|
|
110
|
+
getPoolConfig: import("@trpc/server").TRPCQueryProcedure<{
|
|
111
|
+
input: void;
|
|
112
|
+
output: {
|
|
113
|
+
enabled: boolean;
|
|
114
|
+
poolSize: number;
|
|
115
|
+
warmCount: number;
|
|
116
|
+
};
|
|
117
|
+
meta: object;
|
|
118
|
+
}>;
|
|
119
|
+
setPoolSize: import("@trpc/server").TRPCMutationProcedure<{
|
|
120
|
+
input: {
|
|
121
|
+
size: number;
|
|
122
|
+
};
|
|
123
|
+
output: {
|
|
124
|
+
poolSize: number;
|
|
125
|
+
};
|
|
126
|
+
meta: object;
|
|
127
|
+
}>;
|
|
110
128
|
}>>;
|
|
111
129
|
export {};
|
|
@@ -269,5 +269,26 @@ export function createAdminRouter(container, config) {
|
|
|
269
269
|
orgCount,
|
|
270
270
|
};
|
|
271
271
|
}),
|
|
272
|
+
// -----------------------------------------------------------------------
|
|
273
|
+
// Hot pool config (DB-driven, admin-only)
|
|
274
|
+
// -----------------------------------------------------------------------
|
|
275
|
+
getPoolConfig: adminProcedure.query(async () => {
|
|
276
|
+
if (!container.hotPool) {
|
|
277
|
+
return { enabled: false, poolSize: 0, warmCount: 0 };
|
|
278
|
+
}
|
|
279
|
+
const poolSize = await container.hotPool.getPoolSize();
|
|
280
|
+
const warmRes = await container.pool.query("SELECT COUNT(*)::int AS count FROM pool_instances WHERE status = 'warm'");
|
|
281
|
+
const warmCount = Number(warmRes.rows[0]?.count ?? 0);
|
|
282
|
+
return { enabled: true, poolSize, warmCount };
|
|
283
|
+
}),
|
|
284
|
+
setPoolSize: adminProcedure
|
|
285
|
+
.input(z.object({ size: z.number().int().min(0).max(50) }))
|
|
286
|
+
.mutation(async ({ input }) => {
|
|
287
|
+
if (!container.hotPool) {
|
|
288
|
+
throw new Error("Hot pool not enabled");
|
|
289
|
+
}
|
|
290
|
+
await container.hotPool.setPoolSize(input.size);
|
|
291
|
+
return { poolSize: input.size };
|
|
292
|
+
}),
|
|
272
293
|
});
|
|
273
294
|
}
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Atomic hot pool claiming.
|
|
3
|
+
*
|
|
4
|
+
* Uses IPoolRepository for all DB operations. No raw pool.query().
|
|
5
|
+
*/
|
|
6
|
+
import type { PlatformContainer } from "../container.js";
|
|
7
|
+
import type { IPoolRepository } from "./pool-repository.js";
|
|
8
|
+
export interface ClaimConfig {
|
|
9
|
+
/** Shared secret for provision auth. */
|
|
10
|
+
provisionSecret: string;
|
|
11
|
+
/** Container name prefix. Default: "wopr". */
|
|
12
|
+
containerPrefix?: string;
|
|
13
|
+
}
|
|
14
|
+
export interface ClaimAdminUser {
|
|
15
|
+
id: string;
|
|
16
|
+
email: string;
|
|
17
|
+
name: string;
|
|
18
|
+
}
|
|
19
|
+
export interface ClaimResult {
|
|
20
|
+
id: string;
|
|
21
|
+
name: string;
|
|
22
|
+
subdomain: string;
|
|
23
|
+
}
|
|
24
|
+
/**
|
|
25
|
+
* Claim a warm pool instance, rename it, create a fleet profile,
|
|
26
|
+
* and register the proxy route.
|
|
27
|
+
*
|
|
28
|
+
* Returns the claim result on success, or null if the pool is empty.
|
|
29
|
+
*/
|
|
30
|
+
export declare function claimPoolInstance(container: PlatformContainer, repo: IPoolRepository, name: string, tenantId: string, _adminUser: ClaimAdminUser, config?: ClaimConfig): Promise<ClaimResult | null>;
|
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Atomic hot pool claiming.
|
|
3
|
+
*
|
|
4
|
+
* Uses IPoolRepository for all DB operations. No raw pool.query().
|
|
5
|
+
*/
|
|
6
|
+
import { randomBytes } from "node:crypto";
|
|
7
|
+
import { logger } from "../../config/logger.js";
|
|
8
|
+
import { replenishPool } from "./hot-pool.js";
|
|
9
|
+
/**
|
|
10
|
+
* Claim a warm pool instance, rename it, create a fleet profile,
|
|
11
|
+
* and register the proxy route.
|
|
12
|
+
*
|
|
13
|
+
* Returns the claim result on success, or null if the pool is empty.
|
|
14
|
+
*/
|
|
15
|
+
export async function claimPoolInstance(container, repo, name, tenantId, _adminUser, config) {
|
|
16
|
+
if (!container.fleet)
|
|
17
|
+
throw new Error("Fleet services required for pool claim");
|
|
18
|
+
const pc = container.productConfig;
|
|
19
|
+
const containerPort = pc.fleet?.containerPort ?? 3100;
|
|
20
|
+
const containerImage = pc.fleet?.containerImage ?? "ghcr.io/wopr-network/platform:latest";
|
|
21
|
+
const platformDomain = pc.product?.domain ?? "localhost";
|
|
22
|
+
const prefix = config?.containerPrefix ?? "wopr";
|
|
23
|
+
// ---- Step 1: Atomically claim a warm instance ----
|
|
24
|
+
const claimed = await repo.claimWarm(tenantId, name);
|
|
25
|
+
if (!claimed)
|
|
26
|
+
return null;
|
|
27
|
+
const { id: instanceId, containerId } = claimed;
|
|
28
|
+
// ---- Step 2: Rename Docker container ----
|
|
29
|
+
const docker = container.fleet.docker;
|
|
30
|
+
const containerName = `${prefix}-${name}`;
|
|
31
|
+
try {
|
|
32
|
+
const c = docker.getContainer(containerId);
|
|
33
|
+
await c.rename({ name: containerName });
|
|
34
|
+
logger.info(`Pool claim: renamed container to ${containerName}`);
|
|
35
|
+
}
|
|
36
|
+
catch (err) {
|
|
37
|
+
logger.error("Pool claim: rename failed", { error: err.message });
|
|
38
|
+
await repo.markDead(instanceId);
|
|
39
|
+
return null;
|
|
40
|
+
}
|
|
41
|
+
// ---- Step 3: Create fleet profile ----
|
|
42
|
+
const serviceKeyRepo = container.fleet.serviceKeyRepo;
|
|
43
|
+
const gatewayKey = serviceKeyRepo ? await serviceKeyRepo.generate(tenantId, instanceId) : crypto.randomUUID();
|
|
44
|
+
const store = container.fleet.profileStore;
|
|
45
|
+
const profile = {
|
|
46
|
+
id: instanceId,
|
|
47
|
+
name,
|
|
48
|
+
tenantId,
|
|
49
|
+
image: containerImage,
|
|
50
|
+
description: `Managed instance: ${name}`,
|
|
51
|
+
env: {
|
|
52
|
+
PORT: String(containerPort),
|
|
53
|
+
HOST: "0.0.0.0",
|
|
54
|
+
NODE_ENV: "production",
|
|
55
|
+
PROVISION_SECRET: config?.provisionSecret ?? "",
|
|
56
|
+
BETTER_AUTH_SECRET: randomBytes(32).toString("hex"),
|
|
57
|
+
DATA_HOME: "/data",
|
|
58
|
+
HOSTED_MODE: "true",
|
|
59
|
+
DEPLOYMENT_MODE: "hosted_proxy",
|
|
60
|
+
DEPLOYMENT_EXPOSURE: "private",
|
|
61
|
+
MIGRATION_AUTO_APPLY: "true",
|
|
62
|
+
GATEWAY_KEY: gatewayKey,
|
|
63
|
+
},
|
|
64
|
+
restartPolicy: "unless-stopped",
|
|
65
|
+
releaseChannel: "stable",
|
|
66
|
+
updatePolicy: "manual",
|
|
67
|
+
};
|
|
68
|
+
await store.save(profile);
|
|
69
|
+
logger.info(`Pool claim: saved fleet profile for ${name} (${instanceId})`);
|
|
70
|
+
// ---- Step 4: Register proxy route ----
|
|
71
|
+
try {
|
|
72
|
+
if (container.fleet.proxy.addRoute) {
|
|
73
|
+
await container.fleet.proxy.addRoute({
|
|
74
|
+
instanceId,
|
|
75
|
+
subdomain: name,
|
|
76
|
+
upstreamHost: containerName,
|
|
77
|
+
upstreamPort: containerPort,
|
|
78
|
+
healthy: true,
|
|
79
|
+
});
|
|
80
|
+
logger.info(`Pool claim: registered proxy route ${name}.${platformDomain}`);
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
catch (err) {
|
|
84
|
+
logger.error("Pool claim: proxy route registration failed", { error: err.message });
|
|
85
|
+
}
|
|
86
|
+
// ---- Step 5: Replenish pool in background ----
|
|
87
|
+
replenishPool(container, repo, { provisionSecret: config?.provisionSecret ?? "" }).catch((err) => {
|
|
88
|
+
logger.error("Pool replenish after claim failed", { error: err.message });
|
|
89
|
+
});
|
|
90
|
+
const subdomain = `${name}.${platformDomain}`;
|
|
91
|
+
return { id: instanceId, name, subdomain };
|
|
92
|
+
}
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Hot pool manager — pre-provisions warm containers for instant claiming.
|
|
3
|
+
*
|
|
4
|
+
* Reads desired pool size from DB (`pool_config` table) via IPoolRepository.
|
|
5
|
+
* Periodically replenishes the pool and cleans up dead containers.
|
|
6
|
+
*
|
|
7
|
+
* All config is DB-driven — no env vars for pool size, container image,
|
|
8
|
+
* or port. Admin API updates pool_config, this reads it.
|
|
9
|
+
*/
|
|
10
|
+
import type { PlatformContainer } from "../container.js";
|
|
11
|
+
import type { IPoolRepository } from "./pool-repository.js";
|
|
12
|
+
export interface HotPoolConfig {
|
|
13
|
+
/** Shared secret for provision auth between platform and managed instances. */
|
|
14
|
+
provisionSecret: string;
|
|
15
|
+
/** Replenish interval in ms. Default: 60_000. */
|
|
16
|
+
replenishIntervalMs?: number;
|
|
17
|
+
}
|
|
18
|
+
export interface HotPoolHandles {
|
|
19
|
+
replenishTimer: ReturnType<typeof setInterval>;
|
|
20
|
+
stop: () => void;
|
|
21
|
+
}
|
|
22
|
+
export declare function getPoolSize(repo: IPoolRepository): Promise<number>;
|
|
23
|
+
export declare function setPoolSize(repo: IPoolRepository, size: number): Promise<void>;
|
|
24
|
+
export declare function replenishPool(container: PlatformContainer, repo: IPoolRepository, config: HotPoolConfig): Promise<void>;
|
|
25
|
+
export declare function startHotPool(container: PlatformContainer, repo: IPoolRepository, config: HotPoolConfig): Promise<HotPoolHandles>;
|
|
@@ -0,0 +1,129 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Hot pool manager — pre-provisions warm containers for instant claiming.
|
|
3
|
+
*
|
|
4
|
+
* Reads desired pool size from DB (`pool_config` table) via IPoolRepository.
|
|
5
|
+
* Periodically replenishes the pool and cleans up dead containers.
|
|
6
|
+
*
|
|
7
|
+
* All config is DB-driven — no env vars for pool size, container image,
|
|
8
|
+
* or port. Admin API updates pool_config, this reads it.
|
|
9
|
+
*/
|
|
10
|
+
import { logger } from "../../config/logger.js";
|
|
11
|
+
// ---------------------------------------------------------------------------
|
|
12
|
+
// Pool size — delegates to repository
|
|
13
|
+
// ---------------------------------------------------------------------------
|
|
14
|
+
export async function getPoolSize(repo) {
|
|
15
|
+
return repo.getPoolSize();
|
|
16
|
+
}
|
|
17
|
+
export async function setPoolSize(repo, size) {
|
|
18
|
+
return repo.setPoolSize(size);
|
|
19
|
+
}
|
|
20
|
+
// ---------------------------------------------------------------------------
|
|
21
|
+
// Warm container management
|
|
22
|
+
// ---------------------------------------------------------------------------
|
|
23
|
+
async function createWarmContainer(container, repo, config) {
|
|
24
|
+
if (!container.fleet)
|
|
25
|
+
throw new Error("Fleet services required for hot pool");
|
|
26
|
+
const pc = container.productConfig;
|
|
27
|
+
const containerImage = pc.fleet?.containerImage ?? "ghcr.io/wopr-network/platform:latest";
|
|
28
|
+
const containerPort = pc.fleet?.containerPort ?? 3100;
|
|
29
|
+
const provisionSecret = config.provisionSecret;
|
|
30
|
+
const dockerNetwork = pc.fleet?.dockerNetwork ?? "";
|
|
31
|
+
const docker = container.fleet.docker;
|
|
32
|
+
const id = crypto.randomUUID();
|
|
33
|
+
const containerName = `pool-${id.slice(0, 8)}`;
|
|
34
|
+
const volumeName = `pool-${id.slice(0, 8)}`;
|
|
35
|
+
try {
|
|
36
|
+
// Init volume permissions
|
|
37
|
+
const init = await docker.createContainer({
|
|
38
|
+
Image: containerImage,
|
|
39
|
+
Entrypoint: ["/bin/sh", "-c"],
|
|
40
|
+
Cmd: ["chown -R 999:999 /data"],
|
|
41
|
+
User: "root",
|
|
42
|
+
HostConfig: { Binds: [`${volumeName}:/data`] },
|
|
43
|
+
});
|
|
44
|
+
await init.start();
|
|
45
|
+
await init.wait();
|
|
46
|
+
await init.remove();
|
|
47
|
+
const warmContainer = await docker.createContainer({
|
|
48
|
+
Image: containerImage,
|
|
49
|
+
name: containerName,
|
|
50
|
+
Env: [`PORT=${containerPort}`, `PROVISION_SECRET=${provisionSecret}`, "HOME=/data"],
|
|
51
|
+
HostConfig: {
|
|
52
|
+
Binds: [`${volumeName}:/data`],
|
|
53
|
+
RestartPolicy: { Name: "unless-stopped" },
|
|
54
|
+
},
|
|
55
|
+
});
|
|
56
|
+
await warmContainer.start();
|
|
57
|
+
if (dockerNetwork) {
|
|
58
|
+
const network = docker.getNetwork(dockerNetwork);
|
|
59
|
+
await network.connect({ Container: warmContainer.id });
|
|
60
|
+
}
|
|
61
|
+
await repo.insertWarm(id, warmContainer.id);
|
|
62
|
+
logger.info(`Hot pool: created warm container ${containerName} (${id})`);
|
|
63
|
+
}
|
|
64
|
+
catch (err) {
|
|
65
|
+
logger.error("Hot pool: failed to create warm container", {
|
|
66
|
+
error: err.message,
|
|
67
|
+
});
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
export async function replenishPool(container, repo, config) {
|
|
71
|
+
const desired = await repo.getPoolSize();
|
|
72
|
+
const current = await repo.warmCount();
|
|
73
|
+
const deficit = desired - current;
|
|
74
|
+
if (deficit <= 0)
|
|
75
|
+
return;
|
|
76
|
+
logger.info(`Hot pool: replenishing ${deficit} container(s) (have ${current}, want ${desired})`);
|
|
77
|
+
for (let i = 0; i < deficit; i++) {
|
|
78
|
+
await createWarmContainer(container, repo, config);
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
async function cleanupDead(container, repo) {
|
|
82
|
+
if (!container.fleet)
|
|
83
|
+
return;
|
|
84
|
+
const docker = container.fleet.docker;
|
|
85
|
+
const warmInstances = await repo.listWarm();
|
|
86
|
+
for (const instance of warmInstances) {
|
|
87
|
+
try {
|
|
88
|
+
const c = docker.getContainer(instance.containerId);
|
|
89
|
+
const info = await c.inspect();
|
|
90
|
+
if (!info.State.Running) {
|
|
91
|
+
await repo.markDead(instance.id);
|
|
92
|
+
try {
|
|
93
|
+
await c.remove({ force: true });
|
|
94
|
+
}
|
|
95
|
+
catch {
|
|
96
|
+
/* already gone */
|
|
97
|
+
}
|
|
98
|
+
logger.warn(`Hot pool: marked dead container ${instance.id}`);
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
catch {
|
|
102
|
+
await repo.markDead(instance.id);
|
|
103
|
+
logger.warn(`Hot pool: marked missing container ${instance.id} as dead`);
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
await repo.deleteDead();
|
|
107
|
+
}
|
|
108
|
+
// ---------------------------------------------------------------------------
|
|
109
|
+
// Lifecycle
|
|
110
|
+
// ---------------------------------------------------------------------------
|
|
111
|
+
export async function startHotPool(container, repo, config) {
|
|
112
|
+
await cleanupDead(container, repo);
|
|
113
|
+
await replenishPool(container, repo, config);
|
|
114
|
+
const intervalMs = config.replenishIntervalMs ?? 60_000;
|
|
115
|
+
const replenishTimer = setInterval(async () => {
|
|
116
|
+
try {
|
|
117
|
+
await cleanupDead(container, repo);
|
|
118
|
+
await replenishPool(container, repo, config);
|
|
119
|
+
}
|
|
120
|
+
catch (err) {
|
|
121
|
+
logger.error("Hot pool tick failed", { error: err.message });
|
|
122
|
+
}
|
|
123
|
+
}, intervalMs);
|
|
124
|
+
logger.info("Hot pool manager started");
|
|
125
|
+
return {
|
|
126
|
+
replenishTimer,
|
|
127
|
+
stop: () => clearInterval(replenishTimer),
|
|
128
|
+
};
|
|
129
|
+
}
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Repository for hot pool database operations.
|
|
3
|
+
*
|
|
4
|
+
* Encapsulates all pool_config and pool_instances queries behind
|
|
5
|
+
* a testable interface. No raw pool.query() outside this file.
|
|
6
|
+
*/
|
|
7
|
+
import type { Pool } from "pg";
|
|
8
|
+
export interface PoolInstance {
|
|
9
|
+
id: string;
|
|
10
|
+
containerId: string;
|
|
11
|
+
status: string;
|
|
12
|
+
tenantId: string | null;
|
|
13
|
+
name: string | null;
|
|
14
|
+
}
|
|
15
|
+
export interface IPoolRepository {
|
|
16
|
+
getPoolSize(): Promise<number>;
|
|
17
|
+
setPoolSize(size: number): Promise<void>;
|
|
18
|
+
warmCount(): Promise<number>;
|
|
19
|
+
insertWarm(id: string, containerId: string): Promise<void>;
|
|
20
|
+
listWarm(): Promise<PoolInstance[]>;
|
|
21
|
+
markDead(id: string): Promise<void>;
|
|
22
|
+
deleteDead(): Promise<void>;
|
|
23
|
+
claimWarm(tenantId: string, name: string): Promise<{
|
|
24
|
+
id: string;
|
|
25
|
+
containerId: string;
|
|
26
|
+
} | null>;
|
|
27
|
+
updateInstanceStatus(id: string, status: string): Promise<void>;
|
|
28
|
+
}
|
|
29
|
+
export declare class DrizzlePoolRepository implements IPoolRepository {
|
|
30
|
+
private pool;
|
|
31
|
+
constructor(pool: Pool);
|
|
32
|
+
getPoolSize(): Promise<number>;
|
|
33
|
+
setPoolSize(size: number): Promise<void>;
|
|
34
|
+
warmCount(): Promise<number>;
|
|
35
|
+
insertWarm(id: string, containerId: string): Promise<void>;
|
|
36
|
+
listWarm(): Promise<PoolInstance[]>;
|
|
37
|
+
markDead(id: string): Promise<void>;
|
|
38
|
+
deleteDead(): Promise<void>;
|
|
39
|
+
claimWarm(tenantId: string, name: string): Promise<{
|
|
40
|
+
id: string;
|
|
41
|
+
containerId: string;
|
|
42
|
+
} | null>;
|
|
43
|
+
updateInstanceStatus(id: string, status: string): Promise<void>;
|
|
44
|
+
}
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Repository for hot pool database operations.
|
|
3
|
+
*
|
|
4
|
+
* Encapsulates all pool_config and pool_instances queries behind
|
|
5
|
+
* a testable interface. No raw pool.query() outside this file.
|
|
6
|
+
*/
|
|
7
|
+
export class DrizzlePoolRepository {
|
|
8
|
+
pool;
|
|
9
|
+
constructor(pool) {
|
|
10
|
+
this.pool = pool;
|
|
11
|
+
}
|
|
12
|
+
async getPoolSize() {
|
|
13
|
+
try {
|
|
14
|
+
const res = await this.pool.query("SELECT pool_size FROM pool_config WHERE id = 1");
|
|
15
|
+
return res.rows[0]?.pool_size ?? 2;
|
|
16
|
+
}
|
|
17
|
+
catch {
|
|
18
|
+
return 2;
|
|
19
|
+
}
|
|
20
|
+
}
|
|
21
|
+
async setPoolSize(size) {
|
|
22
|
+
await this.pool.query("INSERT INTO pool_config (id, pool_size) VALUES (1, $1) ON CONFLICT (id) DO UPDATE SET pool_size = $1", [size]);
|
|
23
|
+
}
|
|
24
|
+
async warmCount() {
|
|
25
|
+
const res = await this.pool.query("SELECT COUNT(*)::int AS count FROM pool_instances WHERE status = 'warm'");
|
|
26
|
+
return res.rows[0].count;
|
|
27
|
+
}
|
|
28
|
+
async insertWarm(id, containerId) {
|
|
29
|
+
await this.pool.query("INSERT INTO pool_instances (id, container_id, status) VALUES ($1, $2, 'warm')", [
|
|
30
|
+
id,
|
|
31
|
+
containerId,
|
|
32
|
+
]);
|
|
33
|
+
}
|
|
34
|
+
async listWarm() {
|
|
35
|
+
const res = await this.pool.query("SELECT id, container_id, status, tenant_id, name FROM pool_instances WHERE status = 'warm'");
|
|
36
|
+
return res.rows.map((r) => ({
|
|
37
|
+
id: r.id,
|
|
38
|
+
containerId: r.container_id,
|
|
39
|
+
status: r.status,
|
|
40
|
+
tenantId: r.tenant_id ?? null,
|
|
41
|
+
name: r.name ?? null,
|
|
42
|
+
}));
|
|
43
|
+
}
|
|
44
|
+
async markDead(id) {
|
|
45
|
+
await this.pool.query("UPDATE pool_instances SET status = 'dead' WHERE id = $1", [id]);
|
|
46
|
+
}
|
|
47
|
+
async deleteDead() {
|
|
48
|
+
await this.pool.query("DELETE FROM pool_instances WHERE status = 'dead'");
|
|
49
|
+
}
|
|
50
|
+
async claimWarm(tenantId, name) {
|
|
51
|
+
const res = await this.pool.query(`UPDATE pool_instances
|
|
52
|
+
SET status = 'claimed',
|
|
53
|
+
claimed_at = NOW(),
|
|
54
|
+
tenant_id = $1,
|
|
55
|
+
name = $2
|
|
56
|
+
WHERE id = (
|
|
57
|
+
SELECT id FROM pool_instances
|
|
58
|
+
WHERE status = 'warm'
|
|
59
|
+
ORDER BY created_at ASC
|
|
60
|
+
LIMIT 1
|
|
61
|
+
FOR UPDATE SKIP LOCKED
|
|
62
|
+
)
|
|
63
|
+
RETURNING id, container_id`, [tenantId, name]);
|
|
64
|
+
if (res.rowCount === 0)
|
|
65
|
+
return null;
|
|
66
|
+
const row = res.rows[0];
|
|
67
|
+
return { id: row.id, containerId: row.container_id };
|
|
68
|
+
}
|
|
69
|
+
async updateInstanceStatus(id, status) {
|
|
70
|
+
await this.pool.query("UPDATE pool_instances SET status = $1 WHERE id = $2", [status, id]);
|
|
71
|
+
}
|
|
72
|
+
}
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
-- Hot pool: pre-provisioned warm containers for instant claiming
|
|
2
|
+
CREATE TABLE IF NOT EXISTS "pool_config" (
|
|
3
|
+
"id" integer PRIMARY KEY DEFAULT 1,
|
|
4
|
+
"pool_size" integer NOT NULL DEFAULT 2
|
|
5
|
+
);
|
|
6
|
+
--> statement-breakpoint
|
|
7
|
+
|
|
8
|
+
CREATE TABLE IF NOT EXISTS "pool_instances" (
|
|
9
|
+
"id" text PRIMARY KEY,
|
|
10
|
+
"container_id" text NOT NULL,
|
|
11
|
+
"status" text NOT NULL DEFAULT 'warm',
|
|
12
|
+
"tenant_id" text,
|
|
13
|
+
"name" text,
|
|
14
|
+
"created_at" timestamp NOT NULL DEFAULT now(),
|
|
15
|
+
"claimed_at" timestamp
|
|
16
|
+
);
|
|
17
|
+
--> statement-breakpoint
|
|
18
|
+
|
|
19
|
+
-- Claim query: WHERE status = 'warm' ORDER BY created_at ASC FOR UPDATE SKIP LOCKED
|
|
20
|
+
CREATE INDEX IF NOT EXISTS "pool_instances_status_created" ON "pool_instances" ("status", "created_at");
|
|
21
|
+
--> statement-breakpoint
|
|
22
|
+
|
|
23
|
+
-- Tenant lookup for admin queries
|
|
24
|
+
CREATE INDEX IF NOT EXISTS "pool_instances_tenant" ON "pool_instances" ("tenant_id") WHERE "tenant_id" IS NOT NULL;
|
|
25
|
+
--> statement-breakpoint
|
|
26
|
+
|
|
27
|
+
-- Seed default pool config
|
|
28
|
+
INSERT INTO "pool_config" ("id", "pool_size") VALUES (1, 2)
|
|
29
|
+
ON CONFLICT ("id") DO NOTHING;
|
package/package.json
CHANGED
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import { pgTable, text, timestamp } from "drizzle-orm/pg-core";
|
|
2
|
+
|
|
3
|
+
export const poolInstances = pgTable("pool_instances", {
|
|
4
|
+
id: text("id").primaryKey(),
|
|
5
|
+
containerId: text("container_id").notNull(),
|
|
6
|
+
status: text("status").notNull().default("warm"),
|
|
7
|
+
tenantId: text("tenant_id"),
|
|
8
|
+
name: text("name"),
|
|
9
|
+
createdAt: timestamp("created_at").notNull().defaultNow(),
|
|
10
|
+
claimedAt: timestamp("claimed_at"),
|
|
11
|
+
});
|
|
@@ -172,7 +172,10 @@ describe("createTestContainer", () => {
|
|
|
172
172
|
};
|
|
173
173
|
|
|
174
174
|
const hotPool: HotPoolServices = {
|
|
175
|
-
|
|
175
|
+
start: async () => ({ stop: () => {} }),
|
|
176
|
+
claim: async () => null,
|
|
177
|
+
getPoolSize: async () => 2,
|
|
178
|
+
setPoolSize: async () => {},
|
|
176
179
|
};
|
|
177
180
|
|
|
178
181
|
const c = createTestContainer({ fleet, crypto, stripe, gateway, hotPool });
|
package/src/server/container.ts
CHANGED
|
@@ -53,8 +53,18 @@ export interface GatewayServices {
|
|
|
53
53
|
}
|
|
54
54
|
|
|
55
55
|
export interface HotPoolServices {
|
|
56
|
-
/**
|
|
57
|
-
|
|
56
|
+
/** Start the pool manager (replenish loop + cleanup). */
|
|
57
|
+
start: () => Promise<{ stop: () => void }>;
|
|
58
|
+
/** Claim a warm instance from the pool. Returns null if empty. */
|
|
59
|
+
claim: (
|
|
60
|
+
name: string,
|
|
61
|
+
tenantId: string,
|
|
62
|
+
adminUser: { id: string; email: string; name: string },
|
|
63
|
+
) => Promise<{ id: string; name: string; subdomain: string } | null>;
|
|
64
|
+
/** Get current pool size from DB. */
|
|
65
|
+
getPoolSize: () => Promise<number>;
|
|
66
|
+
/** Set pool size in DB. */
|
|
67
|
+
setPoolSize: (size: number) => Promise<void>;
|
|
58
68
|
}
|
|
59
69
|
|
|
60
70
|
// ---------------------------------------------------------------------------
|
|
@@ -217,10 +227,8 @@ export async function buildContainer(bootConfig: BootConfig): Promise<PlatformCo
|
|
|
217
227
|
gateway = { serviceKeyRepo };
|
|
218
228
|
}
|
|
219
229
|
|
|
220
|
-
//
|
|
221
|
-
const
|
|
222
|
-
|
|
223
|
-
return {
|
|
230
|
+
// 12. Build the container (hotPool bound after construction)
|
|
231
|
+
const result: PlatformContainer = {
|
|
224
232
|
db,
|
|
225
233
|
pool,
|
|
226
234
|
productConfig,
|
|
@@ -232,6 +240,25 @@ export async function buildContainer(bootConfig: BootConfig): Promise<PlatformCo
|
|
|
232
240
|
crypto,
|
|
233
241
|
stripe,
|
|
234
242
|
gateway,
|
|
235
|
-
hotPool,
|
|
243
|
+
hotPool: null,
|
|
236
244
|
};
|
|
245
|
+
|
|
246
|
+
// Bind hot pool after container construction (closures need the full container)
|
|
247
|
+
if (bootConfig.features.hotPool && fleet) {
|
|
248
|
+
const { startHotPool, setPoolSize: setSize, getPoolSize: getSize } = await import("./services/hot-pool.js");
|
|
249
|
+
const { claimPoolInstance } = await import("./services/hot-pool-claim.js");
|
|
250
|
+
const { DrizzlePoolRepository } = await import("./services/pool-repository.js");
|
|
251
|
+
const poolRepo = new DrizzlePoolRepository(pool);
|
|
252
|
+
|
|
253
|
+
const hotPoolConfig = { provisionSecret: bootConfig.provisionSecret };
|
|
254
|
+
result.hotPool = {
|
|
255
|
+
start: () => startHotPool(result, poolRepo, hotPoolConfig),
|
|
256
|
+
claim: (name, tenantId, adminUser) =>
|
|
257
|
+
claimPoolInstance(result, poolRepo, name, tenantId, adminUser, hotPoolConfig),
|
|
258
|
+
getPoolSize: () => getSize(poolRepo),
|
|
259
|
+
setPoolSize: (size) => setSize(poolRepo, size),
|
|
260
|
+
};
|
|
261
|
+
}
|
|
262
|
+
|
|
263
|
+
return result;
|
|
237
264
|
}
|
package/src/server/lifecycle.ts
CHANGED
|
@@ -40,6 +40,16 @@ export async function startBackgroundServices(container: PlatformContainer): Pro
|
|
|
40
40
|
}
|
|
41
41
|
}
|
|
42
42
|
|
|
43
|
+
// Hot pool manager (if enabled)
|
|
44
|
+
if (container.hotPool) {
|
|
45
|
+
try {
|
|
46
|
+
const poolHandles = await container.hotPool.start();
|
|
47
|
+
handles.unsubscribes.push(poolHandles.stop);
|
|
48
|
+
} catch {
|
|
49
|
+
// Non-fatal — pool will be empty but claiming falls back to cold create
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
|
|
43
53
|
return handles;
|
|
44
54
|
}
|
|
45
55
|
|
|
@@ -330,5 +330,31 @@ export function createAdminRouter(container: PlatformContainer, config?: AdminRo
|
|
|
330
330
|
orgCount,
|
|
331
331
|
};
|
|
332
332
|
}),
|
|
333
|
+
|
|
334
|
+
// -----------------------------------------------------------------------
|
|
335
|
+
// Hot pool config (DB-driven, admin-only)
|
|
336
|
+
// -----------------------------------------------------------------------
|
|
337
|
+
|
|
338
|
+
getPoolConfig: adminProcedure.query(async () => {
|
|
339
|
+
if (!container.hotPool) {
|
|
340
|
+
return { enabled: false, poolSize: 0, warmCount: 0 };
|
|
341
|
+
}
|
|
342
|
+
const poolSize = await container.hotPool.getPoolSize();
|
|
343
|
+
const warmRes = await container.pool.query<{ count: string }>(
|
|
344
|
+
"SELECT COUNT(*)::int AS count FROM pool_instances WHERE status = 'warm'",
|
|
345
|
+
);
|
|
346
|
+
const warmCount = Number(warmRes.rows[0]?.count ?? 0);
|
|
347
|
+
return { enabled: true, poolSize, warmCount };
|
|
348
|
+
}),
|
|
349
|
+
|
|
350
|
+
setPoolSize: adminProcedure
|
|
351
|
+
.input(z.object({ size: z.number().int().min(0).max(50) }))
|
|
352
|
+
.mutation(async ({ input }) => {
|
|
353
|
+
if (!container.hotPool) {
|
|
354
|
+
throw new Error("Hot pool not enabled");
|
|
355
|
+
}
|
|
356
|
+
await container.hotPool.setPoolSize(input.size);
|
|
357
|
+
return { poolSize: input.size };
|
|
358
|
+
}),
|
|
333
359
|
});
|
|
334
360
|
}
|
|
@@ -0,0 +1,130 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Atomic hot pool claiming.
|
|
3
|
+
*
|
|
4
|
+
* Uses IPoolRepository for all DB operations. No raw pool.query().
|
|
5
|
+
*/
|
|
6
|
+
|
|
7
|
+
import { randomBytes } from "node:crypto";
|
|
8
|
+
|
|
9
|
+
import { logger } from "../../config/logger.js";
|
|
10
|
+
import type { PlatformContainer } from "../container.js";
|
|
11
|
+
import { replenishPool } from "./hot-pool.js";
|
|
12
|
+
import type { IPoolRepository } from "./pool-repository.js";
|
|
13
|
+
|
|
14
|
+
export interface ClaimConfig {
|
|
15
|
+
/** Shared secret for provision auth. */
|
|
16
|
+
provisionSecret: string;
|
|
17
|
+
/** Container name prefix. Default: "wopr". */
|
|
18
|
+
containerPrefix?: string;
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
export interface ClaimAdminUser {
|
|
22
|
+
id: string;
|
|
23
|
+
email: string;
|
|
24
|
+
name: string;
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
export interface ClaimResult {
|
|
28
|
+
id: string;
|
|
29
|
+
name: string;
|
|
30
|
+
subdomain: string;
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
/**
|
|
34
|
+
* Claim a warm pool instance, rename it, create a fleet profile,
|
|
35
|
+
* and register the proxy route.
|
|
36
|
+
*
|
|
37
|
+
* Returns the claim result on success, or null if the pool is empty.
|
|
38
|
+
*/
|
|
39
|
+
export async function claimPoolInstance(
|
|
40
|
+
container: PlatformContainer,
|
|
41
|
+
repo: IPoolRepository,
|
|
42
|
+
name: string,
|
|
43
|
+
tenantId: string,
|
|
44
|
+
_adminUser: ClaimAdminUser,
|
|
45
|
+
config?: ClaimConfig,
|
|
46
|
+
): Promise<ClaimResult | null> {
|
|
47
|
+
if (!container.fleet) throw new Error("Fleet services required for pool claim");
|
|
48
|
+
|
|
49
|
+
const pc = container.productConfig;
|
|
50
|
+
const containerPort = pc.fleet?.containerPort ?? 3100;
|
|
51
|
+
const containerImage = pc.fleet?.containerImage ?? "ghcr.io/wopr-network/platform:latest";
|
|
52
|
+
const platformDomain = pc.product?.domain ?? "localhost";
|
|
53
|
+
const prefix = config?.containerPrefix ?? "wopr";
|
|
54
|
+
|
|
55
|
+
// ---- Step 1: Atomically claim a warm instance ----
|
|
56
|
+
const claimed = await repo.claimWarm(tenantId, name);
|
|
57
|
+
if (!claimed) return null;
|
|
58
|
+
|
|
59
|
+
const { id: instanceId, containerId } = claimed;
|
|
60
|
+
|
|
61
|
+
// ---- Step 2: Rename Docker container ----
|
|
62
|
+
const docker = container.fleet.docker;
|
|
63
|
+
const containerName = `${prefix}-${name}`;
|
|
64
|
+
|
|
65
|
+
try {
|
|
66
|
+
const c = docker.getContainer(containerId);
|
|
67
|
+
await c.rename({ name: containerName });
|
|
68
|
+
logger.info(`Pool claim: renamed container to ${containerName}`);
|
|
69
|
+
} catch (err) {
|
|
70
|
+
logger.error("Pool claim: rename failed", { error: (err as Error).message });
|
|
71
|
+
await repo.markDead(instanceId);
|
|
72
|
+
return null;
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
// ---- Step 3: Create fleet profile ----
|
|
76
|
+
const serviceKeyRepo = container.fleet.serviceKeyRepo;
|
|
77
|
+
const gatewayKey = serviceKeyRepo ? await serviceKeyRepo.generate(tenantId, instanceId) : crypto.randomUUID();
|
|
78
|
+
|
|
79
|
+
const store = container.fleet.profileStore;
|
|
80
|
+
const profile = {
|
|
81
|
+
id: instanceId,
|
|
82
|
+
name,
|
|
83
|
+
tenantId,
|
|
84
|
+
image: containerImage,
|
|
85
|
+
description: `Managed instance: ${name}`,
|
|
86
|
+
env: {
|
|
87
|
+
PORT: String(containerPort),
|
|
88
|
+
HOST: "0.0.0.0",
|
|
89
|
+
NODE_ENV: "production",
|
|
90
|
+
PROVISION_SECRET: config?.provisionSecret ?? "",
|
|
91
|
+
BETTER_AUTH_SECRET: randomBytes(32).toString("hex"),
|
|
92
|
+
DATA_HOME: "/data",
|
|
93
|
+
HOSTED_MODE: "true",
|
|
94
|
+
DEPLOYMENT_MODE: "hosted_proxy",
|
|
95
|
+
DEPLOYMENT_EXPOSURE: "private",
|
|
96
|
+
MIGRATION_AUTO_APPLY: "true",
|
|
97
|
+
GATEWAY_KEY: gatewayKey,
|
|
98
|
+
},
|
|
99
|
+
restartPolicy: "unless-stopped" as const,
|
|
100
|
+
releaseChannel: "stable" as const,
|
|
101
|
+
updatePolicy: "manual" as const,
|
|
102
|
+
};
|
|
103
|
+
|
|
104
|
+
await store.save(profile);
|
|
105
|
+
logger.info(`Pool claim: saved fleet profile for ${name} (${instanceId})`);
|
|
106
|
+
|
|
107
|
+
// ---- Step 4: Register proxy route ----
|
|
108
|
+
try {
|
|
109
|
+
if (container.fleet.proxy.addRoute) {
|
|
110
|
+
await container.fleet.proxy.addRoute({
|
|
111
|
+
instanceId,
|
|
112
|
+
subdomain: name,
|
|
113
|
+
upstreamHost: containerName,
|
|
114
|
+
upstreamPort: containerPort,
|
|
115
|
+
healthy: true,
|
|
116
|
+
});
|
|
117
|
+
logger.info(`Pool claim: registered proxy route ${name}.${platformDomain}`);
|
|
118
|
+
}
|
|
119
|
+
} catch (err) {
|
|
120
|
+
logger.error("Pool claim: proxy route registration failed", { error: (err as Error).message });
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
// ---- Step 5: Replenish pool in background ----
|
|
124
|
+
replenishPool(container, repo, { provisionSecret: config?.provisionSecret ?? "" }).catch((err) => {
|
|
125
|
+
logger.error("Pool replenish after claim failed", { error: (err as Error).message });
|
|
126
|
+
});
|
|
127
|
+
|
|
128
|
+
const subdomain = `${name}.${platformDomain}`;
|
|
129
|
+
return { id: instanceId, name, subdomain };
|
|
130
|
+
}
|
|
@@ -0,0 +1,174 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Hot pool manager — pre-provisions warm containers for instant claiming.
|
|
3
|
+
*
|
|
4
|
+
* Reads desired pool size from DB (`pool_config` table) via IPoolRepository.
|
|
5
|
+
* Periodically replenishes the pool and cleans up dead containers.
|
|
6
|
+
*
|
|
7
|
+
* All config is DB-driven — no env vars for pool size, container image,
|
|
8
|
+
* or port. Admin API updates pool_config, this reads it.
|
|
9
|
+
*/
|
|
10
|
+
|
|
11
|
+
import { logger } from "../../config/logger.js";
|
|
12
|
+
import type { PlatformContainer } from "../container.js";
|
|
13
|
+
import type { IPoolRepository } from "./pool-repository.js";
|
|
14
|
+
|
|
15
|
+
export interface HotPoolConfig {
|
|
16
|
+
/** Shared secret for provision auth between platform and managed instances. */
|
|
17
|
+
provisionSecret: string;
|
|
18
|
+
/** Replenish interval in ms. Default: 60_000. */
|
|
19
|
+
replenishIntervalMs?: number;
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
export interface HotPoolHandles {
|
|
23
|
+
replenishTimer: ReturnType<typeof setInterval>;
|
|
24
|
+
stop: () => void;
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
// ---------------------------------------------------------------------------
|
|
28
|
+
// Pool size — delegates to repository
|
|
29
|
+
// ---------------------------------------------------------------------------
|
|
30
|
+
|
|
31
|
+
export async function getPoolSize(repo: IPoolRepository): Promise<number> {
|
|
32
|
+
return repo.getPoolSize();
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
export async function setPoolSize(repo: IPoolRepository, size: number): Promise<void> {
|
|
36
|
+
return repo.setPoolSize(size);
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
// ---------------------------------------------------------------------------
|
|
40
|
+
// Warm container management
|
|
41
|
+
// ---------------------------------------------------------------------------
|
|
42
|
+
|
|
43
|
+
async function createWarmContainer(
|
|
44
|
+
container: PlatformContainer,
|
|
45
|
+
repo: IPoolRepository,
|
|
46
|
+
config: HotPoolConfig,
|
|
47
|
+
): Promise<void> {
|
|
48
|
+
if (!container.fleet) throw new Error("Fleet services required for hot pool");
|
|
49
|
+
|
|
50
|
+
const pc = container.productConfig;
|
|
51
|
+
const containerImage = pc.fleet?.containerImage ?? "ghcr.io/wopr-network/platform:latest";
|
|
52
|
+
const containerPort = pc.fleet?.containerPort ?? 3100;
|
|
53
|
+
const provisionSecret = config.provisionSecret;
|
|
54
|
+
const dockerNetwork = pc.fleet?.dockerNetwork ?? "";
|
|
55
|
+
const docker = container.fleet.docker;
|
|
56
|
+
const id = crypto.randomUUID();
|
|
57
|
+
const containerName = `pool-${id.slice(0, 8)}`;
|
|
58
|
+
const volumeName = `pool-${id.slice(0, 8)}`;
|
|
59
|
+
|
|
60
|
+
try {
|
|
61
|
+
// Init volume permissions
|
|
62
|
+
const init = await docker.createContainer({
|
|
63
|
+
Image: containerImage,
|
|
64
|
+
Entrypoint: ["/bin/sh", "-c"],
|
|
65
|
+
Cmd: ["chown -R 999:999 /data"],
|
|
66
|
+
User: "root",
|
|
67
|
+
HostConfig: { Binds: [`${volumeName}:/data`] },
|
|
68
|
+
});
|
|
69
|
+
await init.start();
|
|
70
|
+
await init.wait();
|
|
71
|
+
await init.remove();
|
|
72
|
+
|
|
73
|
+
const warmContainer = await docker.createContainer({
|
|
74
|
+
Image: containerImage,
|
|
75
|
+
name: containerName,
|
|
76
|
+
Env: [`PORT=${containerPort}`, `PROVISION_SECRET=${provisionSecret}`, "HOME=/data"],
|
|
77
|
+
HostConfig: {
|
|
78
|
+
Binds: [`${volumeName}:/data`],
|
|
79
|
+
RestartPolicy: { Name: "unless-stopped" },
|
|
80
|
+
},
|
|
81
|
+
});
|
|
82
|
+
|
|
83
|
+
await warmContainer.start();
|
|
84
|
+
|
|
85
|
+
if (dockerNetwork) {
|
|
86
|
+
const network = docker.getNetwork(dockerNetwork);
|
|
87
|
+
await network.connect({ Container: warmContainer.id });
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
await repo.insertWarm(id, warmContainer.id);
|
|
91
|
+
|
|
92
|
+
logger.info(`Hot pool: created warm container ${containerName} (${id})`);
|
|
93
|
+
} catch (err) {
|
|
94
|
+
logger.error("Hot pool: failed to create warm container", {
|
|
95
|
+
error: (err as Error).message,
|
|
96
|
+
});
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
export async function replenishPool(
|
|
101
|
+
container: PlatformContainer,
|
|
102
|
+
repo: IPoolRepository,
|
|
103
|
+
config: HotPoolConfig,
|
|
104
|
+
): Promise<void> {
|
|
105
|
+
const desired = await repo.getPoolSize();
|
|
106
|
+
const current = await repo.warmCount();
|
|
107
|
+
const deficit = desired - current;
|
|
108
|
+
|
|
109
|
+
if (deficit <= 0) return;
|
|
110
|
+
|
|
111
|
+
logger.info(`Hot pool: replenishing ${deficit} container(s) (have ${current}, want ${desired})`);
|
|
112
|
+
|
|
113
|
+
for (let i = 0; i < deficit; i++) {
|
|
114
|
+
await createWarmContainer(container, repo, config);
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
async function cleanupDead(container: PlatformContainer, repo: IPoolRepository): Promise<void> {
|
|
119
|
+
if (!container.fleet) return;
|
|
120
|
+
|
|
121
|
+
const docker = container.fleet.docker;
|
|
122
|
+
const warmInstances = await repo.listWarm();
|
|
123
|
+
|
|
124
|
+
for (const instance of warmInstances) {
|
|
125
|
+
try {
|
|
126
|
+
const c = docker.getContainer(instance.containerId);
|
|
127
|
+
const info = await c.inspect();
|
|
128
|
+
if (!info.State.Running) {
|
|
129
|
+
await repo.markDead(instance.id);
|
|
130
|
+
try {
|
|
131
|
+
await c.remove({ force: true });
|
|
132
|
+
} catch {
|
|
133
|
+
/* already gone */
|
|
134
|
+
}
|
|
135
|
+
logger.warn(`Hot pool: marked dead container ${instance.id}`);
|
|
136
|
+
}
|
|
137
|
+
} catch {
|
|
138
|
+
await repo.markDead(instance.id);
|
|
139
|
+
logger.warn(`Hot pool: marked missing container ${instance.id} as dead`);
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
await repo.deleteDead();
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
// ---------------------------------------------------------------------------
|
|
147
|
+
// Lifecycle
|
|
148
|
+
// ---------------------------------------------------------------------------
|
|
149
|
+
|
|
150
|
+
export async function startHotPool(
|
|
151
|
+
container: PlatformContainer,
|
|
152
|
+
repo: IPoolRepository,
|
|
153
|
+
config: HotPoolConfig,
|
|
154
|
+
): Promise<HotPoolHandles> {
|
|
155
|
+
await cleanupDead(container, repo);
|
|
156
|
+
await replenishPool(container, repo, config);
|
|
157
|
+
|
|
158
|
+
const intervalMs = config.replenishIntervalMs ?? 60_000;
|
|
159
|
+
const replenishTimer = setInterval(async () => {
|
|
160
|
+
try {
|
|
161
|
+
await cleanupDead(container, repo);
|
|
162
|
+
await replenishPool(container, repo, config);
|
|
163
|
+
} catch (err) {
|
|
164
|
+
logger.error("Hot pool tick failed", { error: (err as Error).message });
|
|
165
|
+
}
|
|
166
|
+
}, intervalMs);
|
|
167
|
+
|
|
168
|
+
logger.info("Hot pool manager started");
|
|
169
|
+
|
|
170
|
+
return {
|
|
171
|
+
replenishTimer,
|
|
172
|
+
stop: () => clearInterval(replenishTimer),
|
|
173
|
+
};
|
|
174
|
+
}
|
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Repository for hot pool database operations.
|
|
3
|
+
*
|
|
4
|
+
* Encapsulates all pool_config and pool_instances queries behind
|
|
5
|
+
* a testable interface. No raw pool.query() outside this file.
|
|
6
|
+
*/
|
|
7
|
+
|
|
8
|
+
import type { Pool } from "pg";
|
|
9
|
+
|
|
10
|
+
export interface PoolInstance {
|
|
11
|
+
id: string;
|
|
12
|
+
containerId: string;
|
|
13
|
+
status: string;
|
|
14
|
+
tenantId: string | null;
|
|
15
|
+
name: string | null;
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
export interface IPoolRepository {
|
|
19
|
+
getPoolSize(): Promise<number>;
|
|
20
|
+
setPoolSize(size: number): Promise<void>;
|
|
21
|
+
warmCount(): Promise<number>;
|
|
22
|
+
insertWarm(id: string, containerId: string): Promise<void>;
|
|
23
|
+
listWarm(): Promise<PoolInstance[]>;
|
|
24
|
+
markDead(id: string): Promise<void>;
|
|
25
|
+
deleteDead(): Promise<void>;
|
|
26
|
+
claimWarm(tenantId: string, name: string): Promise<{ id: string; containerId: string } | null>;
|
|
27
|
+
updateInstanceStatus(id: string, status: string): Promise<void>;
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
export class DrizzlePoolRepository implements IPoolRepository {
|
|
31
|
+
constructor(private pool: Pool) {}
|
|
32
|
+
|
|
33
|
+
async getPoolSize(): Promise<number> {
|
|
34
|
+
try {
|
|
35
|
+
const res = await this.pool.query("SELECT pool_size FROM pool_config WHERE id = 1");
|
|
36
|
+
return res.rows[0]?.pool_size ?? 2;
|
|
37
|
+
} catch {
|
|
38
|
+
return 2;
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
async setPoolSize(size: number): Promise<void> {
|
|
43
|
+
await this.pool.query(
|
|
44
|
+
"INSERT INTO pool_config (id, pool_size) VALUES (1, $1) ON CONFLICT (id) DO UPDATE SET pool_size = $1",
|
|
45
|
+
[size],
|
|
46
|
+
);
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
async warmCount(): Promise<number> {
|
|
50
|
+
const res = await this.pool.query("SELECT COUNT(*)::int AS count FROM pool_instances WHERE status = 'warm'");
|
|
51
|
+
return res.rows[0].count;
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
async insertWarm(id: string, containerId: string): Promise<void> {
|
|
55
|
+
await this.pool.query("INSERT INTO pool_instances (id, container_id, status) VALUES ($1, $2, 'warm')", [
|
|
56
|
+
id,
|
|
57
|
+
containerId,
|
|
58
|
+
]);
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
async listWarm(): Promise<PoolInstance[]> {
|
|
62
|
+
const res = await this.pool.query(
|
|
63
|
+
"SELECT id, container_id, status, tenant_id, name FROM pool_instances WHERE status = 'warm'",
|
|
64
|
+
);
|
|
65
|
+
return res.rows.map((r: Record<string, unknown>) => ({
|
|
66
|
+
id: r.id as string,
|
|
67
|
+
containerId: r.container_id as string,
|
|
68
|
+
status: r.status as string,
|
|
69
|
+
tenantId: (r.tenant_id as string) ?? null,
|
|
70
|
+
name: (r.name as string) ?? null,
|
|
71
|
+
}));
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
async markDead(id: string): Promise<void> {
|
|
75
|
+
await this.pool.query("UPDATE pool_instances SET status = 'dead' WHERE id = $1", [id]);
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
async deleteDead(): Promise<void> {
|
|
79
|
+
await this.pool.query("DELETE FROM pool_instances WHERE status = 'dead'");
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
async claimWarm(tenantId: string, name: string): Promise<{ id: string; containerId: string } | null> {
|
|
83
|
+
const res = await this.pool.query(
|
|
84
|
+
`UPDATE pool_instances
|
|
85
|
+
SET status = 'claimed',
|
|
86
|
+
claimed_at = NOW(),
|
|
87
|
+
tenant_id = $1,
|
|
88
|
+
name = $2
|
|
89
|
+
WHERE id = (
|
|
90
|
+
SELECT id FROM pool_instances
|
|
91
|
+
WHERE status = 'warm'
|
|
92
|
+
ORDER BY created_at ASC
|
|
93
|
+
LIMIT 1
|
|
94
|
+
FOR UPDATE SKIP LOCKED
|
|
95
|
+
)
|
|
96
|
+
RETURNING id, container_id`,
|
|
97
|
+
[tenantId, name],
|
|
98
|
+
);
|
|
99
|
+
if (res.rowCount === 0) return null;
|
|
100
|
+
const row = res.rows[0] as { id: string; container_id: string };
|
|
101
|
+
return { id: row.id, containerId: row.container_id };
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
async updateInstanceStatus(id: string, status: string): Promise<void> {
|
|
105
|
+
await this.pool.query("UPDATE pool_instances SET status = $1 WHERE id = $2", [status, id]);
|
|
106
|
+
}
|
|
107
|
+
}
|