@wopr-network/platform-core 1.22.0 → 1.24.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (33) hide show
  1. package/dist/billing/crypto/btc/watcher.d.ts +5 -1
  2. package/dist/billing/crypto/btc/watcher.js +8 -4
  3. package/dist/billing/crypto/cursor-store.d.ts +28 -0
  4. package/dist/billing/crypto/cursor-store.js +43 -0
  5. package/dist/billing/crypto/evm/eth-watcher.d.ts +12 -4
  6. package/dist/billing/crypto/evm/eth-watcher.js +23 -9
  7. package/dist/billing/crypto/evm/watcher.d.ts +6 -0
  8. package/dist/billing/crypto/evm/watcher.js +54 -17
  9. package/dist/billing/crypto/index.d.ts +2 -0
  10. package/dist/billing/crypto/index.js +1 -0
  11. package/dist/db/schema/crypto.d.ts +121 -0
  12. package/dist/db/schema/crypto.js +16 -1
  13. package/dist/fleet/__tests__/rollout-orchestrator.test.d.ts +1 -0
  14. package/dist/fleet/__tests__/rollout-orchestrator.test.js +262 -0
  15. package/dist/fleet/index.d.ts +1 -0
  16. package/dist/fleet/index.js +1 -0
  17. package/dist/fleet/rollout-orchestrator.d.ts +69 -0
  18. package/dist/fleet/rollout-orchestrator.js +204 -0
  19. package/dist/fleet/services.d.ts +6 -0
  20. package/dist/fleet/services.js +22 -0
  21. package/drizzle/migrations/0007_watcher_cursors.sql +12 -0
  22. package/drizzle/migrations/meta/_journal.json +7 -0
  23. package/package.json +1 -1
  24. package/src/billing/crypto/btc/watcher.ts +11 -4
  25. package/src/billing/crypto/cursor-store.ts +61 -0
  26. package/src/billing/crypto/evm/eth-watcher.ts +25 -10
  27. package/src/billing/crypto/evm/watcher.ts +57 -19
  28. package/src/billing/crypto/index.ts +2 -0
  29. package/src/db/schema/crypto.ts +22 -1
  30. package/src/fleet/__tests__/rollout-orchestrator.test.ts +321 -0
  31. package/src/fleet/index.ts +1 -0
  32. package/src/fleet/rollout-orchestrator.ts +262 -0
  33. package/src/fleet/services.ts +28 -0
@@ -0,0 +1,262 @@
1
+ import { beforeEach, describe, expect, it, vi } from "vitest";
2
+ import { RolloutOrchestrator } from "../rollout-orchestrator.js";
3
+ function makeProfile(id, volumeName) {
4
+ return {
5
+ id,
6
+ tenantId: "tenant-1",
7
+ name: `bot-${id}`,
8
+ description: "",
9
+ image: "ghcr.io/wopr-network/paperclip:managed",
10
+ env: {},
11
+ restartPolicy: "unless-stopped",
12
+ releaseChannel: "stable",
13
+ updatePolicy: "nightly",
14
+ volumeName,
15
+ };
16
+ }
17
+ function makeResult(botId, success) {
18
+ return {
19
+ botId,
20
+ success,
21
+ previousImage: "old:latest",
22
+ newImage: "new:latest",
23
+ previousDigest: "sha256:old",
24
+ newDigest: "sha256:new",
25
+ rolledBack: !success,
26
+ error: success ? undefined : "Health check failed",
27
+ };
28
+ }
29
+ function mockUpdater(results) {
30
+ return {
31
+ updateBot: vi.fn(async (botId) => results.get(botId) ?? makeResult(botId, true)),
32
+ };
33
+ }
34
+ function mockSnapshotManager() {
35
+ return {
36
+ snapshot: vi.fn(async (volumeName) => ({
37
+ id: `${volumeName}-snap`,
38
+ volumeName,
39
+ archivePath: `/backup/${volumeName}-snap.tar`,
40
+ createdAt: new Date(),
41
+ sizeBytes: 1024,
42
+ })),
43
+ restore: vi.fn(async () => { }),
44
+ delete: vi.fn(async () => { }),
45
+ };
46
+ }
47
+ function mockStrategy(overrides = {}) {
48
+ return {
49
+ nextBatch: (remaining) => remaining.slice(0, 2),
50
+ pauseDuration: () => 0,
51
+ onBotFailure: () => "skip",
52
+ maxRetries: () => 2,
53
+ healthCheckTimeout: () => 120_000,
54
+ ...overrides,
55
+ };
56
+ }
57
+ describe("RolloutOrchestrator", () => {
58
+ let updater;
59
+ let snapMgr;
60
+ beforeEach(() => {
61
+ vi.clearAllMocks();
62
+ updater = mockUpdater(new Map());
63
+ snapMgr = mockSnapshotManager();
64
+ });
65
+ it("processes all bots in batches", async () => {
66
+ const profiles = [makeProfile("b1", "vol-1"), makeProfile("b2", "vol-2"), makeProfile("b3", "vol-3")];
67
+ const strategy = mockStrategy({ nextBatch: (r) => r.slice(0, 2) });
68
+ const orch = new RolloutOrchestrator({
69
+ updater,
70
+ snapshotManager: snapMgr,
71
+ strategy,
72
+ getUpdatableProfiles: async () => profiles,
73
+ });
74
+ const result = await orch.rollout();
75
+ expect(result.totalBots).toBe(3);
76
+ expect(result.succeeded).toBe(3);
77
+ expect(result.failed).toBe(0);
78
+ expect(result.aborted).toBe(false);
79
+ expect(updater.updateBot).toHaveBeenCalledTimes(3);
80
+ });
81
+ it("snapshots volumes before updating", async () => {
82
+ const profiles = [makeProfile("b1", "my-volume")];
83
+ const orch = new RolloutOrchestrator({
84
+ updater,
85
+ snapshotManager: snapMgr,
86
+ strategy: mockStrategy(),
87
+ getUpdatableProfiles: async () => profiles,
88
+ });
89
+ await orch.rollout();
90
+ expect(snapMgr.snapshot).toHaveBeenCalledWith("my-volume");
91
+ // On success, snapshot is cleaned up
92
+ expect(snapMgr.delete).toHaveBeenCalledWith("my-volume-snap");
93
+ });
94
+ it("skips snapshot for bots without volumes", async () => {
95
+ const profiles = [makeProfile("b1")]; // no volumeName
96
+ const orch = new RolloutOrchestrator({
97
+ updater,
98
+ snapshotManager: snapMgr,
99
+ strategy: mockStrategy(),
100
+ getUpdatableProfiles: async () => profiles,
101
+ });
102
+ await orch.rollout();
103
+ expect(snapMgr.snapshot).not.toHaveBeenCalled();
104
+ expect(updater.updateBot).toHaveBeenCalledWith("b1");
105
+ });
106
+ it("restores volumes on update failure", async () => {
107
+ const failResults = new Map([["b1", makeResult("b1", false)]]);
108
+ updater = mockUpdater(failResults);
109
+ const profiles = [makeProfile("b1", "my-volume")];
110
+ const orch = new RolloutOrchestrator({
111
+ updater,
112
+ snapshotManager: snapMgr,
113
+ strategy: mockStrategy(),
114
+ getUpdatableProfiles: async () => profiles,
115
+ });
116
+ const result = await orch.rollout();
117
+ expect(result.failed).toBe(1);
118
+ expect(result.results[0].volumeRestored).toBe(true);
119
+ expect(snapMgr.restore).toHaveBeenCalledWith("my-volume-snap");
120
+ // Snapshot NOT deleted on failure (restored instead)
121
+ expect(snapMgr.delete).not.toHaveBeenCalled();
122
+ });
123
+ it("aborts rollout when strategy says abort", async () => {
124
+ const failResults = new Map([["b1", makeResult("b1", false)]]);
125
+ updater = mockUpdater(failResults);
126
+ const profiles = [makeProfile("b1", "v1"), makeProfile("b2", "v2"), makeProfile("b3", "v3")];
127
+ const strategy = mockStrategy({
128
+ nextBatch: (r) => r.slice(0, 1),
129
+ onBotFailure: () => "abort",
130
+ });
131
+ const orch = new RolloutOrchestrator({
132
+ updater,
133
+ snapshotManager: snapMgr,
134
+ strategy,
135
+ getUpdatableProfiles: async () => profiles,
136
+ });
137
+ const result = await orch.rollout();
138
+ expect(result.aborted).toBe(true);
139
+ expect(result.succeeded).toBe(0);
140
+ expect(result.failed).toBe(1);
141
+ expect(result.skipped).toBe(2); // b2, b3 never processed
142
+ expect(updater.updateBot).toHaveBeenCalledTimes(1);
143
+ });
144
+ it("returns empty result when no bots to update", async () => {
145
+ const orch = new RolloutOrchestrator({
146
+ updater,
147
+ snapshotManager: snapMgr,
148
+ strategy: mockStrategy(),
149
+ getUpdatableProfiles: async () => [],
150
+ });
151
+ const result = await orch.rollout();
152
+ expect(result.totalBots).toBe(0);
153
+ expect(result.results).toHaveLength(0);
154
+ });
155
+ it("rejects concurrent rollouts", async () => {
156
+ const profiles = [makeProfile("b1")];
157
+ // Make updateBot slow
158
+ updater = {
159
+ updateBot: vi.fn(async (botId) => {
160
+ await new Promise((r) => setTimeout(r, 100));
161
+ return makeResult(botId, true);
162
+ }),
163
+ };
164
+ const orch = new RolloutOrchestrator({
165
+ updater,
166
+ snapshotManager: snapMgr,
167
+ strategy: mockStrategy(),
168
+ getUpdatableProfiles: async () => profiles,
169
+ });
170
+ const [r1, r2] = await Promise.all([orch.rollout(), orch.rollout()]);
171
+ // One succeeds, one is rejected as already running
172
+ const succeeded = [r1, r2].find((r) => r.totalBots > 0);
173
+ const rejected = [r1, r2].find((r) => r.alreadyRunning);
174
+ expect(succeeded).toBeDefined();
175
+ expect(rejected).toBeDefined();
176
+ expect(rejected?.alreadyRunning).toBe(true);
177
+ expect(rejected?.totalBots).toBe(0);
178
+ });
179
+ it("retries failed bots when strategy says retry", async () => {
180
+ let callCount = 0;
181
+ updater = {
182
+ updateBot: vi.fn(async (botId) => {
183
+ callCount++;
184
+ // Fail first attempt, succeed on retry
185
+ if (botId === "b1" && callCount === 1)
186
+ return makeResult("b1", false);
187
+ return makeResult(botId, true);
188
+ }),
189
+ };
190
+ const profiles = [makeProfile("b1")];
191
+ const strategy = mockStrategy({
192
+ nextBatch: (r) => r.slice(0, 1),
193
+ onBotFailure: (_botId, _err, attempt) => (attempt < 2 ? "retry" : "skip"),
194
+ });
195
+ const orch = new RolloutOrchestrator({
196
+ updater,
197
+ snapshotManager: snapMgr,
198
+ strategy,
199
+ getUpdatableProfiles: async () => profiles,
200
+ });
201
+ const result = await orch.rollout();
202
+ // b1 failed once, retried, succeeded
203
+ expect(updater.updateBot).toHaveBeenCalledTimes(2);
204
+ expect(result.succeeded).toBe(1);
205
+ expect(result.failed).toBe(1); // first attempt counted as failed
206
+ });
207
+ it("calls onBotUpdated callback for each bot", async () => {
208
+ const profiles = [makeProfile("b1"), makeProfile("b2")];
209
+ const onBotUpdated = vi.fn();
210
+ const orch = new RolloutOrchestrator({
211
+ updater,
212
+ snapshotManager: snapMgr,
213
+ strategy: mockStrategy(),
214
+ getUpdatableProfiles: async () => profiles,
215
+ onBotUpdated,
216
+ });
217
+ await orch.rollout();
218
+ expect(onBotUpdated).toHaveBeenCalledTimes(2);
219
+ });
220
+ it("calls onRolloutComplete callback", async () => {
221
+ const profiles = [makeProfile("b1")];
222
+ const onRolloutComplete = vi.fn();
223
+ const orch = new RolloutOrchestrator({
224
+ updater,
225
+ snapshotManager: snapMgr,
226
+ strategy: mockStrategy(),
227
+ getUpdatableProfiles: async () => profiles,
228
+ onRolloutComplete,
229
+ });
230
+ await orch.rollout();
231
+ expect(onRolloutComplete).toHaveBeenCalledTimes(1);
232
+ expect(onRolloutComplete).toHaveBeenCalledWith(expect.objectContaining({ totalBots: 1, succeeded: 1, aborted: false }));
233
+ });
234
+ it("continues on snapshot failure (best-effort)", async () => {
235
+ const profiles = [makeProfile("b1", "my-volume")];
236
+ snapMgr.snapshot = vi.fn().mockRejectedValue(new Error("disk full"));
237
+ const orch = new RolloutOrchestrator({
238
+ updater,
239
+ snapshotManager: snapMgr,
240
+ strategy: mockStrategy(),
241
+ getUpdatableProfiles: async () => profiles,
242
+ });
243
+ const result = await orch.rollout();
244
+ // Update still proceeds despite snapshot failure
245
+ expect(result.succeeded).toBe(1);
246
+ expect(updater.updateBot).toHaveBeenCalledWith("b1");
247
+ });
248
+ it("isRolling reflects rollout state", async () => {
249
+ const profiles = [makeProfile("b1")];
250
+ const orch = new RolloutOrchestrator({
251
+ updater,
252
+ snapshotManager: snapMgr,
253
+ strategy: mockStrategy(),
254
+ getUpdatableProfiles: async () => profiles,
255
+ });
256
+ expect(orch.isRolling).toBe(false);
257
+ const promise = orch.rollout();
258
+ // isRolling is true during rollout (may already be done for sync mocks)
259
+ await promise;
260
+ expect(orch.isRolling).toBe(false);
261
+ });
262
+ });
@@ -1,4 +1,5 @@
1
1
  export * from "./repository-types.js";
2
+ export * from "./rollout-orchestrator.js";
2
3
  export * from "./rollout-strategy.js";
3
4
  export * from "./services.js";
4
5
  export * from "./types.js";
@@ -1,4 +1,5 @@
1
1
  export * from "./repository-types.js";
2
+ export * from "./rollout-orchestrator.js";
2
3
  export * from "./rollout-strategy.js";
3
4
  export * from "./services.js";
4
5
  export * from "./types.js";
@@ -0,0 +1,69 @@
1
+ /**
2
+ * RolloutOrchestrator — coordinates fleet-wide container updates using
3
+ * pluggable rollout strategies and volume snapshots for nuclear rollback.
4
+ *
5
+ * Sits between ImagePoller (detects new digests) and ContainerUpdater
6
+ * (handles per-bot pull/stop/recreate/health). Adds:
7
+ * - Strategy-driven batching (rolling wave, single bot, immediate)
8
+ * - Pre-update volume snapshots via VolumeSnapshotManager
9
+ * - Volume restore on health check failure (nuclear rollback)
10
+ * - Per-tenant update orchestration
11
+ */
12
+ import type { IRolloutStrategy } from "./rollout-strategy.js";
13
+ import type { BotProfile } from "./types.js";
14
+ import type { ContainerUpdater, UpdateResult } from "./updater.js";
15
+ import type { VolumeSnapshotManager } from "./volume-snapshot-manager.js";
16
+ export interface RolloutOrchestratorDeps {
17
+ updater: ContainerUpdater;
18
+ snapshotManager: VolumeSnapshotManager;
19
+ strategy: IRolloutStrategy;
20
+ /** Resolve running profiles that need updating for a given image digest */
21
+ getUpdatableProfiles: () => Promise<BotProfile[]>;
22
+ /** Optional callback after each bot update (success or failure) */
23
+ onBotUpdated?: (result: UpdateResult & {
24
+ volumeRestored: boolean;
25
+ }) => void;
26
+ /** Optional callback when a rollout completes */
27
+ onRolloutComplete?: (results: RolloutResult) => void;
28
+ }
29
+ export interface BotUpdateResult extends UpdateResult {
30
+ volumeRestored: boolean;
31
+ }
32
+ export interface RolloutResult {
33
+ totalBots: number;
34
+ succeeded: number;
35
+ failed: number;
36
+ skipped: number;
37
+ aborted: boolean;
38
+ /** True when a concurrent rollout was already in progress */
39
+ alreadyRunning: boolean;
40
+ results: BotUpdateResult[];
41
+ }
42
+ export declare class RolloutOrchestrator {
43
+ private readonly updater;
44
+ private readonly snapshotManager;
45
+ private readonly strategy;
46
+ private readonly getUpdatableProfiles;
47
+ private readonly onBotUpdated?;
48
+ private readonly onRolloutComplete?;
49
+ private rolling;
50
+ constructor(deps: RolloutOrchestratorDeps);
51
+ /** Whether a rollout is currently in progress. */
52
+ get isRolling(): boolean;
53
+ /**
54
+ * Execute a rollout across all updatable bots.
55
+ * Uses the configured strategy for batching, pausing, and failure handling.
56
+ */
57
+ rollout(): Promise<RolloutResult>;
58
+ /**
59
+ * Update a single bot with volume snapshot + nuclear rollback.
60
+ */
61
+ private updateBot;
62
+ /**
63
+ * Handle a bot failure using the strategy's failure policy.
64
+ * Retries the update up to maxRetries before escalating.
65
+ */
66
+ private handleFailure;
67
+ private restoreVolumes;
68
+ private cleanupSnapshots;
69
+ }
@@ -0,0 +1,204 @@
1
+ /**
2
+ * RolloutOrchestrator — coordinates fleet-wide container updates using
3
+ * pluggable rollout strategies and volume snapshots for nuclear rollback.
4
+ *
5
+ * Sits between ImagePoller (detects new digests) and ContainerUpdater
6
+ * (handles per-bot pull/stop/recreate/health). Adds:
7
+ * - Strategy-driven batching (rolling wave, single bot, immediate)
8
+ * - Pre-update volume snapshots via VolumeSnapshotManager
9
+ * - Volume restore on health check failure (nuclear rollback)
10
+ * - Per-tenant update orchestration
11
+ */
12
+ import { logger } from "../config/logger.js";
13
+ export class RolloutOrchestrator {
14
+ updater;
15
+ snapshotManager;
16
+ strategy;
17
+ getUpdatableProfiles;
18
+ onBotUpdated;
19
+ onRolloutComplete;
20
+ rolling = false;
21
+ constructor(deps) {
22
+ this.updater = deps.updater;
23
+ this.snapshotManager = deps.snapshotManager;
24
+ this.strategy = deps.strategy;
25
+ this.getUpdatableProfiles = deps.getUpdatableProfiles;
26
+ this.onBotUpdated = deps.onBotUpdated;
27
+ this.onRolloutComplete = deps.onRolloutComplete;
28
+ }
29
+ /** Whether a rollout is currently in progress. */
30
+ get isRolling() {
31
+ return this.rolling;
32
+ }
33
+ /**
34
+ * Execute a rollout across all updatable bots.
35
+ * Uses the configured strategy for batching, pausing, and failure handling.
36
+ */
37
+ async rollout() {
38
+ if (this.rolling) {
39
+ logger.warn("Rollout already in progress — skipping");
40
+ return { totalBots: 0, succeeded: 0, failed: 0, skipped: 0, aborted: false, alreadyRunning: true, results: [] };
41
+ }
42
+ this.rolling = true;
43
+ const allResults = [];
44
+ let aborted = false;
45
+ try {
46
+ let remaining = await this.getUpdatableProfiles();
47
+ const totalBots = remaining.length;
48
+ if (totalBots === 0) {
49
+ logger.info("Rollout: no bots to update");
50
+ return {
51
+ totalBots: 0,
52
+ succeeded: 0,
53
+ failed: 0,
54
+ skipped: 0,
55
+ aborted: false,
56
+ alreadyRunning: false,
57
+ results: [],
58
+ };
59
+ }
60
+ logger.info(`Rollout starting: ${totalBots} bots to update`);
61
+ while (remaining.length > 0 && !aborted) {
62
+ const batch = this.strategy.nextBatch(remaining);
63
+ if (batch.length === 0)
64
+ break;
65
+ logger.info(`Rollout wave: ${batch.length} bots (${remaining.length} remaining)`);
66
+ // Process batch — each bot sequentially within a wave for safety
67
+ const retryProfiles = [];
68
+ for (const profile of batch) {
69
+ if (aborted)
70
+ break;
71
+ const result = await this.updateBot(profile);
72
+ allResults.push(result);
73
+ this.onBotUpdated?.(result);
74
+ if (!result.success) {
75
+ const action = this.handleFailure(profile.id, result, allResults);
76
+ if (action === "abort") {
77
+ aborted = true;
78
+ logger.warn(`Rollout aborted after bot ${profile.id} failure`);
79
+ }
80
+ else if (action === "retry") {
81
+ retryProfiles.push(profile);
82
+ }
83
+ // "skip" → don't re-add, bot is dropped
84
+ }
85
+ }
86
+ // Remove processed bots from remaining, but re-add retries
87
+ const processedIds = new Set(batch.map((b) => b.id));
88
+ const retryIds = new Set(retryProfiles.map((b) => b.id));
89
+ remaining = [
90
+ ...remaining.filter((b) => !processedIds.has(b.id)),
91
+ ...retryProfiles.filter((b) => retryIds.has(b.id)),
92
+ ];
93
+ // Pause between waves (unless aborted or done)
94
+ if (remaining.length > 0 && !aborted) {
95
+ const pause = this.strategy.pauseDuration();
96
+ if (pause > 0) {
97
+ logger.info(`Rollout: pausing ${pause}ms before next wave`);
98
+ await sleep(pause);
99
+ }
100
+ }
101
+ }
102
+ const succeeded = allResults.filter((r) => r.success).length;
103
+ const failed = allResults.filter((r) => !r.success).length;
104
+ const skipped = totalBots - allResults.length;
105
+ const rolloutResult = {
106
+ totalBots,
107
+ succeeded,
108
+ failed,
109
+ skipped,
110
+ aborted,
111
+ alreadyRunning: false,
112
+ results: allResults,
113
+ };
114
+ logger.info(`Rollout complete: ${succeeded} succeeded, ${failed} failed, ${skipped} skipped, aborted=${aborted}`);
115
+ this.onRolloutComplete?.(rolloutResult);
116
+ return rolloutResult;
117
+ }
118
+ finally {
119
+ this.rolling = false;
120
+ }
121
+ }
122
+ /**
123
+ * Update a single bot with volume snapshot + nuclear rollback.
124
+ */
125
+ async updateBot(profile) {
126
+ const snapshotIds = [];
127
+ try {
128
+ // Step 1: Snapshot volumes before update
129
+ if (profile.volumeName) {
130
+ try {
131
+ const snap = await this.snapshotManager.snapshot(profile.volumeName);
132
+ snapshotIds.push(snap.id);
133
+ logger.info(`Pre-update snapshot for ${profile.id}: ${snap.id}`);
134
+ }
135
+ catch (err) {
136
+ logger.warn(`Volume snapshot failed for ${profile.id} — proceeding without backup`, { err });
137
+ }
138
+ }
139
+ // Step 2: Delegate to ContainerUpdater
140
+ const result = await this.updater.updateBot(profile.id);
141
+ if (result.success) {
142
+ // Clean up snapshots on success
143
+ await this.cleanupSnapshots(snapshotIds);
144
+ return { ...result, volumeRestored: false };
145
+ }
146
+ // Step 3: Nuclear rollback — restore volumes if update failed
147
+ const volumeRestored = await this.restoreVolumes(profile.id, snapshotIds);
148
+ return { ...result, volumeRestored };
149
+ }
150
+ catch (err) {
151
+ logger.error(`Unexpected error updating bot ${profile.id}`, { err });
152
+ // Attempt volume restore on unexpected errors too
153
+ const volumeRestored = await this.restoreVolumes(profile.id, snapshotIds);
154
+ return {
155
+ botId: profile.id,
156
+ success: false,
157
+ previousImage: profile.image,
158
+ newImage: profile.image,
159
+ previousDigest: null,
160
+ newDigest: null,
161
+ rolledBack: false,
162
+ volumeRestored,
163
+ error: err instanceof Error ? err.message : String(err),
164
+ };
165
+ }
166
+ }
167
+ /**
168
+ * Handle a bot failure using the strategy's failure policy.
169
+ * Retries the update up to maxRetries before escalating.
170
+ */
171
+ handleFailure(botId, result, allResults) {
172
+ const error = new Error(result.error ?? "Unknown error");
173
+ const failCount = allResults.filter((r) => r.botId === botId && !r.success).length;
174
+ return this.strategy.onBotFailure(botId, error, failCount);
175
+ }
176
+ async restoreVolumes(botId, snapshotIds) {
177
+ if (snapshotIds.length === 0)
178
+ return false;
179
+ for (const id of snapshotIds) {
180
+ try {
181
+ await this.snapshotManager.restore(id);
182
+ logger.info(`Volume restored for ${botId} from snapshot ${id}`);
183
+ return true;
184
+ }
185
+ catch (err) {
186
+ logger.error(`Volume restore failed for ${botId} snapshot ${id}`, { err });
187
+ }
188
+ }
189
+ return false;
190
+ }
191
+ async cleanupSnapshots(snapshotIds) {
192
+ for (const id of snapshotIds) {
193
+ try {
194
+ await this.snapshotManager.delete(id);
195
+ }
196
+ catch (err) {
197
+ logger.warn(`Failed to clean up snapshot ${id}`, { err });
198
+ }
199
+ }
200
+ }
201
+ }
202
+ function sleep(ms) {
203
+ return new Promise((resolve) => setTimeout(resolve, ms));
204
+ }
@@ -14,6 +14,8 @@ import type { IAffiliateRepository } from "../monetization/affiliate/drizzle-aff
14
14
  import type { IBotBilling } from "../monetization/credits/bot-billing.js";
15
15
  import type { IPhoneNumberRepository } from "../monetization/credits/drizzle-phone-number-repository.js";
16
16
  import { SystemResourceMonitor } from "../observability/system-resources.js";
17
+ import type { RolloutOrchestrator } from "./rollout-orchestrator.js";
18
+ import type { VolumeSnapshotManager } from "./volume-snapshot-manager.js";
17
19
  import { AdminNotifier } from "./admin-notifier.js";
18
20
  import type { IBotInstanceRepository } from "./bot-instance-repository.js";
19
21
  import type { IBotProfileRepository } from "./bot-profile-repository.js";
@@ -79,6 +81,10 @@ export declare function getCapacityPolicy(configOverrides?: Partial<CapacityPoli
79
81
  export declare function getRestoreLogStore(): IRestoreLogStore;
80
82
  export declare function getBackupStatusStore(): IBackupStatusStore;
81
83
  export declare function getSnapshotManager(): SnapshotManager;
84
+ export declare function getVolumeSnapshotManager(): VolumeSnapshotManager;
85
+ export declare function setVolumeSnapshotManager(mgr: VolumeSnapshotManager): void;
86
+ export declare function getRolloutOrchestrator(): RolloutOrchestrator;
87
+ export declare function setRolloutOrchestrator(orch: RolloutOrchestrator): void;
82
88
  export declare function getRestoreService(): RestoreService;
83
89
  /** Call once at server startup to wire up fleet services. */
84
90
  export declare function initFleet(): void;
@@ -105,6 +105,8 @@ let _restoreLogStore = null;
105
105
  let _restoreService = null;
106
106
  let _backupStatusStore = null;
107
107
  let _snapshotManager = null;
108
+ let _volumeSnapshotManager = null;
109
+ let _rolloutOrchestrator = null;
108
110
  const S3_BUCKET = process.env.S3_BUCKET || "wopr-backups";
109
111
  function envInt(key, fallback) {
110
112
  const raw = process.env[key];
@@ -427,6 +429,24 @@ export function getSnapshotManager() {
427
429
  }
428
430
  return _snapshotManager;
429
431
  }
432
+ export function getVolumeSnapshotManager() {
433
+ if (!_volumeSnapshotManager) {
434
+ throw new Error("VolumeSnapshotManager not initialized — call setVolumeSnapshotManager() first");
435
+ }
436
+ return _volumeSnapshotManager;
437
+ }
438
+ export function setVolumeSnapshotManager(mgr) {
439
+ _volumeSnapshotManager = mgr;
440
+ }
441
+ export function getRolloutOrchestrator() {
442
+ if (!_rolloutOrchestrator) {
443
+ throw new Error("RolloutOrchestrator not initialized — call setRolloutOrchestrator() first");
444
+ }
445
+ return _rolloutOrchestrator;
446
+ }
447
+ export function setRolloutOrchestrator(orch) {
448
+ _rolloutOrchestrator = orch;
449
+ }
430
450
  export function getRestoreService() {
431
451
  if (!_restoreService) {
432
452
  _restoreService = new RestoreService({
@@ -683,6 +703,8 @@ export function _resetForTest() {
683
703
  _restoreService = null;
684
704
  _backupStatusStore = null;
685
705
  _snapshotManager = null;
706
+ _volumeSnapshotManager = null;
707
+ _rolloutOrchestrator = null;
686
708
  _botBilling = null;
687
709
  _phoneNumberRepo = null;
688
710
  _affiliateRepo = null;
@@ -0,0 +1,12 @@
1
+ CREATE TABLE IF NOT EXISTS "watcher_cursors" (
2
+ "watcher_id" text PRIMARY KEY NOT NULL,
3
+ "cursor_block" integer NOT NULL,
4
+ "updated_at" text DEFAULT (now()) NOT NULL
5
+ );
6
+ --> statement-breakpoint
7
+ CREATE TABLE IF NOT EXISTS "watcher_processed" (
8
+ "watcher_id" text NOT NULL,
9
+ "tx_id" text NOT NULL,
10
+ "processed_at" text DEFAULT (now()) NOT NULL,
11
+ CONSTRAINT "watcher_processed_watcher_id_tx_id_pk" PRIMARY KEY ("watcher_id", "tx_id")
12
+ );
@@ -50,6 +50,13 @@
50
50
  "when": 1742140800000,
51
51
  "tag": "0006_invite_acceptance",
52
52
  "breakpoints": true
53
+ },
54
+ {
55
+ "idx": 7,
56
+ "version": "7",
57
+ "when": 1742227200000,
58
+ "tag": "0007_watcher_cursors",
59
+ "breakpoints": true
53
60
  }
54
61
  ]
55
62
  }
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@wopr-network/platform-core",
3
- "version": "1.22.0",
3
+ "version": "1.24.0",
4
4
  "type": "module",
5
5
  "main": "dist/index.js",
6
6
  "types": "dist/index.d.ts",