ovtools 0.0.0-stage → 1.0.0

package/dist/orchestrator/deploy.js

@@ -0,0 +1,313 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.DeployOrchestrator = void 0;
+exports.createDeployOrchestrator = createDeployOrchestrator;
+const base_1 = require("./base");
+const state_1 = require("../utils/state");
+const scripts_1 = require("../utils/scripts");
+const dashboard_1 = require("../utils/dashboard");
+const constants_1 = require("../constants");
+class DeployOrchestrator extends base_1.BaseOrchestrator {
+    config;
+    statePair;
+    managers = null;
+    awaitingEnroll = null;
+    softStop = null;
+    dashboardUrl = null;
+    constructor(config, logger) {
+        super(new state_1.StateManager(constants_1.STATE_FILES.deploy), logger);
+        this.config = config;
+        this.statePair = config.ip;
+        this.program = 'deploy';
+    }
+    async run() {
+        const sshConfig = {
+            host: this.config.ip,
+            port: 22,
+            username: 'root',
+            auth: this.config.auth,
+        };
+        try {
+            this.client = await this.connectSSH(sshConfig, `target ${this.config.ip}`);
+            this.managers = (0, scripts_1.createScriptManager)(this.client);
+            if (this.config.reset) {
+                await this.stateManager.reset(this.statePair);
+                this.log('resume state cleared', 'info');
+            }
+            const phase = this.config.phase || 'deploy';
+            if (phase === 'status') {
+                const st = await this.remoteStatus(this.client, 'deploy');
+                this.log(`target ${this.config.ip}: ${st.state} ${st.detail}`, 'info');
+                return;
+            }
+            await this.preflight(this.client);
+            await this.pushScripts(this.client, ['deploy']);
+            await this.ensureHostTools(this.client, undefined, 'deploy host tools');
+            await this.harden(this.client);
+            if (phase === 'deploy' || phase === 'all') {
+                await this.runAll();
+            }
+            else {
+                await this.runSingle(phase);
+            }
+        }
+        catch (err) {
+            const msg = err instanceof Error ? err.message : String(err);
+            this.log(`Deploy failed: ${msg}`, 'error');
+            throw err;
+        }
+        finally {
+            await this.disconnectSSH(this.client);
+        }
+    }
+    async runAll() {
+        await this.ensure('prep', () => this.stepPrep());
+        await this.ensure('toolchain', () => this.stepToolchain());
+        await this.ensure('setup', () => this.stepSetup());
+        await this.ensure('start', () => this.stepStart());
+        await this.ensure('verify', () => this.stepVerify());
+        if (this.config.role === 'validator' && this.config.bond > 0) {
+            await this.ensure('enroll', () => this.stepEnroll());
+            if (!this.awaitingEnroll) {
+                await this.ensure('promote', () => this.stepPromote());
+            }
+            else {
+                this.log('enrollment is waiting for funding - promote starts after the wallet is funded', 'warn');
+            }
+        }
+        else if (this.config.role === 'validator') {
+            const address = await this.walletAddress(this.client);
+            this.log(`no bond given: the node stays an observer - fund ${address} and enroll when ready`, 'warn');
+            this.awaitingEnroll = { address, bond: 1_000_000 };
+        }
+        if (this.config.domain && !this.config.skipDashboard) {
+            await this.ensure('dashboard', () => this.stepDashboard());
+        }
+        this.log('=== DEPLOY COMPLETE: node ready on ' + this.config.ip + ' ===', 'ok');
+        if (!this.config.bond || this.config.role !== 'validator') {
+            const address = await this.walletAddress(this.client);
+            this.log(`node address: ${address}`, 'info');
+        }
+    }
+    async runSingle(phase) {
+        switch (phase) {
+            case 'prep':
+                await this.stepPrep();
+                break;
+            case 'toolchain':
+                await this.stepToolchain();
+                break;
+            case 'setup':
+                await this.stepSetup();
+                break;
+            case 'start':
+                await this.stepStart();
+                break;
+            case 'verify':
+                await this.stepVerify();
+                break;
+            case 'enroll':
+                await this.stepEnroll();
+                break;
+            case 'promote':
+                await this.stepPromote();
+                break;
+            case 'dashboard':
+                await this.stepDashboard();
+                break;
+            default: throw new Error(`unknown deploy phase: ${phase}`);
+        }
+    }
+    async ensure(name, fn) {
+        if (this.aborted)
+            throw new Error('operation aborted');
+        if (await this.stateManager.isStepDone(this.statePair, name)) {
+            this.log(`${name} already completed - skipping`, 'info');
+            return;
+        }
+        await fn();
+        await this.stateManager.markStepDone(this.statePair, name);
+    }
+    async runStep(options) {
+        this.startPhase(options.phase);
+        this.log(`--> [${this.config.ip}] ${options.label} (detached unit ${options.unit})`, 'step', this.config.ip);
+        try {
+            const result = await this.managers.waitForScript({
+                program: 'deploy',
+                script: options.script,
+                unit: options.unit,
+                env: options.env,
+                timeoutSec: options.timeoutSec,
+                onLine: line => this.log(`    ${line}`, 'remote', this.config.ip),
+            });
+            if (result.state === 'DONE') {
+                this.log(`${options.label} finished${result.detail ? ` (${result.detail})` : ''}`, 'ok', this.config.ip);
+                this.completePhase(true);
+                return result;
+            }
+            if (result.state.startsWith('WAITING-')) {
+                this.log(`${options.label} stopped: ${result.state} ${result.detail}`, 'warn', this.config.ip);
+                this.completePhase(true);
+                this.softStop = { state: result.state, detail: result.detail };
+                return result;
+            }
+            this.log(`${options.label} failed: ${result.detail}`, 'error', this.config.ip);
+            this.completePhase(false, result.detail);
+            throw new Error(`${options.label} failed: ${result.detail}`);
+        }
+        catch (err) {
+            this.completePhase(false, err.message);
+            throw err;
+        }
+    }
+    async stepPrep() {
+        // Deploy is for a fresh node: an existing wallet means migrate instead.
+        const pre = await this.getPrerunState(this.client, '/root/deploy/prerun.env');
+        const walletProbe = await this.client.exec('test -f /opt/octra/libv_litecore/.keys/validator/wallet.json && echo WALLET=yes || echo WALLET=no', { timeout: 60000 });
+        const hasWallet = walletProbe.stdout.includes('WALLET=yes');
+        if (hasWallet && !this.config.wipeVolume) {
+            throw new Error('this host already has a node identity (wallet.json). Deploy is for a FRESH node - use Migrate, or pass wipeVolume to start over');
+        }
+        if (pre.dataSet && !hasWallet && !this.config.keepData) {
+            this.log('chain data found on the data volume - keeping it: the node opens it instead of syncing 38 GB', 'warn');
+        }
+        const wipe = this.config.keepData ? '' : 'WIPE_VOLUME=1';
+        if (!this.config.keepData) {
+            this.log('the data volume will be wiped and formatted (fresh node) - keepData reuses an existing chain', 'warn');
+        }
+        const result = await this.runStep({
+            script: '/root/deploy/10-prepare-host.sh',
+            unit: 'octra-deploy-prepare-host',
+            label: 'prepare host (swap + data store + tooling)',
+            env: {
+                ...(wipe ? { WIPE_VOLUME: '1' } : {}),
+                SWAP_GB: String(this.config.swapGB),
+                BLOCK_DEVICE: this.config.dataDevice,
+                DATA_MIN_GB: String(this.config.dataMinGB),
+            },
+            timeoutSec: 1800,
+            phase: 'prep',
+        });
+        if (result.soft)
+            return;
+        const after = await this.getPrerunState(this.client, '/root/deploy/prerun.env');
+        if (after.dev) {
+            this.log(`data store: ${after.dev} -> ${after.mount} (${after.action}${after.dataSet ? ', data set kept' : ''})`, 'ok');
+        }
+    }
+    async stepToolchain() {
+        const result = await this.runStep({
+            script: '/root/deploy/20-install-toolchain.sh',
+            unit: 'octra-deploy-install-toolchain',
+            label: 'build the runtime (apt + OCaml/dune)',
+            timeoutSec: 7200,
+            phase: 'toolchain',
+        });
+        if (result.soft)
+            throw new Error(`the OCaml toolchain is not usable yet: ${result.detail}`);
+    }
+    async stepSetup() {
+        const result = await this.runStep({
+            script: '/root/deploy/30-setup-node.sh',
+            unit: 'octra-deploy-setup-node',
+            label: 'configure the node (identity + build + sync)',
+            env: this.config.nodeName ? { NODE_NAME: this.config.nodeName } : {},
+            timeoutSec: 7200,
+            phase: 'setup',
+        });
+        if (result.soft)
+            throw new Error(`node setup cannot run yet: ${result.detail}`);
+    }
+    async stepStart() {
+        await this.runStep({
+            script: '/root/deploy/40-start-node.sh',
+            unit: 'octra-deploy-start-node',
+            label: 'start the node (wait for rpc = ready)',
+            timeoutSec: 2400,
+            phase: 'start',
+        });
+    }
+    async stepVerify() {
+        const samples = this.config.samples || constants_1.VERIFY_SAMPLES;
+        const interval = this.config.interval || constants_1.VERIFY_INTERVAL;
+        const peerWait = this.config.peerWaitSec || 300;
+        const voteWait = this.config.voteWaitSec || 900;
+        // A validator without a bond is an observer until the enroll popup runs.
+        // Verifying it as a validator would sit in the fold wait for 15 minutes.
+        const role = this.config.role === 'validator' && this.config.bond <= 0 ? 'observer' : this.config.role;
+        const waitBudget = role === 'validator' ? voteWait : peerWait;
+        const ceiling = 600 + waitBudget + Math.max(1, samples) * Math.max(5, interval);
+        await this.runStep({
+            script: '/root/deploy/50-verify.sh',
+            unit: 'octra-deploy-verify',
+            label: `verify (role=${role})`,
+            env: {
+                SAMPLES: String(samples),
+                INTERVAL: String(interval),
+                ROLE: role,
+                PEER_WAIT: String(peerWait),
+                VOTE_WAIT: String(voteWait),
+            },
+            timeoutSec: ceiling,
+            phase: 'verify',
+        });
+    }
+    async stepEnroll() {
+        if (this.config.bond <= 0) {
+            const address = await this.walletAddress(this.client);
+            this.log(`no bond given: leaving the node as an observer (fund ${address})`, 'info');
+            this.awaitingEnroll = { address, bond: 1_000_000 };
+            return;
+        }
+        const result = await this.runStep({
+            script: '/root/deploy/60-enroll.sh',
+            unit: 'octra-deploy-enroll',
+            label: `enroll with bond ${this.config.bond}`,
+            env: { BOND_AMOUNT: String(this.config.bond) },
+            timeoutSec: 1200,
+            phase: 'enroll',
+        });
+        if (result.soft) {
+            if (result.state === 'WAITING-FUNDING') {
+                this.log(`enrollment needs funding: ${result.detail}`, 'warn');
+                this.awaitingEnroll = { address: result.detail, bond: this.config.bond };
+            }
+            else {
+                this.log(`enrollment is waiting for the chain: ${result.state} ${result.detail}`, 'warn');
+            }
+        }
+    }
+    async stepPromote() {
+        const result = await this.runStep({
+            script: '/root/deploy/70-promote.sh',
+            unit: 'octra-deploy-promote',
+            label: 'wait for the fold, then switch to validator',
+            timeoutSec: 5100,
+            phase: 'promote',
+        });
+        if (result.soft)
+            this.log(`waiting for the set fold: ${result.state} ${result.detail}`, 'warn');
+    }
+    async stepDashboard() {
+        this.startPhase('dashboard');
+        try {
+            const result = await (0, dashboard_1.deployDashboard)(this.client, this.logger, {
+                ip: this.config.ip,
+                domain: this.config.domain,
+                sslMode: 'auto',
+                reset: false,
+            }, 'deploy');
+            this.dashboardUrl = result.url;
+            this.completePhase(true);
+        }
+        catch (err) {
+            this.completePhase(false, err.message);
+            throw err;
+        }
+    }
+}
+exports.DeployOrchestrator = DeployOrchestrator;
+function createDeployOrchestrator(config, logger) {
+    return new DeployOrchestrator(config, logger);
+}
+//# sourceMappingURL=deploy.js.map

package/dist/orchestrator/deploy.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"deploy.js","sourceRoot":"","sources":["../../src/orchestrator/deploy.ts"],"names":[],"mappings":";;;AAqTA,4DAEC;AAvTD,iCAA0C;AAC1C,0CAA8C;AAE9C,8CAAsE;AACtE,kDAAqD;AAErD,4CAA4E;AAE5E,MAAa,kBAAmB,SAAQ,uBAAgB;IAC9C,MAAM,CAAe;IACrB,SAAS,CAAS;IAClB,QAAQ,GAAyB,IAAI,CAAC;IAC9C,cAAc,GAA6C,IAAI,CAAC;IAChE,QAAQ,GAA6C,IAAI,CAAC;IAC1D,YAAY,GAAkB,IAAI,CAAC;IAEnC,YAAY,MAAoB,EAAE,MAAc;QAC9C,KAAK,CAAC,IAAI,oBAAY,CAAC,uBAAW,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC,CAAC;QACpD,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,SAAS,GAAG,MAAM,CAAC,EAAE,CAAC;QAC3B,IAAI,CAAC,OAAO,GAAG,QAAQ,CAAC;IAC1B,CAAC;IAED,KAAK,CAAC,GAAG;QACP,MAAM,SAAS,GAAc;YAC3B,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC,EAAE;YACpB,IAAI,EAAE,EAAE;YACR,QAAQ,EAAE,MAAM;YAChB,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC,IAAI;SACvB,CAAC;QACF,IAAI,CAAC;YACH,IAAI,CAAC,MAAM,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,SAAS,EAAE,UAAU,IAAI,CAAC,MAAM,CAAC,EAAE,EAAE,CAAC,CAAC;YAC3E,IAAI,CAAC,QAAQ,GAAG,IAAA,6BAAmB,EAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YAEjD,IAAI,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;gBACtB,MAAM,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;gBAC9C,IAAI,CAAC,GAAG,CAAC,sBAAsB,EAAE,MAAM,CAAC,CAAC;YAC3C,CAAC;YAED,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,IAAI,QAAQ,CAAC;YAC5C,IAAI,KAAK,KAAK,QAAQ,EAAE,CAAC;gBACvB,MAAM,EAAE,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;gBAC1D,IAAI,CAAC,GAAG,CAAC,UAAU,IAAI,CAAC,MAAM,CAAC,EAAE,KAAK,EAAE,CAAC,KAAK,IAAI,EAAE,CAAC,MAAM,EAAE,EAAE,MAAM,CAAC,CAAC;gBACvE,OAAO;YACT,CAAC;YAED,MAAM,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YAClC,MAAM,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,CAAC,CAAC;YAChD,MAAM,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,MAAM,EAAE,SAAS,EAAE,mBAAmB,CAAC,CAAC;YACxE,MAAM,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YAE/B,IAAI,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,KAAK,EAAE,CAAC;gBAC1C,MAAM,IAAI,CAAC,MAAM,EAAE,CAAC;YACtB,CAAC;iBAAM,CAAC;gBACN,MAAM,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;YAC9B,CAAC;QACH,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,GAAG,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YAC7D,IAAI,CAAC,GAAG,CAAC,kBAAkB,GAAG,EAAE,EAAE,OAAO,CAAC,CAAC;YAC3C,MAAM,GAAG,CAAC;QACZ,CAAC;gBAAS,CAAC;YACT,MAAM,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACxC,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,MAAM;QAClB,MAAM,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC;QACjD,MAAM,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,aAAa,EAAE,CAAC,CAAC;QAC3D,MAAM,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC,CAAC;QACnD,MAAM,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC,CAAC;QACnD,MAAM,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,UAAU,EAAE,CAAC,CAAC;QACrD,IAAI,IAAI,CAAC,MAAM,CAAC,IAAI,KAAK,WAAW,IAAI,IAAI,CAAC,MAAM,CAAC,IAAI,GAAG,CAAC,EAAE,CAAC;YAC7D,MAAM,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,UAAU,EAAE,CAAC,CAAC;YACrD,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,CAAC;gBACzB,MAAM,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,CAAC;YACzD,CAAC;iBAAM,CAAC;gBACN,IAAI,CAAC,GAAG,CAAC,+EAA+E,EAAE,MAAM,CAAC,CAAC;YACpG,CAAC;QACH,CAAC;aAAM,IAAI,IAAI,CAAC,MAAM,CAAC,IAAI,KAAK,WAAW,EAAE,CAAC;YAC5C,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,MAAO,CAAC,CAAC;YACvD,IAAI,CAAC,GAAG,CAAC,oDAAoD,OAAO,wBAAwB,EAAE,MAAM,CAAC,CAAC;YACtG,IAAI,CAAC,cAAc,GAAG,EAAE,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,CAAC;QACrD,CAAC;QACD,IAAI,IAAI,CAAC,MAAM,CAAC,MAAM,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,aAAa,EAAE,CAAC;YACrD,MAAM,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,aAAa,EAAE,CAAC,CAAC;QAC7D,CAAC;QACD,IAAI,CAAC,GAAG,CAAC,qCAAqC,GAAG,IAAI,CAAC,MAAM,CAAC,EAAE,GAAG,MAAM,EAAE,IAAI,CAAC,CAAC;QAChF,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,IAAI,IAAI,CAAC,MAAM,CAAC,IAAI,KAAK,WAAW,EAAE,CAAC;YAC1D,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,MAAO,CAAC,CAAC;YACvD,IAAI,CAAC,GAAG,CAAC,iBAAiB,OAAO,EAAE,EAAE,MAAM,CAAC,CAAC;QAC/C,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,SAAS,CAAC,KAAa;QACnC,QAAQ,KAAK,EAAE,CAAC;YACd,KAAK,MAAM;gBAAE,MAAM,IAAI,CAAC,QAAQ,EAAE,CAAC;gBAAC,MAAM;YAC1C,KAAK,WAAW;gBAAE,MAAM,IAAI,CAAC,aAAa,EAAE,CAAC;gBAAC,MAAM;YACpD,KAAK,OAAO;gBAAE,MAAM,IAAI,CAAC,SAAS,EAAE,CAAC;gBAAC,MAAM;YAC5C,KAAK,OAAO;gBAAE,MAAM,IAAI,CAAC,SAAS,EAAE,CAAC;gBAAC,MAAM;YAC5C,KAAK,QAAQ;gBAAE,MAAM,IAAI,CAAC,UAAU,EAAE,CAAC;gBAAC,MAAM;YAC9C,KAAK,QAAQ;gBAAE,MAAM,IAAI,CAAC,UAAU,EAAE,CAAC;gBAAC,MAAM;YAC9C,KAAK,SAAS;gBAAE,MAAM,IAAI,CAAC,WAAW,EAAE,CAAC;gBAAC,MAAM;YAChD,KAAK,WAAW;gBAAE,MAAM,IAAI,CAAC,aAAa,EAAE,CAAC;gBAAC,MAAM;YACpD,OAAO,CAAC,CAAC,MAAM,IAAI,KAAK,CAAC,yBAAyB,KAAK,EAAE,CAAC,CAAC;QAC7D,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,MAAM,CAAC,IAAY,EAAE,EAAuB;QACxD,IAAI,IAAI,CAAC,OAAO;YAAE,MAAM,IAAI,KAAK,CAAC,mBAAmB,CAAC,CAAC;QACvD,IAAI,MAAM,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,EAAE,CAAC;YAC7D,IAAI,CAAC,GAAG,CAAC,GAAG,IAAI,+BAA+B,EAAE,MAAM,CAAC,CAAC;YACzD,OAAO;QACT,CAAC;QACD,MAAM,EAAE,EAAE,CAAC;QACX,MAAM,IAAI,CAAC,YAAY,CAAC,YAAY,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC;IAC7D,CAAC;IAEO,KAAK,CAAC,OAAO,CAAC,OAOrB;QACC,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;QAC/B,IAAI,CAAC,GAAG,CAAC,QAAQ,IAAI,CAAC,MAAM,CAAC,EAAE,KAAK,OAAO,CAAC,KAAK,mBAAmB,OAAO,CAAC,IAAI,GAAG,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;QAC7G,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,QAAS,CAAC,aAAa,CAAC;gBAChD,OAAO,EAAE,QAAQ;gBACjB,MAAM,EAAE,OAAO,CAAC,MAAM;gBACtB,IAAI,EAAE,OAAO,CAAC,IAAI;gBAClB,GAAG,EAAE,OAAO,CAAC,GAAG;gBAChB,UAAU,EAAE,OAAO,CAAC,UAAU;gBAC9B,MAAM,EAAE,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,OAAO,IAAI,EAAE,EAAE,QAAQ,EAAE,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC;aAClE,CAAC,CAAC;YACH,IAAI,MAAM,CAAC,KAAK,KAAK,MAAM,EAAE,CAAC;gBAC5B,IAAI,CAAC,GAAG,CAAC,GAAG,OAAO,CAAC,KAAK,YAAY,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;gBACzG,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC;gBACzB,OAAO,MAAM,CAAC;YAChB,CAAC;YACD,IAAI,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;gBACxC,IAAI,CAAC,GAAG,CAAC,GAAG,OAAO,CAAC,KAAK,aAAa,MAAM,CAAC,KAAK,IAAI,MAAM,CAAC,MAAM,EAAE,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;gBAC/F,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC;gBACzB,IAAI,CAAC,QAAQ,GAAG,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,CAAC;gBAC/D,OAAO,MAAM,CAAC;YAChB,CAAC;YACD,IAAI,CAAC,GAAG,CAAC,GAAG,OAAO,CAAC,KAAK,YAAY,MAAM,CAAC,MAAM,EAAE,EAAE,OAAO,EAAE,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;YAC/E,IAAI,CAAC,aAAa,CAAC,KAAK,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC;YACzC,MAAM,IAAI,KAAK,CAAC,GAAG,OAAO,CAAC,KAAK,YAAY,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC;QAC/D,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,IAAI,CAAC,aAAa,CAAC,KAAK,EAAG,GAAa,CAAC,OAAO,CAAC,CAAC;YAClD,MAAM,GAAG,CAAC;QACZ,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,QAAQ;QACpB,wEAAwE;QACxE,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,MAAO,EAAE,yBAAyB,CAAC,CAAC;QAC/E,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,MAAO,CAAC,IAAI,CAAC,mGAAmG,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,CAAC;QACrK,MAAM,SAAS,GAAG,WAAW,CAAC,MAAM,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC;QAC5D,IAAI,SAAS,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,UAAU,EAAE,CAAC;YACzC,MAAM,IAAI,KAAK,CAAC,iIAAiI,CAAC,CAAC;QACrJ,CAAC;QACD,IAAI,GAAG,CAAC,OAAO,IAAI,CAAC,SAAS,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC;YACvD,IAAI,CAAC,GAAG,CAAC,8FAA8F,EAAE,MAAM,CAAC,CAAC;QACnH,CAAC;QACD,MAAM,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,eAAe,CAAC;QACzD,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC;YAC1B,IAAI,CAAC,GAAG,CAAC,8FAA8F,EAAE,MAAM,CAAC,CAAC;QACnH,CAAC;QACD,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC;YAChC,MAAM,EAAE,iCAAiC;YACzC,IAAI,EAAE,2BAA2B;YACjC,KAAK,EAAE,4CAA4C;YACnD,GAAG,EAAE;gBACH,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBACrC,OAAO,EAAE,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC;gBACnC,YAAY,EAAE,IAAI,CAAC,MAAM,CAAC,UAAU;gBACpC,WAAW,EAAE,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC;aAC3C;YACD,UAAU,EAAE,IAAI;YAChB,KAAK,EAAE,MAAM;SACd,CAAC,CAAC;QACH,IAAI,MAAM,CAAC,IAAI;YAAE,OAAO;QACxB,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,MAAO,EAAE,yBAAyB,CAAC,CAAC;QACjF,IAAI,KAAK,CAAC,GAAG,EAAE,CAAC;YACd,IAAI,CAAC,GAAG,CAAC,eAAe,KAAK,CAAC,GAAG,OAAO,KAAK,CAAC,KAAK,KAAK,KAAK,CAAC,MAAM,GAAG,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,iBAAiB,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,CAAC;QAC1H,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,aAAa;QACzB,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC;YAChC,MAAM,EAAE,sCAAsC;YAC9C,IAAI,EAAE,gCAAgC;YACtC,KAAK,EAAE,sCAAsC;YAC7C,UAAU,EAAE,IAAI;YAChB,KAAK,EAAE,WAAW;SACnB,CAAC,CAAC;QACH,IAAI,MAAM,CAAC,IAAI;YAAE,MAAM,IAAI,KAAK,CAAC,0CAA0C,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC;IAC9F,CAAC;IAEO,KAAK,CAAC,SAAS;QACrB,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC;YAChC,MAAM,EAAE,+BAA+B;YACvC,IAAI,EAAE,yBAAyB;YAC/B,KAAK,EAAE,8CAA8C;YACrD,GAAG,EAAE,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE;YACpE,UAAU,EAAE,IAAI;YAChB,KAAK,EAAE,OAAO;SACf,CAAC,CAAC;QACH,IAAI,MAAM,CAAC,IAAI;YAAE,MAAM,IAAI,KAAK,CAAC,8BAA8B,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC;IAClF,CAAC;IAEO,KAAK,CAAC,SAAS;QACrB,MAAM,IAAI,CAAC,OAAO,CAAC;YACjB,MAAM,EAAE,+BAA+B;YACvC,IAAI,EAAE,yBAAyB;YAC/B,KAAK,EAAE,uCAAuC;YAC9C,UAAU,EAAE,IAAI;YAChB,KAAK,EAAE,OAAO;SACf,CAAC,CAAC;IACL,CAAC;IAEO,KAAK,CAAC,UAAU;QACtB,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,OAAO,IAAI,0BAAc,CAAC;QACtD,MAAM,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,QAAQ,IAAI,2BAAe,CAAC;QACzD,MAAM,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,WAAW,IAAI,GAAG,CAAC;QAChD,MAAM,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,WAAW,IAAI,GAAG,CAAC;QAChD,yEAAyE;QACzE,yEAAyE;QACzE,MAAM,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,KAAK,WAAW,IAAI,IAAI,CAAC,MAAM,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC;QACvG,MAAM,UAAU,GAAG,IAAI,KAAK,WAAW,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC;QAC9D,MAAM,OAAO,GAAG,GAAG,GAAG,UAAU,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC;QAChF,MAAM,IAAI,CAAC,OAAO,CAAC;YACjB,MAAM,EAAE,2BAA2B;YACnC,IAAI,EAAE,qBAAqB;YAC3B,KAAK,EAAE,gBAAgB,IAAI,GAAG;YAC9B,GAAG,EAAE;gBACH,OAAO,EAAE,MAAM,CAAC,OAAO,CAAC;gBACxB,QAAQ,EAAE,MAAM,CAAC,QAAQ,CAAC;gBAC1B,IAAI,EAAE,IAAI;gBACV,SAAS,EAAE,MAAM,CAAC,QAAQ,CAAC;gBAC3B,SAAS,EAAE,MAAM,CAAC,QAAQ,CAAC;aAC5B;YACD,UAAU,EAAE,OAAO;YACnB,KAAK,EAAE,QAAQ;SAChB,CAAC,CAAC;IACL,CAAC;IAEO,KAAK,CAAC,UAAU;QACtB,IAAI,IAAI,CAAC,MAAM,CAAC,IAAI,IAAI,CAAC,EAAE,CAAC;YAC1B,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,MAAO,CAAC,CAAC;YACvD,IAAI,CAAC,GAAG,CAAC,wDAAwD,OAAO,GAAG,EAAE,MAAM,CAAC,CAAC;YACrF,IAAI,CAAC,cAAc,GAAG,EAAE,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,CAAC;YACnD,OAAO;QACT,CAAC;QACD,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC;YAChC,MAAM,EAAE,2BAA2B;YACnC,IAAI,EAAE,qBAAqB;YAC3B,KAAK,EAAE,oBAAoB,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE;YAC7C,GAAG,EAAE,EAAE,WAAW,EAAE,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE;YAC9C,UAAU,EAAE,IAAI;YAChB,KAAK,EAAE,QAAQ;SAChB,CAAC,CAAC;QACH,IAAI,MAAM,CAAC,IAAI,EAAE,CAAC;YAChB,IAAI,MAAM,CAAC,KAAK,KAAK,iBAAiB,EAAE,CAAC;gBACvC,IAAI,CAAC,GAAG,CAAC,6BAA6B,MAAM,CAAC,MAAM,EAAE,EAAE,MAAM,CAAC,CAAC;gBAC/D,IAAI,CAAC,cAAc,GAAG,EAAE,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;YAC3E,CAAC;iBAAM,CAAC;gBACN,IAAI,CAAC,GAAG,CAAC,wCAAwC,MAAM,CAAC,KAAK,IAAI,MAAM,CAAC,MAAM,EAAE,EAAE,MAAM,CAAC,CAAC;YAC5F,CAAC;QACH,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,WAAW;QACvB,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC;YAChC,MAAM,EAAE,4BAA4B;YACpC,IAAI,EAAE,sBAAsB;YAC5B,KAAK,EAAE,6CAA6C;YACpD,UAAU,EAAE,IAAI;YAChB,KAAK,EAAE,SAAS;SACjB,CAAC,CAAC;QACH,IAAI,MAAM,CAAC,IAAI;YAAE,IAAI,CAAC,GAAG,CAAC,6BAA6B,MAAM,CAAC,KAAK,IAAI,MAAM,CAAC,MAAM,EAAE,EAAE,MAAM,CAAC,CAAC;IAClG,CAAC;IAEO,KAAK,CAAC,aAAa;QACzB,IAAI,CAAC,UAAU,CAAC,WAAW,CAAC,CAAC;QAC7B,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,IAAA,2BAAe,EAClC,IAAI,CAAC,MAAO,EACZ,IAAI,CAAC,MAAM,EACX;gBACE,EAAE,EAAE,IAAI,CAAC,MAAM,CAAC,EAAE;gBAClB,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,MAAM;gBAC1B,OAAO,EAAE,MAAM;gBACf,KAAK,EAAE,KAAK;aACb,EACD,QAAQ,CACT,CAAC;YACF,IAAI,CAAC,YAAY,GAAG,MAAM,CAAC,GAAG,CAAC;YAC/B,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC;QAC3B,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,IAAI,CAAC,aAAa,CAAC,KAAK,EAAG,GAAa,CAAC,OAAO,CAAC,CAAC;YAClD,MAAM,GAAG,CAAC;QACZ,CAAC;IACH,CAAC;CACF;AA3SD,gDA2SC;AAED,SAAgB,wBAAwB,CAAC,MAAoB,EAAE,MAAc;IAC3E,OAAO,IAAI,kBAAkB,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;AAChD,CAAC"}

package/dist/orchestrator/manage.d.ts

@@ -0,0 +1,28 @@
+import { BaseOrchestrator } from './base';
+import { Logger } from '../utils/helpers';
+import type { ManageConfig } from '../types';
+export declare class ManageOrchestrator extends BaseOrchestrator {
+    private config;
+    private statePair;
+    private managers;
+    softStop: {
+        state: string;
+        detail: string;
+    } | null;
+    dashboardUrl: string | null;
+    constructor(config: ManageConfig, logger: Logger);
+    run(): Promise<void>;
+    private runStep;
+    private stepStatus;
+    private stepBond;
+    private stepExit;
+    private stepDiagnose;
+    private stepRecover;
+    private stepPreflightUpgrade;
+    private stepUpgrade;
+    private stepManual;
+    private stepStorage;
+    private stepDashboard;
+}
+export declare function createManageOrchestrator(config: ManageConfig, logger: Logger): ManageOrchestrator;
+//# sourceMappingURL=manage.d.ts.map

package/dist/orchestrator/manage.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"manage.d.ts","sourceRoot":"","sources":["../../src/orchestrator/manage.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,gBAAgB,EAAE,MAAM,QAAQ,CAAC;AAE1C,OAAO,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAG1C,OAAO,KAAK,EAAE,YAAY,EAAa,MAAM,UAAU,CAAC;AAGxD,qBAAa,kBAAmB,SAAQ,gBAAgB;IACtD,OAAO,CAAC,MAAM,CAAe;IAC7B,OAAO,CAAC,SAAS,CAAS;IAC1B,OAAO,CAAC,QAAQ,CAA8B;IAC9C,QAAQ,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI,CAAQ;IAC1D,YAAY,EAAE,MAAM,GAAG,IAAI,CAAQ;gBAEvB,MAAM,EAAE,YAAY,EAAE,MAAM,EAAE,MAAM;IAU1C,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC;YA2DZ,OAAO;YAyCP,UAAU;YAWV,QAAQ;YAiBR,QAAQ;YAcR,YAAY;YAWZ,WAAW;YAaX,oBAAoB;YAWpB,WAAW;YAgBX,UAAU;YAYV,WAAW;YAaX,aAAa;CAuB5B;AAED,wBAAgB,wBAAwB,CAAC,MAAM,EAAE,YAAY,EAAE,MAAM,EAAE,MAAM,GAAG,kBAAkB,CAEjG"}

package/dist/orchestrator/manage.js

@@ -0,0 +1,267 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ManageOrchestrator = void 0;
+exports.createManageOrchestrator = createManageOrchestrator;
+const base_1 = require("./base");
+const state_1 = require("../utils/state");
+const scripts_1 = require("../utils/scripts");
+const dashboard_1 = require("../utils/dashboard");
+const constants_1 = require("../constants");
+class ManageOrchestrator extends base_1.BaseOrchestrator {
+    config;
+    statePair;
+    managers = null;
+    softStop = null;
+    dashboardUrl = null;
+    constructor(config, logger) {
+        const stateFile = config.phase === 'upgrade' || config.phase === 'manual' || config.phase === 'storage'
+            ? constants_1.STATE_FILES.upgrade
+            : constants_1.STATE_FILES.manage;
+        super(new state_1.StateManager(stateFile), logger);
+        this.config = config;
+        this.statePair = config.ip;
+        this.program = 'manage';
+    }
+    async run() {
+        const sshConfig = {
+            host: this.config.ip,
+            port: 22,
+            username: 'root',
+            auth: this.config.auth,
+        };
+        try {
+            this.client = await this.connectSSH(sshConfig, `node ${this.config.ip}`);
+            this.managers = (0, scripts_1.createScriptManager)(this.client);
+            if (this.config.reset) {
+                await this.stateManager.reset(this.statePair);
+                this.log('resume state cleared', 'info');
+            }
+            const phase = this.config.phase;
+            const isUpgrade = phase === 'upgrade' || phase === 'manual' || phase === 'storage';
+            await this.preflight(this.client);
+            await this.pushScripts(this.client, isUpgrade ? ['upgrade'] : ['manage']);
+            if (isUpgrade) {
+                await this.ensureHostTools(this.client, ['bash', 'sed', 'grep', 'awk', 'tar', 'curl', 'git', 'rsync', 'jq', 'pgrep', 'nproc', 'make', 'g++', 'opam', 'du'], 'update host tools');
+            }
+            else {
+                await this.ensureHostTools(this.client, undefined, 'manage host tools');
+            }
+            switch (phase) {
+                case 'status':
+                    await this.stepStatus();
+                    break;
+                case 'bond':
+                    await this.stepBond();
+                    break;
+                case 'exit':
+                    await this.stepExit();
+                    break;
+                case 'dashboard':
+                    await this.stepDashboard();
+                    break;
+                case 'diagnose':
+                    await this.stepDiagnose();
+                    break;
+                case 'recover':
+                    await this.stepRecover();
+                    break;
+                case 'upgrade':
+                    await this.stepPreflightUpgrade();
+                    await this.stepUpgrade();
+                    break;
+                case 'manual':
+                    await this.stepManual();
+                    break;
+                case 'storage':
+                    await this.stepStorage();
+                    break;
+                default: throw new Error(`unknown manage phase: ${phase}`);
+            }
+            if (phase !== 'dashboard') {
+                const st = await this.remoteStatus(this.client, isUpgrade ? 'upgrade' : 'manage').catch(() => ({ state: 'UNKNOWN', detail: '' }));
+                this.log(`node ${this.config.ip}: ${st.state} ${st.detail}`, 'info');
+            }
+        }
+        catch (err) {
+            const msg = err instanceof Error ? err.message : String(err);
+            this.log(`Manage ${this.config.phase} failed: ${msg}`, 'error');
+            throw err;
+        }
+        finally {
+            await this.disconnectSSH(this.client);
+        }
+    }
+    async runStep(options) {
+        this.startPhase(options.phase);
+        const dir = options.program === 'upgrade' ? 'upgrade' : 'manage';
+        await this.log(`--> [${this.config.ip}] ${options.label}`, 'step', this.config.ip);
+        try {
+            const result = await this.managers.waitForScript({
+                program: options.program,
+                script: `/root/${dir}/${options.script}`,
+                unit: options.unit,
+                env: options.env,
+                timeoutSec: options.timeoutSec,
+                onLine: line => this.log(`    ${line}`, 'remote', this.config.ip),
+            });
+            if (result.state === 'DONE') {
+                this.log(`${options.label} finished${result.detail ? ` (${result.detail})` : ''}`, 'ok', this.config.ip);
+                this.completePhase(true);
+                return result;
+            }
+            if (result.state.startsWith('WAITING-')) {
+                this.log(`${options.label} stopped: ${result.state} ${result.detail}`, 'warn', this.config.ip);
+                this.softStop = result;
+                this.completePhase(true);
+                return result;
+            }
+            this.log(`${options.label} failed: ${result.detail}`, 'error', this.config.ip);
+            this.completePhase(false, result.detail);
+            throw new Error(`${options.label} failed: ${result.detail}`);
+        }
+        catch (err) {
+            this.completePhase(false, err.message);
+            throw err;
+        }
+    }
+    async stepStatus() {
+        await this.runStep({
+            program: 'manage',
+            script: '30-status.sh',
+            unit: 'octra-manage-status',
+            label: 'read the active node',
+            timeoutSec: 300,
+            phase: 'status',
+        });
+    }
+    async stepBond() {
+        const amount = (this.config.amount || '').trim();
+        if (!/^[0-9]+(\.[0-9]{1,6})?$/.test(amount) || Number(amount) < 1) {
+            throw new Error('type the amount to add in OCT (min 1, e.g. 100 or 12.5)');
+        }
+        const result = await this.runStep({
+            program: 'manage',
+            script: '10-bond-increase.sh',
+            unit: 'octra-manage-bond-increase',
+            label: `increase bond by ${amount} OCT`,
+            env: { BOND_OCT: amount, ...(this.config.newBond ? { BOND_NEW: '1' } : {}) },
+            timeoutSec: 1200,
+            phase: 'bond',
+        });
+        if (result.soft)
+            this.log(`the increase is not confirmed yet (${result.detail})`, 'warn');
+    }
+    async stepExit() {
+        const result = await this.runStep({
+            program: 'manage',
+            script: '20-exit-validator.sh',
+            unit: 'octra-manage-exit-validator',
+            label: 'exit the validator set',
+            timeoutSec: 3600,
+            phase: 'exit',
+        });
+        if (result.soft) {
+            this.log('=== exit accepted: the stake comes back on its own (unlock epoch above) ===', 'ok');
+        }
+    }
+    async stepDiagnose() {
+        await this.runStep({
+            program: 'manage',
+            script: '10-diagnose-node.sh',
+            unit: 'octra-manage-diagnose-node',
+            label: 'diagnose the node',
+            timeoutSec: 600,
+            phase: 'diagnose',
+        });
+    }
+    async stepRecover() {
+        const result = await this.runStep({
+            program: 'manage',
+            script: '25-recover-node.sh',
+            unit: 'octra-manage-recover-node',
+            label: 'repair the stuck node',
+            env: { ALLOW_RESTART: this.config.noRestart ? '0' : '1' },
+            timeoutSec: 2700,
+            phase: 'recover',
+        });
+        if (result.soft)
+            this.log(`recovery is waiting: ${result.state} ${result.detail}`, 'warn');
+    }
+    async stepPreflightUpgrade() {
+        await this.runStep({
+            program: 'upgrade',
+            script: '10-preflight.sh',
+            unit: 'octra-upgrade-preflight',
+            label: 'preflight (what is on this host)',
+            timeoutSec: 600,
+            phase: 'preflight',
+        });
+    }
+    async stepUpgrade() {
+        const env = { UPGRADE_WAIT_SECONDS: String(this.config.waitSeconds || 900) };
+        if (this.config.testOnly)
+            env.UPGRADE_APPLY = '0';
+        if (this.config.extra)
+            env.UPGRADE_EXTRA = this.config.extra;
+        const result = await this.runStep({
+            program: 'upgrade',
+            script: '20-upgrade.sh',
+            unit: 'octra-upgrade-upgrade',
+            label: 'update the node (official tool)',
+            env,
+            timeoutSec: 7500,
+            phase: 'upgrade',
+        });
+        if (result.soft)
+            this.log('the tool asked to wait (do NOT restart the node by hand) - run the update again later', 'warn');
+    }
+    async stepManual() {
+        const result = await this.runStep({
+            program: 'upgrade',
+            script: '30-manual-upgrade.sh',
+            unit: 'octra-upgrade-manual-upgrade',
+            label: 'update the node (manual fallback)',
+            timeoutSec: 8400,
+            phase: 'manual',
+        });
+        if (result.soft)
+            this.log('the manual update is not finished - re-run later', 'warn');
+    }
+    async stepStorage() {
+        const result = await this.runStep({
+            program: 'upgrade',
+            script: '40-storage-prune.sh',
+            unit: 'octra-upgrade-storage-prune',
+            label: 'remove preserved states',
+            env: this.config.testOnly ? { PRUNE_APPLY: '0' } : {},
+            timeoutSec: 1500,
+            phase: 'storage',
+        });
+        if (result.soft)
+            this.log('the prune stopped - re-run later', 'warn');
+    }
+    async stepDashboard() {
+        this.startPhase('dashboard');
+        try {
+            const result = await (0, dashboard_1.deployDashboard)(this.client, this.logger, {
+                ip: this.config.ip,
+                domain: this.config.domain,
+                sslMode: this.config.sslMode || 'auto',
+                certDir: this.config.certDir,
+                email: this.config.email,
+                reset: this.config.reset,
+            }, 'manage');
+            this.dashboardUrl = result.url;
+            this.completePhase(true);
+        }
+        catch (err) {
+            this.completePhase(false, err.message);
+            throw err;
+        }
+    }
+}
+exports.ManageOrchestrator = ManageOrchestrator;
+function createManageOrchestrator(config, logger) {
+    return new ManageOrchestrator(config, logger);
+}
+//# sourceMappingURL=manage.js.map

package/dist/orchestrator/manage.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"manage.js","sourceRoot":"","sources":["../../src/orchestrator/manage.ts"],"names":[],"mappings":";;;AA4QA,4DAEC;AA9QD,iCAA0C;AAC1C,0CAA8C;AAE9C,8CAAsE;AACtE,kDAAqD;AAErD,4CAA2C;AAE3C,MAAa,kBAAmB,SAAQ,uBAAgB;IAC9C,MAAM,CAAe;IACrB,SAAS,CAAS;IAClB,QAAQ,GAAyB,IAAI,CAAC;IAC9C,QAAQ,GAA6C,IAAI,CAAC;IAC1D,YAAY,GAAkB,IAAI,CAAC;IAEnC,YAAY,MAAoB,EAAE,MAAc;QAC9C,MAAM,SAAS,GAAG,MAAM,CAAC,KAAK,KAAK,SAAS,IAAI,MAAM,CAAC,KAAK,KAAK,QAAQ,IAAI,MAAM,CAAC,KAAK,KAAK,SAAS;YACrG,CAAC,CAAC,uBAAW,CAAC,OAAO;YACrB,CAAC,CAAC,uBAAW,CAAC,MAAM,CAAC;QACvB,KAAK,CAAC,IAAI,oBAAY,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC,CAAC;QAC3C,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,SAAS,GAAG,MAAM,CAAC,EAAE,CAAC;QAC3B,IAAI,CAAC,OAAO,GAAG,QAAQ,CAAC;IAC1B,CAAC;IAED,KAAK,CAAC,GAAG;QACP,MAAM,SAAS,GAAc;YAC3B,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC,EAAE;YACpB,IAAI,EAAE,EAAE;YACR,QAAQ,EAAE,MAAM;YAChB,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC,IAAI;SACvB,CAAC;QACF,IAAI,CAAC;YACH,IAAI,CAAC,MAAM,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,SAAS,EAAE,QAAQ,IAAI,CAAC,MAAM,CAAC,EAAE,EAAE,CAAC,CAAC;YACzE,IAAI,CAAC,QAAQ,GAAG,IAAA,6BAAmB,EAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YAEjD,IAAI,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;gBACtB,MAAM,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;gBAC9C,IAAI,CAAC,GAAG,CAAC,sBAAsB,EAAE,MAAM,CAAC,CAAC;YAC3C,CAAC;YAED,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC;YAChC,MAAM,SAAS,GAAG,KAAK,KAAK,SAAS,IAAI,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,SAAS,CAAC;YACnF,MAAM,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YAClC,MAAM,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC;YAC1E,IAAI,SAAS,EAAE,CAAC;gBACd,MAAM,IAAI,CAAC,eAAe,CACxB,IAAI,CAAC,MAAM,EACX,CAAC,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,CAAC,EAClH,mBAAmB,CACpB,CAAC;YACJ,CAAC;iBAAM,CAAC;gBACN,MAAM,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,MAAM,EAAE,SAAS,EAAE,mBAAmB,CAAC,CAAC;YAC1E,CAAC;YAED,QAAQ,KAAK,EAAE,CAAC;gBACd,KAAK,QAAQ;oBAAE,MAAM,IAAI,CAAC,UAAU,EAAE,CAAC;oBAAC,MAAM;gBAC9C,KAAK,MAAM;oBAAE,MAAM,IAAI,CAAC,QAAQ,EAAE,CAAC;oBAAC,MAAM;gBAC1C,KAAK,MAAM;oBAAE,MAAM,IAAI,CAAC,QAAQ,EAAE,CAAC;oBAAC,MAAM;gBAC1C,KAAK,WAAW;oBAAE,MAAM,IAAI,CAAC,aAAa,EAAE,CAAC;oBAAC,MAAM;gBACpD,KAAK,UAAU;oBAAE,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;oBAAC,MAAM;gBAClD,KAAK,SAAS;oBAAE,MAAM,IAAI,CAAC,WAAW,EAAE,CAAC;oBAAC,MAAM;gBAChD,KAAK,SAAS;oBACZ,MAAM,IAAI,CAAC,oBAAoB,EAAE,CAAC;oBAClC,MAAM,IAAI,CAAC,WAAW,EAAE,CAAC;oBACzB,MAAM;gBACR,KAAK,QAAQ;oBAAE,MAAM,IAAI,CAAC,UAAU,EAAE,CAAC;oBAAC,MAAM;gBAC9C,KAAK,SAAS;oBAAE,MAAM,IAAI,CAAC,WAAW,EAAE,CAAC;oBAAC,MAAM;gBAChD,OAAO,CAAC,CAAC,MAAM,IAAI,KAAK,CAAC,yBAAyB,KAAK,EAAE,CAAC,CAAC;YAC7D,CAAC;YAED,IAAI,KAAK,KAAK,WAAW,EAAE,CAAC;gBAC1B,MAAM,EAAE,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,CAAC,EAAE,KAAK,EAAE,SAAS,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC,CAAC,CAAC;gBAClI,IAAI,CAAC,GAAG,CAAC,QAAQ,IAAI,CAAC,MAAM,CAAC,EAAE,KAAK,EAAE,CAAC,KAAK,IAAI,EAAE,CAAC,MAAM,EAAE,EAAE,MAAM,CAAC,CAAC;YACvE,CAAC;QACH,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,GAAG,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YAC7D,IAAI,CAAC,GAAG,CAAC,UAAU,IAAI,CAAC,MAAM,CAAC,KAAK,YAAY,GAAG,EAAE,EAAE,OAAO,CAAC,CAAC;YAChE,MAAM,GAAG,CAAC;QACZ,CAAC;gBAAS,CAAC;YACT,MAAM,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACxC,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,OAAO,CAAC,OAQrB;QACC,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;QAC/B,MAAM,GAAG,GAAG,OAAO,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,QAAQ,CAAC;QACjE,MAAM,IAAI,CAAC,GAAG,CAAC,QAAQ,IAAI,CAAC,MAAM,CAAC,EAAE,KAAK,OAAO,CAAC,KAAK,EAAE,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;QACnF,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,QAAS,CAAC,aAAa,CAAC;gBAChD,OAAO,EAAE,OAAO,CAAC,OAAO;gBACxB,MAAM,EAAE,SAAS,GAAG,IAAI,OAAO,CAAC,MAAM,EAAE;gBACxC,IAAI,EAAE,OAAO,CAAC,IAAI;gBAClB,GAAG,EAAE,OAAO,CAAC,GAAG;gBAChB,UAAU,EAAE,OAAO,CAAC,UAAU;gBAC9B,MAAM,EAAE,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,OAAO,IAAI,EAAE,EAAE,QAAQ,EAAE,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC;aAClE,CAAC,CAAC;YACH,IAAI,MAAM,CAAC,KAAK,KAAK,MAAM,EAAE,CAAC;gBAC5B,IAAI,CAAC,GAAG,CAAC,GAAG,OAAO,CAAC,KAAK,YAAY,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;gBACzG,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC;gBACzB,OAAO,MAAM,CAAC;YAChB,CAAC;YACD,IAAI,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;gBACxC,IAAI,CAAC,GAAG,CAAC,GAAG,OAAO,CAAC,KAAK,aAAa,MAAM,CAAC,KAAK,IAAI,MAAM,CAAC,MAAM,EAAE,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;gBAC/F,IAAI,CAAC,QAAQ,GAAG,MAAM,CAAC;gBACvB,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC;gBACzB,OAAO,MAAM,CAAC;YAChB,CAAC;YACD,IAAI,CAAC,GAAG,CAAC,GAAG,OAAO,CAAC,KAAK,YAAY,MAAM,CAAC,MAAM,EAAE,EAAE,OAAO,EAAE,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;YAC/E,IAAI,CAAC,aAAa,CAAC,KAAK,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC;YACzC,MAAM,IAAI,KAAK,CAAC,GAAG,OAAO,CAAC,KAAK,YAAY,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC;QAC/D,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,IAAI,CAAC,aAAa,CAAC,KAAK,EAAG,GAAa,CAAC,OAAO,CAAC,CAAC;YAClD,MAAM,GAAG,CAAC;QACZ,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,UAAU;QACtB,MAAM,IAAI,CAAC,OAAO,CAAC;YACjB,OAAO,EAAE,QAAQ;YACjB,MAAM,EAAE,cAAc;YACtB,IAAI,EAAE,qBAAqB;YAC3B,KAAK,EAAE,sBAAsB;YAC7B,UAAU,EAAE,GAAG;YACf,KAAK,EAAE,QAAQ;SAChB,CAAC,CAAC;IACL,CAAC;IAEO,KAAK,CAAC,QAAQ;QACpB,MAAM,MAAM,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;QACjD,IAAI,CAAC,yBAAyB,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC;YAClE,MAAM,IAAI,KAAK,CAAC,yDAAyD,CAAC,CAAC;QAC7E,CAAC;QACD,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC;YAChC,OAAO,EAAE,QAAQ;YACjB,MAAM,EAAE,qBAAqB;YAC7B,IAAI,EAAE,4BAA4B;YAClC,KAAK,EAAE,oBAAoB,MAAM,MAAM;YACvC,GAAG,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE;YAC5E,UAAU,EAAE,IAAI;YAChB,KAAK,EAAE,MAAM;SACd,CAAC,CAAC;QACH,IAAI,MAAM,CAAC,IAAI;YAAE,IAAI,CAAC,GAAG,CAAC,sCAAsC,MAAM,CAAC,MAAM,GAAG,EAAE,MAAM,CAAC,CAAC;IAC5F,CAAC;IAEO,KAAK,CAAC,QAAQ;QACpB,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC;YAChC,OAAO,EAAE,QAAQ;YACjB,MAAM,EAAE,sBAAsB;YAC9B,IAAI,EAAE,6BAA6B;YACnC,KAAK,EAAE,wBAAwB;YAC/B,UAAU,EAAE,IAAI;YAChB,KAAK,EAAE,MAAM;SACd,CAAC,CAAC;QACH,IAAI,MAAM,CAAC,IAAI,EAAE,CAAC;YAChB,IAAI,CAAC,GAAG,CAAC,6EAA6E,EAAE,IAAI,CAAC,CAAC;QAChG,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,YAAY;QACxB,MAAM,IAAI,CAAC,OAAO,CAAC;YACjB,OAAO,EAAE,QAAQ;YACjB,MAAM,EAAE,qBAAqB;YAC7B,IAAI,EAAE,4BAA4B;YAClC,KAAK,EAAE,mBAAmB;YAC1B,UAAU,EAAE,GAAG;YACf,KAAK,EAAE,UAAU;SAClB,CAAC,CAAC;IACL,CAAC;IAEO,KAAK,CAAC,WAAW;QACvB,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC;YAChC,OAAO,EAAE,QAAQ;YACjB,MAAM,EAAE,oBAAoB;YAC5B,IAAI,EAAE,2BAA2B;YACjC,KAAK,EAAE,uBAAuB;YAC9B,GAAG,EAAE,EAAE,aAAa,EAAE,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE;YACzD,UAAU,EAAE,IAAI;YAChB,KAAK,EAAE,SAAS;SACjB,CAAC,CAAC;QACH,IAAI,MAAM,CAAC,IAAI;YAAE,IAAI,CAAC,GAAG,CAAC,wBAAwB,MAAM,CAAC,KAAK,IAAI,MAAM,CAAC,MAAM,EAAE,EAAE,MAAM,CAAC,CAAC;IAC7F,CAAC;IAEO,KAAK,CAAC,oBAAoB;QAChC,MAAM,IAAI,CAAC,OAAO,CAAC;YACjB,OAAO,EAAE,SAAS;YAClB,MAAM,EAAE,iBAAiB;YACzB,IAAI,EAAE,yBAAyB;YAC/B,KAAK,EAAE,kCAAkC;YACzC,UAAU,EAAE,GAAG;YACf,KAAK,EAAE,WAAW;SACnB,CAAC,CAAC;IACL,CAAC;IAEO,KAAK,CAAC,WAAW;QACvB,MAAM,GAAG,GAA2B,EAAE,oBAAoB,EAAE,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,WAAW,IAAI,GAAG,CAAC,EAAE,CAAC;QACrG,IAAI,IAAI,CAAC,MAAM,CAAC,QAAQ;YAAE,GAAG,CAAC,aAAa,GAAG,GAAG,CAAC;QAClD,IAAI,IAAI,CAAC,MAAM,CAAC,KAAK;YAAE,GAAG,CAAC,aAAa,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC;QAC7D,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC;YAChC,OAAO,EAAE,SAAS;YAClB,MAAM,EAAE,eAAe;YACvB,IAAI,EAAE,uBAAuB;YAC7B,KAAK,EAAE,iCAAiC;YACxC,GAAG;YACH,UAAU,EAAE,IAAI;YAChB,KAAK,EAAE,SAAS;SACjB,CAAC,CAAC;QACH,IAAI,MAAM,CAAC,IAAI;YAAE,IAAI,CAAC,GAAG,CAAC,uFAAuF,EAAE,MAAM,CAAC,CAAC;IAC7H,CAAC;IAEO,KAAK,CAAC,UAAU;QACtB,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC;YAChC,OAAO,EAAE,SAAS;YAClB,MAAM,EAAE,sBAAsB;YAC9B,IAAI,EAAE,8BAA8B;YACpC,KAAK,EAAE,mCAAmC;YAC1C,UAAU,EAAE,IAAI;YAChB,KAAK,EAAE,QAAQ;SAChB,CAAC,CAAC;QACH,IAAI,MAAM,CAAC,IAAI;YAAE,IAAI,CAAC,GAAG,CAAC,kDAAkD,EAAE,MAAM,CAAC,CAAC;IACxF,CAAC;IAEO,KAAK,CAAC,WAAW;QACvB,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC;YAChC,OAAO,EAAE,SAAS;YAClB,MAAM,EAAE,qBAAqB;YAC7B,IAAI,EAAE,6BAA6B;YACnC,KAAK,EAAE,yBAAyB;YAChC,GAAG,EAAE,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC,EAAE;YACrD,UAAU,EAAE,IAAI;YAChB,KAAK,EAAE,SAAS;SACjB,CAAC,CAAC;QACH,IAAI,MAAM,CAAC,IAAI;YAAE,IAAI,CAAC,GAAG,CAAC,kCAAkC,EAAE,MAAM,CAAC,CAAC;IACxE,CAAC;IAEO,KAAK,CAAC,aAAa;QACzB,IAAI,CAAC,UAAU,CAAC,WAAW,CAAC,CAAC;QAC7B,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,IAAA,2BAAe,EAClC,IAAI,CAAC,MAAO,EACZ,IAAI,CAAC,MAAM,EACX;gBACE,EAAE,EAAE,IAAI,CAAC,MAAM,CAAC,EAAE;gBAClB,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,MAAM;gBAC1B,OAAO,EAAE,IAAI,CAAC,MAAM,CAAC,OAAO,IAAI,MAAM;gBACtC,OAAO,EAAE,IAAI,CAAC,MAAM,CAAC,OAAO;gBAC5B,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,KAAK;gBACxB,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,KAAK;aACzB,EACD,QAAQ,CACT,CAAC;YACF,IAAI,CAAC,YAAY,GAAG,MAAM,CAAC,GAAG,CAAC;YAC/B,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC;QAC3B,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,IAAI,CAAC,aAAa,CAAC,KAAK,EAAG,GAAa,CAAC,OAAO,CAAC,CAAC;YAClD,MAAM,GAAG,CAAC;QACZ,CAAC;IACH,CAAC;CACF;AAlQD,gDAkQC;AAED,SAAgB,wBAAwB,CAAC,MAAoB,EAAE,MAAc;IAC3E,OAAO,IAAI,kBAAkB,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;AAChD,CAAC"}