@constellation-network/node-pilot 0.0.14 → 0.0.16

package/README.md

@@ -21,7 +21,7 @@ $ npm install -g @constellation-network/node-pilot
 $ cpilot COMMAND
 running command...
 $ cpilot (--version|-v)
-@constellation-network/node-pilot/0.0.14 darwin-arm64 node-v22.15.0
+@constellation-network/node-pilot/0.0.16 darwin-arm64 node-v24.8.0
 $ cpilot --help [COMMAND]
 USAGE
   $ cpilot COMMAND
@@ -62,7 +62,7 @@ EXAMPLES
   $ cpilot config
 ```
 
-_See code: [src/commands/config.ts](https://github.com/Constellation-Labs/node-pilot/blob/v0.0.14/src/commands/config.ts)_
+_See code: [src/commands/config.ts](https://github.com/Constellation-Labs/node-pilot/blob/v0.0.16/src/commands/config.ts)_
 
 ## `cpilot config get [NAME]`
 
@@ -86,7 +86,7 @@ EXAMPLES
   $ cpilot config get gl0:CL_PUBLIC_HTTP_PORT
 ```
 
-_See code: [src/commands/config/get.ts](https://github.com/Constellation-Labs/node-pilot/blob/v0.0.14/src/commands/config/get.ts)_
+_See code: [src/commands/config/get.ts](https://github.com/Constellation-Labs/node-pilot/blob/v0.0.16/src/commands/config/get.ts)_
 
 ## `cpilot config set NAME VALUE`
 
@@ -109,7 +109,7 @@ EXAMPLES
   $ cpilot config set gl0:CL_PUBLIC_HTTP_PORT 9000
 ```
 
-_See code: [src/commands/config/set.ts](https://github.com/Constellation-Labs/node-pilot/blob/v0.0.14/src/commands/config/set.ts)_
+_See code: [src/commands/config/set.ts](https://github.com/Constellation-Labs/node-pilot/blob/v0.0.16/src/commands/config/set.ts)_
 
 ## `cpilot help [COMMAND]`
 
@@ -146,7 +146,7 @@ EXAMPLES
   $ cpilot info
 ```
 
-_See code: [src/commands/info.ts](https://github.com/Constellation-Labs/node-pilot/blob/v0.0.14/src/commands/info.ts)_
+_See code: [src/commands/info.ts](https://github.com/Constellation-Labs/node-pilot/blob/v0.0.16/src/commands/info.ts)_
 
 ## `cpilot logs LAYER`
 
@@ -170,7 +170,7 @@ EXAMPLES
   $ cpilot logs
 ```
 
-_See code: [src/commands/logs.ts](https://github.com/Constellation-Labs/node-pilot/blob/v0.0.14/src/commands/logs.ts)_
+_See code: [src/commands/logs.ts](https://github.com/Constellation-Labs/node-pilot/blob/v0.0.16/src/commands/logs.ts)_
 
 ## `cpilot restart [LAYER]`
 
@@ -198,7 +198,7 @@ EXAMPLES
   $ cpilot restart
 ```
 
-_See code: [src/commands/restart.ts](https://github.com/Constellation-Labs/node-pilot/blob/v0.0.14/src/commands/restart.ts)_
+_See code: [src/commands/restart.ts](https://github.com/Constellation-Labs/node-pilot/blob/v0.0.16/src/commands/restart.ts)_
 
 ## `cpilot shutdown`
 
@@ -215,7 +215,7 @@ EXAMPLES
   $ cpilot shutdown
 ```
 
-_See code: [src/commands/shutdown.ts](https://github.com/Constellation-Labs/node-pilot/blob/v0.0.14/src/commands/shutdown.ts)_
+_See code: [src/commands/shutdown.ts](https://github.com/Constellation-Labs/node-pilot/blob/v0.0.16/src/commands/shutdown.ts)_
 
 ## `cpilot status`
 
@@ -229,5 +229,5 @@ DESCRIPTION
   Display node status and configuration settings
 ```
 
-_See code: [src/commands/status.ts](https://github.com/Constellation-Labs/node-pilot/blob/v0.0.14/src/commands/status.ts)_
+_See code: [src/commands/status.ts](https://github.com/Constellation-Labs/node-pilot/blob/v0.0.16/src/commands/status.ts)_
 <!-- commandsstop -->

package/dist/checks/check-dependencies.js

@@ -30,8 +30,8 @@ export const checkDependencies = async () => {
                 await promptHelper.doYouWishToContinue();
             }
             clm.debug(`Running install-dependencies.sh from ${pilotDir}`);
-            const silent = !process.env.DEBUG;
-            const result = shell.exec('bash install-dependencies.sh', { cwd: pilotDir, silent });
+            // const silent = !process.env.DEBUG;
+            const result = shell.exec('bash install-dependencies.sh', { cwd: pilotDir });
             if (result.code > 0) {
                 console.log(result.stderr);
                 clm.error(`Failed to install dependencies. Please try again after resolving any errors.`);

package/dist/checks/check-hardware.js

@@ -13,9 +13,7 @@ export const checkHardware = {
         const totalSpaceGB = (r.bsize * r.blocks / (1024 * 1024 * 1024)).toFixed(2);
         // const usage = diskusage.checkSync(os.homedir());
         // const usableSpaceGB = (usage.total / (1024 * 1024 * 1024)).toFixed(2);
-        // Retrieve sys memory
-        const totalMemoryBytes = os.totalmem();
-        const totalMemoryGB = (totalMemoryBytes / (1024 * 1024 * 1024)).toFixed(2);
+        const totalMemoryGB = (os.totalmem() / (1024 * 1024 * 1024)).toFixed(2);
         // Retrieve number of CPU cores
         const numOfCores = os.availableParallelism();
         let allPassed = true;

package/dist/checks/check-layers.js

@@ -2,6 +2,7 @@ import { input } from "@inquirer/prompts";
 import chalk from "chalk";
 import { clm } from "../clm.js";
 import { configStore } from "../config-store.js";
+import { StatusTable } from "../helpers/status-table.js";
 import { dockerService } from "../services/docker-service.js";
 import { nodeService } from "../services/node-service.js";
 export const checkLayers = {
@@ -62,17 +63,7 @@ export const checkLayers = {
         }
     },
     async layersStatus() {
-        const { layersToRun } = configStore.getProjectInfo();
-        const promises = layersToRun.map(l => nodeService.getNodeInfo(l));
-        const results = await Promise.all(promises);
-        const states = results.map(r => r.state);
-        clm.preStep("-----------------------");
-        clm.preStep(" VALIDATOR NODE STATUS ");
-        clm.preStep("-----------------------");
-        for (const [i, layer] of layersToRun.entries()) {
-            const state = states[i];
-            clm.step(` "${layer.toUpperCase()}" is "${state}"`);
-        }
+        await StatusTable.run();
     },
     async nodeReadyToJoin(layer) {
         const status = await nodeService.getNodeInfo(layer);

package/dist/checks/check-network.d.ts

@@ -1,4 +1,5 @@
 export declare const checkNetwork: {
+    checkExternalIpAddress(): Promise<void> | undefined;
     checkForExistingNodeIdInCluster(): Promise<void>;
     checkSeedList(): Promise<void>;
     configureIpAddress(): Promise<void>;

package/dist/checks/check-network.js

@@ -8,6 +8,12 @@ import { clusterService } from "../services/cluster-service.js";
 import { dockerService } from "../services/docker-service.js";
 import { shellService } from "../services/shell-service.js";
 export const checkNetwork = {
+    checkExternalIpAddress() {
+        const { CL_EXTERNAL_IP } = configStore.getEnvInfo();
+        if (!CL_EXTERNAL_IP) {
+            return this.configureIpAddress();
+        }
+    },
     async checkForExistingNodeIdInCluster() {
         if (configStore.hasProjectFlag('duplicateNodeIdChecked')) {
             return;

package/dist/checks/check-project.js

@@ -1,6 +1,4 @@
 import { input, select } from "@inquirer/prompts";
-import chalk from "chalk";
-import ora from 'ora';
 import { clm } from "../clm.js";
 import { configStore } from "../config-store.js";
 import { configHelper } from "../helpers/config-helper.js";
@@ -96,29 +94,31 @@ export const checkProject = {
         if (isRunning) {
             await dockerService.dockerDown();
         }
-        const silent = !process.env.DEBUG;
-        const spinner = ora('Running install script...');
-        if (silent) {
-            spinner.start();
-            spinner.color = 'green';
-        }
-        else {
-            clm.preStep('Running install script...');
-        }
+        // const silent = !process.env.DEBUG;
+        //
+        // const spinner = ora('');
+        //
+        // if (silent) {
+        //     spinner.start();
+        //     spinner.color = 'green';
+        // }
+        // else {
+        clm.preStep('Running install script...');
+        // }
         // const node = await clusterService.getClusterNodeInfo();
         // const NODE_URL = `http://${node.host}:${node.publicPort}`;
         // NOTE: may be different for metagraphs
-        await shellService.runProjectCommand(`scripts/install.sh ${nInfo.type}`, undefined, silent)
+        await shellService.runProjectCommand(`scripts/install.sh ${nInfo.type}`, undefined, false)
             .catch(() => {
-            spinner.stop();
-            if (silent) {
-                clm.error(`Install script failed. run: ${chalk.cyan("DEBUG=true cpilot")} for more details`);
-            }
+            // spinner.stop();
+            // if (silent) {
+            //     clm.error(`Install script failed. run: ${chalk.cyan("DEBUG=true cpilot")} for more details`);
+            // }
             clm.error('Install script failed. Please run cpilot again after correcting the error');
         });
-        if (silent) {
-            spinner.stop();
-        }
+        // if (silent) {
+        //     spinner.stop();
+        // }
         rInfo = await configHelper.getReleaseInfo();
         configStore.setNetworkInfo({
             type: rInfo.network,

package/dist/commands/info.js

@@ -20,7 +20,7 @@ export default class Info extends Command {
         // Node ID
         configHelper.showEnvInfo('Node ID', projectInfo.nodeId);
         // Layers to Run
-        configHelper.showEnvInfo('Layers to Run', projectInfo.layersToRun.join(', '));
+        configHelper.showEnvInfo('Layers to Run', projectInfo.layersToRun?.join(', '));
         // Network type
         configHelper.showEnvInfo('Network', networkInfo.type);
         // Network version

package/dist/commands/restart.js

@@ -27,7 +27,7 @@ export default class Restart extends BaseCommand {
             serviceLog.log('Executing "cpilot restart --update" at ' + new Date().toLocaleString('en-US', { timeZone: 'America/Los_Angeles' }));
             const project = configStore.getActiveProject();
             const activeProjects = configStore.getRunningProjects();
-            serviceLog.log(`    Active projects: ${activeProjects.join(', ')}...`);
+            // serviceLog.log(`    Active projects: ${activeProjects.join(', ')}...`);
             for (const project of activeProjects) {
                 configStore.setActiveProject(project);
                 // eslint-disable-next-line no-await-in-loop
@@ -44,7 +44,7 @@ export default class Restart extends BaseCommand {
             serviceLog.log('Executing "cpilot restart --autostart" at ' + new Date().toLocaleString('en-US', { timeZone: 'America/Los_Angeles' }));
             const project = configStore.getActiveProject();
             const activeProjects = configStore.getRunningProjects();
-            serviceLog.log(`    Active projects: ${activeProjects.join(', ')}...`);
+            // serviceLog.log(`    Active projects: ${activeProjects.join(', ')}...`);
             for (const project of activeProjects) {
                 serviceLog.log('    ' + project + ' is restarting...');
                 configStore.setActiveProject(project);

package/dist/commands/status.js

@@ -17,6 +17,7 @@ export default class Status extends Command {
 export async function checkInstallationAndConfigurationStatus() {
     await checkInitialSetup.firstTimeRun();
     await checkProject.projectInstallation();
+    await checkNetwork.checkExternalIpAddress();
     await checkNetwork.isNetworkConnectable();
     await checkProject.releaseVersion();
     await checkNodeCtl.check4Migration();

package/dist/commands/test.js

@@ -4,6 +4,7 @@ import fs from "node:fs";
 import path from "node:path";
 import { fileURLToPath } from "node:url";
 import { getRandomNode } from "../services/get-random-node.js";
+import { StatusTable } from "../helpers/status-table.js";
 export default class Test extends Command {
     static description = 'node pilot test command';
     static hidden = true;
@@ -17,6 +18,7 @@ export default class Test extends Command {
         //
         // await archiverService.syncToLatestSnapshot();
         // await archiverService.checkLogsForMissingSnapshots();
+        await StatusTable.run();
     }
     async testRandomNode() {
         const filePath = path.resolve(path.dirname(fileURLToPath(import.meta.url)), `../../projects/hypergraph/networks/mainnet/source-nodes.env`);

package/dist/helpers/project-helper.js

@@ -6,6 +6,7 @@ import { fileURLToPath } from 'node:url';
 import { clm } from "../clm.js";
 import { configStore } from "../config-store.js";
 import { githubService } from "../services/github-service.js";
+import { shellService } from "../services/shell-service.js";
 import { configHelper } from "./config-helper.js";
 import { getLayerEnvFileContent } from "./env-templates.js";
 export const projectHelper = {
@@ -63,11 +64,17 @@ export const projectHelper = {
     async installHypergraph() {
         await this.installEmbedded('hypergraph');
         const { projectDir } = configStore.getProjectInfo();
-        // Create gl0 folders for fast forward feature before Docker does
+        const { platform } = configStore.getSystemInfo();
+        // Create gl0 folder for the fast-forward feature before Docker does
         const gl0DataDir = path.join(projectDir, 'gl0', 'data');
         fs.mkdirSync(path.join(gl0DataDir, 'incremental_snapshot'), { recursive: true });
         fs.mkdirSync(path.join(gl0DataDir, 'snapshot_info'));
         fs.mkdirSync(path.join(gl0DataDir, 'tmp'));
+        if (platform === 'linux') {
+            const layerDir = path.join(projectDir, 'gl0');
+            // set permission for group "docker" on the layer folder and any subfolders created later
+            await shellService.runCommand(`sudo setfacl -Rm g:docker:rwX -dm g:docker:rwX ${layerDir}`);
+        }
         this.importEnvFiles();
     },
     async installProject(name, projectFolder) {

package/dist/helpers/status-table-helper.d.ts

@@ -0,0 +1,31 @@
+export declare const statusTableHeader: ({
+    color: string;
+    headerColor: string;
+    value: string;
+    formatter?: undefined;
+    width?: undefined;
+} | {
+    color: string;
+    formatter: (startTime: number | string) => string;
+    headerColor: string;
+    value: string;
+    width?: undefined;
+} | {
+    color: string;
+    formatter: (value: string) => string;
+    headerColor: string;
+    value: string;
+    width: number;
+})[];
+export type NodeStatusInfo = {
+    clusterOrdinal: number;
+    clusterSession: string;
+    clusterState: string;
+    cpuUsage: string;
+    error: string;
+    memUsage: string;
+    ordinal: number;
+    pilotSession: string;
+    session: string;
+    state: string;
+};

package/dist/helpers/status-table-helper.js

@@ -0,0 +1,88 @@
+class CellFormatter {
+    formatCpu(value) {
+        if (value === '-')
+            return value;
+        const num = Number.parseInt(value, 10);
+        if (num === 0)
+            return value;
+        if (num < 60)
+            return this.style(value.toString(), "green");
+        if (num < 90)
+            return this.style(value.toString(), "yellow", "bold");
+        return this.style(value.toString(), "red", "bold");
+    }
+    formatDistance(value) {
+        if (value === '-')
+            return value;
+        const num = Number(value);
+        if (num === 0)
+            return value;
+        if (num < 4)
+            return this.style(value.toString(), "bgYellow", "bold");
+        return this.style(value.toString(), "bgRed", "bold");
+    }
+    formatError(value) {
+        if (value === '-')
+            return value;
+        return this.style(value, "bgRed", "bold");
+    }
+    formatMem(value) {
+        if (value === '-')
+            return value;
+        const num = Number.parseInt(value.split('(')[1], 10);
+        if (num === 0)
+            return value;
+        if (num < 70)
+            return this.style(value.toString(), "green");
+        if (num < 90)
+            return this.style(value.toString(), "yellow", "bold");
+        return this.style(value.toString(), "red", "bold");
+    }
+    formatOrdinal(value) {
+        const [v, changed] = value.split(':');
+        if (changed) {
+            return this.style(v, "whiteBright");
+        }
+        return this.style(v, "cyan");
+    }
+    formatState(value) {
+        if (value === 'Offline')
+            return this.style(value, "bgRed", "bold");
+        if (value === 'Ready')
+            return this.style(value, "green");
+        if (value === 'ReadyToJoin')
+            return this.style(value, "yellow", "bold");
+        if (value === 'Restarting')
+            return this.style(value, "yellow", "bold");
+        return this.style(value, "white");
+    }
+    formatUpTIme(startTime) {
+        if (!startTime)
+            return '-';
+        const upTimeMs = Date.now() - Number(startTime);
+        const upTimeSec = Math.floor(upTimeMs / 1000);
+        const hours = Math.floor(upTimeSec / 3600);
+        const minutes = Math.floor((upTimeSec % 3600) / 60);
+        const seconds = upTimeSec % 60;
+        if (hours < 1 && minutes < 1)
+            return `${seconds}s`;
+        return hours > 0 ? `${hours}h ${minutes}m ${seconds}s` : `${minutes}m ${seconds}s`;
+    }
+    style(value, color, style) {
+        return `c${color}v${value}s${style}`;
+    }
+}
+const { formatCpu, formatDistance, formatError, formatMem, formatOrdinal, formatState, formatUpTIme } = new CellFormatter();
+// Layer | Uptime | State | Ordinal | Distance from cluster | Cluster State | CPU Usage | Mem Usage | Error
+export const statusTableHeader = [
+    { color: 'white', headerColor: 'whiteBright', value: 'Network' },
+    { color: 'whiteBright', headerColor: 'whiteBright', value: 'Layer' },
+    { color: 'white', formatter: formatUpTIme, headerColor: 'whiteBright', value: 'Uptime' },
+    { color: 'white', formatter: formatState, headerColor: 'whiteBright', value: 'Node State', width: 24 },
+    { color: 'white', formatter: formatOrdinal, headerColor: 'whiteBright', value: 'Ordinal' },
+    { color: 'white', formatter: formatDistance, headerColor: 'whiteBright', value: 'Distance' },
+    { color: 'white', formatter: formatState, headerColor: 'whiteBright', value: 'Cluster State', width: 16 },
+    { color: 'white', formatter: formatCpu, headerColor: 'whiteBright', value: 'CPU Usage', width: 12 },
+    { color: 'white', formatter: formatMem, headerColor: 'whiteBright', value: 'Mem Usage (GB)', width: 16 },
+    { color: 'white', formatter: formatError, headerColor: 'whiteBright', value: 'Error', width: 22 },
+];

package/dist/helpers/status-table.d.ts

@@ -0,0 +1,8 @@
+export declare class StatusTable {
+    private alreadyRendered;
+    private previousHeight;
+    static run(): Promise<void>;
+    private getProjectInfo;
+    private monitorState;
+    private render;
+}

package/dist/helpers/status-table.js

@@ -0,0 +1,122 @@
+import { createPrompt, useKeypress } from '@inquirer/core';
+import fs from "node:fs";
+import os from "node:os";
+import ttyTable from "tty-table";
+import { clm } from "../clm.js";
+import { configStore } from "../config-store.js";
+import { statusTableHeader } from "./status-table-helper.js";
+export class StatusTable {
+    alreadyRendered = false;
+    previousHeight = 0;
+    static async run() {
+        process.on("exit", () => {
+            // show cursor
+            console.log("\u001B[?25h");
+        });
+        process.on("SIGINT", () => {
+            process.exit();
+        });
+        const table = new StatusTable();
+        table.monitorState();
+        const onKeyPress = createPrompt((_, done) => {
+            useKeypress(() => { done(''); });
+            return '';
+        });
+        await onKeyPress({});
+        console.log("\u001B[?25h");
+        process.exit();
+    }
+    getProjectInfo() {
+        const appPath = os.homedir() + '/.node-pilot';
+        const activeProject = configStore.getActiveProject();
+        const projects = configStore.getRunningProjects();
+        if (!activeProject || projects.length === 0) {
+            clm.error("No running projects found.");
+        }
+        const info = [];
+        for (const project of projects) {
+            configStore.setActiveProject(project);
+            const { layersToRun } = configStore.getProjectInfo();
+            const { type: network } = configStore.getNetworkInfo();
+            for (const layer of layersToRun) {
+                info.push({ layer, network, path: `${appPath}/${project}/${layer}/data/health-check/node`, project });
+            }
+        }
+        configStore.setActiveProject(activeProject);
+        return info;
+    }
+    async monitorState() {
+        const projects = this.getProjectInfo();
+        const values = {};
+        while (true) {
+            const rows = [];
+            for (const p of projects) {
+                if (!values[p.layer])
+                    values[p.layer] = {};
+                const n = JSON.parse(fs.readFileSync(p.path, 'utf8'));
+                const projectName = p.project === 'hypergraph' ? '' : p.project;
+                const network = p.network === 'integrationnet' ? 'intnet' : p.network;
+                const distance = Number.isNaN(n.clusterOrdinal - n.ordinal) ? '-' : String(n.clusterOrdinal - n.ordinal);
+                const ordinal = (values[p.layer].ordinal === n.ordinal) ? String(n.ordinal) : n.ordinal + ':true';
+                rows.push([
+                    projectName + ':' + network,
+                    p.layer,
+                    n.session, // n.pilotSession,
+                    n.state,
+                    n.ordinal ? ordinal : '-',
+                    distance,
+                    n.clusterState || 'Ready',
+                    n.cpuUsage || '-',
+                    n.memUsage || '-',
+                    n.error || '-'
+                ]);
+                values[p.layer].ordinal = n.ordinal;
+            }
+            this.render(rows);
+            process.stdout.write("   * press any key to cancel");
+            // eslint-disable-next-line no-await-in-loop
+            await sleep(1);
+        }
+    }
+    render(rows) {
+        const header = [...statusTableHeader];
+        // const emptyColumns = Array.from({length: rows[0].length}).fill(0) as number[];
+        // for (const [, row] of rows.entries()) {
+        //     for (const [j, cell] of row.entries()) {
+        //         if (cell === '-') emptyColumns[j] += 1;
+        //     }
+        // }
+        //
+        // for (const [j, cell] of emptyColumns.entries()) {
+        //     if(cell === rows.length) {
+        //         header.splice(j, 1);
+        //         for (const [, row] of rows.entries()) {
+        //             row.splice(j, 1);
+        //         }
+        //     }
+        // }
+        const options = { terminalAdapter: true };
+        const t1 = ttyTable(header, rows, options);
+        // hide cursor
+        console.log("\u001B[?25l");
+        // wipe existing if already rendered
+        if (this.alreadyRendered) {
+            // move cursor up number to the top of the previous print before deleting
+            console.log(`\u001B[${this.previousHeight + 3}A`);
+            // delete to end of terminal
+            console.log("\u001B[0J");
+        }
+        else {
+            this.alreadyRendered = true;
+        }
+        console.log(t1.render());
+        // reset the previous height to the height of this output
+        // for when we next clear the print
+        // eslint-disable-next-line @typescript-eslint/ban-ts-comment
+        // @ts-expect-error
+        this.previousHeight = t1.height;
+    }
+}
+function sleep(sec) {
+    return new Promise(resolve => { setTimeout(resolve, sec * 1000); });
+}

package/dist/services/fastforward-service.js

@@ -5,6 +5,7 @@ import 'json-bigint-patch';
 import path from "node:path";
 import { clm } from "../clm.js";
 import { configStore } from "../config-store.js";
+import { promptHelper } from "../helpers/prompt-helper.js";
 const CHUNK_SIZE = 20_000;
 export class FastforwardService {
     dataDir;
@@ -15,21 +16,22 @@ export class FastforwardService {
         const { projectDir } = configStore.getProjectInfo();
         const { type } = configStore.getNetworkInfo();
         this.network = type;
-        this.tmpDir = path.join(projectDir, 'gl0', 'tmp');
         this.dataDir = path.join(projectDir, 'gl0', 'data');
+        this.tmpDir = path.join(projectDir, 'gl0', 'data', 'tmp');
         fs.mkdirSync(this.tmpDir, { recursive: true });
-        fs.mkdirSync(this.dataDir, { recursive: true });
-        const env = configStore.getEnvNetworkInfo(type);
-        // this.lbUrl = `https://l0-lb-${this.network}.constellationnetwork.io`;
-        this.lbUrl = `http://${env.CL_L0_PEER_HTTP_HOST}:${env.CL_L0_PEER_HTTP_PORT}`;
+        // const env = configStore.getEnvNetworkInfo(type);
+        this.lbUrl = `https://l0-lb-${this.network}.constellationnetwork.io`;
+        // this.lbUrl = `http://${env.CL_L0_PEER_HTTP_HOST}:${env.CL_L0_PEER_HTTP_PORT}`;
     }
     static async synctoLatestSnapshot() {
         const ffs = new FastforwardService();
-        await ffs.runFastForwardSnapshot().catch(() => {
+        await ffs.runFastForwardSnapshot().catch(async (error) => {
+            clm.debug(error);
             const { projectDir } = configStore.getProjectInfo();
             const dataDir = path.join(projectDir, 'gl0', 'data', 'incremental_snapshot', 'ordinal');
             if (fs.existsSync(dataDir) && fs.readdirSync(dataDir).length > 0) {
                 clm.warn('Failed to fast forward to latest snapshot. Skipping...');
+                await promptHelper.doYouWishToContinue();
                 return;
             }
             clm.error('Failed to fast forward to latest snapshot. Please try again later.');
@@ -43,7 +45,10 @@ export class FastforwardService {
         // clm.debug('hash', Buffer.from(jsSha256.sha256(compressedSnapshotIncremental)).toString('hex'));
         const snapshotInfoDir = path.join(this.dataDir, 'snapshot_info');
         fs.mkdirSync(snapshotInfoDir, { recursive: true });
-        fs.writeFileSync(path.join(snapshotInfoDir, ordinal.toString()), compressedSnapshotInfo);
+        const ordinalDir = path.join(snapshotInfoDir, ordinal.toString());
+        clm.debug('Saving compressedSnapshotInfo to: ' + ordinalDir);
+        fs.writeFileSync(ordinalDir, compressedSnapshotInfo);
+        clm.debug('Saving compressedSnapshotIncremental to: ' + this.tmpDir);
         fs.writeFileSync(path.join(this.tmpDir, ordinal.toString() + '.c'), compressedSnapshotIncremental);
         await this.saveSnapshotFiles(ordinal.toString(), hash);
         clm.postStep(`Fastforward to snapshot "${ordinal}" completed.`);
@@ -94,9 +99,12 @@ export class FastforwardService {
             clm.warn(`Snapshot ${destOrdinalFile} already exists. Skipping...`);
             return;
         }
+        clm.debug(`Saving snapshot ${ordinal} to ${destOrdinalFile}`);
         fs.copyFileSync(ordinalFile, destOrdinalFile);
         const hashFile = path.join(hashDir, hash);
+        clm.debug(`Saving hashFile to ${hashFile}`);
         fs.linkSync(destOrdinalFile, hashFile);
+        clm.debug(`Saving snapshot version to ${path.join(this.dataDir, 'snapshot-version')}`);
         fs.writeFileSync(path.join(this.dataDir, 'snapshot-version'), `${this.network}:${ordinal}`);
     }
 }

package/dist/services/node-service.js

@@ -13,6 +13,7 @@ export const nodeService = {
             .then(res => {
             if (res.ok)
                 return res.json().then(i => ({ ...i, layer }));
+            clm.debug(`Failed to get node info from ${res.url} (${res.status})`);
             throw new Error(`Failed`);
         })
             .catch(() => ({ layer, state: "Unavailable" }));
@@ -62,7 +63,7 @@ export const nodeService = {
         const url = `http://localhost:${cliPort}/cluster/join`;
         clm.preStep(`Joining ${layer} to cluster`); // on ${cliPort}...${body} ... ${url}`);
         await shellService.execDockerShell(layer, `curl -X POST '${url}' -H 'Content-Type: application/json' --data '${body}'`);
-        await this.pollForState(layer, 'Ready');
+        // await this.pollForState(layer, 'Ready');
     },
     async leaveCluster(layer) {
         const { state } = await this.getNodeInfo(layer);
@@ -103,7 +104,7 @@ export const nodeService = {
     async pollForState(layer, expectedState = 'ReadyToJoin') {
         clm.preStep(`Waiting for ${layer.toUpperCase()} Validator Node to become ${expectedState}...`);
         await sleep(1);
-        for (let i = 1; i <= 60; i++) {
+        for (let i = 1; i <= 24; i++) {
             // eslint-disable-next-line no-await-in-loop
             const { state } = await this.getNodeInfo(layer);
             if (expectedState === 'Offline' && (state === "Unavailable" || state === "ReadyToJoin" || state === "SessionStarted"))
@@ -123,7 +124,7 @@ export const nodeService = {
             // eslint-disable-next-line no-await-in-loop
             await sleep(5);
         }
-        clm.warn(`${layer} is not ${expectedState} after 5 minutes`);
+        clm.warn(`${layer} is not ${expectedState} after 2 minutes`);
         return false;
     }
 };

package/dist/types.d.ts

@@ -38,14 +38,6 @@ export type ClusterStats = {
     startTime: string;
     total: number;
 };
-export type NodeStatusInfo = {
-    clusterSession: string;
-    id: string;
-    inConsensus: boolean;
-    inNetwork: boolean;
-    ip: string;
-    session: string;
-};
 export type NodeDiagnosticInfo = {
     collateral: number;
     hasCollateral: boolean;

package/install-dependencies.sh

@@ -3,6 +3,32 @@
 # Break on any error
 set -e
 
+check_acl() {
+#    echo "Checking if ACL is enabled on filesystem"
+  if [[ "$(getfacl . | grep 'mask:')" != "" ]]; then
+      echo "✅ ACL is enabled on filesystem."
+      return 0
+  else
+      echo "⚠️ ACL is not enabled on filesystem. Attempting to enable it now."
+      if [[ "$(uname)" == "Linux" ]]; then
+          if command -v apt >/dev/null 2>&1; then
+              echo "Enabling ACL using apt..."
+              sudo apt install -y acl
+          else
+              echo "⚠️ Unsupported Linux distribution. Please enable ACL manually."
+              return 1
+          fi
+          echo "✅ ACL installation complete. "
+      elif [[ "$(uname)" == "Darwin" ]]; then
+          echo "⚠️ macOS does not support enabling ACL via this script. Please ensure ACL is enabled manually."
+          return 1
+      else
+          echo "⚠️ Unsupported OS: $(uname). Please enable ACL manually."
+          return 1
+      fi
+  fi
+}
+
 # Check and install curl
 check_curl() {
 #  echo "Checking for curl..."
@@ -62,16 +88,10 @@ check_docker() {
   
   case "$(uname)" in
     Linux)
-      if command -v apt >/dev/null 2>&1; then
-        echo "Installing Docker using script..."
-        curl -fsSL https://get.docker.com -o get-docker.sh
-        sudo sh ./get-docker.sh
-        sudo usermod -aG docker $USER
-        echo "Docker installed. You may need to log out and back in for group changes to take effect."
-      else
-        echo "⚠️ Unsupported Linux distribution. Please install Docker manually."
-        return 1
-      fi
+      echo "Installing Docker using script..."
+      curl -fsSL https://get.docker.com | sudo sh
+      sudo usermod -aG docker $USER
+      echo "Docker installed. You NEED to log out and log back in for the changes to take effect."
       ;;
     Darwin)
         echo "Please install Docker Desktop manually from https://www.docker.com/products/docker-desktop"
@@ -94,5 +114,6 @@ check_docker() {
 
 # Run all checks
 echo "🔍 Checking and installing required dependencies..."
+check_acl
 check_curl
 check_docker

package/oclif.manifest.json

@@ -276,5 +276,5 @@
       ]
     }
   },
-  "version": "0.0.14"
+  "version": "0.0.16"
 }

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@constellation-network/node-pilot",
   "description": "An easy deployment and monitoring tool for Constellation nodes.",
-  "version": "0.0.14",
+  "version": "0.0.16",
   "author": "Frank Fox",
   "bin": {
     "cpilot": "bin/run.js"
@@ -70,7 +70,8 @@
     "projects",
     "scripts",
     "README.md",
-    "LICENSE"
+    "LICENSE",
+    "!**/source-nodes.env"
   ],
   "homepage": "https://github.com/Constellation-Labs/node-pilot",
   "keywords": [

package/projects/hypergraph/docker-compose.yml

@@ -3,10 +3,10 @@ networks:
     driver: bridge
 
 services:
-  autoheal:
-    image: willfarrell/autoheal
-    volumes:
-      - /var/run/docker.sock:/var/run/docker.sock
+#  autoheal:
+#    image: willfarrell/autoheal
+#    volumes:
+#      - /var/run/docker.sock:/var/run/docker.sock
   gl0:
     image: constellationnetwork/tessellation${CL_DOCKER_CORE_REPO:-}:${DOCKER_IMAGE_VERSION:-latest}
     networks:
@@ -30,8 +30,8 @@ services:
       timeout: 15s
       start_period: 60s
     profiles: ["gl0"]
-    labels:
-      autoheal: "true"
+#    labels:
+#      autoheal: "true"
   gl1:
     image: constellationnetwork/tessellation${CL_DOCKER_CORE_REPO:-}:${DOCKER_IMAGE_VERSION:-latest}
     restart: unless-stopped
@@ -46,10 +46,11 @@ services:
       - ./gl1/data:/app/data
       - ./gl1/logs:/app/logs
       - ./key.p12:/app/key.p12:ro
-#    healthcheck:
-#      test: ["CMD", "/app/internal-health-check.sh"]
-#      interval: 3s
-#      timeout: 5s
-#      retries: 5
-#      start_period: 10s
+    healthcheck:
+      test: ["CMD", "cpilotHC"]
+#      test: ["CMD", "/health-check/bin/run.sh"]
+      interval: 15s
+      retries: 1
+      timeout: 15s
+      start_period: 60s
     profiles: ["gl1"]

package/projects/hypergraph/scripts/install-dependencies.sh

@@ -11,7 +11,7 @@ check_java_home() {
         JAVA_HOME_LINE='export JAVA_HOME="$(dirname "$(dirname "$(readlink -f "$(which java)")")")"'
         echo "JAVA_HOME is not set. Attempting to set automatically"
         echo "Please ensure the following line is in your ~/.bashrc or ~/.zshrc file:"
-        echo "Script will attempt to now add it for you and run it, but this only adds to .bashrc" 
+        echo "Script will attempt to now add it for you and run it, but this only adds to .bashrc"
         echo $JAVA_HOME_LINE
         echo "$JAVA_HOME_LINE" >> $HOME/.bashrc
         echo "Adding JAVA_HOME to current environment"
@@ -215,94 +215,6 @@ check_curl() {
   return 0
 }
 
-# Check and install Docker Engine
-check_docker() {
-#  echo "Checking for Docker..."
-  if command -v docker >/dev/null 2>&1; then
-    echo "✅ Docker is already installed."
-    return 0
-  fi
-  
-  echo "⚠️ Docker not found. Will attempt to install Docker."
-  
-  case "$(uname)" in
-    Linux)
-      if command -v apt >/dev/null 2>&1; then
-        echo "Installing Docker using script..."
-        curl -fsSL https://get.docker.com -o get-docker.sh
-        sudo sh ./get-docker.sh
-        sudo usermod -aG docker $USER
-        echo "Docker installed. You may need to log out and back in for group changes to take effect."
-      else
-        echo "⚠️ Unsupported Linux distribution. Please install Docker manually."
-        return 1
-      fi
-      ;;
-    Darwin)
-        echo "Please install Docker Desktop manually from https://www.docker.com/products/docker-desktop"
-        return 1
-      ;;
-    MINGW*|MSYS*|CYGWIN*)
-      echo "On Windows, please install Docker Desktop manually from https://www.docker.com/products/docker-desktop"
-      return 1
-      ;;
-    *)
-      echo "⚠️ Unsupported OS: $(uname). Please install Docker manually."
-      return 1
-      ;;
-  esac
-  
-  echo "✅ Docker installation complete."
-  return 0
-}
-
-# Check and install Node.js
-check_node() {
-#  echo "Checking for Node.js..."
-  if [ -d "$HOME/.nvm" ]; then
-    echo "✅ Node.js is already installed."
-    return 0
-  fi
-  
-  echo "⚠️ Node.js not found. Will attempt to install Node.js."
-  
-  case "$(uname)" in
-    Linux)
-      echo "Installing Node.js using nvm..."
-      # NPM
-      curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.40.3/install.sh | bash
-
-      # Load nvm without needing to open a new terminal
-      export NVM_DIR="$HOME/.nvm"
-      [ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh"  # This loads nvm
-      [ -s "$NVM_DIR/bash_completion" ] && \. "$NVM_DIR/bash_completion"  # This loads nvm bash_completion
-
-      # Install node
-      nvm install 22
-
-      # Verify installation
-      echo "Node version: $(node -v)"
-      echo "NPM version: $(npm -v)"
-      echo "NPX version: $(npx -v)"
-      ;;
-    Darwin)
-      if command -v brew >/dev/null 2>&1; then
-        echo "Installing Node.js using Homebrew..."
-        brew install node
-      else
-        echo "⚠️ Homebrew not found. Please install Node.js manually."
-        return 1
-      fi
-      ;;
-    *)
-      echo "⚠️ No Node.js installation needed for this OS: $(uname)"
-      return 0
-      ;;
-  esac
-  
-  echo "✅ Node.js installation complete."
-  return 0
-}
 
 # Run all checks
 echo "🔍 Checking and installing required dependencies..."
@@ -311,7 +223,5 @@ check_java_home
 check_jq
 check_wget
 check_curl
-check_docker
-check_node
 
 echo "All dependency checks completed."

package/projects/hypergraph/scripts/install.sh

@@ -103,6 +103,11 @@ download_asset() {
 
   # Download the asset
   echo "$ASSET_NAME -> $OUTPUT_NAME"
+#  DEBUG=${DEBUG:-false}
+#  SILENT="-s"
+#  if [ "$DEBUG" = "true" ]; then
+#    SILENT="-#"
+#  fi
   curl -# -L -H "Accept: application/octet-stream" -o "$OUTPUT_DIR/$OUTPUT_NAME" "$DOWNLOAD_URL"
 
   if [ $? -ne 0 ]; then

package/scripts/install_services.sh

@@ -1,71 +1,119 @@
 #!/usr/bin/env bash
 
+# Find cpilot in PATH
 CPILOT=$(command -v cpilot)
 if [ -z "$CPILOT" ]; then
   echo "cpilot command not found. Please ensure Node Pilot is installed and accessible in your PATH."
   exit 1
 fi
 
-WORKING_DIR=/home/$USER/.config/systemd/user
+# Get current user and home directory reliably
+CURRENT_USER=$(id -un)
+if [ "$CURRENT_USER" = "root" ]; then
+  IS_ROOT=true
+  WORKING_DIR="/etc/systemd/system"
+  HOME_DIR="/root"
+  UNIT_TARGET="multi-user.target"
+else
+  IS_ROOT=false
+  HOME_DIR=$(eval echo ~$CURRENT_USER)
+  WORKING_DIR="$HOME_DIR/.config/systemd/user"
+  UNIT_TARGET="default.target"
+fi
+
+BIN_DIRECTORY="$HOME_DIR/.node-pilot/scripts"
 
-mkdir -p $WORKING_DIR
+mkdir -p "$BIN_DIRECTORY"
 
-cat << EOF > $WORKING_DIR/restart-unhealthy.service
+# Create working directory if it doesn't exist
+mkdir -p "$WORKING_DIR"
+
+# Create restart-unhealthy.service
+cat << EOF > "$WORKING_DIR/restart-unhealthy.service"
 [Unit]
 Description=Restart unhealthy docker containers
 After=docker.service
 Wants=docker.service
 
 [Service]
-ExecStart=$WORKING_DIR/restart-unhealthy.sh
-Restart=always
-RestartSec=60s
+Type=oneshot
+ExecStart=$BIN_DIRECTORY/restart-unhealthy.sh
+EOF
+
+# Create restart-unhealthy.timer
+cat <<EOF > "$WORKING_DIR/restart-unhealthy.timer"
+[Unit]
+Description=Run docker unhealthy restart every 5 minutes
+Requires=restart-unhealthy.service
+After=docker.service
+Wants=docker.service
+
+[Timer]
+OnCalendar=*:0/1
+Persistent=true
 
 [Install]
-WantedBy=docker.service
+WantedBy=timers.target
 EOF
 
-cat << EOF > $WORKING_DIR/restart-unhealthy.sh
+# Create restart-unhealthy.sh
+cat << EOF > "$BIN_DIRECTORY/restart-unhealthy.sh"
 #!/usr/bin/env bash
 docker ps -q -f health=unhealthy | xargs --no-run-if-empty docker restart
 EOF
 
-chmod +x $WORKING_DIR/restart-unhealthy.sh
+chmod +x "$BIN_DIRECTORY/restart-unhealthy.sh"
 
-cat << EOF > $WORKING_DIR/node-pilot-autostart.service
+# Create node-pilot-autostart.service
+cat << EOF > "$WORKING_DIR/node-pilot-autostart.service"
 [Unit]
 Description=Constellation Network Node Auto Restart Service
-After=default.target
+After=$UNIT_TARGET
 
 [Service]
-ExecStart=$CPILOT restart --autostart
+ExecStart=cpilot restart --autostart
 
 [Install]
-WantedBy=default.target
+WantedBy=$UNIT_TARGET
 EOF
 
-cat << EOF > $WORKING_DIR/node-pilot-update.service
+# Create node-pilot-update.service
+cat << EOF > "$WORKING_DIR/node-pilot-update.service"
 [Unit]
 Description=Constellation Node Pilot Updater Service
-After=default.target
+After=$UNIT_TARGET
 
 [Service]
-ExecStart=$CPILOT restart --update
+ExecStart=cpilot restart --update
 Restart=always
 RestartSec=5m
 
 [Install]
-WantedBy=default.target
+WantedBy=$UNIT_TARGET
 EOF
 
-systemctl --user daemon-reload
-
-systemctl --user enable restart-unhealthy.service
-systemctl --user enable node-pilot-autostart.service
-systemctl --user enable node-pilot-update.service
-
-systemctl --user start restart-unhealthy.service
-systemctl --user start node-pilot-autostart.service
-systemctl --user start node-pilot-update.service
-
-loginctl enable-linger $USER
+# Enable and start services/timers
+if [ "$IS_ROOT" = true ]; then
+  systemctl daemon-reload
+  systemctl enable restart-unhealthy.timer
+  systemctl enable node-pilot-autostart.service
+  systemctl enable node-pilot-update.service
+
+  systemctl start restart-unhealthy.timer
+  systemctl start node-pilot-autostart.service
+  systemctl start node-pilot-update.service
+else
+  systemctl --user daemon-reload
+  systemctl --user enable restart-unhealthy.timer
+  systemctl --user enable node-pilot-autostart.service
+  systemctl --user enable node-pilot-update.service
+
+  systemctl --user start restart-unhealthy.timer
+  systemctl --user start node-pilot-autostart.service
+  systemctl --user start node-pilot-update.service
+
+  # Enable linger for the user to allow user services to run without login
+  loginctl enable-linger "$CURRENT_USER"
+fi
+#systemctl --user status node-pilot-update.service
+#  systemctl --user status restart-unhealthy

package/scripts/node-pilot-update.service

@@ -0,0 +1,11 @@
+[Unit]
+Description=Constellation Node Pilot Updater Service
+After=default.target
+
+[Service]
+ExecStart=/bin/bash -lc "echo $HOME : $(which node) >> /home/ubuntu/service.log"
+Restart=always
+RestartSec=5m
+
+[Install]
+WantedBy=default.target

package/projects/hypergraph/networks/integrationnet/source-nodes.env

@@ -1,9 +0,0 @@
-SOURCE_NODE_1_PORT=9000
-SOURCE_NODE_1_HOST=13.52.205.240
-SOURCE_NODE_1_ID=e2f4496e5872682d7a55aa06e507a58e96b5d48a5286bfdff7ed780fa464d9e789b2760ecd840f4cb3ee6e1c1d81b2ee844c88dbebf149b1084b7313eb680714
-SOURCE_NODE_2_PORT=9000
-SOURCE_NODE_2_HOST=50.18.155.22
-SOURCE_NODE_2_ID=3458a688925a4bd89f2ac2c695362e44d2e0c2903bdbb41b341a4d39283b22d8c85b487bd33cc5d36dbe5e31b5b00a10a6eab802718ead4ed7192ade5a5d1941
-SOURCE_NODE_3_PORT=9000
-SOURCE_NODE_3_HOST=52.9.216.57
-SOURCE_NODE_3_ID=46daea11ca239cb8c0c8cdeb27db9dbe9c03744908a8a389a60d14df2ddde409260a93334d74957331eec1af323f458b12b3a6c3b8e05885608aae7e3a77eac7

package/projects/hypergraph/networks/mainnet/source-nodes.env

@@ -1,9 +0,0 @@
-SOURCE_NODE_1_PORT=9000
-SOURCE_NODE_1_HOST=52.53.46.33
-SOURCE_NODE_1_ID=e0c1ee6ec43510f0e16d2969a7a7c074a5c8cdb477c074fe9c32a9aad8cbc8ff1dff60bb81923e0db437d2686a9b65b86c403e6a21fa32b6acc4e61be4d70925
-SOURCE_NODE_2_PORT=9000
-SOURCE_NODE_2_HOST=54.215.18.98
-SOURCE_NODE_2_ID=629880a5b8d4cc6d12aec26f24230a463825c429723153aeaff29475b29e39d2406af0f8b034ba7798ae598dbd5f513d642bcbbeef088290abeadac61a0445d6
-SOURCE_NODE_3_PORT=9000
-SOURCE_NODE_3_HOST=54.151.19.111
-SOURCE_NODE_3_ID=710b3dc521b805aea7a798d61f5d4dae39601124f1f34fac9738a78047adeff60931ba522250226b87a2194d3b7d39da8d2cbffa35d6502c70f1a7e97132a4b0

package/projects/hypergraph/networks/testnet/source-nodes.env

@@ -1,9 +0,0 @@
-SOURCE_NODE_1_PORT=9000
-SOURCE_NODE_1_HOST=52.8.132.193
-SOURCE_NODE_1_ID=e2f4496e5872682d7a55aa06e507a58e96b5d48a5286bfdff7ed780fa464d9e789b2760ecd840f4cb3ee6e1c1d81b2ee844c88dbebf149b1084b7313eb680714
-SOURCE_NODE_2_PORT=9000
-SOURCE_NODE_2_HOST=13.57.110.45
-SOURCE_NODE_2_ID=3458a688925a4bd89f2ac2c695362e44d2e0c2903bdbb41b341a4d39283b22d8c85b487bd33cc5d36dbe5e31b5b00a10a6eab802718ead4ed7192ade5a5d1941
-SOURCE_NODE_3_PORT=9000
-SOURCE_NODE_3_HOST=54.153.23.79
-SOURCE_NODE_3_ID=46daea11ca239cb8c0c8cdeb27db9dbe9c03744908a8a389a60d14df2ddde409260a93334d74957331eec1af323f458b12b3a6c3b8e05885608aae7e3a77eac7