@constellation-network/node-pilot 0.0.21 → 0.2.0

package/README.md

@@ -21,7 +21,7 @@ $ npm install -g @constellation-network/node-pilot
 $ cpilot COMMAND
 running command...
 $ cpilot (--version|-v)
-@constellation-network/node-pilot/0.0.21 darwin-arm64 node-v24.8.0
+@constellation-network/node-pilot/0.2.0 darwin-arm64 node-v22.15.0
 $ cpilot --help [COMMAND]
 USAGE
   $ cpilot COMMAND
@@ -71,7 +71,7 @@ EXAMPLES
   $ cpilot clean
 ```
 
-_See code: [src/commands/clean.ts](https://github.com/Constellation-Labs/node-pilot/blob/v0.0.21/src/commands/clean.ts)_
+_See code: [src/commands/clean.ts](https://github.com/Constellation-Labs/node-pilot/blob/v0.2.0/src/commands/clean.ts)_
 
 ## `cpilot config`
 
@@ -88,7 +88,7 @@ EXAMPLES
   $ cpilot config
 ```
 
-_See code: [src/commands/config.ts](https://github.com/Constellation-Labs/node-pilot/blob/v0.0.21/src/commands/config.ts)_
+_See code: [src/commands/config.ts](https://github.com/Constellation-Labs/node-pilot/blob/v0.2.0/src/commands/config.ts)_
 
 ## `cpilot config get [NAME]`
 
@@ -112,7 +112,7 @@ EXAMPLES
   $ cpilot config get gl0:CL_PUBLIC_HTTP_PORT
 ```
 
-_See code: [src/commands/config/get.ts](https://github.com/Constellation-Labs/node-pilot/blob/v0.0.21/src/commands/config/get.ts)_
+_See code: [src/commands/config/get.ts](https://github.com/Constellation-Labs/node-pilot/blob/v0.2.0/src/commands/config/get.ts)_
 
 ## `cpilot config set NAME VALUE`
 
@@ -135,7 +135,7 @@ EXAMPLES
   $ cpilot config set gl0:CL_PUBLIC_HTTP_PORT 9000
 ```
 
-_See code: [src/commands/config/set.ts](https://github.com/Constellation-Labs/node-pilot/blob/v0.0.21/src/commands/config/set.ts)_
+_See code: [src/commands/config/set.ts](https://github.com/Constellation-Labs/node-pilot/blob/v0.2.0/src/commands/config/set.ts)_
 
 ## `cpilot help [COMMAND]`
 
@@ -172,7 +172,7 @@ EXAMPLES
   $ cpilot info
 ```
 
-_See code: [src/commands/info.ts](https://github.com/Constellation-Labs/node-pilot/blob/v0.0.21/src/commands/info.ts)_
+_See code: [src/commands/info.ts](https://github.com/Constellation-Labs/node-pilot/blob/v0.2.0/src/commands/info.ts)_
 
 ## `cpilot logs LAYER`
 
@@ -180,12 +180,14 @@ view validator node runtime logs
 
 ```
 USAGE
-  $ cpilot logs LAYER [-f] [-n <value>]
+  $ cpilot logs LAYER [-a app|hc] [-f] [-n <value>]
 
 ARGUMENTS
   LAYER  network layer to view. e.g. gl0
 
 FLAGS
+  -a, --area=<option>       [default: hc] area to view logs for. e.g. "app" or "hc" for health-check
+                            <options: app|hc>
   -f, --follow              continuously wait for additional data to be appended
   -n, --numOfLines=<value>  number of lines at the end of the log to display
 
@@ -196,7 +198,7 @@ EXAMPLES
   $ cpilot logs
 ```
 
-_See code: [src/commands/logs.ts](https://github.com/Constellation-Labs/node-pilot/blob/v0.0.21/src/commands/logs.ts)_
+_See code: [src/commands/logs.ts](https://github.com/Constellation-Labs/node-pilot/blob/v0.2.0/src/commands/logs.ts)_
 
 ## `cpilot restart [LAYER]`
 
@@ -224,7 +226,7 @@ EXAMPLES
   $ cpilot restart
 ```
 
-_See code: [src/commands/restart.ts](https://github.com/Constellation-Labs/node-pilot/blob/v0.0.21/src/commands/restart.ts)_
+_See code: [src/commands/restart.ts](https://github.com/Constellation-Labs/node-pilot/blob/v0.2.0/src/commands/restart.ts)_
 
 ## `cpilot shutdown`
 
@@ -244,7 +246,7 @@ EXAMPLES
   $ cpilot shutdown
 ```
 
-_See code: [src/commands/shutdown.ts](https://github.com/Constellation-Labs/node-pilot/blob/v0.0.21/src/commands/shutdown.ts)_
+_See code: [src/commands/shutdown.ts](https://github.com/Constellation-Labs/node-pilot/blob/v0.2.0/src/commands/shutdown.ts)_
 
 ## `cpilot status`
 
@@ -258,5 +260,5 @@ DESCRIPTION
   Display node status and configuration settings
 ```
 
-_See code: [src/commands/status.ts](https://github.com/Constellation-Labs/node-pilot/blob/v0.0.21/src/commands/status.ts)_
+_See code: [src/commands/status.ts](https://github.com/Constellation-Labs/node-pilot/blob/v0.2.0/src/commands/status.ts)_
 <!-- commandsstop -->

package/dist/checks/check-pilot.js

@@ -2,11 +2,13 @@ import { JSONStorage } from "node-localstorage";
 import fs from "node:fs";
 import os from "node:os";
 import path from "node:path";
+import semver from "semver";
 import packageJson from '../../package.json' with { type: 'json' };
 import { clm } from "../clm.js";
 import { configStore } from "../config-store.js";
 import { projectHelper } from "../helpers/project-helper.js";
 import { promptHelper } from "../helpers/prompt-helper.js";
+import { dockerService } from "../services/docker-service.js";
 import { shellService } from "../services/shell-service.js";
 const REGISTRY_URL = 'https://registry.npmjs.org/';
 export const checkNodePilot = {
@@ -26,17 +28,44 @@ export const checkNodePilot = {
         if (!result) {
             return;
         }
-        const latestVersion = result['dist-tags'].latest;
-        // console.log(`Current version: "${packageJson.version}", Latest version: "${latestVersion}"`);
-        if (packageJson.version !== latestVersion) {
-            console.log(`There is a new node-pilot version available. Current version: "${packageJson.version}", Latest version: "${latestVersion}"`);
-            if (os.platform() === 'linux' && await promptHelper.confirmPrompt('Do you wish to upgrade now?')) {
-                await shellService.runCommand('sudo npm install -g @constellation-network/node-pilot@latest');
-                await projectHelper.upgradeHypergraph();
-                clm.postStep('Run cpilot again to use the latest version');
-                process.exit(0);
+        const latestVer = semver.parse(result['dist-tags'].latest);
+        const currentVer = semver.parse(packageJson.version);
+        if (!latestVer || !currentVer || latestVer.compare(currentVer) === 0)
+            return;
+        clm.echo(`There is a new node-pilot version available. Current version: "${currentVer.version}", Latest version: "${latestVer.version}"`);
+        if (os.platform() !== 'linux')
+            return;
+        const hasMajorMinorChange = latestVer.major !== currentVer.major || latestVer.minor !== currentVer.minor;
+        if (hasMajorMinorChange) {
+            const dockerIsRunning = await dockerService.isRunning();
+            if (dockerIsRunning) {
+                clm.warn('This update requires the Node to shutdown before proceeding.');
+            }
+            if (await promptHelper.confirmPrompt('Do you wish to upgrade now?')) {
+                if (dockerIsRunning) {
+                    await dockerService.dockerDown();
+                }
+            }
+            else {
+                clm.postStep('Skipping update...');
+                return;
             }
         }
+        else {
+            clm.preStep('This update includes a minor change. There is no need to restart your Node.');
+            if (!await promptHelper.confirmPrompt('Do you wish to upgrade now?')) {
+                clm.postStep('Skipping update...');
+                return;
+            }
+        }
+        clm.step('Updating Node Pilot to the latest version...');
+        await shellService.runCommand(`sudo npm install -g @constellation-network/node-pilot@${latestVer.version}`);
+        if (hasMajorMinorChange) {
+            clm.step('Updating scripts and configuration files...');
+            await projectHelper.upgradeHypergraph();
+        }
+        clm.postStep('Update completed. Run cpilot again to use the latest version');
+        process.exit(0);
     },
     async promptDiscordRegistration() {
         const hcStorage = getHcStorage();

package/dist/checks/check-project.js

@@ -70,7 +70,7 @@ export const checkProject = {
             if (nInfo.version !== clusterVersion) {
                 const answer = await input({
                     default: 'y',
-                    message: `A new required network version has been detected. Do you want to upgrade now? (y/n): `
+                    message: `A new required network version has been detected. ${clusterVersion}. Do you want to upgrade now? (y/n): `
                 });
                 if (answer !== 'y') {
                     process.exit(0);

package/dist/clm.js

@@ -1,4 +1,6 @@
 import chalk from "chalk";
+import { configStore } from "./config-store.js";
+import { serviceLog } from "./helpers/service-log.js";
 /*
     Command Line Messaging with color styling
  */
@@ -9,25 +11,28 @@ export const clm = {
         }
     },
     echo(msg) {
-        console.log(msg);
+        o().log(msg);
     },
     echoRepeatLine(char) {
         process.stdout.write('\r' + char);
     },
     error(msg, silent = true) {
-        console.error(chalk.red(msg));
+        o().error(chalk.red(msg));
         process.exit(silent ? 0 : 1);
     },
     postStep(s) {
-        console.log(chalk.green(s));
+        o().log(chalk.green(s));
     },
     preStep(s) {
-        console.log('\n' + chalk.italic(chalk.green(s)));
+        o().log('\n' + chalk.italic(chalk.green(s)));
     },
     step(msg) {
-        console.warn(chalk.whiteBright(msg));
+        o().warn(chalk.whiteBright(msg));
     },
     warn(msg) {
-        console.warn(chalk.red(msg));
+        o().warn(chalk.red(msg));
     }
 };
+function o() {
+    return configStore.isRestarting() ? serviceLog : console;
+}

package/dist/commands/clean.js

@@ -37,6 +37,7 @@ export default class Clean extends Command {
         if (deleteData) {
             await shellService.runProjectCommand(`sudo rm -rf ${args.layer}/data`);
             projectHelper.prepareDataFolder();
+            configStore.setProjectFlag('discordChecked', false);
         }
         if (deleteLogs) {
             await shellService.runProjectCommand(`sudo rm -rf ${args.layer}/logs`);

package/dist/commands/logs.d.ts

@@ -6,6 +6,7 @@ export default class Logs extends Command {
     static description: string;
     static examples: string[];
     static flags: {
+        area: import("@oclif/core/interfaces").OptionFlag<string, import("@oclif/core/interfaces").CustomOptions>;
         follow: import("@oclif/core/interfaces").BooleanFlag<boolean>;
         numOfLines: import("@oclif/core/interfaces").OptionFlag<string | undefined, import("@oclif/core/interfaces").CustomOptions>;
     };

package/dist/commands/logs.js

@@ -12,6 +12,7 @@ export default class Logs extends Command {
         '<%= config.bin %> <%= command.id %>',
     ];
     static flags = {
+        area: Flags.string({ char: 'a', default: 'hc', description: 'area to view logs for. e.g. "app" or "hc" for health-check', options: ['app', 'hc'] }),
         follow: Flags.boolean({ char: 'f', description: 'continuously wait for additional data to be appended' }),
         numOfLines: Flags.string({ char: 'n', description: 'number of lines at the end of the log to display' }),
     };
@@ -27,8 +28,9 @@ export default class Logs extends Command {
         if (flags.follow) {
             tailFlag = ' -f';
         }
+        const file = flags.area === 'app' ? 'app.log' : 'health-check.log';
         const { projectDir } = configStore.getProjectInfo();
-        const logPath = path.join(projectDir, args.layer, 'logs', 'app.log');
+        const logPath = path.join(projectDir, args.layer, 'logs', file);
         await shellService.runCommand(`tail ${tailFlag} ${logPath}`).catch(() => 1);
     }
 }

package/dist/commands/restart.d.ts

@@ -11,4 +11,5 @@ export default class Restart extends BaseCommand {
     };
     run(): Promise<void>;
     private restart;
+    private setIsRestarting;
 }

package/dist/commands/restart.js

@@ -25,9 +25,9 @@ export default class Restart extends BaseCommand {
         configHelper.assertProject('No project found. ');
         if (flags.update) {
             serviceLog.log('Executing "cpilot restart --update" at ' + new Date().toLocaleString('en-US', { timeZone: 'America/Los_Angeles' }));
+            this.setIsRestarting(true);
             const project = configStore.getActiveProject();
             const activeProjects = configStore.getRunningProjects();
-            // serviceLog.log(`    Active projects: ${activeProjects.join(', ')}...`);
             for (const project of activeProjects) {
                 configStore.setActiveProject(project);
                 // eslint-disable-next-line no-await-in-loop
@@ -41,13 +41,14 @@ export default class Restart extends BaseCommand {
                 }
             }
             configStore.setActiveProject(project);
+            this.setIsRestarting(false);
             return;
         }
         if (flags.autostart) {
             serviceLog.log('Executing "cpilot restart --autostart" at ' + new Date().toLocaleString('en-US', { timeZone: 'America/Los_Angeles' }));
+            this.setIsRestarting(true);
             const project = configStore.getActiveProject();
             const activeProjects = configStore.getRunningProjects();
-            // serviceLog.log(`    Active projects: ${activeProjects.join(', ')}...`);
             for (const project of activeProjects) {
                 serviceLog.log('    ' + project + ' is being auto started...');
                 configStore.setActiveProject(project);
@@ -55,6 +56,7 @@ export default class Restart extends BaseCommand {
                 await this.restart();
             }
             configStore.setActiveProject(project);
+            this.setIsRestarting(false);
             return;
         }
         if (!await dockerService.isRunning()) {
@@ -75,32 +77,27 @@ export default class Restart extends BaseCommand {
         await this.restart();
     }
     async restart() {
-        // const pAll = layersToRun.map(l => nodeService.getNodeInfo(l));
-        // const info = await Promise.all(pAll);
-        // const isRunning = info.some(n => n.state !== 'Unavailable');
-        const startRestarted = configStore.isRestarting();
-        if (startRestarted && (startRestarted + 1000 * 60 * 5 > Date.now())) {
-            serviceLog.log('Restart already ACTIVE');
-            return;
+        if (await dockerService.isRunning()) {
+            await nodeService.leaveClusterAllLayers();
+            const { layersToRun } = configStore.getProjectInfo();
+            await nodeService.pollForLayersState(layersToRun, 'Offline');
+            clm.preStep('Stopping the node...');
+            await dockerService.dockerDown();
         }
-        configStore.setIsRestarting(Date.now());
-        try {
-            if (await dockerService.isRunning()) {
-                await nodeService.leaveClusterAllLayers();
-                const { layersToRun } = configStore.getProjectInfo();
-                await nodeService.pollForLayersState(layersToRun, 'Offline');
-                clm.preStep('Stopping the node...');
-                serviceLog.log('Stopping the node...');
-                await dockerService.dockerDown();
+        clm.preStep('Checking for a new version...');
+        await checkProject.runUpgrade();
+        clm.preStep('Starting the node...');
+        await dockerService.dockerRestartAll();
+    }
+    setIsRestarting(val) {
+        if (val) {
+            if (configStore.isRestarting()) {
+                serviceLog.log('Restart already ACTIVE');
+                process.exit(0);
             }
-            clm.preStep('Checking for a new version...');
-            serviceLog.log('Checking for a new version...');
-            await checkProject.runUpgrade();
-            clm.preStep('Starting the node...');
-            serviceLog.log('Starting the node...');
-            await dockerService.dockerRestartAll();
+            configStore.setIsRestarting(Date.now());
         }
-        finally {
+        else {
             configStore.setIsRestarting(0);
         }
     }

package/dist/commands/test.js

@@ -4,7 +4,7 @@ import fs from "node:fs";
 import path from "node:path";
 import { fileURLToPath } from "node:url";
 import { getRandomNode } from "../services/get-random-node.js";
-import { checkNodePilot } from "../checks/check-pilot.js";
+import { StatusTable } from "../helpers/status-table.js";
 export default class Test extends Command {
     static description = 'node pilot test command';
     static hidden = true;
@@ -18,8 +18,8 @@ export default class Test extends Command {
         //
         // await archiverService.syncToLatestSnapshot();
         // await archiverService.checkLogsForMissingSnapshots();
-        // await StatusTable.run();
-        await checkNodePilot.checkVersion();
+        await StatusTable.run();
+        // await checkNodePilot.checkVersion();
     }
     async testRandomNode() {
         const filePath = path.resolve(path.dirname(fileURLToPath(import.meta.url)), `../../projects/hypergraph/networks/mainnet/source-nodes.env`);

package/dist/config-store.d.ts

@@ -18,7 +18,7 @@ declare class ConfigStore {
     getSystemInfo(): SystemInfo;
     hasProjectFlag(name: string): any;
     hasProjects(): boolean;
-    isRestarting(): number;
+    isRestarting(): boolean;
     setActiveProject(name: string): void;
     setClusterStats(info: Partial<ClusterStats>): void;
     setDockerEnvInfo(info: Partial<{

package/dist/config-store.js

@@ -106,7 +106,11 @@ class ConfigStore {
     }
     isRestarting() {
         const { restarting } = this.pilotStore.getItem('pilot');
-        return restarting;
+        if (restarting && restarting + 1000 * 60 * 5 < Date.now()) {
+            this.setIsRestarting(0);
+            return false;
+        }
+        return restarting > 0;
     }
     setActiveProject(name) {
         const { appDir, project, projects } = this.pilotStore.getItem('pilot');

package/dist/helpers/service-log.d.ts

@@ -1,3 +1,5 @@
 export declare const serviceLog: {
+    error(s: string): void;
     log(s: string): void;
+    warn(s: string): void;
 };

package/dist/helpers/service-log.js

@@ -2,9 +2,15 @@ import fs from "node:fs";
 import path from "node:path";
 import { configStore } from "../config-store.js";
 export const serviceLog = {
+    error(s) {
+        this.log(s);
+    },
     log(s) {
         const appDir = configStore.getAppDir();
         const logFile = path.join(appDir, 'logs', 'service.log');
         fs.appendFileSync(logFile, s + '\n');
     },
+    warn(s) {
+        this.log(s);
+    },
 };

package/dist/helpers/status-table-helper.d.ts

@@ -19,6 +19,20 @@ export declare const statusTableHeader: ({
     value: string;
     width: number;
 })[];
+export declare const glHeader1: {
+    color: string;
+    formatter: (value: string) => string;
+    headerColor: string;
+    value: string;
+    width: number;
+}[];
+export declare const glHeader2: {
+    color: string;
+    formatter: (value: string) => string;
+    headerColor: string;
+    value: string;
+    width: number;
+}[];
 export type NodeStatusInfo = {
     clusterOrdinal: number;
     clusterSession: string;

package/dist/helpers/status-table-helper.js

@@ -57,10 +57,14 @@ class CellFormatter {
             return this.style(value, "bgRed", "bold");
         if (value === 'Ready')
             return this.style(value, "green");
+        if (value === 'HydratingSnapshots')
+            return this.style(value, "cyan");
         if (value === 'ReadyToJoin' || value === 'JoiningCluster')
             return this.style(value, "yellow", "bold");
         if (value === 'Restarting')
             return this.style(value, "yellow", "bold");
+        if (value.startsWith('Ready'))
+            return this.style(value, "green");
         return this.style(value, "white");
     }
     formatUpTIme(startTime) {
@@ -108,3 +112,17 @@ export const statusTableHeader = [
     { color: 'white', formatter: formatMem, headerColor: 'whiteBright', value: 'Mem Usage (GB)', width: 16 },
     { color: 'white', formatter: formatError, headerColor: 'whiteBright', value: 'Error', width: 22 },
 ];
+export const glHeader1 = [
+    // { color: 'white', headerColor: 'whiteBright', value: 'Network' },
+    // { color: 'whiteBright', headerColor: 'whiteBright', value: 'Layer' },
+    { color: 'white', formatter: formatState, headerColor: 'whiteBright', value: 'Node State', width: 22 },
+    { color: 'white', formatter: formatUpTIme, headerColor: 'whiteBright', value: 'Uptime', width: 13 },
+    { color: 'white', formatter: formatOrdinal, headerColor: 'whiteBright', value: 'Ordinal', width: 16 },
+    { color: 'white', formatter: formatDistance, headerColor: 'whiteBright', value: 'Distance', width: 22 },
+];
+export const glHeader2 = [
+    { color: 'white', formatter: formatState, headerColor: 'whiteBright', value: 'Cluster State', width: 22 },
+    { color: 'white', formatter: formatCpu, headerColor: 'whiteBright', value: 'CPU Usage', width: 13 },
+    { color: 'white', formatter: formatMem, headerColor: 'whiteBright', value: 'Mem Usage (GB)', width: 16 },
+    { color: 'white', formatter: formatError, headerColor: 'whiteBright', value: 'Error', width: 22 },
+];

package/dist/helpers/status-table.d.ts

@@ -3,6 +3,8 @@ export declare class StatusTable {
     private previousHeight;
     static run(): Promise<void>;
     private getProjectInfo;
-    private monitorState;
-    private render;
+    private monitorNarrow;
+    private monitorWide;
+    private renderNarrow;
+    private renderWide;
 }

package/dist/helpers/status-table.js

@@ -5,7 +5,7 @@ import os from "node:os";
 import ttyTable from "tty-table";
 import { clm } from "../clm.js";
 import { configStore } from "../config-store.js";
-import { formatTimeAgo, statusTableHeader } from "./status-table-helper.js";
+import { formatTimeAgo, glHeader1, glHeader2, statusTableHeader } from "./status-table-helper.js";
 export class StatusTable {
     alreadyRendered = false;
     previousHeight = 0;
@@ -18,7 +18,14 @@ export class StatusTable {
             process.exit();
         });
         const table = new StatusTable();
-        table.monitorState();
+        const cols = process.stdout.columns;
+        console.log(String('Colspan: ' + cols));
+        if (cols > 120) {
+            table.monitorWide();
+        }
+        else {
+            table.monitorNarrow();
+        }
         const onKeyPress = createPrompt((_, done) => {
             useKeypress(() => { done(''); });
             return '';
@@ -45,23 +52,73 @@ export class StatusTable {
         configStore.setActiveProject(activeProject);
         return info;
     }
-    async monitorState() {
-        const projects = this.getProjectInfo();
+    async monitorNarrow() {
+        const projectInfos = this.getProjectInfo();
+        const values = {};
+        // const err = { date: '', layer: '', msg:'', timeAgo: '' }
+        while (true) {
+            // const rows = [];
+            const tables = [];
+            for (const info of projectInfos) {
+                if (!values[info.layer])
+                    values[info.layer] = {};
+                const n = fs.existsSync(info.path) ? JSON.parse(fs.readFileSync(info.path, 'utf8')) : {};
+                const projectName = info.project === 'hypergraph' ? '' : ':' + info.project;
+                const network = info.network === 'integrationnet' ? 'intnet' : info.network;
+                const distance = Number.isNaN(n.clusterOrdinal - n.ordinal) ? '-' : String(n.clusterOrdinal - n.ordinal);
+                const ordinal = (values[info.layer].ordinal === n.ordinal) ? String(n.ordinal) : n.ordinal + ':true';
+                const label = info.layer + ':' + network + projectName;
+                const row1 = [
+                    n.state,
+                    n.session,
+                    n.ordinal ? ordinal : '-',
+                    distance,
+                ];
+                const row2 = [
+                    n.clusterState || 'Ready',
+                    n.cpuUsage || '-',
+                    n.memUsage || '-',
+                    n.error || '-'
+                ];
+                values[info.layer].ordinal = n.ordinal;
+                let errorMsg = '';
+                if ((info.layer === 'gl0' || info.layer === 'ml0') && n.lastError) {
+                    const d = new Date(n.errorDate);
+                    // if under 8 hours ago
+                    if ((d.getTime() + (8 * 60 * 60 * 1000) > Date.now())) {
+                        const date = new Date(n.errorDate).toISOString();
+                        const timeAgo = formatTimeAgo(Date.now() - n.errorDate) || '';
+                        errorMsg = chalk.green(`   AUTO HEALED (${timeAgo}): `) + chalk.yellowBright(`${n.lastError} - ${date}`);
+                    }
+                }
+                tables.push({ error: errorMsg, label, row1: [row1], row2: [row2] });
+            }
+            this.renderNarrow(tables);
+            process.stdout.write("\n   * press any key to cancel");
+            // eslint-disable-next-line no-await-in-loop
+            await sleep(1);
+        }
+    }
+    async monitorWide() {
+        const projectInfos = this.getProjectInfo();
         const values = {};
-        const err = { date: '', layer: '', msg: '', timeAgo: '' };
+        // const err = { date: '', layer: '', msg:'', timeAgo: '' }
         while (true) {
             const rows = [];
-            for (const p of projects) {
-                if (!values[p.layer])
-                    values[p.layer] = {};
-                const n = fs.existsSync(p.path) ? JSON.parse(fs.readFileSync(p.path, 'utf8')) : {};
-                const projectName = p.project === 'hypergraph' ? '' : p.project;
-                const network = p.network === 'integrationnet' ? 'intnet' : p.network;
+            // const projects: { error: string; label: string, row1: string[][], row2: string[][] }[] = [];
+            let errorMsg = '';
+            for (const info of projectInfos) {
+                if (!values[info.layer])
+                    values[info.layer] = {};
+                const n = fs.existsSync(info.path) ? JSON.parse(fs.readFileSync(info.path, 'utf8')) : {};
+                const projectName = info.project === 'hypergraph' ? '' : ':' + info.project;
+                const network = info.network === 'integrationnet' ? 'intnet' : info.network;
                 const distance = Number.isNaN(n.clusterOrdinal - n.ordinal) ? '-' : String(n.clusterOrdinal - n.ordinal);
-                const ordinal = (values[p.layer].ordinal === n.ordinal) ? String(n.ordinal) : n.ordinal + ':true';
+                const ordinal = (values[info.layer].ordinal === n.ordinal) ? String(n.ordinal) : n.ordinal + ':true';
+                // const label = info.layer + ':' + network + projectName;
                 rows.push([
                     projectName + ':' + network,
-                    p.layer,
+                    info.layer,
                     n.session, // n.pilotSession,
                     n.state,
                     n.ordinal ? ordinal : '-',
@@ -71,30 +128,59 @@ export class StatusTable {
                     n.memUsage || '-',
                     n.error || '-'
                 ]);
-                values[p.layer].ordinal = n.ordinal;
-                if ((p.layer === 'gl0' || p.layer === 'ml0') && n.lastError) {
-                    err.layer = p.layer.toUpperCase();
-                    err.msg = n.lastError;
-                    err.date = new Date(n.errorDate).toISOString();
-                    err.timeAgo = formatTimeAgo(Date.now() - n.errorDate) || '';
+                values[info.layer].ordinal = n.ordinal;
+                if ((info.layer === 'gl0' || info.layer === 'ml0') && n.lastError) {
+                    const d = new Date(n.errorDate);
+                    // if under 8 hours ago
+                    if ((d.getTime() + (8 * 60 * 60 * 1000) > Date.now())) {
+                        const date = new Date(n.errorDate).toISOString();
+                        const timeAgo = formatTimeAgo(Date.now() - n.errorDate) || '';
+                        errorMsg = chalk.green(`   AUTO HEALED (${timeAgo}): `) + chalk.yellowBright(`${n.lastError} - ${date}`);
+                    }
                 }
+                // projects.push({ error: errorMsg, label, row1, row2 });
             }
-            let hasError = err.msg !== '';
-            if (hasError) {
-                const d = new Date(err.date);
-                // if under 8 hours ago
-                hasError = (d.getTime() + (8 * 60 * 60 * 1000) > Date.now());
-            }
-            this.render(rows, hasError);
-            if (hasError) {
-                process.stdout.write(chalk.green(`   AUTO HEALED (${err.timeAgo}): `) + chalk.red(`${err.layer}:${err.msg} - ${err.date}\n`));
-            }
-            process.stdout.write("   * press any key to cancel");
+            this.renderWide({ error: errorMsg, label: '', row1: rows, row2: [] });
+            process.stdout.write("\n   * press any key to cancel");
             // eslint-disable-next-line no-await-in-loop
             await sleep(1);
         }
     }
-    render(rows, hasError = false) {
+    renderNarrow(project) {
+        const options = { terminalAdapter: true };
+        // hide cursor
+        console.log("\u001B[?25l");
+        // wipe existing if already rendered
+        if (this.alreadyRendered) {
+            // move cursor up number to the top of the previous print before deleting
+            console.log(`\u001B[${this.previousHeight}A`);
+            // delete to end of terminal
+            console.log("\u001B[0J");
+        }
+        else {
+            this.alreadyRendered = true;
+        }
+        this.previousHeight = 0;
+        for (const p of project) {
+            const t1 = ttyTable(glHeader1, p.row1, options);
+            const t2 = ttyTable(glHeader2, p.row2, options);
+            process.stdout.write("   " + chalk.cyanBright(chalk.bold(p.label)));
+            process.stdout.write(t1.render());
+            process.stdout.write(t2.render());
+            // reset the previous height to the height of this output
+            // for when we next clear the print
+            // eslint-disable-next-line @typescript-eslint/ban-ts-comment
+            // @ts-expect-error
+            this.previousHeight += t1.height + t2.height + 1;
+            if (p.error) {
+                process.stdout.write("\n" + p.error + "\n");
+                this.previousHeight++;
+                this.previousHeight++;
+            }
+            console.log("");
+        }
+    }
+    renderWide(table) {
         const header = [...statusTableHeader];
         // const emptyColumns = Array.from({length: rows[0].length}).fill(0) as number[];
         // for (const [, row] of rows.entries()) {
@@ -112,13 +198,13 @@ export class StatusTable {
         //     }
         // }
         const options = { terminalAdapter: true };
-        const t1 = ttyTable(header, rows, options);
+        const t1 = ttyTable(header, table.row1, options);
         // hide cursor
         console.log("\u001B[?25l");
         // wipe existing if already rendered
         if (this.alreadyRendered) {
             // move cursor up number to the top of the previous print before deleting
-            console.log(`\u001B[${this.previousHeight + (hasError ? 4 : 3)}A`);
+            console.log(`\u001B[${this.previousHeight + 4}A`);
             // delete to end of terminal
             console.log("\u001B[0J");
         }
@@ -131,6 +217,10 @@ export class StatusTable {
         // eslint-disable-next-line @typescript-eslint/ban-ts-comment
         // @ts-expect-error
         this.previousHeight = t1.height;
+        if (table.error) {
+            console.log(table.error);
+            this.previousHeight++;
+        }
     }
 }
 function sleep(sec) {

package/oclif.manifest.json

@@ -111,6 +111,19 @@
         "<%= config.bin %> <%= command.id %>"
       ],
       "flags": {
+        "area": {
+          "char": "a",
+          "description": "area to view logs for. e.g. \"app\" or \"hc\" for health-check",
+          "name": "area",
+          "default": "hc",
+          "hasDynamicHelp": false,
+          "multiple": false,
+          "options": [
+            "app",
+            "hc"
+          ],
+          "type": "option"
+        },
         "follow": {
           "char": "f",
           "description": "continuously wait for additional data to be appended",
@@ -336,5 +349,5 @@
       ]
     }
   },
-  "version": "0.0.21"
+  "version": "0.2.0"
 }

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@constellation-network/node-pilot",
   "description": "An easy deployment and monitoring tool for Constellation nodes.",
-  "version": "0.0.21",
+  "version": "0.2.0",
   "author": "Frank Fox",
   "bin": {
     "cpilot": "bin/run.js"
@@ -39,6 +39,7 @@
     "json-bigint-patch": "^0.0.8",
     "node-localstorage": "^3.0.5",
     "ora": "^8.2.0",
+    "semver": "^7.7.3",
     "shelljs": "^0.10.0",
     "tty-table": "^4.2.3",
     "yaml": "^2.8.1"
@@ -51,6 +52,7 @@
     "@types/mocha": "^10",
     "@types/node": "^22.17.1",
     "@types/node-localstorage": "^1.3.3",
+    "@types/semver": "^7.7.1",
     "@types/shelljs": "^0.8.17",
     "chai": "^4",
     "eslint": "^9",

package/projects/hypergraph/Dockerfile

@@ -39,7 +39,7 @@ COPY dist/gl0.jar /app/jars/gl0.jar
 #COPY ./health-check /health-check
 #RUN chmod +x /health-check/bin/run.sh
 #RUN chmod +x /health-check/bin/hydrate.sh
-RUN npm install -g "@constellation-network/node-pilot-health-check@0.0.14"
+RUN npm install -g "@constellation-network/node-pilot-health-check@0.0.16"
 
 # Add entrypoint
 COPY ./entrypoint.sh /app/entrypoint.sh