@northflare/runner 0.0.1

package/src/utils/config.ts

@@ -0,0 +1,315 @@
+/**
+ * Configuration management utilities
+ */
+
+import { RunnerConfig, EnvironmentConfig, RetryStrategy } from "../types";
+import fs from "fs/promises";
+import path from "path";
+import { createLogger } from "./logger";
+import { Command } from "commander";
+
+const logger = createLogger("ConfigManager");
+
+export class ConfigManager {
+  private static DEFAULT_CONFIG: Partial<RunnerConfig> = {
+    dataDir: "./data",
+    heartbeatInterval: 120000, // 2 minutes
+    retryStrategy: "exponential" as RetryStrategy,
+    retryIntervalSecs: 60,
+    retryDurationSecs: 900,
+  };
+
+  /**
+   * Load configuration without parsing command line arguments
+   * Used when arguments have already been parsed by the CLI
+   */
+  static async loadConfig(configPath?: string): Promise<RunnerConfig> {
+    // Validate required environment variables
+    this.validateEnvironment();
+
+    // Start with defaults
+    let config: Partial<RunnerConfig> = { ...this.DEFAULT_CONFIG };
+
+    // Determine config path - use provided path or default location
+    let effectiveConfigPath = configPath;
+    if (!effectiveConfigPath) {
+      // Try to use default config location
+      try {
+        const envPaths = require("env-paths").default || require("env-paths");
+        const paths = envPaths("northflare-runner", { suffix: "" });
+        const defaultConfigPath = path.join(paths.config, "config.json");
+
+        // Check if default config exists
+        const fs = require("fs");
+        if (fs.existsSync(defaultConfigPath)) {
+          effectiveConfigPath = defaultConfigPath;
+          logger.info(`Using default config file: ${defaultConfigPath}`);
+        }
+      } catch (error) {
+        // env-paths not available or error accessing default location
+        logger.debug("Could not check default config location:", error);
+      }
+    }
+
+    // Load from config file if we have a path
+    if (effectiveConfigPath) {
+      const fileConfig = await this.loadConfigFile(effectiveConfigPath);
+      config = { ...config, ...fileConfig };
+    }
+
+    // Override with environment variables
+    const envConfig = this.loadFromEnvironment();
+    config = { ...config, ...envConfig };
+
+    // Validate final configuration
+    this.validateConfig(config);
+
+    return config as RunnerConfig;
+  }
+
+  /**
+   * Parse command line arguments and load configuration
+   */
+  static async parseArgsAndLoadConfig(argv: string[]): Promise<RunnerConfig> {
+    const program = new Command();
+
+    program
+      .name("northflare-runner")
+      .description(
+        "Northflare Runner - Executes Claude agents for task processing"
+      )
+      .version("1.0.0")
+      .option("-c, --config <path>", "Path to configuration file")
+      .option(
+        "--retry-strategy <strategy>",
+        "Registration retry strategy (none, interval, exponential)",
+        "exponential"
+      )
+      .option(
+        "--retry-interval-secs <seconds>",
+        "Retry interval in seconds for interval strategy",
+        "60"
+      )
+      .option(
+        "--retry-duration-secs <seconds>",
+        "Max retry duration in seconds for exponential strategy",
+        "900"
+      )
+      .option("--data-dir <path>", "Data directory path", "./data")
+      .option(
+        "--heartbeat-interval <ms>",
+        "Heartbeat interval in milliseconds",
+        "120000"
+      );
+
+    program.parse(argv);
+    const options = program.opts();
+
+    // Validate required environment variables
+    this.validateEnvironment();
+
+    // Start with defaults
+    let config: Partial<RunnerConfig> = { ...this.DEFAULT_CONFIG };
+
+    // Load from config file if provided
+    if (options["config"]) {
+      const fileConfig = await this.loadConfigFile(options["config"]);
+      config = { ...config, ...fileConfig };
+    }
+
+    // Override with environment variables
+    const envConfig = this.loadFromEnvironment();
+    config = { ...config, ...envConfig };
+
+    // Override with CLI arguments (highest priority)
+    if (options["retryStrategy"]) {
+      config.retryStrategy = options["retryStrategy"] as RetryStrategy;
+    }
+    if (options["retryIntervalSecs"]) {
+      config.retryIntervalSecs = parseInt(options["retryIntervalSecs"]);
+    }
+    if (options["retryDurationSecs"]) {
+      config.retryDurationSecs = parseInt(options["retryDurationSecs"]);
+    }
+    if (options["dataDir"]) {
+      config.dataDir = options["dataDir"];
+    }
+    if (options["heartbeatInterval"]) {
+      config.heartbeatInterval = parseInt(options["heartbeatInterval"]);
+    }
+
+    // Validate final configuration
+    this.validateConfig(config);
+
+    return config as RunnerConfig;
+  }
+
+  /**
+   * Validate required environment variables
+   */
+  private static validateEnvironment(): void {
+    // Set default for NORTHFLARE_WORKSPACE_DIR if not provided
+    if (!process.env["NORTHFLARE_WORKSPACE_DIR"]) {
+      try {
+        const envPaths = require("env-paths").default || require("env-paths");
+        const paths = envPaths("northflare-runner", { suffix: "" });
+        process.env["NORTHFLARE_WORKSPACE_DIR"] = paths.data;
+      } catch (error) {
+        // Fallback to original default if env-paths is not available
+        process.env["NORTHFLARE_WORKSPACE_DIR"] = "/workspace";
+      }
+    }
+
+    // Set default for NORTHFLARE_ORCHESTRATOR_URL if not provided
+    if (!process.env["NORTHFLARE_ORCHESTRATOR_URL"]) {
+      process.env["NORTHFLARE_ORCHESTRATOR_URL"] = "https://api.northflare.ai";
+    }
+
+    const required = [
+      "NORTHFLARE_RUNNER_TOKEN",
+      "NORTHFLARE_WORKSPACE_DIR",
+      "NORTHFLARE_ORCHESTRATOR_URL",
+    ];
+
+    const missing = required.filter((key) => !process.env[key]);
+
+    if (missing.length > 0) {
+      throw new Error(
+        `Missing required environment variables: ${missing.join(", ")}\n` +
+          "Please set these environment variables before starting the runner."
+      );
+    }
+  }
+
+  /**
+   * Load configuration from environment variables
+   */
+  private static loadFromEnvironment(): Partial<RunnerConfig> {
+    const config: Partial<RunnerConfig> = {
+      orchestratorUrl: process.env["NORTHFLARE_ORCHESTRATOR_URL"]!,
+    };
+
+    // Optional environment overrides
+    if (process.env["NORTHFLARE_DATA_DIR"]) {
+      config.dataDir = process.env["NORTHFLARE_DATA_DIR"];
+    }
+
+    return config;
+  }
+
+  /**
+   * Load configuration from file
+   */
+  private static async loadConfigFile(
+    configPath: string
+  ): Promise<Partial<RunnerConfig>> {
+    try {
+      const absolutePath = path.resolve(configPath);
+      const content = await fs.readFile(absolutePath, "utf-8");
+
+      // Support both JSON and YAML formats
+      if (configPath.endsWith(".json")) {
+        const config = JSON.parse(content);
+
+        // Log if runnerRepos are found in config
+        if (config.runnerRepos && Array.isArray(config.runnerRepos)) {
+          logger.info(
+            `Found ${config.runnerRepos.length} runner repos in config file`
+          );
+        }
+
+        return config;
+      } else if (configPath.endsWith(".yaml") || configPath.endsWith(".yml")) {
+        // For YAML support, we'd need to add a yaml parser dependency
+        throw new Error("YAML configuration files are not yet supported");
+      } else {
+        throw new Error("Configuration file must be .json format");
+      }
+    } catch (error) {
+      if ((error as any).code === "ENOENT") {
+        logger.warn(`Configuration file not found: ${configPath}`);
+        return {};
+      }
+      throw error;
+    }
+  }
+
+  /**
+   * Validate final configuration
+   */
+  private static validateConfig(config: Partial<RunnerConfig>): void {
+    // Note: Runner ID will be generated by server during registration
+
+    if (!config.orchestratorUrl) {
+      throw new Error("orchestratorUrl is required");
+    }
+
+    if (!config.dataDir) {
+      throw new Error("dataDir is required");
+    }
+
+    if (!config.heartbeatInterval || config.heartbeatInterval < 1000) {
+      throw new Error("heartbeatInterval must be at least 1000ms");
+    }
+
+    // Validate retry strategy
+    const validStrategies: RetryStrategy[] = [
+      "none",
+      "interval",
+      "exponential",
+    ];
+    if (
+      !config.retryStrategy ||
+      !validStrategies.includes(config.retryStrategy)
+    ) {
+      throw new Error(
+        `retryStrategy must be one of: ${validStrategies.join(", ")}`
+      );
+    }
+
+    if (
+      config.retryStrategy === "interval" &&
+      (!config.retryIntervalSecs || config.retryIntervalSecs < 1)
+    ) {
+      throw new Error(
+        "retryIntervalSecs must be at least 1 when using interval strategy"
+      );
+    }
+
+    if (
+      config.retryStrategy === "exponential" &&
+      (!config.retryDurationSecs || config.retryDurationSecs < 1)
+    ) {
+      throw new Error(
+        "retryDurationSecs must be at least 1 when using exponential strategy"
+      );
+    }
+  }
+
+  /**
+   * Get all environment configuration
+   */
+  static getEnvironmentConfig(): EnvironmentConfig {
+    return {
+      NORTHFLARE_RUNNER_TOKEN: process.env["NORTHFLARE_RUNNER_TOKEN"]!,
+      NORTHFLARE_WORKSPACE_DIR: process.env["NORTHFLARE_WORKSPACE_DIR"]!,
+      NORTHFLARE_ORCHESTRATOR_URL: process.env["NORTHFLARE_ORCHESTRATOR_URL"]!,
+      DEBUG: process.env["DEBUG"],
+    };
+  }
+
+  /**
+   * Save configuration file with updated data
+   */
+  static async saveConfigFile(configPath: string, config: any): Promise<void> {
+    try {
+      const absolutePath = path.resolve(configPath);
+      const content = JSON.stringify(config, null, 2);
+      await fs.writeFile(absolutePath, content, "utf-8");
+      logger.info(`Updated configuration file: ${absolutePath}`);
+    } catch (error) {
+      logger.error("Failed to save configuration file", error);
+      throw error;
+    }
+  }
+}

package/src/utils/console.ts

@@ -0,0 +1,13 @@
+/**
+ * Console wrapper that suppresses output when not in debug mode
+ */
+
+const isDebug = process.env["DEBUG"] === "true";
+
+export const console = {
+  log: isDebug ? global.console.log.bind(global.console) : () => {},
+  warn: isDebug ? global.console.warn.bind(global.console) : () => {},
+  error: global.console.error.bind(global.console), // Always show errors
+  info: isDebug ? global.console.info.bind(global.console) : () => {},
+  debug: isDebug ? global.console.debug.bind(global.console) : () => {},
+};

package/src/utils/expand-env.ts

@@ -0,0 +1,22 @@
+export function expandEnv(
+  obj: any,
+  env: Record<string, string | undefined> = process.env
+): any {
+  if (typeof obj === "string") {
+    return obj.replace(/\$\{([^}]+)\}/g, (match, key) => env[key] || match);
+  }
+
+  if (Array.isArray(obj)) {
+    return obj.map((item) => expandEnv(item, env));
+  }
+
+  if (obj && typeof obj === "object") {
+    const result: Record<string, any> = {};
+    for (const [key, value] of Object.entries(obj)) {
+      result[key] = expandEnv(value, env);
+    }
+    return result;
+  }
+
+  return obj;
+}

package/src/utils/logger.ts

@@ -0,0 +1,131 @@
+/**
+ * Logger configuration using Winston
+ */
+
+import winston from "winston";
+import path from "path";
+
+// Custom log levels
+const levels = {
+  error: 0,
+  warn: 1,
+  info: 2,
+  http: 3,
+  verbose: 4,
+  debug: 5,
+  silly: 6,
+};
+
+// Log colors
+const colors = {
+  error: "red",
+  warn: "yellow",
+  info: "green",
+  http: "magenta",
+  verbose: "cyan",
+  debug: "blue",
+  silly: "gray",
+};
+
+winston.addColors(colors);
+
+// Format for console output
+const consoleFormat = winston.format.combine(
+  winston.format.colorize({ all: true }),
+  winston.format.timestamp({ format: "YYYY-MM-DD HH:mm:ss" }),
+  winston.format.printf(({ timestamp, level, message, stack, ...metadata }) => {
+    let msg = `${timestamp} [${level}]: ${message}`;
+    // Only show stack traces and metadata in debug mode
+    if (process.env["DEBUG"] === "true") {
+      if (stack) {
+        msg += `\n${stack}`;
+      }
+      const filteredMetadata = { ...metadata };
+      delete filteredMetadata["component"]; // Keep component info
+      if (Object.keys(filteredMetadata).length > 0) {
+        msg += ` ${JSON.stringify(filteredMetadata)}`;
+      }
+    }
+    return msg;
+  })
+);
+
+// Format for file output
+const fileFormat = winston.format.combine(
+  winston.format.timestamp({ format: "YYYY-MM-DD HH:mm:ss" }),
+  winston.format.errors({ stack: true }),
+  winston.format.json()
+);
+
+// Create logger instance
+export const logger = winston.createLogger({
+  level: process.env["DEBUG"] === "true" ? "debug" : "info",
+  levels,
+  transports: [
+    // Always include console transport for errors
+    new winston.transports.Console({
+      format: consoleFormat,
+      level: process.env["DEBUG"] === "true" ? "debug" : "error",
+    }),
+  ],
+});
+
+// Add file transports if log directory is configured
+export function configureFileLogging(logDir: string): void {
+  // Error log file
+  logger.add(
+    new winston.transports.File({
+      filename: path.join(logDir, "error.log"),
+      level: "error",
+      format: fileFormat,
+      maxsize: 10 * 1024 * 1024, // 10MB
+      maxFiles: 5,
+    })
+  );
+
+  // Combined log file
+  logger.add(
+    new winston.transports.File({
+      filename: path.join(logDir, "combined.log"),
+      format: fileFormat,
+      maxsize: 50 * 1024 * 1024, // 50MB
+      maxFiles: 10,
+    })
+  );
+
+  // Debug log file (only in debug mode)
+  if (process.env["DEBUG"] === "true") {
+    logger.add(
+      new winston.transports.File({
+        filename: path.join(logDir, "debug.log"),
+        level: "debug",
+        format: fileFormat,
+        maxsize: 100 * 1024 * 1024, // 100MB
+        maxFiles: 3,
+      })
+    );
+  }
+}
+
+// Create child logger for specific components
+export function createLogger(component: string): winston.Logger {
+  return logger.child({ component });
+}
+
+// Log unhandled errors
+logger.exceptions.handle(
+  new winston.transports.File({
+    filename: "exceptions.log",
+    format: fileFormat,
+  })
+);
+
+logger.rejections.handle(
+  new winston.transports.File({
+    filename: "rejections.log",
+    format: fileFormat,
+  })
+);
+
+// Export default logger
+export default logger;

package/src/utils/sdk-demo.js

@@ -0,0 +1,34 @@
+import { query } from "./codex-sdk.js";
+
+export default async function getAllOutput(prompt = "Hello!", options = {}) {
+  const defaultOptions = {
+    cwd: "/Users/toby/Code/hypervisual",
+    configOverrides: {
+      model: "gpt-5-codex",
+      model_reasoning_effort: "high",
+      sandbox_mode: "danger-full-access",
+    },
+  };
+
+  const q = query({
+    prompt,
+    options: { ...defaultOptions, ...options },
+  });
+
+  const messages = [];
+  for await (const message of q) {
+    messages.push(message);
+    console.log(message);
+
+    // Check if we've received a task completion
+    if (message.msg?.type === "task_complete") {
+      break;
+    }
+  }
+
+  // Send shutdown command and close
+  await q.shutdown();
+  await q.close();
+
+  return messages;
+}

package/src/utils/status-line.ts

@@ -0,0 +1,121 @@
+/**
+ * StatusLineManager - Manages persistent status line output for active Claude Code processes
+ * 
+ * This component maintains a single-line status output that updates every minute to show
+ * the count of active Claude Code processes. It prevents the machine from entering idle
+ * state while processes are running and provides clear visual feedback without cluttering
+ * the terminal with multiple log lines.
+ */
+
+export class StatusLineManager {
+  private intervalId?: NodeJS.Timeout;
+  private activeCount: number = 0;
+  private lastLine: string = "";
+  private isEnabled: boolean = true;
+
+  constructor() {
+    // Only enable status line when not in debug mode
+    this.isEnabled = process.env["DEBUG"] !== "true";
+  }
+
+  /**
+   * Updates the count of active conversations
+   */
+  updateActiveCount(count: number): void {
+    const wasZero = this.activeCount === 0;
+    const isZero = count === 0;
+    this.activeCount = count;
+
+    // Start interval when transitioning from 0 to active
+    if (wasZero && !isZero) {
+      this.startInterval();
+    }
+    // Stop interval when transitioning to 0
+    else if (!wasZero && isZero) {
+      this.stopInterval();
+    }
+  }
+
+  /**
+   * Starts the status line interval
+   */
+  private startInterval(): void {
+    if (!this.isEnabled || this.intervalId) return;
+
+    // Show initial status immediately
+    this.showStatus();
+
+    // Update every minute on the minute
+    const now = new Date();
+    const msUntilNextMinute = (60 - now.getSeconds()) * 1000 - now.getMilliseconds();
+
+    // First timeout to align with the minute
+    setTimeout(() => {
+      this.showStatus();
+      
+      // Then set up regular interval
+      this.intervalId = setInterval(() => {
+        this.showStatus();
+      }, 60000); // Every 60 seconds
+    }, msUntilNextMinute);
+  }
+
+  /**
+   * Stops the status line interval
+   */
+  private stopInterval(): void {
+    if (this.intervalId) {
+      clearInterval(this.intervalId);
+      this.intervalId = undefined;
+    }
+    
+    // Show final "No active" message
+    this.showStatus();
+  }
+
+  /**
+   * Shows the current status
+   */
+  private showStatus(): void {
+    if (!this.isEnabled) return;
+
+    const now = new Date();
+    const timeStr = now.toTimeString().slice(0, 5); // HH:MM format
+
+    let newLine: string;
+    if (this.activeCount === 0) {
+      newLine = "No active Claude Code processes";
+    } else if (this.activeCount === 1) {
+      newLine = `${timeStr} 1 active Claude Code process`;
+    } else {
+      newLine = `${timeStr} ${this.activeCount} active Claude Code processes`;
+    }
+
+    // If this is the first status line, position it
+    if (!this.lastLine) {
+      // Move to start of line and save cursor position
+      process.stdout.write('\r');
+    } else {
+      // Clear the current line
+      process.stdout.write('\r' + ' '.repeat(this.lastLine.length) + '\r');
+    }
+    
+    // Write the new status without newline
+    process.stdout.write(newLine);
+    this.lastLine = newLine;
+  }
+
+  /**
+   * Cleans up the status line manager
+   */
+  dispose(): void {
+    this.stopInterval();
+    // Add a newline to ensure next output starts on a new line
+    if (this.lastLine) {
+      process.stdout.write('\n');
+    }
+  }
+}
+
+// Singleton instance
+export const statusLineManager = new StatusLineManager();

package/test-debug.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+
+# Test script to demonstrate debug logging in the runner
+
+echo "Starting runner in debug mode to demonstrate enhanced logging..."
+echo "This will show:"
+echo "  - runnerUid on initialization"
+echo "  - lastProcessedAt state for ElectricSQL connection"
+echo "  - ElectricSQL connection details"
+echo "  - Message processing decisions with ownership info"
+echo ""
+echo "Set DEBUG=true to enable verbose logging"
+echo ""
+
+# Run the runner with debug mode enabled
+DEBUG=true npm run start -- --config ./config.json 2>&1 | head -100
+
+echo ""
+echo "Note: The runner will display debug logs including:"
+echo "  - Runner UID and ownership details during registration"
+echo "  - LastProcessedAt watermark for message filtering"
+echo "  - ElectricSQL collection creation with connection details"
+echo "  - Message processing decisions with reasons"
+echo ""
+echo "To run the runner with debug logging enabled, use:"
+echo "  DEBUG=true npm run start"