@northflare/runner 0.0.1

package/src/components/repository-manager.ts

@@ -0,0 +1,337 @@
+/**
+ * RepositoryManager - Manages Git repository checkouts and workspace isolation
+ *
+ * This component handles all Git operations for the runner, including:
+ * - Cloning repositories with authentication
+ * - Updating repositories to latest state
+ * - Workspace isolation via separate directories
+ * - Cleanup of old repositories
+ * - Error handling for Git operations
+ */
+
+import { IRunnerApp } from "../types/runner-interface";
+import { WorkspaceRepository } from "../types";
+import path from "path";
+import fs from "fs/promises";
+import { mkdirSync } from "fs";
+import simpleGit, { SimpleGit, GitError } from "simple-git";
+import { console } from "../utils/console";
+
+export class RepositoryManager {
+  private repositories: Map<string, WorkspaceRepository>;
+  protected repoBasePath: string;
+  private runner: IRunnerApp;
+  private git: SimpleGit;
+
+  constructor(runner: IRunnerApp) {
+    this.runner = runner;
+    this.repositories = new Map();
+
+    // Use workspaceDir from config if available, otherwise use environment variable
+    this.repoBasePath =
+      runner.config_.workspaceDir ||
+      process.env["NORTHFLARE_WORKSPACE_DIR"] ||
+      "/tmp/northflare-workspace";
+    if (!this.repoBasePath) {
+      throw new Error(
+        "NORTHFLARE_WORKSPACE_DIR environment variable must be set"
+      );
+    }
+
+    // Ensure workspace directory exists before initializing simple-git
+    try {
+      mkdirSync(this.repoBasePath, { recursive: true });
+    } catch (error) {
+      console.error(
+        `Failed to create workspace directory ${this.repoBasePath}:`,
+        error
+      );
+      throw error;
+    }
+
+    // Initialize simple-git
+    this.git = simpleGit({
+      baseDir: this.repoBasePath,
+      binary: "git",
+      maxConcurrentProcesses: 3,
+      config: ["user.name=Northflare", "user.email=runner@northflare.ai"],
+    });
+  }
+
+  async checkoutRepository(
+    workspaceId: string,
+    repoUrl: string,
+    branch: string,
+    githubToken?: string
+  ): Promise<string> {
+    const existing = this.repositories.get(workspaceId);
+    if (
+      existing &&
+      existing.repoUrl === repoUrl &&
+      existing.branch === branch
+    ) {
+      await this.updateRepository(existing, githubToken);
+      return existing.localPath;
+    }
+
+    const localPath = path.join(this.repoBasePath, workspaceId);
+
+    // Clean up existing directory if it exists
+    await this.cleanupDirectory(localPath);
+
+    // Clone with authentication
+    const authUrl = this.getAuthenticatedUrl(repoUrl, githubToken);
+
+    try {
+      console.log(`Cloning repository for workspace ${workspaceId}...`);
+      await this.executeGit(["clone", "--branch", branch, authUrl, localPath]);
+
+      // Configure the repository
+      const repoGit = simpleGit(localPath);
+      await repoGit.addConfig("credential.helper", "store");
+
+      const repo: WorkspaceRepository = {
+        workspaceId,
+        repoUrl,
+        branch,
+        localPath,
+        lastAccessed: new Date(),
+      };
+
+      this.repositories.set(workspaceId, repo);
+      console.log(
+        `Successfully cloned repository for workspace ${workspaceId}`
+      );
+      return localPath;
+    } catch (error) {
+      console.error(
+        `Failed to clone repository for workspace ${workspaceId}:`,
+        error
+      );
+      await this.cleanupDirectory(localPath);
+      throw error;
+    }
+  }
+
+  /**
+   * Checkout a local repository (bypasses Git operations)
+   * Used for local workspaces that point to existing directories on the filesystem
+   */
+  async checkoutLocalRepository(
+    workspaceId: string,
+    localRepoPath: string
+  ): Promise<string> {
+    console.log(
+      `Using local repository for workspace ${workspaceId}: ${localRepoPath}`
+    );
+
+    // Verify the path exists
+    try {
+      const stats = await fs.stat(localRepoPath);
+      if (!stats.isDirectory()) {
+        throw new Error(`Path is not a directory: ${localRepoPath}`);
+      }
+    } catch (error) {
+      console.error(`Local repository path not found: ${localRepoPath}`);
+      throw new Error(`Local repository path not found: ${localRepoPath}`);
+    }
+
+    // Store in repositories map for consistency
+    const repo: WorkspaceRepository = {
+      workspaceId,
+      repoUrl: `file://${localRepoPath}`, // Use file:// protocol to indicate local
+      branch: "local", // Local repos don't have branches in the same sense
+      localPath: localRepoPath,
+      lastAccessed: new Date(),
+    };
+
+    this.repositories.set(workspaceId, repo);
+    console.log(
+      `Successfully set up local repository for workspace ${workspaceId}`
+    );
+    return localRepoPath;
+  }
+
+  async getWorkspacePath(workspaceId: string): Promise<string> {
+    const localPath = path.join(this.repoBasePath, workspaceId);
+    await this.ensureDirectory(localPath);
+    return localPath;
+  }
+
+  private async updateRepository(
+    repo: WorkspaceRepository,
+    githubToken?: string
+  ): Promise<void> {
+    repo.lastAccessed = new Date();
+
+    // Skip Git operations for local repositories
+    if (repo.repoUrl.startsWith("file://")) {
+      console.log(
+        `Local repository for workspace ${repo.workspaceId} - no update needed`
+      );
+      return;
+    }
+
+    const repoGit = simpleGit(repo.localPath);
+
+    try {
+      console.log(`Updating repository for workspace ${repo.workspaceId}...`);
+
+      // Update remote URL if token has changed
+      if (githubToken) {
+        const authUrl = this.getAuthenticatedUrl(repo.repoUrl, githubToken);
+        await repoGit.remote(["set-url", "origin", authUrl]);
+      }
+
+      // Fetch latest changes
+      await repoGit.fetch("origin");
+
+      // Reset to latest branch state
+      await repoGit.reset(["--hard", `origin/${repo.branch}`]);
+
+      // Clean untracked files
+      await repoGit.clean("f", ["-d"]);
+
+      console.log(
+        `Successfully updated repository for workspace ${repo.workspaceId}`
+      );
+    } catch (error) {
+      console.error(
+        `Failed to update repository for workspace ${repo.workspaceId}:`,
+        error
+      );
+      throw error;
+    }
+  }
+
+  async cleanupRepository(workspaceId: string): Promise<void> {
+    const repo = this.repositories.get(workspaceId);
+    if (!repo) return;
+
+    console.log(`Cleaning up repository for workspace ${workspaceId}...`);
+
+    // For local repositories, only remove from tracking, don't delete the directory
+    if (repo.repoUrl.startsWith("file://")) {
+      console.log(`Local repository - removing from tracking only`);
+      this.repositories.delete(workspaceId);
+      return;
+    }
+
+    // Remove local directory for cloned repositories
+    await this.cleanupDirectory(repo.localPath);
+
+    this.repositories.delete(workspaceId);
+  }
+
+  async cleanupOldRepositories(
+    maxAge: number = 7 * 24 * 60 * 60 * 1000
+  ): Promise<void> {
+    const now = new Date();
+    const toCleanup: string[] = [];
+
+    for (const [workspaceId, repo] of this.repositories.entries()) {
+      const age = now.getTime() - repo.lastAccessed.getTime();
+      if (age > maxAge) {
+        toCleanup.push(workspaceId);
+      }
+    }
+
+    for (const workspaceId of toCleanup) {
+      await this.cleanupRepository(workspaceId);
+    }
+
+    console.log(`Cleaned up ${toCleanup.length} old repositories`);
+  }
+
+  protected async executeGit(args: string[], cwd?: string): Promise<void> {
+    const git = cwd ? simpleGit(cwd) : this.git;
+    try {
+      await git.raw(args);
+    } catch (error) {
+      if (error instanceof GitError) {
+        throw new Error(`Git operation failed: ${error.message}`);
+      }
+      throw error;
+    }
+  }
+
+  protected getAuthenticatedUrl(repoUrl: string, githubToken?: string): string {
+    if (!githubToken) {
+      return repoUrl;
+    }
+
+    // Handle different URL formats
+    if (repoUrl.startsWith("https://")) {
+      // HTTPS URL
+      return repoUrl.replace("https://", `https://${githubToken}@`);
+    } else if (repoUrl.startsWith("git@")) {
+      // SSH URL - convert to HTTPS with token
+      const httpsUrl = repoUrl
+        .replace("git@github.com:", "https://github.com/")
+        .replace(".git", "");
+      return `https://${githubToken}@${httpsUrl.replace("https://", "")}.git`;
+    }
+
+    return repoUrl;
+  }
+
+  protected async ensureDirectory(dirPath: string): Promise<void> {
+    try {
+      await fs.mkdir(dirPath, { recursive: true });
+    } catch (error) {
+      console.error(`Failed to create directory ${dirPath}:`, error);
+      throw error;
+    }
+  }
+
+  protected async cleanupDirectory(dirPath: string): Promise<void> {
+    try {
+      const stats = await fs.stat(dirPath).catch(() => null);
+      if (stats) {
+        await fs.rm(dirPath, { recursive: true, force: true });
+      }
+    } catch (error) {
+      console.error(`Failed to cleanup directory ${dirPath}:`, error);
+      // Don't throw - cleanup errors are not critical
+    }
+  }
+
+  /**
+   * Get statistics about managed repositories
+   */
+  getRepositoryStats(): {
+    total: number;
+    workspaceIds: string[];
+    oldestAccess: Date | null;
+    newestAccess: Date | null;
+  } {
+    const repos = Array.from(this.repositories.values());
+
+    return {
+      total: repos.length,
+      workspaceIds: Array.from(this.repositories.keys()),
+      oldestAccess:
+        repos.length > 0
+          ? new Date(Math.min(...repos.map((r) => r.lastAccessed.getTime())))
+          : null,
+      newestAccess:
+        repos.length > 0
+          ? new Date(Math.max(...repos.map((r) => r.lastAccessed.getTime())))
+          : null,
+    };
+  }
+
+  /**
+   * Check if a workspace has an existing repository
+   */
+  hasRepository(workspaceId: string): boolean {
+    return this.repositories.has(workspaceId);
+  }
+
+  /**
+   * Get repository info for a workspace
+   */
+  getRepository(workspaceId: string): WorkspaceRepository | undefined {
+    return this.repositories.get(workspaceId);
+  }
+}

package/src/index.ts

@@ -0,0 +1,166 @@
+/**
+ * Main entry point for the Northflare Runner App
+ */
+
+import { RunnerApp } from "./runner-sse";
+import { ConfigManager } from "./utils/config";
+import { logger, configureFileLogging } from "./utils/logger";
+import path from "path";
+import fs from "fs/promises";
+
+let runner: RunnerApp | null = null;
+
+async function main() {
+  try {
+    logger.info("Starting Northflare Runner...");
+
+    // Load configuration (args already parsed by CLI)
+    let configPath = process.argv[2]; // This is set by the CLI if --config was provided
+
+    // If no config path provided, check for default location
+    if (!configPath) {
+      try {
+        const envPaths = require("env-paths").default || require("env-paths");
+        const paths = envPaths("northflare-runner", { suffix: "" });
+        const defaultConfigPath = path.join(paths.config, "config.json");
+
+        // Check if default config exists
+        if (
+          await fs
+            .access(defaultConfigPath)
+            .then(() => true)
+            .catch(() => false)
+        ) {
+          configPath = defaultConfigPath;
+        }
+      } catch (error) {
+        // env-paths not available or error accessing default location
+      }
+    }
+
+    const config = await ConfigManager.loadConfig(configPath);
+
+    // Set up file logging
+    const logDir = path.join(config.dataDir, "logs");
+    await fs.mkdir(logDir, { recursive: true });
+    configureFileLogging(logDir);
+
+    logger.info("Configuration loaded", {
+      orchestratorUrl: config.orchestratorUrl,
+      retryStrategy: config.retryStrategy,
+      retryIntervalSecs: config.retryIntervalSecs,
+      retryDurationSecs: config.retryDurationSecs,
+    });
+
+    // Additional debug logging
+    if (process.env["DEBUG"] === "true") {
+      logger.debug("Debug mode enabled - verbose logging active", {
+        dataDir: config.dataDir,
+        heartbeatInterval: config.heartbeatInterval,
+        nodeVersion: process.version,
+        platform: process.platform,
+        pid: process.pid,
+      });
+    }
+
+    // Create and start runner - pass the resolved config path
+    runner = new RunnerApp(config, configPath);
+    await runner.start();
+
+    // Set up graceful shutdown handlers
+    setupShutdownHandlers();
+
+    logger.info("Northflare Runner is running", {
+      runnerId: runner.getRunnerId() || "pending registration",
+    });
+
+    // Log additional details in debug mode
+    if (process.env["DEBUG"] === "true" && runner) {
+      logger.debug("Runner started with full details", {
+        runnerId: runner.getRunnerId(),
+        runnerUid: runner.getRunnerUid(),
+        lastProcessedAt: runner.getLastProcessedAt()?.toISOString() || "null",
+        isActiveRunner: runner.getIsActiveRunner(),
+        uptime: process.uptime(),
+        memoryUsage: process.memoryUsage(),
+      });
+    }
+  } catch (error) {
+    logger.error("Failed to start runner:", error);
+    process.exit(1);
+  }
+}
+
+function setupShutdownHandlers() {
+  let isShuttingDown = false;
+
+  // Handle various shutdown signals
+  const signals: NodeJS.Signals[] = ["SIGTERM", "SIGINT", "SIGHUP"];
+
+  for (const signal of signals) {
+    process.on(signal, async () => {
+      if (isShuttingDown) {
+        logger.info(`Already shutting down, ignoring ${signal}`);
+        return;
+      }
+      isShuttingDown = true;
+      logger.info(`Received ${signal}, shutting down gracefully...`);
+      await shutdown();
+    });
+  }
+
+  // Handle uncaught exceptions
+  process.on("uncaughtException", async (error) => {
+    logger.error("Uncaught exception:", error);
+    if (!isShuttingDown) {
+      isShuttingDown = true;
+      await shutdown(1);
+    }
+  });
+
+  // Handle unhandled promise rejections
+  process.on("unhandledRejection", async (reason, promise) => {
+    logger.error("Unhandled rejection", { promise, reason });
+    if (!isShuttingDown) {
+      isShuttingDown = true;
+      await shutdown(1);
+    }
+  });
+
+  // Handle process warnings
+  process.on("warning", (warning) => {
+    logger.warn("Process warning", {
+      name: warning.name,
+      message: warning.message,
+      stack: warning.stack,
+    });
+  });
+}
+
+async function shutdown(exitCode: number = 0) {
+  if (runner) {
+    try {
+      await runner.stop();
+      logger.info("Runner stopped successfully");
+    } catch (error) {
+      logger.error("Error during shutdown:", error);
+      exitCode = 1;
+    }
+  }
+
+  process.exit(exitCode);
+}
+
+// Start the runner if this is the main module
+if (require.main === module) {
+  main().catch((error) => {
+    logger.error("Fatal error:", error);
+    process.exit(1);
+  });
+}
+
+// Export for programmatic usage
+export { RunnerApp } from "./runner-sse";
+export { ConfigManager } from "./utils/config";
+export * from "./types";
+export { main };