@forwardimpact/libutil 0.1.60

package/finder.js

@@ -0,0 +1,148 @@
+import fs from "fs";
+import fsAsync from "fs/promises";
+import path from "path";
+import { createRequire } from "node:module";
+
+/**
+ * Finder class for project path resolution and symlink management
+ * Handles filesystem operations for linking generated code to packages
+ */
+export class Finder {
+  #logger;
+  #process;
+
+  /**
+   * Creates a new Finder instance
+   * @param {object} fs - Filesystem module (fs/promises)
+   * @param {object} logger - Logger instance for debug output
+   * @param {object} process - Process environment access (for testing)
+   */
+  constructor(fs, logger, process = global.process) {
+    if (!fs) throw new Error("fs is required");
+    if (!logger) throw new Error("logger is required");
+    if (!process) throw new Error("process is required");
+
+    this.#logger = logger;
+    this.#process = process;
+  }
+
+  /**
+   * Searches upward from one or more roots for a target file or directory.
+   * @param {string} root - Starting directory to search from
+   * @param {string} relativePath - Relative path to append while traversing upward
+   * @param {number} maxDepth - Maximum parent levels to check
+   * @returns {string|null} Found absolute path or null
+   */
+  findUpward(root, relativePath, maxDepth = 3) {
+    let current = root;
+    for (let depth = 0; depth < maxDepth; depth++) {
+      const candidate = path.join(current, relativePath);
+      if (fs.existsSync(candidate)) {
+        return candidate;
+      }
+      const parent = path.dirname(current);
+      if (parent === current) break;
+      current = parent;
+    }
+    return null;
+  }
+
+  /**
+   * Find the project root directory
+   * @param {string} startPath - Starting directory path
+   * @returns {string} Project root directory path
+   */
+  findProjectRoot(startPath) {
+    const projectRoot = this.findUpward(startPath, "package.json", 5);
+    if (projectRoot) {
+      return path.dirname(projectRoot);
+    }
+
+    throw new Error("Could not find project root");
+  }
+
+  /**
+   * Resolve the actual filesystem path to a package
+   * Works both in monorepo (./packages) and when installed as dependency
+   * @param {string} projectRoot - Project root directory path
+   * @param {"libtype"|"librpc"} packageName - Package name without scope
+   * @returns {string} Absolute path to package directory
+   */
+  findPackagePath(projectRoot, packageName) {
+    const fullPackageName = `@forwardimpact/${packageName}`;
+
+    // First try local monorepo structures
+    for (const dir of ["libraries", "packages"]) {
+      const localPath = path.join(projectRoot, dir, packageName);
+      if (fs.existsSync(localPath)) {
+        return localPath;
+      }
+    }
+
+    // Fall back to Node module resolution for installed packages
+    const require = createRequire(path.join(projectRoot, "package.json"));
+
+    // Resolve the package.json path
+    const packageJsonPath = require.resolve(`${fullPackageName}/package.json`);
+    return path.dirname(packageJsonPath);
+  }
+
+  /**
+   * Resolve the generated directory path for a package
+   * @param {string} projectRoot - Project root directory path
+   * @param {"libtype"|"librpc"} packageName - Package name without scope
+   * @returns {string} Absolute path to package's generated directory
+   */
+  findGeneratedPath(projectRoot, packageName) {
+    const packagePath = this.findPackagePath(projectRoot, packageName);
+    return path.join(packagePath, "generated");
+  }
+
+  /**
+   * Create symlink from source to target directory
+   * @param {string} sourcePath - Source directory path
+   * @param {string} targetPath - Target directory path
+   * @returns {Promise<void>}
+   */
+  async createSymlink(sourcePath, targetPath) {
+    // Ensure the source directory exists
+    await fsAsync.mkdir(sourcePath, { recursive: true });
+
+    // Remove the existing target if it exists
+    try {
+      const stats = await fsAsync.lstat(targetPath);
+      if (stats.isSymbolicLink()) {
+        await fsAsync.unlink(targetPath);
+      } else {
+        await fsAsync.rm(targetPath, { recursive: true, force: true });
+      }
+    } catch {
+      // Target doesn't exist, which is fine
+    }
+
+    // Create the symlink
+    await fsAsync.symlink(sourcePath, targetPath, "dir");
+    this.#logger.debug("Finder", "Created symlink", {
+      source_path: sourcePath,
+      target_path: targetPath,
+    });
+  }
+
+  /**
+   * Create symlinks to the generated directory for standard packages
+   * Attempts to find project root and create symlinks, but won't fail in test environments
+   * @param {string} generatedPath - Path to generated code directory
+   * @returns {Promise<void>}
+   */
+  async createPackageSymlinks(generatedPath) {
+    const projectRoot = this.findProjectRoot(this.#process.cwd());
+    const packageNames = ["libtype", "librpc"];
+
+    const promises = packageNames.map(async (packageName) => {
+      const targetPath = this.findGeneratedPath(projectRoot, packageName);
+      await this.createSymlink(generatedPath, targetPath);
+    });
+
+    await Promise.all(promises);
+  }
+}

package/http.js

@@ -0,0 +1,21 @@
+/**
+ * Parses the JSON body from an incoming HTTP request.
+ * @param {import('http').IncomingMessage} req - The HTTP request object.
+ * @returns {Promise<object>} The parsed JSON object, or an empty object if parsing fails.
+ */
+export function parseJsonBody(req) {
+  return new Promise((resolve, reject) => {
+    let body = "";
+    req.on("data", (chunk) => {
+      body += chunk;
+    });
+    req.on("end", () => {
+      try {
+        resolve(JSON.parse(body));
+      } catch {
+        resolve({});
+      }
+    });
+    req.on("error", reject);
+  });
+}

package/index.js

@@ -0,0 +1,154 @@
+import crypto from "crypto";
+import fs from "fs/promises";
+import path from "path";
+import { spawn } from "child_process";
+
+import { Tokenizer, ranks } from "./tokenizer.js";
+import { Finder } from "./finder.js";
+import { BundleDownloader } from "./downloader.js";
+import { TarExtractor } from "./extractor.js";
+
+/**
+ * Updates or creates an environment variable in .env file
+ * @param {string} key - Environment variable name (e.g., "SERVICE_SECRET")
+ * @param {string} value - Environment variable value
+ * @param {string} [envPath] - Path to .env file (defaults to .env in current directory)
+ */
+export async function updateEnvFile(key, value, envPath = ".env") {
+  const fullPath = path.resolve(envPath);
+  let content = "";
+
+  try {
+    content = await fs.readFile(fullPath, "utf8");
+  } catch (error) {
+    // It's ok if the file doesn't exist
+    if (error.code !== "ENOENT") throw error;
+  }
+
+  const envLine = `${key}=${value}`;
+  const lines = content.split("\n");
+  let found = false;
+
+  // Look for existing key line (both active and commented)
+  for (let i = 0; i < lines.length; i++) {
+    if (lines[i].startsWith(`${key}=`) || lines[i].startsWith(`# ${key}=`)) {
+      lines[i] = envLine;
+      found = true;
+      break;
+    }
+  }
+
+  // If not found, add it to the end
+  if (!found) {
+    if (content && !content.endsWith("\n")) {
+      lines.push("");
+    }
+    lines.push(envLine);
+  }
+
+  // Write back to file
+  await fs.writeFile(fullPath, lines.join("\n"));
+}
+
+/**
+ * Generates a deterministic hash from multiple input values
+ * @param {...string} values - Values to hash together
+ * @returns {string} The first 16 characters of SHA256 hash
+ */
+export function generateHash(...values) {
+  const input = values.filter(Boolean).join(".");
+  return crypto
+    .createHash("sha256")
+    .update(input)
+    .digest("hex")
+    .substring(0, 8);
+}
+
+/**
+ * Generates a unique session ID for conversation tracking
+ * @returns {string} Unique session identifier
+ */
+export function generateUUID() {
+  return crypto.randomUUID();
+}
+
+/**
+ * Helper function to count tokens
+ * @param {string} text - Text to count tokens for
+ * @param {Tokenizer} tokenizer - Tokenizer instance
+ * @returns {number} Approximate token count
+ */
+export function countTokens(text, tokenizer) {
+  if (!tokenizer) tokenizer = createTokenizer();
+  return tokenizer.encode(text).length;
+}
+
+/**
+ * Creates a new tokenizer instance
+ * @returns {Tokenizer} New tokenizer instance
+ */
+export function createTokenizer() {
+  return new Tokenizer(ranks);
+}
+
+/**
+ * Creates a BundleDownloader instance configured for generated code management.
+ * Used in containerized deployments to download pre-generated code bundles.
+ * @param {Function} createStorage - Storage factory function from libstorage
+ * @returns {Promise<BundleDownloader>} Configured BundleDownloader instance
+ */
+export async function createBundleDownloader(createStorage) {
+  if (!createStorage) throw new Error("createStorage is required");
+
+  // Dynamic import to avoid circular dependency with libtelemetry
+  const { createLogger } = await import("@forwardimpact/libtelemetry");
+  const logger = createLogger("generated");
+  const finder = new Finder(fs, logger);
+  const extractor = new TarExtractor(fs, path);
+
+  return new BundleDownloader(createStorage, finder, logger, extractor);
+}
+
+/**
+ * Executes command line arguments as child process, similar to execv() in C
+ * @param {number} [shift] - Number of arguments to skip from process.argv before extracting command
+ * @returns {void} Function does not return - exits parent process
+ */
+export function execLine(shift = 0) {
+  const args = process.argv.slice(2 + shift);
+  if (args.length === 0) return;
+
+  // Look for '--' delimiter and use everything after it as the command
+  const index = args.indexOf("--");
+  const line = index !== -1 ? args.slice(index + 1) : args;
+
+  if (line.length === 0) return;
+
+  const [command, ...commandArgs] = line;
+  const child = spawn(command, commandArgs, {
+    stdio: "inherit",
+    env: process.env,
+  });
+
+  // Forward signals to child process
+  ["SIGTERM", "SIGINT", "SIGQUIT"].forEach((signal) => {
+    process.on(signal, () => child.kill(signal));
+  });
+
+  child.on("error", (error) => {
+    console.error("Error:", error);
+    process.exit(1);
+  });
+
+  child.on("exit", (code, signal) => {
+    process.exit(signal ? 1 : code || 0);
+  });
+}
+
+export { Finder } from "./finder.js";
+export { BundleDownloader } from "./downloader.js";
+export { TarExtractor, ZipExtractor } from "./extractor.js";
+export { ProcessorBase } from "./processor.js";
+export { Retry, createRetry } from "./retry.js";
+export { parseJsonBody } from "./http.js";
+export { waitFor } from "./wait.js";

package/package.json

@@ -0,0 +1,24 @@
+{
+  "name": "@forwardimpact/libutil",
+  "version": "0.1.60",
+  "description": "Utility functions and utilities for Guide",
+  "license": "Apache-2.0",
+  "author": "D. Olsson <hi@senzilla.io>",
+  "type": "module",
+  "main": "index.js",
+  "bin": {
+    "fit-download-bundle": "./bin/fit-download-bundle.js"
+  },
+  "engines": {
+    "node": ">=22.0.0"
+  },
+  "scripts": {
+    "test": "node --test test/*.test.js"
+  },
+  "dependencies": {
+    "@forwardimpact/libtelemetry": "^0.1.22"
+  },
+  "devDependencies": {
+    "@forwardimpact/libharness": "^0.1.5"
+  }
+}

package/processor.js

@@ -0,0 +1,125 @@
+/**
+ * Base class for batch processor implementations with common batch management logic
+ */
+export class ProcessorBase {
+  #logger;
+  #batchSize;
+
+  /**
+   * Creates a new processor instance
+   * @param {object} logger - Logger instance for debug output
+   * @param {number} batchSize - Size of batches for processing (default: 4)
+   */
+  constructor(logger, batchSize = 4) {
+    if (!logger) throw new Error("logger is required");
+    if (typeof batchSize !== "number" || batchSize < 1) {
+      throw new Error("batchSize must be a positive number");
+    }
+
+    this.#logger = logger;
+    this.#batchSize = batchSize;
+  }
+
+  /**
+   * Gets the logger instance for use in subclasses
+   * @returns {object} Logger instance
+   * @protected
+   */
+  get logger() {
+    return this.#logger;
+  }
+
+  /**
+   * Processes items in batches
+   * @param {any[]} items - Items to process
+   * @param {string} context - Processing context label
+   * @returns {Promise<any[]>} Processed results
+   */
+  async process(items, context = "items") {
+    if (!Array.isArray(items)) {
+      throw new Error("items must be an array");
+    }
+
+    if (items.length === 0) {
+      this.#logger.debug("Processor", "No items to process", { context });
+      return;
+    }
+
+    this.#logger.debug("Processor", "Starting batch", {
+      total: items.length,
+      context,
+    });
+
+    let currentBatch = [];
+    let processedCount = 0;
+
+    for (let i = 0; i < items.length; i++) {
+      currentBatch.push(items[i]);
+
+      if (currentBatch.length >= this.#batchSize) {
+        await this.processBatch(
+          currentBatch,
+          processedCount,
+          items.length,
+          context,
+        );
+        processedCount += currentBatch.length;
+        currentBatch = [];
+      }
+    }
+
+    if (currentBatch.length > 0) {
+      await this.processBatch(
+        currentBatch,
+        processedCount,
+        items.length,
+        context,
+      );
+    }
+  }
+
+  /**
+   * Processes a batch of items
+   * @param {any[]} batch - Batch to process
+   * @param {number} processed - Number already processed
+   * @param {number} total - Total number of items
+   * @param {object} context - Processing context
+   * @returns {Promise<any[]>} Batch results
+   */
+  async processBatch(batch, processed, total, context) {
+    const batchSize = batch.length;
+
+    this.#logger.debug("Processor", "Processing batch", {
+      items:
+        batchSize > 1
+          ? `${processed + 1}-${processed + batchSize}/${total}`
+          : `${processed + 1}/${total}`,
+      context,
+    });
+
+    const promises = batch.map(async (item, itemIndex) => {
+      const globalIndex = processed + itemIndex;
+      try {
+        return await this.processItem(item);
+      } catch (error) {
+        this.#logger.debug("Processor", "Skipping, failed to process item", {
+          item: `${globalIndex + 1}/${total}`,
+          context,
+          error: error.message,
+        });
+        return null;
+      }
+    });
+
+    await Promise.all(promises);
+  }
+
+  /**
+   * Processes a single item (must be implemented by subclass)
+   * @param {any} _item - Item to process
+   * @returns {Promise<any>} Processed result
+   */
+  async processItem(_item) {
+    throw new Error("processItem must be implemented by subclass");
+  }
+}

package/retry.js

@@ -0,0 +1,126 @@
+/**
+ * Retry class for handling transient errors with exponential backoff
+ */
+export class Retry {
+  #retries;
+  #delay;
+
+  /**
+   * Creates a new Retry instance
+   * @param {object} [config] - Optional configuration object
+   * @param {number} [config.retries] - Maximum number of retry attempts (default: 10)
+   * @param {number} [config.delay] - Initial delay in milliseconds (default: 1000)
+   */
+  constructor(config = {}) {
+    this.#retries = config.retries ?? 10;
+    this.#delay = config.delay ?? 1000;
+  }
+
+  /**
+   * Checks if an HTTP status code or error should trigger a retry
+   * @param {number} status - HTTP status code
+   * @returns {boolean} True if the status should be retried
+   * @private
+   */
+  #isRetryableStatus(status) {
+    // Retry on rate limits, transient server errors, and client timeouts
+    return (
+      status === 429 ||
+      status === 499 ||
+      status === 500 ||
+      status === 502 ||
+      status === 503 ||
+      status === 504
+    );
+  }
+
+  /**
+   * Checks if an error is a network error that should trigger a retry
+   * @param {Error} error - Error object
+   * @returns {boolean} True if the error should be retried
+   * @private
+   */
+  #isRetryableError(error) {
+    const message = error.message.toLowerCase();
+
+    // Check for HTTP status codes in error messages (e.g., "HTTP 499: status code 499")
+    const httpStatusMatch = message.match(/http (\d{3})/);
+    if (httpStatusMatch) {
+      const statusCode = parseInt(httpStatusMatch[1], 10);
+      if (this.#isRetryableStatus(statusCode)) {
+        return true;
+      }
+    }
+
+    return (
+      message.includes("network") ||
+      message.includes("timeout") ||
+      message.includes("econnrefused") ||
+      message.includes("econnreset") ||
+      message.includes("etimedout") ||
+      message.includes("unavailable") ||
+      message.includes("fetch failed") ||
+      message.includes("unexpected eof")
+    );
+  }
+
+  /**
+   * Executes a function with exponential backoff retry logic for transient errors
+   * @param {() => Promise<Response>} requestFn - Function that returns a fetch promise
+   * @returns {Promise<Response>} Response from successful request
+   * @throws {Error} When all retry attempts are exhausted
+   */
+  async execute(requestFn) {
+    let lastError;
+
+    for (let attempt = 0; attempt <= this.#retries; attempt++) {
+      try {
+        const response = await requestFn();
+
+        // Check if we should retry based on status code
+        if (
+          response?.status &&
+          this.#isRetryableStatus(response.status) &&
+          attempt < this.#retries
+        ) {
+          // Add jitter to exponential backoff to avoid thundering herd
+          const exponentialDelay = this.#delay * Math.pow(2, attempt);
+          const jitter = Math.random() * 0.3 * exponentialDelay;
+          const wait = exponentialDelay + jitter;
+          await new Promise((resolve) => setTimeout(resolve, wait));
+          continue;
+        }
+
+        return response;
+      } catch (error) {
+        lastError = error;
+
+        // Check if this is a retryable network error
+        if (this.#isRetryableError(error) && attempt < this.#retries) {
+          const exponentialDelay = this.#delay * Math.pow(2, attempt);
+          const jitter = Math.random() * 0.3 * exponentialDelay;
+          const wait = exponentialDelay + jitter;
+          await new Promise((resolve) => setTimeout(resolve, wait));
+          continue;
+        }
+
+        // Non-retryable error or out of retries
+        throw error;
+      }
+    }
+
+    // This should never be reached, but if it is, throw the last error
+    throw lastError || new Error("Retries exhausted without a valid response");
+  }
+}
+
+/**
+ * Factory function to create a Retry instance with optional configuration
+ * @param {object} [config] - Optional configuration object
+ * @param {number} [config.retries] - Maximum number of retry attempts
+ * @param {number} [config.delay] - Initial delay in milliseconds
+ * @returns {Retry} Configured Retry instance
+ */
+export function createRetry(config) {
+  return new Retry(config);
+}