@cyanautomation/kaseki-agent 1.4.1

package/src/index.ts

@@ -0,0 +1,89 @@
+// Kaseki Agent TypeScript Entry Point
+// Public API exports for library users and service consumers
+
+// Services
+export { JobScheduler } from './job-scheduler';
+export { ResultCache } from './result-cache';
+export { WebhookManager } from './webhook-manager';
+
+// Configuration & Utilities
+export { KasekiApiConfig, validateApiKey, loadConfig } from './kaseki-api-config';
+export { createGracefulShutdown, assertSupportedNodeVersion } from './kaseki-api-service';
+
+// Types
+export type {
+  Job,
+  RunRequest,
+  RunResponse,
+  StatusResponse,
+  StructuredProgress,
+  LogResponse,
+  ArtifactResponse,
+  RunsListResponse,
+  ErrorResponse,
+  ValidationResponse,
+  PreflightCheck,
+  PreflightResponse,
+  AnalysisResponse,
+  WebhookPayload,
+  WebhookEventType,
+} from './kaseki-api-types';
+
+// Client
+export { KasekiApiClient } from './kaseki-api-client';
+
+// Utilities (Phase 1 extractions)
+export { sendErrorResponse, buildStatusResponse, detectContentType } from './utils/response-helpers';
+export {
+  isNonEmptyFile,
+  isNonEmptyFile as fileIsNonEmpty,
+  readFirstLine,
+  readTailLines,
+  commandOutput,
+  fileExists,
+  readFileContent,
+  getFileStats,
+} from './utils/file-helpers';
+export {
+  isString,
+  isNumber,
+  isBoolean,
+  isRecord,
+  isArray,
+  isStringArray,
+  isRecordArray,
+  hasKeys,
+} from './utils/type-guards';
+export {
+  decodeUtf8TailSafely,
+  tailLogByLines,
+  readTailBytes,
+} from './utils/utf8-helpers';
+export { getJobOrRespond } from './utils/route-helpers';
+export { jobLookupMiddleware } from './middleware/job-lookup';
+
+// Utilities (Phase 2 consolidations)
+export {
+  createJobSubmittedEvent,
+  createJobStartedEvent,
+  createJobCompletedEvent,
+  createJobCancelledEvent,
+  createJobFailedEvent,
+} from './utils/webhook-event-builder';
+export { HttpClientFactory } from './utils/http-client-factory';
+export type { HttpRequestOptions, RetryConfig } from './utils/http-client-factory';
+
+// Utilities (Phase 3 refactoring)
+export { FailureArtifactWriter } from './utils/failure-artifact-writer';
+export type { CleanupResult } from './utils/failure-artifact-writer';
+export { StatusResponseBuilder } from './utils/status-response-builder';
+export {
+  normalizeProgressEvent,
+  toStructuredProgress,
+} from './utils/progress-normalizer';
+
+// Route modules (Phase 3 refactoring)
+export { createStatusRoutes } from './routes/status-routes';
+export { createLogRoutes } from './routes/log-routes';
+export { createArtifactRoutes, readArtifactContent } from './routes/artifact-routes';
+export { createWebhookRoutes } from './routes/webhook-routes';

package/src/instance/InstanceManager.ts

@@ -0,0 +1,285 @@
+/**
+ * Instance Manager
+ *
+ * Handles kaseki instance lifecycle: naming, directories, metadata
+ */
+
+import fs from 'fs/promises';
+import path from 'path';
+import { createLogger } from '../logger';
+
+const logger = createLogger('instance');
+
+export interface InstanceMetadata {
+  id: string;
+  createdAt: string;
+  completedAt?: string;
+  status: 'running' | 'completed' | 'failed';
+  exitCode?: number;
+  model?: string;
+  provider?: string;
+  repoUrl?: string;
+  gitRef?: string;
+  stages?: {
+    [stageName: string]: {
+      startTime?: string;
+      endTime?: string;
+      exitCode?: number;
+      duration?: number;
+    };
+  };
+}
+
+export class InstanceManager {
+  private rootDir: string;
+  private instanceId: string | null = null;
+
+  constructor(rootDir: string = '/agents') {
+    this.rootDir = rootDir;
+  }
+
+  /**
+   * Generate or get next instance ID
+   */
+  async getOrCreateInstanceId(): Promise<string> {
+    if (this.instanceId) {
+      return this.instanceId;
+    }
+
+    // Find next available instance number
+    const runsDir = path.join(this.rootDir, 'kaseki-runs');
+    let instanceNum = 1;
+
+    try {
+      const entries = await fs.readdir(runsDir, { withFileTypes: true });
+      const existingNums = entries
+        .filter((entry) => entry.isDirectory() && entry.name.startsWith('kaseki-'))
+        .map((entry) => {
+          const match = entry.name.match(/kaseki-(\d+)/);
+          return match ? parseInt(match[1], 10) : 0;
+        })
+        .filter((num) => num > 0);
+
+      if (existingNums.length > 0) {
+        instanceNum = Math.max(...existingNums) + 1;
+      }
+    } catch {
+      // Directory doesn't exist yet, that's fine
+      instanceNum = 1;
+    }
+
+    this.instanceId = `kaseki-${instanceNum}`;
+    logger.debug(`Generated instance ID: ${this.instanceId}`);
+
+    return this.instanceId;
+  }
+
+  /**
+   * Get workspace directory path
+   */
+  async getWorkspaceDir(): Promise<string> {
+    const id = await this.getOrCreateInstanceId();
+    return path.join(this.rootDir, 'kaseki-runs', id);
+  }
+
+  /**
+   * Get results directory path
+   */
+  async getResultsDir(): Promise<string> {
+    const id = await this.getOrCreateInstanceId();
+    return path.join(this.rootDir, 'kaseki-results', id);
+  }
+
+  /**
+   * Get cache directory path
+   */
+  getCacheDir(): string {
+    return path.join(this.rootDir, 'kaseki-cache');
+  }
+
+  /**
+   * Create instance directories
+   */
+  async createDirectories(): Promise<{ workspace: string; results: string }> {
+    const workspace = await this.getWorkspaceDir();
+    const results = await this.getResultsDir();
+
+    try {
+      await fs.mkdir(workspace, { recursive: true });
+      await fs.mkdir(results, { recursive: true });
+      logger.debug(`Created instance directories for ${this.instanceId}`);
+
+      return { workspace, results };
+    } catch (error) {
+      throw new Error(`Failed to create instance directories: ${error}`);
+    }
+  }
+
+  /**
+   * Initialize metadata file
+   */
+  async initializeMetadata(metadata: Partial<InstanceMetadata>): Promise<void> {
+    const id = await this.getOrCreateInstanceId();
+    const resultsDir = await this.getResultsDir();
+    const metadataPath = path.join(resultsDir, 'metadata.json');
+
+    const fullMetadata: InstanceMetadata = {
+      id,
+      createdAt: new Date().toISOString(),
+      status: 'running',
+      stages: {},
+      ...metadata,
+    };
+
+    try {
+      await fs.writeFile(
+        metadataPath,
+        JSON.stringify(fullMetadata, null, 2),
+        'utf-8'
+      );
+      logger.debug(`Initialized metadata for ${id}`);
+    } catch (error) {
+      throw new Error(`Failed to initialize metadata: ${error}`);
+    }
+  }
+
+  /**
+   * Update metadata
+   */
+  async updateMetadata(updates: Partial<InstanceMetadata>): Promise<void> {
+    const resultsDir = await this.getResultsDir();
+    const metadataPath = path.join(resultsDir, 'metadata.json');
+
+    try {
+      let metadata: InstanceMetadata = {
+        id: this.instanceId || '',
+        createdAt: new Date().toISOString(),
+        status: 'running',
+      };
+
+      // Read existing metadata
+      try {
+        const content = await fs.readFile(metadataPath, 'utf-8');
+        metadata = JSON.parse(content);
+      } catch {
+        // File doesn't exist yet, that's fine
+      }
+
+      // Merge updates
+      const updated = { ...metadata, ...updates };
+
+      // Write back
+      await fs.writeFile(
+        metadataPath,
+        JSON.stringify(updated, null, 2),
+        'utf-8'
+      );
+
+      logger.debug(`Updated metadata for ${this.instanceId}`);
+    } catch (error) {
+      logger.warn(`Failed to update metadata: ${error}`);
+    }
+  }
+
+  /**
+   * Record stage timing
+   */
+  async recordStage(
+    stageName: string,
+    exitCode?: number,
+    startTime?: Date,
+    endTime?: Date
+  ): Promise<void> {
+    const resultsDir = await this.getResultsDir();
+    const metadataPath = path.join(resultsDir, 'metadata.json');
+
+    try {
+      let metadata: InstanceMetadata = {
+        id: this.instanceId || '',
+        createdAt: new Date().toISOString(),
+        status: 'running',
+      };
+
+      try {
+        const content = await fs.readFile(metadataPath, 'utf-8');
+        metadata = JSON.parse(content);
+      } catch {
+        // File doesn't exist
+      }
+
+      if (!metadata.stages) {
+        metadata.stages = {};
+      }
+
+      metadata.stages[stageName] = {
+        startTime: startTime?.toISOString(),
+        endTime: endTime?.toISOString(),
+        exitCode,
+        duration: startTime && endTime ?
+          (endTime.getTime() - startTime.getTime()) / 1000 : undefined,
+      };
+
+      await fs.writeFile(
+        metadataPath,
+        JSON.stringify(metadata, null, 2),
+        'utf-8'
+      );
+
+      logger.debug(`Recorded stage: ${stageName} (exit code: ${exitCode})`);
+    } catch (error) {
+      logger.warn(`Failed to record stage: ${error}`);
+    }
+  }
+
+  /**
+   * Finalize instance metadata
+   */
+  async finalize(exitCode: number): Promise<void> {
+    await this.updateMetadata({
+      completedAt: new Date().toISOString(),
+      status: exitCode === 0 ? 'completed' : 'failed',
+      exitCode,
+    });
+
+    logger.info(`Finalized instance ${this.instanceId}: exit code ${exitCode}`);
+  }
+
+  /**
+   * Get instance metadata
+   */
+  async getMetadata(): Promise<InstanceMetadata | null> {
+    try {
+      const resultsDir = await this.getResultsDir();
+      const metadataPath = path.join(resultsDir, 'metadata.json');
+      const content = await fs.readFile(metadataPath, 'utf-8');
+      return JSON.parse(content);
+    } catch {
+      return null;
+    }
+  }
+
+  /**
+   * Clean up instance workspace (optional)
+   */
+  async cleanup(keepWorkspace: boolean = false): Promise<void> {
+    if (keepWorkspace) {
+      logger.debug(`Keeping workspace for ${this.instanceId}`);
+      return;
+    }
+
+    try {
+      const workspace = await this.getWorkspaceDir();
+      await fs.rm(workspace, { recursive: true, force: true });
+      logger.debug(`Cleaned up workspace for ${this.instanceId}`);
+    } catch (error) {
+      logger.warn(`Failed to cleanup workspace: ${error}`);
+    }
+  }
+
+  /**
+   * Get instance ID
+   */
+  getInstanceId(): string | null {
+    return this.instanceId;
+  }
+}

package/src/instance-metadata-reader.test.ts

@@ -0,0 +1,190 @@
+import fs from 'fs';
+import os from 'os';
+import path from 'path';
+import { readInstanceMetadata } from './instance-metadata-reader';
+
+describe('instance-metadata-reader', () => {
+  let tempDir: string;
+
+  beforeEach(() => {
+    tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'kaseki-test-'));
+  });
+
+  afterEach(() => {
+    fs.rmSync(tempDir, { recursive: true, force: true });
+  });
+
+  it('should read metadata.json and host-start.json', () => {
+    const metadata = {
+      current_stage: 'validation',
+      exit_code: 0,
+      duration_seconds: 120,
+      model: 'claude-3-opus',
+    };
+
+    const hostStart = {
+      model: 'claude-3-opus',
+      repo_url: 'https://github.com/example/repo',
+      git_ref: 'main',
+      agentTimeoutSeconds: 1200,
+    };
+
+    fs.writeFileSync(path.join(tempDir, 'metadata.json'), JSON.stringify(metadata));
+    fs.writeFileSync(path.join(tempDir, 'host-start.json'), JSON.stringify(hostStart));
+
+    const result = readInstanceMetadata(tempDir);
+
+    expect(result.metadata).toEqual(metadata);
+    expect(result.hostStart).toEqual(hostStart);
+    expect(result.elapsedSeconds).toBe(120);
+  });
+
+  it('should return empty objects when files do not exist', () => {
+    const result = readInstanceMetadata(tempDir);
+
+    expect(result.metadata).toEqual({});
+    expect(result.hostStart).toEqual({});
+    expect(result.elapsedSeconds).toBeNull();
+  });
+
+  it('should handle corrupt JSON in metadata.json gracefully', () => {
+    fs.writeFileSync(path.join(tempDir, 'metadata.json'), '{invalid json');
+
+    const result = readInstanceMetadata(tempDir);
+
+    expect(result.metadata).toEqual({});
+  });
+
+  it('should handle corrupt JSON in host-start.json gracefully', () => {
+    fs.writeFileSync(path.join(tempDir, 'host-start.json'), '{invalid json');
+
+    const result = readInstanceMetadata(tempDir);
+
+    expect(result.hostStart).toEqual({});
+  });
+
+  it('should read elapsed_seconds from resource.time when metadata.duration_seconds is missing', () => {
+    const metadata = { current_stage: 'completed' };
+    fs.writeFileSync(
+      path.join(tempDir, 'metadata.json'),
+      JSON.stringify(metadata)
+    );
+    fs.writeFileSync(
+      path.join(tempDir, 'resource.time'),
+      'user_cpu=1.234\nelapsed_seconds=90\nsystem_cpu=0.456'
+    );
+
+    const result = readInstanceMetadata(tempDir);
+
+    expect(result.elapsedSeconds).toBe(90);
+  });
+
+  it('should prefer metadata.duration_seconds over resource.time', () => {
+    const metadata = { current_stage: 'completed', duration_seconds: 120 };
+    fs.writeFileSync(
+      path.join(tempDir, 'metadata.json'),
+      JSON.stringify(metadata)
+    );
+    fs.writeFileSync(
+      path.join(tempDir, 'resource.time'),
+      'elapsed_seconds=90'
+    );
+
+    const result = readInstanceMetadata(tempDir);
+
+    expect(result.elapsedSeconds).toBe(120);
+  });
+
+  it('should handle malformed resource.time gracefully', () => {
+    const metadata = { current_stage: 'completed' };
+    fs.writeFileSync(
+      path.join(tempDir, 'metadata.json'),
+      JSON.stringify(metadata)
+    );
+    fs.writeFileSync(path.join(tempDir, 'resource.time'), 'malformed_content');
+
+    const result = readInstanceMetadata(tempDir);
+
+    expect(result.elapsedSeconds).toBeNull();
+  });
+
+  it('should return null for elapsedSeconds when neither source has valid data', () => {
+    const metadata = { current_stage: 'pending' };
+    fs.writeFileSync(
+      path.join(tempDir, 'metadata.json'),
+      JSON.stringify(metadata)
+    );
+
+    const result = readInstanceMetadata(tempDir);
+
+    expect(result.elapsedSeconds).toBeNull();
+  });
+
+  it('should preserve all metadata fields', () => {
+    const metadata = {
+      current_stage: 'validation',
+      exit_code: 1,
+      duration_seconds: 120,
+      started_at: '2024-01-01T10:00:00Z',
+      model: 'gpt-4',
+      pi_duration_seconds: 60,
+      custom_field: 'custom_value',
+    };
+
+    fs.writeFileSync(path.join(tempDir, 'metadata.json'), JSON.stringify(metadata));
+
+    const result = readInstanceMetadata(tempDir);
+
+    expect(result.metadata).toEqual(metadata);
+    expect(result.metadata.custom_field).toBe('custom_value');
+  });
+
+  it('should propagate ENOENT errors when reading metadata.json', () => {
+    const metadataPath = path.join(tempDir, 'metadata.json');
+
+    // Create file, then delete it during read simulation
+    fs.writeFileSync(metadataPath, '{}');
+
+    // Override readFileSync to throw ENOENT
+    const originalReadFileSync = fs.readFileSync as any;
+    let readCallCount = 0;
+    (fs.readFileSync as any) = jest.fn((filePath: any, ...args: any[]) => {
+      readCallCount++;
+      if (readCallCount === 1 && filePath === metadataPath) {
+        const error = new Error('File not found') as any;
+        error.code = 'ENOENT';
+        throw error;
+      }
+      return originalReadFileSync(filePath, ...args);
+    });
+
+    try {
+      expect(() => readInstanceMetadata(tempDir)).toThrow();
+    } finally {
+      (fs.readFileSync as any) = originalReadFileSync;
+    }
+  });
+
+  it('should propagate ESTALE errors when reading host-start.json', () => {
+    const hostStartPath = path.join(tempDir, 'host-start.json');
+    fs.writeFileSync(hostStartPath, '{}');
+
+    const originalReadFileSync = fs.readFileSync as any;
+    let readCallCount = 0;
+    (fs.readFileSync as any) = jest.fn((filePath: any, ...args: any[]) => {
+      readCallCount++;
+      if (readCallCount === 1 && filePath === hostStartPath) {
+        const error = new Error('Stale NFS file handle') as any;
+        error.code = 'ESTALE';
+        throw error;
+      }
+      return originalReadFileSync(filePath, ...args);
+    });
+
+    try {
+      expect(() => readInstanceMetadata(tempDir)).toThrow();
+    } finally {
+      (fs.readFileSync as any) = originalReadFileSync;
+    }
+  });
+});

package/src/instance-metadata-reader.ts

@@ -0,0 +1,129 @@
+/**
+ * instance-metadata-reader.ts
+ *
+ * Encapsulates metadata reading logic for kaseki instances.
+ * Safely reads and parses metadata.json, host-start.json, and resource.time files
+ * with graceful error handling for transient I/O errors.
+ */
+
+import fs from 'fs';
+import path from 'path';
+
+export interface Metadata {
+  current_stage?: string;
+  exit_code?: number | string;
+  duration_seconds?: number;
+  started_at?: string;
+  start_time?: string;
+  model?: string;
+  pi_duration_seconds?: number;
+  [key: string]: any;
+}
+
+export interface HostStart {
+  model?: string;
+  repo_url?: string;
+  repo?: string;
+  git_ref?: string;
+  ref?: string;
+  agentTimeoutSeconds?: number;
+  [key: string]: any;
+}
+
+export interface InstanceMetadataInfo {
+  metadata: Metadata;
+  hostStart: HostStart;
+  elapsedSeconds: number | null;
+}
+
+/**
+ * Check if an error is a transient I/O error that should be retried or skipped.
+ */
+function isSkippableInstanceIoError(error: any): boolean {
+  return error && (error.code === 'ENOENT' || error.code === 'ESTALE');
+}
+
+/**
+ * Read elapsed seconds from metadata or resource.time file.
+ * Tries metadata.duration_seconds first, then falls back to resource.time.
+ */
+function readElapsedSeconds(resultDir: string, metadata: Metadata): number | null {
+  if (metadata.duration_seconds !== undefined) {
+    return metadata.duration_seconds;
+  }
+
+  const resourceTimePath = path.join(resultDir, 'resource.time');
+  if (!fs.existsSync(resourceTimePath)) {
+    return null;
+  }
+
+  try {
+    const content = fs.readFileSync(resourceTimePath, 'utf8');
+    const match = content.match(/elapsed_seconds=([0-9]+(?:\.[0-9]+)?)/);
+    if (match) {
+      return parseFloat(match[1]);
+    }
+  } catch {
+    // File may be unreadable; return null
+  }
+
+  return null;
+}
+
+/**
+ * Read metadata for a kaseki instance from its results directory.
+ *
+ * Reads three files:
+ * - metadata.json (primary source of truth)
+ * - host-start.json (configuration from run initiation)
+ * - resource.time (timing information if metadata incomplete)
+ *
+ * Handles transient I/O errors (ENOENT, ESTALE) by propagating them
+ * so callers can skip the instance and continue scanning.
+ *
+ * @param resultDir - Path to the kaseki results directory (e.g., /agents/kaseki-results/kaseki-1)
+ * @returns InstanceMetadataInfo with parsed metadata, host config, and elapsed time
+ * @throws Error if I/O error is transient (ENOENT, ESTALE); caller should skip this instance
+ */
+export function readInstanceMetadata(resultDir: string): InstanceMetadataInfo {
+  const metadataPath = path.join(resultDir, 'metadata.json');
+  const hostStartPath = path.join(resultDir, 'host-start.json');
+
+  let metadata: Metadata = {};
+  let hostStart: HostStart = {};
+
+  // Read metadata
+  if (fs.existsSync(metadataPath)) {
+    try {
+      const content = fs.readFileSync(metadataPath, 'utf8');
+      metadata = JSON.parse(content);
+    } catch (e) {
+      if (isSkippableInstanceIoError(e)) {
+        throw e; // Propagate transient errors to caller
+      }
+      // Metadata may still be incomplete if run is in progress; use empty object
+    }
+  }
+
+  // Read host start config
+  if (fs.existsSync(hostStartPath)) {
+    try {
+      const content = fs.readFileSync(hostStartPath, 'utf8');
+      hostStart = JSON.parse(content);
+    } catch (e) {
+      if (isSkippableInstanceIoError(e)) {
+        throw e; // Propagate transient errors to caller
+      }
+      // File may be unreadable; use empty object
+    }
+  }
+
+  // Read elapsed seconds
+  const elapsedSeconds = readElapsedSeconds(resultDir, metadata);
+
+  return {
+    metadata,
+    hostStart,
+    elapsedSeconds,
+  };
+}