@herdctl/core 0.0.1

package/src/state/job-output.ts

@@ -0,0 +1,316 @@
+/**
+ * Job output logging operations
+ *
+ * Provides functions for streaming job output to JSONL files.
+ * Supports real-time monitoring with immediate writes (no buffering).
+ *
+ * Output files are stored at: .herdctl/jobs/job-<id>.jsonl
+ */
+
+import { join } from "node:path";
+import { createReadStream } from "node:fs";
+import { stat } from "node:fs/promises";
+import { createInterface } from "node:readline";
+import { appendJsonl } from "./utils/atomic.js";
+import { StateFileError } from "./errors.js";
+import {
+  type JobOutputMessage,
+  type JobOutputInput,
+  isValidJobOutputInput,
+  JobOutputMessageSchema,
+} from "./schemas/job-output.js";
+
+// =============================================================================
+// Types
+// =============================================================================
+
+/**
+ * Logger interface for job output operations
+ */
+export interface JobOutputLogger {
+  warn: (message: string) => void;
+}
+
+/**
+ * Options for job output operations
+ */
+export interface JobOutputOptions {
+  /** Logger for warnings and errors */
+  logger?: JobOutputLogger;
+}
+
+/**
+ * Options for reading job output
+ */
+export interface ReadJobOutputOptions extends JobOutputOptions {
+  /** Whether to skip invalid lines instead of failing */
+  skipInvalidLines?: boolean;
+}
+
+// =============================================================================
+// Path Utilities
+// =============================================================================
+
+/**
+ * Get the path to a job's output file
+ *
+ * @param jobsDir - Path to the jobs directory (.herdctl/jobs)
+ * @param jobId - Job ID (e.g., job-2024-01-15-abc123)
+ * @returns Path to the JSONL output file
+ */
+export function getJobOutputPath(jobsDir: string, jobId: string): string {
+  return join(jobsDir, `${jobId}.jsonl`);
+}
+
+// =============================================================================
+// Write Operations
+// =============================================================================
+
+/**
+ * Append a single message to a job's output log
+ *
+ * Writes immediately to disk for real-time monitoring (no buffering).
+ * Uses append mode which is safe for concurrent appends.
+ *
+ * @param jobsDir - Path to the jobs directory
+ * @param jobId - Job ID
+ * @param message - Message to append (timestamp added automatically)
+ * @throws StateFileError if write fails or message is invalid
+ *
+ * @example
+ * ```typescript
+ * await appendJobOutput(jobsDir, jobId, {
+ *   type: "assistant",
+ *   content: "Hello, world!"
+ * });
+ * ```
+ */
+export async function appendJobOutput(
+  jobsDir: string,
+  jobId: string,
+  message: JobOutputInput
+): Promise<void> {
+  // Validate message structure
+  if (!isValidJobOutputInput(message)) {
+    throw new StateFileError(
+      `Invalid job output message: must have a valid 'type' field`,
+      getJobOutputPath(jobsDir, jobId),
+      "write"
+    );
+  }
+
+  const outputPath = getJobOutputPath(jobsDir, jobId);
+
+  // Add timestamp to message
+  const messageWithTimestamp: JobOutputMessage = {
+    ...message,
+    timestamp: new Date().toISOString(),
+  } as JobOutputMessage;
+
+  try {
+    await appendJsonl(outputPath, messageWithTimestamp);
+  } catch (error) {
+    throw new StateFileError(
+      `Failed to append to job output: ${(error as Error).message}`,
+      outputPath,
+      "write",
+      error as Error
+    );
+  }
+}
+
+/**
+ * Append multiple messages to a job's output log
+ *
+ * Writes all messages immediately in sequence.
+ * Each message is validated before writing begins.
+ *
+ * @param jobsDir - Path to the jobs directory
+ * @param jobId - Job ID
+ * @param messages - Messages to append (timestamps added automatically)
+ * @throws StateFileError if write fails or any message is invalid
+ *
+ * @example
+ * ```typescript
+ * await appendJobOutputBatch(jobsDir, jobId, [
+ *   { type: "tool_use", tool_name: "read_file", input: { path: "/etc/hosts" } },
+ *   { type: "tool_result", result: "127.0.0.1 localhost", success: true }
+ * ]);
+ * ```
+ */
+export async function appendJobOutputBatch(
+  jobsDir: string,
+  jobId: string,
+  messages: JobOutputInput[]
+): Promise<void> {
+  const outputPath = getJobOutputPath(jobsDir, jobId);
+
+  // Validate all messages first
+  for (let i = 0; i < messages.length; i++) {
+    if (!isValidJobOutputInput(messages[i])) {
+      throw new StateFileError(
+        `Invalid job output message at index ${i}: must have a valid 'type' field`,
+        outputPath,
+        "write"
+      );
+    }
+  }
+
+  // Write all messages
+  const timestamp = new Date().toISOString();
+  for (const message of messages) {
+    const messageWithTimestamp: JobOutputMessage = {
+      ...message,
+      timestamp,
+    } as JobOutputMessage;
+
+    try {
+      await appendJsonl(outputPath, messageWithTimestamp);
+    } catch (error) {
+      throw new StateFileError(
+        `Failed to append to job output: ${(error as Error).message}`,
+        outputPath,
+        "write",
+        error as Error
+      );
+    }
+  }
+}
+
+// =============================================================================
+// Read Operations
+// =============================================================================
+
+/**
+ * Read job output as an async generator for streaming reads
+ *
+ * Yields messages one at a time, allowing for memory-efficient
+ * processing of large output files.
+ *
+ * @param jobsDir - Path to the jobs directory
+ * @param jobId - Job ID
+ * @param options - Read options
+ * @yields JobOutputMessage objects
+ * @throws StateFileError if file cannot be read (except ENOENT which yields nothing)
+ *
+ * @example
+ * ```typescript
+ * for await (const message of readJobOutput(jobsDir, jobId)) {
+ *   console.log(`[${message.type}] ${message.timestamp}`);
+ * }
+ * ```
+ */
+export async function* readJobOutput(
+  jobsDir: string,
+  jobId: string,
+  options: ReadJobOutputOptions = {}
+): AsyncGenerator<JobOutputMessage, void, undefined> {
+  const { skipInvalidLines = false, logger = console } = options;
+  const outputPath = getJobOutputPath(jobsDir, jobId);
+
+  // Check if file exists
+  try {
+    await stat(outputPath);
+  } catch (error) {
+    if ((error as NodeJS.ErrnoException).code === "ENOENT") {
+      // File doesn't exist - yield nothing
+      return;
+    }
+    throw new StateFileError(
+      `Failed to read job output: ${(error as Error).message}`,
+      outputPath,
+      "read",
+      error as Error
+    );
+  }
+
+  // Create readline interface for streaming
+  const fileStream = createReadStream(outputPath, { encoding: "utf-8" });
+  const rl = createInterface({
+    input: fileStream,
+    crlfDelay: Infinity,
+  });
+
+  let lineNumber = 0;
+
+  try {
+    for await (const line of rl) {
+      lineNumber++;
+
+      // Skip empty lines
+      const trimmedLine = line.trim();
+      if (trimmedLine === "") {
+        continue;
+      }
+
+      try {
+        const parsed = JSON.parse(trimmedLine);
+        const validated = JobOutputMessageSchema.safeParse(parsed);
+
+        if (validated.success) {
+          yield validated.data;
+        } else if (skipInvalidLines) {
+          logger.warn(
+            `[herdctl] Skipping invalid message at line ${lineNumber} in ${outputPath}: ${validated.error.message}`
+          );
+        } else {
+          throw new StateFileError(
+            `Invalid job output message at line ${lineNumber}: ${validated.error.message}`,
+            outputPath,
+            "read"
+          );
+        }
+      } catch (error) {
+        if (error instanceof StateFileError) {
+          throw error;
+        }
+
+        if (skipInvalidLines) {
+          logger.warn(
+            `[herdctl] Skipping malformed JSON at line ${lineNumber} in ${outputPath}: ${(error as Error).message}`
+          );
+        } else {
+          throw new StateFileError(
+            `Failed to parse job output at line ${lineNumber}: ${(error as Error).message}`,
+            outputPath,
+            "read",
+            error as Error
+          );
+        }
+      }
+    }
+  } finally {
+    // Ensure resources are cleaned up
+    rl.close();
+    fileStream.destroy();
+  }
+}
+
+/**
+ * Read all job output messages into an array
+ *
+ * Convenience function that collects all messages from readJobOutput.
+ * For large files, prefer using readJobOutput directly as a generator.
+ *
+ * @param jobsDir - Path to the jobs directory
+ * @param jobId - Job ID
+ * @param options - Read options
+ * @returns Array of all job output messages
+ *
+ * @example
+ * ```typescript
+ * const messages = await readJobOutputAll(jobsDir, jobId);
+ * console.log(`Total messages: ${messages.length}`);
+ * ```
+ */
+export async function readJobOutputAll(
+  jobsDir: string,
+  jobId: string,
+  options: ReadJobOutputOptions = {}
+): Promise<JobOutputMessage[]> {
+  const messages: JobOutputMessage[] = [];
+  for await (const message of readJobOutput(jobsDir, jobId, options)) {
+    messages.push(message);
+  }
+  return messages;
+}

package/src/state/schemas/__tests__/job-output.test.ts

@@ -0,0 +1,338 @@
+import { describe, it, expect } from "vitest";
+import {
+  JobOutputTypeSchema,
+  JobOutputMessageSchema,
+  SystemMessageSchema,
+  AssistantMessageSchema,
+  ToolUseMessageSchema,
+  ToolResultMessageSchema,
+  ErrorMessageSchema,
+  validateJobOutputMessage,
+  isValidJobOutputInput,
+  type JobOutputMessage,
+  type JobOutputInput,
+} from "../job-output.js";
+
+describe("JobOutputTypeSchema", () => {
+  it("accepts valid types", () => {
+    expect(JobOutputTypeSchema.parse("system")).toBe("system");
+    expect(JobOutputTypeSchema.parse("assistant")).toBe("assistant");
+    expect(JobOutputTypeSchema.parse("tool_use")).toBe("tool_use");
+    expect(JobOutputTypeSchema.parse("tool_result")).toBe("tool_result");
+    expect(JobOutputTypeSchema.parse("error")).toBe("error");
+  });
+
+  it("rejects invalid types", () => {
+    expect(() => JobOutputTypeSchema.parse("invalid")).toThrow();
+    expect(() => JobOutputTypeSchema.parse("")).toThrow();
+    expect(() => JobOutputTypeSchema.parse(null)).toThrow();
+  });
+});
+
+describe("SystemMessageSchema", () => {
+  it("accepts valid system message", () => {
+    const message = {
+      type: "system" as const,
+      timestamp: "2024-01-15T10:00:00Z",
+      content: "System initialized",
+    };
+    expect(SystemMessageSchema.parse(message)).toEqual(message);
+  });
+
+  it("accepts system message with subtype", () => {
+    const message = {
+      type: "system" as const,
+      timestamp: "2024-01-15T10:00:00Z",
+      subtype: "session_start",
+    };
+    expect(SystemMessageSchema.parse(message).subtype).toBe("session_start");
+  });
+
+  it("accepts minimal system message", () => {
+    const message = {
+      type: "system" as const,
+      timestamp: "2024-01-15T10:00:00Z",
+    };
+    expect(SystemMessageSchema.parse(message)).toEqual(message);
+  });
+});
+
+describe("AssistantMessageSchema", () => {
+  it("accepts valid assistant message", () => {
+    const message = {
+      type: "assistant" as const,
+      timestamp: "2024-01-15T10:00:00Z",
+      content: "Hello, world!",
+    };
+    expect(AssistantMessageSchema.parse(message)).toEqual(message);
+  });
+
+  it("accepts assistant message with partial flag", () => {
+    const message = {
+      type: "assistant" as const,
+      timestamp: "2024-01-15T10:00:00Z",
+      content: "Partial...",
+      partial: true,
+    };
+    expect(AssistantMessageSchema.parse(message).partial).toBe(true);
+  });
+
+  it("accepts assistant message with usage info", () => {
+    const message = {
+      type: "assistant" as const,
+      timestamp: "2024-01-15T10:00:00Z",
+      content: "Response",
+      usage: {
+        input_tokens: 100,
+        output_tokens: 50,
+      },
+    };
+    expect(AssistantMessageSchema.parse(message).usage).toEqual({
+      input_tokens: 100,
+      output_tokens: 50,
+    });
+  });
+});
+
+describe("ToolUseMessageSchema", () => {
+  it("accepts valid tool_use message", () => {
+    const message = {
+      type: "tool_use" as const,
+      timestamp: "2024-01-15T10:00:00Z",
+      tool_name: "read_file",
+    };
+    expect(ToolUseMessageSchema.parse(message)).toEqual(message);
+  });
+
+  it("accepts tool_use with full details", () => {
+    const message = {
+      type: "tool_use" as const,
+      timestamp: "2024-01-15T10:00:00Z",
+      tool_name: "bash",
+      tool_use_id: "tool-123",
+      input: { command: "ls -la" },
+    };
+    const parsed = ToolUseMessageSchema.parse(message);
+    expect(parsed.tool_name).toBe("bash");
+    expect(parsed.tool_use_id).toBe("tool-123");
+    expect(parsed.input).toEqual({ command: "ls -la" });
+  });
+
+  it("rejects tool_use without tool_name", () => {
+    const message = {
+      type: "tool_use" as const,
+      timestamp: "2024-01-15T10:00:00Z",
+    };
+    expect(() => ToolUseMessageSchema.parse(message)).toThrow();
+  });
+});
+
+describe("ToolResultMessageSchema", () => {
+  it("accepts valid tool_result message", () => {
+    const message = {
+      type: "tool_result" as const,
+      timestamp: "2024-01-15T10:00:00Z",
+      result: "file contents",
+      success: true,
+    };
+    expect(ToolResultMessageSchema.parse(message).success).toBe(true);
+  });
+
+  it("accepts tool_result with error", () => {
+    const message = {
+      type: "tool_result" as const,
+      timestamp: "2024-01-15T10:00:00Z",
+      success: false,
+      error: "Command failed with exit code 1",
+    };
+    const parsed = ToolResultMessageSchema.parse(message);
+    expect(parsed.success).toBe(false);
+    expect(parsed.error).toBe("Command failed with exit code 1");
+  });
+
+  it("accepts minimal tool_result", () => {
+    const message = {
+      type: "tool_result" as const,
+      timestamp: "2024-01-15T10:00:00Z",
+    };
+    expect(ToolResultMessageSchema.parse(message)).toEqual(message);
+  });
+});
+
+describe("ErrorMessageSchema", () => {
+  it("accepts valid error message", () => {
+    const message = {
+      type: "error" as const,
+      timestamp: "2024-01-15T10:00:00Z",
+      message: "Something went wrong",
+    };
+    expect(ErrorMessageSchema.parse(message).message).toBe(
+      "Something went wrong"
+    );
+  });
+
+  it("accepts error with code and stack", () => {
+    const message = {
+      type: "error" as const,
+      timestamp: "2024-01-15T10:00:00Z",
+      message: "Timeout",
+      code: "ERR_TIMEOUT",
+      stack: "Error: Timeout\n  at foo.ts:10",
+    };
+    const parsed = ErrorMessageSchema.parse(message);
+    expect(parsed.code).toBe("ERR_TIMEOUT");
+    expect(parsed.stack).toContain("foo.ts:10");
+  });
+
+  it("rejects error without message", () => {
+    const message = {
+      type: "error" as const,
+      timestamp: "2024-01-15T10:00:00Z",
+    };
+    expect(() => ErrorMessageSchema.parse(message)).toThrow();
+  });
+});
+
+describe("JobOutputMessageSchema (discriminated union)", () => {
+  it("parses system message", () => {
+    const msg: JobOutputMessage = JobOutputMessageSchema.parse({
+      type: "system",
+      timestamp: "2024-01-15T10:00:00Z",
+      content: "Init",
+    });
+    expect(msg.type).toBe("system");
+  });
+
+  it("parses assistant message", () => {
+    const msg: JobOutputMessage = JobOutputMessageSchema.parse({
+      type: "assistant",
+      timestamp: "2024-01-15T10:00:00Z",
+      content: "Hello",
+    });
+    expect(msg.type).toBe("assistant");
+  });
+
+  it("parses tool_use message", () => {
+    const msg: JobOutputMessage = JobOutputMessageSchema.parse({
+      type: "tool_use",
+      timestamp: "2024-01-15T10:00:00Z",
+      tool_name: "test",
+    });
+    expect(msg.type).toBe("tool_use");
+  });
+
+  it("parses tool_result message", () => {
+    const msg: JobOutputMessage = JobOutputMessageSchema.parse({
+      type: "tool_result",
+      timestamp: "2024-01-15T10:00:00Z",
+    });
+    expect(msg.type).toBe("tool_result");
+  });
+
+  it("parses error message", () => {
+    const msg: JobOutputMessage = JobOutputMessageSchema.parse({
+      type: "error",
+      timestamp: "2024-01-15T10:00:00Z",
+      message: "Oops",
+    });
+    expect(msg.type).toBe("error");
+  });
+
+  it("rejects invalid type", () => {
+    expect(() =>
+      JobOutputMessageSchema.parse({
+        type: "invalid",
+        timestamp: "2024-01-15T10:00:00Z",
+      })
+    ).toThrow();
+  });
+
+  it("rejects missing type", () => {
+    expect(() =>
+      JobOutputMessageSchema.parse({
+        timestamp: "2024-01-15T10:00:00Z",
+      })
+    ).toThrow();
+  });
+});
+
+describe("validateJobOutputMessage", () => {
+  it("returns parsed message for valid input", () => {
+    const input = {
+      type: "assistant",
+      timestamp: "2024-01-15T10:00:00Z",
+      content: "Hello",
+    };
+    const result = validateJobOutputMessage(input);
+    expect(result).not.toBeNull();
+    expect(result?.type).toBe("assistant");
+  });
+
+  it("returns null for invalid input", () => {
+    expect(validateJobOutputMessage({ type: "invalid" })).toBeNull();
+    expect(validateJobOutputMessage(null)).toBeNull();
+    expect(validateJobOutputMessage("string")).toBeNull();
+    expect(validateJobOutputMessage(123)).toBeNull();
+  });
+
+  it("returns null for missing required fields", () => {
+    expect(
+      validateJobOutputMessage({
+        type: "error",
+        timestamp: "2024-01-15T10:00:00Z",
+        // missing message
+      })
+    ).toBeNull();
+
+    expect(
+      validateJobOutputMessage({
+        type: "tool_use",
+        timestamp: "2024-01-15T10:00:00Z",
+        // missing tool_name
+      })
+    ).toBeNull();
+  });
+});
+
+describe("isValidJobOutputInput", () => {
+  it("returns true for valid input objects", () => {
+    expect(isValidJobOutputInput({ type: "system" })).toBe(true);
+    expect(isValidJobOutputInput({ type: "assistant", content: "Hi" })).toBe(
+      true
+    );
+    expect(isValidJobOutputInput({ type: "tool_use", tool_name: "test" })).toBe(
+      true
+    );
+    expect(isValidJobOutputInput({ type: "tool_result" })).toBe(true);
+    expect(isValidJobOutputInput({ type: "error", message: "Err" })).toBe(true);
+  });
+
+  it("returns false for invalid type", () => {
+    expect(isValidJobOutputInput({ type: "invalid" })).toBe(false);
+    expect(isValidJobOutputInput({ type: "" })).toBe(false);
+  });
+
+  it("returns false for non-objects", () => {
+    expect(isValidJobOutputInput(null)).toBe(false);
+    expect(isValidJobOutputInput(undefined)).toBe(false);
+    expect(isValidJobOutputInput("string")).toBe(false);
+    expect(isValidJobOutputInput(123)).toBe(false);
+    expect(isValidJobOutputInput([])).toBe(false);
+  });
+
+  it("returns false for objects without type", () => {
+    expect(isValidJobOutputInput({})).toBe(false);
+    expect(isValidJobOutputInput({ content: "Hi" })).toBe(false);
+  });
+
+  it("returns false for objects with non-string type", () => {
+    expect(isValidJobOutputInput({ type: 123 })).toBe(false);
+    expect(isValidJobOutputInput({ type: null })).toBe(false);
+    expect(isValidJobOutputInput({ type: {} })).toBe(false);
+  });
+
+  it("allows objects without timestamp (timestamp added later)", () => {
+    const input: JobOutputInput = { type: "assistant", content: "Hi" };
+    expect(isValidJobOutputInput(input)).toBe(true);
+  });
+});