anima-core 1.0.2 → 1.1.0

data/lib/mneme.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+# Mneme — the memory department. Watches for viewport eviction and creates
+# summaries before context is lost. Named after the Greek Titaness of memory.
+#
+# Mneme is the third event bus department alongside Nous (main agent) and
+# the Analytical Brain. It operates as a phantom LLM loop: observes the
+# main session, creates snapshots, but leaves no trace of its own reasoning.
+module Mneme
+  # Dev-only logger that writes to log/mneme.log.
+  # In non-development environments returns a null logger so
+  # call sites don't need conditionals.
+  #
+  # @return [Logger]
+  def self.logger
+    @logger ||= build_logger
+  end
+
+  def self.build_logger
+    return Logger.new(File::NULL) unless Rails.env.development?
+
+    Logger.new(Rails.root.join("log", "mneme.log")).tap do |log|
+      log.formatter = proc { |severity, time, _progname, msg|
+        "[#{time.strftime("%H:%M:%S.%L")}] #{severity}  #{msg}\n"
+      }
+    end
+  end
+  private_class_method :build_logger
+end

data/lib/providers/anthropic.rb

@@ -13,6 +13,10 @@ module Providers
     API_VERSION = "2023-06-01"
     REQUIRED_BETA = "oauth-2025-04-20"
 
+    # Anthropic requires this exact string as the first system block for OAuth
+    # subscription tokens on Sonnet/Opus. Without it, /v1/messages returns 400.
+    OAUTH_PASSPHRASE = "You are Claude Code, Anthropic's official CLI for Claude."
+
     class Error < StandardError; end
     class AuthenticationError < Error; end
     class TokenFormatError < Error; end
@@ -25,11 +29,13 @@ module Providers
     class << self
       def fetch_token
         token = CredentialStore.read("anthropic", "subscription_token")
-        raise AuthenticationError, <<~MSG.strip if token.blank?
+        return token if token.present?
+        return "sk-ant-oat01-#{"0" * 68}" if ENV["CI"]
+
+        raise AuthenticationError, <<~MSG.strip
           No Anthropic subscription token found in credentials.
           Use the TUI token setup (Ctrl+a → a) to configure your token.
         MSG
-        token
       end
 
       def validate_token_format!(token)
@@ -46,6 +52,13 @@ module Providers
         true
       end
 
+      # Validate a token against the live Anthropic API.
+      # Delegates to {#validate_credentials!} on a throwaway instance.
+      #
+      # @param token [String] Anthropic API token to validate
+      # @return [true] when the API accepts the token
+      # @raise [TransientError] on network failures or server errors (retryable)
+      # @raise [AuthenticationError] on 401/403 (permanent)
       def validate_token_api!(token)
         provider = new(token)
         provider.validate_credentials!
@@ -58,7 +71,18 @@ module Providers
       @token = token || self.class.fetch_token
     end
 
+    # Send a message to the Anthropic API and return the parsed response.
+    #
+    # @param model [String] Anthropic model identifier
+    # @param messages [Array<Hash>] conversation messages
+    # @param max_tokens [Integer] maximum tokens in the response
+    # @param options [Hash] additional parameters (e.g. +system:+, +tools:+)
+    # @return [Hash] parsed API response
+    # @raise [TransientError] on network failures or server errors (retryable)
+    # @raise [AuthenticationError] on 401/403 (permanent)
+    # @raise [Error] on other API errors
     def create_message(model:, messages:, max_tokens:, **options)
+      wrap_system_prompt!(options)
       body = {model: model, messages: messages, max_tokens: max_tokens}.merge(options)
 
       response = self.class.post(
@@ -69,8 +93,8 @@ module Providers
       )
 
       handle_response(response)
-    rescue Errno::ECONNRESET, Net::ReadTimeout, Net::OpenTimeout, SocketError, EOFError => e
-      raise TransientError, "#{e.class}: #{e.message}"
+    rescue Errno::ECONNRESET, Net::ReadTimeout, Net::OpenTimeout, SocketError, EOFError => network_error
+      raise TransientError, "#{network_error.class}: #{network_error.message}"
     end
 
     # Count tokens in a message payload without creating a message.
@@ -82,6 +106,7 @@ module Providers
     # @return [Integer] estimated input token count
     # @raise [Error] on API errors
     def count_tokens(model:, messages:, **options)
+      wrap_system_prompt!(options)
       body = {model: model, messages: messages}.merge(options)
 
       response = self.class.post(
@@ -93,18 +118,22 @@ module Providers
 
       result = handle_response(response)
       result["input_tokens"]
-    rescue Errno::ECONNRESET, Net::ReadTimeout, Net::OpenTimeout, SocketError, EOFError => e
-      raise TransientError, "#{e.class}: #{e.message}"
+    rescue Errno::ECONNRESET, Net::ReadTimeout, Net::OpenTimeout, SocketError, EOFError => network_error
+      raise TransientError, "#{network_error.class}: #{network_error.message}"
     end
 
+    # Verify the token is accepted by Anthropic using the free models endpoint.
+    # Returns +true+ on success; raises typed exceptions on failure so callers
+    # can distinguish permanent auth problems from transient outages.
+    #
+    # @return [true] when the API accepts the token
+    # @raise [AuthenticationError] on 401 (invalid token) or 403 (restricted credential)
+    # @raise [RateLimitError] on 429
+    # @raise [ServerError] on 5xx
+    # @raise [TransientError] on network-level failures
     def validate_credentials!
-      response = self.class.post(
-        "/v1/messages",
-        body: {
-          model: Anima::Settings.model,
-          messages: [{role: "user", content: "Hi"}],
-          max_tokens: 1
-        }.to_json,
+      response = self.class.get(
+        "/v1/models",
         headers: request_headers,
         timeout: Anima::Settings.api_timeout
       )
@@ -121,10 +150,25 @@ module Providers
       else
         handle_response(response)
       end
+    rescue Errno::ECONNRESET, Net::ReadTimeout, Net::OpenTimeout, SocketError, EOFError => network_error
+      raise TransientError, "#{network_error.class}: #{network_error.message}"
     end
 
     private
 
+    # Wraps the system parameter in the array-of-blocks format required by
+    # Anthropic for OAuth tokens. The passphrase block is always present;
+    # the caller's prompt (if any) is appended as the second block.
+    #
+    # @param options [Hash] mutable options hash (modified in place)
+    # @return [void]
+    def wrap_system_prompt!(options)
+      prompt = options[:system]
+      blocks = [{type: "text", text: OAUTH_PASSPHRASE}]
+      blocks << {type: "text", text: prompt} if prompt
+      options[:system] = blocks
+    end
+
     def request_headers
       {
         "Authorization" => "Bearer #{token}",

data/lib/shell_session.rb

@@ -3,12 +3,20 @@
 require "io/console"
 require "pty"
 require "securerandom"
-require "timeout"
+require "shellwords"
 
 # Persistent shell session backed by a PTY with FIFO-based stderr separation.
 # Commands share working directory, environment variables, and shell history
 # within a conversation. Multiple tools share the same session.
 #
+# Auto-recovers from timeouts and crashes: if the shell dies, the next command
+# transparently respawns a fresh shell and restores the working directory.
+#
+# Uses IO.select-based deadlines instead of Timeout.timeout for all PTY reads.
+# Timeout.timeout is unsafe with PTY I/O — it uses Thread.raise which can
+# corrupt mutex state, leave resources inconsistent, and cause exceptions
+# to fire outside handler blocks when nested.
+#
 # @example
 #   session = ShellSession.new(session_id: 42)
 #   session.run("cd /tmp")
@@ -25,27 +33,37 @@ class ShellSession
     @mutex = Mutex.new
     @fifo_path = File.join(Dir.tmpdir, "anima-stderr-#{Process.pid}-#{SecureRandom.hex(8)}")
     @alive = false
+    @finalized = false
     @pwd = nil
+    @read_buffer = +""
     self.class.cleanup_orphans
     start
     self.class.register(self)
   end
 
-  # Execute a command in the persistent shell.
+  # Execute a command in the persistent shell. Respawns the shell
+  # automatically if the previous session died (timeout, crash, etc.).
   #
   # @param command [String] bash command to execute
   # @return [Hash] with :stdout, :stderr, :exit_code keys on success
   # @return [Hash] with :error key on failure
   def run(command)
     @mutex.synchronize do
-      return {error: "Shell session is not running"} unless @alive
+      return {error: "Shell session is not running"} if @finalized
+      restart unless @alive
       execute_in_pty(command)
     end
+  rescue => error # rubocop:disable Lint/RescueException -- LLM must always get a result hash, never a stack trace
+    {error: "#{error.class}: #{error.message}"}
   end
 
-  # Clean up PTY, FIFO, and child process.
+  # Clean up PTY, FIFO, and child process. Permanent — the session
+  # will not auto-respawn after this call.
   def finalize
-    @mutex.synchronize { shutdown }
+    @mutex.synchronize do
+      @finalized = true
+      teardown
+    end
     self.class.unregister(self)
   end
 
@@ -73,7 +91,7 @@ class ShellSession
     # Finalize all live sessions. Called automatically via at_exit.
     def cleanup_all
       @sessions_mutex.synchronize do
-        @sessions.each { |session| session.send(:shutdown) }
+        @sessions.each { |session| session.send(:teardown) }
         @sessions.clear
       end
     end
@@ -116,15 +134,54 @@ class ShellSession
     @alive = true
   end
 
+  # Shuts down the current shell and spawns a fresh one, restoring the
+  # previous working directory. Called automatically when @alive is false.
+  def restart
+    saved_pwd = @pwd
+    teardown
+    @fifo_path = File.join(Dir.tmpdir, "anima-stderr-#{Process.pid}-#{SecureRandom.hex(8)}")
+    start
+    restore_working_directory(saved_pwd)
+  end
+
+  # Restores the shell's working directory after a respawn.
+  # Skips silently if the directory no longer exists.
+  #
+  # @param saved_pwd [String, nil] directory path to restore
+  # @return [void]
+  def restore_working_directory(saved_pwd)
+    return unless saved_pwd && File.directory?(saved_pwd)
+    execute_in_pty("cd #{Shellwords.shellescape(saved_pwd)}")
+  end
+
   def create_fifo
-    File.mkfifo(@fifo_path)
+    File.mkfifo(@fifo_path, 0o600)
   rescue Errno::EEXIST
     # FIFO already exists — reuse it
   end
 
+  # Env vars that prevent interactive pagers and credential prompts from
+  # hanging the PTY. We need a PTY (not pipes) for pwd tracking via /proc
+  # and signal handling, but this makes programs think they're on a terminal
+  # and launch pagers. No single switch disables all pagers — each tool has
+  # its own env var — so we set a comprehensive list plus LESS flags as a
+  # safety net for direct `less` invocations.
+  SHELL_ENV = {
+    "TERM" => "dumb",
+    "PAGER" => "cat",                   # Default pager for most Unix tools
+    "LESS" => "-eFRX",                  # Safety net: make less auto-exit at EOF, no screen clear
+    "GIT_PAGER" => "cat",              # Git checks this before PAGER
+    "MANPAGER" => "cat",               # man pages
+    "SYSTEMD_PAGER" => "",             # journalctl, systemctl (empty = disable)
+    "BAT_PAGER" => "cat",             # bat (cat alternative)
+    "AWS_PAGER" => "",                 # AWS CLI v2 (empty = disable)
+    "PSQL_PAGER" => "cat",            # PostgreSQL psql
+    "GIT_TERMINAL_PROMPT" => "0"       # Fail immediately instead of prompting for credentials
+  }.freeze
+
   def spawn_shell
     @pty_stdout, @pty_stdin, @pid = PTY.spawn(
-      {"TERM" => "dumb"},
+      SHELL_ENV,
       "bash", "--norc", "--noprofile"
     )
     # Disable terminal echo via termios before bash can echo our commands.
@@ -165,45 +222,57 @@ class ShellSession
     @pty_stdin.puts "PS1=''"
     @pty_stdin.puts "exec 2>#{@fifo_path}"
     @pty_stdin.puts "echo '#{marker}'"
-    consume_until(marker)
+    unless consume_until(marker, deadline: monotonic_now + 10)
+      raise IOError, "Shell initialization timed out"
+    end
   end
 
   def execute_in_pty(command)
     clear_stderr
     marker = "__ANIMA_#{SecureRandom.hex(8)}__"
     timeout = Anima::Settings.command_timeout
+    deadline = monotonic_now + timeout
 
-    Timeout.timeout(timeout) do
-      # All on one line: run command, capture exit code, ensure newline
-      # before marker so output without trailing newline doesn't merge.
-      @pty_stdin.puts "#{command}; __anima_ec=$?; echo; echo '#{marker}' $__anima_ec"
+    @pty_stdin.puts "#{command}; __anima_ec=$?; echo; echo '#{marker}' $__anima_ec"
 
-      stdout, exit_code = read_until_marker(marker)
-      update_pwd
-      stderr = drain_stderr
+    stdout, exit_code = read_until_marker(marker, deadline: deadline)
 
-      {
-        stdout: truncate(stdout),
-        stderr: truncate(stderr),
-        exit_code: exit_code
-      }
+    if exit_code.nil?
+      recover_from_timeout
+      stderr = drain_stderr
+      parts = ["Command timed out after #{timeout} seconds."]
+      parts << "Partial stdout:\n#{truncate(stdout)}" unless stdout.empty?
+      parts << "stderr:\n#{truncate(stderr)}" unless stderr.empty?
+      return {error: parts.join("\n\n")}
     end
-  rescue Timeout::Error
-    recover_from_timeout
-    {error: "Command timed out after #{timeout} seconds"}
-  rescue Errno::EIO
+
+    update_pwd
+    stderr = drain_stderr
+
+    {
+      stdout: truncate(stdout),
+      stderr: truncate(stderr),
+      exit_code: exit_code
+    }
+  rescue Errno::EIO, IOError
     @alive = false
     {error: "Shell session terminated unexpectedly"}
-  rescue => error
+  rescue => error # rubocop:disable Lint/RescueException -- LLM must always get a result hash, never a stack trace
     {error: "#{error.class}: #{error.message}"}
   end
 
-  def read_until_marker(marker)
+  # Reads lines from the PTY until the marker appears.
+  #
+  # @param marker [String] unique marker to detect command completion
+  # @param deadline [Float] monotonic clock deadline
+  # @return [Array(String, Integer)] stdout and exit code on success
+  # @return [Array(String, nil)] partial stdout and nil exit code on timeout
+  def read_until_marker(marker, deadline:)
     lines = []
     exit_code = nil
 
     loop do
-      line = @pty_stdout.gets
+      line = gets_with_deadline(deadline)
       break if line.nil?
 
       line = line.chomp.delete("\r")
@@ -219,26 +288,70 @@ class ShellSession
     # Strip trailing empty line added by our separator echo
     lines.pop if lines.last == ""
 
-    [lines.join("\n"), exit_code || -1]
+    [lines.join("\n"), exit_code]
   end
 
-  def consume_until(marker)
+  # Reads and discards PTY output until the marker appears or deadline expires.
+  #
+  # @param marker [String] unique marker to wait for
+  # @param deadline [Float] monotonic clock deadline
+  # @return [Boolean] true if marker was found, false if deadline expired
+  # @raise [Errno::EIO] when the PTY child process has exited
+  # @raise [IOError] when the PTY file descriptor is closed
+  def consume_until(marker, deadline:)
     loop do
-      line = @pty_stdout.gets
-      break if line.nil?
-      break if line.chomp.delete("\r").include?(marker)
+      line = gets_with_deadline(deadline)
+      return false if line.nil?
+      return true if line.chomp.delete("\r").include?(marker)
+    end
+  end
+
+  # Reads a single line from the PTY, respecting a deadline.
+  # Caller must hold @mutex — @read_buffer is not independently synchronized.
+  #
+  # Uses IO.select for safe, non-interruptive timeout handling instead of
+  # Timeout.timeout (which uses Thread.raise that can corrupt mutex state
+  # and leave resources inconsistent).
+  #
+  # @param deadline [Float] monotonic clock deadline
+  # @return [String] line including trailing newline
+  # @return [nil] if deadline expired
+  # @raise [Errno::EIO] when the PTY child process exits (Linux)
+  # @raise [IOError] when the PTY file descriptor is closed
+  def gets_with_deadline(deadline)
+    loop do
+      if (idx = @read_buffer.index("\n"))
+        return @read_buffer.slice!(0..idx)
+      end
+
+      remaining = deadline - monotonic_now
+      return nil if remaining <= 0
+
+      ready = IO.select([@pty_stdout], nil, nil, remaining)
+      return nil unless ready
+
+      begin
+        @read_buffer << @pty_stdout.read_nonblock(4096)
+      rescue IO::WaitReadable
+        # Spurious wakeup from IO.select — retry
+      end
     end
   end
 
   # Sends Ctrl+C to interrupt the running command and drains leftover output.
-  # If recovery fails, marks the session as dead.
+  # If recovery fails, marks the session as dead (will be respawned on next run).
+  #
+  # @return [void]
+  # @raise [Errno::EIO] when the PTY child process has exited
+  # @raise [IOError] when the PTY file descriptor is closed
   def recover_from_timeout
     @pty_stdin.write("\x03")
     sleep 0.1
     marker = "__ANIMA_RECOVER_#{SecureRandom.hex(8)}__"
     @pty_stdin.puts "echo '#{marker}'"
-    Timeout.timeout(3) { consume_until(marker) }
-  rescue Timeout::Error, Errno::EIO, IOError
+    recovered = consume_until(marker, deadline: monotonic_now + 3)
+    @alive = false unless recovered
+  rescue Errno::EIO, IOError
     @alive = false
   end
 
@@ -283,15 +396,26 @@ class ShellSession
       "\n\n[Truncated: output exceeded #{max_bytes} bytes]"
   end
 
-  def shutdown
-    return unless @alive
-    @alive = false
+  def monotonic_now
+    Process.clock_gettime(Process::CLOCK_MONOTONIC)
+  end
 
-    begin
-      pgid = Process.getpgid(@pid)
-      Process.kill("TERM", -pgid)
-    rescue Errno::ESRCH, Errno::EPERM
-      # Process group already gone
+  # Unconditionally cleans up all shell resources (PTY, FIFO, child process).
+  # Does NOT short-circuit when @alive is already false — this ensures leaked
+  # processes are reaped even after failed recovery marked the session dead.
+  #
+  # @return [void]
+  def teardown
+    @alive = false
+    @read_buffer = +""
+
+    if @pid
+      begin
+        pgid = Process.getpgid(@pid)
+        Process.kill("TERM", -pgid)
+      rescue Errno::ESRCH, Errno::EPERM
+        # Process group already gone
+      end
     end
 
     begin
@@ -307,28 +431,33 @@ class ShellSession
     end
 
     begin
+      @stderr_thread&.join(1)
       @stderr_thread&.kill
     rescue ThreadError
       # Thread already dead
     end
 
-    File.delete(@fifo_path) if File.exist?(@fifo_path)
-
-    begin
-      # Non-blocking reap with SIGKILL fallback if process doesn't exit in time
-      deadline = Process.clock_gettime(Process::CLOCK_MONOTONIC) + 2
-      loop do
-        _, status = Process.wait2(@pid, Process::WNOHANG)
-        break if status
-        if Process.clock_gettime(Process::CLOCK_MONOTONIC) > deadline
-          Process.kill("KILL", @pid)
-          Process.wait(@pid)
-          break
+    File.delete(@fifo_path) if @fifo_path && File.exist?(@fifo_path)
+
+    if @pid
+      begin
+        # Non-blocking reap with SIGKILL fallback if process doesn't exit in time
+        deadline = monotonic_now + 2
+        loop do
+          _, status = Process.wait2(@pid, Process::WNOHANG)
+          break if status
+          if monotonic_now > deadline
+            Process.kill("KILL", @pid)
+            Process.wait(@pid)
+            break
+          end
+          sleep 0.05
         end
-        sleep 0.05
+      rescue Errno::ECHILD, Errno::ESRCH
+        # Already reaped
       end
-    rescue Errno::ECHILD, Errno::ESRCH
-      # Already reaped
+
+      @pid = nil
     end
   end
 end

data/lib/tasks/fts5.rake

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+
+# No custom Rake tasks needed — FTS5 virtual tables are handled by:
+# - Migration: creates/drops the FTS5 table and triggers
+# - Initializer (config/initializers/fts5_schema_dump.rb): skips virtual
+#   tables during schema dump since they can't be expressed in Ruby DSL