pikuri-core 0.0.3

data/lib/pikuri/subprocess.rb

@@ -0,0 +1,166 @@
+# frozen_string_literal: true
+
+require 'open3'
+require 'set'
+
+module Pikuri
+  # Chokepoint for *all* subprocess spawning in pikuri. Forces a new
+  # process group for each invocation, tracks pgids so descendants of
+  # the direct child (commands backgrounded with +&+) can be cleaned
+  # up at process exit, and captures combined stdout+stderr through a
+  # single pipe.
+  #
+  # == Seam discipline
+  #
+  # All subprocess spawning in +lib/+ goes through {.spawn}. Direct
+  # +Process.spawn+ / +Open3.*+ / +system+ / backticks anywhere in
+  # +lib/+ are bugs. The convention is grep-enforceable:
+  # +grep -rn 'Process\.spawn\|Open3\|system\|backtick' lib/+ should
+  # only hit this file.
+  #
+  # == Timeouts are the caller's job
+  #
+  # {.spawn} does not implement a timeout — Ruby's +Timeout.timeout+
+  # cannot kill subprocesses cleanly. Callers that need a timeout
+  # wrap their argv with coreutils' +timeout+ binary:
+  #
+  #   Pikuri::Subprocess.spawn(
+  #     'timeout', '--signal=TERM', '--kill-after=5s', '120s',
+  #     'bash', '-c', command,
+  #     chdir: workspace.cwd.to_s
+  #   )
+  #
+  # When +timeout+ and its FD-inheriting children die, the combined
+  # output pipe closes and {#wait}'s +io.read+ returns. No Ruby-side
+  # timeout machinery; the +timeout+ binary handles SIGTERM-then-
+  # SIGKILL race-free.
+  #
+  # == Backgrounded subprocesses
+  #
+  # When a shell command backgrounds work with +&+, the resulting
+  # process stays in our pgroup. {#wait} returns as soon as the
+  # direct child exits, but {.active} keeps the pgid in the tracked
+  # set as long as any process in the group is alive (probed with
+  # +kill(0, -pgid)+). On pikuri exit, {.cleanup!} sends SIGTERM to
+  # every tracked group. The model can opt out via +nohup cmd &+ or
+  # +setsid cmd &+ — both detach from our group.
+  #
+  # == State is process-global
+  #
+  # One +@active+ Set and one +at_exit+ for the whole process. A
+  # +Mutex+ guards register/prune/cleanup; v1 is single-threaded, so
+  # this is more for the +at_exit+/register race than for current
+  # callers.
+  #
+  # == Why +Pikuri::Subprocess+, not top-level
+  #
+  # First class actually under the +Pikuri::+ namespace. Domain
+  # classes (+Tool+, +Agent+, +URLCache+) are top-level as a legacy
+  # convention — they predate the namespacing decision and an
+  # eventual refactor moves them too. For now: library-level
+  # infrastructure under +Pikuri::+; domain objects flat. See
+  # +CLAUDE.md+ for the convention.
+  class Subprocess
+    # Combined output + exit status, returned from {#wait}.
+    Result = Data.define(:output, :status)
+
+    # Spawn +argv+ in a new process group, redirecting stderr onto
+    # stdout. Tracked for cleanup.
+    #
+    # @param argv [Array<String>] command and arguments. Caller does
+    #   any shell wrapping (e.g. +'bash', '-c', cmd+) when shell
+    #   interpretation is wanted; +argv+ is passed to +exec+
+    #   directly, so no implicit shell expansion happens here.
+    # @param chdir [String, Pathname] working directory
+    # @return [Subprocess] handle — call {#wait} to block for the
+    #   direct child to exit and read the captured output
+    def self.spawn(*argv, chdir:)
+      stdin, io, wait_thr = Open3.popen2e(*argv, chdir: chdir.to_s, pgroup: true)
+      stdin.close
+      register(wait_thr.pid)
+      new(io: io, wait_thr: wait_thr)
+    end
+
+    # @return [Integer] direct child's pid
+    attr_reader :pid
+
+    # @return [Integer] process group id. Equal to {#pid} since the
+    #   child was spawned with +pgroup: true+ (it's the group leader).
+    attr_reader :pgid
+
+    # @return [IO] read end of the combined stdout+stderr pipe.
+    #   Exposed for future live-streaming consumers; v1 callers go
+    #   straight to {#wait}, which drains it.
+    attr_reader :io
+
+    # @api private — call {.spawn}, not the constructor.
+    def initialize(io:, wait_thr:)
+      @io       = io
+      @wait_thr = wait_thr
+      @pid      = wait_thr.pid
+      @pgid     = wait_thr.pid # pgroup:true → pgid == pid
+    end
+
+    # Block until the direct child exits, read whatever remains on
+    # the combined-output pipe, return a {Result}. The pgid stays
+    # tracked if the group still has live members (backgrounded
+    # children); pruned if everything's gone.
+    #
+    # @return [Result]
+    def wait
+      output = @io.read
+      @io.close
+      Result.new(output: output, status: @wait_thr.value)
+    ensure
+      self.class.send(:prune, @pgid)
+    end
+
+    class << self
+      # Currently-tracked process groups, with dead ones pruned as a
+      # side effect. Useful for a future +/bg+ REPL command or a
+      # between-turn status line.
+      #
+      # @return [Array<Integer>]
+      def active
+        @mutex.synchronize do
+          @active.delete_if { |g| !alive?(g) }
+          @active.to_a
+        end
+      end
+
+      # SIGTERM every tracked process group. Used by +at_exit+
+      # (production) and +after+ blocks (specs). Best-effort —
+      # ignores errors from already-dead groups.
+      #
+      # @return [void]
+      def cleanup!
+        @mutex.synchronize do
+          @active.each { |g| Process.kill('-TERM', g) rescue nil }
+          @active.clear
+        end
+      end
+
+      private
+
+      def register(pgid)
+        @mutex.synchronize { @active << pgid }
+      end
+
+      def prune(pgid)
+        @mutex.synchronize { @active.delete(pgid) unless alive?(pgid) }
+      end
+
+      def alive?(pgid)
+        Process.kill(0, -pgid)
+        true
+      rescue Errno::ESRCH
+        false
+      end
+    end
+
+    @active = Set.new
+    @mutex  = Mutex.new
+  end
+end
+
+at_exit { Pikuri::Subprocess.cleanup! }

data/lib/pikuri/tool/calculator.rb

@@ -0,0 +1,82 @@
+# frozen_string_literal: true
+
+require 'dentaku'
+
+module Pikuri
+  class Tool
+    # Evaluates a basic arithmetic expression using Dentaku, with light
+    # preprocessing so the LLM can emit Python-flavored syntax (notably
+    # +**+ for exponentiation) instead of learning Dentaku's dialect.
+    #
+    # Scope is intentionally narrow: operators (+, -, *, /, **, %),
+    # parentheses, and decimal numbers. No variables, functions, or
+    # booleans — those would mean teaching the model a dialect, which we
+    # specifically want to avoid for this tool.
+    module Calculator
+      # Translate the operator differences between Python and Dentaku. In
+      # practice that is only +**+ → +^+; everything else in the supported
+      # subset is byte-identical.
+      #
+      # @param expression [String] raw expression as the model wrote it
+      # @return [String] expression with Python-style operators rewritten
+      def self.normalize(expression)
+        expression.gsub('**', '^')
+      end
+
+      # Evaluate +expression+ and return the result formatted as a String.
+      # Parse, unbound-variable, and division-by-zero failures are caught
+      # and returned as +"Error: ..."+ strings so the model can read the
+      # failure as the next observation and self-correct rather than
+      # crashing the agent loop.
+      #
+      # @param expression [String]
+      # @return [String] numeric result, or +"Error: ..."+ on failure
+      def self.calculate(expression)
+        result = Dentaku::Calculator.new.evaluate!(normalize(expression))
+        format_result(result)
+      rescue Dentaku::ZeroDivisionError, ZeroDivisionError
+        'Error: division by zero'
+      rescue Dentaku::Error => e
+        "Error: #{e.message}"
+      end
+
+      # Dentaku returns BigDecimal for any expression that touches division
+      # or a decimal literal, with full BigDecimal precision (47-digit tails
+      # for the leopard expression). Round to 3 places and strip the
+      # default scientific-notation formatting so the model sees a short
+      # readable number; integer/other results pass through unchanged.
+      def self.format_result(result)
+        case result
+        when BigDecimal then result.round(3).to_s('F')
+        else result.to_s
+        end
+      end
+      private_class_method :format_result
+    end
+
+    # Arithmetic-evaluation tool backed by {Tool::Calculator.calculate}.
+    # Accepts Python-flavored operator syntax (+, -, *, /, ** for
+    # exponentiation, %, parentheses, decimals) so the model can emit the
+    # syntax it already knows.
+    #
+    # @return [Tool]
+    CALCULATOR = new(
+      name: 'calculator',
+      description: <<~DESC,
+        Evaluates a basic arithmetic expression and returns the numeric result.
+
+        Usage:
+        - Use this for any arithmetic beyond simple mental math — do not eyeball multi-digit work.
+        - Operators supported: +, -, *, /, ** (exponentiation), %, parentheses, decimal numbers.
+        - Decimal results are rounded to 3 places; integer results are exact.
+        - Failures (parse error, division by zero) come back as "Error: ..." — read the message and re-call with a corrected expression.
+      DESC
+      parameters: Parameters.build { |p|
+        p.required_string :expression,
+                          'Arithmetic expression to evaluate, e.g. ' \
+                          '"155 / (58 * 1000.0 / 3600)" or "2**10".'
+      },
+      execute: ->(expression:) { Calculator.calculate(expression) }
+    )
+  end
+end

data/lib/pikuri/tool/fetch.rb

@@ -0,0 +1,171 @@
+# frozen_string_literal: true
+
+module Pikuri
+  class Tool
+    # Truncation policy and Tool spec for the +fetch+ tool. The HTTP work
+    # lives in {Tool::Scraper::Simple.fetch}; this module is a thin
+    # wrapper that accepts only textual content-types, applies a character
+    # cap so the LLM doesn't drown in long-form bodies, and exposes the
+    # result to the agent loop in OpenAI tool-call shape.
+    #
+    # Sister of {Tool::WebScrape}, but without HTML→Markdown or PDF→text
+    # extraction: bodies are returned verbatim. Useful for raw textual
+    # data — JSON APIs, CSV files, +robots.txt+, sitemaps, source files —
+    # where any rendering pass would corrupt the payload.
+    module Fetch
+      # @return [Integer] default character cap on the body returned by
+      #   {.fetch}. Smaller than {Tool::WebScrape::DEFAULT_MAX_CHARS}
+      #   because fetch's content profile is bimodal — most JSON/XML/CSV
+      #   responses are tiny, and the long-tail (large data dumps) is
+      #   better re-requested deliberately than padded into every default.
+      DEFAULT_MAX_CHARS = 5_000
+
+      # @return [Integer] hard ceiling on the +max_chars+ argument to
+      #   {.fetch}. Matches {Tool::WebScrape::MAX_MAX_CHARS}.
+      MAX_MAX_CHARS = 100_000
+
+      # Application content-types that are textual in practice and so
+      # safe to return verbatim to the LLM, despite their +application/+
+      # prefix making them fail the +text/*+ check. Anything outside
+      # +text/*+ and this allowlist is refused.
+      # @return [Array<String>]
+      TEXTUAL_APPLICATION_TYPES = %w[
+        application/json
+        application/xml
+        application/javascript
+        application/xhtml+xml
+        application/rss+xml
+        application/atom+xml
+      ].freeze
+
+      # On-disk cache used by {.fetch} to memoize downloads. Defined as a
+      # method so specs can swap it for an isolated cache or
+      # {UrlCache::NULL} without touching the shared instance. Lives in
+      # its own subdir under {UrlCache::ROOT_DIR} so a +fetch+ on a URL
+      # and a +web_scrape+ on the same URL cannot collide on the same
+      # cache file (one returns the raw body, the other returns extracted
+      # Markdown).
+      #
+      # @return [UrlCache, #fetch]
+      CACHE = UrlCache.new(ttl: UrlCache::DEFAULT_TTL, dir: "#{UrlCache::ROOT_DIR}/fetch")
+      # Accessor for {CACHE}; specs override this to swap in
+      # {UrlCache::NULL} or an isolated cache.
+      #
+      # @return [UrlCache, #fetch]
+      def self.cache
+        CACHE
+      end
+
+      # Download +url+ via {Tool::Scraper::Simple.fetch} and return the
+      # response body verbatim, provided the content-type is one we deem
+      # textual (any +text/*+, plus the formats listed in
+      # {TEXTUAL_APPLICATION_TYPES}). Anything else — PDFs, images, other
+      # binaries — produces an +"Error: ..."+ string in the calculator-
+      # style convention so the agent loop feeds the failure back to the
+      # model as the next observation.
+      #
+      # The body is cached on disk via {.cache}, keyed by URL, so repeat
+      # fetches within the cache TTL skip the network. +max_chars+ is not
+      # part of the cache key — different values for the same URL share
+      # one entry, and truncation runs after the cache lookup. The cache
+      # is only populated on success: {Scraper::FetchError} (HTTP non-2xx,
+      # network failure, redirect-loop exhaustion, refused content-type)
+      # is caught outside the +cache.fetch+ block, so failure strings are
+      # never persisted and a retry on the next call hits the network
+      # again. Other exceptions (parser bugs in our own code) bubble up
+      # unchanged.
+      #
+      # @param url [String] absolute HTTP(S) URL to download
+      # @param max_chars [Integer] character cap on the returned body.
+      #   Clamped to +[1, {MAX_MAX_CHARS}]+; defaults to
+      #   {DEFAULT_MAX_CHARS}. When the body exceeds the cap, output is
+      #   cut and a marker noting the original length is appended.
+      # @return [String] response body, possibly truncated, or
+      #   +"Error: ..."+ on a recoverable failure
+      def self.fetch(url, max_chars: DEFAULT_MAX_CHARS)
+        max_chars = max_chars.clamp(1, MAX_MAX_CHARS)
+        body = cache.fetch(url) { download(url) }
+        truncate(body, max_chars)
+      rescue Scraper::FetchError => e
+        "Error: #{e.message}"
+      end
+
+      # GET +url+ and verify the response's content-type is textual.
+      # Caller is responsible for caching and truncation; this method
+      # always hits the network.
+      #
+      # @param url [String]
+      # @return [String] response body
+      # @raise [Scraper::FetchError] on HTTP non-2xx, network failure,
+      #   redirect-loop exhaustion, missing +Location+ on a 3xx, or a
+      #   non-textual content-type
+      def self.download(url)
+        fetched = Scraper::Simple.fetch(url)
+        return fetched.body if textual?(fetched.content_type)
+
+        raise Scraper::FetchError,
+              "refused to fetch #{url}: content-type #{fetched.content_type.inspect} " \
+              'is not textual (use web_scrape for PDFs or rendered pages)'
+      end
+
+      # @param content_type [String] normalized content-type (no +charset+
+      #   parameter, lowercased) as produced by {Scraper::Simple.fetch}
+      # @return [Boolean] true when the content-type is +text/*+ or one
+      #   of {TEXTUAL_APPLICATION_TYPES}
+      def self.textual?(content_type)
+        content_type.start_with?('text/') ||
+          TEXTUAL_APPLICATION_TYPES.include?(content_type)
+      end
+
+      # Cut +body+ to at most +max_chars+ characters, appending a marker
+      # describing the original length when truncation actually happens.
+      # Returns +body+ unchanged if it already fits. Same shape as
+      # {Tool::WebScrape.truncate} so the LLM sees a consistent
+      # truncation marker across both tools.
+      #
+      # @param body [String] full response body
+      # @param max_chars [Integer] character cap; assumed already clamped
+      # @return [String]
+      def self.truncate(body, max_chars)
+        return body if body.length <= max_chars
+
+        "#{body[0, max_chars]}\n\n" \
+          "... [truncated at #{max_chars} of #{body.length} chars; " \
+          'call again with a larger `max_chars` to see more]'
+      end
+    end
+
+    # Verbatim URL download tool. Thin wrapper over {Tool::Fetch.fetch}
+    # that exposes it to the agent loop in OpenAI tool-call shape. Use for
+    # raw textual payloads (JSON APIs, CSV files, +robots.txt+, source
+    # files); use {Tool::WEB_SCRAPE} for rendered web pages or PDFs where
+    # readability extraction makes the result usable.
+    #
+    # @return [Tool]
+    FETCH = new(
+      name: 'fetch',
+      description: <<~DESC,
+        Downloads the given URL and returns its body verbatim.
+
+        Usage:
+        - Use for raw textual payloads: JSON APIs, CSV files, robots.txt, sitemaps, source files — anywhere a rendering pass would corrupt the data.
+        - For rendered HTML pages or PDFs, use web_scrape — it extracts readable content; fetch returns the raw HTML/PDF bytes unchanged.
+        - Accepts text/* and common textual application/* types (JSON, XML, JS, XHTML, RSS, Atom). Refuses PDFs, images, and other binaries.
+      DESC
+      parameters: Parameters.build { |p|
+        p.required_string :url,
+                          'Absolute URL to download, including the scheme, ' \
+                          'e.g. "https://example.com/data.json".'
+        p.optional_integer :max_chars,
+                           'Maximum number of characters of the body to ' \
+                           'return. Defaults to 5000; hard-capped at ' \
+                           '100000. When the body is longer than this, ' \
+                           'output is cut and a marker reports the full ' \
+                           'length.'
+      },
+      execute: ->(url:, max_chars: Fetch::DEFAULT_MAX_CHARS) {
+        Fetch.fetch(url, max_chars: max_chars)
+      }
+    )
+  end
+end