pikuri-core 0.0.5 → 0.0.7

data/lib/pikuri/subprocess.rb

@@ -7,8 +7,10 @@ module Pikuri
   # Chokepoint for *all* subprocess spawning in pikuri. Forces a new
   # process group for each invocation, tracks pgids so descendants of
   # the direct child (commands backgrounded with +&+) can be cleaned
-  # up at process exit, and captures combined stdout+stderr through a
-  # single pipe.
+  # up at process exit. Two front doors: {.spawn} (combined
+  # stdout+stderr through a single pipe — the shell-command shape) and
+  # {.run} (stdin fed from a String or streamed from an IO, stdout
+  # redirected to a file, stderr captured — the filter shape).
   #
   # == Seam discipline
   #
@@ -91,6 +93,75 @@ module Pikuri
       new(io: io, wait_thr: wait_thr)
     end
 
+    # Run +argv+ as a one-shot *filter*: feed it +stdin_data+, redirect
+    # its stdout straight to +stdout+ (an open +File+), capture stderr
+    # through a pipe, and block until it exits. Built for the
+    # stdin→markdown document converters (pikuri-extractors), where
+    # {.spawn}'s shape is wrong twice over: it closes the child's stdin
+    # immediately, and it merges stderr onto stdout — fatal when stdout
+    # *is* the payload and a converter's warnings would corrupt it.
+    #
+    # Redirecting stdout to a file (not a pipe) is also what makes the
+    # I/O deadlock-free with one writer thread: the child never blocks
+    # writing output, so it keeps draining stdin, while the parent
+    # drains the (low-volume) stderr pipe. The returned
+    # {Result#output} is the captured *stderr* — the diagnostics — not
+    # the payload; the payload is in +stdout+, whose file offset is
+    # shared with the child, so rewind before reading it back.
+    #
+    # Same discipline as {.spawn}: new process group, registered for
+    # the exit sweep, no built-in timeout (wrap +argv+ with coreutils'
+    # +timeout+ — see the class docs).
+    #
+    # @param argv [Array<String>] command and arguments, passed to
+    #   +exec+ directly — no implicit shell.
+    # @param stdin_data [String, IO, StringIO] the child's stdin: a
+    #   String is written as-is, an IO is streamed through
+    #   +IO.copy_stream+ (so a large source file never materialises in
+    #   the Ruby heap) and read from its current position. Either way
+    #   stdin is closed (EOF) afterwards. May be empty.
+    # @param stdout [File] open writable file the child's stdout is
+    #   redirected to.
+    # @param chdir [String, Pathname] working directory.
+    # @param env [Hash{String=>String}] extra environment variables,
+    #   as for {.spawn}.
+    # @return [Result] +output+ is the captured stderr; +status+ the
+    #   child's exit status.
+    # @raise [SystemCallError] whatever an IO +stdin_data+ raises
+    #   mid-stream (disk error, closed handle) — re-raised here after
+    #   the child has been reaped.
+    def self.run(*argv, stdin_data:, stdout:, chdir:, env: {})
+      in_r, in_w = IO.pipe
+      err_r, err_w = IO.pipe
+      pid = Process.spawn(env, *argv, chdir: chdir.to_s, pgroup: true,
+                          in: in_r, out: stdout, err: err_w)
+      in_r.close
+      err_w.close
+      register(pid)
+      writer = Thread.new do
+        in_w.binmode
+        if stdin_data.respond_to?(:read)
+          IO.copy_stream(stdin_data, in_w)
+        else
+          in_w.write(stdin_data)
+        end
+      rescue Errno::EPIPE
+        nil # child exited without draining stdin; its status tells the story
+      ensure
+        in_w.close
+      end
+      stderr = err_r.read
+      err_r.close
+      # Reap before joining the writer: if an IO source raised
+      # mid-stream, #join re-raises it, and the child (already exited —
+      # err_r hit EOF) must not be left a zombie.
+      _, status = Process.waitpid2(pid)
+      writer.join
+      Result.new(output: stderr, status: status)
+    ensure
+      prune(pid) if pid
+    end
+
     # @return [Integer] direct child's pid
     attr_reader :pid
 

data/lib/pikuri/tool/calculator.rb

@@ -1,63 +1,235 @@
 # frozen_string_literal: true
 
-require 'dentaku'
-
 module Pikuri
   class Tool
-    # Evaluates a basic arithmetic expression using Dentaku, with light
-    # preprocessing so the LLM can emit Python-flavored syntax (notably
-    # +**+ for exponentiation) instead of learning Dentaku's dialect.
+    # Evaluates a basic arithmetic expression with Python operator
+    # syntax and semantics, via the hand-rolled recursive-descent
+    # {Parser} below.
+    #
+    # Why hand-rolled rather than a gem: the previous backend, dentaku,
+    # pulled in concurrent-ruby (~16k lines of Ruby — the single
+    # heaviest audit item in pikuri's whole dependency closure) plus
+    # bigdecimal and tsort, all to evaluate four-function arithmetic.
+    # The ~100 lines here implement Python's expression grammar
+    # directly, which also retires the old preprocessing step that
+    # rewrote Python's +**+ into dentaku's +^+ dialect — the model's
+    # native syntax is now simply the grammar.
     #
-    # Scope is intentionally narrow: operators (+, -, *, /, **, %),
-    # parentheses, and decimal numbers. No variables, functions, or
-    # booleans — those would mean teaching the model a dialect, which we
-    # specifically want to avoid for this tool.
+    # Scope is intentionally narrow: operators (+, -, *, /, //, %, **),
+    # unary minus, parentheses, and integer / decimal / e-notation
+    # literals. No variables, functions, or booleans — those would mean
+    # teaching the model a dialect, which we specifically want to avoid
+    # for this tool.
     module Calculator
-      # Translate the operator differences between Python and Dentaku. In
-      # practice that is only +**+ → +^+; everything else in the supported
-      # subset is byte-identical.
-      #
-      # @param expression [String] raw expression as the model wrote it
-      # @return [String] expression with Python-style operators rewritten
-      def self.normalize(expression)
-        expression.gsub('**', '^')
-      end
+      # Raised internally for anything {.calculate} should hand back to
+      # the model as an +"Error: ..."+ observation rather than crash
+      # the agent loop: parse failures, division by zero, complex or
+      # non-finite results. The message always names the offending
+      # token or operands.
+      class Error < StandardError; end
 
-      # Evaluate +expression+ and return the result formatted as a String.
-      # Parse, unbound-variable, and division-by-zero failures are caught
-      # and returned as +"Error: ..."+ strings so the model can read the
-      # failure as the next observation and self-correct rather than
-      # crashing the agent loop.
+      # Evaluate +expression+ and return the result formatted as a
+      # String. Parse and arithmetic failures (division by zero,
+      # overflow to infinity, complex results) are caught and returned
+      # as +"Error: ..."+ strings so the model can read the failure as
+      # the next observation and self-correct rather than crashing the
+      # agent loop.
       #
-      # @param expression [String]
+      # @param expression [String] Python-syntax arithmetic expression
       # @return [String] numeric result, or +"Error: ..."+ on failure
       def self.calculate(expression)
-        result = Dentaku::Calculator.new.evaluate!(normalize(expression))
+        result = Parser.new(expression).parse
+        if result.is_a?(Float) && !result.finite?
+          raise Error, "result of #{expression.inspect} is not a finite number"
+        end
+
         format_result(result)
-      rescue Dentaku::ZeroDivisionError, ZeroDivisionError
-        'Error: division by zero'
-      rescue Dentaku::Error => e
+      rescue Error => e
         "Error: #{e.message}"
       end
 
-      # Dentaku returns BigDecimal for any expression that touches division
-      # or a decimal literal, with full BigDecimal precision (47-digit tails
-      # for the leopard expression). Round to 3 places and strip the
-      # default scientific-notation formatting so the model sees a short
-      # readable number; integer/other results pass through unchanged.
+      # Integers (never produced by division — +/+ is Python-3-style
+      # true division) pass through exact. Floats are rounded to 3
+      # places so the model sees a short readable number, and
+      # whole-valued floats drop the trailing +.0+ (+4 / 2+ renders as
+      # +"2"+, not +"2.0"+).
       def self.format_result(result)
-        case result
-        when BigDecimal then result.round(3).to_s('F')
-        else result.to_s
-        end
+        return result.to_s if result.is_a?(Integer)
+
+        rounded = result.round(3)
+        rounded == rounded.truncate ? rounded.truncate.to_s : rounded.to_s
       end
       private_class_method :format_result
+
+      # Recursive-descent parser-evaluator for Python's arithmetic
+      # expression grammar:
+      #
+      #   additive       := multiplicative (('+' | '-') multiplicative)*
+      #   multiplicative := unary (('*' | '/' | '//' | '%') unary)*
+      #   unary          := ('+' | '-') unary | power
+      #   power          := atom ('**' unary)?
+      #   atom           := NUMBER | '(' additive ')'
+      #
+      # The +power+ → +unary+ recursion on the right operand is what
+      # makes +**+ right-associative (+2**3**2+ is 512) and lets a sign
+      # follow it (+2**-3+); +unary+ sitting *above* +power+ on the
+      # left is what makes +-2**2+ evaluate to -4 — both exactly as
+      # Python parses them.
+      #
+      # Semantics follow Python 3 where Ruby differs: +/+ is always
+      # true (float) division, +//+ floors, +2**-1+ is the Float 0.5
+      # (Ruby would return a Rational), and a negative base under a
+      # fractional exponent is rejected (Ruby would return a Complex).
+      class Parser
+        # One number or operator. +**+ / +//+ listed before their
+        # single-character prefixes so the two-character operators win;
+        # number literals cover +42+, +4.2+, +5.+, +.5+, and e-notation
+        # on any of them. +\G+ anchors each match at the scan position
+        # so nothing between tokens goes unnoticed.
+        TOKEN_RE = %r{\G\s*(\*\*|//|\d+(?:\.\d*)?(?:[eE][+-]?\d+)?|\.\d+(?:[eE][+-]?\d+)?|[-+*/%()])}
+
+        # @param expression [String] raw expression as the model wrote it
+        # @raise [Error] when +expression+ contains a character no token matches
+        def initialize(expression)
+          @tokens = tokenize(expression)
+          @pos = 0
+        end
+
+        # Parse and evaluate the whole token stream.
+        #
+        # @return [Integer, Float] the value of the expression
+        # @raise [Error] on syntax errors, division by zero, or a complex result
+        def parse
+          value = additive
+          raise Error, "unexpected #{peek.inspect} after expression" if peek
+
+          value
+        end
+
+        private
+
+        # @param expression [String]
+        # @return [Array<String>] token strings in source order
+        def tokenize(expression)
+          tokens = []
+          pos = 0
+          while (match = TOKEN_RE.match(expression, pos))
+            tokens << match[1]
+            pos = match.end(0)
+          end
+          rest = expression[pos..].to_s.strip
+          raise Error, "unexpected character #{rest[0].inspect} in #{expression.inspect}" unless rest.empty?
+
+          tokens
+        end
+
+        def additive
+          value = multiplicative
+          while (op = accept('+', '-'))
+            rhs = multiplicative
+            value = op == '+' ? value + rhs : value - rhs
+          end
+          value
+        end
+
+        def multiplicative
+          value = unary
+          while (op = accept('*', '/', '//', '%'))
+            value = apply_multiplicative(op, value, unary)
+          end
+          value
+        end
+
+        def unary
+          op = accept('+', '-')
+          return power unless op
+
+          value = unary
+          op == '-' ? -value : value
+        end
+
+        def power
+          base = atom
+          return base unless accept('**')
+
+          apply_power(base, unary)
+        end
+
+        def atom
+          return parenthesized if accept('(')
+
+          token = peek
+          unless token&.match?(/\A[.\d]/)
+            raise Error, token ? "unexpected #{token.inspect}" : 'unexpected end of expression'
+          end
+
+          @pos += 1
+          token.match?(/[.eE]/) ? token.to_f : token.to_i
+        end
+
+        def parenthesized
+          value = additive
+          raise Error, 'missing closing parenthesis' unless accept(')')
+
+          value
+        end
+
+        # @return [String, nil] the next token without consuming it
+        def peek
+          @tokens[@pos]
+        end
+
+        # Consume and return the next token if it is one of +expected+.
+        #
+        # @return [String, nil] the consumed token, or nil on no match
+        def accept(*expected)
+          token = peek
+          return nil unless expected.include?(token)
+
+          @pos += 1
+          token
+        end
+
+        # +/+ is Python-3 true division (always Float); +//+ floors
+        # (kept exact in Ruby's arbitrary-precision Integer division
+        # when both operands are Integers — Ruby's +Integer#/+ already
+        # floors like Python's +//+); +%+ delegates to Ruby's +%+,
+        # whose sign-of-divisor semantics match Python's exactly.
+        def apply_multiplicative(op, lhs, rhs)
+          return lhs * rhs if op == '*'
+          raise Error, 'division by zero' if rhs.zero?
+
+          case op
+          when '/' then lhs.fdiv(rhs)
+          when '//' then lhs.is_a?(Integer) && rhs.is_a?(Integer) ? lhs / rhs : lhs.fdiv(rhs).floor
+          when '%' then lhs % rhs
+          end
+        end
+
+        # Two Python-compatibility shims over Ruby's +**+: an Integer
+        # raised to a negative Integer yields a Float (Ruby would
+        # return a Rational), and a Complex result — negative base
+        # under a fractional exponent — is rejected loudly.
+        def apply_power(base, exponent)
+          if base.is_a?(Integer) && exponent.is_a?(Integer) && exponent.negative?
+            raise Error, 'division by zero' if base.zero?
+
+            return base.to_f**exponent
+          end
+          result = base**exponent
+          if result.is_a?(Complex)
+            raise Error, "(#{base})**(#{exponent}) is a complex number; only real arithmetic is supported"
+          end
+
+          result
+        end
+      end
     end
 
     # Arithmetic-evaluation tool backed by {Tool::Calculator.calculate}.
-    # Accepts Python-flavored operator syntax (+, -, *, /, ** for
-    # exponentiation, %, parentheses, decimals) so the model can emit the
-    # syntax it already knows.
+    # Accepts Python expression syntax (+, -, *, /, //, %, ** for
+    # exponentiation, unary minus, parentheses, decimals) so the model
+    # can emit the syntax it already knows.
     #
     # @return [Tool]
     CALCULATOR = new(
@@ -67,7 +239,7 @@ module Pikuri
 
         Usage:
         - Use this for any arithmetic beyond simple mental math — do not eyeball multi-digit work.
-        - Operators supported: +, -, *, /, ** (exponentiation), %, parentheses, decimal numbers.
+        - Python expression syntax: +, -, *, / (true division), // (floor division), % (modulo), ** (exponentiation), unary minus, parentheses, decimal numbers.
         - Decimal results are rounded to 3 places; integer results are exact.
         - Failures (parse error, division by zero) come back as "Error: ..." — read the message and re-call with a corrected expression.
       DESC

data/lib/pikuri/tool/fetch.rb

@@ -3,13 +3,14 @@
 module Pikuri
   class Tool
     # Truncation policy and Tool spec for the +fetch+ tool. The HTTP work
-    # lives in {Tool::Scraper::Simple.fetch}; this module is a thin
+    # lives in {Tool::Scraper.fetch}; this module is a thin
     # wrapper that accepts only textual content-types, applies a character
     # cap so the LLM doesn't drown in long-form bodies, and exposes the
     # result to the agent loop in OpenAI tool-call shape.
     #
-    # Sister of {Tool::WebScrape}, but without HTML→Markdown or PDF→text
-    # extraction: bodies are returned verbatim. Useful for raw textual
+    # Sister of {Tool::WebScrape}, but with no extraction pass
+    # (HTML→Markdown, or whatever plug-in extractors are registered):
+    # bodies are returned verbatim. Useful for raw textual
     # data — JSON APIs, CSV files, +robots.txt+, sitemaps, source files —
     # where any rendering pass would corrupt the payload.
     module Fetch
@@ -56,7 +57,7 @@ module Pikuri
         CACHE
       end
 
-      # Download +url+ via {Tool::Scraper::Simple.fetch} and return the
+      # Download +url+ via {Tool::Scraper.fetch} and return the
       # response body verbatim, provided the content-type is one we deem
       # textual (any +text/*+, plus the formats listed in
       # {TEXTUAL_APPLICATION_TYPES}). Anything else — PDFs, images, other
@@ -100,16 +101,16 @@ module Pikuri
       #   redirect-loop exhaustion, missing +Location+ on a 3xx, or a
       #   non-textual content-type
       def self.download(url)
-        fetched = Scraper::Simple.fetch(url)
+        fetched = Scraper.fetch(url)
         return fetched.body if textual?(fetched.content_type)
 
         raise Scraper::FetchError,
               "refused to fetch #{url}: content-type #{fetched.content_type.inspect} " \
-              'is not textual (use web_scrape for PDFs or rendered pages)'
+              'is not textual (use web_scrape for rendered pages)'
       end
 
       # @param content_type [String] normalized content-type (no +charset+
-      #   parameter, lowercased) as produced by {Scraper::Simple.fetch}
+      #   parameter, lowercased) as produced by {Scraper.fetch}
       # @return [Boolean] true when the content-type is +text/*+ or one
       #   of {TEXTUAL_APPLICATION_TYPES}
       def self.textual?(content_type)
@@ -138,7 +139,7 @@ module Pikuri
     # Verbatim URL download tool. Thin wrapper over {Tool::Fetch.fetch}
     # that exposes it to the agent loop in OpenAI tool-call shape. Use for
     # raw textual payloads (JSON APIs, CSV files, +robots.txt+, source
-    # files); use {Tool::WEB_SCRAPE} for rendered web pages or PDFs where
+    # files); use {Tool::WEB_SCRAPE} for rendered web pages where
     # readability extraction makes the result usable.
     #
     # @return [Tool]
@@ -149,7 +150,7 @@ module Pikuri
 
         Usage:
         - Use for raw textual payloads: JSON APIs, CSV files, robots.txt, sitemaps, source files — anywhere a rendering pass would corrupt the data.
-        - For rendered HTML pages or PDFs, use web_scrape — it extracts readable content; fetch returns the raw HTML/PDF bytes unchanged.
+        - For rendered HTML pages, use web_scrape — it extracts readable content; fetch returns the raw HTML bytes unchanged.
         - Accepts text/* and common textual application/* types (JSON, XML, JS, XHTML, RSS, Atom). Refuses PDFs, images, and other binaries.
       DESC
       parameters: Parameters.build { |p|

data/lib/pikuri/tool/parameters.rb

@@ -68,6 +68,36 @@ module Pikuri
         add(name, 'string', description, required: false)
       end
 
+      # Add a required +array+-of-+string+ property — JSON-Schema
+      # +{type: 'array', items: {type: 'string'}}+. The LLM sends a
+      # native JSON array in the tool-call arguments (the shape its
+      # training data overwhelmingly uses for list-valued parameters),
+      # so there is no in-band encoding for it to get wrong.
+      # The value must arrive as an Array — no
+      # JSON-encoded-array-in-a-string fallback. Element coercion
+      # mirrors the scalar fields' one documented leniency, in
+      # reverse: Integers and finite Floats are converted to their
+      # +to_s+ form (a model emitting +["Fix issue 12", 42]+ meant a
+      # string list — the conversion is unambiguous), while booleans,
+      # +nil+, and nested structures are rejected — those signal a
+      # genuinely wrong call shape, not a representational quirk.
+      # An empty array is type-valid; rejecting it (if the tool needs
+      # at least one element) is the tool's job, with a tool-specific
+      # error message.
+      #
+      # @param name [Symbol] property name
+      # @param description [String] human-readable description shown to the LLM
+      # @return [self]
+      def required_string_array(name, description)
+        @properties[name] = {
+          type: 'array',
+          items: { type: 'string' },
+          description: description
+        }
+        @required << name.to_s
+        self
+      end
+
       # Add a required +integer+ property. Accepts Integers, Floats with a
       # zero fractional part (e.g. +1.0+), and base-10 numeric Strings (after
       # trimming) that resolve to whole numbers; rejects everything else.
@@ -260,9 +290,35 @@ module Pikuri
           coerce_number(value)
         when 'boolean'
           coerce_boolean(value)
+        when 'array'
+          coerce_string_array(value)
+        end
+      end
+
+      def coerce_string_array(value)
+        raise CoercionError, "must be an array of strings (got #{value.class}: #{value.inspect})" unless value.is_a?(Array)
+
+        value.each_with_index.map do |element, i|
+          case element
+          when String
+            element
+          when Integer
+            element.to_s
+          when Float
+            raise CoercionError, array_element_message(i, element) unless element.finite?
+
+            element.to_s
+          else
+            raise CoercionError, array_element_message(i, element)
+          end
         end
       end
 
+      def array_element_message(index, element)
+        "must be an array of strings (element #{index} is #{element.class}: #{element.inspect}; " \
+          'numbers are auto-converted, other types are not)'
+      end
+
       def coerce_boolean(value)
         return value if value == true || value == false
 
@@ -341,7 +397,14 @@ module Pikuri
 
       def missing_required_message(name, schema)
         enum_part = schema[:enum] ? ", one of: #{schema[:enum].map { |v| "`#{v}`" }.join(', ')}" : ''
-        "Missing required parameter `#{name}` (#{schema[:type]}#{enum_part}): #{schema[:description]}"
+        "Missing required parameter `#{name}` (#{type_label(schema)}#{enum_part}): #{schema[:description]}"
+      end
+
+      # Human/LLM-facing label for a property's type in error messages:
+      # +"array of strings"+ for array properties, the bare JSON-Schema
+      # type name otherwise.
+      def type_label(schema)
+        schema[:items] ? "array of #{schema[:items][:type]}s" : schema[:type]
       end
 
       def unknown_key_error(unknown)
@@ -366,7 +429,7 @@ module Pikuri
           *@properties.map { |name, prop|
             req = @required.include?(name.to_s) ? 'required' : 'optional'
             enum_part = prop[:enum] ? ", one of: #{prop[:enum].map { |v| "`#{v}`" }.join(', ')}" : ''
-            "  - `#{name}` (#{prop[:type]}, #{req}#{enum_part}): #{prop[:description]}"
+            "  - `#{name}` (#{type_label(prop)}, #{req}#{enum_part}): #{prop[:description]}"
           }
         ].join("\n")
       end