pikuri-core 0.0.4 → 0.0.6

data/lib/pikuri/subprocess.rb

@@ -7,16 +7,20 @@ module Pikuri
   # Chokepoint for *all* subprocess spawning in pikuri. Forces a new
   # process group for each invocation, tracks pgids so descendants of
   # the direct child (commands backgrounded with +&+) can be cleaned
-  # up at process exit, and captures combined stdout+stderr through a
-  # single pipe.
+  # up at process exit. Two front doors: {.spawn} (combined
+  # stdout+stderr through a single pipe — the shell-command shape) and
+  # {.run} (stdin fed from a String or streamed from an IO, stdout
+  # redirected to a file, stderr captured — the filter shape).
   #
   # == Seam discipline
   #
   # All subprocess spawning in +lib/+ goes through {.spawn}. Direct
   # +Process.spawn+ / +Open3.*+ / +system+ / backticks anywhere in
   # +lib/+ are bugs. The convention is grep-enforceable:
-  # +grep -rn 'Process\.spawn\|Open3\|system\|backtick' lib/+ should
-  # only hit this file.
+  # +grep -rnE 'Process\.spawn|Open3\.|\bsystem\(' lib/+ should
+  # only hit this file (plus the comment in
+  # +pikuri-mcp/lib/pikuri/mcp/servers.rb+ explaining the MCP
+  # exception).
   #
   # == Timeouts are the caller's job
   #
@@ -47,10 +51,11 @@ module Pikuri
   #
   # == State is process-global
   #
-  # One +@active+ Set and one +at_exit+ for the whole process. A
-  # +Mutex+ guards register/prune/cleanup; v1 is single-threaded, so
-  # this is more for the +at_exit+/register race than for current
-  # callers.
+  # One +@active+ Set for the whole process, swept once at exit via
+  # {Pikuri::Finalizers} (see the registration at the bottom of this
+  # file). A +Mutex+ guards register/prune/cleanup; v1 is
+  # single-threaded, so this is more for the exit-sweep/register race
+  # than for current callers.
   #
   # == Why +Pikuri::Subprocess+, not top-level
   #
@@ -88,6 +93,75 @@ module Pikuri
       new(io: io, wait_thr: wait_thr)
     end
 
+    # Run +argv+ as a one-shot *filter*: feed it +stdin_data+, redirect
+    # its stdout straight to +stdout+ (an open +File+), capture stderr
+    # through a pipe, and block until it exits. Built for the
+    # stdin→markdown document converters (pikuri-extractors), where
+    # {.spawn}'s shape is wrong twice over: it closes the child's stdin
+    # immediately, and it merges stderr onto stdout — fatal when stdout
+    # *is* the payload and a converter's warnings would corrupt it.
+    #
+    # Redirecting stdout to a file (not a pipe) is also what makes the
+    # I/O deadlock-free with one writer thread: the child never blocks
+    # writing output, so it keeps draining stdin, while the parent
+    # drains the (low-volume) stderr pipe. The returned
+    # {Result#output} is the captured *stderr* — the diagnostics — not
+    # the payload; the payload is in +stdout+, whose file offset is
+    # shared with the child, so rewind before reading it back.
+    #
+    # Same discipline as {.spawn}: new process group, registered for
+    # the exit sweep, no built-in timeout (wrap +argv+ with coreutils'
+    # +timeout+ — see the class docs).
+    #
+    # @param argv [Array<String>] command and arguments, passed to
+    #   +exec+ directly — no implicit shell.
+    # @param stdin_data [String, IO, StringIO] the child's stdin: a
+    #   String is written as-is, an IO is streamed through
+    #   +IO.copy_stream+ (so a large source file never materialises in
+    #   the Ruby heap) and read from its current position. Either way
+    #   stdin is closed (EOF) afterwards. May be empty.
+    # @param stdout [File] open writable file the child's stdout is
+    #   redirected to.
+    # @param chdir [String, Pathname] working directory.
+    # @param env [Hash{String=>String}] extra environment variables,
+    #   as for {.spawn}.
+    # @return [Result] +output+ is the captured stderr; +status+ the
+    #   child's exit status.
+    # @raise [SystemCallError] whatever an IO +stdin_data+ raises
+    #   mid-stream (disk error, closed handle) — re-raised here after
+    #   the child has been reaped.
+    def self.run(*argv, stdin_data:, stdout:, chdir:, env: {})
+      in_r, in_w = IO.pipe
+      err_r, err_w = IO.pipe
+      pid = Process.spawn(env, *argv, chdir: chdir.to_s, pgroup: true,
+                          in: in_r, out: stdout, err: err_w)
+      in_r.close
+      err_w.close
+      register(pid)
+      writer = Thread.new do
+        in_w.binmode
+        if stdin_data.respond_to?(:read)
+          IO.copy_stream(stdin_data, in_w)
+        else
+          in_w.write(stdin_data)
+        end
+      rescue Errno::EPIPE
+        nil # child exited without draining stdin; its status tells the story
+      ensure
+        in_w.close
+      end
+      stderr = err_r.read
+      err_r.close
+      # Reap before joining the writer: if an IO source raised
+      # mid-stream, #join re-raises it, and the child (already exited —
+      # err_r hit EOF) must not be left a zombie.
+      _, status = Process.waitpid2(pid)
+      writer.join
+      Result.new(output: stderr, status: status)
+    ensure
+      prune(pid) if pid
+    end
+
     # @return [Integer] direct child's pid
     attr_reader :pid
 
@@ -122,6 +196,23 @@ module Pikuri
       self.class.send(:prune, @pgid)
     end
 
+    # SIGTERM the whole process group without blocking for output —
+    # the stop button for a *daemon* child (one the caller never
+    # {#wait}s on, e.g. {Pikuri::Memory::Mem0Server}'s socat relay).
+    # Best-effort and idempotent: an already-dead group is a no-op.
+    # The group stays in the exit-sweep set until it actually dies,
+    # so a child that ignores SIGTERM is still re-signalled by
+    # {.cleanup!} at process exit.
+    #
+    # @return [void]
+    def terminate
+      Process.kill('-TERM', @pgid)
+    rescue Errno::ESRCH
+      # already gone
+    ensure
+      self.class.send(:prune, @pgid)
+    end
+
     class << self
       # Currently-tracked process groups, with dead ones pruned as a
       # side effect. Useful for a future +/bg+ REPL command or a
@@ -135,9 +226,9 @@ module Pikuri
         end
       end
 
-      # SIGTERM every tracked process group. Used by +at_exit+
-      # (production) and +after+ blocks (specs). Best-effort —
-      # ignores errors from already-dead groups.
+      # SIGTERM every tracked process group. Run at process exit via
+      # {Pikuri::Finalizers} (production) and from +after+ blocks
+      # (specs). Best-effort — ignores errors from already-dead groups.
       #
       # @return [void]
       def cleanup!
@@ -170,4 +261,10 @@ module Pikuri
   end
 end
 
-at_exit { Pikuri::Subprocess.cleanup! }
+# Registered at load (boot) — before any agent or server registers at
+# construction time — so in {Pikuri::Finalizers}' LIFO sweep this runs
+# LAST: graceful +#close+ on agents and servers first, then SIGTERM
+# whatever child groups are still alive. The reaper, not a peer. (Was a
+# standalone +at_exit+; routed through Finalizers so the order relative
+# to every other teardown is controlled, not left to file load order.)
+Pikuri::Finalizers.register { Pikuri::Subprocess.cleanup! }

data/lib/pikuri/tool/calculator.rb

@@ -1,63 +1,235 @@
 # frozen_string_literal: true
 
-require 'dentaku'
-
 module Pikuri
   class Tool
-    # Evaluates a basic arithmetic expression using Dentaku, with light
-    # preprocessing so the LLM can emit Python-flavored syntax (notably
-    # +**+ for exponentiation) instead of learning Dentaku's dialect.
+    # Evaluates a basic arithmetic expression with Python operator
+    # syntax and semantics, via the hand-rolled recursive-descent
+    # {Parser} below.
+    #
+    # Why hand-rolled rather than a gem: the previous backend, dentaku,
+    # pulled in concurrent-ruby (~16k lines of Ruby — the single
+    # heaviest audit item in pikuri's whole dependency closure) plus
+    # bigdecimal and tsort, all to evaluate four-function arithmetic.
+    # The ~100 lines here implement Python's expression grammar
+    # directly, which also retires the old preprocessing step that
+    # rewrote Python's +**+ into dentaku's +^+ dialect — the model's
+    # native syntax is now simply the grammar.
     #
-    # Scope is intentionally narrow: operators (+, -, *, /, **, %),
-    # parentheses, and decimal numbers. No variables, functions, or
-    # booleans — those would mean teaching the model a dialect, which we
-    # specifically want to avoid for this tool.
+    # Scope is intentionally narrow: operators (+, -, *, /, //, %, **),
+    # unary minus, parentheses, and integer / decimal / e-notation
+    # literals. No variables, functions, or booleans — those would mean
+    # teaching the model a dialect, which we specifically want to avoid
+    # for this tool.
     module Calculator
-      # Translate the operator differences between Python and Dentaku. In
-      # practice that is only +**+ → +^+; everything else in the supported
-      # subset is byte-identical.
-      #
-      # @param expression [String] raw expression as the model wrote it
-      # @return [String] expression with Python-style operators rewritten
-      def self.normalize(expression)
-        expression.gsub('**', '^')
-      end
+      # Raised internally for anything {.calculate} should hand back to
+      # the model as an +"Error: ..."+ observation rather than crash
+      # the agent loop: parse failures, division by zero, complex or
+      # non-finite results. The message always names the offending
+      # token or operands.
+      class Error < StandardError; end
 
-      # Evaluate +expression+ and return the result formatted as a String.
-      # Parse, unbound-variable, and division-by-zero failures are caught
-      # and returned as +"Error: ..."+ strings so the model can read the
-      # failure as the next observation and self-correct rather than
-      # crashing the agent loop.
+      # Evaluate +expression+ and return the result formatted as a
+      # String. Parse and arithmetic failures (division by zero,
+      # overflow to infinity, complex results) are caught and returned
+      # as +"Error: ..."+ strings so the model can read the failure as
+      # the next observation and self-correct rather than crashing the
+      # agent loop.
       #
-      # @param expression [String]
+      # @param expression [String] Python-syntax arithmetic expression
       # @return [String] numeric result, or +"Error: ..."+ on failure
       def self.calculate(expression)
-        result = Dentaku::Calculator.new.evaluate!(normalize(expression))
+        result = Parser.new(expression).parse
+        if result.is_a?(Float) && !result.finite?
+          raise Error, "result of #{expression.inspect} is not a finite number"
+        end
+
         format_result(result)
-      rescue Dentaku::ZeroDivisionError, ZeroDivisionError
-        'Error: division by zero'
-      rescue Dentaku::Error => e
+      rescue Error => e
         "Error: #{e.message}"
       end
 
-      # Dentaku returns BigDecimal for any expression that touches division
-      # or a decimal literal, with full BigDecimal precision (47-digit tails
-      # for the leopard expression). Round to 3 places and strip the
-      # default scientific-notation formatting so the model sees a short
-      # readable number; integer/other results pass through unchanged.
+      # Integers (never produced by division — +/+ is Python-3-style
+      # true division) pass through exact. Floats are rounded to 3
+      # places so the model sees a short readable number, and
+      # whole-valued floats drop the trailing +.0+ (+4 / 2+ renders as
+      # +"2"+, not +"2.0"+).
       def self.format_result(result)
-        case result
-        when BigDecimal then result.round(3).to_s('F')
-        else result.to_s
-        end
+        return result.to_s if result.is_a?(Integer)
+
+        rounded = result.round(3)
+        rounded == rounded.truncate ? rounded.truncate.to_s : rounded.to_s
       end
       private_class_method :format_result
+
+      # Recursive-descent parser-evaluator for Python's arithmetic
+      # expression grammar:
+      #
+      #   additive       := multiplicative (('+' | '-') multiplicative)*
+      #   multiplicative := unary (('*' | '/' | '//' | '%') unary)*
+      #   unary          := ('+' | '-') unary | power
+      #   power          := atom ('**' unary)?
+      #   atom           := NUMBER | '(' additive ')'
+      #
+      # The +power+ → +unary+ recursion on the right operand is what
+      # makes +**+ right-associative (+2**3**2+ is 512) and lets a sign
+      # follow it (+2**-3+); +unary+ sitting *above* +power+ on the
+      # left is what makes +-2**2+ evaluate to -4 — both exactly as
+      # Python parses them.
+      #
+      # Semantics follow Python 3 where Ruby differs: +/+ is always
+      # true (float) division, +//+ floors, +2**-1+ is the Float 0.5
+      # (Ruby would return a Rational), and a negative base under a
+      # fractional exponent is rejected (Ruby would return a Complex).
+      class Parser
+        # One number or operator. +**+ / +//+ listed before their
+        # single-character prefixes so the two-character operators win;
+        # number literals cover +42+, +4.2+, +5.+, +.5+, and e-notation
+        # on any of them. +\G+ anchors each match at the scan position
+        # so nothing between tokens goes unnoticed.
+        TOKEN_RE = %r{\G\s*(\*\*|//|\d+(?:\.\d*)?(?:[eE][+-]?\d+)?|\.\d+(?:[eE][+-]?\d+)?|[-+*/%()])}
+
+        # @param expression [String] raw expression as the model wrote it
+        # @raise [Error] when +expression+ contains a character no token matches
+        def initialize(expression)
+          @tokens = tokenize(expression)
+          @pos = 0
+        end
+
+        # Parse and evaluate the whole token stream.
+        #
+        # @return [Integer, Float] the value of the expression
+        # @raise [Error] on syntax errors, division by zero, or a complex result
+        def parse
+          value = additive
+          raise Error, "unexpected #{peek.inspect} after expression" if peek
+
+          value
+        end
+
+        private
+
+        # @param expression [String]
+        # @return [Array<String>] token strings in source order
+        def tokenize(expression)
+          tokens = []
+          pos = 0
+          while (match = TOKEN_RE.match(expression, pos))
+            tokens << match[1]
+            pos = match.end(0)
+          end
+          rest = expression[pos..].to_s.strip
+          raise Error, "unexpected character #{rest[0].inspect} in #{expression.inspect}" unless rest.empty?
+
+          tokens
+        end
+
+        def additive
+          value = multiplicative
+          while (op = accept('+', '-'))
+            rhs = multiplicative
+            value = op == '+' ? value + rhs : value - rhs
+          end
+          value
+        end
+
+        def multiplicative
+          value = unary
+          while (op = accept('*', '/', '//', '%'))
+            value = apply_multiplicative(op, value, unary)
+          end
+          value
+        end
+
+        def unary
+          op = accept('+', '-')
+          return power unless op
+
+          value = unary
+          op == '-' ? -value : value
+        end
+
+        def power
+          base = atom
+          return base unless accept('**')
+
+          apply_power(base, unary)
+        end
+
+        def atom
+          return parenthesized if accept('(')
+
+          token = peek
+          unless token&.match?(/\A[.\d]/)
+            raise Error, token ? "unexpected #{token.inspect}" : 'unexpected end of expression'
+          end
+
+          @pos += 1
+          token.match?(/[.eE]/) ? token.to_f : token.to_i
+        end
+
+        def parenthesized
+          value = additive
+          raise Error, 'missing closing parenthesis' unless accept(')')
+
+          value
+        end
+
+        # @return [String, nil] the next token without consuming it
+        def peek
+          @tokens[@pos]
+        end
+
+        # Consume and return the next token if it is one of +expected+.
+        #
+        # @return [String, nil] the consumed token, or nil on no match
+        def accept(*expected)
+          token = peek
+          return nil unless expected.include?(token)
+
+          @pos += 1
+          token
+        end
+
+        # +/+ is Python-3 true division (always Float); +//+ floors
+        # (kept exact in Ruby's arbitrary-precision Integer division
+        # when both operands are Integers — Ruby's +Integer#/+ already
+        # floors like Python's +//+); +%+ delegates to Ruby's +%+,
+        # whose sign-of-divisor semantics match Python's exactly.
+        def apply_multiplicative(op, lhs, rhs)
+          return lhs * rhs if op == '*'
+          raise Error, 'division by zero' if rhs.zero?
+
+          case op
+          when '/' then lhs.fdiv(rhs)
+          when '//' then lhs.is_a?(Integer) && rhs.is_a?(Integer) ? lhs / rhs : lhs.fdiv(rhs).floor
+          when '%' then lhs % rhs
+          end
+        end
+
+        # Two Python-compatibility shims over Ruby's +**+: an Integer
+        # raised to a negative Integer yields a Float (Ruby would
+        # return a Rational), and a Complex result — negative base
+        # under a fractional exponent — is rejected loudly.
+        def apply_power(base, exponent)
+          if base.is_a?(Integer) && exponent.is_a?(Integer) && exponent.negative?
+            raise Error, 'division by zero' if base.zero?
+
+            return base.to_f**exponent
+          end
+          result = base**exponent
+          if result.is_a?(Complex)
+            raise Error, "(#{base})**(#{exponent}) is a complex number; only real arithmetic is supported"
+          end
+
+          result
+        end
+      end
     end
 
     # Arithmetic-evaluation tool backed by {Tool::Calculator.calculate}.
-    # Accepts Python-flavored operator syntax (+, -, *, /, ** for
-    # exponentiation, %, parentheses, decimals) so the model can emit the
-    # syntax it already knows.
+    # Accepts Python expression syntax (+, -, *, /, //, %, ** for
+    # exponentiation, unary minus, parentheses, decimals) so the model
+    # can emit the syntax it already knows.
     #
     # @return [Tool]
     CALCULATOR = new(
@@ -67,7 +239,7 @@ module Pikuri
 
         Usage:
         - Use this for any arithmetic beyond simple mental math — do not eyeball multi-digit work.
-        - Operators supported: +, -, *, /, ** (exponentiation), %, parentheses, decimal numbers.
+        - Python expression syntax: +, -, *, / (true division), // (floor division), % (modulo), ** (exponentiation), unary minus, parentheses, decimal numbers.
         - Decimal results are rounded to 3 places; integer results are exact.
         - Failures (parse error, division by zero) come back as "Error: ..." — read the message and re-call with a corrected expression.
       DESC

data/lib/pikuri/tool/fetch.rb

@@ -3,13 +3,14 @@
 module Pikuri
   class Tool
     # Truncation policy and Tool spec for the +fetch+ tool. The HTTP work
-    # lives in {Tool::Scraper::Simple.fetch}; this module is a thin
+    # lives in {Tool::Scraper.fetch}; this module is a thin
     # wrapper that accepts only textual content-types, applies a character
     # cap so the LLM doesn't drown in long-form bodies, and exposes the
     # result to the agent loop in OpenAI tool-call shape.
     #
-    # Sister of {Tool::WebScrape}, but without HTML→Markdown or PDF→text
-    # extraction: bodies are returned verbatim. Useful for raw textual
+    # Sister of {Tool::WebScrape}, but with no extraction pass
+    # (HTML→Markdown, or whatever plug-in extractors are registered):
+    # bodies are returned verbatim. Useful for raw textual
     # data — JSON APIs, CSV files, +robots.txt+, sitemaps, source files —
     # where any rendering pass would corrupt the payload.
     module Fetch
@@ -56,7 +57,7 @@ module Pikuri
         CACHE
       end
 
-      # Download +url+ via {Tool::Scraper::Simple.fetch} and return the
+      # Download +url+ via {Tool::Scraper.fetch} and return the
       # response body verbatim, provided the content-type is one we deem
       # textual (any +text/*+, plus the formats listed in
       # {TEXTUAL_APPLICATION_TYPES}). Anything else — PDFs, images, other
@@ -100,16 +101,16 @@ module Pikuri
       #   redirect-loop exhaustion, missing +Location+ on a 3xx, or a
       #   non-textual content-type
       def self.download(url)
-        fetched = Scraper::Simple.fetch(url)
+        fetched = Scraper.fetch(url)
         return fetched.body if textual?(fetched.content_type)
 
         raise Scraper::FetchError,
               "refused to fetch #{url}: content-type #{fetched.content_type.inspect} " \
-              'is not textual (use web_scrape for PDFs or rendered pages)'
+              'is not textual (use web_scrape for rendered pages)'
       end
 
       # @param content_type [String] normalized content-type (no +charset+
-      #   parameter, lowercased) as produced by {Scraper::Simple.fetch}
+      #   parameter, lowercased) as produced by {Scraper.fetch}
       # @return [Boolean] true when the content-type is +text/*+ or one
       #   of {TEXTUAL_APPLICATION_TYPES}
       def self.textual?(content_type)
@@ -138,7 +139,7 @@ module Pikuri
     # Verbatim URL download tool. Thin wrapper over {Tool::Fetch.fetch}
     # that exposes it to the agent loop in OpenAI tool-call shape. Use for
     # raw textual payloads (JSON APIs, CSV files, +robots.txt+, source
-    # files); use {Tool::WEB_SCRAPE} for rendered web pages or PDFs where
+    # files); use {Tool::WEB_SCRAPE} for rendered web pages where
     # readability extraction makes the result usable.
     #
     # @return [Tool]
@@ -149,7 +150,7 @@ module Pikuri
 
         Usage:
         - Use for raw textual payloads: JSON APIs, CSV files, robots.txt, sitemaps, source files — anywhere a rendering pass would corrupt the data.
-        - For rendered HTML pages or PDFs, use web_scrape — it extracts readable content; fetch returns the raw HTML/PDF bytes unchanged.
+        - For rendered HTML pages, use web_scrape — it extracts readable content; fetch returns the raw HTML bytes unchanged.
         - Accepts text/* and common textual application/* types (JSON, XML, JS, XHTML, RSS, Atom). Refuses PDFs, images, and other binaries.
       DESC
       parameters: Parameters.build { |p|