pikuri-core 0.0.4 → 0.0.6

data/lib/pikuri/tool/scraper/pdf.rb

@@ -1,54 +0,0 @@
-# frozen_string_literal: true
-
-require 'pdf-reader'
-require 'stringio'
-
-module Pikuri
-  class Tool
-    module Scraper
-      # PDF → text extractor used by {Simple.visit} when the fetched
-      # response carries +application/pdf+. Wraps the +pdf-reader+ gem:
-      # walk every page, concatenate the extracted text, hand the result
-      # back as a single string the LLM can read.
-      #
-      # Best-effort by design. +pdf-reader+ produces clean text from PDFs
-      # generated from a digital source (LaTeX, Word export, ...) but
-      # returns nothing useful from scanned documents — there is no OCR
-      # in this path. When extraction yields no text we still return an
-      # empty string rather than raising, so the caller's cache stores a
-      # consistent result and the LLM sees an empty observation it can
-      # react to.
-      #
-      # Pure parser — no I/O. {.extract} takes PDF bytes and returns text,
-      # so tests can drive it against an in-memory fixture without
-      # touching the network.
-      module PDF
-        # Render +bytes+ as plain text, one page per paragraph.
-        #
-        # +pdf-reader+ raises a handful of typed exceptions for documents
-        # it cannot parse — broken xrefs ({::PDF::Reader::MalformedPDFError}),
-        # invalid page references ({::PDF::Reader::InvalidPageError}),
-        # encrypted/XFA files ({::PDF::Reader::UnsupportedFeatureError}).
-        # All three describe a property of the PDF the LLM can react to
-        # ("try a different URL"), so we re-raise them as {FetchError} —
-        # same convention as the HTTP layer in {Simple.fetch}. Genuine
-        # bugs in +pdf-reader+ itself surface as their own classes and
-        # crash loud.
-        #
-        # @param bytes [String] raw PDF document (binary string)
-        # @return [String] concatenated page text; possibly empty when
-        #   the PDF carries no extractable text (scanned image, empty
-        #   document)
-        # @raise [FetchError] when +pdf-reader+ refuses the document
-        def self.extract(bytes)
-          reader = ::PDF::Reader.new(StringIO.new(bytes))
-          reader.pages.map { |p| p.text.strip }.reject(&:empty?).join("\n\n")
-        rescue ::PDF::Reader::MalformedPDFError,
-               ::PDF::Reader::InvalidPageError,
-               ::PDF::Reader::UnsupportedFeatureError => e
-          raise FetchError, "PDF rendering failed: #{e.class.name.split('::').last}: #{e.message}"
-        end
-      end
-    end
-  end
-end

data/lib/pikuri/tool/scraper/simple.rb

@@ -1,183 +0,0 @@
-# frozen_string_literal: true
-
-require 'faraday'
-require 'uri'
-
-module Pikuri
-  class Tool
-    # Namespace for the URL-to-Markdown scraping stack used by
-    # {Tool::WEB_SCRAPE} and {Tool::FETCH}: a content-type-dispatching
-    # fetcher ({Simple}), pure content extractors ({HTML}, {PDF}), and a
-    # shared error type ({FetchError}). Nothing here knows about the LLM
-    # — the tools that wrap these layers turn rendered Markdown (or
-    # +FetchError+) into the next observation.
-    module Scraper
-      # Plain HTTP scraper: GET the URL with a real-browser User-Agent,
-      # follow redirects, and dispatch the response body to the parser
-      # matching its +Content-Type+. HTML and XHTML route to
-      # {HTML.extract}; +application/pdf+ routes to {PDF.extract}; any
-      # other +text/*+ type (plain text, Markdown, source files, …) is
-      # passed through verbatim since the LLM can already read it; the
-      # remaining types raise {FetchError} so the LLM observes the
-      # failure instead of receiving an empty rendering.
-      #
-      # Split into a thin HTTP fetch ({.fetch}) and a content-type
-      # dispatcher ({.visit}) so tests can drive each piece in isolation.
-      # "Simple" because everything happens in one Faraday GET — no
-      # headless browser, no JS execution.
-      module Simple
-        # @return [String] User-Agent sent with each request; many sites
-        #   reject requests with no UA or an obvious bot UA
-        USER_AGENT = 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 ' \
-                     '(KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'
-        # @return [String] +Accept+ header sent with each request. Lists
-        #   every content-type the dispatcher in {.visit} knows how to
-        #   render, so servers that content-negotiate hand back something
-        #   we can use. The trailing +text/*;q=0.8+ covers the verbatim
-        #   pass-through arm (plain text, Markdown, source files, …) at a
-        #   lower preference than rendered HTML/PDF.
-        ACCEPT = 'text/html,application/xhtml+xml,application/pdf,text/*;q=0.8'
-        # @return [Integer] maximum number of HTTP redirects to follow
-        #   before giving up
-        MAX_REDIRECTS = 5
-        # @return [Integer] connect timeout in seconds for the underlying
-        #   Faraday request
-        OPEN_TIMEOUT = 10
-        # @return [Integer] read timeout in seconds for the underlying
-        #   Faraday request
-        READ_TIMEOUT = 20
-
-        # @return [Integer] maximum number of characters of an error
-        #   response body to include in a {FetchError} message. The body is
-        #   often a multi-kilobyte HTML challenge page (Cloudflare, WAF
-        #   interstitial, etc.); a short excerpt tells the LLM what kind of
-        #   page came back without flooding the next observation.
-        ERROR_BODY_EXCERPT = 200
-
-        # Result of a successful {Simple.fetch}: the response body, the
-        # normalized content-type (lower-cased, with any +; charset=...+
-        # parameters stripped), and the final URL after redirects. The
-        # final URL is kept so future scrapers can resolve relative links
-        # against the actual landing page rather than the originally
-        # requested one.
-        Fetched = Data.define(:body, :content_type, :url)
-
-        # Fetch +url+ and render its main content as Markdown.
-        #
-        # No caching here — every call hits the network. Callers that want
-        # to memoize results should wrap this method themselves (see
-        # {Tool::WebScrape.visit}, which does exactly that).
-        #
-        # The dispatcher's output is +String#strip+'d so the LLM never
-        # sees a body that opens or closes with blank lines — common with
-        # +pdf-reader+'s page-feed whitespace and with text bodies that
-        # carry a trailing newline. Interior whitespace is preserved
-        # because Markdown paragraph breaks and source-code indentation
-        # are load-bearing.
-        #
-        # @param url [String] absolute HTTP(S) URL of the page to download
-        # @return [String] full Markdown representation of the page with
-        #   leading/trailing whitespace trimmed, uncapped otherwise —
-        #   caller is responsible for any size limiting before feeding
-        #   the result back to the LLM
-        # @raise [FetchError] on HTTP non-2xx, network failure, redirect
-        #   loop, a 3xx without a +Location+ header, or a response whose
-        #   content-type the dispatcher does not recognize
-        def self.visit(url)
-          dispatch(fetch(url)).strip
-        end
-
-        # Download the body of +url+, manually following up to
-        # {MAX_REDIRECTS} redirects. Faraday is configured with no
-        # middleware so behavior here mirrors the rest of the codebase
-        # (see +Tool::Search::DuckDuckGo.search+).
-        #
-        # All recoverable failures — HTTP 4xx/5xx, +Faraday::Error+ network
-        # blips, exhausted redirect budget, 3xx without a +Location+ —
-        # surface as {FetchError} so the caller has a single exception type
-        # to rescue. Error bodies are trimmed to {ERROR_BODY_EXCERPT}
-        # characters with whitespace collapsed, so a Cloudflare-challenge
-        # response doesn't dump kilobytes of inline HTML into the next LLM
-        # observation.
-        #
-        # @param url [String] absolute HTTP(S) URL to fetch
-        # @param limit [Integer] redirects remaining; recurses with
-        #   +limit - 1+ on each 3xx
-        # @return [Fetched] body, normalized content-type, and final URL
-        #   after redirects
-        # @raise [FetchError] on non-2xx/3xx responses, network errors,
-        #   redirect-loop exhaustion, or 3xx without a +Location+ header
-        def self.fetch(url, limit: MAX_REDIRECTS)
-          raise FetchError, "too many redirects fetching #{url}" if limit.zero?
-
-          response = begin
-            Faraday.new(request: { open_timeout: OPEN_TIMEOUT, timeout: READ_TIMEOUT }).get(url) do |req|
-              req.headers['User-Agent'] = USER_AGENT
-              req.headers['Accept']     = ACCEPT
-            end
-          rescue Faraday::Error => e
-            raise FetchError, "#{e.class.name.split('::').last} fetching #{url}: #{e.message}"
-          end
-
-          case response.status
-          when 200..299
-            Fetched.new(body: response.body, content_type: normalize_content_type(response.headers['content-type']), url: url)
-          when 300..399
-            location = response.headers['location']
-            raise FetchError, "HTTP #{response.status} from #{url} with no Location header" if location.nil? || location.empty?
-
-            fetch(URI.join(url, location).to_s, limit: limit - 1)
-          else
-            raise FetchError, "HTTP #{response.status} fetching #{url}: #{excerpt(response.body)}"
-          end
-        end
-
-        # Route a {Fetched} response to the parser that matches its
-        # content-type. Unknown types raise {FetchError} so the LLM gets a
-        # legible observation instead of an empty string.
-        #
-        # @param fetched [Fetched]
-        # @return [String] Markdown representation produced by the matched
-        #   parser
-        # @raise [FetchError] when no parser matches the response's
-        #   content-type
-        def self.dispatch(fetched)
-          case fetched.content_type
-          when 'text/html', 'application/xhtml+xml'
-            HTML.extract(fetched.body)
-          when 'application/pdf'
-            PDF.extract(fetched.body)
-          when %r{\Atext/}
-            fetched.body
-          else
-            raise FetchError, "unsupported content-type #{fetched.content_type.inspect} for #{fetched.url}"
-          end
-        end
-
-        # Lower-case +raw+ and strip any +; charset=...+ parameters so the
-        # dispatcher can match on a canonical token.
-        #
-        # @param raw [String, nil] raw +Content-Type+ header value
-        # @return [String] normalized content-type, or +""+ when the
-        #   header was missing
-        def self.normalize_content_type(raw)
-          raw.to_s.split(';').first.to_s.strip.downcase
-        end
-        private_class_method :normalize_content_type
-
-        # Whitespace-collapse +body+ and clip to {ERROR_BODY_EXCERPT}
-        # characters, so the {FetchError} message stays a single readable
-        # line even when the server returned a multi-KB HTML challenge
-        # page.
-        #
-        # @param body [String, nil]
-        # @return [String]
-        def self.excerpt(body)
-          text = body.to_s.gsub(/\s+/, ' ').strip
-          text.length > ERROR_BODY_EXCERPT ? "#{text[0, ERROR_BODY_EXCERPT]}..." : text
-        end
-        private_class_method :excerpt
-      end
-    end
-  end
-end