llm_scraper 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 7b23032aa5d35d2f2943290b0f353af606eea3590e4eaf5d7e54a689775a423f
+  data.tar.gz: 11bd8dc76ccda17b63d902bd0ce0d13d9c2badb139359e2448ba97a597ef948f
+SHA512:
+  metadata.gz: a1c6eb718ce331dcb4d895e2b2810d9f513208d8db2f7a1723be84bc41b13047cf13a33f625456183d19bc4b9af001cad578e952eae89e8a0f1d18317791d7a7
+  data.tar.gz: 8e819709d2fe96e9fcafe422e881ec853a13dc5041644ed2cb17e1460942f4a7fa1e859012609f8e4188c3c126e5658fd33de8101b435915c2d483797daf30a2

data/.env.example

@@ -0,0 +1,10 @@
+# Copy this file to .env and fill in your keys
+# .env is git-ignored — never commit real keys
+
+GEMINI_API_KEY=
+DEEPSEEK_API_KEY=
+ANTHROPIC_API_KEY=
+GLM_API_KEY=
+JINA_API_KEY=
+FIRECRAWL_API_KEY=
+MARKDOWNIFY_API_KEY=

data/CHANGELOG.md

@@ -0,0 +1,5 @@
+## [Unreleased]
+
+## [0.1.0] - 2026-06-08
+
+- Initial release

data/CODE_OF_CONDUCT.md

@@ -0,0 +1,10 @@
+# Code of Conduct
+
+"llm_scraper" follows [The Ruby Community Conduct Guideline](https://www.ruby-lang.org/en/conduct) in all "collaborative space", which is defined as community communications channels (such as mailing lists, submitted patches, commit comments, etc.):
+
+* Participants will be tolerant of opposing views.
+* Participants must ensure that their language and actions are free of personal attacks and disparaging personal remarks.
+* When interpreting the words and actions of others, participants should always assume good intentions.
+* Behaviour which can be reasonably considered harassment will not be tolerated.
+
+If you have any concerns about behaviour within this project, please contact us at ["cuongnguyenfu@gmail.com"](mailto:"cuongnguyenfu@gmail.com").

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2026 cuongnc0211
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,268 @@
+# LlmScraper
+
+Extract structured JSON from any web page using LLMs — more reliable than CSS selectors, works even when markup changes.
+
+```
+URL → ContentFetcher (URL → Markdown) → LlmClient (Markdown → JSON) → Result
+```
+
+Two independent layers let you mix any fetcher with any LLM provider.
+
+## Features
+
+- **Schema DSL** — declare fields with `type`, `what`, `how`, `examples`, `enum`, `required`, `default`
+- **Multiple fetchers** — Jina AI, Firecrawl, ScrapeGraphAI Markdownify, or local Nokogiri (+ optional Ferrum for SPA)
+- **Multiple LLM providers** — any OpenAI-compatible API (DeepSeek, Kimi, GLM, Gemini, OpenRouter…) or Anthropic native
+- **Automatic retry** — re-prompts once with stricter instructions on JSON parse failure
+- **Cost estimation** — `result.cost_usd` based on token usage
+- **Minimal dependencies** — Faraday + Nokogiri + Zeitwerk, no Rails required
+
+## Installation
+
+Add to your Gemfile:
+
+```ruby
+gem "llm_scraper"
+```
+
+Or install directly:
+
+```bash
+gem install llm_scraper
+```
+
+## Quick Start
+
+```ruby
+require "llm_scraper"
+
+LlmScraper.configure do |c|
+  c.llm_provider = :openai_compatible
+  c.llm_base_url = "https://api.deepseek.com/v1"
+  c.llm_api_key  = ENV["DEEPSEEK_API_KEY"]
+  c.llm_model    = "deepseek-v4-flash"
+  c.fetcher      = :jina
+  c.jina_api_key = ENV["JINA_API_KEY"]   # optional — 200 req/day free without key
+end
+
+schema = LlmScraper::Schema.define do
+  field :name,  type: :string,  required: true, description: "Full name of the artisan"
+  field :price, type: :number,  what: "Current retail price",
+                                how: "Return CNY value as a number, strip ¥ symbol"
+  field :style, type: :string,  enum: ["yixing", "zhuni", "duanni"]
+end
+
+result = LlmScraper::Scraper.new(schema: schema).scrape("https://example.com/teapot")
+
+result.success?      # => true
+result.data          # => { name: "Gu Jingzhou", price: 15000, style: "yixing" }
+result.tokens_used   # => { input: 4821, output: 87 }
+result.cost_usd      # => 0.0009
+result.fetcher       # => :jina
+result.provider      # => :openai_compatible
+result.duration_ms   # => 1842
+```
+
+## Schema DSL
+
+```ruby
+schema = LlmScraper::Schema.define do
+  # Simple field — description is enough
+  field :name,      type: :string,  required: true, description: "Full artisan name"
+  field :available, type: :boolean, default: true,  description: "In stock status"
+
+  # Complex field — what identifies the field, how tells the LLM how to extract it
+  field :price,
+        type:     :number,
+        what:     "Current retail price (not auction, not historical)",
+        how:      "Return CNY as a plain number, strip ¥. If multiple prices, take the lowest",
+        examples: [1500, 8000, 25000]
+
+  # Closed-set field — LLM must pick from the list
+  field :clay_type,
+        type:     :string,
+        what:     "Clay type used",
+        how:      "Return lowercase English name",
+        enum:     ["zisha", "zhuni", "duanni", "hongni"]
+
+  # Array field
+  field :techniques, type: :array, items: :string,
+        description: "Distinctive crafting techniques"
+end
+```
+
+### Field options
+
+| Option | Purpose |
+|---|---|
+| `type` | `:string`, `:number`, `:boolean`, `:array`, `:object` |
+| `description` | Alias for `what` — use for simple fields |
+| `what` | What this field is (identity, disambiguation) |
+| `how` | Extraction instruction (normalization, format, edge cases) |
+| `examples` | Few-shot values to improve accuracy |
+| `enum` | Closed-set — LLM must pick one of these values |
+| `required` | Raises `ParseError` if null after extraction |
+| `default` | Fallback value when field is missing |
+| `items` | Element type for `type: :array` |
+
+Schema can also be a plain Hash:
+
+```ruby
+schema = {
+  name:  { type: :string,  required: true, description: "Artisan name" },
+  price: { type: :number,  description: "Price in CNY" },
+}
+```
+
+## Fetchers
+
+### Jina AI (default, recommended)
+
+Clean Markdown via `r.jina.ai` — no JS execution needed, generous free tier.
+
+```ruby
+c.fetcher      = :jina
+c.jina_api_key = ENV["JINA_API_KEY"]   # optional, ~200 req/day without key
+```
+
+### Firecrawl
+
+Higher fidelity, handles JS-heavy pages, 1 credit per page.
+
+```ruby
+c.fetcher            = :firecrawl
+c.firecrawl_api_key  = ENV["FIRECRAWL_API_KEY"]
+```
+
+### ScrapeGraphAI Markdownify
+
+```ruby
+c.fetcher               = :markdownify
+c.markdownify_api_key   = ENV["MARKDOWNIFY_API_KEY"]
+```
+
+### Local (Nokogiri)
+
+No external API — fetches directly and strips boilerplate HTML with Nokogiri.
+
+```ruby
+c.fetcher = :local
+```
+
+For SPA pages that require JavaScript, add `ferrum` to your Gemfile:
+
+```ruby
+gem "ferrum"
+```
+
+## LLM Providers
+
+### OpenAI-compatible (DeepSeek, Kimi, GLM, Gemini, OpenRouter…)
+
+```ruby
+# DeepSeek V4 Flash — cheap and accurate
+c.llm_provider = :openai_compatible
+c.llm_base_url = "https://api.deepseek.com/v1"
+c.llm_api_key  = ENV["DEEPSEEK_API_KEY"]
+c.llm_model    = "deepseek-v4-flash"
+
+# GLM-4.7-Flash — free, good for testing
+c.llm_base_url = "https://open.bigmodel.cn/api/paas/v4"
+c.llm_api_key  = ENV["GLM_API_KEY"]
+c.llm_model    = "glm-4.7-flash"
+
+# Gemini 2.5 Flash
+c.llm_base_url = "https://generativelanguage.googleapis.com/v1beta/openai"
+c.llm_api_key  = ENV["GEMINI_API_KEY"]
+c.llm_model    = "gemini-2.5-flash"
+
+# Kimi K2.5 — long context, auto cache
+c.llm_base_url = "https://api.moonshot.ai/v1"
+c.llm_api_key  = ENV["KIMI_API_KEY"]
+c.llm_model    = "kimi-k2.5"
+```
+
+### Anthropic
+
+```ruby
+c.llm_provider = :anthropic
+c.llm_api_key  = ENV["ANTHROPIC_API_KEY"]
+c.llm_model    = "claude-haiku-4-5-20251001"
+```
+
+## API
+
+### `Scraper#scrape(url, rescue_errors: false)`
+
+Fetch URL then extract. Raises on error by default; pass `rescue_errors: true` to get a failure `Result` instead.
+
+### `Scraper#extract(content)`
+
+Extract from raw HTML/Markdown — skips the fetch step.
+
+### `Scraper#scrape_batch(urls)`
+
+Scrapes multiple URLs. Never raises — errors are captured in `result.error` per item.
+
+```ruby
+results = scraper.scrape_batch(["https://...", "https://..."])
+results.each { |r| puts r.data if r.success? }
+```
+
+### `Scraper#with_provider(provider)` / `#with_fetcher(fetcher)`
+
+Return a new `Scraper` with a swapped provider or fetcher — original is unchanged.
+
+```ruby
+cheap    = scraper.with_provider(:openai_compatible)
+accurate = scraper.with_provider(:anthropic)
+offline  = scraper.with_fetcher(:local)
+```
+
+### `Result`
+
+| Field | Type | Description |
+|---|---|---|
+| `data` | `Hash` | Extracted fields (symbol keys) |
+| `success?` | `Boolean` | |
+| `error` | `String\|nil` | Error message on failure |
+| `url` | `String\|nil` | Source URL |
+| `fetcher` | `Symbol` | Fetcher used |
+| `provider` | `Symbol` | LLM provider used |
+| `model` | `String` | Model name |
+| `tokens_used` | `Hash` | `{ input:, output: }` |
+| `cost_usd` | `Float` | Estimated cost |
+| `duration_ms` | `Integer` | Total wall time |
+
+## Estimated Cost (1,000 pages/day)
+
+| Combo | Fetcher | LLM/day | Total/day |
+|---|---|---|---|
+| Jina free + GLM-4.7-Flash | $0 | $0 | **$0** |
+| Jina free + DeepSeek V4 Flash | $0 | ~$0.85 | **~$0.85** |
+| Local + DeepSeek V4 Flash | $0 | ~$2–4 | **~$2–4** |
+| Jina free + Claude Haiku | $0 | ~$3–5 | **~$3–5** |
+
+> Local fetcher produces ~4× more tokens than Jina Markdown.
+
+## Development
+
+```bash
+git clone https://github.com/cuongnc0211/llm_scraper
+cd llm_scraper
+bundle install
+
+cp .env.example .env
+# Add your API keys to .env
+
+bundle exec rspec        # run tests
+bin/console              # interactive console with dotenv loaded
+```
+
+## Contributing
+
+Bug reports and pull requests are welcome at https://github.com/cuongnc0211/llm_scraper.
+
+## License
+
+MIT — see [LICENSE.txt](LICENSE.txt).

data/Rakefile

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+
+require "rubocop/rake_task"
+
+RuboCop::RakeTask.new
+
+task default: %i[spec rubocop]

data/lib/llm_scraper/configuration.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module LlmScraper
+  class Configuration
+    attr_accessor :llm_provider      # :openai_compatible | :anthropic
+    attr_accessor :llm_base_url      # e.g. "https://api.deepseek.com/v1"
+    attr_accessor :llm_api_key
+    attr_accessor :llm_model         # e.g. "deepseek-v4-flash"
+    attr_accessor :llm_timeout       # seconds
+    attr_accessor :max_retries
+
+    attr_accessor :fetcher            # :jina | :firecrawl | :markdownify | :local
+    attr_accessor :jina_api_key       # nil = unauthenticated (~200 req/day limit)
+    attr_accessor :firecrawl_api_key
+    attr_accessor :markdownify_api_key
+
+    def initialize
+      @llm_provider = :openai_compatible
+      @llm_timeout  = 30
+      @max_retries  = 3
+      @fetcher      = :jina
+    end
+  end
+end

data/lib/llm_scraper/content_fetchers/base.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module LlmScraper
+  module ContentFetchers
+    class Base
+      # @param url [String]
+      # @return [String] cleaned text/markdown content
+      # @raise [LlmScraper::FetchError]
+      def fetch(url)
+        raise NotImplementedError, "#{self.class}#fetch not implemented"
+      end
+
+      private
+
+      def build_connection(base_url: nil, timeout: 30)
+        Faraday.new(url: base_url) do |f|
+          f.request  :retry, max: 3, interval: 1, backoff_factor: 2
+          f.options.timeout      = timeout
+          f.options.open_timeout = 10
+          f.adapter  Faraday.default_adapter
+        end
+      end
+    end
+  end
+end

data/lib/llm_scraper/content_fetchers/firecrawl.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module LlmScraper
+  module ContentFetchers
+    class Firecrawl < Base
+      BASE_URL = "https://api.firecrawl.dev"
+
+      def initialize(config = LlmScraper.configuration)
+        @config = config
+        @conn   = build_connection(base_url: BASE_URL, timeout: 60)
+      end
+
+      # @param url [String]
+      # @return [String] markdown content
+      def fetch(url)
+        response = @conn.post("/v1/scrape") do |req|
+          req.headers["Authorization"] = "Bearer #{@config.firecrawl_api_key}"
+          req.headers["Content-Type"]  = "application/json"
+          req.body = JSON.generate(url: url, formats: ["markdown"])
+        end
+
+        raise LlmScraper::FetchError, "Firecrawl error (#{response.status}): #{response.body}" unless response.success?
+
+        body = JSON.parse(response.body)
+        body.dig("data", "markdown") ||
+          raise(LlmScraper::FetchError, "Firecrawl returned no markdown for #{url}")
+      rescue Faraday::Error => e
+        raise LlmScraper::FetchError, "Firecrawl fetch error: #{e.message}"
+      rescue JSON::ParserError => e
+        raise LlmScraper::FetchError, "Firecrawl response parse error: #{e.message}"
+      end
+    end
+  end
+end

data/lib/llm_scraper/content_fetchers/jina.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module LlmScraper
+  module ContentFetchers
+    class Jina < Base
+      BASE_URL = "https://r.jina.ai"
+
+      def initialize(config = LlmScraper.configuration)
+        @config = config
+        @conn   = build_connection(base_url: BASE_URL, timeout: 30)
+      end
+
+      # @param url [String]
+      # @return [String] markdown content
+      def fetch(url)
+        warn "[LlmScraper] No jina_api_key set — unauthenticated (~200 req/day limit)" if @config.jina_api_key.nil?
+
+        response = @conn.get("/#{url}") do |req|
+          req.headers["Accept"]        = "text/markdown"
+          req.headers["Authorization"] = "Bearer #{@config.jina_api_key}" if @config.jina_api_key
+        end
+
+        raise LlmScraper::FetchError, "Jina error (#{response.status}): #{response.body}" unless response.success?
+
+        response.body
+      rescue Faraday::Error => e
+        raise LlmScraper::FetchError, "Jina fetch error: #{e.message}"
+      end
+    end
+  end
+end

data/lib/llm_scraper/content_fetchers/local.rb

@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+module LlmScraper
+  module ContentFetchers
+    class Local < Base
+      def initialize(config = LlmScraper.configuration)
+        @config = config
+      end
+
+      # @param url [String]
+      # @param spa [Boolean] use Ferrum for JS-rendered pages
+      # @return [String] cleaned plain text
+      def fetch(url, spa: false)
+        spa ? fetch_spa(url) : fetch_static(url)
+      end
+
+      private
+
+      def fetch_static(url)
+        conn = build_connection(timeout: 30)
+        response = conn.get(url)
+        raise LlmScraper::FetchError, "HTTP #{response.status} for #{url}" unless response.success?
+
+        clean_html(response.body)
+      rescue Faraday::Error => e
+        raise LlmScraper::FetchError, "Failed to fetch #{url}: #{e.message}"
+      end
+
+      def fetch_spa(url)
+        begin
+          require "ferrum"
+        rescue LoadError
+          raise LlmScraper::FetchError,
+                "Ferrum gem required for SPA fetching. Add `gem 'ferrum'` to your Gemfile."
+        end
+
+        browser = Ferrum::Browser.new(headless: true)
+        browser.go_to(url)
+        browser.network.wait_for_idle
+        html = browser.body
+        browser.quit
+        clean_html(html)
+      rescue LlmScraper::FetchError
+        raise
+      rescue => e
+        raise LlmScraper::FetchError, "SPA fetch failed for #{url}: #{e.message}"
+      end
+
+      # Strips boilerplate, collapses whitespace — reduces ~50k tokens → ~5k
+      def clean_html(html)
+        doc = Nokogiri::HTML(html)
+        doc.css("script, style, nav, footer, header, [aria-hidden]").remove
+        doc.css("body").inner_text.gsub(/\s+/, " ").strip
+      end
+    end
+  end
+end

data/lib/llm_scraper/content_fetchers/markdownify.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module LlmScraper
+  module ContentFetchers
+    class Markdownify < Base
+      BASE_URL = "https://api.scrapegraphai.com"
+
+      def initialize(config = LlmScraper.configuration)
+        @config = config
+        @conn   = build_connection(base_url: BASE_URL, timeout: 60)
+      end
+
+      # @param url [String]
+      # @return [String] markdown content
+      def fetch(url)
+        response = @conn.post("/v1/markdownify") do |req|
+          req.headers["SGAI-APIKEY"]  = @config.markdownify_api_key
+          req.headers["Content-Type"] = "application/json"
+          req.body = JSON.generate(website_url: url)
+        end
+
+        raise LlmScraper::FetchError, "Markdownify error (#{response.status}): #{response.body}" unless response.success?
+
+        body = JSON.parse(response.body)
+        body["result"] ||
+          raise(LlmScraper::FetchError, "Markdownify returned no content for #{url}")
+      rescue Faraday::Error => e
+        raise LlmScraper::FetchError, "Markdownify fetch error: #{e.message}"
+      rescue JSON::ParserError => e
+        raise LlmScraper::FetchError, "Markdownify response parse error: #{e.message}"
+      end
+    end
+  end
+end

data/lib/llm_scraper/llm_clients/anthropic.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+module LlmScraper
+  module LlmClients
+    class Anthropic < Base
+      BASE_URL    = "https://api.anthropic.com"
+      API_VERSION = "2023-06-01"
+
+      def initialize(config = LlmScraper.configuration)
+        @config = config
+        @conn   = build_connection(base_url: BASE_URL, timeout: config.llm_timeout)
+      end
+
+      # @param prompt [String]
+      # @return [Hash] { content:, tokens:, cost_usd: }
+      def complete(prompt)
+        response = @conn.post("v1/messages") do |req|
+          req.headers["x-api-key"]        = @config.llm_api_key
+          req.headers["anthropic-version"] = API_VERSION
+          req.headers["Content-Type"]      = "application/json"
+          req.body = JSON.generate(
+            model:      @config.llm_model,
+            max_tokens: 1024,
+            messages:   [{ role: "user", content: prompt }]
+          )
+        end
+
+        handle_response(response)
+      rescue Faraday::Error => e
+        raise LlmScraper::LlmError, "Anthropic request failed: #{e.message}"
+      end
+
+      private
+
+      def handle_response(response)
+        raise LlmScraper::LlmError, "Anthropic API error #{response.status}: #{response.body}" unless response.success?
+
+        body          = JSON.parse(response.body)
+        content       = body.dig("content", 0, "text")
+        input_tokens  = body.dig("usage", "input_tokens").to_i
+        output_tokens = body.dig("usage", "output_tokens").to_i
+
+        {
+          content:  content,
+          tokens:   { input: input_tokens, output: output_tokens },
+          cost_usd: estimate_cost(@config.llm_model, input_tokens, output_tokens)
+        }
+      rescue JSON::ParserError => e
+        raise LlmScraper::LlmError, "Failed to parse Anthropic response: #{e.message}"
+      end
+    end
+  end
+end

data/lib/llm_scraper/llm_clients/base.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+module LlmScraper
+  module LlmClients
+    class Base
+      # Pricing in USD per 1M tokens (updated 2026-06)
+      PRICING = {
+        "deepseek-v4-flash"        => { input: 0.14, output: 0.28 },
+        "deepseek-v4-pro"          => { input: 1.74, output: 3.48 },
+        "kimi-k2.5"                => { input: 0.60, output: 3.00 },
+        "glm-4.7-flash"            => { input: 0.0,  output: 0.0  },
+        "claude-haiku-4-5"         => { input: 0.80, output: 4.00 },
+        "claude-haiku-4-5-20251001" => { input: 0.80, output: 4.00 },
+        "gemini-2.5-flash"         => { input: 0.15, output: 0.60 },
+      }.freeze
+
+      # @param prompt [String]
+      # @return [Hash] { content: String, tokens: { input: Integer, output: Integer }, cost_usd: Float }
+      # @raise [LlmScraper::LlmError]
+      def complete(prompt)
+        raise NotImplementedError, "#{self.class}#complete not implemented"
+      end
+
+      private
+
+      def estimate_cost(model, input_tokens, output_tokens)
+        pricing = PRICING[model] || { input: 0.0, output: 0.0 }
+        (input_tokens * pricing[:input] + output_tokens * pricing[:output]) / 1_000_000.0
+      end
+
+      def build_connection(base_url:, timeout: 30)
+        # Trailing slash required so relative paths (e.g. "chat/completions")
+        # are appended correctly when base_url itself has a path component
+        url = base_url.end_with?("/") ? base_url : "#{base_url}/"
+        Faraday.new(url: url) do |f|
+          f.request  :retry, max: 2, interval: 1, backoff_factor: 2,
+                     retry_statuses: [429, 500, 502, 503, 504]
+          f.options.timeout      = timeout
+          f.options.open_timeout = 10
+          f.adapter  Faraday.default_adapter
+        end
+      end
+    end
+  end
+end

data/lib/llm_scraper/llm_clients/openai_compatible.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+module LlmScraper
+  module LlmClients
+    class OpenaiCompatible < Base
+      def initialize(config = LlmScraper.configuration)
+        @config = config
+        @conn   = build_connection(base_url: config.llm_base_url, timeout: config.llm_timeout)
+      end
+
+      # @param prompt [String]
+      # @return [Hash] { content:, tokens:, cost_usd: }
+      def complete(prompt)
+        response = @conn.post("chat/completions") do |req|
+          req.headers["Authorization"] = "Bearer #{@config.llm_api_key}"
+          req.headers["Content-Type"]  = "application/json"
+          req.body = JSON.generate(
+            model:           @config.llm_model,
+            messages:        [{ role: "user", content: prompt }],
+            temperature:     0,
+            response_format: { type: "json_object" }
+          )
+        end
+
+        handle_response(response)
+      rescue Faraday::Error => e
+        raise LlmScraper::LlmError, "LLM request failed: #{e.message}"
+      end
+
+      private
+
+      def handle_response(response)
+        raise LlmScraper::LlmError, "LLM API error #{response.status}: #{response.body}" unless response.success?
+
+        body          = JSON.parse(response.body)
+        content       = body.dig("choices", 0, "message", "content")
+        input_tokens  = body.dig("usage", "prompt_tokens").to_i
+        output_tokens = body.dig("usage", "completion_tokens").to_i
+
+        {
+          content:  content,
+          tokens:   { input: input_tokens, output: output_tokens },
+          cost_usd: estimate_cost(@config.llm_model, input_tokens, output_tokens)
+        }
+      rescue JSON::ParserError => e
+        raise LlmScraper::LlmError, "Failed to parse LLM response: #{e.message}"
+      end
+    end
+  end
+end

data/lib/llm_scraper/prompt_builder.rb

@@ -0,0 +1,73 @@
+# frozen_string_literal: true
+
+module LlmScraper
+  class PromptBuilder
+    # @param schema [Schema]
+    # @param content [String]
+    # @return [String]
+    def self.build(schema, content)
+      new(schema).build(content)
+    end
+
+    def initialize(schema)
+      @schema = schema
+    end
+
+    def build(content)
+      <<~PROMPT
+        Extract the following fields from the content below.
+        Return ONLY a valid JSON object. No markdown fences. No explanation.
+
+        Fields:
+        #{render_fields}
+        Rules:
+        - Missing field → null (never omit the key)
+        - Return nothing except the JSON object
+
+        Content:
+        #{content}
+      PROMPT
+    end
+
+    private
+
+    def render_fields
+      @schema.fields.map { |name, field| render_field(name, field) }.join("\n")
+    end
+
+    def render_field(name, field)
+      if field.has_instructions?
+        render_detailed(name, field)
+      else
+        render_inline(name, field)
+      end
+    end
+
+    # Simple fields: single line
+    def render_inline(name, field)
+      label = build_type_label(field)
+      what  = field.what || name.to_s
+      "- #{name} (#{label}): #{what}"
+    end
+
+    # Fields with how/examples/enum: multiline block
+    def render_detailed(name, field)
+      label = build_type_label(field)
+      lines = ["- #{name} (#{label}):"]
+      lines << "    Field: #{field.what}"          if field.what
+      lines << "    Instructions: #{field.how}"    if field.how
+      lines << "    Examples: #{field.examples.join(", ")}" if field.examples
+      lines << "    Allowed values: #{field.enum.join(", ")}" if field.enum
+      lines.join("\n")
+    end
+
+    def build_type_label(field)
+      base = if field.type == :array
+               field.items ? "array of #{field.items}" : "array"
+             else
+               field.type.to_s
+             end
+      field.required ? "#{base}, required" : base
+    end
+  end
+end

data/lib/llm_scraper/response_parser.rb

@@ -0,0 +1,80 @@
+# frozen_string_literal: true
+
+module LlmScraper
+  class ResponseParser
+    # @param content [String] raw LLM response
+    # @param schema [Schema]
+    # @return [Hash]
+    # @raise [LlmScraper::ParseError]
+    def self.parse(content, schema)
+      new(schema).parse(content)
+    end
+
+    def initialize(schema)
+      @schema = schema
+    end
+
+    def parse(content)
+      json    = strip_fences(content)
+      data    = JSON.parse(json)
+      coerced = coerce(data)
+      validate_required!(coerced)
+      coerced
+    rescue JSON::ParserError => e
+      raise LlmScraper::ParseError, "Invalid JSON from LLM: #{e.message}\nRaw: #{content}"
+    end
+
+    private
+
+    # Strip markdown fences in case LLM ignores the instruction
+    def strip_fences(content)
+      content
+        .gsub(/\A```(?:json)?\s*/i, "")
+        .gsub(/\s*```\z/, "")
+        .strip
+    end
+
+    def coerce(data)
+      @schema.fields.each_with_object({}) do |(name, field), result|
+        raw          = data[name.to_s]
+        result[name] = raw.nil? ? field.default : coerce_value(raw, field)
+      end
+    end
+
+    def coerce_value(value, field)
+      return nil if value.nil?
+
+      case field.type
+      when :number  then coerce_number(value)
+      when :boolean then coerce_boolean(value)
+      when :array   then Array(value)
+      else value.to_s
+      end
+    end
+
+    # Strips currency symbols and thousands separators: "¥150,000" → 150000
+    def coerce_number(value)
+      return value if value.is_a?(Numeric)
+
+      cleaned = value.to_s.gsub(/[^\d.\-]/, "")
+      return nil if cleaned.empty?
+
+      cleaned.include?(".") ? cleaned.to_f : cleaned.to_i
+    end
+
+    def coerce_boolean(value)
+      return value if value == true || value == false
+
+      %w[true yes 1].include?(value.to_s.downcase)
+    end
+
+    def validate_required!(data)
+      @schema.fields.each do |name, field|
+        next unless field.required
+        next unless data[name].nil?
+
+        raise LlmScraper::ParseError, "Required field '#{name}' is missing or null in LLM response"
+      end
+    end
+  end
+end

data/lib/llm_scraper/result.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+module LlmScraper
+  Result = Struct.new(
+    :data,         # Hash — extracted fields
+    :success,      # Boolean
+    :error,        # String | nil
+    :url,          # String | nil
+    :fetcher,      # Symbol — fetcher used
+    :provider,     # Symbol — LLM provider used
+    :model,        # String — model name
+    :tokens_used,  # Hash — { input: Integer, output: Integer }
+    :cost_usd,     # Float — estimated cost
+    :duration_ms,  # Integer — total time in milliseconds
+    keyword_init: true
+  ) do
+    def success? = success
+    def failure? = !success
+  end
+end

data/lib/llm_scraper/schema.rb

@@ -0,0 +1,72 @@
+# frozen_string_literal: true
+
+module LlmScraper
+  class Schema
+    attr_reader :fields
+
+    # @param block [Proc] DSL block
+    # @return [Schema]
+    def self.define(&block)
+      schema = new
+      schema.instance_eval(&block)
+      schema
+    end
+
+    # @param hash [Hash] { field_name => { type:, description:, ... } }
+    # @return [Schema]
+    def self.from_hash(hash)
+      schema = new
+      hash.each do |name, opts|
+        schema.field(name.to_sym, **opts.transform_keys(&:to_sym))
+      end
+      schema
+    end
+
+    def initialize
+      @fields = {}
+    end
+
+    # @param name [Symbol]
+    # @param type [Symbol] :string | :number | :boolean | :array | :object
+    # @param options [Hash]
+    def field(name, type:, **options)
+      @fields[name.to_sym] = Field.new(name: name.to_sym, type: type, **options)
+    end
+
+    class Field
+      VALID_TYPES = %i[string number boolean array object].freeze
+
+      attr_reader :name, :type, :what, :how, :examples, :enum,
+                  :required, :default, :items
+
+      def initialize(name:, type:, description: nil, what: nil, how: nil,
+                     examples: nil, enum: nil, required: false,
+                     default: nil, items: nil, **_rest)
+        @name     = name
+        @type     = type.to_sym
+        @what     = what || description
+        @how      = how
+        @examples = examples
+        @enum     = enum
+        @required = required
+        @default  = default
+        @items    = items
+
+        validate!
+      end
+
+      # True when field has extraction instructions beyond just a label
+      def has_instructions?
+        !@how.nil? || !@examples.nil? || !@enum.nil?
+      end
+
+      private
+
+      def validate!
+        return if VALID_TYPES.include?(@type)
+
+        raise LlmScraper::SchemaError, "Field '#{@name}': unsupported type '#{@type}'"
+      end
+    end
+  end
+end

data/lib/llm_scraper/scraper.rb

@@ -0,0 +1,131 @@
+# frozen_string_literal: true
+
+module LlmScraper
+  class Scraper
+    # @param schema [Schema, Hash]
+    # @param config [Configuration]
+    def initialize(schema:, config: LlmScraper.configuration)
+      @schema = normalize_schema(schema)
+      @config = config
+    end
+
+    # @param url [String]
+    # @param rescue_errors [Boolean] return error Result instead of raising
+    # @return [Result]
+    # @raise [LlmScraper::Error] when rescue_errors is false
+    def scrape(url, rescue_errors: false)
+      start = monotonic_now
+      result = run_pipeline(url: url)
+      attach_timing(result, start)
+    rescue LlmScraper::Error => e
+      raise unless rescue_errors
+
+      Result.new(success: false, error: e.message, url: url,
+                 fetcher: @config.fetcher, provider: @config.llm_provider,
+                 model: @config.llm_model)
+    end
+
+    # Extract from raw content — skips fetching step
+    # @param content [String]
+    # @return [Result]
+    def extract(content)
+      start = monotonic_now
+      attach_timing(run_llm_pipeline(content: content), start)
+    end
+
+    # @param urls [Array<String>]
+    # @return [Array<Result>] never raises — errors captured in result.error
+    def scrape_batch(urls)
+      urls.map { |url| scrape(url, rescue_errors: true) }
+    end
+
+    # @param provider [Symbol] :openai_compatible | :anthropic
+    # @return [Scraper] new instance with swapped LLM provider
+    def with_provider(provider)
+      self.class.new(schema: @schema, config: clone_config(llm_provider: provider))
+    end
+
+    # @param fetcher [Symbol] :jina | :firecrawl | :markdownify | :local
+    # @return [Scraper] new instance with swapped fetcher
+    def with_fetcher(fetcher)
+      self.class.new(schema: @schema, config: clone_config(fetcher: fetcher))
+    end
+
+    private
+
+    def run_pipeline(url:)
+      content = build_fetcher.fetch(url)
+      result  = run_llm_pipeline(content: content)
+      result.url     = url
+      result.fetcher = @config.fetcher
+      result
+    end
+
+    def run_llm_pipeline(content:)
+      client     = build_llm_client
+      prompt     = PromptBuilder.build(@schema, content)
+      llm_result = client.complete(prompt)
+      data       = parse_with_retry(llm_result[:content], client, prompt)
+
+      Result.new(
+        data:        data,
+        success:     true,
+        provider:    @config.llm_provider,
+        model:       @config.llm_model,
+        tokens_used: llm_result[:tokens],
+        cost_usd:    llm_result[:cost_usd]
+      )
+    end
+
+    # Retry once with a stricter prompt on JSON parse failure
+    def parse_with_retry(content, client, original_prompt)
+      ResponseParser.parse(content, @schema)
+    rescue LlmScraper::ParseError
+      retry_prompt = original_prompt +
+        "\n\nCRITICAL: Return ONLY the JSON object. Example output: {\"field\": \"value\"}"
+      llm_result = client.complete(retry_prompt)
+      ResponseParser.parse(llm_result[:content], @schema)
+    end
+
+    def build_fetcher
+      case @config.fetcher
+      when :jina        then ContentFetchers::Jina.new(@config)
+      when :firecrawl   then ContentFetchers::Firecrawl.new(@config)
+      when :markdownify then ContentFetchers::Markdownify.new(@config)
+      when :local       then ContentFetchers::Local.new(@config)
+      else raise ConfigurationError, "Unknown fetcher: #{@config.fetcher.inspect}"
+      end
+    end
+
+    def build_llm_client
+      case @config.llm_provider
+      when :openai_compatible then LlmClients::OpenaiCompatible.new(@config)
+      when :anthropic         then LlmClients::Anthropic.new(@config)
+      else raise ConfigurationError, "Unknown LLM provider: #{@config.llm_provider.inspect}"
+      end
+    end
+
+    def normalize_schema(schema)
+      case schema
+      when Schema then schema
+      when Hash   then Schema.from_hash(schema)
+      else raise SchemaError, "schema must be a Hash or LlmScraper::Schema instance"
+      end
+    end
+
+    def clone_config(**overrides)
+      new_config = @config.dup
+      overrides.each { |key, val| new_config.public_send(:"#{key}=", val) }
+      new_config
+    end
+
+    def attach_timing(result, start)
+      result.duration_ms = ((monotonic_now - start) * 1000).round
+      result
+    end
+
+    def monotonic_now
+      Process.clock_gettime(Process::CLOCK_MONOTONIC)
+    end
+  end
+end

data/lib/llm_scraper/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module LlmScraper
+  VERSION = "0.1.0"
+end

data/lib/llm_scraper.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+require "zeitwerk"
+require "faraday"
+require "faraday/retry"
+require "nokogiri"
+require "json"
+
+loader = Zeitwerk::Loader.for_gem
+loader.setup
+
+module LlmScraper
+  class Error              < StandardError; end
+  class FetchError         < Error; end
+  class LlmError           < Error; end
+  class ParseError         < Error; end
+  class SchemaError        < Error; end
+  class ConfigurationError < Error; end
+
+  def self.configure
+    yield configuration
+  end
+
+  def self.configuration
+    @configuration ||= Configuration.new
+  end
+
+  def self.reset_configuration!
+    @configuration = Configuration.new
+  end
+end

data/sig/llm_scraper.rbs

@@ -0,0 +1,4 @@
+module LlmScraper
+  VERSION: String
+  # See the writing guide of rbs: https://github.com/ruby/rbs#guides
+end

metadata

@@ -0,0 +1,179 @@
+--- !ruby/object:Gem::Specification
+name: llm_scraper
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- cuongnc0211
+bindir: exe
+cert_chain: []
+date: 1980-01-02 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: faraday
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.10'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.10'
+- !ruby/object:Gem::Dependency
+  name: faraday-retry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.15'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.15'
+- !ruby/object:Gem::Dependency
+  name: zeitwerk
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '2.6'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '2.6'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: vcr
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '6.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '6.0'
+- !ruby/object:Gem::Dependency
+  name: webmock
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: rubocop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+description: 'Pipeline: URL → ContentFetcher (Markdown) → LlmClient (JSON). Supports
+  Jina/Firecrawl fetchers and OpenAI-compatible/Anthropic LLM providers.'
+email:
+- cuongnguyenfu@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".env.example"
+- CHANGELOG.md
+- CODE_OF_CONDUCT.md
+- LICENSE.txt
+- README.md
+- Rakefile
+- lib/llm_scraper.rb
+- lib/llm_scraper/configuration.rb
+- lib/llm_scraper/content_fetchers/base.rb
+- lib/llm_scraper/content_fetchers/firecrawl.rb
+- lib/llm_scraper/content_fetchers/jina.rb
+- lib/llm_scraper/content_fetchers/local.rb
+- lib/llm_scraper/content_fetchers/markdownify.rb
+- lib/llm_scraper/llm_clients/anthropic.rb
+- lib/llm_scraper/llm_clients/base.rb
+- lib/llm_scraper/llm_clients/openai_compatible.rb
+- lib/llm_scraper/prompt_builder.rb
+- lib/llm_scraper/response_parser.rb
+- lib/llm_scraper/result.rb
+- lib/llm_scraper/schema.rb
+- lib/llm_scraper/scraper.rb
+- lib/llm_scraper/version.rb
+- sig/llm_scraper.rbs
+homepage: https://github.com/cuongnc0211/llm_scraper
+licenses:
+- MIT
+metadata:
+  homepage_uri: https://github.com/cuongnc0211/llm_scraper
+  source_code_uri: https://github.com/cuongnc0211/llm_scraper
+  rubygems_mfa_required: 'true'
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 3.1.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 4.0.3
+specification_version: 4
+summary: Extract structured JSON from web pages using LLMs
+test_files: []