ask-web-fetch 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (8) hide show
  1. checksums.yaml +7 -0
  2. data/LICENSE +21 -0
  3. data/README.md +83 -0
  4. data/lib/ask/web_fetch/tool.rb +122 -0
  5. data/lib/ask/web_fetch/version.rb +7 -0
  6. data/lib/ask/web_fetch.rb +4 -0
  7. data/lib/ask-web-fetch.rb +3 -0
  8. metadata +151 -0
checksums.yaml ADDED
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA256:
3
+ metadata.gz: e750ba3efd9f7947a21e35e92ae5e99d639bc028dd8c2be2a8581707a0fdd908
4
+ data.tar.gz: 5c8d428a526da1e0cc7ac1397adbb73e22eb4714c3576cf2b6ec927bbe5630f8
5
+ SHA512:
6
+ metadata.gz: f770807fc25707aae50934aa6ebe4b2137fea07cda5e4204f9cf34ab441bf3f426a526c61c92edc467e011c32b712d00bf5c948cefafc110bb2f7c9139177725
7
+ data.tar.gz: f6d780234851e1cd40d96ab2f3f75636054e2cff4a13f6591d50caa7c621d09b833021831244e3df90eb4869facae6d50fa664fd8f53386f653a4692b00c1eac
data/LICENSE ADDED
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Kaka Ruto
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
data/README.md ADDED
@@ -0,0 +1,83 @@
1
+ # ask-web-fetch
2
+
3
+ [![Gem Version](https://badge.fury.io/rb/ask-web-fetch.svg)](https://badge.fury.io/rb/ask-web-fetch)
4
+
5
+ A web fetch tool for the ask-rb ecosystem. It provides
6
+ `Ask::Tools::WebFetch`, which fetches a URL and returns its content as clean
7
+ markdown for LLM consumption. It has no Rails dependencies, no external
8
+ service, and no API key required — pure Ruby (`Net::HTTP` + Nokogiri +
9
+ reverse_markdown).
10
+
11
+ ## How it works
12
+
13
+ 1. `Net::HTTP` GET with a browser-like User-Agent (redirects followed, up to 5)
14
+ 2. Nokogiri parses the HTML and picks the main content
15
+ (`<article>` → `<main>` → `<body>`)
16
+ 3. Navigation, scripts, and other chrome are stripped
17
+ 4. reverse_markdown converts the content to markdown (tables become markdown
18
+ tables, links become `[text](url)`)
19
+ 5. Output is truncated to `max_chars` (default 20000)
20
+
21
+ ## Installation
22
+
23
+ ```ruby
24
+ gem "ask-web-fetch"
25
+ ```
26
+
27
+ ## Quick Start
28
+
29
+ ```ruby
30
+ require "ask/web_fetch"
31
+
32
+ tool = Ask::Tools::WebFetch.new
33
+ result = tool.execute(url: "https://www.ruby-lang.org/en/")
34
+ puts result
35
+ ```
36
+
37
+ Results include the page title, source URL, and clean markdown:
38
+
39
+ ```
40
+ # Ruby Programming Language
41
+
42
+ Source: https://www.ruby-lang.org/en/
43
+
44
+ Ruby is a dynamic, open-source programming language with a focus on
45
+ simplicity and productivity...
46
+ ```
47
+
48
+ You can cap the output size:
49
+
50
+ ```ruby
51
+ result = tool.execute(url: "https://example.com", max_chars: 5000)
52
+ ```
53
+
54
+ If the page has no readable content, returns
55
+ `"No readable content found at <url>."`
56
+
57
+ ## Configuration
58
+
59
+ No configuration required. The only knob is the optional `max_chars`
60
+ parameter.
61
+
62
+ ## Known limitations
63
+
64
+ - Pages rendered entirely client-side (JavaScript SPAs) may yield little or
65
+ no content — no JS engine is executed.
66
+ - Some sites block non-browser requests regardless of User-Agent.
67
+
68
+ ## Full documentation
69
+
70
+ The full ask-rb documentation lives at https://ask-rb.github.io/ask-docs.
71
+ [Core: Web Fetch](https://ask-rb.github.io/ask-docs/core/web-fetch) covers
72
+ ask-web-fetch in depth. API reference: https://ask-rb.github.io/ask-docs/reference/api.
73
+
74
+ ## Development
75
+
76
+ ```
77
+ bundle install
78
+ bundle exec rake test
79
+ ```
80
+
81
+ ## License
82
+
83
+ MIT
data/lib/ask/web_fetch/tool.rb ADDED
@@ -0,0 +1,122 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'ask-tools'
4
+ require 'net/http'
5
+ require 'uri'
6
+ require 'nokogiri'
7
+ require 'reverse_markdown'
8
+
9
+ module Ask
10
+ module Tools
11
+ # Fetches a URL and returns its content as clean markdown for LLM
12
+ # consumption. Pure Ruby: Net::HTTP + Nokogiri + reverse_markdown.
13
+ # No external service or API key required.
14
+ class WebFetch < Ask::Tool
15
+ USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) ' \
16
+ 'AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0 Safari/537.36 ' \
17
+ "ask-web-fetch/#{Ask::WebFetch::VERSION}".freeze
18
+ DEFAULT_MAX_CHARS = 20_000
19
+ MAX_REDIRECTS = 5
20
+ OPEN_TIMEOUT = 5
21
+ READ_TIMEOUT = 15
22
+
23
+ # Class/id fragments that mark navigation chrome worth dropping, e.g.
24
+ # "vector-page-toolbar", "sidebar", "toc".
25
+ NAV_CHROME_RE = /
26
+ (^|[\s_-])(nav|menu|toolbar|breadcrumb|sidebar|toc|footer|header|
27
+ banner|pagination|search|cookie|modal|popup)([\s_-]|$)
28
+ /ix
29
+
30
+ description 'Fetch a URL and return its content as clean markdown for LLM consumption. ' \
31
+ 'Use this to read web pages, articles, and documentation.'
32
+
33
+ params(
34
+ type: 'object',
35
+ properties: {
36
+ url: { type: 'string', description: 'The URL to fetch' },
37
+ max_chars: { type: 'integer', description: 'Maximum number of characters to return (default 20000)' }
38
+ },
39
+ required: ['url']
40
+ )
41
+
42
+ def execute(url:, max_chars: DEFAULT_MAX_CHARS)
43
+ body, content_type = fetch(url)
44
+ raise "WebFetch expected HTML from #{url}, got #{content_type}" unless content_type.include?('html')
45
+
46
+ markdown = to_markdown(body, url)
47
+ markdown = truncate(markdown, max_chars) if max_chars&.positive?
48
+ Ask::Result.ok(data: markdown)
49
+ end
50
+
51
+ private
52
+
53
+ # GET with redirect following (max MAX_REDIRECTS hops). Raises on
54
+ # non-2xx responses; Ask::Tool#call wraps exceptions into a failure.
55
+ def fetch(url)
56
+ uri = URI(url)
57
+ hops = 0
58
+ loop do
59
+ http = Net::HTTP.new(uri.host, uri.port)
60
+ http.open_timeout = OPEN_TIMEOUT
61
+ http.read_timeout = READ_TIMEOUT
62
+ http.use_ssl = uri.scheme == 'https'
63
+ req = Net::HTTP::Get.new(uri)
64
+ req['User-Agent'] = USER_AGENT
65
+ req['Accept'] = 'text/html,application/xhtml+xml'
66
+ res = http.request(req)
67
+ return [res.body, res['content-type'].to_s] if res.code.start_with?('2')
68
+
69
+ raise "WebFetch got #{res.code} from #{url}" unless res.code.start_with?('3') && res['location']
70
+ raise "WebFetch hit a redirect loop at #{url}" if (hops += 1) > MAX_REDIRECTS
71
+
72
+ uri = URI.join(uri, res['location'])
73
+ end
74
+ end
75
+
76
+ def to_markdown(html, url)
77
+ doc = Nokogiri::HTML(html)
78
+ candidate = extract_main(doc)
79
+ scrub(candidate)
80
+ markdown = ReverseMarkdown.convert(candidate.to_html, unknown_tags: :bypass, github_flavored: true)
81
+ markdown = clean(markdown)
82
+ return "No readable content found at #{url}." if markdown.empty?
83
+
84
+ header = +''
85
+ title = doc.at('title')&.text&.strip
86
+ header << "# #{title}\n\n" unless title.to_s.empty?
87
+ header << "Source: #{url}\n\n"
88
+ header + markdown
89
+ end
90
+
91
+ def extract_main(doc)
92
+ doc.at('article') || doc.at('main') || doc.at('[role="main"]') || doc.at('body') || doc
93
+ end
94
+
95
+ # Remove chrome INSIDE the candidate only, never ancestors.
96
+ def scrub(candidate)
97
+ candidate.css('script, style, noscript, nav, footer, header, iframe, form, svg, aside').each(&:remove)
98
+ candidate.css('*[id], *[class]').each do |el|
99
+ next if el.equal?(candidate)
100
+
101
+ id_cls = [el['id'], el['class']].compact.join(' ')
102
+ el.remove if id_cls.match?(NAV_CHROME_RE)
103
+ end
104
+ candidate
105
+ end
106
+
107
+ def clean(markdown)
108
+ markdown.gsub(/[ \t]+\n/, "\n")
109
+ .gsub(/\n{3,}/, "\n\n")
110
+ .strip
111
+ end
112
+
113
+ def truncate(markdown, max_chars)
114
+ return markdown if markdown.length <= max_chars
115
+
116
+ "#{markdown[0, max_chars].rstrip}\n\n…(truncated)"
117
+ end
118
+ end
119
+ end
120
+ end
121
+
122
+ Ask::Tools.register(Ask::Tools::WebFetch) if defined?(Ask::Tools)
data/lib/ask/web_fetch/version.rb ADDED
@@ -0,0 +1,7 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Ask
4
+ module WebFetch
5
+ VERSION = '0.1.0'
6
+ end
7
+ end
data/lib/ask/web_fetch.rb ADDED
@@ -0,0 +1,4 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative 'web_fetch/version'
4
+ require_relative 'web_fetch/tool'
data/lib/ask-web-fetch.rb ADDED
@@ -0,0 +1,3 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative 'ask/web_fetch'
metadata ADDED
@@ -0,0 +1,151 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: ask-web-fetch
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.1.0
5
+ platform: ruby
6
+ authors:
7
+ - Kaka Ruto
8
+ bindir: bin
9
+ cert_chain: []
10
+ date: 1980-01-02 00:00:00.000000000 Z
11
+ dependencies:
12
+ - !ruby/object:Gem::Dependency
13
+ name: ask-tools
14
+ requirement: !ruby/object:Gem::Requirement
15
+ requirements:
16
+ - - ">="
17
+ - !ruby/object:Gem::Version
18
+ version: '0.1'
19
+ type: :runtime
20
+ prerelease: false
21
+ version_requirements: !ruby/object:Gem::Requirement
22
+ requirements:
23
+ - - ">="
24
+ - !ruby/object:Gem::Version
25
+ version: '0.1'
26
+ - !ruby/object:Gem::Dependency
27
+ name: nokogiri
28
+ requirement: !ruby/object:Gem::Requirement
29
+ requirements:
30
+ - - ">="
31
+ - !ruby/object:Gem::Version
32
+ version: '1.15'
33
+ type: :runtime
34
+ prerelease: false
35
+ version_requirements: !ruby/object:Gem::Requirement
36
+ requirements:
37
+ - - ">="
38
+ - !ruby/object:Gem::Version
39
+ version: '1.15'
40
+ - !ruby/object:Gem::Dependency
41
+ name: reverse_markdown
42
+ requirement: !ruby/object:Gem::Requirement
43
+ requirements:
44
+ - - ">="
45
+ - !ruby/object:Gem::Version
46
+ version: '2.0'
47
+ type: :runtime
48
+ prerelease: false
49
+ version_requirements: !ruby/object:Gem::Requirement
50
+ requirements:
51
+ - - ">="
52
+ - !ruby/object:Gem::Version
53
+ version: '2.0'
54
+ - !ruby/object:Gem::Dependency
55
+ name: minitest
56
+ requirement: !ruby/object:Gem::Requirement
57
+ requirements:
58
+ - - "~>"
59
+ - !ruby/object:Gem::Version
60
+ version: '5.25'
61
+ type: :development
62
+ prerelease: false
63
+ version_requirements: !ruby/object:Gem::Requirement
64
+ requirements:
65
+ - - "~>"
66
+ - !ruby/object:Gem::Version
67
+ version: '5.25'
68
+ - !ruby/object:Gem::Dependency
69
+ name: rake
70
+ requirement: !ruby/object:Gem::Requirement
71
+ requirements:
72
+ - - "~>"
73
+ - !ruby/object:Gem::Version
74
+ version: '13.0'
75
+ type: :development
76
+ prerelease: false
77
+ version_requirements: !ruby/object:Gem::Requirement
78
+ requirements:
79
+ - - "~>"
80
+ - !ruby/object:Gem::Version
81
+ version: '13.0'
82
+ - !ruby/object:Gem::Dependency
83
+ name: vcr
84
+ requirement: !ruby/object:Gem::Requirement
85
+ requirements:
86
+ - - "~>"
87
+ - !ruby/object:Gem::Version
88
+ version: '6.0'
89
+ type: :development
90
+ prerelease: false
91
+ version_requirements: !ruby/object:Gem::Requirement
92
+ requirements:
93
+ - - "~>"
94
+ - !ruby/object:Gem::Version
95
+ version: '6.0'
96
+ - !ruby/object:Gem::Dependency
97
+ name: webmock
98
+ requirement: !ruby/object:Gem::Requirement
99
+ requirements:
100
+ - - "~>"
101
+ - !ruby/object:Gem::Version
102
+ version: '3.26'
103
+ type: :development
104
+ prerelease: false
105
+ version_requirements: !ruby/object:Gem::Requirement
106
+ requirements:
107
+ - - "~>"
108
+ - !ruby/object:Gem::Version
109
+ version: '3.26'
110
+ description: 'Provides Ask::Tools::WebFetch, a tool that fetches a URL and converts
111
+ its content to clean markdown for LLM consumption. Pure Ruby (Net::HTTP + Nokogiri
112
+ + reverse_markdown): no external service or API key required. Works with any ask-rb
113
+ chat or agent.'
114
+ email:
115
+ - kaka@myrrlabs.com
116
+ executables: []
117
+ extensions: []
118
+ extra_rdoc_files: []
119
+ files:
120
+ - LICENSE
121
+ - README.md
122
+ - lib/ask-web-fetch.rb
123
+ - lib/ask/web_fetch.rb
124
+ - lib/ask/web_fetch/tool.rb
125
+ - lib/ask/web_fetch/version.rb
126
+ homepage: https://github.com/ask-rb/ask-web-fetch
127
+ licenses:
128
+ - MIT
129
+ metadata:
130
+ homepage_uri: https://github.com/ask-rb/ask-web-fetch
131
+ source_code_uri: https://github.com/ask-rb/ask-web-fetch
132
+ changelog_uri: https://github.com/ask-rb/ask-web-fetch/blob/master/CHANGELOG.md
133
+ rubygems_mfa_required: 'true'
134
+ rdoc_options: []
135
+ require_paths:
136
+ - lib
137
+ required_ruby_version: !ruby/object:Gem::Requirement
138
+ requirements:
139
+ - - ">="
140
+ - !ruby/object:Gem::Version
141
+ version: '3.2'
142
+ required_rubygems_version: !ruby/object:Gem::Requirement
143
+ requirements:
144
+ - - ">="
145
+ - !ruby/object:Gem::Version
146
+ version: '0'
147
+ requirements: []
148
+ rubygems_version: 4.0.3
149
+ specification_version: 4
150
+ summary: Web fetch tool for the ask-rb ecosystem
151
+ test_files: []