RubyGems - crawlr - Versions diffs - 0.1.0 - Mend

crawlr 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (21) hide show

checksums.yaml +7 -0
data/.rspec +3 -0
data/.rubocop.yml +9 -0
data/CHANGELOG.md +5 -0
data/LICENSE.txt +21 -0
data/README.md +326 -0
data/Rakefile +12 -0
data/lib/crawlr/callbacks.rb +177 -0
data/lib/crawlr/collector.rb +632 -0
data/lib/crawlr/config.rb +232 -0
data/lib/crawlr/context.rb +80 -0
data/lib/crawlr/domains.rb +166 -0
data/lib/crawlr/hooks.rb +161 -0
data/lib/crawlr/http_interface.rb +286 -0
data/lib/crawlr/parser.rb +242 -0
data/lib/crawlr/robots.rb +329 -0
data/lib/crawlr/version.rb +5 -0
data/lib/crawlr/visits.rb +190 -0
data/lib/crawlr.rb +16 -0
data/sig/crawlr.rbs +4 -0
metadata +209 -0

metadata ADDED Viewed

@@ -0,0 +1,209 @@
+--- !ruby/object:Gem::Specification
+name: crawlr
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Aristotelis Rapai
+bindir: exe
+cert_chain: []
+date: 2025-09-29 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: async
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.32'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.32'
+- !ruby/object:Gem::Dependency
+  name: async-http
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.91.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.91.0
+- !ruby/object:Gem::Dependency
+  name: concurrent-ruby
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
+- !ruby/object:Gem::Dependency
+  name: http-cookie
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.18'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.18'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '13.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '13.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: rubocop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.60'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.60'
+- !ruby/object:Gem::Dependency
+  name: ruby-prof
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.7'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.7'
+- !ruby/object:Gem::Dependency
+  name: webmock
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.25'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.25'
+description: |
+  Crawlr is a modern Ruby web scraping framework built for respectful and efficient data extraction.
+  Features async HTTP requests, robots.txt compliance, cookie management, proxy rotation, flexible
+  CSS/XPath selectors, and comprehensive error handling. Designed for both simple scraping tasks
+  and complex, large-scale data extraction projects.
+email:
+- aristorap.dev@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".rspec"
+- ".rubocop.yml"
+- CHANGELOG.md
+- LICENSE.txt
+- README.md
+- Rakefile
+- lib/crawlr.rb
+- lib/crawlr/callbacks.rb
+- lib/crawlr/collector.rb
+- lib/crawlr/config.rb
+- lib/crawlr/context.rb
+- lib/crawlr/domains.rb
+- lib/crawlr/hooks.rb
+- lib/crawlr/http_interface.rb
+- lib/crawlr/parser.rb
+- lib/crawlr/robots.rb
+- lib/crawlr/version.rb
+- lib/crawlr/visits.rb
+- sig/crawlr.rbs
+homepage: https://github.com/aristorap/crawlr
+licenses:
+- MIT
+metadata:
+  allowed_push_host: https://rubygems.org
+  homepage_uri: https://github.com/aristorap/crawlr
+  source_code_uri: https://github.com/aristorap/crawlr
+  changelog_uri: https://github.com/aristorap/crawlr/blob/main/CHANGELOG.md
+  bug_tracker_uri: https://github.com/aristorap/crawlr/issues
+  rubygems_mfa_required: 'true'
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 3.1.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.6.3
+specification_version: 4
+summary: A powerful, async Ruby web scraping framework
+test_files: []