ReadRobotstxt 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: eb26a56e7fd51ba2787f82bc6aae192ccc5c3b26ffa9c409081b0ad05465a630
+  data.tar.gz: 2466cfcb919e02c5f429685862eec578339cbbd8383543cd372cc8772507b2bb
+SHA512:
+  metadata.gz: 883441daa58f6cafdd058ff4906501783b589ae4d2979b1ca510e0daa4f48dfaa9b43bb1d1df7deb5a5ddb984d796bb77ffe960d194d0f374d5f1aefda928594
+  data.tar.gz: edc78bd52329da3e8f8038146252669e72231527b7423e806d995e5f96f42e241931ebb93b3170dab5c68506c4a4a11e5ce293f23ce12ab15c005c43fd92e3aa

data/README.md

@@ -0,0 +1,34 @@
+# Robots
+Get robots.txt information from a site. 
+
+## Installation
+
+```ruby
+gem install Robots-txt
+```
+
+```bash
+bundle add Robots-txt
+```
+
+If bundler is not being used to manage dependencies, install the gem by executing:
+
+```bash
+gem install Robots-txt
+```
+
+## Usage
+
+```ruby
+require './lib/robots'
+
+u = Robots::Url.new('https://www.ebay.com')
+u.allow
+u.disallow
+u.disallow_removed
+u.allow_removed
+```
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/[USERNAME]/Robots.

data/Rakefile

@@ -0,0 +1,4 @@
+# frozen_string_literal: true
+
+require 'bundler/gem_tasks'
+task default: %i[]

data/lib/Robots/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module Robots
+  VERSION = '0.1.0'
+end

data/lib/Robots.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+require 'net/http'
+require 'uri'
+require_relative 'Robots/version'
+
+module Robots
+  class Url
+    attr_reader :url
+
+    def initialize(url)
+      @url = url
+      puts "kkk"
+      uri = URI(File.join(@url, 'robots.txt'))
+      @response = Net::HTTP.get_response(uri).body.split("\n").map(&:to_s)
+    end
+
+    def disallow
+      for disallow in @response
+        puts disallow
+        if /(Disallow:)/ =~ disallow
+          puts "#{disallow}"
+        end
+      end
+    end
+
+    def allow
+      for allow in @response
+        if /(Allow:)/ =~ allow
+          puts "#{allow}"
+        end
+      end
+    end
+
+    def allow_removed
+      for allow in @response
+        if /(Allow:)/ =~ allow
+          puts "#{allow.gsub('Allow: ', '')}"
+        end
+      end
+    end
+
+    def disallow_removed
+      for disallow in @response
+        if /(Disallow:)/ =~ disallow
+          puts "#{disallow.gsub('Disallow: ', '')}"
+        end
+      end
+    end
+  end
+end

data/sig/Robots.rbs

@@ -0,0 +1,4 @@
+module Robots
+  VERSION: String
+  # See the writing guide of rbs: https://github.com/ruby/rbs#guides
+end

metadata

@@ -0,0 +1,47 @@
+--- !ruby/object:Gem::Specification
+name: ReadRobotstxt
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Michael-Meade
+autorequire:
+bindir: exe
+cert_chain: []
+date: 2025-10-28 00:00:00.000000000 Z
+dependencies: []
+description: Read Robots.txt files
+email:
+- nowaylol@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- README.md
+- Rakefile
+- lib/Robots.rb
+- lib/Robots/version.rb
+- sig/Robots.rbs
+homepage:
+licenses: []
+metadata: {}
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 3.2.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.4.20
+signing_key:
+specification_version: 4
+summary: Read Robots.txt files
+test_files: []