google_robotstxt_parser 0.0.3

data/ext/robotstxt/robotstxt.cc

@@ -0,0 +1,32 @@
+#include <ruby.h>
+#include "robots.h"
+#include <string>
+
+typedef VALUE(ruby_method)(...);
+
+extern "C"
+{
+  static VALUE m_Robotstxt;
+
+  static VALUE robotstxt_allowed_by_robots(VALUE self, VALUE robots_content, VALUE user_agent, VALUE url)
+  {
+    Check_Type(robots_content, T_STRING);
+    Check_Type(user_agent, T_STRING);
+    Check_Type(url, T_STRING);
+
+    std::string rc = RSTRING_PTR(robots_content);
+    std::string ua = RSTRING_PTR(user_agent);
+    std::string ur = RSTRING_PTR(url);
+
+    googlebot::RobotsMatcher matcher;
+    bool result = matcher.OneAgentAllowedByRobots(rc, ua, ur);
+    return result ? Qtrue : Qfalse;
+  }
+
+  void Init_robotstxt()
+  {
+    m_Robotstxt = rb_define_module("Robotstxt");
+
+    rb_define_method(m_Robotstxt, "allowed_by_robots", (ruby_method *)&robotstxt_allowed_by_robots, 3);
+  }
+}

data/google_robotstxt_parser.gemspec

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+lib = File.expand_path('lib', __dir__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+
+require 'google_robotstxt_parser/version'
+
+Gem::Specification.new do |s|
+  s.name        = 'google_robotstxt_parser'
+  s.version     = GoogleRobotstxtParser::VERSION
+  s.date        = '2019-11-25'
+  s.summary     = 'Ruby gem wrapper around Google Robotstxt Parser library'
+  s.description = 'This is a unofficial Ruby gem that provides a wrapper around [Google Robotstxt Parser C++ library]'
+  s.authors     = ['Bastien Montois']
+  s.email       = 'bastien.montois@la-revanche-des-sites.fr'
+  s.files       = Dir['lib/**/*.rb']
+  s.homepage    = 'https://github.com/larevanchedessites/google-robotstxt-ruby'
+  s.license     = 'MIT'
+
+  # normal spec stuff above
+  s.files = `git ls-files`.split("\n")
+
+  # get an array of submodule dirs by executing 'pwd' inside each submodule
+  gem_dir = File.expand_path(File.dirname(__FILE__)) + "/"
+  `git submodule --quiet foreach pwd`.split($\).each do |submodule_path|
+    Dir.chdir(submodule_path) do
+      submodule_relative_path = submodule_path.sub gem_dir, ""
+      # issue git ls-files in submodule's directory and
+      # prepend the submodule path to create absolute file paths
+      `git ls-files`.split($\).each do |filename|
+        s.files << "#{submodule_relative_path}/#{filename}"
+      end
+    end
+  end
+
+  s.require_paths = %w[lib ext]
+  s.extensions = ['ext/robotstxt/extconf.rb']
+
+  s.add_development_dependency 'bundler', '~> 2.0'
+  s.add_development_dependency 'rake', '~> 10.0'
+  s.add_development_dependency 'rake-compiler', '~> 1.0'
+
+  s.add_development_dependency 'guard-rspec', '~> 4.7'
+  s.add_development_dependency 'rspec', '~> 3.0'
+end

data/lib/google_robotstxt_parser/version.rb

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+
+module GoogleRobotstxtParser
+  VERSION = '0.0.3'
+  GOOGLE_ROBOTSTXT_MAJOR_VERSION = 1
+end

data/lib/google_robotstxt_parser.rb

@@ -0,0 +1,4 @@
+# frozen_string_literal: true
+
+module GoogleRobotstxtParser
+end

data/spec/google_robotstxt_parser_spec.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+require 'robotstxt'
+
+RSpec.describe GoogleRobotstxtParser do
+  include Robotstxt
+
+  let(:context) { GoogleRobotstxtParser.create }
+
+  describe '.VERSION' do
+    it 'returns a string' do
+      expect(GoogleRobotstxtParser::VERSION).to be_instance_of String
+    end
+  end
+
+  describe 'allowed_by_robots' do
+    it 'should return the true' do
+      robotstxt_content = "# robotstxt.org/\n\nUser-agent: *\nDisallow: \n\nSitemap: https://www.bqst.fr/sitemap.xml"
+      user_agent = 'GoogleBot'
+      url = 'https://www.bqst.fr'
+
+      expect(Robotstxt.allowed_by_robots(robotstxt_content, user_agent, url)).to eq(true)
+    end
+
+    it 'should return the true' do
+      robotstxt_content = "# robotstxt.org/\n\nUser-agent: *\nDisallow: /\n\nSitemap: https://www.bqst.fr/sitemap.xml"
+      user_agent = 'GoogleBot'
+      url = 'https://www.bqst.fr'
+
+      expect(Robotstxt.allowed_by_robots(robotstxt_content, user_agent, url)).to eq(false)
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+require 'simplecov'
+SimpleCov.start
+
+require 'bundler/setup'
+require 'sound_io'
+
+RSpec.configure do |config|
+  # Enable flags like --only-failures and --next-failure
+  config.example_status_persistence_file_path = '.rspec_status'
+
+  # Disable RSpec exposing methods globally on `Module` and `main`
+  config.disable_monkey_patching!
+
+  config.expect_with :rspec do |c|
+    c.syntax = :expect
+  end
+end

metadata

@@ -0,0 +1,146 @@
+--- !ruby/object:Gem::Specification
+name: google_robotstxt_parser
+version: !ruby/object:Gem::Version
+  version: 0.0.3
+platform: ruby
+authors:
+- Bastien Montois
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2019-11-25 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: rake-compiler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  name: guard-rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.7'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.7'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+description: This is a unofficial Ruby gem that provides a wrapper around [Google
+  Robotstxt Parser C++ library]
+email: bastien.montois@la-revanche-des-sites.fr
+executables: []
+extensions:
+- ext/robotstxt/extconf.rb
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".gitmodules"
+- CHANGELOG.md
+- CODE_OF_CONDUCT.md
+- Gemfile
+- Guardfile
+- LICENSE
+- README.md
+- Rakefile
+- ext/robotstxt/.DS_Store
+- ext/robotstxt/extconf.rb
+- ext/robotstxt/robotstxt.cc
+- ext/robotstxt/robotstxt/.gitignore
+- ext/robotstxt/robotstxt/BUILD
+- ext/robotstxt/robotstxt/CMakeLists.txt
+- ext/robotstxt/robotstxt/CMakeLists.txt.in
+- ext/robotstxt/robotstxt/CONTRIBUTING.md
+- ext/robotstxt/robotstxt/LICENSE
+- ext/robotstxt/robotstxt/README.md
+- ext/robotstxt/robotstxt/WORKSPACE
+- ext/robotstxt/robotstxt/protocol-draft/README.md
+- ext/robotstxt/robotstxt/protocol-draft/draft-koster-rep-00.txt
+- ext/robotstxt/robotstxt/robots.cc
+- ext/robotstxt/robotstxt/robots.h
+- ext/robotstxt/robotstxt/robots_main.cc
+- ext/robotstxt/robotstxt/robots_test.cc
+- google_robotstxt_parser.gemspec
+- lib/google_robotstxt_parser.rb
+- lib/google_robotstxt_parser/version.rb
+- spec/google_robotstxt_parser_spec.rb
+- spec/spec_helper.rb
+homepage: https://github.com/larevanchedessites/google-robotstxt-ruby
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+- ext
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.0.4
+signing_key: 
+specification_version: 4
+summary: Ruby gem wrapper around Google Robotstxt Parser library
+test_files: []