meta_information 1.0.0 → 1.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 1f66071a06238dc6aac14bf94b9c7308c744d87f
-  data.tar.gz: 96dc74d10a6542551ed70c7e891d9932b4930ca2
+SHA256:
+  metadata.gz: a986aa80d954fde7cd758f23b62fa4bc6c6ef564ab06a86b55f8b4980739812e
+  data.tar.gz: f2d878d8d1c57665677608e71e6d5f75dcfccf256ea4b625750695bab8740ca1
 SHA512:
-  metadata.gz: 670b94bc7fdae7a68eed412e51ea0f8648eeab931ff8b6efafde8522694911959d77ae92c779d5e5d5ce77bf5985f3c70c8276fbf16eda1df97fdde6170de4bb
-  data.tar.gz: c83c405927fd5b265f1a8cf7fe18cd1b7fffc013f4cdfad091e5aca505a8b758c56980cc476a4c5725b47341c67f12ac335677178efe2b1a5b8b5338beecda7b
+  metadata.gz: 00ad8369eb9559cb958b4fd2443c5e09d42a2c03d0ed1d4425ca7247f02d2abc90b66ecb4884a6d11faed122d74a812dbb87e36a273c2c0e8bad63f8dfbd1c87
+  data.tar.gz: 32798218b16280ca2cc0e7f58659d04f8b91853935a00646dade439b85e927305cc2a2f4d8e1bc8d5c0ff72cebfc22c29b45a1d18c753b70102ee13db6ec0d54

data/.rspec

@@ -0,0 +1 @@
+--color

data/CHANGELOG.md

@@ -0,0 +1,6 @@
+## Changelog
+
+### 1.1.0
+
+- Added rubocop
+- Replaced open-uri by net/http

data/Gemfile

@@ -1,5 +1,12 @@
+# frozen_string_literal: true
+
 source 'https://rubygems.org'
-ruby '2.3.3'
+ruby ['>= 2.1.0', '<= 2.7.0']
 
 gem 'nokogiri'
-gem 'rspec'
+
+group :development, :test do
+  gem 'pry'
+  gem 'rspec'
+  gem 'rubocop'
+end

data/README.md

@@ -1,16 +1,26 @@
 # MetaInformation
+
+[![Gem Version](https://badge.fury.io/rb/meta_information.svg)](https://badge.fury.io/rb/meta_information)
+
 Simple gem for parsing meta information from websites. It scan all meta-tags by name or property attributes.
+
 ## Instalation
+
 Add this line to your application's Gemfile:
+
 ```ruby
 gem 'meta_information'
 ```
+
 Then run `bundle install`
 Or install it yourself as:
+
 ```sh
 gem install meta_information
 ```
+
 ## Usage
+
 ```ruby
 require 'pp'
 meta = MetaInformation.get_meta('https://www.awesome_site.com/awesome_page')
@@ -21,15 +31,20 @@ pp meta
 # :all_meta=>
 #  [{:type=>"name",
 #    :name=>"viewport",
+#    :property=>nil,
 #    :content=>"width=device-width, initial-scale=1.0"},
-#   {:type=>"name", :name=>"description", :content=>"some description"},
-#   {:type=>"name", :name=>"title", :content=>"i am title"},
-#   {:type=>"name", :name=>"og:title", :content=>"some content"},
-#   {:type=>"name", :name=>"og:description", :content=>"some description"},
+#   {:type=>"name", :name=>"description", :property=>nil, :content=>"some description"},
+#   {:type=>"name", :name=>"title", :property=>nil, :content=>"i am title"},
+#   {:type=>"name", :name=>"og:title", :property=>nil, :content=>"some content"},
+#   {:type=>"name", :name=>"og:description", :property=>nil, :content=>"some description"},
 #   {:type=>"name",
 #    :name=>"og:image",
-#    :content=> "https://www.awesome_site.com/assets/awesome_picture.jpg"}]}
+#    :property=>nil,
+#    :content=> "https://www.awesome_site.com/assets/awesome_picture.jpg"}]},
+#   {:type=>"property", :name=>nil, :property=>"fb:app_id", :content=>"1234567890"},
 ###
 ```
+
 ## License
-MIT License.
+
+MIT License.

data/ROADMAP.md

@@ -0,0 +1,3 @@
+## ROADMAP
+
+- Instead of returing hash, module `MetaInformation` should return struct object as a result. Each node should be a struct with methods to recognize a tag such as `.twitter?`, `.og?`, `.fb?`, '.vk?' (check it by property attr).

data/lib/meta_information.rb

@@ -1,74 +1,101 @@
-require './lib/meta_information/version'
+# frozen_string_literal: true
+
+require_relative 'meta_information/version'
 require 'nokogiri'
-require 'open-uri'
+require 'uri'
+require 'net/http'
 
-# MetaInformation - module for scaning meta information
-# form web page
-# for usage
+# MetaInformation - module for scaning meta information from web page
 # MetaInformation.get_meta('https://some_site.com/some_page')
 module MetaInformation
-  class << self
-    def get_meta(input_url)
-      return not_valid_url_error unless valid_url?(input_url)
+  extend self
 
-      document = create_document(input_url)
-      return nokogiri_error if document == false
+  def get_meta(input_url)
+    return not_valid_url_error unless valid_url?(input_url)
+    return not_valid_url_scheme unless valid_url_scheme?(input_url)
 
-      meta_hash = create_meta_array(document)
-      success_hash.merge(all_meta: meta_hash)
-    end
+    document = create_document(input_url)
+    return nokogiri_error if document == false
+
+    meta_hash = create_meta_array(document)
+    success_hash.merge(all_meta: meta_hash)
+  end
+
+  private
 
-    private
-
-    def create_meta_array(document)
-      array = []
-      document.css('meta').each do |node|
-        if !node['name'].nil?
-          array.push(
-            type: 'name',
-            name: node['name'],
-            content: node['content']
-          )
-        elsif !node['property'].nil?
-          array.push(
-            type: 'property',
-            property: node['property'],
-            content: node['content']
-          )
-        end
+  # TODO: change to struct
+  def create_meta_array(document)
+    document
+      .css('meta').reject { |node| node_type(node).nil? }
+      .map do |node|
+        {
+          type: node_type(node),
+          name: node['name'],
+          property: node['property'],
+          content: node['content'],
+          itemprop: node['itemprop']
+        }
       end
-      array
-    end
+  end
 
-    def valid_url?(uri)
-      !!uri.match(/^(https?:\/\/)?([\w\.]+)\.([a-z]{2,6}\.?)(\/[\w\.]*)*\/?$/)
+  def node_type(node)
+    if !node['name'].nil?
+      'name'
+    elsif !node['property'].nil?
+      'property'
+    elsif !node['itemprop'].nil?
+      'itemprop'
+    else
+      nil
     end
+  end
 
-    def create_document(input_url)
-      Nokogiri::HTML(open(input_url))
-    rescue
-      false
-    end
+  def valid_url?(uri)
+    !(uri =~ URI::DEFAULT_PARSER.make_regexp).nil?
+  end
 
-    def not_valid_url_error
-      {
-        success: false,
-        error: 'url is not valid'
-      }
-    end
+  def valid_url_scheme?(input_url)
+    URI(input_url).is_a?(URI::HTTP)
+  end
 
-    def nokogiri_error
-      {
-        success: false,
-        error: 'error with parsing a document'
-      }
-    end
+  def create_document(input_url)
+    uri = URI(input_url)
+    res = Net::HTTP.get_response(uri)
 
-    def success_hash
-      {
-        succes: 'true',
-        error: ''
-      }
-    end
+    raise 'Response code is not 2xx' if !(res.code.to_i >= 200 && res.code.to_i <= 299)
+    raise 'Response is without body' unless res.class.body_permitted?
+
+    Nokogiri::HTML(res.body)
+  rescue StandardError => e
+    puts e
+    false
+  end
+
+  def not_valid_url_error
+    {
+      success: false,
+      error: 'url is not valid'
+    }
+  end
+
+  def not_valid_url_scheme
+    {
+      success: false,
+      error: 'url must be http(s)'
+    }
+  end
+
+  def nokogiri_error
+    {
+      success: false,
+      error: 'error with parsing a document'
+    }
+  end
+
+  def success_hash
+    {
+      succes: 'true',
+      error: ''
+    }
   end
 end

data/lib/meta_information/version.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module MetaInformation
-  VERSION = '1.0.0'
-end
+  VERSION = '1.1.0'
+end

data/meta_information.gemspec

@@ -1,15 +1,25 @@
-$:.push File.expand_path("../lib", __FILE__)
-require "meta_information/version"
+# frozen_string_literal: true
+
+$LOAD_PATH.push File.expand_path('lib', __dir__)
+require 'meta_information/version'
 
 Gem::Specification.new do |s|
+  s.required_ruby_version = '>= 2.1.0'
   s.name        = 'meta_information'
   s.version     = MetaInformation::VERSION
-  s.date        = '2017-02-26'
+  s.date        = '2021-02-12'
   s.summary     = 'MetaInformation - Simple gem for parsing meta information'
-  s.description = 'Simple gem for parsing meta information from websites. It scan all meta-tags by name or property attributes.'
+  s.description = 'Simple gem for parsing meta information from websites. It scans all meta-tags by name, itemprop or property attributes.'
   s.author      = 'Vladislav Kopylov'
   s.email       = 'kopylov.vlad@gmail.com'
-  s.files       = `git ls-files`.split("\n")
+  s.files       = `git ls-files -z`.split("\x0").reject { |f| f =~ /^bin/ }
+  s.executables = []
   s.homepage    = 'https://github.com/kopylovvlad/meta_information'
   s.license     = 'MIT'
-end
+
+  s.add_dependency('nokogiri')
+
+  s.add_development_dependency('pry')
+  s.add_development_dependency('rspec')
+  s.add_development_dependency('rubocop')
+end

data/spec/config.rb

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+
+RSpec.configure do |rspec|
+  rspec.shared_context_metadata_behavior = :apply_to_host_groups
+  rspec.include_context 'shared stuff', include_shared: true
+end

data/spec/lib/meta_information_spec.rb

@@ -1,38 +1,77 @@
+# frozen_string_literal: true
+
 require './lib/meta_information'
+require './spec/shared_stuff'
+require './spec/config'
 
 RSpec.describe 'MetaInformation' do
-  
+  include_context 'shared stuff'
+
+  describe 'get_meta' do
+    it 'should return not_valid_url_error' do
+      allow(MetaInformation).to receive(:valid_url?).and_return(false)
+      result = MetaInformation.get_meta('http://some_url.com')
+      expect(result).to(
+        eq(
+          success: false,
+          error: 'url is not valid'
+        )
+      )
+    end
+
+    it 'should return not_valid_url_scheme_error' do
+      result = MetaInformation.get_meta('ftp://some_url.com')
+      expect(result).to(
+        eq(success: false, error: 'url must be http(s)')
+      )
+    end
+
+    describe 'with mock for valid_url?' do
+      before do
+        allow(MetaInformation).to receive(:valid_url?).and_return(true)
+      end
+      it 'should return nokogiri_error' do
+        allow(MetaInformation).to receive(:create_document).and_return(false)
+
+        result = MetaInformation.get_meta('http://some_url.com')
+        expect(result).to(
+          eq(
+            success: false,
+            error: 'error with parsing a document'
+          )
+        )
+      end
+
+      it 'should return success_hash' do
+        allow(MetaInformation).to(
+          receive(:create_document).and_return(
+            Nokogiri::HTML(default_html)
+          )
+        )
+
+        result = MetaInformation.get_meta('http://some_url.com')
+        expect(result).to(
+          eq(
+            succes: 'true',
+            error: '',
+            all_meta: default_html_meta_array
+          )
+        )
+      end
+    end
+  end
+
   describe 'create_meta_array' do
     describe 'we have meta' do
       it 'must create array' do
-        document = Nokogiri::HTML('
-          <html>
-            <meta name="description" content="" />
-            <meta name="title" content="some title" />
-            <meta property="author" content="Bob" />
-            <meta property="og:title" content="og_title" />
-            <meta property="twitter:image" content="http://some_host.com/some_path" />
-            <meta property="og:locale" content="ru_RU" />
-            <meta property="al:ios:app_store_id" content="12345678900" />
-            <body>
-              <h1>Mr. Belvedere Fan Club</h1>
-            </body>
-          </html>
-        ')
-        expect(MetaInformation.send(:create_meta_array, document)).to eq([
-          { type: 'name', name: 'description', content: '' },
-          { type: 'name', name: 'title', content: 'some title' },
-          { type: 'property', property: 'author', content: 'Bob' },
-          { type: 'property', property: 'og:title', content: 'og_title' },
-          { type: 'property', property: 'twitter:image', content: 'http://some_host.com/some_path' },
-          { type: 'property', property: 'og:locale', content: 'ru_RU' },
-          { type: 'property', property: 'al:ios:app_store_id', content: '12345678900' }
-        ])
+        document = Nokogiri::HTML(default_html)
+        result = MetaInformation.send(:create_meta_array, document)
+        expect(result).to eq(default_html_meta_array)
       end
     end
-    
-    describe 'without meta' do
-      it 'has empty array' do
+
+    describe 'without meta has empty array' do
+      it 'if have not mate' do
         first_document = Nokogiri::HTML('
           <html>
             <body>
@@ -40,51 +79,95 @@ RSpec.describe 'MetaInformation' do
             </body>
           </html>
         ')
-        second_document = Nokogiri::HTML('')
         expect(MetaInformation.send(:create_meta_array, first_document)).to eq([])
+      end
+
+      it 'if html is empty' do
+        second_document = Nokogiri::HTML('')
         expect(MetaInformation.send(:create_meta_array, second_document)).to eq([])
       end
     end
   end
 
   describe 'valid_url?' do
-    it 'must be valid' do
-      expect(MetaInformation.send(:valid_url?, 'http://www.somesite.com')).to be_truthy
-      expect(MetaInformation.send(:valid_url?, 'https://www.somesite.com')).to be_truthy
-      expect(MetaInformation.send(:valid_url?, 'https://somesite.com')).to be_truthy
-      expect(MetaInformation.send(:valid_url?, 'wwwsome_site.ru')).to be_truthy
-      expect(MetaInformation.send(:valid_url?, 'https://somesite.com/some_page')).to be_truthy
-      expect(MetaInformation.send(:valid_url?, 'https://somesite.com/some_page/page')).to be_truthy
-      expect(MetaInformation.send(:valid_url?, 'https://somesite.com.uk/some_page')).to be_truthy
+    def self.validate_valid_url(url)
+      it "#{url} must be valid" do
+        expect(MetaInformation.send(:valid_url?, url)).to be_truthy
+      end
     end
-    
-    it 'does not valid' do
-      expect(MetaInformation.send(:valid_url?, 'some_site')).to be_falsey
-      expect(MetaInformation.send(:valid_url?, 'http\\:wwwsome_site.ru')).to be_falsey
-      expect(MetaInformation.send(:valid_url?, 'com.some_site')).to be_falsey
+
+    validate_valid_url('http://www.somesite.com')
+    validate_valid_url('https://www.somesite.com')
+    validate_valid_url('https://somesite.com')
+    validate_valid_url('http://www.siteforadmin.ru')
+    validate_valid_url('https://somesite.com/some_page')
+    validate_valid_url('https://somesite.com/some_page/page')
+    validate_valid_url('http://somesite.com/some_page/2012/12/page/another_page')
+    validate_valid_url('http://somesite.com/2012/12/page-page-page')
+    validate_valid_url('https://somesite.com.uk/some_page')
+    validate_valid_url('https://meduza.io/short/2017/03/25/v-londone-proshel-mnogotysyachnyy-marsh-protiv-brekzita-fotografiya')
+
+    def self.validate_invalid_url(url)
+      it "#{url} must be invalid" do
+        expect(MetaInformation.send(:valid_url?, url)).to be_falsey
+      end
     end
+
+    validate_invalid_url('some_site')
+    validate_invalid_url('http\\:wwwsome_site.ru')
+    validate_invalid_url('com.some_site')
   end
-  
+
   describe 'private hash equal' do
     it 'not_valid_url_error hash' do
       expect(MetaInformation.send(:not_valid_url_error)).to eq({
-        success: false,
-        error: 'url is not valid'
-      })
+                                                                 success: false,
+                                                                 error: 'url is not valid'
+                                                               })
     end
 
     it 'nokogiri_error hash' do
       expect(MetaInformation.send(:nokogiri_error)).to eq({
-        success: false,
-        error: 'error with parsing a document'
-      })
+                                                            success: false,
+                                                            error: 'error with parsing a document'
+                                                          })
     end
 
     it 'success_hash hash' do
       expect(MetaInformation.send(:success_hash)).to eq({
-        succes: 'true',
-        error: ''
-      })
+                                                          succes: 'true',
+                                                          error: ''
+                                                        })
+    end
+  end
+
+  describe 'node_type' do
+    it 'must return name' do
+      document = Nokogiri::HTML('<meta name="description" content="" />')
+      node = document.css('meta').first
+      expect(MetaInformation.send(:node_type, node)).to eq('name')
+    end
+
+    it 'must return property' do
+      document = Nokogiri::HTML(
+        '<meta property="og:title" content="og_title" />'
+      )
+      node = document.css('meta').first
+      expect(MetaInformation.send(:node_type, node)).to eq('property')
+    end
+
+    it 'must return property' do
+      document = Nokogiri::HTML(
+        '<meta itemprop="description" content="description" />'
+      )
+      node = document.css('meta').first
+      expect(MetaInformation.send(:node_type, node)).to eq('itemprop')
+    end
+
+    it 'must return nil' do
+      document = Nokogiri::HTML('<meta content="og_title" />')
+      node = document.css('meta').first
+      expect(MetaInformation.send(:node_type, node)).to eq(nil)
     end
   end
 end

data/spec/shared_stuff.rb

@@ -0,0 +1,90 @@
+# frozen_string_literal: true
+
+RSpec.shared_context 'shared stuff', shared_context: :metadata do
+  let(:default_html) do
+    '
+      <html>
+        <meta name="description" content="" />
+        <meta name="title" content="some title" />
+        <meta property="author" content="Bob" />
+        <meta property="og:title" content="og_title" />
+        <meta property="twitter:image" content="http://some_host.com/some_path" />
+        <meta property="og:locale" content="ru_RU" />
+        <meta property="al:ios:app_store_id" content="12345678900" />
+        <meta itemprop="name" content="site name"/>
+        <meta itemprop="description" content="site description"/>
+        <body>
+          <h1>Mr. Belvedere Fan Club</h1>
+        </body>
+      </html>
+    '
+  end
+
+  let(:default_html_meta_array) do
+    [
+      {
+        type: 'name',
+        name: 'description',
+        property: nil,
+        itemprop: nil,
+        content: ''
+      },
+      {
+        type: 'name',
+        name: 'title',
+        property: nil,
+        itemprop: nil,
+        content: 'some title'
+      },
+      {
+        type: 'property',
+        name: nil,
+        property: 'author',
+        itemprop: nil,
+        content: 'Bob'
+      },
+      {
+        type: 'property',
+        name: nil,
+        property: 'og:title',
+        itemprop: nil,
+        content: 'og_title'
+      },
+      {
+        type: 'property',
+        name: nil,
+        property: 'twitter:image',
+        itemprop: nil,
+        content: 'http://some_host.com/some_path'
+      },
+      {
+        type: 'property',
+        name: nil,
+        property: 'og:locale',
+        itemprop: nil,
+        content: 'ru_RU'
+      },
+      {
+        type: 'property',
+        name: nil,
+        property: 'al:ios:app_store_id',
+        itemprop: nil,
+        content: '12345678900'
+      },
+      {
+        type: 'itemprop',
+        name: nil,
+        property: nil,
+        itemprop: 'name',
+        content: 'site name'
+      },
+      {
+        type: 'itemprop',
+        name: nil,
+        property: nil,
+        itemprop: 'description',
+        content: 'site description'
+      }
+    ]
+  end
+end

metadata

@@ -1,31 +1,91 @@
 --- !ruby/object:Gem::Specification
 name: meta_information
 version: !ruby/object:Gem::Version
-  version: 1.0.0
+  version: 1.1.0
 platform: ruby
 authors:
 - Vladislav Kopylov
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-02-26 00:00:00.000000000 Z
-dependencies: []
-description: Simple gem for parsing meta information from websites. It scan all meta-tags
-  by name or property attributes.
+date: 2021-02-12 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rubocop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Simple gem for parsing meta information from websites. It scans all meta-tags
+  by name, itemprop or property attributes.
 email: kopylov.vlad@gmail.com
 executables: []
 extensions: []
 extra_rdoc_files: []
 files:
 - ".gitignore"
-- ".ruby-version"
+- ".rspec"
+- CHANGELOG.md
 - Gemfile
 - LICENSE.txt
 - README.md
+- ROADMAP.md
 - lib/meta_information.rb
 - lib/meta_information/version.rb
 - meta_information.gemspec
+- spec/config.rb
 - spec/lib/meta_information_spec.rb
+- spec/shared_stuff.rb
 homepage: https://github.com/kopylovvlad/meta_information
 licenses:
 - MIT
@@ -38,15 +98,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 2.1.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.6.10
+rubygems_version: 3.1.2
 signing_key: 
 specification_version: 4
 summary: MetaInformation - Simple gem for parsing meta information

data/.ruby-version

@@ -1 +0,0 @@
-ruby-2.3.3