hertools 0.1.0 → 0.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 20f15df456f2d19d57142b6e67310c78cf4d7a80a08d4f3e17ca07764e7bb38f
-  data.tar.gz: 0e0ac2aff27cc5303d8a3ed93e44c075c6bad37cba541f00565eeb00f4f3cb85
+  metadata.gz: '05092964905a17f8eeefae0d133e2ea5ba85db714759ccc42d173b4ec8a5dbbb'
+  data.tar.gz: 1f0a3befe399d98d0c0cccb543a92e6b1674675e898a7c4a437ae7b91e9998c6
 SHA512:
-  metadata.gz: 14a9a96095ad1a146abb92cb5c230366639cb2a4e490822ca8bedd9e6311c9ded6932d500c6b8b6025916d9c874dcef7fb2330c6b3b615cdd122c50990c83b0f
-  data.tar.gz: ab279755ab77ff157d39a9d6ae5317837906c8bf81ecd3d95ee7d92580fe1cafec49f96198ba28f4728082ac76f9526854a0c838a9cf0f47e3e33a29ea526034
+  metadata.gz: 43a337be463c80fd54caba8f05feb63f638f2208a671c45827ce331114f1e40209a68d353feebb302115f91721201bd818b783d628a77b7c1d208725922a4bcc
+  data.tar.gz: 7d0824dc195b48c560009d1a1dba29fc2a479b2cf6a9d8327d769005109696fce926f8184370d6c32ced34adaeb24fdc20051ef84fc63bcdf18d3b9df006f035

data/lib/hertools/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Hertools
-  VERSION = '0.1.0'
+  VERSION = '0.1.1'
 end

data/lib/hertools/website_parser.rb

@@ -2,6 +2,7 @@
 
 require 'base64'
 require 'open-uri'
+require 'net/http'
 require 'httparty'
 require 'htmlentities'
 require 'nokogiri'
@@ -13,7 +14,7 @@ module Hertools
     # Arguments
     # url: the url of a webpage
     # options:
-    #   html_parser: %w[httparty nokogiri]
+    #   html_parser: %w[httparty nokogiri net_http]
     #   root_path: existing file directory
     def crawl_title_and_favicon_file(url, options = {})
       puts '>>> Parsing the arguments <<<'
@@ -27,10 +28,8 @@ module Hertools
       end
 
       puts '>>> Analysing the http response <<<'
-      if httparty?
-        response = HTTParty.get(@url)
-        res = response.body
-      else
+      case @html_parser
+      when 'nokogiri'
         response = HTTParty.head(@url)
         res = begin
                 Nokogiri::HTML(URI.open(url), nil, 'UTF-8')
@@ -38,6 +37,12 @@ module Hertools
                 puts e
                 nil
               end
+      when 'httparty'
+        response = HTTParty.get(@url)
+        res = response.body
+      else
+        response = Net::HTTP.get_response(URI(@url))
+        res = response.body.force_encoding("utf-8")
       end
       puts "HttpCode: #{response.code}"
 
@@ -47,27 +52,27 @@ module Hertools
         @favicon_url = "#{@index_url}/favicon.ico"
         puts "Use the default favicon url: #{@favicon_url}."
       else
-        @title = if httparty?
-                   res[%r{<title>\n*(.*)\n*</title>}, 1].to_s
-                 else
+        @title = if nokogiri?
                    res.xpath('//head/title')[0]&.content.to_s
+                 else
+                   res[%r{<title>\n*(.*)\n*</title>}, 1].to_s
                  end
         if @title.empty?
           puts 'Not found the title!'
           puts 'Use the domain name as the title.'
           @title = @domain_name
         end
-        if httparty?
+        unless nokogiri?
           coder = HTMLEntities.new
           @title = coder.decode(@title)
         end
         puts "Title: #{@title}"
 
-        @favicon_url = if httparty?
-                         res[/<link rel="icon".*href="([^"]+)/, 1].to_s
-                       else
+        @favicon_url = if nokogiri?
                          favicon_links = res.xpath('//head/link[@rel="icon"]')
                          favicon_links.empty? ? '' : favicon_links[0][:href].to_s
+                       else
+                         res[/<link rel="icon".*href="([^"]+)/, 1].to_s
                        end
         if @favicon_url.empty?
           puts 'Not found the favicon url!'
@@ -147,10 +152,10 @@ module Hertools
 
     def parse_options(options)
       options = Hash(options)
-      html_parser = options.fetch(:html_parser) { 'httparty' }
+      html_parser = options.fetch(:html_parser) { 'net_http' }
       @old_html_parser = @html_parser
-      @html_parser = %w[httparty nokogiri].include?(html_parser) ? html_parser : 'httparty'
-      reset_httparty_judge_result if @old_html_parser != @html_parser
+      @html_parser = %w[httparty nokogiri].include?(html_parser) ? html_parser : 'net_http'
+      rejudge_nokogiri if @old_html_parser != @html_parser
       puts "HtmlParser: #{@html_parser}"
       root_path = options.fetch(:root_path) { Dir.pwd }
       @root_path = (File.directory?(root_path) ? root_path : Dir.pwd).chomp('/')
@@ -161,12 +166,12 @@ module Hertools
       false
     end
 
-    def httparty?
-      @httparty_judge_result ||= @html_parser == 'httparty'
+    def nokogiri?
+      @judge_nokogiri ||= @html_parser == 'nokogiri'
     end
 
-    def reset_httparty_judge_result
-      @httparty_judge_result = nil
+    def rejudge_nokogiri
+      @judge_nokogiri = nil
     end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: hertools
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.1
 platform: ruby
 authors:
 - WuDi
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2020-09-25 00:00:00.000000000 Z
+date: 2020-09-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: httparty