title_grabber 0.3.4 → 0.3.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8a8b57f801507ef8c856ebd741b247c30e08472b5df3be45d30e6fd7fa3c0092
-  data.tar.gz: 75ff595060fe8dd71a669ebadcf38f040d5c47492a5c6ea770e893832493ddff
+  metadata.gz: efadff19dbc0622e9188c0349f80ff8d7e5a3040c91db075f3c51529efb39b03
+  data.tar.gz: 3230ec05cd35e8d3b797b9caa69eca8bf30a41598fb98b312b59522b9c348353
 SHA512:
-  metadata.gz: 661dd87e4e60dcfd4a66168799f3b07babf5c21a890b2e61afb99b527b1e15cb7ebd875a60f39bd12cc2f06ef286b603efb89c4908f461606fa3c60a09ad5db6
-  data.tar.gz: b6b1152856c43702082a788a1a056ccc2cef93101faada8890eda27b949d424cf3b9a83991fd91e07733f2c860003b26644510ae9635ea5c6336a1990f997445
+  metadata.gz: 02adc0c9b125a64a11fce2fc6be7ee1ab3d3798045536823f6c2763c7c004c950bfbacafdc443634e43c2f742adb42ba6213a5c1197575648b3ff23c5a2ab1e2
+  data.tar.gz: d487cbd2f084db0f076a42f03fe14912d614bacbf8a7ae33d3eed93f2e6125da69c5ef4433d70e94fbbe4103341bd86f07bd02c725d3721ce6f6816c00227e7e

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    title_grabber (0.3.4)
+    title_grabber (0.3.5)
       http (~> 4.1)
       oga (~> 2.15)
 

data/lib/http_helper.rb

@@ -15,12 +15,11 @@ module HTTPHelper
     retries = 0
 
     begin
-      body = Timeout.timeout(read_to) {
-               HTTP.timeout(write: write_to, connect: connect_to, read: read_to).
-                    follow(max_hops: MAX_HOPS).
-                    get(url, ssl_context: ssl_ctx).
-                    to_s
-             }
+      res = Timeout.timeout(read_to) {
+              HTTP.timeout(write: write_to, connect: connect_to, read: read_to).
+                   follow(max_hops: MAX_HOPS).
+                   get(url, ssl_context: ssl_ctx)
+            }
     rescue HTTP::Error, Timeout::Error => err
       msg = err.message
 
@@ -42,7 +41,7 @@ module HTTPHelper
       logger.error "[#{Thread.current.name}] URL: #{url} [#{err.message}]"
       nil
     else
-      utf8_encode(body)
+      [res.uri.to_s, utf8_encode(res.to_s)]
     end
   end
 

data/lib/title_grabber.rb

@@ -20,9 +20,11 @@ module TitleGrabber
   MAX_THREADS = Etc.nprocessors
   URL_RE = %r(https?://\S+)i
   URL_HEADER = -"url"
+  END_URL_HEAD = -"end_url"
   PAGE_TIT_HEAD = -"page_title"
   ART_TIT_HEAD = -"article_title"
-  HEADERS = [URL_HEADER, PAGE_TIT_HEAD, ART_TIT_HEAD].freeze
+  HEADERS = [URL_HEADER, END_URL_HEAD, PAGE_TIT_HEAD, ART_TIT_HEAD].freeze
+  ART_TIT_SEL = ["article h1", "h1"].freeze
 
   def self.call(lines, options)
     MultiThreadedGrabber.new(lines, options).call
@@ -86,7 +88,8 @@ module TitleGrabber
                   rescue ThreadError; end
 
             while url
-              if (html = open_w_timeout(url, **http_opts)) && !html.empty?
+              end_url, html = open_w_timeout(url, **http_opts)
+              if html && !html&.empty?
                 doc = begin
                         Oga.parse_html(html)
                       rescue ArgumentError, LL::ParserError => err
@@ -97,11 +100,16 @@ module TitleGrabber
                 if doc
                   page_title = doc.at_css('title')&.text || -""
                   clean_up_whitespace(page_title) unless page_title.empty?
-                  article_title = doc.at_css('article h1')&.text
-                  article_title ||= doc.at_css('h1')&.text || -""
+
+                  article_title = nil
+                  ART_TIT_SEL.each do |selector|
+                    article_title = doc.at_css(selector)&.text
+                    break if article_title && !article_title.empty?
+                  end
+                  article_title ||= -""
                   clean_up_whitespace(article_title) unless article_title.empty?
 
-                  csv << [url, page_title, article_title]
+                  csv << [url, end_url, page_title, article_title]
                 end
               end
 
@@ -128,11 +136,13 @@ module TitleGrabber
 
                             if out_path.exist?
                               CSV.foreach(out_path, headers: true) do |r|
+                                end_url = r[END_URL_HEAD]
                                 page_tit = r[PAGE_TIT_HEAD]
                                 art_tit = r[ART_TIT_HEAD]
 
                                 unless page_tit.empty? && art_tit.empty?
-                                  urls[r[URL_HEADER]] = { PAGE_TIT_HEAD => page_tit,
+                                  urls[r[URL_HEADER]] = { END_URL_HEAD => end_url,
+                                                          PAGE_TIT_HEAD => page_tit,
                                                           ART_TIT_HEAD => art_tit }
                                 end
                               end

data/lib/title_grabber/version.rb

@@ -1,3 +1,3 @@
 module TitleGrabber
-  VERSION = "0.3.4"
+  VERSION = "0.3.5"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: title_grabber
 version: !ruby/object:Gem::Version
-  version: 0.3.4
+  version: 0.3.5
 platform: ruby
 authors:
 - Cristian Rasch
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2019-04-07 00:00:00.000000000 Z
+date: 2019-04-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: http