title_grabber 0.3.7 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6642eaa211c58d0debf01ff8b0129832ef2c83d7e280f048bab0656ae7f0aec1
-  data.tar.gz: 3d9bc77e04fd081dd9c3792c9cc378c6eac60a818c8cdd9efdfed134f52ec57d
+  metadata.gz: 58643e0df803b9315f741db2effdcbd8b3d4e52845b06de27f18dd726f732bb6
+  data.tar.gz: 013fcdb1650a497126e11b62845240489c296d8de7b0dcd9bc95ccbafb288633
 SHA512:
-  metadata.gz: e5455e96ff7c4ad6654c0c3e57fff4dc60627c55fe586d7bc72f9492c591765451101bfaee1a3537a881713f469600ffc7343081ff9fcc73d56aa733af42e4e1
-  data.tar.gz: 5474dcf7991beb7ea7c48cfe95fa98ab02cd976d068da9a1246ad518eb4963b5c701f46f54c41735b2d77f5309391dcf8f82692b803b401c1b70ef1e75d3147c
+  metadata.gz: '09167bbc4fcd61034322ab62ec5ca68ebffcb8920c9f07622e6b69367c76a639c03314101a3588ecd0e53e62b08d180e323114e5c9f70760c8416e4a342b9850'
+  data.tar.gz: d2f053afc4fc465049d8068302e1baf56a76ef1c6230813a7facd03e24207c03bfd104ebe45c7b2ae5dbc821140a59d4d6d3c4137e766258eb4cf19fbe0a92de

data/Gemfile.lock

@@ -24,6 +24,8 @@ GEM
     http-form_data (2.1.1)
     http_parser.rb (0.6.0)
     minitest (5.11.3)
+    minitest-line (0.6.5)
+      minitest (~> 5.0)
     oga (2.15)
       ast
       ruby-ll (~> 2.1)
@@ -42,6 +44,7 @@ PLATFORMS
 DEPENDENCIES
   bundler (~> 1.17)
   minitest (~> 5.0)
+  minitest-line (~> 0.6)
   rake (~> 10.0)
   title_grabber!
 

data/README.md

@@ -33,20 +33,10 @@ Data is either recorded to out.csv in the CWD or the file specified using the
 title-grabber -o ~/output.csv /abs/path/2/file1.txt rel/path/2/file2.txt
 ```
 
-### Environment Variables
+See all available CLI switches and env vars
 
 ```
-DEBUG - when set it logs to STDOUT instead of to its default target, title_grabber.log
-```
-
-```
-MAX_THREADS - max. # of threads to use, defaults to the # of CPU cores in the machine
-```
-
-```
-CONNECT_TIMEOUT (in seconds) - defaults to 15
-READ_TIMEOUT (in seconds) - defaults to 15
-WRITE_TIMEOUT (in seconds) - defaults to 15
+title-grabber -h
 ```
 
 ## Development
@@ -55,6 +45,8 @@ After checking out the repo, run `bin/setup` to install dependencies. Then, run
 
 To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
 
+Run rake (the default task runs the test suite) to make sure all tests pass.
+
 ## Contributing
 
 Bug reports and pull requests are welcome on GitHub at https://github.com/cristian-rasch/title_grabber.

data/lib/http_helper.rb

@@ -21,7 +21,8 @@ module HTTPHelper
             }
     rescue HTTP::Redirector::TooManyRedirectsError
       logger.warn "[#{Thread.current.name}] GET #{url} resulted in more than #{max_redirects} redirect#{'s' unless max_redirects == 1}"
-    rescue HTTP::Error, Timeout::Error => err
+      nil
+    rescue => err
       msg = err.message
 
       if err.kind_of?(HTTP::Error) || err.kind_of?(Timeout::Error) ||
@@ -35,19 +36,20 @@ module HTTPHelper
           retry
         else
           logger.error "[#{Thread.current.name}] URL: #{url} [#{msg}]"
-          nil
         end
+      else
+        logger.error "[#{Thread.current.name}] URL: #{url} [#{msg}]"
       end
-    rescue => err
-      logger.error "[#{Thread.current.name}] URL: #{url} [#{err.message}]"
+
       nil
-    else
-      [res.uri.to_s, utf8_encode(res.to_s)]
     end
   end
 
-  def logger
-    @logger ||= Logger.new(STDOUT)
+  def read_w_timeout(url, write_to:, connect_to:, read_to:, max_retries:)
+    if res = open_w_timeout(url, write_to: write_to, connect_to: connect_to,
+                                 read_to: read_to, max_retries: max_retries)
+      [res.uri.to_s, utf8_encode(res.to_s)]
+    end
   end
 
   private

data/lib/text_helper.rb

@@ -6,7 +6,7 @@ module TextHelper
 
     begin
       txt.encode!(-"UTF-8", invalid: :replace, undef: :replace,
-                                    replace: -"")
+                            replace: -"")
     rescue EncodingError
       -""
     else
@@ -18,8 +18,8 @@ module TextHelper
   # document.querySelector('title').textContent.trim().replace(/\n/g, ' ').replace(/\s{2,}/g, ' ')
   def clean_up_whitespace(text)
     text.strip!
+    text.gsub!(/\s{2,}/, SINGLE_SPACE)
     text.gsub!("\n", SINGLE_SPACE)
-    text.gsub(/\s{2,}/, SINGLE_SPACE)
     text
   end
 end

data/lib/title_grabber/version.rb

@@ -1,3 +1,3 @@
 module TitleGrabber
-  VERSION = "0.3.7"
+  VERSION = "0.4.0"
 end

data/lib/title_grabber.rb

@@ -26,6 +26,12 @@ module TitleGrabber
   ART_TIT_HEAD = -"article_title"
   HEADERS = [URL_HEADER, END_URL_HEAD, PAGE_TIT_HEAD, ART_TIT_HEAD].freeze
   ART_TIT_SEL = ["article h1", "h1"].freeze
+  TWEET_PERMA_LINK_SEL = -".tweet.permalink-tweet"
+  TWEET_TXT_SELS = %w(.tweet-text QuoteTweet).freeze
+  TWITTER_HOST = -"twitter.com"
+  TWITTER_STATUS_RE = %r(/status/\d+\z)
+  TWITTER_URL_PREFIX = -"https://#{TWITTER_HOST}"
+  CSV_FIELD_SEP = -","
 
   def self.call(lines, options)
     MultiThreadedGrabber.new(lines, options).call
@@ -72,7 +78,7 @@ module TitleGrabber
 
           url = md.to_s
           if h = processed_urls[url]
-            csv << [url, h[PAGE_TIT_HEAD], h[ART_TIT_HEAD]]
+            csv << [url, h[END_URL_HEAD], h[PAGE_TIT_HEAD], h[ART_TIT_HEAD]]
             next
           end
 
@@ -90,8 +96,9 @@ module TitleGrabber
                   rescue ThreadError; end
 
             while url
-              end_url, html = open_w_timeout(url, **http_opts)
-              if html && !html&.empty?
+              end_url, html = read_w_timeout(url, **http_opts)
+
+              if end_url && html && !html.empty?
                 doc = begin
                         Oga.parse_html(html)
                       rescue ArgumentError, LL::ParserError => err
@@ -100,6 +107,34 @@ module TitleGrabber
                       end
 
                 if doc
+                  tweet_urls = []
+                  TWEET_TXT_SELS.each do |tweet_txt_sel|
+                    tweet_urls.concat(doc.css("#{TWEET_PERMA_LINK_SEL} #{tweet_txt_sel} a").
+                                          map { |a| a[-"href"] })
+
+                  end
+                  tweet_urls.compact!
+                  tweet_urls.uniq!
+                  tweet_urls.map! do |url|
+                    if res = open_w_timeout(url, **http_opts)
+                      uri = res.uri
+                      uri.host == TWITTER_HOST && !uri.to_s.match?(TWITTER_STATUS_RE) ? nil : uri.to_s
+                    else
+                      url
+                    end
+                  end
+                  tweet_urls.compact!
+                  tweet_urls.map! do |url|
+                    url.start_with?("/") ? URI.join(TWITTER_URL_PREFIX, url).to_s : url
+                  end
+                  tweet_urls.delete_if { |url|
+                    uri = URI(url)
+                    uri.host == TWITTER_HOST && uri.path.count("/") > 1 &&
+                      !uri.to_s.match?(TWITTER_STATUS_RE)
+                  }
+                  tweet_urls.sort!
+                  end_url = tweet_urls.join(CSV_FIELD_SEP) unless tweet_urls.empty?
+
                   page_title = doc.at_css('title')&.text || -""
                   clean_up_whitespace(page_title) unless page_title.empty?
 

data/title_grabber.gemspec

@@ -44,4 +44,5 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "bundler", "~> 1.17"
   spec.add_development_dependency "rake", "~> 10.0"
   spec.add_development_dependency "minitest", "~> 5.0"
+  spec.add_development_dependency "minitest-line", "~> 0.6"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: title_grabber
 version: !ruby/object:Gem::Version
-  version: 0.3.7
+  version: 0.4.0
 platform: ruby
 authors:
 - Cristian Rasch
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2019-04-10 00:00:00.000000000 Z
+date: 2019-04-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: http
@@ -80,6 +80,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '5.0'
+- !ruby/object:Gem::Dependency
+  name: minitest-line
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.6'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.6'
 description: 
 email:
 - cristianrasch@fastmail.fm