url_common 0.1.1 → 0.1.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e75ba5e9d78301e4cf753f4ade0a54b036aaca17d7527791b984d8bc81215a5a
-  data.tar.gz: 7bae7598dd4884c9c795e511c32c02cca9095a32f23522e0241842631250488b
+  metadata.gz: bf016ee73428bc8dc46afb13ddf8aa1e21840650e38ac0b6c03d91c69453afca
+  data.tar.gz: '098c5fcaa13ee7bf0b390bfae0af56784ed25547d7b0946591c2619344f004ca'
 SHA512:
-  metadata.gz: d633158a6b206eca935948a72ebcce4bf90a196c373eca02ae9895819f5b027333b60e4dc5ec84f7b6507d4f468ac1752d36ca367abae4b03b65fd6665857095
-  data.tar.gz: a48dc2eace4656f5b2e16957174ae399633d97ddcff830d799ce31dee94ebfcbe648a4e3a1eb78c0ea8a91f63d8df22df783eed7f99d21920a2ea6a3b6edef44
+  metadata.gz: 780507307f3b1c6745cddba4eeb37cbad88a623c8b6f694a51798c0347d80db5fffbc2210ffafe0b272e270ecc48bf036c35c7071384a656bcf0b0ca30cdc791
+  data.tar.gz: bd449bfe93ecdfe735c947c8d6caba91962493778cbfe899cc456a5f056db2102c3ff208084f3fbd3ce47b3383c4046d3a1c4a6c5a5504e6974a81e9b809d0cd

data/Gemfile

@@ -3,10 +3,15 @@ source "https://rubygems.org"
 # Specify your gem's dependencies in url_common.gemspec
 gemspec
 
-ruby "2.7.1"
+ruby "3.1.2"
 
 gem "rake", "~> 12.0"
 gem "rspec", "~> 3.0"
 gem "fuzzyurl", '~> 0.9.0'
-gem 'mechanize', '~> 2.6'
 gem "byebug"
+
+gem "hpricot", "~> 0.8.6"
+gem 'net-http-persistent', github: 'drbrain/net-http-persistent'
+gem "mechanize", "~> 2.7"
+
+gem "webrick", "~> 1.7"

data/Gemfile.lock

@@ -1,7 +1,14 @@
+GIT
+  remote: https://github.com/drbrain/net-http-persistent.git
+  revision: 857c3baaa541644fa437328b535042a500414119
+  specs:
+    net-http-persistent (4.0.1)
+      connection_pool (~> 2.2)
+
 PATH
   remote: .
   specs:
-    url_common (0.1.1)
+    url_common (0.1.5)
       fuzzyurl (~> 0.9.0)
       mechanize (~> 2.6)
 
@@ -9,11 +16,12 @@ GEM
   remote: https://rubygems.org/
   specs:
     byebug (11.1.3)
-    connection_pool (2.2.3)
+    connection_pool (2.2.5)
     diff-lcs (1.4.4)
     domain_name (0.5.20190701)
       unf (>= 0.0.5, < 1.0.0)
     fuzzyurl (0.9.0)
+    hpricot (0.8.6)
     http-cookie (1.0.3)
       domain_name (~> 0.5)
     mechanize (2.7.6)
@@ -30,8 +38,6 @@ GEM
     mime-types-data (3.2020.0512)
     mini_portile2 (2.4.0)
     net-http-digest_auth (1.4.1)
-    net-http-persistent (4.0.0)
-      connection_pool (~> 2.2)
     nokogiri (1.10.10)
       mini_portile2 (~> 2.4.0)
     ntlm-http (0.1.1)
@@ -52,6 +58,7 @@ GEM
     unf (0.1.4)
       unf_ext
     unf_ext (0.0.7.7)
+    webrick (1.7.0)
     webrobots (0.1.2)
 
 PLATFORMS
@@ -60,13 +67,16 @@ PLATFORMS
 DEPENDENCIES
   byebug
   fuzzyurl (~> 0.9.0)
-  mechanize (~> 2.6)
+  hpricot (~> 0.8.6)
+  mechanize (~> 2.7)
+  net-http-persistent!
   rake (~> 12.0)
   rspec (~> 3.0)
   url_common!
+  webrick (~> 1.7)
 
 RUBY VERSION
-   ruby 2.7.1p83
+   ruby 3.1.2p20
 
 BUNDLED WITH
    2.1.4

data/lib/url_common/version.rb

@@ -1,3 +1,3 @@
 module UrlCommon
-  VERSION = "0.1.1"
+  VERSION = "0.1.5"
 end

data/lib/url_common.rb

@@ -30,6 +30,17 @@ module UrlCommon
     end
   end
   
+  # UrlCommon.parse_fid_from_amazon_url("https://www.amazon.com/Original-GEN-2-0-Screwdriver-Industrial-Technician/dp/B0845919P2/?_encoding=UTF8&pd_rd_w=cekvo&content-id=amzn1.sym.bbb6bbd8-d236-47cb-b42f-734cb0cacc1f&pf_rd_p=bbb6bbd8-d236-47cb-b42f-734cb0cacc1f&pf_rd_r=3WP00V89EKYCQ1PB16VY&pd_rd_wg=HlQVt&pd_rd_r=30b33abe-2010-435e-b2cc-338f2ffbf3cf&ref_=pd_gw_ci_mcx_mi")
+  def self.parse_fid_from_amazon_url(url)
+    tmp = /\/dp\/([A-Za-z0-9]+)/.match(url)
+    if tmp && tmp[1]
+      return tmp[1] 
+    else
+      return nil
+    end
+  end
+  
+  
   def self.parse_country_from_itunes_url(url)
     country = /https?:\/\/itunes\.apple\.com\/(..)\//.match(url)
     if country
@@ -39,9 +50,22 @@ module UrlCommon
     return 'us'
   end
   
+  # original
+  # def self.get_base_domain(url)
+  #   parts = URI.parse(url)
+  #   return parts.host.gsub(/^www./,'')
+  # end
+  
   def self.get_base_domain(url)
-    parts = URI.parse(url)
-    return parts.host.gsub(/^www./,'')
+    #debugger if url =~ /c06rh22whx1g/
+    begin
+      url = url.gsub(/ /,'%20')
+      parts = URI.parse(url)
+      return parts.host.gsub(/^www./,'')
+    rescue StandardError => e
+      fu = Fuzzyurl.from_string(url)
+      return fu.hostname.gsub(/^www./,'')
+    end
   end
   
   def self.join(base, rest, debug = false)
@@ -60,9 +84,19 @@ module UrlCommon
     end
   end
   
-  #TODO
   def self.count_links(html)
-    return 0
+    if html =~ /<html/i
+      content_type = "html"
+    else
+      content_type = "ascii"
+    end
+    parts = html.split(" ")
+    link_ctr = 0
+    parts.each do |part|
+      link_ctr = link_ctr + 1 if part =~ /https:?\/\// && content_type == 'ascii'
+      link_ctr = link_ctr + 1 if part =~ /<a [^>]+.+<\/a>/i && content_type == 'html'
+    end
+    link_ctr
   end
   
   def self.agent
@@ -82,13 +116,23 @@ module UrlCommon
   #
   def self.url_base(url, base_domain=nil)
     if base_domain.nil?
-      base_domain = get_base_domain(url)
+      base_domain = UrlCommon.get_base_domain(url)
+    end
+    begin
+      url = url.gsub(/ /,'%20')      
+      parts = URI.parse(url)
+      extra = ""
+      extra = "?#{parts.query}" if parts.query
+      url_base = "#{base_domain}#{parts.path}#{extra}"
+      return url_base[0..254]
+    rescue StandardError => e
+      fu = Fuzzyurl.from_string(url)
+      base_domain = UrlCommon.get_base_domain(url)
+      extra = ""
+      extra = "?#{fu.query}" if fu.query
+      url_base = "#{base_domain}#{fu.path}#{extra}"
+      return url_base[0..254]
     end
-    parts = URI.parse(url)
-    extra = ""
-    extra = "?#{parts.query}" if parts.query
-    url_base = "#{base_domain}#{parts.path}#{extra}"
-    return url_base[0..254]
   end
   
   #tested #https://www.amazon.com/gp/product/B01DT4A2R4/ref=as_li_qf_sp_asin_il_tl?ie=UTF8&tag=nickjanetakis-20&camp=1789&creative=9325&linkCode=as2&creativeASIN=B01DT4A2R4&linkId=496be5e222b6291369c0a393c797c2c0
@@ -262,8 +306,9 @@ module UrlCommon
   #TODO needs tests
   def self.create_mechanize_page_from_html(url, html)
     mechanize_page = Mechanize::Page.new(nil, {'content-type'=>'text/html'}, html, nil, Mechanize.new)
+    url = url.gsub(/ /,'%20')
     mechanize_page.uri = URI.parse(url)    
-  
+    
     return mechanize_page
   end
   
@@ -279,6 +324,7 @@ module UrlCommon
   end
   
   #TODO needs tests
+  # UrlCommon.get_page_title("https://gist.github.com/fuzzygroup/811a9334b1a6dc394de74a23cb7e12fa")
   def self.get_page_title(url, html)
     page = UrlCommon.create_mechanize_page_from_html(url, html)
     title = ""
@@ -496,5 +542,26 @@ module UrlCommon
     
     return UrlCommon.select_best_rssurl_from_rssurls(results)
   end
+
+  #based on this approach: https://medium.com/@sparkboldstudio/building-a-url-shortener-rails-app-96db60d3bf9d
+  #def self.generate_short_fid
+  #def self.create_fid
+  def self.generate_fid
+    rand(36**8).to_s(36)
+  end
+  
+  def self.sanitize(long_url, url_base)
+    long_url.strip!
+    sanitize_url = self.long_url.downcase.gsub(/(https?:\/\/)|(www\.)/,"")
+    #"http://#{sanitize_url}"
+    url_base + sanitize_url
+  end
+  
+  # todo expand on this by checking to make sure it doesn't call again
+  # should be fine but ttfn
+  def self.create_permalink(tbl, fld, url)
+    short_url = UrlCommon.generate_short_url
+    sanitized_url = UrlCommon.sanitize(url, "https://pullquotes.io/pull_quotes/")
+  end
   
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: url_common
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.1.5
 platform: ruby
 authors:
 - Scott Johnson
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2022-06-04 00:00:00.000000000 Z
+date: 2022-07-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fuzzyurl
@@ -84,7 +84,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.2
+rubygems_version: 3.3.7
 signing_key:
 specification_version: 4
 summary: This is a class library designed for common url manipulation and crawling