url_common 0.1.1 → 0.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e75ba5e9d78301e4cf753f4ade0a54b036aaca17d7527791b984d8bc81215a5a
-  data.tar.gz: 7bae7598dd4884c9c795e511c32c02cca9095a32f23522e0241842631250488b
+  metadata.gz: b6ae6f1bbc3ec4b834e05a97701a13746db44ad172869868c962067599c12cc2
+  data.tar.gz: 3194ad31952c2bfc4fc5730ad766104f96a0299b3000a99a6efd32224907adfc
 SHA512:
-  metadata.gz: d633158a6b206eca935948a72ebcce4bf90a196c373eca02ae9895819f5b027333b60e4dc5ec84f7b6507d4f468ac1752d36ca367abae4b03b65fd6665857095
-  data.tar.gz: a48dc2eace4656f5b2e16957174ae399633d97ddcff830d799ce31dee94ebfcbe648a4e3a1eb78c0ea8a91f63d8df22df783eed7f99d21920a2ea6a3b6edef44
+  metadata.gz: 32882ac9c6ee8e890daa00cd7ebedb631b99ad3b57856378a49675586a4b125bef12dd4ab116087684969c600acd63300f9689af8ee223792362aecbbbb282fd
+  data.tar.gz: a01f7d306fbc08f3610ea0732540d0ee80c13703d36fb5d4c3919025df9ccca92707737dfc640016086e7a179437108ddb696144bc55faa789ed6c46aa7f1184

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    url_common (0.1.1)
+    url_common (0.1.2)
       fuzzyurl (~> 0.9.0)
       mechanize (~> 2.6)
 

data/lib/url_common/version.rb

@@ -1,3 +1,3 @@
 module UrlCommon
-  VERSION = "0.1.1"
+  VERSION = "0.1.2"
 end

data/lib/url_common.rb

@@ -39,9 +39,22 @@ module UrlCommon
     return 'us'
   end
   
+  # original
+  # def self.get_base_domain(url)
+  #   parts = URI.parse(url)
+  #   return parts.host.gsub(/^www./,'')
+  # end
+  
   def self.get_base_domain(url)
-    parts = URI.parse(url)
-    return parts.host.gsub(/^www./,'')
+    #debugger if url =~ /c06rh22whx1g/
+    begin
+      url = url.gsub(/ /,'%20')
+      parts = URI.parse(url)
+      return parts.host.gsub(/^www./,'')
+    rescue StandardError => e
+      fu = Fuzzyurl.from_string(url)
+      return fu.hostname.gsub(/^www./,'')
+    end
   end
   
   def self.join(base, rest, debug = false)
@@ -60,9 +73,19 @@ module UrlCommon
     end
   end
   
-  #TODO
   def self.count_links(html)
-    return 0
+    if html =~ /<html/i
+      content_type = "html"
+    else
+      content_type = "ascii"
+    end
+    parts = html.split(" ")
+    link_ctr = 0
+    parts.each do |part|
+      link_ctr = link_ctr + 1 if part =~ /https:?\/\// && content_type == 'ascii'
+      link_ctr = link_ctr + 1 if part =~ /<a [^>]+.+<\/a>/i && content_type == 'html'
+    end
+    link_ctr
   end
   
   def self.agent
@@ -82,13 +105,23 @@ module UrlCommon
   #
   def self.url_base(url, base_domain=nil)
     if base_domain.nil?
-      base_domain = get_base_domain(url)
+      base_domain = UrlCommon.get_base_domain(url)
+    end
+    begin
+      url = url.gsub(/ /,'%20')      
+      parts = URI.parse(url)
+      extra = ""
+      extra = "?#{parts.query}" if parts.query
+      url_base = "#{base_domain}#{parts.path}#{extra}"
+      return url_base[0..254]
+    rescue StandardError => e
+      fu = Fuzzyurl.from_string(url)
+      base_domain = UrlCommon.get_base_domain(url)
+      extra = ""
+      extra = "?#{fu.query}" if fu.query
+      url_base = "#{base_domain}#{fu.path}#{extra}"
+      return url_base[0..254]
     end
-    parts = URI.parse(url)
-    extra = ""
-    extra = "?#{parts.query}" if parts.query
-    url_base = "#{base_domain}#{parts.path}#{extra}"
-    return url_base[0..254]
   end
   
   #tested #https://www.amazon.com/gp/product/B01DT4A2R4/ref=as_li_qf_sp_asin_il_tl?ie=UTF8&tag=nickjanetakis-20&camp=1789&creative=9325&linkCode=as2&creativeASIN=B01DT4A2R4&linkId=496be5e222b6291369c0a393c797c2c0
@@ -262,8 +295,9 @@ module UrlCommon
   #TODO needs tests
   def self.create_mechanize_page_from_html(url, html)
     mechanize_page = Mechanize::Page.new(nil, {'content-type'=>'text/html'}, html, nil, Mechanize.new)
+    url = url.gsub(/ /,'%20')
     mechanize_page.uri = URI.parse(url)    
-  
+    
     return mechanize_page
   end
   
@@ -279,6 +313,7 @@ module UrlCommon
   end
   
   #TODO needs tests
+  # UrlCommon.get_page_title("https://gist.github.com/fuzzygroup/811a9334b1a6dc394de74a23cb7e12fa")
   def self.get_page_title(url, html)
     page = UrlCommon.create_mechanize_page_from_html(url, html)
     title = ""

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: url_common
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.1.2
 platform: ruby
 authors:
 - Scott Johnson
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2022-06-04 00:00:00.000000000 Z
+date: 2022-06-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fuzzyurl