blackstack_commons 0.0.16 → 0.0.20

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 5aca35493bf21923dc3c0479dc67ddb700d34541
-  data.tar.gz: 7f8b76d7fe177684af3b961689852a7a33b4d07b
+  metadata.gz: a6ab4c21a6f6cf0ff81035fa7f09d0be05c113d4
+  data.tar.gz: 31141034849f46b3681e718d854d51b6bca10890
 SHA512:
-  metadata.gz: 7c6eb8bc9526ee4c19181eb48fdc1deea4ed985a4f494ebd9f24915b7856ad732c15ab58ebfb009d88266dfff22f1e829dee5601a5c0438105859bb2d99ebe3a
-  data.tar.gz: 6fe0d63b677c23fcd56cf58b02c88a8e0dbae38c1ae10854a4385dc19bb84b1fbf00476bf407616adf4c815932d39ee6c95ae6992bd210f9d576960b33810617
+  metadata.gz: a455c47b52c54b8d52919b8850593e7e2acb90c5300cb0580fb1e40301357481ac5473bdba81d6a6b302001b5a9f585f277f9248651b4256b89a6277eee470a2
+  data.tar.gz: 94893de40ffbc265b4a4246aa1d23a812ca7336ccea7aecc872ee172c1da97bcf84b4aad169360c9edf992b714b46d16c9d9d11deb87a6436ef8051a525a65c6

data/lib/functions.rb

@@ -453,6 +453,7 @@ module BlackStack
       uri.query = URI.encode_www_form(params)
       Net::HTTP.start(uri.host, uri.port, :use_ssl => uri.scheme == 'https', :verify_mode => ssl_verify_mode) do |http|
         req = Net::HTTP::Get.new uri
+        #req.body = body if !body.nil?
         res = http.request req
         case res
         when Net::HTTPSuccess then res
@@ -470,6 +471,7 @@ module BlackStack
         req = Net::HTTP::Post.new(uri)
         req['Content-Type'] = 'application/json'
         req.set_form_data(params)
+        #req.body = body if !body.nil?
         res = http.request req
         case res 
         when Net::HTTPSuccess then res
@@ -552,6 +554,147 @@ module BlackStack
     def self.file_age(filename)
       (Time.now - File.ctime(filename))/(24*3600)
     end
+
+
+    # TODO: Is not guaranteed this function works with 100% of the redirect-urls. This problem requires analysis and development of a general purpose algorith
+    # This function gets the final url from a redirect url.
+    # Not all the redirect-urls works the same way.
+    # Below are 3 examples. Each one works with 1 of the 2 strategies applied by this funcion.
+    # => url = "https://www.google.com.ar/url?sa=t&rct=j&q=&esrc=s&source=web&cd=1&cad=rja&uact=8&ved=0CB0QFjAAahUKEwjCg8zMsNvGAhXMMj4KHWBfA50&url=https%3A%2F%2Fwww.linkedin.com%2Fpub%2Fdavid-bell%2F5%2F76a%2F12&ei=IGalVcLzFMzl-AHgvo3oCQ&usg=AFQjCNGMbF2vRIOWsRjF-bjjoG6Nl1wg_g&sig2=ZP6ZbZxpmTHw82rIP7YYew&bvm=bv.97653015,d.cWw"
+    # => url = "https://www.google.com.ar/url?q=https://www.linkedin.com/pub/mark-greene/2/bb8/b59&sa=U&ved=0CDoQFjAIahUKEwiqivi5sdvGAhWJg5AKHSzkB5o&usg=AFQjCNGE09H9hf92mfvwPVnComssDjBBCw"
+    # If the url is not a redirect-url, this function returns the same url.
+=begin
+    def get_redirect(url)
+      begin
+        res = nil
+        httpc = HTTPClient.new
+        resp = httpc.get(url)
+        res = resp.header['Location']
+        
+        if res.size == 0
+          uri = URI.parse(url)
+          uri_params = CGI.parse(uri.query)
+          redirected_url = uri_params['url'][0]            
+          
+          if ( redirected_url != nil )
+            res = redirected_url
+          else
+            res = url
+          end
+        else
+          res = res[0]
+        end
+      rescue
+        res = url
+      end
+      return res
+    end
+=end
+    # returns a hash with the parametes in the url
+    def self.params(url)
+      # TODO: Corregir este parche:
+      # => El codigo de abajo usa la URL de una busqueda en google. Esta url generara una excepcion cuando se intenta parsear sus parametros.
+      # => Ejecutar las 2 lineas de abajo para verificar.
+      # => url = "https://www.google.com/webhp#q=[lead+generation]+%22John%22+%22Greater+New+York+City+Area+*+Financial+Services%22+site:linkedin.com%2Fpub+-site:linkedin.com%2Fpub%2Fdir"
+      # => p = CGI::parse(URI.parse(url).query)
+      # => La linea de abajo hace un gsbub que hace que esta url siga funcionando como busqueda de google, y ademas se posible parsearla.
+      url = url.gsub("webhp#q=", "webhp?q=")
+    
+      return CGI::parse(URI.parse(url).query)
+    end
+    
+    # Add a parameter to the url. It doesn't validate if the param already exists.
+    def self.add_param(url, param_name, param_value)
+      uri = URI(url)
+      params = URI.decode_www_form(uri.query || '')
+      
+      if (params.size==0)
+        params << [param_name, param_value]
+        uri.query = URI.encode_www_form(params)
+        return uri.to_s
+      else
+        uri.query = URI.encode_www_form(params)
+        return uri.to_s + "&" + param_name + "=" + param_value    
+      end
+    end
+    
+    # Changes the value of a parameter in the url. It doesn't validate if the param already exists.
+    def self.change_param(url, param_name, param_value)
+      uri = URI(url)
+    #  params = URI.decode_www_form(uri.query || [])
+      params = CGI.parse(uri.query)
+      params["start"] = param_value
+      uri.query = URI.encode_www_form(params)
+      uri.to_s
+    end
+    
+    # Change or add the value of a parameter in the url, depending if the parameter already exists or not.
+    def self.set_param(url, param_name, param_value)
+      params = BlackStack::Netting::params(url) 
+      if ( params.has_key?(param_name) == true )
+        newurl = BlackStack::Netting::change_param(url, param_name, param_value)
+      else
+        newurl = BlackStack::Netting::add_param(url, param_name, param_value)
+      end
+      return newurl
+    end
+    
+    # get the domain from any url
+    def self.getDomainFromUrl(url)
+      if (url !~ /^http:\/\//i && url !~ /^https:\/\//i) 
+        url = "http://#{url}"
+      end
+      
+      if (URI.parse(url).host == nil)
+        raise "Cannot get domain for #{url}" 
+      end
+    
+      if (url.to_s.length>0)
+        return URI.parse(url).host.sub(/^www\./, '')
+      else
+        return nil
+      end
+    end
+  
+    def self.getDomainFromEmail(email)
+      if email.email?
+        return email.split("@").last
+      else
+        raise "getDomainFromEmail: Wrong email format."
+      end
+    end
+  
+    def self.getWhoisDomains(domain, allow_heuristic_to_avoid_hosting_companies=false)
+      a = Array.new
+      c = Whois::Client.new
+      r = c.lookup(domain)
+    
+      res = r.to_s.scan(/Registrant Email: (#{BlackStack::Strings::MATCH_EMAIL})/).first
+      if (res!=nil)
+        a << BlackStack::Netting::getDomainFromEmail(res[0].downcase)
+      end
+    
+      res = r.to_s.scan(/Admin Email: (#{BlackStack::Strings::MATCH_EMAIL})/).first
+      if (res!=nil)
+        a << BlackStack::Netting::getDomainFromEmail(res[0].downcase)
+      end
+    
+      res = r.to_s.scan(/Tech Email: (#{BlackStack::Strings::MATCH_EMAIL})/).first
+      if (res!=nil)
+        a << BlackStack::Netting::getDomainFromEmail(res[0].downcase)
+      end
+    
+      # remover duplicados
+      a = a.uniq
+    
+      # 
+      if (allow_heuristic_to_avoid_hosting_companies==true)
+        # TODO: develop this feature
+      end
+    
+      return a
+    end
+
   end # module Netting
   
 end # module BlackStack

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: blackstack_commons
 version: !ruby/object:Gem::Version
-  version: 0.0.16
+  version: 0.0.20
 platform: ruby
 authors:
 - Leandro Daniel Sardi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-11-15 00:00:00.000000000 Z
+date: 2019-12-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: content_spinning
@@ -50,7 +50,7 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 1.8.1
-description: 'Find documentation here: https://github.com/leandrosardi/blackstack_commons.'
+description: 'THIS GEM IS STILL IN DEVELOPMENT STAGE. Find documentation here: https://github.com/leandrosardi/blackstack_commons.'
 email: leandro.sardi@expandedventure.com
 executables: []
 extensions: []
@@ -89,5 +89,6 @@ rubyforge_project:
 rubygems_version: 2.4.5.1
 signing_key: 
 specification_version: 4
-summary: Commons classes, functions and constants for the BlackStack framework.
+summary: THIS GEM IS STILL IN DEVELOPMENT STAGE. Commons classes, functions and constants
+  for the BlackStack framework.
 test_files: []