whos_using_what 0.3.3 → 1.0.0

data/lib/whos_using_what/api_clients/base_api_client.rb

@@ -5,7 +5,7 @@ class BaseApiClient < Base
   require "uri"
   require "rest-client"
 
-  def arraySearch(array, rawHtml)
+  def self.arraySearch(array, rawHtml)
 
     rawHtml = rawHtml.downcase
     array.each do |token|
@@ -16,7 +16,8 @@ class BaseApiClient < Base
     return false
   end
 
-  def arry_to_str_delim array, delim
+
+  def self.arry_to_str_delim array, delim
 
     str = ""
     i = 0
@@ -34,7 +35,7 @@ class BaseApiClient < Base
   end
 
 
-  def cleanup_url url
+  def self.cleanup_url url
     #clean up url
     url = url.strip
     if url["www."] != nil
@@ -49,19 +50,20 @@ class BaseApiClient < Base
 
   def determineIfUsesTechnology(technology, rawHtml)
 
-    isJobPage = arraySearch(@jobPageTokens, rawHtml)
+    isJobPage = BaseApiClient.arraySearch(@jobPageTokens, rawHtml)
 
     return isJobPage
 
   end
 
 
-  def starts_with?(string, prefix)
+  def self.starts_with?(string, prefix)
     prefix = prefix.to_s
     string[0, prefix.length] == prefix
   end
 
-  def prepare_params_from_map_helper (base_url, params_map)
+
+  def self.prepare_params_from_map_helper (base_url, params_map)
 
     iter = 1
 

data/lib/whos_using_what/api_clients/google_client.rb

@@ -47,17 +47,17 @@ class GoogleClient < BaseApiClient
       return acceptedUrls
     end
 
-    mustContainUrl = cleanup_url mustContainUrl
+    mustContainUrl = BaseApiClient.cleanup_url mustContainUrl
 
     urls.each do |url|
 
-      url = cleanup_url url
+      url = BaseApiClient.cleanup_url url
 
       accept_url_bool = false
 
       @positiveMatchUrlPatterns.each do |token|
-        if (starts_with? url, token) ||
-            (starts_with? url, mustContainUrl)
+        if (BaseApiClient.starts_with? url, token) ||
+            (BaseApiClient.starts_with? url, mustContainUrl)
           accept_url_bool = true
           break
         end
@@ -78,7 +78,7 @@ class GoogleClient < BaseApiClient
         end
       end
 
-      url = cleanup_url url
+      url = BaseApiClient.cleanup_url url
 
       if accept_url_bool &&
           url != nil && !(acceptedUrls.include? url)
@@ -88,33 +88,11 @@ class GoogleClient < BaseApiClient
     acceptedUrls
   end
 
-  def generate_google_url site_url, search_keyword
-    query_url = [
-        "http://www.google.com/search?",
-        "hl=en",
-        "&as_q=" << "hiring+" << search_keyword,
-        "&as_sitesearch=" << (cleanup_url (site_url))
-    ]
 
-    url = arry_to_str_delim query_url, ""
-
-  end
-
-  def generate_duckduckgo_url site_url, search_keyword
-
-    query_url = [
-        "http://duckduckgo.com/?",
-        "q=" <<
-            "site:" << (cleanup_url (site_url)) <<
-            "+hiring+" << search_keyword,
-    ]
-
-    url = arry_to_str_delim query_url, ""
-  end
 
   #performs a search engine search that is restricted to a company's website and then attempts to determine if they have job listings for a given technology.
   #If an ad exists it is returned as part of map
-  def google_search queries, site_url
+  def perform_search queries, site_url, search_query_url_generator_closure
 
     ret_map = Hash.new
 
@@ -125,7 +103,7 @@ class GoogleClient < BaseApiClient
 
       begin
 
-        url = generate_duckduckgo_url site_url, search_keyword
+        url = search_query_url_generator_closure.call( site_url, search_keyword )
 
         #perform initial search engine search
         @browser.goto url

data/lib/whos_using_what/data_gatherers/tech_ad_tagger.rb

@@ -16,7 +16,7 @@ class TechAdTagger < Base
 
 
   #iterates through array and updates company db record with technologies found from ads from their website
-  def tag_company_with_technologies tech_keywords
+  def tag_company_with_technologies tech_keywords, search_query_url_generator_closure
 
     # uncomment if need to clear out all existing technologies
 =begin
@@ -37,7 +37,7 @@ class TechAdTagger < Base
 
       languages = Hash.new
 
-      company_languages_map = @search_client.google_search tech_keywords, company["websiteUrl"]
+      company_languages_map = @search_client.perform_search tech_keywords, company["websiteUrl"], search_query_url_generator_closure
 
       company_languages_map.each do |key, value|
 

data/lib/whos_using_what/scripts/data_populators.rb

@@ -6,6 +6,7 @@ class DataPopulators
 
     $:.unshift(File.expand_path('../../data_gatherers', __FILE__))
     $:.unshift(File.expand_path('../../data_searchers', __FILE__))
+    $:.unshift(File.expand_path('../../api_clients', __FILE__))
     $:.unshift(File.expand_path('../../logging', __FILE__))
 
   end
@@ -18,6 +19,7 @@ class DataPopulators
     require 'logger_factory'
     require 'gather_companies'
     require 'tech_ad_tagger'
+    require 'base_api_client'
 
   end
 
@@ -52,7 +54,7 @@ class DataPopulators
 
       begin
 
-       # @@gather_companies.load_companies_to_db 700, 0, @@facet_location
+        # @@gather_companies.load_companies_to_db 700, 0, @@facet_location
 
       rescue Exception => e
         puts e.message
@@ -64,9 +66,35 @@ class DataPopulators
 
     t2 = Thread.new do
 
+      # def  site_url, search_keyword
+      generate_google_url = lambda { |site_url, search_keyword|
+        query_url = [
+            "http://www.google.com/search?",
+            "hl=en",
+            "&as_q=" << "hiring+" << search_keyword,
+            "&as_sitesearch=" << (BaseApiClient.cleanup_url (site_url))
+        ]
+
+        url = BaseApiClient.arry_to_str_delim query_url, ""
+      }
+
+
+      #   def  site_url, search_keyword
+      generate_duckduckgo_url= lambda { |site_url, search_keyword|
+        query_url = [
+            "http://duckduckgo.com/?",
+            "q=" <<
+                "site:" << (BaseApiClient.cleanup_url (site_url)) <<
+                "+hiring+" << search_keyword,
+        ]
+
+        url = BaseApiClient.arry_to_str_delim query_url, ""
+      }
+
+
       begin
 
-        @@tech_ad_tagger.tag_company_with_technologies @@programming_languages
+        @@tech_ad_tagger.tag_company_with_technologies @@programming_languages, generate_duckduckgo_url
 
       rescue Exception => e
         puts e.message

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: whos_using_what
 version: !ruby/object:Gem::Version
-  version: 0.3.3
+  version: 1.0.0
   prerelease: 
 platform: ruby
 authors: