whos_using_what 0.2.12 → 0.3.0

data/lib/whos_using_what/api_clients/{search_client.rb → whos_using_what_search_client.rb}

@@ -1,6 +1,6 @@
 require_relative "../base"
 
-class SearchClient   < Base
+class WhosUsingWhatSearchClient < Base
 
   require "uri"
   require "rest-client"
@@ -22,11 +22,26 @@ class SearchClient   < Base
 
   end
 
+  private
+
   def extractUrls (rawInput, mustContainUrl)
 
     acceptedUrls = Array.new
 
-    urls = URI.extract(rawInput)
+    if (rawInput == nil)
+      return acceptedUrls
+    end
+
+    urls = []
+
+    begin
+      urls = URI.extract(rawInput)
+    end
+
+    if urls.size < 1
+      return urls
+    end
+
     urls.each do |url|
       add = true
       @negativeMatchUrlPatterns.each do |token|
@@ -75,10 +90,12 @@ class SearchClient   < Base
 
   end
 
+  public
 
-  def search(query, site)
 
-    url = "https://www.google.com/search?hl=en&as_q=" << query << "&as_epq=&as_oq=&as_eq=&as_nlo=&as_nhi=&lr=&cr=&as_qdr=all&as_sitesearch=" << site << "&as_occt=any&safe=off&tbs=&as_filetype=&as_rights="
+  #performs a search engine search that is restricted to a company's website and then attempts to determine if they have job listings for a given technology.
+  #If an ad exists it is returned as part of map
+  def search queries, url
 
     begin
       rawHtml = RestClient.get(url)
@@ -86,31 +103,34 @@ class SearchClient   < Base
 
     end
 
-    urls = extractUrls(rawHtml, site)
+    urls = extractUrls(rawHtml, url)
 
-    isMatch = false
+    matching_url = nil
 
-    at_least_one_nonexception_url = false
+    ret_map = Hash.new
 
     urls.each do |cur_url|
       begin
         html = RestClient.get(cur_url)
-        uses_technology = determineIfUsesTechnology(query, html)
-        at_least_one_nonexception_url = true
-        if (uses_technology)
-          isMatch = true
-          break
+
+        queries.each do |query|
+
+          url = "https://www.google.com/search?hl=en&as_q=" << query << "&as_epq=&as_oq=&as_eq=&as_nlo=&as_nhi=&lr=&cr=&as_qdr=all&as_sitesearch=" << cur_url << "&as_occt=any&safe=off&tbs=&as_filetype=&as_rights="
+
+          uses_technology = determineIfUsesTechnology(query, html)
+
+          if (uses_technology)
+            ret_map[query] = cur_url
+          end
+
         end
+
       rescue Exception => exception
-        #raise exception
+        #don't really care at this point, probably not worth logging as some sites just don't end up loading properly
       end
     end
 
-    if (!at_least_one_nonexception_url)
-      return false
-    end
-
-    return isMatch
+    ret_map
   end
 
 end

data/lib/whos_using_what/data_gatherers/gather_companies.rb

@@ -21,7 +21,7 @@ class GatherCompanies < Base
 
   end
 
-  def load_companies_to_db num_iterations, cur_start_position
+  def load_companies_to_db num_iterations, cur_start_position, facet_location_code
 
     increment = 20
     cnt = 1
@@ -32,7 +32,7 @@ class GatherCompanies < Base
       resp = @@linkedin_client.query_companies ({
           "start" => cur_start_position.to_s << "&count=" << increment.to_s,
           "facet=industry" => @linkedin_tech_industry_codes,
-          "locations:(address:(postal-code))" => "95688"
+          "facet=location"=> facet_location_code
       })
       docs = resp['companies'].values[3]
       if docs != nil

data/lib/whos_using_what/data_gatherers/geo_tagger.rb

@@ -1,6 +1,6 @@
 require_relative "../base"
 
-class GeoTagger  < Base
+class GeoTagger < Base
 
   require 'mongo_helper'
   require 'map_data_extraction_util'
@@ -97,7 +97,9 @@ class GeoTagger  < Base
 
   def load_geolocations_into_db
 
-    @companies_coll.find().to_a.each do |company|
+    companies = @companies_coll.find()
+    companies_arr = companies.to_a
+    companies_arr.each do |company|
 
       if !company
         next

data/lib/whos_using_what/data_gatherers/tech_ad_tagger.rb

@@ -0,0 +1,48 @@
+require_relative "../base"
+
+class TechAdTagger < Base
+
+  require 'whos_using_what_search_client'
+
+
+  def initialize
+    @search_client = WhosUsingWhatSearchClient.new
+
+    @mongo_client = MongoHelper.get_mongo_connection
+    @companies_coll = @mongo_client['companies']
+
+
+  end
+
+
+  #iterates through array and updates company db record with technologies found from ads from their website
+  def tag_company_with_technologies tech_keywords
+
+    companies = @companies_coll.find(
+        "languages" => {"$exists" => false}
+    )
+
+    languages = Hash.new
+
+    companies.each do |company|
+
+      languages = Hash.new
+
+      company_languages_map = @search_client.search tech_keywords, company["websiteUrl"]
+
+      company_languages_map.each do |key, value|
+
+        languages[key] = value
+
+      end
+
+      company['languages'] = languages
+
+      @companies_coll.update({"_id" => company["_id"]}, company)
+
+      puts "updating: " << company.to_s
+
+    end
+  end
+
+end

data/lib/whos_using_what/no_sql/mongo_helper.rb

@@ -1,7 +1,5 @@
 require_relative '../base'
 
-include Mongo
-
 class MongoHelper < Base
 
   require 'mongo'
@@ -9,6 +7,8 @@ class MongoHelper < Base
   require 'json'
   require 'yaml'
 
+  include Mongo
+
   def self.get_connection
     return @db_connection if @db_connection
     db = URI.parse(ENV["mongo.uri"].strip)

data/lib/whos_using_what/scripts/data_populators.rb

@@ -0,0 +1,76 @@
+class DataPopulators
+
+  private
+
+  def self.load_class_paths
+
+    $:.unshift(File.expand_path('../../data_gatherers', __FILE__))
+    $:.unshift(File.expand_path('../../data_searchers', __FILE__))
+    $:.unshift(File.expand_path('../../logging', __FILE__))
+
+  end
+
+  #must be after load_class_paths is called
+  def self.initial_requires
+
+    require 'geo_tagger'
+    require 'companies_searcher'
+    require 'logger_factory'
+    require 'gather_companies'
+    require 'tech_ad_tagger'
+
+  end
+
+
+  def self.static_initialize
+
+    #this must be called first
+    load_class_paths
+    #then this
+    initial_requires
+
+    #objects
+    @@log = LoggerFactory.get_default_logger
+    @@geo_tagger = GeoTagger.new @@log
+    @@gather_companies = GatherCompanies.new
+    @@companies_searcher = CompaniesSearcher.new @@geo_tagger
+    @@ech_ad_tagger = TechAdTagger.new
+
+    #data holders
+    @@facet_location = "us:82" #Sacramento
+    @@programming_languages = ["java", "ruby", "c#", "php", "python", "javascript"]
+
+
+  end
+
+  static_initialize
+
+  if __FILE__ == $PROGRAM_NAME
+
+
+    begin
+      @@geo_tagger.load_geolocations_into_db
+    rescue Exception => e
+      puts e.message
+      puts e.backtrace
+    end
+
+
+  end
+
+  #examples:
+
+  # self_instance.tech_ad_tagger.tag_company_with_technologies self_instance.programming_languages
+
+  # self_instance.gather_companies.load_companies_to_db 700, 0, facet_location
+
+  # self_instance.geo_tagger.load_geolocations_into_db
+
+  # self_instance.geo_tagger.update_companies_with_latitude_longitude
+
+  # near = self_instance.companies_searcher.zip_code_search "95688"
+
+  # near = self_instance.companies_searcher.geospatial_search -122.4099154, 37.8059887
+
+
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: whos_using_what
 version: !ruby/object:Gem::Version
-  version: 0.2.12
+  version: 0.3.0
   prerelease: 
 platform: ruby
 authors:
@@ -116,12 +116,14 @@ files:
 - lib/whos_using_what/data_searchers/companies_searcher.rb
 - lib/whos_using_what/util/map_data_extraction_util.rb
 - lib/whos_using_what/logging/logger_factory.rb
+- lib/whos_using_what/scripts/data_populators.rb
 - lib/whos_using_what/api_clients/base_api_client.rb
-- lib/whos_using_what/api_clients/search_client.rb
 - lib/whos_using_what/api_clients/linkedin_client.rb
+- lib/whos_using_what/api_clients/whos_using_what_search_client.rb
 - lib/whos_using_what/api_clients/google_locations_client.rb
 - lib/whos_using_what/data_gatherers/gather_companies.rb
 - lib/whos_using_what/data_gatherers/geo_tagger.rb
+- lib/whos_using_what/data_gatherers/tech_ad_tagger.rb
 - lib/whos_using_what/no_sql/mongo_helper.rb
 - lib/whos_using_what/base.rb
 homepage: http://rubygems.org/gems/whos_using_what