whos_using_what 0.1.5 → 0.2.5

data/lib/whos_using_what.rb

@@ -3,14 +3,8 @@ require_relative "whos_using_what/search_client"
 
 class WhosUsingWhat
 
-  def test
-    puts "its working"
-  end
-
   if __FILE__ == $PROGRAM_NAME
 
-    client = SearchClient.new()
-    #client.search("ebay.com", "ruby")
 
 
   end

data/lib/whos_using_what/{base_api_client.rb → api_clients/base_api_client.rb}

@@ -7,14 +7,22 @@ class BaseApiClient
 
   def prepare_params_from_map_helper (base_url, params_map)
 
+    iter = 1
+
     base_url = base_url << "?"
     params_map.each do |key, value|
+
+      if iter != 1
+        base_url = base_url << "&"
+      end
+
       if starts_with?(key, "facet")
-        base_url = base_url << "&" << key << "," << value
+
+        base_url = base_url << key << "," << value
       else
-        base_url = base_url << "&" << key << "=" << value
+        base_url = base_url << key << "=" << value
       end
-
+      iter = iter + 1
     end
     base_url
 

data/lib/whos_using_what/api_clients/google_locations_client.rb

@@ -0,0 +1,57 @@
+require 'map_data_extraction_util'
+
+class GoogleLocationsClient
+
+
+  def initialize
+
+    #todo pull this from config file if this becomes needed
+    @google_api_key = ""
+
+  end
+
+
+  def api_get_google_location_data zip_code
+
+    params = "q=" << zip_code
+    "&output=json"
+    "&key=" << @google_api_key
+
+
+    @geo_code_api_url_base = "http://maps.google.com/maps/geo?"
+
+    begin
+      rawHtml = RestClient.get(@geo_code_api_url_base << params)
+    rescue Exception => e
+      puts e
+    end
+
+    json_resp = JSON.parse(rawHtml.body)
+
+    resp_map = nil
+    begin
+      resp_map = json_resp["Placemark"][0]
+    rescue
+
+    end
+
+    if !resp_map
+      return nil
+    end
+    resp_map
+  end
+
+
+  def get_coords_from_google_location_resp_helper resp_map
+
+    keys_arr1 = ["Point", "coordinates", 0]
+    keys_arr2 = ["Point", "coordinates", 1]
+
+    part1 = MapDataExtractionUtil.safe_extract_helper keys_arr1, resp_map, nil, nil
+    part2 = MapDataExtractionUtil.safe_extract_helper keys_arr2, resp_map, nil, nil
+
+    arr = [part1, part2]
+
+  end
+
+end

data/lib/whos_using_what/api_clients/linkedin_client.rb

@@ -0,0 +1,60 @@
+require 'oauth'
+require 'json'
+require 'base_api_client'
+
+class LinkedinClient < BaseApiClient
+
+
+  @@json_indicator = "format=json"
+
+
+  def initialize(api_key, api_secret, user_token, user_secret, url)
+    super()
+
+    consumer = OAuth::Consumer.new(api_key, api_secret, {:site => url})
+
+    @access_token = OAuth::AccessToken.new(consumer, user_token, user_secret)
+
+  end
+
+
+  def query_companies params
+
+    @@base_url = "http://api.linkedin.com/v1/"
+
+    base_url = @@base_url <<
+        "company-search:(
+        companies:(
+        id,
+        name,
+        universal-name,
+        website-url,
+        industries,
+        logo-url,
+        employee-count-range,
+        locations
+      )
+    )"
+
+    json_api_call_helper(base_url, params)
+
+  end
+
+
+  def json_api_call_helper (base_url, params)
+
+    url = prepare_params_from_map_helper(base_url, params)
+
+    #remove white spaces, for ease in reading queries, they may have white spaces / line breaks
+    url = url.gsub(/\s+/, "")
+
+    puts url
+
+    json = @access_token.get(url << "&" << @@json_indicator)
+
+    JSON.parse(json.body)
+
+  end
+
+
+end

data/lib/whos_using_what/data_gatherers/gather_companies.rb

@@ -0,0 +1,54 @@
+require 'mongo_helper'
+require 'linkedin_client'
+
+#meant to be able to be used as long-running process to save company data to DB
+class GatherCompanies
+
+  def initialize
+
+    @linkedin_tech_industry_codes = "4,132,6,96,113";
+
+    @@mongo_client = MongoHelper.get_mongo_connection
+
+    @@companies_coll = @@mongo_client['companies']
+
+    @li_config = YAML.load_file(File.expand_path("../../config/linkedin.env", __FILE__))
+
+    @@linkedin_client = LinkedinClient.new @li_config["api_key"], @li_config["api_secret"], @li_config["user_token"], @li_config["user_secret"], @li_config["url"]
+
+
+  end
+
+  def load_companies_to_db num_iterations, cur_start_position
+
+    increment = 20
+    cnt = 1
+
+    while cnt <= num_iterations do
+      puts cur_start_position.to_s
+
+      resp = @@linkedin_client.query_companies ({
+          "start" => cur_start_position.to_s << "&count=" << increment.to_s,
+          "facet=industry" => @linkedin_tech_industry_codes,
+          "locations:(address:(postal-code))" => "95688"
+      })
+      docs = resp['companies'].values[3]
+      if docs != nil
+        docs.each do |doc|
+          puts doc
+          @@companies_coll.insert(doc)
+        end
+      end
+
+      cur_start_position = cur_start_position + increment
+
+      cnt = cnt + 1
+
+      sleep_seconds = rand(1-35)
+      puts "sleeping for: " << sleep_seconds.to_s << " seconds"
+      sleep(sleep_seconds)
+
+    end
+  end
+
+end

data/lib/whos_using_what/data_gatherers/geo_tagger.rb

@@ -0,0 +1,151 @@
+require 'mongo_helper'
+require 'map_data_extraction_util'
+require 'google_locations_client'
+
+require "rest-client"
+
+class GeoTagger
+
+  def initialize
+    @mongo_client = MongoHelper.get_mongo_connection
+    @companies_coll = @mongo_client['companies']
+    @coords_coll = @mongo_client['coordinates']
+
+    @locations_client = GoogleLocationsClient.new
+
+    #todo this code should possibly be moved somewhere else for clarity / encapsulation
+    @coords_coll.remove({"loc" => nil})
+
+    @coords_coll.ensure_index([["loc", Mongo::GEO2D]])
+    @companies_coll.ensure_index([["loc", Mongo::GEO2D]])
+
+  end
+
+
+  def zip_acceptance_predicate zip_code
+
+    if !zip_code
+      return false
+    end
+
+    accept = true
+
+    if !zip_code.start_with? ("9")
+      accept = false
+    end
+
+    accept
+
+  end
+
+
+  def load_geolocations_into_db
+
+    @companies_coll.find().to_a.each do |company|
+
+      if !company
+        next
+      end
+
+
+      keys_arr = ['locations', 'values']
+      locations = MapDataExtractionUtil.safe_extract_helper keys_arr, company, :nil, nil
+
+      if !locations
+        next
+      end
+      locations.each do |location|
+
+        zip_code = location['address']['postalCode']
+
+        #strip off anything past 5 characters, as we only want main part of zip code
+        if !zip_code || zip_code.size < 5
+          next
+        end
+
+        zip_code = zip_code[0...5]
+
+        if (zip_acceptance_predicate (zip_code))
+
+
+          begin
+            #todo figure how to do this now that we are using instance variables instead of class variables
+            process_zip_closure = lambda {
+
+              resp_map = @locations_client.api_get_google_location_data zip_code
+
+              doc = {}
+
+
+              doc[:zip] = zip_code
+
+              coords = @locations_client.get_coords_from_google_location_resp_helper resp_map
+              if coords[0] && coords[1]
+                doc[:loc] = {"lon" => coords[0], "lat" => coords[1]}
+              end
+
+
+              keys_arr = ["AddressDetails", "Country", "AdministrativeArea", "Locality", "LocalityName"]
+              MapDataExtractionUtil.safe_extract_helper keys_arr, resp_map, :city, doc
+
+              keys_arr = ["AddressDetails", "Country", "AdministrativeArea", "AdministrativeAreaName"]
+              MapDataExtractionUtil.safe_extract_helper keys_arr, resp_map, :state, doc
+
+              keys_arr = ["AddressDetails", "Country", "CountryNameCode"]
+              MapDataExtractionUtil.safe_extract_helper keys_arr, resp_map, :country, doc
+
+
+              if (doc.size > 1 && doc[:country] == "US")
+                coll = @coords_coll.find(zip: zip_code).to_a
+              end
+
+              if coll && coll.size < 1
+
+
+                @coords_coll.insert(doc)
+
+              end
+            }
+
+
+            process_zip_closure.call
+          rescue Exception => e
+            puts e.message
+            puts e.backtrace
+
+          end
+        end
+      end
+    end
+  end
+
+  def update_companies_with_latitude_longitude
+
+    @companies_coll.find().to_a.each do |company|
+
+
+      locations = MapDataExtractionUtil.safe_extract_helper ["locations", "values"], company, nil, nil
+
+      if locations
+
+        locations.each do |location|
+
+          zip = MapDataExtractionUtil.safe_extract_helper ["address", "postalCode"], location, nil, nil
+
+          if zip
+
+            coords = @coords_coll.find_one({:zip => zip})
+            if coords != nil
+
+              company["loc"] = coords["loc"]
+
+              @companies_coll.update({"_id" => company["_id"]}, company)
+
+            end
+          end
+        end
+      end
+    end
+  end
+
+end

data/lib/whos_using_what/data_searchers/companies_searcher.rb

@@ -0,0 +1,41 @@
+class CompaniesSearcher
+
+  def initialize
+
+    @mongo_client = MongoHelper.get_mongo_connection
+    @companies_coll = @mongo_client['companies']
+    @coords_coll = @mongo_client['coordinates']
+
+    @geo_tagger = GeoTagger.new
+
+  end
+
+  def geospatial_search lon, lat
+
+    near = @companies_coll.find({"loc" => {"$near" => [lat, lon]}})
+
+
+  end
+
+  def zip_code_search zip_code
+    zip_doc = @coords_coll.find_one({:zip => zip_code})
+    if zip_doc == nil
+
+      #todo consider making a method in GeoTagger class to do this instead of using closure directly here
+      #todo need to figure out how to call this from other class
+      closure = @geo_tagger.process_zip_closure zip_code
+
+      closure.call
+
+    end
+    zip_doc = @coords_coll.find_one({:zip => zip_code})
+
+    if zip_doc == nil
+      return nil
+    end
+
+    results = geospatial_search zip_doc["loc"]["lon"], zip_doc["loc"]["lat"]
+    results.to_a
+  end
+
+end

data/lib/whos_using_what/logging/logger_factory.rb

@@ -0,0 +1,15 @@
+require 'logger'
+
+class LoggerFactory
+
+  @@log_path = "log.txt"
+  @@log = Logger.new (@@log_path);
+
+  def self.get_default_logger
+
+    @@log
+
+  end
+
+
+end

data/lib/whos_using_what/no_sql/mongo_helper.rb

@@ -0,0 +1,39 @@
+require 'mongo'
+require 'uri'
+require 'json'
+require 'yaml'
+
+include Mongo
+
+class MongoHelper
+
+  def self.get_connection
+    return @db_connection if @db_connection
+    db = URI.parse(ENV["mongo.uri"].strip)
+    db_name = db.path.gsub(/^\//, '')
+    @db_connection = Mongo::Connection.new(db.host, db.port).db(db_name)
+    @db_connection.authenticate(db.user, db.password) unless (db.user.nil? || db.user.nil?)
+    @db_connection
+
+  end
+
+  if (ENV["mongo.uri"] == nil)
+    @li_config = YAML.load_file(File.expand_path("../../config/mongo.env", __FILE__))
+    ENV["mongo.host"]= @li_config["mongo.host"]
+    ENV["mongo.port"]= @li_config["mongo.port"].to_s
+    ENV["mongo.user"]= @li_config["mongo.user"]
+    ENV["mongo.pass"]= @li_config["mongo.pass"]
+    ENV["mongo.dbname"] = @li_config["mongo.dbname"]
+    ENV["mongo.uri"] = "mongodb://" << ENV["mongo.user"] << ":" << ENV["mongo.pass"] << "@" <<
+        ENV["mongo.host"] <<":" << ENV["mongo.port"] <<"/" << ENV["mongo.dbname"]
+
+  end
+
+  get_connection
+
+
+  def self.get_mongo_connection
+    @db_connection
+  end
+
+end

data/lib/whos_using_what/util/map_data_extraction_util.rb

@@ -0,0 +1,25 @@
+class MapDataExtractionUtil
+
+
+  #doc_key = nil && doc=nil to return the end value instead of to add it to the doc
+  def self.safe_extract_helper keys_arr, map, doc_key, doc
+
+    iter = 1
+    val = ""
+    keys_arr.each do |key|
+      val = map[key]
+      if !val
+        return
+      end
+      if (iter == keys_arr.length)
+        if doc
+          doc[doc_key] = val
+        end
+      end
+      iter = iter + 1
+      map = val
+    end
+    val
+  end
+
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: whos_using_what
 version: !ruby/object:Gem::Version
-  version: 0.1.5
+  version: 0.2.5
   prerelease: 
 platform: ruby
 authors:
@@ -91,16 +91,54 @@ dependencies:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: uri
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: mongo
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 description: What companies are using what technologies
 email: r.dane1010@gmail.com
 executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- lib/whos_using_what/base_api_client.rb
-- lib/whos_using_what/search_client.rb
-- lib/whos_using_what/linkedin_client.rb
-- lib/whos_using_what/config_module.rb
+- lib/whos_using_what/data_searchers/companies_searcher.rb
+- lib/whos_using_what/util/map_data_extraction_util.rb
+- lib/whos_using_what/logging/logger_factory.rb
+- lib/whos_using_what/api_clients/base_api_client.rb
+- lib/whos_using_what/api_clients/search_client.rb
+- lib/whos_using_what/api_clients/linkedin_client.rb
+- lib/whos_using_what/api_clients/google_locations_client.rb
+- lib/whos_using_what/data_gatherers/gather_companies.rb
+- lib/whos_using_what/data_gatherers/geo_tagger.rb
+- lib/whos_using_what/no_sql/mongo_helper.rb
 - lib/whos_using_what.rb
 homepage: http://rubygems.org/gems/whos_using_what
 licenses: []

data/lib/whos_using_what/config_module.rb

@@ -1,10 +0,0 @@
-module ConfigModule
-
-  attr_accessor :linkedin_tech_industry_codes
-
-
-  def initialize
-    @linkedin_tech_industry_codes = "4,132,6,96,113";
-  end
-
-end

data/lib/whos_using_what/linkedin_client.rb

@@ -1,98 +0,0 @@
-require 'oauth'
-require 'json'
-require_relative 'config_module'
-require_relative 'base_api_client'
-
-class LinkedinClient < BaseApiClient
-
-  #the company industry codes to search for, see: https://developer.linkedin.com/documents/industry-codes
-  attr :access_token, true
-  attr :companyUrls
-  @@json_indicator = "format=json"
-
-  include ConfigModule
-
-
-  def initialize(api_key, api_secret, user_token, user_secret, url)
-    super()
-
-    @numberResults = "5"
-    @start = "15"
-
-    consumer = OAuth::Consumer.new(api_key, api_secret, {:site => url})
-
-    @access_token = OAuth::AccessToken.new(consumer, user_token, user_secret)
-
-    #this appears to be the most that linkedin will give back per request
-    @max_results = 20
-
-  end
-
-  def add_json_to_map(key_field_name, raw_json_map, output_map)
-    raw_json_map.each do |value|
-      output_map[value[key_field_name]] = value
-    end
-  end
-
-
-  #todo this should be put into module for re-use
-  def json_api_call_helper (base_url, params)
-    url = prepare_params_from_map_helper(base_url, params)
-    json = @access_token.get(url << "&" << @@json_indicator)
-
-    JSON.parse(json.body)
-  end
-
-  # this method searches for people from a specified company for a specific job type
-  def people_search_for_company (location_code, title, company_name)
-
-    params = Hash.new
-    params["facet=location"] = "us:" <<location_code
-    params["current-company"]= "true"
-    params["title"]=title
-    params["company-name"] = company_name
-
-    base_url = "http://api.linkedin.com/v1/people-search:(people:(id,first-name,headline,positions:(company:(name)),public-profile-url,last-name,picture-url),facets)"
-
-    puts json_api_call_helper(base_url, params)['people']['values']
-
-  end
-
-
-  def gather_company_data(start, number_to_collect, industry_codes)
-
-    if number_to_collect == nil
-      number_to_collect = 20
-    end
-
-    request_num = number_to_collect
-    cnt = 0
-    div = number_to_collect / @max_results
-    if (div <1)
-      div = 1
-    end
-
-    results = Hash.new
-
-    if (industry_codes == nil)
-      industry_codes = @linkedin_tech_industry_codes
-    end
-
-    while cnt < div do
-      base_url = "http://api.linkedin.com/v1/company-search:(companies:(universal-name,id,website-url,locations:(address:(city,state))),facets,num-results)"
-
-      params = Hash.new
-      params["start"] = (start * @max_results + 1).to_s
-      params["count"] =  @max_results.to_s
-      params["facet=location"] =   "us:84"
-      params["facet=industry"]  = industry_codes
-
-      raw_json_map = json_api_call_helper(base_url, params)['companies']['values']
-      add_json_to_map("universalName", raw_json_map, results)
-
-      cnt = cnt + 1
-    end
-    results
-  end
-
-end