whos_using_what 1.0.0 → 1.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/lib/whos_using_what/api_clients/base_api_client.rb +3 -0
- data/lib/whos_using_what/api_clients/indeed_api_client.rb +32 -0
- data/lib/whos_using_what/api_clients/linkedin_client.rb +17 -3
- data/lib/whos_using_what/data_gatherers/gather_companies.rb +51 -1
- data/lib/whos_using_what/data_searchers/companies_searcher.rb +7 -0
- data/lib/whos_using_what/scripts/data_populators.rb +9 -2
- metadata +2 -1
@@ -0,0 +1,32 @@
|
|
1
|
+
require_relative "../base"
|
2
|
+
|
3
|
+
class IndeedApiClient < BaseApiClient
|
4
|
+
|
5
|
+
def initialize
|
6
|
+
|
7
|
+
headless = Headless.new
|
8
|
+
headless.start
|
9
|
+
@browser = Watir::Browser.new :firefox
|
10
|
+
|
11
|
+
end
|
12
|
+
|
13
|
+
def perform_search keyword, location_city_state, increment, start
|
14
|
+
|
15
|
+
query_url = [
|
16
|
+
"http://api.indeed.com/ads/apisearch?publisher=8417074034192456",
|
17
|
+
"&q=" << keyword << "+" << location_city_state <<
|
18
|
+
"&format=json" <<
|
19
|
+
"&start=" << start.to_s <<
|
20
|
+
"&limit=" << increment.to_s <<
|
21
|
+
"&v=2"
|
22
|
+
]
|
23
|
+
|
24
|
+
url = (BaseApiClient.arry_to_str_delim query_url, "").strip
|
25
|
+
url = url.tr(" ", "+")
|
26
|
+
|
27
|
+
rawHtml = RestClient.get(url)
|
28
|
+
JSON.parse(rawHtml)
|
29
|
+
|
30
|
+
end
|
31
|
+
|
32
|
+
end
|
@@ -8,6 +8,7 @@ class LinkedinClient < BaseApiClient
|
|
8
8
|
|
9
9
|
|
10
10
|
@@json_indicator = "format=json"
|
11
|
+
@@base_url = "http://api.linkedin.com/v1/"
|
11
12
|
|
12
13
|
|
13
14
|
def initialize(api_key, api_secret, user_token, user_secret, url)
|
@@ -22,8 +23,6 @@ class LinkedinClient < BaseApiClient
|
|
22
23
|
|
23
24
|
def query_companies params
|
24
25
|
|
25
|
-
@@base_url = "http://api.linkedin.com/v1/"
|
26
|
-
|
27
26
|
base_url = @@base_url <<
|
28
27
|
"company-search:(
|
29
28
|
companies:(
|
@@ -43,9 +42,24 @@ class LinkedinClient < BaseApiClient
|
|
43
42
|
end
|
44
43
|
|
45
44
|
|
45
|
+
def query_people_from_company company_name, location
|
46
|
+
|
47
|
+
company_name = company_name.gsub(/\s+/, "+")
|
48
|
+
location = location.gsub(/\s+/, "+")
|
49
|
+
|
50
|
+
url = @@base_url <<
|
51
|
+
"people-search?
|
52
|
+
company-name=" << company_name << ",
|
53
|
+
&location=" << location
|
54
|
+
|
55
|
+
json_api_call_helper url, {}
|
56
|
+
|
57
|
+
end
|
58
|
+
|
59
|
+
|
46
60
|
def json_api_call_helper (base_url, params)
|
47
61
|
|
48
|
-
url = prepare_params_from_map_helper(base_url, params)
|
62
|
+
url = BaseApiClient.prepare_params_from_map_helper(base_url, params)
|
49
63
|
|
50
64
|
#remove white spaces, for ease in reading queries, they may have white spaces / line breaks
|
51
65
|
url = url.gsub(/\s+/, "")
|
@@ -5,11 +5,14 @@ class GatherCompanies < Base
|
|
5
5
|
|
6
6
|
require 'mongo_helper'
|
7
7
|
require 'linkedin_client'
|
8
|
+
require 'indeed_api_client'
|
8
9
|
|
9
10
|
def initialize
|
10
11
|
|
11
12
|
@linkedin_tech_industry_codes = "4,132,6,96,113";
|
12
13
|
|
14
|
+
@indeed_api_client = IndeedApiClient.new
|
15
|
+
|
13
16
|
@@mongo_client = MongoHelper.get_mongo_connection
|
14
17
|
|
15
18
|
@@companies_coll = @@mongo_client['companies']
|
@@ -21,6 +24,53 @@ class GatherCompanies < Base
|
|
21
24
|
|
22
25
|
end
|
23
26
|
|
27
|
+
def load_companies_from_indeed
|
28
|
+
|
29
|
+
num_iterations = 20
|
30
|
+
increment = 20
|
31
|
+
cnt = 15
|
32
|
+
|
33
|
+
while cnt <= num_iterations do
|
34
|
+
|
35
|
+
keyword = "ruby"
|
36
|
+
city_state = "pleasant hill, ca"
|
37
|
+
|
38
|
+
json_resp = @indeed_api_client.perform_search keyword, city_state, increment, (increment * (cnt-1)) + 1
|
39
|
+
|
40
|
+
json_resp['results'].each do |job|
|
41
|
+
|
42
|
+
if @@companies_coll.find_one({'name' => job['company']}) != nil
|
43
|
+
next
|
44
|
+
end
|
45
|
+
|
46
|
+
company = {}
|
47
|
+
|
48
|
+
company['locations'] = {
|
49
|
+
values: [
|
50
|
+
{
|
51
|
+
address: {
|
52
|
+
city: job['city'],
|
53
|
+
state: job['state'],
|
54
|
+
country: job['country']
|
55
|
+
}
|
56
|
+
}
|
57
|
+
]
|
58
|
+
}
|
59
|
+
company['name']= job['company']
|
60
|
+
company['languages'] =
|
61
|
+
{
|
62
|
+
keyword.to_s => job['url']
|
63
|
+
}
|
64
|
+
|
65
|
+
|
66
|
+
@@companies_coll.insert company
|
67
|
+
|
68
|
+
end
|
69
|
+
cnt += cnt
|
70
|
+
end
|
71
|
+
|
72
|
+
end
|
73
|
+
|
24
74
|
def load_companies_to_db num_iterations, cur_start_position, facet_location_code
|
25
75
|
|
26
76
|
increment = 20
|
@@ -32,7 +82,7 @@ class GatherCompanies < Base
|
|
32
82
|
resp = @@linkedin_client.query_companies ({
|
33
83
|
"start" => cur_start_position.to_s << "&count=" << increment.to_s,
|
34
84
|
"facet=industry" => @linkedin_tech_industry_codes,
|
35
|
-
"facet=location"=> facet_location_code
|
85
|
+
"facet=location" => facet_location_code
|
36
86
|
})
|
37
87
|
docs = resp['companies'].values[3]
|
38
88
|
if docs != nil
|
@@ -20,6 +20,13 @@ class CompaniesSearcher < Base
|
|
20
20
|
|
21
21
|
end
|
22
22
|
|
23
|
+
#find people from a company, based off the user's network and connections
|
24
|
+
def find_people_for_company linkedin_client, company_name, location
|
25
|
+
|
26
|
+
puts linkedin_client.query_people_from_company company_name, location
|
27
|
+
|
28
|
+
end
|
29
|
+
|
23
30
|
def zip_code_search zip_code
|
24
31
|
|
25
32
|
zip_doc = @coords_coll.find_one({:zip => zip_code})
|
@@ -20,6 +20,7 @@ class DataPopulators
|
|
20
20
|
require 'gather_companies'
|
21
21
|
require 'tech_ad_tagger'
|
22
22
|
require 'base_api_client'
|
23
|
+
require "linkedin_client"
|
23
24
|
|
24
25
|
end
|
25
26
|
|
@@ -54,7 +55,13 @@ class DataPopulators
|
|
54
55
|
|
55
56
|
begin
|
56
57
|
|
57
|
-
|
58
|
+
#@@gather_companies.load_companies_from_indeed
|
59
|
+
|
60
|
+
@li_config = YAML.load_file(File.expand_path("../../config/linkedin.env", __FILE__))
|
61
|
+
|
62
|
+
@linkedin_client = LinkedinClient.new @li_config["api_key"], @li_config["api_secret"], @li_config["user_token"], @li_config["user_secret"], @li_config["url"]
|
63
|
+
|
64
|
+
@@companies_searcher.find_people_for_company @linkedin_client, "apple", "san jose, ca"
|
58
65
|
|
59
66
|
rescue Exception => e
|
60
67
|
puts e.message
|
@@ -94,7 +101,7 @@ class DataPopulators
|
|
94
101
|
|
95
102
|
begin
|
96
103
|
|
97
|
-
|
104
|
+
# @@tech_ad_tagger.tag_company_with_technologies @@programming_languages, generate_duckduckgo_url
|
98
105
|
|
99
106
|
rescue Exception => e
|
100
107
|
puts e.message
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: whos_using_what
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.0.
|
4
|
+
version: 1.0.1
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -165,6 +165,7 @@ files:
|
|
165
165
|
- lib/whos_using_what/util/map_data_extraction_util.rb
|
166
166
|
- lib/whos_using_what/logging/logger_factory.rb
|
167
167
|
- lib/whos_using_what/scripts/data_populators.rb
|
168
|
+
- lib/whos_using_what/api_clients/indeed_api_client.rb
|
168
169
|
- lib/whos_using_what/api_clients/google_client.rb
|
169
170
|
- lib/whos_using_what/api_clients/base_api_client.rb
|
170
171
|
- lib/whos_using_what/api_clients/linkedin_client.rb
|