gsc-cli 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/LICENSE +21 -0
- data/README.md +500 -0
- data/bin/gsc +8128 -0
- data/dist/gsc +8128 -0
- data/lib/gsc/api.rb +300 -0
- data/lib/gsc/auth.rb +92 -0
- data/lib/gsc/cli.rb +5435 -0
- data/lib/gsc/client.rb +79 -0
- data/lib/gsc/color.rb +22 -0
- data/lib/gsc/command_registry.rb +335 -0
- data/lib/gsc/config.rb +178 -0
- data/lib/gsc/google_trends.rb +187 -0
- data/lib/gsc/keyword_planner.rb +256 -0
- data/lib/gsc/keywords_everywhere.rb +144 -0
- data/lib/gsc/page_analyzer.rb +431 -0
- data/lib/gsc/prompts.rb +285 -0
- data/lib/gsc/site_crawler.rb +260 -0
- data/lib/gsc/sitemap_loader.rb +91 -0
- data/lib/gsc/version.rb +5 -0
- data/lib/gsc.rb +38 -0
- metadata +67 -0
|
@@ -0,0 +1,187 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module GSC
|
|
4
|
+
class GoogleTrends
|
|
5
|
+
TRENDS_HOST = 'trends.google.com'
|
|
6
|
+
|
|
7
|
+
def self.normalize_time(time_str)
|
|
8
|
+
case time_str.to_s.strip.downcase
|
|
9
|
+
when '5y', '5-y', 'past-5-years', '5years' then 'today 5-y'
|
|
10
|
+
when '12m', '1y', 'past-12-months', '12months', '1year' then 'today 12-m'
|
|
11
|
+
when '3m', 'past-3-months', '3months' then 'today 3-m'
|
|
12
|
+
when '1m', 'past-month', '30d' then 'today 1-m'
|
|
13
|
+
when '7d', 'past-week' then 'now 7-d'
|
|
14
|
+
when '1d', '24h', 'today' then 'now 1-d'
|
|
15
|
+
when 'all' then 'all'
|
|
16
|
+
else
|
|
17
|
+
time_str.to_s.strip.empty? ? 'today 5-y' : time_str.to_s.strip
|
|
18
|
+
end
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
def self.render_sparkline(values, max_points: 36)
|
|
22
|
+
return '' if values.nil? || values.empty?
|
|
23
|
+
bucketed = if values.size > max_points
|
|
24
|
+
chunk_size = (values.size.to_f / max_points).ceil
|
|
25
|
+
values.each_slice(chunk_size).map { |slice| (slice.sum.to_f / slice.size).round }
|
|
26
|
+
else
|
|
27
|
+
values
|
|
28
|
+
end
|
|
29
|
+
min_val = bucketed.min.to_f
|
|
30
|
+
max_val = bucketed.max.to_f
|
|
31
|
+
range = max_val - min_val
|
|
32
|
+
range = 1.0 if range.zero?
|
|
33
|
+
sparks = ["\u2581", "\u2582", "\u2583", "\u2584", "\u2585", "\u2586", "\u2587", "\u2588"]
|
|
34
|
+
bucketed.map do |v|
|
|
35
|
+
idx = [((v - min_val) / range * (sparks.length - 1)).round, sparks.length - 1].min
|
|
36
|
+
sparks[idx]
|
|
37
|
+
end.join
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
def self.fetch(keyword, geo: 'US', time: '5y')
|
|
41
|
+
http = Net::HTTP.new(TRENDS_HOST, 443)
|
|
42
|
+
http.use_ssl = true
|
|
43
|
+
http.open_timeout = 10
|
|
44
|
+
http.read_timeout = 25
|
|
45
|
+
|
|
46
|
+
init_req = Net::HTTP::Get.new('/trends/explore')
|
|
47
|
+
init_req['User-Agent'] = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'
|
|
48
|
+
init_res = http.request(init_req)
|
|
49
|
+
cookies = init_res.get_fields('set-cookie')&.map { |c| c.split(';').first }&.join('; ')
|
|
50
|
+
|
|
51
|
+
norm_time = normalize_time(time)
|
|
52
|
+
clean_geo = geo.to_s.upcase == 'WORLDWIDE' ? '' : geo.to_s.strip
|
|
53
|
+
req_obj = {
|
|
54
|
+
comparisonItem: [{ keyword: keyword, geo: clean_geo, time: norm_time }],
|
|
55
|
+
category: 0,
|
|
56
|
+
property: ''
|
|
57
|
+
}
|
|
58
|
+
explore_path = '/trends/api/explore?hl=en-US&tz=-180&req=' + URI.encode_www_form_component(JSON.generate(req_obj))
|
|
59
|
+
explore_req = Net::HTTP::Get.new(explore_path)
|
|
60
|
+
explore_req['User-Agent'] = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'
|
|
61
|
+
explore_req['Accept'] = 'application/json, text/plain, */*'
|
|
62
|
+
explore_req['Referer'] = 'https://trends.google.com/trends/explore'
|
|
63
|
+
explore_req['Cookie'] = cookies if cookies
|
|
64
|
+
|
|
65
|
+
explore_res = http.request(explore_req)
|
|
66
|
+
return { ok: false, error: "Google Trends Explore API error (HTTP #{explore_res.code})" } unless explore_res.is_a?(Net::HTTPSuccess)
|
|
67
|
+
|
|
68
|
+
raw_exp = explore_res.body.to_s
|
|
69
|
+
clean_body = raw_exp.index('{') ? raw_exp[raw_exp.index('{')..] : raw_exp
|
|
70
|
+
explore_data = JSON.parse(clean_body) rescue nil
|
|
71
|
+
widgets = explore_data&.dig('widgets') || []
|
|
72
|
+
|
|
73
|
+
ts_widget = widgets.find { |w| w['id'] == 'TIMESERIES' }
|
|
74
|
+
geo_widget = widgets.find { |w| w['id'] == 'GEO_MAP' }
|
|
75
|
+
queries_widget = widgets.find { |w| w['id'] == 'RELATED_QUERIES' }
|
|
76
|
+
|
|
77
|
+
timeline = []
|
|
78
|
+
if ts_widget
|
|
79
|
+
ts_path = '/trends/api/widgetdata/multiline?hl=en-US&tz=-180&req=' + URI.encode_www_form_component(JSON.generate(ts_widget['request'])) + '&token=' + ts_widget['token']
|
|
80
|
+
ts_req = Net::HTTP::Get.new(ts_path)
|
|
81
|
+
ts_req['User-Agent'] = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'
|
|
82
|
+
ts_req['Accept'] = 'application/json, text/plain, */*'
|
|
83
|
+
ts_req['Referer'] = 'https://trends.google.com/trends/explore'
|
|
84
|
+
ts_req['Cookie'] = cookies if cookies
|
|
85
|
+
ts_res = http.request(ts_req)
|
|
86
|
+
if ts_res.is_a?(Net::HTTPSuccess)
|
|
87
|
+
raw_ts = ts_res.body.to_s
|
|
88
|
+
ts_clean = raw_ts.index('{') ? raw_ts[raw_ts.index('{')..] : raw_ts
|
|
89
|
+
ts_data = JSON.parse(ts_clean) rescue {}
|
|
90
|
+
timeline = ts_data.dig('default', 'timelineData') || []
|
|
91
|
+
end
|
|
92
|
+
end
|
|
93
|
+
|
|
94
|
+
regions = []
|
|
95
|
+
if geo_widget
|
|
96
|
+
geo_path = '/trends/api/widgetdata/comparedgeo?hl=en-US&tz=-180&req=' + URI.encode_www_form_component(JSON.generate(geo_widget['request'])) + '&token=' + geo_widget['token']
|
|
97
|
+
geo_req = Net::HTTP::Get.new(geo_path)
|
|
98
|
+
geo_req['User-Agent'] = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'
|
|
99
|
+
geo_req['Referer'] = 'https://trends.google.com/trends/explore'
|
|
100
|
+
geo_req['Cookie'] = cookies if cookies
|
|
101
|
+
geo_res = http.request(geo_req)
|
|
102
|
+
if geo_res.is_a?(Net::HTTPSuccess)
|
|
103
|
+
raw_geo = geo_res.body.to_s
|
|
104
|
+
geo_clean = raw_geo.index('{') ? raw_geo[raw_geo.index('{')..] : raw_geo
|
|
105
|
+
geo_data = JSON.parse(geo_clean) rescue {}
|
|
106
|
+
raw_regions = geo_data.dig('default', 'geoMapData') || []
|
|
107
|
+
regions = raw_regions.map do |r|
|
|
108
|
+
{ name: r['geoName'], score: (r['value']&.first || 0) }
|
|
109
|
+
end.sort_by { |r| -r[:score] }.reject { |r| r[:score].zero? }
|
|
110
|
+
end
|
|
111
|
+
end
|
|
112
|
+
|
|
113
|
+
top_queries = []
|
|
114
|
+
rising_queries = []
|
|
115
|
+
if queries_widget
|
|
116
|
+
q_path = '/trends/api/widgetdata/relatedsearches?hl=en-US&tz=-180&req=' + URI.encode_www_form_component(JSON.generate(queries_widget['request'])) + '&token=' + queries_widget['token']
|
|
117
|
+
q_req = Net::HTTP::Get.new(q_path)
|
|
118
|
+
q_req['User-Agent'] = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'
|
|
119
|
+
q_req['Accept'] = 'application/json, text/plain, */*'
|
|
120
|
+
q_req['Referer'] = 'https://trends.google.com/trends/explore'
|
|
121
|
+
q_req['Cookie'] = cookies if cookies
|
|
122
|
+
q_res = http.request(q_req)
|
|
123
|
+
if q_res.is_a?(Net::HTTPSuccess)
|
|
124
|
+
raw_q = q_res.body.to_s
|
|
125
|
+
q_clean = raw_q.index('{') ? raw_q[raw_q.index('{')..] : raw_q
|
|
126
|
+
q_data = JSON.parse(q_clean) rescue {}
|
|
127
|
+
ranked = q_data.dig('default', 'rankedList') || []
|
|
128
|
+
top_queries = (ranked.first&.dig('rankedKeyword') || []).map do |q|
|
|
129
|
+
{ query: q['query'], score: q['value'] }
|
|
130
|
+
end
|
|
131
|
+
rising_queries = (ranked.last&.dig('rankedKeyword') || []).map do |q|
|
|
132
|
+
{ query: q['query'], growth: q['formattedValue'] }
|
|
133
|
+
end
|
|
134
|
+
end
|
|
135
|
+
end
|
|
136
|
+
|
|
137
|
+
points = timeline.map do |pt|
|
|
138
|
+
{
|
|
139
|
+
date: pt['formattedTime'] || pt['formattedAxisTime'],
|
|
140
|
+
score: (pt['value']&.first || 0)
|
|
141
|
+
}
|
|
142
|
+
end
|
|
143
|
+
|
|
144
|
+
scores = points.map { |p| p[:score] }
|
|
145
|
+
peak_val = scores.max || 0
|
|
146
|
+
peak_pt = points.find { |p| p[:score] == peak_val }
|
|
147
|
+
curr_val = scores.last || 0
|
|
148
|
+
|
|
149
|
+
split = [points.size / 5, 4].max
|
|
150
|
+
baseline = points.first(split).map { |p| p[:score] }.sum.to_f / split
|
|
151
|
+
recent = points.last(split).map { |p| p[:score] }.sum.to_f / split
|
|
152
|
+
growth_pct = baseline > 0 ? (((recent - baseline) / baseline) * 100).round : 0
|
|
153
|
+
|
|
154
|
+
velocity_badge = if growth_pct >= 250
|
|
155
|
+
"+#{growth_pct}% 🚀 (Explosive Breakout)"
|
|
156
|
+
elsif growth_pct >= 80
|
|
157
|
+
"+#{growth_pct}% 🔥 (Strong Surging)"
|
|
158
|
+
elsif growth_pct >= 15
|
|
159
|
+
"+#{growth_pct}% 📈 (Growing Demand)"
|
|
160
|
+
elsif growth_pct >= -15
|
|
161
|
+
"#{growth_pct}% ⚖️ (Stable Demand)"
|
|
162
|
+
else
|
|
163
|
+
"#{growth_pct}% 📉 (Cooling / Declining)"
|
|
164
|
+
end
|
|
165
|
+
|
|
166
|
+
{
|
|
167
|
+
ok: true,
|
|
168
|
+
keyword: keyword,
|
|
169
|
+
geo: clean_geo.empty? ? 'Worldwide' : clean_geo,
|
|
170
|
+
time: norm_time,
|
|
171
|
+
peak_score: peak_val,
|
|
172
|
+
peak_date: peak_pt ? peak_pt[:date] : nil,
|
|
173
|
+
current_score: curr_val,
|
|
174
|
+
growth_pct: growth_pct,
|
|
175
|
+
velocity_badge: velocity_badge,
|
|
176
|
+
sparkline: render_sparkline(scores),
|
|
177
|
+
points_count: points.size,
|
|
178
|
+
regions: regions.first(10),
|
|
179
|
+
top_queries: top_queries.first(8),
|
|
180
|
+
rising_queries: rising_queries.first(8),
|
|
181
|
+
points: points
|
|
182
|
+
}
|
|
183
|
+
rescue StandardError => e
|
|
184
|
+
{ ok: false, error: e.message }
|
|
185
|
+
end
|
|
186
|
+
end
|
|
187
|
+
end
|
|
@@ -0,0 +1,256 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module GSC
|
|
4
|
+
class KeywordPlanner
|
|
5
|
+
SUGGEST_HOST = 'suggestqueries.google.com'
|
|
6
|
+
|
|
7
|
+
def self.calculate_opportunity(volume, competition)
|
|
8
|
+
vol = [volume.to_i, 0].max
|
|
9
|
+
comp = [[competition.to_f, 0.0].max, 1.0].min
|
|
10
|
+
return 0 if vol.zero?
|
|
11
|
+
|
|
12
|
+
log_vol = Math.log10(vol + 10)
|
|
13
|
+
vol_score = [log_vol * 10.0, 50.0].min
|
|
14
|
+
comp_score = (1.0 - comp) * 50.0
|
|
15
|
+
(vol_score + comp_score).round
|
|
16
|
+
end
|
|
17
|
+
|
|
18
|
+
def self.classify_intent(query)
|
|
19
|
+
q = query.to_s.downcase
|
|
20
|
+
if q =~ /\b(how|what|why|guide|tips|symptoms|remedies|causes|when|is it|can i)\b/
|
|
21
|
+
'Informational'
|
|
22
|
+
elsif q =~ /\b(best|review|reviews|top|vs|compare|alternative|alternatives|cost|pricing)\b/
|
|
23
|
+
'Commercial'
|
|
24
|
+
elsif q =~ /\b(buy|order|hire|cheap|service|services|near me|discount|coupon|deal|store|shop)\b/
|
|
25
|
+
'Transactional'
|
|
26
|
+
else
|
|
27
|
+
'Navigational'
|
|
28
|
+
end
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
def self.fetch_suggest(seed, country: 'us')
|
|
32
|
+
encoded = URI.encode_www_form_component(seed)
|
|
33
|
+
uri = URI("https://#{SUGGEST_HOST}/complete/search?client=chrome&hl=en&gl=#{country}&q=#{encoded}")
|
|
34
|
+
http = Net::HTTP.new(uri.host, uri.port)
|
|
35
|
+
http.use_ssl = true
|
|
36
|
+
http.open_timeout = 5
|
|
37
|
+
http.read_timeout = 10
|
|
38
|
+
req = Net::HTTP::Get.new(uri)
|
|
39
|
+
req['User-Agent'] = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'
|
|
40
|
+
res = http.request(req)
|
|
41
|
+
return [] unless res.is_a?(Net::HTTPSuccess)
|
|
42
|
+
data = JSON.parse(res.body) rescue nil
|
|
43
|
+
(data && data[1].is_a?(Array)) ? data[1] : []
|
|
44
|
+
rescue StandardError
|
|
45
|
+
[]
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
def self.expand(seed, country: 'us', limit: 50)
|
|
49
|
+
queries = []
|
|
50
|
+
queries.concat(fetch_suggest(seed, country: country))
|
|
51
|
+
|
|
52
|
+
modifiers = [
|
|
53
|
+
"how to #{seed}",
|
|
54
|
+
"best #{seed}",
|
|
55
|
+
"#{seed} reviews",
|
|
56
|
+
"#{seed} vs",
|
|
57
|
+
"#{seed} alternatives",
|
|
58
|
+
"cheap #{seed}",
|
|
59
|
+
"#{seed} near me"
|
|
60
|
+
]
|
|
61
|
+
modifiers.each do |mod|
|
|
62
|
+
queries.concat(fetch_suggest(mod, country: country))
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
%w[a b c d f h m p s t w].each do |char|
|
|
66
|
+
queries.concat(fetch_suggest("#{seed} #{char}", country: country))
|
|
67
|
+
end
|
|
68
|
+
|
|
69
|
+
clean_list = queries.map(&:strip).reject(&:empty?).uniq.first(limit)
|
|
70
|
+
clean_list.map do |q|
|
|
71
|
+
{
|
|
72
|
+
keyword: q,
|
|
73
|
+
intent: classify_intent(q)
|
|
74
|
+
}
|
|
75
|
+
end
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
def self.render_sparkline(values, max_points: 12)
|
|
79
|
+
return "" if values.nil? || values.empty?
|
|
80
|
+
sparks = ["\u2581", "\u2582", "\u2583", "\u2584", "\u2585", "\u2586", "\u2587", "\u2588"]
|
|
81
|
+
vals = values.last(max_points)
|
|
82
|
+
min_val = vals.min.to_f
|
|
83
|
+
max_val = vals.max.to_f
|
|
84
|
+
range = max_val - min_val
|
|
85
|
+
if range <= 0
|
|
86
|
+
return sparks[3] * vals.length
|
|
87
|
+
end
|
|
88
|
+
vals.map do |v|
|
|
89
|
+
idx = [((v - min_val) / range * (sparks.length - 1)).round, sparks.length - 1].min
|
|
90
|
+
sparks[idx]
|
|
91
|
+
end.join
|
|
92
|
+
end
|
|
93
|
+
|
|
94
|
+
def self.read_clipboard
|
|
95
|
+
if RUBY_PLATFORM =~ /darwin/i
|
|
96
|
+
`pbpaste 2>/dev/null`
|
|
97
|
+
elsif system('which xclip > /dev/null 2>&1')
|
|
98
|
+
`xclip -selection clipboard -o 2>/dev/null`
|
|
99
|
+
elsif system('which wl-paste > /dev/null 2>&1')
|
|
100
|
+
`wl-paste 2>/dev/null`
|
|
101
|
+
else
|
|
102
|
+
nil
|
|
103
|
+
end
|
|
104
|
+
end
|
|
105
|
+
|
|
106
|
+
def self.import_file(source)
|
|
107
|
+
src_str = source.to_s.strip
|
|
108
|
+
raw_text = if ['clipboard', 'clip', 'paste', 'pbpaste'].include?(src_str.downcase)
|
|
109
|
+
txt = read_clipboard
|
|
110
|
+
raise "Clipboard is empty or could not be read. Copy a table from Keywords Everywhere and run: gsc import clip" if txt.nil? || txt.strip.empty?
|
|
111
|
+
txt
|
|
112
|
+
elsif src_str == '-'
|
|
113
|
+
$stdin.read
|
|
114
|
+
else
|
|
115
|
+
raise "File does not exist: #{src_str}" unless File.exist?(src_str)
|
|
116
|
+
File.read(src_str, encoding: 'UTF-8')
|
|
117
|
+
end
|
|
118
|
+
|
|
119
|
+
raw_lines = raw_text.lines.map(&:strip).reject(&:empty?)
|
|
120
|
+
raise "Import content is empty." if raw_lines.empty?
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
sample = raw_lines.find { |l| l =~ /keyword/i } || raw_lines.first
|
|
124
|
+
delim = if sample.count("\t") > 2
|
|
125
|
+
"\t"
|
|
126
|
+
elsif sample.count(',') > 2
|
|
127
|
+
','
|
|
128
|
+
elsif sample.include?('|')
|
|
129
|
+
'|'
|
|
130
|
+
else
|
|
131
|
+
"\t"
|
|
132
|
+
end
|
|
133
|
+
|
|
134
|
+
header_idx = raw_lines.find_index { |l| l =~ /keyword|query/i } || 0
|
|
135
|
+
header_line = raw_lines[header_idx]
|
|
136
|
+
headers = header_line.split(delim).map { |h| h.strip.gsub(/[`*"]/, '') }
|
|
137
|
+
|
|
138
|
+
kw_col = headers.find_index { |h| h =~ /^(keyword|query|search query)$/i } || 0
|
|
139
|
+
vol_col = headers.find_index { |h| h =~ /(search volume|avg.*monthly searches|volume)/i }
|
|
140
|
+
cpc_col = headers.find_index { |h| h =~ /(cpc|top of page bid)/i }
|
|
141
|
+
comp_col = headers.find_index { |h| h =~ /^(competition|competition \(indexed value\))$/i }
|
|
142
|
+
tier_col = headers.find_index { |h| h =~ /(tier|competition level)/i }
|
|
143
|
+
trend_col = headers.find_index { |h| h =~ /(trend.*%|three month change|yoy)/i }
|
|
144
|
+
|
|
145
|
+
# Detect any month columns (e.g. from Keywords Everywhere 12-month export)
|
|
146
|
+
month_cols = {}
|
|
147
|
+
headers.each_with_index do |h, idx|
|
|
148
|
+
if h =~ /^(jan|feb|mar|apr|may|jun|jul|aug|sep|oct|nov|dec)\s+\d{4}$/i || h =~ /^\d{4}[-\/]\d{2}$/
|
|
149
|
+
month_cols[h] = idx
|
|
150
|
+
end
|
|
151
|
+
end
|
|
152
|
+
|
|
153
|
+
data_rows = raw_lines[(header_idx + 1)..]
|
|
154
|
+
results = []
|
|
155
|
+
seen = {}
|
|
156
|
+
|
|
157
|
+
data_rows.each do |line|
|
|
158
|
+
next if line =~ /^\|?[- :|]+\|?$/
|
|
159
|
+
|
|
160
|
+
parts = line.split(delim).map { |p| p.strip.gsub(/[`*"]/, '') }
|
|
161
|
+
kw = parts[kw_col]
|
|
162
|
+
next if kw.nil? || kw.empty? || kw =~ /^(keyword|query|---|===)$/i
|
|
163
|
+
next if kw.downcase == 'keyword'
|
|
164
|
+
|
|
165
|
+
vol = vol_col ? parts[vol_col].to_s.gsub(/[,+ ]/, '').to_i : 0
|
|
166
|
+
cpc_raw = cpc_col ? parts[cpc_col].to_s.strip : '$0.00'
|
|
167
|
+
comp = comp_col ? parts[comp_col].to_s.to_f : 0.0
|
|
168
|
+
comp_tier = tier_col ? parts[tier_col].to_s.strip : nil
|
|
169
|
+
if comp_tier.nil? || comp_tier.empty?
|
|
170
|
+
comp_tier = if comp < 0.30 then 'Low'
|
|
171
|
+
elsif comp < 0.70 then 'Medium'
|
|
172
|
+
else 'High'
|
|
173
|
+
end
|
|
174
|
+
end
|
|
175
|
+
trend = trend_col ? parts[trend_col].to_s.sub(/%/, '').to_i : 0
|
|
176
|
+
opp_score = calculate_opportunity(vol, comp)
|
|
177
|
+
intent = classify_intent(kw)
|
|
178
|
+
|
|
179
|
+
history = {}
|
|
180
|
+
month_cols.each do |m_name, c_idx|
|
|
181
|
+
m_val = parts[c_idx].to_s.gsub(/[,+ ]/, '')
|
|
182
|
+
history[m_name] = m_val.to_i unless m_val.empty?
|
|
183
|
+
end
|
|
184
|
+
|
|
185
|
+
kw_key = kw.downcase.strip
|
|
186
|
+
if seen[kw_key]
|
|
187
|
+
existing = seen[kw_key]
|
|
188
|
+
if vol > existing[:volume]
|
|
189
|
+
existing[:volume] = vol
|
|
190
|
+
existing[:cpc] = cpc_raw if cpc_raw != '$0.00'
|
|
191
|
+
existing[:competition] = comp
|
|
192
|
+
existing[:competition_tier] = comp_tier
|
|
193
|
+
existing[:trend_pct] = trend
|
|
194
|
+
existing[:opportunity_score] = opp_score
|
|
195
|
+
end
|
|
196
|
+
existing[:monthly_history] = history.merge(existing[:monthly_history] || {}) unless history.empty?
|
|
197
|
+
else
|
|
198
|
+
entry = {
|
|
199
|
+
keyword: kw,
|
|
200
|
+
volume: vol,
|
|
201
|
+
cpc: cpc_raw,
|
|
202
|
+
competition: comp,
|
|
203
|
+
competition_tier: comp_tier,
|
|
204
|
+
trend_pct: trend,
|
|
205
|
+
opportunity_score: opp_score,
|
|
206
|
+
intent: intent
|
|
207
|
+
}
|
|
208
|
+
if !history.empty?
|
|
209
|
+
entry[:monthly_history] = history
|
|
210
|
+
entry[:sparkline] = render_sparkline(history.values)
|
|
211
|
+
end
|
|
212
|
+
seen[kw_key] = entry
|
|
213
|
+
results << entry
|
|
214
|
+
end
|
|
215
|
+
end
|
|
216
|
+
|
|
217
|
+
results.sort_by { |r| [-r[:opportunity_score], -r[:volume]] }
|
|
218
|
+
end
|
|
219
|
+
|
|
220
|
+
def self.correlate_with_gsc(keywords, api, site_url, days: 30)
|
|
221
|
+
return keywords if api.nil? || site_url.nil?
|
|
222
|
+
|
|
223
|
+
res = api.query_analytics(site_url, days: days, dimensions: ['query'], row_limit: 5000)
|
|
224
|
+
return keywords unless res[:ok]
|
|
225
|
+
|
|
226
|
+
gsc_map = {}
|
|
227
|
+
(res.dig(:data, 'rows') || []).each do |r|
|
|
228
|
+
q = r['keys'].first.to_s.downcase.strip
|
|
229
|
+
gsc_map[q] = {
|
|
230
|
+
clicks: r['clicks'],
|
|
231
|
+
impressions: r['impressions'],
|
|
232
|
+
ctr: (r['ctr'] * 100).round(1),
|
|
233
|
+
position: r['position'].round(1)
|
|
234
|
+
}
|
|
235
|
+
end
|
|
236
|
+
|
|
237
|
+
keywords.map do |kw_item|
|
|
238
|
+
item = kw_item.dup
|
|
239
|
+
q = item[:keyword].to_s.downcase.strip
|
|
240
|
+
if gsc_map[q]
|
|
241
|
+
g = gsc_map[q]
|
|
242
|
+
pos = g[:position]
|
|
243
|
+
status = if pos <= 3.0 then "🏆 Top 3 (Pos #{pos})"
|
|
244
|
+
elsif pos <= 10.0 then "🥇 Page 1 (Pos #{pos})"
|
|
245
|
+
elsif pos <= 20.0 then "🎯 Striking Distance (Pos #{pos})"
|
|
246
|
+
else "🔍 Deep SERP (Pos #{pos})"
|
|
247
|
+
end
|
|
248
|
+
item[:gsc] = g.merge(status: status)
|
|
249
|
+
else
|
|
250
|
+
item[:gsc] = { clicks: 0, impressions: 0, ctr: 0.0, position: 0.0, status: '🚀 Untargeted' }
|
|
251
|
+
end
|
|
252
|
+
item
|
|
253
|
+
end
|
|
254
|
+
end
|
|
255
|
+
end
|
|
256
|
+
end
|
|
@@ -0,0 +1,144 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module GSC
|
|
4
|
+
class KeywordsEverywhere
|
|
5
|
+
API_HOST = "api.keywordseverywhere.com"
|
|
6
|
+
|
|
7
|
+
def self.api_key
|
|
8
|
+
Config.keywords_everywhere_api_key
|
|
9
|
+
end
|
|
10
|
+
|
|
11
|
+
def self.check_account(key = nil)
|
|
12
|
+
k = (key && !key.to_s.strip.empty?) ? key.to_s.strip : api_key
|
|
13
|
+
return { ok: false, error: "Keywords Everywhere API key not configured. Run: gsc connect ke" } if k.nil? || k.empty?
|
|
14
|
+
|
|
15
|
+
uri = URI("https://#{API_HOST}/v1/account/credits")
|
|
16
|
+
http = Net::HTTP.new(uri.host, uri.port)
|
|
17
|
+
http.use_ssl = true
|
|
18
|
+
http.open_timeout = 6
|
|
19
|
+
http.read_timeout = 12
|
|
20
|
+
|
|
21
|
+
req = Net::HTTP::Get.new(uri)
|
|
22
|
+
req["Authorization"] = "Bearer #{k}"
|
|
23
|
+
req["Accept"] = "application/json"
|
|
24
|
+
req["User-Agent"] = "gsc-cli/#{GSC::VERSION}"
|
|
25
|
+
|
|
26
|
+
res = http.request(req)
|
|
27
|
+
parsed = begin
|
|
28
|
+
JSON.parse(res.body)
|
|
29
|
+
rescue StandardError
|
|
30
|
+
res.body
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
if res.is_a?(Net::HTTPSuccess)
|
|
34
|
+
# Keywords Everywhere /v1/account/credits returns an array with remaining credits as the only element: [ 95597755 ]
|
|
35
|
+
# Some endpoints or proxies may return a Hash: {"credits": 95597755}
|
|
36
|
+
credits = if parsed.is_a?(Array)
|
|
37
|
+
parsed.first.is_a?(Hash) ? (parsed.first["credits"] || parsed.first["data"] || 0) : parsed.first.to_i
|
|
38
|
+
elsif parsed.is_a?(Hash)
|
|
39
|
+
parsed["credits"] || parsed["data"] || parsed["credits_left"] || 0
|
|
40
|
+
else
|
|
41
|
+
parsed.to_i
|
|
42
|
+
end
|
|
43
|
+
credits = credits.first if credits.is_a?(Array)
|
|
44
|
+
{ ok: true, credits: credits.to_i }
|
|
45
|
+
else
|
|
46
|
+
err_msg = if parsed.is_a?(Hash)
|
|
47
|
+
parsed["message"] || parsed["error"] || parsed["description"] || "HTTP #{res.code}"
|
|
48
|
+
elsif parsed.is_a?(Array)
|
|
49
|
+
parsed.map { |x| x.is_a?(Hash) ? (x["message"] || x["error"]) : x.to_s }.join(", ")
|
|
50
|
+
else
|
|
51
|
+
"HTTP #{res.code}: #{parsed}"
|
|
52
|
+
end
|
|
53
|
+
{ ok: false, error: err_msg }
|
|
54
|
+
end
|
|
55
|
+
rescue StandardError => e
|
|
56
|
+
{ ok: false, error: e.message }
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
def self.fetch_data(keywords, country: "us", currency: "usd", data_source: "gkp", key: nil)
|
|
60
|
+
k = (key && !key.to_s.strip.empty?) ? key.to_s.strip : api_key
|
|
61
|
+
return { ok: false, error: "Keywords Everywhere API key not configured. Run: gsc connect ke" } if k.nil? || k.empty?
|
|
62
|
+
|
|
63
|
+
kw_list = Array(keywords).map(&:to_s).map(&:strip).reject(&:empty?).uniq
|
|
64
|
+
return { ok: true, count: 0, data: [] } if kw_list.empty?
|
|
65
|
+
|
|
66
|
+
results = []
|
|
67
|
+
# Batch up to 100 keywords per request
|
|
68
|
+
kw_list.each_slice(100) do |slice|
|
|
69
|
+
uri = URI("https://#{API_HOST}/v1/get_keyword_data")
|
|
70
|
+
http = Net::HTTP.new(uri.host, uri.port)
|
|
71
|
+
http.use_ssl = true
|
|
72
|
+
http.open_timeout = 10
|
|
73
|
+
http.read_timeout = 30
|
|
74
|
+
|
|
75
|
+
req = Net::HTTP::Post.new(uri)
|
|
76
|
+
req["Authorization"] = "Bearer #{k}"
|
|
77
|
+
req["Accept"] = "application/json"
|
|
78
|
+
req["User-Agent"] = "gsc-cli/#{GSC::VERSION}"
|
|
79
|
+
req["Content-Type"] = "application/x-www-form-urlencoded"
|
|
80
|
+
|
|
81
|
+
form = {
|
|
82
|
+
"country" => country.to_s.downcase,
|
|
83
|
+
"currency" => currency.to_s.downcase,
|
|
84
|
+
"dataSource" => data_source.to_s.downcase
|
|
85
|
+
}
|
|
86
|
+
base_encoded = URI.encode_www_form(form)
|
|
87
|
+
kw_params = slice.map { |kw| "kw[]=#{URI.encode_www_form_component(kw)}" }.join("&")
|
|
88
|
+
req.body = "#{base_encoded}&#{kw_params}"
|
|
89
|
+
|
|
90
|
+
res = http.request(req)
|
|
91
|
+
parsed = begin
|
|
92
|
+
JSON.parse(res.body)
|
|
93
|
+
rescue StandardError
|
|
94
|
+
res.body
|
|
95
|
+
end
|
|
96
|
+
|
|
97
|
+
if res.is_a?(Net::HTTPSuccess)
|
|
98
|
+
raw_items = if parsed.is_a?(Hash)
|
|
99
|
+
parsed["data"] || []
|
|
100
|
+
elsif parsed.is_a?(Array)
|
|
101
|
+
parsed
|
|
102
|
+
else
|
|
103
|
+
[]
|
|
104
|
+
end
|
|
105
|
+
|
|
106
|
+
raw_items.each do |item|
|
|
107
|
+
next unless item.is_a?(Hash)
|
|
108
|
+
vol = item["vol"].to_i
|
|
109
|
+
cpc_val = (item["cpc"].is_a?(Hash) ? item.dig("cpc", "value") : item["cpc"]).to_f
|
|
110
|
+
currency_sym = (item["cpc"].is_a?(Hash) ? item.dig("cpc", "currency") : "$") || "$"
|
|
111
|
+
comp = item["competition"].to_f
|
|
112
|
+
trend = item["trend"] || []
|
|
113
|
+
opp_score = KeywordPlanner.calculate_opportunity(vol, comp)
|
|
114
|
+
intent = KeywordPlanner.classify_intent(item["keyword"])
|
|
115
|
+
|
|
116
|
+
results << {
|
|
117
|
+
keyword: item["keyword"],
|
|
118
|
+
volume: vol,
|
|
119
|
+
cpc: cpc_val,
|
|
120
|
+
currency: currency_sym,
|
|
121
|
+
competition: comp,
|
|
122
|
+
opportunity_score: opp_score,
|
|
123
|
+
intent: intent,
|
|
124
|
+
trend: trend
|
|
125
|
+
}
|
|
126
|
+
end
|
|
127
|
+
else
|
|
128
|
+
err = if parsed.is_a?(Hash)
|
|
129
|
+
parsed["message"] || parsed["error"] || "HTTP #{res.code}"
|
|
130
|
+
elsif parsed.is_a?(Array)
|
|
131
|
+
parsed.map { |x| x.is_a?(Hash) ? (x["message"] || x["error"]) : x.to_s }.join(", ")
|
|
132
|
+
else
|
|
133
|
+
"HTTP #{res.code}"
|
|
134
|
+
end
|
|
135
|
+
return { ok: false, error: "Keywords Everywhere API error: #{err}" }
|
|
136
|
+
end
|
|
137
|
+
end
|
|
138
|
+
|
|
139
|
+
{ ok: true, count: results.size, data: results }
|
|
140
|
+
rescue StandardError => e
|
|
141
|
+
{ ok: false, error: e.message }
|
|
142
|
+
end
|
|
143
|
+
end
|
|
144
|
+
end
|