google-maps-scraper-sdk 0.1.1 → 0.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +23 -10
- data/lib/gmaps_scraper/client.rb +79 -7
- data/lib/gmaps_scraper/version.rb +1 -1
- metadata +2 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 13c138a6dbce983ae0c352135acdf2fed8badb8df956dbef3eed002c210a21b6
|
|
4
|
+
data.tar.gz: 9030a395a1c32e0869dd85adcd39a05365d750f3fe06b6d4262708c905a89d20
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: dedb53fcc0aaca6bef2ff8e675b1fc27fe826a54c1fa9ea48d6416789209d7a99813d4713ab60447a14e609438254ca42862d14590416ef836f8ac306a49a379
|
|
7
|
+
data.tar.gz: 450263e5973986600e611df385d7915a834e2e6189de2ccf9b08698bb435550902ab7b364f2d89fbd15dcc21d210eaca7f46eeebb68476f97c654296c0ada741
|
data/README.md
CHANGED
|
@@ -6,7 +6,7 @@
|
|
|
6
6
|
|
|
7
7
|
**Google Maps Extractor · Google Maps Lead Scraper · Google Maps Lead Extractor**
|
|
8
8
|
|
|
9
|
-
Ruby SDK to scrape Google Maps places
|
|
9
|
+
Ruby SDK to scrape Google Maps places (leads), **reviews**, and **photos** through a hosted Agent HTTP API. Powered by [GMaps Lead Finder](https://gmapsleadfinder.com). This is **not** a local browser crawler — jobs run in the cloud scrape-and-enrich pipeline.
|
|
10
10
|
|
|
11
11
|
- **RubyGems:** [`google-maps-scraper-sdk`](https://rubygems.org/gems/google-maps-scraper-sdk)
|
|
12
12
|
- **Require:** `gmaps_scraper` → `GmapsScraper::Client`
|
|
@@ -42,6 +42,11 @@ puts "#{me['plan']} #{me['creditsRemaining']}"
|
|
|
42
42
|
|
|
43
43
|
rows = client.scrape("dentists in Austin TX")
|
|
44
44
|
puts rows.length
|
|
45
|
+
|
|
46
|
+
# Single-place reviews / photos (run sequentially — one in-flight job per user)
|
|
47
|
+
reviews = client.scrape_reviews("https://maps.google.com/?cid=…")
|
|
48
|
+
photos = client.scrape_photos("ChIJ…") # Place ID, URL, or business_id
|
|
49
|
+
puts reviews.length, photos.length
|
|
45
50
|
```
|
|
46
51
|
|
|
47
52
|
CLI:
|
|
@@ -65,12 +70,20 @@ bundle exec gmaps-scraper me
|
|
|
65
70
|
|--------|-------------|
|
|
66
71
|
| `Client.new(api_key:, base_url:, timeout_s:)` | From env / kwargs |
|
|
67
72
|
| `me` | Plan & credits |
|
|
68
|
-
| `create_job(keyword)` | Queue one-keyword job |
|
|
69
|
-
| `get_job(job_id)` | Poll status |
|
|
70
|
-
| `get_results(job_id, limit:, cursor:)` | Paginated rows |
|
|
71
|
-
| `scrape(keyword, poll_interval_ms:, timeout_ms:, result_limit:)` | Create → poll → all rows |
|
|
72
|
-
|
|
73
|
-
|
|
73
|
+
| `create_job(keyword)` | Queue one-keyword leads job |
|
|
74
|
+
| `get_job(job_id)` | Poll leads status |
|
|
75
|
+
| `get_results(job_id, limit:, cursor:)` | Paginated leads rows |
|
|
76
|
+
| `scrape(keyword, poll_interval_ms:, timeout_ms:, result_limit:)` | Create → poll → all leads rows |
|
|
77
|
+
| `create_reviews_job(place)` | Queue single-place reviews job |
|
|
78
|
+
| `get_reviews_job(job_id)` | Poll reviews status |
|
|
79
|
+
| `get_reviews_results(job_id, limit:, cursor:)` | Paginated review rows |
|
|
80
|
+
| `scrape_reviews(place, …)` | Create → poll → all review rows |
|
|
81
|
+
| `create_photos_job(place)` | Queue single-place photos job |
|
|
82
|
+
| `get_photos_job(job_id)` | Poll photos status |
|
|
83
|
+
| `get_photos_results(job_id, limit:, cursor:)` | Paginated photo rows |
|
|
84
|
+
| `scrape_photos(place, …)` | Create → poll → all photo rows |
|
|
85
|
+
|
|
86
|
+
`scrape` / `scrape_reviews` / `scrape_photos` defaults: `poll_interval_ms` 2000, `timeout_ms` 600000, `result_limit` 100.
|
|
74
87
|
|
|
75
88
|
## Errors
|
|
76
89
|
|
|
@@ -86,8 +99,8 @@ bundle exec gmaps-scraper me
|
|
|
86
99
|
|
|
87
100
|
## Limits
|
|
88
101
|
|
|
89
|
-
- Exactly one keyword per
|
|
90
|
-
- One running job per user (`409` if busy)
|
|
102
|
+
- Exactly one keyword per leads job; exactly one place per reviews or photos job
|
|
103
|
+
- One running job per user (`409` if busy); run keywords/places sequentially
|
|
91
104
|
- 1 credit = 1 place row; Growth+ required
|
|
92
105
|
|
|
93
106
|
## Publish to RubyGems
|
|
@@ -95,7 +108,7 @@ bundle exec gmaps-scraper me
|
|
|
95
108
|
```bash
|
|
96
109
|
cd ruby
|
|
97
110
|
gem build google-maps-scraper-sdk.gemspec
|
|
98
|
-
gem push google-maps-scraper-sdk-0.1.
|
|
111
|
+
gem push google-maps-scraper-sdk-0.1.2.gem
|
|
99
112
|
```
|
|
100
113
|
|
|
101
114
|
## Links
|
data/lib/gmaps_scraper/client.rb
CHANGED
|
@@ -48,6 +48,40 @@ module GmapsScraper
|
|
|
48
48
|
request("GET", "/api/v1/jobs/#{job_id}/results", query: query)
|
|
49
49
|
end
|
|
50
50
|
|
|
51
|
+
def create_reviews_job(place)
|
|
52
|
+
trimmed = place.to_s.strip
|
|
53
|
+
raise BadRequestError, "place is required" if trimmed.empty?
|
|
54
|
+
|
|
55
|
+
request("POST", "/api/v1/review-jobs", json_body: { "place" => trimmed })
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
def get_reviews_job(job_id)
|
|
59
|
+
request("GET", "/api/v1/review-jobs/#{job_id}")
|
|
60
|
+
end
|
|
61
|
+
|
|
62
|
+
def get_reviews_results(job_id, limit: DEFAULT_RESULT_LIMIT, cursor: nil)
|
|
63
|
+
query = { "limit" => limit.to_s }
|
|
64
|
+
query["cursor"] = cursor.to_s unless cursor.nil?
|
|
65
|
+
request("GET", "/api/v1/review-jobs/#{job_id}/results", query: query)
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
def create_photos_job(place)
|
|
69
|
+
trimmed = place.to_s.strip
|
|
70
|
+
raise BadRequestError, "place is required" if trimmed.empty?
|
|
71
|
+
|
|
72
|
+
request("POST", "/api/v1/photo-jobs", json_body: { "place" => trimmed })
|
|
73
|
+
end
|
|
74
|
+
|
|
75
|
+
def get_photos_job(job_id)
|
|
76
|
+
request("GET", "/api/v1/photo-jobs/#{job_id}")
|
|
77
|
+
end
|
|
78
|
+
|
|
79
|
+
def get_photos_results(job_id, limit: DEFAULT_RESULT_LIMIT, cursor: nil)
|
|
80
|
+
query = { "limit" => limit.to_s }
|
|
81
|
+
query["cursor"] = cursor.to_s unless cursor.nil?
|
|
82
|
+
request("GET", "/api/v1/photo-jobs/#{job_id}/results", query: query)
|
|
83
|
+
end
|
|
84
|
+
|
|
51
85
|
def scrape(
|
|
52
86
|
keyword,
|
|
53
87
|
poll_interval_ms: DEFAULT_POLL_INTERVAL_MS,
|
|
@@ -56,11 +90,46 @@ module GmapsScraper
|
|
|
56
90
|
)
|
|
57
91
|
created = create_job(keyword)
|
|
58
92
|
job_id = created["jobId"].to_s
|
|
93
|
+
wait_for_job(job_id, poll_interval_ms: poll_interval_ms, timeout_ms: timeout_ms, kind: "maps")
|
|
94
|
+
fetch_all_rows(job_id, result_limit: result_limit, kind: "maps")
|
|
95
|
+
end
|
|
96
|
+
|
|
97
|
+
def scrape_reviews(
|
|
98
|
+
place,
|
|
99
|
+
poll_interval_ms: DEFAULT_POLL_INTERVAL_MS,
|
|
100
|
+
timeout_ms: DEFAULT_TIMEOUT_MS,
|
|
101
|
+
result_limit: DEFAULT_RESULT_LIMIT
|
|
102
|
+
)
|
|
103
|
+
created = create_reviews_job(place)
|
|
104
|
+
job_id = created["jobId"].to_s
|
|
105
|
+
wait_for_job(job_id, poll_interval_ms: poll_interval_ms, timeout_ms: timeout_ms, kind: "reviews")
|
|
106
|
+
fetch_all_rows(job_id, result_limit: result_limit, kind: "reviews")
|
|
107
|
+
end
|
|
108
|
+
|
|
109
|
+
def scrape_photos(
|
|
110
|
+
place,
|
|
111
|
+
poll_interval_ms: DEFAULT_POLL_INTERVAL_MS,
|
|
112
|
+
timeout_ms: DEFAULT_TIMEOUT_MS,
|
|
113
|
+
result_limit: DEFAULT_RESULT_LIMIT
|
|
114
|
+
)
|
|
115
|
+
created = create_photos_job(place)
|
|
116
|
+
job_id = created["jobId"].to_s
|
|
117
|
+
wait_for_job(job_id, poll_interval_ms: poll_interval_ms, timeout_ms: timeout_ms, kind: "photos")
|
|
118
|
+
fetch_all_rows(job_id, result_limit: result_limit, kind: "photos")
|
|
119
|
+
end
|
|
120
|
+
|
|
121
|
+
private
|
|
122
|
+
|
|
123
|
+
def wait_for_job(job_id, poll_interval_ms:, timeout_ms:, kind:)
|
|
59
124
|
deadline = Process.clock_gettime(Process::CLOCK_MONOTONIC) + (timeout_ms / 1000.0)
|
|
60
125
|
poll_s = [poll_interval_ms, 100].max / 1000.0
|
|
61
126
|
|
|
62
127
|
loop do
|
|
63
|
-
job =
|
|
128
|
+
job = case kind
|
|
129
|
+
when "reviews" then get_reviews_job(job_id)
|
|
130
|
+
when "photos" then get_photos_job(job_id)
|
|
131
|
+
else get_job(job_id)
|
|
132
|
+
end
|
|
64
133
|
status = job["status"].to_s.downcase
|
|
65
134
|
if TERMINAL_STATUSES.include?(status)
|
|
66
135
|
if status == "failed"
|
|
@@ -75,17 +144,20 @@ module GmapsScraper
|
|
|
75
144
|
end
|
|
76
145
|
sleep(poll_s)
|
|
77
146
|
end
|
|
78
|
-
|
|
79
|
-
fetch_all_rows(job_id, result_limit: result_limit)
|
|
80
147
|
end
|
|
81
148
|
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
def fetch_all_rows(job_id, result_limit:)
|
|
149
|
+
def fetch_all_rows(job_id, result_limit:, kind:)
|
|
85
150
|
rows = []
|
|
86
151
|
cursor = "0"
|
|
87
152
|
while cursor
|
|
88
|
-
page =
|
|
153
|
+
page = case kind
|
|
154
|
+
when "reviews"
|
|
155
|
+
get_reviews_results(job_id, limit: result_limit, cursor: cursor)
|
|
156
|
+
when "photos"
|
|
157
|
+
get_photos_results(job_id, limit: result_limit, cursor: cursor)
|
|
158
|
+
else
|
|
159
|
+
get_results(job_id, limit: result_limit, cursor: cursor)
|
|
160
|
+
end
|
|
89
161
|
rows.concat(Array(page["rows"]))
|
|
90
162
|
next_cursor = page["nextCursor"]
|
|
91
163
|
cursor = next_cursor.nil? || next_cursor == "" ? nil : next_cursor.to_s
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: google-maps-scraper-sdk
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.1.
|
|
4
|
+
version: 0.1.2
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- GMaps Lead Finder
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: exe
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2026-09-
|
|
11
|
+
date: 2026-09-19 00:00:00.000000000 Z
|
|
12
12
|
dependencies: []
|
|
13
13
|
description: Official Ruby client for the GMaps Lead Finder Agent HTTP API — scrape
|
|
14
14
|
Google Maps places (name, phone, website, emails) via a hosted cloud pipeline.
|