crawlora 1.5.0.pre.sdk.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/CHANGELOG.md +21 -0
- data/LICENSE +21 -0
- data/README.md +123 -0
- data/docs/operations.md +338 -0
- data/docs/recipes.md +87 -0
- data/examples/bing_search.rb +10 -0
- data/examples/paginate.rb +11 -0
- data/examples/youtube_transcript.rb +10 -0
- data/lib/crawlora/client.rb +626 -0
- data/lib/crawlora/errors.rb +38 -0
- data/lib/crawlora/operations.rb +13841 -0
- data/lib/crawlora/pagination.rb +39 -0
- data/lib/crawlora/version.rb +9 -0
- data/lib/crawlora.rb +31 -0
- data/openapi/public.json +54522 -0
- data/sig/crawlora.rbs +465 -0
- metadata +70 -0
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Crawlora
|
|
4
|
+
# Pagination helpers shared by the client's #paginate / #paginate_items.
|
|
5
|
+
module Pagination
|
|
6
|
+
PAGE_PARAM_NAMES = %w[page offset].freeze
|
|
7
|
+
|
|
8
|
+
module_function
|
|
9
|
+
|
|
10
|
+
# First page/offset query parameter an operation exposes, or nil.
|
|
11
|
+
def detect_page_param(operation)
|
|
12
|
+
names = (operation["queryParams"] || []).map { |p| p["name"] }
|
|
13
|
+
PAGE_PARAM_NAMES.find { |candidate| names.include?(candidate) }
|
|
14
|
+
end
|
|
15
|
+
|
|
16
|
+
# A page is empty when its `data` array (Crawlora envelope) or the page
|
|
17
|
+
# itself is empty/blank.
|
|
18
|
+
def page_empty?(response)
|
|
19
|
+
data = response.is_a?(Hash) && response.key?("data") ? response["data"] : response
|
|
20
|
+
return true if data.nil?
|
|
21
|
+
return data.empty? if data.respond_to?(:empty?)
|
|
22
|
+
|
|
23
|
+
!data
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
def default_start(page_param)
|
|
27
|
+
page_param == "offset" ? 0 : 1
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
# Default item extractor: the response's `data` list (Crawlora envelope), or
|
|
31
|
+
# the response itself when it is already an array.
|
|
32
|
+
def default_items(response)
|
|
33
|
+
return response["data"] if response.is_a?(Hash) && response["data"].is_a?(Array)
|
|
34
|
+
return response if response.is_a?(Array)
|
|
35
|
+
|
|
36
|
+
[]
|
|
37
|
+
end
|
|
38
|
+
end
|
|
39
|
+
end
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Crawlora
|
|
4
|
+
# SDK release version, in the shared `MAJOR.MINOR.PATCH-sdk.N` tag form (same
|
|
5
|
+
# as the Go/Java/PHP SDKs). RubyGems treats it as a prerelease and normalizes
|
|
6
|
+
# the published gem version to `1.5.0.pre.sdk.2`. Bumped across all SDK repos
|
|
7
|
+
# by the API repo's tools/sdkgen/bump_version.py.
|
|
8
|
+
VERSION = "1.5.0-sdk.2"
|
|
9
|
+
end
|
data/lib/crawlora.rb
ADDED
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "crawlora/version"
|
|
4
|
+
require_relative "crawlora/errors"
|
|
5
|
+
require_relative "crawlora/pagination"
|
|
6
|
+
require_relative "crawlora/operations"
|
|
7
|
+
require_relative "crawlora/client"
|
|
8
|
+
|
|
9
|
+
# Ruby SDK for the public Crawlora API.
|
|
10
|
+
#
|
|
11
|
+
# Quick start:
|
|
12
|
+
#
|
|
13
|
+
# require "crawlora"
|
|
14
|
+
#
|
|
15
|
+
# client = Crawlora.client(api_key: ENV["CRAWLORA_API_KEY"])
|
|
16
|
+
# result = client.bing.search(q: "web scraping")
|
|
17
|
+
# puts result["data"]
|
|
18
|
+
module Crawlora
|
|
19
|
+
# Build a Client. When a block is given, the client is yielded and closed
|
|
20
|
+
# afterwards (releasing pooled connections).
|
|
21
|
+
def self.client(**options)
|
|
22
|
+
client = Client.new(**options)
|
|
23
|
+
return client unless block_given?
|
|
24
|
+
|
|
25
|
+
begin
|
|
26
|
+
yield client
|
|
27
|
+
ensure
|
|
28
|
+
client.close
|
|
29
|
+
end
|
|
30
|
+
end
|
|
31
|
+
end
|