cloudscrape-client 0.1.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +9 -0
- data/.rspec +1 -0
- data/.rubocop.yml +7 -0
- data/.simplecov +13 -0
- data/.travis.yml +8 -0
- data/CODE_OF_CONDUCT.md +13 -0
- data/Gemfile +4 -0
- data/LICENSE.txt +21 -0
- data/README.md +154 -0
- data/Rakefile +6 -0
- data/bin/bundler +16 -0
- data/bin/console +14 -0
- data/bin/htmldiff +16 -0
- data/bin/ldiff +16 -0
- data/bin/rake +16 -0
- data/bin/rspec +16 -0
- data/bin/rubocop +16 -0
- data/bin/ruby-parse +16 -0
- data/bin/ruby-rewrite +16 -0
- data/bin/safe_yaml +16 -0
- data/bin/setup +7 -0
- data/bin/test +4 -0
- data/cloudscrape-client-ruby.gemspec +37 -0
- data/lib/cloud_scrape.rb +27 -0
- data/lib/cloud_scrape/api.rb +87 -0
- data/lib/cloud_scrape/configuration.rb +57 -0
- data/lib/cloud_scrape/dto.rb +58 -0
- data/lib/cloud_scrape/execution_dto.rb +17 -0
- data/lib/cloud_scrape/executions.rb +37 -0
- data/lib/cloud_scrape/executions/get.rb +41 -0
- data/lib/cloud_scrape/executions/result.rb +34 -0
- data/lib/cloud_scrape/run_dto.rb +25 -0
- data/lib/cloud_scrape/runs.rb +19 -0
- data/lib/cloud_scrape/validate.rb +38 -0
- data/lib/cloud_scrape/version.rb +3 -0
- data/postman_collection +180 -0
- metadata +234 -0
@@ -0,0 +1,37 @@
|
|
1
|
+
# coding: utf-8
|
2
|
+
lib = File.expand_path("../lib", __FILE__)
|
3
|
+
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
4
|
+
require "cloud_scrape/version"
|
5
|
+
|
6
|
+
Gem::Specification.new do |spec|
|
7
|
+
spec.name = "cloudscrape-client"
|
8
|
+
spec.version = CloudScrape::VERSION
|
9
|
+
spec.authors = ["Charles J Hardy"]
|
10
|
+
spec.email = ["support@cloudscrape.com"]
|
11
|
+
|
12
|
+
spec.summary = "Wrapper for CloudScrape API"
|
13
|
+
spec.description = "Wrapper for CloudScrape API"
|
14
|
+
spec.homepage = "http://cloudscrape.com"
|
15
|
+
spec.license = "MIT"
|
16
|
+
|
17
|
+
spec.files = `git ls-files -z`.split("\x0").reject do |f|
|
18
|
+
f.match(%r{^(test|spec|features)/})
|
19
|
+
end
|
20
|
+
|
21
|
+
spec.bindir = "exe"
|
22
|
+
spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
|
23
|
+
spec.require_paths = ["lib"]
|
24
|
+
|
25
|
+
spec.add_runtime_dependency "faraday", "~> 0.9"
|
26
|
+
spec.add_runtime_dependency "faraday_middleware", "~> 0.10"
|
27
|
+
spec.add_runtime_dependency "faraday-conductivity", "~> 0.3"
|
28
|
+
spec.add_runtime_dependency "faraday_middleware-multi_json", "~> 0.0"
|
29
|
+
|
30
|
+
spec.add_development_dependency "bundler", "~> 1.10"
|
31
|
+
spec.add_development_dependency "rake", "~> 10.0"
|
32
|
+
spec.add_development_dependency "rspec"
|
33
|
+
spec.add_development_dependency "rubocop"
|
34
|
+
spec.add_development_dependency "simplecov"
|
35
|
+
spec.add_development_dependency "vcr"
|
36
|
+
spec.add_development_dependency "webmock"
|
37
|
+
end
|
data/lib/cloud_scrape.rb
ADDED
@@ -0,0 +1,27 @@
|
|
1
|
+
require "logger"
|
2
|
+
|
3
|
+
require "cloud_scrape/version"
|
4
|
+
require "cloud_scrape/configuration"
|
5
|
+
require "cloud_scrape/dto"
|
6
|
+
require "cloud_scrape/runs"
|
7
|
+
require "cloud_scrape/executions"
|
8
|
+
|
9
|
+
class CloudScrape
|
10
|
+
extend Configure
|
11
|
+
|
12
|
+
NotFound = Class.new(StandardError)
|
13
|
+
InternalServerError = Class.new(StandardError)
|
14
|
+
|
15
|
+
def initialize(api_key: nil, account_id: nil)
|
16
|
+
CloudScrape.configuration.api_key = api_key if api_key
|
17
|
+
CloudScrape.configuration.account_id = account_id if account_id
|
18
|
+
end
|
19
|
+
|
20
|
+
def runs(id)
|
21
|
+
Runs.new(id: id)
|
22
|
+
end
|
23
|
+
|
24
|
+
def executions(id)
|
25
|
+
Executions.new(id: id)
|
26
|
+
end
|
27
|
+
end
|
@@ -0,0 +1,87 @@
|
|
1
|
+
require "digest"
|
2
|
+
require "faraday"
|
3
|
+
require "faraday_middleware"
|
4
|
+
require "faraday/conductivity"
|
5
|
+
require "faraday_middleware/multi_json"
|
6
|
+
|
7
|
+
class CloudScrape
|
8
|
+
class API
|
9
|
+
InvalidApiKey = Class.new(StandardError)
|
10
|
+
InvalidAccountId = Class.new(StandardError)
|
11
|
+
|
12
|
+
def self.get(*args)
|
13
|
+
new.get(*args)
|
14
|
+
end
|
15
|
+
|
16
|
+
def self.post(*args)
|
17
|
+
new.post(*args)
|
18
|
+
end
|
19
|
+
|
20
|
+
def self.delete(*args)
|
21
|
+
new.delete(*args)
|
22
|
+
end
|
23
|
+
|
24
|
+
def get(domain:, url:, options: {})
|
25
|
+
connection(domain).get(URI.escape(url), options)
|
26
|
+
end
|
27
|
+
|
28
|
+
def post(domain:, url:, options: {})
|
29
|
+
connection(domain).post do |req|
|
30
|
+
req.url URI.escape(url)
|
31
|
+
req.headers["Content-Type"] = "application/json"
|
32
|
+
req.body = options.to_json
|
33
|
+
end
|
34
|
+
end
|
35
|
+
|
36
|
+
def delete(domain:, url:, options: {})
|
37
|
+
connection(domain).delete(URI.escape(url), options)
|
38
|
+
end
|
39
|
+
|
40
|
+
private
|
41
|
+
|
42
|
+
def access_key
|
43
|
+
fail InvalidAccountId, account_id unless account_id
|
44
|
+
fail InvalidApiKey, api_key unless api_key
|
45
|
+
|
46
|
+
Digest::MD5.hexdigest(account_id + api_key)
|
47
|
+
end
|
48
|
+
|
49
|
+
def account_id
|
50
|
+
CloudScrape.configuration.account_id
|
51
|
+
end
|
52
|
+
|
53
|
+
def api_key
|
54
|
+
CloudScrape.configuration.api_key
|
55
|
+
end
|
56
|
+
|
57
|
+
# rubocop:disable Metrics/AbcSize
|
58
|
+
# rubocop:disable Metrics/MethodLength
|
59
|
+
def connection(domain)
|
60
|
+
Faraday.new(url: domain) do |faraday|
|
61
|
+
faraday.request :url_encoded
|
62
|
+
|
63
|
+
faraday.request :user_agent,
|
64
|
+
app: CloudScrape.configuration.user_agent_app,
|
65
|
+
version: CloudScrape.configuration.user_agent_version
|
66
|
+
|
67
|
+
faraday.request :request_headers,
|
68
|
+
accept: "application/json",
|
69
|
+
"X-CloudScrape-Access" => access_key,
|
70
|
+
"X-CloudScrape-Account" => account_id,
|
71
|
+
content_type: "application/json"
|
72
|
+
|
73
|
+
if CloudScrape.configuration.verbose
|
74
|
+
faraday.response :logger, CloudScrape.configuration.logger
|
75
|
+
end
|
76
|
+
|
77
|
+
faraday.response :multi_json,
|
78
|
+
content_type: /\bjson$/,
|
79
|
+
symbolize_keys: true
|
80
|
+
|
81
|
+
faraday.adapter Faraday.default_adapter
|
82
|
+
end
|
83
|
+
end
|
84
|
+
# rubocop:enable Metrics/AbcSize
|
85
|
+
# rubocop:enable Metrics/MethodLength
|
86
|
+
end
|
87
|
+
end
|
@@ -0,0 +1,57 @@
|
|
1
|
+
class CloudScrape
|
2
|
+
module Configure
|
3
|
+
def configuration
|
4
|
+
@configuration ||= Configuration.new
|
5
|
+
end
|
6
|
+
|
7
|
+
def configure
|
8
|
+
yield(configuration)
|
9
|
+
configuration
|
10
|
+
end
|
11
|
+
|
12
|
+
private
|
13
|
+
|
14
|
+
class Configuration
|
15
|
+
attr_accessor :base_url,
|
16
|
+
:api_key,
|
17
|
+
:account_id,
|
18
|
+
:user_agent_app,
|
19
|
+
:user_agent_version,
|
20
|
+
:timeout,
|
21
|
+
:verbose,
|
22
|
+
:logger
|
23
|
+
|
24
|
+
def initialize # rubocop:disable Metrics/AbcSize
|
25
|
+
self.base_url = base_url_default
|
26
|
+
self.user_agent_app = user_agent_app_default
|
27
|
+
self.user_agent_version = user_agent_version_default
|
28
|
+
self.timeout = timeout_default
|
29
|
+
|
30
|
+
self.api_key = ENV["CLOUD_SCRAPE_CLIENT_API_KEY"]
|
31
|
+
self.account_id = ENV["CLOUD_SCRAPE_CLIENT_ACCOUNT_ID"]
|
32
|
+
self.verbose = ENV["CLOUD_SCRAPE_CLIENT_VERBOSE"] || false
|
33
|
+
|
34
|
+
self.logger = Logger.new(STDOUT)
|
35
|
+
end
|
36
|
+
|
37
|
+
private
|
38
|
+
|
39
|
+
def base_url_default
|
40
|
+
ENV["CLOUD_SCRAPE_CLIENT_BASE_URL"] ||
|
41
|
+
"https://app.cloudscrape.com/api/"
|
42
|
+
end
|
43
|
+
|
44
|
+
def user_agent_app_default
|
45
|
+
ENV["CLOUD_SCRAPE_CLIENT_USER_AGENT_APP"] || "CS-RUBY-CLIENT"
|
46
|
+
end
|
47
|
+
|
48
|
+
def user_agent_version_default
|
49
|
+
ENV["CLOUD_SCRAPE_CLIENT_USER_AGENT_VERSION"] || "1.0"
|
50
|
+
end
|
51
|
+
|
52
|
+
def timeout_default
|
53
|
+
ENV["CLOUD_SCRAPE_CLIENT_TIMEOUT"] || 3600
|
54
|
+
end
|
55
|
+
end
|
56
|
+
end
|
57
|
+
end
|
@@ -0,0 +1,58 @@
|
|
1
|
+
require "cloud_scrape/api"
|
2
|
+
require "cloud_scrape/validate"
|
3
|
+
|
4
|
+
class CloudScrape
|
5
|
+
class DTO
|
6
|
+
attr_reader :options
|
7
|
+
|
8
|
+
def initialize(options:)
|
9
|
+
@options = options
|
10
|
+
end
|
11
|
+
|
12
|
+
def self.for(options)
|
13
|
+
new(options: options).response(options.fetch(:method))
|
14
|
+
end
|
15
|
+
|
16
|
+
def self.get(options = {})
|
17
|
+
new(options: options).response(:get)
|
18
|
+
end
|
19
|
+
|
20
|
+
def self.post(options = {})
|
21
|
+
new(options: options).response(:post)
|
22
|
+
end
|
23
|
+
|
24
|
+
def self.delete(options = {})
|
25
|
+
new(options: options).response(:delete)
|
26
|
+
end
|
27
|
+
|
28
|
+
def response(method)
|
29
|
+
API.public_send(
|
30
|
+
method,
|
31
|
+
domain: domain,
|
32
|
+
url: endpoint,
|
33
|
+
options: {
|
34
|
+
api_key: api_key,
|
35
|
+
format: "json"
|
36
|
+
}.merge(params)
|
37
|
+
).tap(&CloudScrape::Validate).body
|
38
|
+
end
|
39
|
+
|
40
|
+
def params
|
41
|
+
{}
|
42
|
+
end
|
43
|
+
|
44
|
+
def endpoint
|
45
|
+
fail NotImplementedError, "Inheriting class must implement"
|
46
|
+
end
|
47
|
+
|
48
|
+
private
|
49
|
+
|
50
|
+
def domain
|
51
|
+
CloudScrape.configuration.base_url
|
52
|
+
end
|
53
|
+
|
54
|
+
def api_key
|
55
|
+
CloudScrape.configuration.api_key
|
56
|
+
end
|
57
|
+
end
|
58
|
+
end
|
@@ -0,0 +1,37 @@
|
|
1
|
+
require "cloud_scrape/execution_dto"
|
2
|
+
require "cloud_scrape/executions/get"
|
3
|
+
require "cloud_scrape/executions/result"
|
4
|
+
|
5
|
+
class CloudScrape
|
6
|
+
class Executions
|
7
|
+
def initialize(id:)
|
8
|
+
@id = id
|
9
|
+
end
|
10
|
+
|
11
|
+
def get
|
12
|
+
Get.new(response: dto("", :get))
|
13
|
+
end
|
14
|
+
|
15
|
+
def result
|
16
|
+
Result.new(response: dto("result", :get))
|
17
|
+
end
|
18
|
+
|
19
|
+
def remove
|
20
|
+
dto("", :delete)
|
21
|
+
end
|
22
|
+
|
23
|
+
def stop
|
24
|
+
dto("stop", :post)
|
25
|
+
end
|
26
|
+
|
27
|
+
def continue
|
28
|
+
dto("continue", :post)
|
29
|
+
end
|
30
|
+
|
31
|
+
private
|
32
|
+
|
33
|
+
def dto(url, method)
|
34
|
+
ExecutionDTO.for(id: @id, url: url, method: method)
|
35
|
+
end
|
36
|
+
end
|
37
|
+
end
|
@@ -0,0 +1,41 @@
|
|
1
|
+
class CloudScrape
|
2
|
+
class Executions
|
3
|
+
class Get
|
4
|
+
def initialize(response:)
|
5
|
+
@response = response
|
6
|
+
end
|
7
|
+
|
8
|
+
def queued?
|
9
|
+
%w(QUEUED).include?(state)
|
10
|
+
end
|
11
|
+
|
12
|
+
def pending?
|
13
|
+
%w(PENDING).include?(state)
|
14
|
+
end
|
15
|
+
|
16
|
+
def running?
|
17
|
+
%w(RUNNING).include?(state)
|
18
|
+
end
|
19
|
+
|
20
|
+
def failed?
|
21
|
+
%w(FAILED).include?(state)
|
22
|
+
end
|
23
|
+
|
24
|
+
def stopped?
|
25
|
+
%w(STOPPED).include?(state)
|
26
|
+
end
|
27
|
+
|
28
|
+
def ok?
|
29
|
+
%w(OK).include?(state)
|
30
|
+
end
|
31
|
+
|
32
|
+
private
|
33
|
+
|
34
|
+
attr_reader :response
|
35
|
+
|
36
|
+
def state
|
37
|
+
response.fetch(:state)
|
38
|
+
end
|
39
|
+
end
|
40
|
+
end
|
41
|
+
end
|
@@ -0,0 +1,34 @@
|
|
1
|
+
class CloudScrape
|
2
|
+
class Executions
|
3
|
+
class Result
|
4
|
+
attr_reader :response
|
5
|
+
|
6
|
+
def initialize(response:)
|
7
|
+
@response = response
|
8
|
+
build
|
9
|
+
end
|
10
|
+
|
11
|
+
def build
|
12
|
+
as_hash.each(&define_method_for_header)
|
13
|
+
end
|
14
|
+
|
15
|
+
def as_hash
|
16
|
+
@as_hash ||= Hash[headers.zip(rows)]
|
17
|
+
end
|
18
|
+
|
19
|
+
private
|
20
|
+
|
21
|
+
def headers
|
22
|
+
response.fetch(:headers, [])
|
23
|
+
end
|
24
|
+
|
25
|
+
def rows
|
26
|
+
response.fetch(:rows, [[]]).flatten
|
27
|
+
end
|
28
|
+
|
29
|
+
def define_method_for_header
|
30
|
+
->(key, value) { self.class.send(:define_method, key) { value } }
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|
34
|
+
end
|
@@ -0,0 +1,25 @@
|
|
1
|
+
class CloudScrape
|
2
|
+
class RunDTO < DTO
|
3
|
+
def endpoint
|
4
|
+
"runs/#{id}/#{url}?connect=#{connect}"
|
5
|
+
end
|
6
|
+
|
7
|
+
private
|
8
|
+
|
9
|
+
def id
|
10
|
+
options.fetch(:id)
|
11
|
+
end
|
12
|
+
|
13
|
+
def url
|
14
|
+
options.fetch(:url)
|
15
|
+
end
|
16
|
+
|
17
|
+
def connect
|
18
|
+
options.fetch(:connect)
|
19
|
+
end
|
20
|
+
|
21
|
+
def params
|
22
|
+
options.fetch(:input)
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
@@ -0,0 +1,19 @@
|
|
1
|
+
require "cloud_scrape/run_dto"
|
2
|
+
|
3
|
+
class CloudScrape
|
4
|
+
class Runs
|
5
|
+
def initialize(id:)
|
6
|
+
@id = id
|
7
|
+
end
|
8
|
+
|
9
|
+
def execute(input: {}, connect: false)
|
10
|
+
RunDTO.for(
|
11
|
+
id: @id,
|
12
|
+
url: (input.empty? ? "execute" : "execute/inputs"),
|
13
|
+
input: input,
|
14
|
+
connect: connect,
|
15
|
+
method: :post
|
16
|
+
).fetch(:_id)
|
17
|
+
end
|
18
|
+
end
|
19
|
+
end
|