webscraping_ai 2.0.2 → 3.1.2

Sign up to get free protection for your applications and to get access to all the features.
@@ -1,12 +1,12 @@
1
1
  =begin
2
2
  #WebScraping.AI
3
3
 
4
- #A client for https://webscraping.ai API. It provides a web scaping automation API with Chrome JS rendering, rotating proxies and builtin HTML parsing.
4
+ #WebScraping.AI scraping API provides GPT-powered tools with Chromium JavaScript rendering, rotating proxies, and built-in HTML parsing.
5
5
 
6
- The version of the OpenAPI document: 2.0.2
6
+ The version of the OpenAPI document: 3.1.2
7
7
  Contact: support@webscraping.ai
8
8
  Generated by: https://openapi-generator.tech
9
- OpenAPI Generator version: 4.3.1
9
+ OpenAPI Generator version: 7.2.0
10
10
 
11
11
  =end
12
12
 
@@ -17,24 +17,37 @@ require 'date'
17
17
  # Unit tests for WebScrapingAI::Error
18
18
  # Automatically generated by openapi-generator (https://openapi-generator.tech)
19
19
  # Please update as you see appropriate
20
- describe 'Error' do
21
- before do
22
- # run before each test
23
- @instance = WebScrapingAI::Error.new
24
- end
25
-
26
- after do
27
- # run after each test
28
- end
20
+ describe WebScrapingAI::Error do
21
+ let(:instance) { WebScrapingAI::Error.new }
29
22
 
30
23
  describe 'test an instance of Error' do
31
24
  it 'should create an instance of Error' do
32
- expect(@instance).to be_instance_of(WebScrapingAI::Error)
25
+ # uncomment below to test the instance creation
26
+ #expect(instance).to be_instance_of(WebScrapingAI::Error)
33
27
  end
34
28
  end
29
+
35
30
  describe 'test attribute "message"' do
36
31
  it 'should work' do
37
- # assertion here. ref: https://www.relishapp.com/rspec/rspec-expectations/docs/built-in-matchers
32
+ # assertion here. ref: https://rspec.info/features/3-12/rspec-expectations/built-in-matchers/
33
+ end
34
+ end
35
+
36
+ describe 'test attribute "status_code"' do
37
+ it 'should work' do
38
+ # assertion here. ref: https://rspec.info/features/3-12/rspec-expectations/built-in-matchers/
39
+ end
40
+ end
41
+
42
+ describe 'test attribute "status_message"' do
43
+ it 'should work' do
44
+ # assertion here. ref: https://rspec.info/features/3-12/rspec-expectations/built-in-matchers/
45
+ end
46
+ end
47
+
48
+ describe 'test attribute "body"' do
49
+ it 'should work' do
50
+ # assertion here. ref: https://rspec.info/features/3-12/rspec-expectations/built-in-matchers/
38
51
  end
39
52
  end
40
53
 
data/spec/spec_helper.rb CHANGED
@@ -1,12 +1,12 @@
1
1
  =begin
2
2
  #WebScraping.AI
3
3
 
4
- #A client for https://webscraping.ai API. It provides a web scaping automation API with Chrome JS rendering, rotating proxies and builtin HTML parsing.
4
+ #WebScraping.AI scraping API provides GPT-powered tools with Chromium JavaScript rendering, rotating proxies, and built-in HTML parsing.
5
5
 
6
- The version of the OpenAPI document: 2.0.2
6
+ The version of the OpenAPI document: 3.1.2
7
7
  Contact: support@webscraping.ai
8
8
  Generated by: https://openapi-generator.tech
9
- OpenAPI Generator version: 4.3.1
9
+ OpenAPI Generator version: 7.2.0
10
10
 
11
11
  =end
12
12
 
@@ -3,12 +3,12 @@
3
3
  =begin
4
4
  #WebScraping.AI
5
5
 
6
- #A client for https://webscraping.ai API. It provides a web scaping automation API with Chrome JS rendering, rotating proxies and builtin HTML parsing.
6
+ #WebScraping.AI scraping API provides GPT-powered tools with Chromium JavaScript rendering, rotating proxies, and built-in HTML parsing.
7
7
 
8
- The version of the OpenAPI document: 2.0.2
8
+ The version of the OpenAPI document: 3.1.2
9
9
  Contact: support@webscraping.ai
10
10
  Generated by: https://openapi-generator.tech
11
- OpenAPI Generator version: 4.3.1
11
+ OpenAPI Generator version: 7.2.0
12
12
 
13
13
  =end
14
14
 
@@ -23,12 +23,12 @@ Gem::Specification.new do |s|
23
23
  s.email = ["hello@webscraping.ai"]
24
24
  s.homepage = "https://webscraping.ai"
25
25
  s.summary = "WebScraping.AI Ruby Gem"
26
- s.description = "A client for https://webscraping.ai API. It provides a web scaping automation API with Chrome JS rendering, rotating proxies and builtin HTML parsing."
27
- s.license = 'MIT'
28
- s.required_ruby_version = ">= 1.9"
26
+ s.description = "WebScraping.AI scraping API provides GPT-powered tools with Chromium JavaScript rendering, rotating proxies, and built-in HTML parsing."
27
+ s.license = "MIT"
28
+ s.required_ruby_version = ">= 2.7"
29
+ s.metadata = {}
29
30
 
30
31
  s.add_runtime_dependency 'typhoeus', '~> 1.0', '>= 1.0.1'
31
- s.add_runtime_dependency 'json', '~> 2.1', '>= 2.1.0'
32
32
 
33
33
  s.add_development_dependency 'rspec', '~> 3.6', '>= 3.6.0'
34
34
 
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: webscraping_ai
3
3
  version: !ruby/object:Gem::Version
4
- version: 2.0.2
4
+ version: 3.1.2
5
5
  platform: ruby
6
6
  authors:
7
7
  - WebScraping.AI
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2020-09-24 00:00:00.000000000 Z
11
+ date: 2024-01-03 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: typhoeus
@@ -30,26 +30,6 @@ dependencies:
30
30
  - - ">="
31
31
  - !ruby/object:Gem::Version
32
32
  version: 1.0.1
33
- - !ruby/object:Gem::Dependency
34
- name: json
35
- requirement: !ruby/object:Gem::Requirement
36
- requirements:
37
- - - "~>"
38
- - !ruby/object:Gem::Version
39
- version: '2.1'
40
- - - ">="
41
- - !ruby/object:Gem::Version
42
- version: 2.1.0
43
- type: :runtime
44
- prerelease: false
45
- version_requirements: !ruby/object:Gem::Requirement
46
- requirements:
47
- - - "~>"
48
- - !ruby/object:Gem::Version
49
- version: '2.1'
50
- - - ">="
51
- - !ruby/object:Gem::Version
52
- version: 2.1.0
53
33
  - !ruby/object:Gem::Dependency
54
34
  name: rspec
55
35
  requirement: !ruby/object:Gem::Requirement
@@ -70,8 +50,8 @@ dependencies:
70
50
  - - ">="
71
51
  - !ruby/object:Gem::Version
72
52
  version: 3.6.0
73
- description: A client for https://webscraping.ai API. It provides a web scaping automation
74
- API with Chrome JS rendering, rotating proxies and builtin HTML parsing.
53
+ description: WebScraping.AI scraping API provides GPT-powered tools with Chromium
54
+ JavaScript rendering, rotating proxies, and built-in HTML parsing.
75
55
  email:
76
56
  - hello@webscraping.ai
77
57
  executables: []
@@ -81,26 +61,33 @@ files:
81
61
  - Gemfile
82
62
  - README.md
83
63
  - Rakefile
64
+ - docs/AIApi.md
65
+ - docs/Account.md
66
+ - docs/AccountApi.md
84
67
  - docs/Error.md
85
68
  - docs/HTMLApi.md
86
- - docs/PageError.md
87
69
  - docs/SelectedHTMLApi.md
70
+ - docs/TextApi.md
88
71
  - git_push.sh
89
72
  - lib/webscraping_ai.rb
73
+ - lib/webscraping_ai/api/account_api.rb
74
+ - lib/webscraping_ai/api/ai_api.rb
90
75
  - lib/webscraping_ai/api/html_api.rb
91
76
  - lib/webscraping_ai/api/selected_html_api.rb
77
+ - lib/webscraping_ai/api/text_api.rb
92
78
  - lib/webscraping_ai/api_client.rb
93
79
  - lib/webscraping_ai/api_error.rb
94
80
  - lib/webscraping_ai/configuration.rb
81
+ - lib/webscraping_ai/models/account.rb
95
82
  - lib/webscraping_ai/models/error.rb
96
- - lib/webscraping_ai/models/page_error.rb
97
83
  - lib/webscraping_ai/version.rb
84
+ - spec/api/account_api_spec.rb
85
+ - spec/api/ai_api_spec.rb
98
86
  - spec/api/html_api_spec.rb
99
87
  - spec/api/selected_html_api_spec.rb
100
- - spec/api_client_spec.rb
101
- - spec/configuration_spec.rb
88
+ - spec/api/text_api_spec.rb
89
+ - spec/models/account_spec.rb
102
90
  - spec/models/error_spec.rb
103
- - spec/models/page_error_spec.rb
104
91
  - spec/spec_helper.rb
105
92
  - webscraping_ai.gemspec
106
93
  homepage: https://webscraping.ai
@@ -115,22 +102,23 @@ required_ruby_version: !ruby/object:Gem::Requirement
115
102
  requirements:
116
103
  - - ">="
117
104
  - !ruby/object:Gem::Version
118
- version: '1.9'
105
+ version: '2.7'
119
106
  required_rubygems_version: !ruby/object:Gem::Requirement
120
107
  requirements:
121
108
  - - ">="
122
109
  - !ruby/object:Gem::Version
123
110
  version: '0'
124
111
  requirements: []
125
- rubygems_version: 3.1.2
112
+ rubygems_version: 3.4.10
126
113
  signing_key:
127
114
  specification_version: 4
128
115
  summary: WebScraping.AI Ruby Gem
129
116
  test_files:
117
+ - spec/api/ai_api_spec.rb
118
+ - spec/api/text_api_spec.rb
130
119
  - spec/api/html_api_spec.rb
131
120
  - spec/api/selected_html_api_spec.rb
132
- - spec/api_client_spec.rb
133
- - spec/configuration_spec.rb
121
+ - spec/api/account_api_spec.rb
134
122
  - spec/models/error_spec.rb
135
- - spec/models/page_error_spec.rb
123
+ - spec/models/account_spec.rb
136
124
  - spec/spec_helper.rb
data/docs/PageError.md DELETED
@@ -1,19 +0,0 @@
1
- # WebScrapingAI::PageError
2
-
3
- ## Properties
4
-
5
- Name | Type | Description | Notes
6
- ------------ | ------------- | ------------- | -------------
7
- **status_code** | **Integer** | Response HTTP status code (403, 500, etc) | [optional]
8
- **status_message** | **String** | Response HTTP status message | [optional]
9
-
10
- ## Code Sample
11
-
12
- ```ruby
13
- require 'WebScrapingAI'
14
-
15
- instance = WebScrapingAI::PageError.new(status_code: null,
16
- status_message: null)
17
- ```
18
-
19
-
@@ -1,226 +0,0 @@
1
- =begin
2
- #WebScraping.AI
3
-
4
- #A client for https://webscraping.ai API. It provides a web scaping automation API with Chrome JS rendering, rotating proxies and builtin HTML parsing.
5
-
6
- The version of the OpenAPI document: 2.0.2
7
- Contact: support@webscraping.ai
8
- Generated by: https://openapi-generator.tech
9
- OpenAPI Generator version: 4.3.1
10
-
11
- =end
12
-
13
- require 'spec_helper'
14
-
15
- describe WebScrapingAI::ApiClient do
16
- context 'initialization' do
17
- context 'URL stuff' do
18
- context 'host' do
19
- it 'removes http from host' do
20
- WebScrapingAI.configure { |c| c.host = 'http://example.com' }
21
- expect(WebScrapingAI::Configuration.default.host).to eq('example.com')
22
- end
23
-
24
- it 'removes https from host' do
25
- WebScrapingAI.configure { |c| c.host = 'https://wookiee.com' }
26
- expect(WebScrapingAI::ApiClient.default.config.host).to eq('wookiee.com')
27
- end
28
-
29
- it 'removes trailing path from host' do
30
- WebScrapingAI.configure { |c| c.host = 'hobo.com/v4' }
31
- expect(WebScrapingAI::Configuration.default.host).to eq('hobo.com')
32
- end
33
- end
34
-
35
- context 'base_path' do
36
- it "prepends a slash to base_path" do
37
- WebScrapingAI.configure { |c| c.base_path = 'v4/dog' }
38
- expect(WebScrapingAI::Configuration.default.base_path).to eq('/v4/dog')
39
- end
40
-
41
- it "doesn't prepend a slash if one is already there" do
42
- WebScrapingAI.configure { |c| c.base_path = '/v4/dog' }
43
- expect(WebScrapingAI::Configuration.default.base_path).to eq('/v4/dog')
44
- end
45
-
46
- it "ends up as a blank string if nil" do
47
- WebScrapingAI.configure { |c| c.base_path = nil }
48
- expect(WebScrapingAI::Configuration.default.base_path).to eq('')
49
- end
50
- end
51
- end
52
- end
53
-
54
- describe 'params_encoding in #build_request' do
55
- let(:config) { WebScrapingAI::Configuration.new }
56
- let(:api_client) { WebScrapingAI::ApiClient.new(config) }
57
-
58
- it 'defaults to nil' do
59
- expect(WebScrapingAI::Configuration.default.params_encoding).to eq(nil)
60
- expect(config.params_encoding).to eq(nil)
61
-
62
- request = api_client.build_request(:get, '/test')
63
- expect(request.options[:params_encoding]).to eq(nil)
64
- end
65
-
66
- it 'can be customized' do
67
- config.params_encoding = :multi
68
- request = api_client.build_request(:get, '/test')
69
- expect(request.options[:params_encoding]).to eq(:multi)
70
- end
71
- end
72
-
73
- describe 'timeout in #build_request' do
74
- let(:config) { WebScrapingAI::Configuration.new }
75
- let(:api_client) { WebScrapingAI::ApiClient.new(config) }
76
-
77
- it 'defaults to 0' do
78
- expect(WebScrapingAI::Configuration.default.timeout).to eq(0)
79
- expect(config.timeout).to eq(0)
80
-
81
- request = api_client.build_request(:get, '/test')
82
- expect(request.options[:timeout]).to eq(0)
83
- end
84
-
85
- it 'can be customized' do
86
- config.timeout = 100
87
- request = api_client.build_request(:get, '/test')
88
- expect(request.options[:timeout]).to eq(100)
89
- end
90
- end
91
-
92
- describe '#deserialize' do
93
- it "handles Array<Integer>" do
94
- api_client = WebScrapingAI::ApiClient.new
95
- headers = { 'Content-Type' => 'application/json' }
96
- response = double('response', headers: headers, body: '[12, 34]')
97
- data = api_client.deserialize(response, 'Array<Integer>')
98
- expect(data).to be_instance_of(Array)
99
- expect(data).to eq([12, 34])
100
- end
101
-
102
- it 'handles Array<Array<Integer>>' do
103
- api_client = WebScrapingAI::ApiClient.new
104
- headers = { 'Content-Type' => 'application/json' }
105
- response = double('response', headers: headers, body: '[[12, 34], [56]]')
106
- data = api_client.deserialize(response, 'Array<Array<Integer>>')
107
- expect(data).to be_instance_of(Array)
108
- expect(data).to eq([[12, 34], [56]])
109
- end
110
-
111
- it 'handles Hash<String, String>' do
112
- api_client = WebScrapingAI::ApiClient.new
113
- headers = { 'Content-Type' => 'application/json' }
114
- response = double('response', headers: headers, body: '{"message": "Hello"}')
115
- data = api_client.deserialize(response, 'Hash<String, String>')
116
- expect(data).to be_instance_of(Hash)
117
- expect(data).to eq(:message => 'Hello')
118
- end
119
- end
120
-
121
- describe "#object_to_hash" do
122
- it 'ignores nils and includes empty arrays' do
123
- # uncomment below to test object_to_hash for model
124
- # api_client = WebScrapingAI::ApiClient.new
125
- # _model = WebScrapingAI::ModelName.new
126
- # update the model attribute below
127
- # _model.id = 1
128
- # update the expected value (hash) below
129
- # expected = {id: 1, name: '', tags: []}
130
- # expect(api_client.object_to_hash(_model)).to eq(expected)
131
- end
132
- end
133
-
134
- describe '#build_collection_param' do
135
- let(:param) { ['aa', 'bb', 'cc'] }
136
- let(:api_client) { WebScrapingAI::ApiClient.new }
137
-
138
- it 'works for csv' do
139
- expect(api_client.build_collection_param(param, :csv)).to eq('aa,bb,cc')
140
- end
141
-
142
- it 'works for ssv' do
143
- expect(api_client.build_collection_param(param, :ssv)).to eq('aa bb cc')
144
- end
145
-
146
- it 'works for tsv' do
147
- expect(api_client.build_collection_param(param, :tsv)).to eq("aa\tbb\tcc")
148
- end
149
-
150
- it 'works for pipes' do
151
- expect(api_client.build_collection_param(param, :pipes)).to eq('aa|bb|cc')
152
- end
153
-
154
- it 'works for multi' do
155
- expect(api_client.build_collection_param(param, :multi)).to eq(['aa', 'bb', 'cc'])
156
- end
157
-
158
- it 'fails for invalid collection format' do
159
- expect { api_client.build_collection_param(param, :INVALID) }.to raise_error(RuntimeError, 'unknown collection format: :INVALID')
160
- end
161
- end
162
-
163
- describe '#json_mime?' do
164
- let(:api_client) { WebScrapingAI::ApiClient.new }
165
-
166
- it 'works' do
167
- expect(api_client.json_mime?(nil)).to eq false
168
- expect(api_client.json_mime?('')).to eq false
169
-
170
- expect(api_client.json_mime?('application/json')).to eq true
171
- expect(api_client.json_mime?('application/json; charset=UTF8')).to eq true
172
- expect(api_client.json_mime?('APPLICATION/JSON')).to eq true
173
-
174
- expect(api_client.json_mime?('application/xml')).to eq false
175
- expect(api_client.json_mime?('text/plain')).to eq false
176
- expect(api_client.json_mime?('application/jsonp')).to eq false
177
- end
178
- end
179
-
180
- describe '#select_header_accept' do
181
- let(:api_client) { WebScrapingAI::ApiClient.new }
182
-
183
- it 'works' do
184
- expect(api_client.select_header_accept(nil)).to be_nil
185
- expect(api_client.select_header_accept([])).to be_nil
186
-
187
- expect(api_client.select_header_accept(['application/json'])).to eq('application/json')
188
- expect(api_client.select_header_accept(['application/xml', 'application/json; charset=UTF8'])).to eq('application/json; charset=UTF8')
189
- expect(api_client.select_header_accept(['APPLICATION/JSON', 'text/html'])).to eq('APPLICATION/JSON')
190
-
191
- expect(api_client.select_header_accept(['application/xml'])).to eq('application/xml')
192
- expect(api_client.select_header_accept(['text/html', 'application/xml'])).to eq('text/html,application/xml')
193
- end
194
- end
195
-
196
- describe '#select_header_content_type' do
197
- let(:api_client) { WebScrapingAI::ApiClient.new }
198
-
199
- it 'works' do
200
- expect(api_client.select_header_content_type(nil)).to eq('application/json')
201
- expect(api_client.select_header_content_type([])).to eq('application/json')
202
-
203
- expect(api_client.select_header_content_type(['application/json'])).to eq('application/json')
204
- expect(api_client.select_header_content_type(['application/xml', 'application/json; charset=UTF8'])).to eq('application/json; charset=UTF8')
205
- expect(api_client.select_header_content_type(['APPLICATION/JSON', 'text/html'])).to eq('APPLICATION/JSON')
206
- expect(api_client.select_header_content_type(['application/xml'])).to eq('application/xml')
207
- expect(api_client.select_header_content_type(['text/plain', 'application/xml'])).to eq('text/plain')
208
- end
209
- end
210
-
211
- describe '#sanitize_filename' do
212
- let(:api_client) { WebScrapingAI::ApiClient.new }
213
-
214
- it 'works' do
215
- expect(api_client.sanitize_filename('sun')).to eq('sun')
216
- expect(api_client.sanitize_filename('sun.gif')).to eq('sun.gif')
217
- expect(api_client.sanitize_filename('../sun.gif')).to eq('sun.gif')
218
- expect(api_client.sanitize_filename('/var/tmp/sun.gif')).to eq('sun.gif')
219
- expect(api_client.sanitize_filename('./sun.gif')).to eq('sun.gif')
220
- expect(api_client.sanitize_filename('..\sun.gif')).to eq('sun.gif')
221
- expect(api_client.sanitize_filename('\var\tmp\sun.gif')).to eq('sun.gif')
222
- expect(api_client.sanitize_filename('c:\var\tmp\sun.gif')).to eq('sun.gif')
223
- expect(api_client.sanitize_filename('.\sun.gif')).to eq('sun.gif')
224
- end
225
- end
226
- end
@@ -1,42 +0,0 @@
1
- =begin
2
- #WebScraping.AI
3
-
4
- #A client for https://webscraping.ai API. It provides a web scaping automation API with Chrome JS rendering, rotating proxies and builtin HTML parsing.
5
-
6
- The version of the OpenAPI document: 2.0.2
7
- Contact: support@webscraping.ai
8
- Generated by: https://openapi-generator.tech
9
- OpenAPI Generator version: 4.3.1
10
-
11
- =end
12
-
13
- require 'spec_helper'
14
-
15
- describe WebScrapingAI::Configuration do
16
- let(:config) { WebScrapingAI::Configuration.default }
17
-
18
- before(:each) do
19
- # uncomment below to setup host and base_path
20
- # require 'URI'
21
- # uri = URI.parse("https://api.webscraping.ai")
22
- # WebScrapingAI.configure do |c|
23
- # c.host = uri.host
24
- # c.base_path = uri.path
25
- # end
26
- end
27
-
28
- describe '#base_url' do
29
- it 'should have the default value' do
30
- # uncomment below to test default value of the base path
31
- # expect(config.base_url).to eq("https://api.webscraping.ai")
32
- end
33
-
34
- it 'should remove trailing slashes' do
35
- [nil, '', '/', '//'].each do |base_path|
36
- config.base_path = base_path
37
- # uncomment below to test trailing slashes
38
- # expect(config.base_url).to eq("https://api.webscraping.ai")
39
- end
40
- end
41
- end
42
- end
@@ -1,47 +0,0 @@
1
- =begin
2
- #WebScraping.AI
3
-
4
- #A client for https://webscraping.ai API. It provides a web scaping automation API with Chrome JS rendering, rotating proxies and builtin HTML parsing.
5
-
6
- The version of the OpenAPI document: 2.0.2
7
- Contact: support@webscraping.ai
8
- Generated by: https://openapi-generator.tech
9
- OpenAPI Generator version: 4.3.1
10
-
11
- =end
12
-
13
- require 'spec_helper'
14
- require 'json'
15
- require 'date'
16
-
17
- # Unit tests for WebScrapingAI::PageError
18
- # Automatically generated by openapi-generator (https://openapi-generator.tech)
19
- # Please update as you see appropriate
20
- describe 'PageError' do
21
- before do
22
- # run before each test
23
- @instance = WebScrapingAI::PageError.new
24
- end
25
-
26
- after do
27
- # run after each test
28
- end
29
-
30
- describe 'test an instance of PageError' do
31
- it 'should create an instance of PageError' do
32
- expect(@instance).to be_instance_of(WebScrapingAI::PageError)
33
- end
34
- end
35
- describe 'test attribute "status_code"' do
36
- it 'should work' do
37
- # assertion here. ref: https://www.relishapp.com/rspec/rspec-expectations/docs/built-in-matchers
38
- end
39
- end
40
-
41
- describe 'test attribute "status_message"' do
42
- it 'should work' do
43
- # assertion here. ref: https://www.relishapp.com/rspec/rspec-expectations/docs/built-in-matchers
44
- end
45
- end
46
-
47
- end