blackstack-core 1.2.14 → 1.2.16

Sign up to get free protection for your applications and to get access to all the features.
Files changed (4) hide show
  1. checksums.yaml +4 -4
  2. data/lib/blackstack-core.rb +1 -0
  3. data/lib/functions.rb +272 -69
  4. metadata +4 -5
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: b95d9103bd0e1ca4e6592838f394cb973c8da2c95e4c3a3ed85825a2eb300770
4
- data.tar.gz: '08c66d9a524d6095153ed68b573e21c0dbdbe9ffecfb96341a7204559b1707f1'
3
+ metadata.gz: adc18115deed231dca18ea1f2b4d33ae28c13cad49bcf0e31de86f1ca31a2f0d
4
+ data.tar.gz: 1d56c4a2343240ff77716a6a7ea5a8a92e5811180158a09f20235887079f7f36
5
5
  SHA512:
6
- metadata.gz: 4bd8f56f2f3e85079be9d8ca3422a296f0cc8077ecf182ccb07c82001330f24ee106a4c50216c1272f3b8eed469357239e9a2c114038e6920715c7da45fbfc8c
7
- data.tar.gz: e1bfc3cc67e81725ba92ddcc1902f2d32fe104ce7eceadd01ff2597b824832b49bab39c2a1f0644a3b6c88af728983fc1209279d7c315bf6f30d908bdc21b0cd
6
+ metadata.gz: 2b386f7ef47e29f1dbe0e73f49a2e5ee404d71db92c51dfeb5eb599cb2909be3dd7264f70fbc34cf2e57c186410ea8cd90fea5af81b8df99215b20867ed08651
7
+ data.tar.gz: a07ee4d466e602e7e5b6aa6a274eb38266bcb01347ff45fa32c58ac5d2985c61a29a3ca4e8f9a708f89594a82b6a50968b6c8ffe5bd95dd407f11295fe04f26f
@@ -1,3 +1,4 @@
1
+ require 'sequel'
1
2
  require 'pry'
2
3
  require 'content_spinning'
3
4
  require 'uri'
data/lib/functions.rb CHANGED
@@ -6,88 +6,291 @@ module BlackStack
6
6
  File.exists?('./.sandbox')
7
7
  end
8
8
 
9
- module API
10
- # API connection parameters
11
- @@api_key = nil # Api-key of the client who will be the owner of a process.
12
- @@api_protocol = nil # Protocol, HTTP or HTTPS, of the BlackStack server where this process will be registered.
13
- @@api_domain = nil # Domain of the BlackStack server where this process will be registered.
14
- @@api_port = nil # Port of the BlackStack server where this process will be registered.
15
- @@api_less_secure_port = nil # Port of the BlackStack server where this process will be registered.
16
-
17
- # API connection getters and setters
18
- def self.api_key()
19
- @@api_key
20
- end
21
- def self.api_protocol
22
- @@api_protocol
23
- end
24
- def self.api_domain
25
- @@api_domain
26
- end
27
- def self.api_port
28
- @@api_port
29
- end
30
- def self.api_less_secure_port
31
- @@api_less_secure_port
32
- end
33
- def self.api_url()
34
- "#{BlackStack::API.api_protocol}://#{BlackStack::API.api_domain}:#{BlackStack::API.api_port}"
35
- end
36
- def self.api_less_secure_url()
37
- "http://#{BlackStack::API.api_domain}:#{BlackStack::API.api_less_secure_port}"
38
- end
39
- def self.set_api_key(s)
40
- # validate: the parameter s is required
41
- raise 'The parameter s is required' unless s
9
+ module BlackStack
10
+ module API
11
+ @@api_key = nil
12
+ @@api_url = nil
13
+ @@api_port = nil
14
+ @@api_version = '1.0'
15
+ @@classes = {}
42
16
 
43
- # validate: the parameter s must be a string
44
- raise 'The parameter s must be a string' unless s.is_a?(String)
17
+ def self.api_key
18
+ @@api_key
19
+ end # def self.api_key
45
20
 
46
- # map values
47
- @@api_key = s
48
- end
49
- def self.set_api_url(h)
50
- # validate: the parameter h is requred
51
- raise "The parameter h is required." if h.nil?
21
+ def self.api_url
22
+ @@api_url
23
+ end
52
24
 
53
- # validate: the parameter h must be a hash
54
- raise "The parameter h must be a hash" unless h.is_a?(Hash)
25
+ def self.api_version
26
+ @@api_version
27
+ end # def self.api_version
55
28
 
56
- # validate: the :api_key key is required
57
- raise 'The key :api_key is required' unless h.has_key?(:api_key)
29
+ def self.api_port
30
+ @@api_port
31
+ end # def self.api_port
58
32
 
59
- # validate: the :api_protocol key is required
60
- raise 'The key :api_protocol is required' unless h.has_key?(:api_domain)
33
+ def self.api_url()
34
+ "#{BlackStack::API.api_protocol}://#{BlackStack::API.api_domain}:#{BlackStack::API.api_port}"
35
+ end
61
36
 
62
- # validate: the :api_domain key is required
63
- raise 'The key :api_domain is required' unless h.has_key?(:api_domain)
37
+ def self.classes
38
+ @@classes
39
+ end # def self.classes
40
+
41
+ def self.set_client(
42
+ api_key: ,
43
+ api_url: ,
44
+ api_version: '1.0',
45
+ api_port: nil
46
+ )
47
+ @@api_key = api_key
48
+ @@api_url = api_url
49
+ @@api_version = api_version
50
+ @@api_port = api_port
51
+ end # def self.set_client
64
52
 
65
- # validate: the :api_port key is required
66
- raise 'The key :api_port is required' unless h.has_key?(:api_port)
53
+ def self.set_server(
54
+ classes: {}
55
+ )
56
+ @@classes = classes
57
+ end # def self.set_server
67
58
 
68
- # validate: the :api_key key is a string
69
- raise 'The key :api_key must be a string' unless h[:api_key].is_a?(String)
59
+ def self.post(
60
+ endpoint: ,
61
+ params: {}
62
+ )
63
+ begin
64
+ url = "#{@@api_url}:#{@@api_port}/api#{@@api_version}/#{endpoint}.json"
65
+ params['api_key'] = @@api_key
66
+ res = BlackStack::Netting.call_post(url, params)
67
+ parsed = JSON.parse(res.body)
68
+ return JSON.parse(res.body)
70
69
 
71
- # validate: the :api_protocol key is a string
72
- raise 'The key :api_protocol must be a string' unless h[:api_protocol].is_a?(String)
70
+ ## write response.body into a file
71
+ # File.open('response.body.html', 'w') { |file| file.write(res.body) }
72
+ rescue => e
73
+ return {
74
+ 'status' => e.message,
75
+ 'value' => e.to_console
76
+ }
77
+ end
78
+ end # def self.post
73
79
 
74
- # validate: the :api_domain key is a string
75
- raise 'The key :api_domain must be a string' unless h[:api_domain].is_a?(String)
80
+ end # module API
81
+ end # module Mass
76
82
 
77
- # validate: the :api_port key is an integer, or a string that can be converted to an integer
78
- raise 'The key :api_port must be an integer' unless h[:api_port].is_a?(Integer) || (h[:api_port].is_a?(String) && h[:api_port].to_i.to_s == h[:api_port])
79
83
 
80
- # validate: the :api_less_secure_port key is an integer, or a string that can be converted to an integer
81
- raise 'The key :api_less_secure_port must be an integer' unless h[:api_less_secure_port].is_a?(Integer) || (h[:api_less_secure_port].is_a?(String) && h[:api_less_secure_port].to_i.to_s == h[:api_less_secure_port])
84
+ module BlackStack
85
+ # Base class.
86
+ # List of methods you have to overload if you develop a profile type.
87
+ #
88
+ class Base
89
+ # object json descriptor
90
+ attr_accessor :desc
82
91
 
83
- # map the values
84
- @@api_key = h[:api_key]
85
- @@api_protocol = h[:api_protocol]
86
- @@api_domain = h[:api_domain]
87
- @@api_port = h[:api_port].to_i
88
- @@api_less_secure_port = h[:api_less_secure_port].to_i
89
- end
90
- end # module API
92
+ def self.object_name
93
+ raise 'You have to overload this method in your class.'
94
+ end
95
+
96
+ def initialize(h)
97
+ self.desc = h
98
+ end
99
+
100
+ # Crate an instance of a child class using speicfications in the `desc['name']` attribute.
101
+ # By default, returns the same instance.
102
+ def child_class_instance
103
+ return self
104
+ end
105
+
106
+ def self.account_value(field:)
107
+ params = {}
108
+ params['field'] = field
109
+ # call the API
110
+ ret = BlackStack::API.post(
111
+ endpoint: "account_value",
112
+ params: params
113
+ )
114
+ raise "Error calling account_value endpoint: #{ret['status']}" if ret['status'] != 'success'
115
+ return ret['result']
116
+ end # def self.base
117
+
118
+
119
+ # Get array of hash descriptor of profile.
120
+ #
121
+ # Parameters:
122
+ # - page: integer. Page number.
123
+ # - limit: integer. Number of profiles per page.
124
+ # - params: hash. Additional filter parameters used by the specific child class.
125
+ #
126
+ def self.page(page:, limit:, filters: {})
127
+ # add page and limit to the params
128
+ params = {}
129
+ params['page'] = page
130
+ params['limit'] = limit
131
+ params['filters'] = filters
132
+ params['backtrace'] = Mass.backtrace
133
+ # call the API
134
+ ret = BlackStack::API.post(
135
+ endpoint: "#{self.object_name}/page",
136
+ params: params
137
+ )
138
+ raise "Error calling page endpoint: #{ret['status']}" if ret['status'] != 'success'
139
+ return ret['results'].map { |h| self.new(h).child_class_instance }
140
+ end # def self.base
141
+
142
+ # Get array of hash descriptors of profiles.
143
+ #
144
+ # Parameters:
145
+ # - id: guid. Id of the profile to bring.
146
+ #
147
+ def self.count
148
+ params = {}
149
+ params['backtrace'] = Mass.backtrace
150
+ ret = BlackStack::API.post(
151
+ endpoint: "#{self.object_name}/count",
152
+ params: params
153
+ )
154
+ raise "Error calling count endpoint: #{ret['status']}" if ret['status'] != 'success'
155
+ return ret['result'].to_i
156
+ end # def self.count
157
+
158
+ # Get array of hash descriptors of profiles.
159
+ #
160
+ # Parameters:
161
+ # - id: guid. Id of the profile to bring.
162
+ #
163
+ def self.get(id)
164
+ params = {}
165
+ params['id'] = id
166
+ params['backtrace'] = Mass.backtrace
167
+ ret = BlackStack::API.post(
168
+ endpoint: "#{self.object_name}/get",
169
+ params: params
170
+ )
171
+ raise "Error calling get endpoint: #{ret['status']}" if ret['status'] != 'success'
172
+ return self.new(ret['result']).child_class_instance
173
+ end # def self.get
174
+
175
+ # Submit a hash descriptor to the server for an update
176
+ #
177
+ def self.update(desc)
178
+ params = {}
179
+ params['desc'] = desc
180
+ params['backtrace'] = Mass.backtrace
181
+ ret = BlackStack::API.post(
182
+ endpoint: "#{self.object_name}/update",
183
+ params: params
184
+ )
185
+ raise "Error calling update endpoint: #{ret['status']}" if ret['status'] != 'success'
186
+ return self.new(ret['result']).child_class_instance
187
+ end # def self.update
188
+
189
+ # Submit a hash descriptor to the server for an update
190
+ #
191
+ def update
192
+ self.class.update(self.desc)
193
+ end
194
+
195
+ # Submit a hash descriptor to the server for an insert
196
+ #
197
+ def self.insert(desc)
198
+ params = {}
199
+ params['desc'] = desc
200
+ params['backtrace'] = Mass.backtrace
201
+ ret = BlackStack::API.post(
202
+ endpoint: "#{self.object_name}/insert",
203
+ params: params
204
+ )
205
+ raise "Error calling insert endpoint: #{ret['status']}" if ret['status'] != 'success'
206
+ return self.new(ret['result']).child_class_instance
207
+ end # def self.insert
208
+
209
+
210
+ # Submit a hash descriptor to the server for an upsert
211
+ #
212
+ def self.upsert(desc)
213
+ params = {}
214
+ params['desc'] = desc
215
+ params['backtrace'] = Mass.backtrace
216
+ ret = BlackStack::API.post(
217
+ endpoint: "#{self.object_name}/upsert",
218
+ params: params
219
+ )
220
+ raise "Error calling upsert endpoint: #{ret['status']}" if ret['status'] != 'success'
221
+ return self.new(ret['result']).child_class_instance
222
+ end # def self.upsert
223
+
224
+ # Submit a hash descriptor to the server for an upsert
225
+ #
226
+ def upsert
227
+ self.class.upsert(self.desc)
228
+ end
229
+
230
+
231
+ # return the HTML of a page downloaded by Zyte.
232
+ #
233
+ # Parameters:
234
+ # - url: the URL of the page to download.
235
+ # - api_key: the Zyte API key.
236
+ # - options: the options to pass to Zyte.
237
+ #
238
+ def zyte_html(url, api_key:, options:, data_filename:)
239
+ ret = nil
240
+ # getting the HTML
241
+ zyte = ZyteClient.new(key: api_key)
242
+ html = zyte.extract(url: url, options: options, data_filename: data_filename)
243
+ # return the URL of the file in the cloud
244
+ return html
245
+ end # def zyte_html
246
+
247
+ # create a file in the cloud with the HTML of a page downloaded by Zyte.
248
+ # return the URL of the file.
249
+ #
250
+ # Parameters:
251
+ # - url: the URL of the page to download.
252
+ # - api_key: the Zyte API key.
253
+ # - options: the options to pass to Zyte.
254
+ # - dropbox_folder: the folder in the cloud where to store the file. If nil, it will use the self.desc['id_account'] value.
255
+ # - retry_times: the number of times to retry the download until the HTML is valid.
256
+ #
257
+ def zyte_snapshot(url, api_key:, options:, data_filename:, dropbox_folder:nil, retry_times: 3)
258
+ # "The garbage character must be due to the 520 error code which was caused on the second request."
259
+ garbage = "\x9E\xE9e"
260
+
261
+ ret = nil
262
+ raise "Either dropbox_folder parameter or self.desc['id_account'] are required." if dropbox_folder.nil? && self.desc['id_account'].nil?
263
+ dropbox_folder = self.desc['id_account'] if dropbox_folder.nil?
264
+ # build path to the local file in /tmp
265
+ id = SecureRandom.uuid
266
+ filename = "#{id}.html"
267
+ tmp_path = "/tmp/#{filename}"
268
+ # build path to the file in the cloud
269
+ year = Time.now.year.to_s.rjust(4,'0')
270
+ month = Time.now.month.to_s.rjust(2,'0')
271
+ dropbox_folder = "/massprospecting.bots/#{dropbox_folder}.#{year}.#{month}"
272
+ dropbox_path = "#{dropbox_folder}/#{filename}"
273
+ # getting the HTML - Retry mechanism
274
+ zyte = ZyteClient.new(key: api_key)
275
+ try = 0
276
+ html = garbage
277
+ while try < retry_times && html == garbage
278
+ html = zyte.extract(url: url, options: options, data_filename: data_filename)
279
+ try += 1
280
+ end
281
+ # save the HTML in the local file in /tmp
282
+ File.open(tmp_path, 'w') { |file| file.write(html) }
283
+ # create the folder in the cloud and upload the file
284
+ BlackStack::DropBox.dropbox_create_folder(dropbox_folder)
285
+ BlackStack::DropBox.dropbox_upload_file(tmp_path, dropbox_path)
286
+ # delete the local file
287
+ File.delete(tmp_path)
288
+ # return the URL of the file in the cloud
289
+ return BlackStack::DropBox.get_file_url(dropbox_path)
290
+ end # def zyte_snapshot
291
+
292
+ end # class Base
293
+ end # module Mass
91
294
 
92
295
  # -----------------------------------------------------------------------------------------
93
296
  # PRY Supporting Functions
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: blackstack-core
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.2.14
4
+ version: 1.2.16
5
5
  platform: ruby
6
6
  authors:
7
7
  - Leandro Daniel Sardi
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2023-08-17 00:00:00.000000000 Z
11
+ date: 2024-07-26 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: content_spinning
@@ -90,7 +90,7 @@ dependencies:
90
90
  - - ">="
91
91
  - !ruby/object:Gem::Version
92
92
  version: 2.3.0
93
- description: 'THIS GEM IS STILL IN DEVELOPMENT STAGE. Find documentation here: https://github.com/leandrosardi/blackstack-core.'
93
+ description: 'Find documentation here: https://github.com/leandrosardi/blackstack-core.'
94
94
  email: leandro.sardi@expandedventure.com
95
95
  executables: []
96
96
  extensions: []
@@ -128,6 +128,5 @@ requirements: []
128
128
  rubygems_version: 3.3.7
129
129
  signing_key:
130
130
  specification_version: 4
131
- summary: THIS GEM IS STILL IN DEVELOPMENT STAGE. Core classes, functions and constants
132
- for the BlackStack framework.
131
+ summary: Core modules, functions and constants for the BlackStack framework.
133
132
  test_files: []