blackstack-core 1.2.28 → 1.2.30
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/examples/example01.rb +0 -0
- data/examples/example02.rb +0 -0
- data/lib/blackstack-core.rb +0 -0
- data/lib/extend_datetime.rb +0 -0
- data/lib/extend_exception.rb +0 -0
- data/lib/extend_fixnum.rb +0 -0
- data/lib/extend_float.rb +0 -0
- data/lib/extend_string.rb +0 -0
- data/lib/extend_time.rb +0 -0
- data/lib/functions.rb +269 -183
- metadata +3 -6
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 6319585aa6afd55c363b5b7e59acc4afe407a741d22306831b3c5cef48724c76
|
|
4
|
+
data.tar.gz: 2d7e134595693737c9beb9cfabaf05ac502a7631a79bafbbb5b60ae03cd90512
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: f6d4de6e017a5871b60a1d1f3a987e9233ac2df83700ba3d794ddf843f5ffd3ec30de9efd4544eb4415e4f9381898826dd80647d8250c89d6d0f3001c38355ed
|
|
7
|
+
data.tar.gz: 3e8c922cb98658ef9677b4e5badb5e4592f112b71d6a76d79ac59361f19cfbb0ad94558bbaea5a6ad99493a876eda33f224ccdde11e0cebc4d4e09716a70c0a7
|
data/examples/example01.rb
CHANGED
|
File without changes
|
data/examples/example02.rb
CHANGED
|
File without changes
|
data/lib/blackstack-core.rb
CHANGED
|
File without changes
|
data/lib/extend_datetime.rb
CHANGED
|
File without changes
|
data/lib/extend_exception.rb
CHANGED
|
File without changes
|
data/lib/extend_fixnum.rb
CHANGED
|
File without changes
|
data/lib/extend_float.rb
CHANGED
|
File without changes
|
data/lib/extend_string.rb
CHANGED
|
File without changes
|
data/lib/extend_time.rb
CHANGED
|
File without changes
|
data/lib/functions.rb
CHANGED
|
@@ -20,7 +20,7 @@ module BlackStack
|
|
|
20
20
|
|
|
21
21
|
def self.api_url
|
|
22
22
|
@@api_url
|
|
23
|
-
end
|
|
23
|
+
end
|
|
24
24
|
|
|
25
25
|
def self.api_port
|
|
26
26
|
@@api_port
|
|
@@ -37,7 +37,7 @@ module BlackStack
|
|
|
37
37
|
def self.classes
|
|
38
38
|
@@classes
|
|
39
39
|
end # def self.classes
|
|
40
|
-
|
|
40
|
+
|
|
41
41
|
def self.set_client(
|
|
42
42
|
api_key: ,
|
|
43
43
|
api_url: ,
|
|
@@ -60,7 +60,7 @@ module BlackStack
|
|
|
60
60
|
end # def self.set_server
|
|
61
61
|
|
|
62
62
|
def self.post(
|
|
63
|
-
endpoint: ,
|
|
63
|
+
endpoint: ,
|
|
64
64
|
params: {}
|
|
65
65
|
)
|
|
66
66
|
begin
|
|
@@ -84,7 +84,7 @@ module BlackStack
|
|
|
84
84
|
|
|
85
85
|
# Base class.
|
|
86
86
|
# List of methods you have to overload if you develop a profile type.
|
|
87
|
-
#
|
|
87
|
+
#
|
|
88
88
|
class Base
|
|
89
89
|
# object json descriptor
|
|
90
90
|
attr_accessor :desc
|
|
@@ -120,10 +120,10 @@ module BlackStack
|
|
|
120
120
|
|
|
121
121
|
|
|
122
122
|
# Get array of hash descriptor of profile.
|
|
123
|
-
#
|
|
124
|
-
# Parameters:
|
|
123
|
+
#
|
|
124
|
+
# Parameters:
|
|
125
125
|
# - id_account: guid. Id of the account to bring the profiles. Sysowner must provide the id_account for getting an account value. For non sysowner it is assigned to his account.
|
|
126
|
-
# - promiscuous: boolean. It works only for Sysowner. If true, it will bring all non-deleted rows. If false, it will bring only rows matching id_profile. Default: false.
|
|
126
|
+
# - promiscuous: boolean. It works only for Sysowner. If true, it will bring all non-deleted rows, including the ones that are not owned by Sysowner. If false, it will bring only rows matching id_profile. Default: false.
|
|
127
127
|
# - page: integer. Page number.
|
|
128
128
|
# - limit: integer. Number of profiles per page.
|
|
129
129
|
# - params: hash. Additional filter parameters used by the specific child class.
|
|
@@ -147,8 +147,8 @@ module BlackStack
|
|
|
147
147
|
end # def self.base
|
|
148
148
|
|
|
149
149
|
# Get array of hash descriptors of profiles.
|
|
150
|
-
#
|
|
151
|
-
# Parameters:
|
|
150
|
+
#
|
|
151
|
+
# Parameters:
|
|
152
152
|
# - id: guid. Id of the profile to bring.
|
|
153
153
|
#
|
|
154
154
|
def self.count
|
|
@@ -163,8 +163,8 @@ module BlackStack
|
|
|
163
163
|
end # def self.count
|
|
164
164
|
|
|
165
165
|
# Get array of hash descriptors of profiles.
|
|
166
|
-
#
|
|
167
|
-
# Parameters:
|
|
166
|
+
#
|
|
167
|
+
# Parameters:
|
|
168
168
|
# - id: guid. Id of the profile to bring.
|
|
169
169
|
#
|
|
170
170
|
def self.get(id)
|
|
@@ -179,6 +179,23 @@ module BlackStack
|
|
|
179
179
|
return self.new(ret['result']).child_class_instance
|
|
180
180
|
end # def self.get
|
|
181
181
|
|
|
182
|
+
# Delete object.
|
|
183
|
+
#
|
|
184
|
+
# Parameters:
|
|
185
|
+
# - id: guid. Id of the profile to bring.
|
|
186
|
+
#
|
|
187
|
+
def self.delete(id)
|
|
188
|
+
params = {}
|
|
189
|
+
params['id'] = id
|
|
190
|
+
params['backtrace'] = BlackStack::API.backtrace
|
|
191
|
+
ret = BlackStack::API.post(
|
|
192
|
+
endpoint: "#{self.object_name}/delete",
|
|
193
|
+
params: params
|
|
194
|
+
)
|
|
195
|
+
raise "Error calling get endpoint: #{ret['status']}" if ret['status'] != 'success'
|
|
196
|
+
return self.new(ret['result']).child_class_instance
|
|
197
|
+
end # def self.get
|
|
198
|
+
|
|
182
199
|
# Submit a hash descriptor to the server for an update
|
|
183
200
|
#
|
|
184
201
|
def self.update(desc)
|
|
@@ -248,6 +265,20 @@ module BlackStack
|
|
|
248
265
|
return ret['result'] == {} ? nil : self.new(ret['result']).child_class_instance
|
|
249
266
|
end # def self.upsert
|
|
250
267
|
|
|
268
|
+
# Submit a hash descriptor to the server for an upsert
|
|
269
|
+
#
|
|
270
|
+
def self.upsert2(desc)
|
|
271
|
+
params = {}
|
|
272
|
+
params['desc'] = desc
|
|
273
|
+
params['backtrace'] = BlackStack::API.backtrace
|
|
274
|
+
ret = BlackStack::API.post(
|
|
275
|
+
endpoint: "#{self.object_name}/upsert2",
|
|
276
|
+
params: params
|
|
277
|
+
)
|
|
278
|
+
raise "Error calling upsert endpoint: #{ret['status']}" if ret['status'] != 'success'
|
|
279
|
+
return ret['result'] == {} ? nil : self.new(ret['result']).child_class_instance
|
|
280
|
+
end # def self.upsert
|
|
281
|
+
|
|
251
282
|
# Submit a hash descriptor to the server for an upsert
|
|
252
283
|
#
|
|
253
284
|
def upsert
|
|
@@ -266,7 +297,7 @@ module BlackStack
|
|
|
266
297
|
ret = nil
|
|
267
298
|
# getting the HTML
|
|
268
299
|
zyte = ZyteClient.new(key: api_key)
|
|
269
|
-
html = zyte.extract(url: url, options: options, data_filename: data_filename)
|
|
300
|
+
html = zyte.extract(url: url, options: options, data_filename: data_filename)
|
|
270
301
|
# return the URL of the file in the cloud
|
|
271
302
|
return html
|
|
272
303
|
end # def zyte_html
|
|
@@ -284,7 +315,7 @@ module BlackStack
|
|
|
284
315
|
def zyte_snapshot(url, api_key:, options:, data_filename:, dropbox_folder:nil, retry_times: 3)
|
|
285
316
|
# "The garbage character must be due to the 520 error code which was caused on the second request."
|
|
286
317
|
garbage = "\x9E\xE9e"
|
|
287
|
-
|
|
318
|
+
|
|
288
319
|
ret = nil
|
|
289
320
|
raise "Either dropbox_folder parameter or self.desc['id_account'] are required." if dropbox_folder.nil? && self.desc['id_account'].nil?
|
|
290
321
|
dropbox_folder = self.desc['id_account'] if dropbox_folder.nil?
|
|
@@ -302,7 +333,7 @@ module BlackStack
|
|
|
302
333
|
try = 0
|
|
303
334
|
html = garbage
|
|
304
335
|
while try < retry_times && html == garbage
|
|
305
|
-
html = zyte.extract(url: url, options: options, data_filename: data_filename)
|
|
336
|
+
html = zyte.extract(url: url, options: options, data_filename: data_filename)
|
|
306
337
|
try += 1
|
|
307
338
|
end
|
|
308
339
|
# save the HTML in the local file in /tmp
|
|
@@ -318,9 +349,9 @@ module BlackStack
|
|
|
318
349
|
|
|
319
350
|
end # class Base
|
|
320
351
|
|
|
321
|
-
# -----------------------------------------------------------------------------------------
|
|
352
|
+
# -----------------------------------------------------------------------------------------
|
|
322
353
|
# PRY Supporting Functions
|
|
323
|
-
# -----------------------------------------------------------------------------------------
|
|
354
|
+
# -----------------------------------------------------------------------------------------
|
|
324
355
|
module Debugging
|
|
325
356
|
@@allow_breakpoints = false
|
|
326
357
|
@@verbose = false
|
|
@@ -341,7 +372,7 @@ module BlackStack
|
|
|
341
372
|
print "Breakpoint are not allowed" if @@verbose
|
|
342
373
|
end
|
|
343
374
|
|
|
344
|
-
Binding.class_eval do
|
|
375
|
+
Binding.class_eval do
|
|
345
376
|
alias_method :old_pry, :pry
|
|
346
377
|
define_method :pry, new_pry
|
|
347
378
|
end
|
|
@@ -349,21 +380,21 @@ module BlackStack
|
|
|
349
380
|
end
|
|
350
381
|
end
|
|
351
382
|
|
|
352
|
-
# -----------------------------------------------------------------------------------------
|
|
383
|
+
# -----------------------------------------------------------------------------------------
|
|
353
384
|
# OCRA Supporting Functions
|
|
354
|
-
# -----------------------------------------------------------------------------------------
|
|
385
|
+
# -----------------------------------------------------------------------------------------
|
|
355
386
|
module OCRA
|
|
356
387
|
# OCRA files run into a temp folder, where the script is unpacked.
|
|
357
|
-
#
|
|
388
|
+
#
|
|
358
389
|
# This function is useful to require a configuration file when the
|
|
359
390
|
# script is running inside an OCRA temp folder, since the local folder
|
|
360
391
|
# of the running command is not the filder where the exe file is hosted.
|
|
361
|
-
#
|
|
362
|
-
# More information:
|
|
392
|
+
#
|
|
393
|
+
# More information:
|
|
363
394
|
# * https://stackoverflow.com/questions/1937743/how-to-get-the-current-working-directorys-absolute-path-from-irb
|
|
364
395
|
# * https://stackoverflow.com/questions/8577223/ruby-get-the-file-being-executed
|
|
365
396
|
# * https://stackoverflow.com/questions/7399882/ruby-getting-path-from-pathfilename/7400057
|
|
366
|
-
#
|
|
397
|
+
#
|
|
367
398
|
def self.require_in_working_path(filename, path, show_path_info=false)
|
|
368
399
|
puts '' if show_path_info
|
|
369
400
|
path = File.expand_path File.dirname(path)
|
|
@@ -375,13 +406,13 @@ module BlackStack
|
|
|
375
406
|
end
|
|
376
407
|
end # module OCRA
|
|
377
408
|
|
|
378
|
-
# -----------------------------------------------------------------------------------------
|
|
409
|
+
# -----------------------------------------------------------------------------------------
|
|
379
410
|
# DateTime Functions
|
|
380
|
-
# -----------------------------------------------------------------------------------------
|
|
381
|
-
module DateTime
|
|
382
|
-
# -----------------------------------------------------------------------------------------
|
|
411
|
+
# -----------------------------------------------------------------------------------------
|
|
412
|
+
module DateTime
|
|
413
|
+
# -----------------------------------------------------------------------------------------
|
|
383
414
|
# Encoding
|
|
384
|
-
# -----------------------------------------------------------------------------------------
|
|
415
|
+
# -----------------------------------------------------------------------------------------
|
|
385
416
|
module Encoding
|
|
386
417
|
# Convierte un objeto date-time a un string con formato sql-datetime (yyyy-mm-dd hh:mm:ss).
|
|
387
418
|
def self.datetime_to_sql(o)
|
|
@@ -389,50 +420,50 @@ module BlackStack
|
|
|
389
420
|
end
|
|
390
421
|
end # module Encode
|
|
391
422
|
|
|
392
|
-
# -----------------------------------------------------------------------------------------
|
|
423
|
+
# -----------------------------------------------------------------------------------------
|
|
393
424
|
# Miscelaneous
|
|
394
|
-
# -----------------------------------------------------------------------------------------
|
|
425
|
+
# -----------------------------------------------------------------------------------------
|
|
395
426
|
module Misc
|
|
396
427
|
def self.datetime_values_check(year,month,day,hour,minute,second)
|
|
397
428
|
if (year.to_i<1900 || year.to_i>=2100)
|
|
398
429
|
return false
|
|
399
430
|
end
|
|
400
|
-
|
|
431
|
+
|
|
401
432
|
if (month.to_i<1 || month.to_i>12)
|
|
402
433
|
return false
|
|
403
434
|
end
|
|
404
|
-
|
|
435
|
+
|
|
405
436
|
# TODO: Considerar la cantidad de dias de cada mes, y los anios biciestos. Buscar alguna funcion existente.
|
|
406
437
|
if (day.to_i<1 || day.to_i>31)
|
|
407
438
|
return false
|
|
408
439
|
end
|
|
409
|
-
|
|
440
|
+
|
|
410
441
|
if (hour.to_i<0 || hour.to_i>23)
|
|
411
442
|
return false
|
|
412
443
|
end
|
|
413
|
-
|
|
444
|
+
|
|
414
445
|
if (minute.to_i<0 || minute.to_i>59)
|
|
415
446
|
return false
|
|
416
447
|
end
|
|
417
|
-
|
|
448
|
+
|
|
418
449
|
if (second.to_i<0 || second.to_i>59)
|
|
419
450
|
return false
|
|
420
451
|
end
|
|
421
|
-
|
|
452
|
+
|
|
422
453
|
return true
|
|
423
454
|
end # datetime_values_check
|
|
424
455
|
end # module Misc
|
|
425
456
|
end # module DateTime
|
|
426
457
|
|
|
427
|
-
# -----------------------------------------------------------------------------------------
|
|
458
|
+
# -----------------------------------------------------------------------------------------
|
|
428
459
|
# Numeric Functions
|
|
429
|
-
# -----------------------------------------------------------------------------------------
|
|
460
|
+
# -----------------------------------------------------------------------------------------
|
|
430
461
|
module Number
|
|
431
|
-
# -----------------------------------------------------------------------------------------
|
|
462
|
+
# -----------------------------------------------------------------------------------------
|
|
432
463
|
# Encoding
|
|
433
|
-
# -----------------------------------------------------------------------------------------
|
|
464
|
+
# -----------------------------------------------------------------------------------------
|
|
434
465
|
module Encoding
|
|
435
|
-
# Converts number to a string with a format like xx,xxx,xxx.xxxx
|
|
466
|
+
# Converts number to a string with a format like xx,xxx,xxx.xxxx
|
|
436
467
|
# number: it may be int or float
|
|
437
468
|
def self.format_with_separator(number)
|
|
438
469
|
whole_part, decimal_part = number.to_s.split('.')
|
|
@@ -444,7 +475,7 @@ module BlackStack
|
|
|
444
475
|
# Ejemplo: "4 hours, 30 minutes"
|
|
445
476
|
# Ejemplo: "3 days, 4 hour"
|
|
446
477
|
def self.encode_minutes(n)
|
|
447
|
-
# TODO: validar que n sea un entero mayor a 0
|
|
478
|
+
# TODO: validar que n sea un entero mayor a 0
|
|
448
479
|
if (n<0)
|
|
449
480
|
return "?"
|
|
450
481
|
end
|
|
@@ -459,13 +490,13 @@ module BlackStack
|
|
|
459
490
|
end # module Encode
|
|
460
491
|
end # module Number
|
|
461
492
|
|
|
462
|
-
# -----------------------------------------------------------------------------------------
|
|
493
|
+
# -----------------------------------------------------------------------------------------
|
|
463
494
|
# String Functions
|
|
464
|
-
# -----------------------------------------------------------------------------------------
|
|
495
|
+
# -----------------------------------------------------------------------------------------
|
|
465
496
|
module Strings
|
|
466
497
|
|
|
467
498
|
GUID_SIZE = 36
|
|
468
|
-
MATCH_PASSWORD = /(?=.*[a-zA-Z])(?=.*[0-9]).{6,}/
|
|
499
|
+
MATCH_PASSWORD = /(?=.*[a-zA-Z])(?=.*[0-9]).{6,}/
|
|
469
500
|
MATCH_GUID = /{?[0-9a-fA-F][0-9a-fA-F][0-9a-fA-F][0-9a-fA-F][0-9a-fA-F][0-9a-fA-F][0-9a-fA-F][0-9a-fA-F]\-[0-9a-fA-F][0-9a-fA-F][0-9a-fA-F][0-9a-fA-F]\-[0-9a-fA-F][0-9a-fA-F][0-9a-fA-F][0-9a-fA-F]-[0-9a-fA-F][0-9a-fA-F][0-9a-fA-F][0-9a-fA-F]\-[0-9a-fA-F][0-9a-fA-F][0-9a-fA-F][0-9a-fA-F][0-9a-fA-F][0-9a-fA-F][0-9a-fA-F][0-9a-fA-F][0-9a-fA-F][0-9a-fA-F][0-9a-fA-F][0-9a-fA-F]}?/
|
|
470
501
|
MATCH_FILENAME = /[\w\-\_\.]+/
|
|
471
502
|
MATCH_EMAIL = /[A-Z0-9._%a-z\-]+@(?:[A-Z0-9a-z\-]+\.)+[A-Za-z]{1,25}/
|
|
@@ -474,7 +505,7 @@ module BlackStack
|
|
|
474
505
|
MATCH_PHONE = /(?:\+\d{1,2}\s)?\(?\d{3}\)?[\s.-]?\d{3}[\s.-]?\d{4}/
|
|
475
506
|
|
|
476
507
|
# Note: MATCH_URL gets the URL up to '?', but it doesn't retrieves the parameters.
|
|
477
|
-
# Exmaple:
|
|
508
|
+
# Exmaple:
|
|
478
509
|
# https://foo.com/bar?param1=value1¶m2=value2 --> https://foo.com/bar?
|
|
479
510
|
# https://foo.com/bar/?param1=value1¶m2=value2 --> https://foo.com/bar/?
|
|
480
511
|
MATCH_URL = /(https?:\/\/)?([\da-z\.-]+)([\.\:])([\da-z]{2,6})([\/[\da-z\.\-]+]*[\da-z])(\/)?(\?)?/i
|
|
@@ -484,32 +515,32 @@ module BlackStack
|
|
|
484
515
|
MATCH_CONTENT_SPINNING = /{[^}]+}/
|
|
485
516
|
MATCH_SPINNED_TEXT = /code me/ # TODO: define this regex for the issue #1226
|
|
486
517
|
|
|
487
|
-
# -----------------------------------------------------------------------------------------
|
|
518
|
+
# -----------------------------------------------------------------------------------------
|
|
488
519
|
# Fuzzy String Comparsion Functions: How similar are 2 strings that are not exactly equal.
|
|
489
|
-
# -----------------------------------------------------------------------------------------
|
|
520
|
+
# -----------------------------------------------------------------------------------------
|
|
490
521
|
module SQL
|
|
491
522
|
def self.string_to_sql_string(s)
|
|
492
523
|
#return s.force_encoding("UTF-8").gsub("'", "''").to_s
|
|
493
524
|
return s.gsub("'", "''").to_s
|
|
494
525
|
end
|
|
495
526
|
end
|
|
496
|
-
|
|
497
|
-
# -----------------------------------------------------------------------------------------
|
|
527
|
+
|
|
528
|
+
# -----------------------------------------------------------------------------------------
|
|
498
529
|
# Fuzzy String Comparsion Functions: How similar are 2 strings that are not exactly equal.
|
|
499
|
-
# -----------------------------------------------------------------------------------------
|
|
530
|
+
# -----------------------------------------------------------------------------------------
|
|
500
531
|
module Comparing
|
|
501
532
|
# retorna 0 si los strings son iguales
|
|
502
533
|
# https://stackoverflow.com/questions/16323571/measure-the-distance-between-two-strings-with-ruby
|
|
503
|
-
def self.levenshtein_distance(s, t)
|
|
534
|
+
def self.levenshtein_distance(s, t)
|
|
504
535
|
s.downcase!
|
|
505
536
|
t.downcase!
|
|
506
|
-
|
|
537
|
+
|
|
507
538
|
m = s.length
|
|
508
539
|
n = t.length
|
|
509
540
|
return m if n == 0
|
|
510
541
|
return n if m == 0
|
|
511
542
|
d = Array.new(m+1) {Array.new(n+1)}
|
|
512
|
-
|
|
543
|
+
|
|
513
544
|
(0..m).each {|i| d[i][0] = i}
|
|
514
545
|
(0..n).each {|j| d[0][j] = j}
|
|
515
546
|
(1..n).each do |j|
|
|
@@ -526,7 +557,7 @@ module BlackStack
|
|
|
526
557
|
end
|
|
527
558
|
d[m][n]
|
|
528
559
|
end
|
|
529
|
-
|
|
560
|
+
|
|
530
561
|
# retorna la cantidad de palabras con mas de 3 caracteres que se encuentran en el parametro s
|
|
531
562
|
def self.max_sardi_distance(s)
|
|
532
563
|
s.downcase!
|
|
@@ -541,14 +572,14 @@ module BlackStack
|
|
|
541
572
|
}
|
|
542
573
|
n
|
|
543
574
|
end
|
|
544
|
-
|
|
575
|
+
|
|
545
576
|
# retorna la cantidad de palabras con mas de 3 caracteres del parametro s que se encuentran en el parametro t
|
|
546
577
|
def self.sardi_distance(s, t)
|
|
547
578
|
s.downcase!
|
|
548
579
|
t.downcase!
|
|
549
580
|
s.gsub!(/-/,' ')
|
|
550
581
|
t.gsub!(/-/,' ')
|
|
551
|
-
max_distance = max_sardi_distance(s)
|
|
582
|
+
max_distance = max_sardi_distance(s)
|
|
552
583
|
ss = s.scan(/\b([a-z]+)\b/)
|
|
553
584
|
tt = t.scan(/\b([a-z]+)\b/)
|
|
554
585
|
n = 0
|
|
@@ -563,20 +594,20 @@ module BlackStack
|
|
|
563
594
|
return max_distance - n
|
|
564
595
|
end
|
|
565
596
|
end # module Comparing
|
|
566
|
-
|
|
567
|
-
# -----------------------------------------------------------------------------------------
|
|
597
|
+
|
|
598
|
+
# -----------------------------------------------------------------------------------------
|
|
568
599
|
# Encoding: Make a string nice to be shown into an HTML string.
|
|
569
|
-
# -----------------------------------------------------------------------------------------
|
|
600
|
+
# -----------------------------------------------------------------------------------------
|
|
570
601
|
module Encoding
|
|
571
602
|
# Then it makes it compatible with UTF-8.
|
|
572
|
-
# More details here: https://bitbucket.org/leandro_sardi/blackstack/issues/961
|
|
603
|
+
# More details here: https://bitbucket.org/leandro_sardi/blackstack/issues/961
|
|
573
604
|
def self.encode_string(s)
|
|
574
605
|
s.encode("UTF-8")
|
|
575
606
|
end
|
|
576
|
-
|
|
607
|
+
|
|
577
608
|
# Escape the string to be shown into an HTML screen.
|
|
578
609
|
# Then it makes it compatible with UTF-8.
|
|
579
|
-
# More details here: https://bitbucket.org/leandro_sardi/blackstack/issues/961
|
|
610
|
+
# More details here: https://bitbucket.org/leandro_sardi/blackstack/issues/961
|
|
580
611
|
def self.encode_html(s)
|
|
581
612
|
encode_string(CGI.escapeHTML(s.to_s))
|
|
582
613
|
end
|
|
@@ -584,9 +615,9 @@ module BlackStack
|
|
|
584
615
|
# Generates a description string from an exception object.
|
|
585
616
|
# Eescapes the string to be shown into an HTML screen.
|
|
586
617
|
# Makes it compatible with UTF-8.
|
|
587
|
-
# More details here: https://bitbucket.org/leandro_sardi/blackstack/issues/961
|
|
618
|
+
# More details here: https://bitbucket.org/leandro_sardi/blackstack/issues/961
|
|
588
619
|
def self.encode_exception(e, include_backtrace=true)
|
|
589
|
-
ret = encode_html(e.to_s)
|
|
620
|
+
ret = encode_html(e.to_s)
|
|
590
621
|
if (include_backtrace == true)
|
|
591
622
|
e.backtrace.each { |s|
|
|
592
623
|
ret += "<br/>" + encode_html(s)
|
|
@@ -596,8 +627,8 @@ module BlackStack
|
|
|
596
627
|
end
|
|
597
628
|
|
|
598
629
|
# Returns a string with a description of a period of time, to be shown in the screen.
|
|
599
|
-
# period: it may be 'H', 'D', 'W', 'M', 'Y'
|
|
600
|
-
# units: it is a positive integer
|
|
630
|
+
# period: it may be 'H', 'D', 'W', 'M', 'Y'
|
|
631
|
+
# units: it is a positive integer
|
|
601
632
|
def self.encode_period(period, units)
|
|
602
633
|
s = "Last "
|
|
603
634
|
s += units.to_i.to_s + " " if units.to_i > 1
|
|
@@ -626,87 +657,87 @@ module BlackStack
|
|
|
626
657
|
|
|
627
658
|
end # module Encoding
|
|
628
659
|
|
|
629
|
-
# -----------------------------------------------------------------------------------------
|
|
660
|
+
# -----------------------------------------------------------------------------------------
|
|
630
661
|
# DateTime
|
|
631
|
-
# -----------------------------------------------------------------------------------------
|
|
632
|
-
module DateTime
|
|
662
|
+
# -----------------------------------------------------------------------------------------
|
|
663
|
+
module DateTime
|
|
633
664
|
# Check the string has the format yyyymmddhhmmss.
|
|
634
665
|
# => Return true if success. Otherwise, return false.
|
|
635
666
|
# => Year cannot be lower than 1900.
|
|
636
|
-
# => Year cannot be higher or equal than 2100.
|
|
667
|
+
# => Year cannot be higher or equal than 2100.
|
|
637
668
|
def self.datetime_api_check(s)
|
|
638
|
-
return false if (s.size!=14)
|
|
669
|
+
return false if (s.size!=14)
|
|
639
670
|
year = s[0..3]
|
|
640
671
|
month = s[4..5]
|
|
641
|
-
day = s[6..7]
|
|
642
|
-
hour = s[8..9]
|
|
643
|
-
minute = s[10..11]
|
|
644
|
-
second = s[12..13]
|
|
672
|
+
day = s[6..7]
|
|
673
|
+
hour = s[8..9]
|
|
674
|
+
minute = s[10..11]
|
|
675
|
+
second = s[12..13]
|
|
645
676
|
BlackStack::DateTime::Misc::datetime_values_check(year,month,day,hour,minute,second)
|
|
646
677
|
end # def datetime_api_check
|
|
647
678
|
|
|
648
679
|
# Check the string has the format yyyy-mm-dd hh:mm:ss.
|
|
649
680
|
# => Return true if success. Otherwise, return false.
|
|
650
681
|
# => Year cannot be lower than 1900.
|
|
651
|
-
# => Year cannot be higher or equal than 2100.
|
|
682
|
+
# => Year cannot be higher or equal than 2100.
|
|
652
683
|
def self.datetime_sql_check(s)
|
|
653
684
|
return false if (s.size!=19)
|
|
654
685
|
year = s[0..3]
|
|
655
686
|
month = s[5..6]
|
|
656
|
-
day = s[8..9]
|
|
657
|
-
hour = s[11..12]
|
|
658
|
-
minute = s[14..15]
|
|
659
|
-
second = s[17..18]
|
|
687
|
+
day = s[8..9]
|
|
688
|
+
hour = s[11..12]
|
|
689
|
+
minute = s[14..15]
|
|
690
|
+
second = s[17..18]
|
|
660
691
|
BlackStack::DateTime::Misc::datetime_values_check(year,month,day,hour,minute,second)
|
|
661
692
|
end # def datetime_sql_check
|
|
662
|
-
|
|
693
|
+
|
|
663
694
|
# Convierte un string con formato api-datatime (yyyymmddhhmmss) a un string con formato sql-datetime (yyyy-mm-dd hh:mm:ss).
|
|
664
695
|
def self.datetime_api_to_sql(s)
|
|
665
696
|
raise "Wrong Api DataTime Format." if (datetime_api_check(s)==false)
|
|
666
697
|
year = s[0..3]
|
|
667
698
|
month = s[4..5]
|
|
668
|
-
day = s[6..7]
|
|
669
|
-
hour = s[8..9]
|
|
670
|
-
minute = s[10..11]
|
|
671
|
-
second = s[12..13]
|
|
672
|
-
ret = "#{year}-#{month}-#{day} #{hour}:#{minute}:#{second}"
|
|
699
|
+
day = s[6..7]
|
|
700
|
+
hour = s[8..9]
|
|
701
|
+
minute = s[10..11]
|
|
702
|
+
second = s[12..13]
|
|
703
|
+
ret = "#{year}-#{month}-#{day} #{hour}:#{minute}:#{second}"
|
|
673
704
|
return ret
|
|
674
|
-
end # def datetime_api_to_sql
|
|
675
|
-
|
|
705
|
+
end # def datetime_api_to_sql
|
|
706
|
+
|
|
676
707
|
# Convierte un string con formato sql-datatime a un string con formato sql-datetime.
|
|
677
708
|
def self.datetime_sql_to_api(s)
|
|
678
709
|
raise "Wrong SQL DataTime Format." if (datetime_sql_check(s)==false)
|
|
679
710
|
year = s[0..3]
|
|
680
711
|
month = s[5..6]
|
|
681
|
-
day = s[8..9]
|
|
682
|
-
hour = s[11..12]
|
|
683
|
-
minute = s[14..15]
|
|
684
|
-
second = s[17..18]
|
|
685
|
-
ret = "#{year}#{month}#{day}#{hour}#{minute}#{second}"
|
|
712
|
+
day = s[8..9]
|
|
713
|
+
hour = s[11..12]
|
|
714
|
+
minute = s[14..15]
|
|
715
|
+
second = s[17..18]
|
|
716
|
+
ret = "#{year}#{month}#{day}#{hour}#{minute}#{second}"
|
|
686
717
|
return ret
|
|
687
718
|
end # def datetime_sql_to_api
|
|
688
719
|
end # module DateTime
|
|
689
720
|
|
|
690
721
|
|
|
691
|
-
# -----------------------------------------------------------------------------------------
|
|
722
|
+
# -----------------------------------------------------------------------------------------
|
|
692
723
|
# Spinning
|
|
693
|
-
# -----------------------------------------------------------------------------------------
|
|
724
|
+
# -----------------------------------------------------------------------------------------
|
|
694
725
|
module Spinning
|
|
695
726
|
# Esta funcion retorna una variacion al azar del texto que se pasa.
|
|
696
727
|
# Esta funcion se ocupa de dividir el texto en partes, para eviar el error "too big to product" que arroja la libraría.
|
|
697
728
|
def self.random_spinning_variation(text)
|
|
698
729
|
ret = text
|
|
699
|
-
|
|
730
|
+
|
|
700
731
|
text.scan(MATCH_CONTENT_SPINNING).each { |s|
|
|
701
732
|
a = ContentSpinning.new(s).spin
|
|
702
733
|
rep = a[rand(a.size)]
|
|
703
734
|
ret = ret.gsub(s, rep)
|
|
704
735
|
a = nil
|
|
705
736
|
}
|
|
706
|
-
|
|
737
|
+
|
|
707
738
|
return ret
|
|
708
739
|
end
|
|
709
|
-
|
|
740
|
+
|
|
710
741
|
# retorna true si la sintaxis del texto spineado es correcta
|
|
711
742
|
# caso contrario retorna false
|
|
712
743
|
# no soporta spinnings anidados. ejemplo: {my|our|{a car of mine}}
|
|
@@ -714,16 +745,16 @@ module BlackStack
|
|
|
714
745
|
# valido que exste
|
|
715
746
|
n = 0
|
|
716
747
|
s.split('').each { |c|
|
|
717
|
-
n+=1 if c=='{'
|
|
748
|
+
n+=1 if c=='{'
|
|
718
749
|
n-=1 if c=='}'
|
|
719
750
|
if n!=0 && n!=1
|
|
720
751
|
#raise "Closing spining char '}' with not previous opening spining char '{'." if n<0
|
|
721
752
|
#raise "Opening spining char '{' inside another spining block." if n>1
|
|
722
753
|
return false if n<0 # Closing spining char '}' with not previous opening spining char '{'.
|
|
723
754
|
return false if n>1 # Opening spining char '{' inside another spining block.
|
|
724
|
-
end
|
|
755
|
+
end
|
|
725
756
|
}
|
|
726
|
-
|
|
757
|
+
|
|
727
758
|
return false if n!=0
|
|
728
759
|
=begin
|
|
729
760
|
# obtengo cada uno de los spinnings
|
|
@@ -731,21 +762,21 @@ module BlackStack
|
|
|
731
762
|
a = x.split('|')
|
|
732
763
|
raise "No variations delimited by '|' inside spinning block." if a.size <= 1
|
|
733
764
|
}
|
|
734
|
-
=end
|
|
765
|
+
=end
|
|
735
766
|
true
|
|
736
767
|
end
|
|
737
|
-
|
|
768
|
+
|
|
738
769
|
# returns true if the text is spinned.
|
|
739
770
|
# otherwise, returns false.
|
|
740
771
|
def self.spintax?(s)
|
|
741
772
|
s.scan(MATCH_CONTENT_SPINNING).size > 0
|
|
742
773
|
end
|
|
743
774
|
end # module Spinning
|
|
744
|
-
|
|
745
|
-
|
|
746
|
-
# -----------------------------------------------------------------------------------------
|
|
775
|
+
|
|
776
|
+
|
|
777
|
+
# -----------------------------------------------------------------------------------------
|
|
747
778
|
# Miscelaneus
|
|
748
|
-
# -----------------------------------------------------------------------------------------
|
|
779
|
+
# -----------------------------------------------------------------------------------------
|
|
749
780
|
module Misc
|
|
750
781
|
# make a Ruby string safe for a filesystem.
|
|
751
782
|
# References:
|
|
@@ -756,7 +787,7 @@ module BlackStack
|
|
|
756
787
|
# NOTE: File.basename doesn't work right with Windows paths on Unix
|
|
757
788
|
# get only the filename, not the whole path
|
|
758
789
|
name.gsub!(/^.*(\\|\/)/, '')
|
|
759
|
-
|
|
790
|
+
|
|
760
791
|
# Strip out the non-ascii character
|
|
761
792
|
name.gsub!(/[^0-9A-Za-z.\-]/, '_')
|
|
762
793
|
end
|
|
@@ -765,9 +796,9 @@ module BlackStack
|
|
|
765
796
|
end # module Misc
|
|
766
797
|
|
|
767
798
|
|
|
768
|
-
# -----------------------------------------------------------------------------------------
|
|
799
|
+
# -----------------------------------------------------------------------------------------
|
|
769
800
|
# Email Appending Functions
|
|
770
|
-
# -----------------------------------------------------------------------------------------
|
|
801
|
+
# -----------------------------------------------------------------------------------------
|
|
771
802
|
module Appending
|
|
772
803
|
APPEND_PATTERN_FNAME_DOT_LNAME = 0
|
|
773
804
|
APPEND_PATTERN_FNAME = 1
|
|
@@ -775,7 +806,7 @@ module BlackStack
|
|
|
775
806
|
APPEND_PATTERN_F_LNAME = 3
|
|
776
807
|
APPEND_PATTERN_F_DOT_LNAME = 4
|
|
777
808
|
|
|
778
|
-
#
|
|
809
|
+
#
|
|
779
810
|
def self.name_pattern(pattern, fname, lname)
|
|
780
811
|
if (pattern==APPEND_PATTERN_FNAME_DOT_LNAME)
|
|
781
812
|
return "#{fname}.#{lname}"
|
|
@@ -792,13 +823,13 @@ module BlackStack
|
|
|
792
823
|
end
|
|
793
824
|
end
|
|
794
825
|
|
|
795
|
-
#
|
|
826
|
+
#
|
|
796
827
|
def self.get_email_variations(first_name, last_name, domain, is_a_big_company)
|
|
797
828
|
variations = Array.new
|
|
798
829
|
variations << first_name + "." + last_name + "@" + domain
|
|
799
830
|
variations << first_name[0] + last_name + "@" + domain
|
|
800
831
|
variations << first_name + "_" + last_name + "@" + domain
|
|
801
|
-
variations << first_name[0] + "." + last_name + "@" + domain
|
|
832
|
+
variations << first_name[0] + "." + last_name + "@" + domain
|
|
802
833
|
if (is_a_big_company == false)
|
|
803
834
|
variations << last_name + "@" + domain
|
|
804
835
|
variations << first_name + "@" + domain
|
|
@@ -821,20 +852,24 @@ module BlackStack
|
|
|
821
852
|
end
|
|
822
853
|
end # module Appending
|
|
823
854
|
end # module String
|
|
824
|
-
|
|
825
|
-
# -----------------------------------------------------------------------------------------
|
|
855
|
+
|
|
856
|
+
# -----------------------------------------------------------------------------------------
|
|
826
857
|
# Network
|
|
827
|
-
# -----------------------------------------------------------------------------------------
|
|
858
|
+
# -----------------------------------------------------------------------------------------
|
|
828
859
|
module Netting
|
|
829
860
|
CALL_METHOD_GET = 'get'
|
|
830
861
|
CALL_METHOD_POST = 'post'
|
|
831
862
|
DEFAULT_SSL_VERIFY_MODE = OpenSSL::SSL::VERIFY_NONE
|
|
832
863
|
SUCCESS = 'success'
|
|
833
|
-
|
|
834
|
-
|
|
864
|
+
|
|
865
|
+
DEFAULT_OPEN_TIMEOUT = 10 # seconds to wait for the initial connection
|
|
866
|
+
DEFAULT_READ_TIMEOUT = 30 # seconds to wait for each response
|
|
867
|
+
DEFAULT_RETRY_ATTEMPTS = 3
|
|
868
|
+
|
|
869
|
+
@@lockfiles = []
|
|
835
870
|
|
|
836
871
|
@@max_api_call_channels = 0 # 0 means infinite
|
|
837
|
-
|
|
872
|
+
|
|
838
873
|
def self.max_api_call_channels()
|
|
839
874
|
@@max_api_call_channels
|
|
840
875
|
end
|
|
@@ -846,7 +881,7 @@ module BlackStack
|
|
|
846
881
|
def self.set(h)
|
|
847
882
|
@@max_api_call_channels = h[:max_api_call_channels]
|
|
848
883
|
@@lockfiles = []
|
|
849
|
-
|
|
884
|
+
|
|
850
885
|
i = 0
|
|
851
886
|
while i<@@max_api_call_channels
|
|
852
887
|
@@lockfiles << File.open("./apicall.channel_#{i.to_s}.lock", "w")
|
|
@@ -857,18 +892,18 @@ module BlackStack
|
|
|
857
892
|
|
|
858
893
|
class ApiCallException < StandardError
|
|
859
894
|
attr_accessor :description
|
|
860
|
-
|
|
895
|
+
|
|
861
896
|
def initialize(s)
|
|
862
897
|
self.description = s
|
|
863
898
|
end
|
|
864
|
-
|
|
899
|
+
|
|
865
900
|
def to_s
|
|
866
901
|
self.description
|
|
867
902
|
end
|
|
868
903
|
end
|
|
869
|
-
|
|
904
|
+
|
|
870
905
|
# New call_get
|
|
871
|
-
def self.call_get(url, params = {}, ssl_verify_mode=BlackStack::Netting::DEFAULT_SSL_VERIFY_MODE, support_redirections=true)
|
|
906
|
+
def self.call_get(url, params = {}, ssl_verify_mode=BlackStack::Netting::DEFAULT_SSL_VERIFY_MODE, support_redirections=true)
|
|
872
907
|
uri = URI(url)
|
|
873
908
|
uri.query = URI.encode_www_form(params)
|
|
874
909
|
Net::HTTP.start(uri.host, uri.port, :use_ssl => uri.scheme == 'https', :verify_mode => ssl_verify_mode) do |http|
|
|
@@ -883,7 +918,7 @@ module BlackStack
|
|
|
883
918
|
end
|
|
884
919
|
end
|
|
885
920
|
end
|
|
886
|
-
|
|
921
|
+
|
|
887
922
|
# Call the API and return th result.
|
|
888
923
|
#
|
|
889
924
|
# Unlike `Net::HTTP::Post`, this method support complex json descriptors in order to submit complex data strucutres to access points.
|
|
@@ -891,37 +926,88 @@ module BlackStack
|
|
|
891
926
|
#
|
|
892
927
|
# url: valid internet address
|
|
893
928
|
# body: hash of body to attach in the call
|
|
894
|
-
# ssl_verify_mode: you can disabele SSL verification here.
|
|
929
|
+
# ssl_verify_mode: you can disabele SSL verification here.
|
|
895
930
|
# max_channels: this method use lockfiles to prevent an excesive number of API calls from each datacenter. There is not allowed more simultaneous calls than max_channels.
|
|
896
|
-
#
|
|
931
|
+
#
|
|
897
932
|
# TODO: parameter support_redirections has been deprecated.
|
|
898
933
|
#
|
|
899
|
-
def self.call_post(url,
|
|
900
|
-
|
|
901
|
-
|
|
902
|
-
|
|
903
|
-
|
|
904
|
-
|
|
905
|
-
|
|
906
|
-
|
|
907
|
-
|
|
908
|
-
|
|
909
|
-
|
|
910
|
-
|
|
911
|
-
|
|
912
|
-
|
|
913
|
-
|
|
914
|
-
|
|
915
|
-
|
|
916
|
-
|
|
917
|
-
|
|
918
|
-
|
|
919
|
-
|
|
920
|
-
|
|
921
|
-
|
|
934
|
+
def self.call_post(url,
|
|
935
|
+
body = {},
|
|
936
|
+
ssl_verify_mode = BlackStack::Netting::DEFAULT_SSL_VERIFY_MODE,
|
|
937
|
+
support_redirections = true)
|
|
938
|
+
|
|
939
|
+
max_redirects = 5
|
|
940
|
+
attempts = 0
|
|
941
|
+
|
|
942
|
+
while attempts < DEFAULT_RETRY_ATTEMPTS
|
|
943
|
+
begin
|
|
944
|
+
uri = URI.parse(url)
|
|
945
|
+
|
|
946
|
+
# Create a basic Faraday connection
|
|
947
|
+
conn = Faraday.new(
|
|
948
|
+
url: "#{uri.scheme}://#{uri.host}",
|
|
949
|
+
ssl: { verify: ssl_verify_mode != OpenSSL::SSL::VERIFY_NONE },
|
|
950
|
+
request: {
|
|
951
|
+
open_timeout: DEFAULT_OPEN_TIMEOUT,
|
|
952
|
+
timeout: DEFAULT_READ_TIMEOUT
|
|
953
|
+
}
|
|
954
|
+
) do |f|
|
|
955
|
+
f.adapter Faraday.default_adapter
|
|
956
|
+
end
|
|
957
|
+
|
|
958
|
+
# Perform the initial POST request
|
|
959
|
+
response = conn.post(uri.path) do |req|
|
|
960
|
+
req.body = JSON.generate(body) # Manually serialize to JSON
|
|
961
|
+
req.headers['Content-Type'] = 'application/json'
|
|
962
|
+
req.headers['Accept'] = 'application/json'
|
|
963
|
+
end
|
|
964
|
+
|
|
965
|
+
# Manually handle HTTP redirects if requested
|
|
966
|
+
redirect_count = 0
|
|
967
|
+
while support_redirections &&
|
|
968
|
+
response.status.between?(300, 399) &&
|
|
969
|
+
response.headers['location'] &&
|
|
970
|
+
redirect_count < max_redirects
|
|
971
|
+
|
|
972
|
+
redirect_count += 1
|
|
973
|
+
new_url = URI.join(url, response.headers['location']).to_s
|
|
974
|
+
|
|
975
|
+
# Update `url` for the next iteration
|
|
976
|
+
url = new_url
|
|
977
|
+
uri = URI.parse(new_url)
|
|
978
|
+
|
|
979
|
+
response = conn.post(uri.path) do |req|
|
|
980
|
+
req.body = JSON.generate(body)
|
|
981
|
+
req.headers['Content-Type'] = 'application/json'
|
|
982
|
+
req.headers['Accept'] = 'application/json'
|
|
983
|
+
end
|
|
984
|
+
end
|
|
985
|
+
|
|
986
|
+
# Raise an error if the final response is not 2xx
|
|
987
|
+
unless response.success?
|
|
988
|
+
raise "HTTP #{response.status}: #{response.body}"
|
|
989
|
+
end
|
|
990
|
+
|
|
991
|
+
# If all is good, return the Faraday response object
|
|
992
|
+
return response
|
|
993
|
+
|
|
994
|
+
rescue Errno::ETIMEDOUT, Timeout::Error, Faraday::ConnectionFailed => e
|
|
995
|
+
# Basic retry logic for transient network errors
|
|
996
|
+
attempts += 1
|
|
997
|
+
if attempts < DEFAULT_RETRY_ATTEMPTS
|
|
998
|
+
sleep_time = 2**attempts
|
|
999
|
+
sleep(sleep_time) # Exponential backoff
|
|
1000
|
+
retry
|
|
1001
|
+
else
|
|
1002
|
+
# Too many failures, re-raise
|
|
1003
|
+
raise e
|
|
1004
|
+
end
|
|
1005
|
+
rescue => e
|
|
1006
|
+
# For any other error, just re-raise
|
|
1007
|
+
raise e
|
|
1008
|
+
end
|
|
922
1009
|
end
|
|
923
|
-
|
|
924
|
-
end
|
|
1010
|
+
end # def self.call_post
|
|
925
1011
|
|
|
926
1012
|
# TODO: deprecated
|
|
927
1013
|
def self.api_call(url, params={}, method=BlackStack::Netting::CALL_METHOD_POST, ssl_verify_mode=BlackStack::Netting::DEFAULT_SSL_VERIFY_MODE, max_retries=5)
|
|
@@ -939,7 +1025,7 @@ module BlackStack
|
|
|
939
1025
|
if (parsed['status']==BlackStack::Netting::SUCCESS)
|
|
940
1026
|
bSuccess = true
|
|
941
1027
|
else
|
|
942
|
-
sError = "Status: #{parsed['status'].to_s}. Description: #{parsed['value'].to_s}."
|
|
1028
|
+
sError = "Status: #{parsed['status'].to_s}. Description: #{parsed['value'].to_s}."
|
|
943
1029
|
end
|
|
944
1030
|
rescue Errno::ECONNREFUSED => e
|
|
945
1031
|
sError = "Errno::ECONNREFUSED:" + e.to_console
|
|
@@ -947,7 +1033,7 @@ module BlackStack
|
|
|
947
1033
|
sError = "Exception:" + e2.to_console
|
|
948
1034
|
end
|
|
949
1035
|
end # while
|
|
950
|
-
|
|
1036
|
+
|
|
951
1037
|
if (bSuccess==false)
|
|
952
1038
|
raise "#{sError}"
|
|
953
1039
|
end
|
|
@@ -958,7 +1044,7 @@ module BlackStack
|
|
|
958
1044
|
# to: must be a valid path to a folder.
|
|
959
1045
|
def self.download(url, to)
|
|
960
1046
|
uri = URI(url)
|
|
961
|
-
domain = uri.host.start_with?('www.') ? uri.host[4..-1] : uri.host
|
|
1047
|
+
domain = uri.host.start_with?('www.') ? uri.host[4..-1] : uri.host
|
|
962
1048
|
path = uri.path
|
|
963
1049
|
filename = path.split("/").last
|
|
964
1050
|
Net::HTTP.start(domain) do |http|
|
|
@@ -970,7 +1056,7 @@ module BlackStack
|
|
|
970
1056
|
end
|
|
971
1057
|
|
|
972
1058
|
# Return the extension of the last path into an URL.
|
|
973
|
-
# Example: get_url_extension("http://connect.data.com/sitemap_index.xml?foo_param=foo_value") => ".xml"
|
|
1059
|
+
# Example: get_url_extension("http://connect.data.com/sitemap_index.xml?foo_param=foo_value") => ".xml"
|
|
974
1060
|
def self.get_url_extension(url)
|
|
975
1061
|
return File.extname(URI.parse(url).path.to_s)
|
|
976
1062
|
end
|
|
@@ -1011,12 +1097,12 @@ module BlackStack
|
|
|
1011
1097
|
httpc = HTTPClient.new
|
|
1012
1098
|
resp = httpc.get(url)
|
|
1013
1099
|
res = resp.header['Location']
|
|
1014
|
-
|
|
1100
|
+
|
|
1015
1101
|
if res.size == 0
|
|
1016
1102
|
uri = URI.parse(url)
|
|
1017
1103
|
uri_params = CGI.parse(uri.query)
|
|
1018
|
-
redirected_url = uri_params['url'][0]
|
|
1019
|
-
|
|
1104
|
+
redirected_url = uri_params['url'][0]
|
|
1105
|
+
|
|
1020
1106
|
if ( redirected_url != nil )
|
|
1021
1107
|
res = redirected_url
|
|
1022
1108
|
else
|
|
@@ -1040,25 +1126,25 @@ module BlackStack
|
|
|
1040
1126
|
# => p = CGI::parse(URI.parse(url).query)
|
|
1041
1127
|
# => La linea de abajo hace un gsbub que hace que esta url siga funcionando como busqueda de google, y ademas se posible parsearla.
|
|
1042
1128
|
url = url.gsub("webhp#q=", "webhp?q=")
|
|
1043
|
-
|
|
1129
|
+
|
|
1044
1130
|
return CGI::parse(URI.parse(url).query)
|
|
1045
1131
|
end
|
|
1046
|
-
|
|
1132
|
+
|
|
1047
1133
|
# Add a parameter to the url. It doesn't validate if the param already exists.
|
|
1048
1134
|
def self.add_param(url, param_name, param_value)
|
|
1049
1135
|
uri = URI(url)
|
|
1050
1136
|
params = URI.decode_www_form(uri.query || '')
|
|
1051
|
-
|
|
1137
|
+
|
|
1052
1138
|
if (params.size==0)
|
|
1053
1139
|
params << [param_name, param_value]
|
|
1054
1140
|
uri.query = URI.encode_www_form(params)
|
|
1055
1141
|
return uri.to_s
|
|
1056
1142
|
else
|
|
1057
1143
|
uri.query = URI.encode_www_form(params)
|
|
1058
|
-
return uri.to_s + "&" + param_name + "=" + param_value
|
|
1144
|
+
return uri.to_s + "&" + param_name + "=" + param_value
|
|
1059
1145
|
end
|
|
1060
1146
|
end
|
|
1061
|
-
|
|
1147
|
+
|
|
1062
1148
|
# Changes the value of a parameter in the url. It doesn't validate if the param already exists.
|
|
1063
1149
|
def self.change_param(url, param_name, param_value)
|
|
1064
1150
|
uri = URI(url)
|
|
@@ -1068,10 +1154,10 @@ module BlackStack
|
|
|
1068
1154
|
uri.query = URI.encode_www_form(params)
|
|
1069
1155
|
uri.to_s
|
|
1070
1156
|
end
|
|
1071
|
-
|
|
1157
|
+
|
|
1072
1158
|
# Change or add the value of a parameter in the url, depending if the parameter already exists or not.
|
|
1073
1159
|
def self.set_param(url, param_name, param_value)
|
|
1074
|
-
params = BlackStack::Netting::params(url)
|
|
1160
|
+
params = BlackStack::Netting::params(url)
|
|
1075
1161
|
if ( params.has_key?(param_name) == true )
|
|
1076
1162
|
newurl = BlackStack::Netting::change_param(url, param_name, param_value)
|
|
1077
1163
|
else
|
|
@@ -1079,24 +1165,24 @@ module BlackStack
|
|
|
1079
1165
|
end
|
|
1080
1166
|
return newurl
|
|
1081
1167
|
end
|
|
1082
|
-
|
|
1168
|
+
|
|
1083
1169
|
# get the domain from any url
|
|
1084
1170
|
def self.getDomainFromUrl(url)
|
|
1085
|
-
if (url !~ /^http:\/\//i && url !~ /^https:\/\//i)
|
|
1171
|
+
if (url !~ /^http:\/\//i && url !~ /^https:\/\//i)
|
|
1086
1172
|
url = "http://#{url}"
|
|
1087
1173
|
end
|
|
1088
|
-
|
|
1174
|
+
|
|
1089
1175
|
if (URI.parse(url).host == nil)
|
|
1090
|
-
raise "Cannot get domain for #{url}"
|
|
1176
|
+
raise "Cannot get domain for #{url}"
|
|
1091
1177
|
end
|
|
1092
|
-
|
|
1178
|
+
|
|
1093
1179
|
if (url.to_s.length>0)
|
|
1094
1180
|
return URI.parse(url).host.sub(/^www\./, '')
|
|
1095
1181
|
else
|
|
1096
1182
|
return nil
|
|
1097
1183
|
end
|
|
1098
1184
|
end
|
|
1099
|
-
|
|
1185
|
+
|
|
1100
1186
|
def self.getDomainFromEmail(email)
|
|
1101
1187
|
if email.email?
|
|
1102
1188
|
return email.split("@").last
|
|
@@ -1104,35 +1190,35 @@ module BlackStack
|
|
|
1104
1190
|
raise "getDomainFromEmail: Wrong email format."
|
|
1105
1191
|
end
|
|
1106
1192
|
end
|
|
1107
|
-
|
|
1193
|
+
|
|
1108
1194
|
def self.getWhoisDomains(domain, allow_heuristic_to_avoid_hosting_companies=false)
|
|
1109
1195
|
a = Array.new
|
|
1110
1196
|
c = Whois::Client.new
|
|
1111
1197
|
r = c.lookup(domain)
|
|
1112
|
-
|
|
1198
|
+
|
|
1113
1199
|
res = r.to_s.scan(/Registrant Email: (#{BlackStack::Strings::MATCH_EMAIL})/).first
|
|
1114
1200
|
if (res!=nil)
|
|
1115
1201
|
a << BlackStack::Netting::getDomainFromEmail(res[0].downcase)
|
|
1116
1202
|
end
|
|
1117
|
-
|
|
1203
|
+
|
|
1118
1204
|
res = r.to_s.scan(/Admin Email: (#{BlackStack::Strings::MATCH_EMAIL})/).first
|
|
1119
1205
|
if (res!=nil)
|
|
1120
1206
|
a << BlackStack::Netting::getDomainFromEmail(res[0].downcase)
|
|
1121
1207
|
end
|
|
1122
|
-
|
|
1208
|
+
|
|
1123
1209
|
res = r.to_s.scan(/Tech Email: (#{BlackStack::Strings::MATCH_EMAIL})/).first
|
|
1124
1210
|
if (res!=nil)
|
|
1125
1211
|
a << BlackStack::Netting::getDomainFromEmail(res[0].downcase)
|
|
1126
1212
|
end
|
|
1127
|
-
|
|
1213
|
+
|
|
1128
1214
|
# remover duplicados
|
|
1129
1215
|
a = a.uniq
|
|
1130
|
-
|
|
1131
|
-
#
|
|
1216
|
+
|
|
1217
|
+
#
|
|
1132
1218
|
if (allow_heuristic_to_avoid_hosting_companies==true)
|
|
1133
1219
|
# TODO: develop this feature
|
|
1134
1220
|
end
|
|
1135
|
-
|
|
1221
|
+
|
|
1136
1222
|
return a
|
|
1137
1223
|
end
|
|
1138
1224
|
|
|
@@ -1160,11 +1246,11 @@ module BlackStack
|
|
|
1160
1246
|
raise "Email #{value} is not valid" if !value.email?
|
|
1161
1247
|
# extract the domain from the email
|
|
1162
1248
|
domain = value.split('@').last
|
|
1163
|
-
#
|
|
1249
|
+
#
|
|
1164
1250
|
return domain=~/gmail\.com/ || domain=~/hotmail\.com/ || domain=~/outlook\.com/ || domain=~/yahoo\.com/ || domain=~/comcast\.com/ || domain=~/aol\.com/ || domain=~/msn\.com/ || domain=~/sbcglobal\.net/ ? true : false
|
|
1165
1251
|
end
|
|
1166
1252
|
|
|
1167
1253
|
|
|
1168
1254
|
end # module Netting
|
|
1169
|
-
|
|
1255
|
+
|
|
1170
1256
|
end # module BlackStack
|
metadata
CHANGED
|
@@ -1,14 +1,13 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: blackstack-core
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 1.2.
|
|
4
|
+
version: 1.2.30
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Leandro Daniel Sardi
|
|
8
|
-
autorequire:
|
|
9
8
|
bindir: bin
|
|
10
9
|
cert_chain: []
|
|
11
|
-
date:
|
|
10
|
+
date: 2025-01-30 00:00:00.000000000 Z
|
|
12
11
|
dependencies:
|
|
13
12
|
- !ruby/object:Gem::Dependency
|
|
14
13
|
name: content_spinning
|
|
@@ -86,7 +85,6 @@ homepage: https://rubygems.org/gems/blackstack-core
|
|
|
86
85
|
licenses:
|
|
87
86
|
- MIT
|
|
88
87
|
metadata: {}
|
|
89
|
-
post_install_message:
|
|
90
88
|
rdoc_options: []
|
|
91
89
|
require_paths:
|
|
92
90
|
- lib
|
|
@@ -101,8 +99,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
|
101
99
|
- !ruby/object:Gem::Version
|
|
102
100
|
version: '0'
|
|
103
101
|
requirements: []
|
|
104
|
-
rubygems_version: 3.3
|
|
105
|
-
signing_key:
|
|
102
|
+
rubygems_version: 3.6.3
|
|
106
103
|
specification_version: 4
|
|
107
104
|
summary: Core modules, functions and constants for the BlackStack framework.
|
|
108
105
|
test_files: []
|