curlyq 0.0.5 → 0.0.6
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +17 -0
- data/Gemfile.lock +30 -7
- data/README.md +47 -12
- data/Rakefile +32 -4
- data/bin/curlyq +32 -9
- data/curlyq.gemspec +7 -3
- data/lib/curly/array.rb +24 -0
- data/lib/curly/curl/html.rb +13 -13
- data/lib/curly/hash.rb +33 -3
- data/lib/curly/version.rb +1 -1
- data/src/_README.md +38 -4
- data/test/curlyq_extract_test.rb +43 -0
- data/test/curlyq_headlinks_test.rb +32 -0
- data/test/curlyq_html_test.rb +25 -0
- data/test/curlyq_images_test.rb +27 -0
- data/test/curlyq_json_test.rb +33 -0
- data/test/curlyq_links_test.rb +20 -0
- data/test/curlyq_scrape_test.rb +22 -0
- data/test/curlyq_tags_test.rb +31 -0
- data/test/helpers/curlyq-helpers.rb +29 -0
- data/test/helpers/fake_std_out.rb +30 -0
- data/test/helpers/threaded_tests.rb +182 -0
- data/test/test_helper.rb +7 -2
- metadata +99 -8
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 376f17c2844a60ca0932187b1fe2c4c3f487c0bc63fbb44abe3062036b8c394f
|
4
|
+
data.tar.gz: 9b84f93db1c13dabc20f33b394506619c77272cccb4d2e85f70f6e74cb6238dd
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 909543c28f2192856e0168fdd7da20383892a0c8cde67503209bfbbf69831edd79bdd03884426af0e8a78743d3b7344ce4e70a175c059e4c1356eaf949199d01
|
7
|
+
data.tar.gz: e242883b09cae56ba55df004afe9febba753cdd21d46639b328e0df73322eb18d37187ecdbfc47030ff84c0d46540b4ee7acabffd7aecf4a28facfc4af8e0a24
|
data/CHANGELOG.md
CHANGED
@@ -1,3 +1,20 @@
|
|
1
|
+
### 0.0.6
|
2
|
+
|
3
|
+
2024-01-12 14:44
|
4
|
+
|
5
|
+
#### CHANGED
|
6
|
+
|
7
|
+
- Attributes array is now a hash directly keyed to the attribute key
|
8
|
+
|
9
|
+
#### NEW
|
10
|
+
|
11
|
+
- Tags command has option to output only raw html of matched tags
|
12
|
+
|
13
|
+
#### FIXED
|
14
|
+
|
15
|
+
- --query works with --search on scrape and tags command
|
16
|
+
- Json command dot query works now
|
17
|
+
|
1
18
|
### 0.0.5
|
2
19
|
|
3
20
|
2024-01-11 18:06
|
data/Gemfile.lock
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
PATH
|
2
2
|
remote: .
|
3
3
|
specs:
|
4
|
-
curlyq (0.0.
|
4
|
+
curlyq (0.0.6)
|
5
5
|
gli (~> 2.21.0)
|
6
6
|
nokogiri (~> 1.16.0)
|
7
7
|
selenium-webdriver (~> 4.16.0)
|
@@ -11,19 +11,38 @@ GEM
|
|
11
11
|
remote: https://rubygems.org/
|
12
12
|
specs:
|
13
13
|
gli (2.21.1)
|
14
|
-
minitest (5.16.3)
|
15
14
|
nokogiri (1.16.0-arm64-darwin)
|
16
15
|
racc (~> 1.4)
|
16
|
+
parallel (1.23.0)
|
17
|
+
parallel_tests (3.13.0)
|
18
|
+
parallel
|
19
|
+
pastel (0.8.0)
|
20
|
+
tty-color (~> 0.5)
|
21
|
+
power_assert (2.0.3)
|
17
22
|
racc (1.7.3)
|
18
|
-
rake (
|
19
|
-
rdoc (
|
23
|
+
rake (13.1.0)
|
24
|
+
rdoc (6.3.3)
|
20
25
|
rexml (3.2.6)
|
21
26
|
rubyzip (2.3.2)
|
22
27
|
selenium-webdriver (4.16.0)
|
23
28
|
rexml (~> 3.2, >= 3.2.5)
|
24
29
|
rubyzip (>= 1.2.2, < 3.0)
|
25
30
|
websocket (~> 1.0)
|
31
|
+
strings-ansi (0.2.0)
|
32
|
+
test-unit (3.4.9)
|
33
|
+
power_assert
|
34
|
+
tty-color (0.6.0)
|
35
|
+
tty-cursor (0.7.1)
|
36
|
+
tty-progressbar (0.18.2)
|
37
|
+
strings-ansi (~> 0.2)
|
38
|
+
tty-cursor (~> 0.7)
|
39
|
+
tty-screen (~> 0.8)
|
40
|
+
unicode-display_width (>= 1.6, < 3.0)
|
41
|
+
tty-screen (0.8.2)
|
42
|
+
tty-spinner (0.9.3)
|
43
|
+
tty-cursor (~> 0.7)
|
26
44
|
tty-which (0.5.0)
|
45
|
+
unicode-display_width (2.5.0)
|
27
46
|
websocket (1.2.10)
|
28
47
|
yard (0.9.34)
|
29
48
|
|
@@ -32,9 +51,13 @@ PLATFORMS
|
|
32
51
|
|
33
52
|
DEPENDENCIES
|
34
53
|
curlyq!
|
35
|
-
|
36
|
-
|
37
|
-
|
54
|
+
parallel_tests (~> 3.7, >= 3.7.3)
|
55
|
+
pastel (~> 0.8.0)
|
56
|
+
rake (~> 13.0, >= 13.0.1)
|
57
|
+
rdoc (~> 6.3.1)
|
58
|
+
test-unit (~> 3.4.4)
|
59
|
+
tty-progressbar (~> 0.18, >= 0.18.2)
|
60
|
+
tty-spinner (~> 0.9, >= 0.9.3)
|
38
61
|
yard (~> 0.9, >= 0.9.26)
|
39
62
|
|
40
63
|
BUNDLED WITH
|
data/README.md
CHANGED
@@ -10,7 +10,7 @@ _If you find this useful, feel free to [buy me some coffee][donate]._
|
|
10
10
|
[donate]: https://brettterpstra.com/donate
|
11
11
|
|
12
12
|
|
13
|
-
The current version of `curlyq` is 0.0.
|
13
|
+
The current version of `curlyq` is 0.0.6
|
14
14
|
.
|
15
15
|
|
16
16
|
CurlyQ is a utility that provides a simple interface for curl, with additional features for things like extracting images and links, finding elements by CSS selector or XPath, getting detailed header info, and more. It's designed to be part of a scripting pipeline, outputting everything as structured data (JSON or YAML). It also has rudimentary support for making calls to JSON endpoints easier, but it's expected that you'll use something like `jq` to parse the output.
|
@@ -44,7 +44,7 @@ SYNOPSIS
|
|
44
44
|
curlyq [global options] command [command options] [arguments...]
|
45
45
|
|
46
46
|
VERSION
|
47
|
-
0.0.
|
47
|
+
0.0.6
|
48
48
|
|
49
49
|
GLOBAL OPTIONS
|
50
50
|
--help - Show this message
|
@@ -252,10 +252,9 @@ OpenGraph images will be returned with the structure:
|
|
252
252
|
"title": "CurlyQ, curl better",
|
253
253
|
"attrs": [
|
254
254
|
{
|
255
|
-
"
|
256
|
-
"value": [
|
255
|
+
"class": [
|
257
256
|
"aligncenter"
|
258
|
-
|
257
|
+
], // all attributes included
|
259
258
|
}
|
260
259
|
]
|
261
260
|
}
|
@@ -335,7 +334,22 @@ Returns all the links on the page, which can be queried on any attribute.
|
|
335
334
|
|
336
335
|
Example:
|
337
336
|
|
338
|
-
curlyq
|
337
|
+
curlyq links -q '[content*=twitter]' 'https://stackoverflow.com/questions/52428409/get-fully-rendered-html-using-selenium-webdriver-and-python'
|
338
|
+
|
339
|
+
[
|
340
|
+
{
|
341
|
+
"href": "https://twitter.com/stackoverflow",
|
342
|
+
"title": null,
|
343
|
+
"rel": null,
|
344
|
+
"content": "Twitter",
|
345
|
+
"class": [
|
346
|
+
"-link",
|
347
|
+
"js-gps-track"
|
348
|
+
]
|
349
|
+
}
|
350
|
+
]
|
351
|
+
|
352
|
+
This example gets all links from the page but only returns ones with link content containing 'twitter' (`-q '[content*=twitter]'`).
|
339
353
|
|
340
354
|
```
|
341
355
|
NAME
|
@@ -426,6 +440,26 @@ COMMAND OPTIONS
|
|
426
440
|
|
427
441
|
Return a hierarchy of all tags in a page. Use `-t` to limit to a specific tag.
|
428
442
|
|
443
|
+
curlyq tags --search '#main .post h3' -q 'attrs[id*=what]' https://brettterpstra.com/2024/01/10/introducing-curlyq-a-pipeline-oriented-curl-helper/
|
444
|
+
|
445
|
+
[
|
446
|
+
{
|
447
|
+
"tag": "h3",
|
448
|
+
"source": "<h3 id=\"whats-next\">What???s Next</h3>",
|
449
|
+
"attrs": [
|
450
|
+
{
|
451
|
+
"id": "whats-next"
|
452
|
+
}
|
453
|
+
],
|
454
|
+
"content": "What???s Next",
|
455
|
+
"tags": [
|
456
|
+
|
457
|
+
]
|
458
|
+
}
|
459
|
+
]
|
460
|
+
|
461
|
+
The above command filters the tags based on a CSS query, then further filters them to just tags with an id containing 'what'.
|
462
|
+
|
429
463
|
```
|
430
464
|
NAME
|
431
465
|
tags - Extract all instances of a tag
|
@@ -435,12 +469,13 @@ SYNOPSIS
|
|
435
469
|
curlyq [global options] tags [command options] URL...
|
436
470
|
|
437
471
|
COMMAND OPTIONS
|
438
|
-
-c, --[no-]compressed
|
439
|
-
--[no-]clean
|
440
|
-
-h, --header=
|
441
|
-
-q, --query, --filter=
|
442
|
-
--search=
|
443
|
-
-
|
472
|
+
-c, --[no-]compressed - Expect compressed results
|
473
|
+
--[no-]clean - Remove extra whitespace from results
|
474
|
+
-h, --header=KEY=VAL - Define a header to send as key=value (may be used more than once, default: none)
|
475
|
+
-q, --query, --filter=DOT_SYNTAX - Dot syntax query to filter results (default: none)
|
476
|
+
--search=CSS/XPATH - Regurn an array of matches to a CSS or XPath query (default: none)
|
477
|
+
--[no-]source, --[no-]html - Output the HTML source of the results
|
478
|
+
-t, --tag=TAG - Specify a tag to collect (may be used more than once, default: none)
|
444
479
|
```
|
445
480
|
|
446
481
|
|
data/Rakefile
CHANGED
@@ -1,8 +1,12 @@
|
|
1
1
|
require 'rake/clean'
|
2
|
+
require 'rake/testtask'
|
2
3
|
require 'rubygems'
|
3
4
|
require 'rubygems/package_task'
|
4
5
|
require 'rdoc/task'
|
5
6
|
require 'yard'
|
7
|
+
require 'parallel_tests'
|
8
|
+
require 'parallel_tests/tasks'
|
9
|
+
require 'tty-spinner'
|
6
10
|
|
7
11
|
YARD::Rake::YardocTask.new do |t|
|
8
12
|
t.files = ['lib/curly/*.rb']
|
@@ -22,10 +26,34 @@ spec = eval(File.read('curlyq.gemspec'))
|
|
22
26
|
|
23
27
|
Gem::PackageTask.new(spec) do |pkg|
|
24
28
|
end
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
+
|
30
|
+
namespace :test do
|
31
|
+
FileList['test/*_test.rb'].each do |rakefile|
|
32
|
+
test_name = File.basename(rakefile, '.rb').sub(/^.*?_(.*?)_.*?$/, '\1')
|
33
|
+
|
34
|
+
Rake::TestTask.new(:"#{test_name}") do |t|
|
35
|
+
t.libs << ['test', 'test/helpers']
|
36
|
+
t.pattern = rakefile
|
37
|
+
t.verbose = ENV['VERBOSE'] =~ /(true|1)/i ? true : false
|
38
|
+
end
|
39
|
+
# Define default task for :test
|
40
|
+
task default: test_name
|
41
|
+
end
|
42
|
+
end
|
43
|
+
|
44
|
+
desc 'Run one test verbosely'
|
45
|
+
task :test_one, :test do |_, args|
|
46
|
+
args.with_defaults(test: '*')
|
47
|
+
puts `bundle exec rake test TESTOPTS="-v" TEST="test/curlyq_#{args[:test]}_test.rb"`
|
48
|
+
end
|
49
|
+
|
50
|
+
desc 'Run all tests, threaded'
|
51
|
+
task :test, :pattern, :threads, :max_tests do |_, args|
|
52
|
+
args.with_defaults(pattern: '*', threads: 8, max_tests: 0)
|
53
|
+
pattern = args[:pattern] =~ /(n[iu]ll?|0|\.)/i ? '*' : args[:pattern]
|
54
|
+
|
55
|
+
require_relative 'test/helpers/threaded_tests'
|
56
|
+
ThreadedTests.new.run(pattern: pattern, max_threads: args[:threads].to_i, max_tests: args[:max_tests])
|
29
57
|
end
|
30
58
|
|
31
59
|
desc 'Development version check'
|
data/bin/curlyq
CHANGED
@@ -222,12 +222,26 @@ command :json do |c|
|
|
222
222
|
headers: res.headers
|
223
223
|
})
|
224
224
|
else
|
225
|
-
|
225
|
+
if options[:query]
|
226
|
+
if options[:query] =~ /^json$/
|
227
|
+
res = json
|
228
|
+
elsif options[:query] =~ /^json\./
|
229
|
+
query = options[:query].sub(/^json\./, '')
|
230
|
+
else
|
231
|
+
query = options[:query]
|
232
|
+
end
|
233
|
+
|
234
|
+
res = json.dot_query(query)
|
235
|
+
else
|
236
|
+
res = res.to_data
|
237
|
+
end
|
226
238
|
|
227
|
-
output.push(
|
239
|
+
output.push(res)
|
228
240
|
end
|
229
241
|
end
|
230
242
|
|
243
|
+
output = output[0] if output.count == 1
|
244
|
+
|
231
245
|
print_out(output, global_options[:yaml], pretty: global_options[:pretty])
|
232
246
|
end
|
233
247
|
end
|
@@ -290,10 +304,10 @@ desc 'Extract all instances of a tag'
|
|
290
304
|
arg_name 'URL', multiple: true
|
291
305
|
command :tags do |c|
|
292
306
|
c.desc 'Define a header to send as key=value'
|
293
|
-
c.flag %i[h header], multiple: true
|
307
|
+
c.flag %i[h header], multiple: true, arg_name: 'KEY=VAL'
|
294
308
|
|
295
309
|
c.desc 'Specify a tag to collect'
|
296
|
-
c.flag %i[t tag], multiple: true
|
310
|
+
c.flag %i[t tag], multiple: true, arg_name: 'TAG'
|
297
311
|
|
298
312
|
c.desc 'Expect compressed results'
|
299
313
|
c.switch %i[c compressed]
|
@@ -301,11 +315,14 @@ command :tags do |c|
|
|
301
315
|
c.desc 'Remove extra whitespace from results'
|
302
316
|
c.switch %i[clean]
|
303
317
|
|
304
|
-
c.desc '
|
305
|
-
c.
|
318
|
+
c.desc 'Output the HTML source of the results'
|
319
|
+
c.switch %i[source html]
|
320
|
+
|
321
|
+
c.desc 'Dot syntax query to filter results'
|
322
|
+
c.flag %i[q query filter], arg_name: 'DOT_SYNTAX'
|
306
323
|
|
307
324
|
c.desc 'Regurn an array of matches to a CSS or XPath query'
|
308
|
-
c.flag %i[search]
|
325
|
+
c.flag %i[search], arg_name: 'CSS/XPATH'
|
309
326
|
|
310
327
|
c.action do |global_options, options, args|
|
311
328
|
urls = args.join(' ').split(/[, ]+/)
|
@@ -322,7 +339,7 @@ command :tags do |c|
|
|
322
339
|
if options[:search]
|
323
340
|
out = res.search(options[:search])
|
324
341
|
|
325
|
-
|
342
|
+
out = out.dot_query(options[:query]) if options[:query]
|
326
343
|
output.push(out)
|
327
344
|
elsif options[:query]
|
328
345
|
query = options[:query] =~ /^links/ ? options[:query] : "links#{options[:query]}"
|
@@ -335,7 +352,13 @@ command :tags do |c|
|
|
335
352
|
end
|
336
353
|
end
|
337
354
|
|
338
|
-
|
355
|
+
output = output[0] if output.count == 1
|
356
|
+
|
357
|
+
if options[:source]
|
358
|
+
puts output.to_html
|
359
|
+
else
|
360
|
+
print_out(output, global_options[:yaml], pretty: global_options[:pretty])
|
361
|
+
end
|
339
362
|
end
|
340
363
|
end
|
341
364
|
|
data/curlyq.gemspec
CHANGED
@@ -16,10 +16,14 @@ spec = Gem::Specification.new do |s|
|
|
16
16
|
s.rdoc_options << '--title' << 'curlyq' << '--main' << 'README.rdoc' << '-ri'
|
17
17
|
s.bindir = 'bin'
|
18
18
|
s.executables << 'curlyq'
|
19
|
-
s.add_development_dependency('rake','~> 0.
|
20
|
-
s.add_development_dependency('rdoc', '~>
|
21
|
-
s.add_development_dependency('
|
19
|
+
s.add_development_dependency('rake','~> 13.0', '>= 13.0.1')
|
20
|
+
s.add_development_dependency('rdoc', '~> 6.3.1')
|
21
|
+
s.add_development_dependency('test-unit', '~> 3.4.4')
|
22
22
|
s.add_development_dependency('yard', '~> 0.9', '>= 0.9.26')
|
23
|
+
s.add_development_dependency('tty-spinner', '~> 0.9', '>= 0.9.3')
|
24
|
+
s.add_development_dependency('tty-progressbar', '~> 0.18', '>= 0.18.2')
|
25
|
+
s.add_development_dependency('pastel', '~> 0.8.0')
|
26
|
+
s.add_development_dependency('parallel_tests', '~> 3.7', '>= 3.7.3')
|
23
27
|
s.add_runtime_dependency('gli','~> 2.21.0')
|
24
28
|
s.add_runtime_dependency('tty-which','~> 0.5.0')
|
25
29
|
s.add_runtime_dependency('nokogiri','~> 1.16.0')
|
data/lib/curly/array.rb
CHANGED
@@ -66,6 +66,30 @@ class ::Array
|
|
66
66
|
replace dedup_links
|
67
67
|
end
|
68
68
|
|
69
|
+
#---------------------------------------------------------
|
70
|
+
## Run a query on array elements
|
71
|
+
##
|
72
|
+
## @param path [String] dot.syntax path to compare
|
73
|
+
##
|
74
|
+
## @return [Array] elements matching dot query
|
75
|
+
##
|
76
|
+
def dot_query(path)
|
77
|
+
filter! do |tag|
|
78
|
+
r = tag.dot_query(path)
|
79
|
+
if r.is_a?(Array)
|
80
|
+
r.count.positive?
|
81
|
+
else
|
82
|
+
r
|
83
|
+
end
|
84
|
+
end
|
85
|
+
|
86
|
+
return self
|
87
|
+
end
|
88
|
+
|
89
|
+
def to_html
|
90
|
+
map { |el| el.to_html }
|
91
|
+
end
|
92
|
+
|
69
93
|
##
|
70
94
|
## Test if a tag contains an attribute matching filter queries
|
71
95
|
##
|
data/lib/curly/curl/html.rb
CHANGED
@@ -243,11 +243,11 @@ module Curl
|
|
243
243
|
when /source/
|
244
244
|
next unless %i[all srcset].include?(type)
|
245
245
|
|
246
|
-
srcsets = img[:attrs].filter { |k| k
|
246
|
+
srcsets = img[:attrs].filter { |k| k == 'srcset' }
|
247
247
|
if srcsets.count.positive?
|
248
248
|
srcset = []
|
249
|
-
srcsets.each do |
|
250
|
-
|
249
|
+
srcsets.each do |k, v|
|
250
|
+
v.split(/ *, */).each do |s|
|
251
251
|
image, media = s.split(/ /)
|
252
252
|
srcset << {
|
253
253
|
src: image,
|
@@ -263,15 +263,14 @@ module Curl
|
|
263
263
|
end
|
264
264
|
when /img/
|
265
265
|
next unless %i[all img].include?(type)
|
266
|
-
|
267
|
-
|
268
|
-
|
269
|
-
|
270
|
-
title = img[:attrs].select { |a| a[:key] == 'title' }.first[:value]
|
266
|
+
width = img[:attrs]['width']
|
267
|
+
height = img[:attrs]['height']
|
268
|
+
alt = img[:attrs]['alt']
|
269
|
+
title = img[:attrs]['title']
|
271
270
|
|
272
271
|
output << {
|
273
272
|
type: 'img',
|
274
|
-
src: img[:attrs]
|
273
|
+
src: img[:attrs]['src'],
|
275
274
|
width: width || 'unknown',
|
276
275
|
height: height || 'unknown',
|
277
276
|
alt: alt,
|
@@ -324,8 +323,9 @@ module Curl
|
|
324
323
|
## @param el [Nokogiri] element to convert
|
325
324
|
##
|
326
325
|
def nokogiri_to_tag(el)
|
327
|
-
attributes =
|
328
|
-
|
326
|
+
attributes = {}
|
327
|
+
attributes = el.attribute_nodes.each_with_object({}) do |a, hsh|
|
328
|
+
hsh[a.name] = a.name =~ /^(class|rel)$/ ? a.value.split(/ /) : a.value
|
329
329
|
end
|
330
330
|
|
331
331
|
{
|
@@ -405,12 +405,12 @@ module Curl
|
|
405
405
|
attrs = tag['attrs'].strip.to_enum(:scan, /(?ix)
|
406
406
|
(?<key>[@a-z0-9-]+)(?:=(?<quot>["'])
|
407
407
|
(?<value>[^"']+)\k<quot>|[ >])?/i).map { Regexp.last_match }
|
408
|
-
attrs.
|
408
|
+
attributes = attrs.each_with_object({}) { |a, hsh| hsh[a['key']] = a['key'] =~ /^(class|rel)$/ ? a['value'].split(/ /) : a['value'] }
|
409
409
|
end
|
410
410
|
{
|
411
411
|
tag: tag['tag'],
|
412
412
|
source: tag.to_s,
|
413
|
-
attrs:
|
413
|
+
attrs: attributes,
|
414
414
|
content: @clean ? tag['content']&.clean : tag['content'],
|
415
415
|
tags: content_tags(tag['content'])
|
416
416
|
}
|
data/lib/curly/hash.rb
CHANGED
@@ -23,6 +23,12 @@ class ::Hash
|
|
23
23
|
end
|
24
24
|
end
|
25
25
|
|
26
|
+
def to_html
|
27
|
+
if key?(:source)
|
28
|
+
self[:source]
|
29
|
+
end
|
30
|
+
end
|
31
|
+
|
26
32
|
# Extract data using a dot-syntax path
|
27
33
|
#
|
28
34
|
# @param path [String] The path
|
@@ -31,6 +37,7 @@ class ::Hash
|
|
31
37
|
#
|
32
38
|
def dot_query(path)
|
33
39
|
res = stringify_keys
|
40
|
+
|
34
41
|
out = []
|
35
42
|
q = path.split(/(?<![\d.])\./)
|
36
43
|
q.each do |pth|
|
@@ -54,7 +61,10 @@ class ::Hash
|
|
54
61
|
ats.push(at) unless at.empty?
|
55
62
|
pth.sub!(/\[\]/, '')
|
56
63
|
|
64
|
+
res = res[0] if res.is_a?(Array)
|
65
|
+
|
57
66
|
return false if el.nil? && ats.empty? && !res.key?(pth)
|
67
|
+
|
58
68
|
res = res[pth] unless pth.empty?
|
59
69
|
|
60
70
|
return false if res.nil?
|
@@ -62,10 +72,11 @@ class ::Hash
|
|
62
72
|
if ats.count.positive?
|
63
73
|
while ats.count.positive?
|
64
74
|
atr = ats.shift
|
65
|
-
|
75
|
+
res = [res] if res.is_a?(Hash)
|
66
76
|
keepers = res.filter do |r|
|
67
77
|
evaluate_comp(r, atr)
|
68
78
|
end
|
79
|
+
|
69
80
|
out.concat(keepers)
|
70
81
|
end
|
71
82
|
else
|
@@ -74,6 +85,7 @@ class ::Hash
|
|
74
85
|
|
75
86
|
out = out[eval(el)] if out.is_a?(Array) && el =~ /^[\d.,]+$/
|
76
87
|
end
|
88
|
+
|
77
89
|
out
|
78
90
|
end
|
79
91
|
|
@@ -89,6 +101,8 @@ class ::Hash
|
|
89
101
|
def evaluate_comp(r, atr)
|
90
102
|
keep = true
|
91
103
|
|
104
|
+
r = r.symbolize_keys
|
105
|
+
|
92
106
|
atr.each do |a|
|
93
107
|
key = a[0].to_sym
|
94
108
|
val = if a[2] =~ /^\d+$/
|
@@ -206,10 +220,26 @@ class ::Hash
|
|
206
220
|
end
|
207
221
|
end
|
208
222
|
|
209
|
-
# Turn all keys into
|
223
|
+
# Turn all keys into symbols
|
224
|
+
#
|
225
|
+
# If the hash has both a string and a symbol for key,
|
226
|
+
# keep the symbol value, discarding the string value
|
227
|
+
#
|
228
|
+
# @return [Hash] a copy of the hash where all its
|
229
|
+
# keys are strings
|
230
|
+
#
|
231
|
+
def symbolize_keys
|
232
|
+
each_with_object({}) do |(k, v), hsh|
|
233
|
+
next if k.is_a?(String) && key?(k.to_sym)
|
234
|
+
|
235
|
+
hsh[k.to_sym] = v.is_a?(Hash) ? v.symbolize_keys : v
|
236
|
+
end
|
237
|
+
end
|
238
|
+
|
239
|
+
# Turn all keys into strings
|
210
240
|
#
|
211
241
|
# If the hash has both a string and a symbol for key,
|
212
|
-
# keep the string value, discarding the
|
242
|
+
# keep the string value, discarding the symbol value
|
213
243
|
#
|
214
244
|
# @return [Hash] a copy of the hash where all its
|
215
245
|
# keys are strings
|
data/lib/curly/version.rb
CHANGED
data/src/_README.md
CHANGED
@@ -184,10 +184,9 @@ OpenGraph images will be returned with the structure:
|
|
184
184
|
"title": "CurlyQ, curl better",
|
185
185
|
"attrs": [
|
186
186
|
{
|
187
|
-
"
|
188
|
-
"value": [
|
187
|
+
"class": [
|
189
188
|
"aligncenter"
|
190
|
-
|
189
|
+
], // all attributes included
|
191
190
|
}
|
192
191
|
]
|
193
192
|
}
|
@@ -245,7 +244,22 @@ Returns all the links on the page, which can be queried on any attribute.
|
|
245
244
|
|
246
245
|
Example:
|
247
246
|
|
248
|
-
curlyq
|
247
|
+
curlyq links -q '[content*=twitter]' 'https://stackoverflow.com/questions/52428409/get-fully-rendered-html-using-selenium-webdriver-and-python'
|
248
|
+
|
249
|
+
[
|
250
|
+
{
|
251
|
+
"href": "https://twitter.com/stackoverflow",
|
252
|
+
"title": null,
|
253
|
+
"rel": null,
|
254
|
+
"content": "Twitter",
|
255
|
+
"class": [
|
256
|
+
"-link",
|
257
|
+
"js-gps-track"
|
258
|
+
]
|
259
|
+
}
|
260
|
+
]
|
261
|
+
|
262
|
+
This example gets all links from the page but only returns ones with link content containing 'twitter' (`-q '[content*=twitter]'`).
|
249
263
|
|
250
264
|
```
|
251
265
|
@cli(bundle exec bin/curlyq help links)
|
@@ -300,6 +314,26 @@ Example:
|
|
300
314
|
|
301
315
|
Return a hierarchy of all tags in a page. Use `-t` to limit to a specific tag.
|
302
316
|
|
317
|
+
curlyq tags --search '#main .post h3' -q 'attrs[id*=what]' https://brettterpstra.com/2024/01/10/introducing-curlyq-a-pipeline-oriented-curl-helper/
|
318
|
+
|
319
|
+
[
|
320
|
+
{
|
321
|
+
"tag": "h3",
|
322
|
+
"source": "<h3 id=\"whats-next\">What’s Next</h3>",
|
323
|
+
"attrs": [
|
324
|
+
{
|
325
|
+
"id": "whats-next"
|
326
|
+
}
|
327
|
+
],
|
328
|
+
"content": "What’s Next",
|
329
|
+
"tags": [
|
330
|
+
|
331
|
+
]
|
332
|
+
}
|
333
|
+
]
|
334
|
+
|
335
|
+
The above command filters the tags based on a CSS query, then further filters them to just tags with an id containing 'what'.
|
336
|
+
|
303
337
|
```
|
304
338
|
@cli(bundle exec bin/curlyq help tags)
|
305
339
|
```
|
@@ -0,0 +1,43 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'json'
|
4
|
+
require 'yaml'
|
5
|
+
|
6
|
+
require 'helpers/curlyq-helpers'
|
7
|
+
require 'test_helper'
|
8
|
+
|
9
|
+
# Tests for tags command
|
10
|
+
class CurlyQTagsTest < Test::Unit::TestCase
|
11
|
+
include CurlyQHelpers
|
12
|
+
|
13
|
+
def setup
|
14
|
+
end
|
15
|
+
|
16
|
+
def test_extract_inclusive
|
17
|
+
result = curlyq('extract', '-i', '-b', 'Adding', '-a', 'accessing the source.', 'https://stackoverflow.com/questions/52428409/get-fully-rendered-html-using-selenium-webdriver-and-python')
|
18
|
+
json = JSON.parse(result)
|
19
|
+
|
20
|
+
assert_match(/^Adding <code>time.sleep\(10\)<\/code>.*?accessing the source.$/, json[0], 'Match should be found and include the before and after strings')
|
21
|
+
end
|
22
|
+
|
23
|
+
def test_extract_exclusive
|
24
|
+
result = curlyq('extract', '-b', 'Adding', '-a', 'accessing the source.', 'https://stackoverflow.com/questions/52428409/get-fully-rendered-html-using-selenium-webdriver-and-python')
|
25
|
+
json = JSON.parse(result)
|
26
|
+
|
27
|
+
assert_match(/^ <code>time.sleep\(10\)<\/code>.*?when I was $/, json[0], 'Match should be found and not include the before and after strings')
|
28
|
+
end
|
29
|
+
|
30
|
+
def test_extract_regex_inclusive
|
31
|
+
result = curlyq('extract', '-ri', '-b', '.dding <', '-a', 'accessing.*?source.', 'https://stackoverflow.com/questions/52428409/get-fully-rendered-html-using-selenium-webdriver-and-python')
|
32
|
+
json = JSON.parse(result)
|
33
|
+
|
34
|
+
assert_match(/^Adding <code>time.sleep\(10\)<\/code>.*?accessing the source.$/, json[0], 'Match should be found and include the before and after strings')
|
35
|
+
end
|
36
|
+
|
37
|
+
def test_extract_regex_exclusive
|
38
|
+
result = curlyq('extract', '-r', '-b', '.dding <', '-a', 'accessing.*?source.', 'https://stackoverflow.com/questions/52428409/get-fully-rendered-html-using-selenium-webdriver-and-python')
|
39
|
+
json = JSON.parse(result)
|
40
|
+
|
41
|
+
assert_match(/^code>time.sleep\(10\)<\/code>.*?when I was $/, json[0], 'Match should be found and not include the before and after strings')
|
42
|
+
end
|
43
|
+
end
|
@@ -0,0 +1,32 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'json'
|
4
|
+
require 'yaml'
|
5
|
+
|
6
|
+
require 'helpers/curlyq-helpers'
|
7
|
+
require 'test_helper'
|
8
|
+
|
9
|
+
# Tests for tags command
|
10
|
+
class CurlyQHeadlinksTest < Test::Unit::TestCase
|
11
|
+
include CurlyQHelpers
|
12
|
+
|
13
|
+
def setup
|
14
|
+
end
|
15
|
+
|
16
|
+
def test_headlinks_query
|
17
|
+
result = curlyq('headlinks', '-q', '[rel=stylesheet]', 'https://brettterpstra.com')
|
18
|
+
json = JSON.parse(result)
|
19
|
+
|
20
|
+
assert_match(/stylesheet/, json['rel'], 'Should have retrieved a single result with rel stylesheet')
|
21
|
+
assert_match(/screen\.\d+\.css$/, json['href'], 'Stylesheet should be correct primary stylesheet')
|
22
|
+
end
|
23
|
+
|
24
|
+
def test_headlinks
|
25
|
+
result = curlyq('headlinks', 'https://brettterpstra.com')
|
26
|
+
json = JSON.parse(result)
|
27
|
+
|
28
|
+
assert_equal(Array, json.class, 'Should have an array of results')
|
29
|
+
assert(json.count > 1, 'Should have more than one link')
|
30
|
+
# assert(json[0].count.positive?)
|
31
|
+
end
|
32
|
+
end
|
@@ -0,0 +1,25 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'json'
|
4
|
+
require 'yaml'
|
5
|
+
|
6
|
+
require 'helpers/curlyq-helpers'
|
7
|
+
require 'test_helper'
|
8
|
+
|
9
|
+
# Tests for tags command
|
10
|
+
class CurlyQHtmlTest < Test::Unit::TestCase
|
11
|
+
include CurlyQHelpers
|
12
|
+
|
13
|
+
def test_html_search_query
|
14
|
+
result = curlyq('html', '-s', '#main article .aligncenter', '-q', 'images[1]', 'https://brettterpstra.com')
|
15
|
+
json = JSON.parse(result)
|
16
|
+
|
17
|
+
assert_match(/aligncenter/, json[0]['class'], 'Should have found an image with class "aligncenter"')
|
18
|
+
end
|
19
|
+
|
20
|
+
def test_html_query
|
21
|
+
result = curlyq('html', '-q', 'meta.title', 'https://brettterpstra.com/2024/01/10/introducing-curlyq-a-pipeline-oriented-curl-helper/')
|
22
|
+
|
23
|
+
assert_match(/Introducing CurlyQ/, result, 'Should have retrived the page title')
|
24
|
+
end
|
25
|
+
end
|
@@ -0,0 +1,27 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'json'
|
4
|
+
require 'yaml'
|
5
|
+
|
6
|
+
require 'helpers/curlyq-helpers'
|
7
|
+
require 'test_helper'
|
8
|
+
|
9
|
+
# Tests for tags command
|
10
|
+
class CurlyQImagesTest < Test::Unit::TestCase
|
11
|
+
include CurlyQHelpers
|
12
|
+
|
13
|
+
def test_images_query
|
14
|
+
result = curlyq('images', '-t', 'img', '-q', '[alt$=screenshot]', 'https://brettterpstra.com/2024/01/08/keyboard-maestro-giveaway/')
|
15
|
+
json = JSON.parse(result)
|
16
|
+
|
17
|
+
assert(json.count == 1, 'Should have found 1 image')
|
18
|
+
assert_match(/Keyboard Maestro screenshot/, json[0]['alt'], 'Should match Keyboard Meastro screenshot')
|
19
|
+
end
|
20
|
+
|
21
|
+
def test_images_type
|
22
|
+
result = curlyq('images', '-t', 'srcset', 'https://brettterpstra.com/')
|
23
|
+
json = JSON.parse(result)
|
24
|
+
|
25
|
+
assert(json.count.positive?, 'Should have found at least 1 image')
|
26
|
+
end
|
27
|
+
end
|
@@ -0,0 +1,33 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'json'
|
4
|
+
require 'yaml'
|
5
|
+
|
6
|
+
require 'helpers/curlyq-helpers'
|
7
|
+
require 'test_helper'
|
8
|
+
|
9
|
+
# Tests for tags command
|
10
|
+
class CurlyQJsonTest < Test::Unit::TestCase
|
11
|
+
include CurlyQHelpers
|
12
|
+
|
13
|
+
def setup
|
14
|
+
end
|
15
|
+
|
16
|
+
def test_json
|
17
|
+
result = curlyq('json', 'https://brettterpstra.com/scripts/giveaways_wrapper.cgi?v=203495&giveaway=hazel2023&action=count')
|
18
|
+
json = JSON.parse(result)
|
19
|
+
|
20
|
+
assert_equal(json.class, Hash, 'Single result should be a hash')
|
21
|
+
assert_equal(286, json['json']['total'], 'json.total should match 286')
|
22
|
+
end
|
23
|
+
|
24
|
+
def test_query
|
25
|
+
result1 = curlyq('json', '-q', 'total', 'https://brettterpstra.com/scripts/giveaways_wrapper.cgi?v=203495&giveaway=hazel2023&action=count')
|
26
|
+
result2 = curlyq('json', '-q', 'json.total', 'https://brettterpstra.com/scripts/giveaways_wrapper.cgi?v=203495&giveaway=hazel2023&action=count')
|
27
|
+
json1 = JSON.parse(result1)
|
28
|
+
json2 = JSON.parse(result2)
|
29
|
+
|
30
|
+
assert_equal(286, json1, 'Should be 286')
|
31
|
+
assert_equal(286, json2, 'Including json in dot path should yeild same result')
|
32
|
+
end
|
33
|
+
end
|
@@ -0,0 +1,20 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'json'
|
4
|
+
require 'yaml'
|
5
|
+
|
6
|
+
require 'helpers/curlyq-helpers'
|
7
|
+
require 'test_helper'
|
8
|
+
|
9
|
+
# Tests for tags command
|
10
|
+
class CurlyQLinksTest < Test::Unit::TestCase
|
11
|
+
include CurlyQHelpers
|
12
|
+
|
13
|
+
def test_links
|
14
|
+
result = curlyq('links', '-q', '[content*=twitter]', 'https://stackoverflow.com/questions/52428409/get-fully-rendered-html-using-selenium-webdriver-and-python')
|
15
|
+
json = JSON.parse(result)
|
16
|
+
|
17
|
+
assert(json.count.positive?, 'Should be at least 1 match')
|
18
|
+
assert_match(/twitter.com/, json[0]['href'], 'Should be a link to Twitter')
|
19
|
+
end
|
20
|
+
end
|
@@ -0,0 +1,22 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'json'
|
4
|
+
require 'yaml'
|
5
|
+
|
6
|
+
require 'helpers/curlyq-helpers'
|
7
|
+
require 'test_helper'
|
8
|
+
|
9
|
+
# Tests for tags command
|
10
|
+
class CurlyQScrapeTest < Test::Unit::TestCase
|
11
|
+
include CurlyQHelpers
|
12
|
+
|
13
|
+
def setup
|
14
|
+
end
|
15
|
+
|
16
|
+
def test_scrape
|
17
|
+
result = curlyq('scrape', '-b', 'firefox', '-q', 'links[rel=me&content*=mastodon][0]', 'https://brettterpstra.com/2024/01/10/introducing-curlyq-a-pipeline-oriented-curl-helper/')
|
18
|
+
json = JSON.parse(result)
|
19
|
+
|
20
|
+
assert_match(/Mastodon/, json['content'], 'Should have retrieved a Mastodon link')
|
21
|
+
end
|
22
|
+
end
|
@@ -0,0 +1,31 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'json'
|
4
|
+
require 'yaml'
|
5
|
+
|
6
|
+
require 'helpers/curlyq-helpers'
|
7
|
+
require 'test_helper'
|
8
|
+
|
9
|
+
# Tests for tags command
|
10
|
+
class CurlyQTagsTest < Test::Unit::TestCase
|
11
|
+
include CurlyQHelpers
|
12
|
+
|
13
|
+
def setup
|
14
|
+
end
|
15
|
+
|
16
|
+
def test_tags
|
17
|
+
result = curlyq('tags', '--search', '#main .post h3', '-q', 'attrs[id*=what]', 'https://brettterpstra.com/2024/01/10/introducing-curlyq-a-pipeline-oriented-curl-helper/')
|
18
|
+
json = JSON.parse(result)
|
19
|
+
|
20
|
+
assert_equal(json.count, 1, 'Should have 1 result')
|
21
|
+
assert_match(/whats-next/, json[0]['attrs']['id'], 'Should have matched #whats-next')
|
22
|
+
end
|
23
|
+
|
24
|
+
def test_clean
|
25
|
+
result = curlyq('tags', '--search', '#main section.related', '--clean', 'https://brettterpstra.com/2024/01/10/introducing-curlyq-a-pipeline-oriented-curl-helper/')
|
26
|
+
json = JSON.parse(result)
|
27
|
+
|
28
|
+
assert_equal(json.count, 1, 'Should have 1 result')
|
29
|
+
assert_match(%r{Last.fm</h5></a></li>}, json[0]['source'], 'Should have matched #whats-next')
|
30
|
+
end
|
31
|
+
end
|
@@ -0,0 +1,29 @@
|
|
1
|
+
require 'open3'
|
2
|
+
require 'time'
|
3
|
+
$LOAD_PATH.unshift File.join(__dir__, '..', '..', 'lib')
|
4
|
+
require 'curly'
|
5
|
+
|
6
|
+
module CurlyQHelpers
|
7
|
+
CURLYQ_EXEC = File.join(File.dirname(__FILE__), '..', '..', 'bin', 'curlyq')
|
8
|
+
BUNDLE = '/Users/ttscoff/.asdf/shims/bundle'
|
9
|
+
|
10
|
+
def curlyq_with_env(env, *args, stdin: nil)
|
11
|
+
Dir.chdir(File.expand_path('~/Desktop/Code/curlyq'))
|
12
|
+
pread(env, BUNDLE, 'exec', 'bin/curlyq', *args, stdin: stdin)
|
13
|
+
end
|
14
|
+
|
15
|
+
def curlyq(*args)
|
16
|
+
curlyq_with_env({ 'GLI_DEBUG' => 'true' }, *args)
|
17
|
+
end
|
18
|
+
|
19
|
+
def pread(env, *cmd, stdin: nil)
|
20
|
+
out, err, status = Open3.capture3(env, *cmd, stdin_data: stdin)
|
21
|
+
unless status.success?
|
22
|
+
raise [
|
23
|
+
"Error (#{status}): #{cmd.inspect} failed", "STDOUT:", out.inspect, "STDERR:", err.inspect
|
24
|
+
].join("\n")
|
25
|
+
end
|
26
|
+
|
27
|
+
out
|
28
|
+
end
|
29
|
+
end
|
@@ -0,0 +1,30 @@
|
|
1
|
+
class FakeStdOut
|
2
|
+
attr_reader :strings
|
3
|
+
|
4
|
+
def initialize
|
5
|
+
@strings = []
|
6
|
+
end
|
7
|
+
|
8
|
+
def puts(string=nil)
|
9
|
+
@strings << string unless string.nil?
|
10
|
+
end
|
11
|
+
|
12
|
+
def write(x)
|
13
|
+
puts(x)
|
14
|
+
end
|
15
|
+
|
16
|
+
def printf(*args)
|
17
|
+
puts(Kernel.printf(*args))
|
18
|
+
end
|
19
|
+
|
20
|
+
# Returns true if the regexp matches anything in the output
|
21
|
+
def contained?(regexp)
|
22
|
+
strings.find{ |x| x =~ regexp }
|
23
|
+
end
|
24
|
+
|
25
|
+
def flush; end
|
26
|
+
|
27
|
+
def to_s
|
28
|
+
@strings.join("\n")
|
29
|
+
end
|
30
|
+
end
|
@@ -0,0 +1,182 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require 'tty-spinner'
|
4
|
+
require 'tty-progressbar'
|
5
|
+
require 'open3'
|
6
|
+
require 'shellwords'
|
7
|
+
require 'fileutils'
|
8
|
+
require 'pastel'
|
9
|
+
|
10
|
+
class ThreadedTests
|
11
|
+
def run(pattern: '*', max_threads: 8, max_tests: 0)
|
12
|
+
pastel = Pastel.new
|
13
|
+
|
14
|
+
start_time = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
15
|
+
@results = File.expand_path('results.log')
|
16
|
+
|
17
|
+
max_threads = 1000 if max_threads.to_i == 0
|
18
|
+
|
19
|
+
shuffle = false
|
20
|
+
|
21
|
+
unless pattern =~ /shuffle/i
|
22
|
+
pattern = "test/curlyq_*#{pattern}*_test.rb"
|
23
|
+
else
|
24
|
+
pattern = "test/curlyq_*_test.rb"
|
25
|
+
shuffle = true
|
26
|
+
end
|
27
|
+
|
28
|
+
tests = Dir.glob(pattern)
|
29
|
+
|
30
|
+
tests.shuffle! if shuffle
|
31
|
+
|
32
|
+
if max_tests.to_i > 0
|
33
|
+
tests = tests.slice(0, max_tests.to_i - 1)
|
34
|
+
end
|
35
|
+
|
36
|
+
puts pastel.cyan("#{tests.count} test files")
|
37
|
+
|
38
|
+
banner = "Running tests [:bar] T/A (#{max_threads.to_s} threads)"
|
39
|
+
|
40
|
+
progress = TTY::ProgressBar::Multi.new(banner,
|
41
|
+
width: 12,
|
42
|
+
clear: true,
|
43
|
+
hide_cursor: true)
|
44
|
+
@children = []
|
45
|
+
tests.each do |t|
|
46
|
+
test_name = File.basename(t, '.rb').sub(/curlyq_(.*?)_test/, '\1')
|
47
|
+
new_sp = progress.register("[:bar] #{test_name}:status",
|
48
|
+
total: tests.count + 8,
|
49
|
+
width: 1,
|
50
|
+
head: ' ',
|
51
|
+
unknown: ' ',
|
52
|
+
hide_cursor: true,
|
53
|
+
clear: true)
|
54
|
+
status = ': waiting'
|
55
|
+
@children.push([test_name, new_sp, status])
|
56
|
+
end
|
57
|
+
|
58
|
+
@elapsed = 0.0
|
59
|
+
@test_total = 0
|
60
|
+
@assrt_total = 0
|
61
|
+
@error_out = []
|
62
|
+
@threads = []
|
63
|
+
@running_tests = []
|
64
|
+
|
65
|
+
begin
|
66
|
+
finish_time = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
67
|
+
while @children.count.positive?
|
68
|
+
|
69
|
+
slices = @children.slice!(0, max_threads)
|
70
|
+
slices.each { |c| c[1].start }
|
71
|
+
slices.each do |s|
|
72
|
+
@threads << Thread.new do
|
73
|
+
run_test(s)
|
74
|
+
finish_time = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
75
|
+
end
|
76
|
+
end
|
77
|
+
|
78
|
+
@threads.each { |t| t.join }
|
79
|
+
end
|
80
|
+
|
81
|
+
finish_time = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
82
|
+
|
83
|
+
progress.finish
|
84
|
+
rescue
|
85
|
+
progress.stop
|
86
|
+
ensure
|
87
|
+
msg = @running_tests.map { |t| t[1].format.sub(/^\[:bar\] (.*?):status/, "\\1#{t[2]}") }.join("\n")
|
88
|
+
|
89
|
+
output = []
|
90
|
+
output << if @error_out.count.positive?
|
91
|
+
pastel.red("#{@error_out.count} Issues")
|
92
|
+
else
|
93
|
+
pastel.green('Success')
|
94
|
+
end
|
95
|
+
output << pastel.green("#{@test_total} tests")
|
96
|
+
output << pastel.cyan("#{@assrt_total} assertions")
|
97
|
+
output << pastel.yellow("#{(finish_time - start_time).round(3)}s")
|
98
|
+
puts output.join(', ')
|
99
|
+
|
100
|
+
if @error_out.count.positive?
|
101
|
+
puts @error_out.join(pastel.white("\n----\n"))
|
102
|
+
Process.exit 1
|
103
|
+
end
|
104
|
+
end
|
105
|
+
end
|
106
|
+
|
107
|
+
def run_test(s)
|
108
|
+
pastel = Pastel.new
|
109
|
+
|
110
|
+
bar = s[1]
|
111
|
+
s[2] = ": #{pastel.green('running')}"
|
112
|
+
bar.advance(status: s[2])
|
113
|
+
|
114
|
+
if @running_tests.count.positive?
|
115
|
+
@running_tests.each do |b|
|
116
|
+
prev_bar = b[1]
|
117
|
+
if prev_bar.complete?
|
118
|
+
prev_bar.reset
|
119
|
+
prev_bar.advance(status: b[2])
|
120
|
+
prev_bar.finish
|
121
|
+
else
|
122
|
+
prev_bar.update(head: ' ', unfinished: ' ')
|
123
|
+
prev_bar.advance(status: b[2])
|
124
|
+
end
|
125
|
+
end
|
126
|
+
end
|
127
|
+
|
128
|
+
@running_tests.push(s)
|
129
|
+
out, _err, status = Open3.capture3(ENV, 'rake', "test:#{s[0]}", stdin_data: nil)
|
130
|
+
time = out.match(/^Finished in (?<time>\d+\.\d+) seconds\./)
|
131
|
+
count = out.match(/^(?<tests>\d+) tests, (?<assrt>\d+) assertions, (?<fails>\d+) failures, (?<errs>\d+) errors/)
|
132
|
+
|
133
|
+
unless status.success? && !count['fails'].to_i.positive? && !count['errs'].to_i.positive?
|
134
|
+
s[2] = if count
|
135
|
+
": #{paste.red(count['fails'])} #{pastel.red('failures')}, #{pastel.red(count['errs'])} #{pastel.red('errors')}"
|
136
|
+
else
|
137
|
+
": #{pastel.red('Unknown Error')}"
|
138
|
+
end
|
139
|
+
bar.update(head: pastel.red('✖'))
|
140
|
+
bar.advance(head: pastel.red('✖'), status: s[2])
|
141
|
+
|
142
|
+
# errs = out.scan(/(?:Failure|Error): [\w_]+\((?:.*?)\):(?:.*?)(?=\n=======)/m)
|
143
|
+
@error_out.push(out)
|
144
|
+
bar.finish
|
145
|
+
|
146
|
+
next_test
|
147
|
+
Thread.exit
|
148
|
+
end
|
149
|
+
|
150
|
+
s[2] = [
|
151
|
+
': ',
|
152
|
+
pastel.green(count['tests']),
|
153
|
+
'/',
|
154
|
+
pastel.cyan(count['assrt']),
|
155
|
+
' ',
|
156
|
+
pastel.yellow(time['time'].to_f.round(3).to_s),
|
157
|
+
's'
|
158
|
+
].join('')
|
159
|
+
bar.update(head: pastel.green('✔'))
|
160
|
+
bar.advance(head: pastel.green('✔'), status: s[2])
|
161
|
+
@test_total += count['tests'].to_i
|
162
|
+
@assrt_total += count['assrt'].to_i
|
163
|
+
@elapsed += time['time'].to_f
|
164
|
+
|
165
|
+
bar.finish
|
166
|
+
|
167
|
+
next_test
|
168
|
+
end
|
169
|
+
|
170
|
+
def next_test
|
171
|
+
if @children.count.positive?
|
172
|
+
t = Thread.new do
|
173
|
+
s = @children.shift
|
174
|
+
# s[1].start
|
175
|
+
# s[1].advance(status: ": #{'running'.green}")
|
176
|
+
run_test(s)
|
177
|
+
end
|
178
|
+
|
179
|
+
t.join
|
180
|
+
end
|
181
|
+
end
|
182
|
+
end
|
data/test/test_helper.rb
CHANGED
@@ -1,4 +1,9 @@
|
|
1
|
-
|
1
|
+
# frozen_string_literal: true
|
2
2
|
|
3
|
+
require 'test/unit'
|
3
4
|
# Add test libraries you want to use here, e.g. mocha
|
4
|
-
|
5
|
+
|
6
|
+
class Test::Unit::TestCase
|
7
|
+
ENV['TZ'] = 'UTC'
|
8
|
+
# Add global extensions to the test case class here
|
9
|
+
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: curlyq
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.6
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Brett Terpstra
|
@@ -16,42 +16,48 @@ dependencies:
|
|
16
16
|
requirements:
|
17
17
|
- - "~>"
|
18
18
|
- !ruby/object:Gem::Version
|
19
|
-
version: 0
|
19
|
+
version: '13.0'
|
20
|
+
- - ">="
|
21
|
+
- !ruby/object:Gem::Version
|
22
|
+
version: 13.0.1
|
20
23
|
type: :development
|
21
24
|
prerelease: false
|
22
25
|
version_requirements: !ruby/object:Gem::Requirement
|
23
26
|
requirements:
|
24
27
|
- - "~>"
|
25
28
|
- !ruby/object:Gem::Version
|
26
|
-
version: 0
|
29
|
+
version: '13.0'
|
30
|
+
- - ">="
|
31
|
+
- !ruby/object:Gem::Version
|
32
|
+
version: 13.0.1
|
27
33
|
- !ruby/object:Gem::Dependency
|
28
34
|
name: rdoc
|
29
35
|
requirement: !ruby/object:Gem::Requirement
|
30
36
|
requirements:
|
31
37
|
- - "~>"
|
32
38
|
- !ruby/object:Gem::Version
|
33
|
-
version:
|
39
|
+
version: 6.3.1
|
34
40
|
type: :development
|
35
41
|
prerelease: false
|
36
42
|
version_requirements: !ruby/object:Gem::Requirement
|
37
43
|
requirements:
|
38
44
|
- - "~>"
|
39
45
|
- !ruby/object:Gem::Version
|
40
|
-
version:
|
46
|
+
version: 6.3.1
|
41
47
|
- !ruby/object:Gem::Dependency
|
42
|
-
name:
|
48
|
+
name: test-unit
|
43
49
|
requirement: !ruby/object:Gem::Requirement
|
44
50
|
requirements:
|
45
51
|
- - "~>"
|
46
52
|
- !ruby/object:Gem::Version
|
47
|
-
version:
|
53
|
+
version: 3.4.4
|
48
54
|
type: :development
|
49
55
|
prerelease: false
|
50
56
|
version_requirements: !ruby/object:Gem::Requirement
|
51
57
|
requirements:
|
52
58
|
- - "~>"
|
53
59
|
- !ruby/object:Gem::Version
|
54
|
-
version:
|
60
|
+
version: 3.4.4
|
55
61
|
- !ruby/object:Gem::Dependency
|
56
62
|
name: yard
|
57
63
|
requirement: !ruby/object:Gem::Requirement
|
@@ -72,6 +78,80 @@ dependencies:
|
|
72
78
|
- - ">="
|
73
79
|
- !ruby/object:Gem::Version
|
74
80
|
version: 0.9.26
|
81
|
+
- !ruby/object:Gem::Dependency
|
82
|
+
name: tty-spinner
|
83
|
+
requirement: !ruby/object:Gem::Requirement
|
84
|
+
requirements:
|
85
|
+
- - "~>"
|
86
|
+
- !ruby/object:Gem::Version
|
87
|
+
version: '0.9'
|
88
|
+
- - ">="
|
89
|
+
- !ruby/object:Gem::Version
|
90
|
+
version: 0.9.3
|
91
|
+
type: :development
|
92
|
+
prerelease: false
|
93
|
+
version_requirements: !ruby/object:Gem::Requirement
|
94
|
+
requirements:
|
95
|
+
- - "~>"
|
96
|
+
- !ruby/object:Gem::Version
|
97
|
+
version: '0.9'
|
98
|
+
- - ">="
|
99
|
+
- !ruby/object:Gem::Version
|
100
|
+
version: 0.9.3
|
101
|
+
- !ruby/object:Gem::Dependency
|
102
|
+
name: tty-progressbar
|
103
|
+
requirement: !ruby/object:Gem::Requirement
|
104
|
+
requirements:
|
105
|
+
- - "~>"
|
106
|
+
- !ruby/object:Gem::Version
|
107
|
+
version: '0.18'
|
108
|
+
- - ">="
|
109
|
+
- !ruby/object:Gem::Version
|
110
|
+
version: 0.18.2
|
111
|
+
type: :development
|
112
|
+
prerelease: false
|
113
|
+
version_requirements: !ruby/object:Gem::Requirement
|
114
|
+
requirements:
|
115
|
+
- - "~>"
|
116
|
+
- !ruby/object:Gem::Version
|
117
|
+
version: '0.18'
|
118
|
+
- - ">="
|
119
|
+
- !ruby/object:Gem::Version
|
120
|
+
version: 0.18.2
|
121
|
+
- !ruby/object:Gem::Dependency
|
122
|
+
name: pastel
|
123
|
+
requirement: !ruby/object:Gem::Requirement
|
124
|
+
requirements:
|
125
|
+
- - "~>"
|
126
|
+
- !ruby/object:Gem::Version
|
127
|
+
version: 0.8.0
|
128
|
+
type: :development
|
129
|
+
prerelease: false
|
130
|
+
version_requirements: !ruby/object:Gem::Requirement
|
131
|
+
requirements:
|
132
|
+
- - "~>"
|
133
|
+
- !ruby/object:Gem::Version
|
134
|
+
version: 0.8.0
|
135
|
+
- !ruby/object:Gem::Dependency
|
136
|
+
name: parallel_tests
|
137
|
+
requirement: !ruby/object:Gem::Requirement
|
138
|
+
requirements:
|
139
|
+
- - "~>"
|
140
|
+
- !ruby/object:Gem::Version
|
141
|
+
version: '3.7'
|
142
|
+
- - ">="
|
143
|
+
- !ruby/object:Gem::Version
|
144
|
+
version: 3.7.3
|
145
|
+
type: :development
|
146
|
+
prerelease: false
|
147
|
+
version_requirements: !ruby/object:Gem::Requirement
|
148
|
+
requirements:
|
149
|
+
- - "~>"
|
150
|
+
- !ruby/object:Gem::Version
|
151
|
+
version: '3.7'
|
152
|
+
- - ">="
|
153
|
+
- !ruby/object:Gem::Version
|
154
|
+
version: 3.7.3
|
75
155
|
- !ruby/object:Gem::Dependency
|
76
156
|
name: gli
|
77
157
|
requirement: !ruby/object:Gem::Requirement
|
@@ -159,7 +239,18 @@ files:
|
|
159
239
|
- lib/curly/string.rb
|
160
240
|
- lib/curly/version.rb
|
161
241
|
- src/_README.md
|
242
|
+
- test/curlyq_extract_test.rb
|
243
|
+
- test/curlyq_headlinks_test.rb
|
244
|
+
- test/curlyq_html_test.rb
|
245
|
+
- test/curlyq_images_test.rb
|
246
|
+
- test/curlyq_json_test.rb
|
247
|
+
- test/curlyq_links_test.rb
|
248
|
+
- test/curlyq_scrape_test.rb
|
249
|
+
- test/curlyq_tags_test.rb
|
162
250
|
- test/default_test.rb
|
251
|
+
- test/helpers/curlyq-helpers.rb
|
252
|
+
- test/helpers/fake_std_out.rb
|
253
|
+
- test/helpers/threaded_tests.rb
|
163
254
|
- test/test_helper.rb
|
164
255
|
homepage: https://brettterpstra.com
|
165
256
|
licenses:
|