curlyq 0.0.5 → 0.0.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2c5eb3f9a5444f19c44362545b302e3889c4e25dc34d9180452a736b1b80bc34
-  data.tar.gz: 3bf8d1009f493b60c31efb3636c64aa8871656dbcd9cebbeb01800d30fd0761c
+  metadata.gz: 92b27e3065435d17fd5d6129bd640481dee8acf66f9ed82b2b68ae6a7589f463
+  data.tar.gz: a5cd5299248fd01d8f12a80a1c9982de4f8e0160ea5033ab96b60677bb9ea2c8
 SHA512:
-  metadata.gz: 808d8122080450acee5e98e0a6338e887ba5b6e3306764dab79c713052c6e5f6749d8b4ef90f43fcdc2cc7da41766f40e6684e0e40d2de98055e2d71986ac0e8
-  data.tar.gz: d4e17b0cc425cbf7a704cdd188e36f734707cd885a097c6e99cb0f8bc0089e46ffdd99d1e15844a981bbfd9a205778178e45dcaa637cd8a7e761432f2610991e
+  metadata.gz: 1348b97fdf89faf44cd0cfc0f2aecc05a679606f19fe57392d588209da26fc3a5c2407569173d41e1497bdf409d762adee0d2533089b5ce27854e298fe98cc13
+  data.tar.gz: c80ecd381e1d941d8e8e5ead0dd925682e26a2f8cc639f0202d5b5cd30f025582fc5d5a2665daca3e5e7d0f2099066d3ca3d9ad8ad00b07d2ee5673b122bae01

data/CHANGELOG.md

@@ -1,3 +1,28 @@
+### 0.0.7
+
+2024-01-12 17:03
+
+#### FIXED
+
+- Revert back to offering single response (no array) in cases where there are single results (for some commands)
+
+### 0.0.6
+
+2024-01-12 14:44
+
+#### CHANGED
+
+- Attributes array is now a hash directly keyed to the attribute key
+
+#### NEW
+
+- Tags command has option to output only raw html of matched tags
+
+#### FIXED
+
+- --query works with --search on scrape and tags command
+- Json command dot query works now
+
 ### 0.0.5
 
 2024-01-11 18:06

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    curlyq (0.0.5)
+    curlyq (0.0.7)
       gli (~> 2.21.0)
       nokogiri (~> 1.16.0)
       selenium-webdriver (~> 4.16.0)
@@ -11,19 +11,38 @@ GEM
   remote: https://rubygems.org/
   specs:
     gli (2.21.1)
-    minitest (5.16.3)
     nokogiri (1.16.0-arm64-darwin)
       racc (~> 1.4)
+    parallel (1.23.0)
+    parallel_tests (3.13.0)
+      parallel
+    pastel (0.8.0)
+      tty-color (~> 0.5)
+    power_assert (2.0.3)
     racc (1.7.3)
-    rake (0.9.6)
-    rdoc (4.3.0)
+    rake (13.1.0)
+    rdoc (6.3.3)
     rexml (3.2.6)
     rubyzip (2.3.2)
     selenium-webdriver (4.16.0)
       rexml (~> 3.2, >= 3.2.5)
       rubyzip (>= 1.2.2, < 3.0)
       websocket (~> 1.0)
+    strings-ansi (0.2.0)
+    test-unit (3.4.9)
+      power_assert
+    tty-color (0.6.0)
+    tty-cursor (0.7.1)
+    tty-progressbar (0.18.2)
+      strings-ansi (~> 0.2)
+      tty-cursor (~> 0.7)
+      tty-screen (~> 0.8)
+      unicode-display_width (>= 1.6, < 3.0)
+    tty-screen (0.8.2)
+    tty-spinner (0.9.3)
+      tty-cursor (~> 0.7)
     tty-which (0.5.0)
+    unicode-display_width (2.5.0)
     websocket (1.2.10)
     yard (0.9.34)
 
@@ -32,9 +51,13 @@ PLATFORMS
 
 DEPENDENCIES
   curlyq!
-  minitest (~> 5.14)
-  rake (~> 0.9.2)
-  rdoc (~> 4.3)
+  parallel_tests (~> 3.7, >= 3.7.3)
+  pastel (~> 0.8.0)
+  rake (~> 13.0, >= 13.0.1)
+  rdoc (~> 6.3.1)
+  test-unit (~> 3.4.4)
+  tty-progressbar (~> 0.18, >= 0.18.2)
+  tty-spinner (~> 0.9, >= 0.9.3)
   yard (~> 0.9, >= 0.9.26)
 
 BUNDLED WITH

data/README.md

@@ -10,7 +10,7 @@ _If you find this useful, feel free to [buy me some coffee][donate]._
 [donate]: https://brettterpstra.com/donate
 
 
-The current version of `curlyq` is 0.0.5
+The current version of `curlyq` is 0.0.7
 .
 
 CurlyQ is a utility that provides a simple interface for curl, with additional features for things like extracting images and links, finding elements by CSS selector or XPath, getting detailed header info, and more. It's designed to be part of a scripting pipeline, outputting everything as structured data (JSON or YAML). It also has rudimentary support for making calls to JSON endpoints easier, but it's expected that you'll use something like `jq` to parse the output.
@@ -44,7 +44,7 @@ SYNOPSIS
     curlyq [global options] command [command options] [arguments...]
 
 VERSION
-    0.0.5
+    0.0.7
 
 GLOBAL OPTIONS
     --help          - Show this message
@@ -252,10 +252,9 @@ OpenGraph images will be returned with the structure:
         "title": "CurlyQ, curl better",
         "attrs": [
           {
-            "key": "class",
-            "value": [
+            "class": [
               "aligncenter"
-            ], // all attributes included
+             ], // all attributes included
           }
         ]
       }
@@ -335,7 +334,22 @@ Returns all the links on the page, which can be queried on any attribute.
 
 Example:
 
-    curlyq images -t img -q '[width>750]' https://brettterpstra.com
+    curlyq links -q '[content*=twitter]' 'https://stackoverflow.com/questions/52428409/get-fully-rendered-html-using-selenium-webdriver-and-python'
+
+    [
+      {
+        "href": "https://twitter.com/stackoverflow",
+        "title": null,
+        "rel": null,
+        "content": "Twitter",
+        "class": [
+          "-link",
+          "js-gps-track"
+        ]
+      }
+    ]
+
+This example gets all links from the page but only returns ones with link content containing 'twitter' (`-q '[content*=twitter]'`).
 
 ```
 NAME
@@ -426,6 +440,26 @@ COMMAND OPTIONS
 
 Return a hierarchy of all tags in a page. Use `-t` to limit to a specific tag.
 
+    curlyq tags --search '#main .post h3' -q 'attrs[id*=what]' https://brettterpstra.com/2024/01/10/introducing-curlyq-a-pipeline-oriented-curl-helper/
+
+    [
+      {
+        "tag": "h3",
+        "source": "<h3 id=\"whats-next\">What???s Next</h3>",
+        "attrs": [
+          {
+            "id": "whats-next"
+          }
+        ],
+        "content": "What???s Next",
+        "tags": [
+
+        ]
+      }
+    ]
+
+The above command filters the tags based on a CSS query, then further filters them to just tags with an id containing 'what'.
+
 ```
 NAME
     tags - Extract all instances of a tag
@@ -435,12 +469,13 @@ SYNOPSIS
     curlyq [global options] tags [command options] URL...
 
 COMMAND OPTIONS
-    -c, --[no-]compressed     - Expect compressed results
-    --[no-]clean              - Remove extra whitespace from results
-    -h, --header=arg          - Define a header to send as key=value (may be used more than once, default: none)
-    -q, --query, --filter=arg - CSS/XPath query (default: none)
-    --search=arg              - Regurn an array of matches to a CSS or XPath query (default: none)
-    -t, --tag=arg             - Specify a tag to collect (may be used more than once, default: none)
+    -c, --[no-]compressed            - Expect compressed results
+    --[no-]clean                     - Remove extra whitespace from results
+    -h, --header=KEY=VAL             - Define a header to send as key=value (may be used more than once, default: none)
+    -q, --query, --filter=DOT_SYNTAX - Dot syntax query to filter results (default: none)
+    --search=CSS/XPATH               - Regurn an array of matches to a CSS or XPath query (default: none)
+    --[no-]source, --[no-]html       - Output the HTML source of the results
+    -t, --tag=TAG                    - Specify a tag to collect (may be used more than once, default: none)
 ```
 
 

data/Rakefile

@@ -1,8 +1,12 @@
 require 'rake/clean'
+require 'rake/testtask'
 require 'rubygems'
 require 'rubygems/package_task'
 require 'rdoc/task'
 require 'yard'
+require 'parallel_tests'
+require 'parallel_tests/tasks'
+require 'tty-spinner'
 
 YARD::Rake::YardocTask.new do |t|
  t.files = ['lib/curly/*.rb']
@@ -22,10 +26,34 @@ spec = eval(File.read('curlyq.gemspec'))
 
 Gem::PackageTask.new(spec) do |pkg|
 end
-require 'rake/testtask'
-Rake::TestTask.new do |t|
-  t.libs << "test"
-  t.test_files = FileList['test/*_test.rb']
+
+namespace :test do
+  FileList['test/*_test.rb'].each do |rakefile|
+    test_name = File.basename(rakefile, '.rb').sub(/^.*?_(.*?)_.*?$/, '\1')
+
+    Rake::TestTask.new(:"#{test_name}") do |t|
+      t.libs << ['test', 'test/helpers']
+      t.pattern = rakefile
+      t.verbose = ENV['VERBOSE'] =~ /(true|1)/i ? true : false
+    end
+    # Define default task for :test
+    task default: test_name
+  end
+end
+
+desc 'Run one test verbosely'
+task :test_one, :test do |_, args|
+  args.with_defaults(test: '*')
+  puts `bundle exec rake test TESTOPTS="-v" TEST="test/curlyq_#{args[:test]}_test.rb"`
+end
+
+desc 'Run all tests, threaded'
+task :test, :pattern, :threads, :max_tests do |_, args|
+  args.with_defaults(pattern: '*', threads: 8, max_tests: 0)
+  pattern = args[:pattern] =~ /(n[iu]ll?|0|\.)/i ? '*' : args[:pattern]
+
+  require_relative 'test/helpers/threaded_tests'
+  ThreadedTests.new.run(pattern: pattern, max_threads: args[:threads].to_i, max_tests: args[:max_tests])
 end
 
 desc 'Development version check'

data/bin/curlyq

@@ -144,7 +144,7 @@ command %i[html curl] do |c|
     end
     output.delete_if(&:nil?)
     output.delete_if(&:empty?)
-    output = output[0] if output.count == 1
+    # output = output[0] if output.count == 1
     output.map! { |o| o[options[:raw].to_sym] } if options[:raw]
 
     print_out(output, global_options[:yaml], raw: options[:raw], pretty: global_options[:pretty])
@@ -222,12 +222,26 @@ command :json do |c|
           headers: res.headers
         })
       else
-        json = json.dot_query(options[:query]) if options[:query]
+        if options[:query]
+          if options[:query] =~ /^json$/
+            res = json
+          elsif options[:query] =~ /^json\./
+            query = options[:query].sub(/^json\./, '')
+          else
+            query = options[:query]
+          end
+
+          res = json.dot_query(query)
+        else
+          res = res.to_data
+        end
 
-        output.push(json)
+        output.push(res)
       end
     end
 
+    # output = output[0] if output.count == 1
+
     print_out(output, global_options[:yaml], pretty: global_options[:pretty])
   end
 end
@@ -290,10 +304,10 @@ desc 'Extract all instances of a tag'
 arg_name 'URL', multiple: true
 command :tags do |c|
   c.desc 'Define a header to send as key=value'
-  c.flag %i[h header], multiple: true
+  c.flag %i[h header], multiple: true, arg_name: 'KEY=VAL'
 
   c.desc 'Specify a tag to collect'
-  c.flag %i[t tag], multiple: true
+  c.flag %i[t tag], multiple: true, arg_name: 'TAG'
 
   c.desc 'Expect compressed results'
   c.switch %i[c compressed]
@@ -301,11 +315,14 @@ command :tags do |c|
   c.desc 'Remove extra whitespace from results'
   c.switch %i[clean]
 
-  c.desc 'CSS/XPath query'
-  c.flag %i[q query filter]
+  c.desc 'Output the HTML source of the results'
+  c.switch %i[source html]
+
+  c.desc 'Dot syntax query to filter results'
+  c.flag %i[q query filter], arg_name: 'DOT_SYNTAX'
 
   c.desc 'Regurn an array of matches to a CSS or XPath query'
-  c.flag %i[search]
+  c.flag %i[search], arg_name: 'CSS/XPATH'
 
   c.action do |global_options, options, args|
     urls = args.join(' ').split(/[, ]+/)
@@ -322,7 +339,7 @@ command :tags do |c|
       if options[:search]
         out = res.search(options[:search])
 
-        # out = out.dot_query(options[:query]) if options[:query]
+        out = out.dot_query(options[:query]) if options[:query]
         output.push(out)
       elsif options[:query]
         query = options[:query] =~ /^links/ ? options[:query] : "links#{options[:query]}"
@@ -335,7 +352,13 @@ command :tags do |c|
       end
     end
 
-    print_out(output, global_options[:yaml], pretty: global_options[:pretty])
+    output = output[0] if output.count == 1
+
+    if options[:source]
+      puts output.to_html
+    else
+      print_out(output, global_options[:yaml], pretty: global_options[:pretty])
+    end
   end
 end
 

data/curlyq.gemspec

@@ -16,10 +16,14 @@ spec = Gem::Specification.new do |s|
   s.rdoc_options << '--title' << 'curlyq' << '--main' << 'README.rdoc' << '-ri'
   s.bindir = 'bin'
   s.executables << 'curlyq'
-  s.add_development_dependency('rake','~> 0.9.2')
-  s.add_development_dependency('rdoc', '~> 4.3')
-  s.add_development_dependency('minitest', '~> 5.14')
+  s.add_development_dependency('rake','~> 13.0', '>= 13.0.1')
+  s.add_development_dependency('rdoc', '~> 6.3.1')
+  s.add_development_dependency('test-unit', '~> 3.4.4')
   s.add_development_dependency('yard', '~> 0.9', '>= 0.9.26')
+  s.add_development_dependency('tty-spinner', '~> 0.9', '>= 0.9.3')
+  s.add_development_dependency('tty-progressbar', '~> 0.18', '>= 0.18.2')
+  s.add_development_dependency('pastel', '~> 0.8.0')
+  s.add_development_dependency('parallel_tests', '~> 3.7', '>= 3.7.3')
   s.add_runtime_dependency('gli','~> 2.21.0')
   s.add_runtime_dependency('tty-which','~> 0.5.0')
   s.add_runtime_dependency('nokogiri','~> 1.16.0')

data/lib/curly/array.rb

@@ -66,6 +66,30 @@ class ::Array
     replace dedup_links
   end
 
+  #---------------------------------------------------------
+  ## Run a query on array elements
+  ##
+  ## @param      path [String] dot.syntax path to compare
+  ##
+  ## @return [Array] elements matching dot query
+  ##
+  def dot_query(path)
+    filter! do |tag|
+      r = tag.dot_query(path)
+      if r.is_a?(Array)
+        r.count.positive?
+      else
+        r
+      end
+    end
+
+    return self
+  end
+
+  def to_html
+    map { |el| el.to_html }
+  end
+
   ##
   ## Test if a tag contains an attribute matching filter queries
   ##

data/lib/curly/curl/html.rb

@@ -243,11 +243,11 @@ module Curl
           when /source/
             next unless %i[all srcset].include?(type)
 
-            srcsets = img[:attrs].filter { |k| k[:key] =~ /srcset/i }
+            srcsets = img[:attrs].filter { |k| k == 'srcset' }
             if srcsets.count.positive?
               srcset = []
-              srcsets.each do |src|
-                src[:value].split(/ *, */).each do |s|
+              srcsets.each do |k, v|
+                v.split(/ *, */).each do |s|
                   image, media = s.split(/ /)
                   srcset << {
                     src: image,
@@ -263,15 +263,14 @@ module Curl
             end
           when /img/
             next unless %i[all img].include?(type)
-
-            width = img[:attrs].select { |a| a[:key] == 'width' }.first[:value]
-            height = img[:attrs].select { |a| a[:key] == 'height' }.first[:value]
-            alt = img[:attrs].select { |a| a[:key] == 'alt' }.first[:value]
-            title = img[:attrs].select { |a| a[:key] == 'title' }.first[:value]
+            width = img[:attrs]['width']
+            height = img[:attrs]['height']
+            alt = img[:attrs]['alt']
+            title = img[:attrs]['title']
 
             output << {
               type: 'img',
-              src: img[:attrs].filter { |a| a[:key] =~ /src/i }.first[:value],
+              src: img[:attrs]['src'],
               width: width || 'unknown',
               height: height || 'unknown',
               alt: alt,
@@ -324,8 +323,9 @@ module Curl
     ## @param      el    [Nokogiri] element to convert
     ##
     def nokogiri_to_tag(el)
-      attributes = el.attribute_nodes.map do |a|
-        { key: a.name, value: a.name =~ /^(class|rel)$/ ? a.value.split(/ /) : a.value }
+      attributes = {}
+      attributes = el.attribute_nodes.each_with_object({}) do |a, hsh|
+        hsh[a.name] = a.name =~ /^(class|rel)$/ ? a.value.split(/ /) : a.value
       end
 
       {
@@ -405,12 +405,12 @@ module Curl
           attrs = tag['attrs'].strip.to_enum(:scan, /(?ix)
                                              (?<key>[@a-z0-9-]+)(?:=(?<quot>["'])
                                              (?<value>[^"']+)\k<quot>|[ >])?/i).map { Regexp.last_match }
-          attrs.map! { |a| { key: a['key'], value: a['key'] =~ /^(class|rel)$/ ? a['value'].split(/ /) : a['value'] } }
+          attributes = attrs.each_with_object({}) { |a, hsh| hsh[a['key']] = a['key'] =~ /^(class|rel)$/ ? a['value'].split(/ /) : a['value'] }
         end
         {
           tag: tag['tag'],
           source: tag.to_s,
-          attrs: attrs,
+          attrs: attributes,
           content: @clean ? tag['content']&.clean : tag['content'],
           tags: content_tags(tag['content'])
         }

data/lib/curly/hash.rb

@@ -23,6 +23,12 @@ class ::Hash
     end
   end
 
+  def to_html
+    if key?(:source)
+      self[:source]
+    end
+  end
+
   # Extract data using a dot-syntax path
   #
   # @param      path  [String] The path
@@ -31,6 +37,7 @@ class ::Hash
   #
   def dot_query(path)
     res = stringify_keys
+
     out = []
     q = path.split(/(?<![\d.])\./)
     q.each do |pth|
@@ -54,7 +61,10 @@ class ::Hash
       ats.push(at) unless at.empty?
       pth.sub!(/\[\]/, '')
 
+      res = res[0] if res.is_a?(Array)
+
       return false if el.nil? && ats.empty? && !res.key?(pth)
+
       res = res[pth] unless pth.empty?
 
       return false if res.nil?
@@ -62,10 +72,11 @@ class ::Hash
       if ats.count.positive?
         while ats.count.positive?
           atr = ats.shift
-
+          res = [res] if res.is_a?(Hash)
           keepers = res.filter do |r|
             evaluate_comp(r, atr)
           end
+
           out.concat(keepers)
         end
       else
@@ -74,6 +85,7 @@ class ::Hash
 
       out = out[eval(el)] if out.is_a?(Array) && el =~ /^[\d.,]+$/
     end
+
     out
   end
 
@@ -89,6 +101,8 @@ class ::Hash
   def evaluate_comp(r, atr)
     keep = true
 
+    r = r.symbolize_keys
+
     atr.each do |a|
       key = a[0].to_sym
       val = if a[2] =~ /^\d+$/
@@ -206,10 +220,26 @@ class ::Hash
     end
   end
 
-  # Turn all keys into string
+  # Turn all keys into symbols
+  #
+  # If the hash has both a string and a symbol for key,
+  # keep the symbol value, discarding the string value
+  #
+  # @return     [Hash] a copy of the hash where all its
+  #             keys are strings
+  #
+  def symbolize_keys
+    each_with_object({}) do |(k, v), hsh|
+      next if k.is_a?(String) && key?(k.to_sym)
+
+      hsh[k.to_sym] = v.is_a?(Hash) ? v.symbolize_keys : v
+    end
+  end
+
+  # Turn all keys into strings
   #
   # If the hash has both a string and a symbol for key,
-  # keep the string value, discarding the symnbol value
+  # keep the string value, discarding the symbol value
   #
   # @return     [Hash] a copy of the hash where all its
   #             keys are strings

data/lib/curly/version.rb

@@ -1,3 +1,3 @@
 module Curly
-  VERSION = '0.0.5'
+  VERSION = '0.0.7'
 end

data/src/_README.md

@@ -10,7 +10,7 @@ _If you find this useful, feel free to [buy me some coffee][donate]._
 [donate]: https://brettterpstra.com/donate
 <!--END GITHUB-->
 
-The current version of `curlyq` is <!--VER-->0.0.4<!--END VER-->.
+The current version of `curlyq` is <!--VER-->0.0.6<!--END VER-->.
 
 CurlyQ is a utility that provides a simple interface for curl, with additional features for things like extracting images and links, finding elements by CSS selector or XPath, getting detailed header info, and more. It's designed to be part of a scripting pipeline, outputting everything as structured data (JSON or YAML). It also has rudimentary support for making calls to JSON endpoints easier, but it's expected that you'll use something like `jq` to parse the output.
 
@@ -184,10 +184,9 @@ OpenGraph images will be returned with the structure:
         "title": "CurlyQ, curl better",
         "attrs": [
           {
-            "key": "class",
-            "value": [
+            "class": [
               "aligncenter"
-            ], // all attributes included
+             ], // all attributes included
           }
         ]
       }
@@ -245,7 +244,22 @@ Returns all the links on the page, which can be queried on any attribute.
 
 Example:
 
-    curlyq images -t img -q '[width>750]' https://brettterpstra.com
+    curlyq links -q '[content*=twitter]' 'https://stackoverflow.com/questions/52428409/get-fully-rendered-html-using-selenium-webdriver-and-python'
+
+    [
+      {
+        "href": "https://twitter.com/stackoverflow",
+        "title": null,
+        "rel": null,
+        "content": "Twitter",
+        "class": [
+          "-link",
+          "js-gps-track"
+        ]
+      }
+    ]
+
+This example gets all links from the page but only returns ones with link content containing 'twitter' (`-q '[content*=twitter]'`).
 
 ```
 @cli(bundle exec bin/curlyq help links)
@@ -300,6 +314,26 @@ Example:
 
 Return a hierarchy of all tags in a page. Use `-t` to limit to a specific tag.
 
+    curlyq tags --search '#main .post h3' -q 'attrs[id*=what]' https://brettterpstra.com/2024/01/10/introducing-curlyq-a-pipeline-oriented-curl-helper/
+
+    [
+      {
+        "tag": "h3",
+        "source": "<h3 id=\"whats-next\">What’s Next</h3>",
+        "attrs": [
+          {
+            "id": "whats-next"
+          }
+        ],
+        "content": "What’s Next",
+        "tags": [
+
+        ]
+      }
+    ]
+
+The above command filters the tags based on a CSS query, then further filters them to just tags with an id containing 'what'.
+
 ```
 @cli(bundle exec bin/curlyq help tags)
 ```

data/test/curlyq_extract_test.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+require 'json'
+require 'yaml'
+
+require 'helpers/curlyq-helpers'
+require 'test_helper'
+
+# Tests for tags command
+class CurlyQTagsTest < Test::Unit::TestCase
+  include CurlyQHelpers
+
+  def setup
+  end
+
+  def test_extract_inclusive
+    result = curlyq('extract', '-i', '-b', 'Adding', '-a', 'accessing the source.', 'https://stackoverflow.com/questions/52428409/get-fully-rendered-html-using-selenium-webdriver-and-python')
+    json = JSON.parse(result)
+
+    assert_match(/^Adding <code>time.sleep\(10\)<\/code>.*?accessing the source.$/, json[0], 'Match should be found and include the before and after strings')
+  end
+
+  def test_extract_exclusive
+    result = curlyq('extract', '-b', 'Adding', '-a', 'accessing the source.', 'https://stackoverflow.com/questions/52428409/get-fully-rendered-html-using-selenium-webdriver-and-python')
+    json = JSON.parse(result)
+
+    assert_match(/^ <code>time.sleep\(10\)<\/code>.*?when I was $/, json[0], 'Match should be found and not include the before and after strings')
+  end
+
+  def test_extract_regex_inclusive
+    result = curlyq('extract', '-ri', '-b', '.dding <', '-a', 'accessing.*?source.', 'https://stackoverflow.com/questions/52428409/get-fully-rendered-html-using-selenium-webdriver-and-python')
+    json = JSON.parse(result)
+
+    assert_match(/^Adding <code>time.sleep\(10\)<\/code>.*?accessing the source.$/, json[0], 'Match should be found and include the before and after strings')
+  end
+
+  def test_extract_regex_exclusive
+    result = curlyq('extract', '-r', '-b', '.dding <', '-a', 'accessing.*?source.', 'https://stackoverflow.com/questions/52428409/get-fully-rendered-html-using-selenium-webdriver-and-python')
+    json = JSON.parse(result)
+
+    assert_match(/^code>time.sleep\(10\)<\/code>.*?when I was $/, json[0], 'Match should be found and not include the before and after strings')
+  end
+end

data/test/curlyq_headlinks_test.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+require 'json'
+require 'yaml'
+
+require 'helpers/curlyq-helpers'
+require 'test_helper'
+
+# Tests for tags command
+class CurlyQHeadlinksTest < Test::Unit::TestCase
+  include CurlyQHelpers
+
+  def setup
+  end
+
+  def test_headlinks_query
+    result = curlyq('headlinks', '-q', '[rel=stylesheet]', 'https://brettterpstra.com')
+    json = JSON.parse(result)
+
+    assert_match(/stylesheet/, json['rel'], 'Should have retrieved a single result with rel stylesheet')
+    assert_match(/screen\.\d+\.css$/, json['href'], 'Stylesheet should be correct primary stylesheet')
+  end
+
+  def test_headlinks
+    result = curlyq('headlinks', 'https://brettterpstra.com')
+    json = JSON.parse(result)
+
+    assert_equal(Array, json.class, 'Should have an array of results')
+    assert(json.count > 1, 'Should have more than one link')
+    # assert(json[0].count.positive?)
+  end
+end

data/test/curlyq_html_test.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+require 'json'
+require 'yaml'
+
+require 'helpers/curlyq-helpers'
+require 'test_helper'
+
+# Tests for tags command
+class CurlyQHtmlTest < Test::Unit::TestCase
+  include CurlyQHelpers
+
+  def test_html_search_query
+    result = curlyq('html', '-s', '#main article .aligncenter', '-q', 'images[1]', 'https://brettterpstra.com')
+    json = JSON.parse(result)[0]
+
+    assert_match(/aligncenter/, json[0]['class'], 'Should have found an image with class "aligncenter"')
+  end
+
+  def test_html_query
+    result = curlyq('html', '-q', 'meta.title', 'https://brettterpstra.com/2024/01/10/introducing-curlyq-a-pipeline-oriented-curl-helper/')
+
+    assert_match(/Introducing CurlyQ/, result, 'Should have retrived the page title')
+  end
+end

data/test/curlyq_images_test.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+require 'json'
+require 'yaml'
+
+require 'helpers/curlyq-helpers'
+require 'test_helper'
+
+# Tests for tags command
+class CurlyQImagesTest < Test::Unit::TestCase
+  include CurlyQHelpers
+
+  def test_images_query
+    result = curlyq('images', '-t', 'img', '-q', '[alt$=screenshot]', 'https://brettterpstra.com/2024/01/08/keyboard-maestro-giveaway/')
+    json = JSON.parse(result)
+
+    assert(json.count == 1, 'Should have found 1 image')
+    assert_match(/Keyboard Maestro screenshot/, json[0]['alt'], 'Should match Keyboard Meastro screenshot')
+  end
+
+  def test_images_type
+    result = curlyq('images', '-t', 'srcset', 'https://brettterpstra.com/')
+    json = JSON.parse(result)
+
+    assert(json.count.positive?, 'Should have found at least 1 image')
+  end
+end

data/test/curlyq_json_test.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+require 'json'
+require 'yaml'
+
+require 'helpers/curlyq-helpers'
+require 'test_helper'
+
+# Tests for tags command
+class CurlyQJsonTest < Test::Unit::TestCase
+  include CurlyQHelpers
+
+  def setup
+  end
+
+  def test_json
+    result = curlyq('json', 'https://brettterpstra.com/scripts/giveaways_wrapper.cgi?v=203495&giveaway=hazel2023&action=count')
+    json = JSON.parse(result)[0]
+
+    assert_equal(json.class, Hash, 'Single result should be a hash')
+    assert_equal(286, json['json']['total'], 'json.total should match 286')
+  end
+
+  def test_query
+    result1 = curlyq('json', '-q', 'total', 'https://brettterpstra.com/scripts/giveaways_wrapper.cgi?v=203495&giveaway=hazel2023&action=count')
+    result2 = curlyq('json', '-q', 'json.total', 'https://brettterpstra.com/scripts/giveaways_wrapper.cgi?v=203495&giveaway=hazel2023&action=count')
+    json1 = JSON.parse(result1)[0]
+    json2 = JSON.parse(result2)[0]
+
+    assert_equal(286, json1, 'Should be 286')
+    assert_equal(286, json2, 'Including json in dot path should yeild same result')
+  end
+end

data/test/curlyq_links_test.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+require 'json'
+require 'yaml'
+
+require 'helpers/curlyq-helpers'
+require 'test_helper'
+
+# Tests for tags command
+class CurlyQLinksTest < Test::Unit::TestCase
+  include CurlyQHelpers
+
+  def test_links
+    result = curlyq('links', '-q', '[content*=twitter]', 'https://stackoverflow.com/questions/52428409/get-fully-rendered-html-using-selenium-webdriver-and-python')
+    json = JSON.parse(result)
+
+    assert(json.count.positive?, 'Should be at least 1 match')
+    assert_match(/twitter.com/, json[0]['href'], 'Should be a link to Twitter')
+  end
+end

data/test/curlyq_scrape_test.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+require 'json'
+require 'yaml'
+
+require 'helpers/curlyq-helpers'
+require 'test_helper'
+
+# Tests for tags command
+class CurlyQScrapeTest < Test::Unit::TestCase
+  include CurlyQHelpers
+
+  def setup
+  end
+
+  def test_scrape
+    result = curlyq('scrape', '-b', 'firefox', '-q', 'links[rel=me&content*=mastodon][0]', 'https://brettterpstra.com/2024/01/10/introducing-curlyq-a-pipeline-oriented-curl-helper/')
+    json = JSON.parse(result)
+
+    assert_match(/Mastodon/, json['content'], 'Should have retrieved a Mastodon link')
+  end
+end

data/test/curlyq_tags_test.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+require 'json'
+require 'yaml'
+
+require 'helpers/curlyq-helpers'
+require 'test_helper'
+
+# Tests for tags command
+class CurlyQTagsTest < Test::Unit::TestCase
+  include CurlyQHelpers
+
+  def setup
+  end
+
+  def test_tags
+    result = curlyq('tags', '--search', '#main .post h3', '-q', 'attrs[id*=what]', 'https://brettterpstra.com/2024/01/10/introducing-curlyq-a-pipeline-oriented-curl-helper/')
+    json = JSON.parse(result)
+
+    assert_equal(json.count, 1, 'Should have 1 result')
+    assert_match(/whats-next/, json[0]['attrs']['id'], 'Should have matched #whats-next')
+  end
+
+  def test_clean
+    result = curlyq('tags', '--search', '#main section.related', '--clean', 'https://brettterpstra.com/2024/01/10/introducing-curlyq-a-pipeline-oriented-curl-helper/')
+    json = JSON.parse(result)
+
+    assert_equal(json.count, 1, 'Should have 1 result')
+    assert_match(%r{Last.fm</h5></a></li>}, json[0]['source'], 'Should have matched #whats-next')
+  end
+end

data/test/helpers/curlyq-helpers.rb

@@ -0,0 +1,29 @@
+require 'open3'
+require 'time'
+$LOAD_PATH.unshift File.join(__dir__, '..', '..', 'lib')
+require 'curly'
+
+module CurlyQHelpers
+  CURLYQ_EXEC = File.join(File.dirname(__FILE__), '..', '..', 'bin', 'curlyq')
+  BUNDLE = '/Users/ttscoff/.asdf/shims/bundle'
+
+  def curlyq_with_env(env, *args, stdin: nil)
+    Dir.chdir(File.expand_path('~/Desktop/Code/curlyq'))
+    pread(env, BUNDLE, 'exec', 'bin/curlyq', *args, stdin: stdin)
+  end
+
+  def curlyq(*args)
+    curlyq_with_env({ 'GLI_DEBUG' => 'true' }, *args)
+  end
+
+  def pread(env, *cmd, stdin: nil)
+    out, err, status = Open3.capture3(env, *cmd, stdin_data: stdin)
+    unless status.success?
+      raise [
+        "Error (#{status}): #{cmd.inspect} failed", "STDOUT:", out.inspect, "STDERR:", err.inspect
+      ].join("\n")
+    end
+
+    out
+  end
+end

data/test/helpers/fake_std_out.rb

@@ -0,0 +1,30 @@
+class FakeStdOut
+  attr_reader :strings
+
+  def initialize
+    @strings = []
+  end
+
+  def puts(string=nil)
+    @strings << string unless string.nil?
+  end
+
+  def write(x)
+    puts(x)
+  end
+
+  def printf(*args)
+    puts(Kernel.printf(*args))
+  end
+
+  # Returns true if the regexp matches anything in the output
+  def contained?(regexp)
+    strings.find{ |x| x =~ regexp }
+  end
+
+  def flush; end
+
+  def to_s
+    @strings.join("\n")
+  end
+end

data/test/helpers/threaded_tests.rb

@@ -0,0 +1,182 @@
+#!/usr/bin/env ruby
+
+require 'tty-spinner'
+require 'tty-progressbar'
+require 'open3'
+require 'shellwords'
+require 'fileutils'
+require 'pastel'
+
+class ThreadedTests
+  def run(pattern: '*', max_threads: 8, max_tests: 0)
+    pastel = Pastel.new
+
+    start_time = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+    @results = File.expand_path('results.log')
+
+    max_threads = 1000 if max_threads.to_i == 0
+
+    shuffle = false
+
+    unless pattern =~ /shuffle/i
+      pattern = "test/curlyq_*#{pattern}*_test.rb"
+    else
+      pattern = "test/curlyq_*_test.rb"
+      shuffle = true
+    end
+
+    tests = Dir.glob(pattern)
+
+    tests.shuffle! if shuffle
+
+    if max_tests.to_i > 0
+      tests = tests.slice(0, max_tests.to_i - 1)
+    end
+
+    puts pastel.cyan("#{tests.count} test files")
+
+    banner = "Running tests [:bar] T/A (#{max_threads.to_s} threads)"
+
+    progress = TTY::ProgressBar::Multi.new(banner,
+                                           width: 12,
+                                           clear: true,
+                                           hide_cursor: true)
+    @children = []
+    tests.each do |t|
+      test_name = File.basename(t, '.rb').sub(/curlyq_(.*?)_test/, '\1')
+      new_sp = progress.register("[:bar] #{test_name}:status",
+                                 total: tests.count + 8,
+                                 width: 1,
+                                 head: ' ',
+                                 unknown: ' ',
+                                 hide_cursor: true,
+                                 clear: true)
+      status = ': waiting'
+      @children.push([test_name, new_sp, status])
+    end
+
+    @elapsed = 0.0
+    @test_total = 0
+    @assrt_total = 0
+    @error_out = []
+    @threads = []
+    @running_tests = []
+
+    begin
+      finish_time = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+      while @children.count.positive?
+
+        slices = @children.slice!(0, max_threads)
+        slices.each { |c| c[1].start }
+        slices.each do |s|
+          @threads << Thread.new do
+            run_test(s)
+            finish_time = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+          end
+        end
+
+        @threads.each { |t| t.join }
+      end
+
+      finish_time = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+
+      progress.finish
+    rescue
+      progress.stop
+    ensure
+      msg = @running_tests.map { |t| t[1].format.sub(/^\[:bar\] (.*?):status/, "\\1#{t[2]}") }.join("\n")
+
+      output = []
+      output << if @error_out.count.positive?
+                  pastel.red("#{@error_out.count} Issues")
+                else
+                  pastel.green('Success')
+                end
+      output << pastel.green("#{@test_total} tests")
+      output << pastel.cyan("#{@assrt_total} assertions")
+      output << pastel.yellow("#{(finish_time - start_time).round(3)}s")
+      puts output.join(', ')
+
+      if @error_out.count.positive?
+        puts @error_out.join(pastel.white("\n----\n"))
+        Process.exit 1
+      end
+    end
+  end
+
+  def run_test(s)
+    pastel = Pastel.new
+
+    bar = s[1]
+    s[2] = ": #{pastel.green('running')}"
+    bar.advance(status: s[2])
+
+    if @running_tests.count.positive?
+      @running_tests.each do |b|
+        prev_bar = b[1]
+        if prev_bar.complete?
+          prev_bar.reset
+          prev_bar.advance(status: b[2])
+          prev_bar.finish
+        else
+          prev_bar.update(head: ' ', unfinished: ' ')
+          prev_bar.advance(status: b[2])
+        end
+      end
+    end
+
+    @running_tests.push(s)
+    out, _err, status = Open3.capture3(ENV, 'rake', "test:#{s[0]}", stdin_data: nil)
+    time = out.match(/^Finished in (?<time>\d+\.\d+) seconds\./)
+    count = out.match(/^(?<tests>\d+) tests, (?<assrt>\d+) assertions, (?<fails>\d+) failures, (?<errs>\d+) errors/)
+
+    unless status.success? && !count['fails'].to_i.positive? && !count['errs'].to_i.positive?
+      s[2] = if count
+               ": #{pastel.red(count['fails'])} #{pastel.red('failures')}, #{pastel.red(count['errs'])} #{pastel.red('errors')}"
+             else
+               ": #{pastel.red('Unknown Error')}"
+             end
+      bar.update(head: pastel.red('✖'))
+      bar.advance(head: pastel.red('✖'), status: s[2])
+
+      # errs = out.scan(/(?:Failure|Error): [\w_]+\((?:.*?)\):(?:.*?)(?=\n=======)/m)
+      @error_out.push(out)
+      bar.finish
+
+      next_test
+      Thread.exit
+    end
+
+    s[2] = [
+      ': ',
+      pastel.green(count['tests']),
+      '/',
+      pastel.cyan(count['assrt']),
+      ' ',
+      pastel.yellow(time['time'].to_f.round(3).to_s),
+      's'
+    ].join('')
+    bar.update(head: pastel.green('✔'))
+    bar.advance(head: pastel.green('✔'), status: s[2])
+    @test_total += count['tests'].to_i
+    @assrt_total += count['assrt'].to_i
+    @elapsed += time['time'].to_f
+
+    bar.finish
+
+    next_test
+  end
+
+  def next_test
+    if @children.count.positive?
+      t = Thread.new do
+        s = @children.shift
+        # s[1].start
+        # s[1].advance(status: ": #{'running'.green}")
+        run_test(s)
+      end
+
+      t.join
+    end
+  end
+end

data/test/test_helper.rb

@@ -1,4 +1,9 @@
-require "minitest/autorun"
+# frozen_string_literal: true
 
+require 'test/unit'
 # Add test libraries you want to use here, e.g. mocha
-# Add helper classes or methods here, too
+
+class Test::Unit::TestCase
+  ENV['TZ'] = 'UTC'
+  # Add global extensions to the test case class here
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: curlyq
 version: !ruby/object:Gem::Version
-  version: 0.0.5
+  version: 0.0.7
 platform: ruby
 authors:
 - Brett Terpstra
@@ -16,42 +16,48 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.9.2
+        version: '13.0'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 13.0.1
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.9.2
+        version: '13.0'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 13.0.1
 - !ruby/object:Gem::Dependency
   name: rdoc
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '4.3'
+        version: 6.3.1
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '4.3'
+        version: 6.3.1
 - !ruby/object:Gem::Dependency
-  name: minitest
+  name: test-unit
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '5.14'
+        version: 3.4.4
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '5.14'
+        version: 3.4.4
 - !ruby/object:Gem::Dependency
   name: yard
   requirement: !ruby/object:Gem::Requirement
@@ -72,6 +78,80 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 0.9.26
+- !ruby/object:Gem::Dependency
+  name: tty-spinner
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.9'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.9.3
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.9'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.9.3
+- !ruby/object:Gem::Dependency
+  name: tty-progressbar
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.18'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.18.2
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.18'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.18.2
+- !ruby/object:Gem::Dependency
+  name: pastel
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.8.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.8.0
+- !ruby/object:Gem::Dependency
+  name: parallel_tests
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.7'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 3.7.3
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.7'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 3.7.3
 - !ruby/object:Gem::Dependency
   name: gli
   requirement: !ruby/object:Gem::Requirement
@@ -159,7 +239,18 @@ files:
 - lib/curly/string.rb
 - lib/curly/version.rb
 - src/_README.md
+- test/curlyq_extract_test.rb
+- test/curlyq_headlinks_test.rb
+- test/curlyq_html_test.rb
+- test/curlyq_images_test.rb
+- test/curlyq_json_test.rb
+- test/curlyq_links_test.rb
+- test/curlyq_scrape_test.rb
+- test/curlyq_tags_test.rb
 - test/default_test.rb
+- test/helpers/curlyq-helpers.rb
+- test/helpers/fake_std_out.rb
+- test/helpers/threaded_tests.rb
 - test/test_helper.rb
 homepage: https://brettterpstra.com
 licenses: