curlyq 0.0.9 → 0.0.11

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 44e01914de08789721522e24e506fe88a49106610fac9a16736efcba0916be88
-  data.tar.gz: ec2887fee0dab67c64c0095f59091e867445b22559674b31f2eea64d8f4b9fea
+  metadata.gz: a9b0847eb3dd79e15b96bed47858ad0eb0df2ba7db8cf2e3395cb9e08e71c194
+  data.tar.gz: '06623683ff93c02087432750a150ac663c4558b7d18323bbbb367e004abd58ab'
 SHA512:
-  metadata.gz: aa8338482e3d9414e6347195abc7ba8645adbc120f13469f219df7f2aa5fa1ba3c209740c608f728539969502965f6c29afc9d65b5ba411c8387b26ebd640c9d
-  data.tar.gz: 4fe071cb872a259795163da084851afdb5d003f7607a82a5ab6fe868f7f2edae22f0caa31327dfe1bad31e28e48fddde1857d882344359db8bf47b8579aef22c
+  metadata.gz: 8b7098dde55f9b76a53eff1f71a5d821a2db6d5828fb67428f2aa3ef5d6ab8e2bdbb79f5375fb5291b965ff3d0b9677cf0084782c078c2bb5575a8383bd26906
+  data.tar.gz: c0b02267ea0de1c490b2c2dcd171f8a992fa659733aa9bd9e0dc590988af3d7c5f4b6e38e0371ce72c879a1f956ec7f8b87e8432e684d8f7dad4f019314fa834

data/CHANGELOG.md

@@ -1,3 +1,21 @@
+### 0.0.11
+
+2024-01-21 15:29
+
+#### IMPROVED
+
+- Add option for --local_links_only to html and links command, only returning links with the same origin site
+
+### 0.0.10
+
+2024-01-17 13:50
+
+#### IMPROVED
+
+- Update YARD documentation
+- Breaking change, ensure all return types are Arrays, even with single objects, to aid in scriptability
+- Screenshot test suite
+
 ### 0.0.9
 
 2024-01-16 12:38

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    curlyq (0.0.9)
+    curlyq (0.0.11)
       gli (~> 2.21.0)
       nokogiri (~> 1.16.0)
       selenium-webdriver (~> 4.16.0)

data/README.md

@@ -13,7 +13,7 @@ _If you find this useful, feel free to [buy me some coffee][donate]._
 [jq]: https://github.com/jqlang/jq "Command-line JSON processor"
 [yq]: https://github.com/mikefarah/yq "yq is a portable command-line YAML, JSON, XML, CSV, TOML and properties processor"
 
-The current version of `curlyq` is 0.0.9
+The current version of `curlyq` is 0.0.11
 .
 
 CurlyQ is a utility that provides a simple interface for curl, with additional features for things like extracting images and links, finding elements by CSS selector or XPath, getting detailed header info, and more. It's designed to be part of a scripting pipeline, outputting everything as structured data (JSON or YAML). It also has rudimentary support for making calls to JSON endpoints easier, but it's expected that you'll use something like [jq] to parse the output.
@@ -47,7 +47,7 @@ SYNOPSIS
     curlyq [global options] command [command options] [arguments...]
 
 VERSION
-    0.0.9
+    0.0.11
 
 GLOBAL OPTIONS
     --help          - Show this message
@@ -94,11 +94,13 @@ Comparisons can be numeric or string comparisons. A numeric comparison like `cur
 
 You can also use dot syntax inside of comparisons, e.g. `[links.rel*=me]` to target the links object (`html` command), and return only the links with a `rel=me` attribute. If the comparison is to an array object (like `class` or `rel`), it will match if any of the elements of the array match your comparison.
 
-If you end the query with a specific key, only that key will be output. If there's only one match, it will be output as a raw string. If there are multiple matches, output will be an array:
+If you end the query with a specific key, only that key will be output, but it will be in an array. If there's only one match, it will be output as a raw string as a single element in an array.
 
     curlyq tags --search '#main .post h3' -q '[attrs.id*=what].source' 'https://brettterpstra.com/2024/01/10/introducing-curlyq-a-pipeline-oriented-curl-helper/'
     
-    <h3 id="whats-next">What???s Next</h3>
+    [
+      "<h3 id=\"whats-next\">What???s Next</h3>"
+    ]
 
 #### Commands
 
@@ -237,6 +239,7 @@ COMMAND OPTIONS
     -h, --header=arg          - Define a header to send as "key=value" (may be used more than once, default: none)
     --[no-]ignore_fragments   - Ignore fragment hrefs when gathering content links
     --[no-]ignore_relative    - Ignore relative hrefs when gathering content links
+    -l, --local_links_only    - Only gather internal (same-site) links
     -q, --query, --filter=arg - Filter output using dot-syntax path (default: none)
     -r, --raw=arg             - Output a raw value for a key (default: none)
     -s, --search=arg          - Regurn an array of matches to a CSS or XPath query (default: none)
@@ -379,6 +382,7 @@ COMMAND OPTIONS
     -d, --[no-]dedup          - Filter out duplicate links, preserving only first one
     --[no-]ignore_fragments   - Ignore fragment hrefs when gathering content links
     --[no-]ignore_relative    - Ignore relative hrefs when gathering content links
+    -l, --local_links_only    - Only gather internal (same-site) links
     -q, --query, --filter=arg - Filter output using dot-syntax path (default: none)
     -x, --external_links_only - Only gather external links
 ```

data/Rakefile

@@ -56,6 +56,23 @@ task :test, :pattern, :threads, :max_tests do |_, args|
   ThreadedTests.new.run(pattern: pattern, max_threads: args[:threads].to_i, max_tests: args[:max_tests])
 end
 
+desc 'Install current gem in all versions of asdf-controlled ruby'
+task :install do
+  Rake::Task['clobber'].invoke
+  Rake::Task['package'].invoke
+  Dir.chdir 'pkg'
+  file = Dir.glob('*.gem').last
+
+  current_ruby = `asdf current ruby`.match(/(\d.\d+.\d+)/)[1]
+
+  `asdf list ruby`.split.map { |ruby| ruby.strip.sub(/^*/, '') }.each do |ruby|
+    `asdf shell ruby #{ruby}`
+    puts `gem install #{file}`
+  end
+
+  `asdf shell ruby #{current_ruby}`
+end
+
 desc 'Development version check'
 task :ver do
   gver = `git ver`

data/bin/curlyq

@@ -49,7 +49,7 @@ end
 def self.print_out(output, yaml, raw: false, pretty: true)
   output = output.to_data if output.respond_to?(:to_data)
   # Was intended to flatten single responses, but not getting an array back is unpredictable
-  # output = output[0] if output&.is_a?(Array) && output.count == 1
+  output = output.clean_output
   if output.is_a?(String)
     print output
   elsif raw
@@ -103,6 +103,9 @@ command %i[html curl] do |c|
   c.desc 'Only gather external links'
   c.switch %i[x external_links_only], default_value: false, negatable: false
 
+  c.desc 'Only gather internal (same-site) links'
+  c.switch %i[l local_links_only], default_value: false, negatable: false
+
   c.action do |global_options, options, args|
     urls = args.join(' ').split(/[, ]+/)
     headers = break_headers(options[:header])
@@ -115,7 +118,8 @@ command %i[html curl] do |c|
                         compressed: options[:compressed], clean: options[:clean],
                         ignore_local_links: options[:ignore_relative],
                         ignore_fragment_links: options[:ignore_fragments],
-                        external_links_only: options[:external_links_only] }
+                        external_links_only: options[:external_links_only],
+                        local_links_only: options[:local_links_only] }
       res = Curl::Html.new(url, curl_settings)
       res.curl
 
@@ -144,14 +148,9 @@ command %i[html curl] do |c|
     end
     output.delete_if(&:nil?)
     output.delete_if(&:empty?)
-    # output = output[0] if output.count == 1
     output.map! { |o| o[options[:raw].to_sym] } if options[:raw]
 
-    if output.is_a?(Array)
-      while output.length == 1
-        output = output[0]
-      end
-    end
+    output = output.clean_output
 
     print_out(output, global_options[:yaml], raw: options[:raw], pretty: global_options[:pretty])
   end
@@ -246,7 +245,7 @@ command :json do |c|
       end
     end
 
-    # output = output[0] if output.count == 1
+    output = output.clean_output
 
     print_out(output, global_options[:yaml], pretty: global_options[:pretty])
   end
@@ -356,9 +355,7 @@ command :tags do |c|
       end
     end
 
-    while output.is_a?(Array) && output.count == 1
-      output = output[0]
-    end
+    output = output.clean_output
 
     if options[:source]
       puts output.to_html
@@ -424,6 +421,9 @@ command :links do |c|
   c.desc 'Only gather external links'
   c.switch %i[x external_links_only], default_value: false, negatable: false
 
+  c.desc 'Only gather internal (same-site) links'
+  c.switch %i[l local_links_only], default_value: false, negatable: false
+
   c.desc 'Filter output using dot-syntax path'
   c.flag %i[q query filter]
 
@@ -440,7 +440,8 @@ command :links do |c|
                              compressed: options[:compressed], clean: options[:clean],
                              ignore_local_links: options[:ignore_relative],
                              ignore_fragment_links: options[:ignore_fragments],
-                             external_links_only: options[:external_links_only]
+                             external_links_only: options[:external_links_only],
+                             local_links_only: options[:local_links_only]
                            })
       res.curl
 
@@ -482,7 +483,7 @@ command :headlinks do |c|
       end
     end
 
-    output = output[0] if output.count == 1
+    output = output.clean_output
 
     print_out(output, global_options[:yaml], pretty: global_options[:pretty])
   end
@@ -533,7 +534,7 @@ command :scrape do |c|
 
     output.delete_if(&:empty?)
 
-    output = output[0] if output.count == 1
+    output = output.clean_output
 
     if options[:raw]
       output.map! { |o| o[options[:raw].to_sym] }

data/lib/curly/array.rb

@@ -66,7 +66,7 @@ class ::Array
     replace dedup_links
   end
 
-  #---------------------------------------------------------
+  ##
   ## Run a query on array elements
   ##
   ## @param      path [String] dot.syntax path to compare
@@ -80,16 +80,29 @@ class ::Array
     res
   end
 
+  ##
+  ## Gets the value of every item in the array
+  ##
+  ## @param      path  The query path (dot syntax)
+  ##
+  ## @return     [Array] array of values
+  ##
   def get_value(path)
     map { |el| el.get_value(path) }
   end
 
+  ##
+  ## Convert every item in the array to HTML
+  ##
+  ## @return     [String] Html representation of the object.
+  ##
   def to_html
     map(&:to_html)
   end
 
   ##
-  ## Test if a tag contains an attribute matching filter queries
+  ## Test if a tag contains an attribute matching filter
+  ## queries
   ##
   ## @param      tag_name    [String] The tag name
   ## @param      classes     [String] The classes to match
@@ -101,6 +114,8 @@ class ::Array
   ## @param      value       [String] The value to match
   ## @param      descendant  [Boolean] Check descendant tags
   ##
+  ## @return     [Boolean] tag matches
+  ##
   def tag_match(tag_name, classes, id, attribute, operator, value, descendant: false)
     tag = self
     keep = true
@@ -154,4 +169,26 @@ class ::Array
       keep
     end
   end
+
+  ##
+  ## Clean up output, shrink single-item arrays, ensure array output
+  ##
+  ## @return [Array] cleaned up array
+  ##
+  def clean_output
+    output = dup
+    while output.is_a?(Array) && output.count == 1
+      output = output[0]
+    end
+    output.ensure_array
+  end
+
+  ##
+  ## Ensure that an object is an array
+  ##
+  ## @return     [Array] object as Array
+  ##
+  def ensure_array
+    return self
+  end
 end

data/lib/curly/curl/html.rb

@@ -11,11 +11,17 @@ module Curl
   # Class for CURLing an HTML page
   class Html
     attr_accessor :settings, :browser, :source, :headers, :headers_only, :compressed, :clean, :fallback,
-                  :ignore_local_links, :ignore_fragment_links, :external_links_only
+                  :ignore_local_links, :ignore_fragment_links, :external_links_only, :local_links_only
 
     attr_reader :url, :code, :meta, :links, :head, :body,
                 :title, :description, :body_links, :body_images
 
+    # Convert self to a hash of data
+    #
+    # @param      url   [String]  A base url to fall back to
+    #
+    # @return     [Hash] a hash of data
+    #
     def to_data(url: nil)
       {
         url: @url || url,
@@ -63,17 +69,29 @@ module Curl
       @ignore_local_links = options[:ignore_local_links]
       @ignore_fragment_links = options[:ignore_fragment_links]
       @external_links_only = options[:external_links_only]
+      @local_links_only = options[:local_links_only]
 
       @curl = TTY::Which.which('curl')
       @url = url.nil? ? options[:url] : url
     end
 
+    ##
+    # Parse raw HTML source instead of curling
+    #
+    # @param      source  [String] The source
+    #
+    #
+    # @return     [Hash] Hash of data after processing #
+    #
     def parse(source)
       @body = source
       { url: @url, code: @code, headers: @headers, meta: @meta, links: @links, head: @head, body: source,
         source: source.strip, body_links: content_links, body_images: content_images }
     end
 
+    ##
+    ## Curl a url, either with curl or Selenium based on browser settings
+    ##
     def curl
       res = if @url && @browser && @browser != :none
               source = curl_dynamic_html
@@ -283,6 +301,11 @@ module Curl
       output
     end
 
+    ##
+    ## String representation
+    ##
+    ## @return     String representation of the object.
+    ##
     def to_s
       headers = @headers.nil? ? 0 : @headers.count
       meta = @meta.nil? ? 0 : @meta.count
@@ -468,11 +491,19 @@ module Curl
 
         link_href = link_href[2]
 
-        next if link_href =~ /^#/ && (@ignore_fragment_links || @external_links_only)
+        if @local_links_only
+          next if @ignore_fragment_links && link_href =~ /^#/
+
+          next unless same_origin?(link_href)
+
+        else
+          next if link_href =~ /^#/ && (@ignore_fragment_links || @external_links_only)
+
+          next if link_href !~ %r{^(\w+:)?//} && (@ignore_local_links || @external_links_only)
 
-        next if link_href !~ %r{^(\w+:)?//} && (@ignore_local_links || @external_links_only)
+          next if same_origin?(link_href) && @external_links_only
 
-        next if same_origin?(link_href) && @external_links_only
+        end
 
         link_title = tag.match(/title=(['"])(.*?)\1/)
         link_title = link_title.nil? ? nil : link_title[2]
@@ -500,11 +531,19 @@ module Curl
       link_tags.each do |m|
         href = m['tag'].match(/href=(["'])(.*?)\1/)
         href = href[2] unless href.nil?
-        next if href =~ /^#/ && (@ignore_fragment_links || @external_links_only)
+        if @local_links_only
+          next if href =~ /^#/ && @ignore_fragment_links
+
+          next unless same_origin?(href)
 
-        next if href !~ %r{^(\w+:)?//} && (@ignore_local_links || @external_links_only)
+        else
+          next if href =~ /^#/ && (@ignore_fragment_links || @external_links_only)
+
+          next if href !~ %r{^(\w+:)?//} && (@ignore_local_links || @external_links_only)
 
-        next if same_origin?(href) && @external_links_only
+          next if same_origin?(href) && @external_links_only
+
+        end
 
         title = m['tag'].match(/title=(["'])(.*?)\1/)
         title = title[2] unless title.nil?

data/lib/curly/hash.rb

@@ -2,6 +2,14 @@
 
 # Hash helpers
 class ::Hash
+  ## Convert a Curly object to data hash
+  ##
+  ## @return     [Hash] return a hash with keys renamed and
+  ##             cleaned up
+  ##
+  ## @param      url    [String] A url to fall back to
+  ## @param      clean  [Boolean] Clean extra spaces and newlines in sources
+  ##
   def to_data(url: nil, clean: false)
     if key?(:body_links)
       {
@@ -23,22 +31,32 @@ class ::Hash
     end
   end
 
+  ##
+  ## Return the raw HTML of the object
+  ##
+  ## @return    [String] Html representation of the object.
+  ##
   def to_html
     if key?(:source)
       self[:source]
     end
   end
 
+  ##
+  ## Get a value from the hash using a dot-syntax query
+  ##
+  ## @param      query  [String] The query (dot notation)
+  ##
+  ## @return     [Object] result of querying the hash
+  ##
   def get_value(query)
     return nil if self.empty?
     stringify_keys!
 
     query.split('.').inject(self) do |v, k|
-      if v.is_a? Array
-        return v.map { |el| el.get_value(k) }
-      end
+      return v.map { |el| el.get_value(k) } if v.is_a? Array
       # k = k.to_i if v.is_a? Array
-      next unless v.key?(k)
+      next v unless v.key?(k)
 
       v.fetch(k)
     end
@@ -48,7 +66,7 @@ class ::Hash
   #
   # @param      path  [String] The path
   #
-  # @return     Result of path query
+  # @return     [Object] Result of path query
   #
   def dot_query(path, root = nil, full_tag: true)
     res = stringify_keys
@@ -63,7 +81,6 @@ class ::Hash
       "[#{inter}]"
     end
 
-    enumerate = false
     out = []
     q = path.split(/(?<![\d.])\./)
 
@@ -152,6 +169,14 @@ class ::Hash
     out
   end
 
+  ##
+  ## Test if values in an array match an operator
+  ##
+  ## @param      array [Array] The array
+  ## @param      key   [String] The key
+  ## @param      comp  [String] The comparison, e.g. *= or $=
+  ##
+  ## @return [Boolean] true if array contains match
   def array_match(array, key, comp)
     keep = false
     array.each do |el|
@@ -353,7 +378,32 @@ class ::Hash
     end
   end
 
+  ##
+  ## Destructive version of #stringify_keys
+  ##
+  ## @see        #stringify_keys
+  ##
   def stringify_keys!
     replace stringify_keys
   end
+
+  ##
+  ## Clean up empty arrays and return an array with one or
+  ## more elements
+  ##
+  ## @return     [Array] output array
+  ##
+  def clean_output
+    output = ensure_array
+    output.clean_output
+  end
+
+  ##
+  ## Ensure that an object is an array
+  ##
+  ## @return     [Array] object as Array
+  ##
+  def ensure_array
+    return [self]
+  end
 end

data/lib/curly/numeric.rb

@@ -0,0 +1,11 @@
+# Numeric helpers
+class ::Numeric
+  ##
+  ## Return an array version of self
+  ##
+  ## @return     [Array] self enclosed in an array
+  ##
+  def ensure_array
+    [self]
+  end
+end

data/lib/curly/string.rb

@@ -6,6 +6,11 @@
 ## @return     [String] cleaned string
 ##
 class ::String
+  ## Remove extra spaces and newlines, compress space
+  ## between tags
+  ##
+  ## @return     [String] cleaned string
+  ##
   def clean
     gsub(/[\t\n ]+/m, ' ').gsub(/> +</, '><')
   end
@@ -40,7 +45,7 @@ class ::String
   ##
   ## Convert an image type string to a symbol
   ##
-  ## @return     Symbol :srcset, :img, :opengraph, :all
+  ## @return     [Symbol] :srcset, :img, :opengraph, :all
   ##
   def normalize_image_type(default = :all)
     case self.to_s
@@ -58,7 +63,7 @@ class ::String
   ##
   ## Convert a browser type string to a symbol
   ##
-  ## @return     Symbol :chrome, :firefox
+  ## @return     [Symbol] :chrome, :firefox
   ##
   def normalize_browser_type(default = :none)
     case self.to_s
@@ -74,7 +79,7 @@ class ::String
   ##
   ## Convert a screenshot type string to a symbol
   ##
-  ## @return     Symbol :full_page, :print_page, :visible
+  ## @return     [Symbol] :full_page, :print_page, :visible
   ##
   def normalize_screenshot_type(default = :none)
     case self.to_s
@@ -88,4 +93,23 @@ class ::String
       default.is_a?(Symbol) ? default.to_sym : default.normalize_browser_type
     end
   end
+
+  ##
+  ## Clean up output and return a single-item array
+  ##
+  ## @return     [Array] output array
+  ##
+  def clean_output
+    output = ensure_array
+    output.clean_output
+  end
+
+  ##
+  ## Ensure that an object is an array
+  ##
+  ## @return     [Array] object as Array
+  ##
+  def ensure_array
+    return [self]
+  end
 end

data/lib/curly/version.rb

@@ -1,3 +1,5 @@
+# Top level module for CurlyQ
 module Curly
-  VERSION = '0.0.9'
+  # Current version number
+  VERSION = '0.0.11'
 end

data/lib/curly.rb

@@ -4,6 +4,7 @@ require 'curly/version'
 require 'curly/hash'
 require 'curly/string'
 require 'curly/array'
+require 'curly/numeric'
 require 'json'
 require 'yaml'
 require 'uri'

data/src/_README.md

@@ -13,7 +13,7 @@ _If you find this useful, feel free to [buy me some coffee][donate]._
 [jq]: https://github.com/jqlang/jq "Command-line JSON processor"
 [yq]: https://github.com/mikefarah/yq "yq is a portable command-line YAML, JSON, XML, CSV, TOML and properties processor"
 
-The current version of `curlyq` is <!--VER-->0.0.4<!--END VER-->.
+The current version of `curlyq` is <!--VER-->0.0.10<!--END VER-->.
 
 CurlyQ is a utility that provides a simple interface for curl, with additional features for things like extracting images and links, finding elements by CSS selector or XPath, getting detailed header info, and more. It's designed to be part of a scripting pipeline, outputting everything as structured data (JSON or YAML). It also has rudimentary support for making calls to JSON endpoints easier, but it's expected that you'll use something like [jq] to parse the output.
 
@@ -68,11 +68,13 @@ Comparisons can be numeric or string comparisons. A numeric comparison like `cur
 
 You can also use dot syntax inside of comparisons, e.g. `[links.rel*=me]` to target the links object (`html` command), and return only the links with a `rel=me` attribute. If the comparison is to an array object (like `class` or `rel`), it will match if any of the elements of the array match your comparison.
 
-If you end the query with a specific key, only that key will be output. If there's only one match, it will be output as a raw string. If there are multiple matches, output will be an array:
+If you end the query with a specific key, only that key will be output, but it will be in an array. If there's only one match, it will be output as a raw string as a single element in an array.
 
     curlyq tags --search '#main .post h3' -q '[attrs.id*=what].source' 'https://brettterpstra.com/2024/01/10/introducing-curlyq-a-pipeline-oriented-curl-helper/'
     
-    <h3 id="whats-next">What’s Next</h3>
+    [
+      "<h3 id=\"whats-next\">What’s Next</h3>"
+    ]
 
 #### Commands
 

data/test/curlyq_headlinks_test.rb

@@ -17,8 +17,9 @@ class CurlyQHeadlinksTest < Test::Unit::TestCase
     result = curlyq('headlinks', '-q', '[rel=stylesheet]', 'https://brettterpstra.com')
     json = JSON.parse(result)
 
-    assert_match(/stylesheet/, json['rel'], 'Should have retrieved a single result with rel stylesheet')
-    assert_match(/screen\.\d+\.css$/, json['href'], 'Stylesheet should be correct primary stylesheet')
+    assert_equal(Array, json.class, 'Result should be an array')
+    assert_match(/stylesheet/, json[0]['rel'], 'Should have retrieved a single result with rel stylesheet')
+    assert_match(/screen\.\d+\.css$/, json[0]['href'], 'Stylesheet should be correct primary stylesheet')
   end
 
   def test_headlinks

data/test/curlyq_html_test.rb

@@ -14,12 +14,12 @@ class CurlyQHtmlTest < Test::Unit::TestCase
     result = curlyq('html', '-s', '#main article .aligncenter', '-q', 'images[1]', 'https://brettterpstra.com')
     json = JSON.parse(result)
 
-    assert_match(/aligncenter/, json['class'], 'Should have found an image with class "aligncenter"')
+    assert_match(/aligncenter/, json[0]['class'], 'Should have found an image with class "aligncenter"')
   end
 
   def test_html_query
     result = curlyq('html', '-q', 'meta.title', 'https://brettterpstra.com/2024/01/10/introducing-curlyq-a-pipeline-oriented-curl-helper/')
-
-    assert_match(/Introducing CurlyQ/, result, 'Should have retrived the page title')
+    json = JSON.parse(result)
+    assert_match(/Introducing CurlyQ/, json[0], 'Should have retrived the page title')
   end
 end

data/test/curlyq_scrape_test.rb

@@ -11,12 +11,42 @@ class CurlyQScrapeTest < Test::Unit::TestCase
   include CurlyQHelpers
 
   def setup
+    @screenshot = File.join(File.dirname(__FILE__), 'screenshot_test')
+    FileUtils.rm_f("#{@screenshot}.pdf") if File.exist?("#{@screenshot}.pdf")
+    FileUtils.rm_f('screenshot_test.png') if File.exist?("#{@screenshot}.png")
+    FileUtils.rm_f("#{@screenshot}_full.png") if File.exist?("#{@screenshot}_full.png")
   end
 
-  def test_scrape
+  def teardown
+    FileUtils.rm_f("#{@screenshot}.pdf") if File.exist?("#{@screenshot}.pdf")
+    FileUtils.rm_f('screenshot_test.png') if File.exist?("#{@screenshot}.png")
+    FileUtils.rm_f("#{@screenshot}_full.png") if File.exist?("#{@screenshot}_full.png")
+  end
+
+  def test_scrape_firefox
     result = curlyq('scrape', '-b', 'firefox', '-q', 'links[rel=me&content*=mastodon][0]', 'https://brettterpstra.com/2024/01/10/introducing-curlyq-a-pipeline-oriented-curl-helper/')
     json = JSON.parse(result)
 
-    assert_match(/Mastodon/, json['content'], 'Should have retrieved a Mastodon link')
+    assert_equal(Array, json.class, 'Result should be an Array')
+    assert_match(/Mastodon/, json[0]['content'], 'Should have retrieved a Mastodon link')
+  end
+
+  def test_scrape_chrome
+    result = curlyq('scrape', '-b', 'chrome', '-q', 'links[rel=me&content*=mastodon][0]', 'https://brettterpstra.com/2024/01/10/introducing-curlyq-a-pipeline-oriented-curl-helper/')
+    json = JSON.parse(result)
+
+    assert_equal(Array, json.class, 'Result should be an Array')
+    assert_match(/Mastodon/, json[0]['content'], 'Should have retrieved a Mastodon link')
+  end
+
+  def test_screenshot
+    curlyq('screenshot', '-b', 'firefox', '-o', @screenshot, '-t', 'print', 'https://brettterpstra.com')
+    assert(File.exist?("#{@screenshot}.pdf"), 'PDF Screenshot should exist')
+
+    curlyq('screenshot', '-b', 'chrome', '-o', @screenshot, '-t', 'visible', 'https://brettterpstra.com')
+    assert(File.exist?("#{@screenshot}.png"), 'PNG Screenshot should exist')
+
+    curlyq('screenshot', '-b', 'firefox', '-o', "#{@screenshot}_full", '-t', 'full', 'https://brettterpstra.com')
+    assert(File.exist?("#{@screenshot}_full.png"), 'PNG Screenshot should exist')
   end
 end

data/test/curlyq_tags_test.rb

@@ -14,18 +14,26 @@ class CurlyQTagsTest < Test::Unit::TestCase
   end
 
   def test_tags
-    result = curlyq('tags', '--search', '#main .post h3', '-q', 'attrs[id*=what]', 'https://brettterpstra.com/2024/01/10/introducing-curlyq-a-pipeline-oriented-curl-helper/')
+    result = curlyq('tags', '--search', '#main .post h3', 'https://brettterpstra.com/2024/01/10/introducing-curlyq-a-pipeline-oriented-curl-helper/')
     json = JSON.parse(result)
 
-    assert_equal(json.count, 1, 'Should have 1 result')
-    assert_match(/whats-next/, json[0]['attrs']['id'], 'Should have matched #whats-next')
+    assert_equal(Array, json.class, 'Should be an array of matches')
+    assert_equal(6, json.count, 'Should be six results')
   end
 
   def test_clean
     result = curlyq('tags', '--search', '#main section.related', '--clean', 'https://brettterpstra.com/2024/01/10/introducing-curlyq-a-pipeline-oriented-curl-helper/')
     json = JSON.parse(result)
 
-    assert_equal(json.count, 1, 'Should have 1 result')
+    assert_equal(Array, json.class, 'Should be a single Array')
+    assert_equal(1, json.count, 'Should be one element')
     assert_match(%r{Last.fm</h5></a></li>}, json[0]['source'], 'Should have matched #whats-next')
   end
+
+  def test_query
+    result = curlyq('tags', '--search', '#main .post h3', '-q', '[attrs.id*=what].source', 'https://brettterpstra.com/2024/01/10/introducing-curlyq-a-pipeline-oriented-curl-helper/')
+    json = JSON.parse(result)
+    assert_equal(Array, json.class, 'Should be an array')
+    assert_match(%r{^<h3 id="whats-next">What’s Next</h3>$}, json[0], 'Should have returned just source')
+  end
 end

data/test/helpers/curlyq-helpers.rb

@@ -1,5 +1,6 @@
 require 'open3'
 require 'time'
+require 'fileutils'
 $LOAD_PATH.unshift File.join(__dir__, '..', '..', 'lib')
 require 'curly'
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: curlyq
 version: !ruby/object:Gem::Version
-  version: 0.0.9
+  version: 0.0.11
 platform: ruby
 authors:
 - Brett Terpstra
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-01-16 00:00:00.000000000 Z
+date: 2024-01-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -236,6 +236,7 @@ files:
 - lib/curly/curl/html.rb
 - lib/curly/curl/json.rb
 - lib/curly/hash.rb
+- lib/curly/numeric.rb
 - lib/curly/string.rb
 - lib/curly/version.rb
 - src/_README.md