curlyq 0.0.9 → 0.0.10

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 44e01914de08789721522e24e506fe88a49106610fac9a16736efcba0916be88
-  data.tar.gz: ec2887fee0dab67c64c0095f59091e867445b22559674b31f2eea64d8f4b9fea
+  metadata.gz: 6109483b8869733f9e21ecab9bc8bcda0aa3b58ca1f13f9b96fe7739d019df1f
+  data.tar.gz: 98a8d46fe68bc88ea030dfb8e04262fbab5418005390ff79693d6f636a3bf276
 SHA512:
-  metadata.gz: aa8338482e3d9414e6347195abc7ba8645adbc120f13469f219df7f2aa5fa1ba3c209740c608f728539969502965f6c29afc9d65b5ba411c8387b26ebd640c9d
-  data.tar.gz: 4fe071cb872a259795163da084851afdb5d003f7607a82a5ab6fe868f7f2edae22f0caa31327dfe1bad31e28e48fddde1857d882344359db8bf47b8579aef22c
+  metadata.gz: 1d75b4af2d6c1fadb83501fa707184ef41d061c08de14666b86d296048e8f21540fe2ad53a79985d5b042c93fa629cdbe8d101828edbb02832d1b55b920d5834
+  data.tar.gz: 238855918e3e765a2edf1864dd2663a959b099cfa5f1b89942f94eb20ba428c1700adee85590879662f0cf8de659328fbe752e8648ee210eefe0769639c57da2

data/CHANGELOG.md

@@ -1,3 +1,13 @@
+### 0.0.10
+
+2024-01-17 13:50
+
+#### IMPROVED
+
+- Update YARD documentation
+- Breaking change, ensure all return types are Arrays, even with single objects, to aid in scriptability
+- Screenshot test suite
+
 ### 0.0.9
 
 2024-01-16 12:38

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    curlyq (0.0.9)
+    curlyq (0.0.10)
       gli (~> 2.21.0)
       nokogiri (~> 1.16.0)
       selenium-webdriver (~> 4.16.0)

data/README.md

@@ -13,7 +13,7 @@ _If you find this useful, feel free to [buy me some coffee][donate]._
 [jq]: https://github.com/jqlang/jq "Command-line JSON processor"
 [yq]: https://github.com/mikefarah/yq "yq is a portable command-line YAML, JSON, XML, CSV, TOML and properties processor"
 
-The current version of `curlyq` is 0.0.9
+The current version of `curlyq` is 0.0.10
 .
 
 CurlyQ is a utility that provides a simple interface for curl, with additional features for things like extracting images and links, finding elements by CSS selector or XPath, getting detailed header info, and more. It's designed to be part of a scripting pipeline, outputting everything as structured data (JSON or YAML). It also has rudimentary support for making calls to JSON endpoints easier, but it's expected that you'll use something like [jq] to parse the output.
@@ -47,7 +47,7 @@ SYNOPSIS
     curlyq [global options] command [command options] [arguments...]
 
 VERSION
-    0.0.9
+    0.0.10
 
 GLOBAL OPTIONS
     --help          - Show this message

data/Rakefile

@@ -56,6 +56,23 @@ task :test, :pattern, :threads, :max_tests do |_, args|
   ThreadedTests.new.run(pattern: pattern, max_threads: args[:threads].to_i, max_tests: args[:max_tests])
 end
 
+desc 'Install current gem in all versions of asdf-controlled ruby'
+task :install do
+  Rake::Task['clobber'].invoke
+  Rake::Task['package'].invoke
+  Dir.chdir 'pkg'
+  file = Dir.glob('*.gem').last
+
+  current_ruby = `asdf current ruby`.match(/(\d.\d+.\d+)/)[1]
+
+  `asdf list ruby`.split.map { |ruby| ruby.strip.sub(/^*/, '') }.each do |ruby|
+    `asdf shell ruby #{ruby}`
+    puts `gem install #{file}`
+  end
+
+  `asdf shell ruby #{current_ruby}`
+end
+
 desc 'Development version check'
 task :ver do
   gver = `git ver`

data/bin/curlyq

@@ -49,7 +49,7 @@ end
 def self.print_out(output, yaml, raw: false, pretty: true)
   output = output.to_data if output.respond_to?(:to_data)
   # Was intended to flatten single responses, but not getting an array back is unpredictable
-  # output = output[0] if output&.is_a?(Array) && output.count == 1
+  output = output.clean_output
   if output.is_a?(String)
     print output
   elsif raw
@@ -144,14 +144,9 @@ command %i[html curl] do |c|
     end
     output.delete_if(&:nil?)
     output.delete_if(&:empty?)
-    # output = output[0] if output.count == 1
     output.map! { |o| o[options[:raw].to_sym] } if options[:raw]
 
-    if output.is_a?(Array)
-      while output.length == 1
-        output = output[0]
-      end
-    end
+    output = output.clean_output
 
     print_out(output, global_options[:yaml], raw: options[:raw], pretty: global_options[:pretty])
   end
@@ -246,7 +241,7 @@ command :json do |c|
       end
     end
 
-    # output = output[0] if output.count == 1
+    output = output.clean_output
 
     print_out(output, global_options[:yaml], pretty: global_options[:pretty])
   end
@@ -356,9 +351,7 @@ command :tags do |c|
       end
     end
 
-    while output.is_a?(Array) && output.count == 1
-      output = output[0]
-    end
+    output = output.clean_output
 
     if options[:source]
       puts output.to_html
@@ -482,7 +475,7 @@ command :headlinks do |c|
       end
     end
 
-    output = output[0] if output.count == 1
+    output = output.clean_output
 
     print_out(output, global_options[:yaml], pretty: global_options[:pretty])
   end
@@ -533,7 +526,7 @@ command :scrape do |c|
 
     output.delete_if(&:empty?)
 
-    output = output[0] if output.count == 1
+    output = output.clean_output
 
     if options[:raw]
       output.map! { |o| o[options[:raw].to_sym] }

data/lib/curly/array.rb

@@ -66,7 +66,7 @@ class ::Array
     replace dedup_links
   end
 
-  #---------------------------------------------------------
+  ##
   ## Run a query on array elements
   ##
   ## @param      path [String] dot.syntax path to compare
@@ -80,16 +80,29 @@ class ::Array
     res
   end
 
+  ##
+  ## Gets the value of every item in the array
+  ##
+  ## @param      path  The query path (dot syntax)
+  ##
+  ## @return     [Array] array of values
+  ##
   def get_value(path)
     map { |el| el.get_value(path) }
   end
 
+  ##
+  ## Convert every item in the array to HTML
+  ##
+  ## @return     [String] Html representation of the object.
+  ##
   def to_html
     map(&:to_html)
   end
 
   ##
-  ## Test if a tag contains an attribute matching filter queries
+  ## Test if a tag contains an attribute matching filter
+  ## queries
   ##
   ## @param      tag_name    [String] The tag name
   ## @param      classes     [String] The classes to match
@@ -101,6 +114,8 @@ class ::Array
   ## @param      value       [String] The value to match
   ## @param      descendant  [Boolean] Check descendant tags
   ##
+  ## @return     [Boolean] tag matches
+  ##
   def tag_match(tag_name, classes, id, attribute, operator, value, descendant: false)
     tag = self
     keep = true
@@ -154,4 +169,26 @@ class ::Array
       keep
     end
   end
+
+  ##
+  ## Clean up output, shrink single-item arrays, ensure array output
+  ##
+  ## @return [Array] cleaned up array
+  ##
+  def clean_output
+    output = dup
+    while output.is_a?(Array) && output.count == 1
+      output = output[0]
+    end
+    output.ensure_array
+  end
+
+  ##
+  ## Ensure that an object is an array
+  ##
+  ## @return     [Array] object as Array
+  ##
+  def ensure_array
+    return self
+  end
 end

data/lib/curly/curl/html.rb

@@ -16,6 +16,12 @@ module Curl
     attr_reader :url, :code, :meta, :links, :head, :body,
                 :title, :description, :body_links, :body_images
 
+    # Convert self to a hash of data
+    #
+    # @param      url   [String]  A base url to fall back to
+    #
+    # @return     [Hash] a hash of data
+    #
     def to_data(url: nil)
       {
         url: @url || url,
@@ -68,12 +74,23 @@ module Curl
       @url = url.nil? ? options[:url] : url
     end
 
+    ##
+    # Parse raw HTML source instead of curling
+    #
+    # @param      source  [String] The source
+    #
+    #
+    # @return     [Hash] Hash of data after processing #
+    #
     def parse(source)
       @body = source
       { url: @url, code: @code, headers: @headers, meta: @meta, links: @links, head: @head, body: source,
         source: source.strip, body_links: content_links, body_images: content_images }
     end
 
+    ##
+    ## Curl a url, either with curl or Selenium based on browser settings
+    ##
     def curl
       res = if @url && @browser && @browser != :none
               source = curl_dynamic_html
@@ -283,6 +300,11 @@ module Curl
       output
     end
 
+    ##
+    ## String representation
+    ##
+    ## @return     String representation of the object.
+    ##
     def to_s
       headers = @headers.nil? ? 0 : @headers.count
       meta = @meta.nil? ? 0 : @meta.count

data/lib/curly/hash.rb

@@ -2,6 +2,14 @@
 
 # Hash helpers
 class ::Hash
+  ## Convert a Curly object to data hash
+  ##
+  ## @return     [Hash] return a hash with keys renamed and
+  ##             cleaned up
+  ##
+  ## @param      url    [String] A url to fall back to
+  ## @param      clean  [Boolean] Clean extra spaces and newlines in sources
+  ##
   def to_data(url: nil, clean: false)
     if key?(:body_links)
       {
@@ -23,22 +31,32 @@ class ::Hash
     end
   end
 
+  ##
+  ## Return the raw HTML of the object
+  ##
+  ## @return    [String] Html representation of the object.
+  ##
   def to_html
     if key?(:source)
       self[:source]
     end
   end
 
+  ##
+  ## Get a value from the hash using a dot-syntax query
+  ##
+  ## @param      query  [String] The query (dot notation)
+  ##
+  ## @return     [Object] result of querying the hash
+  ##
   def get_value(query)
     return nil if self.empty?
     stringify_keys!
 
     query.split('.').inject(self) do |v, k|
-      if v.is_a? Array
-        return v.map { |el| el.get_value(k) }
-      end
+      return v.map { |el| el.get_value(k) } if v.is_a? Array
       # k = k.to_i if v.is_a? Array
-      next unless v.key?(k)
+      next v unless v.key?(k)
 
       v.fetch(k)
     end
@@ -48,7 +66,7 @@ class ::Hash
   #
   # @param      path  [String] The path
   #
-  # @return     Result of path query
+  # @return     [Object] Result of path query
   #
   def dot_query(path, root = nil, full_tag: true)
     res = stringify_keys
@@ -63,7 +81,6 @@ class ::Hash
       "[#{inter}]"
     end
 
-    enumerate = false
     out = []
     q = path.split(/(?<![\d.])\./)
 
@@ -152,6 +169,14 @@ class ::Hash
     out
   end
 
+  ##
+  ## Test if values in an array match an operator
+  ##
+  ## @param      array [Array] The array
+  ## @param      key   [String] The key
+  ## @param      comp  [String] The comparison, e.g. *= or $=
+  ##
+  ## @return [Boolean] true if array contains match
   def array_match(array, key, comp)
     keep = false
     array.each do |el|
@@ -353,7 +378,32 @@ class ::Hash
     end
   end
 
+  ##
+  ## Destructive version of #stringify_keys
+  ##
+  ## @see        #stringify_keys
+  ##
   def stringify_keys!
     replace stringify_keys
   end
+
+  ##
+  ## Clean up empty arrays and return an array with one or
+  ## more elements
+  ##
+  ## @return     [Array] output array
+  ##
+  def clean_output
+    output = ensure_array
+    output.clean_output
+  end
+
+  ##
+  ## Ensure that an object is an array
+  ##
+  ## @return     [Array] object as Array
+  ##
+  def ensure_array
+    return [self]
+  end
 end

data/lib/curly/numeric.rb

@@ -0,0 +1,11 @@
+# Numeric helpers
+class ::Numeric
+  ##
+  ## Return an array version of self
+  ##
+  ## @return     [Array] self enclosed in an array
+  ##
+  def ensure_array
+    [self]
+  end
+end

data/lib/curly/string.rb

@@ -6,6 +6,11 @@
 ## @return     [String] cleaned string
 ##
 class ::String
+  ## Remove extra spaces and newlines, compress space
+  ## between tags
+  ##
+  ## @return     [String] cleaned string
+  ##
   def clean
     gsub(/[\t\n ]+/m, ' ').gsub(/> +</, '><')
   end
@@ -40,7 +45,7 @@ class ::String
   ##
   ## Convert an image type string to a symbol
   ##
-  ## @return     Symbol :srcset, :img, :opengraph, :all
+  ## @return     [Symbol] :srcset, :img, :opengraph, :all
   ##
   def normalize_image_type(default = :all)
     case self.to_s
@@ -58,7 +63,7 @@ class ::String
   ##
   ## Convert a browser type string to a symbol
   ##
-  ## @return     Symbol :chrome, :firefox
+  ## @return     [Symbol] :chrome, :firefox
   ##
   def normalize_browser_type(default = :none)
     case self.to_s
@@ -74,7 +79,7 @@ class ::String
   ##
   ## Convert a screenshot type string to a symbol
   ##
-  ## @return     Symbol :full_page, :print_page, :visible
+  ## @return     [Symbol] :full_page, :print_page, :visible
   ##
   def normalize_screenshot_type(default = :none)
     case self.to_s
@@ -88,4 +93,23 @@ class ::String
       default.is_a?(Symbol) ? default.to_sym : default.normalize_browser_type
     end
   end
+
+  ##
+  ## Clean up output and return a single-item array
+  ##
+  ## @return     [Array] output array
+  ##
+  def clean_output
+    output = ensure_array
+    output.clean_output
+  end
+
+  ##
+  ## Ensure that an object is an array
+  ##
+  ## @return     [Array] object as Array
+  ##
+  def ensure_array
+    return [self]
+  end
 end

data/lib/curly/version.rb

@@ -1,3 +1,5 @@
+# Top level module for CurlyQ
 module Curly
-  VERSION = '0.0.9'
+  # Current version number
+  VERSION = '0.0.10'
 end

data/lib/curly.rb

@@ -4,6 +4,7 @@ require 'curly/version'
 require 'curly/hash'
 require 'curly/string'
 require 'curly/array'
+require 'curly/numeric'
 require 'json'
 require 'yaml'
 require 'uri'

data/src/_README.md

@@ -13,7 +13,7 @@ _If you find this useful, feel free to [buy me some coffee][donate]._
 [jq]: https://github.com/jqlang/jq "Command-line JSON processor"
 [yq]: https://github.com/mikefarah/yq "yq is a portable command-line YAML, JSON, XML, CSV, TOML and properties processor"
 
-The current version of `curlyq` is <!--VER-->0.0.4<!--END VER-->.
+The current version of `curlyq` is <!--VER-->0.0.9<!--END VER-->.
 
 CurlyQ is a utility that provides a simple interface for curl, with additional features for things like extracting images and links, finding elements by CSS selector or XPath, getting detailed header info, and more. It's designed to be part of a scripting pipeline, outputting everything as structured data (JSON or YAML). It also has rudimentary support for making calls to JSON endpoints easier, but it's expected that you'll use something like [jq] to parse the output.
 

data/test/curlyq_headlinks_test.rb

@@ -17,8 +17,9 @@ class CurlyQHeadlinksTest < Test::Unit::TestCase
     result = curlyq('headlinks', '-q', '[rel=stylesheet]', 'https://brettterpstra.com')
     json = JSON.parse(result)
 
-    assert_match(/stylesheet/, json['rel'], 'Should have retrieved a single result with rel stylesheet')
-    assert_match(/screen\.\d+\.css$/, json['href'], 'Stylesheet should be correct primary stylesheet')
+    assert_equal(Array, json.class, 'Result should be an array')
+    assert_match(/stylesheet/, json[0]['rel'], 'Should have retrieved a single result with rel stylesheet')
+    assert_match(/screen\.\d+\.css$/, json[0]['href'], 'Stylesheet should be correct primary stylesheet')
   end
 
   def test_headlinks

data/test/curlyq_html_test.rb

@@ -14,12 +14,12 @@ class CurlyQHtmlTest < Test::Unit::TestCase
     result = curlyq('html', '-s', '#main article .aligncenter', '-q', 'images[1]', 'https://brettterpstra.com')
     json = JSON.parse(result)
 
-    assert_match(/aligncenter/, json['class'], 'Should have found an image with class "aligncenter"')
+    assert_match(/aligncenter/, json[0]['class'], 'Should have found an image with class "aligncenter"')
   end
 
   def test_html_query
     result = curlyq('html', '-q', 'meta.title', 'https://brettterpstra.com/2024/01/10/introducing-curlyq-a-pipeline-oriented-curl-helper/')
-
-    assert_match(/Introducing CurlyQ/, result, 'Should have retrived the page title')
+    json = JSON.parse(result)
+    assert_match(/Introducing CurlyQ/, json[0], 'Should have retrived the page title')
   end
 end

data/test/curlyq_scrape_test.rb

@@ -11,12 +11,42 @@ class CurlyQScrapeTest < Test::Unit::TestCase
   include CurlyQHelpers
 
   def setup
+    @screenshot = File.join(File.dirname(__FILE__), 'screenshot_test')
+    FileUtils.rm_f("#{@screenshot}.pdf") if File.exist?("#{@screenshot}.pdf")
+    FileUtils.rm_f('screenshot_test.png') if File.exist?("#{@screenshot}.png")
+    FileUtils.rm_f("#{@screenshot}_full.png") if File.exist?("#{@screenshot}_full.png")
   end
 
-  def test_scrape
+  def teardown
+    FileUtils.rm_f("#{@screenshot}.pdf") if File.exist?("#{@screenshot}.pdf")
+    FileUtils.rm_f('screenshot_test.png') if File.exist?("#{@screenshot}.png")
+    FileUtils.rm_f("#{@screenshot}_full.png") if File.exist?("#{@screenshot}_full.png")
+  end
+
+  def test_scrape_firefox
     result = curlyq('scrape', '-b', 'firefox', '-q', 'links[rel=me&content*=mastodon][0]', 'https://brettterpstra.com/2024/01/10/introducing-curlyq-a-pipeline-oriented-curl-helper/')
     json = JSON.parse(result)
 
-    assert_match(/Mastodon/, json['content'], 'Should have retrieved a Mastodon link')
+    assert_equal(Array, json.class, 'Result should be an Array')
+    assert_match(/Mastodon/, json[0]['content'], 'Should have retrieved a Mastodon link')
+  end
+
+  def test_scrape_chrome
+    result = curlyq('scrape', '-b', 'chrome', '-q', 'links[rel=me&content*=mastodon][0]', 'https://brettterpstra.com/2024/01/10/introducing-curlyq-a-pipeline-oriented-curl-helper/')
+    json = JSON.parse(result)
+
+    assert_equal(Array, json.class, 'Result should be an Array')
+    assert_match(/Mastodon/, json[0]['content'], 'Should have retrieved a Mastodon link')
+  end
+
+  def test_screenshot
+    curlyq('screenshot', '-b', 'firefox', '-o', @screenshot, '-t', 'print', 'https://brettterpstra.com')
+    assert(File.exist?("#{@screenshot}.pdf"), 'PDF Screenshot should exist')
+
+    curlyq('screenshot', '-b', 'chrome', '-o', @screenshot, '-t', 'visible', 'https://brettterpstra.com')
+    assert(File.exist?("#{@screenshot}.png"), 'PNG Screenshot should exist')
+
+    curlyq('screenshot', '-b', 'firefox', '-o', "#{@screenshot}_full", '-t', 'full', 'https://brettterpstra.com')
+    assert(File.exist?("#{@screenshot}_full.png"), 'PNG Screenshot should exist')
   end
 end

data/test/curlyq_tags_test.rb

@@ -14,18 +14,26 @@ class CurlyQTagsTest < Test::Unit::TestCase
   end
 
   def test_tags
-    result = curlyq('tags', '--search', '#main .post h3', '-q', 'attrs[id*=what]', 'https://brettterpstra.com/2024/01/10/introducing-curlyq-a-pipeline-oriented-curl-helper/')
+    result = curlyq('tags', '--search', '#main .post h3', 'https://brettterpstra.com/2024/01/10/introducing-curlyq-a-pipeline-oriented-curl-helper/')
     json = JSON.parse(result)
 
-    assert_equal(json.count, 1, 'Should have 1 result')
-    assert_match(/whats-next/, json[0]['attrs']['id'], 'Should have matched #whats-next')
+    assert_equal(Array, json.class, 'Should be an array of matches')
+    assert_equal(6, json.count, 'Should be six results')
   end
 
   def test_clean
     result = curlyq('tags', '--search', '#main section.related', '--clean', 'https://brettterpstra.com/2024/01/10/introducing-curlyq-a-pipeline-oriented-curl-helper/')
     json = JSON.parse(result)
 
-    assert_equal(json.count, 1, 'Should have 1 result')
+    assert_equal(Array, json.class, 'Should be a single Array')
+    assert_equal(1, json.count, 'Should be one element')
     assert_match(%r{Last.fm</h5></a></li>}, json[0]['source'], 'Should have matched #whats-next')
   end
+
+  def test_query
+    result = curlyq('tags', '--search', '#main .post h3', '-q', '[attrs.id*=what].source', 'https://brettterpstra.com/2024/01/10/introducing-curlyq-a-pipeline-oriented-curl-helper/')
+    json = JSON.parse(result)
+    assert_equal(Array, json.class, 'Should be an array')
+    assert_match(%r{^<h3 id="whats-next">What’s Next</h3>$}, json[0], 'Should have returned just source')
+  end
 end

data/test/helpers/curlyq-helpers.rb

@@ -1,5 +1,6 @@
 require 'open3'
 require 'time'
+require 'fileutils'
 $LOAD_PATH.unshift File.join(__dir__, '..', '..', 'lib')
 require 'curly'
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: curlyq
 version: !ruby/object:Gem::Version
-  version: 0.0.9
+  version: 0.0.10
 platform: ruby
 authors:
 - Brett Terpstra
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-01-16 00:00:00.000000000 Z
+date: 2024-01-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -236,6 +236,7 @@ files:
 - lib/curly/curl/html.rb
 - lib/curly/curl/json.rb
 - lib/curly/hash.rb
+- lib/curly/numeric.rb
 - lib/curly/string.rb
 - lib/curly/version.rb
 - src/_README.md