curlyq 0.0.7 → 0.0.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 92b27e3065435d17fd5d6129bd640481dee8acf66f9ed82b2b68ae6a7589f463
-  data.tar.gz: a5cd5299248fd01d8f12a80a1c9982de4f8e0160ea5033ab96b60677bb9ea2c8
+  metadata.gz: d3e32b382d7318b067ee3fb22f2e9057cf6aa9facfac41c74a0ebb5d4fb4743d
+  data.tar.gz: d379da3f0db621052e61230356f5c58b587eefccbb0a4c997216516a4159b44a
 SHA512:
-  metadata.gz: 1348b97fdf89faf44cd0cfc0f2aecc05a679606f19fe57392d588209da26fc3a5c2407569173d41e1497bdf409d762adee0d2533089b5ce27854e298fe98cc13
-  data.tar.gz: c80ecd381e1d941d8e8e5ead0dd925682e26a2f8cc639f0202d5b5cd30f025582fc5d5a2665daca3e5e7d0f2099066d3ca3d9ad8ad00b07d2ee5673b122bae01
+  metadata.gz: ae63654deb943771e5f6f3aa0f6a037b1015336abbd696a8ce77acc22f361a3b6a18b03f3b7d02e5c7d5dcaa8d3608248bed240679acfce22ba2e462d84b529f
+  data.tar.gz: 481f8499e45a65cb3981fcf20ef7fc9f01f97a1b7014c6566aa2f3bf7a6611fd2d5d35f78e742e4063eea192b938c0642f0ca764e5032f330778d2815a191a41

data/CHANGELOG.md

@@ -1,3 +1,13 @@
+### 0.0.8
+
+2024-01-15 16:45
+
+#### IMPROVED
+
+- Dot syntax query can now operate on a full array using empty set []
+- Dot syntax query should output a specific key, e.g. attrs[id*=news].content (work in progress)
+- Dot query syntax handling touch-ups. Piping to jq is still more flexible, but the basics are there.
+
 ### 0.0.7
 
 2024-01-12 17:03

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    curlyq (0.0.7)
+    curlyq (0.0.8)
       gli (~> 2.21.0)
       nokogiri (~> 1.16.0)
       selenium-webdriver (~> 4.16.0)

data/README.md

@@ -10,7 +10,7 @@ _If you find this useful, feel free to [buy me some coffee][donate]._
 [donate]: https://brettterpstra.com/donate
 
 
-The current version of `curlyq` is 0.0.7
+The current version of `curlyq` is 0.0.8
 .
 
 CurlyQ is a utility that provides a simple interface for curl, with additional features for things like extracting images and links, finding elements by CSS selector or XPath, getting detailed header info, and more. It's designed to be part of a scripting pipeline, outputting everything as structured data (JSON or YAML). It also has rudimentary support for making calls to JSON endpoints easier, but it's expected that you'll use something like `jq` to parse the output.
@@ -44,7 +44,7 @@ SYNOPSIS
     curlyq [global options] command [command options] [arguments...]
 
 VERSION
-    0.0.7
+    0.0.8
 
 GLOBAL OPTIONS
     --help          - Show this message

data/bin/curlyq

@@ -147,6 +147,12 @@ command %i[html curl] do |c|
     # output = output[0] if output.count == 1
     output.map! { |o| o[options[:raw].to_sym] } if options[:raw]
 
+    if output.is_a?(Array)
+      while output.length == 1
+        output = output[0]
+      end
+    end
+
     print_out(output, global_options[:yaml], raw: options[:raw], pretty: global_options[:pretty])
   end
 end
@@ -342,9 +348,7 @@ command :tags do |c|
         out = out.dot_query(options[:query]) if options[:query]
         output.push(out)
       elsif options[:query]
-        query = options[:query] =~ /^links/ ? options[:query] : "links#{options[:query]}"
-
-        output = res.to_data.dot_query(query)
+        output = res.to_data.dot_query(options[:query])
       elsif tags.count.positive?
         tags.each { |tag| output.concat(res.tags(tag)) }
       else
@@ -393,13 +397,13 @@ command :images do |c|
       res.curl
 
       res = res.images(types: types)
+      res = { images: res }.dot_query(options[:query], 'images', full_tag: false) if options[:query]
 
-      if options[:query]
-        query = options[:query] =~ /^images/ ? options[:query] : "images#{options[:query]}"
-        res = { images: res }.dot_query(query)
+      if res.is_a?(Array)
+        output.concat(res)
+      else
+        output.push(res)
       end
-
-      output.concat(res)
     end
 
     print_out(output, global_options[:yaml], pretty: global_options[:pretty])
@@ -439,9 +443,9 @@ command :links do |c|
       res.curl
 
       if options[:query]
-        query = options[:query] =~ /^links/ ? options[:query] : "links#{options[:query]}"
-        queried = res.to_data.dot_query(query)
-        output.concat(queried) if queried
+        queried = res.to_data.dot_query(options[:query], 'links', full_tag: false)
+
+        queried.is_a?(Array) ? output.concat(queried) : output.push(queried) if queried
       else
         output.concat(res.body_links)
       end
@@ -469,9 +473,8 @@ command :headlinks do |c|
       res.curl
 
       if options[:query]
-        query = options[:query] =~ /^links/ ? options[:query] : "links#{options[:query]}"
-        queried = { links: res.to_data[:meta_links] }.dot_query(query)
-        output.concat(queried) if queried
+        queried = { links: res.to_data[:meta_links] }.dot_query(options[:query], 'links', full_tag: false)
+        output.push(queried) if queried
       else
         output.push(res.to_data[:meta_links])
       end
@@ -516,10 +519,10 @@ command :scrape do |c|
       if options[:search]
         out = res.search(options[:search])
 
-        out = out.dot_query(options[:query]) if options[:query]
+        out = out.dot_query(options[:query], full_tag: false) if options[:query]
         output.push(out)
       elsif options[:query]
-        queried = res.to_data(url: url).dot_query(options[:query])
+        queried = res.to_data(url: url).dot_query(options[:query], full_tag: false)
         output.push(queried) if queried
       else
         output.push(res.to_data(url: url))

data/lib/curly/array.rb

@@ -74,20 +74,19 @@ class ::Array
   ## @return [Array] elements matching dot query
   ##
   def dot_query(path)
-    filter! do |tag|
-      r = tag.dot_query(path)
-      if r.is_a?(Array)
-        r.count.positive?
-      else
-        r
-      end
-    end
+    res = map { |el| el.dot_query(path) }
+    res.delete_if { |r| !r }
+    res.delete_if(&:empty?)
+    res
+  end
 
-    return self
+  def get_value(path)
+    res = map { |el| el.get_value(path) }
+    res.is_a?(Array) && res.count == 1 ? res[0] : res
   end
 
   def to_html
-    map { |el| el.to_html }
+    map(&:to_html)
   end
 
   ##

data/lib/curly/hash.rb

@@ -29,24 +29,50 @@ class ::Hash
     end
   end
 
+  def get_value(query)
+    return nil if self.empty?
+    query.split('.').inject(self) do |v, k|
+      k = k.to_i if v.is_a? Array
+      next unless v.key?(k)
+      v.fetch(k)
+    end
+  end
+
   # Extract data using a dot-syntax path
   #
   # @param      path  [String] The path
   #
   # @return     Result of path query
   #
-  def dot_query(path)
+  def dot_query(path, root = nil, full_tag: true)
     res = stringify_keys
+    res = res[root] unless root.nil?
 
+    unless path =~ /\[/
+      return res.get_value(path)
+    end
+
+    enumerate = false
     out = []
     q = path.split(/(?<![\d.])\./)
-    q.each do |pth|
-      el = Regexp.last_match(1) if pth =~ /\[([0-9,.]+)\]/
-      pth.sub!(/\[([0-9,.]+)\]/, '')
+
+    while q.count.positive?
+      pth = q.shift
+
+      return nil if res.nil?
+
+      unless pth =~ /\[/
+        return res.get_value(pth)
+      end
+
+      el = Regexp.last_match(1) if pth =~ /\[([0-9,.]+)?\]/
+      pth.sub!(/\[([0-9,.]+)?\]/, '')
+
       ats = []
       at = []
-      while pth =~ /\[[+&,]?\w+ *[\^*$=<>]=? *\w+/
-        m = pth.match(/\[(?<com>[,+&])? *(?<key>\w+) *(?<op>[\^*$=<>]{1,2}) *(?<val>[^,&\]]+) */)
+      while pth =~ /\[[+&,]?\w+( *[\^*$=<>]=? *\w+)?/
+        m = pth.match(/\[(?<com>[,+&])? *(?<key>\w+)( *(?<op>[\^*$=<>]{1,2}) *(?<val>[^,&\]]+))? */)
+
         comp = [m['key'], m['op'], m['val']]
         case m['com']
         when ','
@@ -56,36 +82,61 @@ class ::Hash
           at.push(comp)
         end
 
-        pth.sub!(/\[(?<com>[,&+])? *(?<key>\w+) *(?<op>[\^*$=<>]{1,2}) *(?<val>[^,&\]]+)/, '[')
+        pth.sub!(/\[(?<com>[,&+])? *(?<key>\w+)( *(?<op>[\^*$=<>]{1,2}) *(?<val>[^,&\]]+))?/, '[')
       end
       ats.push(at) unless at.empty?
       pth.sub!(/\[\]/, '')
 
-      res = res[0] if res.is_a?(Array)
+      res = res[0] if res.is_a?(Array) && res.count == 1
+      if ats.empty? && el.nil? && res.is_a?(Array) && res[0]&.key?(pth)
+        res.map! { |r| r[pth] }
+        next
+      end
 
-      return false if el.nil? && ats.empty? && !res.key?(pth)
+      res.map!(&:stringify_keys) if res.is_a?(Array) && res[0].is_a?(Hash)
+      # if res.is_a?(String) || (res.is_a?(Array) && res[0].is_a?(String))
+      #   out.push(res)
+      #   next
+      # end
 
-      res = res[pth] unless pth.empty?
+      # if res.is_a?(Array) && !pth.nil?
+      #   return res.delete_if { |r| !r.key?(pth) }
+      # else
+      #   return false if el.nil? && ats.empty? && res.is_a?(Hash) && (res.nil? || !res.key?(pth))
+      # end
+      tag = res
+      res = res[pth] unless pth.nil? || pth.empty?
 
-      return false if res.nil?
+      pth = ''
 
+      return false if res.nil?
       if ats.count.positive?
         while ats.count.positive?
           atr = ats.shift
           res = [res] if res.is_a?(Hash)
-          keepers = res.filter do |r|
-            evaluate_comp(r, atr)
-          end
 
-          out.concat(keepers)
+          res.each do |r|
+            out.push(full_tag ? tag : r) if evaluate_comp(r, atr)
+          end
         end
       else
         out = res
       end
 
-      out = out[eval(el)] if out.is_a?(Array) && el =~ /^[\d.,]+$/
+      out = out.get_value(pth) unless pth.nil?
+
+      if el.nil? && out.is_a?(Array) && out[0].is_a?(Hash)
+        out.map! { |o|
+          o.stringify_keys
+          # o.key?(pth) ? o[pth] : o
+        }
+      elsif out.is_a?(Array) && el =~ /^[\d.,]+$/
+        out = out[eval(el)]
+      end
+      res = out
     end
 
+    out = out[0] if out&.count == 1
     out
   end
 
@@ -112,6 +163,9 @@ class ::Hash
             else
               a[2]
             end
+      r = r.get_value(key.to_s) if key.to_s =~ /\./
+
+      return r.key?(key) && !r[key].nil? && !r[key].empty? if val.nil?
 
       if !r.key?(key)
         keep = false
@@ -251,4 +305,8 @@ class ::Hash
       hsh[k.to_s] = v.is_a?(Hash) ? v.stringify_keys : v
     end
   end
+
+  def stringify_keys!
+    replace stringify_keys
+  end
 end

data/lib/curly/version.rb

@@ -1,3 +1,3 @@
 module Curly
-  VERSION = '0.0.7'
+  VERSION = '0.0.8'
 end

data/src/_README.md

@@ -10,7 +10,7 @@ _If you find this useful, feel free to [buy me some coffee][donate]._
 [donate]: https://brettterpstra.com/donate
 <!--END GITHUB-->
 
-The current version of `curlyq` is <!--VER-->0.0.6<!--END VER-->.
+The current version of `curlyq` is <!--VER-->0.0.4<!--END VER-->.
 
 CurlyQ is a utility that provides a simple interface for curl, with additional features for things like extracting images and links, finding elements by CSS selector or XPath, getting detailed header info, and more. It's designed to be part of a scripting pipeline, outputting everything as structured data (JSON or YAML). It also has rudimentary support for making calls to JSON endpoints easier, but it's expected that you'll use something like `jq` to parse the output.
 

data/test/curlyq_extract_test.rb

@@ -7,7 +7,7 @@ require 'helpers/curlyq-helpers'
 require 'test_helper'
 
 # Tests for tags command
-class CurlyQTagsTest < Test::Unit::TestCase
+class CurlyQExtractTest < Test::Unit::TestCase
   include CurlyQHelpers
 
   def setup

data/test/curlyq_html_test.rb

@@ -12,9 +12,9 @@ class CurlyQHtmlTest < Test::Unit::TestCase
 
   def test_html_search_query
     result = curlyq('html', '-s', '#main article .aligncenter', '-q', 'images[1]', 'https://brettterpstra.com')
-    json = JSON.parse(result)[0]
+    json = JSON.parse(result)
 
-    assert_match(/aligncenter/, json[0]['class'], 'Should have found an image with class "aligncenter"')
+    assert_match(/aligncenter/, json['class'], 'Should have found an image with class "aligncenter"')
   end
 
   def test_html_query

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: curlyq
 version: !ruby/object:Gem::Version
-  version: 0.0.7
+  version: 0.0.8
 platform: ruby
 authors:
 - Brett Terpstra
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-01-12 00:00:00.000000000 Z
+date: 2024-01-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake