curlyq 0.0.4 → 0.0.6

data/bin/curlyq

@@ -71,7 +71,7 @@ command %i[html curl] do |c|
   c.switch %i[I info], negatable: false
 
   c.desc 'Regurn an array of matches to a CSS or XPath query'
-  c.flag %i[search]
+  c.flag %i[s search]
 
   c.desc 'Define a header to send as "key=value"'
   c.flag %i[h header], multiple: true
@@ -110,25 +110,31 @@ command %i[html curl] do |c|
     output = []
 
     urls.each do |url|
-      res = Curl::Html.new(url, { browser: options[:browser], fallback: options[:fallback],
-                                  headers: headers, headers_only: options[:info],
-                                  compressed: options[:compressed], clean: options[:clean],
-                                  ignore_local_links: options[:ignore_relative],
-                                  ignore_fragment_links: options[:ignore_fragments],
-                                  external_links_only: options[:external_links_only] })
+      curl_settings = { browser: options[:browser], fallback: options[:fallback],
+                        headers: headers, headers_only: options[:info],
+                        compressed: options[:compressed], clean: options[:clean],
+                        ignore_local_links: options[:ignore_relative],
+                        ignore_fragment_links: options[:ignore_fragments],
+                        external_links_only: options[:external_links_only] }
+      res = Curl::Html.new(url, curl_settings)
       res.curl
 
       if options[:info]
         output.push(res.headers)
-        # print_out(res.headers, global_options[:yaml], raw: options[:raw], pretty: global_options[:pretty])
         next
       end
 
       if options[:search]
-        out = res.search(options[:search])
+        source = res.search(options[:search], return_source: true)
 
-        out = out.dot_query(options[:query]) if options[:query]
-        output.push(out)
+        out = res.parse(source)
+
+        if options[:query]
+          out = out.to_data(url: url, clean: options[:clean]).dot_query(options[:query])
+        else
+          out = out.to_data
+        end
+        output.push([out])
       elsif options[:query]
         queried = res.to_data.dot_query(options[:query])
         output.push(queried) if queried
@@ -136,7 +142,7 @@ command %i[html curl] do |c|
         output.push(res.to_data(url: url))
       end
     end
-
+    output.delete_if(&:nil?)
     output.delete_if(&:empty?)
     output = output[0] if output.count == 1
     output.map! { |o| o[options[:raw].to_sym] } if options[:raw]
@@ -149,13 +155,13 @@ desc 'Save a screenshot of a URL'
 arg_name 'URL', multiple: true
 command :screenshot do |c|
   c.desc 'Type of screenshot to save (full (requires firefox), print, visible)'
-  c.flag %i[t type], type: ScreenshotType, must_match: /^[fpv].*?$/, default_value: 'full'
+  c.flag %i[t type], type: ScreenshotType, must_match: /^[fpv].*?$/, default_value: 'visible'
 
   c.desc 'Browser to use (firefox, chrome)'
   c.flag %i[b browser], type: BrowserType, must_match: /^[fc].*?$/, default_value: 'chrome'
 
   c.desc 'File destination'
-  c.flag %i[o out file]
+  c.flag %i[o out file], required: true
 
   c.desc 'Define a header to send as key=value'
   c.flag %i[h header], multiple: true
@@ -164,11 +170,19 @@ command :screenshot do |c|
     urls = args.join(' ').split(/[, ]+/)
     headers = break_headers(options[:header])
 
+    type = options[:type]
+    browser = options[:browser]
+
+    type = type.is_a?(Symbol) ? type : type.normalize_screenshot_type
+    browser = browser.is_a?(Symbol) ? browser : browser.normalize_browser_type
+
+    raise 'Full page screen shots only available with Firefox' if type == :full_page && browser != :firefox
+
     urls.each do |url|
       c = Curl::Html.new(url)
       c.headers = headers
-      c.browser = options[:browser]
-      c.screenshot(options[:out], type: options[:type])
+      c.browser = browser
+      c.screenshot(options[:out], type: type)
     end
   end
 end
@@ -208,12 +222,26 @@ command :json do |c|
           headers: res.headers
         })
       else
-        json = json.dot_query(options[:query]) if options[:query]
+        if options[:query]
+          if options[:query] =~ /^json$/
+            res = json
+          elsif options[:query] =~ /^json\./
+            query = options[:query].sub(/^json\./, '')
+          else
+            query = options[:query]
+          end
+
+          res = json.dot_query(query)
+        else
+          res = res.to_data
+        end
 
-        output.push(json)
+        output.push(res)
       end
     end
 
+    output = output[0] if output.count == 1
+
     print_out(output, global_options[:yaml], pretty: global_options[:pretty])
   end
 end
@@ -221,12 +249,18 @@ end
 desc 'Extract contents between two regular expressions'
 arg_name 'URL', multiple: true
 command :extract do |c|
-  c.desc 'Text before extraction, parsed as regex'
+  c.desc 'Text before extraction'
   c.flag %i[b before]
 
-  c.desc 'Text after extraction, parsed as regex'
+  c.desc 'Text after extraction'
   c.flag %i[a after]
 
+  c.desc 'Process before/after strings as regular expressions'
+  c.switch %i[r regex]
+
+  c.desc 'Include the before/after matches in the result'
+  c.switch %i[i include]
+
   c.desc 'Define a header to send as key=value'
   c.flag %i[h header], multiple: true
 
@@ -249,7 +283,15 @@ command :extract do |c|
       res = Curl::Html.new(url, { headers: headers, headers_only: false,
                                   compressed: options[:compressed], clean: options[:clean] })
       res.curl
-      extracted = res.extract(options[:before], options[:after])
+      if options[:regex]
+        before = Regexp.new(options[:before])
+        after = Regexp.new(options[:after])
+      else
+        before = /#{Regexp.escape(options[:before])}/
+        after = /#{Regexp.escape(options[:after])}/
+      end
+
+      extracted = res.extract(before, after, inclusive: options[:include])
       extracted.strip_tags! if options[:strip]
       output.concat(extracted)
     end
@@ -262,10 +304,10 @@ desc 'Extract all instances of a tag'
 arg_name 'URL', multiple: true
 command :tags do |c|
   c.desc 'Define a header to send as key=value'
-  c.flag %i[h header], multiple: true
+  c.flag %i[h header], multiple: true, arg_name: 'KEY=VAL'
 
   c.desc 'Specify a tag to collect'
-  c.flag %i[t tag], multiple: true
+  c.flag %i[t tag], multiple: true, arg_name: 'TAG'
 
   c.desc 'Expect compressed results'
   c.switch %i[c compressed]
@@ -273,8 +315,14 @@ command :tags do |c|
   c.desc 'Remove extra whitespace from results'
   c.switch %i[clean]
 
-  c.desc 'CSS/XPath query'
-  c.flag %i[q query search]
+  c.desc 'Output the HTML source of the results'
+  c.switch %i[source html]
+
+  c.desc 'Dot syntax query to filter results'
+  c.flag %i[q query filter], arg_name: 'DOT_SYNTAX'
+
+  c.desc 'Regurn an array of matches to a CSS or XPath query'
+  c.flag %i[search], arg_name: 'CSS/XPATH'
 
   c.action do |global_options, options, args|
     urls = args.join(' ').split(/[, ]+/)
@@ -286,9 +334,17 @@ command :tags do |c|
       res = Curl::Html.new(url, { headers: headers, headers_only: options[:headers],
                                   compressed: options[:compressed], clean: options[:clean] })
       res.curl
+
       output = []
       if options[:search]
-        output = res.tags.search(options[:search])
+        out = res.search(options[:search])
+
+        out = out.dot_query(options[:query]) if options[:query]
+        output.push(out)
+      elsif options[:query]
+        query = options[:query] =~ /^links/ ? options[:query] : "links#{options[:query]}"
+
+        output = res.to_data.dot_query(query)
       elsif tags.count.positive?
         tags.each { |tag| output.concat(res.tags(tag)) }
       else
@@ -296,7 +352,13 @@ command :tags do |c|
       end
     end
 
-    print_out(output, global_options[:yaml], pretty: global_options[:pretty])
+    output = output[0] if output.count == 1
+
+    if options[:source]
+      puts output.to_html
+    else
+      print_out(output, global_options[:yaml], pretty: global_options[:pretty])
+    end
   end
 end
 
@@ -312,6 +374,9 @@ command :images do |c|
   c.desc 'Remove extra whitespace from results'
   c.switch %i[clean]
 
+  c.desc 'Filter output using dot-syntax path'
+  c.flag %i[q query filter]
+
   c.desc 'Define a header to send as key=value'
   c.flag %i[h header], multiple: true
 
@@ -326,7 +391,15 @@ command :images do |c|
     urls.each do |url|
       res = Curl::Html.new(url, { compressed: options[:compressed], clean: options[:clean] })
       res.curl
-      output.concat(res.images(types: types))
+
+      res = res.images(types: types)
+
+      if options[:query]
+        query = options[:query] =~ /^images/ ? options[:query] : "images#{options[:query]}"
+        res = { images: res }.dot_query(query)
+      end
+
+      output.concat(res)
     end
 
     print_out(output, global_options[:yaml], pretty: global_options[:pretty])
@@ -367,7 +440,7 @@ command :links do |c|
 
       if options[:query]
         query = options[:query] =~ /^links/ ? options[:query] : "links#{options[:query]}"
-        queried = { links: res.to_data[:links] }.dot_query(query)
+        queried = res.to_data.dot_query(query)
         output.concat(queried) if queried
       else
         output.concat(res.body_links)
@@ -414,7 +487,7 @@ desc %(Scrape a page using a web browser, for dynamic (JS) pages. Be sure to hav
 arg_name 'URL', multiple: true
 command :scrape do |c|
   c.desc 'Browser to use (firefox, chrome)'
-  c.flag %i[b browser], type: BrowserType
+  c.flag %i[b browser], type: BrowserType, required: true
 
   c.desc 'Regurn an array of matches to a CSS or XPath query'
   c.flag %i[search]
@@ -437,30 +510,19 @@ command :scrape do |c|
     output = []
 
     urls.each do |url|
-      driver = Selenium::WebDriver.for options[:browser]
-      begin
-        driver.get url
-        res = driver.page_source
-
-        res = Curl::Html.new(nil, { source: res, clean: options[:clean] })
-        res.curl
-        if options[:search]
-          out = res.search(options[:search])
-
-          out = out.dot_query(options[:query]) if options[:query]
-          output.push(out)
-        elsif options[:query]
-          queried = res.to_data(url: url).dot_query(options[:query])
-          output = queried if queried
-        else
-          output.push(res.to_data(url: url))
-        end
+      res = Curl::Html.new(url, { browser: options[:browser], clean: options[:clean] })
+      res.curl
 
-        # elements = driver.find_elements(css: options[:query])
+      if options[:search]
+        out = res.search(options[:search])
 
-        # elements.each { |e| output.push(e.text.strip) }
-      ensure
-        driver.quit
+        out = out.dot_query(options[:query]) if options[:query]
+        output.push(out)
+      elsif options[:query]
+        queried = res.to_data(url: url).dot_query(options[:query])
+        output.push(queried) if queried
+      else
+        output.push(res.to_data(url: url))
       end
     end
 

data/curlyq.gemspec

@@ -16,10 +16,14 @@ spec = Gem::Specification.new do |s|
   s.rdoc_options << '--title' << 'curlyq' << '--main' << 'README.rdoc' << '-ri'
   s.bindir = 'bin'
   s.executables << 'curlyq'
-  s.add_development_dependency('rake','~> 0.9.2')
-  s.add_development_dependency('rdoc', '~> 4.3')
-  s.add_development_dependency('minitest', '~> 5.14')
+  s.add_development_dependency('rake','~> 13.0', '>= 13.0.1')
+  s.add_development_dependency('rdoc', '~> 6.3.1')
+  s.add_development_dependency('test-unit', '~> 3.4.4')
   s.add_development_dependency('yard', '~> 0.9', '>= 0.9.26')
+  s.add_development_dependency('tty-spinner', '~> 0.9', '>= 0.9.3')
+  s.add_development_dependency('tty-progressbar', '~> 0.18', '>= 0.18.2')
+  s.add_development_dependency('pastel', '~> 0.8.0')
+  s.add_development_dependency('parallel_tests', '~> 3.7', '>= 3.7.3')
   s.add_runtime_dependency('gli','~> 2.21.0')
   s.add_runtime_dependency('tty-which','~> 0.5.0')
   s.add_runtime_dependency('nokogiri','~> 1.16.0')

data/lib/curly/array.rb

@@ -66,69 +66,94 @@ class ::Array
     replace dedup_links
   end
 
+  #---------------------------------------------------------
+  ## Run a query on array elements
   ##
-  ## Convert and execute a dot-syntax query on the array
+  ## @param      path [String] dot.syntax path to compare
   ##
-  ## @param      path  [String]  The dot-syntax path
-  ##
-  ## @return     [Array] Matching elements
+  ## @return [Array] elements matching dot query
   ##
   def dot_query(path)
-    output = []
-    if path =~ /^\[([\d+.])\]\.?/
-      int = Regexp.last_match(1)
-      path.sub!(/^\[[\d.]+\]\.?/, '')
-      items = self[eval(int)]
-    else
-      items = self
+    filter! do |tag|
+      r = tag.dot_query(path)
+      if r.is_a?(Array)
+        r.count.positive?
+      else
+        r
+      end
     end
 
-    if items.is_a? Hash
-      output = items.dot_query(path)
-    else
-      items.each do |item|
-        res = item.is_a?(Hash) ? item.stringify_keys : item
-        out = []
-        q = path.split(/(?<![\d.])\./)
-        q.each do |pth|
-          el = Regexp.last_match(1) if pth =~ /\[([0-9,.]+)\]/
-          pth.sub!(/\[([0-9,.]+)\]/, '')
-          ats = []
-          at = []
-          while pth =~ /\[[+&,]?\w+ *[\^*$=<>]=? *\w+/
-            m = pth.match(/\[(?<com>[,+&])? *(?<key>\w+) *(?<op>[\^*$=<>]{1,2}) *(?<val>\w+) */)
-            comp = [m['key'], m['op'], m['val']]
-            case m['com']
-            when ','
-              ats.push(comp)
-              at = []
-            else
-              at.push(comp)
-            end
-
-            pth.sub!(/\[(?<com>[,&+])? *(?<key>\w+) *(?<op>[\^*$=<>]{1,2}) *(?<val>\w+)/, '[')
-          end
-          ats.push(at) unless at.empty?
-          pth.sub!(/\[\]/, '')
-
-          return false if el.nil? && ats.empty? && !res.key?(pth)
-
-          res = res[pth] unless pth.empty?
-
-          while ats.count.positive?
-            atr = ats.shift
-
-            keepers = res.filter do |r|
-              evaluate_comp(r, atr)
-            end
-            out.concat(keepers)
-          end
-
-          out = out[eval(el)] if out.is_a?(Array) && el =~ /^[\d.,]+$/
+    return self
+  end
+
+  def to_html
+    map { |el| el.to_html }
+  end
+
+  ##
+  ## Test if a tag contains an attribute matching filter queries
+  ##
+  ## @param      tag_name    [String] The tag name
+  ## @param      classes     [String] The classes to match
+  ## @param      id          [String] The id attribute to
+  ##                         match
+  ## @param      attribute   [String] The attribute
+  ## @param      operator    [String] The operator, <>= *=
+  ##                         $= ^=
+  ## @param      value       [String] The value to match
+  ## @param      descendant  [Boolean] Check descendant tags
+  ##
+  def tag_match(tag_name, classes, id, attribute, operator, value, descendant: false)
+    tag = self
+    keep = true
+
+    keep = false if tag_name && !tag['tag'] =~ /^#{tag_name}$/i
+
+    if tag.key?('attrs') && tag['attrs']
+      if keep && id
+        tag_id = tag['attrs'].filter { |a| a['key'] == 'id' }.first['value']
+        keep = tag_id && tag_id =~ /#{id}/i
+      end
+
+      if keep && classes
+        cls = tag['attrs'].filter { |a| a['key'] == 'class' }.first
+        if cls
+          all = true
+          classes.each { |c| all = cls['value'].include?(c) }
+          keep = all
+        else
+          keep = false
         end
-        output.push(out)
       end
+
+      if keep && attribute
+        attributes = tag['attrs'].filter { |a| a['key'] =~ /^#{attribute}$/i }
+        any = false
+        attributes.each do |a|
+          break if any
+
+          any = case operator
+                when /^*/
+                  a['value'] =~ /#{value}/i
+                when /^\^/
+                  a['value'] =~ /^#{value}/i
+                when /^\$/
+                  a['value'] =~ /#{value}$/i
+                else
+                  a['value'] =~ /^#{value}$/i
+                end
+        end
+        keep = any
+      end
+    end
+
+    return false if descendant && !keep
+
+    if !descendant && tag.key?('tags')
+      tags = tag['tags'].filter { |t| t.tag_match(tag_name, classes, id, attribute, operator, value) }
+      tags.count.positive?
+    else
+      keep
     end
-    output
   end
 end