curlyq 0.0.2

data/lib/curly/curl/json.rb

@@ -0,0 +1,108 @@
+# frozen_string_literal: true
+
+module Curl
+  # Class for CURLing a JSON response
+  class Json
+    attr_reader :url, :code, :json, :headers
+
+    def to_data
+      {
+        url: @url,
+        code: @code,
+        json: @json,
+        headers: @headers
+      }
+    end
+
+    ##
+    ## Create a new Curl::Json page object
+    ##
+    ## @param      url         [String] The url to curl
+    ## @param      headers     [Hash] The headers to send
+    ## @param      compressed  [Boolean] Expect compressed results
+    ##
+    ## @return     [Curl::Json] Curl::Json object with url, code, parsed json, and response headers
+    ##
+    def initialize(url, headers: nil, compressed: false, symbolize_names: false)
+      @curl = TTY::Which.which('curl')
+      page = curl_json(url, headers: headers, compressed: compressed, symbolize_names: symbolize_names)
+
+      raise "Error retrieving #{url}" if page.nil? || page.empty?
+
+      @url = page[:url]
+      @code = page[:code]
+      @json = page[:json]
+      @headers = page[:headers]
+    end
+
+    def path(path, json = @json)
+      parts = path.split(/./)
+      target = json
+      parts.each do |part|
+        if part =~ /(?<key>[^\[]+)\[(?<int>\d+)\]/
+          target = target[key][int.to_i]
+        else
+          target = target[part]
+        end
+      end
+
+      target
+    end
+
+    private
+
+    ##
+    ## Curl the JSON contents
+    ##
+    ## @param      url         [String] The url
+    ## @param      headers     [Hash] The headers to send
+    ## @param      compressed  [Boolean] Expect compressed results
+    ##
+    ## @return     [Hash] hash of url, code, headers, and parsed json
+    ##
+    def curl_json(url, headers: nil, compressed: false, symbolize_names: false)
+      flags = 'SsLi'
+      agents = [
+        'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.6 Safari/605.1.1',
+        'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:121.0) Gecko/20100101 Firefox/121.',
+        'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.3',
+        'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36 Edg/120.0.0.'
+      ]
+
+      headers = headers.nil? ? '' : headers.map { |h, v| %(-H "#{h}: #{v}") }.join(' ')
+      compress = compressed ? '--compressed' : ''
+      source = `#{@curl} -#{flags} #{compress} #{headers} '#{url}' 2>/dev/null`
+      agent = 0
+      while source.nil? || source.empty?
+        source = `#{@curl} -#{flags} #{compress} -A "#{agents[agent]}" #{headers} '#{url}' 2>/dev/null`
+        break if agent >= agents.count - 1
+      end
+
+      return false if source.nil? || source.empty?
+
+      source.strip!
+
+      headers = {}
+      lines = source.split(/\r\n/)
+      code = lines[0].match(/(\d\d\d)/)[1]
+      lines.shift
+      lines.each_with_index do |line, idx|
+        if line =~ /^([\w-]+): (.*?)$/
+          m = Regexp.last_match
+          headers[m[1]] = m[2]
+        else
+          source = lines[idx..].join("\n")
+          break
+        end
+      end
+
+      json = source.strip.force_encoding('utf-8')
+      begin
+        json.gsub!(/[\u{1F600}-\u{1F6FF}]/, '')
+        { url: url, code: code, headers: headers, json: JSON.parse(json, symbolize_names: symbolize_names) }
+      rescue StandardError => e
+        { url: url, code: code, headers: headers, json: nil}
+      end
+    end
+  end
+end

data/lib/curly/curl.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+# import
+require_relative 'curl/html'
+
+# import
+require_relative 'curl/json'

data/lib/curly/hash.rb

@@ -0,0 +1,200 @@
+# frozen_string_literal: true
+
+# Hash helpers
+class ::Hash
+  # Extract data using a dot-syntax path
+  #
+  # @param      path  [String] The path
+  #
+  # @return     Result of path query
+  #
+  def dot_query(path)
+    res = stringify_keys
+    out = []
+    q = path.split(/(?<![\d.])\./)
+    q.each do |pth|
+      el = Regexp.last_match(1) if pth =~ /\[([0-9,.]+)\]/
+      pth.sub!(/\[([0-9,.]+)\]/, '')
+      ats = []
+      at = []
+      while pth =~ /\[[+&,]?\w+ *[\^*$=<>]=? *\w+/
+        m = pth.match(/\[(?<com>[,+&])? *(?<key>\w+) *(?<op>[\^*$=<>]{1,2}) *(?<val>\w+) */)
+        comp = [m['key'], m['op'], m['val']]
+        case m['com']
+        when ','
+          ats.push(comp)
+          at = []
+        else
+          at.push(comp)
+        end
+
+        pth.sub!(/\[(?<com>[,&+])? *(?<key>\w+) *(?<op>[\^*$=<>]{1,2}) *(?<val>\w+)/, '[')
+      end
+      ats.push(at) unless at.empty?
+      pth.sub!(/\[\]/, '')
+
+      return false if el.nil? && ats.empty? && !res.key?(pth)
+
+      res = res[pth] unless pth.empty?
+
+      if ats.count.positive?
+        while ats.count.positive?
+          atr = ats.shift
+
+          keepers = res.filter do |r|
+            evaluate_comp(r, atr)
+          end
+          out.concat(keepers)
+        end
+      else
+        out = res
+      end
+
+      out = out[eval(el)] if out.is_a?(Array) && el =~ /^[\d.,]+$/
+    end
+    out
+  end
+
+  ##
+  ## Evaluate a comparison
+  ##
+  ## @param      r     [Hash] hash of source elements and
+  ##                   comparison operators
+  ## @param      atr   [String] The attribute to compare
+  ##
+  ## @return     [Boolean] whether the comparison passes or fails
+  ##
+  def evaluate_comp(r, atr)
+    keep = true
+
+    atr.each do |a|
+      key = a[0].to_sym
+      val = if a[2] =~ /^\d+$/
+              a[2].to_i
+            elsif a[2] =~ /^\d+\.\d+$/
+              a[2].to_f
+            else
+              a[2]
+            end
+
+      if !r.key?(key)
+        keep = false
+      elsif r[key].is_a?(Array)
+        valid = r[key].filter do |k|
+          case a[1]
+          when /^\^/
+            k =~ /^#{a[2]}/i ? true : false
+          when /^\$/
+            k =~ /#{a[2]}$/i ? true : false
+          when /^\*/
+            k =~ /#{a[2]}/i ? true : false
+          else
+            k =~ /^#{a[2]}$/i ? true : false
+          end
+        end
+
+        keep = valid.count.positive?
+      elsif val.is_a?(Numeric) && a[1] =~ /^[<>=]{1,2}$/
+        k = r[key].to_i
+        comp = a[1] =~ /^=$/ ? '==' : a[1]
+        keep = eval("#{k}#{comp}#{val}")
+      else
+        keep = case a[1]
+               when /^\^/
+                 r[key] =~ /^#{a[2]}/i ? true : false
+               when /^\$/
+                 r[key] =~ /#{a[2]}$/i ? true : false
+               when /^\*/
+                 r[key] =~ /#{a[2]}/i ? true : false
+               else
+                 r[key] =~ /^#{a[2]}$/i ? true : false
+               end
+      end
+    end
+
+    keep
+  end
+
+  ##
+  ## Test if a hash contains a tag matching filter queries
+  ##
+  ## @param      tag_name    [String] The tag name
+  ## @param      classes     [String] The classes to match
+  ## @param      id          [String] The id attribute to
+  ##                         match
+  ## @param      attribute   [String] The attribute
+  ## @param      operator    [String] The operator, <>= *=
+  ##                         $= ^=
+  ## @param      value       [String] The value to match
+  ## @param      descendant  [Boolean] Check descendant tags
+  ##
+  def tag_match(tag_name, classes, id, attribute, operator, value, descendant: false)
+    tag = self
+    keep = true
+
+    keep = false if tag_name && !tag['tag'] =~ /^#{tag_name}$/i
+
+    if tag.key?('attrs') && tag['attrs']
+      if keep && id
+        tag_id = tag['attrs'].filter { |a| a['key'] == 'id' }.first['value']
+        keep = tag_id && tag_id =~ /#{id}/i
+      end
+
+      if keep && classes
+        cls = tag['attrs'].filter { |a| a['key'] == 'class' }.first
+        if cls
+          all = true
+          classes.each { |c| all = cls['value'].include?(c) }
+          keep = all
+        else
+          keep = false
+        end
+      end
+
+      if keep && attribute
+        attributes = tag['attrs'].filter { |a| a['key'] =~ /^#{attribute}$/i }
+        any = false
+        attributes.each do |a|
+          break if any
+
+          any = case operator
+                when /^*/
+                  a['value'] =~ /#{value}/i
+                when /^\^/
+                  a['value'] =~ /^#{value}/i
+                when /^\$/
+                  a['value'] =~ /#{value}$/i
+                else
+                  a['value'] =~ /^#{value}$/i
+                end
+        end
+        keep = any
+      end
+    end
+
+    return false if descendant && !keep
+
+    if !descendant && tag.key?('tags')
+      tags = tag['tags'].filter { |t| t.tag_match(tag_name, classes, id, attribute, operator, value) }
+      tags.count.positive?
+    else
+      keep
+    end
+  end
+
+  # Turn all keys into string
+  #
+  # If the hash has both a string and a symbol for key,
+  # keep the string value, discarding the symnbol value
+  #
+  # @return     [Hash] a copy of the hash where all its
+  #             keys are strings
+  #
+  def stringify_keys
+    each_with_object({}) do |(k, v), hsh|
+      next if k.is_a?(Symbol) && key?(k.to_s)
+
+      hsh[k.to_s] = v.is_a?(Hash) ? v.stringify_keys : v
+    end
+  end
+end

data/lib/curly/string.rb

@@ -0,0 +1,91 @@
+# frozen_string_literal: true
+
+##
+## Remove extra spaces and newlines from a string
+##
+## @return     [String] cleaned string
+##
+class ::String
+  def clean
+    gsub(/[\n ]+/m, ' ').gsub(/> +</, '><')
+  end
+
+  ##
+  ## Remove HTML tags from a string
+  ##
+  ## @return     [String] stripped string
+  ##
+  def strip_tags
+    gsub(%r{</?.*?>}, '')
+  end
+
+  ##
+  ## Destructive version of #clean
+  ##
+  ## @see #clean
+  ##
+  def clean!
+    replace clean
+  end
+
+  ##
+  ## Destructive version of #strip_tags
+  ##
+  ## @see #strip_tags
+  ##
+  def strip_tags!
+    replace strip_tags
+  end
+
+  ##
+  ## Convert an image type string to a symbol
+  ##
+  ## @return     Symbol :srcset, :img, :opengraph, :all
+  ##
+  def normalize_image_type(default = :all)
+    case self.to_s
+    when /^[sp]/i
+      :srcset
+    when /^i/i
+      :img
+    when /^o/i
+      :opengraph
+    else
+      default.is_a?(Symbol) ? default.to_sym : default.normalize_image_type
+    end
+  end
+
+  ##
+  ## Convert a browser type string to a symbol
+  ##
+  ## @return     Symbol :chrome, :firefox
+  ##
+  def normalize_browser_type(default = :none)
+    case self.to_s
+    when /^c/i
+      :chrome
+    when /^f/i
+      :firefox
+    else
+      default.is_a?(Symbol) ? default.to_sym : default.normalize_browser_type
+    end
+  end
+
+  ##
+  ## Convert a screenshot type string to a symbol
+  ##
+  ## @return     Symbol :full_page, :print_page, :visible
+  ##
+  def normalize_screenshot_type(default = :none)
+    case self.to_s
+    when /^f/i
+      :full_page
+    when /^p/i
+      :print_page
+    when /^v/i
+      :visible
+    else
+      default.is_a?(Symbol) ? default.to_sym : default.normalize_browser_type
+    end
+  end
+end

data/lib/curly/version.rb

@@ -0,0 +1,3 @@
+module Curly
+  VERSION = '0.0.2'
+end

data/lib/curly.rb

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+require 'curly/version'
+require 'curly/hash'
+require 'curly/string'
+require 'curly/array'
+require 'json'
+require 'yaml'
+require 'uri'
+require 'tty-which'
+require 'nokogiri'
+require 'selenium-webdriver'

data/src/_README.md

@@ -0,0 +1,101 @@
+<!--README--><!--GITHUB--># curlyq
+
+[![Gem](https://img.shields.io/gem/v/na.svg)](https://rubygems.org/gems/curlyq)
+[![GitHub license](https://img.shields.io/github/license/ttscoff/curlyq.svg)](./LICENSE.txt)
+
+**A command line helper for curl and web scraping**
+
+_If you find this useful, feel free to [buy me some coffee][donate]._
+<!--END GITHUB-->
+
+The current version of `curlyq` is <!--VER--><!--END VER-->.
+
+`curlyq` is a command that provides a simple interface for curl, with additional features for things like extracting images and links, finding elements by CSS selector or XPath, getting detailed header info, and more. It also has rudimentary support for making calls to JSON endpoints easier, but it's expected that you'll use something like `jq` to parse the output.
+
+[github]: https://github.com/ttscoff/curlyq/
+
+### Installation
+
+Assuming you have Ruby and RubyGems installed, you can just run `gem install curlyq`. If you run into errors, try `gem install --user-install curlyq`, or use `sudo gem install curlyq`.
+
+If you're using Homebrew, you have the option to install via [brew-gem](https://github.com/sportngin/brew-gem):
+
+    brew install brew-gem
+    brew gem install curlyq
+
+If you don't have Ruby/RubyGems, you can install them pretty easily with Homebrew, rvm, or asdf.
+
+### Usage
+
+Run `curlyq help` for a list of commands. Run `curlyq help SUBCOMMAND` for details on a particular subcommand and its options.
+
+```
+@cli(bundle exec bin/curlyq help)
+```
+
+#### Commands
+
+curlyq makes use of subcommands, e.g. `curlyq html` or `curlyq extract`. Each subcommand takes its own options, but I've made an effort to standardize the choices between each command.
+
+##### extract
+
+```
+@cli(bundle exec bin/curlyq help extract)
+```
+
+
+##### headlinks
+
+```
+@cli(bundle exec bin/curlyq help headlinks)
+```
+
+##### html
+
+```
+@cli(bundle exec bin/curlyq help html)
+```
+
+##### images
+
+```
+@cli(bundle exec bin/curlyq help images)
+```
+
+##### json
+
+```
+@cli(bundle exec bin/curlyq help json)
+```
+
+##### links
+
+```
+@cli(bundle exec bin/curlyq help links)
+```
+
+##### scrape
+
+```
+@cli(bundle exec bin/curlyq help scrape)
+```
+
+##### screenshot
+
+```
+@cli(bundle exec bin/curlyq help screenshot)
+```
+
+##### tags
+
+```
+@cli(bundle exec bin/curlyq help tags)
+```
+
+<!--GITHUB-->
+PayPal link: [paypal.me/ttscoff](https://paypal.me/ttscoff)
+
+## Changelog
+
+See [CHANGELOG.md](https://github.com/ttscoff/na_gem/blob/master/CHANGELOG.md)
+<!--END GITHUB--><!--END README-->

data/test/default_test.rb

@@ -0,0 +1,14 @@
+require_relative "test_helper"
+
+class DefaultTest < Minitest::Test
+
+  def setup
+  end
+
+  def teardown
+  end
+
+  def test_the_truth
+    assert true
+  end
+end

data/test/test_helper.rb

@@ -0,0 +1,4 @@
+require "minitest/autorun"
+
+# Add test libraries you want to use here, e.g. mocha
+# Add helper classes or methods here, too