curlyq 0.0.2

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 4a73a5990b9c07f4d564216cd13c1ea0d73a833191c3f7734e7e3e5af2954b40
+  data.tar.gz: 8444276e61febd7b3e517eec56155a4f8754809fa8dc46c0d6e173737bca79e0
+SHA512:
+  metadata.gz: ca1a8c0bfc122e8020b356018276e27647449834f30eb66d7561acf187ec6cd837b59564a722ceaad5b3e99ac47de4a9944dfc370e69b92575155988a81fcfd4
+  data.tar.gz: 3bc9ed736378cc70607d4f42ecbe1f8cc91fbe87243d0dda4f7dc9ff6e44f5cc33f687d00e188f25ae3494f47bbbfedd2f2e27e8b008048e22c6c10ce2dc3b7f

data/.gitignore

@@ -0,0 +1 @@
+html

data/CHANGELOG.md

@@ -0,0 +1,8 @@
+### 0.0.2
+
+2024-01-10 09:18
+
+### 0.0.1
+
+2024-01-10 08:20
+

data/Gemfile

@@ -0,0 +1,2 @@
+source 'https://rubygems.org'
+gemspec

data/Gemfile.lock

@@ -0,0 +1,41 @@
+PATH
+  remote: .
+  specs:
+    curlyq (0.0.2)
+      gli (~> 2.21.0)
+      nokogiri (~> 1.16.0)
+      selenium-webdriver (~> 4.16.0)
+      tty-which (~> 0.5.0)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    gli (2.21.1)
+    minitest (5.16.3)
+    nokogiri (1.16.0-arm64-darwin)
+      racc (~> 1.4)
+    racc (1.7.3)
+    rake (0.9.6)
+    rdoc (4.3.0)
+    rexml (3.2.6)
+    rubyzip (2.3.2)
+    selenium-webdriver (4.16.0)
+      rexml (~> 3.2, >= 3.2.5)
+      rubyzip (>= 1.2.2, < 3.0)
+      websocket (~> 1.0)
+    tty-which (0.5.0)
+    websocket (1.2.10)
+    yard (0.9.34)
+
+PLATFORMS
+  arm64-darwin-20
+
+DEPENDENCIES
+  curlyq!
+  minitest (~> 5.14)
+  rake (~> 0.9.2)
+  rdoc (~> 4.3)
+  yard (~> 0.9, >= 0.9.26)
+
+BUNDLED WITH
+   2.2.29

data/LICENSE.txt

@@ -0,0 +1,19 @@
+MIT License 
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is furnished
+to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice (including the next
+paragraph) shall be included in all copies or substantial portions of the
+Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS
+OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF
+OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,233 @@
+# curlyq
+
+[![Gem](https://img.shields.io/gem/v/na.svg)](https://rubygems.org/gems/curlyq)
+[![GitHub license](https://img.shields.io/github/license/ttscoff/curlyq.svg)](./LICENSE.txt)
+
+**A command line helper for curl and web scraping**
+
+_If you find this useful, feel free to [buy me some coffee][donate]._
+
+
+The current version of `curlyq` is 0.0.2
+.
+
+`curlyq` is a command that provides a simple interface for curl, with additional features for things like extracting images and links, finding elements by CSS selector or XPath, getting detailed header info, and more. It also has rudimentary support for making calls to JSON endpoints easier, but it's expected that you'll use something like `jq` to parse the output.
+
+[github]: https://github.com/ttscoff/curlyq/
+
+### Installation
+
+Assuming you have Ruby and RubyGems installed, you can just run `gem install curlyq`. If you run into errors, try `gem install --user-install curlyq`, or use `sudo gem install curlyq`.
+
+If you're using Homebrew, you have the option to install via [brew-gem](https://github.com/sportngin/brew-gem):
+
+    brew install brew-gem
+    brew gem install curlyq
+
+If you don't have Ruby/RubyGems, you can install them pretty easily with Homebrew, rvm, or asdf.
+
+### Usage
+
+Run `curlyq help` for a list of commands. Run `curlyq help SUBCOMMAND` for details on a particular subcommand and its options.
+
+```
+NAME
+    curlyq - A scriptable interface to curl
+
+SYNOPSIS
+    curlyq [global options] command [command options] [arguments...]
+
+VERSION
+    0.0.2
+
+GLOBAL OPTIONS
+    --help          - Show this message
+    --[no-]pretty   - Output "pretty" JSON (default: enabled)
+    --version       - Display the program version
+    -y, --[no-]yaml - Output YAML instead of json
+
+COMMANDS
+    extract    - Extract contents between two regular expressions
+    headlinks  - Return all <head> links on URL's page
+    help       - Shows a list of commands or help for one command
+    html, curl - Curl URL and output its elements, multiple URLs allowed
+    images     - Extract all images from a URL
+    json       - Get a JSON response from a URL, multiple URLs allowed
+    links      - Return all links on a URL's page
+    scrape     - Scrape a page using a web browser, for dynamic (JS) pages. Be sure to have the selected --browser installed.
+    screenshot - Save a screenshot of a URL
+    tags       - Extract all instances of a tag
+```
+
+#### Commands
+
+curlyq makes use of subcommands, e.g. `curlyq html` or `curlyq extract`. Each subcommand takes its own options, but I've made an effort to standardize the choices between each command.
+
+##### extract
+
+```
+NAME
+    extract - Extract contents between two regular expressions
+
+SYNOPSIS
+
+    curlyq [global options] extract [command options] URL...
+
+COMMAND OPTIONS
+    -a, --after=arg       - Text after extraction, parsed as regex (default: none)
+    -b, --before=arg      - Text before extraction, parsed as regex (default: none)
+    -c, --[no-]compressed - Expect compressed results
+    --[no-]clean          - Remove extra whitespace from results
+    -h, --header=arg      - Define a header to send as key=value (may be used more than once, default: none)
+    --[no-]strip          - Strip HTML tags from results
+```
+
+
+##### headlinks
+
+```
+NAME
+    headlinks - Return all <head> links on URL's page
+
+SYNOPSIS
+
+    curlyq [global options] headlinks [command options] URL...
+
+COMMAND OPTIONS
+    -q, --query, --filter=arg - Filter output using dot-syntax path (default: none)
+```
+
+##### html
+
+```
+NAME
+    html - Curl URL and output its elements, multiple URLs allowed
+
+SYNOPSIS
+
+    curlyq [global options] html [command options] URL...
+
+COMMAND OPTIONS
+    -I, --info                - Only retrieve headers/info
+    -b, --browser=arg         - Use a browser to retrieve a dynamic web page (firefox, chrome) (default: none)
+    -c, --compressed          - Expect compressed results
+    --[no-]clean              - Remove extra whitespace from results
+    -f, --fallback=arg        - If curl doesn't work, use a fallback browser (firefox, chrome) (default: none)
+    -h, --header=arg          - Define a header to send as "key=value" (may be used more than once, default: none)
+    --[no-]ignore_fragments   - Ignore fragment hrefs when gathering content links
+    --[no-]ignore_relative    - Ignore relative hrefs when gathering content links
+    -q, --query, --filter=arg - Filter output using dot-syntax path (default: none)
+    -r, --raw=arg             - Output a raw value for a key (default: none)
+    --search=arg              - Regurn an array of matches to a CSS or XPath query (default: none)
+    -x, --external_links_only - Only gather external links
+```
+
+##### images
+
+```
+NAME
+    images - Extract all images from a URL
+
+SYNOPSIS
+
+    curlyq [global options] images [command options] URL...
+
+COMMAND OPTIONS
+    -c, --[no-]compressed - Expect compressed results
+    --[no-]clean          - Remove extra whitespace from results
+    -t, --type=arg        - Type of images to return (img, srcset, opengraph, all) (may be used more than once, default: ["all"])
+```
+
+##### json
+
+```
+NAME
+    json - Get a JSON response from a URL, multiple URLs allowed
+
+SYNOPSIS
+
+    curlyq [global options] json [command options] URL...
+
+COMMAND OPTIONS
+    -c, --[no-]compressed     - Expect compressed results
+    -h, --header=arg          - Define a header to send as key=value (may be used more than once, default: none)
+    -q, --query, --filter=arg - Filter output using dot-syntax path (default: none)
+```
+
+##### links
+
+```
+NAME
+    links - Return all links on a URL's page
+
+SYNOPSIS
+
+    curlyq [global options] links [command options] URL...
+
+COMMAND OPTIONS
+    -d, --[no-]dedup          - Filter out duplicate links, preserving only first one
+    --[no-]ignore_fragments   - Ignore fragment hrefs when gathering content links
+    --[no-]ignore_relative    - Ignore relative hrefs when gathering content links
+    -q, --query, --filter=arg - Filter output using dot-syntax path (default: none)
+    -x, --external_links_only - Only gather external links
+```
+
+##### scrape
+
+```
+NAME
+    scrape - Scrape a page using a web browser, for dynamic (JS) pages. Be sure to have the selected --browser installed.
+
+SYNOPSIS
+
+    curlyq [global options] scrape [command options] URL...
+
+COMMAND OPTIONS
+    -b, --browser=arg         - Browser to use (firefox, chrome) (default: none)
+    --[no-]clean              - Remove extra whitespace from results
+    -h, --header=arg          - Define a header to send as "key=value" (may be used more than once, default: none)
+    -q, --query, --filter=arg - Filter output using dot-syntax path (default: none)
+    -r, --raw=arg             - Output a raw value for a key (default: none)
+    --search=arg              - Regurn an array of matches to a CSS or XPath query (default: none)
+```
+
+##### screenshot
+
+```
+NAME
+    screenshot - Save a screenshot of a URL
+
+SYNOPSIS
+
+    curlyq [global options] screenshot [command options] URL...
+
+COMMAND OPTIONS
+    -b, --browser=arg     - Browser to use (firefox, chrome) (default: chrome)
+    -o, --out, --file=arg - File destination (default: none)
+    -t, --type=arg        - Type of screenshot to save (full (requires firefox), print, visible) (default: full)
+```
+
+##### tags
+
+```
+NAME
+    tags - Extract all instances of a tag
+
+SYNOPSIS
+
+    curlyq [global options] tags [command options] URL...
+
+COMMAND OPTIONS
+    -c, --[no-]compressed     - Expect compressed results
+    --[no-]clean              - Remove extra whitespace from results
+    -h, --header=arg          - Define a header to send as key=value (may be used more than once, default: none)
+    -q, --query, --search=arg - CSS/XPath query (default: none)
+    -t, --tag=arg             - Specify a tag to collect (may be used more than once, default: none)
+```
+
+
+PayPal link: [paypal.me/ttscoff](https://paypal.me/ttscoff)
+
+## Changelog
+
+See [CHANGELOG.md](https://github.com/ttscoff/na_gem/blob/master/CHANGELOG.md)

data/README.rdoc

@@ -0,0 +1,6 @@
+= curly
+
+A CLI helper for curl and web scraping
+
+:include:curlyq.rdoc
+

data/Rakefile

@@ -0,0 +1,77 @@
+require 'rake/clean'
+require 'rubygems'
+require 'rubygems/package_task'
+require 'rdoc/task'
+require 'yard'
+
+YARD::Rake::YardocTask.new do |t|
+ t.files = ['lib/curly/*.rb']
+ t.options = ['--markup=markdown', '--no-private', '-p', 'yard_templates']
+ # t.stats_options = ['--list-undoc']
+end
+
+task :doc, [*Rake.application[:yard].arg_names] => [:yard]
+
+Rake::RDocTask.new do |rd|
+  rd.main = "README.rdoc"
+  rd.rdoc_files.include("README.rdoc","lib/**/*.rb","bin/**/*")
+  rd.title = 'curlyq'
+end
+
+spec = eval(File.read('curlyq.gemspec'))
+
+Gem::PackageTask.new(spec) do |pkg|
+end
+require 'rake/testtask'
+Rake::TestTask.new do |t|
+  t.libs << "test"
+  t.test_files = FileList['test/*_test.rb']
+end
+
+desc 'Development version check'
+task :ver do
+  gver = `git ver`
+  cver = IO.read(File.join(File.dirname(__FILE__), 'CHANGELOG.md')).match(/^#+ (\d+\.\d+\.\d+(\w+)?)/)[1]
+  res = `grep VERSION lib/curly/version.rb`
+  version = res.match(/VERSION *= *['"](\d+\.\d+\.\d+(\w+)?)/)[1]
+  puts "git tag: #{gver}"
+  puts "version.rb: #{version}"
+  puts "changelog: #{cver}"
+end
+
+desc 'Changelog version check'
+task :cver do
+  puts IO.read(File.join(File.dirname(__FILE__), 'CHANGELOG.md')).match(/^#+ (\d+\.\d+\.\d+(\w+)?)/)[1]
+end
+
+desc 'Bump incremental version number'
+task :bump, :type do |_, args|
+  args.with_defaults(type: 'inc')
+  version_file = 'lib/curly/version.rb'
+  content = IO.read(version_file)
+  content.sub!(/VERSION = '(?<major>\d+)\.(?<minor>\d+)\.(?<inc>\d+)(?<pre>\S+)?'/) do
+    m = Regexp.last_match
+    major = m['major'].to_i
+    minor = m['minor'].to_i
+    inc = m['inc'].to_i
+    pre = m['pre']
+
+    case args[:type]
+    when /^maj/
+      major += 1
+      minor = 0
+      inc = 0
+    when /^min/
+      minor += 1
+      inc = 0
+    else
+      inc += 1
+    end
+
+    $stdout.puts "At version #{major}.#{minor}.#{inc}#{pre}"
+    "VERSION = '#{major}.#{minor}.#{inc}#{pre}'"
+  end
+  File.open(version_file, 'w+') { |f| f.puts content }
+end
+
+task default: %i[test clobber package]