curlyq 0.0.4 → 0.0.6

data/test/curlyq_extract_test.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+require 'json'
+require 'yaml'
+
+require 'helpers/curlyq-helpers'
+require 'test_helper'
+
+# Tests for tags command
+class CurlyQTagsTest < Test::Unit::TestCase
+  include CurlyQHelpers
+
+  def setup
+  end
+
+  def test_extract_inclusive
+    result = curlyq('extract', '-i', '-b', 'Adding', '-a', 'accessing the source.', 'https://stackoverflow.com/questions/52428409/get-fully-rendered-html-using-selenium-webdriver-and-python')
+    json = JSON.parse(result)
+
+    assert_match(/^Adding <code>time.sleep\(10\)<\/code>.*?accessing the source.$/, json[0], 'Match should be found and include the before and after strings')
+  end
+
+  def test_extract_exclusive
+    result = curlyq('extract', '-b', 'Adding', '-a', 'accessing the source.', 'https://stackoverflow.com/questions/52428409/get-fully-rendered-html-using-selenium-webdriver-and-python')
+    json = JSON.parse(result)
+
+    assert_match(/^ <code>time.sleep\(10\)<\/code>.*?when I was $/, json[0], 'Match should be found and not include the before and after strings')
+  end
+
+  def test_extract_regex_inclusive
+    result = curlyq('extract', '-ri', '-b', '.dding <', '-a', 'accessing.*?source.', 'https://stackoverflow.com/questions/52428409/get-fully-rendered-html-using-selenium-webdriver-and-python')
+    json = JSON.parse(result)
+
+    assert_match(/^Adding <code>time.sleep\(10\)<\/code>.*?accessing the source.$/, json[0], 'Match should be found and include the before and after strings')
+  end
+
+  def test_extract_regex_exclusive
+    result = curlyq('extract', '-r', '-b', '.dding <', '-a', 'accessing.*?source.', 'https://stackoverflow.com/questions/52428409/get-fully-rendered-html-using-selenium-webdriver-and-python')
+    json = JSON.parse(result)
+
+    assert_match(/^code>time.sleep\(10\)<\/code>.*?when I was $/, json[0], 'Match should be found and not include the before and after strings')
+  end
+end

data/test/curlyq_headlinks_test.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+require 'json'
+require 'yaml'
+
+require 'helpers/curlyq-helpers'
+require 'test_helper'
+
+# Tests for tags command
+class CurlyQHeadlinksTest < Test::Unit::TestCase
+  include CurlyQHelpers
+
+  def setup
+  end
+
+  def test_headlinks_query
+    result = curlyq('headlinks', '-q', '[rel=stylesheet]', 'https://brettterpstra.com')
+    json = JSON.parse(result)
+
+    assert_match(/stylesheet/, json['rel'], 'Should have retrieved a single result with rel stylesheet')
+    assert_match(/screen\.\d+\.css$/, json['href'], 'Stylesheet should be correct primary stylesheet')
+  end
+
+  def test_headlinks
+    result = curlyq('headlinks', 'https://brettterpstra.com')
+    json = JSON.parse(result)
+
+    assert_equal(Array, json.class, 'Should have an array of results')
+    assert(json.count > 1, 'Should have more than one link')
+    # assert(json[0].count.positive?)
+  end
+end

data/test/curlyq_html_test.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+require 'json'
+require 'yaml'
+
+require 'helpers/curlyq-helpers'
+require 'test_helper'
+
+# Tests for tags command
+class CurlyQHtmlTest < Test::Unit::TestCase
+  include CurlyQHelpers
+
+  def test_html_search_query
+    result = curlyq('html', '-s', '#main article .aligncenter', '-q', 'images[1]', 'https://brettterpstra.com')
+    json = JSON.parse(result)
+
+    assert_match(/aligncenter/, json[0]['class'], 'Should have found an image with class "aligncenter"')
+  end
+
+  def test_html_query
+    result = curlyq('html', '-q', 'meta.title', 'https://brettterpstra.com/2024/01/10/introducing-curlyq-a-pipeline-oriented-curl-helper/')
+
+    assert_match(/Introducing CurlyQ/, result, 'Should have retrived the page title')
+  end
+end

data/test/curlyq_images_test.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+require 'json'
+require 'yaml'
+
+require 'helpers/curlyq-helpers'
+require 'test_helper'
+
+# Tests for tags command
+class CurlyQImagesTest < Test::Unit::TestCase
+  include CurlyQHelpers
+
+  def test_images_query
+    result = curlyq('images', '-t', 'img', '-q', '[alt$=screenshot]', 'https://brettterpstra.com/2024/01/08/keyboard-maestro-giveaway/')
+    json = JSON.parse(result)
+
+    assert(json.count == 1, 'Should have found 1 image')
+    assert_match(/Keyboard Maestro screenshot/, json[0]['alt'], 'Should match Keyboard Meastro screenshot')
+  end
+
+  def test_images_type
+    result = curlyq('images', '-t', 'srcset', 'https://brettterpstra.com/')
+    json = JSON.parse(result)
+
+    assert(json.count.positive?, 'Should have found at least 1 image')
+  end
+end

data/test/curlyq_json_test.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+require 'json'
+require 'yaml'
+
+require 'helpers/curlyq-helpers'
+require 'test_helper'
+
+# Tests for tags command
+class CurlyQJsonTest < Test::Unit::TestCase
+  include CurlyQHelpers
+
+  def setup
+  end
+
+  def test_json    
+    result = curlyq('json', 'https://brettterpstra.com/scripts/giveaways_wrapper.cgi?v=203495&giveaway=hazel2023&action=count')
+    json = JSON.parse(result)
+
+    assert_equal(json.class, Hash, 'Single result should be a hash')
+    assert_equal(286, json['json']['total'], 'json.total should match 286')
+  end
+
+  def test_query
+    result1 = curlyq('json', '-q', 'total', 'https://brettterpstra.com/scripts/giveaways_wrapper.cgi?v=203495&giveaway=hazel2023&action=count')
+    result2 = curlyq('json', '-q', 'json.total', 'https://brettterpstra.com/scripts/giveaways_wrapper.cgi?v=203495&giveaway=hazel2023&action=count')
+    json1 = JSON.parse(result1)
+    json2 = JSON.parse(result2)
+
+    assert_equal(286, json1, 'Should be 286')
+    assert_equal(286, json2, 'Including json in dot path should yeild same result')
+  end
+end

data/test/curlyq_links_test.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+require 'json'
+require 'yaml'
+
+require 'helpers/curlyq-helpers'
+require 'test_helper'
+
+# Tests for tags command
+class CurlyQLinksTest < Test::Unit::TestCase
+  include CurlyQHelpers
+
+  def test_links
+    result = curlyq('links', '-q', '[content*=twitter]', 'https://stackoverflow.com/questions/52428409/get-fully-rendered-html-using-selenium-webdriver-and-python')
+    json = JSON.parse(result)
+
+    assert(json.count.positive?, 'Should be at least 1 match')
+    assert_match(/twitter.com/, json[0]['href'], 'Should be a link to Twitter')
+  end
+end

data/test/curlyq_scrape_test.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+require 'json'
+require 'yaml'
+
+require 'helpers/curlyq-helpers'
+require 'test_helper'
+
+# Tests for tags command
+class CurlyQScrapeTest < Test::Unit::TestCase
+  include CurlyQHelpers
+
+  def setup
+  end
+
+  def test_scrape
+    result = curlyq('scrape', '-b', 'firefox', '-q', 'links[rel=me&content*=mastodon][0]', 'https://brettterpstra.com/2024/01/10/introducing-curlyq-a-pipeline-oriented-curl-helper/')
+    json = JSON.parse(result)
+
+    assert_match(/Mastodon/, json['content'], 'Should have retrieved a Mastodon link')
+  end
+end

data/test/curlyq_tags_test.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+require 'json'
+require 'yaml'
+
+require 'helpers/curlyq-helpers'
+require 'test_helper'
+
+# Tests for tags command
+class CurlyQTagsTest < Test::Unit::TestCase
+  include CurlyQHelpers
+
+  def setup
+  end
+
+  def test_tags
+    result = curlyq('tags', '--search', '#main .post h3', '-q', 'attrs[id*=what]', 'https://brettterpstra.com/2024/01/10/introducing-curlyq-a-pipeline-oriented-curl-helper/')
+    json = JSON.parse(result)
+
+    assert_equal(json.count, 1, 'Should have 1 result')
+    assert_match(/whats-next/, json[0]['attrs']['id'], 'Should have matched #whats-next')
+  end
+
+  def test_clean
+    result = curlyq('tags', '--search', '#main section.related', '--clean', 'https://brettterpstra.com/2024/01/10/introducing-curlyq-a-pipeline-oriented-curl-helper/')
+    json = JSON.parse(result)
+
+    assert_equal(json.count, 1, 'Should have 1 result')
+    assert_match(%r{Last.fm</h5></a></li>}, json[0]['source'], 'Should have matched #whats-next')
+  end
+end

data/test/helpers/curlyq-helpers.rb

@@ -0,0 +1,29 @@
+require 'open3'
+require 'time'
+$LOAD_PATH.unshift File.join(__dir__, '..', '..', 'lib')
+require 'curly'
+
+module CurlyQHelpers
+  CURLYQ_EXEC = File.join(File.dirname(__FILE__), '..', '..', 'bin', 'curlyq')
+  BUNDLE = '/Users/ttscoff/.asdf/shims/bundle'
+
+  def curlyq_with_env(env, *args, stdin: nil)
+    Dir.chdir(File.expand_path('~/Desktop/Code/curlyq'))
+    pread(env, BUNDLE, 'exec', 'bin/curlyq', *args, stdin: stdin)
+  end
+
+  def curlyq(*args)
+    curlyq_with_env({ 'GLI_DEBUG' => 'true' }, *args)
+  end
+
+  def pread(env, *cmd, stdin: nil)
+    out, err, status = Open3.capture3(env, *cmd, stdin_data: stdin)
+    unless status.success?
+      raise [
+        "Error (#{status}): #{cmd.inspect} failed", "STDOUT:", out.inspect, "STDERR:", err.inspect
+      ].join("\n")
+    end
+
+    out
+  end
+end

data/test/helpers/fake_std_out.rb

@@ -0,0 +1,30 @@
+class FakeStdOut
+  attr_reader :strings
+
+  def initialize
+    @strings = []
+  end
+
+  def puts(string=nil)
+    @strings << string unless string.nil?
+  end
+
+  def write(x)
+    puts(x)
+  end
+
+  def printf(*args)
+    puts(Kernel.printf(*args))
+  end
+
+  # Returns true if the regexp matches anything in the output
+  def contained?(regexp)
+    strings.find{ |x| x =~ regexp }
+  end
+
+  def flush; end
+
+  def to_s
+    @strings.join("\n")
+  end
+end

data/test/helpers/threaded_tests.rb

@@ -0,0 +1,182 @@
+#!/usr/bin/env ruby
+
+require 'tty-spinner'
+require 'tty-progressbar'
+require 'open3'
+require 'shellwords'
+require 'fileutils'
+require 'pastel'
+
+class ThreadedTests
+  def run(pattern: '*', max_threads: 8, max_tests: 0)
+    pastel = Pastel.new
+
+    start_time = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+    @results = File.expand_path('results.log')
+
+    max_threads = 1000 if max_threads.to_i == 0
+
+    shuffle = false
+
+    unless pattern =~ /shuffle/i
+      pattern = "test/curlyq_*#{pattern}*_test.rb"
+    else
+      pattern = "test/curlyq_*_test.rb"
+      shuffle = true
+    end
+
+    tests = Dir.glob(pattern)
+
+    tests.shuffle! if shuffle
+
+    if max_tests.to_i > 0
+      tests = tests.slice(0, max_tests.to_i - 1)
+    end
+
+    puts pastel.cyan("#{tests.count} test files")
+
+    banner = "Running tests [:bar] T/A (#{max_threads.to_s} threads)"
+
+    progress = TTY::ProgressBar::Multi.new(banner,
+                                           width: 12,
+                                           clear: true,
+                                           hide_cursor: true)
+    @children = []
+    tests.each do |t|
+      test_name = File.basename(t, '.rb').sub(/curlyq_(.*?)_test/, '\1')
+      new_sp = progress.register("[:bar] #{test_name}:status",
+                                 total: tests.count + 8,
+                                 width: 1,
+                                 head: ' ',
+                                 unknown: ' ',
+                                 hide_cursor: true,
+                                 clear: true)
+      status = ': waiting'
+      @children.push([test_name, new_sp, status])
+    end
+
+    @elapsed = 0.0
+    @test_total = 0
+    @assrt_total = 0
+    @error_out = []
+    @threads = []
+    @running_tests = []
+
+    begin
+      finish_time = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+      while @children.count.positive?
+
+        slices = @children.slice!(0, max_threads)
+        slices.each { |c| c[1].start }
+        slices.each do |s|
+          @threads << Thread.new do
+            run_test(s)
+            finish_time = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+          end
+        end
+
+        @threads.each { |t| t.join }
+      end
+
+      finish_time = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+
+      progress.finish
+    rescue
+      progress.stop
+    ensure
+      msg = @running_tests.map { |t| t[1].format.sub(/^\[:bar\] (.*?):status/, "\\1#{t[2]}") }.join("\n")
+
+      output = []
+      output << if @error_out.count.positive?
+                  pastel.red("#{@error_out.count} Issues")
+                else
+                  pastel.green('Success')
+                end
+      output << pastel.green("#{@test_total} tests")
+      output << pastel.cyan("#{@assrt_total} assertions")
+      output << pastel.yellow("#{(finish_time - start_time).round(3)}s")
+      puts output.join(', ')
+
+      if @error_out.count.positive?
+        puts @error_out.join(pastel.white("\n----\n"))
+        Process.exit 1
+      end
+    end
+  end
+
+  def run_test(s)
+    pastel = Pastel.new
+
+    bar = s[1]
+    s[2] = ": #{pastel.green('running')}"
+    bar.advance(status: s[2])
+
+    if @running_tests.count.positive?
+      @running_tests.each do |b|
+        prev_bar = b[1]
+        if prev_bar.complete?
+          prev_bar.reset
+          prev_bar.advance(status: b[2])
+          prev_bar.finish
+        else
+          prev_bar.update(head: ' ', unfinished: ' ')
+          prev_bar.advance(status: b[2])
+        end
+      end
+    end
+
+    @running_tests.push(s)
+    out, _err, status = Open3.capture3(ENV, 'rake', "test:#{s[0]}", stdin_data: nil)
+    time = out.match(/^Finished in (?<time>\d+\.\d+) seconds\./)
+    count = out.match(/^(?<tests>\d+) tests, (?<assrt>\d+) assertions, (?<fails>\d+) failures, (?<errs>\d+) errors/)
+
+    unless status.success? && !count['fails'].to_i.positive? && !count['errs'].to_i.positive?
+      s[2] = if count
+               ": #{paste.red(count['fails'])} #{pastel.red('failures')}, #{pastel.red(count['errs'])} #{pastel.red('errors')}"
+             else
+               ": #{pastel.red('Unknown Error')}"
+             end
+      bar.update(head: pastel.red('✖'))
+      bar.advance(head: pastel.red('✖'), status: s[2])
+
+      # errs = out.scan(/(?:Failure|Error): [\w_]+\((?:.*?)\):(?:.*?)(?=\n=======)/m)
+      @error_out.push(out)
+      bar.finish
+
+      next_test
+      Thread.exit
+    end
+
+    s[2] = [
+      ': ',
+      pastel.green(count['tests']),
+      '/',
+      pastel.cyan(count['assrt']),
+      ' ',
+      pastel.yellow(time['time'].to_f.round(3).to_s),
+      's'
+    ].join('')
+    bar.update(head: pastel.green('✔'))
+    bar.advance(head: pastel.green('✔'), status: s[2])
+    @test_total += count['tests'].to_i
+    @assrt_total += count['assrt'].to_i
+    @elapsed += time['time'].to_f
+
+    bar.finish
+
+    next_test
+  end
+
+  def next_test
+    if @children.count.positive?
+      t = Thread.new do
+        s = @children.shift
+        # s[1].start
+        # s[1].advance(status: ": #{'running'.green}")
+        run_test(s)
+      end
+
+      t.join
+    end
+  end
+end

data/test/test_helper.rb

@@ -1,4 +1,9 @@
-require "minitest/autorun"
+# frozen_string_literal: true
 
+require 'test/unit'
 # Add test libraries you want to use here, e.g. mocha
-# Add helper classes or methods here, too
+
+class Test::Unit::TestCase
+  ENV['TZ'] = 'UTC'
+  # Add global extensions to the test case class here
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: curlyq
 version: !ruby/object:Gem::Version
-  version: 0.0.4
+  version: 0.0.6
 platform: ruby
 authors:
 - Brett Terpstra
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-01-10 00:00:00.000000000 Z
+date: 2024-01-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -16,42 +16,48 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.9.2
+        version: '13.0'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 13.0.1
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.9.2
+        version: '13.0'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 13.0.1
 - !ruby/object:Gem::Dependency
   name: rdoc
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '4.3'
+        version: 6.3.1
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '4.3'
+        version: 6.3.1
 - !ruby/object:Gem::Dependency
-  name: minitest
+  name: test-unit
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '5.14'
+        version: 3.4.4
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '5.14'
+        version: 3.4.4
 - !ruby/object:Gem::Dependency
   name: yard
   requirement: !ruby/object:Gem::Requirement
@@ -72,6 +78,80 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 0.9.26
+- !ruby/object:Gem::Dependency
+  name: tty-spinner
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.9'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.9.3
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.9'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.9.3
+- !ruby/object:Gem::Dependency
+  name: tty-progressbar
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.18'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.18.2
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.18'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.18.2
+- !ruby/object:Gem::Dependency
+  name: pastel
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.8.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.8.0
+- !ruby/object:Gem::Dependency
+  name: parallel_tests
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.7'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 3.7.3
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.7'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 3.7.3
 - !ruby/object:Gem::Dependency
   name: gli
   requirement: !ruby/object:Gem::Requirement
@@ -139,6 +219,7 @@ extra_rdoc_files:
 files:
 - ".github/FUNDING.yml"
 - ".gitignore"
+- ".irbrc"
 - CHANGELOG.md
 - Gemfile
 - Gemfile.lock
@@ -158,7 +239,18 @@ files:
 - lib/curly/string.rb
 - lib/curly/version.rb
 - src/_README.md
+- test/curlyq_extract_test.rb
+- test/curlyq_headlinks_test.rb
+- test/curlyq_html_test.rb
+- test/curlyq_images_test.rb
+- test/curlyq_json_test.rb
+- test/curlyq_links_test.rb
+- test/curlyq_scrape_test.rb
+- test/curlyq_tags_test.rb
 - test/default_test.rb
+- test/helpers/curlyq-helpers.rb
+- test/helpers/fake_std_out.rb
+- test/helpers/threaded_tests.rb
 - test/test_helper.rb
 homepage: https://brettterpstra.com
 licenses: