RubyGems - sinew - Versions diffs - 1.0.4 → 2.0.0 - Mend

sinew 1.0.4 → 2.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (38) hide show

checksums.yaml +5 -5
data/.gitignore +1 -0
data/.rubocop.yml +49 -0
data/.travis.yml +4 -0
data/.vscode/extensions.json +3 -0
data/.vscode/settings.json +15 -0
data/Gemfile +1 -1
data/README.md +153 -12
data/Rakefile +13 -14
data/bin/sinew +40 -20
data/lib/sinew.rb +10 -6
data/lib/sinew/cache.rb +79 -0
data/lib/sinew/core_ext.rb +59 -0
data/lib/sinew/dsl.rb +98 -0
data/lib/sinew/main.rb +80 -149
data/lib/sinew/nokogiri_ext.rb +10 -9
data/lib/sinew/output.rb +126 -0
data/lib/sinew/request.rb +148 -0
data/lib/sinew/response.rb +75 -0
data/lib/sinew/runtime_options.rb +26 -0
data/lib/sinew/version.rb +1 -1
data/sample.sinew +5 -3
data/sinew.gemspec +24 -19
data/test/test.html +40 -34
data/test/test_cache.rb +69 -0
data/test/test_helper.rb +113 -0
data/test/test_main.rb +36 -91
data/test/test_nokogiri_ext.rb +14 -15
data/test/test_output.rb +73 -0
data/test/test_requests.rb +135 -0
data/test/test_utf8.rb +39 -0
metadata +103 -48
data/lib/sinew/curler.rb +0 -173
data/lib/sinew/text_util.rb +0 -101
data/lib/sinew/util.rb +0 -236
data/test/helper.rb +0 -64
data/test/test_curler.rb +0 -70
data/test/test_text_util.rb +0 -23

data/lib/sinew/text_util.rb DELETED

@@ -1,101 +0,0 @@
-require "active_support/core_ext"
-require "set"
-module Sinew
-  module TextUtil
-    extend self
-    ATTRS_KEEP = Set.new %w(a img iframe)
-    TIDY_OPTIONS = {
-      "-asxml" => nil,
-      "-bare" => nil,
-      "-quiet" => nil,
-      "-utf8" => nil,
-      "-wrap" =>  0,
-      "--doctype" => "omit",
-      "--hide-comments" => "yes",
-      "--force-output" => "yes",
-      "-f" => "/dev/null",
-    }
-    XML_ENTITIES = { "&"=>"&amp;", "<"=>"&lt;", ">"=>"&gt;", "'"=>"&apos;", '"'=>"&quot;" }
-    XML_ENTITIES_INV = XML_ENTITIES.invert
-    COMMON_ENTITIES_INV = XML_ENTITIES_INV.merge(
-                                                 "&frac12;" => "1/2",
-                                                 "&frac14;" => "1/4",
-                                                 "&frac34;" => "3/4",
-                                                 "&ldquo;" => '"',
-                                                 "&lsquo;" => "'",
-                                                 "&mdash;" => "-",
-                                                 "&nbsp;" => " ",
-                                                 "&ndash;" => "-",
-                                                 "&rdquo;" => '"',
-                                                 "&rsquo;" => "'",
-                                                 "&tilde;" => "~",
-                                                 "&#34;" => '"',
-                                                 "&#39;" => "'",
-                                                 "&#160;" => " ",
-                                                 "&#8232;" => "\n"
-                                                 )
-    #
-    # tidy/clean
-    #
-    def html_tidy(s)
-      # run tidy
-      args = TIDY_OPTIONS.map { |k, v| "#{k} #{v}" }.join(" ")
-      s = IO.popen("tidy #{args}", "rb+") do |f|
-        f.write(s)
-        f.close_write
-        f.read
-      end
-      raise "could not run tidy" if ($? >> 8) > 2
-      # now kill some tags
-      s.sub!(/<html\b[^>]+>/, "<html>")
-      s.gsub!(/<\/?(meta|link)\b[^>]*>/m, "")
-      s.gsub!(/<(style|script)\b[^>]*(\/>|>.*?<\/\1\b>)/m, "")
-      s.gsub!(/<\?[^>]*>/m, "")
-      s.squish!
-      # kill whitespace around tags
-      s.gsub!(/ ?<([^>]+)> ?/, "<\\1>")
-      s
-    end
-    def html_clean(s)
-      html_clean_from_tidy(html_tidy(s))
-    end
-    def html_clean_from_tidy(s)
-      # then kill most attrs
-      s = s.dup
-      s.gsub!(/<([^\s>]+)[^>]*?(\/)?>/) do |i|
-        ATTRS_KEEP.include?($1) ? i : "<#{$1}#{$2}>"
-      end
-      s
-    end
-    #
-    # untag/unent
-    #
-    def xml_escape(s)
-      s.gsub(/[&<>'"]/) { |i| XML_ENTITIES[i] }
-    end
-    def xml_unescape(s)
-      s.gsub(/&(amp|lt|gt|apos|quot);/) { |i| XML_ENTITIES_INV[i] }
-    end
-    def untag(s)
-      s.gsub(/<[^>]+>/, " ")
-    end
-    def unent(s)
-      s.gsub(/&#?[a-z0-9]{2,};/) { |i| COMMON_ENTITIES_INV[i] }
-    end
-  end
-end

data/lib/sinew/util.rb DELETED

@@ -1,236 +0,0 @@
-require "digest/md5"
-require "etc"
-require "fileutils"
-module Sinew
-  # Helper module for executing commands and printing stuff
-  # out.
-  #
-  # The general idea is to only print commands that are actually
-  # interesting. For example, mkdir_if_necessary won't print anything
-  # if the directory already exists. That way we can scan output and
-  # see what changes were made without getting lost in repetitive
-  # commands that had no actual effect.
-  module Util
-    class RunError < StandardError ; end
-    extend self
-    RESET   = "\e[0m"
-    RED     = "\e[1;37;41m"
-    GREEN   = "\e[1;37;42m"
-    YELLOW  = "\e[1;37;43m"
-    BLUE    = "\e[1;37;44m"
-    MAGENTA = "\e[1;37;45m"
-    CYAN    = "\e[1;37;46m"
-    #
-    # running commands
-    #
-    # Make all commands echo before running.
-    def run_verbose!
-      @run_verbose = true
-    end
-    # Run a command, raise an error upon failure. Output goes to
-    # $stdout/$stderr.
-    def run(command, args = nil)
-      line = nil
-      if args
-        args = args.map(&:to_s)
-        line = "#{command} #{args.join(" ")}"
-        vputs line
-        system(command, *args)
-      else
-        line = command
-        vputs line
-        system(command)
-      end
-      if $? != 0
-        if $?.termsig == Signal.list["INT"]
-          raise "#{line} interrupted"
-        end
-        raise RunError, "#{line} failed : #{$?.to_i / 256}"
-      end
-    end
-    # Like mkdir -p. Optionally, set the owner and mode.
-    def mkdir(dir, owner = nil, mode = nil)
-      FileUtils.mkdir_p(dir, :verbose => verbose?)
-      chmod(dir, mode) if mode
-      chown(dir, owner) if owner
-    end
-    # mkdir only if the directory doesn't already exist. Optionally,
-    # set the owner and mode.
-    def mkdir_if_necessary(dir, owner = nil, mode = nil)
-      mkdir(dir, owner, mode) if !(File.exists?(dir) || File.symlink?(dir))
-    end
-    # rm a dir and recreate it.
-    def rm_and_mkdir(dir)
-      raise "don't do this" if dir == ""
-      run "rm -rf #{dir} && mkdir -p #{dir}"
-    end
-    # Are two files different?
-    def different?(a, b)
-      !FileUtils.compare_file(a, b)
-    end
-    # Copy file or dir from src to dst. Optionally, set the mode and
-    # owner of dst.
-    def cp(src, dst, owner = nil, mode = nil)
-      FileUtils.cp_r(src, dst, :preserve => true, :verbose => verbose?)
-      if owner && !File.symlink?(dst)
-        chown(dst, owner)
-      end
-      if mode
-        chmod(dst, mode)
-      end
-    end
-    # Copy file or dir from src to dst, but create the dst directory
-    # first if necessary. Optionally, set the mode and owner of dst.
-    def cp_with_mkdir(src, dst, owner = nil, mode = nil)
-      mkdir_if_necessary(File.dirname(dst))
-      cp(src, dst, owner, mode)
-    end
-    # Copy file or dir from src to dst, but ONLY if dst doesn't exist
-    # or has different contents than src. Optionally, set the mode and
-    # owner of dst.
-    def cp_if_necessary(src, dst, owner = nil, mode = nil)
-      if !File.exists?(dst) || different?(src, dst)
-        cp(src, dst, owner, mode)
-        true
-      end
-    end
-    # Move src to dst. Because this uses FileUtils, it works even if
-    # dst is on a different partition.
-    def mv(src, dst)
-      FileUtils.mv(src, dst, :verbose => verbose?)
-    end
-    # Move src to dst, but create the dst directory first if
-    # necessary.
-    def mv_with_mkdir(src, dst)
-      mkdir_if_necessary(File.dirname(dst))
-      mv(src, dst)
-    end
-    # Chown file to be owned by user.
-    def chown(file, user)
-      user = user.to_s
-      # who is the current owner?
-      @uids ||= {}
-      @uids[user] ||= Etc.getpwnam(user).uid
-      uid = @uids[user]
-      if File.stat(file).uid != uid
-        run "chown #{user}:#{user} '#{file}'"
-      end
-    end
-    # Chmod file to a new mode.
-    def chmod(file, mode)
-      if File.stat(file).mode != mode
-        FileUtils.chmod(mode, file, :verbose => verbose?)
-      end
-    end
-    # rm a file
-    def rm(file)
-      FileUtils.rm(file, :force => true, :verbose => verbose?)
-    end
-    # rm a file, but only if it exists.
-    def rm_if_necessary(file)
-      if File.exists?(file)
-        rm(file)
-        true
-      end
-    end
-    # Create a symlink from src to dst.
-    def ln(src, dst)
-      FileUtils.ln_sf(src, dst, :verbose => verbose?)
-    end
-    # Create a symlink from src to dst, but only if it hasn't already
-    # been created.
-    def ln_if_necessary(src, dst)
-      ln = false
-      if !File.symlink?(dst)
-        ln = true
-      elsif File.readlink(dst) != src
-        rm(dst)
-        ln = true
-      end
-      if ln
-        ln(src, dst)
-        true
-      end
-    end
-    # Touch a file
-    def touch(file)
-      FileUtils.touch(file)
-    end
-    # A nice printout in green.
-    def banner(s, color = GREEN)
-      s = "#{s} ".ljust(72, " ")
-      $stderr.write "#{color}[#{Time.new.strftime('%H:%M:%S')}] #{s}#{RESET}\n"
-      $stderr.flush
-    end
-    # Print a warning in yellow.
-    def warning(msg)
-      banner("Warning: #{msg}", YELLOW)
-    end
-    # Print a fatal error in red, then exit.
-    def fatal(msg)
-      banner(msg, RED)
-      exit(1)
-    end
-    # Generate some random text
-    def random_text(len)
-      chars = ("A".."Z").to_a + ("a".."z").to_a + ("0".."9").to_a
-      (1..len).map { chars[rand(chars.length - 1)] }.join("")
-    end
-    # Convert a string into something that could be a path segment
-    def pathify(s)
-      s = s.gsub(/^\//, "")
-      s = s.gsub("..", ",")
-      s = s.gsub(/[?\/&]/, ",")
-      s = s.gsub(/[^A-Za-z0-9_.,=-]/) do |i|
-        hex = i.unpack("H2").first
-        "%#{hex}"
-      end
-      s = "_root_" if s.empty?
-      s = s.downcase
-      s
-    end
-    # checksum some text
-    def md5(s)
-      Digest::MD5.hexdigest(s.to_s)
-    end
-    private
-    # Returns true if verbosity is turned on.
-    def verbose?
-      @run_verbose ||= nil
-    end
-    def vputs(s)
-      $stderr.puts s if verbose?
-    end
-  end
-end

data/test/helper.rb DELETED

@@ -1,64 +0,0 @@
-require "active_support/core_ext"
-require "test/unit"
-require "sinew"
-module Sinew
-  class TestCase < Test::Unit::TestCase
-    TMP = "/tmp/_test_sinew"
-    HTML_FILE = File.expand_path("#{File.dirname(__FILE__)}/test.html")
-    HTML = File.read(HTML_FILE)
-    #
-    # for mocking curl
-    #
-    def mock_curl_200
-      Proc.new do |cmd, args|
-        mock_curl(args, HTML, "HTTP/1.1 200 OK")
-      end
-    end
-    def mock_curl_302
-      Proc.new do |cmd, args|
-        mock_curl(args, "", "HTTP/1.1 302 Moved Temporarily\r\nLocation: http://www.gub.com")
-      end
-    end
-    def mock_curl_500
-      Proc.new do |cmd, args|
-        raise Util::RunError, "curl error"
-      end
-    end
-    def mock_curl(args, body, head)
-      File.write(args[args.index("--output") + 1], body)
-      File.write(args[args.index("--dump-header") + 1], "#{head}\r\n\r\n")
-    end
-  end
-end
-#
-# from MiniTest, but not in the gem yet
-#
-class Object
-  def stub name, val_or_callable, &block
-    new_name = "__minitest_stub__#{name}"
-    metaclass = class << self; self; end
-    metaclass.send :alias_method, new_name, name
-    metaclass.send :define_method, name do |*args|
-      if val_or_callable.respond_to? :call then
-        val_or_callable.call(*args)
-      else
-        val_or_callable
-      end
-    end
-    yield
-  ensure
-    metaclass.send :undef_method, name
-    metaclass.send :alias_method, name, new_name
-    metaclass.send :undef_method, new_name
-  end
-end

data/test/test_curler.rb DELETED

@@ -1,70 +0,0 @@
-require "helper"
-module Sinew
-  class TestCurler < TestCase
-    def setup
-      # create TMP dir
-      FileUtils.rm_rf(TMP) if File.exists?(TMP)
-      FileUtils.mkdir_p(TMP)
-      # curler, pointed at TMP
-      @curler = Curler.new(dir: TMP, verbose: false)
-    end
-    #
-    # tests
-    #
-    def test_200
-      Util.stub(:run, mock_curl_200) do
-        path = @curler.get("http://www.example.com")
-        assert_equal(HTML, File.read(path))
-      end
-    end
-    def test_500
-      assert_raises(Curler::Error) do
-        Util.stub(:run, mock_curl_500) do
-          @curler.get("http://www.example.com")
-        end
-      end
-    end
-    def test_cached
-      Util.stub(:run, mock_curl_200) do
-        assert_equal(HTML, File.read(@curler.get("http://www.example.com")))
-      end
-      # the file is cached, so this shouldn't produce an error
-      Util.stub(:run, mock_curl_500) do
-        @curler.get("http://www.example.com")
-      end
-    end
-    def test_302
-      Util.stub(:run, mock_curl_302) do
-        @curler.get("http://www.example.com")
-        assert_equal("http://www.gub.com", @curler.url)
-      end
-    end
-    def test_rate_limit
-      slept = false
-      # change Kernel#sleep to not really sleep!
-      Kernel.send(:alias_method, :old_sleep, :sleep)
-      Kernel.send(:define_method, :sleep) do |x|
-        slept = true
-      end
-      Util.stub(:run, mock_curl_200) do
-        @curler.get("http://www.example.com/1")
-        @curler.get("http://www.example.com/2")
-      end
-      assert(slept)
-      # restore old Kernel#sleep
-      Kernel.send(:alias_method, :sleep, :old_sleep)
-      Kernel.send(:undef_method, :old_sleep)
-    end
-  end
-end