RubyGems - sinew - Versions diffs - 1.0.4 → 2.0.0 - Mend

sinew 1.0.4 → 2.0.0

Files changed (38) hide show

checksums.yaml +5 -5
data/.gitignore +1 -0
data/.rubocop.yml +49 -0
data/.travis.yml +4 -0
data/.vscode/extensions.json +3 -0
data/.vscode/settings.json +15 -0
data/Gemfile +1 -1
data/README.md +153 -12
data/Rakefile +13 -14
data/bin/sinew +40 -20
data/lib/sinew.rb +10 -6
data/lib/sinew/cache.rb +79 -0
data/lib/sinew/core_ext.rb +59 -0
data/lib/sinew/dsl.rb +98 -0
data/lib/sinew/main.rb +80 -149
data/lib/sinew/nokogiri_ext.rb +10 -9
data/lib/sinew/output.rb +126 -0
data/lib/sinew/request.rb +148 -0
data/lib/sinew/response.rb +75 -0
data/lib/sinew/runtime_options.rb +26 -0
data/lib/sinew/version.rb +1 -1
data/sample.sinew +5 -3
data/sinew.gemspec +24 -19
data/test/test.html +40 -34
data/test/test_cache.rb +69 -0
data/test/test_helper.rb +113 -0
data/test/test_main.rb +36 -91
data/test/test_nokogiri_ext.rb +14 -15
data/test/test_output.rb +73 -0
data/test/test_requests.rb +135 -0
data/test/test_utf8.rb +39 -0
metadata +103 -48
data/lib/sinew/curler.rb +0 -173
data/lib/sinew/text_util.rb +0 -101
data/lib/sinew/util.rb +0 -236
data/test/helper.rb +0 -64
data/test/test_curler.rb +0 -70
data/test/test_text_util.rb +0 -23

@@ -1,101 +0,0 @@
-require "active_support/core_ext"
-require "set"
-module Sinew
-  module TextUtil
-    extend self
-    ATTRS_KEEP = Set.new %w(a img iframe)
-    TIDY_OPTIONS = {
-      "-asxml" => nil,
-      "-bare" => nil,
-      "-quiet" => nil,
-      "-utf8" => nil,
-      "-wrap" =>  0,
-      "--doctype" => "omit",
-      "--hide-comments" => "yes",
-      "--force-output" => "yes",
-      "-f" => "/dev/null",
-    }
-    XML_ENTITIES = { "&"=>"&amp;", "<"=>"&lt;", ">"=>"&gt;", "'"=>"&apos;", '"'=>"&quot;" }
-    XML_ENTITIES_INV = XML_ENTITIES.invert
-    COMMON_ENTITIES_INV = XML_ENTITIES_INV.merge(
-                                                 "&frac12;" => "1/2",
-                                                 "&frac14;" => "1/4",
-                                                 "&frac34;" => "3/4",
-                                                 "&ldquo;" => '"',
-                                                 "&lsquo;" => "'",
-                                                 "&mdash;" => "-",
-                                                 "&nbsp;" => " ",
-                                                 "&ndash;" => "-",
-                                                 "&rdquo;" => '"',
-                                                 "&rsquo;" => "'",
-                                                 "&tilde;" => "~",
-                                                 "&#34;" => '"',
-                                                 "&#39;" => "'",
-                                                 "&#160;" => " ",
-                                                 "&#8232;" => "\n"
-                                                 )
-    #
-    # tidy/clean
-    #
-    def html_tidy(s)
-      # run tidy
-      args = TIDY_OPTIONS.map { |k, v| "#{k} #{v}" }.join(" ")
-      s = IO.popen("tidy #{args}", "rb+") do |f|
-        f.write(s)
-        f.close_write
-        f.read
-      end
-      raise "could not run tidy" if ($? >> 8) > 2
-      # now kill some tags
-      s.sub!(/<html\b[^>]+>/, "<html>")
-      s.gsub!(/<\/?(meta|link)\b[^>]*>/m, "")
-      s.gsub!(/<(style|script)\b[^>]*(\/>|>.*?<\/\1\b>)/m, "")
-      s.gsub!(/<\?[^>]*>/m, "")
-      s.squish!
-      # kill whitespace around tags
-      s.gsub!(/ ?<([^>]+)> ?/, "<\\1>")
-      s
-    end
-    def html_clean(s)
-      html_clean_from_tidy(html_tidy(s))
-    end
-    def html_clean_from_tidy(s)
-      # then kill most attrs
-      s = s.dup
-      s.gsub!(/<([^\s>]+)[^>]*?(\/)?>/) do |i|
-        ATTRS_KEEP.include?($1) ? i : "<#{$1}#{$2}>"
-      end
-      s
-    end
-    #
-    # untag/unent
-    #
-    def xml_escape(s)
-      s.gsub(/[&<>'"]/) { |i| XML_ENTITIES[i] }
-    end
-    def xml_unescape(s)
-      s.gsub(/&(amp|lt|gt|apos|quot);/) { |i| XML_ENTITIES_INV[i] }
-    end
-    def untag(s)
-      s.gsub(/<[^>]+>/, " ")
-    end
-    def unent(s)
-      s.gsub(/&#?[a-z0-9]{2,};/) { |i| COMMON_ENTITIES_INV[i] }
-    end
-  end
-end

data/lib/sinew/util.rb DELETED

@@ -1,236 +0,0 @@
-require "digest/md5"
-require "etc"
-require "fileutils"
-module Sinew
-  # Helper module for executing commands and printing stuff
-  # out.
-  #
-  # The general idea is to only print commands that are actually
-  # interesting. For example, mkdir_if_necessary won't print anything
-  # if the directory already exists. That way we can scan output and
-  # see what changes were made without getting lost in repetitive
-  # commands that had no actual effect.
-  module Util
-    class RunError < StandardError ; end
-    extend self
-    RESET   = "\e[0m"
-    RED     = "\e[1;37;41m"
-    GREEN   = "\e[1;37;42m"
-    YELLOW  = "\e[1;37;43m"
-    BLUE    = "\e[1;37;44m"
-    MAGENTA = "\e[1;37;45m"
-    CYAN    = "\e[1;37;46m"
-    #
-    # running commands
-    #
-    # Make all commands echo before running.
-    def run_verbose!
-      @run_verbose = true
-    end
-    # Run a command, raise an error upon failure. Output goes to
-    # $stdout/$stderr.
-    def run(command, args = nil)
-      line = nil
-      if args
-        args = args.map(&:to_s)
-        line = "#{command} #{args.join(" ")}"
-        vputs line
-        system(command, *args)
-      else
-        line = command
-        vputs line
-        system(command)
-      end
-      if $? != 0
-        if $?.termsig == Signal.list["INT"]
-          raise "#{line} interrupted"
-        end
-        raise RunError, "#{line} failed : #{$?.to_i / 256}"
-      end
-    end
-    # Like mkdir -p. Optionally, set the owner and mode.
-    def mkdir(dir, owner = nil, mode = nil)
-      FileUtils.mkdir_p(dir, :verbose => verbose?)
-      chmod(dir, mode) if mode
-      chown(dir, owner) if owner
-    end
-    # mkdir only if the directory doesn't already exist. Optionally,
-    # set the owner and mode.
-    def mkdir_if_necessary(dir, owner = nil, mode = nil)
-      mkdir(dir, owner, mode) if !(File.exists?(dir) || File.symlink?(dir))
-    end
-    # rm a dir and recreate it.
-    def rm_and_mkdir(dir)
-      raise "don't do this" if dir == ""
-      run "rm -rf #{dir} && mkdir -p #{dir}"
-    end
-    # Are two files different?
-    def different?(a, b)
-      !FileUtils.compare_file(a, b)
-    end
-    # Copy file or dir from src to dst. Optionally, set the mode and
-    # owner of dst.
-    def cp(src, dst, owner = nil, mode = nil)
-      FileUtils.cp_r(src, dst, :preserve => true, :verbose => verbose?)
-      if owner && !File.symlink?(dst)
-        chown(dst, owner)
-      end
-      if mode
-        chmod(dst, mode)
-      end
-    end
-    # Copy file or dir from src to dst, but create the dst directory
-    # first if necessary. Optionally, set the mode and owner of dst.
-    def cp_with_mkdir(src, dst, owner = nil, mode = nil)
-      mkdir_if_necessary(File.dirname(dst))
-      cp(src, dst, owner, mode)
-    end
-    # Copy file or dir from src to dst, but ONLY if dst doesn't exist
-    # or has different contents than src. Optionally, set the mode and
-    # owner of dst.
-    def cp_if_necessary(src, dst, owner = nil, mode = nil)
-      if !File.exists?(dst) || different?(src, dst)
-        cp(src, dst, owner, mode)
-        true
-      end
-    end
-    # Move src to dst. Because this uses FileUtils, it works even if
-    # dst is on a different partition.
-    def mv(src, dst)
-      FileUtils.mv(src, dst, :verbose => verbose?)
-    end
-    # Move src to dst, but create the dst directory first if
-    # necessary.
-    def mv_with_mkdir(src, dst)
-      mkdir_if_necessary(File.dirname(dst))
-      mv(src, dst)
-    end
-    # Chown file to be owned by user.
-    def chown(file, user)
-      user = user.to_s
-      # who is the current owner?
-      @uids ||= {}
-      @uids[user] ||= Etc.getpwnam(user).uid
-      uid = @uids[user]
-      if File.stat(file).uid != uid
-        run "chown #{user}:#{user} '#{file}'"
-      end
-    end
-    # Chmod file to a new mode.
-    def chmod(file, mode)
-      if File.stat(file).mode != mode
-        FileUtils.chmod(mode, file, :verbose => verbose?)
-      end
-    end
-    # rm a file
-    def rm(file)
-      FileUtils.rm(file, :force => true, :verbose => verbose?)
-    end
-    # rm a file, but only if it exists.
-    def rm_if_necessary(file)
-      if File.exists?(file)
-        rm(file)
-        true
-      end
-    end
-    # Create a symlink from src to dst.
-    def ln(src, dst)
-      FileUtils.ln_sf(src, dst, :verbose => verbose?)
-    end
-    # Create a symlink from src to dst, but only if it hasn't already
-    # been created.
-    def ln_if_necessary(src, dst)
-      ln = false
-      if !File.symlink?(dst)
-        ln = true
-      elsif File.readlink(dst) != src
-        rm(dst)
-        ln = true
-      end
-      if ln
-        ln(src, dst)
-        true
-      end
-    end
-    # Touch a file
-    def touch(file)
-      FileUtils.touch(file)
-    end
-    # A nice printout in green.
-    def banner(s, color = GREEN)
-      s = "#{s} ".ljust(72, " ")
-      $stderr.write "#{color}[#{Time.new.strftime('%H:%M:%S')}] #{s}#{RESET}\n"
-      $stderr.flush
-    end
-    # Print a warning in yellow.
-    def warning(msg)
-      banner("Warning: #{msg}", YELLOW)
-    end
-    # Print a fatal error in red, then exit.
-    def fatal(msg)
-      banner(msg, RED)
-      exit(1)
-    end
-    # Generate some random text
-    def random_text(len)
-      chars = ("A".."Z").to_a + ("a".."z").to_a + ("0".."9").to_a
-      (1..len).map { chars[rand(chars.length - 1)] }.join("")
-    end
-    # Convert a string into something that could be a path segment
-    def pathify(s)
-      s = s.gsub(/^\//, "")
-      s = s.gsub("..", ",")
-      s = s.gsub(/[?\/&]/, ",")
-      s = s.gsub(/[^A-Za-z0-9_.,=-]/) do |i|
-        hex = i.unpack("H2").first
-        "%#{hex}"
-      end
-      s = "_root_" if s.empty?
-      s = s.downcase
-      s
-    end
-    # checksum some text
-    def md5(s)
-      Digest::MD5.hexdigest(s.to_s)
-    end
-    private
-    # Returns true if verbosity is turned on.
-    def verbose?
-      @run_verbose ||= nil
-    end
-    def vputs(s)
-      $stderr.puts s if verbose?
-    end
-  end
-end

data/test/helper.rb DELETED

@@ -1,64 +0,0 @@
-require "active_support/core_ext"
-require "test/unit"
-require "sinew"
-module Sinew
-  class TestCase < Test::Unit::TestCase
-    TMP = "/tmp/_test_sinew"
-    HTML_FILE = File.expand_path("#{File.dirname(__FILE__)}/test.html")
-    HTML = File.read(HTML_FILE)
-    #
-    # for mocking curl
-    #
-    def mock_curl_200
-      Proc.new do |cmd, args|
-        mock_curl(args, HTML, "HTTP/1.1 200 OK")
-      end
-    end
-    def mock_curl_302
-      Proc.new do |cmd, args|
-        mock_curl(args, "", "HTTP/1.1 302 Moved Temporarily\r\nLocation: http://www.gub.com")
-      end
-    end
-    def mock_curl_500
-      Proc.new do |cmd, args|
-        raise Util::RunError, "curl error"
-      end
-    end
-    def mock_curl(args, body, head)
-      File.write(args[args.index("--output") + 1], body)
-      File.write(args[args.index("--dump-header") + 1], "#{head}\r\n\r\n")
-    end
-  end
-end
-#
-# from MiniTest, but not in the gem yet
-#
-class Object
-  def stub name, val_or_callable, &block
-    new_name = "__minitest_stub__#{name}"
-    metaclass = class << self; self; end
-    metaclass.send :alias_method, new_name, name
-    metaclass.send :define_method, name do |*args|
-      if val_or_callable.respond_to? :call then
-        val_or_callable.call(*args)
-      else
-        val_or_callable
-      end
-    end
-    yield
-  ensure
-    metaclass.send :undef_method, name
-    metaclass.send :alias_method, name, new_name
-    metaclass.send :undef_method, new_name
-  end
-end

data/test/test_curler.rb DELETED

@@ -1,70 +0,0 @@
-require "helper"
-module Sinew
-  class TestCurler < TestCase
-    def setup
-      # create TMP dir
-      FileUtils.rm_rf(TMP) if File.exists?(TMP)
-      FileUtils.mkdir_p(TMP)
-      # curler, pointed at TMP
-      @curler = Curler.new(dir: TMP, verbose: false)
-    end
-    #
-    # tests
-    #
-    def test_200
-      Util.stub(:run, mock_curl_200) do
-        path = @curler.get("http://www.example.com")
-        assert_equal(HTML, File.read(path))
-      end
-    end
-    def test_500
-      assert_raises(Curler::Error) do
-        Util.stub(:run, mock_curl_500) do
-          @curler.get("http://www.example.com")
-        end
-      end
-    end
-    def test_cached
-      Util.stub(:run, mock_curl_200) do
-        assert_equal(HTML, File.read(@curler.get("http://www.example.com")))
-      end
-      # the file is cached, so this shouldn't produce an error
-      Util.stub(:run, mock_curl_500) do
-        @curler.get("http://www.example.com")
-      end
-    end
-    def test_302
-      Util.stub(:run, mock_curl_302) do
-        @curler.get("http://www.example.com")
-        assert_equal("http://www.gub.com", @curler.url)
-      end
-    end
-    def test_rate_limit
-      slept = false
-      # change Kernel#sleep to not really sleep!
-      Kernel.send(:alias_method, :old_sleep, :sleep)
-      Kernel.send(:define_method, :sleep) do |x|
-        slept = true
-      end
-      Util.stub(:run, mock_curl_200) do
-        @curler.get("http://www.example.com/1")
-        @curler.get("http://www.example.com/2")
-      end
-      assert(slept)
-      # restore old Kernel#sleep
-      Kernel.send(:alias_method, :sleep, :old_sleep)
-      Kernel.send(:undef_method, :old_sleep)
-    end
-  end
-end