RubyGems - sinew - Versions diffs - 1.0.4 → 2.0.0 - Mend

sinew 1.0.4 → 2.0.0

Files changed (38) hide show

checksums.yaml +5 -5
data/.gitignore +1 -0
data/.rubocop.yml +49 -0
data/.travis.yml +4 -0
data/.vscode/extensions.json +3 -0
data/.vscode/settings.json +15 -0
data/Gemfile +1 -1
data/README.md +153 -12
data/Rakefile +13 -14
data/bin/sinew +40 -20
data/lib/sinew.rb +10 -6
data/lib/sinew/cache.rb +79 -0
data/lib/sinew/core_ext.rb +59 -0
data/lib/sinew/dsl.rb +98 -0
data/lib/sinew/main.rb +80 -149
data/lib/sinew/nokogiri_ext.rb +10 -9
data/lib/sinew/output.rb +126 -0
data/lib/sinew/request.rb +148 -0
data/lib/sinew/response.rb +75 -0
data/lib/sinew/runtime_options.rb +26 -0
data/lib/sinew/version.rb +1 -1
data/sample.sinew +5 -3
data/sinew.gemspec +24 -19
data/test/test.html +40 -34
data/test/test_cache.rb +69 -0
data/test/test_helper.rb +113 -0
data/test/test_main.rb +36 -91
data/test/test_nokogiri_ext.rb +14 -15
data/test/test_output.rb +73 -0
data/test/test_requests.rb +135 -0
data/test/test_utf8.rb +39 -0
metadata +103 -48
data/lib/sinew/curler.rb +0 -173
data/lib/sinew/text_util.rb +0 -101
data/lib/sinew/util.rb +0 -236
data/test/helper.rb +0 -64
data/test/test_curler.rb +0 -70
data/test/test_text_util.rb +0 -23

data/test/test.html CHANGED

@@ -1,39 +1,45 @@
 <html>
-  <head>
-    <title>Title</title>
-    <script>
-      alert("alert 1");
-      alert("alert 2");
-    </script>
-  </head>
-  <body>
-    <div id="main">
-      <span class="class1"> text1 </span>
-      <span class="class2"> text2 </span>
-      <!-- for test_normalize -->
-      <div id="element">text</div>
-      <div class="e">text1</div>
-      <div class="e">text2</div>
-    </div>
-    <div id="nokogiri_ext">
-      <ul>
-        <li>hello</li>
-        <li>world</li>
-      </ul>
-      <div>
-        a
-        <p>b<span>c</span></p>
-        <p>b<span>c</span></p>
-      </div>
-    </div>
+<head>
+  <title>Title</title>
+  <script>
+    alert("alert 1");
+    alert("alert 2");
+  </script>
+</head>
+<body>
+  <div id="main">
+    <span class="class1"> text1 </span>
+    <span class="class2"> text2 </span>
-    <div id="text_util">
-      <!-- a comment that should be removed -->
-      <div class="will_be_removed"/>
-      <a class="will_be_preserved"/>
+    <!-- for test_normalize -->
+    <div id="element"> text </div>
+    <div class="e"> text1 </div>
+    <div class="e"> text2 </div>
+  </div>
+  <div id="nokogiri_ext">
+    <ul>
+      <li>hello</li>
+      <li>world</li>
+    </ul>
+    <div>
+      a
+      <p>b
+        <span>c</span>
+      </p>
+      <p>b
+        <span>c</span>
+      </p>
     </div>
-  </body>
+  </div>
+  <div id="text_util">
+    <!-- a comment that should be removed -->
+    <div class="will_be_removed" />
+    <a class="will_be_preserved" />
+  </div>
+</body>
 </html>

data/test/test_cache.rb ADDED

@@ -0,0 +1,69 @@
+require_relative 'test_helper'
+class TestCache < MiniTest::Test
+  def test_get
+    2.times do
+      sinew.dsl.get('http://httpbin.org/get', c: 3, d: 4)
+    end
+    if !test_network?
+      assert_requested :get, 'http://httpbin.org/get?c=3&d=4', times: 1
+    end
+    assert_equal 1, sinew.request_count
+    assert_equal({ c: '3', d: '4' }, sinew.dsl.json[:args])
+    assert File.exist?("#{TMP}/httpbin.org/get,c=3,d=4")
+    assert !File.exist?("#{TMP}/httpbin.org/head/get,c=3,d=4")
+  end
+  def test_post
+    2.times do
+      sinew.dsl.post('http://httpbin.org/post', c: 5, d: 6)
+    end
+    if !test_network?
+      assert_requested :post, 'http://httpbin.org/post', times: 1
+    end
+    assert_equal 1, sinew.request_count
+    assert_equal({ c: '5', d: '6' }, sinew.dsl.json[:form])
+  end
+  def test_redirect
+    2.times do
+      sinew.dsl.get('http://httpbin.org/redirect/2')
+    end
+    if !test_network?
+      assert_requested :get, 'http://httpbin.org/redirect/2', times: 1
+      assert_requested :get, 'http://httpbin.org/redirect/1', times: 1
+      assert_requested :get, 'http://httpbin.org/get', times: 1
+    end
+    assert_equal 1, sinew.request_count
+    assert_equal 'http://httpbin.org/get', sinew.dsl.url
+  end
+  def test_error
+    # gotta set this or the retries mess up our request counts
+    sinew.runtime_options.retries = 0
+    assert_output(/failed with 500/) do
+      2.times do
+        sinew.dsl.get('http://httpbin.org/status/500')
+      end
+    end
+    if !test_network?
+      assert_requested :get, 'http://httpbin.org/status/500', times: 1
+      assert_equal '500', sinew.dsl.raw
+    end
+    assert_equal 1, sinew.request_count
+  end
+  def test_timeout
+    return if test_network?
+    # gotta set this or the retries mess up our request counts
+    sinew.runtime_options.retries = 0
+    assert_output(/failed with 999/) do
+      2.times do
+        sinew.dsl.get('http://httpbin.org/delay/1')
+      end
+    end
+    assert_requested :get, 'http://httpbin.org/delay/1', times: 1
+    assert_equal 'timeout', sinew.dsl.raw
+  end
+end

data/test/test_helper.rb ADDED

@@ -0,0 +1,113 @@
+require 'minitest/autorun'
+require 'minitest/pride'
+require 'webmock/minitest' unless ENV['SINEW_TEST_NETWORK']
+# a hint to sinew, so that it'll do things like set rate limit to zero
+ENV['SINEW_TEST'] = '1'
+# Normally the Rakefile takes care of this, but it's handy to have it here when
+# running tests individually.
+$LOAD_PATH.unshift("#{__dir__}/../lib")
+require 'sinew'
+class MiniTest::Test
+  TMP = '/tmp/_test_sinew'.freeze
+  RECIPE = "#{TMP}/test.sinew".freeze
+  CSV = "#{TMP}/test.csv".freeze
+  HTML = File.read("#{__dir__}/test.html")
+  def setup
+    super
+    # prepare TMP
+    FileUtils.rm_rf(TMP)
+    FileUtils.mkdir_p(TMP)
+    stub_network unless test_network?
+  end
+  def sinew
+    @sinew ||= Sinew::Main.new(cache: TMP, quiet: true, recipe: RECIPE)
+  end
+  protected :sinew
+  def run_recipe(recipe)
+    File.write(RECIPE, recipe)
+    sinew.run
+  end
+  protected :run_recipe
+  def test_network?
+    !!ENV['SINEW_TEST_NETWORK']
+  end
+  protected :test_network?
+  # mock requests, patterned on httpbin
+  def stub_network
+    stub_request(:get, %r{http://[^/]+/html}).to_return(method(:respond_html))
+    stub_request(:get, %r{http://[^/]+/get\b}).to_return(method(:respond_echo))
+    stub_request(:post, %r{http://[^/]+/post\b}).to_return(method(:respond_echo))
+    stub_request(:get, %r{http://[^/]+/status/\d+}).to_return(method(:respond_status))
+    stub_request(:get, %r{http://[^/]+/(relative-)?redirect/\d+}).to_return(method(:respond_redirect))
+    stub_request(:get, %r{http://[^/]+/delay/\d+}).to_timeout
+  end
+  protected :stub_network
+  #
+  # respond_xxx helpers
+  #
+  def respond_html(_request)
+    # this html was carefully chosen to match httpbin.org/html
+    html = <<~EOF
+      <body>
+        <h1>Herman Melville - Moby-Dick</h1>
+      </body>
+    EOF
+    { body: html }
+  end
+  protected :respond_html
+  def respond_echo(request)
+    response = {}
+    response[:headers] = request.headers
+    # args
+    response[:args] = if request.uri.query
+      CGI.parse(request.uri.query).map { |k, v| [k, v.first] }.to_h
+    else
+      {}
+    end
+    # form
+    if request.headers['Content-Type'] == 'application/x-www-form-urlencoded'
+      response[:form] = CGI.parse(request.body).map { |k, v| [k, v.first] }.to_h
+    end
+    # json
+    if request.headers['Content-Type'] == 'application/json'
+      response[:json] = JSON.parse(request.body)
+    end
+    {
+      headers: { 'Content-Type' => 'application/json' },
+      body: response.to_json,
+    }
+  end
+  protected :respond_echo
+  def respond_status(request)
+    status = request.uri.to_s.split('/').last.to_i
+    { body: status.to_s, status: status }
+  end
+  protected :respond_status
+  def respond_redirect(request)
+    parts = request.uri.to_s.split('/')
+    path, count = parts[-2], parts[-1].to_i
+    url = count == 1 ? '/get' : "/#{path}/#{count - 1}"
+    url = "http://example#{url}" if path =~ /absolute/
+    { status: 302, headers: { 'Location' => url } }
+  end
+  protected :respond_redirect
+end

data/test/test_main.rb CHANGED

@@ -1,101 +1,46 @@
-# encoding: UTF-8
-require "helper"
-module Sinew
-  class TestMain < TestCase
-    RECIPE = "#{TMP}/test.sinew"
-    CSV    = "#{TMP}/test.csv"
-    def setup
-      # create TMP dir
-      FileUtils.rm_rf(TMP) if File.exists?(TMP)
-      FileUtils.mkdir_p(TMP)
-    end
-    def run_recipe(recipe)
-      File.write(RECIPE, recipe)
-      Util.stub(:run, mock_curl_200) do
-        Sinew::Main.new(cache: TMP, file: RECIPE, quiet: true)
+require_relative 'test_helper'
+class TestMain < MiniTest::Test
+  def test_noko
+    run_recipe <<~'EOF'
+      get 'http://httpbin.org/html'
+      noko.css("h1").each do |h1|
+        csv_emit(h1: h1.text)
       end
-    end
+    EOF
+    assert_equal("h1\nHerman Melville - Moby-Dick\n", File.read(CSV))
+  end
-    def test_noko
-      run_recipe <<'EOF'
-get "http://www.example.com"
-csv_header(:class, :text)
-noko.css("#main span").each do |span|
-  csv_emit(class: span[:class], text: span.text)
-end
-EOF
-      assert_equal("class,text\nclass1,text1\nclass2,text2\n", File.read(CSV))
-    end
+  def test_raw
+    run_recipe <<~'EOF'
+      get "http://httpbin.org/html"
+      raw.scan(/<h1>([^<]+)/) do
+        csv_emit(h1: $1)
+      end
+    EOF
+    assert_equal("h1\nHerman Melville - Moby-Dick\n", File.read(CSV))
+  end
-    def test_raw
-      # test javascript, which is only crawlable with raw
-      run_recipe <<'EOF'
-get "http://www.example.com"
-raw.scan(/alert\("([^"]+)/) do
-  csv_emit(alert: $1)
-end
-EOF
-      assert_equal("alert\nalert 1\nalert 2\n", File.read(CSV))
-    end
+  def test_rate_limit
+    # true network requests call sleep for timeouts, which interferes with our
+    # instrumentation of Kernel#sleep
+    skip if test_network?
-    def test_html
-      # note the cleaned up whitespace
-      run_recipe <<'EOF'
-get "http://www.example.com"
-csv_header(:class, :text)
-html.scan(/<span class="(\w+)">(\w+)/) do
-  csv_emit(class: $1, text: $2)
-end
-EOF
-      assert_equal("class,text\nclass1,text1\nclass2,text2\n", File.read(CSV))
-    end
+    slept = false
-    def test_clean
-      # note the removed attributes from span
-      run_recipe <<'EOF'
-get "http://www.example.com"
-clean.scan(/<span>(text\d)/) do
-  csv_emit(text: $1)
-end
-EOF
-      assert_equal("text\ntext1\ntext2\n", File.read(CSV))
+    # change Kernel#sleep to not really sleep!
+    Kernel.send(:alias_method, :old_sleep, :sleep)
+    Kernel.send(:define_method, :sleep) do |_duration|
+      slept = true
     end
-    def test_normalize
-      s = Sinew::Main.new(test: true)
-      #
-      # non-strings
-      #
-      noko = Nokogiri::HTML(HTML).css("#main")
-      # node => text
-      assert_equal("text", s.send(:_normalize, noko.css("#element")))
-      # nodes => text joined with space
-      assert_equal("text1 text2", s.send(:_normalize, noko.css(".e")))
-      # array => text joined with pipe
-      assert_equal("1|2", s.send(:_normalize, [1,2]))
+    sinew.runtime_options.rate_limit = 1
+    sinew.dsl.get('http://httpbin.org/html')
+    sinew.dsl.get('http://httpbin.org/get')
+    assert(slept)
-      #
-      # string cleanups
-      #
-      # untag
-      assert_equal("gub", s.send(:_normalize, "<tag>gub</tag>"))
-      # convert_accented_entities
-      assert_equal("a", s.send(:_normalize, "&aacute;"))
-      # unent
-      assert_equal("<>", s.send(:_normalize, "&lt;&gt;"))
-      # to_ascii
-      assert_equal("cafe", s.send(:_normalize, "caf\xc3\xa9"))
-      # squish
-      assert_equal("hello world", s.send(:_normalize, "\nhello \t \rworld"))
-    end
+    # restore old Kernel#sleep
+    Kernel.send(:alias_method, :sleep, :old_sleep)
+    Kernel.send(:undef_method, :old_sleep)
   end
 end

data/test/test_nokogiri_ext.rb CHANGED

@@ -1,19 +1,18 @@
-require "helper"
+require_relative 'test_helper'
-module Sinew
-  class TestNokogiriExt < TestCase
-    def setup
-      @noko = Nokogiri::HTML(HTML).css("#nokogiri_ext")
-    end
-    def test_inner_text
-      assert_equal("hello world", @noko.css("li").inner_text)
-      assert_equal("<li>hello</li> <li>world</li>", @noko.css("ul").inner_html.squish)
-    end
+class TestNokogiriExt < MiniTest::Test
+  def test_inner_text
+    assert_equal('hello world', noko.css('li').inner_text)
+    assert_equal('<li>hello</li> <li>world</li>', noko.css('ul').inner_html.squish)
+  end
+  def test_just_me
+    assert_equal('a', noko.css('div').text_just_me.squish)
+    assert_equal('b b', noko.css('p').text_just_me.squish)
+  end
-    def test_just_me
-      assert_equal("a", @noko.css("div").text_just_me.squish)
-      assert_equal("b b", @noko.css("p").text_just_me.squish)
-    end
+  def noko
+    @noko ||= Nokogiri::HTML(HTML).css('#nokogiri_ext')
   end
+  protected :noko
 end

data/test/test_output.rb ADDED

@@ -0,0 +1,73 @@
+require_relative 'test_helper'
+class TestOutput < MiniTest::Test
+  def test_output
+    sinew.dsl.csv_header(:n, :a, :p)
+    sinew.dsl.csv_emit(n: 'n1', a: 'a1')
+    sinew.dsl.csv_emit(n: 'n2', a: 'a2')
+    assert_equal 2, sinew.output.count
+    assert_equal "n,a,p\nn1,a1,\"\"\nn2,a2,\"\"\n", File.read(CSV)
+  end
+  def test_implicit_header
+    sinew.dsl.csv_emit(name: 'bob', address: 'main')
+    assert_equal "name,address\nbob,main\n", File.read(CSV)
+  end
+  def test_array_header
+    sinew.dsl.csv_header(%i[n a p])
+    sinew.dsl.csv_emit(n: 'n1', a: 'a1')
+    assert_equal "n,a,p\nn1,a1,\"\"\n", File.read(CSV)
+  end
+  def test_filenames
+    sinew = Sinew::Main.new(recipe: 'gub.sinew')
+    assert_equal 'gub.csv', sinew.output.filename
+    sinew = Sinew::Main.new(recipe: 'gub')
+    assert_equal 'gub.csv', sinew.output.filename
+    sinew = Sinew::Main.new(recipe: '/somewhere/gub.sinew')
+    assert_equal '/somewhere/gub.csv', sinew.output.filename
+  end
+  def test_normalization
+    output = Sinew::Output.new(nil)
+    #
+    # simple types
+    #
+    assert_equal '', output.send(:normalize, nil)
+    assert_equal '', output.send(:normalize, '')
+    assert_equal 'text', output.send(:normalize, 'text')
+    assert_equal '123', output.send(:normalize, 123)
+    assert_equal('1|2', output.send(:normalize, [ 1, 2 ]))
+    #
+    # nokogiri
+    #
+    noko = Nokogiri::HTML(HTML)
+    # node => text
+    assert_equal('text', output.send(:normalize, noko.css('#element')))
+    # nodes => text joined with space
+    assert_equal('text1 text2', output.send(:normalize, noko.css('.e')))
+    #
+    # string cleanups
+    #
+    # strip_html_tags
+    assert_equal('gub', output.send(:normalize, '<tag>gub</tag>'))
+    # convert_smart_punctuation
+    assert_equal('"gub"', output.send(:normalize, "\302\223gub\302\224"))
+    # convert_accented_html_entities
+    assert_equal('a', output.send(:normalize, '&aacute;'))
+    # convert_miscellaneous_html_entities
+    assert_equal('<>', output.send(:normalize, '&lt;&gt;'))
+    # to_ascii
+    assert_equal('cafe', output.send(:normalize, "caf\xc3\xa9"))
+    # squish
+    assert_equal('hello world', output.send(:normalize, "\nhello \t \rworld"))
+  end
+end