RubyGems - sinew - Versions diffs - 1.0.4 → 2.0.0 - Mend

sinew 1.0.4 → 2.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (38) hide show

checksums.yaml +5 -5
data/.gitignore +1 -0
data/.rubocop.yml +49 -0
data/.travis.yml +4 -0
data/.vscode/extensions.json +3 -0
data/.vscode/settings.json +15 -0
data/Gemfile +1 -1
data/README.md +153 -12
data/Rakefile +13 -14
data/bin/sinew +40 -20
data/lib/sinew.rb +10 -6
data/lib/sinew/cache.rb +79 -0
data/lib/sinew/core_ext.rb +59 -0
data/lib/sinew/dsl.rb +98 -0
data/lib/sinew/main.rb +80 -149
data/lib/sinew/nokogiri_ext.rb +10 -9
data/lib/sinew/output.rb +126 -0
data/lib/sinew/request.rb +148 -0
data/lib/sinew/response.rb +75 -0
data/lib/sinew/runtime_options.rb +26 -0
data/lib/sinew/version.rb +1 -1
data/sample.sinew +5 -3
data/sinew.gemspec +24 -19
data/test/test.html +40 -34
data/test/test_cache.rb +69 -0
data/test/test_helper.rb +113 -0
data/test/test_main.rb +36 -91
data/test/test_nokogiri_ext.rb +14 -15
data/test/test_output.rb +73 -0
data/test/test_requests.rb +135 -0
data/test/test_utf8.rb +39 -0
metadata +103 -48
data/lib/sinew/curler.rb +0 -173
data/lib/sinew/text_util.rb +0 -101
data/lib/sinew/util.rb +0 -236
data/test/helper.rb +0 -64
data/test/test_curler.rb +0 -70
data/test/test_text_util.rb +0 -23

data/test/test.html CHANGED

@@ -1,39 +1,45 @@
 <html>
-  <head>
-    <title>Title</title>
-    <script>
-      alert("alert 1");
-      alert("alert 2");
-    </script>
-  </head>
-  <body>
-    <div id="main">
-      <span class="class1"> text1 </span>
-      <span class="class2"> text2 </span>
-      <!-- for test_normalize -->
-      <div id="element">text</div>
-      <div class="e">text1</div>
-      <div class="e">text2</div>
-    </div>
-    <div id="nokogiri_ext">
-      <ul>
-        <li>hello</li>
-        <li>world</li>
-      </ul>
-      <div>
-        a
-        <p>b<span>c</span></p>
-        <p>b<span>c</span></p>
-      </div>
-    </div>
+<head>
+  <title>Title</title>
+  <script>
+    alert("alert 1");
+    alert("alert 2");
+  </script>
+</head>
+<body>
+  <div id="main">
+    <span class="class1"> text1 </span>
+    <span class="class2"> text2 </span>
-    <div id="text_util">
-      <!-- a comment that should be removed -->
-      <div class="will_be_removed"/>
-      <a class="will_be_preserved"/>
+    <!-- for test_normalize -->
+    <div id="element"> text </div>
+    <div class="e"> text1 </div>
+    <div class="e"> text2 </div>
+  </div>
+  <div id="nokogiri_ext">
+    <ul>
+      <li>hello</li>
+      <li>world</li>
+    </ul>
+    <div>
+      a
+      <p>b
+        <span>c</span>
+      </p>
+      <p>b
+        <span>c</span>
+      </p>
     </div>
-  </body>
+  </div>
+  <div id="text_util">
+    <!-- a comment that should be removed -->
+    <div class="will_be_removed" />
+    <a class="will_be_preserved" />
+  </div>
+</body>
 </html>

data/test/test_cache.rb ADDED

@@ -0,0 +1,69 @@
+require_relative 'test_helper'
+class TestCache < MiniTest::Test
+  def test_get
+    2.times do
+      sinew.dsl.get('http://httpbin.org/get', c: 3, d: 4)
+    end
+    if !test_network?
+      assert_requested :get, 'http://httpbin.org/get?c=3&d=4', times: 1
+    end
+    assert_equal 1, sinew.request_count
+    assert_equal({ c: '3', d: '4' }, sinew.dsl.json[:args])
+    assert File.exist?("#{TMP}/httpbin.org/get,c=3,d=4")
+    assert !File.exist?("#{TMP}/httpbin.org/head/get,c=3,d=4")
+  end
+  def test_post
+    2.times do
+      sinew.dsl.post('http://httpbin.org/post', c: 5, d: 6)
+    end
+    if !test_network?
+      assert_requested :post, 'http://httpbin.org/post', times: 1
+    end
+    assert_equal 1, sinew.request_count
+    assert_equal({ c: '5', d: '6' }, sinew.dsl.json[:form])
+  end
+  def test_redirect
+    2.times do
+      sinew.dsl.get('http://httpbin.org/redirect/2')
+    end
+    if !test_network?
+      assert_requested :get, 'http://httpbin.org/redirect/2', times: 1
+      assert_requested :get, 'http://httpbin.org/redirect/1', times: 1
+      assert_requested :get, 'http://httpbin.org/get', times: 1
+    end
+    assert_equal 1, sinew.request_count
+    assert_equal 'http://httpbin.org/get', sinew.dsl.url
+  end
+  def test_error
+    # gotta set this or the retries mess up our request counts
+    sinew.runtime_options.retries = 0
+    assert_output(/failed with 500/) do
+      2.times do
+        sinew.dsl.get('http://httpbin.org/status/500')
+      end
+    end
+    if !test_network?
+      assert_requested :get, 'http://httpbin.org/status/500', times: 1
+      assert_equal '500', sinew.dsl.raw
+    end
+    assert_equal 1, sinew.request_count
+  end
+  def test_timeout
+    return if test_network?
+    # gotta set this or the retries mess up our request counts
+    sinew.runtime_options.retries = 0
+    assert_output(/failed with 999/) do
+      2.times do
+        sinew.dsl.get('http://httpbin.org/delay/1')
+      end
+    end
+    assert_requested :get, 'http://httpbin.org/delay/1', times: 1
+    assert_equal 'timeout', sinew.dsl.raw
+  end
+end

data/test/test_helper.rb ADDED

@@ -0,0 +1,113 @@
+require 'minitest/autorun'
+require 'minitest/pride'
+require 'webmock/minitest' unless ENV['SINEW_TEST_NETWORK']
+# a hint to sinew, so that it'll do things like set rate limit to zero
+ENV['SINEW_TEST'] = '1'
+# Normally the Rakefile takes care of this, but it's handy to have it here when
+# running tests individually.
+$LOAD_PATH.unshift("#{__dir__}/../lib")
+require 'sinew'
+class MiniTest::Test
+  TMP = '/tmp/_test_sinew'.freeze
+  RECIPE = "#{TMP}/test.sinew".freeze
+  CSV = "#{TMP}/test.csv".freeze
+  HTML = File.read("#{__dir__}/test.html")
+  def setup
+    super
+    # prepare TMP
+    FileUtils.rm_rf(TMP)
+    FileUtils.mkdir_p(TMP)
+    stub_network unless test_network?
+  end
+  def sinew
+    @sinew ||= Sinew::Main.new(cache: TMP, quiet: true, recipe: RECIPE)
+  end
+  protected :sinew
+  def run_recipe(recipe)
+    File.write(RECIPE, recipe)
+    sinew.run
+  end
+  protected :run_recipe
+  def test_network?
+    !!ENV['SINEW_TEST_NETWORK']
+  end
+  protected :test_network?
+  # mock requests, patterned on httpbin
+  def stub_network
+    stub_request(:get, %r{http://[^/]+/html}).to_return(method(:respond_html))
+    stub_request(:get, %r{http://[^/]+/get\b}).to_return(method(:respond_echo))
+    stub_request(:post, %r{http://[^/]+/post\b}).to_return(method(:respond_echo))
+    stub_request(:get, %r{http://[^/]+/status/\d+}).to_return(method(:respond_status))
+    stub_request(:get, %r{http://[^/]+/(relative-)?redirect/\d+}).to_return(method(:respond_redirect))
+    stub_request(:get, %r{http://[^/]+/delay/\d+}).to_timeout
+  end
+  protected :stub_network
+  #
+  # respond_xxx helpers
+  #
+  def respond_html(_request)
+    # this html was carefully chosen to match httpbin.org/html
+    html = <<~EOF
+      <body>
+        <h1>Herman Melville - Moby-Dick</h1>
+      </body>
+    EOF
+    { body: html }
+  end
+  protected :respond_html
+  def respond_echo(request)
+    response = {}
+    response[:headers] = request.headers
+    # args
+    response[:args] = if request.uri.query
+      CGI.parse(request.uri.query).map { |k, v| [k, v.first] }.to_h
+    else
+      {}
+    end
+    # form
+    if request.headers['Content-Type'] == 'application/x-www-form-urlencoded'
+      response[:form] = CGI.parse(request.body).map { |k, v| [k, v.first] }.to_h
+    end
+    # json
+    if request.headers['Content-Type'] == 'application/json'
+      response[:json] = JSON.parse(request.body)
+    end
+    {
+      headers: { 'Content-Type' => 'application/json' },
+      body: response.to_json,
+    }
+  end
+  protected :respond_echo
+  def respond_status(request)
+    status = request.uri.to_s.split('/').last.to_i
+    { body: status.to_s, status: status }
+  end
+  protected :respond_status
+  def respond_redirect(request)
+    parts = request.uri.to_s.split('/')
+    path, count = parts[-2], parts[-1].to_i
+    url = count == 1 ? '/get' : "/#{path}/#{count - 1}"
+    url = "http://example#{url}" if path =~ /absolute/
+    { status: 302, headers: { 'Location' => url } }
+  end
+  protected :respond_redirect
+end

data/test/test_main.rb CHANGED

@@ -1,101 +1,46 @@
-# encoding: UTF-8
-require "helper"
-module Sinew
-  class TestMain < TestCase
-    RECIPE = "#{TMP}/test.sinew"
-    CSV    = "#{TMP}/test.csv"
-    def setup
-      # create TMP dir
-      FileUtils.rm_rf(TMP) if File.exists?(TMP)
-      FileUtils.mkdir_p(TMP)
-    end
-    def run_recipe(recipe)
-      File.write(RECIPE, recipe)
-      Util.stub(:run, mock_curl_200) do
-        Sinew::Main.new(cache: TMP, file: RECIPE, quiet: true)
+require_relative 'test_helper'
+class TestMain < MiniTest::Test
+  def test_noko
+    run_recipe <<~'EOF'
+      get 'http://httpbin.org/html'
+      noko.css("h1").each do |h1|
+        csv_emit(h1: h1.text)
       end
-    end
+    EOF
+    assert_equal("h1\nHerman Melville - Moby-Dick\n", File.read(CSV))
+  end
-    def test_noko
-      run_recipe <<'EOF'
-get "http://www.example.com"
-csv_header(:class, :text)
-noko.css("#main span").each do |span|
-  csv_emit(class: span[:class], text: span.text)
-end
-EOF
-      assert_equal("class,text\nclass1,text1\nclass2,text2\n", File.read(CSV))
-    end
+  def test_raw
+    run_recipe <<~'EOF'
+      get "http://httpbin.org/html"
+      raw.scan(/<h1>([^<]+)/) do
+        csv_emit(h1: $1)
+      end
+    EOF
+    assert_equal("h1\nHerman Melville - Moby-Dick\n", File.read(CSV))
+  end
-    def test_raw
-      # test javascript, which is only crawlable with raw
-      run_recipe <<'EOF'
-get "http://www.example.com"
-raw.scan(/alert\("([^"]+)/) do
-  csv_emit(alert: $1)
-end
-EOF
-      assert_equal("alert\nalert 1\nalert 2\n", File.read(CSV))
-    end
+  def test_rate_limit
+    # true network requests call sleep for timeouts, which interferes with our
+    # instrumentation of Kernel#sleep
+    skip if test_network?
-    def test_html
-      # note the cleaned up whitespace
-      run_recipe <<'EOF'
-get "http://www.example.com"
-csv_header(:class, :text)
-html.scan(/<span class="(\w+)">(\w+)/) do
-  csv_emit(class: $1, text: $2)
-end
-EOF
-      assert_equal("class,text\nclass1,text1\nclass2,text2\n", File.read(CSV))
-    end
+    slept = false
-    def test_clean
-      # note the removed attributes from span
-      run_recipe <<'EOF'
-get "http://www.example.com"
-clean.scan(/<span>(text\d)/) do
-  csv_emit(text: $1)
-end
-EOF
-      assert_equal("text\ntext1\ntext2\n", File.read(CSV))
+    # change Kernel#sleep to not really sleep!
+    Kernel.send(:alias_method, :old_sleep, :sleep)
+    Kernel.send(:define_method, :sleep) do |_duration|
+      slept = true
     end
-    def test_normalize
-      s = Sinew::Main.new(test: true)
-      #
-      # non-strings
-      #
-      noko = Nokogiri::HTML(HTML).css("#main")
-      # node => text
-      assert_equal("text", s.send(:_normalize, noko.css("#element")))
-      # nodes => text joined with space
-      assert_equal("text1 text2", s.send(:_normalize, noko.css(".e")))
-      # array => text joined with pipe
-      assert_equal("1|2", s.send(:_normalize, [1,2]))
+    sinew.runtime_options.rate_limit = 1
+    sinew.dsl.get('http://httpbin.org/html')
+    sinew.dsl.get('http://httpbin.org/get')
+    assert(slept)
-      #
-      # string cleanups
-      #
-      # untag
-      assert_equal("gub", s.send(:_normalize, "<tag>gub</tag>"))
-      # convert_accented_entities
-      assert_equal("a", s.send(:_normalize, "&aacute;"))
-      # unent
-      assert_equal("<>", s.send(:_normalize, "&lt;&gt;"))
-      # to_ascii
-      assert_equal("cafe", s.send(:_normalize, "caf\xc3\xa9"))
-      # squish
-      assert_equal("hello world", s.send(:_normalize, "\nhello \t \rworld"))
-    end
+    # restore old Kernel#sleep
+    Kernel.send(:alias_method, :sleep, :old_sleep)
+    Kernel.send(:undef_method, :old_sleep)
   end
 end

data/test/test_nokogiri_ext.rb CHANGED

@@ -1,19 +1,18 @@
-require "helper"
+require_relative 'test_helper'
-module Sinew
-  class TestNokogiriExt < TestCase
-    def setup
-      @noko = Nokogiri::HTML(HTML).css("#nokogiri_ext")
-    end
-    def test_inner_text
-      assert_equal("hello world", @noko.css("li").inner_text)
-      assert_equal("<li>hello</li> <li>world</li>", @noko.css("ul").inner_html.squish)
-    end
+class TestNokogiriExt < MiniTest::Test
+  def test_inner_text
+    assert_equal('hello world', noko.css('li').inner_text)
+    assert_equal('<li>hello</li> <li>world</li>', noko.css('ul').inner_html.squish)
+  end
+  def test_just_me
+    assert_equal('a', noko.css('div').text_just_me.squish)
+    assert_equal('b b', noko.css('p').text_just_me.squish)
+  end
-    def test_just_me
-      assert_equal("a", @noko.css("div").text_just_me.squish)
-      assert_equal("b b", @noko.css("p").text_just_me.squish)
-    end
+  def noko
+    @noko ||= Nokogiri::HTML(HTML).css('#nokogiri_ext')
   end
+  protected :noko
 end

data/test/test_output.rb ADDED

@@ -0,0 +1,73 @@
+require_relative 'test_helper'
+class TestOutput < MiniTest::Test
+  def test_output
+    sinew.dsl.csv_header(:n, :a, :p)
+    sinew.dsl.csv_emit(n: 'n1', a: 'a1')
+    sinew.dsl.csv_emit(n: 'n2', a: 'a2')
+    assert_equal 2, sinew.output.count
+    assert_equal "n,a,p\nn1,a1,\"\"\nn2,a2,\"\"\n", File.read(CSV)
+  end
+  def test_implicit_header
+    sinew.dsl.csv_emit(name: 'bob', address: 'main')
+    assert_equal "name,address\nbob,main\n", File.read(CSV)
+  end
+  def test_array_header
+    sinew.dsl.csv_header(%i[n a p])
+    sinew.dsl.csv_emit(n: 'n1', a: 'a1')
+    assert_equal "n,a,p\nn1,a1,\"\"\n", File.read(CSV)
+  end
+  def test_filenames
+    sinew = Sinew::Main.new(recipe: 'gub.sinew')
+    assert_equal 'gub.csv', sinew.output.filename
+    sinew = Sinew::Main.new(recipe: 'gub')
+    assert_equal 'gub.csv', sinew.output.filename
+    sinew = Sinew::Main.new(recipe: '/somewhere/gub.sinew')
+    assert_equal '/somewhere/gub.csv', sinew.output.filename
+  end
+  def test_normalization
+    output = Sinew::Output.new(nil)
+    #
+    # simple types
+    #
+    assert_equal '', output.send(:normalize, nil)
+    assert_equal '', output.send(:normalize, '')
+    assert_equal 'text', output.send(:normalize, 'text')
+    assert_equal '123', output.send(:normalize, 123)
+    assert_equal('1|2', output.send(:normalize, [ 1, 2 ]))
+    #
+    # nokogiri
+    #
+    noko = Nokogiri::HTML(HTML)
+    # node => text
+    assert_equal('text', output.send(:normalize, noko.css('#element')))
+    # nodes => text joined with space
+    assert_equal('text1 text2', output.send(:normalize, noko.css('.e')))
+    #
+    # string cleanups
+    #
+    # strip_html_tags
+    assert_equal('gub', output.send(:normalize, '<tag>gub</tag>'))
+    # convert_smart_punctuation
+    assert_equal('"gub"', output.send(:normalize, "\302\223gub\302\224"))
+    # convert_accented_html_entities
+    assert_equal('a', output.send(:normalize, '&aacute;'))
+    # convert_miscellaneous_html_entities
+    assert_equal('<>', output.send(:normalize, '&lt;&gt;'))
+    # to_ascii
+    assert_equal('cafe', output.send(:normalize, "caf\xc3\xa9"))
+    # squish
+    assert_equal('hello world', output.send(:normalize, "\nhello \t \rworld"))
+  end
+end