sinew 2.0.5 → 3.0.0

data/.travis.yml

@@ -1,4 +0,0 @@
-language: ruby
-rvm:
-  - 2.3.7
-  - 2.5.1

data/lib/sinew/cache.rb

@@ -1,79 +0,0 @@
-require 'fileutils'
-require 'tempfile'
-
-#
-# This class handles the caching of http responses on disk.
-#
-
-module Sinew
-  class Cache
-    attr_reader :sinew
-
-    def initialize(sinew)
-      @sinew = sinew
-    end
-
-    def get(request)
-      body = read_if_exist(body_path(request))
-      return nil if !body
-
-      head = read_if_exist(head_path(request))
-      Response.from_cache(request, body, head)
-    end
-
-    def set(response)
-      body_path = body_path(response.request)
-      head_path = head_path(response.request)
-
-      FileUtils.mkdir_p(File.dirname(body_path))
-      FileUtils.mkdir_p(File.dirname(head_path))
-
-      # write body, and head if necessary
-      atomic_write(body_path, response.body)
-      if head_necessary?(response)
-        head = JSON.pretty_generate(response.head_as_json)
-        atomic_write(head_path, head)
-      end
-    end
-
-    def root_dir
-      sinew.options[:cache]
-    end
-    protected :root_dir
-
-    def head_necessary?(response)
-      response.error? || response.redirected?
-    end
-    protected :head_necessary?
-
-    def body_path(request)
-      "#{root_dir}/#{request.cache_key}"
-    end
-    protected :body_path
-
-    def head_path(request)
-      body_path = body_path(request)
-      dir, base = File.dirname(body_path), File.basename(body_path)
-      "#{dir}/head/#{base}"
-    end
-    protected :head_path
-
-    def read_if_exist(path)
-      if File.exist?(path)
-        IO.read(path, mode: 'r:UTF-8')
-      end
-    end
-    protected :read_if_exist
-
-    def atomic_write(path, data)
-      tmp = Tempfile.new('sinew', encoding: 'UTF-8')
-      tmp.write(data)
-      tmp.close
-      FileUtils.chmod(0o644, tmp.path)
-      FileUtils.mv(tmp.path, path)
-    ensure
-      FileUtils.rm(tmp.path, force: true)
-    end
-    protected :atomic_write
-  end
-end

data/test/legacy/eu.httpbin.org/head/redirect,3

@@ -1,51 +0,0 @@
-HTTP/1.1 302 FOUND
-Connection: keep-alive
-Server: gunicorn/19.7.1
-Date: Wed, 02 May 2018 20:55:20 GMT
-Content-Type: text/html; charset=utf-8
-Content-Length: 247
-Location: /relative-redirect/2
-Access-Control-Allow-Origin: *
-Access-Control-Allow-Credentials: true
-X-Powered-By: Flask
-X-Processed-Time: 0
-Via: 1.1 vegur
-
-HTTP/1.1 302 FOUND
-Connection: keep-alive
-Server: gunicorn/19.7.1
-Date: Wed, 02 May 2018 20:55:20 GMT
-Content-Type: text/html; charset=utf-8
-Content-Length: 0
-Location: /relative-redirect/1
-Access-Control-Allow-Origin: *
-Access-Control-Allow-Credentials: true
-X-Powered-By: Flask
-X-Processed-Time: 0
-Via: 1.1 vegur
-
-HTTP/1.1 302 FOUND
-Connection: keep-alive
-Server: gunicorn/19.7.1
-Date: Wed, 02 May 2018 20:55:20 GMT
-Content-Type: text/html; charset=utf-8
-Content-Length: 0
-Location: /get
-Access-Control-Allow-Origin: *
-Access-Control-Allow-Credentials: true
-X-Powered-By: Flask
-X-Processed-Time: 0
-Via: 1.1 vegur
-
-HTTP/1.1 200 OK
-Connection: keep-alive
-Server: gunicorn/19.7.1
-Date: Wed, 02 May 2018 20:55:20 GMT
-Content-Type: application/json
-Access-Control-Allow-Origin: *
-Access-Control-Allow-Credentials: true
-X-Powered-By: Flask
-X-Processed-Time: 0
-Content-Length: 220
-Via: 1.1 vegur
-

data/test/legacy/eu.httpbin.org/head/status,500

@@ -1 +0,0 @@
-CURLER_ERROR	curl error (22)

data/test/legacy/eu.httpbin.org/redirect,3

@@ -1,11 +0,0 @@
-{
-  "args": {}, 
-  "headers": {
-    "Accept": "*/*", 
-    "Connection": "close", 
-    "Host": "eu.httpbin.org", 
-    "User-Agent": "sinew/1.0.4"
-  }, 
-  "origin": "104.36.46.249", 
-  "url": "http://eu.httpbin.org/get"
-}

data/test/legacy/eu.httpbin.org/status,500

@@ -1 +0,0 @@
-

data/test/legacy/legacy.sinew

@@ -1,2 +0,0 @@
-get 'http://eu.httpbin.org/status/500'
-get 'http://eu.httpbin.org/redirect/3'

data/test/recipes/array_header.sinew

@@ -1,6 +0,0 @@
-csv_header(%i[n a p])
-csv_emit(n: 'n1', a: 'a1')
-
-# OUTPUT
-# n,a,p
-# n1,a1,""

data/test/recipes/basic.sinew

@@ -1,8 +0,0 @@
-get 'http://httpbin.org/html'
-raw.scan(/<h1>([^<]+)/) do
-  csv_emit(h1: $1)
-end
-
-# OUTPUT
-# h1
-# Herman Melville - Moby-Dick

data/test/recipes/dups.sinew

@@ -1,7 +0,0 @@
-5.times do
-  csv_emit(url: 'https://gub')
-end
-
-# OUTPUT
-# url
-# https://gub

data/test/recipes/implicit_header.sinew

@@ -1,5 +0,0 @@
-csv_emit(name: 'bob', address: 'main')
-
-# OUTPUT
-# name,address
-# bob,main

data/test/recipes/limit.sinew

@@ -1,11 +0,0 @@
-# OPTIONS { limit: 3 }
-
-(1..5).each do |i|
-  csv_emit(i: i)
-end
-
-# OUTPUT
-# i
-# 1
-# 2
-# 3

data/test/recipes/noko.sinew

@@ -1,9 +0,0 @@
-get 'http://httpbin.org/xml'
-noko.css('slide title').each do |title|
-  csv_emit(title: title.text)
-end
-
-# OUTPUT
-# title
-# Wake up to WonderWidgets!
-# Overview

data/test/recipes/uri.sinew

@@ -1,11 +0,0 @@
-# This tests get by URI, URI math, and csv_emit with uri
-get(URI.parse('http://httpbin.org/html'))
-csv_emit(url: uri)
-
-get(uri + '../get')
-csv_emit(url: uri)
-
-# OUTPUT
-# url
-# http://httpbin.org/html
-# http://httpbin.org/get

data/test/recipes/xml.sinew

@@ -1,8 +0,0 @@
-get 'http://httpbin.org/html'
-noko.css('h1').each do |h1|
-  csv_emit(h1: h1.text)
-end
-
-# OUTPUT
-# h1
-# Herman Melville - Moby-Dick

data/test/test.html

@@ -1,45 +0,0 @@
-<html>
-
-<head>
-  <title>Title</title>
-  <script>
-    alert("alert 1");
-    alert("alert 2");
-  </script>
-</head>
-
-<body>
-  <div id="main">
-    <span class="class1"> text1 </span>
-    <span class="class2"> text2 </span>
-
-    <!-- for test_normalize -->
-    <div id="element"> text </div>
-    <div class="e"> text1 </div>
-    <div class="e"> text2 </div>
-  </div>
-
-  <div id="nokogiri_ext">
-    <ul>
-      <li>hello</li>
-      <li>world</li>
-    </ul>
-    <div>
-      a
-      <p>b
-        <span>c</span>
-      </p>
-      <p>b
-        <span>c</span>
-      </p>
-    </div>
-  </div>
-
-  <div id="text_util">
-    <!-- a comment that should be removed -->
-    <div class="will_be_removed" />
-    <a class="will_be_preserved" />
-  </div>
-</body>
-
-</html>

data/test/test_cache.rb

@@ -1,69 +0,0 @@
-require_relative 'test_helper'
-
-class TestCache < MiniTest::Test
-  def test_get
-    2.times do
-      sinew.dsl.get('http://httpbin.org/get', c: 3, d: 4)
-    end
-    if !test_network?
-      assert_requested :get, 'http://httpbin.org/get?c=3&d=4', times: 1
-    end
-    assert_equal 1, sinew.request_count
-    assert_equal({ c: '3', d: '4' }, sinew.dsl.json[:args])
-    assert File.exist?("#{TMP}/httpbin.org/get,c=3,d=4")
-    assert !File.exist?("#{TMP}/httpbin.org/head/get,c=3,d=4")
-  end
-
-  def test_post
-    2.times do
-      sinew.dsl.post('http://httpbin.org/post', c: 5, d: 6)
-    end
-    if !test_network?
-      assert_requested :post, 'http://httpbin.org/post', times: 1
-    end
-    assert_equal 1, sinew.request_count
-    assert_equal({ c: '5', d: '6' }, sinew.dsl.json[:form])
-  end
-
-  def test_redirect
-    2.times do
-      sinew.dsl.get('http://httpbin.org/redirect/2')
-    end
-    if !test_network?
-      assert_requested :get, 'http://httpbin.org/redirect/2', times: 1
-      assert_requested :get, 'http://httpbin.org/redirect/1', times: 1
-      assert_requested :get, 'http://httpbin.org/get', times: 1
-    end
-    assert_equal 1, sinew.request_count
-    assert_equal 'http://httpbin.org/get', sinew.dsl.url
-  end
-
-  def test_error
-    # gotta set this or the retries mess up our request counts
-    sinew.runtime_options.retries = 0
-    assert_output(/failed with 500/) do
-      2.times do
-        sinew.dsl.get('http://httpbin.org/status/500')
-      end
-    end
-    if !test_network?
-      assert_requested :get, 'http://httpbin.org/status/500', times: 1
-      assert_equal '500', sinew.dsl.raw
-    end
-    assert_equal 1, sinew.request_count
-  end
-
-  def test_timeout
-    return if test_network?
-
-    # gotta set this or the retries mess up our request counts
-    sinew.runtime_options.retries = 0
-    assert_output(/failed with 999/) do
-      2.times do
-        sinew.dsl.get('http://httpbin.org/delay/1')
-      end
-    end
-    assert_requested :get, 'http://httpbin.org/delay/1', times: 1
-    assert_equal 'execution expired', sinew.dsl.raw
-  end
-end

data/test/test_helper.rb

@@ -1,126 +0,0 @@
-require 'minitest/autorun'
-require 'minitest/pride'
-require 'webmock/minitest' unless ENV['SINEW_TEST_NETWORK']
-
-# to run one test, do this:
-# TEST=test/unit/query_test.rb TESTOPTS="--name=test_parse" rake
-
-# a hint to sinew, so that it'll do things like set rate limit to zero
-ENV['SINEW_TEST'] = '1'
-
-# Normally the Rakefile takes care of this, but it's handy to have it here when
-# running tests individually.
-$LOAD_PATH.unshift("#{__dir__}/../lib")
-require 'sinew'
-
-class MiniTest::Test
-  TMP = '/tmp/_test_sinew'.freeze
-  HTML = File.read("#{__dir__}/test.html")
-
-  def setup
-    super
-
-    # prepare TMP
-    FileUtils.rm_rf(TMP)
-    FileUtils.mkdir_p(TMP)
-
-    stub_network unless test_network?
-  end
-
-  def sinew
-    @sinew ||= Sinew::Main.new(cache: TMP, quiet: true, recipe: "#{TMP}/ignore.sinew")
-  end
-  protected :sinew
-
-  def test_network?
-    !!ENV['SINEW_TEST_NETWORK']
-  end
-  protected :test_network?
-
-  # mock requests, patterned on httpbin
-  def stub_network
-    stub_request(:get, %r{http://[^/]+/html}).to_return(method(:respond_html))
-    stub_request(:get, %r{http://[^/]+/get\b}).to_return(method(:respond_echo))
-    stub_request(:post, %r{http://[^/]+/post\b}).to_return(method(:respond_echo))
-    stub_request(:get, %r{http://[^/]+/status/\d+}).to_return(method(:respond_status))
-    stub_request(:get, %r{http://[^/]+/(relative-)?redirect/\d+}).to_return(method(:respond_redirect))
-    stub_request(:get, %r{http://[^/]+/delay/\d+}).to_timeout
-    stub_request(:get, %r{http://[^/]+/xml}).to_return(method(:respond_xml))
-  end
-  protected :stub_network
-
-  #
-  # respond_xxx helpers
-  #
-
-  def respond_html(_request)
-    # this html was carefully chosen to somewhat match httpbin.org/html
-    html = <<~EOF
-      <body>
-        <h1>Herman Melville - Moby-Dick</h1>
-      </body>
-    EOF
-    { body: html }
-  end
-  protected :respond_html
-
-  def respond_xml(_request)
-    # this xml was carefully chosen to somewhat match httpbin.org/xml
-    xml = <<~EOF
-      <!--   A SAMPLE set of slides   -->
-      <slideshow>
-        <slide type="all">
-          <title>Wake up to WonderWidgets!</title>
-        </slide>
-        <slide type="all">
-          <title>Overview</title>
-        </slide>
-      </slideshow>
-    EOF
-    { body: xml }
-  end
-  protected :respond_xml
-
-  def respond_echo(request)
-    response = {}
-    response[:headers] = request.headers
-
-    # args
-    response[:args] = if request.uri.query
-      CGI.parse(request.uri.query).map { |k, v| [ k, v.first ] }.to_h
-    else
-      {}
-    end
-
-    # form
-    if request.headers['Content-Type'] == 'application/x-www-form-urlencoded'
-      response[:form] = CGI.parse(request.body).map { |k, v| [ k, v.first ] }.to_h
-    end
-
-    # json
-    if request.headers['Content-Type'] == 'application/json'
-      response[:json] = JSON.parse(request.body)
-    end
-
-    {
-      headers: { 'Content-Type' => 'application/json' },
-      body: response.to_json,
-    }
-  end
-  protected :respond_echo
-
-  def respond_status(request)
-    status = request.uri.to_s.split('/').last.to_i
-    { body: status.to_s, status: status }
-  end
-  protected :respond_status
-
-  def respond_redirect(request)
-    parts = request.uri.to_s.split('/')
-    path, count = parts[-2], parts[-1].to_i
-    url = count == 1 ? '/get' : "/#{path}/#{count - 1}"
-    url = "http://example#{url}" if path =~ /absolute/
-    { status: 302, headers: { 'Location' => url } }
-  end
-  protected :respond_redirect
-end