sinew 1.0.4 → 2.0.0

data/lib/sinew/core_ext.rb

@@ -0,0 +1,59 @@
+#
+# A few core extensions brought over from ActiveSupport. These are handy for
+# parsing.
+#
+
+class String
+  def squish
+    dup.squish!
+  end
+
+  def squish!
+    strip!
+    gsub!(/\s+/, ' ')
+    self
+  end
+
+  def first(limit = 1)
+    if limit == 0
+      ''
+    elsif limit >= size
+      dup
+    else
+      self[0..limit - 1]
+    end
+  end
+
+  def last(limit = 1)
+    if limit == 0
+      ''
+    elsif limit >= size
+      dup
+    else
+      self[-limit..-1]
+    end
+  end
+
+  alias starts_with? start_with?
+  alias ends_with? end_with?
+end
+
+#
+# blank?/present?
+#
+
+class Object
+  def blank?
+    respond_to?(:empty?) ? !!empty? : !self
+  end
+
+  def present?
+    !blank?
+  end
+end
+
+class String
+  def blank?
+    !!(self =~ /\A\s*\z/)
+  end
+end

data/lib/sinew/dsl.rb

@@ -0,0 +1,98 @@
+require 'awesome_print'
+require 'cgi'
+
+#
+# The DSL available to .sinew files.
+#
+
+module Sinew
+  class DSL
+    attr_reader :sinew, :raw, :uri, :elapsed
+
+    def initialize(sinew)
+      @sinew = sinew
+    end
+
+    def run
+      tm = Time.now
+      recipe = sinew.options[:recipe]
+      instance_eval(File.read(recipe, mode: 'rb'), recipe)
+      @elapsed = Time.now - tm
+    end
+
+    #
+    # request
+    #
+
+    def get(url, query = {})
+      http('get', url, query: query)
+    end
+
+    def post(url, form = {})
+      body = form
+      headers = {
+        'Content-Type' => 'application/x-www-form-urlencoded',
+      }
+      http('post', url, body: body, headers: headers)
+    end
+
+    def post_json(url, json = {})
+      body = json.to_json
+      headers = {
+        'Content-Type' => 'application/json',
+      }
+      http('post', url, body: body, headers: headers)
+    end
+
+    def http(method, url, options = {})
+      # reset
+      @html = @noko = @json = @url = nil
+
+      # fetch
+      response = sinew.http(method, url, options)
+
+      # respond
+      @uri = response.uri
+      @raw = response.body
+    end
+
+    #
+    # response
+    #
+
+    def html
+      @html ||= begin
+        s = raw.dup
+        # squish!
+        s.squish!
+        # kill whitespace around tags
+        s.gsub!(/ ?<([^>]+)> ?/, '<\\1>')
+        s
+      end
+    end
+
+    def noko
+      @noko ||= Nokogiri::HTML(html)
+    end
+
+    def json
+      @json ||= JSON.parse(raw, symbolize_names: true)
+    end
+
+    def url
+      uri.to_s
+    end
+
+    #
+    # csv
+    #
+
+    def csv_header(*args)
+      sinew.output.header(args)
+    end
+
+    def csv_emit(row)
+      sinew.output.emit(row)
+    end
+  end
+end

data/lib/sinew/main.rb

@@ -1,188 +1,119 @@
-require "nokogiri" # must be loaded before awesome_print
-require "awesome_print"
-require "cgi"
-require "csv"
-require "htmlentities"
-require "stringex"
+require 'scripto'
 
-module Sinew
-  class Main
-    CODER = HTMLEntities.new
+#
+# Main sinew entry point.
+#
 
-    attr_accessor :url, :uri, :raw
+module Sinew
+  class Main < Scripto::Main
+    attr_reader :runtime_options, :request_tm, :request_count
 
     def initialize(options)
-      @options = options.dup
-      _run if !@options[:test]
-    end
-
-    def get(url, params = nil)
-      _http(url, params, :get)
-    end
+      super(options)
 
-    def post(url, params = nil)
-      _http(url, params, :post)
+      # init
+      @runtime_options = RuntimeOptions.new
+      @request_tm = Time.at(0)
+      @request_count = 0
     end
 
-    #
-    # lazy accessors for cleaned up version
-    #
-
-    def html
-      @html ||= begin
-        s = TextUtil.html_tidy(@raw)
-        nelements = @raw.count("<")
-        if nelements > 1
-          # is there a problem with tidy?
-          percent = 100 * s.count("<") / nelements
-          if percent < 80
-            # bad xml processing instruction? Try fixing it.
-            maybe = TextUtil.html_tidy(@raw.gsub(/<\?[^>]*?>/, ""))
-            new_percent = 100 * maybe.count("<") / nelements
-            if new_percent > 80
-              # yes!
-              s = maybe
-            else
-              Util.warning "Hm - it looks like tidy ate some of your file (#{percent}%)" if percent < 90
-            end
-          end
-        end
-        s
-      end
+    def run
+      dsl.run
+      footer if !quiet?
     end
 
-    def clean
-      @clean ||= TextUtil.html_clean_from_tidy(self.html)
+    def quiet?
+      options[:quiet]
     end
 
-    def noko
-      @noko ||= Nokogiri::HTML(html)
+    def dsl
+      @dsl ||= DSL.new(self)
     end
 
     #
-    # csv
+    # http requests and caching
     #
 
-    def csv_header(*args)
-      args = args.flatten
-      if args.first.is_a?(String)
-        file = args.shift
-        if file !~ /^\//
-          file = "#{File.dirname(@options[:file])}/#{file}"
-        end
-      else
-        file = @options[:file]
-      end
-      ext = File.extname(file)
-      file = ext.empty? ? "#{file}.csv" : file.gsub(ext, ".csv")
-
-      @path = file
-      @csv = CSV.open(file, "wb")
-      @csv_keys = args
-      @csv << @csv_keys
-      _banner("Writing to #{@path}...")
+    def cache
+      @cache ||= Cache.new(self)
     end
 
-    def csv_emit(row, options = {})
-      csv_header(row.keys.sort) if !@csv
+    def http(method, url, options = {})
+      request = Request.new(self, method, url, options)
 
-      print = { }
-      row = @csv_keys.map do |i|
-        s = _normalize(row[i], i)
-        print[i] = s if !s.empty?
-        s
-      end
-      $stderr.puts print.ai if @options[:verbose]
-      @csv << row
-      @csv.flush
-    end
+      # try to get from cache
+      response = cache.get(request)
 
-    protected
+      # perform if necessary
+      if !response
+        response = perform(request)
+        cache.set(response)
+      end
 
-    def _curler
-      @curler ||= begin
-        # curler
-        options = { user_agent: "sinew/#{VERSION}" }
-        options[:dir] = @options[:cache] if @options[:cache]
-        options[:verbose] = false if @options[:quiet]
-        Curler.new(options)
+      # always log error messages
+      if response.error?
+        puts "xxx http request failed with #{response.code}"
       end
+
+      response
     end
 
-    def _run
-      @csv = @path = nil
+    def perform(request)
+      before_perform_request(request)
 
-      file = @options[:file]
-      if !File.exists?(file)
-        Util.fatal("#{file} not found")
-      end
+      response = nil
 
-      tm = Time.now
-      instance_eval(File.read(file, mode: "rb"), file)
-      if @path
-        _banner("Finished #{@path} in #{(Time.now - tm).to_i}s.")
-      else
-        _banner("Finished in #{(Time.now - tm).to_i}s.")
+      tries = runtime_options.retries + 1
+      while tries > 0
+        tries -= 1
+        begin
+          @request_count += 1
+          response = request.perform
+        rescue Timeout::Error
+          response = Response.from_timeout(request)
+        end
+        break if !response.error_500?
       end
+
+      response
     end
+    protected :perform
 
-    def _http(url, params, method)
-      url = url.to_s
-      raise "invalid url #{url.inspect}" if url !~ /^http/i
-
-      # decode entities
-      url = CODER.decode(url)
-
-      # handle params
-      body = nil
-      if params
-        q = params.map { |key, value| [CGI.escape(key.to_s), CGI.escape(value.to_s)] }.sort
-        q = q.map { |key, value| "#{key}=#{value}" }.join("&")
-        if method == :get
-          separator = url.include?(??) ? "&" : "?"
-          url = "#{url}#{separator}#{q}"
-        else
-          body = q
-        end
-      end
+    #
+    # output
+    #
 
-      begin
-        if method == :get
-          path = _curler.get(url)
+    def output
+      @output ||= Output.new(self)
+    end
+
+    #
+    # helpers
+    #
+
+    def before_perform_request(request)
+      # log
+      if !quiet?
+        msg = if request.method != 'get'
+          "req #{request.uri} (#{request.method})"
         else
-          path = _curler.post(url, body)
+          "req #{request.uri}"
         end
-        @raw = File.read(path, mode: "rb")
-      rescue Curler::Error => e
-        $stderr.puts "xxx #{e.message}"
-        @raw = ""
+        $stderr.puts msg
       end
 
-      # setup local variables
-      @url, @uri = _curler.url, _curler.uri
-      @html = nil
-      @clean = nil
-      @noko = nil
-    end
-
-    def _normalize(s, key = nil)
-      case s
-      when Nokogiri::XML::Element, Nokogiri::XML::NodeSet
-        s = s.inner_html
-      when Array
-        s = s.map { |j| j.to_s }.join("|")
-      else
-        s = s.to_s
-      end
-      s = TextUtil.untag(s)
-      s = s.convert_accented_html_entities
-      s = TextUtil.unent(s)
-      s = s.to_ascii.squish
-      s
+      # rate limit
+      sleep = (request_tm + runtime_options.rate_limit) - Time.now
+      sleep(sleep) if sleep > 0
+      @request_tm = Time.now
     end
+    protected :before_perform_request
 
-    def _banner(s)
-      Util.banner(s) if !@options[:quiet]
+    def footer
+      output.report
+      finished = output.count > 0 ? "Finished #{output.filename}" : 'Finished'
+      banner("#{finished} in #{dsl.elapsed.to_i}s.")
     end
+    protected :footer
   end
 end

data/lib/sinew/nokogiri_ext.rb

@@ -1,15 +1,16 @@
-require "nokogiri"
+require 'nokogiri'
 
 # modify NodeSet to join with SPACE instead of empty string
 class Nokogiri::XML::NodeSet
-  alias :old_inner_html :inner_html
-  alias :old_inner_text :inner_text  
-  
+  alias old_inner_html inner_html
+  alias old_inner_text inner_text
+
   def inner_text
-    collect { |i| i.inner_text }.join(" ")
+    map(&:inner_text).join(' ')
   end
-  def inner_html *args
-    collect { |i| i.inner_html(*args) }.join(" ")
+
+  def inner_html(*args)
+    map { |i| i.inner_html(*args) }.join(' ')
   end
 end
 
@@ -17,11 +18,11 @@ end
 class Nokogiri::XML::Node
   def text_just_me
     t = children.find { |i| i.node_type == Nokogiri::XML::Node::TEXT_NODE }
-    t && t.text
+    t&.text
   end
 end
 class Nokogiri::XML::NodeSet
   def text_just_me
-    map { |i| i.text_just_me }.join(" ")
+    map(&:text_just_me).join(' ')
   end
 end