stark-rack 1.0.0

data/.autotest

@@ -0,0 +1,23 @@
+# -*- ruby -*-
+
+require 'autotest/restart'
+
+# Autotest.add_hook :initialize do |at|
+#   at.extra_files << "../some/external/dependency.rb"
+#
+#   at.libs << ":../some/external"
+#
+#   at.add_exception 'vendor'
+#
+#   at.add_mapping(/dependency.rb/) do |f, _|
+#     at.files_matching(/test_.*rb$/)
+#   end
+#
+#   %w(TestA TestB).each do |klass|
+#     at.extra_class_map[klass] = "test/test_misc.rb"
+#   end
+# end
+
+# Autotest.add_hook :run_command do |at|
+#   system "rake build"
+# end

data/History.txt

@@ -0,0 +1,6 @@
+=== 1.0.0 / 2013-05-22
+
+* 1 major enhancement
+
+  * Birthday!
+

data/Manifest.txt

@@ -0,0 +1,23 @@
+.autotest
+.gemtest
+History.txt
+Manifest.txt
+README.txt
+Rakefile
+lib/stark/rack.rb
+lib/stark/rack/content_negotiation.rb
+lib/stark/rack/logging_processor.rb
+lib/stark/rack/metadata.rb
+lib/stark/rack/rest.rb
+lib/stark/rack/verbose_protocol.rb
+stark-rack.gemspec
+test/calc-opt.rb
+test/calc.thrift
+test/config.ru
+test/gen-rb/calc.rb
+test/gen-rb/calc_constants.rb
+test/gen-rb/calc_types.rb
+test/helper.rb
+test/test_metadata.rb
+test/test_rack.rb
+test/test_rest.rb

data/README.txt

@@ -0,0 +1,61 @@
+= stark-rack
+
+* https://github.com/evanphx/stark-rack
+
+== DESCRIPTION:
+
+Provides middleware for mounting Stark/Thrift services as Rack endpoints.
+
+== FEATURES/PROBLEMS:
+
+* Thrift binary protocol by default
+* JSON protocol with 'Accept: application/json' in request headers
+
+== SYNOPSIS:
+
+  # config.ru
+  use Stark::Rack
+  use MyService::Processor  # Stark/Thrift processor
+  run MyHandler.new         # Handler that implements service
+
+== REQUIREMENTS:
+
+* Rack (optional; only required with Stark::Rack::REST middleware)
+
+== INSTALL:
+
+* gem install stark-rack
+
+== DEVELOPERS:
+
+After checking out the source, run:
+
+  $ rake newb
+
+This task will install any missing dependencies, run the tests/specs,
+and generate the RDoc.
+
+== LICENSE:
+
+(The MIT License)
+
+Copyright (c) 2012, 2013 Evan Phoenix
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+'Software'), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/Rakefile

@@ -0,0 +1,21 @@
+# -*- ruby -*-
+
+require 'rubygems'
+require 'hoe'
+
+# Don't turn on warnings, output is very ugly w/ generated code
+Hoe::RUBY_FLAGS.sub! /-w/, ''
+# Add stark to path if we have it checked out locally
+stark_local_path = File.expand_path('../../stark/lib', __FILE__)
+Hoe::RUBY_FLAGS.concat " -I#{stark_local_path}" if File.directory?(stark_local_path)
+
+Hoe.plugin :git
+Hoe.plugin :gemspec
+
+Hoe.spec 'stark-rack' do
+  developer('Evan Phoenix', 'evan@phx.io')
+  dependency 'stark', '< 2.0.0'
+  dependency 'rack', '>= 1.5.0', :dev
+end
+
+# vim: syntax=ruby

data/lib/stark/rack.rb

@@ -0,0 +1,94 @@
+require 'stark'
+require 'stark/rack/content_negotiation'
+require 'stark/rack/logging_processor'
+require 'stark/rack/verbose_protocol'
+
+class Stark::Rack
+
+  VERSION = '1.0.0'
+
+  FORMAT = %{when: %0.4f, client: "%s", path: "%s%s", type: "%s", name: "%s", seqid: %d, error: %s\n}
+
+  TYPES = {
+    1 => "CALL",
+    2 => "REPLY",
+    3 => "EXCEPTION",
+    4 => "ONEWAY"
+  }
+
+  include ContentNegotiation
+
+  def initialize(processor, options={})
+    @log = options[:log]
+    @logger = STDERR
+    @app_processor = processor
+  end
+
+  attr_accessor :log
+
+  def processor
+    @processor ||= LoggingProcessor.new(@app_processor, error_capture)
+  end
+
+  def call(env)
+    dup._call(env)
+  end
+
+  def _call(env)
+    path = env['PATH_INFO'] || ""
+    path << "/" if path.empty?
+
+    if env["REQUEST_METHOD"] != "POST"
+      return [405, {"Content-Type" => "text/plain"},
+              ["Method #{env["REQUEST_METHOD"]} not allowed, must be POST\n"]]
+    end
+
+    unless path == "/"
+      return [404, {"Content-Type" => "text/plain"}, ["Nothing at #{path}\n"]]
+    end
+
+    out = StringIO.new
+
+    transport = Thrift::IOStreamTransport.new env['rack.input'], out
+    protocol  = protocol_factory(env).get_protocol transport
+
+    if @log
+      name, type, seqid, err = processor.process protocol, protocol
+
+      @logger.write FORMAT % [
+        Time.now.to_f,
+        env['HTTP_X_FORWARDED_FOR'] || env["REMOTE_ADDR"] || "-",
+        env["PATH_INFO"],
+        env["QUERY_STRING"].empty? ? "" : "?"+env["QUERY_STRING"],
+        TYPES[type],
+        name,
+        seqid,
+        err ? "'#{err.message} (#{err.class})'" : "null"
+      ]
+    else
+      processor.process protocol, protocol
+    end
+
+    [status_from_last_error, headers(env), [out.string]]
+  end
+
+  def error_capture
+    lambda do |m,*args|
+      @last_error = [m, args]
+    end.tap do |x|
+      (class << x; self; end).instance_eval {
+        alias_method :method_missing, :call }
+    end
+  end
+
+  def status_from_last_error
+    return 200 if @last_error.nil? || @last_error.first == :success
+    x = @last_error.last[3]
+    case x.type
+    when Thrift::ApplicationException::UNKNOWN_METHOD
+      404
+    else
+      500
+    end
+  end
+end

data/lib/stark/rack/content_negotiation.rb

@@ -0,0 +1,34 @@
+class Stark::Rack
+  module ContentNegotiation
+    THRIFT_CONTENT_TYPE = 'application/x-thrift'
+    THRIFT_JSON_CONTENT_TYPE = 'application/vnd.thrift+json'
+
+    def accept_json?(env)
+      env['HTTP_ACCEPT'] == THRIFT_JSON_CONTENT_TYPE ||
+        env['HTTP_CONTENT_TYPE'] == THRIFT_JSON_CONTENT_TYPE
+    end
+
+    def headers(env)
+      headers = { 'Content-Type' => THRIFT_CONTENT_TYPE }
+      if accept_json?(env)
+        headers['Content-Type'] = THRIFT_JSON_CONTENT_TYPE
+      end
+      headers
+    end
+
+    def protocol_factory(env)
+      if env['stark.protocol.factory']
+        env['stark.protocol.factory']
+      else
+        if accept_json?(env)
+          f = Thrift::JsonProtocolFactory.new
+          env['stark.protocol'] = :json
+        else
+          f = Thrift::BinaryProtocolFactory.new
+          env['stark.protocol'] = :binary
+        end
+        env['stark.protocol.factory'] = f
+      end
+    end
+  end
+end

data/lib/stark/rack/logging_processor.rb

@@ -0,0 +1,50 @@
+class Stark::Rack
+  class LoggingProcessor
+    def initialize(handler, secondary=nil)
+      @handler = handler
+      @secondary = secondary
+    end
+
+    def process(iprot, oprot)
+      name, type, seqid = iprot.read_message_begin
+      x = nil
+      if @handler.respond_to?("process_#{name}")
+        begin
+          @handler.send("process_#{name}", seqid, iprot, oprot)
+        rescue StandardError => e
+          Stark.logger.error "#{@handler.class.name}#process_#{name}: #{e.message}\n  " + e.backtrace.join("\n  ")
+          x = Thrift::ApplicationException.new(
+                           Thrift::ApplicationException::UNKNOWN,
+                           "#{e.message} (#{e.class})")
+          oprot.write_message_begin(name, Thrift::MessageTypes::EXCEPTION, seqid)
+          x.write(oprot)
+          oprot.write_message_end
+          oprot.trans.flush
+        end
+
+        if s = @secondary
+          if x
+            s.error name, type, seqid, x
+          else
+            s.success name, type, seqid
+          end
+        end
+
+        [name, type, seqid, x]
+      else
+        iprot.skip(Thrift::Types::STRUCT)
+        iprot.read_message_end
+        x = Thrift::ApplicationException.new(Thrift::ApplicationException::UNKNOWN_METHOD,
+                                     "Unknown function: #{name}")
+        oprot.write_message_begin(name, Thrift::MessageTypes::EXCEPTION, seqid)
+        x.write(oprot)
+        oprot.write_message_end
+        oprot.trans.flush
+        if s = @secondary
+          s.error name, type, seqid, x
+        end
+        false
+      end
+    end
+  end
+end

data/lib/stark/rack/metadata.rb

@@ -0,0 +1,89 @@
+require 'stark'
+require 'composite_io'
+
+class Stark::Rack
+  module Metadata
+    DEFAULT_METADATA = { 'version' => Stark::Rack::VERSION }
+    METADATA_IDL = "service Metadata {\nmap<string,string> metadata()\n}\n"
+    Stark.materialize StringIO.new(METADATA_IDL), Stark::Rack
+
+    def self.new(app, metadata = {})
+      Middleware.new app, metadata
+    end
+
+    class Middleware
+      include ContentNegotiation
+
+      def initialize(app, metadata)
+        @app = app
+        @handler = Handler.new metadata
+        @processor = Processor.new @handler
+      end
+
+      def call(env)
+        env['rack.input'] = RewindableInput.new(env['rack.input'])
+        status, hdr, body = @app.call env
+
+        if status == 404
+          env['rack.input'].rewind
+
+          out       = StringIO.new
+          transport = Thrift::IOStreamTransport.new env['rack.input'], out
+          protocol  = protocol_factory(env).get_protocol transport
+
+          if @processor.process(protocol, protocol)
+            return [200, headers(env), [out.string]]
+          end
+        end
+
+        [status, hdr, body]
+      end
+    end
+
+    class Handler
+      attr_reader :metadata
+      def initialize(metadata)
+        @metadata = DEFAULT_METADATA.merge(metadata || {})
+        @metadata.keys.each do |k|
+          unless String === @metadata[k]
+            @metadata[k] = @metadata[k].to_s # stringify values
+          end
+          unless String === k              # stringify keys
+            @metadata[k.to_s] = @metadata.delete k
+          end
+        end
+      end
+    end
+
+    class RewindableInput
+      def initialize(io)
+        @io = io
+        @buffered = StringIO.new
+      end
+
+      def read(n)
+        if @io
+          @io.read(n).tap do |s|
+            if s
+              @buffered.write s
+            else
+              @io.close
+              @io = nil
+            end
+          end
+        else
+          @buffered.read(n)
+        end
+      end
+
+      def rewind
+        if @io
+          @io = CompositeReadIO.new([StringIO.new(@buffered.string), @io])
+          @buffered = StringIO.new
+        else
+          @buffered.rewind
+        end
+      end
+    end
+  end
+end

data/lib/stark/rack/rest.rb

@@ -0,0 +1,327 @@
+require 'rack/request'
+require 'rack/utils/okjson'
+require 'stark/rack'
+
+class Stark::Rack
+  # Public: This middleware translates "REST-y" requests into Thrift requests
+  # and Thrift responses to a simplified, intuitive JSON response.
+  #
+  # In order for a request to qualify as "REST-y" its PATH_INFO must contain a
+  # method name and must either be:
+  #
+  # - a GET request, possibly with a query string
+  # - a POST or PUT request with body containing form data
+  #   (application/x-www-form-urlencoded or multipart/form-data) or JSON
+  #   (application/json)
+  #
+  # When converting parameters or JSON into Thrift, several conventions and
+  # assumptions are made. If your input does not follow these conventions, the
+  # conversion may succeed but the underlying thrift call may fail.
+  #
+  # - Hashes are converted into maps.
+  # - Arrays are converted into lists.
+  # - Structs can be passed as a hash that includes a '_struct_' key.
+  # - Everything else is converted into strings and makes use of Stark's
+  #   facility to coerce strings into other types (numbers, booleans).
+  #
+  # Structs additionally must follow these conventions.
+  # - The keys of a struct hash should start with the field numbers. Anything
+  #   after the number is discarded (e.g., for "1" or "1last_result" or
+  #   "1-last_result", "last_result" or "-last_result" are discarded).
+  # - Field names not prefixed with a number can be used provided that the
+  #   fields declared in the IDL are numbered starting at 1, increase
+  #   monotonically, and the request struct key/values appear in the order in
+  #   which they are declared in the IDL.
+  #
+  # Given the following thrift definition:
+  #
+  #     struct State {
+  #       1: i32 last_result
+  #       2: map<string,i32> vars
+  #     }
+  #
+  #     service Calc {
+  #       i32 add(1: i32 lhs, 2: i32 rhs)
+  #       i32 last_result()
+  #       void store_vars(1: map<string,i32> vars)
+  #       i32 get_var(1: string name)
+  #       void set_state(1: State state)
+  #       State get_state()
+  #     }
+  #
+  # When the Calc service is mounted in a Stark::Rack endpoint at the `/calc` path,
+  # all of the examples below demonstrate valid requests.
+  #
+  # Examples
+  #
+  #     # Calls last_result()
+  #     GET /calc/last_result
+  #
+  #     # Also calls last_result(). Non-alphanumeric are converted to
+  #     # underscore.
+  #     GET /calc/last-result
+  #
+  #     # Calls get_var("a") using an indexed-hash parameter with keys
+  #     # corresponding to argument field numbers.
+  #     # Effective params hash: {"arg" => {"1" => "a"}}
+  #     GET /calc/get_var?arg[1]=a
+  #
+  #     # Calls get_var("a") using open-ended array parameter.
+  #     # Argument field numbers are assumed to start with 1 and increase
+  #     # monotonically.
+  #     # Effective params hash: {"arg" => ["a"]}
+  #     GET /calc/get_var?arg[]=a
+  #
+  #     # Calls add(1, 1) using an open-ended array parameter.
+  #     # Effective params hash: {"arg" => ["1", "1"]}
+  #     GET /calc/add?arg[]=1&arg[]=2
+  #
+  #     # Calls store_vars({"a" => 1, "b" => 2, "c" => 3}),
+  #     # treating query parameters as a single map argument.
+  #     GET /calc/store_vars?a=1&b=2&c=3
+  #
+  #     # Calls set_state(State.new(:last_result => 0, :vars => {"a" => 1, "b"=> 2})).
+  #     # Note the presence of a "_struct_" key in the parameters, marking a
+  #     # struct instead of a map.
+  #     GET /calc/set_state?_struct_=State&last_result=0&vars[a]=1&vars[b]=2
+  #
+  #     # Calls set_state(State.new(:last_result => 0, :vars => {"a" => 1, "b"=> 2})),
+  #     # using indexed-hash format.
+  #     # Effective params hash:
+  #     # {"arg" => {"1" => {"_struct_" => "State", "last_result" => "0", "vars" => {"a" => "1", "b" => "2"}}}}
+  #     GET /calc/set_state?arg[1][_struct_]=State&arg[1][last_result]=0&arg[1][vars][a]=1&arg[1][vars][b]=2
+  #
+  #     # Calls set_state(State.new(:last_result => 0, :vars => {"a" => 1, "b"=> 2})),
+  #     # using JSON.
+  #     POST /calc/set_state
+  #     Content-Type: application/json
+  #
+  #     [{"_struct_":"State","last_result":0,"vars":{"a":1,"b":2}}]
+  #
+  #     # Calls set_state(State.new(:last_result => 0, :vars => {"a" => 1, "b"=> 2})),
+  #     # using JSON with indexed-hash format.
+  #     POST /calc/set_state
+  #     Content-Type: application/json
+  #
+  #     {"arg":{"1":{"_struct_":"State","last_result":0,"vars":{"a":1,"b":2}}}}
+  #
+  class REST
+    include ContentNegotiation
+
+    # Name of marker key in a hash that indicates it represents a struct. The
+    # struct name is the corresponding value.
+    STRUCT = '_struct_'
+
+    def initialize(app)
+      @app = app
+    end
+
+    def call(env)
+      if applies?(env)
+        env['stark.protocol.factory'] = VerboseProtocolFactory.new
+        if send("create_thrift_call_from_#{env['stark.rest.input.format']}", env)
+          status, headers, body = @app.call env
+          headers["Content-Type"] = 'application/json'
+          [status, headers, unmarshal_result(env, body)]
+        else
+          [400, {}, []]
+        end
+      else
+        @app.call env
+      end
+    end
+
+    def applies?(env)
+      path         = env["PATH_INFO"]
+      content_type = env["HTTP_CONTENT_TYPE"]
+
+      if path && path.length > 1 # need a method name
+        env['stark.rest.method.name'] = path.split('/')[1].gsub(/[^a-zA-Z0-9_]/, '_')
+        if content_type == 'application/json'
+          env['stark.rest.input.format'] = 'json'
+        elsif env["REQUEST_METHOD"] == "GET" || # pure GET, no body
+            # posted content looks like form data
+            Rack::Request::FORM_DATA_MEDIA_TYPES.include?(content_type)
+          env['stark.rest.input.format'] = 'params'
+        end
+      end
+    end
+
+    def create_thrift_call_from_json(env)
+      params = Rack::Utils::OkJson.decode(env['rack.input'].read)
+      params = { 'args' => params } if Array === params
+      encode_thrift_call env, params
+    rescue
+      false
+    end
+
+    def create_thrift_call_from_params(env)
+      encode_thrift_call env, ::Rack::Request.new(env).params
+    end
+
+    def encode_thrift_call(env, params)
+      name = env['stark.rest.method.name']
+      input = StringIO.new
+      proto = protocol_factory(env).get_protocol(Thrift::IOStreamTransport.new(input, input))
+      proto.write_message_begin name, Thrift::MessageTypes::CALL, 0
+
+      obj = { STRUCT => "#{name}_args" }
+      if !params.empty?
+        arguments = params['arg'] || params['args']
+        if Hash === arguments
+          obj.update(arguments)
+        elsif Array === arguments
+          arguments.each_with_index do |v,i|
+            obj["#{i+1}"] = v
+          end
+        else
+          obj["1"] = params
+        end
+      end
+
+      encode_thrift_obj proto, obj
+
+      proto.write_message_end
+      proto.trans.flush
+
+      input.rewind
+      env['rack.input'] = input
+      env['PATH_INFO'] = '/'
+      env['REQUEST_METHOD'] = 'POST'
+      true
+    end
+
+    # Determine a thrift type to use from an array of values.
+    def value_type(vals)
+      types = vals.map {|v| v.class }.uniq
+
+      # Convert everything to string if there isn't a single unique type
+      return Thrift::Types::STRING if types.size > 1
+
+      type = types.first
+
+      # Array -> LIST
+      return Thrift::Types::LIST if type == Array
+
+      # Hash can be a MAP or STRUCT
+      if type == Hash
+        if vals.first.has_key?(STRUCT)
+          return Thrift::Types::STRUCT
+        else
+          return Thrift::Types::MAP
+        end
+      end
+
+      Thrift::Types::STRING
+    end
+
+    def encode_thrift_obj(proto, obj)
+      case obj
+      when Hash
+        if struct = obj.delete(STRUCT)
+          proto.write_struct_begin struct
+          idx = 1
+          obj.each do |k,v|
+            _, number, name = /^(\d*)(.*?)$/.match(k).to_a
+            if number.nil? || number.empty?
+              number = idx
+            else
+              number = number.to_i
+            end
+            name = "field#{number}" if name.nil? || name.empty?
+
+            proto.write_field_begin name, value_type([v]), number
+
+            encode_thrift_obj proto, v
+            proto.write_field_end
+            idx += 1
+          end
+          proto.write_field_stop
+          proto.write_struct_end
+        else
+          proto.write_map_begin Thrift::Types::STRING, value_type(obj.values), obj.size
+          obj.each do |k,v|
+            proto.write_string k
+            encode_thrift_obj proto, v
+          end
+          proto.write_map_end
+        end
+      when Array
+        proto.write_list_begin value_type(obj), obj.size
+        obj.each {|v| encode_thrift_obj proto, v }
+        proto.write_list_end
+      else
+        proto.write_string obj.to_s
+      end
+    end
+
+    def decode_thrift_obj(proto, type)
+      case type
+      when Thrift::Types::BOOL
+        proto.read_bool
+      when Thrift::Types::BYTE
+        proto.read_i8
+      when Thrift::Types::I16
+        proto.read_i16
+      when Thrift::Types::I32
+        proto.read_i32
+      when Thrift::Types::I64
+        proto.read_i64
+      when Thrift::Types::DOUBLE
+        proto.read_double
+      when Thrift::Types::STRING
+        proto.read_string
+      when Thrift::Types::STRUCT
+        Hash.new.tap do |hash|
+          struct = proto.read_struct_begin
+          hash[STRUCT] = struct if struct
+          while true
+            name, type, id = proto.read_field_begin
+            break if type == Thrift::Types::STOP
+            hash["#{id}#{':'+name if name}"] = decode_thrift_obj proto, type
+            proto.read_field_end
+          end
+          proto.read_struct_end
+        end
+      when Thrift::Types::MAP
+        Hash.new.tap do |hash|
+          kt, vt, size = proto.read_map_begin
+          size.times do
+            hash[decode_thrift_obj(proto, kt).to_s] = decode_thrift_obj(proto, vt)
+          end
+          proto.read_map_end
+        end
+      when Thrift::Types::SET
+        Set.new.tap do |set|
+          vt, size = proto.read_set_begin
+          size.times do
+            set << decode_thrift_obj(proto, vt)
+          end
+          proto.read_set_end
+        end
+      when Thrift::Types::LIST
+        Array.new.tap do |list|
+          vt, size = proto.read_list_begin
+          size.times do
+            list << decode_thrift_obj(proto, vt)
+          end
+          proto.read_list_end
+        end
+      when Thrift::Types::STOP
+        nil
+      else
+        raise NotImplementedError, type
+      end
+    end
+
+    def unmarshal_result(env, body)
+      out = StringIO.new(body.join)
+      proto = protocol_factory(env).get_protocol(Thrift::IOStreamTransport.new(out, out))
+      proto.read_message_begin
+      proto.read_struct_begin
+      _, type, id = proto.read_field_begin
+      result = decode_thrift_obj(proto, type)
+      [Rack::Utils::OkJson.encode((id == 0 ? "result" : "error") => result)]
+    end
+  end
+end