ffi-yajl 0.0.1

data/lib/ffi_yajl/ext.rb

@@ -0,0 +1,22 @@
+require 'rubygems'
+
+require 'ffi_yajl/encoder'
+require 'ffi_yajl/parser'
+
+unless RUBY_VERSION.to_f >= 1.9
+  # segfaults on ruby 1.8 and this is an exceedingly low priority to fix, use ffi instead
+  raise NotImplementedError, "The C-extension is disabled on Ruby 1.8"
+end
+
+module FFI_Yajl
+  class Parser
+    require 'ffi_yajl/ext/parser'
+    include FFI_Yajl::Ext::Parser
+  end
+
+  class Encoder
+    require 'ffi_yajl/ext/encoder'
+    include FFI_Yajl::Ext::Encoder
+  end
+end
+

data/lib/ffi_yajl/ffi.rb

@@ -0,0 +1,129 @@
+require 'rubygems'
+
+require 'ffi'
+
+module FFI_Yajl
+  extend ::FFI::Library
+
+  libname = ::FFI.map_library_name("yajl")
+  libpath = File.expand_path(File.join(File.dirname(__FILE__), "..", libname))
+
+  if File.file?(libpath)
+    # use our vendored version of libyajl2 if we find it installed
+    ffi_lib libpath
+  else
+    ffi_lib 'yajl'
+  end
+
+  class YajlCallbacks < ::FFI::Struct
+    layout :yajl_null, :pointer,
+      :yajl_boolean, :pointer,
+      :yajl_integer, :pointer,
+      :yajl_double, :pointer,
+      :yajl_number, :pointer,
+      :yajl_string, :pointer,
+      :yajl_start_map, :pointer,
+      :yajl_map_key, :pointer,
+      :yajl_end_map, :pointer,
+      :yajl_start_array, :pointer,
+      :yajl_end_array, :pointer
+  end
+
+  enum :yajl_status, [
+    :yajl_status_ok,
+    :yajl_status_client_canceled,
+    :yajl_status_insufficient_data,
+    :yajl_status_error,
+  ]
+
+# FFI::Enums are slow, should remove the rest
+#  enum :yajl_gen_status, [
+#    :yajl_gen_status_ok,
+#    :yajl_gen_keys_must_be_strings,
+#    :yajl_max_depth_exceeded,
+#    :yajl_gen_in_error_state,
+#    :yajl_gen_generation_complete,
+#    :yajl_gen_invalid_number,
+#    :yajl_gen_no_buf,
+#  ]
+
+  enum :yajl_option, [
+    :yajl_allow_comments, 0x01,
+    :yajl_dont_validate_strings, 0x02,
+    :yajl_allow_trailing_garbage, 0x04,
+    :yajl_allow_multiple_values, 0x08,
+    :yajl_allow_partial_values, 0x10,
+  ]
+
+  enum :yajl_gen_option, [
+    :yajl_gen_beautify, 0x01,
+    :yajl_gen_indent_string, 0x02,
+    :yajl_gen_print_callback, 0x04,
+    :yajl_gen_validate_utf8, 0x08,
+  ]
+
+  typedef :pointer, :yajl_handle
+  typedef :pointer, :yajl_gen
+
+  # yajl uses unsinged char *'s consistently
+  typedef :pointer, :ustring_pointer
+  typedef :string, :ustring
+
+  # const char *yajl_status_to_string (yajl_status code)
+  attach_function :yajl_status_to_string, [ :yajl_status ], :string
+  # yajl_handle yajl_alloc(const yajl_callbacks * callbacks, yajl_alloc_funcs * afs, void * ctx)
+  attach_function :yajl_alloc, [:pointer, :pointer, :pointer], :yajl_handle
+  # void yajl_free (yajl_handle handle)
+  attach_function :yajl_free, [:yajl_handle], :void
+  # yajl_status yajl_parse (yajl_handle hand, const unsigned char *jsonText, unsigned int jsonTextLength)
+  attach_function :yajl_parse, [:yajl_handle, :ustring, :uint], :yajl_status
+  # yajl_status yajl_parse_complete (yajl_handle hand)
+  attach_function :yajl_complete_parse, [:yajl_handle], :yajl_status
+  # unsigned char *yajl_get_error (yajl_handle hand, int verbose, const unsigned char *jsonText, unsigned int jsonTextLength)
+  attach_function :yajl_get_error, [:yajl_handle, :int, :ustring, :int], :ustring
+  # void yajl_free_error (yajl_handle hand, unsigned char *str)
+  attach_function :yajl_free_error, [:yajl_handle, :ustring], :void
+
+  #
+  attach_function :yajl_config, [:yajl_handle, :yajl_option, :int], :int
+
+  attach_function :yajl_gen_config, [:yajl_gen, :yajl_gen_option, :varargs], :int
+
+  # yajl_gen yajl_gen_alloc (const yajl_alloc_funcs *allocFuncs)
+  attach_function :yajl_gen_alloc, [:pointer], :yajl_gen
+  # yajl_gen yajl_gen_alloc2 (const yajl_print_t callback, const yajl_gen_config *config, const yajl_alloc_funcs *allocFuncs, void *ctx)
+  # attach_function :yajl_gen_alloc2, [:pointer, :pointer, :pointer, :pointer], :yajl_gen
+  # void  yajl_gen_free (yajl_gen handle)
+  attach_function :yajl_gen_free, [:yajl_gen], :void
+
+  attach_function :yajl_gen_integer, [:yajl_gen, :long_long], :int
+  attach_function :yajl_gen_double, [:yajl_gen, :double], :int
+  attach_function :yajl_gen_number, [:yajl_gen, :ustring, :int], :int
+  attach_function :yajl_gen_string, [:yajl_gen, :ustring, :int], :int
+  attach_function :yajl_gen_null, [:yajl_gen], :int
+  attach_function :yajl_gen_bool, [:yajl_gen, :int], :int
+  attach_function :yajl_gen_map_open, [:yajl_gen], :int
+  attach_function :yajl_gen_map_close, [:yajl_gen], :int
+  attach_function :yajl_gen_array_open, [:yajl_gen], :int
+  attach_function :yajl_gen_array_close, [:yajl_gen], :int
+  # yajl_gen_status yajl_gen_get_buf (yajl_gen hand, const unsigned char **buf, unsigned int *len)
+  attach_function :yajl_gen_get_buf, [:yajl_gen, :pointer ,:pointer], :int
+  # void yajl_gen_clear (yajl_gen hand)
+  attach_function :yajl_gen_clear, [:yajl_gen], :void
+
+end
+
+require 'ffi_yajl/encoder'
+require 'ffi_yajl/parser'
+
+module FFI_Yajl
+  class Parser
+    require 'ffi_yajl/ffi/parser'
+    include FFI_Yajl::FFI::Parser
+  end
+
+  class Encoder
+    require 'ffi_yajl/ffi/encoder'
+    include FFI_Yajl::FFI::Encoder
+  end
+end

data/lib/ffi_yajl/ffi/encoder.rb

@@ -0,0 +1,175 @@
+
+require 'ffi_yajl/ffi'
+
+module FFI_Yajl
+  module FFI
+    module Encoder
+      def do_yajl_encode(obj, yajl_gen_opts = {})
+
+        yajl_gen = FFI_Yajl.yajl_gen_alloc(nil);
+
+        # configure the yajl encoder
+        if yajl_gen_opts[:yajl_gen_beautify]
+          FFI_Yajl.yajl_gen_config(yajl_gen, :yajl_gen_beautify, :int, 1)
+        end
+        if yajl_gen_opts[:yajl_gen_validate_utf8]
+          FFI_Yajl.yajl_gen_config(yajl_gen, :yajl_gen_validate_utf8, :int, 1)
+        end
+        indent = yajl_gen_opts[:yajl_gen_indent_string] || " "
+        FFI_Yajl.yajl_gen_config(yajl_gen, :yajl_gen_indent_string, :string, indent)
+
+        # setup our own state
+        state = {
+          :json_opts => opts,
+          :processing_key => false,
+        }
+
+        # do the encoding
+        obj.ffi_yajl(yajl_gen, state)
+
+        # get back our encoded JSON
+        string_ptr = ::FFI::MemoryPointer.new(:string)
+        length_ptr = ::FFI::MemoryPointer.new(:int)
+        if ( status = FFI_Yajl.yajl_gen_get_buf(yajl_gen, string_ptr, length_ptr) ) != 0
+          FFI_Yajl::Encoder.raise_error_for_status(status)
+        end
+        length = length_ptr.read_int
+        string = string_ptr.get_pointer(0).read_string
+
+        FFI_Yajl.yajl_gen_free(yajl_gen)
+
+        return string
+      end
+
+    end
+  end
+end
+
+class Hash
+  def ffi_yajl(yajl_gen, state)
+    if ( status = FFI_Yajl.yajl_gen_map_open(yajl_gen) ) != 0
+      FFI_Yajl::Encoder.raise_error_for_status(status)
+    end
+    self.each do |key, value|
+      # Perf Fix: mutate state hash rather than creating new copy
+      state[:processing_key] = true
+      key.ffi_yajl(yajl_gen, state)
+      state[:processing_key] = false
+      value.ffi_yajl(yajl_gen, state)
+    end
+    if ( status = FFI_Yajl.yajl_gen_map_close(yajl_gen) ) != 0
+      FFI_Yajl::Encoder.raise_error_for_status(status)
+    end
+  end
+end
+
+class Array
+  def ffi_yajl(yajl_gen, state)
+    if ( status = FFI_Yajl.yajl_gen_array_open(yajl_gen) ) != 0
+      FFI_Yajl::Encoder.raise_error_for_status(status)
+    end
+    self.each do |value|
+      value.ffi_yajl(yajl_gen, state)
+    end
+    if ( status = FFI_Yajl.yajl_gen_array_close(yajl_gen) ) != 0
+      FFI_Yajl::Encoder.raise_error_for_status(status)
+    end
+  end
+end
+
+class NilClass
+  def ffi_yajl(yajl_gen, state)
+    if ( status = FFI_Yajl.yajl_gen_null(yajl_gen) ) != 0
+      FFI_Yajl::Encoder.raise_error_for_status(status)
+    end
+  end
+end
+
+class TrueClass
+  def ffi_yajl(yajl_gen, state)
+    if ( status = FFI_Yajl.yajl_gen_bool(yajl_gen, 1) ) != 0
+      FFI_Yajl::Encoder.raise_error_for_status(status)
+    end
+  end
+end
+
+class FalseClass
+  def ffi_yajl(yajl_gen, state)
+    if ( status = FFI_Yajl.yajl_gen_bool(yajl_gen, 0) ) != 0
+      FFI_Yajl::Encoder.raise_error_for_status(status)
+    end
+  end
+end
+
+class Fixnum
+  def ffi_yajl(yajl_gen, state)
+    str = self.to_s
+    if str == "NaN" || str == "Infinity" || str == "-Infinity"
+      raise ::FFI_Yajl::EncodeError.new("'#{str}' is an invalid number")
+    end
+    if state[:processing_key]
+      if ( status = FFI_Yajl.yajl_gen_string(yajl_gen, str, str.bytesize) ) != 0
+        FFI_Yajl::Encoder.raise_error_for_status(status)
+      end
+    else
+      if ( status = FFI_Yajl.yajl_gen_integer(yajl_gen, self) ) != 0
+        FFI_Yajl::Encoder.raise_error_for_status(status)
+      end
+    end
+  end
+end
+
+class Bignum
+  def ffi_yajl(yajl_gen, state)
+    str = self.to_s
+    if str == "NaN" || str == "Infinity" || str == "-Infinity"
+      raise ::FFI_Yajl::EncodeError.new("'#{str}' is an invalid number")
+    end
+    if state[:processing_key]
+      if ( status = FFI_Yajl.yajl_gen_string(yajl_gen, str, str.bytesize) ) != 0
+        FFI_Yajl::Encoder.raise_error_for_status(status)
+      end
+    else
+      if ( status = FFI_Yajl.yajl_gen_number(yajl_gen, str, str.bytesize) ) != 0
+        FFI_Yajl::Encoder.raise_error_for_status(status)
+      end
+    end
+  end
+end
+
+class Float
+  def ffi_yajl(yajl_gen, state)
+    str = self.to_s
+    if str == "NaN" || str == "Infinity" || str == "-Infinity"
+      raise ::FFI_Yajl::EncodeError.new("'#{str}' is an invalid number")
+    end
+    if state[:processing_key]
+      if ( status = FFI_Yajl.yajl_gen_string(yajl_gen, str, str.bytesize) ) != 0
+        FFI_Yajl::Encoder.raise_error_for_status(status)
+      end
+    else
+      if ( status = FFI_Yajl.yajl_gen_number(yajl_gen, str, str.bytesize) ) != 0
+        FFI_Yajl::Encoder.raise_error_for_status(status)
+      end
+    end
+  end
+end
+
+class String
+  def ffi_yajl(yajl_gen, state)
+    if ( status = FFI_Yajl.yajl_gen_string(yajl_gen, self, self.bytesize) ) != 0
+      FFI_Yajl::Encoder.raise_error_for_status(status)
+    end
+  end
+end
+
+# I feel dirty
+class Object
+  def ffi_yajl(yajl_gen, state)
+    json = self.to_json(state[:json_opts])
+    if ( status = FFI_Yajl.yajl_gen_number(yajl_gen, json, json.bytesize) ) != 0
+      FFI_Yajl::Encoder.raise_error_for_status(status)
+    end
+  end
+end
+

data/lib/ffi_yajl/ffi/parser.rb

@@ -0,0 +1,145 @@
+
+require 'ffi_yajl/ffi'
+
+module FFI_Yajl
+  module FFI
+    module Parser
+        attr_accessor :stack, :key_stack, :key, :finished
+
+        #
+        # stack used to build up our complex object
+        #
+        def stack
+          @stack ||= Array.new
+        end
+
+        def set_value(val)
+          case stack.last
+          when Hash
+            raise FFI_Yajl::ParseError.new("internal error: missing key in parse") if key.nil?
+            stack.last[key] = val
+          when Array
+            stack.last.push(val)
+          else
+            raise FFI_Yajl::ParseError.new("internal error: object not a hash or array")
+          end
+        end
+
+        def stack_pop
+          if stack.length > 1
+            set_value( stack.pop )
+          else
+            @finished = stack.pop
+          end
+        end
+
+        #
+        # stack to keep track of keys as we create nested hashes
+        #
+        def key_stack
+          @key_stack ||= Array.new
+        end
+
+        def key_push
+          key_stack.push(key)
+        end
+
+        def key_pop
+          @key = key_stack.pop()
+        end
+
+
+      def setup_callbacks
+        @null_callback = ::FFI::Function.new(:int, [:pointer]) do |ctx|
+          set_value(nil)
+          1
+        end
+        @boolean_callback = ::FFI::Function.new(:int, [:pointer, :int]) do |ctx, boolval|
+          set_value(boolval == 1 ? true : false)
+          1
+        end
+        @integer_callback = ::FFI::Function.new(:int, [:pointer, :long_long]) do |ctx, intval|
+          set_value(intval)
+          1
+        end
+        @number_callback = ::FFI::Function.new(:int, [:pointer, :string, :size_t ]) do |ctx, stringval, stringlen|
+          s = stringval.slice(0,stringlen)
+          s.force_encoding('UTF-8') if defined? Encoding
+          # XXX: I can't think of a better way to do this right now.  need to call to_f if and only if its a float.
+          v = ( s =~ /\./ ) ? s.to_f : s.to_i
+          set_value(v)
+          1
+        end
+        @double_callback = ::FFI::Function.new(:int, [:pointer, :double]) do |ctx, doubleval|
+          set_value(doubleval)
+          1
+        end
+        @string_callback = ::FFI::Function.new(:int, [:pointer, :string, :size_t]) do |ctx, stringval, stringlen|
+          s = stringval.slice(0,stringlen)
+          s.force_encoding('UTF-8') if defined? Encoding
+          set_value(s)
+          1
+        end
+        @start_map_callback = ::FFI::Function.new(:int, [:pointer]) do |ctx|
+          key_push  # for key => { } case, save the key
+          stack.push(Hash.new)
+          1
+        end
+        @map_key_callback = ::FFI::Function.new(:int, [:pointer, :string, :size_t]) do |ctx, key, keylen|
+          s = key.slice(0,keylen)
+          s.force_encoding('UTF-8') if defined? Encoding
+          self.key = s
+          1
+        end
+        @end_map_callback = ::FFI::Function.new(:int, [:pointer]) do |ctx|
+          key_pop
+          stack_pop
+          1
+        end
+        @start_array_callback = ::FFI::Function.new(:int, [:pointer]) do |ctx|
+          key_push  # for key => [ ] case, save the key
+          stack.push(Array.new)
+          1
+        end
+        @end_array_callback = ::FFI::Function.new(:int, [:pointer]) do |ctx|
+          key_pop
+          stack_pop
+          1
+        end
+      end
+
+
+      def do_yajl_parse(str, opts = {})
+        setup_callbacks
+        callback_ptr = ::FFI::MemoryPointer.new(::FFI_Yajl::YajlCallbacks)
+        callbacks = ::FFI_Yajl::YajlCallbacks.new(callback_ptr)
+        callbacks[:yajl_null] = @null_callback
+        callbacks[:yajl_boolean] = @boolean_callback
+        callbacks[:yajl_integer] = @integer_callback
+        callbacks[:yajl_double] = @double_callback
+        callbacks[:yajl_number] = @number_callback
+        callbacks[:yajl_string] = @string_callback
+        callbacks[:yajl_start_map] = @start_map_callback
+        callbacks[:yajl_map_key] = @map_key_callback
+        callbacks[:yajl_end_map] = @end_map_callback
+        callbacks[:yajl_start_array] = @start_array_callback
+        callbacks[:yajl_end_array] = @end_array_callback
+        yajl_handle = ::FFI_Yajl.yajl_alloc(callback_ptr, nil, nil)
+        if ( stat = ::FFI_Yajl.yajl_parse(yajl_handle, str, str.bytesize) != :yajl_status_ok )
+          # FIXME: dup the error and call yajl_free_error?
+          error = ::FFI_Yajl.yajl_get_error(yajl_handle, 1, str, str.bytesize)
+          raise ::FFI_Yajl::ParseError.new(error)
+        end
+        if ( stat = FFI_Yajl.yajl_complete_parse(yajl_handle) != :yajl_status_ok )
+          # FIXME: dup the error and call yajl_free_error?
+          error = ::FFI_Yajl.yajl_get_error(yajl_handle, 1, str, str.bytesize)
+          raise ::FFI_Yajl::ParseError.new(error)
+        end
+        finished
+      ensure
+        ::FFI_Yajl.yajl_free(yajl_handle) if yajl_handle
+      end
+    end
+  end
+end
+