zookeeper-ng 1.5

data/ext/event_lib.h

@@ -0,0 +1,175 @@
+#ifndef ZKRB_EVENT_LIB_H
+#define ZKRB_EVENT_LIB_H
+
+#include "ruby.h"
+#include "zookeeper/zookeeper.h"
+#include <errno.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+
+#define ZK_TRUE 1
+#define ZK_FALSE 0
+#define ZKRB_GLOBAL_REQ -1
+
+// (slyphon): this RC value does not conflict with any of the ZOO_ERRORS
+// but need to find a better way of formalizing and defining this stuff
+#define ZKRB_ERR_REQ -2
+#define ZKRB_ERR_RC -15
+
+#ifndef RSTRING_LEN
+# define RSTRING_LEN(x) RSTRING(x)->len
+#endif
+#ifndef RSTRING_PTR
+# define RSTRING_PTR(x) RSTRING(x)->ptr
+#endif
+#ifndef RARRAY_LEN
+# define RARRAY_LEN(x) RARRAY(x)->len
+#endif
+
+extern int ZKRBDebugging;
+extern pthread_mutex_t zkrb_q_mutex;
+
+struct zkrb_data_completion {
+  char *data;
+  int data_len;
+  struct Stat *stat;
+};
+
+struct zkrb_stat_completion {
+  struct Stat *stat;
+};
+
+struct zkrb_void_completion {
+};
+
+struct zkrb_string_completion {
+  char *value;  
+};
+
+struct zkrb_strings_completion {
+  struct String_vector *values;
+};
+
+struct zkrb_strings_stat_completion {
+  struct String_vector *values;
+  struct Stat *stat;
+};
+
+struct zkrb_acl_completion {
+  struct ACL_vector *acl;
+  struct Stat *stat;
+};
+
+struct zkrb_watcher_completion {
+  int type;
+  int state;
+  char *path;
+};
+
+typedef struct {
+  int64_t req_id;
+  int rc;
+
+  enum {
+    ZKRB_DATA         = 0,
+    ZKRB_STAT         = 1,
+    ZKRB_VOID         = 2,
+    ZKRB_STRING       = 3,
+    ZKRB_STRINGS      = 4,
+    ZKRB_STRINGS_STAT = 5,
+    ZKRB_ACL          = 6,
+    ZKRB_WATCHER      = 7
+  } type;
+  
+  union {
+    struct zkrb_data_completion         *data_completion;
+    struct zkrb_stat_completion         *stat_completion;
+    struct zkrb_void_completion         *void_completion;
+    struct zkrb_string_completion       *string_completion;
+    struct zkrb_strings_completion      *strings_completion;
+    struct zkrb_strings_stat_completion *strings_stat_completion;
+    struct zkrb_acl_completion          *acl_completion;
+    struct zkrb_watcher_completion      *watcher_completion;
+  } completion;
+} zkrb_event_t;
+
+struct zkrb_event_ll {
+  zkrb_event_t         *event;
+  struct zkrb_event_ll *next;
+};
+
+typedef struct zkrb_event_ll zkrb_event_ll_t;
+
+typedef struct {
+  zkrb_event_ll_t *head;
+  zkrb_event_ll_t *tail;
+  int             pipe_read;
+  int             pipe_write;
+  pid_t           orig_pid;
+} zkrb_queue_t;
+
+zkrb_queue_t * zkrb_queue_alloc(void);
+void           zkrb_queue_free(zkrb_queue_t *queue);
+zkrb_event_t * zkrb_event_alloc(void);
+void           zkrb_event_free(zkrb_event_t *ptr);
+
+void                 zkrb_enqueue(zkrb_queue_t *queue, zkrb_event_t *elt);
+zkrb_event_t *       zkrb_peek(zkrb_queue_t *queue);
+zkrb_event_t *       zkrb_dequeue(zkrb_queue_t *queue, int need_lock);
+void                 zkrb_signal(zkrb_queue_t *queue);
+
+void zkrb_print_stat(const struct Stat *s);
+
+typedef struct {
+  int64_t        req_id;
+  zkrb_queue_t   *queue;
+} zkrb_calling_context;
+
+void zkrb_print_calling_context(zkrb_calling_context *ctx);
+zkrb_calling_context *zkrb_calling_context_alloc(int64_t req_id, zkrb_queue_t *queue);
+void zkrb_calling_context_free(zkrb_calling_context *ctx);
+
+/*
+  default process completions that get queued into the ruby client event queue
+*/
+
+void zkrb_state_callback(
+    zhandle_t *zh, int type, int state, const char *path, void *calling_ctx);
+
+void zkrb_data_callback(
+    int rc, const char *value, int value_len, const struct Stat *stat, const void *calling_ctx);
+
+void zkrb_stat_callback(
+    int rc, const struct Stat *stat, const void *calling_ctx);
+
+void zkrb_string_callback(
+    int rc, const char *string, const void *calling_ctx);
+
+void zkrb_strings_callback(
+    int rc, const struct String_vector *strings, const void *calling_ctx);
+
+void zkrb_strings_stat_callback(
+    int rc, const struct String_vector *strings, const struct Stat* stat, const void *calling_ctx);
+
+void zkrb_void_callback(
+    int rc, const void *calling_ctx);
+
+void zkrb_acl_callback(
+    int rc, struct ACL_vector *acls, struct Stat *stat, const void *calling_ctx);
+
+VALUE zkrb_event_to_ruby(zkrb_event_t *event);
+VALUE zkrb_acl_to_ruby(struct ACL *acl);
+VALUE zkrb_acl_vector_to_ruby(struct ACL_vector *acl_vector);
+VALUE zkrb_id_to_ruby(struct Id *id);
+VALUE zkrb_string_vector_to_ruby(struct String_vector *string_vector);
+VALUE zkrb_stat_to_rarray(const struct Stat *stat);
+VALUE zkrb_stat_to_rhash(const struct Stat* stat);
+
+struct ACL_vector *    zkrb_ruby_to_aclvector(VALUE acl_ary);
+struct ACL_vector *    zkrb_clone_acl_vector(struct ACL_vector * src);
+struct String_vector * zkrb_clone_string_vector(const struct String_vector * src);
+struct ACL             zkrb_ruby_to_acl(VALUE rubyacl);
+struct Id              zkrb_ruby_to_id(VALUE rubyid);
+
+#endif  /* ZKRB_EVENT_LIB_H */

data/ext/extconf.rb

@@ -0,0 +1,103 @@
+require 'mkmf'
+require 'rbconfig'
+require 'fileutils'
+
+HERE = File.expand_path(File.dirname(__FILE__))
+BUNDLE = Dir.glob("zkc-*.tar.gz").sort.last
+ZKC_VERSION = BUNDLE[/(zkc-.*?)\.tar.gz$/, 1]
+PATCHES = Dir.glob("patches/#{ZKC_VERSION}*.patch")
+
+BUNDLE_PATH = File.join(HERE, ZKC_VERSION, 'c')
+
+$EXTRA_CONF = ''
+
+# CLANG!!!! jeez, if apple would only *stop* "thinking different"
+if cc = RbConfig::CONFIG['CC'] && cc =~ /^gcc/
+  $CC = cc
+  $EXTRA_CONF = "#{$EXTRA_CONF} CC=#{$CC}"
+end
+
+$CFLAGS = "#{$CFLAGS}".gsub("$(cflags)", "").gsub("-arch ppc", "")
+$LDFLAGS = "#{$LDFLAGS}".gsub("$(ldflags)", "").gsub("-arch ppc", "")
+$CXXFLAGS = " -std=gnu++98 #{$CFLAGS}"
+$CPPFLAGS = $ARCH_FLAG = ""
+
+if RUBY_VERSION == '1.8.7'
+  $CFLAGS << ' -DZKRB_RUBY_187'
+end
+
+ZK_DEBUG = (ENV['DEBUG'] or ARGV.any? { |arg| arg == '--debug' })
+ZK_DEV = ENV['ZK_DEV']
+DEBUG_CFLAGS = " -O0 -ggdb3 -DHAVE_DEBUG -fstack-protector-all"
+
+if ZK_DEBUG
+  $stderr.puts "*** Setting debug flags. ***"
+  $EXTRA_CONF = "#{$EXTRA_CONF} --enable-debug"
+  $CFLAGS.gsub!(/ -O[^0] /, ' ')
+  $CFLAGS << DEBUG_CFLAGS
+end
+
+$includes = " -I#{HERE}/include"
+$libraries = " -L#{HERE}/lib -L#{RbConfig::CONFIG['libdir']}"
+$CFLAGS = "#{$includes} #{$libraries} #{$CFLAGS}"
+$LDFLAGS = "#{$libraries} #{$LDFLAGS}"
+$LIBPATH = ["#{HERE}/lib"]
+$DEFLIBPATH = []
+
+def safe_sh(cmd)
+  puts cmd
+  system(cmd)
+  unless $?.exited? and $?.success?
+    raise "command failed! #{cmd}"
+  end
+end
+
+Dir.chdir(HERE) do
+  if File.exist?("lib")
+    puts "Zkc already built; run 'rake clobber' in ext/ first if you need to rebuild."
+  else
+    puts "Building zkc."
+
+    unless File.exists?('c')
+      safe_sh "tar xzf #{BUNDLE} 2>&1"
+      PATCHES.each do |patch|
+        safe_sh "patch -p0 < #{patch} 2>&1"
+      end
+    end
+
+    # clean up stupid apple rsrc fork bullshit
+    FileUtils.rm_f(Dir['**/._*'].select{|p| test(?f, p)})
+
+    Dir.chdir(BUNDLE_PATH) do
+      configure = "./configure --prefix=#{HERE} --with-pic --without-cppunit --disable-dependency-tracking #{$EXTRA_CONF} 2>&1"
+      configure = "env CFLAGS='#{DEBUG_CFLAGS}' #{configure}" if ZK_DEBUG
+
+      safe_sh(configure)
+      safe_sh("make  2>&1")
+      safe_sh("make install 2>&1")
+    end
+
+    system("rm -rf #{BUNDLE_PATH}") unless ZK_DEBUG or ZK_DEV
+  end
+end
+
+# Absolutely prevent the linker from picking up any other zookeeper_mt
+Dir.chdir("#{HERE}/lib") do
+  %w[st mt].each do |stmt|
+    %w[a la dylib].each do |ext|
+      origin_lib_name = "libzookeeper_#{stmt}.#{ext}"
+      dest_lib_name = "libzookeeper_#{stmt}_gem.#{ext}"
+      system("cp -f #{origin_lib_name} #{dest_lib_name}") if File.exists?(origin_lib_name)
+    end
+  end
+end
+
+# -lm must come after lzookeeper_st_gem to ensure proper link
+$LIBS << " -lzookeeper_st_gem -lm"
+
+have_func('rb_thread_blocking_region')
+have_func('rb_thread_fd_select')
+
+$CFLAGS << ' -Wall' if ZK_DEV
+create_makefile 'zookeeper_c'
+

data/ext/generate_gvl_code.rb

@@ -0,0 +1,321 @@
+#!/usr/bin/env ruby
+
+=begin
+the idea here is to take each zoo_* function declaration in the header file, and turn it into
+a calling arguments struct, a wrapper function and a macro for packing the values.
+
+so for:
+
+  ZOOAPI int zoo_acreate(zhandle_t *zh, const char *path, const char *value, 
+          int valuelen, const struct ACL_vector *acl, int flags,
+          string_completion_t completion, const void *data);
+
+we want
+
+  typedef struct {
+    zhandle_t *zh;
+    const char *path;
+    const char *value;
+    int valuelen;
+    const struct ACL_vector *acl;
+    int flags;
+    string_completion_t completion;
+    const void *data;
+  } zkrb_zoo_acreate_args_t;
+
+  static VALUE zkrb_gvl_zoo_acreate(void *data) {
+    zkrb_zoo_acreate_args_t *a = (zkrb_zoo_acreate_args_t *)data;
+  
+    a->rc = zoo_acreate(a->zh, a->path, a->value, a->valuelen, a->acl, a->flags, a->completion, a->data);
+  
+    return Qnil;
+  }
+
+  static int zkrb_call_zoo_acreate(zhandle_t *zh, const char *path, const char *value, 
+                int valuelen, const struct ACL_vector *acl, int flags,
+                string_completion_t completion, const void *data) {
+
+    zkrb_zoo_acreate_args_t args = {
+      .rc = ZKRB_FAIL,
+      .zh = zh,
+      .path = path,
+      .value = value,
+      .valuelen = valuelen,
+      .acl = acl,
+      .flags = flags,
+      .completion = completion,
+      .data = data
+    };
+
+    zkrb_thread_blocking_region(zkrb_gvl_zoo_acreate, (void *)&args);
+
+    return args.rc;
+  }
+
+=end
+
+REGEXP = /^ZOOAPI int (zoo_[^(]+)\(([^)]+)\);$/m
+
+require 'forwardable'
+require 'stringio'
+
+THIS_DIR = File.expand_path('..', __FILE__)
+
+ZKRB_WRAPPER_H_PATH = File.expand_path('../zkrb_wrapper.h', __FILE__)
+ZKRB_WRAPPER_C_PATH = File.expand_path('../zkrb_wrapper.c', __FILE__)
+
+# the struct that holds the call args for zoo_fn_name
+class CallStruct
+  attr_reader :zoo_fn_name, :typed_args, :name
+
+  def initialize(zoo_fn_name, typed_args)
+    @zoo_fn_name, @typed_args = zoo_fn_name, typed_args
+    @name = "zkrb_#{zoo_fn_name}_args_t"
+  end
+
+  def body
+    @body ||= (
+      lines = ["typedef struct {"]
+      lines += typed_args.map{|n| "  #{n};"}
+      lines << "  int rc;"
+      lines << "} #{name};"
+      lines.join("\n")
+    )
+  end
+end
+
+module MemberNames
+  def member_names
+    @member_names ||= typed_args.map { |n| n.split(/[ *]/).last }
+  end
+end
+
+# the zkrb_gvl_zoo_* function
+class WrapperFunction
+  extend Forwardable
+  include MemberNames
+
+  PREFIX = 'zkrb_gvl'
+
+  def_delegators :struct, :typed_args
+
+  attr_reader :struct, :name, :zoo_fn_name
+
+  def initialize(zoo_fn_name, struct)
+    @zoo_fn_name = zoo_fn_name
+    @struct = struct
+    @name = "#{PREFIX}_#{zoo_fn_name}"
+  end
+
+  def fn_signature
+    @fn_signature ||= "static VALUE #{name}(void *data)"
+  end
+
+  def body
+    @body ||= (
+      lines = ["#{fn_signature} {"]
+      lines << "  #{struct.name} *a = (#{struct.name} *)data;"
+
+      funcall = "  a->rc = #{zoo_fn_name}("
+      funcall << member_names.map { |m| "a->#{m}" }.join(', ')
+      funcall << ');'
+
+      lines << funcall
+
+      lines << "  return Qnil;"
+      lines << "}"
+      lines.join("\n")
+    )
+  end
+end
+
+# the zkrb_call_zoo_* function
+class CallingFunction
+  extend Forwardable
+  include MemberNames
+  
+  PREFIX = 'zkrb_call'
+
+  def_delegators :struct, :typed_args
+
+  attr_reader :struct, :wrapper_fn, :zoo_fn_name, :name
+
+  def initialize(zoo_fn_name, struct, wrapper_fn)
+    @zoo_fn_name, @struct, @wrapper_fn = zoo_fn_name, struct, wrapper_fn
+
+    @name = "#{PREFIX}_#{zoo_fn_name}"
+  end
+
+  def fn_signature
+    @fn_signature ||= "int #{name}(#{typed_args.join(', ')})"
+  end
+
+  def initializer_lines
+    @initializer_lines ||= member_names.map { |n| "    .#{n} = #{n}" }.join(",\n")
+  end
+
+  def top
+    <<-EOS
+// wrapper that calls #{zoo_fn_name} via #{wrapper_fn.name} inside rb_thread_blocking_region
+#{fn_signature} {
+  #{struct.name} args = {
+    .rc = ZKRB_FAIL,
+#{initializer_lines}
+  };
+    EOS
+  end
+
+  def rb_thread_blocking_region_call
+    "  zkrb_thread_blocking_region(#{wrapper_fn.name}, (void *)&args);"
+  end
+
+  def bottom
+    <<-EOS
+
+  return args.rc;
+}
+    EOS
+  end
+
+  def body
+    @body ||= [top, rb_thread_blocking_region_call, bottom].join("\n")
+  end
+end
+
+class GeneratedCode < Struct.new(:structs, :wrapper_fns, :calling_fns)
+  def initialize(*a)
+    super
+
+    self.structs     ||= []
+    self.wrapper_fns ||= []
+    self.calling_fns ||= []
+  end
+
+  def self.from_zookeeper_h(text)
+    new.tap do |code|
+      while true
+        break unless text =~ REGEXP
+        text = $~.post_match
+
+        zoo_fn_name, argstr = $1
+        argstr = $2
+
+        typed_args = argstr.split(',').map(&:strip)
+
+        # gah, fix up functions which have a void_completion_t with no name assigned
+        if idx = typed_args.index('void_completion_t')
+          typed_args[idx] = 'void_completion_t completion'
+        end
+
+        struct = CallStruct.new(zoo_fn_name, typed_args)
+        wrapper_fn = WrapperFunction.new(zoo_fn_name, struct)
+        calling_fn = CallingFunction.new(zoo_fn_name, struct, wrapper_fn)
+
+        code.structs << struct
+        code.wrapper_fns << wrapper_fn
+        code.calling_fns << calling_fn
+      end
+    end
+  end
+end
+
+def render_header_file(code)
+  StringIO.new('zkrb_wrapper.h', 'w').tap do |fp|
+    fp.puts <<-EOS
+#ifndef ZKRB_WRAPPER_H
+#define ZKRB_WRAPPER_H
+#if 0
+
+  AUTOGENERATED BY #{File.basename(__FILE__)} 
+
+#endif
+
+#include "ruby.h"
+#include "zookeeper/zookeeper.h"
+#include "zkrb_wrapper_compat.h"
+#include "dbg.h"
+
+#define ZKRB_FAIL -1
+
+    EOS
+
+    code.structs.each do |struct|
+      fp.puts(struct.body)
+      fp.puts
+    end
+
+    code.calling_fns.each do |cf|
+      fp.puts "#{cf.fn_signature};"
+    end
+
+    fp.puts <<-EOS
+
+#endif /* ZKRB_WRAPPER_H */
+    EOS
+
+  end.string
+end
+
+def render_c_file(code)
+  StringIO.new('zkrb_wrapper.c', 'w').tap do |fp|
+    fp.puts <<-EOS
+/*
+
+Autogenerated boilerplate wrappers around zoo_* function calls necessary for using
+rb_thread_blocking_region to release the GIL when calling native code.
+
+generated by ext/#{File.basename(__FILE__)}
+
+*/
+
+#include "ruby.h"
+#include "zkrb_wrapper.h"
+#include <errno.h>
+#include <stdio.h>
+#include <stdlib.h>
+
+    EOS
+
+    code.wrapper_fns.zip(code.calling_fns) do |wrap_fn, call_fn|
+      fp.puts "#{wrap_fn.body}\n\n"
+      fp.puts "#{call_fn.body}\n\n"
+    end
+
+  end.string
+end
+
+
+def help!
+  $stderr.puts "usage: #{File.basename(__FILE__)} {all|headers|code}"
+  exit 1
+end
+
+def main
+  help! if ARGV.empty?
+  opts = []
+
+  zookeeper_h_path = Dir[File.join(THIS_DIR, "**/zookeeper.h")].first
+
+  raise "Could not locate zookeeper.h!" unless zookeeper_h_path
+
+  text = File.read(zookeeper_h_path)
+  code = GeneratedCode.from_zookeeper_h(text)
+
+  cmd = ARGV.first
+
+  help! unless %w[headers all code].include?(cmd)
+
+  if %w[headers all].include?(cmd)
+    $stderr.puts "writing #{ZKRB_WRAPPER_H_PATH}"
+    File.open(ZKRB_WRAPPER_H_PATH, 'w') { |fp| fp.write(render_header_file(code)) }
+  end
+
+  if %w[code all].include?(cmd)
+    $stderr.puts "writing #{ZKRB_WRAPPER_C_PATH}"
+    File.open(ZKRB_WRAPPER_C_PATH, 'w') { |fp| fp.write(render_c_file(code)) }
+  end
+
+end
+
+main
+