c_buffer 0.1.2

data/README

@@ -0,0 +1,22 @@
+##
+ #  
+ #                          CBuffer 
+ #    
+ #  Simple circular buffer for staging data before writing to disk
+ #  
+ #  author      hugo benichi
+ #  email       hugo.benichi@m4x.org
+ #  copyright   2012 hugo benichi  
+ #  version     0.1.0
+ #
+ ##
+ 
+  installation:  
+ 
+    run "rake install" in the root directory
+    it will compile the gem library and produce a .gem package for ruby
+    it will then install the .gem automatically
+    
+  usage, test: 
+  
+    cf test/test_cbuffer.rb

data/ext/c_buffer/c_buffer.c

@@ -0,0 +1,166 @@
+/*
+ *  
+ *                          CBuffer 
+ *    
+ *  Simple circular buffer for staging data before writing to disk
+ *  
+ *  author      hugo benichi
+ *  email       hugo.benichi@m4x.org
+ *  copyright   2012 hugo benichi  
+ *  version     0.1.1
+ *
+ */
+ 
+
+#include <stdlib.h>
+#include <stdio.h>
+#include <c_buffer.h>
+#include <assert.h>
+
+
+DLL cbuffer*
+  cbuffer_new(
+    int chunk,      // the size in byte of one chunk of buffer memory
+    int size        // the total size in byte of the buffer, at creation
+  )
+{
+  assert(chunk > 0);
+  assert(size > 0);
+ 
+  int n = size / chunk;
+  if (!n) n = 2;
+  
+  link*  lnk_ary = malloc( n * sizeof(link) );  
+      
+  if ( !lnk_ary )
+    fprintf(stderr,"cbuffer.c: error while allocating memory for new cbuffer\n");
+
+  int i = 0;
+  while( n - i)
+  {
+    lnk_ary[i].mem = malloc(chunk);
+    if( !( lnk_ary[i].mem ) )
+      fprintf(stderr,"cbuffer.c: error while allocating memory for new cbuffer\n");
+    if(i+1 < n) 
+      lnk_ary[i].next = &lnk_ary[i+1];
+    i++;
+  } 
+  lnk_ary[n-1].next = lnk_ary;    
+      
+      
+  cbuffer* cbuf = malloc(sizeof(cbuffer));
+  if ( !cbuf )
+    fprintf(stderr,"cbuffer.c: error while allocating memory for new cbuffer\n");  
+
+  cbuf->inbound   = lnk_ary;      // start feeding at mem[0]
+  //cbuf->start     = lnk_ary;      // remember mem[0]
+  cbuf->start     = NULL;      // remember mem[0]
+  cbuf->outbound  = NULL;         // null until inbound != start
+  cbuf->stop      = NULL;         // when not null inbound as stopped at stop
+  cbuf->output    = NULL;
+  cbuf->chunk     = chunk;
+
+  return cbuf;
+}
+
+
+DLL void    
+  cbuffer_free(cbuffer* buf)
+{
+  link* l = buf->start;
+  link* s = buf->start;
+  while( l != s)
+  {
+    free(l->mem);
+    l = l->next;
+  }
+  free( buf->start );
+  free( buf );
+}
+
+
+DLL void    
+  cbuffer_open(
+    cbuffer*      buf, 
+    const char*   address
+  )
+{
+  buf-> output = fopen(address, "w" );
+  
+  if( !(buf->output) )
+    fprintf(stderr, "cbuffer.c: error while opening file %s\n", address);
+    
+}
+
+
+DLL char*
+  cbuffer_next(cbuffer* buf)
+{
+  if ( buf->start  ) // wait until the next buffer is free
+  {
+    while( buf->inbound->next == buf->outbound ); //busy wait until the next buffer is free
+    buf->inbound = buf->inbound->next;  
+  } else { 
+    buf->start = buf->inbound;  
+  }
+  return buf->inbound->mem;
+}  
+
+  
+DLL char*
+  cbuffer_has_next(cbuffer* buf)
+{
+  char* out = NULL;
+  if ( buf->start  )
+  { 
+    if( buf->inbound != buf->outbound )
+    {
+      buf->inbound = buf->inbound->next;
+      out = buf->inbound->mem;
+    }
+  } else {
+    buf->start = buf->inbound; 
+    out = buf->inbound->mem;
+  }
+  return out;
+}
+
+  
+DLL void
+  cbuffer_stop(cbuffer* buf) 
+{
+  buf->stop = buf->inbound;
+  buf->inbound = buf->inbound->next; 
+}
+
+  
+DLL int
+  cbuffer_write(cbuffer* buf)
+{  
+  while( !( buf->start ) || buf->inbound == buf->start); 
+  // busy wait while the first buffer is written
+
+  buf->outbound = buf->start;
+  
+  int count = 0;
+  while( !( buf->stop ) )
+  { 
+    fwrite( buf->outbound->mem, 1, buf-> chunk, buf-> output);
+    count += buf->chunk;
+
+    buf->outbound = buf->outbound->next;
+    while(buf->outbound == buf->inbound); //busy wait
+  }
+  
+  while(1)
+  { 
+    fwrite( buf->outbound->mem, 1, buf-> chunk, buf-> output);
+    count += 1;
+    if( buf->stop == buf->outbound )
+      break;
+    buf->outbound = buf->outbound->next;
+  }
+   
+  fclose(buf->output);
+  return count;
+}

data/ext/c_buffer/c_buffer.h

@@ -0,0 +1,42 @@
+#include <stdio.h>
+
+
+#if defined(_WIN32)
+  #define 	DLL     extern __declspec(dllexport)
+#else
+  #define   DLL
+#endif
+
+
+typedef   struct cbuffer    cbuffer;
+typedef   struct link       link;
+  
+    
+struct
+  cbuffer               // 192 bytes on 32bits machines and 352 bytes on 64bits machines
+  {
+    link*   inbound;    // next buffer memory for incoming data
+    link*   outbound;   // next buffer memory for writing
+    link*   start;      // start of the memory (used for cleaning and init)
+    link*   stop;       // where to write to
+    FILE*   output;     // file 
+    int     chunk;      // size of one chunk
+  };
+
+
+struct   
+  link
+  {
+    char* mem;
+    link* next;
+  };
+
+
+DLL   cbuffer*  cbuffer_new       (int chunk, int size);    //  size is n_chunk * chunk
+DLL   void      cbuffer_free      (cbuffer*);               // don't forget to close output if open
+DLL   void      cbuffer_open      (cbuffer*, const char* ); // open a new file at the given path
+
+DLL   char*     cbuffer_next      (cbuffer*);               // blocking, return the next buffer memory inline
+DLL   char*     cbuffer_has_next  (cbuffer*);               // non-blocking, return null if not ready
+DLL   void      cbuffer_stop      (cbuffer*);               // flag for stopping (cut the circular list)
+DLL   int       cbuffer_write     (cbuffer*);               // blocking, write everything untill stop

data/lib/c_buffer.rb

@@ -0,0 +1,79 @@
+##
+ #  
+ #                          CBuffer 
+ #    
+ #  Simple circular buffer for staging data before writing to disk
+ #  
+ #  author      hugo benichi
+ #  email       hugo.benichi@m4x.org
+ #  copyright   2012 hugo benichi  
+ #  version     0.1.1
+ #
+ ##
+ 
+# Main class and namespace of the program
+# This class should be instancied to use a buffer
+class CBuffer
+
+  require 'ffi'
+
+  # the API module wraps the native functions of cbuffer.c
+  module API
+    extend FFI::Library
+    lib_type = ENV["OS"] ? "dll" : "so"
+    #ffi_lib $LOAD_PATH.map{ |p| p + "/cbuffer/cbuffer.#{lib_type}" }    
+    ffi_lib File.dirname(__FILE__) + "/c_buffer/c_buffer.#{lib_type}"
+    [
+      [ :cbuffer_new,      [:int32, :int32],      :pointer                    ],
+      [ :cbuffer_free,     [:pointer],            :void                       ],
+      [ :cbuffer_open,     [:pointer, :pointer],  :void                       ],
+      [ :cbuffer_next,     [:pointer],            :pointer, :blocking => true ],
+      [ :cbuffer_has_next, [:pointer],            :pointer                    ],
+      [ :cbuffer_stop,     [:pointer],            :void                       ],
+      [ :cbuffer_write,    [:pointer],            :int32,   :blocking => true ],
+    ].each{ |sig| attach_function *sig }
+  end
+  
+  # the CBufferRaw class wraps the native struct memory used to store the buffer memory
+  # not used at the momentm here for reference only
+  class CBufferRaw < FFI::Struct
+    layout  :inbound,   :pointer,
+            :outbound,  :pointer, 
+            :start,     :pointer,
+            :stop,      :pointer,
+            :output,    :pointer,
+            :chunk,     :int   
+  end
+  
+  def initialize args 
+    chunk   = args[:chunk]  || 1
+    length  = args[:length] || 2
+    path    = args[:path]   || 1
+    @struct = API::cbuffer_new chunk, length
+    API::cbuffer_open @struct, path
+    self
+  end
+
+  def write &block
+    running = true
+    self.define_singleton_method :stop do
+      API.cbuffer_stop @struct
+      sleep 1 while running
+      API.cbuffer_free @struct
+    end
+    Thread.new do
+      begin
+        count = API.cbuffer_write @struct
+        running = false
+        block.call(count) if block 
+      catch
+        puts "error in writing thread"
+      end
+    end
+  end
+
+  def next
+    API.cbuffer_next @struct
+  end
+
+end

data/rakefile.rb

@@ -0,0 +1,32 @@
+
+name = "c_buffer"
+
+lib_type = ENV["OS"] ? "dll" : "so"
+lib_file = "./lib/%s/%s.%s" % [name,name,lib_type]
+src_file = "./ext/%s/%s.c"  % [name,name]
+hdr_path = "./ext/%s"       % name
+
+flags = ENV["OS"] ? "-D _WIN32" : "-fPIC"
+
+task :compile => lib_file
+
+task lib_file => src_file do
+  sh "gcc -shared -o #{lib_file} #{src_file} -I#{hdr_path} #{flags}"
+end
+
+task :test_global do
+  ruby "test/test_%s.rb" % name
+end
+
+task :test_local do
+  ruby "-I./lib test/test_%s.rb" % name
+end
+
+task :gem_install => :gem_build do
+  gemfile = Dir.new("./").entries.select{ |f| f =~ /cbuffer-[\d]+\.[\d]+\.[\d]+.gem/ }[0]
+  sh "gem install %s" % gemfile
+end
+
+task :gem_build => :compile do
+  sh "gem build %s.gemspec" % name
+end

data/test/test_c_buffer.rb

@@ -0,0 +1,17 @@
+require 'c_buffer'
+
+repet   = 126
+chunk   = 1024*126
+length  = chunk * 64
+
+
+buf = CBuffer.new( path: "./data", chunk: chunk, length: length )
+buf.write{ |count| puts "wrote #{count} bytes of data" }
+
+data = Array.new(chunk){rand 255}
+
+repet.times do |i|
+  buf.next.put_array_of_uchar 0, data
+end
+
+buf.stop

metadata

@@ -0,0 +1,64 @@
+--- !ruby/object:Gem::Specification
+name: c_buffer
+version: !ruby/object:Gem::Version
+  version: 0.1.2
+  prerelease: 
+platform: ruby
+authors:
+- Hugo Benichi
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2012-07-05 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: ffi
+  requirement: &19284820 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *19284820
+description: A simple circular buffer and its Ruby interface to go along RbScope and
+  RbVisa. Perfect for staging high throughput input data to low throughput I/O.
+email: hugo.benichi@m4x.org
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/c_buffer.rb
+- lib/c_buffer/c_buffer.so
+- lib/c_buffer/c_buffer.dll
+- ext/c_buffer/c_buffer.c
+- ext/c_buffer/c_buffer.h
+- rakefile.rb
+- README
+- test/test_c_buffer.rb
+homepage: http://github.com/hugobenichi/cbuffer
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.11
+signing_key: 
+specification_version: 3
+summary: A simple circular buffer and its Ruby interface
+test_files: []