oinky 0.1.0

data/lib/oinky/rb_emitter.rb

@@ -0,0 +1,147 @@
+# This source is distributed under the terms of the MIT License.  Refer 
+# to the 'LICENSE' file for details.
+# 
+# Copyright (c) Jacob Lacouture, 2012
+
+require 'oinky/normalize.rb'
+
+module Oinky
+  module Model
+    class Ruby
+      def initialize(schema)
+        @schema = Oinky::Model.normalize_schema(schema)
+      end
+
+      def rbify_method(str)
+        str = str.gsub(/[^a-z0-9A-Z_]/,'_')
+        return str unless /$[^a-z]/ =~ str
+        return "_" + str
+      end
+      
+      def rbify_chars(str)
+        str = str.gsub(/[^a-z0-9A-Z_]/,'_')
+      end
+      
+      def schema_classname
+        cn = @schema[:classname]
+        return "S#{@schema[:name].to_s}Adapter" unless cn
+        return cn[:ruby] if cn.is_a? Hash
+        return cn.call(:ruby) if cn.is_a? Proc
+        return cn.to_s
+      end
+
+      def tbl_classname(str) ; "Table_#{rbify_chars(str)}" ; end
+      #def handle_from_tablename(str) ; "#{cppify(str)}_handle" ; end
+      #def accessor_from_tablename(str) ; cppify(str) ; end
+
+      def default_value_expression(col)
+        value = col
+        value = col[:default] if col.is_a? Hash
+        return "DateTime.parse(#{value.to_s.inspect})" if value.is_a? DateTime
+        return value.inspect
+      end
+
+      def to_ruby(h)
+        str = ['{']
+        h.each {|k,v|
+          str << k.inspect + "=>"
+          if v.is_a? Hash
+            str << to_ruby(v)
+          elsif v.is_a? DateTime
+            str << default_value_expression(v)
+          else
+            str << v.inspect
+          end
+          str << ','
+        }
+        str << '}'
+        return str * ''
+      end
+      
+      # Generate a string, containing the Ruby code for the model classes.
+      # this can be eval'd directly in ruby, or saved and required 
+      # more normally.  Supporting both lends itself to static analysis 
+      # and transformation moreso than if we solved this with ruby 
+      # metaprogramming.
+      #
+      def emit
+        p = Oinky::Detail::Builder.new
+        schemacls = self.schema_classname
+        p.next("class #{schemacls}", "end ##{schemacls}") {
+          tables = @schema[:tables]
+          p.next("def self.schema", "end") {
+            p << to_ruby(@schema.merge(:classname=>schemacls))
+          }
+          p.next("def schema", "end") {
+            p << 'self.class.schema'
+          }
+          p << ''
+          p.next("def inspect", "end") {
+            # Hide all the internal state, which gets really obnoxious
+            p << '"#<#{self.class}:0x#{self.__id__.to_s(16)}>"'
+            # This is just a bit long to include.
+            #schema=#{self.schema.to_s}>"'
+          }
+          p << ''
+          tables.each{|nm,t|
+            clsname = tbl_classname(nm)
+            p.next("class #{clsname}","end ##{clsname}") {
+              p << "include Oinky::Model::Table"
+              p << ''
+              p.next("def initialize(tbl)","end") {
+                p << "@tbl = tbl"
+                p << "self.class.migrate(tbl)"
+                p << ''
+                t[:indices].each {|k,v|
+                  accessor = rbify_method(v[:accessor])
+                  p << "@#{accessor} = @tbl.indices[#{k.inspect}]"
+                }
+              }
+              p << ''
+              p.next("def self.migrate(tbl)","end") {
+                p.next("tbl.db.atomic {", "}") {
+                  p << "cols = tbl.columns"
+                  t[:columns].each{|k,v|
+                    p.next("if c = cols[#{k.inspect}]","end") {
+                      p << "raise OinkyException.new('Mismatched column type for column [#{k.to_s}]') unless c[:type] == :#{v[:type]}"
+                      p.write("else",-1)
+                      p << "tbl.add_column(#{k.inspect},#{v[:type].inspect},#{default_value_expression(v)})"
+                    }                  
+                  }
+                  p << "ixs = tbl.indices"
+                  t[:indices].each{|k,v|
+                    p.next("unless ixs[#{k.inspect}]","end") {
+                      p << "tbl.add_index(:name=>#{k.inspect},:unique=>#{v[:unique]},:columns=>#{v[:columns]})"
+                    }                  
+                  }
+                }
+              }
+              p << ''
+              t[:indices].each {|k,v|
+                accessor = rbify_method(v[:accessor])
+                p << "attr_accessor :#{accessor}"
+              }
+              p << "def table ; @tbl ; end"
+              p << "alias :t :table"
+            } #tablecls
+            p << ''
+          } #tables.each
+
+          p.next("def initialize(db, prefix = '')","end") {
+            tables.each{|nm,t|
+              accessor = rbify_method(t[:accessor])
+              p << "tn = prefix + #{nm.inspect}"
+              p << "db[tn] or db.create_table(tn)"
+              p << "@#{accessor} = #{tbl_classname(nm)}.new(db[tn])"
+            }
+          }
+          tables.each{|nm,t|
+            accessor = rbify_method(t[:accessor])
+            p << "attr_accessor :#{accessor}"
+          }
+        } #schemacls
+        return p.format
+      end #emit
+    end #class Ruby
+  end #Model
+end #Oinky

data/lib/oinky/shard.rb

@@ -0,0 +1,40 @@
+s = build_schema {
+  create_table(:foo) {
+    string :key
+    string :type
+    string :payload
+
+    datetime :timestamp
+
+    # Create an index on this table (called 'id') which is unique
+    # over the given columnset.  The resolver requires that this field
+    # be identified.
+    identity [:key]
+
+    # When facing two identical 
+    resolve_on :timestamp
+
+    #shard_on :key
+    #shard_on lambda {|r| r[:key]}
+  }
+
+  create_table(:footype) {
+    # Make sure that on any shard, all types are available
+    shard_join([:foo,:type], [:footype,:type])
+  }
+
+  # This is how we generate the names of the storage objects.
+  # fooset_0, fooset_1, etc.
+  shardset_name :fooset
+
+}
+
+# A table spans many shards.  To search for an exact row, we can generate
+# a shard key from the given row.  However, to enumerate a range of rows,
+# we need to generate a range of shard keys.  (The range may only be one
+# shard...but in general we'll need to do enumerations across shards).
+# The atlantic DB is 150BM with just half of 2012.
+
+#In the implementation, we just need a set of shard keys.
+
+

data/lib/oinky/testsup.rb

@@ -0,0 +1,104 @@
+# Test-related support code for oinky
+
+module Oinky
+module Test
+
+class DBCompare
+  def check_eq(l,r)
+    raise RuntimeError.new("Inequality: [#{l.inspect}] != [#{r.inspect}]  Context:[#{@context.to_s}]") unless l == r
+  end
+
+  def kv_compare(ls, rs, fn)
+    check_eq(ls.size,rs.size)
+    ls.each{|k,v|
+      @context.push k
+      fn.call(v, rs[k])
+      @context.pop
+    }
+  end
+  def seq_compare(ls, rs, fn, neq_prev = false)
+    l = ls.each
+    r = rs.each
+    count = 0
+    last = self
+    begin
+      while true
+        left = true
+        lv = l.next
+        left = false
+        rv = r.next
+        @context.push count
+        fn.call(lv,rv)
+
+        # If we are verifying that each list is also unique, then
+        # check against the previous value
+        if neq_prev and (last != self)
+          # We EXPECT this comparison to raise NEQ
+          peq = false
+          begin
+            fn.call(last, lv)
+            peq = true
+          rescue
+          end
+          if peq
+            check_eq("Sequence is not unique",nil)
+          end
+        end
+        last = lv
+
+        @context.pop
+        count += 1
+      end
+    rescue StopIteration
+      # Left should have raised.
+      check_eq(left, true)
+    end
+    # verify that the right enumerator is also done
+    begin
+      r.next
+      # These are obviously not equal if the above didn't raise.
+      check_eq("nonempty enumerator",nil)
+    rescue StopIteration
+    end
+  end
+  
+  def table_compare(l, r)
+    kv_compare(l.columns, r.columns, self.method(:check_eq))
+    check_eq(l.row_count, r.row_count)
+    # Table ordering is not guaranteed to match.  We have to sort.
+    # However, we may have to sort over variant types, so convert
+    # everything to a string before ordering.
+    seq_compare(
+                l.cursor_each.map {|c| c.select_all_values.map{|v| v.to_s} }.sort, 
+                r.cursor_each.map {|c| c.select_all_values.map{|v| v.to_s} }.sort, 
+                self.method(:check_eq))
+    kv_compare(l.indices, r.indices, self.method(:index_compare))
+  end
+  
+  def index_compare(l, r)
+    kv_compare(l.definition, r.definition, self.method(:check_eq))
+    check_eq(l.row_count, r.row_count)
+    # We will only compare the indexed columns.  Building the selectors
+    # outside of the loop saves work.
+    ls = l.selector
+    rs = r.selector
+    seq_compare(l.each(ls), r.each(rs), self.method(:check_eq), l.definition[:unique])
+  end
+
+  def compare(l, r)
+    begin
+      kv_compare(l.tables, r.tables, self.method(:table_compare))
+    rescue StandardError => e
+      puts "Exception comparing database instances: [#{@context}]"
+      raise e
+    end
+    self
+  end
+  def initialize
+    @context = []
+  end
+end
+
+
+end #module Test
+end #module Oinky

data/lib/oinky/version.rb

@@ -0,0 +1,9 @@
+# This source is distributed under the terms of the MIT License.  Refer 
+# to the 'LICENSE' file for details.
+# 
+# Copyright (c) Jacob Lacouture, 2012
+
+module Oinky
+  VERSION = "0.1.0"
+end
+

data/oinky.gemspec

@@ -0,0 +1,36 @@
+# -*- coding: utf-8 -*-
+
+class T
+def self.expand_dirname(__file__)
+  File.expand_path(__file__).sub(/\/[^\/]+$/,'')
+end
+end
+
+gem_root = T.expand_dirname(__FILE__)
+
+require(File.join(gem_root,'lib/oinky/version.rb'))
+ 
+Gem::Specification.new do |s|
+  s.name        = "oinky"
+  s.version     = Oinky::VERSION
+  # There's no version of FFI that's consistent across MRI, JRuby, and RBX.
+  # Hopefully that will get fixed, but for now it has to be MRI.
+  #   The relevant issues are FFI::Struct.by_value and MemoryPointer.get_int64
+  s.platform    = Gem::Platform::RUBY #['mri_19'] <== this isn't recognized??
+  s.authors     = ["Jacob Lacouture"]
+  s.email       = ["jacobx984@gmail.com"]
+  s.homepage    = "http://github.com/oinky/oinky"
+  s.summary     = "Relational data serialization library"
+  s.description = "Oinky is an in-memory relational database which can serialize and mount extremely quickly, making it an ideal solution for relational microsharding atop distributed key-value storage."
+ 
+  s.rubyforge_project         = "oinky"
+ 
+  s.add_development_dependency "rake"
+  s.add_dependency('ffi','>= 1.0.11')
+
+  s.files        = Dir.glob("**/*")
+  s.executables  = []
+  s.extensions << "ext/extconf.rb"
+
+  s.require_path = 'lib'
+end

metadata

@@ -0,0 +1,120 @@
+--- !ruby/object:Gem::Specification
+name: oinky
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+  prerelease: 
+platform: ruby
+authors:
+- Jacob Lacouture
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2012-06-06 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: ffi
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 1.0.11
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 1.0.11
+description: Oinky is an in-memory relational database which can serialize and mount
+  extremely quickly, making it an ideal solution for relational microsharding atop
+  distributed key-value storage.
+email:
+- jacobx984@gmail.com
+executables: []
+extensions:
+- ext/extconf.rb
+extra_rdoc_files: []
+files:
+- ext/extconf.rb
+- ext/include/oinky/nky_base.hpp
+- ext/include/oinky/nky_core.hpp
+- ext/include/oinky/nky_cursor.hpp
+- ext/include/oinky/nky_dialect.hpp
+- ext/include/oinky/nky_error.hpp
+- ext/include/oinky/nky_fixed_table.hpp
+- ext/include/oinky/nky_handle.hpp
+- ext/include/oinky/nky_index.hpp
+- ext/include/oinky/nky_log.hpp
+- ext/include/oinky/nky_merge_itr.hpp
+- ext/include/oinky/nky_model.hpp
+- ext/include/oinky/nky_pool.hpp
+- ext/include/oinky/nky_public.hpp
+- ext/include/oinky/nky_serializer.hpp
+- ext/include/oinky/nky_strtable.hpp
+- ext/include/oinky/nky_table.hpp
+- ext/include/oinky.h
+- ext/include/oinky.hpp
+- ext/nky_lib.cpp
+- ext/nky_lib_core.hpp
+- ext/nky_lib_index.cpp
+- ext/nky_lib_table.cpp
+- lib/oinky/compiler.rb
+- lib/oinky/cpp_emitter.rb
+- lib/oinky/dsl.rb
+- lib/oinky/error.rb
+- lib/oinky/modelbase.rb
+- lib/oinky/nbuffer.rb
+- lib/oinky/normalize.rb
+- lib/oinky/oc_builder.rb
+- lib/oinky/query.rb
+- lib/oinky/rb_emitter.rb
+- lib/oinky/shard.rb
+- lib/oinky/testsup.rb
+- lib/oinky/version.rb
+- lib/oinky.rb
+- LICENSE
+- oinky.gemspec
+- README.md
+homepage: http://github.com/oinky/oinky
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: oinky
+rubygems_version: 1.8.24
+signing_key: 
+specification_version: 3
+summary: Relational data serialization library
+test_files: []