tiny_ds 0.0.1

data/LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2009-2010 Takeru Sasaki, sasaki.takeru@gmail.com
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.rdoc

@@ -0,0 +1,25 @@
+* TinyDS -- tiny datastore library for GAE/JRuby
+  github: http://github.com/takeru/tiny_ds
+
+ - CRUD like a ActiveRecord or DataMapepr
+ - set parent, key, id, name, and find by these, YES entity-group-transaction!!
+ - BASE transaction : http://blog.notdot.net/2009/9/Distributed-Transactions-on-App-Engine
+ - query : very thin wrapper of low level API.
+ - Shout demo app. (sinatra)
+ - some specs..
+  
+** run shout demo
+ % dev_appserver.rb .
+
+** how to run specs
+install jruby 1.4.0
+ % source set_classpath_for_jruby.sh (set classpath, please read spec/spec_helper.rb)
+ % jruby -S spec -c -b spec/basic_spec.rb
+   or 
+ % jruby -e "ENV['RSPEC']='true'; ENV['AUTOTEST']='true'; system('jruby -S autotest', *ARGV)"
+
+** demo
+Rubyist Social Graph : http://rubyist-sg.appspot.com/ is running with GAE/JRuby+Rails2.3.5+TinyDS.
+
+** author
+  takeru : sasaki.takeru@gmail.com, twitter:@urekat (almost japanese!), blog:http://d.hatena.ne.jp/urekat

data/Rakefile

@@ -0,0 +1,45 @@
+require 'rubygems'
+require 'rake/rdoctask'
+require 'rake/gempackagetask'
+require 'rubygems/specification'
+require 'date'
+
+require File.dirname(__FILE__) + '/lib/tiny_ds/version'
+
+# set up pretty rdoc if possible
+begin
+  gem 'rdoc'
+  require 'sdoc'
+  ENV['RDOCOPT'] = '-T lightblue'
+rescue Exception
+end
+
+spec = Gem::Specification.new do |s|
+  s.name = "tiny_ds"
+  s.version = TinyDS::VERSION
+  s.platform = Gem::Platform::RUBY
+  s.has_rdoc = true
+  s.extra_rdoc_files = ["README.rdoc", "LICENSE"]
+  s.description = "Tiny datastore library for Google App Engine with JRuby"
+  s.summary = "Supports CRUD like a ActiveRecord or DataMapepr " +
+              "but with parent/child and entity-group-transaction"
+  s.author = "Takeru Sasaki"
+  s.email = "sasaki.takeru@gmail.com"
+  s.homepage = "http://github.com/takeru/tiny_ds"
+  s.require_path = 'lib'
+  s.files = %w(LICENSE README.rdoc Rakefile) +
+            Dir.glob("spec/**/*") + Dir.glob("lib/**/*") 
+  s.add_dependency('appengine-apis')
+end
+
+task :default => :gem
+
+Rake::GemPackageTask.new(spec) do |pkg|
+  pkg.gem_spec = spec
+end
+
+Rake::RDocTask.new do |rd|
+   rd.main = "README.rdoc"
+   rd.rdoc_files.include("README.rdoc", "lib/**/*.rb")
+end
+

data/lib/tiny_ds/base.rb

@@ -0,0 +1,290 @@
+# TODO
+#   callback before/after
+#   validations
+#   associations
+#x   default value
+#x   property list
+#   property Key
+#   property others
+#   nil=>false
+#   index=>false setUnindexedProperty
+#   version
+#   schema_version
+#x   new_record?
+#   find_by_xxx
+#x   get_by_id(id)         get_by_name(name)         : root entity only!!
+#x   get_by_id(parent, id) get_by_name(parent, name) : ancestor+Kind => key get
+#x   Foo.query.filter(:key=>123)
+#   Foo.filter(:key=>123)
+#   logger
+#   logging low level API calls.
+
+require "time"
+
+module TinyDS
+class Base
+  class << self; attr_accessor :_property_definitions; end
+  RESERVED_PROPERTY_NAME = [:id, :name, :key, :entity, :parent_key, :parent]
+  def self.property(pname, ptype, opts={})
+    pname = pname.to_sym
+    if RESERVED_PROPERTY_NAME.include?(pname)
+      raise "property name '#{pname}' is reserved."
+    end
+    property_definitions[pname] = PropertyDefinition.new(pname, ptype, opts)
+  end
+
+  def self.property_definitions
+    self._property_definitions ||= {}
+  end
+
+  def self.property_definition(name)
+    property_definitions[name.to_sym] or raise "unknown property='#{name}'"
+  end
+
+  def self.default_attrs
+    attrs = {}
+    property_definitions.each do |pname,pdef|
+      if pdef.has_default?
+        attrs[pname] = pdef.default_value
+      end
+    end
+    attrs
+  end
+
+  # kind-string of entity
+  def self.kind
+    name
+  end
+
+  #include ActiveModel::Naming
+#  def self.model_name # for form-builder
+#    @model_name ||= ::ActiveModel::Name.new(self, kind)
+#  end
+
+  # foo.key
+  def key
+    @entity.key
+  end
+  def id
+    key.id
+  end
+  def name
+    key.name
+  end
+
+  # foo.parent_key
+  def parent_key
+    @entity.parent
+  end
+
+  def self.to_key(m)
+    case m
+    when AppEngine::Datastore::Key
+      m
+    when AppEngine::Datastore::Entity
+      m.key
+    when String
+      LowDS::KeyFactory.stringToKey(m)
+    else
+      if m.kind_of?(Base)
+        m.key
+      else
+        raise "unknown key type #{m.class}/#{m.inspect}"
+      end
+    end
+  end
+
+  # Foo.create({:title=>"hello",...}, :parent=>aaa, :id=>bbb, :name=>ccc, :key=>...)
+  def self.create(attrs={}, opts={})
+    m = new(attrs, opts)
+    m.save
+    m
+  end
+
+  # Foo.new
+  def initialize(attrs={}, opts={})
+    @entity = opts.delete(:entity)
+    unless @entity
+      if opts[:parent] && opts[:parent].kind_of?(Base)
+        opts = opts.dup
+        opts[:parent] = opts[:parent].entity
+      end
+      @entity = LowDS.build(self.class.kind, {}, opts)
+      self.attributes = self.class.default_attrs.merge(attrs || {})
+      @new_record = true
+    else
+      @new_record = false
+    end
+  end
+
+  def self.new_from_entity(_entity)
+    new(nil, :entity=>_entity)
+  end
+  attr_reader :entity
+
+  def new_record?
+    @new_record
+  end
+
+  # foo.save
+  def save
+    do_save
+    true
+  end
+
+  def do_save
+    __before_save_set_timestamps
+#    if @new_record && @entity.key && parent
+#      TinyDS.tx{
+#        if LowDS.get(@entity.key)
+#          raise KeyIsAlreadyTaken
+#        end
+#        LowDS.save(@entity)
+#      }
+#    else
+      LowDS.save(@entity)
+#    end
+    @new_record = false
+    nil
+  end
+#  class KeyIsAlreadyTaken < StandardError
+#  end
+
+  def __before_save_set_timestamps
+    if self.class.property_definitions[:created_at] && new_record?
+      self.created_at = Time.now
+    end
+    if self.class.property_definitions[:updated_at]
+      self.updated_at = Time.now
+    end
+  end
+
+  # Foo.get(key)
+  def self.get!(key)
+    self.new_from_entity(LowDS.get(key, :kind=>self.kind))
+  end
+  def self.get(key)
+    get!(key)
+  rescue AppEngine::Datastore::EntityNotFound => e
+    nil
+  end
+
+  def self._get_by_id_or_name!(id_or_name, parent)
+    key = if parent
+            parent = to_key(parent)
+            kfb = LowDS::KeyFactory::Builder.new(parent)
+            kfb.addChild(kind, id_or_name)
+            kfb.key
+          else
+            LowDS::KeyFactory::Builder.new(kind, id_or_name).key
+          end
+    get!(key)
+  end
+
+  def self.get_by_id!(id, parent=nil)
+    if id.kind_of?(String) && id==id.to_i.to_s
+      id = id.to_i
+    end
+    raise "id is not Integer" unless id.kind_of?(Integer)
+    _get_by_id_or_name!(id, parent)
+  end
+  def self.get_by_name!(name, parent=nil)
+    raise "id is not String" unless name.kind_of?(String)
+    _get_by_id_or_name!(name, parent)
+  end
+  def self.get_by_id(id, parent=nil)
+    if id.kind_of?(String) && id==id.to_i.to_s
+      id = id.to_i
+    end
+    _get_by_id_or_name!(id, parent)
+  rescue AppEngine::Datastore::EntityNotFound => e
+    nil
+  end
+  def self.get_by_name(name, parent=nil)
+    _get_by_id_or_name!(name, parent)
+  rescue AppEngine::Datastore::EntityNotFound => e
+    nil
+  end
+
+#  # Foo.find
+#  def self.find(*args)
+#    raise "todo"
+#    direction = dire==:desc ? AppEngine::Datastore::Query::DESCENDING : AppEngine::Datastore::Query::ASCENDING
+#    AppEngine::Datastore::Query.new("TxSrc").
+#      filter(:tx_kind, AppEngine::Datastore::Query::EQUAL, tx_kind).
+#      filter(:status,  AppEngine::Datastore::Query::EQUAL, status).
+#      sort(:created_at, direction)
+#  end
+
+  def self.query
+    Query.new(self)
+  end
+
+  def self.count
+    query.count
+  end
+
+  # foo.destroy
+  def destroy
+    self.class.destroy(self)
+  end
+
+  # Foo.destroy([model, entity, key, ...])
+  def self.destroy(array)
+    array = [array] unless array.kind_of?(Array)
+    keys = array.collect do |m|
+      to_key(m)
+    end
+    AppEngine::Datastore.delete(keys)
+  end
+  def self.destroy_all
+    destroy(query.keys)
+  end
+
+  # set attributes
+  def attributes=(attrs)
+    attrs.each do |k,v|
+      set_property(k, v)
+    end
+    nil
+  end
+
+  # set property-value into @entity
+  def set_property(k,v)
+    prop_def = self.class.property_definition(k)
+    ds_v = prop_def.to_ds_value(v)
+    if ds_v.nil?
+      @entity.removeProperty(k)
+    else
+      @entity[k] = ds_v
+    end
+    # todo cache value read/write
+  end
+
+  # get property-value from @entity
+  def get_property(k)
+    prop_def = self.class.property_definition(k)
+    prop_def.to_ruby_value(@entity[k])
+  end
+
+  def method_missing(m, *args)
+    k, is_set = if m.to_s =~ /(.+)=$/
+                  [$1.to_sym, true]
+                else
+                  [m.to_sym, false]
+                end
+    if prop_def = self.class.property_definitions[k]
+      # TODO define method.
+      if is_set
+        raise if args.size!=1
+        set_property(k, args.first)
+      else
+        raise if args.size!=0
+        get_property(k)
+      end
+    else
+      super(m, *args)
+    end
+  end
+end
+end

data/lib/tiny_ds/base.rb~

@@ -0,0 +1,115 @@
+=begin
+  special pname
+    - key
+    - parent
+    - id
+    - name
+=end
+
+
+class ModelBase
+  @@properties = {}
+  def self.property(pname, ptype, opts={})
+    pname = pname.to_sym
+    if @@properties[pname]
+      raise "duplicated pname=#{pname}"
+    end
+    @@properties[pname] = Property.new(pname, ptype, opts)
+  end
+
+  # kind-string of entity
+  def self.kind
+    name
+  end
+
+  def self.tx(retries=0, &block)
+    transaction{
+      yield(block)
+    }
+  end
+
+  # Foo.create(attrs
+  def self.create(attrs={})
+    m = new(attrs)
+    m.save
+    m
+  end
+
+  # Foo.new
+  def initialize(attrs={})
+    attributes = attrs
+  end
+
+  def self.new_from_entity(entity)
+    m = new()
+  end
+
+  # foo.save
+  def save
+    _save
+  end
+
+  def _save
+  end
+
+  # Foo.get(key)
+  def self.get(key)
+    if key.kind_of?(String)
+      key = LowDS::KeyFactory.stringToKey(key)
+    end
+    raise "invalid key=#{key}" unless key.kind_of?(AppEngine::Datastore::Key)
+    self.new_from_entity(LowDS.get(key, :kind=>self.kind))
+  end
+
+  def self.find(*args)
+    
+  end
+
+  # foo.destroy
+  def destroy
+  end
+
+  def attributes=(attrs)
+    # TODO key, parent, id, name
+    
+    attrs.each do |k,v|
+      prop = @@properties[k.to_sym]
+      prop.value = v
+    end
+    nil
+  end
+
+  def method_missing(m, *args)
+    prop, is_set = if m =~ /(.+)=$/
+                     [@@properties[$1.to_sym], true]
+                   else
+                     [@@properties[m.to_sym], false]
+                   end
+    if prop
+      # TODO define method.
+      if is_set
+        raise if args.size!=1
+        prop.value = args.first
+      else
+        raise if args.size!=0
+        prop.value
+      end
+    else
+      super(m, *args)
+    end
+  end
+
+  class Property
+    def initialize(pname, ptype, opts)
+      @pname = pname
+      @ptype = ptype
+      @opts  = opts
+    end
+    def value=(v)
+      # check_type or cast
+      @value = v
+    end
+    attr_reader :value
+  end
+end
+

data/lib/tiny_ds/base_tx.rb

@@ -0,0 +1,161 @@
+# memo デプロイ時に未完了のトランザクションがあってトランザクション定義がかわったら？version???
+module TinyDS
+class BaseTx
+  def src_phase(src, args)
+    raise "no impl."
+  end
+  def dest_phase(dest, args)
+    raise "no impl."
+  end
+  def self.tx_kind
+    name
+  end
+  def roll_forward_retries_limit
+    100
+  end
+  def tx_retries
+    3
+  end
+  attr_reader :tx_key
+
+  # トランザクション実行
+  #          :  src_phase   dest_phase
+  #   raised :    failed      ----
+  #   false  :    OK          failed
+  #   true   :    OK          OK
+  def self.exec(src, dest, args={})
+    tx = new
+    tx.create_tx(src, dest, args)
+    begin
+      tx.roll_forward
+    rescue AppEngine::Datastore::TransactionFailed => e
+      return false
+    end
+    return true
+  end
+
+  def self.roll_forward_all(limit=50)
+    pending_tx_query.each(:limit=>limit) do |tx_src|
+      tx = new
+      tx.restore_tx(tx_src)
+      begin
+        tx.roll_forward
+      rescue => e
+        #p "roll_forward failed. tx=[#{tx.tx_key}] e=[#{e.inspect}]"
+      end
+    end
+    nil
+  end
+
+  def self.pending_tx_query(status="pending", dire=:asc) # pending/done/failed
+    TxSrc.query.
+      filter(:tx_kind, "==", tx_kind).
+      filter(:status,  "==", status).
+      sort(:created_at, dire)
+  end
+
+  # doneなTxSrcと対応するTxDoneを削除する
+#  def self.delete_done_tx
+#  end
+
+  class TxSrc < Base
+    property :tx_kind,                   :string
+    property :dest_key,                  :string
+    property :status,                    :string,  :default=>"pending"
+    property :roll_forward_failed_count, :integer, :default=>0
+    property :args,                      :text
+    property :created_at,                :time
+    property :done_at,                   :time
+  end
+
+  # トランザクション前半
+  #   TODO TxIDを指定できるようにする。TxSrc#key.nameにTxIDを指定して重複実行防止
+  def create_tx(src, dest, args)
+    tx_src = nil
+    TinyDS.tx(tx_retries){
+      src = src.class.get(src.key)
+      src_phase(src, args)
+      src.save!
+
+      attrs = {
+        :tx_kind  => self.class.tx_kind,
+        :dest_key => dest.key.to_s,
+        :args     => args.to_yaml,
+      }
+      tx_src = TxSrc.create!(attrs, :parent=>src) # srcがparent, tx_srcがchild
+      # COMMIT:「srcの処理(=src_phase)、TxSrc作成」
+    }
+    @tx_key = tx_src.key
+    nil
+  end
+
+  def restore_tx(tx_src)
+    @tx_key = tx_src.key
+    nil
+  end
+
+  class TxDone < Base
+    property :done_at, :time
+  end
+
+  # トランザクション後半
+  def roll_forward
+    tx_src = TxSrc.get(@tx_key)
+
+    TinyDS.tx(tx_retries){
+      dest_key = LowDS::KeyFactory.stringToKey(tx_src.dest_key)
+      dest = dest_key.kind.constantize.get(dest_key)
+      done_name = "TxDone_#{@tx_key.to_s}"
+      done_key = LowDS::KeyFactory.createKey(dest.key, TxDone.kind, done_name)
+      begin
+        TxDone.get!(done_key)
+        # なにもしない : TxDoneが存在しているということはdest_phaseは処理済み
+      rescue AppEngine::Datastore::EntityNotFound => e
+        # TxDoneが無い→dest_phaseが未実行
+        attrs = {:done_at=>Time.now}
+        tx_done = TxDone.create!(attrs, :parent=>dest, :name=>done_name)
+        dest_phase(dest, YAML.load(tx_src.args)) # destの処理を実行
+        dest.save!
+      end
+      # memo: done_keyが同じTxDoneをcommitしようとするとTransactionFailedになるはず→dest_phaseもキャンセル
+      # COMMIT:「destの処理(=dest_phase)、TxDone作成」
+    }
+
+    # TxSrc#statusをdoneに
+    TinyDS.tx(tx_retries){
+      tx_src = TxSrc.get!(@tx_key)
+      if tx_src.status=="pending"
+        tx_src.status = "done"
+        tx_src.done_at = Time.now
+        tx_src.save!
+      end
+    }
+    return true
+  rescue => e
+    puts e.inspect
+    TinyDS.tx(tx_retries){
+      tx_src = TxSrc.get!(@tx_key)
+      tx_src.roll_forward_failed_count += 1
+      if roll_forward_retries_limit < tx_src.roll_forward_failed_count
+        tx_src.status = "failed"
+      end
+      tx_src.save!
+    }
+    return false
+  end
+
+  if false
+    require "benchmark"
+    def create_tx(src, dest, args)
+      RAILS_DEFAULT_LOGGER.info ["create_tx", Benchmark.measure{
+        _create_tx(src, dest, args)
+      }].inspect
+    end
+    def roll_forward
+      RAILS_DEFAULT_LOGGER.info ["roll_forward", Benchmark.measure{
+        _roll_forward
+      }].inspect
+    end
+  end
+end
+end